Line data Source code
1 : // SPDX-License-Identifier: GPL-2.0-only
2 : /*
3 : * cpu_rmap.c: CPU affinity reverse-map support
4 : * Copyright 2011 Solarflare Communications Inc.
5 : */
6 :
7 : #include <linux/cpu_rmap.h>
8 : #include <linux/interrupt.h>
9 : #include <linux/export.h>
10 :
11 : /*
12 : * These functions maintain a mapping from CPUs to some ordered set of
13 : * objects with CPU affinities. This can be seen as a reverse-map of
14 : * CPU affinity. However, we do not assume that the object affinities
15 : * cover all CPUs in the system. For those CPUs not directly covered
16 : * by object affinities, we attempt to find a nearest object based on
17 : * CPU topology.
18 : */
19 :
20 : /**
21 : * alloc_cpu_rmap - allocate CPU affinity reverse-map
22 : * @size: Number of objects to be mapped
23 : * @flags: Allocation flags e.g. %GFP_KERNEL
24 : */
25 0 : struct cpu_rmap *alloc_cpu_rmap(unsigned int size, gfp_t flags)
26 : {
27 0 : struct cpu_rmap *rmap;
28 0 : unsigned int cpu;
29 0 : size_t obj_offset;
30 :
31 : /* This is a silly number of objects, and we use u16 indices. */
32 0 : if (size > 0xffff)
33 : return NULL;
34 :
35 : /* Offset of object pointer array from base structure */
36 0 : obj_offset = ALIGN(offsetof(struct cpu_rmap, near[nr_cpu_ids]),
37 : sizeof(void *));
38 :
39 0 : rmap = kzalloc(obj_offset + size * sizeof(rmap->obj[0]), flags);
40 0 : if (!rmap)
41 : return NULL;
42 :
43 0 : kref_init(&rmap->refcount);
44 0 : rmap->obj = (void **)((char *)rmap + obj_offset);
45 :
46 : /* Initially assign CPUs to objects on a rota, since we have
47 : * no idea where the objects are. Use infinite distance, so
48 : * any object with known distance is preferable. Include the
49 : * CPUs that are not present/online, since we definitely want
50 : * any newly-hotplugged CPUs to have some object assigned.
51 : */
52 0 : for_each_possible_cpu(cpu) {
53 0 : rmap->near[cpu].index = cpu % size;
54 0 : rmap->near[cpu].dist = CPU_RMAP_DIST_INF;
55 : }
56 :
57 0 : rmap->size = size;
58 0 : return rmap;
59 : }
60 : EXPORT_SYMBOL(alloc_cpu_rmap);
61 :
62 : /**
63 : * cpu_rmap_release - internal reclaiming helper called from kref_put
64 : * @ref: kref to struct cpu_rmap
65 : */
66 0 : static void cpu_rmap_release(struct kref *ref)
67 : {
68 0 : struct cpu_rmap *rmap = container_of(ref, struct cpu_rmap, refcount);
69 0 : kfree(rmap);
70 0 : }
71 :
72 : /**
73 : * cpu_rmap_get - internal helper to get new ref on a cpu_rmap
74 : * @rmap: reverse-map allocated with alloc_cpu_rmap()
75 : */
76 0 : static inline void cpu_rmap_get(struct cpu_rmap *rmap)
77 : {
78 0 : kref_get(&rmap->refcount);
79 : }
80 :
81 : /**
82 : * cpu_rmap_put - release ref on a cpu_rmap
83 : * @rmap: reverse-map allocated with alloc_cpu_rmap()
84 : */
85 0 : int cpu_rmap_put(struct cpu_rmap *rmap)
86 : {
87 0 : return kref_put(&rmap->refcount, cpu_rmap_release);
88 : }
89 : EXPORT_SYMBOL(cpu_rmap_put);
90 :
91 : /* Reevaluate nearest object for given CPU, comparing with the given
92 : * neighbours at the given distance.
93 : */
94 0 : static bool cpu_rmap_copy_neigh(struct cpu_rmap *rmap, unsigned int cpu,
95 : const struct cpumask *mask, u16 dist)
96 : {
97 0 : int neigh;
98 :
99 0 : for_each_cpu(neigh, mask) {
100 0 : if (rmap->near[cpu].dist > dist &&
101 0 : rmap->near[neigh].dist <= dist) {
102 0 : rmap->near[cpu].index = rmap->near[neigh].index;
103 0 : rmap->near[cpu].dist = dist;
104 0 : return true;
105 : }
106 : }
107 : return false;
108 : }
109 :
110 : #ifdef DEBUG
111 : static void debug_print_rmap(const struct cpu_rmap *rmap, const char *prefix)
112 : {
113 : unsigned index;
114 : unsigned int cpu;
115 :
116 : pr_info("cpu_rmap %p, %s:\n", rmap, prefix);
117 :
118 : for_each_possible_cpu(cpu) {
119 : index = rmap->near[cpu].index;
120 : pr_info("cpu %d -> obj %u (distance %u)\n",
121 : cpu, index, rmap->near[cpu].dist);
122 : }
123 : }
124 : #else
125 : static inline void
126 0 : debug_print_rmap(const struct cpu_rmap *rmap, const char *prefix)
127 : {
128 0 : }
129 : #endif
130 :
131 : /**
132 : * cpu_rmap_add - add object to a rmap
133 : * @rmap: CPU rmap allocated with alloc_cpu_rmap()
134 : * @obj: Object to add to rmap
135 : *
136 : * Return index of object.
137 : */
138 0 : int cpu_rmap_add(struct cpu_rmap *rmap, void *obj)
139 : {
140 0 : u16 index;
141 :
142 0 : BUG_ON(rmap->used >= rmap->size);
143 0 : index = rmap->used++;
144 0 : rmap->obj[index] = obj;
145 0 : return index;
146 : }
147 : EXPORT_SYMBOL(cpu_rmap_add);
148 :
149 : /**
150 : * cpu_rmap_update - update CPU rmap following a change of object affinity
151 : * @rmap: CPU rmap to update
152 : * @index: Index of object whose affinity changed
153 : * @affinity: New CPU affinity of object
154 : */
155 0 : int cpu_rmap_update(struct cpu_rmap *rmap, u16 index,
156 : const struct cpumask *affinity)
157 : {
158 0 : cpumask_var_t update_mask;
159 0 : unsigned int cpu;
160 :
161 0 : if (unlikely(!zalloc_cpumask_var(&update_mask, GFP_KERNEL)))
162 : return -ENOMEM;
163 :
164 : /* Invalidate distance for all CPUs for which this used to be
165 : * the nearest object. Mark those CPUs for update.
166 : */
167 0 : for_each_online_cpu(cpu) {
168 0 : if (rmap->near[cpu].index == index) {
169 0 : rmap->near[cpu].dist = CPU_RMAP_DIST_INF;
170 0 : cpumask_set_cpu(cpu, update_mask);
171 : }
172 : }
173 :
174 0 : debug_print_rmap(rmap, "after invalidating old distances");
175 :
176 : /* Set distance to 0 for all CPUs in the new affinity mask.
177 : * Mark all CPUs within their NUMA nodes for update.
178 : */
179 0 : for_each_cpu(cpu, affinity) {
180 0 : rmap->near[cpu].index = index;
181 0 : rmap->near[cpu].dist = 0;
182 0 : cpumask_or(update_mask, update_mask,
183 : cpumask_of_node(cpu_to_node(cpu)));
184 : }
185 :
186 0 : debug_print_rmap(rmap, "after updating neighbours");
187 :
188 : /* Update distances based on topology */
189 0 : for_each_cpu(cpu, update_mask) {
190 0 : if (cpu_rmap_copy_neigh(rmap, cpu,
191 0 : topology_sibling_cpumask(cpu), 1))
192 0 : continue;
193 0 : if (cpu_rmap_copy_neigh(rmap, cpu,
194 0 : topology_core_cpumask(cpu), 2))
195 0 : continue;
196 0 : if (cpu_rmap_copy_neigh(rmap, cpu,
197 : cpumask_of_node(cpu_to_node(cpu)), 3))
198 : continue;
199 : /* We could continue into NUMA node distances, but for now
200 : * we give up.
201 : */
202 : }
203 :
204 0 : debug_print_rmap(rmap, "after copying neighbours");
205 :
206 0 : free_cpumask_var(update_mask);
207 0 : return 0;
208 : }
209 : EXPORT_SYMBOL(cpu_rmap_update);
210 :
211 : /* Glue between IRQ affinity notifiers and CPU rmaps */
212 :
213 : struct irq_glue {
214 : struct irq_affinity_notify notify;
215 : struct cpu_rmap *rmap;
216 : u16 index;
217 : };
218 :
219 : /**
220 : * free_irq_cpu_rmap - free a CPU affinity reverse-map used for IRQs
221 : * @rmap: Reverse-map allocated with alloc_irq_cpu_map(), or %NULL
222 : *
223 : * Must be called in process context, before freeing the IRQs.
224 : */
225 0 : void free_irq_cpu_rmap(struct cpu_rmap *rmap)
226 : {
227 0 : struct irq_glue *glue;
228 0 : u16 index;
229 :
230 0 : if (!rmap)
231 : return;
232 :
233 0 : for (index = 0; index < rmap->used; index++) {
234 0 : glue = rmap->obj[index];
235 0 : irq_set_affinity_notifier(glue->notify.irq, NULL);
236 : }
237 :
238 0 : cpu_rmap_put(rmap);
239 : }
240 : EXPORT_SYMBOL(free_irq_cpu_rmap);
241 :
242 : /**
243 : * irq_cpu_rmap_notify - callback for IRQ subsystem when IRQ affinity updated
244 : * @notify: struct irq_affinity_notify passed by irq/manage.c
245 : * @mask: cpu mask for new SMP affinity
246 : *
247 : * This is executed in workqueue context.
248 : */
249 : static void
250 0 : irq_cpu_rmap_notify(struct irq_affinity_notify *notify, const cpumask_t *mask)
251 : {
252 0 : struct irq_glue *glue =
253 0 : container_of(notify, struct irq_glue, notify);
254 0 : int rc;
255 :
256 0 : rc = cpu_rmap_update(glue->rmap, glue->index, mask);
257 0 : if (rc)
258 0 : pr_warn("irq_cpu_rmap_notify: update failed: %d\n", rc);
259 0 : }
260 :
261 : /**
262 : * irq_cpu_rmap_release - reclaiming callback for IRQ subsystem
263 : * @ref: kref to struct irq_affinity_notify passed by irq/manage.c
264 : */
265 0 : static void irq_cpu_rmap_release(struct kref *ref)
266 : {
267 0 : struct irq_glue *glue =
268 0 : container_of(ref, struct irq_glue, notify.kref);
269 :
270 0 : cpu_rmap_put(glue->rmap);
271 0 : kfree(glue);
272 0 : }
273 :
274 : /**
275 : * irq_cpu_rmap_add - add an IRQ to a CPU affinity reverse-map
276 : * @rmap: The reverse-map
277 : * @irq: The IRQ number
278 : *
279 : * This adds an IRQ affinity notifier that will update the reverse-map
280 : * automatically.
281 : *
282 : * Must be called in process context, after the IRQ is allocated but
283 : * before it is bound with request_irq().
284 : */
285 0 : int irq_cpu_rmap_add(struct cpu_rmap *rmap, int irq)
286 : {
287 0 : struct irq_glue *glue = kzalloc(sizeof(*glue), GFP_KERNEL);
288 0 : int rc;
289 :
290 0 : if (!glue)
291 : return -ENOMEM;
292 0 : glue->notify.notify = irq_cpu_rmap_notify;
293 0 : glue->notify.release = irq_cpu_rmap_release;
294 0 : glue->rmap = rmap;
295 0 : cpu_rmap_get(rmap);
296 0 : glue->index = cpu_rmap_add(rmap, glue);
297 0 : rc = irq_set_affinity_notifier(irq, &glue->notify);
298 0 : if (rc) {
299 0 : cpu_rmap_put(glue->rmap);
300 0 : kfree(glue);
301 : }
302 : return rc;
303 : }
304 : EXPORT_SYMBOL(irq_cpu_rmap_add);
|
