Line data Source code
1 : /* SPDX-License-Identifier: GPL-2.0 */
2 : #ifndef __LINUX_CPUMASK_H
3 : #define __LINUX_CPUMASK_H
4 :
5 : /*
6 : * Cpumasks provide a bitmap suitable for representing the
7 : * set of CPU's in a system, one bit position per CPU number. In general,
8 : * only nr_cpu_ids (<= NR_CPUS) bits are valid.
9 : */
10 : #include <linux/kernel.h>
11 : #include <linux/threads.h>
12 : #include <linux/bitmap.h>
13 : #include <linux/atomic.h>
14 : #include <linux/bug.h>
15 :
16 : /* Don't assign or return these: may not be this big! */
17 : typedef struct cpumask { DECLARE_BITMAP(bits, NR_CPUS); } cpumask_t;
18 :
19 : /**
20 : * cpumask_bits - get the bits in a cpumask
21 : * @maskp: the struct cpumask *
22 : *
23 : * You should only assume nr_cpu_ids bits of this mask are valid. This is
24 : * a macro so it's const-correct.
25 : */
26 : #define cpumask_bits(maskp) ((maskp)->bits)
27 :
28 : /**
29 : * cpumask_pr_args - printf args to output a cpumask
30 : * @maskp: cpumask to be printed
31 : *
32 : * Can be used to provide arguments for '%*pb[l]' when printing a cpumask.
33 : */
34 : #define cpumask_pr_args(maskp) nr_cpu_ids, cpumask_bits(maskp)
35 :
36 : #if NR_CPUS == 1
37 : #define nr_cpu_ids 1U
38 : #else
39 : extern unsigned int nr_cpu_ids;
40 : #endif
41 :
42 : #ifdef CONFIG_CPUMASK_OFFSTACK
43 : /* Assuming NR_CPUS is huge, a runtime limit is more efficient. Also,
44 : * not all bits may be allocated. */
45 : #define nr_cpumask_bits nr_cpu_ids
46 : #else
47 : #define nr_cpumask_bits ((unsigned int)NR_CPUS)
48 : #endif
49 :
50 : /*
51 : * The following particular system cpumasks and operations manage
52 : * possible, present, active and online cpus.
53 : *
54 : * cpu_possible_mask- has bit 'cpu' set iff cpu is populatable
55 : * cpu_present_mask - has bit 'cpu' set iff cpu is populated
56 : * cpu_online_mask - has bit 'cpu' set iff cpu available to scheduler
57 : * cpu_active_mask - has bit 'cpu' set iff cpu available to migration
58 : *
59 : * If !CONFIG_HOTPLUG_CPU, present == possible, and active == online.
60 : *
61 : * The cpu_possible_mask is fixed at boot time, as the set of CPU id's
62 : * that it is possible might ever be plugged in at anytime during the
63 : * life of that system boot. The cpu_present_mask is dynamic(*),
64 : * representing which CPUs are currently plugged in. And
65 : * cpu_online_mask is the dynamic subset of cpu_present_mask,
66 : * indicating those CPUs available for scheduling.
67 : *
68 : * If HOTPLUG is enabled, then cpu_possible_mask is forced to have
69 : * all NR_CPUS bits set, otherwise it is just the set of CPUs that
70 : * ACPI reports present at boot.
71 : *
72 : * If HOTPLUG is enabled, then cpu_present_mask varies dynamically,
73 : * depending on what ACPI reports as currently plugged in, otherwise
74 : * cpu_present_mask is just a copy of cpu_possible_mask.
75 : *
76 : * (*) Well, cpu_present_mask is dynamic in the hotplug case. If not
77 : * hotplug, it's a copy of cpu_possible_mask, hence fixed at boot.
78 : *
79 : * Subtleties:
80 : * 1) UP arch's (NR_CPUS == 1, CONFIG_SMP not defined) hardcode
81 : * assumption that their single CPU is online. The UP
82 : * cpu_{online,possible,present}_masks are placebos. Changing them
83 : * will have no useful affect on the following num_*_cpus()
84 : * and cpu_*() macros in the UP case. This ugliness is a UP
85 : * optimization - don't waste any instructions or memory references
86 : * asking if you're online or how many CPUs there are if there is
87 : * only one CPU.
88 : */
89 :
90 : extern struct cpumask __cpu_possible_mask;
91 : extern struct cpumask __cpu_online_mask;
92 : extern struct cpumask __cpu_present_mask;
93 : extern struct cpumask __cpu_active_mask;
94 : #define cpu_possible_mask ((const struct cpumask *)&__cpu_possible_mask)
95 : #define cpu_online_mask ((const struct cpumask *)&__cpu_online_mask)
96 : #define cpu_present_mask ((const struct cpumask *)&__cpu_present_mask)
97 : #define cpu_active_mask ((const struct cpumask *)&__cpu_active_mask)
98 :
99 : extern atomic_t __num_online_cpus;
100 :
101 : #if NR_CPUS > 1
102 : /**
103 : * num_online_cpus() - Read the number of online CPUs
104 : *
105 : * Despite the fact that __num_online_cpus is of type atomic_t, this
106 : * interface gives only a momentary snapshot and is not protected against
107 : * concurrent CPU hotplug operations unless invoked from a cpuhp_lock held
108 : * region.
109 : */
110 13537 : static inline unsigned int num_online_cpus(void)
111 : {
112 13537 : return atomic_read(&__num_online_cpus);
113 : }
114 : #define num_possible_cpus() cpumask_weight(cpu_possible_mask)
115 : #define num_present_cpus() cpumask_weight(cpu_present_mask)
116 : #define num_active_cpus() cpumask_weight(cpu_active_mask)
117 : #define cpu_online(cpu) cpumask_test_cpu((cpu), cpu_online_mask)
118 : #define cpu_possible(cpu) cpumask_test_cpu((cpu), cpu_possible_mask)
119 : #define cpu_present(cpu) cpumask_test_cpu((cpu), cpu_present_mask)
120 : #define cpu_active(cpu) cpumask_test_cpu((cpu), cpu_active_mask)
121 : #else
122 : #define num_online_cpus() 1U
123 : #define num_possible_cpus() 1U
124 : #define num_present_cpus() 1U
125 : #define num_active_cpus() 1U
126 : #define cpu_online(cpu) ((cpu) == 0)
127 : #define cpu_possible(cpu) ((cpu) == 0)
128 : #define cpu_present(cpu) ((cpu) == 0)
129 : #define cpu_active(cpu) ((cpu) == 0)
130 : #endif
131 :
132 : extern cpumask_t cpus_booted_once_mask;
133 :
134 33932592 : static inline void cpu_max_bits_warn(unsigned int cpu, unsigned int bits)
135 : {
136 : #ifdef CONFIG_DEBUG_PER_CPU_MAPS
137 : WARN_ON_ONCE(cpu >= bits);
138 : #endif /* CONFIG_DEBUG_PER_CPU_MAPS */
139 33928491 : }
140 :
141 : /* verify cpu argument to cpumask_* operators */
142 33932576 : static inline unsigned int cpumask_check(unsigned int cpu)
143 : {
144 33932576 : cpu_max_bits_warn(cpu, nr_cpumask_bits);
145 33932576 : return cpu;
146 : }
147 :
148 : #if NR_CPUS == 1
149 : /* Uniprocessor. Assume all masks are "1". */
150 : static inline unsigned int cpumask_first(const struct cpumask *srcp)
151 : {
152 : return 0;
153 : }
154 :
155 : static inline unsigned int cpumask_last(const struct cpumask *srcp)
156 : {
157 : return 0;
158 : }
159 :
160 : /* Valid inputs for n are -1 and 0. */
161 : static inline unsigned int cpumask_next(int n, const struct cpumask *srcp)
162 : {
163 : return n+1;
164 : }
165 :
166 : static inline unsigned int cpumask_next_zero(int n, const struct cpumask *srcp)
167 : {
168 : return n+1;
169 : }
170 :
171 : static inline unsigned int cpumask_next_and(int n,
172 : const struct cpumask *srcp,
173 : const struct cpumask *andp)
174 : {
175 : return n+1;
176 : }
177 :
178 : static inline unsigned int cpumask_next_wrap(int n, const struct cpumask *mask,
179 : int start, bool wrap)
180 : {
181 : /* cpu0 unless stop condition, wrap and at cpu0, then nr_cpumask_bits */
182 : return (wrap && n == 0);
183 : }
184 :
185 : /* cpu must be a valid cpu, ie 0, so there's no other choice. */
186 : static inline unsigned int cpumask_any_but(const struct cpumask *mask,
187 : unsigned int cpu)
188 : {
189 : return 1;
190 : }
191 :
192 : static inline unsigned int cpumask_local_spread(unsigned int i, int node)
193 : {
194 : return 0;
195 : }
196 :
197 : static inline int cpumask_any_and_distribute(const struct cpumask *src1p,
198 : const struct cpumask *src2p) {
199 : return cpumask_next_and(-1, src1p, src2p);
200 : }
201 :
202 : static inline int cpumask_any_distribute(const struct cpumask *srcp)
203 : {
204 : return cpumask_first(srcp);
205 : }
206 :
207 : #define for_each_cpu(cpu, mask) \
208 : for ((cpu) = 0; (cpu) < 1; (cpu)++, (void)mask)
209 : #define for_each_cpu_not(cpu, mask) \
210 : for ((cpu) = 0; (cpu) < 1; (cpu)++, (void)mask)
211 : #define for_each_cpu_wrap(cpu, mask, start) \
212 : for ((cpu) = 0; (cpu) < 1; (cpu)++, (void)mask, (void)(start))
213 : #define for_each_cpu_and(cpu, mask1, mask2) \
214 : for ((cpu) = 0; (cpu) < 1; (cpu)++, (void)mask1, (void)mask2)
215 : #else
216 : /**
217 : * cpumask_first - get the first cpu in a cpumask
218 : * @srcp: the cpumask pointer
219 : *
220 : * Returns >= nr_cpu_ids if no cpus set.
221 : */
222 6390 : static inline unsigned int cpumask_first(const struct cpumask *srcp)
223 : {
224 6390 : return find_first_bit(cpumask_bits(srcp), nr_cpumask_bits);
225 : }
226 :
227 : /**
228 : * cpumask_last - get the last CPU in a cpumask
229 : * @srcp: - the cpumask pointer
230 : *
231 : * Returns >= nr_cpumask_bits if no CPUs set.
232 : */
233 : static inline unsigned int cpumask_last(const struct cpumask *srcp)
234 : {
235 : return find_last_bit(cpumask_bits(srcp), nr_cpumask_bits);
236 : }
237 :
238 : unsigned int cpumask_next(int n, const struct cpumask *srcp);
239 :
240 : /**
241 : * cpumask_next_zero - get the next unset cpu in a cpumask
242 : * @n: the cpu prior to the place to search (ie. return will be > @n)
243 : * @srcp: the cpumask pointer
244 : *
245 : * Returns >= nr_cpu_ids if no further cpus unset.
246 : */
247 : static inline unsigned int cpumask_next_zero(int n, const struct cpumask *srcp)
248 : {
249 : /* -1 is a legal arg here. */
250 : if (n != -1)
251 : cpumask_check(n);
252 : return find_next_zero_bit(cpumask_bits(srcp), nr_cpumask_bits, n+1);
253 : }
254 :
255 : int cpumask_next_and(int n, const struct cpumask *, const struct cpumask *);
256 : int cpumask_any_but(const struct cpumask *mask, unsigned int cpu);
257 : unsigned int cpumask_local_spread(unsigned int i, int node);
258 : int cpumask_any_and_distribute(const struct cpumask *src1p,
259 : const struct cpumask *src2p);
260 : int cpumask_any_distribute(const struct cpumask *srcp);
261 :
262 : /**
263 : * for_each_cpu - iterate over every cpu in a mask
264 : * @cpu: the (optionally unsigned) integer iterator
265 : * @mask: the cpumask pointer
266 : *
267 : * After the loop, cpu is >= nr_cpu_ids.
268 : */
269 : #define for_each_cpu(cpu, mask) \
270 : for ((cpu) = -1; \
271 : (cpu) = cpumask_next((cpu), (mask)), \
272 : (cpu) < nr_cpu_ids;)
273 :
274 : /**
275 : * for_each_cpu_not - iterate over every cpu in a complemented mask
276 : * @cpu: the (optionally unsigned) integer iterator
277 : * @mask: the cpumask pointer
278 : *
279 : * After the loop, cpu is >= nr_cpu_ids.
280 : */
281 : #define for_each_cpu_not(cpu, mask) \
282 : for ((cpu) = -1; \
283 : (cpu) = cpumask_next_zero((cpu), (mask)), \
284 : (cpu) < nr_cpu_ids;)
285 :
286 : extern int cpumask_next_wrap(int n, const struct cpumask *mask, int start, bool wrap);
287 :
288 : /**
289 : * for_each_cpu_wrap - iterate over every cpu in a mask, starting at a specified location
290 : * @cpu: the (optionally unsigned) integer iterator
291 : * @mask: the cpumask poiter
292 : * @start: the start location
293 : *
294 : * The implementation does not assume any bit in @mask is set (including @start).
295 : *
296 : * After the loop, cpu is >= nr_cpu_ids.
297 : */
298 : #define for_each_cpu_wrap(cpu, mask, start) \
299 : for ((cpu) = cpumask_next_wrap((start)-1, (mask), (start), false); \
300 : (cpu) < nr_cpumask_bits; \
301 : (cpu) = cpumask_next_wrap((cpu), (mask), (start), true))
302 :
303 : /**
304 : * for_each_cpu_and - iterate over every cpu in both masks
305 : * @cpu: the (optionally unsigned) integer iterator
306 : * @mask1: the first cpumask pointer
307 : * @mask2: the second cpumask pointer
308 : *
309 : * This saves a temporary CPU mask in many places. It is equivalent to:
310 : * struct cpumask tmp;
311 : * cpumask_and(&tmp, &mask1, &mask2);
312 : * for_each_cpu(cpu, &tmp)
313 : * ...
314 : *
315 : * After the loop, cpu is >= nr_cpu_ids.
316 : */
317 : #define for_each_cpu_and(cpu, mask1, mask2) \
318 : for ((cpu) = -1; \
319 : (cpu) = cpumask_next_and((cpu), (mask1), (mask2)), \
320 : (cpu) < nr_cpu_ids;)
321 : #endif /* SMP */
322 :
323 : #define CPU_BITS_NONE \
324 : { \
325 : [0 ... BITS_TO_LONGS(NR_CPUS)-1] = 0UL \
326 : }
327 :
328 : #define CPU_BITS_CPU0 \
329 : { \
330 : [0] = 1UL \
331 : }
332 :
333 : /**
334 : * cpumask_set_cpu - set a cpu in a cpumask
335 : * @cpu: cpu number (< nr_cpu_ids)
336 : * @dstp: the cpumask pointer
337 : */
338 10752 : static inline void cpumask_set_cpu(unsigned int cpu, struct cpumask *dstp)
339 : {
340 10726 : set_bit(cpumask_check(cpu), cpumask_bits(dstp));
341 10401 : }
342 :
343 121 : static inline void __cpumask_set_cpu(unsigned int cpu, struct cpumask *dstp)
344 : {
345 121 : __set_bit(cpumask_check(cpu), cpumask_bits(dstp));
346 121 : }
347 :
348 :
349 : /**
350 : * cpumask_clear_cpu - clear a cpu in a cpumask
351 : * @cpu: cpu number (< nr_cpu_ids)
352 : * @dstp: the cpumask pointer
353 : */
354 10611 : static inline void cpumask_clear_cpu(int cpu, struct cpumask *dstp)
355 : {
356 10615 : clear_bit(cpumask_check(cpu), cpumask_bits(dstp));
357 10385 : }
358 :
359 2793 : static inline void __cpumask_clear_cpu(int cpu, struct cpumask *dstp)
360 : {
361 2793 : __clear_bit(cpumask_check(cpu), cpumask_bits(dstp));
362 2677 : }
363 :
364 : /**
365 : * cpumask_test_cpu - test for a cpu in a cpumask
366 : * @cpu: cpu number (< nr_cpu_ids)
367 : * @cpumask: the cpumask pointer
368 : *
369 : * Returns 1 if @cpu is set in @cpumask, else returns 0
370 : */
371 33832060 : static inline int cpumask_test_cpu(int cpu, const struct cpumask *cpumask)
372 : {
373 33832973 : return test_bit(cpumask_check(cpu), cpumask_bits((cpumask)));
374 : }
375 :
376 : /**
377 : * cpumask_test_and_set_cpu - atomically test and set a cpu in a cpumask
378 : * @cpu: cpu number (< nr_cpu_ids)
379 : * @cpumask: the cpumask pointer
380 : *
381 : * Returns 1 if @cpu is set in old bitmap of @cpumask, else returns 0
382 : *
383 : * test_and_set_bit wrapper for cpumasks.
384 : */
385 8 : static inline int cpumask_test_and_set_cpu(int cpu, struct cpumask *cpumask)
386 : {
387 8 : return test_and_set_bit(cpumask_check(cpu), cpumask_bits(cpumask));
388 : }
389 :
390 : /**
391 : * cpumask_test_and_clear_cpu - atomically test and clear a cpu in a cpumask
392 : * @cpu: cpu number (< nr_cpu_ids)
393 : * @cpumask: the cpumask pointer
394 : *
395 : * Returns 1 if @cpu is set in old bitmap of @cpumask, else returns 0
396 : *
397 : * test_and_clear_bit wrapper for cpumasks.
398 : */
399 0 : static inline int cpumask_test_and_clear_cpu(int cpu, struct cpumask *cpumask)
400 : {
401 0 : return test_and_clear_bit(cpumask_check(cpu), cpumask_bits(cpumask));
402 : }
403 :
404 : /**
405 : * cpumask_setall - set all cpus (< nr_cpu_ids) in a cpumask
406 : * @dstp: the cpumask pointer
407 : */
408 1 : static inline void cpumask_setall(struct cpumask *dstp)
409 : {
410 1 : bitmap_fill(cpumask_bits(dstp), nr_cpumask_bits);
411 1 : }
412 :
413 : /**
414 : * cpumask_clear - clear all cpus (< nr_cpu_ids) in a cpumask
415 : * @dstp: the cpumask pointer
416 : */
417 2361 : static inline void cpumask_clear(struct cpumask *dstp)
418 : {
419 2339 : bitmap_zero(cpumask_bits(dstp), nr_cpumask_bits);
420 0 : }
421 :
422 : /**
423 : * cpumask_and - *dstp = *src1p & *src2p
424 : * @dstp: the cpumask result
425 : * @src1p: the first input
426 : * @src2p: the second input
427 : *
428 : * If *@dstp is empty, returns 0, else returns 1
429 : */
430 9358 : static inline int cpumask_and(struct cpumask *dstp,
431 : const struct cpumask *src1p,
432 : const struct cpumask *src2p)
433 : {
434 9358 : return bitmap_and(cpumask_bits(dstp), cpumask_bits(src1p),
435 9358 : cpumask_bits(src2p), nr_cpumask_bits);
436 : }
437 :
438 : /**
439 : * cpumask_or - *dstp = *src1p | *src2p
440 : * @dstp: the cpumask result
441 : * @src1p: the first input
442 : * @src2p: the second input
443 : */
444 25 : static inline void cpumask_or(struct cpumask *dstp, const struct cpumask *src1p,
445 : const struct cpumask *src2p)
446 : {
447 25 : bitmap_or(cpumask_bits(dstp), cpumask_bits(src1p),
448 25 : cpumask_bits(src2p), nr_cpumask_bits);
449 1 : }
450 :
451 : /**
452 : * cpumask_xor - *dstp = *src1p ^ *src2p
453 : * @dstp: the cpumask result
454 : * @src1p: the first input
455 : * @src2p: the second input
456 : */
457 : static inline void cpumask_xor(struct cpumask *dstp,
458 : const struct cpumask *src1p,
459 : const struct cpumask *src2p)
460 : {
461 : bitmap_xor(cpumask_bits(dstp), cpumask_bits(src1p),
462 : cpumask_bits(src2p), nr_cpumask_bits);
463 : }
464 :
465 : /**
466 : * cpumask_andnot - *dstp = *src1p & ~*src2p
467 : * @dstp: the cpumask result
468 : * @src1p: the first input
469 : * @src2p: the second input
470 : *
471 : * If *@dstp is empty, returns 0, else returns 1
472 : */
473 0 : static inline int cpumask_andnot(struct cpumask *dstp,
474 : const struct cpumask *src1p,
475 : const struct cpumask *src2p)
476 : {
477 0 : return bitmap_andnot(cpumask_bits(dstp), cpumask_bits(src1p),
478 0 : cpumask_bits(src2p), nr_cpumask_bits);
479 : }
480 :
481 : /**
482 : * cpumask_complement - *dstp = ~*srcp
483 : * @dstp: the cpumask result
484 : * @srcp: the input to invert
485 : */
486 : static inline void cpumask_complement(struct cpumask *dstp,
487 : const struct cpumask *srcp)
488 : {
489 : bitmap_complement(cpumask_bits(dstp), cpumask_bits(srcp),
490 : nr_cpumask_bits);
491 : }
492 :
493 : /**
494 : * cpumask_equal - *src1p == *src2p
495 : * @src1p: the first input
496 : * @src2p: the second input
497 : */
498 129 : static inline bool cpumask_equal(const struct cpumask *src1p,
499 : const struct cpumask *src2p)
500 : {
501 125 : return bitmap_equal(cpumask_bits(src1p), cpumask_bits(src2p),
502 : nr_cpumask_bits);
503 : }
504 :
505 : /**
506 : * cpumask_or_equal - *src1p | *src2p == *src3p
507 : * @src1p: the first input
508 : * @src2p: the second input
509 : * @src3p: the third input
510 : */
511 62 : static inline bool cpumask_or_equal(const struct cpumask *src1p,
512 : const struct cpumask *src2p,
513 : const struct cpumask *src3p)
514 : {
515 62 : return bitmap_or_equal(cpumask_bits(src1p), cpumask_bits(src2p),
516 : cpumask_bits(src3p), nr_cpumask_bits);
517 : }
518 :
519 : /**
520 : * cpumask_intersects - (*src1p & *src2p) != 0
521 : * @src1p: the first input
522 : * @src2p: the second input
523 : */
524 7947 : static inline bool cpumask_intersects(const struct cpumask *src1p,
525 : const struct cpumask *src2p)
526 : {
527 7947 : return bitmap_intersects(cpumask_bits(src1p), cpumask_bits(src2p),
528 : nr_cpumask_bits);
529 : }
530 :
531 : /**
532 : * cpumask_subset - (*src1p & ~*src2p) == 0
533 : * @src1p: the first input
534 : * @src2p: the second input
535 : *
536 : * Returns 1 if *@src1p is a subset of *@src2p, else returns 0
537 : */
538 512 : static inline int cpumask_subset(const struct cpumask *src1p,
539 : const struct cpumask *src2p)
540 : {
541 512 : return bitmap_subset(cpumask_bits(src1p), cpumask_bits(src2p),
542 : nr_cpumask_bits);
543 : }
544 :
545 : /**
546 : * cpumask_empty - *srcp == 0
547 : * @srcp: the cpumask to that all cpus < nr_cpu_ids are clear.
548 : */
549 14487 : static inline bool cpumask_empty(const struct cpumask *srcp)
550 : {
551 14487 : return bitmap_empty(cpumask_bits(srcp), nr_cpumask_bits);
552 : }
553 :
554 : /**
555 : * cpumask_full - *srcp == 0xFFFFFFFF...
556 : * @srcp: the cpumask to that all cpus < nr_cpu_ids are set.
557 : */
558 : static inline bool cpumask_full(const struct cpumask *srcp)
559 : {
560 : return bitmap_full(cpumask_bits(srcp), nr_cpumask_bits);
561 : }
562 :
563 : /**
564 : * cpumask_weight - Count of bits in *srcp
565 : * @srcp: the cpumask to count bits (< nr_cpu_ids) in.
566 : */
567 380 : static inline unsigned int cpumask_weight(const struct cpumask *srcp)
568 : {
569 382 : return bitmap_weight(cpumask_bits(srcp), nr_cpumask_bits);
570 : }
571 :
572 : /**
573 : * cpumask_shift_right - *dstp = *srcp >> n
574 : * @dstp: the cpumask result
575 : * @srcp: the input to shift
576 : * @n: the number of bits to shift by
577 : */
578 : static inline void cpumask_shift_right(struct cpumask *dstp,
579 : const struct cpumask *srcp, int n)
580 : {
581 : bitmap_shift_right(cpumask_bits(dstp), cpumask_bits(srcp), n,
582 : nr_cpumask_bits);
583 : }
584 :
585 : /**
586 : * cpumask_shift_left - *dstp = *srcp << n
587 : * @dstp: the cpumask result
588 : * @srcp: the input to shift
589 : * @n: the number of bits to shift by
590 : */
591 : static inline void cpumask_shift_left(struct cpumask *dstp,
592 : const struct cpumask *srcp, int n)
593 : {
594 : bitmap_shift_left(cpumask_bits(dstp), cpumask_bits(srcp), n,
595 : nr_cpumask_bits);
596 : }
597 :
598 : /**
599 : * cpumask_copy - *dstp = *srcp
600 : * @dstp: the result
601 : * @srcp: the input cpumask
602 : */
603 3439 : static inline void cpumask_copy(struct cpumask *dstp,
604 : const struct cpumask *srcp)
605 : {
606 3410 : bitmap_copy(cpumask_bits(dstp), cpumask_bits(srcp), nr_cpumask_bits);
607 11 : }
608 :
609 : /**
610 : * cpumask_any - pick a "random" cpu from *srcp
611 : * @srcp: the input cpumask
612 : *
613 : * Returns >= nr_cpu_ids if no cpus set.
614 : */
615 : #define cpumask_any(srcp) cpumask_first(srcp)
616 :
617 : /**
618 : * cpumask_first_and - return the first cpu from *srcp1 & *srcp2
619 : * @src1p: the first input
620 : * @src2p: the second input
621 : *
622 : * Returns >= nr_cpu_ids if no cpus set in both. See also cpumask_next_and().
623 : */
624 : #define cpumask_first_and(src1p, src2p) cpumask_next_and(-1, (src1p), (src2p))
625 :
626 : /**
627 : * cpumask_any_and - pick a "random" cpu from *mask1 & *mask2
628 : * @mask1: the first input cpumask
629 : * @mask2: the second input cpumask
630 : *
631 : * Returns >= nr_cpu_ids if no cpus set.
632 : */
633 : #define cpumask_any_and(mask1, mask2) cpumask_first_and((mask1), (mask2))
634 :
635 : /**
636 : * cpumask_of - the cpumask containing just a given cpu
637 : * @cpu: the cpu (<= nr_cpu_ids)
638 : */
639 : #define cpumask_of(cpu) (get_cpu_mask(cpu))
640 :
641 : /**
642 : * cpumask_parse_user - extract a cpumask from a user string
643 : * @buf: the buffer to extract from
644 : * @len: the length of the buffer
645 : * @dstp: the cpumask to set.
646 : *
647 : * Returns -errno, or 0 for success.
648 : */
649 0 : static inline int cpumask_parse_user(const char __user *buf, int len,
650 : struct cpumask *dstp)
651 : {
652 0 : return bitmap_parse_user(buf, len, cpumask_bits(dstp), nr_cpumask_bits);
653 : }
654 :
655 : /**
656 : * cpumask_parselist_user - extract a cpumask from a user string
657 : * @buf: the buffer to extract from
658 : * @len: the length of the buffer
659 : * @dstp: the cpumask to set.
660 : *
661 : * Returns -errno, or 0 for success.
662 : */
663 0 : static inline int cpumask_parselist_user(const char __user *buf, int len,
664 : struct cpumask *dstp)
665 : {
666 0 : return bitmap_parselist_user(buf, len, cpumask_bits(dstp),
667 : nr_cpumask_bits);
668 : }
669 :
670 : /**
671 : * cpumask_parse - extract a cpumask from a string
672 : * @buf: the buffer to extract from
673 : * @dstp: the cpumask to set.
674 : *
675 : * Returns -errno, or 0 for success.
676 : */
677 0 : static inline int cpumask_parse(const char *buf, struct cpumask *dstp)
678 : {
679 0 : return bitmap_parse(buf, UINT_MAX, cpumask_bits(dstp), nr_cpumask_bits);
680 : }
681 :
682 : /**
683 : * cpulist_parse - extract a cpumask from a user string of ranges
684 : * @buf: the buffer to extract from
685 : * @dstp: the cpumask to set.
686 : *
687 : * Returns -errno, or 0 for success.
688 : */
689 0 : static inline int cpulist_parse(const char *buf, struct cpumask *dstp)
690 : {
691 0 : return bitmap_parselist(buf, cpumask_bits(dstp), nr_cpumask_bits);
692 : }
693 :
694 : /**
695 : * cpumask_size - size to allocate for a 'struct cpumask' in bytes
696 : */
697 51 : static inline unsigned int cpumask_size(void)
698 : {
699 51 : return BITS_TO_LONGS(nr_cpumask_bits) * sizeof(long);
700 : }
701 :
702 : /*
703 : * cpumask_var_t: struct cpumask for stack usage.
704 : *
705 : * Oh, the wicked games we play! In order to make kernel coding a
706 : * little more difficult, we typedef cpumask_var_t to an array or a
707 : * pointer: doing &mask on an array is a noop, so it still works.
708 : *
709 : * ie.
710 : * cpumask_var_t tmpmask;
711 : * if (!alloc_cpumask_var(&tmpmask, GFP_KERNEL))
712 : * return -ENOMEM;
713 : *
714 : * ... use 'tmpmask' like a normal struct cpumask * ...
715 : *
716 : * free_cpumask_var(tmpmask);
717 : *
718 : *
719 : * However, one notable exception is there. alloc_cpumask_var() allocates
720 : * only nr_cpumask_bits bits (in the other hand, real cpumask_t always has
721 : * NR_CPUS bits). Therefore you don't have to dereference cpumask_var_t.
722 : *
723 : * cpumask_var_t tmpmask;
724 : * if (!alloc_cpumask_var(&tmpmask, GFP_KERNEL))
725 : * return -ENOMEM;
726 : *
727 : * var = *tmpmask;
728 : *
729 : * This code makes NR_CPUS length memcopy and brings to a memory corruption.
730 : * cpumask_copy() provide safe copy functionality.
731 : *
732 : * Note that there is another evil here: If you define a cpumask_var_t
733 : * as a percpu variable then the way to obtain the address of the cpumask
734 : * structure differently influences what this_cpu_* operation needs to be
735 : * used. Please use this_cpu_cpumask_var_t in those cases. The direct use
736 : * of this_cpu_ptr() or this_cpu_read() will lead to failures when the
737 : * other type of cpumask_var_t implementation is configured.
738 : *
739 : * Please also note that __cpumask_var_read_mostly can be used to declare
740 : * a cpumask_var_t variable itself (not its content) as read mostly.
741 : */
742 : #ifdef CONFIG_CPUMASK_OFFSTACK
743 : typedef struct cpumask *cpumask_var_t;
744 :
745 : #define this_cpu_cpumask_var_ptr(x) this_cpu_read(x)
746 : #define __cpumask_var_read_mostly __read_mostly
747 :
748 : bool alloc_cpumask_var_node(cpumask_var_t *mask, gfp_t flags, int node);
749 : bool alloc_cpumask_var(cpumask_var_t *mask, gfp_t flags);
750 : bool zalloc_cpumask_var_node(cpumask_var_t *mask, gfp_t flags, int node);
751 : bool zalloc_cpumask_var(cpumask_var_t *mask, gfp_t flags);
752 : void alloc_bootmem_cpumask_var(cpumask_var_t *mask);
753 : void free_cpumask_var(cpumask_var_t mask);
754 : void free_bootmem_cpumask_var(cpumask_var_t mask);
755 :
756 : static inline bool cpumask_available(cpumask_var_t mask)
757 : {
758 : return mask != NULL;
759 : }
760 :
761 : #else
762 : typedef struct cpumask cpumask_var_t[1];
763 :
764 : #define this_cpu_cpumask_var_ptr(x) this_cpu_ptr(x)
765 : #define __cpumask_var_read_mostly
766 :
767 32 : static inline bool alloc_cpumask_var(cpumask_var_t *mask, gfp_t flags)
768 : {
769 32 : return true;
770 : }
771 :
772 : static inline bool alloc_cpumask_var_node(cpumask_var_t *mask, gfp_t flags,
773 : int node)
774 : {
775 : return true;
776 : }
777 :
778 217 : static inline bool zalloc_cpumask_var(cpumask_var_t *mask, gfp_t flags)
779 : {
780 217 : cpumask_clear(*mask);
781 217 : return true;
782 : }
783 :
784 41 : static inline bool zalloc_cpumask_var_node(cpumask_var_t *mask, gfp_t flags,
785 : int node)
786 : {
787 41 : cpumask_clear(*mask);
788 41 : return true;
789 : }
790 :
791 2 : static inline void alloc_bootmem_cpumask_var(cpumask_var_t *mask)
792 : {
793 2 : }
794 :
795 13 : static inline void free_cpumask_var(cpumask_var_t mask)
796 : {
797 13 : }
798 :
799 : static inline void free_bootmem_cpumask_var(cpumask_var_t mask)
800 : {
801 : }
802 :
803 1 : static inline bool cpumask_available(cpumask_var_t mask)
804 : {
805 1 : return true;
806 : }
807 : #endif /* CONFIG_CPUMASK_OFFSTACK */
808 :
809 : /* It's common to want to use cpu_all_mask in struct member initializers,
810 : * so it has to refer to an address rather than a pointer. */
811 : extern const DECLARE_BITMAP(cpu_all_bits, NR_CPUS);
812 : #define cpu_all_mask to_cpumask(cpu_all_bits)
813 :
814 : /* First bits of cpu_bit_bitmap are in fact unset. */
815 : #define cpu_none_mask to_cpumask(cpu_bit_bitmap[0])
816 :
817 : #define for_each_possible_cpu(cpu) for_each_cpu((cpu), cpu_possible_mask)
818 : #define for_each_online_cpu(cpu) for_each_cpu((cpu), cpu_online_mask)
819 : #define for_each_present_cpu(cpu) for_each_cpu((cpu), cpu_present_mask)
820 :
821 : /* Wrappers for arch boot code to manipulate normally-constant masks */
822 : void init_cpu_present(const struct cpumask *src);
823 : void init_cpu_possible(const struct cpumask *src);
824 : void init_cpu_online(const struct cpumask *src);
825 :
826 1 : static inline void reset_cpu_possible_mask(void)
827 : {
828 1 : bitmap_zero(cpumask_bits(&__cpu_possible_mask), NR_CPUS);
829 : }
830 :
831 : static inline void
832 9 : set_cpu_possible(unsigned int cpu, bool possible)
833 : {
834 9 : if (possible)
835 9 : cpumask_set_cpu(cpu, &__cpu_possible_mask);
836 : else
837 0 : cpumask_clear_cpu(cpu, &__cpu_possible_mask);
838 9 : }
839 :
840 : static inline void
841 5 : set_cpu_present(unsigned int cpu, bool present)
842 : {
843 5 : if (present)
844 5 : cpumask_set_cpu(cpu, &__cpu_present_mask);
845 : else
846 0 : cpumask_clear_cpu(cpu, &__cpu_present_mask);
847 5 : }
848 :
849 : void set_cpu_online(unsigned int cpu, bool online);
850 :
851 : static inline void
852 4 : set_cpu_active(unsigned int cpu, bool active)
853 : {
854 4 : if (active)
855 4 : cpumask_set_cpu(cpu, &__cpu_active_mask);
856 : else
857 0 : cpumask_clear_cpu(cpu, &__cpu_active_mask);
858 4 : }
859 :
860 :
861 : /**
862 : * to_cpumask - convert an NR_CPUS bitmap to a struct cpumask *
863 : * @bitmap: the bitmap
864 : *
865 : * There are a few places where cpumask_var_t isn't appropriate and
866 : * static cpumasks must be used (eg. very early boot), yet we don't
867 : * expose the definition of 'struct cpumask'.
868 : *
869 : * This does the conversion, and can be used as a constant initializer.
870 : */
871 : #define to_cpumask(bitmap) \
872 : ((struct cpumask *)(1 ? (bitmap) \
873 : : (void *)sizeof(__check_is_bitmap(bitmap))))
874 :
875 : static inline int __check_is_bitmap(const unsigned long *bitmap)
876 : {
877 : return 1;
878 : }
879 :
880 : /*
881 : * Special-case data structure for "single bit set only" constant CPU masks.
882 : *
883 : * We pre-generate all the 64 (or 32) possible bit positions, with enough
884 : * padding to the left and the right, and return the constant pointer
885 : * appropriately offset.
886 : */
887 : extern const unsigned long
888 : cpu_bit_bitmap[BITS_PER_LONG+1][BITS_TO_LONGS(NR_CPUS)];
889 :
890 14492 : static inline const struct cpumask *get_cpu_mask(unsigned int cpu)
891 : {
892 14492 : const unsigned long *p = cpu_bit_bitmap[1 + cpu % BITS_PER_LONG];
893 14492 : p -= cpu / BITS_PER_LONG;
894 14492 : return to_cpumask(p);
895 : }
896 :
897 : #define cpu_is_offline(cpu) unlikely(!cpu_online(cpu))
898 :
899 : #if NR_CPUS <= BITS_PER_LONG
900 : #define CPU_BITS_ALL \
901 : { \
902 : [BITS_TO_LONGS(NR_CPUS)-1] = BITMAP_LAST_WORD_MASK(NR_CPUS) \
903 : }
904 :
905 : #else /* NR_CPUS > BITS_PER_LONG */
906 :
907 : #define CPU_BITS_ALL \
908 : { \
909 : [0 ... BITS_TO_LONGS(NR_CPUS)-2] = ~0UL, \
910 : [BITS_TO_LONGS(NR_CPUS)-1] = BITMAP_LAST_WORD_MASK(NR_CPUS) \
911 : }
912 : #endif /* NR_CPUS > BITS_PER_LONG */
913 :
914 : /**
915 : * cpumap_print_to_pagebuf - copies the cpumask into the buffer either
916 : * as comma-separated list of cpus or hex values of cpumask
917 : * @list: indicates whether the cpumap must be list
918 : * @mask: the cpumask to copy
919 : * @buf: the buffer to copy into
920 : *
921 : * Returns the length of the (null-terminated) @buf string, zero if
922 : * nothing is copied.
923 : */
924 : static inline ssize_t
925 0 : cpumap_print_to_pagebuf(bool list, char *buf, const struct cpumask *mask)
926 : {
927 0 : return bitmap_print_to_pagebuf(list, buf, cpumask_bits(mask),
928 : nr_cpu_ids);
929 : }
930 :
931 : #if NR_CPUS <= BITS_PER_LONG
932 : #define CPU_MASK_ALL \
933 : (cpumask_t) { { \
934 : [BITS_TO_LONGS(NR_CPUS)-1] = BITMAP_LAST_WORD_MASK(NR_CPUS) \
935 : } }
936 : #else
937 : #define CPU_MASK_ALL \
938 : (cpumask_t) { { \
939 : [0 ... BITS_TO_LONGS(NR_CPUS)-2] = ~0UL, \
940 : [BITS_TO_LONGS(NR_CPUS)-1] = BITMAP_LAST_WORD_MASK(NR_CPUS) \
941 : } }
942 : #endif /* NR_CPUS > BITS_PER_LONG */
943 :
944 : #define CPU_MASK_NONE \
945 : (cpumask_t) { { \
946 : [0 ... BITS_TO_LONGS(NR_CPUS)-1] = 0UL \
947 : } }
948 :
949 : #define CPU_MASK_CPU0 \
950 : (cpumask_t) { { \
951 : [0] = 1UL \
952 : } }
953 :
954 : #endif /* __LINUX_CPUMASK_H */
|
