Line data Source code
1 : /* CPU control.
2 : * (C) 2001, 2002, 2003, 2004 Rusty Russell
3 : *
4 : * This code is licenced under the GPL.
5 : */
6 : #include <linux/sched/mm.h>
7 : #include <linux/proc_fs.h>
8 : #include <linux/smp.h>
9 : #include <linux/init.h>
10 : #include <linux/notifier.h>
11 : #include <linux/sched/signal.h>
12 : #include <linux/sched/hotplug.h>
13 : #include <linux/sched/isolation.h>
14 : #include <linux/sched/task.h>
15 : #include <linux/sched/smt.h>
16 : #include <linux/unistd.h>
17 : #include <linux/cpu.h>
18 : #include <linux/oom.h>
19 : #include <linux/rcupdate.h>
20 : #include <linux/export.h>
21 : #include <linux/bug.h>
22 : #include <linux/kthread.h>
23 : #include <linux/stop_machine.h>
24 : #include <linux/mutex.h>
25 : #include <linux/gfp.h>
26 : #include <linux/suspend.h>
27 : #include <linux/lockdep.h>
28 : #include <linux/tick.h>
29 : #include <linux/irq.h>
30 : #include <linux/nmi.h>
31 : #include <linux/smpboot.h>
32 : #include <linux/relay.h>
33 : #include <linux/slab.h>
34 : #include <linux/percpu-rwsem.h>
35 :
36 : #include <trace/events/power.h>
37 : #define CREATE_TRACE_POINTS
38 : #include <trace/events/cpuhp.h>
39 :
40 : #include "smpboot.h"
41 :
42 : /**
43 : * cpuhp_cpu_state - Per cpu hotplug state storage
44 : * @state: The current cpu state
45 : * @target: The target state
46 : * @thread: Pointer to the hotplug thread
47 : * @should_run: Thread should execute
48 : * @rollback: Perform a rollback
49 : * @single: Single callback invocation
50 : * @bringup: Single callback bringup or teardown selector
51 : * @cb_state: The state for a single callback (install/uninstall)
52 : * @result: Result of the operation
53 : * @done_up: Signal completion to the issuer of the task for cpu-up
54 : * @done_down: Signal completion to the issuer of the task for cpu-down
55 : */
56 : struct cpuhp_cpu_state {
57 : enum cpuhp_state state;
58 : enum cpuhp_state target;
59 : enum cpuhp_state fail;
60 : #ifdef CONFIG_SMP
61 : struct task_struct *thread;
62 : bool should_run;
63 : bool rollback;
64 : bool single;
65 : bool bringup;
66 : struct hlist_node *node;
67 : struct hlist_node *last;
68 : enum cpuhp_state cb_state;
69 : int result;
70 : struct completion done_up;
71 : struct completion done_down;
72 : #endif
73 : };
74 :
75 : static DEFINE_PER_CPU(struct cpuhp_cpu_state, cpuhp_state) = {
76 : .fail = CPUHP_INVALID,
77 : };
78 :
79 : #ifdef CONFIG_SMP
80 : cpumask_t cpus_booted_once_mask;
81 : #endif
82 :
83 : #if defined(CONFIG_LOCKDEP) && defined(CONFIG_SMP)
84 : static struct lockdep_map cpuhp_state_up_map =
85 : STATIC_LOCKDEP_MAP_INIT("cpuhp_state-up", &cpuhp_state_up_map);
86 : static struct lockdep_map cpuhp_state_down_map =
87 : STATIC_LOCKDEP_MAP_INIT("cpuhp_state-down", &cpuhp_state_down_map);
88 :
89 :
90 267 : static inline void cpuhp_lock_acquire(bool bringup)
91 : {
92 282 : lock_map_acquire(bringup ? &cpuhp_state_up_map : &cpuhp_state_down_map);
93 267 : }
94 :
95 267 : static inline void cpuhp_lock_release(bool bringup)
96 : {
97 267 : lock_map_release(bringup ? &cpuhp_state_up_map : &cpuhp_state_down_map);
98 237 : }
99 : #else
100 :
101 : static inline void cpuhp_lock_acquire(bool bringup) { }
102 : static inline void cpuhp_lock_release(bool bringup) { }
103 :
104 : #endif
105 :
106 : /**
107 : * cpuhp_step - Hotplug state machine step
108 : * @name: Name of the step
109 : * @startup: Startup function of the step
110 : * @teardown: Teardown function of the step
111 : * @cant_stop: Bringup/teardown can't be stopped at this step
112 : */
113 : struct cpuhp_step {
114 : const char *name;
115 : union {
116 : int (*single)(unsigned int cpu);
117 : int (*multi)(unsigned int cpu,
118 : struct hlist_node *node);
119 : } startup;
120 : union {
121 : int (*single)(unsigned int cpu);
122 : int (*multi)(unsigned int cpu,
123 : struct hlist_node *node);
124 : } teardown;
125 : struct hlist_head list;
126 : bool cant_stop;
127 : bool multi_instance;
128 : };
129 :
130 : static DEFINE_MUTEX(cpuhp_state_mutex);
131 : static struct cpuhp_step cpuhp_hp_states[];
132 :
133 742 : static struct cpuhp_step *cpuhp_get_step(enum cpuhp_state state)
134 : {
135 742 : return cpuhp_hp_states + state;
136 : }
137 :
138 : /**
139 : * cpuhp_invoke_callback _ Invoke the callbacks for a given state
140 : * @cpu: The cpu for which the callback should be invoked
141 : * @state: The state to do callbacks for
142 : * @bringup: True if the bringup callback should be invoked
143 : * @node: For multi-instance, do a single entry callback for install/remove
144 : * @lastp: For multi-instance rollback, remember how far we got
145 : *
146 : * Called from cpu hotplug and from the state register machinery.
147 : */
148 676 : static int cpuhp_invoke_callback(unsigned int cpu, enum cpuhp_state state,
149 : bool bringup, struct hlist_node *node,
150 : struct hlist_node **lastp)
151 : {
152 676 : struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
153 676 : struct cpuhp_step *step = cpuhp_get_step(state);
154 676 : int (*cbm)(unsigned int cpu, struct hlist_node *node);
155 676 : int (*cb)(unsigned int cpu);
156 676 : int ret, cnt;
157 :
158 676 : if (st->fail == state) {
159 0 : st->fail = CPUHP_INVALID;
160 :
161 0 : if (!(bringup ? step->startup.single : step->teardown.single))
162 : return 0;
163 :
164 0 : return -EAGAIN;
165 : }
166 :
167 676 : if (!step->multi_instance) {
168 1342 : WARN_ON_ONCE(lastp && *lastp);
169 671 : cb = bringup ? step->startup.single : step->teardown.single;
170 671 : if (!cb)
171 : return 0;
172 86 : trace_cpuhp_enter(cpu, st->target, state, cb);
173 86 : ret = cb(cpu);
174 86 : trace_cpuhp_exit(cpu, st->state, state, ret);
175 86 : return ret;
176 : }
177 5 : cbm = bringup ? step->startup.multi : step->teardown.multi;
178 5 : if (!cbm)
179 : return 0;
180 :
181 : /* Single invocation for instance add/remove */
182 5 : if (node) {
183 4 : WARN_ON_ONCE(lastp && *lastp);
184 2 : trace_cpuhp_multi_enter(cpu, st->target, state, cbm, node);
185 2 : ret = cbm(cpu, node);
186 2 : trace_cpuhp_exit(cpu, st->state, state, ret);
187 2 : return ret;
188 : }
189 :
190 : /* State transition. Invoke on all instances */
191 3 : cnt = 0;
192 9 : hlist_for_each(node, &step->list) {
193 6 : if (lastp && node == *lastp)
194 : break;
195 :
196 6 : trace_cpuhp_multi_enter(cpu, st->target, state, cbm, node);
197 6 : ret = cbm(cpu, node);
198 6 : trace_cpuhp_exit(cpu, st->state, state, ret);
199 6 : if (ret) {
200 0 : if (!lastp)
201 0 : goto err;
202 :
203 0 : *lastp = node;
204 0 : return ret;
205 : }
206 6 : cnt++;
207 : }
208 3 : if (lastp)
209 0 : *lastp = NULL;
210 : return 0;
211 0 : err:
212 : /* Rollback the instances if one failed */
213 0 : cbm = !bringup ? step->startup.multi : step->teardown.multi;
214 0 : if (!cbm)
215 : return ret;
216 :
217 0 : hlist_for_each(node, &step->list) {
218 0 : if (!cnt--)
219 : break;
220 :
221 0 : trace_cpuhp_multi_enter(cpu, st->target, state, cbm, node);
222 0 : ret = cbm(cpu, node);
223 0 : trace_cpuhp_exit(cpu, st->state, state, ret);
224 : /*
225 : * Rollback must not fail,
226 : */
227 0 : WARN_ON_ONCE(ret);
228 : }
229 : return ret;
230 : }
231 :
232 : #ifdef CONFIG_SMP
233 259 : static bool cpuhp_is_ap_state(enum cpuhp_state state)
234 : {
235 : /*
236 : * The extra check for CPUHP_TEARDOWN_CPU is only for documentation
237 : * purposes as that state is handled explicitly in cpu_down.
238 : */
239 259 : return state > CPUHP_BRINGUP_CPU && state != CPUHP_TEARDOWN_CPU;
240 : }
241 :
242 18 : static inline void wait_for_ap_thread(struct cpuhp_cpu_state *st, bool bringup)
243 : {
244 15 : struct completion *done = bringup ? &st->done_up : &st->done_down;
245 15 : wait_for_completion(done);
246 15 : }
247 :
248 18 : static inline void complete_ap_thread(struct cpuhp_cpu_state *st, bool bringup)
249 : {
250 15 : struct completion *done = bringup ? &st->done_up : &st->done_down;
251 15 : complete(done);
252 18 : }
253 :
254 : /*
255 : * The former STARTING/DYING states, ran with IRQs disabled and must not fail.
256 : */
257 237 : static bool cpuhp_is_atomic_state(enum cpuhp_state state)
258 : {
259 237 : return CPUHP_AP_IDLE_DEAD <= state && state < CPUHP_AP_ONLINE;
260 : }
261 :
262 : /* Serializes the updates to cpu_online_mask, cpu_present_mask */
263 : static DEFINE_MUTEX(cpu_add_remove_lock);
264 : bool cpuhp_tasks_frozen;
265 : EXPORT_SYMBOL_GPL(cpuhp_tasks_frozen);
266 :
267 : /*
268 : * The following two APIs (cpu_maps_update_begin/done) must be used when
269 : * attempting to serialize the updates to cpu_online_mask & cpu_present_mask.
270 : */
271 3 : void cpu_maps_update_begin(void)
272 : {
273 0 : mutex_lock(&cpu_add_remove_lock);
274 0 : }
275 :
276 3 : void cpu_maps_update_done(void)
277 : {
278 0 : mutex_unlock(&cpu_add_remove_lock);
279 0 : }
280 :
281 : /*
282 : * If set, cpu_up and cpu_down will return -EBUSY and do nothing.
283 : * Should always be manipulated under cpu_add_remove_lock
284 : */
285 : static int cpu_hotplug_disabled;
286 :
287 : #ifdef CONFIG_HOTPLUG_CPU
288 :
289 : DEFINE_STATIC_PERCPU_RWSEM(cpu_hotplug_lock);
290 :
291 106 : void cpus_read_lock(void)
292 : {
293 62 : percpu_down_read(&cpu_hotplug_lock);
294 62 : }
295 : EXPORT_SYMBOL_GPL(cpus_read_lock);
296 :
297 0 : int cpus_read_trylock(void)
298 : {
299 0 : return percpu_down_read_trylock(&cpu_hotplug_lock);
300 : }
301 : EXPORT_SYMBOL_GPL(cpus_read_trylock);
302 :
303 106 : void cpus_read_unlock(void)
304 : {
305 62 : percpu_up_read(&cpu_hotplug_lock);
306 62 : }
307 : EXPORT_SYMBOL_GPL(cpus_read_unlock);
308 :
309 3 : void cpus_write_lock(void)
310 : {
311 0 : percpu_down_write(&cpu_hotplug_lock);
312 0 : }
313 :
314 3 : void cpus_write_unlock(void)
315 : {
316 0 : percpu_up_write(&cpu_hotplug_lock);
317 0 : }
318 :
319 50 : void lockdep_assert_cpus_held(void)
320 : {
321 : /*
322 : * We can't have hotplug operations before userspace starts running,
323 : * and some init codepaths will knowingly not take the hotplug lock.
324 : * This is all valid, so mute lockdep until it makes sense to report
325 : * unheld locks.
326 : */
327 50 : if (system_state < SYSTEM_RUNNING)
328 : return;
329 :
330 0 : percpu_rwsem_assert_held(&cpu_hotplug_lock);
331 : }
332 :
333 : #ifdef CONFIG_LOCKDEP
334 0 : int lockdep_is_cpus_held(void)
335 : {
336 0 : return percpu_rwsem_is_held(&cpu_hotplug_lock);
337 : }
338 : #endif
339 :
340 237 : static void lockdep_acquire_cpus_lock(void)
341 : {
342 237 : rwsem_acquire(&cpu_hotplug_lock.dep_map, 0, 0, _THIS_IP_);
343 237 : }
344 :
345 237 : static void lockdep_release_cpus_lock(void)
346 : {
347 237 : rwsem_release(&cpu_hotplug_lock.dep_map, _THIS_IP_);
348 : }
349 :
350 : /*
351 : * Wait for currently running CPU hotplug operations to complete (if any) and
352 : * disable future CPU hotplug (from sysfs). The 'cpu_add_remove_lock' protects
353 : * the 'cpu_hotplug_disabled' flag. The same lock is also acquired by the
354 : * hotplug path before performing hotplug operations. So acquiring that lock
355 : * guarantees mutual exclusion from any currently running hotplug operations.
356 : */
357 0 : void cpu_hotplug_disable(void)
358 : {
359 0 : cpu_maps_update_begin();
360 0 : cpu_hotplug_disabled++;
361 0 : cpu_maps_update_done();
362 0 : }
363 : EXPORT_SYMBOL_GPL(cpu_hotplug_disable);
364 :
365 0 : static void __cpu_hotplug_enable(void)
366 : {
367 0 : if (WARN_ONCE(!cpu_hotplug_disabled, "Unbalanced cpu hotplug enable\n"))
368 : return;
369 0 : cpu_hotplug_disabled--;
370 : }
371 :
372 0 : void cpu_hotplug_enable(void)
373 : {
374 0 : cpu_maps_update_begin();
375 0 : __cpu_hotplug_enable();
376 0 : cpu_maps_update_done();
377 0 : }
378 : EXPORT_SYMBOL_GPL(cpu_hotplug_enable);
379 :
380 : #else
381 :
382 : static void lockdep_acquire_cpus_lock(void)
383 : {
384 : }
385 :
386 : static void lockdep_release_cpus_lock(void)
387 : {
388 : }
389 :
390 : #endif /* CONFIG_HOTPLUG_CPU */
391 :
392 : /*
393 : * Architectures that need SMT-specific errata handling during SMT hotplug
394 : * should override this.
395 : */
396 0 : void __weak arch_smt_update(void) { }
397 :
398 : #ifdef CONFIG_HOTPLUG_SMT
399 : enum cpuhp_smt_control cpu_smt_control __read_mostly = CPU_SMT_ENABLED;
400 :
401 0 : void __init cpu_smt_disable(bool force)
402 : {
403 0 : if (!cpu_smt_possible())
404 : return;
405 :
406 0 : if (force) {
407 0 : pr_info("SMT: Force disabled\n");
408 0 : cpu_smt_control = CPU_SMT_FORCE_DISABLED;
409 : } else {
410 0 : pr_info("SMT: disabled\n");
411 0 : cpu_smt_control = CPU_SMT_DISABLED;
412 : }
413 : }
414 :
415 : /*
416 : * The decision whether SMT is supported can only be done after the full
417 : * CPU identification. Called from architecture code.
418 : */
419 1 : void __init cpu_smt_check_topology(void)
420 : {
421 1 : if (!topology_smt_supported())
422 1 : cpu_smt_control = CPU_SMT_NOT_SUPPORTED;
423 1 : }
424 :
425 0 : static int __init smt_cmdline_disable(char *str)
426 : {
427 0 : cpu_smt_disable(str && !strcmp(str, "force"));
428 0 : return 0;
429 : }
430 : early_param("nosmt", smt_cmdline_disable);
431 :
432 6 : static inline bool cpu_smt_allowed(unsigned int cpu)
433 : {
434 6 : if (cpu_smt_control == CPU_SMT_ENABLED)
435 : return true;
436 :
437 6 : if (topology_is_primary_thread(cpu))
438 : return true;
439 :
440 : /*
441 : * On x86 it's required to boot all logical CPUs at least once so
442 : * that the init code can get a chance to set CR4.MCE on each
443 : * CPU. Otherwise, a broadcasted MCE observing CR4.MCE=0b on any
444 : * core will shutdown the machine.
445 : */
446 0 : return !cpumask_test_cpu(cpu, &cpus_booted_once_mask);
447 : }
448 :
449 : /* Returns true if SMT is not supported of forcefully (irreversibly) disabled */
450 0 : bool cpu_smt_possible(void)
451 : {
452 0 : return cpu_smt_control != CPU_SMT_FORCE_DISABLED &&
453 : cpu_smt_control != CPU_SMT_NOT_SUPPORTED;
454 : }
455 : EXPORT_SYMBOL_GPL(cpu_smt_possible);
456 : #else
457 : static inline bool cpu_smt_allowed(unsigned int cpu) { return true; }
458 : #endif
459 :
460 : static inline enum cpuhp_state
461 6 : cpuhp_set_state(struct cpuhp_cpu_state *st, enum cpuhp_state target)
462 : {
463 6 : enum cpuhp_state prev_state = st->state;
464 :
465 6 : st->rollback = false;
466 6 : st->last = NULL;
467 :
468 6 : st->target = target;
469 6 : st->single = false;
470 6 : st->bringup = st->state < target;
471 :
472 6 : return prev_state;
473 : }
474 :
475 : static inline void
476 0 : cpuhp_reset_state(struct cpuhp_cpu_state *st, enum cpuhp_state prev_state)
477 : {
478 0 : st->rollback = true;
479 :
480 : /*
481 : * If we have st->last we need to undo partial multi_instance of this
482 : * state first. Otherwise start undo at the previous state.
483 : */
484 0 : if (!st->last) {
485 0 : if (st->bringup)
486 0 : st->state--;
487 : else
488 0 : st->state++;
489 : }
490 :
491 0 : st->target = prev_state;
492 0 : st->bringup = !st->bringup;
493 0 : }
494 :
495 : /* Regular hotplug invocation of the AP hotplug thread */
496 15 : static void __cpuhp_kick_ap(struct cpuhp_cpu_state *st)
497 : {
498 15 : if (!st->single && st->state == st->target)
499 : return;
500 :
501 15 : st->result = 0;
502 : /*
503 : * Make sure the above stores are visible before should_run becomes
504 : * true. Paired with the mb() above in cpuhp_thread_fun()
505 : */
506 15 : smp_mb();
507 15 : st->should_run = true;
508 15 : wake_up_process(st->thread);
509 15 : wait_for_ap_thread(st, st->bringup);
510 : }
511 :
512 3 : static int cpuhp_kick_ap(struct cpuhp_cpu_state *st, enum cpuhp_state target)
513 : {
514 3 : enum cpuhp_state prev_state;
515 3 : int ret;
516 :
517 3 : prev_state = cpuhp_set_state(st, target);
518 3 : __cpuhp_kick_ap(st);
519 3 : if ((ret = st->result)) {
520 0 : cpuhp_reset_state(st, prev_state);
521 0 : __cpuhp_kick_ap(st);
522 : }
523 :
524 3 : return ret;
525 : }
526 :
527 3 : static int bringup_wait_for_ap(unsigned int cpu)
528 : {
529 3 : struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
530 :
531 : /* Wait for the CPU to reach CPUHP_AP_ONLINE_IDLE */
532 3 : wait_for_ap_thread(st, true);
533 3 : if (WARN_ON_ONCE((!cpu_online(cpu))))
534 : return -ECANCELED;
535 :
536 : /* Unpark the hotplug thread of the target cpu */
537 3 : kthread_unpark(st->thread);
538 :
539 : /*
540 : * SMT soft disabling on X86 requires to bring the CPU out of the
541 : * BIOS 'wait for SIPI' state in order to set the CR4.MCE bit. The
542 : * CPU marked itself as booted_once in notify_cpu_starting() so the
543 : * cpu_smt_allowed() check will now return false if this is not the
544 : * primary sibling.
545 : */
546 3 : if (!cpu_smt_allowed(cpu))
547 : return -ECANCELED;
548 :
549 3 : if (st->target <= CPUHP_AP_ONLINE_IDLE)
550 : return 0;
551 :
552 3 : return cpuhp_kick_ap(st, st->target);
553 : }
554 :
555 3 : static int bringup_cpu(unsigned int cpu)
556 : {
557 3 : struct task_struct *idle = idle_thread_get(cpu);
558 3 : int ret;
559 :
560 : /*
561 : * Some architectures have to walk the irq descriptors to
562 : * setup the vector space for the cpu which comes online.
563 : * Prevent irq alloc/free across the bringup.
564 : */
565 3 : irq_lock_sparse();
566 :
567 : /* Arch-specific enabling code. */
568 3 : ret = __cpu_up(cpu, idle);
569 3 : irq_unlock_sparse();
570 3 : if (ret)
571 : return ret;
572 3 : return bringup_wait_for_ap(cpu);
573 : }
574 :
575 0 : static int finish_cpu(unsigned int cpu)
576 : {
577 0 : struct task_struct *idle = idle_thread_get(cpu);
578 0 : struct mm_struct *mm = idle->active_mm;
579 :
580 : /*
581 : * idle_task_exit() will have switched to &init_mm, now
582 : * clean up any remaining active_mm state.
583 : */
584 0 : if (mm != &init_mm)
585 0 : idle->active_mm = &init_mm;
586 0 : mmdrop(mm);
587 0 : return 0;
588 : }
589 :
590 : /*
591 : * Hotplug state machine related functions
592 : */
593 :
594 0 : static void undo_cpu_up(unsigned int cpu, struct cpuhp_cpu_state *st)
595 : {
596 0 : for (st->state--; st->state > st->target; st->state--)
597 0 : cpuhp_invoke_callback(cpu, st->state, false, NULL, NULL);
598 0 : }
599 :
600 0 : static inline bool can_rollback_cpu(struct cpuhp_cpu_state *st)
601 : {
602 0 : if (IS_ENABLED(CONFIG_HOTPLUG_CPU))
603 0 : return true;
604 : /*
605 : * When CPU hotplug is disabled, then taking the CPU down is not
606 : * possible because takedown_cpu() and the architecture and
607 : * subsystem specific mechanisms are not available. So the CPU
608 : * which would be completely unplugged again needs to stay around
609 : * in the current state.
610 : */
611 : return st->state <= CPUHP_BRINGUP_CPU;
612 : }
613 :
614 3 : static int cpuhp_up_callbacks(unsigned int cpu, struct cpuhp_cpu_state *st,
615 : enum cpuhp_state target)
616 : {
617 3 : enum cpuhp_state prev_state = st->state;
618 3 : int ret = 0;
619 :
620 264 : while (st->state < target) {
621 261 : st->state++;
622 261 : ret = cpuhp_invoke_callback(cpu, st->state, true, NULL, NULL);
623 261 : if (ret) {
624 0 : if (can_rollback_cpu(st)) {
625 0 : st->target = prev_state;
626 0 : undo_cpu_up(cpu, st);
627 : }
628 : break;
629 : }
630 : }
631 3 : return ret;
632 : }
633 :
634 : /*
635 : * The cpu hotplug threads manage the bringup and teardown of the cpus
636 : */
637 4 : static void cpuhp_create(unsigned int cpu)
638 : {
639 4 : struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
640 :
641 4 : init_completion(&st->done_up);
642 4 : init_completion(&st->done_down);
643 4 : }
644 :
645 253 : static int cpuhp_should_run(unsigned int cpu)
646 : {
647 253 : struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state);
648 :
649 253 : return st->should_run;
650 : }
651 :
652 : /*
653 : * Execute teardown/startup callbacks on the plugged cpu. Also used to invoke
654 : * callbacks when a state gets [un]installed at runtime.
655 : *
656 : * Each invocation of this function by the smpboot thread does a single AP
657 : * state callback.
658 : *
659 : * It has 3 modes of operation:
660 : * - single: runs st->cb_state
661 : * - up: runs ++st->state, while st->state < st->target
662 : * - down: runs st->state--, while st->state > st->target
663 : *
664 : * When complete or on error, should_run is cleared and the completion is fired.
665 : */
666 237 : static void cpuhp_thread_fun(unsigned int cpu)
667 : {
668 237 : struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state);
669 237 : bool bringup = st->bringup;
670 237 : enum cpuhp_state state;
671 :
672 237 : if (WARN_ON_ONCE(!st->should_run))
673 : return;
674 :
675 : /*
676 : * ACQUIRE for the cpuhp_should_run() load of ->should_run. Ensures
677 : * that if we see ->should_run we also see the rest of the state.
678 : */
679 237 : smp_mb();
680 :
681 : /*
682 : * The BP holds the hotplug lock, but we're now running on the AP,
683 : * ensure that anybody asserting the lock is held, will actually find
684 : * it so.
685 : */
686 237 : lockdep_acquire_cpus_lock();
687 237 : cpuhp_lock_acquire(bringup);
688 :
689 237 : if (st->single) {
690 12 : state = st->cb_state;
691 12 : st->should_run = false;
692 : } else {
693 225 : if (bringup) {
694 225 : st->state++;
695 225 : state = st->state;
696 225 : st->should_run = (st->state < st->target);
697 225 : WARN_ON_ONCE(st->state > st->target);
698 : } else {
699 0 : state = st->state;
700 0 : st->state--;
701 0 : st->should_run = (st->state > st->target);
702 0 : WARN_ON_ONCE(st->state < st->target);
703 : }
704 : }
705 :
706 237 : WARN_ON_ONCE(!cpuhp_is_ap_state(state));
707 :
708 237 : if (cpuhp_is_atomic_state(state)) {
709 0 : local_irq_disable();
710 0 : st->result = cpuhp_invoke_callback(cpu, state, bringup, st->node, &st->last);
711 0 : local_irq_enable();
712 :
713 : /*
714 : * STARTING/DYING must not fail!
715 : */
716 0 : WARN_ON_ONCE(st->result);
717 : } else {
718 237 : st->result = cpuhp_invoke_callback(cpu, state, bringup, st->node, &st->last);
719 : }
720 :
721 237 : if (st->result) {
722 : /*
723 : * If we fail on a rollback, we're up a creek without no
724 : * paddle, no way forward, no way back. We loose, thanks for
725 : * playing.
726 : */
727 0 : WARN_ON_ONCE(st->rollback);
728 0 : st->should_run = false;
729 : }
730 :
731 237 : cpuhp_lock_release(bringup);
732 237 : lockdep_release_cpus_lock();
733 :
734 237 : if (!st->should_run)
735 15 : complete_ap_thread(st, bringup);
736 : }
737 :
738 : /* Invoke a single callback on a remote cpu */
739 : static int
740 15 : cpuhp_invoke_ap_callback(int cpu, enum cpuhp_state state, bool bringup,
741 : struct hlist_node *node)
742 : {
743 15 : struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
744 15 : int ret;
745 :
746 15 : if (!cpu_online(cpu))
747 : return 0;
748 :
749 15 : cpuhp_lock_acquire(false);
750 15 : cpuhp_lock_release(false);
751 :
752 15 : cpuhp_lock_acquire(true);
753 15 : cpuhp_lock_release(true);
754 :
755 : /*
756 : * If we are up and running, use the hotplug thread. For early calls
757 : * we invoke the thread function directly.
758 : */
759 15 : if (!st->thread)
760 3 : return cpuhp_invoke_callback(cpu, state, bringup, node, NULL);
761 :
762 12 : st->rollback = false;
763 12 : st->last = NULL;
764 :
765 12 : st->node = node;
766 12 : st->bringup = bringup;
767 12 : st->cb_state = state;
768 12 : st->single = true;
769 :
770 12 : __cpuhp_kick_ap(st);
771 :
772 : /*
773 : * If we failed and did a partial, do a rollback.
774 : */
775 12 : if ((ret = st->result) && st->last) {
776 0 : st->rollback = true;
777 0 : st->bringup = !bringup;
778 :
779 0 : __cpuhp_kick_ap(st);
780 : }
781 :
782 : /*
783 : * Clean up the leftovers so the next hotplug operation wont use stale
784 : * data.
785 : */
786 12 : st->node = st->last = NULL;
787 12 : return ret;
788 : }
789 :
790 0 : static int cpuhp_kick_ap_work(unsigned int cpu)
791 : {
792 0 : struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
793 0 : enum cpuhp_state prev_state = st->state;
794 0 : int ret;
795 :
796 0 : cpuhp_lock_acquire(false);
797 0 : cpuhp_lock_release(false);
798 :
799 0 : cpuhp_lock_acquire(true);
800 0 : cpuhp_lock_release(true);
801 :
802 0 : trace_cpuhp_enter(cpu, st->target, prev_state, cpuhp_kick_ap_work);
803 0 : ret = cpuhp_kick_ap(st, st->target);
804 0 : trace_cpuhp_exit(cpu, st->state, prev_state, ret);
805 :
806 0 : return ret;
807 : }
808 :
809 : static struct smp_hotplug_thread cpuhp_threads = {
810 : .store = &cpuhp_state.thread,
811 : .create = &cpuhp_create,
812 : .thread_should_run = cpuhp_should_run,
813 : .thread_fn = cpuhp_thread_fun,
814 : .thread_comm = "cpuhp/%u",
815 : .selfparking = true,
816 : };
817 :
818 1 : void __init cpuhp_threads_init(void)
819 : {
820 1 : BUG_ON(smpboot_register_percpu_thread(&cpuhp_threads));
821 1 : kthread_unpark(this_cpu_read(cpuhp_state.thread));
822 1 : }
823 :
824 : #ifdef CONFIG_HOTPLUG_CPU
825 : #ifndef arch_clear_mm_cpumask_cpu
826 : #define arch_clear_mm_cpumask_cpu(cpu, mm) cpumask_clear_cpu(cpu, mm_cpumask(mm))
827 : #endif
828 :
829 : /**
830 : * clear_tasks_mm_cpumask - Safely clear tasks' mm_cpumask for a CPU
831 : * @cpu: a CPU id
832 : *
833 : * This function walks all processes, finds a valid mm struct for each one and
834 : * then clears a corresponding bit in mm's cpumask. While this all sounds
835 : * trivial, there are various non-obvious corner cases, which this function
836 : * tries to solve in a safe manner.
837 : *
838 : * Also note that the function uses a somewhat relaxed locking scheme, so it may
839 : * be called only for an already offlined CPU.
840 : */
841 0 : void clear_tasks_mm_cpumask(int cpu)
842 : {
843 0 : struct task_struct *p;
844 :
845 : /*
846 : * This function is called after the cpu is taken down and marked
847 : * offline, so its not like new tasks will ever get this cpu set in
848 : * their mm mask. -- Peter Zijlstra
849 : * Thus, we may use rcu_read_lock() here, instead of grabbing
850 : * full-fledged tasklist_lock.
851 : */
852 0 : WARN_ON(cpu_online(cpu));
853 0 : rcu_read_lock();
854 0 : for_each_process(p) {
855 0 : struct task_struct *t;
856 :
857 : /*
858 : * Main thread might exit, but other threads may still have
859 : * a valid mm. Find one.
860 : */
861 0 : t = find_lock_task_mm(p);
862 0 : if (!t)
863 0 : continue;
864 0 : arch_clear_mm_cpumask_cpu(cpu, t->mm);
865 0 : task_unlock(t);
866 : }
867 0 : rcu_read_unlock();
868 0 : }
869 :
870 : /* Take this CPU down. */
871 0 : static int take_cpu_down(void *_param)
872 : {
873 0 : struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state);
874 0 : enum cpuhp_state target = max((int)st->target, CPUHP_AP_OFFLINE);
875 0 : int err, cpu = smp_processor_id();
876 0 : int ret;
877 :
878 : /* Ensure this CPU doesn't handle any more interrupts. */
879 0 : err = __cpu_disable();
880 0 : if (err < 0)
881 : return err;
882 :
883 : /*
884 : * We get here while we are in CPUHP_TEARDOWN_CPU state and we must not
885 : * do this step again.
886 : */
887 0 : WARN_ON(st->state != CPUHP_TEARDOWN_CPU);
888 0 : st->state--;
889 : /* Invoke the former CPU_DYING callbacks */
890 0 : for (; st->state > target; st->state--) {
891 0 : ret = cpuhp_invoke_callback(cpu, st->state, false, NULL, NULL);
892 : /*
893 : * DYING must not fail!
894 : */
895 0 : WARN_ON_ONCE(ret);
896 : }
897 :
898 : /* Give up timekeeping duties */
899 0 : tick_handover_do_timer();
900 : /* Remove CPU from timer broadcasting */
901 0 : tick_offline_cpu(cpu);
902 : /* Park the stopper thread */
903 0 : stop_machine_park(cpu);
904 0 : return 0;
905 : }
906 :
907 0 : static int takedown_cpu(unsigned int cpu)
908 : {
909 0 : struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
910 0 : int err;
911 :
912 : /* Park the smpboot threads */
913 0 : kthread_park(per_cpu_ptr(&cpuhp_state, cpu)->thread);
914 :
915 : /*
916 : * Prevent irq alloc/free while the dying cpu reorganizes the
917 : * interrupt affinities.
918 : */
919 0 : irq_lock_sparse();
920 :
921 : /*
922 : * So now all preempt/rcu users must observe !cpu_active().
923 : */
924 0 : err = stop_machine_cpuslocked(take_cpu_down, NULL, cpumask_of(cpu));
925 0 : if (err) {
926 : /* CPU refused to die */
927 0 : irq_unlock_sparse();
928 : /* Unpark the hotplug thread so we can rollback there */
929 0 : kthread_unpark(per_cpu_ptr(&cpuhp_state, cpu)->thread);
930 0 : return err;
931 : }
932 0 : BUG_ON(cpu_online(cpu));
933 :
934 : /*
935 : * The teardown callback for CPUHP_AP_SCHED_STARTING will have removed
936 : * all runnable tasks from the CPU, there's only the idle task left now
937 : * that the migration thread is done doing the stop_machine thing.
938 : *
939 : * Wait for the stop thread to go away.
940 : */
941 0 : wait_for_ap_thread(st, false);
942 0 : BUG_ON(st->state != CPUHP_AP_IDLE_DEAD);
943 :
944 : /* Interrupts are moved away from the dying cpu, reenable alloc/free */
945 0 : irq_unlock_sparse();
946 :
947 0 : hotplug_cpu__broadcast_tick_pull(cpu);
948 : /* This actually kills the CPU. */
949 0 : __cpu_die(cpu);
950 :
951 0 : tick_cleanup_dead_cpu(cpu);
952 0 : rcutree_migrate_callbacks(cpu);
953 0 : return 0;
954 : }
955 :
956 0 : static void cpuhp_complete_idle_dead(void *arg)
957 : {
958 0 : struct cpuhp_cpu_state *st = arg;
959 :
960 0 : complete_ap_thread(st, false);
961 0 : }
962 :
963 0 : void cpuhp_report_idle_dead(void)
964 : {
965 0 : struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state);
966 :
967 0 : BUG_ON(st->state != CPUHP_AP_OFFLINE);
968 0 : rcu_report_dead(smp_processor_id());
969 0 : st->state = CPUHP_AP_IDLE_DEAD;
970 : /*
971 : * We cannot call complete after rcu_report_dead() so we delegate it
972 : * to an online cpu.
973 : */
974 0 : smp_call_function_single(cpumask_first(cpu_online_mask),
975 : cpuhp_complete_idle_dead, st, 0);
976 0 : }
977 :
978 0 : static void undo_cpu_down(unsigned int cpu, struct cpuhp_cpu_state *st)
979 : {
980 0 : for (st->state++; st->state < st->target; st->state++)
981 0 : cpuhp_invoke_callback(cpu, st->state, true, NULL, NULL);
982 0 : }
983 :
984 0 : static int cpuhp_down_callbacks(unsigned int cpu, struct cpuhp_cpu_state *st,
985 : enum cpuhp_state target)
986 : {
987 0 : enum cpuhp_state prev_state = st->state;
988 0 : int ret = 0;
989 :
990 0 : for (; st->state > target; st->state--) {
991 0 : ret = cpuhp_invoke_callback(cpu, st->state, false, NULL, NULL);
992 0 : if (ret) {
993 0 : st->target = prev_state;
994 0 : if (st->state < prev_state)
995 0 : undo_cpu_down(cpu, st);
996 : break;
997 : }
998 : }
999 0 : return ret;
1000 : }
1001 :
1002 : /* Requires cpu_add_remove_lock to be held */
1003 0 : static int __ref _cpu_down(unsigned int cpu, int tasks_frozen,
1004 : enum cpuhp_state target)
1005 : {
1006 0 : struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
1007 0 : int prev_state, ret = 0;
1008 :
1009 0 : if (num_online_cpus() == 1)
1010 : return -EBUSY;
1011 :
1012 0 : if (!cpu_present(cpu))
1013 : return -EINVAL;
1014 :
1015 0 : cpus_write_lock();
1016 :
1017 0 : cpuhp_tasks_frozen = tasks_frozen;
1018 :
1019 0 : prev_state = cpuhp_set_state(st, target);
1020 : /*
1021 : * If the current CPU state is in the range of the AP hotplug thread,
1022 : * then we need to kick the thread.
1023 : */
1024 0 : if (st->state > CPUHP_TEARDOWN_CPU) {
1025 0 : st->target = max((int)target, CPUHP_TEARDOWN_CPU);
1026 0 : ret = cpuhp_kick_ap_work(cpu);
1027 : /*
1028 : * The AP side has done the error rollback already. Just
1029 : * return the error code..
1030 : */
1031 0 : if (ret)
1032 0 : goto out;
1033 :
1034 : /*
1035 : * We might have stopped still in the range of the AP hotplug
1036 : * thread. Nothing to do anymore.
1037 : */
1038 0 : if (st->state > CPUHP_TEARDOWN_CPU)
1039 0 : goto out;
1040 :
1041 0 : st->target = target;
1042 : }
1043 : /*
1044 : * The AP brought itself down to CPUHP_TEARDOWN_CPU. So we need
1045 : * to do the further cleanups.
1046 : */
1047 0 : ret = cpuhp_down_callbacks(cpu, st, target);
1048 0 : if (ret && st->state == CPUHP_TEARDOWN_CPU && st->state < prev_state) {
1049 0 : cpuhp_reset_state(st, prev_state);
1050 0 : __cpuhp_kick_ap(st);
1051 : }
1052 :
1053 0 : out:
1054 0 : cpus_write_unlock();
1055 : /*
1056 : * Do post unplug cleanup. This is still protected against
1057 : * concurrent CPU hotplug via cpu_add_remove_lock.
1058 : */
1059 0 : lockup_detector_cleanup();
1060 0 : arch_smt_update();
1061 0 : return ret;
1062 : }
1063 :
1064 0 : static int cpu_down_maps_locked(unsigned int cpu, enum cpuhp_state target)
1065 : {
1066 0 : if (cpu_hotplug_disabled)
1067 : return -EBUSY;
1068 0 : return _cpu_down(cpu, 0, target);
1069 : }
1070 :
1071 0 : static int cpu_down(unsigned int cpu, enum cpuhp_state target)
1072 : {
1073 0 : int err;
1074 :
1075 0 : cpu_maps_update_begin();
1076 0 : err = cpu_down_maps_locked(cpu, target);
1077 0 : cpu_maps_update_done();
1078 0 : return err;
1079 : }
1080 :
1081 : /**
1082 : * cpu_device_down - Bring down a cpu device
1083 : * @dev: Pointer to the cpu device to offline
1084 : *
1085 : * This function is meant to be used by device core cpu subsystem only.
1086 : *
1087 : * Other subsystems should use remove_cpu() instead.
1088 : */
1089 0 : int cpu_device_down(struct device *dev)
1090 : {
1091 0 : return cpu_down(dev->id, CPUHP_OFFLINE);
1092 : }
1093 :
1094 0 : int remove_cpu(unsigned int cpu)
1095 : {
1096 0 : int ret;
1097 :
1098 0 : lock_device_hotplug();
1099 0 : ret = device_offline(get_cpu_device(cpu));
1100 0 : unlock_device_hotplug();
1101 :
1102 0 : return ret;
1103 : }
1104 : EXPORT_SYMBOL_GPL(remove_cpu);
1105 :
1106 0 : void smp_shutdown_nonboot_cpus(unsigned int primary_cpu)
1107 : {
1108 0 : unsigned int cpu;
1109 0 : int error;
1110 :
1111 0 : cpu_maps_update_begin();
1112 :
1113 : /*
1114 : * Make certain the cpu I'm about to reboot on is online.
1115 : *
1116 : * This is inline to what migrate_to_reboot_cpu() already do.
1117 : */
1118 0 : if (!cpu_online(primary_cpu))
1119 0 : primary_cpu = cpumask_first(cpu_online_mask);
1120 :
1121 0 : for_each_online_cpu(cpu) {
1122 0 : if (cpu == primary_cpu)
1123 0 : continue;
1124 :
1125 0 : error = cpu_down_maps_locked(cpu, CPUHP_OFFLINE);
1126 0 : if (error) {
1127 0 : pr_err("Failed to offline CPU%d - error=%d",
1128 : cpu, error);
1129 0 : break;
1130 : }
1131 : }
1132 :
1133 : /*
1134 : * Ensure all but the reboot CPU are offline.
1135 : */
1136 0 : BUG_ON(num_online_cpus() > 1);
1137 :
1138 : /*
1139 : * Make sure the CPUs won't be enabled by someone else after this
1140 : * point. Kexec will reboot to a new kernel shortly resetting
1141 : * everything along the way.
1142 : */
1143 0 : cpu_hotplug_disabled++;
1144 :
1145 0 : cpu_maps_update_done();
1146 0 : }
1147 :
1148 : #else
1149 : #define takedown_cpu NULL
1150 : #endif /*CONFIG_HOTPLUG_CPU*/
1151 :
1152 : /**
1153 : * notify_cpu_starting(cpu) - Invoke the callbacks on the starting CPU
1154 : * @cpu: cpu that just started
1155 : *
1156 : * It must be called by the arch code on the new cpu, before the new cpu
1157 : * enables interrupts and before the "boot" cpu returns from __cpu_up().
1158 : */
1159 3 : void notify_cpu_starting(unsigned int cpu)
1160 : {
1161 3 : struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
1162 3 : enum cpuhp_state target = min((int)st->target, CPUHP_AP_ONLINE);
1163 3 : int ret;
1164 :
1165 3 : rcu_cpu_starting(cpu); /* Enables RCU usage on this CPU. */
1166 3 : cpumask_set_cpu(cpu, &cpus_booted_once_mask);
1167 171 : while (st->state < target) {
1168 168 : st->state++;
1169 168 : ret = cpuhp_invoke_callback(cpu, st->state, true, NULL, NULL);
1170 : /*
1171 : * STARTING must not fail!
1172 : */
1173 168 : WARN_ON_ONCE(ret);
1174 : }
1175 3 : }
1176 :
1177 : /*
1178 : * Called from the idle task. Wake up the controlling task which brings the
1179 : * hotplug thread of the upcoming CPU up and then delegates the rest of the
1180 : * online bringup to the hotplug thread.
1181 : */
1182 4 : void cpuhp_online_idle(enum cpuhp_state state)
1183 : {
1184 4 : struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state);
1185 :
1186 : /* Happens for the boot cpu */
1187 4 : if (state != CPUHP_AP_ONLINE_IDLE)
1188 : return;
1189 :
1190 : /*
1191 : * Unpart the stopper thread before we start the idle loop (and start
1192 : * scheduling); this ensures the stopper task is always available.
1193 : */
1194 3 : stop_machine_unpark(smp_processor_id());
1195 :
1196 3 : st->state = CPUHP_AP_ONLINE_IDLE;
1197 3 : complete_ap_thread(st, true);
1198 : }
1199 :
1200 : /* Requires cpu_add_remove_lock to be held */
1201 3 : static int _cpu_up(unsigned int cpu, int tasks_frozen, enum cpuhp_state target)
1202 : {
1203 3 : struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
1204 3 : struct task_struct *idle;
1205 3 : int ret = 0;
1206 :
1207 3 : cpus_write_lock();
1208 :
1209 3 : if (!cpu_present(cpu)) {
1210 0 : ret = -EINVAL;
1211 0 : goto out;
1212 : }
1213 :
1214 : /*
1215 : * The caller of cpu_up() might have raced with another
1216 : * caller. Nothing to do.
1217 : */
1218 3 : if (st->state >= target)
1219 0 : goto out;
1220 :
1221 3 : if (st->state == CPUHP_OFFLINE) {
1222 : /* Let it fail before we try to bring the cpu up */
1223 3 : idle = idle_thread_get(cpu);
1224 3 : if (IS_ERR(idle)) {
1225 0 : ret = PTR_ERR(idle);
1226 0 : goto out;
1227 : }
1228 : }
1229 :
1230 3 : cpuhp_tasks_frozen = tasks_frozen;
1231 :
1232 3 : cpuhp_set_state(st, target);
1233 : /*
1234 : * If the current CPU state is in the range of the AP hotplug thread,
1235 : * then we need to kick the thread once more.
1236 : */
1237 3 : if (st->state > CPUHP_BRINGUP_CPU) {
1238 0 : ret = cpuhp_kick_ap_work(cpu);
1239 : /*
1240 : * The AP side has done the error rollback already. Just
1241 : * return the error code..
1242 : */
1243 0 : if (ret)
1244 0 : goto out;
1245 : }
1246 :
1247 : /*
1248 : * Try to reach the target state. We max out on the BP at
1249 : * CPUHP_BRINGUP_CPU. After that the AP hotplug thread is
1250 : * responsible for bringing it up to the target state.
1251 : */
1252 3 : target = min((int)target, CPUHP_BRINGUP_CPU);
1253 3 : ret = cpuhp_up_callbacks(cpu, st, target);
1254 3 : out:
1255 3 : cpus_write_unlock();
1256 3 : arch_smt_update();
1257 3 : return ret;
1258 : }
1259 :
1260 3 : static int cpu_up(unsigned int cpu, enum cpuhp_state target)
1261 : {
1262 3 : int err = 0;
1263 :
1264 3 : if (!cpu_possible(cpu)) {
1265 0 : pr_err("can't online cpu %d because it is not configured as may-hotadd at boot time\n",
1266 : cpu);
1267 : #if defined(CONFIG_IA64)
1268 : pr_err("please check additional_cpus= boot parameter\n");
1269 : #endif
1270 0 : return -EINVAL;
1271 : }
1272 :
1273 3 : err = try_online_node(cpu_to_node(cpu));
1274 3 : if (err)
1275 : return err;
1276 :
1277 3 : cpu_maps_update_begin();
1278 :
1279 3 : if (cpu_hotplug_disabled) {
1280 0 : err = -EBUSY;
1281 0 : goto out;
1282 : }
1283 3 : if (!cpu_smt_allowed(cpu)) {
1284 0 : err = -EPERM;
1285 0 : goto out;
1286 : }
1287 :
1288 3 : err = _cpu_up(cpu, 0, target);
1289 3 : out:
1290 3 : cpu_maps_update_done();
1291 3 : return err;
1292 : }
1293 :
1294 : /**
1295 : * cpu_device_up - Bring up a cpu device
1296 : * @dev: Pointer to the cpu device to online
1297 : *
1298 : * This function is meant to be used by device core cpu subsystem only.
1299 : *
1300 : * Other subsystems should use add_cpu() instead.
1301 : */
1302 0 : int cpu_device_up(struct device *dev)
1303 : {
1304 0 : return cpu_up(dev->id, CPUHP_ONLINE);
1305 : }
1306 :
1307 0 : int add_cpu(unsigned int cpu)
1308 : {
1309 0 : int ret;
1310 :
1311 0 : lock_device_hotplug();
1312 0 : ret = device_online(get_cpu_device(cpu));
1313 0 : unlock_device_hotplug();
1314 :
1315 0 : return ret;
1316 : }
1317 : EXPORT_SYMBOL_GPL(add_cpu);
1318 :
1319 : /**
1320 : * bringup_hibernate_cpu - Bring up the CPU that we hibernated on
1321 : * @sleep_cpu: The cpu we hibernated on and should be brought up.
1322 : *
1323 : * On some architectures like arm64, we can hibernate on any CPU, but on
1324 : * wake up the CPU we hibernated on might be offline as a side effect of
1325 : * using maxcpus= for example.
1326 : */
1327 0 : int bringup_hibernate_cpu(unsigned int sleep_cpu)
1328 : {
1329 0 : int ret;
1330 :
1331 0 : if (!cpu_online(sleep_cpu)) {
1332 0 : pr_info("Hibernated on a CPU that is offline! Bringing CPU up.\n");
1333 0 : ret = cpu_up(sleep_cpu, CPUHP_ONLINE);
1334 0 : if (ret) {
1335 0 : pr_err("Failed to bring hibernate-CPU up!\n");
1336 0 : return ret;
1337 : }
1338 : }
1339 : return 0;
1340 : }
1341 :
1342 1 : void bringup_nonboot_cpus(unsigned int setup_max_cpus)
1343 : {
1344 1 : unsigned int cpu;
1345 :
1346 5 : for_each_present_cpu(cpu) {
1347 4 : if (num_online_cpus() >= setup_max_cpus)
1348 : break;
1349 4 : if (!cpu_online(cpu))
1350 3 : cpu_up(cpu, CPUHP_ONLINE);
1351 : }
1352 1 : }
1353 :
1354 : #ifdef CONFIG_PM_SLEEP_SMP
1355 : static cpumask_var_t frozen_cpus;
1356 :
1357 : int freeze_secondary_cpus(int primary)
1358 : {
1359 : int cpu, error = 0;
1360 :
1361 : cpu_maps_update_begin();
1362 : if (primary == -1) {
1363 : primary = cpumask_first(cpu_online_mask);
1364 : if (!housekeeping_cpu(primary, HK_FLAG_TIMER))
1365 : primary = housekeeping_any_cpu(HK_FLAG_TIMER);
1366 : } else {
1367 : if (!cpu_online(primary))
1368 : primary = cpumask_first(cpu_online_mask);
1369 : }
1370 :
1371 : /*
1372 : * We take down all of the non-boot CPUs in one shot to avoid races
1373 : * with the userspace trying to use the CPU hotplug at the same time
1374 : */
1375 : cpumask_clear(frozen_cpus);
1376 :
1377 : pr_info("Disabling non-boot CPUs ...\n");
1378 : for_each_online_cpu(cpu) {
1379 : if (cpu == primary)
1380 : continue;
1381 :
1382 : if (pm_wakeup_pending()) {
1383 : pr_info("Wakeup pending. Abort CPU freeze\n");
1384 : error = -EBUSY;
1385 : break;
1386 : }
1387 :
1388 : trace_suspend_resume(TPS("CPU_OFF"), cpu, true);
1389 : error = _cpu_down(cpu, 1, CPUHP_OFFLINE);
1390 : trace_suspend_resume(TPS("CPU_OFF"), cpu, false);
1391 : if (!error)
1392 : cpumask_set_cpu(cpu, frozen_cpus);
1393 : else {
1394 : pr_err("Error taking CPU%d down: %d\n", cpu, error);
1395 : break;
1396 : }
1397 : }
1398 :
1399 : if (!error)
1400 : BUG_ON(num_online_cpus() > 1);
1401 : else
1402 : pr_err("Non-boot CPUs are not disabled\n");
1403 :
1404 : /*
1405 : * Make sure the CPUs won't be enabled by someone else. We need to do
1406 : * this even in case of failure as all freeze_secondary_cpus() users are
1407 : * supposed to do thaw_secondary_cpus() on the failure path.
1408 : */
1409 : cpu_hotplug_disabled++;
1410 :
1411 : cpu_maps_update_done();
1412 : return error;
1413 : }
1414 :
1415 : void __weak arch_thaw_secondary_cpus_begin(void)
1416 : {
1417 : }
1418 :
1419 : void __weak arch_thaw_secondary_cpus_end(void)
1420 : {
1421 : }
1422 :
1423 : void thaw_secondary_cpus(void)
1424 : {
1425 : int cpu, error;
1426 :
1427 : /* Allow everyone to use the CPU hotplug again */
1428 : cpu_maps_update_begin();
1429 : __cpu_hotplug_enable();
1430 : if (cpumask_empty(frozen_cpus))
1431 : goto out;
1432 :
1433 : pr_info("Enabling non-boot CPUs ...\n");
1434 :
1435 : arch_thaw_secondary_cpus_begin();
1436 :
1437 : for_each_cpu(cpu, frozen_cpus) {
1438 : trace_suspend_resume(TPS("CPU_ON"), cpu, true);
1439 : error = _cpu_up(cpu, 1, CPUHP_ONLINE);
1440 : trace_suspend_resume(TPS("CPU_ON"), cpu, false);
1441 : if (!error) {
1442 : pr_info("CPU%d is up\n", cpu);
1443 : continue;
1444 : }
1445 : pr_warn("Error taking CPU%d up: %d\n", cpu, error);
1446 : }
1447 :
1448 : arch_thaw_secondary_cpus_end();
1449 :
1450 : cpumask_clear(frozen_cpus);
1451 : out:
1452 : cpu_maps_update_done();
1453 : }
1454 :
1455 : static int __init alloc_frozen_cpus(void)
1456 : {
1457 : if (!alloc_cpumask_var(&frozen_cpus, GFP_KERNEL|__GFP_ZERO))
1458 : return -ENOMEM;
1459 : return 0;
1460 : }
1461 : core_initcall(alloc_frozen_cpus);
1462 :
1463 : /*
1464 : * When callbacks for CPU hotplug notifications are being executed, we must
1465 : * ensure that the state of the system with respect to the tasks being frozen
1466 : * or not, as reported by the notification, remains unchanged *throughout the
1467 : * duration* of the execution of the callbacks.
1468 : * Hence we need to prevent the freezer from racing with regular CPU hotplug.
1469 : *
1470 : * This synchronization is implemented by mutually excluding regular CPU
1471 : * hotplug and Suspend/Hibernate call paths by hooking onto the Suspend/
1472 : * Hibernate notifications.
1473 : */
1474 : static int
1475 : cpu_hotplug_pm_callback(struct notifier_block *nb,
1476 : unsigned long action, void *ptr)
1477 : {
1478 : switch (action) {
1479 :
1480 : case PM_SUSPEND_PREPARE:
1481 : case PM_HIBERNATION_PREPARE:
1482 : cpu_hotplug_disable();
1483 : break;
1484 :
1485 : case PM_POST_SUSPEND:
1486 : case PM_POST_HIBERNATION:
1487 : cpu_hotplug_enable();
1488 : break;
1489 :
1490 : default:
1491 : return NOTIFY_DONE;
1492 : }
1493 :
1494 : return NOTIFY_OK;
1495 : }
1496 :
1497 :
1498 : static int __init cpu_hotplug_pm_sync_init(void)
1499 : {
1500 : /*
1501 : * cpu_hotplug_pm_callback has higher priority than x86
1502 : * bsp_pm_callback which depends on cpu_hotplug_pm_callback
1503 : * to disable cpu hotplug to avoid cpu hotplug race.
1504 : */
1505 : pm_notifier(cpu_hotplug_pm_callback, 0);
1506 : return 0;
1507 : }
1508 : core_initcall(cpu_hotplug_pm_sync_init);
1509 :
1510 : #endif /* CONFIG_PM_SLEEP_SMP */
1511 :
1512 : int __boot_cpu_id;
1513 :
1514 : #endif /* CONFIG_SMP */
1515 :
1516 : /* Boot processor state steps */
1517 : static struct cpuhp_step cpuhp_hp_states[] = {
1518 : [CPUHP_OFFLINE] = {
1519 : .name = "offline",
1520 : .startup.single = NULL,
1521 : .teardown.single = NULL,
1522 : },
1523 : #ifdef CONFIG_SMP
1524 : [CPUHP_CREATE_THREADS]= {
1525 : .name = "threads:prepare",
1526 : .startup.single = smpboot_create_threads,
1527 : .teardown.single = NULL,
1528 : .cant_stop = true,
1529 : },
1530 : [CPUHP_PERF_PREPARE] = {
1531 : .name = "perf:prepare",
1532 : .startup.single = perf_event_init_cpu,
1533 : .teardown.single = perf_event_exit_cpu,
1534 : },
1535 : [CPUHP_WORKQUEUE_PREP] = {
1536 : .name = "workqueue:prepare",
1537 : .startup.single = workqueue_prepare_cpu,
1538 : .teardown.single = NULL,
1539 : },
1540 : [CPUHP_HRTIMERS_PREPARE] = {
1541 : .name = "hrtimers:prepare",
1542 : .startup.single = hrtimers_prepare_cpu,
1543 : .teardown.single = hrtimers_dead_cpu,
1544 : },
1545 : [CPUHP_SMPCFD_PREPARE] = {
1546 : .name = "smpcfd:prepare",
1547 : .startup.single = smpcfd_prepare_cpu,
1548 : .teardown.single = smpcfd_dead_cpu,
1549 : },
1550 : [CPUHP_RELAY_PREPARE] = {
1551 : .name = "relay:prepare",
1552 : .startup.single = relay_prepare_cpu,
1553 : .teardown.single = NULL,
1554 : },
1555 : [CPUHP_SLAB_PREPARE] = {
1556 : .name = "slab:prepare",
1557 : .startup.single = slab_prepare_cpu,
1558 : .teardown.single = slab_dead_cpu,
1559 : },
1560 : [CPUHP_RCUTREE_PREP] = {
1561 : .name = "RCU/tree:prepare",
1562 : .startup.single = rcutree_prepare_cpu,
1563 : .teardown.single = rcutree_dead_cpu,
1564 : },
1565 : /*
1566 : * On the tear-down path, timers_dead_cpu() must be invoked
1567 : * before blk_mq_queue_reinit_notify() from notify_dead(),
1568 : * otherwise a RCU stall occurs.
1569 : */
1570 : [CPUHP_TIMERS_PREPARE] = {
1571 : .name = "timers:prepare",
1572 : .startup.single = timers_prepare_cpu,
1573 : .teardown.single = timers_dead_cpu,
1574 : },
1575 : /* Kicks the plugged cpu into life */
1576 : [CPUHP_BRINGUP_CPU] = {
1577 : .name = "cpu:bringup",
1578 : .startup.single = bringup_cpu,
1579 : .teardown.single = finish_cpu,
1580 : .cant_stop = true,
1581 : },
1582 : /* Final state before CPU kills itself */
1583 : [CPUHP_AP_IDLE_DEAD] = {
1584 : .name = "idle:dead",
1585 : },
1586 : /*
1587 : * Last state before CPU enters the idle loop to die. Transient state
1588 : * for synchronization.
1589 : */
1590 : [CPUHP_AP_OFFLINE] = {
1591 : .name = "ap:offline",
1592 : .cant_stop = true,
1593 : },
1594 : /* First state is scheduler control. Interrupts are disabled */
1595 : [CPUHP_AP_SCHED_STARTING] = {
1596 : .name = "sched:starting",
1597 : .startup.single = sched_cpu_starting,
1598 : .teardown.single = sched_cpu_dying,
1599 : },
1600 : [CPUHP_AP_RCUTREE_DYING] = {
1601 : .name = "RCU/tree:dying",
1602 : .startup.single = NULL,
1603 : .teardown.single = rcutree_dying_cpu,
1604 : },
1605 : [CPUHP_AP_SMPCFD_DYING] = {
1606 : .name = "smpcfd:dying",
1607 : .startup.single = NULL,
1608 : .teardown.single = smpcfd_dying_cpu,
1609 : },
1610 : /* Entry state on starting. Interrupts enabled from here on. Transient
1611 : * state for synchronsization */
1612 : [CPUHP_AP_ONLINE] = {
1613 : .name = "ap:online",
1614 : },
1615 : /*
1616 : * Handled on control processor until the plugged processor manages
1617 : * this itself.
1618 : */
1619 : [CPUHP_TEARDOWN_CPU] = {
1620 : .name = "cpu:teardown",
1621 : .startup.single = NULL,
1622 : .teardown.single = takedown_cpu,
1623 : .cant_stop = true,
1624 : },
1625 :
1626 : [CPUHP_AP_SCHED_WAIT_EMPTY] = {
1627 : .name = "sched:waitempty",
1628 : .startup.single = NULL,
1629 : .teardown.single = sched_cpu_wait_empty,
1630 : },
1631 :
1632 : /* Handle smpboot threads park/unpark */
1633 : [CPUHP_AP_SMPBOOT_THREADS] = {
1634 : .name = "smpboot/threads:online",
1635 : .startup.single = smpboot_unpark_threads,
1636 : .teardown.single = smpboot_park_threads,
1637 : },
1638 : [CPUHP_AP_IRQ_AFFINITY_ONLINE] = {
1639 : .name = "irq/affinity:online",
1640 : .startup.single = irq_affinity_online_cpu,
1641 : .teardown.single = NULL,
1642 : },
1643 : [CPUHP_AP_PERF_ONLINE] = {
1644 : .name = "perf:online",
1645 : .startup.single = perf_event_init_cpu,
1646 : .teardown.single = perf_event_exit_cpu,
1647 : },
1648 : [CPUHP_AP_WATCHDOG_ONLINE] = {
1649 : .name = "lockup_detector:online",
1650 : .startup.single = lockup_detector_online_cpu,
1651 : .teardown.single = lockup_detector_offline_cpu,
1652 : },
1653 : [CPUHP_AP_WORKQUEUE_ONLINE] = {
1654 : .name = "workqueue:online",
1655 : .startup.single = workqueue_online_cpu,
1656 : .teardown.single = workqueue_offline_cpu,
1657 : },
1658 : [CPUHP_AP_RCUTREE_ONLINE] = {
1659 : .name = "RCU/tree:online",
1660 : .startup.single = rcutree_online_cpu,
1661 : .teardown.single = rcutree_offline_cpu,
1662 : },
1663 : #endif
1664 : /*
1665 : * The dynamically registered state space is here
1666 : */
1667 :
1668 : #ifdef CONFIG_SMP
1669 : /* Last state is scheduler control setting the cpu active */
1670 : [CPUHP_AP_ACTIVE] = {
1671 : .name = "sched:active",
1672 : .startup.single = sched_cpu_activate,
1673 : .teardown.single = sched_cpu_deactivate,
1674 : },
1675 : #endif
1676 :
1677 : /* CPU is fully up and running. */
1678 : [CPUHP_ONLINE] = {
1679 : .name = "online",
1680 : .startup.single = NULL,
1681 : .teardown.single = NULL,
1682 : },
1683 : };
1684 :
1685 : /* Sanity check for callbacks */
1686 30 : static int cpuhp_cb_check(enum cpuhp_state state)
1687 : {
1688 30 : if (state <= CPUHP_OFFLINE || state >= CPUHP_ONLINE)
1689 0 : return -EINVAL;
1690 : return 0;
1691 : }
1692 :
1693 : /*
1694 : * Returns a free for dynamic slot assignment of the Online state. The states
1695 : * are protected by the cpuhp_slot_states mutex and an empty slot is identified
1696 : * by having no name assigned.
1697 : */
1698 9 : static int cpuhp_reserve_state(enum cpuhp_state state)
1699 : {
1700 9 : enum cpuhp_state i, end;
1701 9 : struct cpuhp_step *step;
1702 :
1703 9 : switch (state) {
1704 : case CPUHP_AP_ONLINE_DYN:
1705 : step = cpuhp_hp_states + CPUHP_AP_ONLINE_DYN;
1706 : end = CPUHP_AP_ONLINE_DYN_END;
1707 : break;
1708 1 : case CPUHP_BP_PREPARE_DYN:
1709 1 : step = cpuhp_hp_states + CPUHP_BP_PREPARE_DYN;
1710 1 : end = CPUHP_BP_PREPARE_DYN_END;
1711 1 : break;
1712 : default:
1713 : return -EINVAL;
1714 : }
1715 :
1716 37 : for (i = state; i <= end; i++, step++) {
1717 37 : if (!step->name)
1718 9 : return i;
1719 : }
1720 0 : WARN(1, "No more dynamic states available for CPU hotplug\n");
1721 0 : return -ENOSPC;
1722 : }
1723 :
1724 30 : static int cpuhp_store_callbacks(enum cpuhp_state state, const char *name,
1725 : int (*startup)(unsigned int cpu),
1726 : int (*teardown)(unsigned int cpu),
1727 : bool multi_instance)
1728 : {
1729 : /* (Un)Install the callbacks for further cpu hotplug operations */
1730 30 : struct cpuhp_step *sp;
1731 30 : int ret = 0;
1732 :
1733 : /*
1734 : * If name is NULL, then the state gets removed.
1735 : *
1736 : * CPUHP_AP_ONLINE_DYN and CPUHP_BP_PREPARE_DYN are handed out on
1737 : * the first allocation from these dynamic ranges, so the removal
1738 : * would trigger a new allocation and clear the wrong (already
1739 : * empty) state, leaving the callbacks of the to be cleared state
1740 : * dangling, which causes wreckage on the next hotplug operation.
1741 : */
1742 30 : if (name && (state == CPUHP_AP_ONLINE_DYN ||
1743 30 : state == CPUHP_BP_PREPARE_DYN)) {
1744 9 : ret = cpuhp_reserve_state(state);
1745 9 : if (ret < 0)
1746 : return ret;
1747 : state = ret;
1748 : }
1749 30 : sp = cpuhp_get_step(state);
1750 30 : if (name && sp->name)
1751 : return -EBUSY;
1752 :
1753 30 : sp->startup.single = startup;
1754 30 : sp->teardown.single = teardown;
1755 30 : sp->name = name;
1756 30 : sp->multi_instance = multi_instance;
1757 30 : INIT_HLIST_HEAD(&sp->list);
1758 30 : return ret;
1759 : }
1760 :
1761 0 : static void *cpuhp_get_teardown_cb(enum cpuhp_state state)
1762 : {
1763 0 : return cpuhp_get_step(state)->teardown.single;
1764 : }
1765 :
1766 : /*
1767 : * Call the startup/teardown function for a step either on the AP or
1768 : * on the current CPU.
1769 : */
1770 22 : static int cpuhp_issue_call(int cpu, enum cpuhp_state state, bool bringup,
1771 : struct hlist_node *node)
1772 : {
1773 22 : struct cpuhp_step *sp = cpuhp_get_step(state);
1774 22 : int ret;
1775 :
1776 : /*
1777 : * If there's nothing to do, we done.
1778 : * Relies on the union for multi_instance.
1779 : */
1780 22 : if ((bringup && !sp->startup.single) ||
1781 22 : (!bringup && !sp->teardown.single))
1782 : return 0;
1783 : /*
1784 : * The non AP bound callbacks can fail on bringup. On teardown
1785 : * e.g. module removal we crash for now.
1786 : */
1787 : #ifdef CONFIG_SMP
1788 22 : if (cpuhp_is_ap_state(state))
1789 15 : ret = cpuhp_invoke_ap_callback(cpu, state, bringup, node);
1790 : else
1791 7 : ret = cpuhp_invoke_callback(cpu, state, bringup, node, NULL);
1792 : #else
1793 : ret = cpuhp_invoke_callback(cpu, state, bringup, node, NULL);
1794 : #endif
1795 22 : BUG_ON(ret && !bringup);
1796 : return ret;
1797 : }
1798 :
1799 : /*
1800 : * Called from __cpuhp_setup_state on a recoverable failure.
1801 : *
1802 : * Note: The teardown callbacks for rollback are not allowed to fail!
1803 : */
1804 0 : static void cpuhp_rollback_install(int failedcpu, enum cpuhp_state state,
1805 : struct hlist_node *node)
1806 : {
1807 0 : int cpu;
1808 :
1809 : /* Roll back the already executed steps on the other cpus */
1810 0 : for_each_present_cpu(cpu) {
1811 0 : struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
1812 0 : int cpustate = st->state;
1813 :
1814 0 : if (cpu >= failedcpu)
1815 : break;
1816 :
1817 : /* Did we invoke the startup call on that cpu ? */
1818 0 : if (cpustate >= state)
1819 0 : cpuhp_issue_call(cpu, state, false, node);
1820 : }
1821 0 : }
1822 :
1823 14 : int __cpuhp_state_add_instance_cpuslocked(enum cpuhp_state state,
1824 : struct hlist_node *node,
1825 : bool invoke)
1826 : {
1827 14 : struct cpuhp_step *sp;
1828 14 : int cpu;
1829 14 : int ret;
1830 :
1831 14 : lockdep_assert_cpus_held();
1832 :
1833 14 : sp = cpuhp_get_step(state);
1834 14 : if (sp->multi_instance == false)
1835 : return -EINVAL;
1836 :
1837 14 : mutex_lock(&cpuhp_state_mutex);
1838 :
1839 14 : if (!invoke || !sp->startup.multi)
1840 12 : goto add_node;
1841 :
1842 : /*
1843 : * Try to call the startup callback for each present cpu
1844 : * depending on the hotplug state of the cpu.
1845 : */
1846 10 : for_each_present_cpu(cpu) {
1847 8 : struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
1848 8 : int cpustate = st->state;
1849 :
1850 8 : if (cpustate < state)
1851 6 : continue;
1852 :
1853 2 : ret = cpuhp_issue_call(cpu, state, true, node);
1854 2 : if (ret) {
1855 0 : if (sp->teardown.multi)
1856 0 : cpuhp_rollback_install(cpu, state, node);
1857 0 : goto unlock;
1858 : }
1859 : }
1860 2 : add_node:
1861 14 : ret = 0;
1862 14 : hlist_add_head(node, &sp->list);
1863 14 : unlock:
1864 14 : mutex_unlock(&cpuhp_state_mutex);
1865 14 : return ret;
1866 : }
1867 :
1868 14 : int __cpuhp_state_add_instance(enum cpuhp_state state, struct hlist_node *node,
1869 : bool invoke)
1870 : {
1871 14 : int ret;
1872 :
1873 14 : cpus_read_lock();
1874 14 : ret = __cpuhp_state_add_instance_cpuslocked(state, node, invoke);
1875 14 : cpus_read_unlock();
1876 14 : return ret;
1877 : }
1878 : EXPORT_SYMBOL_GPL(__cpuhp_state_add_instance);
1879 :
1880 : /**
1881 : * __cpuhp_setup_state_cpuslocked - Setup the callbacks for an hotplug machine state
1882 : * @state: The state to setup
1883 : * @invoke: If true, the startup function is invoked for cpus where
1884 : * cpu state >= @state
1885 : * @startup: startup callback function
1886 : * @teardown: teardown callback function
1887 : * @multi_instance: State is set up for multiple instances which get
1888 : * added afterwards.
1889 : *
1890 : * The caller needs to hold cpus read locked while calling this function.
1891 : * Returns:
1892 : * On success:
1893 : * Positive state number if @state is CPUHP_AP_ONLINE_DYN
1894 : * 0 for all other states
1895 : * On failure: proper (negative) error code
1896 : */
1897 30 : int __cpuhp_setup_state_cpuslocked(enum cpuhp_state state,
1898 : const char *name, bool invoke,
1899 : int (*startup)(unsigned int cpu),
1900 : int (*teardown)(unsigned int cpu),
1901 : bool multi_instance)
1902 : {
1903 30 : int cpu, ret = 0;
1904 30 : bool dynstate;
1905 :
1906 30 : lockdep_assert_cpus_held();
1907 :
1908 30 : if (cpuhp_cb_check(state) || !name)
1909 : return -EINVAL;
1910 :
1911 30 : mutex_lock(&cpuhp_state_mutex);
1912 :
1913 30 : ret = cpuhp_store_callbacks(state, name, startup, teardown,
1914 : multi_instance);
1915 :
1916 30 : dynstate = state == CPUHP_AP_ONLINE_DYN;
1917 30 : if (ret > 0 && dynstate) {
1918 8 : state = ret;
1919 8 : ret = 0;
1920 : }
1921 :
1922 30 : if (ret || !invoke || !startup)
1923 22 : goto out;
1924 :
1925 : /*
1926 : * Try to call the startup callback for each present cpu
1927 : * depending on the hotplug state of the cpu.
1928 : */
1929 40 : for_each_present_cpu(cpu) {
1930 32 : struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
1931 32 : int cpustate = st->state;
1932 :
1933 32 : if (cpustate < state)
1934 12 : continue;
1935 :
1936 20 : ret = cpuhp_issue_call(cpu, state, true, NULL);
1937 20 : if (ret) {
1938 0 : if (teardown)
1939 0 : cpuhp_rollback_install(cpu, state, NULL);
1940 0 : cpuhp_store_callbacks(state, NULL, NULL, NULL, false);
1941 0 : goto out;
1942 : }
1943 : }
1944 8 : out:
1945 30 : mutex_unlock(&cpuhp_state_mutex);
1946 : /*
1947 : * If the requested state is CPUHP_AP_ONLINE_DYN, return the
1948 : * dynamically allocated state in case of success.
1949 : */
1950 30 : if (!ret && dynstate)
1951 8 : return state;
1952 : return ret;
1953 : }
1954 : EXPORT_SYMBOL(__cpuhp_setup_state_cpuslocked);
1955 :
1956 30 : int __cpuhp_setup_state(enum cpuhp_state state,
1957 : const char *name, bool invoke,
1958 : int (*startup)(unsigned int cpu),
1959 : int (*teardown)(unsigned int cpu),
1960 : bool multi_instance)
1961 : {
1962 30 : int ret;
1963 :
1964 30 : cpus_read_lock();
1965 30 : ret = __cpuhp_setup_state_cpuslocked(state, name, invoke, startup,
1966 : teardown, multi_instance);
1967 30 : cpus_read_unlock();
1968 30 : return ret;
1969 : }
1970 : EXPORT_SYMBOL(__cpuhp_setup_state);
1971 :
1972 0 : int __cpuhp_state_remove_instance(enum cpuhp_state state,
1973 : struct hlist_node *node, bool invoke)
1974 : {
1975 0 : struct cpuhp_step *sp = cpuhp_get_step(state);
1976 0 : int cpu;
1977 :
1978 0 : BUG_ON(cpuhp_cb_check(state));
1979 :
1980 0 : if (!sp->multi_instance)
1981 : return -EINVAL;
1982 :
1983 0 : cpus_read_lock();
1984 0 : mutex_lock(&cpuhp_state_mutex);
1985 :
1986 0 : if (!invoke || !cpuhp_get_teardown_cb(state))
1987 0 : goto remove;
1988 : /*
1989 : * Call the teardown callback for each present cpu depending
1990 : * on the hotplug state of the cpu. This function is not
1991 : * allowed to fail currently!
1992 : */
1993 0 : for_each_present_cpu(cpu) {
1994 0 : struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
1995 0 : int cpustate = st->state;
1996 :
1997 0 : if (cpustate >= state)
1998 0 : cpuhp_issue_call(cpu, state, false, node);
1999 : }
2000 :
2001 0 : remove:
2002 0 : hlist_del(node);
2003 0 : mutex_unlock(&cpuhp_state_mutex);
2004 0 : cpus_read_unlock();
2005 :
2006 0 : return 0;
2007 : }
2008 : EXPORT_SYMBOL_GPL(__cpuhp_state_remove_instance);
2009 :
2010 : /**
2011 : * __cpuhp_remove_state_cpuslocked - Remove the callbacks for an hotplug machine state
2012 : * @state: The state to remove
2013 : * @invoke: If true, the teardown function is invoked for cpus where
2014 : * cpu state >= @state
2015 : *
2016 : * The caller needs to hold cpus read locked while calling this function.
2017 : * The teardown callback is currently not allowed to fail. Think
2018 : * about module removal!
2019 : */
2020 0 : void __cpuhp_remove_state_cpuslocked(enum cpuhp_state state, bool invoke)
2021 : {
2022 0 : struct cpuhp_step *sp = cpuhp_get_step(state);
2023 0 : int cpu;
2024 :
2025 0 : BUG_ON(cpuhp_cb_check(state));
2026 :
2027 0 : lockdep_assert_cpus_held();
2028 :
2029 0 : mutex_lock(&cpuhp_state_mutex);
2030 0 : if (sp->multi_instance) {
2031 0 : WARN(!hlist_empty(&sp->list),
2032 : "Error: Removing state %d which has instances left.\n",
2033 : state);
2034 0 : goto remove;
2035 : }
2036 :
2037 0 : if (!invoke || !cpuhp_get_teardown_cb(state))
2038 0 : goto remove;
2039 :
2040 : /*
2041 : * Call the teardown callback for each present cpu depending
2042 : * on the hotplug state of the cpu. This function is not
2043 : * allowed to fail currently!
2044 : */
2045 0 : for_each_present_cpu(cpu) {
2046 0 : struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
2047 0 : int cpustate = st->state;
2048 :
2049 0 : if (cpustate >= state)
2050 0 : cpuhp_issue_call(cpu, state, false, NULL);
2051 : }
2052 0 : remove:
2053 0 : cpuhp_store_callbacks(state, NULL, NULL, NULL, false);
2054 0 : mutex_unlock(&cpuhp_state_mutex);
2055 0 : }
2056 : EXPORT_SYMBOL(__cpuhp_remove_state_cpuslocked);
2057 :
2058 0 : void __cpuhp_remove_state(enum cpuhp_state state, bool invoke)
2059 : {
2060 0 : cpus_read_lock();
2061 0 : __cpuhp_remove_state_cpuslocked(state, invoke);
2062 0 : cpus_read_unlock();
2063 0 : }
2064 : EXPORT_SYMBOL(__cpuhp_remove_state);
2065 :
2066 : #ifdef CONFIG_HOTPLUG_SMT
2067 0 : static void cpuhp_offline_cpu_device(unsigned int cpu)
2068 : {
2069 0 : struct device *dev = get_cpu_device(cpu);
2070 :
2071 0 : dev->offline = true;
2072 : /* Tell user space about the state change */
2073 0 : kobject_uevent(&dev->kobj, KOBJ_OFFLINE);
2074 0 : }
2075 :
2076 0 : static void cpuhp_online_cpu_device(unsigned int cpu)
2077 : {
2078 0 : struct device *dev = get_cpu_device(cpu);
2079 :
2080 0 : dev->offline = false;
2081 : /* Tell user space about the state change */
2082 0 : kobject_uevent(&dev->kobj, KOBJ_ONLINE);
2083 0 : }
2084 :
2085 0 : int cpuhp_smt_disable(enum cpuhp_smt_control ctrlval)
2086 : {
2087 0 : int cpu, ret = 0;
2088 :
2089 0 : cpu_maps_update_begin();
2090 0 : for_each_online_cpu(cpu) {
2091 0 : if (topology_is_primary_thread(cpu))
2092 0 : continue;
2093 0 : ret = cpu_down_maps_locked(cpu, CPUHP_OFFLINE);
2094 0 : if (ret)
2095 : break;
2096 : /*
2097 : * As this needs to hold the cpu maps lock it's impossible
2098 : * to call device_offline() because that ends up calling
2099 : * cpu_down() which takes cpu maps lock. cpu maps lock
2100 : * needs to be held as this might race against in kernel
2101 : * abusers of the hotplug machinery (thermal management).
2102 : *
2103 : * So nothing would update device:offline state. That would
2104 : * leave the sysfs entry stale and prevent onlining after
2105 : * smt control has been changed to 'off' again. This is
2106 : * called under the sysfs hotplug lock, so it is properly
2107 : * serialized against the regular offline usage.
2108 : */
2109 0 : cpuhp_offline_cpu_device(cpu);
2110 : }
2111 0 : if (!ret)
2112 0 : cpu_smt_control = ctrlval;
2113 0 : cpu_maps_update_done();
2114 0 : return ret;
2115 : }
2116 :
2117 0 : int cpuhp_smt_enable(void)
2118 : {
2119 0 : int cpu, ret = 0;
2120 :
2121 0 : cpu_maps_update_begin();
2122 0 : cpu_smt_control = CPU_SMT_ENABLED;
2123 0 : for_each_present_cpu(cpu) {
2124 : /* Skip online CPUs and CPUs on offline nodes */
2125 0 : if (cpu_online(cpu) || !node_online(cpu_to_node(cpu)))
2126 0 : continue;
2127 0 : ret = _cpu_up(cpu, 0, CPUHP_ONLINE);
2128 0 : if (ret)
2129 : break;
2130 : /* See comment in cpuhp_smt_disable() */
2131 0 : cpuhp_online_cpu_device(cpu);
2132 : }
2133 0 : cpu_maps_update_done();
2134 0 : return ret;
2135 : }
2136 : #endif
2137 :
2138 : #if defined(CONFIG_SYSFS) && defined(CONFIG_HOTPLUG_CPU)
2139 0 : static ssize_t show_cpuhp_state(struct device *dev,
2140 : struct device_attribute *attr, char *buf)
2141 : {
2142 0 : struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id);
2143 :
2144 0 : return sprintf(buf, "%d\n", st->state);
2145 : }
2146 : static DEVICE_ATTR(state, 0444, show_cpuhp_state, NULL);
2147 :
2148 0 : static ssize_t write_cpuhp_target(struct device *dev,
2149 : struct device_attribute *attr,
2150 : const char *buf, size_t count)
2151 : {
2152 0 : struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id);
2153 0 : struct cpuhp_step *sp;
2154 0 : int target, ret;
2155 :
2156 0 : ret = kstrtoint(buf, 10, &target);
2157 0 : if (ret)
2158 0 : return ret;
2159 :
2160 : #ifdef CONFIG_CPU_HOTPLUG_STATE_CONTROL
2161 : if (target < CPUHP_OFFLINE || target > CPUHP_ONLINE)
2162 : return -EINVAL;
2163 : #else
2164 0 : if (target != CPUHP_OFFLINE && target != CPUHP_ONLINE)
2165 : return -EINVAL;
2166 : #endif
2167 :
2168 0 : ret = lock_device_hotplug_sysfs();
2169 0 : if (ret)
2170 0 : return ret;
2171 :
2172 0 : mutex_lock(&cpuhp_state_mutex);
2173 0 : sp = cpuhp_get_step(target);
2174 0 : ret = !sp->name || sp->cant_stop ? -EINVAL : 0;
2175 0 : mutex_unlock(&cpuhp_state_mutex);
2176 0 : if (ret)
2177 0 : goto out;
2178 :
2179 0 : if (st->state < target)
2180 0 : ret = cpu_up(dev->id, target);
2181 : else
2182 0 : ret = cpu_down(dev->id, target);
2183 0 : out:
2184 0 : unlock_device_hotplug();
2185 0 : return ret ? ret : count;
2186 : }
2187 :
2188 0 : static ssize_t show_cpuhp_target(struct device *dev,
2189 : struct device_attribute *attr, char *buf)
2190 : {
2191 0 : struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id);
2192 :
2193 0 : return sprintf(buf, "%d\n", st->target);
2194 : }
2195 : static DEVICE_ATTR(target, 0644, show_cpuhp_target, write_cpuhp_target);
2196 :
2197 :
2198 0 : static ssize_t write_cpuhp_fail(struct device *dev,
2199 : struct device_attribute *attr,
2200 : const char *buf, size_t count)
2201 : {
2202 0 : struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id);
2203 0 : struct cpuhp_step *sp;
2204 0 : int fail, ret;
2205 :
2206 0 : ret = kstrtoint(buf, 10, &fail);
2207 0 : if (ret)
2208 0 : return ret;
2209 :
2210 0 : if (fail < CPUHP_OFFLINE || fail > CPUHP_ONLINE)
2211 : return -EINVAL;
2212 :
2213 : /*
2214 : * Cannot fail STARTING/DYING callbacks.
2215 : */
2216 0 : if (cpuhp_is_atomic_state(fail))
2217 : return -EINVAL;
2218 :
2219 : /*
2220 : * Cannot fail anything that doesn't have callbacks.
2221 : */
2222 0 : mutex_lock(&cpuhp_state_mutex);
2223 0 : sp = cpuhp_get_step(fail);
2224 0 : if (!sp->startup.single && !sp->teardown.single)
2225 0 : ret = -EINVAL;
2226 0 : mutex_unlock(&cpuhp_state_mutex);
2227 0 : if (ret)
2228 0 : return ret;
2229 :
2230 0 : st->fail = fail;
2231 :
2232 0 : return count;
2233 : }
2234 :
2235 0 : static ssize_t show_cpuhp_fail(struct device *dev,
2236 : struct device_attribute *attr, char *buf)
2237 : {
2238 0 : struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id);
2239 :
2240 0 : return sprintf(buf, "%d\n", st->fail);
2241 : }
2242 :
2243 : static DEVICE_ATTR(fail, 0644, show_cpuhp_fail, write_cpuhp_fail);
2244 :
2245 : static struct attribute *cpuhp_cpu_attrs[] = {
2246 : &dev_attr_state.attr,
2247 : &dev_attr_target.attr,
2248 : &dev_attr_fail.attr,
2249 : NULL
2250 : };
2251 :
2252 : static const struct attribute_group cpuhp_cpu_attr_group = {
2253 : .attrs = cpuhp_cpu_attrs,
2254 : .name = "hotplug",
2255 : NULL
2256 : };
2257 :
2258 0 : static ssize_t show_cpuhp_states(struct device *dev,
2259 : struct device_attribute *attr, char *buf)
2260 : {
2261 0 : ssize_t cur, res = 0;
2262 0 : int i;
2263 :
2264 0 : mutex_lock(&cpuhp_state_mutex);
2265 0 : for (i = CPUHP_OFFLINE; i <= CPUHP_ONLINE; i++) {
2266 0 : struct cpuhp_step *sp = cpuhp_get_step(i);
2267 :
2268 0 : if (sp->name) {
2269 0 : cur = sprintf(buf, "%3d: %s\n", i, sp->name);
2270 0 : buf += cur;
2271 0 : res += cur;
2272 : }
2273 : }
2274 0 : mutex_unlock(&cpuhp_state_mutex);
2275 0 : return res;
2276 : }
2277 : static DEVICE_ATTR(states, 0444, show_cpuhp_states, NULL);
2278 :
2279 : static struct attribute *cpuhp_cpu_root_attrs[] = {
2280 : &dev_attr_states.attr,
2281 : NULL
2282 : };
2283 :
2284 : static const struct attribute_group cpuhp_cpu_root_attr_group = {
2285 : .attrs = cpuhp_cpu_root_attrs,
2286 : .name = "hotplug",
2287 : NULL
2288 : };
2289 :
2290 : #ifdef CONFIG_HOTPLUG_SMT
2291 :
2292 : static ssize_t
2293 0 : __store_smt_control(struct device *dev, struct device_attribute *attr,
2294 : const char *buf, size_t count)
2295 : {
2296 0 : int ctrlval, ret;
2297 :
2298 0 : if (sysfs_streq(buf, "on"))
2299 : ctrlval = CPU_SMT_ENABLED;
2300 0 : else if (sysfs_streq(buf, "off"))
2301 : ctrlval = CPU_SMT_DISABLED;
2302 0 : else if (sysfs_streq(buf, "forceoff"))
2303 : ctrlval = CPU_SMT_FORCE_DISABLED;
2304 : else
2305 : return -EINVAL;
2306 :
2307 0 : if (cpu_smt_control == CPU_SMT_FORCE_DISABLED)
2308 : return -EPERM;
2309 :
2310 0 : if (cpu_smt_control == CPU_SMT_NOT_SUPPORTED)
2311 : return -ENODEV;
2312 :
2313 0 : ret = lock_device_hotplug_sysfs();
2314 0 : if (ret)
2315 0 : return ret;
2316 :
2317 0 : if (ctrlval != cpu_smt_control) {
2318 0 : switch (ctrlval) {
2319 0 : case CPU_SMT_ENABLED:
2320 0 : ret = cpuhp_smt_enable();
2321 0 : break;
2322 0 : case CPU_SMT_DISABLED:
2323 : case CPU_SMT_FORCE_DISABLED:
2324 0 : ret = cpuhp_smt_disable(ctrlval);
2325 0 : break;
2326 : }
2327 0 : }
2328 :
2329 0 : unlock_device_hotplug();
2330 0 : return ret ? ret : count;
2331 : }
2332 :
2333 : #else /* !CONFIG_HOTPLUG_SMT */
2334 : static ssize_t
2335 : __store_smt_control(struct device *dev, struct device_attribute *attr,
2336 : const char *buf, size_t count)
2337 : {
2338 : return -ENODEV;
2339 : }
2340 : #endif /* CONFIG_HOTPLUG_SMT */
2341 :
2342 : static const char *smt_states[] = {
2343 : [CPU_SMT_ENABLED] = "on",
2344 : [CPU_SMT_DISABLED] = "off",
2345 : [CPU_SMT_FORCE_DISABLED] = "forceoff",
2346 : [CPU_SMT_NOT_SUPPORTED] = "notsupported",
2347 : [CPU_SMT_NOT_IMPLEMENTED] = "notimplemented",
2348 : };
2349 :
2350 : static ssize_t
2351 0 : show_smt_control(struct device *dev, struct device_attribute *attr, char *buf)
2352 : {
2353 0 : const char *state = smt_states[cpu_smt_control];
2354 :
2355 0 : return snprintf(buf, PAGE_SIZE - 2, "%s\n", state);
2356 : }
2357 :
2358 : static ssize_t
2359 0 : store_smt_control(struct device *dev, struct device_attribute *attr,
2360 : const char *buf, size_t count)
2361 : {
2362 0 : return __store_smt_control(dev, attr, buf, count);
2363 : }
2364 : static DEVICE_ATTR(control, 0644, show_smt_control, store_smt_control);
2365 :
2366 : static ssize_t
2367 0 : show_smt_active(struct device *dev, struct device_attribute *attr, char *buf)
2368 : {
2369 0 : return snprintf(buf, PAGE_SIZE - 2, "%d\n", sched_smt_active());
2370 : }
2371 : static DEVICE_ATTR(active, 0444, show_smt_active, NULL);
2372 :
2373 : static struct attribute *cpuhp_smt_attrs[] = {
2374 : &dev_attr_control.attr,
2375 : &dev_attr_active.attr,
2376 : NULL
2377 : };
2378 :
2379 : static const struct attribute_group cpuhp_smt_attr_group = {
2380 : .attrs = cpuhp_smt_attrs,
2381 : .name = "smt",
2382 : NULL
2383 : };
2384 :
2385 1 : static int __init cpu_smt_sysfs_init(void)
2386 : {
2387 1 : return sysfs_create_group(&cpu_subsys.dev_root->kobj,
2388 : &cpuhp_smt_attr_group);
2389 : }
2390 :
2391 1 : static int __init cpuhp_sysfs_init(void)
2392 : {
2393 1 : int cpu, ret;
2394 :
2395 1 : ret = cpu_smt_sysfs_init();
2396 1 : if (ret)
2397 : return ret;
2398 :
2399 1 : ret = sysfs_create_group(&cpu_subsys.dev_root->kobj,
2400 : &cpuhp_cpu_root_attr_group);
2401 1 : if (ret)
2402 : return ret;
2403 :
2404 5 : for_each_possible_cpu(cpu) {
2405 4 : struct device *dev = get_cpu_device(cpu);
2406 :
2407 4 : if (!dev)
2408 0 : continue;
2409 4 : ret = sysfs_create_group(&dev->kobj, &cpuhp_cpu_attr_group);
2410 4 : if (ret)
2411 0 : return ret;
2412 : }
2413 : return 0;
2414 : }
2415 : device_initcall(cpuhp_sysfs_init);
2416 : #endif /* CONFIG_SYSFS && CONFIG_HOTPLUG_CPU */
2417 :
2418 : /*
2419 : * cpu_bit_bitmap[] is a special, "compressed" data structure that
2420 : * represents all NR_CPUS bits binary values of 1<<nr.
2421 : *
2422 : * It is used by cpumask_of() to get a constant address to a CPU
2423 : * mask value that has a single bit set only.
2424 : */
2425 :
2426 : /* cpu_bit_bitmap[0] is empty - so we can back into it */
2427 : #define MASK_DECLARE_1(x) [x+1][0] = (1UL << (x))
2428 : #define MASK_DECLARE_2(x) MASK_DECLARE_1(x), MASK_DECLARE_1(x+1)
2429 : #define MASK_DECLARE_4(x) MASK_DECLARE_2(x), MASK_DECLARE_2(x+2)
2430 : #define MASK_DECLARE_8(x) MASK_DECLARE_4(x), MASK_DECLARE_4(x+4)
2431 :
2432 : const unsigned long cpu_bit_bitmap[BITS_PER_LONG+1][BITS_TO_LONGS(NR_CPUS)] = {
2433 :
2434 : MASK_DECLARE_8(0), MASK_DECLARE_8(8),
2435 : MASK_DECLARE_8(16), MASK_DECLARE_8(24),
2436 : #if BITS_PER_LONG > 32
2437 : MASK_DECLARE_8(32), MASK_DECLARE_8(40),
2438 : MASK_DECLARE_8(48), MASK_DECLARE_8(56),
2439 : #endif
2440 : };
2441 : EXPORT_SYMBOL_GPL(cpu_bit_bitmap);
2442 :
2443 : const DECLARE_BITMAP(cpu_all_bits, NR_CPUS) = CPU_BITS_ALL;
2444 : EXPORT_SYMBOL(cpu_all_bits);
2445 :
2446 : #ifdef CONFIG_INIT_ALL_POSSIBLE
2447 : struct cpumask __cpu_possible_mask __read_mostly
2448 : = {CPU_BITS_ALL};
2449 : #else
2450 : struct cpumask __cpu_possible_mask __read_mostly;
2451 : #endif
2452 : EXPORT_SYMBOL(__cpu_possible_mask);
2453 :
2454 : struct cpumask __cpu_online_mask __read_mostly;
2455 : EXPORT_SYMBOL(__cpu_online_mask);
2456 :
2457 : struct cpumask __cpu_present_mask __read_mostly;
2458 : EXPORT_SYMBOL(__cpu_present_mask);
2459 :
2460 : struct cpumask __cpu_active_mask __read_mostly;
2461 : EXPORT_SYMBOL(__cpu_active_mask);
2462 :
2463 : atomic_t __num_online_cpus __read_mostly;
2464 : EXPORT_SYMBOL(__num_online_cpus);
2465 :
2466 0 : void init_cpu_present(const struct cpumask *src)
2467 : {
2468 0 : cpumask_copy(&__cpu_present_mask, src);
2469 0 : }
2470 :
2471 0 : void init_cpu_possible(const struct cpumask *src)
2472 : {
2473 0 : cpumask_copy(&__cpu_possible_mask, src);
2474 0 : }
2475 :
2476 0 : void init_cpu_online(const struct cpumask *src)
2477 : {
2478 0 : cpumask_copy(&__cpu_online_mask, src);
2479 0 : }
2480 :
2481 4 : void set_cpu_online(unsigned int cpu, bool online)
2482 : {
2483 : /*
2484 : * atomic_inc/dec() is required to handle the horrid abuse of this
2485 : * function by the reboot and kexec code which invoke it from
2486 : * IPI/NMI broadcasts when shutting down CPUs. Invocation from
2487 : * regular CPU hotplug is properly serialized.
2488 : *
2489 : * Note, that the fact that __num_online_cpus is of type atomic_t
2490 : * does not protect readers which are not serialized against
2491 : * concurrent hotplug operations.
2492 : */
2493 4 : if (online) {
2494 4 : if (!cpumask_test_and_set_cpu(cpu, &__cpu_online_mask))
2495 4 : atomic_inc(&__num_online_cpus);
2496 : } else {
2497 0 : if (cpumask_test_and_clear_cpu(cpu, &__cpu_online_mask))
2498 0 : atomic_dec(&__num_online_cpus);
2499 : }
2500 4 : }
2501 :
2502 : /*
2503 : * Activate the first processor.
2504 : */
2505 1 : void __init boot_cpu_init(void)
2506 : {
2507 1 : int cpu = smp_processor_id();
2508 :
2509 : /* Mark the boot cpu "present", "online" etc for SMP and UP case */
2510 1 : set_cpu_online(cpu, true);
2511 1 : set_cpu_active(cpu, true);
2512 1 : set_cpu_present(cpu, true);
2513 1 : set_cpu_possible(cpu, true);
2514 :
2515 : #ifdef CONFIG_SMP
2516 1 : __boot_cpu_id = cpu;
2517 : #endif
2518 1 : }
2519 :
2520 : /*
2521 : * Must be called _AFTER_ setting up the per_cpu areas
2522 : */
2523 1 : void __init boot_cpu_hotplug_init(void)
2524 : {
2525 : #ifdef CONFIG_SMP
2526 1 : cpumask_set_cpu(smp_processor_id(), &cpus_booted_once_mask);
2527 : #endif
2528 1 : this_cpu_write(cpuhp_state.state, CPUHP_ONLINE);
2529 1 : }
2530 :
2531 : /*
2532 : * These are used for a global "mitigations=" cmdline option for toggling
2533 : * optional CPU mitigations.
2534 : */
2535 : enum cpu_mitigations {
2536 : CPU_MITIGATIONS_OFF,
2537 : CPU_MITIGATIONS_AUTO,
2538 : CPU_MITIGATIONS_AUTO_NOSMT,
2539 : };
2540 :
2541 : static enum cpu_mitigations cpu_mitigations __ro_after_init =
2542 : CPU_MITIGATIONS_AUTO;
2543 :
2544 0 : static int __init mitigations_parse_cmdline(char *arg)
2545 : {
2546 0 : if (!strcmp(arg, "off"))
2547 0 : cpu_mitigations = CPU_MITIGATIONS_OFF;
2548 0 : else if (!strcmp(arg, "auto"))
2549 0 : cpu_mitigations = CPU_MITIGATIONS_AUTO;
2550 0 : else if (!strcmp(arg, "auto,nosmt"))
2551 0 : cpu_mitigations = CPU_MITIGATIONS_AUTO_NOSMT;
2552 : else
2553 0 : pr_crit("Unsupported mitigations=%s, system may still be vulnerable\n",
2554 : arg);
2555 :
2556 0 : return 0;
2557 : }
2558 : early_param("mitigations", mitigations_parse_cmdline);
2559 :
2560 : /* mitigations=off */
2561 6 : bool cpu_mitigations_off(void)
2562 : {
2563 6 : return cpu_mitigations == CPU_MITIGATIONS_OFF;
2564 : }
2565 : EXPORT_SYMBOL_GPL(cpu_mitigations_off);
2566 :
2567 : /* mitigations=auto,nosmt */
2568 2 : bool cpu_mitigations_auto_nosmt(void)
2569 : {
2570 2 : return cpu_mitigations == CPU_MITIGATIONS_AUTO_NOSMT;
2571 : }
2572 : EXPORT_SYMBOL_GPL(cpu_mitigations_auto_nosmt);
|
