LCOV - code coverage report
Current view: top level - kernel - cpu.c (source / functions) Hit Total Coverage
Test: landlock.info Lines: 363 847 42.9 %
Date: 2021-04-22 12:43:58 Functions: 37 92 40.2 %

          Line data    Source code
       1             : /* CPU control.
       2             :  * (C) 2001, 2002, 2003, 2004 Rusty Russell
       3             :  *
       4             :  * This code is licenced under the GPL.
       5             :  */
       6             : #include <linux/sched/mm.h>
       7             : #include <linux/proc_fs.h>
       8             : #include <linux/smp.h>
       9             : #include <linux/init.h>
      10             : #include <linux/notifier.h>
      11             : #include <linux/sched/signal.h>
      12             : #include <linux/sched/hotplug.h>
      13             : #include <linux/sched/isolation.h>
      14             : #include <linux/sched/task.h>
      15             : #include <linux/sched/smt.h>
      16             : #include <linux/unistd.h>
      17             : #include <linux/cpu.h>
      18             : #include <linux/oom.h>
      19             : #include <linux/rcupdate.h>
      20             : #include <linux/export.h>
      21             : #include <linux/bug.h>
      22             : #include <linux/kthread.h>
      23             : #include <linux/stop_machine.h>
      24             : #include <linux/mutex.h>
      25             : #include <linux/gfp.h>
      26             : #include <linux/suspend.h>
      27             : #include <linux/lockdep.h>
      28             : #include <linux/tick.h>
      29             : #include <linux/irq.h>
      30             : #include <linux/nmi.h>
      31             : #include <linux/smpboot.h>
      32             : #include <linux/relay.h>
      33             : #include <linux/slab.h>
      34             : #include <linux/percpu-rwsem.h>
      35             : 
      36             : #include <trace/events/power.h>
      37             : #define CREATE_TRACE_POINTS
      38             : #include <trace/events/cpuhp.h>
      39             : 
      40             : #include "smpboot.h"
      41             : 
      42             : /**
      43             :  * cpuhp_cpu_state - Per cpu hotplug state storage
      44             :  * @state:      The current cpu state
      45             :  * @target:     The target state
      46             :  * @thread:     Pointer to the hotplug thread
      47             :  * @should_run: Thread should execute
      48             :  * @rollback:   Perform a rollback
      49             :  * @single:     Single callback invocation
      50             :  * @bringup:    Single callback bringup or teardown selector
      51             :  * @cb_state:   The state for a single callback (install/uninstall)
      52             :  * @result:     Result of the operation
      53             :  * @done_up:    Signal completion to the issuer of the task for cpu-up
      54             :  * @done_down:  Signal completion to the issuer of the task for cpu-down
      55             :  */
      56             : struct cpuhp_cpu_state {
      57             :         enum cpuhp_state        state;
      58             :         enum cpuhp_state        target;
      59             :         enum cpuhp_state        fail;
      60             : #ifdef CONFIG_SMP
      61             :         struct task_struct      *thread;
      62             :         bool                    should_run;
      63             :         bool                    rollback;
      64             :         bool                    single;
      65             :         bool                    bringup;
      66             :         struct hlist_node       *node;
      67             :         struct hlist_node       *last;
      68             :         enum cpuhp_state        cb_state;
      69             :         int                     result;
      70             :         struct completion       done_up;
      71             :         struct completion       done_down;
      72             : #endif
      73             : };
      74             : 
      75             : static DEFINE_PER_CPU(struct cpuhp_cpu_state, cpuhp_state) = {
      76             :         .fail = CPUHP_INVALID,
      77             : };
      78             : 
      79             : #ifdef CONFIG_SMP
      80             : cpumask_t cpus_booted_once_mask;
      81             : #endif
      82             : 
      83             : #if defined(CONFIG_LOCKDEP) && defined(CONFIG_SMP)
      84             : static struct lockdep_map cpuhp_state_up_map =
      85             :         STATIC_LOCKDEP_MAP_INIT("cpuhp_state-up", &cpuhp_state_up_map);
      86             : static struct lockdep_map cpuhp_state_down_map =
      87             :         STATIC_LOCKDEP_MAP_INIT("cpuhp_state-down", &cpuhp_state_down_map);
      88             : 
      89             : 
      90         267 : static inline void cpuhp_lock_acquire(bool bringup)
      91             : {
      92         282 :         lock_map_acquire(bringup ? &cpuhp_state_up_map : &cpuhp_state_down_map);
      93         267 : }
      94             : 
      95         267 : static inline void cpuhp_lock_release(bool bringup)
      96             : {
      97         267 :         lock_map_release(bringup ? &cpuhp_state_up_map : &cpuhp_state_down_map);
      98         237 : }
      99             : #else
     100             : 
     101             : static inline void cpuhp_lock_acquire(bool bringup) { }
     102             : static inline void cpuhp_lock_release(bool bringup) { }
     103             : 
     104             : #endif
     105             : 
     106             : /**
     107             :  * cpuhp_step - Hotplug state machine step
     108             :  * @name:       Name of the step
     109             :  * @startup:    Startup function of the step
     110             :  * @teardown:   Teardown function of the step
     111             :  * @cant_stop:  Bringup/teardown can't be stopped at this step
     112             :  */
     113             : struct cpuhp_step {
     114             :         const char              *name;
     115             :         union {
     116             :                 int             (*single)(unsigned int cpu);
     117             :                 int             (*multi)(unsigned int cpu,
     118             :                                          struct hlist_node *node);
     119             :         } startup;
     120             :         union {
     121             :                 int             (*single)(unsigned int cpu);
     122             :                 int             (*multi)(unsigned int cpu,
     123             :                                          struct hlist_node *node);
     124             :         } teardown;
     125             :         struct hlist_head       list;
     126             :         bool                    cant_stop;
     127             :         bool                    multi_instance;
     128             : };
     129             : 
     130             : static DEFINE_MUTEX(cpuhp_state_mutex);
     131             : static struct cpuhp_step cpuhp_hp_states[];
     132             : 
     133         742 : static struct cpuhp_step *cpuhp_get_step(enum cpuhp_state state)
     134             : {
     135         742 :         return cpuhp_hp_states + state;
     136             : }
     137             : 
     138             : /**
     139             :  * cpuhp_invoke_callback _ Invoke the callbacks for a given state
     140             :  * @cpu:        The cpu for which the callback should be invoked
     141             :  * @state:      The state to do callbacks for
     142             :  * @bringup:    True if the bringup callback should be invoked
     143             :  * @node:       For multi-instance, do a single entry callback for install/remove
     144             :  * @lastp:      For multi-instance rollback, remember how far we got
     145             :  *
     146             :  * Called from cpu hotplug and from the state register machinery.
     147             :  */
     148         676 : static int cpuhp_invoke_callback(unsigned int cpu, enum cpuhp_state state,
     149             :                                  bool bringup, struct hlist_node *node,
     150             :                                  struct hlist_node **lastp)
     151             : {
     152         676 :         struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
     153         676 :         struct cpuhp_step *step = cpuhp_get_step(state);
     154         676 :         int (*cbm)(unsigned int cpu, struct hlist_node *node);
     155         676 :         int (*cb)(unsigned int cpu);
     156         676 :         int ret, cnt;
     157             : 
     158         676 :         if (st->fail == state) {
     159           0 :                 st->fail = CPUHP_INVALID;
     160             : 
     161           0 :                 if (!(bringup ? step->startup.single : step->teardown.single))
     162             :                         return 0;
     163             : 
     164           0 :                 return -EAGAIN;
     165             :         }
     166             : 
     167         676 :         if (!step->multi_instance) {
     168        1342 :                 WARN_ON_ONCE(lastp && *lastp);
     169         671 :                 cb = bringup ? step->startup.single : step->teardown.single;
     170         671 :                 if (!cb)
     171             :                         return 0;
     172          86 :                 trace_cpuhp_enter(cpu, st->target, state, cb);
     173          86 :                 ret = cb(cpu);
     174          86 :                 trace_cpuhp_exit(cpu, st->state, state, ret);
     175          86 :                 return ret;
     176             :         }
     177           5 :         cbm = bringup ? step->startup.multi : step->teardown.multi;
     178           5 :         if (!cbm)
     179             :                 return 0;
     180             : 
     181             :         /* Single invocation for instance add/remove */
     182           5 :         if (node) {
     183           4 :                 WARN_ON_ONCE(lastp && *lastp);
     184           2 :                 trace_cpuhp_multi_enter(cpu, st->target, state, cbm, node);
     185           2 :                 ret = cbm(cpu, node);
     186           2 :                 trace_cpuhp_exit(cpu, st->state, state, ret);
     187           2 :                 return ret;
     188             :         }
     189             : 
     190             :         /* State transition. Invoke on all instances */
     191           3 :         cnt = 0;
     192           9 :         hlist_for_each(node, &step->list) {
     193           6 :                 if (lastp && node == *lastp)
     194             :                         break;
     195             : 
     196           6 :                 trace_cpuhp_multi_enter(cpu, st->target, state, cbm, node);
     197           6 :                 ret = cbm(cpu, node);
     198           6 :                 trace_cpuhp_exit(cpu, st->state, state, ret);
     199           6 :                 if (ret) {
     200           0 :                         if (!lastp)
     201           0 :                                 goto err;
     202             : 
     203           0 :                         *lastp = node;
     204           0 :                         return ret;
     205             :                 }
     206           6 :                 cnt++;
     207             :         }
     208           3 :         if (lastp)
     209           0 :                 *lastp = NULL;
     210             :         return 0;
     211           0 : err:
     212             :         /* Rollback the instances if one failed */
     213           0 :         cbm = !bringup ? step->startup.multi : step->teardown.multi;
     214           0 :         if (!cbm)
     215             :                 return ret;
     216             : 
     217           0 :         hlist_for_each(node, &step->list) {
     218           0 :                 if (!cnt--)
     219             :                         break;
     220             : 
     221           0 :                 trace_cpuhp_multi_enter(cpu, st->target, state, cbm, node);
     222           0 :                 ret = cbm(cpu, node);
     223           0 :                 trace_cpuhp_exit(cpu, st->state, state, ret);
     224             :                 /*
     225             :                  * Rollback must not fail,
     226             :                  */
     227           0 :                 WARN_ON_ONCE(ret);
     228             :         }
     229             :         return ret;
     230             : }
     231             : 
     232             : #ifdef CONFIG_SMP
     233         259 : static bool cpuhp_is_ap_state(enum cpuhp_state state)
     234             : {
     235             :         /*
     236             :          * The extra check for CPUHP_TEARDOWN_CPU is only for documentation
     237             :          * purposes as that state is handled explicitly in cpu_down.
     238             :          */
     239         259 :         return state > CPUHP_BRINGUP_CPU && state != CPUHP_TEARDOWN_CPU;
     240             : }
     241             : 
     242          18 : static inline void wait_for_ap_thread(struct cpuhp_cpu_state *st, bool bringup)
     243             : {
     244          15 :         struct completion *done = bringup ? &st->done_up : &st->done_down;
     245          15 :         wait_for_completion(done);
     246          15 : }
     247             : 
     248          18 : static inline void complete_ap_thread(struct cpuhp_cpu_state *st, bool bringup)
     249             : {
     250          15 :         struct completion *done = bringup ? &st->done_up : &st->done_down;
     251          15 :         complete(done);
     252          18 : }
     253             : 
     254             : /*
     255             :  * The former STARTING/DYING states, ran with IRQs disabled and must not fail.
     256             :  */
     257         237 : static bool cpuhp_is_atomic_state(enum cpuhp_state state)
     258             : {
     259         237 :         return CPUHP_AP_IDLE_DEAD <= state && state < CPUHP_AP_ONLINE;
     260             : }
     261             : 
     262             : /* Serializes the updates to cpu_online_mask, cpu_present_mask */
     263             : static DEFINE_MUTEX(cpu_add_remove_lock);
     264             : bool cpuhp_tasks_frozen;
     265             : EXPORT_SYMBOL_GPL(cpuhp_tasks_frozen);
     266             : 
     267             : /*
     268             :  * The following two APIs (cpu_maps_update_begin/done) must be used when
     269             :  * attempting to serialize the updates to cpu_online_mask & cpu_present_mask.
     270             :  */
     271           3 : void cpu_maps_update_begin(void)
     272             : {
     273           0 :         mutex_lock(&cpu_add_remove_lock);
     274           0 : }
     275             : 
     276           3 : void cpu_maps_update_done(void)
     277             : {
     278           0 :         mutex_unlock(&cpu_add_remove_lock);
     279           0 : }
     280             : 
     281             : /*
     282             :  * If set, cpu_up and cpu_down will return -EBUSY and do nothing.
     283             :  * Should always be manipulated under cpu_add_remove_lock
     284             :  */
     285             : static int cpu_hotplug_disabled;
     286             : 
     287             : #ifdef CONFIG_HOTPLUG_CPU
     288             : 
     289             : DEFINE_STATIC_PERCPU_RWSEM(cpu_hotplug_lock);
     290             : 
     291         106 : void cpus_read_lock(void)
     292             : {
     293          62 :         percpu_down_read(&cpu_hotplug_lock);
     294          62 : }
     295             : EXPORT_SYMBOL_GPL(cpus_read_lock);
     296             : 
     297           0 : int cpus_read_trylock(void)
     298             : {
     299           0 :         return percpu_down_read_trylock(&cpu_hotplug_lock);
     300             : }
     301             : EXPORT_SYMBOL_GPL(cpus_read_trylock);
     302             : 
     303         106 : void cpus_read_unlock(void)
     304             : {
     305          62 :         percpu_up_read(&cpu_hotplug_lock);
     306          62 : }
     307             : EXPORT_SYMBOL_GPL(cpus_read_unlock);
     308             : 
     309           3 : void cpus_write_lock(void)
     310             : {
     311           0 :         percpu_down_write(&cpu_hotplug_lock);
     312           0 : }
     313             : 
     314           3 : void cpus_write_unlock(void)
     315             : {
     316           0 :         percpu_up_write(&cpu_hotplug_lock);
     317           0 : }
     318             : 
     319          50 : void lockdep_assert_cpus_held(void)
     320             : {
     321             :         /*
     322             :          * We can't have hotplug operations before userspace starts running,
     323             :          * and some init codepaths will knowingly not take the hotplug lock.
     324             :          * This is all valid, so mute lockdep until it makes sense to report
     325             :          * unheld locks.
     326             :          */
     327          50 :         if (system_state < SYSTEM_RUNNING)
     328             :                 return;
     329             : 
     330           0 :         percpu_rwsem_assert_held(&cpu_hotplug_lock);
     331             : }
     332             : 
     333             : #ifdef CONFIG_LOCKDEP
     334           0 : int lockdep_is_cpus_held(void)
     335             : {
     336           0 :         return percpu_rwsem_is_held(&cpu_hotplug_lock);
     337             : }
     338             : #endif
     339             : 
     340         237 : static void lockdep_acquire_cpus_lock(void)
     341             : {
     342         237 :         rwsem_acquire(&cpu_hotplug_lock.dep_map, 0, 0, _THIS_IP_);
     343         237 : }
     344             : 
     345         237 : static void lockdep_release_cpus_lock(void)
     346             : {
     347         237 :         rwsem_release(&cpu_hotplug_lock.dep_map, _THIS_IP_);
     348             : }
     349             : 
     350             : /*
     351             :  * Wait for currently running CPU hotplug operations to complete (if any) and
     352             :  * disable future CPU hotplug (from sysfs). The 'cpu_add_remove_lock' protects
     353             :  * the 'cpu_hotplug_disabled' flag. The same lock is also acquired by the
     354             :  * hotplug path before performing hotplug operations. So acquiring that lock
     355             :  * guarantees mutual exclusion from any currently running hotplug operations.
     356             :  */
     357           0 : void cpu_hotplug_disable(void)
     358             : {
     359           0 :         cpu_maps_update_begin();
     360           0 :         cpu_hotplug_disabled++;
     361           0 :         cpu_maps_update_done();
     362           0 : }
     363             : EXPORT_SYMBOL_GPL(cpu_hotplug_disable);
     364             : 
     365           0 : static void __cpu_hotplug_enable(void)
     366             : {
     367           0 :         if (WARN_ONCE(!cpu_hotplug_disabled, "Unbalanced cpu hotplug enable\n"))
     368             :                 return;
     369           0 :         cpu_hotplug_disabled--;
     370             : }
     371             : 
     372           0 : void cpu_hotplug_enable(void)
     373             : {
     374           0 :         cpu_maps_update_begin();
     375           0 :         __cpu_hotplug_enable();
     376           0 :         cpu_maps_update_done();
     377           0 : }
     378             : EXPORT_SYMBOL_GPL(cpu_hotplug_enable);
     379             : 
     380             : #else
     381             : 
     382             : static void lockdep_acquire_cpus_lock(void)
     383             : {
     384             : }
     385             : 
     386             : static void lockdep_release_cpus_lock(void)
     387             : {
     388             : }
     389             : 
     390             : #endif  /* CONFIG_HOTPLUG_CPU */
     391             : 
     392             : /*
     393             :  * Architectures that need SMT-specific errata handling during SMT hotplug
     394             :  * should override this.
     395             :  */
     396           0 : void __weak arch_smt_update(void) { }
     397             : 
     398             : #ifdef CONFIG_HOTPLUG_SMT
     399             : enum cpuhp_smt_control cpu_smt_control __read_mostly = CPU_SMT_ENABLED;
     400             : 
     401           0 : void __init cpu_smt_disable(bool force)
     402             : {
     403           0 :         if (!cpu_smt_possible())
     404             :                 return;
     405             : 
     406           0 :         if (force) {
     407           0 :                 pr_info("SMT: Force disabled\n");
     408           0 :                 cpu_smt_control = CPU_SMT_FORCE_DISABLED;
     409             :         } else {
     410           0 :                 pr_info("SMT: disabled\n");
     411           0 :                 cpu_smt_control = CPU_SMT_DISABLED;
     412             :         }
     413             : }
     414             : 
     415             : /*
     416             :  * The decision whether SMT is supported can only be done after the full
     417             :  * CPU identification. Called from architecture code.
     418             :  */
     419           1 : void __init cpu_smt_check_topology(void)
     420             : {
     421           1 :         if (!topology_smt_supported())
     422           1 :                 cpu_smt_control = CPU_SMT_NOT_SUPPORTED;
     423           1 : }
     424             : 
     425           0 : static int __init smt_cmdline_disable(char *str)
     426             : {
     427           0 :         cpu_smt_disable(str && !strcmp(str, "force"));
     428           0 :         return 0;
     429             : }
     430             : early_param("nosmt", smt_cmdline_disable);
     431             : 
     432           6 : static inline bool cpu_smt_allowed(unsigned int cpu)
     433             : {
     434           6 :         if (cpu_smt_control == CPU_SMT_ENABLED)
     435             :                 return true;
     436             : 
     437           6 :         if (topology_is_primary_thread(cpu))
     438             :                 return true;
     439             : 
     440             :         /*
     441             :          * On x86 it's required to boot all logical CPUs at least once so
     442             :          * that the init code can get a chance to set CR4.MCE on each
     443             :          * CPU. Otherwise, a broadcasted MCE observing CR4.MCE=0b on any
     444             :          * core will shutdown the machine.
     445             :          */
     446           0 :         return !cpumask_test_cpu(cpu, &cpus_booted_once_mask);
     447             : }
     448             : 
     449             : /* Returns true if SMT is not supported of forcefully (irreversibly) disabled */
     450           0 : bool cpu_smt_possible(void)
     451             : {
     452           0 :         return cpu_smt_control != CPU_SMT_FORCE_DISABLED &&
     453             :                 cpu_smt_control != CPU_SMT_NOT_SUPPORTED;
     454             : }
     455             : EXPORT_SYMBOL_GPL(cpu_smt_possible);
     456             : #else
     457             : static inline bool cpu_smt_allowed(unsigned int cpu) { return true; }
     458             : #endif
     459             : 
     460             : static inline enum cpuhp_state
     461           6 : cpuhp_set_state(struct cpuhp_cpu_state *st, enum cpuhp_state target)
     462             : {
     463           6 :         enum cpuhp_state prev_state = st->state;
     464             : 
     465           6 :         st->rollback = false;
     466           6 :         st->last = NULL;
     467             : 
     468           6 :         st->target = target;
     469           6 :         st->single = false;
     470           6 :         st->bringup = st->state < target;
     471             : 
     472           6 :         return prev_state;
     473             : }
     474             : 
     475             : static inline void
     476           0 : cpuhp_reset_state(struct cpuhp_cpu_state *st, enum cpuhp_state prev_state)
     477             : {
     478           0 :         st->rollback = true;
     479             : 
     480             :         /*
     481             :          * If we have st->last we need to undo partial multi_instance of this
     482             :          * state first. Otherwise start undo at the previous state.
     483             :          */
     484           0 :         if (!st->last) {
     485           0 :                 if (st->bringup)
     486           0 :                         st->state--;
     487             :                 else
     488           0 :                         st->state++;
     489             :         }
     490             : 
     491           0 :         st->target = prev_state;
     492           0 :         st->bringup = !st->bringup;
     493           0 : }
     494             : 
     495             : /* Regular hotplug invocation of the AP hotplug thread */
     496          15 : static void __cpuhp_kick_ap(struct cpuhp_cpu_state *st)
     497             : {
     498          15 :         if (!st->single && st->state == st->target)
     499             :                 return;
     500             : 
     501          15 :         st->result = 0;
     502             :         /*
     503             :          * Make sure the above stores are visible before should_run becomes
     504             :          * true. Paired with the mb() above in cpuhp_thread_fun()
     505             :          */
     506          15 :         smp_mb();
     507          15 :         st->should_run = true;
     508          15 :         wake_up_process(st->thread);
     509          15 :         wait_for_ap_thread(st, st->bringup);
     510             : }
     511             : 
     512           3 : static int cpuhp_kick_ap(struct cpuhp_cpu_state *st, enum cpuhp_state target)
     513             : {
     514           3 :         enum cpuhp_state prev_state;
     515           3 :         int ret;
     516             : 
     517           3 :         prev_state = cpuhp_set_state(st, target);
     518           3 :         __cpuhp_kick_ap(st);
     519           3 :         if ((ret = st->result)) {
     520           0 :                 cpuhp_reset_state(st, prev_state);
     521           0 :                 __cpuhp_kick_ap(st);
     522             :         }
     523             : 
     524           3 :         return ret;
     525             : }
     526             : 
     527           3 : static int bringup_wait_for_ap(unsigned int cpu)
     528             : {
     529           3 :         struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
     530             : 
     531             :         /* Wait for the CPU to reach CPUHP_AP_ONLINE_IDLE */
     532           3 :         wait_for_ap_thread(st, true);
     533           3 :         if (WARN_ON_ONCE((!cpu_online(cpu))))
     534             :                 return -ECANCELED;
     535             : 
     536             :         /* Unpark the hotplug thread of the target cpu */
     537           3 :         kthread_unpark(st->thread);
     538             : 
     539             :         /*
     540             :          * SMT soft disabling on X86 requires to bring the CPU out of the
     541             :          * BIOS 'wait for SIPI' state in order to set the CR4.MCE bit.  The
     542             :          * CPU marked itself as booted_once in notify_cpu_starting() so the
     543             :          * cpu_smt_allowed() check will now return false if this is not the
     544             :          * primary sibling.
     545             :          */
     546           3 :         if (!cpu_smt_allowed(cpu))
     547             :                 return -ECANCELED;
     548             : 
     549           3 :         if (st->target <= CPUHP_AP_ONLINE_IDLE)
     550             :                 return 0;
     551             : 
     552           3 :         return cpuhp_kick_ap(st, st->target);
     553             : }
     554             : 
     555           3 : static int bringup_cpu(unsigned int cpu)
     556             : {
     557           3 :         struct task_struct *idle = idle_thread_get(cpu);
     558           3 :         int ret;
     559             : 
     560             :         /*
     561             :          * Some architectures have to walk the irq descriptors to
     562             :          * setup the vector space for the cpu which comes online.
     563             :          * Prevent irq alloc/free across the bringup.
     564             :          */
     565           3 :         irq_lock_sparse();
     566             : 
     567             :         /* Arch-specific enabling code. */
     568           3 :         ret = __cpu_up(cpu, idle);
     569           3 :         irq_unlock_sparse();
     570           3 :         if (ret)
     571             :                 return ret;
     572           3 :         return bringup_wait_for_ap(cpu);
     573             : }
     574             : 
     575           0 : static int finish_cpu(unsigned int cpu)
     576             : {
     577           0 :         struct task_struct *idle = idle_thread_get(cpu);
     578           0 :         struct mm_struct *mm = idle->active_mm;
     579             : 
     580             :         /*
     581             :          * idle_task_exit() will have switched to &init_mm, now
     582             :          * clean up any remaining active_mm state.
     583             :          */
     584           0 :         if (mm != &init_mm)
     585           0 :                 idle->active_mm = &init_mm;
     586           0 :         mmdrop(mm);
     587           0 :         return 0;
     588             : }
     589             : 
     590             : /*
     591             :  * Hotplug state machine related functions
     592             :  */
     593             : 
     594           0 : static void undo_cpu_up(unsigned int cpu, struct cpuhp_cpu_state *st)
     595             : {
     596           0 :         for (st->state--; st->state > st->target; st->state--)
     597           0 :                 cpuhp_invoke_callback(cpu, st->state, false, NULL, NULL);
     598           0 : }
     599             : 
     600           0 : static inline bool can_rollback_cpu(struct cpuhp_cpu_state *st)
     601             : {
     602           0 :         if (IS_ENABLED(CONFIG_HOTPLUG_CPU))
     603           0 :                 return true;
     604             :         /*
     605             :          * When CPU hotplug is disabled, then taking the CPU down is not
     606             :          * possible because takedown_cpu() and the architecture and
     607             :          * subsystem specific mechanisms are not available. So the CPU
     608             :          * which would be completely unplugged again needs to stay around
     609             :          * in the current state.
     610             :          */
     611             :         return st->state <= CPUHP_BRINGUP_CPU;
     612             : }
     613             : 
     614           3 : static int cpuhp_up_callbacks(unsigned int cpu, struct cpuhp_cpu_state *st,
     615             :                               enum cpuhp_state target)
     616             : {
     617           3 :         enum cpuhp_state prev_state = st->state;
     618           3 :         int ret = 0;
     619             : 
     620         264 :         while (st->state < target) {
     621         261 :                 st->state++;
     622         261 :                 ret = cpuhp_invoke_callback(cpu, st->state, true, NULL, NULL);
     623         261 :                 if (ret) {
     624           0 :                         if (can_rollback_cpu(st)) {
     625           0 :                                 st->target = prev_state;
     626           0 :                                 undo_cpu_up(cpu, st);
     627             :                         }
     628             :                         break;
     629             :                 }
     630             :         }
     631           3 :         return ret;
     632             : }
     633             : 
     634             : /*
     635             :  * The cpu hotplug threads manage the bringup and teardown of the cpus
     636             :  */
     637           4 : static void cpuhp_create(unsigned int cpu)
     638             : {
     639           4 :         struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
     640             : 
     641           4 :         init_completion(&st->done_up);
     642           4 :         init_completion(&st->done_down);
     643           4 : }
     644             : 
     645         253 : static int cpuhp_should_run(unsigned int cpu)
     646             : {
     647         253 :         struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state);
     648             : 
     649         253 :         return st->should_run;
     650             : }
     651             : 
     652             : /*
     653             :  * Execute teardown/startup callbacks on the plugged cpu. Also used to invoke
     654             :  * callbacks when a state gets [un]installed at runtime.
     655             :  *
     656             :  * Each invocation of this function by the smpboot thread does a single AP
     657             :  * state callback.
     658             :  *
     659             :  * It has 3 modes of operation:
     660             :  *  - single: runs st->cb_state
     661             :  *  - up:     runs ++st->state, while st->state < st->target
     662             :  *  - down:   runs st->state--, while st->state > st->target
     663             :  *
     664             :  * When complete or on error, should_run is cleared and the completion is fired.
     665             :  */
     666         237 : static void cpuhp_thread_fun(unsigned int cpu)
     667             : {
     668         237 :         struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state);
     669         237 :         bool bringup = st->bringup;
     670         237 :         enum cpuhp_state state;
     671             : 
     672         237 :         if (WARN_ON_ONCE(!st->should_run))
     673             :                 return;
     674             : 
     675             :         /*
     676             :          * ACQUIRE for the cpuhp_should_run() load of ->should_run. Ensures
     677             :          * that if we see ->should_run we also see the rest of the state.
     678             :          */
     679         237 :         smp_mb();
     680             : 
     681             :         /*
     682             :          * The BP holds the hotplug lock, but we're now running on the AP,
     683             :          * ensure that anybody asserting the lock is held, will actually find
     684             :          * it so.
     685             :          */
     686         237 :         lockdep_acquire_cpus_lock();
     687         237 :         cpuhp_lock_acquire(bringup);
     688             : 
     689         237 :         if (st->single) {
     690          12 :                 state = st->cb_state;
     691          12 :                 st->should_run = false;
     692             :         } else {
     693         225 :                 if (bringup) {
     694         225 :                         st->state++;
     695         225 :                         state = st->state;
     696         225 :                         st->should_run = (st->state < st->target);
     697         225 :                         WARN_ON_ONCE(st->state > st->target);
     698             :                 } else {
     699           0 :                         state = st->state;
     700           0 :                         st->state--;
     701           0 :                         st->should_run = (st->state > st->target);
     702           0 :                         WARN_ON_ONCE(st->state < st->target);
     703             :                 }
     704             :         }
     705             : 
     706         237 :         WARN_ON_ONCE(!cpuhp_is_ap_state(state));
     707             : 
     708         237 :         if (cpuhp_is_atomic_state(state)) {
     709           0 :                 local_irq_disable();
     710           0 :                 st->result = cpuhp_invoke_callback(cpu, state, bringup, st->node, &st->last);
     711           0 :                 local_irq_enable();
     712             : 
     713             :                 /*
     714             :                  * STARTING/DYING must not fail!
     715             :                  */
     716           0 :                 WARN_ON_ONCE(st->result);
     717             :         } else {
     718         237 :                 st->result = cpuhp_invoke_callback(cpu, state, bringup, st->node, &st->last);
     719             :         }
     720             : 
     721         237 :         if (st->result) {
     722             :                 /*
     723             :                  * If we fail on a rollback, we're up a creek without no
     724             :                  * paddle, no way forward, no way back. We loose, thanks for
     725             :                  * playing.
     726             :                  */
     727           0 :                 WARN_ON_ONCE(st->rollback);
     728           0 :                 st->should_run = false;
     729             :         }
     730             : 
     731         237 :         cpuhp_lock_release(bringup);
     732         237 :         lockdep_release_cpus_lock();
     733             : 
     734         237 :         if (!st->should_run)
     735          15 :                 complete_ap_thread(st, bringup);
     736             : }
     737             : 
     738             : /* Invoke a single callback on a remote cpu */
     739             : static int
     740          15 : cpuhp_invoke_ap_callback(int cpu, enum cpuhp_state state, bool bringup,
     741             :                          struct hlist_node *node)
     742             : {
     743          15 :         struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
     744          15 :         int ret;
     745             : 
     746          15 :         if (!cpu_online(cpu))
     747             :                 return 0;
     748             : 
     749          15 :         cpuhp_lock_acquire(false);
     750          15 :         cpuhp_lock_release(false);
     751             : 
     752          15 :         cpuhp_lock_acquire(true);
     753          15 :         cpuhp_lock_release(true);
     754             : 
     755             :         /*
     756             :          * If we are up and running, use the hotplug thread. For early calls
     757             :          * we invoke the thread function directly.
     758             :          */
     759          15 :         if (!st->thread)
     760           3 :                 return cpuhp_invoke_callback(cpu, state, bringup, node, NULL);
     761             : 
     762          12 :         st->rollback = false;
     763          12 :         st->last = NULL;
     764             : 
     765          12 :         st->node = node;
     766          12 :         st->bringup = bringup;
     767          12 :         st->cb_state = state;
     768          12 :         st->single = true;
     769             : 
     770          12 :         __cpuhp_kick_ap(st);
     771             : 
     772             :         /*
     773             :          * If we failed and did a partial, do a rollback.
     774             :          */
     775          12 :         if ((ret = st->result) && st->last) {
     776           0 :                 st->rollback = true;
     777           0 :                 st->bringup = !bringup;
     778             : 
     779           0 :                 __cpuhp_kick_ap(st);
     780             :         }
     781             : 
     782             :         /*
     783             :          * Clean up the leftovers so the next hotplug operation wont use stale
     784             :          * data.
     785             :          */
     786          12 :         st->node = st->last = NULL;
     787          12 :         return ret;
     788             : }
     789             : 
     790           0 : static int cpuhp_kick_ap_work(unsigned int cpu)
     791             : {
     792           0 :         struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
     793           0 :         enum cpuhp_state prev_state = st->state;
     794           0 :         int ret;
     795             : 
     796           0 :         cpuhp_lock_acquire(false);
     797           0 :         cpuhp_lock_release(false);
     798             : 
     799           0 :         cpuhp_lock_acquire(true);
     800           0 :         cpuhp_lock_release(true);
     801             : 
     802           0 :         trace_cpuhp_enter(cpu, st->target, prev_state, cpuhp_kick_ap_work);
     803           0 :         ret = cpuhp_kick_ap(st, st->target);
     804           0 :         trace_cpuhp_exit(cpu, st->state, prev_state, ret);
     805             : 
     806           0 :         return ret;
     807             : }
     808             : 
     809             : static struct smp_hotplug_thread cpuhp_threads = {
     810             :         .store                  = &cpuhp_state.thread,
     811             :         .create                 = &cpuhp_create,
     812             :         .thread_should_run      = cpuhp_should_run,
     813             :         .thread_fn              = cpuhp_thread_fun,
     814             :         .thread_comm            = "cpuhp/%u",
     815             :         .selfparking            = true,
     816             : };
     817             : 
     818           1 : void __init cpuhp_threads_init(void)
     819             : {
     820           1 :         BUG_ON(smpboot_register_percpu_thread(&cpuhp_threads));
     821           1 :         kthread_unpark(this_cpu_read(cpuhp_state.thread));
     822           1 : }
     823             : 
     824             : #ifdef CONFIG_HOTPLUG_CPU
     825             : #ifndef arch_clear_mm_cpumask_cpu
     826             : #define arch_clear_mm_cpumask_cpu(cpu, mm) cpumask_clear_cpu(cpu, mm_cpumask(mm))
     827             : #endif
     828             : 
     829             : /**
     830             :  * clear_tasks_mm_cpumask - Safely clear tasks' mm_cpumask for a CPU
     831             :  * @cpu: a CPU id
     832             :  *
     833             :  * This function walks all processes, finds a valid mm struct for each one and
     834             :  * then clears a corresponding bit in mm's cpumask.  While this all sounds
     835             :  * trivial, there are various non-obvious corner cases, which this function
     836             :  * tries to solve in a safe manner.
     837             :  *
     838             :  * Also note that the function uses a somewhat relaxed locking scheme, so it may
     839             :  * be called only for an already offlined CPU.
     840             :  */
     841           0 : void clear_tasks_mm_cpumask(int cpu)
     842             : {
     843           0 :         struct task_struct *p;
     844             : 
     845             :         /*
     846             :          * This function is called after the cpu is taken down and marked
     847             :          * offline, so its not like new tasks will ever get this cpu set in
     848             :          * their mm mask. -- Peter Zijlstra
     849             :          * Thus, we may use rcu_read_lock() here, instead of grabbing
     850             :          * full-fledged tasklist_lock.
     851             :          */
     852           0 :         WARN_ON(cpu_online(cpu));
     853           0 :         rcu_read_lock();
     854           0 :         for_each_process(p) {
     855           0 :                 struct task_struct *t;
     856             : 
     857             :                 /*
     858             :                  * Main thread might exit, but other threads may still have
     859             :                  * a valid mm. Find one.
     860             :                  */
     861           0 :                 t = find_lock_task_mm(p);
     862           0 :                 if (!t)
     863           0 :                         continue;
     864           0 :                 arch_clear_mm_cpumask_cpu(cpu, t->mm);
     865           0 :                 task_unlock(t);
     866             :         }
     867           0 :         rcu_read_unlock();
     868           0 : }
     869             : 
     870             : /* Take this CPU down. */
     871           0 : static int take_cpu_down(void *_param)
     872             : {
     873           0 :         struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state);
     874           0 :         enum cpuhp_state target = max((int)st->target, CPUHP_AP_OFFLINE);
     875           0 :         int err, cpu = smp_processor_id();
     876           0 :         int ret;
     877             : 
     878             :         /* Ensure this CPU doesn't handle any more interrupts. */
     879           0 :         err = __cpu_disable();
     880           0 :         if (err < 0)
     881             :                 return err;
     882             : 
     883             :         /*
     884             :          * We get here while we are in CPUHP_TEARDOWN_CPU state and we must not
     885             :          * do this step again.
     886             :          */
     887           0 :         WARN_ON(st->state != CPUHP_TEARDOWN_CPU);
     888           0 :         st->state--;
     889             :         /* Invoke the former CPU_DYING callbacks */
     890           0 :         for (; st->state > target; st->state--) {
     891           0 :                 ret = cpuhp_invoke_callback(cpu, st->state, false, NULL, NULL);
     892             :                 /*
     893             :                  * DYING must not fail!
     894             :                  */
     895           0 :                 WARN_ON_ONCE(ret);
     896             :         }
     897             : 
     898             :         /* Give up timekeeping duties */
     899           0 :         tick_handover_do_timer();
     900             :         /* Remove CPU from timer broadcasting */
     901           0 :         tick_offline_cpu(cpu);
     902             :         /* Park the stopper thread */
     903           0 :         stop_machine_park(cpu);
     904           0 :         return 0;
     905             : }
     906             : 
     907           0 : static int takedown_cpu(unsigned int cpu)
     908             : {
     909           0 :         struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
     910           0 :         int err;
     911             : 
     912             :         /* Park the smpboot threads */
     913           0 :         kthread_park(per_cpu_ptr(&cpuhp_state, cpu)->thread);
     914             : 
     915             :         /*
     916             :          * Prevent irq alloc/free while the dying cpu reorganizes the
     917             :          * interrupt affinities.
     918             :          */
     919           0 :         irq_lock_sparse();
     920             : 
     921             :         /*
     922             :          * So now all preempt/rcu users must observe !cpu_active().
     923             :          */
     924           0 :         err = stop_machine_cpuslocked(take_cpu_down, NULL, cpumask_of(cpu));
     925           0 :         if (err) {
     926             :                 /* CPU refused to die */
     927           0 :                 irq_unlock_sparse();
     928             :                 /* Unpark the hotplug thread so we can rollback there */
     929           0 :                 kthread_unpark(per_cpu_ptr(&cpuhp_state, cpu)->thread);
     930           0 :                 return err;
     931             :         }
     932           0 :         BUG_ON(cpu_online(cpu));
     933             : 
     934             :         /*
     935             :          * The teardown callback for CPUHP_AP_SCHED_STARTING will have removed
     936             :          * all runnable tasks from the CPU, there's only the idle task left now
     937             :          * that the migration thread is done doing the stop_machine thing.
     938             :          *
     939             :          * Wait for the stop thread to go away.
     940             :          */
     941           0 :         wait_for_ap_thread(st, false);
     942           0 :         BUG_ON(st->state != CPUHP_AP_IDLE_DEAD);
     943             : 
     944             :         /* Interrupts are moved away from the dying cpu, reenable alloc/free */
     945           0 :         irq_unlock_sparse();
     946             : 
     947           0 :         hotplug_cpu__broadcast_tick_pull(cpu);
     948             :         /* This actually kills the CPU. */
     949           0 :         __cpu_die(cpu);
     950             : 
     951           0 :         tick_cleanup_dead_cpu(cpu);
     952           0 :         rcutree_migrate_callbacks(cpu);
     953           0 :         return 0;
     954             : }
     955             : 
     956           0 : static void cpuhp_complete_idle_dead(void *arg)
     957             : {
     958           0 :         struct cpuhp_cpu_state *st = arg;
     959             : 
     960           0 :         complete_ap_thread(st, false);
     961           0 : }
     962             : 
     963           0 : void cpuhp_report_idle_dead(void)
     964             : {
     965           0 :         struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state);
     966             : 
     967           0 :         BUG_ON(st->state != CPUHP_AP_OFFLINE);
     968           0 :         rcu_report_dead(smp_processor_id());
     969           0 :         st->state = CPUHP_AP_IDLE_DEAD;
     970             :         /*
     971             :          * We cannot call complete after rcu_report_dead() so we delegate it
     972             :          * to an online cpu.
     973             :          */
     974           0 :         smp_call_function_single(cpumask_first(cpu_online_mask),
     975             :                                  cpuhp_complete_idle_dead, st, 0);
     976           0 : }
     977             : 
     978           0 : static void undo_cpu_down(unsigned int cpu, struct cpuhp_cpu_state *st)
     979             : {
     980           0 :         for (st->state++; st->state < st->target; st->state++)
     981           0 :                 cpuhp_invoke_callback(cpu, st->state, true, NULL, NULL);
     982           0 : }
     983             : 
     984           0 : static int cpuhp_down_callbacks(unsigned int cpu, struct cpuhp_cpu_state *st,
     985             :                                 enum cpuhp_state target)
     986             : {
     987           0 :         enum cpuhp_state prev_state = st->state;
     988           0 :         int ret = 0;
     989             : 
     990           0 :         for (; st->state > target; st->state--) {
     991           0 :                 ret = cpuhp_invoke_callback(cpu, st->state, false, NULL, NULL);
     992           0 :                 if (ret) {
     993           0 :                         st->target = prev_state;
     994           0 :                         if (st->state < prev_state)
     995           0 :                                 undo_cpu_down(cpu, st);
     996             :                         break;
     997             :                 }
     998             :         }
     999           0 :         return ret;
    1000             : }
    1001             : 
    1002             : /* Requires cpu_add_remove_lock to be held */
    1003           0 : static int __ref _cpu_down(unsigned int cpu, int tasks_frozen,
    1004             :                            enum cpuhp_state target)
    1005             : {
    1006           0 :         struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
    1007           0 :         int prev_state, ret = 0;
    1008             : 
    1009           0 :         if (num_online_cpus() == 1)
    1010             :                 return -EBUSY;
    1011             : 
    1012           0 :         if (!cpu_present(cpu))
    1013             :                 return -EINVAL;
    1014             : 
    1015           0 :         cpus_write_lock();
    1016             : 
    1017           0 :         cpuhp_tasks_frozen = tasks_frozen;
    1018             : 
    1019           0 :         prev_state = cpuhp_set_state(st, target);
    1020             :         /*
    1021             :          * If the current CPU state is in the range of the AP hotplug thread,
    1022             :          * then we need to kick the thread.
    1023             :          */
    1024           0 :         if (st->state > CPUHP_TEARDOWN_CPU) {
    1025           0 :                 st->target = max((int)target, CPUHP_TEARDOWN_CPU);
    1026           0 :                 ret = cpuhp_kick_ap_work(cpu);
    1027             :                 /*
    1028             :                  * The AP side has done the error rollback already. Just
    1029             :                  * return the error code..
    1030             :                  */
    1031           0 :                 if (ret)
    1032           0 :                         goto out;
    1033             : 
    1034             :                 /*
    1035             :                  * We might have stopped still in the range of the AP hotplug
    1036             :                  * thread. Nothing to do anymore.
    1037             :                  */
    1038           0 :                 if (st->state > CPUHP_TEARDOWN_CPU)
    1039           0 :                         goto out;
    1040             : 
    1041           0 :                 st->target = target;
    1042             :         }
    1043             :         /*
    1044             :          * The AP brought itself down to CPUHP_TEARDOWN_CPU. So we need
    1045             :          * to do the further cleanups.
    1046             :          */
    1047           0 :         ret = cpuhp_down_callbacks(cpu, st, target);
    1048           0 :         if (ret && st->state == CPUHP_TEARDOWN_CPU && st->state < prev_state) {
    1049           0 :                 cpuhp_reset_state(st, prev_state);
    1050           0 :                 __cpuhp_kick_ap(st);
    1051             :         }
    1052             : 
    1053           0 : out:
    1054           0 :         cpus_write_unlock();
    1055             :         /*
    1056             :          * Do post unplug cleanup. This is still protected against
    1057             :          * concurrent CPU hotplug via cpu_add_remove_lock.
    1058             :          */
    1059           0 :         lockup_detector_cleanup();
    1060           0 :         arch_smt_update();
    1061           0 :         return ret;
    1062             : }
    1063             : 
    1064           0 : static int cpu_down_maps_locked(unsigned int cpu, enum cpuhp_state target)
    1065             : {
    1066           0 :         if (cpu_hotplug_disabled)
    1067             :                 return -EBUSY;
    1068           0 :         return _cpu_down(cpu, 0, target);
    1069             : }
    1070             : 
    1071           0 : static int cpu_down(unsigned int cpu, enum cpuhp_state target)
    1072             : {
    1073           0 :         int err;
    1074             : 
    1075           0 :         cpu_maps_update_begin();
    1076           0 :         err = cpu_down_maps_locked(cpu, target);
    1077           0 :         cpu_maps_update_done();
    1078           0 :         return err;
    1079             : }
    1080             : 
    1081             : /**
    1082             :  * cpu_device_down - Bring down a cpu device
    1083             :  * @dev: Pointer to the cpu device to offline
    1084             :  *
    1085             :  * This function is meant to be used by device core cpu subsystem only.
    1086             :  *
    1087             :  * Other subsystems should use remove_cpu() instead.
    1088             :  */
    1089           0 : int cpu_device_down(struct device *dev)
    1090             : {
    1091           0 :         return cpu_down(dev->id, CPUHP_OFFLINE);
    1092             : }
    1093             : 
    1094           0 : int remove_cpu(unsigned int cpu)
    1095             : {
    1096           0 :         int ret;
    1097             : 
    1098           0 :         lock_device_hotplug();
    1099           0 :         ret = device_offline(get_cpu_device(cpu));
    1100           0 :         unlock_device_hotplug();
    1101             : 
    1102           0 :         return ret;
    1103             : }
    1104             : EXPORT_SYMBOL_GPL(remove_cpu);
    1105             : 
    1106           0 : void smp_shutdown_nonboot_cpus(unsigned int primary_cpu)
    1107             : {
    1108           0 :         unsigned int cpu;
    1109           0 :         int error;
    1110             : 
    1111           0 :         cpu_maps_update_begin();
    1112             : 
    1113             :         /*
    1114             :          * Make certain the cpu I'm about to reboot on is online.
    1115             :          *
    1116             :          * This is inline to what migrate_to_reboot_cpu() already do.
    1117             :          */
    1118           0 :         if (!cpu_online(primary_cpu))
    1119           0 :                 primary_cpu = cpumask_first(cpu_online_mask);
    1120             : 
    1121           0 :         for_each_online_cpu(cpu) {
    1122           0 :                 if (cpu == primary_cpu)
    1123           0 :                         continue;
    1124             : 
    1125           0 :                 error = cpu_down_maps_locked(cpu, CPUHP_OFFLINE);
    1126           0 :                 if (error) {
    1127           0 :                         pr_err("Failed to offline CPU%d - error=%d",
    1128             :                                 cpu, error);
    1129           0 :                         break;
    1130             :                 }
    1131             :         }
    1132             : 
    1133             :         /*
    1134             :          * Ensure all but the reboot CPU are offline.
    1135             :          */
    1136           0 :         BUG_ON(num_online_cpus() > 1);
    1137             : 
    1138             :         /*
    1139             :          * Make sure the CPUs won't be enabled by someone else after this
    1140             :          * point. Kexec will reboot to a new kernel shortly resetting
    1141             :          * everything along the way.
    1142             :          */
    1143           0 :         cpu_hotplug_disabled++;
    1144             : 
    1145           0 :         cpu_maps_update_done();
    1146           0 : }
    1147             : 
    1148             : #else
    1149             : #define takedown_cpu            NULL
    1150             : #endif /*CONFIG_HOTPLUG_CPU*/
    1151             : 
    1152             : /**
    1153             :  * notify_cpu_starting(cpu) - Invoke the callbacks on the starting CPU
    1154             :  * @cpu: cpu that just started
    1155             :  *
    1156             :  * It must be called by the arch code on the new cpu, before the new cpu
    1157             :  * enables interrupts and before the "boot" cpu returns from __cpu_up().
    1158             :  */
    1159           3 : void notify_cpu_starting(unsigned int cpu)
    1160             : {
    1161           3 :         struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
    1162           3 :         enum cpuhp_state target = min((int)st->target, CPUHP_AP_ONLINE);
    1163           3 :         int ret;
    1164             : 
    1165           3 :         rcu_cpu_starting(cpu);  /* Enables RCU usage on this CPU. */
    1166           3 :         cpumask_set_cpu(cpu, &cpus_booted_once_mask);
    1167         171 :         while (st->state < target) {
    1168         168 :                 st->state++;
    1169         168 :                 ret = cpuhp_invoke_callback(cpu, st->state, true, NULL, NULL);
    1170             :                 /*
    1171             :                  * STARTING must not fail!
    1172             :                  */
    1173         168 :                 WARN_ON_ONCE(ret);
    1174             :         }
    1175           3 : }
    1176             : 
    1177             : /*
    1178             :  * Called from the idle task. Wake up the controlling task which brings the
    1179             :  * hotplug thread of the upcoming CPU up and then delegates the rest of the
    1180             :  * online bringup to the hotplug thread.
    1181             :  */
    1182           4 : void cpuhp_online_idle(enum cpuhp_state state)
    1183             : {
    1184           4 :         struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state);
    1185             : 
    1186             :         /* Happens for the boot cpu */
    1187           4 :         if (state != CPUHP_AP_ONLINE_IDLE)
    1188             :                 return;
    1189             : 
    1190             :         /*
    1191             :          * Unpart the stopper thread before we start the idle loop (and start
    1192             :          * scheduling); this ensures the stopper task is always available.
    1193             :          */
    1194           3 :         stop_machine_unpark(smp_processor_id());
    1195             : 
    1196           3 :         st->state = CPUHP_AP_ONLINE_IDLE;
    1197           3 :         complete_ap_thread(st, true);
    1198             : }
    1199             : 
    1200             : /* Requires cpu_add_remove_lock to be held */
    1201           3 : static int _cpu_up(unsigned int cpu, int tasks_frozen, enum cpuhp_state target)
    1202             : {
    1203           3 :         struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
    1204           3 :         struct task_struct *idle;
    1205           3 :         int ret = 0;
    1206             : 
    1207           3 :         cpus_write_lock();
    1208             : 
    1209           3 :         if (!cpu_present(cpu)) {
    1210           0 :                 ret = -EINVAL;
    1211           0 :                 goto out;
    1212             :         }
    1213             : 
    1214             :         /*
    1215             :          * The caller of cpu_up() might have raced with another
    1216             :          * caller. Nothing to do.
    1217             :          */
    1218           3 :         if (st->state >= target)
    1219           0 :                 goto out;
    1220             : 
    1221           3 :         if (st->state == CPUHP_OFFLINE) {
    1222             :                 /* Let it fail before we try to bring the cpu up */
    1223           3 :                 idle = idle_thread_get(cpu);
    1224           3 :                 if (IS_ERR(idle)) {
    1225           0 :                         ret = PTR_ERR(idle);
    1226           0 :                         goto out;
    1227             :                 }
    1228             :         }
    1229             : 
    1230           3 :         cpuhp_tasks_frozen = tasks_frozen;
    1231             : 
    1232           3 :         cpuhp_set_state(st, target);
    1233             :         /*
    1234             :          * If the current CPU state is in the range of the AP hotplug thread,
    1235             :          * then we need to kick the thread once more.
    1236             :          */
    1237           3 :         if (st->state > CPUHP_BRINGUP_CPU) {
    1238           0 :                 ret = cpuhp_kick_ap_work(cpu);
    1239             :                 /*
    1240             :                  * The AP side has done the error rollback already. Just
    1241             :                  * return the error code..
    1242             :                  */
    1243           0 :                 if (ret)
    1244           0 :                         goto out;
    1245             :         }
    1246             : 
    1247             :         /*
    1248             :          * Try to reach the target state. We max out on the BP at
    1249             :          * CPUHP_BRINGUP_CPU. After that the AP hotplug thread is
    1250             :          * responsible for bringing it up to the target state.
    1251             :          */
    1252           3 :         target = min((int)target, CPUHP_BRINGUP_CPU);
    1253           3 :         ret = cpuhp_up_callbacks(cpu, st, target);
    1254           3 : out:
    1255           3 :         cpus_write_unlock();
    1256           3 :         arch_smt_update();
    1257           3 :         return ret;
    1258             : }
    1259             : 
    1260           3 : static int cpu_up(unsigned int cpu, enum cpuhp_state target)
    1261             : {
    1262           3 :         int err = 0;
    1263             : 
    1264           3 :         if (!cpu_possible(cpu)) {
    1265           0 :                 pr_err("can't online cpu %d because it is not configured as may-hotadd at boot time\n",
    1266             :                        cpu);
    1267             : #if defined(CONFIG_IA64)
    1268             :                 pr_err("please check additional_cpus= boot parameter\n");
    1269             : #endif
    1270           0 :                 return -EINVAL;
    1271             :         }
    1272             : 
    1273           3 :         err = try_online_node(cpu_to_node(cpu));
    1274           3 :         if (err)
    1275             :                 return err;
    1276             : 
    1277           3 :         cpu_maps_update_begin();
    1278             : 
    1279           3 :         if (cpu_hotplug_disabled) {
    1280           0 :                 err = -EBUSY;
    1281           0 :                 goto out;
    1282             :         }
    1283           3 :         if (!cpu_smt_allowed(cpu)) {
    1284           0 :                 err = -EPERM;
    1285           0 :                 goto out;
    1286             :         }
    1287             : 
    1288           3 :         err = _cpu_up(cpu, 0, target);
    1289           3 : out:
    1290           3 :         cpu_maps_update_done();
    1291           3 :         return err;
    1292             : }
    1293             : 
    1294             : /**
    1295             :  * cpu_device_up - Bring up a cpu device
    1296             :  * @dev: Pointer to the cpu device to online
    1297             :  *
    1298             :  * This function is meant to be used by device core cpu subsystem only.
    1299             :  *
    1300             :  * Other subsystems should use add_cpu() instead.
    1301             :  */
    1302           0 : int cpu_device_up(struct device *dev)
    1303             : {
    1304           0 :         return cpu_up(dev->id, CPUHP_ONLINE);
    1305             : }
    1306             : 
    1307           0 : int add_cpu(unsigned int cpu)
    1308             : {
    1309           0 :         int ret;
    1310             : 
    1311           0 :         lock_device_hotplug();
    1312           0 :         ret = device_online(get_cpu_device(cpu));
    1313           0 :         unlock_device_hotplug();
    1314             : 
    1315           0 :         return ret;
    1316             : }
    1317             : EXPORT_SYMBOL_GPL(add_cpu);
    1318             : 
    1319             : /**
    1320             :  * bringup_hibernate_cpu - Bring up the CPU that we hibernated on
    1321             :  * @sleep_cpu: The cpu we hibernated on and should be brought up.
    1322             :  *
    1323             :  * On some architectures like arm64, we can hibernate on any CPU, but on
    1324             :  * wake up the CPU we hibernated on might be offline as a side effect of
    1325             :  * using maxcpus= for example.
    1326             :  */
    1327           0 : int bringup_hibernate_cpu(unsigned int sleep_cpu)
    1328             : {
    1329           0 :         int ret;
    1330             : 
    1331           0 :         if (!cpu_online(sleep_cpu)) {
    1332           0 :                 pr_info("Hibernated on a CPU that is offline! Bringing CPU up.\n");
    1333           0 :                 ret = cpu_up(sleep_cpu, CPUHP_ONLINE);
    1334           0 :                 if (ret) {
    1335           0 :                         pr_err("Failed to bring hibernate-CPU up!\n");
    1336           0 :                         return ret;
    1337             :                 }
    1338             :         }
    1339             :         return 0;
    1340             : }
    1341             : 
    1342           1 : void bringup_nonboot_cpus(unsigned int setup_max_cpus)
    1343             : {
    1344           1 :         unsigned int cpu;
    1345             : 
    1346           5 :         for_each_present_cpu(cpu) {
    1347           4 :                 if (num_online_cpus() >= setup_max_cpus)
    1348             :                         break;
    1349           4 :                 if (!cpu_online(cpu))
    1350           3 :                         cpu_up(cpu, CPUHP_ONLINE);
    1351             :         }
    1352           1 : }
    1353             : 
    1354             : #ifdef CONFIG_PM_SLEEP_SMP
    1355             : static cpumask_var_t frozen_cpus;
    1356             : 
    1357             : int freeze_secondary_cpus(int primary)
    1358             : {
    1359             :         int cpu, error = 0;
    1360             : 
    1361             :         cpu_maps_update_begin();
    1362             :         if (primary == -1) {
    1363             :                 primary = cpumask_first(cpu_online_mask);
    1364             :                 if (!housekeeping_cpu(primary, HK_FLAG_TIMER))
    1365             :                         primary = housekeeping_any_cpu(HK_FLAG_TIMER);
    1366             :         } else {
    1367             :                 if (!cpu_online(primary))
    1368             :                         primary = cpumask_first(cpu_online_mask);
    1369             :         }
    1370             : 
    1371             :         /*
    1372             :          * We take down all of the non-boot CPUs in one shot to avoid races
    1373             :          * with the userspace trying to use the CPU hotplug at the same time
    1374             :          */
    1375             :         cpumask_clear(frozen_cpus);
    1376             : 
    1377             :         pr_info("Disabling non-boot CPUs ...\n");
    1378             :         for_each_online_cpu(cpu) {
    1379             :                 if (cpu == primary)
    1380             :                         continue;
    1381             : 
    1382             :                 if (pm_wakeup_pending()) {
    1383             :                         pr_info("Wakeup pending. Abort CPU freeze\n");
    1384             :                         error = -EBUSY;
    1385             :                         break;
    1386             :                 }
    1387             : 
    1388             :                 trace_suspend_resume(TPS("CPU_OFF"), cpu, true);
    1389             :                 error = _cpu_down(cpu, 1, CPUHP_OFFLINE);
    1390             :                 trace_suspend_resume(TPS("CPU_OFF"), cpu, false);
    1391             :                 if (!error)
    1392             :                         cpumask_set_cpu(cpu, frozen_cpus);
    1393             :                 else {
    1394             :                         pr_err("Error taking CPU%d down: %d\n", cpu, error);
    1395             :                         break;
    1396             :                 }
    1397             :         }
    1398             : 
    1399             :         if (!error)
    1400             :                 BUG_ON(num_online_cpus() > 1);
    1401             :         else
    1402             :                 pr_err("Non-boot CPUs are not disabled\n");
    1403             : 
    1404             :         /*
    1405             :          * Make sure the CPUs won't be enabled by someone else. We need to do
    1406             :          * this even in case of failure as all freeze_secondary_cpus() users are
    1407             :          * supposed to do thaw_secondary_cpus() on the failure path.
    1408             :          */
    1409             :         cpu_hotplug_disabled++;
    1410             : 
    1411             :         cpu_maps_update_done();
    1412             :         return error;
    1413             : }
    1414             : 
    1415             : void __weak arch_thaw_secondary_cpus_begin(void)
    1416             : {
    1417             : }
    1418             : 
    1419             : void __weak arch_thaw_secondary_cpus_end(void)
    1420             : {
    1421             : }
    1422             : 
    1423             : void thaw_secondary_cpus(void)
    1424             : {
    1425             :         int cpu, error;
    1426             : 
    1427             :         /* Allow everyone to use the CPU hotplug again */
    1428             :         cpu_maps_update_begin();
    1429             :         __cpu_hotplug_enable();
    1430             :         if (cpumask_empty(frozen_cpus))
    1431             :                 goto out;
    1432             : 
    1433             :         pr_info("Enabling non-boot CPUs ...\n");
    1434             : 
    1435             :         arch_thaw_secondary_cpus_begin();
    1436             : 
    1437             :         for_each_cpu(cpu, frozen_cpus) {
    1438             :                 trace_suspend_resume(TPS("CPU_ON"), cpu, true);
    1439             :                 error = _cpu_up(cpu, 1, CPUHP_ONLINE);
    1440             :                 trace_suspend_resume(TPS("CPU_ON"), cpu, false);
    1441             :                 if (!error) {
    1442             :                         pr_info("CPU%d is up\n", cpu);
    1443             :                         continue;
    1444             :                 }
    1445             :                 pr_warn("Error taking CPU%d up: %d\n", cpu, error);
    1446             :         }
    1447             : 
    1448             :         arch_thaw_secondary_cpus_end();
    1449             : 
    1450             :         cpumask_clear(frozen_cpus);
    1451             : out:
    1452             :         cpu_maps_update_done();
    1453             : }
    1454             : 
    1455             : static int __init alloc_frozen_cpus(void)
    1456             : {
    1457             :         if (!alloc_cpumask_var(&frozen_cpus, GFP_KERNEL|__GFP_ZERO))
    1458             :                 return -ENOMEM;
    1459             :         return 0;
    1460             : }
    1461             : core_initcall(alloc_frozen_cpus);
    1462             : 
    1463             : /*
    1464             :  * When callbacks for CPU hotplug notifications are being executed, we must
    1465             :  * ensure that the state of the system with respect to the tasks being frozen
    1466             :  * or not, as reported by the notification, remains unchanged *throughout the
    1467             :  * duration* of the execution of the callbacks.
    1468             :  * Hence we need to prevent the freezer from racing with regular CPU hotplug.
    1469             :  *
    1470             :  * This synchronization is implemented by mutually excluding regular CPU
    1471             :  * hotplug and Suspend/Hibernate call paths by hooking onto the Suspend/
    1472             :  * Hibernate notifications.
    1473             :  */
    1474             : static int
    1475             : cpu_hotplug_pm_callback(struct notifier_block *nb,
    1476             :                         unsigned long action, void *ptr)
    1477             : {
    1478             :         switch (action) {
    1479             : 
    1480             :         case PM_SUSPEND_PREPARE:
    1481             :         case PM_HIBERNATION_PREPARE:
    1482             :                 cpu_hotplug_disable();
    1483             :                 break;
    1484             : 
    1485             :         case PM_POST_SUSPEND:
    1486             :         case PM_POST_HIBERNATION:
    1487             :                 cpu_hotplug_enable();
    1488             :                 break;
    1489             : 
    1490             :         default:
    1491             :                 return NOTIFY_DONE;
    1492             :         }
    1493             : 
    1494             :         return NOTIFY_OK;
    1495             : }
    1496             : 
    1497             : 
    1498             : static int __init cpu_hotplug_pm_sync_init(void)
    1499             : {
    1500             :         /*
    1501             :          * cpu_hotplug_pm_callback has higher priority than x86
    1502             :          * bsp_pm_callback which depends on cpu_hotplug_pm_callback
    1503             :          * to disable cpu hotplug to avoid cpu hotplug race.
    1504             :          */
    1505             :         pm_notifier(cpu_hotplug_pm_callback, 0);
    1506             :         return 0;
    1507             : }
    1508             : core_initcall(cpu_hotplug_pm_sync_init);
    1509             : 
    1510             : #endif /* CONFIG_PM_SLEEP_SMP */
    1511             : 
    1512             : int __boot_cpu_id;
    1513             : 
    1514             : #endif /* CONFIG_SMP */
    1515             : 
    1516             : /* Boot processor state steps */
    1517             : static struct cpuhp_step cpuhp_hp_states[] = {
    1518             :         [CPUHP_OFFLINE] = {
    1519             :                 .name                   = "offline",
    1520             :                 .startup.single         = NULL,
    1521             :                 .teardown.single        = NULL,
    1522             :         },
    1523             : #ifdef CONFIG_SMP
    1524             :         [CPUHP_CREATE_THREADS]= {
    1525             :                 .name                   = "threads:prepare",
    1526             :                 .startup.single         = smpboot_create_threads,
    1527             :                 .teardown.single        = NULL,
    1528             :                 .cant_stop              = true,
    1529             :         },
    1530             :         [CPUHP_PERF_PREPARE] = {
    1531             :                 .name                   = "perf:prepare",
    1532             :                 .startup.single         = perf_event_init_cpu,
    1533             :                 .teardown.single        = perf_event_exit_cpu,
    1534             :         },
    1535             :         [CPUHP_WORKQUEUE_PREP] = {
    1536             :                 .name                   = "workqueue:prepare",
    1537             :                 .startup.single         = workqueue_prepare_cpu,
    1538             :                 .teardown.single        = NULL,
    1539             :         },
    1540             :         [CPUHP_HRTIMERS_PREPARE] = {
    1541             :                 .name                   = "hrtimers:prepare",
    1542             :                 .startup.single         = hrtimers_prepare_cpu,
    1543             :                 .teardown.single        = hrtimers_dead_cpu,
    1544             :         },
    1545             :         [CPUHP_SMPCFD_PREPARE] = {
    1546             :                 .name                   = "smpcfd:prepare",
    1547             :                 .startup.single         = smpcfd_prepare_cpu,
    1548             :                 .teardown.single        = smpcfd_dead_cpu,
    1549             :         },
    1550             :         [CPUHP_RELAY_PREPARE] = {
    1551             :                 .name                   = "relay:prepare",
    1552             :                 .startup.single         = relay_prepare_cpu,
    1553             :                 .teardown.single        = NULL,
    1554             :         },
    1555             :         [CPUHP_SLAB_PREPARE] = {
    1556             :                 .name                   = "slab:prepare",
    1557             :                 .startup.single         = slab_prepare_cpu,
    1558             :                 .teardown.single        = slab_dead_cpu,
    1559             :         },
    1560             :         [CPUHP_RCUTREE_PREP] = {
    1561             :                 .name                   = "RCU/tree:prepare",
    1562             :                 .startup.single         = rcutree_prepare_cpu,
    1563             :                 .teardown.single        = rcutree_dead_cpu,
    1564             :         },
    1565             :         /*
    1566             :          * On the tear-down path, timers_dead_cpu() must be invoked
    1567             :          * before blk_mq_queue_reinit_notify() from notify_dead(),
    1568             :          * otherwise a RCU stall occurs.
    1569             :          */
    1570             :         [CPUHP_TIMERS_PREPARE] = {
    1571             :                 .name                   = "timers:prepare",
    1572             :                 .startup.single         = timers_prepare_cpu,
    1573             :                 .teardown.single        = timers_dead_cpu,
    1574             :         },
    1575             :         /* Kicks the plugged cpu into life */
    1576             :         [CPUHP_BRINGUP_CPU] = {
    1577             :                 .name                   = "cpu:bringup",
    1578             :                 .startup.single         = bringup_cpu,
    1579             :                 .teardown.single        = finish_cpu,
    1580             :                 .cant_stop              = true,
    1581             :         },
    1582             :         /* Final state before CPU kills itself */
    1583             :         [CPUHP_AP_IDLE_DEAD] = {
    1584             :                 .name                   = "idle:dead",
    1585             :         },
    1586             :         /*
    1587             :          * Last state before CPU enters the idle loop to die. Transient state
    1588             :          * for synchronization.
    1589             :          */
    1590             :         [CPUHP_AP_OFFLINE] = {
    1591             :                 .name                   = "ap:offline",
    1592             :                 .cant_stop              = true,
    1593             :         },
    1594             :         /* First state is scheduler control. Interrupts are disabled */
    1595             :         [CPUHP_AP_SCHED_STARTING] = {
    1596             :                 .name                   = "sched:starting",
    1597             :                 .startup.single         = sched_cpu_starting,
    1598             :                 .teardown.single        = sched_cpu_dying,
    1599             :         },
    1600             :         [CPUHP_AP_RCUTREE_DYING] = {
    1601             :                 .name                   = "RCU/tree:dying",
    1602             :                 .startup.single         = NULL,
    1603             :                 .teardown.single        = rcutree_dying_cpu,
    1604             :         },
    1605             :         [CPUHP_AP_SMPCFD_DYING] = {
    1606             :                 .name                   = "smpcfd:dying",
    1607             :                 .startup.single         = NULL,
    1608             :                 .teardown.single        = smpcfd_dying_cpu,
    1609             :         },
    1610             :         /* Entry state on starting. Interrupts enabled from here on. Transient
    1611             :          * state for synchronsization */
    1612             :         [CPUHP_AP_ONLINE] = {
    1613             :                 .name                   = "ap:online",
    1614             :         },
    1615             :         /*
    1616             :          * Handled on control processor until the plugged processor manages
    1617             :          * this itself.
    1618             :          */
    1619             :         [CPUHP_TEARDOWN_CPU] = {
    1620             :                 .name                   = "cpu:teardown",
    1621             :                 .startup.single         = NULL,
    1622             :                 .teardown.single        = takedown_cpu,
    1623             :                 .cant_stop              = true,
    1624             :         },
    1625             : 
    1626             :         [CPUHP_AP_SCHED_WAIT_EMPTY] = {
    1627             :                 .name                   = "sched:waitempty",
    1628             :                 .startup.single         = NULL,
    1629             :                 .teardown.single        = sched_cpu_wait_empty,
    1630             :         },
    1631             : 
    1632             :         /* Handle smpboot threads park/unpark */
    1633             :         [CPUHP_AP_SMPBOOT_THREADS] = {
    1634             :                 .name                   = "smpboot/threads:online",
    1635             :                 .startup.single         = smpboot_unpark_threads,
    1636             :                 .teardown.single        = smpboot_park_threads,
    1637             :         },
    1638             :         [CPUHP_AP_IRQ_AFFINITY_ONLINE] = {
    1639             :                 .name                   = "irq/affinity:online",
    1640             :                 .startup.single         = irq_affinity_online_cpu,
    1641             :                 .teardown.single        = NULL,
    1642             :         },
    1643             :         [CPUHP_AP_PERF_ONLINE] = {
    1644             :                 .name                   = "perf:online",
    1645             :                 .startup.single         = perf_event_init_cpu,
    1646             :                 .teardown.single        = perf_event_exit_cpu,
    1647             :         },
    1648             :         [CPUHP_AP_WATCHDOG_ONLINE] = {
    1649             :                 .name                   = "lockup_detector:online",
    1650             :                 .startup.single         = lockup_detector_online_cpu,
    1651             :                 .teardown.single        = lockup_detector_offline_cpu,
    1652             :         },
    1653             :         [CPUHP_AP_WORKQUEUE_ONLINE] = {
    1654             :                 .name                   = "workqueue:online",
    1655             :                 .startup.single         = workqueue_online_cpu,
    1656             :                 .teardown.single        = workqueue_offline_cpu,
    1657             :         },
    1658             :         [CPUHP_AP_RCUTREE_ONLINE] = {
    1659             :                 .name                   = "RCU/tree:online",
    1660             :                 .startup.single         = rcutree_online_cpu,
    1661             :                 .teardown.single        = rcutree_offline_cpu,
    1662             :         },
    1663             : #endif
    1664             :         /*
    1665             :          * The dynamically registered state space is here
    1666             :          */
    1667             : 
    1668             : #ifdef CONFIG_SMP
    1669             :         /* Last state is scheduler control setting the cpu active */
    1670             :         [CPUHP_AP_ACTIVE] = {
    1671             :                 .name                   = "sched:active",
    1672             :                 .startup.single         = sched_cpu_activate,
    1673             :                 .teardown.single        = sched_cpu_deactivate,
    1674             :         },
    1675             : #endif
    1676             : 
    1677             :         /* CPU is fully up and running. */
    1678             :         [CPUHP_ONLINE] = {
    1679             :                 .name                   = "online",
    1680             :                 .startup.single         = NULL,
    1681             :                 .teardown.single        = NULL,
    1682             :         },
    1683             : };
    1684             : 
    1685             : /* Sanity check for callbacks */
    1686          30 : static int cpuhp_cb_check(enum cpuhp_state state)
    1687             : {
    1688          30 :         if (state <= CPUHP_OFFLINE || state >= CPUHP_ONLINE)
    1689           0 :                 return -EINVAL;
    1690             :         return 0;
    1691             : }
    1692             : 
    1693             : /*
    1694             :  * Returns a free for dynamic slot assignment of the Online state. The states
    1695             :  * are protected by the cpuhp_slot_states mutex and an empty slot is identified
    1696             :  * by having no name assigned.
    1697             :  */
    1698           9 : static int cpuhp_reserve_state(enum cpuhp_state state)
    1699             : {
    1700           9 :         enum cpuhp_state i, end;
    1701           9 :         struct cpuhp_step *step;
    1702             : 
    1703           9 :         switch (state) {
    1704             :         case CPUHP_AP_ONLINE_DYN:
    1705             :                 step = cpuhp_hp_states + CPUHP_AP_ONLINE_DYN;
    1706             :                 end = CPUHP_AP_ONLINE_DYN_END;
    1707             :                 break;
    1708           1 :         case CPUHP_BP_PREPARE_DYN:
    1709           1 :                 step = cpuhp_hp_states + CPUHP_BP_PREPARE_DYN;
    1710           1 :                 end = CPUHP_BP_PREPARE_DYN_END;
    1711           1 :                 break;
    1712             :         default:
    1713             :                 return -EINVAL;
    1714             :         }
    1715             : 
    1716          37 :         for (i = state; i <= end; i++, step++) {
    1717          37 :                 if (!step->name)
    1718           9 :                         return i;
    1719             :         }
    1720           0 :         WARN(1, "No more dynamic states available for CPU hotplug\n");
    1721           0 :         return -ENOSPC;
    1722             : }
    1723             : 
    1724          30 : static int cpuhp_store_callbacks(enum cpuhp_state state, const char *name,
    1725             :                                  int (*startup)(unsigned int cpu),
    1726             :                                  int (*teardown)(unsigned int cpu),
    1727             :                                  bool multi_instance)
    1728             : {
    1729             :         /* (Un)Install the callbacks for further cpu hotplug operations */
    1730          30 :         struct cpuhp_step *sp;
    1731          30 :         int ret = 0;
    1732             : 
    1733             :         /*
    1734             :          * If name is NULL, then the state gets removed.
    1735             :          *
    1736             :          * CPUHP_AP_ONLINE_DYN and CPUHP_BP_PREPARE_DYN are handed out on
    1737             :          * the first allocation from these dynamic ranges, so the removal
    1738             :          * would trigger a new allocation and clear the wrong (already
    1739             :          * empty) state, leaving the callbacks of the to be cleared state
    1740             :          * dangling, which causes wreckage on the next hotplug operation.
    1741             :          */
    1742          30 :         if (name && (state == CPUHP_AP_ONLINE_DYN ||
    1743          30 :                      state == CPUHP_BP_PREPARE_DYN)) {
    1744           9 :                 ret = cpuhp_reserve_state(state);
    1745           9 :                 if (ret < 0)
    1746             :                         return ret;
    1747             :                 state = ret;
    1748             :         }
    1749          30 :         sp = cpuhp_get_step(state);
    1750          30 :         if (name && sp->name)
    1751             :                 return -EBUSY;
    1752             : 
    1753          30 :         sp->startup.single = startup;
    1754          30 :         sp->teardown.single = teardown;
    1755          30 :         sp->name = name;
    1756          30 :         sp->multi_instance = multi_instance;
    1757          30 :         INIT_HLIST_HEAD(&sp->list);
    1758          30 :         return ret;
    1759             : }
    1760             : 
    1761           0 : static void *cpuhp_get_teardown_cb(enum cpuhp_state state)
    1762             : {
    1763           0 :         return cpuhp_get_step(state)->teardown.single;
    1764             : }
    1765             : 
    1766             : /*
    1767             :  * Call the startup/teardown function for a step either on the AP or
    1768             :  * on the current CPU.
    1769             :  */
    1770          22 : static int cpuhp_issue_call(int cpu, enum cpuhp_state state, bool bringup,
    1771             :                             struct hlist_node *node)
    1772             : {
    1773          22 :         struct cpuhp_step *sp = cpuhp_get_step(state);
    1774          22 :         int ret;
    1775             : 
    1776             :         /*
    1777             :          * If there's nothing to do, we done.
    1778             :          * Relies on the union for multi_instance.
    1779             :          */
    1780          22 :         if ((bringup && !sp->startup.single) ||
    1781          22 :             (!bringup && !sp->teardown.single))
    1782             :                 return 0;
    1783             :         /*
    1784             :          * The non AP bound callbacks can fail on bringup. On teardown
    1785             :          * e.g. module removal we crash for now.
    1786             :          */
    1787             : #ifdef CONFIG_SMP
    1788          22 :         if (cpuhp_is_ap_state(state))
    1789          15 :                 ret = cpuhp_invoke_ap_callback(cpu, state, bringup, node);
    1790             :         else
    1791           7 :                 ret = cpuhp_invoke_callback(cpu, state, bringup, node, NULL);
    1792             : #else
    1793             :         ret = cpuhp_invoke_callback(cpu, state, bringup, node, NULL);
    1794             : #endif
    1795          22 :         BUG_ON(ret && !bringup);
    1796             :         return ret;
    1797             : }
    1798             : 
    1799             : /*
    1800             :  * Called from __cpuhp_setup_state on a recoverable failure.
    1801             :  *
    1802             :  * Note: The teardown callbacks for rollback are not allowed to fail!
    1803             :  */
    1804           0 : static void cpuhp_rollback_install(int failedcpu, enum cpuhp_state state,
    1805             :                                    struct hlist_node *node)
    1806             : {
    1807           0 :         int cpu;
    1808             : 
    1809             :         /* Roll back the already executed steps on the other cpus */
    1810           0 :         for_each_present_cpu(cpu) {
    1811           0 :                 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
    1812           0 :                 int cpustate = st->state;
    1813             : 
    1814           0 :                 if (cpu >= failedcpu)
    1815             :                         break;
    1816             : 
    1817             :                 /* Did we invoke the startup call on that cpu ? */
    1818           0 :                 if (cpustate >= state)
    1819           0 :                         cpuhp_issue_call(cpu, state, false, node);
    1820             :         }
    1821           0 : }
    1822             : 
    1823          14 : int __cpuhp_state_add_instance_cpuslocked(enum cpuhp_state state,
    1824             :                                           struct hlist_node *node,
    1825             :                                           bool invoke)
    1826             : {
    1827          14 :         struct cpuhp_step *sp;
    1828          14 :         int cpu;
    1829          14 :         int ret;
    1830             : 
    1831          14 :         lockdep_assert_cpus_held();
    1832             : 
    1833          14 :         sp = cpuhp_get_step(state);
    1834          14 :         if (sp->multi_instance == false)
    1835             :                 return -EINVAL;
    1836             : 
    1837          14 :         mutex_lock(&cpuhp_state_mutex);
    1838             : 
    1839          14 :         if (!invoke || !sp->startup.multi)
    1840          12 :                 goto add_node;
    1841             : 
    1842             :         /*
    1843             :          * Try to call the startup callback for each present cpu
    1844             :          * depending on the hotplug state of the cpu.
    1845             :          */
    1846          10 :         for_each_present_cpu(cpu) {
    1847           8 :                 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
    1848           8 :                 int cpustate = st->state;
    1849             : 
    1850           8 :                 if (cpustate < state)
    1851           6 :                         continue;
    1852             : 
    1853           2 :                 ret = cpuhp_issue_call(cpu, state, true, node);
    1854           2 :                 if (ret) {
    1855           0 :                         if (sp->teardown.multi)
    1856           0 :                                 cpuhp_rollback_install(cpu, state, node);
    1857           0 :                         goto unlock;
    1858             :                 }
    1859             :         }
    1860           2 : add_node:
    1861          14 :         ret = 0;
    1862          14 :         hlist_add_head(node, &sp->list);
    1863          14 : unlock:
    1864          14 :         mutex_unlock(&cpuhp_state_mutex);
    1865          14 :         return ret;
    1866             : }
    1867             : 
    1868          14 : int __cpuhp_state_add_instance(enum cpuhp_state state, struct hlist_node *node,
    1869             :                                bool invoke)
    1870             : {
    1871          14 :         int ret;
    1872             : 
    1873          14 :         cpus_read_lock();
    1874          14 :         ret = __cpuhp_state_add_instance_cpuslocked(state, node, invoke);
    1875          14 :         cpus_read_unlock();
    1876          14 :         return ret;
    1877             : }
    1878             : EXPORT_SYMBOL_GPL(__cpuhp_state_add_instance);
    1879             : 
    1880             : /**
    1881             :  * __cpuhp_setup_state_cpuslocked - Setup the callbacks for an hotplug machine state
    1882             :  * @state:              The state to setup
    1883             :  * @invoke:             If true, the startup function is invoked for cpus where
    1884             :  *                      cpu state >= @state
    1885             :  * @startup:            startup callback function
    1886             :  * @teardown:           teardown callback function
    1887             :  * @multi_instance:     State is set up for multiple instances which get
    1888             :  *                      added afterwards.
    1889             :  *
    1890             :  * The caller needs to hold cpus read locked while calling this function.
    1891             :  * Returns:
    1892             :  *   On success:
    1893             :  *      Positive state number if @state is CPUHP_AP_ONLINE_DYN
    1894             :  *      0 for all other states
    1895             :  *   On failure: proper (negative) error code
    1896             :  */
    1897          30 : int __cpuhp_setup_state_cpuslocked(enum cpuhp_state state,
    1898             :                                    const char *name, bool invoke,
    1899             :                                    int (*startup)(unsigned int cpu),
    1900             :                                    int (*teardown)(unsigned int cpu),
    1901             :                                    bool multi_instance)
    1902             : {
    1903          30 :         int cpu, ret = 0;
    1904          30 :         bool dynstate;
    1905             : 
    1906          30 :         lockdep_assert_cpus_held();
    1907             : 
    1908          30 :         if (cpuhp_cb_check(state) || !name)
    1909             :                 return -EINVAL;
    1910             : 
    1911          30 :         mutex_lock(&cpuhp_state_mutex);
    1912             : 
    1913          30 :         ret = cpuhp_store_callbacks(state, name, startup, teardown,
    1914             :                                     multi_instance);
    1915             : 
    1916          30 :         dynstate = state == CPUHP_AP_ONLINE_DYN;
    1917          30 :         if (ret > 0 && dynstate) {
    1918           8 :                 state = ret;
    1919           8 :                 ret = 0;
    1920             :         }
    1921             : 
    1922          30 :         if (ret || !invoke || !startup)
    1923          22 :                 goto out;
    1924             : 
    1925             :         /*
    1926             :          * Try to call the startup callback for each present cpu
    1927             :          * depending on the hotplug state of the cpu.
    1928             :          */
    1929          40 :         for_each_present_cpu(cpu) {
    1930          32 :                 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
    1931          32 :                 int cpustate = st->state;
    1932             : 
    1933          32 :                 if (cpustate < state)
    1934          12 :                         continue;
    1935             : 
    1936          20 :                 ret = cpuhp_issue_call(cpu, state, true, NULL);
    1937          20 :                 if (ret) {
    1938           0 :                         if (teardown)
    1939           0 :                                 cpuhp_rollback_install(cpu, state, NULL);
    1940           0 :                         cpuhp_store_callbacks(state, NULL, NULL, NULL, false);
    1941           0 :                         goto out;
    1942             :                 }
    1943             :         }
    1944           8 : out:
    1945          30 :         mutex_unlock(&cpuhp_state_mutex);
    1946             :         /*
    1947             :          * If the requested state is CPUHP_AP_ONLINE_DYN, return the
    1948             :          * dynamically allocated state in case of success.
    1949             :          */
    1950          30 :         if (!ret && dynstate)
    1951           8 :                 return state;
    1952             :         return ret;
    1953             : }
    1954             : EXPORT_SYMBOL(__cpuhp_setup_state_cpuslocked);
    1955             : 
    1956          30 : int __cpuhp_setup_state(enum cpuhp_state state,
    1957             :                         const char *name, bool invoke,
    1958             :                         int (*startup)(unsigned int cpu),
    1959             :                         int (*teardown)(unsigned int cpu),
    1960             :                         bool multi_instance)
    1961             : {
    1962          30 :         int ret;
    1963             : 
    1964          30 :         cpus_read_lock();
    1965          30 :         ret = __cpuhp_setup_state_cpuslocked(state, name, invoke, startup,
    1966             :                                              teardown, multi_instance);
    1967          30 :         cpus_read_unlock();
    1968          30 :         return ret;
    1969             : }
    1970             : EXPORT_SYMBOL(__cpuhp_setup_state);
    1971             : 
    1972           0 : int __cpuhp_state_remove_instance(enum cpuhp_state state,
    1973             :                                   struct hlist_node *node, bool invoke)
    1974             : {
    1975           0 :         struct cpuhp_step *sp = cpuhp_get_step(state);
    1976           0 :         int cpu;
    1977             : 
    1978           0 :         BUG_ON(cpuhp_cb_check(state));
    1979             : 
    1980           0 :         if (!sp->multi_instance)
    1981             :                 return -EINVAL;
    1982             : 
    1983           0 :         cpus_read_lock();
    1984           0 :         mutex_lock(&cpuhp_state_mutex);
    1985             : 
    1986           0 :         if (!invoke || !cpuhp_get_teardown_cb(state))
    1987           0 :                 goto remove;
    1988             :         /*
    1989             :          * Call the teardown callback for each present cpu depending
    1990             :          * on the hotplug state of the cpu. This function is not
    1991             :          * allowed to fail currently!
    1992             :          */
    1993           0 :         for_each_present_cpu(cpu) {
    1994           0 :                 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
    1995           0 :                 int cpustate = st->state;
    1996             : 
    1997           0 :                 if (cpustate >= state)
    1998           0 :                         cpuhp_issue_call(cpu, state, false, node);
    1999             :         }
    2000             : 
    2001           0 : remove:
    2002           0 :         hlist_del(node);
    2003           0 :         mutex_unlock(&cpuhp_state_mutex);
    2004           0 :         cpus_read_unlock();
    2005             : 
    2006           0 :         return 0;
    2007             : }
    2008             : EXPORT_SYMBOL_GPL(__cpuhp_state_remove_instance);
    2009             : 
    2010             : /**
    2011             :  * __cpuhp_remove_state_cpuslocked - Remove the callbacks for an hotplug machine state
    2012             :  * @state:      The state to remove
    2013             :  * @invoke:     If true, the teardown function is invoked for cpus where
    2014             :  *              cpu state >= @state
    2015             :  *
    2016             :  * The caller needs to hold cpus read locked while calling this function.
    2017             :  * The teardown callback is currently not allowed to fail. Think
    2018             :  * about module removal!
    2019             :  */
    2020           0 : void __cpuhp_remove_state_cpuslocked(enum cpuhp_state state, bool invoke)
    2021             : {
    2022           0 :         struct cpuhp_step *sp = cpuhp_get_step(state);
    2023           0 :         int cpu;
    2024             : 
    2025           0 :         BUG_ON(cpuhp_cb_check(state));
    2026             : 
    2027           0 :         lockdep_assert_cpus_held();
    2028             : 
    2029           0 :         mutex_lock(&cpuhp_state_mutex);
    2030           0 :         if (sp->multi_instance) {
    2031           0 :                 WARN(!hlist_empty(&sp->list),
    2032             :                      "Error: Removing state %d which has instances left.\n",
    2033             :                      state);
    2034           0 :                 goto remove;
    2035             :         }
    2036             : 
    2037           0 :         if (!invoke || !cpuhp_get_teardown_cb(state))
    2038           0 :                 goto remove;
    2039             : 
    2040             :         /*
    2041             :          * Call the teardown callback for each present cpu depending
    2042             :          * on the hotplug state of the cpu. This function is not
    2043             :          * allowed to fail currently!
    2044             :          */
    2045           0 :         for_each_present_cpu(cpu) {
    2046           0 :                 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
    2047           0 :                 int cpustate = st->state;
    2048             : 
    2049           0 :                 if (cpustate >= state)
    2050           0 :                         cpuhp_issue_call(cpu, state, false, NULL);
    2051             :         }
    2052           0 : remove:
    2053           0 :         cpuhp_store_callbacks(state, NULL, NULL, NULL, false);
    2054           0 :         mutex_unlock(&cpuhp_state_mutex);
    2055           0 : }
    2056             : EXPORT_SYMBOL(__cpuhp_remove_state_cpuslocked);
    2057             : 
    2058           0 : void __cpuhp_remove_state(enum cpuhp_state state, bool invoke)
    2059             : {
    2060           0 :         cpus_read_lock();
    2061           0 :         __cpuhp_remove_state_cpuslocked(state, invoke);
    2062           0 :         cpus_read_unlock();
    2063           0 : }
    2064             : EXPORT_SYMBOL(__cpuhp_remove_state);
    2065             : 
    2066             : #ifdef CONFIG_HOTPLUG_SMT
    2067           0 : static void cpuhp_offline_cpu_device(unsigned int cpu)
    2068             : {
    2069           0 :         struct device *dev = get_cpu_device(cpu);
    2070             : 
    2071           0 :         dev->offline = true;
    2072             :         /* Tell user space about the state change */
    2073           0 :         kobject_uevent(&dev->kobj, KOBJ_OFFLINE);
    2074           0 : }
    2075             : 
    2076           0 : static void cpuhp_online_cpu_device(unsigned int cpu)
    2077             : {
    2078           0 :         struct device *dev = get_cpu_device(cpu);
    2079             : 
    2080           0 :         dev->offline = false;
    2081             :         /* Tell user space about the state change */
    2082           0 :         kobject_uevent(&dev->kobj, KOBJ_ONLINE);
    2083           0 : }
    2084             : 
    2085           0 : int cpuhp_smt_disable(enum cpuhp_smt_control ctrlval)
    2086             : {
    2087           0 :         int cpu, ret = 0;
    2088             : 
    2089           0 :         cpu_maps_update_begin();
    2090           0 :         for_each_online_cpu(cpu) {
    2091           0 :                 if (topology_is_primary_thread(cpu))
    2092           0 :                         continue;
    2093           0 :                 ret = cpu_down_maps_locked(cpu, CPUHP_OFFLINE);
    2094           0 :                 if (ret)
    2095             :                         break;
    2096             :                 /*
    2097             :                  * As this needs to hold the cpu maps lock it's impossible
    2098             :                  * to call device_offline() because that ends up calling
    2099             :                  * cpu_down() which takes cpu maps lock. cpu maps lock
    2100             :                  * needs to be held as this might race against in kernel
    2101             :                  * abusers of the hotplug machinery (thermal management).
    2102             :                  *
    2103             :                  * So nothing would update device:offline state. That would
    2104             :                  * leave the sysfs entry stale and prevent onlining after
    2105             :                  * smt control has been changed to 'off' again. This is
    2106             :                  * called under the sysfs hotplug lock, so it is properly
    2107             :                  * serialized against the regular offline usage.
    2108             :                  */
    2109           0 :                 cpuhp_offline_cpu_device(cpu);
    2110             :         }
    2111           0 :         if (!ret)
    2112           0 :                 cpu_smt_control = ctrlval;
    2113           0 :         cpu_maps_update_done();
    2114           0 :         return ret;
    2115             : }
    2116             : 
    2117           0 : int cpuhp_smt_enable(void)
    2118             : {
    2119           0 :         int cpu, ret = 0;
    2120             : 
    2121           0 :         cpu_maps_update_begin();
    2122           0 :         cpu_smt_control = CPU_SMT_ENABLED;
    2123           0 :         for_each_present_cpu(cpu) {
    2124             :                 /* Skip online CPUs and CPUs on offline nodes */
    2125           0 :                 if (cpu_online(cpu) || !node_online(cpu_to_node(cpu)))
    2126           0 :                         continue;
    2127           0 :                 ret = _cpu_up(cpu, 0, CPUHP_ONLINE);
    2128           0 :                 if (ret)
    2129             :                         break;
    2130             :                 /* See comment in cpuhp_smt_disable() */
    2131           0 :                 cpuhp_online_cpu_device(cpu);
    2132             :         }
    2133           0 :         cpu_maps_update_done();
    2134           0 :         return ret;
    2135             : }
    2136             : #endif
    2137             : 
    2138             : #if defined(CONFIG_SYSFS) && defined(CONFIG_HOTPLUG_CPU)
    2139           0 : static ssize_t show_cpuhp_state(struct device *dev,
    2140             :                                 struct device_attribute *attr, char *buf)
    2141             : {
    2142           0 :         struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id);
    2143             : 
    2144           0 :         return sprintf(buf, "%d\n", st->state);
    2145             : }
    2146             : static DEVICE_ATTR(state, 0444, show_cpuhp_state, NULL);
    2147             : 
    2148           0 : static ssize_t write_cpuhp_target(struct device *dev,
    2149             :                                   struct device_attribute *attr,
    2150             :                                   const char *buf, size_t count)
    2151             : {
    2152           0 :         struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id);
    2153           0 :         struct cpuhp_step *sp;
    2154           0 :         int target, ret;
    2155             : 
    2156           0 :         ret = kstrtoint(buf, 10, &target);
    2157           0 :         if (ret)
    2158           0 :                 return ret;
    2159             : 
    2160             : #ifdef CONFIG_CPU_HOTPLUG_STATE_CONTROL
    2161             :         if (target < CPUHP_OFFLINE || target > CPUHP_ONLINE)
    2162             :                 return -EINVAL;
    2163             : #else
    2164           0 :         if (target != CPUHP_OFFLINE && target != CPUHP_ONLINE)
    2165             :                 return -EINVAL;
    2166             : #endif
    2167             : 
    2168           0 :         ret = lock_device_hotplug_sysfs();
    2169           0 :         if (ret)
    2170           0 :                 return ret;
    2171             : 
    2172           0 :         mutex_lock(&cpuhp_state_mutex);
    2173           0 :         sp = cpuhp_get_step(target);
    2174           0 :         ret = !sp->name || sp->cant_stop ? -EINVAL : 0;
    2175           0 :         mutex_unlock(&cpuhp_state_mutex);
    2176           0 :         if (ret)
    2177           0 :                 goto out;
    2178             : 
    2179           0 :         if (st->state < target)
    2180           0 :                 ret = cpu_up(dev->id, target);
    2181             :         else
    2182           0 :                 ret = cpu_down(dev->id, target);
    2183           0 : out:
    2184           0 :         unlock_device_hotplug();
    2185           0 :         return ret ? ret : count;
    2186             : }
    2187             : 
    2188           0 : static ssize_t show_cpuhp_target(struct device *dev,
    2189             :                                  struct device_attribute *attr, char *buf)
    2190             : {
    2191           0 :         struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id);
    2192             : 
    2193           0 :         return sprintf(buf, "%d\n", st->target);
    2194             : }
    2195             : static DEVICE_ATTR(target, 0644, show_cpuhp_target, write_cpuhp_target);
    2196             : 
    2197             : 
    2198           0 : static ssize_t write_cpuhp_fail(struct device *dev,
    2199             :                                 struct device_attribute *attr,
    2200             :                                 const char *buf, size_t count)
    2201             : {
    2202           0 :         struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id);
    2203           0 :         struct cpuhp_step *sp;
    2204           0 :         int fail, ret;
    2205             : 
    2206           0 :         ret = kstrtoint(buf, 10, &fail);
    2207           0 :         if (ret)
    2208           0 :                 return ret;
    2209             : 
    2210           0 :         if (fail < CPUHP_OFFLINE || fail > CPUHP_ONLINE)
    2211             :                 return -EINVAL;
    2212             : 
    2213             :         /*
    2214             :          * Cannot fail STARTING/DYING callbacks.
    2215             :          */
    2216           0 :         if (cpuhp_is_atomic_state(fail))
    2217             :                 return -EINVAL;
    2218             : 
    2219             :         /*
    2220             :          * Cannot fail anything that doesn't have callbacks.
    2221             :          */
    2222           0 :         mutex_lock(&cpuhp_state_mutex);
    2223           0 :         sp = cpuhp_get_step(fail);
    2224           0 :         if (!sp->startup.single && !sp->teardown.single)
    2225           0 :                 ret = -EINVAL;
    2226           0 :         mutex_unlock(&cpuhp_state_mutex);
    2227           0 :         if (ret)
    2228           0 :                 return ret;
    2229             : 
    2230           0 :         st->fail = fail;
    2231             : 
    2232           0 :         return count;
    2233             : }
    2234             : 
    2235           0 : static ssize_t show_cpuhp_fail(struct device *dev,
    2236             :                                struct device_attribute *attr, char *buf)
    2237             : {
    2238           0 :         struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id);
    2239             : 
    2240           0 :         return sprintf(buf, "%d\n", st->fail);
    2241             : }
    2242             : 
    2243             : static DEVICE_ATTR(fail, 0644, show_cpuhp_fail, write_cpuhp_fail);
    2244             : 
    2245             : static struct attribute *cpuhp_cpu_attrs[] = {
    2246             :         &dev_attr_state.attr,
    2247             :         &dev_attr_target.attr,
    2248             :         &dev_attr_fail.attr,
    2249             :         NULL
    2250             : };
    2251             : 
    2252             : static const struct attribute_group cpuhp_cpu_attr_group = {
    2253             :         .attrs = cpuhp_cpu_attrs,
    2254             :         .name = "hotplug",
    2255             :         NULL
    2256             : };
    2257             : 
    2258           0 : static ssize_t show_cpuhp_states(struct device *dev,
    2259             :                                  struct device_attribute *attr, char *buf)
    2260             : {
    2261           0 :         ssize_t cur, res = 0;
    2262           0 :         int i;
    2263             : 
    2264           0 :         mutex_lock(&cpuhp_state_mutex);
    2265           0 :         for (i = CPUHP_OFFLINE; i <= CPUHP_ONLINE; i++) {
    2266           0 :                 struct cpuhp_step *sp = cpuhp_get_step(i);
    2267             : 
    2268           0 :                 if (sp->name) {
    2269           0 :                         cur = sprintf(buf, "%3d: %s\n", i, sp->name);
    2270           0 :                         buf += cur;
    2271           0 :                         res += cur;
    2272             :                 }
    2273             :         }
    2274           0 :         mutex_unlock(&cpuhp_state_mutex);
    2275           0 :         return res;
    2276             : }
    2277             : static DEVICE_ATTR(states, 0444, show_cpuhp_states, NULL);
    2278             : 
    2279             : static struct attribute *cpuhp_cpu_root_attrs[] = {
    2280             :         &dev_attr_states.attr,
    2281             :         NULL
    2282             : };
    2283             : 
    2284             : static const struct attribute_group cpuhp_cpu_root_attr_group = {
    2285             :         .attrs = cpuhp_cpu_root_attrs,
    2286             :         .name = "hotplug",
    2287             :         NULL
    2288             : };
    2289             : 
    2290             : #ifdef CONFIG_HOTPLUG_SMT
    2291             : 
    2292             : static ssize_t
    2293           0 : __store_smt_control(struct device *dev, struct device_attribute *attr,
    2294             :                     const char *buf, size_t count)
    2295             : {
    2296           0 :         int ctrlval, ret;
    2297             : 
    2298           0 :         if (sysfs_streq(buf, "on"))
    2299             :                 ctrlval = CPU_SMT_ENABLED;
    2300           0 :         else if (sysfs_streq(buf, "off"))
    2301             :                 ctrlval = CPU_SMT_DISABLED;
    2302           0 :         else if (sysfs_streq(buf, "forceoff"))
    2303             :                 ctrlval = CPU_SMT_FORCE_DISABLED;
    2304             :         else
    2305             :                 return -EINVAL;
    2306             : 
    2307           0 :         if (cpu_smt_control == CPU_SMT_FORCE_DISABLED)
    2308             :                 return -EPERM;
    2309             : 
    2310           0 :         if (cpu_smt_control == CPU_SMT_NOT_SUPPORTED)
    2311             :                 return -ENODEV;
    2312             : 
    2313           0 :         ret = lock_device_hotplug_sysfs();
    2314           0 :         if (ret)
    2315           0 :                 return ret;
    2316             : 
    2317           0 :         if (ctrlval != cpu_smt_control) {
    2318           0 :                 switch (ctrlval) {
    2319           0 :                 case CPU_SMT_ENABLED:
    2320           0 :                         ret = cpuhp_smt_enable();
    2321           0 :                         break;
    2322           0 :                 case CPU_SMT_DISABLED:
    2323             :                 case CPU_SMT_FORCE_DISABLED:
    2324           0 :                         ret = cpuhp_smt_disable(ctrlval);
    2325           0 :                         break;
    2326             :                 }
    2327           0 :         }
    2328             : 
    2329           0 :         unlock_device_hotplug();
    2330           0 :         return ret ? ret : count;
    2331             : }
    2332             : 
    2333             : #else /* !CONFIG_HOTPLUG_SMT */
    2334             : static ssize_t
    2335             : __store_smt_control(struct device *dev, struct device_attribute *attr,
    2336             :                     const char *buf, size_t count)
    2337             : {
    2338             :         return -ENODEV;
    2339             : }
    2340             : #endif /* CONFIG_HOTPLUG_SMT */
    2341             : 
    2342             : static const char *smt_states[] = {
    2343             :         [CPU_SMT_ENABLED]               = "on",
    2344             :         [CPU_SMT_DISABLED]              = "off",
    2345             :         [CPU_SMT_FORCE_DISABLED]        = "forceoff",
    2346             :         [CPU_SMT_NOT_SUPPORTED]         = "notsupported",
    2347             :         [CPU_SMT_NOT_IMPLEMENTED]       = "notimplemented",
    2348             : };
    2349             : 
    2350             : static ssize_t
    2351           0 : show_smt_control(struct device *dev, struct device_attribute *attr, char *buf)
    2352             : {
    2353           0 :         const char *state = smt_states[cpu_smt_control];
    2354             : 
    2355           0 :         return snprintf(buf, PAGE_SIZE - 2, "%s\n", state);
    2356             : }
    2357             : 
    2358             : static ssize_t
    2359           0 : store_smt_control(struct device *dev, struct device_attribute *attr,
    2360             :                   const char *buf, size_t count)
    2361             : {
    2362           0 :         return __store_smt_control(dev, attr, buf, count);
    2363             : }
    2364             : static DEVICE_ATTR(control, 0644, show_smt_control, store_smt_control);
    2365             : 
    2366             : static ssize_t
    2367           0 : show_smt_active(struct device *dev, struct device_attribute *attr, char *buf)
    2368             : {
    2369           0 :         return snprintf(buf, PAGE_SIZE - 2, "%d\n", sched_smt_active());
    2370             : }
    2371             : static DEVICE_ATTR(active, 0444, show_smt_active, NULL);
    2372             : 
    2373             : static struct attribute *cpuhp_smt_attrs[] = {
    2374             :         &dev_attr_control.attr,
    2375             :         &dev_attr_active.attr,
    2376             :         NULL
    2377             : };
    2378             : 
    2379             : static const struct attribute_group cpuhp_smt_attr_group = {
    2380             :         .attrs = cpuhp_smt_attrs,
    2381             :         .name = "smt",
    2382             :         NULL
    2383             : };
    2384             : 
    2385           1 : static int __init cpu_smt_sysfs_init(void)
    2386             : {
    2387           1 :         return sysfs_create_group(&cpu_subsys.dev_root->kobj,
    2388             :                                   &cpuhp_smt_attr_group);
    2389             : }
    2390             : 
    2391           1 : static int __init cpuhp_sysfs_init(void)
    2392             : {
    2393           1 :         int cpu, ret;
    2394             : 
    2395           1 :         ret = cpu_smt_sysfs_init();
    2396           1 :         if (ret)
    2397             :                 return ret;
    2398             : 
    2399           1 :         ret = sysfs_create_group(&cpu_subsys.dev_root->kobj,
    2400             :                                  &cpuhp_cpu_root_attr_group);
    2401           1 :         if (ret)
    2402             :                 return ret;
    2403             : 
    2404           5 :         for_each_possible_cpu(cpu) {
    2405           4 :                 struct device *dev = get_cpu_device(cpu);
    2406             : 
    2407           4 :                 if (!dev)
    2408           0 :                         continue;
    2409           4 :                 ret = sysfs_create_group(&dev->kobj, &cpuhp_cpu_attr_group);
    2410           4 :                 if (ret)
    2411           0 :                         return ret;
    2412             :         }
    2413             :         return 0;
    2414             : }
    2415             : device_initcall(cpuhp_sysfs_init);
    2416             : #endif /* CONFIG_SYSFS && CONFIG_HOTPLUG_CPU */
    2417             : 
    2418             : /*
    2419             :  * cpu_bit_bitmap[] is a special, "compressed" data structure that
    2420             :  * represents all NR_CPUS bits binary values of 1<<nr.
    2421             :  *
    2422             :  * It is used by cpumask_of() to get a constant address to a CPU
    2423             :  * mask value that has a single bit set only.
    2424             :  */
    2425             : 
    2426             : /* cpu_bit_bitmap[0] is empty - so we can back into it */
    2427             : #define MASK_DECLARE_1(x)       [x+1][0] = (1UL << (x))
    2428             : #define MASK_DECLARE_2(x)       MASK_DECLARE_1(x), MASK_DECLARE_1(x+1)
    2429             : #define MASK_DECLARE_4(x)       MASK_DECLARE_2(x), MASK_DECLARE_2(x+2)
    2430             : #define MASK_DECLARE_8(x)       MASK_DECLARE_4(x), MASK_DECLARE_4(x+4)
    2431             : 
    2432             : const unsigned long cpu_bit_bitmap[BITS_PER_LONG+1][BITS_TO_LONGS(NR_CPUS)] = {
    2433             : 
    2434             :         MASK_DECLARE_8(0),      MASK_DECLARE_8(8),
    2435             :         MASK_DECLARE_8(16),     MASK_DECLARE_8(24),
    2436             : #if BITS_PER_LONG > 32
    2437             :         MASK_DECLARE_8(32),     MASK_DECLARE_8(40),
    2438             :         MASK_DECLARE_8(48),     MASK_DECLARE_8(56),
    2439             : #endif
    2440             : };
    2441             : EXPORT_SYMBOL_GPL(cpu_bit_bitmap);
    2442             : 
    2443             : const DECLARE_BITMAP(cpu_all_bits, NR_CPUS) = CPU_BITS_ALL;
    2444             : EXPORT_SYMBOL(cpu_all_bits);
    2445             : 
    2446             : #ifdef CONFIG_INIT_ALL_POSSIBLE
    2447             : struct cpumask __cpu_possible_mask __read_mostly
    2448             :         = {CPU_BITS_ALL};
    2449             : #else
    2450             : struct cpumask __cpu_possible_mask __read_mostly;
    2451             : #endif
    2452             : EXPORT_SYMBOL(__cpu_possible_mask);
    2453             : 
    2454             : struct cpumask __cpu_online_mask __read_mostly;
    2455             : EXPORT_SYMBOL(__cpu_online_mask);
    2456             : 
    2457             : struct cpumask __cpu_present_mask __read_mostly;
    2458             : EXPORT_SYMBOL(__cpu_present_mask);
    2459             : 
    2460             : struct cpumask __cpu_active_mask __read_mostly;
    2461             : EXPORT_SYMBOL(__cpu_active_mask);
    2462             : 
    2463             : atomic_t __num_online_cpus __read_mostly;
    2464             : EXPORT_SYMBOL(__num_online_cpus);
    2465             : 
    2466           0 : void init_cpu_present(const struct cpumask *src)
    2467             : {
    2468           0 :         cpumask_copy(&__cpu_present_mask, src);
    2469           0 : }
    2470             : 
    2471           0 : void init_cpu_possible(const struct cpumask *src)
    2472             : {
    2473           0 :         cpumask_copy(&__cpu_possible_mask, src);
    2474           0 : }
    2475             : 
    2476           0 : void init_cpu_online(const struct cpumask *src)
    2477             : {
    2478           0 :         cpumask_copy(&__cpu_online_mask, src);
    2479           0 : }
    2480             : 
    2481           4 : void set_cpu_online(unsigned int cpu, bool online)
    2482             : {
    2483             :         /*
    2484             :          * atomic_inc/dec() is required to handle the horrid abuse of this
    2485             :          * function by the reboot and kexec code which invoke it from
    2486             :          * IPI/NMI broadcasts when shutting down CPUs. Invocation from
    2487             :          * regular CPU hotplug is properly serialized.
    2488             :          *
    2489             :          * Note, that the fact that __num_online_cpus is of type atomic_t
    2490             :          * does not protect readers which are not serialized against
    2491             :          * concurrent hotplug operations.
    2492             :          */
    2493           4 :         if (online) {
    2494           4 :                 if (!cpumask_test_and_set_cpu(cpu, &__cpu_online_mask))
    2495           4 :                         atomic_inc(&__num_online_cpus);
    2496             :         } else {
    2497           0 :                 if (cpumask_test_and_clear_cpu(cpu, &__cpu_online_mask))
    2498           0 :                         atomic_dec(&__num_online_cpus);
    2499             :         }
    2500           4 : }
    2501             : 
    2502             : /*
    2503             :  * Activate the first processor.
    2504             :  */
    2505           1 : void __init boot_cpu_init(void)
    2506             : {
    2507           1 :         int cpu = smp_processor_id();
    2508             : 
    2509             :         /* Mark the boot cpu "present", "online" etc for SMP and UP case */
    2510           1 :         set_cpu_online(cpu, true);
    2511           1 :         set_cpu_active(cpu, true);
    2512           1 :         set_cpu_present(cpu, true);
    2513           1 :         set_cpu_possible(cpu, true);
    2514             : 
    2515             : #ifdef CONFIG_SMP
    2516           1 :         __boot_cpu_id = cpu;
    2517             : #endif
    2518           1 : }
    2519             : 
    2520             : /*
    2521             :  * Must be called _AFTER_ setting up the per_cpu areas
    2522             :  */
    2523           1 : void __init boot_cpu_hotplug_init(void)
    2524             : {
    2525             : #ifdef CONFIG_SMP
    2526           1 :         cpumask_set_cpu(smp_processor_id(), &cpus_booted_once_mask);
    2527             : #endif
    2528           1 :         this_cpu_write(cpuhp_state.state, CPUHP_ONLINE);
    2529           1 : }
    2530             : 
    2531             : /*
    2532             :  * These are used for a global "mitigations=" cmdline option for toggling
    2533             :  * optional CPU mitigations.
    2534             :  */
    2535             : enum cpu_mitigations {
    2536             :         CPU_MITIGATIONS_OFF,
    2537             :         CPU_MITIGATIONS_AUTO,
    2538             :         CPU_MITIGATIONS_AUTO_NOSMT,
    2539             : };
    2540             : 
    2541             : static enum cpu_mitigations cpu_mitigations __ro_after_init =
    2542             :         CPU_MITIGATIONS_AUTO;
    2543             : 
    2544           0 : static int __init mitigations_parse_cmdline(char *arg)
    2545             : {
    2546           0 :         if (!strcmp(arg, "off"))
    2547           0 :                 cpu_mitigations = CPU_MITIGATIONS_OFF;
    2548           0 :         else if (!strcmp(arg, "auto"))
    2549           0 :                 cpu_mitigations = CPU_MITIGATIONS_AUTO;
    2550           0 :         else if (!strcmp(arg, "auto,nosmt"))
    2551           0 :                 cpu_mitigations = CPU_MITIGATIONS_AUTO_NOSMT;
    2552             :         else
    2553           0 :                 pr_crit("Unsupported mitigations=%s, system may still be vulnerable\n",
    2554             :                         arg);
    2555             : 
    2556           0 :         return 0;
    2557             : }
    2558             : early_param("mitigations", mitigations_parse_cmdline);
    2559             : 
    2560             : /* mitigations=off */
    2561           6 : bool cpu_mitigations_off(void)
    2562             : {
    2563           6 :         return cpu_mitigations == CPU_MITIGATIONS_OFF;
    2564             : }
    2565             : EXPORT_SYMBOL_GPL(cpu_mitigations_off);
    2566             : 
    2567             : /* mitigations=auto,nosmt */
    2568           2 : bool cpu_mitigations_auto_nosmt(void)
    2569             : {
    2570           2 :         return cpu_mitigations == CPU_MITIGATIONS_AUTO_NOSMT;
    2571             : }
    2572             : EXPORT_SYMBOL_GPL(cpu_mitigations_auto_nosmt);

Generated by: LCOV version 1.14