LCOV - code coverage report
Current view: top level - arch/x86/kernel/cpu - umwait.c (source / functions) Hit Total Coverage
Test: landlock.info Lines: 4 68 5.9 %
Date: 2021-04-22 12:43:58 Functions: 1 10 10.0 %

          Line data    Source code
       1             : // SPDX-License-Identifier: GPL-2.0
       2             : #include <linux/syscore_ops.h>
       3             : #include <linux/suspend.h>
       4             : #include <linux/cpu.h>
       5             : 
       6             : #include <asm/msr.h>
       7             : #include <asm/mwait.h>
       8             : 
       9             : #define UMWAIT_C02_ENABLE       0
      10             : 
      11             : #define UMWAIT_CTRL_VAL(max_time, c02_disable)                          \
      12             :         (((max_time) & MSR_IA32_UMWAIT_CONTROL_TIME_MASK) |         \
      13             :         ((c02_disable) & MSR_IA32_UMWAIT_CONTROL_C02_DISABLE))
      14             : 
      15             : /*
      16             :  * Cache IA32_UMWAIT_CONTROL MSR. This is a systemwide control. By default,
      17             :  * umwait max time is 100000 in TSC-quanta and C0.2 is enabled
      18             :  */
      19             : static u32 umwait_control_cached = UMWAIT_CTRL_VAL(100000, UMWAIT_C02_ENABLE);
      20             : 
      21             : /*
      22             :  * Cache the original IA32_UMWAIT_CONTROL MSR value which is configured by
      23             :  * hardware or BIOS before kernel boot.
      24             :  */
      25             : static u32 orig_umwait_control_cached __ro_after_init;
      26             : 
      27             : /*
      28             :  * Serialize access to umwait_control_cached and IA32_UMWAIT_CONTROL MSR in
      29             :  * the sysfs write functions.
      30             :  */
      31             : static DEFINE_MUTEX(umwait_lock);
      32             : 
      33           0 : static void umwait_update_control_msr(void * unused)
      34             : {
      35           0 :         lockdep_assert_irqs_disabled();
      36           0 :         wrmsr(MSR_IA32_UMWAIT_CONTROL, READ_ONCE(umwait_control_cached), 0);
      37           0 : }
      38             : 
      39             : /*
      40             :  * The CPU hotplug callback sets the control MSR to the global control
      41             :  * value.
      42             :  *
      43             :  * Disable interrupts so the read of umwait_control_cached and the WRMSR
      44             :  * are protected against a concurrent sysfs write. Otherwise the sysfs
      45             :  * write could update the cached value after it had been read on this CPU
      46             :  * and issue the IPI before the old value had been written. The IPI would
      47             :  * interrupt, write the new value and after return from IPI the previous
      48             :  * value would be written by this CPU.
      49             :  *
      50             :  * With interrupts disabled the upcoming CPU either sees the new control
      51             :  * value or the IPI is updating this CPU to the new control value after
      52             :  * interrupts have been reenabled.
      53             :  */
      54           0 : static int umwait_cpu_online(unsigned int cpu)
      55             : {
      56           0 :         local_irq_disable();
      57           0 :         umwait_update_control_msr(NULL);
      58           0 :         local_irq_enable();
      59           0 :         return 0;
      60             : }
      61             : 
      62             : /*
      63             :  * The CPU hotplug callback sets the control MSR to the original control
      64             :  * value.
      65             :  */
      66           0 : static int umwait_cpu_offline(unsigned int cpu)
      67             : {
      68             :         /*
      69             :          * This code is protected by the CPU hotplug already and
      70             :          * orig_umwait_control_cached is never changed after it caches
      71             :          * the original control MSR value in umwait_init(). So there
      72             :          * is no race condition here.
      73             :          */
      74           0 :         wrmsr(MSR_IA32_UMWAIT_CONTROL, orig_umwait_control_cached, 0);
      75             : 
      76           0 :         return 0;
      77             : }
      78             : 
      79             : /*
      80             :  * On resume, restore IA32_UMWAIT_CONTROL MSR on the boot processor which
      81             :  * is the only active CPU at this time. The MSR is set up on the APs via the
      82             :  * CPU hotplug callback.
      83             :  *
      84             :  * This function is invoked on resume from suspend and hibernation. On
      85             :  * resume from suspend the restore should be not required, but we neither
      86             :  * trust the firmware nor does it matter if the same value is written
      87             :  * again.
      88             :  */
      89           0 : static void umwait_syscore_resume(void)
      90             : {
      91           0 :         umwait_update_control_msr(NULL);
      92           0 : }
      93             : 
      94             : static struct syscore_ops umwait_syscore_ops = {
      95             :         .resume = umwait_syscore_resume,
      96             : };
      97             : 
      98             : /* sysfs interface */
      99             : 
     100             : /*
     101             :  * When bit 0 in IA32_UMWAIT_CONTROL MSR is 1, C0.2 is disabled.
     102             :  * Otherwise, C0.2 is enabled.
     103             :  */
     104           0 : static inline bool umwait_ctrl_c02_enabled(u32 ctrl)
     105             : {
     106           0 :         return !(ctrl & MSR_IA32_UMWAIT_CONTROL_C02_DISABLE);
     107             : }
     108             : 
     109           0 : static inline u32 umwait_ctrl_max_time(u32 ctrl)
     110             : {
     111           0 :         return ctrl & MSR_IA32_UMWAIT_CONTROL_TIME_MASK;
     112             : }
     113             : 
     114           0 : static inline void umwait_update_control(u32 maxtime, bool c02_enable)
     115             : {
     116           0 :         u32 ctrl = maxtime & MSR_IA32_UMWAIT_CONTROL_TIME_MASK;
     117             : 
     118           0 :         if (!c02_enable)
     119           0 :                 ctrl |= MSR_IA32_UMWAIT_CONTROL_C02_DISABLE;
     120             : 
     121           0 :         WRITE_ONCE(umwait_control_cached, ctrl);
     122             :         /* Propagate to all CPUs */
     123           0 :         on_each_cpu(umwait_update_control_msr, NULL, 1);
     124           0 : }
     125             : 
     126             : static ssize_t
     127           0 : enable_c02_show(struct device *dev, struct device_attribute *attr, char *buf)
     128             : {
     129           0 :         u32 ctrl = READ_ONCE(umwait_control_cached);
     130             : 
     131           0 :         return sprintf(buf, "%d\n", umwait_ctrl_c02_enabled(ctrl));
     132             : }
     133             : 
     134           0 : static ssize_t enable_c02_store(struct device *dev,
     135             :                                 struct device_attribute *attr,
     136             :                                 const char *buf, size_t count)
     137             : {
     138           0 :         bool c02_enable;
     139           0 :         u32 ctrl;
     140           0 :         int ret;
     141             : 
     142           0 :         ret = kstrtobool(buf, &c02_enable);
     143           0 :         if (ret)
     144           0 :                 return ret;
     145             : 
     146           0 :         mutex_lock(&umwait_lock);
     147             : 
     148           0 :         ctrl = READ_ONCE(umwait_control_cached);
     149           0 :         if (c02_enable != umwait_ctrl_c02_enabled(ctrl))
     150           0 :                 umwait_update_control(ctrl, c02_enable);
     151             : 
     152           0 :         mutex_unlock(&umwait_lock);
     153             : 
     154           0 :         return count;
     155             : }
     156             : static DEVICE_ATTR_RW(enable_c02);
     157             : 
     158             : static ssize_t
     159           0 : max_time_show(struct device *kobj, struct device_attribute *attr, char *buf)
     160             : {
     161           0 :         u32 ctrl = READ_ONCE(umwait_control_cached);
     162             : 
     163           0 :         return sprintf(buf, "%u\n", umwait_ctrl_max_time(ctrl));
     164             : }
     165             : 
     166           0 : static ssize_t max_time_store(struct device *kobj,
     167             :                               struct device_attribute *attr,
     168             :                               const char *buf, size_t count)
     169             : {
     170           0 :         u32 max_time, ctrl;
     171           0 :         int ret;
     172             : 
     173           0 :         ret = kstrtou32(buf, 0, &max_time);
     174           0 :         if (ret)
     175           0 :                 return ret;
     176             : 
     177             :         /* bits[1:0] must be zero */
     178           0 :         if (max_time & ~MSR_IA32_UMWAIT_CONTROL_TIME_MASK)
     179             :                 return -EINVAL;
     180             : 
     181           0 :         mutex_lock(&umwait_lock);
     182             : 
     183           0 :         ctrl = READ_ONCE(umwait_control_cached);
     184           0 :         if (max_time != umwait_ctrl_max_time(ctrl))
     185           0 :                 umwait_update_control(max_time, umwait_ctrl_c02_enabled(ctrl));
     186             : 
     187           0 :         mutex_unlock(&umwait_lock);
     188             : 
     189           0 :         return count;
     190             : }
     191             : static DEVICE_ATTR_RW(max_time);
     192             : 
     193             : static struct attribute *umwait_attrs[] = {
     194             :         &dev_attr_enable_c02.attr,
     195             :         &dev_attr_max_time.attr,
     196             :         NULL
     197             : };
     198             : 
     199             : static struct attribute_group umwait_attr_group = {
     200             :         .attrs = umwait_attrs,
     201             :         .name = "umwait_control",
     202             : };
     203             : 
     204           1 : static int __init umwait_init(void)
     205             : {
     206           1 :         struct device *dev;
     207           1 :         int ret;
     208             : 
     209           1 :         if (!boot_cpu_has(X86_FEATURE_WAITPKG))
     210             :                 return -ENODEV;
     211             : 
     212             :         /*
     213             :          * Cache the original control MSR value before the control MSR is
     214             :          * changed. This is the only place where orig_umwait_control_cached
     215             :          * is modified.
     216             :          */
     217           0 :         rdmsrl(MSR_IA32_UMWAIT_CONTROL, orig_umwait_control_cached);
     218             : 
     219           0 :         ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "umwait:online",
     220             :                                 umwait_cpu_online, umwait_cpu_offline);
     221           0 :         if (ret < 0) {
     222             :                 /*
     223             :                  * On failure, the control MSR on all CPUs has the
     224             :                  * original control value.
     225             :                  */
     226             :                 return ret;
     227             :         }
     228             : 
     229           0 :         register_syscore_ops(&umwait_syscore_ops);
     230             : 
     231             :         /*
     232             :          * Add umwait control interface. Ignore failure, so at least the
     233             :          * default values are set up in case the machine manages to boot.
     234             :          */
     235           0 :         dev = cpu_subsys.dev_root;
     236           0 :         return sysfs_create_group(&dev->kobj, &umwait_attr_group);
     237             : }
     238             : device_initcall(umwait_init);

Generated by: LCOV version 1.14