Line data Source code
1 : // SPDX-License-Identifier: GPL-2.0
2 : #include <linux/syscore_ops.h>
3 : #include <linux/suspend.h>
4 : #include <linux/cpu.h>
5 :
6 : #include <asm/msr.h>
7 : #include <asm/mwait.h>
8 :
9 : #define UMWAIT_C02_ENABLE 0
10 :
11 : #define UMWAIT_CTRL_VAL(max_time, c02_disable) \
12 : (((max_time) & MSR_IA32_UMWAIT_CONTROL_TIME_MASK) | \
13 : ((c02_disable) & MSR_IA32_UMWAIT_CONTROL_C02_DISABLE))
14 :
15 : /*
16 : * Cache IA32_UMWAIT_CONTROL MSR. This is a systemwide control. By default,
17 : * umwait max time is 100000 in TSC-quanta and C0.2 is enabled
18 : */
19 : static u32 umwait_control_cached = UMWAIT_CTRL_VAL(100000, UMWAIT_C02_ENABLE);
20 :
21 : /*
22 : * Cache the original IA32_UMWAIT_CONTROL MSR value which is configured by
23 : * hardware or BIOS before kernel boot.
24 : */
25 : static u32 orig_umwait_control_cached __ro_after_init;
26 :
27 : /*
28 : * Serialize access to umwait_control_cached and IA32_UMWAIT_CONTROL MSR in
29 : * the sysfs write functions.
30 : */
31 : static DEFINE_MUTEX(umwait_lock);
32 :
33 0 : static void umwait_update_control_msr(void * unused)
34 : {
35 0 : lockdep_assert_irqs_disabled();
36 0 : wrmsr(MSR_IA32_UMWAIT_CONTROL, READ_ONCE(umwait_control_cached), 0);
37 0 : }
38 :
39 : /*
40 : * The CPU hotplug callback sets the control MSR to the global control
41 : * value.
42 : *
43 : * Disable interrupts so the read of umwait_control_cached and the WRMSR
44 : * are protected against a concurrent sysfs write. Otherwise the sysfs
45 : * write could update the cached value after it had been read on this CPU
46 : * and issue the IPI before the old value had been written. The IPI would
47 : * interrupt, write the new value and after return from IPI the previous
48 : * value would be written by this CPU.
49 : *
50 : * With interrupts disabled the upcoming CPU either sees the new control
51 : * value or the IPI is updating this CPU to the new control value after
52 : * interrupts have been reenabled.
53 : */
54 0 : static int umwait_cpu_online(unsigned int cpu)
55 : {
56 0 : local_irq_disable();
57 0 : umwait_update_control_msr(NULL);
58 0 : local_irq_enable();
59 0 : return 0;
60 : }
61 :
62 : /*
63 : * The CPU hotplug callback sets the control MSR to the original control
64 : * value.
65 : */
66 0 : static int umwait_cpu_offline(unsigned int cpu)
67 : {
68 : /*
69 : * This code is protected by the CPU hotplug already and
70 : * orig_umwait_control_cached is never changed after it caches
71 : * the original control MSR value in umwait_init(). So there
72 : * is no race condition here.
73 : */
74 0 : wrmsr(MSR_IA32_UMWAIT_CONTROL, orig_umwait_control_cached, 0);
75 :
76 0 : return 0;
77 : }
78 :
79 : /*
80 : * On resume, restore IA32_UMWAIT_CONTROL MSR on the boot processor which
81 : * is the only active CPU at this time. The MSR is set up on the APs via the
82 : * CPU hotplug callback.
83 : *
84 : * This function is invoked on resume from suspend and hibernation. On
85 : * resume from suspend the restore should be not required, but we neither
86 : * trust the firmware nor does it matter if the same value is written
87 : * again.
88 : */
89 0 : static void umwait_syscore_resume(void)
90 : {
91 0 : umwait_update_control_msr(NULL);
92 0 : }
93 :
94 : static struct syscore_ops umwait_syscore_ops = {
95 : .resume = umwait_syscore_resume,
96 : };
97 :
98 : /* sysfs interface */
99 :
100 : /*
101 : * When bit 0 in IA32_UMWAIT_CONTROL MSR is 1, C0.2 is disabled.
102 : * Otherwise, C0.2 is enabled.
103 : */
104 0 : static inline bool umwait_ctrl_c02_enabled(u32 ctrl)
105 : {
106 0 : return !(ctrl & MSR_IA32_UMWAIT_CONTROL_C02_DISABLE);
107 : }
108 :
109 0 : static inline u32 umwait_ctrl_max_time(u32 ctrl)
110 : {
111 0 : return ctrl & MSR_IA32_UMWAIT_CONTROL_TIME_MASK;
112 : }
113 :
114 0 : static inline void umwait_update_control(u32 maxtime, bool c02_enable)
115 : {
116 0 : u32 ctrl = maxtime & MSR_IA32_UMWAIT_CONTROL_TIME_MASK;
117 :
118 0 : if (!c02_enable)
119 0 : ctrl |= MSR_IA32_UMWAIT_CONTROL_C02_DISABLE;
120 :
121 0 : WRITE_ONCE(umwait_control_cached, ctrl);
122 : /* Propagate to all CPUs */
123 0 : on_each_cpu(umwait_update_control_msr, NULL, 1);
124 0 : }
125 :
126 : static ssize_t
127 0 : enable_c02_show(struct device *dev, struct device_attribute *attr, char *buf)
128 : {
129 0 : u32 ctrl = READ_ONCE(umwait_control_cached);
130 :
131 0 : return sprintf(buf, "%d\n", umwait_ctrl_c02_enabled(ctrl));
132 : }
133 :
134 0 : static ssize_t enable_c02_store(struct device *dev,
135 : struct device_attribute *attr,
136 : const char *buf, size_t count)
137 : {
138 0 : bool c02_enable;
139 0 : u32 ctrl;
140 0 : int ret;
141 :
142 0 : ret = kstrtobool(buf, &c02_enable);
143 0 : if (ret)
144 0 : return ret;
145 :
146 0 : mutex_lock(&umwait_lock);
147 :
148 0 : ctrl = READ_ONCE(umwait_control_cached);
149 0 : if (c02_enable != umwait_ctrl_c02_enabled(ctrl))
150 0 : umwait_update_control(ctrl, c02_enable);
151 :
152 0 : mutex_unlock(&umwait_lock);
153 :
154 0 : return count;
155 : }
156 : static DEVICE_ATTR_RW(enable_c02);
157 :
158 : static ssize_t
159 0 : max_time_show(struct device *kobj, struct device_attribute *attr, char *buf)
160 : {
161 0 : u32 ctrl = READ_ONCE(umwait_control_cached);
162 :
163 0 : return sprintf(buf, "%u\n", umwait_ctrl_max_time(ctrl));
164 : }
165 :
166 0 : static ssize_t max_time_store(struct device *kobj,
167 : struct device_attribute *attr,
168 : const char *buf, size_t count)
169 : {
170 0 : u32 max_time, ctrl;
171 0 : int ret;
172 :
173 0 : ret = kstrtou32(buf, 0, &max_time);
174 0 : if (ret)
175 0 : return ret;
176 :
177 : /* bits[1:0] must be zero */
178 0 : if (max_time & ~MSR_IA32_UMWAIT_CONTROL_TIME_MASK)
179 : return -EINVAL;
180 :
181 0 : mutex_lock(&umwait_lock);
182 :
183 0 : ctrl = READ_ONCE(umwait_control_cached);
184 0 : if (max_time != umwait_ctrl_max_time(ctrl))
185 0 : umwait_update_control(max_time, umwait_ctrl_c02_enabled(ctrl));
186 :
187 0 : mutex_unlock(&umwait_lock);
188 :
189 0 : return count;
190 : }
191 : static DEVICE_ATTR_RW(max_time);
192 :
193 : static struct attribute *umwait_attrs[] = {
194 : &dev_attr_enable_c02.attr,
195 : &dev_attr_max_time.attr,
196 : NULL
197 : };
198 :
199 : static struct attribute_group umwait_attr_group = {
200 : .attrs = umwait_attrs,
201 : .name = "umwait_control",
202 : };
203 :
204 1 : static int __init umwait_init(void)
205 : {
206 1 : struct device *dev;
207 1 : int ret;
208 :
209 1 : if (!boot_cpu_has(X86_FEATURE_WAITPKG))
210 : return -ENODEV;
211 :
212 : /*
213 : * Cache the original control MSR value before the control MSR is
214 : * changed. This is the only place where orig_umwait_control_cached
215 : * is modified.
216 : */
217 0 : rdmsrl(MSR_IA32_UMWAIT_CONTROL, orig_umwait_control_cached);
218 :
219 0 : ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "umwait:online",
220 : umwait_cpu_online, umwait_cpu_offline);
221 0 : if (ret < 0) {
222 : /*
223 : * On failure, the control MSR on all CPUs has the
224 : * original control value.
225 : */
226 : return ret;
227 : }
228 :
229 0 : register_syscore_ops(&umwait_syscore_ops);
230 :
231 : /*
232 : * Add umwait control interface. Ignore failure, so at least the
233 : * default values are set up in case the machine manages to boot.
234 : */
235 0 : dev = cpu_subsys.dev_root;
236 0 : return sysfs_create_group(&dev->kobj, &umwait_attr_group);
237 : }
238 : device_initcall(umwait_init);
|
