Line data Source code
1 : // SPDX-License-Identifier: GPL-2.0-only
2 : /*
3 : * Local APIC handling, local APIC timers
4 : *
5 : * (c) 1999, 2000, 2009 Ingo Molnar <mingo@redhat.com>
6 : *
7 : * Fixes
8 : * Maciej W. Rozycki : Bits for genuine 82489DX APICs;
9 : * thanks to Eric Gilmore
10 : * and Rolf G. Tews
11 : * for testing these extensively.
12 : * Maciej W. Rozycki : Various updates and fixes.
13 : * Mikael Pettersson : Power Management for UP-APIC.
14 : * Pavel Machek and
15 : * Mikael Pettersson : PM converted to driver model.
16 : */
17 :
18 : #include <linux/perf_event.h>
19 : #include <linux/kernel_stat.h>
20 : #include <linux/mc146818rtc.h>
21 : #include <linux/acpi_pmtmr.h>
22 : #include <linux/clockchips.h>
23 : #include <linux/interrupt.h>
24 : #include <linux/memblock.h>
25 : #include <linux/ftrace.h>
26 : #include <linux/ioport.h>
27 : #include <linux/export.h>
28 : #include <linux/syscore_ops.h>
29 : #include <linux/delay.h>
30 : #include <linux/timex.h>
31 : #include <linux/i8253.h>
32 : #include <linux/dmar.h>
33 : #include <linux/init.h>
34 : #include <linux/cpu.h>
35 : #include <linux/dmi.h>
36 : #include <linux/smp.h>
37 : #include <linux/mm.h>
38 :
39 : #include <asm/trace/irq_vectors.h>
40 : #include <asm/irq_remapping.h>
41 : #include <asm/perf_event.h>
42 : #include <asm/x86_init.h>
43 : #include <linux/atomic.h>
44 : #include <asm/barrier.h>
45 : #include <asm/mpspec.h>
46 : #include <asm/i8259.h>
47 : #include <asm/proto.h>
48 : #include <asm/traps.h>
49 : #include <asm/apic.h>
50 : #include <asm/acpi.h>
51 : #include <asm/io_apic.h>
52 : #include <asm/desc.h>
53 : #include <asm/hpet.h>
54 : #include <asm/mtrr.h>
55 : #include <asm/time.h>
56 : #include <asm/smp.h>
57 : #include <asm/mce.h>
58 : #include <asm/tsc.h>
59 : #include <asm/hypervisor.h>
60 : #include <asm/cpu_device_id.h>
61 : #include <asm/intel-family.h>
62 : #include <asm/irq_regs.h>
63 :
64 : unsigned int num_processors;
65 :
66 : unsigned disabled_cpus;
67 :
68 : /* Processor that is doing the boot up */
69 : unsigned int boot_cpu_physical_apicid __ro_after_init = -1U;
70 : EXPORT_SYMBOL_GPL(boot_cpu_physical_apicid);
71 :
72 : u8 boot_cpu_apic_version __ro_after_init;
73 :
74 : /*
75 : * The highest APIC ID seen during enumeration.
76 : */
77 : static unsigned int max_physical_apicid;
78 :
79 : /*
80 : * Bitmask of physically existing CPUs:
81 : */
82 : physid_mask_t phys_cpu_present_map;
83 :
84 : /*
85 : * Processor to be disabled specified by kernel parameter
86 : * disable_cpu_apicid=<int>, mostly used for the kdump 2nd kernel to
87 : * avoid undefined behaviour caused by sending INIT from AP to BSP.
88 : */
89 : static unsigned int disabled_cpu_apicid __ro_after_init = BAD_APICID;
90 :
91 : /*
92 : * This variable controls which CPUs receive external NMIs. By default,
93 : * external NMIs are delivered only to the BSP.
94 : */
95 : static int apic_extnmi __ro_after_init = APIC_EXTNMI_BSP;
96 :
97 : /*
98 : * Hypervisor supports 15 bits of APIC ID in MSI Extended Destination ID
99 : */
100 : static bool virt_ext_dest_id __ro_after_init;
101 :
102 : /*
103 : * Map cpu index to physical APIC ID
104 : */
105 : DEFINE_EARLY_PER_CPU_READ_MOSTLY(u16, x86_cpu_to_apicid, BAD_APICID);
106 : DEFINE_EARLY_PER_CPU_READ_MOSTLY(u16, x86_bios_cpu_apicid, BAD_APICID);
107 : DEFINE_EARLY_PER_CPU_READ_MOSTLY(u32, x86_cpu_to_acpiid, U32_MAX);
108 : EXPORT_EARLY_PER_CPU_SYMBOL(x86_cpu_to_apicid);
109 : EXPORT_EARLY_PER_CPU_SYMBOL(x86_bios_cpu_apicid);
110 : EXPORT_EARLY_PER_CPU_SYMBOL(x86_cpu_to_acpiid);
111 :
112 : #ifdef CONFIG_X86_32
113 :
114 : /*
115 : * On x86_32, the mapping between cpu and logical apicid may vary
116 : * depending on apic in use. The following early percpu variable is
117 : * used for the mapping. This is where the behaviors of x86_64 and 32
118 : * actually diverge. Let's keep it ugly for now.
119 : */
120 : DEFINE_EARLY_PER_CPU_READ_MOSTLY(int, x86_cpu_to_logical_apicid, BAD_APICID);
121 :
122 : /* Local APIC was disabled by the BIOS and enabled by the kernel */
123 : static int enabled_via_apicbase __ro_after_init;
124 :
125 : /*
126 : * Handle interrupt mode configuration register (IMCR).
127 : * This register controls whether the interrupt signals
128 : * that reach the BSP come from the master PIC or from the
129 : * local APIC. Before entering Symmetric I/O Mode, either
130 : * the BIOS or the operating system must switch out of
131 : * PIC Mode by changing the IMCR.
132 : */
133 : static inline void imcr_pic_to_apic(void)
134 : {
135 : /* select IMCR register */
136 : outb(0x70, 0x22);
137 : /* NMI and 8259 INTR go through APIC */
138 : outb(0x01, 0x23);
139 : }
140 :
141 : static inline void imcr_apic_to_pic(void)
142 : {
143 : /* select IMCR register */
144 : outb(0x70, 0x22);
145 : /* NMI and 8259 INTR go directly to BSP */
146 : outb(0x00, 0x23);
147 : }
148 : #endif
149 :
150 : /*
151 : * Knob to control our willingness to enable the local APIC.
152 : *
153 : * +1=force-enable
154 : */
155 : static int force_enable_local_apic __initdata;
156 :
157 : /*
158 : * APIC command line parameters
159 : */
160 0 : static int __init parse_lapic(char *arg)
161 : {
162 0 : if (IS_ENABLED(CONFIG_X86_32) && !arg)
163 : force_enable_local_apic = 1;
164 0 : else if (arg && !strncmp(arg, "notscdeadline", 13))
165 0 : setup_clear_cpu_cap(X86_FEATURE_TSC_DEADLINE_TIMER);
166 0 : return 0;
167 : }
168 : early_param("lapic", parse_lapic);
169 :
170 : #ifdef CONFIG_X86_64
171 : static int apic_calibrate_pmtmr __initdata;
172 0 : static __init int setup_apicpmtimer(char *s)
173 : {
174 0 : apic_calibrate_pmtmr = 1;
175 0 : notsc_setup(NULL);
176 0 : return 0;
177 : }
178 : __setup("apicpmtimer", setup_apicpmtimer);
179 : #endif
180 :
181 : unsigned long mp_lapic_addr __ro_after_init;
182 : int disable_apic __ro_after_init;
183 : /* Disable local APIC timer from the kernel commandline or via dmi quirk */
184 : static int disable_apic_timer __initdata;
185 : /* Local APIC timer works in C2 */
186 : int local_apic_timer_c2_ok __ro_after_init;
187 : EXPORT_SYMBOL_GPL(local_apic_timer_c2_ok);
188 :
189 : /*
190 : * Debug level, exported for io_apic.c
191 : */
192 : int apic_verbosity __ro_after_init;
193 :
194 : int pic_mode __ro_after_init;
195 :
196 : /* Have we found an MP table */
197 : int smp_found_config __ro_after_init;
198 :
199 : static struct resource lapic_resource = {
200 : .name = "Local APIC",
201 : .flags = IORESOURCE_MEM | IORESOURCE_BUSY,
202 : };
203 :
204 : unsigned int lapic_timer_period = 0;
205 :
206 : static void apic_pm_activate(void);
207 :
208 : static unsigned long apic_phys __ro_after_init;
209 :
210 : /*
211 : * Get the LAPIC version
212 : */
213 1 : static inline int lapic_get_version(void)
214 : {
215 2 : return GET_APIC_VERSION(apic_read(APIC_LVR));
216 : }
217 :
218 : /*
219 : * Check, if the APIC is integrated or a separate chip
220 : */
221 19 : static inline int lapic_is_integrated(void)
222 : {
223 19 : return APIC_INTEGRATED(lapic_get_version());
224 : }
225 :
226 : /*
227 : * Check, whether this is a modern or a first generation APIC
228 : */
229 1 : static int modern_apic(void)
230 : {
231 : /* AMD systems use old APIC versions, so check the CPU */
232 1 : if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD &&
233 0 : boot_cpu_data.x86 >= 0xf)
234 : return 1;
235 :
236 : /* Hygon systems use modern APIC */
237 1 : if (boot_cpu_data.x86_vendor == X86_VENDOR_HYGON)
238 : return 1;
239 :
240 1 : return lapic_get_version() >= 0x14;
241 : }
242 :
243 : /*
244 : * right after this call apic become NOOP driven
245 : * so apic->write/read doesn't do anything
246 : */
247 0 : static void __init apic_disable(void)
248 : {
249 0 : pr_info("APIC: switched to apic NOOP\n");
250 0 : apic = &apic_noop;
251 0 : }
252 :
253 0 : void native_apic_wait_icr_idle(void)
254 : {
255 0 : while (apic_read(APIC_ICR) & APIC_ICR_BUSY)
256 0 : cpu_relax();
257 0 : }
258 :
259 12 : u32 native_safe_apic_wait_icr_idle(void)
260 : {
261 12 : u32 send_status;
262 12 : int timeout;
263 :
264 12 : timeout = 0;
265 12 : do {
266 12 : send_status = apic_read(APIC_ICR) & APIC_ICR_BUSY;
267 12 : if (!send_status)
268 : break;
269 0 : inc_irq_stat(icr_read_retry_count);
270 0 : udelay(100);
271 0 : } while (timeout++ < 1000);
272 :
273 12 : return send_status;
274 : }
275 :
276 12 : void native_apic_icr_write(u32 low, u32 id)
277 : {
278 12 : unsigned long flags;
279 :
280 24 : local_irq_save(flags);
281 12 : apic_write(APIC_ICR2, SET_APIC_DEST_FIELD(id));
282 12 : apic_write(APIC_ICR, low);
283 12 : local_irq_restore(flags);
284 12 : }
285 :
286 0 : u64 native_apic_icr_read(void)
287 : {
288 0 : u32 icr1, icr2;
289 :
290 0 : icr2 = apic_read(APIC_ICR2);
291 0 : icr1 = apic_read(APIC_ICR);
292 :
293 0 : return icr1 | ((u64)icr2 << 32);
294 : }
295 :
296 : #ifdef CONFIG_X86_32
297 : /**
298 : * get_physical_broadcast - Get number of physical broadcast IDs
299 : */
300 : int get_physical_broadcast(void)
301 : {
302 : return modern_apic() ? 0xff : 0xf;
303 : }
304 : #endif
305 :
306 : /**
307 : * lapic_get_maxlvt - get the maximum number of local vector table entries
308 : */
309 7 : int lapic_get_maxlvt(void)
310 : {
311 : /*
312 : * - we always have APIC integrated on 64bit mode
313 : * - 82489DXs do not report # of LVT entries
314 : */
315 7 : return lapic_is_integrated() ? GET_APIC_MAXLVT(apic_read(APIC_LVR)) : 2;
316 : }
317 :
318 : /*
319 : * Local APIC timer
320 : */
321 :
322 : /* Clock divisor */
323 : #define APIC_DIVISOR 16
324 : #define TSC_DIVISOR 8
325 :
326 : /*
327 : * This function sets up the local APIC timer, with a timeout of
328 : * 'clocks' APIC bus clock. During calibration we actually call
329 : * this function twice on the boot CPU, once with a bogus timeout
330 : * value, second time for real. The other (noncalibrating) CPUs
331 : * call this function only once, with the real, calibrated value.
332 : *
333 : * We do reads before writes even if unnecessary, to get around the
334 : * P5 APIC double write bug.
335 : */
336 4 : static void __setup_APIC_LVTT(unsigned int clocks, int oneshot, int irqen)
337 : {
338 4 : unsigned int lvtt_value, tmp_value;
339 :
340 4 : lvtt_value = LOCAL_TIMER_VECTOR;
341 4 : if (!oneshot)
342 : lvtt_value |= APIC_LVT_TIMER_PERIODIC;
343 4 : else if (boot_cpu_has(X86_FEATURE_TSC_DEADLINE_TIMER))
344 4 : lvtt_value |= APIC_LVT_TIMER_TSCDEADLINE;
345 :
346 4 : if (!lapic_is_integrated())
347 : lvtt_value |= SET_APIC_TIMER_BASE(APIC_TIMER_BASE_DIV);
348 :
349 4 : if (!irqen)
350 0 : lvtt_value |= APIC_LVT_MASKED;
351 :
352 4 : apic_write(APIC_LVTT, lvtt_value);
353 :
354 4 : if (lvtt_value & APIC_LVT_TIMER_TSCDEADLINE) {
355 : /*
356 : * See Intel SDM: TSC-Deadline Mode chapter. In xAPIC mode,
357 : * writing to the APIC LVTT and TSC_DEADLINE MSR isn't serialized.
358 : * According to Intel, MFENCE can do the serialization here.
359 : */
360 4 : asm volatile("mfence" : : : "memory");
361 4 : return;
362 : }
363 :
364 : /*
365 : * Divide PICLK by 16
366 : */
367 0 : tmp_value = apic_read(APIC_TDCR);
368 0 : apic_write(APIC_TDCR,
369 0 : (tmp_value & ~(APIC_TDR_DIV_1 | APIC_TDR_DIV_TMBASE)) |
370 : APIC_TDR_DIV_16);
371 :
372 0 : if (!oneshot)
373 0 : apic_write(APIC_TMICT, clocks / APIC_DIVISOR);
374 : }
375 :
376 : /*
377 : * Setup extended LVT, AMD specific
378 : *
379 : * Software should use the LVT offsets the BIOS provides. The offsets
380 : * are determined by the subsystems using it like those for MCE
381 : * threshold or IBS. On K8 only offset 0 (APIC500) and MCE interrupts
382 : * are supported. Beginning with family 10h at least 4 offsets are
383 : * available.
384 : *
385 : * Since the offsets must be consistent for all cores, we keep track
386 : * of the LVT offsets in software and reserve the offset for the same
387 : * vector also to be used on other cores. An offset is freed by
388 : * setting the entry to APIC_EILVT_MASKED.
389 : *
390 : * If the BIOS is right, there should be no conflicts. Otherwise a
391 : * "[Firmware Bug]: ..." error message is generated. However, if
392 : * software does not properly determines the offsets, it is not
393 : * necessarily a BIOS bug.
394 : */
395 :
396 : static atomic_t eilvt_offsets[APIC_EILVT_NR_MAX];
397 :
398 0 : static inline int eilvt_entry_is_changeable(unsigned int old, unsigned int new)
399 : {
400 0 : return (old & APIC_EILVT_MASKED)
401 0 : || (new == APIC_EILVT_MASKED)
402 0 : || ((new & ~APIC_EILVT_MASKED) == old);
403 : }
404 :
405 0 : static unsigned int reserve_eilvt_offset(int offset, unsigned int new)
406 : {
407 0 : unsigned int rsvd, vector;
408 :
409 0 : if (offset >= APIC_EILVT_NR_MAX)
410 : return ~0;
411 :
412 0 : rsvd = atomic_read(&eilvt_offsets[offset]);
413 0 : do {
414 0 : vector = rsvd & ~APIC_EILVT_MASKED; /* 0: unassigned */
415 0 : if (vector && !eilvt_entry_is_changeable(vector, new))
416 : /* may not change if vectors are different */
417 0 : return rsvd;
418 0 : rsvd = atomic_cmpxchg(&eilvt_offsets[offset], rsvd, new);
419 0 : } while (rsvd != new);
420 :
421 0 : rsvd &= ~APIC_EILVT_MASKED;
422 0 : if (rsvd && rsvd != vector)
423 0 : pr_info("LVT offset %d assigned for vector 0x%02x\n",
424 : offset, rsvd);
425 :
426 : return new;
427 : }
428 :
429 : /*
430 : * If mask=1, the LVT entry does not generate interrupts while mask=0
431 : * enables the vector. See also the BKDGs. Must be called with
432 : * preemption disabled.
433 : */
434 :
435 0 : int setup_APIC_eilvt(u8 offset, u8 vector, u8 msg_type, u8 mask)
436 : {
437 0 : unsigned long reg = APIC_EILVTn(offset);
438 0 : unsigned int new, old, reserved;
439 :
440 0 : new = (mask << 16) | (msg_type << 8) | vector;
441 0 : old = apic_read(reg);
442 0 : reserved = reserve_eilvt_offset(offset, new);
443 :
444 0 : if (reserved != new) {
445 0 : pr_err(FW_BUG "cpu %d, try to use APIC%lX (LVT offset %d) for "
446 : "vector 0x%x, but the register is already in use for "
447 : "vector 0x%x on another cpu\n",
448 : smp_processor_id(), reg, offset, new, reserved);
449 0 : return -EINVAL;
450 : }
451 :
452 0 : if (!eilvt_entry_is_changeable(old, new)) {
453 0 : pr_err(FW_BUG "cpu %d, try to use APIC%lX (LVT offset %d) for "
454 : "vector 0x%x, but the register is already in use for "
455 : "vector 0x%x on this cpu\n",
456 : smp_processor_id(), reg, offset, new, old);
457 0 : return -EBUSY;
458 : }
459 :
460 0 : apic_write(reg, new);
461 :
462 0 : return 0;
463 : }
464 : EXPORT_SYMBOL_GPL(setup_APIC_eilvt);
465 :
466 : /*
467 : * Program the next event, relative to now
468 : */
469 0 : static int lapic_next_event(unsigned long delta,
470 : struct clock_event_device *evt)
471 : {
472 0 : apic_write(APIC_TMICT, delta);
473 0 : return 0;
474 : }
475 :
476 80044 : static int lapic_next_deadline(unsigned long delta,
477 : struct clock_event_device *evt)
478 : {
479 80044 : u64 tsc;
480 :
481 : /* This MSR is special and need a special fence: */
482 80044 : weak_wrmsr_fence();
483 :
484 83071 : tsc = rdtsc();
485 83181 : wrmsrl(MSR_IA32_TSC_DEADLINE, tsc + (((u64) delta) * TSC_DIVISOR));
486 78892 : return 0;
487 : }
488 :
489 4 : static int lapic_timer_shutdown(struct clock_event_device *evt)
490 : {
491 4 : unsigned int v;
492 :
493 : /* Lapic used as dummy for broadcast ? */
494 4 : if (evt->features & CLOCK_EVT_FEAT_DUMMY)
495 : return 0;
496 :
497 4 : v = apic_read(APIC_LVTT);
498 4 : v |= (APIC_LVT_MASKED | LOCAL_TIMER_VECTOR);
499 4 : apic_write(APIC_LVTT, v);
500 4 : apic_write(APIC_TMICT, 0);
501 4 : return 0;
502 : }
503 :
504 : static inline int
505 4 : lapic_timer_set_periodic_oneshot(struct clock_event_device *evt, bool oneshot)
506 : {
507 : /* Lapic used as dummy for broadcast ? */
508 4 : if (evt->features & CLOCK_EVT_FEAT_DUMMY)
509 : return 0;
510 :
511 4 : __setup_APIC_LVTT(lapic_timer_period, oneshot, 1);
512 4 : return 0;
513 : }
514 :
515 0 : static int lapic_timer_set_periodic(struct clock_event_device *evt)
516 : {
517 0 : return lapic_timer_set_periodic_oneshot(evt, false);
518 : }
519 :
520 4 : static int lapic_timer_set_oneshot(struct clock_event_device *evt)
521 : {
522 4 : return lapic_timer_set_periodic_oneshot(evt, true);
523 : }
524 :
525 : /*
526 : * Local APIC timer broadcast function
527 : */
528 0 : static void lapic_timer_broadcast(const struct cpumask *mask)
529 : {
530 : #ifdef CONFIG_SMP
531 0 : apic->send_IPI_mask(mask, LOCAL_TIMER_VECTOR);
532 : #endif
533 0 : }
534 :
535 :
536 : /*
537 : * The local apic timer can be used for any function which is CPU local.
538 : */
539 : static struct clock_event_device lapic_clockevent = {
540 : .name = "lapic",
541 : .features = CLOCK_EVT_FEAT_PERIODIC |
542 : CLOCK_EVT_FEAT_ONESHOT | CLOCK_EVT_FEAT_C3STOP
543 : | CLOCK_EVT_FEAT_DUMMY,
544 : .shift = 32,
545 : .set_state_shutdown = lapic_timer_shutdown,
546 : .set_state_periodic = lapic_timer_set_periodic,
547 : .set_state_oneshot = lapic_timer_set_oneshot,
548 : .set_state_oneshot_stopped = lapic_timer_shutdown,
549 : .set_next_event = lapic_next_event,
550 : .broadcast = lapic_timer_broadcast,
551 : .rating = 100,
552 : .irq = -1,
553 : };
554 : static DEFINE_PER_CPU(struct clock_event_device, lapic_events);
555 :
556 : static const struct x86_cpu_id deadline_match[] __initconst = {
557 : X86_MATCH_INTEL_FAM6_MODEL_STEPPINGS(HASWELL_X, X86_STEPPINGS(0x2, 0x2), 0x3a), /* EP */
558 : X86_MATCH_INTEL_FAM6_MODEL_STEPPINGS(HASWELL_X, X86_STEPPINGS(0x4, 0x4), 0x0f), /* EX */
559 :
560 : X86_MATCH_INTEL_FAM6_MODEL( BROADWELL_X, 0x0b000020),
561 :
562 : X86_MATCH_INTEL_FAM6_MODEL_STEPPINGS(BROADWELL_D, X86_STEPPINGS(0x2, 0x2), 0x00000011),
563 : X86_MATCH_INTEL_FAM6_MODEL_STEPPINGS(BROADWELL_D, X86_STEPPINGS(0x3, 0x3), 0x0700000e),
564 : X86_MATCH_INTEL_FAM6_MODEL_STEPPINGS(BROADWELL_D, X86_STEPPINGS(0x4, 0x4), 0x0f00000c),
565 : X86_MATCH_INTEL_FAM6_MODEL_STEPPINGS(BROADWELL_D, X86_STEPPINGS(0x5, 0x5), 0x0e000003),
566 :
567 : X86_MATCH_INTEL_FAM6_MODEL_STEPPINGS(SKYLAKE_X, X86_STEPPINGS(0x3, 0x3), 0x01000136),
568 : X86_MATCH_INTEL_FAM6_MODEL_STEPPINGS(SKYLAKE_X, X86_STEPPINGS(0x4, 0x4), 0x02000014),
569 : X86_MATCH_INTEL_FAM6_MODEL_STEPPINGS(SKYLAKE_X, X86_STEPPINGS(0x5, 0xf), 0),
570 :
571 : X86_MATCH_INTEL_FAM6_MODEL( HASWELL, 0x22),
572 : X86_MATCH_INTEL_FAM6_MODEL( HASWELL_L, 0x20),
573 : X86_MATCH_INTEL_FAM6_MODEL( HASWELL_G, 0x17),
574 :
575 : X86_MATCH_INTEL_FAM6_MODEL( BROADWELL, 0x25),
576 : X86_MATCH_INTEL_FAM6_MODEL( BROADWELL_G, 0x17),
577 :
578 : X86_MATCH_INTEL_FAM6_MODEL( SKYLAKE_L, 0xb2),
579 : X86_MATCH_INTEL_FAM6_MODEL( SKYLAKE, 0xb2),
580 :
581 : X86_MATCH_INTEL_FAM6_MODEL( KABYLAKE_L, 0x52),
582 : X86_MATCH_INTEL_FAM6_MODEL( KABYLAKE, 0x52),
583 :
584 : {},
585 : };
586 :
587 1 : static __init bool apic_validate_deadline_timer(void)
588 : {
589 1 : const struct x86_cpu_id *m;
590 1 : u32 rev;
591 :
592 1 : if (!boot_cpu_has(X86_FEATURE_TSC_DEADLINE_TIMER))
593 : return false;
594 1 : if (boot_cpu_has(X86_FEATURE_HYPERVISOR))
595 : return true;
596 :
597 0 : m = x86_match_cpu(deadline_match);
598 0 : if (!m)
599 : return true;
600 :
601 0 : rev = (u32)m->driver_data;
602 :
603 0 : if (boot_cpu_data.microcode >= rev)
604 : return true;
605 :
606 0 : setup_clear_cpu_cap(X86_FEATURE_TSC_DEADLINE_TIMER);
607 0 : pr_err(FW_BUG "TSC_DEADLINE disabled due to Errata; "
608 : "please update microcode to version: 0x%x (or later)\n", rev);
609 0 : return false;
610 : }
611 :
612 : /*
613 : * Setup the local APIC timer for this CPU. Copy the initialized values
614 : * of the boot CPU and register the clock event in the framework.
615 : */
616 4 : static void setup_APIC_timer(void)
617 : {
618 4 : struct clock_event_device *levt = this_cpu_ptr(&lapic_events);
619 :
620 4 : if (this_cpu_has(X86_FEATURE_ARAT)) {
621 4 : lapic_clockevent.features &= ~CLOCK_EVT_FEAT_C3STOP;
622 : /* Make LAPIC timer preferrable over percpu HPET */
623 4 : lapic_clockevent.rating = 150;
624 : }
625 :
626 4 : memcpy(levt, &lapic_clockevent, sizeof(*levt));
627 4 : levt->cpumask = cpumask_of(smp_processor_id());
628 :
629 4 : if (this_cpu_has(X86_FEATURE_TSC_DEADLINE_TIMER)) {
630 4 : levt->name = "lapic-deadline";
631 4 : levt->features &= ~(CLOCK_EVT_FEAT_PERIODIC |
632 : CLOCK_EVT_FEAT_DUMMY);
633 4 : levt->set_next_event = lapic_next_deadline;
634 4 : clockevents_config_and_register(levt,
635 : tsc_khz * (1000 / TSC_DIVISOR),
636 : 0xF, ~0UL);
637 : } else
638 0 : clockevents_register_device(levt);
639 4 : }
640 :
641 : /*
642 : * Install the updated TSC frequency from recalibration at the TSC
643 : * deadline clockevent devices.
644 : */
645 0 : static void __lapic_update_tsc_freq(void *info)
646 : {
647 0 : struct clock_event_device *levt = this_cpu_ptr(&lapic_events);
648 :
649 0 : if (!this_cpu_has(X86_FEATURE_TSC_DEADLINE_TIMER))
650 : return;
651 :
652 0 : clockevents_update_freq(levt, tsc_khz * (1000 / TSC_DIVISOR));
653 : }
654 :
655 0 : void lapic_update_tsc_freq(void)
656 : {
657 : /*
658 : * The clockevent device's ->mult and ->shift can both be
659 : * changed. In order to avoid races, schedule the frequency
660 : * update code on each CPU.
661 : */
662 0 : on_each_cpu(__lapic_update_tsc_freq, NULL, 0);
663 0 : }
664 :
665 : /*
666 : * In this functions we calibrate APIC bus clocks to the external timer.
667 : *
668 : * We want to do the calibration only once since we want to have local timer
669 : * irqs syncron. CPUs connected by the same APIC bus have the very same bus
670 : * frequency.
671 : *
672 : * This was previously done by reading the PIT/HPET and waiting for a wrap
673 : * around to find out, that a tick has elapsed. I have a box, where the PIT
674 : * readout is broken, so it never gets out of the wait loop again. This was
675 : * also reported by others.
676 : *
677 : * Monitoring the jiffies value is inaccurate and the clockevents
678 : * infrastructure allows us to do a simple substitution of the interrupt
679 : * handler.
680 : *
681 : * The calibration routine also uses the pm_timer when possible, as the PIT
682 : * happens to run way too slow (factor 2.3 on my VAIO CoreDuo, which goes
683 : * back to normal later in the boot process).
684 : */
685 :
686 : #define LAPIC_CAL_LOOPS (HZ/10)
687 :
688 : static __initdata int lapic_cal_loops = -1;
689 : static __initdata long lapic_cal_t1, lapic_cal_t2;
690 : static __initdata unsigned long long lapic_cal_tsc1, lapic_cal_tsc2;
691 : static __initdata unsigned long lapic_cal_pm1, lapic_cal_pm2;
692 : static __initdata unsigned long lapic_cal_j1, lapic_cal_j2;
693 :
694 : /*
695 : * Temporary interrupt handler and polled calibration function.
696 : */
697 0 : static void __init lapic_cal_handler(struct clock_event_device *dev)
698 : {
699 0 : unsigned long long tsc = 0;
700 0 : long tapic = apic_read(APIC_TMCCT);
701 0 : unsigned long pm = acpi_pm_read_early();
702 :
703 0 : if (boot_cpu_has(X86_FEATURE_TSC))
704 0 : tsc = rdtsc();
705 :
706 0 : switch (lapic_cal_loops++) {
707 0 : case 0:
708 0 : lapic_cal_t1 = tapic;
709 0 : lapic_cal_tsc1 = tsc;
710 0 : lapic_cal_pm1 = pm;
711 0 : lapic_cal_j1 = jiffies;
712 0 : break;
713 :
714 0 : case LAPIC_CAL_LOOPS:
715 0 : lapic_cal_t2 = tapic;
716 0 : lapic_cal_tsc2 = tsc;
717 0 : if (pm < lapic_cal_pm1)
718 0 : pm += ACPI_PM_OVRRUN;
719 0 : lapic_cal_pm2 = pm;
720 0 : lapic_cal_j2 = jiffies;
721 0 : break;
722 : }
723 0 : }
724 :
725 : static int __init
726 0 : calibrate_by_pmtimer(long deltapm, long *delta, long *deltatsc)
727 : {
728 0 : const long pm_100ms = PMTMR_TICKS_PER_SEC / 10;
729 0 : const long pm_thresh = pm_100ms / 100;
730 0 : unsigned long mult;
731 0 : u64 res;
732 :
733 : #ifndef CONFIG_X86_PM_TIMER
734 0 : return -1;
735 : #endif
736 :
737 : apic_printk(APIC_VERBOSE, "... PM-Timer delta = %ld\n", deltapm);
738 :
739 : /* Check, if the PM timer is available */
740 : if (!deltapm)
741 : return -1;
742 :
743 : mult = clocksource_hz2mult(PMTMR_TICKS_PER_SEC, 22);
744 :
745 : if (deltapm > (pm_100ms - pm_thresh) &&
746 : deltapm < (pm_100ms + pm_thresh)) {
747 : apic_printk(APIC_VERBOSE, "... PM-Timer result ok\n");
748 : return 0;
749 : }
750 :
751 : res = (((u64)deltapm) * mult) >> 22;
752 : do_div(res, 1000000);
753 : pr_warn("APIC calibration not consistent "
754 : "with PM-Timer: %ldms instead of 100ms\n", (long)res);
755 :
756 : /* Correct the lapic counter value */
757 : res = (((u64)(*delta)) * pm_100ms);
758 : do_div(res, deltapm);
759 : pr_info("APIC delta adjusted to PM-Timer: "
760 : "%lu (%ld)\n", (unsigned long)res, *delta);
761 : *delta = (long)res;
762 :
763 : /* Correct the tsc counter value */
764 : if (boot_cpu_has(X86_FEATURE_TSC)) {
765 : res = (((u64)(*deltatsc)) * pm_100ms);
766 : do_div(res, deltapm);
767 : apic_printk(APIC_VERBOSE, "TSC delta adjusted to "
768 : "PM-Timer: %lu (%ld)\n",
769 : (unsigned long)res, *deltatsc);
770 : *deltatsc = (long)res;
771 : }
772 :
773 : return 0;
774 : }
775 :
776 0 : static int __init lapic_init_clockevent(void)
777 : {
778 0 : if (!lapic_timer_period)
779 : return -1;
780 :
781 : /* Calculate the scaled math multiplication factor */
782 0 : lapic_clockevent.mult = div_sc(lapic_timer_period/APIC_DIVISOR,
783 0 : TICK_NSEC, lapic_clockevent.shift);
784 0 : lapic_clockevent.max_delta_ns =
785 0 : clockevent_delta2ns(0x7FFFFFFF, &lapic_clockevent);
786 0 : lapic_clockevent.max_delta_ticks = 0x7FFFFFFF;
787 0 : lapic_clockevent.min_delta_ns =
788 0 : clockevent_delta2ns(0xF, &lapic_clockevent);
789 0 : lapic_clockevent.min_delta_ticks = 0xF;
790 :
791 0 : return 0;
792 : }
793 :
794 1 : bool __init apic_needs_pit(void)
795 : {
796 : /*
797 : * If the frequencies are not known, PIT is required for both TSC
798 : * and apic timer calibration.
799 : */
800 1 : if (!tsc_khz || !cpu_khz)
801 : return true;
802 :
803 : /* Is there an APIC at all or is it disabled? */
804 1 : if (!boot_cpu_has(X86_FEATURE_APIC) || disable_apic)
805 : return true;
806 :
807 : /*
808 : * If interrupt delivery mode is legacy PIC or virtual wire without
809 : * configuration, the local APIC timer wont be set up. Make sure
810 : * that the PIT is initialized.
811 : */
812 1 : if (apic_intr_mode == APIC_PIC ||
813 : apic_intr_mode == APIC_VIRTUAL_WIRE_NO_CONFIG)
814 : return true;
815 :
816 : /* Virt guests may lack ARAT, but still have DEADLINE */
817 1 : if (!boot_cpu_has(X86_FEATURE_ARAT))
818 : return true;
819 :
820 : /* Deadline timer is based on TSC so no further PIT action required */
821 1 : if (boot_cpu_has(X86_FEATURE_TSC_DEADLINE_TIMER))
822 : return false;
823 :
824 : /* APIC timer disabled? */
825 0 : if (disable_apic_timer)
826 : return true;
827 : /*
828 : * The APIC timer frequency is known already, no PIT calibration
829 : * required. If unknown, let the PIT be initialized.
830 : */
831 0 : return lapic_timer_period == 0;
832 : }
833 :
834 1 : static int __init calibrate_APIC_clock(void)
835 : {
836 1 : struct clock_event_device *levt = this_cpu_ptr(&lapic_events);
837 1 : u64 tsc_perj = 0, tsc_start = 0;
838 1 : unsigned long jif_start;
839 1 : unsigned long deltaj;
840 1 : long delta, deltatsc;
841 1 : int pm_referenced = 0;
842 :
843 1 : if (boot_cpu_has(X86_FEATURE_TSC_DEADLINE_TIMER))
844 : return 0;
845 :
846 : /*
847 : * Check if lapic timer has already been calibrated by platform
848 : * specific routine, such as tsc calibration code. If so just fill
849 : * in the clockevent structure and return.
850 : */
851 0 : if (!lapic_init_clockevent()) {
852 0 : apic_printk(APIC_VERBOSE, "lapic timer already calibrated %d\n",
853 : lapic_timer_period);
854 : /*
855 : * Direct calibration methods must have an always running
856 : * local APIC timer, no need for broadcast timer.
857 : */
858 0 : lapic_clockevent.features &= ~CLOCK_EVT_FEAT_DUMMY;
859 0 : return 0;
860 : }
861 :
862 0 : apic_printk(APIC_VERBOSE, "Using local APIC timer interrupts.\n"
863 : "calibrating APIC timer ...\n");
864 :
865 : /*
866 : * There are platforms w/o global clockevent devices. Instead of
867 : * making the calibration conditional on that, use a polling based
868 : * approach everywhere.
869 : */
870 0 : local_irq_disable();
871 :
872 : /*
873 : * Setup the APIC counter to maximum. There is no way the lapic
874 : * can underflow in the 100ms detection time frame
875 : */
876 0 : __setup_APIC_LVTT(0xffffffff, 0, 0);
877 :
878 : /*
879 : * Methods to terminate the calibration loop:
880 : * 1) Global clockevent if available (jiffies)
881 : * 2) TSC if available and frequency is known
882 : */
883 0 : jif_start = READ_ONCE(jiffies);
884 :
885 0 : if (tsc_khz) {
886 0 : tsc_start = rdtsc();
887 0 : tsc_perj = div_u64((u64)tsc_khz * 1000, HZ);
888 : }
889 :
890 : /*
891 : * Enable interrupts so the tick can fire, if a global
892 : * clockevent device is available
893 : */
894 0 : local_irq_enable();
895 :
896 0 : while (lapic_cal_loops <= LAPIC_CAL_LOOPS) {
897 : /* Wait for a tick to elapse */
898 0 : while (1) {
899 0 : if (tsc_khz) {
900 0 : u64 tsc_now = rdtsc();
901 0 : if ((tsc_now - tsc_start) >= tsc_perj) {
902 0 : tsc_start += tsc_perj;
903 0 : break;
904 : }
905 : } else {
906 0 : unsigned long jif_now = READ_ONCE(jiffies);
907 :
908 0 : if (time_after(jif_now, jif_start)) {
909 : jif_start = jif_now;
910 : break;
911 : }
912 : }
913 0 : cpu_relax();
914 : }
915 :
916 : /* Invoke the calibration routine */
917 0 : local_irq_disable();
918 0 : lapic_cal_handler(NULL);
919 0 : local_irq_enable();
920 : }
921 :
922 0 : local_irq_disable();
923 :
924 : /* Build delta t1-t2 as apic timer counts down */
925 0 : delta = lapic_cal_t1 - lapic_cal_t2;
926 0 : apic_printk(APIC_VERBOSE, "... lapic delta = %ld\n", delta);
927 :
928 0 : deltatsc = (long)(lapic_cal_tsc2 - lapic_cal_tsc1);
929 :
930 : /* we trust the PM based calibration if possible */
931 0 : pm_referenced = !calibrate_by_pmtimer(lapic_cal_pm2 - lapic_cal_pm1,
932 : &delta, &deltatsc);
933 :
934 0 : lapic_timer_period = (delta * APIC_DIVISOR) / LAPIC_CAL_LOOPS;
935 0 : lapic_init_clockevent();
936 :
937 0 : apic_printk(APIC_VERBOSE, "..... delta %ld\n", delta);
938 0 : apic_printk(APIC_VERBOSE, "..... mult: %u\n", lapic_clockevent.mult);
939 0 : apic_printk(APIC_VERBOSE, "..... calibration result: %u\n",
940 : lapic_timer_period);
941 :
942 0 : if (boot_cpu_has(X86_FEATURE_TSC)) {
943 0 : apic_printk(APIC_VERBOSE, "..... CPU clock speed is "
944 : "%ld.%04ld MHz.\n",
945 : (deltatsc / LAPIC_CAL_LOOPS) / (1000000 / HZ),
946 : (deltatsc / LAPIC_CAL_LOOPS) % (1000000 / HZ));
947 : }
948 :
949 0 : apic_printk(APIC_VERBOSE, "..... host bus clock speed is "
950 : "%u.%04u MHz.\n",
951 : lapic_timer_period / (1000000 / HZ),
952 : lapic_timer_period % (1000000 / HZ));
953 :
954 : /*
955 : * Do a sanity check on the APIC calibration result
956 : */
957 0 : if (lapic_timer_period < (1000000 / HZ)) {
958 0 : local_irq_enable();
959 0 : pr_warn("APIC frequency too slow, disabling apic timer\n");
960 0 : return -1;
961 : }
962 :
963 0 : levt->features &= ~CLOCK_EVT_FEAT_DUMMY;
964 :
965 : /*
966 : * PM timer calibration failed or not turned on so lets try APIC
967 : * timer based calibration, if a global clockevent device is
968 : * available.
969 : */
970 0 : if (!pm_referenced && global_clock_event) {
971 0 : apic_printk(APIC_VERBOSE, "... verify APIC timer\n");
972 :
973 : /*
974 : * Setup the apic timer manually
975 : */
976 0 : levt->event_handler = lapic_cal_handler;
977 0 : lapic_timer_set_periodic(levt);
978 0 : lapic_cal_loops = -1;
979 :
980 : /* Let the interrupts run */
981 0 : local_irq_enable();
982 :
983 0 : while (lapic_cal_loops <= LAPIC_CAL_LOOPS)
984 0 : cpu_relax();
985 :
986 : /* Stop the lapic timer */
987 0 : local_irq_disable();
988 0 : lapic_timer_shutdown(levt);
989 :
990 : /* Jiffies delta */
991 0 : deltaj = lapic_cal_j2 - lapic_cal_j1;
992 0 : apic_printk(APIC_VERBOSE, "... jiffies delta = %lu\n", deltaj);
993 :
994 : /* Check, if the jiffies result is consistent */
995 0 : if (deltaj >= LAPIC_CAL_LOOPS-2 && deltaj <= LAPIC_CAL_LOOPS+2)
996 0 : apic_printk(APIC_VERBOSE, "... jiffies result ok\n");
997 : else
998 0 : levt->features |= CLOCK_EVT_FEAT_DUMMY;
999 : }
1000 0 : local_irq_enable();
1001 :
1002 0 : if (levt->features & CLOCK_EVT_FEAT_DUMMY) {
1003 0 : pr_warn("APIC timer disabled due to verification failure\n");
1004 0 : return -1;
1005 : }
1006 :
1007 : return 0;
1008 : }
1009 :
1010 : /*
1011 : * Setup the boot APIC
1012 : *
1013 : * Calibrate and verify the result.
1014 : */
1015 1 : void __init setup_boot_APIC_clock(void)
1016 : {
1017 : /*
1018 : * The local apic timer can be disabled via the kernel
1019 : * commandline or from the CPU detection code. Register the lapic
1020 : * timer as a dummy clock event source on SMP systems, so the
1021 : * broadcast mechanism is used. On UP systems simply ignore it.
1022 : */
1023 1 : if (disable_apic_timer) {
1024 0 : pr_info("Disabling APIC timer\n");
1025 : /* No broadcast on UP ! */
1026 0 : if (num_possible_cpus() > 1) {
1027 0 : lapic_clockevent.mult = 1;
1028 0 : setup_APIC_timer();
1029 : }
1030 0 : return;
1031 : }
1032 :
1033 1 : if (calibrate_APIC_clock()) {
1034 : /* No broadcast on UP ! */
1035 0 : if (num_possible_cpus() > 1)
1036 0 : setup_APIC_timer();
1037 0 : return;
1038 : }
1039 :
1040 : /*
1041 : * If nmi_watchdog is set to IO_APIC, we need the
1042 : * PIT/HPET going. Otherwise register lapic as a dummy
1043 : * device.
1044 : */
1045 1 : lapic_clockevent.features &= ~CLOCK_EVT_FEAT_DUMMY;
1046 :
1047 : /* Setup the lapic or request the broadcast */
1048 1 : setup_APIC_timer();
1049 1 : amd_e400_c1e_apic_setup();
1050 : }
1051 :
1052 3 : void setup_secondary_APIC_clock(void)
1053 : {
1054 3 : setup_APIC_timer();
1055 3 : amd_e400_c1e_apic_setup();
1056 3 : }
1057 :
1058 : /*
1059 : * The guts of the apic timer interrupt
1060 : */
1061 73901 : static void local_apic_timer_interrupt(void)
1062 : {
1063 73901 : struct clock_event_device *evt = this_cpu_ptr(&lapic_events);
1064 :
1065 : /*
1066 : * Normally we should not be here till LAPIC has been initialized but
1067 : * in some cases like kdump, its possible that there is a pending LAPIC
1068 : * timer interrupt from previous kernel's context and is delivered in
1069 : * new kernel the moment interrupts are enabled.
1070 : *
1071 : * Interrupts are enabled early and LAPIC is setup much later, hence
1072 : * its possible that when we get here evt->event_handler is NULL.
1073 : * Check for event_handler being NULL and discard the interrupt as
1074 : * spurious.
1075 : */
1076 74026 : if (!evt->event_handler) {
1077 0 : pr_warn("Spurious LAPIC timer interrupt on cpu %d\n",
1078 : smp_processor_id());
1079 : /* Switch it off */
1080 0 : lapic_timer_shutdown(evt);
1081 0 : return;
1082 : }
1083 :
1084 : /*
1085 : * the NMI deadlock-detector uses this.
1086 : */
1087 74026 : inc_irq_stat(apic_timer_irqs);
1088 :
1089 74219 : evt->event_handler(evt);
1090 : }
1091 :
1092 : /*
1093 : * Local APIC timer interrupt. This is the most natural way for doing
1094 : * local interrupts, but local timer interrupts can be emulated by
1095 : * broadcast interrupts too. [in case the hw doesn't support APIC timers]
1096 : *
1097 : * [ if a single-CPU system runs an SMP kernel then we call the local
1098 : * interrupt as well. Thus we cannot inline the local irq ... ]
1099 : */
1100 142111 : DEFINE_IDTENTRY_SYSVEC(sysvec_apic_timer_interrupt)
1101 : {
1102 72197 : struct pt_regs *old_regs = set_irq_regs(regs);
1103 :
1104 72197 : ack_APIC_irq();
1105 72826 : trace_local_timer_entry(LOCAL_TIMER_VECTOR);
1106 74022 : local_apic_timer_interrupt();
1107 74080 : trace_local_timer_exit(LOCAL_TIMER_VECTOR);
1108 :
1109 75125 : set_irq_regs(old_regs);
1110 75125 : }
1111 :
1112 0 : int setup_profiling_timer(unsigned int multiplier)
1113 : {
1114 0 : return -EINVAL;
1115 : }
1116 :
1117 : /*
1118 : * Local APIC start and shutdown
1119 : */
1120 :
1121 : /**
1122 : * clear_local_APIC - shutdown the local APIC
1123 : *
1124 : * This is called, when a CPU is disabled and before rebooting, so the state of
1125 : * the local APIC has no dangling leftovers. Also used to cleanout any BIOS
1126 : * leftovers during boot.
1127 : */
1128 0 : void clear_local_APIC(void)
1129 : {
1130 0 : int maxlvt;
1131 0 : u32 v;
1132 :
1133 : /* APIC hasn't been mapped yet */
1134 0 : if (!x2apic_mode && !apic_phys)
1135 : return;
1136 :
1137 0 : maxlvt = lapic_get_maxlvt();
1138 : /*
1139 : * Masking an LVT entry can trigger a local APIC error
1140 : * if the vector is zero. Mask LVTERR first to prevent this.
1141 : */
1142 0 : if (maxlvt >= 3) {
1143 0 : v = ERROR_APIC_VECTOR; /* any non-zero vector will do */
1144 0 : apic_write(APIC_LVTERR, v | APIC_LVT_MASKED);
1145 : }
1146 : /*
1147 : * Careful: we have to set masks only first to deassert
1148 : * any level-triggered sources.
1149 : */
1150 0 : v = apic_read(APIC_LVTT);
1151 0 : apic_write(APIC_LVTT, v | APIC_LVT_MASKED);
1152 0 : v = apic_read(APIC_LVT0);
1153 0 : apic_write(APIC_LVT0, v | APIC_LVT_MASKED);
1154 0 : v = apic_read(APIC_LVT1);
1155 0 : apic_write(APIC_LVT1, v | APIC_LVT_MASKED);
1156 0 : if (maxlvt >= 4) {
1157 0 : v = apic_read(APIC_LVTPC);
1158 0 : apic_write(APIC_LVTPC, v | APIC_LVT_MASKED);
1159 : }
1160 :
1161 : /* lets not touch this if we didn't frob it */
1162 : #ifdef CONFIG_X86_THERMAL_VECTOR
1163 : if (maxlvt >= 5) {
1164 : v = apic_read(APIC_LVTTHMR);
1165 : apic_write(APIC_LVTTHMR, v | APIC_LVT_MASKED);
1166 : }
1167 : #endif
1168 : #ifdef CONFIG_X86_MCE_INTEL
1169 : if (maxlvt >= 6) {
1170 : v = apic_read(APIC_LVTCMCI);
1171 : if (!(v & APIC_LVT_MASKED))
1172 : apic_write(APIC_LVTCMCI, v | APIC_LVT_MASKED);
1173 : }
1174 : #endif
1175 :
1176 : /*
1177 : * Clean APIC state for other OSs:
1178 : */
1179 0 : apic_write(APIC_LVTT, APIC_LVT_MASKED);
1180 0 : apic_write(APIC_LVT0, APIC_LVT_MASKED);
1181 0 : apic_write(APIC_LVT1, APIC_LVT_MASKED);
1182 0 : if (maxlvt >= 3)
1183 0 : apic_write(APIC_LVTERR, APIC_LVT_MASKED);
1184 0 : if (maxlvt >= 4)
1185 0 : apic_write(APIC_LVTPC, APIC_LVT_MASKED);
1186 :
1187 : /* Integrated APIC (!82489DX) ? */
1188 0 : if (lapic_is_integrated()) {
1189 0 : if (maxlvt > 3)
1190 : /* Clear ESR due to Pentium errata 3AP and 11AP */
1191 0 : apic_write(APIC_ESR, 0);
1192 0 : apic_read(APIC_ESR);
1193 : }
1194 : }
1195 :
1196 : /**
1197 : * apic_soft_disable - Clears and software disables the local APIC on hotplug
1198 : *
1199 : * Contrary to disable_local_APIC() this does not touch the enable bit in
1200 : * MSR_IA32_APICBASE. Clearing that bit on systems based on the 3 wire APIC
1201 : * bus would require a hardware reset as the APIC would lose track of bus
1202 : * arbitration. On systems with FSB delivery APICBASE could be disabled,
1203 : * but it has to be guaranteed that no interrupt is sent to the APIC while
1204 : * in that state and it's not clear from the SDM whether it still responds
1205 : * to INIT/SIPI messages. Stay on the safe side and use software disable.
1206 : */
1207 0 : void apic_soft_disable(void)
1208 : {
1209 0 : u32 value;
1210 :
1211 0 : clear_local_APIC();
1212 :
1213 : /* Soft disable APIC (implies clearing of registers for 82489DX!). */
1214 0 : value = apic_read(APIC_SPIV);
1215 0 : value &= ~APIC_SPIV_APIC_ENABLED;
1216 0 : apic_write(APIC_SPIV, value);
1217 0 : }
1218 :
1219 : /**
1220 : * disable_local_APIC - clear and disable the local APIC
1221 : */
1222 0 : void disable_local_APIC(void)
1223 : {
1224 : /* APIC hasn't been mapped yet */
1225 0 : if (!x2apic_mode && !apic_phys)
1226 : return;
1227 :
1228 0 : apic_soft_disable();
1229 :
1230 : #ifdef CONFIG_X86_32
1231 : /*
1232 : * When LAPIC was disabled by the BIOS and enabled by the kernel,
1233 : * restore the disabled state.
1234 : */
1235 : if (enabled_via_apicbase) {
1236 : unsigned int l, h;
1237 :
1238 : rdmsr(MSR_IA32_APICBASE, l, h);
1239 : l &= ~MSR_IA32_APICBASE_ENABLE;
1240 : wrmsr(MSR_IA32_APICBASE, l, h);
1241 : }
1242 : #endif
1243 : }
1244 :
1245 : /*
1246 : * If Linux enabled the LAPIC against the BIOS default disable it down before
1247 : * re-entering the BIOS on shutdown. Otherwise the BIOS may get confused and
1248 : * not power-off. Additionally clear all LVT entries before disable_local_APIC
1249 : * for the case where Linux didn't enable the LAPIC.
1250 : */
1251 0 : void lapic_shutdown(void)
1252 : {
1253 0 : unsigned long flags;
1254 :
1255 0 : if (!boot_cpu_has(X86_FEATURE_APIC) && !apic_from_smp_config())
1256 : return;
1257 :
1258 0 : local_irq_save(flags);
1259 :
1260 : #ifdef CONFIG_X86_32
1261 : if (!enabled_via_apicbase)
1262 : clear_local_APIC();
1263 : else
1264 : #endif
1265 0 : disable_local_APIC();
1266 :
1267 :
1268 0 : local_irq_restore(flags);
1269 : }
1270 :
1271 : /**
1272 : * sync_Arb_IDs - synchronize APIC bus arbitration IDs
1273 : */
1274 1 : void __init sync_Arb_IDs(void)
1275 : {
1276 : /*
1277 : * Unsupported on P4 - see Intel Dev. Manual Vol. 3, Ch. 8.6.1 And not
1278 : * needed on AMD.
1279 : */
1280 1 : if (modern_apic() || boot_cpu_data.x86_vendor == X86_VENDOR_AMD)
1281 : return;
1282 :
1283 : /*
1284 : * Wait for idle.
1285 : */
1286 0 : apic_wait_icr_idle();
1287 :
1288 0 : apic_printk(APIC_DEBUG, "Synchronizing Arb IDs.\n");
1289 0 : apic_write(APIC_ICR, APIC_DEST_ALLINC |
1290 : APIC_INT_LEVELTRIG | APIC_DM_INIT);
1291 : }
1292 :
1293 : enum apic_intr_mode_id apic_intr_mode __ro_after_init;
1294 :
1295 1 : static int __init __apic_intr_mode_select(void)
1296 : {
1297 : /* Check kernel option */
1298 1 : if (disable_apic) {
1299 0 : pr_info("APIC disabled via kernel command line\n");
1300 0 : return APIC_PIC;
1301 : }
1302 :
1303 : /* Check BIOS */
1304 : #ifdef CONFIG_X86_64
1305 : /* On 64-bit, the APIC must be integrated, Check local APIC only */
1306 1 : if (!boot_cpu_has(X86_FEATURE_APIC)) {
1307 0 : disable_apic = 1;
1308 0 : pr_info("APIC disabled by BIOS\n");
1309 0 : return APIC_PIC;
1310 : }
1311 : #else
1312 : /* On 32-bit, the APIC may be integrated APIC or 82489DX */
1313 :
1314 : /* Neither 82489DX nor integrated APIC ? */
1315 : if (!boot_cpu_has(X86_FEATURE_APIC) && !smp_found_config) {
1316 : disable_apic = 1;
1317 : return APIC_PIC;
1318 : }
1319 :
1320 : /* If the BIOS pretends there is an integrated APIC ? */
1321 : if (!boot_cpu_has(X86_FEATURE_APIC) &&
1322 : APIC_INTEGRATED(boot_cpu_apic_version)) {
1323 : disable_apic = 1;
1324 : pr_err(FW_BUG "Local APIC %d not detected, force emulation\n",
1325 : boot_cpu_physical_apicid);
1326 : return APIC_PIC;
1327 : }
1328 : #endif
1329 :
1330 : /* Check MP table or ACPI MADT configuration */
1331 1 : if (!smp_found_config) {
1332 0 : disable_ioapic_support();
1333 0 : if (!acpi_lapic) {
1334 0 : pr_info("APIC: ACPI MADT or MP tables are not detected\n");
1335 0 : return APIC_VIRTUAL_WIRE_NO_CONFIG;
1336 : }
1337 : return APIC_VIRTUAL_WIRE;
1338 : }
1339 :
1340 : #ifdef CONFIG_SMP
1341 : /* If SMP should be disabled, then really disable it! */
1342 1 : if (!setup_max_cpus) {
1343 0 : pr_info("APIC: SMP mode deactivated\n");
1344 0 : return APIC_SYMMETRIC_IO_NO_ROUTING;
1345 : }
1346 :
1347 1 : if (read_apic_id() != boot_cpu_physical_apicid) {
1348 0 : panic("Boot APIC ID in local APIC unexpected (%d vs %d)",
1349 : read_apic_id(), boot_cpu_physical_apicid);
1350 : /* Or can we switch back to PIC here? */
1351 : }
1352 : #endif
1353 :
1354 : return APIC_SYMMETRIC_IO;
1355 : }
1356 :
1357 : /* Select the interrupt delivery mode for the BSP */
1358 1 : void __init apic_intr_mode_select(void)
1359 : {
1360 1 : apic_intr_mode = __apic_intr_mode_select();
1361 1 : }
1362 :
1363 : /*
1364 : * An initial setup of the virtual wire mode.
1365 : */
1366 1 : void __init init_bsp_APIC(void)
1367 : {
1368 1 : unsigned int value;
1369 :
1370 : /*
1371 : * Don't do the setup now if we have a SMP BIOS as the
1372 : * through-I/O-APIC virtual wire mode might be active.
1373 : */
1374 1 : if (smp_found_config || !boot_cpu_has(X86_FEATURE_APIC))
1375 1 : return;
1376 :
1377 : /*
1378 : * Do not trust the local APIC being empty at bootup.
1379 : */
1380 0 : clear_local_APIC();
1381 :
1382 : /*
1383 : * Enable APIC.
1384 : */
1385 0 : value = apic_read(APIC_SPIV);
1386 0 : value &= ~APIC_VECTOR_MASK;
1387 0 : value |= APIC_SPIV_APIC_ENABLED;
1388 :
1389 : #ifdef CONFIG_X86_32
1390 : /* This bit is reserved on P4/Xeon and should be cleared */
1391 : if ((boot_cpu_data.x86_vendor == X86_VENDOR_INTEL) &&
1392 : (boot_cpu_data.x86 == 15))
1393 : value &= ~APIC_SPIV_FOCUS_DISABLED;
1394 : else
1395 : #endif
1396 0 : value |= APIC_SPIV_FOCUS_DISABLED;
1397 0 : value |= SPURIOUS_APIC_VECTOR;
1398 0 : apic_write(APIC_SPIV, value);
1399 :
1400 : /*
1401 : * Set up the virtual wire mode.
1402 : */
1403 0 : apic_write(APIC_LVT0, APIC_DM_EXTINT);
1404 0 : value = APIC_DM_NMI;
1405 0 : if (!lapic_is_integrated()) /* 82489DX */
1406 : value |= APIC_LVT_LEVEL_TRIGGER;
1407 0 : if (apic_extnmi == APIC_EXTNMI_NONE)
1408 0 : value |= APIC_LVT_MASKED;
1409 0 : apic_write(APIC_LVT1, value);
1410 : }
1411 :
1412 : static void __init apic_bsp_setup(bool upmode);
1413 :
1414 : /* Init the interrupt delivery mode for the BSP */
1415 1 : void __init apic_intr_mode_init(void)
1416 : {
1417 1 : bool upmode = IS_ENABLED(CONFIG_UP_LATE_INIT);
1418 :
1419 1 : switch (apic_intr_mode) {
1420 0 : case APIC_PIC:
1421 0 : pr_info("APIC: Keep in PIC mode(8259)\n");
1422 0 : return;
1423 0 : case APIC_VIRTUAL_WIRE:
1424 0 : pr_info("APIC: Switch to virtual wire mode setup\n");
1425 0 : default_setup_apic_routing();
1426 0 : break;
1427 0 : case APIC_VIRTUAL_WIRE_NO_CONFIG:
1428 0 : pr_info("APIC: Switch to virtual wire mode setup with no configuration\n");
1429 0 : upmode = true;
1430 0 : default_setup_apic_routing();
1431 0 : break;
1432 1 : case APIC_SYMMETRIC_IO:
1433 1 : pr_info("APIC: Switch to symmetric I/O mode setup\n");
1434 1 : default_setup_apic_routing();
1435 1 : break;
1436 0 : case APIC_SYMMETRIC_IO_NO_ROUTING:
1437 0 : pr_info("APIC: Switch to symmetric I/O mode setup in no SMP routine\n");
1438 0 : break;
1439 : }
1440 :
1441 1 : if (x86_platform.apic_post_init)
1442 1 : x86_platform.apic_post_init();
1443 :
1444 1 : apic_bsp_setup(upmode);
1445 : }
1446 :
1447 4 : static void lapic_setup_esr(void)
1448 : {
1449 4 : unsigned int oldvalue, value, maxlvt;
1450 :
1451 4 : if (!lapic_is_integrated()) {
1452 : pr_info("No ESR for 82489DX.\n");
1453 : return;
1454 : }
1455 :
1456 4 : if (apic->disable_esr) {
1457 : /*
1458 : * Something untraceable is creating bad interrupts on
1459 : * secondary quads ... for the moment, just leave the
1460 : * ESR disabled - we can't do anything useful with the
1461 : * errors anyway - mbligh
1462 : */
1463 0 : pr_info("Leaving ESR disabled.\n");
1464 0 : return;
1465 : }
1466 :
1467 4 : maxlvt = lapic_get_maxlvt();
1468 4 : if (maxlvt > 3) /* Due to the Pentium erratum 3AP. */
1469 4 : apic_write(APIC_ESR, 0);
1470 4 : oldvalue = apic_read(APIC_ESR);
1471 :
1472 : /* enables sending errors */
1473 4 : value = ERROR_APIC_VECTOR;
1474 4 : apic_write(APIC_LVTERR, value);
1475 :
1476 : /*
1477 : * spec says clear errors after enabling vector.
1478 : */
1479 4 : if (maxlvt > 3)
1480 4 : apic_write(APIC_ESR, 0);
1481 4 : value = apic_read(APIC_ESR);
1482 4 : if (value != oldvalue)
1483 0 : apic_printk(APIC_VERBOSE, "ESR value before enabling "
1484 : "vector: 0x%08x after: 0x%08x\n",
1485 : oldvalue, value);
1486 : }
1487 :
1488 : #define APIC_IR_REGS APIC_ISR_NR
1489 : #define APIC_IR_BITS (APIC_IR_REGS * 32)
1490 : #define APIC_IR_MAPSIZE (APIC_IR_BITS / BITS_PER_LONG)
1491 :
1492 : union apic_ir {
1493 : unsigned long map[APIC_IR_MAPSIZE];
1494 : u32 regs[APIC_IR_REGS];
1495 : };
1496 :
1497 4 : static bool apic_check_and_ack(union apic_ir *irr, union apic_ir *isr)
1498 : {
1499 4 : int i, bit;
1500 :
1501 : /* Read the IRRs */
1502 36 : for (i = 0; i < APIC_IR_REGS; i++)
1503 32 : irr->regs[i] = apic_read(APIC_IRR + i * 0x10);
1504 :
1505 : /* Read the ISRs */
1506 36 : for (i = 0; i < APIC_IR_REGS; i++)
1507 32 : isr->regs[i] = apic_read(APIC_ISR + i * 0x10);
1508 :
1509 : /*
1510 : * If the ISR map is not empty. ACK the APIC and run another round
1511 : * to verify whether a pending IRR has been unblocked and turned
1512 : * into a ISR.
1513 : */
1514 4 : if (!bitmap_empty(isr->map, APIC_IR_BITS)) {
1515 : /*
1516 : * There can be multiple ISR bits set when a high priority
1517 : * interrupt preempted a lower priority one. Issue an ACK
1518 : * per set bit.
1519 : */
1520 0 : for_each_set_bit(bit, isr->map, APIC_IR_BITS)
1521 0 : ack_APIC_irq();
1522 : return true;
1523 : }
1524 :
1525 4 : return !bitmap_empty(irr->map, APIC_IR_BITS);
1526 : }
1527 :
1528 : /*
1529 : * After a crash, we no longer service the interrupts and a pending
1530 : * interrupt from previous kernel might still have ISR bit set.
1531 : *
1532 : * Most probably by now the CPU has serviced that pending interrupt and it
1533 : * might not have done the ack_APIC_irq() because it thought, interrupt
1534 : * came from i8259 as ExtInt. LAPIC did not get EOI so it does not clear
1535 : * the ISR bit and cpu thinks it has already serivced the interrupt. Hence
1536 : * a vector might get locked. It was noticed for timer irq (vector
1537 : * 0x31). Issue an extra EOI to clear ISR.
1538 : *
1539 : * If there are pending IRR bits they turn into ISR bits after a higher
1540 : * priority ISR bit has been acked.
1541 : */
1542 4 : static void apic_pending_intr_clear(void)
1543 : {
1544 4 : union apic_ir irr, isr;
1545 4 : unsigned int i;
1546 :
1547 : /* 512 loops are way oversized and give the APIC a chance to obey. */
1548 4 : for (i = 0; i < 512; i++) {
1549 4 : if (!apic_check_and_ack(&irr, &isr))
1550 4 : return;
1551 : }
1552 : /* Dump the IRR/ISR content if that failed */
1553 0 : pr_warn("APIC: Stale IRR: %256pb ISR: %256pb\n", irr.map, isr.map);
1554 : }
1555 :
1556 : /**
1557 : * setup_local_APIC - setup the local APIC
1558 : *
1559 : * Used to setup local APIC while initializing BSP or bringing up APs.
1560 : * Always called with preemption disabled.
1561 : */
1562 4 : static void setup_local_APIC(void)
1563 : {
1564 4 : int cpu = smp_processor_id();
1565 4 : unsigned int value;
1566 :
1567 4 : if (disable_apic) {
1568 0 : disable_ioapic_support();
1569 0 : return;
1570 : }
1571 :
1572 : /*
1573 : * If this comes from kexec/kcrash the APIC might be enabled in
1574 : * SPIV. Soft disable it before doing further initialization.
1575 : */
1576 4 : value = apic_read(APIC_SPIV);
1577 4 : value &= ~APIC_SPIV_APIC_ENABLED;
1578 4 : apic_write(APIC_SPIV, value);
1579 :
1580 : #ifdef CONFIG_X86_32
1581 : /* Pound the ESR really hard over the head with a big hammer - mbligh */
1582 : if (lapic_is_integrated() && apic->disable_esr) {
1583 : apic_write(APIC_ESR, 0);
1584 : apic_write(APIC_ESR, 0);
1585 : apic_write(APIC_ESR, 0);
1586 : apic_write(APIC_ESR, 0);
1587 : }
1588 : #endif
1589 : /*
1590 : * Double-check whether this APIC is really registered.
1591 : * This is meaningless in clustered apic mode, so we skip it.
1592 : */
1593 4 : BUG_ON(!apic->apic_id_registered());
1594 :
1595 : /*
1596 : * Intel recommends to set DFR, LDR and TPR before enabling
1597 : * an APIC. See e.g. "AP-388 82489DX User's Manual" (Intel
1598 : * document number 292116). So here it goes...
1599 : */
1600 4 : apic->init_apic_ldr();
1601 :
1602 : #ifdef CONFIG_X86_32
1603 : if (apic->dest_mode_logical) {
1604 : int logical_apicid, ldr_apicid;
1605 :
1606 : /*
1607 : * APIC LDR is initialized. If logical_apicid mapping was
1608 : * initialized during get_smp_config(), make sure it matches
1609 : * the actual value.
1610 : */
1611 : logical_apicid = early_per_cpu(x86_cpu_to_logical_apicid, cpu);
1612 : ldr_apicid = GET_APIC_LOGICAL_ID(apic_read(APIC_LDR));
1613 : if (logical_apicid != BAD_APICID)
1614 : WARN_ON(logical_apicid != ldr_apicid);
1615 : /* Always use the value from LDR. */
1616 : early_per_cpu(x86_cpu_to_logical_apicid, cpu) = ldr_apicid;
1617 : }
1618 : #endif
1619 :
1620 : /*
1621 : * Set Task Priority to 'accept all except vectors 0-31'. An APIC
1622 : * vector in the 16-31 range could be delivered if TPR == 0, but we
1623 : * would think it's an exception and terrible things will happen. We
1624 : * never change this later on.
1625 : */
1626 4 : value = apic_read(APIC_TASKPRI);
1627 4 : value &= ~APIC_TPRI_MASK;
1628 4 : value |= 0x10;
1629 4 : apic_write(APIC_TASKPRI, value);
1630 :
1631 : /* Clear eventually stale ISR/IRR bits */
1632 4 : apic_pending_intr_clear();
1633 :
1634 : /*
1635 : * Now that we are all set up, enable the APIC
1636 : */
1637 4 : value = apic_read(APIC_SPIV);
1638 4 : value &= ~APIC_VECTOR_MASK;
1639 : /*
1640 : * Enable APIC
1641 : */
1642 4 : value |= APIC_SPIV_APIC_ENABLED;
1643 :
1644 : #ifdef CONFIG_X86_32
1645 : /*
1646 : * Some unknown Intel IO/APIC (or APIC) errata is biting us with
1647 : * certain networking cards. If high frequency interrupts are
1648 : * happening on a particular IOAPIC pin, plus the IOAPIC routing
1649 : * entry is masked/unmasked at a high rate as well then sooner or
1650 : * later IOAPIC line gets 'stuck', no more interrupts are received
1651 : * from the device. If focus CPU is disabled then the hang goes
1652 : * away, oh well :-(
1653 : *
1654 : * [ This bug can be reproduced easily with a level-triggered
1655 : * PCI Ne2000 networking cards and PII/PIII processors, dual
1656 : * BX chipset. ]
1657 : */
1658 : /*
1659 : * Actually disabling the focus CPU check just makes the hang less
1660 : * frequent as it makes the interrupt distributon model be more
1661 : * like LRU than MRU (the short-term load is more even across CPUs).
1662 : */
1663 :
1664 : /*
1665 : * - enable focus processor (bit==0)
1666 : * - 64bit mode always use processor focus
1667 : * so no need to set it
1668 : */
1669 : value &= ~APIC_SPIV_FOCUS_DISABLED;
1670 : #endif
1671 :
1672 : /*
1673 : * Set spurious IRQ vector
1674 : */
1675 4 : value |= SPURIOUS_APIC_VECTOR;
1676 4 : apic_write(APIC_SPIV, value);
1677 :
1678 4 : perf_events_lapic_init();
1679 :
1680 : /*
1681 : * Set up LVT0, LVT1:
1682 : *
1683 : * set up through-local-APIC on the boot CPU's LINT0. This is not
1684 : * strictly necessary in pure symmetric-IO mode, but sometimes
1685 : * we delegate interrupts to the 8259A.
1686 : */
1687 : /*
1688 : * TODO: set up through-local-APIC from through-I/O-APIC? --macro
1689 : */
1690 4 : value = apic_read(APIC_LVT0) & APIC_LVT_MASKED;
1691 4 : if (!cpu && (pic_mode || !value || skip_ioapic_setup)) {
1692 0 : value = APIC_DM_EXTINT;
1693 0 : apic_printk(APIC_VERBOSE, "enabled ExtINT on CPU#%d\n", cpu);
1694 : } else {
1695 4 : value = APIC_DM_EXTINT | APIC_LVT_MASKED;
1696 4 : apic_printk(APIC_VERBOSE, "masked ExtINT on CPU#%d\n", cpu);
1697 : }
1698 4 : apic_write(APIC_LVT0, value);
1699 :
1700 : /*
1701 : * Only the BSP sees the LINT1 NMI signal by default. This can be
1702 : * modified by apic_extnmi= boot option.
1703 : */
1704 4 : if ((!cpu && apic_extnmi != APIC_EXTNMI_NONE) ||
1705 3 : apic_extnmi == APIC_EXTNMI_ALL)
1706 : value = APIC_DM_NMI;
1707 : else
1708 3 : value = APIC_DM_NMI | APIC_LVT_MASKED;
1709 :
1710 : /* Is 82489DX ? */
1711 4 : if (!lapic_is_integrated())
1712 : value |= APIC_LVT_LEVEL_TRIGGER;
1713 4 : apic_write(APIC_LVT1, value);
1714 :
1715 : #ifdef CONFIG_X86_MCE_INTEL
1716 : /* Recheck CMCI information after local APIC is up on CPU #0 */
1717 : if (!cpu)
1718 : cmci_recheck();
1719 : #endif
1720 : }
1721 :
1722 4 : static void end_local_APIC_setup(void)
1723 : {
1724 4 : lapic_setup_esr();
1725 :
1726 : #ifdef CONFIG_X86_32
1727 : {
1728 : unsigned int value;
1729 : /* Disable the local apic timer */
1730 : value = apic_read(APIC_LVTT);
1731 : value |= (APIC_LVT_MASKED | LOCAL_TIMER_VECTOR);
1732 : apic_write(APIC_LVTT, value);
1733 : }
1734 : #endif
1735 :
1736 4 : apic_pm_activate();
1737 : }
1738 :
1739 : /*
1740 : * APIC setup function for application processors. Called from smpboot.c
1741 : */
1742 3 : void apic_ap_setup(void)
1743 : {
1744 3 : setup_local_APIC();
1745 3 : end_local_APIC_setup();
1746 3 : }
1747 :
1748 : #ifdef CONFIG_X86_X2APIC
1749 : int x2apic_mode;
1750 : EXPORT_SYMBOL_GPL(x2apic_mode);
1751 :
1752 : enum {
1753 : X2APIC_OFF,
1754 : X2APIC_ON,
1755 : X2APIC_DISABLED,
1756 : };
1757 : static int x2apic_state;
1758 :
1759 : static void __x2apic_disable(void)
1760 : {
1761 : u64 msr;
1762 :
1763 : if (!boot_cpu_has(X86_FEATURE_APIC))
1764 : return;
1765 :
1766 : rdmsrl(MSR_IA32_APICBASE, msr);
1767 : if (!(msr & X2APIC_ENABLE))
1768 : return;
1769 : /* Disable xapic and x2apic first and then reenable xapic mode */
1770 : wrmsrl(MSR_IA32_APICBASE, msr & ~(X2APIC_ENABLE | XAPIC_ENABLE));
1771 : wrmsrl(MSR_IA32_APICBASE, msr & ~X2APIC_ENABLE);
1772 : printk_once(KERN_INFO "x2apic disabled\n");
1773 : }
1774 :
1775 : static void __x2apic_enable(void)
1776 : {
1777 : u64 msr;
1778 :
1779 : rdmsrl(MSR_IA32_APICBASE, msr);
1780 : if (msr & X2APIC_ENABLE)
1781 : return;
1782 : wrmsrl(MSR_IA32_APICBASE, msr | X2APIC_ENABLE);
1783 : printk_once(KERN_INFO "x2apic enabled\n");
1784 : }
1785 :
1786 : static int __init setup_nox2apic(char *str)
1787 : {
1788 : if (x2apic_enabled()) {
1789 : int apicid = native_apic_msr_read(APIC_ID);
1790 :
1791 : if (apicid >= 255) {
1792 : pr_warn("Apicid: %08x, cannot enforce nox2apic\n",
1793 : apicid);
1794 : return 0;
1795 : }
1796 : pr_warn("x2apic already enabled.\n");
1797 : __x2apic_disable();
1798 : }
1799 : setup_clear_cpu_cap(X86_FEATURE_X2APIC);
1800 : x2apic_state = X2APIC_DISABLED;
1801 : x2apic_mode = 0;
1802 : return 0;
1803 : }
1804 : early_param("nox2apic", setup_nox2apic);
1805 :
1806 : /* Called from cpu_init() to enable x2apic on (secondary) cpus */
1807 : void x2apic_setup(void)
1808 : {
1809 : /*
1810 : * If x2apic is not in ON state, disable it if already enabled
1811 : * from BIOS.
1812 : */
1813 : if (x2apic_state != X2APIC_ON) {
1814 : __x2apic_disable();
1815 : return;
1816 : }
1817 : __x2apic_enable();
1818 : }
1819 :
1820 : static __init void x2apic_disable(void)
1821 : {
1822 : u32 x2apic_id, state = x2apic_state;
1823 :
1824 : x2apic_mode = 0;
1825 : x2apic_state = X2APIC_DISABLED;
1826 :
1827 : if (state != X2APIC_ON)
1828 : return;
1829 :
1830 : x2apic_id = read_apic_id();
1831 : if (x2apic_id >= 255)
1832 : panic("Cannot disable x2apic, id: %08x\n", x2apic_id);
1833 :
1834 : __x2apic_disable();
1835 : register_lapic_address(mp_lapic_addr);
1836 : }
1837 :
1838 : static __init void x2apic_enable(void)
1839 : {
1840 : if (x2apic_state != X2APIC_OFF)
1841 : return;
1842 :
1843 : x2apic_mode = 1;
1844 : x2apic_state = X2APIC_ON;
1845 : __x2apic_enable();
1846 : }
1847 :
1848 : static __init void try_to_enable_x2apic(int remap_mode)
1849 : {
1850 : if (x2apic_state == X2APIC_DISABLED)
1851 : return;
1852 :
1853 : if (remap_mode != IRQ_REMAP_X2APIC_MODE) {
1854 : u32 apic_limit = 255;
1855 :
1856 : /*
1857 : * Using X2APIC without IR is not architecturally supported
1858 : * on bare metal but may be supported in guests.
1859 : */
1860 : if (!x86_init.hyper.x2apic_available()) {
1861 : pr_info("x2apic: IRQ remapping doesn't support X2APIC mode\n");
1862 : x2apic_disable();
1863 : return;
1864 : }
1865 :
1866 : /*
1867 : * If the hypervisor supports extended destination ID in
1868 : * MSI, that increases the maximum APIC ID that can be
1869 : * used for non-remapped IRQ domains.
1870 : */
1871 : if (x86_init.hyper.msi_ext_dest_id()) {
1872 : virt_ext_dest_id = 1;
1873 : apic_limit = 32767;
1874 : }
1875 :
1876 : /*
1877 : * Without IR, all CPUs can be addressed by IOAPIC/MSI only
1878 : * in physical mode, and CPUs with an APIC ID that cannnot
1879 : * be addressed must not be brought online.
1880 : */
1881 : x2apic_set_max_apicid(apic_limit);
1882 : x2apic_phys = 1;
1883 : }
1884 : x2apic_enable();
1885 : }
1886 :
1887 : void __init check_x2apic(void)
1888 : {
1889 : if (x2apic_enabled()) {
1890 : pr_info("x2apic: enabled by BIOS, switching to x2apic ops\n");
1891 : x2apic_mode = 1;
1892 : x2apic_state = X2APIC_ON;
1893 : } else if (!boot_cpu_has(X86_FEATURE_X2APIC)) {
1894 : x2apic_state = X2APIC_DISABLED;
1895 : }
1896 : }
1897 : #else /* CONFIG_X86_X2APIC */
1898 1 : static int __init validate_x2apic(void)
1899 : {
1900 1 : if (!apic_is_x2apic_enabled())
1901 1 : return 0;
1902 : /*
1903 : * Checkme: Can we simply turn off x2apic here instead of panic?
1904 : */
1905 0 : panic("BIOS has enabled x2apic but kernel doesn't support x2apic, please disable x2apic in BIOS.\n");
1906 : }
1907 : early_initcall(validate_x2apic);
1908 :
1909 : static inline void try_to_enable_x2apic(int remap_mode) { }
1910 : static inline void __x2apic_enable(void) { }
1911 : #endif /* !CONFIG_X86_X2APIC */
1912 :
1913 1 : void __init enable_IR_x2apic(void)
1914 : {
1915 1 : unsigned long flags;
1916 1 : int ret, ir_stat;
1917 :
1918 1 : if (skip_ioapic_setup) {
1919 0 : pr_info("Not enabling interrupt remapping due to skipped IO-APIC setup\n");
1920 0 : return;
1921 : }
1922 :
1923 1 : ir_stat = irq_remapping_prepare();
1924 : if (ir_stat < 0 && !x2apic_supported())
1925 : return;
1926 :
1927 : ret = save_ioapic_entries();
1928 : if (ret) {
1929 : pr_info("Saving IO-APIC state failed: %d\n", ret);
1930 : return;
1931 : }
1932 :
1933 : local_irq_save(flags);
1934 : legacy_pic->mask_all();
1935 : mask_ioapic_entries();
1936 :
1937 : /* If irq_remapping_prepare() succeeded, try to enable it */
1938 : if (ir_stat >= 0)
1939 : ir_stat = irq_remapping_enable();
1940 : /* ir_stat contains the remap mode or an error code */
1941 : try_to_enable_x2apic(ir_stat);
1942 :
1943 : if (ir_stat < 0)
1944 : restore_ioapic_entries();
1945 : legacy_pic->restore_mask();
1946 1 : local_irq_restore(flags);
1947 : }
1948 :
1949 : #ifdef CONFIG_X86_64
1950 : /*
1951 : * Detect and enable local APICs on non-SMP boards.
1952 : * Original code written by Keir Fraser.
1953 : * On AMD64 we trust the BIOS - if it says no APIC it is likely
1954 : * not correctly set up (usually the APIC timer won't work etc.)
1955 : */
1956 0 : static int __init detect_init_APIC(void)
1957 : {
1958 0 : if (!boot_cpu_has(X86_FEATURE_APIC)) {
1959 0 : pr_info("No local APIC present\n");
1960 0 : return -1;
1961 : }
1962 :
1963 0 : mp_lapic_addr = APIC_DEFAULT_PHYS_BASE;
1964 0 : return 0;
1965 : }
1966 : #else
1967 :
1968 : static int __init apic_verify(void)
1969 : {
1970 : u32 features, h, l;
1971 :
1972 : /*
1973 : * The APIC feature bit should now be enabled
1974 : * in `cpuid'
1975 : */
1976 : features = cpuid_edx(1);
1977 : if (!(features & (1 << X86_FEATURE_APIC))) {
1978 : pr_warn("Could not enable APIC!\n");
1979 : return -1;
1980 : }
1981 : set_cpu_cap(&boot_cpu_data, X86_FEATURE_APIC);
1982 : mp_lapic_addr = APIC_DEFAULT_PHYS_BASE;
1983 :
1984 : /* The BIOS may have set up the APIC at some other address */
1985 : if (boot_cpu_data.x86 >= 6) {
1986 : rdmsr(MSR_IA32_APICBASE, l, h);
1987 : if (l & MSR_IA32_APICBASE_ENABLE)
1988 : mp_lapic_addr = l & MSR_IA32_APICBASE_BASE;
1989 : }
1990 :
1991 : pr_info("Found and enabled local APIC!\n");
1992 : return 0;
1993 : }
1994 :
1995 : int __init apic_force_enable(unsigned long addr)
1996 : {
1997 : u32 h, l;
1998 :
1999 : if (disable_apic)
2000 : return -1;
2001 :
2002 : /*
2003 : * Some BIOSes disable the local APIC in the APIC_BASE
2004 : * MSR. This can only be done in software for Intel P6 or later
2005 : * and AMD K7 (Model > 1) or later.
2006 : */
2007 : if (boot_cpu_data.x86 >= 6) {
2008 : rdmsr(MSR_IA32_APICBASE, l, h);
2009 : if (!(l & MSR_IA32_APICBASE_ENABLE)) {
2010 : pr_info("Local APIC disabled by BIOS -- reenabling.\n");
2011 : l &= ~MSR_IA32_APICBASE_BASE;
2012 : l |= MSR_IA32_APICBASE_ENABLE | addr;
2013 : wrmsr(MSR_IA32_APICBASE, l, h);
2014 : enabled_via_apicbase = 1;
2015 : }
2016 : }
2017 : return apic_verify();
2018 : }
2019 :
2020 : /*
2021 : * Detect and initialize APIC
2022 : */
2023 : static int __init detect_init_APIC(void)
2024 : {
2025 : /* Disabled by kernel option? */
2026 : if (disable_apic)
2027 : return -1;
2028 :
2029 : switch (boot_cpu_data.x86_vendor) {
2030 : case X86_VENDOR_AMD:
2031 : if ((boot_cpu_data.x86 == 6 && boot_cpu_data.x86_model > 1) ||
2032 : (boot_cpu_data.x86 >= 15))
2033 : break;
2034 : goto no_apic;
2035 : case X86_VENDOR_HYGON:
2036 : break;
2037 : case X86_VENDOR_INTEL:
2038 : if (boot_cpu_data.x86 == 6 || boot_cpu_data.x86 == 15 ||
2039 : (boot_cpu_data.x86 == 5 && boot_cpu_has(X86_FEATURE_APIC)))
2040 : break;
2041 : goto no_apic;
2042 : default:
2043 : goto no_apic;
2044 : }
2045 :
2046 : if (!boot_cpu_has(X86_FEATURE_APIC)) {
2047 : /*
2048 : * Over-ride BIOS and try to enable the local APIC only if
2049 : * "lapic" specified.
2050 : */
2051 : if (!force_enable_local_apic) {
2052 : pr_info("Local APIC disabled by BIOS -- "
2053 : "you can enable it with \"lapic\"\n");
2054 : return -1;
2055 : }
2056 : if (apic_force_enable(APIC_DEFAULT_PHYS_BASE))
2057 : return -1;
2058 : } else {
2059 : if (apic_verify())
2060 : return -1;
2061 : }
2062 :
2063 : apic_pm_activate();
2064 :
2065 : return 0;
2066 :
2067 : no_apic:
2068 : pr_info("No local APIC present or hardware disabled\n");
2069 : return -1;
2070 : }
2071 : #endif
2072 :
2073 : /**
2074 : * init_apic_mappings - initialize APIC mappings
2075 : */
2076 1 : void __init init_apic_mappings(void)
2077 : {
2078 1 : unsigned int new_apicid;
2079 :
2080 1 : if (apic_validate_deadline_timer())
2081 1 : pr_info("TSC deadline timer available\n");
2082 :
2083 1 : if (x2apic_mode) {
2084 : boot_cpu_physical_apicid = read_apic_id();
2085 : return;
2086 : }
2087 :
2088 : /* If no local APIC can be found return early */
2089 1 : if (!smp_found_config && detect_init_APIC()) {
2090 : /* lets NOP'ify apic operations */
2091 0 : pr_info("APIC: disable apic facility\n");
2092 0 : apic_disable();
2093 : } else {
2094 1 : apic_phys = mp_lapic_addr;
2095 :
2096 : /*
2097 : * If the system has ACPI MADT tables or MP info, the LAPIC
2098 : * address is already registered.
2099 : */
2100 1 : if (!acpi_lapic && !smp_found_config)
2101 0 : register_lapic_address(apic_phys);
2102 : }
2103 :
2104 : /*
2105 : * Fetch the APIC ID of the BSP in case we have a
2106 : * default configuration (or the MP table is broken).
2107 : */
2108 1 : new_apicid = read_apic_id();
2109 1 : if (boot_cpu_physical_apicid != new_apicid) {
2110 0 : boot_cpu_physical_apicid = new_apicid;
2111 : /*
2112 : * yeah -- we lie about apic_version
2113 : * in case if apic was disabled via boot option
2114 : * but it's not a problem for SMP compiled kernel
2115 : * since apic_intr_mode_select is prepared for such
2116 : * a case and disable smp mode
2117 : */
2118 0 : boot_cpu_apic_version = GET_APIC_VERSION(apic_read(APIC_LVR));
2119 : }
2120 : }
2121 :
2122 1 : void __init register_lapic_address(unsigned long address)
2123 : {
2124 1 : mp_lapic_addr = address;
2125 :
2126 1 : if (!x2apic_mode) {
2127 1 : set_fixmap_nocache(FIX_APIC_BASE, address);
2128 1 : apic_printk(APIC_VERBOSE, "mapped APIC to %16lx (%16lx)\n",
2129 : APIC_BASE, address);
2130 : }
2131 1 : if (boot_cpu_physical_apicid == -1U) {
2132 1 : boot_cpu_physical_apicid = read_apic_id();
2133 1 : boot_cpu_apic_version = GET_APIC_VERSION(apic_read(APIC_LVR));
2134 : }
2135 1 : }
2136 :
2137 : /*
2138 : * Local APIC interrupts
2139 : */
2140 :
2141 : /*
2142 : * Common handling code for spurious_interrupt and spurious_vector entry
2143 : * points below. No point in allowing the compiler to inline it twice.
2144 : */
2145 0 : static noinline void handle_spurious_interrupt(u8 vector)
2146 : {
2147 0 : u32 v;
2148 :
2149 0 : trace_spurious_apic_entry(vector);
2150 :
2151 0 : inc_irq_stat(irq_spurious_count);
2152 :
2153 : /*
2154 : * If this is a spurious interrupt then do not acknowledge
2155 : */
2156 0 : if (vector == SPURIOUS_APIC_VECTOR) {
2157 : /* See SDM vol 3 */
2158 0 : pr_info("Spurious APIC interrupt (vector 0xFF) on CPU#%d, should never happen.\n",
2159 : smp_processor_id());
2160 0 : goto out;
2161 : }
2162 :
2163 : /*
2164 : * If it is a vectored one, verify it's set in the ISR. If set,
2165 : * acknowledge it.
2166 : */
2167 0 : v = apic_read(APIC_ISR + ((vector & ~0x1f) >> 1));
2168 0 : if (v & (1 << (vector & 0x1f))) {
2169 0 : pr_info("Spurious interrupt (vector 0x%02x) on CPU#%d. Acked\n",
2170 : vector, smp_processor_id());
2171 0 : ack_APIC_irq();
2172 : } else {
2173 0 : pr_info("Spurious interrupt (vector 0x%02x) on CPU#%d. Not pending!\n",
2174 : vector, smp_processor_id());
2175 : }
2176 0 : out:
2177 0 : trace_spurious_apic_exit(vector);
2178 0 : }
2179 :
2180 : /**
2181 : * spurious_interrupt - Catch all for interrupts raised on unused vectors
2182 : * @regs: Pointer to pt_regs on stack
2183 : * @vector: The vector number
2184 : *
2185 : * This is invoked from ASM entry code to catch all interrupts which
2186 : * trigger on an entry which is routed to the common_spurious idtentry
2187 : * point.
2188 : */
2189 0 : DEFINE_IDTENTRY_IRQ(spurious_interrupt)
2190 : {
2191 0 : handle_spurious_interrupt(vector);
2192 0 : }
2193 :
2194 0 : DEFINE_IDTENTRY_SYSVEC(sysvec_spurious_apic_interrupt)
2195 : {
2196 0 : handle_spurious_interrupt(SPURIOUS_APIC_VECTOR);
2197 0 : }
2198 :
2199 : /*
2200 : * This interrupt should never happen with our APIC/SMP architecture
2201 : */
2202 0 : DEFINE_IDTENTRY_SYSVEC(sysvec_error_interrupt)
2203 : {
2204 0 : static const char * const error_interrupt_reason[] = {
2205 : "Send CS error", /* APIC Error Bit 0 */
2206 : "Receive CS error", /* APIC Error Bit 1 */
2207 : "Send accept error", /* APIC Error Bit 2 */
2208 : "Receive accept error", /* APIC Error Bit 3 */
2209 : "Redirectable IPI", /* APIC Error Bit 4 */
2210 : "Send illegal vector", /* APIC Error Bit 5 */
2211 : "Received illegal vector", /* APIC Error Bit 6 */
2212 : "Illegal register address", /* APIC Error Bit 7 */
2213 : };
2214 0 : u32 v, i = 0;
2215 :
2216 0 : trace_error_apic_entry(ERROR_APIC_VECTOR);
2217 :
2218 : /* First tickle the hardware, only then report what went on. -- REW */
2219 0 : if (lapic_get_maxlvt() > 3) /* Due to the Pentium erratum 3AP. */
2220 0 : apic_write(APIC_ESR, 0);
2221 0 : v = apic_read(APIC_ESR);
2222 0 : ack_APIC_irq();
2223 0 : atomic_inc(&irq_err_count);
2224 :
2225 0 : apic_printk(APIC_DEBUG, KERN_DEBUG "APIC error on CPU%d: %02x",
2226 : smp_processor_id(), v);
2227 :
2228 0 : v &= 0xff;
2229 0 : while (v) {
2230 0 : if (v & 0x1)
2231 0 : apic_printk(APIC_DEBUG, KERN_CONT " : %s", error_interrupt_reason[i]);
2232 0 : i++;
2233 0 : v >>= 1;
2234 : }
2235 :
2236 0 : apic_printk(APIC_DEBUG, KERN_CONT "\n");
2237 :
2238 0 : trace_error_apic_exit(ERROR_APIC_VECTOR);
2239 0 : }
2240 :
2241 : /**
2242 : * connect_bsp_APIC - attach the APIC to the interrupt system
2243 : */
2244 1 : static void __init connect_bsp_APIC(void)
2245 : {
2246 : #ifdef CONFIG_X86_32
2247 : if (pic_mode) {
2248 : /*
2249 : * Do not trust the local APIC being empty at bootup.
2250 : */
2251 : clear_local_APIC();
2252 : /*
2253 : * PIC mode, enable APIC mode in the IMCR, i.e. connect BSP's
2254 : * local APIC to INT and NMI lines.
2255 : */
2256 : apic_printk(APIC_VERBOSE, "leaving PIC mode, "
2257 : "enabling APIC mode.\n");
2258 : imcr_pic_to_apic();
2259 : }
2260 : #endif
2261 1 : }
2262 :
2263 : /**
2264 : * disconnect_bsp_APIC - detach the APIC from the interrupt system
2265 : * @virt_wire_setup: indicates, whether virtual wire mode is selected
2266 : *
2267 : * Virtual wire mode is necessary to deliver legacy interrupts even when the
2268 : * APIC is disabled.
2269 : */
2270 0 : void disconnect_bsp_APIC(int virt_wire_setup)
2271 : {
2272 0 : unsigned int value;
2273 :
2274 : #ifdef CONFIG_X86_32
2275 : if (pic_mode) {
2276 : /*
2277 : * Put the board back into PIC mode (has an effect only on
2278 : * certain older boards). Note that APIC interrupts, including
2279 : * IPIs, won't work beyond this point! The only exception are
2280 : * INIT IPIs.
2281 : */
2282 : apic_printk(APIC_VERBOSE, "disabling APIC mode, "
2283 : "entering PIC mode.\n");
2284 : imcr_apic_to_pic();
2285 : return;
2286 : }
2287 : #endif
2288 :
2289 : /* Go back to Virtual Wire compatibility mode */
2290 :
2291 : /* For the spurious interrupt use vector F, and enable it */
2292 0 : value = apic_read(APIC_SPIV);
2293 0 : value &= ~APIC_VECTOR_MASK;
2294 0 : value |= APIC_SPIV_APIC_ENABLED;
2295 0 : value |= 0xf;
2296 0 : apic_write(APIC_SPIV, value);
2297 :
2298 0 : if (!virt_wire_setup) {
2299 : /*
2300 : * For LVT0 make it edge triggered, active high,
2301 : * external and enabled
2302 : */
2303 0 : value = apic_read(APIC_LVT0);
2304 0 : value &= ~(APIC_MODE_MASK | APIC_SEND_PENDING |
2305 : APIC_INPUT_POLARITY | APIC_LVT_REMOTE_IRR |
2306 : APIC_LVT_LEVEL_TRIGGER | APIC_LVT_MASKED);
2307 0 : value |= APIC_LVT_REMOTE_IRR | APIC_SEND_PENDING;
2308 0 : value = SET_APIC_DELIVERY_MODE(value, APIC_MODE_EXTINT);
2309 0 : apic_write(APIC_LVT0, value);
2310 : } else {
2311 : /* Disable LVT0 */
2312 0 : apic_write(APIC_LVT0, APIC_LVT_MASKED);
2313 : }
2314 :
2315 : /*
2316 : * For LVT1 make it edge triggered, active high,
2317 : * nmi and enabled
2318 : */
2319 0 : value = apic_read(APIC_LVT1);
2320 0 : value &= ~(APIC_MODE_MASK | APIC_SEND_PENDING |
2321 : APIC_INPUT_POLARITY | APIC_LVT_REMOTE_IRR |
2322 : APIC_LVT_LEVEL_TRIGGER | APIC_LVT_MASKED);
2323 0 : value |= APIC_LVT_REMOTE_IRR | APIC_SEND_PENDING;
2324 0 : value = SET_APIC_DELIVERY_MODE(value, APIC_MODE_NMI);
2325 0 : apic_write(APIC_LVT1, value);
2326 0 : }
2327 :
2328 : /*
2329 : * The number of allocated logical CPU IDs. Since logical CPU IDs are allocated
2330 : * contiguously, it equals to current allocated max logical CPU ID plus 1.
2331 : * All allocated CPU IDs should be in the [0, nr_logical_cpuids) range,
2332 : * so the maximum of nr_logical_cpuids is nr_cpu_ids.
2333 : *
2334 : * NOTE: Reserve 0 for BSP.
2335 : */
2336 : static int nr_logical_cpuids = 1;
2337 :
2338 : /*
2339 : * Used to store mapping between logical CPU IDs and APIC IDs.
2340 : */
2341 : static int cpuid_to_apicid[] = {
2342 : [0 ... NR_CPUS - 1] = -1,
2343 : };
2344 :
2345 : #ifdef CONFIG_SMP
2346 : /**
2347 : * apic_id_is_primary_thread - Check whether APIC ID belongs to a primary thread
2348 : * @apicid: APIC ID to check
2349 : */
2350 6 : bool apic_id_is_primary_thread(unsigned int apicid)
2351 : {
2352 6 : u32 mask;
2353 :
2354 6 : if (smp_num_siblings == 1)
2355 : return true;
2356 : /* Isolate the SMT bit(s) in the APICID and check for 0 */
2357 0 : mask = (1U << (fls(smp_num_siblings) - 1)) - 1;
2358 0 : return !(apicid & mask);
2359 : }
2360 : #endif
2361 :
2362 : /*
2363 : * Should use this API to allocate logical CPU IDs to keep nr_logical_cpuids
2364 : * and cpuid_to_apicid[] synchronized.
2365 : */
2366 3 : static int allocate_logical_cpuid(int apicid)
2367 : {
2368 3 : int i;
2369 :
2370 : /*
2371 : * cpuid <-> apicid mapping is persistent, so when a cpu is up,
2372 : * check if the kernel has allocated a cpuid for it.
2373 : */
2374 9 : for (i = 0; i < nr_logical_cpuids; i++) {
2375 6 : if (cpuid_to_apicid[i] == apicid)
2376 0 : return i;
2377 : }
2378 :
2379 : /* Allocate a new cpuid. */
2380 3 : if (nr_logical_cpuids >= nr_cpu_ids) {
2381 0 : WARN_ONCE(1, "APIC: NR_CPUS/possible_cpus limit of %u reached. "
2382 : "Processor %d/0x%x and the rest are ignored.\n",
2383 : nr_cpu_ids, nr_logical_cpuids, apicid);
2384 0 : return -EINVAL;
2385 : }
2386 :
2387 3 : cpuid_to_apicid[nr_logical_cpuids] = apicid;
2388 3 : return nr_logical_cpuids++;
2389 : }
2390 :
2391 4 : int generic_processor_info(int apicid, int version)
2392 : {
2393 4 : int cpu, max = nr_cpu_ids;
2394 4 : bool boot_cpu_detected = physid_isset(boot_cpu_physical_apicid,
2395 : phys_cpu_present_map);
2396 :
2397 : /*
2398 : * boot_cpu_physical_apicid is designed to have the apicid
2399 : * returned by read_apic_id(), i.e, the apicid of the
2400 : * currently booting-up processor. However, on some platforms,
2401 : * it is temporarily modified by the apicid reported as BSP
2402 : * through MP table. Concretely:
2403 : *
2404 : * - arch/x86/kernel/mpparse.c: MP_processor_info()
2405 : * - arch/x86/mm/amdtopology.c: amd_numa_init()
2406 : *
2407 : * This function is executed with the modified
2408 : * boot_cpu_physical_apicid. So, disabled_cpu_apicid kernel
2409 : * parameter doesn't work to disable APs on kdump 2nd kernel.
2410 : *
2411 : * Since fixing handling of boot_cpu_physical_apicid requires
2412 : * another discussion and tests on each platform, we leave it
2413 : * for now and here we use read_apic_id() directly in this
2414 : * function, generic_processor_info().
2415 : */
2416 4 : if (disabled_cpu_apicid != BAD_APICID &&
2417 0 : disabled_cpu_apicid != read_apic_id() &&
2418 0 : disabled_cpu_apicid == apicid) {
2419 0 : int thiscpu = num_processors + disabled_cpus;
2420 :
2421 0 : pr_warn("APIC: Disabling requested cpu."
2422 : " Processor %d/0x%x ignored.\n", thiscpu, apicid);
2423 :
2424 0 : disabled_cpus++;
2425 0 : return -ENODEV;
2426 : }
2427 :
2428 : /*
2429 : * If boot cpu has not been detected yet, then only allow upto
2430 : * nr_cpu_ids - 1 processors and keep one slot free for boot cpu
2431 : */
2432 4 : if (!boot_cpu_detected && num_processors >= nr_cpu_ids - 1 &&
2433 0 : apicid != boot_cpu_physical_apicid) {
2434 0 : int thiscpu = max + disabled_cpus - 1;
2435 :
2436 0 : pr_warn("APIC: NR_CPUS/possible_cpus limit of %i almost"
2437 : " reached. Keeping one slot for boot cpu."
2438 : " Processor %d/0x%x ignored.\n", max, thiscpu, apicid);
2439 :
2440 0 : disabled_cpus++;
2441 0 : return -ENODEV;
2442 : }
2443 :
2444 4 : if (num_processors >= nr_cpu_ids) {
2445 0 : int thiscpu = max + disabled_cpus;
2446 :
2447 0 : pr_warn("APIC: NR_CPUS/possible_cpus limit of %i reached. "
2448 : "Processor %d/0x%x ignored.\n", max, thiscpu, apicid);
2449 :
2450 0 : disabled_cpus++;
2451 0 : return -EINVAL;
2452 : }
2453 :
2454 4 : if (apicid == boot_cpu_physical_apicid) {
2455 : /*
2456 : * x86_bios_cpu_apicid is required to have processors listed
2457 : * in same order as logical cpu numbers. Hence the first
2458 : * entry is BSP, and so on.
2459 : * boot_cpu_init() already hold bit 0 in cpu_present_mask
2460 : * for BSP.
2461 : */
2462 1 : cpu = 0;
2463 :
2464 : /* Logical cpuid 0 is reserved for BSP. */
2465 1 : cpuid_to_apicid[0] = apicid;
2466 : } else {
2467 3 : cpu = allocate_logical_cpuid(apicid);
2468 3 : if (cpu < 0) {
2469 0 : disabled_cpus++;
2470 0 : return -EINVAL;
2471 : }
2472 : }
2473 :
2474 : /*
2475 : * Validate version
2476 : */
2477 4 : if (version == 0x0) {
2478 0 : pr_warn("BIOS bug: APIC version is 0 for CPU %d/0x%x, fixing up to 0x10\n",
2479 : cpu, apicid);
2480 0 : version = 0x10;
2481 : }
2482 :
2483 4 : if (version != boot_cpu_apic_version) {
2484 0 : pr_warn("BIOS bug: APIC version mismatch, boot CPU: %x, CPU %d: version %x\n",
2485 : boot_cpu_apic_version, cpu, version);
2486 : }
2487 :
2488 4 : if (apicid > max_physical_apicid)
2489 3 : max_physical_apicid = apicid;
2490 :
2491 : #if defined(CONFIG_SMP) || defined(CONFIG_X86_64)
2492 4 : early_per_cpu(x86_cpu_to_apicid, cpu) = apicid;
2493 4 : early_per_cpu(x86_bios_cpu_apicid, cpu) = apicid;
2494 : #endif
2495 : #ifdef CONFIG_X86_32
2496 : early_per_cpu(x86_cpu_to_logical_apicid, cpu) =
2497 : apic->x86_32_early_logical_apicid(cpu);
2498 : #endif
2499 4 : set_cpu_possible(cpu, true);
2500 4 : physid_set(apicid, phys_cpu_present_map);
2501 4 : set_cpu_present(cpu, true);
2502 4 : num_processors++;
2503 :
2504 4 : return cpu;
2505 : }
2506 :
2507 1 : int hard_smp_processor_id(void)
2508 : {
2509 1 : return read_apic_id();
2510 : }
2511 :
2512 6 : void __irq_msi_compose_msg(struct irq_cfg *cfg, struct msi_msg *msg,
2513 : bool dmar)
2514 : {
2515 6 : memset(msg, 0, sizeof(*msg));
2516 :
2517 6 : msg->arch_addr_lo.base_address = X86_MSI_BASE_ADDRESS_LOW;
2518 6 : msg->arch_addr_lo.dest_mode_logical = apic->dest_mode_logical;
2519 6 : msg->arch_addr_lo.destid_0_7 = cfg->dest_apicid & 0xFF;
2520 :
2521 6 : msg->arch_data.delivery_mode = APIC_DELIVERY_MODE_FIXED;
2522 6 : msg->arch_data.vector = cfg->vector;
2523 :
2524 6 : msg->address_hi = X86_MSI_BASE_ADDRESS_HIGH;
2525 : /*
2526 : * Only the IOMMU itself can use the trick of putting destination
2527 : * APIC ID into the high bits of the address. Anything else would
2528 : * just be writing to memory if it tried that, and needs IR to
2529 : * address APICs which can't be addressed in the normal 32-bit
2530 : * address range at 0xFFExxxxx. That is typically just 8 bits, but
2531 : * some hypervisors allow the extended destination ID field in bits
2532 : * 5-11 to be used, giving support for 15 bits of APIC IDs in total.
2533 : */
2534 6 : if (dmar)
2535 0 : msg->arch_addr_hi.destid_8_31 = cfg->dest_apicid >> 8;
2536 6 : else if (virt_ext_dest_id && cfg->dest_apicid < 0x8000)
2537 0 : msg->arch_addr_lo.virt_destid_8_14 = cfg->dest_apicid >> 8;
2538 : else
2539 6 : WARN_ON_ONCE(cfg->dest_apicid > 0xFF);
2540 6 : }
2541 :
2542 0 : u32 x86_msi_msg_get_destid(struct msi_msg *msg, bool extid)
2543 : {
2544 0 : u32 dest = msg->arch_addr_lo.destid_0_7;
2545 :
2546 0 : if (extid)
2547 0 : dest |= msg->arch_addr_hi.destid_8_31 << 8;
2548 0 : return dest;
2549 : }
2550 : EXPORT_SYMBOL_GPL(x86_msi_msg_get_destid);
2551 :
2552 : /*
2553 : * Override the generic EOI implementation with an optimized version.
2554 : * Only called during early boot when only one CPU is active and with
2555 : * interrupts disabled, so we know this does not race with actual APIC driver
2556 : * use.
2557 : */
2558 1 : void __init apic_set_eoi_write(void (*eoi_write)(u32 reg, u32 v))
2559 : {
2560 1 : struct apic **drv;
2561 :
2562 3 : for (drv = __apicdrivers; drv < __apicdrivers_end; drv++) {
2563 : /* Should happen once for each apic */
2564 2 : WARN_ON((*drv)->eoi_write == eoi_write);
2565 2 : (*drv)->native_eoi_write = (*drv)->eoi_write;
2566 2 : (*drv)->eoi_write = eoi_write;
2567 : }
2568 1 : }
2569 :
2570 0 : static void __init apic_bsp_up_setup(void)
2571 : {
2572 : #ifdef CONFIG_X86_64
2573 0 : apic_write(APIC_ID, apic->set_apic_id(boot_cpu_physical_apicid));
2574 : #else
2575 : /*
2576 : * Hack: In case of kdump, after a crash, kernel might be booting
2577 : * on a cpu with non-zero lapic id. But boot_cpu_physical_apicid
2578 : * might be zero if read from MP tables. Get it from LAPIC.
2579 : */
2580 : # ifdef CONFIG_CRASH_DUMP
2581 : boot_cpu_physical_apicid = read_apic_id();
2582 : # endif
2583 : #endif
2584 0 : physid_set_mask_of_physid(boot_cpu_physical_apicid, &phys_cpu_present_map);
2585 0 : }
2586 :
2587 : /**
2588 : * apic_bsp_setup - Setup function for local apic and io-apic
2589 : * @upmode: Force UP mode (for APIC_init_uniprocessor)
2590 : */
2591 1 : static void __init apic_bsp_setup(bool upmode)
2592 : {
2593 1 : connect_bsp_APIC();
2594 1 : if (upmode)
2595 0 : apic_bsp_up_setup();
2596 1 : setup_local_APIC();
2597 :
2598 1 : enable_IO_APIC();
2599 1 : end_local_APIC_setup();
2600 1 : irq_remap_enable_fault_handling();
2601 1 : setup_IO_APIC();
2602 1 : }
2603 :
2604 : #ifdef CONFIG_UP_LATE_INIT
2605 : void __init up_late_init(void)
2606 : {
2607 : if (apic_intr_mode == APIC_PIC)
2608 : return;
2609 :
2610 : /* Setup local timer */
2611 : x86_init.timers.setup_percpu_clockev();
2612 : }
2613 : #endif
2614 :
2615 : /*
2616 : * Power management
2617 : */
2618 : #ifdef CONFIG_PM
2619 :
2620 : static struct {
2621 : /*
2622 : * 'active' is true if the local APIC was enabled by us and
2623 : * not the BIOS; this signifies that we are also responsible
2624 : * for disabling it before entering apm/acpi suspend
2625 : */
2626 : int active;
2627 : /* r/w apic fields */
2628 : unsigned int apic_id;
2629 : unsigned int apic_taskpri;
2630 : unsigned int apic_ldr;
2631 : unsigned int apic_dfr;
2632 : unsigned int apic_spiv;
2633 : unsigned int apic_lvtt;
2634 : unsigned int apic_lvtpc;
2635 : unsigned int apic_lvt0;
2636 : unsigned int apic_lvt1;
2637 : unsigned int apic_lvterr;
2638 : unsigned int apic_tmict;
2639 : unsigned int apic_tdcr;
2640 : unsigned int apic_thmr;
2641 : unsigned int apic_cmci;
2642 : } apic_pm_state;
2643 :
2644 : static int lapic_suspend(void)
2645 : {
2646 : unsigned long flags;
2647 : int maxlvt;
2648 :
2649 : if (!apic_pm_state.active)
2650 : return 0;
2651 :
2652 : maxlvt = lapic_get_maxlvt();
2653 :
2654 : apic_pm_state.apic_id = apic_read(APIC_ID);
2655 : apic_pm_state.apic_taskpri = apic_read(APIC_TASKPRI);
2656 : apic_pm_state.apic_ldr = apic_read(APIC_LDR);
2657 : apic_pm_state.apic_dfr = apic_read(APIC_DFR);
2658 : apic_pm_state.apic_spiv = apic_read(APIC_SPIV);
2659 : apic_pm_state.apic_lvtt = apic_read(APIC_LVTT);
2660 : if (maxlvt >= 4)
2661 : apic_pm_state.apic_lvtpc = apic_read(APIC_LVTPC);
2662 : apic_pm_state.apic_lvt0 = apic_read(APIC_LVT0);
2663 : apic_pm_state.apic_lvt1 = apic_read(APIC_LVT1);
2664 : apic_pm_state.apic_lvterr = apic_read(APIC_LVTERR);
2665 : apic_pm_state.apic_tmict = apic_read(APIC_TMICT);
2666 : apic_pm_state.apic_tdcr = apic_read(APIC_TDCR);
2667 : #ifdef CONFIG_X86_THERMAL_VECTOR
2668 : if (maxlvt >= 5)
2669 : apic_pm_state.apic_thmr = apic_read(APIC_LVTTHMR);
2670 : #endif
2671 : #ifdef CONFIG_X86_MCE_INTEL
2672 : if (maxlvt >= 6)
2673 : apic_pm_state.apic_cmci = apic_read(APIC_LVTCMCI);
2674 : #endif
2675 :
2676 : local_irq_save(flags);
2677 :
2678 : /*
2679 : * Mask IOAPIC before disabling the local APIC to prevent stale IRR
2680 : * entries on some implementations.
2681 : */
2682 : mask_ioapic_entries();
2683 :
2684 : disable_local_APIC();
2685 :
2686 : irq_remapping_disable();
2687 :
2688 : local_irq_restore(flags);
2689 : return 0;
2690 : }
2691 :
2692 : static void lapic_resume(void)
2693 : {
2694 : unsigned int l, h;
2695 : unsigned long flags;
2696 : int maxlvt;
2697 :
2698 : if (!apic_pm_state.active)
2699 : return;
2700 :
2701 : local_irq_save(flags);
2702 :
2703 : /*
2704 : * IO-APIC and PIC have their own resume routines.
2705 : * We just mask them here to make sure the interrupt
2706 : * subsystem is completely quiet while we enable x2apic
2707 : * and interrupt-remapping.
2708 : */
2709 : mask_ioapic_entries();
2710 : legacy_pic->mask_all();
2711 :
2712 : if (x2apic_mode) {
2713 : __x2apic_enable();
2714 : } else {
2715 : /*
2716 : * Make sure the APICBASE points to the right address
2717 : *
2718 : * FIXME! This will be wrong if we ever support suspend on
2719 : * SMP! We'll need to do this as part of the CPU restore!
2720 : */
2721 : if (boot_cpu_data.x86 >= 6) {
2722 : rdmsr(MSR_IA32_APICBASE, l, h);
2723 : l &= ~MSR_IA32_APICBASE_BASE;
2724 : l |= MSR_IA32_APICBASE_ENABLE | mp_lapic_addr;
2725 : wrmsr(MSR_IA32_APICBASE, l, h);
2726 : }
2727 : }
2728 :
2729 : maxlvt = lapic_get_maxlvt();
2730 : apic_write(APIC_LVTERR, ERROR_APIC_VECTOR | APIC_LVT_MASKED);
2731 : apic_write(APIC_ID, apic_pm_state.apic_id);
2732 : apic_write(APIC_DFR, apic_pm_state.apic_dfr);
2733 : apic_write(APIC_LDR, apic_pm_state.apic_ldr);
2734 : apic_write(APIC_TASKPRI, apic_pm_state.apic_taskpri);
2735 : apic_write(APIC_SPIV, apic_pm_state.apic_spiv);
2736 : apic_write(APIC_LVT0, apic_pm_state.apic_lvt0);
2737 : apic_write(APIC_LVT1, apic_pm_state.apic_lvt1);
2738 : #ifdef CONFIG_X86_THERMAL_VECTOR
2739 : if (maxlvt >= 5)
2740 : apic_write(APIC_LVTTHMR, apic_pm_state.apic_thmr);
2741 : #endif
2742 : #ifdef CONFIG_X86_MCE_INTEL
2743 : if (maxlvt >= 6)
2744 : apic_write(APIC_LVTCMCI, apic_pm_state.apic_cmci);
2745 : #endif
2746 : if (maxlvt >= 4)
2747 : apic_write(APIC_LVTPC, apic_pm_state.apic_lvtpc);
2748 : apic_write(APIC_LVTT, apic_pm_state.apic_lvtt);
2749 : apic_write(APIC_TDCR, apic_pm_state.apic_tdcr);
2750 : apic_write(APIC_TMICT, apic_pm_state.apic_tmict);
2751 : apic_write(APIC_ESR, 0);
2752 : apic_read(APIC_ESR);
2753 : apic_write(APIC_LVTERR, apic_pm_state.apic_lvterr);
2754 : apic_write(APIC_ESR, 0);
2755 : apic_read(APIC_ESR);
2756 :
2757 : irq_remapping_reenable(x2apic_mode);
2758 :
2759 : local_irq_restore(flags);
2760 : }
2761 :
2762 : /*
2763 : * This device has no shutdown method - fully functioning local APICs
2764 : * are needed on every CPU up until machine_halt/restart/poweroff.
2765 : */
2766 :
2767 : static struct syscore_ops lapic_syscore_ops = {
2768 : .resume = lapic_resume,
2769 : .suspend = lapic_suspend,
2770 : };
2771 :
2772 : static void apic_pm_activate(void)
2773 : {
2774 : apic_pm_state.active = 1;
2775 : }
2776 :
2777 : static int __init init_lapic_sysfs(void)
2778 : {
2779 : /* XXX: remove suspend/resume procs if !apic_pm_state.active? */
2780 : if (boot_cpu_has(X86_FEATURE_APIC))
2781 : register_syscore_ops(&lapic_syscore_ops);
2782 :
2783 : return 0;
2784 : }
2785 :
2786 : /* local apic needs to resume before other devices access its registers. */
2787 : core_initcall(init_lapic_sysfs);
2788 :
2789 : #else /* CONFIG_PM */
2790 :
2791 1 : static void apic_pm_activate(void) { }
2792 :
2793 : #endif /* CONFIG_PM */
2794 :
2795 : #ifdef CONFIG_X86_64
2796 :
2797 : static int multi_checked;
2798 : static int multi;
2799 :
2800 : static int set_multi(const struct dmi_system_id *d)
2801 : {
2802 : if (multi)
2803 : return 0;
2804 : pr_info("APIC: %s detected, Multi Chassis\n", d->ident);
2805 : multi = 1;
2806 : return 0;
2807 : }
2808 :
2809 : static const struct dmi_system_id multi_dmi_table[] = {
2810 : {
2811 : .callback = set_multi,
2812 : .ident = "IBM System Summit2",
2813 : .matches = {
2814 : DMI_MATCH(DMI_SYS_VENDOR, "IBM"),
2815 : DMI_MATCH(DMI_PRODUCT_NAME, "Summit2"),
2816 : },
2817 : },
2818 : {}
2819 : };
2820 :
2821 7 : static void dmi_check_multi(void)
2822 : {
2823 7 : if (multi_checked)
2824 : return;
2825 :
2826 1 : dmi_check_system(multi_dmi_table);
2827 1 : multi_checked = 1;
2828 : }
2829 :
2830 : /*
2831 : * apic_is_clustered_box() -- Check if we can expect good TSC
2832 : *
2833 : * Thus far, the major user of this is IBM's Summit2 series:
2834 : * Clustered boxes may have unsynced TSC problems if they are
2835 : * multi-chassis.
2836 : * Use DMI to check them
2837 : */
2838 7 : int apic_is_clustered_box(void)
2839 : {
2840 7 : dmi_check_multi();
2841 7 : return multi;
2842 : }
2843 : #endif
2844 :
2845 : /*
2846 : * APIC command line parameters
2847 : */
2848 0 : static int __init setup_disableapic(char *arg)
2849 : {
2850 0 : disable_apic = 1;
2851 0 : setup_clear_cpu_cap(X86_FEATURE_APIC);
2852 0 : return 0;
2853 : }
2854 : early_param("disableapic", setup_disableapic);
2855 :
2856 : /* same as disableapic, for compatibility */
2857 0 : static int __init setup_nolapic(char *arg)
2858 : {
2859 0 : return setup_disableapic(arg);
2860 : }
2861 : early_param("nolapic", setup_nolapic);
2862 :
2863 0 : static int __init parse_lapic_timer_c2_ok(char *arg)
2864 : {
2865 0 : local_apic_timer_c2_ok = 1;
2866 0 : return 0;
2867 : }
2868 : early_param("lapic_timer_c2_ok", parse_lapic_timer_c2_ok);
2869 :
2870 0 : static int __init parse_disable_apic_timer(char *arg)
2871 : {
2872 0 : disable_apic_timer = 1;
2873 0 : return 0;
2874 : }
2875 : early_param("noapictimer", parse_disable_apic_timer);
2876 :
2877 0 : static int __init parse_nolapic_timer(char *arg)
2878 : {
2879 0 : disable_apic_timer = 1;
2880 0 : return 0;
2881 : }
2882 : early_param("nolapic_timer", parse_nolapic_timer);
2883 :
2884 0 : static int __init apic_set_verbosity(char *arg)
2885 : {
2886 0 : if (!arg) {
2887 : #ifdef CONFIG_X86_64
2888 0 : skip_ioapic_setup = 0;
2889 0 : return 0;
2890 : #endif
2891 : return -EINVAL;
2892 : }
2893 :
2894 0 : if (strcmp("debug", arg) == 0)
2895 0 : apic_verbosity = APIC_DEBUG;
2896 0 : else if (strcmp("verbose", arg) == 0)
2897 0 : apic_verbosity = APIC_VERBOSE;
2898 : #ifdef CONFIG_X86_64
2899 : else {
2900 0 : pr_warn("APIC Verbosity level %s not recognised"
2901 : " use apic=verbose or apic=debug\n", arg);
2902 0 : return -EINVAL;
2903 : }
2904 : #endif
2905 :
2906 : return 0;
2907 : }
2908 : early_param("apic", apic_set_verbosity);
2909 :
2910 1 : static int __init lapic_insert_resource(void)
2911 : {
2912 1 : if (!apic_phys)
2913 : return -1;
2914 :
2915 : /* Put local APIC into the resource map. */
2916 1 : lapic_resource.start = apic_phys;
2917 1 : lapic_resource.end = lapic_resource.start + PAGE_SIZE - 1;
2918 1 : insert_resource(&iomem_resource, &lapic_resource);
2919 :
2920 1 : return 0;
2921 : }
2922 :
2923 : /*
2924 : * need call insert after e820__reserve_resources()
2925 : * that is using request_resource
2926 : */
2927 : late_initcall(lapic_insert_resource);
2928 :
2929 0 : static int __init apic_set_disabled_cpu_apicid(char *arg)
2930 : {
2931 0 : if (!arg || !get_option(&arg, &disabled_cpu_apicid))
2932 0 : return -EINVAL;
2933 :
2934 : return 0;
2935 : }
2936 : early_param("disable_cpu_apicid", apic_set_disabled_cpu_apicid);
2937 :
2938 0 : static int __init apic_set_extnmi(char *arg)
2939 : {
2940 0 : if (!arg)
2941 : return -EINVAL;
2942 :
2943 0 : if (!strncmp("all", arg, 3))
2944 0 : apic_extnmi = APIC_EXTNMI_ALL;
2945 0 : else if (!strncmp("none", arg, 4))
2946 0 : apic_extnmi = APIC_EXTNMI_NONE;
2947 0 : else if (!strncmp("bsp", arg, 3))
2948 0 : apic_extnmi = APIC_EXTNMI_BSP;
2949 : else {
2950 0 : pr_warn("Unknown external NMI delivery mode `%s' ignored\n", arg);
2951 0 : return -EINVAL;
2952 : }
2953 :
2954 : return 0;
2955 : }
2956 : early_param("apic_extnmi", apic_set_extnmi);
|
