Line data Source code
1 : // SPDX-License-Identifier: GPL-2.0
2 : /*
3 : * Intel IO-APIC support for multi-Pentium hosts.
4 : *
5 : * Copyright (C) 1997, 1998, 1999, 2000, 2009 Ingo Molnar, Hajnalka Szabo
6 : *
7 : * Many thanks to Stig Venaas for trying out countless experimental
8 : * patches and reporting/debugging problems patiently!
9 : *
10 : * (c) 1999, Multiple IO-APIC support, developed by
11 : * Ken-ichi Yaku <yaku@css1.kbnes.nec.co.jp> and
12 : * Hidemi Kishimoto <kisimoto@css1.kbnes.nec.co.jp>,
13 : * further tested and cleaned up by Zach Brown <zab@redhat.com>
14 : * and Ingo Molnar <mingo@redhat.com>
15 : *
16 : * Fixes
17 : * Maciej W. Rozycki : Bits for genuine 82489DX APICs;
18 : * thanks to Eric Gilmore
19 : * and Rolf G. Tews
20 : * for testing these extensively
21 : * Paul Diefenbaugh : Added full ACPI support
22 : *
23 : * Historical information which is worth to be preserved:
24 : *
25 : * - SiS APIC rmw bug:
26 : *
27 : * We used to have a workaround for a bug in SiS chips which
28 : * required to rewrite the index register for a read-modify-write
29 : * operation as the chip lost the index information which was
30 : * setup for the read already. We cache the data now, so that
31 : * workaround has been removed.
32 : */
33 :
34 : #include <linux/mm.h>
35 : #include <linux/interrupt.h>
36 : #include <linux/irq.h>
37 : #include <linux/init.h>
38 : #include <linux/delay.h>
39 : #include <linux/sched.h>
40 : #include <linux/pci.h>
41 : #include <linux/mc146818rtc.h>
42 : #include <linux/compiler.h>
43 : #include <linux/acpi.h>
44 : #include <linux/export.h>
45 : #include <linux/syscore_ops.h>
46 : #include <linux/freezer.h>
47 : #include <linux/kthread.h>
48 : #include <linux/jiffies.h> /* time_after() */
49 : #include <linux/slab.h>
50 : #include <linux/memblock.h>
51 : #include <linux/msi.h>
52 :
53 : #include <asm/irqdomain.h>
54 : #include <asm/io.h>
55 : #include <asm/smp.h>
56 : #include <asm/cpu.h>
57 : #include <asm/desc.h>
58 : #include <asm/proto.h>
59 : #include <asm/acpi.h>
60 : #include <asm/dma.h>
61 : #include <asm/timer.h>
62 : #include <asm/time.h>
63 : #include <asm/i8259.h>
64 : #include <asm/setup.h>
65 : #include <asm/irq_remapping.h>
66 : #include <asm/hw_irq.h>
67 : #include <asm/apic.h>
68 :
69 : #define for_each_ioapic(idx) \
70 : for ((idx) = 0; (idx) < nr_ioapics; (idx)++)
71 : #define for_each_ioapic_reverse(idx) \
72 : for ((idx) = nr_ioapics - 1; (idx) >= 0; (idx)--)
73 : #define for_each_pin(idx, pin) \
74 : for ((pin) = 0; (pin) < ioapics[(idx)].nr_registers; (pin)++)
75 : #define for_each_ioapic_pin(idx, pin) \
76 : for_each_ioapic((idx)) \
77 : for_each_pin((idx), (pin))
78 : #define for_each_irq_pin(entry, head) \
79 : list_for_each_entry(entry, &head, list)
80 :
81 : static DEFINE_RAW_SPINLOCK(ioapic_lock);
82 : static DEFINE_MUTEX(ioapic_mutex);
83 : static unsigned int ioapic_dynirq_base;
84 : static int ioapic_initialized;
85 :
86 : struct irq_pin_list {
87 : struct list_head list;
88 : int apic, pin;
89 : };
90 :
91 : struct mp_chip_data {
92 : struct list_head irq_2_pin;
93 : struct IO_APIC_route_entry entry;
94 : bool is_level;
95 : bool active_low;
96 : bool isa_irq;
97 : u32 count;
98 : };
99 :
100 : struct mp_ioapic_gsi {
101 : u32 gsi_base;
102 : u32 gsi_end;
103 : };
104 :
105 : static struct ioapic {
106 : /*
107 : * # of IRQ routing registers
108 : */
109 : int nr_registers;
110 : /*
111 : * Saved state during suspend/resume, or while enabling intr-remap.
112 : */
113 : struct IO_APIC_route_entry *saved_registers;
114 : /* I/O APIC config */
115 : struct mpc_ioapic mp_config;
116 : /* IO APIC gsi routing info */
117 : struct mp_ioapic_gsi gsi_config;
118 : struct ioapic_domain_cfg irqdomain_cfg;
119 : struct irq_domain *irqdomain;
120 : struct resource *iomem_res;
121 : } ioapics[MAX_IO_APICS];
122 :
123 : #define mpc_ioapic_ver(ioapic_idx) ioapics[ioapic_idx].mp_config.apicver
124 :
125 393 : int mpc_ioapic_id(int ioapic_idx)
126 : {
127 393 : return ioapics[ioapic_idx].mp_config.apicid;
128 : }
129 :
130 214 : unsigned int mpc_ioapic_addr(int ioapic_idx)
131 : {
132 214 : return ioapics[ioapic_idx].mp_config.apicaddr;
133 : }
134 :
135 62 : static inline struct mp_ioapic_gsi *mp_ioapic_gsi_routing(int ioapic_idx)
136 : {
137 62 : return &ioapics[ioapic_idx].gsi_config;
138 : }
139 :
140 1 : static inline int mp_ioapic_pin_count(int ioapic)
141 : {
142 1 : struct mp_ioapic_gsi *gsi_cfg = mp_ioapic_gsi_routing(ioapic);
143 :
144 1 : return gsi_cfg->gsi_end - gsi_cfg->gsi_base + 1;
145 : }
146 :
147 30 : static inline u32 mp_pin_to_gsi(int ioapic, int pin)
148 : {
149 30 : return mp_ioapic_gsi_routing(ioapic)->gsi_base + pin;
150 : }
151 :
152 15 : static inline bool mp_is_legacy_irq(int irq)
153 : {
154 0 : return irq >= 0 && irq < nr_legacy_irqs();
155 : }
156 :
157 15 : static inline struct irq_domain *mp_ioapic_irqdomain(int ioapic)
158 : {
159 15 : return ioapics[ioapic].irqdomain;
160 : }
161 :
162 : int nr_ioapics;
163 :
164 : /* The one past the highest gsi number used */
165 : u32 gsi_top;
166 :
167 : /* MP IRQ source entries */
168 : struct mpc_intsrc mp_irqs[MAX_IRQ_SOURCES];
169 :
170 : /* # of MP IRQ source entries */
171 : int mp_irq_entries;
172 :
173 : #ifdef CONFIG_EISA
174 : int mp_bus_id_to_type[MAX_MP_BUSSES];
175 : #endif
176 :
177 : DECLARE_BITMAP(mp_bus_not_pci, MAX_MP_BUSSES);
178 :
179 : int skip_ioapic_setup;
180 :
181 : /**
182 : * disable_ioapic_support() - disables ioapic support at runtime
183 : */
184 0 : void disable_ioapic_support(void)
185 : {
186 : #ifdef CONFIG_PCI
187 : noioapicquirk = 1;
188 : noioapicreroute = -1;
189 : #endif
190 0 : skip_ioapic_setup = 1;
191 0 : }
192 :
193 0 : static int __init parse_noapic(char *str)
194 : {
195 : /* disable IO-APIC */
196 0 : disable_ioapic_support();
197 0 : return 0;
198 : }
199 : early_param("noapic", parse_noapic);
200 :
201 : /* Will be called in mpparse/ACPI codes for saving IRQ info */
202 15 : void mp_save_irq(struct mpc_intsrc *m)
203 : {
204 15 : int i;
205 :
206 15 : apic_printk(APIC_VERBOSE, "Int: type %d, pol %d, trig %d, bus %02x,"
207 : " IRQ %02x, APIC ID %x, APIC INT %02x\n",
208 : m->irqtype, m->irqflag & 3, (m->irqflag >> 2) & 3, m->srcbus,
209 : m->srcbusirq, m->dstapic, m->dstirq);
210 :
211 120 : for (i = 0; i < mp_irq_entries; i++) {
212 105 : if (!memcmp(&mp_irqs[i], m, sizeof(*m)))
213 : return;
214 : }
215 :
216 15 : memcpy(&mp_irqs[mp_irq_entries], m, sizeof(*m));
217 15 : if (++mp_irq_entries == MAX_IRQ_SOURCES)
218 0 : panic("Max # of irq sources exceeded!!\n");
219 : }
220 :
221 1 : static void alloc_ioapic_saved_registers(int idx)
222 : {
223 1 : size_t size;
224 :
225 1 : if (ioapics[idx].saved_registers)
226 : return;
227 :
228 1 : size = sizeof(struct IO_APIC_route_entry) * ioapics[idx].nr_registers;
229 1 : ioapics[idx].saved_registers = kzalloc(size, GFP_KERNEL);
230 1 : if (!ioapics[idx].saved_registers)
231 0 : pr_err("IOAPIC %d: suspend/resume impossible!\n", idx);
232 : }
233 :
234 0 : static void free_ioapic_saved_registers(int idx)
235 : {
236 0 : kfree(ioapics[idx].saved_registers);
237 0 : ioapics[idx].saved_registers = NULL;
238 0 : }
239 :
240 1 : int __init arch_early_ioapic_init(void)
241 : {
242 1 : int i;
243 :
244 1 : if (!nr_legacy_irqs())
245 0 : io_apic_irqs = ~0UL;
246 :
247 2 : for_each_ioapic(i)
248 1 : alloc_ioapic_saved_registers(i);
249 :
250 1 : return 0;
251 : }
252 :
253 : struct io_apic {
254 : unsigned int index;
255 : unsigned int unused[3];
256 : unsigned int data;
257 : unsigned int unused2[11];
258 : unsigned int eoi;
259 : };
260 :
261 212 : static __attribute_const__ struct io_apic __iomem *io_apic_base(int idx)
262 : {
263 212 : return (void __iomem *) __fix_to_virt(FIX_IO_APIC_BASE_0 + idx)
264 212 : + (mpc_ioapic_addr(idx) & ~PAGE_MASK);
265 : }
266 :
267 0 : static inline void io_apic_eoi(unsigned int apic, unsigned int vector)
268 : {
269 0 : struct io_apic __iomem *io_apic = io_apic_base(apic);
270 0 : writel(vector, &io_apic->eoi);
271 0 : }
272 :
273 149 : unsigned int native_io_apic_read(unsigned int apic, unsigned int reg)
274 : {
275 149 : struct io_apic __iomem *io_apic = io_apic_base(apic);
276 149 : writel(reg, &io_apic->index);
277 149 : return readl(&io_apic->data);
278 : }
279 :
280 63 : static void io_apic_write(unsigned int apic, unsigned int reg,
281 : unsigned int value)
282 : {
283 63 : struct io_apic __iomem *io_apic = io_apic_base(apic);
284 :
285 126 : writel(reg, &io_apic->index);
286 63 : writel(value, &io_apic->data);
287 0 : }
288 :
289 72 : static struct IO_APIC_route_entry __ioapic_read_entry(int apic, int pin)
290 : {
291 72 : struct IO_APIC_route_entry entry;
292 :
293 72 : entry.w1 = io_apic_read(apic, 0x10 + 2 * pin);
294 72 : entry.w2 = io_apic_read(apic, 0x11 + 2 * pin);
295 :
296 72 : return entry;
297 : }
298 :
299 72 : static struct IO_APIC_route_entry ioapic_read_entry(int apic, int pin)
300 : {
301 72 : struct IO_APIC_route_entry entry;
302 72 : unsigned long flags;
303 :
304 72 : raw_spin_lock_irqsave(&ioapic_lock, flags);
305 72 : entry = __ioapic_read_entry(apic, pin);
306 72 : raw_spin_unlock_irqrestore(&ioapic_lock, flags);
307 :
308 72 : return entry;
309 : }
310 :
311 : /*
312 : * When we write a new IO APIC routing entry, we need to write the high
313 : * word first! If the mask bit in the low word is clear, we will enable
314 : * the interrupt, and we need to make sure the entry is fully populated
315 : * before that happens.
316 : */
317 6 : static void __ioapic_write_entry(int apic, int pin, struct IO_APIC_route_entry e)
318 : {
319 6 : io_apic_write(apic, 0x11 + 2*pin, e.w2);
320 6 : io_apic_write(apic, 0x10 + 2*pin, e.w1);
321 6 : }
322 :
323 0 : static void ioapic_write_entry(int apic, int pin, struct IO_APIC_route_entry e)
324 : {
325 0 : unsigned long flags;
326 :
327 0 : raw_spin_lock_irqsave(&ioapic_lock, flags);
328 0 : __ioapic_write_entry(apic, pin, e);
329 0 : raw_spin_unlock_irqrestore(&ioapic_lock, flags);
330 0 : }
331 :
332 : /*
333 : * When we mask an IO APIC routing entry, we need to write the low
334 : * word first, in order to set the mask bit before we change the
335 : * high bits!
336 : */
337 24 : static void ioapic_mask_entry(int apic, int pin)
338 : {
339 24 : struct IO_APIC_route_entry e = { .masked = true };
340 24 : unsigned long flags;
341 :
342 24 : raw_spin_lock_irqsave(&ioapic_lock, flags);
343 24 : io_apic_write(apic, 0x10 + 2*pin, e.w1);
344 24 : io_apic_write(apic, 0x11 + 2*pin, e.w2);
345 24 : raw_spin_unlock_irqrestore(&ioapic_lock, flags);
346 24 : }
347 :
348 : /*
349 : * The common case is 1:1 IRQ<->pin mappings. Sometimes there are
350 : * shared ISA-space IRQs, so we have to support them. We are super
351 : * fast in the common case, and fast for shared ISA-space IRQs.
352 : */
353 15 : static int __add_pin_to_irq_node(struct mp_chip_data *data,
354 : int node, int apic, int pin)
355 : {
356 15 : struct irq_pin_list *entry;
357 :
358 : /* don't allow duplicates */
359 15 : for_each_irq_pin(entry, data->irq_2_pin)
360 0 : if (entry->apic == apic && entry->pin == pin)
361 : return 0;
362 :
363 15 : entry = kzalloc_node(sizeof(struct irq_pin_list), GFP_ATOMIC, node);
364 15 : if (!entry) {
365 0 : pr_err("can not alloc irq_pin_list (%d,%d,%d)\n",
366 : node, apic, pin);
367 0 : return -ENOMEM;
368 : }
369 15 : entry->apic = apic;
370 15 : entry->pin = pin;
371 15 : list_add_tail(&entry->list, &data->irq_2_pin);
372 :
373 15 : return 0;
374 : }
375 :
376 0 : static void __remove_pin_from_irq(struct mp_chip_data *data, int apic, int pin)
377 : {
378 0 : struct irq_pin_list *tmp, *entry;
379 :
380 0 : list_for_each_entry_safe(entry, tmp, &data->irq_2_pin, list)
381 0 : if (entry->apic == apic && entry->pin == pin) {
382 0 : list_del(&entry->list);
383 0 : kfree(entry);
384 0 : return;
385 : }
386 : }
387 :
388 15 : static void add_pin_to_irq_node(struct mp_chip_data *data,
389 : int node, int apic, int pin)
390 : {
391 15 : if (__add_pin_to_irq_node(data, node, apic, pin))
392 0 : panic("IO-APIC: failed to add irq-pin. Can not proceed\n");
393 15 : }
394 :
395 : /*
396 : * Reroute an IRQ to a different pin.
397 : */
398 0 : static void __init replace_pin_at_irq_node(struct mp_chip_data *data, int node,
399 : int oldapic, int oldpin,
400 : int newapic, int newpin)
401 : {
402 0 : struct irq_pin_list *entry;
403 :
404 0 : for_each_irq_pin(entry, data->irq_2_pin) {
405 0 : if (entry->apic == oldapic && entry->pin == oldpin) {
406 0 : entry->apic = newapic;
407 0 : entry->pin = newpin;
408 : /* every one is different, right? */
409 0 : return;
410 : }
411 : }
412 :
413 : /* old apic/pin didn't exist, so just add new ones */
414 0 : add_pin_to_irq_node(data, node, newapic, newpin);
415 : }
416 :
417 3 : static void io_apic_modify_irq(struct mp_chip_data *data, bool masked,
418 : void (*final)(struct irq_pin_list *entry))
419 : {
420 3 : struct irq_pin_list *entry;
421 :
422 3 : data->entry.masked = masked;
423 :
424 6 : for_each_irq_pin(entry, data->irq_2_pin) {
425 3 : io_apic_write(entry->apic, 0x10 + 2 * entry->pin, data->entry.w1);
426 3 : if (final)
427 0 : final(entry);
428 : }
429 3 : }
430 :
431 0 : static void io_apic_sync(struct irq_pin_list *entry)
432 : {
433 : /*
434 : * Synchronize the IO-APIC and the CPU by doing
435 : * a dummy read from the IO-APIC
436 : */
437 0 : struct io_apic __iomem *io_apic;
438 :
439 0 : io_apic = io_apic_base(entry->apic);
440 0 : readl(&io_apic->data);
441 0 : }
442 :
443 0 : static void mask_ioapic_irq(struct irq_data *irq_data)
444 : {
445 0 : struct mp_chip_data *data = irq_data->chip_data;
446 0 : unsigned long flags;
447 :
448 0 : raw_spin_lock_irqsave(&ioapic_lock, flags);
449 0 : io_apic_modify_irq(data, true, &io_apic_sync);
450 0 : raw_spin_unlock_irqrestore(&ioapic_lock, flags);
451 0 : }
452 :
453 3 : static void __unmask_ioapic(struct mp_chip_data *data)
454 : {
455 3 : io_apic_modify_irq(data, false, NULL);
456 : }
457 :
458 0 : static void unmask_ioapic_irq(struct irq_data *irq_data)
459 : {
460 0 : struct mp_chip_data *data = irq_data->chip_data;
461 0 : unsigned long flags;
462 :
463 0 : raw_spin_lock_irqsave(&ioapic_lock, flags);
464 0 : __unmask_ioapic(data);
465 0 : raw_spin_unlock_irqrestore(&ioapic_lock, flags);
466 0 : }
467 :
468 : /*
469 : * IO-APIC versions below 0x20 don't support EOI register.
470 : * For the record, here is the information about various versions:
471 : * 0Xh 82489DX
472 : * 1Xh I/OAPIC or I/O(x)APIC which are not PCI 2.2 Compliant
473 : * 2Xh I/O(x)APIC which is PCI 2.2 Compliant
474 : * 30h-FFh Reserved
475 : *
476 : * Some of the Intel ICH Specs (ICH2 to ICH5) documents the io-apic
477 : * version as 0x2. This is an error with documentation and these ICH chips
478 : * use io-apic's of version 0x20.
479 : *
480 : * For IO-APIC's with EOI register, we use that to do an explicit EOI.
481 : * Otherwise, we simulate the EOI message manually by changing the trigger
482 : * mode to edge and then back to level, with RTE being masked during this.
483 : */
484 0 : static void __eoi_ioapic_pin(int apic, int pin, int vector)
485 : {
486 0 : if (mpc_ioapic_ver(apic) >= 0x20) {
487 0 : io_apic_eoi(apic, vector);
488 : } else {
489 0 : struct IO_APIC_route_entry entry, entry1;
490 :
491 0 : entry = entry1 = __ioapic_read_entry(apic, pin);
492 :
493 : /*
494 : * Mask the entry and change the trigger mode to edge.
495 : */
496 0 : entry1.masked = true;
497 0 : entry1.is_level = false;
498 :
499 0 : __ioapic_write_entry(apic, pin, entry1);
500 :
501 : /*
502 : * Restore the previous level triggered entry.
503 : */
504 0 : __ioapic_write_entry(apic, pin, entry);
505 : }
506 0 : }
507 :
508 0 : static void eoi_ioapic_pin(int vector, struct mp_chip_data *data)
509 : {
510 0 : unsigned long flags;
511 0 : struct irq_pin_list *entry;
512 :
513 0 : raw_spin_lock_irqsave(&ioapic_lock, flags);
514 0 : for_each_irq_pin(entry, data->irq_2_pin)
515 0 : __eoi_ioapic_pin(entry->apic, entry->pin, vector);
516 0 : raw_spin_unlock_irqrestore(&ioapic_lock, flags);
517 0 : }
518 :
519 24 : static void clear_IO_APIC_pin(unsigned int apic, unsigned int pin)
520 : {
521 24 : struct IO_APIC_route_entry entry;
522 :
523 : /* Check delivery_mode to be sure we're not clearing an SMI pin */
524 24 : entry = ioapic_read_entry(apic, pin);
525 24 : if (entry.delivery_mode == APIC_DELIVERY_MODE_SMI)
526 0 : return;
527 :
528 : /*
529 : * Make sure the entry is masked and re-read the contents to check
530 : * if it is a level triggered pin and if the remote-IRR is set.
531 : */
532 24 : if (!entry.masked) {
533 0 : entry.masked = true;
534 0 : ioapic_write_entry(apic, pin, entry);
535 0 : entry = ioapic_read_entry(apic, pin);
536 : }
537 :
538 24 : if (entry.irr) {
539 0 : unsigned long flags;
540 :
541 : /*
542 : * Make sure the trigger mode is set to level. Explicit EOI
543 : * doesn't clear the remote-IRR if the trigger mode is not
544 : * set to level.
545 : */
546 0 : if (!entry.is_level) {
547 0 : entry.is_level = true;
548 0 : ioapic_write_entry(apic, pin, entry);
549 : }
550 0 : raw_spin_lock_irqsave(&ioapic_lock, flags);
551 0 : __eoi_ioapic_pin(apic, pin, entry.vector);
552 0 : raw_spin_unlock_irqrestore(&ioapic_lock, flags);
553 : }
554 :
555 : /*
556 : * Clear the rest of the bits in the IO-APIC RTE except for the mask
557 : * bit.
558 : */
559 24 : ioapic_mask_entry(apic, pin);
560 24 : entry = ioapic_read_entry(apic, pin);
561 24 : if (entry.irr)
562 0 : pr_err("Unable to reset IRR for apic: %d, pin :%d\n",
563 : mpc_ioapic_id(apic), pin);
564 : }
565 :
566 1 : void clear_IO_APIC (void)
567 : {
568 1 : int apic, pin;
569 :
570 26 : for_each_ioapic_pin(apic, pin)
571 24 : clear_IO_APIC_pin(apic, pin);
572 1 : }
573 :
574 : #ifdef CONFIG_X86_32
575 : /*
576 : * support for broken MP BIOSs, enables hand-redirection of PIRQ0-7 to
577 : * specific CPU-side IRQs.
578 : */
579 :
580 : #define MAX_PIRQS 8
581 : static int pirq_entries[MAX_PIRQS] = {
582 : [0 ... MAX_PIRQS - 1] = -1
583 : };
584 :
585 : static int __init ioapic_pirq_setup(char *str)
586 : {
587 : int i, max;
588 : int ints[MAX_PIRQS+1];
589 :
590 : get_options(str, ARRAY_SIZE(ints), ints);
591 :
592 : apic_printk(APIC_VERBOSE, KERN_INFO
593 : "PIRQ redirection, working around broken MP-BIOS.\n");
594 : max = MAX_PIRQS;
595 : if (ints[0] < MAX_PIRQS)
596 : max = ints[0];
597 :
598 : for (i = 0; i < max; i++) {
599 : apic_printk(APIC_VERBOSE, KERN_DEBUG
600 : "... PIRQ%d -> IRQ %d\n", i, ints[i+1]);
601 : /*
602 : * PIRQs are mapped upside down, usually.
603 : */
604 : pirq_entries[MAX_PIRQS-i-1] = ints[i+1];
605 : }
606 : return 1;
607 : }
608 :
609 : __setup("pirq=", ioapic_pirq_setup);
610 : #endif /* CONFIG_X86_32 */
611 :
612 : /*
613 : * Saves all the IO-APIC RTE's
614 : */
615 0 : int save_ioapic_entries(void)
616 : {
617 0 : int apic, pin;
618 0 : int err = 0;
619 :
620 0 : for_each_ioapic(apic) {
621 0 : if (!ioapics[apic].saved_registers) {
622 0 : err = -ENOMEM;
623 0 : continue;
624 : }
625 :
626 0 : for_each_pin(apic, pin)
627 0 : ioapics[apic].saved_registers[pin] =
628 0 : ioapic_read_entry(apic, pin);
629 : }
630 :
631 0 : return err;
632 : }
633 :
634 : /*
635 : * Mask all IO APIC entries.
636 : */
637 0 : void mask_ioapic_entries(void)
638 : {
639 0 : int apic, pin;
640 :
641 0 : for_each_ioapic(apic) {
642 0 : if (!ioapics[apic].saved_registers)
643 0 : continue;
644 :
645 0 : for_each_pin(apic, pin) {
646 0 : struct IO_APIC_route_entry entry;
647 :
648 0 : entry = ioapics[apic].saved_registers[pin];
649 0 : if (!entry.masked) {
650 0 : entry.masked = true;
651 0 : ioapic_write_entry(apic, pin, entry);
652 : }
653 : }
654 : }
655 0 : }
656 :
657 : /*
658 : * Restore IO APIC entries which was saved in the ioapic structure.
659 : */
660 0 : int restore_ioapic_entries(void)
661 : {
662 0 : int apic, pin;
663 :
664 0 : for_each_ioapic(apic) {
665 0 : if (!ioapics[apic].saved_registers)
666 0 : continue;
667 :
668 0 : for_each_pin(apic, pin)
669 0 : ioapic_write_entry(apic, pin,
670 0 : ioapics[apic].saved_registers[pin]);
671 : }
672 0 : return 0;
673 : }
674 :
675 : /*
676 : * Find the IRQ entry number of a certain pin.
677 : */
678 39 : static int find_irq_entry(int ioapic_idx, int pin, int type)
679 : {
680 39 : int i;
681 :
682 384 : for (i = 0; i < mp_irq_entries; i++)
683 375 : if (mp_irqs[i].irqtype == type &&
684 375 : (mp_irqs[i].dstapic == mpc_ioapic_id(ioapic_idx) ||
685 375 : mp_irqs[i].dstapic == MP_APIC_ALL) &&
686 375 : mp_irqs[i].dstirq == pin)
687 30 : return i;
688 :
689 : return -1;
690 : }
691 :
692 : /*
693 : * Find the pin to which IRQ[irq] (ISA) is connected
694 : */
695 1 : static int __init find_isa_irq_pin(int irq, int type)
696 : {
697 1 : int i;
698 :
699 16 : for (i = 0; i < mp_irq_entries; i++) {
700 15 : int lbus = mp_irqs[i].srcbus;
701 :
702 15 : if (test_bit(lbus, mp_bus_not_pci) &&
703 15 : (mp_irqs[i].irqtype == type) &&
704 0 : (mp_irqs[i].srcbusirq == irq))
705 :
706 0 : return mp_irqs[i].dstirq;
707 : }
708 : return -1;
709 : }
710 :
711 1 : static int __init find_isa_irq_apic(int irq, int type)
712 : {
713 1 : int i;
714 :
715 16 : for (i = 0; i < mp_irq_entries; i++) {
716 15 : int lbus = mp_irqs[i].srcbus;
717 :
718 15 : if (test_bit(lbus, mp_bus_not_pci) &&
719 15 : (mp_irqs[i].irqtype == type) &&
720 0 : (mp_irqs[i].srcbusirq == irq))
721 : break;
722 : }
723 :
724 1 : if (i < mp_irq_entries) {
725 : int ioapic_idx;
726 :
727 0 : for_each_ioapic(ioapic_idx)
728 0 : if (mpc_ioapic_id(ioapic_idx) == mp_irqs[i].dstapic)
729 0 : return ioapic_idx;
730 : }
731 :
732 : return -1;
733 : }
734 :
735 15 : static bool irq_active_low(int idx)
736 : {
737 15 : int bus = mp_irqs[idx].srcbus;
738 :
739 : /*
740 : * Determine IRQ line polarity (high active or low active):
741 : */
742 15 : switch (mp_irqs[idx].irqflag & MP_IRQPOL_MASK) {
743 15 : case MP_IRQPOL_DEFAULT:
744 : /*
745 : * Conforms to spec, ie. bus-type dependent polarity. PCI
746 : * defaults to low active. [E]ISA defaults to high active.
747 : */
748 15 : return !test_bit(bus, mp_bus_not_pci);
749 : case MP_IRQPOL_ACTIVE_HIGH:
750 : return false;
751 0 : case MP_IRQPOL_RESERVED:
752 0 : pr_warn("IOAPIC: Invalid polarity: 2, defaulting to low\n");
753 : fallthrough;
754 : case MP_IRQPOL_ACTIVE_LOW:
755 : default: /* Pointless default required due to do gcc stupidity */
756 : return true;
757 : }
758 : }
759 :
760 : #ifdef CONFIG_EISA
761 : /*
762 : * EISA Edge/Level control register, ELCR
763 : */
764 : static bool EISA_ELCR(unsigned int irq)
765 : {
766 : if (irq < nr_legacy_irqs()) {
767 : unsigned int port = 0x4d0 + (irq >> 3);
768 : return (inb(port) >> (irq & 7)) & 1;
769 : }
770 : apic_printk(APIC_VERBOSE, KERN_INFO
771 : "Broken MPtable reports ISA irq %d\n", irq);
772 : return false;
773 : }
774 :
775 : /*
776 : * EISA interrupts are always active high and can be edge or level
777 : * triggered depending on the ELCR value. If an interrupt is listed as
778 : * EISA conforming in the MP table, that means its trigger type must be
779 : * read in from the ELCR.
780 : */
781 : static bool eisa_irq_is_level(int idx, int bus, bool level)
782 : {
783 : switch (mp_bus_id_to_type[bus]) {
784 : case MP_BUS_PCI:
785 : case MP_BUS_ISA:
786 : return level;
787 : case MP_BUS_EISA:
788 : return EISA_ELCR(mp_irqs[idx].srcbusirq);
789 : }
790 : pr_warn("IOAPIC: Invalid srcbus: %d defaulting to level\n", bus);
791 : return true;
792 : }
793 : #else
794 15 : static inline int eisa_irq_is_level(int idx, int bus, bool level)
795 : {
796 15 : return level;
797 : }
798 : #endif
799 :
800 15 : static bool irq_is_level(int idx)
801 : {
802 15 : int bus = mp_irqs[idx].srcbus;
803 15 : bool level;
804 :
805 : /*
806 : * Determine IRQ trigger mode (edge or level sensitive):
807 : */
808 15 : switch (mp_irqs[idx].irqflag & MP_IRQTRIG_MASK) {
809 15 : case MP_IRQTRIG_DEFAULT:
810 : /*
811 : * Conforms to spec, ie. bus-type dependent trigger
812 : * mode. PCI defaults to level, ISA to edge.
813 : */
814 15 : level = !test_bit(bus, mp_bus_not_pci);
815 : /* Take EISA into account */
816 15 : return eisa_irq_is_level(idx, bus, level);
817 : case MP_IRQTRIG_EDGE:
818 : return false;
819 0 : case MP_IRQTRIG_RESERVED:
820 0 : pr_warn("IOAPIC: Invalid trigger mode 2 defaulting to level\n");
821 : fallthrough;
822 : case MP_IRQTRIG_LEVEL:
823 : default: /* Pointless default required due to do gcc stupidity */
824 : return true;
825 : }
826 : }
827 :
828 15 : static int __acpi_get_override_irq(u32 gsi, bool *trigger, bool *polarity)
829 : {
830 15 : int ioapic, pin, idx;
831 :
832 15 : if (skip_ioapic_setup)
833 : return -1;
834 :
835 15 : ioapic = mp_find_ioapic(gsi);
836 15 : if (ioapic < 0)
837 : return -1;
838 :
839 15 : pin = mp_find_ioapic_pin(ioapic, gsi);
840 15 : if (pin < 0)
841 : return -1;
842 :
843 15 : idx = find_irq_entry(ioapic, pin, mp_INT);
844 15 : if (idx < 0)
845 : return -1;
846 :
847 15 : *trigger = irq_is_level(idx);
848 15 : *polarity = irq_active_low(idx);
849 15 : return 0;
850 : }
851 :
852 : #ifdef CONFIG_ACPI
853 : int acpi_get_override_irq(u32 gsi, int *is_level, int *active_low)
854 : {
855 : *is_level = *active_low = 0;
856 : return __acpi_get_override_irq(gsi, (bool *)is_level,
857 : (bool *)active_low);
858 : }
859 : #endif
860 :
861 0 : void ioapic_set_alloc_attr(struct irq_alloc_info *info, int node,
862 : int trigger, int polarity)
863 : {
864 0 : init_irq_alloc_info(info, NULL);
865 0 : info->type = X86_IRQ_ALLOC_TYPE_IOAPIC;
866 0 : info->ioapic.node = node;
867 0 : info->ioapic.is_level = trigger;
868 0 : info->ioapic.active_low = polarity;
869 0 : info->ioapic.valid = 1;
870 0 : }
871 :
872 15 : static void ioapic_copy_alloc_attr(struct irq_alloc_info *dst,
873 : struct irq_alloc_info *src,
874 : u32 gsi, int ioapic_idx, int pin)
875 : {
876 15 : bool level, pol_low;
877 :
878 15 : copy_irq_alloc_info(dst, src);
879 15 : dst->type = X86_IRQ_ALLOC_TYPE_IOAPIC;
880 15 : dst->devid = mpc_ioapic_id(ioapic_idx);
881 15 : dst->ioapic.pin = pin;
882 15 : dst->ioapic.valid = 1;
883 15 : if (src && src->ioapic.valid) {
884 0 : dst->ioapic.node = src->ioapic.node;
885 0 : dst->ioapic.is_level = src->ioapic.is_level;
886 0 : dst->ioapic.active_low = src->ioapic.active_low;
887 : } else {
888 15 : dst->ioapic.node = NUMA_NO_NODE;
889 15 : if (__acpi_get_override_irq(gsi, &level, &pol_low) >= 0) {
890 15 : dst->ioapic.is_level = level;
891 15 : dst->ioapic.active_low = pol_low;
892 : } else {
893 : /*
894 : * PCI interrupts are always active low level
895 : * triggered.
896 : */
897 0 : dst->ioapic.is_level = true;
898 0 : dst->ioapic.active_low = true;
899 : }
900 : }
901 15 : }
902 :
903 30 : static int ioapic_alloc_attr_node(struct irq_alloc_info *info)
904 : {
905 30 : return (info && info->ioapic.valid) ? info->ioapic.node : NUMA_NO_NODE;
906 : }
907 :
908 15 : static void mp_register_handler(unsigned int irq, bool level)
909 : {
910 15 : irq_flow_handler_t hdl;
911 15 : bool fasteoi;
912 :
913 15 : if (level) {
914 0 : irq_set_status_flags(irq, IRQ_LEVEL);
915 0 : fasteoi = true;
916 : } else {
917 15 : irq_clear_status_flags(irq, IRQ_LEVEL);
918 15 : fasteoi = false;
919 : }
920 :
921 15 : hdl = fasteoi ? handle_fasteoi_irq : handle_edge_irq;
922 30 : __irq_set_handler(irq, hdl, 0, fasteoi ? "fasteoi" : "edge");
923 15 : }
924 :
925 0 : static bool mp_check_pin_attr(int irq, struct irq_alloc_info *info)
926 : {
927 0 : struct mp_chip_data *data = irq_get_chip_data(irq);
928 :
929 : /*
930 : * setup_IO_APIC_irqs() programs all legacy IRQs with default trigger
931 : * and polarity attirbutes. So allow the first user to reprogram the
932 : * pin with real trigger and polarity attributes.
933 : */
934 0 : if (irq < nr_legacy_irqs() && data->count == 1) {
935 0 : if (info->ioapic.is_level != data->is_level)
936 0 : mp_register_handler(irq, info->ioapic.is_level);
937 0 : data->entry.is_level = data->is_level = info->ioapic.is_level;
938 0 : data->entry.active_low = data->active_low = info->ioapic.active_low;
939 : }
940 :
941 0 : return data->is_level == info->ioapic.is_level &&
942 0 : data->active_low == info->ioapic.active_low;
943 : }
944 :
945 0 : static int alloc_irq_from_domain(struct irq_domain *domain, int ioapic, u32 gsi,
946 : struct irq_alloc_info *info)
947 : {
948 0 : bool legacy = false;
949 0 : int irq = -1;
950 0 : int type = ioapics[ioapic].irqdomain_cfg.type;
951 :
952 0 : switch (type) {
953 0 : case IOAPIC_DOMAIN_LEGACY:
954 : /*
955 : * Dynamically allocate IRQ number for non-ISA IRQs in the first
956 : * 16 GSIs on some weird platforms.
957 : */
958 0 : if (!ioapic_initialized || gsi >= nr_legacy_irqs())
959 0 : irq = gsi;
960 0 : legacy = mp_is_legacy_irq(irq);
961 0 : break;
962 0 : case IOAPIC_DOMAIN_STRICT:
963 0 : irq = gsi;
964 0 : break;
965 : case IOAPIC_DOMAIN_DYNAMIC:
966 : break;
967 : default:
968 0 : WARN(1, "ioapic: unknown irqdomain type %d\n", type);
969 0 : return -1;
970 : }
971 :
972 0 : return __irq_domain_alloc_irqs(domain, irq, 1,
973 : ioapic_alloc_attr_node(info),
974 : info, legacy, NULL);
975 : }
976 :
977 : /*
978 : * Need special handling for ISA IRQs because there may be multiple IOAPIC pins
979 : * sharing the same ISA IRQ number and irqdomain only supports 1:1 mapping
980 : * between IOAPIC pin and IRQ number. A typical IOAPIC has 24 pins, pin 0-15 are
981 : * used for legacy IRQs and pin 16-23 are used for PCI IRQs (PIRQ A-H).
982 : * When ACPI is disabled, only legacy IRQ numbers (IRQ0-15) are available, and
983 : * some BIOSes may use MP Interrupt Source records to override IRQ numbers for
984 : * PIRQs instead of reprogramming the interrupt routing logic. Thus there may be
985 : * multiple pins sharing the same legacy IRQ number when ACPI is disabled.
986 : */
987 15 : static int alloc_isa_irq_from_domain(struct irq_domain *domain,
988 : int irq, int ioapic, int pin,
989 : struct irq_alloc_info *info)
990 : {
991 15 : struct mp_chip_data *data;
992 15 : struct irq_data *irq_data = irq_get_irq_data(irq);
993 15 : int node = ioapic_alloc_attr_node(info);
994 :
995 : /*
996 : * Legacy ISA IRQ has already been allocated, just add pin to
997 : * the pin list assoicated with this IRQ and program the IOAPIC
998 : * entry. The IOAPIC entry
999 : */
1000 15 : if (irq_data && irq_data->parent_data) {
1001 0 : if (!mp_check_pin_attr(irq, info))
1002 : return -EBUSY;
1003 0 : if (__add_pin_to_irq_node(irq_data->chip_data, node, ioapic,
1004 : info->ioapic.pin))
1005 0 : return -ENOMEM;
1006 : } else {
1007 15 : info->flags |= X86_IRQ_ALLOC_LEGACY;
1008 15 : irq = __irq_domain_alloc_irqs(domain, irq, 1, node, info, true,
1009 : NULL);
1010 15 : if (irq >= 0) {
1011 15 : irq_data = irq_domain_get_irq_data(domain, irq);
1012 15 : data = irq_data->chip_data;
1013 15 : data->isa_irq = true;
1014 : }
1015 : }
1016 :
1017 : return irq;
1018 : }
1019 :
1020 15 : static int mp_map_pin_to_irq(u32 gsi, int idx, int ioapic, int pin,
1021 : unsigned int flags, struct irq_alloc_info *info)
1022 : {
1023 15 : int irq;
1024 15 : bool legacy = false;
1025 15 : struct irq_alloc_info tmp;
1026 15 : struct mp_chip_data *data;
1027 15 : struct irq_domain *domain = mp_ioapic_irqdomain(ioapic);
1028 :
1029 15 : if (!domain)
1030 : return -ENOSYS;
1031 :
1032 15 : if (idx >= 0 && test_bit(mp_irqs[idx].srcbus, mp_bus_not_pci)) {
1033 15 : irq = mp_irqs[idx].srcbusirq;
1034 15 : legacy = mp_is_legacy_irq(irq);
1035 : }
1036 :
1037 15 : mutex_lock(&ioapic_mutex);
1038 15 : if (!(flags & IOAPIC_MAP_ALLOC)) {
1039 0 : if (!legacy) {
1040 0 : irq = irq_find_mapping(domain, pin);
1041 0 : if (irq == 0)
1042 0 : irq = -ENOENT;
1043 : }
1044 : } else {
1045 15 : ioapic_copy_alloc_attr(&tmp, info, gsi, ioapic, pin);
1046 15 : if (legacy)
1047 15 : irq = alloc_isa_irq_from_domain(domain, irq,
1048 : ioapic, pin, &tmp);
1049 0 : else if ((irq = irq_find_mapping(domain, pin)) == 0)
1050 0 : irq = alloc_irq_from_domain(domain, ioapic, gsi, &tmp);
1051 0 : else if (!mp_check_pin_attr(irq, &tmp))
1052 : irq = -EBUSY;
1053 15 : if (irq >= 0) {
1054 15 : data = irq_get_chip_data(irq);
1055 15 : data->count++;
1056 : }
1057 : }
1058 15 : mutex_unlock(&ioapic_mutex);
1059 :
1060 15 : return irq;
1061 : }
1062 :
1063 15 : static int pin_2_irq(int idx, int ioapic, int pin, unsigned int flags)
1064 : {
1065 15 : u32 gsi = mp_pin_to_gsi(ioapic, pin);
1066 :
1067 : /*
1068 : * Debugging check, we are in big trouble if this message pops up!
1069 : */
1070 15 : if (mp_irqs[idx].dstirq != pin)
1071 0 : pr_err("broken BIOS or MPTABLE parser, ayiee!!\n");
1072 :
1073 : #ifdef CONFIG_X86_32
1074 : /*
1075 : * PCI IRQ command line redirection. Yes, limits are hardcoded.
1076 : */
1077 : if ((pin >= 16) && (pin <= 23)) {
1078 : if (pirq_entries[pin-16] != -1) {
1079 : if (!pirq_entries[pin-16]) {
1080 : apic_printk(APIC_VERBOSE, KERN_DEBUG
1081 : "disabling PIRQ%d\n", pin-16);
1082 : } else {
1083 : int irq = pirq_entries[pin-16];
1084 : apic_printk(APIC_VERBOSE, KERN_DEBUG
1085 : "using PIRQ%d -> IRQ %d\n",
1086 : pin-16, irq);
1087 : return irq;
1088 : }
1089 : }
1090 : }
1091 : #endif
1092 :
1093 15 : return mp_map_pin_to_irq(gsi, idx, ioapic, pin, flags, NULL);
1094 : }
1095 :
1096 0 : int mp_map_gsi_to_irq(u32 gsi, unsigned int flags, struct irq_alloc_info *info)
1097 : {
1098 0 : int ioapic, pin, idx;
1099 :
1100 0 : ioapic = mp_find_ioapic(gsi);
1101 0 : if (ioapic < 0)
1102 : return -ENODEV;
1103 :
1104 0 : pin = mp_find_ioapic_pin(ioapic, gsi);
1105 0 : idx = find_irq_entry(ioapic, pin, mp_INT);
1106 0 : if ((flags & IOAPIC_MAP_CHECK) && idx < 0)
1107 : return -ENODEV;
1108 :
1109 0 : return mp_map_pin_to_irq(gsi, idx, ioapic, pin, flags, info);
1110 : }
1111 :
1112 0 : void mp_unmap_irq(int irq)
1113 : {
1114 0 : struct irq_data *irq_data = irq_get_irq_data(irq);
1115 0 : struct mp_chip_data *data;
1116 :
1117 0 : if (!irq_data || !irq_data->domain)
1118 : return;
1119 :
1120 0 : data = irq_data->chip_data;
1121 0 : if (!data || data->isa_irq)
1122 : return;
1123 :
1124 0 : mutex_lock(&ioapic_mutex);
1125 0 : if (--data->count == 0)
1126 0 : irq_domain_free_irqs(irq, 1);
1127 0 : mutex_unlock(&ioapic_mutex);
1128 : }
1129 :
1130 : /*
1131 : * Find a specific PCI IRQ entry.
1132 : * Not an __init, possibly needed by modules
1133 : */
1134 0 : int IO_APIC_get_PCI_irq_vector(int bus, int slot, int pin)
1135 : {
1136 0 : int irq, i, best_ioapic = -1, best_idx = -1;
1137 :
1138 0 : apic_printk(APIC_DEBUG,
1139 : "querying PCI -> IRQ mapping bus:%d, slot:%d, pin:%d.\n",
1140 : bus, slot, pin);
1141 0 : if (test_bit(bus, mp_bus_not_pci)) {
1142 0 : apic_printk(APIC_VERBOSE,
1143 : "PCI BIOS passed nonexistent PCI bus %d!\n", bus);
1144 0 : return -1;
1145 : }
1146 :
1147 0 : for (i = 0; i < mp_irq_entries; i++) {
1148 0 : int lbus = mp_irqs[i].srcbus;
1149 0 : int ioapic_idx, found = 0;
1150 :
1151 0 : if (bus != lbus || mp_irqs[i].irqtype != mp_INT ||
1152 0 : slot != ((mp_irqs[i].srcbusirq >> 2) & 0x1f))
1153 0 : continue;
1154 :
1155 0 : for_each_ioapic(ioapic_idx)
1156 0 : if (mpc_ioapic_id(ioapic_idx) == mp_irqs[i].dstapic ||
1157 : mp_irqs[i].dstapic == MP_APIC_ALL) {
1158 : found = 1;
1159 : break;
1160 : }
1161 0 : if (!found)
1162 0 : continue;
1163 :
1164 : /* Skip ISA IRQs */
1165 0 : irq = pin_2_irq(i, ioapic_idx, mp_irqs[i].dstirq, 0);
1166 0 : if (irq > 0 && !IO_APIC_IRQ(irq))
1167 0 : continue;
1168 :
1169 0 : if (pin == (mp_irqs[i].srcbusirq & 3)) {
1170 0 : best_idx = i;
1171 0 : best_ioapic = ioapic_idx;
1172 0 : goto out;
1173 : }
1174 :
1175 : /*
1176 : * Use the first all-but-pin matching entry as a
1177 : * best-guess fuzzy result for broken mptables.
1178 : */
1179 0 : if (best_idx < 0) {
1180 0 : best_idx = i;
1181 0 : best_ioapic = ioapic_idx;
1182 : }
1183 : }
1184 0 : if (best_idx < 0)
1185 : return -1;
1186 :
1187 0 : out:
1188 0 : return pin_2_irq(best_idx, best_ioapic, mp_irqs[best_idx].dstirq,
1189 : IOAPIC_MAP_ALLOC);
1190 : }
1191 : EXPORT_SYMBOL(IO_APIC_get_PCI_irq_vector);
1192 :
1193 : static struct irq_chip ioapic_chip, ioapic_ir_chip;
1194 :
1195 1 : static void __init setup_IO_APIC_irqs(void)
1196 : {
1197 1 : unsigned int ioapic, pin;
1198 1 : int idx;
1199 :
1200 1 : apic_printk(APIC_VERBOSE, KERN_DEBUG "init IO_APIC IRQs\n");
1201 :
1202 26 : for_each_ioapic_pin(ioapic, pin) {
1203 24 : idx = find_irq_entry(ioapic, pin, mp_INT);
1204 24 : if (idx < 0)
1205 9 : apic_printk(APIC_VERBOSE,
1206 : KERN_DEBUG " apic %d pin %d not connected\n",
1207 : mpc_ioapic_id(ioapic), pin);
1208 : else
1209 15 : pin_2_irq(idx, ioapic, pin,
1210 : ioapic ? 0 : IOAPIC_MAP_ALLOC);
1211 : }
1212 1 : }
1213 :
1214 0 : void ioapic_zap_locks(void)
1215 : {
1216 0 : raw_spin_lock_init(&ioapic_lock);
1217 0 : }
1218 :
1219 0 : static void io_apic_print_entries(unsigned int apic, unsigned int nr_entries)
1220 : {
1221 0 : struct IO_APIC_route_entry entry;
1222 0 : char buf[256];
1223 0 : int i;
1224 :
1225 0 : printk(KERN_DEBUG "IOAPIC %d:\n", apic);
1226 0 : for (i = 0; i <= nr_entries; i++) {
1227 0 : entry = ioapic_read_entry(apic, i);
1228 0 : snprintf(buf, sizeof(buf),
1229 : " pin%02x, %s, %s, %s, V(%02X), IRR(%1d), S(%1d)",
1230 : i,
1231 0 : entry.masked ? "disabled" : "enabled ",
1232 : entry.is_level ? "level" : "edge ",
1233 0 : entry.active_low ? "low " : "high",
1234 0 : entry.vector, entry.irr, entry.delivery_status);
1235 0 : if (entry.ir_format) {
1236 0 : printk(KERN_DEBUG "%s, remapped, I(%04X), Z(%X)\n",
1237 : buf,
1238 0 : (entry.ir_index_15 << 15) | entry.ir_index_0_14,
1239 : entry.ir_zero);
1240 : } else {
1241 0 : printk(KERN_DEBUG "%s, %s, D(%02X%02X), M(%1d)\n", buf,
1242 0 : entry.dest_mode_logical ? "logical " : "physical",
1243 0 : entry.virt_destid_8_14, entry.destid_0_7,
1244 : entry.delivery_mode);
1245 : }
1246 : }
1247 0 : }
1248 :
1249 0 : static void __init print_IO_APIC(int ioapic_idx)
1250 : {
1251 0 : union IO_APIC_reg_00 reg_00;
1252 0 : union IO_APIC_reg_01 reg_01;
1253 0 : union IO_APIC_reg_02 reg_02;
1254 0 : union IO_APIC_reg_03 reg_03;
1255 0 : unsigned long flags;
1256 :
1257 0 : raw_spin_lock_irqsave(&ioapic_lock, flags);
1258 0 : reg_00.raw = io_apic_read(ioapic_idx, 0);
1259 0 : reg_01.raw = io_apic_read(ioapic_idx, 1);
1260 0 : if (reg_01.bits.version >= 0x10)
1261 0 : reg_02.raw = io_apic_read(ioapic_idx, 2);
1262 0 : if (reg_01.bits.version >= 0x20)
1263 0 : reg_03.raw = io_apic_read(ioapic_idx, 3);
1264 0 : raw_spin_unlock_irqrestore(&ioapic_lock, flags);
1265 :
1266 0 : printk(KERN_DEBUG "IO APIC #%d......\n", mpc_ioapic_id(ioapic_idx));
1267 0 : printk(KERN_DEBUG ".... register #00: %08X\n", reg_00.raw);
1268 0 : printk(KERN_DEBUG "....... : physical APIC id: %02X\n", reg_00.bits.ID);
1269 0 : printk(KERN_DEBUG "....... : Delivery Type: %X\n", reg_00.bits.delivery_type);
1270 0 : printk(KERN_DEBUG "....... : LTS : %X\n", reg_00.bits.LTS);
1271 :
1272 0 : printk(KERN_DEBUG ".... register #01: %08X\n", *(int *)®_01);
1273 0 : printk(KERN_DEBUG "....... : max redirection entries: %02X\n",
1274 : reg_01.bits.entries);
1275 :
1276 0 : printk(KERN_DEBUG "....... : PRQ implemented: %X\n", reg_01.bits.PRQ);
1277 0 : printk(KERN_DEBUG "....... : IO APIC version: %02X\n",
1278 : reg_01.bits.version);
1279 :
1280 : /*
1281 : * Some Intel chipsets with IO APIC VERSION of 0x1? don't have reg_02,
1282 : * but the value of reg_02 is read as the previous read register
1283 : * value, so ignore it if reg_02 == reg_01.
1284 : */
1285 0 : if (reg_01.bits.version >= 0x10 && reg_02.raw != reg_01.raw) {
1286 0 : printk(KERN_DEBUG ".... register #02: %08X\n", reg_02.raw);
1287 0 : printk(KERN_DEBUG "....... : arbitration: %02X\n", reg_02.bits.arbitration);
1288 : }
1289 :
1290 : /*
1291 : * Some Intel chipsets with IO APIC VERSION of 0x2? don't have reg_02
1292 : * or reg_03, but the value of reg_0[23] is read as the previous read
1293 : * register value, so ignore it if reg_03 == reg_0[12].
1294 : */
1295 0 : if (reg_01.bits.version >= 0x20 && reg_03.raw != reg_02.raw &&
1296 : reg_03.raw != reg_01.raw) {
1297 0 : printk(KERN_DEBUG ".... register #03: %08X\n", reg_03.raw);
1298 0 : printk(KERN_DEBUG "....... : Boot DT : %X\n", reg_03.bits.boot_DT);
1299 : }
1300 :
1301 0 : printk(KERN_DEBUG ".... IRQ redirection table:\n");
1302 0 : io_apic_print_entries(ioapic_idx, reg_01.bits.entries);
1303 0 : }
1304 :
1305 0 : void __init print_IO_APICs(void)
1306 : {
1307 0 : int ioapic_idx;
1308 0 : unsigned int irq;
1309 :
1310 0 : printk(KERN_DEBUG "number of MP IRQ sources: %d.\n", mp_irq_entries);
1311 0 : for_each_ioapic(ioapic_idx)
1312 0 : printk(KERN_DEBUG "number of IO-APIC #%d registers: %d.\n",
1313 : mpc_ioapic_id(ioapic_idx),
1314 : ioapics[ioapic_idx].nr_registers);
1315 :
1316 : /*
1317 : * We are a bit conservative about what we expect. We have to
1318 : * know about every hardware change ASAP.
1319 : */
1320 0 : printk(KERN_INFO "testing the IO APIC.......................\n");
1321 :
1322 0 : for_each_ioapic(ioapic_idx)
1323 0 : print_IO_APIC(ioapic_idx);
1324 :
1325 0 : printk(KERN_DEBUG "IRQ to pin mappings:\n");
1326 0 : for_each_active_irq(irq) {
1327 0 : struct irq_pin_list *entry;
1328 0 : struct irq_chip *chip;
1329 0 : struct mp_chip_data *data;
1330 :
1331 0 : chip = irq_get_chip(irq);
1332 0 : if (chip != &ioapic_chip && chip != &ioapic_ir_chip)
1333 0 : continue;
1334 0 : data = irq_get_chip_data(irq);
1335 0 : if (!data)
1336 0 : continue;
1337 0 : if (list_empty(&data->irq_2_pin))
1338 0 : continue;
1339 :
1340 0 : printk(KERN_DEBUG "IRQ%d ", irq);
1341 0 : for_each_irq_pin(entry, data->irq_2_pin)
1342 0 : pr_cont("-> %d:%d", entry->apic, entry->pin);
1343 0 : pr_cont("\n");
1344 : }
1345 :
1346 0 : printk(KERN_INFO ".................................... done.\n");
1347 0 : }
1348 :
1349 : /* Where if anywhere is the i8259 connect in external int mode */
1350 : static struct { int pin, apic; } ioapic_i8259 = { -1, -1 };
1351 :
1352 1 : void __init enable_IO_APIC(void)
1353 : {
1354 1 : int i8259_apic, i8259_pin;
1355 1 : int apic, pin;
1356 :
1357 1 : if (skip_ioapic_setup)
1358 0 : nr_ioapics = 0;
1359 :
1360 1 : if (!nr_legacy_irqs() || !nr_ioapics)
1361 : return;
1362 :
1363 26 : for_each_ioapic_pin(apic, pin) {
1364 : /* See if any of the pins is in ExtINT mode */
1365 24 : struct IO_APIC_route_entry entry = ioapic_read_entry(apic, pin);
1366 :
1367 : /* If the interrupt line is enabled and in ExtInt mode
1368 : * I have found the pin where the i8259 is connected.
1369 : */
1370 24 : if (!entry.masked &&
1371 0 : entry.delivery_mode == APIC_DELIVERY_MODE_EXTINT) {
1372 0 : ioapic_i8259.apic = apic;
1373 0 : ioapic_i8259.pin = pin;
1374 0 : goto found_i8259;
1375 : }
1376 : }
1377 1 : found_i8259:
1378 : /* Look to see what if the MP table has reported the ExtINT */
1379 : /* If we could not find the appropriate pin by looking at the ioapic
1380 : * the i8259 probably is not connected the ioapic but give the
1381 : * mptable a chance anyway.
1382 : */
1383 1 : i8259_pin = find_isa_irq_pin(0, mp_ExtINT);
1384 1 : i8259_apic = find_isa_irq_apic(0, mp_ExtINT);
1385 : /* Trust the MP table if nothing is setup in the hardware */
1386 1 : if ((ioapic_i8259.pin == -1) && (i8259_pin >= 0)) {
1387 0 : printk(KERN_WARNING "ExtINT not setup in hardware but reported by MP table\n");
1388 0 : ioapic_i8259.pin = i8259_pin;
1389 0 : ioapic_i8259.apic = i8259_apic;
1390 : }
1391 : /* Complain if the MP table and the hardware disagree */
1392 1 : if (((ioapic_i8259.apic != i8259_apic) || (ioapic_i8259.pin != i8259_pin)) &&
1393 0 : (i8259_pin >= 0) && (ioapic_i8259.pin >= 0))
1394 : {
1395 0 : printk(KERN_WARNING "ExtINT in hardware and MP table differ\n");
1396 : }
1397 :
1398 : /*
1399 : * Do not trust the IO-APIC being empty at bootup
1400 : */
1401 1 : clear_IO_APIC();
1402 : }
1403 :
1404 0 : void native_restore_boot_irq_mode(void)
1405 : {
1406 : /*
1407 : * If the i8259 is routed through an IOAPIC
1408 : * Put that IOAPIC in virtual wire mode
1409 : * so legacy interrupts can be delivered.
1410 : */
1411 0 : if (ioapic_i8259.pin != -1) {
1412 0 : struct IO_APIC_route_entry entry;
1413 0 : u32 apic_id = read_apic_id();
1414 :
1415 0 : memset(&entry, 0, sizeof(entry));
1416 0 : entry.masked = false;
1417 0 : entry.is_level = false;
1418 0 : entry.active_low = false;
1419 0 : entry.dest_mode_logical = false;
1420 0 : entry.delivery_mode = APIC_DELIVERY_MODE_EXTINT;
1421 0 : entry.destid_0_7 = apic_id & 0xFF;
1422 0 : entry.virt_destid_8_14 = apic_id >> 8;
1423 :
1424 : /*
1425 : * Add it to the IO-APIC irq-routing table:
1426 : */
1427 0 : ioapic_write_entry(ioapic_i8259.apic, ioapic_i8259.pin, entry);
1428 : }
1429 :
1430 0 : if (boot_cpu_has(X86_FEATURE_APIC) || apic_from_smp_config())
1431 0 : disconnect_bsp_APIC(ioapic_i8259.pin != -1);
1432 0 : }
1433 :
1434 0 : void restore_boot_irq_mode(void)
1435 : {
1436 0 : if (!nr_legacy_irqs())
1437 : return;
1438 :
1439 0 : x86_apic_ops.restore();
1440 : }
1441 :
1442 : #ifdef CONFIG_X86_32
1443 : /*
1444 : * function to set the IO-APIC physical IDs based on the
1445 : * values stored in the MPC table.
1446 : *
1447 : * by Matt Domsch <Matt_Domsch@dell.com> Tue Dec 21 12:25:05 CST 1999
1448 : */
1449 : void __init setup_ioapic_ids_from_mpc_nocheck(void)
1450 : {
1451 : union IO_APIC_reg_00 reg_00;
1452 : physid_mask_t phys_id_present_map;
1453 : int ioapic_idx;
1454 : int i;
1455 : unsigned char old_id;
1456 : unsigned long flags;
1457 :
1458 : /*
1459 : * This is broken; anything with a real cpu count has to
1460 : * circumvent this idiocy regardless.
1461 : */
1462 : apic->ioapic_phys_id_map(&phys_cpu_present_map, &phys_id_present_map);
1463 :
1464 : /*
1465 : * Set the IOAPIC ID to the value stored in the MPC table.
1466 : */
1467 : for_each_ioapic(ioapic_idx) {
1468 : /* Read the register 0 value */
1469 : raw_spin_lock_irqsave(&ioapic_lock, flags);
1470 : reg_00.raw = io_apic_read(ioapic_idx, 0);
1471 : raw_spin_unlock_irqrestore(&ioapic_lock, flags);
1472 :
1473 : old_id = mpc_ioapic_id(ioapic_idx);
1474 :
1475 : if (mpc_ioapic_id(ioapic_idx) >= get_physical_broadcast()) {
1476 : printk(KERN_ERR "BIOS bug, IO-APIC#%d ID is %d in the MPC table!...\n",
1477 : ioapic_idx, mpc_ioapic_id(ioapic_idx));
1478 : printk(KERN_ERR "... fixing up to %d. (tell your hw vendor)\n",
1479 : reg_00.bits.ID);
1480 : ioapics[ioapic_idx].mp_config.apicid = reg_00.bits.ID;
1481 : }
1482 :
1483 : /*
1484 : * Sanity check, is the ID really free? Every APIC in a
1485 : * system must have a unique ID or we get lots of nice
1486 : * 'stuck on smp_invalidate_needed IPI wait' messages.
1487 : */
1488 : if (apic->check_apicid_used(&phys_id_present_map,
1489 : mpc_ioapic_id(ioapic_idx))) {
1490 : printk(KERN_ERR "BIOS bug, IO-APIC#%d ID %d is already used!...\n",
1491 : ioapic_idx, mpc_ioapic_id(ioapic_idx));
1492 : for (i = 0; i < get_physical_broadcast(); i++)
1493 : if (!physid_isset(i, phys_id_present_map))
1494 : break;
1495 : if (i >= get_physical_broadcast())
1496 : panic("Max APIC ID exceeded!\n");
1497 : printk(KERN_ERR "... fixing up to %d. (tell your hw vendor)\n",
1498 : i);
1499 : physid_set(i, phys_id_present_map);
1500 : ioapics[ioapic_idx].mp_config.apicid = i;
1501 : } else {
1502 : physid_mask_t tmp;
1503 : apic->apicid_to_cpu_present(mpc_ioapic_id(ioapic_idx),
1504 : &tmp);
1505 : apic_printk(APIC_VERBOSE, "Setting %d in the "
1506 : "phys_id_present_map\n",
1507 : mpc_ioapic_id(ioapic_idx));
1508 : physids_or(phys_id_present_map, phys_id_present_map, tmp);
1509 : }
1510 :
1511 : /*
1512 : * We need to adjust the IRQ routing table
1513 : * if the ID changed.
1514 : */
1515 : if (old_id != mpc_ioapic_id(ioapic_idx))
1516 : for (i = 0; i < mp_irq_entries; i++)
1517 : if (mp_irqs[i].dstapic == old_id)
1518 : mp_irqs[i].dstapic
1519 : = mpc_ioapic_id(ioapic_idx);
1520 :
1521 : /*
1522 : * Update the ID register according to the right value
1523 : * from the MPC table if they are different.
1524 : */
1525 : if (mpc_ioapic_id(ioapic_idx) == reg_00.bits.ID)
1526 : continue;
1527 :
1528 : apic_printk(APIC_VERBOSE, KERN_INFO
1529 : "...changing IO-APIC physical APIC ID to %d ...",
1530 : mpc_ioapic_id(ioapic_idx));
1531 :
1532 : reg_00.bits.ID = mpc_ioapic_id(ioapic_idx);
1533 : raw_spin_lock_irqsave(&ioapic_lock, flags);
1534 : io_apic_write(ioapic_idx, 0, reg_00.raw);
1535 : raw_spin_unlock_irqrestore(&ioapic_lock, flags);
1536 :
1537 : /*
1538 : * Sanity check
1539 : */
1540 : raw_spin_lock_irqsave(&ioapic_lock, flags);
1541 : reg_00.raw = io_apic_read(ioapic_idx, 0);
1542 : raw_spin_unlock_irqrestore(&ioapic_lock, flags);
1543 : if (reg_00.bits.ID != mpc_ioapic_id(ioapic_idx))
1544 : pr_cont("could not set ID!\n");
1545 : else
1546 : apic_printk(APIC_VERBOSE, " ok.\n");
1547 : }
1548 : }
1549 :
1550 : void __init setup_ioapic_ids_from_mpc(void)
1551 : {
1552 :
1553 : if (acpi_ioapic)
1554 : return;
1555 : /*
1556 : * Don't check I/O APIC IDs for xAPIC systems. They have
1557 : * no meaning without the serial APIC bus.
1558 : */
1559 : if (!(boot_cpu_data.x86_vendor == X86_VENDOR_INTEL)
1560 : || APIC_XAPIC(boot_cpu_apic_version))
1561 : return;
1562 : setup_ioapic_ids_from_mpc_nocheck();
1563 : }
1564 : #endif
1565 :
1566 : int no_timer_check __initdata;
1567 :
1568 0 : static int __init notimercheck(char *s)
1569 : {
1570 0 : no_timer_check = 1;
1571 0 : return 1;
1572 : }
1573 : __setup("no_timer_check", notimercheck);
1574 :
1575 0 : static void __init delay_with_tsc(void)
1576 : {
1577 0 : unsigned long long start, now;
1578 0 : unsigned long end = jiffies + 4;
1579 :
1580 0 : start = rdtsc();
1581 :
1582 : /*
1583 : * We don't know the TSC frequency yet, but waiting for
1584 : * 40000000000/HZ TSC cycles is safe:
1585 : * 4 GHz == 10 jiffies
1586 : * 1 GHz == 40 jiffies
1587 : */
1588 0 : do {
1589 0 : rep_nop();
1590 0 : now = rdtsc();
1591 0 : } while ((now - start) < 40000000000ULL / HZ &&
1592 0 : time_before_eq(jiffies, end));
1593 0 : }
1594 :
1595 0 : static void __init delay_without_tsc(void)
1596 : {
1597 0 : unsigned long end = jiffies + 4;
1598 0 : int band = 1;
1599 :
1600 : /*
1601 : * We don't know any frequency yet, but waiting for
1602 : * 40940000000/HZ cycles is safe:
1603 : * 4 GHz == 10 jiffies
1604 : * 1 GHz == 40 jiffies
1605 : * 1 << 1 + 1 << 2 +...+ 1 << 11 = 4094
1606 : */
1607 0 : do {
1608 0 : __delay(((1U << band++) * 10000000UL) / HZ);
1609 0 : } while (band < 12 && time_before_eq(jiffies, end));
1610 0 : }
1611 :
1612 : /*
1613 : * There is a nasty bug in some older SMP boards, their mptable lies
1614 : * about the timer IRQ. We do the following to work around the situation:
1615 : *
1616 : * - timer IRQ defaults to IO-APIC IRQ
1617 : * - if this function detects that timer IRQs are defunct, then we fall
1618 : * back to ISA timer IRQs
1619 : */
1620 0 : static int __init timer_irq_works(void)
1621 : {
1622 0 : unsigned long t1 = jiffies;
1623 :
1624 0 : if (no_timer_check)
1625 : return 1;
1626 :
1627 0 : local_irq_enable();
1628 0 : if (boot_cpu_has(X86_FEATURE_TSC))
1629 0 : delay_with_tsc();
1630 : else
1631 0 : delay_without_tsc();
1632 :
1633 : /*
1634 : * Expect a few ticks at least, to be sure some possible
1635 : * glue logic does not lock up after one or two first
1636 : * ticks in a non-ExtINT mode. Also the local APIC
1637 : * might have cached one ExtINT interrupt. Finally, at
1638 : * least one tick may be lost due to delays.
1639 : */
1640 :
1641 0 : local_irq_disable();
1642 :
1643 : /* Did jiffies advance? */
1644 0 : return time_after(jiffies, t1 + 4);
1645 : }
1646 :
1647 : /*
1648 : * In the SMP+IOAPIC case it might happen that there are an unspecified
1649 : * number of pending IRQ events unhandled. These cases are very rare,
1650 : * so we 'resend' these IRQs via IPIs, to the same CPU. It's much
1651 : * better to do it this way as thus we do not have to be aware of
1652 : * 'pending' interrupts in the IRQ path, except at this point.
1653 : */
1654 : /*
1655 : * Edge triggered needs to resend any interrupt
1656 : * that was delayed but this is now handled in the device
1657 : * independent code.
1658 : */
1659 :
1660 : /*
1661 : * Starting up a edge-triggered IO-APIC interrupt is
1662 : * nasty - we need to make sure that we get the edge.
1663 : * If it is already asserted for some reason, we need
1664 : * return 1 to indicate that is was pending.
1665 : *
1666 : * This is not complete - we should be able to fake
1667 : * an edge even if it isn't on the 8259A...
1668 : */
1669 3 : static unsigned int startup_ioapic_irq(struct irq_data *data)
1670 : {
1671 3 : int was_pending = 0, irq = data->irq;
1672 3 : unsigned long flags;
1673 :
1674 3 : raw_spin_lock_irqsave(&ioapic_lock, flags);
1675 3 : if (irq < nr_legacy_irqs()) {
1676 3 : legacy_pic->mask(irq);
1677 3 : if (legacy_pic->irq_pending(irq))
1678 1 : was_pending = 1;
1679 : }
1680 3 : __unmask_ioapic(data->chip_data);
1681 3 : raw_spin_unlock_irqrestore(&ioapic_lock, flags);
1682 :
1683 3 : return was_pending;
1684 : }
1685 :
1686 : atomic_t irq_mis_count;
1687 :
1688 : #ifdef CONFIG_GENERIC_PENDING_IRQ
1689 0 : static bool io_apic_level_ack_pending(struct mp_chip_data *data)
1690 : {
1691 0 : struct irq_pin_list *entry;
1692 0 : unsigned long flags;
1693 :
1694 0 : raw_spin_lock_irqsave(&ioapic_lock, flags);
1695 0 : for_each_irq_pin(entry, data->irq_2_pin) {
1696 0 : struct IO_APIC_route_entry e;
1697 0 : int pin;
1698 :
1699 0 : pin = entry->pin;
1700 0 : e.w1 = io_apic_read(entry->apic, 0x10 + pin*2);
1701 : /* Is the remote IRR bit set? */
1702 0 : if (e.irr) {
1703 0 : raw_spin_unlock_irqrestore(&ioapic_lock, flags);
1704 0 : return true;
1705 : }
1706 : }
1707 0 : raw_spin_unlock_irqrestore(&ioapic_lock, flags);
1708 :
1709 0 : return false;
1710 : }
1711 :
1712 0 : static inline bool ioapic_prepare_move(struct irq_data *data)
1713 : {
1714 : /* If we are moving the IRQ we need to mask it */
1715 0 : if (unlikely(irqd_is_setaffinity_pending(data))) {
1716 0 : if (!irqd_irq_masked(data))
1717 0 : mask_ioapic_irq(data);
1718 0 : return true;
1719 : }
1720 : return false;
1721 : }
1722 :
1723 0 : static inline void ioapic_finish_move(struct irq_data *data, bool moveit)
1724 : {
1725 0 : if (unlikely(moveit)) {
1726 : /* Only migrate the irq if the ack has been received.
1727 : *
1728 : * On rare occasions the broadcast level triggered ack gets
1729 : * delayed going to ioapics, and if we reprogram the
1730 : * vector while Remote IRR is still set the irq will never
1731 : * fire again.
1732 : *
1733 : * To prevent this scenario we read the Remote IRR bit
1734 : * of the ioapic. This has two effects.
1735 : * - On any sane system the read of the ioapic will
1736 : * flush writes (and acks) going to the ioapic from
1737 : * this cpu.
1738 : * - We get to see if the ACK has actually been delivered.
1739 : *
1740 : * Based on failed experiments of reprogramming the
1741 : * ioapic entry from outside of irq context starting
1742 : * with masking the ioapic entry and then polling until
1743 : * Remote IRR was clear before reprogramming the
1744 : * ioapic I don't trust the Remote IRR bit to be
1745 : * completey accurate.
1746 : *
1747 : * However there appears to be no other way to plug
1748 : * this race, so if the Remote IRR bit is not
1749 : * accurate and is causing problems then it is a hardware bug
1750 : * and you can go talk to the chipset vendor about it.
1751 : */
1752 0 : if (!io_apic_level_ack_pending(data->chip_data))
1753 0 : irq_move_masked_irq(data);
1754 : /* If the IRQ is masked in the core, leave it: */
1755 0 : if (!irqd_irq_masked(data))
1756 0 : unmask_ioapic_irq(data);
1757 : }
1758 0 : }
1759 : #else
1760 : static inline bool ioapic_prepare_move(struct irq_data *data)
1761 : {
1762 : return false;
1763 : }
1764 : static inline void ioapic_finish_move(struct irq_data *data, bool moveit)
1765 : {
1766 : }
1767 : #endif
1768 :
1769 0 : static void ioapic_ack_level(struct irq_data *irq_data)
1770 : {
1771 0 : struct irq_cfg *cfg = irqd_cfg(irq_data);
1772 0 : unsigned long v;
1773 0 : bool moveit;
1774 0 : int i;
1775 :
1776 0 : irq_complete_move(cfg);
1777 0 : moveit = ioapic_prepare_move(irq_data);
1778 :
1779 : /*
1780 : * It appears there is an erratum which affects at least version 0x11
1781 : * of I/O APIC (that's the 82093AA and cores integrated into various
1782 : * chipsets). Under certain conditions a level-triggered interrupt is
1783 : * erroneously delivered as edge-triggered one but the respective IRR
1784 : * bit gets set nevertheless. As a result the I/O unit expects an EOI
1785 : * message but it will never arrive and further interrupts are blocked
1786 : * from the source. The exact reason is so far unknown, but the
1787 : * phenomenon was observed when two consecutive interrupt requests
1788 : * from a given source get delivered to the same CPU and the source is
1789 : * temporarily disabled in between.
1790 : *
1791 : * A workaround is to simulate an EOI message manually. We achieve it
1792 : * by setting the trigger mode to edge and then to level when the edge
1793 : * trigger mode gets detected in the TMR of a local APIC for a
1794 : * level-triggered interrupt. We mask the source for the time of the
1795 : * operation to prevent an edge-triggered interrupt escaping meanwhile.
1796 : * The idea is from Manfred Spraul. --macro
1797 : *
1798 : * Also in the case when cpu goes offline, fixup_irqs() will forward
1799 : * any unhandled interrupt on the offlined cpu to the new cpu
1800 : * destination that is handling the corresponding interrupt. This
1801 : * interrupt forwarding is done via IPI's. Hence, in this case also
1802 : * level-triggered io-apic interrupt will be seen as an edge
1803 : * interrupt in the IRR. And we can't rely on the cpu's EOI
1804 : * to be broadcasted to the IO-APIC's which will clear the remoteIRR
1805 : * corresponding to the level-triggered interrupt. Hence on IO-APIC's
1806 : * supporting EOI register, we do an explicit EOI to clear the
1807 : * remote IRR and on IO-APIC's which don't have an EOI register,
1808 : * we use the above logic (mask+edge followed by unmask+level) from
1809 : * Manfred Spraul to clear the remote IRR.
1810 : */
1811 0 : i = cfg->vector;
1812 0 : v = apic_read(APIC_TMR + ((i & ~0x1f) >> 1));
1813 :
1814 : /*
1815 : * We must acknowledge the irq before we move it or the acknowledge will
1816 : * not propagate properly.
1817 : */
1818 0 : ack_APIC_irq();
1819 :
1820 : /*
1821 : * Tail end of clearing remote IRR bit (either by delivering the EOI
1822 : * message via io-apic EOI register write or simulating it using
1823 : * mask+edge followed by unnask+level logic) manually when the
1824 : * level triggered interrupt is seen as the edge triggered interrupt
1825 : * at the cpu.
1826 : */
1827 0 : if (!(v & (1 << (i & 0x1f)))) {
1828 0 : atomic_inc(&irq_mis_count);
1829 0 : eoi_ioapic_pin(cfg->vector, irq_data->chip_data);
1830 : }
1831 :
1832 0 : ioapic_finish_move(irq_data, moveit);
1833 0 : }
1834 :
1835 0 : static void ioapic_ir_ack_level(struct irq_data *irq_data)
1836 : {
1837 0 : struct mp_chip_data *data = irq_data->chip_data;
1838 :
1839 : /*
1840 : * Intr-remapping uses pin number as the virtual vector
1841 : * in the RTE. Actual vector is programmed in
1842 : * intr-remapping table entry. Hence for the io-apic
1843 : * EOI we use the pin number.
1844 : */
1845 0 : apic_ack_irq(irq_data);
1846 0 : eoi_ioapic_pin(data->entry.vector, data);
1847 0 : }
1848 :
1849 : /*
1850 : * The I/OAPIC is just a device for generating MSI messages from legacy
1851 : * interrupt pins. Various fields of the RTE translate into bits of the
1852 : * resulting MSI which had a historical meaning.
1853 : *
1854 : * With interrupt remapping, many of those bits have different meanings
1855 : * in the underlying MSI, but the way that the I/OAPIC transforms them
1856 : * from its RTE to the MSI message is the same. This function allows
1857 : * the parent IRQ domain to compose the MSI message, then takes the
1858 : * relevant bits to put them in the appropriate places in the RTE in
1859 : * order to generate that message when the IRQ happens.
1860 : *
1861 : * The setup here relies on a preconfigured route entry (is_level,
1862 : * active_low, masked) because the parent domain is merely composing the
1863 : * generic message routing information which is used for the MSI.
1864 : */
1865 6 : static void ioapic_setup_msg_from_msi(struct irq_data *irq_data,
1866 : struct IO_APIC_route_entry *entry)
1867 : {
1868 6 : struct msi_msg msg;
1869 :
1870 : /* Let the parent domain compose the MSI message */
1871 6 : irq_chip_compose_msi_msg(irq_data, &msg);
1872 :
1873 : /*
1874 : * - Real vector
1875 : * - DMAR/IR: 8bit subhandle (ioapic.pin)
1876 : * - AMD/IR: 8bit IRTE index
1877 : */
1878 6 : entry->vector = msg.arch_data.vector;
1879 : /* Delivery mode (for DMAR/IR all 0) */
1880 6 : entry->delivery_mode = msg.arch_data.delivery_mode;
1881 : /* Destination mode or DMAR/IR index bit 15 */
1882 6 : entry->dest_mode_logical = msg.arch_addr_lo.dest_mode_logical;
1883 : /* DMAR/IR: 1, 0 for all other modes */
1884 6 : entry->ir_format = msg.arch_addr_lo.dmar_format;
1885 : /*
1886 : * - DMAR/IR: index bit 0-14.
1887 : *
1888 : * - Virt: If the host supports x2apic without a virtualized IR
1889 : * unit then bit 0-6 of dmar_index_0_14 are providing bit
1890 : * 8-14 of the destination id.
1891 : *
1892 : * All other modes have bit 0-6 of dmar_index_0_14 cleared and the
1893 : * topmost 8 bits are destination id bit 0-7 (entry::destid_0_7).
1894 : */
1895 6 : entry->ir_index_0_14 = msg.arch_addr_lo.dmar_index_0_14;
1896 6 : }
1897 :
1898 6 : static void ioapic_configure_entry(struct irq_data *irqd)
1899 : {
1900 6 : struct mp_chip_data *mpd = irqd->chip_data;
1901 6 : struct irq_pin_list *entry;
1902 :
1903 6 : ioapic_setup_msg_from_msi(irqd, &mpd->entry);
1904 :
1905 12 : for_each_irq_pin(entry, mpd->irq_2_pin)
1906 6 : __ioapic_write_entry(entry->apic, entry->pin, mpd->entry);
1907 6 : }
1908 :
1909 3 : static int ioapic_set_affinity(struct irq_data *irq_data,
1910 : const struct cpumask *mask, bool force)
1911 : {
1912 3 : struct irq_data *parent = irq_data->parent_data;
1913 3 : unsigned long flags;
1914 3 : int ret;
1915 :
1916 3 : ret = parent->chip->irq_set_affinity(parent, mask, force);
1917 3 : raw_spin_lock_irqsave(&ioapic_lock, flags);
1918 3 : if (ret >= 0 && ret != IRQ_SET_MASK_OK_DONE)
1919 3 : ioapic_configure_entry(irq_data);
1920 3 : raw_spin_unlock_irqrestore(&ioapic_lock, flags);
1921 :
1922 3 : return ret;
1923 : }
1924 :
1925 : /*
1926 : * Interrupt shutdown masks the ioapic pin, but the interrupt might already
1927 : * be in flight, but not yet serviced by the target CPU. That means
1928 : * __synchronize_hardirq() would return and claim that everything is calmed
1929 : * down. So free_irq() would proceed and deactivate the interrupt and free
1930 : * resources.
1931 : *
1932 : * Once the target CPU comes around to service it it will find a cleared
1933 : * vector and complain. While the spurious interrupt is harmless, the full
1934 : * release of resources might prevent the interrupt from being acknowledged
1935 : * which keeps the hardware in a weird state.
1936 : *
1937 : * Verify that the corresponding Remote-IRR bits are clear.
1938 : */
1939 0 : static int ioapic_irq_get_chip_state(struct irq_data *irqd,
1940 : enum irqchip_irq_state which,
1941 : bool *state)
1942 : {
1943 0 : struct mp_chip_data *mcd = irqd->chip_data;
1944 0 : struct IO_APIC_route_entry rentry;
1945 0 : struct irq_pin_list *p;
1946 :
1947 0 : if (which != IRQCHIP_STATE_ACTIVE)
1948 : return -EINVAL;
1949 :
1950 0 : *state = false;
1951 0 : raw_spin_lock(&ioapic_lock);
1952 0 : for_each_irq_pin(p, mcd->irq_2_pin) {
1953 0 : rentry = __ioapic_read_entry(p->apic, p->pin);
1954 : /*
1955 : * The remote IRR is only valid in level trigger mode. It's
1956 : * meaning is undefined for edge triggered interrupts and
1957 : * irrelevant because the IO-APIC treats them as fire and
1958 : * forget.
1959 : */
1960 0 : if (rentry.irr && rentry.is_level) {
1961 0 : *state = true;
1962 0 : break;
1963 : }
1964 : }
1965 0 : raw_spin_unlock(&ioapic_lock);
1966 0 : return 0;
1967 : }
1968 :
1969 : static struct irq_chip ioapic_chip __read_mostly = {
1970 : .name = "IO-APIC",
1971 : .irq_startup = startup_ioapic_irq,
1972 : .irq_mask = mask_ioapic_irq,
1973 : .irq_unmask = unmask_ioapic_irq,
1974 : .irq_ack = irq_chip_ack_parent,
1975 : .irq_eoi = ioapic_ack_level,
1976 : .irq_set_affinity = ioapic_set_affinity,
1977 : .irq_retrigger = irq_chip_retrigger_hierarchy,
1978 : .irq_get_irqchip_state = ioapic_irq_get_chip_state,
1979 : .flags = IRQCHIP_SKIP_SET_WAKE,
1980 : };
1981 :
1982 : static struct irq_chip ioapic_ir_chip __read_mostly = {
1983 : .name = "IR-IO-APIC",
1984 : .irq_startup = startup_ioapic_irq,
1985 : .irq_mask = mask_ioapic_irq,
1986 : .irq_unmask = unmask_ioapic_irq,
1987 : .irq_ack = irq_chip_ack_parent,
1988 : .irq_eoi = ioapic_ir_ack_level,
1989 : .irq_set_affinity = ioapic_set_affinity,
1990 : .irq_retrigger = irq_chip_retrigger_hierarchy,
1991 : .irq_get_irqchip_state = ioapic_irq_get_chip_state,
1992 : .flags = IRQCHIP_SKIP_SET_WAKE,
1993 : };
1994 :
1995 1 : static inline void init_IO_APIC_traps(void)
1996 : {
1997 1 : struct irq_cfg *cfg;
1998 1 : unsigned int irq;
1999 :
2000 17 : for_each_active_irq(irq) {
2001 16 : cfg = irq_cfg(irq);
2002 16 : if (IO_APIC_IRQ(irq) && cfg && !cfg->vector) {
2003 : /*
2004 : * Hmm.. We don't have an entry for this,
2005 : * so default to an old-fashioned 8259
2006 : * interrupt if we can..
2007 : */
2008 0 : if (irq < nr_legacy_irqs())
2009 0 : legacy_pic->make_irq(irq);
2010 : else
2011 : /* Strange. Oh, well.. */
2012 0 : irq_set_chip(irq, &no_irq_chip);
2013 : }
2014 : }
2015 1 : }
2016 :
2017 : /*
2018 : * The local APIC irq-chip implementation:
2019 : */
2020 :
2021 0 : static void mask_lapic_irq(struct irq_data *data)
2022 : {
2023 0 : unsigned long v;
2024 :
2025 0 : v = apic_read(APIC_LVT0);
2026 0 : apic_write(APIC_LVT0, v | APIC_LVT_MASKED);
2027 0 : }
2028 :
2029 0 : static void unmask_lapic_irq(struct irq_data *data)
2030 : {
2031 0 : unsigned long v;
2032 :
2033 0 : v = apic_read(APIC_LVT0);
2034 0 : apic_write(APIC_LVT0, v & ~APIC_LVT_MASKED);
2035 0 : }
2036 :
2037 0 : static void ack_lapic_irq(struct irq_data *data)
2038 : {
2039 0 : ack_APIC_irq();
2040 0 : }
2041 :
2042 : static struct irq_chip lapic_chip __read_mostly = {
2043 : .name = "local-APIC",
2044 : .irq_mask = mask_lapic_irq,
2045 : .irq_unmask = unmask_lapic_irq,
2046 : .irq_ack = ack_lapic_irq,
2047 : };
2048 :
2049 0 : static void lapic_register_intr(int irq)
2050 : {
2051 0 : irq_clear_status_flags(irq, IRQ_LEVEL);
2052 0 : irq_set_chip_and_handler_name(irq, &lapic_chip, handle_edge_irq,
2053 : "edge");
2054 0 : }
2055 :
2056 : /*
2057 : * This looks a bit hackish but it's about the only one way of sending
2058 : * a few INTA cycles to 8259As and any associated glue logic. ICR does
2059 : * not support the ExtINT mode, unfortunately. We need to send these
2060 : * cycles as some i82489DX-based boards have glue logic that keeps the
2061 : * 8259A interrupt line asserted until INTA. --macro
2062 : */
2063 0 : static inline void __init unlock_ExtINT_logic(void)
2064 : {
2065 0 : int apic, pin, i;
2066 0 : struct IO_APIC_route_entry entry0, entry1;
2067 0 : unsigned char save_control, save_freq_select;
2068 0 : u32 apic_id;
2069 :
2070 0 : pin = find_isa_irq_pin(8, mp_INT);
2071 0 : if (pin == -1) {
2072 0 : WARN_ON_ONCE(1);
2073 0 : return;
2074 : }
2075 0 : apic = find_isa_irq_apic(8, mp_INT);
2076 0 : if (apic == -1) {
2077 0 : WARN_ON_ONCE(1);
2078 0 : return;
2079 : }
2080 :
2081 0 : entry0 = ioapic_read_entry(apic, pin);
2082 0 : clear_IO_APIC_pin(apic, pin);
2083 :
2084 0 : apic_id = hard_smp_processor_id();
2085 0 : memset(&entry1, 0, sizeof(entry1));
2086 :
2087 0 : entry1.dest_mode_logical = true;
2088 0 : entry1.masked = false;
2089 0 : entry1.destid_0_7 = apic_id & 0xFF;
2090 0 : entry1.virt_destid_8_14 = apic_id >> 8;
2091 0 : entry1.delivery_mode = APIC_DELIVERY_MODE_EXTINT;
2092 0 : entry1.active_low = entry0.active_low;
2093 0 : entry1.is_level = false;
2094 0 : entry1.vector = 0;
2095 :
2096 0 : ioapic_write_entry(apic, pin, entry1);
2097 :
2098 0 : save_control = CMOS_READ(RTC_CONTROL);
2099 0 : save_freq_select = CMOS_READ(RTC_FREQ_SELECT);
2100 0 : CMOS_WRITE((save_freq_select & ~RTC_RATE_SELECT) | 0x6,
2101 : RTC_FREQ_SELECT);
2102 0 : CMOS_WRITE(save_control | RTC_PIE, RTC_CONTROL);
2103 :
2104 0 : i = 100;
2105 0 : while (i-- > 0) {
2106 0 : mdelay(10);
2107 0 : if ((CMOS_READ(RTC_INTR_FLAGS) & RTC_PF) == RTC_PF)
2108 0 : i -= 10;
2109 : }
2110 :
2111 0 : CMOS_WRITE(save_control, RTC_CONTROL);
2112 0 : CMOS_WRITE(save_freq_select, RTC_FREQ_SELECT);
2113 0 : clear_IO_APIC_pin(apic, pin);
2114 :
2115 0 : ioapic_write_entry(apic, pin, entry0);
2116 : }
2117 :
2118 : static int disable_timer_pin_1 __initdata;
2119 : /* Actually the next is obsolete, but keep it for paranoid reasons -AK */
2120 0 : static int __init disable_timer_pin_setup(char *arg)
2121 : {
2122 0 : disable_timer_pin_1 = 1;
2123 0 : return 0;
2124 : }
2125 : early_param("disable_timer_pin_1", disable_timer_pin_setup);
2126 :
2127 0 : static int mp_alloc_timer_irq(int ioapic, int pin)
2128 : {
2129 0 : int irq = -1;
2130 0 : struct irq_domain *domain = mp_ioapic_irqdomain(ioapic);
2131 :
2132 0 : if (domain) {
2133 0 : struct irq_alloc_info info;
2134 :
2135 0 : ioapic_set_alloc_attr(&info, NUMA_NO_NODE, 0, 0);
2136 0 : info.devid = mpc_ioapic_id(ioapic);
2137 0 : info.ioapic.pin = pin;
2138 0 : mutex_lock(&ioapic_mutex);
2139 0 : irq = alloc_isa_irq_from_domain(domain, 0, ioapic, pin, &info);
2140 0 : mutex_unlock(&ioapic_mutex);
2141 : }
2142 :
2143 0 : return irq;
2144 : }
2145 :
2146 : /*
2147 : * This code may look a bit paranoid, but it's supposed to cooperate with
2148 : * a wide range of boards and BIOS bugs. Fortunately only the timer IRQ
2149 : * is so screwy. Thanks to Brian Perkins for testing/hacking this beast
2150 : * fanatically on his truly buggy board.
2151 : *
2152 : * FIXME: really need to revamp this for all platforms.
2153 : */
2154 1 : static inline void __init check_timer(void)
2155 : {
2156 1 : struct irq_data *irq_data = irq_get_irq_data(0);
2157 1 : struct mp_chip_data *data = irq_data->chip_data;
2158 1 : struct irq_cfg *cfg = irqd_cfg(irq_data);
2159 1 : int node = cpu_to_node(0);
2160 1 : int apic1, pin1, apic2, pin2;
2161 1 : int no_pin1 = 0;
2162 :
2163 1 : if (!global_clock_event)
2164 : return;
2165 :
2166 0 : local_irq_disable();
2167 :
2168 : /*
2169 : * get/set the timer IRQ vector:
2170 : */
2171 0 : legacy_pic->mask(0);
2172 :
2173 : /*
2174 : * As IRQ0 is to be enabled in the 8259A, the virtual
2175 : * wire has to be disabled in the local APIC. Also
2176 : * timer interrupts need to be acknowledged manually in
2177 : * the 8259A for the i82489DX when using the NMI
2178 : * watchdog as that APIC treats NMIs as level-triggered.
2179 : * The AEOI mode will finish them in the 8259A
2180 : * automatically.
2181 : */
2182 0 : apic_write(APIC_LVT0, APIC_LVT_MASKED | APIC_DM_EXTINT);
2183 0 : legacy_pic->init(1);
2184 :
2185 0 : pin1 = find_isa_irq_pin(0, mp_INT);
2186 0 : apic1 = find_isa_irq_apic(0, mp_INT);
2187 0 : pin2 = ioapic_i8259.pin;
2188 0 : apic2 = ioapic_i8259.apic;
2189 :
2190 0 : apic_printk(APIC_QUIET, KERN_INFO "..TIMER: vector=0x%02X "
2191 : "apic1=%d pin1=%d apic2=%d pin2=%d\n",
2192 : cfg->vector, apic1, pin1, apic2, pin2);
2193 :
2194 : /*
2195 : * Some BIOS writers are clueless and report the ExtINTA
2196 : * I/O APIC input from the cascaded 8259A as the timer
2197 : * interrupt input. So just in case, if only one pin
2198 : * was found above, try it both directly and through the
2199 : * 8259A.
2200 : */
2201 0 : if (pin1 == -1) {
2202 0 : panic_if_irq_remap("BIOS bug: timer not connected to IO-APIC");
2203 : pin1 = pin2;
2204 : apic1 = apic2;
2205 : no_pin1 = 1;
2206 0 : } else if (pin2 == -1) {
2207 0 : pin2 = pin1;
2208 0 : apic2 = apic1;
2209 : }
2210 :
2211 0 : if (pin1 != -1) {
2212 : /* Ok, does IRQ0 through the IOAPIC work? */
2213 0 : if (no_pin1) {
2214 0 : mp_alloc_timer_irq(apic1, pin1);
2215 : } else {
2216 : /*
2217 : * for edge trigger, it's already unmasked,
2218 : * so only need to unmask if it is level-trigger
2219 : * do we really have level trigger timer?
2220 : */
2221 0 : int idx = find_irq_entry(apic1, pin1, mp_INT);
2222 :
2223 0 : if (idx != -1 && irq_is_level(idx))
2224 0 : unmask_ioapic_irq(irq_get_irq_data(0));
2225 : }
2226 0 : irq_domain_deactivate_irq(irq_data);
2227 0 : irq_domain_activate_irq(irq_data, false);
2228 0 : if (timer_irq_works()) {
2229 0 : if (disable_timer_pin_1 > 0)
2230 0 : clear_IO_APIC_pin(0, pin1);
2231 0 : goto out;
2232 : }
2233 0 : panic_if_irq_remap("timer doesn't work through Interrupt-remapped IO-APIC");
2234 0 : clear_IO_APIC_pin(apic1, pin1);
2235 0 : if (!no_pin1)
2236 0 : apic_printk(APIC_QUIET, KERN_ERR "..MP-BIOS bug: "
2237 : "8254 timer not connected to IO-APIC\n");
2238 :
2239 0 : apic_printk(APIC_QUIET, KERN_INFO "...trying to set up timer "
2240 : "(IRQ0) through the 8259A ...\n");
2241 0 : apic_printk(APIC_QUIET, KERN_INFO
2242 : "..... (found apic %d pin %d) ...\n", apic2, pin2);
2243 : /*
2244 : * legacy devices should be connected to IO APIC #0
2245 : */
2246 0 : replace_pin_at_irq_node(data, node, apic1, pin1, apic2, pin2);
2247 0 : irq_domain_deactivate_irq(irq_data);
2248 0 : irq_domain_activate_irq(irq_data, false);
2249 0 : legacy_pic->unmask(0);
2250 0 : if (timer_irq_works()) {
2251 0 : apic_printk(APIC_QUIET, KERN_INFO "....... works.\n");
2252 0 : goto out;
2253 : }
2254 : /*
2255 : * Cleanup, just in case ...
2256 : */
2257 0 : legacy_pic->mask(0);
2258 0 : clear_IO_APIC_pin(apic2, pin2);
2259 0 : apic_printk(APIC_QUIET, KERN_INFO "....... failed.\n");
2260 : }
2261 :
2262 0 : apic_printk(APIC_QUIET, KERN_INFO
2263 : "...trying to set up timer as Virtual Wire IRQ...\n");
2264 :
2265 0 : lapic_register_intr(0);
2266 0 : apic_write(APIC_LVT0, APIC_DM_FIXED | cfg->vector); /* Fixed mode */
2267 0 : legacy_pic->unmask(0);
2268 :
2269 0 : if (timer_irq_works()) {
2270 0 : apic_printk(APIC_QUIET, KERN_INFO "..... works.\n");
2271 0 : goto out;
2272 : }
2273 0 : legacy_pic->mask(0);
2274 0 : apic_write(APIC_LVT0, APIC_LVT_MASKED | APIC_DM_FIXED | cfg->vector);
2275 0 : apic_printk(APIC_QUIET, KERN_INFO "..... failed.\n");
2276 :
2277 0 : apic_printk(APIC_QUIET, KERN_INFO
2278 : "...trying to set up timer as ExtINT IRQ...\n");
2279 :
2280 0 : legacy_pic->init(0);
2281 0 : legacy_pic->make_irq(0);
2282 0 : apic_write(APIC_LVT0, APIC_DM_EXTINT);
2283 0 : legacy_pic->unmask(0);
2284 :
2285 0 : unlock_ExtINT_logic();
2286 :
2287 0 : if (timer_irq_works()) {
2288 0 : apic_printk(APIC_QUIET, KERN_INFO "..... works.\n");
2289 0 : goto out;
2290 : }
2291 0 : apic_printk(APIC_QUIET, KERN_INFO "..... failed :(.\n");
2292 0 : if (apic_is_x2apic_enabled())
2293 0 : apic_printk(APIC_QUIET, KERN_INFO
2294 : "Perhaps problem with the pre-enabled x2apic mode\n"
2295 : "Try booting with x2apic and interrupt-remapping disabled in the bios.\n");
2296 0 : panic("IO-APIC + timer doesn't work! Boot with apic=debug and send a "
2297 : "report. Then try booting with the 'noapic' option.\n");
2298 0 : out:
2299 0 : local_irq_enable();
2300 : }
2301 :
2302 : /*
2303 : * Traditionally ISA IRQ2 is the cascade IRQ, and is not available
2304 : * to devices. However there may be an I/O APIC pin available for
2305 : * this interrupt regardless. The pin may be left unconnected, but
2306 : * typically it will be reused as an ExtINT cascade interrupt for
2307 : * the master 8259A. In the MPS case such a pin will normally be
2308 : * reported as an ExtINT interrupt in the MP table. With ACPI
2309 : * there is no provision for ExtINT interrupts, and in the absence
2310 : * of an override it would be treated as an ordinary ISA I/O APIC
2311 : * interrupt, that is edge-triggered and unmasked by default. We
2312 : * used to do this, but it caused problems on some systems because
2313 : * of the NMI watchdog and sometimes IRQ0 of the 8254 timer using
2314 : * the same ExtINT cascade interrupt to drive the local APIC of the
2315 : * bootstrap processor. Therefore we refrain from routing IRQ2 to
2316 : * the I/O APIC in all cases now. No actual device should request
2317 : * it anyway. --macro
2318 : */
2319 : #define PIC_IRQS (1UL << PIC_CASCADE_IR)
2320 :
2321 1 : static int mp_irqdomain_create(int ioapic)
2322 : {
2323 1 : struct irq_domain *parent;
2324 1 : int hwirqs = mp_ioapic_pin_count(ioapic);
2325 1 : struct ioapic *ip = &ioapics[ioapic];
2326 1 : struct ioapic_domain_cfg *cfg = &ip->irqdomain_cfg;
2327 1 : struct mp_ioapic_gsi *gsi_cfg = mp_ioapic_gsi_routing(ioapic);
2328 1 : struct fwnode_handle *fn;
2329 1 : struct irq_fwspec fwspec;
2330 :
2331 1 : if (cfg->type == IOAPIC_DOMAIN_INVALID)
2332 : return 0;
2333 :
2334 : /* Handle device tree enumerated APICs proper */
2335 1 : if (cfg->dev) {
2336 0 : fn = of_node_to_fwnode(cfg->dev);
2337 : } else {
2338 1 : fn = irq_domain_alloc_named_id_fwnode("IO-APIC", mpc_ioapic_id(ioapic));
2339 1 : if (!fn)
2340 : return -ENOMEM;
2341 : }
2342 :
2343 1 : fwspec.fwnode = fn;
2344 1 : fwspec.param_count = 1;
2345 1 : fwspec.param[0] = mpc_ioapic_id(ioapic);
2346 :
2347 1 : parent = irq_find_matching_fwspec(&fwspec, DOMAIN_BUS_ANY);
2348 1 : if (!parent) {
2349 0 : if (!cfg->dev)
2350 0 : irq_domain_free_fwnode(fn);
2351 0 : return -ENODEV;
2352 : }
2353 :
2354 2 : ip->irqdomain = irq_domain_create_linear(fn, hwirqs, cfg->ops,
2355 1 : (void *)(long)ioapic);
2356 :
2357 1 : if (!ip->irqdomain) {
2358 : /* Release fw handle if it was allocated above */
2359 0 : if (!cfg->dev)
2360 0 : irq_domain_free_fwnode(fn);
2361 0 : return -ENOMEM;
2362 : }
2363 :
2364 1 : ip->irqdomain->parent = parent;
2365 :
2366 1 : if (cfg->type == IOAPIC_DOMAIN_LEGACY ||
2367 : cfg->type == IOAPIC_DOMAIN_STRICT)
2368 1 : ioapic_dynirq_base = max(ioapic_dynirq_base,
2369 : gsi_cfg->gsi_end + 1);
2370 :
2371 : return 0;
2372 : }
2373 :
2374 0 : static void ioapic_destroy_irqdomain(int idx)
2375 : {
2376 0 : struct ioapic_domain_cfg *cfg = &ioapics[idx].irqdomain_cfg;
2377 0 : struct fwnode_handle *fn = ioapics[idx].irqdomain->fwnode;
2378 :
2379 0 : if (ioapics[idx].irqdomain) {
2380 0 : irq_domain_remove(ioapics[idx].irqdomain);
2381 0 : if (!cfg->dev)
2382 0 : irq_domain_free_fwnode(fn);
2383 0 : ioapics[idx].irqdomain = NULL;
2384 : }
2385 0 : }
2386 :
2387 1 : void __init setup_IO_APIC(void)
2388 : {
2389 1 : int ioapic;
2390 :
2391 1 : if (skip_ioapic_setup || !nr_ioapics)
2392 : return;
2393 :
2394 1 : io_apic_irqs = nr_legacy_irqs() ? ~PIC_IRQS : ~0UL;
2395 :
2396 1 : apic_printk(APIC_VERBOSE, "ENABLING IO-APIC IRQs\n");
2397 2 : for_each_ioapic(ioapic)
2398 1 : BUG_ON(mp_irqdomain_create(ioapic));
2399 :
2400 : /*
2401 : * Set up IO-APIC IRQ routing.
2402 : */
2403 1 : x86_init.mpparse.setup_ioapic_ids();
2404 :
2405 1 : sync_Arb_IDs();
2406 1 : setup_IO_APIC_irqs();
2407 1 : init_IO_APIC_traps();
2408 1 : if (nr_legacy_irqs())
2409 1 : check_timer();
2410 :
2411 1 : ioapic_initialized = 1;
2412 : }
2413 :
2414 0 : static void resume_ioapic_id(int ioapic_idx)
2415 : {
2416 0 : unsigned long flags;
2417 0 : union IO_APIC_reg_00 reg_00;
2418 :
2419 0 : raw_spin_lock_irqsave(&ioapic_lock, flags);
2420 0 : reg_00.raw = io_apic_read(ioapic_idx, 0);
2421 0 : if (reg_00.bits.ID != mpc_ioapic_id(ioapic_idx)) {
2422 0 : reg_00.bits.ID = mpc_ioapic_id(ioapic_idx);
2423 0 : io_apic_write(ioapic_idx, 0, reg_00.raw);
2424 : }
2425 0 : raw_spin_unlock_irqrestore(&ioapic_lock, flags);
2426 0 : }
2427 :
2428 0 : static void ioapic_resume(void)
2429 : {
2430 0 : int ioapic_idx;
2431 :
2432 0 : for_each_ioapic_reverse(ioapic_idx)
2433 0 : resume_ioapic_id(ioapic_idx);
2434 :
2435 0 : restore_ioapic_entries();
2436 0 : }
2437 :
2438 : static struct syscore_ops ioapic_syscore_ops = {
2439 : .suspend = save_ioapic_entries,
2440 : .resume = ioapic_resume,
2441 : };
2442 :
2443 1 : static int __init ioapic_init_ops(void)
2444 : {
2445 1 : register_syscore_ops(&ioapic_syscore_ops);
2446 :
2447 1 : return 0;
2448 : }
2449 :
2450 : device_initcall(ioapic_init_ops);
2451 :
2452 1 : static int io_apic_get_redir_entries(int ioapic)
2453 : {
2454 1 : union IO_APIC_reg_01 reg_01;
2455 1 : unsigned long flags;
2456 :
2457 1 : raw_spin_lock_irqsave(&ioapic_lock, flags);
2458 1 : reg_01.raw = io_apic_read(ioapic, 1);
2459 1 : raw_spin_unlock_irqrestore(&ioapic_lock, flags);
2460 :
2461 : /* The register returns the maximum index redir index
2462 : * supported, which is one less than the total number of redir
2463 : * entries.
2464 : */
2465 1 : return reg_01.bits.entries + 1;
2466 : }
2467 :
2468 0 : unsigned int arch_dynirq_lower_bound(unsigned int from)
2469 : {
2470 : /*
2471 : * dmar_alloc_hwirq() may be called before setup_IO_APIC(), so use
2472 : * gsi_top if ioapic_dynirq_base hasn't been initialized yet.
2473 : */
2474 0 : if (!ioapic_initialized)
2475 0 : return gsi_top;
2476 : /*
2477 : * For DT enabled machines ioapic_dynirq_base is irrelevant and not
2478 : * updated. So simply return @from if ioapic_dynirq_base == 0.
2479 : */
2480 0 : return ioapic_dynirq_base ? : from;
2481 : }
2482 :
2483 : #ifdef CONFIG_X86_32
2484 : static int io_apic_get_unique_id(int ioapic, int apic_id)
2485 : {
2486 : union IO_APIC_reg_00 reg_00;
2487 : static physid_mask_t apic_id_map = PHYSID_MASK_NONE;
2488 : physid_mask_t tmp;
2489 : unsigned long flags;
2490 : int i = 0;
2491 :
2492 : /*
2493 : * The P4 platform supports up to 256 APIC IDs on two separate APIC
2494 : * buses (one for LAPICs, one for IOAPICs), where predecessors only
2495 : * supports up to 16 on one shared APIC bus.
2496 : *
2497 : * TBD: Expand LAPIC/IOAPIC support on P4-class systems to take full
2498 : * advantage of new APIC bus architecture.
2499 : */
2500 :
2501 : if (physids_empty(apic_id_map))
2502 : apic->ioapic_phys_id_map(&phys_cpu_present_map, &apic_id_map);
2503 :
2504 : raw_spin_lock_irqsave(&ioapic_lock, flags);
2505 : reg_00.raw = io_apic_read(ioapic, 0);
2506 : raw_spin_unlock_irqrestore(&ioapic_lock, flags);
2507 :
2508 : if (apic_id >= get_physical_broadcast()) {
2509 : printk(KERN_WARNING "IOAPIC[%d]: Invalid apic_id %d, trying "
2510 : "%d\n", ioapic, apic_id, reg_00.bits.ID);
2511 : apic_id = reg_00.bits.ID;
2512 : }
2513 :
2514 : /*
2515 : * Every APIC in a system must have a unique ID or we get lots of nice
2516 : * 'stuck on smp_invalidate_needed IPI wait' messages.
2517 : */
2518 : if (apic->check_apicid_used(&apic_id_map, apic_id)) {
2519 :
2520 : for (i = 0; i < get_physical_broadcast(); i++) {
2521 : if (!apic->check_apicid_used(&apic_id_map, i))
2522 : break;
2523 : }
2524 :
2525 : if (i == get_physical_broadcast())
2526 : panic("Max apic_id exceeded!\n");
2527 :
2528 : printk(KERN_WARNING "IOAPIC[%d]: apic_id %d already used, "
2529 : "trying %d\n", ioapic, apic_id, i);
2530 :
2531 : apic_id = i;
2532 : }
2533 :
2534 : apic->apicid_to_cpu_present(apic_id, &tmp);
2535 : physids_or(apic_id_map, apic_id_map, tmp);
2536 :
2537 : if (reg_00.bits.ID != apic_id) {
2538 : reg_00.bits.ID = apic_id;
2539 :
2540 : raw_spin_lock_irqsave(&ioapic_lock, flags);
2541 : io_apic_write(ioapic, 0, reg_00.raw);
2542 : reg_00.raw = io_apic_read(ioapic, 0);
2543 : raw_spin_unlock_irqrestore(&ioapic_lock, flags);
2544 :
2545 : /* Sanity check */
2546 : if (reg_00.bits.ID != apic_id) {
2547 : pr_err("IOAPIC[%d]: Unable to change apic_id!\n",
2548 : ioapic);
2549 : return -1;
2550 : }
2551 : }
2552 :
2553 : apic_printk(APIC_VERBOSE, KERN_INFO
2554 : "IOAPIC[%d]: Assigned apic_id %d\n", ioapic, apic_id);
2555 :
2556 : return apic_id;
2557 : }
2558 :
2559 : static u8 io_apic_unique_id(int idx, u8 id)
2560 : {
2561 : if ((boot_cpu_data.x86_vendor == X86_VENDOR_INTEL) &&
2562 : !APIC_XAPIC(boot_cpu_apic_version))
2563 : return io_apic_get_unique_id(idx, id);
2564 : else
2565 : return id;
2566 : }
2567 : #else
2568 1 : static u8 io_apic_unique_id(int idx, u8 id)
2569 : {
2570 1 : union IO_APIC_reg_00 reg_00;
2571 1 : DECLARE_BITMAP(used, 256);
2572 1 : unsigned long flags;
2573 1 : u8 new_id;
2574 1 : int i;
2575 :
2576 1 : bitmap_zero(used, 256);
2577 1 : for_each_ioapic(i)
2578 0 : __set_bit(mpc_ioapic_id(i), used);
2579 :
2580 : /* Hand out the requested id if available */
2581 1 : if (!test_bit(id, used))
2582 : return id;
2583 :
2584 : /*
2585 : * Read the current id from the ioapic and keep it if
2586 : * available.
2587 : */
2588 0 : raw_spin_lock_irqsave(&ioapic_lock, flags);
2589 0 : reg_00.raw = io_apic_read(idx, 0);
2590 0 : raw_spin_unlock_irqrestore(&ioapic_lock, flags);
2591 0 : new_id = reg_00.bits.ID;
2592 0 : if (!test_bit(new_id, used)) {
2593 0 : apic_printk(APIC_VERBOSE, KERN_INFO
2594 : "IOAPIC[%d]: Using reg apic_id %d instead of %d\n",
2595 : idx, new_id, id);
2596 0 : return new_id;
2597 : }
2598 :
2599 : /*
2600 : * Get the next free id and write it to the ioapic.
2601 : */
2602 0 : new_id = find_first_zero_bit(used, 256);
2603 0 : reg_00.bits.ID = new_id;
2604 0 : raw_spin_lock_irqsave(&ioapic_lock, flags);
2605 0 : io_apic_write(idx, 0, reg_00.raw);
2606 0 : reg_00.raw = io_apic_read(idx, 0);
2607 0 : raw_spin_unlock_irqrestore(&ioapic_lock, flags);
2608 : /* Sanity check */
2609 0 : BUG_ON(reg_00.bits.ID != new_id);
2610 :
2611 : return new_id;
2612 : }
2613 : #endif
2614 :
2615 1 : static int io_apic_get_version(int ioapic)
2616 : {
2617 1 : union IO_APIC_reg_01 reg_01;
2618 1 : unsigned long flags;
2619 :
2620 1 : raw_spin_lock_irqsave(&ioapic_lock, flags);
2621 1 : reg_01.raw = io_apic_read(ioapic, 1);
2622 1 : raw_spin_unlock_irqrestore(&ioapic_lock, flags);
2623 :
2624 1 : return reg_01.bits.version;
2625 : }
2626 :
2627 : /*
2628 : * This function updates target affinity of IOAPIC interrupts to include
2629 : * the CPUs which came online during SMP bringup.
2630 : */
2631 : #define IOAPIC_RESOURCE_NAME_SIZE 11
2632 :
2633 : static struct resource *ioapic_resources;
2634 :
2635 1 : static struct resource * __init ioapic_setup_resources(void)
2636 : {
2637 1 : unsigned long n;
2638 1 : struct resource *res;
2639 1 : char *mem;
2640 1 : int i;
2641 :
2642 1 : if (nr_ioapics == 0)
2643 : return NULL;
2644 :
2645 1 : n = IOAPIC_RESOURCE_NAME_SIZE + sizeof(struct resource);
2646 1 : n *= nr_ioapics;
2647 :
2648 1 : mem = memblock_alloc(n, SMP_CACHE_BYTES);
2649 1 : if (!mem)
2650 0 : panic("%s: Failed to allocate %lu bytes\n", __func__, n);
2651 1 : res = (void *)mem;
2652 :
2653 1 : mem += sizeof(struct resource) * nr_ioapics;
2654 :
2655 2 : for_each_ioapic(i) {
2656 1 : res[i].name = mem;
2657 1 : res[i].flags = IORESOURCE_MEM | IORESOURCE_BUSY;
2658 1 : snprintf(mem, IOAPIC_RESOURCE_NAME_SIZE, "IOAPIC %u", i);
2659 1 : mem += IOAPIC_RESOURCE_NAME_SIZE;
2660 1 : ioapics[i].iomem_res = &res[i];
2661 : }
2662 :
2663 1 : ioapic_resources = res;
2664 :
2665 1 : return res;
2666 : }
2667 :
2668 1 : void __init io_apic_init_mappings(void)
2669 : {
2670 1 : unsigned long ioapic_phys, idx = FIX_IO_APIC_BASE_0;
2671 1 : struct resource *ioapic_res;
2672 1 : int i;
2673 :
2674 1 : ioapic_res = ioapic_setup_resources();
2675 3 : for_each_ioapic(i) {
2676 1 : if (smp_found_config) {
2677 1 : ioapic_phys = mpc_ioapic_addr(i);
2678 : #ifdef CONFIG_X86_32
2679 : if (!ioapic_phys) {
2680 : printk(KERN_ERR
2681 : "WARNING: bogus zero IO-APIC "
2682 : "address found in MPTABLE, "
2683 : "disabling IO/APIC support!\n");
2684 : smp_found_config = 0;
2685 : skip_ioapic_setup = 1;
2686 : goto fake_ioapic_page;
2687 : }
2688 : #endif
2689 : } else {
2690 : #ifdef CONFIG_X86_32
2691 : fake_ioapic_page:
2692 : #endif
2693 0 : ioapic_phys = (unsigned long)memblock_alloc(PAGE_SIZE,
2694 : PAGE_SIZE);
2695 0 : if (!ioapic_phys)
2696 0 : panic("%s: Failed to allocate %lu bytes align=0x%lx\n",
2697 : __func__, PAGE_SIZE, PAGE_SIZE);
2698 0 : ioapic_phys = __pa(ioapic_phys);
2699 : }
2700 1 : set_fixmap_nocache(idx, ioapic_phys);
2701 1 : apic_printk(APIC_VERBOSE, "mapped IOAPIC to %08lx (%08lx)\n",
2702 : __fix_to_virt(idx) + (ioapic_phys & ~PAGE_MASK),
2703 : ioapic_phys);
2704 1 : idx++;
2705 :
2706 1 : ioapic_res->start = ioapic_phys;
2707 1 : ioapic_res->end = ioapic_phys + IO_APIC_SLOT_SIZE - 1;
2708 1 : ioapic_res++;
2709 : }
2710 1 : }
2711 :
2712 0 : void __init ioapic_insert_resources(void)
2713 : {
2714 0 : int i;
2715 0 : struct resource *r = ioapic_resources;
2716 :
2717 0 : if (!r) {
2718 0 : if (nr_ioapics > 0)
2719 0 : printk(KERN_ERR
2720 : "IO APIC resources couldn't be allocated.\n");
2721 0 : return;
2722 : }
2723 :
2724 0 : for_each_ioapic(i) {
2725 0 : insert_resource(&iomem_resource, r);
2726 0 : r++;
2727 : }
2728 : }
2729 :
2730 15 : int mp_find_ioapic(u32 gsi)
2731 : {
2732 15 : int i;
2733 :
2734 15 : if (nr_ioapics == 0)
2735 : return -1;
2736 :
2737 : /* Find the IOAPIC that manages this GSI. */
2738 15 : for_each_ioapic(i) {
2739 15 : struct mp_ioapic_gsi *gsi_cfg = mp_ioapic_gsi_routing(i);
2740 15 : if (gsi >= gsi_cfg->gsi_base && gsi <= gsi_cfg->gsi_end)
2741 15 : return i;
2742 : }
2743 :
2744 0 : printk(KERN_ERR "ERROR: Unable to locate IOAPIC for GSI %d\n", gsi);
2745 0 : return -1;
2746 : }
2747 :
2748 15 : int mp_find_ioapic_pin(int ioapic, u32 gsi)
2749 : {
2750 15 : struct mp_ioapic_gsi *gsi_cfg;
2751 :
2752 15 : if (WARN_ON(ioapic < 0))
2753 : return -1;
2754 :
2755 15 : gsi_cfg = mp_ioapic_gsi_routing(ioapic);
2756 15 : if (WARN_ON(gsi > gsi_cfg->gsi_end))
2757 : return -1;
2758 :
2759 15 : return gsi - gsi_cfg->gsi_base;
2760 : }
2761 :
2762 1 : static int bad_ioapic_register(int idx)
2763 : {
2764 1 : union IO_APIC_reg_00 reg_00;
2765 1 : union IO_APIC_reg_01 reg_01;
2766 1 : union IO_APIC_reg_02 reg_02;
2767 :
2768 1 : reg_00.raw = io_apic_read(idx, 0);
2769 1 : reg_01.raw = io_apic_read(idx, 1);
2770 1 : reg_02.raw = io_apic_read(idx, 2);
2771 :
2772 1 : if (reg_00.raw == -1 && reg_01.raw == -1 && reg_02.raw == -1) {
2773 0 : pr_warn("I/O APIC 0x%x registers return all ones, skipping!\n",
2774 : mpc_ioapic_addr(idx));
2775 0 : return 1;
2776 : }
2777 :
2778 : return 0;
2779 : }
2780 :
2781 : static int find_free_ioapic_entry(void)
2782 : {
2783 : int idx;
2784 :
2785 1 : for (idx = 0; idx < MAX_IO_APICS; idx++)
2786 1 : if (ioapics[idx].nr_registers == 0)
2787 : return idx;
2788 :
2789 : return MAX_IO_APICS;
2790 : }
2791 :
2792 : /**
2793 : * mp_register_ioapic - Register an IOAPIC device
2794 : * @id: hardware IOAPIC ID
2795 : * @address: physical address of IOAPIC register area
2796 : * @gsi_base: base of GSI associated with the IOAPIC
2797 : * @cfg: configuration information for the IOAPIC
2798 : */
2799 1 : int mp_register_ioapic(int id, u32 address, u32 gsi_base,
2800 : struct ioapic_domain_cfg *cfg)
2801 : {
2802 1 : bool hotplug = !!ioapic_initialized;
2803 1 : struct mp_ioapic_gsi *gsi_cfg;
2804 1 : int idx, ioapic, entries;
2805 1 : u32 gsi_end;
2806 :
2807 1 : if (!address) {
2808 0 : pr_warn("Bogus (zero) I/O APIC address found, skipping!\n");
2809 0 : return -EINVAL;
2810 : }
2811 1 : for_each_ioapic(ioapic)
2812 0 : if (ioapics[ioapic].mp_config.apicaddr == address) {
2813 0 : pr_warn("address 0x%x conflicts with IOAPIC%d\n",
2814 : address, ioapic);
2815 0 : return -EEXIST;
2816 : }
2817 :
2818 1 : idx = find_free_ioapic_entry();
2819 1 : if (idx >= MAX_IO_APICS) {
2820 0 : pr_warn("Max # of I/O APICs (%d) exceeded (found %d), skipping\n",
2821 : MAX_IO_APICS, idx);
2822 0 : return -ENOSPC;
2823 : }
2824 :
2825 1 : ioapics[idx].mp_config.type = MP_IOAPIC;
2826 1 : ioapics[idx].mp_config.flags = MPC_APIC_USABLE;
2827 1 : ioapics[idx].mp_config.apicaddr = address;
2828 :
2829 1 : set_fixmap_nocache(FIX_IO_APIC_BASE_0 + idx, address);
2830 1 : if (bad_ioapic_register(idx)) {
2831 0 : clear_fixmap(FIX_IO_APIC_BASE_0 + idx);
2832 0 : return -ENODEV;
2833 : }
2834 :
2835 1 : ioapics[idx].mp_config.apicid = io_apic_unique_id(idx, id);
2836 1 : ioapics[idx].mp_config.apicver = io_apic_get_version(idx);
2837 :
2838 : /*
2839 : * Build basic GSI lookup table to facilitate gsi->io_apic lookups
2840 : * and to prevent reprogramming of IOAPIC pins (PCI GSIs).
2841 : */
2842 1 : entries = io_apic_get_redir_entries(idx);
2843 1 : gsi_end = gsi_base + entries - 1;
2844 1 : for_each_ioapic(ioapic) {
2845 0 : gsi_cfg = mp_ioapic_gsi_routing(ioapic);
2846 0 : if ((gsi_base >= gsi_cfg->gsi_base &&
2847 0 : gsi_base <= gsi_cfg->gsi_end) ||
2848 0 : (gsi_end >= gsi_cfg->gsi_base &&
2849 0 : gsi_end <= gsi_cfg->gsi_end)) {
2850 0 : pr_warn("GSI range [%u-%u] for new IOAPIC conflicts with GSI[%u-%u]\n",
2851 : gsi_base, gsi_end,
2852 : gsi_cfg->gsi_base, gsi_cfg->gsi_end);
2853 0 : clear_fixmap(FIX_IO_APIC_BASE_0 + idx);
2854 0 : return -ENOSPC;
2855 : }
2856 : }
2857 1 : gsi_cfg = mp_ioapic_gsi_routing(idx);
2858 1 : gsi_cfg->gsi_base = gsi_base;
2859 1 : gsi_cfg->gsi_end = gsi_end;
2860 :
2861 1 : ioapics[idx].irqdomain = NULL;
2862 1 : ioapics[idx].irqdomain_cfg = *cfg;
2863 :
2864 : /*
2865 : * If mp_register_ioapic() is called during early boot stage when
2866 : * walking ACPI/DT tables, it's too early to create irqdomain,
2867 : * we are still using bootmem allocator. So delay it to setup_IO_APIC().
2868 : */
2869 1 : if (hotplug) {
2870 0 : if (mp_irqdomain_create(idx)) {
2871 0 : clear_fixmap(FIX_IO_APIC_BASE_0 + idx);
2872 0 : return -ENOMEM;
2873 : }
2874 0 : alloc_ioapic_saved_registers(idx);
2875 : }
2876 :
2877 1 : if (gsi_cfg->gsi_end >= gsi_top)
2878 1 : gsi_top = gsi_cfg->gsi_end + 1;
2879 1 : if (nr_ioapics <= idx)
2880 1 : nr_ioapics = idx + 1;
2881 :
2882 : /* Set nr_registers to mark entry present */
2883 1 : ioapics[idx].nr_registers = entries;
2884 :
2885 1 : pr_info("IOAPIC[%d]: apic_id %d, version %d, address 0x%x, GSI %d-%d\n",
2886 : idx, mpc_ioapic_id(idx),
2887 : mpc_ioapic_ver(idx), mpc_ioapic_addr(idx),
2888 : gsi_cfg->gsi_base, gsi_cfg->gsi_end);
2889 :
2890 1 : return 0;
2891 : }
2892 :
2893 0 : int mp_unregister_ioapic(u32 gsi_base)
2894 : {
2895 0 : int ioapic, pin;
2896 0 : int found = 0;
2897 :
2898 0 : for_each_ioapic(ioapic)
2899 0 : if (ioapics[ioapic].gsi_config.gsi_base == gsi_base) {
2900 : found = 1;
2901 : break;
2902 : }
2903 0 : if (!found) {
2904 0 : pr_warn("can't find IOAPIC for GSI %d\n", gsi_base);
2905 0 : return -ENODEV;
2906 : }
2907 :
2908 0 : for_each_pin(ioapic, pin) {
2909 0 : u32 gsi = mp_pin_to_gsi(ioapic, pin);
2910 0 : int irq = mp_map_gsi_to_irq(gsi, 0, NULL);
2911 0 : struct mp_chip_data *data;
2912 :
2913 0 : if (irq >= 0) {
2914 0 : data = irq_get_chip_data(irq);
2915 0 : if (data && data->count) {
2916 0 : pr_warn("pin%d on IOAPIC%d is still in use.\n",
2917 : pin, ioapic);
2918 0 : return -EBUSY;
2919 : }
2920 : }
2921 : }
2922 :
2923 : /* Mark entry not present */
2924 0 : ioapics[ioapic].nr_registers = 0;
2925 0 : ioapic_destroy_irqdomain(ioapic);
2926 0 : free_ioapic_saved_registers(ioapic);
2927 0 : if (ioapics[ioapic].iomem_res)
2928 0 : release_resource(ioapics[ioapic].iomem_res);
2929 0 : clear_fixmap(FIX_IO_APIC_BASE_0 + ioapic);
2930 0 : memset(&ioapics[ioapic], 0, sizeof(ioapics[ioapic]));
2931 :
2932 0 : return 0;
2933 : }
2934 :
2935 0 : int mp_ioapic_registered(u32 gsi_base)
2936 : {
2937 0 : int ioapic;
2938 :
2939 0 : for_each_ioapic(ioapic)
2940 0 : if (ioapics[ioapic].gsi_config.gsi_base == gsi_base)
2941 : return 1;
2942 :
2943 : return 0;
2944 : }
2945 :
2946 15 : static void mp_irqdomain_get_attr(u32 gsi, struct mp_chip_data *data,
2947 : struct irq_alloc_info *info)
2948 : {
2949 15 : if (info && info->ioapic.valid) {
2950 15 : data->is_level = info->ioapic.is_level;
2951 15 : data->active_low = info->ioapic.active_low;
2952 0 : } else if (__acpi_get_override_irq(gsi, &data->is_level,
2953 : &data->active_low) < 0) {
2954 : /* PCI interrupts are always active low level triggered. */
2955 0 : data->is_level = true;
2956 0 : data->active_low = true;
2957 : }
2958 15 : }
2959 :
2960 : /*
2961 : * Configure the I/O-APIC specific fields in the routing entry.
2962 : *
2963 : * This is important to setup the I/O-APIC specific bits (is_level,
2964 : * active_low, masked) because the underlying parent domain will only
2965 : * provide the routing information and is oblivious of the I/O-APIC
2966 : * specific bits.
2967 : *
2968 : * The entry is just preconfigured at this point and not written into the
2969 : * RTE. This happens later during activation which will fill in the actual
2970 : * routing information.
2971 : */
2972 15 : static void mp_preconfigure_entry(struct mp_chip_data *data)
2973 : {
2974 15 : struct IO_APIC_route_entry *entry = &data->entry;
2975 :
2976 15 : memset(entry, 0, sizeof(*entry));
2977 15 : entry->is_level = data->is_level;
2978 15 : entry->active_low = data->active_low;
2979 : /*
2980 : * Mask level triggered irqs. Edge triggered irqs are masked
2981 : * by the irq core code in case they fire.
2982 : */
2983 15 : entry->masked = data->is_level;
2984 15 : }
2985 :
2986 15 : int mp_irqdomain_alloc(struct irq_domain *domain, unsigned int virq,
2987 : unsigned int nr_irqs, void *arg)
2988 : {
2989 15 : struct irq_alloc_info *info = arg;
2990 15 : struct mp_chip_data *data;
2991 15 : struct irq_data *irq_data;
2992 15 : int ret, ioapic, pin;
2993 15 : unsigned long flags;
2994 :
2995 15 : if (!info || nr_irqs > 1)
2996 : return -EINVAL;
2997 15 : irq_data = irq_domain_get_irq_data(domain, virq);
2998 15 : if (!irq_data)
2999 : return -EINVAL;
3000 :
3001 15 : ioapic = mp_irqdomain_ioapic_idx(domain);
3002 15 : pin = info->ioapic.pin;
3003 15 : if (irq_find_mapping(domain, (irq_hw_number_t)pin) > 0)
3004 : return -EEXIST;
3005 :
3006 15 : data = kzalloc(sizeof(*data), GFP_KERNEL);
3007 15 : if (!data)
3008 : return -ENOMEM;
3009 :
3010 15 : ret = irq_domain_alloc_irqs_parent(domain, virq, nr_irqs, info);
3011 15 : if (ret < 0) {
3012 0 : kfree(data);
3013 0 : return ret;
3014 : }
3015 :
3016 15 : INIT_LIST_HEAD(&data->irq_2_pin);
3017 15 : irq_data->hwirq = info->ioapic.pin;
3018 30 : irq_data->chip = (domain->parent == x86_vector_domain) ?
3019 15 : &ioapic_chip : &ioapic_ir_chip;
3020 15 : irq_data->chip_data = data;
3021 15 : mp_irqdomain_get_attr(mp_pin_to_gsi(ioapic, pin), data, info);
3022 :
3023 30 : add_pin_to_irq_node(data, ioapic_alloc_attr_node(info), ioapic, pin);
3024 :
3025 15 : mp_preconfigure_entry(data);
3026 15 : mp_register_handler(virq, data->is_level);
3027 :
3028 30 : local_irq_save(flags);
3029 15 : if (virq < nr_legacy_irqs())
3030 15 : legacy_pic->mask(virq);
3031 15 : local_irq_restore(flags);
3032 :
3033 15 : apic_printk(APIC_VERBOSE, KERN_DEBUG
3034 : "IOAPIC[%d]: Preconfigured routing entry (%d-%d -> IRQ %d Level:%i ActiveLow:%i)\n",
3035 : ioapic, mpc_ioapic_id(ioapic), pin, virq,
3036 : data->is_level, data->active_low);
3037 : return 0;
3038 : }
3039 :
3040 0 : void mp_irqdomain_free(struct irq_domain *domain, unsigned int virq,
3041 : unsigned int nr_irqs)
3042 : {
3043 0 : struct irq_data *irq_data;
3044 0 : struct mp_chip_data *data;
3045 :
3046 0 : BUG_ON(nr_irqs != 1);
3047 0 : irq_data = irq_domain_get_irq_data(domain, virq);
3048 0 : if (irq_data && irq_data->chip_data) {
3049 0 : data = irq_data->chip_data;
3050 0 : __remove_pin_from_irq(data, mp_irqdomain_ioapic_idx(domain),
3051 0 : (int)irq_data->hwirq);
3052 0 : WARN_ON(!list_empty(&data->irq_2_pin));
3053 0 : kfree(irq_data->chip_data);
3054 : }
3055 0 : irq_domain_free_irqs_top(domain, virq, nr_irqs);
3056 0 : }
3057 :
3058 3 : int mp_irqdomain_activate(struct irq_domain *domain,
3059 : struct irq_data *irq_data, bool reserve)
3060 : {
3061 3 : unsigned long flags;
3062 :
3063 3 : raw_spin_lock_irqsave(&ioapic_lock, flags);
3064 3 : ioapic_configure_entry(irq_data);
3065 3 : raw_spin_unlock_irqrestore(&ioapic_lock, flags);
3066 3 : return 0;
3067 : }
3068 :
3069 0 : void mp_irqdomain_deactivate(struct irq_domain *domain,
3070 : struct irq_data *irq_data)
3071 : {
3072 : /* It won't be called for IRQ with multiple IOAPIC pins associated */
3073 0 : ioapic_mask_entry(mp_irqdomain_ioapic_idx(domain),
3074 0 : (int)irq_data->hwirq);
3075 0 : }
3076 :
3077 15 : int mp_irqdomain_ioapic_idx(struct irq_domain *domain)
3078 : {
3079 15 : return (int)(long)domain->host_data;
3080 : }
3081 :
3082 : const struct irq_domain_ops mp_ioapic_irqdomain_ops = {
3083 : .alloc = mp_irqdomain_alloc,
3084 : .free = mp_irqdomain_free,
3085 : .activate = mp_irqdomain_activate,
3086 : .deactivate = mp_irqdomain_deactivate,
3087 : };
|
