Line data Source code
1 : // SPDX-License-Identifier: GPL-2.0-only
2 : /*
3 : * Interrupt descriptor table related code
4 : */
5 : #include <linux/interrupt.h>
6 :
7 : #include <asm/cpu_entry_area.h>
8 : #include <asm/set_memory.h>
9 : #include <asm/traps.h>
10 : #include <asm/proto.h>
11 : #include <asm/desc.h>
12 : #include <asm/hw_irq.h>
13 :
14 : #define DPL0 0x0
15 : #define DPL3 0x3
16 :
17 : #define DEFAULT_STACK 0
18 :
19 : #define G(_vector, _addr, _ist, _type, _dpl, _segment) \
20 : { \
21 : .vector = _vector, \
22 : .bits.ist = _ist, \
23 : .bits.type = _type, \
24 : .bits.dpl = _dpl, \
25 : .bits.p = 1, \
26 : .addr = _addr, \
27 : .segment = _segment, \
28 : }
29 :
30 : /* Interrupt gate */
31 : #define INTG(_vector, _addr) \
32 : G(_vector, _addr, DEFAULT_STACK, GATE_INTERRUPT, DPL0, __KERNEL_CS)
33 :
34 : /* System interrupt gate */
35 : #define SYSG(_vector, _addr) \
36 : G(_vector, _addr, DEFAULT_STACK, GATE_INTERRUPT, DPL3, __KERNEL_CS)
37 :
38 : /*
39 : * Interrupt gate with interrupt stack. The _ist index is the index in
40 : * the tss.ist[] array, but for the descriptor it needs to start at 1.
41 : */
42 : #define ISTG(_vector, _addr, _ist) \
43 : G(_vector, _addr, _ist + 1, GATE_INTERRUPT, DPL0, __KERNEL_CS)
44 :
45 : /* Task gate */
46 : #define TSKG(_vector, _gdt) \
47 : G(_vector, NULL, DEFAULT_STACK, GATE_TASK, DPL0, _gdt << 3)
48 :
49 : #define IDT_TABLE_SIZE (IDT_ENTRIES * sizeof(gate_desc))
50 :
51 : static bool idt_setup_done __initdata;
52 :
53 : /*
54 : * Early traps running on the DEFAULT_STACK because the other interrupt
55 : * stacks work only after cpu_init().
56 : */
57 : static const __initconst struct idt_data early_idts[] = {
58 : INTG(X86_TRAP_DB, asm_exc_debug),
59 : SYSG(X86_TRAP_BP, asm_exc_int3),
60 :
61 : #ifdef CONFIG_X86_32
62 : /*
63 : * Not possible on 64-bit. See idt_setup_early_pf() for details.
64 : */
65 : INTG(X86_TRAP_PF, asm_exc_page_fault),
66 : #endif
67 : };
68 :
69 : /*
70 : * The default IDT entries which are set up in trap_init() before
71 : * cpu_init() is invoked. Interrupt stacks cannot be used at that point and
72 : * the traps which use them are reinitialized with IST after cpu_init() has
73 : * set up TSS.
74 : */
75 : static const __initconst struct idt_data def_idts[] = {
76 : INTG(X86_TRAP_DE, asm_exc_divide_error),
77 : INTG(X86_TRAP_NMI, asm_exc_nmi),
78 : INTG(X86_TRAP_BR, asm_exc_bounds),
79 : INTG(X86_TRAP_UD, asm_exc_invalid_op),
80 : INTG(X86_TRAP_NM, asm_exc_device_not_available),
81 : INTG(X86_TRAP_OLD_MF, asm_exc_coproc_segment_overrun),
82 : INTG(X86_TRAP_TS, asm_exc_invalid_tss),
83 : INTG(X86_TRAP_NP, asm_exc_segment_not_present),
84 : INTG(X86_TRAP_SS, asm_exc_stack_segment),
85 : INTG(X86_TRAP_GP, asm_exc_general_protection),
86 : INTG(X86_TRAP_SPURIOUS, asm_exc_spurious_interrupt_bug),
87 : INTG(X86_TRAP_MF, asm_exc_coprocessor_error),
88 : INTG(X86_TRAP_AC, asm_exc_alignment_check),
89 : INTG(X86_TRAP_XF, asm_exc_simd_coprocessor_error),
90 :
91 : #ifdef CONFIG_X86_32
92 : TSKG(X86_TRAP_DF, GDT_ENTRY_DOUBLEFAULT_TSS),
93 : #else
94 : INTG(X86_TRAP_DF, asm_exc_double_fault),
95 : #endif
96 : INTG(X86_TRAP_DB, asm_exc_debug),
97 :
98 : #ifdef CONFIG_X86_MCE
99 : INTG(X86_TRAP_MC, asm_exc_machine_check),
100 : #endif
101 :
102 : SYSG(X86_TRAP_OF, asm_exc_overflow),
103 : #if defined(CONFIG_IA32_EMULATION)
104 : SYSG(IA32_SYSCALL_VECTOR, entry_INT80_compat),
105 : #elif defined(CONFIG_X86_32)
106 : SYSG(IA32_SYSCALL_VECTOR, entry_INT80_32),
107 : #endif
108 : };
109 :
110 : /*
111 : * The APIC and SMP idt entries
112 : */
113 : static const __initconst struct idt_data apic_idts[] = {
114 : #ifdef CONFIG_SMP
115 : INTG(RESCHEDULE_VECTOR, asm_sysvec_reschedule_ipi),
116 : INTG(CALL_FUNCTION_VECTOR, asm_sysvec_call_function),
117 : INTG(CALL_FUNCTION_SINGLE_VECTOR, asm_sysvec_call_function_single),
118 : INTG(IRQ_MOVE_CLEANUP_VECTOR, asm_sysvec_irq_move_cleanup),
119 : INTG(REBOOT_VECTOR, asm_sysvec_reboot),
120 : #endif
121 :
122 : #ifdef CONFIG_X86_THERMAL_VECTOR
123 : INTG(THERMAL_APIC_VECTOR, asm_sysvec_thermal),
124 : #endif
125 :
126 : #ifdef CONFIG_X86_MCE_THRESHOLD
127 : INTG(THRESHOLD_APIC_VECTOR, asm_sysvec_threshold),
128 : #endif
129 :
130 : #ifdef CONFIG_X86_MCE_AMD
131 : INTG(DEFERRED_ERROR_VECTOR, asm_sysvec_deferred_error),
132 : #endif
133 :
134 : #ifdef CONFIG_X86_LOCAL_APIC
135 : INTG(LOCAL_TIMER_VECTOR, asm_sysvec_apic_timer_interrupt),
136 : INTG(X86_PLATFORM_IPI_VECTOR, asm_sysvec_x86_platform_ipi),
137 : # ifdef CONFIG_HAVE_KVM
138 : INTG(POSTED_INTR_VECTOR, asm_sysvec_kvm_posted_intr_ipi),
139 : INTG(POSTED_INTR_WAKEUP_VECTOR, asm_sysvec_kvm_posted_intr_wakeup_ipi),
140 : INTG(POSTED_INTR_NESTED_VECTOR, asm_sysvec_kvm_posted_intr_nested_ipi),
141 : # endif
142 : # ifdef CONFIG_IRQ_WORK
143 : INTG(IRQ_WORK_VECTOR, asm_sysvec_irq_work),
144 : # endif
145 : INTG(SPURIOUS_APIC_VECTOR, asm_sysvec_spurious_apic_interrupt),
146 : INTG(ERROR_APIC_VECTOR, asm_sysvec_error_interrupt),
147 : #endif
148 : };
149 :
150 : /* Must be page-aligned because the real IDT is used in the cpu entry area */
151 : static gate_desc idt_table[IDT_ENTRIES] __page_aligned_bss;
152 :
153 : static struct desc_ptr idt_descr __ro_after_init = {
154 : .size = IDT_TABLE_SIZE - 1,
155 : .address = (unsigned long) idt_table,
156 : };
157 :
158 7 : void load_current_idt(void)
159 : {
160 14 : lockdep_assert_irqs_disabled();
161 7 : load_idt(&idt_descr);
162 7 : }
163 :
164 : #ifdef CONFIG_X86_F00F_BUG
165 : bool idt_is_f00f_address(unsigned long address)
166 : {
167 : return ((address - idt_descr.address) >> 3) == 6;
168 : }
169 : #endif
170 :
171 : static __init void
172 247 : idt_setup_from_table(gate_desc *idt, const struct idt_data *t, int size, bool sys)
173 : {
174 247 : gate_desc desc;
175 :
176 526 : for (; size > 0; t++, size--) {
177 279 : idt_init_desc(&desc, t);
178 279 : write_idt_entry(idt, t->vector, &desc);
179 279 : if (sys)
180 37 : set_bit(t->vector, system_vectors);
181 : }
182 247 : }
183 :
184 242 : static __init void set_intr_gate(unsigned int n, const void *addr)
185 : {
186 242 : struct idt_data data;
187 :
188 242 : init_idt_data(&data, n, addr);
189 :
190 242 : idt_setup_from_table(idt_table, &data, 1, false);
191 242 : }
192 :
193 : /**
194 : * idt_setup_early_traps - Initialize the idt table with early traps
195 : *
196 : * On X8664 these traps do not use interrupt stacks as they can't work
197 : * before cpu_init() is invoked and sets up TSS. The IST variants are
198 : * installed after that.
199 : */
200 1 : void __init idt_setup_early_traps(void)
201 : {
202 1 : idt_setup_from_table(idt_table, early_idts, ARRAY_SIZE(early_idts),
203 : true);
204 1 : load_idt(&idt_descr);
205 1 : }
206 :
207 : /**
208 : * idt_setup_traps - Initialize the idt table with default traps
209 : */
210 1 : void __init idt_setup_traps(void)
211 : {
212 1 : idt_setup_from_table(idt_table, def_idts, ARRAY_SIZE(def_idts), true);
213 1 : }
214 :
215 : #ifdef CONFIG_X86_64
216 : /*
217 : * Early traps running on the DEFAULT_STACK because the other interrupt
218 : * stacks work only after cpu_init().
219 : */
220 : static const __initconst struct idt_data early_pf_idts[] = {
221 : INTG(X86_TRAP_PF, asm_exc_page_fault),
222 : };
223 :
224 : /*
225 : * The exceptions which use Interrupt stacks. They are setup after
226 : * cpu_init() when the TSS has been initialized.
227 : */
228 : static const __initconst struct idt_data ist_idts[] = {
229 : ISTG(X86_TRAP_DB, asm_exc_debug, IST_INDEX_DB),
230 : ISTG(X86_TRAP_NMI, asm_exc_nmi, IST_INDEX_NMI),
231 : ISTG(X86_TRAP_DF, asm_exc_double_fault, IST_INDEX_DF),
232 : #ifdef CONFIG_X86_MCE
233 : ISTG(X86_TRAP_MC, asm_exc_machine_check, IST_INDEX_MCE),
234 : #endif
235 : #ifdef CONFIG_AMD_MEM_ENCRYPT
236 : ISTG(X86_TRAP_VC, asm_exc_vmm_communication, IST_INDEX_VC),
237 : #endif
238 : };
239 :
240 : /**
241 : * idt_setup_early_pf - Initialize the idt table with early pagefault handler
242 : *
243 : * On X8664 this does not use interrupt stacks as they can't work before
244 : * cpu_init() is invoked and sets up TSS. The IST variant is installed
245 : * after that.
246 : *
247 : * Note, that X86_64 cannot install the real #PF handler in
248 : * idt_setup_early_traps() because the memory intialization needs the #PF
249 : * handler from the early_idt_handler_array to initialize the early page
250 : * tables.
251 : */
252 1 : void __init idt_setup_early_pf(void)
253 : {
254 1 : idt_setup_from_table(idt_table, early_pf_idts,
255 : ARRAY_SIZE(early_pf_idts), true);
256 1 : }
257 :
258 : /**
259 : * idt_setup_ist_traps - Initialize the idt table with traps using IST
260 : */
261 1 : void __init idt_setup_ist_traps(void)
262 : {
263 1 : idt_setup_from_table(idt_table, ist_idts, ARRAY_SIZE(ist_idts), true);
264 1 : }
265 : #endif
266 :
267 1 : static void __init idt_map_in_cea(void)
268 : {
269 : /*
270 : * Set the IDT descriptor to a fixed read-only location in the cpu
271 : * entry area, so that the "sidt" instruction will not leak the
272 : * location of the kernel, and to defend the IDT against arbitrary
273 : * memory write vulnerabilities.
274 : */
275 1 : cea_set_pte(CPU_ENTRY_AREA_RO_IDT_VADDR, __pa_symbol(idt_table),
276 1 : PAGE_KERNEL_RO);
277 1 : idt_descr.address = CPU_ENTRY_AREA_RO_IDT;
278 1 : }
279 :
280 : /**
281 : * idt_setup_apic_and_irq_gates - Setup APIC/SMP and normal interrupt gates
282 : */
283 1 : void __init idt_setup_apic_and_irq_gates(void)
284 : {
285 1 : int i = FIRST_EXTERNAL_VECTOR;
286 1 : void *entry;
287 :
288 1 : idt_setup_from_table(idt_table, apic_idts, ARRAY_SIZE(apic_idts), true);
289 :
290 203 : for_each_clear_bit_from(i, system_vectors, FIRST_SYSTEM_VECTOR) {
291 202 : entry = irq_entries_start + 8 * (i - FIRST_EXTERNAL_VECTOR);
292 202 : set_intr_gate(i, entry);
293 : }
294 :
295 : #ifdef CONFIG_X86_LOCAL_APIC
296 9 : for_each_clear_bit_from(i, system_vectors, NR_VECTORS) {
297 : /*
298 : * Don't set the non assigned system vectors in the
299 : * system_vectors bitmap. Otherwise they show up in
300 : * /proc/interrupts.
301 : */
302 8 : entry = spurious_entries_start + 8 * (i - FIRST_SYSTEM_VECTOR);
303 8 : set_intr_gate(i, entry);
304 : }
305 : #endif
306 : /* Map IDT into CPU entry area and reload it. */
307 1 : idt_map_in_cea();
308 1 : load_idt(&idt_descr);
309 :
310 : /* Make the IDT table read only */
311 1 : set_memory_ro((unsigned long)&idt_table, 1);
312 :
313 1 : idt_setup_done = true;
314 1 : }
315 :
316 : /**
317 : * idt_setup_early_handler - Initializes the idt table with early handlers
318 : */
319 1 : void __init idt_setup_early_handler(void)
320 : {
321 1 : int i;
322 :
323 33 : for (i = 0; i < NUM_EXCEPTION_VECTORS; i++)
324 32 : set_intr_gate(i, early_idt_handler_array[i]);
325 : #ifdef CONFIG_X86_32
326 : for ( ; i < NR_VECTORS; i++)
327 : set_intr_gate(i, early_ignore_irq);
328 : #endif
329 1 : load_idt(&idt_descr);
330 1 : }
331 :
332 : /**
333 : * idt_invalidate - Invalidate interrupt descriptor table
334 : * @addr: The virtual address of the 'invalid' IDT
335 : */
336 0 : void idt_invalidate(void *addr)
337 : {
338 0 : struct desc_ptr idt = { .address = (unsigned long) addr, .size = 0 };
339 :
340 0 : load_idt(&idt);
341 0 : }
342 :
343 0 : void __init alloc_intr_gate(unsigned int n, const void *addr)
344 : {
345 0 : if (WARN_ON(n < FIRST_SYSTEM_VECTOR))
346 : return;
347 :
348 0 : if (WARN_ON(idt_setup_done))
349 : return;
350 :
351 0 : if (!WARN_ON(test_and_set_bit(n, system_vectors)))
352 0 : set_intr_gate(n, addr);
353 : }
|
