LCOV - code coverage report
Current view: top level - arch/x86/kernel - idt.c (source / functions) Hit Total Coverage
Test: landlock.info Lines: 56 65 86.2 %
Date: 2021-04-22 12:43:58 Functions: 10 12 83.3 %

          Line data    Source code
       1             : // SPDX-License-Identifier: GPL-2.0-only
       2             : /*
       3             :  * Interrupt descriptor table related code
       4             :  */
       5             : #include <linux/interrupt.h>
       6             : 
       7             : #include <asm/cpu_entry_area.h>
       8             : #include <asm/set_memory.h>
       9             : #include <asm/traps.h>
      10             : #include <asm/proto.h>
      11             : #include <asm/desc.h>
      12             : #include <asm/hw_irq.h>
      13             : 
      14             : #define DPL0            0x0
      15             : #define DPL3            0x3
      16             : 
      17             : #define DEFAULT_STACK   0
      18             : 
      19             : #define G(_vector, _addr, _ist, _type, _dpl, _segment)  \
      20             :         {                                               \
      21             :                 .vector         = _vector,              \
      22             :                 .bits.ist       = _ist,                 \
      23             :                 .bits.type      = _type,                \
      24             :                 .bits.dpl       = _dpl,                 \
      25             :                 .bits.p         = 1,                    \
      26             :                 .addr           = _addr,                \
      27             :                 .segment        = _segment,             \
      28             :         }
      29             : 
      30             : /* Interrupt gate */
      31             : #define INTG(_vector, _addr)                            \
      32             :         G(_vector, _addr, DEFAULT_STACK, GATE_INTERRUPT, DPL0, __KERNEL_CS)
      33             : 
      34             : /* System interrupt gate */
      35             : #define SYSG(_vector, _addr)                            \
      36             :         G(_vector, _addr, DEFAULT_STACK, GATE_INTERRUPT, DPL3, __KERNEL_CS)
      37             : 
      38             : /*
      39             :  * Interrupt gate with interrupt stack. The _ist index is the index in
      40             :  * the tss.ist[] array, but for the descriptor it needs to start at 1.
      41             :  */
      42             : #define ISTG(_vector, _addr, _ist)                      \
      43             :         G(_vector, _addr, _ist + 1, GATE_INTERRUPT, DPL0, __KERNEL_CS)
      44             : 
      45             : /* Task gate */
      46             : #define TSKG(_vector, _gdt)                             \
      47             :         G(_vector, NULL, DEFAULT_STACK, GATE_TASK, DPL0, _gdt << 3)
      48             : 
      49             : #define IDT_TABLE_SIZE          (IDT_ENTRIES * sizeof(gate_desc))
      50             : 
      51             : static bool idt_setup_done __initdata;
      52             : 
      53             : /*
      54             :  * Early traps running on the DEFAULT_STACK because the other interrupt
      55             :  * stacks work only after cpu_init().
      56             :  */
      57             : static const __initconst struct idt_data early_idts[] = {
      58             :         INTG(X86_TRAP_DB,               asm_exc_debug),
      59             :         SYSG(X86_TRAP_BP,               asm_exc_int3),
      60             : 
      61             : #ifdef CONFIG_X86_32
      62             :         /*
      63             :          * Not possible on 64-bit. See idt_setup_early_pf() for details.
      64             :          */
      65             :         INTG(X86_TRAP_PF,               asm_exc_page_fault),
      66             : #endif
      67             : };
      68             : 
      69             : /*
      70             :  * The default IDT entries which are set up in trap_init() before
      71             :  * cpu_init() is invoked. Interrupt stacks cannot be used at that point and
      72             :  * the traps which use them are reinitialized with IST after cpu_init() has
      73             :  * set up TSS.
      74             :  */
      75             : static const __initconst struct idt_data def_idts[] = {
      76             :         INTG(X86_TRAP_DE,               asm_exc_divide_error),
      77             :         INTG(X86_TRAP_NMI,              asm_exc_nmi),
      78             :         INTG(X86_TRAP_BR,               asm_exc_bounds),
      79             :         INTG(X86_TRAP_UD,               asm_exc_invalid_op),
      80             :         INTG(X86_TRAP_NM,               asm_exc_device_not_available),
      81             :         INTG(X86_TRAP_OLD_MF,           asm_exc_coproc_segment_overrun),
      82             :         INTG(X86_TRAP_TS,               asm_exc_invalid_tss),
      83             :         INTG(X86_TRAP_NP,               asm_exc_segment_not_present),
      84             :         INTG(X86_TRAP_SS,               asm_exc_stack_segment),
      85             :         INTG(X86_TRAP_GP,               asm_exc_general_protection),
      86             :         INTG(X86_TRAP_SPURIOUS,         asm_exc_spurious_interrupt_bug),
      87             :         INTG(X86_TRAP_MF,               asm_exc_coprocessor_error),
      88             :         INTG(X86_TRAP_AC,               asm_exc_alignment_check),
      89             :         INTG(X86_TRAP_XF,               asm_exc_simd_coprocessor_error),
      90             : 
      91             : #ifdef CONFIG_X86_32
      92             :         TSKG(X86_TRAP_DF,               GDT_ENTRY_DOUBLEFAULT_TSS),
      93             : #else
      94             :         INTG(X86_TRAP_DF,               asm_exc_double_fault),
      95             : #endif
      96             :         INTG(X86_TRAP_DB,               asm_exc_debug),
      97             : 
      98             : #ifdef CONFIG_X86_MCE
      99             :         INTG(X86_TRAP_MC,               asm_exc_machine_check),
     100             : #endif
     101             : 
     102             :         SYSG(X86_TRAP_OF,               asm_exc_overflow),
     103             : #if defined(CONFIG_IA32_EMULATION)
     104             :         SYSG(IA32_SYSCALL_VECTOR,       entry_INT80_compat),
     105             : #elif defined(CONFIG_X86_32)
     106             :         SYSG(IA32_SYSCALL_VECTOR,       entry_INT80_32),
     107             : #endif
     108             : };
     109             : 
     110             : /*
     111             :  * The APIC and SMP idt entries
     112             :  */
     113             : static const __initconst struct idt_data apic_idts[] = {
     114             : #ifdef CONFIG_SMP
     115             :         INTG(RESCHEDULE_VECTOR,                 asm_sysvec_reschedule_ipi),
     116             :         INTG(CALL_FUNCTION_VECTOR,              asm_sysvec_call_function),
     117             :         INTG(CALL_FUNCTION_SINGLE_VECTOR,       asm_sysvec_call_function_single),
     118             :         INTG(IRQ_MOVE_CLEANUP_VECTOR,           asm_sysvec_irq_move_cleanup),
     119             :         INTG(REBOOT_VECTOR,                     asm_sysvec_reboot),
     120             : #endif
     121             : 
     122             : #ifdef CONFIG_X86_THERMAL_VECTOR
     123             :         INTG(THERMAL_APIC_VECTOR,               asm_sysvec_thermal),
     124             : #endif
     125             : 
     126             : #ifdef CONFIG_X86_MCE_THRESHOLD
     127             :         INTG(THRESHOLD_APIC_VECTOR,             asm_sysvec_threshold),
     128             : #endif
     129             : 
     130             : #ifdef CONFIG_X86_MCE_AMD
     131             :         INTG(DEFERRED_ERROR_VECTOR,             asm_sysvec_deferred_error),
     132             : #endif
     133             : 
     134             : #ifdef CONFIG_X86_LOCAL_APIC
     135             :         INTG(LOCAL_TIMER_VECTOR,                asm_sysvec_apic_timer_interrupt),
     136             :         INTG(X86_PLATFORM_IPI_VECTOR,           asm_sysvec_x86_platform_ipi),
     137             : # ifdef CONFIG_HAVE_KVM
     138             :         INTG(POSTED_INTR_VECTOR,                asm_sysvec_kvm_posted_intr_ipi),
     139             :         INTG(POSTED_INTR_WAKEUP_VECTOR,         asm_sysvec_kvm_posted_intr_wakeup_ipi),
     140             :         INTG(POSTED_INTR_NESTED_VECTOR,         asm_sysvec_kvm_posted_intr_nested_ipi),
     141             : # endif
     142             : # ifdef CONFIG_IRQ_WORK
     143             :         INTG(IRQ_WORK_VECTOR,                   asm_sysvec_irq_work),
     144             : # endif
     145             :         INTG(SPURIOUS_APIC_VECTOR,              asm_sysvec_spurious_apic_interrupt),
     146             :         INTG(ERROR_APIC_VECTOR,                 asm_sysvec_error_interrupt),
     147             : #endif
     148             : };
     149             : 
     150             : /* Must be page-aligned because the real IDT is used in the cpu entry area */
     151             : static gate_desc idt_table[IDT_ENTRIES] __page_aligned_bss;
     152             : 
     153             : static struct desc_ptr idt_descr __ro_after_init = {
     154             :         .size           = IDT_TABLE_SIZE - 1,
     155             :         .address        = (unsigned long) idt_table,
     156             : };
     157             : 
     158           7 : void load_current_idt(void)
     159             : {
     160          14 :         lockdep_assert_irqs_disabled();
     161           7 :         load_idt(&idt_descr);
     162           7 : }
     163             : 
     164             : #ifdef CONFIG_X86_F00F_BUG
     165             : bool idt_is_f00f_address(unsigned long address)
     166             : {
     167             :         return ((address - idt_descr.address) >> 3) == 6;
     168             : }
     169             : #endif
     170             : 
     171             : static __init void
     172         247 : idt_setup_from_table(gate_desc *idt, const struct idt_data *t, int size, bool sys)
     173             : {
     174         247 :         gate_desc desc;
     175             : 
     176         526 :         for (; size > 0; t++, size--) {
     177         279 :                 idt_init_desc(&desc, t);
     178         279 :                 write_idt_entry(idt, t->vector, &desc);
     179         279 :                 if (sys)
     180          37 :                         set_bit(t->vector, system_vectors);
     181             :         }
     182         247 : }
     183             : 
     184         242 : static __init void set_intr_gate(unsigned int n, const void *addr)
     185             : {
     186         242 :         struct idt_data data;
     187             : 
     188         242 :         init_idt_data(&data, n, addr);
     189             : 
     190         242 :         idt_setup_from_table(idt_table, &data, 1, false);
     191         242 : }
     192             : 
     193             : /**
     194             :  * idt_setup_early_traps - Initialize the idt table with early traps
     195             :  *
     196             :  * On X8664 these traps do not use interrupt stacks as they can't work
     197             :  * before cpu_init() is invoked and sets up TSS. The IST variants are
     198             :  * installed after that.
     199             :  */
     200           1 : void __init idt_setup_early_traps(void)
     201             : {
     202           1 :         idt_setup_from_table(idt_table, early_idts, ARRAY_SIZE(early_idts),
     203             :                              true);
     204           1 :         load_idt(&idt_descr);
     205           1 : }
     206             : 
     207             : /**
     208             :  * idt_setup_traps - Initialize the idt table with default traps
     209             :  */
     210           1 : void __init idt_setup_traps(void)
     211             : {
     212           1 :         idt_setup_from_table(idt_table, def_idts, ARRAY_SIZE(def_idts), true);
     213           1 : }
     214             : 
     215             : #ifdef CONFIG_X86_64
     216             : /*
     217             :  * Early traps running on the DEFAULT_STACK because the other interrupt
     218             :  * stacks work only after cpu_init().
     219             :  */
     220             : static const __initconst struct idt_data early_pf_idts[] = {
     221             :         INTG(X86_TRAP_PF,               asm_exc_page_fault),
     222             : };
     223             : 
     224             : /*
     225             :  * The exceptions which use Interrupt stacks. They are setup after
     226             :  * cpu_init() when the TSS has been initialized.
     227             :  */
     228             : static const __initconst struct idt_data ist_idts[] = {
     229             :         ISTG(X86_TRAP_DB,       asm_exc_debug,                  IST_INDEX_DB),
     230             :         ISTG(X86_TRAP_NMI,      asm_exc_nmi,                    IST_INDEX_NMI),
     231             :         ISTG(X86_TRAP_DF,       asm_exc_double_fault,           IST_INDEX_DF),
     232             : #ifdef CONFIG_X86_MCE
     233             :         ISTG(X86_TRAP_MC,       asm_exc_machine_check,          IST_INDEX_MCE),
     234             : #endif
     235             : #ifdef CONFIG_AMD_MEM_ENCRYPT
     236             :         ISTG(X86_TRAP_VC,       asm_exc_vmm_communication,      IST_INDEX_VC),
     237             : #endif
     238             : };
     239             : 
     240             : /**
     241             :  * idt_setup_early_pf - Initialize the idt table with early pagefault handler
     242             :  *
     243             :  * On X8664 this does not use interrupt stacks as they can't work before
     244             :  * cpu_init() is invoked and sets up TSS. The IST variant is installed
     245             :  * after that.
     246             :  *
     247             :  * Note, that X86_64 cannot install the real #PF handler in
     248             :  * idt_setup_early_traps() because the memory intialization needs the #PF
     249             :  * handler from the early_idt_handler_array to initialize the early page
     250             :  * tables.
     251             :  */
     252           1 : void __init idt_setup_early_pf(void)
     253             : {
     254           1 :         idt_setup_from_table(idt_table, early_pf_idts,
     255             :                              ARRAY_SIZE(early_pf_idts), true);
     256           1 : }
     257             : 
     258             : /**
     259             :  * idt_setup_ist_traps - Initialize the idt table with traps using IST
     260             :  */
     261           1 : void __init idt_setup_ist_traps(void)
     262             : {
     263           1 :         idt_setup_from_table(idt_table, ist_idts, ARRAY_SIZE(ist_idts), true);
     264           1 : }
     265             : #endif
     266             : 
     267           1 : static void __init idt_map_in_cea(void)
     268             : {
     269             :         /*
     270             :          * Set the IDT descriptor to a fixed read-only location in the cpu
     271             :          * entry area, so that the "sidt" instruction will not leak the
     272             :          * location of the kernel, and to defend the IDT against arbitrary
     273             :          * memory write vulnerabilities.
     274             :          */
     275           1 :         cea_set_pte(CPU_ENTRY_AREA_RO_IDT_VADDR, __pa_symbol(idt_table),
     276           1 :                     PAGE_KERNEL_RO);
     277           1 :         idt_descr.address = CPU_ENTRY_AREA_RO_IDT;
     278           1 : }
     279             : 
     280             : /**
     281             :  * idt_setup_apic_and_irq_gates - Setup APIC/SMP and normal interrupt gates
     282             :  */
     283           1 : void __init idt_setup_apic_and_irq_gates(void)
     284             : {
     285           1 :         int i = FIRST_EXTERNAL_VECTOR;
     286           1 :         void *entry;
     287             : 
     288           1 :         idt_setup_from_table(idt_table, apic_idts, ARRAY_SIZE(apic_idts), true);
     289             : 
     290         203 :         for_each_clear_bit_from(i, system_vectors, FIRST_SYSTEM_VECTOR) {
     291         202 :                 entry = irq_entries_start + 8 * (i - FIRST_EXTERNAL_VECTOR);
     292         202 :                 set_intr_gate(i, entry);
     293             :         }
     294             : 
     295             : #ifdef CONFIG_X86_LOCAL_APIC
     296           9 :         for_each_clear_bit_from(i, system_vectors, NR_VECTORS) {
     297             :                 /*
     298             :                  * Don't set the non assigned system vectors in the
     299             :                  * system_vectors bitmap. Otherwise they show up in
     300             :                  * /proc/interrupts.
     301             :                  */
     302           8 :                 entry = spurious_entries_start + 8 * (i - FIRST_SYSTEM_VECTOR);
     303           8 :                 set_intr_gate(i, entry);
     304             :         }
     305             : #endif
     306             :         /* Map IDT into CPU entry area and reload it. */
     307           1 :         idt_map_in_cea();
     308           1 :         load_idt(&idt_descr);
     309             : 
     310             :         /* Make the IDT table read only */
     311           1 :         set_memory_ro((unsigned long)&idt_table, 1);
     312             : 
     313           1 :         idt_setup_done = true;
     314           1 : }
     315             : 
     316             : /**
     317             :  * idt_setup_early_handler - Initializes the idt table with early handlers
     318             :  */
     319           1 : void __init idt_setup_early_handler(void)
     320             : {
     321           1 :         int i;
     322             : 
     323          33 :         for (i = 0; i < NUM_EXCEPTION_VECTORS; i++)
     324          32 :                 set_intr_gate(i, early_idt_handler_array[i]);
     325             : #ifdef CONFIG_X86_32
     326             :         for ( ; i < NR_VECTORS; i++)
     327             :                 set_intr_gate(i, early_ignore_irq);
     328             : #endif
     329           1 :         load_idt(&idt_descr);
     330           1 : }
     331             : 
     332             : /**
     333             :  * idt_invalidate - Invalidate interrupt descriptor table
     334             :  * @addr:       The virtual address of the 'invalid' IDT
     335             :  */
     336           0 : void idt_invalidate(void *addr)
     337             : {
     338           0 :         struct desc_ptr idt = { .address = (unsigned long) addr, .size = 0 };
     339             : 
     340           0 :         load_idt(&idt);
     341           0 : }
     342             : 
     343           0 : void __init alloc_intr_gate(unsigned int n, const void *addr)
     344             : {
     345           0 :         if (WARN_ON(n < FIRST_SYSTEM_VECTOR))
     346             :                 return;
     347             : 
     348           0 :         if (WARN_ON(idt_setup_done))
     349             :                 return;
     350             : 
     351           0 :         if (!WARN_ON(test_and_set_bit(n, system_vectors)))
     352           0 :                 set_intr_gate(n, addr);
     353             : }

Generated by: LCOV version 1.14