LCOV - code coverage report
Current view: top level - arch/x86/kernel - irq.c (source / functions) Hit Total Coverage
Test: landlock.info Lines: 12 152 7.9 %
Date: 2021-04-22 12:43:58 Functions: 2 15 13.3 %

          Line data    Source code
       1             : // SPDX-License-Identifier: GPL-2.0-only
       2             : /*
       3             :  * Common interrupt code for 32 and 64 bit
       4             :  */
       5             : #include <linux/cpu.h>
       6             : #include <linux/interrupt.h>
       7             : #include <linux/kernel_stat.h>
       8             : #include <linux/of.h>
       9             : #include <linux/seq_file.h>
      10             : #include <linux/smp.h>
      11             : #include <linux/ftrace.h>
      12             : #include <linux/delay.h>
      13             : #include <linux/export.h>
      14             : #include <linux/irq.h>
      15             : 
      16             : #include <asm/irq_stack.h>
      17             : #include <asm/apic.h>
      18             : #include <asm/io_apic.h>
      19             : #include <asm/irq.h>
      20             : #include <asm/mce.h>
      21             : #include <asm/hw_irq.h>
      22             : #include <asm/desc.h>
      23             : #include <asm/traps.h>
      24             : #include <asm/thermal.h>
      25             : 
      26             : #define CREATE_TRACE_POINTS
      27             : #include <asm/trace/irq_vectors.h>
      28             : 
      29             : DEFINE_PER_CPU_SHARED_ALIGNED(irq_cpustat_t, irq_stat);
      30             : EXPORT_PER_CPU_SYMBOL(irq_stat);
      31             : 
      32             : atomic_t irq_err_count;
      33             : 
      34             : /*
      35             :  * 'what should we do if we get a hw irq event on an illegal vector'.
      36             :  * each architecture has to answer this themselves.
      37             :  */
      38           0 : void ack_bad_irq(unsigned int irq)
      39             : {
      40           0 :         if (printk_ratelimit())
      41           0 :                 pr_err("unexpected IRQ trap at vector %02x\n", irq);
      42             : 
      43             :         /*
      44             :          * Currently unexpected vectors happen only on SMP and APIC.
      45             :          * We _must_ ack these because every local APIC has only N
      46             :          * irq slots per priority level, and a 'hanging, unacked' IRQ
      47             :          * holds up an irq slot - in excessive cases (when multiple
      48             :          * unexpected vectors occur) that might lock up the APIC
      49             :          * completely.
      50             :          * But only ack when the APIC is enabled -AK
      51             :          */
      52           0 :         ack_APIC_irq();
      53           0 : }
      54             : 
      55             : #define irq_stats(x)            (&per_cpu(irq_stat, x))
      56             : /*
      57             :  * /proc/interrupts printing for arch specific interrupts
      58             :  */
      59           0 : int arch_show_interrupts(struct seq_file *p, int prec)
      60             : {
      61           0 :         int j;
      62             : 
      63           0 :         seq_printf(p, "%*s: ", prec, "NMI");
      64           0 :         for_each_online_cpu(j)
      65           0 :                 seq_printf(p, "%10u ", irq_stats(j)->__nmi_count);
      66           0 :         seq_puts(p, "  Non-maskable interrupts\n");
      67             : #ifdef CONFIG_X86_LOCAL_APIC
      68           0 :         seq_printf(p, "%*s: ", prec, "LOC");
      69           0 :         for_each_online_cpu(j)
      70           0 :                 seq_printf(p, "%10u ", irq_stats(j)->apic_timer_irqs);
      71           0 :         seq_puts(p, "  Local timer interrupts\n");
      72             : 
      73           0 :         seq_printf(p, "%*s: ", prec, "SPU");
      74           0 :         for_each_online_cpu(j)
      75           0 :                 seq_printf(p, "%10u ", irq_stats(j)->irq_spurious_count);
      76           0 :         seq_puts(p, "  Spurious interrupts\n");
      77           0 :         seq_printf(p, "%*s: ", prec, "PMI");
      78           0 :         for_each_online_cpu(j)
      79           0 :                 seq_printf(p, "%10u ", irq_stats(j)->apic_perf_irqs);
      80           0 :         seq_puts(p, "  Performance monitoring interrupts\n");
      81           0 :         seq_printf(p, "%*s: ", prec, "IWI");
      82           0 :         for_each_online_cpu(j)
      83           0 :                 seq_printf(p, "%10u ", irq_stats(j)->apic_irq_work_irqs);
      84           0 :         seq_puts(p, "  IRQ work interrupts\n");
      85           0 :         seq_printf(p, "%*s: ", prec, "RTR");
      86           0 :         for_each_online_cpu(j)
      87           0 :                 seq_printf(p, "%10u ", irq_stats(j)->icr_read_retry_count);
      88           0 :         seq_puts(p, "  APIC ICR read retries\n");
      89           0 :         if (x86_platform_ipi_callback) {
      90           0 :                 seq_printf(p, "%*s: ", prec, "PLT");
      91           0 :                 for_each_online_cpu(j)
      92           0 :                         seq_printf(p, "%10u ", irq_stats(j)->x86_platform_ipis);
      93           0 :                 seq_puts(p, "  Platform interrupts\n");
      94             :         }
      95             : #endif
      96             : #ifdef CONFIG_SMP
      97           0 :         seq_printf(p, "%*s: ", prec, "RES");
      98           0 :         for_each_online_cpu(j)
      99           0 :                 seq_printf(p, "%10u ", irq_stats(j)->irq_resched_count);
     100           0 :         seq_puts(p, "  Rescheduling interrupts\n");
     101           0 :         seq_printf(p, "%*s: ", prec, "CAL");
     102           0 :         for_each_online_cpu(j)
     103           0 :                 seq_printf(p, "%10u ", irq_stats(j)->irq_call_count);
     104           0 :         seq_puts(p, "  Function call interrupts\n");
     105           0 :         seq_printf(p, "%*s: ", prec, "TLB");
     106           0 :         for_each_online_cpu(j)
     107           0 :                 seq_printf(p, "%10u ", irq_stats(j)->irq_tlb_count);
     108           0 :         seq_puts(p, "  TLB shootdowns\n");
     109             : #endif
     110             : #ifdef CONFIG_X86_THERMAL_VECTOR
     111             :         seq_printf(p, "%*s: ", prec, "TRM");
     112             :         for_each_online_cpu(j)
     113             :                 seq_printf(p, "%10u ", irq_stats(j)->irq_thermal_count);
     114             :         seq_puts(p, "  Thermal event interrupts\n");
     115             : #endif
     116             : #ifdef CONFIG_X86_MCE_THRESHOLD
     117             :         seq_printf(p, "%*s: ", prec, "THR");
     118             :         for_each_online_cpu(j)
     119             :                 seq_printf(p, "%10u ", irq_stats(j)->irq_threshold_count);
     120             :         seq_puts(p, "  Threshold APIC interrupts\n");
     121             : #endif
     122             : #ifdef CONFIG_X86_MCE_AMD
     123             :         seq_printf(p, "%*s: ", prec, "DFR");
     124             :         for_each_online_cpu(j)
     125             :                 seq_printf(p, "%10u ", irq_stats(j)->irq_deferred_error_count);
     126             :         seq_puts(p, "  Deferred Error APIC interrupts\n");
     127             : #endif
     128             : #ifdef CONFIG_X86_MCE
     129             :         seq_printf(p, "%*s: ", prec, "MCE");
     130             :         for_each_online_cpu(j)
     131             :                 seq_printf(p, "%10u ", per_cpu(mce_exception_count, j));
     132             :         seq_puts(p, "  Machine check exceptions\n");
     133             :         seq_printf(p, "%*s: ", prec, "MCP");
     134             :         for_each_online_cpu(j)
     135             :                 seq_printf(p, "%10u ", per_cpu(mce_poll_count, j));
     136             :         seq_puts(p, "  Machine check polls\n");
     137             : #endif
     138             : #ifdef CONFIG_X86_HV_CALLBACK_VECTOR
     139           0 :         if (test_bit(HYPERVISOR_CALLBACK_VECTOR, system_vectors)) {
     140           0 :                 seq_printf(p, "%*s: ", prec, "HYP");
     141           0 :                 for_each_online_cpu(j)
     142           0 :                         seq_printf(p, "%10u ",
     143           0 :                                    irq_stats(j)->irq_hv_callback_count);
     144           0 :                 seq_puts(p, "  Hypervisor callback interrupts\n");
     145             :         }
     146             : #endif
     147             : #if IS_ENABLED(CONFIG_HYPERV)
     148             :         if (test_bit(HYPERV_REENLIGHTENMENT_VECTOR, system_vectors)) {
     149             :                 seq_printf(p, "%*s: ", prec, "HRE");
     150             :                 for_each_online_cpu(j)
     151             :                         seq_printf(p, "%10u ",
     152             :                                    irq_stats(j)->irq_hv_reenlightenment_count);
     153             :                 seq_puts(p, "  Hyper-V reenlightenment interrupts\n");
     154             :         }
     155             :         if (test_bit(HYPERV_STIMER0_VECTOR, system_vectors)) {
     156             :                 seq_printf(p, "%*s: ", prec, "HVS");
     157             :                 for_each_online_cpu(j)
     158             :                         seq_printf(p, "%10u ",
     159             :                                    irq_stats(j)->hyperv_stimer0_count);
     160             :                 seq_puts(p, "  Hyper-V stimer0 interrupts\n");
     161             :         }
     162             : #endif
     163           0 :         seq_printf(p, "%*s: %10u\n", prec, "ERR", atomic_read(&irq_err_count));
     164             : #if defined(CONFIG_X86_IO_APIC)
     165           0 :         seq_printf(p, "%*s: %10u\n", prec, "MIS", atomic_read(&irq_mis_count));
     166             : #endif
     167             : #ifdef CONFIG_HAVE_KVM
     168           0 :         seq_printf(p, "%*s: ", prec, "PIN");
     169           0 :         for_each_online_cpu(j)
     170           0 :                 seq_printf(p, "%10u ", irq_stats(j)->kvm_posted_intr_ipis);
     171           0 :         seq_puts(p, "  Posted-interrupt notification event\n");
     172             : 
     173           0 :         seq_printf(p, "%*s: ", prec, "NPI");
     174           0 :         for_each_online_cpu(j)
     175           0 :                 seq_printf(p, "%10u ",
     176           0 :                            irq_stats(j)->kvm_posted_intr_nested_ipis);
     177           0 :         seq_puts(p, "  Nested posted-interrupt event\n");
     178             : 
     179           0 :         seq_printf(p, "%*s: ", prec, "PIW");
     180           0 :         for_each_online_cpu(j)
     181           0 :                 seq_printf(p, "%10u ",
     182           0 :                            irq_stats(j)->kvm_posted_intr_wakeup_ipis);
     183           0 :         seq_puts(p, "  Posted-interrupt wakeup event\n");
     184             : #endif
     185           0 :         return 0;
     186             : }
     187             : 
     188             : /*
     189             :  * /proc/stat helpers
     190             :  */
     191           0 : u64 arch_irq_stat_cpu(unsigned int cpu)
     192             : {
     193           0 :         u64 sum = irq_stats(cpu)->__nmi_count;
     194             : 
     195             : #ifdef CONFIG_X86_LOCAL_APIC
     196           0 :         sum += irq_stats(cpu)->apic_timer_irqs;
     197           0 :         sum += irq_stats(cpu)->irq_spurious_count;
     198           0 :         sum += irq_stats(cpu)->apic_perf_irqs;
     199           0 :         sum += irq_stats(cpu)->apic_irq_work_irqs;
     200           0 :         sum += irq_stats(cpu)->icr_read_retry_count;
     201           0 :         if (x86_platform_ipi_callback)
     202           0 :                 sum += irq_stats(cpu)->x86_platform_ipis;
     203             : #endif
     204             : #ifdef CONFIG_SMP
     205           0 :         sum += irq_stats(cpu)->irq_resched_count;
     206           0 :         sum += irq_stats(cpu)->irq_call_count;
     207             : #endif
     208             : #ifdef CONFIG_X86_THERMAL_VECTOR
     209             :         sum += irq_stats(cpu)->irq_thermal_count;
     210             : #endif
     211             : #ifdef CONFIG_X86_MCE_THRESHOLD
     212             :         sum += irq_stats(cpu)->irq_threshold_count;
     213             : #endif
     214             : #ifdef CONFIG_X86_MCE
     215             :         sum += per_cpu(mce_exception_count, cpu);
     216             :         sum += per_cpu(mce_poll_count, cpu);
     217             : #endif
     218           0 :         return sum;
     219             : }
     220             : 
     221           0 : u64 arch_irq_stat(void)
     222             : {
     223           0 :         u64 sum = atomic_read(&irq_err_count);
     224           0 :         return sum;
     225             : }
     226             : 
     227        3719 : static __always_inline void handle_irq(struct irq_desc *desc,
     228             :                                        struct pt_regs *regs)
     229             : {
     230        3719 :         if (IS_ENABLED(CONFIG_X86_64))
     231        7438 :                 generic_handle_irq_desc(desc);
     232             :         else
     233             :                 __handle_irq(desc, regs);
     234             : }
     235             : 
     236             : /*
     237             :  * common_interrupt() handles all normal device IRQ's (the special SMP
     238             :  * cross-CPU interrupts have their own entry points).
     239             :  */
     240        7438 : DEFINE_IDTENTRY_IRQ(common_interrupt)
     241             : {
     242        3719 :         struct pt_regs *old_regs = set_irq_regs(regs);
     243        3719 :         struct irq_desc *desc;
     244             : 
     245             :         /* entry code tells RCU that we're not quiescent.  Check it. */
     246        3719 :         RCU_LOCKDEP_WARN(!rcu_is_watching(), "IRQ failed to wake up RCU");
     247             : 
     248        3719 :         desc = __this_cpu_read(vector_irq[vector]);
     249        7438 :         if (likely(!IS_ERR_OR_NULL(desc))) {
     250        3719 :                 handle_irq(desc, regs);
     251             :         } else {
     252           0 :                 ack_APIC_irq();
     253             : 
     254           0 :                 if (desc == VECTOR_UNUSED) {
     255           0 :                         pr_emerg_ratelimited("%s: %d.%u No irq handler for vector\n",
     256             :                                              __func__, smp_processor_id(),
     257             :                                              vector);
     258             :                 } else {
     259           0 :                         __this_cpu_write(vector_irq[vector], VECTOR_UNUSED);
     260             :                 }
     261             :         }
     262             : 
     263        3719 :         set_irq_regs(old_regs);
     264        3719 : }
     265             : 
     266             : #ifdef CONFIG_X86_LOCAL_APIC
     267             : /* Function pointer for generic interrupt vector handling */
     268             : void (*x86_platform_ipi_callback)(void) = NULL;
     269             : /*
     270             :  * Handler for X86_PLATFORM_IPI_VECTOR.
     271             :  */
     272           0 : DEFINE_IDTENTRY_SYSVEC(sysvec_x86_platform_ipi)
     273             : {
     274           0 :         struct pt_regs *old_regs = set_irq_regs(regs);
     275             : 
     276           0 :         ack_APIC_irq();
     277           0 :         trace_x86_platform_ipi_entry(X86_PLATFORM_IPI_VECTOR);
     278           0 :         inc_irq_stat(x86_platform_ipis);
     279           0 :         if (x86_platform_ipi_callback)
     280           0 :                 x86_platform_ipi_callback();
     281           0 :         trace_x86_platform_ipi_exit(X86_PLATFORM_IPI_VECTOR);
     282           0 :         set_irq_regs(old_regs);
     283           0 : }
     284             : #endif
     285             : 
     286             : #ifdef CONFIG_HAVE_KVM
     287           0 : static void dummy_handler(void) {}
     288             : static void (*kvm_posted_intr_wakeup_handler)(void) = dummy_handler;
     289             : 
     290           0 : void kvm_set_posted_intr_wakeup_handler(void (*handler)(void))
     291             : {
     292           0 :         if (handler)
     293           0 :                 kvm_posted_intr_wakeup_handler = handler;
     294             :         else
     295           0 :                 kvm_posted_intr_wakeup_handler = dummy_handler;
     296           0 : }
     297             : EXPORT_SYMBOL_GPL(kvm_set_posted_intr_wakeup_handler);
     298             : 
     299             : /*
     300             :  * Handler for POSTED_INTERRUPT_VECTOR.
     301             :  */
     302           0 : DEFINE_IDTENTRY_SYSVEC_SIMPLE(sysvec_kvm_posted_intr_ipi)
     303             : {
     304           0 :         ack_APIC_irq();
     305           0 :         inc_irq_stat(kvm_posted_intr_ipis);
     306             : }
     307             : 
     308             : /*
     309             :  * Handler for POSTED_INTERRUPT_WAKEUP_VECTOR.
     310             :  */
     311           0 : DEFINE_IDTENTRY_SYSVEC(sysvec_kvm_posted_intr_wakeup_ipi)
     312             : {
     313           0 :         ack_APIC_irq();
     314           0 :         inc_irq_stat(kvm_posted_intr_wakeup_ipis);
     315           0 :         kvm_posted_intr_wakeup_handler();
     316           0 : }
     317             : 
     318             : /*
     319             :  * Handler for POSTED_INTERRUPT_NESTED_VECTOR.
     320             :  */
     321           0 : DEFINE_IDTENTRY_SYSVEC_SIMPLE(sysvec_kvm_posted_intr_nested_ipi)
     322             : {
     323           0 :         ack_APIC_irq();
     324           0 :         inc_irq_stat(kvm_posted_intr_nested_ipis);
     325             : }
     326             : #endif
     327             : 
     328             : 
     329             : #ifdef CONFIG_HOTPLUG_CPU
     330             : /* A cpu has been removed from cpu_online_mask.  Reset irq affinities. */
     331           0 : void fixup_irqs(void)
     332             : {
     333           0 :         unsigned int irr, vector;
     334           0 :         struct irq_desc *desc;
     335           0 :         struct irq_data *data;
     336           0 :         struct irq_chip *chip;
     337             : 
     338           0 :         irq_migrate_all_off_this_cpu();
     339             : 
     340             :         /*
     341             :          * We can remove mdelay() and then send spuriuous interrupts to
     342             :          * new cpu targets for all the irqs that were handled previously by
     343             :          * this cpu. While it works, I have seen spurious interrupt messages
     344             :          * (nothing wrong but still...).
     345             :          *
     346             :          * So for now, retain mdelay(1) and check the IRR and then send those
     347             :          * interrupts to new targets as this cpu is already offlined...
     348             :          */
     349           0 :         mdelay(1);
     350             : 
     351             :         /*
     352             :          * We can walk the vector array of this cpu without holding
     353             :          * vector_lock because the cpu is already marked !online, so
     354             :          * nothing else will touch it.
     355             :          */
     356           0 :         for (vector = FIRST_EXTERNAL_VECTOR; vector < NR_VECTORS; vector++) {
     357           0 :                 if (IS_ERR_OR_NULL(__this_cpu_read(vector_irq[vector])))
     358           0 :                         continue;
     359             : 
     360           0 :                 irr = apic_read(APIC_IRR + (vector / 32 * 0x10));
     361           0 :                 if (irr  & (1 << (vector % 32))) {
     362           0 :                         desc = __this_cpu_read(vector_irq[vector]);
     363             : 
     364           0 :                         raw_spin_lock(&desc->lock);
     365           0 :                         data = irq_desc_get_irq_data(desc);
     366           0 :                         chip = irq_data_get_irq_chip(data);
     367           0 :                         if (chip->irq_retrigger) {
     368           0 :                                 chip->irq_retrigger(data);
     369           0 :                                 __this_cpu_write(vector_irq[vector], VECTOR_RETRIGGERED);
     370             :                         }
     371           0 :                         raw_spin_unlock(&desc->lock);
     372             :                 }
     373           0 :                 if (__this_cpu_read(vector_irq[vector]) != VECTOR_RETRIGGERED)
     374           0 :                         __this_cpu_write(vector_irq[vector], VECTOR_UNUSED);
     375             :         }
     376           0 : }
     377             : #endif
     378             : 
     379             : #ifdef CONFIG_X86_THERMAL_VECTOR
     380             : static void smp_thermal_vector(void)
     381             : {
     382             :         if (x86_thermal_enabled())
     383             :                 intel_thermal_interrupt();
     384             :         else
     385             :                 pr_err("CPU%d: Unexpected LVT thermal interrupt!\n",
     386             :                        smp_processor_id());
     387             : }
     388             : 
     389             : DEFINE_IDTENTRY_SYSVEC(sysvec_thermal)
     390             : {
     391             :         trace_thermal_apic_entry(THERMAL_APIC_VECTOR);
     392             :         inc_irq_stat(irq_thermal_count);
     393             :         smp_thermal_vector();
     394             :         trace_thermal_apic_exit(THERMAL_APIC_VECTOR);
     395             :         ack_APIC_irq();
     396             : }
     397             : #endif

Generated by: LCOV version 1.14