Line data    Source code

       1             : // SPDX-License-Identifier: GPL-2.0
       2             : /*
       3             :  *      Copyright (C) 1992, 1998 Linus Torvalds, Ingo Molnar
       4             :  *
       5             :  * This file contains the lowest level x86_64-specific interrupt
       6             :  * entry and irq statistics code. All the remaining irq logic is
       7             :  * done by the generic kernel/irq/ code and in the
       8             :  * x86_64-specific irq controller code. (e.g. i8259.c and
       9             :  * io_apic.c.)
      10             :  */
      11             : 
      12             : #include <linux/kernel_stat.h>
      13             : #include <linux/interrupt.h>
      14             : #include <linux/irq.h>
      15             : #include <linux/seq_file.h>
      16             : #include <linux/delay.h>
      17             : #include <linux/ftrace.h>
      18             : #include <linux/uaccess.h>
      19             : #include <linux/smp.h>
      20             : #include <linux/sched/task_stack.h>
      21             : 
      22             : #include <asm/cpu_entry_area.h>
      23             : #include <asm/softirq_stack.h>
      24             : #include <asm/irq_stack.h>
      25             : #include <asm/io_apic.h>
      26             : #include <asm/apic.h>
      27             : 
      28             : DEFINE_PER_CPU_PAGE_ALIGNED(struct irq_stack, irq_stack_backing_store) __visible;
      29             : DECLARE_INIT_PER_CPU(irq_stack_backing_store);
      30             : 
      31             : #ifdef CONFIG_VMAP_STACK
      32             : /*
      33             :  * VMAP the backing store with guard pages
      34             :  */
      35             : static int map_irq_stack(unsigned int cpu)
      36             : {
      37             :         char *stack = (char *)per_cpu_ptr(&irq_stack_backing_store, cpu);
      38             :         struct page *pages[IRQ_STACK_SIZE / PAGE_SIZE];
      39             :         void *va;
      40             :         int i;
      41             : 
      42             :         for (i = 0; i < IRQ_STACK_SIZE / PAGE_SIZE; i++) {
      43             :                 phys_addr_t pa = per_cpu_ptr_to_phys(stack + (i << PAGE_SHIFT));
      44             : 
      45             :                 pages[i] = pfn_to_page(pa >> PAGE_SHIFT);
      46             :         }
      47             : 
      48             :         va = vmap(pages, IRQ_STACK_SIZE / PAGE_SIZE, VM_MAP, PAGE_KERNEL);
      49             :         if (!va)
      50             :                 return -ENOMEM;
      51             : 
      52             :         /* Store actual TOS to avoid adjustment in the hotpath */
      53             :         per_cpu(hardirq_stack_ptr, cpu) = va + IRQ_STACK_SIZE - 8;
      54             :         return 0;
      55             : }
      56             : #else
      57             : /*
      58             :  * If VMAP stacks are disabled due to KASAN, just use the per cpu
      59             :  * backing store without guard pages.
      60             :  */
      61           4 : static int map_irq_stack(unsigned int cpu)
      62             : {
      63           4 :         void *va = per_cpu_ptr(&irq_stack_backing_store, cpu);
      64             : 
      65             :         /* Store actual TOS to avoid adjustment in the hotpath */
      66           4 :         per_cpu(hardirq_stack_ptr, cpu) = va + IRQ_STACK_SIZE - 8;
      67           4 :         return 0;
      68             : }
      69             : #endif
      70             : 
      71           4 : int irq_init_percpu_irqstack(unsigned int cpu)
      72             : {
      73           4 :         if (per_cpu(hardirq_stack_ptr, cpu))
      74             :                 return 0;
      75           4 :         return map_irq_stack(cpu);
      76             : }