Line data    Source code

       1             : // SPDX-License-Identifier: GPL-2.0
       2             : /*
       3             :  * local apic based NMI watchdog for various CPUs.
       4             :  *
       5             :  * This file also handles reservation of performance counters for coordination
       6             :  * with other users.
       7             :  *
       8             :  * Note that these events normally don't tick when the CPU idles. This means
       9             :  * the frequency varies with CPU load.
      10             :  *
      11             :  * Original code for K7/P6 written by Keith Owens
      12             :  *
      13             :  */
      14             : 
      15             : #include <linux/percpu.h>
      16             : #include <linux/export.h>
      17             : #include <linux/kernel.h>
      18             : #include <linux/bitops.h>
      19             : #include <linux/smp.h>
      20             : #include <asm/nmi.h>
      21             : #include <linux/kprobes.h>
      22             : 
      23             : #include <asm/apic.h>
      24             : #include <asm/perf_event.h>
      25             : 
      26             : /*
      27             :  * this number is calculated from Intel's MSR_P4_CRU_ESCR5 register and it's
      28             :  * offset from MSR_P4_BSU_ESCR0.
      29             :  *
      30             :  * It will be the max for all platforms (for now)
      31             :  */
      32             : #define NMI_MAX_COUNTER_BITS 66
      33             : 
      34             : /*
      35             :  * perfctr_nmi_owner tracks the ownership of the perfctr registers:
      36             :  * evtsel_nmi_owner tracks the ownership of the event selection
      37             :  * - different performance counters/ event selection may be reserved for
      38             :  *   different subsystems this reservation system just tries to coordinate
      39             :  *   things a little
      40             :  */
      41             : static DECLARE_BITMAP(perfctr_nmi_owner, NMI_MAX_COUNTER_BITS);
      42             : static DECLARE_BITMAP(evntsel_nmi_owner, NMI_MAX_COUNTER_BITS);
      43             : 
      44             : /* converts an msr to an appropriate reservation bit */
      45           0 : static inline unsigned int nmi_perfctr_msr_to_bit(unsigned int msr)
      46             : {
      47             :         /* returns the bit offset of the performance counter register */
      48           0 :         switch (boot_cpu_data.x86_vendor) {
      49           0 :         case X86_VENDOR_HYGON:
      50             :         case X86_VENDOR_AMD:
      51           0 :                 if (msr >= MSR_F15H_PERF_CTR)
      52           0 :                         return (msr - MSR_F15H_PERF_CTR) >> 1;
      53           0 :                 return msr - MSR_K7_PERFCTR0;
      54             :         case X86_VENDOR_INTEL:
      55           0 :                 if (cpu_has(&boot_cpu_data, X86_FEATURE_ARCH_PERFMON))
      56           0 :                         return msr - MSR_ARCH_PERFMON_PERFCTR0;
      57             : 
      58           0 :                 switch (boot_cpu_data.x86) {
      59           0 :                 case 6:
      60           0 :                         return msr - MSR_P6_PERFCTR0;
      61           0 :                 case 11:
      62           0 :                         return msr - MSR_KNC_PERFCTR0;
      63           0 :                 case 15:
      64           0 :                         return msr - MSR_P4_BPU_PERFCTR0;
      65             :                 }
      66           0 :                 fallthrough;
      67             :         case X86_VENDOR_ZHAOXIN:
      68             :         case X86_VENDOR_CENTAUR:
      69           0 :                 return msr - MSR_ARCH_PERFMON_PERFCTR0;
      70             :         }
      71             :         return 0;
      72             : }
      73             : 
      74             : /*
      75             :  * converts an msr to an appropriate reservation bit
      76             :  * returns the bit offset of the event selection register
      77             :  */
      78           0 : static inline unsigned int nmi_evntsel_msr_to_bit(unsigned int msr)
      79             : {
      80             :         /* returns the bit offset of the event selection register */
      81           0 :         switch (boot_cpu_data.x86_vendor) {
      82           0 :         case X86_VENDOR_HYGON:
      83             :         case X86_VENDOR_AMD:
      84           0 :                 if (msr >= MSR_F15H_PERF_CTL)
      85           0 :                         return (msr - MSR_F15H_PERF_CTL) >> 1;
      86           0 :                 return msr - MSR_K7_EVNTSEL0;
      87             :         case X86_VENDOR_INTEL:
      88           0 :                 if (cpu_has(&boot_cpu_data, X86_FEATURE_ARCH_PERFMON))
      89           0 :                         return msr - MSR_ARCH_PERFMON_EVENTSEL0;
      90             : 
      91           0 :                 switch (boot_cpu_data.x86) {
      92           0 :                 case 6:
      93           0 :                         return msr - MSR_P6_EVNTSEL0;
      94           0 :                 case 11:
      95           0 :                         return msr - MSR_KNC_EVNTSEL0;
      96           0 :                 case 15:
      97           0 :                         return msr - MSR_P4_BSU_ESCR0;
      98             :                 }
      99           0 :                 fallthrough;
     100             :         case X86_VENDOR_ZHAOXIN:
     101             :         case X86_VENDOR_CENTAUR:
     102           0 :                 return msr - MSR_ARCH_PERFMON_EVENTSEL0;
     103             :         }
     104             :         return 0;
     105             : 
     106             : }
     107             : 
     108           0 : int reserve_perfctr_nmi(unsigned int msr)
     109             : {
     110           0 :         unsigned int counter;
     111             : 
     112           0 :         counter = nmi_perfctr_msr_to_bit(msr);
     113             :         /* register not managed by the allocator? */
     114           0 :         if (counter > NMI_MAX_COUNTER_BITS)
     115             :                 return 1;
     116             : 
     117           0 :         if (!test_and_set_bit(counter, perfctr_nmi_owner))
     118           0 :                 return 1;
     119             :         return 0;
     120             : }
     121             : EXPORT_SYMBOL(reserve_perfctr_nmi);
     122             : 
     123           0 : void release_perfctr_nmi(unsigned int msr)
     124             : {
     125           0 :         unsigned int counter;
     126             : 
     127           0 :         counter = nmi_perfctr_msr_to_bit(msr);
     128             :         /* register not managed by the allocator? */
     129           0 :         if (counter > NMI_MAX_COUNTER_BITS)
     130             :                 return;
     131             : 
     132           0 :         clear_bit(counter, perfctr_nmi_owner);
     133             : }
     134             : EXPORT_SYMBOL(release_perfctr_nmi);
     135             : 
     136           0 : int reserve_evntsel_nmi(unsigned int msr)
     137             : {
     138           0 :         unsigned int counter;
     139             : 
     140           0 :         counter = nmi_evntsel_msr_to_bit(msr);
     141             :         /* register not managed by the allocator? */
     142           0 :         if (counter > NMI_MAX_COUNTER_BITS)
     143             :                 return 1;
     144             : 
     145           0 :         if (!test_and_set_bit(counter, evntsel_nmi_owner))
     146           0 :                 return 1;
     147             :         return 0;
     148             : }
     149             : EXPORT_SYMBOL(reserve_evntsel_nmi);
     150             : 
     151           0 : void release_evntsel_nmi(unsigned int msr)
     152             : {
     153           0 :         unsigned int counter;
     154             : 
     155           0 :         counter = nmi_evntsel_msr_to_bit(msr);
     156             :         /* register not managed by the allocator? */
     157           0 :         if (counter > NMI_MAX_COUNTER_BITS)
     158             :                 return;
     159             : 
     160           0 :         clear_bit(counter, evntsel_nmi_owner);
     161             : }
     162             : EXPORT_SYMBOL(release_evntsel_nmi);