LCOV - code coverage report
Current view: top level - arch/x86/events - perf_event.h (source / functions) Hit Total Coverage
Test: landlock.info Lines: 0 65 0.0 %
Date: 2021-04-22 12:43:58 Functions: 0 6 0.0 %

          Line data    Source code
       1             : /*
       2             :  * Performance events x86 architecture header
       3             :  *
       4             :  *  Copyright (C) 2008 Thomas Gleixner <tglx@linutronix.de>
       5             :  *  Copyright (C) 2008-2009 Red Hat, Inc., Ingo Molnar
       6             :  *  Copyright (C) 2009 Jaswinder Singh Rajput
       7             :  *  Copyright (C) 2009 Advanced Micro Devices, Inc., Robert Richter
       8             :  *  Copyright (C) 2008-2009 Red Hat, Inc., Peter Zijlstra
       9             :  *  Copyright (C) 2009 Intel Corporation, <markus.t.metzger@intel.com>
      10             :  *  Copyright (C) 2009 Google, Inc., Stephane Eranian
      11             :  *
      12             :  *  For licencing details see kernel-base/COPYING
      13             :  */
      14             : 
      15             : #include <linux/perf_event.h>
      16             : 
      17             : #include <asm/intel_ds.h>
      18             : 
      19             : /* To enable MSR tracing please use the generic trace points. */
      20             : 
      21             : /*
      22             :  *          |   NHM/WSM    |      SNB     |
      23             :  * register -------------------------------
      24             :  *          |  HT  | no HT |  HT  | no HT |
      25             :  *-----------------------------------------
      26             :  * offcore  | core | core  | cpu  | core  |
      27             :  * lbr_sel  | core | core  | cpu  | core  |
      28             :  * ld_lat   | cpu  | core  | cpu  | core  |
      29             :  *-----------------------------------------
      30             :  *
      31             :  * Given that there is a small number of shared regs,
      32             :  * we can pre-allocate their slot in the per-cpu
      33             :  * per-core reg tables.
      34             :  */
      35             : enum extra_reg_type {
      36             :         EXTRA_REG_NONE  = -1,   /* not used */
      37             : 
      38             :         EXTRA_REG_RSP_0 = 0,    /* offcore_response_0 */
      39             :         EXTRA_REG_RSP_1 = 1,    /* offcore_response_1 */
      40             :         EXTRA_REG_LBR   = 2,    /* lbr_select */
      41             :         EXTRA_REG_LDLAT = 3,    /* ld_lat_threshold */
      42             :         EXTRA_REG_FE    = 4,    /* fe_* */
      43             : 
      44             :         EXTRA_REG_MAX           /* number of entries needed */
      45             : };
      46             : 
      47             : struct event_constraint {
      48             :         union {
      49             :                 unsigned long   idxmsk[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
      50             :                 u64             idxmsk64;
      51             :         };
      52             :         u64             code;
      53             :         u64             cmask;
      54             :         int             weight;
      55             :         int             overlap;
      56             :         int             flags;
      57             :         unsigned int    size;
      58             : };
      59             : 
      60           0 : static inline bool constraint_match(struct event_constraint *c, u64 ecode)
      61             : {
      62           0 :         return ((ecode & c->cmask) - c->code) <= (u64)c->size;
      63             : }
      64             : 
      65             : /*
      66             :  * struct hw_perf_event.flags flags
      67             :  */
      68             : #define PERF_X86_EVENT_PEBS_LDLAT       0x0001 /* ld+ldlat data address sampling */
      69             : #define PERF_X86_EVENT_PEBS_ST          0x0002 /* st data address sampling */
      70             : #define PERF_X86_EVENT_PEBS_ST_HSW      0x0004 /* haswell style datala, store */
      71             : #define PERF_X86_EVENT_PEBS_LD_HSW      0x0008 /* haswell style datala, load */
      72             : #define PERF_X86_EVENT_PEBS_NA_HSW      0x0010 /* haswell style datala, unknown */
      73             : #define PERF_X86_EVENT_EXCL             0x0020 /* HT exclusivity on counter */
      74             : #define PERF_X86_EVENT_DYNAMIC          0x0040 /* dynamic alloc'd constraint */
      75             : #define PERF_X86_EVENT_RDPMC_ALLOWED    0x0080 /* grant rdpmc permission */
      76             : #define PERF_X86_EVENT_EXCL_ACCT        0x0100 /* accounted EXCL event */
      77             : #define PERF_X86_EVENT_AUTO_RELOAD      0x0200 /* use PEBS auto-reload */
      78             : #define PERF_X86_EVENT_LARGE_PEBS       0x0400 /* use large PEBS */
      79             : #define PERF_X86_EVENT_PEBS_VIA_PT      0x0800 /* use PT buffer for PEBS */
      80             : #define PERF_X86_EVENT_PAIR             0x1000 /* Large Increment per Cycle */
      81             : #define PERF_X86_EVENT_LBR_SELECT       0x2000 /* Save/Restore MSR_LBR_SELECT */
      82             : #define PERF_X86_EVENT_TOPDOWN          0x4000 /* Count Topdown slots/metrics events */
      83             : #define PERF_X86_EVENT_PEBS_STLAT       0x8000 /* st+stlat data address sampling */
      84             : 
      85           0 : static inline bool is_topdown_count(struct perf_event *event)
      86             : {
      87           0 :         return event->hw.flags & PERF_X86_EVENT_TOPDOWN;
      88             : }
      89             : 
      90           0 : static inline bool is_metric_event(struct perf_event *event)
      91             : {
      92           0 :         u64 config = event->attr.config;
      93             : 
      94           0 :         return ((config & ARCH_PERFMON_EVENTSEL_EVENT) == 0) &&
      95           0 :                 ((config & INTEL_ARCH_EVENT_MASK) >= INTEL_TD_METRIC_RETIRING)  &&
      96           0 :                 ((config & INTEL_ARCH_EVENT_MASK) <= INTEL_TD_METRIC_MAX);
      97             : }
      98             : 
      99           0 : static inline bool is_slots_event(struct perf_event *event)
     100             : {
     101           0 :         return (event->attr.config & INTEL_ARCH_EVENT_MASK) == INTEL_TD_SLOTS;
     102             : }
     103             : 
     104           0 : static inline bool is_topdown_event(struct perf_event *event)
     105             : {
     106           0 :         return is_metric_event(event) || is_slots_event(event);
     107             : }
     108             : 
     109             : struct amd_nb {
     110             :         int nb_id;  /* NorthBridge id */
     111             :         int refcnt; /* reference count */
     112             :         struct perf_event *owners[X86_PMC_IDX_MAX];
     113             :         struct event_constraint event_constraints[X86_PMC_IDX_MAX];
     114             : };
     115             : 
     116             : #define PEBS_COUNTER_MASK       ((1ULL << MAX_PEBS_EVENTS) - 1)
     117             : #define PEBS_PMI_AFTER_EACH_RECORD BIT_ULL(60)
     118             : #define PEBS_OUTPUT_OFFSET      61
     119             : #define PEBS_OUTPUT_MASK        (3ull << PEBS_OUTPUT_OFFSET)
     120             : #define PEBS_OUTPUT_PT          (1ull << PEBS_OUTPUT_OFFSET)
     121             : #define PEBS_VIA_PT_MASK        (PEBS_OUTPUT_PT | PEBS_PMI_AFTER_EACH_RECORD)
     122             : 
     123             : /*
     124             :  * Flags PEBS can handle without an PMI.
     125             :  *
     126             :  * TID can only be handled by flushing at context switch.
     127             :  * REGS_USER can be handled for events limited to ring 3.
     128             :  *
     129             :  */
     130             : #define LARGE_PEBS_FLAGS \
     131             :         (PERF_SAMPLE_IP | PERF_SAMPLE_TID | PERF_SAMPLE_ADDR | \
     132             :         PERF_SAMPLE_ID | PERF_SAMPLE_CPU | PERF_SAMPLE_STREAM_ID | \
     133             :         PERF_SAMPLE_DATA_SRC | PERF_SAMPLE_IDENTIFIER | \
     134             :         PERF_SAMPLE_TRANSACTION | PERF_SAMPLE_PHYS_ADDR | \
     135             :         PERF_SAMPLE_REGS_INTR | PERF_SAMPLE_REGS_USER | \
     136             :         PERF_SAMPLE_PERIOD | PERF_SAMPLE_CODE_PAGE_SIZE)
     137             : 
     138             : #define PEBS_GP_REGS                    \
     139             :         ((1ULL << PERF_REG_X86_AX)    | \
     140             :          (1ULL << PERF_REG_X86_BX)    | \
     141             :          (1ULL << PERF_REG_X86_CX)    | \
     142             :          (1ULL << PERF_REG_X86_DX)    | \
     143             :          (1ULL << PERF_REG_X86_DI)    | \
     144             :          (1ULL << PERF_REG_X86_SI)    | \
     145             :          (1ULL << PERF_REG_X86_SP)    | \
     146             :          (1ULL << PERF_REG_X86_BP)    | \
     147             :          (1ULL << PERF_REG_X86_IP)    | \
     148             :          (1ULL << PERF_REG_X86_FLAGS) | \
     149             :          (1ULL << PERF_REG_X86_R8)    | \
     150             :          (1ULL << PERF_REG_X86_R9)    | \
     151             :          (1ULL << PERF_REG_X86_R10)   | \
     152             :          (1ULL << PERF_REG_X86_R11)   | \
     153             :          (1ULL << PERF_REG_X86_R12)   | \
     154             :          (1ULL << PERF_REG_X86_R13)   | \
     155             :          (1ULL << PERF_REG_X86_R14)   | \
     156             :          (1ULL << PERF_REG_X86_R15))
     157             : 
     158             : /*
     159             :  * Per register state.
     160             :  */
     161             : struct er_account {
     162             :         raw_spinlock_t      lock;       /* per-core: protect structure */
     163             :         u64                 config;     /* extra MSR config */
     164             :         u64                 reg;        /* extra MSR number */
     165             :         atomic_t            ref;        /* reference count */
     166             : };
     167             : 
     168             : /*
     169             :  * Per core/cpu state
     170             :  *
     171             :  * Used to coordinate shared registers between HT threads or
     172             :  * among events on a single PMU.
     173             :  */
     174             : struct intel_shared_regs {
     175             :         struct er_account       regs[EXTRA_REG_MAX];
     176             :         int                     refcnt;         /* per-core: #HT threads */
     177             :         unsigned                core_id;        /* per-core: core id */
     178             : };
     179             : 
     180             : enum intel_excl_state_type {
     181             :         INTEL_EXCL_UNUSED    = 0, /* counter is unused */
     182             :         INTEL_EXCL_SHARED    = 1, /* counter can be used by both threads */
     183             :         INTEL_EXCL_EXCLUSIVE = 2, /* counter can be used by one thread only */
     184             : };
     185             : 
     186             : struct intel_excl_states {
     187             :         enum intel_excl_state_type state[X86_PMC_IDX_MAX];
     188             :         bool sched_started; /* true if scheduling has started */
     189             : };
     190             : 
     191             : struct intel_excl_cntrs {
     192             :         raw_spinlock_t  lock;
     193             : 
     194             :         struct intel_excl_states states[2];
     195             : 
     196             :         union {
     197             :                 u16     has_exclusive[2];
     198             :                 u32     exclusive_present;
     199             :         };
     200             : 
     201             :         int             refcnt;         /* per-core: #HT threads */
     202             :         unsigned        core_id;        /* per-core: core id */
     203             : };
     204             : 
     205             : struct x86_perf_task_context;
     206             : #define MAX_LBR_ENTRIES         32
     207             : 
     208             : enum {
     209             :         LBR_FORMAT_32           = 0x00,
     210             :         LBR_FORMAT_LIP          = 0x01,
     211             :         LBR_FORMAT_EIP          = 0x02,
     212             :         LBR_FORMAT_EIP_FLAGS    = 0x03,
     213             :         LBR_FORMAT_EIP_FLAGS2   = 0x04,
     214             :         LBR_FORMAT_INFO         = 0x05,
     215             :         LBR_FORMAT_TIME         = 0x06,
     216             :         LBR_FORMAT_MAX_KNOWN    = LBR_FORMAT_TIME,
     217             : };
     218             : 
     219             : enum {
     220             :         X86_PERF_KFREE_SHARED = 0,
     221             :         X86_PERF_KFREE_EXCL   = 1,
     222             :         X86_PERF_KFREE_MAX
     223             : };
     224             : 
     225             : struct cpu_hw_events {
     226             :         /*
     227             :          * Generic x86 PMC bits
     228             :          */
     229             :         struct perf_event       *events[X86_PMC_IDX_MAX]; /* in counter order */
     230             :         unsigned long           active_mask[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
     231             :         unsigned long           running[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
     232             :         int                     enabled;
     233             : 
     234             :         int                     n_events; /* the # of events in the below arrays */
     235             :         int                     n_added;  /* the # last events in the below arrays;
     236             :                                              they've never been enabled yet */
     237             :         int                     n_txn;    /* the # last events in the below arrays;
     238             :                                              added in the current transaction */
     239             :         int                     n_txn_pair;
     240             :         int                     n_txn_metric;
     241             :         int                     assign[X86_PMC_IDX_MAX]; /* event to counter assignment */
     242             :         u64                     tags[X86_PMC_IDX_MAX];
     243             : 
     244             :         struct perf_event       *event_list[X86_PMC_IDX_MAX]; /* in enabled order */
     245             :         struct event_constraint *event_constraint[X86_PMC_IDX_MAX];
     246             : 
     247             :         int                     n_excl; /* the number of exclusive events */
     248             : 
     249             :         unsigned int            txn_flags;
     250             :         int                     is_fake;
     251             : 
     252             :         /*
     253             :          * Intel DebugStore bits
     254             :          */
     255             :         struct debug_store      *ds;
     256             :         void                    *ds_pebs_vaddr;
     257             :         void                    *ds_bts_vaddr;
     258             :         u64                     pebs_enabled;
     259             :         int                     n_pebs;
     260             :         int                     n_large_pebs;
     261             :         int                     n_pebs_via_pt;
     262             :         int                     pebs_output;
     263             : 
     264             :         /* Current super set of events hardware configuration */
     265             :         u64                     pebs_data_cfg;
     266             :         u64                     active_pebs_data_cfg;
     267             :         int                     pebs_record_size;
     268             : 
     269             :         /*
     270             :          * Intel LBR bits
     271             :          */
     272             :         int                             lbr_users;
     273             :         int                             lbr_pebs_users;
     274             :         struct perf_branch_stack        lbr_stack;
     275             :         struct perf_branch_entry        lbr_entries[MAX_LBR_ENTRIES];
     276             :         union {
     277             :                 struct er_account               *lbr_sel;
     278             :                 struct er_account               *lbr_ctl;
     279             :         };
     280             :         u64                             br_sel;
     281             :         void                            *last_task_ctx;
     282             :         int                             last_log_id;
     283             :         int                             lbr_select;
     284             :         void                            *lbr_xsave;
     285             : 
     286             :         /*
     287             :          * Intel host/guest exclude bits
     288             :          */
     289             :         u64                             intel_ctrl_guest_mask;
     290             :         u64                             intel_ctrl_host_mask;
     291             :         struct perf_guest_switch_msr    guest_switch_msrs[X86_PMC_IDX_MAX];
     292             : 
     293             :         /*
     294             :          * Intel checkpoint mask
     295             :          */
     296             :         u64                             intel_cp_status;
     297             : 
     298             :         /*
     299             :          * manage shared (per-core, per-cpu) registers
     300             :          * used on Intel NHM/WSM/SNB
     301             :          */
     302             :         struct intel_shared_regs        *shared_regs;
     303             :         /*
     304             :          * manage exclusive counter access between hyperthread
     305             :          */
     306             :         struct event_constraint *constraint_list; /* in enable order */
     307             :         struct intel_excl_cntrs         *excl_cntrs;
     308             :         int excl_thread_id; /* 0 or 1 */
     309             : 
     310             :         /*
     311             :          * SKL TSX_FORCE_ABORT shadow
     312             :          */
     313             :         u64                             tfa_shadow;
     314             : 
     315             :         /*
     316             :          * Perf Metrics
     317             :          */
     318             :         /* number of accepted metrics events */
     319             :         int                             n_metric;
     320             : 
     321             :         /*
     322             :          * AMD specific bits
     323             :          */
     324             :         struct amd_nb                   *amd_nb;
     325             :         /* Inverted mask of bits to clear in the perf_ctr ctrl registers */
     326             :         u64                             perf_ctr_virt_mask;
     327             :         int                             n_pair; /* Large increment events */
     328             : 
     329             :         void                            *kfree_on_online[X86_PERF_KFREE_MAX];
     330             : };
     331             : 
     332             : #define __EVENT_CONSTRAINT_RANGE(c, e, n, m, w, o, f) { \
     333             :         { .idxmsk64 = (n) },            \
     334             :         .code = (c),                    \
     335             :         .size = (e) - (c),              \
     336             :         .cmask = (m),                   \
     337             :         .weight = (w),                  \
     338             :         .overlap = (o),                 \
     339             :         .flags = f,                     \
     340             : }
     341             : 
     342             : #define __EVENT_CONSTRAINT(c, n, m, w, o, f) \
     343             :         __EVENT_CONSTRAINT_RANGE(c, c, n, m, w, o, f)
     344             : 
     345             : #define EVENT_CONSTRAINT(c, n, m)       \
     346             :         __EVENT_CONSTRAINT(c, n, m, HWEIGHT(n), 0, 0)
     347             : 
     348             : /*
     349             :  * The constraint_match() function only works for 'simple' event codes
     350             :  * and not for extended (AMD64_EVENTSEL_EVENT) events codes.
     351             :  */
     352             : #define EVENT_CONSTRAINT_RANGE(c, e, n, m) \
     353             :         __EVENT_CONSTRAINT_RANGE(c, e, n, m, HWEIGHT(n), 0, 0)
     354             : 
     355             : #define INTEL_EXCLEVT_CONSTRAINT(c, n)  \
     356             :         __EVENT_CONSTRAINT(c, n, ARCH_PERFMON_EVENTSEL_EVENT, HWEIGHT(n),\
     357             :                            0, PERF_X86_EVENT_EXCL)
     358             : 
     359             : /*
     360             :  * The overlap flag marks event constraints with overlapping counter
     361             :  * masks. This is the case if the counter mask of such an event is not
     362             :  * a subset of any other counter mask of a constraint with an equal or
     363             :  * higher weight, e.g.:
     364             :  *
     365             :  *  c_overlaps = EVENT_CONSTRAINT_OVERLAP(0, 0x09, 0);
     366             :  *  c_another1 = EVENT_CONSTRAINT(0, 0x07, 0);
     367             :  *  c_another2 = EVENT_CONSTRAINT(0, 0x38, 0);
     368             :  *
     369             :  * The event scheduler may not select the correct counter in the first
     370             :  * cycle because it needs to know which subsequent events will be
     371             :  * scheduled. It may fail to schedule the events then. So we set the
     372             :  * overlap flag for such constraints to give the scheduler a hint which
     373             :  * events to select for counter rescheduling.
     374             :  *
     375             :  * Care must be taken as the rescheduling algorithm is O(n!) which
     376             :  * will increase scheduling cycles for an over-committed system
     377             :  * dramatically.  The number of such EVENT_CONSTRAINT_OVERLAP() macros
     378             :  * and its counter masks must be kept at a minimum.
     379             :  */
     380             : #define EVENT_CONSTRAINT_OVERLAP(c, n, m)       \
     381             :         __EVENT_CONSTRAINT(c, n, m, HWEIGHT(n), 1, 0)
     382             : 
     383             : /*
     384             :  * Constraint on the Event code.
     385             :  */
     386             : #define INTEL_EVENT_CONSTRAINT(c, n)    \
     387             :         EVENT_CONSTRAINT(c, n, ARCH_PERFMON_EVENTSEL_EVENT)
     388             : 
     389             : /*
     390             :  * Constraint on a range of Event codes
     391             :  */
     392             : #define INTEL_EVENT_CONSTRAINT_RANGE(c, e, n)                   \
     393             :         EVENT_CONSTRAINT_RANGE(c, e, n, ARCH_PERFMON_EVENTSEL_EVENT)
     394             : 
     395             : /*
     396             :  * Constraint on the Event code + UMask + fixed-mask
     397             :  *
     398             :  * filter mask to validate fixed counter events.
     399             :  * the following filters disqualify for fixed counters:
     400             :  *  - inv
     401             :  *  - edge
     402             :  *  - cnt-mask
     403             :  *  - in_tx
     404             :  *  - in_tx_checkpointed
     405             :  *  The other filters are supported by fixed counters.
     406             :  *  The any-thread option is supported starting with v3.
     407             :  */
     408             : #define FIXED_EVENT_FLAGS (X86_RAW_EVENT_MASK|HSW_IN_TX|HSW_IN_TX_CHECKPOINTED)
     409             : #define FIXED_EVENT_CONSTRAINT(c, n)    \
     410             :         EVENT_CONSTRAINT(c, (1ULL << (32+n)), FIXED_EVENT_FLAGS)
     411             : 
     412             : /*
     413             :  * The special metric counters do not actually exist. They are calculated from
     414             :  * the combination of the FxCtr3 + MSR_PERF_METRICS.
     415             :  *
     416             :  * The special metric counters are mapped to a dummy offset for the scheduler.
     417             :  * The sharing between multiple users of the same metric without multiplexing
     418             :  * is not allowed, even though the hardware supports that in principle.
     419             :  */
     420             : 
     421             : #define METRIC_EVENT_CONSTRAINT(c, n)                                   \
     422             :         EVENT_CONSTRAINT(c, (1ULL << (INTEL_PMC_IDX_METRIC_BASE + n)),    \
     423             :                          INTEL_ARCH_EVENT_MASK)
     424             : 
     425             : /*
     426             :  * Constraint on the Event code + UMask
     427             :  */
     428             : #define INTEL_UEVENT_CONSTRAINT(c, n)   \
     429             :         EVENT_CONSTRAINT(c, n, INTEL_ARCH_EVENT_MASK)
     430             : 
     431             : /* Constraint on specific umask bit only + event */
     432             : #define INTEL_UBIT_EVENT_CONSTRAINT(c, n)       \
     433             :         EVENT_CONSTRAINT(c, n, ARCH_PERFMON_EVENTSEL_EVENT|(c))
     434             : 
     435             : /* Like UEVENT_CONSTRAINT, but match flags too */
     436             : #define INTEL_FLAGS_UEVENT_CONSTRAINT(c, n)     \
     437             :         EVENT_CONSTRAINT(c, n, INTEL_ARCH_EVENT_MASK|X86_ALL_EVENT_FLAGS)
     438             : 
     439             : #define INTEL_EXCLUEVT_CONSTRAINT(c, n) \
     440             :         __EVENT_CONSTRAINT(c, n, INTEL_ARCH_EVENT_MASK, \
     441             :                            HWEIGHT(n), 0, PERF_X86_EVENT_EXCL)
     442             : 
     443             : #define INTEL_PLD_CONSTRAINT(c, n)      \
     444             :         __EVENT_CONSTRAINT(c, n, INTEL_ARCH_EVENT_MASK|X86_ALL_EVENT_FLAGS, \
     445             :                            HWEIGHT(n), 0, PERF_X86_EVENT_PEBS_LDLAT)
     446             : 
     447             : #define INTEL_PSD_CONSTRAINT(c, n)      \
     448             :         __EVENT_CONSTRAINT(c, n, INTEL_ARCH_EVENT_MASK|X86_ALL_EVENT_FLAGS, \
     449             :                            HWEIGHT(n), 0, PERF_X86_EVENT_PEBS_STLAT)
     450             : 
     451             : #define INTEL_PST_CONSTRAINT(c, n)      \
     452             :         __EVENT_CONSTRAINT(c, n, INTEL_ARCH_EVENT_MASK|X86_ALL_EVENT_FLAGS, \
     453             :                           HWEIGHT(n), 0, PERF_X86_EVENT_PEBS_ST)
     454             : 
     455             : /* Event constraint, but match on all event flags too. */
     456             : #define INTEL_FLAGS_EVENT_CONSTRAINT(c, n) \
     457             :         EVENT_CONSTRAINT(c, n, ARCH_PERFMON_EVENTSEL_EVENT|X86_ALL_EVENT_FLAGS)
     458             : 
     459             : #define INTEL_FLAGS_EVENT_CONSTRAINT_RANGE(c, e, n)                     \
     460             :         EVENT_CONSTRAINT_RANGE(c, e, n, ARCH_PERFMON_EVENTSEL_EVENT|X86_ALL_EVENT_FLAGS)
     461             : 
     462             : /* Check only flags, but allow all event/umask */
     463             : #define INTEL_ALL_EVENT_CONSTRAINT(code, n)     \
     464             :         EVENT_CONSTRAINT(code, n, X86_ALL_EVENT_FLAGS)
     465             : 
     466             : /* Check flags and event code, and set the HSW store flag */
     467             : #define INTEL_FLAGS_EVENT_CONSTRAINT_DATALA_ST(code, n) \
     468             :         __EVENT_CONSTRAINT(code, n,                     \
     469             :                           ARCH_PERFMON_EVENTSEL_EVENT|X86_ALL_EVENT_FLAGS, \
     470             :                           HWEIGHT(n), 0, PERF_X86_EVENT_PEBS_ST_HSW)
     471             : 
     472             : /* Check flags and event code, and set the HSW load flag */
     473             : #define INTEL_FLAGS_EVENT_CONSTRAINT_DATALA_LD(code, n) \
     474             :         __EVENT_CONSTRAINT(code, n,                     \
     475             :                           ARCH_PERFMON_EVENTSEL_EVENT|X86_ALL_EVENT_FLAGS, \
     476             :                           HWEIGHT(n), 0, PERF_X86_EVENT_PEBS_LD_HSW)
     477             : 
     478             : #define INTEL_FLAGS_EVENT_CONSTRAINT_DATALA_LD_RANGE(code, end, n) \
     479             :         __EVENT_CONSTRAINT_RANGE(code, end, n,                          \
     480             :                           ARCH_PERFMON_EVENTSEL_EVENT|X86_ALL_EVENT_FLAGS, \
     481             :                           HWEIGHT(n), 0, PERF_X86_EVENT_PEBS_LD_HSW)
     482             : 
     483             : #define INTEL_FLAGS_EVENT_CONSTRAINT_DATALA_XLD(code, n) \
     484             :         __EVENT_CONSTRAINT(code, n,                     \
     485             :                           ARCH_PERFMON_EVENTSEL_EVENT|X86_ALL_EVENT_FLAGS, \
     486             :                           HWEIGHT(n), 0, \
     487             :                           PERF_X86_EVENT_PEBS_LD_HSW|PERF_X86_EVENT_EXCL)
     488             : 
     489             : /* Check flags and event code/umask, and set the HSW store flag */
     490             : #define INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_ST(code, n) \
     491             :         __EVENT_CONSTRAINT(code, n,                     \
     492             :                           INTEL_ARCH_EVENT_MASK|X86_ALL_EVENT_FLAGS, \
     493             :                           HWEIGHT(n), 0, PERF_X86_EVENT_PEBS_ST_HSW)
     494             : 
     495             : #define INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_XST(code, n) \
     496             :         __EVENT_CONSTRAINT(code, n,                     \
     497             :                           INTEL_ARCH_EVENT_MASK|X86_ALL_EVENT_FLAGS, \
     498             :                           HWEIGHT(n), 0, \
     499             :                           PERF_X86_EVENT_PEBS_ST_HSW|PERF_X86_EVENT_EXCL)
     500             : 
     501             : /* Check flags and event code/umask, and set the HSW load flag */
     502             : #define INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_LD(code, n) \
     503             :         __EVENT_CONSTRAINT(code, n,                     \
     504             :                           INTEL_ARCH_EVENT_MASK|X86_ALL_EVENT_FLAGS, \
     505             :                           HWEIGHT(n), 0, PERF_X86_EVENT_PEBS_LD_HSW)
     506             : 
     507             : #define INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_XLD(code, n) \
     508             :         __EVENT_CONSTRAINT(code, n,                     \
     509             :                           INTEL_ARCH_EVENT_MASK|X86_ALL_EVENT_FLAGS, \
     510             :                           HWEIGHT(n), 0, \
     511             :                           PERF_X86_EVENT_PEBS_LD_HSW|PERF_X86_EVENT_EXCL)
     512             : 
     513             : /* Check flags and event code/umask, and set the HSW N/A flag */
     514             : #define INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_NA(code, n) \
     515             :         __EVENT_CONSTRAINT(code, n,                     \
     516             :                           INTEL_ARCH_EVENT_MASK|X86_ALL_EVENT_FLAGS, \
     517             :                           HWEIGHT(n), 0, PERF_X86_EVENT_PEBS_NA_HSW)
     518             : 
     519             : 
     520             : /*
     521             :  * We define the end marker as having a weight of -1
     522             :  * to enable blacklisting of events using a counter bitmask
     523             :  * of zero and thus a weight of zero.
     524             :  * The end marker has a weight that cannot possibly be
     525             :  * obtained from counting the bits in the bitmask.
     526             :  */
     527             : #define EVENT_CONSTRAINT_END { .weight = -1 }
     528             : 
     529             : /*
     530             :  * Check for end marker with weight == -1
     531             :  */
     532             : #define for_each_event_constraint(e, c) \
     533             :         for ((e) = (c); (e)->weight != -1; (e)++)
     534             : 
     535             : /*
     536             :  * Extra registers for specific events.
     537             :  *
     538             :  * Some events need large masks and require external MSRs.
     539             :  * Those extra MSRs end up being shared for all events on
     540             :  * a PMU and sometimes between PMU of sibling HT threads.
     541             :  * In either case, the kernel needs to handle conflicting
     542             :  * accesses to those extra, shared, regs. The data structure
     543             :  * to manage those registers is stored in cpu_hw_event.
     544             :  */
     545             : struct extra_reg {
     546             :         unsigned int            event;
     547             :         unsigned int            msr;
     548             :         u64                     config_mask;
     549             :         u64                     valid_mask;
     550             :         int                     idx;  /* per_xxx->regs[] reg index */
     551             :         bool                    extra_msr_access;
     552             : };
     553             : 
     554             : #define EVENT_EXTRA_REG(e, ms, m, vm, i) {      \
     555             :         .event = (e),                   \
     556             :         .msr = (ms),                    \
     557             :         .config_mask = (m),             \
     558             :         .valid_mask = (vm),             \
     559             :         .idx = EXTRA_REG_##i,           \
     560             :         .extra_msr_access = true,       \
     561             :         }
     562             : 
     563             : #define INTEL_EVENT_EXTRA_REG(event, msr, vm, idx)      \
     564             :         EVENT_EXTRA_REG(event, msr, ARCH_PERFMON_EVENTSEL_EVENT, vm, idx)
     565             : 
     566             : #define INTEL_UEVENT_EXTRA_REG(event, msr, vm, idx) \
     567             :         EVENT_EXTRA_REG(event, msr, ARCH_PERFMON_EVENTSEL_EVENT | \
     568             :                         ARCH_PERFMON_EVENTSEL_UMASK, vm, idx)
     569             : 
     570             : #define INTEL_UEVENT_PEBS_LDLAT_EXTRA_REG(c) \
     571             :         INTEL_UEVENT_EXTRA_REG(c, \
     572             :                                MSR_PEBS_LD_LAT_THRESHOLD, \
     573             :                                0xffff, \
     574             :                                LDLAT)
     575             : 
     576             : #define EVENT_EXTRA_END EVENT_EXTRA_REG(0, 0, 0, 0, RSP_0)
     577             : 
     578             : union perf_capabilities {
     579             :         struct {
     580             :                 u64     lbr_format:6;
     581             :                 u64     pebs_trap:1;
     582             :                 u64     pebs_arch_reg:1;
     583             :                 u64     pebs_format:4;
     584             :                 u64     smm_freeze:1;
     585             :                 /*
     586             :                  * PMU supports separate counter range for writing
     587             :                  * values > 32bit.
     588             :                  */
     589             :                 u64     full_width_write:1;
     590             :                 u64     pebs_baseline:1;
     591             :                 u64     perf_metrics:1;
     592             :                 u64     pebs_output_pt_available:1;
     593             :                 u64     anythread_deprecated:1;
     594             :         };
     595             :         u64     capabilities;
     596             : };
     597             : 
     598             : struct x86_pmu_quirk {
     599             :         struct x86_pmu_quirk *next;
     600             :         void (*func)(void);
     601             : };
     602             : 
     603             : union x86_pmu_config {
     604             :         struct {
     605             :                 u64 event:8,
     606             :                     umask:8,
     607             :                     usr:1,
     608             :                     os:1,
     609             :                     edge:1,
     610             :                     pc:1,
     611             :                     interrupt:1,
     612             :                     __reserved1:1,
     613             :                     en:1,
     614             :                     inv:1,
     615             :                     cmask:8,
     616             :                     event2:4,
     617             :                     __reserved2:4,
     618             :                     go:1,
     619             :                     ho:1;
     620             :         } bits;
     621             :         u64 value;
     622             : };
     623             : 
     624             : #define X86_CONFIG(args...) ((union x86_pmu_config){.bits = {args}}).value
     625             : 
     626             : enum {
     627             :         x86_lbr_exclusive_lbr,
     628             :         x86_lbr_exclusive_bts,
     629             :         x86_lbr_exclusive_pt,
     630             :         x86_lbr_exclusive_max,
     631             : };
     632             : 
     633             : /*
     634             :  * struct x86_pmu - generic x86 pmu
     635             :  */
     636             : struct x86_pmu {
     637             :         /*
     638             :          * Generic x86 PMC bits
     639             :          */
     640             :         const char      *name;
     641             :         int             version;
     642             :         int             (*handle_irq)(struct pt_regs *);
     643             :         void            (*disable_all)(void);
     644             :         void            (*enable_all)(int added);
     645             :         void            (*enable)(struct perf_event *);
     646             :         void            (*disable)(struct perf_event *);
     647             :         void            (*add)(struct perf_event *);
     648             :         void            (*del)(struct perf_event *);
     649             :         void            (*read)(struct perf_event *event);
     650             :         int             (*hw_config)(struct perf_event *event);
     651             :         int             (*schedule_events)(struct cpu_hw_events *cpuc, int n, int *assign);
     652             :         unsigned        eventsel;
     653             :         unsigned        perfctr;
     654             :         int             (*addr_offset)(int index, bool eventsel);
     655             :         int             (*rdpmc_index)(int index);
     656             :         u64             (*event_map)(int);
     657             :         int             max_events;
     658             :         int             num_counters;
     659             :         int             num_counters_fixed;
     660             :         int             cntval_bits;
     661             :         u64             cntval_mask;
     662             :         union {
     663             :                         unsigned long events_maskl;
     664             :                         unsigned long events_mask[BITS_TO_LONGS(ARCH_PERFMON_EVENTS_COUNT)];
     665             :         };
     666             :         int             events_mask_len;
     667             :         int             apic;
     668             :         u64             max_period;
     669             :         struct event_constraint *
     670             :                         (*get_event_constraints)(struct cpu_hw_events *cpuc,
     671             :                                                  int idx,
     672             :                                                  struct perf_event *event);
     673             : 
     674             :         void            (*put_event_constraints)(struct cpu_hw_events *cpuc,
     675             :                                                  struct perf_event *event);
     676             : 
     677             :         void            (*start_scheduling)(struct cpu_hw_events *cpuc);
     678             : 
     679             :         void            (*commit_scheduling)(struct cpu_hw_events *cpuc, int idx, int cntr);
     680             : 
     681             :         void            (*stop_scheduling)(struct cpu_hw_events *cpuc);
     682             : 
     683             :         struct event_constraint *event_constraints;
     684             :         struct x86_pmu_quirk *quirks;
     685             :         int             perfctr_second_write;
     686             :         u64             (*limit_period)(struct perf_event *event, u64 l);
     687             : 
     688             :         /* PMI handler bits */
     689             :         unsigned int    late_ack                :1,
     690             :                         enabled_ack             :1;
     691             :         /*
     692             :          * sysfs attrs
     693             :          */
     694             :         int             attr_rdpmc_broken;
     695             :         int             attr_rdpmc;
     696             :         struct attribute **format_attrs;
     697             : 
     698             :         ssize_t         (*events_sysfs_show)(char *page, u64 config);
     699             :         const struct attribute_group **attr_update;
     700             : 
     701             :         unsigned long   attr_freeze_on_smi;
     702             : 
     703             :         /*
     704             :          * CPU Hotplug hooks
     705             :          */
     706             :         int             (*cpu_prepare)(int cpu);
     707             :         void            (*cpu_starting)(int cpu);
     708             :         void            (*cpu_dying)(int cpu);
     709             :         void            (*cpu_dead)(int cpu);
     710             : 
     711             :         void            (*check_microcode)(void);
     712             :         void            (*sched_task)(struct perf_event_context *ctx,
     713             :                                       bool sched_in);
     714             : 
     715             :         /*
     716             :          * Intel Arch Perfmon v2+
     717             :          */
     718             :         u64                     intel_ctrl;
     719             :         union perf_capabilities intel_cap;
     720             : 
     721             :         /*
     722             :          * Intel DebugStore bits
     723             :          */
     724             :         unsigned int    bts                     :1,
     725             :                         bts_active              :1,
     726             :                         pebs                    :1,
     727             :                         pebs_active             :1,
     728             :                         pebs_broken             :1,
     729             :                         pebs_prec_dist          :1,
     730             :                         pebs_no_tlb             :1,
     731             :                         pebs_no_isolation       :1,
     732             :                         pebs_block              :1;
     733             :         int             pebs_record_size;
     734             :         int             pebs_buffer_size;
     735             :         int             max_pebs_events;
     736             :         void            (*drain_pebs)(struct pt_regs *regs, struct perf_sample_data *data);
     737             :         struct event_constraint *pebs_constraints;
     738             :         void            (*pebs_aliases)(struct perf_event *event);
     739             :         unsigned long   large_pebs_flags;
     740             :         u64             rtm_abort_event;
     741             : 
     742             :         /*
     743             :          * Intel LBR
     744             :          */
     745             :         unsigned int    lbr_tos, lbr_from, lbr_to,
     746             :                         lbr_info, lbr_nr;          /* LBR base regs and size */
     747             :         union {
     748             :                 u64     lbr_sel_mask;              /* LBR_SELECT valid bits */
     749             :                 u64     lbr_ctl_mask;              /* LBR_CTL valid bits */
     750             :         };
     751             :         union {
     752             :                 const int       *lbr_sel_map;      /* lbr_select mappings */
     753             :                 int             *lbr_ctl_map;      /* LBR_CTL mappings */
     754             :         };
     755             :         bool            lbr_double_abort;          /* duplicated lbr aborts */
     756             :         bool            lbr_pt_coexist;            /* (LBR|BTS) may coexist with PT */
     757             : 
     758             :         /*
     759             :          * Intel Architectural LBR CPUID Enumeration
     760             :          */
     761             :         unsigned int    lbr_depth_mask:8;
     762             :         unsigned int    lbr_deep_c_reset:1;
     763             :         unsigned int    lbr_lip:1;
     764             :         unsigned int    lbr_cpl:1;
     765             :         unsigned int    lbr_filter:1;
     766             :         unsigned int    lbr_call_stack:1;
     767             :         unsigned int    lbr_mispred:1;
     768             :         unsigned int    lbr_timed_lbr:1;
     769             :         unsigned int    lbr_br_type:1;
     770             : 
     771             :         void            (*lbr_reset)(void);
     772             :         void            (*lbr_read)(struct cpu_hw_events *cpuc);
     773             :         void            (*lbr_save)(void *ctx);
     774             :         void            (*lbr_restore)(void *ctx);
     775             : 
     776             :         /*
     777             :          * Intel PT/LBR/BTS are exclusive
     778             :          */
     779             :         atomic_t        lbr_exclusive[x86_lbr_exclusive_max];
     780             : 
     781             :         /*
     782             :          * Intel perf metrics
     783             :          */
     784             :         int             num_topdown_events;
     785             :         u64             (*update_topdown_event)(struct perf_event *event);
     786             :         int             (*set_topdown_event_period)(struct perf_event *event);
     787             : 
     788             :         /*
     789             :          * perf task context (i.e. struct perf_event_context::task_ctx_data)
     790             :          * switch helper to bridge calls from perf/core to perf/x86.
     791             :          * See struct pmu::swap_task_ctx() usage for examples;
     792             :          */
     793             :         void            (*swap_task_ctx)(struct perf_event_context *prev,
     794             :                                          struct perf_event_context *next);
     795             : 
     796             :         /*
     797             :          * AMD bits
     798             :          */
     799             :         unsigned int    amd_nb_constraints : 1;
     800             :         u64             perf_ctr_pair_en;
     801             : 
     802             :         /*
     803             :          * Extra registers for events
     804             :          */
     805             :         struct extra_reg *extra_regs;
     806             :         unsigned int flags;
     807             : 
     808             :         /*
     809             :          * Intel host/guest support (KVM)
     810             :          */
     811             :         struct perf_guest_switch_msr *(*guest_get_msrs)(int *nr);
     812             : 
     813             :         /*
     814             :          * Check period value for PERF_EVENT_IOC_PERIOD ioctl.
     815             :          */
     816             :         int (*check_period) (struct perf_event *event, u64 period);
     817             : 
     818             :         int (*aux_output_match) (struct perf_event *event);
     819             : };
     820             : 
     821             : struct x86_perf_task_context_opt {
     822             :         int lbr_callstack_users;
     823             :         int lbr_stack_state;
     824             :         int log_id;
     825             : };
     826             : 
     827             : struct x86_perf_task_context {
     828             :         u64 lbr_sel;
     829             :         int tos;
     830             :         int valid_lbrs;
     831             :         struct x86_perf_task_context_opt opt;
     832             :         struct lbr_entry lbr[MAX_LBR_ENTRIES];
     833             : };
     834             : 
     835             : struct x86_perf_task_context_arch_lbr {
     836             :         struct x86_perf_task_context_opt opt;
     837             :         struct lbr_entry entries[];
     838             : };
     839             : 
     840             : /*
     841             :  * Add padding to guarantee the 64-byte alignment of the state buffer.
     842             :  *
     843             :  * The structure is dynamically allocated. The size of the LBR state may vary
     844             :  * based on the number of LBR registers.
     845             :  *
     846             :  * Do not put anything after the LBR state.
     847             :  */
     848             : struct x86_perf_task_context_arch_lbr_xsave {
     849             :         struct x86_perf_task_context_opt                opt;
     850             : 
     851             :         union {
     852             :                 struct xregs_state                      xsave;
     853             :                 struct {
     854             :                         struct fxregs_state             i387;
     855             :                         struct xstate_header            header;
     856             :                         struct arch_lbr_state           lbr;
     857             :                 } __attribute__ ((packed, aligned (XSAVE_ALIGNMENT)));
     858             :         };
     859             : };
     860             : 
     861             : #define x86_add_quirk(func_)                                            \
     862             : do {                                                                    \
     863             :         static struct x86_pmu_quirk __quirk __initdata = {              \
     864             :                 .func = func_,                                          \
     865             :         };                                                              \
     866             :         __quirk.next = x86_pmu.quirks;                                  \
     867             :         x86_pmu.quirks = &__quirk;                                  \
     868             : } while (0)
     869             : 
     870             : /*
     871             :  * x86_pmu flags
     872             :  */
     873             : #define PMU_FL_NO_HT_SHARING    0x1 /* no hyper-threading resource sharing */
     874             : #define PMU_FL_HAS_RSP_1        0x2 /* has 2 equivalent offcore_rsp regs   */
     875             : #define PMU_FL_EXCL_CNTRS       0x4 /* has exclusive counter requirements  */
     876             : #define PMU_FL_EXCL_ENABLED     0x8 /* exclusive counter active */
     877             : #define PMU_FL_PEBS_ALL         0x10 /* all events are valid PEBS events */
     878             : #define PMU_FL_TFA              0x20 /* deal with TSX force abort */
     879             : #define PMU_FL_PAIR             0x40 /* merge counters for large incr. events */
     880             : #define PMU_FL_INSTR_LATENCY    0x80 /* Support Instruction Latency in PEBS Memory Info Record */
     881             : #define PMU_FL_MEM_LOADS_AUX    0x100 /* Require an auxiliary event for the complete memory info */
     882             : 
     883             : #define EVENT_VAR(_id)  event_attr_##_id
     884             : #define EVENT_PTR(_id) &event_attr_##_id.attr.attr
     885             : 
     886             : #define EVENT_ATTR(_name, _id)                                          \
     887             : static struct perf_pmu_events_attr EVENT_VAR(_id) = {                   \
     888             :         .attr           = __ATTR(_name, 0444, events_sysfs_show, NULL), \
     889             :         .id             = PERF_COUNT_HW_##_id,                          \
     890             :         .event_str      = NULL,                                         \
     891             : };
     892             : 
     893             : #define EVENT_ATTR_STR(_name, v, str)                                   \
     894             : static struct perf_pmu_events_attr event_attr_##v = {                   \
     895             :         .attr           = __ATTR(_name, 0444, events_sysfs_show, NULL), \
     896             :         .id             = 0,                                            \
     897             :         .event_str      = str,                                          \
     898             : };
     899             : 
     900             : #define EVENT_ATTR_STR_HT(_name, v, noht, ht)                           \
     901             : static struct perf_pmu_events_ht_attr event_attr_##v = {                \
     902             :         .attr           = __ATTR(_name, 0444, events_ht_sysfs_show, NULL),\
     903             :         .id             = 0,                                            \
     904             :         .event_str_noht = noht,                                         \
     905             :         .event_str_ht   = ht,                                           \
     906             : }
     907             : 
     908             : struct pmu *x86_get_pmu(void);
     909             : extern struct x86_pmu x86_pmu __read_mostly;
     910             : 
     911           0 : static __always_inline struct x86_perf_task_context_opt *task_context_opt(void *ctx)
     912             : {
     913           0 :         if (static_cpu_has(X86_FEATURE_ARCH_LBR))
     914           0 :                 return &((struct x86_perf_task_context_arch_lbr *)ctx)->opt;
     915             : 
     916           0 :         return &((struct x86_perf_task_context *)ctx)->opt;
     917             : }
     918             : 
     919           0 : static inline bool x86_pmu_has_lbr_callstack(void)
     920             : {
     921           0 :         return  x86_pmu.lbr_sel_map &&
     922           0 :                 x86_pmu.lbr_sel_map[PERF_SAMPLE_BRANCH_CALL_STACK_SHIFT] > 0;
     923             : }
     924             : 
     925             : DECLARE_PER_CPU(struct cpu_hw_events, cpu_hw_events);
     926             : 
     927             : int x86_perf_event_set_period(struct perf_event *event);
     928             : 
     929             : /*
     930             :  * Generalized hw caching related hw_event table, filled
     931             :  * in on a per model basis. A value of 0 means
     932             :  * 'not supported', -1 means 'hw_event makes no sense on
     933             :  * this CPU', any other value means the raw hw_event
     934             :  * ID.
     935             :  */
     936             : 
     937             : #define C(x) PERF_COUNT_HW_CACHE_##x
     938             : 
     939             : extern u64 __read_mostly hw_cache_event_ids
     940             :                                 [PERF_COUNT_HW_CACHE_MAX]
     941             :                                 [PERF_COUNT_HW_CACHE_OP_MAX]
     942             :                                 [PERF_COUNT_HW_CACHE_RESULT_MAX];
     943             : extern u64 __read_mostly hw_cache_extra_regs
     944             :                                 [PERF_COUNT_HW_CACHE_MAX]
     945             :                                 [PERF_COUNT_HW_CACHE_OP_MAX]
     946             :                                 [PERF_COUNT_HW_CACHE_RESULT_MAX];
     947             : 
     948             : u64 x86_perf_event_update(struct perf_event *event);
     949             : 
     950           0 : static inline unsigned int x86_pmu_config_addr(int index)
     951             : {
     952           0 :         return x86_pmu.eventsel + (x86_pmu.addr_offset ?
     953           0 :                                    x86_pmu.addr_offset(index, true) : index);
     954             : }
     955             : 
     956           0 : static inline unsigned int x86_pmu_event_addr(int index)
     957             : {
     958           0 :         return x86_pmu.perfctr + (x86_pmu.addr_offset ?
     959           0 :                                   x86_pmu.addr_offset(index, false) : index);
     960             : }
     961             : 
     962             : static inline int x86_pmu_rdpmc_index(int index)
     963             : {
     964             :         return x86_pmu.rdpmc_index ? x86_pmu.rdpmc_index(index) : index;
     965             : }
     966             : 
     967             : int x86_add_exclusive(unsigned int what);
     968             : 
     969             : void x86_del_exclusive(unsigned int what);
     970             : 
     971             : int x86_reserve_hardware(void);
     972             : 
     973             : void x86_release_hardware(void);
     974             : 
     975             : int x86_pmu_max_precise(void);
     976             : 
     977             : void hw_perf_lbr_event_destroy(struct perf_event *event);
     978             : 
     979             : int x86_setup_perfctr(struct perf_event *event);
     980             : 
     981             : int x86_pmu_hw_config(struct perf_event *event);
     982             : 
     983             : void x86_pmu_disable_all(void);
     984             : 
     985           0 : static inline bool is_counter_pair(struct hw_perf_event *hwc)
     986             : {
     987           0 :         return hwc->flags & PERF_X86_EVENT_PAIR;
     988             : }
     989             : 
     990           0 : static inline void __x86_pmu_enable_event(struct hw_perf_event *hwc,
     991             :                                           u64 enable_mask)
     992             : {
     993           0 :         u64 disable_mask = __this_cpu_read(cpu_hw_events.perf_ctr_virt_mask);
     994             : 
     995           0 :         if (hwc->extra_reg.reg)
     996           0 :                 wrmsrl(hwc->extra_reg.reg, hwc->extra_reg.config);
     997             : 
     998             :         /*
     999             :          * Add enabled Merge event on next counter
    1000             :          * if large increment event being enabled on this counter
    1001             :          */
    1002           0 :         if (is_counter_pair(hwc))
    1003           0 :                 wrmsrl(x86_pmu_config_addr(hwc->idx + 1), x86_pmu.perf_ctr_pair_en);
    1004             : 
    1005           0 :         wrmsrl(hwc->config_base, (hwc->config | enable_mask) & ~disable_mask);
    1006           0 : }
    1007             : 
    1008             : void x86_pmu_enable_all(int added);
    1009             : 
    1010             : int perf_assign_events(struct event_constraint **constraints, int n,
    1011             :                         int wmin, int wmax, int gpmax, int *assign);
    1012             : int x86_schedule_events(struct cpu_hw_events *cpuc, int n, int *assign);
    1013             : 
    1014             : void x86_pmu_stop(struct perf_event *event, int flags);
    1015             : 
    1016           0 : static inline void x86_pmu_disable_event(struct perf_event *event)
    1017             : {
    1018           0 :         struct hw_perf_event *hwc = &event->hw;
    1019             : 
    1020           0 :         wrmsrl(hwc->config_base, hwc->config);
    1021             : 
    1022           0 :         if (is_counter_pair(hwc))
    1023           0 :                 wrmsrl(x86_pmu_config_addr(hwc->idx + 1), 0);
    1024           0 : }
    1025             : 
    1026             : void x86_pmu_enable_event(struct perf_event *event);
    1027             : 
    1028             : int x86_pmu_handle_irq(struct pt_regs *regs);
    1029             : 
    1030             : extern struct event_constraint emptyconstraint;
    1031             : 
    1032             : extern struct event_constraint unconstrained;
    1033             : 
    1034           0 : static inline bool kernel_ip(unsigned long ip)
    1035             : {
    1036             : #ifdef CONFIG_X86_32
    1037             :         return ip > PAGE_OFFSET;
    1038             : #else
    1039           0 :         return (long)ip < 0;
    1040             : #endif
    1041             : }
    1042             : 
    1043             : /*
    1044             :  * Not all PMUs provide the right context information to place the reported IP
    1045             :  * into full context. Specifically segment registers are typically not
    1046             :  * supplied.
    1047             :  *
    1048             :  * Assuming the address is a linear address (it is for IBS), we fake the CS and
    1049             :  * vm86 mode using the known zero-based code segment and 'fix up' the registers
    1050             :  * to reflect this.
    1051             :  *
    1052             :  * Intel PEBS/LBR appear to typically provide the effective address, nothing
    1053             :  * much we can do about that but pray and treat it like a linear address.
    1054             :  */
    1055           0 : static inline void set_linear_ip(struct pt_regs *regs, unsigned long ip)
    1056             : {
    1057           0 :         regs->cs = kernel_ip(ip) ? __KERNEL_CS : __USER_CS;
    1058           0 :         if (regs->flags & X86_VM_MASK)
    1059             :                 regs->flags ^= (PERF_EFLAGS_VM | X86_VM_MASK);
    1060           0 :         regs->ip = ip;
    1061           0 : }
    1062             : 
    1063             : ssize_t x86_event_sysfs_show(char *page, u64 config, u64 event);
    1064             : ssize_t intel_event_sysfs_show(char *page, u64 config);
    1065             : 
    1066             : ssize_t events_sysfs_show(struct device *dev, struct device_attribute *attr,
    1067             :                           char *page);
    1068             : ssize_t events_ht_sysfs_show(struct device *dev, struct device_attribute *attr,
    1069             :                           char *page);
    1070             : 
    1071           0 : static inline bool fixed_counter_disabled(int i)
    1072             : {
    1073           0 :         return !(x86_pmu.intel_ctrl >> (i + INTEL_PMC_IDX_FIXED));
    1074             : }
    1075             : 
    1076             : #ifdef CONFIG_CPU_SUP_AMD
    1077             : 
    1078             : int amd_pmu_init(void);
    1079             : 
    1080             : #else /* CONFIG_CPU_SUP_AMD */
    1081             : 
    1082             : static inline int amd_pmu_init(void)
    1083             : {
    1084             :         return 0;
    1085             : }
    1086             : 
    1087             : #endif /* CONFIG_CPU_SUP_AMD */
    1088             : 
    1089           0 : static inline int is_pebs_pt(struct perf_event *event)
    1090             : {
    1091           0 :         return !!(event->hw.flags & PERF_X86_EVENT_PEBS_VIA_PT);
    1092             : }
    1093             : 
    1094             : #ifdef CONFIG_CPU_SUP_INTEL
    1095             : 
    1096           0 : static inline bool intel_pmu_has_bts_period(struct perf_event *event, u64 period)
    1097             : {
    1098           0 :         struct hw_perf_event *hwc = &event->hw;
    1099           0 :         unsigned int hw_event, bts_event;
    1100             : 
    1101           0 :         if (event->attr.freq)
    1102             :                 return false;
    1103             : 
    1104           0 :         hw_event = hwc->config & INTEL_ARCH_EVENT_MASK;
    1105           0 :         bts_event = x86_pmu.event_map(PERF_COUNT_HW_BRANCH_INSTRUCTIONS);
    1106             : 
    1107           0 :         return hw_event == bts_event && period == 1;
    1108             : }
    1109             : 
    1110           0 : static inline bool intel_pmu_has_bts(struct perf_event *event)
    1111             : {
    1112           0 :         struct hw_perf_event *hwc = &event->hw;
    1113             : 
    1114           0 :         return intel_pmu_has_bts_period(event, hwc->sample_period);
    1115             : }
    1116             : 
    1117             : int intel_pmu_save_and_restart(struct perf_event *event);
    1118             : 
    1119             : struct event_constraint *
    1120             : x86_get_event_constraints(struct cpu_hw_events *cpuc, int idx,
    1121             :                           struct perf_event *event);
    1122             : 
    1123             : extern int intel_cpuc_prepare(struct cpu_hw_events *cpuc, int cpu);
    1124             : extern void intel_cpuc_finish(struct cpu_hw_events *cpuc);
    1125             : 
    1126             : int intel_pmu_init(void);
    1127             : 
    1128             : void init_debug_store_on_cpu(int cpu);
    1129             : 
    1130             : void fini_debug_store_on_cpu(int cpu);
    1131             : 
    1132             : void release_ds_buffers(void);
    1133             : 
    1134             : void reserve_ds_buffers(void);
    1135             : 
    1136             : void release_lbr_buffers(void);
    1137             : 
    1138             : extern struct event_constraint bts_constraint;
    1139             : extern struct event_constraint vlbr_constraint;
    1140             : 
    1141             : void intel_pmu_enable_bts(u64 config);
    1142             : 
    1143             : void intel_pmu_disable_bts(void);
    1144             : 
    1145             : int intel_pmu_drain_bts_buffer(void);
    1146             : 
    1147             : extern struct event_constraint intel_core2_pebs_event_constraints[];
    1148             : 
    1149             : extern struct event_constraint intel_atom_pebs_event_constraints[];
    1150             : 
    1151             : extern struct event_constraint intel_slm_pebs_event_constraints[];
    1152             : 
    1153             : extern struct event_constraint intel_glm_pebs_event_constraints[];
    1154             : 
    1155             : extern struct event_constraint intel_glp_pebs_event_constraints[];
    1156             : 
    1157             : extern struct event_constraint intel_nehalem_pebs_event_constraints[];
    1158             : 
    1159             : extern struct event_constraint intel_westmere_pebs_event_constraints[];
    1160             : 
    1161             : extern struct event_constraint intel_snb_pebs_event_constraints[];
    1162             : 
    1163             : extern struct event_constraint intel_ivb_pebs_event_constraints[];
    1164             : 
    1165             : extern struct event_constraint intel_hsw_pebs_event_constraints[];
    1166             : 
    1167             : extern struct event_constraint intel_bdw_pebs_event_constraints[];
    1168             : 
    1169             : extern struct event_constraint intel_skl_pebs_event_constraints[];
    1170             : 
    1171             : extern struct event_constraint intel_icl_pebs_event_constraints[];
    1172             : 
    1173             : extern struct event_constraint intel_spr_pebs_event_constraints[];
    1174             : 
    1175             : struct event_constraint *intel_pebs_constraints(struct perf_event *event);
    1176             : 
    1177             : void intel_pmu_pebs_add(struct perf_event *event);
    1178             : 
    1179             : void intel_pmu_pebs_del(struct perf_event *event);
    1180             : 
    1181             : void intel_pmu_pebs_enable(struct perf_event *event);
    1182             : 
    1183             : void intel_pmu_pebs_disable(struct perf_event *event);
    1184             : 
    1185             : void intel_pmu_pebs_enable_all(void);
    1186             : 
    1187             : void intel_pmu_pebs_disable_all(void);
    1188             : 
    1189             : void intel_pmu_pebs_sched_task(struct perf_event_context *ctx, bool sched_in);
    1190             : 
    1191             : void intel_pmu_auto_reload_read(struct perf_event *event);
    1192             : 
    1193             : void intel_pmu_store_pebs_lbrs(struct lbr_entry *lbr);
    1194             : 
    1195             : void intel_ds_init(void);
    1196             : 
    1197             : void intel_pmu_lbr_swap_task_ctx(struct perf_event_context *prev,
    1198             :                                  struct perf_event_context *next);
    1199             : 
    1200             : void intel_pmu_lbr_sched_task(struct perf_event_context *ctx, bool sched_in);
    1201             : 
    1202             : u64 lbr_from_signext_quirk_wr(u64 val);
    1203             : 
    1204             : void intel_pmu_lbr_reset(void);
    1205             : 
    1206             : void intel_pmu_lbr_reset_32(void);
    1207             : 
    1208             : void intel_pmu_lbr_reset_64(void);
    1209             : 
    1210             : void intel_pmu_lbr_add(struct perf_event *event);
    1211             : 
    1212             : void intel_pmu_lbr_del(struct perf_event *event);
    1213             : 
    1214             : void intel_pmu_lbr_enable_all(bool pmi);
    1215             : 
    1216             : void intel_pmu_lbr_disable_all(void);
    1217             : 
    1218             : void intel_pmu_lbr_read(void);
    1219             : 
    1220             : void intel_pmu_lbr_read_32(struct cpu_hw_events *cpuc);
    1221             : 
    1222             : void intel_pmu_lbr_read_64(struct cpu_hw_events *cpuc);
    1223             : 
    1224             : void intel_pmu_lbr_save(void *ctx);
    1225             : 
    1226             : void intel_pmu_lbr_restore(void *ctx);
    1227             : 
    1228             : void intel_pmu_lbr_init_core(void);
    1229             : 
    1230             : void intel_pmu_lbr_init_nhm(void);
    1231             : 
    1232             : void intel_pmu_lbr_init_atom(void);
    1233             : 
    1234             : void intel_pmu_lbr_init_slm(void);
    1235             : 
    1236             : void intel_pmu_lbr_init_snb(void);
    1237             : 
    1238             : void intel_pmu_lbr_init_hsw(void);
    1239             : 
    1240             : void intel_pmu_lbr_init_skl(void);
    1241             : 
    1242             : void intel_pmu_lbr_init_knl(void);
    1243             : 
    1244             : void intel_pmu_arch_lbr_init(void);
    1245             : 
    1246             : void intel_pmu_pebs_data_source_nhm(void);
    1247             : 
    1248             : void intel_pmu_pebs_data_source_skl(bool pmem);
    1249             : 
    1250             : int intel_pmu_setup_lbr_filter(struct perf_event *event);
    1251             : 
    1252             : void intel_pt_interrupt(void);
    1253             : 
    1254             : int intel_bts_interrupt(void);
    1255             : 
    1256             : void intel_bts_enable_local(void);
    1257             : 
    1258             : void intel_bts_disable_local(void);
    1259             : 
    1260             : int p4_pmu_init(void);
    1261             : 
    1262             : int p6_pmu_init(void);
    1263             : 
    1264             : int knc_pmu_init(void);
    1265             : 
    1266           0 : static inline int is_ht_workaround_enabled(void)
    1267             : {
    1268           0 :         return !!(x86_pmu.flags & PMU_FL_EXCL_ENABLED);
    1269             : }
    1270             : 
    1271             : #else /* CONFIG_CPU_SUP_INTEL */
    1272             : 
    1273             : static inline void reserve_ds_buffers(void)
    1274             : {
    1275             : }
    1276             : 
    1277             : static inline void release_ds_buffers(void)
    1278             : {
    1279             : }
    1280             : 
    1281             : static inline void release_lbr_buffers(void)
    1282             : {
    1283             : }
    1284             : 
    1285             : static inline int intel_pmu_init(void)
    1286             : {
    1287             :         return 0;
    1288             : }
    1289             : 
    1290             : static inline int intel_cpuc_prepare(struct cpu_hw_events *cpuc, int cpu)
    1291             : {
    1292             :         return 0;
    1293             : }
    1294             : 
    1295             : static inline void intel_cpuc_finish(struct cpu_hw_events *cpuc)
    1296             : {
    1297             : }
    1298             : 
    1299             : static inline int is_ht_workaround_enabled(void)
    1300             : {
    1301             :         return 0;
    1302             : }
    1303             : #endif /* CONFIG_CPU_SUP_INTEL */
    1304             : 
    1305             : #if ((defined CONFIG_CPU_SUP_CENTAUR) || (defined CONFIG_CPU_SUP_ZHAOXIN))
    1306             : int zhaoxin_pmu_init(void);
    1307             : #else
    1308             : static inline int zhaoxin_pmu_init(void)
    1309             : {
    1310             :         return 0;
    1311             : }
    1312             : #endif /*CONFIG_CPU_SUP_CENTAUR or CONFIG_CPU_SUP_ZHAOXIN*/

Generated by: LCOV version 1.14