Line data Source code
1 : /*
2 : * Performance events x86 architecture header
3 : *
4 : * Copyright (C) 2008 Thomas Gleixner <tglx@linutronix.de>
5 : * Copyright (C) 2008-2009 Red Hat, Inc., Ingo Molnar
6 : * Copyright (C) 2009 Jaswinder Singh Rajput
7 : * Copyright (C) 2009 Advanced Micro Devices, Inc., Robert Richter
8 : * Copyright (C) 2008-2009 Red Hat, Inc., Peter Zijlstra
9 : * Copyright (C) 2009 Intel Corporation, <markus.t.metzger@intel.com>
10 : * Copyright (C) 2009 Google, Inc., Stephane Eranian
11 : *
12 : * For licencing details see kernel-base/COPYING
13 : */
14 :
15 : #include <linux/perf_event.h>
16 :
17 : #include <asm/intel_ds.h>
18 :
19 : /* To enable MSR tracing please use the generic trace points. */
20 :
21 : /*
22 : * | NHM/WSM | SNB |
23 : * register -------------------------------
24 : * | HT | no HT | HT | no HT |
25 : *-----------------------------------------
26 : * offcore | core | core | cpu | core |
27 : * lbr_sel | core | core | cpu | core |
28 : * ld_lat | cpu | core | cpu | core |
29 : *-----------------------------------------
30 : *
31 : * Given that there is a small number of shared regs,
32 : * we can pre-allocate their slot in the per-cpu
33 : * per-core reg tables.
34 : */
35 : enum extra_reg_type {
36 : EXTRA_REG_NONE = -1, /* not used */
37 :
38 : EXTRA_REG_RSP_0 = 0, /* offcore_response_0 */
39 : EXTRA_REG_RSP_1 = 1, /* offcore_response_1 */
40 : EXTRA_REG_LBR = 2, /* lbr_select */
41 : EXTRA_REG_LDLAT = 3, /* ld_lat_threshold */
42 : EXTRA_REG_FE = 4, /* fe_* */
43 :
44 : EXTRA_REG_MAX /* number of entries needed */
45 : };
46 :
47 : struct event_constraint {
48 : union {
49 : unsigned long idxmsk[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
50 : u64 idxmsk64;
51 : };
52 : u64 code;
53 : u64 cmask;
54 : int weight;
55 : int overlap;
56 : int flags;
57 : unsigned int size;
58 : };
59 :
60 0 : static inline bool constraint_match(struct event_constraint *c, u64 ecode)
61 : {
62 0 : return ((ecode & c->cmask) - c->code) <= (u64)c->size;
63 : }
64 :
65 : /*
66 : * struct hw_perf_event.flags flags
67 : */
68 : #define PERF_X86_EVENT_PEBS_LDLAT 0x0001 /* ld+ldlat data address sampling */
69 : #define PERF_X86_EVENT_PEBS_ST 0x0002 /* st data address sampling */
70 : #define PERF_X86_EVENT_PEBS_ST_HSW 0x0004 /* haswell style datala, store */
71 : #define PERF_X86_EVENT_PEBS_LD_HSW 0x0008 /* haswell style datala, load */
72 : #define PERF_X86_EVENT_PEBS_NA_HSW 0x0010 /* haswell style datala, unknown */
73 : #define PERF_X86_EVENT_EXCL 0x0020 /* HT exclusivity on counter */
74 : #define PERF_X86_EVENT_DYNAMIC 0x0040 /* dynamic alloc'd constraint */
75 : #define PERF_X86_EVENT_RDPMC_ALLOWED 0x0080 /* grant rdpmc permission */
76 : #define PERF_X86_EVENT_EXCL_ACCT 0x0100 /* accounted EXCL event */
77 : #define PERF_X86_EVENT_AUTO_RELOAD 0x0200 /* use PEBS auto-reload */
78 : #define PERF_X86_EVENT_LARGE_PEBS 0x0400 /* use large PEBS */
79 : #define PERF_X86_EVENT_PEBS_VIA_PT 0x0800 /* use PT buffer for PEBS */
80 : #define PERF_X86_EVENT_PAIR 0x1000 /* Large Increment per Cycle */
81 : #define PERF_X86_EVENT_LBR_SELECT 0x2000 /* Save/Restore MSR_LBR_SELECT */
82 : #define PERF_X86_EVENT_TOPDOWN 0x4000 /* Count Topdown slots/metrics events */
83 : #define PERF_X86_EVENT_PEBS_STLAT 0x8000 /* st+stlat data address sampling */
84 :
85 0 : static inline bool is_topdown_count(struct perf_event *event)
86 : {
87 0 : return event->hw.flags & PERF_X86_EVENT_TOPDOWN;
88 : }
89 :
90 0 : static inline bool is_metric_event(struct perf_event *event)
91 : {
92 0 : u64 config = event->attr.config;
93 :
94 0 : return ((config & ARCH_PERFMON_EVENTSEL_EVENT) == 0) &&
95 0 : ((config & INTEL_ARCH_EVENT_MASK) >= INTEL_TD_METRIC_RETIRING) &&
96 0 : ((config & INTEL_ARCH_EVENT_MASK) <= INTEL_TD_METRIC_MAX);
97 : }
98 :
99 0 : static inline bool is_slots_event(struct perf_event *event)
100 : {
101 0 : return (event->attr.config & INTEL_ARCH_EVENT_MASK) == INTEL_TD_SLOTS;
102 : }
103 :
104 0 : static inline bool is_topdown_event(struct perf_event *event)
105 : {
106 0 : return is_metric_event(event) || is_slots_event(event);
107 : }
108 :
109 : struct amd_nb {
110 : int nb_id; /* NorthBridge id */
111 : int refcnt; /* reference count */
112 : struct perf_event *owners[X86_PMC_IDX_MAX];
113 : struct event_constraint event_constraints[X86_PMC_IDX_MAX];
114 : };
115 :
116 : #define PEBS_COUNTER_MASK ((1ULL << MAX_PEBS_EVENTS) - 1)
117 : #define PEBS_PMI_AFTER_EACH_RECORD BIT_ULL(60)
118 : #define PEBS_OUTPUT_OFFSET 61
119 : #define PEBS_OUTPUT_MASK (3ull << PEBS_OUTPUT_OFFSET)
120 : #define PEBS_OUTPUT_PT (1ull << PEBS_OUTPUT_OFFSET)
121 : #define PEBS_VIA_PT_MASK (PEBS_OUTPUT_PT | PEBS_PMI_AFTER_EACH_RECORD)
122 :
123 : /*
124 : * Flags PEBS can handle without an PMI.
125 : *
126 : * TID can only be handled by flushing at context switch.
127 : * REGS_USER can be handled for events limited to ring 3.
128 : *
129 : */
130 : #define LARGE_PEBS_FLAGS \
131 : (PERF_SAMPLE_IP | PERF_SAMPLE_TID | PERF_SAMPLE_ADDR | \
132 : PERF_SAMPLE_ID | PERF_SAMPLE_CPU | PERF_SAMPLE_STREAM_ID | \
133 : PERF_SAMPLE_DATA_SRC | PERF_SAMPLE_IDENTIFIER | \
134 : PERF_SAMPLE_TRANSACTION | PERF_SAMPLE_PHYS_ADDR | \
135 : PERF_SAMPLE_REGS_INTR | PERF_SAMPLE_REGS_USER | \
136 : PERF_SAMPLE_PERIOD | PERF_SAMPLE_CODE_PAGE_SIZE)
137 :
138 : #define PEBS_GP_REGS \
139 : ((1ULL << PERF_REG_X86_AX) | \
140 : (1ULL << PERF_REG_X86_BX) | \
141 : (1ULL << PERF_REG_X86_CX) | \
142 : (1ULL << PERF_REG_X86_DX) | \
143 : (1ULL << PERF_REG_X86_DI) | \
144 : (1ULL << PERF_REG_X86_SI) | \
145 : (1ULL << PERF_REG_X86_SP) | \
146 : (1ULL << PERF_REG_X86_BP) | \
147 : (1ULL << PERF_REG_X86_IP) | \
148 : (1ULL << PERF_REG_X86_FLAGS) | \
149 : (1ULL << PERF_REG_X86_R8) | \
150 : (1ULL << PERF_REG_X86_R9) | \
151 : (1ULL << PERF_REG_X86_R10) | \
152 : (1ULL << PERF_REG_X86_R11) | \
153 : (1ULL << PERF_REG_X86_R12) | \
154 : (1ULL << PERF_REG_X86_R13) | \
155 : (1ULL << PERF_REG_X86_R14) | \
156 : (1ULL << PERF_REG_X86_R15))
157 :
158 : /*
159 : * Per register state.
160 : */
161 : struct er_account {
162 : raw_spinlock_t lock; /* per-core: protect structure */
163 : u64 config; /* extra MSR config */
164 : u64 reg; /* extra MSR number */
165 : atomic_t ref; /* reference count */
166 : };
167 :
168 : /*
169 : * Per core/cpu state
170 : *
171 : * Used to coordinate shared registers between HT threads or
172 : * among events on a single PMU.
173 : */
174 : struct intel_shared_regs {
175 : struct er_account regs[EXTRA_REG_MAX];
176 : int refcnt; /* per-core: #HT threads */
177 : unsigned core_id; /* per-core: core id */
178 : };
179 :
180 : enum intel_excl_state_type {
181 : INTEL_EXCL_UNUSED = 0, /* counter is unused */
182 : INTEL_EXCL_SHARED = 1, /* counter can be used by both threads */
183 : INTEL_EXCL_EXCLUSIVE = 2, /* counter can be used by one thread only */
184 : };
185 :
186 : struct intel_excl_states {
187 : enum intel_excl_state_type state[X86_PMC_IDX_MAX];
188 : bool sched_started; /* true if scheduling has started */
189 : };
190 :
191 : struct intel_excl_cntrs {
192 : raw_spinlock_t lock;
193 :
194 : struct intel_excl_states states[2];
195 :
196 : union {
197 : u16 has_exclusive[2];
198 : u32 exclusive_present;
199 : };
200 :
201 : int refcnt; /* per-core: #HT threads */
202 : unsigned core_id; /* per-core: core id */
203 : };
204 :
205 : struct x86_perf_task_context;
206 : #define MAX_LBR_ENTRIES 32
207 :
208 : enum {
209 : LBR_FORMAT_32 = 0x00,
210 : LBR_FORMAT_LIP = 0x01,
211 : LBR_FORMAT_EIP = 0x02,
212 : LBR_FORMAT_EIP_FLAGS = 0x03,
213 : LBR_FORMAT_EIP_FLAGS2 = 0x04,
214 : LBR_FORMAT_INFO = 0x05,
215 : LBR_FORMAT_TIME = 0x06,
216 : LBR_FORMAT_MAX_KNOWN = LBR_FORMAT_TIME,
217 : };
218 :
219 : enum {
220 : X86_PERF_KFREE_SHARED = 0,
221 : X86_PERF_KFREE_EXCL = 1,
222 : X86_PERF_KFREE_MAX
223 : };
224 :
225 : struct cpu_hw_events {
226 : /*
227 : * Generic x86 PMC bits
228 : */
229 : struct perf_event *events[X86_PMC_IDX_MAX]; /* in counter order */
230 : unsigned long active_mask[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
231 : unsigned long running[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
232 : int enabled;
233 :
234 : int n_events; /* the # of events in the below arrays */
235 : int n_added; /* the # last events in the below arrays;
236 : they've never been enabled yet */
237 : int n_txn; /* the # last events in the below arrays;
238 : added in the current transaction */
239 : int n_txn_pair;
240 : int n_txn_metric;
241 : int assign[X86_PMC_IDX_MAX]; /* event to counter assignment */
242 : u64 tags[X86_PMC_IDX_MAX];
243 :
244 : struct perf_event *event_list[X86_PMC_IDX_MAX]; /* in enabled order */
245 : struct event_constraint *event_constraint[X86_PMC_IDX_MAX];
246 :
247 : int n_excl; /* the number of exclusive events */
248 :
249 : unsigned int txn_flags;
250 : int is_fake;
251 :
252 : /*
253 : * Intel DebugStore bits
254 : */
255 : struct debug_store *ds;
256 : void *ds_pebs_vaddr;
257 : void *ds_bts_vaddr;
258 : u64 pebs_enabled;
259 : int n_pebs;
260 : int n_large_pebs;
261 : int n_pebs_via_pt;
262 : int pebs_output;
263 :
264 : /* Current super set of events hardware configuration */
265 : u64 pebs_data_cfg;
266 : u64 active_pebs_data_cfg;
267 : int pebs_record_size;
268 :
269 : /*
270 : * Intel LBR bits
271 : */
272 : int lbr_users;
273 : int lbr_pebs_users;
274 : struct perf_branch_stack lbr_stack;
275 : struct perf_branch_entry lbr_entries[MAX_LBR_ENTRIES];
276 : union {
277 : struct er_account *lbr_sel;
278 : struct er_account *lbr_ctl;
279 : };
280 : u64 br_sel;
281 : void *last_task_ctx;
282 : int last_log_id;
283 : int lbr_select;
284 : void *lbr_xsave;
285 :
286 : /*
287 : * Intel host/guest exclude bits
288 : */
289 : u64 intel_ctrl_guest_mask;
290 : u64 intel_ctrl_host_mask;
291 : struct perf_guest_switch_msr guest_switch_msrs[X86_PMC_IDX_MAX];
292 :
293 : /*
294 : * Intel checkpoint mask
295 : */
296 : u64 intel_cp_status;
297 :
298 : /*
299 : * manage shared (per-core, per-cpu) registers
300 : * used on Intel NHM/WSM/SNB
301 : */
302 : struct intel_shared_regs *shared_regs;
303 : /*
304 : * manage exclusive counter access between hyperthread
305 : */
306 : struct event_constraint *constraint_list; /* in enable order */
307 : struct intel_excl_cntrs *excl_cntrs;
308 : int excl_thread_id; /* 0 or 1 */
309 :
310 : /*
311 : * SKL TSX_FORCE_ABORT shadow
312 : */
313 : u64 tfa_shadow;
314 :
315 : /*
316 : * Perf Metrics
317 : */
318 : /* number of accepted metrics events */
319 : int n_metric;
320 :
321 : /*
322 : * AMD specific bits
323 : */
324 : struct amd_nb *amd_nb;
325 : /* Inverted mask of bits to clear in the perf_ctr ctrl registers */
326 : u64 perf_ctr_virt_mask;
327 : int n_pair; /* Large increment events */
328 :
329 : void *kfree_on_online[X86_PERF_KFREE_MAX];
330 : };
331 :
332 : #define __EVENT_CONSTRAINT_RANGE(c, e, n, m, w, o, f) { \
333 : { .idxmsk64 = (n) }, \
334 : .code = (c), \
335 : .size = (e) - (c), \
336 : .cmask = (m), \
337 : .weight = (w), \
338 : .overlap = (o), \
339 : .flags = f, \
340 : }
341 :
342 : #define __EVENT_CONSTRAINT(c, n, m, w, o, f) \
343 : __EVENT_CONSTRAINT_RANGE(c, c, n, m, w, o, f)
344 :
345 : #define EVENT_CONSTRAINT(c, n, m) \
346 : __EVENT_CONSTRAINT(c, n, m, HWEIGHT(n), 0, 0)
347 :
348 : /*
349 : * The constraint_match() function only works for 'simple' event codes
350 : * and not for extended (AMD64_EVENTSEL_EVENT) events codes.
351 : */
352 : #define EVENT_CONSTRAINT_RANGE(c, e, n, m) \
353 : __EVENT_CONSTRAINT_RANGE(c, e, n, m, HWEIGHT(n), 0, 0)
354 :
355 : #define INTEL_EXCLEVT_CONSTRAINT(c, n) \
356 : __EVENT_CONSTRAINT(c, n, ARCH_PERFMON_EVENTSEL_EVENT, HWEIGHT(n),\
357 : 0, PERF_X86_EVENT_EXCL)
358 :
359 : /*
360 : * The overlap flag marks event constraints with overlapping counter
361 : * masks. This is the case if the counter mask of such an event is not
362 : * a subset of any other counter mask of a constraint with an equal or
363 : * higher weight, e.g.:
364 : *
365 : * c_overlaps = EVENT_CONSTRAINT_OVERLAP(0, 0x09, 0);
366 : * c_another1 = EVENT_CONSTRAINT(0, 0x07, 0);
367 : * c_another2 = EVENT_CONSTRAINT(0, 0x38, 0);
368 : *
369 : * The event scheduler may not select the correct counter in the first
370 : * cycle because it needs to know which subsequent events will be
371 : * scheduled. It may fail to schedule the events then. So we set the
372 : * overlap flag for such constraints to give the scheduler a hint which
373 : * events to select for counter rescheduling.
374 : *
375 : * Care must be taken as the rescheduling algorithm is O(n!) which
376 : * will increase scheduling cycles for an over-committed system
377 : * dramatically. The number of such EVENT_CONSTRAINT_OVERLAP() macros
378 : * and its counter masks must be kept at a minimum.
379 : */
380 : #define EVENT_CONSTRAINT_OVERLAP(c, n, m) \
381 : __EVENT_CONSTRAINT(c, n, m, HWEIGHT(n), 1, 0)
382 :
383 : /*
384 : * Constraint on the Event code.
385 : */
386 : #define INTEL_EVENT_CONSTRAINT(c, n) \
387 : EVENT_CONSTRAINT(c, n, ARCH_PERFMON_EVENTSEL_EVENT)
388 :
389 : /*
390 : * Constraint on a range of Event codes
391 : */
392 : #define INTEL_EVENT_CONSTRAINT_RANGE(c, e, n) \
393 : EVENT_CONSTRAINT_RANGE(c, e, n, ARCH_PERFMON_EVENTSEL_EVENT)
394 :
395 : /*
396 : * Constraint on the Event code + UMask + fixed-mask
397 : *
398 : * filter mask to validate fixed counter events.
399 : * the following filters disqualify for fixed counters:
400 : * - inv
401 : * - edge
402 : * - cnt-mask
403 : * - in_tx
404 : * - in_tx_checkpointed
405 : * The other filters are supported by fixed counters.
406 : * The any-thread option is supported starting with v3.
407 : */
408 : #define FIXED_EVENT_FLAGS (X86_RAW_EVENT_MASK|HSW_IN_TX|HSW_IN_TX_CHECKPOINTED)
409 : #define FIXED_EVENT_CONSTRAINT(c, n) \
410 : EVENT_CONSTRAINT(c, (1ULL << (32+n)), FIXED_EVENT_FLAGS)
411 :
412 : /*
413 : * The special metric counters do not actually exist. They are calculated from
414 : * the combination of the FxCtr3 + MSR_PERF_METRICS.
415 : *
416 : * The special metric counters are mapped to a dummy offset for the scheduler.
417 : * The sharing between multiple users of the same metric without multiplexing
418 : * is not allowed, even though the hardware supports that in principle.
419 : */
420 :
421 : #define METRIC_EVENT_CONSTRAINT(c, n) \
422 : EVENT_CONSTRAINT(c, (1ULL << (INTEL_PMC_IDX_METRIC_BASE + n)), \
423 : INTEL_ARCH_EVENT_MASK)
424 :
425 : /*
426 : * Constraint on the Event code + UMask
427 : */
428 : #define INTEL_UEVENT_CONSTRAINT(c, n) \
429 : EVENT_CONSTRAINT(c, n, INTEL_ARCH_EVENT_MASK)
430 :
431 : /* Constraint on specific umask bit only + event */
432 : #define INTEL_UBIT_EVENT_CONSTRAINT(c, n) \
433 : EVENT_CONSTRAINT(c, n, ARCH_PERFMON_EVENTSEL_EVENT|(c))
434 :
435 : /* Like UEVENT_CONSTRAINT, but match flags too */
436 : #define INTEL_FLAGS_UEVENT_CONSTRAINT(c, n) \
437 : EVENT_CONSTRAINT(c, n, INTEL_ARCH_EVENT_MASK|X86_ALL_EVENT_FLAGS)
438 :
439 : #define INTEL_EXCLUEVT_CONSTRAINT(c, n) \
440 : __EVENT_CONSTRAINT(c, n, INTEL_ARCH_EVENT_MASK, \
441 : HWEIGHT(n), 0, PERF_X86_EVENT_EXCL)
442 :
443 : #define INTEL_PLD_CONSTRAINT(c, n) \
444 : __EVENT_CONSTRAINT(c, n, INTEL_ARCH_EVENT_MASK|X86_ALL_EVENT_FLAGS, \
445 : HWEIGHT(n), 0, PERF_X86_EVENT_PEBS_LDLAT)
446 :
447 : #define INTEL_PSD_CONSTRAINT(c, n) \
448 : __EVENT_CONSTRAINT(c, n, INTEL_ARCH_EVENT_MASK|X86_ALL_EVENT_FLAGS, \
449 : HWEIGHT(n), 0, PERF_X86_EVENT_PEBS_STLAT)
450 :
451 : #define INTEL_PST_CONSTRAINT(c, n) \
452 : __EVENT_CONSTRAINT(c, n, INTEL_ARCH_EVENT_MASK|X86_ALL_EVENT_FLAGS, \
453 : HWEIGHT(n), 0, PERF_X86_EVENT_PEBS_ST)
454 :
455 : /* Event constraint, but match on all event flags too. */
456 : #define INTEL_FLAGS_EVENT_CONSTRAINT(c, n) \
457 : EVENT_CONSTRAINT(c, n, ARCH_PERFMON_EVENTSEL_EVENT|X86_ALL_EVENT_FLAGS)
458 :
459 : #define INTEL_FLAGS_EVENT_CONSTRAINT_RANGE(c, e, n) \
460 : EVENT_CONSTRAINT_RANGE(c, e, n, ARCH_PERFMON_EVENTSEL_EVENT|X86_ALL_EVENT_FLAGS)
461 :
462 : /* Check only flags, but allow all event/umask */
463 : #define INTEL_ALL_EVENT_CONSTRAINT(code, n) \
464 : EVENT_CONSTRAINT(code, n, X86_ALL_EVENT_FLAGS)
465 :
466 : /* Check flags and event code, and set the HSW store flag */
467 : #define INTEL_FLAGS_EVENT_CONSTRAINT_DATALA_ST(code, n) \
468 : __EVENT_CONSTRAINT(code, n, \
469 : ARCH_PERFMON_EVENTSEL_EVENT|X86_ALL_EVENT_FLAGS, \
470 : HWEIGHT(n), 0, PERF_X86_EVENT_PEBS_ST_HSW)
471 :
472 : /* Check flags and event code, and set the HSW load flag */
473 : #define INTEL_FLAGS_EVENT_CONSTRAINT_DATALA_LD(code, n) \
474 : __EVENT_CONSTRAINT(code, n, \
475 : ARCH_PERFMON_EVENTSEL_EVENT|X86_ALL_EVENT_FLAGS, \
476 : HWEIGHT(n), 0, PERF_X86_EVENT_PEBS_LD_HSW)
477 :
478 : #define INTEL_FLAGS_EVENT_CONSTRAINT_DATALA_LD_RANGE(code, end, n) \
479 : __EVENT_CONSTRAINT_RANGE(code, end, n, \
480 : ARCH_PERFMON_EVENTSEL_EVENT|X86_ALL_EVENT_FLAGS, \
481 : HWEIGHT(n), 0, PERF_X86_EVENT_PEBS_LD_HSW)
482 :
483 : #define INTEL_FLAGS_EVENT_CONSTRAINT_DATALA_XLD(code, n) \
484 : __EVENT_CONSTRAINT(code, n, \
485 : ARCH_PERFMON_EVENTSEL_EVENT|X86_ALL_EVENT_FLAGS, \
486 : HWEIGHT(n), 0, \
487 : PERF_X86_EVENT_PEBS_LD_HSW|PERF_X86_EVENT_EXCL)
488 :
489 : /* Check flags and event code/umask, and set the HSW store flag */
490 : #define INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_ST(code, n) \
491 : __EVENT_CONSTRAINT(code, n, \
492 : INTEL_ARCH_EVENT_MASK|X86_ALL_EVENT_FLAGS, \
493 : HWEIGHT(n), 0, PERF_X86_EVENT_PEBS_ST_HSW)
494 :
495 : #define INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_XST(code, n) \
496 : __EVENT_CONSTRAINT(code, n, \
497 : INTEL_ARCH_EVENT_MASK|X86_ALL_EVENT_FLAGS, \
498 : HWEIGHT(n), 0, \
499 : PERF_X86_EVENT_PEBS_ST_HSW|PERF_X86_EVENT_EXCL)
500 :
501 : /* Check flags and event code/umask, and set the HSW load flag */
502 : #define INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_LD(code, n) \
503 : __EVENT_CONSTRAINT(code, n, \
504 : INTEL_ARCH_EVENT_MASK|X86_ALL_EVENT_FLAGS, \
505 : HWEIGHT(n), 0, PERF_X86_EVENT_PEBS_LD_HSW)
506 :
507 : #define INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_XLD(code, n) \
508 : __EVENT_CONSTRAINT(code, n, \
509 : INTEL_ARCH_EVENT_MASK|X86_ALL_EVENT_FLAGS, \
510 : HWEIGHT(n), 0, \
511 : PERF_X86_EVENT_PEBS_LD_HSW|PERF_X86_EVENT_EXCL)
512 :
513 : /* Check flags and event code/umask, and set the HSW N/A flag */
514 : #define INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_NA(code, n) \
515 : __EVENT_CONSTRAINT(code, n, \
516 : INTEL_ARCH_EVENT_MASK|X86_ALL_EVENT_FLAGS, \
517 : HWEIGHT(n), 0, PERF_X86_EVENT_PEBS_NA_HSW)
518 :
519 :
520 : /*
521 : * We define the end marker as having a weight of -1
522 : * to enable blacklisting of events using a counter bitmask
523 : * of zero and thus a weight of zero.
524 : * The end marker has a weight that cannot possibly be
525 : * obtained from counting the bits in the bitmask.
526 : */
527 : #define EVENT_CONSTRAINT_END { .weight = -1 }
528 :
529 : /*
530 : * Check for end marker with weight == -1
531 : */
532 : #define for_each_event_constraint(e, c) \
533 : for ((e) = (c); (e)->weight != -1; (e)++)
534 :
535 : /*
536 : * Extra registers for specific events.
537 : *
538 : * Some events need large masks and require external MSRs.
539 : * Those extra MSRs end up being shared for all events on
540 : * a PMU and sometimes between PMU of sibling HT threads.
541 : * In either case, the kernel needs to handle conflicting
542 : * accesses to those extra, shared, regs. The data structure
543 : * to manage those registers is stored in cpu_hw_event.
544 : */
545 : struct extra_reg {
546 : unsigned int event;
547 : unsigned int msr;
548 : u64 config_mask;
549 : u64 valid_mask;
550 : int idx; /* per_xxx->regs[] reg index */
551 : bool extra_msr_access;
552 : };
553 :
554 : #define EVENT_EXTRA_REG(e, ms, m, vm, i) { \
555 : .event = (e), \
556 : .msr = (ms), \
557 : .config_mask = (m), \
558 : .valid_mask = (vm), \
559 : .idx = EXTRA_REG_##i, \
560 : .extra_msr_access = true, \
561 : }
562 :
563 : #define INTEL_EVENT_EXTRA_REG(event, msr, vm, idx) \
564 : EVENT_EXTRA_REG(event, msr, ARCH_PERFMON_EVENTSEL_EVENT, vm, idx)
565 :
566 : #define INTEL_UEVENT_EXTRA_REG(event, msr, vm, idx) \
567 : EVENT_EXTRA_REG(event, msr, ARCH_PERFMON_EVENTSEL_EVENT | \
568 : ARCH_PERFMON_EVENTSEL_UMASK, vm, idx)
569 :
570 : #define INTEL_UEVENT_PEBS_LDLAT_EXTRA_REG(c) \
571 : INTEL_UEVENT_EXTRA_REG(c, \
572 : MSR_PEBS_LD_LAT_THRESHOLD, \
573 : 0xffff, \
574 : LDLAT)
575 :
576 : #define EVENT_EXTRA_END EVENT_EXTRA_REG(0, 0, 0, 0, RSP_0)
577 :
578 : union perf_capabilities {
579 : struct {
580 : u64 lbr_format:6;
581 : u64 pebs_trap:1;
582 : u64 pebs_arch_reg:1;
583 : u64 pebs_format:4;
584 : u64 smm_freeze:1;
585 : /*
586 : * PMU supports separate counter range for writing
587 : * values > 32bit.
588 : */
589 : u64 full_width_write:1;
590 : u64 pebs_baseline:1;
591 : u64 perf_metrics:1;
592 : u64 pebs_output_pt_available:1;
593 : u64 anythread_deprecated:1;
594 : };
595 : u64 capabilities;
596 : };
597 :
598 : struct x86_pmu_quirk {
599 : struct x86_pmu_quirk *next;
600 : void (*func)(void);
601 : };
602 :
603 : union x86_pmu_config {
604 : struct {
605 : u64 event:8,
606 : umask:8,
607 : usr:1,
608 : os:1,
609 : edge:1,
610 : pc:1,
611 : interrupt:1,
612 : __reserved1:1,
613 : en:1,
614 : inv:1,
615 : cmask:8,
616 : event2:4,
617 : __reserved2:4,
618 : go:1,
619 : ho:1;
620 : } bits;
621 : u64 value;
622 : };
623 :
624 : #define X86_CONFIG(args...) ((union x86_pmu_config){.bits = {args}}).value
625 :
626 : enum {
627 : x86_lbr_exclusive_lbr,
628 : x86_lbr_exclusive_bts,
629 : x86_lbr_exclusive_pt,
630 : x86_lbr_exclusive_max,
631 : };
632 :
633 : /*
634 : * struct x86_pmu - generic x86 pmu
635 : */
636 : struct x86_pmu {
637 : /*
638 : * Generic x86 PMC bits
639 : */
640 : const char *name;
641 : int version;
642 : int (*handle_irq)(struct pt_regs *);
643 : void (*disable_all)(void);
644 : void (*enable_all)(int added);
645 : void (*enable)(struct perf_event *);
646 : void (*disable)(struct perf_event *);
647 : void (*add)(struct perf_event *);
648 : void (*del)(struct perf_event *);
649 : void (*read)(struct perf_event *event);
650 : int (*hw_config)(struct perf_event *event);
651 : int (*schedule_events)(struct cpu_hw_events *cpuc, int n, int *assign);
652 : unsigned eventsel;
653 : unsigned perfctr;
654 : int (*addr_offset)(int index, bool eventsel);
655 : int (*rdpmc_index)(int index);
656 : u64 (*event_map)(int);
657 : int max_events;
658 : int num_counters;
659 : int num_counters_fixed;
660 : int cntval_bits;
661 : u64 cntval_mask;
662 : union {
663 : unsigned long events_maskl;
664 : unsigned long events_mask[BITS_TO_LONGS(ARCH_PERFMON_EVENTS_COUNT)];
665 : };
666 : int events_mask_len;
667 : int apic;
668 : u64 max_period;
669 : struct event_constraint *
670 : (*get_event_constraints)(struct cpu_hw_events *cpuc,
671 : int idx,
672 : struct perf_event *event);
673 :
674 : void (*put_event_constraints)(struct cpu_hw_events *cpuc,
675 : struct perf_event *event);
676 :
677 : void (*start_scheduling)(struct cpu_hw_events *cpuc);
678 :
679 : void (*commit_scheduling)(struct cpu_hw_events *cpuc, int idx, int cntr);
680 :
681 : void (*stop_scheduling)(struct cpu_hw_events *cpuc);
682 :
683 : struct event_constraint *event_constraints;
684 : struct x86_pmu_quirk *quirks;
685 : int perfctr_second_write;
686 : u64 (*limit_period)(struct perf_event *event, u64 l);
687 :
688 : /* PMI handler bits */
689 : unsigned int late_ack :1,
690 : enabled_ack :1;
691 : /*
692 : * sysfs attrs
693 : */
694 : int attr_rdpmc_broken;
695 : int attr_rdpmc;
696 : struct attribute **format_attrs;
697 :
698 : ssize_t (*events_sysfs_show)(char *page, u64 config);
699 : const struct attribute_group **attr_update;
700 :
701 : unsigned long attr_freeze_on_smi;
702 :
703 : /*
704 : * CPU Hotplug hooks
705 : */
706 : int (*cpu_prepare)(int cpu);
707 : void (*cpu_starting)(int cpu);
708 : void (*cpu_dying)(int cpu);
709 : void (*cpu_dead)(int cpu);
710 :
711 : void (*check_microcode)(void);
712 : void (*sched_task)(struct perf_event_context *ctx,
713 : bool sched_in);
714 :
715 : /*
716 : * Intel Arch Perfmon v2+
717 : */
718 : u64 intel_ctrl;
719 : union perf_capabilities intel_cap;
720 :
721 : /*
722 : * Intel DebugStore bits
723 : */
724 : unsigned int bts :1,
725 : bts_active :1,
726 : pebs :1,
727 : pebs_active :1,
728 : pebs_broken :1,
729 : pebs_prec_dist :1,
730 : pebs_no_tlb :1,
731 : pebs_no_isolation :1,
732 : pebs_block :1;
733 : int pebs_record_size;
734 : int pebs_buffer_size;
735 : int max_pebs_events;
736 : void (*drain_pebs)(struct pt_regs *regs, struct perf_sample_data *data);
737 : struct event_constraint *pebs_constraints;
738 : void (*pebs_aliases)(struct perf_event *event);
739 : unsigned long large_pebs_flags;
740 : u64 rtm_abort_event;
741 :
742 : /*
743 : * Intel LBR
744 : */
745 : unsigned int lbr_tos, lbr_from, lbr_to,
746 : lbr_info, lbr_nr; /* LBR base regs and size */
747 : union {
748 : u64 lbr_sel_mask; /* LBR_SELECT valid bits */
749 : u64 lbr_ctl_mask; /* LBR_CTL valid bits */
750 : };
751 : union {
752 : const int *lbr_sel_map; /* lbr_select mappings */
753 : int *lbr_ctl_map; /* LBR_CTL mappings */
754 : };
755 : bool lbr_double_abort; /* duplicated lbr aborts */
756 : bool lbr_pt_coexist; /* (LBR|BTS) may coexist with PT */
757 :
758 : /*
759 : * Intel Architectural LBR CPUID Enumeration
760 : */
761 : unsigned int lbr_depth_mask:8;
762 : unsigned int lbr_deep_c_reset:1;
763 : unsigned int lbr_lip:1;
764 : unsigned int lbr_cpl:1;
765 : unsigned int lbr_filter:1;
766 : unsigned int lbr_call_stack:1;
767 : unsigned int lbr_mispred:1;
768 : unsigned int lbr_timed_lbr:1;
769 : unsigned int lbr_br_type:1;
770 :
771 : void (*lbr_reset)(void);
772 : void (*lbr_read)(struct cpu_hw_events *cpuc);
773 : void (*lbr_save)(void *ctx);
774 : void (*lbr_restore)(void *ctx);
775 :
776 : /*
777 : * Intel PT/LBR/BTS are exclusive
778 : */
779 : atomic_t lbr_exclusive[x86_lbr_exclusive_max];
780 :
781 : /*
782 : * Intel perf metrics
783 : */
784 : int num_topdown_events;
785 : u64 (*update_topdown_event)(struct perf_event *event);
786 : int (*set_topdown_event_period)(struct perf_event *event);
787 :
788 : /*
789 : * perf task context (i.e. struct perf_event_context::task_ctx_data)
790 : * switch helper to bridge calls from perf/core to perf/x86.
791 : * See struct pmu::swap_task_ctx() usage for examples;
792 : */
793 : void (*swap_task_ctx)(struct perf_event_context *prev,
794 : struct perf_event_context *next);
795 :
796 : /*
797 : * AMD bits
798 : */
799 : unsigned int amd_nb_constraints : 1;
800 : u64 perf_ctr_pair_en;
801 :
802 : /*
803 : * Extra registers for events
804 : */
805 : struct extra_reg *extra_regs;
806 : unsigned int flags;
807 :
808 : /*
809 : * Intel host/guest support (KVM)
810 : */
811 : struct perf_guest_switch_msr *(*guest_get_msrs)(int *nr);
812 :
813 : /*
814 : * Check period value for PERF_EVENT_IOC_PERIOD ioctl.
815 : */
816 : int (*check_period) (struct perf_event *event, u64 period);
817 :
818 : int (*aux_output_match) (struct perf_event *event);
819 : };
820 :
821 : struct x86_perf_task_context_opt {
822 : int lbr_callstack_users;
823 : int lbr_stack_state;
824 : int log_id;
825 : };
826 :
827 : struct x86_perf_task_context {
828 : u64 lbr_sel;
829 : int tos;
830 : int valid_lbrs;
831 : struct x86_perf_task_context_opt opt;
832 : struct lbr_entry lbr[MAX_LBR_ENTRIES];
833 : };
834 :
835 : struct x86_perf_task_context_arch_lbr {
836 : struct x86_perf_task_context_opt opt;
837 : struct lbr_entry entries[];
838 : };
839 :
840 : /*
841 : * Add padding to guarantee the 64-byte alignment of the state buffer.
842 : *
843 : * The structure is dynamically allocated. The size of the LBR state may vary
844 : * based on the number of LBR registers.
845 : *
846 : * Do not put anything after the LBR state.
847 : */
848 : struct x86_perf_task_context_arch_lbr_xsave {
849 : struct x86_perf_task_context_opt opt;
850 :
851 : union {
852 : struct xregs_state xsave;
853 : struct {
854 : struct fxregs_state i387;
855 : struct xstate_header header;
856 : struct arch_lbr_state lbr;
857 : } __attribute__ ((packed, aligned (XSAVE_ALIGNMENT)));
858 : };
859 : };
860 :
861 : #define x86_add_quirk(func_) \
862 : do { \
863 : static struct x86_pmu_quirk __quirk __initdata = { \
864 : .func = func_, \
865 : }; \
866 : __quirk.next = x86_pmu.quirks; \
867 : x86_pmu.quirks = &__quirk; \
868 : } while (0)
869 :
870 : /*
871 : * x86_pmu flags
872 : */
873 : #define PMU_FL_NO_HT_SHARING 0x1 /* no hyper-threading resource sharing */
874 : #define PMU_FL_HAS_RSP_1 0x2 /* has 2 equivalent offcore_rsp regs */
875 : #define PMU_FL_EXCL_CNTRS 0x4 /* has exclusive counter requirements */
876 : #define PMU_FL_EXCL_ENABLED 0x8 /* exclusive counter active */
877 : #define PMU_FL_PEBS_ALL 0x10 /* all events are valid PEBS events */
878 : #define PMU_FL_TFA 0x20 /* deal with TSX force abort */
879 : #define PMU_FL_PAIR 0x40 /* merge counters for large incr. events */
880 : #define PMU_FL_INSTR_LATENCY 0x80 /* Support Instruction Latency in PEBS Memory Info Record */
881 : #define PMU_FL_MEM_LOADS_AUX 0x100 /* Require an auxiliary event for the complete memory info */
882 :
883 : #define EVENT_VAR(_id) event_attr_##_id
884 : #define EVENT_PTR(_id) &event_attr_##_id.attr.attr
885 :
886 : #define EVENT_ATTR(_name, _id) \
887 : static struct perf_pmu_events_attr EVENT_VAR(_id) = { \
888 : .attr = __ATTR(_name, 0444, events_sysfs_show, NULL), \
889 : .id = PERF_COUNT_HW_##_id, \
890 : .event_str = NULL, \
891 : };
892 :
893 : #define EVENT_ATTR_STR(_name, v, str) \
894 : static struct perf_pmu_events_attr event_attr_##v = { \
895 : .attr = __ATTR(_name, 0444, events_sysfs_show, NULL), \
896 : .id = 0, \
897 : .event_str = str, \
898 : };
899 :
900 : #define EVENT_ATTR_STR_HT(_name, v, noht, ht) \
901 : static struct perf_pmu_events_ht_attr event_attr_##v = { \
902 : .attr = __ATTR(_name, 0444, events_ht_sysfs_show, NULL),\
903 : .id = 0, \
904 : .event_str_noht = noht, \
905 : .event_str_ht = ht, \
906 : }
907 :
908 : struct pmu *x86_get_pmu(void);
909 : extern struct x86_pmu x86_pmu __read_mostly;
910 :
911 0 : static __always_inline struct x86_perf_task_context_opt *task_context_opt(void *ctx)
912 : {
913 0 : if (static_cpu_has(X86_FEATURE_ARCH_LBR))
914 0 : return &((struct x86_perf_task_context_arch_lbr *)ctx)->opt;
915 :
916 0 : return &((struct x86_perf_task_context *)ctx)->opt;
917 : }
918 :
919 0 : static inline bool x86_pmu_has_lbr_callstack(void)
920 : {
921 0 : return x86_pmu.lbr_sel_map &&
922 0 : x86_pmu.lbr_sel_map[PERF_SAMPLE_BRANCH_CALL_STACK_SHIFT] > 0;
923 : }
924 :
925 : DECLARE_PER_CPU(struct cpu_hw_events, cpu_hw_events);
926 :
927 : int x86_perf_event_set_period(struct perf_event *event);
928 :
929 : /*
930 : * Generalized hw caching related hw_event table, filled
931 : * in on a per model basis. A value of 0 means
932 : * 'not supported', -1 means 'hw_event makes no sense on
933 : * this CPU', any other value means the raw hw_event
934 : * ID.
935 : */
936 :
937 : #define C(x) PERF_COUNT_HW_CACHE_##x
938 :
939 : extern u64 __read_mostly hw_cache_event_ids
940 : [PERF_COUNT_HW_CACHE_MAX]
941 : [PERF_COUNT_HW_CACHE_OP_MAX]
942 : [PERF_COUNT_HW_CACHE_RESULT_MAX];
943 : extern u64 __read_mostly hw_cache_extra_regs
944 : [PERF_COUNT_HW_CACHE_MAX]
945 : [PERF_COUNT_HW_CACHE_OP_MAX]
946 : [PERF_COUNT_HW_CACHE_RESULT_MAX];
947 :
948 : u64 x86_perf_event_update(struct perf_event *event);
949 :
950 0 : static inline unsigned int x86_pmu_config_addr(int index)
951 : {
952 0 : return x86_pmu.eventsel + (x86_pmu.addr_offset ?
953 0 : x86_pmu.addr_offset(index, true) : index);
954 : }
955 :
956 0 : static inline unsigned int x86_pmu_event_addr(int index)
957 : {
958 0 : return x86_pmu.perfctr + (x86_pmu.addr_offset ?
959 0 : x86_pmu.addr_offset(index, false) : index);
960 : }
961 :
962 : static inline int x86_pmu_rdpmc_index(int index)
963 : {
964 : return x86_pmu.rdpmc_index ? x86_pmu.rdpmc_index(index) : index;
965 : }
966 :
967 : int x86_add_exclusive(unsigned int what);
968 :
969 : void x86_del_exclusive(unsigned int what);
970 :
971 : int x86_reserve_hardware(void);
972 :
973 : void x86_release_hardware(void);
974 :
975 : int x86_pmu_max_precise(void);
976 :
977 : void hw_perf_lbr_event_destroy(struct perf_event *event);
978 :
979 : int x86_setup_perfctr(struct perf_event *event);
980 :
981 : int x86_pmu_hw_config(struct perf_event *event);
982 :
983 : void x86_pmu_disable_all(void);
984 :
985 0 : static inline bool is_counter_pair(struct hw_perf_event *hwc)
986 : {
987 0 : return hwc->flags & PERF_X86_EVENT_PAIR;
988 : }
989 :
990 0 : static inline void __x86_pmu_enable_event(struct hw_perf_event *hwc,
991 : u64 enable_mask)
992 : {
993 0 : u64 disable_mask = __this_cpu_read(cpu_hw_events.perf_ctr_virt_mask);
994 :
995 0 : if (hwc->extra_reg.reg)
996 0 : wrmsrl(hwc->extra_reg.reg, hwc->extra_reg.config);
997 :
998 : /*
999 : * Add enabled Merge event on next counter
1000 : * if large increment event being enabled on this counter
1001 : */
1002 0 : if (is_counter_pair(hwc))
1003 0 : wrmsrl(x86_pmu_config_addr(hwc->idx + 1), x86_pmu.perf_ctr_pair_en);
1004 :
1005 0 : wrmsrl(hwc->config_base, (hwc->config | enable_mask) & ~disable_mask);
1006 0 : }
1007 :
1008 : void x86_pmu_enable_all(int added);
1009 :
1010 : int perf_assign_events(struct event_constraint **constraints, int n,
1011 : int wmin, int wmax, int gpmax, int *assign);
1012 : int x86_schedule_events(struct cpu_hw_events *cpuc, int n, int *assign);
1013 :
1014 : void x86_pmu_stop(struct perf_event *event, int flags);
1015 :
1016 0 : static inline void x86_pmu_disable_event(struct perf_event *event)
1017 : {
1018 0 : struct hw_perf_event *hwc = &event->hw;
1019 :
1020 0 : wrmsrl(hwc->config_base, hwc->config);
1021 :
1022 0 : if (is_counter_pair(hwc))
1023 0 : wrmsrl(x86_pmu_config_addr(hwc->idx + 1), 0);
1024 0 : }
1025 :
1026 : void x86_pmu_enable_event(struct perf_event *event);
1027 :
1028 : int x86_pmu_handle_irq(struct pt_regs *regs);
1029 :
1030 : extern struct event_constraint emptyconstraint;
1031 :
1032 : extern struct event_constraint unconstrained;
1033 :
1034 0 : static inline bool kernel_ip(unsigned long ip)
1035 : {
1036 : #ifdef CONFIG_X86_32
1037 : return ip > PAGE_OFFSET;
1038 : #else
1039 0 : return (long)ip < 0;
1040 : #endif
1041 : }
1042 :
1043 : /*
1044 : * Not all PMUs provide the right context information to place the reported IP
1045 : * into full context. Specifically segment registers are typically not
1046 : * supplied.
1047 : *
1048 : * Assuming the address is a linear address (it is for IBS), we fake the CS and
1049 : * vm86 mode using the known zero-based code segment and 'fix up' the registers
1050 : * to reflect this.
1051 : *
1052 : * Intel PEBS/LBR appear to typically provide the effective address, nothing
1053 : * much we can do about that but pray and treat it like a linear address.
1054 : */
1055 0 : static inline void set_linear_ip(struct pt_regs *regs, unsigned long ip)
1056 : {
1057 0 : regs->cs = kernel_ip(ip) ? __KERNEL_CS : __USER_CS;
1058 0 : if (regs->flags & X86_VM_MASK)
1059 : regs->flags ^= (PERF_EFLAGS_VM | X86_VM_MASK);
1060 0 : regs->ip = ip;
1061 0 : }
1062 :
1063 : ssize_t x86_event_sysfs_show(char *page, u64 config, u64 event);
1064 : ssize_t intel_event_sysfs_show(char *page, u64 config);
1065 :
1066 : ssize_t events_sysfs_show(struct device *dev, struct device_attribute *attr,
1067 : char *page);
1068 : ssize_t events_ht_sysfs_show(struct device *dev, struct device_attribute *attr,
1069 : char *page);
1070 :
1071 0 : static inline bool fixed_counter_disabled(int i)
1072 : {
1073 0 : return !(x86_pmu.intel_ctrl >> (i + INTEL_PMC_IDX_FIXED));
1074 : }
1075 :
1076 : #ifdef CONFIG_CPU_SUP_AMD
1077 :
1078 : int amd_pmu_init(void);
1079 :
1080 : #else /* CONFIG_CPU_SUP_AMD */
1081 :
1082 : static inline int amd_pmu_init(void)
1083 : {
1084 : return 0;
1085 : }
1086 :
1087 : #endif /* CONFIG_CPU_SUP_AMD */
1088 :
1089 0 : static inline int is_pebs_pt(struct perf_event *event)
1090 : {
1091 0 : return !!(event->hw.flags & PERF_X86_EVENT_PEBS_VIA_PT);
1092 : }
1093 :
1094 : #ifdef CONFIG_CPU_SUP_INTEL
1095 :
1096 0 : static inline bool intel_pmu_has_bts_period(struct perf_event *event, u64 period)
1097 : {
1098 0 : struct hw_perf_event *hwc = &event->hw;
1099 0 : unsigned int hw_event, bts_event;
1100 :
1101 0 : if (event->attr.freq)
1102 : return false;
1103 :
1104 0 : hw_event = hwc->config & INTEL_ARCH_EVENT_MASK;
1105 0 : bts_event = x86_pmu.event_map(PERF_COUNT_HW_BRANCH_INSTRUCTIONS);
1106 :
1107 0 : return hw_event == bts_event && period == 1;
1108 : }
1109 :
1110 0 : static inline bool intel_pmu_has_bts(struct perf_event *event)
1111 : {
1112 0 : struct hw_perf_event *hwc = &event->hw;
1113 :
1114 0 : return intel_pmu_has_bts_period(event, hwc->sample_period);
1115 : }
1116 :
1117 : int intel_pmu_save_and_restart(struct perf_event *event);
1118 :
1119 : struct event_constraint *
1120 : x86_get_event_constraints(struct cpu_hw_events *cpuc, int idx,
1121 : struct perf_event *event);
1122 :
1123 : extern int intel_cpuc_prepare(struct cpu_hw_events *cpuc, int cpu);
1124 : extern void intel_cpuc_finish(struct cpu_hw_events *cpuc);
1125 :
1126 : int intel_pmu_init(void);
1127 :
1128 : void init_debug_store_on_cpu(int cpu);
1129 :
1130 : void fini_debug_store_on_cpu(int cpu);
1131 :
1132 : void release_ds_buffers(void);
1133 :
1134 : void reserve_ds_buffers(void);
1135 :
1136 : void release_lbr_buffers(void);
1137 :
1138 : extern struct event_constraint bts_constraint;
1139 : extern struct event_constraint vlbr_constraint;
1140 :
1141 : void intel_pmu_enable_bts(u64 config);
1142 :
1143 : void intel_pmu_disable_bts(void);
1144 :
1145 : int intel_pmu_drain_bts_buffer(void);
1146 :
1147 : extern struct event_constraint intel_core2_pebs_event_constraints[];
1148 :
1149 : extern struct event_constraint intel_atom_pebs_event_constraints[];
1150 :
1151 : extern struct event_constraint intel_slm_pebs_event_constraints[];
1152 :
1153 : extern struct event_constraint intel_glm_pebs_event_constraints[];
1154 :
1155 : extern struct event_constraint intel_glp_pebs_event_constraints[];
1156 :
1157 : extern struct event_constraint intel_nehalem_pebs_event_constraints[];
1158 :
1159 : extern struct event_constraint intel_westmere_pebs_event_constraints[];
1160 :
1161 : extern struct event_constraint intel_snb_pebs_event_constraints[];
1162 :
1163 : extern struct event_constraint intel_ivb_pebs_event_constraints[];
1164 :
1165 : extern struct event_constraint intel_hsw_pebs_event_constraints[];
1166 :
1167 : extern struct event_constraint intel_bdw_pebs_event_constraints[];
1168 :
1169 : extern struct event_constraint intel_skl_pebs_event_constraints[];
1170 :
1171 : extern struct event_constraint intel_icl_pebs_event_constraints[];
1172 :
1173 : extern struct event_constraint intel_spr_pebs_event_constraints[];
1174 :
1175 : struct event_constraint *intel_pebs_constraints(struct perf_event *event);
1176 :
1177 : void intel_pmu_pebs_add(struct perf_event *event);
1178 :
1179 : void intel_pmu_pebs_del(struct perf_event *event);
1180 :
1181 : void intel_pmu_pebs_enable(struct perf_event *event);
1182 :
1183 : void intel_pmu_pebs_disable(struct perf_event *event);
1184 :
1185 : void intel_pmu_pebs_enable_all(void);
1186 :
1187 : void intel_pmu_pebs_disable_all(void);
1188 :
1189 : void intel_pmu_pebs_sched_task(struct perf_event_context *ctx, bool sched_in);
1190 :
1191 : void intel_pmu_auto_reload_read(struct perf_event *event);
1192 :
1193 : void intel_pmu_store_pebs_lbrs(struct lbr_entry *lbr);
1194 :
1195 : void intel_ds_init(void);
1196 :
1197 : void intel_pmu_lbr_swap_task_ctx(struct perf_event_context *prev,
1198 : struct perf_event_context *next);
1199 :
1200 : void intel_pmu_lbr_sched_task(struct perf_event_context *ctx, bool sched_in);
1201 :
1202 : u64 lbr_from_signext_quirk_wr(u64 val);
1203 :
1204 : void intel_pmu_lbr_reset(void);
1205 :
1206 : void intel_pmu_lbr_reset_32(void);
1207 :
1208 : void intel_pmu_lbr_reset_64(void);
1209 :
1210 : void intel_pmu_lbr_add(struct perf_event *event);
1211 :
1212 : void intel_pmu_lbr_del(struct perf_event *event);
1213 :
1214 : void intel_pmu_lbr_enable_all(bool pmi);
1215 :
1216 : void intel_pmu_lbr_disable_all(void);
1217 :
1218 : void intel_pmu_lbr_read(void);
1219 :
1220 : void intel_pmu_lbr_read_32(struct cpu_hw_events *cpuc);
1221 :
1222 : void intel_pmu_lbr_read_64(struct cpu_hw_events *cpuc);
1223 :
1224 : void intel_pmu_lbr_save(void *ctx);
1225 :
1226 : void intel_pmu_lbr_restore(void *ctx);
1227 :
1228 : void intel_pmu_lbr_init_core(void);
1229 :
1230 : void intel_pmu_lbr_init_nhm(void);
1231 :
1232 : void intel_pmu_lbr_init_atom(void);
1233 :
1234 : void intel_pmu_lbr_init_slm(void);
1235 :
1236 : void intel_pmu_lbr_init_snb(void);
1237 :
1238 : void intel_pmu_lbr_init_hsw(void);
1239 :
1240 : void intel_pmu_lbr_init_skl(void);
1241 :
1242 : void intel_pmu_lbr_init_knl(void);
1243 :
1244 : void intel_pmu_arch_lbr_init(void);
1245 :
1246 : void intel_pmu_pebs_data_source_nhm(void);
1247 :
1248 : void intel_pmu_pebs_data_source_skl(bool pmem);
1249 :
1250 : int intel_pmu_setup_lbr_filter(struct perf_event *event);
1251 :
1252 : void intel_pt_interrupt(void);
1253 :
1254 : int intel_bts_interrupt(void);
1255 :
1256 : void intel_bts_enable_local(void);
1257 :
1258 : void intel_bts_disable_local(void);
1259 :
1260 : int p4_pmu_init(void);
1261 :
1262 : int p6_pmu_init(void);
1263 :
1264 : int knc_pmu_init(void);
1265 :
1266 0 : static inline int is_ht_workaround_enabled(void)
1267 : {
1268 0 : return !!(x86_pmu.flags & PMU_FL_EXCL_ENABLED);
1269 : }
1270 :
1271 : #else /* CONFIG_CPU_SUP_INTEL */
1272 :
1273 : static inline void reserve_ds_buffers(void)
1274 : {
1275 : }
1276 :
1277 : static inline void release_ds_buffers(void)
1278 : {
1279 : }
1280 :
1281 : static inline void release_lbr_buffers(void)
1282 : {
1283 : }
1284 :
1285 : static inline int intel_pmu_init(void)
1286 : {
1287 : return 0;
1288 : }
1289 :
1290 : static inline int intel_cpuc_prepare(struct cpu_hw_events *cpuc, int cpu)
1291 : {
1292 : return 0;
1293 : }
1294 :
1295 : static inline void intel_cpuc_finish(struct cpu_hw_events *cpuc)
1296 : {
1297 : }
1298 :
1299 : static inline int is_ht_workaround_enabled(void)
1300 : {
1301 : return 0;
1302 : }
1303 : #endif /* CONFIG_CPU_SUP_INTEL */
1304 :
1305 : #if ((defined CONFIG_CPU_SUP_CENTAUR) || (defined CONFIG_CPU_SUP_ZHAOXIN))
1306 : int zhaoxin_pmu_init(void);
1307 : #else
1308 : static inline int zhaoxin_pmu_init(void)
1309 : {
1310 : return 0;
1311 : }
1312 : #endif /*CONFIG_CPU_SUP_CENTAUR or CONFIG_CPU_SUP_ZHAOXIN*/
|
