Line data Source code
1 : // SPDX-License-Identifier: GPL-2.0-only
2 : /*
3 : * Copyright (C) 2013 Advanced Micro Devices, Inc.
4 : *
5 : * Author: Jacob Shin <jacob.shin@amd.com>
6 : */
7 :
8 : #include <linux/perf_event.h>
9 : #include <linux/percpu.h>
10 : #include <linux/types.h>
11 : #include <linux/slab.h>
12 : #include <linux/init.h>
13 : #include <linux/cpu.h>
14 : #include <linux/cpumask.h>
15 :
16 : #include <asm/cpufeature.h>
17 : #include <asm/perf_event.h>
18 : #include <asm/msr.h>
19 : #include <asm/smp.h>
20 :
21 : #define NUM_COUNTERS_NB 4
22 : #define NUM_COUNTERS_L2 4
23 : #define NUM_COUNTERS_L3 6
24 : #define MAX_COUNTERS 6
25 :
26 : #define RDPMC_BASE_NB 6
27 : #define RDPMC_BASE_LLC 10
28 :
29 : #define COUNTER_SHIFT 16
30 :
31 : #undef pr_fmt
32 : #define pr_fmt(fmt) "amd_uncore: " fmt
33 :
34 : static int num_counters_llc;
35 : static int num_counters_nb;
36 : static bool l3_mask;
37 :
38 : static HLIST_HEAD(uncore_unused_list);
39 :
40 : struct amd_uncore {
41 : int id;
42 : int refcnt;
43 : int cpu;
44 : int num_counters;
45 : int rdpmc_base;
46 : u32 msr_base;
47 : cpumask_t *active_mask;
48 : struct pmu *pmu;
49 : struct perf_event *events[MAX_COUNTERS];
50 : struct hlist_node node;
51 : };
52 :
53 : static struct amd_uncore * __percpu *amd_uncore_nb;
54 : static struct amd_uncore * __percpu *amd_uncore_llc;
55 :
56 : static struct pmu amd_nb_pmu;
57 : static struct pmu amd_llc_pmu;
58 :
59 : static cpumask_t amd_nb_active_mask;
60 : static cpumask_t amd_llc_active_mask;
61 :
62 0 : static bool is_nb_event(struct perf_event *event)
63 : {
64 0 : return event->pmu->type == amd_nb_pmu.type;
65 : }
66 :
67 0 : static bool is_llc_event(struct perf_event *event)
68 : {
69 0 : return event->pmu->type == amd_llc_pmu.type;
70 : }
71 :
72 0 : static struct amd_uncore *event_to_amd_uncore(struct perf_event *event)
73 : {
74 0 : if (is_nb_event(event) && amd_uncore_nb)
75 0 : return *per_cpu_ptr(amd_uncore_nb, event->cpu);
76 0 : else if (is_llc_event(event) && amd_uncore_llc)
77 0 : return *per_cpu_ptr(amd_uncore_llc, event->cpu);
78 :
79 : return NULL;
80 : }
81 :
82 0 : static void amd_uncore_read(struct perf_event *event)
83 : {
84 0 : struct hw_perf_event *hwc = &event->hw;
85 0 : u64 prev, new;
86 0 : s64 delta;
87 :
88 : /*
89 : * since we do not enable counter overflow interrupts,
90 : * we do not have to worry about prev_count changing on us
91 : */
92 :
93 0 : prev = local64_read(&hwc->prev_count);
94 0 : rdpmcl(hwc->event_base_rdpmc, new);
95 0 : local64_set(&hwc->prev_count, new);
96 0 : delta = (new << COUNTER_SHIFT) - (prev << COUNTER_SHIFT);
97 0 : delta >>= COUNTER_SHIFT;
98 0 : local64_add(delta, &event->count);
99 0 : }
100 :
101 0 : static void amd_uncore_start(struct perf_event *event, int flags)
102 : {
103 0 : struct hw_perf_event *hwc = &event->hw;
104 :
105 0 : if (flags & PERF_EF_RELOAD)
106 0 : wrmsrl(hwc->event_base, (u64)local64_read(&hwc->prev_count));
107 :
108 0 : hwc->state = 0;
109 0 : wrmsrl(hwc->config_base, (hwc->config | ARCH_PERFMON_EVENTSEL_ENABLE));
110 0 : perf_event_update_userpage(event);
111 0 : }
112 :
113 0 : static void amd_uncore_stop(struct perf_event *event, int flags)
114 : {
115 0 : struct hw_perf_event *hwc = &event->hw;
116 :
117 0 : wrmsrl(hwc->config_base, hwc->config);
118 0 : hwc->state |= PERF_HES_STOPPED;
119 :
120 0 : if ((flags & PERF_EF_UPDATE) && !(hwc->state & PERF_HES_UPTODATE)) {
121 0 : amd_uncore_read(event);
122 0 : hwc->state |= PERF_HES_UPTODATE;
123 : }
124 0 : }
125 :
126 0 : static int amd_uncore_add(struct perf_event *event, int flags)
127 : {
128 0 : int i;
129 0 : struct amd_uncore *uncore = event_to_amd_uncore(event);
130 0 : struct hw_perf_event *hwc = &event->hw;
131 :
132 : /* are we already assigned? */
133 0 : if (hwc->idx != -1 && uncore->events[hwc->idx] == event)
134 0 : goto out;
135 :
136 0 : for (i = 0; i < uncore->num_counters; i++) {
137 0 : if (uncore->events[i] == event) {
138 0 : hwc->idx = i;
139 0 : goto out;
140 : }
141 : }
142 :
143 : /* if not, take the first available counter */
144 0 : hwc->idx = -1;
145 0 : for (i = 0; i < uncore->num_counters; i++) {
146 0 : if (cmpxchg(&uncore->events[i], NULL, event) == NULL) {
147 0 : hwc->idx = i;
148 0 : break;
149 : }
150 : }
151 :
152 0 : out:
153 0 : if (hwc->idx == -1)
154 : return -EBUSY;
155 :
156 0 : hwc->config_base = uncore->msr_base + (2 * hwc->idx);
157 0 : hwc->event_base = uncore->msr_base + 1 + (2 * hwc->idx);
158 0 : hwc->event_base_rdpmc = uncore->rdpmc_base + hwc->idx;
159 0 : hwc->state = PERF_HES_UPTODATE | PERF_HES_STOPPED;
160 :
161 0 : if (flags & PERF_EF_START)
162 0 : amd_uncore_start(event, PERF_EF_RELOAD);
163 :
164 : return 0;
165 : }
166 :
167 0 : static void amd_uncore_del(struct perf_event *event, int flags)
168 : {
169 0 : int i;
170 0 : struct amd_uncore *uncore = event_to_amd_uncore(event);
171 0 : struct hw_perf_event *hwc = &event->hw;
172 :
173 0 : amd_uncore_stop(event, PERF_EF_UPDATE);
174 :
175 0 : for (i = 0; i < uncore->num_counters; i++) {
176 0 : if (cmpxchg(&uncore->events[i], event, NULL) == event)
177 : break;
178 : }
179 :
180 0 : hwc->idx = -1;
181 0 : }
182 :
183 : /*
184 : * Return a full thread and slice mask unless user
185 : * has provided them
186 : */
187 0 : static u64 l3_thread_slice_mask(u64 config)
188 : {
189 0 : if (boot_cpu_data.x86 <= 0x18)
190 0 : return ((config & AMD64_L3_SLICE_MASK) ? : AMD64_L3_SLICE_MASK) |
191 0 : ((config & AMD64_L3_THREAD_MASK) ? : AMD64_L3_THREAD_MASK);
192 :
193 : /*
194 : * If the user doesn't specify a threadmask, they're not trying to
195 : * count core 0, so we enable all cores & threads.
196 : * We'll also assume that they want to count slice 0 if they specify
197 : * a threadmask and leave sliceid and enallslices unpopulated.
198 : */
199 0 : if (!(config & AMD64_L3_F19H_THREAD_MASK))
200 : return AMD64_L3_F19H_THREAD_MASK | AMD64_L3_EN_ALL_SLICES |
201 : AMD64_L3_EN_ALL_CORES;
202 :
203 0 : return config & (AMD64_L3_F19H_THREAD_MASK | AMD64_L3_SLICEID_MASK |
204 : AMD64_L3_EN_ALL_CORES | AMD64_L3_EN_ALL_SLICES |
205 : AMD64_L3_COREID_MASK);
206 : }
207 :
208 0 : static int amd_uncore_event_init(struct perf_event *event)
209 : {
210 0 : struct amd_uncore *uncore;
211 0 : struct hw_perf_event *hwc = &event->hw;
212 :
213 0 : if (event->attr.type != event->pmu->type)
214 : return -ENOENT;
215 :
216 : /*
217 : * NB and Last level cache counters (MSRs) are shared across all cores
218 : * that share the same NB / Last level cache. On family 16h and below,
219 : * Interrupts can be directed to a single target core, however, event
220 : * counts generated by processes running on other cores cannot be masked
221 : * out. So we do not support sampling and per-thread events via
222 : * CAP_NO_INTERRUPT, and we do not enable counter overflow interrupts:
223 : */
224 0 : hwc->config = event->attr.config & AMD64_RAW_EVENT_MASK_NB;
225 0 : hwc->idx = -1;
226 :
227 0 : if (event->cpu < 0)
228 : return -EINVAL;
229 :
230 : /*
231 : * SliceMask and ThreadMask need to be set for certain L3 events.
232 : * For other events, the two fields do not affect the count.
233 : */
234 0 : if (l3_mask && is_llc_event(event))
235 0 : hwc->config |= l3_thread_slice_mask(event->attr.config);
236 :
237 0 : uncore = event_to_amd_uncore(event);
238 0 : if (!uncore)
239 : return -ENODEV;
240 :
241 : /*
242 : * since request can come in to any of the shared cores, we will remap
243 : * to a single common cpu.
244 : */
245 0 : event->cpu = uncore->cpu;
246 :
247 0 : return 0;
248 : }
249 :
250 0 : static ssize_t amd_uncore_attr_show_cpumask(struct device *dev,
251 : struct device_attribute *attr,
252 : char *buf)
253 : {
254 0 : cpumask_t *active_mask;
255 0 : struct pmu *pmu = dev_get_drvdata(dev);
256 :
257 0 : if (pmu->type == amd_nb_pmu.type)
258 : active_mask = &amd_nb_active_mask;
259 0 : else if (pmu->type == amd_llc_pmu.type)
260 : active_mask = &amd_llc_active_mask;
261 : else
262 : return 0;
263 :
264 0 : return cpumap_print_to_pagebuf(true, buf, active_mask);
265 : }
266 : static DEVICE_ATTR(cpumask, S_IRUGO, amd_uncore_attr_show_cpumask, NULL);
267 :
268 : static struct attribute *amd_uncore_attrs[] = {
269 : &dev_attr_cpumask.attr,
270 : NULL,
271 : };
272 :
273 : static struct attribute_group amd_uncore_attr_group = {
274 : .attrs = amd_uncore_attrs,
275 : };
276 :
277 : #define DEFINE_UNCORE_FORMAT_ATTR(_var, _name, _format) \
278 : static ssize_t __uncore_##_var##_show(struct kobject *kobj, \
279 : struct kobj_attribute *attr, \
280 : char *page) \
281 : { \
282 : BUILD_BUG_ON(sizeof(_format) >= PAGE_SIZE); \
283 : return sprintf(page, _format "\n"); \
284 : } \
285 : static struct kobj_attribute format_attr_##_var = \
286 : __ATTR(_name, 0444, __uncore_##_var##_show, NULL)
287 :
288 0 : DEFINE_UNCORE_FORMAT_ATTR(event12, event, "config:0-7,32-35");
289 0 : DEFINE_UNCORE_FORMAT_ATTR(event14, event, "config:0-7,32-35,59-60"); /* F17h+ DF */
290 0 : DEFINE_UNCORE_FORMAT_ATTR(event8, event, "config:0-7"); /* F17h+ L3 */
291 0 : DEFINE_UNCORE_FORMAT_ATTR(umask, umask, "config:8-15");
292 0 : DEFINE_UNCORE_FORMAT_ATTR(coreid, coreid, "config:42-44"); /* F19h L3 */
293 0 : DEFINE_UNCORE_FORMAT_ATTR(slicemask, slicemask, "config:48-51"); /* F17h L3 */
294 0 : DEFINE_UNCORE_FORMAT_ATTR(threadmask8, threadmask, "config:56-63"); /* F17h L3 */
295 0 : DEFINE_UNCORE_FORMAT_ATTR(threadmask2, threadmask, "config:56-57"); /* F19h L3 */
296 0 : DEFINE_UNCORE_FORMAT_ATTR(enallslices, enallslices, "config:46"); /* F19h L3 */
297 0 : DEFINE_UNCORE_FORMAT_ATTR(enallcores, enallcores, "config:47"); /* F19h L3 */
298 0 : DEFINE_UNCORE_FORMAT_ATTR(sliceid, sliceid, "config:48-50"); /* F19h L3 */
299 :
300 : static struct attribute *amd_uncore_df_format_attr[] = {
301 : &format_attr_event12.attr, /* event14 if F17h+ */
302 : &format_attr_umask.attr,
303 : NULL,
304 : };
305 :
306 : static struct attribute *amd_uncore_l3_format_attr[] = {
307 : &format_attr_event12.attr, /* event8 if F17h+ */
308 : &format_attr_umask.attr,
309 : NULL, /* slicemask if F17h, coreid if F19h */
310 : NULL, /* threadmask8 if F17h, enallslices if F19h */
311 : NULL, /* enallcores if F19h */
312 : NULL, /* sliceid if F19h */
313 : NULL, /* threadmask2 if F19h */
314 : NULL,
315 : };
316 :
317 : static struct attribute_group amd_uncore_df_format_group = {
318 : .name = "format",
319 : .attrs = amd_uncore_df_format_attr,
320 : };
321 :
322 : static struct attribute_group amd_uncore_l3_format_group = {
323 : .name = "format",
324 : .attrs = amd_uncore_l3_format_attr,
325 : };
326 :
327 : static const struct attribute_group *amd_uncore_df_attr_groups[] = {
328 : &amd_uncore_attr_group,
329 : &amd_uncore_df_format_group,
330 : NULL,
331 : };
332 :
333 : static const struct attribute_group *amd_uncore_l3_attr_groups[] = {
334 : &amd_uncore_attr_group,
335 : &amd_uncore_l3_format_group,
336 : NULL,
337 : };
338 :
339 : static struct pmu amd_nb_pmu = {
340 : .task_ctx_nr = perf_invalid_context,
341 : .attr_groups = amd_uncore_df_attr_groups,
342 : .name = "amd_nb",
343 : .event_init = amd_uncore_event_init,
344 : .add = amd_uncore_add,
345 : .del = amd_uncore_del,
346 : .start = amd_uncore_start,
347 : .stop = amd_uncore_stop,
348 : .read = amd_uncore_read,
349 : .capabilities = PERF_PMU_CAP_NO_EXCLUDE | PERF_PMU_CAP_NO_INTERRUPT,
350 : };
351 :
352 : static struct pmu amd_llc_pmu = {
353 : .task_ctx_nr = perf_invalid_context,
354 : .attr_groups = amd_uncore_l3_attr_groups,
355 : .name = "amd_l2",
356 : .event_init = amd_uncore_event_init,
357 : .add = amd_uncore_add,
358 : .del = amd_uncore_del,
359 : .start = amd_uncore_start,
360 : .stop = amd_uncore_stop,
361 : .read = amd_uncore_read,
362 : .capabilities = PERF_PMU_CAP_NO_EXCLUDE | PERF_PMU_CAP_NO_INTERRUPT,
363 : };
364 :
365 0 : static struct amd_uncore *amd_uncore_alloc(unsigned int cpu)
366 : {
367 0 : return kzalloc_node(sizeof(struct amd_uncore), GFP_KERNEL,
368 : cpu_to_node(cpu));
369 : }
370 :
371 0 : static int amd_uncore_cpu_up_prepare(unsigned int cpu)
372 : {
373 0 : struct amd_uncore *uncore_nb = NULL, *uncore_llc;
374 :
375 0 : if (amd_uncore_nb) {
376 0 : uncore_nb = amd_uncore_alloc(cpu);
377 0 : if (!uncore_nb)
378 0 : goto fail;
379 0 : uncore_nb->cpu = cpu;
380 0 : uncore_nb->num_counters = num_counters_nb;
381 0 : uncore_nb->rdpmc_base = RDPMC_BASE_NB;
382 0 : uncore_nb->msr_base = MSR_F15H_NB_PERF_CTL;
383 0 : uncore_nb->active_mask = &amd_nb_active_mask;
384 0 : uncore_nb->pmu = &amd_nb_pmu;
385 0 : uncore_nb->id = -1;
386 0 : *per_cpu_ptr(amd_uncore_nb, cpu) = uncore_nb;
387 : }
388 :
389 0 : if (amd_uncore_llc) {
390 0 : uncore_llc = amd_uncore_alloc(cpu);
391 0 : if (!uncore_llc)
392 0 : goto fail;
393 0 : uncore_llc->cpu = cpu;
394 0 : uncore_llc->num_counters = num_counters_llc;
395 0 : uncore_llc->rdpmc_base = RDPMC_BASE_LLC;
396 0 : uncore_llc->msr_base = MSR_F16H_L2I_PERF_CTL;
397 0 : uncore_llc->active_mask = &amd_llc_active_mask;
398 0 : uncore_llc->pmu = &amd_llc_pmu;
399 0 : uncore_llc->id = -1;
400 0 : *per_cpu_ptr(amd_uncore_llc, cpu) = uncore_llc;
401 : }
402 :
403 : return 0;
404 :
405 0 : fail:
406 0 : if (amd_uncore_nb)
407 0 : *per_cpu_ptr(amd_uncore_nb, cpu) = NULL;
408 0 : kfree(uncore_nb);
409 0 : return -ENOMEM;
410 : }
411 :
412 : static struct amd_uncore *
413 0 : amd_uncore_find_online_sibling(struct amd_uncore *this,
414 : struct amd_uncore * __percpu *uncores)
415 : {
416 0 : unsigned int cpu;
417 0 : struct amd_uncore *that;
418 :
419 0 : for_each_online_cpu(cpu) {
420 0 : that = *per_cpu_ptr(uncores, cpu);
421 :
422 0 : if (!that)
423 0 : continue;
424 :
425 0 : if (this == that)
426 0 : continue;
427 :
428 0 : if (this->id == that->id) {
429 0 : hlist_add_head(&this->node, &uncore_unused_list);
430 0 : this = that;
431 0 : break;
432 : }
433 : }
434 :
435 0 : this->refcnt++;
436 0 : return this;
437 : }
438 :
439 0 : static int amd_uncore_cpu_starting(unsigned int cpu)
440 : {
441 0 : unsigned int eax, ebx, ecx, edx;
442 0 : struct amd_uncore *uncore;
443 :
444 0 : if (amd_uncore_nb) {
445 0 : uncore = *per_cpu_ptr(amd_uncore_nb, cpu);
446 0 : cpuid(0x8000001e, &eax, &ebx, &ecx, &edx);
447 0 : uncore->id = ecx & 0xff;
448 :
449 0 : uncore = amd_uncore_find_online_sibling(uncore, amd_uncore_nb);
450 0 : *per_cpu_ptr(amd_uncore_nb, cpu) = uncore;
451 : }
452 :
453 0 : if (amd_uncore_llc) {
454 0 : uncore = *per_cpu_ptr(amd_uncore_llc, cpu);
455 0 : uncore->id = per_cpu(cpu_llc_id, cpu);
456 :
457 0 : uncore = amd_uncore_find_online_sibling(uncore, amd_uncore_llc);
458 0 : *per_cpu_ptr(amd_uncore_llc, cpu) = uncore;
459 : }
460 :
461 0 : return 0;
462 : }
463 :
464 0 : static void uncore_clean_online(void)
465 : {
466 0 : struct amd_uncore *uncore;
467 0 : struct hlist_node *n;
468 :
469 0 : hlist_for_each_entry_safe(uncore, n, &uncore_unused_list, node) {
470 0 : hlist_del(&uncore->node);
471 0 : kfree(uncore);
472 : }
473 0 : }
474 :
475 0 : static void uncore_online(unsigned int cpu,
476 : struct amd_uncore * __percpu *uncores)
477 : {
478 0 : struct amd_uncore *uncore = *per_cpu_ptr(uncores, cpu);
479 :
480 0 : uncore_clean_online();
481 :
482 0 : if (cpu == uncore->cpu)
483 0 : cpumask_set_cpu(cpu, uncore->active_mask);
484 0 : }
485 :
486 0 : static int amd_uncore_cpu_online(unsigned int cpu)
487 : {
488 0 : if (amd_uncore_nb)
489 0 : uncore_online(cpu, amd_uncore_nb);
490 :
491 0 : if (amd_uncore_llc)
492 0 : uncore_online(cpu, amd_uncore_llc);
493 :
494 0 : return 0;
495 : }
496 :
497 0 : static void uncore_down_prepare(unsigned int cpu,
498 : struct amd_uncore * __percpu *uncores)
499 : {
500 0 : unsigned int i;
501 0 : struct amd_uncore *this = *per_cpu_ptr(uncores, cpu);
502 :
503 0 : if (this->cpu != cpu)
504 : return;
505 :
506 : /* this cpu is going down, migrate to a shared sibling if possible */
507 0 : for_each_online_cpu(i) {
508 0 : struct amd_uncore *that = *per_cpu_ptr(uncores, i);
509 :
510 0 : if (cpu == i)
511 0 : continue;
512 :
513 0 : if (this == that) {
514 0 : perf_pmu_migrate_context(this->pmu, cpu, i);
515 0 : cpumask_clear_cpu(cpu, that->active_mask);
516 0 : cpumask_set_cpu(i, that->active_mask);
517 0 : that->cpu = i;
518 0 : break;
519 : }
520 : }
521 : }
522 :
523 0 : static int amd_uncore_cpu_down_prepare(unsigned int cpu)
524 : {
525 0 : if (amd_uncore_nb)
526 0 : uncore_down_prepare(cpu, amd_uncore_nb);
527 :
528 0 : if (amd_uncore_llc)
529 0 : uncore_down_prepare(cpu, amd_uncore_llc);
530 :
531 0 : return 0;
532 : }
533 :
534 0 : static void uncore_dead(unsigned int cpu, struct amd_uncore * __percpu *uncores)
535 : {
536 0 : struct amd_uncore *uncore = *per_cpu_ptr(uncores, cpu);
537 :
538 0 : if (cpu == uncore->cpu)
539 0 : cpumask_clear_cpu(cpu, uncore->active_mask);
540 :
541 0 : if (!--uncore->refcnt)
542 0 : kfree(uncore);
543 0 : *per_cpu_ptr(uncores, cpu) = NULL;
544 0 : }
545 :
546 0 : static int amd_uncore_cpu_dead(unsigned int cpu)
547 : {
548 0 : if (amd_uncore_nb)
549 0 : uncore_dead(cpu, amd_uncore_nb);
550 :
551 0 : if (amd_uncore_llc)
552 0 : uncore_dead(cpu, amd_uncore_llc);
553 :
554 0 : return 0;
555 : }
556 :
557 1 : static int __init amd_uncore_init(void)
558 : {
559 1 : struct attribute **df_attr = amd_uncore_df_format_attr;
560 1 : struct attribute **l3_attr = amd_uncore_l3_format_attr;
561 1 : int ret = -ENODEV;
562 :
563 1 : if (boot_cpu_data.x86_vendor != X86_VENDOR_AMD &&
564 : boot_cpu_data.x86_vendor != X86_VENDOR_HYGON)
565 : return -ENODEV;
566 :
567 0 : if (!boot_cpu_has(X86_FEATURE_TOPOEXT))
568 : return -ENODEV;
569 :
570 0 : num_counters_nb = NUM_COUNTERS_NB;
571 0 : num_counters_llc = NUM_COUNTERS_L2;
572 0 : if (boot_cpu_data.x86 >= 0x17) {
573 : /*
574 : * For F17h and above, the Northbridge counters are
575 : * repurposed as Data Fabric counters. Also, L3
576 : * counters are supported too. The PMUs are exported
577 : * based on family as either L2 or L3 and NB or DF.
578 : */
579 0 : num_counters_llc = NUM_COUNTERS_L3;
580 0 : amd_nb_pmu.name = "amd_df";
581 0 : amd_llc_pmu.name = "amd_l3";
582 0 : l3_mask = true;
583 : }
584 :
585 0 : if (boot_cpu_has(X86_FEATURE_PERFCTR_NB)) {
586 0 : if (boot_cpu_data.x86 >= 0x17)
587 0 : *df_attr = &format_attr_event14.attr;
588 :
589 0 : amd_uncore_nb = alloc_percpu(struct amd_uncore *);
590 0 : if (!amd_uncore_nb) {
591 0 : ret = -ENOMEM;
592 0 : goto fail_nb;
593 : }
594 0 : ret = perf_pmu_register(&amd_nb_pmu, amd_nb_pmu.name, -1);
595 0 : if (ret)
596 0 : goto fail_nb;
597 :
598 0 : pr_info("%d %s %s counters detected\n", num_counters_nb,
599 : boot_cpu_data.x86_vendor == X86_VENDOR_HYGON ? "HYGON" : "",
600 : amd_nb_pmu.name);
601 :
602 0 : ret = 0;
603 : }
604 :
605 0 : if (boot_cpu_has(X86_FEATURE_PERFCTR_LLC)) {
606 0 : if (boot_cpu_data.x86 >= 0x19) {
607 0 : *l3_attr++ = &format_attr_event8.attr;
608 0 : *l3_attr++ = &format_attr_umask.attr;
609 0 : *l3_attr++ = &format_attr_coreid.attr;
610 0 : *l3_attr++ = &format_attr_enallslices.attr;
611 0 : *l3_attr++ = &format_attr_enallcores.attr;
612 0 : *l3_attr++ = &format_attr_sliceid.attr;
613 0 : *l3_attr++ = &format_attr_threadmask2.attr;
614 0 : } else if (boot_cpu_data.x86 >= 0x17) {
615 0 : *l3_attr++ = &format_attr_event8.attr;
616 0 : *l3_attr++ = &format_attr_umask.attr;
617 0 : *l3_attr++ = &format_attr_slicemask.attr;
618 0 : *l3_attr++ = &format_attr_threadmask8.attr;
619 : }
620 :
621 0 : amd_uncore_llc = alloc_percpu(struct amd_uncore *);
622 0 : if (!amd_uncore_llc) {
623 0 : ret = -ENOMEM;
624 0 : goto fail_llc;
625 : }
626 0 : ret = perf_pmu_register(&amd_llc_pmu, amd_llc_pmu.name, -1);
627 0 : if (ret)
628 0 : goto fail_llc;
629 :
630 0 : pr_info("%d %s %s counters detected\n", num_counters_llc,
631 : boot_cpu_data.x86_vendor == X86_VENDOR_HYGON ? "HYGON" : "",
632 : amd_llc_pmu.name);
633 0 : ret = 0;
634 : }
635 :
636 : /*
637 : * Install callbacks. Core will call them for each online cpu.
638 : */
639 0 : if (cpuhp_setup_state(CPUHP_PERF_X86_AMD_UNCORE_PREP,
640 : "perf/x86/amd/uncore:prepare",
641 : amd_uncore_cpu_up_prepare, amd_uncore_cpu_dead))
642 0 : goto fail_llc;
643 :
644 0 : if (cpuhp_setup_state(CPUHP_AP_PERF_X86_AMD_UNCORE_STARTING,
645 : "perf/x86/amd/uncore:starting",
646 : amd_uncore_cpu_starting, NULL))
647 0 : goto fail_prep;
648 0 : if (cpuhp_setup_state(CPUHP_AP_PERF_X86_AMD_UNCORE_ONLINE,
649 : "perf/x86/amd/uncore:online",
650 : amd_uncore_cpu_online,
651 : amd_uncore_cpu_down_prepare))
652 0 : goto fail_start;
653 : return 0;
654 :
655 0 : fail_start:
656 0 : cpuhp_remove_state(CPUHP_AP_PERF_X86_AMD_UNCORE_STARTING);
657 0 : fail_prep:
658 0 : cpuhp_remove_state(CPUHP_PERF_X86_AMD_UNCORE_PREP);
659 0 : fail_llc:
660 0 : if (boot_cpu_has(X86_FEATURE_PERFCTR_NB))
661 0 : perf_pmu_unregister(&amd_nb_pmu);
662 0 : if (amd_uncore_llc)
663 0 : free_percpu(amd_uncore_llc);
664 0 : fail_nb:
665 0 : if (amd_uncore_nb)
666 0 : free_percpu(amd_uncore_nb);
667 :
668 : return ret;
669 : }
670 : device_initcall(amd_uncore_init);
|
