Line data Source code
1 : // SPDX-License-Identifier: GPL-2.0
2 : /*
3 : * Performance events callchain code, extracted from core.c:
4 : *
5 : * Copyright (C) 2008 Thomas Gleixner <tglx@linutronix.de>
6 : * Copyright (C) 2008-2011 Red Hat, Inc., Ingo Molnar
7 : * Copyright (C) 2008-2011 Red Hat, Inc., Peter Zijlstra
8 : * Copyright © 2009 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com>
9 : */
10 :
11 : #include <linux/perf_event.h>
12 : #include <linux/slab.h>
13 : #include <linux/sched/task_stack.h>
14 :
15 : #include "internal.h"
16 :
17 : struct callchain_cpus_entries {
18 : struct rcu_head rcu_head;
19 : struct perf_callchain_entry *cpu_entries[];
20 : };
21 :
22 : int sysctl_perf_event_max_stack __read_mostly = PERF_MAX_STACK_DEPTH;
23 : int sysctl_perf_event_max_contexts_per_stack __read_mostly = PERF_MAX_CONTEXTS_PER_STACK;
24 :
25 0 : static inline size_t perf_callchain_entry__sizeof(void)
26 : {
27 0 : return (sizeof(struct perf_callchain_entry) +
28 0 : sizeof(__u64) * (sysctl_perf_event_max_stack +
29 : sysctl_perf_event_max_contexts_per_stack));
30 : }
31 :
32 : static DEFINE_PER_CPU(int, callchain_recursion[PERF_NR_CONTEXTS]);
33 : static atomic_t nr_callchain_events;
34 : static DEFINE_MUTEX(callchain_mutex);
35 : static struct callchain_cpus_entries *callchain_cpus_entries;
36 :
37 :
38 0 : __weak void perf_callchain_kernel(struct perf_callchain_entry_ctx *entry,
39 : struct pt_regs *regs)
40 : {
41 0 : }
42 :
43 0 : __weak void perf_callchain_user(struct perf_callchain_entry_ctx *entry,
44 : struct pt_regs *regs)
45 : {
46 0 : }
47 :
48 0 : static void release_callchain_buffers_rcu(struct rcu_head *head)
49 : {
50 0 : struct callchain_cpus_entries *entries;
51 0 : int cpu;
52 :
53 0 : entries = container_of(head, struct callchain_cpus_entries, rcu_head);
54 :
55 0 : for_each_possible_cpu(cpu)
56 0 : kfree(entries->cpu_entries[cpu]);
57 :
58 0 : kfree(entries);
59 0 : }
60 :
61 0 : static void release_callchain_buffers(void)
62 : {
63 0 : struct callchain_cpus_entries *entries;
64 :
65 0 : entries = callchain_cpus_entries;
66 0 : RCU_INIT_POINTER(callchain_cpus_entries, NULL);
67 0 : call_rcu(&entries->rcu_head, release_callchain_buffers_rcu);
68 0 : }
69 :
70 0 : static int alloc_callchain_buffers(void)
71 : {
72 0 : int cpu;
73 0 : int size;
74 0 : struct callchain_cpus_entries *entries;
75 :
76 : /*
77 : * We can't use the percpu allocation API for data that can be
78 : * accessed from NMI. Use a temporary manual per cpu allocation
79 : * until that gets sorted out.
80 : */
81 0 : size = offsetof(struct callchain_cpus_entries, cpu_entries[nr_cpu_ids]);
82 :
83 0 : entries = kzalloc(size, GFP_KERNEL);
84 0 : if (!entries)
85 : return -ENOMEM;
86 :
87 0 : size = perf_callchain_entry__sizeof() * PERF_NR_CONTEXTS;
88 :
89 0 : for_each_possible_cpu(cpu) {
90 0 : entries->cpu_entries[cpu] = kmalloc_node(size, GFP_KERNEL,
91 : cpu_to_node(cpu));
92 0 : if (!entries->cpu_entries[cpu])
93 0 : goto fail;
94 : }
95 :
96 0 : rcu_assign_pointer(callchain_cpus_entries, entries);
97 :
98 0 : return 0;
99 :
100 0 : fail:
101 0 : for_each_possible_cpu(cpu)
102 0 : kfree(entries->cpu_entries[cpu]);
103 0 : kfree(entries);
104 :
105 0 : return -ENOMEM;
106 : }
107 :
108 0 : int get_callchain_buffers(int event_max_stack)
109 : {
110 0 : int err = 0;
111 0 : int count;
112 :
113 0 : mutex_lock(&callchain_mutex);
114 :
115 0 : count = atomic_inc_return(&nr_callchain_events);
116 0 : if (WARN_ON_ONCE(count < 1)) {
117 0 : err = -EINVAL;
118 0 : goto exit;
119 : }
120 :
121 : /*
122 : * If requesting per event more than the global cap,
123 : * return a different error to help userspace figure
124 : * this out.
125 : *
126 : * And also do it here so that we have &callchain_mutex held.
127 : */
128 0 : if (event_max_stack > sysctl_perf_event_max_stack) {
129 0 : err = -EOVERFLOW;
130 0 : goto exit;
131 : }
132 :
133 0 : if (count == 1)
134 0 : err = alloc_callchain_buffers();
135 0 : exit:
136 0 : if (err)
137 0 : atomic_dec(&nr_callchain_events);
138 :
139 0 : mutex_unlock(&callchain_mutex);
140 :
141 0 : return err;
142 : }
143 :
144 0 : void put_callchain_buffers(void)
145 : {
146 0 : if (atomic_dec_and_mutex_lock(&nr_callchain_events, &callchain_mutex)) {
147 0 : release_callchain_buffers();
148 0 : mutex_unlock(&callchain_mutex);
149 : }
150 0 : }
151 :
152 0 : struct perf_callchain_entry *get_callchain_entry(int *rctx)
153 : {
154 0 : int cpu;
155 0 : struct callchain_cpus_entries *entries;
156 :
157 0 : *rctx = get_recursion_context(this_cpu_ptr(callchain_recursion));
158 0 : if (*rctx == -1)
159 : return NULL;
160 :
161 0 : entries = rcu_dereference(callchain_cpus_entries);
162 0 : if (!entries) {
163 0 : put_recursion_context(this_cpu_ptr(callchain_recursion), *rctx);
164 0 : return NULL;
165 : }
166 :
167 0 : cpu = smp_processor_id();
168 :
169 0 : return (((void *)entries->cpu_entries[cpu]) +
170 0 : (*rctx * perf_callchain_entry__sizeof()));
171 : }
172 :
173 : void
174 0 : put_callchain_entry(int rctx)
175 : {
176 0 : put_recursion_context(this_cpu_ptr(callchain_recursion), rctx);
177 0 : }
178 :
179 : struct perf_callchain_entry *
180 0 : get_perf_callchain(struct pt_regs *regs, u32 init_nr, bool kernel, bool user,
181 : u32 max_stack, bool crosstask, bool add_mark)
182 : {
183 0 : struct perf_callchain_entry *entry;
184 0 : struct perf_callchain_entry_ctx ctx;
185 0 : int rctx;
186 :
187 0 : entry = get_callchain_entry(&rctx);
188 0 : if (!entry)
189 : return NULL;
190 :
191 0 : ctx.entry = entry;
192 0 : ctx.max_stack = max_stack;
193 0 : ctx.nr = entry->nr = init_nr;
194 0 : ctx.contexts = 0;
195 0 : ctx.contexts_maxed = false;
196 :
197 0 : if (kernel && !user_mode(regs)) {
198 0 : if (add_mark)
199 0 : perf_callchain_store_context(&ctx, PERF_CONTEXT_KERNEL);
200 0 : perf_callchain_kernel(&ctx, regs);
201 : }
202 :
203 0 : if (user) {
204 0 : if (!user_mode(regs)) {
205 0 : if (current->mm)
206 0 : regs = task_pt_regs(current);
207 : else
208 : regs = NULL;
209 : }
210 :
211 0 : if (regs) {
212 0 : mm_segment_t fs;
213 :
214 0 : if (crosstask)
215 0 : goto exit_put;
216 :
217 0 : if (add_mark)
218 0 : perf_callchain_store_context(&ctx, PERF_CONTEXT_USER);
219 :
220 0 : fs = force_uaccess_begin();
221 0 : perf_callchain_user(&ctx, regs);
222 0 : force_uaccess_end(fs);
223 : }
224 : }
225 :
226 0 : exit_put:
227 0 : put_callchain_entry(rctx);
228 :
229 0 : return entry;
230 : }
231 :
232 : /*
233 : * Used for sysctl_perf_event_max_stack and
234 : * sysctl_perf_event_max_contexts_per_stack.
235 : */
236 0 : int perf_event_max_stack_handler(struct ctl_table *table, int write,
237 : void *buffer, size_t *lenp, loff_t *ppos)
238 : {
239 0 : int *value = table->data;
240 0 : int new_value = *value, ret;
241 0 : struct ctl_table new_table = *table;
242 :
243 0 : new_table.data = &new_value;
244 0 : ret = proc_dointvec_minmax(&new_table, write, buffer, lenp, ppos);
245 0 : if (ret || !write)
246 : return ret;
247 :
248 0 : mutex_lock(&callchain_mutex);
249 0 : if (atomic_read(&nr_callchain_events))
250 : ret = -EBUSY;
251 : else
252 0 : *value = new_value;
253 :
254 0 : mutex_unlock(&callchain_mutex);
255 :
256 0 : return ret;
257 : }
|
