Line data Source code
1 : // SPDX-License-Identifier: GPL-2.0-only
2 : /*
3 : * async.c: Asynchronous function calls for boot performance
4 : *
5 : * (C) Copyright 2009 Intel Corporation
6 : * Author: Arjan van de Ven <arjan@linux.intel.com>
7 : */
8 :
9 :
10 : /*
11 :
12 : Goals and Theory of Operation
13 :
14 : The primary goal of this feature is to reduce the kernel boot time,
15 : by doing various independent hardware delays and discovery operations
16 : decoupled and not strictly serialized.
17 :
18 : More specifically, the asynchronous function call concept allows
19 : certain operations (primarily during system boot) to happen
20 : asynchronously, out of order, while these operations still
21 : have their externally visible parts happen sequentially and in-order.
22 : (not unlike how out-of-order CPUs retire their instructions in order)
23 :
24 : Key to the asynchronous function call implementation is the concept of
25 : a "sequence cookie" (which, although it has an abstracted type, can be
26 : thought of as a monotonically incrementing number).
27 :
28 : The async core will assign each scheduled event such a sequence cookie and
29 : pass this to the called functions.
30 :
31 : The asynchronously called function should before doing a globally visible
32 : operation, such as registering device numbers, call the
33 : async_synchronize_cookie() function and pass in its own cookie. The
34 : async_synchronize_cookie() function will make sure that all asynchronous
35 : operations that were scheduled prior to the operation corresponding with the
36 : cookie have completed.
37 :
38 : Subsystem/driver initialization code that scheduled asynchronous probe
39 : functions, but which shares global resources with other drivers/subsystems
40 : that do not use the asynchronous call feature, need to do a full
41 : synchronization with the async_synchronize_full() function, before returning
42 : from their init function. This is to maintain strict ordering between the
43 : asynchronous and synchronous parts of the kernel.
44 :
45 : */
46 :
47 : #include <linux/async.h>
48 : #include <linux/atomic.h>
49 : #include <linux/ktime.h>
50 : #include <linux/export.h>
51 : #include <linux/wait.h>
52 : #include <linux/sched.h>
53 : #include <linux/slab.h>
54 : #include <linux/workqueue.h>
55 :
56 : #include "workqueue_internal.h"
57 :
58 : static async_cookie_t next_cookie = 1;
59 :
60 : #define MAX_WORK 32768
61 : #define ASYNC_COOKIE_MAX ULLONG_MAX /* infinity cookie */
62 :
63 : static LIST_HEAD(async_global_pending); /* pending from all registered doms */
64 : static ASYNC_DOMAIN(async_dfl_domain);
65 : static DEFINE_SPINLOCK(async_lock);
66 :
67 : struct async_entry {
68 : struct list_head domain_list;
69 : struct list_head global_list;
70 : struct work_struct work;
71 : async_cookie_t cookie;
72 : async_func_t func;
73 : void *data;
74 : struct async_domain *domain;
75 : };
76 :
77 : static DECLARE_WAIT_QUEUE_HEAD(async_done);
78 :
79 : static atomic_t entry_count;
80 :
81 2 : static async_cookie_t lowest_in_progress(struct async_domain *domain)
82 : {
83 2 : struct async_entry *first = NULL;
84 2 : async_cookie_t ret = ASYNC_COOKIE_MAX;
85 2 : unsigned long flags;
86 :
87 2 : spin_lock_irqsave(&async_lock, flags);
88 :
89 2 : if (domain) {
90 0 : if (!list_empty(&domain->pending))
91 0 : first = list_first_entry(&domain->pending,
92 : struct async_entry, domain_list);
93 : } else {
94 2 : if (!list_empty(&async_global_pending))
95 0 : first = list_first_entry(&async_global_pending,
96 : struct async_entry, global_list);
97 : }
98 :
99 0 : if (first)
100 0 : ret = first->cookie;
101 :
102 2 : spin_unlock_irqrestore(&async_lock, flags);
103 2 : return ret;
104 : }
105 :
106 : /*
107 : * pick the first pending entry and run it
108 : */
109 0 : static void async_run_entry_fn(struct work_struct *work)
110 : {
111 0 : struct async_entry *entry =
112 0 : container_of(work, struct async_entry, work);
113 0 : unsigned long flags;
114 0 : ktime_t calltime, delta, rettime;
115 :
116 : /* 1) run (and print duration) */
117 0 : if (initcall_debug && system_state < SYSTEM_RUNNING) {
118 0 : pr_debug("calling %lli_%pS @ %i\n",
119 : (long long)entry->cookie,
120 : entry->func, task_pid_nr(current));
121 0 : calltime = ktime_get();
122 : }
123 0 : entry->func(entry->data, entry->cookie);
124 0 : if (initcall_debug && system_state < SYSTEM_RUNNING) {
125 0 : rettime = ktime_get();
126 0 : delta = ktime_sub(rettime, calltime);
127 0 : pr_debug("initcall %lli_%pS returned 0 after %lld usecs\n",
128 : (long long)entry->cookie,
129 : entry->func,
130 : (long long)ktime_to_ns(delta) >> 10);
131 : }
132 :
133 : /* 2) remove self from the pending queues */
134 0 : spin_lock_irqsave(&async_lock, flags);
135 0 : list_del_init(&entry->domain_list);
136 0 : list_del_init(&entry->global_list);
137 :
138 : /* 3) free the entry */
139 0 : kfree(entry);
140 0 : atomic_dec(&entry_count);
141 :
142 0 : spin_unlock_irqrestore(&async_lock, flags);
143 :
144 : /* 4) wake up any waiters */
145 0 : wake_up(&async_done);
146 0 : }
147 :
148 : /**
149 : * async_schedule_node_domain - NUMA specific version of async_schedule_domain
150 : * @func: function to execute asynchronously
151 : * @data: data pointer to pass to the function
152 : * @node: NUMA node that we want to schedule this on or close to
153 : * @domain: the domain
154 : *
155 : * Returns an async_cookie_t that may be used for checkpointing later.
156 : * @domain may be used in the async_synchronize_*_domain() functions to
157 : * wait within a certain synchronization domain rather than globally.
158 : *
159 : * Note: This function may be called from atomic or non-atomic contexts.
160 : *
161 : * The node requested will be honored on a best effort basis. If the node
162 : * has no CPUs associated with it then the work is distributed among all
163 : * available CPUs.
164 : */
165 0 : async_cookie_t async_schedule_node_domain(async_func_t func, void *data,
166 : int node, struct async_domain *domain)
167 : {
168 0 : struct async_entry *entry;
169 0 : unsigned long flags;
170 0 : async_cookie_t newcookie;
171 :
172 : /* allow irq-off callers */
173 0 : entry = kzalloc(sizeof(struct async_entry), GFP_ATOMIC);
174 :
175 : /*
176 : * If we're out of memory or if there's too much work
177 : * pending already, we execute synchronously.
178 : */
179 0 : if (!entry || atomic_read(&entry_count) > MAX_WORK) {
180 0 : kfree(entry);
181 0 : spin_lock_irqsave(&async_lock, flags);
182 0 : newcookie = next_cookie++;
183 0 : spin_unlock_irqrestore(&async_lock, flags);
184 :
185 : /* low on memory.. run synchronously */
186 0 : func(data, newcookie);
187 0 : return newcookie;
188 : }
189 0 : INIT_LIST_HEAD(&entry->domain_list);
190 0 : INIT_LIST_HEAD(&entry->global_list);
191 0 : INIT_WORK(&entry->work, async_run_entry_fn);
192 0 : entry->func = func;
193 0 : entry->data = data;
194 0 : entry->domain = domain;
195 :
196 0 : spin_lock_irqsave(&async_lock, flags);
197 :
198 : /* allocate cookie and queue */
199 0 : newcookie = entry->cookie = next_cookie++;
200 :
201 0 : list_add_tail(&entry->domain_list, &domain->pending);
202 0 : if (domain->registered)
203 0 : list_add_tail(&entry->global_list, &async_global_pending);
204 :
205 0 : atomic_inc(&entry_count);
206 0 : spin_unlock_irqrestore(&async_lock, flags);
207 :
208 : /* mark that this task has queued an async job, used by module init */
209 0 : current->flags |= PF_USED_ASYNC;
210 :
211 : /* schedule for execution */
212 0 : queue_work_node(node, system_unbound_wq, &entry->work);
213 :
214 0 : return newcookie;
215 : }
216 : EXPORT_SYMBOL_GPL(async_schedule_node_domain);
217 :
218 : /**
219 : * async_schedule_node - NUMA specific version of async_schedule
220 : * @func: function to execute asynchronously
221 : * @data: data pointer to pass to the function
222 : * @node: NUMA node that we want to schedule this on or close to
223 : *
224 : * Returns an async_cookie_t that may be used for checkpointing later.
225 : * Note: This function may be called from atomic or non-atomic contexts.
226 : *
227 : * The node requested will be honored on a best effort basis. If the node
228 : * has no CPUs associated with it then the work is distributed among all
229 : * available CPUs.
230 : */
231 0 : async_cookie_t async_schedule_node(async_func_t func, void *data, int node)
232 : {
233 0 : return async_schedule_node_domain(func, data, node, &async_dfl_domain);
234 : }
235 : EXPORT_SYMBOL_GPL(async_schedule_node);
236 :
237 : /**
238 : * async_synchronize_full - synchronize all asynchronous function calls
239 : *
240 : * This function waits until all asynchronous function calls have been done.
241 : */
242 2 : void async_synchronize_full(void)
243 : {
244 4 : async_synchronize_full_domain(NULL);
245 2 : }
246 : EXPORT_SYMBOL_GPL(async_synchronize_full);
247 :
248 : /**
249 : * async_unregister_domain - ensure no more anonymous waiters on this domain
250 : * @domain: idle domain to flush out of any async_synchronize_full instances
251 : *
252 : * async_synchronize_{cookie|full}_domain() are not flushed since callers
253 : * of these routines should know the lifetime of @domain
254 : *
255 : * Prefer ASYNC_DOMAIN_EXCLUSIVE() declarations over flushing
256 : */
257 0 : void async_unregister_domain(struct async_domain *domain)
258 : {
259 0 : spin_lock_irq(&async_lock);
260 0 : WARN_ON(!domain->registered || !list_empty(&domain->pending));
261 0 : domain->registered = 0;
262 0 : spin_unlock_irq(&async_lock);
263 0 : }
264 : EXPORT_SYMBOL_GPL(async_unregister_domain);
265 :
266 : /**
267 : * async_synchronize_full_domain - synchronize all asynchronous function within a certain domain
268 : * @domain: the domain to synchronize
269 : *
270 : * This function waits until all asynchronous function calls for the
271 : * synchronization domain specified by @domain have been done.
272 : */
273 2 : void async_synchronize_full_domain(struct async_domain *domain)
274 : {
275 2 : async_synchronize_cookie_domain(ASYNC_COOKIE_MAX, domain);
276 0 : }
277 : EXPORT_SYMBOL_GPL(async_synchronize_full_domain);
278 :
279 : /**
280 : * async_synchronize_cookie_domain - synchronize asynchronous function calls within a certain domain with cookie checkpointing
281 : * @cookie: async_cookie_t to use as checkpoint
282 : * @domain: the domain to synchronize (%NULL for all registered domains)
283 : *
284 : * This function waits until all asynchronous function calls for the
285 : * synchronization domain specified by @domain submitted prior to @cookie
286 : * have been done.
287 : */
288 2 : void async_synchronize_cookie_domain(async_cookie_t cookie, struct async_domain *domain)
289 : {
290 2 : ktime_t starttime, delta, endtime;
291 :
292 2 : if (initcall_debug && system_state < SYSTEM_RUNNING) {
293 0 : pr_debug("async_waiting @ %i\n", task_pid_nr(current));
294 0 : starttime = ktime_get();
295 : }
296 :
297 2 : wait_event(async_done, lowest_in_progress(domain) >= cookie);
298 :
299 2 : if (initcall_debug && system_state < SYSTEM_RUNNING) {
300 0 : endtime = ktime_get();
301 0 : delta = ktime_sub(endtime, starttime);
302 :
303 0 : pr_debug("async_continuing @ %i after %lli usec\n",
304 : task_pid_nr(current),
305 : (long long)ktime_to_ns(delta) >> 10);
306 : }
307 2 : }
308 : EXPORT_SYMBOL_GPL(async_synchronize_cookie_domain);
309 :
310 : /**
311 : * async_synchronize_cookie - synchronize asynchronous function calls with cookie checkpointing
312 : * @cookie: async_cookie_t to use as checkpoint
313 : *
314 : * This function waits until all asynchronous function calls prior to @cookie
315 : * have been done.
316 : */
317 0 : void async_synchronize_cookie(async_cookie_t cookie)
318 : {
319 0 : async_synchronize_cookie_domain(cookie, &async_dfl_domain);
320 0 : }
321 : EXPORT_SYMBOL_GPL(async_synchronize_cookie);
322 :
323 : /**
324 : * current_is_async - is %current an async worker task?
325 : *
326 : * Returns %true if %current is an async worker task.
327 : */
328 0 : bool current_is_async(void)
329 : {
330 0 : struct worker *worker = current_wq_worker();
331 :
332 0 : return worker && worker->current_func == async_run_entry_fn;
333 : }
334 : EXPORT_SYMBOL_GPL(current_is_async);
|
