Line data Source code
1 : /* SPDX-License-Identifier: GPL-2.0 */
2 : #ifndef __LINUX_SPINLOCK_H
3 : #define __LINUX_SPINLOCK_H
4 :
5 : /*
6 : * include/linux/spinlock.h - generic spinlock/rwlock declarations
7 : *
8 : * here's the role of the various spinlock/rwlock related include files:
9 : *
10 : * on SMP builds:
11 : *
12 : * asm/spinlock_types.h: contains the arch_spinlock_t/arch_rwlock_t and the
13 : * initializers
14 : *
15 : * linux/spinlock_types.h:
16 : * defines the generic type and initializers
17 : *
18 : * asm/spinlock.h: contains the arch_spin_*()/etc. lowlevel
19 : * implementations, mostly inline assembly code
20 : *
21 : * (also included on UP-debug builds:)
22 : *
23 : * linux/spinlock_api_smp.h:
24 : * contains the prototypes for the _spin_*() APIs.
25 : *
26 : * linux/spinlock.h: builds the final spin_*() APIs.
27 : *
28 : * on UP builds:
29 : *
30 : * linux/spinlock_type_up.h:
31 : * contains the generic, simplified UP spinlock type.
32 : * (which is an empty structure on non-debug builds)
33 : *
34 : * linux/spinlock_types.h:
35 : * defines the generic type and initializers
36 : *
37 : * linux/spinlock_up.h:
38 : * contains the arch_spin_*()/etc. version of UP
39 : * builds. (which are NOPs on non-debug, non-preempt
40 : * builds)
41 : *
42 : * (included on UP-non-debug builds:)
43 : *
44 : * linux/spinlock_api_up.h:
45 : * builds the _spin_*() APIs.
46 : *
47 : * linux/spinlock.h: builds the final spin_*() APIs.
48 : */
49 :
50 : #include <linux/typecheck.h>
51 : #include <linux/preempt.h>
52 : #include <linux/linkage.h>
53 : #include <linux/compiler.h>
54 : #include <linux/irqflags.h>
55 : #include <linux/thread_info.h>
56 : #include <linux/kernel.h>
57 : #include <linux/stringify.h>
58 : #include <linux/bottom_half.h>
59 : #include <linux/lockdep.h>
60 : #include <asm/barrier.h>
61 : #include <asm/mmiowb.h>
62 :
63 :
64 : /*
65 : * Must define these before including other files, inline functions need them
66 : */
67 : #define LOCK_SECTION_NAME ".text..lock."KBUILD_BASENAME
68 :
69 : #define LOCK_SECTION_START(extra) \
70 : ".subsection 1\n\t" \
71 : extra \
72 : ".ifndef " LOCK_SECTION_NAME "\n\t" \
73 : LOCK_SECTION_NAME ":\n\t" \
74 : ".endif\n"
75 :
76 : #define LOCK_SECTION_END \
77 : ".previous\n\t"
78 :
79 : #define __lockfunc __section(".spinlock.text")
80 :
81 : /*
82 : * Pull the arch_spinlock_t and arch_rwlock_t definitions:
83 : */
84 : #include <linux/spinlock_types.h>
85 :
86 : /*
87 : * Pull the arch_spin*() functions/declarations (UP-nondebug doesn't need them):
88 : */
89 : #ifdef CONFIG_SMP
90 : # include <asm/spinlock.h>
91 : #else
92 : # include <linux/spinlock_up.h>
93 : #endif
94 :
95 : #ifdef CONFIG_DEBUG_SPINLOCK
96 : extern void __raw_spin_lock_init(raw_spinlock_t *lock, const char *name,
97 : struct lock_class_key *key, short inner);
98 :
99 : # define raw_spin_lock_init(lock) \
100 : do { \
101 : static struct lock_class_key __key; \
102 : \
103 : __raw_spin_lock_init((lock), #lock, &__key, LD_WAIT_SPIN); \
104 : } while (0)
105 :
106 : #else
107 : # define raw_spin_lock_init(lock) \
108 : do { *(lock) = __RAW_SPIN_LOCK_UNLOCKED(lock); } while (0)
109 : #endif
110 :
111 : #define raw_spin_is_locked(lock) arch_spin_is_locked(&(lock)->raw_lock)
112 :
113 : #ifdef arch_spin_is_contended
114 : #define raw_spin_is_contended(lock) arch_spin_is_contended(&(lock)->raw_lock)
115 : #else
116 : #define raw_spin_is_contended(lock) (((void)(lock), 0))
117 : #endif /*arch_spin_is_contended*/
118 :
119 : /*
120 : * smp_mb__after_spinlock() provides the equivalent of a full memory barrier
121 : * between program-order earlier lock acquisitions and program-order later
122 : * memory accesses.
123 : *
124 : * This guarantees that the following two properties hold:
125 : *
126 : * 1) Given the snippet:
127 : *
128 : * { X = 0; Y = 0; }
129 : *
130 : * CPU0 CPU1
131 : *
132 : * WRITE_ONCE(X, 1); WRITE_ONCE(Y, 1);
133 : * spin_lock(S); smp_mb();
134 : * smp_mb__after_spinlock(); r1 = READ_ONCE(X);
135 : * r0 = READ_ONCE(Y);
136 : * spin_unlock(S);
137 : *
138 : * it is forbidden that CPU0 does not observe CPU1's store to Y (r0 = 0)
139 : * and CPU1 does not observe CPU0's store to X (r1 = 0); see the comments
140 : * preceding the call to smp_mb__after_spinlock() in __schedule() and in
141 : * try_to_wake_up().
142 : *
143 : * 2) Given the snippet:
144 : *
145 : * { X = 0; Y = 0; }
146 : *
147 : * CPU0 CPU1 CPU2
148 : *
149 : * spin_lock(S); spin_lock(S); r1 = READ_ONCE(Y);
150 : * WRITE_ONCE(X, 1); smp_mb__after_spinlock(); smp_rmb();
151 : * spin_unlock(S); r0 = READ_ONCE(X); r2 = READ_ONCE(X);
152 : * WRITE_ONCE(Y, 1);
153 : * spin_unlock(S);
154 : *
155 : * it is forbidden that CPU0's critical section executes before CPU1's
156 : * critical section (r0 = 1), CPU2 observes CPU1's store to Y (r1 = 1)
157 : * and CPU2 does not observe CPU0's store to X (r2 = 0); see the comments
158 : * preceding the calls to smp_rmb() in try_to_wake_up() for similar
159 : * snippets but "projected" onto two CPUs.
160 : *
161 : * Property (2) upgrades the lock to an RCsc lock.
162 : *
163 : * Since most load-store architectures implement ACQUIRE with an smp_mb() after
164 : * the LL/SC loop, they need no further barriers. Similarly all our TSO
165 : * architectures imply an smp_mb() for each atomic instruction and equally don't
166 : * need more.
167 : *
168 : * Architectures that can implement ACQUIRE better need to take care.
169 : */
170 : #ifndef smp_mb__after_spinlock
171 : #define smp_mb__after_spinlock() do { } while (0)
172 : #endif
173 :
174 : #ifdef CONFIG_DEBUG_SPINLOCK
175 : extern void do_raw_spin_lock(raw_spinlock_t *lock) __acquires(lock);
176 : #define do_raw_spin_lock_flags(lock, flags) do_raw_spin_lock(lock)
177 : extern int do_raw_spin_trylock(raw_spinlock_t *lock);
178 : extern void do_raw_spin_unlock(raw_spinlock_t *lock) __releases(lock);
179 : #else
180 : static inline void do_raw_spin_lock(raw_spinlock_t *lock) __acquires(lock)
181 : {
182 : __acquire(lock);
183 : arch_spin_lock(&lock->raw_lock);
184 : mmiowb_spin_lock();
185 : }
186 :
187 : #ifndef arch_spin_lock_flags
188 : #define arch_spin_lock_flags(lock, flags) arch_spin_lock(lock)
189 : #endif
190 :
191 : static inline void
192 : do_raw_spin_lock_flags(raw_spinlock_t *lock, unsigned long *flags) __acquires(lock)
193 : {
194 : __acquire(lock);
195 : arch_spin_lock_flags(&lock->raw_lock, *flags);
196 : mmiowb_spin_lock();
197 : }
198 :
199 : static inline int do_raw_spin_trylock(raw_spinlock_t *lock)
200 : {
201 : int ret = arch_spin_trylock(&(lock)->raw_lock);
202 :
203 : if (ret)
204 : mmiowb_spin_lock();
205 :
206 : return ret;
207 : }
208 :
209 : static inline void do_raw_spin_unlock(raw_spinlock_t *lock) __releases(lock)
210 : {
211 : mmiowb_spin_unlock();
212 : arch_spin_unlock(&lock->raw_lock);
213 : __release(lock);
214 : }
215 : #endif
216 :
217 : /*
218 : * Define the various spin_lock methods. Note we define these
219 : * regardless of whether CONFIG_SMP or CONFIG_PREEMPTION are set. The
220 : * various methods are defined as nops in the case they are not
221 : * required.
222 : */
223 : #define raw_spin_trylock(lock) __cond_lock(lock, _raw_spin_trylock(lock))
224 :
225 : #define raw_spin_lock(lock) _raw_spin_lock(lock)
226 :
227 : #ifdef CONFIG_DEBUG_LOCK_ALLOC
228 : # define raw_spin_lock_nested(lock, subclass) \
229 : _raw_spin_lock_nested(lock, subclass)
230 :
231 : # define raw_spin_lock_nest_lock(lock, nest_lock) \
232 : do { \
233 : typecheck(struct lockdep_map *, &(nest_lock)->dep_map);\
234 : _raw_spin_lock_nest_lock(lock, &(nest_lock)->dep_map); \
235 : } while (0)
236 : #else
237 : /*
238 : * Always evaluate the 'subclass' argument to avoid that the compiler
239 : * warns about set-but-not-used variables when building with
240 : * CONFIG_DEBUG_LOCK_ALLOC=n and with W=1.
241 : */
242 : # define raw_spin_lock_nested(lock, subclass) \
243 : _raw_spin_lock(((void)(subclass), (lock)))
244 : # define raw_spin_lock_nest_lock(lock, nest_lock) _raw_spin_lock(lock)
245 : #endif
246 :
247 : #if defined(CONFIG_SMP) || defined(CONFIG_DEBUG_SPINLOCK)
248 :
249 : #define raw_spin_lock_irqsave(lock, flags) \
250 : do { \
251 : typecheck(unsigned long, flags); \
252 : flags = _raw_spin_lock_irqsave(lock); \
253 : } while (0)
254 :
255 : #ifdef CONFIG_DEBUG_LOCK_ALLOC
256 : #define raw_spin_lock_irqsave_nested(lock, flags, subclass) \
257 : do { \
258 : typecheck(unsigned long, flags); \
259 : flags = _raw_spin_lock_irqsave_nested(lock, subclass); \
260 : } while (0)
261 : #else
262 : #define raw_spin_lock_irqsave_nested(lock, flags, subclass) \
263 : do { \
264 : typecheck(unsigned long, flags); \
265 : flags = _raw_spin_lock_irqsave(lock); \
266 : } while (0)
267 : #endif
268 :
269 : #else
270 :
271 : #define raw_spin_lock_irqsave(lock, flags) \
272 : do { \
273 : typecheck(unsigned long, flags); \
274 : _raw_spin_lock_irqsave(lock, flags); \
275 : } while (0)
276 :
277 : #define raw_spin_lock_irqsave_nested(lock, flags, subclass) \
278 : raw_spin_lock_irqsave(lock, flags)
279 :
280 : #endif
281 :
282 : #define raw_spin_lock_irq(lock) _raw_spin_lock_irq(lock)
283 : #define raw_spin_lock_bh(lock) _raw_spin_lock_bh(lock)
284 : #define raw_spin_unlock(lock) _raw_spin_unlock(lock)
285 : #define raw_spin_unlock_irq(lock) _raw_spin_unlock_irq(lock)
286 :
287 : #define raw_spin_unlock_irqrestore(lock, flags) \
288 : do { \
289 : typecheck(unsigned long, flags); \
290 : _raw_spin_unlock_irqrestore(lock, flags); \
291 : } while (0)
292 : #define raw_spin_unlock_bh(lock) _raw_spin_unlock_bh(lock)
293 :
294 : #define raw_spin_trylock_bh(lock) \
295 : __cond_lock(lock, _raw_spin_trylock_bh(lock))
296 :
297 : #define raw_spin_trylock_irq(lock) \
298 : ({ \
299 : local_irq_disable(); \
300 : raw_spin_trylock(lock) ? \
301 : 1 : ({ local_irq_enable(); 0; }); \
302 : })
303 :
304 : #define raw_spin_trylock_irqsave(lock, flags) \
305 : ({ \
306 : local_irq_save(flags); \
307 : raw_spin_trylock(lock) ? \
308 : 1 : ({ local_irq_restore(flags); 0; }); \
309 : })
310 :
311 : /* Include rwlock functions */
312 : #include <linux/rwlock.h>
313 :
314 : /*
315 : * Pull the _spin_*()/_read_*()/_write_*() functions/declarations:
316 : */
317 : #if defined(CONFIG_SMP) || defined(CONFIG_DEBUG_SPINLOCK)
318 : # include <linux/spinlock_api_smp.h>
319 : #else
320 : # include <linux/spinlock_api_up.h>
321 : #endif
322 :
323 : /*
324 : * Map the spin_lock functions to the raw variants for PREEMPT_RT=n
325 : */
326 :
327 851867 : static __always_inline raw_spinlock_t *spinlock_check(spinlock_t *lock)
328 : {
329 848821 : return &lock->rlock;
330 : }
331 :
332 : #ifdef CONFIG_DEBUG_SPINLOCK
333 :
334 : # define spin_lock_init(lock) \
335 : do { \
336 : static struct lock_class_key __key; \
337 : \
338 : __raw_spin_lock_init(spinlock_check(lock), \
339 : #lock, &__key, LD_WAIT_CONFIG); \
340 : } while (0)
341 :
342 : #else
343 :
344 : # define spin_lock_init(_lock) \
345 : do { \
346 : spinlock_check(_lock); \
347 : *(_lock) = __SPIN_LOCK_UNLOCKED(_lock); \
348 : } while (0)
349 :
350 : #endif
351 :
352 1599547 : static __always_inline void spin_lock(spinlock_t *lock)
353 : {
354 1592700 : raw_spin_lock(&lock->rlock);
355 69217 : }
356 :
357 2590 : static __always_inline void spin_lock_bh(spinlock_t *lock)
358 : {
359 2590 : raw_spin_lock_bh(&lock->rlock);
360 141 : }
361 :
362 21526 : static __always_inline int spin_trylock(spinlock_t *lock)
363 : {
364 21526 : return raw_spin_trylock(&lock->rlock);
365 : }
366 :
367 : #define spin_lock_nested(lock, subclass) \
368 : do { \
369 : raw_spin_lock_nested(spinlock_check(lock), subclass); \
370 : } while (0)
371 :
372 : #define spin_lock_nest_lock(lock, nest_lock) \
373 : do { \
374 : raw_spin_lock_nest_lock(spinlock_check(lock), nest_lock); \
375 : } while (0)
376 :
377 36075 : static __always_inline void spin_lock_irq(spinlock_t *lock)
378 : {
379 36075 : raw_spin_lock_irq(&lock->rlock);
380 2 : }
381 :
382 : #define spin_lock_irqsave(lock, flags) \
383 : do { \
384 : raw_spin_lock_irqsave(spinlock_check(lock), flags); \
385 : } while (0)
386 :
387 : #define spin_lock_irqsave_nested(lock, flags, subclass) \
388 : do { \
389 : raw_spin_lock_irqsave_nested(spinlock_check(lock), flags, subclass); \
390 : } while (0)
391 :
392 1682558 : static __always_inline void spin_unlock(spinlock_t *lock)
393 : {
394 1671319 : raw_spin_unlock(&lock->rlock);
395 109274 : }
396 :
397 1475 : static __always_inline void spin_unlock_bh(spinlock_t *lock)
398 : {
399 1475 : raw_spin_unlock_bh(&lock->rlock);
400 270 : }
401 :
402 36076 : static __always_inline void spin_unlock_irq(spinlock_t *lock)
403 : {
404 36076 : raw_spin_unlock_irq(&lock->rlock);
405 4148 : }
406 :
407 410607 : static __always_inline void spin_unlock_irqrestore(spinlock_t *lock, unsigned long flags)
408 : {
409 410607 : raw_spin_unlock_irqrestore(&lock->rlock, flags);
410 233404 : }
411 :
412 : static __always_inline int spin_trylock_bh(spinlock_t *lock)
413 : {
414 : return raw_spin_trylock_bh(&lock->rlock);
415 : }
416 :
417 : static __always_inline int spin_trylock_irq(spinlock_t *lock)
418 : {
419 : return raw_spin_trylock_irq(&lock->rlock);
420 : }
421 :
422 : #define spin_trylock_irqsave(lock, flags) \
423 : ({ \
424 : raw_spin_trylock_irqsave(spinlock_check(lock), flags); \
425 : })
426 :
427 : /**
428 : * spin_is_locked() - Check whether a spinlock is locked.
429 : * @lock: Pointer to the spinlock.
430 : *
431 : * This function is NOT required to provide any memory ordering
432 : * guarantees; it could be used for debugging purposes or, when
433 : * additional synchronization is needed, accompanied with other
434 : * constructs (memory barriers) enforcing the synchronization.
435 : *
436 : * Returns: 1 if @lock is locked, 0 otherwise.
437 : *
438 : * Note that the function only tells you that the spinlock is
439 : * seen to be locked, not that it is locked on your CPU.
440 : *
441 : * Further, on CONFIG_SMP=n builds with CONFIG_DEBUG_SPINLOCK=n,
442 : * the return value is always 0 (see include/linux/spinlock_up.h).
443 : * Therefore you should not rely heavily on the return value.
444 : */
445 0 : static __always_inline int spin_is_locked(spinlock_t *lock)
446 : {
447 0 : return raw_spin_is_locked(&lock->rlock);
448 : }
449 :
450 : static __always_inline int spin_is_contended(spinlock_t *lock)
451 : {
452 : return raw_spin_is_contended(&lock->rlock);
453 : }
454 :
455 : #define assert_spin_locked(lock) assert_raw_spin_locked(&(lock)->rlock)
456 :
457 : /*
458 : * Pull the atomic_t declaration:
459 : * (asm-mips/atomic.h needs above definitions)
460 : */
461 : #include <linux/atomic.h>
462 : /**
463 : * atomic_dec_and_lock - lock on reaching reference count zero
464 : * @atomic: the atomic counter
465 : * @lock: the spinlock in question
466 : *
467 : * Decrements @atomic by 1. If the result is 0, returns true and locks
468 : * @lock. Returns false for all other cases.
469 : */
470 : extern int _atomic_dec_and_lock(atomic_t *atomic, spinlock_t *lock);
471 : #define atomic_dec_and_lock(atomic, lock) \
472 : __cond_lock(lock, _atomic_dec_and_lock(atomic, lock))
473 :
474 : extern int _atomic_dec_and_lock_irqsave(atomic_t *atomic, spinlock_t *lock,
475 : unsigned long *flags);
476 : #define atomic_dec_and_lock_irqsave(atomic, lock, flags) \
477 : __cond_lock(lock, _atomic_dec_and_lock_irqsave(atomic, lock, &(flags)))
478 :
479 : int __alloc_bucket_spinlocks(spinlock_t **locks, unsigned int *lock_mask,
480 : size_t max_size, unsigned int cpu_mult,
481 : gfp_t gfp, const char *name,
482 : struct lock_class_key *key);
483 :
484 : #define alloc_bucket_spinlocks(locks, lock_mask, max_size, cpu_mult, gfp) \
485 : ({ \
486 : static struct lock_class_key key; \
487 : int ret; \
488 : \
489 : ret = __alloc_bucket_spinlocks(locks, lock_mask, max_size, \
490 : cpu_mult, gfp, #locks, &key); \
491 : ret; \
492 : })
493 :
494 : void free_bucket_spinlocks(spinlock_t *locks);
495 :
496 : #endif /* __LINUX_SPINLOCK_H */
|
