Line data Source code
1 : // SPDX-License-Identifier: GPL-2.0
2 :
3 : // Generated by scripts/atomic/gen-atomic-instrumented.sh
4 : // DO NOT MODIFY THIS FILE DIRECTLY
5 :
6 : /*
7 : * This file provides wrappers with KASAN instrumentation for atomic operations.
8 : * To use this functionality an arch's atomic.h file needs to define all
9 : * atomic operations with arch_ prefix (e.g. arch_atomic_read()) and include
10 : * this file at the end. This file provides atomic_read() that forwards to
11 : * arch_atomic_read() for actual atomic operation.
12 : * Note: if an arch atomic operation is implemented by means of other atomic
13 : * operations (e.g. atomic_read()/atomic_cmpxchg() loop), then it needs to use
14 : * arch_ variants (i.e. arch_atomic_read()/arch_atomic_cmpxchg()) to avoid
15 : * double instrumentation.
16 : */
17 : #ifndef _ASM_GENERIC_ATOMIC_INSTRUMENTED_H
18 : #define _ASM_GENERIC_ATOMIC_INSTRUMENTED_H
19 :
20 : #include <linux/build_bug.h>
21 : #include <linux/compiler.h>
22 : #include <linux/instrumented.h>
23 :
24 : static __always_inline int
25 86826126 : atomic_read(const atomic_t *v)
26 : {
27 86845614 : instrument_atomic_read(v, sizeof(*v));
28 85449730 : return arch_atomic_read(v);
29 : }
30 : #define atomic_read atomic_read
31 :
32 : #if defined(arch_atomic_read_acquire)
33 : static __always_inline int
34 1971 : atomic_read_acquire(const atomic_t *v)
35 : {
36 1971 : instrument_atomic_read(v, sizeof(*v));
37 1971 : return arch_atomic_read_acquire(v);
38 : }
39 : #define atomic_read_acquire atomic_read_acquire
40 : #endif
41 :
42 : static __always_inline void
43 1372584 : atomic_set(atomic_t *v, int i)
44 : {
45 1909123 : instrument_atomic_write(v, sizeof(*v));
46 1710173 : arch_atomic_set(v, i);
47 79625 : }
48 : #define atomic_set atomic_set
49 :
50 : #if defined(arch_atomic_set_release)
51 : static __always_inline void
52 201 : atomic_set_release(atomic_t *v, int i)
53 : {
54 201 : instrument_atomic_write(v, sizeof(*v));
55 201 : arch_atomic_set_release(v, i);
56 0 : }
57 : #define atomic_set_release atomic_set_release
58 : #endif
59 :
60 : static __always_inline void
61 3847 : atomic_add(int i, atomic_t *v)
62 : {
63 3847 : instrument_atomic_read_write(v, sizeof(*v));
64 3847 : arch_atomic_add(i, v);
65 1044 : }
66 : #define atomic_add atomic_add
67 :
68 : #if !defined(arch_atomic_add_return_relaxed) || defined(arch_atomic_add_return)
69 : static __always_inline int
70 10654 : atomic_add_return(int i, atomic_t *v)
71 : {
72 10654 : instrument_atomic_read_write(v, sizeof(*v));
73 10651 : return arch_atomic_add_return(i, v);
74 : }
75 : #define atomic_add_return atomic_add_return
76 : #endif
77 :
78 : #if defined(arch_atomic_add_return_acquire)
79 : static __always_inline int
80 41882 : atomic_add_return_acquire(int i, atomic_t *v)
81 : {
82 41882 : instrument_atomic_read_write(v, sizeof(*v));
83 41882 : return arch_atomic_add_return_acquire(i, v);
84 : }
85 : #define atomic_add_return_acquire atomic_add_return_acquire
86 : #endif
87 :
88 : #if defined(arch_atomic_add_return_release)
89 : static __always_inline int
90 : atomic_add_return_release(int i, atomic_t *v)
91 : {
92 : instrument_atomic_read_write(v, sizeof(*v));
93 : return arch_atomic_add_return_release(i, v);
94 : }
95 : #define atomic_add_return_release atomic_add_return_release
96 : #endif
97 :
98 : #if defined(arch_atomic_add_return_relaxed)
99 : static __always_inline int
100 : atomic_add_return_relaxed(int i, atomic_t *v)
101 : {
102 : instrument_atomic_read_write(v, sizeof(*v));
103 : return arch_atomic_add_return_relaxed(i, v);
104 : }
105 : #define atomic_add_return_relaxed atomic_add_return_relaxed
106 : #endif
107 :
108 : #if !defined(arch_atomic_fetch_add_relaxed) || defined(arch_atomic_fetch_add)
109 : static __always_inline int
110 0 : atomic_fetch_add(int i, atomic_t *v)
111 : {
112 0 : instrument_atomic_read_write(v, sizeof(*v));
113 0 : return arch_atomic_fetch_add(i, v);
114 : }
115 : #define atomic_fetch_add atomic_fetch_add
116 : #endif
117 :
118 : #if defined(arch_atomic_fetch_add_acquire)
119 : static __always_inline int
120 : atomic_fetch_add_acquire(int i, atomic_t *v)
121 : {
122 : instrument_atomic_read_write(v, sizeof(*v));
123 : return arch_atomic_fetch_add_acquire(i, v);
124 : }
125 : #define atomic_fetch_add_acquire atomic_fetch_add_acquire
126 : #endif
127 :
128 : #if defined(arch_atomic_fetch_add_release)
129 : static __always_inline int
130 : atomic_fetch_add_release(int i, atomic_t *v)
131 : {
132 : instrument_atomic_read_write(v, sizeof(*v));
133 : return arch_atomic_fetch_add_release(i, v);
134 : }
135 : #define atomic_fetch_add_release atomic_fetch_add_release
136 : #endif
137 :
138 : #if defined(arch_atomic_fetch_add_relaxed)
139 : static __always_inline int
140 50099 : atomic_fetch_add_relaxed(int i, atomic_t *v)
141 : {
142 50099 : instrument_atomic_read_write(v, sizeof(*v));
143 50101 : return arch_atomic_fetch_add_relaxed(i, v);
144 : }
145 : #define atomic_fetch_add_relaxed atomic_fetch_add_relaxed
146 : #endif
147 :
148 : static __always_inline void
149 15766 : atomic_sub(int i, atomic_t *v)
150 : {
151 15766 : instrument_atomic_read_write(v, sizeof(*v));
152 15766 : arch_atomic_sub(i, v);
153 54 : }
154 : #define atomic_sub atomic_sub
155 :
156 : #if !defined(arch_atomic_sub_return_relaxed) || defined(arch_atomic_sub_return)
157 : static __always_inline int
158 2363 : atomic_sub_return(int i, atomic_t *v)
159 : {
160 2363 : instrument_atomic_read_write(v, sizeof(*v));
161 2363 : return arch_atomic_sub_return(i, v);
162 : }
163 : #define atomic_sub_return atomic_sub_return
164 : #endif
165 :
166 : #if defined(arch_atomic_sub_return_acquire)
167 : static __always_inline int
168 : atomic_sub_return_acquire(int i, atomic_t *v)
169 : {
170 : instrument_atomic_read_write(v, sizeof(*v));
171 : return arch_atomic_sub_return_acquire(i, v);
172 : }
173 : #define atomic_sub_return_acquire atomic_sub_return_acquire
174 : #endif
175 :
176 : #if defined(arch_atomic_sub_return_release)
177 : static __always_inline int
178 41881 : atomic_sub_return_release(int i, atomic_t *v)
179 : {
180 41881 : instrument_atomic_read_write(v, sizeof(*v));
181 41881 : return arch_atomic_sub_return_release(i, v);
182 : }
183 : #define atomic_sub_return_release atomic_sub_return_release
184 : #endif
185 :
186 : #if defined(arch_atomic_sub_return_relaxed)
187 : static __always_inline int
188 : atomic_sub_return_relaxed(int i, atomic_t *v)
189 : {
190 : instrument_atomic_read_write(v, sizeof(*v));
191 : return arch_atomic_sub_return_relaxed(i, v);
192 : }
193 : #define atomic_sub_return_relaxed atomic_sub_return_relaxed
194 : #endif
195 :
196 : #if !defined(arch_atomic_fetch_sub_relaxed) || defined(arch_atomic_fetch_sub)
197 : static __always_inline int
198 : atomic_fetch_sub(int i, atomic_t *v)
199 : {
200 : instrument_atomic_read_write(v, sizeof(*v));
201 : return arch_atomic_fetch_sub(i, v);
202 : }
203 : #define atomic_fetch_sub atomic_fetch_sub
204 : #endif
205 :
206 : #if defined(arch_atomic_fetch_sub_acquire)
207 : static __always_inline int
208 : atomic_fetch_sub_acquire(int i, atomic_t *v)
209 : {
210 : instrument_atomic_read_write(v, sizeof(*v));
211 : return arch_atomic_fetch_sub_acquire(i, v);
212 : }
213 : #define atomic_fetch_sub_acquire atomic_fetch_sub_acquire
214 : #endif
215 :
216 : #if defined(arch_atomic_fetch_sub_release)
217 : static __always_inline int
218 54452 : atomic_fetch_sub_release(int i, atomic_t *v)
219 : {
220 54452 : instrument_atomic_read_write(v, sizeof(*v));
221 54452 : return arch_atomic_fetch_sub_release(i, v);
222 : }
223 : #define atomic_fetch_sub_release atomic_fetch_sub_release
224 : #endif
225 :
226 : #if defined(arch_atomic_fetch_sub_relaxed)
227 : static __always_inline int
228 : atomic_fetch_sub_relaxed(int i, atomic_t *v)
229 : {
230 : instrument_atomic_read_write(v, sizeof(*v));
231 : return arch_atomic_fetch_sub_relaxed(i, v);
232 : }
233 : #define atomic_fetch_sub_relaxed atomic_fetch_sub_relaxed
234 : #endif
235 :
236 : #if defined(arch_atomic_inc)
237 : static __always_inline void
238 612636 : atomic_inc(atomic_t *v)
239 : {
240 612636 : instrument_atomic_read_write(v, sizeof(*v));
241 606376 : arch_atomic_inc(v);
242 50153 : }
243 : #define atomic_inc atomic_inc
244 : #endif
245 :
246 : #if defined(arch_atomic_inc_return)
247 : static __always_inline int
248 2072 : atomic_inc_return(atomic_t *v)
249 : {
250 2072 : instrument_atomic_read_write(v, sizeof(*v));
251 2048 : return arch_atomic_inc_return(v);
252 : }
253 : #define atomic_inc_return atomic_inc_return
254 : #endif
255 :
256 : #if defined(arch_atomic_inc_return_acquire)
257 : static __always_inline int
258 : atomic_inc_return_acquire(atomic_t *v)
259 : {
260 : instrument_atomic_read_write(v, sizeof(*v));
261 : return arch_atomic_inc_return_acquire(v);
262 : }
263 : #define atomic_inc_return_acquire atomic_inc_return_acquire
264 : #endif
265 :
266 : #if defined(arch_atomic_inc_return_release)
267 : static __always_inline int
268 : atomic_inc_return_release(atomic_t *v)
269 : {
270 : instrument_atomic_read_write(v, sizeof(*v));
271 : return arch_atomic_inc_return_release(v);
272 : }
273 : #define atomic_inc_return_release atomic_inc_return_release
274 : #endif
275 :
276 : #if defined(arch_atomic_inc_return_relaxed)
277 : static __always_inline int
278 : atomic_inc_return_relaxed(atomic_t *v)
279 : {
280 : instrument_atomic_read_write(v, sizeof(*v));
281 : return arch_atomic_inc_return_relaxed(v);
282 : }
283 : #define atomic_inc_return_relaxed atomic_inc_return_relaxed
284 : #endif
285 :
286 : #if defined(arch_atomic_fetch_inc)
287 : static __always_inline int
288 : atomic_fetch_inc(atomic_t *v)
289 : {
290 : instrument_atomic_read_write(v, sizeof(*v));
291 : return arch_atomic_fetch_inc(v);
292 : }
293 : #define atomic_fetch_inc atomic_fetch_inc
294 : #endif
295 :
296 : #if defined(arch_atomic_fetch_inc_acquire)
297 : static __always_inline int
298 : atomic_fetch_inc_acquire(atomic_t *v)
299 : {
300 : instrument_atomic_read_write(v, sizeof(*v));
301 : return arch_atomic_fetch_inc_acquire(v);
302 : }
303 : #define atomic_fetch_inc_acquire atomic_fetch_inc_acquire
304 : #endif
305 :
306 : #if defined(arch_atomic_fetch_inc_release)
307 : static __always_inline int
308 : atomic_fetch_inc_release(atomic_t *v)
309 : {
310 : instrument_atomic_read_write(v, sizeof(*v));
311 : return arch_atomic_fetch_inc_release(v);
312 : }
313 : #define atomic_fetch_inc_release atomic_fetch_inc_release
314 : #endif
315 :
316 : #if defined(arch_atomic_fetch_inc_relaxed)
317 : static __always_inline int
318 : atomic_fetch_inc_relaxed(atomic_t *v)
319 : {
320 : instrument_atomic_read_write(v, sizeof(*v));
321 : return arch_atomic_fetch_inc_relaxed(v);
322 : }
323 : #define atomic_fetch_inc_relaxed atomic_fetch_inc_relaxed
324 : #endif
325 :
326 : #if defined(arch_atomic_dec)
327 : static __always_inline void
328 136851 : atomic_dec(atomic_t *v)
329 : {
330 136851 : instrument_atomic_read_write(v, sizeof(*v));
331 136850 : arch_atomic_dec(v);
332 4123 : }
333 : #define atomic_dec atomic_dec
334 : #endif
335 :
336 : #if defined(arch_atomic_dec_return)
337 : static __always_inline int
338 4687 : atomic_dec_return(atomic_t *v)
339 : {
340 4687 : instrument_atomic_read_write(v, sizeof(*v));
341 4687 : return arch_atomic_dec_return(v);
342 : }
343 : #define atomic_dec_return atomic_dec_return
344 : #endif
345 :
346 : #if defined(arch_atomic_dec_return_acquire)
347 : static __always_inline int
348 : atomic_dec_return_acquire(atomic_t *v)
349 : {
350 : instrument_atomic_read_write(v, sizeof(*v));
351 : return arch_atomic_dec_return_acquire(v);
352 : }
353 : #define atomic_dec_return_acquire atomic_dec_return_acquire
354 : #endif
355 :
356 : #if defined(arch_atomic_dec_return_release)
357 : static __always_inline int
358 : atomic_dec_return_release(atomic_t *v)
359 : {
360 : instrument_atomic_read_write(v, sizeof(*v));
361 : return arch_atomic_dec_return_release(v);
362 : }
363 : #define atomic_dec_return_release atomic_dec_return_release
364 : #endif
365 :
366 : #if defined(arch_atomic_dec_return_relaxed)
367 : static __always_inline int
368 : atomic_dec_return_relaxed(atomic_t *v)
369 : {
370 : instrument_atomic_read_write(v, sizeof(*v));
371 : return arch_atomic_dec_return_relaxed(v);
372 : }
373 : #define atomic_dec_return_relaxed atomic_dec_return_relaxed
374 : #endif
375 :
376 : #if defined(arch_atomic_fetch_dec)
377 : static __always_inline int
378 : atomic_fetch_dec(atomic_t *v)
379 : {
380 : instrument_atomic_read_write(v, sizeof(*v));
381 : return arch_atomic_fetch_dec(v);
382 : }
383 : #define atomic_fetch_dec atomic_fetch_dec
384 : #endif
385 :
386 : #if defined(arch_atomic_fetch_dec_acquire)
387 : static __always_inline int
388 : atomic_fetch_dec_acquire(atomic_t *v)
389 : {
390 : instrument_atomic_read_write(v, sizeof(*v));
391 : return arch_atomic_fetch_dec_acquire(v);
392 : }
393 : #define atomic_fetch_dec_acquire atomic_fetch_dec_acquire
394 : #endif
395 :
396 : #if defined(arch_atomic_fetch_dec_release)
397 : static __always_inline int
398 : atomic_fetch_dec_release(atomic_t *v)
399 : {
400 : instrument_atomic_read_write(v, sizeof(*v));
401 : return arch_atomic_fetch_dec_release(v);
402 : }
403 : #define atomic_fetch_dec_release atomic_fetch_dec_release
404 : #endif
405 :
406 : #if defined(arch_atomic_fetch_dec_relaxed)
407 : static __always_inline int
408 : atomic_fetch_dec_relaxed(atomic_t *v)
409 : {
410 : instrument_atomic_read_write(v, sizeof(*v));
411 : return arch_atomic_fetch_dec_relaxed(v);
412 : }
413 : #define atomic_fetch_dec_relaxed atomic_fetch_dec_relaxed
414 : #endif
415 :
416 : static __always_inline void
417 0 : atomic_and(int i, atomic_t *v)
418 : {
419 0 : instrument_atomic_read_write(v, sizeof(*v));
420 0 : arch_atomic_and(i, v);
421 0 : }
422 : #define atomic_and atomic_and
423 :
424 : #if !defined(arch_atomic_fetch_and_relaxed) || defined(arch_atomic_fetch_and)
425 : static __always_inline int
426 0 : atomic_fetch_and(int i, atomic_t *v)
427 : {
428 0 : instrument_atomic_read_write(v, sizeof(*v));
429 0 : return arch_atomic_fetch_and(i, v);
430 : }
431 : #define atomic_fetch_and atomic_fetch_and
432 : #endif
433 :
434 : #if defined(arch_atomic_fetch_and_acquire)
435 : static __always_inline int
436 : atomic_fetch_and_acquire(int i, atomic_t *v)
437 : {
438 : instrument_atomic_read_write(v, sizeof(*v));
439 : return arch_atomic_fetch_and_acquire(i, v);
440 : }
441 : #define atomic_fetch_and_acquire atomic_fetch_and_acquire
442 : #endif
443 :
444 : #if defined(arch_atomic_fetch_and_release)
445 : static __always_inline int
446 : atomic_fetch_and_release(int i, atomic_t *v)
447 : {
448 : instrument_atomic_read_write(v, sizeof(*v));
449 : return arch_atomic_fetch_and_release(i, v);
450 : }
451 : #define atomic_fetch_and_release atomic_fetch_and_release
452 : #endif
453 :
454 : #if defined(arch_atomic_fetch_and_relaxed)
455 : static __always_inline int
456 : atomic_fetch_and_relaxed(int i, atomic_t *v)
457 : {
458 : instrument_atomic_read_write(v, sizeof(*v));
459 : return arch_atomic_fetch_and_relaxed(i, v);
460 : }
461 : #define atomic_fetch_and_relaxed atomic_fetch_and_relaxed
462 : #endif
463 :
464 : #if defined(arch_atomic_andnot)
465 : static __always_inline void
466 : atomic_andnot(int i, atomic_t *v)
467 : {
468 : instrument_atomic_read_write(v, sizeof(*v));
469 : arch_atomic_andnot(i, v);
470 : }
471 : #define atomic_andnot atomic_andnot
472 : #endif
473 :
474 : #if defined(arch_atomic_fetch_andnot)
475 : static __always_inline int
476 220 : atomic_fetch_andnot(int i, atomic_t *v)
477 : {
478 220 : instrument_atomic_read_write(v, sizeof(*v));
479 440 : return arch_atomic_fetch_andnot(i, v);
480 : }
481 : #define atomic_fetch_andnot atomic_fetch_andnot
482 : #endif
483 :
484 : #if defined(arch_atomic_fetch_andnot_acquire)
485 : static __always_inline int
486 : atomic_fetch_andnot_acquire(int i, atomic_t *v)
487 : {
488 : instrument_atomic_read_write(v, sizeof(*v));
489 : return arch_atomic_fetch_andnot_acquire(i, v);
490 : }
491 : #define atomic_fetch_andnot_acquire atomic_fetch_andnot_acquire
492 : #endif
493 :
494 : #if defined(arch_atomic_fetch_andnot_release)
495 : static __always_inline int
496 : atomic_fetch_andnot_release(int i, atomic_t *v)
497 : {
498 : instrument_atomic_read_write(v, sizeof(*v));
499 : return arch_atomic_fetch_andnot_release(i, v);
500 : }
501 : #define atomic_fetch_andnot_release atomic_fetch_andnot_release
502 : #endif
503 :
504 : #if defined(arch_atomic_fetch_andnot_relaxed)
505 : static __always_inline int
506 : atomic_fetch_andnot_relaxed(int i, atomic_t *v)
507 : {
508 : instrument_atomic_read_write(v, sizeof(*v));
509 : return arch_atomic_fetch_andnot_relaxed(i, v);
510 : }
511 : #define atomic_fetch_andnot_relaxed atomic_fetch_andnot_relaxed
512 : #endif
513 :
514 : static __always_inline void
515 0 : atomic_or(int i, atomic_t *v)
516 : {
517 0 : instrument_atomic_read_write(v, sizeof(*v));
518 0 : arch_atomic_or(i, v);
519 0 : }
520 : #define atomic_or atomic_or
521 :
522 : #if !defined(arch_atomic_fetch_or_relaxed) || defined(arch_atomic_fetch_or)
523 : static __always_inline int
524 303 : atomic_fetch_or(int i, atomic_t *v)
525 : {
526 303 : instrument_atomic_read_write(v, sizeof(*v));
527 536 : return arch_atomic_fetch_or(i, v);
528 : }
529 : #define atomic_fetch_or atomic_fetch_or
530 : #endif
531 :
532 : #if defined(arch_atomic_fetch_or_acquire)
533 : static __always_inline int
534 : atomic_fetch_or_acquire(int i, atomic_t *v)
535 : {
536 : instrument_atomic_read_write(v, sizeof(*v));
537 : return arch_atomic_fetch_or_acquire(i, v);
538 : }
539 : #define atomic_fetch_or_acquire atomic_fetch_or_acquire
540 : #endif
541 :
542 : #if defined(arch_atomic_fetch_or_release)
543 : static __always_inline int
544 : atomic_fetch_or_release(int i, atomic_t *v)
545 : {
546 : instrument_atomic_read_write(v, sizeof(*v));
547 : return arch_atomic_fetch_or_release(i, v);
548 : }
549 : #define atomic_fetch_or_release atomic_fetch_or_release
550 : #endif
551 :
552 : #if defined(arch_atomic_fetch_or_relaxed)
553 : static __always_inline int
554 : atomic_fetch_or_relaxed(int i, atomic_t *v)
555 : {
556 : instrument_atomic_read_write(v, sizeof(*v));
557 : return arch_atomic_fetch_or_relaxed(i, v);
558 : }
559 : #define atomic_fetch_or_relaxed atomic_fetch_or_relaxed
560 : #endif
561 :
562 : static __always_inline void
563 0 : atomic_xor(int i, atomic_t *v)
564 : {
565 0 : instrument_atomic_read_write(v, sizeof(*v));
566 0 : arch_atomic_xor(i, v);
567 0 : }
568 : #define atomic_xor atomic_xor
569 :
570 : #if !defined(arch_atomic_fetch_xor_relaxed) || defined(arch_atomic_fetch_xor)
571 : static __always_inline int
572 0 : atomic_fetch_xor(int i, atomic_t *v)
573 : {
574 0 : instrument_atomic_read_write(v, sizeof(*v));
575 0 : return arch_atomic_fetch_xor(i, v);
576 : }
577 : #define atomic_fetch_xor atomic_fetch_xor
578 : #endif
579 :
580 : #if defined(arch_atomic_fetch_xor_acquire)
581 : static __always_inline int
582 : atomic_fetch_xor_acquire(int i, atomic_t *v)
583 : {
584 : instrument_atomic_read_write(v, sizeof(*v));
585 : return arch_atomic_fetch_xor_acquire(i, v);
586 : }
587 : #define atomic_fetch_xor_acquire atomic_fetch_xor_acquire
588 : #endif
589 :
590 : #if defined(arch_atomic_fetch_xor_release)
591 : static __always_inline int
592 : atomic_fetch_xor_release(int i, atomic_t *v)
593 : {
594 : instrument_atomic_read_write(v, sizeof(*v));
595 : return arch_atomic_fetch_xor_release(i, v);
596 : }
597 : #define atomic_fetch_xor_release atomic_fetch_xor_release
598 : #endif
599 :
600 : #if defined(arch_atomic_fetch_xor_relaxed)
601 : static __always_inline int
602 : atomic_fetch_xor_relaxed(int i, atomic_t *v)
603 : {
604 : instrument_atomic_read_write(v, sizeof(*v));
605 : return arch_atomic_fetch_xor_relaxed(i, v);
606 : }
607 : #define atomic_fetch_xor_relaxed atomic_fetch_xor_relaxed
608 : #endif
609 :
610 : #if !defined(arch_atomic_xchg_relaxed) || defined(arch_atomic_xchg)
611 : static __always_inline int
612 1229 : atomic_xchg(atomic_t *v, int i)
613 : {
614 1229 : instrument_atomic_read_write(v, sizeof(*v));
615 1229 : return arch_atomic_xchg(v, i);
616 : }
617 : #define atomic_xchg atomic_xchg
618 : #endif
619 :
620 : #if defined(arch_atomic_xchg_acquire)
621 : static __always_inline int
622 : atomic_xchg_acquire(atomic_t *v, int i)
623 : {
624 : instrument_atomic_read_write(v, sizeof(*v));
625 : return arch_atomic_xchg_acquire(v, i);
626 : }
627 : #define atomic_xchg_acquire atomic_xchg_acquire
628 : #endif
629 :
630 : #if defined(arch_atomic_xchg_release)
631 : static __always_inline int
632 : atomic_xchg_release(atomic_t *v, int i)
633 : {
634 : instrument_atomic_read_write(v, sizeof(*v));
635 : return arch_atomic_xchg_release(v, i);
636 : }
637 : #define atomic_xchg_release atomic_xchg_release
638 : #endif
639 :
640 : #if defined(arch_atomic_xchg_relaxed)
641 : static __always_inline int
642 : atomic_xchg_relaxed(atomic_t *v, int i)
643 : {
644 : instrument_atomic_read_write(v, sizeof(*v));
645 : return arch_atomic_xchg_relaxed(v, i);
646 : }
647 : #define atomic_xchg_relaxed atomic_xchg_relaxed
648 : #endif
649 :
650 : #if !defined(arch_atomic_cmpxchg_relaxed) || defined(arch_atomic_cmpxchg)
651 : static __always_inline int
652 194 : atomic_cmpxchg(atomic_t *v, int old, int new)
653 : {
654 194 : instrument_atomic_read_write(v, sizeof(*v));
655 194 : return arch_atomic_cmpxchg(v, old, new);
656 : }
657 : #define atomic_cmpxchg atomic_cmpxchg
658 : #endif
659 :
660 : #if defined(arch_atomic_cmpxchg_acquire)
661 : static __always_inline int
662 3 : atomic_cmpxchg_acquire(atomic_t *v, int old, int new)
663 : {
664 3 : instrument_atomic_read_write(v, sizeof(*v));
665 3 : return arch_atomic_cmpxchg_acquire(v, old, new);
666 : }
667 : #define atomic_cmpxchg_acquire atomic_cmpxchg_acquire
668 : #endif
669 :
670 : #if defined(arch_atomic_cmpxchg_release)
671 : static __always_inline int
672 1012 : atomic_cmpxchg_release(atomic_t *v, int old, int new)
673 : {
674 1012 : instrument_atomic_read_write(v, sizeof(*v));
675 1012 : return arch_atomic_cmpxchg_release(v, old, new);
676 : }
677 : #define atomic_cmpxchg_release atomic_cmpxchg_release
678 : #endif
679 :
680 : #if defined(arch_atomic_cmpxchg_relaxed)
681 : static __always_inline int
682 19 : atomic_cmpxchg_relaxed(atomic_t *v, int old, int new)
683 : {
684 19 : instrument_atomic_read_write(v, sizeof(*v));
685 19 : return arch_atomic_cmpxchg_relaxed(v, old, new);
686 : }
687 : #define atomic_cmpxchg_relaxed atomic_cmpxchg_relaxed
688 : #endif
689 :
690 : #if defined(arch_atomic_try_cmpxchg)
691 : static __always_inline bool
692 : atomic_try_cmpxchg(atomic_t *v, int *old, int new)
693 : {
694 : instrument_atomic_read_write(v, sizeof(*v));
695 : instrument_atomic_read_write(old, sizeof(*old));
696 : return arch_atomic_try_cmpxchg(v, old, new);
697 : }
698 : #define atomic_try_cmpxchg atomic_try_cmpxchg
699 : #endif
700 :
701 : #if defined(arch_atomic_try_cmpxchg_acquire)
702 : static __always_inline bool
703 9535918 : atomic_try_cmpxchg_acquire(atomic_t *v, int *old, int new)
704 : {
705 9535918 : instrument_atomic_read_write(v, sizeof(*v));
706 9528612 : instrument_atomic_read_write(old, sizeof(*old));
707 19087258 : return arch_atomic_try_cmpxchg_acquire(v, old, new);
708 : }
709 : #define atomic_try_cmpxchg_acquire atomic_try_cmpxchg_acquire
710 : #endif
711 :
712 : #if defined(arch_atomic_try_cmpxchg_release)
713 : static __always_inline bool
714 18536 : atomic_try_cmpxchg_release(atomic_t *v, int *old, int new)
715 : {
716 18536 : instrument_atomic_read_write(v, sizeof(*v));
717 18535 : instrument_atomic_read_write(old, sizeof(*old));
718 37074 : return arch_atomic_try_cmpxchg_release(v, old, new);
719 : }
720 : #define atomic_try_cmpxchg_release atomic_try_cmpxchg_release
721 : #endif
722 :
723 : #if defined(arch_atomic_try_cmpxchg_relaxed)
724 : static __always_inline bool
725 322034 : atomic_try_cmpxchg_relaxed(atomic_t *v, int *old, int new)
726 : {
727 322034 : instrument_atomic_read_write(v, sizeof(*v));
728 322034 : instrument_atomic_read_write(old, sizeof(*old));
729 644068 : return arch_atomic_try_cmpxchg_relaxed(v, old, new);
730 : }
731 : #define atomic_try_cmpxchg_relaxed atomic_try_cmpxchg_relaxed
732 : #endif
733 :
734 : #if defined(arch_atomic_sub_and_test)
735 : static __always_inline bool
736 1 : atomic_sub_and_test(int i, atomic_t *v)
737 : {
738 1 : instrument_atomic_read_write(v, sizeof(*v));
739 1 : return arch_atomic_sub_and_test(i, v);
740 : }
741 : #define atomic_sub_and_test atomic_sub_and_test
742 : #endif
743 :
744 : #if defined(arch_atomic_dec_and_test)
745 : static __always_inline bool
746 2025041 : atomic_dec_and_test(atomic_t *v)
747 : {
748 2025041 : instrument_atomic_read_write(v, sizeof(*v));
749 2025038 : return arch_atomic_dec_and_test(v);
750 : }
751 : #define atomic_dec_and_test atomic_dec_and_test
752 : #endif
753 :
754 : #if defined(arch_atomic_inc_and_test)
755 : static __always_inline bool
756 798005 : atomic_inc_and_test(atomic_t *v)
757 : {
758 798005 : instrument_atomic_read_write(v, sizeof(*v));
759 797991 : return arch_atomic_inc_and_test(v);
760 : }
761 : #define atomic_inc_and_test atomic_inc_and_test
762 : #endif
763 :
764 : #if defined(arch_atomic_add_negative)
765 : static __always_inline bool
766 923073 : atomic_add_negative(int i, atomic_t *v)
767 : {
768 923073 : instrument_atomic_read_write(v, sizeof(*v));
769 923058 : return arch_atomic_add_negative(i, v);
770 : }
771 : #define atomic_add_negative atomic_add_negative
772 : #endif
773 :
774 : #if defined(arch_atomic_fetch_add_unless)
775 : static __always_inline int
776 : atomic_fetch_add_unless(atomic_t *v, int a, int u)
777 : {
778 : instrument_atomic_read_write(v, sizeof(*v));
779 : return arch_atomic_fetch_add_unless(v, a, u);
780 : }
781 : #define atomic_fetch_add_unless atomic_fetch_add_unless
782 : #endif
783 :
784 : #if defined(arch_atomic_add_unless)
785 : static __always_inline bool
786 12645 : atomic_add_unless(atomic_t *v, int a, int u)
787 : {
788 12645 : instrument_atomic_read_write(v, sizeof(*v));
789 25288 : return arch_atomic_add_unless(v, a, u);
790 : }
791 : #define atomic_add_unless atomic_add_unless
792 : #endif
793 :
794 : #if defined(arch_atomic_inc_not_zero)
795 : static __always_inline bool
796 24879 : atomic_inc_not_zero(atomic_t *v)
797 : {
798 24879 : instrument_atomic_read_write(v, sizeof(*v));
799 25221 : return arch_atomic_inc_not_zero(v);
800 : }
801 : #define atomic_inc_not_zero atomic_inc_not_zero
802 : #endif
803 :
804 : #if defined(arch_atomic_inc_unless_negative)
805 : static __always_inline bool
806 5640 : atomic_inc_unless_negative(atomic_t *v)
807 : {
808 5640 : instrument_atomic_read_write(v, sizeof(*v));
809 9688 : return arch_atomic_inc_unless_negative(v);
810 : }
811 : #define atomic_inc_unless_negative atomic_inc_unless_negative
812 : #endif
813 :
814 : #if defined(arch_atomic_dec_unless_positive)
815 : static __always_inline bool
816 9576 : atomic_dec_unless_positive(atomic_t *v)
817 : {
818 9576 : instrument_atomic_read_write(v, sizeof(*v));
819 19152 : return arch_atomic_dec_unless_positive(v);
820 : }
821 : #define atomic_dec_unless_positive atomic_dec_unless_positive
822 : #endif
823 :
824 : #if defined(arch_atomic_dec_if_positive)
825 : static __always_inline int
826 86 : atomic_dec_if_positive(atomic_t *v)
827 : {
828 86 : instrument_atomic_read_write(v, sizeof(*v));
829 172 : return arch_atomic_dec_if_positive(v);
830 : }
831 : #define atomic_dec_if_positive atomic_dec_if_positive
832 : #endif
833 :
834 : static __always_inline s64
835 689827 : atomic64_read(const atomic64_t *v)
836 : {
837 2249659 : instrument_atomic_read(v, sizeof(*v));
838 2240752 : return arch_atomic64_read(v);
839 : }
840 : #define atomic64_read atomic64_read
841 :
842 : #if defined(arch_atomic64_read_acquire)
843 : static __always_inline s64
844 : atomic64_read_acquire(const atomic64_t *v)
845 : {
846 : instrument_atomic_read(v, sizeof(*v));
847 : return arch_atomic64_read_acquire(v);
848 : }
849 : #define atomic64_read_acquire atomic64_read_acquire
850 : #endif
851 :
852 : static __always_inline void
853 1131948 : atomic64_set(atomic64_t *v, s64 i)
854 : {
855 1136083 : instrument_atomic_write(v, sizeof(*v));
856 948519 : arch_atomic64_set(v, i);
857 : }
858 : #define atomic64_set atomic64_set
859 :
860 : #if defined(arch_atomic64_set_release)
861 : static __always_inline void
862 : atomic64_set_release(atomic64_t *v, s64 i)
863 : {
864 : instrument_atomic_write(v, sizeof(*v));
865 : arch_atomic64_set_release(v, i);
866 : }
867 : #define atomic64_set_release atomic64_set_release
868 : #endif
869 :
870 : static __always_inline void
871 177676 : atomic64_add(s64 i, atomic64_t *v)
872 : {
873 177676 : instrument_atomic_read_write(v, sizeof(*v));
874 141321 : arch_atomic64_add(i, v);
875 206 : }
876 : #define atomic64_add atomic64_add
877 :
878 : #if !defined(arch_atomic64_add_return_relaxed) || defined(arch_atomic64_add_return)
879 : static __always_inline s64
880 138302 : atomic64_add_return(s64 i, atomic64_t *v)
881 : {
882 138302 : instrument_atomic_read_write(v, sizeof(*v));
883 138303 : return arch_atomic64_add_return(i, v);
884 : }
885 : #define atomic64_add_return atomic64_add_return
886 : #endif
887 :
888 : #if defined(arch_atomic64_add_return_acquire)
889 : static __always_inline s64
890 35892 : atomic64_add_return_acquire(s64 i, atomic64_t *v)
891 : {
892 35892 : instrument_atomic_read_write(v, sizeof(*v));
893 35893 : return arch_atomic64_add_return_acquire(i, v);
894 : }
895 : #define atomic64_add_return_acquire atomic64_add_return_acquire
896 : #endif
897 :
898 : #if defined(arch_atomic64_add_return_release)
899 : static __always_inline s64
900 181456 : atomic64_add_return_release(s64 i, atomic64_t *v)
901 : {
902 181456 : instrument_atomic_read_write(v, sizeof(*v));
903 181449 : return arch_atomic64_add_return_release(i, v);
904 : }
905 : #define atomic64_add_return_release atomic64_add_return_release
906 : #endif
907 :
908 : #if defined(arch_atomic64_add_return_relaxed)
909 : static __always_inline s64
910 : atomic64_add_return_relaxed(s64 i, atomic64_t *v)
911 : {
912 : instrument_atomic_read_write(v, sizeof(*v));
913 : return arch_atomic64_add_return_relaxed(i, v);
914 : }
915 : #define atomic64_add_return_relaxed atomic64_add_return_relaxed
916 : #endif
917 :
918 : #if !defined(arch_atomic64_fetch_add_relaxed) || defined(arch_atomic64_fetch_add)
919 : static __always_inline s64
920 45 : atomic64_fetch_add(s64 i, atomic64_t *v)
921 : {
922 45 : instrument_atomic_read_write(v, sizeof(*v));
923 45 : return arch_atomic64_fetch_add(i, v);
924 : }
925 : #define atomic64_fetch_add atomic64_fetch_add
926 : #endif
927 :
928 : #if defined(arch_atomic64_fetch_add_acquire)
929 : static __always_inline s64
930 : atomic64_fetch_add_acquire(s64 i, atomic64_t *v)
931 : {
932 : instrument_atomic_read_write(v, sizeof(*v));
933 : return arch_atomic64_fetch_add_acquire(i, v);
934 : }
935 : #define atomic64_fetch_add_acquire atomic64_fetch_add_acquire
936 : #endif
937 :
938 : #if defined(arch_atomic64_fetch_add_release)
939 : static __always_inline s64
940 328809 : atomic64_fetch_add_release(s64 i, atomic64_t *v)
941 : {
942 328809 : instrument_atomic_read_write(v, sizeof(*v));
943 328803 : return arch_atomic64_fetch_add_release(i, v);
944 : }
945 : #define atomic64_fetch_add_release atomic64_fetch_add_release
946 : #endif
947 :
948 : #if defined(arch_atomic64_fetch_add_relaxed)
949 : static __always_inline s64
950 : atomic64_fetch_add_relaxed(s64 i, atomic64_t *v)
951 : {
952 : instrument_atomic_read_write(v, sizeof(*v));
953 : return arch_atomic64_fetch_add_relaxed(i, v);
954 : }
955 : #define atomic64_fetch_add_relaxed atomic64_fetch_add_relaxed
956 : #endif
957 :
958 : static __always_inline void
959 61053 : atomic64_sub(s64 i, atomic64_t *v)
960 : {
961 61053 : instrument_atomic_read_write(v, sizeof(*v));
962 61298 : arch_atomic64_sub(i, v);
963 : }
964 : #define atomic64_sub atomic64_sub
965 :
966 : #if !defined(arch_atomic64_sub_return_relaxed) || defined(arch_atomic64_sub_return)
967 : static __always_inline s64
968 0 : atomic64_sub_return(s64 i, atomic64_t *v)
969 : {
970 0 : instrument_atomic_read_write(v, sizeof(*v));
971 0 : return arch_atomic64_sub_return(i, v);
972 : }
973 : #define atomic64_sub_return atomic64_sub_return
974 : #endif
975 :
976 : #if defined(arch_atomic64_sub_return_acquire)
977 : static __always_inline s64
978 : atomic64_sub_return_acquire(s64 i, atomic64_t *v)
979 : {
980 : instrument_atomic_read_write(v, sizeof(*v));
981 : return arch_atomic64_sub_return_acquire(i, v);
982 : }
983 : #define atomic64_sub_return_acquire atomic64_sub_return_acquire
984 : #endif
985 :
986 : #if defined(arch_atomic64_sub_return_release)
987 : static __always_inline s64
988 : atomic64_sub_return_release(s64 i, atomic64_t *v)
989 : {
990 : instrument_atomic_read_write(v, sizeof(*v));
991 : return arch_atomic64_sub_return_release(i, v);
992 : }
993 : #define atomic64_sub_return_release atomic64_sub_return_release
994 : #endif
995 :
996 : #if defined(arch_atomic64_sub_return_relaxed)
997 : static __always_inline s64
998 : atomic64_sub_return_relaxed(s64 i, atomic64_t *v)
999 : {
1000 : instrument_atomic_read_write(v, sizeof(*v));
1001 : return arch_atomic64_sub_return_relaxed(i, v);
1002 : }
1003 : #define atomic64_sub_return_relaxed atomic64_sub_return_relaxed
1004 : #endif
1005 :
1006 : #if !defined(arch_atomic64_fetch_sub_relaxed) || defined(arch_atomic64_fetch_sub)
1007 : static __always_inline s64
1008 : atomic64_fetch_sub(s64 i, atomic64_t *v)
1009 : {
1010 : instrument_atomic_read_write(v, sizeof(*v));
1011 : return arch_atomic64_fetch_sub(i, v);
1012 : }
1013 : #define atomic64_fetch_sub atomic64_fetch_sub
1014 : #endif
1015 :
1016 : #if defined(arch_atomic64_fetch_sub_acquire)
1017 : static __always_inline s64
1018 : atomic64_fetch_sub_acquire(s64 i, atomic64_t *v)
1019 : {
1020 : instrument_atomic_read_write(v, sizeof(*v));
1021 : return arch_atomic64_fetch_sub_acquire(i, v);
1022 : }
1023 : #define atomic64_fetch_sub_acquire atomic64_fetch_sub_acquire
1024 : #endif
1025 :
1026 : #if defined(arch_atomic64_fetch_sub_release)
1027 : static __always_inline s64
1028 : atomic64_fetch_sub_release(s64 i, atomic64_t *v)
1029 : {
1030 : instrument_atomic_read_write(v, sizeof(*v));
1031 : return arch_atomic64_fetch_sub_release(i, v);
1032 : }
1033 : #define atomic64_fetch_sub_release atomic64_fetch_sub_release
1034 : #endif
1035 :
1036 : #if defined(arch_atomic64_fetch_sub_relaxed)
1037 : static __always_inline s64
1038 : atomic64_fetch_sub_relaxed(s64 i, atomic64_t *v)
1039 : {
1040 : instrument_atomic_read_write(v, sizeof(*v));
1041 : return arch_atomic64_fetch_sub_relaxed(i, v);
1042 : }
1043 : #define atomic64_fetch_sub_relaxed atomic64_fetch_sub_relaxed
1044 : #endif
1045 :
1046 : #if defined(arch_atomic64_inc)
1047 : static __always_inline void
1048 131532 : atomic64_inc(atomic64_t *v)
1049 : {
1050 131532 : instrument_atomic_read_write(v, sizeof(*v));
1051 131532 : arch_atomic64_inc(v);
1052 : }
1053 : #define atomic64_inc atomic64_inc
1054 : #endif
1055 :
1056 : #if defined(arch_atomic64_inc_return)
1057 : static __always_inline s64
1058 85916 : atomic64_inc_return(atomic64_t *v)
1059 : {
1060 85916 : instrument_atomic_read_write(v, sizeof(*v));
1061 85910 : return arch_atomic64_inc_return(v);
1062 : }
1063 : #define atomic64_inc_return atomic64_inc_return
1064 : #endif
1065 :
1066 : #if defined(arch_atomic64_inc_return_acquire)
1067 : static __always_inline s64
1068 : atomic64_inc_return_acquire(atomic64_t *v)
1069 : {
1070 : instrument_atomic_read_write(v, sizeof(*v));
1071 : return arch_atomic64_inc_return_acquire(v);
1072 : }
1073 : #define atomic64_inc_return_acquire atomic64_inc_return_acquire
1074 : #endif
1075 :
1076 : #if defined(arch_atomic64_inc_return_release)
1077 : static __always_inline s64
1078 : atomic64_inc_return_release(atomic64_t *v)
1079 : {
1080 : instrument_atomic_read_write(v, sizeof(*v));
1081 : return arch_atomic64_inc_return_release(v);
1082 : }
1083 : #define atomic64_inc_return_release atomic64_inc_return_release
1084 : #endif
1085 :
1086 : #if defined(arch_atomic64_inc_return_relaxed)
1087 : static __always_inline s64
1088 : atomic64_inc_return_relaxed(atomic64_t *v)
1089 : {
1090 : instrument_atomic_read_write(v, sizeof(*v));
1091 : return arch_atomic64_inc_return_relaxed(v);
1092 : }
1093 : #define atomic64_inc_return_relaxed atomic64_inc_return_relaxed
1094 : #endif
1095 :
1096 : #if defined(arch_atomic64_fetch_inc)
1097 : static __always_inline s64
1098 : atomic64_fetch_inc(atomic64_t *v)
1099 : {
1100 : instrument_atomic_read_write(v, sizeof(*v));
1101 : return arch_atomic64_fetch_inc(v);
1102 : }
1103 : #define atomic64_fetch_inc atomic64_fetch_inc
1104 : #endif
1105 :
1106 : #if defined(arch_atomic64_fetch_inc_acquire)
1107 : static __always_inline s64
1108 : atomic64_fetch_inc_acquire(atomic64_t *v)
1109 : {
1110 : instrument_atomic_read_write(v, sizeof(*v));
1111 : return arch_atomic64_fetch_inc_acquire(v);
1112 : }
1113 : #define atomic64_fetch_inc_acquire atomic64_fetch_inc_acquire
1114 : #endif
1115 :
1116 : #if defined(arch_atomic64_fetch_inc_release)
1117 : static __always_inline s64
1118 : atomic64_fetch_inc_release(atomic64_t *v)
1119 : {
1120 : instrument_atomic_read_write(v, sizeof(*v));
1121 : return arch_atomic64_fetch_inc_release(v);
1122 : }
1123 : #define atomic64_fetch_inc_release atomic64_fetch_inc_release
1124 : #endif
1125 :
1126 : #if defined(arch_atomic64_fetch_inc_relaxed)
1127 : static __always_inline s64
1128 : atomic64_fetch_inc_relaxed(atomic64_t *v)
1129 : {
1130 : instrument_atomic_read_write(v, sizeof(*v));
1131 : return arch_atomic64_fetch_inc_relaxed(v);
1132 : }
1133 : #define atomic64_fetch_inc_relaxed atomic64_fetch_inc_relaxed
1134 : #endif
1135 :
1136 : #if defined(arch_atomic64_dec)
1137 : static __always_inline void
1138 19450 : atomic64_dec(atomic64_t *v)
1139 : {
1140 19450 : instrument_atomic_read_write(v, sizeof(*v));
1141 19450 : arch_atomic64_dec(v);
1142 : }
1143 : #define atomic64_dec atomic64_dec
1144 : #endif
1145 :
1146 : #if defined(arch_atomic64_dec_return)
1147 : static __always_inline s64
1148 0 : atomic64_dec_return(atomic64_t *v)
1149 : {
1150 0 : instrument_atomic_read_write(v, sizeof(*v));
1151 0 : return arch_atomic64_dec_return(v);
1152 : }
1153 : #define atomic64_dec_return atomic64_dec_return
1154 : #endif
1155 :
1156 : #if defined(arch_atomic64_dec_return_acquire)
1157 : static __always_inline s64
1158 : atomic64_dec_return_acquire(atomic64_t *v)
1159 : {
1160 : instrument_atomic_read_write(v, sizeof(*v));
1161 : return arch_atomic64_dec_return_acquire(v);
1162 : }
1163 : #define atomic64_dec_return_acquire atomic64_dec_return_acquire
1164 : #endif
1165 :
1166 : #if defined(arch_atomic64_dec_return_release)
1167 : static __always_inline s64
1168 : atomic64_dec_return_release(atomic64_t *v)
1169 : {
1170 : instrument_atomic_read_write(v, sizeof(*v));
1171 : return arch_atomic64_dec_return_release(v);
1172 : }
1173 : #define atomic64_dec_return_release atomic64_dec_return_release
1174 : #endif
1175 :
1176 : #if defined(arch_atomic64_dec_return_relaxed)
1177 : static __always_inline s64
1178 : atomic64_dec_return_relaxed(atomic64_t *v)
1179 : {
1180 : instrument_atomic_read_write(v, sizeof(*v));
1181 : return arch_atomic64_dec_return_relaxed(v);
1182 : }
1183 : #define atomic64_dec_return_relaxed atomic64_dec_return_relaxed
1184 : #endif
1185 :
1186 : #if defined(arch_atomic64_fetch_dec)
1187 : static __always_inline s64
1188 : atomic64_fetch_dec(atomic64_t *v)
1189 : {
1190 : instrument_atomic_read_write(v, sizeof(*v));
1191 : return arch_atomic64_fetch_dec(v);
1192 : }
1193 : #define atomic64_fetch_dec atomic64_fetch_dec
1194 : #endif
1195 :
1196 : #if defined(arch_atomic64_fetch_dec_acquire)
1197 : static __always_inline s64
1198 : atomic64_fetch_dec_acquire(atomic64_t *v)
1199 : {
1200 : instrument_atomic_read_write(v, sizeof(*v));
1201 : return arch_atomic64_fetch_dec_acquire(v);
1202 : }
1203 : #define atomic64_fetch_dec_acquire atomic64_fetch_dec_acquire
1204 : #endif
1205 :
1206 : #if defined(arch_atomic64_fetch_dec_release)
1207 : static __always_inline s64
1208 : atomic64_fetch_dec_release(atomic64_t *v)
1209 : {
1210 : instrument_atomic_read_write(v, sizeof(*v));
1211 : return arch_atomic64_fetch_dec_release(v);
1212 : }
1213 : #define atomic64_fetch_dec_release atomic64_fetch_dec_release
1214 : #endif
1215 :
1216 : #if defined(arch_atomic64_fetch_dec_relaxed)
1217 : static __always_inline s64
1218 : atomic64_fetch_dec_relaxed(atomic64_t *v)
1219 : {
1220 : instrument_atomic_read_write(v, sizeof(*v));
1221 : return arch_atomic64_fetch_dec_relaxed(v);
1222 : }
1223 : #define atomic64_fetch_dec_relaxed atomic64_fetch_dec_relaxed
1224 : #endif
1225 :
1226 : static __always_inline void
1227 0 : atomic64_and(s64 i, atomic64_t *v)
1228 : {
1229 0 : instrument_atomic_read_write(v, sizeof(*v));
1230 0 : arch_atomic64_and(i, v);
1231 : }
1232 : #define atomic64_and atomic64_and
1233 :
1234 : #if !defined(arch_atomic64_fetch_and_relaxed) || defined(arch_atomic64_fetch_and)
1235 : static __always_inline s64
1236 0 : atomic64_fetch_and(s64 i, atomic64_t *v)
1237 : {
1238 0 : instrument_atomic_read_write(v, sizeof(*v));
1239 0 : return arch_atomic64_fetch_and(i, v);
1240 : }
1241 : #define atomic64_fetch_and atomic64_fetch_and
1242 : #endif
1243 :
1244 : #if defined(arch_atomic64_fetch_and_acquire)
1245 : static __always_inline s64
1246 : atomic64_fetch_and_acquire(s64 i, atomic64_t *v)
1247 : {
1248 : instrument_atomic_read_write(v, sizeof(*v));
1249 : return arch_atomic64_fetch_and_acquire(i, v);
1250 : }
1251 : #define atomic64_fetch_and_acquire atomic64_fetch_and_acquire
1252 : #endif
1253 :
1254 : #if defined(arch_atomic64_fetch_and_release)
1255 : static __always_inline s64
1256 : atomic64_fetch_and_release(s64 i, atomic64_t *v)
1257 : {
1258 : instrument_atomic_read_write(v, sizeof(*v));
1259 : return arch_atomic64_fetch_and_release(i, v);
1260 : }
1261 : #define atomic64_fetch_and_release atomic64_fetch_and_release
1262 : #endif
1263 :
1264 : #if defined(arch_atomic64_fetch_and_relaxed)
1265 : static __always_inline s64
1266 : atomic64_fetch_and_relaxed(s64 i, atomic64_t *v)
1267 : {
1268 : instrument_atomic_read_write(v, sizeof(*v));
1269 : return arch_atomic64_fetch_and_relaxed(i, v);
1270 : }
1271 : #define atomic64_fetch_and_relaxed atomic64_fetch_and_relaxed
1272 : #endif
1273 :
1274 : #if defined(arch_atomic64_andnot)
1275 : static __always_inline void
1276 112 : atomic64_andnot(s64 i, atomic64_t *v)
1277 : {
1278 112 : instrument_atomic_read_write(v, sizeof(*v));
1279 112 : arch_atomic64_andnot(i, v);
1280 : }
1281 : #define atomic64_andnot atomic64_andnot
1282 : #endif
1283 :
1284 : #if defined(arch_atomic64_fetch_andnot)
1285 : static __always_inline s64
1286 : atomic64_fetch_andnot(s64 i, atomic64_t *v)
1287 : {
1288 : instrument_atomic_read_write(v, sizeof(*v));
1289 : return arch_atomic64_fetch_andnot(i, v);
1290 : }
1291 : #define atomic64_fetch_andnot atomic64_fetch_andnot
1292 : #endif
1293 :
1294 : #if defined(arch_atomic64_fetch_andnot_acquire)
1295 : static __always_inline s64
1296 : atomic64_fetch_andnot_acquire(s64 i, atomic64_t *v)
1297 : {
1298 : instrument_atomic_read_write(v, sizeof(*v));
1299 : return arch_atomic64_fetch_andnot_acquire(i, v);
1300 : }
1301 : #define atomic64_fetch_andnot_acquire atomic64_fetch_andnot_acquire
1302 : #endif
1303 :
1304 : #if defined(arch_atomic64_fetch_andnot_release)
1305 : static __always_inline s64
1306 : atomic64_fetch_andnot_release(s64 i, atomic64_t *v)
1307 : {
1308 : instrument_atomic_read_write(v, sizeof(*v));
1309 : return arch_atomic64_fetch_andnot_release(i, v);
1310 : }
1311 : #define atomic64_fetch_andnot_release atomic64_fetch_andnot_release
1312 : #endif
1313 :
1314 : #if defined(arch_atomic64_fetch_andnot_relaxed)
1315 : static __always_inline s64
1316 : atomic64_fetch_andnot_relaxed(s64 i, atomic64_t *v)
1317 : {
1318 : instrument_atomic_read_write(v, sizeof(*v));
1319 : return arch_atomic64_fetch_andnot_relaxed(i, v);
1320 : }
1321 : #define atomic64_fetch_andnot_relaxed atomic64_fetch_andnot_relaxed
1322 : #endif
1323 :
1324 : static __always_inline void
1325 105 : atomic64_or(s64 i, atomic64_t *v)
1326 : {
1327 105 : instrument_atomic_read_write(v, sizeof(*v));
1328 194 : arch_atomic64_or(i, v);
1329 : }
1330 : #define atomic64_or atomic64_or
1331 :
1332 : #if !defined(arch_atomic64_fetch_or_relaxed) || defined(arch_atomic64_fetch_or)
1333 : static __always_inline s64
1334 0 : atomic64_fetch_or(s64 i, atomic64_t *v)
1335 : {
1336 0 : instrument_atomic_read_write(v, sizeof(*v));
1337 0 : return arch_atomic64_fetch_or(i, v);
1338 : }
1339 : #define atomic64_fetch_or atomic64_fetch_or
1340 : #endif
1341 :
1342 : #if defined(arch_atomic64_fetch_or_acquire)
1343 : static __always_inline s64
1344 : atomic64_fetch_or_acquire(s64 i, atomic64_t *v)
1345 : {
1346 : instrument_atomic_read_write(v, sizeof(*v));
1347 : return arch_atomic64_fetch_or_acquire(i, v);
1348 : }
1349 : #define atomic64_fetch_or_acquire atomic64_fetch_or_acquire
1350 : #endif
1351 :
1352 : #if defined(arch_atomic64_fetch_or_release)
1353 : static __always_inline s64
1354 : atomic64_fetch_or_release(s64 i, atomic64_t *v)
1355 : {
1356 : instrument_atomic_read_write(v, sizeof(*v));
1357 : return arch_atomic64_fetch_or_release(i, v);
1358 : }
1359 : #define atomic64_fetch_or_release atomic64_fetch_or_release
1360 : #endif
1361 :
1362 : #if defined(arch_atomic64_fetch_or_relaxed)
1363 : static __always_inline s64
1364 : atomic64_fetch_or_relaxed(s64 i, atomic64_t *v)
1365 : {
1366 : instrument_atomic_read_write(v, sizeof(*v));
1367 : return arch_atomic64_fetch_or_relaxed(i, v);
1368 : }
1369 : #define atomic64_fetch_or_relaxed atomic64_fetch_or_relaxed
1370 : #endif
1371 :
1372 : static __always_inline void
1373 0 : atomic64_xor(s64 i, atomic64_t *v)
1374 : {
1375 0 : instrument_atomic_read_write(v, sizeof(*v));
1376 0 : arch_atomic64_xor(i, v);
1377 : }
1378 : #define atomic64_xor atomic64_xor
1379 :
1380 : #if !defined(arch_atomic64_fetch_xor_relaxed) || defined(arch_atomic64_fetch_xor)
1381 : static __always_inline s64
1382 0 : atomic64_fetch_xor(s64 i, atomic64_t *v)
1383 : {
1384 0 : instrument_atomic_read_write(v, sizeof(*v));
1385 0 : return arch_atomic64_fetch_xor(i, v);
1386 : }
1387 : #define atomic64_fetch_xor atomic64_fetch_xor
1388 : #endif
1389 :
1390 : #if defined(arch_atomic64_fetch_xor_acquire)
1391 : static __always_inline s64
1392 : atomic64_fetch_xor_acquire(s64 i, atomic64_t *v)
1393 : {
1394 : instrument_atomic_read_write(v, sizeof(*v));
1395 : return arch_atomic64_fetch_xor_acquire(i, v);
1396 : }
1397 : #define atomic64_fetch_xor_acquire atomic64_fetch_xor_acquire
1398 : #endif
1399 :
1400 : #if defined(arch_atomic64_fetch_xor_release)
1401 : static __always_inline s64
1402 : atomic64_fetch_xor_release(s64 i, atomic64_t *v)
1403 : {
1404 : instrument_atomic_read_write(v, sizeof(*v));
1405 : return arch_atomic64_fetch_xor_release(i, v);
1406 : }
1407 : #define atomic64_fetch_xor_release atomic64_fetch_xor_release
1408 : #endif
1409 :
1410 : #if defined(arch_atomic64_fetch_xor_relaxed)
1411 : static __always_inline s64
1412 : atomic64_fetch_xor_relaxed(s64 i, atomic64_t *v)
1413 : {
1414 : instrument_atomic_read_write(v, sizeof(*v));
1415 : return arch_atomic64_fetch_xor_relaxed(i, v);
1416 : }
1417 : #define atomic64_fetch_xor_relaxed atomic64_fetch_xor_relaxed
1418 : #endif
1419 :
1420 : #if !defined(arch_atomic64_xchg_relaxed) || defined(arch_atomic64_xchg)
1421 : static __always_inline s64
1422 3 : atomic64_xchg(atomic64_t *v, s64 i)
1423 : {
1424 3 : instrument_atomic_read_write(v, sizeof(*v));
1425 3 : return arch_atomic64_xchg(v, i);
1426 : }
1427 : #define atomic64_xchg atomic64_xchg
1428 : #endif
1429 :
1430 : #if defined(arch_atomic64_xchg_acquire)
1431 : static __always_inline s64
1432 : atomic64_xchg_acquire(atomic64_t *v, s64 i)
1433 : {
1434 : instrument_atomic_read_write(v, sizeof(*v));
1435 : return arch_atomic64_xchg_acquire(v, i);
1436 : }
1437 : #define atomic64_xchg_acquire atomic64_xchg_acquire
1438 : #endif
1439 :
1440 : #if defined(arch_atomic64_xchg_release)
1441 : static __always_inline s64
1442 : atomic64_xchg_release(atomic64_t *v, s64 i)
1443 : {
1444 : instrument_atomic_read_write(v, sizeof(*v));
1445 : return arch_atomic64_xchg_release(v, i);
1446 : }
1447 : #define atomic64_xchg_release atomic64_xchg_release
1448 : #endif
1449 :
1450 : #if defined(arch_atomic64_xchg_relaxed)
1451 : static __always_inline s64
1452 : atomic64_xchg_relaxed(atomic64_t *v, s64 i)
1453 : {
1454 : instrument_atomic_read_write(v, sizeof(*v));
1455 : return arch_atomic64_xchg_relaxed(v, i);
1456 : }
1457 : #define atomic64_xchg_relaxed atomic64_xchg_relaxed
1458 : #endif
1459 :
1460 : #if !defined(arch_atomic64_cmpxchg_relaxed) || defined(arch_atomic64_cmpxchg)
1461 : static __always_inline s64
1462 1107 : atomic64_cmpxchg(atomic64_t *v, s64 old, s64 new)
1463 : {
1464 1107 : instrument_atomic_read_write(v, sizeof(*v));
1465 1107 : return arch_atomic64_cmpxchg(v, old, new);
1466 : }
1467 : #define atomic64_cmpxchg atomic64_cmpxchg
1468 : #endif
1469 :
1470 : #if defined(arch_atomic64_cmpxchg_acquire)
1471 : static __always_inline s64
1472 128396 : atomic64_cmpxchg_acquire(atomic64_t *v, s64 old, s64 new)
1473 : {
1474 128396 : instrument_atomic_read_write(v, sizeof(*v));
1475 128403 : return arch_atomic64_cmpxchg_acquire(v, old, new);
1476 : }
1477 : #define atomic64_cmpxchg_acquire atomic64_cmpxchg_acquire
1478 : #endif
1479 :
1480 : #if defined(arch_atomic64_cmpxchg_release)
1481 : static __always_inline s64
1482 128343 : atomic64_cmpxchg_release(atomic64_t *v, s64 old, s64 new)
1483 : {
1484 128343 : instrument_atomic_read_write(v, sizeof(*v));
1485 128343 : return arch_atomic64_cmpxchg_release(v, old, new);
1486 : }
1487 : #define atomic64_cmpxchg_release atomic64_cmpxchg_release
1488 : #endif
1489 :
1490 : #if defined(arch_atomic64_cmpxchg_relaxed)
1491 : static __always_inline s64
1492 268 : atomic64_cmpxchg_relaxed(atomic64_t *v, s64 old, s64 new)
1493 : {
1494 268 : instrument_atomic_read_write(v, sizeof(*v));
1495 268 : return arch_atomic64_cmpxchg_relaxed(v, old, new);
1496 : }
1497 : #define atomic64_cmpxchg_relaxed atomic64_cmpxchg_relaxed
1498 : #endif
1499 :
1500 : #if defined(arch_atomic64_try_cmpxchg)
1501 : static __always_inline bool
1502 182815 : atomic64_try_cmpxchg(atomic64_t *v, s64 *old, s64 new)
1503 : {
1504 182815 : instrument_atomic_read_write(v, sizeof(*v));
1505 182817 : instrument_atomic_read_write(old, sizeof(*old));
1506 365344 : return arch_atomic64_try_cmpxchg(v, old, new);
1507 : }
1508 : #define atomic64_try_cmpxchg atomic64_try_cmpxchg
1509 : #endif
1510 :
1511 : #if defined(arch_atomic64_try_cmpxchg_acquire)
1512 : static __always_inline bool
1513 473788 : atomic64_try_cmpxchg_acquire(atomic64_t *v, s64 *old, s64 new)
1514 : {
1515 473788 : instrument_atomic_read_write(v, sizeof(*v));
1516 473763 : instrument_atomic_read_write(old, sizeof(*old));
1517 947613 : return arch_atomic64_try_cmpxchg_acquire(v, old, new);
1518 : }
1519 : #define atomic64_try_cmpxchg_acquire atomic64_try_cmpxchg_acquire
1520 : #endif
1521 :
1522 : #if defined(arch_atomic64_try_cmpxchg_release)
1523 : static __always_inline bool
1524 : atomic64_try_cmpxchg_release(atomic64_t *v, s64 *old, s64 new)
1525 : {
1526 : instrument_atomic_read_write(v, sizeof(*v));
1527 : instrument_atomic_read_write(old, sizeof(*old));
1528 : return arch_atomic64_try_cmpxchg_release(v, old, new);
1529 : }
1530 : #define atomic64_try_cmpxchg_release atomic64_try_cmpxchg_release
1531 : #endif
1532 :
1533 : #if defined(arch_atomic64_try_cmpxchg_relaxed)
1534 : static __always_inline bool
1535 : atomic64_try_cmpxchg_relaxed(atomic64_t *v, s64 *old, s64 new)
1536 : {
1537 : instrument_atomic_read_write(v, sizeof(*v));
1538 : instrument_atomic_read_write(old, sizeof(*old));
1539 : return arch_atomic64_try_cmpxchg_relaxed(v, old, new);
1540 : }
1541 : #define atomic64_try_cmpxchg_relaxed atomic64_try_cmpxchg_relaxed
1542 : #endif
1543 :
1544 : #if defined(arch_atomic64_sub_and_test)
1545 : static __always_inline bool
1546 197711 : atomic64_sub_and_test(s64 i, atomic64_t *v)
1547 : {
1548 197711 : instrument_atomic_read_write(v, sizeof(*v));
1549 197703 : return arch_atomic64_sub_and_test(i, v);
1550 : }
1551 : #define atomic64_sub_and_test atomic64_sub_and_test
1552 : #endif
1553 :
1554 : #if defined(arch_atomic64_dec_and_test)
1555 : static __always_inline bool
1556 251 : atomic64_dec_and_test(atomic64_t *v)
1557 : {
1558 251 : instrument_atomic_read_write(v, sizeof(*v));
1559 251 : return arch_atomic64_dec_and_test(v);
1560 : }
1561 : #define atomic64_dec_and_test atomic64_dec_and_test
1562 : #endif
1563 :
1564 : #if defined(arch_atomic64_inc_and_test)
1565 : static __always_inline bool
1566 : atomic64_inc_and_test(atomic64_t *v)
1567 : {
1568 : instrument_atomic_read_write(v, sizeof(*v));
1569 : return arch_atomic64_inc_and_test(v);
1570 : }
1571 : #define atomic64_inc_and_test atomic64_inc_and_test
1572 : #endif
1573 :
1574 : #if defined(arch_atomic64_add_negative)
1575 : static __always_inline bool
1576 : atomic64_add_negative(s64 i, atomic64_t *v)
1577 : {
1578 : instrument_atomic_read_write(v, sizeof(*v));
1579 : return arch_atomic64_add_negative(i, v);
1580 : }
1581 : #define atomic64_add_negative atomic64_add_negative
1582 : #endif
1583 :
1584 : #if defined(arch_atomic64_fetch_add_unless)
1585 : static __always_inline s64
1586 : atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u)
1587 : {
1588 : instrument_atomic_read_write(v, sizeof(*v));
1589 : return arch_atomic64_fetch_add_unless(v, a, u);
1590 : }
1591 : #define atomic64_fetch_add_unless atomic64_fetch_add_unless
1592 : #endif
1593 :
1594 : #if defined(arch_atomic64_add_unless)
1595 : static __always_inline bool
1596 20516 : atomic64_add_unless(atomic64_t *v, s64 a, s64 u)
1597 : {
1598 20516 : instrument_atomic_read_write(v, sizeof(*v));
1599 41032 : return arch_atomic64_add_unless(v, a, u);
1600 : }
1601 : #define atomic64_add_unless atomic64_add_unless
1602 : #endif
1603 :
1604 : #if defined(arch_atomic64_inc_not_zero)
1605 : static __always_inline bool
1606 1 : atomic64_inc_not_zero(atomic64_t *v)
1607 : {
1608 1 : instrument_atomic_read_write(v, sizeof(*v));
1609 2 : return arch_atomic64_inc_not_zero(v);
1610 : }
1611 : #define atomic64_inc_not_zero atomic64_inc_not_zero
1612 : #endif
1613 :
1614 : #if defined(arch_atomic64_inc_unless_negative)
1615 : static __always_inline bool
1616 : atomic64_inc_unless_negative(atomic64_t *v)
1617 : {
1618 : instrument_atomic_read_write(v, sizeof(*v));
1619 : return arch_atomic64_inc_unless_negative(v);
1620 : }
1621 : #define atomic64_inc_unless_negative atomic64_inc_unless_negative
1622 : #endif
1623 :
1624 : #if defined(arch_atomic64_dec_unless_positive)
1625 : static __always_inline bool
1626 : atomic64_dec_unless_positive(atomic64_t *v)
1627 : {
1628 : instrument_atomic_read_write(v, sizeof(*v));
1629 : return arch_atomic64_dec_unless_positive(v);
1630 : }
1631 : #define atomic64_dec_unless_positive atomic64_dec_unless_positive
1632 : #endif
1633 :
1634 : #if defined(arch_atomic64_dec_if_positive)
1635 : static __always_inline s64
1636 : atomic64_dec_if_positive(atomic64_t *v)
1637 : {
1638 : instrument_atomic_read_write(v, sizeof(*v));
1639 : return arch_atomic64_dec_if_positive(v);
1640 : }
1641 : #define atomic64_dec_if_positive atomic64_dec_if_positive
1642 : #endif
1643 :
1644 : #if !defined(arch_xchg_relaxed) || defined(arch_xchg)
1645 : #define xchg(ptr, ...) \
1646 : ({ \
1647 : typeof(ptr) __ai_ptr = (ptr); \
1648 : instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1649 : arch_xchg(__ai_ptr, __VA_ARGS__); \
1650 : })
1651 : #endif
1652 :
1653 : #if defined(arch_xchg_acquire)
1654 : #define xchg_acquire(ptr, ...) \
1655 : ({ \
1656 : typeof(ptr) __ai_ptr = (ptr); \
1657 : instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1658 : arch_xchg_acquire(__ai_ptr, __VA_ARGS__); \
1659 : })
1660 : #endif
1661 :
1662 : #if defined(arch_xchg_release)
1663 : #define xchg_release(ptr, ...) \
1664 : ({ \
1665 : typeof(ptr) __ai_ptr = (ptr); \
1666 : instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1667 : arch_xchg_release(__ai_ptr, __VA_ARGS__); \
1668 : })
1669 : #endif
1670 :
1671 : #if defined(arch_xchg_relaxed)
1672 : #define xchg_relaxed(ptr, ...) \
1673 : ({ \
1674 : typeof(ptr) __ai_ptr = (ptr); \
1675 : instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1676 : arch_xchg_relaxed(__ai_ptr, __VA_ARGS__); \
1677 : })
1678 : #endif
1679 :
1680 : #if !defined(arch_cmpxchg_relaxed) || defined(arch_cmpxchg)
1681 : #define cmpxchg(ptr, ...) \
1682 : ({ \
1683 : typeof(ptr) __ai_ptr = (ptr); \
1684 : instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1685 : arch_cmpxchg(__ai_ptr, __VA_ARGS__); \
1686 : })
1687 : #endif
1688 :
1689 : #if defined(arch_cmpxchg_acquire)
1690 : #define cmpxchg_acquire(ptr, ...) \
1691 : ({ \
1692 : typeof(ptr) __ai_ptr = (ptr); \
1693 : instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1694 : arch_cmpxchg_acquire(__ai_ptr, __VA_ARGS__); \
1695 : })
1696 : #endif
1697 :
1698 : #if defined(arch_cmpxchg_release)
1699 : #define cmpxchg_release(ptr, ...) \
1700 : ({ \
1701 : typeof(ptr) __ai_ptr = (ptr); \
1702 : instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1703 : arch_cmpxchg_release(__ai_ptr, __VA_ARGS__); \
1704 : })
1705 : #endif
1706 :
1707 : #if defined(arch_cmpxchg_relaxed)
1708 : #define cmpxchg_relaxed(ptr, ...) \
1709 : ({ \
1710 : typeof(ptr) __ai_ptr = (ptr); \
1711 : instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1712 : arch_cmpxchg_relaxed(__ai_ptr, __VA_ARGS__); \
1713 : })
1714 : #endif
1715 :
1716 : #if !defined(arch_cmpxchg64_relaxed) || defined(arch_cmpxchg64)
1717 : #define cmpxchg64(ptr, ...) \
1718 : ({ \
1719 : typeof(ptr) __ai_ptr = (ptr); \
1720 : instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1721 : arch_cmpxchg64(__ai_ptr, __VA_ARGS__); \
1722 : })
1723 : #endif
1724 :
1725 : #if defined(arch_cmpxchg64_acquire)
1726 : #define cmpxchg64_acquire(ptr, ...) \
1727 : ({ \
1728 : typeof(ptr) __ai_ptr = (ptr); \
1729 : instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1730 : arch_cmpxchg64_acquire(__ai_ptr, __VA_ARGS__); \
1731 : })
1732 : #endif
1733 :
1734 : #if defined(arch_cmpxchg64_release)
1735 : #define cmpxchg64_release(ptr, ...) \
1736 : ({ \
1737 : typeof(ptr) __ai_ptr = (ptr); \
1738 : instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1739 : arch_cmpxchg64_release(__ai_ptr, __VA_ARGS__); \
1740 : })
1741 : #endif
1742 :
1743 : #if defined(arch_cmpxchg64_relaxed)
1744 : #define cmpxchg64_relaxed(ptr, ...) \
1745 : ({ \
1746 : typeof(ptr) __ai_ptr = (ptr); \
1747 : instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1748 : arch_cmpxchg64_relaxed(__ai_ptr, __VA_ARGS__); \
1749 : })
1750 : #endif
1751 :
1752 : #if !defined(arch_try_cmpxchg_relaxed) || defined(arch_try_cmpxchg)
1753 : #define try_cmpxchg(ptr, oldp, ...) \
1754 : ({ \
1755 : typeof(ptr) __ai_ptr = (ptr); \
1756 : typeof(oldp) __ai_oldp = (oldp); \
1757 : instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1758 : instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \
1759 : arch_try_cmpxchg(__ai_ptr, __ai_oldp, __VA_ARGS__); \
1760 : })
1761 : #endif
1762 :
1763 : #if defined(arch_try_cmpxchg_acquire)
1764 : #define try_cmpxchg_acquire(ptr, oldp, ...) \
1765 : ({ \
1766 : typeof(ptr) __ai_ptr = (ptr); \
1767 : typeof(oldp) __ai_oldp = (oldp); \
1768 : instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1769 : instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \
1770 : arch_try_cmpxchg_acquire(__ai_ptr, __ai_oldp, __VA_ARGS__); \
1771 : })
1772 : #endif
1773 :
1774 : #if defined(arch_try_cmpxchg_release)
1775 : #define try_cmpxchg_release(ptr, oldp, ...) \
1776 : ({ \
1777 : typeof(ptr) __ai_ptr = (ptr); \
1778 : typeof(oldp) __ai_oldp = (oldp); \
1779 : instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1780 : instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \
1781 : arch_try_cmpxchg_release(__ai_ptr, __ai_oldp, __VA_ARGS__); \
1782 : })
1783 : #endif
1784 :
1785 : #if defined(arch_try_cmpxchg_relaxed)
1786 : #define try_cmpxchg_relaxed(ptr, oldp, ...) \
1787 : ({ \
1788 : typeof(ptr) __ai_ptr = (ptr); \
1789 : typeof(oldp) __ai_oldp = (oldp); \
1790 : instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1791 : instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \
1792 : arch_try_cmpxchg_relaxed(__ai_ptr, __ai_oldp, __VA_ARGS__); \
1793 : })
1794 : #endif
1795 :
1796 : #define cmpxchg_local(ptr, ...) \
1797 : ({ \
1798 : typeof(ptr) __ai_ptr = (ptr); \
1799 : instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1800 : arch_cmpxchg_local(__ai_ptr, __VA_ARGS__); \
1801 : })
1802 :
1803 : #define cmpxchg64_local(ptr, ...) \
1804 : ({ \
1805 : typeof(ptr) __ai_ptr = (ptr); \
1806 : instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1807 : arch_cmpxchg64_local(__ai_ptr, __VA_ARGS__); \
1808 : })
1809 :
1810 : #define sync_cmpxchg(ptr, ...) \
1811 : ({ \
1812 : typeof(ptr) __ai_ptr = (ptr); \
1813 : instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1814 : arch_sync_cmpxchg(__ai_ptr, __VA_ARGS__); \
1815 : })
1816 :
1817 : #define cmpxchg_double(ptr, ...) \
1818 : ({ \
1819 : typeof(ptr) __ai_ptr = (ptr); \
1820 : instrument_atomic_write(__ai_ptr, 2 * sizeof(*__ai_ptr)); \
1821 : arch_cmpxchg_double(__ai_ptr, __VA_ARGS__); \
1822 : })
1823 :
1824 :
1825 : #define cmpxchg_double_local(ptr, ...) \
1826 : ({ \
1827 : typeof(ptr) __ai_ptr = (ptr); \
1828 : instrument_atomic_write(__ai_ptr, 2 * sizeof(*__ai_ptr)); \
1829 : arch_cmpxchg_double_local(__ai_ptr, __VA_ARGS__); \
1830 : })
1831 :
1832 : #endif /* _ASM_GENERIC_ATOMIC_INSTRUMENTED_H */
1833 : // 4bec382e44520f4d8267e42620054db26a659ea3
|
