LCOV - code coverage report
Current view: top level - include/linux - atomic-arch-fallback.h (source / functions) Hit Total Coverage
Test: landlock.info Lines: 51 56 91.1 %
Date: 2021-04-22 12:43:58 Functions: 0 0 -

          Line data    Source code
       1             : // SPDX-License-Identifier: GPL-2.0
       2             : 
       3             : // Generated by scripts/atomic/gen-atomic-fallback.sh
       4             : // DO NOT MODIFY THIS FILE DIRECTLY
       5             : 
       6             : #ifndef _LINUX_ATOMIC_FALLBACK_H
       7             : #define _LINUX_ATOMIC_FALLBACK_H
       8             : 
       9             : #include <linux/compiler.h>
      10             : 
      11             : #ifndef arch_xchg_relaxed
      12             : #define arch_xchg_acquire arch_xchg
      13             : #define arch_xchg_release arch_xchg
      14             : #define arch_xchg_relaxed arch_xchg
      15             : #else /* arch_xchg_relaxed */
      16             : 
      17             : #ifndef arch_xchg_acquire
      18             : #define arch_xchg_acquire(...) \
      19             :         __atomic_op_acquire(arch_xchg, __VA_ARGS__)
      20             : #endif
      21             : 
      22             : #ifndef arch_xchg_release
      23             : #define arch_xchg_release(...) \
      24             :         __atomic_op_release(arch_xchg, __VA_ARGS__)
      25             : #endif
      26             : 
      27             : #ifndef arch_xchg
      28             : #define arch_xchg(...) \
      29             :         __atomic_op_fence(arch_xchg, __VA_ARGS__)
      30             : #endif
      31             : 
      32             : #endif /* arch_xchg_relaxed */
      33             : 
      34             : #ifndef arch_cmpxchg_relaxed
      35             : #define arch_cmpxchg_acquire arch_cmpxchg
      36             : #define arch_cmpxchg_release arch_cmpxchg
      37             : #define arch_cmpxchg_relaxed arch_cmpxchg
      38             : #else /* arch_cmpxchg_relaxed */
      39             : 
      40             : #ifndef arch_cmpxchg_acquire
      41             : #define arch_cmpxchg_acquire(...) \
      42             :         __atomic_op_acquire(arch_cmpxchg, __VA_ARGS__)
      43             : #endif
      44             : 
      45             : #ifndef arch_cmpxchg_release
      46             : #define arch_cmpxchg_release(...) \
      47             :         __atomic_op_release(arch_cmpxchg, __VA_ARGS__)
      48             : #endif
      49             : 
      50             : #ifndef arch_cmpxchg
      51             : #define arch_cmpxchg(...) \
      52             :         __atomic_op_fence(arch_cmpxchg, __VA_ARGS__)
      53             : #endif
      54             : 
      55             : #endif /* arch_cmpxchg_relaxed */
      56             : 
      57             : #ifndef arch_cmpxchg64_relaxed
      58             : #define arch_cmpxchg64_acquire arch_cmpxchg64
      59             : #define arch_cmpxchg64_release arch_cmpxchg64
      60             : #define arch_cmpxchg64_relaxed arch_cmpxchg64
      61             : #else /* arch_cmpxchg64_relaxed */
      62             : 
      63             : #ifndef arch_cmpxchg64_acquire
      64             : #define arch_cmpxchg64_acquire(...) \
      65             :         __atomic_op_acquire(arch_cmpxchg64, __VA_ARGS__)
      66             : #endif
      67             : 
      68             : #ifndef arch_cmpxchg64_release
      69             : #define arch_cmpxchg64_release(...) \
      70             :         __atomic_op_release(arch_cmpxchg64, __VA_ARGS__)
      71             : #endif
      72             : 
      73             : #ifndef arch_cmpxchg64
      74             : #define arch_cmpxchg64(...) \
      75             :         __atomic_op_fence(arch_cmpxchg64, __VA_ARGS__)
      76             : #endif
      77             : 
      78             : #endif /* arch_cmpxchg64_relaxed */
      79             : 
      80             : #ifndef arch_try_cmpxchg_relaxed
      81             : #ifdef arch_try_cmpxchg
      82             : #define arch_try_cmpxchg_acquire arch_try_cmpxchg
      83             : #define arch_try_cmpxchg_release arch_try_cmpxchg
      84             : #define arch_try_cmpxchg_relaxed arch_try_cmpxchg
      85             : #endif /* arch_try_cmpxchg */
      86             : 
      87             : #ifndef arch_try_cmpxchg
      88             : #define arch_try_cmpxchg(_ptr, _oldp, _new) \
      89             : ({ \
      90             :         typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
      91             :         ___r = arch_cmpxchg((_ptr), ___o, (_new)); \
      92             :         if (unlikely(___r != ___o)) \
      93             :                 *___op = ___r; \
      94             :         likely(___r == ___o); \
      95             : })
      96             : #endif /* arch_try_cmpxchg */
      97             : 
      98             : #ifndef arch_try_cmpxchg_acquire
      99             : #define arch_try_cmpxchg_acquire(_ptr, _oldp, _new) \
     100             : ({ \
     101             :         typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
     102             :         ___r = arch_cmpxchg_acquire((_ptr), ___o, (_new)); \
     103             :         if (unlikely(___r != ___o)) \
     104             :                 *___op = ___r; \
     105             :         likely(___r == ___o); \
     106             : })
     107             : #endif /* arch_try_cmpxchg_acquire */
     108             : 
     109             : #ifndef arch_try_cmpxchg_release
     110             : #define arch_try_cmpxchg_release(_ptr, _oldp, _new) \
     111             : ({ \
     112             :         typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
     113             :         ___r = arch_cmpxchg_release((_ptr), ___o, (_new)); \
     114             :         if (unlikely(___r != ___o)) \
     115             :                 *___op = ___r; \
     116             :         likely(___r == ___o); \
     117             : })
     118             : #endif /* arch_try_cmpxchg_release */
     119             : 
     120             : #ifndef arch_try_cmpxchg_relaxed
     121             : #define arch_try_cmpxchg_relaxed(_ptr, _oldp, _new) \
     122             : ({ \
     123             :         typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
     124             :         ___r = arch_cmpxchg_relaxed((_ptr), ___o, (_new)); \
     125             :         if (unlikely(___r != ___o)) \
     126             :                 *___op = ___r; \
     127             :         likely(___r == ___o); \
     128             : })
     129             : #endif /* arch_try_cmpxchg_relaxed */
     130             : 
     131             : #else /* arch_try_cmpxchg_relaxed */
     132             : 
     133             : #ifndef arch_try_cmpxchg_acquire
     134             : #define arch_try_cmpxchg_acquire(...) \
     135             :         __atomic_op_acquire(arch_try_cmpxchg, __VA_ARGS__)
     136             : #endif
     137             : 
     138             : #ifndef arch_try_cmpxchg_release
     139             : #define arch_try_cmpxchg_release(...) \
     140             :         __atomic_op_release(arch_try_cmpxchg, __VA_ARGS__)
     141             : #endif
     142             : 
     143             : #ifndef arch_try_cmpxchg
     144             : #define arch_try_cmpxchg(...) \
     145             :         __atomic_op_fence(arch_try_cmpxchg, __VA_ARGS__)
     146             : #endif
     147             : 
     148             : #endif /* arch_try_cmpxchg_relaxed */
     149             : 
     150             : #ifndef arch_atomic_read_acquire
     151             : static __always_inline int
     152        1971 : arch_atomic_read_acquire(const atomic_t *v)
     153             : {
     154        1971 :         return smp_load_acquire(&(v)->counter);
     155             : }
     156             : #define arch_atomic_read_acquire arch_atomic_read_acquire
     157             : #endif
     158             : 
     159             : #ifndef arch_atomic_set_release
     160             : static __always_inline void
     161         201 : arch_atomic_set_release(atomic_t *v, int i)
     162             : {
     163         201 :         smp_store_release(&(v)->counter, i);
     164             : }
     165             : #define arch_atomic_set_release arch_atomic_set_release
     166             : #endif
     167             : 
     168             : #ifndef arch_atomic_add_return_relaxed
     169             : #define arch_atomic_add_return_acquire arch_atomic_add_return
     170             : #define arch_atomic_add_return_release arch_atomic_add_return
     171             : #define arch_atomic_add_return_relaxed arch_atomic_add_return
     172             : #else /* arch_atomic_add_return_relaxed */
     173             : 
     174             : #ifndef arch_atomic_add_return_acquire
     175             : static __always_inline int
     176             : arch_atomic_add_return_acquire(int i, atomic_t *v)
     177             : {
     178             :         int ret = arch_atomic_add_return_relaxed(i, v);
     179             :         __atomic_acquire_fence();
     180             :         return ret;
     181             : }
     182             : #define arch_atomic_add_return_acquire arch_atomic_add_return_acquire
     183             : #endif
     184             : 
     185             : #ifndef arch_atomic_add_return_release
     186             : static __always_inline int
     187             : arch_atomic_add_return_release(int i, atomic_t *v)
     188             : {
     189             :         __atomic_release_fence();
     190             :         return arch_atomic_add_return_relaxed(i, v);
     191             : }
     192             : #define arch_atomic_add_return_release arch_atomic_add_return_release
     193             : #endif
     194             : 
     195             : #ifndef arch_atomic_add_return
     196             : static __always_inline int
     197             : arch_atomic_add_return(int i, atomic_t *v)
     198             : {
     199             :         int ret;
     200             :         __atomic_pre_full_fence();
     201             :         ret = arch_atomic_add_return_relaxed(i, v);
     202             :         __atomic_post_full_fence();
     203             :         return ret;
     204             : }
     205             : #define arch_atomic_add_return arch_atomic_add_return
     206             : #endif
     207             : 
     208             : #endif /* arch_atomic_add_return_relaxed */
     209             : 
     210             : #ifndef arch_atomic_fetch_add_relaxed
     211             : #define arch_atomic_fetch_add_acquire arch_atomic_fetch_add
     212             : #define arch_atomic_fetch_add_release arch_atomic_fetch_add
     213             : #define arch_atomic_fetch_add_relaxed arch_atomic_fetch_add
     214             : #else /* arch_atomic_fetch_add_relaxed */
     215             : 
     216             : #ifndef arch_atomic_fetch_add_acquire
     217             : static __always_inline int
     218             : arch_atomic_fetch_add_acquire(int i, atomic_t *v)
     219             : {
     220             :         int ret = arch_atomic_fetch_add_relaxed(i, v);
     221             :         __atomic_acquire_fence();
     222             :         return ret;
     223             : }
     224             : #define arch_atomic_fetch_add_acquire arch_atomic_fetch_add_acquire
     225             : #endif
     226             : 
     227             : #ifndef arch_atomic_fetch_add_release
     228             : static __always_inline int
     229             : arch_atomic_fetch_add_release(int i, atomic_t *v)
     230             : {
     231             :         __atomic_release_fence();
     232             :         return arch_atomic_fetch_add_relaxed(i, v);
     233             : }
     234             : #define arch_atomic_fetch_add_release arch_atomic_fetch_add_release
     235             : #endif
     236             : 
     237             : #ifndef arch_atomic_fetch_add
     238             : static __always_inline int
     239             : arch_atomic_fetch_add(int i, atomic_t *v)
     240             : {
     241             :         int ret;
     242             :         __atomic_pre_full_fence();
     243             :         ret = arch_atomic_fetch_add_relaxed(i, v);
     244             :         __atomic_post_full_fence();
     245             :         return ret;
     246             : }
     247             : #define arch_atomic_fetch_add arch_atomic_fetch_add
     248             : #endif
     249             : 
     250             : #endif /* arch_atomic_fetch_add_relaxed */
     251             : 
     252             : #ifndef arch_atomic_sub_return_relaxed
     253             : #define arch_atomic_sub_return_acquire arch_atomic_sub_return
     254             : #define arch_atomic_sub_return_release arch_atomic_sub_return
     255             : #define arch_atomic_sub_return_relaxed arch_atomic_sub_return
     256             : #else /* arch_atomic_sub_return_relaxed */
     257             : 
     258             : #ifndef arch_atomic_sub_return_acquire
     259             : static __always_inline int
     260             : arch_atomic_sub_return_acquire(int i, atomic_t *v)
     261             : {
     262             :         int ret = arch_atomic_sub_return_relaxed(i, v);
     263             :         __atomic_acquire_fence();
     264             :         return ret;
     265             : }
     266             : #define arch_atomic_sub_return_acquire arch_atomic_sub_return_acquire
     267             : #endif
     268             : 
     269             : #ifndef arch_atomic_sub_return_release
     270             : static __always_inline int
     271             : arch_atomic_sub_return_release(int i, atomic_t *v)
     272             : {
     273             :         __atomic_release_fence();
     274             :         return arch_atomic_sub_return_relaxed(i, v);
     275             : }
     276             : #define arch_atomic_sub_return_release arch_atomic_sub_return_release
     277             : #endif
     278             : 
     279             : #ifndef arch_atomic_sub_return
     280             : static __always_inline int
     281             : arch_atomic_sub_return(int i, atomic_t *v)
     282             : {
     283             :         int ret;
     284             :         __atomic_pre_full_fence();
     285             :         ret = arch_atomic_sub_return_relaxed(i, v);
     286             :         __atomic_post_full_fence();
     287             :         return ret;
     288             : }
     289             : #define arch_atomic_sub_return arch_atomic_sub_return
     290             : #endif
     291             : 
     292             : #endif /* arch_atomic_sub_return_relaxed */
     293             : 
     294             : #ifndef arch_atomic_fetch_sub_relaxed
     295             : #define arch_atomic_fetch_sub_acquire arch_atomic_fetch_sub
     296             : #define arch_atomic_fetch_sub_release arch_atomic_fetch_sub
     297             : #define arch_atomic_fetch_sub_relaxed arch_atomic_fetch_sub
     298             : #else /* arch_atomic_fetch_sub_relaxed */
     299             : 
     300             : #ifndef arch_atomic_fetch_sub_acquire
     301             : static __always_inline int
     302             : arch_atomic_fetch_sub_acquire(int i, atomic_t *v)
     303             : {
     304             :         int ret = arch_atomic_fetch_sub_relaxed(i, v);
     305             :         __atomic_acquire_fence();
     306             :         return ret;
     307             : }
     308             : #define arch_atomic_fetch_sub_acquire arch_atomic_fetch_sub_acquire
     309             : #endif
     310             : 
     311             : #ifndef arch_atomic_fetch_sub_release
     312             : static __always_inline int
     313             : arch_atomic_fetch_sub_release(int i, atomic_t *v)
     314             : {
     315             :         __atomic_release_fence();
     316             :         return arch_atomic_fetch_sub_relaxed(i, v);
     317             : }
     318             : #define arch_atomic_fetch_sub_release arch_atomic_fetch_sub_release
     319             : #endif
     320             : 
     321             : #ifndef arch_atomic_fetch_sub
     322             : static __always_inline int
     323             : arch_atomic_fetch_sub(int i, atomic_t *v)
     324             : {
     325             :         int ret;
     326             :         __atomic_pre_full_fence();
     327             :         ret = arch_atomic_fetch_sub_relaxed(i, v);
     328             :         __atomic_post_full_fence();
     329             :         return ret;
     330             : }
     331             : #define arch_atomic_fetch_sub arch_atomic_fetch_sub
     332             : #endif
     333             : 
     334             : #endif /* arch_atomic_fetch_sub_relaxed */
     335             : 
     336             : #ifndef arch_atomic_inc
     337             : static __always_inline void
     338             : arch_atomic_inc(atomic_t *v)
     339             : {
     340             :         arch_atomic_add(1, v);
     341             : }
     342             : #define arch_atomic_inc arch_atomic_inc
     343             : #endif
     344             : 
     345             : #ifndef arch_atomic_inc_return_relaxed
     346             : #ifdef arch_atomic_inc_return
     347             : #define arch_atomic_inc_return_acquire arch_atomic_inc_return
     348             : #define arch_atomic_inc_return_release arch_atomic_inc_return
     349             : #define arch_atomic_inc_return_relaxed arch_atomic_inc_return
     350             : #endif /* arch_atomic_inc_return */
     351             : 
     352             : #ifndef arch_atomic_inc_return
     353             : static __always_inline int
     354        2072 : arch_atomic_inc_return(atomic_t *v)
     355             : {
     356        2048 :         return arch_atomic_add_return(1, v);
     357             : }
     358             : #define arch_atomic_inc_return arch_atomic_inc_return
     359             : #endif
     360             : 
     361             : #ifndef arch_atomic_inc_return_acquire
     362             : static __always_inline int
     363             : arch_atomic_inc_return_acquire(atomic_t *v)
     364             : {
     365             :         return arch_atomic_add_return_acquire(1, v);
     366             : }
     367             : #define arch_atomic_inc_return_acquire arch_atomic_inc_return_acquire
     368             : #endif
     369             : 
     370             : #ifndef arch_atomic_inc_return_release
     371             : static __always_inline int
     372             : arch_atomic_inc_return_release(atomic_t *v)
     373             : {
     374             :         return arch_atomic_add_return_release(1, v);
     375             : }
     376             : #define arch_atomic_inc_return_release arch_atomic_inc_return_release
     377             : #endif
     378             : 
     379             : #ifndef arch_atomic_inc_return_relaxed
     380             : static __always_inline int
     381             : arch_atomic_inc_return_relaxed(atomic_t *v)
     382             : {
     383             :         return arch_atomic_add_return_relaxed(1, v);
     384             : }
     385             : #define arch_atomic_inc_return_relaxed arch_atomic_inc_return_relaxed
     386             : #endif
     387             : 
     388             : #else /* arch_atomic_inc_return_relaxed */
     389             : 
     390             : #ifndef arch_atomic_inc_return_acquire
     391             : static __always_inline int
     392             : arch_atomic_inc_return_acquire(atomic_t *v)
     393             : {
     394             :         int ret = arch_atomic_inc_return_relaxed(v);
     395             :         __atomic_acquire_fence();
     396             :         return ret;
     397             : }
     398             : #define arch_atomic_inc_return_acquire arch_atomic_inc_return_acquire
     399             : #endif
     400             : 
     401             : #ifndef arch_atomic_inc_return_release
     402             : static __always_inline int
     403             : arch_atomic_inc_return_release(atomic_t *v)
     404             : {
     405             :         __atomic_release_fence();
     406             :         return arch_atomic_inc_return_relaxed(v);
     407             : }
     408             : #define arch_atomic_inc_return_release arch_atomic_inc_return_release
     409             : #endif
     410             : 
     411             : #ifndef arch_atomic_inc_return
     412             : static __always_inline int
     413             : arch_atomic_inc_return(atomic_t *v)
     414             : {
     415             :         int ret;
     416             :         __atomic_pre_full_fence();
     417             :         ret = arch_atomic_inc_return_relaxed(v);
     418             :         __atomic_post_full_fence();
     419             :         return ret;
     420             : }
     421             : #define arch_atomic_inc_return arch_atomic_inc_return
     422             : #endif
     423             : 
     424             : #endif /* arch_atomic_inc_return_relaxed */
     425             : 
     426             : #ifndef arch_atomic_fetch_inc_relaxed
     427             : #ifdef arch_atomic_fetch_inc
     428             : #define arch_atomic_fetch_inc_acquire arch_atomic_fetch_inc
     429             : #define arch_atomic_fetch_inc_release arch_atomic_fetch_inc
     430             : #define arch_atomic_fetch_inc_relaxed arch_atomic_fetch_inc
     431             : #endif /* arch_atomic_fetch_inc */
     432             : 
     433             : #ifndef arch_atomic_fetch_inc
     434             : static __always_inline int
     435             : arch_atomic_fetch_inc(atomic_t *v)
     436             : {
     437             :         return arch_atomic_fetch_add(1, v);
     438             : }
     439             : #define arch_atomic_fetch_inc arch_atomic_fetch_inc
     440             : #endif
     441             : 
     442             : #ifndef arch_atomic_fetch_inc_acquire
     443             : static __always_inline int
     444             : arch_atomic_fetch_inc_acquire(atomic_t *v)
     445             : {
     446             :         return arch_atomic_fetch_add_acquire(1, v);
     447             : }
     448             : #define arch_atomic_fetch_inc_acquire arch_atomic_fetch_inc_acquire
     449             : #endif
     450             : 
     451             : #ifndef arch_atomic_fetch_inc_release
     452             : static __always_inline int
     453             : arch_atomic_fetch_inc_release(atomic_t *v)
     454             : {
     455             :         return arch_atomic_fetch_add_release(1, v);
     456             : }
     457             : #define arch_atomic_fetch_inc_release arch_atomic_fetch_inc_release
     458             : #endif
     459             : 
     460             : #ifndef arch_atomic_fetch_inc_relaxed
     461             : static __always_inline int
     462             : arch_atomic_fetch_inc_relaxed(atomic_t *v)
     463             : {
     464             :         return arch_atomic_fetch_add_relaxed(1, v);
     465             : }
     466             : #define arch_atomic_fetch_inc_relaxed arch_atomic_fetch_inc_relaxed
     467             : #endif
     468             : 
     469             : #else /* arch_atomic_fetch_inc_relaxed */
     470             : 
     471             : #ifndef arch_atomic_fetch_inc_acquire
     472             : static __always_inline int
     473             : arch_atomic_fetch_inc_acquire(atomic_t *v)
     474             : {
     475             :         int ret = arch_atomic_fetch_inc_relaxed(v);
     476             :         __atomic_acquire_fence();
     477             :         return ret;
     478             : }
     479             : #define arch_atomic_fetch_inc_acquire arch_atomic_fetch_inc_acquire
     480             : #endif
     481             : 
     482             : #ifndef arch_atomic_fetch_inc_release
     483             : static __always_inline int
     484             : arch_atomic_fetch_inc_release(atomic_t *v)
     485             : {
     486             :         __atomic_release_fence();
     487             :         return arch_atomic_fetch_inc_relaxed(v);
     488             : }
     489             : #define arch_atomic_fetch_inc_release arch_atomic_fetch_inc_release
     490             : #endif
     491             : 
     492             : #ifndef arch_atomic_fetch_inc
     493             : static __always_inline int
     494             : arch_atomic_fetch_inc(atomic_t *v)
     495             : {
     496             :         int ret;
     497             :         __atomic_pre_full_fence();
     498             :         ret = arch_atomic_fetch_inc_relaxed(v);
     499             :         __atomic_post_full_fence();
     500             :         return ret;
     501             : }
     502             : #define arch_atomic_fetch_inc arch_atomic_fetch_inc
     503             : #endif
     504             : 
     505             : #endif /* arch_atomic_fetch_inc_relaxed */
     506             : 
     507             : #ifndef arch_atomic_dec
     508             : static __always_inline void
     509             : arch_atomic_dec(atomic_t *v)
     510             : {
     511             :         arch_atomic_sub(1, v);
     512             : }
     513             : #define arch_atomic_dec arch_atomic_dec
     514             : #endif
     515             : 
     516             : #ifndef arch_atomic_dec_return_relaxed
     517             : #ifdef arch_atomic_dec_return
     518             : #define arch_atomic_dec_return_acquire arch_atomic_dec_return
     519             : #define arch_atomic_dec_return_release arch_atomic_dec_return
     520             : #define arch_atomic_dec_return_relaxed arch_atomic_dec_return
     521             : #endif /* arch_atomic_dec_return */
     522             : 
     523             : #ifndef arch_atomic_dec_return
     524             : static __always_inline int
     525        4687 : arch_atomic_dec_return(atomic_t *v)
     526             : {
     527        4687 :         return arch_atomic_sub_return(1, v);
     528             : }
     529             : #define arch_atomic_dec_return arch_atomic_dec_return
     530             : #endif
     531             : 
     532             : #ifndef arch_atomic_dec_return_acquire
     533             : static __always_inline int
     534             : arch_atomic_dec_return_acquire(atomic_t *v)
     535             : {
     536             :         return arch_atomic_sub_return_acquire(1, v);
     537             : }
     538             : #define arch_atomic_dec_return_acquire arch_atomic_dec_return_acquire
     539             : #endif
     540             : 
     541             : #ifndef arch_atomic_dec_return_release
     542             : static __always_inline int
     543             : arch_atomic_dec_return_release(atomic_t *v)
     544             : {
     545             :         return arch_atomic_sub_return_release(1, v);
     546             : }
     547             : #define arch_atomic_dec_return_release arch_atomic_dec_return_release
     548             : #endif
     549             : 
     550             : #ifndef arch_atomic_dec_return_relaxed
     551             : static __always_inline int
     552             : arch_atomic_dec_return_relaxed(atomic_t *v)
     553             : {
     554             :         return arch_atomic_sub_return_relaxed(1, v);
     555             : }
     556             : #define arch_atomic_dec_return_relaxed arch_atomic_dec_return_relaxed
     557             : #endif
     558             : 
     559             : #else /* arch_atomic_dec_return_relaxed */
     560             : 
     561             : #ifndef arch_atomic_dec_return_acquire
     562             : static __always_inline int
     563             : arch_atomic_dec_return_acquire(atomic_t *v)
     564             : {
     565             :         int ret = arch_atomic_dec_return_relaxed(v);
     566             :         __atomic_acquire_fence();
     567             :         return ret;
     568             : }
     569             : #define arch_atomic_dec_return_acquire arch_atomic_dec_return_acquire
     570             : #endif
     571             : 
     572             : #ifndef arch_atomic_dec_return_release
     573             : static __always_inline int
     574             : arch_atomic_dec_return_release(atomic_t *v)
     575             : {
     576             :         __atomic_release_fence();
     577             :         return arch_atomic_dec_return_relaxed(v);
     578             : }
     579             : #define arch_atomic_dec_return_release arch_atomic_dec_return_release
     580             : #endif
     581             : 
     582             : #ifndef arch_atomic_dec_return
     583             : static __always_inline int
     584             : arch_atomic_dec_return(atomic_t *v)
     585             : {
     586             :         int ret;
     587             :         __atomic_pre_full_fence();
     588             :         ret = arch_atomic_dec_return_relaxed(v);
     589             :         __atomic_post_full_fence();
     590             :         return ret;
     591             : }
     592             : #define arch_atomic_dec_return arch_atomic_dec_return
     593             : #endif
     594             : 
     595             : #endif /* arch_atomic_dec_return_relaxed */
     596             : 
     597             : #ifndef arch_atomic_fetch_dec_relaxed
     598             : #ifdef arch_atomic_fetch_dec
     599             : #define arch_atomic_fetch_dec_acquire arch_atomic_fetch_dec
     600             : #define arch_atomic_fetch_dec_release arch_atomic_fetch_dec
     601             : #define arch_atomic_fetch_dec_relaxed arch_atomic_fetch_dec
     602             : #endif /* arch_atomic_fetch_dec */
     603             : 
     604             : #ifndef arch_atomic_fetch_dec
     605             : static __always_inline int
     606             : arch_atomic_fetch_dec(atomic_t *v)
     607             : {
     608             :         return arch_atomic_fetch_sub(1, v);
     609             : }
     610             : #define arch_atomic_fetch_dec arch_atomic_fetch_dec
     611             : #endif
     612             : 
     613             : #ifndef arch_atomic_fetch_dec_acquire
     614             : static __always_inline int
     615             : arch_atomic_fetch_dec_acquire(atomic_t *v)
     616             : {
     617             :         return arch_atomic_fetch_sub_acquire(1, v);
     618             : }
     619             : #define arch_atomic_fetch_dec_acquire arch_atomic_fetch_dec_acquire
     620             : #endif
     621             : 
     622             : #ifndef arch_atomic_fetch_dec_release
     623             : static __always_inline int
     624             : arch_atomic_fetch_dec_release(atomic_t *v)
     625             : {
     626             :         return arch_atomic_fetch_sub_release(1, v);
     627             : }
     628             : #define arch_atomic_fetch_dec_release arch_atomic_fetch_dec_release
     629             : #endif
     630             : 
     631             : #ifndef arch_atomic_fetch_dec_relaxed
     632             : static __always_inline int
     633             : arch_atomic_fetch_dec_relaxed(atomic_t *v)
     634             : {
     635             :         return arch_atomic_fetch_sub_relaxed(1, v);
     636             : }
     637             : #define arch_atomic_fetch_dec_relaxed arch_atomic_fetch_dec_relaxed
     638             : #endif
     639             : 
     640             : #else /* arch_atomic_fetch_dec_relaxed */
     641             : 
     642             : #ifndef arch_atomic_fetch_dec_acquire
     643             : static __always_inline int
     644             : arch_atomic_fetch_dec_acquire(atomic_t *v)
     645             : {
     646             :         int ret = arch_atomic_fetch_dec_relaxed(v);
     647             :         __atomic_acquire_fence();
     648             :         return ret;
     649             : }
     650             : #define arch_atomic_fetch_dec_acquire arch_atomic_fetch_dec_acquire
     651             : #endif
     652             : 
     653             : #ifndef arch_atomic_fetch_dec_release
     654             : static __always_inline int
     655             : arch_atomic_fetch_dec_release(atomic_t *v)
     656             : {
     657             :         __atomic_release_fence();
     658             :         return arch_atomic_fetch_dec_relaxed(v);
     659             : }
     660             : #define arch_atomic_fetch_dec_release arch_atomic_fetch_dec_release
     661             : #endif
     662             : 
     663             : #ifndef arch_atomic_fetch_dec
     664             : static __always_inline int
     665             : arch_atomic_fetch_dec(atomic_t *v)
     666             : {
     667             :         int ret;
     668             :         __atomic_pre_full_fence();
     669             :         ret = arch_atomic_fetch_dec_relaxed(v);
     670             :         __atomic_post_full_fence();
     671             :         return ret;
     672             : }
     673             : #define arch_atomic_fetch_dec arch_atomic_fetch_dec
     674             : #endif
     675             : 
     676             : #endif /* arch_atomic_fetch_dec_relaxed */
     677             : 
     678             : #ifndef arch_atomic_fetch_and_relaxed
     679             : #define arch_atomic_fetch_and_acquire arch_atomic_fetch_and
     680             : #define arch_atomic_fetch_and_release arch_atomic_fetch_and
     681             : #define arch_atomic_fetch_and_relaxed arch_atomic_fetch_and
     682             : #else /* arch_atomic_fetch_and_relaxed */
     683             : 
     684             : #ifndef arch_atomic_fetch_and_acquire
     685             : static __always_inline int
     686             : arch_atomic_fetch_and_acquire(int i, atomic_t *v)
     687             : {
     688             :         int ret = arch_atomic_fetch_and_relaxed(i, v);
     689             :         __atomic_acquire_fence();
     690             :         return ret;
     691             : }
     692             : #define arch_atomic_fetch_and_acquire arch_atomic_fetch_and_acquire
     693             : #endif
     694             : 
     695             : #ifndef arch_atomic_fetch_and_release
     696             : static __always_inline int
     697             : arch_atomic_fetch_and_release(int i, atomic_t *v)
     698             : {
     699             :         __atomic_release_fence();
     700             :         return arch_atomic_fetch_and_relaxed(i, v);
     701             : }
     702             : #define arch_atomic_fetch_and_release arch_atomic_fetch_and_release
     703             : #endif
     704             : 
     705             : #ifndef arch_atomic_fetch_and
     706             : static __always_inline int
     707             : arch_atomic_fetch_and(int i, atomic_t *v)
     708             : {
     709             :         int ret;
     710             :         __atomic_pre_full_fence();
     711             :         ret = arch_atomic_fetch_and_relaxed(i, v);
     712             :         __atomic_post_full_fence();
     713             :         return ret;
     714             : }
     715             : #define arch_atomic_fetch_and arch_atomic_fetch_and
     716             : #endif
     717             : 
     718             : #endif /* arch_atomic_fetch_and_relaxed */
     719             : 
     720             : #ifndef arch_atomic_andnot
     721             : static __always_inline void
     722           0 : arch_atomic_andnot(int i, atomic_t *v)
     723             : {
     724           0 :         arch_atomic_and(~i, v);
     725           0 : }
     726             : #define arch_atomic_andnot arch_atomic_andnot
     727             : #endif
     728             : 
     729             : #ifndef arch_atomic_fetch_andnot_relaxed
     730             : #ifdef arch_atomic_fetch_andnot
     731             : #define arch_atomic_fetch_andnot_acquire arch_atomic_fetch_andnot
     732             : #define arch_atomic_fetch_andnot_release arch_atomic_fetch_andnot
     733             : #define arch_atomic_fetch_andnot_relaxed arch_atomic_fetch_andnot
     734             : #endif /* arch_atomic_fetch_andnot */
     735             : 
     736             : #ifndef arch_atomic_fetch_andnot
     737             : static __always_inline int
     738         220 : arch_atomic_fetch_andnot(int i, atomic_t *v)
     739             : {
     740         440 :         return arch_atomic_fetch_and(~i, v);
     741             : }
     742             : #define arch_atomic_fetch_andnot arch_atomic_fetch_andnot
     743             : #endif
     744             : 
     745             : #ifndef arch_atomic_fetch_andnot_acquire
     746             : static __always_inline int
     747             : arch_atomic_fetch_andnot_acquire(int i, atomic_t *v)
     748             : {
     749             :         return arch_atomic_fetch_and_acquire(~i, v);
     750             : }
     751             : #define arch_atomic_fetch_andnot_acquire arch_atomic_fetch_andnot_acquire
     752             : #endif
     753             : 
     754             : #ifndef arch_atomic_fetch_andnot_release
     755             : static __always_inline int
     756             : arch_atomic_fetch_andnot_release(int i, atomic_t *v)
     757             : {
     758             :         return arch_atomic_fetch_and_release(~i, v);
     759             : }
     760             : #define arch_atomic_fetch_andnot_release arch_atomic_fetch_andnot_release
     761             : #endif
     762             : 
     763             : #ifndef arch_atomic_fetch_andnot_relaxed
     764             : static __always_inline int
     765             : arch_atomic_fetch_andnot_relaxed(int i, atomic_t *v)
     766             : {
     767             :         return arch_atomic_fetch_and_relaxed(~i, v);
     768             : }
     769             : #define arch_atomic_fetch_andnot_relaxed arch_atomic_fetch_andnot_relaxed
     770             : #endif
     771             : 
     772             : #else /* arch_atomic_fetch_andnot_relaxed */
     773             : 
     774             : #ifndef arch_atomic_fetch_andnot_acquire
     775             : static __always_inline int
     776             : arch_atomic_fetch_andnot_acquire(int i, atomic_t *v)
     777             : {
     778             :         int ret = arch_atomic_fetch_andnot_relaxed(i, v);
     779             :         __atomic_acquire_fence();
     780             :         return ret;
     781             : }
     782             : #define arch_atomic_fetch_andnot_acquire arch_atomic_fetch_andnot_acquire
     783             : #endif
     784             : 
     785             : #ifndef arch_atomic_fetch_andnot_release
     786             : static __always_inline int
     787             : arch_atomic_fetch_andnot_release(int i, atomic_t *v)
     788             : {
     789             :         __atomic_release_fence();
     790             :         return arch_atomic_fetch_andnot_relaxed(i, v);
     791             : }
     792             : #define arch_atomic_fetch_andnot_release arch_atomic_fetch_andnot_release
     793             : #endif
     794             : 
     795             : #ifndef arch_atomic_fetch_andnot
     796             : static __always_inline int
     797             : arch_atomic_fetch_andnot(int i, atomic_t *v)
     798             : {
     799             :         int ret;
     800             :         __atomic_pre_full_fence();
     801             :         ret = arch_atomic_fetch_andnot_relaxed(i, v);
     802             :         __atomic_post_full_fence();
     803             :         return ret;
     804             : }
     805             : #define arch_atomic_fetch_andnot arch_atomic_fetch_andnot
     806             : #endif
     807             : 
     808             : #endif /* arch_atomic_fetch_andnot_relaxed */
     809             : 
     810             : #ifndef arch_atomic_fetch_or_relaxed
     811             : #define arch_atomic_fetch_or_acquire arch_atomic_fetch_or
     812             : #define arch_atomic_fetch_or_release arch_atomic_fetch_or
     813             : #define arch_atomic_fetch_or_relaxed arch_atomic_fetch_or
     814             : #else /* arch_atomic_fetch_or_relaxed */
     815             : 
     816             : #ifndef arch_atomic_fetch_or_acquire
     817             : static __always_inline int
     818             : arch_atomic_fetch_or_acquire(int i, atomic_t *v)
     819             : {
     820             :         int ret = arch_atomic_fetch_or_relaxed(i, v);
     821             :         __atomic_acquire_fence();
     822             :         return ret;
     823             : }
     824             : #define arch_atomic_fetch_or_acquire arch_atomic_fetch_or_acquire
     825             : #endif
     826             : 
     827             : #ifndef arch_atomic_fetch_or_release
     828             : static __always_inline int
     829             : arch_atomic_fetch_or_release(int i, atomic_t *v)
     830             : {
     831             :         __atomic_release_fence();
     832             :         return arch_atomic_fetch_or_relaxed(i, v);
     833             : }
     834             : #define arch_atomic_fetch_or_release arch_atomic_fetch_or_release
     835             : #endif
     836             : 
     837             : #ifndef arch_atomic_fetch_or
     838             : static __always_inline int
     839             : arch_atomic_fetch_or(int i, atomic_t *v)
     840             : {
     841             :         int ret;
     842             :         __atomic_pre_full_fence();
     843             :         ret = arch_atomic_fetch_or_relaxed(i, v);
     844             :         __atomic_post_full_fence();
     845             :         return ret;
     846             : }
     847             : #define arch_atomic_fetch_or arch_atomic_fetch_or
     848             : #endif
     849             : 
     850             : #endif /* arch_atomic_fetch_or_relaxed */
     851             : 
     852             : #ifndef arch_atomic_fetch_xor_relaxed
     853             : #define arch_atomic_fetch_xor_acquire arch_atomic_fetch_xor
     854             : #define arch_atomic_fetch_xor_release arch_atomic_fetch_xor
     855             : #define arch_atomic_fetch_xor_relaxed arch_atomic_fetch_xor
     856             : #else /* arch_atomic_fetch_xor_relaxed */
     857             : 
     858             : #ifndef arch_atomic_fetch_xor_acquire
     859             : static __always_inline int
     860             : arch_atomic_fetch_xor_acquire(int i, atomic_t *v)
     861             : {
     862             :         int ret = arch_atomic_fetch_xor_relaxed(i, v);
     863             :         __atomic_acquire_fence();
     864             :         return ret;
     865             : }
     866             : #define arch_atomic_fetch_xor_acquire arch_atomic_fetch_xor_acquire
     867             : #endif
     868             : 
     869             : #ifndef arch_atomic_fetch_xor_release
     870             : static __always_inline int
     871             : arch_atomic_fetch_xor_release(int i, atomic_t *v)
     872             : {
     873             :         __atomic_release_fence();
     874             :         return arch_atomic_fetch_xor_relaxed(i, v);
     875             : }
     876             : #define arch_atomic_fetch_xor_release arch_atomic_fetch_xor_release
     877             : #endif
     878             : 
     879             : #ifndef arch_atomic_fetch_xor
     880             : static __always_inline int
     881             : arch_atomic_fetch_xor(int i, atomic_t *v)
     882             : {
     883             :         int ret;
     884             :         __atomic_pre_full_fence();
     885             :         ret = arch_atomic_fetch_xor_relaxed(i, v);
     886             :         __atomic_post_full_fence();
     887             :         return ret;
     888             : }
     889             : #define arch_atomic_fetch_xor arch_atomic_fetch_xor
     890             : #endif
     891             : 
     892             : #endif /* arch_atomic_fetch_xor_relaxed */
     893             : 
     894             : #ifndef arch_atomic_xchg_relaxed
     895             : #define arch_atomic_xchg_acquire arch_atomic_xchg
     896             : #define arch_atomic_xchg_release arch_atomic_xchg
     897             : #define arch_atomic_xchg_relaxed arch_atomic_xchg
     898             : #else /* arch_atomic_xchg_relaxed */
     899             : 
     900             : #ifndef arch_atomic_xchg_acquire
     901             : static __always_inline int
     902             : arch_atomic_xchg_acquire(atomic_t *v, int i)
     903             : {
     904             :         int ret = arch_atomic_xchg_relaxed(v, i);
     905             :         __atomic_acquire_fence();
     906             :         return ret;
     907             : }
     908             : #define arch_atomic_xchg_acquire arch_atomic_xchg_acquire
     909             : #endif
     910             : 
     911             : #ifndef arch_atomic_xchg_release
     912             : static __always_inline int
     913             : arch_atomic_xchg_release(atomic_t *v, int i)
     914             : {
     915             :         __atomic_release_fence();
     916             :         return arch_atomic_xchg_relaxed(v, i);
     917             : }
     918             : #define arch_atomic_xchg_release arch_atomic_xchg_release
     919             : #endif
     920             : 
     921             : #ifndef arch_atomic_xchg
     922             : static __always_inline int
     923             : arch_atomic_xchg(atomic_t *v, int i)
     924             : {
     925             :         int ret;
     926             :         __atomic_pre_full_fence();
     927             :         ret = arch_atomic_xchg_relaxed(v, i);
     928             :         __atomic_post_full_fence();
     929             :         return ret;
     930             : }
     931             : #define arch_atomic_xchg arch_atomic_xchg
     932             : #endif
     933             : 
     934             : #endif /* arch_atomic_xchg_relaxed */
     935             : 
     936             : #ifndef arch_atomic_cmpxchg_relaxed
     937             : #define arch_atomic_cmpxchg_acquire arch_atomic_cmpxchg
     938             : #define arch_atomic_cmpxchg_release arch_atomic_cmpxchg
     939             : #define arch_atomic_cmpxchg_relaxed arch_atomic_cmpxchg
     940             : #else /* arch_atomic_cmpxchg_relaxed */
     941             : 
     942             : #ifndef arch_atomic_cmpxchg_acquire
     943             : static __always_inline int
     944             : arch_atomic_cmpxchg_acquire(atomic_t *v, int old, int new)
     945             : {
     946             :         int ret = arch_atomic_cmpxchg_relaxed(v, old, new);
     947             :         __atomic_acquire_fence();
     948             :         return ret;
     949             : }
     950             : #define arch_atomic_cmpxchg_acquire arch_atomic_cmpxchg_acquire
     951             : #endif
     952             : 
     953             : #ifndef arch_atomic_cmpxchg_release
     954             : static __always_inline int
     955             : arch_atomic_cmpxchg_release(atomic_t *v, int old, int new)
     956             : {
     957             :         __atomic_release_fence();
     958             :         return arch_atomic_cmpxchg_relaxed(v, old, new);
     959             : }
     960             : #define arch_atomic_cmpxchg_release arch_atomic_cmpxchg_release
     961             : #endif
     962             : 
     963             : #ifndef arch_atomic_cmpxchg
     964             : static __always_inline int
     965             : arch_atomic_cmpxchg(atomic_t *v, int old, int new)
     966             : {
     967             :         int ret;
     968             :         __atomic_pre_full_fence();
     969             :         ret = arch_atomic_cmpxchg_relaxed(v, old, new);
     970             :         __atomic_post_full_fence();
     971             :         return ret;
     972             : }
     973             : #define arch_atomic_cmpxchg arch_atomic_cmpxchg
     974             : #endif
     975             : 
     976             : #endif /* arch_atomic_cmpxchg_relaxed */
     977             : 
     978             : #ifndef arch_atomic_try_cmpxchg_relaxed
     979             : #ifdef arch_atomic_try_cmpxchg
     980             : #define arch_atomic_try_cmpxchg_acquire arch_atomic_try_cmpxchg
     981             : #define arch_atomic_try_cmpxchg_release arch_atomic_try_cmpxchg
     982             : #define arch_atomic_try_cmpxchg_relaxed arch_atomic_try_cmpxchg
     983             : #endif /* arch_atomic_try_cmpxchg */
     984             : 
     985             : #ifndef arch_atomic_try_cmpxchg
     986             : static __always_inline bool
     987             : arch_atomic_try_cmpxchg(atomic_t *v, int *old, int new)
     988             : {
     989             :         int r, o = *old;
     990             :         r = arch_atomic_cmpxchg(v, o, new);
     991             :         if (unlikely(r != o))
     992             :                 *old = r;
     993             :         return likely(r == o);
     994             : }
     995             : #define arch_atomic_try_cmpxchg arch_atomic_try_cmpxchg
     996             : #endif
     997             : 
     998             : #ifndef arch_atomic_try_cmpxchg_acquire
     999             : static __always_inline bool
    1000             : arch_atomic_try_cmpxchg_acquire(atomic_t *v, int *old, int new)
    1001             : {
    1002             :         int r, o = *old;
    1003             :         r = arch_atomic_cmpxchg_acquire(v, o, new);
    1004             :         if (unlikely(r != o))
    1005             :                 *old = r;
    1006             :         return likely(r == o);
    1007             : }
    1008             : #define arch_atomic_try_cmpxchg_acquire arch_atomic_try_cmpxchg_acquire
    1009             : #endif
    1010             : 
    1011             : #ifndef arch_atomic_try_cmpxchg_release
    1012             : static __always_inline bool
    1013             : arch_atomic_try_cmpxchg_release(atomic_t *v, int *old, int new)
    1014             : {
    1015             :         int r, o = *old;
    1016             :         r = arch_atomic_cmpxchg_release(v, o, new);
    1017             :         if (unlikely(r != o))
    1018             :                 *old = r;
    1019             :         return likely(r == o);
    1020             : }
    1021             : #define arch_atomic_try_cmpxchg_release arch_atomic_try_cmpxchg_release
    1022             : #endif
    1023             : 
    1024             : #ifndef arch_atomic_try_cmpxchg_relaxed
    1025             : static __always_inline bool
    1026             : arch_atomic_try_cmpxchg_relaxed(atomic_t *v, int *old, int new)
    1027             : {
    1028             :         int r, o = *old;
    1029             :         r = arch_atomic_cmpxchg_relaxed(v, o, new);
    1030             :         if (unlikely(r != o))
    1031             :                 *old = r;
    1032             :         return likely(r == o);
    1033             : }
    1034             : #define arch_atomic_try_cmpxchg_relaxed arch_atomic_try_cmpxchg_relaxed
    1035             : #endif
    1036             : 
    1037             : #else /* arch_atomic_try_cmpxchg_relaxed */
    1038             : 
    1039             : #ifndef arch_atomic_try_cmpxchg_acquire
    1040             : static __always_inline bool
    1041             : arch_atomic_try_cmpxchg_acquire(atomic_t *v, int *old, int new)
    1042             : {
    1043             :         bool ret = arch_atomic_try_cmpxchg_relaxed(v, old, new);
    1044             :         __atomic_acquire_fence();
    1045             :         return ret;
    1046             : }
    1047             : #define arch_atomic_try_cmpxchg_acquire arch_atomic_try_cmpxchg_acquire
    1048             : #endif
    1049             : 
    1050             : #ifndef arch_atomic_try_cmpxchg_release
    1051             : static __always_inline bool
    1052             : arch_atomic_try_cmpxchg_release(atomic_t *v, int *old, int new)
    1053             : {
    1054             :         __atomic_release_fence();
    1055             :         return arch_atomic_try_cmpxchg_relaxed(v, old, new);
    1056             : }
    1057             : #define arch_atomic_try_cmpxchg_release arch_atomic_try_cmpxchg_release
    1058             : #endif
    1059             : 
    1060             : #ifndef arch_atomic_try_cmpxchg
    1061             : static __always_inline bool
    1062             : arch_atomic_try_cmpxchg(atomic_t *v, int *old, int new)
    1063             : {
    1064             :         bool ret;
    1065             :         __atomic_pre_full_fence();
    1066             :         ret = arch_atomic_try_cmpxchg_relaxed(v, old, new);
    1067             :         __atomic_post_full_fence();
    1068             :         return ret;
    1069             : }
    1070             : #define arch_atomic_try_cmpxchg arch_atomic_try_cmpxchg
    1071             : #endif
    1072             : 
    1073             : #endif /* arch_atomic_try_cmpxchg_relaxed */
    1074             : 
    1075             : #ifndef arch_atomic_sub_and_test
    1076             : /**
    1077             :  * arch_atomic_sub_and_test - subtract value from variable and test result
    1078             :  * @i: integer value to subtract
    1079             :  * @v: pointer of type atomic_t
    1080             :  *
    1081             :  * Atomically subtracts @i from @v and returns
    1082             :  * true if the result is zero, or false for all
    1083             :  * other cases.
    1084             :  */
    1085             : static __always_inline bool
    1086             : arch_atomic_sub_and_test(int i, atomic_t *v)
    1087             : {
    1088             :         return arch_atomic_sub_return(i, v) == 0;
    1089             : }
    1090             : #define arch_atomic_sub_and_test arch_atomic_sub_and_test
    1091             : #endif
    1092             : 
    1093             : #ifndef arch_atomic_dec_and_test
    1094             : /**
    1095             :  * arch_atomic_dec_and_test - decrement and test
    1096             :  * @v: pointer of type atomic_t
    1097             :  *
    1098             :  * Atomically decrements @v by 1 and
    1099             :  * returns true if the result is 0, or false for all other
    1100             :  * cases.
    1101             :  */
    1102             : static __always_inline bool
    1103             : arch_atomic_dec_and_test(atomic_t *v)
    1104             : {
    1105             :         return arch_atomic_dec_return(v) == 0;
    1106             : }
    1107             : #define arch_atomic_dec_and_test arch_atomic_dec_and_test
    1108             : #endif
    1109             : 
    1110             : #ifndef arch_atomic_inc_and_test
    1111             : /**
    1112             :  * arch_atomic_inc_and_test - increment and test
    1113             :  * @v: pointer of type atomic_t
    1114             :  *
    1115             :  * Atomically increments @v by 1
    1116             :  * and returns true if the result is zero, or false for all
    1117             :  * other cases.
    1118             :  */
    1119             : static __always_inline bool
    1120             : arch_atomic_inc_and_test(atomic_t *v)
    1121             : {
    1122             :         return arch_atomic_inc_return(v) == 0;
    1123             : }
    1124             : #define arch_atomic_inc_and_test arch_atomic_inc_and_test
    1125             : #endif
    1126             : 
    1127             : #ifndef arch_atomic_add_negative
    1128             : /**
    1129             :  * arch_atomic_add_negative - add and test if negative
    1130             :  * @i: integer value to add
    1131             :  * @v: pointer of type atomic_t
    1132             :  *
    1133             :  * Atomically adds @i to @v and returns true
    1134             :  * if the result is negative, or false when
    1135             :  * result is greater than or equal to zero.
    1136             :  */
    1137             : static __always_inline bool
    1138             : arch_atomic_add_negative(int i, atomic_t *v)
    1139             : {
    1140             :         return arch_atomic_add_return(i, v) < 0;
    1141             : }
    1142             : #define arch_atomic_add_negative arch_atomic_add_negative
    1143             : #endif
    1144             : 
    1145             : #ifndef arch_atomic_fetch_add_unless
    1146             : /**
    1147             :  * arch_atomic_fetch_add_unless - add unless the number is already a given value
    1148             :  * @v: pointer of type atomic_t
    1149             :  * @a: the amount to add to v...
    1150             :  * @u: ...unless v is equal to u.
    1151             :  *
    1152             :  * Atomically adds @a to @v, so long as @v was not already @u.
    1153             :  * Returns original value of @v
    1154             :  */
    1155             : static __always_inline int
    1156       37524 : arch_atomic_fetch_add_unless(atomic_t *v, int a, int u)
    1157             : {
    1158       37524 :         int c = arch_atomic_read(v);
    1159             : 
    1160       37524 :         do {
    1161       37524 :                 if (unlikely(c == u))
    1162             :                         break;
    1163       58664 :         } while (!arch_atomic_try_cmpxchg(v, &c, c + a));
    1164             : 
    1165       37524 :         return c;
    1166             : }
    1167             : #define arch_atomic_fetch_add_unless arch_atomic_fetch_add_unless
    1168             : #endif
    1169             : 
    1170             : #ifndef arch_atomic_add_unless
    1171             : /**
    1172             :  * arch_atomic_add_unless - add unless the number is already a given value
    1173             :  * @v: pointer of type atomic_t
    1174             :  * @a: the amount to add to v...
    1175             :  * @u: ...unless v is equal to u.
    1176             :  *
    1177             :  * Atomically adds @a to @v, if @v was not already @u.
    1178             :  * Returns true if the addition was done.
    1179             :  */
    1180             : static __always_inline bool
    1181       37524 : arch_atomic_add_unless(atomic_t *v, int a, int u)
    1182             : {
    1183       75040 :         return arch_atomic_fetch_add_unless(v, a, u) != u;
    1184             : }
    1185             : #define arch_atomic_add_unless arch_atomic_add_unless
    1186             : #endif
    1187             : 
    1188             : #ifndef arch_atomic_inc_not_zero
    1189             : /**
    1190             :  * arch_atomic_inc_not_zero - increment unless the number is zero
    1191             :  * @v: pointer of type atomic_t
    1192             :  *
    1193             :  * Atomically increments @v by 1, if @v is non-zero.
    1194             :  * Returns true if the increment was done.
    1195             :  */
    1196             : static __always_inline bool
    1197       24879 : arch_atomic_inc_not_zero(atomic_t *v)
    1198             : {
    1199       25221 :         return arch_atomic_add_unless(v, 1, 0);
    1200             : }
    1201             : #define arch_atomic_inc_not_zero arch_atomic_inc_not_zero
    1202             : #endif
    1203             : 
    1204             : #ifndef arch_atomic_inc_unless_negative
    1205             : static __always_inline bool
    1206        5640 : arch_atomic_inc_unless_negative(atomic_t *v)
    1207             : {
    1208        5640 :         int c = arch_atomic_read(v);
    1209             : 
    1210        5640 :         do {
    1211        5640 :                 if (unlikely(c < 0))
    1212             :                         return false;
    1213       11280 :         } while (!arch_atomic_try_cmpxchg(v, &c, c + 1));
    1214             : 
    1215             :         return true;
    1216             : }
    1217             : #define arch_atomic_inc_unless_negative arch_atomic_inc_unless_negative
    1218             : #endif
    1219             : 
    1220             : #ifndef arch_atomic_dec_unless_positive
    1221             : static __always_inline bool
    1222        9576 : arch_atomic_dec_unless_positive(atomic_t *v)
    1223             : {
    1224        9576 :         int c = arch_atomic_read(v);
    1225             : 
    1226        9576 :         do {
    1227        9576 :                 if (unlikely(c > 0))
    1228             :                         return false;
    1229       19152 :         } while (!arch_atomic_try_cmpxchg(v, &c, c - 1));
    1230             : 
    1231             :         return true;
    1232             : }
    1233             : #define arch_atomic_dec_unless_positive arch_atomic_dec_unless_positive
    1234             : #endif
    1235             : 
    1236             : #ifndef arch_atomic_dec_if_positive
    1237             : static __always_inline int
    1238          86 : arch_atomic_dec_if_positive(atomic_t *v)
    1239             : {
    1240          86 :         int dec, c = arch_atomic_read(v);
    1241             : 
    1242          86 :         do {
    1243          86 :                 dec = c - 1;
    1244          86 :                 if (unlikely(dec < 0))
    1245             :                         break;
    1246         172 :         } while (!arch_atomic_try_cmpxchg(v, &c, dec));
    1247             : 
    1248          86 :         return dec;
    1249             : }
    1250             : #define arch_atomic_dec_if_positive arch_atomic_dec_if_positive
    1251             : #endif
    1252             : 
    1253             : #ifdef CONFIG_GENERIC_ATOMIC64
    1254             : #include <asm-generic/atomic64.h>
    1255             : #endif
    1256             : 
    1257             : #ifndef arch_atomic64_read_acquire
    1258             : static __always_inline s64
    1259             : arch_atomic64_read_acquire(const atomic64_t *v)
    1260             : {
    1261             :         return smp_load_acquire(&(v)->counter);
    1262             : }
    1263             : #define arch_atomic64_read_acquire arch_atomic64_read_acquire
    1264             : #endif
    1265             : 
    1266             : #ifndef arch_atomic64_set_release
    1267             : static __always_inline void
    1268             : arch_atomic64_set_release(atomic64_t *v, s64 i)
    1269             : {
    1270             :         smp_store_release(&(v)->counter, i);
    1271             : }
    1272             : #define arch_atomic64_set_release arch_atomic64_set_release
    1273             : #endif
    1274             : 
    1275             : #ifndef arch_atomic64_add_return_relaxed
    1276             : #define arch_atomic64_add_return_acquire arch_atomic64_add_return
    1277             : #define arch_atomic64_add_return_release arch_atomic64_add_return
    1278             : #define arch_atomic64_add_return_relaxed arch_atomic64_add_return
    1279             : #else /* arch_atomic64_add_return_relaxed */
    1280             : 
    1281             : #ifndef arch_atomic64_add_return_acquire
    1282             : static __always_inline s64
    1283             : arch_atomic64_add_return_acquire(s64 i, atomic64_t *v)
    1284             : {
    1285             :         s64 ret = arch_atomic64_add_return_relaxed(i, v);
    1286             :         __atomic_acquire_fence();
    1287             :         return ret;
    1288             : }
    1289             : #define arch_atomic64_add_return_acquire arch_atomic64_add_return_acquire
    1290             : #endif
    1291             : 
    1292             : #ifndef arch_atomic64_add_return_release
    1293             : static __always_inline s64
    1294             : arch_atomic64_add_return_release(s64 i, atomic64_t *v)
    1295             : {
    1296             :         __atomic_release_fence();
    1297             :         return arch_atomic64_add_return_relaxed(i, v);
    1298             : }
    1299             : #define arch_atomic64_add_return_release arch_atomic64_add_return_release
    1300             : #endif
    1301             : 
    1302             : #ifndef arch_atomic64_add_return
    1303             : static __always_inline s64
    1304             : arch_atomic64_add_return(s64 i, atomic64_t *v)
    1305             : {
    1306             :         s64 ret;
    1307             :         __atomic_pre_full_fence();
    1308             :         ret = arch_atomic64_add_return_relaxed(i, v);
    1309             :         __atomic_post_full_fence();
    1310             :         return ret;
    1311             : }
    1312             : #define arch_atomic64_add_return arch_atomic64_add_return
    1313             : #endif
    1314             : 
    1315             : #endif /* arch_atomic64_add_return_relaxed */
    1316             : 
    1317             : #ifndef arch_atomic64_fetch_add_relaxed
    1318             : #define arch_atomic64_fetch_add_acquire arch_atomic64_fetch_add
    1319             : #define arch_atomic64_fetch_add_release arch_atomic64_fetch_add
    1320             : #define arch_atomic64_fetch_add_relaxed arch_atomic64_fetch_add
    1321             : #else /* arch_atomic64_fetch_add_relaxed */
    1322             : 
    1323             : #ifndef arch_atomic64_fetch_add_acquire
    1324             : static __always_inline s64
    1325             : arch_atomic64_fetch_add_acquire(s64 i, atomic64_t *v)
    1326             : {
    1327             :         s64 ret = arch_atomic64_fetch_add_relaxed(i, v);
    1328             :         __atomic_acquire_fence();
    1329             :         return ret;
    1330             : }
    1331             : #define arch_atomic64_fetch_add_acquire arch_atomic64_fetch_add_acquire
    1332             : #endif
    1333             : 
    1334             : #ifndef arch_atomic64_fetch_add_release
    1335             : static __always_inline s64
    1336             : arch_atomic64_fetch_add_release(s64 i, atomic64_t *v)
    1337             : {
    1338             :         __atomic_release_fence();
    1339             :         return arch_atomic64_fetch_add_relaxed(i, v);
    1340             : }
    1341             : #define arch_atomic64_fetch_add_release arch_atomic64_fetch_add_release
    1342             : #endif
    1343             : 
    1344             : #ifndef arch_atomic64_fetch_add
    1345             : static __always_inline s64
    1346             : arch_atomic64_fetch_add(s64 i, atomic64_t *v)
    1347             : {
    1348             :         s64 ret;
    1349             :         __atomic_pre_full_fence();
    1350             :         ret = arch_atomic64_fetch_add_relaxed(i, v);
    1351             :         __atomic_post_full_fence();
    1352             :         return ret;
    1353             : }
    1354             : #define arch_atomic64_fetch_add arch_atomic64_fetch_add
    1355             : #endif
    1356             : 
    1357             : #endif /* arch_atomic64_fetch_add_relaxed */
    1358             : 
    1359             : #ifndef arch_atomic64_sub_return_relaxed
    1360             : #define arch_atomic64_sub_return_acquire arch_atomic64_sub_return
    1361             : #define arch_atomic64_sub_return_release arch_atomic64_sub_return
    1362             : #define arch_atomic64_sub_return_relaxed arch_atomic64_sub_return
    1363             : #else /* arch_atomic64_sub_return_relaxed */
    1364             : 
    1365             : #ifndef arch_atomic64_sub_return_acquire
    1366             : static __always_inline s64
    1367             : arch_atomic64_sub_return_acquire(s64 i, atomic64_t *v)
    1368             : {
    1369             :         s64 ret = arch_atomic64_sub_return_relaxed(i, v);
    1370             :         __atomic_acquire_fence();
    1371             :         return ret;
    1372             : }
    1373             : #define arch_atomic64_sub_return_acquire arch_atomic64_sub_return_acquire
    1374             : #endif
    1375             : 
    1376             : #ifndef arch_atomic64_sub_return_release
    1377             : static __always_inline s64
    1378             : arch_atomic64_sub_return_release(s64 i, atomic64_t *v)
    1379             : {
    1380             :         __atomic_release_fence();
    1381             :         return arch_atomic64_sub_return_relaxed(i, v);
    1382             : }
    1383             : #define arch_atomic64_sub_return_release arch_atomic64_sub_return_release
    1384             : #endif
    1385             : 
    1386             : #ifndef arch_atomic64_sub_return
    1387             : static __always_inline s64
    1388             : arch_atomic64_sub_return(s64 i, atomic64_t *v)
    1389             : {
    1390             :         s64 ret;
    1391             :         __atomic_pre_full_fence();
    1392             :         ret = arch_atomic64_sub_return_relaxed(i, v);
    1393             :         __atomic_post_full_fence();
    1394             :         return ret;
    1395             : }
    1396             : #define arch_atomic64_sub_return arch_atomic64_sub_return
    1397             : #endif
    1398             : 
    1399             : #endif /* arch_atomic64_sub_return_relaxed */
    1400             : 
    1401             : #ifndef arch_atomic64_fetch_sub_relaxed
    1402             : #define arch_atomic64_fetch_sub_acquire arch_atomic64_fetch_sub
    1403             : #define arch_atomic64_fetch_sub_release arch_atomic64_fetch_sub
    1404             : #define arch_atomic64_fetch_sub_relaxed arch_atomic64_fetch_sub
    1405             : #else /* arch_atomic64_fetch_sub_relaxed */
    1406             : 
    1407             : #ifndef arch_atomic64_fetch_sub_acquire
    1408             : static __always_inline s64
    1409             : arch_atomic64_fetch_sub_acquire(s64 i, atomic64_t *v)
    1410             : {
    1411             :         s64 ret = arch_atomic64_fetch_sub_relaxed(i, v);
    1412             :         __atomic_acquire_fence();
    1413             :         return ret;
    1414             : }
    1415             : #define arch_atomic64_fetch_sub_acquire arch_atomic64_fetch_sub_acquire
    1416             : #endif
    1417             : 
    1418             : #ifndef arch_atomic64_fetch_sub_release
    1419             : static __always_inline s64
    1420             : arch_atomic64_fetch_sub_release(s64 i, atomic64_t *v)
    1421             : {
    1422             :         __atomic_release_fence();
    1423             :         return arch_atomic64_fetch_sub_relaxed(i, v);
    1424             : }
    1425             : #define arch_atomic64_fetch_sub_release arch_atomic64_fetch_sub_release
    1426             : #endif
    1427             : 
    1428             : #ifndef arch_atomic64_fetch_sub
    1429             : static __always_inline s64
    1430             : arch_atomic64_fetch_sub(s64 i, atomic64_t *v)
    1431             : {
    1432             :         s64 ret;
    1433             :         __atomic_pre_full_fence();
    1434             :         ret = arch_atomic64_fetch_sub_relaxed(i, v);
    1435             :         __atomic_post_full_fence();
    1436             :         return ret;
    1437             : }
    1438             : #define arch_atomic64_fetch_sub arch_atomic64_fetch_sub
    1439             : #endif
    1440             : 
    1441             : #endif /* arch_atomic64_fetch_sub_relaxed */
    1442             : 
    1443             : #ifndef arch_atomic64_inc
    1444             : static __always_inline void
    1445             : arch_atomic64_inc(atomic64_t *v)
    1446             : {
    1447             :         arch_atomic64_add(1, v);
    1448             : }
    1449             : #define arch_atomic64_inc arch_atomic64_inc
    1450             : #endif
    1451             : 
    1452             : #ifndef arch_atomic64_inc_return_relaxed
    1453             : #ifdef arch_atomic64_inc_return
    1454             : #define arch_atomic64_inc_return_acquire arch_atomic64_inc_return
    1455             : #define arch_atomic64_inc_return_release arch_atomic64_inc_return
    1456             : #define arch_atomic64_inc_return_relaxed arch_atomic64_inc_return
    1457             : #endif /* arch_atomic64_inc_return */
    1458             : 
    1459             : #ifndef arch_atomic64_inc_return
    1460             : static __always_inline s64
    1461       85910 : arch_atomic64_inc_return(atomic64_t *v)
    1462             : {
    1463       85910 :         return arch_atomic64_add_return(1, v);
    1464             : }
    1465             : #define arch_atomic64_inc_return arch_atomic64_inc_return
    1466             : #endif
    1467             : 
    1468             : #ifndef arch_atomic64_inc_return_acquire
    1469             : static __always_inline s64
    1470             : arch_atomic64_inc_return_acquire(atomic64_t *v)
    1471             : {
    1472             :         return arch_atomic64_add_return_acquire(1, v);
    1473             : }
    1474             : #define arch_atomic64_inc_return_acquire arch_atomic64_inc_return_acquire
    1475             : #endif
    1476             : 
    1477             : #ifndef arch_atomic64_inc_return_release
    1478             : static __always_inline s64
    1479             : arch_atomic64_inc_return_release(atomic64_t *v)
    1480             : {
    1481             :         return arch_atomic64_add_return_release(1, v);
    1482             : }
    1483             : #define arch_atomic64_inc_return_release arch_atomic64_inc_return_release
    1484             : #endif
    1485             : 
    1486             : #ifndef arch_atomic64_inc_return_relaxed
    1487             : static __always_inline s64
    1488             : arch_atomic64_inc_return_relaxed(atomic64_t *v)
    1489             : {
    1490             :         return arch_atomic64_add_return_relaxed(1, v);
    1491             : }
    1492             : #define arch_atomic64_inc_return_relaxed arch_atomic64_inc_return_relaxed
    1493             : #endif
    1494             : 
    1495             : #else /* arch_atomic64_inc_return_relaxed */
    1496             : 
    1497             : #ifndef arch_atomic64_inc_return_acquire
    1498             : static __always_inline s64
    1499             : arch_atomic64_inc_return_acquire(atomic64_t *v)
    1500             : {
    1501             :         s64 ret = arch_atomic64_inc_return_relaxed(v);
    1502             :         __atomic_acquire_fence();
    1503             :         return ret;
    1504             : }
    1505             : #define arch_atomic64_inc_return_acquire arch_atomic64_inc_return_acquire
    1506             : #endif
    1507             : 
    1508             : #ifndef arch_atomic64_inc_return_release
    1509             : static __always_inline s64
    1510             : arch_atomic64_inc_return_release(atomic64_t *v)
    1511             : {
    1512             :         __atomic_release_fence();
    1513             :         return arch_atomic64_inc_return_relaxed(v);
    1514             : }
    1515             : #define arch_atomic64_inc_return_release arch_atomic64_inc_return_release
    1516             : #endif
    1517             : 
    1518             : #ifndef arch_atomic64_inc_return
    1519             : static __always_inline s64
    1520             : arch_atomic64_inc_return(atomic64_t *v)
    1521             : {
    1522             :         s64 ret;
    1523             :         __atomic_pre_full_fence();
    1524             :         ret = arch_atomic64_inc_return_relaxed(v);
    1525             :         __atomic_post_full_fence();
    1526             :         return ret;
    1527             : }
    1528             : #define arch_atomic64_inc_return arch_atomic64_inc_return
    1529             : #endif
    1530             : 
    1531             : #endif /* arch_atomic64_inc_return_relaxed */
    1532             : 
    1533             : #ifndef arch_atomic64_fetch_inc_relaxed
    1534             : #ifdef arch_atomic64_fetch_inc
    1535             : #define arch_atomic64_fetch_inc_acquire arch_atomic64_fetch_inc
    1536             : #define arch_atomic64_fetch_inc_release arch_atomic64_fetch_inc
    1537             : #define arch_atomic64_fetch_inc_relaxed arch_atomic64_fetch_inc
    1538             : #endif /* arch_atomic64_fetch_inc */
    1539             : 
    1540             : #ifndef arch_atomic64_fetch_inc
    1541             : static __always_inline s64
    1542             : arch_atomic64_fetch_inc(atomic64_t *v)
    1543             : {
    1544             :         return arch_atomic64_fetch_add(1, v);
    1545             : }
    1546             : #define arch_atomic64_fetch_inc arch_atomic64_fetch_inc
    1547             : #endif
    1548             : 
    1549             : #ifndef arch_atomic64_fetch_inc_acquire
    1550             : static __always_inline s64
    1551             : arch_atomic64_fetch_inc_acquire(atomic64_t *v)
    1552             : {
    1553             :         return arch_atomic64_fetch_add_acquire(1, v);
    1554             : }
    1555             : #define arch_atomic64_fetch_inc_acquire arch_atomic64_fetch_inc_acquire
    1556             : #endif
    1557             : 
    1558             : #ifndef arch_atomic64_fetch_inc_release
    1559             : static __always_inline s64
    1560             : arch_atomic64_fetch_inc_release(atomic64_t *v)
    1561             : {
    1562             :         return arch_atomic64_fetch_add_release(1, v);
    1563             : }
    1564             : #define arch_atomic64_fetch_inc_release arch_atomic64_fetch_inc_release
    1565             : #endif
    1566             : 
    1567             : #ifndef arch_atomic64_fetch_inc_relaxed
    1568             : static __always_inline s64
    1569             : arch_atomic64_fetch_inc_relaxed(atomic64_t *v)
    1570             : {
    1571             :         return arch_atomic64_fetch_add_relaxed(1, v);
    1572             : }
    1573             : #define arch_atomic64_fetch_inc_relaxed arch_atomic64_fetch_inc_relaxed
    1574             : #endif
    1575             : 
    1576             : #else /* arch_atomic64_fetch_inc_relaxed */
    1577             : 
    1578             : #ifndef arch_atomic64_fetch_inc_acquire
    1579             : static __always_inline s64
    1580             : arch_atomic64_fetch_inc_acquire(atomic64_t *v)
    1581             : {
    1582             :         s64 ret = arch_atomic64_fetch_inc_relaxed(v);
    1583             :         __atomic_acquire_fence();
    1584             :         return ret;
    1585             : }
    1586             : #define arch_atomic64_fetch_inc_acquire arch_atomic64_fetch_inc_acquire
    1587             : #endif
    1588             : 
    1589             : #ifndef arch_atomic64_fetch_inc_release
    1590             : static __always_inline s64
    1591             : arch_atomic64_fetch_inc_release(atomic64_t *v)
    1592             : {
    1593             :         __atomic_release_fence();
    1594             :         return arch_atomic64_fetch_inc_relaxed(v);
    1595             : }
    1596             : #define arch_atomic64_fetch_inc_release arch_atomic64_fetch_inc_release
    1597             : #endif
    1598             : 
    1599             : #ifndef arch_atomic64_fetch_inc
    1600             : static __always_inline s64
    1601             : arch_atomic64_fetch_inc(atomic64_t *v)
    1602             : {
    1603             :         s64 ret;
    1604             :         __atomic_pre_full_fence();
    1605             :         ret = arch_atomic64_fetch_inc_relaxed(v);
    1606             :         __atomic_post_full_fence();
    1607             :         return ret;
    1608             : }
    1609             : #define arch_atomic64_fetch_inc arch_atomic64_fetch_inc
    1610             : #endif
    1611             : 
    1612             : #endif /* arch_atomic64_fetch_inc_relaxed */
    1613             : 
    1614             : #ifndef arch_atomic64_dec
    1615             : static __always_inline void
    1616             : arch_atomic64_dec(atomic64_t *v)
    1617             : {
    1618             :         arch_atomic64_sub(1, v);
    1619             : }
    1620             : #define arch_atomic64_dec arch_atomic64_dec
    1621             : #endif
    1622             : 
    1623             : #ifndef arch_atomic64_dec_return_relaxed
    1624             : #ifdef arch_atomic64_dec_return
    1625             : #define arch_atomic64_dec_return_acquire arch_atomic64_dec_return
    1626             : #define arch_atomic64_dec_return_release arch_atomic64_dec_return
    1627             : #define arch_atomic64_dec_return_relaxed arch_atomic64_dec_return
    1628             : #endif /* arch_atomic64_dec_return */
    1629             : 
    1630             : #ifndef arch_atomic64_dec_return
    1631             : static __always_inline s64
    1632           0 : arch_atomic64_dec_return(atomic64_t *v)
    1633             : {
    1634           0 :         return arch_atomic64_sub_return(1, v);
    1635             : }
    1636             : #define arch_atomic64_dec_return arch_atomic64_dec_return
    1637             : #endif
    1638             : 
    1639             : #ifndef arch_atomic64_dec_return_acquire
    1640             : static __always_inline s64
    1641             : arch_atomic64_dec_return_acquire(atomic64_t *v)
    1642             : {
    1643             :         return arch_atomic64_sub_return_acquire(1, v);
    1644             : }
    1645             : #define arch_atomic64_dec_return_acquire arch_atomic64_dec_return_acquire
    1646             : #endif
    1647             : 
    1648             : #ifndef arch_atomic64_dec_return_release
    1649             : static __always_inline s64
    1650             : arch_atomic64_dec_return_release(atomic64_t *v)
    1651             : {
    1652             :         return arch_atomic64_sub_return_release(1, v);
    1653             : }
    1654             : #define arch_atomic64_dec_return_release arch_atomic64_dec_return_release
    1655             : #endif
    1656             : 
    1657             : #ifndef arch_atomic64_dec_return_relaxed
    1658             : static __always_inline s64
    1659             : arch_atomic64_dec_return_relaxed(atomic64_t *v)
    1660             : {
    1661             :         return arch_atomic64_sub_return_relaxed(1, v);
    1662             : }
    1663             : #define arch_atomic64_dec_return_relaxed arch_atomic64_dec_return_relaxed
    1664             : #endif
    1665             : 
    1666             : #else /* arch_atomic64_dec_return_relaxed */
    1667             : 
    1668             : #ifndef arch_atomic64_dec_return_acquire
    1669             : static __always_inline s64
    1670             : arch_atomic64_dec_return_acquire(atomic64_t *v)
    1671             : {
    1672             :         s64 ret = arch_atomic64_dec_return_relaxed(v);
    1673             :         __atomic_acquire_fence();
    1674             :         return ret;
    1675             : }
    1676             : #define arch_atomic64_dec_return_acquire arch_atomic64_dec_return_acquire
    1677             : #endif
    1678             : 
    1679             : #ifndef arch_atomic64_dec_return_release
    1680             : static __always_inline s64
    1681             : arch_atomic64_dec_return_release(atomic64_t *v)
    1682             : {
    1683             :         __atomic_release_fence();
    1684             :         return arch_atomic64_dec_return_relaxed(v);
    1685             : }
    1686             : #define arch_atomic64_dec_return_release arch_atomic64_dec_return_release
    1687             : #endif
    1688             : 
    1689             : #ifndef arch_atomic64_dec_return
    1690             : static __always_inline s64
    1691             : arch_atomic64_dec_return(atomic64_t *v)
    1692             : {
    1693             :         s64 ret;
    1694             :         __atomic_pre_full_fence();
    1695             :         ret = arch_atomic64_dec_return_relaxed(v);
    1696             :         __atomic_post_full_fence();
    1697             :         return ret;
    1698             : }
    1699             : #define arch_atomic64_dec_return arch_atomic64_dec_return
    1700             : #endif
    1701             : 
    1702             : #endif /* arch_atomic64_dec_return_relaxed */
    1703             : 
    1704             : #ifndef arch_atomic64_fetch_dec_relaxed
    1705             : #ifdef arch_atomic64_fetch_dec
    1706             : #define arch_atomic64_fetch_dec_acquire arch_atomic64_fetch_dec
    1707             : #define arch_atomic64_fetch_dec_release arch_atomic64_fetch_dec
    1708             : #define arch_atomic64_fetch_dec_relaxed arch_atomic64_fetch_dec
    1709             : #endif /* arch_atomic64_fetch_dec */
    1710             : 
    1711             : #ifndef arch_atomic64_fetch_dec
    1712             : static __always_inline s64
    1713             : arch_atomic64_fetch_dec(atomic64_t *v)
    1714             : {
    1715             :         return arch_atomic64_fetch_sub(1, v);
    1716             : }
    1717             : #define arch_atomic64_fetch_dec arch_atomic64_fetch_dec
    1718             : #endif
    1719             : 
    1720             : #ifndef arch_atomic64_fetch_dec_acquire
    1721             : static __always_inline s64
    1722             : arch_atomic64_fetch_dec_acquire(atomic64_t *v)
    1723             : {
    1724             :         return arch_atomic64_fetch_sub_acquire(1, v);
    1725             : }
    1726             : #define arch_atomic64_fetch_dec_acquire arch_atomic64_fetch_dec_acquire
    1727             : #endif
    1728             : 
    1729             : #ifndef arch_atomic64_fetch_dec_release
    1730             : static __always_inline s64
    1731             : arch_atomic64_fetch_dec_release(atomic64_t *v)
    1732             : {
    1733             :         return arch_atomic64_fetch_sub_release(1, v);
    1734             : }
    1735             : #define arch_atomic64_fetch_dec_release arch_atomic64_fetch_dec_release
    1736             : #endif
    1737             : 
    1738             : #ifndef arch_atomic64_fetch_dec_relaxed
    1739             : static __always_inline s64
    1740             : arch_atomic64_fetch_dec_relaxed(atomic64_t *v)
    1741             : {
    1742             :         return arch_atomic64_fetch_sub_relaxed(1, v);
    1743             : }
    1744             : #define arch_atomic64_fetch_dec_relaxed arch_atomic64_fetch_dec_relaxed
    1745             : #endif
    1746             : 
    1747             : #else /* arch_atomic64_fetch_dec_relaxed */
    1748             : 
    1749             : #ifndef arch_atomic64_fetch_dec_acquire
    1750             : static __always_inline s64
    1751             : arch_atomic64_fetch_dec_acquire(atomic64_t *v)
    1752             : {
    1753             :         s64 ret = arch_atomic64_fetch_dec_relaxed(v);
    1754             :         __atomic_acquire_fence();
    1755             :         return ret;
    1756             : }
    1757             : #define arch_atomic64_fetch_dec_acquire arch_atomic64_fetch_dec_acquire
    1758             : #endif
    1759             : 
    1760             : #ifndef arch_atomic64_fetch_dec_release
    1761             : static __always_inline s64
    1762             : arch_atomic64_fetch_dec_release(atomic64_t *v)
    1763             : {
    1764             :         __atomic_release_fence();
    1765             :         return arch_atomic64_fetch_dec_relaxed(v);
    1766             : }
    1767             : #define arch_atomic64_fetch_dec_release arch_atomic64_fetch_dec_release
    1768             : #endif
    1769             : 
    1770             : #ifndef arch_atomic64_fetch_dec
    1771             : static __always_inline s64
    1772             : arch_atomic64_fetch_dec(atomic64_t *v)
    1773             : {
    1774             :         s64 ret;
    1775             :         __atomic_pre_full_fence();
    1776             :         ret = arch_atomic64_fetch_dec_relaxed(v);
    1777             :         __atomic_post_full_fence();
    1778             :         return ret;
    1779             : }
    1780             : #define arch_atomic64_fetch_dec arch_atomic64_fetch_dec
    1781             : #endif
    1782             : 
    1783             : #endif /* arch_atomic64_fetch_dec_relaxed */
    1784             : 
    1785             : #ifndef arch_atomic64_fetch_and_relaxed
    1786             : #define arch_atomic64_fetch_and_acquire arch_atomic64_fetch_and
    1787             : #define arch_atomic64_fetch_and_release arch_atomic64_fetch_and
    1788             : #define arch_atomic64_fetch_and_relaxed arch_atomic64_fetch_and
    1789             : #else /* arch_atomic64_fetch_and_relaxed */
    1790             : 
    1791             : #ifndef arch_atomic64_fetch_and_acquire
    1792             : static __always_inline s64
    1793             : arch_atomic64_fetch_and_acquire(s64 i, atomic64_t *v)
    1794             : {
    1795             :         s64 ret = arch_atomic64_fetch_and_relaxed(i, v);
    1796             :         __atomic_acquire_fence();
    1797             :         return ret;
    1798             : }
    1799             : #define arch_atomic64_fetch_and_acquire arch_atomic64_fetch_and_acquire
    1800             : #endif
    1801             : 
    1802             : #ifndef arch_atomic64_fetch_and_release
    1803             : static __always_inline s64
    1804             : arch_atomic64_fetch_and_release(s64 i, atomic64_t *v)
    1805             : {
    1806             :         __atomic_release_fence();
    1807             :         return arch_atomic64_fetch_and_relaxed(i, v);
    1808             : }
    1809             : #define arch_atomic64_fetch_and_release arch_atomic64_fetch_and_release
    1810             : #endif
    1811             : 
    1812             : #ifndef arch_atomic64_fetch_and
    1813             : static __always_inline s64
    1814             : arch_atomic64_fetch_and(s64 i, atomic64_t *v)
    1815             : {
    1816             :         s64 ret;
    1817             :         __atomic_pre_full_fence();
    1818             :         ret = arch_atomic64_fetch_and_relaxed(i, v);
    1819             :         __atomic_post_full_fence();
    1820             :         return ret;
    1821             : }
    1822             : #define arch_atomic64_fetch_and arch_atomic64_fetch_and
    1823             : #endif
    1824             : 
    1825             : #endif /* arch_atomic64_fetch_and_relaxed */
    1826             : 
    1827             : #ifndef arch_atomic64_andnot
    1828             : static __always_inline void
    1829         112 : arch_atomic64_andnot(s64 i, atomic64_t *v)
    1830             : {
    1831         112 :         arch_atomic64_and(~i, v);
    1832             : }
    1833             : #define arch_atomic64_andnot arch_atomic64_andnot
    1834             : #endif
    1835             : 
    1836             : #ifndef arch_atomic64_fetch_andnot_relaxed
    1837             : #ifdef arch_atomic64_fetch_andnot
    1838             : #define arch_atomic64_fetch_andnot_acquire arch_atomic64_fetch_andnot
    1839             : #define arch_atomic64_fetch_andnot_release arch_atomic64_fetch_andnot
    1840             : #define arch_atomic64_fetch_andnot_relaxed arch_atomic64_fetch_andnot
    1841             : #endif /* arch_atomic64_fetch_andnot */
    1842             : 
    1843             : #ifndef arch_atomic64_fetch_andnot
    1844             : static __always_inline s64
    1845             : arch_atomic64_fetch_andnot(s64 i, atomic64_t *v)
    1846             : {
    1847             :         return arch_atomic64_fetch_and(~i, v);
    1848             : }
    1849             : #define arch_atomic64_fetch_andnot arch_atomic64_fetch_andnot
    1850             : #endif
    1851             : 
    1852             : #ifndef arch_atomic64_fetch_andnot_acquire
    1853             : static __always_inline s64
    1854             : arch_atomic64_fetch_andnot_acquire(s64 i, atomic64_t *v)
    1855             : {
    1856             :         return arch_atomic64_fetch_and_acquire(~i, v);
    1857             : }
    1858             : #define arch_atomic64_fetch_andnot_acquire arch_atomic64_fetch_andnot_acquire
    1859             : #endif
    1860             : 
    1861             : #ifndef arch_atomic64_fetch_andnot_release
    1862             : static __always_inline s64
    1863             : arch_atomic64_fetch_andnot_release(s64 i, atomic64_t *v)
    1864             : {
    1865             :         return arch_atomic64_fetch_and_release(~i, v);
    1866             : }
    1867             : #define arch_atomic64_fetch_andnot_release arch_atomic64_fetch_andnot_release
    1868             : #endif
    1869             : 
    1870             : #ifndef arch_atomic64_fetch_andnot_relaxed
    1871             : static __always_inline s64
    1872             : arch_atomic64_fetch_andnot_relaxed(s64 i, atomic64_t *v)
    1873             : {
    1874             :         return arch_atomic64_fetch_and_relaxed(~i, v);
    1875             : }
    1876             : #define arch_atomic64_fetch_andnot_relaxed arch_atomic64_fetch_andnot_relaxed
    1877             : #endif
    1878             : 
    1879             : #else /* arch_atomic64_fetch_andnot_relaxed */
    1880             : 
    1881             : #ifndef arch_atomic64_fetch_andnot_acquire
    1882             : static __always_inline s64
    1883             : arch_atomic64_fetch_andnot_acquire(s64 i, atomic64_t *v)
    1884             : {
    1885             :         s64 ret = arch_atomic64_fetch_andnot_relaxed(i, v);
    1886             :         __atomic_acquire_fence();
    1887             :         return ret;
    1888             : }
    1889             : #define arch_atomic64_fetch_andnot_acquire arch_atomic64_fetch_andnot_acquire
    1890             : #endif
    1891             : 
    1892             : #ifndef arch_atomic64_fetch_andnot_release
    1893             : static __always_inline s64
    1894             : arch_atomic64_fetch_andnot_release(s64 i, atomic64_t *v)
    1895             : {
    1896             :         __atomic_release_fence();
    1897             :         return arch_atomic64_fetch_andnot_relaxed(i, v);
    1898             : }
    1899             : #define arch_atomic64_fetch_andnot_release arch_atomic64_fetch_andnot_release
    1900             : #endif
    1901             : 
    1902             : #ifndef arch_atomic64_fetch_andnot
    1903             : static __always_inline s64
    1904             : arch_atomic64_fetch_andnot(s64 i, atomic64_t *v)
    1905             : {
    1906             :         s64 ret;
    1907             :         __atomic_pre_full_fence();
    1908             :         ret = arch_atomic64_fetch_andnot_relaxed(i, v);
    1909             :         __atomic_post_full_fence();
    1910             :         return ret;
    1911             : }
    1912             : #define arch_atomic64_fetch_andnot arch_atomic64_fetch_andnot
    1913             : #endif
    1914             : 
    1915             : #endif /* arch_atomic64_fetch_andnot_relaxed */
    1916             : 
    1917             : #ifndef arch_atomic64_fetch_or_relaxed
    1918             : #define arch_atomic64_fetch_or_acquire arch_atomic64_fetch_or
    1919             : #define arch_atomic64_fetch_or_release arch_atomic64_fetch_or
    1920             : #define arch_atomic64_fetch_or_relaxed arch_atomic64_fetch_or
    1921             : #else /* arch_atomic64_fetch_or_relaxed */
    1922             : 
    1923             : #ifndef arch_atomic64_fetch_or_acquire
    1924             : static __always_inline s64
    1925             : arch_atomic64_fetch_or_acquire(s64 i, atomic64_t *v)
    1926             : {
    1927             :         s64 ret = arch_atomic64_fetch_or_relaxed(i, v);
    1928             :         __atomic_acquire_fence();
    1929             :         return ret;
    1930             : }
    1931             : #define arch_atomic64_fetch_or_acquire arch_atomic64_fetch_or_acquire
    1932             : #endif
    1933             : 
    1934             : #ifndef arch_atomic64_fetch_or_release
    1935             : static __always_inline s64
    1936             : arch_atomic64_fetch_or_release(s64 i, atomic64_t *v)
    1937             : {
    1938             :         __atomic_release_fence();
    1939             :         return arch_atomic64_fetch_or_relaxed(i, v);
    1940             : }
    1941             : #define arch_atomic64_fetch_or_release arch_atomic64_fetch_or_release
    1942             : #endif
    1943             : 
    1944             : #ifndef arch_atomic64_fetch_or
    1945             : static __always_inline s64
    1946             : arch_atomic64_fetch_or(s64 i, atomic64_t *v)
    1947             : {
    1948             :         s64 ret;
    1949             :         __atomic_pre_full_fence();
    1950             :         ret = arch_atomic64_fetch_or_relaxed(i, v);
    1951             :         __atomic_post_full_fence();
    1952             :         return ret;
    1953             : }
    1954             : #define arch_atomic64_fetch_or arch_atomic64_fetch_or
    1955             : #endif
    1956             : 
    1957             : #endif /* arch_atomic64_fetch_or_relaxed */
    1958             : 
    1959             : #ifndef arch_atomic64_fetch_xor_relaxed
    1960             : #define arch_atomic64_fetch_xor_acquire arch_atomic64_fetch_xor
    1961             : #define arch_atomic64_fetch_xor_release arch_atomic64_fetch_xor
    1962             : #define arch_atomic64_fetch_xor_relaxed arch_atomic64_fetch_xor
    1963             : #else /* arch_atomic64_fetch_xor_relaxed */
    1964             : 
    1965             : #ifndef arch_atomic64_fetch_xor_acquire
    1966             : static __always_inline s64
    1967             : arch_atomic64_fetch_xor_acquire(s64 i, atomic64_t *v)
    1968             : {
    1969             :         s64 ret = arch_atomic64_fetch_xor_relaxed(i, v);
    1970             :         __atomic_acquire_fence();
    1971             :         return ret;
    1972             : }
    1973             : #define arch_atomic64_fetch_xor_acquire arch_atomic64_fetch_xor_acquire
    1974             : #endif
    1975             : 
    1976             : #ifndef arch_atomic64_fetch_xor_release
    1977             : static __always_inline s64
    1978             : arch_atomic64_fetch_xor_release(s64 i, atomic64_t *v)
    1979             : {
    1980             :         __atomic_release_fence();
    1981             :         return arch_atomic64_fetch_xor_relaxed(i, v);
    1982             : }
    1983             : #define arch_atomic64_fetch_xor_release arch_atomic64_fetch_xor_release
    1984             : #endif
    1985             : 
    1986             : #ifndef arch_atomic64_fetch_xor
    1987             : static __always_inline s64
    1988             : arch_atomic64_fetch_xor(s64 i, atomic64_t *v)
    1989             : {
    1990             :         s64 ret;
    1991             :         __atomic_pre_full_fence();
    1992             :         ret = arch_atomic64_fetch_xor_relaxed(i, v);
    1993             :         __atomic_post_full_fence();
    1994             :         return ret;
    1995             : }
    1996             : #define arch_atomic64_fetch_xor arch_atomic64_fetch_xor
    1997             : #endif
    1998             : 
    1999             : #endif /* arch_atomic64_fetch_xor_relaxed */
    2000             : 
    2001             : #ifndef arch_atomic64_xchg_relaxed
    2002             : #define arch_atomic64_xchg_acquire arch_atomic64_xchg
    2003             : #define arch_atomic64_xchg_release arch_atomic64_xchg
    2004             : #define arch_atomic64_xchg_relaxed arch_atomic64_xchg
    2005             : #else /* arch_atomic64_xchg_relaxed */
    2006             : 
    2007             : #ifndef arch_atomic64_xchg_acquire
    2008             : static __always_inline s64
    2009             : arch_atomic64_xchg_acquire(atomic64_t *v, s64 i)
    2010             : {
    2011             :         s64 ret = arch_atomic64_xchg_relaxed(v, i);
    2012             :         __atomic_acquire_fence();
    2013             :         return ret;
    2014             : }
    2015             : #define arch_atomic64_xchg_acquire arch_atomic64_xchg_acquire
    2016             : #endif
    2017             : 
    2018             : #ifndef arch_atomic64_xchg_release
    2019             : static __always_inline s64
    2020             : arch_atomic64_xchg_release(atomic64_t *v, s64 i)
    2021             : {
    2022             :         __atomic_release_fence();
    2023             :         return arch_atomic64_xchg_relaxed(v, i);
    2024             : }
    2025             : #define arch_atomic64_xchg_release arch_atomic64_xchg_release
    2026             : #endif
    2027             : 
    2028             : #ifndef arch_atomic64_xchg
    2029             : static __always_inline s64
    2030             : arch_atomic64_xchg(atomic64_t *v, s64 i)
    2031             : {
    2032             :         s64 ret;
    2033             :         __atomic_pre_full_fence();
    2034             :         ret = arch_atomic64_xchg_relaxed(v, i);
    2035             :         __atomic_post_full_fence();
    2036             :         return ret;
    2037             : }
    2038             : #define arch_atomic64_xchg arch_atomic64_xchg
    2039             : #endif
    2040             : 
    2041             : #endif /* arch_atomic64_xchg_relaxed */
    2042             : 
    2043             : #ifndef arch_atomic64_cmpxchg_relaxed
    2044             : #define arch_atomic64_cmpxchg_acquire arch_atomic64_cmpxchg
    2045             : #define arch_atomic64_cmpxchg_release arch_atomic64_cmpxchg
    2046             : #define arch_atomic64_cmpxchg_relaxed arch_atomic64_cmpxchg
    2047             : #else /* arch_atomic64_cmpxchg_relaxed */
    2048             : 
    2049             : #ifndef arch_atomic64_cmpxchg_acquire
    2050             : static __always_inline s64
    2051             : arch_atomic64_cmpxchg_acquire(atomic64_t *v, s64 old, s64 new)
    2052             : {
    2053             :         s64 ret = arch_atomic64_cmpxchg_relaxed(v, old, new);
    2054             :         __atomic_acquire_fence();
    2055             :         return ret;
    2056             : }
    2057             : #define arch_atomic64_cmpxchg_acquire arch_atomic64_cmpxchg_acquire
    2058             : #endif
    2059             : 
    2060             : #ifndef arch_atomic64_cmpxchg_release
    2061             : static __always_inline s64
    2062             : arch_atomic64_cmpxchg_release(atomic64_t *v, s64 old, s64 new)
    2063             : {
    2064             :         __atomic_release_fence();
    2065             :         return arch_atomic64_cmpxchg_relaxed(v, old, new);
    2066             : }
    2067             : #define arch_atomic64_cmpxchg_release arch_atomic64_cmpxchg_release
    2068             : #endif
    2069             : 
    2070             : #ifndef arch_atomic64_cmpxchg
    2071             : static __always_inline s64
    2072             : arch_atomic64_cmpxchg(atomic64_t *v, s64 old, s64 new)
    2073             : {
    2074             :         s64 ret;
    2075             :         __atomic_pre_full_fence();
    2076             :         ret = arch_atomic64_cmpxchg_relaxed(v, old, new);
    2077             :         __atomic_post_full_fence();
    2078             :         return ret;
    2079             : }
    2080             : #define arch_atomic64_cmpxchg arch_atomic64_cmpxchg
    2081             : #endif
    2082             : 
    2083             : #endif /* arch_atomic64_cmpxchg_relaxed */
    2084             : 
    2085             : #ifndef arch_atomic64_try_cmpxchg_relaxed
    2086             : #ifdef arch_atomic64_try_cmpxchg
    2087             : #define arch_atomic64_try_cmpxchg_acquire arch_atomic64_try_cmpxchg
    2088             : #define arch_atomic64_try_cmpxchg_release arch_atomic64_try_cmpxchg
    2089             : #define arch_atomic64_try_cmpxchg_relaxed arch_atomic64_try_cmpxchg
    2090             : #endif /* arch_atomic64_try_cmpxchg */
    2091             : 
    2092             : #ifndef arch_atomic64_try_cmpxchg
    2093             : static __always_inline bool
    2094             : arch_atomic64_try_cmpxchg(atomic64_t *v, s64 *old, s64 new)
    2095             : {
    2096             :         s64 r, o = *old;
    2097             :         r = arch_atomic64_cmpxchg(v, o, new);
    2098             :         if (unlikely(r != o))
    2099             :                 *old = r;
    2100             :         return likely(r == o);
    2101             : }
    2102             : #define arch_atomic64_try_cmpxchg arch_atomic64_try_cmpxchg
    2103             : #endif
    2104             : 
    2105             : #ifndef arch_atomic64_try_cmpxchg_acquire
    2106             : static __always_inline bool
    2107             : arch_atomic64_try_cmpxchg_acquire(atomic64_t *v, s64 *old, s64 new)
    2108             : {
    2109             :         s64 r, o = *old;
    2110             :         r = arch_atomic64_cmpxchg_acquire(v, o, new);
    2111             :         if (unlikely(r != o))
    2112             :                 *old = r;
    2113             :         return likely(r == o);
    2114             : }
    2115             : #define arch_atomic64_try_cmpxchg_acquire arch_atomic64_try_cmpxchg_acquire
    2116             : #endif
    2117             : 
    2118             : #ifndef arch_atomic64_try_cmpxchg_release
    2119             : static __always_inline bool
    2120             : arch_atomic64_try_cmpxchg_release(atomic64_t *v, s64 *old, s64 new)
    2121             : {
    2122             :         s64 r, o = *old;
    2123             :         r = arch_atomic64_cmpxchg_release(v, o, new);
    2124             :         if (unlikely(r != o))
    2125             :                 *old = r;
    2126             :         return likely(r == o);
    2127             : }
    2128             : #define arch_atomic64_try_cmpxchg_release arch_atomic64_try_cmpxchg_release
    2129             : #endif
    2130             : 
    2131             : #ifndef arch_atomic64_try_cmpxchg_relaxed
    2132             : static __always_inline bool
    2133             : arch_atomic64_try_cmpxchg_relaxed(atomic64_t *v, s64 *old, s64 new)
    2134             : {
    2135             :         s64 r, o = *old;
    2136             :         r = arch_atomic64_cmpxchg_relaxed(v, o, new);
    2137             :         if (unlikely(r != o))
    2138             :                 *old = r;
    2139             :         return likely(r == o);
    2140             : }
    2141             : #define arch_atomic64_try_cmpxchg_relaxed arch_atomic64_try_cmpxchg_relaxed
    2142             : #endif
    2143             : 
    2144             : #else /* arch_atomic64_try_cmpxchg_relaxed */
    2145             : 
    2146             : #ifndef arch_atomic64_try_cmpxchg_acquire
    2147             : static __always_inline bool
    2148             : arch_atomic64_try_cmpxchg_acquire(atomic64_t *v, s64 *old, s64 new)
    2149             : {
    2150             :         bool ret = arch_atomic64_try_cmpxchg_relaxed(v, old, new);
    2151             :         __atomic_acquire_fence();
    2152             :         return ret;
    2153             : }
    2154             : #define arch_atomic64_try_cmpxchg_acquire arch_atomic64_try_cmpxchg_acquire
    2155             : #endif
    2156             : 
    2157             : #ifndef arch_atomic64_try_cmpxchg_release
    2158             : static __always_inline bool
    2159             : arch_atomic64_try_cmpxchg_release(atomic64_t *v, s64 *old, s64 new)
    2160             : {
    2161             :         __atomic_release_fence();
    2162             :         return arch_atomic64_try_cmpxchg_relaxed(v, old, new);
    2163             : }
    2164             : #define arch_atomic64_try_cmpxchg_release arch_atomic64_try_cmpxchg_release
    2165             : #endif
    2166             : 
    2167             : #ifndef arch_atomic64_try_cmpxchg
    2168             : static __always_inline bool
    2169             : arch_atomic64_try_cmpxchg(atomic64_t *v, s64 *old, s64 new)
    2170             : {
    2171             :         bool ret;
    2172             :         __atomic_pre_full_fence();
    2173             :         ret = arch_atomic64_try_cmpxchg_relaxed(v, old, new);
    2174             :         __atomic_post_full_fence();
    2175             :         return ret;
    2176             : }
    2177             : #define arch_atomic64_try_cmpxchg arch_atomic64_try_cmpxchg
    2178             : #endif
    2179             : 
    2180             : #endif /* arch_atomic64_try_cmpxchg_relaxed */
    2181             : 
    2182             : #ifndef arch_atomic64_sub_and_test
    2183             : /**
    2184             :  * arch_atomic64_sub_and_test - subtract value from variable and test result
    2185             :  * @i: integer value to subtract
    2186             :  * @v: pointer of type atomic64_t
    2187             :  *
    2188             :  * Atomically subtracts @i from @v and returns
    2189             :  * true if the result is zero, or false for all
    2190             :  * other cases.
    2191             :  */
    2192             : static __always_inline bool
    2193             : arch_atomic64_sub_and_test(s64 i, atomic64_t *v)
    2194             : {
    2195             :         return arch_atomic64_sub_return(i, v) == 0;
    2196             : }
    2197             : #define arch_atomic64_sub_and_test arch_atomic64_sub_and_test
    2198             : #endif
    2199             : 
    2200             : #ifndef arch_atomic64_dec_and_test
    2201             : /**
    2202             :  * arch_atomic64_dec_and_test - decrement and test
    2203             :  * @v: pointer of type atomic64_t
    2204             :  *
    2205             :  * Atomically decrements @v by 1 and
    2206             :  * returns true if the result is 0, or false for all other
    2207             :  * cases.
    2208             :  */
    2209             : static __always_inline bool
    2210             : arch_atomic64_dec_and_test(atomic64_t *v)
    2211             : {
    2212             :         return arch_atomic64_dec_return(v) == 0;
    2213             : }
    2214             : #define arch_atomic64_dec_and_test arch_atomic64_dec_and_test
    2215             : #endif
    2216             : 
    2217             : #ifndef arch_atomic64_inc_and_test
    2218             : /**
    2219             :  * arch_atomic64_inc_and_test - increment and test
    2220             :  * @v: pointer of type atomic64_t
    2221             :  *
    2222             :  * Atomically increments @v by 1
    2223             :  * and returns true if the result is zero, or false for all
    2224             :  * other cases.
    2225             :  */
    2226             : static __always_inline bool
    2227             : arch_atomic64_inc_and_test(atomic64_t *v)
    2228             : {
    2229             :         return arch_atomic64_inc_return(v) == 0;
    2230             : }
    2231             : #define arch_atomic64_inc_and_test arch_atomic64_inc_and_test
    2232             : #endif
    2233             : 
    2234             : #ifndef arch_atomic64_add_negative
    2235             : /**
    2236             :  * arch_atomic64_add_negative - add and test if negative
    2237             :  * @i: integer value to add
    2238             :  * @v: pointer of type atomic64_t
    2239             :  *
    2240             :  * Atomically adds @i to @v and returns true
    2241             :  * if the result is negative, or false when
    2242             :  * result is greater than or equal to zero.
    2243             :  */
    2244             : static __always_inline bool
    2245             : arch_atomic64_add_negative(s64 i, atomic64_t *v)
    2246             : {
    2247             :         return arch_atomic64_add_return(i, v) < 0;
    2248             : }
    2249             : #define arch_atomic64_add_negative arch_atomic64_add_negative
    2250             : #endif
    2251             : 
    2252             : #ifndef arch_atomic64_fetch_add_unless
    2253             : /**
    2254             :  * arch_atomic64_fetch_add_unless - add unless the number is already a given value
    2255             :  * @v: pointer of type atomic64_t
    2256             :  * @a: the amount to add to v...
    2257             :  * @u: ...unless v is equal to u.
    2258             :  *
    2259             :  * Atomically adds @a to @v, so long as @v was not already @u.
    2260             :  * Returns original value of @v
    2261             :  */
    2262             : static __always_inline s64
    2263       20517 : arch_atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u)
    2264             : {
    2265       20517 :         s64 c = arch_atomic64_read(v);
    2266             : 
    2267       20517 :         do {
    2268       20517 :                 if (unlikely(c == u))
    2269             :                         break;
    2270       41034 :         } while (!arch_atomic64_try_cmpxchg(v, &c, c + a));
    2271             : 
    2272       20517 :         return c;
    2273             : }
    2274             : #define arch_atomic64_fetch_add_unless arch_atomic64_fetch_add_unless
    2275             : #endif
    2276             : 
    2277             : #ifndef arch_atomic64_add_unless
    2278             : /**
    2279             :  * arch_atomic64_add_unless - add unless the number is already a given value
    2280             :  * @v: pointer of type atomic64_t
    2281             :  * @a: the amount to add to v...
    2282             :  * @u: ...unless v is equal to u.
    2283             :  *
    2284             :  * Atomically adds @a to @v, if @v was not already @u.
    2285             :  * Returns true if the addition was done.
    2286             :  */
    2287             : static __always_inline bool
    2288       20517 : arch_atomic64_add_unless(atomic64_t *v, s64 a, s64 u)
    2289             : {
    2290       41034 :         return arch_atomic64_fetch_add_unless(v, a, u) != u;
    2291             : }
    2292             : #define arch_atomic64_add_unless arch_atomic64_add_unless
    2293             : #endif
    2294             : 
    2295             : #ifndef arch_atomic64_inc_not_zero
    2296             : /**
    2297             :  * arch_atomic64_inc_not_zero - increment unless the number is zero
    2298             :  * @v: pointer of type atomic64_t
    2299             :  *
    2300             :  * Atomically increments @v by 1, if @v is non-zero.
    2301             :  * Returns true if the increment was done.
    2302             :  */
    2303             : static __always_inline bool
    2304           1 : arch_atomic64_inc_not_zero(atomic64_t *v)
    2305             : {
    2306           2 :         return arch_atomic64_add_unless(v, 1, 0);
    2307             : }
    2308             : #define arch_atomic64_inc_not_zero arch_atomic64_inc_not_zero
    2309             : #endif
    2310             : 
    2311             : #ifndef arch_atomic64_inc_unless_negative
    2312             : static __always_inline bool
    2313             : arch_atomic64_inc_unless_negative(atomic64_t *v)
    2314             : {
    2315             :         s64 c = arch_atomic64_read(v);
    2316             : 
    2317             :         do {
    2318             :                 if (unlikely(c < 0))
    2319             :                         return false;
    2320             :         } while (!arch_atomic64_try_cmpxchg(v, &c, c + 1));
    2321             : 
    2322             :         return true;
    2323             : }
    2324             : #define arch_atomic64_inc_unless_negative arch_atomic64_inc_unless_negative
    2325             : #endif
    2326             : 
    2327             : #ifndef arch_atomic64_dec_unless_positive
    2328             : static __always_inline bool
    2329             : arch_atomic64_dec_unless_positive(atomic64_t *v)
    2330             : {
    2331             :         s64 c = arch_atomic64_read(v);
    2332             : 
    2333             :         do {
    2334             :                 if (unlikely(c > 0))
    2335             :                         return false;
    2336             :         } while (!arch_atomic64_try_cmpxchg(v, &c, c - 1));
    2337             : 
    2338             :         return true;
    2339             : }
    2340             : #define arch_atomic64_dec_unless_positive arch_atomic64_dec_unless_positive
    2341             : #endif
    2342             : 
    2343             : #ifndef arch_atomic64_dec_if_positive
    2344             : static __always_inline s64
    2345             : arch_atomic64_dec_if_positive(atomic64_t *v)
    2346             : {
    2347             :         s64 dec, c = arch_atomic64_read(v);
    2348             : 
    2349             :         do {
    2350             :                 dec = c - 1;
    2351             :                 if (unlikely(dec < 0))
    2352             :                         break;
    2353             :         } while (!arch_atomic64_try_cmpxchg(v, &c, dec));
    2354             : 
    2355             :         return dec;
    2356             : }
    2357             : #define arch_atomic64_dec_if_positive arch_atomic64_dec_if_positive
    2358             : #endif
    2359             : 
    2360             : #endif /* _LINUX_ATOMIC_FALLBACK_H */
    2361             : // cca554917d7ea73d5e3e7397dd70c484cad9b2c4

Generated by: LCOV version 1.14