Line data    Source code

       1             : /* SPDX-License-Identifier: GPL-2.0 */
       2             : #ifndef _ASM_X86_ATOMIC_H
       3             : #define _ASM_X86_ATOMIC_H
       4             : 
       5             : #include <linux/compiler.h>
       6             : #include <linux/types.h>
       7             : #include <asm/alternative.h>
       8             : #include <asm/cmpxchg.h>
       9             : #include <asm/rmwcc.h>
      10             : #include <asm/barrier.h>
      11             : 
      12             : /*
      13             :  * Atomic operations that C can't guarantee us.  Useful for
      14             :  * resource counting etc..
      15             :  */
      16             : 
      17             : /**
      18             :  * arch_atomic_read - read atomic variable
      19             :  * @v: pointer of type atomic_t
      20             :  *
      21             :  * Atomically reads the value of @v.
      22             :  */
      23   127520518 : static __always_inline int arch_atomic_read(const atomic_t *v)
      24             : {
      25             :         /*
      26             :          * Note for KASAN: we deliberately don't use READ_ONCE_NOCHECK() here,
      27             :          * it's non-inlined function that increases binary size and stack usage.
      28             :          */
      29   126127067 :         return __READ_ONCE((v)->counter);
      30             : }
      31             : 
      32             : /**
      33             :  * arch_atomic_set - set atomic variable
      34             :  * @v: pointer of type atomic_t
      35             :  * @i: required value
      36             :  *
      37             :  * Atomically sets the value of @v to @i.
      38             :  */
      39     2199851 : static __always_inline void arch_atomic_set(atomic_t *v, int i)
      40             : {
      41     1710173 :         __WRITE_ONCE(v->counter, i);
      42             : }
      43             : 
      44             : /**
      45             :  * arch_atomic_add - add integer to atomic variable
      46             :  * @i: integer value to add
      47             :  * @v: pointer of type atomic_t
      48             :  *
      49             :  * Atomically adds @i to @v.
      50             :  */
      51        3847 : static __always_inline void arch_atomic_add(int i, atomic_t *v)
      52             : {
      53        3847 :         asm volatile(LOCK_PREFIX "addl %1,%0"
      54             :                      : "+m" (v->counter)
      55             :                      : "ir" (i) : "memory");
      56             : }
      57             : 
      58             : /**
      59             :  * arch_atomic_sub - subtract integer from atomic variable
      60             :  * @i: integer value to subtract
      61             :  * @v: pointer of type atomic_t
      62             :  *
      63             :  * Atomically subtracts @i from @v.
      64             :  */
      65       15766 : static __always_inline void arch_atomic_sub(int i, atomic_t *v)
      66             : {
      67       15766 :         asm volatile(LOCK_PREFIX "subl %1,%0"
      68             :                      : "+m" (v->counter)
      69             :                      : "ir" (i) : "memory");
      70             : }
      71             : 
      72             : /**
      73             :  * arch_atomic_sub_and_test - subtract value from variable and test result
      74             :  * @i: integer value to subtract
      75             :  * @v: pointer of type atomic_t
      76             :  *
      77             :  * Atomically subtracts @i from @v and returns
      78             :  * true if the result is zero, or false for all
      79             :  * other cases.
      80             :  */
      81           1 : static __always_inline bool arch_atomic_sub_and_test(int i, atomic_t *v)
      82             : {
      83           1 :         return GEN_BINARY_RMWcc(LOCK_PREFIX "subl", v->counter, e, "er", i);
      84             : }
      85             : #define arch_atomic_sub_and_test arch_atomic_sub_and_test
      86             : 
      87             : /**
      88             :  * arch_atomic_inc - increment atomic variable
      89             :  * @v: pointer of type atomic_t
      90             :  *
      91             :  * Atomically increments @v by 1.
      92             :  */
      93      612635 : static __always_inline void arch_atomic_inc(atomic_t *v)
      94             : {
      95      612635 :         asm volatile(LOCK_PREFIX "incl %0"
      96             :                      : "+m" (v->counter) :: "memory");
      97             : }
      98             : #define arch_atomic_inc arch_atomic_inc
      99             : 
     100             : /**
     101             :  * arch_atomic_dec - decrement atomic variable
     102             :  * @v: pointer of type atomic_t
     103             :  *
     104             :  * Atomically decrements @v by 1.
     105             :  */
     106      136850 : static __always_inline void arch_atomic_dec(atomic_t *v)
     107             : {
     108      136850 :         asm volatile(LOCK_PREFIX "decl %0"
     109             :                      : "+m" (v->counter) :: "memory");
     110             : }
     111             : #define arch_atomic_dec arch_atomic_dec
     112             : 
     113             : /**
     114             :  * arch_atomic_dec_and_test - decrement and test
     115             :  * @v: pointer of type atomic_t
     116             :  *
     117             :  * Atomically decrements @v by 1 and
     118             :  * returns true if the result is 0, or false for all other
     119             :  * cases.
     120             :  */
     121     2025038 : static __always_inline bool arch_atomic_dec_and_test(atomic_t *v)
     122             : {
     123     2025038 :         return GEN_UNARY_RMWcc(LOCK_PREFIX "decl", v->counter, e);
     124             : }
     125             : #define arch_atomic_dec_and_test arch_atomic_dec_and_test
     126             : 
     127             : /**
     128             :  * arch_atomic_inc_and_test - increment and test
     129             :  * @v: pointer of type atomic_t
     130             :  *
     131             :  * Atomically increments @v by 1
     132             :  * and returns true if the result is zero, or false for all
     133             :  * other cases.
     134             :  */
     135      797991 : static __always_inline bool arch_atomic_inc_and_test(atomic_t *v)
     136             : {
     137      797991 :         return GEN_UNARY_RMWcc(LOCK_PREFIX "incl", v->counter, e);
     138             : }
     139             : #define arch_atomic_inc_and_test arch_atomic_inc_and_test
     140             : 
     141             : /**
     142             :  * arch_atomic_add_negative - add and test if negative
     143             :  * @i: integer value to add
     144             :  * @v: pointer of type atomic_t
     145             :  *
     146             :  * Atomically adds @i to @v and returns true
     147             :  * if the result is negative, or false when
     148             :  * result is greater than or equal to zero.
     149             :  */
     150      923058 : static __always_inline bool arch_atomic_add_negative(int i, atomic_t *v)
     151             : {
     152      923058 :         return GEN_BINARY_RMWcc(LOCK_PREFIX "addl", v->counter, s, "er", i);
     153             : }
     154             : #define arch_atomic_add_negative arch_atomic_add_negative
     155             : 
     156             : /**
     157             :  * arch_atomic_add_return - add integer and return
     158             :  * @i: integer value to add
     159             :  * @v: pointer of type atomic_t
     160             :  *
     161             :  * Atomically adds @i to @v and returns @i + @v
     162             :  */
     163      173046 : static __always_inline int arch_atomic_add_return(int i, atomic_t *v)
     164             : {
     165      130281 :         return i + xadd(&v->counter, i);
     166             : }
     167             : #define arch_atomic_add_return arch_atomic_add_return
     168             : 
     169             : /**
     170             :  * arch_atomic_sub_return - subtract integer and return
     171             :  * @v: pointer of type atomic_t
     172             :  * @i: integer value to subtract
     173             :  *
     174             :  * Atomically subtracts @i from @v and returns @v - @i
     175             :  */
     176       48931 : static __always_inline int arch_atomic_sub_return(int i, atomic_t *v)
     177             : {
     178       48931 :         return arch_atomic_add_return(-i, v);
     179             : }
     180             : #define arch_atomic_sub_return arch_atomic_sub_return
     181             : 
     182       50101 : static __always_inline int arch_atomic_fetch_add(int i, atomic_t *v)
     183             : {
     184       50101 :         return xadd(&v->counter, i);
     185             : }
     186             : #define arch_atomic_fetch_add arch_atomic_fetch_add
     187             : 
     188       54452 : static __always_inline int arch_atomic_fetch_sub(int i, atomic_t *v)
     189             : {
     190       54452 :         return xadd(&v->counter, -i);
     191             : }
     192             : #define arch_atomic_fetch_sub arch_atomic_fetch_sub
     193             : 
     194        1228 : static __always_inline int arch_atomic_cmpxchg(atomic_t *v, int old, int new)
     195             : {
     196        1228 :         return arch_cmpxchg(&v->counter, old, new);
     197             : }
     198             : #define arch_atomic_cmpxchg arch_atomic_cmpxchg
     199             : 
     200     9918031 : static __always_inline bool arch_atomic_try_cmpxchg(atomic_t *v, int *old, int new)
     201             : {
     202     9918032 :         return arch_try_cmpxchg(&v->counter, old, new);
     203             : }
     204             : #define arch_atomic_try_cmpxchg arch_atomic_try_cmpxchg
     205             : 
     206        1229 : static __always_inline int arch_atomic_xchg(atomic_t *v, int new)
     207             : {
     208        1229 :         return arch_xchg(&v->counter, new);
     209             : }
     210             : #define arch_atomic_xchg arch_atomic_xchg
     211             : 
     212           0 : static __always_inline void arch_atomic_and(int i, atomic_t *v)
     213             : {
     214           0 :         asm volatile(LOCK_PREFIX "andl %1,%0"
     215             :                         : "+m" (v->counter)
     216             :                         : "ir" (i)
     217             :                         : "memory");
     218             : }
     219             : 
     220         220 : static __always_inline int arch_atomic_fetch_and(int i, atomic_t *v)
     221             : {
     222         220 :         int val = arch_atomic_read(v);
     223             : 
     224         440 :         do { } while (!arch_atomic_try_cmpxchg(v, &val, val & i));
     225             : 
     226         220 :         return val;
     227             : }
     228             : #define arch_atomic_fetch_and arch_atomic_fetch_and
     229             : 
     230           0 : static __always_inline void arch_atomic_or(int i, atomic_t *v)
     231             : {
     232           0 :         asm volatile(LOCK_PREFIX "orl %1,%0"
     233             :                         : "+m" (v->counter)
     234             :                         : "ir" (i)
     235             :                         : "memory");
     236             : }
     237             : 
     238         302 : static __always_inline int arch_atomic_fetch_or(int i, atomic_t *v)
     239             : {
     240         302 :         int val = arch_atomic_read(v);
     241             : 
     242         606 :         do { } while (!arch_atomic_try_cmpxchg(v, &val, val | i));
     243             : 
     244         303 :         return val;
     245             : }
     246             : #define arch_atomic_fetch_or arch_atomic_fetch_or
     247             : 
     248           0 : static __always_inline void arch_atomic_xor(int i, atomic_t *v)
     249             : {
     250           0 :         asm volatile(LOCK_PREFIX "xorl %1,%0"
     251             :                         : "+m" (v->counter)
     252             :                         : "ir" (i)
     253             :                         : "memory");
     254             : }
     255             : 
     256           0 : static __always_inline int arch_atomic_fetch_xor(int i, atomic_t *v)
     257             : {
     258           0 :         int val = arch_atomic_read(v);
     259             : 
     260           0 :         do { } while (!arch_atomic_try_cmpxchg(v, &val, val ^ i));
     261             : 
     262           0 :         return val;
     263             : }
     264             : #define arch_atomic_fetch_xor arch_atomic_fetch_xor
     265             : 
     266             : #ifdef CONFIG_X86_32
     267             : # include <asm/atomic64_32.h>
     268             : #else
     269             : # include <asm/atomic64_64.h>
     270             : #endif
     271             : 
     272             : #define ARCH_ATOMIC
     273             : 
     274             : #endif /* _ASM_X86_ATOMIC_H */