Line data    Source code

       1             : /* SPDX-License-Identifier: GPL-2.0 */
       2             : 
       3             : /*
       4             :  * This file provides wrappers with sanitizer instrumentation for non-atomic
       5             :  * bit operations.
       6             :  *
       7             :  * To use this functionality, an arch's bitops.h file needs to define each of
       8             :  * the below bit operations with an arch_ prefix (e.g. arch_set_bit(),
       9             :  * arch___set_bit(), etc.).
      10             :  */
      11             : #ifndef _ASM_GENERIC_BITOPS_INSTRUMENTED_NON_ATOMIC_H
      12             : #define _ASM_GENERIC_BITOPS_INSTRUMENTED_NON_ATOMIC_H
      13             : 
      14             : #include <linux/instrumented.h>
      15             : 
      16             : /**
      17             :  * __set_bit - Set a bit in memory
      18             :  * @nr: the bit to set
      19             :  * @addr: the address to start counting from
      20             :  *
      21             :  * Unlike set_bit(), this function is non-atomic. If it is called on the same
      22             :  * region of memory concurrently, the effect may be that only one operation
      23             :  * succeeds.
      24             :  */
      25      414131 : static inline void __set_bit(long nr, volatile unsigned long *addr)
      26             : {
      27      414131 :         instrument_write(addr + BIT_WORD(nr), sizeof(long));
      28      414019 :         arch___set_bit(nr, addr);
      29      222567 : }
      30             : 
      31             : /**
      32             :  * __clear_bit - Clears a bit in memory
      33             :  * @nr: the bit to clear
      34             :  * @addr: the address to start counting from
      35             :  *
      36             :  * Unlike clear_bit(), this function is non-atomic. If it is called on the same
      37             :  * region of memory concurrently, the effect may be that only one operation
      38             :  * succeeds.
      39             :  */
      40      844342 : static inline void __clear_bit(long nr, volatile unsigned long *addr)
      41             : {
      42      844342 :         instrument_write(addr + BIT_WORD(nr), sizeof(long));
      43      844347 :         arch___clear_bit(nr, addr);
      44      379791 : }
      45             : 
      46             : /**
      47             :  * __change_bit - Toggle a bit in memory
      48             :  * @nr: the bit to change
      49             :  * @addr: the address to start counting from
      50             :  *
      51             :  * Unlike change_bit(), this function is non-atomic. If it is called on the same
      52             :  * region of memory concurrently, the effect may be that only one operation
      53             :  * succeeds.
      54             :  */
      55           0 : static inline void __change_bit(long nr, volatile unsigned long *addr)
      56             : {
      57           0 :         instrument_write(addr + BIT_WORD(nr), sizeof(long));
      58           0 :         arch___change_bit(nr, addr);
      59           0 : }
      60             : 
      61      120852 : static inline void __instrument_read_write_bitop(long nr, volatile unsigned long *addr)
      62             : {
      63      120852 :         if (IS_ENABLED(CONFIG_KCSAN_ASSUME_PLAIN_WRITES_ATOMIC)) {
      64             :                 /*
      65             :                  * We treat non-atomic read-write bitops a little more special.
      66             :                  * Given the operations here only modify a single bit, assuming
      67             :                  * non-atomicity of the writer is sufficient may be reasonable
      68             :                  * for certain usage (and follows the permissible nature of the
      69             :                  * assume-plain-writes-atomic rule):
      70             :                  * 1. report read-modify-write races -> check read;
      71             :                  * 2. do not report races with marked readers, but do report
      72             :                  *    races with unmarked readers -> check "atomic" write.
      73             :                  */
      74             :                 kcsan_check_read(addr + BIT_WORD(nr), sizeof(long));
      75             :                 /*
      76             :                  * Use generic write instrumentation, in case other sanitizers
      77             :                  * or tools are enabled alongside KCSAN.
      78             :                  */
      79             :                 instrument_write(addr + BIT_WORD(nr), sizeof(long));
      80             :         } else {
      81      242586 :                 instrument_read_write(addr + BIT_WORD(nr), sizeof(long));
      82             :         }
      83             : }
      84             : 
      85             : /**
      86             :  * __test_and_set_bit - Set a bit and return its old value
      87             :  * @nr: Bit to set
      88             :  * @addr: Address to count from
      89             :  *
      90             :  * This operation is non-atomic. If two instances of this operation race, one
      91             :  * can appear to succeed but actually fail.
      92             :  */
      93        9429 : static inline bool __test_and_set_bit(long nr, volatile unsigned long *addr)
      94             : {
      95        9429 :         __instrument_read_write_bitop(nr, addr);
      96        9429 :         return arch___test_and_set_bit(nr, addr);
      97             : }
      98             : 
      99             : /**
     100             :  * __test_and_clear_bit - Clear a bit and return its old value
     101             :  * @nr: Bit to clear
     102             :  * @addr: Address to count from
     103             :  *
     104             :  * This operation is non-atomic. If two instances of this operation race, one
     105             :  * can appear to succeed but actually fail.
     106             :  */
     107      111423 : static inline bool __test_and_clear_bit(long nr, volatile unsigned long *addr)
     108             : {
     109      111423 :         __instrument_read_write_bitop(nr, addr);
     110      112305 :         return arch___test_and_clear_bit(nr, addr);
     111             : }
     112             : 
     113             : /**
     114             :  * __test_and_change_bit - Change a bit and return its old value
     115             :  * @nr: Bit to change
     116             :  * @addr: Address to count from
     117             :  *
     118             :  * This operation is non-atomic. If two instances of this operation race, one
     119             :  * can appear to succeed but actually fail.
     120             :  */
     121             : static inline bool __test_and_change_bit(long nr, volatile unsigned long *addr)
     122             : {
     123             :         __instrument_read_write_bitop(nr, addr);
     124             :         return arch___test_and_change_bit(nr, addr);
     125             : }
     126             : 
     127             : /**
     128             :  * test_bit - Determine whether a bit is set
     129             :  * @nr: bit number to test
     130             :  * @addr: Address to start counting from
     131             :  */
     132   184583362 : static inline bool test_bit(long nr, const volatile unsigned long *addr)
     133             : {
     134   185148755 :         instrument_atomic_read(addr + BIT_WORD(nr), sizeof(long));
     135   185443673 :         return arch_test_bit(nr, addr);
     136             : }
     137             : 
     138             : #endif /* _ASM_GENERIC_BITOPS_INSTRUMENTED_NON_ATOMIC_H */