Line data    Source code

       1             : // SPDX-License-Identifier: GPL-2.0
       2             : #include <linux/export.h>
       3             : #include <linux/lockref.h>
       4             : 
       5             : #if USE_CMPXCHG_LOCKREF
       6             : 
       7             : /*
       8             :  * Note that the "cmpxchg()" reloads the "old" value for the
       9             :  * failure case.
      10             :  */
      11             : #define CMPXCHG_LOOP(CODE, SUCCESS) do {                                        \
      12             :         int retry = 100;                                                        \
      13             :         struct lockref old;                                                     \
      14             :         BUILD_BUG_ON(sizeof(old) != 8);                                         \
      15             :         old.lock_count = READ_ONCE(lockref->lock_count);                     \
      16             :         while (likely(arch_spin_value_unlocked(old.lock.rlock.raw_lock))) {     \
      17             :                 struct lockref new = old, prev = old;                           \
      18             :                 CODE                                                            \
      19             :                 old.lock_count = cmpxchg64_relaxed(&lockref->lock_count, \
      20             :                                                    old.lock_count,              \
      21             :                                                    new.lock_count);             \
      22             :                 if (likely(old.lock_count == prev.lock_count)) {                \
      23             :                         SUCCESS;                                                \
      24             :                 }                                                               \
      25             :                 if (!--retry)                                                   \
      26             :                         break;                                                  \
      27             :                 cpu_relax();                                                    \
      28             :         }                                                                       \
      29             : } while (0)
      30             : 
      31             : #else
      32             : 
      33             : #define CMPXCHG_LOOP(CODE, SUCCESS) do { } while (0)
      34             : 
      35             : #endif
      36             : 
      37             : /**
      38             :  * lockref_get - Increments reference count unconditionally
      39             :  * @lockref: pointer to lockref structure
      40             :  *
      41             :  * This operation is only valid if you already hold a reference
      42             :  * to the object, so you know the count cannot be zero.
      43             :  */
      44       69636 : void lockref_get(struct lockref *lockref)
      45             : {
      46       69636 :         CMPXCHG_LOOP(
      47             :                 new.count++;
      48             :         ,
      49             :                 return;
      50             :         );
      51             : 
      52       69636 :         spin_lock(&lockref->lock);
      53       69653 :         lockref->count++;
      54       69653 :         spin_unlock(&lockref->lock);
      55       69657 : }
      56             : EXPORT_SYMBOL(lockref_get);
      57             : 
      58             : /**
      59             :  * lockref_get_not_zero - Increments count unless the count is 0 or dead
      60             :  * @lockref: pointer to lockref structure
      61             :  * Return: 1 if count updated successfully or 0 if count was zero
      62             :  */
      63        2489 : int lockref_get_not_zero(struct lockref *lockref)
      64             : {
      65        2489 :         int retval;
      66             : 
      67        2489 :         CMPXCHG_LOOP(
      68             :                 new.count++;
      69             :                 if (old.count <= 0)
      70             :                         return 0;
      71             :         ,
      72             :                 return 1;
      73             :         );
      74             : 
      75        2489 :         spin_lock(&lockref->lock);
      76        2489 :         retval = 0;
      77        2489 :         if (lockref->count > 0) {
      78        2489 :                 lockref->count++;
      79        2489 :                 retval = 1;
      80             :         }
      81        2489 :         spin_unlock(&lockref->lock);
      82        2489 :         return retval;
      83             : }
      84             : EXPORT_SYMBOL(lockref_get_not_zero);
      85             : 
      86             : /**
      87             :  * lockref_put_not_zero - Decrements count unless count <= 1 before decrement
      88             :  * @lockref: pointer to lockref structure
      89             :  * Return: 1 if count updated successfully or 0 if count would become zero
      90             :  */
      91           0 : int lockref_put_not_zero(struct lockref *lockref)
      92             : {
      93           0 :         int retval;
      94             : 
      95           0 :         CMPXCHG_LOOP(
      96             :                 new.count--;
      97             :                 if (old.count <= 1)
      98             :                         return 0;
      99             :         ,
     100             :                 return 1;
     101             :         );
     102             : 
     103           0 :         spin_lock(&lockref->lock);
     104           0 :         retval = 0;
     105           0 :         if (lockref->count > 1) {
     106           0 :                 lockref->count--;
     107           0 :                 retval = 1;
     108             :         }
     109           0 :         spin_unlock(&lockref->lock);
     110           0 :         return retval;
     111             : }
     112             : EXPORT_SYMBOL(lockref_put_not_zero);
     113             : 
     114             : /**
     115             :  * lockref_get_or_lock - Increments count unless the count is 0 or dead
     116             :  * @lockref: pointer to lockref structure
     117             :  * Return: 1 if count updated successfully or 0 if count was zero
     118             :  * and we got the lock instead.
     119             :  */
     120           0 : int lockref_get_or_lock(struct lockref *lockref)
     121             : {
     122           0 :         CMPXCHG_LOOP(
     123             :                 new.count++;
     124             :                 if (old.count <= 0)
     125             :                         break;
     126             :         ,
     127             :                 return 1;
     128             :         );
     129             : 
     130           0 :         spin_lock(&lockref->lock);
     131           0 :         if (lockref->count <= 0)
     132             :                 return 0;
     133           0 :         lockref->count++;
     134           0 :         spin_unlock(&lockref->lock);
     135           0 :         return 1;
     136             : }
     137             : EXPORT_SYMBOL(lockref_get_or_lock);
     138             : 
     139             : /**
     140             :  * lockref_put_return - Decrement reference count if possible
     141             :  * @lockref: pointer to lockref structure
     142             :  *
     143             :  * Decrement the reference count and return the new value.
     144             :  * If the lockref was dead or locked, return an error.
     145             :  */
     146      177602 : int lockref_put_return(struct lockref *lockref)
     147             : {
     148      177602 :         CMPXCHG_LOOP(
     149             :                 new.count--;
     150             :                 if (old.count <= 0)
     151             :                         return -1;
     152             :         ,
     153             :                 return new.count;
     154             :         );
     155      177602 :         return -1;
     156             : }
     157             : EXPORT_SYMBOL(lockref_put_return);
     158             : 
     159             : /**
     160             :  * lockref_put_or_lock - decrements count unless count <= 1 before decrement
     161             :  * @lockref: pointer to lockref structure
     162             :  * Return: 1 if count updated successfully or 0 if count <= 1 and lock taken
     163             :  */
     164       69714 : int lockref_put_or_lock(struct lockref *lockref)
     165             : {
     166       69714 :         CMPXCHG_LOOP(
     167             :                 new.count--;
     168             :                 if (old.count <= 1)
     169             :                         break;
     170             :         ,
     171             :                 return 1;
     172             :         );
     173             : 
     174       69714 :         spin_lock(&lockref->lock);
     175       69719 :         if (lockref->count <= 1)
     176             :                 return 0;
     177       57453 :         lockref->count--;
     178       57453 :         spin_unlock(&lockref->lock);
     179       57453 :         return 1;
     180             : }
     181             : EXPORT_SYMBOL(lockref_put_or_lock);
     182             : 
     183             : /**
     184             :  * lockref_mark_dead - mark lockref dead
     185             :  * @lockref: pointer to lockref structure
     186             :  */
     187       14149 : void lockref_mark_dead(struct lockref *lockref)
     188             : {
     189       14149 :         assert_spin_locked(&lockref->lock);
     190       14149 :         lockref->count = -128;
     191       14149 : }
     192             : EXPORT_SYMBOL(lockref_mark_dead);
     193             : 
     194             : /**
     195             :  * lockref_get_not_dead - Increments count unless the ref is dead
     196             :  * @lockref: pointer to lockref structure
     197             :  * Return: 1 if count updated successfully or 0 if lockref was dead
     198             :  */
     199      103315 : int lockref_get_not_dead(struct lockref *lockref)
     200             : {
     201      103315 :         int retval;
     202             : 
     203      103315 :         CMPXCHG_LOOP(
     204             :                 new.count++;
     205             :                 if (old.count < 0)
     206             :                         return 0;
     207             :         ,
     208             :                 return 1;
     209             :         );
     210             : 
     211      103315 :         spin_lock(&lockref->lock);
     212      103316 :         retval = 0;
     213      103316 :         if (lockref->count >= 0) {
     214      103314 :                 lockref->count++;
     215      103314 :                 retval = 1;
     216             :         }
     217      103316 :         spin_unlock(&lockref->lock);
     218      103313 :         return retval;
     219             : }
     220             : EXPORT_SYMBOL(lockref_get_not_dead);