Line data    Source code

       1             : // SPDX-License-Identifier: GPL-2.0
       2             : /*
       3             :  * Fast batching percpu counters.
       4             :  */
       5             : 
       6             : #include <linux/percpu_counter.h>
       7             : #include <linux/mutex.h>
       8             : #include <linux/init.h>
       9             : #include <linux/cpu.h>
      10             : #include <linux/module.h>
      11             : #include <linux/debugobjects.h>
      12             : 
      13             : #ifdef CONFIG_HOTPLUG_CPU
      14             : static LIST_HEAD(percpu_counters);
      15             : static DEFINE_SPINLOCK(percpu_counters_lock);
      16             : #endif
      17             : 
      18             : #ifdef CONFIG_DEBUG_OBJECTS_PERCPU_COUNTER
      19             : 
      20             : static const struct debug_obj_descr percpu_counter_debug_descr;
      21             : 
      22             : static bool percpu_counter_fixup_free(void *addr, enum debug_obj_state state)
      23             : {
      24             :         struct percpu_counter *fbc = addr;
      25             : 
      26             :         switch (state) {
      27             :         case ODEBUG_STATE_ACTIVE:
      28             :                 percpu_counter_destroy(fbc);
      29             :                 debug_object_free(fbc, &percpu_counter_debug_descr);
      30             :                 return true;
      31             :         default:
      32             :                 return false;
      33             :         }
      34             : }
      35             : 
      36             : static const struct debug_obj_descr percpu_counter_debug_descr = {
      37             :         .name           = "percpu_counter",
      38             :         .fixup_free     = percpu_counter_fixup_free,
      39             : };
      40             : 
      41             : static inline void debug_percpu_counter_activate(struct percpu_counter *fbc)
      42             : {
      43             :         debug_object_init(fbc, &percpu_counter_debug_descr);
      44             :         debug_object_activate(fbc, &percpu_counter_debug_descr);
      45             : }
      46             : 
      47             : static inline void debug_percpu_counter_deactivate(struct percpu_counter *fbc)
      48             : {
      49             :         debug_object_deactivate(fbc, &percpu_counter_debug_descr);
      50             :         debug_object_free(fbc, &percpu_counter_debug_descr);
      51             : }
      52             : 
      53             : #else   /* CONFIG_DEBUG_OBJECTS_PERCPU_COUNTER */
      54         168 : static inline void debug_percpu_counter_activate(struct percpu_counter *fbc)
      55         168 : { }
      56          95 : static inline void debug_percpu_counter_deactivate(struct percpu_counter *fbc)
      57          95 : { }
      58             : #endif  /* CONFIG_DEBUG_OBJECTS_PERCPU_COUNTER */
      59             : 
      60           0 : void percpu_counter_set(struct percpu_counter *fbc, s64 amount)
      61             : {
      62           0 :         int cpu;
      63           0 :         unsigned long flags;
      64             : 
      65           0 :         raw_spin_lock_irqsave(&fbc->lock, flags);
      66           0 :         for_each_possible_cpu(cpu) {
      67           0 :                 s32 *pcount = per_cpu_ptr(fbc->counters, cpu);
      68           0 :                 *pcount = 0;
      69             :         }
      70           0 :         fbc->count = amount;
      71           0 :         raw_spin_unlock_irqrestore(&fbc->lock, flags);
      72           0 : }
      73             : EXPORT_SYMBOL(percpu_counter_set);
      74             : 
      75             : /**
      76             :  * This function is both preempt and irq safe. The former is due to explicit
      77             :  * preemption disable. The latter is guaranteed by the fact that the slow path
      78             :  * is explicitly protected by an irq-safe spinlock whereas the fast patch uses
      79             :  * this_cpu_add which is irq-safe by definition. Hence there is no need muck
      80             :  * with irq state before calling this one
      81             :  */
      82      322013 : void percpu_counter_add_batch(struct percpu_counter *fbc, s64 amount, s32 batch)
      83             : {
      84      322013 :         s64 count;
      85             : 
      86      322013 :         preempt_disable();
      87      322028 :         count = __this_cpu_read(*fbc->counters) + amount;
      88      322028 :         if (abs(count) >= batch) {
      89        1832 :                 unsigned long flags;
      90        1832 :                 raw_spin_lock_irqsave(&fbc->lock, flags);
      91        1832 :                 fbc->count += count;
      92        1832 :                 __this_cpu_sub(*fbc->counters, count - amount);
      93        1832 :                 raw_spin_unlock_irqrestore(&fbc->lock, flags);
      94             :         } else {
      95      320196 :                 this_cpu_add(*fbc->counters, amount);
      96             :         }
      97      322030 :         preempt_enable();
      98      322027 : }
      99             : EXPORT_SYMBOL(percpu_counter_add_batch);
     100             : 
     101             : /*
     102             :  * For percpu_counter with a big batch, the devication of its count could
     103             :  * be big, and there is requirement to reduce the deviation, like when the
     104             :  * counter's batch could be runtime decreased to get a better accuracy,
     105             :  * which can be achieved by running this sync function on each CPU.
     106             :  */
     107           0 : void percpu_counter_sync(struct percpu_counter *fbc)
     108             : {
     109           0 :         unsigned long flags;
     110           0 :         s64 count;
     111             : 
     112           0 :         raw_spin_lock_irqsave(&fbc->lock, flags);
     113           0 :         count = __this_cpu_read(*fbc->counters);
     114           0 :         fbc->count += count;
     115           0 :         __this_cpu_sub(*fbc->counters, count);
     116           0 :         raw_spin_unlock_irqrestore(&fbc->lock, flags);
     117           0 : }
     118             : EXPORT_SYMBOL(percpu_counter_sync);
     119             : 
     120             : /*
     121             :  * Add up all the per-cpu counts, return the result.  This is a more accurate
     122             :  * but much slower version of percpu_counter_read_positive()
     123             :  */
     124         815 : s64 __percpu_counter_sum(struct percpu_counter *fbc)
     125             : {
     126         815 :         s64 ret;
     127         815 :         int cpu;
     128         815 :         unsigned long flags;
     129             : 
     130         815 :         raw_spin_lock_irqsave(&fbc->lock, flags);
     131         815 :         ret = fbc->count;
     132        4075 :         for_each_online_cpu(cpu) {
     133        3260 :                 s32 *pcount = per_cpu_ptr(fbc->counters, cpu);
     134        3260 :                 ret += *pcount;
     135             :         }
     136         815 :         raw_spin_unlock_irqrestore(&fbc->lock, flags);
     137         815 :         return ret;
     138             : }
     139             : EXPORT_SYMBOL(__percpu_counter_sum);
     140             : 
     141         168 : int __percpu_counter_init(struct percpu_counter *fbc, s64 amount, gfp_t gfp,
     142             :                           struct lock_class_key *key)
     143             : {
     144         168 :         unsigned long flags __maybe_unused;
     145             : 
     146         168 :         raw_spin_lock_init(&fbc->lock);
     147         168 :         lockdep_set_class(&fbc->lock, key);
     148         168 :         fbc->count = amount;
     149         168 :         fbc->counters = alloc_percpu_gfp(s32, gfp);
     150         168 :         if (!fbc->counters)
     151             :                 return -ENOMEM;
     152             : 
     153         168 :         debug_percpu_counter_activate(fbc);
     154             : 
     155             : #ifdef CONFIG_HOTPLUG_CPU
     156         168 :         INIT_LIST_HEAD(&fbc->list);
     157         168 :         spin_lock_irqsave(&percpu_counters_lock, flags);
     158         168 :         list_add(&fbc->list, &percpu_counters);
     159         168 :         spin_unlock_irqrestore(&percpu_counters_lock, flags);
     160             : #endif
     161         168 :         return 0;
     162             : }
     163             : EXPORT_SYMBOL(__percpu_counter_init);
     164             : 
     165          95 : void percpu_counter_destroy(struct percpu_counter *fbc)
     166             : {
     167          95 :         unsigned long flags __maybe_unused;
     168             : 
     169          95 :         if (!fbc->counters)
     170             :                 return;
     171             : 
     172          95 :         debug_percpu_counter_deactivate(fbc);
     173             : 
     174             : #ifdef CONFIG_HOTPLUG_CPU
     175          95 :         spin_lock_irqsave(&percpu_counters_lock, flags);
     176          95 :         list_del(&fbc->list);
     177          95 :         spin_unlock_irqrestore(&percpu_counters_lock, flags);
     178             : #endif
     179          95 :         free_percpu(fbc->counters);
     180          95 :         fbc->counters = NULL;
     181             : }
     182             : EXPORT_SYMBOL(percpu_counter_destroy);
     183             : 
     184             : int percpu_counter_batch __read_mostly = 32;
     185             : EXPORT_SYMBOL(percpu_counter_batch);
     186             : 
     187           4 : static int compute_batch_value(unsigned int cpu)
     188             : {
     189           4 :         int nr = num_online_cpus();
     190             : 
     191           4 :         percpu_counter_batch = max(32, nr*2);
     192           4 :         return 0;
     193             : }
     194             : 
     195           0 : static int percpu_counter_cpu_dead(unsigned int cpu)
     196             : {
     197             : #ifdef CONFIG_HOTPLUG_CPU
     198           0 :         struct percpu_counter *fbc;
     199             : 
     200           0 :         compute_batch_value(cpu);
     201             : 
     202           0 :         spin_lock_irq(&percpu_counters_lock);
     203           0 :         list_for_each_entry(fbc, &percpu_counters, list) {
     204           0 :                 s32 *pcount;
     205             : 
     206           0 :                 raw_spin_lock(&fbc->lock);
     207           0 :                 pcount = per_cpu_ptr(fbc->counters, cpu);
     208           0 :                 fbc->count += *pcount;
     209           0 :                 *pcount = 0;
     210           0 :                 raw_spin_unlock(&fbc->lock);
     211             :         }
     212           0 :         spin_unlock_irq(&percpu_counters_lock);
     213             : #endif
     214           0 :         return 0;
     215             : }
     216             : 
     217             : /*
     218             :  * Compare counter against given value.
     219             :  * Return 1 if greater, 0 if equal and -1 if less
     220             :  */
     221        1983 : int __percpu_counter_compare(struct percpu_counter *fbc, s64 rhs, s32 batch)
     222             : {
     223        1983 :         s64     count;
     224             : 
     225        1983 :         count = percpu_counter_read(fbc);
     226             :         /* Check to see if rough count will be sufficient for comparison */
     227        1983 :         if (abs(count - rhs) > (batch * num_online_cpus())) {
     228        1983 :                 if (count > rhs)
     229             :                         return 1;
     230             :                 else
     231        1983 :                         return -1;
     232             :         }
     233             :         /* Need to use precise count */
     234           0 :         count = percpu_counter_sum(fbc);
     235           0 :         if (count > rhs)
     236             :                 return 1;
     237           0 :         else if (count < rhs)
     238             :                 return -1;
     239             :         else
     240           0 :                 return 0;
     241             : }
     242             : EXPORT_SYMBOL(__percpu_counter_compare);
     243             : 
     244           1 : static int __init percpu_counter_startup(void)
     245             : {
     246           1 :         int ret;
     247             : 
     248           1 :         ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "lib/percpu_cnt:online",
     249             :                                 compute_batch_value, NULL);
     250           1 :         WARN_ON(ret < 0);
     251           1 :         ret = cpuhp_setup_state_nocalls(CPUHP_PERCPU_CNT_DEAD,
     252             :                                         "lib/percpu_cnt:dead", NULL,
     253             :                                         percpu_counter_cpu_dead);
     254           1 :         WARN_ON(ret < 0);
     255           1 :         return 0;
     256             : }
     257             : module_init(percpu_counter_startup);