Line data    Source code

       1             : // SPDX-License-Identifier: GPL-2.0
       2             : /*
       3             :  * Block stat tracking code
       4             :  *
       5             :  * Copyright (C) 2016 Jens Axboe
       6             :  */
       7             : #include <linux/kernel.h>
       8             : #include <linux/rculist.h>
       9             : #include <linux/blk-mq.h>
      10             : 
      11             : #include "blk-stat.h"
      12             : #include "blk-mq.h"
      13             : #include "blk.h"
      14             : 
      15             : struct blk_queue_stats {
      16             :         struct list_head callbacks;
      17             :         spinlock_t lock;
      18             :         bool enable_accounting;
      19             : };
      20             : 
      21           0 : void blk_rq_stat_init(struct blk_rq_stat *stat)
      22             : {
      23           0 :         stat->min = -1ULL;
      24           0 :         stat->max = stat->nr_samples = stat->mean = 0;
      25           0 :         stat->batch = 0;
      26           0 : }
      27             : 
      28             : /* src is a per-cpu stat, mean isn't initialized */
      29           0 : void blk_rq_stat_sum(struct blk_rq_stat *dst, struct blk_rq_stat *src)
      30             : {
      31           0 :         if (!src->nr_samples)
      32             :                 return;
      33             : 
      34           0 :         dst->min = min(dst->min, src->min);
      35           0 :         dst->max = max(dst->max, src->max);
      36             : 
      37           0 :         dst->mean = div_u64(src->batch + dst->mean * dst->nr_samples,
      38           0 :                                 dst->nr_samples + src->nr_samples);
      39             : 
      40           0 :         dst->nr_samples += src->nr_samples;
      41             : }
      42             : 
      43           0 : void blk_rq_stat_add(struct blk_rq_stat *stat, u64 value)
      44             : {
      45           0 :         stat->min = min(stat->min, value);
      46           0 :         stat->max = max(stat->max, value);
      47           0 :         stat->batch += value;
      48           0 :         stat->nr_samples++;
      49           0 : }
      50             : 
      51           0 : void blk_stat_add(struct request *rq, u64 now)
      52             : {
      53           0 :         struct request_queue *q = rq->q;
      54           0 :         struct blk_stat_callback *cb;
      55           0 :         struct blk_rq_stat *stat;
      56           0 :         int bucket, cpu;
      57           0 :         u64 value;
      58             : 
      59           0 :         value = (now >= rq->io_start_time_ns) ? now - rq->io_start_time_ns : 0;
      60             : 
      61           0 :         blk_throtl_stat_add(rq, value);
      62             : 
      63           0 :         rcu_read_lock();
      64           0 :         cpu = get_cpu();
      65           0 :         list_for_each_entry_rcu(cb, &q->stats->callbacks, list) {
      66           0 :                 if (!blk_stat_is_active(cb))
      67           0 :                         continue;
      68             : 
      69           0 :                 bucket = cb->bucket_fn(rq);
      70           0 :                 if (bucket < 0)
      71           0 :                         continue;
      72             : 
      73           0 :                 stat = &per_cpu_ptr(cb->cpu_stat, cpu)[bucket];
      74           0 :                 blk_rq_stat_add(stat, value);
      75             :         }
      76           0 :         put_cpu();
      77           0 :         rcu_read_unlock();
      78           0 : }
      79             : 
      80           0 : static void blk_stat_timer_fn(struct timer_list *t)
      81             : {
      82           0 :         struct blk_stat_callback *cb = from_timer(cb, t, timer);
      83           0 :         unsigned int bucket;
      84           0 :         int cpu;
      85             : 
      86           0 :         for (bucket = 0; bucket < cb->buckets; bucket++)
      87           0 :                 blk_rq_stat_init(&cb->stat[bucket]);
      88             : 
      89           0 :         for_each_online_cpu(cpu) {
      90           0 :                 struct blk_rq_stat *cpu_stat;
      91             : 
      92           0 :                 cpu_stat = per_cpu_ptr(cb->cpu_stat, cpu);
      93           0 :                 for (bucket = 0; bucket < cb->buckets; bucket++) {
      94           0 :                         blk_rq_stat_sum(&cb->stat[bucket], &cpu_stat[bucket]);
      95           0 :                         blk_rq_stat_init(&cpu_stat[bucket]);
      96             :                 }
      97             :         }
      98             : 
      99           0 :         cb->timer_fn(cb);
     100           0 : }
     101             : 
     102             : struct blk_stat_callback *
     103           9 : blk_stat_alloc_callback(void (*timer_fn)(struct blk_stat_callback *),
     104             :                         int (*bucket_fn)(const struct request *),
     105             :                         unsigned int buckets, void *data)
     106             : {
     107           9 :         struct blk_stat_callback *cb;
     108             : 
     109           9 :         cb = kmalloc(sizeof(*cb), GFP_KERNEL);
     110           9 :         if (!cb)
     111             :                 return NULL;
     112             : 
     113           9 :         cb->stat = kmalloc_array(buckets, sizeof(struct blk_rq_stat),
     114             :                                  GFP_KERNEL);
     115           9 :         if (!cb->stat) {
     116           0 :                 kfree(cb);
     117           0 :                 return NULL;
     118             :         }
     119           9 :         cb->cpu_stat = __alloc_percpu(buckets * sizeof(struct blk_rq_stat),
     120             :                                       __alignof__(struct blk_rq_stat));
     121           9 :         if (!cb->cpu_stat) {
     122           0 :                 kfree(cb->stat);
     123           0 :                 kfree(cb);
     124           0 :                 return NULL;
     125             :         }
     126             : 
     127           9 :         cb->timer_fn = timer_fn;
     128           9 :         cb->bucket_fn = bucket_fn;
     129           9 :         cb->data = data;
     130           9 :         cb->buckets = buckets;
     131           9 :         timer_setup(&cb->timer, blk_stat_timer_fn, 0);
     132             : 
     133           9 :         return cb;
     134             : }
     135             : 
     136           0 : void blk_stat_add_callback(struct request_queue *q,
     137             :                            struct blk_stat_callback *cb)
     138             : {
     139           0 :         unsigned int bucket;
     140           0 :         unsigned long flags;
     141           0 :         int cpu;
     142             : 
     143           0 :         for_each_possible_cpu(cpu) {
     144           0 :                 struct blk_rq_stat *cpu_stat;
     145             : 
     146           0 :                 cpu_stat = per_cpu_ptr(cb->cpu_stat, cpu);
     147           0 :                 for (bucket = 0; bucket < cb->buckets; bucket++)
     148           0 :                         blk_rq_stat_init(&cpu_stat[bucket]);
     149             :         }
     150             : 
     151           0 :         spin_lock_irqsave(&q->stats->lock, flags);
     152           0 :         list_add_tail_rcu(&cb->list, &q->stats->callbacks);
     153           0 :         blk_queue_flag_set(QUEUE_FLAG_STATS, q);
     154           0 :         spin_unlock_irqrestore(&q->stats->lock, flags);
     155           0 : }
     156             : 
     157           0 : void blk_stat_remove_callback(struct request_queue *q,
     158             :                               struct blk_stat_callback *cb)
     159             : {
     160           0 :         unsigned long flags;
     161             : 
     162           0 :         spin_lock_irqsave(&q->stats->lock, flags);
     163           0 :         list_del_rcu(&cb->list);
     164           0 :         if (list_empty(&q->stats->callbacks) && !q->stats->enable_accounting)
     165           0 :                 blk_queue_flag_clear(QUEUE_FLAG_STATS, q);
     166           0 :         spin_unlock_irqrestore(&q->stats->lock, flags);
     167             : 
     168           0 :         del_timer_sync(&cb->timer);
     169           0 : }
     170             : 
     171           0 : static void blk_stat_free_callback_rcu(struct rcu_head *head)
     172             : {
     173           0 :         struct blk_stat_callback *cb;
     174             : 
     175           0 :         cb = container_of(head, struct blk_stat_callback, rcu);
     176           0 :         free_percpu(cb->cpu_stat);
     177           0 :         kfree(cb->stat);
     178           0 :         kfree(cb);
     179           0 : }
     180             : 
     181           0 : void blk_stat_free_callback(struct blk_stat_callback *cb)
     182             : {
     183           0 :         if (cb)
     184           0 :                 call_rcu(&cb->rcu, blk_stat_free_callback_rcu);
     185           0 : }
     186             : 
     187           0 : void blk_stat_enable_accounting(struct request_queue *q)
     188             : {
     189           0 :         unsigned long flags;
     190             : 
     191           0 :         spin_lock_irqsave(&q->stats->lock, flags);
     192           0 :         q->stats->enable_accounting = true;
     193           0 :         blk_queue_flag_set(QUEUE_FLAG_STATS, q);
     194           0 :         spin_unlock_irqrestore(&q->stats->lock, flags);
     195           0 : }
     196             : EXPORT_SYMBOL_GPL(blk_stat_enable_accounting);
     197             : 
     198           9 : struct blk_queue_stats *blk_alloc_queue_stats(void)
     199             : {
     200           9 :         struct blk_queue_stats *stats;
     201             : 
     202           9 :         stats = kmalloc(sizeof(*stats), GFP_KERNEL);
     203           9 :         if (!stats)
     204             :                 return NULL;
     205             : 
     206           9 :         INIT_LIST_HEAD(&stats->callbacks);
     207           9 :         spin_lock_init(&stats->lock);
     208           9 :         stats->enable_accounting = false;
     209             : 
     210           9 :         return stats;
     211             : }
     212             : 
     213           0 : void blk_free_queue_stats(struct blk_queue_stats *stats)
     214             : {
     215           0 :         if (!stats)
     216             :                 return;
     217             : 
     218           0 :         WARN_ON(!list_empty(&stats->callbacks));
     219             : 
     220           0 :         kfree(stats);
     221             : }