Line data    Source code

       1             : // SPDX-License-Identifier: GPL-2.0-or-later
       2             : /* delayacct.c - per-task delay accounting
       3             :  *
       4             :  * Copyright (C) Shailabh Nagar, IBM Corp. 2006
       5             :  */
       6             : 
       7             : #include <linux/sched.h>
       8             : #include <linux/sched/task.h>
       9             : #include <linux/sched/cputime.h>
      10             : #include <linux/slab.h>
      11             : #include <linux/taskstats.h>
      12             : #include <linux/time.h>
      13             : #include <linux/sysctl.h>
      14             : #include <linux/delayacct.h>
      15             : #include <linux/module.h>
      16             : 
      17             : int delayacct_on __read_mostly = 1;     /* Delay accounting turned on/off */
      18             : EXPORT_SYMBOL_GPL(delayacct_on);
      19             : struct kmem_cache *delayacct_cache;
      20             : 
      21           0 : static int __init delayacct_setup_disable(char *str)
      22             : {
      23           0 :         delayacct_on = 0;
      24           0 :         return 1;
      25             : }
      26             : __setup("nodelayacct", delayacct_setup_disable);
      27             : 
      28           1 : void delayacct_init(void)
      29             : {
      30           1 :         delayacct_cache = KMEM_CACHE(task_delay_info, SLAB_PANIC|SLAB_ACCOUNT);
      31           1 :         delayacct_tsk_init(&init_task);
      32           1 : }
      33             : 
      34        1185 : void __delayacct_tsk_init(struct task_struct *tsk)
      35             : {
      36        1185 :         tsk->delays = kmem_cache_zalloc(delayacct_cache, GFP_KERNEL);
      37        1185 :         if (tsk->delays)
      38        1185 :                 raw_spin_lock_init(&tsk->delays->lock);
      39        1185 : }
      40             : 
      41             : /*
      42             :  * Finish delay accounting for a statistic using its timestamps (@start),
      43             :  * accumalator (@total) and @count
      44             :  */
      45        1876 : static void delayacct_end(raw_spinlock_t *lock, u64 *start, u64 *total,
      46             :                           u32 *count)
      47             : {
      48        1876 :         s64 ns = ktime_get_ns() - *start;
      49        1876 :         unsigned long flags;
      50             : 
      51        1876 :         if (ns > 0) {
      52        1876 :                 raw_spin_lock_irqsave(lock, flags);
      53        1876 :                 *total += ns;
      54        1876 :                 (*count)++;
      55        1876 :                 raw_spin_unlock_irqrestore(lock, flags);
      56             :         }
      57        1875 : }
      58             : 
      59        1876 : void __delayacct_blkio_start(void)
      60             : {
      61        1876 :         current->delays->blkio_start = ktime_get_ns();
      62        1876 : }
      63             : 
      64             : /*
      65             :  * We cannot rely on the `current` macro, as we haven't yet switched back to
      66             :  * the process being woken.
      67             :  */
      68        1876 : void __delayacct_blkio_end(struct task_struct *p)
      69             : {
      70        1876 :         struct task_delay_info *delays = p->delays;
      71        1876 :         u64 *total;
      72        1876 :         u32 *count;
      73             : 
      74        1876 :         if (p->delays->flags & DELAYACCT_PF_SWAPIN) {
      75           0 :                 total = &delays->swapin_delay;
      76           0 :                 count = &delays->swapin_count;
      77             :         } else {
      78        1876 :                 total = &delays->blkio_delay;
      79        1876 :                 count = &delays->blkio_count;
      80             :         }
      81             : 
      82        1876 :         delayacct_end(&delays->lock, &delays->blkio_start, total, count);
      83        1875 : }
      84             : 
      85           2 : int __delayacct_add_tsk(struct taskstats *d, struct task_struct *tsk)
      86             : {
      87           2 :         u64 utime, stime, stimescaled, utimescaled;
      88           2 :         unsigned long long t2, t3;
      89           2 :         unsigned long flags, t1;
      90           2 :         s64 tmp;
      91             : 
      92           2 :         task_cputime(tsk, &utime, &stime);
      93           2 :         tmp = (s64)d->cpu_run_real_total;
      94           2 :         tmp += utime + stime;
      95           2 :         d->cpu_run_real_total = (tmp < (s64)d->cpu_run_real_total) ? 0 : tmp;
      96             : 
      97           2 :         task_cputime_scaled(tsk, &utimescaled, &stimescaled);
      98           2 :         tmp = (s64)d->cpu_scaled_run_real_total;
      99           2 :         tmp += utimescaled + stimescaled;
     100           4 :         d->cpu_scaled_run_real_total =
     101           2 :                 (tmp < (s64)d->cpu_scaled_run_real_total) ? 0 : tmp;
     102             : 
     103             :         /*
     104             :          * No locking available for sched_info (and too expensive to add one)
     105             :          * Mitigate by taking snapshot of values
     106             :          */
     107           2 :         t1 = tsk->sched_info.pcount;
     108           2 :         t2 = tsk->sched_info.run_delay;
     109           2 :         t3 = tsk->se.sum_exec_runtime;
     110             : 
     111           2 :         d->cpu_count += t1;
     112             : 
     113           2 :         tmp = (s64)d->cpu_delay_total + t2;
     114           2 :         d->cpu_delay_total = (tmp < (s64)d->cpu_delay_total) ? 0 : tmp;
     115             : 
     116           2 :         tmp = (s64)d->cpu_run_virtual_total + t3;
     117           4 :         d->cpu_run_virtual_total =
     118           2 :                 (tmp < (s64)d->cpu_run_virtual_total) ?   0 : tmp;
     119             : 
     120             :         /* zero XXX_total, non-zero XXX_count implies XXX stat overflowed */
     121             : 
     122           2 :         raw_spin_lock_irqsave(&tsk->delays->lock, flags);
     123           2 :         tmp = d->blkio_delay_total + tsk->delays->blkio_delay;
     124           2 :         d->blkio_delay_total = (tmp < d->blkio_delay_total) ? 0 : tmp;
     125           2 :         tmp = d->swapin_delay_total + tsk->delays->swapin_delay;
     126           2 :         d->swapin_delay_total = (tmp < d->swapin_delay_total) ? 0 : tmp;
     127           2 :         tmp = d->freepages_delay_total + tsk->delays->freepages_delay;
     128           2 :         d->freepages_delay_total = (tmp < d->freepages_delay_total) ? 0 : tmp;
     129           2 :         tmp = d->thrashing_delay_total + tsk->delays->thrashing_delay;
     130           2 :         d->thrashing_delay_total = (tmp < d->thrashing_delay_total) ? 0 : tmp;
     131           2 :         d->blkio_count += tsk->delays->blkio_count;
     132           2 :         d->swapin_count += tsk->delays->swapin_count;
     133           2 :         d->freepages_count += tsk->delays->freepages_count;
     134           2 :         d->thrashing_count += tsk->delays->thrashing_count;
     135           2 :         raw_spin_unlock_irqrestore(&tsk->delays->lock, flags);
     136             : 
     137           2 :         return 0;
     138             : }
     139             : 
     140         139 : __u64 __delayacct_blkio_ticks(struct task_struct *tsk)
     141             : {
     142         139 :         __u64 ret;
     143         139 :         unsigned long flags;
     144             : 
     145         139 :         raw_spin_lock_irqsave(&tsk->delays->lock, flags);
     146         278 :         ret = nsec_to_clock_t(tsk->delays->blkio_delay +
     147         139 :                                 tsk->delays->swapin_delay);
     148         139 :         raw_spin_unlock_irqrestore(&tsk->delays->lock, flags);
     149         139 :         return ret;
     150             : }
     151             : 
     152           0 : void __delayacct_freepages_start(void)
     153             : {
     154           0 :         current->delays->freepages_start = ktime_get_ns();
     155           0 : }
     156             : 
     157           0 : void __delayacct_freepages_end(void)
     158             : {
     159           0 :         delayacct_end(
     160           0 :                 &current->delays->lock,
     161           0 :                 &current->delays->freepages_start,
     162           0 :                 &current->delays->freepages_delay,
     163           0 :                 &current->delays->freepages_count);
     164           0 : }
     165             : 
     166           0 : void __delayacct_thrashing_start(void)
     167             : {
     168           0 :         current->delays->thrashing_start = ktime_get_ns();
     169           0 : }
     170             : 
     171           0 : void __delayacct_thrashing_end(void)
     172             : {
     173           0 :         delayacct_end(&current->delays->lock,
     174           0 :                       &current->delays->thrashing_start,
     175           0 :                       &current->delays->thrashing_delay,
     176           0 :                       &current->delays->thrashing_count);
     177           0 : }