Line data    Source code

       1             : // SPDX-License-Identifier: GPL-2.0
       2             : /*
       3             :  * preemptoff and irqoff tracepoints
       4             :  *
       5             :  * Copyright (C) Joel Fernandes (Google) <joel@joelfernandes.org>
       6             :  */
       7             : 
       8             : #include <linux/kallsyms.h>
       9             : #include <linux/uaccess.h>
      10             : #include <linux/module.h>
      11             : #include <linux/ftrace.h>
      12             : #include <linux/kprobes.h>
      13             : #include "trace.h"
      14             : 
      15             : #define CREATE_TRACE_POINTS
      16             : #include <trace/events/preemptirq.h>
      17             : 
      18             : #ifdef CONFIG_TRACE_IRQFLAGS
      19             : /* Per-cpu variable to prevent redundant calls when IRQs already off */
      20             : static DEFINE_PER_CPU(int, tracing_irq_cpu);
      21             : 
      22             : /*
      23             :  * Like trace_hardirqs_on() but without the lockdep invocation. This is
      24             :  * used in the low level entry code where the ordering vs. RCU is important
      25             :  * and lockdep uses a staged approach which splits the lockdep hardirq
      26             :  * tracking into a RCU on and a RCU off section.
      27             :  */
      28      500019 : void trace_hardirqs_on_prepare(void)
      29             : {
      30      500019 :         if (this_cpu_read(tracing_irq_cpu)) {
      31      500079 :                 if (!in_nmi())
      32      500090 :                         trace_irq_enable(CALLER_ADDR0, CALLER_ADDR1);
      33      500114 :                 tracer_hardirqs_on(CALLER_ADDR0, CALLER_ADDR1);
      34      500128 :                 this_cpu_write(tracing_irq_cpu, 0);
      35             :         }
      36      500130 : }
      37             : EXPORT_SYMBOL(trace_hardirqs_on_prepare);
      38             : NOKPROBE_SYMBOL(trace_hardirqs_on_prepare);
      39             : 
      40    12450595 : void trace_hardirqs_on(void)
      41             : {
      42    12450595 :         if (this_cpu_read(tracing_irq_cpu)) {
      43    12456436 :                 if (!in_nmi())
      44    12452698 :                         trace_irq_enable_rcuidle(CALLER_ADDR0, CALLER_ADDR1);
      45    12438235 :                 tracer_hardirqs_on(CALLER_ADDR0, CALLER_ADDR1);
      46    12437961 :                 this_cpu_write(tracing_irq_cpu, 0);
      47             :         }
      48             : 
      49    12439398 :         lockdep_hardirqs_on_prepare(CALLER_ADDR0);
      50    12427562 :         lockdep_hardirqs_on(CALLER_ADDR0);
      51    12422794 : }
      52             : EXPORT_SYMBOL(trace_hardirqs_on);
      53             : NOKPROBE_SYMBOL(trace_hardirqs_on);
      54             : 
      55             : /*
      56             :  * Like trace_hardirqs_off() but without the lockdep invocation. This is
      57             :  * used in the low level entry code where the ordering vs. RCU is important
      58             :  * and lockdep uses a staged approach which splits the lockdep hardirq
      59             :  * tracking into a RCU on and a RCU off section.
      60             :  */
      61      506969 : void trace_hardirqs_off_finish(void)
      62             : {
      63      506969 :         if (!this_cpu_read(tracing_irq_cpu)) {
      64      507701 :                 this_cpu_write(tracing_irq_cpu, 1);
      65      507682 :                 tracer_hardirqs_off(CALLER_ADDR0, CALLER_ADDR1);
      66      507677 :                 if (!in_nmi())
      67      507660 :                         trace_irq_disable(CALLER_ADDR0, CALLER_ADDR1);
      68             :         }
      69             : 
      70      507323 : }
      71             : EXPORT_SYMBOL(trace_hardirqs_off_finish);
      72             : NOKPROBE_SYMBOL(trace_hardirqs_off_finish);
      73             : 
      74    12377762 : void trace_hardirqs_off(void)
      75             : {
      76    12377762 :         lockdep_hardirqs_off(CALLER_ADDR0);
      77             : 
      78    12390156 :         if (!this_cpu_read(tracing_irq_cpu)) {
      79    12394366 :                 this_cpu_write(tracing_irq_cpu, 1);
      80    12395783 :                 tracer_hardirqs_off(CALLER_ADDR0, CALLER_ADDR1);
      81    12396675 :                 if (!in_nmi())
      82    12400490 :                         trace_irq_disable_rcuidle(CALLER_ADDR0, CALLER_ADDR1);
      83             :         }
      84    12398722 : }
      85             : EXPORT_SYMBOL(trace_hardirqs_off);
      86             : NOKPROBE_SYMBOL(trace_hardirqs_off);
      87             : 
      88           0 : __visible void trace_hardirqs_on_caller(unsigned long caller_addr)
      89             : {
      90           0 :         if (this_cpu_read(tracing_irq_cpu)) {
      91           0 :                 if (!in_nmi())
      92           0 :                         trace_irq_enable_rcuidle(CALLER_ADDR0, caller_addr);
      93           0 :                 tracer_hardirqs_on(CALLER_ADDR0, caller_addr);
      94           0 :                 this_cpu_write(tracing_irq_cpu, 0);
      95             :         }
      96             : 
      97           0 :         lockdep_hardirqs_on_prepare(CALLER_ADDR0);
      98           0 :         lockdep_hardirqs_on(CALLER_ADDR0);
      99           0 : }
     100             : EXPORT_SYMBOL(trace_hardirqs_on_caller);
     101             : NOKPROBE_SYMBOL(trace_hardirqs_on_caller);
     102             : 
     103           0 : __visible void trace_hardirqs_off_caller(unsigned long caller_addr)
     104             : {
     105           0 :         lockdep_hardirqs_off(CALLER_ADDR0);
     106             : 
     107           0 :         if (!this_cpu_read(tracing_irq_cpu)) {
     108           0 :                 this_cpu_write(tracing_irq_cpu, 1);
     109           0 :                 tracer_hardirqs_off(CALLER_ADDR0, caller_addr);
     110           0 :                 if (!in_nmi())
     111           0 :                         trace_irq_disable_rcuidle(CALLER_ADDR0, caller_addr);
     112             :         }
     113           0 : }
     114             : EXPORT_SYMBOL(trace_hardirqs_off_caller);
     115             : NOKPROBE_SYMBOL(trace_hardirqs_off_caller);
     116             : #endif /* CONFIG_TRACE_IRQFLAGS */
     117             : 
     118             : #ifdef CONFIG_TRACE_PREEMPT_TOGGLE
     119             : 
     120             : void trace_preempt_on(unsigned long a0, unsigned long a1)
     121             : {
     122             :         if (!in_nmi())
     123             :                 trace_preempt_enable_rcuidle(a0, a1);
     124             :         tracer_preempt_on(a0, a1);
     125             : }
     126             : 
     127             : void trace_preempt_off(unsigned long a0, unsigned long a1)
     128             : {
     129             :         if (!in_nmi())
     130             :                 trace_preempt_disable_rcuidle(a0, a1);
     131             :         tracer_preempt_off(a0, a1);
     132             : }
     133             : #endif