Line data    Source code

       1             : // SPDX-License-Identifier: GPL-2.0-or-later
       2             : /* Rewritten by Rusty Russell, on the backs of many others...
       3             :    Copyright (C) 2001 Rusty Russell, 2002 Rusty Russell IBM.
       4             : 
       5             : */
       6             : #include <linux/ftrace.h>
       7             : #include <linux/memory.h>
       8             : #include <linux/extable.h>
       9             : #include <linux/module.h>
      10             : #include <linux/mutex.h>
      11             : #include <linux/init.h>
      12             : #include <linux/kprobes.h>
      13             : #include <linux/filter.h>
      14             : 
      15             : #include <asm/sections.h>
      16             : #include <linux/uaccess.h>
      17             : 
      18             : /*
      19             :  * mutex protecting text section modification (dynamic code patching).
      20             :  * some users need to sleep (allocating memory...) while they hold this lock.
      21             :  *
      22             :  * Note: Also protects SMP-alternatives modification on x86.
      23             :  *
      24             :  * NOT exported to modules - patching kernel text is a really delicate matter.
      25             :  */
      26             : DEFINE_MUTEX(text_mutex);
      27             : 
      28             : extern struct exception_table_entry __start___ex_table[];
      29             : extern struct exception_table_entry __stop___ex_table[];
      30             : 
      31             : /* Cleared by build time tools if the table is already sorted. */
      32             : u32 __initdata __visible main_extable_sort_needed = 1;
      33             : 
      34             : /* Sort the kernel's built-in exception table */
      35           1 : void __init sort_main_extable(void)
      36             : {
      37           1 :         if (main_extable_sort_needed &&
      38             :             &__stop___ex_table > &__start___ex_table) {
      39           0 :                 pr_notice("Sorting __ex_table...\n");
      40           0 :                 sort_extable(__start___ex_table, __stop___ex_table);
      41             :         }
      42           1 : }
      43             : 
      44             : /* Given an address, look for it in the kernel exception table */
      45             : const
      46         265 : struct exception_table_entry *search_kernel_exception_table(unsigned long addr)
      47             : {
      48         265 :         return search_extable(__start___ex_table,
      49           0 :                               __stop___ex_table - __start___ex_table, addr);
      50             : }
      51             : 
      52             : /* Given an address, look for it in the exception tables. */
      53         265 : const struct exception_table_entry *search_exception_tables(unsigned long addr)
      54             : {
      55         265 :         const struct exception_table_entry *e;
      56             : 
      57         265 :         e = search_kernel_exception_table(addr);
      58         265 :         if (!e)
      59         265 :                 e = search_module_extables(addr);
      60             :         if (!e)
      61         265 :                 e = search_bpf_extables(addr);
      62         265 :         return e;
      63             : }
      64             : 
      65     3698414 : int init_kernel_text(unsigned long addr)
      66             : {
      67     1064070 :         if (addr >= (unsigned long)_sinittext &&
      68     1242863 :             addr < (unsigned long)_einittext)
      69      618695 :                 return 1;
      70             :         return 0;
      71             : }
      72             : 
      73    57390949 : int notrace core_kernel_text(unsigned long addr)
      74             : {
      75    57390949 :         if (addr >= (unsigned long)_stext &&
      76    54895239 :             addr < (unsigned long)_etext)
      77             :                 return 1;
      78             : 
      79     3079574 :         if (system_state < SYSTEM_RUNNING &&
      80    57836324 :             init_kernel_text(addr))
      81      445375 :                 return 1;
      82             :         return 0;
      83             : }
      84             : 
      85             : /**
      86             :  * core_kernel_data - tell if addr points to kernel data
      87             :  * @addr: address to test
      88             :  *
      89             :  * Returns true if @addr passed in is from the core kernel data
      90             :  * section.
      91             :  *
      92             :  * Note: On some archs it may return true for core RODATA, and false
      93             :  *  for others. But will always be true for core RW data.
      94             :  */
      95           0 : int core_kernel_data(unsigned long addr)
      96             : {
      97           0 :         if (addr >= (unsigned long)_sdata &&
      98           0 :             addr < (unsigned long)_edata)
      99           0 :                 return 1;
     100             :         return 0;
     101             : }
     102             : 
     103    57174119 : int __kernel_text_address(unsigned long addr)
     104             : {
     105    57174119 :         if (kernel_text_address(addr))
     106             :                 return 1;
     107             :         /*
     108             :          * There might be init symbols in saved stacktraces.
     109             :          * Give those symbols a chance to be printed in
     110             :          * backtraces (such as lockdep traces).
     111             :          *
     112             :          * Since we are after the module-symbols check, there's
     113             :          * no danger of address overlap:
     114             :          */
     115     2634344 :         if (init_kernel_text(addr))
     116      178792 :                 return 1;
     117             :         return 0;
     118             : }
     119             : 
     120    57125758 : int kernel_text_address(unsigned long addr)
     121             : {
     122    57125758 :         bool no_rcu;
     123    57125758 :         int ret = 1;
     124             : 
     125    57125758 :         if (core_kernel_text(addr))
     126             :                 return 1;
     127             : 
     128             :         /*
     129             :          * If a stack dump happens while RCU is not watching, then
     130             :          * RCU needs to be notified that it requires to start
     131             :          * watching again. This can happen either by tracing that
     132             :          * triggers a stack trace, or a WARN() that happens during
     133             :          * coming back from idle, or cpu on or offlining.
     134             :          *
     135             :          * is_module_text_address() as well as the kprobe slots,
     136             :          * is_bpf_text_address() and is_bpf_image_address require
     137             :          * RCU to be watching.
     138             :          */
     139     2634202 :         no_rcu = !rcu_is_watching();
     140             : 
     141             :         /* Treat this like an NMI as it can happen anywhere */
     142     2634183 :         if (no_rcu)
     143           0 :                 rcu_nmi_enter();
     144             : 
     145     2634183 :         if (is_module_text_address(addr))
     146             :                 goto out;
     147     2634183 :         if (is_ftrace_trampoline(addr))
     148             :                 goto out;
     149     2634183 :         if (is_kprobe_optinsn_slot(addr) || is_kprobe_insn_slot(addr))
     150             :                 goto out;
     151     2634183 :         if (is_bpf_text_address(addr))
     152             :                 goto out;
     153     2634183 :         ret = 0;
     154     2634183 : out:
     155     2634183 :         if (no_rcu)
     156           0 :                 rcu_nmi_exit();
     157             : 
     158             :         return ret;
     159             : }
     160             : 
     161             : /*
     162             :  * On some architectures (PPC64, IA64) function pointers
     163             :  * are actually only tokens to some data that then holds the
     164             :  * real function address. As a result, to find if a function
     165             :  * pointer is part of the kernel text, we need to do some
     166             :  * special dereferencing first.
     167             :  */
     168           0 : int func_ptr_is_kernel_text(void *ptr)
     169             : {
     170           0 :         unsigned long addr;
     171           0 :         addr = (unsigned long) dereference_function_descriptor(ptr);
     172           0 :         if (core_kernel_text(addr))
     173           0 :                 return 1;
     174           0 :         return is_module_text_address(addr);
     175             : }