Line data    Source code

       1             : /* SPDX-License-Identifier: GPL-2.0 */
       2             : /*
       3             :  * Kernel Electric-Fence (KFENCE). Public interface for allocator and fault
       4             :  * handler integration. For more info see Documentation/dev-tools/kfence.rst.
       5             :  *
       6             :  * Copyright (C) 2020, Google LLC.
       7             :  */
       8             : 
       9             : #ifndef _LINUX_KFENCE_H
      10             : #define _LINUX_KFENCE_H
      11             : 
      12             : #include <linux/mm.h>
      13             : #include <linux/types.h>
      14             : 
      15             : #ifdef CONFIG_KFENCE
      16             : 
      17             : /*
      18             :  * We allocate an even number of pages, as it simplifies calculations to map
      19             :  * address to metadata indices; effectively, the very first page serves as an
      20             :  * extended guard page, but otherwise has no special purpose.
      21             :  */
      22             : #define KFENCE_POOL_SIZE ((CONFIG_KFENCE_NUM_OBJECTS + 1) * 2 * PAGE_SIZE)
      23             : extern char *__kfence_pool;
      24             : 
      25             : #ifdef CONFIG_KFENCE_STATIC_KEYS
      26             : #include <linux/static_key.h>
      27             : DECLARE_STATIC_KEY_FALSE(kfence_allocation_key);
      28             : #else
      29             : #include <linux/atomic.h>
      30             : extern atomic_t kfence_allocation_gate;
      31             : #endif
      32             : 
      33             : /**
      34             :  * is_kfence_address() - check if an address belongs to KFENCE pool
      35             :  * @addr: address to check
      36             :  *
      37             :  * Return: true or false depending on whether the address is within the KFENCE
      38             :  * object range.
      39             :  *
      40             :  * KFENCE objects live in a separate page range and are not to be intermixed
      41             :  * with regular heap objects (e.g. KFENCE objects must never be added to the
      42             :  * allocator freelists). Failing to do so may and will result in heap
      43             :  * corruptions, therefore is_kfence_address() must be used to check whether
      44             :  * an object requires specific handling.
      45             :  *
      46             :  * Note: This function may be used in fast-paths, and is performance critical.
      47             :  * Future changes should take this into account; for instance, we want to avoid
      48             :  * introducing another load and therefore need to keep KFENCE_POOL_SIZE a
      49             :  * constant (until immediate patching support is added to the kernel).
      50             :  */
      51             : static __always_inline bool is_kfence_address(const void *addr)
      52             : {
      53             :         /*
      54             :          * The non-NULL check is required in case the __kfence_pool pointer was
      55             :          * never initialized; keep it in the slow-path after the range-check.
      56             :          */
      57             :         return unlikely((unsigned long)((char *)addr - __kfence_pool) < KFENCE_POOL_SIZE && addr);
      58             : }
      59             : 
      60             : /**
      61             :  * kfence_alloc_pool() - allocate the KFENCE pool via memblock
      62             :  */
      63             : void __init kfence_alloc_pool(void);
      64             : 
      65             : /**
      66             :  * kfence_init() - perform KFENCE initialization at boot time
      67             :  *
      68             :  * Requires that kfence_alloc_pool() was called before. This sets up the
      69             :  * allocation gate timer, and requires that workqueues are available.
      70             :  */
      71             : void __init kfence_init(void);
      72             : 
      73             : /**
      74             :  * kfence_shutdown_cache() - handle shutdown_cache() for KFENCE objects
      75             :  * @s: cache being shut down
      76             :  *
      77             :  * Before shutting down a cache, one must ensure there are no remaining objects
      78             :  * allocated from it. Because KFENCE objects are not referenced from the cache
      79             :  * directly, we need to check them here.
      80             :  *
      81             :  * Note that shutdown_cache() is internal to SL*B, and kmem_cache_destroy() does
      82             :  * not return if allocated objects still exist: it prints an error message and
      83             :  * simply aborts destruction of a cache, leaking memory.
      84             :  *
      85             :  * If the only such objects are KFENCE objects, we will not leak the entire
      86             :  * cache, but instead try to provide more useful debug info by making allocated
      87             :  * objects "zombie allocations". Objects may then still be used or freed (which
      88             :  * is handled gracefully), but usage will result in showing KFENCE error reports
      89             :  * which include stack traces to the user of the object, the original allocation
      90             :  * site, and caller to shutdown_cache().
      91             :  */
      92             : void kfence_shutdown_cache(struct kmem_cache *s);
      93             : 
      94             : /*
      95             :  * Allocate a KFENCE object. Allocators must not call this function directly,
      96             :  * use kfence_alloc() instead.
      97             :  */
      98             : void *__kfence_alloc(struct kmem_cache *s, size_t size, gfp_t flags);
      99             : 
     100             : /**
     101             :  * kfence_alloc() - allocate a KFENCE object with a low probability
     102             :  * @s:     struct kmem_cache with object requirements
     103             :  * @size:  exact size of the object to allocate (can be less than @s->size
     104             :  *         e.g. for kmalloc caches)
     105             :  * @flags: GFP flags
     106             :  *
     107             :  * Return:
     108             :  * * NULL     - must proceed with allocating as usual,
     109             :  * * non-NULL - pointer to a KFENCE object.
     110             :  *
     111             :  * kfence_alloc() should be inserted into the heap allocation fast path,
     112             :  * allowing it to transparently return KFENCE-allocated objects with a low
     113             :  * probability using a static branch (the probability is controlled by the
     114             :  * kfence.sample_interval boot parameter).
     115             :  */
     116             : static __always_inline void *kfence_alloc(struct kmem_cache *s, size_t size, gfp_t flags)
     117             : {
     118             : #ifdef CONFIG_KFENCE_STATIC_KEYS
     119             :         if (static_branch_unlikely(&kfence_allocation_key))
     120             : #else
     121             :         if (unlikely(!atomic_read(&kfence_allocation_gate)))
     122             : #endif
     123             :                 return __kfence_alloc(s, size, flags);
     124             :         return NULL;
     125             : }
     126             : 
     127             : /**
     128             :  * kfence_ksize() - get actual amount of memory allocated for a KFENCE object
     129             :  * @addr: pointer to a heap object
     130             :  *
     131             :  * Return:
     132             :  * * 0     - not a KFENCE object, must call __ksize() instead,
     133             :  * * non-0 - this many bytes can be accessed without causing a memory error.
     134             :  *
     135             :  * kfence_ksize() returns the number of bytes requested for a KFENCE object at
     136             :  * allocation time. This number may be less than the object size of the
     137             :  * corresponding struct kmem_cache.
     138             :  */
     139             : size_t kfence_ksize(const void *addr);
     140             : 
     141             : /**
     142             :  * kfence_object_start() - find the beginning of a KFENCE object
     143             :  * @addr: address within a KFENCE-allocated object
     144             :  *
     145             :  * Return: address of the beginning of the object.
     146             :  *
     147             :  * SL[AU]B-allocated objects are laid out within a page one by one, so it is
     148             :  * easy to calculate the beginning of an object given a pointer inside it and
     149             :  * the object size. The same is not true for KFENCE, which places a single
     150             :  * object at either end of the page. This helper function is used to find the
     151             :  * beginning of a KFENCE-allocated object.
     152             :  */
     153             : void *kfence_object_start(const void *addr);
     154             : 
     155             : /**
     156             :  * __kfence_free() - release a KFENCE heap object to KFENCE pool
     157             :  * @addr: object to be freed
     158             :  *
     159             :  * Requires: is_kfence_address(addr)
     160             :  *
     161             :  * Release a KFENCE object and mark it as freed.
     162             :  */
     163             : void __kfence_free(void *addr);
     164             : 
     165             : /**
     166             :  * kfence_free() - try to release an arbitrary heap object to KFENCE pool
     167             :  * @addr: object to be freed
     168             :  *
     169             :  * Return:
     170             :  * * false - object doesn't belong to KFENCE pool and was ignored,
     171             :  * * true  - object was released to KFENCE pool.
     172             :  *
     173             :  * Release a KFENCE object and mark it as freed. May be called on any object,
     174             :  * even non-KFENCE objects, to simplify integration of the hooks into the
     175             :  * allocator's free codepath. The allocator must check the return value to
     176             :  * determine if it was a KFENCE object or not.
     177             :  */
     178             : static __always_inline __must_check bool kfence_free(void *addr)
     179             : {
     180             :         if (!is_kfence_address(addr))
     181             :                 return false;
     182             :         __kfence_free(addr);
     183             :         return true;
     184             : }
     185             : 
     186             : /**
     187             :  * kfence_handle_page_fault() - perform page fault handling for KFENCE pages
     188             :  * @addr: faulting address
     189             :  * @is_write: is access a write
     190             :  * @regs: current struct pt_regs (can be NULL, but shows full stack trace)
     191             :  *
     192             :  * Return:
     193             :  * * false - address outside KFENCE pool,
     194             :  * * true  - page fault handled by KFENCE, no additional handling required.
     195             :  *
     196             :  * A page fault inside KFENCE pool indicates a memory error, such as an
     197             :  * out-of-bounds access, a use-after-free or an invalid memory access. In these
     198             :  * cases KFENCE prints an error message and marks the offending page as
     199             :  * present, so that the kernel can proceed.
     200             :  */
     201             : bool __must_check kfence_handle_page_fault(unsigned long addr, bool is_write, struct pt_regs *regs);
     202             : 
     203             : #else /* CONFIG_KFENCE */
     204             : 
     205    10232524 : static inline bool is_kfence_address(const void *addr) { return false; }
     206           1 : static inline void kfence_alloc_pool(void) { }
     207             : static inline void kfence_init(void) { }
     208           0 : static inline void kfence_shutdown_cache(struct kmem_cache *s) { }
     209     1471092 : static inline void *kfence_alloc(struct kmem_cache *s, size_t size, gfp_t flags) { return NULL; }
     210       10081 : static inline size_t kfence_ksize(const void *addr) { return 0; }
     211             : static inline void *kfence_object_start(const void *addr) { return NULL; }
     212             : static inline void __kfence_free(void *addr) { }
     213     1235789 : static inline bool __must_check kfence_free(void *addr) { return false; }
     214             : static inline bool __must_check kfence_handle_page_fault(unsigned long addr, bool is_write,
     215             :                                                          struct pt_regs *regs)
     216             : {
     217             :         return false;
     218             : }
     219             : 
     220             : #endif
     221             : 
     222             : #endif /* _LINUX_KFENCE_H */