Line data    Source code

       1             : // SPDX-License-Identifier: GPL-2.0
       2             : /* Copyright(c) 2016-2020 Intel Corporation. All rights reserved. */
       3             : 
       4             : #include <linux/jump_label.h>
       5             : #include <linux/uaccess.h>
       6             : #include <linux/export.h>
       7             : #include <linux/string.h>
       8             : #include <linux/types.h>
       9             : 
      10             : #include <asm/mce.h>
      11             : 
      12             : #ifdef CONFIG_X86_MCE
      13             : static DEFINE_STATIC_KEY_FALSE(copy_mc_fragile_key);
      14             : 
      15             : void enable_copy_mc_fragile(void)
      16             : {
      17             :         static_branch_inc(&copy_mc_fragile_key);
      18             : }
      19             : #define copy_mc_fragile_enabled (static_branch_unlikely(&copy_mc_fragile_key))
      20             : 
      21             : /*
      22             :  * Similar to copy_user_handle_tail, probe for the write fault point, or
      23             :  * source exception point.
      24             :  */
      25             : __visible notrace unsigned long
      26             : copy_mc_fragile_handle_tail(char *to, char *from, unsigned len)
      27             : {
      28             :         for (; len; --len, to++, from++)
      29             :                 if (copy_mc_fragile(to, from, 1))
      30             :                         break;
      31             :         return len;
      32             : }
      33             : #else
      34             : /*
      35             :  * No point in doing careful copying, or consulting a static key when
      36             :  * there is no #MC handler in the CONFIG_X86_MCE=n case.
      37             :  */
      38           0 : void enable_copy_mc_fragile(void)
      39             : {
      40           0 : }
      41             : #define copy_mc_fragile_enabled (0)
      42             : #endif
      43             : 
      44             : unsigned long copy_mc_enhanced_fast_string(void *dst, const void *src, unsigned len);
      45             : 
      46             : /**
      47             :  * copy_mc_to_kernel - memory copy that handles source exceptions
      48             :  *
      49             :  * @dst:        destination address
      50             :  * @src:        source address
      51             :  * @len:        number of bytes to copy
      52             :  *
      53             :  * Call into the 'fragile' version on systems that benefit from avoiding
      54             :  * corner case poison consumption scenarios, For example, accessing
      55             :  * poison across 2 cachelines with a single instruction. Almost all
      56             :  * other uses case can use copy_mc_enhanced_fast_string() for a fast
      57             :  * recoverable copy, or fallback to plain memcpy.
      58             :  *
      59             :  * Return 0 for success, or number of bytes not copied if there was an
      60             :  * exception.
      61             :  */
      62           0 : unsigned long __must_check copy_mc_to_kernel(void *dst, const void *src, unsigned len)
      63             : {
      64           0 :         if (copy_mc_fragile_enabled)
      65             :                 return copy_mc_fragile(dst, src, len);
      66           0 :         if (static_cpu_has(X86_FEATURE_ERMS))
      67           0 :                 return copy_mc_enhanced_fast_string(dst, src, len);
      68           0 :         memcpy(dst, src, len);
      69           0 :         return 0;
      70             : }
      71             : EXPORT_SYMBOL_GPL(copy_mc_to_kernel);
      72             : 
      73           0 : unsigned long __must_check copy_mc_to_user(void *dst, const void *src, unsigned len)
      74             : {
      75           0 :         unsigned long ret;
      76             : 
      77           0 :         if (copy_mc_fragile_enabled) {
      78             :                 __uaccess_begin();
      79             :                 ret = copy_mc_fragile(dst, src, len);
      80             :                 __uaccess_end();
      81             :                 return ret;
      82             :         }
      83             : 
      84           0 :         if (static_cpu_has(X86_FEATURE_ERMS)) {
      85           0 :                 __uaccess_begin();
      86           0 :                 ret = copy_mc_enhanced_fast_string(dst, src, len);
      87           0 :                 __uaccess_end();
      88           0 :                 return ret;
      89             :         }
      90             : 
      91           0 :         return copy_user_generic(dst, src, len);
      92             : }