LCOV - code coverage report
Current view: top level - arch/x86/lib - usercopy_64.c (source / functions) Hit Total Coverage
Test: landlock.info Lines: 11 70 15.7 %
Date: 2021-04-22 12:43:58 Functions: 2 7 28.6 %

          Line data    Source code
       1             : // SPDX-License-Identifier: GPL-2.0-only
       2             : /* 
       3             :  * User address space access functions.
       4             :  *
       5             :  * Copyright 1997 Andi Kleen <ak@muc.de>
       6             :  * Copyright 1997 Linus Torvalds
       7             :  * Copyright 2002 Andi Kleen <ak@suse.de>
       8             :  */
       9             : #include <linux/export.h>
      10             : #include <linux/uaccess.h>
      11             : #include <linux/highmem.h>
      12             : 
      13             : /*
      14             :  * Zero Userspace
      15             :  */
      16             : 
      17        7049 : unsigned long __clear_user(void __user *addr, unsigned long size)
      18             : {
      19        7049 :         long __d0;
      20        7049 :         might_fault();
      21             :         /* no memory constraint because it doesn't change any memory gcc knows
      22             :            about */
      23        7049 :         stac();
      24       14098 :         asm volatile(
      25             :                 "  testq  %[size8],%[size8]\n"
      26             :                 "  jz     4f\n"
      27             :                 "  .align 16\n"
      28             :                 "0:        movq $0,(%[dst])\n"
      29             :                 "  addq   $8,%[dst]\n"
      30             :                 "  decl %%ecx ; jnz   0b\n"
      31             :                 "4:        movq  %[size1],%%rcx\n"
      32             :                 "  testl %%ecx,%%ecx\n"
      33             :                 "  jz     2f\n"
      34             :                 "1:        movb   $0,(%[dst])\n"
      35             :                 "  incq   %[dst]\n"
      36             :                 "  decl %%ecx ; jnz  1b\n"
      37             :                 "2:\n"
      38             :                 ".section .fixup,\"ax\"\n"
      39             :                 "3:        lea 0(%[size1],%[size8],8),%[size8]\n"
      40             :                 "  jmp 2b\n"
      41             :                 ".previous\n"
      42             :                 _ASM_EXTABLE_UA(0b, 3b)
      43             :                 _ASM_EXTABLE_UA(1b, 2b)
      44             :                 : [size8] "=&c"(size), [dst] "=&D" (__d0)
      45        7049 :                 : [size1] "r"(size & 7), "[size8]" (size / 8), "[dst]"(addr));
      46        7049 :         clac();
      47        7049 :         return size;
      48             : }
      49             : EXPORT_SYMBOL(__clear_user);
      50             : 
      51        4908 : unsigned long clear_user(void __user *to, unsigned long n)
      52             : {
      53        9816 :         if (access_ok(to, n))
      54        4908 :                 return __clear_user(to, n);
      55             :         return n;
      56             : }
      57             : EXPORT_SYMBOL(clear_user);
      58             : 
      59             : #ifdef CONFIG_ARCH_HAS_UACCESS_FLUSHCACHE
      60             : /**
      61             :  * clean_cache_range - write back a cache range with CLWB
      62             :  * @vaddr:      virtual start address
      63             :  * @size:       number of bytes to write back
      64             :  *
      65             :  * Write back a cache range using the CLWB (cache line write back)
      66             :  * instruction. Note that @size is internally rounded up to be cache
      67             :  * line size aligned.
      68             :  */
      69           0 : static void clean_cache_range(void *addr, size_t size)
      70             : {
      71           0 :         u16 x86_clflush_size = boot_cpu_data.x86_clflush_size;
      72           0 :         unsigned long clflush_mask = x86_clflush_size - 1;
      73           0 :         void *vend = addr + size;
      74           0 :         void *p;
      75             : 
      76           0 :         for (p = (void *)((unsigned long)addr & ~clflush_mask);
      77           0 :              p < vend; p += x86_clflush_size)
      78           0 :                 clwb(p);
      79           0 : }
      80             : 
      81           0 : void arch_wb_cache_pmem(void *addr, size_t size)
      82             : {
      83           0 :         clean_cache_range(addr, size);
      84           0 : }
      85             : EXPORT_SYMBOL_GPL(arch_wb_cache_pmem);
      86             : 
      87           0 : long __copy_user_flushcache(void *dst, const void __user *src, unsigned size)
      88             : {
      89           0 :         unsigned long flushed, dest = (unsigned long) dst;
      90           0 :         long rc = __copy_user_nocache(dst, src, size, 0);
      91             : 
      92             :         /*
      93             :          * __copy_user_nocache() uses non-temporal stores for the bulk
      94             :          * of the transfer, but we need to manually flush if the
      95             :          * transfer is unaligned. A cached memory copy is used when
      96             :          * destination or size is not naturally aligned. That is:
      97             :          *   - Require 8-byte alignment when size is 8 bytes or larger.
      98             :          *   - Require 4-byte alignment when size is 4 bytes.
      99             :          */
     100           0 :         if (size < 8) {
     101           0 :                 if (!IS_ALIGNED(dest, 4) || size != 4)
     102           0 :                         clean_cache_range(dst, size);
     103             :         } else {
     104           0 :                 if (!IS_ALIGNED(dest, 8)) {
     105           0 :                         dest = ALIGN(dest, boot_cpu_data.x86_clflush_size);
     106           0 :                         clean_cache_range(dst, 1);
     107             :                 }
     108             : 
     109           0 :                 flushed = dest - (unsigned long) dst;
     110           0 :                 if (size > flushed && !IS_ALIGNED(size - flushed, 8))
     111           0 :                         clean_cache_range(dst + size - 1, 1);
     112             :         }
     113             : 
     114           0 :         return rc;
     115             : }
     116             : 
     117           0 : void __memcpy_flushcache(void *_dst, const void *_src, size_t size)
     118             : {
     119           0 :         unsigned long dest = (unsigned long) _dst;
     120           0 :         unsigned long source = (unsigned long) _src;
     121             : 
     122             :         /* cache copy and flush to align dest */
     123           0 :         if (!IS_ALIGNED(dest, 8)) {
     124           0 :                 unsigned len = min_t(unsigned, size, ALIGN(dest, 8) - dest);
     125             : 
     126           0 :                 memcpy((void *) dest, (void *) source, len);
     127           0 :                 clean_cache_range((void *) dest, len);
     128           0 :                 dest += len;
     129           0 :                 source += len;
     130           0 :                 size -= len;
     131           0 :                 if (!size)
     132             :                         return;
     133             :         }
     134             : 
     135             :         /* 4x8 movnti loop */
     136           0 :         while (size >= 32) {
     137           0 :                 asm("movq    (%0), %%r8\n"
     138             :                     "movq   8(%0), %%r9\n"
     139             :                     "movq  16(%0), %%r10\n"
     140             :                     "movq  24(%0), %%r11\n"
     141             :                     "movnti  %%r8,   (%1)\n"
     142             :                     "movnti  %%r9,  8(%1)\n"
     143             :                     "movnti %%r10, 16(%1)\n"
     144             :                     "movnti %%r11, 24(%1)\n"
     145             :                     :: "r" (source), "r" (dest)
     146             :                     : "memory", "r8", "r9", "r10", "r11");
     147           0 :                 dest += 32;
     148           0 :                 source += 32;
     149           0 :                 size -= 32;
     150             :         }
     151             : 
     152             :         /* 1x8 movnti loop */
     153           0 :         while (size >= 8) {
     154           0 :                 asm("movq    (%0), %%r8\n"
     155             :                     "movnti  %%r8,   (%1)\n"
     156             :                     :: "r" (source), "r" (dest)
     157             :                     : "memory", "r8");
     158           0 :                 dest += 8;
     159           0 :                 source += 8;
     160           0 :                 size -= 8;
     161             :         }
     162             : 
     163             :         /* 1x4 movnti loop */
     164           0 :         while (size >= 4) {
     165           0 :                 asm("movl    (%0), %%r8d\n"
     166             :                     "movnti  %%r8d,   (%1)\n"
     167             :                     :: "r" (source), "r" (dest)
     168             :                     : "memory", "r8");
     169           0 :                 dest += 4;
     170           0 :                 source += 4;
     171           0 :                 size -= 4;
     172             :         }
     173             : 
     174             :         /* cache copy for remaining bytes */
     175           0 :         if (size) {
     176           0 :                 memcpy((void *) dest, (void *) source, size);
     177           0 :                 clean_cache_range((void *) dest, size);
     178             :         }
     179             : }
     180             : EXPORT_SYMBOL_GPL(__memcpy_flushcache);
     181             : 
     182           0 : void memcpy_page_flushcache(char *to, struct page *page, size_t offset,
     183             :                 size_t len)
     184             : {
     185           0 :         char *from = kmap_atomic(page);
     186             : 
     187           0 :         memcpy_flushcache(to, from + offset, len);
     188           0 :         kunmap_atomic(from);
     189           0 : }
     190             : #endif

Generated by: LCOV version 1.14