Line data Source code
1 : // SPDX-License-Identifier: GPL-2.0-only
2 : /*
3 : * User address space access functions.
4 : *
5 : * Copyright 1997 Andi Kleen <ak@muc.de>
6 : * Copyright 1997 Linus Torvalds
7 : * Copyright 2002 Andi Kleen <ak@suse.de>
8 : */
9 : #include <linux/export.h>
10 : #include <linux/uaccess.h>
11 : #include <linux/highmem.h>
12 :
13 : /*
14 : * Zero Userspace
15 : */
16 :
17 7049 : unsigned long __clear_user(void __user *addr, unsigned long size)
18 : {
19 7049 : long __d0;
20 7049 : might_fault();
21 : /* no memory constraint because it doesn't change any memory gcc knows
22 : about */
23 7049 : stac();
24 14098 : asm volatile(
25 : " testq %[size8],%[size8]\n"
26 : " jz 4f\n"
27 : " .align 16\n"
28 : "0: movq $0,(%[dst])\n"
29 : " addq $8,%[dst]\n"
30 : " decl %%ecx ; jnz 0b\n"
31 : "4: movq %[size1],%%rcx\n"
32 : " testl %%ecx,%%ecx\n"
33 : " jz 2f\n"
34 : "1: movb $0,(%[dst])\n"
35 : " incq %[dst]\n"
36 : " decl %%ecx ; jnz 1b\n"
37 : "2:\n"
38 : ".section .fixup,\"ax\"\n"
39 : "3: lea 0(%[size1],%[size8],8),%[size8]\n"
40 : " jmp 2b\n"
41 : ".previous\n"
42 : _ASM_EXTABLE_UA(0b, 3b)
43 : _ASM_EXTABLE_UA(1b, 2b)
44 : : [size8] "=&c"(size), [dst] "=&D" (__d0)
45 7049 : : [size1] "r"(size & 7), "[size8]" (size / 8), "[dst]"(addr));
46 7049 : clac();
47 7049 : return size;
48 : }
49 : EXPORT_SYMBOL(__clear_user);
50 :
51 4908 : unsigned long clear_user(void __user *to, unsigned long n)
52 : {
53 9816 : if (access_ok(to, n))
54 4908 : return __clear_user(to, n);
55 : return n;
56 : }
57 : EXPORT_SYMBOL(clear_user);
58 :
59 : #ifdef CONFIG_ARCH_HAS_UACCESS_FLUSHCACHE
60 : /**
61 : * clean_cache_range - write back a cache range with CLWB
62 : * @vaddr: virtual start address
63 : * @size: number of bytes to write back
64 : *
65 : * Write back a cache range using the CLWB (cache line write back)
66 : * instruction. Note that @size is internally rounded up to be cache
67 : * line size aligned.
68 : */
69 0 : static void clean_cache_range(void *addr, size_t size)
70 : {
71 0 : u16 x86_clflush_size = boot_cpu_data.x86_clflush_size;
72 0 : unsigned long clflush_mask = x86_clflush_size - 1;
73 0 : void *vend = addr + size;
74 0 : void *p;
75 :
76 0 : for (p = (void *)((unsigned long)addr & ~clflush_mask);
77 0 : p < vend; p += x86_clflush_size)
78 0 : clwb(p);
79 0 : }
80 :
81 0 : void arch_wb_cache_pmem(void *addr, size_t size)
82 : {
83 0 : clean_cache_range(addr, size);
84 0 : }
85 : EXPORT_SYMBOL_GPL(arch_wb_cache_pmem);
86 :
87 0 : long __copy_user_flushcache(void *dst, const void __user *src, unsigned size)
88 : {
89 0 : unsigned long flushed, dest = (unsigned long) dst;
90 0 : long rc = __copy_user_nocache(dst, src, size, 0);
91 :
92 : /*
93 : * __copy_user_nocache() uses non-temporal stores for the bulk
94 : * of the transfer, but we need to manually flush if the
95 : * transfer is unaligned. A cached memory copy is used when
96 : * destination or size is not naturally aligned. That is:
97 : * - Require 8-byte alignment when size is 8 bytes or larger.
98 : * - Require 4-byte alignment when size is 4 bytes.
99 : */
100 0 : if (size < 8) {
101 0 : if (!IS_ALIGNED(dest, 4) || size != 4)
102 0 : clean_cache_range(dst, size);
103 : } else {
104 0 : if (!IS_ALIGNED(dest, 8)) {
105 0 : dest = ALIGN(dest, boot_cpu_data.x86_clflush_size);
106 0 : clean_cache_range(dst, 1);
107 : }
108 :
109 0 : flushed = dest - (unsigned long) dst;
110 0 : if (size > flushed && !IS_ALIGNED(size - flushed, 8))
111 0 : clean_cache_range(dst + size - 1, 1);
112 : }
113 :
114 0 : return rc;
115 : }
116 :
117 0 : void __memcpy_flushcache(void *_dst, const void *_src, size_t size)
118 : {
119 0 : unsigned long dest = (unsigned long) _dst;
120 0 : unsigned long source = (unsigned long) _src;
121 :
122 : /* cache copy and flush to align dest */
123 0 : if (!IS_ALIGNED(dest, 8)) {
124 0 : unsigned len = min_t(unsigned, size, ALIGN(dest, 8) - dest);
125 :
126 0 : memcpy((void *) dest, (void *) source, len);
127 0 : clean_cache_range((void *) dest, len);
128 0 : dest += len;
129 0 : source += len;
130 0 : size -= len;
131 0 : if (!size)
132 : return;
133 : }
134 :
135 : /* 4x8 movnti loop */
136 0 : while (size >= 32) {
137 0 : asm("movq (%0), %%r8\n"
138 : "movq 8(%0), %%r9\n"
139 : "movq 16(%0), %%r10\n"
140 : "movq 24(%0), %%r11\n"
141 : "movnti %%r8, (%1)\n"
142 : "movnti %%r9, 8(%1)\n"
143 : "movnti %%r10, 16(%1)\n"
144 : "movnti %%r11, 24(%1)\n"
145 : :: "r" (source), "r" (dest)
146 : : "memory", "r8", "r9", "r10", "r11");
147 0 : dest += 32;
148 0 : source += 32;
149 0 : size -= 32;
150 : }
151 :
152 : /* 1x8 movnti loop */
153 0 : while (size >= 8) {
154 0 : asm("movq (%0), %%r8\n"
155 : "movnti %%r8, (%1)\n"
156 : :: "r" (source), "r" (dest)
157 : : "memory", "r8");
158 0 : dest += 8;
159 0 : source += 8;
160 0 : size -= 8;
161 : }
162 :
163 : /* 1x4 movnti loop */
164 0 : while (size >= 4) {
165 0 : asm("movl (%0), %%r8d\n"
166 : "movnti %%r8d, (%1)\n"
167 : :: "r" (source), "r" (dest)
168 : : "memory", "r8");
169 0 : dest += 4;
170 0 : source += 4;
171 0 : size -= 4;
172 : }
173 :
174 : /* cache copy for remaining bytes */
175 0 : if (size) {
176 0 : memcpy((void *) dest, (void *) source, size);
177 0 : clean_cache_range((void *) dest, size);
178 : }
179 : }
180 : EXPORT_SYMBOL_GPL(__memcpy_flushcache);
181 :
182 0 : void memcpy_page_flushcache(char *to, struct page *page, size_t offset,
183 : size_t len)
184 : {
185 0 : char *from = kmap_atomic(page);
186 :
187 0 : memcpy_flushcache(to, from + offset, len);
188 0 : kunmap_atomic(from);
189 0 : }
190 : #endif
|
