Line data Source code
1 : // SPDX-License-Identifier: GPL-2.0-only
2 : /*
3 : * Access kernel or user memory without faulting.
4 : */
5 : #include <linux/export.h>
6 : #include <linux/mm.h>
7 : #include <linux/uaccess.h>
8 :
9 0 : bool __weak copy_from_kernel_nofault_allowed(const void *unsafe_src,
10 : size_t size)
11 : {
12 0 : return true;
13 : }
14 :
15 : #ifdef HAVE_GET_KERNEL_NOFAULT
16 :
17 : #define copy_from_kernel_nofault_loop(dst, src, len, type, err_label) \
18 : while (len >= sizeof(type)) { \
19 : __get_kernel_nofault(dst, src, type, err_label); \
20 : dst += sizeof(type); \
21 : src += sizeof(type); \
22 : len -= sizeof(type); \
23 : }
24 :
25 98 : long copy_from_kernel_nofault(void *dst, const void *src, size_t size)
26 : {
27 98 : if (!copy_from_kernel_nofault_allowed(src, size))
28 : return -ERANGE;
29 :
30 98 : pagefault_disable();
31 106 : copy_from_kernel_nofault_loop(dst, src, size, u64, Efault);
32 98 : copy_from_kernel_nofault_loop(dst, src, size, u32, Efault);
33 195 : copy_from_kernel_nofault_loop(dst, src, size, u16, Efault);
34 98 : copy_from_kernel_nofault_loop(dst, src, size, u8, Efault);
35 98 : pagefault_enable();
36 98 : return 0;
37 0 : Efault:
38 0 : pagefault_enable();
39 0 : return -EFAULT;
40 : }
41 : EXPORT_SYMBOL_GPL(copy_from_kernel_nofault);
42 :
43 : #define copy_to_kernel_nofault_loop(dst, src, len, type, err_label) \
44 : while (len >= sizeof(type)) { \
45 : __put_kernel_nofault(dst, src, type, err_label); \
46 : dst += sizeof(type); \
47 : src += sizeof(type); \
48 : len -= sizeof(type); \
49 : }
50 :
51 0 : long copy_to_kernel_nofault(void *dst, const void *src, size_t size)
52 : {
53 0 : pagefault_disable();
54 0 : copy_to_kernel_nofault_loop(dst, src, size, u64, Efault);
55 0 : copy_to_kernel_nofault_loop(dst, src, size, u32, Efault);
56 0 : copy_to_kernel_nofault_loop(dst, src, size, u16, Efault);
57 0 : copy_to_kernel_nofault_loop(dst, src, size, u8, Efault);
58 0 : pagefault_enable();
59 0 : return 0;
60 0 : Efault:
61 0 : pagefault_enable();
62 0 : return -EFAULT;
63 : }
64 :
65 0 : long strncpy_from_kernel_nofault(char *dst, const void *unsafe_addr, long count)
66 : {
67 0 : const void *src = unsafe_addr;
68 :
69 0 : if (unlikely(count <= 0))
70 : return 0;
71 0 : if (!copy_from_kernel_nofault_allowed(unsafe_addr, count))
72 : return -ERANGE;
73 :
74 0 : pagefault_disable();
75 0 : do {
76 0 : __get_kernel_nofault(dst, src, u8, Efault);
77 0 : dst++;
78 0 : src++;
79 0 : } while (dst[-1] && src - unsafe_addr < count);
80 0 : pagefault_enable();
81 :
82 0 : dst[-1] = '\0';
83 0 : return src - unsafe_addr;
84 0 : Efault:
85 0 : pagefault_enable();
86 0 : dst[-1] = '\0';
87 0 : return -EFAULT;
88 : }
89 : #else /* HAVE_GET_KERNEL_NOFAULT */
90 : /**
91 : * copy_from_kernel_nofault(): safely attempt to read from kernel-space
92 : * @dst: pointer to the buffer that shall take the data
93 : * @src: address to read from
94 : * @size: size of the data chunk
95 : *
96 : * Safely read from kernel address @src to the buffer at @dst. If a kernel
97 : * fault happens, handle that and return -EFAULT. If @src is not a valid kernel
98 : * address, return -ERANGE.
99 : *
100 : * We ensure that the copy_from_user is executed in atomic context so that
101 : * do_page_fault() doesn't attempt to take mmap_lock. This makes
102 : * copy_from_kernel_nofault() suitable for use within regions where the caller
103 : * already holds mmap_lock, or other locks which nest inside mmap_lock.
104 : */
105 : long copy_from_kernel_nofault(void *dst, const void *src, size_t size)
106 : {
107 : long ret;
108 : mm_segment_t old_fs = get_fs();
109 :
110 : if (!copy_from_kernel_nofault_allowed(src, size))
111 : return -ERANGE;
112 :
113 : set_fs(KERNEL_DS);
114 : pagefault_disable();
115 : ret = __copy_from_user_inatomic(dst, (__force const void __user *)src,
116 : size);
117 : pagefault_enable();
118 : set_fs(old_fs);
119 :
120 : if (ret)
121 : return -EFAULT;
122 : return 0;
123 : }
124 : EXPORT_SYMBOL_GPL(copy_from_kernel_nofault);
125 :
126 : /**
127 : * copy_to_kernel_nofault(): safely attempt to write to a location
128 : * @dst: address to write to
129 : * @src: pointer to the data that shall be written
130 : * @size: size of the data chunk
131 : *
132 : * Safely write to address @dst from the buffer at @src. If a kernel fault
133 : * happens, handle that and return -EFAULT.
134 : */
135 : long copy_to_kernel_nofault(void *dst, const void *src, size_t size)
136 : {
137 : long ret;
138 : mm_segment_t old_fs = get_fs();
139 :
140 : set_fs(KERNEL_DS);
141 : pagefault_disable();
142 : ret = __copy_to_user_inatomic((__force void __user *)dst, src, size);
143 : pagefault_enable();
144 : set_fs(old_fs);
145 :
146 : if (ret)
147 : return -EFAULT;
148 : return 0;
149 : }
150 :
151 : /**
152 : * strncpy_from_kernel_nofault: - Copy a NUL terminated string from unsafe
153 : * address.
154 : * @dst: Destination address, in kernel space. This buffer must be at
155 : * least @count bytes long.
156 : * @unsafe_addr: Unsafe address.
157 : * @count: Maximum number of bytes to copy, including the trailing NUL.
158 : *
159 : * Copies a NUL-terminated string from unsafe address to kernel buffer.
160 : *
161 : * On success, returns the length of the string INCLUDING the trailing NUL.
162 : *
163 : * If access fails, returns -EFAULT (some data may have been copied and the
164 : * trailing NUL added). If @unsafe_addr is not a valid kernel address, return
165 : * -ERANGE.
166 : *
167 : * If @count is smaller than the length of the string, copies @count-1 bytes,
168 : * sets the last byte of @dst buffer to NUL and returns @count.
169 : */
170 : long strncpy_from_kernel_nofault(char *dst, const void *unsafe_addr, long count)
171 : {
172 : mm_segment_t old_fs = get_fs();
173 : const void *src = unsafe_addr;
174 : long ret;
175 :
176 : if (unlikely(count <= 0))
177 : return 0;
178 : if (!copy_from_kernel_nofault_allowed(unsafe_addr, count))
179 : return -ERANGE;
180 :
181 : set_fs(KERNEL_DS);
182 : pagefault_disable();
183 :
184 : do {
185 : ret = __get_user(*dst++, (const char __user __force *)src++);
186 : } while (dst[-1] && ret == 0 && src - unsafe_addr < count);
187 :
188 : dst[-1] = '\0';
189 : pagefault_enable();
190 : set_fs(old_fs);
191 :
192 : return ret ? -EFAULT : src - unsafe_addr;
193 : }
194 : #endif /* HAVE_GET_KERNEL_NOFAULT */
195 :
196 : /**
197 : * copy_from_user_nofault(): safely attempt to read from a user-space location
198 : * @dst: pointer to the buffer that shall take the data
199 : * @src: address to read from. This must be a user address.
200 : * @size: size of the data chunk
201 : *
202 : * Safely read from user address @src to the buffer at @dst. If a kernel fault
203 : * happens, handle that and return -EFAULT.
204 : */
205 0 : long copy_from_user_nofault(void *dst, const void __user *src, size_t size)
206 : {
207 0 : long ret = -EFAULT;
208 0 : mm_segment_t old_fs = force_uaccess_begin();
209 :
210 0 : if (access_ok(src, size)) {
211 0 : pagefault_disable();
212 0 : ret = __copy_from_user_inatomic(dst, src, size);
213 0 : pagefault_enable();
214 : }
215 0 : force_uaccess_end(old_fs);
216 :
217 0 : if (ret)
218 0 : return -EFAULT;
219 : return 0;
220 : }
221 : EXPORT_SYMBOL_GPL(copy_from_user_nofault);
222 :
223 : /**
224 : * copy_to_user_nofault(): safely attempt to write to a user-space location
225 : * @dst: address to write to
226 : * @src: pointer to the data that shall be written
227 : * @size: size of the data chunk
228 : *
229 : * Safely write to address @dst from the buffer at @src. If a kernel fault
230 : * happens, handle that and return -EFAULT.
231 : */
232 0 : long copy_to_user_nofault(void __user *dst, const void *src, size_t size)
233 : {
234 0 : long ret = -EFAULT;
235 0 : mm_segment_t old_fs = force_uaccess_begin();
236 :
237 0 : if (access_ok(dst, size)) {
238 0 : pagefault_disable();
239 0 : ret = __copy_to_user_inatomic(dst, src, size);
240 0 : pagefault_enable();
241 : }
242 0 : force_uaccess_end(old_fs);
243 :
244 0 : if (ret)
245 0 : return -EFAULT;
246 : return 0;
247 : }
248 : EXPORT_SYMBOL_GPL(copy_to_user_nofault);
249 :
250 : /**
251 : * strncpy_from_user_nofault: - Copy a NUL terminated string from unsafe user
252 : * address.
253 : * @dst: Destination address, in kernel space. This buffer must be at
254 : * least @count bytes long.
255 : * @unsafe_addr: Unsafe user address.
256 : * @count: Maximum number of bytes to copy, including the trailing NUL.
257 : *
258 : * Copies a NUL-terminated string from unsafe user address to kernel buffer.
259 : *
260 : * On success, returns the length of the string INCLUDING the trailing NUL.
261 : *
262 : * If access fails, returns -EFAULT (some data may have been copied
263 : * and the trailing NUL added).
264 : *
265 : * If @count is smaller than the length of the string, copies @count-1 bytes,
266 : * sets the last byte of @dst buffer to NUL and returns @count.
267 : */
268 0 : long strncpy_from_user_nofault(char *dst, const void __user *unsafe_addr,
269 : long count)
270 : {
271 0 : mm_segment_t old_fs;
272 0 : long ret;
273 :
274 0 : if (unlikely(count <= 0))
275 : return 0;
276 :
277 0 : old_fs = force_uaccess_begin();
278 0 : pagefault_disable();
279 0 : ret = strncpy_from_user(dst, unsafe_addr, count);
280 0 : pagefault_enable();
281 0 : force_uaccess_end(old_fs);
282 :
283 0 : if (ret >= count) {
284 0 : ret = count;
285 0 : dst[ret - 1] = '\0';
286 0 : } else if (ret > 0) {
287 0 : ret++;
288 : }
289 :
290 : return ret;
291 : }
292 :
293 : /**
294 : * strnlen_user_nofault: - Get the size of a user string INCLUDING final NUL.
295 : * @unsafe_addr: The string to measure.
296 : * @count: Maximum count (including NUL)
297 : *
298 : * Get the size of a NUL-terminated string in user space without pagefault.
299 : *
300 : * Returns the size of the string INCLUDING the terminating NUL.
301 : *
302 : * If the string is too long, returns a number larger than @count. User
303 : * has to check the return value against "> count".
304 : * On exception (or invalid count), returns 0.
305 : *
306 : * Unlike strnlen_user, this can be used from IRQ handler etc. because
307 : * it disables pagefaults.
308 : */
309 0 : long strnlen_user_nofault(const void __user *unsafe_addr, long count)
310 : {
311 0 : mm_segment_t old_fs;
312 0 : int ret;
313 :
314 0 : old_fs = force_uaccess_begin();
315 0 : pagefault_disable();
316 0 : ret = strnlen_user(unsafe_addr, count);
317 0 : pagefault_enable();
318 0 : force_uaccess_end(old_fs);
319 :
320 0 : return ret;
321 : }
|
