Line data Source code
1 : // SPDX-License-Identifier: GPL-2.0-only
2 : #include <crypto/hash.h>
3 : #include <linux/export.h>
4 : #include <linux/bvec.h>
5 : #include <linux/fault-inject-usercopy.h>
6 : #include <linux/uio.h>
7 : #include <linux/pagemap.h>
8 : #include <linux/slab.h>
9 : #include <linux/vmalloc.h>
10 : #include <linux/splice.h>
11 : #include <linux/compat.h>
12 : #include <net/checksum.h>
13 : #include <linux/scatterlist.h>
14 : #include <linux/instrumented.h>
15 :
16 : #define PIPE_PARANOIA /* for now */
17 :
18 : #define iterate_iovec(i, n, __v, __p, skip, STEP) { \
19 : size_t left; \
20 : size_t wanted = n; \
21 : __p = i->iov; \
22 : __v.iov_len = min(n, __p->iov_len - skip); \
23 : if (likely(__v.iov_len)) { \
24 : __v.iov_base = __p->iov_base + skip; \
25 : left = (STEP); \
26 : __v.iov_len -= left; \
27 : skip += __v.iov_len; \
28 : n -= __v.iov_len; \
29 : } else { \
30 : left = 0; \
31 : } \
32 : while (unlikely(!left && n)) { \
33 : __p++; \
34 : __v.iov_len = min(n, __p->iov_len); \
35 : if (unlikely(!__v.iov_len)) \
36 : continue; \
37 : __v.iov_base = __p->iov_base; \
38 : left = (STEP); \
39 : __v.iov_len -= left; \
40 : skip = __v.iov_len; \
41 : n -= __v.iov_len; \
42 : } \
43 : n = wanted - n; \
44 : }
45 :
46 : #define iterate_kvec(i, n, __v, __p, skip, STEP) { \
47 : size_t wanted = n; \
48 : __p = i->kvec; \
49 : __v.iov_len = min(n, __p->iov_len - skip); \
50 : if (likely(__v.iov_len)) { \
51 : __v.iov_base = __p->iov_base + skip; \
52 : (void)(STEP); \
53 : skip += __v.iov_len; \
54 : n -= __v.iov_len; \
55 : } \
56 : while (unlikely(n)) { \
57 : __p++; \
58 : __v.iov_len = min(n, __p->iov_len); \
59 : if (unlikely(!__v.iov_len)) \
60 : continue; \
61 : __v.iov_base = __p->iov_base; \
62 : (void)(STEP); \
63 : skip = __v.iov_len; \
64 : n -= __v.iov_len; \
65 : } \
66 : n = wanted; \
67 : }
68 :
69 : #define iterate_bvec(i, n, __v, __bi, skip, STEP) { \
70 : struct bvec_iter __start; \
71 : __start.bi_size = n; \
72 : __start.bi_bvec_done = skip; \
73 : __start.bi_idx = 0; \
74 : for_each_bvec(__v, i->bvec, __bi, __start) { \
75 : (void)(STEP); \
76 : } \
77 : }
78 :
79 : #define iterate_all_kinds(i, n, v, I, B, K) { \
80 : if (likely(n)) { \
81 : size_t skip = i->iov_offset; \
82 : if (unlikely(i->type & ITER_BVEC)) { \
83 : struct bio_vec v; \
84 : struct bvec_iter __bi; \
85 : iterate_bvec(i, n, v, __bi, skip, (B)) \
86 : } else if (unlikely(i->type & ITER_KVEC)) { \
87 : const struct kvec *kvec; \
88 : struct kvec v; \
89 : iterate_kvec(i, n, v, kvec, skip, (K)) \
90 : } else if (unlikely(i->type & ITER_DISCARD)) { \
91 : } else { \
92 : const struct iovec *iov; \
93 : struct iovec v; \
94 : iterate_iovec(i, n, v, iov, skip, (I)) \
95 : } \
96 : } \
97 : }
98 :
99 : #define iterate_and_advance(i, n, v, I, B, K) { \
100 : if (unlikely(i->count < n)) \
101 : n = i->count; \
102 : if (i->count) { \
103 : size_t skip = i->iov_offset; \
104 : if (unlikely(i->type & ITER_BVEC)) { \
105 : const struct bio_vec *bvec = i->bvec; \
106 : struct bio_vec v; \
107 : struct bvec_iter __bi; \
108 : iterate_bvec(i, n, v, __bi, skip, (B)) \
109 : i->bvec = __bvec_iter_bvec(i->bvec, __bi); \
110 : i->nr_segs -= i->bvec - bvec; \
111 : skip = __bi.bi_bvec_done; \
112 : } else if (unlikely(i->type & ITER_KVEC)) { \
113 : const struct kvec *kvec; \
114 : struct kvec v; \
115 : iterate_kvec(i, n, v, kvec, skip, (K)) \
116 : if (skip == kvec->iov_len) { \
117 : kvec++; \
118 : skip = 0; \
119 : } \
120 : i->nr_segs -= kvec - i->kvec; \
121 : i->kvec = kvec; \
122 : } else if (unlikely(i->type & ITER_DISCARD)) { \
123 : skip += n; \
124 : } else { \
125 : const struct iovec *iov; \
126 : struct iovec v; \
127 : iterate_iovec(i, n, v, iov, skip, (I)) \
128 : if (skip == iov->iov_len) { \
129 : iov++; \
130 : skip = 0; \
131 : } \
132 : i->nr_segs -= iov - i->iov; \
133 : i->iov = iov; \
134 : } \
135 : i->count -= n; \
136 : i->iov_offset = skip; \
137 : } \
138 : }
139 :
140 29075 : static int copyout(void __user *to, const void *from, size_t n)
141 : {
142 29075 : if (should_fail_usercopy())
143 : return n;
144 58150 : if (access_ok(to, n)) {
145 29075 : instrument_copy_to_user(to, from, n);
146 29077 : n = raw_copy_to_user(to, from, n);
147 : }
148 29077 : return n;
149 : }
150 :
151 21087 : static int copyin(void *to, const void __user *from, size_t n)
152 : {
153 21087 : if (should_fail_usercopy())
154 : return n;
155 42174 : if (access_ok(from, n)) {
156 21087 : instrument_copy_from_user(to, from, n);
157 21087 : n = raw_copy_from_user(to, from, n);
158 : }
159 21087 : return n;
160 : }
161 :
162 21344 : static size_t copy_page_to_iter_iovec(struct page *page, size_t offset, size_t bytes,
163 : struct iov_iter *i)
164 : {
165 21344 : size_t skip, copy, left, wanted;
166 21344 : const struct iovec *iov;
167 21344 : char __user *buf;
168 21344 : void *kaddr, *from;
169 :
170 21344 : if (unlikely(bytes > i->count))
171 12 : bytes = i->count;
172 :
173 21344 : if (unlikely(!bytes))
174 : return 0;
175 :
176 20680 : might_fault();
177 20679 : wanted = bytes;
178 20679 : iov = i->iov;
179 20679 : skip = i->iov_offset;
180 20679 : buf = iov->iov_base + skip;
181 20679 : copy = min(bytes, iov->iov_len - skip);
182 :
183 20679 : if (IS_ENABLED(CONFIG_HIGHMEM) && !fault_in_pages_writeable(buf, copy)) {
184 : kaddr = kmap_atomic(page);
185 : from = kaddr + offset;
186 :
187 : /* first chunk, usually the only one */
188 : left = copyout(buf, from, copy);
189 : copy -= left;
190 : skip += copy;
191 : from += copy;
192 : bytes -= copy;
193 :
194 : while (unlikely(!left && bytes)) {
195 : iov++;
196 : buf = iov->iov_base;
197 : copy = min(bytes, iov->iov_len);
198 : left = copyout(buf, from, copy);
199 : copy -= left;
200 : skip = copy;
201 : from += copy;
202 : bytes -= copy;
203 : }
204 : if (likely(!bytes)) {
205 : kunmap_atomic(kaddr);
206 : goto done;
207 : }
208 : offset = from - kaddr;
209 : buf += copy;
210 : kunmap_atomic(kaddr);
211 : copy = min(bytes, iov->iov_len - skip);
212 : }
213 : /* Too bad - revert to non-atomic kmap */
214 :
215 20679 : kaddr = kmap(page);
216 20680 : from = kaddr + offset;
217 20680 : left = copyout(buf, from, copy);
218 20679 : copy -= left;
219 20679 : skip += copy;
220 20679 : from += copy;
221 20679 : bytes -= copy;
222 20679 : while (unlikely(!left && bytes)) {
223 0 : iov++;
224 0 : buf = iov->iov_base;
225 0 : copy = min(bytes, iov->iov_len);
226 0 : left = copyout(buf, from, copy);
227 0 : copy -= left;
228 0 : skip = copy;
229 0 : from += copy;
230 0 : bytes -= copy;
231 : }
232 20679 : kunmap(page);
233 :
234 20679 : done:
235 20679 : if (skip == iov->iov_len) {
236 14965 : iov++;
237 14965 : skip = 0;
238 : }
239 20679 : i->count -= wanted - bytes;
240 20679 : i->nr_segs -= iov - i->iov;
241 20679 : i->iov = iov;
242 20679 : i->iov_offset = skip;
243 20679 : return wanted - bytes;
244 : }
245 :
246 9046 : static size_t copy_page_from_iter_iovec(struct page *page, size_t offset, size_t bytes,
247 : struct iov_iter *i)
248 : {
249 9046 : size_t skip, copy, left, wanted;
250 9046 : const struct iovec *iov;
251 9046 : char __user *buf;
252 9046 : void *kaddr, *to;
253 :
254 9046 : if (unlikely(bytes > i->count))
255 649 : bytes = i->count;
256 :
257 9046 : if (unlikely(!bytes))
258 : return 0;
259 :
260 9046 : might_fault();
261 9046 : wanted = bytes;
262 9046 : iov = i->iov;
263 9046 : skip = i->iov_offset;
264 9046 : buf = iov->iov_base + skip;
265 9046 : copy = min(bytes, iov->iov_len - skip);
266 :
267 9046 : if (IS_ENABLED(CONFIG_HIGHMEM) && !fault_in_pages_readable(buf, copy)) {
268 : kaddr = kmap_atomic(page);
269 : to = kaddr + offset;
270 :
271 : /* first chunk, usually the only one */
272 : left = copyin(to, buf, copy);
273 : copy -= left;
274 : skip += copy;
275 : to += copy;
276 : bytes -= copy;
277 :
278 : while (unlikely(!left && bytes)) {
279 : iov++;
280 : buf = iov->iov_base;
281 : copy = min(bytes, iov->iov_len);
282 : left = copyin(to, buf, copy);
283 : copy -= left;
284 : skip = copy;
285 : to += copy;
286 : bytes -= copy;
287 : }
288 : if (likely(!bytes)) {
289 : kunmap_atomic(kaddr);
290 : goto done;
291 : }
292 : offset = to - kaddr;
293 : buf += copy;
294 : kunmap_atomic(kaddr);
295 : copy = min(bytes, iov->iov_len - skip);
296 : }
297 : /* Too bad - revert to non-atomic kmap */
298 :
299 9046 : kaddr = kmap(page);
300 9046 : to = kaddr + offset;
301 9046 : left = copyin(to, buf, copy);
302 9046 : copy -= left;
303 9046 : skip += copy;
304 9046 : to += copy;
305 9046 : bytes -= copy;
306 9046 : while (unlikely(!left && bytes)) {
307 0 : iov++;
308 0 : buf = iov->iov_base;
309 0 : copy = min(bytes, iov->iov_len);
310 0 : left = copyin(to, buf, copy);
311 0 : copy -= left;
312 0 : skip = copy;
313 0 : to += copy;
314 0 : bytes -= copy;
315 : }
316 9046 : kunmap(page);
317 :
318 9046 : done:
319 9046 : if (skip == iov->iov_len) {
320 8689 : iov++;
321 8689 : skip = 0;
322 : }
323 9046 : i->count -= wanted - bytes;
324 9046 : i->nr_segs -= iov - i->iov;
325 9046 : i->iov = iov;
326 9046 : i->iov_offset = skip;
327 9046 : return wanted - bytes;
328 : }
329 :
330 : #ifdef PIPE_PARANOIA
331 576 : static bool sanity(const struct iov_iter *i)
332 : {
333 576 : struct pipe_inode_info *pipe = i->pipe;
334 576 : unsigned int p_head = pipe->head;
335 576 : unsigned int p_tail = pipe->tail;
336 576 : unsigned int p_mask = pipe->ring_size - 1;
337 576 : unsigned int p_occupancy = pipe_occupancy(p_head, p_tail);
338 576 : unsigned int i_head = i->head;
339 576 : unsigned int idx;
340 :
341 576 : if (i->iov_offset) {
342 540 : struct pipe_buffer *p;
343 540 : if (unlikely(p_occupancy == 0))
344 0 : goto Bad; // pipe must be non-empty
345 540 : if (unlikely(i_head != p_head - 1))
346 0 : goto Bad; // must be at the last buffer...
347 :
348 540 : p = &pipe->bufs[i_head & p_mask];
349 540 : if (unlikely(p->offset + p->len != i->iov_offset))
350 0 : goto Bad; // ... at the end of segment
351 : } else {
352 36 : if (i_head != p_head)
353 0 : goto Bad; // must be right after the last buffer
354 : }
355 : return true;
356 0 : Bad:
357 0 : printk(KERN_ERR "idx = %d, offset = %zd\n", i_head, i->iov_offset);
358 0 : printk(KERN_ERR "head = %d, tail = %d, buffers = %d\n",
359 : p_head, p_tail, pipe->ring_size);
360 0 : for (idx = 0; idx < pipe->ring_size; idx++)
361 0 : printk(KERN_ERR "[%p %p %d %d]\n",
362 : pipe->bufs[idx].ops,
363 : pipe->bufs[idx].page,
364 : pipe->bufs[idx].offset,
365 0 : pipe->bufs[idx].len);
366 0 : WARN_ON(1);
367 0 : return false;
368 : }
369 : #else
370 : #define sanity(i) true
371 : #endif
372 :
373 609 : static size_t copy_page_to_iter_pipe(struct page *page, size_t offset, size_t bytes,
374 : struct iov_iter *i)
375 : {
376 609 : struct pipe_inode_info *pipe = i->pipe;
377 609 : struct pipe_buffer *buf;
378 609 : unsigned int p_tail = pipe->tail;
379 609 : unsigned int p_mask = pipe->ring_size - 1;
380 609 : unsigned int i_head = i->head;
381 609 : size_t off;
382 :
383 609 : if (unlikely(bytes > i->count))
384 0 : bytes = i->count;
385 :
386 609 : if (unlikely(!bytes))
387 : return 0;
388 :
389 576 : if (!sanity(i))
390 : return 0;
391 :
392 576 : off = i->iov_offset;
393 576 : buf = &pipe->bufs[i_head & p_mask];
394 576 : if (off) {
395 540 : if (offset == off && buf->page == page) {
396 : /* merge with the last one */
397 0 : buf->len += bytes;
398 0 : i->iov_offset += bytes;
399 0 : goto out;
400 : }
401 540 : i_head++;
402 540 : buf = &pipe->bufs[i_head & p_mask];
403 : }
404 576 : if (pipe_full(i_head, p_tail, pipe->max_usage))
405 : return 0;
406 :
407 576 : buf->ops = &page_cache_pipe_buf_ops;
408 576 : get_page(page);
409 576 : buf->page = page;
410 576 : buf->offset = offset;
411 576 : buf->len = bytes;
412 :
413 576 : pipe->head = i_head + 1;
414 576 : i->iov_offset = offset + bytes;
415 576 : i->head = i_head;
416 576 : out:
417 576 : i->count -= bytes;
418 576 : return bytes;
419 : }
420 :
421 : /*
422 : * Fault in one or more iovecs of the given iov_iter, to a maximum length of
423 : * bytes. For each iovec, fault in each page that constitutes the iovec.
424 : *
425 : * Return 0 on success, or non-zero if the memory could not be accessed (i.e.
426 : * because it is an invalid address).
427 : */
428 3512 : int iov_iter_fault_in_readable(struct iov_iter *i, size_t bytes)
429 : {
430 3512 : size_t skip = i->iov_offset;
431 3512 : const struct iovec *iov;
432 3512 : int err;
433 3512 : struct iovec v;
434 :
435 3512 : if (!(i->type & (ITER_BVEC|ITER_KVEC))) {
436 5872 : iterate_iovec(i, bytes, v, iov, skip, ({
437 : err = fault_in_pages_readable(v.iov_base, v.iov_len);
438 : if (unlikely(err))
439 : return err;
440 : 0;}))
441 : }
442 : return 0;
443 : }
444 : EXPORT_SYMBOL(iov_iter_fault_in_readable);
445 :
446 46221 : void iov_iter_init(struct iov_iter *i, unsigned int direction,
447 : const struct iovec *iov, unsigned long nr_segs,
448 : size_t count)
449 : {
450 46221 : WARN_ON(direction & ~(READ | WRITE));
451 46221 : direction &= READ | WRITE;
452 :
453 : /* It will get better. Eventually... */
454 46221 : if (uaccess_kernel()) {
455 : i->type = ITER_KVEC | direction;
456 : i->kvec = (struct kvec *)iov;
457 : } else {
458 46221 : i->type = ITER_IOVEC | direction;
459 46221 : i->iov = iov;
460 : }
461 46221 : i->nr_segs = nr_segs;
462 46221 : i->iov_offset = 0;
463 46221 : i->count = count;
464 46221 : }
465 : EXPORT_SYMBOL(iov_iter_init);
466 :
467 0 : static void memzero_page(struct page *page, size_t offset, size_t len)
468 : {
469 0 : char *addr = kmap_atomic(page);
470 0 : memset(addr + offset, 0, len);
471 0 : kunmap_atomic(addr);
472 0 : }
473 :
474 0 : static inline bool allocated(struct pipe_buffer *buf)
475 : {
476 0 : return buf->ops == &default_pipe_buf_ops;
477 : }
478 :
479 0 : static inline void data_start(const struct iov_iter *i,
480 : unsigned int *iter_headp, size_t *offp)
481 : {
482 0 : unsigned int p_mask = i->pipe->ring_size - 1;
483 0 : unsigned int iter_head = i->head;
484 0 : size_t off = i->iov_offset;
485 :
486 0 : if (off && (!allocated(&i->pipe->bufs[iter_head & p_mask]) ||
487 : off == PAGE_SIZE)) {
488 0 : iter_head++;
489 0 : off = 0;
490 : }
491 0 : *iter_headp = iter_head;
492 0 : *offp = off;
493 0 : }
494 :
495 0 : static size_t push_pipe(struct iov_iter *i, size_t size,
496 : int *iter_headp, size_t *offp)
497 : {
498 0 : struct pipe_inode_info *pipe = i->pipe;
499 0 : unsigned int p_tail = pipe->tail;
500 0 : unsigned int p_mask = pipe->ring_size - 1;
501 0 : unsigned int iter_head;
502 0 : size_t off;
503 0 : ssize_t left;
504 :
505 0 : if (unlikely(size > i->count))
506 0 : size = i->count;
507 0 : if (unlikely(!size))
508 : return 0;
509 :
510 0 : left = size;
511 0 : data_start(i, &iter_head, &off);
512 0 : *iter_headp = iter_head;
513 0 : *offp = off;
514 0 : if (off) {
515 0 : left -= PAGE_SIZE - off;
516 0 : if (left <= 0) {
517 0 : pipe->bufs[iter_head & p_mask].len += size;
518 0 : return size;
519 : }
520 0 : pipe->bufs[iter_head & p_mask].len = PAGE_SIZE;
521 0 : iter_head++;
522 : }
523 0 : while (!pipe_full(iter_head, p_tail, pipe->max_usage)) {
524 0 : struct pipe_buffer *buf = &pipe->bufs[iter_head & p_mask];
525 0 : struct page *page = alloc_page(GFP_USER);
526 0 : if (!page)
527 : break;
528 :
529 0 : buf->ops = &default_pipe_buf_ops;
530 0 : buf->page = page;
531 0 : buf->offset = 0;
532 0 : buf->len = min_t(ssize_t, left, PAGE_SIZE);
533 0 : left -= buf->len;
534 0 : iter_head++;
535 0 : pipe->head = iter_head;
536 :
537 0 : if (left == 0)
538 : return size;
539 : }
540 0 : return size - left;
541 : }
542 :
543 0 : static size_t copy_pipe_to_iter(const void *addr, size_t bytes,
544 : struct iov_iter *i)
545 : {
546 0 : struct pipe_inode_info *pipe = i->pipe;
547 0 : unsigned int p_mask = pipe->ring_size - 1;
548 0 : unsigned int i_head;
549 0 : size_t n, off;
550 :
551 0 : if (!sanity(i))
552 : return 0;
553 :
554 0 : bytes = n = push_pipe(i, bytes, &i_head, &off);
555 0 : if (unlikely(!n))
556 : return 0;
557 0 : do {
558 0 : size_t chunk = min_t(size_t, n, PAGE_SIZE - off);
559 0 : memcpy_to_page(pipe->bufs[i_head & p_mask].page, off, addr, chunk);
560 0 : i->head = i_head;
561 0 : i->iov_offset = off + chunk;
562 0 : n -= chunk;
563 0 : addr += chunk;
564 0 : off = 0;
565 0 : i_head++;
566 0 : } while (n);
567 0 : i->count -= bytes;
568 0 : return bytes;
569 : }
570 :
571 0 : static __wsum csum_and_memcpy(void *to, const void *from, size_t len,
572 : __wsum sum, size_t off)
573 : {
574 0 : __wsum next = csum_partial_copy_nocheck(from, to, len);
575 0 : return csum_block_add(sum, next, off);
576 : }
577 :
578 0 : static size_t csum_and_copy_to_pipe_iter(const void *addr, size_t bytes,
579 : struct csum_state *csstate,
580 : struct iov_iter *i)
581 : {
582 0 : struct pipe_inode_info *pipe = i->pipe;
583 0 : unsigned int p_mask = pipe->ring_size - 1;
584 0 : __wsum sum = csstate->csum;
585 0 : size_t off = csstate->off;
586 0 : unsigned int i_head;
587 0 : size_t n, r;
588 :
589 0 : if (!sanity(i))
590 : return 0;
591 :
592 0 : bytes = n = push_pipe(i, bytes, &i_head, &r);
593 0 : if (unlikely(!n))
594 : return 0;
595 0 : do {
596 0 : size_t chunk = min_t(size_t, n, PAGE_SIZE - r);
597 0 : char *p = kmap_atomic(pipe->bufs[i_head & p_mask].page);
598 0 : sum = csum_and_memcpy(p + r, addr, chunk, sum, off);
599 0 : kunmap_atomic(p);
600 0 : i->head = i_head;
601 0 : i->iov_offset = r + chunk;
602 0 : n -= chunk;
603 0 : off += chunk;
604 0 : addr += chunk;
605 0 : r = 0;
606 0 : i_head++;
607 0 : } while (n);
608 0 : i->count -= bytes;
609 0 : csstate->csum = sum;
610 0 : csstate->off = off;
611 0 : return bytes;
612 : }
613 :
614 18331 : size_t _copy_to_iter(const void *addr, size_t bytes, struct iov_iter *i)
615 : {
616 18331 : const char *from = addr;
617 18331 : if (unlikely(iov_iter_is_pipe(i)))
618 0 : return copy_pipe_to_iter(addr, bytes, i);
619 18331 : if (iter_is_iovec(i))
620 8430 : might_fault();
621 36651 : iterate_and_advance(i, bytes, v,
622 : copyout(v.iov_base, (from += v.iov_len) - v.iov_len, v.iov_len),
623 : memcpy_to_page(v.bv_page, v.bv_offset,
624 : (from += v.bv_len) - v.bv_len, v.bv_len),
625 : memcpy(v.iov_base, (from += v.iov_len) - v.iov_len, v.iov_len)
626 : )
627 :
628 : return bytes;
629 : }
630 : EXPORT_SYMBOL(_copy_to_iter);
631 :
632 : #ifdef CONFIG_ARCH_HAS_COPY_MC
633 0 : static int copyout_mc(void __user *to, const void *from, size_t n)
634 : {
635 0 : if (access_ok(to, n)) {
636 0 : instrument_copy_to_user(to, from, n);
637 0 : n = copy_mc_to_user((__force void *) to, from, n);
638 : }
639 0 : return n;
640 : }
641 :
642 0 : static unsigned long copy_mc_to_page(struct page *page, size_t offset,
643 : const char *from, size_t len)
644 : {
645 0 : unsigned long ret;
646 0 : char *to;
647 :
648 0 : to = kmap_atomic(page);
649 0 : ret = copy_mc_to_kernel(to + offset, from, len);
650 0 : kunmap_atomic(to);
651 :
652 0 : return ret;
653 : }
654 :
655 0 : static size_t copy_mc_pipe_to_iter(const void *addr, size_t bytes,
656 : struct iov_iter *i)
657 : {
658 0 : struct pipe_inode_info *pipe = i->pipe;
659 0 : unsigned int p_mask = pipe->ring_size - 1;
660 0 : unsigned int i_head;
661 0 : size_t n, off, xfer = 0;
662 :
663 0 : if (!sanity(i))
664 : return 0;
665 :
666 0 : bytes = n = push_pipe(i, bytes, &i_head, &off);
667 0 : if (unlikely(!n))
668 : return 0;
669 0 : do {
670 0 : size_t chunk = min_t(size_t, n, PAGE_SIZE - off);
671 0 : unsigned long rem;
672 :
673 0 : rem = copy_mc_to_page(pipe->bufs[i_head & p_mask].page,
674 : off, addr, chunk);
675 0 : i->head = i_head;
676 0 : i->iov_offset = off + chunk - rem;
677 0 : xfer += chunk - rem;
678 0 : if (rem)
679 : break;
680 0 : n -= chunk;
681 0 : addr += chunk;
682 0 : off = 0;
683 0 : i_head++;
684 0 : } while (n);
685 0 : i->count -= xfer;
686 0 : return xfer;
687 : }
688 :
689 : /**
690 : * _copy_mc_to_iter - copy to iter with source memory error exception handling
691 : * @addr: source kernel address
692 : * @bytes: total transfer length
693 : * @iter: destination iterator
694 : *
695 : * The pmem driver deploys this for the dax operation
696 : * (dax_copy_to_iter()) for dax reads (bypass page-cache and the
697 : * block-layer). Upon #MC read(2) aborts and returns EIO or the bytes
698 : * successfully copied.
699 : *
700 : * The main differences between this and typical _copy_to_iter().
701 : *
702 : * * Typical tail/residue handling after a fault retries the copy
703 : * byte-by-byte until the fault happens again. Re-triggering machine
704 : * checks is potentially fatal so the implementation uses source
705 : * alignment and poison alignment assumptions to avoid re-triggering
706 : * hardware exceptions.
707 : *
708 : * * ITER_KVEC, ITER_PIPE, and ITER_BVEC can return short copies.
709 : * Compare to copy_to_iter() where only ITER_IOVEC attempts might return
710 : * a short copy.
711 : */
712 0 : size_t _copy_mc_to_iter(const void *addr, size_t bytes, struct iov_iter *i)
713 : {
714 0 : const char *from = addr;
715 0 : unsigned long rem, curr_addr, s_addr = (unsigned long) addr;
716 :
717 0 : if (unlikely(iov_iter_is_pipe(i)))
718 0 : return copy_mc_pipe_to_iter(addr, bytes, i);
719 0 : if (iter_is_iovec(i))
720 0 : might_fault();
721 0 : iterate_and_advance(i, bytes, v,
722 : copyout_mc(v.iov_base, (from += v.iov_len) - v.iov_len,
723 : v.iov_len),
724 : ({
725 : rem = copy_mc_to_page(v.bv_page, v.bv_offset,
726 : (from += v.bv_len) - v.bv_len, v.bv_len);
727 : if (rem) {
728 : curr_addr = (unsigned long) from;
729 : bytes = curr_addr - s_addr - rem;
730 : return bytes;
731 : }
732 : }),
733 : ({
734 : rem = copy_mc_to_kernel(v.iov_base, (from += v.iov_len)
735 : - v.iov_len, v.iov_len);
736 : if (rem) {
737 : curr_addr = (unsigned long) from;
738 : bytes = curr_addr - s_addr - rem;
739 : return bytes;
740 : }
741 : })
742 : )
743 :
744 : return bytes;
745 : }
746 : EXPORT_SYMBOL_GPL(_copy_mc_to_iter);
747 : #endif /* CONFIG_ARCH_HAS_COPY_MC */
748 :
749 3048 : size_t _copy_from_iter(void *addr, size_t bytes, struct iov_iter *i)
750 : {
751 3048 : char *to = addr;
752 3048 : if (unlikely(iov_iter_is_pipe(i))) {
753 0 : WARN_ON(1);
754 0 : return 0;
755 : }
756 3048 : if (iter_is_iovec(i))
757 3048 : might_fault();
758 10910 : iterate_and_advance(i, bytes, v,
759 : copyin((to += v.iov_len) - v.iov_len, v.iov_base, v.iov_len),
760 : memcpy_from_page((to += v.bv_len) - v.bv_len, v.bv_page,
761 : v.bv_offset, v.bv_len),
762 : memcpy((to += v.iov_len) - v.iov_len, v.iov_base, v.iov_len)
763 : )
764 :
765 : return bytes;
766 : }
767 : EXPORT_SYMBOL(_copy_from_iter);
768 :
769 834 : bool _copy_from_iter_full(void *addr, size_t bytes, struct iov_iter *i)
770 : {
771 834 : char *to = addr;
772 834 : if (unlikely(iov_iter_is_pipe(i))) {
773 0 : WARN_ON(1);
774 0 : return false;
775 : }
776 834 : if (unlikely(i->count < bytes))
777 : return false;
778 :
779 834 : if (iter_is_iovec(i))
780 834 : might_fault();
781 2078 : iterate_all_kinds(i, bytes, v, ({
782 : if (copyin((to += v.iov_len) - v.iov_len,
783 : v.iov_base, v.iov_len))
784 : return false;
785 : 0;}),
786 : memcpy_from_page((to += v.bv_len) - v.bv_len, v.bv_page,
787 : v.bv_offset, v.bv_len),
788 : memcpy((to += v.iov_len) - v.iov_len, v.iov_base, v.iov_len)
789 : )
790 :
791 834 : iov_iter_advance(i, bytes);
792 834 : return true;
793 : }
794 : EXPORT_SYMBOL(_copy_from_iter_full);
795 :
796 0 : size_t _copy_from_iter_nocache(void *addr, size_t bytes, struct iov_iter *i)
797 : {
798 0 : char *to = addr;
799 0 : if (unlikely(iov_iter_is_pipe(i))) {
800 0 : WARN_ON(1);
801 0 : return 0;
802 : }
803 0 : iterate_and_advance(i, bytes, v,
804 : __copy_from_user_inatomic_nocache((to += v.iov_len) - v.iov_len,
805 : v.iov_base, v.iov_len),
806 : memcpy_from_page((to += v.bv_len) - v.bv_len, v.bv_page,
807 : v.bv_offset, v.bv_len),
808 : memcpy((to += v.iov_len) - v.iov_len, v.iov_base, v.iov_len)
809 : )
810 :
811 : return bytes;
812 : }
813 : EXPORT_SYMBOL(_copy_from_iter_nocache);
814 :
815 : #ifdef CONFIG_ARCH_HAS_UACCESS_FLUSHCACHE
816 : /**
817 : * _copy_from_iter_flushcache - write destination through cpu cache
818 : * @addr: destination kernel address
819 : * @bytes: total transfer length
820 : * @iter: source iterator
821 : *
822 : * The pmem driver arranges for filesystem-dax to use this facility via
823 : * dax_copy_from_iter() for ensuring that writes to persistent memory
824 : * are flushed through the CPU cache. It is differentiated from
825 : * _copy_from_iter_nocache() in that guarantees all data is flushed for
826 : * all iterator types. The _copy_from_iter_nocache() only attempts to
827 : * bypass the cache for the ITER_IOVEC case, and on some archs may use
828 : * instructions that strand dirty-data in the cache.
829 : */
830 0 : size_t _copy_from_iter_flushcache(void *addr, size_t bytes, struct iov_iter *i)
831 : {
832 0 : char *to = addr;
833 0 : if (unlikely(iov_iter_is_pipe(i))) {
834 0 : WARN_ON(1);
835 0 : return 0;
836 : }
837 0 : iterate_and_advance(i, bytes, v,
838 : __copy_from_user_flushcache((to += v.iov_len) - v.iov_len,
839 : v.iov_base, v.iov_len),
840 : memcpy_page_flushcache((to += v.bv_len) - v.bv_len, v.bv_page,
841 : v.bv_offset, v.bv_len),
842 : memcpy_flushcache((to += v.iov_len) - v.iov_len, v.iov_base,
843 : v.iov_len)
844 : )
845 :
846 : return bytes;
847 : }
848 : EXPORT_SYMBOL_GPL(_copy_from_iter_flushcache);
849 : #endif
850 :
851 0 : bool _copy_from_iter_full_nocache(void *addr, size_t bytes, struct iov_iter *i)
852 : {
853 0 : char *to = addr;
854 0 : if (unlikely(iov_iter_is_pipe(i))) {
855 0 : WARN_ON(1);
856 0 : return false;
857 : }
858 0 : if (unlikely(i->count < bytes))
859 : return false;
860 0 : iterate_all_kinds(i, bytes, v, ({
861 : if (__copy_from_user_inatomic_nocache((to += v.iov_len) - v.iov_len,
862 : v.iov_base, v.iov_len))
863 : return false;
864 : 0;}),
865 : memcpy_from_page((to += v.bv_len) - v.bv_len, v.bv_page,
866 : v.bv_offset, v.bv_len),
867 : memcpy((to += v.iov_len) - v.iov_len, v.iov_base, v.iov_len)
868 : )
869 :
870 0 : iov_iter_advance(i, bytes);
871 0 : return true;
872 : }
873 : EXPORT_SYMBOL(_copy_from_iter_full_nocache);
874 :
875 44412 : static inline bool page_copy_sane(struct page *page, size_t offset, size_t n)
876 : {
877 44412 : struct page *head;
878 44412 : size_t v = n + offset;
879 :
880 : /*
881 : * The general case needs to access the page order in order
882 : * to compute the page size.
883 : * However, we mostly deal with order-0 pages and thus can
884 : * avoid a possible cache line miss for requests that fit all
885 : * page orders.
886 : */
887 44412 : if (n <= v && v <= PAGE_SIZE)
888 : return true;
889 :
890 5 : head = compound_head(page);
891 5 : v += (page - head) << PAGE_SHIFT;
892 :
893 10 : if (likely(n <= v && v <= (page_size(head))))
894 : return true;
895 0 : WARN_ON(1);
896 : return false;
897 : }
898 :
899 31854 : size_t copy_page_to_iter(struct page *page, size_t offset, size_t bytes,
900 : struct iov_iter *i)
901 : {
902 31854 : if (unlikely(!page_copy_sane(page, offset, bytes)))
903 : return 0;
904 31854 : if (i->type & (ITER_BVEC|ITER_KVEC)) {
905 9901 : void *kaddr = kmap_atomic(page);
906 9901 : size_t wanted = copy_to_iter(kaddr + offset, bytes, i);
907 9901 : kunmap_atomic(kaddr);
908 9901 : return wanted;
909 21953 : } else if (unlikely(iov_iter_is_discard(i)))
910 : return bytes;
911 21953 : else if (likely(!iov_iter_is_pipe(i)))
912 21344 : return copy_page_to_iter_iovec(page, offset, bytes, i);
913 : else
914 609 : return copy_page_to_iter_pipe(page, offset, bytes, i);
915 : }
916 : EXPORT_SYMBOL(copy_page_to_iter);
917 :
918 9046 : size_t copy_page_from_iter(struct page *page, size_t offset, size_t bytes,
919 : struct iov_iter *i)
920 : {
921 9046 : if (unlikely(!page_copy_sane(page, offset, bytes)))
922 : return 0;
923 9046 : if (unlikely(iov_iter_is_pipe(i) || iov_iter_is_discard(i))) {
924 0 : WARN_ON(1);
925 0 : return 0;
926 : }
927 9046 : if (i->type & (ITER_BVEC|ITER_KVEC)) {
928 0 : void *kaddr = kmap_atomic(page);
929 0 : size_t wanted = _copy_from_iter(kaddr + offset, bytes, i);
930 0 : kunmap_atomic(kaddr);
931 0 : return wanted;
932 : } else
933 9046 : return copy_page_from_iter_iovec(page, offset, bytes, i);
934 : }
935 : EXPORT_SYMBOL(copy_page_from_iter);
936 :
937 0 : static size_t pipe_zero(size_t bytes, struct iov_iter *i)
938 : {
939 0 : struct pipe_inode_info *pipe = i->pipe;
940 0 : unsigned int p_mask = pipe->ring_size - 1;
941 0 : unsigned int i_head;
942 0 : size_t n, off;
943 :
944 0 : if (!sanity(i))
945 : return 0;
946 :
947 0 : bytes = n = push_pipe(i, bytes, &i_head, &off);
948 0 : if (unlikely(!n))
949 : return 0;
950 :
951 0 : do {
952 0 : size_t chunk = min_t(size_t, n, PAGE_SIZE - off);
953 0 : memzero_page(pipe->bufs[i_head & p_mask].page, off, chunk);
954 0 : i->head = i_head;
955 0 : i->iov_offset = off + chunk;
956 0 : n -= chunk;
957 0 : off = 0;
958 0 : i_head++;
959 0 : } while (n);
960 0 : i->count -= bytes;
961 0 : return bytes;
962 : }
963 :
964 0 : size_t iov_iter_zero(size_t bytes, struct iov_iter *i)
965 : {
966 0 : if (unlikely(iov_iter_is_pipe(i)))
967 0 : return pipe_zero(bytes, i);
968 0 : iterate_and_advance(i, bytes, v,
969 : clear_user(v.iov_base, v.iov_len),
970 : memzero_page(v.bv_page, v.bv_offset, v.bv_len),
971 : memset(v.iov_base, 0, v.iov_len)
972 : )
973 :
974 : return bytes;
975 : }
976 : EXPORT_SYMBOL(iov_iter_zero);
977 :
978 3512 : size_t iov_iter_copy_from_user_atomic(struct page *page,
979 : struct iov_iter *i, unsigned long offset, size_t bytes)
980 : {
981 3512 : char *kaddr = kmap_atomic(page), *p = kaddr + offset;
982 3512 : if (unlikely(!page_copy_sane(page, offset, bytes))) {
983 0 : kunmap_atomic(kaddr);
984 0 : return 0;
985 : }
986 3512 : if (unlikely(iov_iter_is_pipe(i) || iov_iter_is_discard(i))) {
987 0 : kunmap_atomic(kaddr);
988 0 : WARN_ON(1);
989 0 : return 0;
990 : }
991 7024 : iterate_all_kinds(i, bytes, v,
992 : copyin((p += v.iov_len) - v.iov_len, v.iov_base, v.iov_len),
993 : memcpy_from_page((p += v.bv_len) - v.bv_len, v.bv_page,
994 : v.bv_offset, v.bv_len),
995 : memcpy((p += v.iov_len) - v.iov_len, v.iov_base, v.iov_len)
996 : )
997 3512 : kunmap_atomic(kaddr);
998 3512 : return bytes;
999 : }
1000 : EXPORT_SYMBOL(iov_iter_copy_from_user_atomic);
1001 :
1002 0 : static inline void pipe_truncate(struct iov_iter *i)
1003 : {
1004 0 : struct pipe_inode_info *pipe = i->pipe;
1005 0 : unsigned int p_tail = pipe->tail;
1006 0 : unsigned int p_head = pipe->head;
1007 0 : unsigned int p_mask = pipe->ring_size - 1;
1008 :
1009 0 : if (!pipe_empty(p_head, p_tail)) {
1010 0 : struct pipe_buffer *buf;
1011 0 : unsigned int i_head = i->head;
1012 0 : size_t off = i->iov_offset;
1013 :
1014 0 : if (off) {
1015 0 : buf = &pipe->bufs[i_head & p_mask];
1016 0 : buf->len = off - buf->offset;
1017 0 : i_head++;
1018 : }
1019 0 : while (p_head != i_head) {
1020 0 : p_head--;
1021 0 : pipe_buf_release(pipe, &pipe->bufs[p_head & p_mask]);
1022 : }
1023 :
1024 0 : pipe->head = p_head;
1025 : }
1026 0 : }
1027 :
1028 0 : static void pipe_advance(struct iov_iter *i, size_t size)
1029 : {
1030 0 : struct pipe_inode_info *pipe = i->pipe;
1031 0 : if (unlikely(i->count < size))
1032 0 : size = i->count;
1033 0 : if (size) {
1034 0 : struct pipe_buffer *buf;
1035 0 : unsigned int p_mask = pipe->ring_size - 1;
1036 0 : unsigned int i_head = i->head;
1037 0 : size_t off = i->iov_offset, left = size;
1038 :
1039 0 : if (off) /* make it relative to the beginning of buffer */
1040 0 : left += off - pipe->bufs[i_head & p_mask].offset;
1041 0 : while (1) {
1042 0 : buf = &pipe->bufs[i_head & p_mask];
1043 0 : if (left <= buf->len)
1044 : break;
1045 0 : left -= buf->len;
1046 0 : i_head++;
1047 : }
1048 0 : i->head = i_head;
1049 0 : i->iov_offset = buf->offset + left;
1050 : }
1051 0 : i->count -= size;
1052 : /* ... and discard everything past that point */
1053 0 : pipe_truncate(i);
1054 0 : }
1055 :
1056 576 : static void iov_iter_bvec_advance(struct iov_iter *i, size_t size)
1057 : {
1058 576 : struct bvec_iter bi;
1059 :
1060 576 : bi.bi_size = i->count;
1061 576 : bi.bi_bvec_done = i->iov_offset;
1062 576 : bi.bi_idx = 0;
1063 576 : bvec_iter_advance(i->bvec, &bi, size);
1064 :
1065 576 : i->bvec += bi.bi_idx;
1066 576 : i->nr_segs -= bi.bi_idx;
1067 576 : i->count = bi.bi_size;
1068 576 : i->iov_offset = bi.bi_bvec_done;
1069 576 : }
1070 :
1071 4361 : void iov_iter_advance(struct iov_iter *i, size_t size)
1072 : {
1073 4361 : if (unlikely(iov_iter_is_pipe(i))) {
1074 0 : pipe_advance(i, size);
1075 0 : return;
1076 : }
1077 4361 : if (unlikely(iov_iter_is_discard(i))) {
1078 0 : i->count -= size;
1079 0 : return;
1080 : }
1081 4361 : if (iov_iter_is_bvec(i)) {
1082 576 : iov_iter_bvec_advance(i, size);
1083 576 : return;
1084 : }
1085 7980 : iterate_and_advance(i, size, v, 0, 0, 0)
1086 : }
1087 : EXPORT_SYMBOL(iov_iter_advance);
1088 :
1089 0 : void iov_iter_revert(struct iov_iter *i, size_t unroll)
1090 : {
1091 0 : if (!unroll)
1092 : return;
1093 0 : if (WARN_ON(unroll > MAX_RW_COUNT))
1094 : return;
1095 0 : i->count += unroll;
1096 0 : if (unlikely(iov_iter_is_pipe(i))) {
1097 0 : struct pipe_inode_info *pipe = i->pipe;
1098 0 : unsigned int p_mask = pipe->ring_size - 1;
1099 0 : unsigned int i_head = i->head;
1100 0 : size_t off = i->iov_offset;
1101 0 : while (1) {
1102 0 : struct pipe_buffer *b = &pipe->bufs[i_head & p_mask];
1103 0 : size_t n = off - b->offset;
1104 0 : if (unroll < n) {
1105 0 : off -= unroll;
1106 0 : break;
1107 : }
1108 0 : unroll -= n;
1109 0 : if (!unroll && i_head == i->start_head) {
1110 : off = 0;
1111 : break;
1112 : }
1113 0 : i_head--;
1114 0 : b = &pipe->bufs[i_head & p_mask];
1115 0 : off = b->offset + b->len;
1116 : }
1117 0 : i->iov_offset = off;
1118 0 : i->head = i_head;
1119 0 : pipe_truncate(i);
1120 0 : return;
1121 : }
1122 0 : if (unlikely(iov_iter_is_discard(i)))
1123 : return;
1124 0 : if (unroll <= i->iov_offset) {
1125 0 : i->iov_offset -= unroll;
1126 0 : return;
1127 : }
1128 0 : unroll -= i->iov_offset;
1129 0 : if (iov_iter_is_bvec(i)) {
1130 0 : const struct bio_vec *bvec = i->bvec;
1131 0 : while (1) {
1132 0 : size_t n = (--bvec)->bv_len;
1133 0 : i->nr_segs++;
1134 0 : if (unroll <= n) {
1135 0 : i->bvec = bvec;
1136 0 : i->iov_offset = n - unroll;
1137 0 : return;
1138 : }
1139 0 : unroll -= n;
1140 : }
1141 : } else { /* same logics for iovec and kvec */
1142 0 : const struct iovec *iov = i->iov;
1143 0 : while (1) {
1144 0 : size_t n = (--iov)->iov_len;
1145 0 : i->nr_segs++;
1146 0 : if (unroll <= n) {
1147 0 : i->iov = iov;
1148 0 : i->iov_offset = n - unroll;
1149 0 : return;
1150 : }
1151 0 : unroll -= n;
1152 : }
1153 : }
1154 : }
1155 : EXPORT_SYMBOL(iov_iter_revert);
1156 :
1157 : /*
1158 : * Return the count of just the current iov_iter segment.
1159 : */
1160 0 : size_t iov_iter_single_seg_count(const struct iov_iter *i)
1161 : {
1162 0 : if (unlikely(iov_iter_is_pipe(i)))
1163 0 : return i->count; // it is a silly place, anyway
1164 0 : if (i->nr_segs == 1)
1165 0 : return i->count;
1166 0 : if (unlikely(iov_iter_is_discard(i)))
1167 0 : return i->count;
1168 0 : else if (iov_iter_is_bvec(i))
1169 0 : return min(i->count, i->bvec->bv_len - i->iov_offset);
1170 : else
1171 0 : return min(i->count, i->iov->iov_len - i->iov_offset);
1172 : }
1173 : EXPORT_SYMBOL(iov_iter_single_seg_count);
1174 :
1175 9901 : void iov_iter_kvec(struct iov_iter *i, unsigned int direction,
1176 : const struct kvec *kvec, unsigned long nr_segs,
1177 : size_t count)
1178 : {
1179 9901 : WARN_ON(direction & ~(READ | WRITE));
1180 9901 : i->type = ITER_KVEC | (direction & (READ | WRITE));
1181 9901 : i->kvec = kvec;
1182 9901 : i->nr_segs = nr_segs;
1183 9901 : i->iov_offset = 0;
1184 9901 : i->count = count;
1185 9901 : }
1186 : EXPORT_SYMBOL(iov_iter_kvec);
1187 :
1188 36 : void iov_iter_bvec(struct iov_iter *i, unsigned int direction,
1189 : const struct bio_vec *bvec, unsigned long nr_segs,
1190 : size_t count)
1191 : {
1192 36 : WARN_ON(direction & ~(READ | WRITE));
1193 36 : i->type = ITER_BVEC | (direction & (READ | WRITE));
1194 36 : i->bvec = bvec;
1195 36 : i->nr_segs = nr_segs;
1196 36 : i->iov_offset = 0;
1197 36 : i->count = count;
1198 36 : }
1199 : EXPORT_SYMBOL(iov_iter_bvec);
1200 :
1201 36 : void iov_iter_pipe(struct iov_iter *i, unsigned int direction,
1202 : struct pipe_inode_info *pipe,
1203 : size_t count)
1204 : {
1205 36 : BUG_ON(direction != READ);
1206 36 : WARN_ON(pipe_full(pipe->head, pipe->tail, pipe->ring_size));
1207 36 : i->type = ITER_PIPE | READ;
1208 36 : i->pipe = pipe;
1209 36 : i->head = pipe->head;
1210 36 : i->iov_offset = 0;
1211 36 : i->count = count;
1212 36 : i->start_head = i->head;
1213 36 : }
1214 : EXPORT_SYMBOL(iov_iter_pipe);
1215 :
1216 : /**
1217 : * iov_iter_discard - Initialise an I/O iterator that discards data
1218 : * @i: The iterator to initialise.
1219 : * @direction: The direction of the transfer.
1220 : * @count: The size of the I/O buffer in bytes.
1221 : *
1222 : * Set up an I/O iterator that just discards everything that's written to it.
1223 : * It's only available as a READ iterator.
1224 : */
1225 0 : void iov_iter_discard(struct iov_iter *i, unsigned int direction, size_t count)
1226 : {
1227 0 : BUG_ON(direction != READ);
1228 0 : i->type = ITER_DISCARD | READ;
1229 0 : i->count = count;
1230 0 : i->iov_offset = 0;
1231 0 : }
1232 : EXPORT_SYMBOL(iov_iter_discard);
1233 :
1234 0 : unsigned long iov_iter_alignment(const struct iov_iter *i)
1235 : {
1236 0 : unsigned long res = 0;
1237 0 : size_t size = i->count;
1238 :
1239 0 : if (unlikely(iov_iter_is_pipe(i))) {
1240 0 : unsigned int p_mask = i->pipe->ring_size - 1;
1241 :
1242 0 : if (size && i->iov_offset && allocated(&i->pipe->bufs[i->head & p_mask]))
1243 0 : return size | i->iov_offset;
1244 : return size;
1245 : }
1246 0 : iterate_all_kinds(i, size, v,
1247 : (res |= (unsigned long)v.iov_base | v.iov_len, 0),
1248 : res |= v.bv_offset | v.bv_len,
1249 : res |= (unsigned long)v.iov_base | v.iov_len
1250 : )
1251 : return res;
1252 : }
1253 : EXPORT_SYMBOL(iov_iter_alignment);
1254 :
1255 0 : unsigned long iov_iter_gap_alignment(const struct iov_iter *i)
1256 : {
1257 0 : unsigned long res = 0;
1258 0 : size_t size = i->count;
1259 :
1260 0 : if (unlikely(iov_iter_is_pipe(i) || iov_iter_is_discard(i))) {
1261 0 : WARN_ON(1);
1262 0 : return ~0U;
1263 : }
1264 :
1265 0 : iterate_all_kinds(i, size, v,
1266 : (res |= (!res ? 0 : (unsigned long)v.iov_base) |
1267 : (size != v.iov_len ? size : 0), 0),
1268 : (res |= (!res ? 0 : (unsigned long)v.bv_offset) |
1269 : (size != v.bv_len ? size : 0)),
1270 : (res |= (!res ? 0 : (unsigned long)v.iov_base) |
1271 : (size != v.iov_len ? size : 0))
1272 : );
1273 : return res;
1274 : }
1275 : EXPORT_SYMBOL(iov_iter_gap_alignment);
1276 :
1277 0 : static inline ssize_t __pipe_get_pages(struct iov_iter *i,
1278 : size_t maxsize,
1279 : struct page **pages,
1280 : int iter_head,
1281 : size_t *start)
1282 : {
1283 0 : struct pipe_inode_info *pipe = i->pipe;
1284 0 : unsigned int p_mask = pipe->ring_size - 1;
1285 0 : ssize_t n = push_pipe(i, maxsize, &iter_head, start);
1286 0 : if (!n)
1287 : return -EFAULT;
1288 :
1289 0 : maxsize = n;
1290 0 : n += *start;
1291 0 : while (n > 0) {
1292 0 : get_page(*pages++ = pipe->bufs[iter_head & p_mask].page);
1293 0 : iter_head++;
1294 0 : n -= PAGE_SIZE;
1295 : }
1296 :
1297 : return maxsize;
1298 : }
1299 :
1300 0 : static ssize_t pipe_get_pages(struct iov_iter *i,
1301 : struct page **pages, size_t maxsize, unsigned maxpages,
1302 : size_t *start)
1303 : {
1304 0 : unsigned int iter_head, npages;
1305 0 : size_t capacity;
1306 :
1307 0 : if (!maxsize)
1308 : return 0;
1309 :
1310 0 : if (!sanity(i))
1311 : return -EFAULT;
1312 :
1313 0 : data_start(i, &iter_head, start);
1314 : /* Amount of free space: some of this one + all after this one */
1315 0 : npages = pipe_space_for_user(iter_head, i->pipe->tail, i->pipe);
1316 0 : capacity = min(npages, maxpages) * PAGE_SIZE - *start;
1317 :
1318 0 : return __pipe_get_pages(i, min(maxsize, capacity), pages, iter_head, start);
1319 : }
1320 :
1321 0 : ssize_t iov_iter_get_pages(struct iov_iter *i,
1322 : struct page **pages, size_t maxsize, unsigned maxpages,
1323 : size_t *start)
1324 : {
1325 0 : if (maxsize > i->count)
1326 : maxsize = i->count;
1327 :
1328 0 : if (unlikely(iov_iter_is_pipe(i)))
1329 0 : return pipe_get_pages(i, pages, maxsize, maxpages, start);
1330 0 : if (unlikely(iov_iter_is_discard(i)))
1331 : return -EFAULT;
1332 :
1333 0 : iterate_all_kinds(i, maxsize, v, ({
1334 : unsigned long addr = (unsigned long)v.iov_base;
1335 : size_t len = v.iov_len + (*start = addr & (PAGE_SIZE - 1));
1336 : int n;
1337 : int res;
1338 :
1339 : if (len > maxpages * PAGE_SIZE)
1340 : len = maxpages * PAGE_SIZE;
1341 : addr &= ~(PAGE_SIZE - 1);
1342 : n = DIV_ROUND_UP(len, PAGE_SIZE);
1343 : res = get_user_pages_fast(addr, n,
1344 : iov_iter_rw(i) != WRITE ? FOLL_WRITE : 0,
1345 : pages);
1346 : if (unlikely(res < 0))
1347 : return res;
1348 : return (res == n ? len : res * PAGE_SIZE) - *start;
1349 : 0;}),({
1350 : /* can't be more than PAGE_SIZE */
1351 : *start = v.bv_offset;
1352 : get_page(*pages = v.bv_page);
1353 : return v.bv_len;
1354 : }),({
1355 : return -EFAULT;
1356 : })
1357 : )
1358 : return 0;
1359 : }
1360 : EXPORT_SYMBOL(iov_iter_get_pages);
1361 :
1362 0 : static struct page **get_pages_array(size_t n)
1363 : {
1364 0 : return kvmalloc_array(n, sizeof(struct page *), GFP_KERNEL);
1365 : }
1366 :
1367 0 : static ssize_t pipe_get_pages_alloc(struct iov_iter *i,
1368 : struct page ***pages, size_t maxsize,
1369 : size_t *start)
1370 : {
1371 0 : struct page **p;
1372 0 : unsigned int iter_head, npages;
1373 0 : ssize_t n;
1374 :
1375 0 : if (!maxsize)
1376 : return 0;
1377 :
1378 0 : if (!sanity(i))
1379 : return -EFAULT;
1380 :
1381 0 : data_start(i, &iter_head, start);
1382 : /* Amount of free space: some of this one + all after this one */
1383 0 : npages = pipe_space_for_user(iter_head, i->pipe->tail, i->pipe);
1384 0 : n = npages * PAGE_SIZE - *start;
1385 0 : if (maxsize > n)
1386 : maxsize = n;
1387 : else
1388 0 : npages = DIV_ROUND_UP(maxsize + *start, PAGE_SIZE);
1389 0 : p = get_pages_array(npages);
1390 0 : if (!p)
1391 : return -ENOMEM;
1392 0 : n = __pipe_get_pages(i, maxsize, p, iter_head, start);
1393 0 : if (n > 0)
1394 0 : *pages = p;
1395 : else
1396 0 : kvfree(p);
1397 : return n;
1398 : }
1399 :
1400 0 : ssize_t iov_iter_get_pages_alloc(struct iov_iter *i,
1401 : struct page ***pages, size_t maxsize,
1402 : size_t *start)
1403 : {
1404 0 : struct page **p;
1405 :
1406 0 : if (maxsize > i->count)
1407 : maxsize = i->count;
1408 :
1409 0 : if (unlikely(iov_iter_is_pipe(i)))
1410 0 : return pipe_get_pages_alloc(i, pages, maxsize, start);
1411 0 : if (unlikely(iov_iter_is_discard(i)))
1412 : return -EFAULT;
1413 :
1414 0 : iterate_all_kinds(i, maxsize, v, ({
1415 : unsigned long addr = (unsigned long)v.iov_base;
1416 : size_t len = v.iov_len + (*start = addr & (PAGE_SIZE - 1));
1417 : int n;
1418 : int res;
1419 :
1420 : addr &= ~(PAGE_SIZE - 1);
1421 : n = DIV_ROUND_UP(len, PAGE_SIZE);
1422 : p = get_pages_array(n);
1423 : if (!p)
1424 : return -ENOMEM;
1425 : res = get_user_pages_fast(addr, n,
1426 : iov_iter_rw(i) != WRITE ? FOLL_WRITE : 0, p);
1427 : if (unlikely(res < 0)) {
1428 : kvfree(p);
1429 : return res;
1430 : }
1431 : *pages = p;
1432 : return (res == n ? len : res * PAGE_SIZE) - *start;
1433 : 0;}),({
1434 : /* can't be more than PAGE_SIZE */
1435 : *start = v.bv_offset;
1436 : *pages = p = get_pages_array(1);
1437 : if (!p)
1438 : return -ENOMEM;
1439 : get_page(*p = v.bv_page);
1440 : return v.bv_len;
1441 : }),({
1442 : return -EFAULT;
1443 : })
1444 : )
1445 : return 0;
1446 : }
1447 : EXPORT_SYMBOL(iov_iter_get_pages_alloc);
1448 :
1449 0 : size_t csum_and_copy_from_iter(void *addr, size_t bytes, __wsum *csum,
1450 : struct iov_iter *i)
1451 : {
1452 0 : char *to = addr;
1453 0 : __wsum sum, next;
1454 0 : size_t off = 0;
1455 0 : sum = *csum;
1456 0 : if (unlikely(iov_iter_is_pipe(i) || iov_iter_is_discard(i))) {
1457 0 : WARN_ON(1);
1458 0 : return 0;
1459 : }
1460 0 : iterate_and_advance(i, bytes, v, ({
1461 : next = csum_and_copy_from_user(v.iov_base,
1462 : (to += v.iov_len) - v.iov_len,
1463 : v.iov_len);
1464 : if (next) {
1465 : sum = csum_block_add(sum, next, off);
1466 : off += v.iov_len;
1467 : }
1468 : next ? 0 : v.iov_len;
1469 : }), ({
1470 : char *p = kmap_atomic(v.bv_page);
1471 : sum = csum_and_memcpy((to += v.bv_len) - v.bv_len,
1472 : p + v.bv_offset, v.bv_len,
1473 : sum, off);
1474 : kunmap_atomic(p);
1475 : off += v.bv_len;
1476 : }),({
1477 : sum = csum_and_memcpy((to += v.iov_len) - v.iov_len,
1478 : v.iov_base, v.iov_len,
1479 : sum, off);
1480 : off += v.iov_len;
1481 : })
1482 : )
1483 0 : *csum = sum;
1484 0 : return bytes;
1485 : }
1486 : EXPORT_SYMBOL(csum_and_copy_from_iter);
1487 :
1488 15 : bool csum_and_copy_from_iter_full(void *addr, size_t bytes, __wsum *csum,
1489 : struct iov_iter *i)
1490 : {
1491 15 : char *to = addr;
1492 15 : __wsum sum, next;
1493 15 : size_t off = 0;
1494 15 : sum = *csum;
1495 15 : if (unlikely(iov_iter_is_pipe(i) || iov_iter_is_discard(i))) {
1496 0 : WARN_ON(1);
1497 0 : return false;
1498 : }
1499 15 : if (unlikely(i->count < bytes))
1500 : return false;
1501 30 : iterate_all_kinds(i, bytes, v, ({
1502 : next = csum_and_copy_from_user(v.iov_base,
1503 : (to += v.iov_len) - v.iov_len,
1504 : v.iov_len);
1505 : if (!next)
1506 : return false;
1507 : sum = csum_block_add(sum, next, off);
1508 : off += v.iov_len;
1509 : 0;
1510 : }), ({
1511 : char *p = kmap_atomic(v.bv_page);
1512 : sum = csum_and_memcpy((to += v.bv_len) - v.bv_len,
1513 : p + v.bv_offset, v.bv_len,
1514 : sum, off);
1515 : kunmap_atomic(p);
1516 : off += v.bv_len;
1517 : }),({
1518 : sum = csum_and_memcpy((to += v.iov_len) - v.iov_len,
1519 : v.iov_base, v.iov_len,
1520 : sum, off);
1521 : off += v.iov_len;
1522 : })
1523 : )
1524 15 : *csum = sum;
1525 15 : iov_iter_advance(i, bytes);
1526 15 : return true;
1527 : }
1528 : EXPORT_SYMBOL(csum_and_copy_from_iter_full);
1529 :
1530 0 : size_t csum_and_copy_to_iter(const void *addr, size_t bytes, void *_csstate,
1531 : struct iov_iter *i)
1532 : {
1533 0 : struct csum_state *csstate = _csstate;
1534 0 : const char *from = addr;
1535 0 : __wsum sum, next;
1536 0 : size_t off;
1537 :
1538 0 : if (unlikely(iov_iter_is_pipe(i)))
1539 0 : return csum_and_copy_to_pipe_iter(addr, bytes, _csstate, i);
1540 :
1541 0 : sum = csstate->csum;
1542 0 : off = csstate->off;
1543 0 : if (unlikely(iov_iter_is_discard(i))) {
1544 0 : WARN_ON(1); /* for now */
1545 0 : return 0;
1546 : }
1547 0 : iterate_and_advance(i, bytes, v, ({
1548 : next = csum_and_copy_to_user((from += v.iov_len) - v.iov_len,
1549 : v.iov_base,
1550 : v.iov_len);
1551 : if (next) {
1552 : sum = csum_block_add(sum, next, off);
1553 : off += v.iov_len;
1554 : }
1555 : next ? 0 : v.iov_len;
1556 : }), ({
1557 : char *p = kmap_atomic(v.bv_page);
1558 : sum = csum_and_memcpy(p + v.bv_offset,
1559 : (from += v.bv_len) - v.bv_len,
1560 : v.bv_len, sum, off);
1561 : kunmap_atomic(p);
1562 : off += v.bv_len;
1563 : }),({
1564 : sum = csum_and_memcpy(v.iov_base,
1565 : (from += v.iov_len) - v.iov_len,
1566 : v.iov_len, sum, off);
1567 : off += v.iov_len;
1568 : })
1569 : )
1570 0 : csstate->csum = sum;
1571 0 : csstate->off = off;
1572 0 : return bytes;
1573 : }
1574 : EXPORT_SYMBOL(csum_and_copy_to_iter);
1575 :
1576 0 : size_t hash_and_copy_to_iter(const void *addr, size_t bytes, void *hashp,
1577 : struct iov_iter *i)
1578 : {
1579 : #ifdef CONFIG_CRYPTO_HASH
1580 0 : struct ahash_request *hash = hashp;
1581 0 : struct scatterlist sg;
1582 0 : size_t copied;
1583 :
1584 0 : copied = copy_to_iter(addr, bytes, i);
1585 0 : sg_init_one(&sg, addr, copied);
1586 0 : ahash_request_set_crypt(hash, &sg, NULL, copied);
1587 0 : crypto_ahash_update(hash);
1588 0 : return copied;
1589 : #else
1590 : return 0;
1591 : #endif
1592 : }
1593 : EXPORT_SYMBOL(hash_and_copy_to_iter);
1594 :
1595 0 : int iov_iter_npages(const struct iov_iter *i, int maxpages)
1596 : {
1597 0 : size_t size = i->count;
1598 0 : int npages = 0;
1599 :
1600 0 : if (!size)
1601 : return 0;
1602 0 : if (unlikely(iov_iter_is_discard(i)))
1603 : return 0;
1604 :
1605 0 : if (unlikely(iov_iter_is_pipe(i))) {
1606 0 : struct pipe_inode_info *pipe = i->pipe;
1607 0 : unsigned int iter_head;
1608 0 : size_t off;
1609 :
1610 0 : if (!sanity(i))
1611 0 : return 0;
1612 :
1613 0 : data_start(i, &iter_head, &off);
1614 : /* some of this one + all after this one */
1615 0 : npages = pipe_space_for_user(iter_head, pipe->tail, pipe);
1616 0 : if (npages >= maxpages)
1617 : return maxpages;
1618 0 : } else iterate_all_kinds(i, size, v, ({
1619 : unsigned long p = (unsigned long)v.iov_base;
1620 : npages += DIV_ROUND_UP(p + v.iov_len, PAGE_SIZE)
1621 : - p / PAGE_SIZE;
1622 : if (npages >= maxpages)
1623 : return maxpages;
1624 : 0;}),({
1625 : npages++;
1626 : if (npages >= maxpages)
1627 : return maxpages;
1628 : }),({
1629 : unsigned long p = (unsigned long)v.iov_base;
1630 : npages += DIV_ROUND_UP(p + v.iov_len, PAGE_SIZE)
1631 : - p / PAGE_SIZE;
1632 : if (npages >= maxpages)
1633 : return maxpages;
1634 : })
1635 : )
1636 : return npages;
1637 : }
1638 : EXPORT_SYMBOL(iov_iter_npages);
1639 :
1640 0 : const void *dup_iter(struct iov_iter *new, struct iov_iter *old, gfp_t flags)
1641 : {
1642 0 : *new = *old;
1643 0 : if (unlikely(iov_iter_is_pipe(new))) {
1644 0 : WARN_ON(1);
1645 0 : return NULL;
1646 : }
1647 0 : if (unlikely(iov_iter_is_discard(new)))
1648 : return NULL;
1649 0 : if (iov_iter_is_bvec(new))
1650 0 : return new->bvec = kmemdup(new->bvec,
1651 0 : new->nr_segs * sizeof(struct bio_vec),
1652 : flags);
1653 : else
1654 : /* iovec and kvec have identical layout */
1655 0 : return new->iov = kmemdup(new->iov,
1656 0 : new->nr_segs * sizeof(struct iovec),
1657 : flags);
1658 : }
1659 : EXPORT_SYMBOL(dup_iter);
1660 :
1661 0 : static int copy_compat_iovec_from_user(struct iovec *iov,
1662 : const struct iovec __user *uvec, unsigned long nr_segs)
1663 : {
1664 0 : const struct compat_iovec __user *uiov =
1665 : (const struct compat_iovec __user *)uvec;
1666 0 : int ret = -EFAULT, i;
1667 :
1668 0 : if (!user_access_begin(uiov, nr_segs * sizeof(*uiov)))
1669 : return -EFAULT;
1670 :
1671 0 : for (i = 0; i < nr_segs; i++) {
1672 0 : compat_uptr_t buf;
1673 0 : compat_ssize_t len;
1674 :
1675 0 : unsafe_get_user(len, &uiov[i].iov_len, uaccess_end);
1676 0 : unsafe_get_user(buf, &uiov[i].iov_base, uaccess_end);
1677 :
1678 : /* check for compat_size_t not fitting in compat_ssize_t .. */
1679 0 : if (len < 0) {
1680 0 : ret = -EINVAL;
1681 0 : goto uaccess_end;
1682 : }
1683 0 : iov[i].iov_base = compat_ptr(buf);
1684 0 : iov[i].iov_len = len;
1685 : }
1686 :
1687 : ret = 0;
1688 : uaccess_end:
1689 : user_access_end();
1690 : return ret;
1691 : }
1692 :
1693 6928 : static int copy_iovec_from_user(struct iovec *iov,
1694 : const struct iovec __user *uvec, unsigned long nr_segs)
1695 : {
1696 6928 : unsigned long seg;
1697 :
1698 13859 : if (copy_from_user(iov, uvec, nr_segs * sizeof(*uvec)))
1699 : return -EFAULT;
1700 19318 : for (seg = 0; seg < nr_segs; seg++) {
1701 12387 : if ((ssize_t)iov[seg].iov_len < 0)
1702 : return -EINVAL;
1703 : }
1704 :
1705 : return 0;
1706 : }
1707 :
1708 6928 : struct iovec *iovec_from_user(const struct iovec __user *uvec,
1709 : unsigned long nr_segs, unsigned long fast_segs,
1710 : struct iovec *fast_iov, bool compat)
1711 : {
1712 6928 : struct iovec *iov = fast_iov;
1713 6928 : int ret;
1714 :
1715 : /*
1716 : * SuS says "The readv() function *may* fail if the iovcnt argument was
1717 : * less than or equal to 0, or greater than {IOV_MAX}. Linux has
1718 : * traditionally returned zero for zero segments, so...
1719 : */
1720 6928 : if (nr_segs == 0)
1721 : return iov;
1722 6928 : if (nr_segs > UIO_MAXIOV)
1723 6930 : return ERR_PTR(-EINVAL);
1724 6928 : if (nr_segs > fast_segs) {
1725 165 : iov = kmalloc_array(nr_segs, sizeof(struct iovec), GFP_KERNEL);
1726 165 : if (!iov)
1727 6930 : return ERR_PTR(-ENOMEM);
1728 : }
1729 :
1730 6928 : if (compat)
1731 0 : ret = copy_compat_iovec_from_user(iov, uvec, nr_segs);
1732 : else
1733 6928 : ret = copy_iovec_from_user(iov, uvec, nr_segs);
1734 6930 : if (ret) {
1735 0 : if (iov != fast_iov)
1736 0 : kfree(iov);
1737 0 : return ERR_PTR(ret);
1738 : }
1739 :
1740 : return iov;
1741 : }
1742 :
1743 6928 : ssize_t __import_iovec(int type, const struct iovec __user *uvec,
1744 : unsigned nr_segs, unsigned fast_segs, struct iovec **iovp,
1745 : struct iov_iter *i, bool compat)
1746 : {
1747 6928 : ssize_t total_len = 0;
1748 6928 : unsigned long seg;
1749 6928 : struct iovec *iov;
1750 :
1751 6928 : iov = iovec_from_user(uvec, nr_segs, fast_segs, *iovp, compat);
1752 6931 : if (IS_ERR(iov)) {
1753 0 : *iovp = NULL;
1754 0 : return PTR_ERR(iov);
1755 : }
1756 :
1757 : /*
1758 : * According to the Single Unix Specification we should return EINVAL if
1759 : * an element length is < 0 when cast to ssize_t or if the total length
1760 : * would overflow the ssize_t return value of the system call.
1761 : *
1762 : * Linux caps all read/write calls to MAX_RW_COUNT, and avoids the
1763 : * overflow case.
1764 : */
1765 19317 : for (seg = 0; seg < nr_segs; seg++) {
1766 12385 : ssize_t len = (ssize_t)iov[seg].iov_len;
1767 :
1768 24771 : if (!access_ok(iov[seg].iov_base, len)) {
1769 0 : if (iov != *iovp)
1770 0 : kfree(iov);
1771 0 : *iovp = NULL;
1772 0 : return -EFAULT;
1773 : }
1774 :
1775 12386 : if (len > MAX_RW_COUNT - total_len) {
1776 0 : len = MAX_RW_COUNT - total_len;
1777 0 : iov[seg].iov_len = len;
1778 : }
1779 12386 : total_len += len;
1780 : }
1781 :
1782 6932 : iov_iter_init(i, type, iov, nr_segs, total_len);
1783 6931 : if (iov == *iovp)
1784 6766 : *iovp = NULL;
1785 : else
1786 165 : *iovp = iov;
1787 : return total_len;
1788 : }
1789 :
1790 : /**
1791 : * import_iovec() - Copy an array of &struct iovec from userspace
1792 : * into the kernel, check that it is valid, and initialize a new
1793 : * &struct iov_iter iterator to access it.
1794 : *
1795 : * @type: One of %READ or %WRITE.
1796 : * @uvec: Pointer to the userspace array.
1797 : * @nr_segs: Number of elements in userspace array.
1798 : * @fast_segs: Number of elements in @iov.
1799 : * @iovp: (input and output parameter) Pointer to pointer to (usually small
1800 : * on-stack) kernel array.
1801 : * @i: Pointer to iterator that will be initialized on success.
1802 : *
1803 : * If the array pointed to by *@iov is large enough to hold all @nr_segs,
1804 : * then this function places %NULL in *@iov on return. Otherwise, a new
1805 : * array will be allocated and the result placed in *@iov. This means that
1806 : * the caller may call kfree() on *@iov regardless of whether the small
1807 : * on-stack array was used or not (and regardless of whether this function
1808 : * returns an error or not).
1809 : *
1810 : * Return: Negative error code on error, bytes imported on success
1811 : */
1812 6929 : ssize_t import_iovec(int type, const struct iovec __user *uvec,
1813 : unsigned nr_segs, unsigned fast_segs,
1814 : struct iovec **iovp, struct iov_iter *i)
1815 : {
1816 6929 : return __import_iovec(type, uvec, nr_segs, fast_segs, iovp, i,
1817 6929 : in_compat_syscall());
1818 : }
1819 : EXPORT_SYMBOL(import_iovec);
1820 :
1821 191 : int import_single_range(int rw, void __user *buf, size_t len,
1822 : struct iovec *iov, struct iov_iter *i)
1823 : {
1824 191 : if (len > MAX_RW_COUNT)
1825 : len = MAX_RW_COUNT;
1826 382 : if (unlikely(!access_ok(buf, len)))
1827 : return -EFAULT;
1828 :
1829 191 : iov->iov_base = buf;
1830 191 : iov->iov_len = len;
1831 191 : iov_iter_init(i, rw, iov, 1, len);
1832 191 : return 0;
1833 : }
1834 : EXPORT_SYMBOL(import_single_range);
1835 :
1836 0 : int iov_iter_for_each_range(struct iov_iter *i, size_t bytes,
1837 : int (*f)(struct kvec *vec, void *context),
1838 : void *context)
1839 : {
1840 0 : struct kvec w;
1841 0 : int err = -EINVAL;
1842 0 : if (!bytes)
1843 : return 0;
1844 :
1845 0 : iterate_all_kinds(i, bytes, v, -EINVAL, ({
1846 : w.iov_base = kmap(v.bv_page) + v.bv_offset;
1847 : w.iov_len = v.bv_len;
1848 : err = f(&w, context);
1849 : kunmap(v.bv_page);
1850 : err;}), ({
1851 : w = v;
1852 : err = f(&w, context);})
1853 : )
1854 : return err;
1855 : }
1856 : EXPORT_SYMBOL(iov_iter_for_each_range);
|
