Line data Source code
1 : /* SPDX-License-Identifier: GPL-2.0 */
2 : /*
3 : * bvec iterator
4 : *
5 : * Copyright (C) 2001 Ming Lei <ming.lei@canonical.com>
6 : */
7 : #ifndef __LINUX_BVEC_ITER_H
8 : #define __LINUX_BVEC_ITER_H
9 :
10 : #include <linux/bug.h>
11 : #include <linux/errno.h>
12 : #include <linux/limits.h>
13 : #include <linux/minmax.h>
14 : #include <linux/mm.h>
15 : #include <linux/types.h>
16 :
17 : struct page;
18 :
19 : /**
20 : * struct bio_vec - a contiguous range of physical memory addresses
21 : * @bv_page: First page associated with the address range.
22 : * @bv_len: Number of bytes in the address range.
23 : * @bv_offset: Start of the address range relative to the start of @bv_page.
24 : *
25 : * The following holds for a bvec if n * PAGE_SIZE < bv_offset + bv_len:
26 : *
27 : * nth_page(@bv_page, n) == @bv_page + n
28 : *
29 : * This holds because page_is_mergeable() checks the above property.
30 : */
31 : struct bio_vec {
32 : struct page *bv_page;
33 : unsigned int bv_len;
34 : unsigned int bv_offset;
35 : };
36 :
37 : struct bvec_iter {
38 : sector_t bi_sector; /* device address in 512 byte
39 : sectors */
40 : unsigned int bi_size; /* residual I/O count */
41 :
42 : unsigned int bi_idx; /* current index into bvl_vec */
43 :
44 : unsigned int bi_bvec_done; /* number of bytes completed in
45 : current bvec */
46 : };
47 :
48 : struct bvec_iter_all {
49 : struct bio_vec bv;
50 : int idx;
51 : unsigned done;
52 : };
53 :
54 : /*
55 : * various member access, note that bio_data should of course not be used
56 : * on highmem page vectors
57 : */
58 : #define __bvec_iter_bvec(bvec, iter) (&(bvec)[(iter).bi_idx])
59 :
60 : /* multi-page (mp_bvec) helpers */
61 : #define mp_bvec_iter_page(bvec, iter) \
62 : (__bvec_iter_bvec((bvec), (iter))->bv_page)
63 :
64 : #define mp_bvec_iter_len(bvec, iter) \
65 : min((iter).bi_size, \
66 : __bvec_iter_bvec((bvec), (iter))->bv_len - (iter).bi_bvec_done)
67 :
68 : #define mp_bvec_iter_offset(bvec, iter) \
69 : (__bvec_iter_bvec((bvec), (iter))->bv_offset + (iter).bi_bvec_done)
70 :
71 : #define mp_bvec_iter_page_idx(bvec, iter) \
72 : (mp_bvec_iter_offset((bvec), (iter)) / PAGE_SIZE)
73 :
74 : #define mp_bvec_iter_bvec(bvec, iter) \
75 : ((struct bio_vec) { \
76 : .bv_page = mp_bvec_iter_page((bvec), (iter)), \
77 : .bv_len = mp_bvec_iter_len((bvec), (iter)), \
78 : .bv_offset = mp_bvec_iter_offset((bvec), (iter)), \
79 : })
80 :
81 : /* For building single-page bvec in flight */
82 : #define bvec_iter_offset(bvec, iter) \
83 : (mp_bvec_iter_offset((bvec), (iter)) % PAGE_SIZE)
84 :
85 : #define bvec_iter_len(bvec, iter) \
86 : min_t(unsigned, mp_bvec_iter_len((bvec), (iter)), \
87 : PAGE_SIZE - bvec_iter_offset((bvec), (iter)))
88 :
89 : #define bvec_iter_page(bvec, iter) \
90 : (mp_bvec_iter_page((bvec), (iter)) + \
91 : mp_bvec_iter_page_idx((bvec), (iter)))
92 :
93 : #define bvec_iter_bvec(bvec, iter) \
94 : ((struct bio_vec) { \
95 : .bv_page = bvec_iter_page((bvec), (iter)), \
96 : .bv_len = bvec_iter_len((bvec), (iter)), \
97 : .bv_offset = bvec_iter_offset((bvec), (iter)), \
98 : })
99 :
100 9374 : static inline bool bvec_iter_advance(const struct bio_vec *bv,
101 : struct bvec_iter *iter, unsigned bytes)
102 : {
103 9374 : unsigned int idx = iter->bi_idx;
104 :
105 9374 : if (WARN_ONCE(bytes > iter->bi_size,
106 : "Attempted to advance past end of bvec iter\n")) {
107 0 : iter->bi_size = 0;
108 0 : return false;
109 : }
110 :
111 9374 : iter->bi_size -= bytes;
112 9374 : bytes += iter->bi_bvec_done;
113 :
114 22540 : while (bytes && bytes >= bv[idx].bv_len) {
115 13166 : bytes -= bv[idx].bv_len;
116 13166 : idx++;
117 : }
118 :
119 9374 : iter->bi_idx = idx;
120 9374 : iter->bi_bvec_done = bytes;
121 9374 : return true;
122 : }
123 :
124 : /*
125 : * A simpler version of bvec_iter_advance(), @bytes should not span
126 : * across multiple bvec entries, i.e. bytes <= bv[i->bi_idx].bv_len
127 : */
128 18483 : static inline void bvec_iter_advance_single(const struct bio_vec *bv,
129 : struct bvec_iter *iter, unsigned int bytes)
130 : {
131 18483 : unsigned int done = iter->bi_bvec_done + bytes;
132 :
133 18483 : if (done == bv[iter->bi_idx].bv_len) {
134 18483 : done = 0;
135 18483 : iter->bi_idx++;
136 : }
137 18483 : iter->bi_bvec_done = done;
138 18483 : iter->bi_size -= bytes;
139 18483 : }
140 :
141 : #define for_each_bvec(bvl, bio_vec, iter, start) \
142 : for (iter = (start); \
143 : (iter).bi_size && \
144 : ((bvl = bvec_iter_bvec((bio_vec), (iter))), 1); \
145 : bvec_iter_advance_single((bio_vec), &(iter), (bvl).bv_len))
146 :
147 : /* for iterating one bio from start to end */
148 : #define BVEC_ITER_ALL_INIT (struct bvec_iter) \
149 : { \
150 : .bi_sector = 0, \
151 : .bi_size = UINT_MAX, \
152 : .bi_idx = 0, \
153 : .bi_bvec_done = 0, \
154 : }
155 :
156 1869 : static inline struct bio_vec *bvec_init_iter_all(struct bvec_iter_all *iter_all)
157 : {
158 1869 : iter_all->done = 0;
159 1869 : iter_all->idx = 0;
160 :
161 1869 : return &iter_all->bv;
162 : }
163 :
164 22054 : static inline void bvec_advance(const struct bio_vec *bvec,
165 : struct bvec_iter_all *iter_all)
166 : {
167 22054 : struct bio_vec *bv = &iter_all->bv;
168 :
169 22054 : if (iter_all->done) {
170 16263 : bv->bv_page++;
171 16263 : bv->bv_offset = 0;
172 : } else {
173 5791 : bv->bv_page = bvec->bv_page + (bvec->bv_offset >> PAGE_SHIFT);
174 5791 : bv->bv_offset = bvec->bv_offset & ~PAGE_MASK;
175 : }
176 22054 : bv->bv_len = min_t(unsigned int, PAGE_SIZE - bv->bv_offset,
177 : bvec->bv_len - iter_all->done);
178 22054 : iter_all->done += bv->bv_len;
179 :
180 22054 : if (iter_all->done == bvec->bv_len) {
181 5791 : iter_all->idx++;
182 5791 : iter_all->done = 0;
183 : }
184 22054 : }
185 :
186 : #endif /* __LINUX_BVEC_ITER_H */
|
