projects/linux·blob master · refs

308 lines · 7.9 KiB

1// SPDX-License-Identifier: GPL-2.0-or-later
2/* Iterator helpers.
3 *
4 * Copyright (C) 2022 Red Hat, Inc. All Rights Reserved.
5 * Written by David Howells ([email protected])
6 */
7
8#include <linux/export.h>
9#include <linux/slab.h>
10#include <linux/mm.h>
11#include <linux/uio.h>
12#include <linux/scatterlist.h>
13#include <linux/netfs.h>
14#include "internal.h"
15
16/**
17 * netfs_extract_user_iter - Extract the pages from a user iterator into a bvec
18 * @orig: The original iterator
19 * @orig_len: The amount of iterator to copy
20 * @new: The iterator to be set up
21 * @extraction_flags: Flags to qualify the request
22 *
23 * Extract the page fragments from the given amount of the source iterator and
24 * build up a second iterator that refers to all of those bits. This allows
25 * the original iterator to be disposed of.
26 *
27 * @extraction_flags can have ITER_ALLOW_P2PDMA set to request peer-to-peer DMA be
28 * allowed on the pages extracted.
29 *
30 * On success, the number of elements in the bvec is returned, the original
31 * iterator will have been advanced by the amount extracted.
32 *
33 * The iov_iter_extract_mode() function should be used to query how cleanup
34 * should be performed.
35 */
36ssize_t netfs_extract_user_iter(struct iov_iter *orig, size_t orig_len,
37 struct iov_iter *new,
38 iov_iter_extraction_t extraction_flags)
39{
40 struct bio_vec *bv = NULL;
41 struct page **pages;
42 unsigned int cur_npages;
43 unsigned int max_pages;
44 unsigned int npages = 0;
45 unsigned int i;
46 ssize_t ret = 0;
47 size_t count = orig_len, offset, len;
48 size_t bv_size, pg_size;
49
50 if (WARN_ON_ONCE(!iter_is_ubuf(orig) && !iter_is_iovec(orig)))
51 return -EIO;
52
53 max_pages = iov_iter_npages(orig, INT_MAX);
54 bv_size = array_size(max_pages, sizeof(*bv));
55 bv = kvmalloc(bv_size, GFP_KERNEL);
56 if (!bv)
57 return -ENOMEM;
58
59 /* Put the page list at the end of the bvec list storage. bvec
60 * elements are larger than page pointers, so as long as we work
61 * 0->last, we should be fine.
62 */
63 pg_size = array_size(max_pages, sizeof(*pages));
64 pages = (void *)bv + bv_size - pg_size;
65
66 while (count && npages < max_pages) {
67 ret = iov_iter_extract_pages(orig, &pages, count,
68 max_pages - npages, extraction_flags,
69 &offset);
70 if (unlikely(ret <= 0)) {
71 ret = ret ?: -EIO;
72 break;
73 }
74
75 if (WARN(ret > count,
76 "%s: extract_pages overrun %zd > %zu bytes\n",
77 __func__, ret, count)) {
78 ret = -EIO;
79 break;
80 }
81
82 cur_npages = DIV_ROUND_UP(offset + ret, PAGE_SIZE);
83 if (WARN(cur_npages > max_pages - npages,
84 "%s: extract_pages overrun %u > %u pages\n",
85 __func__, npages + cur_npages, max_pages)) {
86 ret = -EIO;
87 break;
88 }
89
90 count -= ret;
91 ret += offset;
92
93 for (i = 0; i < cur_npages; i++) {
94 len = ret > PAGE_SIZE ? PAGE_SIZE : ret;
95 bvec_set_page(bv + npages + i, *pages++, len - offset, offset);
96 ret -= len;
97 offset = 0;
98 }
99
100 npages += cur_npages;
101 }
102
103 /* Note: Don't try to clean up after EIO. Either we got no pages, so
104 * nothing to clean up, or we got a buffer overrun, memory corruption
105 * and can't trust the stuff in the buffer (a WARN was emitted).
106 */
107
108 if (ret < 0 && (ret == -ENOMEM || npages == 0)) {
109 for (i = 0; i < npages; i++)
110 unpin_user_page(bv[i].bv_page);
111 kvfree(bv);
112 return ret;
113 }
114
115 iov_iter_bvec(new, orig->data_source, bv, npages, orig_len - count);
116 return npages;
117}
118EXPORT_SYMBOL_GPL(netfs_extract_user_iter);
119
120/*
121 * Select the span of a bvec iterator we're going to use. Limit it by both maximum
122 * size and maximum number of segments. Returns the size of the span in bytes.
123 */
124static size_t netfs_limit_bvec(const struct iov_iter *iter, size_t start_offset,
125 size_t max_size, size_t max_segs)
126{
127 const struct bio_vec *bvecs = iter->bvec;
128 unsigned int nbv = iter->nr_segs, ix = 0, nsegs = 0;
129 size_t len, span = 0, n = iter->count;
130 size_t skip = iter->iov_offset + start_offset;
131
132 if (WARN_ON(!iov_iter_is_bvec(iter)) ||
133 WARN_ON(start_offset > n) ||
134 n == 0)
135 return 0;
136
137 while (n && ix < nbv && skip) {
138 len = bvecs[ix].bv_len;
139 if (skip < len)
140 break;
141 skip -= len;
142 n -= len;
143 ix++;
144 }
145
146 while (n && ix < nbv) {
147 len = min3(n, bvecs[ix].bv_len - skip, max_size);
148 span += len;
149 nsegs++;
150 ix++;
151 if (span >= max_size || nsegs >= max_segs)
152 break;
153 skip = 0;
154 n -= len;
155 }
156
157 return min(span, max_size);
158}
159
160/*
161 * Select the span of a kvec iterator we're going to use. Limit it by both
162 * maximum size and maximum number of segments. Returns the size of the span
163 * in bytes.
164 */
165static size_t netfs_limit_kvec(const struct iov_iter *iter, size_t start_offset,
166 size_t max_size, size_t max_segs)
167{
168 const struct kvec *kvecs = iter->kvec;
169 unsigned int nkv = iter->nr_segs, ix = 0, nsegs = 0;
170 size_t len, span = 0, n = iter->count;
171 size_t skip = iter->iov_offset + start_offset;
172
173 if (WARN_ON(!iov_iter_is_kvec(iter)) ||
174 WARN_ON(start_offset > n) ||
175 n == 0)
176 return 0;
177
178 while (n && ix < nkv && skip) {
179 len = kvecs[ix].iov_len;
180 if (skip < len)
181 break;
182 skip -= len;
183 n -= len;
184 ix++;
185 }
186
187 while (n && ix < nkv) {
188 len = min3(n, kvecs[ix].iov_len - skip, max_size);
189 span += len;
190 nsegs++;
191 ix++;
192 if (span >= max_size || nsegs >= max_segs)
193 break;
194 skip = 0;
195 n -= len;
196 }
197
198 return min(span, max_size);
199}
200
201/*
202 * Select the span of an xarray iterator we're going to use. Limit it by both
203 * maximum size and maximum number of segments. It is assumed that segments
204 * can be larger than a page in size, provided they're physically contiguous.
205 * Returns the size of the span in bytes.
206 */
207static size_t netfs_limit_xarray(const struct iov_iter *iter, size_t start_offset,
208 size_t max_size, size_t max_segs)
209{
210 struct folio *folio;
211 unsigned int nsegs = 0;
212 uoff_t pos = iter->xarray_start + iter->iov_offset;
213 pgoff_t index = pos / PAGE_SIZE;
214 size_t span = 0, n = iter->count;
215
216 XA_STATE(xas, iter->xarray, index);
217
218 if (WARN_ON(!iov_iter_is_xarray(iter)) ||
219 WARN_ON(start_offset > n) ||
220 n == 0)
221 return 0;
222 max_size = min(max_size, n - start_offset);
223
224 rcu_read_lock();
225 xas_for_each(&xas, folio, ULONG_MAX) {
226 size_t offset, flen, len;
227 if (xas_retry(&xas, folio))
228 continue;
229 if (WARN_ON(xa_is_value(folio)))
230 break;
231 if (WARN_ON(folio_test_hugetlb(folio)))
232 break;
233
234 flen = folio_size(folio);
235 offset = offset_in_folio(folio, pos);
236 len = min(max_size, flen - offset);
237 span += len;
238 nsegs++;
239 if (span >= max_size || nsegs >= max_segs)
240 break;
241 }
242
243 rcu_read_unlock();
244 return min(span, max_size);
245}
246
247/*
248 * Select the span of a folio queue iterator we're going to use. Limit it by
249 * both maximum size and maximum number of segments. Returns the size of the
250 * span in bytes.
251 */
252static size_t netfs_limit_folioq(const struct iov_iter *iter, size_t start_offset,
253 size_t max_size, size_t max_segs)
254{
255 const struct folio_queue *folioq = iter->folioq;
256 unsigned int nsegs = 0;
257 unsigned int slot = iter->folioq_slot;
258 size_t span = 0, n = iter->count;
259
260 if (WARN_ON(!iov_iter_is_folioq(iter)) ||
261 WARN_ON(start_offset > n) ||
262 n == 0)
263 return 0;
264 max_size = umin(max_size, n - start_offset);
265
266 if (slot >= folioq_nr_slots(folioq)) {
267 folioq = folioq->next;
268 slot = 0;
269 }
270
271 start_offset += iter->iov_offset;
272 do {
273 size_t flen = folioq_folio_size(folioq, slot);
274
275 if (start_offset < flen) {
276 span += flen - start_offset;
277 nsegs++;
278 start_offset = 0;
279 } else {
280 start_offset -= flen;
281 }
282 if (span >= max_size || nsegs >= max_segs)
283 break;
284
285 slot++;
286 if (slot >= folioq_nr_slots(folioq)) {
287 folioq = folioq->next;
288 slot = 0;
289 }
290 } while (folioq);
291
292 return umin(span, max_size);
293}
294
295size_t netfs_limit_iter(const struct iov_iter *iter, size_t start_offset,
296 size_t max_size, size_t max_segs)
297{
298 if (iov_iter_is_folioq(iter))
299 return netfs_limit_folioq(iter, start_offset, max_size, max_segs);
300 if (iov_iter_is_bvec(iter))
301 return netfs_limit_bvec(iter, start_offset, max_size, max_segs);
302 if (iov_iter_is_xarray(iter))
303 return netfs_limit_xarray(iter, start_offset, max_size, max_segs);
304 if (iov_iter_is_kvec(iter))
305 return netfs_limit_kvec(iter, start_offset, max_size, max_segs);
306 BUG();
307}
308EXPORT_SYMBOL(netfs_limit_iter);