projects/linux·blob master · refs

1250 lines · 30.7 KiB

1// SPDX-License-Identifier: GPL-2.0-only
2/* I/O iterator tests. This can only test kernel-backed iterator types.
3 *
4 * Copyright (C) 2023 Red Hat, Inc. All Rights Reserved.
5 * Written by David Howells ([email protected])
6 */
7
8#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
9
10#include <linux/module.h>
11#include <linux/vmalloc.h>
12#include <linux/mm.h>
13#include <linux/uio.h>
14#include <linux/bvec.h>
15#include <linux/folio_queue.h>
16#include <linux/scatterlist.h>
17#include <linux/minmax.h>
18#include <linux/mman.h>
19#include <kunit/test.h>
20
21MODULE_DESCRIPTION("iov_iter testing");
22MODULE_AUTHOR("David Howells <[email protected]>");
23MODULE_LICENSE("GPL");
24
25struct kvec_test_range {
26 int from, to;
27};
28
29static const struct kvec_test_range kvec_test_ranges[] = {
30 { 0x00002, 0x00002 },
31 { 0x00027, 0x03000 },
32 { 0x05193, 0x18794 },
33 { 0x20000, 0x20000 },
34 { 0x20000, 0x24000 },
35 { 0x24000, 0x27001 },
36 { 0x29000, 0xffffb },
37 { 0xffffd, 0xffffe },
38 { -1 }
39};
40
41static inline u8 pattern(unsigned long x)
42{
43 return (u8)x + (u8)(x >> 8) + (u8)(x >> 16);
44}
45
46static void iov_kunit_unmap(void *data)
47{
48 vfree(data);
49}
50
51static void *__init iov_kunit_create_buffer(struct kunit *test,
52 struct page ***ppages,
53 size_t npages)
54{
55 struct page **pages;
56 unsigned long got, last;
57 void *buffer;
58 unsigned int i;
59
60 pages = kzalloc_objs(struct page *, npages);
61 KUNIT_ASSERT_NOT_ERR_OR_NULL(test, pages);
62 *ppages = pages;
63
64 got = 0;
65 while (true) {
66 last = got;
67 got = alloc_pages_bulk(GFP_KERNEL, npages, pages);
68
69 if (last == got || got == npages)
70 break;
71 }
72
73 if (got != npages) {
74 release_pages(pages, got);
75 kvfree(pages);
76 KUNIT_ASSERT_EQ(test, got, npages);
77 }
78 /* Make sure that we don't get a physically contiguous buffer. */
79 for (i = 0; i < npages / 4; ++i)
80 swap(pages[i], pages[i + npages / 2]);
81
82 buffer = vmap(pages, npages, VM_MAP | VM_MAP_PUT_PAGES, PAGE_KERNEL);
83 if (buffer == NULL) {
84 release_pages(pages, got);
85 kvfree(pages);
86 }
87 KUNIT_ASSERT_NOT_ERR_OR_NULL(test, buffer);
88
89 kunit_add_action_or_reset(test, iov_kunit_unmap, buffer);
90 return buffer;
91}
92
93static void __init iov_kunit_load_kvec(struct kunit *test,
94 struct iov_iter *iter, int dir,
95 struct kvec *kvec, unsigned int kvmax,
96 void *buffer, size_t bufsize,
97 const struct kvec_test_range *pr)
98{
99 size_t size = 0;
100 int i;
101
102 for (i = 0; i < kvmax; i++, pr++) {
103 if (pr->from < 0)
104 break;
105 KUNIT_ASSERT_GE(test, pr->to, pr->from);
106 KUNIT_ASSERT_LE(test, pr->to, bufsize);
107 kvec[i].iov_base = buffer + pr->from;
108 kvec[i].iov_len = pr->to - pr->from;
109 size += pr->to - pr->from;
110 }
111 KUNIT_ASSERT_LE(test, size, bufsize);
112
113 iov_iter_kvec(iter, dir, kvec, i, size);
114}
115
116/*
117 * Test copying to a ITER_KVEC-type iterator.
118 */
119static void __init iov_kunit_copy_to_kvec(struct kunit *test)
120{
121 const struct kvec_test_range *pr;
122 struct iov_iter iter;
123 struct page **spages, **bpages;
124 struct kvec kvec[8];
125 u8 *scratch, *buffer;
126 size_t bufsize, npages, size, copied;
127 int i, patt;
128
129 bufsize = 0x100000;
130 npages = bufsize / PAGE_SIZE;
131
132 scratch = iov_kunit_create_buffer(test, &spages, npages);
133 for (i = 0; i < bufsize; i++)
134 scratch[i] = pattern(i);
135
136 buffer = iov_kunit_create_buffer(test, &bpages, npages);
137 memset(buffer, 0, bufsize);
138
139 iov_kunit_load_kvec(test, &iter, READ, kvec, ARRAY_SIZE(kvec),
140 buffer, bufsize, kvec_test_ranges);
141 size = iter.count;
142
143 copied = copy_to_iter(scratch, size, &iter);
144
145 KUNIT_EXPECT_EQ(test, copied, size);
146 KUNIT_EXPECT_EQ(test, iter.count, 0);
147 KUNIT_EXPECT_EQ(test, iter.nr_segs, 0);
148
149 /* Build the expected image in the scratch buffer. */
150 patt = 0;
151 memset(scratch, 0, bufsize);
152 for (pr = kvec_test_ranges; pr->from >= 0; pr++)
153 for (i = pr->from; i < pr->to; i++)
154 scratch[i] = pattern(patt++);
155
156 /* Compare the images */
157 for (i = 0; i < bufsize; i++) {
158 KUNIT_EXPECT_EQ_MSG(test, buffer[i], scratch[i], "at i=%x", i);
159 if (buffer[i] != scratch[i])
160 return;
161 }
162
163 KUNIT_SUCCEED(test);
164}
165
166/*
167 * Test copying from a ITER_KVEC-type iterator.
168 */
169static void __init iov_kunit_copy_from_kvec(struct kunit *test)
170{
171 const struct kvec_test_range *pr;
172 struct iov_iter iter;
173 struct page **spages, **bpages;
174 struct kvec kvec[8];
175 u8 *scratch, *buffer;
176 size_t bufsize, npages, size, copied;
177 int i, j;
178
179 bufsize = 0x100000;
180 npages = bufsize / PAGE_SIZE;
181
182 buffer = iov_kunit_create_buffer(test, &bpages, npages);
183 for (i = 0; i < bufsize; i++)
184 buffer[i] = pattern(i);
185
186 scratch = iov_kunit_create_buffer(test, &spages, npages);
187 memset(scratch, 0, bufsize);
188
189 iov_kunit_load_kvec(test, &iter, WRITE, kvec, ARRAY_SIZE(kvec),
190 buffer, bufsize, kvec_test_ranges);
191 size = min(iter.count, bufsize);
192
193 copied = copy_from_iter(scratch, size, &iter);
194
195 KUNIT_EXPECT_EQ(test, copied, size);
196 KUNIT_EXPECT_EQ(test, iter.count, 0);
197 KUNIT_EXPECT_EQ(test, iter.nr_segs, 0);
198
199 /* Build the expected image in the main buffer. */
200 i = 0;
201 memset(buffer, 0, bufsize);
202 for (pr = kvec_test_ranges; pr->from >= 0; pr++) {
203 for (j = pr->from; j < pr->to; j++) {
204 buffer[i++] = pattern(j);
205 if (i >= bufsize)
206 goto stop;
207 }
208 }
209stop:
210
211 /* Compare the images */
212 for (i = 0; i < bufsize; i++) {
213 KUNIT_EXPECT_EQ_MSG(test, scratch[i], buffer[i], "at i=%x", i);
214 if (scratch[i] != buffer[i])
215 return;
216 }
217
218 KUNIT_SUCCEED(test);
219}
220
221struct bvec_test_range {
222 int page, from, to;
223};
224
225static const struct bvec_test_range bvec_test_ranges[] = {
226 { 0, 0x0002, 0x0002 },
227 { 1, 0x0027, 0x0893 },
228 { 2, 0x0193, 0x0794 },
229 { 3, 0x0000, 0x1000 },
230 { 4, 0x0000, 0x1000 },
231 { 5, 0x0000, 0x1000 },
232 { 6, 0x0000, 0x0ffb },
233 { 6, 0x0ffd, 0x0ffe },
234 { -1, -1, -1 }
235};
236
237static void __init iov_kunit_load_bvec(struct kunit *test,
238 struct iov_iter *iter, int dir,
239 struct bio_vec *bvec, unsigned int bvmax,
240 struct page **pages, size_t npages,
241 size_t bufsize,
242 const struct bvec_test_range *pr)
243{
244 struct page *can_merge = NULL, *page;
245 size_t size = 0;
246 int i;
247
248 for (i = 0; i < bvmax; i++, pr++) {
249 if (pr->from < 0)
250 break;
251 KUNIT_ASSERT_LT(test, pr->page, npages);
252 KUNIT_ASSERT_LT(test, pr->page * PAGE_SIZE, bufsize);
253 KUNIT_ASSERT_GE(test, pr->from, 0);
254 KUNIT_ASSERT_GE(test, pr->to, pr->from);
255 KUNIT_ASSERT_LE(test, pr->to, PAGE_SIZE);
256
257 page = pages[pr->page];
258 if (pr->from == 0 && pr->from != pr->to && page == can_merge) {
259 i--;
260 bvec[i].bv_len += pr->to;
261 } else {
262 bvec_set_page(&bvec[i], page, pr->to - pr->from, pr->from);
263 }
264
265 size += pr->to - pr->from;
266 if ((pr->to & ~PAGE_MASK) == 0)
267 can_merge = page + pr->to / PAGE_SIZE;
268 else
269 can_merge = NULL;
270 }
271
272 iov_iter_bvec(iter, dir, bvec, i, size);
273}
274
275/*
276 * Test copying to a ITER_BVEC-type iterator.
277 */
278static void __init iov_kunit_copy_to_bvec(struct kunit *test)
279{
280 const struct bvec_test_range *pr;
281 struct iov_iter iter;
282 struct bio_vec bvec[8];
283 struct page **spages, **bpages;
284 u8 *scratch, *buffer;
285 size_t bufsize, npages, size, copied;
286 int i, patt;
287
288 bufsize = 0x100000;
289 npages = bufsize / PAGE_SIZE;
290
291 scratch = iov_kunit_create_buffer(test, &spages, npages);
292 for (i = 0; i < bufsize; i++)
293 scratch[i] = pattern(i);
294
295 buffer = iov_kunit_create_buffer(test, &bpages, npages);
296 memset(buffer, 0, bufsize);
297
298 iov_kunit_load_bvec(test, &iter, READ, bvec, ARRAY_SIZE(bvec),
299 bpages, npages, bufsize, bvec_test_ranges);
300 size = iter.count;
301
302 copied = copy_to_iter(scratch, size, &iter);
303
304 KUNIT_EXPECT_EQ(test, copied, size);
305 KUNIT_EXPECT_EQ(test, iter.count, 0);
306 KUNIT_EXPECT_EQ(test, iter.nr_segs, 0);
307
308 /* Build the expected image in the scratch buffer. */
309 patt = 0;
310 memset(scratch, 0, bufsize);
311 for (pr = bvec_test_ranges; pr->from >= 0; pr++) {
312 u8 *p = scratch + pr->page * PAGE_SIZE;
313
314 for (i = pr->from; i < pr->to; i++)
315 p[i] = pattern(patt++);
316 }
317
318 /* Compare the images */
319 for (i = 0; i < bufsize; i++) {
320 KUNIT_EXPECT_EQ_MSG(test, buffer[i], scratch[i], "at i=%x", i);
321 if (buffer[i] != scratch[i])
322 return;
323 }
324
325 KUNIT_SUCCEED(test);
326}
327
328/*
329 * Test copying from a ITER_BVEC-type iterator.
330 */
331static void __init iov_kunit_copy_from_bvec(struct kunit *test)
332{
333 const struct bvec_test_range *pr;
334 struct iov_iter iter;
335 struct bio_vec bvec[8];
336 struct page **spages, **bpages;
337 u8 *scratch, *buffer;
338 size_t bufsize, npages, size, copied;
339 int i, j;
340
341 bufsize = 0x100000;
342 npages = bufsize / PAGE_SIZE;
343
344 buffer = iov_kunit_create_buffer(test, &bpages, npages);
345 for (i = 0; i < bufsize; i++)
346 buffer[i] = pattern(i);
347
348 scratch = iov_kunit_create_buffer(test, &spages, npages);
349 memset(scratch, 0, bufsize);
350
351 iov_kunit_load_bvec(test, &iter, WRITE, bvec, ARRAY_SIZE(bvec),
352 bpages, npages, bufsize, bvec_test_ranges);
353 size = iter.count;
354
355 copied = copy_from_iter(scratch, size, &iter);
356
357 KUNIT_EXPECT_EQ(test, copied, size);
358 KUNIT_EXPECT_EQ(test, iter.count, 0);
359 KUNIT_EXPECT_EQ(test, iter.nr_segs, 0);
360
361 /* Build the expected image in the main buffer. */
362 i = 0;
363 memset(buffer, 0, bufsize);
364 for (pr = bvec_test_ranges; pr->from >= 0; pr++) {
365 size_t patt = pr->page * PAGE_SIZE;
366
367 for (j = pr->from; j < pr->to; j++) {
368 buffer[i++] = pattern(patt + j);
369 if (i >= bufsize)
370 goto stop;
371 }
372 }
373stop:
374
375 /* Compare the images */
376 for (i = 0; i < bufsize; i++) {
377 KUNIT_EXPECT_EQ_MSG(test, scratch[i], buffer[i], "at i=%x", i);
378 if (scratch[i] != buffer[i])
379 return;
380 }
381
382 KUNIT_SUCCEED(test);
383}
384
385static void iov_kunit_destroy_folioq(void *data)
386{
387 struct folio_queue *folioq, *next;
388
389 for (folioq = data; folioq; folioq = next) {
390 next = folioq->next;
391 kfree(folioq);
392 }
393}
394
395static void __init iov_kunit_load_folioq(struct kunit *test,
396 struct iov_iter *iter, int dir,
397 struct folio_queue *folioq,
398 struct page **pages, size_t npages)
399{
400 struct folio_queue *p = folioq;
401 size_t size = 0;
402 int i;
403
404 for (i = 0; i < npages; i++) {
405 if (folioq_full(p)) {
406 p->next = kzalloc_obj(struct folio_queue);
407 KUNIT_ASSERT_NOT_ERR_OR_NULL(test, p->next);
408 folioq_init(p->next, 0);
409 p->next->prev = p;
410 p = p->next;
411 }
412 folioq_append(p, page_folio(pages[i]));
413 size += PAGE_SIZE;
414 }
415 iov_iter_folio_queue(iter, dir, folioq, 0, 0, size);
416}
417
418static struct folio_queue *iov_kunit_create_folioq(struct kunit *test)
419{
420 struct folio_queue *folioq;
421
422 folioq = kzalloc_obj(struct folio_queue);
423 KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folioq);
424 kunit_add_action_or_reset(test, iov_kunit_destroy_folioq, folioq);
425 folioq_init(folioq, 0);
426 return folioq;
427}
428
429/*
430 * Test copying to a ITER_FOLIOQ-type iterator.
431 */
432static void __init iov_kunit_copy_to_folioq(struct kunit *test)
433{
434 const struct kvec_test_range *pr;
435 struct iov_iter iter;
436 struct folio_queue *folioq;
437 struct page **spages, **bpages;
438 u8 *scratch, *buffer;
439 size_t bufsize, npages, size, copied;
440 int i, patt;
441
442 bufsize = 0x100000;
443 npages = bufsize / PAGE_SIZE;
444
445 folioq = iov_kunit_create_folioq(test);
446
447 scratch = iov_kunit_create_buffer(test, &spages, npages);
448 for (i = 0; i < bufsize; i++)
449 scratch[i] = pattern(i);
450
451 buffer = iov_kunit_create_buffer(test, &bpages, npages);
452 memset(buffer, 0, bufsize);
453
454 iov_kunit_load_folioq(test, &iter, READ, folioq, bpages, npages);
455
456 i = 0;
457 for (pr = kvec_test_ranges; pr->from >= 0; pr++) {
458 size = pr->to - pr->from;
459 KUNIT_ASSERT_LE(test, pr->to, bufsize);
460
461 iov_iter_folio_queue(&iter, READ, folioq, 0, 0, pr->to);
462 iov_iter_advance(&iter, pr->from);
463 copied = copy_to_iter(scratch + i, size, &iter);
464
465 KUNIT_EXPECT_EQ(test, copied, size);
466 KUNIT_EXPECT_EQ(test, iter.count, 0);
467 KUNIT_EXPECT_EQ(test, iter.iov_offset, pr->to % PAGE_SIZE);
468 i += size;
469 if (test->status == KUNIT_FAILURE)
470 goto stop;
471 }
472
473 /* Build the expected image in the scratch buffer. */
474 patt = 0;
475 memset(scratch, 0, bufsize);
476 for (pr = kvec_test_ranges; pr->from >= 0; pr++)
477 for (i = pr->from; i < pr->to; i++)
478 scratch[i] = pattern(patt++);
479
480 /* Compare the images */
481 for (i = 0; i < bufsize; i++) {
482 KUNIT_EXPECT_EQ_MSG(test, buffer[i], scratch[i], "at i=%x", i);
483 if (buffer[i] != scratch[i])
484 return;
485 }
486
487stop:
488 KUNIT_SUCCEED(test);
489}
490
491/*
492 * Test copying from a ITER_FOLIOQ-type iterator.
493 */
494static void __init iov_kunit_copy_from_folioq(struct kunit *test)
495{
496 const struct kvec_test_range *pr;
497 struct iov_iter iter;
498 struct folio_queue *folioq;
499 struct page **spages, **bpages;
500 u8 *scratch, *buffer;
501 size_t bufsize, npages, size, copied;
502 int i, j;
503
504 bufsize = 0x100000;
505 npages = bufsize / PAGE_SIZE;
506
507 folioq = iov_kunit_create_folioq(test);
508
509 buffer = iov_kunit_create_buffer(test, &bpages, npages);
510 for (i = 0; i < bufsize; i++)
511 buffer[i] = pattern(i);
512
513 scratch = iov_kunit_create_buffer(test, &spages, npages);
514 memset(scratch, 0, bufsize);
515
516 iov_kunit_load_folioq(test, &iter, READ, folioq, bpages, npages);
517
518 i = 0;
519 for (pr = kvec_test_ranges; pr->from >= 0; pr++) {
520 size = pr->to - pr->from;
521 KUNIT_ASSERT_LE(test, pr->to, bufsize);
522
523 iov_iter_folio_queue(&iter, WRITE, folioq, 0, 0, pr->to);
524 iov_iter_advance(&iter, pr->from);
525 copied = copy_from_iter(scratch + i, size, &iter);
526
527 KUNIT_EXPECT_EQ(test, copied, size);
528 KUNIT_EXPECT_EQ(test, iter.count, 0);
529 KUNIT_EXPECT_EQ(test, iter.iov_offset, pr->to % PAGE_SIZE);
530 i += size;
531 }
532
533 /* Build the expected image in the main buffer. */
534 i = 0;
535 memset(buffer, 0, bufsize);
536 for (pr = kvec_test_ranges; pr->from >= 0; pr++) {
537 for (j = pr->from; j < pr->to; j++) {
538 buffer[i++] = pattern(j);
539 if (i >= bufsize)
540 goto stop;
541 }
542 }
543stop:
544
545 /* Compare the images */
546 for (i = 0; i < bufsize; i++) {
547 KUNIT_EXPECT_EQ_MSG(test, scratch[i], buffer[i], "at i=%x", i);
548 if (scratch[i] != buffer[i])
549 return;
550 }
551
552 KUNIT_SUCCEED(test);
553}
554
555static void iov_kunit_destroy_xarray(void *data)
556{
557 struct xarray *xarray = data;
558
559 xa_destroy(xarray);
560 kfree(xarray);
561}
562
563static void __init iov_kunit_load_xarray(struct kunit *test,
564 struct iov_iter *iter, int dir,
565 struct xarray *xarray,
566 struct page **pages, size_t npages)
567{
568 size_t size = 0;
569 int i;
570
571 for (i = 0; i < npages; i++) {
572 void *x = xa_store(xarray, i, pages[i], GFP_KERNEL);
573
574 KUNIT_ASSERT_FALSE(test, xa_is_err(x));
575 size += PAGE_SIZE;
576 }
577 iov_iter_xarray(iter, dir, xarray, 0, size);
578}
579
580static struct xarray *iov_kunit_create_xarray(struct kunit *test)
581{
582 struct xarray *xarray;
583
584 xarray = kzalloc_obj(struct xarray);
585 xa_init(xarray);
586 KUNIT_ASSERT_NOT_ERR_OR_NULL(test, xarray);
587 kunit_add_action_or_reset(test, iov_kunit_destroy_xarray, xarray);
588 return xarray;
589}
590
591/*
592 * Test copying to a ITER_XARRAY-type iterator.
593 */
594static void __init iov_kunit_copy_to_xarray(struct kunit *test)
595{
596 const struct kvec_test_range *pr;
597 struct iov_iter iter;
598 struct xarray *xarray;
599 struct page **spages, **bpages;
600 u8 *scratch, *buffer;
601 size_t bufsize, npages, size, copied;
602 int i, patt;
603
604 bufsize = 0x100000;
605 npages = bufsize / PAGE_SIZE;
606
607 xarray = iov_kunit_create_xarray(test);
608
609 scratch = iov_kunit_create_buffer(test, &spages, npages);
610 for (i = 0; i < bufsize; i++)
611 scratch[i] = pattern(i);
612
613 buffer = iov_kunit_create_buffer(test, &bpages, npages);
614 memset(buffer, 0, bufsize);
615
616 iov_kunit_load_xarray(test, &iter, READ, xarray, bpages, npages);
617
618 i = 0;
619 for (pr = kvec_test_ranges; pr->from >= 0; pr++) {
620 size = pr->to - pr->from;
621 KUNIT_ASSERT_LE(test, pr->to, bufsize);
622
623 iov_iter_xarray(&iter, READ, xarray, pr->from, size);
624 copied = copy_to_iter(scratch + i, size, &iter);
625
626 KUNIT_EXPECT_EQ(test, copied, size);
627 KUNIT_EXPECT_EQ(test, iter.count, 0);
628 KUNIT_EXPECT_EQ(test, iter.iov_offset, size);
629 i += size;
630 }
631
632 /* Build the expected image in the scratch buffer. */
633 patt = 0;
634 memset(scratch, 0, bufsize);
635 for (pr = kvec_test_ranges; pr->from >= 0; pr++)
636 for (i = pr->from; i < pr->to; i++)
637 scratch[i] = pattern(patt++);
638
639 /* Compare the images */
640 for (i = 0; i < bufsize; i++) {
641 KUNIT_EXPECT_EQ_MSG(test, buffer[i], scratch[i], "at i=%x", i);
642 if (buffer[i] != scratch[i])
643 return;
644 }
645
646 KUNIT_SUCCEED(test);
647}
648
649/*
650 * Test copying from a ITER_XARRAY-type iterator.
651 */
652static void __init iov_kunit_copy_from_xarray(struct kunit *test)
653{
654 const struct kvec_test_range *pr;
655 struct iov_iter iter;
656 struct xarray *xarray;
657 struct page **spages, **bpages;
658 u8 *scratch, *buffer;
659 size_t bufsize, npages, size, copied;
660 int i, j;
661
662 bufsize = 0x100000;
663 npages = bufsize / PAGE_SIZE;
664
665 xarray = iov_kunit_create_xarray(test);
666
667 buffer = iov_kunit_create_buffer(test, &bpages, npages);
668 for (i = 0; i < bufsize; i++)
669 buffer[i] = pattern(i);
670
671 scratch = iov_kunit_create_buffer(test, &spages, npages);
672 memset(scratch, 0, bufsize);
673
674 iov_kunit_load_xarray(test, &iter, READ, xarray, bpages, npages);
675
676 i = 0;
677 for (pr = kvec_test_ranges; pr->from >= 0; pr++) {
678 size = pr->to - pr->from;
679 KUNIT_ASSERT_LE(test, pr->to, bufsize);
680
681 iov_iter_xarray(&iter, WRITE, xarray, pr->from, size);
682 copied = copy_from_iter(scratch + i, size, &iter);
683
684 KUNIT_EXPECT_EQ(test, copied, size);
685 KUNIT_EXPECT_EQ(test, iter.count, 0);
686 KUNIT_EXPECT_EQ(test, iter.iov_offset, size);
687 i += size;
688 }
689
690 /* Build the expected image in the main buffer. */
691 i = 0;
692 memset(buffer, 0, bufsize);
693 for (pr = kvec_test_ranges; pr->from >= 0; pr++) {
694 for (j = pr->from; j < pr->to; j++) {
695 buffer[i++] = pattern(j);
696 if (i >= bufsize)
697 goto stop;
698 }
699 }
700stop:
701
702 /* Compare the images */
703 for (i = 0; i < bufsize; i++) {
704 KUNIT_EXPECT_EQ_MSG(test, scratch[i], buffer[i], "at i=%x", i);
705 if (scratch[i] != buffer[i])
706 return;
707 }
708
709 KUNIT_SUCCEED(test);
710}
711
712/*
713 * Test the extraction of ITER_KVEC-type iterators.
714 */
715static void __init iov_kunit_extract_pages_kvec(struct kunit *test)
716{
717 const struct kvec_test_range *pr;
718 struct iov_iter iter;
719 struct page **bpages, *pagelist[8], **pages = pagelist;
720 struct kvec kvec[8];
721 u8 *buffer;
722 ssize_t len;
723 size_t bufsize, size = 0, npages;
724 int i, from;
725
726 bufsize = 0x100000;
727 npages = bufsize / PAGE_SIZE;
728
729 buffer = iov_kunit_create_buffer(test, &bpages, npages);
730
731 iov_kunit_load_kvec(test, &iter, READ, kvec, ARRAY_SIZE(kvec),
732 buffer, bufsize, kvec_test_ranges);
733 size = iter.count;
734
735 pr = kvec_test_ranges;
736 from = pr->from;
737 do {
738 size_t offset0 = LONG_MAX;
739
740 for (i = 0; i < ARRAY_SIZE(pagelist); i++)
741 pagelist[i] = (void *)(unsigned long)0xaa55aa55aa55aa55ULL;
742
743 len = iov_iter_extract_pages(&iter, &pages, 100 * 1024,
744 ARRAY_SIZE(pagelist), 0, &offset0);
745 KUNIT_EXPECT_GE(test, len, 0);
746 if (len < 0)
747 break;
748 KUNIT_EXPECT_GE(test, (ssize_t)offset0, 0);
749 KUNIT_EXPECT_LT(test, offset0, PAGE_SIZE);
750 KUNIT_EXPECT_LE(test, len, size);
751 KUNIT_EXPECT_EQ(test, iter.count, size - len);
752 size -= len;
753
754 if (len == 0)
755 break;
756
757 for (i = 0; i < ARRAY_SIZE(pagelist); i++) {
758 struct page *p;
759 ssize_t part = min_t(ssize_t, len, PAGE_SIZE - offset0);
760 int ix;
761
762 KUNIT_ASSERT_GE(test, part, 0);
763 while (from == pr->to) {
764 pr++;
765 from = pr->from;
766 if (from < 0)
767 goto stop;
768 }
769 ix = from / PAGE_SIZE;
770 KUNIT_ASSERT_LT(test, ix, npages);
771 p = bpages[ix];
772 KUNIT_EXPECT_PTR_EQ(test, pagelist[i], p);
773 KUNIT_EXPECT_EQ(test, offset0, from % PAGE_SIZE);
774 from += part;
775 len -= part;
776 KUNIT_ASSERT_GE(test, len, 0);
777 if (len == 0)
778 break;
779 offset0 = 0;
780 }
781
782 if (test->status == KUNIT_FAILURE)
783 break;
784 } while (iov_iter_count(&iter) > 0);
785
786stop:
787 KUNIT_EXPECT_EQ(test, size, 0);
788 KUNIT_EXPECT_EQ(test, iter.count, 0);
789 KUNIT_SUCCEED(test);
790}
791
792/*
793 * Test the extraction of ITER_BVEC-type iterators.
794 */
795static void __init iov_kunit_extract_pages_bvec(struct kunit *test)
796{
797 const struct bvec_test_range *pr;
798 struct iov_iter iter;
799 struct page **bpages, *pagelist[8], **pages = pagelist;
800 struct bio_vec bvec[8];
801 ssize_t len;
802 size_t bufsize, size = 0, npages;
803 int i, from;
804
805 bufsize = 0x100000;
806 npages = bufsize / PAGE_SIZE;
807
808 iov_kunit_create_buffer(test, &bpages, npages);
809 iov_kunit_load_bvec(test, &iter, READ, bvec, ARRAY_SIZE(bvec),
810 bpages, npages, bufsize, bvec_test_ranges);
811 size = iter.count;
812
813 pr = bvec_test_ranges;
814 from = pr->from;
815 do {
816 size_t offset0 = LONG_MAX;
817
818 for (i = 0; i < ARRAY_SIZE(pagelist); i++)
819 pagelist[i] = (void *)(unsigned long)0xaa55aa55aa55aa55ULL;
820
821 len = iov_iter_extract_pages(&iter, &pages, 100 * 1024,
822 ARRAY_SIZE(pagelist), 0, &offset0);
823 KUNIT_EXPECT_GE(test, len, 0);
824 if (len < 0)
825 break;
826 KUNIT_EXPECT_GE(test, (ssize_t)offset0, 0);
827 KUNIT_EXPECT_LT(test, offset0, PAGE_SIZE);
828 KUNIT_EXPECT_LE(test, len, size);
829 KUNIT_EXPECT_EQ(test, iter.count, size - len);
830 size -= len;
831
832 if (len == 0)
833 break;
834
835 for (i = 0; i < ARRAY_SIZE(pagelist); i++) {
836 struct page *p;
837 ssize_t part = min_t(ssize_t, len, PAGE_SIZE - offset0);
838 int ix;
839
840 KUNIT_ASSERT_GE(test, part, 0);
841 while (from == pr->to) {
842 pr++;
843 from = pr->from;
844 if (from < 0)
845 goto stop;
846 }
847 ix = pr->page + from / PAGE_SIZE;
848 KUNIT_ASSERT_LT(test, ix, npages);
849 p = bpages[ix];
850 KUNIT_EXPECT_PTR_EQ(test, pagelist[i], p);
851 KUNIT_EXPECT_EQ(test, offset0, from % PAGE_SIZE);
852 from += part;
853 len -= part;
854 KUNIT_ASSERT_GE(test, len, 0);
855 if (len == 0)
856 break;
857 offset0 = 0;
858 }
859
860 if (test->status == KUNIT_FAILURE)
861 break;
862 } while (iov_iter_count(&iter) > 0);
863
864stop:
865 KUNIT_EXPECT_EQ(test, size, 0);
866 KUNIT_EXPECT_EQ(test, iter.count, 0);
867 KUNIT_SUCCEED(test);
868}
869
870/*
871 * Test the extraction of ITER_FOLIOQ-type iterators.
872 */
873static void __init iov_kunit_extract_pages_folioq(struct kunit *test)
874{
875 const struct kvec_test_range *pr;
876 struct folio_queue *folioq;
877 struct iov_iter iter;
878 struct page **bpages, *pagelist[8], **pages = pagelist;
879 ssize_t len;
880 size_t bufsize, size = 0, npages;
881 int i, from;
882
883 bufsize = 0x100000;
884 npages = bufsize / PAGE_SIZE;
885
886 folioq = iov_kunit_create_folioq(test);
887
888 iov_kunit_create_buffer(test, &bpages, npages);
889 iov_kunit_load_folioq(test, &iter, READ, folioq, bpages, npages);
890
891 for (pr = kvec_test_ranges; pr->from >= 0; pr++) {
892 from = pr->from;
893 size = pr->to - from;
894 KUNIT_ASSERT_LE(test, pr->to, bufsize);
895
896 iov_iter_folio_queue(&iter, WRITE, folioq, 0, 0, pr->to);
897 iov_iter_advance(&iter, from);
898
899 do {
900 size_t offset0 = LONG_MAX;
901
902 for (i = 0; i < ARRAY_SIZE(pagelist); i++)
903 pagelist[i] = (void *)(unsigned long)0xaa55aa55aa55aa55ULL;
904
905 len = iov_iter_extract_pages(&iter, &pages, 100 * 1024,
906 ARRAY_SIZE(pagelist), 0, &offset0);
907 KUNIT_EXPECT_GE(test, len, 0);
908 if (len < 0)
909 break;
910 KUNIT_EXPECT_LE(test, len, size);
911 KUNIT_EXPECT_EQ(test, iter.count, size - len);
912 if (len == 0)
913 break;
914 size -= len;
915 KUNIT_EXPECT_GE(test, (ssize_t)offset0, 0);
916 KUNIT_EXPECT_LT(test, offset0, PAGE_SIZE);
917
918 for (i = 0; i < ARRAY_SIZE(pagelist); i++) {
919 struct page *p;
920 ssize_t part = min_t(ssize_t, len, PAGE_SIZE - offset0);
921 int ix;
922
923 KUNIT_ASSERT_GE(test, part, 0);
924 ix = from / PAGE_SIZE;
925 KUNIT_ASSERT_LT(test, ix, npages);
926 p = bpages[ix];
927 KUNIT_EXPECT_PTR_EQ(test, pagelist[i], p);
928 KUNIT_EXPECT_EQ(test, offset0, from % PAGE_SIZE);
929 from += part;
930 len -= part;
931 KUNIT_ASSERT_GE(test, len, 0);
932 if (len == 0)
933 break;
934 offset0 = 0;
935 }
936
937 if (test->status == KUNIT_FAILURE)
938 goto stop;
939 } while (iov_iter_count(&iter) > 0);
940
941 KUNIT_EXPECT_EQ(test, size, 0);
942 KUNIT_EXPECT_EQ(test, iter.count, 0);
943 }
944
945stop:
946 KUNIT_SUCCEED(test);
947}
948
949/*
950 * Test the extraction of ITER_XARRAY-type iterators.
951 */
952static void __init iov_kunit_extract_pages_xarray(struct kunit *test)
953{
954 const struct kvec_test_range *pr;
955 struct iov_iter iter;
956 struct xarray *xarray;
957 struct page **bpages, *pagelist[8], **pages = pagelist;
958 ssize_t len;
959 size_t bufsize, size = 0, npages;
960 int i, from;
961
962 bufsize = 0x100000;
963 npages = bufsize / PAGE_SIZE;
964
965 xarray = iov_kunit_create_xarray(test);
966
967 iov_kunit_create_buffer(test, &bpages, npages);
968 iov_kunit_load_xarray(test, &iter, READ, xarray, bpages, npages);
969
970 for (pr = kvec_test_ranges; pr->from >= 0; pr++) {
971 from = pr->from;
972 size = pr->to - from;
973 KUNIT_ASSERT_LE(test, pr->to, bufsize);
974
975 iov_iter_xarray(&iter, WRITE, xarray, from, size);
976
977 do {
978 size_t offset0 = LONG_MAX;
979
980 for (i = 0; i < ARRAY_SIZE(pagelist); i++)
981 pagelist[i] = (void *)(unsigned long)0xaa55aa55aa55aa55ULL;
982
983 len = iov_iter_extract_pages(&iter, &pages, 100 * 1024,
984 ARRAY_SIZE(pagelist), 0, &offset0);
985 KUNIT_EXPECT_GE(test, len, 0);
986 if (len < 0)
987 break;
988 KUNIT_EXPECT_LE(test, len, size);
989 KUNIT_EXPECT_EQ(test, iter.count, size - len);
990 if (len == 0)
991 break;
992 size -= len;
993 KUNIT_EXPECT_GE(test, (ssize_t)offset0, 0);
994 KUNIT_EXPECT_LT(test, offset0, PAGE_SIZE);
995
996 for (i = 0; i < ARRAY_SIZE(pagelist); i++) {
997 struct page *p;
998 ssize_t part = min_t(ssize_t, len, PAGE_SIZE - offset0);
999 int ix;
1000
1001 KUNIT_ASSERT_GE(test, part, 0);
1002 ix = from / PAGE_SIZE;
1003 KUNIT_ASSERT_LT(test, ix, npages);
1004 p = bpages[ix];
1005 KUNIT_EXPECT_PTR_EQ(test, pagelist[i], p);
1006 KUNIT_EXPECT_EQ(test, offset0, from % PAGE_SIZE);
1007 from += part;
1008 len -= part;
1009 KUNIT_ASSERT_GE(test, len, 0);
1010 if (len == 0)
1011 break;
1012 offset0 = 0;
1013 }
1014
1015 if (test->status == KUNIT_FAILURE)
1016 goto stop;
1017 } while (iov_iter_count(&iter) > 0);
1018
1019 KUNIT_EXPECT_EQ(test, size, 0);
1020 KUNIT_EXPECT_EQ(test, iter.count, 0);
1021 KUNIT_EXPECT_EQ(test, iter.iov_offset, pr->to - pr->from);
1022 }
1023
1024stop:
1025 KUNIT_SUCCEED(test);
1026}
1027
1028struct iov_kunit_iter_to_sg_data {
1029 struct sg_table *sgt;
1030 u8 *buffer, *scratch;
1031 u8 __user *ubuf;
1032 struct page **pages;
1033 size_t npages;
1034};
1035
1036static void __init
1037iov_kunit_iter_unpin_sgt(void *data)
1038{
1039 struct sg_table *sgt = data;
1040
1041 for (unsigned int i = 0; i < sgt->nents; ++i)
1042 unpin_user_page(sg_page(&sgt->sgl[i]));
1043}
1044
1045static void __init
1046iov_kunit_iter_to_sg_init(struct kunit *test, size_t bufsize, bool user,
1047 struct iov_kunit_iter_to_sg_data *data)
1048{
1049 struct page **spages;
1050 struct scatterlist *sg;
1051 unsigned long uaddr;
1052 size_t i;
1053
1054 data->npages = bufsize / PAGE_SIZE;
1055 sg = kunit_kmalloc_array(test, data->npages, sizeof(*sg), GFP_KERNEL);
1056 KUNIT_ASSERT_NOT_ERR_OR_NULL(test, sg);
1057 sg_init_table(sg, data->npages);
1058 data->sgt = kunit_kzalloc(test, sizeof(*data->sgt), GFP_KERNEL);
1059 KUNIT_ASSERT_NOT_ERR_OR_NULL(test, data->sgt);
1060 data->sgt->orig_nents = 0;
1061 data->sgt->sgl = sg;
1062
1063 data->buffer = NULL;
1064 data->ubuf = NULL;
1065 if (user) {
1066 uaddr = kunit_vm_mmap(test, NULL, 0, bufsize,
1067 PROT_READ | PROT_WRITE,
1068 MAP_ANONYMOUS | MAP_PRIVATE, 0);
1069 KUNIT_ASSERT_NE(test, uaddr, 0);
1070 data->ubuf = (u8 __user *)uaddr;
1071 for (i = 0; i < bufsize; ++i)
1072 put_user(pattern(i), data->ubuf + i);
1073 } else {
1074 data->buffer = iov_kunit_create_buffer(test, &data->pages,
1075 data->npages);
1076 for (i = 0; i < bufsize; ++i)
1077 data->buffer[i] = pattern(i);
1078 }
1079 data->scratch = iov_kunit_create_buffer(test, &spages, data->npages);
1080 memset(data->scratch, 0, bufsize);
1081}
1082
1083static void __init
1084iov_kunit_iter_to_sg_check(struct kunit *test, struct iov_iter *iter,
1085 size_t bufsize,
1086 struct iov_kunit_iter_to_sg_data *data)
1087{
1088 static const size_t tail = 16 * PAGE_SIZE;
1089 size_t i;
1090
1091 KUNIT_ASSERT_LT(test, tail, bufsize);
1092
1093 if (iov_iter_extract_will_pin(iter))
1094 kunit_add_action_or_reset(test, iov_kunit_iter_unpin_sgt,
1095 data->sgt);
1096
1097 i = extract_iter_to_sg(iter, bufsize, data->sgt, 0, 0);
1098 KUNIT_ASSERT_EQ(test, i, 0);
1099 KUNIT_ASSERT_EQ(test, data->sgt->nents, 0);
1100
1101 i = extract_iter_to_sg(iter, bufsize - tail, data->sgt, 1, 0);
1102 KUNIT_ASSERT_LE(test, i, bufsize - tail);
1103 KUNIT_ASSERT_EQ(test, data->sgt->nents, 1);
1104
1105 i += extract_iter_to_sg(iter, bufsize - tail - i, data->sgt,
1106 data->npages - data->sgt->nents, 0);
1107 KUNIT_ASSERT_EQ(test, i, bufsize - tail);
1108 KUNIT_ASSERT_LE(test, data->sgt->nents, data->npages);
1109
1110 i += extract_iter_to_sg(iter, tail, data->sgt,
1111 data->npages - data->sgt->nents, 0);
1112 KUNIT_ASSERT_EQ(test, i, bufsize);
1113 KUNIT_ASSERT_LE(test, data->sgt->nents, data->npages);
1114
1115 sg_mark_end(&data->sgt->sgl[data->sgt->nents - 1]);
1116
1117 i = sg_copy_to_buffer(data->sgt->sgl, data->sgt->nents,
1118 data->scratch, bufsize);
1119 KUNIT_ASSERT_EQ(test, i, bufsize);
1120
1121 for (i = 0; i < bufsize; ++i) {
1122 KUNIT_EXPECT_EQ_MSG(test, data->scratch[i], pattern(i),
1123 "at i=%zx", i);
1124 if (data->scratch[i] != pattern(i))
1125 break;
1126 }
1127
1128 KUNIT_EXPECT_EQ(test, i, bufsize);
1129}
1130
1131static void __init iov_kunit_iter_to_sg_kvec(struct kunit *test)
1132{
1133 struct iov_kunit_iter_to_sg_data data;
1134 struct iov_iter iter;
1135 struct kvec kvec;
1136 size_t bufsize;
1137
1138 bufsize = 0x200000;
1139 iov_kunit_iter_to_sg_init(test, bufsize, false, &data);
1140
1141 kvec.iov_base = data.buffer;
1142 kvec.iov_len = bufsize;
1143 iov_iter_kvec(&iter, READ, &kvec, 1, bufsize);
1144
1145 iov_kunit_iter_to_sg_check(test, &iter, bufsize, &data);
1146}
1147
1148static void __init iov_kunit_iter_to_sg_bvec(struct kunit *test)
1149{
1150 struct iov_kunit_iter_to_sg_data data;
1151 struct page *p, *can_merge = NULL;
1152 size_t i, k, bufsize;
1153 struct bio_vec *bvec;
1154 struct iov_iter iter;
1155
1156 bufsize = 0x200000;
1157 iov_kunit_iter_to_sg_init(test, bufsize, false, &data);
1158
1159 bvec = kunit_kmalloc_array(test, data.npages, sizeof(*bvec),
1160 GFP_KERNEL);
1161 KUNIT_ASSERT_NOT_ERR_OR_NULL(test, bvec);
1162 k = 0;
1163 for (i = 0; i < data.npages; ++i) {
1164 p = data.pages[i];
1165 if (p == can_merge)
1166 bvec[k-1].bv_len += PAGE_SIZE;
1167 else
1168 bvec_set_page(&bvec[k++], p, PAGE_SIZE, 0);
1169 can_merge = p + 1;
1170 }
1171 iov_iter_bvec(&iter, READ, bvec, k, bufsize);
1172
1173 iov_kunit_iter_to_sg_check(test, &iter, bufsize, &data);
1174}
1175
1176static void __init iov_kunit_iter_to_sg_folioq(struct kunit *test)
1177{
1178 struct iov_kunit_iter_to_sg_data data;
1179 struct folio_queue *folioq;
1180 struct iov_iter iter;
1181 size_t bufsize;
1182
1183 bufsize = 0x200000;
1184 iov_kunit_iter_to_sg_init(test, bufsize, false, &data);
1185
1186 folioq = iov_kunit_create_folioq(test);
1187 iov_kunit_load_folioq(test, &iter, READ, folioq, data.pages,
1188 data.npages);
1189
1190 iov_kunit_iter_to_sg_check(test, &iter, bufsize, &data);
1191}
1192
1193static void __init iov_kunit_iter_to_sg_xarray(struct kunit *test)
1194{
1195 struct iov_kunit_iter_to_sg_data data;
1196 struct xarray *xarray;
1197 struct iov_iter iter;
1198 size_t bufsize;
1199
1200 bufsize = 0x200000;
1201 iov_kunit_iter_to_sg_init(test, bufsize, false, &data);
1202
1203 xarray = iov_kunit_create_xarray(test);
1204 iov_kunit_load_xarray(test, &iter, READ, xarray, data.pages,
1205 data.npages);
1206
1207 iov_kunit_iter_to_sg_check(test, &iter, bufsize, &data);
1208}
1209
1210static void __init iov_kunit_iter_to_sg_ubuf(struct kunit *test)
1211{
1212 struct iov_kunit_iter_to_sg_data data;
1213 struct iov_iter iter;
1214 size_t bufsize;
1215
1216 bufsize = 0x200000;
1217 iov_kunit_iter_to_sg_init(test, bufsize, true, &data);
1218
1219 iov_iter_ubuf(&iter, READ, data.ubuf, bufsize);
1220
1221 iov_kunit_iter_to_sg_check(test, &iter, bufsize, &data);
1222}
1223
1224static struct kunit_case __refdata iov_kunit_cases[] = {
1225 KUNIT_CASE(iov_kunit_copy_to_kvec),
1226 KUNIT_CASE(iov_kunit_copy_from_kvec),
1227 KUNIT_CASE(iov_kunit_copy_to_bvec),
1228 KUNIT_CASE(iov_kunit_copy_from_bvec),
1229 KUNIT_CASE(iov_kunit_copy_to_folioq),
1230 KUNIT_CASE(iov_kunit_copy_from_folioq),
1231 KUNIT_CASE(iov_kunit_copy_to_xarray),
1232 KUNIT_CASE(iov_kunit_copy_from_xarray),
1233 KUNIT_CASE(iov_kunit_extract_pages_kvec),
1234 KUNIT_CASE(iov_kunit_extract_pages_bvec),
1235 KUNIT_CASE(iov_kunit_extract_pages_folioq),
1236 KUNIT_CASE(iov_kunit_extract_pages_xarray),
1237 KUNIT_CASE(iov_kunit_iter_to_sg_kvec),
1238 KUNIT_CASE(iov_kunit_iter_to_sg_bvec),
1239 KUNIT_CASE(iov_kunit_iter_to_sg_folioq),
1240 KUNIT_CASE(iov_kunit_iter_to_sg_xarray),
1241 KUNIT_CASE(iov_kunit_iter_to_sg_ubuf),
1242 {}
1243};
1244
1245static struct kunit_suite iov_kunit_suite = {
1246 .name = "iov_iter",
1247 .test_cases = iov_kunit_cases,
1248};
1249
1250kunit_test_suites(&iov_kunit_suite);