projects/linux·blob master · refs

2754 lines · 75.1 KiB

1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Userfaultfd unit tests.
4 *
5 * Copyright (C) 2015-2023 Red Hat, Inc.
6 */
7
8#include "uffd-common.h"
9
10#include <linux/fs.h>
11#include <sys/uio.h>
12#include "../../../../mm/gup_test.h"
13
14#ifdef __NR_userfaultfd
15
16/* The unit test doesn't need a large or random size, make it 32MB for now */
17#define UFFD_TEST_MEM_SIZE (32UL << 20)
18
19#define MEM_ANON BIT_ULL(0)
20#define MEM_SHMEM BIT_ULL(1)
21#define MEM_SHMEM_PRIVATE BIT_ULL(2)
22#define MEM_HUGETLB BIT_ULL(3)
23#define MEM_HUGETLB_PRIVATE BIT_ULL(4)
24
25#define MEM_ALL (MEM_ANON | MEM_SHMEM | MEM_SHMEM_PRIVATE | \
26 MEM_HUGETLB | MEM_HUGETLB_PRIVATE)
27
28#define ALIGN_UP(x, align_to) \
29 ((__typeof__(x))((((unsigned long)(x)) + ((align_to)-1)) & ~((align_to)-1)))
30
31#define MAX(a, b) (((a) > (b)) ? (a) : (b))
32
33struct mem_type {
34 const char *name;
35 unsigned int mem_flag;
36 uffd_test_ops_t *mem_ops;
37 bool shared;
38};
39typedef struct mem_type mem_type_t;
40
41mem_type_t mem_types[] = {
42 {
43 .name = "anon",
44 .mem_flag = MEM_ANON,
45 .mem_ops = &anon_uffd_test_ops,
46 .shared = false,
47 },
48 {
49 .name = "shmem",
50 .mem_flag = MEM_SHMEM,
51 .mem_ops = &shmem_uffd_test_ops,
52 .shared = true,
53 },
54 {
55 .name = "shmem-private",
56 .mem_flag = MEM_SHMEM_PRIVATE,
57 .mem_ops = &shmem_uffd_test_ops,
58 .shared = false,
59 },
60 {
61 .name = "hugetlb",
62 .mem_flag = MEM_HUGETLB,
63 .mem_ops = &hugetlb_uffd_test_ops,
64 .shared = true,
65 },
66 {
67 .name = "hugetlb-private",
68 .mem_flag = MEM_HUGETLB_PRIVATE,
69 .mem_ops = &hugetlb_uffd_test_ops,
70 .shared = false,
71 },
72};
73
74/* Arguments to be passed over to each uffd unit test */
75struct uffd_test_args {
76 mem_type_t *mem_type;
77};
78typedef struct uffd_test_args uffd_test_args_t;
79
80/* Returns: UFFD_TEST_* */
81typedef void (*uffd_test_fn)(uffd_global_test_opts_t *, uffd_test_args_t *);
82
83typedef struct {
84 const char *name;
85 uffd_test_fn uffd_fn;
86 unsigned int mem_targets;
87 uint64_t uffd_feature_required;
88 uffd_test_case_ops_t *test_case_ops;
89} uffd_test_case_t;
90
91static char current_test[256];
92
93static void uffd_test_pass(void)
94{
95 ksft_test_result_pass("%s\n", current_test);
96}
97
98#define uffd_test_start(...) do { \
99 snprintf(current_test, sizeof(current_test), __VA_ARGS__); \
100 } while (0)
101
102#define uffd_test_fail(fmt, ...) do { \
103 ksft_print_msg("failed reason: [" fmt "]\n", ##__VA_ARGS__); \
104 ksft_test_result_fail("%s\n", current_test); \
105 } while (0)
106
107static void uffd_test_skip(const char *message)
108{
109 ksft_test_result_skip("%s (%s)\n", current_test, message);
110}
111
112static void test_uffd_api(bool use_dev)
113{
114 const uint64_t expected_ioctls =
115 BIT_ULL(_UFFDIO_REGISTER) |
116 BIT_ULL(_UFFDIO_UNREGISTER) |
117 BIT_ULL(_UFFDIO_API);
118 struct uffdio_api uffdio_api;
119 int uffd;
120
121 uffd_test_start("UFFDIO_API (with %s)",
122 use_dev ? "/dev/userfaultfd" : "syscall");
123
124 if (use_dev)
125 uffd = uffd_open_dev(UFFD_FLAGS);
126 else
127 uffd = uffd_open_sys(UFFD_FLAGS);
128 if (uffd < 0) {
129 uffd_test_skip("cannot open userfaultfd handle");
130 return;
131 }
132
133 /* Test wrong UFFD_API */
134 uffdio_api.api = 0xab;
135 uffdio_api.features = 0;
136 if (ioctl(uffd, UFFDIO_API, &uffdio_api) == 0) {
137 uffd_test_fail("UFFDIO_API should fail with wrong api but didn't");
138 goto out;
139 }
140
141 /* Test wrong feature bit */
142 uffdio_api.api = UFFD_API;
143 uffdio_api.features = BIT_ULL(63);
144 if (ioctl(uffd, UFFDIO_API, &uffdio_api) == 0) {
145 uffd_test_fail("UFFDIO_API should fail with wrong feature but didn't");
146 goto out;
147 }
148
149 /* Test normal UFFDIO_API */
150 uffdio_api.api = UFFD_API;
151 uffdio_api.features = 0;
152 if (ioctl(uffd, UFFDIO_API, &uffdio_api)) {
153 uffd_test_fail("UFFDIO_API should succeed but failed");
154 goto out;
155 }
156
157 /* Verify returned fd-level ioctls bitmask */
158 if ((uffdio_api.ioctls & expected_ioctls) != expected_ioctls) {
159 uffd_test_fail("UFFDIO_API missing expected ioctls: "
160 "got=0x%"PRIx64", expected=0x%"PRIx64,
161 (uint64_t)uffdio_api.ioctls,
162 expected_ioctls);
163 goto out;
164 }
165
166 /* Test double requests of UFFDIO_API with a random feature set */
167 uffdio_api.features = BIT_ULL(0);
168 if (ioctl(uffd, UFFDIO_API, &uffdio_api) == 0) {
169 uffd_test_fail("UFFDIO_API should reject initialized uffd");
170 goto out;
171 }
172
173 uffd_test_pass();
174out:
175 close(uffd);
176}
177
178
179static bool uffd_feature_supported(uffd_test_case_t *test)
180{
181 uint64_t features;
182
183 if (uffd_get_features(&features))
184 return false;
185
186 return (features & test->uffd_feature_required) ==
187 test->uffd_feature_required;
188}
189
190static int pagemap_open(void)
191{
192 int fd = open("/proc/self/pagemap", O_RDONLY);
193
194 if (fd < 0)
195 err("open pagemap");
196
197 return fd;
198}
199
200/* This macro let __LINE__ works in err() */
201#define pagemap_check_wp(value, wp) do { \
202 if (!!(value & PM_UFFD_WP) != wp) \
203 err("pagemap uffd-wp bit error: 0x%"PRIx64, value); \
204 } while (0)
205
206typedef struct {
207 uffd_global_test_opts_t *gopts;
208 int child_uffd;
209} fork_event_args;
210
211static void *fork_event_consumer(void *data)
212{
213 fork_event_args *args = data;
214 struct uffd_msg msg = { 0 };
215
216 args->gopts->ready_for_fork = true;
217
218 /* Read until a full msg received */
219 while (uffd_read_msg(args->gopts, &msg));
220
221 if (msg.event != UFFD_EVENT_FORK)
222 err("wrong message: %u\n", msg.event);
223
224 /* Just to be properly freed later */
225 args->child_uffd = msg.arg.fork.ufd;
226 return NULL;
227}
228
229typedef struct {
230 int gup_fd;
231 bool pinned;
232} pin_args;
233
234/*
235 * Returns 0 if succeed, <0 for errors. pin_pages() needs to be paired
236 * with unpin_pages(). Currently it needs to be RO longterm pin to satisfy
237 * all needs of the test cases (e.g., trigger unshare, trigger fork() early
238 * CoW, etc.).
239 */
240static int pin_pages(pin_args *args, void *buffer, size_t size)
241{
242 struct pin_longterm_test test = {
243 .addr = (uintptr_t)buffer,
244 .size = size,
245 /* Read-only pins */
246 .flags = 0,
247 };
248
249 if (args->pinned)
250 err("already pinned");
251
252 args->gup_fd = open("/sys/kernel/debug/gup_test", O_RDWR);
253 if (args->gup_fd < 0)
254 return -errno;
255
256 if (ioctl(args->gup_fd, PIN_LONGTERM_TEST_START, &test)) {
257 /* Even if gup_test existed, can be an old gup_test / kernel */
258 close(args->gup_fd);
259 return -errno;
260 }
261 args->pinned = true;
262 return 0;
263}
264
265static void unpin_pages(pin_args *args)
266{
267 if (!args->pinned)
268 err("unpin without pin first");
269 if (ioctl(args->gup_fd, PIN_LONGTERM_TEST_STOP))
270 err("PIN_LONGTERM_TEST_STOP");
271 close(args->gup_fd);
272 args->pinned = false;
273}
274
275static int pagemap_test_fork(uffd_global_test_opts_t *gopts, bool with_event, bool test_pin)
276{
277 fork_event_args args = { .gopts = gopts, .child_uffd = -1 };
278 pthread_t thread;
279 pid_t child;
280 uint64_t value;
281 int fd, result;
282
283 /* Prepare a thread to resolve EVENT_FORK */
284 if (with_event) {
285 gopts->ready_for_fork = false;
286 if (pthread_create(&thread, NULL, fork_event_consumer, &args))
287 err("pthread_create()");
288 while (!gopts->ready_for_fork)
289 ; /* Wait for the poll_thread to start executing before forking */
290 }
291
292 child = fork();
293 if (!child) {
294 /* Open the pagemap fd of the child itself */
295 pin_args args = {};
296
297 fd = pagemap_open();
298
299 if (test_pin && pin_pages(&args, gopts->area_dst, gopts->page_size))
300 /*
301 * Normally when reach here we have pinned in
302 * previous tests, so shouldn't fail anymore
303 */
304 err("pin page failed in child");
305
306 value = pagemap_get_entry(fd, gopts->area_dst);
307 /*
308 * After fork(), we should handle uffd-wp bit differently:
309 *
310 * (1) when with EVENT_FORK, it should persist
311 * (2) when without EVENT_FORK, it should be dropped
312 */
313 pagemap_check_wp(value, with_event);
314 if (test_pin)
315 unpin_pages(&args);
316 /* Succeed */
317 _exit(0);
318 }
319 waitpid(child, &result, 0);
320
321 if (with_event) {
322 if (pthread_join(thread, NULL))
323 err("pthread_join()");
324 if (args.child_uffd < 0)
325 err("Didn't receive child uffd");
326 close(args.child_uffd);
327 }
328
329 return result;
330}
331
332static void uffd_wp_unpopulated_test(uffd_global_test_opts_t *gopts, uffd_test_args_t *args)
333{
334 uint64_t value;
335 int pagemap_fd;
336
337 if (uffd_register(gopts->uffd, gopts->area_dst, gopts->nr_pages * gopts->page_size,
338 false, true, false))
339 err("register failed");
340
341 pagemap_fd = pagemap_open();
342
343 /* Test applying pte marker to anon unpopulated */
344 wp_range(gopts->uffd, (uint64_t)gopts->area_dst, gopts->page_size, true);
345 value = pagemap_get_entry(pagemap_fd, gopts->area_dst);
346 pagemap_check_wp(value, true);
347
348 /* Test unprotect on anon pte marker */
349 wp_range(gopts->uffd, (uint64_t)gopts->area_dst, gopts->page_size, false);
350 value = pagemap_get_entry(pagemap_fd, gopts->area_dst);
351 pagemap_check_wp(value, false);
352
353 /* Test zap on anon marker */
354 wp_range(gopts->uffd, (uint64_t)gopts->area_dst, gopts->page_size, true);
355 if (madvise(gopts->area_dst, gopts->page_size, MADV_DONTNEED))
356 err("madvise(MADV_DONTNEED) failed");
357 value = pagemap_get_entry(pagemap_fd, gopts->area_dst);
358 pagemap_check_wp(value, false);
359
360 /* Test fault in after marker removed */
361 *gopts->area_dst = 1;
362 value = pagemap_get_entry(pagemap_fd, gopts->area_dst);
363 pagemap_check_wp(value, false);
364 /* Drop it to make pte none again */
365 if (madvise(gopts->area_dst, gopts->page_size, MADV_DONTNEED))
366 err("madvise(MADV_DONTNEED) failed");
367
368 /* Test read-zero-page upon pte marker */
369 wp_range(gopts->uffd, (uint64_t)gopts->area_dst, gopts->page_size, true);
370 *(volatile char *)gopts->area_dst;
371 /* Drop it to make pte none again */
372 if (madvise(gopts->area_dst, gopts->page_size, MADV_DONTNEED))
373 err("madvise(MADV_DONTNEED) failed");
374
375 uffd_test_pass();
376}
377
378static void uffd_wp_fork_test_common(uffd_global_test_opts_t *gopts, uffd_test_args_t *args,
379 bool with_event)
380{
381 int pagemap_fd;
382 uint64_t value;
383
384 if (uffd_register(gopts->uffd, gopts->area_dst, gopts->nr_pages * gopts->page_size,
385 false, true, false))
386 err("register failed");
387
388 pagemap_fd = pagemap_open();
389
390 /* Touch the page */
391 *gopts->area_dst = 1;
392 wp_range(gopts->uffd, (uint64_t)gopts->area_dst, gopts->page_size, true);
393 value = pagemap_get_entry(pagemap_fd, gopts->area_dst);
394 pagemap_check_wp(value, true);
395 if (pagemap_test_fork(gopts, with_event, false)) {
396 uffd_test_fail("Detected %s uffd-wp bit in child in present pte",
397 with_event ? "missing" : "stall");
398 goto out;
399 }
400
401 /*
402 * This is an attempt for zapping the pgtable so as to test the
403 * markers.
404 *
405 * For private mappings, PAGEOUT will only work on exclusive ptes
406 * (PM_MMAP_EXCLUSIVE) which we should satisfy.
407 *
408 * For shared, PAGEOUT may not work. Use DONTNEED instead which
409 * plays a similar role of zapping (rather than freeing the page)
410 * to expose pte markers.
411 */
412 if (args->mem_type->shared) {
413 if (madvise(gopts->area_dst, gopts->page_size, MADV_DONTNEED))
414 err("MADV_DONTNEED");
415 } else {
416 /*
417 * NOTE: ignore retval because private-hugetlb doesn't yet
418 * support swapping, so it could fail.
419 */
420 madvise(gopts->area_dst, gopts->page_size, MADV_PAGEOUT);
421 }
422
423 /* Uffd-wp should persist even swapped out */
424 value = pagemap_get_entry(pagemap_fd, gopts->area_dst);
425 pagemap_check_wp(value, true);
426 if (pagemap_test_fork(gopts, with_event, false)) {
427 uffd_test_fail("Detected %s uffd-wp bit in child in zapped pte",
428 with_event ? "missing" : "stall");
429 goto out;
430 }
431
432 /* Unprotect; this tests swap pte modifications */
433 wp_range(gopts->uffd, (uint64_t)gopts->area_dst, gopts->page_size, false);
434 value = pagemap_get_entry(pagemap_fd, gopts->area_dst);
435 pagemap_check_wp(value, false);
436
437 /* Fault in the page from disk */
438 *gopts->area_dst = 2;
439 value = pagemap_get_entry(pagemap_fd, gopts->area_dst);
440 pagemap_check_wp(value, false);
441 uffd_test_pass();
442out:
443 if (uffd_unregister(gopts->uffd, gopts->area_dst, gopts->nr_pages * gopts->page_size))
444 err("unregister failed");
445 close(pagemap_fd);
446}
447
448static void uffd_wp_fork_test(uffd_global_test_opts_t *gopts, uffd_test_args_t *args)
449{
450 uffd_wp_fork_test_common(gopts, args, false);
451}
452
453static void uffd_wp_fork_with_event_test(uffd_global_test_opts_t *gopts, uffd_test_args_t *args)
454{
455 uffd_wp_fork_test_common(gopts, args, true);
456}
457
458static void uffd_wp_fork_pin_test_common(uffd_global_test_opts_t *gopts,
459 uffd_test_args_t *args,
460 bool with_event)
461{
462 int pagemap_fd;
463 pin_args pin_args = {};
464
465 if (uffd_register(gopts->uffd, gopts->area_dst, gopts->page_size, false, true, false))
466 err("register failed");
467
468 pagemap_fd = pagemap_open();
469
470 /* Touch the page */
471 *gopts->area_dst = 1;
472 wp_range(gopts->uffd, (uint64_t)gopts->area_dst, gopts->page_size, true);
473
474 /*
475 * 1. First pin, then fork(). This tests fork() special path when
476 * doing early CoW if the page is private.
477 */
478 if (pin_pages(&pin_args, gopts->area_dst, gopts->page_size)) {
479 uffd_test_skip("Possibly CONFIG_GUP_TEST missing "
480 "or unprivileged");
481 close(pagemap_fd);
482 uffd_unregister(gopts->uffd, gopts->area_dst, gopts->page_size);
483 return;
484 }
485
486 if (pagemap_test_fork(gopts, with_event, false)) {
487 uffd_test_fail("Detected %s uffd-wp bit in early CoW of fork()",
488 with_event ? "missing" : "stall");
489 unpin_pages(&pin_args);
490 goto out;
491 }
492
493 unpin_pages(&pin_args);
494
495 /*
496 * 2. First fork(), then pin (in the child, where test_pin==true).
497 * This tests COR, aka, page unsharing on private memories.
498 */
499 if (pagemap_test_fork(gopts, with_event, true)) {
500 uffd_test_fail("Detected %s uffd-wp bit when RO pin",
501 with_event ? "missing" : "stall");
502 goto out;
503 }
504 uffd_test_pass();
505out:
506 if (uffd_unregister(gopts->uffd, gopts->area_dst, gopts->page_size))
507 err("register failed");
508 close(pagemap_fd);
509}
510
511static void uffd_wp_fork_pin_test(uffd_global_test_opts_t *gopts, uffd_test_args_t *args)
512{
513 uffd_wp_fork_pin_test_common(gopts, args, false);
514}
515
516static void uffd_wp_fork_pin_with_event_test(uffd_global_test_opts_t *gopts, uffd_test_args_t *args)
517{
518 uffd_wp_fork_pin_test_common(gopts, args, true);
519}
520
521static void check_memory_contents(uffd_global_test_opts_t *gopts, char *p)
522{
523 unsigned long i, j;
524 uint8_t expected_byte;
525
526 for (i = 0; i < gopts->nr_pages; ++i) {
527 expected_byte = ~((uint8_t)(i % ((uint8_t)-1)));
528 for (j = 0; j < gopts->page_size; j++) {
529 uint8_t v = *(uint8_t *)(p + (i * gopts->page_size) + j);
530 if (v != expected_byte)
531 err("unexpected page contents");
532 }
533 }
534}
535
536static void uffd_minor_test_common(uffd_global_test_opts_t *gopts, bool test_collapse, bool test_wp)
537{
538 unsigned long p;
539 pthread_t uffd_mon;
540 char c = '\0';
541 struct uffd_args args = { 0 };
542 args.gopts = gopts;
543
544 /*
545 * NOTE: MADV_COLLAPSE is not yet compatible with WP, so testing
546 * both do not make much sense.
547 */
548 assert(!(test_collapse && test_wp));
549
550 if (uffd_register(gopts->uffd, gopts->area_dst_alias, gopts->nr_pages * gopts->page_size,
551 /* NOTE! MADV_COLLAPSE may not work with uffd-wp */
552 false, test_wp, true))
553 err("register failure");
554
555 /*
556 * After registering with UFFD, populate the non-UFFD-registered side of
557 * the shared mapping. This should *not* trigger any UFFD minor faults.
558 */
559 for (p = 0; p < gopts->nr_pages; ++p)
560 memset(gopts->area_dst + (p * gopts->page_size), p % ((uint8_t)-1),
561 gopts->page_size);
562
563 args.apply_wp = test_wp;
564 if (pthread_create(&uffd_mon, NULL, uffd_poll_thread, &args))
565 err("uffd_poll_thread create");
566
567 /*
568 * Read each of the pages back using the UFFD-registered mapping. We
569 * expect that the first time we touch a page, it will result in a minor
570 * fault. uffd_poll_thread will resolve the fault by bit-flipping the
571 * page's contents, and then issuing a CONTINUE ioctl.
572 */
573 check_memory_contents(gopts, gopts->area_dst_alias);
574
575 if (write(gopts->pipefd[1], &c, sizeof(c)) != sizeof(c))
576 err("pipe write");
577 if (pthread_join(uffd_mon, NULL))
578 err("join() failed");
579
580 if (test_collapse) {
581 if (madvise(gopts->area_dst_alias, gopts->nr_pages * gopts->page_size,
582 MADV_COLLAPSE)) {
583 /* It's fine to fail for this one... */
584 uffd_test_skip("MADV_COLLAPSE failed");
585 return;
586 }
587
588 uffd_test_ops->check_pmd_mapping(gopts,
589 gopts->area_dst,
590 gopts->nr_pages * gopts->page_size /
591 read_pmd_pagesize());
592 /*
593 * This won't cause uffd-fault - it purely just makes sure there
594 * was no corruption.
595 */
596 check_memory_contents(gopts, gopts->area_dst_alias);
597 }
598
599 if (args.missing_faults != 0 || args.minor_faults != gopts->nr_pages)
600 uffd_test_fail("stats check error");
601 else
602 uffd_test_pass();
603}
604
605void uffd_minor_test(uffd_global_test_opts_t *gopts, uffd_test_args_t *args)
606{
607 uffd_minor_test_common(gopts, false, false);
608}
609
610void uffd_minor_wp_test(uffd_global_test_opts_t *gopts, uffd_test_args_t *args)
611{
612 uffd_minor_test_common(gopts, false, true);
613}
614
615void uffd_minor_collapse_test(uffd_global_test_opts_t *gopts, uffd_test_args_t *args)
616{
617 uffd_minor_test_common(gopts, true, false);
618}
619
620static int uffd_register_rwp(int uffd, void *addr, uint64_t len)
621{
622 struct uffdio_register reg = {
623 .range = { .start = (unsigned long)addr, .len = len },
624 .mode = UFFDIO_REGISTER_MODE_RWP,
625 };
626
627 if (ioctl(uffd, UFFDIO_REGISTER, &reg) == -1)
628 return -errno;
629 return 0;
630}
631
632static void rwprotect_range(int uffd, __u64 start, __u64 len, bool protect)
633{
634 struct uffdio_rwprotect rwp = {
635 .range = { .start = start, .len = len },
636 .mode = protect ? UFFDIO_RWPROTECT_MODE_RWP : 0,
637 };
638
639 if (ioctl(uffd, UFFDIO_RWPROTECT, &rwp))
640 err("UFFDIO_RWPROTECT failed");
641}
642
643static void set_async_mode(int uffd, bool enable)
644{
645 struct uffdio_set_mode mode = { };
646
647 if (enable)
648 mode.enable = UFFD_FEATURE_RWP_ASYNC;
649 else
650 mode.disable = UFFD_FEATURE_RWP_ASYNC;
651
652 if (ioctl(uffd, UFFDIO_SET_MODE, &mode))
653 err("UFFDIO_SET_MODE failed");
654}
655
656/*
657 * Test async RWP faults on anonymous memory.
658 * Populate pages, register MODE_RWP with RWP_ASYNC,
659 * RW-protect, re-access, verify content preserved and no faults delivered.
660 */
661static void uffd_rwp_async_test(uffd_global_test_opts_t *gopts,
662 uffd_test_args_t *args)
663{
664 unsigned long nr_pages = gopts->nr_pages;
665 unsigned long page_size = gopts->page_size;
666 unsigned long p;
667
668 /* Populate all pages with known content */
669 for (p = 0; p < nr_pages; p++)
670 memset(gopts->area_dst + p * page_size, p % 255 + 1, page_size);
671
672 /* Register MODE_RWP */
673 if (uffd_register_rwp(gopts->uffd, gopts->area_dst,
674 nr_pages * page_size))
675 err("register failure");
676
677 /* RW-protect all pages (sets protnone) */
678 rwprotect_range(gopts->uffd, (uint64_t)gopts->area_dst,
679 nr_pages * page_size, true);
680
681 /* Access all pages — should auto-resolve, no faults */
682 for (p = 0; p < nr_pages; p++) {
683 unsigned char *page = (unsigned char *)gopts->area_dst +
684 p * page_size;
685 unsigned char expected = p % 255 + 1;
686
687 if (page[0] != expected) {
688 uffd_test_fail("page %lu content mismatch: %u != %u",
689 p, page[0], expected);
690 return;
691 }
692 }
693
694 uffd_test_pass();
695}
696
697/*
698 * Fault handler for RWP — unprotect the page via UFFDIO_RWPROTECT.
699 */
700static void uffd_handle_rwp_fault(uffd_global_test_opts_t *gopts,
701 struct uffd_msg *msg,
702 struct uffd_args *uargs)
703{
704 if (!(msg->arg.pagefault.flags & UFFD_PAGEFAULT_FLAG_RWP))
705 err("expected RWP fault, got 0x%llx",
706 msg->arg.pagefault.flags);
707
708 rwprotect_range(gopts->uffd, msg->arg.pagefault.address,
709 gopts->page_size, false);
710 uargs->minor_faults++;
711}
712
713/*
714 * Test sync RWP faults on anonymous memory.
715 * Populate pages, register MODE_RWP (sync), RW-protect,
716 * access from worker thread, verify fault delivered, UFFDIO_RWPROTECT resolves.
717 */
718static void uffd_rwp_sync_test(uffd_global_test_opts_t *gopts,
719 uffd_test_args_t *args)
720{
721 unsigned long nr_pages = gopts->nr_pages;
722 unsigned long page_size = gopts->page_size;
723 pthread_t uffd_mon;
724 struct uffd_args uargs = { };
725 bool failed = false;
726 char c = '\0';
727 unsigned long p;
728
729 uargs.gopts = gopts;
730 uargs.handle_fault = uffd_handle_rwp_fault;
731
732 /* Populate all pages */
733 for (p = 0; p < nr_pages; p++)
734 memset(gopts->area_dst + p * page_size, p % 255 + 1, page_size);
735
736 /* Register MODE_RWP */
737 if (uffd_register_rwp(gopts->uffd, gopts->area_dst,
738 nr_pages * page_size))
739 err("register failure");
740
741 /* RW-protect all pages */
742 rwprotect_range(gopts->uffd, (uint64_t)gopts->area_dst,
743 nr_pages * page_size, true);
744
745 /* Start fault handler thread */
746 if (pthread_create(&uffd_mon, NULL, uffd_poll_thread, &uargs))
747 err("uffd_poll_thread create");
748
749 /* Access all pages — triggers sync RWP faults, handler unprotects */
750 for (p = 0; p < nr_pages; p++) {
751 unsigned char *page = (unsigned char *)gopts->area_dst +
752 p * page_size;
753
754 if (page[0] != (p % 255 + 1)) {
755 uffd_test_fail("page %lu content mismatch", p);
756 failed = true;
757 goto out;
758 }
759 }
760
761out:
762 /*
763 * Stop the handler before reading minor_faults: the last fault
764 * resolution rwprotect_range()s before incrementing the counter,
765 * so the main thread can race ahead of the increment.
766 */
767 if (write(gopts->pipefd[1], &c, sizeof(c)) != sizeof(c))
768 err("pipe write");
769 if (pthread_join(uffd_mon, NULL))
770 err("join() failed");
771
772 if (failed)
773 return;
774 if (uargs.minor_faults == 0)
775 uffd_test_fail("expected RWP faults, got 0");
776 else
777 uffd_test_pass();
778}
779
780/*
781 * Test PAGEMAP_SCAN working-set discovery via the "hot" (accessed) scan.
782 *
783 * The working-set primitive is to find pages that were accessed: scan for
784 * PAGE_IS_ACCESSED, which is set once an access clears the protnone+uffd
785 * marker. A VMM treats every access fault as "hot" (RWP here; MINOR/MISSING
786 * for non-resident pages) and reclaims the rest from the backing file.
787 *
788 * We deliberately do NOT use an inverted "cold" scan: that only sees
789 * VMA-resident ptes, so for a file mapping it misses cached-but-unmapped (and
790 * never-faulted, pre-populated) pages, which are pte_none and thus invisible.
791 * Hot tracking + file-level reclaim covers them; a cold pte scan cannot.
792 */
793static void uffd_rwp_pagemap_test(uffd_global_test_opts_t *gopts,
794 uffd_test_args_t *args)
795{
796 unsigned long nr_pages = gopts->nr_pages;
797 unsigned long page_size = gopts->page_size;
798 unsigned long p;
799 struct page_region regions[16];
800 struct pm_scan_arg pm_arg;
801 int pagemap_fd;
802 long ret;
803
804 /* Need at least 4 pages */
805 if (nr_pages < 4) {
806 uffd_test_skip("need at least 4 pages");
807 return;
808 }
809
810 /* Populate all pages */
811 for (p = 0; p < nr_pages; p++)
812 memset(gopts->area_dst + p * page_size, 0xab, page_size);
813
814 /* Register and RW-protect */
815 if (uffd_register_rwp(gopts->uffd, gopts->area_dst,
816 nr_pages * page_size))
817 err("register failure");
818
819 rwprotect_range(gopts->uffd, (uint64_t)gopts->area_dst,
820 nr_pages * page_size, true);
821
822 /* Touch first half of pages to re-activate them (async auto-resolve) */
823 for (p = 0; p < nr_pages / 2; p++) {
824 volatile char *page = gopts->area_dst + p * page_size;
825 (void)*page;
826 }
827
828 uint64_t start = (uint64_t)gopts->area_dst;
829 uint64_t boundary = start + (nr_pages / 2) * page_size;
830 uint64_t end = start + nr_pages * page_size;
831
832 pagemap_fd = open("/proc/self/pagemap", O_RDONLY);
833 if (pagemap_fd < 0)
834 err("open pagemap");
835
836 /*
837 * Hot scan: report the pages that were accessed. PAGE_IS_ACCESSED is
838 * set once the protnone+uffd marker is cleared (by the access, async
839 * auto-resolve here). The touched first half must come back as exactly
840 * one hot region [start, boundary); the untouched second half must not
841 * appear.
842 */
843 memset(&pm_arg, 0, sizeof(pm_arg));
844 pm_arg.size = sizeof(pm_arg);
845 pm_arg.start = start;
846 pm_arg.end = end;
847 pm_arg.vec = (uint64_t)regions;
848 pm_arg.vec_len = ARRAY_SIZE(regions);
849 pm_arg.category_mask = PAGE_IS_ACCESSED;
850 pm_arg.return_mask = PAGE_IS_ACCESSED;
851
852 ret = ioctl(pagemap_fd, PAGEMAP_SCAN, &pm_arg);
853 close(pagemap_fd);
854
855 if (ret < 0) {
856 uffd_test_fail("PAGEMAP_SCAN failed: %s", strerror(errno));
857 return;
858 }
859
860 if (ret != 1 || regions[0].start != start ||
861 regions[0].end != boundary) {
862 uffd_test_fail("hot set wrong: got %ld regions [0x%lx,0x%lx), expected 1 [0x%lx,0x%lx)",
863 ret, (unsigned long)regions[0].start,
864 (unsigned long)regions[0].end,
865 (unsigned long)start, (unsigned long)boundary);
866 return;
867 }
868
869 uffd_test_pass();
870}
871
872/*
873 * Test that RWP protection survives a mprotect(PROT_NONE) ->
874 * mprotect(PROT_READ|PROT_WRITE) round-trip. The uffd-wp bit on a
875 * VM_UFFD_RWP VMA must continue to carry PROT_NONE semantics after
876 * mprotect() changes the base protection; otherwise accesses would
877 * silently succeed and the pagemap bit would stick without a fault
878 * ever clearing it.
879 */
880static void uffd_rwp_mprotect_test(uffd_global_test_opts_t *gopts,
881 uffd_test_args_t *args)
882{
883 unsigned long nr_pages = gopts->nr_pages;
884 unsigned long page_size = gopts->page_size;
885 unsigned long p;
886 struct page_region regions[16];
887 struct pm_scan_arg pm_arg;
888 int pagemap_fd;
889 uint64_t value;
890 long ret;
891
892 /* Populate all pages */
893 for (p = 0; p < nr_pages; p++)
894 memset(gopts->area_dst + p * page_size, 0xab, page_size);
895
896 /* Register and RW-protect the whole range */
897 if (uffd_register_rwp(gopts->uffd, gopts->area_dst,
898 nr_pages * page_size))
899 err("register failure");
900 rwprotect_range(gopts->uffd, (uint64_t)gopts->area_dst,
901 nr_pages * page_size, true);
902
903 /* Round-trip mprotect(): PROT_NONE -> PROT_READ|PROT_WRITE */
904 if (mprotect(gopts->area_dst, nr_pages * page_size, PROT_NONE))
905 err("mprotect() PROT_NONE");
906 if (mprotect(gopts->area_dst, nr_pages * page_size,
907 PROT_READ | PROT_WRITE))
908 err("mprotect() PROT_READ|PROT_WRITE");
909
910 /*
911 * The marker must survive the round-trip; if mprotect() dropped it,
912 * the touches below would not fault and the scan would pass
913 * vacuously.
914 */
915 pagemap_fd = pagemap_open();
916 value = pagemap_get_entry(pagemap_fd, gopts->area_dst);
917 close(pagemap_fd);
918 if (!(value & PM_UFFD_WP)) {
919 uffd_test_fail("RWP marker lost across mprotect()");
920 return;
921 }
922
923 /* Touch every page. Async RWP must auto-resolve each fault. */
924 for (p = 0; p < nr_pages; p++) {
925 volatile char *page = gopts->area_dst + p * page_size;
926 (void)*page;
927 }
928
929 /*
930 * After touching, no page should remain RW-protected. A stuck
931 * uffd-wp bit would mean mprotect() silently dropped PROT_NONE and
932 * the access never faulted.
933 */
934 pagemap_fd = open("/proc/self/pagemap", O_RDONLY);
935 if (pagemap_fd < 0)
936 err("open pagemap");
937
938 memset(&pm_arg, 0, sizeof(pm_arg));
939 pm_arg.size = sizeof(pm_arg);
940 pm_arg.start = (uint64_t)gopts->area_dst;
941 pm_arg.end = (uint64_t)gopts->area_dst + nr_pages * page_size;
942 pm_arg.vec = (uint64_t)regions;
943 pm_arg.vec_len = ARRAY_SIZE(regions);
944 pm_arg.category_mask = PAGE_IS_ACCESSED;
945 pm_arg.category_inverted = PAGE_IS_ACCESSED;
946 pm_arg.return_mask = PAGE_IS_ACCESSED;
947
948 ret = ioctl(pagemap_fd, PAGEMAP_SCAN, &pm_arg);
949 close(pagemap_fd);
950
951 if (ret < 0) {
952 uffd_test_fail("PAGEMAP_SCAN failed: %s", strerror(errno));
953 return;
954 }
955 if (ret != 0) {
956 uffd_test_fail("expected no cold pages after mprotect()+touch, got %ld regions",
957 ret);
958 return;
959 }
960
961 uffd_test_pass();
962}
963
964/*
965 * Test that GUP resolves through protnone PTEs (async mode).
966 * vmsplice() into a pipe pins user pages via get_user_pages_fast() --
967 * unlike write(), which goes through copy_from_user() and ordinary
968 * hardware page faults -- so it exercises gup_can_follow_protnone() on
969 * the RW-protected PTE. In async mode the kernel auto-restores
970 * permissions and GUP returns the page.
971 */
972static void uffd_rwp_gup_test(uffd_global_test_opts_t *gopts,
973 uffd_test_args_t *args)
974{
975 struct iovec iov;
976 char buf;
977 int pipefd[2];
978
979 /* Populate first page with known content */
980 memset(gopts->area_dst, 0xCD, gopts->page_size);
981
982 if (uffd_register_rwp(gopts->uffd, gopts->area_dst, gopts->page_size))
983 err("register failure");
984
985 rwprotect_range(gopts->uffd, (uint64_t)gopts->area_dst,
986 gopts->page_size, true);
987
988 if (pipe(pipefd))
989 err("pipe");
990
991 /*
992 * One byte's worth of iov is enough to GUP the containing page and
993 * keeps the pipe transfer well under any pipe-capacity limit even on
994 * hugetlb-backed runs.
995 */
996 iov.iov_base = gopts->area_dst;
997 iov.iov_len = 1;
998 if (vmsplice(pipefd[1], &iov, 1, 0) != 1) {
999 uffd_test_fail("vmsplice from RW-protected page failed: %s",
1000 strerror(errno));
1001 goto out;
1002 }
1003
1004 if (read(pipefd[0], &buf, 1) != 1) {
1005 uffd_test_fail("read from pipe failed");
1006 goto out;
1007 }
1008
1009 if (buf != (char)0xCD) {
1010 uffd_test_fail("content mismatch: got 0x%02x, expected 0xCD",
1011 (unsigned char)buf);
1012 goto out;
1013 }
1014
1015 uffd_test_pass();
1016out:
1017 close(pipefd[0]);
1018 close(pipefd[1]);
1019}
1020
1021/*
1022 * Test runtime toggle between async and sync modes.
1023 * Start in async mode (detection), flip to sync (eviction), verify faults
1024 * block, resolve them, flip back to async.
1025 */
1026static void uffd_rwp_async_toggle_test(uffd_global_test_opts_t *gopts,
1027 uffd_test_args_t *args)
1028{
1029 unsigned long nr_pages = gopts->nr_pages;
1030 unsigned long page_size = gopts->page_size;
1031 struct uffd_args uargs = { };
1032 pthread_t uffd_mon;
1033 char c = '\0';
1034 unsigned long p;
1035
1036 uargs.gopts = gopts;
1037 uargs.handle_fault = uffd_handle_rwp_fault;
1038
1039 /* Populate */
1040 for (p = 0; p < nr_pages; p++)
1041 memset(gopts->area_dst + p * page_size, p % 255 + 1, page_size);
1042
1043 if (uffd_register_rwp(gopts->uffd, gopts->area_dst,
1044 nr_pages * page_size))
1045 err("register failure");
1046
1047 /* Phase 1: async detection — RW-protect, access first half */
1048 rwprotect_range(gopts->uffd, (uint64_t)gopts->area_dst,
1049 nr_pages * page_size, true);
1050
1051 for (p = 0; p < nr_pages / 2; p++) {
1052 volatile char *page = gopts->area_dst + p * page_size;
1053 (void)*page; /* auto-resolves in async mode */
1054 }
1055
1056 /* Phase 2: flip to sync for eviction */
1057 set_async_mode(gopts->uffd, false);
1058
1059 /* Start handler — will receive faults for cold pages */
1060 if (pthread_create(&uffd_mon, NULL, uffd_poll_thread, &uargs))
1061 err("uffd_poll_thread create");
1062
1063 /* Access second half (cold pages) — should trigger sync faults */
1064 for (p = nr_pages / 2; p < nr_pages; p++) {
1065 unsigned char *page = (unsigned char *)gopts->area_dst +
1066 p * page_size;
1067 if (page[0] != (p % 255 + 1)) {
1068 uffd_test_fail("page %lu content mismatch", p);
1069 goto out;
1070 }
1071 }
1072
1073 /*
1074 * Stop the handler before reading minor_faults: the last fault
1075 * resolution rwprotect_range()s before incrementing the counter,
1076 * so the main thread can race ahead of the increment. Stopping
1077 * here also makes Phase 3 a clean async-only test -- with the
1078 * handler still running it would silently resolve any sync fault
1079 * the kernel erroneously delivers, masking a regression.
1080 */
1081 if (write(gopts->pipefd[1], &c, sizeof(c)) != sizeof(c))
1082 err("pipe write");
1083 if (pthread_join(uffd_mon, NULL))
1084 err("join() failed");
1085
1086 if (uargs.minor_faults == 0) {
1087 uffd_test_fail("expected sync faults, got 0");
1088 return;
1089 }
1090
1091 /* Phase 3: flip back to async */
1092 set_async_mode(gopts->uffd, true);
1093
1094 /* RW-protect and access again — should auto-resolve */
1095 rwprotect_range(gopts->uffd, (uint64_t)gopts->area_dst,
1096 nr_pages * page_size, true);
1097
1098 for (p = 0; p < nr_pages; p++) {
1099 volatile char *page = gopts->area_dst + p * page_size;
1100 (void)*page;
1101 }
1102
1103 uffd_test_pass();
1104 return;
1105out:
1106 if (write(gopts->pipefd[1], &c, sizeof(c)) != sizeof(c))
1107 err("pipe write");
1108 if (pthread_join(uffd_mon, NULL))
1109 err("join() failed");
1110}
1111
1112/*
1113 * Test that RW-protected pages become accessible after closing uffd.
1114 */
1115static void uffd_rwp_close_test(uffd_global_test_opts_t *gopts,
1116 uffd_test_args_t *args)
1117{
1118 unsigned long nr_pages = gopts->nr_pages;
1119 unsigned long page_size = gopts->page_size;
1120 unsigned long p;
1121
1122 /* Populate */
1123 for (p = 0; p < nr_pages; p++)
1124 memset(gopts->area_dst + p * page_size, p % 255 + 1, page_size);
1125
1126 if (uffd_register_rwp(gopts->uffd, gopts->area_dst,
1127 nr_pages * page_size))
1128 err("register failure");
1129
1130 rwprotect_range(gopts->uffd, (uint64_t)gopts->area_dst,
1131 nr_pages * page_size, true);
1132
1133 /* Close uffd — should restore protnone PTEs */
1134 close(gopts->uffd);
1135 gopts->uffd = -1;
1136
1137 /* All pages should be accessible with original content */
1138 for (p = 0; p < nr_pages; p++) {
1139 unsigned char *page = (unsigned char *)gopts->area_dst +
1140 p * page_size;
1141 unsigned char expected = p % 255 + 1;
1142
1143 if (page[0] != expected) {
1144 uffd_test_fail("page %lu not accessible after close", p);
1145 return;
1146 }
1147 }
1148
1149 uffd_test_pass();
1150}
1151
1152/*
1153 * Test that RWP protection is preserved across fork() when
1154 * UFFD_FEATURE_EVENT_FORK is enabled. Without preservation, the child's
1155 * PTEs would lose the uffd-wp marker and RWP-protected accesses would
1156 * silently fall through to do_numa_page().
1157 */
1158static void uffd_rwp_fork_test(uffd_global_test_opts_t *gopts,
1159 uffd_test_args_t *args)
1160{
1161 unsigned long nr_pages = gopts->nr_pages;
1162 unsigned long page_size = gopts->page_size;
1163 int pagemap_fd;
1164 uint64_t value;
1165
1166 if (uffd_register_rwp(gopts->uffd, gopts->area_dst,
1167 nr_pages * page_size))
1168 err("register failed");
1169
1170 /* Populate + RWP-protect */
1171 *gopts->area_dst = 1;
1172 rwprotect_range(gopts->uffd, (uint64_t)gopts->area_dst,
1173 page_size, true);
1174
1175 /* Parent: verify uffd-wp bit is set before fork */
1176 pagemap_fd = pagemap_open();
1177 value = pagemap_get_entry(pagemap_fd, gopts->area_dst);
1178 pagemap_check_wp(value, true);
1179
1180 /*
1181 * Fork with EVENT_FORK: child inherits VM_UFFD_RWP. Child reads
1182 * its own pagemap and must still see the uffd-wp bit set.
1183 */
1184 if (pagemap_test_fork(gopts, true, false)) {
1185 uffd_test_fail("RWP marker lost in child after fork");
1186 goto out;
1187 }
1188
1189 uffd_test_pass();
1190out:
1191 close(pagemap_fd);
1192}
1193
1194/*
1195 * Test that RWP protection on a pinned anon page is preserved across fork().
1196 * Pinning forces copy_present_page() in the child path, which must restore
1197 * PAGE_NONE on top of the uffd bit. Using async mode, a read in the child
1198 * auto-resolves if — and only if — the PTE was actually protnone+uffd; the
1199 * cleared uffd bit afterward proves the fault path ran.
1200 */
1201static void uffd_rwp_fork_pin_test(uffd_global_test_opts_t *gopts,
1202 uffd_test_args_t *args)
1203{
1204 unsigned long page_size = gopts->page_size;
1205 fork_event_args fevent_args = { .gopts = gopts, .child_uffd = -1 };
1206 pin_args pin_args = {};
1207 int pagemap_fd, status;
1208 pthread_t fevent_thread;
1209 uint64_t value;
1210 pid_t child;
1211
1212 if (uffd_register_rwp(gopts->uffd, gopts->area_dst, page_size))
1213 err("register failed");
1214
1215 /* Populate. */
1216 *gopts->area_dst = 1;
1217
1218 /* RO-longterm pin so fork() takes copy_present_page() for this PTE. */
1219 if (pin_pages(&pin_args, gopts->area_dst, page_size)) {
1220 uffd_test_skip("Possibly CONFIG_GUP_TEST missing or unprivileged");
1221 uffd_unregister(gopts->uffd, gopts->area_dst, page_size);
1222 return;
1223 }
1224
1225 /* RWP-protect: PTE is now PAGE_NONE + uffd bit. */
1226 rwprotect_range(gopts->uffd, (uint64_t)gopts->area_dst, page_size, true);
1227
1228 pagemap_fd = pagemap_open();
1229 value = pagemap_get_entry(pagemap_fd, gopts->area_dst);
1230 pagemap_check_wp(value, true);
1231
1232 /*
1233 * UFFD_FEATURE_EVENT_FORK is required so the child inherits
1234 * VM_UFFD_RWP and the marker; without it dup_userfaultfd() resets
1235 * the child VMA and the test would pass for the wrong reason.
1236 * dup_userfaultfd() blocks until the EVENT_FORK message is consumed,
1237 * so spawn a reader before the fork().
1238 */
1239 gopts->ready_for_fork = false;
1240 if (pthread_create(&fevent_thread, NULL, fork_event_consumer,
1241 &fevent_args))
1242 err("pthread_create() for fork event consumer");
1243 while (!gopts->ready_for_fork)
1244 ; /* Wait for consumer to start polling. */
1245
1246 child = fork();
1247 if (child < 0)
1248 err("fork");
1249 if (child == 0) {
1250 volatile char c;
1251 int cfd;
1252
1253 /*
1254 * Precondition: the child must have inherited the marker.
1255 * If copy_present_page() dropped it together with PAGE_NONE,
1256 * the read below would succeed without the fault path and
1257 * the after-read check would pass for the wrong reason.
1258 */
1259 cfd = pagemap_open();
1260 value = pagemap_get_entry(cfd, gopts->area_dst);
1261 if (!(value & PM_UFFD_WP)) {
1262 close(cfd);
1263 _exit(2);
1264 }
1265
1266 /*
1267 * Read the pinned page. Only reaches the fault path if the
1268 * child PTE is protnone + uffd; async mode auto-resolves and
1269 * clears the uffd bit. If copy_present_page() dropped
1270 * PAGE_NONE, the read would silently succeed and the bit
1271 * would still be set.
1272 */
1273 c = *(volatile char *)gopts->area_dst;
1274 (void)c;
1275
1276 value = pagemap_get_entry(cfd, gopts->area_dst);
1277 close(cfd);
1278 _exit((value & PM_UFFD_WP) ? 1 : 0);
1279 }
1280 if (waitpid(child, &status, 0) < 0)
1281 err("waitpid");
1282 if (pthread_join(fevent_thread, NULL))
1283 err("pthread_join() for fork event consumer");
1284 if (fevent_args.child_uffd >= 0)
1285 close(fevent_args.child_uffd);
1286
1287 unpin_pages(&pin_args);
1288 close(pagemap_fd);
1289 if (uffd_unregister(gopts->uffd, gopts->area_dst, page_size))
1290 err("unregister failed");
1291
1292 if (WIFEXITED(status) && WEXITSTATUS(status) == 2) {
1293 uffd_test_fail("RWP marker not inherited by child");
1294 return;
1295 }
1296 if (!WIFEXITED(status) || WEXITSTATUS(status) != 0) {
1297 uffd_test_fail("RWP not enforced in child after pinned fork");
1298 return;
1299 }
1300
1301 uffd_test_pass();
1302}
1303
1304/*
1305 * A non-exclusive (forked, COW-shared) anon page that is RWP-protected and
1306 * then swapped out must keep tracking across swap-in. On the write that
1307 * swaps it back in, do_swap_page() restores PAGE_NONE and the access retries
1308 * through the RWP fault path, instead of being COWed straight to a fresh
1309 * accessible page -- which would silently drop the marker for a non-exclusive
1310 * folio. Sync mode lets us observe the fault directly: with the bug, the
1311 * write COWs without delivering any RWP fault.
1312 *
1313 * Needs a swap device; skipped if MADV_PAGEOUT cannot evict the page.
1314 */
1315static void uffd_rwp_swap_cow_test(uffd_global_test_opts_t *gopts,
1316 uffd_test_args_t *args)
1317{
1318 unsigned long page_size = gopts->page_size;
1319 struct uffd_args uargs = { };
1320 int pagemap_fd, go[2], i;
1321 pthread_t uffd_mon;
1322 char c = '\0';
1323 pid_t child;
1324
1325 uargs.gopts = gopts;
1326 uargs.handle_fault = uffd_handle_rwp_fault;
1327
1328 if (uffd_register_rwp(gopts->uffd, gopts->area_dst, page_size))
1329 err("register failed");
1330
1331 /* Populate one page (exclusive at this point). */
1332 *gopts->area_dst = 0x11;
1333
1334 /* RWP-protect: PTE becomes PAGE_NONE + uffd bit (still exclusive). */
1335 rwprotect_range(gopts->uffd, (uint64_t)gopts->area_dst, page_size, true);
1336
1337 /*
1338 * Swap the page out while it is still exclusive: a shared (forked)
1339 * folio does not get reclaimed by MADV_PAGEOUT. Retry, since a hot
1340 * page may just be rotated on the first reclaim pass.
1341 */
1342 pagemap_fd = pagemap_open();
1343 for (i = 0; i < 100; i++) {
1344 if (madvise(gopts->area_dst, page_size, MADV_PAGEOUT))
1345 err("MADV_PAGEOUT");
1346 if (pagemap_is_swapped(pagemap_fd, gopts->area_dst))
1347 break;
1348 usleep(10000);
1349 }
1350 if (!pagemap_is_swapped(pagemap_fd, gopts->area_dst)) {
1351 uffd_test_skip("MADV_PAGEOUT did not swap the page; is swap enabled?");
1352 close(pagemap_fd);
1353 uffd_unregister(gopts->uffd, gopts->area_dst, page_size);
1354 return;
1355 }
1356
1357 /*
1358 * fork() now: the child duplicates the swap entry, so the slot becomes
1359 * non-exclusive. The child parks (keeping the reference) until the
1360 * parent has faulted the page back in.
1361 */
1362 if (pipe(go))
1363 err("pipe");
1364 child = fork();
1365 if (child < 0)
1366 err("fork");
1367 if (child == 0) {
1368 close(go[1]);
1369 read(go[0], &c, 1);
1370 _exit(0);
1371 }
1372 close(go[0]);
1373
1374 if (pthread_create(&uffd_mon, NULL, uffd_poll_thread, &uargs))
1375 err("uffd_poll_thread create");
1376
1377 /*
1378 * Write the page: swaps it back in (do_swap_page) on a non-exclusive
1379 * folio with FAULT_FLAG_WRITE. The marker must survive and deliver an
1380 * RWP fault rather than COW silently.
1381 */
1382 *gopts->area_dst = 0x22;
1383
1384 if (write(gopts->pipefd[1], &c, sizeof(c)) != sizeof(c))
1385 err("pipe write");
1386 if (pthread_join(uffd_mon, NULL))
1387 err("join failed");
1388
1389 if (uargs.minor_faults == 0)
1390 uffd_test_fail("no RWP fault on swapped-in non-exclusive page");
1391 else
1392 uffd_test_pass();
1393
1394 close(pagemap_fd);
1395 if (write(go[1], &c, 1) != 1)
1396 err("child release");
1397 close(go[1]);
1398 waitpid(child, NULL, 0);
1399}
1400
1401/*
1402 * WP and RWP share the uffd-wp PTE bit and cannot coexist in the same VMA.
1403 * Registration requesting both modes must be rejected.
1404 */
1405static void uffd_rwp_wp_exclusive_test(uffd_global_test_opts_t *gopts,
1406 uffd_test_args_t *args)
1407{
1408 unsigned long nr_pages = gopts->nr_pages;
1409 unsigned long page_size = gopts->page_size;
1410 struct uffdio_register reg = { };
1411
1412 reg.range.start = (unsigned long)gopts->area_dst;
1413 reg.range.len = nr_pages * page_size;
1414 reg.mode = UFFDIO_REGISTER_MODE_WP | UFFDIO_REGISTER_MODE_RWP;
1415
1416 if (ioctl(gopts->uffd, UFFDIO_REGISTER, &reg) == 0) {
1417 uffd_test_fail("register with WP|RWP unexpectedly succeeded");
1418 return;
1419 }
1420 if (errno != EINVAL) {
1421 uffd_test_fail("register with WP|RWP: expected EINVAL, got %d",
1422 errno);
1423 return;
1424 }
1425 uffd_test_pass();
1426}
1427
1428static sigjmp_buf jbuf, *sigbuf;
1429
1430static void sighndl(int sig, siginfo_t *siginfo, void *ptr)
1431{
1432 if (sig == SIGBUS) {
1433 if (sigbuf)
1434 siglongjmp(*sigbuf, 1);
1435 abort();
1436 }
1437}
1438
1439/*
1440 * For non-cooperative userfaultfd test we fork() a process that will
1441 * generate pagefaults, will mremap the area monitored by the
1442 * userfaultfd and at last this process will release the monitored
1443 * area.
1444 * For the anonymous and shared memory the area is divided into two
1445 * parts, the first part is accessed before mremap, and the second
1446 * part is accessed after mremap. Since hugetlbfs does not support
1447 * mremap, the entire monitored area is accessed in a single pass for
1448 * HUGETLB_TEST.
1449 * The release of the pages currently generates event for shmem and
1450 * anonymous memory (UFFD_EVENT_REMOVE), hence it is not checked
1451 * for hugetlb.
1452 * For signal test(UFFD_FEATURE_SIGBUS), signal_test = 1, we register
1453 * monitored area, generate pagefaults and test that signal is delivered.
1454 * Use UFFDIO_COPY to allocate missing page and retry. For signal_test = 2
1455 * test robustness use case - we release monitored area, fork a process
1456 * that will generate pagefaults and verify signal is generated.
1457 * This also tests UFFD_FEATURE_EVENT_FORK event along with the signal
1458 * feature. Using monitor thread, verify no userfault events are generated.
1459 */
1460static int faulting_process(uffd_global_test_opts_t *gopts, int signal_test, bool wp)
1461{
1462 unsigned long nr, i;
1463 unsigned long long count;
1464 unsigned long split_nr_pages;
1465 unsigned long lastnr;
1466 struct sigaction act;
1467 volatile unsigned long signalled = 0;
1468
1469 split_nr_pages = (gopts->nr_pages + 1) / 2;
1470
1471 if (signal_test) {
1472 sigbuf = &jbuf;
1473 memset(&act, 0, sizeof(act));
1474 act.sa_sigaction = sighndl;
1475 act.sa_flags = SA_SIGINFO;
1476 if (sigaction(SIGBUS, &act, 0))
1477 err("sigaction");
1478 lastnr = (unsigned long)-1;
1479 }
1480
1481 for (nr = 0; nr < split_nr_pages; nr++) {
1482 volatile int steps = 1;
1483 unsigned long offset = nr * gopts->page_size;
1484
1485 if (signal_test) {
1486 if (sigsetjmp(*sigbuf, 1) != 0) {
1487 if (steps == 1 && nr == lastnr)
1488 err("Signal repeated");
1489
1490 lastnr = nr;
1491 if (signal_test == 1) {
1492 if (steps == 1) {
1493 /* This is a MISSING request */
1494 steps++;
1495 if (copy_page(gopts, offset, wp))
1496 signalled++;
1497 } else {
1498 /* This is a WP request */
1499 assert(steps == 2);
1500 wp_range(gopts->uffd,
1501 (__u64)gopts->area_dst +
1502 offset,
1503 gopts->page_size, false);
1504 }
1505 } else {
1506 signalled++;
1507 continue;
1508 }
1509 }
1510 }
1511
1512 count = *area_count(gopts->area_dst, nr, gopts);
1513 if (count != gopts->count_verify[nr])
1514 err("nr %lu memory corruption %llu %llu\n",
1515 nr, count, gopts->count_verify[nr]);
1516 /*
1517 * Trigger write protection if there is by writing
1518 * the same value back.
1519 */
1520 *area_count(gopts->area_dst, nr, gopts) = count;
1521 }
1522
1523 if (signal_test)
1524 return signalled != split_nr_pages;
1525
1526 gopts->area_dst = mremap(gopts->area_dst, gopts->nr_pages * gopts->page_size,
1527 gopts->nr_pages * gopts->page_size,
1528 MREMAP_MAYMOVE | MREMAP_FIXED,
1529 gopts->area_src);
1530 if (gopts->area_dst == MAP_FAILED)
1531 err("mremap");
1532 /* Reset area_src since we just clobbered it */
1533 gopts->area_src = NULL;
1534
1535 for (; nr < gopts->nr_pages; nr++) {
1536 count = *area_count(gopts->area_dst, nr, gopts);
1537 if (count != gopts->count_verify[nr]) {
1538 err("nr %lu memory corruption %llu %llu\n",
1539 nr, count, gopts->count_verify[nr]);
1540 }
1541 /*
1542 * Trigger write protection if there is by writing
1543 * the same value back.
1544 */
1545 *area_count(gopts->area_dst, nr, gopts) = count;
1546 }
1547
1548 uffd_test_ops->release_pages(gopts, gopts->area_dst);
1549
1550 for (nr = 0; nr < gopts->nr_pages; nr++)
1551 for (i = 0; i < gopts->page_size; i++)
1552 if (*(gopts->area_dst + nr * gopts->page_size + i) != 0)
1553 err("page %lu offset %lu is not zero", nr, i);
1554
1555 return 0;
1556}
1557
1558static void uffd_sigbus_test_common(uffd_global_test_opts_t *gopts, bool wp)
1559{
1560 unsigned long userfaults;
1561 pthread_t uffd_mon;
1562 pid_t pid;
1563 int err;
1564 char c = '\0';
1565 struct uffd_args args = { 0 };
1566 args.gopts = gopts;
1567
1568 gopts->ready_for_fork = false;
1569
1570 fcntl(gopts->uffd, F_SETFL, gopts->uffd_flags | O_NONBLOCK);
1571
1572 if (uffd_register(gopts->uffd, gopts->area_dst, gopts->nr_pages * gopts->page_size,
1573 true, wp, false))
1574 err("register failure");
1575
1576 if (faulting_process(gopts, 1, wp))
1577 err("faulting process failed");
1578
1579 uffd_test_ops->release_pages(gopts, gopts->area_dst);
1580
1581 args.apply_wp = wp;
1582 if (pthread_create(&uffd_mon, NULL, uffd_poll_thread, &args))
1583 err("uffd_poll_thread create");
1584
1585 while (!gopts->ready_for_fork)
1586 ; /* Wait for the poll_thread to start executing before forking */
1587
1588 pid = fork();
1589 if (pid < 0)
1590 err("fork");
1591
1592 if (!pid)
1593 _exit(faulting_process(gopts, 2, wp));
1594
1595 waitpid(pid, &err, 0);
1596 if (err)
1597 err("faulting process failed");
1598 if (write(gopts->pipefd[1], &c, sizeof(c)) != sizeof(c))
1599 err("pipe write");
1600 if (pthread_join(uffd_mon, (void **)&userfaults))
1601 err("pthread_join()");
1602
1603 if (userfaults)
1604 uffd_test_fail("Signal test failed, userfaults: %ld", userfaults);
1605 else
1606 uffd_test_pass();
1607}
1608
1609static void uffd_sigbus_test(uffd_global_test_opts_t *gopts, uffd_test_args_t *args)
1610{
1611 uffd_sigbus_test_common(gopts, false);
1612}
1613
1614static void uffd_sigbus_wp_test(uffd_global_test_opts_t *gopts, uffd_test_args_t *args)
1615{
1616 uffd_sigbus_test_common(gopts, true);
1617}
1618
1619static void uffd_events_test_common(uffd_global_test_opts_t *gopts, bool wp)
1620{
1621 pthread_t uffd_mon;
1622 pid_t pid;
1623 int err;
1624 char c = '\0';
1625 struct uffd_args args = { 0 };
1626 args.gopts = gopts;
1627
1628 gopts->ready_for_fork = false;
1629
1630 fcntl(gopts->uffd, F_SETFL, gopts->uffd_flags | O_NONBLOCK);
1631 if (uffd_register(gopts->uffd, gopts->area_dst, gopts->nr_pages * gopts->page_size,
1632 true, wp, false))
1633 err("register failure");
1634
1635 args.apply_wp = wp;
1636 if (pthread_create(&uffd_mon, NULL, uffd_poll_thread, &args))
1637 err("uffd_poll_thread create");
1638
1639 while (!gopts->ready_for_fork)
1640 ; /* Wait for the poll_thread to start executing before forking */
1641
1642 pid = fork();
1643 if (pid < 0)
1644 err("fork");
1645
1646 if (!pid)
1647 _exit(faulting_process(gopts, 0, wp));
1648
1649 waitpid(pid, &err, 0);
1650 if (err)
1651 err("faulting process failed");
1652 if (write(gopts->pipefd[1], &c, sizeof(c)) != sizeof(c))
1653 err("pipe write");
1654 if (pthread_join(uffd_mon, NULL))
1655 err("pthread_join()");
1656
1657 if (args.missing_faults != gopts->nr_pages)
1658 uffd_test_fail("Fault counts wrong");
1659 else
1660 uffd_test_pass();
1661}
1662
1663static void uffd_events_test(uffd_global_test_opts_t *gopts, uffd_test_args_t *args)
1664{
1665 uffd_events_test_common(gopts, false);
1666}
1667
1668static void uffd_events_wp_test(uffd_global_test_opts_t *gopts, uffd_test_args_t *args)
1669{
1670 uffd_events_test_common(gopts, true);
1671}
1672
1673static void retry_uffdio_zeropage(uffd_global_test_opts_t *gopts,
1674 struct uffdio_zeropage *uffdio_zeropage)
1675{
1676 uffd_test_ops->alias_mapping(gopts, &uffdio_zeropage->range.start,
1677 uffdio_zeropage->range.len,
1678 0);
1679 if (ioctl(gopts->uffd, UFFDIO_ZEROPAGE, uffdio_zeropage)) {
1680 if (uffdio_zeropage->zeropage != -EEXIST)
1681 err("UFFDIO_ZEROPAGE error: %"PRId64,
1682 (int64_t)uffdio_zeropage->zeropage);
1683 } else {
1684 err("UFFDIO_ZEROPAGE error: %"PRId64,
1685 (int64_t)uffdio_zeropage->zeropage);
1686 }
1687}
1688
1689static bool do_uffdio_zeropage(uffd_global_test_opts_t *gopts, bool has_zeropage)
1690{
1691 struct uffdio_zeropage uffdio_zeropage = { 0 };
1692 int ret;
1693 __s64 res;
1694
1695 uffdio_zeropage.range.start = (unsigned long) gopts->area_dst;
1696 uffdio_zeropage.range.len = gopts->page_size;
1697 uffdio_zeropage.mode = 0;
1698 ret = ioctl(gopts->uffd, UFFDIO_ZEROPAGE, &uffdio_zeropage);
1699 res = uffdio_zeropage.zeropage;
1700 if (ret) {
1701 /* real retval in ufdio_zeropage.zeropage */
1702 if (has_zeropage)
1703 err("UFFDIO_ZEROPAGE error: %"PRId64, (int64_t)res);
1704 else if (res != -EINVAL)
1705 err("UFFDIO_ZEROPAGE not -EINVAL");
1706 } else if (has_zeropage) {
1707 if (res != gopts->page_size)
1708 err("UFFDIO_ZEROPAGE unexpected size");
1709 else
1710 retry_uffdio_zeropage(gopts, &uffdio_zeropage);
1711 return true;
1712 } else
1713 err("UFFDIO_ZEROPAGE succeeded");
1714
1715 return false;
1716}
1717
1718/*
1719 * Registers a range with MISSING mode only for zeropage test. Return true
1720 * if UFFDIO_ZEROPAGE supported, false otherwise. Can't use uffd_register()
1721 * because we want to detect .ioctls along the way.
1722 */
1723static bool
1724uffd_register_detect_zeropage(int uffd, void *addr, uint64_t len)
1725{
1726 uint64_t ioctls = 0;
1727
1728 if (uffd_register_with_ioctls(uffd, addr, len, true,
1729 false, false, &ioctls))
1730 err("zeropage register fail");
1731
1732 return ioctls & (1 << _UFFDIO_ZEROPAGE);
1733}
1734
1735/* exercise UFFDIO_ZEROPAGE */
1736static void uffd_zeropage_test(uffd_global_test_opts_t *gopts, uffd_test_args_t *args)
1737{
1738 bool has_zeropage;
1739 int i;
1740
1741 has_zeropage = uffd_register_detect_zeropage(gopts->uffd,
1742 gopts->area_dst,
1743 gopts->page_size);
1744 if (gopts->area_dst_alias)
1745 /* Ignore the retval; we already have it */
1746 uffd_register_detect_zeropage(gopts->uffd, gopts->area_dst_alias, gopts->page_size);
1747
1748 if (do_uffdio_zeropage(gopts, has_zeropage))
1749 for (i = 0; i < gopts->page_size; i++)
1750 if (gopts->area_dst[i] != 0)
1751 err("data non-zero at offset %d\n", i);
1752
1753 if (uffd_unregister(gopts->uffd, gopts->area_dst, gopts->page_size))
1754 err("unregister");
1755
1756 if (gopts->area_dst_alias && uffd_unregister(gopts->uffd,
1757 gopts->area_dst_alias,
1758 gopts->page_size))
1759 err("unregister");
1760
1761 uffd_test_pass();
1762}
1763
1764static void uffd_register_poison(int uffd, void *addr, uint64_t len)
1765{
1766 uint64_t ioctls = 0;
1767 uint64_t expected = (1 << _UFFDIO_COPY) | (1 << _UFFDIO_POISON);
1768
1769 if (uffd_register_with_ioctls(uffd, addr, len, true,
1770 false, false, &ioctls))
1771 err("poison register fail");
1772
1773 if ((ioctls & expected) != expected)
1774 err("registered area doesn't support COPY and POISON ioctls");
1775}
1776
1777static void do_uffdio_poison(uffd_global_test_opts_t *gopts, unsigned long offset)
1778{
1779 struct uffdio_poison uffdio_poison = { 0 };
1780 int ret;
1781 __s64 res;
1782
1783 uffdio_poison.range.start = (unsigned long) gopts->area_dst + offset;
1784 uffdio_poison.range.len = gopts->page_size;
1785 uffdio_poison.mode = 0;
1786 ret = ioctl(gopts->uffd, UFFDIO_POISON, &uffdio_poison);
1787 res = uffdio_poison.updated;
1788
1789 if (ret)
1790 err("UFFDIO_POISON error: %"PRId64, (int64_t)res);
1791 else if (res != gopts->page_size)
1792 err("UFFDIO_POISON unexpected size: %"PRId64, (int64_t)res);
1793}
1794
1795static void uffd_poison_handle_fault(uffd_global_test_opts_t *gopts,
1796 struct uffd_msg *msg,
1797 struct uffd_args *args)
1798{
1799 unsigned long offset;
1800
1801 if (msg->event != UFFD_EVENT_PAGEFAULT)
1802 err("unexpected msg event %u", msg->event);
1803
1804 if (msg->arg.pagefault.flags &
1805 (UFFD_PAGEFAULT_FLAG_WP | UFFD_PAGEFAULT_FLAG_MINOR))
1806 err("unexpected fault type %llu", msg->arg.pagefault.flags);
1807
1808 offset = (char *)(unsigned long)msg->arg.pagefault.address - gopts->area_dst;
1809 offset &= ~(gopts->page_size-1);
1810
1811 /* Odd pages -> copy zeroed page; even pages -> poison. */
1812 if (offset & gopts->page_size)
1813 copy_page(gopts, offset, false);
1814 else
1815 do_uffdio_poison(gopts, offset);
1816}
1817
1818/* Make sure to cover odd/even, and minimum duplications */
1819#define UFFD_POISON_TEST_NPAGES 4
1820
1821static void uffd_poison_test(uffd_global_test_opts_t *gopts, uffd_test_args_t *targs)
1822{
1823 pthread_t uffd_mon;
1824 char c;
1825 struct uffd_args args = { 0 };
1826 struct sigaction act = { 0 };
1827 unsigned long nr_sigbus = 0;
1828 unsigned long nr, poison_pages = UFFD_POISON_TEST_NPAGES;
1829
1830 if (gopts->nr_pages < poison_pages) {
1831 uffd_test_skip("Too less pages for POISON test");
1832 return;
1833 }
1834
1835 args.gopts = gopts;
1836
1837 fcntl(gopts->uffd, F_SETFL, gopts->uffd_flags | O_NONBLOCK);
1838
1839 uffd_register_poison(gopts->uffd, gopts->area_dst, poison_pages * gopts->page_size);
1840 memset(gopts->area_src, 0, poison_pages * gopts->page_size);
1841
1842 args.handle_fault = uffd_poison_handle_fault;
1843 if (pthread_create(&uffd_mon, NULL, uffd_poll_thread, &args))
1844 err("uffd_poll_thread create");
1845
1846 sigbuf = &jbuf;
1847 act.sa_sigaction = sighndl;
1848 act.sa_flags = SA_SIGINFO;
1849 if (sigaction(SIGBUS, &act, 0))
1850 err("sigaction");
1851
1852 for (nr = 0; nr < poison_pages; ++nr) {
1853 unsigned long offset = nr * gopts->page_size;
1854 const char *bytes = (const char *) gopts->area_dst + offset;
1855 const char *i;
1856
1857 if (sigsetjmp(*sigbuf, 1)) {
1858 /*
1859 * Access below triggered a SIGBUS, which was caught by
1860 * sighndl, which then jumped here. Count this SIGBUS,
1861 * and move on to next page.
1862 */
1863 ++nr_sigbus;
1864 continue;
1865 }
1866
1867 for (i = bytes; i < bytes + gopts->page_size; ++i) {
1868 if (*i)
1869 err("nonzero byte in area_dst (%p) at %p: %u",
1870 gopts->area_dst, i, *i);
1871 }
1872 }
1873
1874 if (write(gopts->pipefd[1], &c, sizeof(c)) != sizeof(c))
1875 err("pipe write");
1876 if (pthread_join(uffd_mon, NULL))
1877 err("pthread_join()");
1878
1879 if (nr_sigbus != poison_pages / 2)
1880 err("expected to receive %lu SIGBUS, actually received %lu",
1881 poison_pages / 2, nr_sigbus);
1882
1883 uffd_test_pass();
1884}
1885
1886static void
1887uffd_move_handle_fault_common(uffd_global_test_opts_t *gopts,
1888 struct uffd_msg *msg,
1889 struct uffd_args *args,
1890 unsigned long len)
1891{
1892 unsigned long offset;
1893
1894 if (msg->event != UFFD_EVENT_PAGEFAULT)
1895 err("unexpected msg event %u", msg->event);
1896
1897 if (msg->arg.pagefault.flags &
1898 (UFFD_PAGEFAULT_FLAG_WP | UFFD_PAGEFAULT_FLAG_MINOR | UFFD_PAGEFAULT_FLAG_WRITE))
1899 err("unexpected fault type %llu", msg->arg.pagefault.flags);
1900
1901 offset = (char *)(unsigned long)msg->arg.pagefault.address - gopts->area_dst;
1902 offset &= ~(len-1);
1903
1904 if (move_page(gopts, offset, len))
1905 args->missing_faults++;
1906}
1907
1908static void uffd_move_handle_fault(uffd_global_test_opts_t *gopts, struct uffd_msg *msg,
1909 struct uffd_args *args)
1910{
1911 uffd_move_handle_fault_common(gopts, msg, args, gopts->page_size);
1912}
1913
1914static void uffd_move_pmd_handle_fault(uffd_global_test_opts_t *gopts, struct uffd_msg *msg,
1915 struct uffd_args *args)
1916{
1917 uffd_move_handle_fault_common(gopts, msg, args, read_pmd_pagesize());
1918}
1919
1920static void
1921uffd_move_test_common(uffd_global_test_opts_t *gopts,
1922 uffd_test_args_t *targs,
1923 unsigned long chunk_size,
1924 void (*handle_fault)(struct uffd_global_test_opts *gopts,
1925 struct uffd_msg *msg, struct uffd_args *args)
1926)
1927{
1928 unsigned long nr;
1929 pthread_t uffd_mon;
1930 char c = '\0';
1931 unsigned long long count;
1932 struct uffd_args args = { 0 };
1933 char *orig_area_src = NULL, *orig_area_dst = NULL;
1934 unsigned long step_size, step_count;
1935 unsigned long src_offs = 0;
1936 unsigned long dst_offs = 0;
1937
1938 args.gopts = gopts;
1939
1940 /* Prevent source pages from being mapped more than once */
1941 if (madvise(gopts->area_src, gopts->nr_pages * gopts->page_size, MADV_DONTFORK))
1942 err("madvise(MADV_DONTFORK) failure");
1943
1944 if (uffd_register(gopts->uffd, gopts->area_dst, gopts->nr_pages * gopts->page_size,
1945 true, false, false))
1946 err("register failure");
1947
1948 args.handle_fault = handle_fault;
1949 if (pthread_create(&uffd_mon, NULL, uffd_poll_thread, &args))
1950 err("uffd_poll_thread create");
1951
1952 step_size = chunk_size / gopts->page_size;
1953 step_count = gopts->nr_pages / step_size;
1954
1955 if (chunk_size > gopts->page_size) {
1956 char *aligned_src = ALIGN_UP(gopts->area_src, chunk_size);
1957 char *aligned_dst = ALIGN_UP(gopts->area_dst, chunk_size);
1958
1959 if (aligned_src != gopts->area_src || aligned_dst != gopts->area_dst) {
1960 src_offs = (aligned_src - gopts->area_src) / gopts->page_size;
1961 dst_offs = (aligned_dst - gopts->area_dst) / gopts->page_size;
1962 step_count--;
1963 }
1964 orig_area_src = gopts->area_src;
1965 orig_area_dst = gopts->area_dst;
1966 gopts->area_src = aligned_src;
1967 gopts->area_dst = aligned_dst;
1968 }
1969
1970 /*
1971 * Read each of the pages back using the UFFD-registered mapping. We
1972 * expect that the first time we touch a page, it will result in a missing
1973 * fault. uffd_poll_thread will resolve the fault by moving source
1974 * page to destination.
1975 */
1976 for (nr = 0; nr < step_count * step_size; nr += step_size) {
1977 unsigned long i;
1978
1979 /* Check area_src content */
1980 for (i = 0; i < step_size; i++) {
1981 count = *area_count(gopts->area_src, nr + i, gopts);
1982 if (count != gopts->count_verify[src_offs + nr + i])
1983 err("nr %lu source memory invalid %llu %llu\n",
1984 nr + i, count, gopts->count_verify[src_offs + nr + i]);
1985 }
1986
1987 /* Faulting into area_dst should move the page or the huge page */
1988 for (i = 0; i < step_size; i++) {
1989 count = *area_count(gopts->area_dst, nr + i, gopts);
1990 if (count != gopts->count_verify[dst_offs + nr + i])
1991 err("nr %lu memory corruption %llu %llu\n",
1992 nr, count, gopts->count_verify[dst_offs + nr + i]);
1993 }
1994
1995 /* Re-check area_src content which should be empty */
1996 for (i = 0; i < step_size; i++) {
1997 count = *area_count(gopts->area_src, nr + i, gopts);
1998 if (count != 0)
1999 err("nr %lu move failed %llu %llu\n",
2000 nr, count, gopts->count_verify[src_offs + nr + i]);
2001 }
2002 }
2003 if (chunk_size > gopts->page_size) {
2004 gopts->area_src = orig_area_src;
2005 gopts->area_dst = orig_area_dst;
2006 }
2007
2008 if (write(gopts->pipefd[1], &c, sizeof(c)) != sizeof(c))
2009 err("pipe write");
2010 if (pthread_join(uffd_mon, NULL))
2011 err("join() failed");
2012
2013 if (args.missing_faults != step_count || args.minor_faults != 0)
2014 uffd_test_fail("stats check error");
2015 else
2016 uffd_test_pass();
2017}
2018
2019static void uffd_move_test(uffd_global_test_opts_t *gopts, uffd_test_args_t *targs)
2020{
2021 uffd_move_test_common(gopts, targs, gopts->page_size, uffd_move_handle_fault);
2022}
2023
2024static void uffd_move_pmd_test(uffd_global_test_opts_t *gopts, uffd_test_args_t *targs)
2025{
2026 if (madvise(gopts->area_dst, gopts->nr_pages * gopts->page_size, MADV_HUGEPAGE))
2027 err("madvise(MADV_HUGEPAGE) failure");
2028 uffd_move_test_common(gopts, targs, read_pmd_pagesize(),
2029 uffd_move_pmd_handle_fault);
2030}
2031
2032static void uffd_move_pmd_split_test(uffd_global_test_opts_t *gopts, uffd_test_args_t *targs)
2033{
2034 if (madvise(gopts->area_dst, gopts->nr_pages * gopts->page_size, MADV_NOHUGEPAGE))
2035 err("madvise(MADV_NOHUGEPAGE) failure");
2036 uffd_move_test_common(gopts, targs, read_pmd_pagesize(),
2037 uffd_move_pmd_handle_fault);
2038}
2039
2040static bool
2041uffdio_verify_results(const char *name, int ret, int error, long result)
2042{
2043 /*
2044 * Should always return -1 with errno=EAGAIN, with corresponding
2045 * result field updated in ioctl() args to be -EAGAIN too
2046 * (e.g. copy.copy field for UFFDIO_COPY).
2047 */
2048 if (ret != -1) {
2049 uffd_test_fail("%s should have returned -1", name);
2050 return false;
2051 }
2052
2053 if (error != EAGAIN) {
2054 uffd_test_fail("%s should have errno==EAGAIN", name);
2055 return false;
2056 }
2057
2058 if (result != -EAGAIN) {
2059 uffd_test_fail("%s should have been updated for -EAGAIN",
2060 name);
2061 return false;
2062 }
2063
2064 return true;
2065}
2066
2067/*
2068 * This defines a function to test one ioctl. Note that here "field" can
2069 * be 1 or anything not -EAGAIN. With that initial value set, we can
2070 * verify later that it should be updated by kernel (when -EAGAIN
2071 * returned), by checking whether it is also updated to -EAGAIN.
2072 */
2073#define DEFINE_MMAP_CHANGING_TEST(name, ioctl_name, field) \
2074 static bool uffdio_mmap_changing_test_##name(int fd) \
2075 { \
2076 int ret; \
2077 struct uffdio_##name args = { \
2078 .field = 1, \
2079 }; \
2080 ret = ioctl(fd, ioctl_name, &args); \
2081 return uffdio_verify_results(#ioctl_name, ret, errno, args.field); \
2082 }
2083
2084DEFINE_MMAP_CHANGING_TEST(zeropage, UFFDIO_ZEROPAGE, zeropage)
2085DEFINE_MMAP_CHANGING_TEST(copy, UFFDIO_COPY, copy)
2086DEFINE_MMAP_CHANGING_TEST(move, UFFDIO_MOVE, move)
2087DEFINE_MMAP_CHANGING_TEST(poison, UFFDIO_POISON, updated)
2088DEFINE_MMAP_CHANGING_TEST(continue, UFFDIO_CONTINUE, mapped)
2089
2090typedef enum {
2091 /* We actually do not care about any state except UNINTERRUPTIBLE.. */
2092 THR_STATE_UNKNOWN = 0,
2093 THR_STATE_UNINTERRUPTIBLE,
2094} thread_state;
2095
2096typedef struct {
2097 uffd_global_test_opts_t *gopts;
2098 volatile pid_t *pid;
2099} mmap_changing_thread_args;
2100
2101static void sleep_short(void)
2102{
2103 usleep(1000);
2104}
2105
2106static thread_state thread_state_get(pid_t tid)
2107{
2108 const char *header = "State:\t";
2109 char tmp[256], *p, c;
2110 FILE *fp;
2111
2112 snprintf(tmp, sizeof(tmp), "/proc/%d/status", tid);
2113 fp = fopen(tmp, "r");
2114
2115 if (!fp)
2116 return THR_STATE_UNKNOWN;
2117
2118 while (fgets(tmp, sizeof(tmp), fp)) {
2119 p = strstr(tmp, header);
2120 if (p) {
2121 /* For example, "State:\tD (disk sleep)" */
2122 c = *(p + strlen(header));
2123 return c == 'D' ?
2124 THR_STATE_UNINTERRUPTIBLE : THR_STATE_UNKNOWN;
2125 }
2126 }
2127
2128 return THR_STATE_UNKNOWN;
2129}
2130
2131static void thread_state_until(pid_t tid, thread_state state)
2132{
2133 thread_state s;
2134
2135 do {
2136 s = thread_state_get(tid);
2137 sleep_short();
2138 } while (s != state);
2139}
2140
2141static void *uffd_mmap_changing_thread(void *opaque)
2142{
2143 mmap_changing_thread_args *args = opaque;
2144 uffd_global_test_opts_t *gopts = args->gopts;
2145 volatile pid_t *pid = args->pid;
2146 int ret;
2147
2148 /* Unfortunately, it's only fetch-able from the thread itself.. */
2149 assert(*pid == 0);
2150 *pid = syscall(SYS_gettid);
2151
2152 /* Inject an event, this will hang solid until the event read */
2153 ret = madvise(gopts->area_dst, gopts->page_size, MADV_REMOVE);
2154 if (ret)
2155 err("madvise(MADV_REMOVE) failed");
2156
2157 return NULL;
2158}
2159
2160static void uffd_consume_message(uffd_global_test_opts_t *gopts)
2161{
2162 struct uffd_msg msg = { 0 };
2163
2164 while (uffd_read_msg(gopts, &msg));
2165}
2166
2167static void uffd_mmap_changing_test(uffd_global_test_opts_t *gopts, uffd_test_args_t *targs)
2168{
2169 /*
2170 * This stores the real PID (which can be different from how tid is
2171 * defined..) for the child thread, 0 means not initialized.
2172 */
2173 pid_t pid = 0;
2174 pthread_t tid;
2175 int ret;
2176 mmap_changing_thread_args args = { gopts, &pid };
2177
2178 if (uffd_register(gopts->uffd, gopts->area_dst, gopts->nr_pages * gopts->page_size,
2179 true, false, false))
2180 err("uffd_register() failed");
2181
2182 /* Create a thread to generate the racy event */
2183 ret = pthread_create(&tid, NULL, uffd_mmap_changing_thread, &args);
2184 if (ret)
2185 err("pthread_create() failed");
2186
2187 /*
2188 * Wait until the thread setup the pid. Use volatile to make sure
2189 * it reads from RAM not regs.
2190 */
2191 while (!(volatile pid_t)pid)
2192 sleep_short();
2193
2194 /* Wait until the thread hangs at REMOVE event */
2195 thread_state_until(pid, THR_STATE_UNINTERRUPTIBLE);
2196
2197 if (!uffdio_mmap_changing_test_copy(gopts->uffd))
2198 return;
2199
2200 if (!uffdio_mmap_changing_test_zeropage(gopts->uffd))
2201 return;
2202
2203 if (!uffdio_mmap_changing_test_move(gopts->uffd))
2204 return;
2205
2206 if (!uffdio_mmap_changing_test_poison(gopts->uffd))
2207 return;
2208
2209 if (!uffdio_mmap_changing_test_continue(gopts->uffd))
2210 return;
2211
2212 /*
2213 * All succeeded above! Recycle everything. Start by reading the
2214 * event so as to kick the thread roll again..
2215 */
2216 uffd_consume_message(gopts);
2217
2218 ret = pthread_join(tid, NULL);
2219 assert(ret == 0);
2220
2221 uffd_test_pass();
2222}
2223
2224static int prevent_hugepages(uffd_global_test_opts_t *gopts, const char **errmsg)
2225{
2226 /* This should be done before source area is populated */
2227 if (madvise(gopts->area_src, gopts->nr_pages * gopts->page_size, MADV_NOHUGEPAGE)) {
2228 /* Ignore only if CONFIG_TRANSPARENT_HUGEPAGE=n */
2229 if (errno != EINVAL) {
2230 if (errmsg)
2231 *errmsg = "madvise(MADV_NOHUGEPAGE) failed";
2232 return -errno;
2233 }
2234 }
2235 return 0;
2236}
2237
2238static int request_hugepages(uffd_global_test_opts_t *gopts, const char **errmsg)
2239{
2240 /* This should be done before source area is populated */
2241 if (madvise(gopts->area_src, gopts->nr_pages * gopts->page_size, MADV_HUGEPAGE)) {
2242 if (errmsg) {
2243 *errmsg = (errno == EINVAL) ?
2244 "CONFIG_TRANSPARENT_HUGEPAGE is not set" :
2245 "madvise(MADV_HUGEPAGE) failed";
2246 }
2247 return -errno;
2248 }
2249 return 0;
2250}
2251
2252struct uffd_test_case_ops uffd_move_test_case_ops = {
2253 .post_alloc = prevent_hugepages,
2254};
2255
2256struct uffd_test_case_ops uffd_move_test_pmd_case_ops = {
2257 .post_alloc = request_hugepages,
2258};
2259
2260/*
2261 * Test the returned uffdio_register.ioctls with different register modes.
2262 * Note that _UFFDIO_ZEROPAGE is tested separately in the zeropage test.
2263 */
2264static void
2265do_register_ioctls_test(uffd_global_test_opts_t *gopts,
2266 uffd_test_args_t *args,
2267 bool miss,
2268 bool wp,
2269 bool minor)
2270{
2271 uint64_t ioctls = 0, expected = BIT_ULL(_UFFDIO_WAKE);
2272 mem_type_t *mem_type = args->mem_type;
2273 int ret;
2274
2275 ret = uffd_register_with_ioctls(gopts->uffd, gopts->area_dst, gopts->page_size,
2276 miss, wp, minor, &ioctls);
2277
2278 /*
2279 * Handle special cases of UFFDIO_REGISTER here where it should
2280 * just fail with -EINVAL first..
2281 *
2282 * Case 1: register MINOR on anon
2283 * Case 2: register with no mode selected
2284 */
2285 if ((minor && (mem_type->mem_flag == MEM_ANON)) ||
2286 (!miss && !wp && !minor)) {
2287 if (ret != -EINVAL)
2288 err("register (miss=%d, wp=%d, minor=%d) failed "
2289 "with wrong errno=%d", miss, wp, minor, ret);
2290 return;
2291 }
2292
2293 /* UFFDIO_REGISTER should succeed, then check ioctls returned */
2294 if (miss)
2295 expected |= BIT_ULL(_UFFDIO_COPY);
2296 if (wp)
2297 expected |= BIT_ULL(_UFFDIO_WRITEPROTECT);
2298 if (minor)
2299 expected |= BIT_ULL(_UFFDIO_CONTINUE);
2300
2301 if ((ioctls & expected) != expected)
2302 err("unexpected uffdio_register.ioctls "
2303 "(miss=%d, wp=%d, minor=%d): expected=0x%"PRIx64", "
2304 "returned=0x%"PRIx64, miss, wp, minor, expected, ioctls);
2305
2306 if (uffd_unregister(gopts->uffd, gopts->area_dst, gopts->page_size))
2307 err("unregister");
2308}
2309
2310static void uffd_register_ioctls_test(uffd_global_test_opts_t *gopts, uffd_test_args_t *args)
2311{
2312 int miss, wp, minor;
2313
2314 for (miss = 0; miss <= 1; miss++)
2315 for (wp = 0; wp <= 1; wp++)
2316 for (minor = 0; minor <= 1; minor++)
2317 do_register_ioctls_test(gopts, args, miss, wp, minor);
2318
2319 uffd_test_pass();
2320}
2321
2322uffd_test_case_t uffd_tests[] = {
2323 {
2324 /* Test returned uffdio_register.ioctls. */
2325 .name = "register-ioctls",
2326 .uffd_fn = uffd_register_ioctls_test,
2327 .mem_targets = MEM_ALL,
2328 .uffd_feature_required = UFFD_FEATURE_MISSING_HUGETLBFS |
2329 UFFD_FEATURE_MISSING_SHMEM |
2330 UFFD_FEATURE_PAGEFAULT_FLAG_WP |
2331 UFFD_FEATURE_WP_HUGETLBFS_SHMEM |
2332 UFFD_FEATURE_MINOR_HUGETLBFS |
2333 UFFD_FEATURE_MINOR_SHMEM,
2334 },
2335 {
2336 .name = "zeropage",
2337 .uffd_fn = uffd_zeropage_test,
2338 .mem_targets = MEM_ALL,
2339 .uffd_feature_required = 0,
2340 },
2341 {
2342 .name = "move",
2343 .uffd_fn = uffd_move_test,
2344 .mem_targets = MEM_ANON,
2345 .uffd_feature_required = UFFD_FEATURE_MOVE,
2346 .test_case_ops = &uffd_move_test_case_ops,
2347 },
2348 {
2349 .name = "move-pmd",
2350 .uffd_fn = uffd_move_pmd_test,
2351 .mem_targets = MEM_ANON,
2352 .uffd_feature_required = UFFD_FEATURE_MOVE,
2353 .test_case_ops = &uffd_move_test_pmd_case_ops,
2354 },
2355 {
2356 .name = "move-pmd-split",
2357 .uffd_fn = uffd_move_pmd_split_test,
2358 .mem_targets = MEM_ANON,
2359 .uffd_feature_required = UFFD_FEATURE_MOVE,
2360 .test_case_ops = &uffd_move_test_pmd_case_ops,
2361 },
2362 {
2363 .name = "wp-fork",
2364 .uffd_fn = uffd_wp_fork_test,
2365 .mem_targets = MEM_ALL,
2366 .uffd_feature_required = UFFD_FEATURE_PAGEFAULT_FLAG_WP |
2367 UFFD_FEATURE_WP_HUGETLBFS_SHMEM,
2368 },
2369 {
2370 .name = "wp-fork-with-event",
2371 .uffd_fn = uffd_wp_fork_with_event_test,
2372 .mem_targets = MEM_ALL,
2373 .uffd_feature_required = UFFD_FEATURE_PAGEFAULT_FLAG_WP |
2374 UFFD_FEATURE_WP_HUGETLBFS_SHMEM |
2375 /* when set, child process should inherit uffd-wp bits */
2376 UFFD_FEATURE_EVENT_FORK,
2377 },
2378 {
2379 .name = "wp-fork-pin",
2380 .uffd_fn = uffd_wp_fork_pin_test,
2381 .mem_targets = MEM_ALL,
2382 .uffd_feature_required = UFFD_FEATURE_PAGEFAULT_FLAG_WP |
2383 UFFD_FEATURE_WP_HUGETLBFS_SHMEM,
2384 },
2385 {
2386 .name = "wp-fork-pin-with-event",
2387 .uffd_fn = uffd_wp_fork_pin_with_event_test,
2388 .mem_targets = MEM_ALL,
2389 .uffd_feature_required = UFFD_FEATURE_PAGEFAULT_FLAG_WP |
2390 UFFD_FEATURE_WP_HUGETLBFS_SHMEM |
2391 /* when set, child process should inherit uffd-wp bits */
2392 UFFD_FEATURE_EVENT_FORK,
2393 },
2394 {
2395 .name = "wp-unpopulated",
2396 .uffd_fn = uffd_wp_unpopulated_test,
2397 .mem_targets = MEM_ANON,
2398 .uffd_feature_required =
2399 UFFD_FEATURE_PAGEFAULT_FLAG_WP | UFFD_FEATURE_WP_UNPOPULATED,
2400 },
2401 {
2402 .name = "minor",
2403 .uffd_fn = uffd_minor_test,
2404 .mem_targets = MEM_SHMEM | MEM_HUGETLB,
2405 .uffd_feature_required =
2406 UFFD_FEATURE_MINOR_HUGETLBFS | UFFD_FEATURE_MINOR_SHMEM,
2407 },
2408 {
2409 .name = "minor-wp",
2410 .uffd_fn = uffd_minor_wp_test,
2411 .mem_targets = MEM_SHMEM | MEM_HUGETLB,
2412 .uffd_feature_required =
2413 UFFD_FEATURE_MINOR_HUGETLBFS | UFFD_FEATURE_MINOR_SHMEM |
2414 UFFD_FEATURE_PAGEFAULT_FLAG_WP |
2415 /*
2416 * HACK: here we leveraged WP_UNPOPULATED to detect whether
2417 * minor mode supports wr-protect. There's no feature flag
2418 * for it so this is the best we can test against.
2419 */
2420 UFFD_FEATURE_WP_UNPOPULATED,
2421 },
2422 {
2423 .name = "minor-collapse",
2424 .uffd_fn = uffd_minor_collapse_test,
2425 /* MADV_COLLAPSE only works with shmem */
2426 .mem_targets = MEM_SHMEM,
2427 /* We can't test MADV_COLLAPSE, so try our luck */
2428 .uffd_feature_required = UFFD_FEATURE_MINOR_SHMEM,
2429 },
2430 {
2431 .name = "rwp-async",
2432 .uffd_fn = uffd_rwp_async_test,
2433 .mem_targets = MEM_ALL,
2434 .uffd_feature_required =
2435 UFFD_FEATURE_RWP | UFFD_FEATURE_RWP_ASYNC,
2436 },
2437 {
2438 .name = "rwp-sync",
2439 .uffd_fn = uffd_rwp_sync_test,
2440 .mem_targets = MEM_ALL,
2441 .uffd_feature_required = UFFD_FEATURE_RWP,
2442 },
2443 {
2444 .name = "rwp-pagemap",
2445 .uffd_fn = uffd_rwp_pagemap_test,
2446 .mem_targets = MEM_ALL,
2447 .uffd_feature_required =
2448 UFFD_FEATURE_RWP | UFFD_FEATURE_RWP_ASYNC,
2449 },
2450 {
2451 .name = "rwp-mprotect",
2452 .uffd_fn = uffd_rwp_mprotect_test,
2453 .mem_targets = MEM_ALL,
2454 .uffd_feature_required =
2455 UFFD_FEATURE_RWP | UFFD_FEATURE_RWP_ASYNC,
2456 },
2457 {
2458 .name = "rwp-gup",
2459 .uffd_fn = uffd_rwp_gup_test,
2460 .mem_targets = MEM_ALL,
2461 .uffd_feature_required =
2462 UFFD_FEATURE_RWP | UFFD_FEATURE_RWP_ASYNC,
2463 },
2464 {
2465 .name = "rwp-async-toggle",
2466 .uffd_fn = uffd_rwp_async_toggle_test,
2467 .mem_targets = MEM_ALL,
2468 .uffd_feature_required =
2469 UFFD_FEATURE_RWP | UFFD_FEATURE_RWP_ASYNC,
2470 },
2471 {
2472 .name = "rwp-close",
2473 .uffd_fn = uffd_rwp_close_test,
2474 .mem_targets = MEM_ALL,
2475 .uffd_feature_required = UFFD_FEATURE_RWP,
2476 },
2477 {
2478 .name = "rwp-fork",
2479 .uffd_fn = uffd_rwp_fork_test,
2480 .mem_targets = MEM_ALL,
2481 .uffd_feature_required =
2482 UFFD_FEATURE_RWP | UFFD_FEATURE_EVENT_FORK,
2483 },
2484 {
2485 .name = "rwp-fork-pin",
2486 .uffd_fn = uffd_rwp_fork_pin_test,
2487 .mem_targets = MEM_ANON,
2488 .uffd_feature_required =
2489 UFFD_FEATURE_RWP | UFFD_FEATURE_RWP_ASYNC |
2490 UFFD_FEATURE_EVENT_FORK,
2491 },
2492 {
2493 .name = "rwp-swap-cow",
2494 .uffd_fn = uffd_rwp_swap_cow_test,
2495 .mem_targets = MEM_ANON,
2496 .uffd_feature_required = UFFD_FEATURE_RWP,
2497 },
2498 {
2499 .name = "rwp-wp-exclusive",
2500 .uffd_fn = uffd_rwp_wp_exclusive_test,
2501 .mem_targets = MEM_ALL,
2502 .uffd_feature_required =
2503 UFFD_FEATURE_RWP |
2504 UFFD_FEATURE_PAGEFAULT_FLAG_WP |
2505 UFFD_FEATURE_WP_HUGETLBFS_SHMEM,
2506 },
2507 {
2508 .name = "sigbus",
2509 .uffd_fn = uffd_sigbus_test,
2510 .mem_targets = MEM_ALL,
2511 .uffd_feature_required = UFFD_FEATURE_SIGBUS |
2512 UFFD_FEATURE_EVENT_FORK,
2513 },
2514 {
2515 .name = "sigbus-wp",
2516 .uffd_fn = uffd_sigbus_wp_test,
2517 .mem_targets = MEM_ALL,
2518 .uffd_feature_required = UFFD_FEATURE_SIGBUS |
2519 UFFD_FEATURE_EVENT_FORK | UFFD_FEATURE_PAGEFAULT_FLAG_WP |
2520 UFFD_FEATURE_WP_HUGETLBFS_SHMEM,
2521 },
2522 {
2523 .name = "events",
2524 .uffd_fn = uffd_events_test,
2525 .mem_targets = MEM_ALL,
2526 .uffd_feature_required = UFFD_FEATURE_EVENT_FORK |
2527 UFFD_FEATURE_EVENT_REMAP | UFFD_FEATURE_EVENT_REMOVE,
2528 },
2529 {
2530 .name = "events-wp",
2531 .uffd_fn = uffd_events_wp_test,
2532 .mem_targets = MEM_ALL,
2533 .uffd_feature_required = UFFD_FEATURE_EVENT_FORK |
2534 UFFD_FEATURE_EVENT_REMAP | UFFD_FEATURE_EVENT_REMOVE |
2535 UFFD_FEATURE_PAGEFAULT_FLAG_WP |
2536 UFFD_FEATURE_WP_HUGETLBFS_SHMEM,
2537 },
2538 {
2539 .name = "poison",
2540 .uffd_fn = uffd_poison_test,
2541 .mem_targets = MEM_ALL,
2542 .uffd_feature_required = UFFD_FEATURE_POISON,
2543 },
2544 {
2545 .name = "mmap-changing",
2546 .uffd_fn = uffd_mmap_changing_test,
2547 /*
2548 * There's no point running this test over all mem types as
2549 * they share the same code paths.
2550 *
2551 * Choose shmem for simplicity, because (1) shmem supports
2552 * MINOR mode to cover UFFDIO_CONTINUE, and (2) shmem is
2553 * almost always available (unlike hugetlb). Here we
2554 * abused SHMEM for UFFDIO_MOVE, but the test we want to
2555 * cover doesn't yet need the correct memory type..
2556 */
2557 .mem_targets = MEM_SHMEM,
2558 /*
2559 * Any UFFD_FEATURE_EVENT_* should work to trigger the
2560 * race logically, but choose the simplest (REMOVE).
2561 *
2562 * Meanwhile, since we'll cover quite a few new ioctl()s
2563 * (CONTINUE, POISON, MOVE), skip this test for old kernels
2564 * by choosing all of them.
2565 */
2566 .uffd_feature_required = UFFD_FEATURE_EVENT_REMOVE |
2567 UFFD_FEATURE_MOVE | UFFD_FEATURE_POISON |
2568 UFFD_FEATURE_MINOR_SHMEM,
2569 },
2570};
2571
2572static void usage(const char *prog)
2573{
2574 printf("usage: %s [-f TESTNAME]\n", prog);
2575 puts("");
2576 puts(" -f: test name to filter (e.g., event)");
2577 puts(" -h: show the help msg");
2578 puts(" -l: list tests only");
2579 puts("");
2580 exit(KSFT_FAIL);
2581}
2582
2583static int uffd_count_tests(int n_tests, int n_mems, const char *test_filter)
2584{
2585 uffd_test_case_t *test;
2586 int i, j, count = 0;
2587
2588 if (!test_filter)
2589 count += 2; /* test_uffd_api(false) + test_uffd_api(true) */
2590
2591 for (i = 0; i < n_tests; i++) {
2592 test = &uffd_tests[i];
2593 if (test_filter && !strstr(test->name, test_filter))
2594 continue;
2595 for (j = 0; j < n_mems; j++)
2596 if (test->mem_targets & mem_types[j].mem_flag)
2597 count++;
2598 }
2599
2600 return count;
2601}
2602
2603static unsigned long uffd_setup_hugetlb(void)
2604{
2605 unsigned long nr_hugepages, hp_size;
2606
2607 hugetlb_save_settings();
2608 hp_size = default_huge_page_size();
2609
2610 if (!hp_size)
2611 return 0;
2612
2613 /* need twice UFFD_TEST_MEM_SIZE, one for src area and one for dst */
2614 nr_hugepages = 2 * MAX(UFFD_TEST_MEM_SIZE, hp_size * 2) / hp_size;
2615 hugetlb_set_nr_default_pages(nr_hugepages);
2616
2617 if (hugetlb_free_default_pages() < nr_hugepages)
2618 return 0;
2619
2620 return hp_size;
2621}
2622
2623int main(int argc, char *argv[])
2624{
2625 int n_tests = sizeof(uffd_tests) / sizeof(uffd_test_case_t);
2626 int n_mems = sizeof(mem_types) / sizeof(mem_type_t);
2627 const char *test_filter = NULL;
2628 unsigned long hugepage_size;
2629 bool list_only = false;
2630 uffd_test_case_t *test;
2631 mem_type_t *mem_type;
2632 uffd_test_args_t args;
2633 const char *errmsg;
2634 int i, j, opt;
2635
2636 while ((opt = getopt(argc, argv, "f:hl")) != -1) {
2637 switch (opt) {
2638 case 'f':
2639 test_filter = optarg;
2640 break;
2641 case 'l':
2642 list_only = true;
2643 break;
2644 case 'h':
2645 default:
2646 /* Unknown */
2647 usage(argv[0]);
2648 break;
2649 }
2650 }
2651
2652 if (list_only) {
2653 for (i = 0; i < n_tests; i++) {
2654 test = &uffd_tests[i];
2655 if (test_filter && !strstr(test->name, test_filter))
2656 continue;
2657 printf("%s\n", test->name);
2658 }
2659 return KSFT_PASS;
2660 }
2661
2662 hugepage_size = uffd_setup_hugetlb();
2663
2664 ksft_print_header();
2665 ksft_set_plan(uffd_count_tests(n_tests, n_mems, test_filter));
2666
2667 if (!test_filter) {
2668 test_uffd_api(false);
2669 test_uffd_api(true);
2670 }
2671
2672 for (i = 0; i < n_tests; i++) {
2673 test = &uffd_tests[i];
2674 if (test_filter && !strstr(test->name, test_filter))
2675 continue;
2676 for (j = 0; j < n_mems; j++) {
2677 mem_type = &mem_types[j];
2678
2679 /* Initialize global test options */
2680 uffd_global_test_opts_t gopts = { 0 };
2681
2682 gopts.map_shared = mem_type->shared;
2683 uffd_test_ops = mem_type->mem_ops;
2684 uffd_test_case_ops = test->test_case_ops;
2685
2686 if (!(test->mem_targets & mem_type->mem_flag))
2687 continue;
2688
2689 uffd_test_start("%s on %s", test->name, mem_type->name);
2690 if (mem_type->mem_flag & (MEM_HUGETLB_PRIVATE | MEM_HUGETLB)) {
2691 gopts.page_size = hugepage_size;
2692 if (gopts.page_size == 0) {
2693 uffd_test_skip("not enough HugeTLB pages");
2694 continue;
2695 }
2696 } else {
2697 gopts.page_size = psize();
2698 }
2699
2700 /* Ensure we have at least 2 pages */
2701 gopts.nr_pages = MAX(UFFD_TEST_MEM_SIZE, gopts.page_size * 2)
2702 / gopts.page_size;
2703
2704 gopts.nr_parallel = 1;
2705
2706 /* Initialize test arguments */
2707 args.mem_type = mem_type;
2708
2709 if (!uffd_feature_supported(test)) {
2710 uffd_test_skip("feature missing");
2711 continue;
2712 }
2713 if (uffd_test_ctx_init(&gopts, test->uffd_feature_required, &errmsg)) {
2714 uffd_test_skip(errmsg);
2715 continue;
2716 }
2717 /*
2718 * RWP tracks protection on ptes; a THP-backed shmem/anon
2719 * range (e.g. shmem_enabled=always) would split on
2720 * rwprotect and change behaviour under the test. Keep
2721 * such ranges off THP. hugetlb is huge by definition and
2722 * rejects MADV_NOHUGEPAGE, so skip it.
2723 */
2724 if ((test->uffd_feature_required & UFFD_FEATURE_RWP) &&
2725 !(mem_type->mem_flag & (MEM_HUGETLB | MEM_HUGETLB_PRIVATE))) {
2726 unsigned long len = gopts.nr_pages * gopts.page_size;
2727
2728 /*
2729 * EINVAL means CONFIG_TRANSPARENT_HUGEPAGE=n:
2730 * nothing to opt out of.
2731 */
2732 if (madvise(gopts.area_dst, len, MADV_NOHUGEPAGE) &&
2733 errno != EINVAL)
2734 err("madvise(MADV_NOHUGEPAGE)");
2735 }
2736 test->uffd_fn(&gopts, &args);
2737 uffd_test_ctx_clear(&gopts);
2738 }
2739 }
2740
2741 ksft_finished();
2742}
2743
2744#else /* __NR_userfaultfd */
2745
2746#warning "missing __NR_userfaultfd definition"
2747
2748int main(void)
2749{
2750 ksft_print_header();
2751 ksft_exit_skip("missing __NR_userfaultfd definition\n");
2752}
2753
2754#endif /* __NR_userfaultfd */