projects/linux·blob master · refs

525 lines · 11.5 KiB

1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * GUP long-term page pinning tests.
4 *
5 * Copyright 2023, Red Hat, Inc.
6 *
7 * Author(s): David Hildenbrand <[email protected]>
8 */
9#define _GNU_SOURCE
10#include <stdlib.h>
11#include <string.h>
12#include <stdbool.h>
13#include <stdint.h>
14#include <unistd.h>
15#include <errno.h>
16#include <fcntl.h>
17#include <assert.h>
18#include <sys/mman.h>
19#include <sys/ioctl.h>
20#include <sys/vfs.h>
21#include <linux/magic.h>
22#include <linux/memfd.h>
23
24#include "local_config.h_gen"
25#ifdef LOCAL_CONFIG_HAVE_LIBURING
26#include <liburing.h>
27#endif /* LOCAL_CONFIG_HAVE_LIBURING */
28
29#include "../../../../mm/gup_test.h"
30#include "kselftest.h"
31#include "vm_util.h"
32#include "hugepage_settings.h"
33
34static size_t pagesize;
35static int nr_hugetlbsizes;
36static unsigned long hugetlbsizes[10];
37static int gup_fd;
38
39static __fsword_t get_fs_type(int fd)
40{
41 struct statfs fs;
42 int ret;
43
44 do {
45 ret = fstatfs(fd, &fs);
46 } while (ret && errno == EINTR);
47
48 return ret ? 0 : fs.f_type;
49}
50
51static bool fs_is_unknown(__fsword_t fs_type)
52{
53 /*
54 * We only support some filesystems in our tests when dealing with
55 * R/W long-term pinning. For these filesystems, we can be fairly sure
56 * whether they support it or not.
57 */
58 switch (fs_type) {
59 case TMPFS_MAGIC:
60 case HUGETLBFS_MAGIC:
61 case BTRFS_SUPER_MAGIC:
62 case EXT4_SUPER_MAGIC:
63 case XFS_SUPER_MAGIC:
64 return false;
65 default:
66 return true;
67 }
68}
69
70static bool fs_supports_writable_longterm_pinning(__fsword_t fs_type)
71{
72 assert(!fs_is_unknown(fs_type));
73 switch (fs_type) {
74 case TMPFS_MAGIC:
75 case HUGETLBFS_MAGIC:
76 return true;
77 default:
78 return false;
79 }
80}
81
82enum test_type {
83 TEST_TYPE_RO,
84 TEST_TYPE_RO_FAST,
85 TEST_TYPE_RW,
86 TEST_TYPE_RW_FAST,
87#ifdef LOCAL_CONFIG_HAVE_LIBURING
88 TEST_TYPE_IOURING,
89#endif /* LOCAL_CONFIG_HAVE_LIBURING */
90};
91
92static void do_test(int fd, size_t size, enum test_type type, bool shared)
93{
94 __fsword_t fs_type = get_fs_type(fd);
95 bool should_work;
96 char *mem;
97 int result = KSFT_PASS;
98 int ret;
99
100 if (fd < 0) {
101 result = KSFT_FAIL;
102 goto report;
103 }
104
105 if (ftruncate(fd, size)) {
106 if (errno == ENOENT) {
107 skip_test_dodgy_fs("ftruncate()");
108 } else {
109 ksft_print_msg("ftruncate() failed (%s)\n",
110 strerror(errno));
111 result = KSFT_FAIL;
112 goto report;
113 }
114 return;
115 }
116
117 if (fallocate(fd, 0, 0, size)) {
118 /*
119 * Some filesystems (eg, NFSv3) don't support
120 * fallocate(), report this as a skip rather than a
121 * test failure.
122 */
123 if (errno == EOPNOTSUPP) {
124 ksft_print_msg("fallocate() not supported by filesystem\n");
125 result = KSFT_SKIP;
126 } else if (size == pagesize) {
127 ksft_print_msg("fallocate() failed (%s)\n", strerror(errno));
128 result = KSFT_FAIL;
129 } else {
130 ksft_print_msg("need more free huge pages\n");
131 result = KSFT_SKIP;
132 }
133 goto report;
134 }
135
136 mem = mmap(NULL, size, PROT_READ | PROT_WRITE,
137 shared ? MAP_SHARED : MAP_PRIVATE, fd, 0);
138 if (mem == MAP_FAILED) {
139 if (size == pagesize || shared) {
140 ksft_print_msg("mmap() failed (%s)\n", strerror(errno));
141 result = KSFT_FAIL;
142 } else {
143 ksft_print_msg("need more free huge pages\n");
144 result = KSFT_SKIP;
145 }
146 goto report;
147 }
148
149 /* Fault in the page such that GUP-fast can pin it directly. */
150 memset(mem, 0, size);
151
152 switch (type) {
153 case TEST_TYPE_RO:
154 case TEST_TYPE_RO_FAST:
155 /*
156 * Cover more cases regarding unsharing decisions when
157 * long-term R/O pinning by mapping the page R/O.
158 */
159 ret = mprotect(mem, size, PROT_READ);
160 if (ret) {
161 ksft_print_msg("mprotect() failed (%s)\n", strerror(errno));
162 result = KSFT_FAIL;
163 goto munmap;
164 }
165 /* FALLTHROUGH */
166 case TEST_TYPE_RW:
167 case TEST_TYPE_RW_FAST: {
168 struct pin_longterm_test args;
169 const bool fast = type == TEST_TYPE_RO_FAST ||
170 type == TEST_TYPE_RW_FAST;
171 const bool rw = type == TEST_TYPE_RW ||
172 type == TEST_TYPE_RW_FAST;
173
174 if (gup_fd < 0) {
175 ksft_print_msg("gup_test not available\n");
176 result = KSFT_SKIP;
177 break;
178 }
179
180 if (rw && shared && fs_is_unknown(fs_type)) {
181 ksft_print_msg("Unknown filesystem\n");
182 result = KSFT_SKIP;
183 break;
184 }
185 /*
186 * R/O pinning or pinning in a private mapping is always
187 * expected to work. Otherwise, we expect long-term R/W pinning
188 * to only succeed for special filesystems.
189 */
190 should_work = !shared || !rw ||
191 fs_supports_writable_longterm_pinning(fs_type);
192
193 args.addr = (__u64)(uintptr_t)mem;
194 args.size = size;
195 args.flags = fast ? PIN_LONGTERM_TEST_FLAG_USE_FAST : 0;
196 args.flags |= rw ? PIN_LONGTERM_TEST_FLAG_USE_WRITE : 0;
197 ret = ioctl(gup_fd, PIN_LONGTERM_TEST_START, &args);
198 if (ret && errno == EINVAL) {
199 ksft_print_msg("PIN_LONGTERM_TEST_START failed (EINVAL)\n");
200 result = KSFT_SKIP;
201 break;
202 } else if (ret && errno == EFAULT) {
203 if (should_work)
204 result = KSFT_FAIL;
205 else
206 result = KSFT_PASS;
207 break;
208 } else if (ret) {
209 ksft_print_msg("PIN_LONGTERM_TEST_START failed (%s)\n",
210 strerror(errno));
211 result = KSFT_FAIL;
212 break;
213 }
214
215 if (ioctl(gup_fd, PIN_LONGTERM_TEST_STOP))
216 ksft_print_msg("[INFO] PIN_LONGTERM_TEST_STOP failed (%s)\n",
217 strerror(errno));
218
219 /*
220 * TODO: if the kernel ever supports long-term R/W pinning on
221 * some previously unsupported filesystems, we might want to
222 * perform some additional tests for possible data corruptions.
223 */
224 if (should_work)
225 result = KSFT_PASS;
226 else
227 result = KSFT_FAIL;
228 break;
229 }
230#ifdef LOCAL_CONFIG_HAVE_LIBURING
231 case TEST_TYPE_IOURING: {
232 struct io_uring ring;
233 struct iovec iov;
234
235 /* io_uring always pins pages writable. */
236 if (shared && fs_is_unknown(fs_type)) {
237 ksft_print_msg("Unknown filesystem\n");
238 result = KSFT_SKIP;
239 goto report;
240 }
241 should_work = !shared ||
242 fs_supports_writable_longterm_pinning(fs_type);
243
244 /* Skip on errors, as we might just lack kernel support. */
245 ret = io_uring_queue_init(1, &ring, 0);
246 if (ret < 0) {
247 ksft_print_msg("io_uring_queue_init() failed (%s)\n",
248 strerror(-ret));
249 result = KSFT_SKIP;
250 break;
251 }
252 /*
253 * Register the range as a fixed buffer. This will FOLL_WRITE |
254 * FOLL_PIN | FOLL_LONGTERM the range.
255 */
256 iov.iov_base = mem;
257 iov.iov_len = size;
258 ret = io_uring_register_buffers(&ring, &iov, 1);
259 /* Only new kernels return EFAULT. */
260 if (ret && (errno == ENOSPC || errno == EOPNOTSUPP ||
261 errno == EFAULT)) {
262 if (should_work) {
263 ksft_print_msg("Should have failed (%s)\n",
264 strerror(errno));
265 result = KSFT_FAIL;
266 } else {
267 result = KSFT_PASS;
268 }
269 } else if (ret) {
270 /*
271 * We might just lack support or have insufficient
272 * MEMLOCK limits.
273 */
274 ksft_print_msg("io_uring_register_buffers() failed (%s)\n",
275 strerror(-ret));
276 result = KSFT_SKIP;
277 } else {
278 if (should_work) {
279 result = KSFT_PASS;
280 } else {
281 ksft_print_msg("Should have worked\n");
282 result = KSFT_FAIL;
283 }
284 io_uring_unregister_buffers(&ring);
285 }
286
287 io_uring_queue_exit(&ring);
288 break;
289 }
290#endif /* LOCAL_CONFIG_HAVE_LIBURING */
291 default:
292 assert(false);
293 }
294
295munmap:
296 munmap(mem, size);
297report:
298 log_test_result(result);
299}
300
301typedef void (*test_fn)(int fd, size_t size);
302
303static void run_with_memfd(test_fn fn, const char *desc)
304{
305 int fd;
306
307 log_test_start("%s ... with memfd", desc);
308
309 fd = memfd_create("test", 0);
310 if (fd < 0) {
311 ksft_print_msg("memfd_create() failed (%s)\n", strerror(errno));
312 log_test_result(KSFT_SKIP);
313 return;
314 }
315
316 fn(fd, pagesize);
317 close(fd);
318}
319
320static void run_with_tmpfile(test_fn fn, const char *desc)
321{
322 FILE *file;
323 int fd;
324
325 log_test_start("%s ... with tmpfile", desc);
326
327 file = tmpfile();
328 if (!file) {
329 ksft_print_msg("tmpfile() failed (%s)\n", strerror(errno));
330 fd = -1;
331 } else {
332 fd = fileno(file);
333 if (fd < 0) {
334 ksft_print_msg("fileno() failed (%s)\n", strerror(errno));
335 }
336 }
337
338 fn(fd, pagesize);
339
340 if (file)
341 fclose(file);
342}
343
344static void run_with_local_tmpfile(test_fn fn, const char *desc)
345{
346 char filename[] = __FILE__"_tmpfile_XXXXXX";
347 int fd;
348
349 log_test_start("%s ... with local tmpfile", desc);
350
351 fd = mkstemp(filename);
352 if (fd < 0)
353 ksft_print_msg("mkstemp() failed (%s)\n", strerror(errno));
354
355 if (unlink(filename)) {
356 ksft_print_msg("unlink() failed (%s)\n", strerror(errno));
357 close(fd);
358 fd = -1;
359 }
360
361 fn(fd, pagesize);
362
363 if (fd >= 0)
364 close(fd);
365}
366
367static void run_with_memfd_hugetlb(test_fn fn, const char *desc,
368 size_t hugetlbsize)
369{
370 int flags = MFD_HUGETLB;
371 int fd;
372
373 log_test_start("%s ... with memfd hugetlb (%zu kB)", desc,
374 hugetlbsize / 1024);
375
376 flags |= __builtin_ctzll(hugetlbsize) << MFD_HUGE_SHIFT;
377
378 fd = memfd_create("test", flags);
379 if (fd < 0) {
380 ksft_print_msg("memfd_create() failed (%s)\n", strerror(errno));
381 log_test_result(KSFT_SKIP);
382 return;
383 }
384
385 fn(fd, hugetlbsize);
386 close(fd);
387}
388
389struct test_case {
390 const char *desc;
391 test_fn fn;
392};
393
394static void test_shared_rw_pin(int fd, size_t size)
395{
396 do_test(fd, size, TEST_TYPE_RW, true);
397}
398
399static void test_shared_rw_fast_pin(int fd, size_t size)
400{
401 do_test(fd, size, TEST_TYPE_RW_FAST, true);
402}
403
404static void test_shared_ro_pin(int fd, size_t size)
405{
406 do_test(fd, size, TEST_TYPE_RO, true);
407}
408
409static void test_shared_ro_fast_pin(int fd, size_t size)
410{
411 do_test(fd, size, TEST_TYPE_RO_FAST, true);
412}
413
414static void test_private_rw_pin(int fd, size_t size)
415{
416 do_test(fd, size, TEST_TYPE_RW, false);
417}
418
419static void test_private_rw_fast_pin(int fd, size_t size)
420{
421 do_test(fd, size, TEST_TYPE_RW_FAST, false);
422}
423
424static void test_private_ro_pin(int fd, size_t size)
425{
426 do_test(fd, size, TEST_TYPE_RO, false);
427}
428
429static void test_private_ro_fast_pin(int fd, size_t size)
430{
431 do_test(fd, size, TEST_TYPE_RO_FAST, false);
432}
433
434#ifdef LOCAL_CONFIG_HAVE_LIBURING
435static void test_shared_iouring(int fd, size_t size)
436{
437 do_test(fd, size, TEST_TYPE_IOURING, true);
438}
439
440static void test_private_iouring(int fd, size_t size)
441{
442 do_test(fd, size, TEST_TYPE_IOURING, false);
443}
444#endif /* LOCAL_CONFIG_HAVE_LIBURING */
445
446static const struct test_case test_cases[] = {
447 {
448 "R/W longterm GUP pin in MAP_SHARED file mapping",
449 test_shared_rw_pin,
450 },
451 {
452 "R/W longterm GUP-fast pin in MAP_SHARED file mapping",
453 test_shared_rw_fast_pin,
454 },
455 {
456 "R/O longterm GUP pin in MAP_SHARED file mapping",
457 test_shared_ro_pin,
458 },
459 {
460 "R/O longterm GUP-fast pin in MAP_SHARED file mapping",
461 test_shared_ro_fast_pin,
462 },
463 {
464 "R/W longterm GUP pin in MAP_PRIVATE file mapping",
465 test_private_rw_pin,
466 },
467 {
468 "R/W longterm GUP-fast pin in MAP_PRIVATE file mapping",
469 test_private_rw_fast_pin,
470 },
471 {
472 "R/O longterm GUP pin in MAP_PRIVATE file mapping",
473 test_private_ro_pin,
474 },
475 {
476 "R/O longterm GUP-fast pin in MAP_PRIVATE file mapping",
477 test_private_ro_fast_pin,
478 },
479#ifdef LOCAL_CONFIG_HAVE_LIBURING
480 {
481 "io_uring fixed buffer with MAP_SHARED file mapping",
482 test_shared_iouring,
483 },
484 {
485 "io_uring fixed buffer with MAP_PRIVATE file mapping",
486 test_private_iouring,
487 },
488#endif /* LOCAL_CONFIG_HAVE_LIBURING */
489};
490
491static void run_test_case(struct test_case const *test_case)
492{
493 int i;
494
495 run_with_memfd(test_case->fn, test_case->desc);
496 run_with_tmpfile(test_case->fn, test_case->desc);
497 run_with_local_tmpfile(test_case->fn, test_case->desc);
498 for (i = 0; i < nr_hugetlbsizes; i++)
499 run_with_memfd_hugetlb(test_case->fn, test_case->desc,
500 hugetlbsizes[i]);
501}
502
503static int tests_per_test_case(void)
504{
505 return 3 + nr_hugetlbsizes;
506}
507
508int main(int argc, char **argv)
509{
510 int i;
511
512 pagesize = getpagesize();
513 nr_hugetlbsizes = hugetlb_setup(2, hugetlbsizes,
514 ARRAY_SIZE(hugetlbsizes));
515
516 ksft_print_header();
517 ksft_set_plan(ARRAY_SIZE(test_cases) * tests_per_test_case());
518
519 gup_fd = open("/sys/kernel/debug/gup_test", O_RDWR);
520
521 for (i = 0; i < ARRAY_SIZE(test_cases); i++)
522 run_test_case(&test_cases[i]);
523
524 ksft_finished();
525}