projects/linux·blob master · refs

514 lines · 11.3 KiB

1// SPDX-License-Identifier: GPL-2.0
2/*
3 * RMAP functional tests
4 *
5 * Author(s): Wei Yang <[email protected]>
6 */
7
8#include "kselftest_harness.h"
9#include <strings.h>
10#include <pthread.h>
11#include <numa.h>
12#include <numaif.h>
13#include <sys/mman.h>
14#include <sys/prctl.h>
15#include <sys/types.h>
16#include <signal.h>
17#include <time.h>
18#include <sys/sem.h>
19#include <unistd.h>
20#include <fcntl.h>
21
22#include "vm_util.h"
23
24#define TOTAL_LEVEL 5
25#define MAX_CHILDREN 3
26
27#define FAIL_ON_CHECK (1 << 0)
28#define FAIL_ON_WORK (1 << 1)
29
30struct sembuf sem_wait = {0, -1, 0};
31struct sembuf sem_signal = {0, 1, 0};
32
33enum backend_type {
34 ANON,
35 SHM,
36 NORM_FILE,
37};
38
39#define PREFIX "kst_rmap"
40#define MAX_FILENAME_LEN 256
41const char *suffixes[] = {
42 "",
43 "_shm",
44 "_file",
45};
46
47struct global_data;
48typedef int (*work_fn)(struct global_data *data);
49typedef int (*check_fn)(struct global_data *data);
50typedef void (*prepare_fn)(struct global_data *data);
51
52struct global_data {
53 int worker_level;
54
55 int semid;
56 int pipefd[2];
57
58 unsigned int mapsize;
59 unsigned int rand_seed;
60 char *region;
61
62 prepare_fn do_prepare;
63 work_fn do_work;
64 check_fn do_check;
65
66 enum backend_type backend;
67 char filename[MAX_FILENAME_LEN];
68
69 unsigned long *expected_pfn;
70};
71
72/*
73 * Create a process tree with TOTAL_LEVEL height and at most MAX_CHILDREN
74 * children for each.
75 *
76 * It will randomly select one process as 'worker' process which will
77 * 'do_work' until all processes are created. And all other processes will
78 * wait until 'worker' finish its work.
79 */
80void propagate_children(struct __test_metadata *_metadata, struct global_data *data)
81{
82 pid_t root_pid, pid;
83 unsigned int num_child;
84 int status;
85 int ret = 0;
86 int curr_child, worker_child;
87 int curr_level = 1;
88 bool is_worker = true;
89
90 root_pid = getpid();
91repeat:
92 num_child = rand_r(&data->rand_seed) % MAX_CHILDREN + 1;
93 worker_child = is_worker ? rand_r(&data->rand_seed) % num_child : -1;
94
95 for (curr_child = 0; curr_child < num_child; curr_child++) {
96 pid = fork();
97
98 if (pid < 0) {
99 perror("Error: fork\n");
100 } else if (pid == 0) {
101 curr_level++;
102
103 if (curr_child != worker_child)
104 is_worker = false;
105
106 if (curr_level == TOTAL_LEVEL)
107 break;
108
109 data->rand_seed += curr_child;
110 goto repeat;
111 }
112 }
113
114 if (data->do_prepare)
115 data->do_prepare(data);
116
117 close(data->pipefd[1]);
118
119 if (is_worker && curr_level == data->worker_level) {
120 /* This is the worker process, first wait last process created */
121 char buf;
122
123 while (read(data->pipefd[0], &buf, 1) > 0)
124 ;
125
126 if (data->do_work)
127 ret = data->do_work(data);
128
129 /* Kick others */
130 semctl(data->semid, 0, IPC_RMID);
131 } else {
132 /* Wait worker finish */
133 semop(data->semid, &sem_wait, 1);
134 if (data->do_check)
135 ret = data->do_check(data);
136 }
137
138 /* Wait all child to quit */
139 while (wait(&status) > 0) {
140 if (WIFEXITED(status))
141 ret |= WEXITSTATUS(status);
142 }
143
144 if (getpid() == root_pid) {
145 if (ret & FAIL_ON_WORK)
146 SKIP(return, "Failed in worker");
147
148 ASSERT_EQ(ret, 0);
149 } else {
150 exit(ret);
151 }
152}
153
154FIXTURE(migrate)
155{
156 struct global_data data;
157};
158
159FIXTURE_SETUP(migrate)
160{
161 struct global_data *data = &self->data;
162
163 if (numa_available() < 0)
164 SKIP(return, "NUMA not available");
165 if (numa_bitmask_weight(numa_all_nodes_ptr) <= 1)
166 SKIP(return, "Not enough NUMA nodes available");
167
168 data->mapsize = getpagesize();
169
170 data->expected_pfn = mmap(0, sizeof(unsigned long),
171 PROT_READ | PROT_WRITE,
172 MAP_SHARED | MAP_ANONYMOUS, -1, 0);
173 ASSERT_NE(data->expected_pfn, MAP_FAILED);
174
175 /* Prepare semaphore */
176 data->semid = semget(IPC_PRIVATE, 1, 0666 | IPC_CREAT);
177 ASSERT_NE(data->semid, -1);
178 ASSERT_NE(semctl(data->semid, 0, SETVAL, 0), -1);
179
180 /* Prepare pipe */
181 ASSERT_NE(pipe(data->pipefd), -1);
182
183 data->rand_seed = time(NULL);
184 srand(data->rand_seed);
185
186 data->worker_level = rand() % TOTAL_LEVEL + 1;
187
188 data->do_prepare = NULL;
189 data->do_work = NULL;
190 data->do_check = NULL;
191
192 data->backend = ANON;
193};
194
195FIXTURE_TEARDOWN(migrate)
196{
197 struct global_data *data = &self->data;
198
199 if (data->region != MAP_FAILED)
200 munmap(data->region, data->mapsize);
201 data->region = MAP_FAILED;
202 if (data->expected_pfn != MAP_FAILED)
203 munmap(data->expected_pfn, sizeof(unsigned long));
204 data->expected_pfn = MAP_FAILED;
205 semctl(data->semid, 0, IPC_RMID);
206 data->semid = -1;
207
208 close(data->pipefd[0]);
209
210 switch (data->backend) {
211 case ANON:
212 break;
213 case SHM:
214 shm_unlink(data->filename);
215 break;
216 case NORM_FILE:
217 unlink(data->filename);
218 break;
219 }
220}
221
222void access_region(struct global_data *data)
223{
224 /*
225 * Force read "region" to make sure page fault in.
226 */
227 FORCE_READ(*data->region);
228}
229
230int try_to_move_page(char *region)
231{
232 int ret;
233 int node;
234 int status = 0;
235 int failures = 0;
236
237 ret = move_pages(0, 1, (void **)&region, NULL, &status, MPOL_MF_MOVE_ALL);
238 if (ret != 0) {
239 perror("Failed to get original numa");
240 return FAIL_ON_WORK;
241 }
242
243 /* Pick up a different target node */
244 for (node = 0; node <= numa_max_node(); node++) {
245 if (numa_bitmask_isbitset(numa_all_nodes_ptr, node) && node != status)
246 break;
247 }
248
249 if (node > numa_max_node()) {
250 ksft_print_msg("Couldn't find available numa node for testing\n");
251 return FAIL_ON_WORK;
252 }
253
254 while (1) {
255 ret = move_pages(0, 1, (void **)&region, &node, &status, MPOL_MF_MOVE_ALL);
256
257 /* migrate successfully */
258 if (!ret)
259 break;
260
261 /* error happened */
262 if (ret < 0) {
263 ksft_perror("Failed to move pages");
264 return FAIL_ON_WORK;
265 }
266
267 /* migration is best effort; try again */
268 if (++failures >= 100)
269 return FAIL_ON_WORK;
270 }
271
272 return 0;
273}
274
275int move_region(struct global_data *data)
276{
277 int ret;
278 int pagemap_fd;
279
280 ret = try_to_move_page(data->region);
281 if (ret != 0)
282 return ret;
283
284 pagemap_fd = open("/proc/self/pagemap", O_RDONLY);
285 if (pagemap_fd == -1)
286 return FAIL_ON_WORK;
287 *data->expected_pfn = pagemap_get_pfn(pagemap_fd, data->region);
288
289 return 0;
290}
291
292int has_same_pfn(struct global_data *data)
293{
294 unsigned long pfn;
295 int pagemap_fd;
296
297 if (data->region == MAP_FAILED)
298 return 0;
299
300 pagemap_fd = open("/proc/self/pagemap", O_RDONLY);
301 if (pagemap_fd == -1)
302 return FAIL_ON_CHECK;
303
304 pfn = pagemap_get_pfn(pagemap_fd, data->region);
305 if (pfn != *data->expected_pfn)
306 return FAIL_ON_CHECK;
307
308 return 0;
309}
310
311TEST_F(migrate, anon)
312{
313 struct global_data *data = &self->data;
314
315 /* Map an area and fault in */
316 data->region = mmap(0, data->mapsize, PROT_READ | PROT_WRITE,
317 MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
318 ASSERT_NE(data->region, MAP_FAILED);
319 memset(data->region, 0xcf, data->mapsize);
320
321 data->do_prepare = access_region;
322 data->do_work = move_region;
323 data->do_check = has_same_pfn;
324
325 propagate_children(_metadata, data);
326}
327
328TEST_F(migrate, shm)
329{
330 int shm_fd;
331 struct global_data *data = &self->data;
332
333 snprintf(data->filename, MAX_FILENAME_LEN, "%s%s", PREFIX, suffixes[SHM]);
334 shm_fd = shm_open(data->filename, O_CREAT | O_RDWR, 0666);
335 ASSERT_NE(shm_fd, -1);
336 ftruncate(shm_fd, data->mapsize);
337 data->backend = SHM;
338
339 /* Map a shared area and fault in */
340 data->region = mmap(0, data->mapsize, PROT_READ | PROT_WRITE,
341 MAP_SHARED, shm_fd, 0);
342 ASSERT_NE(data->region, MAP_FAILED);
343 memset(data->region, 0xcf, data->mapsize);
344 close(shm_fd);
345
346 data->do_prepare = access_region;
347 data->do_work = move_region;
348 data->do_check = has_same_pfn;
349
350 propagate_children(_metadata, data);
351}
352
353TEST_F(migrate, file)
354{
355 int fd;
356 struct global_data *data = &self->data;
357
358 snprintf(data->filename, MAX_FILENAME_LEN, "%s%s", PREFIX, suffixes[NORM_FILE]);
359 fd = open(data->filename, O_CREAT | O_RDWR | O_EXCL, 0666);
360 ASSERT_NE(fd, -1);
361 ftruncate(fd, data->mapsize);
362 data->backend = NORM_FILE;
363
364 /* Map a shared area and fault in */
365 data->region = mmap(0, data->mapsize, PROT_READ | PROT_WRITE,
366 MAP_SHARED, fd, 0);
367 ASSERT_NE(data->region, MAP_FAILED);
368 memset(data->region, 0xcf, data->mapsize);
369 close(fd);
370
371 data->do_prepare = access_region;
372 data->do_work = move_region;
373 data->do_check = has_same_pfn;
374
375 propagate_children(_metadata, data);
376}
377
378void prepare_local_region(struct global_data *data)
379{
380 /* Allocate range and set the same data */
381 data->region = mmap(NULL, data->mapsize, PROT_READ|PROT_WRITE,
382 MAP_PRIVATE|MAP_ANON, -1, 0);
383 if (data->region == MAP_FAILED)
384 return;
385
386 memset(data->region, 0xcf, data->mapsize);
387}
388
389int merge_and_migrate(struct global_data *data)
390{
391 int pagemap_fd;
392 int ret = 0;
393
394 if (data->region == MAP_FAILED)
395 return FAIL_ON_WORK;
396
397 if (ksm_start() < 0)
398 return FAIL_ON_WORK;
399
400 ret = try_to_move_page(data->region);
401
402 pagemap_fd = open("/proc/self/pagemap", O_RDONLY);
403 if (pagemap_fd == -1)
404 return FAIL_ON_WORK;
405 *data->expected_pfn = pagemap_get_pfn(pagemap_fd, data->region);
406
407 return ret;
408}
409
410TEST_F(migrate, ksm)
411{
412 int ret;
413 struct global_data *data = &self->data;
414
415 if (ksm_stop() < 0)
416 SKIP(return, "accessing \"/sys/kernel/mm/ksm/run\") failed");
417 if (ksm_get_full_scans() < 0)
418 SKIP(return, "accessing \"/sys/kernel/mm/ksm/full_scan\") failed");
419
420 ret = prctl(PR_SET_MEMORY_MERGE, 1, 0, 0, 0);
421 if (ret < 0 && errno == EINVAL)
422 SKIP(return, "PR_SET_MEMORY_MERGE not supported");
423 else if (ret)
424 ksft_exit_fail_perror("PR_SET_MEMORY_MERGE=1 failed");
425
426 data->do_prepare = prepare_local_region;
427 data->do_work = merge_and_migrate;
428 data->do_check = has_same_pfn;
429
430 propagate_children(_metadata, data);
431}
432
433static bool range_maps_the_same_pfn(int pagemap_fd, void *region, int nr_pages)
434{
435 int i;
436 int retries = 0;
437 unsigned long first_pfn;
438
439retry:
440 if (retries > 10)
441 return false;
442
443 first_pfn = pagemap_get_pfn(pagemap_fd, region);
444 for (i = 0; i < nr_pages; i++) {
445 if (pagemap_get_pfn(pagemap_fd, region + i * getpagesize()) != first_pfn) {
446 /*
447 * Retry up to 10 times at most in case of the low chance of page
448 * compaction migrating the page while we check for pfn.
449 */
450 retries++;
451 goto retry;
452 }
453 }
454
455 return true;
456}
457
458TEST_F(migrate, ksm_and_mremap)
459{
460 unsigned long old_pfn, new_pfn;
461 void *region, *mremap_region;
462 const int nr_pages = 16;
463 size_t mmap_size;
464 int pagemap_fd;
465
466 /* Skip if KSM is not available */
467 if (ksm_stop() < 0)
468 SKIP(return, "accessing \"/sys/kernel/mm/ksm/run\" failed");
469 if (ksm_get_full_scans() < 0)
470 SKIP(return, "accessing \"/sys/kernel/mm/ksm/full_scan\" failed");
471
472 pagemap_fd = open("/proc/self/pagemap", O_RDONLY);
473 if (pagemap_fd < 0)
474 SKIP(return, "opening pagemap failed");
475
476 /* Allocate and populate twice the anon pages initially. */
477 mmap_size = 2 * nr_pages * getpagesize();
478 region = mmap(NULL, mmap_size, PROT_READ | PROT_WRITE,
479 MAP_PRIVATE | MAP_ANON, -1, 0);
480 ASSERT_NE(region, MAP_FAILED);
481 memset(region, 0x77, mmap_size);
482
483 /* mremap the second half over the first half, to stress rmap handling */
484 mmap_size /= 2;
485 mremap_region = mremap(region + mmap_size, mmap_size, mmap_size,
486 MREMAP_MAYMOVE | MREMAP_FIXED, region);
487 ASSERT_EQ(mremap_region, region);
488
489 /* Merge all pages into a single KSM page. */
490 madvise(region, mmap_size, MADV_MERGEABLE);
491 ASSERT_EQ(ksm_start(), 0);
492
493 /* The whole range should map the same KSM page. */
494 old_pfn = pagemap_get_pfn(pagemap_fd, region);
495 if (old_pfn == -1ul)
496 SKIP(return, "Obtaining PFN failed");
497 ksm_start();
498 ASSERT_TRUE(range_maps_the_same_pfn(pagemap_fd, region, nr_pages));
499
500 /*
501 * Migrate the KSM page; the whole range should map the new (migrated)
502 * KSM page.
503 */
504 ASSERT_EQ(try_to_move_page(region), 0);
505
506 new_pfn = pagemap_get_pfn(pagemap_fd, region);
507 if (new_pfn == -1ul)
508 SKIP(return, "Obtaining PFN failed");
509 ASSERT_NE(new_pfn, old_pfn);
510 ASSERT_TRUE(range_maps_the_same_pfn(pagemap_fd, region, nr_pages));
511}
512
513
514TEST_HARNESS_MAIN