projects/linux·blob master · refs

995 lines · 21.8 KiB

1// SPDX-License-Identifier: GPL-2.0
2#include <string.h>
3#include <errno.h>
4#include <fcntl.h>
5#include <inttypes.h>
6#include <sys/ioctl.h>
7#include <linux/userfaultfd.h>
8#include <linux/fs.h>
9#include <sys/syscall.h>
10#include <unistd.h>
11#include "kselftest.h"
12#include "vm_util.h"
13
14#define PMD_SIZE_FILE_PATH "/sys/kernel/mm/transparent_hugepage/hpage_pmd_size"
15#define SMAP_FILE_PATH "/proc/self/smaps"
16#define STATUS_FILE_PATH "/proc/self/status"
17#define MAX_LINE_LENGTH 500
18#define PAGEMAP_PATH "/proc/self/pagemap"
19#define KPAGEFLAGS_PATH "/proc/kpageflags"
20#define MAX_NR_ORDERS 20
21
22unsigned int __page_size;
23unsigned int __page_shift;
24
25uint64_t pagemap_get_entry(int fd, char *start)
26{
27 const unsigned long pfn = (unsigned long)start / getpagesize();
28 uint64_t entry;
29 int ret;
30
31 ret = pread(fd, &entry, sizeof(entry), pfn * sizeof(entry));
32 if (ret != sizeof(entry))
33 ksft_exit_fail_msg("reading pagemap failed\n");
34 return entry;
35}
36
37static int __pagemap_scan_get_categories(int fd, char *start, struct page_region *r)
38{
39 struct pm_scan_arg arg;
40
41 arg.start = (uintptr_t)start;
42 arg.end = (uintptr_t)(start + psize());
43 arg.vec = (uintptr_t)r;
44 arg.vec_len = 1;
45 arg.flags = 0;
46 arg.size = sizeof(struct pm_scan_arg);
47 arg.max_pages = 0;
48 arg.category_inverted = 0;
49 arg.category_mask = 0;
50 arg.category_anyof_mask = PAGE_IS_WPALLOWED | PAGE_IS_WRITTEN | PAGE_IS_FILE |
51 PAGE_IS_PRESENT | PAGE_IS_SWAPPED | PAGE_IS_PFNZERO |
52 PAGE_IS_HUGE | PAGE_IS_SOFT_DIRTY;
53 arg.return_mask = arg.category_anyof_mask;
54
55 return ioctl(fd, PAGEMAP_SCAN, &arg);
56}
57
58static uint64_t pagemap_scan_get_categories(int fd, char *start)
59{
60 struct page_region r;
61 int ret;
62
63 ret = __pagemap_scan_get_categories(fd, start, &r);
64 if (ret < 0)
65 ksft_exit_fail_msg("PAGEMAP_SCAN failed: %s\n", strerror(errno));
66 if (ret == 0)
67 return 0;
68 return r.categories;
69}
70
71/* `start` is any valid address. */
72static bool pagemap_scan_supported(int fd, char *start)
73{
74 static int supported = -1;
75 int ret;
76
77 if (supported != -1)
78 return supported;
79
80 /* Provide an invalid address in order to trigger EFAULT. */
81 ret = __pagemap_scan_get_categories(fd, start, (struct page_region *) ~0UL);
82 if (ret == 0)
83 ksft_exit_fail_msg("PAGEMAP_SCAN succeeded unexpectedly\n");
84
85 supported = errno == EFAULT;
86
87 return supported;
88}
89
90static bool page_entry_is(int fd, char *start, char *desc,
91 uint64_t pagemap_flags, uint64_t pagescan_flags)
92{
93 bool m = pagemap_get_entry(fd, start) & pagemap_flags;
94
95 if (pagemap_scan_supported(fd, start)) {
96 bool s = pagemap_scan_get_categories(fd, start) & pagescan_flags;
97
98 if (m == s)
99 return m;
100
101 ksft_exit_fail_msg(
102 "read and ioctl return unmatched results for %s: %d %d", desc, m, s);
103 }
104 return m;
105}
106
107bool pagemap_is_softdirty(int fd, char *start)
108{
109 return page_entry_is(fd, start, "soft-dirty",
110 PM_SOFT_DIRTY, PAGE_IS_SOFT_DIRTY);
111}
112
113bool pagemap_is_swapped(int fd, char *start)
114{
115 return page_entry_is(fd, start, "swap", PM_SWAP, PAGE_IS_SWAPPED);
116}
117
118bool pagemap_is_populated(int fd, char *start)
119{
120 return page_entry_is(fd, start, "populated",
121 PM_PRESENT | PM_SWAP,
122 PAGE_IS_PRESENT | PAGE_IS_SWAPPED);
123}
124
125unsigned long pagemap_get_pfn(int fd, char *start)
126{
127 uint64_t entry = pagemap_get_entry(fd, start);
128
129 /* If present (63th bit), PFN is at bit 0 -- 54. */
130 if (entry & PM_PRESENT)
131 return entry & 0x007fffffffffffffull;
132 return -1ul;
133}
134
135void clear_softdirty(void)
136{
137 int ret;
138 const char *ctrl = "4";
139 int fd = open("/proc/self/clear_refs", O_WRONLY);
140
141 if (fd < 0)
142 ksft_exit_fail_msg("opening clear_refs failed\n");
143 ret = write(fd, ctrl, strlen(ctrl));
144 close(fd);
145 if (ret != (signed int)strlen(ctrl))
146 ksft_exit_fail_msg("writing clear_refs failed\n");
147}
148
149bool check_for_pattern(FILE *fp, const char *pattern, char *buf, size_t len)
150{
151 while (fgets(buf, len, fp)) {
152 if (!strncmp(buf, pattern, strlen(pattern)))
153 return true;
154 }
155 return false;
156}
157
158uint64_t read_pmd_pagesize(void)
159{
160 int fd;
161 char buf[20];
162 ssize_t num_read;
163
164 fd = open(PMD_SIZE_FILE_PATH, O_RDONLY);
165 if (fd == -1)
166 return 0;
167
168 num_read = read(fd, buf, 19);
169 if (num_read < 1) {
170 close(fd);
171 return 0;
172 }
173 buf[num_read] = '\0';
174 close(fd);
175
176 return strtoul(buf, NULL, 10);
177}
178
179unsigned long rss_anon(void)
180{
181 unsigned long rss_anon = 0;
182 FILE *fp;
183 char buffer[MAX_LINE_LENGTH];
184
185 fp = fopen(STATUS_FILE_PATH, "r");
186 if (!fp)
187 ksft_exit_fail_msg("%s: Failed to open file %s\n", __func__, STATUS_FILE_PATH);
188
189 if (!check_for_pattern(fp, "RssAnon:", buffer, sizeof(buffer)))
190 goto err_out;
191
192 if (sscanf(buffer, "RssAnon:%10lu kB", &rss_anon) != 1)
193 ksft_exit_fail_msg("Reading status error\n");
194
195err_out:
196 fclose(fp);
197 return rss_anon;
198}
199
200static int vaddr_pageflags_get(char *vaddr, int pagemap_fd, int kpageflags_fd,
201 uint64_t *flags)
202{
203 unsigned long pfn;
204
205 pfn = pagemap_get_pfn(pagemap_fd, vaddr);
206
207 /* non-present PFN */
208 if (pfn == -1UL)
209 return 1;
210
211 if (pageflags_get(pfn, kpageflags_fd, flags))
212 return -1;
213
214 return 0;
215}
216
217/*
218 * gather_folio_orders - scan through [vaddr_start, len) and record
219 * folio orders
220 *
221 * @vaddr_start: start vaddr
222 * @len: range length
223 * @pagemap_fd: file descriptor to /proc/<pid>/pagemap
224 * @kpageflags_fd: file descriptor to /proc/kpageflags
225 * @orders: output folio order array
226 * @nr_orders: folio order array size
227 *
228 * gather_folio_orders() scan through [vaddr_start, len) and check
229 * all folios within the range and record their orders. All order-0 pages will
230 * be recorded. Non-present vaddr is skipped.
231 *
232 * Return: 0 - no error, -1 - unhandled cases
233 */
234int gather_folio_orders(char *vaddr_start, size_t len,
235 int pagemap_fd, int kpageflags_fd, int orders[], int nr_orders)
236{
237 uint64_t page_flags = 0;
238 int cur_order = -1;
239 char *vaddr;
240
241 if (pagemap_fd == -1 || kpageflags_fd == -1)
242 return -1;
243 if (!orders)
244 return -1;
245 if (nr_orders <= 0)
246 return -1;
247
248 for (vaddr = vaddr_start; vaddr < vaddr_start + len;) {
249 char *next_folio_vaddr;
250 int status;
251
252 status = vaddr_pageflags_get(vaddr, pagemap_fd, kpageflags_fd,
253 &page_flags);
254 if (status < 0)
255 return -1;
256
257 /* skip non present vaddr */
258 if (status == 1) {
259 vaddr += psize();
260 continue;
261 }
262
263 /* all order-0 pages with possible false postive (non folio) */
264 if (!(page_flags & (KPF_COMPOUND_HEAD | KPF_COMPOUND_TAIL))) {
265 orders[0]++;
266 vaddr += psize();
267 continue;
268 }
269
270 /* skip non thp compound pages */
271 if (!(page_flags & KPF_THP)) {
272 vaddr += psize();
273 continue;
274 }
275
276 /* vpn points to part of a THP at this point */
277 if (page_flags & KPF_COMPOUND_HEAD)
278 cur_order = 1;
279 else {
280 vaddr += psize();
281 continue;
282 }
283
284 next_folio_vaddr = vaddr + (1UL << (cur_order + pshift()));
285
286 if (next_folio_vaddr >= vaddr_start + len)
287 break;
288
289 while ((status = vaddr_pageflags_get(next_folio_vaddr,
290 pagemap_fd, kpageflags_fd,
291 &page_flags)) >= 0) {
292 /*
293 * non present vaddr, next compound head page, or
294 * order-0 page
295 */
296 if (status == 1 ||
297 (page_flags & KPF_COMPOUND_HEAD) ||
298 !(page_flags & (KPF_COMPOUND_HEAD | KPF_COMPOUND_TAIL))) {
299 if (cur_order < nr_orders) {
300 orders[cur_order]++;
301 cur_order = -1;
302 vaddr = next_folio_vaddr;
303 }
304 break;
305 }
306
307 cur_order++;
308 next_folio_vaddr = vaddr + (1UL << (cur_order + pshift()));
309 }
310
311 if (status < 0)
312 return status;
313 }
314 if (cur_order > 0 && cur_order < nr_orders)
315 orders[cur_order]++;
316 return 0;
317}
318
319char *__get_smap_entry(void *addr, const char *pattern, char *buf, size_t len)
320{
321 int ret;
322 FILE *fp;
323 char *entry = NULL;
324 char addr_pattern[MAX_LINE_LENGTH];
325
326 ret = snprintf(addr_pattern, MAX_LINE_LENGTH, "%08lx-",
327 (unsigned long) addr);
328 if (ret >= MAX_LINE_LENGTH)
329 ksft_exit_fail_msg("%s: Pattern is too long\n", __func__);
330
331 fp = fopen(SMAP_FILE_PATH, "r");
332 if (!fp)
333 ksft_exit_fail_msg("%s: Failed to open file %s\n", __func__, SMAP_FILE_PATH);
334
335 if (!check_for_pattern(fp, addr_pattern, buf, len))
336 goto err_out;
337
338 /* Fetch the pattern in the same block */
339 if (!check_for_pattern(fp, pattern, buf, len))
340 goto err_out;
341
342 /* Trim trailing newline */
343 entry = strchr(buf, '\n');
344 if (entry)
345 *entry = '\0';
346
347 entry = buf + strlen(pattern);
348
349err_out:
350 fclose(fp);
351 return entry;
352}
353
354static bool __check_pmd_huge(void *addr, char *pattern, int nr_hpages,
355 uint64_t hpage_size)
356{
357 char buffer[MAX_LINE_LENGTH];
358 uint64_t thp = -1;
359 char *entry;
360
361 entry = __get_smap_entry(addr, pattern, buffer, sizeof(buffer));
362 if (!entry)
363 goto err_out;
364
365 if (sscanf(entry, "%9" SCNu64 " kB", &thp) != 1)
366 ksft_exit_fail_msg("Reading smap error\n");
367
368err_out:
369 return thp == (nr_hpages * (hpage_size >> 10));
370}
371
372static bool check_large_folios(void *addr, size_t len, int nr_hpages,
373 uint64_t hpage_size)
374{
375 int order = 0, pagesize = getpagesize();
376 unsigned int nr_pages = hpage_size / pagesize;
377 int orders[MAX_NR_ORDERS], status;
378 int pagemap_fd, kpageflags_fd;
379 bool ret = false;
380
381 if (!nr_pages)
382 ksft_exit_fail_msg("invalid hugepage size\n");
383
384 order = 31 - __builtin_clz(nr_pages);
385 if (!order || order >= MAX_NR_ORDERS)
386 ksft_exit_fail_msg("invalid order\n");
387
388 memset(orders, 0, sizeof(int) * MAX_NR_ORDERS);
389 pagemap_fd = open(PAGEMAP_PATH, O_RDONLY);
390 if (pagemap_fd == -1)
391 ksft_exit_fail_msg("read pagemap fail\n");
392
393 kpageflags_fd = open(KPAGEFLAGS_PATH, O_RDONLY);
394 if (kpageflags_fd == -1) {
395 close(pagemap_fd);
396 ksft_exit_fail_msg("read kpageflags fail\n");
397 }
398
399 status = gather_folio_orders(addr, len, pagemap_fd,
400 kpageflags_fd, orders, MAX_NR_ORDERS);
401 if (status)
402 goto out;
403
404 if (orders[order] == nr_hpages)
405 ret = true;
406
407out:
408 close(pagemap_fd);
409 close(kpageflags_fd);
410 return ret;
411}
412
413bool check_huge_anon(void *addr, size_t len, int nr_hpages, uint64_t hpage_size)
414{
415 uint64_t pmd_pagesize = read_pmd_pagesize();
416
417 if (!pmd_pagesize)
418 ksft_exit_fail_msg("reading PMD pagesize failed\n");
419
420 if (hpage_size == pmd_pagesize)
421 return __check_pmd_huge(addr, "AnonHugePages: ", nr_hpages, hpage_size);
422
423 return check_large_folios(addr, len, nr_hpages, hpage_size);
424}
425
426bool check_huge_file(void *addr, size_t len, int nr_hpages, uint64_t hpage_size)
427{
428 uint64_t pmd_pagesize = read_pmd_pagesize();
429
430 if (!pmd_pagesize)
431 ksft_exit_fail_msg("reading PMD pagesize failed\n");
432
433 if (hpage_size == pmd_pagesize)
434 return __check_pmd_huge(addr, "FilePmdMapped:", nr_hpages, hpage_size);
435
436 return check_large_folios(addr, len, nr_hpages, hpage_size);
437}
438
439bool check_huge_shmem(void *addr, size_t len, int nr_hpages, uint64_t hpage_size)
440{
441 uint64_t pmd_pagesize = read_pmd_pagesize();
442
443 if (!pmd_pagesize)
444 ksft_exit_fail_msg("reading PMD pagesize failed\n");
445
446 if (hpage_size == pmd_pagesize)
447 return __check_pmd_huge(addr, "ShmemPmdMapped:", nr_hpages, hpage_size);
448
449 return check_large_folios(addr, len, nr_hpages, hpage_size);
450}
451
452int64_t allocate_transhuge(void *ptr, int pagemap_fd)
453{
454 uint64_t ent[2];
455
456 /* drop pmd */
457 if (mmap(ptr, HPAGE_SIZE, PROT_READ | PROT_WRITE,
458 MAP_FIXED | MAP_ANONYMOUS |
459 MAP_NORESERVE | MAP_PRIVATE, -1, 0) != ptr)
460 ksft_exit_fail_msg("mmap transhuge\n");
461
462 if (madvise(ptr, HPAGE_SIZE, MADV_HUGEPAGE))
463 ksft_exit_fail_msg("MADV_HUGEPAGE\n");
464
465 /* allocate transparent huge page */
466 *(volatile void **)ptr = ptr;
467
468 if (pread(pagemap_fd, ent, sizeof(ent),
469 (uintptr_t)ptr >> (pshift() - 3)) != sizeof(ent))
470 ksft_exit_fail_msg("read pagemap\n");
471
472 if (PAGEMAP_PRESENT(ent[0]) && PAGEMAP_PRESENT(ent[1]) &&
473 PAGEMAP_PFN(ent[0]) + 1 == PAGEMAP_PFN(ent[1]) &&
474 !(PAGEMAP_PFN(ent[0]) & ((1 << (HPAGE_SHIFT - pshift())) - 1)))
475 return PAGEMAP_PFN(ent[0]);
476
477 return -1;
478}
479
480int pageflags_get(unsigned long pfn, int kpageflags_fd, uint64_t *flags)
481{
482 size_t count;
483
484 count = pread(kpageflags_fd, flags, sizeof(*flags),
485 pfn * sizeof(*flags));
486
487 if (count != sizeof(*flags))
488 return -1;
489
490 return 0;
491}
492
493/* If `ioctls' non-NULL, the allowed ioctls will be returned into the var */
494int uffd_register_with_ioctls(int uffd, void *addr, uint64_t len,
495 bool miss, bool wp, bool minor, uint64_t *ioctls)
496{
497 struct uffdio_register uffdio_register = { 0 };
498 uint64_t mode = 0;
499 int ret = 0;
500
501 if (miss)
502 mode |= UFFDIO_REGISTER_MODE_MISSING;
503 if (wp)
504 mode |= UFFDIO_REGISTER_MODE_WP;
505 if (minor)
506 mode |= UFFDIO_REGISTER_MODE_MINOR;
507
508 uffdio_register.range.start = (unsigned long)addr;
509 uffdio_register.range.len = len;
510 uffdio_register.mode = mode;
511
512 if (ioctl(uffd, UFFDIO_REGISTER, &uffdio_register) == -1)
513 ret = -errno;
514 else if (ioctls)
515 *ioctls = uffdio_register.ioctls;
516
517 return ret;
518}
519
520int uffd_register(int uffd, void *addr, uint64_t len,
521 bool miss, bool wp, bool minor)
522{
523 return uffd_register_with_ioctls(uffd, addr, len,
524 miss, wp, minor, NULL);
525}
526
527int uffd_unregister(int uffd, void *addr, uint64_t len)
528{
529 struct uffdio_range range = { .start = (uintptr_t)addr, .len = len };
530 int ret = 0;
531
532 if (ioctl(uffd, UFFDIO_UNREGISTER, &range) == -1)
533 ret = -errno;
534
535 return ret;
536}
537
538static bool check_vmflag(void *addr, const char *flag)
539{
540 char buffer[MAX_LINE_LENGTH];
541 const char *flags;
542 size_t flaglen;
543
544 flags = __get_smap_entry(addr, "VmFlags:", buffer, sizeof(buffer));
545 if (!flags)
546 ksft_exit_fail_msg("%s: No VmFlags for %p\n", __func__, addr);
547
548 while (true) {
549 flags += strspn(flags, " ");
550
551 flaglen = strcspn(flags, " ");
552 if (!flaglen)
553 return false;
554
555 if (flaglen == strlen(flag) && !memcmp(flags, flag, flaglen))
556 return true;
557
558 flags += flaglen;
559 }
560}
561
562bool check_vmflag_io(void *addr)
563{
564 return check_vmflag(addr, "io");
565}
566
567bool check_vmflag_pfnmap(void *addr)
568{
569 return check_vmflag(addr, "pf");
570}
571
572bool check_vmflag_guard(void *addr)
573{
574 return check_vmflag(addr, "gu");
575}
576
577bool softdirty_supported(void)
578{
579 char *addr;
580 bool supported = false;
581 const size_t pagesize = getpagesize();
582
583 /* New mappings are expected to be marked with VM_SOFTDIRTY (sd). */
584 addr = mmap(0, pagesize, PROT_READ | PROT_WRITE,
585 MAP_ANONYMOUS | MAP_PRIVATE, 0, 0);
586 if (addr == MAP_FAILED)
587 ksft_exit_fail_msg("mmap failed\n");
588
589 supported = check_vmflag(addr, "sd");
590 munmap(addr, pagesize);
591 return supported;
592}
593
594/*
595 * Open an fd at /proc/$pid/maps and configure procmap_out ready for
596 * PROCMAP_QUERY query. Returns 0 on success, or an error code otherwise.
597 */
598int open_procmap(pid_t pid, struct procmap_fd *procmap_out)
599{
600 char path[256];
601 int ret = 0;
602
603 memset(procmap_out, '\0', sizeof(*procmap_out));
604 sprintf(path, "/proc/%d/maps", pid);
605 procmap_out->query.size = sizeof(procmap_out->query);
606 procmap_out->fd = open(path, O_RDONLY);
607 if (procmap_out->fd < 0)
608 ret = -errno;
609
610 return ret;
611}
612
613/* Perform PROCMAP_QUERY. Returns 0 on success, or an error code otherwise. */
614int query_procmap(struct procmap_fd *procmap)
615{
616 int ret = 0;
617
618 if (ioctl(procmap->fd, PROCMAP_QUERY, &procmap->query) == -1)
619 ret = -errno;
620
621 return ret;
622}
623
624/*
625 * Try to find the VMA at specified address, returns true if found, false if not
626 * found, and the test is failed if any other error occurs.
627 *
628 * On success, procmap->query is populated with the results.
629 */
630bool find_vma_procmap(struct procmap_fd *procmap, void *address)
631{
632 int err;
633
634 procmap->query.query_flags = 0;
635 procmap->query.query_addr = (unsigned long)address;
636 err = query_procmap(procmap);
637 if (!err)
638 return true;
639
640 if (err != -ENOENT)
641 ksft_exit_fail_msg("%s: Error %d on ioctl(PROCMAP_QUERY)\n",
642 __func__, err);
643 return false;
644}
645
646/*
647 * Close fd used by PROCMAP_QUERY mechanism. Returns 0 on success, or an error
648 * code otherwise.
649 */
650int close_procmap(struct procmap_fd *procmap)
651{
652 return close(procmap->fd);
653}
654
655int write_sysfs(const char *file_path, unsigned long val)
656{
657 FILE *f = fopen(file_path, "w");
658
659 if (!f) {
660 fprintf(stderr, "f %s\n", file_path);
661 perror("fopen");
662 return 1;
663 }
664 if (fprintf(f, "%lu", val) < 0) {
665 perror("fprintf");
666 fclose(f);
667 return 1;
668 }
669 fclose(f);
670
671 return 0;
672}
673
674int read_sysfs(const char *file_path, unsigned long *val)
675{
676 FILE *f = fopen(file_path, "r");
677
678 if (!f) {
679 fprintf(stderr, "f %s\n", file_path);
680 perror("fopen");
681 return 1;
682 }
683 if (fscanf(f, "%lu", val) != 1) {
684 perror("fscanf");
685 fclose(f);
686 return 1;
687 }
688 fclose(f);
689
690 return 0;
691}
692
693void *sys_mremap(void *old_address, unsigned long old_size,
694 unsigned long new_size, int flags, void *new_address)
695{
696 return (void *)syscall(__NR_mremap, (unsigned long)old_address,
697 old_size, new_size, flags,
698 (unsigned long)new_address);
699}
700
701bool detect_huge_zeropage(void)
702{
703 int fd = open("/sys/kernel/mm/transparent_hugepage/use_zero_page",
704 O_RDONLY);
705 bool enabled = 0;
706 char buf[15];
707 int ret;
708
709 if (fd < 0)
710 return 0;
711
712 ret = pread(fd, buf, sizeof(buf), 0);
713 if (ret > 0 && ret < sizeof(buf)) {
714 buf[ret] = 0;
715
716 if (strtoul(buf, NULL, 10) == 1)
717 enabled = 1;
718 }
719
720 close(fd);
721 return enabled;
722}
723
724long ksm_get_self_zero_pages(void)
725{
726 int proc_self_ksm_stat_fd;
727 char buf[200];
728 char *substr_ksm_zero;
729 size_t value_pos;
730 ssize_t read_size;
731
732 proc_self_ksm_stat_fd = open("/proc/self/ksm_stat", O_RDONLY);
733 if (proc_self_ksm_stat_fd < 0)
734 return -errno;
735
736 read_size = pread(proc_self_ksm_stat_fd, buf, sizeof(buf) - 1, 0);
737 close(proc_self_ksm_stat_fd);
738 if (read_size < 0)
739 return -errno;
740
741 buf[read_size] = 0;
742
743 substr_ksm_zero = strstr(buf, "ksm_zero_pages");
744 if (!substr_ksm_zero)
745 return 0;
746
747 value_pos = strcspn(substr_ksm_zero, "0123456789");
748 return strtol(substr_ksm_zero + value_pos, NULL, 10);
749}
750
751long ksm_get_self_merging_pages(void)
752{
753 int proc_self_ksm_merging_pages_fd;
754 char buf[10];
755 ssize_t ret;
756
757 proc_self_ksm_merging_pages_fd = open("/proc/self/ksm_merging_pages",
758 O_RDONLY);
759 if (proc_self_ksm_merging_pages_fd < 0)
760 return -errno;
761
762 ret = pread(proc_self_ksm_merging_pages_fd, buf, sizeof(buf) - 1, 0);
763 close(proc_self_ksm_merging_pages_fd);
764 if (ret <= 0)
765 return -errno;
766 buf[ret] = 0;
767
768 return strtol(buf, NULL, 10);
769}
770
771long ksm_get_full_scans(void)
772{
773 int ksm_full_scans_fd;
774 char buf[10];
775 ssize_t ret;
776
777 ksm_full_scans_fd = open("/sys/kernel/mm/ksm/full_scans", O_RDONLY);
778 if (ksm_full_scans_fd < 0)
779 return -errno;
780
781 ret = pread(ksm_full_scans_fd, buf, sizeof(buf) - 1, 0);
782 close(ksm_full_scans_fd);
783 if (ret <= 0)
784 return -errno;
785 buf[ret] = 0;
786
787 return strtol(buf, NULL, 10);
788}
789
790int ksm_use_zero_pages(void)
791{
792 int ksm_use_zero_pages_fd;
793 ssize_t ret;
794
795 ksm_use_zero_pages_fd = open("/sys/kernel/mm/ksm/use_zero_pages", O_RDWR);
796 if (ksm_use_zero_pages_fd < 0)
797 return -errno;
798
799 ret = write(ksm_use_zero_pages_fd, "1", 1);
800 close(ksm_use_zero_pages_fd);
801 return ret == 1 ? 0 : -errno;
802}
803
804int ksm_start(void)
805{
806 int ksm_fd;
807 ssize_t ret;
808 long start_scans, end_scans;
809
810 ksm_fd = open("/sys/kernel/mm/ksm/run", O_RDWR);
811 if (ksm_fd < 0)
812 return -errno;
813
814 /* Wait for two full scans such that any possible merging happened. */
815 start_scans = ksm_get_full_scans();
816 if (start_scans < 0) {
817 close(ksm_fd);
818 return start_scans;
819 }
820 ret = write(ksm_fd, "1", 1);
821 close(ksm_fd);
822 if (ret != 1)
823 return -errno;
824 do {
825 end_scans = ksm_get_full_scans();
826 if (end_scans < 0)
827 return end_scans;
828 } while (end_scans < start_scans + 2);
829
830 return 0;
831}
832
833int ksm_stop(void)
834{
835 int ksm_fd;
836 ssize_t ret;
837
838 ksm_fd = open("/sys/kernel/mm/ksm/run", O_RDWR);
839 if (ksm_fd < 0)
840 return -errno;
841
842 ret = write(ksm_fd, "2", 1);
843 close(ksm_fd);
844 return ret == 1 ? 0 : -errno;
845}
846
847int get_hardware_corrupted_size(unsigned long *val)
848{
849 unsigned long size;
850 char *line = NULL;
851 size_t linelen = 0;
852 FILE *f = fopen("/proc/meminfo", "r");
853 int ret = -1;
854
855 if (!f)
856 return ret;
857
858 while (getline(&line, &linelen, f) > 0) {
859 if (sscanf(line, "HardwareCorrupted: %12lu kB", &size) == 1) {
860 *val = size;
861 ret = 0;
862 break;
863 }
864 }
865
866 free(line);
867 fclose(f);
868 return ret;
869}
870
871int unpoison_memory(unsigned long pfn)
872{
873 int unpoison_fd, len;
874 char buf[32];
875 ssize_t ret;
876
877 unpoison_fd = open("/sys/kernel/debug/hwpoison/unpoison-pfn", O_WRONLY);
878 if (unpoison_fd < 0)
879 return -errno;
880
881 len = sprintf(buf, "0x%lx\n", pfn);
882 ret = write(unpoison_fd, buf, len);
883 close(unpoison_fd);
884
885 return ret > 0 ? 0 : -errno;
886}
887
888int read_file(const char *path, char *buf, size_t buflen)
889{
890 int fd;
891 ssize_t numread;
892
893 fd = open(path, O_RDONLY);
894 if (fd == -1)
895 return 0;
896
897 numread = read(fd, buf, buflen - 1);
898 if (numread < 1) {
899 close(fd);
900 return 0;
901 }
902
903 buf[numread] = '\0';
904 close(fd);
905
906 return (unsigned int) numread;
907}
908
909static void __write_file(const char *path, const char *buf, size_t buflen, bool ignore_einval)
910{
911 int fd, saved_errno;
912 ssize_t numwritten;
913
914 if (buflen < 2)
915 ksft_exit_fail_msg("Incorrect buffer len: %zu\n", buflen);
916
917 fd = open(path, O_WRONLY);
918 if (fd == -1)
919 ksft_exit_fail_msg("%s open failed: %s\n", path, strerror(errno));
920
921 numwritten = write(fd, buf, buflen - 1);
922 saved_errno = errno;
923 close(fd);
924 errno = saved_errno;
925 if (numwritten < 0) {
926 if (ignore_einval && errno == EINVAL)
927 return;
928 ksft_exit_fail_msg("%s write(%.*s) failed: %s\n", path, (int)(buflen - 1),
929 buf, strerror(errno));
930 }
931 if (numwritten != buflen - 1)
932 ksft_exit_fail_msg("%s write(%.*s) is truncated, expected %zu bytes, got %zd bytes\n",
933 path, (int)(buflen - 1), buf, buflen - 1, numwritten);
934}
935
936void write_file(const char *path, const char *buf, size_t buflen)
937{
938 __write_file(path, buf, buflen, /* ignore_einval = */ false);
939}
940
941unsigned long read_num(const char *path)
942{
943 char buf[21];
944
945 if (!read_file(path, buf, sizeof(buf)))
946 ksft_exit_fail_perror("read_file()");
947
948 return strtoul(buf, NULL, 10);
949}
950
951static void __write_num(const char *path, unsigned long num, bool ignore_einval)
952{
953 char buf[21];
954
955 sprintf(buf, "%lu", num);
956 __write_file(path, buf, strlen(buf) + 1, ignore_einval);
957}
958
959void write_num(const char *path, unsigned long num)
960{
961 return __write_num(path, num, /* ignore_einval = */ false);
962}
963
964void write_num_ignore_einval(const char *path, unsigned long num)
965{
966 return __write_num(path, num, /* ignore_einval = */ true);
967}
968
969static unsigned long shmall, shmmax;
970
971void __shm_limits_restore(void)
972{
973 if (shmmax)
974 write_num("/proc/sys/kernel/shmmax", shmmax);
975 if (shmall)
976 write_num("/proc/sys/kernel/shmall", shmall);
977}
978
979void shm_limits_prepare(unsigned long length)
980{
981 unsigned long nr = length / psize();
982 unsigned long val;
983
984 val = read_num("/proc/sys/kernel/shmmax");
985 if (val < length) {
986 write_num("/proc/sys/kernel/shmmax", length);
987 shmmax = val;
988 }
989
990 val = read_num("/proc/sys/kernel/shmall");
991 if (val < nr) {
992 write_num("/proc/sys/kernel/shmall", nr);
993 shmall = val;
994 }
995}