| 1 | |
| 2 | #include <linux/kernel.h>
|
| 3 | #include <linux/mm.h>
|
| 4 | #include <linux/slab.h>
|
| 5 | #include <linux/uaccess.h>
|
| 6 | #include <linux/ktime.h>
|
| 7 | #include <linux/debugfs.h>
|
| 8 | #include <linux/highmem.h>
|
| 9 | #include "gup_test.h"
|
| 10 |
|
| 11 | struct gup_test_data {
|
| 12 | struct mutex longterm_mutex;
|
| 13 | struct page **longterm_pages;
|
| 14 | unsigned long longterm_nr_pages;
|
| 15 | };
|
| 16 |
|
| 17 | static void put_back_pages(unsigned int cmd, struct page **pages,
|
| 18 | unsigned long nr_pages, unsigned int gup_test_flags)
|
| 19 | {
|
| 20 | unsigned long i;
|
| 21 |
|
| 22 | switch (cmd) {
|
| 23 | case GUP_FAST_BENCHMARK:
|
| 24 | case GUP_BASIC_TEST:
|
| 25 | for (i = 0; i < nr_pages; i++)
|
| 26 | put_page(pages[i]);
|
| 27 | break;
|
| 28 |
|
| 29 | case PIN_FAST_BENCHMARK:
|
| 30 | case PIN_BASIC_TEST:
|
| 31 | case PIN_LONGTERM_BENCHMARK:
|
| 32 | unpin_user_pages(pages, nr_pages);
|
| 33 | break;
|
| 34 | case DUMP_USER_PAGES_TEST:
|
| 35 | if (gup_test_flags & GUP_TEST_FLAG_DUMP_PAGES_USE_PIN) {
|
| 36 | unpin_user_pages(pages, nr_pages);
|
| 37 | } else {
|
| 38 | for (i = 0; i < nr_pages; i++)
|
| 39 | put_page(pages[i]);
|
| 40 |
|
| 41 | }
|
| 42 | break;
|
| 43 | }
|
| 44 | }
|
| 45 |
|
| 46 | static void verify_dma_pinned(unsigned int cmd, struct page **pages,
|
| 47 | unsigned long nr_pages)
|
| 48 | {
|
| 49 | unsigned long i;
|
| 50 | struct folio *folio;
|
| 51 |
|
| 52 | switch (cmd) {
|
| 53 | case PIN_FAST_BENCHMARK:
|
| 54 | case PIN_BASIC_TEST:
|
| 55 | case PIN_LONGTERM_BENCHMARK:
|
| 56 | for (i = 0; i < nr_pages; i++) {
|
| 57 | folio = page_folio(pages[i]);
|
| 58 |
|
| 59 | if (WARN(!folio_maybe_dma_pinned(folio),
|
| 60 | "pages[%lu] is NOT dma-pinned\n", i)) {
|
| 61 |
|
| 62 | dump_page(&folio->page, "gup_test failure");
|
| 63 | break;
|
| 64 | } else if (cmd == PIN_LONGTERM_BENCHMARK &&
|
| 65 | WARN(!folio_is_longterm_pinnable(folio),
|
| 66 | "pages[%lu] is NOT pinnable but pinned\n",
|
| 67 | i)) {
|
| 68 | dump_page(&folio->page, "gup_test failure");
|
| 69 | break;
|
| 70 | }
|
| 71 | }
|
| 72 | break;
|
| 73 | }
|
| 74 | }
|
| 75 |
|
| 76 | static void dump_pages_test(struct gup_test *gup, struct page **pages,
|
| 77 | unsigned long nr_pages)
|
| 78 | {
|
| 79 | unsigned int index_to_dump;
|
| 80 | unsigned int i;
|
| 81 |
|
| 82 | |
| 83 | * Zero out any user-supplied page index that is out of range. Remember:
|
| 84 | * .which_pages[] contains a 1-based set of page indices.
|
| 85 |
|
| 86 | for (i = 0; i < GUP_TEST_MAX_PAGES_TO_DUMP; i++) {
|
| 87 | if (gup->which_pages[i] > nr_pages) {
|
| 88 | pr_warn("ZEROING due to out of range: .which_pages[%u]: %u\n",
|
| 89 | i, gup->which_pages[i]);
|
| 90 | gup->which_pages[i] = 0;
|
| 91 | }
|
| 92 | }
|
| 93 |
|
| 94 | for (i = 0; i < GUP_TEST_MAX_PAGES_TO_DUMP; i++) {
|
| 95 | index_to_dump = gup->which_pages[i];
|
| 96 |
|
| 97 | if (index_to_dump) {
|
| 98 | index_to_dump--; |
| 99 | pr_info("---- page #%u, starting from user virt addr: 0x%llx\n",
|
| 100 | index_to_dump, gup->addr);
|
| 101 | dump_page(pages[index_to_dump],
|
| 102 | "gup_test: dump_pages() test");
|
| 103 | }
|
| 104 | }
|
| 105 | }
|
| 106 |
|
| 107 | static int __gup_test_ioctl(unsigned int cmd,
|
| 108 | struct gup_test *gup)
|
| 109 | {
|
| 110 | ktime_t start_time, end_time;
|
| 111 | unsigned long i, nr_pages, addr, next;
|
| 112 | long nr;
|
| 113 | struct page **pages;
|
| 114 | unsigned long end;
|
| 115 | int ret = 0;
|
| 116 | bool needs_mmap_lock =
|
| 117 | cmd != GUP_FAST_BENCHMARK && cmd != PIN_FAST_BENCHMARK;
|
| 118 |
|
| 119 | if (gup->addr > ULONG_MAX || gup->size > ULONG_MAX)
|
| 120 | return -EINVAL;
|
| 121 | if (check_add_overflow((unsigned long)gup->addr,
|
| 122 | (unsigned long)gup->size, &end))
|
| 123 | return -EINVAL;
|
| 124 |
|
| 125 | nr_pages = gup->size / PAGE_SIZE;
|
| 126 | pages = kvcalloc(nr_pages, sizeof(void *), GFP_KERNEL);
|
| 127 | if (!pages)
|
| 128 | return -ENOMEM;
|
| 129 |
|
| 130 | if (needs_mmap_lock && mmap_read_lock_killable(current->mm)) {
|
| 131 | ret = -EINTR;
|
| 132 | goto free_pages;
|
| 133 | }
|
| 134 |
|
| 135 | i = 0;
|
| 136 | nr = gup->nr_pages_per_call;
|
| 137 | start_time = ktime_get();
|
| 138 | for (addr = gup->addr; addr < end; addr = next) {
|
| 139 | if (nr != gup->nr_pages_per_call)
|
| 140 | break;
|
| 141 |
|
| 142 | next = addr + nr * PAGE_SIZE;
|
| 143 | if (next > end) {
|
| 144 | next = end;
|
| 145 | nr = (next - addr) / PAGE_SIZE;
|
| 146 | }
|
| 147 |
|
| 148 | switch (cmd) {
|
| 149 | case GUP_FAST_BENCHMARK:
|
| 150 | nr = get_user_pages_fast(addr, nr, gup->gup_flags,
|
| 151 | pages + i);
|
| 152 | break;
|
| 153 | case GUP_BASIC_TEST:
|
| 154 | nr = get_user_pages(addr, nr, gup->gup_flags, pages + i);
|
| 155 | break;
|
| 156 | case PIN_FAST_BENCHMARK:
|
| 157 | nr = pin_user_pages_fast(addr, nr, gup->gup_flags,
|
| 158 | pages + i);
|
| 159 | break;
|
| 160 | case PIN_BASIC_TEST:
|
| 161 | nr = pin_user_pages(addr, nr, gup->gup_flags, pages + i);
|
| 162 | break;
|
| 163 | case PIN_LONGTERM_BENCHMARK:
|
| 164 | nr = pin_user_pages(addr, nr,
|
| 165 | gup->gup_flags | FOLL_LONGTERM,
|
| 166 | pages + i);
|
| 167 | break;
|
| 168 | case DUMP_USER_PAGES_TEST:
|
| 169 | if (gup->test_flags & GUP_TEST_FLAG_DUMP_PAGES_USE_PIN)
|
| 170 | nr = pin_user_pages(addr, nr, gup->gup_flags,
|
| 171 | pages + i);
|
| 172 | else
|
| 173 | nr = get_user_pages(addr, nr, gup->gup_flags,
|
| 174 | pages + i);
|
| 175 | break;
|
| 176 | default:
|
| 177 | ret = -EINVAL;
|
| 178 | goto unlock;
|
| 179 | }
|
| 180 |
|
| 181 | if (nr <= 0)
|
| 182 | break;
|
| 183 | i += nr;
|
| 184 | }
|
| 185 | end_time = ktime_get();
|
| 186 |
|
| 187 |
|
| 188 | nr_pages = i;
|
| 189 |
|
| 190 | gup->get_delta_usec = ktime_us_delta(end_time, start_time);
|
| 191 | gup->size = addr - gup->addr;
|
| 192 |
|
| 193 | |
| 194 | * Take an un-benchmark-timed moment to verify DMA pinned
|
| 195 | * state: print a warning if any non-dma-pinned pages are found:
|
| 196 |
|
| 197 | verify_dma_pinned(cmd, pages, nr_pages);
|
| 198 |
|
| 199 | if (cmd == DUMP_USER_PAGES_TEST)
|
| 200 | dump_pages_test(gup, pages, nr_pages);
|
| 201 |
|
| 202 | start_time = ktime_get();
|
| 203 |
|
| 204 | put_back_pages(cmd, pages, nr_pages, gup->test_flags);
|
| 205 |
|
| 206 | end_time = ktime_get();
|
| 207 | gup->put_delta_usec = ktime_us_delta(end_time, start_time);
|
| 208 |
|
| 209 | unlock:
|
| 210 | if (needs_mmap_lock)
|
| 211 | mmap_read_unlock(current->mm);
|
| 212 | free_pages:
|
| 213 | kvfree(pages);
|
| 214 | return ret;
|
| 215 | }
|
| 216 |
|
| 217 | static inline void pin_longterm_test_stop(struct gup_test_data *data)
|
| 218 | {
|
| 219 | if (data->longterm_pages) {
|
| 220 | if (data->longterm_nr_pages)
|
| 221 | unpin_user_pages(data->longterm_pages,
|
| 222 | data->longterm_nr_pages);
|
| 223 | kvfree(data->longterm_pages);
|
| 224 | data->longterm_pages = NULL;
|
| 225 | data->longterm_nr_pages = 0;
|
| 226 | }
|
| 227 | }
|
| 228 |
|
| 229 | static inline int pin_longterm_test_start(struct gup_test_data *data,
|
| 230 | unsigned long arg)
|
| 231 | {
|
| 232 | long nr_pages, cur_pages, addr, remaining_pages;
|
| 233 | int gup_flags = FOLL_LONGTERM;
|
| 234 | struct pin_longterm_test args;
|
| 235 | struct page **pages;
|
| 236 | int ret = 0;
|
| 237 | bool fast;
|
| 238 |
|
| 239 | if (data->longterm_pages)
|
| 240 | return -EINVAL;
|
| 241 |
|
| 242 | if (copy_from_user(&args, (void __user *)arg, sizeof(args)))
|
| 243 | return -EFAULT;
|
| 244 |
|
| 245 | if (args.flags &
|
| 246 | ~(PIN_LONGTERM_TEST_FLAG_USE_WRITE|PIN_LONGTERM_TEST_FLAG_USE_FAST))
|
| 247 | return -EINVAL;
|
| 248 | if (!IS_ALIGNED(args.addr | args.size, PAGE_SIZE))
|
| 249 | return -EINVAL;
|
| 250 | if (args.size > LONG_MAX)
|
| 251 | return -EINVAL;
|
| 252 | nr_pages = args.size / PAGE_SIZE;
|
| 253 | if (!nr_pages)
|
| 254 | return -EINVAL;
|
| 255 |
|
| 256 | pages = kvcalloc(nr_pages, sizeof(void *), GFP_KERNEL);
|
| 257 | if (!pages)
|
| 258 | return -ENOMEM;
|
| 259 |
|
| 260 | if (args.flags & PIN_LONGTERM_TEST_FLAG_USE_WRITE)
|
| 261 | gup_flags |= FOLL_WRITE;
|
| 262 | fast = !!(args.flags & PIN_LONGTERM_TEST_FLAG_USE_FAST);
|
| 263 |
|
| 264 | if (!fast && mmap_read_lock_killable(current->mm)) {
|
| 265 | kvfree(pages);
|
| 266 | return -EINTR;
|
| 267 | }
|
| 268 |
|
| 269 | data->longterm_pages = pages;
|
| 270 | data->longterm_nr_pages = 0;
|
| 271 |
|
| 272 | while (nr_pages - data->longterm_nr_pages) {
|
| 273 | remaining_pages = nr_pages - data->longterm_nr_pages;
|
| 274 | addr = args.addr + data->longterm_nr_pages * PAGE_SIZE;
|
| 275 |
|
| 276 | if (fast)
|
| 277 | cur_pages = pin_user_pages_fast(addr, remaining_pages,
|
| 278 | gup_flags, pages);
|
| 279 | else
|
| 280 | cur_pages = pin_user_pages(addr, remaining_pages,
|
| 281 | gup_flags, pages);
|
| 282 | if (cur_pages < 0) {
|
| 283 | pin_longterm_test_stop(data);
|
| 284 | ret = cur_pages;
|
| 285 | break;
|
| 286 | }
|
| 287 | data->longterm_nr_pages += cur_pages;
|
| 288 | pages += cur_pages;
|
| 289 | }
|
| 290 |
|
| 291 | if (!fast)
|
| 292 | mmap_read_unlock(current->mm);
|
| 293 | return ret;
|
| 294 | }
|
| 295 |
|
| 296 | static inline int pin_longterm_test_read(struct gup_test_data *data,
|
| 297 | unsigned long arg)
|
| 298 | {
|
| 299 | __u64 user_addr;
|
| 300 | unsigned long i;
|
| 301 |
|
| 302 | if (!data->longterm_pages)
|
| 303 | return -EINVAL;
|
| 304 |
|
| 305 | if (copy_from_user(&user_addr, (void __user *)arg, sizeof(user_addr)))
|
| 306 | return -EFAULT;
|
| 307 |
|
| 308 | for (i = 0; i < data->longterm_nr_pages; i++) {
|
| 309 | void *addr = kmap_local_page(data->longterm_pages[i]);
|
| 310 | unsigned long ret;
|
| 311 |
|
| 312 | ret = copy_to_user((void __user *)(unsigned long)user_addr, addr,
|
| 313 | PAGE_SIZE);
|
| 314 | kunmap_local(addr);
|
| 315 | if (ret)
|
| 316 | return -EFAULT;
|
| 317 | user_addr += PAGE_SIZE;
|
| 318 | }
|
| 319 | return 0;
|
| 320 | }
|
| 321 |
|
| 322 | static long pin_longterm_test_ioctl(struct file *filep, unsigned int cmd,
|
| 323 | unsigned long arg)
|
| 324 | {
|
| 325 | struct gup_test_data *data = filep->private_data;
|
| 326 | int ret = -EINVAL;
|
| 327 |
|
| 328 | if (mutex_lock_killable(&data->longterm_mutex))
|
| 329 | return -EINTR;
|
| 330 |
|
| 331 | switch (cmd) {
|
| 332 | case PIN_LONGTERM_TEST_START:
|
| 333 | ret = pin_longterm_test_start(data, arg);
|
| 334 | break;
|
| 335 | case PIN_LONGTERM_TEST_STOP:
|
| 336 | pin_longterm_test_stop(data);
|
| 337 | ret = 0;
|
| 338 | break;
|
| 339 | case PIN_LONGTERM_TEST_READ:
|
| 340 | ret = pin_longterm_test_read(data, arg);
|
| 341 | break;
|
| 342 | }
|
| 343 |
|
| 344 | mutex_unlock(&data->longterm_mutex);
|
| 345 | return ret;
|
| 346 | }
|
| 347 |
|
| 348 | static long gup_test_ioctl(struct file *filep, unsigned int cmd,
|
| 349 | unsigned long arg)
|
| 350 | {
|
| 351 | struct gup_test gup;
|
| 352 | int ret;
|
| 353 |
|
| 354 | switch (cmd) {
|
| 355 | case GUP_FAST_BENCHMARK:
|
| 356 | case PIN_FAST_BENCHMARK:
|
| 357 | case PIN_LONGTERM_BENCHMARK:
|
| 358 | case GUP_BASIC_TEST:
|
| 359 | case PIN_BASIC_TEST:
|
| 360 | case DUMP_USER_PAGES_TEST:
|
| 361 | break;
|
| 362 | case PIN_LONGTERM_TEST_START:
|
| 363 | case PIN_LONGTERM_TEST_STOP:
|
| 364 | case PIN_LONGTERM_TEST_READ:
|
| 365 | return pin_longterm_test_ioctl(filep, cmd, arg);
|
| 366 | default:
|
| 367 | return -EINVAL;
|
| 368 | }
|
| 369 |
|
| 370 | if (copy_from_user(&gup, (void __user *)arg, sizeof(gup)))
|
| 371 | return -EFAULT;
|
| 372 |
|
| 373 | ret = __gup_test_ioctl(cmd, &gup);
|
| 374 | if (ret)
|
| 375 | return ret;
|
| 376 |
|
| 377 | if (copy_to_user((void __user *)arg, &gup, sizeof(gup)))
|
| 378 | return -EFAULT;
|
| 379 |
|
| 380 | return 0;
|
| 381 | }
|
| 382 |
|
| 383 | static int gup_test_open(struct inode *inode, struct file *file)
|
| 384 | {
|
| 385 | struct gup_test_data *data;
|
| 386 | int ret;
|
| 387 |
|
| 388 | data = kzalloc_obj(*data);
|
| 389 | if (!data)
|
| 390 | return -ENOMEM;
|
| 391 |
|
| 392 | ret = nonseekable_open(inode, file);
|
| 393 | if (ret) {
|
| 394 | kfree(data);
|
| 395 | return ret;
|
| 396 | }
|
| 397 |
|
| 398 | mutex_init(&data->longterm_mutex);
|
| 399 | file->private_data = data;
|
| 400 | return 0;
|
| 401 | }
|
| 402 |
|
| 403 | static int gup_test_release(struct inode *inode, struct file *file)
|
| 404 | {
|
| 405 | struct gup_test_data *data = file->private_data;
|
| 406 |
|
| 407 | pin_longterm_test_stop(data);
|
| 408 | mutex_destroy(&data->longterm_mutex);
|
| 409 | kfree(data);
|
| 410 | file->private_data = NULL;
|
| 411 |
|
| 412 | return 0;
|
| 413 | }
|
| 414 |
|
| 415 | static const struct file_operations gup_test_fops = {
|
| 416 | .open = gup_test_open,
|
| 417 | .unlocked_ioctl = gup_test_ioctl,
|
| 418 | .compat_ioctl = compat_ptr_ioctl,
|
| 419 | .release = gup_test_release,
|
| 420 | };
|
| 421 |
|
| 422 | static int __init gup_test_init(void)
|
| 423 | {
|
| 424 | debugfs_create_file_unsafe("gup_test", 0600, NULL, NULL,
|
| 425 | &gup_test_fops);
|
| 426 |
|
| 427 | return 0;
|
| 428 | }
|
| 429 |
|
| 430 | late_initcall(gup_test_init);
|