projects/linux·blob master · refs

1971 lines · 47.6 KiB

1// SPDX-License-Identifier: GPL-2.0
2/*
3 * This is a module to test the HMM (Heterogeneous Memory Management)
4 * mirror and zone device private memory migration APIs of the kernel.
5 * Userspace programs can register with the driver to mirror their own address
6 * space and can use the device to read/write any valid virtual address.
7 */
8#include <linux/init.h>
9#include <linux/fs.h>
10#include <linux/mm.h>
11#include <linux/module.h>
12#include <linux/kernel.h>
13#include <linux/cdev.h>
14#include <linux/device.h>
15#include <linux/memremap.h>
16#include <linux/mutex.h>
17#include <linux/rwsem.h>
18#include <linux/sched.h>
19#include <linux/slab.h>
20#include <linux/highmem.h>
21#include <linux/delay.h>
22#include <linux/pagemap.h>
23#include <linux/hmm.h>
24#include <linux/vmalloc.h>
25#include <linux/swap.h>
26#include <linux/swapops.h>
27#include <linux/sched/mm.h>
28#include <linux/platform_device.h>
29#include <linux/rmap.h>
30#include <linux/mmu_notifier.h>
31#include <linux/migrate.h>
32
33#include "test_hmm_uapi.h"
34
35#define DMIRROR_NDEVICES 4
36#define DMIRROR_RANGE_FAULT_TIMEOUT 1000
37#define DEVMEM_CHUNK_SIZE (256 * 1024 * 1024U)
38#define DEVMEM_CHUNKS_RESERVE 16
39
40/*
41 * For device_private pages, dpage is just a dummy struct page
42 * representing a piece of device memory. dmirror_devmem_alloc_page
43 * allocates a real system memory page as backing storage to fake a
44 * real device. zone_device_data points to that backing page. But
45 * for device_coherent memory, the struct page represents real
46 * physical CPU-accessible memory that we can use directly.
47 */
48#define BACKING_PAGE(page) (is_device_private_page((page)) ? \
49 (page)->zone_device_data : (page))
50
51static unsigned long spm_addr_dev0;
52module_param(spm_addr_dev0, long, 0644);
53MODULE_PARM_DESC(spm_addr_dev0,
54 "Specify start address for SPM (special purpose memory) used for device 0. By setting this Coherent device type will be used. Make sure spm_addr_dev1 is set too. Minimum SPM size should be DEVMEM_CHUNK_SIZE.");
55
56static unsigned long spm_addr_dev1;
57module_param(spm_addr_dev1, long, 0644);
58MODULE_PARM_DESC(spm_addr_dev1,
59 "Specify start address for SPM (special purpose memory) used for device 1. By setting this Coherent device type will be used. Make sure spm_addr_dev0 is set too. Minimum SPM size should be DEVMEM_CHUNK_SIZE.");
60
61static const struct dev_pagemap_ops dmirror_devmem_ops;
62static const struct mmu_interval_notifier_ops dmirror_min_ops;
63static dev_t dmirror_dev;
64
65struct dmirror_device;
66
67struct dmirror_bounce {
68 void *ptr;
69 unsigned long size;
70 unsigned long addr;
71 unsigned long cpages;
72};
73
74#define DPT_XA_TAG_ATOMIC 1UL
75#define DPT_XA_TAG_WRITE 3UL
76
77/*
78 * Data structure to track address ranges and register for mmu interval
79 * notifier updates.
80 */
81struct dmirror_interval {
82 struct mmu_interval_notifier notifier;
83 struct dmirror *dmirror;
84};
85
86/*
87 * Data attached to the open device file.
88 * Note that it might be shared after a fork().
89 */
90struct dmirror {
91 struct dmirror_device *mdevice;
92 struct xarray pt;
93 struct mmu_interval_notifier notifier;
94 struct mutex mutex;
95 __u64 flags;
96};
97
98/*
99 * ZONE_DEVICE pages for migration and simulating device memory.
100 */
101struct dmirror_chunk {
102 struct dev_pagemap pagemap;
103 struct dmirror_device *mdevice;
104 bool remove;
105};
106
107/*
108 * Per device data.
109 */
110struct dmirror_device {
111 struct cdev cdevice;
112 unsigned int zone_device_type;
113 struct device device;
114
115 unsigned int devmem_capacity;
116 unsigned int devmem_count;
117 struct dmirror_chunk **devmem_chunks;
118 struct mutex devmem_lock; /* protects the above */
119
120 unsigned long calloc;
121 unsigned long cfree;
122 struct page *free_pages;
123 struct folio *free_folios;
124 spinlock_t lock; /* protects the above */
125};
126
127static struct dmirror_device dmirror_devices[DMIRROR_NDEVICES];
128
129static int dmirror_bounce_init(struct dmirror_bounce *bounce,
130 unsigned long addr,
131 unsigned long size)
132{
133 bounce->addr = addr;
134 bounce->size = size;
135 bounce->cpages = 0;
136 bounce->ptr = vmalloc(size);
137 if (!bounce->ptr)
138 return -ENOMEM;
139 return 0;
140}
141
142static bool dmirror_is_private_zone(struct dmirror_device *mdevice)
143{
144 return (mdevice->zone_device_type ==
145 HMM_DMIRROR_MEMORY_DEVICE_PRIVATE);
146}
147
148static enum migrate_vma_direction
149dmirror_select_device(struct dmirror *dmirror)
150{
151 return (dmirror->mdevice->zone_device_type ==
152 HMM_DMIRROR_MEMORY_DEVICE_PRIVATE) ?
153 MIGRATE_VMA_SELECT_DEVICE_PRIVATE :
154 MIGRATE_VMA_SELECT_DEVICE_COHERENT;
155}
156
157static void dmirror_bounce_fini(struct dmirror_bounce *bounce)
158{
159 vfree(bounce->ptr);
160}
161
162static int dmirror_fops_open(struct inode *inode, struct file *filp)
163{
164 struct cdev *cdev = inode->i_cdev;
165 struct dmirror *dmirror;
166 int ret;
167
168 /* Mirror this process address space */
169 dmirror = kzalloc_obj(*dmirror);
170 if (dmirror == NULL)
171 return -ENOMEM;
172
173 dmirror->mdevice = container_of(cdev, struct dmirror_device, cdevice);
174 mutex_init(&dmirror->mutex);
175 xa_init(&dmirror->pt);
176
177 ret = mmu_interval_notifier_insert(&dmirror->notifier, current->mm,
178 0, ULONG_MAX & PAGE_MASK, &dmirror_min_ops);
179 if (ret) {
180 kfree(dmirror);
181 return ret;
182 }
183
184 filp->private_data = dmirror;
185 return 0;
186}
187
188static void dmirror_device_evict_chunk(struct dmirror_chunk *chunk)
189{
190 unsigned long start_pfn = chunk->pagemap.range.start >> PAGE_SHIFT;
191 unsigned long end_pfn = chunk->pagemap.range.end >> PAGE_SHIFT;
192 unsigned long npages = end_pfn - start_pfn + 1;
193 unsigned long i;
194 unsigned long *src_pfns;
195 unsigned long *dst_pfns;
196 unsigned int order = 0;
197
198 src_pfns = kvcalloc(npages, sizeof(*src_pfns), GFP_KERNEL | __GFP_NOFAIL);
199 dst_pfns = kvcalloc(npages, sizeof(*dst_pfns), GFP_KERNEL | __GFP_NOFAIL);
200
201 migrate_device_range(src_pfns, start_pfn, npages);
202 for (i = 0; i < npages; i++) {
203 struct page *dpage, *spage;
204
205 spage = migrate_pfn_to_page(src_pfns[i]);
206 if (!spage || !(src_pfns[i] & MIGRATE_PFN_MIGRATE))
207 continue;
208
209 if (WARN_ON(!is_device_private_page(spage) &&
210 !is_device_coherent_page(spage)))
211 continue;
212
213 order = folio_order(page_folio(spage));
214 spage = BACKING_PAGE(spage);
215 if (src_pfns[i] & MIGRATE_PFN_COMPOUND) {
216 dpage = folio_page(folio_alloc(GFP_HIGHUSER_MOVABLE,
217 order), 0);
218 } else {
219 dpage = alloc_page(GFP_HIGHUSER_MOVABLE | __GFP_NOFAIL);
220 order = 0;
221 }
222
223 /* TODO Support splitting here */
224 lock_page(dpage);
225 dst_pfns[i] = migrate_pfn(page_to_pfn(dpage));
226 if (src_pfns[i] & MIGRATE_PFN_WRITE)
227 dst_pfns[i] |= MIGRATE_PFN_WRITE;
228 if (order)
229 dst_pfns[i] |= MIGRATE_PFN_COMPOUND;
230 folio_copy(page_folio(dpage), page_folio(spage));
231 }
232 migrate_device_pages(src_pfns, dst_pfns, npages);
233 migrate_device_finalize(src_pfns, dst_pfns, npages);
234 kvfree(src_pfns);
235 kvfree(dst_pfns);
236}
237
238static int dmirror_fops_release(struct inode *inode, struct file *filp)
239{
240 struct dmirror *dmirror = filp->private_data;
241 struct dmirror_device *mdevice = dmirror->mdevice;
242 int i;
243
244 mmu_interval_notifier_remove(&dmirror->notifier);
245
246 if (mdevice->devmem_chunks) {
247 for (i = 0; i < mdevice->devmem_count; i++) {
248 struct dmirror_chunk *devmem =
249 mdevice->devmem_chunks[i];
250
251 dmirror_device_evict_chunk(devmem);
252 }
253 }
254
255 xa_destroy(&dmirror->pt);
256 kfree(dmirror);
257 return 0;
258}
259
260static struct dmirror_chunk *dmirror_page_to_chunk(struct page *page)
261{
262 return container_of(page_pgmap(page), struct dmirror_chunk,
263 pagemap);
264}
265
266static struct dmirror_device *dmirror_page_to_device(struct page *page)
267
268{
269 return dmirror_page_to_chunk(page)->mdevice;
270}
271
272static int dmirror_do_fault(struct dmirror *dmirror, struct hmm_range *range)
273{
274 unsigned long *pfns = range->hmm_pfns;
275 unsigned long pfn;
276
277 for (pfn = (range->start >> PAGE_SHIFT);
278 pfn < (range->end >> PAGE_SHIFT);
279 pfn++, pfns++) {
280 struct page *page;
281 void *entry;
282
283 /*
284 * Since we asked for hmm_range_fault() to populate pages,
285 * it shouldn't return an error entry on success.
286 */
287 WARN_ON(*pfns & HMM_PFN_ERROR);
288 WARN_ON(!(*pfns & HMM_PFN_VALID));
289
290 page = hmm_pfn_to_page(*pfns);
291 WARN_ON(!page);
292
293 entry = page;
294 if (*pfns & HMM_PFN_WRITE)
295 entry = xa_tag_pointer(entry, DPT_XA_TAG_WRITE);
296 else if (WARN_ON(range->default_flags & HMM_PFN_WRITE))
297 return -EFAULT;
298 entry = xa_store(&dmirror->pt, pfn, entry, GFP_ATOMIC);
299 if (xa_is_err(entry))
300 return xa_err(entry);
301 }
302
303 return 0;
304}
305
306static void dmirror_do_update(struct dmirror *dmirror, unsigned long start,
307 unsigned long end)
308{
309 unsigned long pfn;
310 void *entry;
311
312 /*
313 * The XArray doesn't hold references to pages since it relies on
314 * the mmu notifier to clear page pointers when they become stale.
315 * Therefore, it is OK to just clear the entry.
316 */
317 xa_for_each_range(&dmirror->pt, pfn, entry, start >> PAGE_SHIFT,
318 end >> PAGE_SHIFT)
319 xa_erase(&dmirror->pt, pfn);
320}
321
322static bool dmirror_interval_invalidate(struct mmu_interval_notifier *mni,
323 const struct mmu_notifier_range *range,
324 unsigned long cur_seq)
325{
326 struct dmirror *dmirror = container_of(mni, struct dmirror, notifier);
327
328 /*
329 * Ignore invalidation callbacks for device private pages since
330 * the invalidation is handled as part of the migration process.
331 */
332 if (range->event == MMU_NOTIFY_MIGRATE &&
333 range->owner == dmirror->mdevice)
334 return true;
335
336 if (mmu_notifier_range_blockable(range))
337 mutex_lock(&dmirror->mutex);
338 else if (!mutex_trylock(&dmirror->mutex))
339 return false;
340
341 mmu_interval_set_seq(mni, cur_seq);
342 dmirror_do_update(dmirror, range->start, range->end);
343
344 mutex_unlock(&dmirror->mutex);
345 return true;
346}
347
348static const struct mmu_interval_notifier_ops dmirror_min_ops = {
349 .invalidate = dmirror_interval_invalidate,
350};
351
352static int dmirror_range_fault(struct dmirror *dmirror,
353 struct hmm_range *range)
354{
355 struct mm_struct *mm = dmirror->notifier.mm;
356 unsigned long timeout =
357 jiffies + msecs_to_jiffies(HMM_RANGE_DEFAULT_TIMEOUT);
358 int ret;
359
360 while (true) {
361 if (time_after(jiffies, timeout)) {
362 ret = -EBUSY;
363 goto out;
364 }
365
366 range->notifier_seq = mmu_interval_read_begin(range->notifier);
367 mmap_read_lock(mm);
368 ret = hmm_range_fault(range);
369 mmap_read_unlock(mm);
370 if (ret) {
371 if (ret == -EBUSY)
372 continue;
373 goto out;
374 }
375
376 mutex_lock(&dmirror->mutex);
377 if (mmu_interval_read_retry(range->notifier,
378 range->notifier_seq)) {
379 mutex_unlock(&dmirror->mutex);
380 continue;
381 }
382 break;
383 }
384
385 ret = dmirror_do_fault(dmirror, range);
386
387 mutex_unlock(&dmirror->mutex);
388out:
389 return ret;
390}
391
392static int dmirror_range_fault_unlocked(struct dmirror *dmirror,
393 struct hmm_range *range,
394 unsigned long timeout)
395{
396 int ret;
397
398 while (true) {
399 ret = hmm_range_fault_unlocked_timeout(range, timeout);
400 if (ret)
401 goto out;
402
403 mutex_lock(&dmirror->mutex);
404 if (mmu_interval_read_retry(range->notifier,
405 range->notifier_seq)) {
406 mutex_unlock(&dmirror->mutex);
407 continue;
408 }
409 break;
410 }
411
412 ret = dmirror_do_fault(dmirror, range);
413
414 mutex_unlock(&dmirror->mutex);
415out:
416 return ret;
417}
418
419static int dmirror_fault_unlocked(struct dmirror *dmirror,
420 unsigned long start,
421 unsigned long end, bool write,
422 unsigned long timeout)
423{
424 struct mm_struct *mm = dmirror->notifier.mm;
425 unsigned long addr;
426 unsigned long pfns[32];
427 struct hmm_range range = {
428 .notifier = &dmirror->notifier,
429 .hmm_pfns = pfns,
430 .pfn_flags_mask = 0,
431 .default_flags =
432 HMM_PFN_REQ_FAULT | (write ? HMM_PFN_REQ_WRITE : 0),
433 .dev_private_owner = dmirror->mdevice,
434 };
435 int ret = 0;
436
437 if (!mmget_not_zero(mm))
438 return -EFAULT;
439
440 for (addr = start; addr < end; addr = range.end) {
441 range.start = addr;
442 range.end = min(addr + (ARRAY_SIZE(pfns) << PAGE_SHIFT), end);
443
444 ret = dmirror_range_fault_unlocked(dmirror, &range, timeout);
445 if (ret)
446 break;
447 }
448
449 mmput(mm);
450 return ret;
451}
452
453static int dmirror_fault(struct dmirror *dmirror, unsigned long start,
454 unsigned long end, bool write)
455{
456 struct mm_struct *mm = dmirror->notifier.mm;
457 unsigned long addr;
458 unsigned long pfns[32];
459 struct hmm_range range = {
460 .notifier = &dmirror->notifier,
461 .hmm_pfns = pfns,
462 .pfn_flags_mask = 0,
463 .default_flags =
464 HMM_PFN_REQ_FAULT | (write ? HMM_PFN_REQ_WRITE : 0),
465 .dev_private_owner = dmirror->mdevice,
466 };
467 int ret = 0;
468
469 /* Since the mm is for the mirrored process, get a reference first. */
470 if (!mmget_not_zero(mm))
471 return -EFAULT;
472
473 for (addr = start; addr < end; addr = range.end) {
474 range.start = addr;
475 range.end = min(addr + (ARRAY_SIZE(pfns) << PAGE_SHIFT), end);
476
477 ret = dmirror_range_fault(dmirror, &range);
478 if (ret)
479 break;
480 }
481
482 mmput(mm);
483 return ret;
484}
485
486static int dmirror_do_read(struct dmirror *dmirror, unsigned long start,
487 unsigned long end, struct dmirror_bounce *bounce)
488{
489 unsigned long pfn;
490 void *ptr;
491
492 ptr = bounce->ptr + ((start - bounce->addr) & PAGE_MASK);
493
494 for (pfn = start >> PAGE_SHIFT; pfn < (end >> PAGE_SHIFT); pfn++) {
495 void *entry;
496 struct page *page;
497
498 entry = xa_load(&dmirror->pt, pfn);
499 page = xa_untag_pointer(entry);
500 if (!page)
501 return -ENOENT;
502
503 memcpy_from_page(ptr, page, 0, PAGE_SIZE);
504
505 ptr += PAGE_SIZE;
506 bounce->cpages++;
507 }
508
509 return 0;
510}
511
512static int dmirror_read(struct dmirror *dmirror, struct hmm_dmirror_cmd *cmd)
513{
514 struct dmirror_bounce bounce;
515 unsigned long start, end;
516 unsigned long size = cmd->npages << PAGE_SHIFT;
517 int ret;
518
519 start = cmd->addr;
520 end = start + size;
521 if (end < start)
522 return -EINVAL;
523
524 ret = dmirror_bounce_init(&bounce, start, size);
525 if (ret)
526 return ret;
527
528 while (1) {
529 mutex_lock(&dmirror->mutex);
530 ret = dmirror_do_read(dmirror, start, end, &bounce);
531 mutex_unlock(&dmirror->mutex);
532 if (ret != -ENOENT)
533 break;
534
535 start = cmd->addr + (bounce.cpages << PAGE_SHIFT);
536 ret = dmirror_fault(dmirror, start, end, false);
537 if (ret)
538 break;
539 cmd->faults++;
540 }
541
542 if (ret == 0) {
543 if (copy_to_user(u64_to_user_ptr(cmd->ptr), bounce.ptr,
544 bounce.size))
545 ret = -EFAULT;
546 }
547 cmd->cpages = bounce.cpages;
548 dmirror_bounce_fini(&bounce);
549 return ret;
550}
551
552static int dmirror_read_unlocked(struct dmirror *dmirror,
553 struct hmm_dmirror_cmd *cmd,
554 unsigned long timeout)
555{
556 struct dmirror_bounce bounce;
557 unsigned long start, end;
558 unsigned long size = cmd->npages << PAGE_SHIFT;
559 int ret;
560
561 start = cmd->addr;
562 end = start + size;
563 if (end < start)
564 return -EINVAL;
565
566 ret = dmirror_bounce_init(&bounce, start, size);
567 if (ret)
568 return ret;
569
570 while (1) {
571 mutex_lock(&dmirror->mutex);
572 ret = dmirror_do_read(dmirror, start, end, &bounce);
573 mutex_unlock(&dmirror->mutex);
574 if (ret != -ENOENT)
575 break;
576
577 start = cmd->addr + (bounce.cpages << PAGE_SHIFT);
578 ret = dmirror_fault_unlocked(dmirror, start, end, false, timeout);
579 if (ret)
580 break;
581 cmd->faults++;
582 }
583
584 if (ret == 0) {
585 if (copy_to_user(u64_to_user_ptr(cmd->ptr), bounce.ptr,
586 bounce.size))
587 ret = -EFAULT;
588 }
589 cmd->cpages = bounce.cpages;
590 dmirror_bounce_fini(&bounce);
591 return ret;
592}
593
594static int dmirror_do_write(struct dmirror *dmirror, unsigned long start,
595 unsigned long end, struct dmirror_bounce *bounce)
596{
597 unsigned long pfn;
598 void *ptr;
599
600 ptr = bounce->ptr + ((start - bounce->addr) & PAGE_MASK);
601
602 for (pfn = start >> PAGE_SHIFT; pfn < (end >> PAGE_SHIFT); pfn++) {
603 void *entry;
604 struct page *page;
605
606 entry = xa_load(&dmirror->pt, pfn);
607 page = xa_untag_pointer(entry);
608 if (!page || xa_pointer_tag(entry) != DPT_XA_TAG_WRITE)
609 return -ENOENT;
610
611 memcpy_to_page(page, 0, ptr, PAGE_SIZE);
612
613 ptr += PAGE_SIZE;
614 bounce->cpages++;
615 }
616
617 return 0;
618}
619
620static int dmirror_write(struct dmirror *dmirror, struct hmm_dmirror_cmd *cmd)
621{
622 struct dmirror_bounce bounce;
623 unsigned long start, end;
624 unsigned long size = cmd->npages << PAGE_SHIFT;
625 int ret;
626
627 start = cmd->addr;
628 end = start + size;
629 if (end < start)
630 return -EINVAL;
631
632 ret = dmirror_bounce_init(&bounce, start, size);
633 if (ret)
634 return ret;
635 if (copy_from_user(bounce.ptr, u64_to_user_ptr(cmd->ptr),
636 bounce.size)) {
637 ret = -EFAULT;
638 goto fini;
639 }
640
641 while (1) {
642 mutex_lock(&dmirror->mutex);
643 ret = dmirror_do_write(dmirror, start, end, &bounce);
644 mutex_unlock(&dmirror->mutex);
645 if (ret != -ENOENT)
646 break;
647
648 start = cmd->addr + (bounce.cpages << PAGE_SHIFT);
649 ret = dmirror_fault(dmirror, start, end, true);
650 if (ret)
651 break;
652 cmd->faults++;
653 }
654
655fini:
656 cmd->cpages = bounce.cpages;
657 dmirror_bounce_fini(&bounce);
658 return ret;
659}
660
661static int dmirror_allocate_chunk(struct dmirror_device *mdevice,
662 struct page **ppage, bool is_large)
663{
664 struct dmirror_chunk *devmem;
665 struct resource *res = NULL;
666 unsigned long pfn;
667 unsigned long pfn_first;
668 unsigned long pfn_last;
669 void *ptr;
670 int ret = -ENOMEM;
671
672 devmem = kzalloc_obj(*devmem);
673 if (!devmem)
674 return ret;
675
676 switch (mdevice->zone_device_type) {
677 case HMM_DMIRROR_MEMORY_DEVICE_PRIVATE:
678 res = request_free_mem_region(&iomem_resource, DEVMEM_CHUNK_SIZE,
679 "hmm_dmirror");
680 if (IS_ERR_OR_NULL(res))
681 goto err_devmem;
682 devmem->pagemap.range.start = res->start;
683 devmem->pagemap.range.end = res->end;
684 devmem->pagemap.type = MEMORY_DEVICE_PRIVATE;
685 break;
686 case HMM_DMIRROR_MEMORY_DEVICE_COHERENT:
687 devmem->pagemap.range.start = (MINOR(mdevice->device.devt) - 2) ?
688 spm_addr_dev0 :
689 spm_addr_dev1;
690 devmem->pagemap.range.end = devmem->pagemap.range.start +
691 DEVMEM_CHUNK_SIZE - 1;
692 devmem->pagemap.type = MEMORY_DEVICE_COHERENT;
693 break;
694 default:
695 ret = -EINVAL;
696 goto err_devmem;
697 }
698
699 devmem->pagemap.nr_range = 1;
700 devmem->pagemap.ops = &dmirror_devmem_ops;
701 devmem->pagemap.owner = mdevice;
702
703 mutex_lock(&mdevice->devmem_lock);
704
705 if (mdevice->devmem_count == mdevice->devmem_capacity) {
706 struct dmirror_chunk **new_chunks;
707 unsigned int new_capacity;
708
709 new_capacity = mdevice->devmem_capacity +
710 DEVMEM_CHUNKS_RESERVE;
711 new_chunks = krealloc(mdevice->devmem_chunks,
712 sizeof(new_chunks[0]) * new_capacity,
713 GFP_KERNEL);
714 if (!new_chunks)
715 goto err_release;
716 mdevice->devmem_capacity = new_capacity;
717 mdevice->devmem_chunks = new_chunks;
718 }
719 ptr = memremap_pages(&devmem->pagemap, numa_node_id());
720 if (IS_ERR_OR_NULL(ptr)) {
721 if (ptr)
722 ret = PTR_ERR(ptr);
723 else
724 ret = -EFAULT;
725 goto err_release;
726 }
727
728 devmem->mdevice = mdevice;
729 pfn_first = devmem->pagemap.range.start >> PAGE_SHIFT;
730 pfn_last = pfn_first + (range_len(&devmem->pagemap.range) >> PAGE_SHIFT);
731 mdevice->devmem_chunks[mdevice->devmem_count++] = devmem;
732
733 mutex_unlock(&mdevice->devmem_lock);
734
735 pr_info("added new %u MB chunk (total %u chunks, %u MB) PFNs [0x%lx 0x%lx)\n",
736 DEVMEM_CHUNK_SIZE / (1024 * 1024),
737 mdevice->devmem_count,
738 mdevice->devmem_count * (DEVMEM_CHUNK_SIZE / (1024 * 1024)),
739 pfn_first, pfn_last);
740
741 spin_lock(&mdevice->lock);
742 for (pfn = pfn_first; pfn < pfn_last; ) {
743 struct page *page = pfn_to_page(pfn);
744
745 if (is_large && IS_ALIGNED(pfn, HPAGE_PMD_NR)
746 && (pfn + HPAGE_PMD_NR <= pfn_last)) {
747 page->zone_device_data = mdevice->free_folios;
748 mdevice->free_folios = page_folio(page);
749 pfn += HPAGE_PMD_NR;
750 continue;
751 }
752
753 page->zone_device_data = mdevice->free_pages;
754 mdevice->free_pages = page;
755 pfn++;
756 }
757
758 ret = 0;
759 if (ppage) {
760 if (is_large) {
761 if (!mdevice->free_folios) {
762 ret = -ENOMEM;
763 goto err_unlock;
764 }
765 *ppage = folio_page(mdevice->free_folios, 0);
766 mdevice->free_folios = (*ppage)->zone_device_data;
767 mdevice->calloc += HPAGE_PMD_NR;
768 } else if (mdevice->free_pages) {
769 *ppage = mdevice->free_pages;
770 mdevice->free_pages = (*ppage)->zone_device_data;
771 mdevice->calloc++;
772 } else {
773 ret = -ENOMEM;
774 goto err_unlock;
775 }
776 }
777err_unlock:
778 spin_unlock(&mdevice->lock);
779
780 return ret;
781
782err_release:
783 mutex_unlock(&mdevice->devmem_lock);
784 if (res && devmem->pagemap.type == MEMORY_DEVICE_PRIVATE)
785 release_mem_region(devmem->pagemap.range.start,
786 range_len(&devmem->pagemap.range));
787err_devmem:
788 kfree(devmem);
789
790 return ret;
791}
792
793static struct page *dmirror_devmem_alloc_page(struct dmirror *dmirror,
794 bool is_large)
795{
796 struct page *dpage = NULL;
797 struct page *rpage = NULL;
798 unsigned int order = is_large ? HPAGE_PMD_ORDER : 0;
799 struct dmirror_device *mdevice = dmirror->mdevice;
800
801 /*
802 * For ZONE_DEVICE private type, this is a fake device so we allocate
803 * real system memory to store our device memory.
804 * For ZONE_DEVICE coherent type we use the actual dpage to store the
805 * data and ignore rpage.
806 */
807 if (dmirror_is_private_zone(mdevice)) {
808 rpage = folio_page(folio_alloc(GFP_HIGHUSER, order), 0);
809 if (!rpage)
810 return NULL;
811 }
812 spin_lock(&mdevice->lock);
813
814 if (is_large && mdevice->free_folios) {
815 dpage = folio_page(mdevice->free_folios, 0);
816 mdevice->free_folios = dpage->zone_device_data;
817 mdevice->calloc += 1 << order;
818 spin_unlock(&mdevice->lock);
819 } else if (!is_large && mdevice->free_pages) {
820 dpage = mdevice->free_pages;
821 mdevice->free_pages = dpage->zone_device_data;
822 mdevice->calloc++;
823 spin_unlock(&mdevice->lock);
824 } else {
825 spin_unlock(&mdevice->lock);
826 if (dmirror_allocate_chunk(mdevice, &dpage, is_large))
827 goto error;
828 }
829
830 zone_device_folio_init(page_folio(dpage),
831 page_pgmap(folio_page(page_folio(dpage), 0)),
832 order);
833 dpage->zone_device_data = rpage;
834 return dpage;
835
836error:
837 if (rpage)
838 __free_pages(rpage, order);
839 return NULL;
840}
841
842static void dmirror_migrate_alloc_and_copy(struct migrate_vma *args,
843 struct dmirror *dmirror)
844{
845 const unsigned long *src = args->src;
846 unsigned long *dst = args->dst;
847 unsigned long addr;
848
849 for (addr = args->start; addr < args->end; ) {
850 struct page *spage;
851 struct page *dpage;
852 struct page *rpage;
853 bool is_large = *src & MIGRATE_PFN_COMPOUND;
854 int write = (*src & MIGRATE_PFN_WRITE) ? MIGRATE_PFN_WRITE : 0;
855 unsigned long nr = 1;
856
857 if (!(*src & MIGRATE_PFN_MIGRATE))
858 goto next;
859
860 /*
861 * Note that spage might be NULL which is OK since it is an
862 * unallocated pte_none() or read-only zero page.
863 */
864 spage = migrate_pfn_to_page(*src);
865 if (WARN(spage && is_zone_device_page(spage),
866 "page already in device spage pfn: 0x%lx\n",
867 page_to_pfn(spage)))
868 goto next;
869
870 if (dmirror->flags & HMM_DMIRROR_FLAG_FAIL_ALLOC) {
871 dmirror->flags &= ~HMM_DMIRROR_FLAG_FAIL_ALLOC;
872 dpage = NULL;
873 } else
874 dpage = dmirror_devmem_alloc_page(dmirror, is_large);
875
876 if (!dpage) {
877 struct folio *folio;
878 unsigned long i;
879 unsigned long spfn = *src >> MIGRATE_PFN_SHIFT;
880 struct page *src_page;
881
882 if (!is_large)
883 goto next;
884
885 if (!spage && is_large) {
886 nr = HPAGE_PMD_NR;
887 } else {
888 folio = page_folio(spage);
889 nr = folio_nr_pages(folio);
890 }
891
892 for (i = 0; i < nr && addr < args->end; i++) {
893 dpage = dmirror_devmem_alloc_page(dmirror, false);
894 rpage = BACKING_PAGE(dpage);
895 rpage->zone_device_data = dmirror;
896
897 *dst = migrate_pfn(page_to_pfn(dpage)) | write;
898 src_page = pfn_to_page(spfn + i);
899
900 if (spage)
901 copy_highpage(rpage, src_page);
902 else
903 clear_highpage(rpage);
904 src++;
905 dst++;
906 addr += PAGE_SIZE;
907 }
908 continue;
909 }
910
911 rpage = BACKING_PAGE(dpage);
912
913 /*
914 * Normally, a device would use the page->zone_device_data to
915 * point to the mirror but here we use it to hold the page for
916 * the simulated device memory and that page holds the pointer
917 * to the mirror.
918 */
919 rpage->zone_device_data = dmirror;
920
921 pr_debug("migrating from sys to dev pfn src: 0x%lx pfn dst: 0x%lx\n",
922 page_to_pfn(spage), page_to_pfn(dpage));
923
924 *dst = migrate_pfn(page_to_pfn(dpage)) | write;
925
926 if (is_large) {
927 int i;
928 struct folio *folio = page_folio(dpage);
929 *dst |= MIGRATE_PFN_COMPOUND;
930
931 if (folio_test_large(folio)) {
932 for (i = 0; i < folio_nr_pages(folio); i++) {
933 struct page *dst_page =
934 pfn_to_page(page_to_pfn(rpage) + i);
935 struct page *src_page =
936 pfn_to_page(page_to_pfn(spage) + i);
937
938 if (spage)
939 copy_highpage(dst_page, src_page);
940 else
941 clear_highpage(dst_page);
942 src++;
943 dst++;
944 addr += PAGE_SIZE;
945 }
946 continue;
947 }
948 }
949
950 if (spage)
951 copy_highpage(rpage, spage);
952 else
953 clear_highpage(rpage);
954
955next:
956 src++;
957 dst++;
958 addr += PAGE_SIZE;
959 }
960}
961
962static int dmirror_check_atomic(struct dmirror *dmirror, unsigned long start,
963 unsigned long end)
964{
965 unsigned long pfn;
966
967 for (pfn = start >> PAGE_SHIFT; pfn < (end >> PAGE_SHIFT); pfn++) {
968 void *entry;
969
970 entry = xa_load(&dmirror->pt, pfn);
971 if (xa_pointer_tag(entry) == DPT_XA_TAG_ATOMIC)
972 return -EPERM;
973 }
974
975 return 0;
976}
977
978static int dmirror_atomic_map(unsigned long addr, struct page *page,
979 struct dmirror *dmirror)
980{
981 void *entry;
982
983 /* Map the migrated pages into the device's page tables. */
984 mutex_lock(&dmirror->mutex);
985
986 entry = xa_tag_pointer(page, DPT_XA_TAG_ATOMIC);
987 entry = xa_store(&dmirror->pt, addr >> PAGE_SHIFT, entry, GFP_ATOMIC);
988 if (xa_is_err(entry)) {
989 mutex_unlock(&dmirror->mutex);
990 return xa_err(entry);
991 }
992
993 mutex_unlock(&dmirror->mutex);
994 return 0;
995}
996
997static int dmirror_migrate_finalize_and_map(struct migrate_vma *args,
998 struct dmirror *dmirror)
999{
1000 unsigned long start = args->start;
1001 unsigned long end = args->end;
1002 const unsigned long *src = args->src;
1003 const unsigned long *dst = args->dst;
1004 unsigned long pfn;
1005 const unsigned long start_pfn = start >> PAGE_SHIFT;
1006 const unsigned long end_pfn = end >> PAGE_SHIFT;
1007
1008 /* Map the migrated pages into the device's page tables. */
1009 mutex_lock(&dmirror->mutex);
1010
1011 for (pfn = start_pfn; pfn < end_pfn; pfn++, src++, dst++) {
1012 struct page *dpage;
1013 void *entry;
1014 int nr, i;
1015 struct page *rpage;
1016
1017 if (!(*src & MIGRATE_PFN_MIGRATE))
1018 continue;
1019
1020 dpage = migrate_pfn_to_page(*dst);
1021 if (!dpage)
1022 continue;
1023
1024 if (*dst & MIGRATE_PFN_COMPOUND)
1025 nr = folio_nr_pages(page_folio(dpage));
1026 else
1027 nr = 1;
1028
1029 WARN_ON_ONCE(end_pfn < start_pfn + nr);
1030
1031 rpage = BACKING_PAGE(dpage);
1032 VM_WARN_ON(folio_nr_pages(page_folio(rpage)) != nr);
1033
1034 for (i = 0; i < nr; i++) {
1035 entry = folio_page(page_folio(rpage), i);
1036 if (*dst & MIGRATE_PFN_WRITE)
1037 entry = xa_tag_pointer(entry, DPT_XA_TAG_WRITE);
1038 entry = xa_store(&dmirror->pt, pfn + i, entry, GFP_ATOMIC);
1039 if (xa_is_err(entry)) {
1040 mutex_unlock(&dmirror->mutex);
1041 return xa_err(entry);
1042 }
1043 }
1044 }
1045
1046 mutex_unlock(&dmirror->mutex);
1047 return 0;
1048}
1049
1050static int dmirror_exclusive(struct dmirror *dmirror,
1051 struct hmm_dmirror_cmd *cmd)
1052{
1053 unsigned long start, end, addr;
1054 unsigned long size = cmd->npages << PAGE_SHIFT;
1055 struct mm_struct *mm = dmirror->notifier.mm;
1056 struct dmirror_bounce bounce;
1057 int ret = 0;
1058
1059 start = cmd->addr;
1060 end = start + size;
1061 if (end < start)
1062 return -EINVAL;
1063
1064 /* Since the mm is for the mirrored process, get a reference first. */
1065 if (!mmget_not_zero(mm))
1066 return -EINVAL;
1067
1068 mmap_read_lock(mm);
1069 for (addr = start; !ret && addr < end; addr += PAGE_SIZE) {
1070 struct folio *folio;
1071 struct page *page;
1072
1073 page = make_device_exclusive(mm, addr, NULL, &folio);
1074 if (IS_ERR(page)) {
1075 ret = PTR_ERR(page);
1076 break;
1077 }
1078
1079 ret = dmirror_atomic_map(addr, page, dmirror);
1080 folio_unlock(folio);
1081 folio_put(folio);
1082 }
1083 mmap_read_unlock(mm);
1084 mmput(mm);
1085
1086 if (ret)
1087 return ret;
1088
1089 /* Return the migrated data for verification. */
1090 ret = dmirror_bounce_init(&bounce, start, size);
1091 if (ret)
1092 return ret;
1093 mutex_lock(&dmirror->mutex);
1094 ret = dmirror_do_read(dmirror, start, end, &bounce);
1095 mutex_unlock(&dmirror->mutex);
1096 if (ret == 0) {
1097 if (copy_to_user(u64_to_user_ptr(cmd->ptr), bounce.ptr,
1098 bounce.size))
1099 ret = -EFAULT;
1100 }
1101
1102 cmd->cpages = bounce.cpages;
1103 dmirror_bounce_fini(&bounce);
1104 return ret;
1105}
1106
1107static vm_fault_t dmirror_devmem_fault_alloc_and_copy(struct migrate_vma *args,
1108 struct dmirror *dmirror)
1109{
1110 const unsigned long *src = args->src;
1111 unsigned long *dst = args->dst;
1112 unsigned long start = args->start;
1113 unsigned long end = args->end;
1114 unsigned long addr;
1115 unsigned int order = 0;
1116 int i;
1117
1118 for (addr = start; addr < end; ) {
1119 struct page *dpage, *spage;
1120
1121 spage = migrate_pfn_to_page(*src);
1122 if (!spage || !(*src & MIGRATE_PFN_MIGRATE)) {
1123 addr += PAGE_SIZE;
1124 goto next;
1125 }
1126
1127 if (WARN_ON(!is_device_private_page(spage) &&
1128 !is_device_coherent_page(spage))) {
1129 addr += PAGE_SIZE;
1130 goto next;
1131 }
1132
1133 spage = BACKING_PAGE(spage);
1134 order = folio_order(page_folio(spage));
1135 if (order)
1136 *dst = MIGRATE_PFN_COMPOUND;
1137 if (*src & MIGRATE_PFN_WRITE)
1138 *dst |= MIGRATE_PFN_WRITE;
1139
1140 if (dmirror->flags & HMM_DMIRROR_FLAG_FAIL_ALLOC) {
1141 dmirror->flags &= ~HMM_DMIRROR_FLAG_FAIL_ALLOC;
1142 *dst &= ~MIGRATE_PFN_COMPOUND;
1143 dpage = NULL;
1144 } else if (order) {
1145 dpage = folio_page(vma_alloc_folio(GFP_HIGHUSER_MOVABLE,
1146 order, args->vma, addr), 0);
1147 } else {
1148 dpage = alloc_page_vma(GFP_HIGHUSER_MOVABLE, args->vma, addr);
1149 }
1150
1151 if (!dpage && !order)
1152 return VM_FAULT_OOM;
1153
1154 if (dpage) {
1155 pr_debug("migrating from dev to sys pfn src: 0x%lx pfn dst: 0x%lx\n",
1156 page_to_pfn(spage), page_to_pfn(dpage));
1157 lock_page(dpage);
1158 *dst |= migrate_pfn(page_to_pfn(dpage));
1159 }
1160
1161 for (i = 0; i < (1 << order); i++) {
1162 struct page *src_page;
1163 struct page *dst_page;
1164
1165 /* Try with smaller pages if large allocation fails */
1166 if (!dpage && order) {
1167 dpage = alloc_page_vma(GFP_HIGHUSER_MOVABLE, args->vma, addr);
1168 if (!dpage) {
1169 /* Unlock and free pages already allocated. */
1170 while (i > 0) {
1171 struct page *fpage;
1172
1173 fpage = migrate_pfn_to_page(dst[--i]);
1174 unlock_page(fpage);
1175 __free_page(fpage);
1176 }
1177 /* Clear remaining dst entries to avoid
1178 * migrate_vma_pages/finalize() using
1179 * uninitialized values.
1180 */
1181 while (i < (1 << order)) {
1182 dst[i] = 0;
1183 i++;
1184 }
1185 return VM_FAULT_OOM;
1186 }
1187 lock_page(dpage);
1188 dst[i] = migrate_pfn(page_to_pfn(dpage));
1189 dst_page = pfn_to_page(page_to_pfn(dpage));
1190 dpage = NULL; /* For the next iteration */
1191 } else {
1192 dst_page = pfn_to_page(page_to_pfn(dpage) + i);
1193 }
1194
1195 src_page = pfn_to_page(page_to_pfn(spage) + i);
1196
1197 xa_erase(&dmirror->pt, addr >> PAGE_SHIFT);
1198 addr += PAGE_SIZE;
1199 copy_highpage(dst_page, src_page);
1200 }
1201next:
1202 src += 1 << order;
1203 dst += 1 << order;
1204 }
1205 return 0;
1206}
1207
1208static unsigned long
1209dmirror_successful_migrated_pages(struct migrate_vma *migrate)
1210{
1211 unsigned long cpages = 0;
1212 unsigned long i;
1213
1214 for (i = 0; i < migrate->npages; i++) {
1215 if (migrate->src[i] & MIGRATE_PFN_VALID &&
1216 migrate->src[i] & MIGRATE_PFN_MIGRATE)
1217 cpages++;
1218 }
1219 return cpages;
1220}
1221
1222static int dmirror_migrate_to_system(struct dmirror *dmirror,
1223 struct hmm_dmirror_cmd *cmd)
1224{
1225 unsigned long start, end, addr;
1226 unsigned long size = cmd->npages << PAGE_SHIFT;
1227 struct mm_struct *mm = dmirror->notifier.mm;
1228 struct vm_area_struct *vma;
1229 struct migrate_vma args = { 0 };
1230 unsigned long next;
1231 int ret;
1232 unsigned long *src_pfns;
1233 unsigned long *dst_pfns;
1234
1235 start = cmd->addr;
1236 end = start + size;
1237 if (end < start)
1238 return -EINVAL;
1239
1240 /* Since the mm is for the mirrored process, get a reference first. */
1241 if (!mmget_not_zero(mm))
1242 return -EINVAL;
1243
1244 src_pfns = kvcalloc(PTRS_PER_PTE, sizeof(*src_pfns), GFP_KERNEL | __GFP_NOFAIL);
1245 dst_pfns = kvcalloc(PTRS_PER_PTE, sizeof(*dst_pfns), GFP_KERNEL | __GFP_NOFAIL);
1246
1247 cmd->cpages = 0;
1248 mmap_read_lock(mm);
1249 for (addr = start; addr < end; addr = next) {
1250 vma = vma_lookup(mm, addr);
1251 if (!vma || !(vma->vm_flags & VM_READ)) {
1252 ret = -EINVAL;
1253 goto out;
1254 }
1255 next = min(end, addr + (PTRS_PER_PTE << PAGE_SHIFT));
1256 if (next > vma->vm_end)
1257 next = vma->vm_end;
1258
1259 args.vma = vma;
1260 args.src = src_pfns;
1261 args.dst = dst_pfns;
1262 args.start = addr;
1263 args.end = next;
1264 args.pgmap_owner = dmirror->mdevice;
1265 args.flags = dmirror_select_device(dmirror) | MIGRATE_VMA_SELECT_COMPOUND;
1266
1267 ret = migrate_vma_setup(&args);
1268 if (ret)
1269 goto out;
1270
1271 pr_debug("Migrating from device mem to sys mem\n");
1272 if (dmirror_devmem_fault_alloc_and_copy(&args, dmirror)) {
1273 migrate_vma_finalize(&args);
1274 ret = -ENOMEM;
1275 goto out;
1276 }
1277
1278 migrate_vma_pages(&args);
1279 cmd->cpages += dmirror_successful_migrated_pages(&args);
1280 migrate_vma_finalize(&args);
1281 }
1282out:
1283 mmap_read_unlock(mm);
1284 mmput(mm);
1285 kvfree(src_pfns);
1286 kvfree(dst_pfns);
1287
1288 return ret;
1289}
1290
1291static int dmirror_migrate_to_device(struct dmirror *dmirror,
1292 struct hmm_dmirror_cmd *cmd)
1293{
1294 unsigned long start, end, addr;
1295 unsigned long size = cmd->npages << PAGE_SHIFT;
1296 struct mm_struct *mm = dmirror->notifier.mm;
1297 struct vm_area_struct *vma;
1298 struct dmirror_bounce bounce;
1299 struct migrate_vma args = { 0 };
1300 unsigned long next;
1301 int ret;
1302 unsigned long *src_pfns = NULL;
1303 unsigned long *dst_pfns = NULL;
1304
1305 start = cmd->addr;
1306 end = start + size;
1307 if (end < start)
1308 return -EINVAL;
1309
1310 /* Since the mm is for the mirrored process, get a reference first. */
1311 if (!mmget_not_zero(mm))
1312 return -EINVAL;
1313
1314 src_pfns = kvcalloc(PTRS_PER_PTE, sizeof(*src_pfns),
1315 GFP_KERNEL | __GFP_NOFAIL);
1316 dst_pfns = kvcalloc(PTRS_PER_PTE, sizeof(*dst_pfns),
1317 GFP_KERNEL | __GFP_NOFAIL);
1318
1319 ret = 0;
1320 mmap_read_lock(mm);
1321 for (addr = start; addr < end; addr = next) {
1322 vma = vma_lookup(mm, addr);
1323 if (!vma || !(vma->vm_flags & VM_READ)) {
1324 ret = -EINVAL;
1325 goto out;
1326 }
1327 next = min(end, addr + (PTRS_PER_PTE << PAGE_SHIFT));
1328 if (next > vma->vm_end)
1329 next = vma->vm_end;
1330
1331 args.vma = vma;
1332 args.src = src_pfns;
1333 args.dst = dst_pfns;
1334 args.start = addr;
1335 args.end = next;
1336 args.pgmap_owner = dmirror->mdevice;
1337 args.flags = MIGRATE_VMA_SELECT_SYSTEM |
1338 MIGRATE_VMA_SELECT_COMPOUND;
1339 ret = migrate_vma_setup(&args);
1340 if (ret)
1341 goto out;
1342
1343 pr_debug("Migrating from sys mem to device mem\n");
1344 dmirror_migrate_alloc_and_copy(&args, dmirror);
1345 migrate_vma_pages(&args);
1346 dmirror_migrate_finalize_and_map(&args, dmirror);
1347 migrate_vma_finalize(&args);
1348 }
1349 mmap_read_unlock(mm);
1350 mmput(mm);
1351
1352 /*
1353 * Return the migrated data for verification.
1354 * Only for pages in device zone
1355 */
1356 ret = dmirror_bounce_init(&bounce, start, size);
1357 if (ret)
1358 goto free_mem;
1359 mutex_lock(&dmirror->mutex);
1360 ret = dmirror_do_read(dmirror, start, end, &bounce);
1361 mutex_unlock(&dmirror->mutex);
1362 if (ret == 0) {
1363 if (copy_to_user(u64_to_user_ptr(cmd->ptr), bounce.ptr,
1364 bounce.size))
1365 ret = -EFAULT;
1366 }
1367 cmd->cpages = bounce.cpages;
1368 dmirror_bounce_fini(&bounce);
1369 goto free_mem;
1370
1371out:
1372 mmap_read_unlock(mm);
1373 mmput(mm);
1374free_mem:
1375 kvfree(src_pfns);
1376 kvfree(dst_pfns);
1377 return ret;
1378}
1379
1380static void dmirror_mkentry(struct dmirror *dmirror, struct hmm_range *range,
1381 unsigned char *perm, unsigned long entry)
1382{
1383 struct page *page;
1384
1385 if (entry & HMM_PFN_ERROR) {
1386 *perm = HMM_DMIRROR_PROT_ERROR;
1387 return;
1388 }
1389 if (!(entry & HMM_PFN_VALID)) {
1390 *perm = HMM_DMIRROR_PROT_NONE;
1391 return;
1392 }
1393
1394 page = hmm_pfn_to_page(entry);
1395 if (is_device_private_page(page)) {
1396 /* Is the page migrated to this device or some other? */
1397 if (dmirror->mdevice == dmirror_page_to_device(page))
1398 *perm = HMM_DMIRROR_PROT_DEV_PRIVATE_LOCAL;
1399 else
1400 *perm = HMM_DMIRROR_PROT_DEV_PRIVATE_REMOTE;
1401 } else if (is_device_coherent_page(page)) {
1402 /* Is the page migrated to this device or some other? */
1403 if (dmirror->mdevice == dmirror_page_to_device(page))
1404 *perm = HMM_DMIRROR_PROT_DEV_COHERENT_LOCAL;
1405 else
1406 *perm = HMM_DMIRROR_PROT_DEV_COHERENT_REMOTE;
1407 } else if (is_zero_pfn(page_to_pfn(page)))
1408 *perm = HMM_DMIRROR_PROT_ZERO;
1409 else
1410 *perm = HMM_DMIRROR_PROT_NONE;
1411 if (entry & HMM_PFN_WRITE)
1412 *perm |= HMM_DMIRROR_PROT_WRITE;
1413 else
1414 *perm |= HMM_DMIRROR_PROT_READ;
1415 if (hmm_pfn_to_map_order(entry) + PAGE_SHIFT == PMD_SHIFT)
1416 *perm |= HMM_DMIRROR_PROT_PMD;
1417 else if (hmm_pfn_to_map_order(entry) + PAGE_SHIFT == PUD_SHIFT)
1418 *perm |= HMM_DMIRROR_PROT_PUD;
1419}
1420
1421static bool dmirror_snapshot_invalidate(struct mmu_interval_notifier *mni,
1422 const struct mmu_notifier_range *range,
1423 unsigned long cur_seq)
1424{
1425 struct dmirror_interval *dmi =
1426 container_of(mni, struct dmirror_interval, notifier);
1427 struct dmirror *dmirror = dmi->dmirror;
1428
1429 if (mmu_notifier_range_blockable(range))
1430 mutex_lock(&dmirror->mutex);
1431 else if (!mutex_trylock(&dmirror->mutex))
1432 return false;
1433
1434 /*
1435 * Snapshots only need to set the sequence number since any
1436 * invalidation in the interval invalidates the whole snapshot.
1437 */
1438 mmu_interval_set_seq(mni, cur_seq);
1439
1440 mutex_unlock(&dmirror->mutex);
1441 return true;
1442}
1443
1444static const struct mmu_interval_notifier_ops dmirror_mrn_ops = {
1445 .invalidate = dmirror_snapshot_invalidate,
1446};
1447
1448static int dmirror_range_snapshot(struct dmirror *dmirror,
1449 struct hmm_range *range,
1450 unsigned char *perm)
1451{
1452 struct mm_struct *mm = dmirror->notifier.mm;
1453 struct dmirror_interval notifier;
1454 unsigned long timeout =
1455 jiffies + msecs_to_jiffies(HMM_RANGE_DEFAULT_TIMEOUT);
1456 unsigned long i;
1457 unsigned long n;
1458 int ret = 0;
1459
1460 notifier.dmirror = dmirror;
1461 range->notifier = &notifier.notifier;
1462
1463 ret = mmu_interval_notifier_insert(range->notifier, mm,
1464 range->start, range->end - range->start,
1465 &dmirror_mrn_ops);
1466 if (ret)
1467 return ret;
1468
1469 while (true) {
1470 if (time_after(jiffies, timeout)) {
1471 ret = -EBUSY;
1472 goto out;
1473 }
1474
1475 range->notifier_seq = mmu_interval_read_begin(range->notifier);
1476
1477 mmap_read_lock(mm);
1478 ret = hmm_range_fault(range);
1479 mmap_read_unlock(mm);
1480 if (ret) {
1481 if (ret == -EBUSY)
1482 continue;
1483 goto out;
1484 }
1485
1486 mutex_lock(&dmirror->mutex);
1487 if (mmu_interval_read_retry(range->notifier,
1488 range->notifier_seq)) {
1489 mutex_unlock(&dmirror->mutex);
1490 continue;
1491 }
1492 break;
1493 }
1494
1495 n = (range->end - range->start) >> PAGE_SHIFT;
1496 for (i = 0; i < n; i++)
1497 dmirror_mkentry(dmirror, range, perm + i, range->hmm_pfns[i]);
1498
1499 mutex_unlock(&dmirror->mutex);
1500out:
1501 mmu_interval_notifier_remove(range->notifier);
1502 return ret;
1503}
1504
1505static int dmirror_snapshot(struct dmirror *dmirror,
1506 struct hmm_dmirror_cmd *cmd)
1507{
1508 struct mm_struct *mm = dmirror->notifier.mm;
1509 unsigned long start, end;
1510 unsigned long size = cmd->npages << PAGE_SHIFT;
1511 unsigned long addr;
1512 unsigned long next;
1513 unsigned long pfns[32];
1514 unsigned char perm[32];
1515 char __user *uptr;
1516 struct hmm_range range = {
1517 .hmm_pfns = pfns,
1518 .dev_private_owner = dmirror->mdevice,
1519 };
1520 int ret = 0;
1521
1522 start = cmd->addr;
1523 end = start + size;
1524 if (end < start)
1525 return -EINVAL;
1526
1527 /* Since the mm is for the mirrored process, get a reference first. */
1528 if (!mmget_not_zero(mm))
1529 return -EINVAL;
1530
1531 /*
1532 * Register a temporary notifier to detect invalidations even if it
1533 * overlaps with other mmu_interval_notifiers.
1534 */
1535 uptr = u64_to_user_ptr(cmd->ptr);
1536 for (addr = start; addr < end; addr = next) {
1537 unsigned long n;
1538
1539 next = min(addr + (ARRAY_SIZE(pfns) << PAGE_SHIFT), end);
1540 range.start = addr;
1541 range.end = next;
1542
1543 ret = dmirror_range_snapshot(dmirror, &range, perm);
1544 if (ret)
1545 break;
1546
1547 n = (range.end - range.start) >> PAGE_SHIFT;
1548 if (copy_to_user(uptr, perm, n)) {
1549 ret = -EFAULT;
1550 break;
1551 }
1552
1553 cmd->cpages += n;
1554 uptr += n;
1555 }
1556 mmput(mm);
1557
1558 return ret;
1559}
1560
1561/* Removes free pages from the free list so they can't be re-allocated */
1562static void dmirror_remove_free_pages(struct dmirror_chunk *devmem)
1563{
1564 struct dmirror_device *mdevice = devmem->mdevice;
1565 struct page *page;
1566 struct folio *folio;
1567
1568
1569 for (folio = mdevice->free_folios; folio; folio = folio_zone_device_data(folio))
1570 if (dmirror_page_to_chunk(folio_page(folio, 0)) == devmem)
1571 mdevice->free_folios = folio_zone_device_data(folio);
1572 for (page = mdevice->free_pages; page; page = page->zone_device_data)
1573 if (dmirror_page_to_chunk(page) == devmem)
1574 mdevice->free_pages = page->zone_device_data;
1575}
1576
1577static void dmirror_device_remove_chunks(struct dmirror_device *mdevice)
1578{
1579 unsigned int i;
1580
1581 mutex_lock(&mdevice->devmem_lock);
1582 if (mdevice->devmem_chunks) {
1583 for (i = 0; i < mdevice->devmem_count; i++) {
1584 struct dmirror_chunk *devmem =
1585 mdevice->devmem_chunks[i];
1586
1587 spin_lock(&mdevice->lock);
1588 devmem->remove = true;
1589 dmirror_remove_free_pages(devmem);
1590 spin_unlock(&mdevice->lock);
1591
1592 dmirror_device_evict_chunk(devmem);
1593 memunmap_pages(&devmem->pagemap);
1594 if (devmem->pagemap.type == MEMORY_DEVICE_PRIVATE)
1595 release_mem_region(devmem->pagemap.range.start,
1596 range_len(&devmem->pagemap.range));
1597 kfree(devmem);
1598 }
1599 mdevice->devmem_count = 0;
1600 mdevice->devmem_capacity = 0;
1601 mdevice->free_pages = NULL;
1602 mdevice->free_folios = NULL;
1603 kfree(mdevice->devmem_chunks);
1604 mdevice->devmem_chunks = NULL;
1605 }
1606 mutex_unlock(&mdevice->devmem_lock);
1607}
1608
1609static long dmirror_fops_unlocked_ioctl(struct file *filp,
1610 unsigned int command,
1611 unsigned long arg)
1612{
1613 void __user *uarg = (void __user *)arg;
1614 struct hmm_dmirror_cmd cmd;
1615 struct dmirror *dmirror;
1616 int ret;
1617
1618 dmirror = filp->private_data;
1619 if (!dmirror)
1620 return -EINVAL;
1621
1622 if (copy_from_user(&cmd, uarg, sizeof(cmd)))
1623 return -EFAULT;
1624
1625 if (cmd.addr & ~PAGE_MASK)
1626 return -EINVAL;
1627 if (cmd.addr >= (cmd.addr + (cmd.npages << PAGE_SHIFT)))
1628 return -EINVAL;
1629
1630 cmd.cpages = 0;
1631 cmd.faults = 0;
1632
1633 switch (command) {
1634 case HMM_DMIRROR_READ:
1635 ret = dmirror_read(dmirror, &cmd);
1636 break;
1637
1638 case HMM_DMIRROR_WRITE:
1639 ret = dmirror_write(dmirror, &cmd);
1640 break;
1641
1642 case HMM_DMIRROR_MIGRATE_TO_DEV:
1643 ret = dmirror_migrate_to_device(dmirror, &cmd);
1644 break;
1645
1646 case HMM_DMIRROR_MIGRATE_TO_SYS:
1647 ret = dmirror_migrate_to_system(dmirror, &cmd);
1648 break;
1649
1650 case HMM_DMIRROR_EXCLUSIVE:
1651 ret = dmirror_exclusive(dmirror, &cmd);
1652 break;
1653
1654 case HMM_DMIRROR_CHECK_EXCLUSIVE:
1655 ret = dmirror_check_atomic(dmirror, cmd.addr,
1656 cmd.addr + (cmd.npages << PAGE_SHIFT));
1657 break;
1658
1659 case HMM_DMIRROR_SNAPSHOT:
1660 ret = dmirror_snapshot(dmirror, &cmd);
1661 break;
1662
1663 case HMM_DMIRROR_RELEASE:
1664 dmirror_device_remove_chunks(dmirror->mdevice);
1665 ret = 0;
1666 break;
1667 case HMM_DMIRROR_FLAGS:
1668 dmirror->flags = cmd.npages;
1669 ret = 0;
1670 break;
1671 case HMM_DMIRROR_READ_UNLOCKED:
1672 ret = dmirror_read_unlocked(dmirror, &cmd, 0);
1673 break;
1674 default:
1675 return -EINVAL;
1676 }
1677 if (ret)
1678 return ret;
1679
1680 if (copy_to_user(uarg, &cmd, sizeof(cmd)))
1681 return -EFAULT;
1682
1683 return 0;
1684}
1685
1686static int dmirror_fops_mmap(struct file *file, struct vm_area_struct *vma)
1687{
1688 unsigned long addr;
1689
1690 for (addr = vma->vm_start; addr < vma->vm_end; addr += PAGE_SIZE) {
1691 struct page *page;
1692 int ret;
1693
1694 page = alloc_page(GFP_KERNEL | __GFP_ZERO);
1695 if (!page)
1696 return -ENOMEM;
1697
1698 ret = vm_insert_page(vma, addr, page);
1699 if (ret) {
1700 __free_page(page);
1701 return ret;
1702 }
1703 put_page(page);
1704 }
1705
1706 return 0;
1707}
1708
1709static const struct file_operations dmirror_fops = {
1710 .open = dmirror_fops_open,
1711 .release = dmirror_fops_release,
1712 .mmap = dmirror_fops_mmap,
1713 .unlocked_ioctl = dmirror_fops_unlocked_ioctl,
1714 .llseek = default_llseek,
1715 .owner = THIS_MODULE,
1716};
1717
1718static void dmirror_devmem_free(struct folio *folio)
1719{
1720 struct page *page = &folio->page;
1721 struct page *rpage = BACKING_PAGE(page);
1722 struct dmirror_device *mdevice;
1723 struct folio *rfolio = page_folio(rpage);
1724 unsigned int order = folio_order(rfolio);
1725
1726 if (rpage != page) {
1727 if (order)
1728 __free_pages(rpage, order);
1729 else
1730 __free_page(rpage);
1731 rpage = NULL;
1732 }
1733
1734 mdevice = dmirror_page_to_device(page);
1735 spin_lock(&mdevice->lock);
1736
1737 /* Return page to our allocator if not freeing the chunk */
1738 if (!dmirror_page_to_chunk(page)->remove) {
1739 mdevice->cfree += 1 << order;
1740 if (order) {
1741 page->zone_device_data = mdevice->free_folios;
1742 mdevice->free_folios = page_folio(page);
1743 } else {
1744 page->zone_device_data = mdevice->free_pages;
1745 mdevice->free_pages = page;
1746 }
1747 }
1748 spin_unlock(&mdevice->lock);
1749}
1750
1751static vm_fault_t dmirror_devmem_fault(struct vm_fault *vmf)
1752{
1753 struct migrate_vma args = { 0 };
1754 struct page *rpage;
1755 struct dmirror *dmirror;
1756 vm_fault_t ret = 0;
1757 unsigned int order, nr;
1758
1759 /*
1760 * Normally, a device would use the page->zone_device_data to point to
1761 * the mirror but here we use it to hold the page for the simulated
1762 * device memory and that page holds the pointer to the mirror.
1763 */
1764 rpage = folio_zone_device_data(page_folio(vmf->page));
1765 dmirror = rpage->zone_device_data;
1766
1767 /* FIXME demonstrate how we can adjust migrate range */
1768 order = folio_order(page_folio(vmf->page));
1769 nr = 1 << order;
1770
1771 /*
1772 * When folios are partially mapped, we can't rely on the folio
1773 * order of vmf->page as the folio might not be fully split yet
1774 */
1775 if (vmf->pte) {
1776 order = 0;
1777 nr = 1;
1778 }
1779
1780 /*
1781 * Consider a per-cpu cache of src and dst pfns, but with
1782 * large number of cpus that might not scale well.
1783 */
1784 args.start = ALIGN_DOWN(vmf->address, (PAGE_SIZE << order));
1785 args.vma = vmf->vma;
1786 args.end = args.start + (PAGE_SIZE << order);
1787
1788 nr = (args.end - args.start) >> PAGE_SHIFT;
1789 args.src = kcalloc(nr, sizeof(unsigned long), GFP_KERNEL);
1790 args.dst = kcalloc(nr, sizeof(unsigned long), GFP_KERNEL);
1791 args.pgmap_owner = dmirror->mdevice;
1792 args.flags = dmirror_select_device(dmirror);
1793 args.fault_page = vmf->page;
1794
1795 if (!args.src || !args.dst) {
1796 ret = VM_FAULT_OOM;
1797 goto err;
1798 }
1799
1800 if (order)
1801 args.flags |= MIGRATE_VMA_SELECT_COMPOUND;
1802
1803 /*
1804 * In practice migrate_vma_setup() should never fail unless the
1805 * test is wrong as it just tests some static VMA properties.
1806 */
1807 if (migrate_vma_setup(&args)) {
1808 ret = VM_FAULT_SIGBUS;
1809 goto err;
1810 }
1811
1812 ret = dmirror_devmem_fault_alloc_and_copy(&args, dmirror);
1813 if (ret) {
1814 migrate_vma_finalize(&args);
1815 goto err;
1816 }
1817 migrate_vma_pages(&args);
1818 /*
1819 * No device finalize step is needed since
1820 * dmirror_devmem_fault_alloc_and_copy() will have already
1821 * invalidated the device page table.
1822 */
1823 migrate_vma_finalize(&args);
1824err:
1825 kfree(args.src);
1826 kfree(args.dst);
1827 return ret;
1828}
1829
1830static void dmirror_devmem_folio_split(struct folio *head, struct folio *tail)
1831{
1832 struct page *rpage = BACKING_PAGE(folio_page(head, 0));
1833 struct page *rpage_tail;
1834 struct folio *rfolio;
1835 unsigned long offset = 0;
1836
1837 if (!rpage) {
1838 tail->page.zone_device_data = NULL;
1839 return;
1840 }
1841
1842 rfolio = page_folio(rpage);
1843
1844 if (tail == NULL) {
1845 folio_reset_order(rfolio);
1846 rfolio->mapping = NULL;
1847 folio_set_count(rfolio, 1);
1848 return;
1849 }
1850
1851 offset = folio_pfn(tail) - folio_pfn(head);
1852
1853 rpage_tail = folio_page(rfolio, offset);
1854 tail->page.zone_device_data = rpage_tail;
1855 rpage_tail->zone_device_data = rpage->zone_device_data;
1856 clear_compound_head(rpage_tail);
1857 rpage_tail->mapping = NULL;
1858
1859 folio_page(tail, 0)->mapping = folio_page(head, 0)->mapping;
1860 tail->pgmap = head->pgmap;
1861 folio_set_count(page_folio(rpage_tail), 1);
1862}
1863
1864static const struct dev_pagemap_ops dmirror_devmem_ops = {
1865 .folio_free = dmirror_devmem_free,
1866 .migrate_to_ram = dmirror_devmem_fault,
1867 .folio_split = dmirror_devmem_folio_split,
1868};
1869
1870static void dmirror_device_release(struct device *dev)
1871{
1872 struct dmirror_device *mdevice = container_of(dev, struct dmirror_device, device);
1873
1874 dmirror_device_remove_chunks(mdevice);
1875}
1876
1877static int dmirror_device_init(struct dmirror_device *mdevice, int id)
1878{
1879 dev_t dev;
1880 int ret;
1881
1882 dev = MKDEV(MAJOR(dmirror_dev), id);
1883 mutex_init(&mdevice->devmem_lock);
1884 spin_lock_init(&mdevice->lock);
1885
1886 cdev_init(&mdevice->cdevice, &dmirror_fops);
1887 mdevice->cdevice.owner = THIS_MODULE;
1888 mdevice->device.release = dmirror_device_release;
1889
1890 device_initialize(&mdevice->device);
1891 mdevice->device.devt = dev;
1892
1893 ret = dev_set_name(&mdevice->device, "hmm_dmirror%u", id);
1894 if (ret)
1895 goto put_device;
1896
1897 /* Build a list of free ZONE_DEVICE struct pages */
1898 ret = dmirror_allocate_chunk(mdevice, NULL, false);
1899 if (ret)
1900 goto put_device;
1901
1902 ret = cdev_device_add(&mdevice->cdevice, &mdevice->device);
1903 if (ret)
1904 goto put_device;
1905
1906 return 0;
1907
1908put_device:
1909 put_device(&mdevice->device);
1910 return ret;
1911}
1912
1913static void dmirror_device_remove(struct dmirror_device *mdevice)
1914{
1915 cdev_device_del(&mdevice->cdevice, &mdevice->device);
1916 put_device(&mdevice->device);
1917}
1918
1919static int __init hmm_dmirror_init(void)
1920{
1921 int ret;
1922 int id = 0;
1923 int ndevices = 0;
1924
1925 ret = alloc_chrdev_region(&dmirror_dev, 0, DMIRROR_NDEVICES,
1926 "HMM_DMIRROR");
1927 if (ret)
1928 goto err_unreg;
1929
1930 memset(dmirror_devices, 0, DMIRROR_NDEVICES * sizeof(dmirror_devices[0]));
1931 dmirror_devices[ndevices++].zone_device_type =
1932 HMM_DMIRROR_MEMORY_DEVICE_PRIVATE;
1933 dmirror_devices[ndevices++].zone_device_type =
1934 HMM_DMIRROR_MEMORY_DEVICE_PRIVATE;
1935 if (spm_addr_dev0 && spm_addr_dev1) {
1936 dmirror_devices[ndevices++].zone_device_type =
1937 HMM_DMIRROR_MEMORY_DEVICE_COHERENT;
1938 dmirror_devices[ndevices++].zone_device_type =
1939 HMM_DMIRROR_MEMORY_DEVICE_COHERENT;
1940 }
1941 for (id = 0; id < ndevices; id++) {
1942 ret = dmirror_device_init(dmirror_devices + id, id);
1943 if (ret)
1944 goto err_chrdev;
1945 }
1946
1947 pr_info("HMM test module loaded. This is only for testing HMM.\n");
1948 return 0;
1949
1950err_chrdev:
1951 while (--id >= 0)
1952 dmirror_device_remove(dmirror_devices + id);
1953 unregister_chrdev_region(dmirror_dev, DMIRROR_NDEVICES);
1954err_unreg:
1955 return ret;
1956}
1957
1958static void __exit hmm_dmirror_exit(void)
1959{
1960 int id;
1961
1962 for (id = 0; id < DMIRROR_NDEVICES; id++)
1963 if (dmirror_devices[id].zone_device_type)
1964 dmirror_device_remove(dmirror_devices + id);
1965 unregister_chrdev_region(dmirror_dev, DMIRROR_NDEVICES);
1966}
1967
1968module_init(hmm_dmirror_init);
1969module_exit(hmm_dmirror_exit);
1970MODULE_DESCRIPTION("HMM (Heterogeneous Memory Management) test module");
1971MODULE_LICENSE("GPL");