projects/linux·blob master · refs

1040 lines · 29.9 KiB

1// SPDX-License-Identifier: GPL-2.0-or-later
2/*
3 * Copyright 2013 Red Hat Inc.
4 *
5 * Authors: Jérôme Glisse <[email protected]>
6 */
7/*
8 * Refer to include/linux/hmm.h for information about heterogeneous memory
9 * management or HMM for short.
10 */
11#include <linux/pagewalk.h>
12#include <linux/hmm.h>
13#include <linux/hmm-dma.h>
14#include <linux/init.h>
15#include <linux/rmap.h>
16#include <linux/swap.h>
17#include <linux/slab.h>
18#include <linux/sched.h>
19#include <linux/mmzone.h>
20#include <linux/oom.h>
21#include <linux/pagemap.h>
22#include <linux/leafops.h>
23#include <linux/hugetlb.h>
24#include <linux/memremap.h>
25#include <linux/sched/mm.h>
26#include <linux/jump_label.h>
27#include <linux/dma-mapping.h>
28#include <linux/pci-p2pdma.h>
29#include <linux/mmu_notifier.h>
30#include <linux/memory_hotplug.h>
31
32#include "internal.h"
33
34struct hmm_vma_walk {
35 struct hmm_range *range;
36 bool *locked;
37 unsigned long last;
38 unsigned long end;
39 unsigned int required_fault;
40};
41
42/*
43 * Internal sentinel returned by walk callbacks when they need a page fault.
44 * The callback stores end/required_fault in hmm_vma_walk; the outer loop
45 * consumes the sentinel and never propagates it to the caller.
46 */
47#define HMM_FAULT_PENDING -EAGAIN
48
49/*
50 * Internal sentinel returned by hmm_do_fault() when handle_mm_fault()
51 * completes a page fault with the mmap lock dropped. hmm_do_fault() sets
52 * *locked = false; the outer loop consumes the sentinel and never propagates
53 * it to the caller.
54 */
55#define HMM_FAULT_UNLOCKED -ENOLCK
56
57enum {
58 HMM_NEED_FAULT = 1 << 0,
59 HMM_NEED_WRITE_FAULT = 1 << 1,
60 HMM_NEED_ALL_BITS = HMM_NEED_FAULT | HMM_NEED_WRITE_FAULT,
61};
62
63enum {
64 /* These flags are carried from input-to-output */
65 HMM_PFN_INOUT_FLAGS = HMM_PFN_DMA_MAPPED | HMM_PFN_P2PDMA |
66 HMM_PFN_P2PDMA_BUS,
67};
68
69static int hmm_pfns_fill(unsigned long addr, unsigned long end,
70 struct hmm_range *range, unsigned long cpu_flags)
71{
72 unsigned long i = (addr - range->start) >> PAGE_SHIFT;
73
74 for (; addr < end; addr += PAGE_SIZE, i++) {
75 range->hmm_pfns[i] &= HMM_PFN_INOUT_FLAGS;
76 range->hmm_pfns[i] |= cpu_flags;
77 }
78 return 0;
79}
80
81/*
82 * hmm_record_fault() - record a range that needs to be faulted in
83 *
84 * Called by the walk callbacks when they discover that part of the range
85 * needs a page fault. The callback records what to fault and returns
86 * HMM_FAULT_PENDING; the outer loop in hmm_range_fault_locked() drops
87 * back out of walk_page_range() and invokes handle_mm_fault() from a context
88 * where no page-table or hugetlb_vma_lock is held.
89 */
90static int hmm_record_fault(unsigned long addr, unsigned long end,
91 unsigned int required_fault,
92 struct mm_walk *walk)
93{
94 struct hmm_vma_walk *hmm_vma_walk = walk->private;
95
96 WARN_ON_ONCE(!required_fault);
97 hmm_vma_walk->last = addr;
98 hmm_vma_walk->end = end;
99 hmm_vma_walk->required_fault = required_fault;
100 return HMM_FAULT_PENDING;
101}
102
103static unsigned int hmm_pte_need_fault(const struct hmm_vma_walk *hmm_vma_walk,
104 unsigned long pfn_req_flags,
105 unsigned long cpu_flags)
106{
107 struct hmm_range *range = hmm_vma_walk->range;
108
109 /*
110 * So we not only consider the individual per page request we also
111 * consider the default flags requested for the range. The API can
112 * be used 2 ways. The first one where the HMM user coalesces
113 * multiple page faults into one request and sets flags per pfn for
114 * those faults. The second one where the HMM user wants to pre-
115 * fault a range with specific flags. For the latter one it is a
116 * waste to have the user pre-fill the pfn arrays with a default
117 * flags value.
118 */
119 pfn_req_flags &= range->pfn_flags_mask;
120 pfn_req_flags |= range->default_flags;
121
122 /* We aren't ask to do anything ... */
123 if (!(pfn_req_flags & HMM_PFN_REQ_FAULT))
124 return 0;
125
126 /* Need to write fault ? */
127 if ((pfn_req_flags & HMM_PFN_REQ_WRITE) &&
128 !(cpu_flags & HMM_PFN_WRITE))
129 return HMM_NEED_FAULT | HMM_NEED_WRITE_FAULT;
130
131 /* If CPU page table is not valid then we need to fault */
132 if (!(cpu_flags & HMM_PFN_VALID))
133 return HMM_NEED_FAULT;
134 return 0;
135}
136
137static unsigned int
138hmm_range_need_fault(const struct hmm_vma_walk *hmm_vma_walk,
139 const unsigned long hmm_pfns[], unsigned long npages,
140 unsigned long cpu_flags)
141{
142 struct hmm_range *range = hmm_vma_walk->range;
143 unsigned int required_fault = 0;
144 unsigned long i;
145
146 /*
147 * If the default flags do not request to fault pages, and the mask does
148 * not allow for individual pages to be faulted, then
149 * hmm_pte_need_fault() will always return 0.
150 */
151 if (!((range->default_flags | range->pfn_flags_mask) &
152 HMM_PFN_REQ_FAULT))
153 return 0;
154
155 for (i = 0; i < npages; ++i) {
156 required_fault |= hmm_pte_need_fault(hmm_vma_walk, hmm_pfns[i],
157 cpu_flags);
158 if (required_fault == HMM_NEED_ALL_BITS)
159 return required_fault;
160 }
161 return required_fault;
162}
163
164static int hmm_vma_walk_hole(unsigned long addr, unsigned long end,
165 __always_unused int depth, struct mm_walk *walk)
166{
167 struct hmm_vma_walk *hmm_vma_walk = walk->private;
168 struct hmm_range *range = hmm_vma_walk->range;
169 unsigned int required_fault;
170 unsigned long i, npages;
171 unsigned long *hmm_pfns;
172
173 i = (addr - range->start) >> PAGE_SHIFT;
174 npages = (end - addr) >> PAGE_SHIFT;
175 hmm_pfns = &range->hmm_pfns[i];
176 required_fault =
177 hmm_range_need_fault(hmm_vma_walk, hmm_pfns, npages, 0);
178 if (!walk->vma) {
179 if (required_fault)
180 return -EFAULT;
181 return hmm_pfns_fill(addr, end, range, HMM_PFN_ERROR);
182 }
183 if (required_fault)
184 return hmm_record_fault(addr, end, required_fault, walk);
185 return hmm_pfns_fill(addr, end, range, 0);
186}
187
188static inline unsigned long hmm_pfn_flags_order(unsigned long order)
189{
190 return order << HMM_PFN_ORDER_SHIFT;
191}
192
193#ifdef CONFIG_TRANSPARENT_HUGEPAGE
194static inline unsigned long pmd_to_hmm_pfn_flags(struct hmm_range *range,
195 pmd_t pmd)
196{
197 if (pmd_protnone(pmd))
198 return 0;
199 return (pmd_write(pmd) ? (HMM_PFN_VALID | HMM_PFN_WRITE) :
200 HMM_PFN_VALID) |
201 hmm_pfn_flags_order(PMD_SHIFT - PAGE_SHIFT);
202}
203
204static int hmm_vma_handle_pmd(struct mm_walk *walk, unsigned long addr,
205 unsigned long end, unsigned long hmm_pfns[],
206 pmd_t pmd)
207{
208 struct hmm_vma_walk *hmm_vma_walk = walk->private;
209 struct hmm_range *range = hmm_vma_walk->range;
210 unsigned long pfn, npages, i;
211 unsigned int required_fault;
212 unsigned long cpu_flags;
213
214 npages = (end - addr) >> PAGE_SHIFT;
215 cpu_flags = pmd_to_hmm_pfn_flags(range, pmd);
216 required_fault =
217 hmm_range_need_fault(hmm_vma_walk, hmm_pfns, npages, cpu_flags);
218 if (required_fault)
219 return hmm_record_fault(addr, end, required_fault, walk);
220
221 pfn = pmd_pfn(pmd) + ((addr & ~PMD_MASK) >> PAGE_SHIFT);
222 for (i = 0; addr < end; addr += PAGE_SIZE, i++, pfn++) {
223 hmm_pfns[i] &= HMM_PFN_INOUT_FLAGS;
224 hmm_pfns[i] |= pfn | cpu_flags;
225 }
226 return 0;
227}
228#else /* CONFIG_TRANSPARENT_HUGEPAGE */
229/* stub to allow the code below to compile */
230int hmm_vma_handle_pmd(struct mm_walk *walk, unsigned long addr,
231 unsigned long end, unsigned long hmm_pfns[], pmd_t pmd);
232#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
233
234static inline unsigned long pte_to_hmm_pfn_flags(struct hmm_range *range,
235 pte_t pte)
236{
237 if (pte_none(pte) || !pte_present(pte) || pte_protnone(pte))
238 return 0;
239 return pte_write(pte) ? (HMM_PFN_VALID | HMM_PFN_WRITE) : HMM_PFN_VALID;
240}
241
242static int hmm_vma_handle_pte(struct mm_walk *walk, unsigned long addr,
243 unsigned long end, pmd_t *pmdp, pte_t *ptep,
244 unsigned long *hmm_pfn)
245{
246 struct hmm_vma_walk *hmm_vma_walk = walk->private;
247 struct hmm_range *range = hmm_vma_walk->range;
248 unsigned int required_fault;
249 unsigned long cpu_flags;
250 pte_t pte = ptep_get(ptep);
251 uint64_t pfn_req_flags = *hmm_pfn;
252 uint64_t new_pfn_flags = 0;
253
254 /*
255 * Any other marker than a UFFD WP marker will result in a fault error
256 * that will be correctly handled, so we need only check for UFFD WP
257 * here.
258 */
259 if (pte_none(pte) || pte_is_uffd_wp_marker(pte)) {
260 required_fault =
261 hmm_pte_need_fault(hmm_vma_walk, pfn_req_flags, 0);
262 if (required_fault)
263 goto fault;
264 goto out;
265 }
266
267 if (!pte_present(pte)) {
268 const softleaf_t entry = softleaf_from_pte(pte);
269
270 /*
271 * Don't fault in device private pages owned by the caller,
272 * just report the PFN.
273 */
274 if (softleaf_is_device_private(entry) &&
275 page_pgmap(softleaf_to_page(entry))->owner ==
276 range->dev_private_owner) {
277 cpu_flags = HMM_PFN_VALID;
278 if (softleaf_is_device_private_write(entry))
279 cpu_flags |= HMM_PFN_WRITE;
280 new_pfn_flags = softleaf_to_pfn(entry) | cpu_flags;
281 goto out;
282 }
283
284 required_fault =
285 hmm_pte_need_fault(hmm_vma_walk, pfn_req_flags, 0);
286 if (!required_fault)
287 goto out;
288
289 if (softleaf_is_swap(entry))
290 goto fault;
291
292 if (softleaf_is_device_private(entry))
293 goto fault;
294
295 if (softleaf_is_device_exclusive(entry))
296 goto fault;
297
298 if (softleaf_is_migration(entry)) {
299 pte_unmap(ptep);
300 hmm_vma_walk->last = addr;
301 migration_entry_wait(walk->mm, pmdp, addr);
302 return -EBUSY;
303 }
304
305 /* Report error for everything else */
306 pte_unmap(ptep);
307 return -EFAULT;
308 }
309
310 cpu_flags = pte_to_hmm_pfn_flags(range, pte);
311 required_fault =
312 hmm_pte_need_fault(hmm_vma_walk, pfn_req_flags, cpu_flags);
313 if (required_fault)
314 goto fault;
315
316 /*
317 * Since each architecture defines a struct page for the zero page, just
318 * fall through and treat it like a normal page.
319 */
320 if (!vm_normal_page(walk->vma, addr, pte) &&
321 !is_zero_pfn(pte_pfn(pte))) {
322 if (hmm_pte_need_fault(hmm_vma_walk, pfn_req_flags, 0)) {
323 pte_unmap(ptep);
324 return -EFAULT;
325 }
326 new_pfn_flags = HMM_PFN_ERROR;
327 goto out;
328 }
329
330 new_pfn_flags = pte_pfn(pte) | cpu_flags;
331out:
332 *hmm_pfn = (*hmm_pfn & HMM_PFN_INOUT_FLAGS) | new_pfn_flags;
333 return 0;
334
335fault:
336 pte_unmap(ptep);
337 /* Fault any virtual address we were asked to fault */
338 return hmm_record_fault(addr, end, required_fault, walk);
339}
340
341#ifdef CONFIG_ARCH_HAS_PMD_SOFTLEAVES
342static int hmm_vma_handle_absent_pmd(struct mm_walk *walk, unsigned long start,
343 unsigned long end, unsigned long *hmm_pfns,
344 pmd_t pmd)
345{
346 struct hmm_vma_walk *hmm_vma_walk = walk->private;
347 struct hmm_range *range = hmm_vma_walk->range;
348 unsigned long npages = (end - start) >> PAGE_SHIFT;
349 const softleaf_t entry = softleaf_from_pmd(pmd);
350 unsigned long addr = start;
351 unsigned int required_fault;
352
353 if (softleaf_is_device_private(entry) &&
354 softleaf_to_folio(entry)->pgmap->owner ==
355 range->dev_private_owner) {
356 unsigned long cpu_flags = HMM_PFN_VALID |
357 hmm_pfn_flags_order(PMD_SHIFT - PAGE_SHIFT);
358 unsigned long pfn = softleaf_to_pfn(entry);
359 unsigned long i;
360
361 if (softleaf_is_device_private_write(entry))
362 cpu_flags |= HMM_PFN_WRITE;
363
364 /*
365 * Fully populate the PFN list though subsequent PFNs could be
366 * inferred, because drivers which are not yet aware of large
367 * folios probably do not support sparsely populated PFN lists.
368 */
369 for (i = 0; addr < end; addr += PAGE_SIZE, i++, pfn++) {
370 hmm_pfns[i] &= HMM_PFN_INOUT_FLAGS;
371 hmm_pfns[i] |= pfn | cpu_flags;
372 }
373
374 return 0;
375 }
376
377 required_fault = hmm_range_need_fault(hmm_vma_walk, hmm_pfns,
378 npages, 0);
379 if (required_fault) {
380 if (softleaf_is_device_private(entry))
381 return hmm_record_fault(addr, end, required_fault, walk);
382 else
383 return -EFAULT;
384 }
385
386 return hmm_pfns_fill(start, end, range, HMM_PFN_ERROR);
387}
388#else
389static int hmm_vma_handle_absent_pmd(struct mm_walk *walk, unsigned long start,
390 unsigned long end, unsigned long *hmm_pfns,
391 pmd_t pmd)
392{
393 struct hmm_vma_walk *hmm_vma_walk = walk->private;
394 struct hmm_range *range = hmm_vma_walk->range;
395 unsigned long npages = (end - start) >> PAGE_SHIFT;
396
397 if (hmm_range_need_fault(hmm_vma_walk, hmm_pfns, npages, 0))
398 return -EFAULT;
399 return hmm_pfns_fill(start, end, range, HMM_PFN_ERROR);
400}
401#endif /* CONFIG_ARCH_HAS_PMD_SOFTLEAVES */
402
403static int hmm_vma_walk_pmd(pmd_t *pmdp,
404 unsigned long start,
405 unsigned long end,
406 struct mm_walk *walk)
407{
408 struct hmm_vma_walk *hmm_vma_walk = walk->private;
409 struct hmm_range *range = hmm_vma_walk->range;
410 unsigned long *hmm_pfns =
411 &range->hmm_pfns[(start - range->start) >> PAGE_SHIFT];
412 unsigned long npages = (end - start) >> PAGE_SHIFT;
413 unsigned long addr = start;
414 pte_t *ptep;
415 pmd_t pmd;
416
417again:
418 pmd = pmdp_get_lockless(pmdp);
419 if (pmd_none(pmd))
420 return hmm_vma_walk_hole(start, end, -1, walk);
421
422 if (thp_migration_supported() && pmd_is_migration_entry(pmd)) {
423 if (hmm_range_need_fault(hmm_vma_walk, hmm_pfns, npages, 0)) {
424 hmm_vma_walk->last = addr;
425 pmd_migration_entry_wait(walk->mm, pmdp);
426 return -EBUSY;
427 }
428 return hmm_pfns_fill(start, end, range, 0);
429 }
430
431 if (!pmd_present(pmd))
432 return hmm_vma_handle_absent_pmd(walk, start, end, hmm_pfns,
433 pmd);
434
435 if (pmd_trans_huge(pmd)) {
436 /*
437 * No need to take pmd_lock here, even if some other thread
438 * is splitting the huge pmd we will get that event through
439 * mmu_notifier callback.
440 *
441 * So just read pmd value and check again it's a transparent
442 * huge or device mapping one and compute corresponding pfn
443 * values.
444 */
445 pmd = pmdp_get_lockless(pmdp);
446 if (!pmd_trans_huge(pmd))
447 goto again;
448
449 return hmm_vma_handle_pmd(walk, addr, end, hmm_pfns, pmd);
450 }
451
452 /*
453 * We have handled all the valid cases above ie either none, migration,
454 * huge or transparent huge. At this point either it is a valid pmd
455 * entry pointing to pte directory or it is a bad pmd that will not
456 * recover.
457 */
458 if (pmd_bad(pmd)) {
459 if (hmm_range_need_fault(hmm_vma_walk, hmm_pfns, npages, 0))
460 return -EFAULT;
461 return hmm_pfns_fill(start, end, range, HMM_PFN_ERROR);
462 }
463
464 ptep = pte_offset_map(pmdp, addr);
465 if (!ptep)
466 goto again;
467 for (; addr < end; addr += PAGE_SIZE, ptep++, hmm_pfns++) {
468 int r;
469
470 r = hmm_vma_handle_pte(walk, addr, end, pmdp, ptep, hmm_pfns);
471 if (r) {
472 /* hmm_vma_handle_pte() did pte_unmap() */
473 return r;
474 }
475 }
476 pte_unmap(ptep - 1);
477 return 0;
478}
479
480#if defined(CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD)
481static inline unsigned long pud_to_hmm_pfn_flags(struct hmm_range *range,
482 pud_t pud)
483{
484 if (!pud_present(pud))
485 return 0;
486 return (pud_write(pud) ? (HMM_PFN_VALID | HMM_PFN_WRITE) :
487 HMM_PFN_VALID) |
488 hmm_pfn_flags_order(PUD_SHIFT - PAGE_SHIFT);
489}
490
491static int hmm_vma_walk_pud(pud_t *pudp, unsigned long start, unsigned long end,
492 struct mm_walk *walk)
493{
494 struct hmm_vma_walk *hmm_vma_walk = walk->private;
495 struct hmm_range *range = hmm_vma_walk->range;
496 unsigned long addr = start;
497 pud_t pud;
498 spinlock_t *ptl = pud_trans_huge_lock(pudp, walk->vma);
499
500 if (!ptl)
501 return 0;
502
503 /* Normally we don't want to split the huge page */
504 walk->action = ACTION_CONTINUE;
505
506 pud = pudp_get(pudp);
507 if (!pud_present(pud)) {
508 spin_unlock(ptl);
509 return hmm_vma_walk_hole(start, end, -1, walk);
510 }
511
512 if (pud_leaf(pud)) {
513 unsigned long i, npages, pfn;
514 unsigned int required_fault;
515 unsigned long *hmm_pfns;
516 unsigned long cpu_flags;
517
518 i = (addr - range->start) >> PAGE_SHIFT;
519 npages = (end - addr) >> PAGE_SHIFT;
520 hmm_pfns = &range->hmm_pfns[i];
521
522 cpu_flags = pud_to_hmm_pfn_flags(range, pud);
523 required_fault = hmm_range_need_fault(hmm_vma_walk, hmm_pfns,
524 npages, cpu_flags);
525 if (required_fault) {
526 spin_unlock(ptl);
527 return hmm_record_fault(addr, end, required_fault, walk);
528 }
529
530 pfn = pud_pfn(pud) + ((addr & ~PUD_MASK) >> PAGE_SHIFT);
531 for (i = 0; i < npages; ++i, ++pfn) {
532 hmm_pfns[i] &= HMM_PFN_INOUT_FLAGS;
533 hmm_pfns[i] |= pfn | cpu_flags;
534 }
535 goto out_unlock;
536 }
537
538 /* Ask for the PUD to be split */
539 walk->action = ACTION_SUBTREE;
540
541out_unlock:
542 spin_unlock(ptl);
543 return 0;
544}
545#else
546#define hmm_vma_walk_pud NULL
547#endif
548
549#ifdef CONFIG_HUGETLB_PAGE
550static int hmm_vma_walk_hugetlb_entry(pte_t *pte, unsigned long hmask,
551 unsigned long start, unsigned long end,
552 struct mm_walk *walk)
553{
554 unsigned long addr = start, i, pfn;
555 struct hmm_vma_walk *hmm_vma_walk = walk->private;
556 struct hmm_range *range = hmm_vma_walk->range;
557 struct vm_area_struct *vma = walk->vma;
558 unsigned int required_fault;
559 unsigned long pfn_req_flags;
560 unsigned long cpu_flags;
561 spinlock_t *ptl;
562 pte_t entry;
563
564 ptl = huge_pte_lock(hstate_vma(vma), walk->mm, pte);
565 entry = huge_ptep_get(walk->mm, addr, pte);
566
567 i = (start - range->start) >> PAGE_SHIFT;
568 pfn_req_flags = range->hmm_pfns[i];
569 cpu_flags = pte_to_hmm_pfn_flags(range, entry) |
570 hmm_pfn_flags_order(huge_page_order(hstate_vma(vma)));
571 required_fault =
572 hmm_pte_need_fault(hmm_vma_walk, pfn_req_flags, cpu_flags);
573 if (required_fault) {
574 spin_unlock(ptl);
575 return hmm_record_fault(addr, end, required_fault, walk);
576 }
577
578 pfn = pte_pfn(entry) + ((start & ~hmask) >> PAGE_SHIFT);
579 for (; addr < end; addr += PAGE_SIZE, i++, pfn++) {
580 range->hmm_pfns[i] &= HMM_PFN_INOUT_FLAGS;
581 range->hmm_pfns[i] |= pfn | cpu_flags;
582 }
583
584 spin_unlock(ptl);
585 return 0;
586}
587#else
588#define hmm_vma_walk_hugetlb_entry NULL
589#endif /* CONFIG_HUGETLB_PAGE */
590
591static int hmm_vma_walk_test(unsigned long start, unsigned long end,
592 struct mm_walk *walk)
593{
594 struct hmm_vma_walk *hmm_vma_walk = walk->private;
595 struct hmm_range *range = hmm_vma_walk->range;
596 struct vm_area_struct *vma = walk->vma;
597
598 if (!(vma->vm_flags & (VM_IO | VM_PFNMAP)) &&
599 vma->vm_flags & VM_READ)
600 return 0;
601
602 /*
603 * vma ranges that don't have struct page backing them or map I/O
604 * devices directly cannot be handled by hmm_range_fault().
605 *
606 * If the vma does not allow read access, then assume that it does not
607 * allow write access either. HMM does not support architectures that
608 * allow write without read.
609 *
610 * If a fault is requested for an unsupported range then it is a hard
611 * failure.
612 */
613 if (hmm_range_need_fault(hmm_vma_walk,
614 range->hmm_pfns +
615 ((start - range->start) >> PAGE_SHIFT),
616 (end - start) >> PAGE_SHIFT, 0))
617 return -EFAULT;
618
619 hmm_pfns_fill(start, end, range, HMM_PFN_ERROR);
620
621 /* Skip this vma and continue processing the next vma. */
622 return 1;
623}
624
625static const struct mm_walk_ops hmm_walk_ops = {
626 .pud_entry = hmm_vma_walk_pud,
627 .pmd_entry = hmm_vma_walk_pmd,
628 .pte_hole = hmm_vma_walk_hole,
629 .hugetlb_entry = hmm_vma_walk_hugetlb_entry,
630 .test_walk = hmm_vma_walk_test,
631 .walk_lock = PGWALK_RDLOCK,
632};
633
634/*
635 * hmm_do_fault - fault in a range recorded by a walk callback
636 *
637 * Called from the outer loop in hmm_range_fault_locked() after a callback
638 * returned HMM_FAULT_PENDING. At this point we hold only mmap_lock;
639 * the page-table spinlock and any hugetlb_vma_lock acquired by the walk
640 * framework have already been released by the unwind.
641 *
642 * Returns -EBUSY on success (all pages faulted, caller should re-walk).
643 * Returns a negative errno on failure.
644 */
645static int hmm_do_fault(struct mm_struct *mm,
646 struct hmm_vma_walk *hmm_vma_walk)
647{
648 unsigned long addr = hmm_vma_walk->last;
649 unsigned long end = hmm_vma_walk->end;
650 unsigned int required_fault = hmm_vma_walk->required_fault;
651 unsigned int fault_flags = FAULT_FLAG_REMOTE;
652 struct vm_area_struct *vma;
653
654 if (hmm_vma_walk->locked)
655 fault_flags |= FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE;
656
657 vma = vma_lookup(mm, addr);
658 if (!vma)
659 return -EFAULT;
660
661 if (required_fault & HMM_NEED_WRITE_FAULT) {
662 if (!(vma->vm_flags & VM_WRITE))
663 return -EPERM;
664 fault_flags |= FAULT_FLAG_WRITE;
665 }
666
667 for (; addr < end; addr += PAGE_SIZE) {
668 vm_fault_t ret;
669
670 ret = handle_mm_fault(vma, addr, fault_flags, NULL);
671
672 if (ret & (VM_FAULT_COMPLETED | VM_FAULT_RETRY)) {
673 if (hmm_vma_walk->locked) /* needed by sparse */
674 *hmm_vma_walk->locked = false;
675 else
676 WARN_ON_ONCE(1); /* broken fault handler */
677 return HMM_FAULT_UNLOCKED;
678 }
679
680 if (ret & VM_FAULT_ERROR) {
681 int err = vm_fault_to_errno(ret, 0);
682
683 if (WARN_ON(!err))
684 err = -EINVAL;
685
686 return err;
687 }
688 }
689
690 return -EBUSY;
691}
692
693static int hmm_range_fault_locked(struct hmm_range *range, bool *locked)
694{
695 struct hmm_vma_walk hmm_vma_walk = {
696 .range = range,
697 .locked = locked,
698 .last = range->start,
699 };
700 struct mm_struct *mm = range->notifier->mm;
701 int ret;
702
703 mmap_assert_locked(mm);
704
705 do {
706 /* If range is no longer valid force retry. */
707 if (mmu_interval_check_retry(range->notifier,
708 range->notifier_seq))
709 return -EBUSY;
710 ret = walk_page_range(mm, hmm_vma_walk.last, range->end,
711 &hmm_walk_ops, &hmm_vma_walk);
712 /*
713 * When HMM_FAULT_PENDING is returned a walk callback
714 * recorded a range that needs handle_mm_fault();
715 * hmm_do_fault() runs the fault outside walk_page_range()
716 * (so no page-table or hugetlb_vma_lock is held) and
717 * returns -EBUSY so the loop re-walks and picks up the
718 * now-present entries.
719 */
720 if (ret == HMM_FAULT_PENDING) {
721 ret = hmm_do_fault(mm, &hmm_vma_walk);
722 if (ret == HMM_FAULT_UNLOCKED) {
723 if (fatal_signal_pending(current))
724 return -EINTR;
725 return -EBUSY;
726 }
727 }
728 /*
729 * When -EBUSY is returned the loop restarts with
730 * hmm_vma_walk.last set to an address that has not been stored
731 * in pfns. All entries < last in the pfn array are set to their
732 * output, and all >= are still at their input values.
733 */
734 } while (ret == -EBUSY);
735 return ret;
736}
737
738/**
739 * hmm_range_fault - try to fault some address in a virtual address range
740 * @range: argument structure
741 *
742 * Returns 0 on success or one of the following error codes:
743 *
744 * -EINVAL: Invalid arguments or mm or virtual address is in an invalid vma
745 * (e.g., device file vma).
746 * -ENOMEM: Out of memory.
747 * -EPERM: Invalid permission (e.g., asking for write and range is read
748 * only).
749 * -EBUSY: The range has been invalidated and the caller needs to wait for
750 * the invalidation to finish.
751 * -EFAULT: A page was requested to be valid and could not be made valid
752 * ie it has no backing VMA or it is illegal to access
753 *
754 * This is similar to get_user_pages(), except that it can read the page tables
755 * without mutating them (ie causing faults).
756 *
757 * The mmap lock must be held by the caller and will remain held on return.
758 * New users should prefer hmm_range_fault_unlocked_timeout() unless they
759 * specifically need to keep the mmap lock held across the call. This helper
760 * cannot support VMAs whose fault handlers need to drop the mmap lock.
761 */
762int hmm_range_fault(struct hmm_range *range)
763{
764 return hmm_range_fault_locked(range, NULL);
765}
766EXPORT_SYMBOL(hmm_range_fault);
767
768/**
769 * hmm_range_fault_unlocked_timeout - fault in a range with a retry timeout
770 * @range: argument structure
771 * @timeout: timeout in jiffies for internal -EBUSY retries, or 0 to retry
772 * indefinitely
773 *
774 * The caller must not hold the mmap lock. The function takes the mmap read
775 * lock internally and allows handle_mm_fault() to drop it during faults. If
776 * the mmap lock is dropped or the range is invalidated, the function refreshes
777 * range->notifier_seq and restarts the walk internally.
778 *
779 * Passing 0 for @timeout retries indefinitely. A non-zero @timeout is a caller
780 * policy limit for repeated mmu-notifier invalidation retries. HMM does not
781 * interrupt page fault handling when the timeout expires, but returns -EBUSY
782 * if the retry budget is exhausted before a stable range is obtained.
783 *
784 * Returns 0 on success or one of the error codes documented for
785 * hmm_range_fault(). -EINTR is returned if mmap_lock acquisition is
786 * interrupted or a fatal signal is pending during retry handling.
787 */
788int hmm_range_fault_unlocked_timeout(struct hmm_range *range,
789 unsigned long timeout)
790{
791 struct mm_struct *mm = range->notifier->mm;
792 unsigned long deadline = 0;
793 bool locked = false;
794 int ret;
795
796 do {
797 /*
798 * If the previous fault dropped mmap_lock, then the fault
799 * handler made progress. Restart the retry timeout in that
800 * case, but keep the existing deadline for ordinary -EBUSY
801 * retries.
802 */
803 if (timeout && !locked)
804 deadline = jiffies + timeout;
805
806 range->notifier_seq =
807 mmu_interval_read_begin(range->notifier);
808
809 ret = mmap_read_lock_killable(mm);
810 if (ret)
811 return ret;
812
813 if (check_stable_address_space(mm)) {
814 mmap_read_unlock(mm);
815 return -EFAULT;
816 }
817
818 if (timeout && time_after(jiffies, deadline)) {
819 mmap_read_unlock(mm);
820 return -EBUSY;
821 }
822
823 locked = true;
824 ret = hmm_range_fault_locked(range, &locked);
825 if (locked)
826 mmap_read_unlock(mm);
827 } while (ret == -EBUSY);
828
829 return ret;
830}
831EXPORT_SYMBOL(hmm_range_fault_unlocked_timeout);
832
833/**
834 * hmm_dma_map_alloc - Allocate HMM map structure
835 * @dev: device to allocate structure for
836 * @map: HMM map to allocate
837 * @nr_entries: number of entries in the map
838 * @dma_entry_size: size of the DMA entry in the map
839 *
840 * Allocate the HMM map structure and all the lists it contains.
841 * Return 0 on success, -ENOMEM on failure.
842 */
843int hmm_dma_map_alloc(struct device *dev, struct hmm_dma_map *map,
844 size_t nr_entries, size_t dma_entry_size)
845{
846 bool dma_need_sync = false;
847 bool use_iova;
848
849 WARN_ON_ONCE(!(nr_entries * PAGE_SIZE / dma_entry_size));
850
851 /*
852 * The HMM API violates our normal DMA buffer ownership rules and can't
853 * transfer buffer ownership. The dma_addressing_limited() check is a
854 * best approximation to ensure no swiotlb buffering happens.
855 */
856#ifdef CONFIG_DMA_NEED_SYNC
857 dma_need_sync = !dev_dma_skip_sync(dev);
858#endif /* CONFIG_DMA_NEED_SYNC */
859 if (dma_need_sync || dma_addressing_limited(dev))
860 return -EOPNOTSUPP;
861
862 map->dma_entry_size = dma_entry_size;
863 map->pfn_list = kvcalloc(nr_entries, sizeof(*map->pfn_list),
864 GFP_KERNEL | __GFP_NOWARN);
865 if (!map->pfn_list)
866 return -ENOMEM;
867
868 use_iova = dma_iova_try_alloc(dev, &map->state, 0,
869 nr_entries * PAGE_SIZE);
870 if (!use_iova && dma_need_unmap(dev)) {
871 map->dma_list = kvzalloc_objs(*map->dma_list, nr_entries,
872 GFP_KERNEL | __GFP_NOWARN);
873 if (!map->dma_list)
874 goto err_dma;
875 }
876 return 0;
877
878err_dma:
879 kvfree(map->pfn_list);
880 return -ENOMEM;
881}
882EXPORT_SYMBOL_GPL(hmm_dma_map_alloc);
883
884/**
885 * hmm_dma_map_free - iFree HMM map structure
886 * @dev: device to free structure from
887 * @map: HMM map containing the various lists and state
888 *
889 * Free the HMM map structure and all the lists it contains.
890 */
891void hmm_dma_map_free(struct device *dev, struct hmm_dma_map *map)
892{
893 if (dma_use_iova(&map->state))
894 dma_iova_free(dev, &map->state);
895 kvfree(map->pfn_list);
896 kvfree(map->dma_list);
897}
898EXPORT_SYMBOL_GPL(hmm_dma_map_free);
899
900/**
901 * hmm_dma_map_pfn - Map a physical HMM page to DMA address
902 * @dev: Device to map the page for
903 * @map: HMM map
904 * @idx: Index into the PFN and dma address arrays
905 * @p2pdma_state: PCI P2P state.
906 *
907 * dma_alloc_iova() allocates IOVA based on the size specified by their use in
908 * iova->size. Call this function after IOVA allocation to link whole @page
909 * to get the DMA address. Note that very first call to this function
910 * will have @offset set to 0 in the IOVA space allocated from
911 * dma_alloc_iova(). For subsequent calls to this function on same @iova,
912 * @offset needs to be advanced by the caller with the size of previous
913 * page that was linked + DMA address returned for the previous page that was
914 * linked by this function.
915 */
916dma_addr_t hmm_dma_map_pfn(struct device *dev, struct hmm_dma_map *map,
917 size_t idx,
918 struct pci_p2pdma_map_state *p2pdma_state)
919{
920 struct dma_iova_state *state = &map->state;
921 dma_addr_t *dma_addrs = map->dma_list;
922 unsigned long *pfns = map->pfn_list;
923 struct page *page = hmm_pfn_to_page(pfns[idx]);
924 phys_addr_t paddr = hmm_pfn_to_phys(pfns[idx]);
925 size_t offset = idx * map->dma_entry_size;
926 unsigned long attrs = DMA_ATTR_REQUIRE_COHERENT;
927 dma_addr_t dma_addr;
928 int ret;
929
930 if ((pfns[idx] & HMM_PFN_DMA_MAPPED) &&
931 !(pfns[idx] & HMM_PFN_P2PDMA_BUS)) {
932 /*
933 * We are in this flow when there is a need to resync flags,
934 * for example when page was already linked in prefetch call
935 * with READ flag and now we need to add WRITE flag
936 *
937 * This page was already programmed to HW and we don't want/need
938 * to unlink and link it again just to resync flags.
939 */
940 if (dma_use_iova(state))
941 return state->addr + offset;
942
943 /*
944 * Without dma_need_unmap, the dma_addrs array is NULL, thus we
945 * need to regenerate the address below even if there already
946 * was a mapping. But !dma_need_unmap implies that the
947 * mapping stateless, so this is fine.
948 */
949 if (dma_need_unmap(dev))
950 return dma_addrs[idx];
951
952 /* Continue to remapping */
953 }
954
955 switch (pci_p2pdma_state(p2pdma_state, dev, page)) {
956 case PCI_P2PDMA_MAP_NONE:
957 break;
958 case PCI_P2PDMA_MAP_THRU_HOST_BRIDGE:
959 attrs |= DMA_ATTR_MMIO;
960 pfns[idx] |= HMM_PFN_P2PDMA;
961 break;
962 case PCI_P2PDMA_MAP_BUS_ADDR:
963 pfns[idx] |= HMM_PFN_P2PDMA_BUS | HMM_PFN_DMA_MAPPED;
964 return pci_p2pdma_bus_addr_map(p2pdma_state->mem, paddr);
965 default:
966 return DMA_MAPPING_ERROR;
967 }
968
969 if (dma_use_iova(state)) {
970 ret = dma_iova_link(dev, state, paddr, offset,
971 map->dma_entry_size, DMA_BIDIRECTIONAL,
972 attrs);
973 if (ret)
974 goto error;
975
976 ret = dma_iova_sync(dev, state, offset, map->dma_entry_size);
977 if (ret) {
978 dma_iova_unlink(dev, state, offset, map->dma_entry_size,
979 DMA_BIDIRECTIONAL, attrs);
980 goto error;
981 }
982
983 dma_addr = state->addr + offset;
984 } else {
985 if (WARN_ON_ONCE(dma_need_unmap(dev) && !dma_addrs))
986 goto error;
987
988 dma_addr = dma_map_phys(dev, paddr, map->dma_entry_size,
989 DMA_BIDIRECTIONAL, attrs);
990 if (dma_mapping_error(dev, dma_addr))
991 goto error;
992
993 if (dma_need_unmap(dev))
994 dma_addrs[idx] = dma_addr;
995 }
996 pfns[idx] |= HMM_PFN_DMA_MAPPED;
997 return dma_addr;
998error:
999 pfns[idx] &= ~HMM_PFN_P2PDMA;
1000 return DMA_MAPPING_ERROR;
1001
1002}
1003EXPORT_SYMBOL_GPL(hmm_dma_map_pfn);
1004
1005/**
1006 * hmm_dma_unmap_pfn - Unmap a physical HMM page from DMA address
1007 * @dev: Device to unmap the page from
1008 * @map: HMM map
1009 * @idx: Index of the PFN to unmap
1010 *
1011 * Returns true if the PFN was mapped and has been unmapped, false otherwise.
1012 */
1013bool hmm_dma_unmap_pfn(struct device *dev, struct hmm_dma_map *map, size_t idx)
1014{
1015 const unsigned long valid_dma = HMM_PFN_VALID | HMM_PFN_DMA_MAPPED;
1016 struct dma_iova_state *state = &map->state;
1017 dma_addr_t *dma_addrs = map->dma_list;
1018 unsigned long *pfns = map->pfn_list;
1019 unsigned long attrs = DMA_ATTR_REQUIRE_COHERENT;
1020
1021 if ((pfns[idx] & valid_dma) != valid_dma)
1022 return false;
1023
1024 if (pfns[idx] & HMM_PFN_P2PDMA)
1025 attrs |= DMA_ATTR_MMIO;
1026
1027 if (pfns[idx] & HMM_PFN_P2PDMA_BUS)
1028 ; /* no need to unmap bus address P2P mappings */
1029 else if (dma_use_iova(state))
1030 dma_iova_unlink(dev, state, idx * map->dma_entry_size,
1031 map->dma_entry_size, DMA_BIDIRECTIONAL, attrs);
1032 else if (dma_need_unmap(dev))
1033 dma_unmap_phys(dev, dma_addrs[idx], map->dma_entry_size,
1034 DMA_BIDIRECTIONAL, attrs);
1035
1036 pfns[idx] &=
1037 ~(HMM_PFN_DMA_MAPPED | HMM_PFN_P2PDMA | HMM_PFN_P2PDMA_BUS);
1038 return true;
1039}
1040EXPORT_SYMBOL_GPL(hmm_dma_unmap_pfn);