projects/linux·blob master · refs

1833 lines · 54.8 KiB

1// SPDX-License-Identifier: GPL-2.0-only OR MIT
2/*
3 * Copyright © 2024 Intel Corporation
4 *
5 * Authors:
6 * Matthew Brost <[email protected]>
7 */
8
9#include <linux/dma-mapping.h>
10#include <linux/export.h>
11#include <linux/hmm.h>
12#include <linux/hugetlb_inline.h>
13#include <linux/memremap.h>
14#include <linux/mm_types.h>
15#include <linux/slab.h>
16
17#include <drm/drm_device.h>
18#include <drm/drm_gpusvm.h>
19#include <drm/drm_pagemap.h>
20#include <drm/drm_print.h>
21
22/**
23 * DOC: Overview
24 *
25 * GPU Shared Virtual Memory (GPU SVM) layer for the Direct Rendering Manager (DRM)
26 * is a component of the DRM framework designed to manage shared virtual memory
27 * between the CPU and GPU. It enables efficient data exchange and processing
28 * for GPU-accelerated applications by allowing memory sharing and
29 * synchronization between the CPU's and GPU's virtual address spaces.
30 *
31 * Key GPU SVM Components:
32 *
33 * - Notifiers:
34 * Used for tracking memory intervals and notifying the GPU of changes,
35 * notifiers are sized based on a GPU SVM initialization parameter, with a
36 * recommendation of 512M or larger. They maintain a Red-BlacK tree and a
37 * list of ranges that fall within the notifier interval. Notifiers are
38 * tracked within a GPU SVM Red-BlacK tree and list and are dynamically
39 * inserted or removed as ranges within the interval are created or
40 * destroyed.
41 * - Ranges:
42 * Represent memory ranges mapped in a DRM device and managed by GPU SVM.
43 * They are sized based on an array of chunk sizes, which is a GPU SVM
44 * initialization parameter, and the CPU address space. Upon GPU fault,
45 * the largest aligned chunk that fits within the faulting CPU address
46 * space is chosen for the range size. Ranges are expected to be
47 * dynamically allocated on GPU fault and removed on an MMU notifier UNMAP
48 * event. As mentioned above, ranges are tracked in a notifier's Red-Black
49 * tree.
50 *
51 * - Pages:
52 * struct drm_gpusvm_pages holds the DMA mapping state for a range of
53 * CPU virtual addresses: the DMA mapped device addresses,
54 * the device private pagemap, the IOVA state, the per mapping
55 * notifier sequence number, and the drm_device that owns the DMA
56 * mappings.
57 * A driver embeds one or more struct drm_gpusvm_pages alongside its
58 * struct drm_gpusvm_range, choosing one of two layouts:
59 *
60 * 1:1 - one drm_gpusvm_pages per range (one drm_device). Simplest
61 * layout; to mirror a VA range on several devices a driver uses a
62 * separate range (and notifier) per device, so the HMM fault is taken
63 * once per device.
64 *
65 * N:1 - one drm_gpusvm_pages per drm_device, all sharing one range and
66 * notifier; only the per-device DMA mapping differs. The instances must
67 * sit in contiguous memory so a single drm_gpusvm_range_set_unmapped()
68 * can mark them all. A driver can keep one instance inline for the single
69 * device case and switch to a heap array only when more devices join,
70 * e.g.:
71 *
72 * .. code-block:: c
73 *
74 * struct driver_range {
75 * struct drm_gpusvm_range base;
76 * unsigned int num_pages; // 1: inline_pages, >1: pages[]
77 * union {
78 * struct drm_gpusvm_pages inline_pages;
79 * struct drm_gpusvm_pages *pages;
80 * };
81 * };
82 *
83 * In the N:1 case the driver allocates the pages array with a zeroing
84 * allocator (e.g. kcalloc(num_pages, ...)), initialises each entry with
85 * drm_gpusvm_init_pages(), and frees each entry with
86 * drm_gpusvm_free_pages() plus the array itself, from its range free
87 * callback. Each drm_gpusvm_pages is mapped independently by their own
88 * drm_device.
89 * Each drm_gpusvm_pages must be zero-initialised and initialised with
90 * drm_gpusvm_init_pages(), called once per entry.
91 *
92 * - Operations:
93 * Define the interface for driver-specific GPU SVM operations such as
94 * range allocation, notifier allocation, and invalidations.
95 *
96 * - Device Memory Allocations:
97 * Embedded structure containing enough information for GPU SVM to migrate
98 * to / from device memory.
99 *
100 * - Device Memory Operations:
101 * Define the interface for driver-specific device memory operations
102 * release memory, populate pfns, and copy to / from device memory.
103 *
104 * This layer provides interfaces for allocating, mapping, migrating, and
105 * releasing memory ranges between the CPU and GPU. It handles all core memory
106 * management interactions (DMA mapping, HMM, and migration) and provides
107 * driver-specific virtual functions (vfuncs). This infrastructure is sufficient
108 * to build the expected driver components for an SVM implementation as detailed
109 * below.
110 *
111 * Expected Driver Components:
112 *
113 * - GPU page fault handler:
114 * Used to create ranges and notifiers based on the fault address,
115 * optionally migrate the range to device memory, and create GPU bindings.
116 *
117 * - Garbage collector:
118 * Used to unmap and destroy GPU bindings for ranges. Ranges are expected
119 * to be added to the garbage collector upon a MMU_NOTIFY_UNMAP event in
120 * notifier callback.
121 *
122 * - Notifier callback:
123 * Used to invalidate and DMA unmap GPU bindings for ranges.
124 */
125
126/**
127 * DOC: Locking
128 *
129 * GPU SVM handles locking for core MM interactions, i.e., it locks/unlocks the
130 * mmap lock as needed.
131 *
132 * GPU SVM introduces a global notifier lock, which safeguards the notifier's
133 * range RB tree and list, as well as the range's DMA mappings and sequence
134 * number. GPU SVM manages all necessary locking and unlocking operations,
135 * except for the recheck range's pages being valid
136 * (drm_gpusvm_pages_valid) when the driver is committing GPU bindings.
137 * This lock corresponds to the ``driver->update`` lock mentioned in
138 * Documentation/mm/hmm.rst. Future revisions may transition from a GPU SVM
139 * global lock to a per-notifier lock if finer-grained locking is deemed
140 * necessary.
141 *
142 * In addition to the locking mentioned above, the driver should implement a
143 * lock to safeguard core GPU SVM function calls that modify state, such as
144 * drm_gpusvm_range_find_or_insert and drm_gpusvm_range_remove. This lock is
145 * denoted as 'driver_svm_lock' in code examples. Finer grained driver side
146 * locking should also be possible for concurrent GPU fault processing within a
147 * single GPU SVM. The 'driver_svm_lock' can be via drm_gpusvm_driver_set_lock
148 * to add annotations to GPU SVM.
149 */
150
151/**
152 * DOC: Partial Unmapping of Ranges
153 *
154 * Partial unmapping of ranges (e.g., 1M out of 2M is unmapped by CPU resulting
155 * in MMU_NOTIFY_UNMAP event) presents several challenges, with the main one
156 * being that a subset of the range still has CPU and GPU mappings. If the
157 * backing store for the range is in device memory, a subset of the backing
158 * store has references. One option would be to split the range and device
159 * memory backing store, but the implementation for this would be quite
160 * complicated. Given that partial unmappings are rare and driver-defined range
161 * sizes are relatively small, GPU SVM does not support splitting of ranges.
162 *
163 * With no support for range splitting, upon partial unmapping of a range, the
164 * driver is expected to invalidate and destroy the entire range. If the range
165 * has device memory as its backing, the driver is also expected to migrate any
166 * remaining pages back to RAM.
167 */
168
169/**
170 * DOC: Examples
171 *
172 * This section provides three examples of how to build the expected driver
173 * components: the GPU page fault handler, the garbage collector, and the
174 * notifier callback.
175 *
176 * The generic code provided does not include logic for complex migration
177 * policies, optimized invalidations, fined grained driver locking, or other
178 * potentially required driver locking (e.g., DMA-resv locks).
179 *
180 * 1) GPU page fault handler
181 *
182 * .. code-block:: c
183 *
184 * struct driver_range {
185 * struct drm_gpusvm_range base;
186 * struct drm_gpusvm_pages pages;
187 * };
188 *
189 * int driver_bind_range(struct drm_gpusvm *gpusvm, struct driver_range *drange)
190 * {
191 * int err = 0;
192 *
193 * driver_alloc_and_setup_memory_for_bind(gpusvm, drange);
194 *
195 * drm_gpusvm_notifier_lock(gpusvm);
196 * if (drm_gpusvm_pages_valid(gpusvm, &drange->pages))
197 * driver_commit_bind(gpusvm, drange);
198 * else
199 * err = -EAGAIN;
200 * drm_gpusvm_notifier_unlock(gpusvm);
201 *
202 * return err;
203 * }
204 *
205 * int driver_gpu_fault(struct drm_gpusvm *gpusvm, unsigned long fault_addr,
206 * unsigned long gpuva_start, unsigned long gpuva_end)
207 * {
208 * struct drm_gpusvm_ctx ctx = {};
209 * struct driver_range *drange;
210 * struct drm_gpusvm_range *range;
211 * int err;
212 *
213 * driver_svm_lock();
214 * retry:
215 * // Always process UNMAPs first so view of GPU SVM ranges is current
216 * driver_garbage_collector(gpusvm);
217 *
218 * range = drm_gpusvm_range_find_or_insert(gpusvm, fault_addr,
219 * gpuva_start, gpuva_end,
220 * &ctx);
221 * if (IS_ERR(range)) {
222 * err = PTR_ERR(range);
223 * goto unlock;
224 * }
225 * drange = container_of(range, struct driver_range, base);
226 *
227 * if (driver_migration_policy(range)) {
228 * err = drm_pagemap_populate_mm(driver_choose_drm_pagemap(),
229 * gpuva_start, gpuva_end, gpusvm->mm,
230 * ctx->timeslice_ms);
231 * if (err) // CPU mappings may have changed
232 * goto retry;
233 * }
234 *
235 * err = drm_gpusvm_get_pages(gpusvm, &drange->pages,
236 * gpusvm->mm, &range->notifier->notifier,
237 * drm_gpusvm_range_start(range),
238 * drm_gpusvm_range_end(range), &ctx);
239 * if (err == -EOPNOTSUPP || err == -EFAULT || err == -EPERM) { // CPU mappings changed
240 * if (err == -EOPNOTSUPP)
241 * drm_gpusvm_range_evict(gpusvm, range);
242 * goto retry;
243 * } else if (err) {
244 * goto unlock;
245 * }
246 *
247 * err = driver_bind_range(gpusvm, drange);
248 * if (err == -EAGAIN) // CPU mappings changed
249 * goto retry
250 *
251 * unlock:
252 * driver_svm_unlock();
253 * return err;
254 * }
255 *
256 * 2) Garbage Collector
257 *
258 * .. code-block:: c
259 *
260 * // The driver owns the drm_gpusvm_pages lifecycle. ops->range_free is
261 * // the final fallback: drm_gpusvm_free_pages() unmaps any
262 * // lingering DMA mapping and a no-op if already unmapped and frees the
263 * // dma_addr array. The normal flow is to DMA unmap before
264 * // drm_gpusvm_range_remove() (before the range leaves the tree).
265 * void driver_range_free(struct drm_gpusvm_range *range)
266 * {
267 * struct driver_range *drange =
268 * container_of(range, struct driver_range, base);
269 *
270 * drm_gpusvm_free_pages(range->gpusvm, &drange->pages,
271 * drm_gpusvm_range_size(range) >> PAGE_SHIFT);
272 * kfree(drange);
273 * }
274 *
275 * void __driver_garbage_collector(struct drm_gpusvm *gpusvm,
276 * struct drm_gpusvm_range *range)
277 * {
278 * assert_driver_svm_locked(gpusvm);
279 *
280 * // Partial unmap, migrate any remaining device memory pages back to RAM
281 * if (range->flags.partial_unmap)
282 * drm_gpusvm_range_evict(gpusvm, range);
283 *
284 * driver_unbind_range(range);
285 * // The pages must be DMA unmapped before drm_gpusvm_range_remove()
286 * // , so a range is never off the MMU interval tree while still DMA
287 * // mapped as the original drmsvm design flow. Otherwise a concurrent CPU
288 * // munmap's notifier could miss this range and free pages still mapped
289 * // for device DMA. This is the normal unmap point.
290 * drm_gpusvm_unmap_pages(gpusvm, &drange->pages,
291 * drm_gpusvm_range_size(range) >> PAGE_SHIFT,
292 * &(struct drm_gpusvm_ctx){ .in_notifier = false });
293 * drm_gpusvm_range_remove(gpusvm, range);
294 * }
295 *
296 * void driver_garbage_collector(struct drm_gpusvm *gpusvm)
297 * {
298 * assert_driver_svm_locked(gpusvm);
299 *
300 * for_each_range_in_garbage_collector(gpusvm, range)
301 * __driver_garbage_collector(gpusvm, range);
302 * }
303 *
304 * 3) Notifier callback
305 *
306 * .. code-block:: c
307 *
308 * void driver_invalidation(struct drm_gpusvm *gpusvm,
309 * struct drm_gpusvm_notifier *notifier,
310 * const struct mmu_notifier_range *mmu_range)
311 * {
312 * struct drm_gpusvm_ctx ctx = { .in_notifier = true, };
313 * struct drm_gpusvm_range *range = NULL;
314 * struct driver_range *drange;
315 *
316 * driver_invalidate_device_pages(gpusvm, mmu_range->start, mmu_range->end);
317 *
318 * drm_gpusvm_for_each_range(range, notifier, mmu_range->start,
319 * mmu_range->end) {
320 * drange = container_of(range, struct driver_range, base);
321 *
322 * drm_gpusvm_unmap_pages(gpusvm, &drange->pages,
323 * drm_gpusvm_range_size(range) >> PAGE_SHIFT,
324 * &ctx);
325 *
326 * if (mmu_range->event != MMU_NOTIFY_UNMAP)
327 * continue;
328 *
329 * drm_gpusvm_range_set_unmapped(range, &drange->pages, 1, mmu_range);
330 * driver_garbage_collector_add(gpusvm, range);
331 * }
332 * }
333 */
334
335/**
336 * npages_in_range() - Calculate the number of pages in a given range
337 * @start: The start address of the range
338 * @end: The end address of the range
339 *
340 * This macro calculates the number of pages in a given memory range,
341 * specified by the start and end addresses. It divides the difference
342 * between the end and start addresses by the page size (PAGE_SIZE) to
343 * determine the number of pages in the range.
344 *
345 * Return: The number of pages in the specified range.
346 */
347static unsigned long
348npages_in_range(unsigned long start, unsigned long end)
349{
350 return (end - start) >> PAGE_SHIFT;
351}
352
353/**
354 * drm_gpusvm_notifier_find() - Find GPU SVM notifier from GPU SVM
355 * @gpusvm: Pointer to the GPU SVM structure.
356 * @start: Start address of the notifier
357 * @end: End address of the notifier
358 *
359 * Return: A pointer to the drm_gpusvm_notifier if found or NULL
360 */
361struct drm_gpusvm_notifier *
362drm_gpusvm_notifier_find(struct drm_gpusvm *gpusvm, unsigned long start,
363 unsigned long end)
364{
365 struct interval_tree_node *itree;
366
367 itree = interval_tree_iter_first(&gpusvm->root, start, end - 1);
368
369 if (itree)
370 return container_of(itree, struct drm_gpusvm_notifier, itree);
371 else
372 return NULL;
373}
374EXPORT_SYMBOL_GPL(drm_gpusvm_notifier_find);
375
376/**
377 * drm_gpusvm_range_find() - Find GPU SVM range from GPU SVM notifier
378 * @notifier: Pointer to the GPU SVM notifier structure.
379 * @start: Start address of the range
380 * @end: End address of the range
381 *
382 * Return: A pointer to the drm_gpusvm_range if found or NULL
383 */
384struct drm_gpusvm_range *
385drm_gpusvm_range_find(struct drm_gpusvm_notifier *notifier, unsigned long start,
386 unsigned long end)
387{
388 struct interval_tree_node *itree;
389
390 itree = interval_tree_iter_first(&notifier->root, start, end - 1);
391
392 if (itree)
393 return container_of(itree, struct drm_gpusvm_range, itree);
394 else
395 return NULL;
396}
397EXPORT_SYMBOL_GPL(drm_gpusvm_range_find);
398
399/**
400 * drm_gpusvm_notifier_invalidate() - Invalidate a GPU SVM notifier.
401 * @mni: Pointer to the mmu_interval_notifier structure.
402 * @mmu_range: Pointer to the mmu_notifier_range structure.
403 * @cur_seq: Current sequence number.
404 *
405 * This function serves as a generic MMU notifier for GPU SVM. It sets the MMU
406 * notifier sequence number and calls the driver invalidate vfunc under
407 * gpusvm->notifier_lock.
408 *
409 * Return: true if the operation succeeds, false otherwise.
410 */
411static bool
412drm_gpusvm_notifier_invalidate(struct mmu_interval_notifier *mni,
413 const struct mmu_notifier_range *mmu_range,
414 unsigned long cur_seq)
415{
416 struct drm_gpusvm_notifier *notifier =
417 container_of(mni, typeof(*notifier), notifier);
418 struct drm_gpusvm *gpusvm = notifier->gpusvm;
419
420 if (!mmu_notifier_range_blockable(mmu_range))
421 return false;
422
423 down_write(&gpusvm->notifier_lock);
424 mmu_interval_set_seq(mni, cur_seq);
425 gpusvm->ops->invalidate(gpusvm, notifier, mmu_range);
426 up_write(&gpusvm->notifier_lock);
427
428 return true;
429}
430
431/*
432 * drm_gpusvm_notifier_ops - MMU interval notifier operations for GPU SVM
433 */
434static const struct mmu_interval_notifier_ops drm_gpusvm_notifier_ops = {
435 .invalidate = drm_gpusvm_notifier_invalidate,
436};
437
438/**
439 * drm_gpusvm_init() - Initialize the GPU SVM.
440 * @gpusvm: Pointer to the GPU SVM structure.
441 * @name: Name of the GPU SVM.
442 * @mm: Pointer to the mm_struct for the address space.
443 * @mm_start: Start address of GPU SVM.
444 * @mm_range: Range of the GPU SVM.
445 * @notifier_size: Size of individual notifiers.
446 * @ops: Pointer to the operations structure for GPU SVM.
447 * @chunk_sizes: Pointer to the array of chunk sizes used in range allocation.
448 * Entries should be powers of 2 in descending order with last
449 * entry being SZ_4K.
450 * @num_chunks: Number of chunks.
451 *
452 * This function initializes the GPU SVM.
453 *
454 * Note: If only using the simple drm_gpusvm_pages API (get/unmap/free),
455 * then only @gpusvm and @name are expected. The @drm drm_device for dma
456 * mappings is bound per-pages via drm_gpusvm_init_pages() before the first
457 * drm_gpusvm_get_pages() call. However, the same base
458 * @gpusvm can also be used with both modes together in which case the full
459 * setup is needed, where the core drm_gpusvm_pages API will simply never use
460 * the other fields.
461 *
462 * Return: 0 on success, a negative error code on failure.
463 */
464int drm_gpusvm_init(struct drm_gpusvm *gpusvm,
465 const char *name,
466 struct mm_struct *mm,
467 unsigned long mm_start, unsigned long mm_range,
468 unsigned long notifier_size,
469 const struct drm_gpusvm_ops *ops,
470 const unsigned long *chunk_sizes, int num_chunks)
471{
472 if (mm) {
473 if (!ops->invalidate || !num_chunks)
474 return -EINVAL;
475 mmgrab(mm);
476 } else {
477 /* No full SVM mode, only core drm_gpusvm_pages API. */
478 if (ops || num_chunks || mm_range || notifier_size)
479 return -EINVAL;
480 }
481
482 gpusvm->name = name;
483 gpusvm->mm = mm;
484 gpusvm->mm_start = mm_start;
485 gpusvm->mm_range = mm_range;
486 gpusvm->notifier_size = notifier_size;
487 gpusvm->ops = ops;
488 gpusvm->chunk_sizes = chunk_sizes;
489 gpusvm->num_chunks = num_chunks;
490
491 gpusvm->root = RB_ROOT_CACHED;
492 INIT_LIST_HEAD(&gpusvm->notifier_list);
493
494 init_rwsem(&gpusvm->notifier_lock);
495
496 fs_reclaim_acquire(GFP_KERNEL);
497 might_lock(&gpusvm->notifier_lock);
498 fs_reclaim_release(GFP_KERNEL);
499
500#ifdef CONFIG_LOCKDEP
501 gpusvm->lock_dep_map = NULL;
502#endif
503
504 return 0;
505}
506EXPORT_SYMBOL_GPL(drm_gpusvm_init);
507
508/**
509 * to_drm_gpusvm_notifier() - retrieve the container struct for a given rbtree node
510 * @node: a pointer to the rbtree node embedded within a drm_gpusvm_notifier struct
511 *
512 * Return: A pointer to the containing drm_gpusvm_notifier structure.
513 */
514static struct drm_gpusvm_notifier *to_drm_gpusvm_notifier(struct rb_node *node)
515{
516 return container_of(node, struct drm_gpusvm_notifier, itree.rb);
517}
518
519/**
520 * drm_gpusvm_notifier_insert() - Insert GPU SVM notifier
521 * @gpusvm: Pointer to the GPU SVM structure
522 * @notifier: Pointer to the GPU SVM notifier structure
523 *
524 * This function inserts the GPU SVM notifier into the GPU SVM RB tree and list.
525 */
526static void drm_gpusvm_notifier_insert(struct drm_gpusvm *gpusvm,
527 struct drm_gpusvm_notifier *notifier)
528{
529 struct rb_node *node;
530 struct list_head *head;
531
532 interval_tree_insert(&notifier->itree, &gpusvm->root);
533
534 node = rb_prev(&notifier->itree.rb);
535 if (node)
536 head = &(to_drm_gpusvm_notifier(node))->entry;
537 else
538 head = &gpusvm->notifier_list;
539
540 list_add(&notifier->entry, head);
541}
542
543/**
544 * drm_gpusvm_notifier_remove() - Remove GPU SVM notifier
545 * @gpusvm: Pointer to the GPU SVM tructure
546 * @notifier: Pointer to the GPU SVM notifier structure
547 *
548 * This function removes the GPU SVM notifier from the GPU SVM RB tree and list.
549 */
550static void drm_gpusvm_notifier_remove(struct drm_gpusvm *gpusvm,
551 struct drm_gpusvm_notifier *notifier)
552{
553 interval_tree_remove(&notifier->itree, &gpusvm->root);
554 list_del(&notifier->entry);
555}
556
557/**
558 * drm_gpusvm_fini() - Finalize the GPU SVM.
559 * @gpusvm: Pointer to the GPU SVM structure.
560 *
561 * This function finalizes the GPU SVM by cleaning up any remaining ranges and
562 * notifiers, and dropping a reference to struct MM.
563 */
564void drm_gpusvm_fini(struct drm_gpusvm *gpusvm)
565{
566 struct drm_gpusvm_notifier *notifier, *next;
567
568 drm_gpusvm_for_each_notifier_safe(notifier, next, gpusvm, 0, LONG_MAX) {
569 struct drm_gpusvm_range *range, *__next;
570
571 /*
572 * Remove notifier first to avoid racing with any invalidation
573 */
574 mmu_interval_notifier_remove(&notifier->notifier);
575 notifier->flags.removed = true;
576
577 drm_gpusvm_for_each_range_safe(range, __next, notifier, 0,
578 LONG_MAX)
579 drm_gpusvm_range_remove(gpusvm, range);
580 }
581
582 if (gpusvm->mm)
583 mmdrop(gpusvm->mm);
584 WARN_ON(!RB_EMPTY_ROOT(&gpusvm->root.rb_root));
585}
586EXPORT_SYMBOL_GPL(drm_gpusvm_fini);
587
588/**
589 * drm_gpusvm_notifier_alloc() - Allocate GPU SVM notifier
590 * @gpusvm: Pointer to the GPU SVM structure
591 * @fault_addr: Fault address
592 *
593 * This function allocates and initializes the GPU SVM notifier structure.
594 *
595 * Return: Pointer to the allocated GPU SVM notifier on success, ERR_PTR() on failure.
596 */
597static struct drm_gpusvm_notifier *
598drm_gpusvm_notifier_alloc(struct drm_gpusvm *gpusvm, unsigned long fault_addr)
599{
600 struct drm_gpusvm_notifier *notifier;
601
602 if (gpusvm->ops->notifier_alloc)
603 notifier = gpusvm->ops->notifier_alloc();
604 else
605 notifier = kzalloc_obj(*notifier);
606
607 if (!notifier)
608 return ERR_PTR(-ENOMEM);
609
610 notifier->gpusvm = gpusvm;
611 notifier->itree.start = ALIGN_DOWN(fault_addr, gpusvm->notifier_size);
612 notifier->itree.last = ALIGN(fault_addr + 1, gpusvm->notifier_size) - 1;
613 INIT_LIST_HEAD(&notifier->entry);
614 notifier->root = RB_ROOT_CACHED;
615 INIT_LIST_HEAD(&notifier->range_list);
616
617 return notifier;
618}
619
620/**
621 * drm_gpusvm_notifier_free() - Free GPU SVM notifier
622 * @gpusvm: Pointer to the GPU SVM structure
623 * @notifier: Pointer to the GPU SVM notifier structure
624 *
625 * This function frees the GPU SVM notifier structure.
626 */
627static void drm_gpusvm_notifier_free(struct drm_gpusvm *gpusvm,
628 struct drm_gpusvm_notifier *notifier)
629{
630 WARN_ON(!RB_EMPTY_ROOT(&notifier->root.rb_root));
631
632 if (gpusvm->ops->notifier_free)
633 gpusvm->ops->notifier_free(notifier);
634 else
635 kfree(notifier);
636}
637
638/**
639 * to_drm_gpusvm_range() - retrieve the container struct for a given rbtree node
640 * @node: a pointer to the rbtree node embedded within a drm_gpusvm_range struct
641 *
642 * Return: A pointer to the containing drm_gpusvm_range structure.
643 */
644static struct drm_gpusvm_range *to_drm_gpusvm_range(struct rb_node *node)
645{
646 return container_of(node, struct drm_gpusvm_range, itree.rb);
647}
648
649/**
650 * drm_gpusvm_range_insert() - Insert GPU SVM range
651 * @notifier: Pointer to the GPU SVM notifier structure
652 * @range: Pointer to the GPU SVM range structure
653 *
654 * This function inserts the GPU SVM range into the notifier RB tree and list.
655 */
656static void drm_gpusvm_range_insert(struct drm_gpusvm_notifier *notifier,
657 struct drm_gpusvm_range *range)
658{
659 struct rb_node *node;
660 struct list_head *head;
661
662 drm_gpusvm_notifier_lock(notifier->gpusvm);
663 interval_tree_insert(&range->itree, &notifier->root);
664
665 node = rb_prev(&range->itree.rb);
666 if (node)
667 head = &(to_drm_gpusvm_range(node))->entry;
668 else
669 head = &notifier->range_list;
670
671 list_add(&range->entry, head);
672 drm_gpusvm_notifier_unlock(notifier->gpusvm);
673}
674
675/**
676 * __drm_gpusvm_range_remove() - Remove GPU SVM range
677 * @notifier: Pointer to the GPU SVM notifier structure
678 * @range: Pointer to the GPU SVM range structure
679 *
680 * This macro removes the GPU SVM range from the notifier RB tree and list.
681 */
682static void __drm_gpusvm_range_remove(struct drm_gpusvm_notifier *notifier,
683 struct drm_gpusvm_range *range)
684{
685 interval_tree_remove(&range->itree, &notifier->root);
686 list_del(&range->entry);
687}
688
689/**
690 * drm_gpusvm_range_alloc() - Allocate GPU SVM range
691 * @gpusvm: Pointer to the GPU SVM structure
692 * @notifier: Pointer to the GPU SVM notifier structure
693 * @fault_addr: Fault address
694 * @chunk_size: Chunk size
695 * @migrate_devmem: Flag indicating whether to migrate device memory
696 *
697 * This function allocates and initializes the GPU SVM range structure.
698 *
699 * Return: Pointer to the allocated GPU SVM range on success, ERR_PTR() on failure.
700 */
701static struct drm_gpusvm_range *
702drm_gpusvm_range_alloc(struct drm_gpusvm *gpusvm,
703 struct drm_gpusvm_notifier *notifier,
704 unsigned long fault_addr, unsigned long chunk_size,
705 bool migrate_devmem)
706{
707 struct drm_gpusvm_range *range;
708
709 if (gpusvm->ops->range_alloc)
710 range = gpusvm->ops->range_alloc(gpusvm);
711 else
712 range = kzalloc_obj(*range);
713
714 if (!range)
715 return ERR_PTR(-ENOMEM);
716
717 kref_init(&range->refcount);
718 range->gpusvm = gpusvm;
719 range->notifier = notifier;
720 range->itree.start = ALIGN_DOWN(fault_addr, chunk_size);
721 range->itree.last = ALIGN(fault_addr + 1, chunk_size) - 1;
722 INIT_LIST_HEAD(&range->entry);
723 range->flags.migrate_devmem = migrate_devmem ? 1 : 0;
724
725 return range;
726}
727
728/**
729 * drm_gpusvm_hmm_pfn_to_order() - Get the largest CPU mapping order.
730 * @hmm_pfn: The current hmm_pfn.
731 * @hmm_pfn_index: Index of the @hmm_pfn within the pfn array.
732 * @npages: Number of pages within the pfn array i.e the hmm range size.
733 *
734 * To allow skipping PFNs with the same flags (like when they belong to
735 * the same huge PTE) when looping over the pfn array, take a given a hmm_pfn,
736 * and return the largest order that will fit inside the CPU PTE, but also
737 * crucially accounting for the original hmm range boundaries.
738 *
739 * Return: The largest order that will safely fit within the size of the hmm_pfn
740 * CPU PTE.
741 */
742static unsigned int drm_gpusvm_hmm_pfn_to_order(unsigned long hmm_pfn,
743 unsigned long hmm_pfn_index,
744 unsigned long npages)
745{
746 unsigned long size;
747
748 size = 1UL << hmm_pfn_to_map_order(hmm_pfn);
749 size -= (hmm_pfn & ~HMM_PFN_FLAGS) & (size - 1);
750 hmm_pfn_index += size;
751 if (hmm_pfn_index > npages)
752 size -= (hmm_pfn_index - npages);
753
754 return ilog2(size);
755}
756
757/**
758 * drm_gpusvm_check_pages() - Check pages
759 * @gpusvm: Pointer to the GPU SVM structure
760 * @notifier: Pointer to the GPU SVM notifier structure
761 * @start: Start address
762 * @end: End address
763 * @dev_private_owner: The device private page owner
764 *
765 * Check if pages between start and end have been faulted in on the CPU. Use to
766 * prevent migration of pages without CPU backing store.
767 *
768 * Return: True if pages have been faulted into CPU, False otherwise
769 */
770static bool drm_gpusvm_check_pages(struct drm_gpusvm *gpusvm,
771 struct drm_gpusvm_notifier *notifier,
772 unsigned long start, unsigned long end,
773 void *dev_private_owner)
774{
775 struct hmm_range hmm_range = {
776 .default_flags = 0,
777 .notifier = &notifier->notifier,
778 .start = start,
779 .end = end,
780 .dev_private_owner = dev_private_owner,
781 };
782 unsigned long timeout =
783 jiffies + msecs_to_jiffies(HMM_RANGE_DEFAULT_TIMEOUT);
784 unsigned long *pfns;
785 unsigned long npages = npages_in_range(start, end);
786 int err, i;
787
788 mmap_assert_locked(gpusvm->mm);
789
790 pfns = kvmalloc_array(npages, sizeof(*pfns), GFP_KERNEL);
791 if (!pfns)
792 return false;
793
794 hmm_range.notifier_seq = mmu_interval_read_begin(&notifier->notifier);
795 hmm_range.hmm_pfns = pfns;
796
797 while (true) {
798 err = hmm_range_fault(&hmm_range);
799 if (err == -EBUSY) {
800 if (time_after(jiffies, timeout))
801 break;
802
803 hmm_range.notifier_seq =
804 mmu_interval_read_begin(&notifier->notifier);
805 continue;
806 }
807 break;
808 }
809 if (err)
810 goto err_free;
811
812 for (i = 0; i < npages;) {
813 if (!(pfns[i] & HMM_PFN_VALID)) {
814 err = -EFAULT;
815 goto err_free;
816 }
817 i += 0x1 << drm_gpusvm_hmm_pfn_to_order(pfns[i], i, npages);
818 }
819
820err_free:
821 kvfree(pfns);
822 return err ? false : true;
823}
824
825/**
826 * drm_gpusvm_scan_mm() - Check the migration state of a drm_gpusvm_range
827 * @range: Pointer to the struct drm_gpusvm_range to check.
828 * @dev_private_owner: The struct dev_private_owner to use to determine
829 * compatible device-private pages.
830 * @pagemap: The struct dev_pagemap pointer to use for pagemap-specific
831 * checks.
832 *
833 * Scan the CPU address space corresponding to @range and return the
834 * current migration state. Note that the result may be invalid as
835 * soon as the function returns. It's an advisory check.
836 *
837 * TODO: Bail early and call hmm_range_fault() for subranges.
838 *
839 * Return: See &enum drm_gpusvm_scan_result.
840 */
841enum drm_gpusvm_scan_result drm_gpusvm_scan_mm(struct drm_gpusvm_range *range,
842 void *dev_private_owner,
843 const struct dev_pagemap *pagemap)
844{
845 struct mmu_interval_notifier *notifier = &range->notifier->notifier;
846 unsigned long start = drm_gpusvm_range_start(range);
847 unsigned long end = drm_gpusvm_range_end(range);
848 struct hmm_range hmm_range = {
849 .default_flags = 0,
850 .notifier = notifier,
851 .start = start,
852 .end = end,
853 .dev_private_owner = dev_private_owner,
854 };
855 unsigned long timeout = msecs_to_jiffies(HMM_RANGE_DEFAULT_TIMEOUT);
856 enum drm_gpusvm_scan_result state = DRM_GPUSVM_SCAN_UNPOPULATED, new_state;
857 unsigned long *pfns;
858 unsigned long npages = npages_in_range(start, end);
859 const struct dev_pagemap *other = NULL;
860 int err, i;
861
862 pfns = kvcalloc(npages, sizeof(*pfns), GFP_KERNEL);
863 if (!pfns)
864 return DRM_GPUSVM_SCAN_UNPOPULATED;
865
866 hmm_range.hmm_pfns = pfns;
867
868retry:
869 err = hmm_range_fault_unlocked_timeout(&hmm_range, timeout);
870 if (err)
871 goto err_free;
872
873 drm_gpusvm_notifier_lock(range->gpusvm);
874 if (mmu_interval_read_retry(notifier, hmm_range.notifier_seq)) {
875 drm_gpusvm_notifier_unlock(range->gpusvm);
876 goto retry;
877 }
878
879 for (i = 0; i < npages;) {
880 struct page *page;
881 const struct dev_pagemap *cur = NULL;
882
883 if (!(pfns[i] & HMM_PFN_VALID)) {
884 state = DRM_GPUSVM_SCAN_UNPOPULATED;
885 break;
886 }
887
888 page = hmm_pfn_to_page(pfns[i]);
889 if (is_device_private_page(page) ||
890 is_device_coherent_page(page))
891 cur = page_pgmap(page);
892
893 if (cur == pagemap) {
894 new_state = DRM_GPUSVM_SCAN_EQUAL;
895 } else if (cur && (cur == other || !other)) {
896 new_state = DRM_GPUSVM_SCAN_OTHER;
897 other = cur;
898 } else if (cur) {
899 new_state = DRM_GPUSVM_SCAN_MIXED_DEVICE;
900 } else {
901 new_state = DRM_GPUSVM_SCAN_SYSTEM;
902 }
903
904 /*
905 * TODO: Could use an array for state
906 * transitions, and caller might want it
907 * to bail early for some results.
908 */
909 if (state == DRM_GPUSVM_SCAN_UNPOPULATED) {
910 state = new_state;
911 } else if (state != new_state) {
912 if (new_state == DRM_GPUSVM_SCAN_SYSTEM ||
913 state == DRM_GPUSVM_SCAN_SYSTEM)
914 state = DRM_GPUSVM_SCAN_MIXED;
915 else if (state != DRM_GPUSVM_SCAN_MIXED)
916 state = DRM_GPUSVM_SCAN_MIXED_DEVICE;
917 }
918
919 i += 1ul << drm_gpusvm_hmm_pfn_to_order(pfns[i], i, npages);
920 }
921
922 drm_gpusvm_notifier_unlock(range->gpusvm);
923
924err_free:
925 kvfree(pfns);
926 return state;
927}
928EXPORT_SYMBOL(drm_gpusvm_scan_mm);
929
930/**
931 * drm_gpusvm_range_chunk_size() - Determine chunk size for GPU SVM range
932 * @gpusvm: Pointer to the GPU SVM structure
933 * @notifier: Pointer to the GPU SVM notifier structure
934 * @vas: Pointer to the virtual memory area structure
935 * @fault_addr: Fault address
936 * @gpuva_start: Start address of GPUVA which mirrors CPU
937 * @gpuva_end: End address of GPUVA which mirrors CPU
938 * @check_pages_threshold: Check CPU pages for present threshold
939 * @dev_private_owner: The device private page owner
940 *
941 * This function determines the chunk size for the GPU SVM range based on the
942 * fault address, GPU SVM chunk sizes, existing GPU SVM ranges, and the virtual
943 * memory area boundaries.
944 *
945 * Return: Chunk size on success, LONG_MAX on failure.
946 */
947static unsigned long
948drm_gpusvm_range_chunk_size(struct drm_gpusvm *gpusvm,
949 struct drm_gpusvm_notifier *notifier,
950 struct vm_area_struct *vas,
951 unsigned long fault_addr,
952 unsigned long gpuva_start,
953 unsigned long gpuva_end,
954 unsigned long check_pages_threshold,
955 void *dev_private_owner)
956{
957 unsigned long start, end;
958 int i = 0;
959
960retry:
961 for (; i < gpusvm->num_chunks; ++i) {
962 start = ALIGN_DOWN(fault_addr, gpusvm->chunk_sizes[i]);
963 end = ALIGN(fault_addr + 1, gpusvm->chunk_sizes[i]);
964
965 if (start >= vas->vm_start && end <= vas->vm_end &&
966 start >= drm_gpusvm_notifier_start(notifier) &&
967 end <= drm_gpusvm_notifier_end(notifier) &&
968 start >= gpuva_start && end <= gpuva_end)
969 break;
970 }
971
972 if (i == gpusvm->num_chunks)
973 return LONG_MAX;
974
975 /*
976 * If allocation more than page, ensure not to overlap with existing
977 * ranges.
978 */
979 if (end - start != SZ_4K) {
980 struct drm_gpusvm_range *range;
981
982 range = drm_gpusvm_range_find(notifier, start, end);
983 if (range) {
984 ++i;
985 goto retry;
986 }
987
988 /*
989 * XXX: Only create range on pages CPU has faulted in. Without
990 * this check, or prefault, on BMG 'xe_exec_system_allocator --r
991 * process-many-malloc' fails. In the failure case, each process
992 * mallocs 16k but the CPU VMA is ~128k which results in 64k SVM
993 * ranges. When migrating the SVM ranges, some processes fail in
994 * drm_pagemap_migrate_to_devmem with 'migrate.cpages != npages'
995 * and then upon drm_gpusvm_get_pages device pages from
996 * other processes are collected + faulted in which creates all
997 * sorts of problems. Unsure exactly how this happening, also
998 * problem goes away if 'xe_exec_system_allocator --r
999 * process-many-malloc' mallocs at least 64k at a time.
1000 */
1001 if (end - start <= check_pages_threshold &&
1002 !drm_gpusvm_check_pages(gpusvm, notifier, start, end, dev_private_owner)) {
1003 ++i;
1004 goto retry;
1005 }
1006 }
1007
1008 return end - start;
1009}
1010
1011#ifdef CONFIG_LOCKDEP
1012/**
1013 * drm_gpusvm_driver_lock_held() - Assert GPU SVM driver lock is held
1014 * @gpusvm: Pointer to the GPU SVM structure.
1015 *
1016 * Ensure driver lock is held.
1017 */
1018static void drm_gpusvm_driver_lock_held(struct drm_gpusvm *gpusvm)
1019{
1020 if ((gpusvm)->lock_dep_map)
1021 lockdep_assert(lock_is_held_type((gpusvm)->lock_dep_map, 0));
1022}
1023#else
1024static void drm_gpusvm_driver_lock_held(struct drm_gpusvm *gpusvm)
1025{
1026}
1027#endif
1028
1029/**
1030 * drm_gpusvm_find_vma_start() - Find start address for first VMA in range
1031 * @gpusvm: Pointer to the GPU SVM structure
1032 * @start: The inclusive start user address.
1033 * @end: The exclusive end user address.
1034 *
1035 * Returns: The start address of first VMA within the provided range,
1036 * ULONG_MAX otherwise. Assumes start_addr < end_addr.
1037 */
1038unsigned long
1039drm_gpusvm_find_vma_start(struct drm_gpusvm *gpusvm,
1040 unsigned long start,
1041 unsigned long end)
1042{
1043 struct mm_struct *mm = gpusvm->mm;
1044 struct vm_area_struct *vma;
1045 unsigned long addr = ULONG_MAX;
1046
1047 if (!mmget_not_zero(mm))
1048 return addr;
1049
1050 mmap_read_lock(mm);
1051
1052 vma = find_vma_intersection(mm, start, end);
1053 if (vma)
1054 addr = vma->vm_start;
1055
1056 mmap_read_unlock(mm);
1057 mmput(mm);
1058
1059 return addr;
1060}
1061EXPORT_SYMBOL_GPL(drm_gpusvm_find_vma_start);
1062
1063/**
1064 * drm_gpusvm_range_find_or_insert() - Find or insert GPU SVM range
1065 * @gpusvm: Pointer to the GPU SVM structure
1066 * @fault_addr: Fault address
1067 * @gpuva_start: Start address of GPUVA which mirrors CPU
1068 * @gpuva_end: End address of GPUVA which mirrors CPU
1069 * @ctx: GPU SVM context
1070 *
1071 * This function finds or inserts a newly allocated a GPU SVM range based on the
1072 * fault address. Caller must hold a lock to protect range lookup and insertion.
1073 *
1074 * Return: Pointer to the GPU SVM range on success, ERR_PTR() on failure.
1075 */
1076struct drm_gpusvm_range *
1077drm_gpusvm_range_find_or_insert(struct drm_gpusvm *gpusvm,
1078 unsigned long fault_addr,
1079 unsigned long gpuva_start,
1080 unsigned long gpuva_end,
1081 const struct drm_gpusvm_ctx *ctx)
1082{
1083 struct drm_gpusvm_notifier *notifier;
1084 struct drm_gpusvm_range *range;
1085 struct mm_struct *mm = gpusvm->mm;
1086 struct vm_area_struct *vas;
1087 bool notifier_alloc = false;
1088 unsigned long chunk_size;
1089 int err;
1090 bool migrate_devmem;
1091
1092 drm_gpusvm_driver_lock_held(gpusvm);
1093
1094 if (fault_addr < gpusvm->mm_start ||
1095 fault_addr > gpusvm->mm_start + gpusvm->mm_range)
1096 return ERR_PTR(-EINVAL);
1097
1098 if (!mmget_not_zero(mm))
1099 return ERR_PTR(-EFAULT);
1100
1101 notifier = drm_gpusvm_notifier_find(gpusvm, fault_addr, fault_addr + 1);
1102 if (!notifier) {
1103 notifier = drm_gpusvm_notifier_alloc(gpusvm, fault_addr);
1104 if (IS_ERR(notifier)) {
1105 err = PTR_ERR(notifier);
1106 goto err_mmunlock;
1107 }
1108 notifier_alloc = true;
1109 err = mmu_interval_notifier_insert(&notifier->notifier,
1110 mm,
1111 drm_gpusvm_notifier_start(notifier),
1112 drm_gpusvm_notifier_size(notifier),
1113 &drm_gpusvm_notifier_ops);
1114 if (err)
1115 goto err_notifier;
1116 }
1117
1118 mmap_read_lock(mm);
1119
1120 vas = vma_lookup(mm, fault_addr);
1121 if (!vas) {
1122 err = -ENOENT;
1123 goto err_notifier_remove;
1124 }
1125
1126 if (!ctx->read_only && !(vas->vm_flags & VM_WRITE)) {
1127 err = -EPERM;
1128 goto err_notifier_remove;
1129 }
1130
1131 if (vas->vm_flags & (VM_IO | VM_PFNMAP)) {
1132 err = -EIO;
1133 goto err_notifier_remove;
1134 }
1135
1136 range = drm_gpusvm_range_find(notifier, fault_addr, fault_addr + 1);
1137 if (range)
1138 goto out_mmunlock;
1139 /*
1140 * XXX: Short-circuiting migration based on migrate_vma_* current
1141 * limitations. If/when migrate_vma_* add more support, this logic will
1142 * have to change.
1143 */
1144 migrate_devmem = ctx->devmem_possible &&
1145 vma_is_anonymous(vas) && !is_vm_hugetlb_page(vas);
1146
1147 chunk_size = drm_gpusvm_range_chunk_size(gpusvm, notifier, vas,
1148 fault_addr, gpuva_start,
1149 gpuva_end,
1150 ctx->check_pages_threshold,
1151 ctx->device_private_page_owner);
1152 if (chunk_size == LONG_MAX) {
1153 err = -EINVAL;
1154 goto err_notifier_remove;
1155 }
1156
1157 range = drm_gpusvm_range_alloc(gpusvm, notifier, fault_addr, chunk_size,
1158 migrate_devmem);
1159 if (IS_ERR(range)) {
1160 err = PTR_ERR(range);
1161 goto err_notifier_remove;
1162 }
1163
1164 drm_gpusvm_range_insert(notifier, range);
1165 if (notifier_alloc)
1166 drm_gpusvm_notifier_insert(gpusvm, notifier);
1167
1168out_mmunlock:
1169 mmap_read_unlock(mm);
1170 mmput(mm);
1171
1172 return range;
1173
1174err_notifier_remove:
1175 mmap_read_unlock(mm);
1176 if (notifier_alloc)
1177 mmu_interval_notifier_remove(&notifier->notifier);
1178err_notifier:
1179 if (notifier_alloc)
1180 drm_gpusvm_notifier_free(gpusvm, notifier);
1181err_mmunlock:
1182 mmput(mm);
1183 return ERR_PTR(err);
1184}
1185EXPORT_SYMBOL_GPL(drm_gpusvm_range_find_or_insert);
1186
1187/**
1188 * __drm_gpusvm_unmap_pages() - Unmap pages associated with GPU SVM pages (internal)
1189 * @gpusvm: Pointer to the GPU SVM structure
1190 * @svm_pages: Pointer to the GPU SVM pages structure
1191 * @npages: Number of pages to unmap
1192 *
1193 * This function unmap pages associated with a GPU SVM pages struct. Assumes and
1194 * asserts correct locking is in place when called.
1195 */
1196static void __drm_gpusvm_unmap_pages(struct drm_gpusvm *gpusvm,
1197 struct drm_gpusvm_pages *svm_pages,
1198 unsigned long npages)
1199{
1200 struct drm_pagemap *dpagemap = svm_pages->dpagemap;
1201 struct device *dev;
1202 unsigned long i, j;
1203
1204 lockdep_assert_held(&gpusvm->notifier_lock);
1205
1206 if (!svm_pages->drm)
1207 return;
1208
1209 dev = svm_pages->drm->dev;
1210
1211 if (svm_pages->flags.has_dma_mapping) {
1212 struct drm_gpusvm_pages_flags flags = {
1213 .__flags = svm_pages->flags.__flags,
1214 };
1215 bool use_iova = dma_use_iova(&svm_pages->state);
1216
1217 /*
1218 * IOVA is reserved for the whole range but only the linked
1219 * system pages (state_offset bytes) need unlinking; free the
1220 * entire reservation to avoid leaking the device-page part.
1221 * On the error path state_offset is 0, so just free it.
1222 */
1223 if (use_iova) {
1224 if (svm_pages->state_offset)
1225 dma_iova_unlink(dev, &svm_pages->state, 0,
1226 svm_pages->state_offset,
1227 svm_pages->dma_addr[0].dir, 0);
1228 dma_iova_free(dev, &svm_pages->state);
1229 }
1230
1231 for (i = 0, j = 0; i < npages; j++) {
1232 struct drm_pagemap_addr *addr = &svm_pages->dma_addr[j];
1233
1234 if (addr->proto == DRM_INTERCONNECT_SYSTEM) {
1235 /*
1236 * Linked IOVA pages were already torn down by
1237 * the dma_iova_unlink()/dma_iova_free() above;
1238 * only the non-IOVA mappings need unmap here.
1239 */
1240 if (!use_iova)
1241 dma_unmap_page(dev,
1242 addr->addr,
1243 PAGE_SIZE << addr->order,
1244 addr->dir);
1245 } else if (dpagemap && dpagemap->ops->device_unmap)
1246 dpagemap->ops->device_unmap(dpagemap,
1247 dev, addr);
1248 i += 1 << addr->order;
1249 }
1250
1251 /* WRITE_ONCE pairs with READ_ONCE for opportunistic checks */
1252 flags.has_devmem_pages = false;
1253 flags.has_dma_mapping = false;
1254 WRITE_ONCE(svm_pages->flags.__flags, flags.__flags);
1255
1256 drm_pagemap_put(svm_pages->dpagemap);
1257 svm_pages->dpagemap = NULL;
1258 }
1259}
1260
1261/**
1262 * __drm_gpusvm_free_pages() - Free dma array associated with GPU SVM pages
1263 * @gpusvm: Pointer to the GPU SVM structure
1264 * @svm_pages: Pointer to the GPU SVM pages structure
1265 *
1266 * This function frees the dma address array associated with a GPU SVM range.
1267 */
1268static void __drm_gpusvm_free_pages(struct drm_gpusvm *gpusvm,
1269 struct drm_gpusvm_pages *svm_pages)
1270{
1271 lockdep_assert_held(&gpusvm->notifier_lock);
1272
1273 if (svm_pages->dma_addr) {
1274 kvfree(svm_pages->dma_addr);
1275 svm_pages->dma_addr = NULL;
1276 }
1277}
1278
1279/**
1280 * drm_gpusvm_free_pages() - Free dma-mapping associated with GPU SVM pages
1281 * struct
1282 * @gpusvm: Pointer to the GPU SVM structure
1283 * @svm_pages: Pointer to the GPU SVM pages structure
1284 * @npages: Number of mapped pages
1285 *
1286 * This function unmaps and frees the dma address array associated with a GPU
1287 * SVM pages struct.
1288 */
1289void drm_gpusvm_free_pages(struct drm_gpusvm *gpusvm,
1290 struct drm_gpusvm_pages *svm_pages,
1291 unsigned long npages)
1292{
1293 drm_gpusvm_notifier_lock(gpusvm);
1294 __drm_gpusvm_unmap_pages(gpusvm, svm_pages, npages);
1295 __drm_gpusvm_free_pages(gpusvm, svm_pages);
1296 drm_gpusvm_notifier_unlock(gpusvm);
1297}
1298EXPORT_SYMBOL_GPL(drm_gpusvm_free_pages);
1299
1300/**
1301 * drm_gpusvm_range_remove() - Remove GPU SVM range
1302 * @gpusvm: Pointer to the GPU SVM structure
1303 * @range: Pointer to the GPU SVM range to be removed
1304 *
1305 * This function removes the specified GPU SVM range and also removes the parent
1306 * GPU SVM notifier if no more ranges remain in the notifier. The caller must
1307 * hold a lock to protect range and notifier removal.
1308 *
1309 * This function does not unmap or free the drm_gpusvm_pages, the driver owns
1310 * that lifecycle. The caller must DMA unmap the range's pages before calling
1311 * this function, so a range is never removed from the MMU interval tree while
1312 * still DMA mapped. Typically the driver calls drm_gpusvm_unmap_pages() first.
1313 * And the range_free callback's drm_gpusvm_free_pages() is a final fallback safe
1314 * net.
1315 */
1316void drm_gpusvm_range_remove(struct drm_gpusvm *gpusvm,
1317 struct drm_gpusvm_range *range)
1318{
1319 struct drm_gpusvm_notifier *notifier;
1320
1321 drm_gpusvm_driver_lock_held(gpusvm);
1322
1323 notifier = drm_gpusvm_notifier_find(gpusvm,
1324 drm_gpusvm_range_start(range),
1325 drm_gpusvm_range_start(range) + 1);
1326 if (WARN_ON_ONCE(!notifier))
1327 return;
1328
1329 drm_gpusvm_notifier_lock(gpusvm);
1330 __drm_gpusvm_range_remove(notifier, range);
1331 drm_gpusvm_notifier_unlock(gpusvm);
1332
1333 drm_gpusvm_range_put(range);
1334
1335 if (RB_EMPTY_ROOT(&notifier->root.rb_root)) {
1336 if (!notifier->flags.removed)
1337 mmu_interval_notifier_remove(&notifier->notifier);
1338 drm_gpusvm_notifier_remove(gpusvm, notifier);
1339 drm_gpusvm_notifier_free(gpusvm, notifier);
1340 }
1341}
1342EXPORT_SYMBOL_GPL(drm_gpusvm_range_remove);
1343
1344/**
1345 * drm_gpusvm_range_get() - Get a reference to GPU SVM range
1346 * @range: Pointer to the GPU SVM range
1347 *
1348 * This function increments the reference count of the specified GPU SVM range.
1349 *
1350 * Return: Pointer to the GPU SVM range.
1351 */
1352struct drm_gpusvm_range *
1353drm_gpusvm_range_get(struct drm_gpusvm_range *range)
1354{
1355 kref_get(&range->refcount);
1356
1357 return range;
1358}
1359EXPORT_SYMBOL_GPL(drm_gpusvm_range_get);
1360
1361/**
1362 * drm_gpusvm_range_destroy() - Destroy GPU SVM range
1363 * @refcount: Pointer to the reference counter embedded in the GPU SVM range
1364 *
1365 * This function destroys the specified GPU SVM range when its reference count
1366 * reaches zero. If a custom range-free function is provided, it is invoked to
1367 * free the range; otherwise, the range is deallocated using kfree().
1368 */
1369static void drm_gpusvm_range_destroy(struct kref *refcount)
1370{
1371 struct drm_gpusvm_range *range =
1372 container_of(refcount, struct drm_gpusvm_range, refcount);
1373 struct drm_gpusvm *gpusvm = range->gpusvm;
1374
1375 if (gpusvm->ops->range_free)
1376 gpusvm->ops->range_free(range);
1377 else
1378 kfree(range);
1379}
1380
1381/**
1382 * drm_gpusvm_range_put() - Put a reference to GPU SVM range
1383 * @range: Pointer to the GPU SVM range
1384 *
1385 * This function decrements the reference count of the specified GPU SVM range
1386 * and frees it when the count reaches zero.
1387 */
1388void drm_gpusvm_range_put(struct drm_gpusvm_range *range)
1389{
1390 kref_put(&range->refcount, drm_gpusvm_range_destroy);
1391}
1392EXPORT_SYMBOL_GPL(drm_gpusvm_range_put);
1393
1394/**
1395 * drm_gpusvm_pages_valid() - GPU SVM range pages valid
1396 * @gpusvm: Pointer to the GPU SVM structure
1397 * @svm_pages: Pointer to the GPU SVM pages structure
1398 *
1399 * This function determines if a GPU SVM range pages are valid. Expected be
1400 * called holding gpusvm->notifier_lock and as the last step before committing a
1401 * GPU binding. This is akin to a notifier seqno check in the HMM documentation
1402 * but due to wider notifiers (i.e., notifiers which span multiple ranges) this
1403 * function is required for finer grained checking (i.e., per range) if pages
1404 * are valid.
1405 *
1406 * Return: True if GPU SVM range has valid pages, False otherwise
1407 */
1408bool drm_gpusvm_pages_valid(struct drm_gpusvm *gpusvm,
1409 struct drm_gpusvm_pages *svm_pages)
1410{
1411 lockdep_assert_held(&gpusvm->notifier_lock);
1412
1413 return svm_pages->flags.has_devmem_pages || svm_pages->flags.has_dma_mapping;
1414}
1415EXPORT_SYMBOL_GPL(drm_gpusvm_pages_valid);
1416
1417/**
1418 * drm_gpusvm_pages_valid_unlocked() - GPU SVM pages valid unlocked
1419 * @gpusvm: Pointer to the GPU SVM structure
1420 * @svm_pages: Pointer to the GPU SVM pages structure
1421 *
1422 * This function determines if a GPU SVM pages are valid. Expected be called
1423 * without holding gpusvm->notifier_lock.
1424 *
1425 * Return: True if GPU SVM pages are valid, False otherwise
1426 */
1427static bool drm_gpusvm_pages_valid_unlocked(struct drm_gpusvm *gpusvm,
1428 struct drm_gpusvm_pages *svm_pages)
1429{
1430 bool pages_valid;
1431
1432 if (!svm_pages->dma_addr)
1433 return false;
1434
1435 drm_gpusvm_notifier_lock(gpusvm);
1436 pages_valid = drm_gpusvm_pages_valid(gpusvm, svm_pages);
1437 if (!pages_valid)
1438 __drm_gpusvm_free_pages(gpusvm, svm_pages);
1439 drm_gpusvm_notifier_unlock(gpusvm);
1440
1441 return pages_valid;
1442}
1443
1444/**
1445 * drm_gpusvm_get_pages() - Get pages and populate GPU SVM pages struct
1446 * @gpusvm: Pointer to the GPU SVM structure
1447 * @svm_pages: The SVM pages to populate. This will contain the dma-addresses
1448 * @mm: The mm corresponding to the CPU range
1449 * @notifier: The corresponding notifier for the given CPU range
1450 * @pages_start: Start CPU address for the pages
1451 * @pages_end: End CPU address for the pages (exclusive)
1452 * @ctx: GPU SVM context
1453 *
1454 * This function gets and maps pages for CPU range and ensures they are
1455 * mapped for DMA access.
1456 *
1457 * Return: 0 on success, negative error code on failure.
1458 */
1459int drm_gpusvm_get_pages(struct drm_gpusvm *gpusvm,
1460 struct drm_gpusvm_pages *svm_pages,
1461 struct mm_struct *mm,
1462 struct mmu_interval_notifier *notifier,
1463 unsigned long pages_start, unsigned long pages_end,
1464 const struct drm_gpusvm_ctx *ctx)
1465{
1466 struct hmm_range hmm_range = {
1467 .default_flags = HMM_PFN_REQ_FAULT | (ctx->read_only ? 0 :
1468 HMM_PFN_REQ_WRITE),
1469 .notifier = notifier,
1470 .start = pages_start,
1471 .end = pages_end,
1472 .dev_private_owner = ctx->device_private_page_owner,
1473 };
1474 void *zdd;
1475 unsigned long timeout =
1476 jiffies + msecs_to_jiffies(HMM_RANGE_DEFAULT_TIMEOUT);
1477 unsigned long remaining;
1478 unsigned long i, j;
1479 unsigned long npages = npages_in_range(pages_start, pages_end);
1480 unsigned long num_dma_mapped;
1481 unsigned int order = 0;
1482 unsigned long *pfns;
1483 int err = 0;
1484 struct dev_pagemap *pagemap;
1485 struct drm_pagemap *dpagemap;
1486 struct drm_gpusvm_pages_flags flags;
1487 enum dma_data_direction dma_dir = ctx->read_only ? DMA_TO_DEVICE :
1488 DMA_BIDIRECTIONAL;
1489 struct dma_iova_state *state = &svm_pages->state;
1490
1491 if (!svm_pages->drm)
1492 return -EINVAL;
1493
1494retry:
1495 remaining = timeout - jiffies;
1496
1497 if (time_after_eq(jiffies, timeout))
1498 return -EBUSY;
1499
1500 hmm_range.notifier_seq = mmu_interval_read_begin(notifier);
1501 if (drm_gpusvm_pages_valid_unlocked(gpusvm, svm_pages))
1502 goto set_seqno;
1503
1504 pfns = kvmalloc_array(npages, sizeof(*pfns), GFP_KERNEL);
1505 if (!pfns)
1506 return -ENOMEM;
1507
1508 if (!mmget_not_zero(mm)) {
1509 err = -EFAULT;
1510 goto err_free;
1511 }
1512
1513 hmm_range.hmm_pfns = pfns;
1514 err = hmm_range_fault_unlocked_timeout(&hmm_range, remaining);
1515 mmput(mm);
1516 if (err)
1517 goto err_free;
1518
1519 *state = (struct dma_iova_state){};
1520 svm_pages->state_offset = 0;
1521
1522map_pages:
1523 /*
1524 * Perform all dma mappings under the notifier lock to not
1525 * access freed pages. A notifier will either block on
1526 * the notifier lock or unmap dma.
1527 */
1528 drm_gpusvm_notifier_lock(gpusvm);
1529
1530 flags.__flags = svm_pages->flags.__flags;
1531 if (flags.unmapped) {
1532 drm_gpusvm_notifier_unlock(gpusvm);
1533 err = -EFAULT;
1534 goto err_free;
1535 }
1536
1537 if (mmu_interval_read_retry(notifier, hmm_range.notifier_seq)) {
1538 drm_gpusvm_notifier_unlock(gpusvm);
1539 kvfree(pfns);
1540 goto retry;
1541 }
1542
1543 if (!svm_pages->dma_addr) {
1544 /* Unlock and restart mapping to allocate memory. */
1545 drm_gpusvm_notifier_unlock(gpusvm);
1546 svm_pages->dma_addr =
1547 kvzalloc_objs(*svm_pages->dma_addr, npages);
1548 if (!svm_pages->dma_addr) {
1549 err = -ENOMEM;
1550 goto err_free;
1551 }
1552 goto map_pages;
1553 }
1554
1555 zdd = NULL;
1556 pagemap = NULL;
1557 num_dma_mapped = 0;
1558 for (i = 0, j = 0; i < npages; ++j) {
1559 struct page *page = hmm_pfn_to_page(pfns[i]);
1560
1561 order = drm_gpusvm_hmm_pfn_to_order(pfns[i], i, npages);
1562 if (is_device_private_page(page) ||
1563 is_device_coherent_page(page)) {
1564 struct drm_pagemap_zdd *__zdd =
1565 drm_pagemap_page_zone_device_data(page);
1566
1567 if (!ctx->allow_mixed &&
1568 zdd != __zdd && i > 0) {
1569 err = -EOPNOTSUPP;
1570 goto err_unmap;
1571 }
1572 zdd = __zdd;
1573 if (pagemap != page_pgmap(page)) {
1574 if (pagemap) {
1575 err = -EOPNOTSUPP;
1576 goto err_unmap;
1577 }
1578
1579 pagemap = page_pgmap(page);
1580 dpagemap = drm_pagemap_page_to_dpagemap(page);
1581 if (drm_WARN_ON(svm_pages->drm, !dpagemap)) {
1582 /*
1583 * Raced. This is not supposed to happen
1584 * since hmm_range_fault() should've migrated
1585 * this page to system.
1586 */
1587 err = -EAGAIN;
1588 goto err_unmap;
1589 }
1590
1591 /*
1592 * Set the dpagemap as soon as the first
1593 * device page is mapped so the err_unmap path
1594 * can device_unmap() the device mappings that
1595 * have already been created.
1596 */
1597 drm_pagemap_get(dpagemap);
1598 drm_pagemap_put(svm_pages->dpagemap);
1599 svm_pages->dpagemap = dpagemap;
1600 }
1601 svm_pages->dma_addr[j] =
1602 dpagemap->ops->device_map(dpagemap,
1603 svm_pages->drm->dev,
1604 page, order,
1605 dma_dir);
1606 if (dma_mapping_error(svm_pages->drm->dev,
1607 svm_pages->dma_addr[j].addr)) {
1608 err = -EFAULT;
1609 goto err_unmap;
1610 }
1611 } else {
1612 dma_addr_t addr;
1613
1614 if (is_zone_device_page(page) ||
1615 (pagemap && !ctx->allow_mixed)) {
1616 err = -EOPNOTSUPP;
1617 goto err_unmap;
1618 }
1619
1620 if (ctx->devmem_only) {
1621 err = -EFAULT;
1622 goto err_unmap;
1623 }
1624
1625 if (!i)
1626 dma_iova_try_alloc(svm_pages->drm->dev, state,
1627 0, npages * PAGE_SIZE);
1628
1629 if (dma_use_iova(state)) {
1630 err = dma_iova_link(svm_pages->drm->dev, state,
1631 hmm_pfn_to_phys(pfns[i]),
1632 svm_pages->state_offset,
1633 PAGE_SIZE << order,
1634 dma_dir, 0);
1635 if (err)
1636 goto err_unmap;
1637
1638 addr = state->addr + svm_pages->state_offset;
1639 svm_pages->state_offset += PAGE_SIZE << order;
1640 } else {
1641 addr = dma_map_page(svm_pages->drm->dev,
1642 page, 0,
1643 PAGE_SIZE << order,
1644 dma_dir);
1645 if (dma_mapping_error(svm_pages->drm->dev, addr)) {
1646 err = -EFAULT;
1647 goto err_unmap;
1648 }
1649 }
1650
1651 svm_pages->dma_addr[j] = drm_pagemap_addr_encode
1652 (addr, DRM_INTERCONNECT_SYSTEM, order,
1653 dma_dir);
1654 }
1655 i += 1 << order;
1656 num_dma_mapped = i;
1657 flags.has_dma_mapping = true;
1658 }
1659
1660 if (dma_use_iova(state)) {
1661 err = dma_iova_sync(svm_pages->drm->dev, state, 0,
1662 svm_pages->state_offset);
1663 if (err)
1664 goto err_unmap;
1665 }
1666
1667 if (pagemap)
1668 flags.has_devmem_pages = true;
1669
1670 /* WRITE_ONCE pairs with READ_ONCE for opportunistic checks */
1671 WRITE_ONCE(svm_pages->flags.__flags, flags.__flags);
1672
1673 drm_gpusvm_notifier_unlock(gpusvm);
1674 kvfree(pfns);
1675set_seqno:
1676 svm_pages->notifier_seq = hmm_range.notifier_seq;
1677
1678 return 0;
1679
1680err_unmap:
1681 svm_pages->flags.has_dma_mapping = true;
1682 __drm_gpusvm_unmap_pages(gpusvm, svm_pages, num_dma_mapped);
1683 drm_gpusvm_notifier_unlock(gpusvm);
1684err_free:
1685 kvfree(pfns);
1686 if (err == -EAGAIN)
1687 goto retry;
1688 return err;
1689}
1690EXPORT_SYMBOL_GPL(drm_gpusvm_get_pages);
1691
1692/**
1693 * drm_gpusvm_unmap_pages() - Unmap GPU svm pages
1694 * @gpusvm: Pointer to the GPU SVM structure
1695 * @svm_pages: Pointer to the GPU SVM pages structure
1696 * @npages: Number of pages in @svm_pages.
1697 * @ctx: GPU SVM context
1698 *
1699 * This function unmaps pages associated with a GPU SVM pages struct. If
1700 * @in_notifier is set, it is assumed that gpusvm->notifier_lock is held in
1701 * write mode; if it is clear, it acquires gpusvm->notifier_lock in read mode.
1702 * Must be called in the invalidate() callback of the corresponding notifier for
1703 * IOMMU security model.
1704 */
1705void drm_gpusvm_unmap_pages(struct drm_gpusvm *gpusvm,
1706 struct drm_gpusvm_pages *svm_pages,
1707 unsigned long npages,
1708 const struct drm_gpusvm_ctx *ctx)
1709{
1710 if (ctx->in_notifier)
1711 lockdep_assert_held_write(&gpusvm->notifier_lock);
1712 else
1713 drm_gpusvm_notifier_lock(gpusvm);
1714
1715 __drm_gpusvm_unmap_pages(gpusvm, svm_pages, npages);
1716
1717 if (!ctx->in_notifier)
1718 drm_gpusvm_notifier_unlock(gpusvm);
1719}
1720EXPORT_SYMBOL_GPL(drm_gpusvm_unmap_pages);
1721
1722/**
1723 * drm_gpusvm_range_evict() - Evict GPU SVM range
1724 * @gpusvm: Pointer to the GPU SVM structure
1725 * @range: Pointer to the GPU SVM range to be removed
1726 *
1727 * This function evicts the specified GPU SVM range.
1728 *
1729 * Return: 0 on success, a negative error code on failure.
1730 */
1731int drm_gpusvm_range_evict(struct drm_gpusvm *gpusvm,
1732 struct drm_gpusvm_range *range)
1733{
1734 struct mmu_interval_notifier *notifier = &range->notifier->notifier;
1735 struct hmm_range hmm_range = {
1736 .default_flags = HMM_PFN_REQ_FAULT,
1737 .notifier = notifier,
1738 .start = drm_gpusvm_range_start(range),
1739 .end = drm_gpusvm_range_end(range),
1740 .dev_private_owner = NULL,
1741 };
1742 unsigned long timeout = msecs_to_jiffies(HMM_RANGE_DEFAULT_TIMEOUT);
1743 unsigned long *pfns;
1744 unsigned long npages = npages_in_range(drm_gpusvm_range_start(range),
1745 drm_gpusvm_range_end(range));
1746 int err = 0;
1747 struct mm_struct *mm = gpusvm->mm;
1748
1749 if (!mmget_not_zero(mm))
1750 return -EFAULT;
1751
1752 pfns = kvmalloc_array(npages, sizeof(*pfns), GFP_KERNEL);
1753 if (!pfns) {
1754 mmput(mm);
1755 return -ENOMEM;
1756 }
1757
1758 hmm_range.hmm_pfns = pfns;
1759 err = hmm_range_fault_unlocked_timeout(&hmm_range, timeout);
1760
1761 kvfree(pfns);
1762 mmput(mm);
1763
1764 return err == -EBUSY ? -ETIME : err;
1765}
1766EXPORT_SYMBOL_GPL(drm_gpusvm_range_evict);
1767
1768/**
1769 * drm_gpusvm_has_mapping() - Check if GPU SVM has mapping for the given address range
1770 * @gpusvm: Pointer to the GPU SVM structure.
1771 * @start: Start address
1772 * @end: End address
1773 *
1774 * Return: True if GPU SVM has mapping, False otherwise
1775 */
1776bool drm_gpusvm_has_mapping(struct drm_gpusvm *gpusvm, unsigned long start,
1777 unsigned long end)
1778{
1779 struct drm_gpusvm_notifier *notifier;
1780
1781 drm_gpusvm_for_each_notifier(notifier, gpusvm, start, end) {
1782 struct drm_gpusvm_range *range = NULL;
1783
1784 drm_gpusvm_for_each_range(range, notifier, start, end)
1785 return true;
1786 }
1787
1788 return false;
1789}
1790EXPORT_SYMBOL_GPL(drm_gpusvm_has_mapping);
1791
1792/**
1793 * drm_gpusvm_range_set_unmapped() - Mark a GPU SVM range as unmapped
1794 * @range: Pointer to the GPU SVM range structure.
1795 * @pages: Pointer to the GPU SVM pages structure(s).
1796 * @pages_count: Number of GPU SVM pages structure(s) passed in.
1797 * @mmu_range: Pointer to the MMU notifier range structure.
1798 *
1799 * This function marks a GPU SVM range as unmapped and sets the partial_unmap flag
1800 * if the range partially falls within the provided MMU notifier range.
1801 */
1802void drm_gpusvm_range_set_unmapped(struct drm_gpusvm_range *range,
1803 struct drm_gpusvm_pages *pages,
1804 unsigned int pages_count,
1805 const struct mmu_notifier_range *mmu_range)
1806{
1807 struct drm_gpusvm_range_flags range_flags = {
1808 .__flags = range->flags.__flags,
1809 };
1810 unsigned int i;
1811
1812 lockdep_assert_held_write(&range->gpusvm->notifier_lock);
1813
1814 range_flags.unmapped = true;
1815 for (i = 0; i < pages_count; ++i) {
1816 struct drm_gpusvm_pages_flags flags = {
1817 .__flags = pages[i].flags.__flags,
1818 };
1819
1820 flags.unmapped = true;
1821 /* WRITE_ONCE pairs with READ_ONCE for opportunistic checks */
1822 WRITE_ONCE(pages[i].flags.__flags, flags.__flags);
1823 }
1824 if (drm_gpusvm_range_start(range) < mmu_range->start ||
1825 drm_gpusvm_range_end(range) > mmu_range->end)
1826 range_flags.partial_unmap = true;
1827 /* WRITE_ONCE pairs with READ_ONCE for opportunistic checks */
1828 WRITE_ONCE(range->flags.__flags, range_flags.__flags);
1829}
1830EXPORT_SYMBOL_GPL(drm_gpusvm_range_set_unmapped);
1831
1832MODULE_DESCRIPTION("DRM GPUSVM");
1833MODULE_LICENSE("GPL");