projects/linux·blob master · refs

598 lines · 19.6 KiB

1/* SPDX-License-Identifier: GPL-2.0-only OR MIT */
2/*
3 * Copyright © 2024 Intel Corporation
4 */
5
6#ifndef __DRM_GPUSVM_H__
7#define __DRM_GPUSVM_H__
8
9#include <linux/dma-mapping.h>
10#include <linux/kref.h>
11#include <linux/interval_tree.h>
12#include <linux/mmu_notifier.h>
13
14struct dev_pagemap_ops;
15struct drm_device;
16struct drm_gpusvm;
17struct drm_gpusvm_notifier;
18struct drm_gpusvm_ops;
19struct drm_gpusvm_range;
20struct drm_pagemap;
21struct drm_pagemap_addr;
22
23/**
24 * struct drm_gpusvm_ops - Operations structure for GPU SVM
25 *
26 * This structure defines the operations for GPU Shared Virtual Memory (SVM).
27 * These operations are provided by the GPU driver to manage SVM ranges and
28 * notifiers.
29 */
30struct drm_gpusvm_ops {
31 /**
32 * @notifier_alloc: Allocate a GPU SVM notifier (optional)
33 *
34 * Allocate a GPU SVM notifier.
35 *
36 * Return: Pointer to the allocated GPU SVM notifier on success, NULL on failure.
37 */
38 struct drm_gpusvm_notifier *(*notifier_alloc)(void);
39
40 /**
41 * @notifier_free: Free a GPU SVM notifier (optional)
42 * @notifier: Pointer to the GPU SVM notifier to be freed
43 *
44 * Free a GPU SVM notifier.
45 */
46 void (*notifier_free)(struct drm_gpusvm_notifier *notifier);
47
48 /**
49 * @range_alloc: Allocate a GPU SVM range (optional)
50 * @gpusvm: Pointer to the GPU SVM
51 *
52 * Allocate a GPU SVM range.
53 *
54 * Return: Pointer to the allocated GPU SVM range on success, NULL on failure.
55 */
56 struct drm_gpusvm_range *(*range_alloc)(struct drm_gpusvm *gpusvm);
57
58 /**
59 * @range_free: Free a GPU SVM range (optional)
60 * @range: Pointer to the GPU SVM range to be freed
61 *
62 * Free a GPU SVM range.
63 */
64 void (*range_free)(struct drm_gpusvm_range *range);
65
66 /**
67 * @invalidate: Invalidate GPU SVM notifier (required)
68 * @gpusvm: Pointer to the GPU SVM
69 * @notifier: Pointer to the GPU SVM notifier
70 * @mmu_range: Pointer to the mmu_notifier_range structure
71 *
72 * Invalidate the GPU page tables. It can safely walk the notifier range
73 * RB tree/list in this function. Called while holding the notifier lock.
74 */
75 void (*invalidate)(struct drm_gpusvm *gpusvm,
76 struct drm_gpusvm_notifier *notifier,
77 const struct mmu_notifier_range *mmu_range);
78};
79
80/**
81 * struct drm_gpusvm_notifier - Structure representing a GPU SVM notifier
82 *
83 * @gpusvm: Pointer to the GPU SVM structure
84 * @notifier: MMU interval notifier
85 * @itree: Interval tree node for the notifier (inserted in GPU SVM)
86 * @entry: List entry to fast interval tree traversal
87 * @root: Cached root node of the RB tree containing ranges
88 * @range_list: List head containing of ranges in the same order they appear in
89 * interval tree. This is useful to keep iterating ranges while
90 * doing modifications to RB tree.
91 * @flags: Flags for notifier
92 * @flags.removed: Flag indicating whether the MMU interval notifier has been
93 * removed
94 *
95 * This structure represents a GPU SVM notifier.
96 */
97struct drm_gpusvm_notifier {
98 struct drm_gpusvm *gpusvm;
99 struct mmu_interval_notifier notifier;
100 struct interval_tree_node itree;
101 struct list_head entry;
102 struct rb_root_cached root;
103 struct list_head range_list;
104 struct {
105 u32 removed : 1;
106 } flags;
107};
108
109/**
110 * struct drm_gpusvm_pages_flags - Structure representing a GPU SVM pages flags
111 *
112 * @unmapped: Flag indicating if the pages has been unmapped
113 * @has_devmem_pages: Flag indicating if the pages has devmem pages
114 * @has_dma_mapping: Flag indicating if the pages has a DMA mapping
115 * @__flags: Flags for pages in u16 form (used for READ_ONCE)
116 */
117struct drm_gpusvm_pages_flags {
118 union {
119 struct {
120 /* All flags below must be set / cleared under notifier lock */
121 u16 unmapped : 1;
122 u16 has_devmem_pages : 1;
123 u16 has_dma_mapping : 1;
124 };
125 u16 __flags;
126 };
127};
128
129/**
130 * struct drm_gpusvm_pages - Structure representing a GPU SVM mapped pages
131 *
132 * @drm: The DRM device that owns the dma mappings
133 * @dma_addr: Device address array
134 * @dpagemap: The struct drm_pagemap of the device pages we're dma-mapping.
135 * Note this is assuming only one drm_pagemap per range is allowed.
136 * @state: DMA IOVA state for mapping.
137 * @state_offset: DMA IOVA offset for mapping.
138 * @notifier_seq: Notifier sequence number of the range's pages
139 * @flags: Flags for the range; see &struct drm_gpusvm_pages_flags
140 */
141struct drm_gpusvm_pages {
142 struct drm_device *drm;
143 struct drm_pagemap_addr *dma_addr;
144 struct drm_pagemap *dpagemap;
145 struct dma_iova_state state;
146 unsigned long state_offset;
147 unsigned long notifier_seq;
148 struct drm_gpusvm_pages_flags flags;
149};
150
151/**
152 * struct drm_gpusvm_range_flags - Range-level GPU SVM flags
153 *
154 * @migrate_devmem: Flag indicating whether the range can be migrated to device memory
155 * @unmapped: Flag indicating if the range has been unmapped
156 * @partial_unmap: Flag indicating if the range has been partially unmapped
157 * @__flags: All flags in u16 form (used for READ_ONCE)
158 */
159struct drm_gpusvm_range_flags {
160 union {
161 struct {
162 /* All flags below must be set upon creation */
163 u16 migrate_devmem : 1;
164 /* All flags below must be set / cleared under notifier lock */
165 u16 unmapped : 1;
166 u16 partial_unmap : 1;
167 };
168 u16 __flags;
169 };
170};
171
172/**
173 * struct drm_gpusvm_range - Structure representing a GPU SVM range
174 *
175 * @gpusvm: Pointer to the GPU SVM structure
176 * @notifier: Pointer to the GPU SVM notifier
177 * @refcount: Reference count for the range
178 * @itree: Interval tree node for the range (inserted in GPU SVM notifier)
179 * @entry: List entry to fast interval tree traversal
180 * @flags: Flags for range see &struct drm_gpusvm_range_flags
181 *
182 * This structure represents a GPU SVM range used for tracking memory ranges
183 * mapped in a DRM device.
184 */
185struct drm_gpusvm_range {
186 struct drm_gpusvm *gpusvm;
187 struct drm_gpusvm_notifier *notifier;
188 struct kref refcount;
189 struct interval_tree_node itree;
190 struct list_head entry;
191 struct drm_gpusvm_range_flags flags;
192};
193
194/**
195 * struct drm_gpusvm - GPU SVM structure
196 *
197 * @name: Name of the GPU SVM
198 * @mm: Pointer to the mm_struct for the address space
199 * @mm_start: Start address of GPU SVM
200 * @mm_range: Range of the GPU SVM
201 * @notifier_size: Size of individual notifiers
202 * @ops: Pointer to the operations structure for GPU SVM
203 * @chunk_sizes: Pointer to the array of chunk sizes used in range allocation.
204 * Entries should be powers of 2 in descending order.
205 * @num_chunks: Number of chunks
206 * @notifier_lock: Read-write semaphore for protecting notifier operations
207 * @root: Cached root node of the Red-Black tree containing GPU SVM notifiers
208 * @notifier_list: list head containing of notifiers in the same order they
209 * appear in interval tree. This is useful to keep iterating
210 * notifiers while doing modifications to RB tree.
211 *
212 * This structure represents a GPU SVM (Shared Virtual Memory) used for tracking
213 * memory ranges mapped in a DRM (Direct Rendering Manager) device.
214 *
215 * No reference counting is provided, as this is expected to be embedded in the
216 * driver VM structure along with the struct drm_gpuvm, which handles reference
217 * counting.
218 */
219struct drm_gpusvm {
220 const char *name;
221 struct mm_struct *mm;
222 unsigned long mm_start;
223 unsigned long mm_range;
224 unsigned long notifier_size;
225 const struct drm_gpusvm_ops *ops;
226 const unsigned long *chunk_sizes;
227 int num_chunks;
228 struct rw_semaphore notifier_lock;
229 struct rb_root_cached root;
230 struct list_head notifier_list;
231#ifdef CONFIG_LOCKDEP
232 /**
233 * @lock_dep_map: Annotates drm_gpusvm_range_find_or_insert and
234 * drm_gpusvm_range_remove with a driver provided lock.
235 */
236 struct lockdep_map *lock_dep_map;
237#endif
238};
239
240/**
241 * struct drm_gpusvm_ctx - DRM GPU SVM context
242 *
243 * @device_private_page_owner: The device-private page owner to use for
244 * this operation
245 * @check_pages_threshold: Check CPU pages for present if chunk is less than or
246 * equal to threshold. If not present, reduce chunk
247 * size.
248 * @timeslice_ms: The timeslice MS which in minimum time a piece of memory
249 * remains with either exclusive GPU or CPU access.
250 * @in_notifier: entering from a MMU notifier
251 * @read_only: operating on read-only memory
252 * @devmem_possible: possible to use device memory
253 * @devmem_only: use only device memory
254 * @allow_mixed: Allow mixed mappings in get pages. Mixing between system and
255 * single dpagemap is supported, mixing between multiple dpagemap
256 * is unsupported.
257 *
258 * Context that is DRM GPUSVM is operating in (i.e. user arguments).
259 */
260struct drm_gpusvm_ctx {
261 void *device_private_page_owner;
262 unsigned long check_pages_threshold;
263 unsigned long timeslice_ms;
264 unsigned int in_notifier :1;
265 unsigned int read_only :1;
266 unsigned int devmem_possible :1;
267 unsigned int devmem_only :1;
268 unsigned int allow_mixed :1;
269};
270
271int drm_gpusvm_init(struct drm_gpusvm *gpusvm,
272 const char *name,
273 struct mm_struct *mm,
274 unsigned long mm_start, unsigned long mm_range,
275 unsigned long notifier_size,
276 const struct drm_gpusvm_ops *ops,
277 const unsigned long *chunk_sizes, int num_chunks);
278
279void drm_gpusvm_fini(struct drm_gpusvm *gpusvm);
280
281void drm_gpusvm_free(struct drm_gpusvm *gpusvm);
282
283unsigned long
284drm_gpusvm_find_vma_start(struct drm_gpusvm *gpusvm,
285 unsigned long start,
286 unsigned long end);
287
288struct drm_gpusvm_range *
289drm_gpusvm_range_find_or_insert(struct drm_gpusvm *gpusvm,
290 unsigned long fault_addr,
291 unsigned long gpuva_start,
292 unsigned long gpuva_end,
293 const struct drm_gpusvm_ctx *ctx);
294
295void drm_gpusvm_range_remove(struct drm_gpusvm *gpusvm,
296 struct drm_gpusvm_range *range);
297
298int drm_gpusvm_range_evict(struct drm_gpusvm *gpusvm,
299 struct drm_gpusvm_range *range);
300
301struct drm_gpusvm_range *
302drm_gpusvm_range_get(struct drm_gpusvm_range *range);
303
304void drm_gpusvm_range_put(struct drm_gpusvm_range *range);
305
306bool drm_gpusvm_pages_valid(struct drm_gpusvm *gpusvm,
307 struct drm_gpusvm_pages *svm_pages);
308
309bool drm_gpusvm_has_mapping(struct drm_gpusvm *gpusvm, unsigned long start,
310 unsigned long end);
311
312struct drm_gpusvm_notifier *
313drm_gpusvm_notifier_find(struct drm_gpusvm *gpusvm, unsigned long start,
314 unsigned long end);
315
316struct drm_gpusvm_range *
317drm_gpusvm_range_find(struct drm_gpusvm_notifier *notifier, unsigned long start,
318 unsigned long end);
319
320void drm_gpusvm_range_set_unmapped(struct drm_gpusvm_range *range,
321 struct drm_gpusvm_pages *pages,
322 unsigned int pages_count,
323 const struct mmu_notifier_range *mmu_range);
324
325int drm_gpusvm_get_pages(struct drm_gpusvm *gpusvm,
326 struct drm_gpusvm_pages *svm_pages,
327 struct mm_struct *mm,
328 struct mmu_interval_notifier *notifier,
329 unsigned long pages_start, unsigned long pages_end,
330 const struct drm_gpusvm_ctx *ctx);
331
332void drm_gpusvm_unmap_pages(struct drm_gpusvm *gpusvm,
333 struct drm_gpusvm_pages *svm_pages,
334 unsigned long npages,
335 const struct drm_gpusvm_ctx *ctx);
336
337void drm_gpusvm_free_pages(struct drm_gpusvm *gpusvm,
338 struct drm_gpusvm_pages *svm_pages,
339 unsigned long npages);
340
341/**
342 * drm_gpusvm_init_pages() - Initialize a freshly allocated drm_gpusvm_pages
343 * @svm_pages: Pointer to the drm_gpusvm_pages to initialize.
344 * @drm: The DRM device that will own DMA mappings for this pages object.
345 *
346 * Drivers that embed one or more drm_gpusvm_pages in their own range
347 * structure must call this once on each pages instance after allocation,
348 * before the first drm_gpusvm_get_pages() / unmap / free.
349 */
350static inline void drm_gpusvm_init_pages(struct drm_gpusvm_pages *svm_pages,
351 struct drm_device *drm)
352{
353 memset(svm_pages, 0, sizeof(*svm_pages));
354 svm_pages->drm = drm;
355 svm_pages->notifier_seq = LONG_MAX;
356}
357
358/**
359 * enum drm_gpusvm_scan_result - Scan result from the drm_gpusvm_scan_mm() function.
360 * @DRM_GPUSVM_SCAN_UNPOPULATED: At least one page was not present or inaccessible.
361 * @DRM_GPUSVM_SCAN_EQUAL: All pages belong to the struct dev_pagemap indicated as
362 * the @pagemap argument to the drm_gpusvm_scan_mm() function.
363 * @DRM_GPUSVM_SCAN_OTHER: All pages belong to exactly one dev_pagemap, which is
364 * *NOT* the @pagemap argument to the drm_gpusvm_scan_mm(). All pages belong to
365 * the same device private owner.
366 * @DRM_GPUSVM_SCAN_SYSTEM: All pages are present and system pages.
367 * @DRM_GPUSVM_SCAN_MIXED_DEVICE: All pages are device pages and belong to at least
368 * two different struct dev_pagemaps. All pages belong to the same device private
369 * owner.
370 * @DRM_GPUSVM_SCAN_MIXED: Pages are present and are a mix of system pages
371 * and device-private pages. All device-private pages belong to the same device
372 * private owner.
373 */
374enum drm_gpusvm_scan_result {
375 DRM_GPUSVM_SCAN_UNPOPULATED,
376 DRM_GPUSVM_SCAN_EQUAL,
377 DRM_GPUSVM_SCAN_OTHER,
378 DRM_GPUSVM_SCAN_SYSTEM,
379 DRM_GPUSVM_SCAN_MIXED_DEVICE,
380 DRM_GPUSVM_SCAN_MIXED,
381};
382
383enum drm_gpusvm_scan_result drm_gpusvm_scan_mm(struct drm_gpusvm_range *range,
384 void *dev_private_owner,
385 const struct dev_pagemap *pagemap);
386
387#ifdef CONFIG_LOCKDEP
388/**
389 * drm_gpusvm_driver_set_lock() - Set the lock protecting accesses to GPU SVM
390 * @gpusvm: Pointer to the GPU SVM structure.
391 * @lock: the lock used to protect the gpuva list. The locking primitive
392 * must contain a dep_map field.
393 *
394 * Call this to annotate drm_gpusvm_range_find_or_insert and
395 * drm_gpusvm_range_remove.
396 */
397#define drm_gpusvm_driver_set_lock(gpusvm, lock) \
398 do { \
399 if (!WARN((gpusvm)->lock_dep_map, \
400 "GPUSVM range lock should be set only once."))\
401 (gpusvm)->lock_dep_map = &(lock)->dep_map; \
402 } while (0)
403#else
404#define drm_gpusvm_driver_set_lock(gpusvm, lock) do {} while (0)
405#endif
406
407/**
408 * drm_gpusvm_notifier_lock() - Lock GPU SVM notifier
409 * @gpusvm__: Pointer to the GPU SVM structure.
410 *
411 * Abstract client usage GPU SVM notifier lock, take lock
412 */
413#define drm_gpusvm_notifier_lock(gpusvm__) \
414 down_read(&(gpusvm__)->notifier_lock)
415
416/**
417 * drm_gpusvm_notifier_unlock() - Unlock GPU SVM notifier
418 * @gpusvm__: Pointer to the GPU SVM structure.
419 *
420 * Abstract client usage GPU SVM notifier lock, drop lock
421 */
422#define drm_gpusvm_notifier_unlock(gpusvm__) \
423 up_read(&(gpusvm__)->notifier_lock)
424
425/**
426 * drm_gpusvm_range_start() - GPU SVM range start address
427 * @range: Pointer to the GPU SVM range
428 *
429 * Return: GPU SVM range start address
430 */
431static inline unsigned long
432drm_gpusvm_range_start(struct drm_gpusvm_range *range)
433{
434 return range->itree.start;
435}
436
437/**
438 * drm_gpusvm_range_end() - GPU SVM range end address
439 * @range: Pointer to the GPU SVM range
440 *
441 * Return: GPU SVM range end address
442 */
443static inline unsigned long
444drm_gpusvm_range_end(struct drm_gpusvm_range *range)
445{
446 return range->itree.last + 1;
447}
448
449/**
450 * drm_gpusvm_range_size() - GPU SVM range size
451 * @range: Pointer to the GPU SVM range
452 *
453 * Return: GPU SVM range size
454 */
455static inline unsigned long
456drm_gpusvm_range_size(struct drm_gpusvm_range *range)
457{
458 return drm_gpusvm_range_end(range) - drm_gpusvm_range_start(range);
459}
460
461/**
462 * drm_gpusvm_notifier_start() - GPU SVM notifier start address
463 * @notifier: Pointer to the GPU SVM notifier
464 *
465 * Return: GPU SVM notifier start address
466 */
467static inline unsigned long
468drm_gpusvm_notifier_start(struct drm_gpusvm_notifier *notifier)
469{
470 return notifier->itree.start;
471}
472
473/**
474 * drm_gpusvm_notifier_end() - GPU SVM notifier end address
475 * @notifier: Pointer to the GPU SVM notifier
476 *
477 * Return: GPU SVM notifier end address
478 */
479static inline unsigned long
480drm_gpusvm_notifier_end(struct drm_gpusvm_notifier *notifier)
481{
482 return notifier->itree.last + 1;
483}
484
485/**
486 * drm_gpusvm_notifier_size() - GPU SVM notifier size
487 * @notifier: Pointer to the GPU SVM notifier
488 *
489 * Return: GPU SVM notifier size
490 */
491static inline unsigned long
492drm_gpusvm_notifier_size(struct drm_gpusvm_notifier *notifier)
493{
494 return drm_gpusvm_notifier_end(notifier) -
495 drm_gpusvm_notifier_start(notifier);
496}
497
498/**
499 * __drm_gpusvm_range_next() - Get the next GPU SVM range in the list
500 * @range: a pointer to the current GPU SVM range
501 *
502 * Return: A pointer to the next drm_gpusvm_range if available, or NULL if the
503 * current range is the last one or if the input range is NULL.
504 */
505static inline struct drm_gpusvm_range *
506__drm_gpusvm_range_next(struct drm_gpusvm_range *range)
507{
508 if (range && !list_is_last(&range->entry,
509 &range->notifier->range_list))
510 return list_next_entry(range, entry);
511
512 return NULL;
513}
514
515/**
516 * drm_gpusvm_for_each_range() - Iterate over GPU SVM ranges in a notifier
517 * @range__: Iterator variable for the ranges. If set, it indicates the start of
518 * the iterator. If NULL, call drm_gpusvm_range_find() to get the range.
519 * @notifier__: Pointer to the GPU SVM notifier
520 * @start__: Start address of the range
521 * @end__: End address of the range
522 *
523 * This macro is used to iterate over GPU SVM ranges in a notifier. It is safe
524 * to use while holding the driver SVM lock or the notifier lock.
525 */
526#define drm_gpusvm_for_each_range(range__, notifier__, start__, end__) \
527 for ((range__) = (range__) ?: \
528 drm_gpusvm_range_find((notifier__), (start__), (end__)); \
529 (range__) && (drm_gpusvm_range_start(range__) < (end__)); \
530 (range__) = __drm_gpusvm_range_next(range__))
531
532/**
533 * drm_gpusvm_for_each_range_safe() - Safely iterate over GPU SVM ranges in a notifier
534 * @range__: Iterator variable for the ranges
535 * @next__: Iterator variable for the ranges temporay storage
536 * @notifier__: Pointer to the GPU SVM notifier
537 * @start__: Start address of the range
538 * @end__: End address of the range
539 *
540 * This macro is used to iterate over GPU SVM ranges in a notifier while
541 * removing ranges from it.
542 */
543#define drm_gpusvm_for_each_range_safe(range__, next__, notifier__, start__, end__) \
544 for ((range__) = drm_gpusvm_range_find((notifier__), (start__), (end__)), \
545 (next__) = __drm_gpusvm_range_next(range__); \
546 (range__) && (drm_gpusvm_range_start(range__) < (end__)); \
547 (range__) = (next__), (next__) = __drm_gpusvm_range_next(range__))
548
549/**
550 * __drm_gpusvm_notifier_next() - get the next drm_gpusvm_notifier in the list
551 * @notifier: a pointer to the current drm_gpusvm_notifier
552 *
553 * Return: A pointer to the next drm_gpusvm_notifier if available, or NULL if
554 * the current notifier is the last one or if the input notifier is
555 * NULL.
556 */
557static inline struct drm_gpusvm_notifier *
558__drm_gpusvm_notifier_next(struct drm_gpusvm_notifier *notifier)
559{
560 if (notifier && !list_is_last(&notifier->entry,
561 &notifier->gpusvm->notifier_list))
562 return list_next_entry(notifier, entry);
563
564 return NULL;
565}
566
567/**
568 * drm_gpusvm_for_each_notifier() - Iterate over GPU SVM notifiers in a gpusvm
569 * @notifier__: Iterator variable for the notifiers
570 * @gpusvm__: Pointer to the GPU SVM notifier
571 * @start__: Start address of the notifier
572 * @end__: End address of the notifier
573 *
574 * This macro is used to iterate over GPU SVM notifiers in a gpusvm.
575 */
576#define drm_gpusvm_for_each_notifier(notifier__, gpusvm__, start__, end__) \
577 for ((notifier__) = drm_gpusvm_notifier_find((gpusvm__), (start__), (end__)); \
578 (notifier__) && (drm_gpusvm_notifier_start(notifier__) < (end__)); \
579 (notifier__) = __drm_gpusvm_notifier_next(notifier__))
580
581/**
582 * drm_gpusvm_for_each_notifier_safe() - Safely iterate over GPU SVM notifiers in a gpusvm
583 * @notifier__: Iterator variable for the notifiers
584 * @next__: Iterator variable for the notifiers temporay storage
585 * @gpusvm__: Pointer to the GPU SVM notifier
586 * @start__: Start address of the notifier
587 * @end__: End address of the notifier
588 *
589 * This macro is used to iterate over GPU SVM notifiers in a gpusvm while
590 * removing notifiers from it.
591 */
592#define drm_gpusvm_for_each_notifier_safe(notifier__, next__, gpusvm__, start__, end__) \
593 for ((notifier__) = drm_gpusvm_notifier_find((gpusvm__), (start__), (end__)), \
594 (next__) = __drm_gpusvm_notifier_next(notifier__); \
595 (notifier__) && (drm_gpusvm_notifier_start(notifier__) < (end__)); \
596 (notifier__) = (next__), (next__) = __drm_gpusvm_notifier_next(notifier__))
597
598#endif /* __DRM_GPUSVM_H__ */