projects/linux·blob master · refs

394 lines · 12.8 KiB

1/* SPDX-License-Identifier: MIT */
2#ifndef _DRM_PAGEMAP_H_
3#define _DRM_PAGEMAP_H_
4
5#include <linux/bits.h>
6#include <linux/dma-direction.h>
7#include <linux/hmm.h>
8#include <linux/memremap.h>
9#include <linux/types.h>
10
11#define NR_PAGES(order) (1U << (order))
12
13struct dma_fence;
14struct drm_pagemap;
15struct drm_pagemap_cache;
16struct drm_pagemap_dev_hold;
17struct drm_pagemap_zdd;
18struct device;
19
20/**
21 * enum drm_interconnect_protocol - Used to identify an interconnect protocol.
22 *
23 * @DRM_INTERCONNECT_SYSTEM: DMA map is system pages
24 * @DRM_INTERCONNECT_DRIVER: DMA map is driver defined
25 */
26enum drm_interconnect_protocol {
27 DRM_INTERCONNECT_SYSTEM,
28 DRM_INTERCONNECT_DRIVER,
29 /* A driver can add private values beyond DRM_INTERCONNECT_DRIVER */
30};
31
32/**
33 * struct drm_pagemap_addr - Address representation.
34 * @addr: The dma address or driver-defined address for driver private interconnects.
35 * @proto: The interconnect protocol.
36 * @order: The page order of the device mapping. (Size is PAGE_SIZE << order).
37 * @dir: The DMA direction.
38 *
39 * Note: There is room for improvement here. We should be able to pack into
40 * 64 bits.
41 */
42struct drm_pagemap_addr {
43 dma_addr_t addr;
44 u64 proto : 54;
45 u64 order : 8;
46 u64 dir : 2;
47};
48
49/**
50 * drm_pagemap_addr_encode() - Encode a dma address with metadata
51 * @addr: The dma address or driver-defined address for driver private interconnects.
52 * @proto: The interconnect protocol.
53 * @order: The page order of the dma mapping. (Size is PAGE_SIZE << order).
54 * @dir: The DMA direction.
55 *
56 * Return: A struct drm_pagemap_addr encoding the above information.
57 */
58static inline struct drm_pagemap_addr
59drm_pagemap_addr_encode(dma_addr_t addr,
60 enum drm_interconnect_protocol proto,
61 unsigned int order,
62 enum dma_data_direction dir)
63{
64 return (struct drm_pagemap_addr) {
65 .addr = addr,
66 .proto = proto,
67 .order = order,
68 .dir = dir,
69 };
70}
71
72/**
73 * struct drm_pagemap_ops: Ops for a drm-pagemap.
74 */
75struct drm_pagemap_ops {
76 /**
77 * @device_map: Map for device access or provide a virtual address suitable for
78 *
79 * @dpagemap: The struct drm_pagemap for the page.
80 * @dev: The device mapper.
81 * @page: The page to map.
82 * @order: The page order of the device mapping. (Size is PAGE_SIZE << order).
83 * @dir: The transfer direction.
84 */
85 struct drm_pagemap_addr (*device_map)(struct drm_pagemap *dpagemap,
86 struct device *dev,
87 struct page *page,
88 unsigned int order,
89 enum dma_data_direction dir);
90
91 /**
92 * @device_unmap: Unmap a device address previously obtained using @device_map.
93 *
94 * @dpagemap: The struct drm_pagemap for the mapping.
95 * @dev: The device unmapper.
96 * @addr: The device address obtained when mapping.
97 */
98 void (*device_unmap)(struct drm_pagemap *dpagemap,
99 struct device *dev,
100 const struct drm_pagemap_addr *addr);
101
102 /**
103 * @populate_mm: Populate part of the mm with @dpagemap memory,
104 * migrating existing data.
105 * @dpagemap: The struct drm_pagemap managing the memory.
106 * @start: The virtual start address in @mm
107 * @end: The virtual end address in @mm
108 * @mm: Pointer to a live mm. The caller must have an mmget()
109 * reference.
110 *
111 * The caller will have the mm lock at least in read mode.
112 * Note that there is no guarantee that the memory is resident
113 * after the function returns, it's best effort only.
114 * When the mm is not using the memory anymore,
115 * it will be released. The struct drm_pagemap might have a
116 * mechanism in place to reclaim the memory and the data will
117 * then be migrated. Typically to system memory.
118 * The implementation should hold sufficient runtime power-
119 * references while pages are used in an address space and
120 * should ideally guard against hardware device unbind in
121 * a way such that device pages are migrated back to system
122 * followed by device page removal. The implementation should
123 * return -ENODEV after device removal.
124 *
125 * Return: 0 if successful. Negative error code on error.
126 */
127 int (*populate_mm)(struct drm_pagemap *dpagemap,
128 unsigned long start, unsigned long end,
129 struct mm_struct *mm,
130 unsigned long timeslice_ms);
131 /**
132 * @destroy: Destroy the drm_pagemap and associated resources.
133 * @dpagemap: The drm_pagemap to destroy.
134 * @is_atomic_or_reclaim: The function may be called from
135 * atomic- or reclaim context.
136 *
137 * The implementation should take care not to attempt to
138 * destroy resources that may already have been destroyed
139 * using devm_ callbacks, since this function may be called
140 * after the underlying struct device has been unbound.
141 * If the implementation defers the execution to a work item
142 * to avoid locking issues, then it must make sure the work
143 * items are flushed before module exit. If the destroy call
144 * happens after the provider's pci_remove() callback has
145 * been executed, a module reference and drm device reference is
146 * held across the destroy callback.
147 */
148 void (*destroy)(struct drm_pagemap *dpagemap,
149 bool is_atomic_or_reclaim);
150};
151
152/**
153 * struct drm_pagemap: Additional information for a struct dev_pagemap
154 * used for device p2p handshaking.
155 * @ops: The struct drm_pagemap_ops.
156 * @ref: Reference count.
157 * @drm: The struct drm device owning the device-private memory.
158 * @pagemap: Pointer to the underlying dev_pagemap.
159 * @dev_hold: Pointer to a struct drm_pagemap_dev_hold for
160 * device referencing.
161 * @cache: Back-pointer to the &struct drm_pagemap_cache used for this
162 * &struct drm_pagemap. May be NULL if no cache is used.
163 * @shrink_link: Link into the shrinker's list of drm_pagemaps. Only
164 * used if also using a pagemap cache.
165 */
166struct drm_pagemap {
167 const struct drm_pagemap_ops *ops;
168 struct kref ref;
169 struct drm_device *drm;
170 struct dev_pagemap *pagemap;
171 struct drm_pagemap_dev_hold *dev_hold;
172 struct drm_pagemap_cache *cache;
173 struct list_head shrink_link;
174};
175
176struct drm_pagemap_devmem;
177
178/**
179 * struct drm_pagemap_devmem_ops - Operations structure for GPU SVM device memory
180 *
181 * This structure defines the operations for GPU Shared Virtual Memory (SVM)
182 * device memory. These operations are provided by the GPU driver to manage device memory
183 * allocations and perform operations such as migration between device memory and system
184 * RAM.
185 */
186struct drm_pagemap_devmem_ops {
187 /**
188 * @devmem_release: Release device memory allocation (optional)
189 * @devmem_allocation: device memory allocation
190 *
191 * Release device memory allocation and drop a reference to device
192 * memory allocation.
193 */
194 void (*devmem_release)(struct drm_pagemap_devmem *devmem_allocation);
195
196 /**
197 * @populate_devmem_pfn: Populate device memory PFN (required for migration)
198 * @devmem_allocation: device memory allocation
199 * @npages: Number of pages to populate
200 * @pfn: Array of page frame numbers to populate
201 *
202 * Populate device memory page frame numbers (PFN).
203 *
204 * Return: 0 on success, a negative error code on failure.
205 */
206 int (*populate_devmem_pfn)(struct drm_pagemap_devmem *devmem_allocation,
207 unsigned long npages, unsigned long *pfn);
208
209 /**
210 * @copy_to_devmem: Copy to device memory (required for migration)
211 * @pages: Pointer to array of device memory pages (destination)
212 * @pagemap_addr: Pointer to array of DMA information (source)
213 * @npages: Number of pages to copy
214 * @pre_migrate_fence: dma-fence to wait for before migration start.
215 * May be NULL.
216 *
217 * Copy pages to device memory. If the order of a @pagemap_addr entry
218 * is greater than 0, the entry is populated but subsequent entries
219 * within the range of that order are not populated.
220 *
221 * Return: 0 on success, a negative error code on failure.
222 */
223 int (*copy_to_devmem)(struct page **pages,
224 struct drm_pagemap_addr *pagemap_addr,
225 unsigned long npages,
226 struct dma_fence *pre_migrate_fence);
227
228 /**
229 * @copy_to_ram: Copy to system RAM (required for migration)
230 * @pages: Pointer to array of device memory pages (source)
231 * @pagemap_addr: Pointer to array of DMA information (destination)
232 * @npages: Number of pages to copy
233 * @pre_migrate_fence: dma-fence to wait for before migration start.
234 * May be NULL.
235 *
236 * Copy pages to system RAM. If the order of a @pagemap_addr entry
237 * is greater than 0, the entry is populated but subsequent entries
238 * within the range of that order are not populated.
239 *
240 * Return: 0 on success, a negative error code on failure.
241 */
242 int (*copy_to_ram)(struct page **pages,
243 struct drm_pagemap_addr *pagemap_addr,
244 unsigned long npages,
245 struct dma_fence *pre_migrate_fence);
246};
247
248#if IS_ENABLED(CONFIG_ZONE_DEVICE)
249
250int drm_pagemap_init(struct drm_pagemap *dpagemap,
251 struct dev_pagemap *pagemap,
252 struct drm_device *drm,
253 const struct drm_pagemap_ops *ops);
254
255struct drm_pagemap *drm_pagemap_create(struct drm_device *drm,
256 struct dev_pagemap *pagemap,
257 const struct drm_pagemap_ops *ops);
258
259struct drm_pagemap *drm_pagemap_page_to_dpagemap(struct page *page);
260
261void drm_pagemap_put(struct drm_pagemap *dpagemap);
262
263#else
264
265static inline struct drm_pagemap *drm_pagemap_page_to_dpagemap(struct page *page)
266{
267 return NULL;
268}
269
270static inline void drm_pagemap_put(struct drm_pagemap *dpagemap)
271{
272}
273
274#endif /* IS_ENABLED(CONFIG_ZONE_DEVICE) */
275
276/**
277 * drm_pagemap_get() - Obtain a reference on a struct drm_pagemap
278 * @dpagemap: Pointer to the struct drm_pagemap, or NULL.
279 *
280 * Return: Pointer to the struct drm_pagemap, or NULL.
281 */
282static inline struct drm_pagemap *
283drm_pagemap_get(struct drm_pagemap *dpagemap)
284{
285 if (likely(dpagemap))
286 kref_get(&dpagemap->ref);
287
288 return dpagemap;
289}
290
291/**
292 * drm_pagemap_get_unless_zero() - Obtain a reference on a struct drm_pagemap
293 * unless the current reference count is zero.
294 * @dpagemap: Pointer to the drm_pagemap or NULL.
295 *
296 * Return: A pointer to @dpagemap if the reference count was successfully
297 * incremented. NULL if @dpagemap was NULL, or its refcount was 0.
298 */
299static inline struct drm_pagemap * __must_check
300drm_pagemap_get_unless_zero(struct drm_pagemap *dpagemap)
301{
302 return (dpagemap && kref_get_unless_zero(&dpagemap->ref)) ? dpagemap : NULL;
303}
304
305/**
306 * struct drm_pagemap_devmem - Structure representing a GPU SVM device memory allocation
307 *
308 * @dev: Pointer to the device structure which device memory allocation belongs to
309 * @mm: Pointer to the mm_struct for the address space
310 * @detached: device memory allocations is detached from device pages
311 * @ops: Pointer to the operations structure for GPU SVM device memory
312 * @dpagemap: The struct drm_pagemap of the pages this allocation belongs to.
313 * @size: Size of device memory allocation
314 * @timeslice_expiration: Timeslice expiration in jiffies
315 * @pre_migrate_fence: Fence to wait for or pipeline behind before migration starts.
316 * (May be NULL).
317 */
318struct drm_pagemap_devmem {
319 struct device *dev;
320 struct mm_struct *mm;
321 struct completion detached;
322 const struct drm_pagemap_devmem_ops *ops;
323 struct drm_pagemap *dpagemap;
324 size_t size;
325 u64 timeslice_expiration;
326 struct dma_fence *pre_migrate_fence;
327};
328
329/**
330 * struct drm_pagemap_migrate_details - Details to govern migration.
331 * @timeslice_ms: The time requested for the migrated pagemap pages to
332 * be present in @mm before being allowed to be migrated back.
333 * @can_migrate_same_pagemap: Whether the copy function can migrate
334 * device pages within a single drm_pagemap.
335 */
336struct drm_pagemap_migrate_details {
337 unsigned long timeslice_ms;
338 u32 can_migrate_same_pagemap : 1;
339};
340
341#if IS_ENABLED(CONFIG_ZONE_DEVICE)
342
343#define DRM_PAGEMAP_ZDD_FLAG_MIGRATED BIT(0)
344#define DRM_PAGEMAP_ZDD_FLAG_MASK DRM_PAGEMAP_ZDD_FLAG_MIGRATED
345
346int drm_pagemap_migrate_to_devmem(struct drm_pagemap_devmem *devmem_allocation,
347 struct mm_struct *mm,
348 unsigned long start, unsigned long end,
349 const struct drm_pagemap_migrate_details *mdetails);
350
351int drm_pagemap_evict_to_ram(struct drm_pagemap_devmem *devmem_allocation);
352
353const struct dev_pagemap_ops *drm_pagemap_pagemap_ops_get(void);
354
355void drm_pagemap_devmem_init(struct drm_pagemap_devmem *devmem_allocation,
356 struct device *dev, struct mm_struct *mm,
357 const struct drm_pagemap_devmem_ops *ops,
358 struct drm_pagemap *dpagemap, size_t size,
359 struct dma_fence *pre_migrate_fence);
360
361int drm_pagemap_populate_mm(struct drm_pagemap *dpagemap,
362 unsigned long start, unsigned long end,
363 struct mm_struct *mm,
364 unsigned long timeslice_ms);
365
366void drm_pagemap_destroy(struct drm_pagemap *dpagemap, bool is_atomic_or_reclaim);
367
368int drm_pagemap_reinit(struct drm_pagemap *dpagemap);
369
370/**
371 * drm_pagemap_page_zone_device_data() - Page to zone_device_data
372 * @page: Pointer to the page
373 *
374 * Return: Page's zone_device_data
375 */
376static inline struct drm_pagemap_zdd *drm_pagemap_page_zone_device_data(struct page *page)
377{
378 struct folio *folio = page_folio(page);
379
380 return (struct drm_pagemap_zdd *)
381 ((unsigned long)folio_zone_device_data(folio) &
382 ~DRM_PAGEMAP_ZDD_FLAG_MASK);
383}
384
385#else
386
387static inline struct drm_pagemap_zdd *drm_pagemap_page_zone_device_data(struct page *page)
388{
389 return NULL;
390}
391
392#endif /* IS_ENABLED(CONFIG_ZONE_DEVICE) */
393
394#endif