projects/linux·blob master · refs

1768 lines · 47.6 KiB

1/*
2 * Copyright © 2008 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 *
23 * Authors:
24 * Eric Anholt <[email protected]>
25 *
26 */
27
28#include <linux/dma-buf.h>
29#include <linux/export.h>
30#include <linux/file.h>
31#include <linux/fs.h>
32#ifdef CONFIG_TRANSPARENT_HUGEPAGE
33#include <linux/fs_context.h>
34#endif
35#include <linux/iosys-map.h>
36#include <linux/mem_encrypt.h>
37#include <linux/mm.h>
38#include <linux/mman.h>
39#include <linux/module.h>
40#include <linux/pagemap.h>
41#include <linux/folio_batch.h>
42#include <linux/sched/mm.h>
43#include <linux/shmem_fs.h>
44#include <linux/slab.h>
45#include <linux/string_helpers.h>
46#include <linux/types.h>
47#include <linux/uaccess.h>
48
49#include <drm/drm.h>
50#include <drm/drm_device.h>
51#include <drm/drm_drv.h>
52#include <drm/drm_file.h>
53#include <drm/drm_gem.h>
54#include <drm/drm_managed.h>
55#include <drm/drm_print.h>
56#include <drm/drm_vma_manager.h>
57
58#include "drm_internal.h"
59
60/** @file drm_gem.c
61 *
62 * This file provides some of the base ioctls and library routines for
63 * the graphics memory manager implemented by each device driver.
64 *
65 * Because various devices have different requirements in terms of
66 * synchronization and migration strategies, implementing that is left up to
67 * the driver, and all that the general API provides should be generic --
68 * allocating objects, reading/writing data with the cpu, freeing objects.
69 * Even there, platform-dependent optimizations for reading/writing data with
70 * the CPU mean we'll likely hook those out to driver-specific calls. However,
71 * the DRI2 implementation wants to have at least allocate/mmap be generic.
72 *
73 * The goal was to have swap-backed object allocation managed through
74 * struct file. However, file descriptors as handles to a struct file have
75 * two major failings:
76 * - Process limits prevent more than 1024 or so being used at a time by
77 * default.
78 * - Inability to allocate high fds will aggravate the X Server's select()
79 * handling, and likely that of many GL client applications as well.
80 *
81 * This led to a plan of using our own integer IDs (called handles, following
82 * DRM terminology) to mimic fds, and implement the fd syscalls we need as
83 * ioctls. The objects themselves will still include the struct file so
84 * that we can transition to fds if the required kernel infrastructure shows
85 * up at a later date, and as our interface with shmfs for memory allocation.
86 */
87
88#ifdef CONFIG_TRANSPARENT_HUGEPAGE
89static void drm_gem_huge_mnt_free(struct drm_device *dev, void *data)
90{
91 kern_unmount(dev->huge_mnt);
92}
93
94/**
95 * drm_gem_huge_mnt_create - Create, mount and use a huge tmpfs mountpoint
96 * @dev: DRM device that will use the huge tmpfs mountpoint
97 * @value: huge tmpfs mount option value
98 *
99 * This function creates and mounts a dedicated huge tmpfs mountpoint for the
100 * lifetime of the DRM device @dev which is used at GEM object initialization
101 * with drm_gem_object_init().
102 *
103 * The most common option for @value is "within_size" which only allocates huge
104 * pages if the page will be fully within the GEM object size. "always",
105 * "advise" and "never" are supported too but the latter would just create a
106 * mountpoint similar to the default one (`shm_mnt`). See shmemfs and
107 * Transparent Hugepage for more information.
108 *
109 * Returns:
110 * 0 on success or a negative error code on failure.
111 */
112int drm_gem_huge_mnt_create(struct drm_device *dev, const char *value)
113{
114 struct file_system_type *type;
115 struct fs_context *fc;
116 int ret;
117
118 if (unlikely(drm_gem_get_huge_mnt(dev)))
119 return 0;
120
121 type = get_fs_type("tmpfs");
122 if (unlikely(!type))
123 return -EOPNOTSUPP;
124 fc = fs_context_for_mount(type, SB_KERNMOUNT);
125 if (IS_ERR(fc))
126 return PTR_ERR(fc);
127 ret = vfs_parse_fs_string(fc, "source", "tmpfs");
128 if (unlikely(ret))
129 return -ENOPARAM;
130 ret = vfs_parse_fs_string(fc, "huge", value);
131 if (unlikely(ret))
132 return -ENOPARAM;
133
134 dev->huge_mnt = fc_mount_longterm(fc);
135 put_fs_context(fc);
136
137 return drmm_add_action_or_reset(dev, drm_gem_huge_mnt_free, NULL);
138}
139EXPORT_SYMBOL_GPL(drm_gem_huge_mnt_create);
140#endif
141
142static void
143drm_gem_init_release(struct drm_device *dev, void *ptr)
144{
145 drm_vma_offset_manager_destroy(dev->vma_offset_manager);
146}
147
148/**
149 * drm_gem_init - Initialize the GEM device fields
150 * @dev: drm_devic structure to initialize
151 */
152int
153drm_gem_init(struct drm_device *dev)
154{
155 struct drm_vma_offset_manager *vma_offset_manager;
156
157 mutex_init(&dev->object_name_lock);
158 idr_init_base(&dev->object_name_idr, 1);
159
160 vma_offset_manager = drmm_kzalloc(dev, sizeof(*vma_offset_manager),
161 GFP_KERNEL);
162 if (!vma_offset_manager)
163 return -ENOMEM;
164
165 dev->vma_offset_manager = vma_offset_manager;
166 drm_vma_offset_manager_init(vma_offset_manager,
167 DRM_FILE_PAGE_OFFSET_START,
168 DRM_FILE_PAGE_OFFSET_SIZE);
169
170 return drmm_add_action(dev, drm_gem_init_release, NULL);
171}
172
173/**
174 * drm_gem_object_init - initialize an allocated shmem-backed GEM object
175 *
176 * @dev: drm_device the object should be initialized for
177 * @obj: drm_gem_object to initialize
178 * @size: object size
179 *
180 * Initialize an already allocated GEM object of the specified size with
181 * shmfs backing store. A huge mountpoint can be used by calling
182 * drm_gem_huge_mnt_create() beforehand.
183 */
184int drm_gem_object_init(struct drm_device *dev, struct drm_gem_object *obj,
185 size_t size)
186{
187 struct vfsmount *huge_mnt;
188 struct file *filp;
189 const vma_flags_t flags = mk_vma_flags(VMA_NORESERVE_BIT);
190
191 drm_gem_private_object_init(dev, obj, size);
192
193 huge_mnt = drm_gem_get_huge_mnt(dev);
194 if (huge_mnt)
195 filp = shmem_file_setup_with_mnt(huge_mnt, "drm mm object",
196 size, flags);
197 else
198 filp = shmem_file_setup("drm mm object", size, flags);
199
200 if (IS_ERR(filp))
201 return PTR_ERR(filp);
202
203 obj->filp = filp;
204
205 return 0;
206}
207EXPORT_SYMBOL(drm_gem_object_init);
208
209/**
210 * drm_gem_private_object_init - initialize an allocated private GEM object
211 * @dev: drm_device the object should be initialized for
212 * @obj: drm_gem_object to initialize
213 * @size: object size
214 *
215 * Initialize an already allocated GEM object of the specified size with
216 * no GEM provided backing store. Instead the caller is responsible for
217 * backing the object and handling it.
218 */
219void drm_gem_private_object_init(struct drm_device *dev,
220 struct drm_gem_object *obj, size_t size)
221{
222 BUG_ON((size & (PAGE_SIZE - 1)) != 0);
223
224 obj->dev = dev;
225 obj->filp = NULL;
226
227 kref_init(&obj->refcount);
228 obj->handle_count = 0;
229 obj->size = size;
230 mutex_init(&obj->gpuva.lock);
231 dma_resv_init(&obj->_resv);
232 if (!obj->resv)
233 obj->resv = &obj->_resv;
234
235 drm_gem_gpuva_init(obj);
236
237 drm_vma_node_reset(&obj->vma_node);
238 INIT_LIST_HEAD(&obj->lru_node);
239}
240EXPORT_SYMBOL(drm_gem_private_object_init);
241
242/**
243 * drm_gem_private_object_fini - Finalize a failed drm_gem_object
244 * @obj: drm_gem_object
245 *
246 * Uninitialize an already allocated GEM object when it initialized failed
247 */
248void drm_gem_private_object_fini(struct drm_gem_object *obj)
249{
250 WARN_ON(obj->dma_buf);
251
252 dma_resv_fini(&obj->_resv);
253 mutex_destroy(&obj->gpuva.lock);
254}
255EXPORT_SYMBOL(drm_gem_private_object_fini);
256
257static void drm_gem_object_handle_get(struct drm_gem_object *obj)
258{
259 struct drm_device *dev = obj->dev;
260
261 drm_WARN_ON(dev, !mutex_is_locked(&dev->object_name_lock));
262
263 if (obj->handle_count++ == 0)
264 drm_gem_object_get(obj);
265}
266
267/**
268 * drm_gem_object_handle_get_if_exists_unlocked - acquire reference on user-space handle, if any
269 * @obj: GEM object
270 *
271 * Acquires a reference on the GEM buffer object's handle. Required to keep
272 * the GEM object alive. Call drm_gem_object_handle_put_if_exists_unlocked()
273 * to release the reference. Does nothing if the buffer object has no handle.
274 *
275 * Returns:
276 * True if a handle exists, or false otherwise
277 */
278bool drm_gem_object_handle_get_if_exists_unlocked(struct drm_gem_object *obj)
279{
280 struct drm_device *dev = obj->dev;
281
282 guard(mutex)(&dev->object_name_lock);
283
284 /*
285 * First ref taken during GEM object creation, if any. Some
286 * drivers set up internal framebuffers with GEM objects that
287 * do not have a GEM handle. Hence, this counter can be zero.
288 */
289 if (!obj->handle_count)
290 return false;
291
292 drm_gem_object_handle_get(obj);
293
294 return true;
295}
296
297/**
298 * drm_gem_object_handle_free - release resources bound to userspace handles
299 * @obj: GEM object to clean up.
300 *
301 * Called after the last handle to the object has been closed
302 *
303 * Removes any name for the object. Note that this must be
304 * called before drm_gem_object_free or we'll be touching
305 * freed memory
306 */
307static void drm_gem_object_handle_free(struct drm_gem_object *obj)
308{
309 struct drm_device *dev = obj->dev;
310
311 /* Remove any name for this object */
312 if (obj->name) {
313 idr_remove(&dev->object_name_idr, obj->name);
314 obj->name = 0;
315 }
316}
317
318static void drm_gem_object_exported_dma_buf_free(struct drm_gem_object *obj)
319{
320 /* Unbreak the reference cycle if we have an exported dma_buf. */
321 if (obj->dma_buf) {
322 dma_buf_put(obj->dma_buf);
323 obj->dma_buf = NULL;
324 }
325}
326
327/**
328 * drm_gem_object_handle_put_unlocked - releases reference on user-space handle
329 * @obj: GEM object
330 *
331 * Releases a reference on the GEM buffer object's handle. Possibly releases
332 * the GEM buffer object and associated dma-buf objects.
333 */
334void drm_gem_object_handle_put_unlocked(struct drm_gem_object *obj)
335{
336 struct drm_device *dev = obj->dev;
337 bool final = false;
338
339 if (drm_WARN_ON(dev, READ_ONCE(obj->handle_count) == 0))
340 return;
341
342 /*
343 * Must bump handle count first as this may be the last
344 * ref, in which case the object would disappear before
345 * we checked for a name.
346 */
347
348 mutex_lock(&dev->object_name_lock);
349 if (--obj->handle_count == 0) {
350 drm_gem_object_handle_free(obj);
351 drm_gem_object_exported_dma_buf_free(obj);
352 final = true;
353 }
354 mutex_unlock(&dev->object_name_lock);
355
356 if (final)
357 drm_gem_object_put(obj);
358}
359
360/*
361 * Called at device or object close to release the file's
362 * handle references on objects.
363 */
364static int
365drm_gem_object_release_handle(int id, void *ptr, void *data)
366{
367 struct drm_file *file_priv = data;
368 struct drm_gem_object *obj = ptr;
369
370 if (drm_WARN_ON(obj->dev, !data))
371 return 0;
372
373 if (obj->funcs->close)
374 obj->funcs->close(obj, file_priv);
375
376 mutex_lock(&file_priv->prime.lock);
377
378 drm_prime_remove_buf_handle(&file_priv->prime, id);
379
380 mutex_unlock(&file_priv->prime.lock);
381
382 drm_vma_node_revoke(&obj->vma_node, file_priv);
383
384 drm_gem_object_handle_put_unlocked(obj);
385
386 return 0;
387}
388
389/**
390 * drm_gem_handle_delete - deletes the given file-private handle
391 * @filp: drm file-private structure to use for the handle look up
392 * @handle: userspace handle to delete
393 *
394 * Removes the GEM handle from the @filp lookup table which has been added with
395 * drm_gem_handle_create(). If this is the last handle also cleans up linked
396 * resources like GEM names.
397 */
398int
399drm_gem_handle_delete(struct drm_file *filp, u32 handle)
400{
401 struct drm_gem_object *obj;
402
403 spin_lock(&filp->table_lock);
404
405 /* Check if we currently have a reference on the object */
406 obj = idr_replace(&filp->object_idr, NULL, handle);
407 spin_unlock(&filp->table_lock);
408 if (IS_ERR_OR_NULL(obj))
409 return -EINVAL;
410
411 /* Release driver's reference and decrement refcount. */
412 drm_gem_object_release_handle(handle, obj, filp);
413
414 /* And finally make the handle available for future allocations. */
415 spin_lock(&filp->table_lock);
416 idr_remove(&filp->object_idr, handle);
417 spin_unlock(&filp->table_lock);
418
419 return 0;
420}
421EXPORT_SYMBOL(drm_gem_handle_delete);
422
423/**
424 * drm_gem_dumb_map_offset - return the fake mmap offset for a gem object
425 * @file: drm file-private structure containing the gem object
426 * @dev: corresponding drm_device
427 * @handle: gem object handle
428 * @offset: return location for the fake mmap offset
429 *
430 * This implements the &drm_driver.dumb_map_offset kms driver callback for
431 * drivers which use gem to manage their backing storage.
432 *
433 * Returns:
434 * 0 on success or a negative error code on failure.
435 */
436int drm_gem_dumb_map_offset(struct drm_file *file, struct drm_device *dev,
437 u32 handle, u64 *offset)
438{
439 struct drm_gem_object *obj;
440 int ret;
441
442 obj = drm_gem_object_lookup(file, handle);
443 if (!obj)
444 return -ENOENT;
445
446 /* Don't allow imported objects to be mapped */
447 if (drm_gem_is_imported(obj)) {
448 ret = -EINVAL;
449 goto out;
450 }
451
452 ret = drm_gem_create_mmap_offset(obj);
453 if (ret)
454 goto out;
455
456 *offset = drm_vma_node_offset_addr(&obj->vma_node);
457out:
458 drm_gem_object_put(obj);
459
460 return ret;
461}
462EXPORT_SYMBOL_GPL(drm_gem_dumb_map_offset);
463
464/**
465 * drm_gem_handle_create_tail - internal functions to create a handle
466 * @file_priv: drm file-private structure to register the handle for
467 * @obj: object to register
468 * @handlep: pointer to return the created handle to the caller
469 *
470 * This expects the &drm_device.object_name_lock to be held already and will
471 * drop it before returning. Used to avoid races in establishing new handles
472 * when importing an object from either an flink name or a dma-buf.
473 *
474 * Handles must be release again through drm_gem_handle_delete(). This is done
475 * when userspace closes @file_priv for all attached handles, or through the
476 * GEM_CLOSE ioctl for individual handles.
477 */
478int
479drm_gem_handle_create_tail(struct drm_file *file_priv,
480 struct drm_gem_object *obj,
481 u32 *handlep)
482{
483 struct drm_device *dev = obj->dev;
484 u32 handle;
485 int ret;
486
487 WARN_ON(!mutex_is_locked(&dev->object_name_lock));
488
489 drm_gem_object_handle_get(obj);
490
491 /*
492 * Get the user-visible handle using idr. Preload and perform
493 * allocation under our spinlock.
494 */
495 idr_preload(GFP_KERNEL);
496 spin_lock(&file_priv->table_lock);
497
498 ret = idr_alloc(&file_priv->object_idr, NULL, 1, 0, GFP_NOWAIT);
499
500 spin_unlock(&file_priv->table_lock);
501 idr_preload_end();
502
503 mutex_unlock(&dev->object_name_lock);
504 if (ret < 0)
505 goto err_unref;
506
507 handle = ret;
508
509 ret = drm_vma_node_allow(&obj->vma_node, file_priv);
510 if (ret)
511 goto err_remove;
512
513 if (obj->funcs->open) {
514 ret = obj->funcs->open(obj, file_priv);
515 if (ret)
516 goto err_revoke;
517 }
518
519 /* mirrors drm_gem_handle_delete to avoid races */
520 spin_lock(&file_priv->table_lock);
521 obj = idr_replace(&file_priv->object_idr, obj, handle);
522 WARN_ON(obj != NULL);
523 spin_unlock(&file_priv->table_lock);
524 *handlep = handle;
525 return 0;
526
527err_revoke:
528 drm_vma_node_revoke(&obj->vma_node, file_priv);
529err_remove:
530 spin_lock(&file_priv->table_lock);
531 idr_remove(&file_priv->object_idr, handle);
532 spin_unlock(&file_priv->table_lock);
533err_unref:
534 drm_gem_object_handle_put_unlocked(obj);
535 return ret;
536}
537
538/**
539 * drm_gem_handle_create - create a gem handle for an object
540 * @file_priv: drm file-private structure to register the handle for
541 * @obj: object to register
542 * @handlep: pointer to return the created handle to the caller
543 *
544 * Create a handle for this object. This adds a handle reference to the object,
545 * which includes a regular reference count. Callers will likely want to
546 * dereference the object afterwards.
547 *
548 * Since this publishes @obj to userspace it must be fully set up by this point,
549 * drivers must call this last in their buffer object creation callbacks.
550 */
551int drm_gem_handle_create(struct drm_file *file_priv,
552 struct drm_gem_object *obj,
553 u32 *handlep)
554{
555 mutex_lock(&obj->dev->object_name_lock);
556
557 return drm_gem_handle_create_tail(file_priv, obj, handlep);
558}
559EXPORT_SYMBOL(drm_gem_handle_create);
560
561
562/**
563 * drm_gem_free_mmap_offset - release a fake mmap offset for an object
564 * @obj: obj in question
565 *
566 * This routine frees fake offsets allocated by drm_gem_create_mmap_offset().
567 *
568 * Note that drm_gem_object_release() already calls this function, so drivers
569 * don't have to take care of releasing the mmap offset themselves when freeing
570 * the GEM object.
571 */
572void
573drm_gem_free_mmap_offset(struct drm_gem_object *obj)
574{
575 struct drm_device *dev = obj->dev;
576
577 drm_vma_offset_remove(dev->vma_offset_manager, &obj->vma_node);
578}
579EXPORT_SYMBOL(drm_gem_free_mmap_offset);
580
581/**
582 * drm_gem_create_mmap_offset_size - create a fake mmap offset for an object
583 * @obj: obj in question
584 * @size: the virtual size
585 *
586 * GEM memory mapping works by handing back to userspace a fake mmap offset
587 * it can use in a subsequent mmap(2) call. The DRM core code then looks
588 * up the object based on the offset and sets up the various memory mapping
589 * structures.
590 *
591 * This routine allocates and attaches a fake offset for @obj, in cases where
592 * the virtual size differs from the physical size (ie. &drm_gem_object.size).
593 * Otherwise just use drm_gem_create_mmap_offset().
594 *
595 * This function is idempotent and handles an already allocated mmap offset
596 * transparently. Drivers do not need to check for this case.
597 */
598int
599drm_gem_create_mmap_offset_size(struct drm_gem_object *obj, size_t size)
600{
601 struct drm_device *dev = obj->dev;
602
603 return drm_vma_offset_add(dev->vma_offset_manager, &obj->vma_node,
604 size / PAGE_SIZE);
605}
606EXPORT_SYMBOL(drm_gem_create_mmap_offset_size);
607
608/**
609 * drm_gem_create_mmap_offset - create a fake mmap offset for an object
610 * @obj: obj in question
611 *
612 * GEM memory mapping works by handing back to userspace a fake mmap offset
613 * it can use in a subsequent mmap(2) call. The DRM core code then looks
614 * up the object based on the offset and sets up the various memory mapping
615 * structures.
616 *
617 * This routine allocates and attaches a fake offset for @obj.
618 *
619 * Drivers can call drm_gem_free_mmap_offset() before freeing @obj to release
620 * the fake offset again.
621 */
622int drm_gem_create_mmap_offset(struct drm_gem_object *obj)
623{
624 return drm_gem_create_mmap_offset_size(obj, obj->size);
625}
626EXPORT_SYMBOL(drm_gem_create_mmap_offset);
627
628/*
629 * Move folios to appropriate lru and release the folios, decrementing the
630 * ref count of those folios.
631 */
632static void drm_gem_check_release_batch(struct folio_batch *fbatch)
633{
634 check_move_unevictable_folios(fbatch);
635 __folio_batch_release(fbatch);
636 cond_resched();
637}
638
639/**
640 * drm_gem_get_pages - helper to allocate backing pages for a GEM object
641 * from shmem
642 * @obj: obj in question
643 *
644 * This reads the page-array of the shmem-backing storage of the given gem
645 * object. An array of pages is returned. If a page is not allocated or
646 * swapped-out, this will allocate/swap-in the required pages. Note that the
647 * whole object is covered by the page-array and pinned in memory.
648 *
649 * Use drm_gem_put_pages() to release the array and unpin all pages.
650 *
651 * This uses the GFP-mask set on the shmem-mapping (see mapping_set_gfp_mask()).
652 * If you require other GFP-masks, you have to do those allocations yourself.
653 *
654 * Note that you are not allowed to change gfp-zones during runtime. That is,
655 * shmem_read_mapping_page_gfp() must be called with the same gfp_zone(gfp) as
656 * set during initialization. If you have special zone constraints, set them
657 * after drm_gem_object_init() via mapping_set_gfp_mask(). shmem-core takes care
658 * to keep pages in the required zone during swap-in.
659 *
660 * This function is only valid on objects initialized with
661 * drm_gem_object_init(), but not for those initialized with
662 * drm_gem_private_object_init() only.
663 */
664struct page **drm_gem_get_pages(struct drm_gem_object *obj)
665{
666 struct address_space *mapping;
667 struct page **pages;
668 struct folio *folio;
669 struct folio_batch fbatch;
670 unsigned long i, j, npages;
671
672 if (WARN_ON(!obj->filp))
673 return ERR_PTR(-EINVAL);
674
675 /* This is the shared memory object that backs the GEM resource */
676 mapping = obj->filp->f_mapping;
677
678 /* We already BUG_ON() for non-page-aligned sizes in
679 * drm_gem_object_init(), so we should never hit this unless
680 * driver author is doing something really wrong:
681 */
682 WARN_ON((obj->size & (PAGE_SIZE - 1)) != 0);
683
684 npages = obj->size >> PAGE_SHIFT;
685
686 pages = kvmalloc_objs(struct page *, npages);
687 if (pages == NULL)
688 return ERR_PTR(-ENOMEM);
689
690 mapping_set_unevictable(mapping);
691
692 i = 0;
693 while (i < npages) {
694 unsigned long nr;
695 folio = shmem_read_folio_gfp(mapping, i,
696 mapping_gfp_mask(mapping));
697 if (IS_ERR(folio))
698 goto fail;
699 nr = min(npages - i, folio_nr_pages(folio));
700 for (j = 0; j < nr; j++, i++)
701 pages[i] = folio_file_page(folio, i);
702
703 /* Make sure shmem keeps __GFP_DMA32 allocated pages in the
704 * correct region during swapin. Note that this requires
705 * __GFP_DMA32 to be set in mapping_gfp_mask(inode->i_mapping)
706 * so shmem can relocate pages during swapin if required.
707 */
708 BUG_ON(mapping_gfp_constraint(mapping, __GFP_DMA32) &&
709 (folio_pfn(folio) >= 0x00100000UL));
710 }
711
712 return pages;
713
714fail:
715 mapping_clear_unevictable(mapping);
716 folio_batch_init(&fbatch);
717 j = 0;
718 while (j < i) {
719 struct folio *f = page_folio(pages[j]);
720 if (!folio_batch_add(&fbatch, f))
721 drm_gem_check_release_batch(&fbatch);
722 j += folio_nr_pages(f);
723 }
724 if (fbatch.nr)
725 drm_gem_check_release_batch(&fbatch);
726
727 kvfree(pages);
728 return ERR_CAST(folio);
729}
730EXPORT_SYMBOL(drm_gem_get_pages);
731
732/**
733 * drm_gem_put_pages - helper to free backing pages for a GEM object
734 * @obj: obj in question
735 * @pages: pages to free
736 * @dirty: if true, pages will be marked as dirty
737 * @accessed: if true, the pages will be marked as accessed
738 */
739void drm_gem_put_pages(struct drm_gem_object *obj, struct page **pages,
740 bool dirty, bool accessed)
741{
742 int i, npages;
743 struct address_space *mapping;
744 struct folio_batch fbatch;
745
746 mapping = file_inode(obj->filp)->i_mapping;
747 mapping_clear_unevictable(mapping);
748
749 /* We already BUG_ON() for non-page-aligned sizes in
750 * drm_gem_object_init(), so we should never hit this unless
751 * driver author is doing something really wrong:
752 */
753 WARN_ON((obj->size & (PAGE_SIZE - 1)) != 0);
754
755 npages = obj->size >> PAGE_SHIFT;
756
757 folio_batch_init(&fbatch);
758 for (i = 0; i < npages; i++) {
759 struct folio *folio;
760
761 if (!pages[i])
762 continue;
763 folio = page_folio(pages[i]);
764
765 if (dirty)
766 folio_mark_dirty(folio);
767
768 if (accessed)
769 folio_mark_accessed(folio);
770
771 /* Undo the reference we took when populating the table */
772 if (!folio_batch_add(&fbatch, folio))
773 drm_gem_check_release_batch(&fbatch);
774 i += folio_nr_pages(folio) - 1;
775 }
776 if (folio_batch_count(&fbatch))
777 drm_gem_check_release_batch(&fbatch);
778
779 kvfree(pages);
780}
781EXPORT_SYMBOL(drm_gem_put_pages);
782
783static int objects_lookup(struct drm_file *filp, u32 *handle, int count,
784 struct drm_gem_object **objs)
785{
786 int i;
787 struct drm_gem_object *obj;
788
789 spin_lock(&filp->table_lock);
790
791 for (i = 0; i < count; i++) {
792 /* Check if we currently have a reference on the object */
793 obj = idr_find(&filp->object_idr, handle[i]);
794 if (!obj)
795 goto err;
796
797 drm_gem_object_get(obj);
798 objs[i] = obj;
799 }
800
801 spin_unlock(&filp->table_lock);
802 return 0;
803
804err:
805 spin_unlock(&filp->table_lock);
806
807 while (i--)
808 drm_gem_object_put(objs[i]);
809
810 return -ENOENT;
811}
812
813/**
814 * drm_gem_objects_lookup - look up GEM objects from an array of handles
815 * @filp: DRM file private date
816 * @bo_handles: user pointer to array of userspace handle
817 * @count: size of handle array
818 * @objs_out: returned pointer to array of drm_gem_object pointers
819 *
820 * Takes an array of userspace handles and returns a newly allocated array of
821 * GEM objects.
822 *
823 * After a successful lookup GEM objects need to be released using
824 * drm_gem_object_put() and the array returned in @objs_out must be freed using
825 * kvfree().
826 *
827 * For a single handle lookup, use drm_gem_object_lookup().
828 *
829 * Return:
830 * Zero on success or a negative error code.
831 */
832int drm_gem_objects_lookup(struct drm_file *filp, void __user *bo_handles,
833 int count, struct drm_gem_object ***objs_out)
834{
835 struct drm_gem_object **objs;
836 u32 *handles;
837 int ret;
838
839 *objs_out = NULL;
840
841 if (!count)
842 return 0;
843
844 objs = kvmalloc_objs(*objs, count);
845 if (!objs)
846 return -ENOMEM;
847
848 handles = vmemdup_array_user(bo_handles, count, sizeof(u32));
849 if (IS_ERR(handles)) {
850 ret = PTR_ERR(handles);
851 goto err_free_objs;
852 }
853
854 ret = objects_lookup(filp, handles, count, objs);
855 if (ret)
856 goto err_free_handles;
857
858 kvfree(handles);
859 *objs_out = objs;
860 return 0;
861
862err_free_handles:
863 kvfree(handles);
864err_free_objs:
865 kvfree(objs);
866 return ret;
867}
868EXPORT_SYMBOL(drm_gem_objects_lookup);
869
870/**
871 * drm_gem_object_lookup - look up a GEM object from its handle
872 * @filp: DRM file private date
873 * @handle: userspace handle
874 *
875 * If looking up an array of handles, use drm_gem_objects_lookup().
876 *
877 * Returns:
878 * A reference to the object named by the handle if such exists on @filp, NULL
879 * otherwise.
880 */
881struct drm_gem_object *
882drm_gem_object_lookup(struct drm_file *filp, u32 handle)
883{
884 struct drm_gem_object *obj = NULL;
885
886 objects_lookup(filp, &handle, 1, &obj);
887 return obj;
888}
889EXPORT_SYMBOL(drm_gem_object_lookup);
890
891/**
892 * drm_gem_dma_resv_wait - Wait on GEM object's reservation's objects
893 * shared and/or exclusive fences.
894 * @filep: DRM file private date
895 * @handle: userspace handle
896 * @wait_all: if true, wait on all fences, else wait on just exclusive fence
897 * @timeout: timeout value in jiffies or zero to return immediately
898 *
899 * Returns:
900 * Returns -ERESTARTSYS if interrupted, 0 if the wait timed out, or
901 * greater than 0 on success.
902 */
903long drm_gem_dma_resv_wait(struct drm_file *filep, u32 handle,
904 bool wait_all, unsigned long timeout)
905{
906 struct drm_device *dev = filep->minor->dev;
907 struct drm_gem_object *obj;
908 long ret;
909
910 obj = drm_gem_object_lookup(filep, handle);
911 if (!obj) {
912 drm_dbg_core(dev, "Failed to look up GEM BO %d\n", handle);
913 return -EINVAL;
914 }
915
916 ret = dma_resv_wait_timeout(obj->resv, dma_resv_usage_rw(wait_all),
917 true, timeout);
918 if (ret == 0)
919 ret = -ETIME;
920 else if (ret > 0)
921 ret = 0;
922
923 drm_gem_object_put(obj);
924
925 return ret;
926}
927EXPORT_SYMBOL(drm_gem_dma_resv_wait);
928
929int
930drm_gem_close_ioctl(struct drm_device *dev, void *data,
931 struct drm_file *file_priv)
932{
933 struct drm_gem_close *args = data;
934 int ret;
935
936 if (!drm_core_check_feature(dev, DRIVER_GEM))
937 return -EOPNOTSUPP;
938
939 ret = drm_gem_handle_delete(file_priv, args->handle);
940
941 return ret;
942}
943
944int
945drm_gem_flink_ioctl(struct drm_device *dev, void *data,
946 struct drm_file *file_priv)
947{
948 struct drm_gem_flink *args = data;
949 struct drm_gem_object *obj;
950 int ret;
951
952 if (!drm_core_check_feature(dev, DRIVER_GEM))
953 return -EOPNOTSUPP;
954
955 obj = drm_gem_object_lookup(file_priv, args->handle);
956 if (obj == NULL)
957 return -ENOENT;
958
959 mutex_lock(&dev->object_name_lock);
960 /* prevent races with concurrent gem_close. */
961 if (obj->handle_count == 0) {
962 ret = -ENOENT;
963 goto err;
964 }
965
966 if (!obj->name) {
967 ret = idr_alloc(&dev->object_name_idr, obj, 1, 0, GFP_KERNEL);
968 if (ret < 0)
969 goto err;
970
971 obj->name = ret;
972 }
973
974 args->name = (uint64_t) obj->name;
975 ret = 0;
976
977err:
978 mutex_unlock(&dev->object_name_lock);
979 drm_gem_object_put(obj);
980 return ret;
981}
982
983int
984drm_gem_open_ioctl(struct drm_device *dev, void *data,
985 struct drm_file *file_priv)
986{
987 struct drm_gem_open *args = data;
988 struct drm_gem_object *obj;
989 int ret;
990 u32 handle;
991
992 if (!drm_core_check_feature(dev, DRIVER_GEM))
993 return -EOPNOTSUPP;
994
995 mutex_lock(&dev->object_name_lock);
996 obj = idr_find(&dev->object_name_idr, (int) args->name);
997 if (obj) {
998 drm_gem_object_get(obj);
999 } else {
1000 mutex_unlock(&dev->object_name_lock);
1001 return -ENOENT;
1002 }
1003
1004 /* drm_gem_handle_create_tail unlocks dev->object_name_lock. */
1005 ret = drm_gem_handle_create_tail(file_priv, obj, &handle);
1006 if (ret)
1007 goto err;
1008
1009 args->handle = handle;
1010 args->size = obj->size;
1011
1012err:
1013 drm_gem_object_put(obj);
1014 return ret;
1015}
1016
1017/*
1018 * This ioctl is disabled for security reasons but also it failed
1019 * to follow process in terms of adding testing in igt and verifying
1020 * all the corner cases which made fixing security bugs in it even
1021 * harder than necessary.
1022 *
1023 * To re-enable this ioctl
1024 * 1. land working IGT tests in igt-gpu-tools that cover
1025 * all corner cases and race conditions.
1026 * 2. handle idr_preload
1027 * 3. handle == 0
1028 * 4. handle == new_handle semantics definition.
1029 */
1030int drm_gem_change_handle_ioctl(struct drm_device *dev, void *data,
1031 struct drm_file *file_priv)
1032{
1033 struct drm_gem_change_handle *args = data;
1034 struct drm_gem_object *obj;
1035 int new_handle, ret;
1036
1037 if (!drm_core_check_feature(dev, DRIVER_GEM))
1038 return -EOPNOTSUPP;
1039
1040 /* idr_alloc() limitation. */
1041 if (args->new_handle > INT_MAX)
1042 return -EINVAL;
1043 new_handle = args->new_handle;
1044
1045 if (args->handle == new_handle)
1046 return 0;
1047
1048 mutex_lock(&file_priv->prime.lock);
1049 spin_lock(&file_priv->table_lock);
1050 ret = idr_alloc(&file_priv->object_idr, NULL, new_handle, new_handle + 1,
1051 GFP_NOWAIT);
1052
1053 if (ret < 0) {
1054 spin_unlock(&file_priv->table_lock);
1055 goto out_unlock;
1056 }
1057
1058 obj = idr_replace(&file_priv->object_idr, NULL, args->handle);
1059 if (IS_ERR_OR_NULL(obj)) {
1060 idr_remove(&file_priv->object_idr, new_handle);
1061 spin_unlock(&file_priv->table_lock);
1062 ret = -ENOENT;
1063 goto out_unlock;
1064 }
1065 spin_unlock(&file_priv->table_lock);
1066
1067 if (obj->dma_buf) {
1068 ret = drm_prime_add_buf_handle(&file_priv->prime, obj->dma_buf,
1069 new_handle);
1070 if (ret < 0) {
1071 spin_lock(&file_priv->table_lock);
1072 idr_remove(&file_priv->object_idr, new_handle);
1073 idr_replace(&file_priv->object_idr, obj, args->handle);
1074 spin_unlock(&file_priv->table_lock);
1075 goto out_unlock;
1076 }
1077
1078 drm_prime_remove_buf_handle(&file_priv->prime, args->handle);
1079 }
1080
1081 ret = 0;
1082
1083 spin_lock(&file_priv->table_lock);
1084 idr_remove(&file_priv->object_idr, args->handle);
1085 obj = idr_replace(&file_priv->object_idr, obj, new_handle);
1086 spin_unlock(&file_priv->table_lock);
1087 WARN_ON(obj != NULL);
1088
1089out_unlock:
1090 mutex_unlock(&file_priv->prime.lock);
1091
1092 return ret;
1093}
1094
1095/**
1096 * drm_gem_open - initializes GEM file-private structures at devnode open time
1097 * @dev: drm_device which is being opened by userspace
1098 * @file_private: drm file-private structure to set up
1099 *
1100 * Called at device open time, sets up the structure for handling refcounting
1101 * of mm objects.
1102 */
1103void
1104drm_gem_open(struct drm_device *dev, struct drm_file *file_private)
1105{
1106 idr_init_base(&file_private->object_idr, 1);
1107 spin_lock_init(&file_private->table_lock);
1108}
1109
1110/**
1111 * drm_gem_release - release file-private GEM resources
1112 * @dev: drm_device which is being closed by userspace
1113 * @file_private: drm file-private structure to clean up
1114 *
1115 * Called at close time when the filp is going away.
1116 *
1117 * Releases any remaining references on objects by this filp.
1118 */
1119void
1120drm_gem_release(struct drm_device *dev, struct drm_file *file_private)
1121{
1122 idr_for_each(&file_private->object_idr,
1123 &drm_gem_object_release_handle, file_private);
1124 idr_destroy(&file_private->object_idr);
1125}
1126
1127/**
1128 * drm_gem_object_release - release GEM buffer object resources
1129 * @obj: GEM buffer object
1130 *
1131 * This releases any structures and resources used by @obj and is the inverse of
1132 * drm_gem_object_init().
1133 */
1134void
1135drm_gem_object_release(struct drm_gem_object *obj)
1136{
1137 if (obj->filp)
1138 fput(obj->filp);
1139
1140 drm_gem_private_object_fini(obj);
1141
1142 drm_gem_free_mmap_offset(obj);
1143 drm_gem_lru_remove(obj);
1144}
1145EXPORT_SYMBOL(drm_gem_object_release);
1146
1147/**
1148 * drm_gem_object_free - free a GEM object
1149 * @kref: kref of the object to free
1150 *
1151 * Called after the last reference to the object has been lost.
1152 *
1153 * Frees the object
1154 */
1155void
1156drm_gem_object_free(struct kref *kref)
1157{
1158 struct drm_gem_object *obj =
1159 container_of(kref, struct drm_gem_object, refcount);
1160
1161 if (WARN_ON(!obj->funcs->free))
1162 return;
1163
1164 obj->funcs->free(obj);
1165}
1166EXPORT_SYMBOL(drm_gem_object_free);
1167
1168/**
1169 * drm_gem_vm_open - vma->ops->open implementation for GEM
1170 * @vma: VM area structure
1171 *
1172 * This function implements the #vm_operations_struct open() callback for GEM
1173 * drivers. This must be used together with drm_gem_vm_close().
1174 */
1175void drm_gem_vm_open(struct vm_area_struct *vma)
1176{
1177 struct drm_gem_object *obj = vma->vm_private_data;
1178
1179 drm_gem_object_get(obj);
1180}
1181EXPORT_SYMBOL(drm_gem_vm_open);
1182
1183/**
1184 * drm_gem_vm_close - vma->ops->close implementation for GEM
1185 * @vma: VM area structure
1186 *
1187 * This function implements the #vm_operations_struct close() callback for GEM
1188 * drivers. This must be used together with drm_gem_vm_open().
1189 */
1190void drm_gem_vm_close(struct vm_area_struct *vma)
1191{
1192 struct drm_gem_object *obj = vma->vm_private_data;
1193
1194 drm_gem_object_put(obj);
1195}
1196EXPORT_SYMBOL(drm_gem_vm_close);
1197
1198/**
1199 * drm_gem_mmap_obj - memory map a GEM object
1200 * @obj: the GEM object to map
1201 * @obj_size: the object size to be mapped, in bytes
1202 * @vma: VMA for the area to be mapped
1203 *
1204 * Set up the VMA to prepare mapping of the GEM object using the GEM object's
1205 * vm_ops. Depending on their requirements, GEM objects can either
1206 * provide a fault handler in their vm_ops (in which case any accesses to
1207 * the object will be trapped, to perform migration, GTT binding, surface
1208 * register allocation, or performance monitoring), or mmap the buffer memory
1209 * synchronously after calling drm_gem_mmap_obj.
1210 *
1211 * This function is mainly intended to implement the DMABUF mmap operation, when
1212 * the GEM object is not looked up based on its fake offset. To implement the
1213 * DRM mmap operation, drivers should use the drm_gem_mmap() function.
1214 *
1215 * drm_gem_mmap_obj() assumes the user is granted access to the buffer while
1216 * drm_gem_mmap() prevents unprivileged users from mapping random objects. So
1217 * callers must verify access restrictions before calling this helper.
1218 *
1219 * Return 0 or success or -EINVAL if the object size is smaller than the VMA
1220 * size, or if no vm_ops are provided.
1221 */
1222int drm_gem_mmap_obj(struct drm_gem_object *obj, unsigned long obj_size,
1223 struct vm_area_struct *vma)
1224{
1225 int ret;
1226
1227 /* Check for valid size. */
1228 if (obj_size < vma->vm_end - vma->vm_start)
1229 return -EINVAL;
1230
1231 /* Take a ref for this mapping of the object, so that the fault
1232 * handler can dereference the mmap offset's pointer to the object.
1233 * This reference is cleaned up by the corresponding vm_close
1234 * (which should happen whether the vma was created by this call, or
1235 * by a vm_open due to mremap or partial unmap or whatever).
1236 */
1237 drm_gem_object_get(obj);
1238
1239 vma->vm_private_data = obj;
1240 vma->vm_ops = obj->funcs->vm_ops;
1241
1242 if (obj->funcs->mmap) {
1243 ret = obj->funcs->mmap(obj, vma);
1244 if (ret)
1245 goto err_drm_gem_object_put;
1246 WARN_ON(!(vma->vm_flags & VM_DONTEXPAND));
1247 } else {
1248 if (!vma->vm_ops) {
1249 ret = -EINVAL;
1250 goto err_drm_gem_object_put;
1251 }
1252
1253 vm_flags_set(vma, VM_IO | VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP);
1254 vma->vm_page_prot = pgprot_writecombine(vma_get_page_prot(vma));
1255 vma->vm_page_prot = pgprot_decrypted(vma->vm_page_prot);
1256 }
1257
1258 return 0;
1259
1260err_drm_gem_object_put:
1261 drm_gem_object_put(obj);
1262 return ret;
1263}
1264EXPORT_SYMBOL(drm_gem_mmap_obj);
1265
1266/*
1267 * Look up a GEM object in offset space based on the exact start address. The
1268 * caller must be granted access to the object. Returns a GEM object on success
1269 * or a negative error code on failure. The returned GEM object needs to be
1270 * released with drm_gem_object_put().
1271 */
1272static struct drm_gem_object *
1273drm_gem_object_lookup_at_offset(struct file *filp, unsigned long start,
1274 unsigned long pages)
1275{
1276 struct drm_file *priv = filp->private_data;
1277 struct drm_device *dev = priv->minor->dev;
1278 struct drm_gem_object *obj = NULL;
1279 struct drm_vma_offset_node *node;
1280
1281 if (drm_dev_is_unplugged(dev))
1282 return ERR_PTR(-ENODEV);
1283
1284 drm_vma_offset_lock_lookup(dev->vma_offset_manager);
1285 node = drm_vma_offset_exact_lookup_locked(dev->vma_offset_manager,
1286 start, pages);
1287 if (likely(node)) {
1288 obj = container_of(node, struct drm_gem_object, vma_node);
1289 /*
1290 * When the object is being freed, after it hits 0-refcnt it
1291 * proceeds to tear down the object. In the process it will
1292 * attempt to remove the VMA offset and so acquire this
1293 * mgr->vm_lock. Therefore if we find an object with a 0-refcnt
1294 * that matches our range, we know it is in the process of being
1295 * destroyed and will be freed as soon as we release the lock -
1296 * so we have to check for the 0-refcnted object and treat it as
1297 * invalid.
1298 */
1299 if (!kref_get_unless_zero(&obj->refcount))
1300 obj = NULL;
1301 }
1302 drm_vma_offset_unlock_lookup(dev->vma_offset_manager);
1303
1304 if (!obj)
1305 return ERR_PTR(-EINVAL);
1306
1307 if (!drm_vma_node_is_allowed(node, priv)) {
1308 drm_gem_object_put(obj);
1309 return ERR_PTR(-EACCES);
1310 }
1311
1312 return obj;
1313}
1314
1315#ifdef CONFIG_MMU
1316/**
1317 * drm_gem_get_unmapped_area - get memory mapping region routine for GEM objects
1318 * @filp: DRM file pointer
1319 * @uaddr: User address hint
1320 * @len: Mapping length
1321 * @pgoff: Offset (in pages)
1322 * @flags: Mapping flags
1323 *
1324 * If a driver supports GEM object mapping, before ending up in drm_gem_mmap(),
1325 * mmap calls on the DRM file descriptor will first try to find a free linear
1326 * address space large enough for a mapping. Since GEM objects are backed by
1327 * shmem buffers, this should preferably be handled by the shmem virtual memory
1328 * filesystem which can appropriately align addresses to huge page sizes when
1329 * needed.
1330 *
1331 * Look up the GEM object based on the offset passed in (vma->vm_pgoff will
1332 * contain the fake offset we created) and call shmem_get_unmapped_area() with
1333 * the right file pointer.
1334 *
1335 * If a GEM object is not available at the given offset or if the caller is not
1336 * granted access to it, fall back to mm_get_unmapped_area().
1337 */
1338unsigned long drm_gem_get_unmapped_area(struct file *filp, unsigned long uaddr,
1339 unsigned long len, unsigned long pgoff,
1340 unsigned long flags)
1341{
1342 struct drm_gem_object *obj;
1343 unsigned long ret;
1344
1345 obj = drm_gem_object_lookup_at_offset(filp, pgoff, len >> PAGE_SHIFT);
1346 if (IS_ERR(obj))
1347 obj = NULL;
1348
1349 if (!obj || !obj->filp || !obj->filp->f_op->get_unmapped_area)
1350 ret = mm_get_unmapped_area(filp, uaddr, len, 0, flags);
1351 else
1352 ret = obj->filp->f_op->get_unmapped_area(obj->filp, uaddr, len, 0, flags);
1353
1354 drm_gem_object_put(obj);
1355
1356 return ret;
1357}
1358EXPORT_SYMBOL_GPL(drm_gem_get_unmapped_area);
1359#endif
1360
1361/**
1362 * drm_gem_mmap - memory map routine for GEM objects
1363 * @filp: DRM file pointer
1364 * @vma: VMA for the area to be mapped
1365 *
1366 * If a driver supports GEM object mapping, mmap calls on the DRM file
1367 * descriptor will end up here.
1368 *
1369 * Look up the GEM object based on the offset passed in (vma->vm_pgoff will
1370 * contain the fake offset we created) and map it with a call to
1371 * drm_gem_mmap_obj().
1372 *
1373 * If the caller is not granted access to the buffer object, the mmap will fail
1374 * with EACCES. Please see the vma manager for more information.
1375 */
1376int drm_gem_mmap(struct file *filp, struct vm_area_struct *vma)
1377{
1378 struct drm_gem_object *obj;
1379 int ret;
1380
1381 obj = drm_gem_object_lookup_at_offset(filp, vma->vm_pgoff,
1382 vma_pages(vma));
1383 if (IS_ERR(obj))
1384 return PTR_ERR(obj);
1385
1386 ret = drm_gem_mmap_obj(obj,
1387 drm_vma_node_size(&obj->vma_node) << PAGE_SHIFT,
1388 vma);
1389
1390 drm_gem_object_put(obj);
1391
1392 return ret;
1393}
1394EXPORT_SYMBOL(drm_gem_mmap);
1395
1396void drm_gem_print_info(struct drm_printer *p, unsigned int indent,
1397 const struct drm_gem_object *obj)
1398{
1399 drm_printf_indent(p, indent, "name=%d\n", obj->name);
1400 drm_printf_indent(p, indent, "refcount=%u\n",
1401 kref_read(&obj->refcount));
1402 drm_printf_indent(p, indent, "start=%08lx\n",
1403 drm_vma_node_start(&obj->vma_node));
1404 drm_printf_indent(p, indent, "size=%zu\n", obj->size);
1405 drm_printf_indent(p, indent, "imported=%s\n",
1406 str_yes_no(drm_gem_is_imported(obj)));
1407
1408 if (obj->funcs->print_info)
1409 obj->funcs->print_info(p, indent, obj);
1410}
1411
1412int drm_gem_vmap_locked(struct drm_gem_object *obj, struct iosys_map *map)
1413{
1414 int ret;
1415
1416 dma_resv_assert_held(obj->resv);
1417
1418 if (!obj->funcs->vmap)
1419 return -EOPNOTSUPP;
1420
1421 ret = obj->funcs->vmap(obj, map);
1422 if (ret)
1423 return ret;
1424 else if (iosys_map_is_null(map))
1425 return -ENOMEM;
1426
1427 return 0;
1428}
1429EXPORT_SYMBOL(drm_gem_vmap_locked);
1430
1431void drm_gem_vunmap_locked(struct drm_gem_object *obj, struct iosys_map *map)
1432{
1433 dma_resv_assert_held(obj->resv);
1434
1435 if (iosys_map_is_null(map))
1436 return;
1437
1438 if (obj->funcs->vunmap)
1439 obj->funcs->vunmap(obj, map);
1440
1441 /* Always set the mapping to NULL. Callers may rely on this. */
1442 iosys_map_clear(map);
1443}
1444EXPORT_SYMBOL(drm_gem_vunmap_locked);
1445
1446void drm_gem_lock(struct drm_gem_object *obj)
1447{
1448 dma_resv_lock(obj->resv, NULL);
1449}
1450EXPORT_SYMBOL(drm_gem_lock);
1451
1452void drm_gem_unlock(struct drm_gem_object *obj)
1453{
1454 dma_resv_unlock(obj->resv);
1455}
1456EXPORT_SYMBOL(drm_gem_unlock);
1457
1458int drm_gem_vmap(struct drm_gem_object *obj, struct iosys_map *map)
1459{
1460 int ret;
1461
1462 dma_resv_lock(obj->resv, NULL);
1463 ret = drm_gem_vmap_locked(obj, map);
1464 dma_resv_unlock(obj->resv);
1465
1466 return ret;
1467}
1468EXPORT_SYMBOL(drm_gem_vmap);
1469
1470void drm_gem_vunmap(struct drm_gem_object *obj, struct iosys_map *map)
1471{
1472 dma_resv_lock(obj->resv, NULL);
1473 drm_gem_vunmap_locked(obj, map);
1474 dma_resv_unlock(obj->resv);
1475}
1476EXPORT_SYMBOL(drm_gem_vunmap);
1477
1478/**
1479 * drm_gem_lock_reservations - Sets up the ww context and acquires
1480 * the lock on an array of GEM objects.
1481 *
1482 * Once you've locked your reservations, you'll want to set up space
1483 * for your shared fences (if applicable), submit your job, then
1484 * drm_gem_unlock_reservations().
1485 *
1486 * @objs: drm_gem_objects to lock
1487 * @count: Number of objects in @objs
1488 * @acquire_ctx: struct ww_acquire_ctx that will be initialized as
1489 * part of tracking this set of locked reservations.
1490 */
1491int
1492drm_gem_lock_reservations(struct drm_gem_object **objs, int count,
1493 struct ww_acquire_ctx *acquire_ctx)
1494{
1495 int contended = -1;
1496 int i, ret;
1497
1498 ww_acquire_init(acquire_ctx, &reservation_ww_class);
1499
1500retry:
1501 if (contended != -1) {
1502 struct drm_gem_object *obj = objs[contended];
1503
1504 ret = dma_resv_lock_slow_interruptible(obj->resv,
1505 acquire_ctx);
1506 if (ret) {
1507 ww_acquire_fini(acquire_ctx);
1508 return ret;
1509 }
1510 }
1511
1512 for (i = 0; i < count; i++) {
1513 if (i == contended)
1514 continue;
1515
1516 ret = dma_resv_lock_interruptible(objs[i]->resv,
1517 acquire_ctx);
1518 if (ret) {
1519 int j;
1520
1521 for (j = 0; j < i; j++)
1522 dma_resv_unlock(objs[j]->resv);
1523
1524 if (contended != -1 && contended >= i)
1525 dma_resv_unlock(objs[contended]->resv);
1526
1527 if (ret == -EDEADLK) {
1528 contended = i;
1529 goto retry;
1530 }
1531
1532 ww_acquire_fini(acquire_ctx);
1533 return ret;
1534 }
1535 }
1536
1537 ww_acquire_done(acquire_ctx);
1538
1539 return 0;
1540}
1541EXPORT_SYMBOL(drm_gem_lock_reservations);
1542
1543void
1544drm_gem_unlock_reservations(struct drm_gem_object **objs, int count,
1545 struct ww_acquire_ctx *acquire_ctx)
1546{
1547 int i;
1548
1549 for (i = 0; i < count; i++)
1550 dma_resv_unlock(objs[i]->resv);
1551
1552 ww_acquire_fini(acquire_ctx);
1553}
1554EXPORT_SYMBOL(drm_gem_unlock_reservations);
1555
1556/**
1557 * drm_gem_lru_init - initialize a LRU
1558 *
1559 * @lru: The LRU to initialize
1560 */
1561void
1562drm_gem_lru_init(struct drm_gem_lru *lru)
1563{
1564 lru->count = 0;
1565 INIT_LIST_HEAD(&lru->list);
1566}
1567EXPORT_SYMBOL(drm_gem_lru_init);
1568
1569static void
1570drm_gem_lru_remove_locked(struct drm_gem_object *obj)
1571{
1572 obj->lru->count -= obj->size >> PAGE_SHIFT;
1573 WARN_ON(obj->lru->count < 0);
1574 list_del(&obj->lru_node);
1575 obj->lru = NULL;
1576}
1577
1578/**
1579 * drm_gem_lru_remove - remove object from whatever LRU it is in
1580 *
1581 * If the object is currently in any LRU, remove it.
1582 *
1583 * @obj: The GEM object to remove from current LRU
1584 */
1585void
1586drm_gem_lru_remove(struct drm_gem_object *obj)
1587{
1588 mutex_lock(&obj->dev->gem_lru_mutex);
1589 if (obj->lru)
1590 drm_gem_lru_remove_locked(obj);
1591 mutex_unlock(&obj->dev->gem_lru_mutex);
1592}
1593EXPORT_SYMBOL(drm_gem_lru_remove);
1594
1595/**
1596 * drm_gem_lru_move_tail_locked - move the object to the tail of the LRU
1597 *
1598 * Like &drm_gem_lru_move_tail but lru lock must be held
1599 *
1600 * @lru: The LRU to move the object into.
1601 * @obj: The GEM object to move into this LRU
1602 */
1603void
1604drm_gem_lru_move_tail_locked(struct drm_gem_lru *lru, struct drm_gem_object *obj)
1605{
1606 lockdep_assert_held_once(&obj->dev->gem_lru_mutex);
1607
1608 if (obj->lru)
1609 drm_gem_lru_remove_locked(obj);
1610
1611 lru->count += obj->size >> PAGE_SHIFT;
1612 list_add_tail(&obj->lru_node, &lru->list);
1613 obj->lru = lru;
1614}
1615EXPORT_SYMBOL(drm_gem_lru_move_tail_locked);
1616
1617/**
1618 * drm_gem_lru_move_tail - move the object to the tail of the LRU
1619 *
1620 * If the object is already in this LRU it will be moved to the
1621 * tail. Otherwise it will be removed from whichever other LRU
1622 * it is in (if any) and moved into this LRU.
1623 *
1624 * @lru: The LRU to move the object into.
1625 * @obj: The GEM object to move into this LRU
1626 */
1627void
1628drm_gem_lru_move_tail(struct drm_gem_lru *lru, struct drm_gem_object *obj)
1629{
1630 mutex_lock(&obj->dev->gem_lru_mutex);
1631 drm_gem_lru_move_tail_locked(lru, obj);
1632 mutex_unlock(&obj->dev->gem_lru_mutex);
1633}
1634EXPORT_SYMBOL(drm_gem_lru_move_tail);
1635
1636/**
1637 * drm_gem_lru_scan - helper to implement shrinker.scan_objects
1638 *
1639 * If the shrink callback succeeds, it is expected that the driver
1640 * move the object out of this LRU.
1641 *
1642 * If the LRU possibly contain active buffers, it is the responsibility
1643 * of the shrink callback to check for this (ie. dma_resv_test_signaled())
1644 * or if necessary block until the buffer becomes idle.
1645 *
1646 * @dev: DRM device the LRU belongs to
1647 * @lru: The LRU to scan
1648 * @nr_to_scan: The number of pages to try to reclaim
1649 * @remaining: The number of pages left to reclaim, should be initialized by caller
1650 * @shrink: Callback to try to shrink/reclaim the object.
1651 * @ticket: Optional ww_acquire_ctx context to use for locking
1652 */
1653unsigned long
1654drm_gem_lru_scan(struct drm_device *dev,
1655 struct drm_gem_lru *lru,
1656 unsigned int nr_to_scan,
1657 unsigned long *remaining,
1658 bool (*shrink)(struct drm_gem_object *obj, struct ww_acquire_ctx *ticket),
1659 struct ww_acquire_ctx *ticket)
1660{
1661 struct drm_gem_lru still_in_lru;
1662 struct drm_gem_object *obj;
1663 unsigned freed = 0;
1664
1665 drm_gem_lru_init(&still_in_lru);
1666
1667 mutex_lock(&dev->gem_lru_mutex);
1668
1669 while (freed < nr_to_scan) {
1670 obj = list_first_entry_or_null(&lru->list, typeof(*obj), lru_node);
1671
1672 if (!obj)
1673 break;
1674
1675 drm_gem_lru_move_tail_locked(&still_in_lru, obj);
1676
1677 /*
1678 * If it's in the process of being freed, gem_object->free()
1679 * may be blocked on lock waiting to remove it. So just
1680 * skip it.
1681 */
1682 if (!kref_get_unless_zero(&obj->refcount))
1683 continue;
1684
1685 /*
1686 * Now that we own a reference, we can drop the lock for the
1687 * rest of the loop body, to reduce contention with other
1688 * code paths that need the LRU lock
1689 */
1690 mutex_unlock(&dev->gem_lru_mutex);
1691
1692 if (ticket)
1693 ww_acquire_init(ticket, &reservation_ww_class);
1694
1695 /*
1696 * Note that this still needs to be trylock, since we can
1697 * hit shrinker in response to trying to get backing pages
1698 * for this obj (ie. while it's lock is already held)
1699 */
1700 if (!ww_mutex_trylock(&obj->resv->lock, ticket)) {
1701 *remaining += obj->size >> PAGE_SHIFT;
1702 goto tail;
1703 }
1704
1705 if (shrink(obj, ticket)) {
1706 freed += obj->size >> PAGE_SHIFT;
1707
1708 /*
1709 * If we succeeded in releasing the object's backing
1710 * pages, we expect the driver to have moved the object
1711 * out of this LRU
1712 */
1713 WARN_ON(obj->lru == &still_in_lru);
1714 WARN_ON(obj->lru == lru);
1715 } else if (obj->lru == &still_in_lru) {
1716 /*
1717 * If the object wasn't moved and wasn't shrunk either,
1718 * it's still remaining as reclaimable. Note that
1719 * obj->lru is supposed to be checked with the LRU lock
1720 * held for an accurate result, but we don't care about
1721 * accuracy here. Worst thing that could happen is an
1722 * extra scan.
1723 */
1724 *remaining += obj->size >> PAGE_SHIFT;
1725 }
1726
1727 dma_resv_unlock(obj->resv);
1728
1729 if (ticket)
1730 ww_acquire_fini(ticket);
1731
1732tail:
1733 drm_gem_object_put(obj);
1734 mutex_lock(&dev->gem_lru_mutex);
1735 }
1736
1737 /*
1738 * Move objects we've skipped over out of the temporary still_in_lru
1739 * back into this LRU
1740 */
1741 list_for_each_entry (obj, &still_in_lru.list, lru_node)
1742 obj->lru = lru;
1743 list_splice_tail(&still_in_lru.list, &lru->list);
1744 lru->count += still_in_lru.count;
1745
1746 mutex_unlock(&dev->gem_lru_mutex);
1747
1748 return freed;
1749}
1750EXPORT_SYMBOL(drm_gem_lru_scan);
1751
1752/**
1753 * drm_gem_evict_locked - helper to evict backing pages for a GEM object
1754 * @obj: obj in question
1755 */
1756int drm_gem_evict_locked(struct drm_gem_object *obj)
1757{
1758 dma_resv_assert_held(obj->resv);
1759
1760 if (!dma_resv_test_signaled(obj->resv, DMA_RESV_USAGE_READ))
1761 return -EBUSY;
1762
1763 if (obj->funcs->evict)
1764 return obj->funcs->evict(obj);
1765
1766 return 0;
1767}
1768EXPORT_SYMBOL(drm_gem_evict_locked);