projects/linux·blob master · refs

1398 lines · 36.5 KiB

1/*
2 * Copyright 2008 Advanced Micro Devices, Inc.
3 * Copyright 2008 Red Hat Inc.
4 * Copyright 2009 Jerome Glisse.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the "Software"),
8 * to deal in the Software without restriction, including without limitation
9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10 * and/or sell copies of the Software, and to permit persons to whom the
11 * Software is furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
20 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
21 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
22 * OTHER DEALINGS IN THE SOFTWARE.
23 *
24 * Authors: Dave Airlie
25 * Alex Deucher
26 * Jerome Glisse
27 */
28#include <linux/ktime.h>
29#include <linux/module.h>
30#include <linux/overflow.h>
31#include <linux/pagemap.h>
32#include <linux/pci.h>
33#include <linux/dma-buf.h>
34#include <linux/dma-fence-unwrap.h>
35#include <linux/uaccess.h>
36
37#include <drm/amdgpu_drm.h>
38#include <drm/drm_drv.h>
39#include <drm/drm_exec.h>
40#include <drm/drm_gem_ttm_helper.h>
41#include <drm/ttm/ttm_tt.h>
42#include <drm/drm_syncobj.h>
43
44#include "amdgpu.h"
45#include "amdgpu_display.h"
46#include "amdgpu_dma_buf.h"
47#include "amdgpu_hmm.h"
48#include "amdgpu_xgmi.h"
49#include "amdgpu_vm.h"
50
51static int
52amdgpu_gem_add_input_fence(struct drm_file *filp,
53 uint64_t syncobj_handles_array,
54 uint32_t num_syncobj_handles)
55{
56 struct dma_fence *fence;
57 uint32_t *syncobj_handles;
58 int ret, i;
59
60 if (!num_syncobj_handles)
61 return 0;
62
63 syncobj_handles = memdup_user(u64_to_user_ptr(syncobj_handles_array),
64 size_mul(sizeof(uint32_t), num_syncobj_handles));
65 if (IS_ERR(syncobj_handles))
66 return PTR_ERR(syncobj_handles);
67
68 for (i = 0; i < num_syncobj_handles; i++) {
69
70 if (!syncobj_handles[i]) {
71 ret = -EINVAL;
72 goto free_memdup;
73 }
74
75 ret = drm_syncobj_find_fence(filp, syncobj_handles[i], 0, 0, &fence);
76 if (ret)
77 goto free_memdup;
78
79 dma_fence_wait(fence, false);
80
81 /* TODO: optimize async handling */
82 dma_fence_put(fence);
83 }
84
85free_memdup:
86 kfree(syncobj_handles);
87 return ret;
88}
89
90static int
91amdgpu_gem_update_timeline_node(struct drm_file *filp,
92 uint32_t syncobj_handle,
93 uint64_t point,
94 struct drm_syncobj **syncobj,
95 struct dma_fence_chain **chain)
96{
97 if (!syncobj_handle)
98 return 0;
99
100 /* Find the sync object */
101 *syncobj = drm_syncobj_find(filp, syncobj_handle);
102 if (!*syncobj)
103 return -ENOENT;
104
105 if (!point)
106 return 0;
107
108 /* Allocate the chain node */
109 *chain = dma_fence_chain_alloc();
110 if (!*chain) {
111 drm_syncobj_put(*syncobj);
112 *syncobj = NULL;
113 return -ENOMEM;
114 }
115
116 return 0;
117}
118
119static vm_fault_t amdgpu_gem_fault(struct vm_fault *vmf)
120{
121 struct ttm_buffer_object *bo = vmf->vma->vm_private_data;
122 struct drm_device *ddev = bo->base.dev;
123 vm_fault_t ret;
124 int idx;
125
126 ret = ttm_bo_vm_reserve(bo, vmf);
127 if (ret)
128 return ret;
129
130 if (drm_dev_enter(ddev, &idx)) {
131 ret = amdgpu_bo_fault_reserve_notify(bo);
132 if (ret) {
133 drm_dev_exit(idx);
134 goto unlock;
135 }
136
137 ret = ttm_bo_vm_fault_reserved(vmf, vmf->vma->vm_page_prot,
138 TTM_BO_VM_NUM_PREFAULT);
139
140 drm_dev_exit(idx);
141 } else {
142 ret = ttm_bo_vm_dummy_page(vmf, vmf->vma->vm_page_prot);
143 }
144 if (ret == VM_FAULT_RETRY && !(vmf->flags & FAULT_FLAG_RETRY_NOWAIT))
145 return ret;
146
147unlock:
148 dma_resv_unlock(bo->base.resv);
149 return ret;
150}
151
152static const struct vm_operations_struct amdgpu_gem_vm_ops = {
153 .fault = amdgpu_gem_fault,
154 .open = ttm_bo_vm_open,
155 .close = ttm_bo_vm_close,
156 .access = ttm_bo_vm_access
157};
158
159static void amdgpu_gem_object_free(struct drm_gem_object *gobj)
160{
161 struct amdgpu_bo *aobj = gem_to_amdgpu_bo(gobj);
162
163 amdgpu_hmm_unregister(aobj);
164 ttm_bo_fini(&aobj->tbo);
165}
166
167int amdgpu_gem_object_create(struct amdgpu_device *adev, unsigned long size,
168 int alignment, u32 initial_domain,
169 u64 flags, enum ttm_bo_type type,
170 struct dma_resv *resv,
171 struct drm_gem_object **obj, int8_t xcp_id_plus1)
172{
173 struct amdgpu_bo *bo;
174 struct amdgpu_bo_user *ubo;
175 struct amdgpu_bo_param bp;
176 int r;
177
178 memset(&bp, 0, sizeof(bp));
179 *obj = NULL;
180 flags |= AMDGPU_GEM_CREATE_VRAM_WIPE_ON_RELEASE;
181
182 bp.size = size;
183 bp.byte_align = alignment;
184 bp.type = type;
185 bp.resv = resv;
186 bp.preferred_domain = initial_domain;
187 bp.flags = flags;
188 bp.domain = initial_domain;
189 bp.bo_ptr_size = sizeof(struct amdgpu_bo);
190 bp.xcp_id_plus1 = xcp_id_plus1;
191
192 r = amdgpu_bo_create_user(adev, &bp, &ubo);
193 if (r)
194 return r;
195
196 bo = &ubo->bo;
197 *obj = &bo->tbo.base;
198
199 return 0;
200}
201
202void amdgpu_gem_force_release(struct amdgpu_device *adev)
203{
204 struct drm_device *ddev = adev_to_drm(adev);
205 struct drm_file *file;
206
207 mutex_lock(&ddev->filelist_mutex);
208
209 list_for_each_entry(file, &ddev->filelist, lhead) {
210 struct drm_gem_object *gobj;
211 int handle;
212
213 WARN_ONCE(1, "Still active user space clients!\n");
214 spin_lock(&file->table_lock);
215 idr_for_each_entry(&file->object_idr, gobj, handle) {
216 WARN_ONCE(1, "And also active allocations!\n");
217 drm_gem_object_put(gobj);
218 }
219 idr_destroy(&file->object_idr);
220 spin_unlock(&file->table_lock);
221 }
222
223 mutex_unlock(&ddev->filelist_mutex);
224}
225
226/*
227 * Call from drm_gem_handle_create which appear in both new and open ioctl
228 * case.
229 */
230static int amdgpu_gem_object_open(struct drm_gem_object *obj,
231 struct drm_file *file_priv)
232{
233 struct amdgpu_bo *abo = gem_to_amdgpu_bo(obj);
234 struct amdgpu_device *adev = amdgpu_ttm_adev(abo->tbo.bdev);
235 struct amdgpu_fpriv *fpriv = file_priv->driver_priv;
236 struct amdgpu_vm *vm = &fpriv->vm;
237 struct amdgpu_bo_va *bo_va;
238 struct mm_struct *mm;
239 struct drm_exec exec;
240 int r;
241
242 mm = amdgpu_ttm_tt_get_usermm(abo->tbo.ttm);
243 if (mm && mm != current->mm)
244 return -EPERM;
245
246 if (abo->flags & AMDGPU_GEM_CREATE_VM_ALWAYS_VALID &&
247 !amdgpu_vm_is_bo_always_valid(vm, abo))
248 return -EPERM;
249
250 drm_exec_init(&exec, DRM_EXEC_IGNORE_DUPLICATES, 0);
251 drm_exec_until_all_locked(&exec) {
252 r = drm_exec_prepare_obj(&exec, &abo->tbo.base, TTM_NUM_MOVE_FENCES + 1);
253 drm_exec_retry_on_contention(&exec);
254 if (unlikely(r))
255 goto out_unlock;
256
257 r = amdgpu_vm_lock_pd(vm, &exec, 0);
258 drm_exec_retry_on_contention(&exec);
259 if (unlikely(r))
260 goto out_unlock;
261 }
262
263 amdgpu_vm_bo_update_shared(abo);
264 bo_va = amdgpu_vm_bo_find(vm, abo);
265 if (!bo_va) {
266 bo_va = amdgpu_vm_bo_add(adev, vm, abo);
267 r = amdgpu_evf_mgr_attach_fence(&fpriv->evf_mgr, abo);
268 if (r)
269 goto out_unlock;
270 } else {
271 ++bo_va->ref_count;
272 }
273
274 drm_exec_fini(&exec);
275
276 /* Validate and add eviction fence to DMABuf imports with dynamic
277 * attachment in compute VMs. Re-validation will be done by
278 * amdgpu_vm_validate. Fences are on the reservation shared with the
279 * export, which is currently required to be validated and fenced
280 * already by amdgpu_amdkfd_gpuvm_restore_process_bos.
281 *
282 * Nested locking below for the case that a GEM object is opened in
283 * kfd_mem_export_dmabuf. Since the lock below is only taken for imports,
284 * but not for export, this is a different lock class that cannot lead to
285 * circular lock dependencies.
286 */
287 if (!vm->is_compute_context || !vm->process_info)
288 return 0;
289 if (!drm_gem_is_imported(obj) ||
290 !dma_buf_is_dynamic(obj->import_attach->dmabuf))
291 return 0;
292 mutex_lock_nested(&vm->process_info->lock, 1);
293 if (!WARN_ON(!vm->process_info->eviction_fence)) {
294 r = amdgpu_amdkfd_bo_validate_and_fence(abo, AMDGPU_GEM_DOMAIN_GTT,
295 &vm->process_info->eviction_fence->base);
296 if (r) {
297 struct amdgpu_task_info *ti = amdgpu_vm_get_task_info_vm(vm);
298
299 dev_warn(adev->dev, "validate_and_fence failed: %d\n", r);
300 if (ti) {
301 dev_warn(adev->dev, "pid %d\n", ti->task.pid);
302 amdgpu_vm_put_task_info(ti);
303 }
304 }
305 }
306 mutex_unlock(&vm->process_info->lock);
307 return r;
308
309out_unlock:
310 drm_exec_fini(&exec);
311 return r;
312}
313
314static void amdgpu_gem_object_close(struct drm_gem_object *obj,
315 struct drm_file *file_priv)
316{
317 struct amdgpu_bo *bo = gem_to_amdgpu_bo(obj);
318 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
319 struct amdgpu_fpriv *fpriv = file_priv->driver_priv;
320 struct amdgpu_vm *vm = &fpriv->vm;
321
322 struct dma_fence *fence = NULL;
323 struct amdgpu_bo_va *bo_va;
324 struct drm_exec exec;
325 long r;
326
327 drm_exec_init(&exec, DRM_EXEC_IGNORE_DUPLICATES, 0);
328 drm_exec_until_all_locked(&exec) {
329 r = drm_exec_prepare_obj(&exec, &bo->tbo.base, 1);
330 drm_exec_retry_on_contention(&exec);
331 if (unlikely(r))
332 goto out_unlock;
333
334 r = amdgpu_vm_lock_pd(vm, &exec, 0);
335 drm_exec_retry_on_contention(&exec);
336 if (unlikely(r))
337 goto out_unlock;
338 }
339
340 bo_va = amdgpu_vm_bo_find(vm, bo);
341 /* the fence is attached once per bo_va, so keep it for other handles */
342 if (bo_va && --bo_va->ref_count)
343 goto out_unlock;
344
345 if (!amdgpu_vm_is_bo_always_valid(vm, bo))
346 amdgpu_evf_mgr_detach_fence(&fpriv->evf_mgr, bo);
347 if (!bo_va)
348 goto out_unlock;
349
350 amdgpu_vm_bo_del(adev, bo_va);
351 amdgpu_vm_bo_update_shared(bo);
352 if (!amdgpu_vm_ready(vm))
353 goto out_unlock;
354
355 r = amdgpu_vm_clear_freed(adev, vm, &fence);
356 if (unlikely(r < 0) && !drm_dev_is_unplugged(adev_to_drm(adev)))
357 dev_err(adev->dev, "failed to clear page "
358 "tables on GEM object close (%ld)\n", r);
359 if (r || !fence)
360 goto out_unlock;
361
362 amdgpu_bo_fence(bo, fence, true);
363 dma_fence_put(fence);
364
365out_unlock:
366 if (r && !drm_dev_is_unplugged(adev_to_drm(adev)))
367 dev_err(adev->dev, "leaking bo va (%ld)\n", r);
368 drm_exec_fini(&exec);
369}
370
371static int amdgpu_gem_object_mmap(struct drm_gem_object *obj, struct vm_area_struct *vma)
372{
373 struct amdgpu_bo *bo = gem_to_amdgpu_bo(obj);
374
375 if (amdgpu_ttm_tt_get_usermm(bo->tbo.ttm))
376 return -EPERM;
377 if (bo->flags & AMDGPU_GEM_CREATE_NO_CPU_ACCESS)
378 return -EPERM;
379
380 /* Workaround for Thunk bug creating PROT_NONE,MAP_PRIVATE mappings
381 * for debugger access to invisible VRAM. Should have used MAP_SHARED
382 * instead. Clearing VM_MAYWRITE prevents the mapping from ever
383 * becoming writable and makes vma_is_cow_mapping(vma) false.
384 */
385 if (vma_is_cow_mapping(vma) &&
386 !(vma->vm_flags & VM_ACCESS_FLAGS))
387 vm_flags_clear(vma, VM_MAYWRITE);
388
389 return drm_gem_ttm_mmap(obj, vma);
390}
391
392const struct drm_gem_object_funcs amdgpu_gem_object_funcs = {
393 .free = amdgpu_gem_object_free,
394 .open = amdgpu_gem_object_open,
395 .close = amdgpu_gem_object_close,
396 .export = amdgpu_gem_prime_export,
397 .vmap = drm_gem_ttm_vmap,
398 .vunmap = drm_gem_ttm_vunmap,
399 .mmap = amdgpu_gem_object_mmap,
400 .vm_ops = &amdgpu_gem_vm_ops,
401};
402
403static bool amdgpu_gem_are_domains_valid(u32 domains)
404{
405 u32 normal = AMDGPU_GEM_DOMAIN_CPU |
406 AMDGPU_GEM_DOMAIN_GTT |
407 AMDGPU_GEM_DOMAIN_VRAM;
408 /* Treat all non CPU/GTT/VRAM domains as special domains. */
409 u32 special = AMDGPU_GEM_DOMAIN_MASK & ~normal;
410 u32 normal_mask = domains & normal;
411 u32 special_mask = domains & special;
412
413 if (!special_mask)
414 return true;
415
416 if (normal_mask)
417 return false;
418
419 return !(special_mask & (special_mask - 1));
420}
421
422/*
423 * GEM ioctls.
424 */
425int amdgpu_gem_create_ioctl(struct drm_device *dev, void *data,
426 struct drm_file *filp)
427{
428 struct amdgpu_device *adev = drm_to_adev(dev);
429 struct amdgpu_fpriv *fpriv = filp->driver_priv;
430 struct amdgpu_vm *vm = &fpriv->vm;
431 union drm_amdgpu_gem_create *args = data;
432 uint64_t flags = args->in.domain_flags;
433 uint64_t size = args->in.bo_size;
434 struct dma_resv *resv = NULL;
435 struct drm_gem_object *gobj;
436 uint32_t handle, initial_domain;
437 int r;
438
439 /* reject invalid gem flags */
440 if (flags & ~AMDGPU_GEM_CREATE_SETTABLE_MASK)
441 return -EINVAL;
442
443 /* reject invalid gem domains */
444 if (args->in.domains & ~AMDGPU_GEM_DOMAIN_MASK)
445 return -EINVAL;
446 if (!amdgpu_gem_are_domains_valid(args->in.domains))
447 return -EINVAL;
448
449 if (!amdgpu_is_tmz(adev) && (flags & AMDGPU_GEM_CREATE_ENCRYPTED)) {
450 DRM_NOTE_ONCE("Cannot allocate secure buffer since TMZ is disabled\n");
451 return -EINVAL;
452 }
453
454 /* always clear VRAM */
455 flags |= AMDGPU_GEM_CREATE_VRAM_CLEARED;
456
457 /* create a gem object to contain this object in */
458 if (args->in.domains & (AMDGPU_GEM_DOMAIN_GDS |
459 AMDGPU_GEM_DOMAIN_GWS | AMDGPU_GEM_DOMAIN_OA)) {
460 if (flags & AMDGPU_GEM_CREATE_VM_ALWAYS_VALID) {
461 /* if gds bo is created from user space, it must be
462 * passed to bo list
463 */
464 DRM_ERROR("GDS bo cannot be per-vm-bo\n");
465 return -EINVAL;
466 }
467 flags |= AMDGPU_GEM_CREATE_NO_CPU_ACCESS;
468 }
469
470 if (flags & AMDGPU_GEM_CREATE_VM_ALWAYS_VALID) {
471 r = amdgpu_bo_reserve(vm->root.bo, false);
472 if (r)
473 return r;
474
475 resv = vm->root.bo->tbo.base.resv;
476 }
477
478 initial_domain = (u32)(0xffffffff & args->in.domains);
479retry:
480 r = amdgpu_gem_object_create(adev, size, args->in.alignment,
481 initial_domain,
482 flags, ttm_bo_type_device, resv, &gobj, fpriv->xcp_id + 1);
483 if (r && r != -ERESTARTSYS) {
484 if (flags & AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED) {
485 flags &= ~AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED;
486 goto retry;
487 }
488
489 if (initial_domain == AMDGPU_GEM_DOMAIN_VRAM) {
490 initial_domain |= AMDGPU_GEM_DOMAIN_GTT;
491 goto retry;
492 }
493 DRM_DEBUG("Failed to allocate GEM object (%llu, %d, %llu, %d)\n",
494 size, initial_domain, args->in.alignment, r);
495 }
496
497 if (flags & AMDGPU_GEM_CREATE_VM_ALWAYS_VALID) {
498 if (!r) {
499 struct amdgpu_bo *abo = gem_to_amdgpu_bo(gobj);
500
501 abo->parent = amdgpu_bo_ref(vm->root.bo);
502 }
503 amdgpu_bo_unreserve(vm->root.bo);
504 }
505 if (r)
506 return r;
507
508 r = drm_gem_handle_create(filp, gobj, &handle);
509 /* drop reference from allocate - handle holds it now */
510 drm_gem_object_put(gobj);
511 if (r)
512 return r;
513
514 memset(args, 0, sizeof(*args));
515 args->out.handle = handle;
516 return 0;
517}
518
519int amdgpu_gem_userptr_ioctl(struct drm_device *dev, void *data,
520 struct drm_file *filp)
521{
522 struct ttm_operation_ctx ctx = { true, false };
523 struct amdgpu_device *adev = drm_to_adev(dev);
524 struct drm_amdgpu_gem_userptr *args = data;
525 struct amdgpu_fpriv *fpriv = filp->driver_priv;
526 struct drm_gem_object *gobj;
527 struct amdgpu_hmm_range *range;
528 struct amdgpu_bo *bo;
529 uint32_t handle;
530 int r;
531
532 args->addr = untagged_addr(args->addr);
533
534 if (offset_in_page(args->addr | args->size))
535 return -EINVAL;
536
537 if (!access_ok((void __user *)(uintptr_t)args->addr, args->size))
538 return -EFAULT;
539
540 /* reject unknown flag values */
541 if (args->flags & ~(AMDGPU_GEM_USERPTR_READONLY |
542 AMDGPU_GEM_USERPTR_ANONONLY | AMDGPU_GEM_USERPTR_VALIDATE |
543 AMDGPU_GEM_USERPTR_REGISTER))
544 return -EINVAL;
545
546 if (!(args->flags & AMDGPU_GEM_USERPTR_READONLY) &&
547 !(args->flags & AMDGPU_GEM_USERPTR_REGISTER)) {
548
549 /* if we want to write to it we must install a MMU notifier */
550 return -EACCES;
551 }
552
553 /* create a gem object to contain this object in */
554 r = amdgpu_gem_object_create(adev, args->size, 0, AMDGPU_GEM_DOMAIN_CPU,
555 0, ttm_bo_type_device, NULL, &gobj, fpriv->xcp_id + 1);
556 if (r)
557 return r;
558
559 bo = gem_to_amdgpu_bo(gobj);
560 bo->preferred_domains = AMDGPU_GEM_DOMAIN_GTT;
561 bo->allowed_domains = AMDGPU_GEM_DOMAIN_GTT;
562 bo->parent = amdgpu_bo_ref(fpriv->vm.root.bo);
563 r = amdgpu_ttm_tt_set_userptr(&bo->tbo, args->addr, args->flags);
564 if (r)
565 goto release_object;
566
567 r = amdgpu_hmm_register(bo, args->addr);
568 if (r)
569 goto release_object;
570
571 if (args->flags & AMDGPU_GEM_USERPTR_VALIDATE) {
572 range = amdgpu_hmm_range_alloc(NULL);
573 if (unlikely(!range)) {
574 r = -ENOMEM;
575 goto release_object;
576 }
577 r = amdgpu_ttm_tt_get_user_pages(bo, range);
578 if (r) {
579 amdgpu_hmm_range_free(range);
580 goto release_object;
581 }
582 r = amdgpu_bo_reserve(bo, true);
583 if (r)
584 goto user_pages_done;
585
586 amdgpu_ttm_tt_set_user_pages(bo->tbo.ttm, range);
587
588 amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_GTT);
589 r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
590 amdgpu_bo_unreserve(bo);
591 if (r)
592 goto user_pages_done;
593 }
594
595 r = drm_gem_handle_create(filp, gobj, &handle);
596 if (r)
597 goto user_pages_done;
598
599 args->handle = handle;
600
601user_pages_done:
602 if (args->flags & AMDGPU_GEM_USERPTR_VALIDATE)
603 amdgpu_hmm_range_free(range);
604release_object:
605 drm_gem_object_put(gobj);
606
607 return r;
608}
609
610int amdgpu_mode_dumb_mmap(struct drm_file *filp,
611 struct drm_device *dev,
612 uint32_t handle, uint64_t *offset_p)
613{
614 struct drm_gem_object *gobj;
615 struct amdgpu_bo *robj;
616
617 gobj = drm_gem_object_lookup(filp, handle);
618 if (!gobj)
619 return -ENOENT;
620
621 robj = gem_to_amdgpu_bo(gobj);
622 if (amdgpu_ttm_tt_get_usermm(robj->tbo.ttm) ||
623 (robj->flags & AMDGPU_GEM_CREATE_NO_CPU_ACCESS)) {
624 drm_gem_object_put(gobj);
625 return -EPERM;
626 }
627 *offset_p = amdgpu_bo_mmap_offset(robj);
628 drm_gem_object_put(gobj);
629 return 0;
630}
631
632int amdgpu_gem_mmap_ioctl(struct drm_device *dev, void *data,
633 struct drm_file *filp)
634{
635 union drm_amdgpu_gem_mmap *args = data;
636 uint32_t handle = args->in.handle;
637
638 memset(args, 0, sizeof(*args));
639 return amdgpu_mode_dumb_mmap(filp, dev, handle, &args->out.addr_ptr);
640}
641
642/**
643 * amdgpu_gem_timeout - calculate jiffies timeout from absolute value
644 *
645 * @timeout_ns: timeout in ns
646 *
647 * Calculate the timeout in jiffies from an absolute timeout in ns.
648 */
649unsigned long amdgpu_gem_timeout(uint64_t timeout_ns)
650{
651 unsigned long timeout_jiffies;
652 ktime_t timeout;
653
654 /* clamp timeout if it's to large */
655 if (((int64_t)timeout_ns) < 0)
656 return MAX_SCHEDULE_TIMEOUT;
657
658 timeout = ktime_sub(ns_to_ktime(timeout_ns), ktime_get());
659 if (ktime_to_ns(timeout) < 0)
660 return 0;
661
662 timeout_jiffies = nsecs_to_jiffies(ktime_to_ns(timeout));
663 /* clamp timeout to avoid unsigned-> signed overflow */
664 if (timeout_jiffies > MAX_SCHEDULE_TIMEOUT)
665 return MAX_SCHEDULE_TIMEOUT - 1;
666
667 return timeout_jiffies;
668}
669
670int amdgpu_gem_wait_idle_ioctl(struct drm_device *dev, void *data,
671 struct drm_file *filp)
672{
673 union drm_amdgpu_gem_wait_idle *args = data;
674 struct drm_gem_object *gobj;
675 struct amdgpu_bo *robj;
676 uint32_t handle = args->in.handle;
677 unsigned long timeout = amdgpu_gem_timeout(args->in.timeout);
678 int r = 0;
679 long ret;
680
681 gobj = drm_gem_object_lookup(filp, handle);
682 if (!gobj)
683 return -ENOENT;
684
685 robj = gem_to_amdgpu_bo(gobj);
686 ret = dma_resv_wait_timeout(robj->tbo.base.resv, DMA_RESV_USAGE_READ,
687 true, timeout);
688
689 /* ret == 0 means not signaled,
690 * ret > 0 means signaled
691 * ret < 0 means interrupted before timeout
692 */
693 if (ret >= 0) {
694 memset(args, 0, sizeof(*args));
695 args->out.status = (ret == 0);
696 } else
697 r = ret;
698
699 drm_gem_object_put(gobj);
700 return r;
701}
702
703int amdgpu_gem_metadata_ioctl(struct drm_device *dev, void *data,
704 struct drm_file *filp)
705{
706 struct drm_amdgpu_gem_metadata *args = data;
707 struct drm_gem_object *gobj;
708 struct amdgpu_bo *robj;
709 int r = -1;
710
711 DRM_DEBUG("%d\n", args->handle);
712 gobj = drm_gem_object_lookup(filp, args->handle);
713 if (gobj == NULL)
714 return -ENOENT;
715 robj = gem_to_amdgpu_bo(gobj);
716
717 r = amdgpu_bo_reserve(robj, false);
718 if (unlikely(r != 0))
719 goto out;
720
721 /* Reject MMIO_REMAP BOs at IOCTL level: metadata/tiling does not apply. */
722 if (robj->tbo.resource &&
723 robj->tbo.resource->mem_type == AMDGPU_PL_MMIO_REMAP) {
724 DRM_WARN("metadata ioctl on MMIO_REMAP BO (handle %d)\n",
725 args->handle);
726 r = -EINVAL;
727 goto unreserve;
728 }
729
730 if (args->op == AMDGPU_GEM_METADATA_OP_GET_METADATA) {
731 amdgpu_bo_get_tiling_flags(robj, &args->data.tiling_info);
732 r = amdgpu_bo_get_metadata(robj, args->data.data,
733 sizeof(args->data.data),
734 &args->data.data_size_bytes,
735 &args->data.flags);
736 } else if (args->op == AMDGPU_GEM_METADATA_OP_SET_METADATA) {
737 if (args->data.data_size_bytes > sizeof(args->data.data)) {
738 r = -EINVAL;
739 goto unreserve;
740 }
741 r = amdgpu_bo_set_tiling_flags(robj, args->data.tiling_info);
742 if (!r)
743 r = amdgpu_bo_set_metadata(robj, args->data.data,
744 args->data.data_size_bytes,
745 args->data.flags);
746 }
747
748unreserve:
749 amdgpu_bo_unreserve(robj);
750out:
751 drm_gem_object_put(gobj);
752 return r;
753}
754
755/**
756 * amdgpu_gem_va_update_vm -update the bo_va in its VM
757 *
758 * @adev: amdgpu_device pointer
759 * @vm: vm to update
760 * @bo_va: bo_va to update
761 * @operation: map, unmap or clear
762 *
763 * Update the bo_va directly after setting its address. Errors are not
764 * vital here, so they are not reported back to userspace.
765 *
766 * Returns resulting fence if freed BO(s) got cleared from the PT.
767 * otherwise stub fence in case of error.
768 */
769static struct dma_fence *
770amdgpu_gem_va_update_vm(struct amdgpu_device *adev,
771 struct amdgpu_vm *vm,
772 struct amdgpu_bo_va *bo_va,
773 uint32_t operation)
774{
775 struct dma_fence *fence;
776 int r = 0;
777
778 /* If the VM is not ready return only a stub. */
779 if (!amdgpu_vm_ready(vm))
780 return dma_fence_get_stub();
781
782
783 /*
784 * First clean up any freed mappings in the VM.
785 *
786 * amdgpu_vm_clear_freed() may replace @fence with a new fence if it
787 * schedules GPU work. If nothing needs clearing, @fence can remain as
788 * the original vm->last_update.
789 */
790 r = amdgpu_vm_clear_freed(adev, vm, &vm->last_update);
791 if (r)
792 goto error;
793
794 /* For MAP/REPLACE we also need to update the BO mappings. */
795 if (operation == AMDGPU_VA_OP_MAP ||
796 operation == AMDGPU_VA_OP_REPLACE) {
797 r = amdgpu_vm_bo_update(adev, bo_va, false);
798 if (r)
799 goto error;
800 }
801
802 /* Always update PDEs after we touched the mappings. */
803 r = amdgpu_vm_update_pdes(adev, vm, false);
804 if (r)
805 goto error;
806
807 if ((operation == AMDGPU_VA_OP_MAP ||
808 operation == AMDGPU_VA_OP_REPLACE) &&
809 !amdgpu_vm_is_bo_always_valid(vm, bo_va->base.bo)) {
810
811 /*
812 * For MAP/REPLACE of non per-VM BOs we need to sync to both the
813 * bo_va->last_pt_update and vm->last_update or otherwise we
814 * potentially miss the PDE updates.
815 */
816 fence = dma_fence_unwrap_merge(vm->last_update,
817 bo_va->last_pt_update);
818 if (!fence) {
819 /* As fallback in OOM situations */
820 dma_fence_wait(vm->last_update, false);
821 dma_fence_wait(bo_va->last_pt_update, false);
822 fence = dma_fence_get_stub();
823 }
824 } else {
825 fence = dma_fence_get(vm->last_update);
826 }
827
828 return fence;
829
830error:
831 if (r && r != -ERESTARTSYS)
832 DRM_ERROR("Couldn't update BO_VA (%d)\n", r);
833
834 return dma_fence_get(vm->last_update);
835}
836
837int amdgpu_gem_va_ioctl(struct drm_device *dev, void *data,
838 struct drm_file *filp)
839{
840 const uint32_t valid_flags = AMDGPU_VM_DELAY_UPDATE |
841 AMDGPU_VM_PAGE_READABLE | AMDGPU_VM_PAGE_WRITEABLE |
842 AMDGPU_VM_PAGE_EXECUTABLE | AMDGPU_VM_MTYPE_MASK |
843 AMDGPU_VM_PAGE_NOALLOC;
844 const uint32_t prt_flags = AMDGPU_VM_DELAY_UPDATE |
845 AMDGPU_VM_PAGE_PRT;
846
847 struct drm_amdgpu_gem_va *args = data;
848 struct drm_gem_object *gobj;
849 struct amdgpu_device *adev = drm_to_adev(dev);
850 struct amdgpu_fpriv *fpriv = filp->driver_priv;
851 struct amdgpu_bo *abo;
852 struct amdgpu_bo_va *bo_va;
853 struct drm_syncobj *timeline_syncobj = NULL;
854 struct dma_fence_chain *timeline_chain = NULL;
855 struct drm_exec exec;
856 uint64_t vm_size, tmp;
857 int r = 0;
858
859 /* Validate virtual address range against reserved regions. */
860 if (args->va_address < AMDGPU_VA_RESERVED_BOTTOM) {
861 dev_dbg(dev->dev,
862 "va_address 0x%llx is in reserved area 0x%llx\n",
863 args->va_address, AMDGPU_VA_RESERVED_BOTTOM);
864 return -EINVAL;
865 }
866
867 if (args->va_address >= AMDGPU_GMC_HOLE_START &&
868 args->va_address < AMDGPU_GMC_HOLE_END) {
869 dev_dbg(dev->dev,
870 "va_address 0x%llx is in VA hole 0x%llx-0x%llx\n",
871 args->va_address, AMDGPU_GMC_HOLE_START,
872 AMDGPU_GMC_HOLE_END);
873 return -EINVAL;
874 }
875
876 args->va_address &= AMDGPU_GMC_HOLE_MASK;
877
878 vm_size = adev->vm_manager.max_pfn * AMDGPU_GPU_PAGE_SIZE;
879 vm_size -= AMDGPU_VA_RESERVED_TOP;
880 if (check_add_overflow(args->va_address, args->map_size, &tmp) || tmp > vm_size) {
881 dev_dbg(dev->dev,
882 "va_address 0x%llx is in top reserved area 0x%llx\n",
883 args->va_address + args->map_size, vm_size);
884 return -EINVAL;
885 }
886
887 if ((args->flags & ~valid_flags) && (args->flags & ~prt_flags)) {
888 dev_dbg(dev->dev, "invalid flags combination 0x%08X\n",
889 args->flags);
890 return -EINVAL;
891 }
892
893 /* Validate operation type. */
894 switch (args->operation) {
895 case AMDGPU_VA_OP_MAP:
896 case AMDGPU_VA_OP_UNMAP:
897 case AMDGPU_VA_OP_CLEAR:
898 case AMDGPU_VA_OP_REPLACE:
899 break;
900 default:
901 dev_dbg(dev->dev, "unsupported operation %d\n",
902 args->operation);
903 return -EINVAL;
904 }
905
906 if (args->flags & AMDGPU_VM_DELAY_UPDATE &&
907 args->vm_timeline_syncobj_out)
908 return -EINVAL;
909
910 if ((args->operation != AMDGPU_VA_OP_CLEAR) &&
911 !(args->flags & AMDGPU_VM_PAGE_PRT)) {
912 gobj = drm_gem_object_lookup(filp, args->handle);
913 if (gobj == NULL)
914 return -ENOENT;
915 abo = gem_to_amdgpu_bo(gobj);
916 } else {
917 gobj = NULL;
918 abo = NULL;
919 }
920
921 /* Add input syncobj fences (if any) for synchronization. */
922 r = amdgpu_gem_add_input_fence(filp,
923 args->input_fence_syncobj_handles,
924 args->num_syncobj_handles);
925 if (r)
926 goto error_put_gobj;
927
928 drm_exec_init(&exec, DRM_EXEC_INTERRUPTIBLE_WAIT |
929 DRM_EXEC_IGNORE_DUPLICATES, 0);
930 drm_exec_until_all_locked(&exec) {
931 if (gobj) {
932 r = drm_exec_lock_obj(&exec, gobj);
933 drm_exec_retry_on_contention(&exec);
934 if (unlikely(r))
935 goto error;
936 }
937
938 r = amdgpu_vm_lock_pd(&fpriv->vm, &exec, 2);
939 drm_exec_retry_on_contention(&exec);
940 if (unlikely(r))
941 goto error;
942 }
943
944 /* Resolve the BO-VA mapping for this VM/BO combination. */
945 if (abo) {
946 bo_va = amdgpu_vm_bo_find(&fpriv->vm, abo);
947 if (!bo_va) {
948 r = -ENOENT;
949 goto error;
950 }
951 } else if (args->operation != AMDGPU_VA_OP_CLEAR) {
952 bo_va = fpriv->prt_va;
953 } else {
954 bo_va = NULL;
955 }
956
957 /*
958 * Prepare the timeline syncobj node if the user requested a VM
959 * timeline update. This only allocates/looks up the syncobj and
960 * chain node; the actual fence is attached later.
961 */
962 r = amdgpu_gem_update_timeline_node(filp,
963 args->vm_timeline_syncobj_out,
964 args->vm_timeline_point,
965 &timeline_syncobj,
966 &timeline_chain);
967 if (r)
968 goto error;
969
970 switch (args->operation) {
971 case AMDGPU_VA_OP_MAP:
972 r = amdgpu_vm_bo_map(adev, bo_va, args->va_address,
973 args->offset_in_bo, args->map_size,
974 args->flags);
975 break;
976 case AMDGPU_VA_OP_UNMAP:
977 r = amdgpu_vm_bo_unmap(adev, bo_va, args->va_address);
978 break;
979
980 case AMDGPU_VA_OP_CLEAR:
981 r = amdgpu_vm_bo_clear_mappings(adev, &fpriv->vm,
982 args->va_address,
983 args->map_size);
984 break;
985 case AMDGPU_VA_OP_REPLACE:
986 r = amdgpu_vm_bo_replace_map(adev, bo_va, args->va_address,
987 args->offset_in_bo, args->map_size,
988 args->flags);
989 break;
990 default:
991 break;
992 }
993
994 /*
995 * Once the VA operation is done, update the VM and obtain the fence
996 * that represents the last relevant update for this mapping. This
997 * fence can then be exported to the user-visible VM timeline.
998 */
999 if (!r && !(args->flags & AMDGPU_VM_DELAY_UPDATE) &&
1000 (!adev->debug_vm || timeline_syncobj)) {
1001 struct dma_fence *fence;
1002
1003 fence = amdgpu_gem_va_update_vm(adev, &fpriv->vm, bo_va,
1004 args->operation);
1005 if (timeline_syncobj) {
1006 if (!args->vm_timeline_point) {
1007 /* Replace the existing fence when no point is given. */
1008 drm_syncobj_replace_fence(timeline_syncobj,
1009 fence);
1010 } else {
1011 /* Attach the last-update fence at a specific point. */
1012 drm_syncobj_add_point(timeline_syncobj,
1013 timeline_chain,
1014 fence,
1015 args->vm_timeline_point);
1016 timeline_chain = NULL;
1017 }
1018 }
1019 dma_fence_put(fence);
1020
1021 }
1022
1023error:
1024 dma_fence_chain_free(timeline_chain);
1025 if (timeline_syncobj)
1026 drm_syncobj_put(timeline_syncobj);
1027 drm_exec_fini(&exec);
1028error_put_gobj:
1029 drm_gem_object_put(gobj);
1030 return r;
1031}
1032
1033int amdgpu_gem_op_ioctl(struct drm_device *dev, void *data,
1034 struct drm_file *filp)
1035{
1036 struct drm_amdgpu_gem_op *args = data;
1037 struct drm_gem_object *gobj;
1038 struct amdgpu_vm_bo_base *base;
1039 struct amdgpu_bo *robj;
1040 struct drm_exec exec;
1041 struct amdgpu_fpriv *fpriv = filp->driver_priv;
1042 int r;
1043
1044 if (args->padding)
1045 return -EINVAL;
1046
1047 gobj = drm_gem_object_lookup(filp, args->handle);
1048 if (!gobj)
1049 return -ENOENT;
1050
1051 robj = gem_to_amdgpu_bo(gobj);
1052
1053 drm_exec_init(&exec, DRM_EXEC_INTERRUPTIBLE_WAIT |
1054 DRM_EXEC_IGNORE_DUPLICATES, 0);
1055 drm_exec_until_all_locked(&exec) {
1056 r = drm_exec_lock_obj(&exec, gobj);
1057 drm_exec_retry_on_contention(&exec);
1058 if (r)
1059 goto out_exec;
1060
1061 if (args->op == AMDGPU_GEM_OP_GET_MAPPING_INFO) {
1062 r = amdgpu_vm_lock_pd(&fpriv->vm, &exec, 0);
1063 drm_exec_retry_on_contention(&exec);
1064 if (r)
1065 goto out_exec;
1066 }
1067 }
1068
1069 switch (args->op) {
1070 case AMDGPU_GEM_OP_GET_GEM_CREATE_INFO: {
1071 struct drm_amdgpu_gem_create_in info;
1072 void __user *out = u64_to_user_ptr(args->value);
1073
1074 info.bo_size = robj->tbo.base.size;
1075 info.alignment = robj->tbo.page_alignment << PAGE_SHIFT;
1076 info.domains = robj->preferred_domains;
1077 info.domain_flags = robj->flags;
1078 drm_exec_fini(&exec);
1079 if (copy_to_user(out, &info, sizeof(info)))
1080 r = -EFAULT;
1081 break;
1082 }
1083 case AMDGPU_GEM_OP_SET_PLACEMENT:
1084 if (drm_gem_is_imported(&robj->tbo.base) &&
1085 args->value & AMDGPU_GEM_DOMAIN_VRAM) {
1086 r = -EINVAL;
1087 goto out_exec;
1088 }
1089 if (amdgpu_ttm_tt_get_usermm(robj->tbo.ttm)) {
1090 r = -EPERM;
1091 goto out_exec;
1092 }
1093 for (base = robj->vm_bo; base; base = base->next)
1094 if (amdgpu_xgmi_same_hive(amdgpu_ttm_adev(robj->tbo.bdev),
1095 amdgpu_ttm_adev(base->vm->root.bo->tbo.bdev))) {
1096 r = -EINVAL;
1097 goto out_exec;
1098 }
1099
1100
1101 robj->preferred_domains = args->value & (AMDGPU_GEM_DOMAIN_VRAM |
1102 AMDGPU_GEM_DOMAIN_GTT |
1103 AMDGPU_GEM_DOMAIN_CPU);
1104 robj->allowed_domains = robj->preferred_domains;
1105 if (robj->allowed_domains == AMDGPU_GEM_DOMAIN_VRAM)
1106 robj->allowed_domains |= AMDGPU_GEM_DOMAIN_GTT;
1107
1108 if (robj->flags & AMDGPU_GEM_CREATE_VM_ALWAYS_VALID)
1109 amdgpu_vm_bo_invalidate(robj, true);
1110 drm_exec_fini(&exec);
1111 break;
1112 case AMDGPU_GEM_OP_GET_MAPPING_INFO: {
1113 struct amdgpu_bo_va *bo_va = amdgpu_vm_bo_find(&fpriv->vm, robj);
1114 struct drm_amdgpu_gem_vm_entry *vm_entries;
1115 struct amdgpu_bo_va_mapping *mapping;
1116 int num_mappings = 0;
1117 /*
1118 * num_entries is set as an input to the size of the user-allocated array of
1119 * drm_amdgpu_gem_vm_entry stored at args->value.
1120 * num_entries is sent back as output as the number of mappings the bo has.
1121 * If that number is larger than the size of the array, the ioctl must
1122 * be retried.
1123 */
1124 if (!bo_va) {
1125 r = -ENOENT;
1126 goto out_exec;
1127 }
1128
1129 if (args->num_entries > INT_MAX / sizeof(*vm_entries)) {
1130 r = -EINVAL;
1131 goto out_exec;
1132 }
1133
1134 vm_entries = kvcalloc(args->num_entries, sizeof(*vm_entries), GFP_KERNEL);
1135 if (!vm_entries) {
1136 r = -ENOMEM;
1137 goto out_exec;
1138 }
1139
1140 amdgpu_vm_bo_va_for_each_valid_mapping(bo_va, mapping) {
1141 if (num_mappings < args->num_entries) {
1142 vm_entries[num_mappings].addr = mapping->start * AMDGPU_GPU_PAGE_SIZE;
1143 vm_entries[num_mappings].size = (mapping->last - mapping->start + 1) * AMDGPU_GPU_PAGE_SIZE;
1144 vm_entries[num_mappings].offset = mapping->offset;
1145 vm_entries[num_mappings].flags = mapping->flags;
1146 }
1147 num_mappings += 1;
1148 }
1149
1150 amdgpu_vm_bo_va_for_each_invalid_mapping(bo_va, mapping) {
1151 if (num_mappings < args->num_entries) {
1152 vm_entries[num_mappings].addr = mapping->start * AMDGPU_GPU_PAGE_SIZE;
1153 vm_entries[num_mappings].size = (mapping->last - mapping->start + 1) * AMDGPU_GPU_PAGE_SIZE;
1154 vm_entries[num_mappings].offset = mapping->offset;
1155 vm_entries[num_mappings].flags = mapping->flags;
1156 }
1157 num_mappings += 1;
1158 }
1159
1160 drm_exec_fini(&exec);
1161
1162 if (num_mappings > 0 && num_mappings <= args->num_entries)
1163 if (copy_to_user(u64_to_user_ptr(args->value), vm_entries, num_mappings * sizeof(*vm_entries)))
1164 r = -EFAULT;
1165
1166 args->num_entries = num_mappings;
1167
1168 kvfree(vm_entries);
1169 break;
1170 }
1171 default:
1172 drm_exec_fini(&exec);
1173 r = -EINVAL;
1174 }
1175
1176 drm_gem_object_put(gobj);
1177 return r;
1178out_exec:
1179 drm_exec_fini(&exec);
1180 drm_gem_object_put(gobj);
1181 return r;
1182}
1183
1184/**
1185 * amdgpu_gem_list_handles_ioctl - get information about a process' buffer objects
1186 *
1187 * @dev: drm device pointer
1188 * @data: drm_amdgpu_gem_list_handles
1189 * @filp: drm file pointer
1190 *
1191 * num_entries is set as an input to the size of the entries array.
1192 * num_entries is sent back as output as the number of bos in the process.
1193 * If that number is larger than the size of the array, the ioctl must
1194 * be retried.
1195 *
1196 * Returns:
1197 * 0 for success, -errno for errors.
1198 */
1199int amdgpu_gem_list_handles_ioctl(struct drm_device *dev, void *data,
1200 struct drm_file *filp)
1201{
1202 struct drm_amdgpu_gem_list_handles *args = data;
1203 struct drm_amdgpu_gem_list_handles_entry *bo_entries;
1204 struct drm_gem_object *gobj;
1205 int id, ret = 0;
1206 int bo_index = 0;
1207 int num_bos = 0;
1208
1209 spin_lock(&filp->table_lock);
1210 idr_for_each_entry(&filp->object_idr, gobj, id)
1211 num_bos += 1;
1212 spin_unlock(&filp->table_lock);
1213
1214 if (args->num_entries < num_bos) {
1215 args->num_entries = num_bos;
1216 return 0;
1217 }
1218
1219 if (num_bos == 0) {
1220 args->num_entries = 0;
1221 return 0;
1222 }
1223
1224 bo_entries = kvzalloc_objs(*bo_entries, num_bos);
1225 if (!bo_entries)
1226 return -ENOMEM;
1227
1228 spin_lock(&filp->table_lock);
1229 idr_for_each_entry(&filp->object_idr, gobj, id) {
1230 struct amdgpu_bo *bo = gem_to_amdgpu_bo(gobj);
1231 struct drm_amdgpu_gem_list_handles_entry *bo_entry;
1232
1233 if (bo_index >= num_bos) {
1234 ret = -EAGAIN;
1235 break;
1236 }
1237
1238 bo_entry = &bo_entries[bo_index];
1239
1240 bo_entry->size = amdgpu_bo_size(bo);
1241 bo_entry->alloc_flags = bo->flags & AMDGPU_GEM_CREATE_SETTABLE_MASK;
1242 bo_entry->preferred_domains = bo->preferred_domains;
1243 bo_entry->gem_handle = id;
1244 bo_entry->alignment = (u64)bo->tbo.page_alignment << PAGE_SHIFT;
1245
1246 if (bo->tbo.base.import_attach)
1247 bo_entry->flags |= AMDGPU_GEM_LIST_HANDLES_FLAG_IS_IMPORT;
1248
1249 bo_index += 1;
1250 }
1251 spin_unlock(&filp->table_lock);
1252
1253 args->num_entries = bo_index;
1254
1255 if (!ret)
1256 if (copy_to_user(u64_to_user_ptr(args->entries), bo_entries, num_bos * sizeof(*bo_entries)))
1257 ret = -EFAULT;
1258
1259 kvfree(bo_entries);
1260
1261 return ret;
1262}
1263
1264static unsigned int amdgpu_gem_align_pitch(struct amdgpu_device *adev,
1265 unsigned int width,
1266 unsigned int cpp,
1267 bool tiled)
1268{
1269 unsigned int aligned = width;
1270 unsigned int pitch_mask = 0;
1271 unsigned int pitch;
1272
1273 switch (cpp) {
1274 case 1:
1275 pitch_mask = 255;
1276 break;
1277 case 2:
1278 pitch_mask = 127;
1279 break;
1280 case 3:
1281 case 4:
1282 pitch_mask = 63;
1283 break;
1284 }
1285
1286 if (check_add_overflow(aligned, pitch_mask, &aligned))
1287 return 0;
1288 aligned &= ~pitch_mask;
1289 if (check_mul_overflow(aligned, cpp, &pitch))
1290 return 0;
1291 return pitch;
1292}
1293
1294int amdgpu_mode_dumb_create(struct drm_file *file_priv,
1295 struct drm_device *dev,
1296 struct drm_mode_create_dumb *args)
1297{
1298 struct amdgpu_device *adev = drm_to_adev(dev);
1299 struct amdgpu_fpriv *fpriv = file_priv->driver_priv;
1300 struct drm_gem_object *gobj;
1301 uint32_t handle;
1302 u64 flags = AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED |
1303 AMDGPU_GEM_CREATE_CPU_GTT_USWC |
1304 AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS;
1305 u32 domain;
1306 int r;
1307
1308 /*
1309 * The buffer returned from this function should be cleared, but
1310 * it can only be done if the ring is enabled or we'll fail to
1311 * create the buffer.
1312 */
1313 if (adev->mman.buffer_funcs_enabled)
1314 flags |= AMDGPU_GEM_CREATE_VRAM_CLEARED;
1315
1316 args->pitch = amdgpu_gem_align_pitch(adev, args->width,
1317 DIV_ROUND_UP(args->bpp, 8), 0);
1318 if (!args->pitch)
1319 return -EINVAL;
1320 args->size = (u64)args->pitch * args->height;
1321 args->size = ALIGN(args->size, PAGE_SIZE);
1322 if (!args->size)
1323 return -EINVAL;
1324 domain = amdgpu_bo_get_preferred_domain(adev,
1325 amdgpu_display_supported_domains(adev, flags));
1326 r = amdgpu_gem_object_create(adev, args->size, 0, domain, flags,
1327 ttm_bo_type_device, NULL, &gobj, fpriv->xcp_id + 1);
1328 if (r)
1329 return -ENOMEM;
1330
1331 r = drm_gem_handle_create(file_priv, gobj, &handle);
1332 /* drop reference from allocate - handle holds it now */
1333 drm_gem_object_put(gobj);
1334 if (r)
1335 return r;
1336
1337 args->handle = handle;
1338 return 0;
1339}
1340
1341#if defined(CONFIG_DEBUG_FS)
1342static int amdgpu_debugfs_gem_info_show(struct seq_file *m, void *unused)
1343{
1344 struct amdgpu_device *adev = m->private;
1345 struct drm_device *dev = adev_to_drm(adev);
1346 struct drm_file *file;
1347 int r;
1348
1349 r = mutex_lock_interruptible(&dev->filelist_mutex);
1350 if (r)
1351 return r;
1352
1353 list_for_each_entry(file, &dev->filelist, lhead) {
1354 struct task_struct *task;
1355 struct drm_gem_object *gobj;
1356 struct pid *pid;
1357 int id;
1358
1359 /*
1360 * Although we have a valid reference on file->pid, that does
1361 * not guarantee that the task_struct who called get_pid() is
1362 * still alive (e.g. get_pid(current) => fork() => exit()).
1363 * Therefore, we need to protect this ->comm access using RCU.
1364 */
1365 rcu_read_lock();
1366 pid = rcu_dereference(file->pid);
1367 task = pid_task(pid, PIDTYPE_TGID);
1368 seq_printf(m, "pid %8d command %s:\n", pid_nr(pid),
1369 task ? task->comm : "<unknown>");
1370 rcu_read_unlock();
1371
1372 spin_lock(&file->table_lock);
1373 idr_for_each_entry(&file->object_idr, gobj, id) {
1374 struct amdgpu_bo *bo = gem_to_amdgpu_bo(gobj);
1375
1376 amdgpu_bo_print_info(id, bo, m);
1377 }
1378 spin_unlock(&file->table_lock);
1379 }
1380
1381 mutex_unlock(&dev->filelist_mutex);
1382 return 0;
1383}
1384
1385DEFINE_SHOW_ATTRIBUTE(amdgpu_debugfs_gem_info);
1386
1387#endif
1388
1389void amdgpu_debugfs_gem_init(struct amdgpu_device *adev)
1390{
1391#if defined(CONFIG_DEBUG_FS)
1392 struct drm_minor *minor = adev_to_drm(adev)->primary;
1393 struct dentry *root = minor->debugfs_root;
1394
1395 debugfs_create_file("amdgpu_gem_info", 0444, root, adev,
1396 &amdgpu_debugfs_gem_info_fops);
1397#endif
1398}