projects/linux·blob master · refs

1004 lines · 26.9 KiB

1// SPDX-License-Identifier: MIT
2/*
3 * Copyright 2023 Advanced Micro Devices, Inc.
4 *
5 * Permission is hereby granted, free of charge, to any person obtaining a
6 * copy of this software and associated documentation files (the "Software"),
7 * to deal in the Software without restriction, including without limitation
8 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
9 * and/or sell copies of the Software, and to permit persons to whom the
10 * Software is furnished to do so, subject to the following conditions:
11 *
12 * The above copyright notice and this permission notice shall be included in
13 * all copies or substantial portions of the Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
19 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
20 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
21 * OTHER DEALINGS IN THE SOFTWARE.
22 *
23 */
24
25#include <linux/kref.h>
26#include <linux/slab.h>
27#include <linux/dma-fence-unwrap.h>
28
29#include <drm/drm_exec.h>
30#include <drm/drm_syncobj.h>
31
32#include "amdgpu.h"
33#include "amdgpu_trace.h"
34
35#define AMDGPU_USERQ_MAX_HANDLES (1U << 16)
36
37static const struct dma_fence_ops amdgpu_userq_fence_ops;
38
39static inline struct amdgpu_userq_fence *to_amdgpu_userq_fence(struct dma_fence *f)
40{
41 if (!f || f->ops != &amdgpu_userq_fence_ops)
42 return NULL;
43
44 return container_of(f, struct amdgpu_userq_fence, base);
45}
46
47static u64 amdgpu_userq_fence_read(struct amdgpu_userq_fence_driver *fence_drv)
48{
49 return le64_to_cpu(*fence_drv->cpu_addr);
50}
51
52static void
53amdgpu_userq_fence_write(struct amdgpu_userq_fence_driver *fence_drv,
54 u64 seq)
55{
56 if (fence_drv->cpu_addr)
57 *fence_drv->cpu_addr = cpu_to_le64(seq);
58}
59
60int amdgpu_userq_fence_driver_alloc(struct amdgpu_device *adev,
61 struct amdgpu_userq_fence_driver **fence_drv_req)
62{
63 struct amdgpu_userq_fence_driver *fence_drv;
64 int r;
65
66 if (!fence_drv_req)
67 return -EINVAL;
68 *fence_drv_req = NULL;
69
70 fence_drv = kzalloc_obj(*fence_drv);
71 if (!fence_drv)
72 return -ENOMEM;
73
74 /* Acquire seq64 memory */
75 r = amdgpu_seq64_alloc(adev, &fence_drv->va, &fence_drv->gpu_addr,
76 &fence_drv->cpu_addr);
77 if (r)
78 goto free_fence_drv;
79
80 memset(fence_drv->cpu_addr, 0, sizeof(u64));
81
82 kref_init(&fence_drv->refcount);
83 INIT_LIST_HEAD(&fence_drv->fences);
84 spin_lock_init(&fence_drv->fence_list_lock);
85
86 fence_drv->adev = adev;
87 fence_drv->context = dma_fence_context_alloc(1);
88 get_task_comm(fence_drv->timeline_name, current);
89
90 *fence_drv_req = fence_drv;
91
92 return 0;
93
94free_fence_drv:
95 kfree(fence_drv);
96
97 return r;
98}
99
100static void amdgpu_userq_walk_and_drop_fence_drv(struct xarray *xa)
101{
102 struct amdgpu_userq_fence_driver *fence_drv;
103 unsigned long index;
104
105 if (xa_empty(xa))
106 return;
107
108 xa_lock(xa);
109 xa_for_each(xa, index, fence_drv) {
110 __xa_erase(xa, index);
111 amdgpu_userq_fence_driver_put(fence_drv);
112 }
113
114 xa_unlock(xa);
115}
116
117void
118amdgpu_userq_fence_driver_free(struct amdgpu_usermode_queue *userq)
119{
120 dma_fence_put(userq->last_fence);
121 userq->last_fence = NULL;
122 amdgpu_userq_walk_and_drop_fence_drv(&userq->fence_drv_xa);
123 xa_destroy(&userq->fence_drv_xa);
124 mutex_destroy(&userq->fence_drv_lock);
125 /* Drop the queue's ownership reference to fence_drv explicitly */
126 amdgpu_userq_fence_driver_put(userq->fence_drv);
127}
128
129static void
130amdgpu_userq_fence_put_fence_drv_array(struct amdgpu_userq_fence *userq_fence)
131{
132 unsigned long i;
133 for (i = 0; i < userq_fence->fence_drv_array_count; i++)
134 amdgpu_userq_fence_driver_put(userq_fence->fence_drv_array[i]);
135 userq_fence->fence_drv_array_count = 0;
136}
137
138/*
139 * Returns:
140 * -ENOENT when no fences were processes
141 * 1 when more fences are pending
142 * 0 when no fences are pending any more
143 */
144int
145amdgpu_userq_fence_driver_process(struct amdgpu_userq_fence_driver *fence_drv)
146{
147 struct amdgpu_userq_fence *userq_fence, *tmp;
148 LIST_HEAD(to_be_signaled);
149 struct dma_fence *fence;
150 unsigned long flags;
151 u64 rptr;
152
153 spin_lock_irqsave(&fence_drv->fence_list_lock, flags);
154 rptr = amdgpu_userq_fence_read(fence_drv);
155
156 list_for_each_entry(userq_fence, &fence_drv->fences, link) {
157 if (rptr < userq_fence->base.seqno)
158 break;
159 }
160
161 list_cut_before(&to_be_signaled, &fence_drv->fences,
162 &userq_fence->link);
163 spin_unlock_irqrestore(&fence_drv->fence_list_lock, flags);
164
165 if (list_empty(&to_be_signaled))
166 return -ENOENT;
167
168 list_for_each_entry_safe(userq_fence, tmp, &to_be_signaled, link) {
169 fence = &userq_fence->base;
170 list_del_init(&userq_fence->link);
171 dma_fence_signal(fence);
172 /* Drop fence_drv_array outside fence_list_lock
173 * to avoid the recursion lock.
174 */
175 amdgpu_userq_fence_put_fence_drv_array(userq_fence);
176 dma_fence_put(fence);
177 }
178
179 /* That doesn't need to be accurate so no locking */
180 return list_empty(&fence_drv->fences) ? 0 : 1;
181}
182
183void amdgpu_userq_fence_driver_destroy(struct kref *ref)
184{
185 struct amdgpu_userq_fence_driver *fence_drv = container_of(ref,
186 struct amdgpu_userq_fence_driver,
187 refcount);
188 struct amdgpu_device *adev = fence_drv->adev;
189 struct amdgpu_userq_fence *fence, *tmp;
190 unsigned long flags;
191 struct dma_fence *f;
192
193 spin_lock_irqsave(&fence_drv->fence_list_lock, flags);
194 lockdep_assert_held(&fence_drv->fence_list_lock);
195 list_for_each_entry_safe(fence, tmp, &fence_drv->fences, link) {
196 f = &fence->base;
197 spin_lock(dma_fence_spinlock(f));
198 if (!dma_fence_is_signaled_locked(f)) {
199 dma_fence_set_error(f, -ECANCELED);
200 dma_fence_signal_locked(f);
201 }
202 spin_unlock(dma_fence_spinlock(f));
203 list_del(&fence->link);
204 dma_fence_put(f);
205 }
206 spin_unlock_irqrestore(&fence_drv->fence_list_lock, flags);
207
208 /* Free seq64 memory */
209 amdgpu_seq64_free(adev, fence_drv->va);
210 kfree(fence_drv);
211}
212
213void amdgpu_userq_fence_driver_get(struct amdgpu_userq_fence_driver *fence_drv)
214{
215 kref_get(&fence_drv->refcount);
216}
217
218void amdgpu_userq_fence_driver_put(struct amdgpu_userq_fence_driver *fence_drv)
219{
220 kref_put(&fence_drv->refcount, amdgpu_userq_fence_driver_destroy);
221}
222
223static int amdgpu_userq_fence_alloc(struct amdgpu_usermode_queue *userq,
224 struct amdgpu_userq_fence **pfence)
225{
226 struct amdgpu_userq_fence_driver *fence_drv = userq->fence_drv;
227 struct amdgpu_userq_fence *userq_fence;
228 void *entry;
229
230 userq_fence = kmalloc_obj(*userq_fence);
231 if (!userq_fence)
232 return -ENOMEM;
233
234 /*
235 * Get the next unused entry, since we fill from the start this can be
236 * used as size to allocate the array.
237 */
238 mutex_lock(&userq->fence_drv_lock);
239 XA_STATE(xas, &userq->fence_drv_xa, 0);
240
241 rcu_read_lock();
242 do {
243 entry = xas_find_marked(&xas, ULONG_MAX, XA_FREE_MARK);
244 } while (xas_retry(&xas, entry));
245 rcu_read_unlock();
246
247 userq_fence->fence_drv_array = kvmalloc_objs(fence_drv, xas.xa_index);
248 if (!userq_fence->fence_drv_array) {
249 mutex_unlock(&userq->fence_drv_lock);
250 kfree(userq_fence);
251 return -ENOMEM;
252 }
253
254 userq_fence->fence_drv_array_count = xas.xa_index;
255 xa_extract(&userq->fence_drv_xa, (void **)userq_fence->fence_drv_array,
256 0, ULONG_MAX, xas.xa_index, XA_PRESENT);
257 xa_destroy(&userq->fence_drv_xa);
258
259 mutex_unlock(&userq->fence_drv_lock);
260
261 amdgpu_userq_fence_driver_get(fence_drv);
262 userq_fence->fence_drv = fence_drv;
263
264 *pfence = userq_fence;
265 return 0;
266}
267
268static void amdgpu_userq_fence_init(struct amdgpu_usermode_queue *userq,
269 struct amdgpu_userq_fence *fence,
270 u64 seq)
271{
272 struct amdgpu_userq_fence_driver *fence_drv = userq->fence_drv;
273 unsigned long flags;
274 bool signaled = false;
275
276 spin_lock_init(&fence->lock);
277 dma_fence_init64(&fence->base, &amdgpu_userq_fence_ops, &fence->lock,
278 fence_drv->context, seq);
279
280 /* Make sure the fence is visible to the hang detect worker */
281 dma_fence_put(userq->last_fence);
282 userq->last_fence = dma_fence_get(&fence->base);
283
284 /* Check if hardware has already processed the fence */
285 spin_lock_irqsave(&fence_drv->fence_list_lock, flags);
286 if (!dma_fence_is_signaled(&fence->base)) {
287 dma_fence_get(&fence->base);
288 list_add_tail(&fence->link, &fence_drv->fences);
289 } else {
290 INIT_LIST_HEAD(&fence->link);
291 signaled = true;
292 }
293 spin_unlock_irqrestore(&fence_drv->fence_list_lock, flags);
294
295 if (signaled)
296 amdgpu_userq_fence_put_fence_drv_array(fence);
297 else
298 amdgpu_userq_start_hang_detect_work(userq);
299}
300
301static const char *amdgpu_userq_fence_get_driver_name(struct dma_fence *f)
302{
303 return "amdgpu_userq_fence";
304}
305
306static const char *amdgpu_userq_fence_get_timeline_name(struct dma_fence *f)
307{
308 struct amdgpu_userq_fence *fence = to_amdgpu_userq_fence(f);
309
310 return fence->fence_drv->timeline_name;
311}
312
313static bool amdgpu_userq_fence_signaled(struct dma_fence *f)
314{
315 struct amdgpu_userq_fence *fence = to_amdgpu_userq_fence(f);
316 struct amdgpu_userq_fence_driver *fence_drv = fence->fence_drv;
317 u64 rptr, wptr;
318
319 rptr = amdgpu_userq_fence_read(fence_drv);
320 wptr = fence->base.seqno;
321
322 if (rptr >= wptr)
323 return true;
324
325 return false;
326}
327
328static void amdgpu_userq_fence_free(struct rcu_head *rcu)
329{
330 struct dma_fence *fence = container_of(rcu, struct dma_fence, rcu);
331 struct amdgpu_userq_fence *userq_fence = to_amdgpu_userq_fence(fence);
332 struct amdgpu_userq_fence_driver *fence_drv = userq_fence->fence_drv;
333
334 /* Release the fence driver reference */
335 amdgpu_userq_fence_driver_put(fence_drv);
336
337 kvfree(userq_fence->fence_drv_array);
338 kfree(userq_fence);
339}
340
341static void amdgpu_userq_fence_release(struct dma_fence *f)
342{
343 call_rcu(&f->rcu, amdgpu_userq_fence_free);
344}
345
346static const struct dma_fence_ops amdgpu_userq_fence_ops = {
347 .get_driver_name = amdgpu_userq_fence_get_driver_name,
348 .get_timeline_name = amdgpu_userq_fence_get_timeline_name,
349 .signaled = amdgpu_userq_fence_signaled,
350 .release = amdgpu_userq_fence_release,
351};
352
353/**
354 * amdgpu_userq_fence_read_wptr - Read the userq wptr value
355 *
356 * @adev: amdgpu_device pointer
357 * @queue: user mode queue structure pointer
358 * @wptr: write pointer value
359 *
360 * Read the wptr value from userq's MQD. The userq signal IOCTL
361 * creates a dma_fence for the shared buffers that expects the
362 * RPTR value written to seq64 memory >= WPTR.
363 *
364 * Returns wptr value on success, error on failure.
365 */
366static int amdgpu_userq_fence_read_wptr(struct amdgpu_device *adev,
367 struct amdgpu_usermode_queue *queue,
368 u64 *wptr)
369{
370 struct amdgpu_bo_va_mapping *mapping;
371 struct amdgpu_bo *bo;
372 struct drm_exec exec;
373 u64 addr, *ptr;
374 int ret;
375
376 addr = queue->userq_prop->wptr_gpu_addr;
377 addr &= AMDGPU_GMC_HOLE_MASK;
378
379 drm_exec_init(&exec, DRM_EXEC_IGNORE_DUPLICATES, 2);
380 drm_exec_until_all_locked(&exec) {
381 ret = amdgpu_vm_lock_pd(queue->vm, &exec, 1);
382 drm_exec_retry_on_contention(&exec);
383 if (unlikely(ret))
384 goto lock_error;
385
386 mapping = amdgpu_vm_bo_lookup_mapping(queue->vm, addr >> PAGE_SHIFT);
387 if (!mapping) {
388 ret = -EINVAL;
389 goto lock_error;
390 }
391
392 ret = drm_exec_lock_obj(&exec, &mapping->bo_va->base.bo->tbo.base);
393 drm_exec_retry_on_contention(&exec);
394 if (unlikely(ret))
395 goto lock_error;
396 }
397
398 bo = mapping->bo_va->base.bo;
399 ret = amdgpu_bo_kmap(bo, (void **)&ptr);
400 if (ret) {
401 DRM_ERROR("Failed mapping the userqueue wptr bo");
402 goto lock_error;
403 }
404
405 *wptr = le64_to_cpu(*ptr);
406
407 amdgpu_bo_kunmap(bo);
408 drm_exec_fini(&exec);
409 return 0;
410
411lock_error:
412 drm_exec_fini(&exec);
413 return ret;
414}
415
416static void
417amdgpu_userq_fence_driver_set_error(struct amdgpu_userq_fence *fence,
418 int error)
419{
420 struct amdgpu_userq_fence_driver *fence_drv = fence->fence_drv;
421 unsigned long flags;
422 struct dma_fence *f;
423
424 spin_lock_irqsave(&fence_drv->fence_list_lock, flags);
425 lockdep_assert_held(&fence_drv->fence_list_lock);
426 f = rcu_dereference_protected(&fence->base,
427 lockdep_is_held(&fence_drv->fence_list_lock));
428 if (f) {
429 /* nest f->lock inside fence_list_lock */
430 spin_lock(dma_fence_spinlock(f));
431 if (!dma_fence_is_signaled_locked(f))
432 dma_fence_set_error(f, error);
433 spin_unlock(dma_fence_spinlock(f));
434 }
435 spin_unlock_irqrestore(&fence_drv->fence_list_lock, flags);
436}
437
438void
439amdgpu_userq_fence_driver_force_completion(struct amdgpu_usermode_queue *userq)
440{
441 struct dma_fence *f = userq->last_fence;
442
443 if (f) {
444 struct amdgpu_userq_fence *fence = to_amdgpu_userq_fence(f);
445 struct amdgpu_userq_fence_driver *fence_drv = fence->fence_drv;
446 u64 wptr = fence->base.seqno;
447
448 amdgpu_userq_fence_driver_set_error(fence, -ECANCELED);
449 amdgpu_userq_fence_write(fence_drv, wptr);
450 amdgpu_userq_fence_driver_process(fence_drv);
451
452 }
453}
454
455int amdgpu_userq_signal_ioctl(struct drm_device *dev, void *data,
456 struct drm_file *filp)
457{
458 struct amdgpu_device *adev = drm_to_adev(dev);
459 struct drm_amdgpu_userq_signal *args = data;
460 const unsigned int num_write_bo_handles = args->num_bo_write_handles;
461 const unsigned int num_read_bo_handles = args->num_bo_read_handles;
462 struct amdgpu_fpriv *fpriv = filp->driver_priv;
463 struct amdgpu_userq_mgr *userq_mgr = &fpriv->userq_mgr;
464
465 struct drm_gem_object **gobj_write, **gobj_read;
466 u32 *syncobj_handles, num_syncobj_handles;
467 struct amdgpu_usermode_queue *queue;
468 struct amdgpu_userq_fence *fence;
469 struct drm_syncobj **syncobj;
470 struct drm_exec exec;
471 void __user *ptr;
472 int r, i, entry;
473 u64 wptr;
474
475 if (!amdgpu_userq_enabled(dev))
476 return -ENOTSUPP;
477
478 if (args->num_bo_write_handles > AMDGPU_USERQ_MAX_HANDLES ||
479 args->num_bo_read_handles > AMDGPU_USERQ_MAX_HANDLES)
480 return -EINVAL;
481
482 num_syncobj_handles = args->num_syncobj_handles;
483 ptr = u64_to_user_ptr(args->syncobj_handles);
484 syncobj_handles = memdup_array_user(ptr, num_syncobj_handles,
485 sizeof(u32));
486 if (IS_ERR(syncobj_handles))
487 return PTR_ERR(syncobj_handles);
488
489 syncobj = kmalloc_array(num_syncobj_handles, sizeof(*syncobj),
490 GFP_KERNEL);
491 if (!syncobj) {
492 r = -ENOMEM;
493 goto free_syncobj_handles;
494 }
495
496 for (entry = 0; entry < num_syncobj_handles; entry++) {
497 syncobj[entry] = drm_syncobj_find(filp, syncobj_handles[entry]);
498 if (!syncobj[entry]) {
499 r = -ENOENT;
500 goto free_syncobj;
501 }
502 }
503
504 ptr = u64_to_user_ptr(args->bo_read_handles);
505 r = drm_gem_objects_lookup(filp, ptr, num_read_bo_handles, &gobj_read);
506 if (r)
507 goto free_syncobj;
508
509 ptr = u64_to_user_ptr(args->bo_write_handles);
510 r = drm_gem_objects_lookup(filp, ptr, num_write_bo_handles,
511 &gobj_write);
512 if (r)
513 goto put_gobj_read;
514
515 queue = amdgpu_userq_get(userq_mgr, args->queue_id);
516 if (!queue) {
517 r = -ENOENT;
518 goto put_gobj_write;
519 }
520
521 r = amdgpu_userq_fence_read_wptr(adev, queue, &wptr);
522 if (r)
523 goto put_queue;
524
525 r = amdgpu_userq_fence_alloc(queue, &fence);
526 if (r)
527 goto put_queue;
528
529 /* We are here means UQ is active, make sure the eviction fence is valid */
530 amdgpu_userq_ensure_ev_fence(&fpriv->userq_mgr, &fpriv->evf_mgr);
531
532 /* Create the new fence */
533 amdgpu_userq_fence_init(queue, fence, wptr);
534
535 trace_amdgpu_userq_emit_fence(dev->dev, queue, fence);
536
537 mutex_unlock(&userq_mgr->userq_mutex);
538
539 /*
540 * This needs to come after the fence is created since
541 * amdgpu_userq_ensure_ev_fence() can't be called while holding the resv
542 * locks.
543 */
544 drm_exec_init(&exec, DRM_EXEC_INTERRUPTIBLE_WAIT | DRM_EXEC_IGNORE_DUPLICATES,
545 (num_read_bo_handles + num_write_bo_handles));
546
547 drm_exec_until_all_locked(&exec) {
548 r = drm_exec_prepare_array(&exec, gobj_read,
549 num_read_bo_handles, 1);
550 drm_exec_retry_on_contention(&exec);
551 if (r)
552 goto exec_fini;
553
554 r = drm_exec_prepare_array(&exec, gobj_write,
555 num_write_bo_handles, 1);
556 drm_exec_retry_on_contention(&exec);
557 if (r)
558 goto exec_fini;
559 }
560
561 /* And publish the new fence in the BOs and syncobj */
562 for (i = 0; i < num_read_bo_handles; i++)
563 dma_resv_add_fence(gobj_read[i]->resv, &fence->base,
564 DMA_RESV_USAGE_READ);
565
566 for (i = 0; i < num_write_bo_handles; i++)
567 dma_resv_add_fence(gobj_write[i]->resv, &fence->base,
568 DMA_RESV_USAGE_WRITE);
569
570 for (i = 0; i < num_syncobj_handles; i++)
571 drm_syncobj_replace_fence(syncobj[i], &fence->base);
572
573exec_fini:
574 /* drop the reference acquired in fence creation function */
575 dma_fence_put(&fence->base);
576
577 drm_exec_fini(&exec);
578put_queue:
579 amdgpu_userq_put(queue);
580put_gobj_write:
581 for (i = 0; i < num_write_bo_handles; i++)
582 drm_gem_object_put(gobj_write[i]);
583 kvfree(gobj_write);
584put_gobj_read:
585 for (i = 0; i < num_read_bo_handles; i++)
586 drm_gem_object_put(gobj_read[i]);
587 kvfree(gobj_read);
588free_syncobj:
589 while (entry-- > 0)
590 drm_syncobj_put(syncobj[entry]);
591 kfree(syncobj);
592free_syncobj_handles:
593 kfree(syncobj_handles);
594
595 return r;
596}
597
598/* Count the number of expected fences so userspace can alloc a buffer */
599static int
600amdgpu_userq_wait_count_fences(struct drm_file *filp,
601 struct drm_amdgpu_userq_wait *wait_info,
602 u32 *syncobj_handles, u64 *timeline_points,
603 u32 *timeline_handles,
604 struct drm_gem_object **gobj_write,
605 struct drm_gem_object **gobj_read)
606{
607 int num_read_bo_handles, num_write_bo_handles;
608 struct dma_fence_unwrap iter;
609 struct dma_fence *fence, *f;
610 unsigned int num_fences = 0;
611 struct drm_exec exec;
612 int i, r;
613
614 /*
615 * This needs to be outside of the lock provided by drm_exec for
616 * DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT to work correctly.
617 */
618
619 /* Count timeline fences */
620 for (i = 0; i < wait_info->num_syncobj_timeline_handles; i++) {
621 r = drm_syncobj_find_fence(filp, timeline_handles[i],
622 timeline_points[i],
623 DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT,
624 &fence);
625 if (r)
626 return r;
627
628 dma_fence_unwrap_for_each(f, &iter, fence)
629 num_fences++;
630
631 dma_fence_put(fence);
632 }
633
634 /* Count boolean fences */
635 for (i = 0; i < wait_info->num_syncobj_handles; i++) {
636 r = drm_syncobj_find_fence(filp, syncobj_handles[i], 0,
637 DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT,
638 &fence);
639 if (r)
640 return r;
641
642 num_fences++;
643 dma_fence_put(fence);
644 }
645
646 /* Lock all the GEM objects */
647 /* TODO: It is actually not necessary to lock them */
648 num_read_bo_handles = wait_info->num_bo_read_handles;
649 num_write_bo_handles = wait_info->num_bo_write_handles;
650 drm_exec_init(&exec, DRM_EXEC_INTERRUPTIBLE_WAIT | DRM_EXEC_IGNORE_DUPLICATES,
651 num_read_bo_handles + num_write_bo_handles);
652
653 drm_exec_until_all_locked(&exec) {
654 r = drm_exec_prepare_array(&exec, gobj_read,
655 num_read_bo_handles, 1);
656 drm_exec_retry_on_contention(&exec);
657 if (r)
658 goto error_unlock;
659
660 r = drm_exec_prepare_array(&exec, gobj_write,
661 num_write_bo_handles, 1);
662 drm_exec_retry_on_contention(&exec);
663 if (r)
664 goto error_unlock;
665 }
666
667 /* Count read fences */
668 for (i = 0; i < num_read_bo_handles; i++) {
669 struct dma_resv_iter resv_cursor;
670 struct dma_fence *fence;
671
672 dma_resv_for_each_fence(&resv_cursor, gobj_read[i]->resv,
673 DMA_RESV_USAGE_READ, fence)
674 num_fences++;
675 }
676
677 /* Count write fences */
678 for (i = 0; i < num_write_bo_handles; i++) {
679 struct dma_resv_iter resv_cursor;
680 struct dma_fence *fence;
681
682 dma_resv_for_each_fence(&resv_cursor, gobj_write[i]->resv,
683 DMA_RESV_USAGE_WRITE, fence)
684 num_fences++;
685 }
686
687 wait_info->num_fences = min(num_fences, USHRT_MAX);
688 r = 0;
689
690error_unlock:
691 /* Unlock all the GEM objects */
692 drm_exec_fini(&exec);
693 return r;
694}
695
696static int
697amdgpu_userq_wait_add_fence(struct drm_amdgpu_userq_wait *wait_info,
698 struct dma_fence **fences, unsigned int *num_fences,
699 struct dma_fence *fence)
700{
701 /* As fallback shouldn't userspace allocate enough space */
702 if (*num_fences >= wait_info->num_fences)
703 return dma_fence_wait(fence, true);
704
705 fences[(*num_fences)++] = dma_fence_get(fence);
706 return 0;
707}
708
709static int
710amdgpu_userq_wait_return_fence_info(struct drm_device *dev, struct drm_file *filp,
711 struct drm_amdgpu_userq_wait *wait_info,
712 u32 *syncobj_handles, u64 *timeline_points,
713 u32 *timeline_handles,
714 struct drm_gem_object **gobj_write,
715 struct drm_gem_object **gobj_read)
716{
717 struct amdgpu_fpriv *fpriv = filp->driver_priv;
718 struct amdgpu_userq_mgr *userq_mgr = &fpriv->userq_mgr;
719 struct drm_amdgpu_userq_fence_info *fence_info;
720 int num_read_bo_handles, num_write_bo_handles;
721 struct amdgpu_usermode_queue *waitq;
722 struct dma_fence **fences, *fence, *f;
723 struct dma_fence_unwrap iter;
724 int num_points, num_syncobj;
725 unsigned int num_fences = 0;
726 struct drm_exec exec;
727 int i, cnt, r;
728
729 fence_info = kmalloc_array(wait_info->num_fences, sizeof(*fence_info),
730 GFP_KERNEL);
731 if (!fence_info)
732 return -ENOMEM;
733
734 fences = kmalloc_array(wait_info->num_fences, sizeof(*fences),
735 GFP_KERNEL);
736 if (!fences) {
737 r = -ENOMEM;
738 goto free_fence_info;
739 }
740
741 /* Retrieve timeline fences */
742 num_points = wait_info->num_syncobj_timeline_handles;
743 for (i = 0; i < num_points; i++) {
744 r = drm_syncobj_find_fence(filp, timeline_handles[i],
745 timeline_points[i],
746 DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT,
747 &fence);
748 if (r)
749 goto free_fences;
750
751 dma_fence_unwrap_for_each(f, &iter, fence) {
752 r = amdgpu_userq_wait_add_fence(wait_info, fences,
753 &num_fences, f);
754 if (r) {
755 dma_fence_put(fence);
756 goto free_fences;
757 }
758 }
759
760 dma_fence_put(fence);
761 }
762
763 /* Retrieve boolean fences */
764 num_syncobj = wait_info->num_syncobj_handles;
765 for (i = 0; i < num_syncobj; i++) {
766 struct dma_fence *fence;
767
768 r = drm_syncobj_find_fence(filp, syncobj_handles[i], 0,
769 DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT,
770 &fence);
771 if (r)
772 goto free_fences;
773
774 r = amdgpu_userq_wait_add_fence(wait_info, fences,
775 &num_fences, fence);
776 dma_fence_put(fence);
777 if (r)
778 goto free_fences;
779
780 }
781
782 /* Lock all the GEM objects */
783 num_read_bo_handles = wait_info->num_bo_read_handles;
784 num_write_bo_handles = wait_info->num_bo_write_handles;
785 drm_exec_init(&exec, DRM_EXEC_INTERRUPTIBLE_WAIT | DRM_EXEC_IGNORE_DUPLICATES,
786 num_read_bo_handles + num_write_bo_handles);
787
788 drm_exec_until_all_locked(&exec) {
789 r = drm_exec_prepare_array(&exec, gobj_read,
790 num_read_bo_handles, 1);
791 drm_exec_retry_on_contention(&exec);
792 if (r)
793 goto error_unlock;
794
795 r = drm_exec_prepare_array(&exec, gobj_write,
796 num_write_bo_handles, 1);
797 drm_exec_retry_on_contention(&exec);
798 if (r)
799 goto error_unlock;
800 }
801
802 /* Retrieve GEM read objects fence */
803 for (i = 0; i < num_read_bo_handles; i++) {
804 struct dma_resv_iter resv_cursor;
805 struct dma_fence *fence;
806
807 dma_resv_for_each_fence(&resv_cursor, gobj_read[i]->resv,
808 DMA_RESV_USAGE_READ, fence) {
809 r = amdgpu_userq_wait_add_fence(wait_info, fences,
810 &num_fences, fence);
811 if (r)
812 goto error_unlock;
813 }
814 }
815
816 /* Retrieve GEM write objects fence */
817 for (i = 0; i < num_write_bo_handles; i++) {
818 struct dma_resv_iter resv_cursor;
819 struct dma_fence *fence;
820
821 dma_resv_for_each_fence(&resv_cursor, gobj_write[i]->resv,
822 DMA_RESV_USAGE_WRITE, fence) {
823 r = amdgpu_userq_wait_add_fence(wait_info, fences,
824 &num_fences, fence);
825 if (r)
826 goto error_unlock;
827 }
828 }
829
830 drm_exec_fini(&exec);
831
832 /*
833 * Keep only the latest fences to reduce the number of values
834 * given back to userspace.
835 */
836 num_fences = dma_fence_dedup_array(fences, num_fences);
837
838 waitq = amdgpu_userq_get(userq_mgr, wait_info->waitq_id);
839 if (!waitq) {
840 r = -EINVAL;
841 goto free_fences;
842 }
843
844 for (i = 0, cnt = 0; i < num_fences; i++) {
845 struct amdgpu_userq_fence_driver *fence_drv;
846 struct amdgpu_userq_fence *userq_fence;
847 u32 index;
848
849 userq_fence = to_amdgpu_userq_fence(fences[i]);
850 if (!userq_fence) {
851 /*
852 * Just waiting on other driver fences should
853 * be good for now
854 */
855 r = dma_fence_wait(fences[i], true);
856 if (r)
857 goto put_waitq;
858
859 continue;
860 }
861
862 fence_drv = userq_fence->fence_drv;
863 /*
864 * We need to make sure the user queue release their reference
865 * to the fence drivers at some point before queue destruction.
866 * Otherwise, we would gather those references until we don't
867 * have any more space left and crash.
868 */
869 mutex_lock(&waitq->fence_drv_lock);
870 r = xa_alloc(&waitq->fence_drv_xa, &index, fence_drv,
871 xa_limit_32b, GFP_KERNEL);
872 mutex_unlock(&waitq->fence_drv_lock);
873 if (r)
874 goto put_waitq;
875
876 amdgpu_userq_fence_driver_get(fence_drv);
877
878 trace_amdgpu_userq_wait_deps(dev->dev, waitq, userq_fence);
879
880 /* Store drm syncobj's gpu va address and value */
881 fence_info[cnt].va = fence_drv->va;
882 fence_info[cnt].value = fences[i]->seqno;
883
884 /* Increment the actual userq fence count */
885 cnt++;
886 }
887 wait_info->num_fences = cnt;
888
889 /* Copy userq fence info to user space */
890 if (copy_to_user(u64_to_user_ptr(wait_info->out_fences),
891 fence_info, cnt * sizeof(*fence_info)))
892 r = -EFAULT;
893 else
894 r = 0;
895
896put_waitq:
897 amdgpu_userq_put(waitq);
898
899free_fences:
900 while (num_fences--)
901 dma_fence_put(fences[num_fences]);
902 kfree(fences);
903
904free_fence_info:
905 kfree(fence_info);
906 return r;
907
908error_unlock:
909 drm_exec_fini(&exec);
910 goto free_fences;
911}
912
913int amdgpu_userq_wait_ioctl(struct drm_device *dev, void *data,
914 struct drm_file *filp)
915{
916 int num_points, num_syncobj, num_read_bo_handles, num_write_bo_handles;
917 u32 *syncobj_handles, *timeline_handles;
918 u64 *timeline_points;
919 struct drm_amdgpu_userq_wait *wait_info = data;
920 struct drm_gem_object **gobj_write;
921 struct drm_gem_object **gobj_read;
922 void __user *ptr;
923 int r;
924
925 if (!amdgpu_userq_enabled(dev))
926 return -ENOTSUPP;
927
928 if (wait_info->num_bo_write_handles > AMDGPU_USERQ_MAX_HANDLES ||
929 wait_info->num_bo_read_handles > AMDGPU_USERQ_MAX_HANDLES)
930 return -EINVAL;
931
932 num_syncobj = wait_info->num_syncobj_handles;
933 ptr = u64_to_user_ptr(wait_info->syncobj_handles);
934 syncobj_handles = memdup_array_user(ptr, num_syncobj, sizeof(u32));
935 if (IS_ERR(syncobj_handles))
936 return PTR_ERR(syncobj_handles);
937
938 num_points = wait_info->num_syncobj_timeline_handles;
939 ptr = u64_to_user_ptr(wait_info->syncobj_timeline_handles);
940 timeline_handles = memdup_array_user(ptr, num_points, sizeof(u32));
941 if (IS_ERR(timeline_handles)) {
942 r = PTR_ERR(timeline_handles);
943 goto free_syncobj_handles;
944 }
945
946 ptr = u64_to_user_ptr(wait_info->syncobj_timeline_points);
947 timeline_points = memdup_array_user(ptr, num_points, sizeof(u64));
948 if (IS_ERR(timeline_points)) {
949 r = PTR_ERR(timeline_points);
950 goto free_timeline_handles;
951 }
952
953 num_read_bo_handles = wait_info->num_bo_read_handles;
954 ptr = u64_to_user_ptr(wait_info->bo_read_handles);
955 r = drm_gem_objects_lookup(filp, ptr, num_read_bo_handles, &gobj_read);
956 if (r)
957 goto free_timeline_points;
958
959 num_write_bo_handles = wait_info->num_bo_write_handles;
960 ptr = u64_to_user_ptr(wait_info->bo_write_handles);
961 r = drm_gem_objects_lookup(filp, ptr, num_write_bo_handles,
962 &gobj_write);
963 if (r)
964 goto put_gobj_read;
965
966 /*
967 * Passing num_fences = 0 means that userspace doesn't want to
968 * retrieve userq_fence_info. If num_fences = 0 we skip filling
969 * userq_fence_info and return the actual number of fences on
970 * args->num_fences.
971 */
972 if (!wait_info->num_fences) {
973 r = amdgpu_userq_wait_count_fences(filp, wait_info,
974 syncobj_handles,
975 timeline_points,
976 timeline_handles,
977 gobj_write,
978 gobj_read);
979 } else {
980 r = amdgpu_userq_wait_return_fence_info(dev, filp, wait_info,
981 syncobj_handles,
982 timeline_points,
983 timeline_handles,
984 gobj_write,
985 gobj_read);
986 }
987
988 while (num_write_bo_handles--)
989 drm_gem_object_put(gobj_write[num_write_bo_handles]);
990 kvfree(gobj_write);
991
992put_gobj_read:
993 while (num_read_bo_handles--)
994 drm_gem_object_put(gobj_read[num_read_bo_handles]);
995 kvfree(gobj_read);
996
997free_timeline_points:
998 kfree(timeline_points);
999free_timeline_handles:
1000 kfree(timeline_handles);
1001free_syncobj_handles:
1002 kfree(syncobj_handles);
1003 return r;
1004}