projects/linux·blob master · refs

1841 lines · 45.4 KiB

1/*
2 * Copyright 2008 Jerome Glisse.
3 * All Rights Reserved.
4 *
5 * Permission is hereby granted, free of charge, to any person obtaining a
6 * copy of this software and associated documentation files (the "Software"),
7 * to deal in the Software without restriction, including without limitation
8 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
9 * and/or sell copies of the Software, and to permit persons to whom the
10 * Software is furnished to do so, subject to the following conditions:
11 *
12 * The above copyright notice and this permission notice (including the next
13 * paragraph) shall be included in all copies or substantial portions of the
14 * Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * PRECISION INSIGHT AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
20 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
21 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
22 * DEALINGS IN THE SOFTWARE.
23 *
24 * Authors:
25 * Jerome Glisse <[email protected]>
26 */
27
28#include <linux/file.h>
29#include <linux/pagemap.h>
30#include <linux/sync_file.h>
31#include <linux/dma-buf.h>
32
33#include <drm/amdgpu_drm.h>
34#include <drm/drm_syncobj.h>
35#include <drm/ttm/ttm_tt.h>
36
37#include "amdgpu_cs.h"
38#include "amdgpu.h"
39#include "amdgpu_trace.h"
40#include "amdgpu_gmc.h"
41#include "amdgpu_gem.h"
42#include "amdgpu_ras.h"
43#include "amdgpu_hmm.h"
44
45/*
46 * Maximum IB length (dwords) for rings whose emit_ib packet format
47 * documents a 20-bit size field.
48 */
49#define AMDGPU_GFX_SDMA_IB_PACKET_SIZE_MAX_DW 0xFFFFF
50#define AMDGPU_MM_IB_PACKET_SIZE_MAX_DW 0x7FFFF0
51
52static u32 amdgpu_cs_ib_packet_size_max_dw(enum amdgpu_ring_type type)
53{
54 switch (type) {
55 case AMDGPU_RING_TYPE_GFX:
56 case AMDGPU_RING_TYPE_COMPUTE:
57 case AMDGPU_RING_TYPE_SDMA:
58 case AMDGPU_RING_TYPE_VPE:
59 return AMDGPU_GFX_SDMA_IB_PACKET_SIZE_MAX_DW;
60 default:
61 return AMDGPU_MM_IB_PACKET_SIZE_MAX_DW;
62 }
63}
64
65static int amdgpu_cs_parser_init(struct amdgpu_cs_parser *p,
66 struct amdgpu_device *adev,
67 struct drm_file *filp,
68 union drm_amdgpu_cs *cs)
69{
70 struct amdgpu_fpriv *fpriv = filp->driver_priv;
71
72 if (cs->in.num_chunks == 0)
73 return -EINVAL;
74
75 memset(p, 0, sizeof(*p));
76 p->adev = adev;
77 p->filp = filp;
78
79 p->ctx = amdgpu_ctx_get(fpriv, cs->in.ctx_id);
80 if (!p->ctx)
81 return -EINVAL;
82
83 amdgpu_sync_create(&p->sync);
84 drm_exec_init(&p->exec, DRM_EXEC_INTERRUPTIBLE_WAIT |
85 DRM_EXEC_IGNORE_DUPLICATES, 0);
86 return 0;
87}
88
89static int amdgpu_cs_job_idx(struct amdgpu_cs_parser *p,
90 struct drm_amdgpu_cs_chunk_ib *chunk_ib)
91{
92 struct drm_sched_entity *entity;
93 unsigned int i;
94 int r;
95
96 r = amdgpu_ctx_get_entity(p->ctx, chunk_ib->ip_type,
97 chunk_ib->ip_instance,
98 chunk_ib->ring, &entity);
99 if (r)
100 return r;
101
102 /* Check if we can add this IB to some existing job */
103 for (i = 0; i < p->gang_size; ++i)
104 if (p->entities[i] == entity)
105 return i;
106
107 /* If not increase the gang size if possible */
108 if (i == AMDGPU_CS_GANG_SIZE)
109 return -EINVAL;
110
111 p->entities[i] = entity;
112 p->gang_size = i + 1;
113 return i;
114}
115
116static int amdgpu_cs_p1_ib(struct amdgpu_cs_parser *p,
117 struct drm_amdgpu_cs_chunk_ib *chunk_ib,
118 unsigned int *num_ibs)
119{
120 int r;
121
122 r = amdgpu_cs_job_idx(p, chunk_ib);
123 if (r < 0)
124 return r;
125
126 if (num_ibs[r] >= amdgpu_ring_max_ibs(chunk_ib->ip_type))
127 return -EINVAL;
128
129 ++(num_ibs[r]);
130 p->gang_leader_idx = r;
131 return 0;
132}
133
134static int amdgpu_cs_p1_user_fence(struct amdgpu_cs_parser *p,
135 struct drm_amdgpu_cs_chunk_fence *data,
136 uint32_t *offset)
137{
138 struct drm_gem_object *gobj;
139 unsigned long size;
140
141 gobj = drm_gem_object_lookup(p->filp, data->handle);
142 if (gobj == NULL)
143 return -EINVAL;
144
145 p->uf_bo = amdgpu_bo_ref(gem_to_amdgpu_bo(gobj));
146 drm_gem_object_put(gobj);
147
148 size = amdgpu_bo_size(p->uf_bo);
149 if (size != PAGE_SIZE || data->offset > (size - 8))
150 return -EINVAL;
151
152 if (amdgpu_ttm_tt_get_usermm(p->uf_bo->tbo.ttm))
153 return -EINVAL;
154
155 *offset = data->offset;
156 return 0;
157}
158
159static int amdgpu_cs_p1_bo_handles(struct amdgpu_cs_parser *p,
160 struct drm_amdgpu_bo_list_in *data)
161{
162 struct drm_amdgpu_bo_list_entry *info;
163 struct amdgpu_bo_list *list;
164
165 info = amdgpu_bo_create_list_entry_array(data);
166 if (IS_ERR(info))
167 return PTR_ERR(info);
168
169 list = amdgpu_bo_list_create(p->adev, p->filp, info, data->bo_number);
170 kvfree(info);
171 if (IS_ERR(list))
172 return PTR_ERR(list);
173
174 p->bo_list = list;
175 return 0;
176}
177
178/* Copy the data from userspace and go over it the first time */
179static int amdgpu_cs_pass1(struct amdgpu_cs_parser *p,
180 union drm_amdgpu_cs *cs)
181{
182 struct amdgpu_fpriv *fpriv = p->filp->driver_priv;
183 unsigned int num_ibs[AMDGPU_CS_GANG_SIZE] = { };
184 struct amdgpu_vm *vm = &fpriv->vm;
185 uint64_t *chunk_array;
186 uint32_t uf_offset = 0;
187 size_t size;
188 int ret;
189 int i;
190
191 chunk_array = memdup_array_user(u64_to_user_ptr(cs->in.chunks),
192 cs->in.num_chunks,
193 sizeof(uint64_t));
194 if (IS_ERR(chunk_array))
195 return PTR_ERR(chunk_array);
196
197 p->nchunks = cs->in.num_chunks;
198 p->chunks = kvmalloc_objs(struct amdgpu_cs_chunk, p->nchunks);
199 if (!p->chunks) {
200 ret = -ENOMEM;
201 goto free_chunk;
202 }
203
204 for (i = 0; i < p->nchunks; i++) {
205 struct drm_amdgpu_cs_chunk __user *chunk_ptr = NULL;
206 struct drm_amdgpu_cs_chunk user_chunk;
207
208 chunk_ptr = u64_to_user_ptr(chunk_array[i]);
209 if (copy_from_user(&user_chunk, chunk_ptr,
210 sizeof(struct drm_amdgpu_cs_chunk))) {
211 ret = -EFAULT;
212 i--;
213 goto free_partial_kdata;
214 }
215 p->chunks[i].chunk_id = user_chunk.chunk_id;
216 p->chunks[i].length_dw = user_chunk.length_dw;
217
218 size = p->chunks[i].length_dw;
219
220 p->chunks[i].kdata = vmemdup_array_user(u64_to_user_ptr(user_chunk.chunk_data),
221 size,
222 sizeof(uint32_t));
223 if (IS_ERR(p->chunks[i].kdata)) {
224 ret = PTR_ERR(p->chunks[i].kdata);
225 i--;
226 goto free_partial_kdata;
227 }
228 size *= sizeof(uint32_t);
229
230 /* Assume the worst on the following checks */
231 ret = -EINVAL;
232 switch (p->chunks[i].chunk_id) {
233 case AMDGPU_CHUNK_ID_IB:
234 if (size < sizeof(struct drm_amdgpu_cs_chunk_ib))
235 goto free_partial_kdata;
236
237 ret = amdgpu_cs_p1_ib(p, p->chunks[i].kdata, num_ibs);
238 if (ret)
239 goto free_partial_kdata;
240 break;
241
242 case AMDGPU_CHUNK_ID_FENCE:
243 if (size < sizeof(struct drm_amdgpu_cs_chunk_fence))
244 goto free_partial_kdata;
245
246 /* Only a single user fence is allowed to simplify handling. */
247 if (p->uf_bo)
248 goto free_partial_kdata;
249
250 ret = amdgpu_cs_p1_user_fence(p, p->chunks[i].kdata,
251 &uf_offset);
252 if (ret)
253 goto free_partial_kdata;
254 break;
255
256 case AMDGPU_CHUNK_ID_BO_HANDLES:
257 if (size < sizeof(struct drm_amdgpu_bo_list_in))
258 goto free_partial_kdata;
259
260 /* Only a single BO list is allowed to simplify handling. */
261 if (p->bo_list)
262 goto free_partial_kdata;
263
264 ret = amdgpu_cs_p1_bo_handles(p, p->chunks[i].kdata);
265 if (ret)
266 goto free_partial_kdata;
267 break;
268
269 case AMDGPU_CHUNK_ID_CP_GFX_SHADOW:
270 if (size < sizeof(struct drm_amdgpu_cs_chunk_cp_gfx_shadow))
271 goto free_partial_kdata;
272 break;
273
274 case AMDGPU_CHUNK_ID_DEPENDENCIES:
275 case AMDGPU_CHUNK_ID_SYNCOBJ_IN:
276 case AMDGPU_CHUNK_ID_SYNCOBJ_OUT:
277 case AMDGPU_CHUNK_ID_SCHEDULED_DEPENDENCIES:
278 case AMDGPU_CHUNK_ID_SYNCOBJ_TIMELINE_WAIT:
279 case AMDGPU_CHUNK_ID_SYNCOBJ_TIMELINE_SIGNAL:
280 break;
281
282 default:
283 goto free_partial_kdata;
284 }
285 }
286
287 if (!p->gang_size || (amdgpu_sriov_vf(p->adev) && p->gang_size > 1)) {
288 ret = -EINVAL;
289 goto free_all_kdata;
290 }
291
292 for (i = 0; i < p->gang_size; ++i) {
293 ret = amdgpu_job_alloc(p->adev, vm, p->entities[i], vm,
294 num_ibs[i], p->filp->client_id,
295 GFP_KERNEL, &p->jobs[i]);
296 if (ret)
297 goto free_all_kdata;
298 switch (p->adev->enforce_isolation[fpriv->xcp_id]) {
299 case AMDGPU_ENFORCE_ISOLATION_DISABLE:
300 default:
301 p->jobs[i]->enforce_isolation = false;
302 p->jobs[i]->run_cleaner_shader = false;
303 break;
304 case AMDGPU_ENFORCE_ISOLATION_ENABLE:
305 p->jobs[i]->enforce_isolation = true;
306 p->jobs[i]->run_cleaner_shader = true;
307 break;
308 case AMDGPU_ENFORCE_ISOLATION_ENABLE_LEGACY:
309 p->jobs[i]->enforce_isolation = true;
310 p->jobs[i]->run_cleaner_shader = false;
311 break;
312 case AMDGPU_ENFORCE_ISOLATION_NO_CLEANER_SHADER:
313 p->jobs[i]->enforce_isolation = true;
314 p->jobs[i]->run_cleaner_shader = false;
315 break;
316 }
317 }
318 p->gang_leader = p->jobs[p->gang_leader_idx];
319
320 if (p->ctx->generation != p->gang_leader->generation) {
321 ret = -ECANCELED;
322 goto free_all_kdata;
323 }
324
325 if (p->uf_bo)
326 p->gang_leader->uf_addr = uf_offset;
327 kvfree(chunk_array);
328
329 /* Use this opportunity to fill in task info for the vm */
330 amdgpu_vm_set_task_info(vm);
331
332 return 0;
333
334free_all_kdata:
335 i = p->nchunks - 1;
336free_partial_kdata:
337 for (; i >= 0; i--)
338 kvfree(p->chunks[i].kdata);
339 kvfree(p->chunks);
340 p->chunks = NULL;
341 p->nchunks = 0;
342free_chunk:
343 kvfree(chunk_array);
344
345 return ret;
346}
347
348static int amdgpu_cs_p2_ib(struct amdgpu_cs_parser *p,
349 struct amdgpu_cs_chunk *chunk,
350 unsigned int *ce_preempt,
351 unsigned int *de_preempt)
352{
353 struct drm_amdgpu_cs_chunk_ib *chunk_ib = chunk->kdata;
354 struct amdgpu_fpriv *fpriv = p->filp->driver_priv;
355 struct amdgpu_vm *vm = &fpriv->vm;
356 struct amdgpu_ring *ring;
357 struct amdgpu_job *job;
358 struct amdgpu_ib *ib;
359 int r;
360
361 r = amdgpu_cs_job_idx(p, chunk_ib);
362 if (r < 0)
363 return r;
364
365 job = p->jobs[r];
366 ring = amdgpu_job_ring(job);
367
368 /* submissions to kernel queues are disabled */
369 if (ring->no_user_submission)
370 return -EINVAL;
371
372 /* MM engine doesn't support user fences */
373 if (p->uf_bo && ring->funcs->no_user_fence)
374 return -EINVAL;
375
376 if (!p->adev->debug_enable_ce_cs &&
377 chunk_ib->flags & AMDGPU_IB_FLAG_CE) {
378 dev_err_ratelimited(p->adev->dev, "CE CS is blocked, use debug=0x400 to override\n");
379 return -EINVAL;
380 }
381
382 if (chunk_ib->ip_type == AMDGPU_HW_IP_GFX &&
383 chunk_ib->flags & AMDGPU_IB_FLAG_PREEMPT) {
384 if (chunk_ib->flags & AMDGPU_IB_FLAG_CE)
385 (*ce_preempt)++;
386 else
387 (*de_preempt)++;
388
389 /* Each GFX command submit allows only 1 IB max
390 * preemptible for CE & DE */
391 if (*ce_preempt > 1 || *de_preempt > 1)
392 return -EINVAL;
393 }
394
395 if (chunk_ib->ib_bytes / 4 >
396 amdgpu_cs_ib_packet_size_max_dw(ring->funcs->type))
397 return -EINVAL;
398
399 ib = &job->ibs[job->num_ibs++];
400
401 if (chunk_ib->flags & AMDGPU_IB_FLAG_PREAMBLE)
402 job->preamble_status |= AMDGPU_PREAMBLE_IB_PRESENT;
403
404 r = amdgpu_ib_get(p->adev, vm, ring->funcs->parse_cs ?
405 chunk_ib->ib_bytes : 0,
406 AMDGPU_IB_POOL_DELAYED, ib);
407 if (r) {
408 drm_err(adev_to_drm(p->adev), "Failed to get ib !\n");
409 return r;
410 }
411
412 ib->gpu_addr = chunk_ib->va_start;
413 ib->length_dw = chunk_ib->ib_bytes / 4;
414 ib->flags = chunk_ib->flags;
415 return 0;
416}
417
418static int amdgpu_cs_p2_dependencies(struct amdgpu_cs_parser *p,
419 struct amdgpu_cs_chunk *chunk)
420{
421 struct drm_amdgpu_cs_chunk_dep *deps = chunk->kdata;
422 struct amdgpu_fpriv *fpriv = p->filp->driver_priv;
423 unsigned int num_deps;
424 int i, r;
425
426 num_deps = chunk->length_dw * 4 /
427 sizeof(struct drm_amdgpu_cs_chunk_dep);
428
429 for (i = 0; i < num_deps; ++i) {
430 struct amdgpu_ctx *ctx;
431 struct drm_sched_entity *entity;
432 struct dma_fence *fence;
433
434 ctx = amdgpu_ctx_get(fpriv, deps[i].ctx_id);
435 if (ctx == NULL)
436 return -EINVAL;
437
438 r = amdgpu_ctx_get_entity(ctx, deps[i].ip_type,
439 deps[i].ip_instance,
440 deps[i].ring, &entity);
441 if (r) {
442 amdgpu_ctx_put(ctx);
443 return r;
444 }
445
446 fence = amdgpu_ctx_get_fence(ctx, entity, deps[i].handle);
447 amdgpu_ctx_put(ctx);
448
449 if (IS_ERR(fence))
450 return PTR_ERR(fence);
451 else if (!fence)
452 continue;
453
454 if (chunk->chunk_id == AMDGPU_CHUNK_ID_SCHEDULED_DEPENDENCIES) {
455 struct drm_sched_fence *s_fence;
456 struct dma_fence *old = fence;
457
458 s_fence = to_drm_sched_fence(fence);
459 fence = dma_fence_get(&s_fence->scheduled);
460 dma_fence_put(old);
461 }
462
463 r = amdgpu_sync_fence(&p->sync, fence, GFP_KERNEL);
464 dma_fence_put(fence);
465 if (r)
466 return r;
467 }
468 return 0;
469}
470
471static int amdgpu_syncobj_lookup_and_add(struct amdgpu_cs_parser *p,
472 uint32_t handle, u64 point,
473 u64 flags)
474{
475 struct dma_fence *fence;
476 int r;
477
478 r = drm_syncobj_find_fence(p->filp, handle, point, flags, &fence);
479 if (r) {
480 drm_err(adev_to_drm(p->adev), "syncobj %u failed to find fence @ %llu (%d)!\n",
481 handle, point, r);
482 return r;
483 }
484
485 r = amdgpu_sync_fence(&p->sync, fence, GFP_KERNEL);
486 dma_fence_put(fence);
487 return r;
488}
489
490static int amdgpu_cs_p2_syncobj_in(struct amdgpu_cs_parser *p,
491 struct amdgpu_cs_chunk *chunk)
492{
493 struct drm_amdgpu_cs_chunk_sem *deps = chunk->kdata;
494 unsigned int num_deps;
495 int i, r;
496
497 num_deps = chunk->length_dw * 4 /
498 sizeof(struct drm_amdgpu_cs_chunk_sem);
499 for (i = 0; i < num_deps; ++i) {
500 r = amdgpu_syncobj_lookup_and_add(p, deps[i].handle, 0, 0);
501 if (r)
502 return r;
503 }
504
505 return 0;
506}
507
508static int amdgpu_cs_p2_syncobj_timeline_wait(struct amdgpu_cs_parser *p,
509 struct amdgpu_cs_chunk *chunk)
510{
511 struct drm_amdgpu_cs_chunk_syncobj *syncobj_deps = chunk->kdata;
512 unsigned int num_deps;
513 int i, r;
514
515 num_deps = chunk->length_dw * 4 /
516 sizeof(struct drm_amdgpu_cs_chunk_syncobj);
517 for (i = 0; i < num_deps; ++i) {
518 r = amdgpu_syncobj_lookup_and_add(p, syncobj_deps[i].handle,
519 syncobj_deps[i].point,
520 syncobj_deps[i].flags);
521 if (r)
522 return r;
523 }
524
525 return 0;
526}
527
528static int amdgpu_cs_p2_syncobj_out(struct amdgpu_cs_parser *p,
529 struct amdgpu_cs_chunk *chunk)
530{
531 struct drm_amdgpu_cs_chunk_sem *deps = chunk->kdata;
532 unsigned int num_deps;
533 int i;
534
535 num_deps = chunk->length_dw * 4 /
536 sizeof(struct drm_amdgpu_cs_chunk_sem);
537
538 if (p->post_deps)
539 return -EINVAL;
540
541 p->post_deps = kmalloc_objs(*p->post_deps, num_deps);
542 p->num_post_deps = 0;
543
544 if (!p->post_deps)
545 return -ENOMEM;
546
547
548 for (i = 0; i < num_deps; ++i) {
549 p->post_deps[i].syncobj =
550 drm_syncobj_find(p->filp, deps[i].handle);
551 if (!p->post_deps[i].syncobj)
552 return -EINVAL;
553 p->post_deps[i].chain = NULL;
554 p->post_deps[i].point = 0;
555 p->num_post_deps++;
556 }
557
558 return 0;
559}
560
561static int amdgpu_cs_p2_syncobj_timeline_signal(struct amdgpu_cs_parser *p,
562 struct amdgpu_cs_chunk *chunk)
563{
564 struct drm_amdgpu_cs_chunk_syncobj *syncobj_deps = chunk->kdata;
565 unsigned int num_deps;
566 int i;
567
568 num_deps = chunk->length_dw * 4 /
569 sizeof(struct drm_amdgpu_cs_chunk_syncobj);
570
571 if (p->post_deps)
572 return -EINVAL;
573
574 p->post_deps = kmalloc_objs(*p->post_deps, num_deps);
575 p->num_post_deps = 0;
576
577 if (!p->post_deps)
578 return -ENOMEM;
579
580 for (i = 0; i < num_deps; ++i) {
581 struct amdgpu_cs_post_dep *dep = &p->post_deps[i];
582
583 dep->chain = NULL;
584 if (syncobj_deps[i].point) {
585 dep->chain = dma_fence_chain_alloc();
586 if (!dep->chain)
587 return -ENOMEM;
588 }
589
590 dep->syncobj = drm_syncobj_find(p->filp,
591 syncobj_deps[i].handle);
592 if (!dep->syncobj) {
593 dma_fence_chain_free(dep->chain);
594 return -EINVAL;
595 }
596 dep->point = syncobj_deps[i].point;
597 p->num_post_deps++;
598 }
599
600 return 0;
601}
602
603static int amdgpu_cs_p2_shadow(struct amdgpu_cs_parser *p,
604 struct amdgpu_cs_chunk *chunk)
605{
606 struct drm_amdgpu_cs_chunk_cp_gfx_shadow *shadow = chunk->kdata;
607 int i;
608
609 if (shadow->flags & ~AMDGPU_CS_CHUNK_CP_GFX_SHADOW_FLAGS_INIT_SHADOW)
610 return -EINVAL;
611
612 for (i = 0; i < p->gang_size; ++i) {
613 p->jobs[i]->shadow_va = shadow->shadow_va;
614 p->jobs[i]->csa_va = shadow->csa_va;
615 p->jobs[i]->gds_va = shadow->gds_va;
616 p->jobs[i]->init_shadow =
617 shadow->flags & AMDGPU_CS_CHUNK_CP_GFX_SHADOW_FLAGS_INIT_SHADOW;
618 }
619
620 return 0;
621}
622
623static int amdgpu_cs_pass2(struct amdgpu_cs_parser *p)
624{
625 unsigned int ce_preempt = 0, de_preempt = 0;
626 int i, r;
627
628 for (i = 0; i < p->nchunks; ++i) {
629 struct amdgpu_cs_chunk *chunk;
630
631 chunk = &p->chunks[i];
632
633 switch (chunk->chunk_id) {
634 case AMDGPU_CHUNK_ID_IB:
635 r = amdgpu_cs_p2_ib(p, chunk, &ce_preempt, &de_preempt);
636 if (r)
637 return r;
638 break;
639 case AMDGPU_CHUNK_ID_DEPENDENCIES:
640 case AMDGPU_CHUNK_ID_SCHEDULED_DEPENDENCIES:
641 r = amdgpu_cs_p2_dependencies(p, chunk);
642 if (r)
643 return r;
644 break;
645 case AMDGPU_CHUNK_ID_SYNCOBJ_IN:
646 r = amdgpu_cs_p2_syncobj_in(p, chunk);
647 if (r)
648 return r;
649 break;
650 case AMDGPU_CHUNK_ID_SYNCOBJ_OUT:
651 r = amdgpu_cs_p2_syncobj_out(p, chunk);
652 if (r)
653 return r;
654 break;
655 case AMDGPU_CHUNK_ID_SYNCOBJ_TIMELINE_WAIT:
656 r = amdgpu_cs_p2_syncobj_timeline_wait(p, chunk);
657 if (r)
658 return r;
659 break;
660 case AMDGPU_CHUNK_ID_SYNCOBJ_TIMELINE_SIGNAL:
661 r = amdgpu_cs_p2_syncobj_timeline_signal(p, chunk);
662 if (r)
663 return r;
664 break;
665 case AMDGPU_CHUNK_ID_CP_GFX_SHADOW:
666 r = amdgpu_cs_p2_shadow(p, chunk);
667 if (r)
668 return r;
669 break;
670 }
671 }
672
673 return 0;
674}
675
676/* Convert microseconds to bytes. */
677static u64 us_to_bytes(struct amdgpu_device *adev, s64 us)
678{
679 if (us <= 0 || !adev->mm_stats.log2_max_MBps)
680 return 0;
681
682 /* Since accum_us is incremented by a million per second, just
683 * multiply it by the number of MB/s to get the number of bytes.
684 */
685 return us << adev->mm_stats.log2_max_MBps;
686}
687
688static s64 bytes_to_us(struct amdgpu_device *adev, u64 bytes)
689{
690 if (!adev->mm_stats.log2_max_MBps)
691 return 0;
692
693 return bytes >> adev->mm_stats.log2_max_MBps;
694}
695
696/* Returns how many bytes TTM can move right now. If no bytes can be moved,
697 * it returns 0. If it returns non-zero, it's OK to move at least one buffer,
698 * which means it can go over the threshold once. If that happens, the driver
699 * will be in debt and no other buffer migrations can be done until that debt
700 * is repaid.
701 *
702 * This approach allows moving a buffer of any size (it's important to allow
703 * that).
704 *
705 * The currency is simply time in microseconds and it increases as the clock
706 * ticks. The accumulated microseconds (us) are converted to bytes and
707 * returned.
708 */
709static void amdgpu_cs_get_threshold_for_moves(struct amdgpu_device *adev,
710 u64 *max_bytes,
711 u64 *max_vis_bytes)
712{
713 s64 time_us, increment_us;
714 u64 free_vram, total_vram, used_vram;
715 /* Allow a maximum of 200 accumulated ms. This is basically per-IB
716 * throttling.
717 *
718 * It means that in order to get full max MBps, at least 5 IBs per
719 * second must be submitted and not more than 200ms apart from each
720 * other.
721 */
722 const s64 us_upper_bound = 200000;
723
724 if ((!adev->mm_stats.log2_max_MBps) || !ttm_resource_manager_used(&adev->mman.vram_mgr.manager)) {
725 *max_bytes = 0;
726 *max_vis_bytes = 0;
727 return;
728 }
729
730 total_vram = adev->gmc.real_vram_size - atomic64_read(&adev->vram_pin_size);
731 used_vram = ttm_resource_manager_usage(&adev->mman.vram_mgr.manager);
732 free_vram = used_vram >= total_vram ? 0 : total_vram - used_vram;
733
734 spin_lock(&adev->mm_stats.lock);
735
736 /* Increase the amount of accumulated us. */
737 time_us = ktime_to_us(ktime_get());
738 increment_us = time_us - adev->mm_stats.last_update_us;
739 adev->mm_stats.last_update_us = time_us;
740 adev->mm_stats.accum_us = min(adev->mm_stats.accum_us + increment_us,
741 us_upper_bound);
742
743 /* This prevents the short period of low performance when the VRAM
744 * usage is low and the driver is in debt or doesn't have enough
745 * accumulated us to fill VRAM quickly.
746 *
747 * The situation can occur in these cases:
748 * - a lot of VRAM is freed by userspace
749 * - the presence of a big buffer causes a lot of evictions
750 * (solution: split buffers into smaller ones)
751 *
752 * If 128 MB or 1/8th of VRAM is free, start filling it now by setting
753 * accum_us to a positive number.
754 */
755 if (free_vram >= 128 * 1024 * 1024 || free_vram >= total_vram / 8) {
756 s64 min_us;
757
758 /* Be more aggressive on dGPUs. Try to fill a portion of free
759 * VRAM now.
760 */
761 if (!(adev->flags & AMD_IS_APU))
762 min_us = bytes_to_us(adev, free_vram / 4);
763 else
764 min_us = 0; /* Reset accum_us on APUs. */
765
766 adev->mm_stats.accum_us = max(min_us, adev->mm_stats.accum_us);
767 }
768
769 /* This is set to 0 if the driver is in debt to disallow (optional)
770 * buffer moves.
771 */
772 *max_bytes = us_to_bytes(adev, adev->mm_stats.accum_us);
773
774 /* Do the same for visible VRAM if half of it is free */
775 if (!amdgpu_gmc_vram_full_visible(&adev->gmc)) {
776 u64 total_vis_vram = adev->gmc.visible_vram_size;
777 u64 used_vis_vram =
778 amdgpu_vram_mgr_vis_usage(&adev->mman.vram_mgr);
779
780 if (used_vis_vram < total_vis_vram) {
781 u64 free_vis_vram = total_vis_vram - used_vis_vram;
782
783 adev->mm_stats.accum_us_vis = min(adev->mm_stats.accum_us_vis +
784 increment_us, us_upper_bound);
785
786 if (free_vis_vram >= total_vis_vram / 2)
787 adev->mm_stats.accum_us_vis =
788 max(bytes_to_us(adev, free_vis_vram / 2),
789 adev->mm_stats.accum_us_vis);
790 }
791
792 *max_vis_bytes = us_to_bytes(adev, adev->mm_stats.accum_us_vis);
793 } else {
794 *max_vis_bytes = 0;
795 }
796
797 spin_unlock(&adev->mm_stats.lock);
798}
799
800/* Report how many bytes have really been moved for the last command
801 * submission. This can result in a debt that can stop buffer migrations
802 * temporarily.
803 */
804void amdgpu_cs_report_moved_bytes(struct amdgpu_device *adev, u64 num_bytes,
805 u64 num_vis_bytes)
806{
807 spin_lock(&adev->mm_stats.lock);
808 adev->mm_stats.accum_us -= bytes_to_us(adev, num_bytes);
809 adev->mm_stats.accum_us_vis -= bytes_to_us(adev, num_vis_bytes);
810 spin_unlock(&adev->mm_stats.lock);
811}
812
813static int amdgpu_cs_bo_validate(void *param, struct amdgpu_bo *bo)
814{
815 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
816 struct amdgpu_cs_parser *p = param;
817 struct ttm_operation_ctx ctx = {
818 .interruptible = true,
819 .no_wait_gpu = false,
820 .resv = bo->tbo.base.resv
821 };
822 uint32_t domain;
823 int r;
824
825 if (bo->tbo.pin_count)
826 return 0;
827
828 /* Don't move this buffer if we have depleted our allowance
829 * to move it. Don't move anything if the threshold is zero.
830 */
831 if (p->bytes_moved < p->bytes_moved_threshold &&
832 (!bo->tbo.base.dma_buf ||
833 list_empty(&bo->tbo.base.dma_buf->attachments))) {
834 if (!amdgpu_gmc_vram_full_visible(&adev->gmc) &&
835 (bo->flags & AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED)) {
836 /* And don't move a CPU_ACCESS_REQUIRED BO to limited
837 * visible VRAM if we've depleted our allowance to do
838 * that.
839 */
840 if (p->bytes_moved_vis < p->bytes_moved_vis_threshold)
841 domain = bo->preferred_domains;
842 else
843 domain = bo->allowed_domains;
844 } else {
845 domain = bo->preferred_domains;
846 }
847 } else {
848 domain = bo->allowed_domains;
849 }
850
851retry:
852 amdgpu_bo_placement_from_domain(bo, domain);
853 r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
854
855 p->bytes_moved += ctx.bytes_moved;
856 if (!amdgpu_gmc_vram_full_visible(&adev->gmc) &&
857 amdgpu_res_cpu_visible(adev, bo->tbo.resource))
858 p->bytes_moved_vis += ctx.bytes_moved;
859
860 if (unlikely(r == -ENOMEM) && domain != bo->allowed_domains) {
861 domain = bo->allowed_domains;
862 goto retry;
863 }
864
865 return r;
866}
867
868static int amdgpu_cs_parser_bos(struct amdgpu_cs_parser *p,
869 union drm_amdgpu_cs *cs)
870{
871 struct amdgpu_fpriv *fpriv = p->filp->driver_priv;
872 struct ttm_operation_ctx ctx = { true, false };
873 struct amdgpu_bo_list *list = NULL;
874 struct amdgpu_vm *vm = &fpriv->vm;
875 struct amdgpu_bo_list_entry *e;
876 struct drm_gem_object *obj;
877 unsigned int i;
878 int r;
879
880 /* p->bo_list could already be assigned if AMDGPU_CHUNK_ID_BO_HANDLES is present */
881 if (cs->in.bo_list_handle) {
882 if (p->bo_list)
883 return -EINVAL;
884
885 list = amdgpu_bo_list_get(fpriv, cs->in.bo_list_handle);
886 } else if (!p->bo_list) {
887 /* Create a empty bo_list when no handle is provided */
888 list = amdgpu_bo_list_create(p->adev, p->filp, NULL, 0);
889 }
890
891 if (IS_ERR(list))
892 return PTR_ERR(list);
893 else if (list)
894 p->bo_list = list;
895 else
896 list = p->bo_list;
897
898 /* Get userptr backing pages. If pages are updated after registered
899 * in amdgpu_gem_userptr_ioctl(), amdgpu_cs_list_validate() will do
900 * amdgpu_ttm_backend_bind() to flush and invalidate new pages
901 */
902 amdgpu_bo_list_for_each_userptr_entry(e, list) {
903 bool userpage_invalidated = false;
904 struct amdgpu_bo *bo = e->bo;
905
906 e->range = amdgpu_hmm_range_alloc(NULL);
907 if (unlikely(!e->range)) {
908 r = -ENOMEM;
909 goto out_free_user_pages;
910 }
911
912 r = amdgpu_ttm_tt_get_user_pages(bo, e->range);
913 if (r)
914 goto out_free_user_pages;
915
916 for (i = 0; i < bo->tbo.ttm->num_pages; i++) {
917 if (bo->tbo.ttm->pages[i] !=
918 hmm_pfn_to_page(e->range->hmm_range.hmm_pfns[i])) {
919 userpage_invalidated = true;
920 break;
921 }
922 }
923 e->user_invalidated = userpage_invalidated;
924 }
925
926 drm_exec_until_all_locked(&p->exec) {
927 r = amdgpu_vm_lock_pd(&fpriv->vm, &p->exec, 1 + p->gang_size);
928 drm_exec_retry_on_contention(&p->exec);
929 if (unlikely(r))
930 goto out_free_user_pages;
931
932 amdgpu_bo_list_for_each_entry(e, list) {
933 r = drm_exec_prepare_obj(&p->exec, &e->bo->tbo.base,
934 TTM_NUM_MOVE_FENCES + p->gang_size);
935 drm_exec_retry_on_contention(&p->exec);
936 if (unlikely(r))
937 goto out_free_user_pages;
938
939 e->bo_va = amdgpu_vm_bo_find(vm, e->bo);
940 }
941
942 if (p->uf_bo) {
943 r = drm_exec_prepare_obj(&p->exec, &p->uf_bo->tbo.base,
944 TTM_NUM_MOVE_FENCES + p->gang_size);
945 drm_exec_retry_on_contention(&p->exec);
946 if (unlikely(r))
947 goto out_free_user_pages;
948 }
949 }
950
951 amdgpu_bo_list_for_each_userptr_entry(e, list) {
952 struct mm_struct *usermm;
953
954 usermm = amdgpu_ttm_tt_get_usermm(e->bo->tbo.ttm);
955 if (usermm && usermm != current->mm) {
956 r = -EPERM;
957 goto out_free_user_pages;
958 }
959
960 if (amdgpu_ttm_tt_is_userptr(e->bo->tbo.ttm) &&
961 e->user_invalidated) {
962 amdgpu_bo_placement_from_domain(e->bo,
963 AMDGPU_GEM_DOMAIN_CPU);
964 r = ttm_bo_validate(&e->bo->tbo, &e->bo->placement,
965 &ctx);
966 if (r)
967 goto out_free_user_pages;
968
969 amdgpu_ttm_tt_set_user_pages(e->bo->tbo.ttm,
970 e->range);
971 }
972 }
973
974 amdgpu_cs_get_threshold_for_moves(p->adev, &p->bytes_moved_threshold,
975 &p->bytes_moved_vis_threshold);
976 p->bytes_moved = 0;
977 p->bytes_moved_vis = 0;
978
979 r = amdgpu_vm_validate(p->adev, &fpriv->vm, NULL,
980 amdgpu_cs_bo_validate, p);
981 if (r) {
982 drm_err(adev_to_drm(p->adev), "amdgpu_vm_validate() failed.\n");
983 goto out_free_user_pages;
984 }
985
986 drm_exec_for_each_locked_object(&p->exec, obj) {
987 r = amdgpu_cs_bo_validate(p, gem_to_amdgpu_bo(obj));
988 if (unlikely(r))
989 goto out_free_user_pages;
990 }
991
992 if (p->uf_bo) {
993 r = amdgpu_ttm_alloc_gart(&p->uf_bo->tbo);
994 if (unlikely(r))
995 goto out_free_user_pages;
996
997 p->gang_leader->uf_addr += amdgpu_bo_gpu_offset(p->uf_bo);
998 }
999
1000 amdgpu_cs_report_moved_bytes(p->adev, p->bytes_moved,
1001 p->bytes_moved_vis);
1002
1003 for (i = 0; i < p->gang_size; ++i)
1004 amdgpu_job_set_resources(p->jobs[i], list->gds_obj,
1005 list->gws_obj, list->oa_obj);
1006 return 0;
1007
1008out_free_user_pages:
1009 amdgpu_bo_list_for_each_userptr_entry(e, list) {
1010 amdgpu_hmm_range_free(e->range);
1011 e->range = NULL;
1012 }
1013 return r;
1014}
1015
1016static void trace_amdgpu_cs_ibs(struct amdgpu_cs_parser *p)
1017{
1018 int i, j;
1019
1020 if (!trace_amdgpu_cs_enabled())
1021 return;
1022
1023 for (i = 0; i < p->gang_size; ++i) {
1024 struct amdgpu_job *job = p->jobs[i];
1025
1026 for (j = 0; j < job->num_ibs; ++j)
1027 trace_amdgpu_cs(p, job, &job->ibs[j]);
1028 }
1029}
1030
1031static int amdgpu_cs_patch_ibs(struct amdgpu_cs_parser *p,
1032 struct amdgpu_job *job)
1033{
1034 struct amdgpu_ring *ring = amdgpu_job_ring(job);
1035 struct amdgpu_device *adev = ring->adev;
1036 unsigned int i;
1037 int r;
1038
1039 /* Only for UVD/VCE VM emulation */
1040 if (!ring->funcs->parse_cs && !ring->funcs->patch_cs_in_place)
1041 return 0;
1042
1043 for (i = 0; i < job->num_ibs; ++i) {
1044 struct amdgpu_ib *ib = &job->ibs[i];
1045 struct amdgpu_bo_va_mapping *m;
1046 struct amdgpu_bo *aobj;
1047 uint64_t va_start;
1048 uint8_t *kptr;
1049
1050 va_start = ib->gpu_addr & AMDGPU_GMC_HOLE_MASK;
1051 r = amdgpu_cs_find_mapping(p, va_start, &aobj, &m);
1052 if (r) {
1053 drm_err(adev_to_drm(p->adev), "IB va_start is invalid\n");
1054 return r;
1055 }
1056
1057 if ((va_start + ib->length_dw * 4) >
1058 (m->last + 1) * AMDGPU_GPU_PAGE_SIZE) {
1059 drm_err(adev_to_drm(p->adev), "IB va_start+ib_bytes is invalid\n");
1060 return -EINVAL;
1061 }
1062
1063 /* the IB should be reserved at this point */
1064 r = amdgpu_bo_kmap(aobj, (void **)&kptr);
1065 if (r)
1066 return r;
1067
1068 kptr += va_start - (m->start * AMDGPU_GPU_PAGE_SIZE);
1069
1070 if (ring->funcs->parse_cs) {
1071 memcpy(ib->ptr, kptr, ib->length_dw * 4);
1072 amdgpu_bo_kunmap(aobj);
1073
1074 r = amdgpu_ring_parse_cs(ring, p, job, ib);
1075 if (r)
1076 return r;
1077
1078 if (ib->sa_bo)
1079 ib->gpu_addr = amdgpu_sa_bo_gpu_addr(ib->sa_bo);
1080 } else {
1081 ib->ptr = (uint32_t *)kptr;
1082 r = amdgpu_ring_patch_cs_in_place(ring, p, job, ib);
1083 amdgpu_bo_kunmap(aobj);
1084 if (r)
1085 return r;
1086 }
1087 }
1088
1089 return 0;
1090}
1091
1092static int amdgpu_cs_patch_jobs(struct amdgpu_cs_parser *p)
1093{
1094 unsigned int i;
1095 int r;
1096
1097 for (i = 0; i < p->gang_size; ++i) {
1098 r = amdgpu_cs_patch_ibs(p, p->jobs[i]);
1099 if (r)
1100 return r;
1101 }
1102 return 0;
1103}
1104
1105static int amdgpu_cs_vm_handling(struct amdgpu_cs_parser *p)
1106{
1107 struct amdgpu_fpriv *fpriv = p->filp->driver_priv;
1108 struct amdgpu_job *job = p->gang_leader;
1109 struct amdgpu_device *adev = p->adev;
1110 struct amdgpu_vm *vm = &fpriv->vm;
1111 struct amdgpu_bo_list_entry *e;
1112 struct amdgpu_bo_va *bo_va;
1113 unsigned int i;
1114 int r;
1115
1116 /*
1117 * We can't use gang submit on with reserved VMIDs when the VM changes
1118 * can't be invalidated by more than one engine at the same time.
1119 */
1120 if (p->gang_size > 1 && !adev->vm_manager.concurrent_flush) {
1121 for (i = 0; i < p->gang_size; ++i) {
1122 struct drm_sched_entity *entity = p->entities[i];
1123 struct drm_gpu_scheduler *sched = entity->rq->sched;
1124 struct amdgpu_ring *ring = to_amdgpu_ring(sched);
1125
1126 if (amdgpu_vmid_uses_reserved(vm, ring->vm_hub))
1127 return -EINVAL;
1128 }
1129 }
1130
1131 if (!amdgpu_vm_ready(vm))
1132 return -EINVAL;
1133
1134 r = amdgpu_vm_clear_freed(adev, vm, NULL);
1135 if (r)
1136 return r;
1137
1138 r = amdgpu_vm_bo_update(adev, fpriv->prt_va, false);
1139 if (r)
1140 return r;
1141
1142 r = amdgpu_sync_fence(&p->sync, fpriv->prt_va->last_pt_update,
1143 GFP_KERNEL);
1144 if (r)
1145 return r;
1146
1147 if (fpriv->csa_va) {
1148 bo_va = fpriv->csa_va;
1149 if (!bo_va)
1150 return -ENOMEM;
1151 r = amdgpu_vm_bo_update(adev, bo_va, false);
1152 if (r)
1153 return r;
1154
1155 r = amdgpu_sync_fence(&p->sync, bo_va->last_pt_update,
1156 GFP_KERNEL);
1157 if (r)
1158 return r;
1159 }
1160
1161 /* FIXME: In theory this loop shouldn't be needed any more when
1162 * amdgpu_vm_handle_moved handles all moved BOs that are reserved
1163 * with p->ticket. But removing it caused test regressions, so I'm
1164 * leaving it here for now.
1165 */
1166 amdgpu_bo_list_for_each_entry(e, p->bo_list) {
1167 bo_va = e->bo_va;
1168 if (bo_va == NULL)
1169 continue;
1170
1171 r = amdgpu_vm_bo_update(adev, bo_va, false);
1172 if (r)
1173 return r;
1174
1175 r = amdgpu_sync_fence(&p->sync, bo_va->last_pt_update,
1176 GFP_KERNEL);
1177 if (r)
1178 return r;
1179 }
1180
1181 r = amdgpu_vm_handle_moved(adev, vm, drm_exec_ticket(&p->exec));
1182 if (r)
1183 return r;
1184
1185 r = amdgpu_vm_update_pdes(adev, vm, false);
1186 if (r)
1187 return r;
1188
1189 r = amdgpu_sync_fence(&p->sync, vm->last_update, GFP_KERNEL);
1190 if (r)
1191 return r;
1192
1193 for (i = 0; i < p->gang_size; ++i) {
1194 job = p->jobs[i];
1195
1196 if (!job->vm)
1197 continue;
1198
1199 job->vm_pd_addr = amdgpu_gmc_pd_addr(vm->root.bo);
1200 }
1201
1202 return 0;
1203}
1204
1205static int amdgpu_cs_sync_rings(struct amdgpu_cs_parser *p)
1206{
1207 struct amdgpu_fpriv *fpriv = p->filp->driver_priv;
1208 struct drm_gpu_scheduler *sched;
1209 struct drm_gem_object *obj;
1210 struct dma_fence *fence;
1211 unsigned int i;
1212 int r;
1213
1214 r = amdgpu_ctx_wait_prev_fence(p->ctx, p->entities[p->gang_leader_idx]);
1215 if (r) {
1216 if (r != -ERESTARTSYS)
1217 drm_err(adev_to_drm(p->adev), "amdgpu_ctx_wait_prev_fence failed.\n");
1218 return r;
1219 }
1220
1221 drm_exec_for_each_locked_object(&p->exec, obj) {
1222 struct amdgpu_bo *bo = gem_to_amdgpu_bo(obj);
1223
1224 struct dma_resv *resv = bo->tbo.base.resv;
1225 enum amdgpu_sync_mode sync_mode;
1226
1227 sync_mode = amdgpu_bo_explicit_sync(bo) ?
1228 AMDGPU_SYNC_EXPLICIT : AMDGPU_SYNC_NE_OWNER;
1229 r = amdgpu_sync_resv(p->adev, &p->sync, resv, sync_mode,
1230 &fpriv->vm);
1231 if (r)
1232 return r;
1233 }
1234
1235 for (i = 0; i < p->gang_size; ++i) {
1236 r = amdgpu_sync_push_to_job(&p->sync, p->jobs[i]);
1237 if (r)
1238 return r;
1239 }
1240
1241 sched = p->gang_leader->base.entity->rq->sched;
1242 while ((fence = amdgpu_sync_get_fence(&p->sync))) {
1243 struct drm_sched_fence *s_fence = to_drm_sched_fence(fence);
1244
1245 /*
1246 * When we have an dependency it might be necessary to insert a
1247 * pipeline sync to make sure that all caches etc are flushed and the
1248 * next job actually sees the results from the previous one
1249 * before we start executing on the same scheduler ring.
1250 */
1251 if (!s_fence || s_fence->sched != sched) {
1252 dma_fence_put(fence);
1253 continue;
1254 }
1255
1256 r = amdgpu_sync_fence(&p->gang_leader->explicit_sync, fence,
1257 GFP_KERNEL);
1258 dma_fence_put(fence);
1259 if (r)
1260 return r;
1261 }
1262 return 0;
1263}
1264
1265static void amdgpu_cs_post_dependencies(struct amdgpu_cs_parser *p)
1266{
1267 int i;
1268
1269 for (i = 0; i < p->num_post_deps; ++i) {
1270 if (p->post_deps[i].chain && p->post_deps[i].point) {
1271 drm_syncobj_add_point(p->post_deps[i].syncobj,
1272 p->post_deps[i].chain,
1273 p->fence, p->post_deps[i].point);
1274 p->post_deps[i].chain = NULL;
1275 } else {
1276 drm_syncobj_replace_fence(p->post_deps[i].syncobj,
1277 p->fence);
1278 }
1279 }
1280}
1281
1282static int amdgpu_cs_submit(struct amdgpu_cs_parser *p,
1283 union drm_amdgpu_cs *cs)
1284{
1285 struct amdgpu_fpriv *fpriv = p->filp->driver_priv;
1286 struct amdgpu_job *leader = p->gang_leader;
1287 struct amdgpu_vm *vm = &fpriv->vm;
1288 struct amdgpu_bo_list_entry *e;
1289 struct drm_gem_object *gobj;
1290 unsigned int i;
1291 uint64_t seq;
1292 int r;
1293
1294 for (i = 0; i < p->gang_size; ++i)
1295 drm_sched_job_arm(&p->jobs[i]->base);
1296
1297 for (i = 0; i < p->gang_size; ++i) {
1298 struct dma_fence *fence;
1299
1300 if (p->jobs[i] == leader)
1301 continue;
1302
1303 fence = &p->jobs[i]->base.s_fence->scheduled;
1304 dma_fence_get(fence);
1305 r = drm_sched_job_add_dependency(&leader->base, fence);
1306 if (r) {
1307 dma_fence_put(fence);
1308 return r;
1309 }
1310 }
1311
1312 if (p->gang_size > 1) {
1313 for (i = 0; i < p->gang_size; ++i)
1314 amdgpu_job_set_gang_leader(p->jobs[i], leader);
1315 }
1316
1317 /* No memory allocation is allowed while holding the notifier lock.
1318 * The lock is held until amdgpu_cs_submit is finished and fence is
1319 * added to BOs.
1320 */
1321 mutex_lock(&p->adev->notifier_lock);
1322
1323 /* If userptr are invalidated after amdgpu_cs_parser_bos(), return
1324 * -EAGAIN, drmIoctl in libdrm will restart the amdgpu_cs_ioctl.
1325 */
1326 r = 0;
1327 amdgpu_bo_list_for_each_userptr_entry(e, p->bo_list) {
1328 r |= !amdgpu_hmm_range_valid(e->range);
1329 amdgpu_hmm_range_free(e->range);
1330 e->range = NULL;
1331 }
1332
1333 if (r || !list_empty(&vm->individual.needs_update)) {
1334 r = -EAGAIN;
1335 mutex_unlock(&p->adev->notifier_lock);
1336 return r;
1337 }
1338
1339 p->fence = dma_fence_get(&leader->base.s_fence->finished);
1340 drm_exec_for_each_locked_object(&p->exec, gobj) {
1341
1342 ttm_bo_move_to_lru_tail_unlocked(&gem_to_amdgpu_bo(gobj)->tbo);
1343
1344 /* Everybody except for the gang leader uses READ */
1345 for (i = 0; i < p->gang_size; ++i) {
1346 if (p->jobs[i] == leader)
1347 continue;
1348
1349 dma_resv_add_fence(gobj->resv,
1350 &p->jobs[i]->base.s_fence->finished,
1351 DMA_RESV_USAGE_READ);
1352 }
1353
1354 /* The gang leader as remembered as writer */
1355 dma_resv_add_fence(gobj->resv, p->fence, DMA_RESV_USAGE_WRITE);
1356 }
1357
1358 seq = amdgpu_ctx_add_fence(p->ctx, p->entities[p->gang_leader_idx],
1359 p->fence);
1360 amdgpu_cs_post_dependencies(p);
1361
1362 if ((leader->preamble_status & AMDGPU_PREAMBLE_IB_PRESENT) &&
1363 !p->ctx->preamble_presented) {
1364 leader->preamble_status |= AMDGPU_PREAMBLE_IB_PRESENT_FIRST;
1365 p->ctx->preamble_presented = true;
1366 }
1367
1368 cs->out.handle = seq;
1369 leader->uf_sequence = seq;
1370
1371 amdgpu_vm_bo_trace_cs(&fpriv->vm, drm_exec_ticket(&p->exec));
1372 for (i = 0; i < p->gang_size; ++i) {
1373 amdgpu_job_free_resources(p->jobs[i]);
1374 trace_amdgpu_cs_ioctl(p->jobs[i]);
1375 drm_sched_entity_push_job(&p->jobs[i]->base);
1376 p->jobs[i] = NULL;
1377 }
1378
1379 amdgpu_vm_move_to_lru_tail(p->adev, &fpriv->vm);
1380
1381 mutex_unlock(&p->adev->notifier_lock);
1382 return 0;
1383}
1384
1385/* Cleanup the parser structure */
1386static void amdgpu_cs_parser_fini(struct amdgpu_cs_parser *parser)
1387{
1388 struct amdgpu_device *adev = parser->adev;
1389 struct amdgpu_bo_list_entry *e;
1390 unsigned int i;
1391
1392 amdgpu_sync_free(&parser->sync);
1393 drm_exec_fini(&parser->exec);
1394
1395 for (i = 0; i < parser->num_post_deps; i++) {
1396 drm_syncobj_put(parser->post_deps[i].syncobj);
1397 kfree(parser->post_deps[i].chain);
1398 }
1399 kfree(parser->post_deps);
1400
1401 dma_fence_put(parser->fence);
1402
1403 if (parser->ctx)
1404 amdgpu_ctx_put(parser->ctx);
1405 if (parser->bo_list) {
1406 if (adev->debug_vm) {
1407 /* Invalidate all BOs to test for userspace bugs */
1408 amdgpu_bo_list_for_each_entry(e, parser->bo_list) {
1409 struct amdgpu_bo *bo = e->bo;
1410
1411 /* ignore duplicates */
1412 if (!bo)
1413 continue;
1414
1415 amdgpu_vm_bo_invalidate(bo, false);
1416 }
1417 }
1418 amdgpu_bo_list_put(parser->bo_list);
1419 }
1420
1421 for (i = 0; i < parser->nchunks; i++)
1422 kvfree(parser->chunks[i].kdata);
1423 kvfree(parser->chunks);
1424 for (i = 0; i < parser->gang_size; ++i) {
1425 if (parser->jobs[i])
1426 amdgpu_job_free(parser->jobs[i]);
1427 }
1428 amdgpu_bo_unref(&parser->uf_bo);
1429}
1430
1431int amdgpu_cs_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
1432{
1433 struct amdgpu_device *adev = drm_to_adev(dev);
1434 struct amdgpu_bo_list_entry *e;
1435 struct amdgpu_cs_parser parser;
1436 int r;
1437
1438 if (amdgpu_ras_intr_triggered())
1439 return -EHWPOISON;
1440
1441 if (!adev->accel_working)
1442 return -EBUSY;
1443
1444 r = amdgpu_cs_parser_init(&parser, adev, filp, data);
1445 if (r) {
1446 drm_err_ratelimited(dev, "Failed to initialize parser %d!\n", r);
1447 return r;
1448 }
1449
1450 r = amdgpu_cs_pass1(&parser, data);
1451 if (r)
1452 goto error_fini;
1453
1454 r = amdgpu_cs_pass2(&parser);
1455 if (r)
1456 goto error_fini;
1457
1458 r = amdgpu_cs_parser_bos(&parser, data);
1459 if (r) {
1460 if (r == -ENOMEM)
1461 drm_err(dev, "Not enough memory for command submission!\n");
1462 else if (r != -ERESTARTSYS && r != -EAGAIN)
1463 drm_dbg(dev, "Failed to process the buffer list %d!\n", r);
1464 goto error_fini;
1465 }
1466
1467 r = amdgpu_cs_patch_jobs(&parser);
1468 if (r)
1469 goto error_free_user_pages;
1470
1471 r = amdgpu_cs_vm_handling(&parser);
1472 if (r)
1473 goto error_free_user_pages;
1474
1475 r = amdgpu_cs_sync_rings(&parser);
1476 if (r)
1477 goto error_free_user_pages;
1478
1479 trace_amdgpu_cs_ibs(&parser);
1480
1481 r = amdgpu_cs_submit(&parser, data);
1482 if (r)
1483 goto error_free_user_pages;
1484
1485 amdgpu_cs_parser_fini(&parser);
1486 return 0;
1487
1488error_free_user_pages:
1489 /* amdgpu_cs_submit() only drops the ranges once it got that far. */
1490 amdgpu_bo_list_for_each_userptr_entry(e, parser.bo_list) {
1491 amdgpu_hmm_range_free(e->range);
1492 e->range = NULL;
1493 }
1494error_fini:
1495 amdgpu_cs_parser_fini(&parser);
1496 return r;
1497}
1498
1499/**
1500 * amdgpu_cs_wait_ioctl - wait for a command submission to finish
1501 *
1502 * @dev: drm device
1503 * @data: data from userspace
1504 * @filp: file private
1505 *
1506 * Wait for the command submission identified by handle to finish.
1507 */
1508int amdgpu_cs_wait_ioctl(struct drm_device *dev, void *data,
1509 struct drm_file *filp)
1510{
1511 union drm_amdgpu_wait_cs *wait = data;
1512 unsigned long timeout = amdgpu_gem_timeout(wait->in.timeout);
1513 struct drm_sched_entity *entity;
1514 struct amdgpu_ctx *ctx;
1515 struct dma_fence *fence;
1516 long r;
1517
1518 ctx = amdgpu_ctx_get(filp->driver_priv, wait->in.ctx_id);
1519 if (ctx == NULL)
1520 return -EINVAL;
1521
1522 r = amdgpu_ctx_get_entity(ctx, wait->in.ip_type, wait->in.ip_instance,
1523 wait->in.ring, &entity);
1524 if (r) {
1525 amdgpu_ctx_put(ctx);
1526 return r;
1527 }
1528
1529 fence = amdgpu_ctx_get_fence(ctx, entity, wait->in.handle);
1530 if (IS_ERR(fence))
1531 r = PTR_ERR(fence);
1532 else if (fence) {
1533 r = dma_fence_wait_timeout(fence, true, timeout);
1534 if (r > 0 && fence->error)
1535 r = fence->error;
1536 dma_fence_put(fence);
1537 } else
1538 r = 1;
1539
1540 amdgpu_ctx_put(ctx);
1541 if (r < 0)
1542 return r;
1543
1544 memset(wait, 0, sizeof(*wait));
1545 wait->out.status = (r == 0);
1546
1547 return 0;
1548}
1549
1550/**
1551 * amdgpu_cs_get_fence - helper to get fence from drm_amdgpu_fence
1552 *
1553 * @adev: amdgpu device
1554 * @filp: file private
1555 * @user: drm_amdgpu_fence copied from user space
1556 */
1557static struct dma_fence *amdgpu_cs_get_fence(struct amdgpu_device *adev,
1558 struct drm_file *filp,
1559 struct drm_amdgpu_fence *user)
1560{
1561 struct drm_sched_entity *entity;
1562 struct amdgpu_ctx *ctx;
1563 struct dma_fence *fence;
1564 int r;
1565
1566 ctx = amdgpu_ctx_get(filp->driver_priv, user->ctx_id);
1567 if (ctx == NULL)
1568 return ERR_PTR(-EINVAL);
1569
1570 r = amdgpu_ctx_get_entity(ctx, user->ip_type, user->ip_instance,
1571 user->ring, &entity);
1572 if (r) {
1573 amdgpu_ctx_put(ctx);
1574 return ERR_PTR(r);
1575 }
1576
1577 fence = amdgpu_ctx_get_fence(ctx, entity, user->seq_no);
1578 amdgpu_ctx_put(ctx);
1579
1580 return fence;
1581}
1582
1583int amdgpu_cs_fence_to_handle_ioctl(struct drm_device *dev, void *data,
1584 struct drm_file *filp)
1585{
1586 struct amdgpu_device *adev = drm_to_adev(dev);
1587 union drm_amdgpu_fence_to_handle *info = data;
1588 struct dma_fence *fence;
1589 struct drm_syncobj *syncobj;
1590 struct sync_file *sync_file;
1591 int fd, r;
1592
1593 fence = amdgpu_cs_get_fence(adev, filp, &info->in.fence);
1594 if (IS_ERR(fence))
1595 return PTR_ERR(fence);
1596
1597 if (!fence)
1598 fence = dma_fence_get_stub();
1599
1600 switch (info->in.what) {
1601 case AMDGPU_FENCE_TO_HANDLE_GET_SYNCOBJ:
1602 r = drm_syncobj_create(&syncobj, 0, fence);
1603 dma_fence_put(fence);
1604 if (r)
1605 return r;
1606 r = drm_syncobj_get_handle(filp, syncobj, &info->out.handle);
1607 drm_syncobj_put(syncobj);
1608 return r;
1609
1610 case AMDGPU_FENCE_TO_HANDLE_GET_SYNCOBJ_FD:
1611 r = drm_syncobj_create(&syncobj, 0, fence);
1612 dma_fence_put(fence);
1613 if (r)
1614 return r;
1615 r = drm_syncobj_get_fd(syncobj, (int *)&info->out.handle);
1616 drm_syncobj_put(syncobj);
1617 return r;
1618
1619 case AMDGPU_FENCE_TO_HANDLE_GET_SYNC_FILE_FD:
1620 fd = get_unused_fd_flags(O_CLOEXEC);
1621 if (fd < 0) {
1622 dma_fence_put(fence);
1623 return fd;
1624 }
1625
1626 sync_file = sync_file_create(fence);
1627 dma_fence_put(fence);
1628 if (!sync_file) {
1629 put_unused_fd(fd);
1630 return -ENOMEM;
1631 }
1632
1633 fd_install(fd, sync_file->file);
1634 info->out.handle = fd;
1635 return 0;
1636
1637 default:
1638 dma_fence_put(fence);
1639 return -EINVAL;
1640 }
1641}
1642
1643/**
1644 * amdgpu_cs_wait_all_fences - wait on all fences to signal
1645 *
1646 * @adev: amdgpu device
1647 * @filp: file private
1648 * @wait: wait parameters
1649 * @fences: array of drm_amdgpu_fence
1650 */
1651static int amdgpu_cs_wait_all_fences(struct amdgpu_device *adev,
1652 struct drm_file *filp,
1653 union drm_amdgpu_wait_fences *wait,
1654 struct drm_amdgpu_fence *fences)
1655{
1656 uint32_t fence_count = wait->in.fence_count;
1657 unsigned int i;
1658 long r = 1;
1659
1660 for (i = 0; i < fence_count; i++) {
1661 struct dma_fence *fence;
1662 unsigned long timeout = amdgpu_gem_timeout(wait->in.timeout_ns);
1663
1664 fence = amdgpu_cs_get_fence(adev, filp, &fences[i]);
1665 if (IS_ERR(fence))
1666 return PTR_ERR(fence);
1667 else if (!fence)
1668 continue;
1669
1670 r = dma_fence_wait_timeout(fence, true, timeout);
1671 if (r > 0 && fence->error)
1672 r = fence->error;
1673
1674 dma_fence_put(fence);
1675 if (r < 0)
1676 return r;
1677
1678 if (r == 0)
1679 break;
1680 }
1681
1682 memset(wait, 0, sizeof(*wait));
1683 wait->out.status = (r > 0);
1684
1685 return 0;
1686}
1687
1688/**
1689 * amdgpu_cs_wait_any_fence - wait on any fence to signal
1690 *
1691 * @adev: amdgpu device
1692 * @filp: file private
1693 * @wait: wait parameters
1694 * @fences: array of drm_amdgpu_fence
1695 */
1696static int amdgpu_cs_wait_any_fence(struct amdgpu_device *adev,
1697 struct drm_file *filp,
1698 union drm_amdgpu_wait_fences *wait,
1699 struct drm_amdgpu_fence *fences)
1700{
1701 unsigned long timeout = amdgpu_gem_timeout(wait->in.timeout_ns);
1702 uint32_t fence_count = wait->in.fence_count;
1703 uint32_t first = ~0;
1704 struct dma_fence **array;
1705 unsigned int i;
1706 long r;
1707
1708 /* Prepare the fence array */
1709 array = kzalloc_objs(struct dma_fence *, fence_count);
1710
1711 if (array == NULL)
1712 return -ENOMEM;
1713
1714 for (i = 0; i < fence_count; i++) {
1715 struct dma_fence *fence;
1716
1717 fence = amdgpu_cs_get_fence(adev, filp, &fences[i]);
1718 if (IS_ERR(fence)) {
1719 r = PTR_ERR(fence);
1720 goto err_free_fence_array;
1721 } else if (fence) {
1722 array[i] = fence;
1723 } else { /* NULL, the fence has been already signaled */
1724 r = 1;
1725 first = i;
1726 goto out;
1727 }
1728 }
1729
1730 r = dma_fence_wait_any_timeout(array, fence_count, true, timeout,
1731 &first);
1732 if (r < 0)
1733 goto err_free_fence_array;
1734
1735out:
1736 memset(wait, 0, sizeof(*wait));
1737 wait->out.status = (r > 0);
1738 wait->out.first_signaled = first;
1739
1740 if (first < fence_count && array[first])
1741 r = array[first]->error;
1742 else
1743 r = 0;
1744
1745err_free_fence_array:
1746 for (i = 0; i < fence_count; i++)
1747 dma_fence_put(array[i]);
1748 kfree(array);
1749
1750 return r;
1751}
1752
1753/**
1754 * amdgpu_cs_wait_fences_ioctl - wait for multiple command submissions to finish
1755 *
1756 * @dev: drm device
1757 * @data: data from userspace
1758 * @filp: file private
1759 */
1760int amdgpu_cs_wait_fences_ioctl(struct drm_device *dev, void *data,
1761 struct drm_file *filp)
1762{
1763 struct amdgpu_device *adev = drm_to_adev(dev);
1764 union drm_amdgpu_wait_fences *wait = data;
1765 struct drm_amdgpu_fence *fences;
1766 int r;
1767
1768 /*
1769 * fence_count must be non-zero; dma_fence_wait_any_timeout()
1770 * does not accept an empty fence array.
1771 */
1772 if (!wait->in.fence_count)
1773 return -EINVAL;
1774
1775 /* Get the fences from userspace */
1776 fences = memdup_array_user(u64_to_user_ptr(wait->in.fences),
1777 wait->in.fence_count,
1778 sizeof(struct drm_amdgpu_fence));
1779 if (IS_ERR(fences))
1780 return PTR_ERR(fences);
1781
1782 if (wait->in.wait_all)
1783 r = amdgpu_cs_wait_all_fences(adev, filp, wait, fences);
1784 else
1785 r = amdgpu_cs_wait_any_fence(adev, filp, wait, fences);
1786
1787 kfree(fences);
1788
1789 return r;
1790}
1791
1792/**
1793 * amdgpu_cs_find_mapping - find bo_va for VM address
1794 *
1795 * @parser: command submission parser context
1796 * @addr: VM address
1797 * @bo: resulting BO of the mapping found
1798 * @map: Placeholder to return found BO mapping
1799 *
1800 * Search the buffer objects in the command submission context for a certain
1801 * virtual memory address. Returns allocation structure when found, NULL
1802 * otherwise.
1803 */
1804int amdgpu_cs_find_mapping(struct amdgpu_cs_parser *parser,
1805 uint64_t addr, struct amdgpu_bo **bo,
1806 struct amdgpu_bo_va_mapping **map)
1807{
1808 struct amdgpu_fpriv *fpriv = parser->filp->driver_priv;
1809 struct ttm_operation_ctx ctx = { false, false };
1810 struct amdgpu_vm *vm = &fpriv->vm;
1811 struct amdgpu_bo_va_mapping *mapping;
1812 int i, r;
1813
1814 addr /= AMDGPU_GPU_PAGE_SIZE;
1815
1816 mapping = amdgpu_vm_bo_lookup_mapping(vm, addr);
1817 if (!mapping || !mapping->bo_va || !mapping->bo_va->base.bo)
1818 return -EINVAL;
1819
1820 *bo = mapping->bo_va->base.bo;
1821 *map = mapping;
1822
1823 /* Double check that the BO is reserved by this CS */
1824 if (dma_resv_locking_ctx((*bo)->tbo.base.resv) != drm_exec_ticket(&parser->exec))
1825 return -EINVAL;
1826
1827 /* Make sure VRAM is allocated contigiously */
1828 (*bo)->flags |= AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS;
1829 if ((*bo)->tbo.resource->mem_type == TTM_PL_VRAM &&
1830 !((*bo)->tbo.resource->placement & TTM_PL_FLAG_CONTIGUOUS)) {
1831
1832 amdgpu_bo_placement_from_domain(*bo, (*bo)->allowed_domains);
1833 for (i = 0; i < (*bo)->placement.num_placement; i++)
1834 (*bo)->placements[i].flags |= TTM_PL_FLAG_CONTIGUOUS;
1835 r = ttm_bo_validate(&(*bo)->tbo, &(*bo)->placement, &ctx);
1836 if (r)
1837 return r;
1838 }
1839
1840 return amdgpu_ttm_alloc_gart(&(*bo)->tbo);
1841}