projects/linux·blob master · refs

1645 lines · 40.5 KiB

1// SPDX-License-Identifier: GPL-2.0+
2/*
3 * Copyright (C) 2014-2018 Broadcom
4 * Copyright (C) 2023 Raspberry Pi
5 */
6
7#include <linux/dma-fence-unwrap.h>
8#include <linux/overflow.h>
9
10#include <drm/drm_print.h>
11#include <drm/drm_syncobj.h>
12
13#include "v3d_drv.h"
14#include "v3d_regs.h"
15#include "v3d_trace.h"
16
17/* Takes the reservation lock on all the BOs being referenced, so that
18 * we can attach fences and update the reservations after pushing the job
19 * to the queue.
20 *
21 * We don't lock the RCL the tile alloc/state BOs, or overflow memory
22 * (all of which are on render->unref_list). They're entirely private
23 * to v3d, so we don't attach dma-buf fences to them.
24 */
25static int
26v3d_submit_lock_reservations(struct v3d_submit *submit)
27{
28 int i, j, ret;
29
30 drm_exec_init(&submit->exec,
31 DRM_EXEC_INTERRUPTIBLE_WAIT | DRM_EXEC_IGNORE_DUPLICATES, 0);
32 drm_exec_until_all_locked(&submit->exec) {
33 for (i = 0; i < submit->job_count; i++) {
34 struct v3d_job *job = submit->jobs[i];
35
36 ret = drm_exec_prepare_array(&submit->exec, job->bo,
37 job->bo_count, 1);
38 if (ret)
39 break;
40 }
41 drm_exec_retry_on_contention(&submit->exec);
42 if (ret)
43 goto fail;
44 }
45
46 for (i = 0; i < submit->job_count; i++) {
47 struct v3d_job *job = submit->jobs[i];
48
49 if (!job->has_implicit_dep)
50 continue;
51
52 for (j = 0; j < job->bo_count; j++) {
53 ret = drm_sched_job_add_implicit_dependencies(&job->base,
54 job->bo[j],
55 true);
56 if (ret)
57 goto fail;
58 }
59 }
60
61 return 0;
62
63fail:
64 drm_exec_fini(&submit->exec);
65 return ret;
66}
67
68static void
69v3d_submit_unlock_reservations(struct v3d_submit *submit)
70{
71 drm_exec_fini(&submit->exec);
72}
73
74/**
75 * v3d_lookup_bos() - Sets up job->bo[] with the GEM objects
76 * referenced by the job.
77 * @file_priv: DRM file for this fd
78 * @job: V3D job being set up
79 * @bo_handles: GEM handles
80 * @bo_count: Number of GEM handles passed in
81 *
82 * The command validator needs to reference BOs by their index within
83 * the submitted job's BO list. This does the validation of the job's
84 * BO list and reference counting for the lifetime of the job.
85 *
86 * Note that this function doesn't need to unreference the BOs on
87 * failure, because that will happen at `v3d_job_free()`.
88 */
89static int
90v3d_lookup_bos(struct drm_file *file_priv, struct v3d_job *job,
91 u64 bo_handles, u32 bo_count)
92{
93 if (!bo_count) {
94 drm_warn(&job->v3d->drm, "Rendering requires BOs\n");
95 return -EINVAL;
96 }
97
98 job->bo_count = bo_count;
99
100 return drm_gem_objects_lookup(file_priv,
101 (void __user *)(uintptr_t)bo_handles,
102 job->bo_count, &job->bo);
103}
104
105/**
106 * v3d_job_reference_bos() - Share another job's BOs with @dst
107 * @dst: job that acquires references to the BOs
108 * @src: job whose already-resolved BO list is shared
109 *
110 * For submissions with multiple jobs that use the same BOs, a trailing job
111 * shouldn't look the handles up again, as it could cause inconsistencies.
112 * Instead, it should reference the previous job's BOs.
113 */
114static int
115v3d_job_reference_bos(struct v3d_job *dst, struct v3d_job *src)
116{
117 dst->bo = kvmalloc_objs(*dst->bo, src->bo_count);
118 if (!dst->bo)
119 return -ENOMEM;
120
121 dst->bo_count = src->bo_count;
122 for (int i = 0; i < dst->bo_count; i++) {
123 dst->bo[i] = src->bo[i];
124 drm_gem_object_get(dst->bo[i]);
125 }
126
127 return 0;
128}
129
130static void
131v3d_job_free(struct kref *ref)
132{
133 struct v3d_job *job = container_of(ref, struct v3d_job, refcount);
134 int i;
135
136 if (job->bo) {
137 for (i = 0; i < job->bo_count; i++)
138 drm_gem_object_put(job->bo[i]);
139 kvfree(job->bo);
140 }
141
142 dma_fence_put(job->irq_fence);
143 dma_fence_put(job->done_fence);
144
145 if (job->perfmon)
146 v3d_perfmon_put(job->perfmon);
147
148 v3d_stats_put(job->client_stats);
149 v3d_stats_put(job->global_stats);
150
151 if (job->has_pm_ref)
152 v3d_pm_runtime_put(job->v3d);
153
154 kfree(job);
155}
156
157static void
158v3d_render_job_free(struct kref *ref)
159{
160 struct v3d_render_job *job = container_of(ref, struct v3d_render_job,
161 base.refcount);
162 struct v3d_bo *bo, *save;
163
164 list_for_each_entry_safe(bo, save, &job->unref_list, unref_head) {
165 drm_gem_object_put(&bo->base.base);
166 }
167
168 v3d_job_free(ref);
169}
170
171static void
172v3d_cpu_job_free(struct kref *ref)
173{
174 struct v3d_cpu_job *job = container_of(ref, struct v3d_cpu_job,
175 base.refcount);
176
177 v3d_timestamp_query_info_free(&job->timestamp_query,
178 job->timestamp_query.count);
179
180 v3d_performance_query_info_free(&job->performance_query,
181 job->performance_query.count);
182
183 if (job->indirect_csd.indirect)
184 drm_gem_object_put(job->indirect_csd.indirect);
185
186 v3d_job_free(ref);
187}
188
189void v3d_job_cleanup(struct v3d_job *job)
190{
191 if (!job)
192 return;
193
194 drm_sched_job_cleanup(&job->base);
195 v3d_job_put(job);
196}
197
198void v3d_job_put(struct v3d_job *job)
199{
200 if (!job)
201 return;
202
203 kref_put(&job->refcount, job->free);
204}
205
206static int
207v3d_job_add_syncobjs(struct v3d_job *job, struct drm_file *file_priv,
208 u32 in_sync, struct v3d_submit_ext *se)
209{
210 bool has_multisync = se && (se->flags & DRM_V3D_EXT_ID_MULTI_SYNC);
211 struct v3d_dev *v3d = job->v3d;
212 int ret = 0;
213
214 if (!has_multisync) {
215 /* Ignore syncobj if its handle is zero */
216 if (in_sync)
217 ret = drm_sched_job_add_syncobj_dependency(&job->base, file_priv,
218 in_sync, 0);
219 return ret;
220 }
221
222 if (se->in_sync_count && se->wait_stage == job->queue) {
223 struct drm_v3d_sem __user *handle = u64_to_user_ptr(se->in_syncs);
224
225 for (int i = 0; i < se->in_sync_count; i++) {
226 struct drm_v3d_sem in;
227
228 if (copy_from_user(&in, handle++, sizeof(in))) {
229 drm_dbg(&v3d->drm, "Failed to copy wait dep handle.\n");
230 return -EFAULT;
231 }
232
233 /* Ignore syncobj if its handle is zero */
234 if (in.handle) {
235 ret = drm_sched_job_add_syncobj_dependency(&job->base,
236 file_priv, in.handle, 0);
237 if (ret)
238 return ret;
239 }
240 }
241 }
242
243 return 0;
244}
245
246static const struct {
247 size_t size;
248 void (*free)(struct kref *ref);
249 bool has_implicit_dep;
250} v3d_job_types[] = {
251 [V3D_BIN] = { sizeof(struct v3d_bin_job), v3d_job_free, false },
252 [V3D_RENDER] = { sizeof(struct v3d_render_job), v3d_render_job_free, true },
253 [V3D_TFU] = { sizeof(struct v3d_tfu_job), v3d_job_free, true },
254 [V3D_CSD] = { sizeof(struct v3d_csd_job), v3d_job_free, true },
255 [V3D_CACHE_CLEAN] = { sizeof(struct v3d_job), v3d_job_free, false },
256 [V3D_CPU] = { sizeof(struct v3d_cpu_job), v3d_cpu_job_free, true },
257};
258
259static struct v3d_job *
260v3d_submit_add_job(struct v3d_submit *submit, enum v3d_queue queue)
261{
262 struct v3d_file_priv *v3d_priv = submit->file_priv->driver_priv;
263 struct v3d_dev *v3d = submit->v3d;
264 struct v3d_job *job;
265 int ret;
266
267 if (queue >= V3D_MAX_QUEUES)
268 return ERR_PTR(-EINVAL);
269
270 job = kzalloc(v3d_job_types[queue].size, GFP_KERNEL);
271 if (!job)
272 return ERR_PTR(-ENOMEM);
273
274 job->v3d = v3d;
275 job->queue = queue;
276 job->file_priv = v3d_priv;
277 job->free = v3d_job_types[queue].free;
278 job->has_implicit_dep = v3d_job_types[queue].has_implicit_dep;
279
280 ret = drm_sched_job_init(&job->base, &v3d_priv->sched_entity[queue],
281 1, v3d_priv, submit->file_priv->client_id);
282 if (ret)
283 goto fail_free;
284
285 /* CPU jobs don't require hardware resources */
286 if (queue != V3D_CPU) {
287 ret = v3d_pm_runtime_get(v3d);
288 if (ret)
289 goto fail_sched_job;
290 job->has_pm_ref = true;
291 }
292
293 kref_init(&job->refcount);
294
295 job->client_stats = v3d_stats_get(v3d_priv->stats[queue]);
296 job->global_stats = v3d_stats_get(v3d->queue[queue].stats);
297
298 submit->jobs[submit->job_count++] = job;
299
300 return job;
301
302fail_sched_job:
303 drm_sched_job_cleanup(&job->base);
304fail_free:
305 kfree(job);
306 return ERR_PTR(ret);
307}
308
309static void
310v3d_submit_put_jobs(struct v3d_submit *submit)
311{
312 for (int i = 0; i < submit->job_count; i++)
313 v3d_job_put(submit->jobs[i]);
314}
315
316static void
317v3d_submit_cleanup_jobs(struct v3d_submit *submit)
318{
319 for (int i = 0; i < submit->job_count; i++)
320 v3d_job_cleanup(submit->jobs[i]);
321}
322
323static int
324v3d_attach_perfmon_to_jobs(struct v3d_submit *submit, u32 perfmon_id)
325{
326 struct v3d_file_priv *v3d_priv = submit->file_priv->driver_priv;
327 struct v3d_dev *v3d = submit->v3d;
328 struct v3d_perfmon *perfmon;
329
330 if (!perfmon_id)
331 return 0;
332
333 scoped_guard(spinlock_irqsave, &v3d->perfmon_state.lock) {
334 if (v3d->global_perfmon)
335 return -EAGAIN;
336 }
337
338 perfmon = v3d_perfmon_find(v3d_priv, perfmon_id);
339 if (!perfmon)
340 return -ENOENT;
341
342 for (int i = 0; i < submit->job_count; i++) {
343 submit->jobs[i]->perfmon = perfmon;
344 if (i != 0)
345 v3d_perfmon_get(perfmon);
346 }
347
348 return 0;
349}
350
351/*
352 * Prepare fences to enforce job serialization when a perfmon is active. A job
353 * that carries a non-global perfmon must wait for every job currently in-flight
354 * across all HW queues to finish, otherwise concurrent unrelated work on the
355 * same core would pollute the performance counters. Symmetrically, while such a
356 * job is still in-flight, all subsequently submitted jobs must wait for it.
357 *
358 * We don't serialize the jobs when using a global perfmon as it's expected to
359 * track concurrent activity from all jobs.
360 */
361static int
362v3d_serialize_for_perfmon(struct v3d_job *job)
363{
364 struct v3d_dev *v3d = job->v3d;
365 struct dma_fence *merged;
366 bool is_global_perfmon;
367 int ret;
368
369 lockdep_assert_held(&v3d->sched_lock);
370
371 scoped_guard(spinlock_irqsave, &v3d->perfmon_state.lock)
372 is_global_perfmon = !!v3d->global_perfmon;
373
374 if (is_global_perfmon)
375 goto publish;
376
377 if (job->perfmon) {
378 for (enum v3d_queue q = 0; q < V3D_MAX_QUEUES; q++) {
379 struct dma_fence *f = v3d->perfmon_state.last_hw_fence[q];
380
381 if (!f || dma_fence_is_signaled(f))
382 continue;
383
384 ret = drm_sched_job_add_dependency(&job->base, dma_fence_get(f));
385 if (ret)
386 return ret;
387 }
388 } else if (v3d->perfmon_state.fence &&
389 !dma_fence_is_signaled(v3d->perfmon_state.fence)) {
390 ret = drm_sched_job_add_dependency(&job->base,
391 dma_fence_get(v3d->perfmon_state.fence));
392 if (ret)
393 return ret;
394 }
395
396publish:
397 /*
398 * Accumulate every in-flight job on this queue into one merged fence.
399 * A HW queue is fed by several scheduler entities (one per-fd), so jobs
400 * on it can complete out of order.
401 */
402 merged = dma_fence_unwrap_merge(v3d->perfmon_state.last_hw_fence[job->queue],
403 job->done_fence);
404 if (!merged)
405 return -ENOMEM;
406
407 dma_fence_put(v3d->perfmon_state.last_hw_fence[job->queue]);
408 v3d->perfmon_state.last_hw_fence[job->queue] = merged;
409
410 if (job->perfmon && !is_global_perfmon) {
411 dma_fence_put(v3d->perfmon_state.fence);
412 v3d->perfmon_state.fence = dma_fence_get(job->done_fence);
413 }
414
415 return 0;
416}
417
418static void
419v3d_submit_attach_object_fences(struct v3d_submit *submit)
420{
421 struct v3d_job *last_job = submit->jobs[submit->job_count - 1];
422
423 /* The submission's last fence covers the entire submission. Attach it
424 * to every BO touched by any job in the submission.
425 */
426 for (int i = 0; i < submit->job_count; i++) {
427 struct v3d_job *job = submit->jobs[i];
428
429 for (int j = 0; j < job->bo_count; j++) {
430 /* XXX: Use shared fences for read-only objects. */
431 dma_resv_add_fence(job->bo[j]->resv, last_job->done_fence,
432 DMA_RESV_USAGE_WRITE);
433 }
434 }
435}
436
437static void
438v3d_submit_process_post_deps(struct v3d_submit *submit, struct drm_syncobj *sync_out,
439 struct v3d_submit_ext *se)
440{
441 bool has_multisync = se && (se->flags & DRM_V3D_EXT_ID_MULTI_SYNC);
442 struct v3d_job *last_job = submit->jobs[submit->job_count - 1];
443
444 /* Make sure single syncobj and multisync are mutually exclusive */
445 WARN_ON_ONCE(sync_out && has_multisync);
446
447 /* Update the return sync object for the job */
448 /* If it only supports a single signal semaphore*/
449 if (!has_multisync) {
450 if (sync_out) {
451 drm_syncobj_replace_fence(sync_out, last_job->done_fence);
452 drm_syncobj_put(sync_out);
453 }
454 return;
455 }
456
457 /* If multiple semaphores extension is supported */
458 if (se->out_sync_count) {
459 for (int i = 0; i < se->out_sync_count; i++) {
460 drm_syncobj_replace_fence(se->out_syncs[i].syncobj,
461 last_job->done_fence);
462 drm_syncobj_put(se->out_syncs[i].syncobj);
463 }
464 kvfree(se->out_syncs);
465 }
466}
467
468static int
469v3d_submit_jobs(struct v3d_submit *submit, struct drm_syncobj *sync_out,
470 struct v3d_submit_ext *se)
471{
472 struct v3d_dev *v3d = submit->v3d;
473 int ret = 0;
474
475 mutex_lock(&v3d->sched_lock);
476
477 for (int i = 0; i < submit->job_count; i++) {
478 struct v3d_job *job = submit->jobs[i];
479
480 drm_sched_job_arm(&job->base);
481 job->done_fence = dma_fence_get(&job->base.s_fence->finished);
482
483 /* put by scheduler job completion */
484 kref_get(&job->refcount);
485 }
486
487 for (int i = 1; i < submit->job_count; i++) {
488 ret = drm_sched_job_add_dependency(&submit->jobs[i]->base,
489 dma_fence_get(submit->jobs[i - 1]->done_fence));
490 if (ret)
491 goto err;
492 }
493
494 for (int i = 0; i < submit->job_count; i++) {
495 ret = v3d_serialize_for_perfmon(submit->jobs[i]);
496 if (ret)
497 goto err;
498 }
499
500 for (int i = 0; i < submit->job_count; i++)
501 drm_sched_entity_push_job(&submit->jobs[i]->base);
502
503 mutex_unlock(&v3d->sched_lock);
504
505 v3d_submit_attach_object_fences(submit);
506 v3d_submit_unlock_reservations(submit);
507 v3d_submit_process_post_deps(submit, sync_out, se);
508
509 v3d_submit_put_jobs(submit);
510
511 return 0;
512
513err:
514 /* Mark every armed job as failed so run_job() skips execution */
515 for (int i = 0; i < submit->job_count; i++)
516 dma_fence_set_error(&submit->jobs[i]->base.s_fence->finished, ret);
517
518 for (int i = 0; i < submit->job_count; i++)
519 drm_sched_entity_push_job(&submit->jobs[i]->base);
520
521 mutex_unlock(&v3d->sched_lock);
522
523 v3d_submit_unlock_reservations(submit);
524 v3d_submit_put_jobs(submit);
525
526 return ret;
527}
528
529static int
530v3d_setup_csd_jobs_and_bos(struct v3d_submit *submit,
531 struct drm_v3d_submit_csd *args,
532 struct v3d_submit_ext *se)
533{
534 struct v3d_csd_job *job;
535 struct v3d_job *clean_job;
536 int ret;
537
538 job = (struct v3d_csd_job *)v3d_submit_add_job(submit, V3D_CSD);
539 if (IS_ERR(job))
540 return PTR_ERR(job);
541
542 ret = v3d_job_add_syncobjs(&job->base, submit->file_priv, args->in_sync, se);
543 if (ret)
544 return ret;
545
546 ret = v3d_lookup_bos(submit->file_priv, &job->base, args->bo_handles,
547 args->bo_handle_count);
548 if (ret)
549 return ret;
550
551 job->args = *args;
552
553 clean_job = v3d_submit_add_job(submit, V3D_CACHE_CLEAN);
554 if (IS_ERR(clean_job))
555 return PTR_ERR(clean_job);
556
557 return v3d_job_reference_bos(clean_job, &job->base);
558}
559
560static void
561v3d_submit_put_post_deps(struct drm_syncobj *sync_out, struct v3d_submit_ext *se)
562{
563 unsigned int i;
564
565 if (sync_out)
566 drm_syncobj_put(sync_out);
567
568 if (!(se && se->out_sync_count))
569 return;
570
571 for (i = 0; i < se->out_sync_count; i++)
572 drm_syncobj_put(se->out_syncs[i].syncobj);
573 kvfree(se->out_syncs);
574}
575
576static int
577v3d_get_multisync_post_deps(struct drm_file *file_priv,
578 struct v3d_submit_ext *se,
579 u32 count, u64 handles)
580{
581 struct v3d_file_priv *v3d_priv = file_priv->driver_priv;
582 struct v3d_dev *v3d = v3d_priv->v3d;
583 struct drm_v3d_sem __user *post_deps;
584 int i, ret;
585
586 if (!count)
587 return 0;
588
589 se->out_syncs = (struct v3d_submit_outsync *)
590 kvmalloc_objs(struct v3d_submit_outsync, count);
591 if (!se->out_syncs)
592 return -ENOMEM;
593
594 post_deps = u64_to_user_ptr(handles);
595
596 for (i = 0; i < count; i++) {
597 struct drm_v3d_sem out;
598
599 if (copy_from_user(&out, post_deps++, sizeof(out))) {
600 ret = -EFAULT;
601 drm_dbg(&v3d->drm, "Failed to copy post dep handles\n");
602 goto fail;
603 }
604
605 se->out_syncs[i].syncobj = drm_syncobj_find(file_priv,
606 out.handle);
607 if (!se->out_syncs[i].syncobj) {
608 ret = -EINVAL;
609 goto fail;
610 }
611 }
612 se->out_sync_count = count;
613
614 return 0;
615
616fail:
617 for (i--; i >= 0; i--)
618 drm_syncobj_put(se->out_syncs[i].syncobj);
619 kvfree(se->out_syncs);
620
621 return ret;
622}
623
624/* Get data for multiple binary semaphores synchronization. Parse syncobj
625 * to be signaled when job completes (out_sync).
626 */
627static int
628v3d_get_multisync_submit_deps(struct drm_file *file_priv,
629 struct drm_v3d_extension __user *ext,
630 struct v3d_submit_ext *se)
631{
632 struct v3d_file_priv *v3d_priv = file_priv->driver_priv;
633 struct v3d_dev *v3d = v3d_priv->v3d;
634 struct drm_v3d_multi_sync multisync;
635 int ret;
636
637 if (se->in_sync_count || se->out_sync_count) {
638 drm_dbg(&v3d->drm, "Two multisync extensions were added to the same job.");
639 return -EINVAL;
640 }
641
642 if (copy_from_user(&multisync, ext, sizeof(multisync)))
643 return -EFAULT;
644
645 if (multisync.pad)
646 return -EINVAL;
647
648 if (!multisync.in_sync_count && !multisync.out_sync_count) {
649 drm_dbg(&v3d->drm, "Empty multisync extension\n");
650 return -EINVAL;
651 }
652
653 ret = v3d_get_multisync_post_deps(file_priv, se, multisync.out_sync_count,
654 multisync.out_syncs);
655 if (ret)
656 return ret;
657
658 se->in_sync_count = multisync.in_sync_count;
659 se->in_syncs = multisync.in_syncs;
660 se->flags |= DRM_V3D_EXT_ID_MULTI_SYNC;
661 se->wait_stage = multisync.wait_stage;
662
663 return 0;
664}
665
666/* Returns false if the CPU job has an invalid configuration. */
667static bool
668v3d_validate_cpu_job(struct drm_file *file_priv, struct v3d_cpu_job *job)
669{
670 struct v3d_file_priv *v3d_priv = file_priv->driver_priv;
671 struct v3d_dev *v3d = v3d_priv->v3d;
672
673 if (!job) {
674 drm_dbg(&v3d->drm, "CPU job extension was attached to a GPU job.\n");
675 return false;
676 }
677
678 if (job->job_type) {
679 drm_dbg(&v3d->drm, "Two CPU job extensions were added to the same CPU job.\n");
680 return false;
681 }
682
683 return true;
684}
685
686/* Get data for the indirect CSD job submission. */
687static int
688v3d_get_cpu_indirect_csd_params(struct drm_file *file_priv,
689 struct drm_v3d_extension __user *ext,
690 struct v3d_cpu_job *job)
691{
692 struct v3d_file_priv *v3d_priv = file_priv->driver_priv;
693 struct v3d_dev *v3d = v3d_priv->v3d;
694 struct drm_v3d_indirect_csd indirect_csd;
695 struct v3d_indirect_csd_info *info = &job->indirect_csd;
696
697 if (!v3d_validate_cpu_job(file_priv, job))
698 return -EINVAL;
699
700 if (copy_from_user(&indirect_csd, ext, sizeof(indirect_csd)))
701 return -EFAULT;
702
703 if (!v3d_has_csd(v3d)) {
704 drm_warn(&v3d->drm, "Attempting CSD submit on non-CSD hardware.\n");
705 return -EINVAL;
706 }
707
708 job->job_type = V3D_CPU_JOB_TYPE_INDIRECT_CSD;
709 info->args = indirect_csd.submit;
710 info->offset = indirect_csd.offset;
711 info->wg_size = indirect_csd.wg_size;
712 memcpy(&info->wg_uniform_offsets, &indirect_csd.wg_uniform_offsets,
713 sizeof(indirect_csd.wg_uniform_offsets));
714
715 info->indirect = drm_gem_object_lookup(file_priv, indirect_csd.indirect);
716 if (!info->indirect)
717 return -ENOENT;
718
719 return 0;
720}
721
722/* Get data for the query timestamp job submission. */
723static int
724v3d_get_cpu_timestamp_query_params(struct drm_file *file_priv,
725 struct drm_v3d_extension __user *ext,
726 struct v3d_cpu_job *job)
727{
728 u32 __user *offsets, *syncs;
729 struct drm_v3d_timestamp_query timestamp;
730 struct v3d_timestamp_query_info *query_info = &job->timestamp_query;
731 unsigned int i;
732 int err;
733
734 if (!v3d_validate_cpu_job(file_priv, job))
735 return -EINVAL;
736
737 if (copy_from_user(&timestamp, ext, sizeof(timestamp)))
738 return -EFAULT;
739
740 if (timestamp.pad)
741 return -EINVAL;
742
743 job->job_type = V3D_CPU_JOB_TYPE_TIMESTAMP_QUERY;
744
745 query_info->queries = kvmalloc_objs(struct v3d_timestamp_query,
746 timestamp.count);
747 if (!query_info->queries)
748 return -ENOMEM;
749
750 offsets = u64_to_user_ptr(timestamp.offsets);
751 syncs = u64_to_user_ptr(timestamp.syncs);
752
753 for (i = 0; i < timestamp.count; i++) {
754 u32 offset, sync;
755
756 if (get_user(offset, offsets++)) {
757 err = -EFAULT;
758 goto error;
759 }
760
761 query_info->queries[i].offset = offset;
762
763 if (get_user(sync, syncs++)) {
764 err = -EFAULT;
765 goto error;
766 }
767
768 query_info->queries[i].syncobj = drm_syncobj_find(file_priv,
769 sync);
770 if (!query_info->queries[i].syncobj) {
771 err = -ENOENT;
772 goto error;
773 }
774 }
775 query_info->count = timestamp.count;
776
777 return 0;
778
779error:
780 v3d_timestamp_query_info_free(&job->timestamp_query, i);
781 return err;
782}
783
784static int
785v3d_get_cpu_reset_timestamp_params(struct drm_file *file_priv,
786 struct drm_v3d_extension __user *ext,
787 struct v3d_cpu_job *job)
788{
789 u32 __user *syncs;
790 struct drm_v3d_reset_timestamp_query reset;
791 struct v3d_timestamp_query_info *query_info = &job->timestamp_query;
792 unsigned int i;
793 int err;
794
795 if (!v3d_validate_cpu_job(file_priv, job))
796 return -EINVAL;
797
798 if (copy_from_user(&reset, ext, sizeof(reset)))
799 return -EFAULT;
800
801 job->job_type = V3D_CPU_JOB_TYPE_RESET_TIMESTAMP_QUERY;
802
803 query_info->queries = kvmalloc_objs(struct v3d_timestamp_query,
804 reset.count);
805 if (!query_info->queries)
806 return -ENOMEM;
807
808 syncs = u64_to_user_ptr(reset.syncs);
809
810 for (i = 0; i < reset.count; i++) {
811 u32 sync;
812
813 query_info->queries[i].offset = reset.offset + 8 * i;
814
815 if (get_user(sync, syncs++)) {
816 err = -EFAULT;
817 goto error;
818 }
819
820 query_info->queries[i].syncobj = drm_syncobj_find(file_priv,
821 sync);
822 if (!query_info->queries[i].syncobj) {
823 err = -ENOENT;
824 goto error;
825 }
826 }
827 query_info->count = reset.count;
828
829 return 0;
830
831error:
832 v3d_timestamp_query_info_free(&job->timestamp_query, i);
833 return err;
834}
835
836/* Get data for the copy timestamp query results job submission. */
837static int
838v3d_get_cpu_copy_query_results_params(struct drm_file *file_priv,
839 struct drm_v3d_extension __user *ext,
840 struct v3d_cpu_job *job)
841{
842 u32 __user *offsets, *syncs;
843 struct drm_v3d_copy_timestamp_query copy;
844 struct v3d_timestamp_query_info *query_info = &job->timestamp_query;
845 unsigned int i;
846 int err;
847
848 if (!v3d_validate_cpu_job(file_priv, job))
849 return -EINVAL;
850
851 if (copy_from_user(&copy, ext, sizeof(copy)))
852 return -EFAULT;
853
854 if (copy.pad)
855 return -EINVAL;
856
857 job->job_type = V3D_CPU_JOB_TYPE_COPY_TIMESTAMP_QUERY;
858
859 query_info->queries = kvmalloc_objs(struct v3d_timestamp_query,
860 copy.count);
861 if (!query_info->queries)
862 return -ENOMEM;
863
864 offsets = u64_to_user_ptr(copy.offsets);
865 syncs = u64_to_user_ptr(copy.syncs);
866
867 for (i = 0; i < copy.count; i++) {
868 u32 offset, sync;
869
870 if (get_user(offset, offsets++)) {
871 err = -EFAULT;
872 goto error;
873 }
874
875 query_info->queries[i].offset = offset;
876
877 if (get_user(sync, syncs++)) {
878 err = -EFAULT;
879 goto error;
880 }
881
882 query_info->queries[i].syncobj = drm_syncobj_find(file_priv,
883 sync);
884 if (!query_info->queries[i].syncobj) {
885 err = -ENOENT;
886 goto error;
887 }
888 }
889 query_info->count = copy.count;
890
891 job->copy.do_64bit = copy.do_64bit;
892 job->copy.do_partial = copy.do_partial;
893 job->copy.availability_bit = copy.availability_bit;
894 job->copy.offset = copy.offset;
895 job->copy.stride = copy.stride;
896
897 return 0;
898
899error:
900 v3d_timestamp_query_info_free(&job->timestamp_query, i);
901 return err;
902}
903
904static int
905v3d_copy_query_info(struct v3d_performance_query_info *query_info,
906 unsigned int count,
907 unsigned int nperfmons,
908 u32 __user *syncs,
909 u64 __user *kperfmon_ids,
910 struct drm_file *file_priv)
911{
912 unsigned int i, j;
913 int err;
914
915 for (i = 0; i < count; i++) {
916 struct v3d_performance_query *query = &query_info->queries[i];
917 u32 __user *ids_pointer;
918 u32 sync, id;
919 u64 ids;
920
921 if (get_user(sync, syncs++)) {
922 err = -EFAULT;
923 goto error;
924 }
925
926 if (get_user(ids, kperfmon_ids++)) {
927 err = -EFAULT;
928 goto error;
929 }
930
931 query->kperfmon_ids =
932 kvmalloc_array(nperfmons,
933 sizeof(struct v3d_performance_query *),
934 GFP_KERNEL);
935 if (!query->kperfmon_ids) {
936 err = -ENOMEM;
937 goto error;
938 }
939
940 ids_pointer = u64_to_user_ptr(ids);
941
942 for (j = 0; j < nperfmons; j++) {
943 if (get_user(id, ids_pointer++)) {
944 kvfree(query->kperfmon_ids);
945 err = -EFAULT;
946 goto error;
947 }
948
949 query->kperfmon_ids[j] = id;
950 }
951
952 query->syncobj = drm_syncobj_find(file_priv, sync);
953 if (!query->syncobj) {
954 kvfree(query->kperfmon_ids);
955 err = -ENOENT;
956 goto error;
957 }
958 }
959
960 return 0;
961
962error:
963 v3d_performance_query_info_free(query_info, i);
964 return err;
965}
966
967static int
968v3d_get_cpu_reset_performance_params(struct drm_file *file_priv,
969 struct drm_v3d_extension __user *ext,
970 struct v3d_cpu_job *job)
971{
972 struct v3d_performance_query_info *query_info = &job->performance_query;
973 struct drm_v3d_reset_performance_query reset;
974 int err;
975
976 if (!v3d_validate_cpu_job(file_priv, job))
977 return -EINVAL;
978
979 if (copy_from_user(&reset, ext, sizeof(reset)))
980 return -EFAULT;
981
982 job->job_type = V3D_CPU_JOB_TYPE_RESET_PERFORMANCE_QUERY;
983
984 query_info->queries =
985 kvmalloc_objs(struct v3d_performance_query, reset.count);
986 if (!query_info->queries)
987 return -ENOMEM;
988
989 err = v3d_copy_query_info(query_info,
990 reset.count,
991 reset.nperfmons,
992 u64_to_user_ptr(reset.syncs),
993 u64_to_user_ptr(reset.kperfmon_ids),
994 file_priv);
995 if (err)
996 return err;
997
998 query_info->count = reset.count;
999 query_info->nperfmons = reset.nperfmons;
1000
1001 return 0;
1002}
1003
1004static int
1005v3d_get_cpu_copy_performance_query_params(struct drm_file *file_priv,
1006 struct drm_v3d_extension __user *ext,
1007 struct v3d_cpu_job *job)
1008{
1009 struct v3d_performance_query_info *query_info = &job->performance_query;
1010 struct drm_v3d_copy_performance_query copy;
1011 int err;
1012
1013 if (!v3d_validate_cpu_job(file_priv, job))
1014 return -EINVAL;
1015
1016 if (copy_from_user(&copy, ext, sizeof(copy)))
1017 return -EFAULT;
1018
1019 if (copy.pad)
1020 return -EINVAL;
1021
1022 job->job_type = V3D_CPU_JOB_TYPE_COPY_PERFORMANCE_QUERY;
1023
1024 query_info->queries =
1025 kvmalloc_objs(struct v3d_performance_query, copy.count);
1026 if (!query_info->queries)
1027 return -ENOMEM;
1028
1029 err = v3d_copy_query_info(query_info,
1030 copy.count,
1031 copy.nperfmons,
1032 u64_to_user_ptr(copy.syncs),
1033 u64_to_user_ptr(copy.kperfmon_ids),
1034 file_priv);
1035 if (err)
1036 return err;
1037
1038 query_info->count = copy.count;
1039 query_info->nperfmons = copy.nperfmons;
1040 query_info->ncounters = copy.ncounters;
1041
1042 job->copy.do_64bit = copy.do_64bit;
1043 job->copy.do_partial = copy.do_partial;
1044 job->copy.availability_bit = copy.availability_bit;
1045 job->copy.offset = copy.offset;
1046 job->copy.stride = copy.stride;
1047
1048 return 0;
1049}
1050
1051/* Whenever userspace sets ioctl extensions, v3d_get_extensions parses data
1052 * according to the extension id (name).
1053 */
1054static int
1055v3d_get_extensions(struct drm_file *file_priv,
1056 u64 ext_handles,
1057 struct v3d_submit_ext *se,
1058 struct v3d_cpu_job *job)
1059{
1060 struct v3d_file_priv *v3d_priv = file_priv->driver_priv;
1061 struct v3d_dev *v3d = v3d_priv->v3d;
1062 struct drm_v3d_extension __user *user_ext;
1063 int ret;
1064
1065 user_ext = u64_to_user_ptr(ext_handles);
1066 while (user_ext) {
1067 struct drm_v3d_extension ext;
1068
1069 if (copy_from_user(&ext, user_ext, sizeof(ext))) {
1070 drm_dbg(&v3d->drm, "Failed to copy submit extension\n");
1071 return -EFAULT;
1072 }
1073
1074 switch (ext.id) {
1075 case DRM_V3D_EXT_ID_MULTI_SYNC:
1076 ret = v3d_get_multisync_submit_deps(file_priv, user_ext, se);
1077 break;
1078 case DRM_V3D_EXT_ID_CPU_INDIRECT_CSD:
1079 ret = v3d_get_cpu_indirect_csd_params(file_priv, user_ext, job);
1080 break;
1081 case DRM_V3D_EXT_ID_CPU_TIMESTAMP_QUERY:
1082 ret = v3d_get_cpu_timestamp_query_params(file_priv, user_ext, job);
1083 break;
1084 case DRM_V3D_EXT_ID_CPU_RESET_TIMESTAMP_QUERY:
1085 ret = v3d_get_cpu_reset_timestamp_params(file_priv, user_ext, job);
1086 break;
1087 case DRM_V3D_EXT_ID_CPU_COPY_TIMESTAMP_QUERY:
1088 ret = v3d_get_cpu_copy_query_results_params(file_priv, user_ext, job);
1089 break;
1090 case DRM_V3D_EXT_ID_CPU_RESET_PERFORMANCE_QUERY:
1091 ret = v3d_get_cpu_reset_performance_params(file_priv, user_ext, job);
1092 break;
1093 case DRM_V3D_EXT_ID_CPU_COPY_PERFORMANCE_QUERY:
1094 ret = v3d_get_cpu_copy_performance_query_params(file_priv, user_ext, job);
1095 break;
1096 default:
1097 drm_dbg(&v3d->drm, "Unknown V3D extension ID: %d\n", ext.id);
1098 return -EINVAL;
1099 }
1100
1101 if (ret)
1102 return ret;
1103
1104 user_ext = u64_to_user_ptr(ext.next);
1105 }
1106
1107 return 0;
1108}
1109
1110/**
1111 * v3d_submit_cl_ioctl() - Submits a job (frame) to the V3D.
1112 * @dev: DRM device
1113 * @data: ioctl argument
1114 * @file_priv: DRM file for this fd
1115 *
1116 * This is the main entrypoint for userspace to submit a 3D frame to
1117 * the GPU. Userspace provides the binner command list (if
1118 * applicable), and the kernel sets up the render command list to draw
1119 * to the framebuffer described in the ioctl, using the command lists
1120 * that the 3D engine's binner will produce.
1121 */
1122int
1123v3d_submit_cl_ioctl(struct drm_device *dev, void *data,
1124 struct drm_file *file_priv)
1125{
1126 struct v3d_submit submit = { .v3d = to_v3d_dev(dev), .file_priv = file_priv };
1127 struct drm_v3d_submit_cl *args = data;
1128 struct drm_syncobj *sync_out = NULL;
1129 struct v3d_submit_ext se = {0};
1130 struct v3d_bin_job *bin = NULL;
1131 struct v3d_render_job *render;
1132 struct v3d_job *clean_job;
1133 int ret;
1134
1135 trace_v3d_submit_cl_ioctl(dev, args->rcl_start, args->rcl_end);
1136
1137 if (args->pad)
1138 return -EINVAL;
1139
1140 if (args->flags &&
1141 args->flags & ~(DRM_V3D_SUBMIT_CL_FLUSH_CACHE |
1142 DRM_V3D_SUBMIT_EXTENSION)) {
1143 drm_dbg(dev, "invalid flags: %d\n", args->flags);
1144 return -EINVAL;
1145 }
1146
1147 if (args->flags & DRM_V3D_SUBMIT_EXTENSION) {
1148 ret = v3d_get_extensions(file_priv, args->extensions, &se, NULL);
1149 if (ret) {
1150 drm_dbg(dev, "Failed to get extensions.\n");
1151 return ret;
1152 }
1153 }
1154
1155 /* If multisync is configured, give priority to it and ignore out_sync. */
1156 if (args->out_sync && !(se.flags & DRM_V3D_EXT_ID_MULTI_SYNC)) {
1157 sync_out = drm_syncobj_find(file_priv, args->out_sync);
1158 if (!sync_out)
1159 return -ENOENT;
1160 }
1161
1162 if (args->bcl_start != args->bcl_end) {
1163 bin = (struct v3d_bin_job *)v3d_submit_add_job(&submit, V3D_BIN);
1164 if (IS_ERR(bin)) {
1165 ret = PTR_ERR(bin);
1166 goto fail;
1167 }
1168
1169 bin->start = args->bcl_start;
1170 bin->end = args->bcl_end;
1171 bin->qma = args->qma;
1172 bin->qms = args->qms;
1173 bin->qts = args->qts;
1174
1175 ret = v3d_job_add_syncobjs(&bin->base, file_priv, args->in_sync_bcl,
1176 &se);
1177 if (ret)
1178 goto fail;
1179 }
1180
1181 render = (struct v3d_render_job *)v3d_submit_add_job(&submit, V3D_RENDER);
1182 if (IS_ERR(render)) {
1183 ret = PTR_ERR(render);
1184 goto fail;
1185 }
1186
1187 INIT_LIST_HEAD(&render->unref_list);
1188 render->start = args->rcl_start;
1189 render->end = args->rcl_end;
1190
1191 if (bin)
1192 bin->render = render;
1193
1194 ret = v3d_job_add_syncobjs(&render->base, file_priv, args->in_sync_rcl, &se);
1195 if (ret)
1196 goto fail;
1197
1198 /*
1199 * We don't associate the BOs with the BIN job. Fences are only
1200 * attached to the last job in the submission chain, and BIN jobs
1201 * don't need implicit dependencies since depending on results from
1202 * another context is not a realistic scenario for binning.
1203 */
1204 ret = v3d_lookup_bos(submit.file_priv, &render->base,
1205 args->bo_handles, args->bo_handle_count);
1206 if (ret)
1207 goto fail;
1208
1209 if (args->flags & DRM_V3D_SUBMIT_CL_FLUSH_CACHE) {
1210 clean_job = v3d_submit_add_job(&submit, V3D_CACHE_CLEAN);
1211 if (IS_ERR(clean_job)) {
1212 ret = PTR_ERR(clean_job);
1213 goto fail;
1214 }
1215
1216 ret = v3d_job_reference_bos(clean_job, &render->base);
1217 if (ret)
1218 goto fail;
1219 }
1220
1221 ret = v3d_attach_perfmon_to_jobs(&submit, args->perfmon_id);
1222 if (ret)
1223 goto fail;
1224
1225 ret = v3d_submit_lock_reservations(&submit);
1226 if (ret)
1227 goto fail;
1228
1229 ret = v3d_submit_jobs(&submit, sync_out, &se);
1230 if (ret)
1231 goto fail_submit;
1232
1233 return 0;
1234
1235fail:
1236 v3d_submit_cleanup_jobs(&submit);
1237fail_submit:
1238 v3d_submit_put_post_deps(sync_out, &se);
1239
1240 return ret;
1241}
1242
1243/**
1244 * v3d_submit_tfu_ioctl() - Submits a TFU (texture formatting) job to the V3D.
1245 * @dev: DRM device
1246 * @data: ioctl argument
1247 * @file_priv: DRM file for this fd
1248 *
1249 * Userspace provides the register setup for the TFU, which we don't
1250 * need to validate since the TFU is behind the MMU.
1251 */
1252int
1253v3d_submit_tfu_ioctl(struct drm_device *dev, void *data,
1254 struct drm_file *file_priv)
1255{
1256 struct v3d_submit submit = { .v3d = to_v3d_dev(dev), .file_priv = file_priv };
1257 struct drm_v3d_submit_tfu *args = data;
1258 struct drm_syncobj *sync_out = NULL;
1259 struct v3d_submit_ext se = {0};
1260 struct v3d_tfu_job *job;
1261 int ret = 0;
1262
1263 trace_v3d_submit_tfu_ioctl(dev, args->iia);
1264
1265 if (args->flags && !(args->flags & DRM_V3D_SUBMIT_EXTENSION)) {
1266 drm_dbg(dev, "invalid flags: %d\n", args->flags);
1267 return -EINVAL;
1268 }
1269
1270 if (args->flags & DRM_V3D_SUBMIT_EXTENSION) {
1271 ret = v3d_get_extensions(file_priv, args->extensions, &se, NULL);
1272 if (ret) {
1273 drm_dbg(dev, "Failed to get extensions.\n");
1274 return ret;
1275 }
1276 }
1277
1278 /* If multisync is configured, give priority to it and ignore out_sync. */
1279 if (args->out_sync && !(se.flags & DRM_V3D_EXT_ID_MULTI_SYNC)) {
1280 sync_out = drm_syncobj_find(file_priv, args->out_sync);
1281 if (!sync_out)
1282 return -ENOENT;
1283 }
1284
1285 job = (struct v3d_tfu_job *)v3d_submit_add_job(&submit, V3D_TFU);
1286 if (IS_ERR(job)) {
1287 ret = PTR_ERR(job);
1288 goto fail;
1289 }
1290
1291 ret = v3d_job_add_syncobjs(&job->base, file_priv, args->in_sync, &se);
1292 if (ret)
1293 goto fail;
1294
1295 job->base.bo = kzalloc_objs(*job->base.bo, ARRAY_SIZE(args->bo_handles));
1296 if (!job->base.bo) {
1297 ret = -ENOMEM;
1298 goto fail;
1299 }
1300
1301 job->args = *args;
1302
1303 for (job->base.bo_count = 0;
1304 job->base.bo_count < ARRAY_SIZE(args->bo_handles);
1305 job->base.bo_count++) {
1306 struct drm_gem_object *bo;
1307
1308 if (!args->bo_handles[job->base.bo_count])
1309 break;
1310
1311 bo = drm_gem_object_lookup(file_priv, args->bo_handles[job->base.bo_count]);
1312 if (!bo) {
1313 drm_dbg(dev, "Failed to look up GEM BO %d: %d\n",
1314 job->base.bo_count,
1315 args->bo_handles[job->base.bo_count]);
1316 ret = -ENOENT;
1317 goto fail;
1318 }
1319 job->base.bo[job->base.bo_count] = bo;
1320 }
1321
1322 ret = v3d_submit_lock_reservations(&submit);
1323 if (ret)
1324 goto fail;
1325
1326 ret = v3d_submit_jobs(&submit, sync_out, &se);
1327 if (ret)
1328 goto fail_submit;
1329
1330 return 0;
1331
1332fail:
1333 v3d_submit_cleanup_jobs(&submit);
1334fail_submit:
1335 v3d_submit_put_post_deps(sync_out, &se);
1336
1337 return ret;
1338}
1339
1340/**
1341 * v3d_submit_csd_ioctl() - Submits a CSD (compute shader) job to the V3D.
1342 * @dev: DRM device
1343 * @data: ioctl argument
1344 * @file_priv: DRM file for this fd
1345 *
1346 * Userspace provides the register setup for the CSD, which we don't
1347 * need to validate since the CSD is behind the MMU.
1348 */
1349int
1350v3d_submit_csd_ioctl(struct drm_device *dev, void *data,
1351 struct drm_file *file_priv)
1352{
1353 struct v3d_submit submit = { .v3d = to_v3d_dev(dev), .file_priv = file_priv };
1354 struct drm_v3d_submit_csd *args = data;
1355 struct drm_syncobj *sync_out = NULL;
1356 struct v3d_submit_ext se = {0};
1357 int ret;
1358
1359 trace_v3d_submit_csd_ioctl(dev, args->cfg[5], args->cfg[6]);
1360
1361 if (args->pad)
1362 return -EINVAL;
1363
1364 if (!v3d_has_csd(submit.v3d)) {
1365 drm_warn(dev, "Attempting CSD submit on non-CSD hardware\n");
1366 return -EINVAL;
1367 }
1368
1369 if (args->flags && !(args->flags & DRM_V3D_SUBMIT_EXTENSION)) {
1370 drm_dbg(dev, "invalid flags: %d\n", args->flags);
1371 return -EINVAL;
1372 }
1373
1374 if (args->flags & DRM_V3D_SUBMIT_EXTENSION) {
1375 ret = v3d_get_extensions(file_priv, args->extensions, &se, NULL);
1376 if (ret) {
1377 drm_dbg(dev, "Failed to get extensions.\n");
1378 return ret;
1379 }
1380 }
1381
1382 /* If multisync is configured, give priority to it and ignore out_sync. */
1383 if (args->out_sync && !(se.flags & DRM_V3D_EXT_ID_MULTI_SYNC)) {
1384 sync_out = drm_syncobj_find(file_priv, args->out_sync);
1385 if (!sync_out)
1386 return -ENOENT;
1387 }
1388
1389 ret = v3d_setup_csd_jobs_and_bos(&submit, args, &se);
1390 if (ret)
1391 goto fail;
1392
1393 ret = v3d_attach_perfmon_to_jobs(&submit, args->perfmon_id);
1394 if (ret)
1395 goto fail;
1396
1397 ret = v3d_submit_lock_reservations(&submit);
1398 if (ret)
1399 goto fail;
1400
1401 ret = v3d_submit_jobs(&submit, sync_out, &se);
1402 if (ret)
1403 goto fail_submit;
1404
1405 return 0;
1406
1407fail:
1408 v3d_submit_cleanup_jobs(&submit);
1409fail_submit:
1410 v3d_submit_put_post_deps(sync_out, &se);
1411
1412 return ret;
1413}
1414
1415static const unsigned int cpu_job_bo_handle_count[] = {
1416 [V3D_CPU_JOB_TYPE_INDIRECT_CSD] = 1,
1417 [V3D_CPU_JOB_TYPE_TIMESTAMP_QUERY] = 1,
1418 [V3D_CPU_JOB_TYPE_RESET_TIMESTAMP_QUERY] = 1,
1419 [V3D_CPU_JOB_TYPE_COPY_TIMESTAMP_QUERY] = 2,
1420 [V3D_CPU_JOB_TYPE_RESET_PERFORMANCE_QUERY] = 0,
1421 [V3D_CPU_JOB_TYPE_COPY_PERFORMANCE_QUERY] = 1,
1422};
1423
1424/* Reject offset + (count - 1) * stride + write_size if it leaves the BO. */
1425static int
1426v3d_check_copy_extent(struct drm_device *dev, size_t bo_size,
1427 u32 offset, u32 stride, u32 count, u64 write_size)
1428{
1429 u64 last;
1430
1431 if (!count)
1432 return 0;
1433
1434 /*
1435 * The executors walk a u8 * cursor, so the furthest written byte is
1436 * offset + (count - 1) * stride + write_size, matching the pointer
1437 * arithmetic in v3d_copy_query_results()/v3d_copy_performance_query().
1438 * (count - 1) * stride is a u32 * u32 product that is exact in u64,
1439 * and offset + write_size stays far below the u64 range, so a single
1440 * overflow check guards the total.
1441 */
1442 last = write_size + offset;
1443 if (check_add_overflow((u64)(count - 1) * stride, last, &last) ||
1444 last > bo_size) {
1445 drm_dbg(dev, "CPU job copy buffer exceeds the destination BO.\n");
1446 return -EINVAL;
1447 }
1448
1449 return 0;
1450}
1451
1452/* Reject a query CPU job whose writes would land outside their BO. */
1453static int
1454v3d_cpu_job_bounds_check(struct v3d_cpu_job *job)
1455{
1456 struct drm_device *dev = &job->base.v3d->drm;
1457 struct v3d_timestamp_query_info *tquery = &job->timestamp_query;
1458 struct v3d_copy_query_results_info *copy = &job->copy;
1459 u32 elem = copy->do_64bit ? sizeof(u64) : sizeof(u32);
1460 struct v3d_bo *dst, *src;
1461 u64 slots, write_size;
1462 u32 i;
1463
1464 switch (job->job_type) {
1465 case V3D_CPU_JOB_TYPE_TIMESTAMP_QUERY:
1466 case V3D_CPU_JOB_TYPE_RESET_TIMESTAMP_QUERY:
1467 /* Each query writes one u64 timestamp slot into bo[0]. */
1468 dst = to_v3d_bo(job->base.bo[0]);
1469
1470 for (i = 0; i < tquery->count; i++) {
1471 if ((u64)tquery->queries[i].offset + sizeof(u64) >
1472 dst->base.base.size)
1473 goto err_range;
1474 }
1475 return 0;
1476 case V3D_CPU_JOB_TYPE_COPY_TIMESTAMP_QUERY:
1477 /* Copies one u64 per query from bo[1] into bo[0]. */
1478 dst = to_v3d_bo(job->base.bo[0]);
1479 src = to_v3d_bo(job->base.bo[1]);
1480
1481 for (i = 0; i < tquery->count; i++) {
1482 if ((u64)tquery->queries[i].offset + sizeof(u64) >
1483 src->base.base.size)
1484 goto err_range;
1485 }
1486
1487 write_size = (copy->availability_bit ? 2 : 1) * elem;
1488 return v3d_check_copy_extent(dev, dst->base.base.size,
1489 copy->offset, copy->stride,
1490 tquery->count, write_size);
1491 case V3D_CPU_JOB_TYPE_COPY_PERFORMANCE_QUERY:
1492 /*
1493 * Each query writes nperfmons * DRM_V3D_MAX_PERF_COUNTERS
1494 * counter slots into bo[0], plus an availability slot at
1495 * index ncounters. nperfmons and ncounters are user values,
1496 * so the slot count is computed overflow-safe.
1497 */
1498 dst = to_v3d_bo(job->base.bo[0]);
1499
1500 slots = (u64)job->performance_query.nperfmons *
1501 DRM_V3D_MAX_PERF_COUNTERS;
1502 if (copy->availability_bit)
1503 slots = max(slots,
1504 (u64)job->performance_query.ncounters + 1);
1505
1506 write_size = slots * elem;
1507 return v3d_check_copy_extent(dev, dst->base.base.size,
1508 copy->offset, copy->stride,
1509 job->performance_query.count,
1510 write_size);
1511 case V3D_CPU_JOB_TYPE_INDIRECT_CSD: {
1512 struct v3d_indirect_csd_info *indirect_csd = &job->indirect_csd;
1513
1514 /* 3 is the three dimensions (x, y, z) of the workgroup counts. */
1515 src = to_v3d_bo(job->base.bo[0]);
1516 if ((u64)indirect_csd->offset + 3 * sizeof(u32) >
1517 src->base.base.size)
1518 goto err_range;
1519
1520 dst = to_v3d_bo(indirect_csd->indirect);
1521 for (i = 0; i < 3; i++) {
1522 u32 uidx = indirect_csd->wg_uniform_offsets[i];
1523
1524 /*
1525 * 0xffffffff means "skip this rewrite", so the exec
1526 * path never writes that index and it needs no check.
1527 */
1528 if (uidx != 0xffffffff &&
1529 (u64)uidx * sizeof(u32) + sizeof(u32) >
1530 dst->base.base.size)
1531 goto err_range;
1532 }
1533 return 0;
1534 }
1535 default:
1536 return 0;
1537 }
1538
1539err_range:
1540 drm_dbg(dev, "CPU job query offset exceeds the BO.\n");
1541 return -EINVAL;
1542}
1543
1544/**
1545 * v3d_submit_cpu_ioctl() - Submits a CPU job to the V3D.
1546 * @dev: DRM device
1547 * @data: ioctl argument
1548 * @file_priv: DRM file for this fd
1549 *
1550 * Userspace specifies the CPU job type and data required to perform its
1551 * operations through the drm_v3d_extension struct.
1552 */
1553int
1554v3d_submit_cpu_ioctl(struct drm_device *dev, void *data,
1555 struct drm_file *file_priv)
1556{
1557 struct v3d_submit submit = { .v3d = to_v3d_dev(dev), .file_priv = file_priv };
1558 struct drm_v3d_submit_cpu *args = data;
1559 struct v3d_submit_ext se = {0};
1560 struct v3d_cpu_job *cpu_job = NULL;
1561 int ret;
1562
1563 if (args->flags && !(args->flags & DRM_V3D_SUBMIT_EXTENSION)) {
1564 drm_dbg(dev, "Invalid flags: %d\n", args->flags);
1565 return -EINVAL;
1566 }
1567
1568 cpu_job = (struct v3d_cpu_job *)v3d_submit_add_job(&submit, V3D_CPU);
1569 if (IS_ERR(cpu_job))
1570 return PTR_ERR(cpu_job);
1571
1572 if (args->flags & DRM_V3D_SUBMIT_EXTENSION) {
1573 ret = v3d_get_extensions(file_priv, args->extensions, &se, cpu_job);
1574 if (ret) {
1575 drm_dbg(dev, "Failed to get extensions.\n");
1576 goto fail;
1577 }
1578 }
1579
1580 /* Every CPU job must have a CPU job user extension */
1581 if (!cpu_job->job_type) {
1582 drm_dbg(dev, "CPU job must have a CPU job user extension.\n");
1583 ret = -EINVAL;
1584 goto fail;
1585 }
1586
1587 if (args->bo_handle_count != cpu_job_bo_handle_count[cpu_job->job_type]) {
1588 drm_dbg(dev, "This CPU job was not submitted with the proper number of BOs.\n");
1589 ret = -EINVAL;
1590 goto fail;
1591 }
1592
1593 trace_v3d_submit_cpu_ioctl(dev, cpu_job->job_type);
1594
1595 ret = v3d_job_add_syncobjs(&cpu_job->base, file_priv, 0, &se);
1596 if (ret)
1597 goto fail;
1598
1599 /* Look up the CPU jobs' BOs before v3d_setup_csd_jobs_and_bos() appends
1600 * the CSD and clean jobs in the case of indirect CSD job.
1601 */
1602 if (args->bo_handle_count) {
1603 ret = v3d_lookup_bos(submit.file_priv, &cpu_job->base,
1604 args->bo_handles, args->bo_handle_count);
1605 if (ret)
1606 goto fail;
1607
1608 ret = v3d_cpu_job_bounds_check(cpu_job);
1609 if (ret)
1610 goto fail;
1611 }
1612
1613 if (cpu_job->job_type == V3D_CPU_JOB_TYPE_INDIRECT_CSD) {
1614 ret = v3d_setup_csd_jobs_and_bos(&submit, &cpu_job->indirect_csd.args,
1615 NULL);
1616 if (ret)
1617 goto fail;
1618
1619 /* The CSD job was appended at jobs[1] */
1620 if (WARN_ON(submit.jobs[1]->queue != V3D_CSD)) {
1621 ret = -EINVAL;
1622 goto fail;
1623 }
1624
1625 cpu_job->indirect_csd.job = container_of(submit.jobs[1], struct v3d_csd_job,
1626 base);
1627 }
1628
1629 ret = v3d_submit_lock_reservations(&submit);
1630 if (ret)
1631 goto fail;
1632
1633 ret = v3d_submit_jobs(&submit, NULL, &se);
1634 if (ret)
1635 goto fail_submit;
1636
1637 return 0;
1638
1639fail:
1640 v3d_submit_cleanup_jobs(&submit);
1641fail_submit:
1642 v3d_submit_put_post_deps(NULL, &se);
1643
1644 return ret;
1645}