projects/linux·blob master · refs

683 lines · 16.6 KiB

1// SPDX-License-Identifier: GPL-2.0-only
2/* Copyright (c) 2020 NVIDIA Corporation */
3
4#include <linux/dma-fence-array.h>
5#include <linux/dma-mapping.h>
6#include <linux/file.h>
7#include <linux/host1x.h>
8#include <linux/iommu.h>
9#include <linux/kref.h>
10#include <linux/list.h>
11#include <linux/nospec.h>
12#include <linux/pm_runtime.h>
13#include <linux/scatterlist.h>
14#include <linux/slab.h>
15#include <linux/sync_file.h>
16
17#include <drm/drm_drv.h>
18#include <drm/drm_file.h>
19#include <drm/drm_syncobj.h>
20
21#include "drm.h"
22#include "gem.h"
23#include "submit.h"
24#include "uapi.h"
25
26#define SUBMIT_ERR(context, fmt, ...) \
27 dev_err_ratelimited(context->client->base.dev, \
28 "%s: job submission failed: " fmt "\n", \
29 current->comm, ##__VA_ARGS__)
30
31struct gather_bo {
32 struct host1x_bo base;
33
34 struct kref ref;
35
36 struct device *dev;
37 u32 *gather_data;
38 dma_addr_t gather_data_dma;
39 size_t gather_data_words;
40};
41
42static struct host1x_bo *gather_bo_get(struct host1x_bo *host_bo)
43{
44 struct gather_bo *bo = container_of(host_bo, struct gather_bo, base);
45
46 kref_get(&bo->ref);
47
48 return host_bo;
49}
50
51static void gather_bo_release(struct kref *ref)
52{
53 struct gather_bo *bo = container_of(ref, struct gather_bo, ref);
54
55 dma_free_attrs(bo->dev, bo->gather_data_words * 4, bo->gather_data, bo->gather_data_dma,
56 0);
57 kfree(bo);
58}
59
60static void gather_bo_put(struct host1x_bo *host_bo)
61{
62 struct gather_bo *bo = container_of(host_bo, struct gather_bo, base);
63
64 kref_put(&bo->ref, gather_bo_release);
65}
66
67static struct host1x_bo_mapping *
68gather_bo_pin(struct device *dev, struct host1x_bo *bo, enum dma_data_direction direction)
69{
70 struct gather_bo *gather = container_of(bo, struct gather_bo, base);
71 struct host1x_bo_mapping *map;
72 int err;
73
74 map = kzalloc_obj(*map);
75 if (!map)
76 return ERR_PTR(-ENOMEM);
77
78 kref_init(&map->ref);
79 map->bo = bo;
80 map->direction = direction;
81 map->dev = dev;
82
83 map->sgt = kzalloc_obj(*map->sgt);
84 if (!map->sgt) {
85 err = -ENOMEM;
86 goto free;
87 }
88
89 err = dma_get_sgtable(gather->dev, map->sgt, gather->gather_data, gather->gather_data_dma,
90 gather->gather_data_words * 4);
91 if (err)
92 goto free_sgt;
93
94 err = dma_map_sgtable(dev, map->sgt, direction, 0);
95 if (err)
96 goto free_sgt;
97
98 map->phys = sg_dma_address(map->sgt->sgl);
99 map->size = gather->gather_data_words * 4;
100 map->chunks = err;
101
102 return map;
103
104free_sgt:
105 sg_free_table(map->sgt);
106 kfree(map->sgt);
107free:
108 kfree(map);
109 return ERR_PTR(err);
110}
111
112static void gather_bo_unpin(struct host1x_bo_mapping *map)
113{
114 if (!map)
115 return;
116
117 dma_unmap_sgtable(map->dev, map->sgt, map->direction, 0);
118 sg_free_table(map->sgt);
119 kfree(map->sgt);
120
121 kfree(map);
122}
123
124static void *gather_bo_mmap(struct host1x_bo *host_bo)
125{
126 struct gather_bo *bo = container_of(host_bo, struct gather_bo, base);
127
128 return bo->gather_data;
129}
130
131static void gather_bo_munmap(struct host1x_bo *host_bo, void *addr)
132{
133}
134
135static const struct host1x_bo_ops gather_bo_ops = {
136 .get = gather_bo_get,
137 .put = gather_bo_put,
138 .pin = gather_bo_pin,
139 .unpin = gather_bo_unpin,
140 .mmap = gather_bo_mmap,
141 .munmap = gather_bo_munmap,
142};
143
144static struct tegra_drm_mapping *
145tegra_drm_mapping_get(struct tegra_drm_context *context, u32 id)
146{
147 struct tegra_drm_mapping *mapping;
148
149 xa_lock(&context->mappings);
150
151 mapping = xa_load(&context->mappings, id);
152 if (mapping)
153 kref_get(&mapping->ref);
154
155 xa_unlock(&context->mappings);
156
157 return mapping;
158}
159
160static void *alloc_copy_user_array(void __user *from, size_t count, size_t size)
161{
162 size_t copy_len;
163 void *data;
164
165 if (check_mul_overflow(count, size, &copy_len))
166 return ERR_PTR(-EINVAL);
167
168 if (copy_len > 0x4000)
169 return ERR_PTR(-E2BIG);
170
171 data = vmemdup_user(from, copy_len);
172 if (IS_ERR(data))
173 return ERR_CAST(data);
174
175 return data;
176}
177
178static int submit_copy_gather_data(struct gather_bo **pbo, struct device *dev,
179 struct tegra_drm_context *context,
180 struct drm_tegra_channel_submit *args)
181{
182 struct gather_bo *bo;
183 size_t copy_len;
184
185 if (args->gather_data_words == 0) {
186 SUBMIT_ERR(context, "gather_data_words cannot be zero");
187 return -EINVAL;
188 }
189
190 if (check_mul_overflow((size_t)args->gather_data_words, (size_t)4, &copy_len)) {
191 SUBMIT_ERR(context, "gather_data_words is too large");
192 return -EINVAL;
193 }
194
195 bo = kzalloc_obj(*bo);
196 if (!bo) {
197 SUBMIT_ERR(context, "failed to allocate memory for bo info");
198 return -ENOMEM;
199 }
200
201 host1x_bo_init(&bo->base, &gather_bo_ops);
202 kref_init(&bo->ref);
203 bo->dev = dev;
204
205 bo->gather_data = dma_alloc_attrs(dev, copy_len, &bo->gather_data_dma,
206 GFP_KERNEL | __GFP_NOWARN, 0);
207 if (!bo->gather_data) {
208 SUBMIT_ERR(context, "failed to allocate memory for gather data");
209 kfree(bo);
210 return -ENOMEM;
211 }
212
213 if (copy_from_user(bo->gather_data, u64_to_user_ptr(args->gather_data_ptr), copy_len)) {
214 SUBMIT_ERR(context, "failed to copy gather data from userspace");
215 dma_free_attrs(dev, copy_len, bo->gather_data, bo->gather_data_dma, 0);
216 kfree(bo);
217 return -EFAULT;
218 }
219
220 bo->gather_data_words = args->gather_data_words;
221
222 *pbo = bo;
223
224 return 0;
225}
226
227static int submit_write_reloc(struct tegra_drm_context *context, struct gather_bo *bo,
228 struct drm_tegra_submit_buf *buf, struct tegra_drm_mapping *mapping)
229{
230 /* TODO check that target_offset is within bounds */
231 dma_addr_t iova = mapping->iova + buf->reloc.target_offset;
232 u32 written_ptr;
233
234#ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT
235 if (buf->flags & DRM_TEGRA_SUBMIT_RELOC_SECTOR_LAYOUT)
236 iova |= BIT_ULL(39);
237#endif
238
239 written_ptr = iova >> buf->reloc.shift;
240
241 if (buf->reloc.gather_offset_words >= bo->gather_data_words) {
242 SUBMIT_ERR(context,
243 "relocation has too large gather offset (%u vs gather length %zu)",
244 buf->reloc.gather_offset_words, bo->gather_data_words);
245 return -EINVAL;
246 }
247
248 buf->reloc.gather_offset_words = array_index_nospec(buf->reloc.gather_offset_words,
249 bo->gather_data_words);
250
251 bo->gather_data[buf->reloc.gather_offset_words] = written_ptr;
252
253 return 0;
254}
255
256static int submit_process_bufs(struct tegra_drm_context *context, struct gather_bo *bo,
257 struct drm_tegra_channel_submit *args,
258 struct tegra_drm_submit_data *job_data)
259{
260 struct tegra_drm_used_mapping *mappings;
261 struct drm_tegra_submit_buf *bufs;
262 int err;
263 u32 i;
264
265 bufs = alloc_copy_user_array(u64_to_user_ptr(args->bufs_ptr), args->num_bufs,
266 sizeof(*bufs));
267 if (IS_ERR(bufs)) {
268 SUBMIT_ERR(context, "failed to copy bufs array from userspace");
269 return PTR_ERR(bufs);
270 }
271
272 mappings = kzalloc_objs(*mappings, args->num_bufs);
273 if (!mappings) {
274 SUBMIT_ERR(context, "failed to allocate memory for mapping info");
275 err = -ENOMEM;
276 goto done;
277 }
278
279 for (i = 0; i < args->num_bufs; i++) {
280 struct drm_tegra_submit_buf *buf = &bufs[i];
281 struct tegra_drm_mapping *mapping;
282
283 if (buf->flags & ~DRM_TEGRA_SUBMIT_RELOC_SECTOR_LAYOUT) {
284 SUBMIT_ERR(context, "invalid flag specified for buffer");
285 err = -EINVAL;
286 goto drop_refs;
287 }
288
289 mapping = tegra_drm_mapping_get(context, buf->mapping);
290 if (!mapping) {
291 SUBMIT_ERR(context, "invalid mapping ID '%u' for buffer", buf->mapping);
292 err = -EINVAL;
293 goto drop_refs;
294 }
295
296 err = submit_write_reloc(context, bo, buf, mapping);
297 if (err) {
298 tegra_drm_mapping_put(mapping);
299 goto drop_refs;
300 }
301
302 mappings[i].mapping = mapping;
303 mappings[i].flags = buf->flags;
304 }
305
306 job_data->used_mappings = mappings;
307 job_data->num_used_mappings = i;
308
309 err = 0;
310
311 goto done;
312
313drop_refs:
314 while (i--)
315 tegra_drm_mapping_put(mappings[i].mapping);
316
317 kfree(mappings);
318 job_data->used_mappings = NULL;
319
320done:
321 kvfree(bufs);
322
323 return err;
324}
325
326static int submit_get_syncpt(struct tegra_drm_context *context, struct host1x_job *job,
327 struct xarray *syncpoints, struct drm_tegra_channel_submit *args)
328{
329 struct host1x_syncpt *sp;
330
331 if (args->syncpt.flags) {
332 SUBMIT_ERR(context, "invalid flag specified for syncpt");
333 return -EINVAL;
334 }
335
336 /* Syncpt ref will be dropped on job release */
337 sp = xa_load(syncpoints, args->syncpt.id);
338 if (!sp) {
339 SUBMIT_ERR(context, "syncpoint specified in syncpt was not allocated");
340 return -EINVAL;
341 }
342
343 job->syncpt = host1x_syncpt_get(sp);
344 job->syncpt_incrs = args->syncpt.increments;
345
346 return 0;
347}
348
349static int submit_job_add_gather(struct host1x_job *job, struct tegra_drm_context *context,
350 struct drm_tegra_submit_cmd_gather_uptr *cmd,
351 struct gather_bo *bo, u32 *offset,
352 struct tegra_drm_submit_data *job_data,
353 u32 *class)
354{
355 u32 next_offset;
356
357 if (cmd->reserved[0] || cmd->reserved[1] || cmd->reserved[2]) {
358 SUBMIT_ERR(context, "non-zero reserved field in GATHER_UPTR command");
359 return -EINVAL;
360 }
361
362 /* Check for maximum gather size */
363 if (cmd->words > 16383) {
364 SUBMIT_ERR(context, "too many words in GATHER_UPTR command");
365 return -EINVAL;
366 }
367
368 if (check_add_overflow(*offset, cmd->words, &next_offset)) {
369 SUBMIT_ERR(context, "too many total words in job");
370 return -EINVAL;
371 }
372
373 if (next_offset > bo->gather_data_words) {
374 SUBMIT_ERR(context, "GATHER_UPTR command overflows gather data");
375 return -EINVAL;
376 }
377
378 if (tegra_drm_fw_validate(context->client, bo->gather_data, *offset,
379 cmd->words, job_data, class)) {
380 SUBMIT_ERR(context, "job was rejected by firewall");
381 return -EINVAL;
382 }
383
384 host1x_job_add_gather(job, &bo->base, cmd->words, *offset * 4);
385
386 *offset = next_offset;
387
388 return 0;
389}
390
391static struct host1x_job *
392submit_create_job(struct tegra_drm_context *context, struct gather_bo *bo,
393 struct drm_tegra_channel_submit *args, struct tegra_drm_submit_data *job_data,
394 struct xarray *syncpoints)
395{
396 struct drm_tegra_submit_cmd *cmds;
397 u32 i, gather_offset = 0, class;
398 struct host1x_job *job;
399 int err;
400
401 /* Set initial class for firewall. */
402 class = context->client->base.class;
403
404 cmds = alloc_copy_user_array(u64_to_user_ptr(args->cmds_ptr), args->num_cmds,
405 sizeof(*cmds));
406 if (IS_ERR(cmds)) {
407 SUBMIT_ERR(context, "failed to copy cmds array from userspace");
408 return ERR_CAST(cmds);
409 }
410
411 job = host1x_job_alloc(context->channel, args->num_cmds, 0, true);
412 if (!job) {
413 SUBMIT_ERR(context, "failed to allocate memory for job");
414 job = ERR_PTR(-ENOMEM);
415 goto done;
416 }
417
418 err = submit_get_syncpt(context, job, syncpoints, args);
419 if (err < 0)
420 goto free_job;
421
422 job->client = &context->client->base;
423 job->class = context->client->base.class;
424 job->serialize = true;
425
426 for (i = 0; i < args->num_cmds; i++) {
427 struct drm_tegra_submit_cmd *cmd = &cmds[i];
428
429 if (cmd->flags) {
430 SUBMIT_ERR(context, "unknown flags given for cmd");
431 err = -EINVAL;
432 goto free_job;
433 }
434
435 if (cmd->type == DRM_TEGRA_SUBMIT_CMD_GATHER_UPTR) {
436 err = submit_job_add_gather(job, context, &cmd->gather_uptr, bo,
437 &gather_offset, job_data, &class);
438 if (err)
439 goto free_job;
440 } else if (cmd->type == DRM_TEGRA_SUBMIT_CMD_WAIT_SYNCPT) {
441 if (cmd->wait_syncpt.reserved[0] || cmd->wait_syncpt.reserved[1]) {
442 SUBMIT_ERR(context, "non-zero reserved value");
443 err = -EINVAL;
444 goto free_job;
445 }
446
447 host1x_job_add_wait(job, cmd->wait_syncpt.id, cmd->wait_syncpt.value,
448 false, class);
449 } else if (cmd->type == DRM_TEGRA_SUBMIT_CMD_WAIT_SYNCPT_RELATIVE) {
450 if (cmd->wait_syncpt.reserved[0] || cmd->wait_syncpt.reserved[1]) {
451 SUBMIT_ERR(context, "non-zero reserved value");
452 err = -EINVAL;
453 goto free_job;
454 }
455
456 if (cmd->wait_syncpt.id != args->syncpt.id) {
457 SUBMIT_ERR(context, "syncpoint ID in CMD_WAIT_SYNCPT_RELATIVE is not used by the job");
458 err = -EINVAL;
459 goto free_job;
460 }
461
462 host1x_job_add_wait(job, cmd->wait_syncpt.id, cmd->wait_syncpt.value,
463 true, class);
464 } else {
465 SUBMIT_ERR(context, "unknown cmd type");
466 err = -EINVAL;
467 goto free_job;
468 }
469 }
470
471 if (gather_offset == 0) {
472 SUBMIT_ERR(context, "job must have at least one gather");
473 err = -EINVAL;
474 goto free_job;
475 }
476
477 goto done;
478
479free_job:
480 host1x_job_put(job);
481 job = ERR_PTR(err);
482
483done:
484 kvfree(cmds);
485
486 return job;
487}
488
489static void release_job(struct host1x_job *job)
490{
491 struct tegra_drm_client *client = container_of(job->client, struct tegra_drm_client, base);
492 struct tegra_drm_submit_data *job_data = job->user_data;
493 u32 i;
494
495 if (job->memory_context)
496 host1x_memory_context_put(job->memory_context);
497
498 for (i = 0; i < job_data->num_used_mappings; i++)
499 tegra_drm_mapping_put(job_data->used_mappings[i].mapping);
500
501 kfree(job_data->used_mappings);
502 kfree(job_data);
503
504 pm_runtime_mark_last_busy(client->base.dev);
505 pm_runtime_put_autosuspend(client->base.dev);
506}
507
508int tegra_drm_ioctl_channel_submit(struct drm_device *drm, void *data,
509 struct drm_file *file)
510{
511 struct tegra_drm_file *fpriv = file->driver_priv;
512 struct drm_tegra_channel_submit *args = data;
513 struct tegra_drm_submit_data *job_data;
514 struct drm_syncobj *syncobj = NULL;
515 struct tegra_drm_context *context;
516 struct host1x_job *job;
517 struct gather_bo *bo;
518 u32 i;
519 int err;
520
521 mutex_lock(&fpriv->lock);
522
523 context = xa_load(&fpriv->contexts, args->context);
524 if (!context) {
525 mutex_unlock(&fpriv->lock);
526 pr_err_ratelimited("%s: %s: invalid channel context '%#x'", __func__,
527 current->comm, args->context);
528 return -EINVAL;
529 }
530
531 if (args->syncobj_in) {
532 struct dma_fence *fence;
533
534 err = drm_syncobj_find_fence(file, args->syncobj_in, 0, 0, &fence);
535 if (err) {
536 SUBMIT_ERR(context, "invalid syncobj_in '%#x'", args->syncobj_in);
537 goto unlock;
538 }
539
540 err = dma_fence_wait_timeout(fence, true, msecs_to_jiffies(10000));
541 dma_fence_put(fence);
542 if (err) {
543 SUBMIT_ERR(context, "wait for syncobj_in timed out");
544 goto unlock;
545 }
546 }
547
548 if (args->syncobj_out) {
549 syncobj = drm_syncobj_find(file, args->syncobj_out);
550 if (!syncobj) {
551 SUBMIT_ERR(context, "invalid syncobj_out '%#x'", args->syncobj_out);
552 err = -ENOENT;
553 goto unlock;
554 }
555 }
556
557 /* Allocate gather BO and copy gather words in. */
558 err = submit_copy_gather_data(&bo, drm->dev, context, args);
559 if (err)
560 goto unlock;
561
562 job_data = kzalloc_obj(*job_data);
563 if (!job_data) {
564 SUBMIT_ERR(context, "failed to allocate memory for job data");
565 err = -ENOMEM;
566 goto put_bo;
567 }
568
569 /* Get data buffer mappings and do relocation patching. */
570 err = submit_process_bufs(context, bo, args, job_data);
571 if (err)
572 goto free_job_data;
573
574 /* Allocate host1x_job and add gathers and waits to it. */
575 job = submit_create_job(context, bo, args, job_data, &fpriv->syncpoints);
576 if (IS_ERR(job)) {
577 err = PTR_ERR(job);
578 goto free_job_data;
579 }
580
581 /* Map gather data for Host1x. */
582 err = host1x_job_pin(job, context->client->base.dev);
583 if (err) {
584 SUBMIT_ERR(context, "failed to pin job: %d", err);
585 goto put_job;
586 }
587
588 if (context->client->ops->get_streamid_offset) {
589 err = context->client->ops->get_streamid_offset(
590 context->client, &job->engine_streamid_offset);
591 if (err) {
592 SUBMIT_ERR(context, "failed to get streamid offset: %d", err);
593 goto unpin_job;
594 }
595 }
596
597 if (context->memory_context && context->client->ops->can_use_memory_ctx) {
598 bool supported;
599
600 err = context->client->ops->can_use_memory_ctx(context->client, &supported);
601 if (err) {
602 SUBMIT_ERR(context, "failed to detect if engine can use memory context: %d", err);
603 goto unpin_job;
604 }
605
606 if (supported) {
607 job->memory_context = context->memory_context;
608 host1x_memory_context_get(job->memory_context);
609 }
610 } else if (context->client->ops->get_streamid_offset) {
611 /*
612 * Job submission will need to temporarily change stream ID,
613 * so need to tell it what to change it back to.
614 */
615 if (!tegra_dev_iommu_get_stream_id(context->client->base.dev,
616 &job->engine_fallback_streamid))
617 job->engine_fallback_streamid = TEGRA_STREAM_ID_BYPASS;
618 }
619
620 /* Boot engine. */
621 err = pm_runtime_resume_and_get(context->client->base.dev);
622 if (err < 0) {
623 SUBMIT_ERR(context, "could not power up engine: %d", err);
624 goto put_memory_context;
625 }
626
627 job->user_data = job_data;
628 job->release = release_job;
629 job->timeout = 10000;
630
631 /*
632 * job_data is now part of job reference counting, so don't release
633 * it from here.
634 */
635 job_data = NULL;
636
637 /* Submit job to hardware. */
638 err = host1x_job_submit(job);
639 if (err) {
640 SUBMIT_ERR(context, "host1x job submission failed: %d", err);
641 goto unpin_job;
642 }
643
644 /* Return postfences to userspace and add fences to DMA reservations. */
645 args->syncpt.value = job->syncpt_end;
646
647 if (syncobj) {
648 struct dma_fence *fence = host1x_fence_create(job->syncpt, job->syncpt_end, true);
649 if (IS_ERR(fence)) {
650 err = PTR_ERR(fence);
651 SUBMIT_ERR(context, "failed to create postfence: %d", err);
652 }
653
654 drm_syncobj_replace_fence(syncobj, fence);
655 }
656
657 goto put_job;
658
659put_memory_context:
660 if (job->memory_context)
661 host1x_memory_context_put(job->memory_context);
662unpin_job:
663 host1x_job_unpin(job);
664put_job:
665 host1x_job_put(job);
666free_job_data:
667 if (job_data && job_data->used_mappings) {
668 for (i = 0; i < job_data->num_used_mappings; i++)
669 tegra_drm_mapping_put(job_data->used_mappings[i].mapping);
670
671 kfree(job_data->used_mappings);
672 }
673
674 kfree(job_data);
675put_bo:
676 gather_bo_put(&bo->base);
677unlock:
678 if (syncobj)
679 drm_syncobj_put(syncobj);
680
681 mutex_unlock(&fpriv->lock);
682 return err;
683}