projects/linux·blob master · refs

688 lines · 16.4 KiB

1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Tegra host1x Command DMA
4 *
5 * Copyright (c) 2010-2013, NVIDIA Corporation.
6 */
7
8
9#include <asm/cacheflush.h>
10#include <linux/device.h>
11#include <linux/dma-mapping.h>
12#include <linux/host1x.h>
13#include <linux/interrupt.h>
14#include <linux/kernel.h>
15#include <linux/kfifo.h>
16#include <linux/overflow.h>
17#include <linux/slab.h>
18#include <trace/events/host1x.h>
19
20#include "cdma.h"
21#include "channel.h"
22#include "dev.h"
23#include "debug.h"
24#include "job.h"
25
26/*
27 * push_buffer
28 *
29 * The push buffer is a circular array of words to be fetched by command DMA.
30 * Note that it works slightly differently to the sync queue; fence == pos
31 * means that the push buffer is full, not empty.
32 */
33
34/*
35 * Typically the commands written into the push buffer are a pair of words. We
36 * use slots to represent each of these pairs and to simplify things. Note the
37 * strange number of slots allocated here. 512 slots will fit exactly within a
38 * single memory page. We also need one additional word at the end of the push
39 * buffer for the RESTART opcode that will instruct the CDMA to jump back to
40 * the beginning of the push buffer. With 512 slots, this means that we'll use
41 * 2 memory pages and waste 4092 bytes of the second page that will never be
42 * used.
43 */
44#define HOST1X_PUSHBUFFER_SLOTS 511
45
46/*
47 * Clean up push buffer resources
48 */
49static void host1x_pushbuffer_destroy(struct push_buffer *pb)
50{
51 struct host1x_cdma *cdma = pb_to_cdma(pb);
52 struct host1x *host1x = cdma_to_host1x(cdma);
53
54 if (!pb->mapped)
55 return;
56
57 if (host1x->domain) {
58 iommu_unmap(host1x->domain, pb->dma, pb->alloc_size);
59 free_iova(&host1x->iova, iova_pfn(&host1x->iova, pb->dma));
60 }
61
62 dma_free_wc(host1x->dev, pb->alloc_size, pb->mapped, pb->phys);
63
64 pb->mapped = NULL;
65 pb->phys = 0;
66}
67
68/*
69 * Init push buffer resources
70 */
71static int host1x_pushbuffer_init(struct push_buffer *pb)
72{
73 struct host1x_cdma *cdma = pb_to_cdma(pb);
74 struct host1x *host1x = cdma_to_host1x(cdma);
75 struct iova *alloc;
76 u32 size;
77 int err;
78
79 pb->mapped = NULL;
80 pb->phys = 0;
81 pb->size = HOST1X_PUSHBUFFER_SLOTS * 8;
82
83 size = pb->size + 4;
84
85 /* initialize buffer pointers */
86 pb->fence = pb->size - 8;
87 pb->pos = 0;
88
89 if (host1x->domain) {
90 unsigned long shift;
91
92 size = iova_align(&host1x->iova, size);
93
94 pb->mapped = dma_alloc_wc(host1x->dev, size, &pb->phys,
95 GFP_KERNEL);
96 if (!pb->mapped)
97 return -ENOMEM;
98
99 shift = iova_shift(&host1x->iova);
100 alloc = alloc_iova(&host1x->iova, size >> shift,
101 host1x->iova_end >> shift, true);
102 if (!alloc) {
103 err = -ENOMEM;
104 goto iommu_free_mem;
105 }
106
107 pb->dma = iova_dma_addr(&host1x->iova, alloc);
108 err = iommu_map(host1x->domain, pb->dma, pb->phys, size,
109 IOMMU_READ, GFP_KERNEL);
110 if (err)
111 goto iommu_free_iova;
112 } else {
113 pb->mapped = dma_alloc_wc(host1x->dev, size, &pb->phys,
114 GFP_KERNEL);
115 if (!pb->mapped)
116 return -ENOMEM;
117
118 pb->dma = pb->phys;
119 }
120
121 pb->alloc_size = size;
122
123 host1x_hw_pushbuffer_init(host1x, pb);
124
125 return 0;
126
127iommu_free_iova:
128 __free_iova(&host1x->iova, alloc);
129iommu_free_mem:
130 dma_free_wc(host1x->dev, size, pb->mapped, pb->phys);
131
132 return err;
133}
134
135/*
136 * Push two words to the push buffer
137 * Caller must ensure push buffer is not full
138 */
139static void host1x_pushbuffer_push(struct push_buffer *pb, u32 op1, u32 op2)
140{
141 u32 *p = (u32 *)((void *)pb->mapped + pb->pos);
142
143 WARN_ON(pb->pos == pb->fence);
144 *(p++) = op1;
145 *(p++) = op2;
146 pb->pos += 8;
147
148 if (pb->pos >= pb->size)
149 pb->pos -= pb->size;
150}
151
152/*
153 * Pop a number of two word slots from the push buffer
154 * Caller must ensure push buffer is not empty
155 */
156static void host1x_pushbuffer_pop(struct push_buffer *pb, unsigned int slots)
157{
158 /* Advance the next write position */
159 pb->fence += slots * 8;
160
161 if (pb->fence >= pb->size)
162 pb->fence -= pb->size;
163}
164
165/*
166 * Return the number of two word slots free in the push buffer
167 */
168static u32 host1x_pushbuffer_space(struct push_buffer *pb)
169{
170 unsigned int fence = pb->fence;
171
172 if (pb->fence < pb->pos)
173 fence += pb->size;
174
175 return (fence - pb->pos) / 8;
176}
177
178/*
179 * Sleep (if necessary) until the requested event happens
180 * - CDMA_EVENT_SYNC_QUEUE_EMPTY : sync queue is completely empty.
181 * - Returns 1
182 * - CDMA_EVENT_PUSH_BUFFER_SPACE : there is space in the push buffer
183 * - Return the amount of space (> 0)
184 * Must be called with the cdma lock held.
185 */
186unsigned int host1x_cdma_wait_locked(struct host1x_cdma *cdma,
187 enum cdma_event event)
188{
189 for (;;) {
190 struct push_buffer *pb = &cdma->push_buffer;
191 unsigned int space;
192
193 switch (event) {
194 case CDMA_EVENT_SYNC_QUEUE_EMPTY:
195 space = list_empty(&cdma->sync_queue) ? 1 : 0;
196 break;
197
198 case CDMA_EVENT_PUSH_BUFFER_SPACE:
199 space = host1x_pushbuffer_space(pb);
200 break;
201
202 default:
203 WARN_ON(1);
204 return -EINVAL;
205 }
206
207 if (space)
208 return space;
209
210 trace_host1x_wait_cdma(dev_name(cdma_to_channel(cdma)->dev),
211 event);
212
213 /* If somebody has managed to already start waiting, yield */
214 if (cdma->event != CDMA_EVENT_NONE) {
215 mutex_unlock(&cdma->lock);
216 schedule();
217 mutex_lock(&cdma->lock);
218 continue;
219 }
220
221 cdma->event = event;
222
223 mutex_unlock(&cdma->lock);
224 wait_for_completion(&cdma->complete);
225 mutex_lock(&cdma->lock);
226 }
227
228 return 0;
229}
230
231/*
232 * Sleep (if necessary) until the push buffer has enough free space.
233 *
234 * Must be called with the cdma lock held.
235 */
236static int host1x_cdma_wait_pushbuffer_space(struct host1x *host1x,
237 struct host1x_cdma *cdma,
238 unsigned int needed)
239{
240 while (true) {
241 struct push_buffer *pb = &cdma->push_buffer;
242 unsigned int space;
243
244 space = host1x_pushbuffer_space(pb);
245 if (space >= needed)
246 break;
247
248 trace_host1x_wait_cdma(dev_name(cdma_to_channel(cdma)->dev),
249 CDMA_EVENT_PUSH_BUFFER_SPACE);
250
251 /* If somebody has managed to already start waiting, yield */
252 if (cdma->event != CDMA_EVENT_NONE) {
253 mutex_unlock(&cdma->lock);
254 schedule();
255 mutex_lock(&cdma->lock);
256 continue;
257 }
258
259 cdma->event = CDMA_EVENT_PUSH_BUFFER_SPACE;
260
261 mutex_unlock(&cdma->lock);
262 wait_for_completion(&cdma->complete);
263 mutex_lock(&cdma->lock);
264 }
265
266 return 0;
267}
268/*
269 * Start timer that tracks the time spent by the job.
270 * Must be called with the cdma lock held.
271 */
272static void cdma_start_timer_locked(struct host1x_cdma *cdma,
273 struct host1x_job *job)
274{
275 if (cdma->timeout.client) {
276 /* timer already started */
277 return;
278 }
279
280 cdma->timeout.client = job->client;
281 cdma->timeout.syncpt = job->syncpt;
282 cdma->timeout.syncpt_val = job->syncpt_end;
283 cdma->timeout.start_ktime = ktime_get();
284
285 schedule_delayed_work(&cdma->timeout.wq,
286 msecs_to_jiffies(job->timeout));
287}
288
289/*
290 * Stop timer when a buffer submission completes.
291 * Must be called with the cdma lock held.
292 */
293static void stop_cdma_timer_locked(struct host1x_cdma *cdma)
294{
295 cancel_delayed_work(&cdma->timeout.wq);
296 cdma->timeout.client = NULL;
297}
298
299/*
300 * For all sync queue entries that have already finished according to the
301 * current sync point registers:
302 * - unpin & unref their mems
303 * - pop their push buffer slots
304 * - remove them from the sync queue
305 * This is normally called from the host code's worker thread, but can be
306 * called manually if necessary.
307 * Must be called with the cdma lock held.
308 */
309static void update_cdma_locked(struct host1x_cdma *cdma)
310{
311 bool signal = false;
312 struct host1x_job *job, *n;
313
314 /*
315 * Walk the sync queue, reading the sync point registers as necessary,
316 * to consume as many sync queue entries as possible without blocking
317 */
318 list_for_each_entry_safe(job, n, &cdma->sync_queue, list) {
319 struct host1x_syncpt *sp = job->syncpt;
320
321 /* Check whether this syncpt has completed, and bail if not */
322 if (!host1x_syncpt_is_expired(sp, job->syncpt_end) &&
323 !job->cancelled) {
324 /* Start timer on next pending syncpt */
325 if (job->timeout)
326 cdma_start_timer_locked(cdma, job);
327
328 break;
329 }
330
331 /* Cancel timeout, when a buffer completes */
332 if (cdma->timeout.client)
333 stop_cdma_timer_locked(cdma);
334
335 /* Unpin the memory */
336 host1x_job_unpin(job);
337
338 /* Pop push buffer slots */
339 if (job->num_slots) {
340 struct push_buffer *pb = &cdma->push_buffer;
341
342 host1x_pushbuffer_pop(pb, job->num_slots);
343
344 if (cdma->event == CDMA_EVENT_PUSH_BUFFER_SPACE)
345 signal = true;
346 }
347
348 list_del(&job->list);
349 host1x_job_put(job);
350 }
351
352 if (cdma->event == CDMA_EVENT_SYNC_QUEUE_EMPTY &&
353 list_empty(&cdma->sync_queue))
354 signal = true;
355
356 if (signal) {
357 cdma->event = CDMA_EVENT_NONE;
358 complete(&cdma->complete);
359 }
360}
361
362void host1x_cdma_update_sync_queue(struct host1x_cdma *cdma,
363 struct device *dev)
364{
365 struct host1x *host1x = cdma_to_host1x(cdma);
366 u32 restart_addr, syncpt_incrs, syncpt_val;
367 struct host1x_job *job, *next_job = NULL;
368
369 syncpt_val = host1x_syncpt_load(cdma->timeout.syncpt);
370
371 dev_dbg(dev, "%s: starting cleanup (thresh %d)\n",
372 __func__, syncpt_val);
373
374 /*
375 * Move the sync_queue read pointer to the first entry that hasn't
376 * completed based on the current HW syncpt value. It's likely there
377 * won't be any (i.e. we're still at the head), but covers the case
378 * where a syncpt incr happens just prior/during the teardown.
379 */
380
381 dev_dbg(dev, "%s: skip completed buffers still in sync_queue\n",
382 __func__);
383
384 list_for_each_entry(job, &cdma->sync_queue, list) {
385 if (syncpt_val < job->syncpt_end) {
386
387 if (!list_is_last(&job->list, &cdma->sync_queue))
388 next_job = list_next_entry(job, list);
389
390 goto syncpt_incr;
391 }
392
393 host1x_job_dump(dev, job);
394 }
395
396 /* all jobs have been completed */
397 job = NULL;
398
399syncpt_incr:
400
401 /*
402 * Increment with CPU the remaining syncpts of a partially executed job.
403 *
404 * CDMA will continue execution starting with the next job or will get
405 * into idle state.
406 */
407 if (next_job)
408 restart_addr = next_job->first_get;
409 else
410 restart_addr = cdma->last_pos;
411
412 if (!job)
413 goto resume;
414
415 /* do CPU increments for the remaining syncpts */
416 if (job->syncpt_recovery) {
417 dev_dbg(dev, "%s: perform CPU incr on pending buffers\n",
418 __func__);
419
420 /* won't need a timeout when replayed */
421 job->timeout = 0;
422
423 syncpt_incrs = wrapping_sub(u32, job->syncpt_end, syncpt_val);
424 dev_dbg(dev, "%s: CPU incr (%d)\n", __func__, syncpt_incrs);
425
426 host1x_job_dump(dev, job);
427
428 /* safe to use CPU to incr syncpts */
429 host1x_hw_cdma_timeout_cpu_incr(host1x, cdma, job->first_get,
430 syncpt_incrs, job->syncpt_end,
431 job->num_slots);
432
433 dev_dbg(dev, "%s: finished sync_queue modification\n",
434 __func__);
435 } else {
436 struct host1x_job *failed_job = job;
437
438 host1x_job_dump(dev, job);
439
440 host1x_syncpt_set_locked(job->syncpt);
441 failed_job->cancelled = true;
442
443 list_for_each_entry_continue(job, &cdma->sync_queue, list) {
444 unsigned int i;
445
446 if (job->syncpt != failed_job->syncpt)
447 continue;
448
449 for (i = 0; i < job->num_slots; i++) {
450 unsigned int slot = (job->first_get/8 + i) %
451 HOST1X_PUSHBUFFER_SLOTS;
452 u32 *mapped = cdma->push_buffer.mapped;
453
454 /*
455 * Overwrite opcodes with 0 word writes
456 * to offset 0xbad. This does nothing but
457 * has a easily detected signature in debug
458 * traces.
459 *
460 * On systems with MLOCK enforcement enabled,
461 * the above 0 word writes would fall foul of
462 * the enforcement. As such, in the first slot
463 * put a RESTART_W opcode to the beginning
464 * of the next job. We don't use this for older
465 * chips since those only support the RESTART
466 * opcode with inconvenient alignment requirements.
467 */
468 if (i == 0 && host1x->info->has_wide_gather) {
469 unsigned int next_job = (job->first_get/8 + job->num_slots)
470 % HOST1X_PUSHBUFFER_SLOTS;
471 mapped[2*slot+0] = (0xd << 28) | (next_job * 2);
472 mapped[2*slot+1] = 0x0;
473 } else {
474 mapped[2*slot+0] = 0x1bad0000;
475 mapped[2*slot+1] = 0x1bad0000;
476 }
477 }
478
479 job->cancelled = true;
480 }
481
482 wmb();
483
484 update_cdma_locked(cdma);
485 }
486
487resume:
488 /* roll back DMAGET and start up channel again */
489 host1x_hw_cdma_resume(host1x, cdma, restart_addr);
490}
491
492static void cdma_update_work(struct work_struct *work)
493{
494 struct host1x_cdma *cdma = container_of(work, struct host1x_cdma, update_work);
495
496 mutex_lock(&cdma->lock);
497 update_cdma_locked(cdma);
498 mutex_unlock(&cdma->lock);
499}
500
501/*
502 * Create a cdma
503 */
504int host1x_cdma_init(struct host1x_cdma *cdma)
505{
506 int err;
507
508 mutex_init(&cdma->lock);
509 init_completion(&cdma->complete);
510 INIT_WORK(&cdma->update_work, cdma_update_work);
511
512 INIT_LIST_HEAD(&cdma->sync_queue);
513
514 cdma->event = CDMA_EVENT_NONE;
515 cdma->running = false;
516 cdma->torndown = false;
517
518 err = host1x_pushbuffer_init(&cdma->push_buffer);
519 if (err)
520 return err;
521
522 return 0;
523}
524
525/*
526 * Destroy a cdma
527 */
528int host1x_cdma_deinit(struct host1x_cdma *cdma)
529{
530 struct push_buffer *pb = &cdma->push_buffer;
531 struct host1x *host1x = cdma_to_host1x(cdma);
532
533 if (cdma->running) {
534 pr_warn("%s: CDMA still running\n", __func__);
535 return -EBUSY;
536 }
537
538 host1x_pushbuffer_destroy(pb);
539 host1x_hw_cdma_timeout_destroy(host1x, cdma);
540
541 return 0;
542}
543
544/*
545 * Begin a cdma submit
546 */
547int host1x_cdma_begin(struct host1x_cdma *cdma, struct host1x_job *job)
548{
549 struct host1x *host1x = cdma_to_host1x(cdma);
550
551 mutex_lock(&cdma->lock);
552
553 /*
554 * Check if syncpoint was locked due to previous job timeout.
555 * This needs to be done within the cdma lock to avoid a race
556 * with the timeout handler.
557 */
558 if (job->syncpt->locked) {
559 mutex_unlock(&cdma->lock);
560 return -EPERM;
561 }
562
563 if (job->timeout) {
564 /* init state on first submit with timeout value */
565 if (!cdma->timeout.initialized) {
566 int err;
567
568 err = host1x_hw_cdma_timeout_init(host1x, cdma);
569 if (err) {
570 mutex_unlock(&cdma->lock);
571 return err;
572 }
573 }
574 }
575
576 if (!cdma->running)
577 host1x_hw_cdma_start(host1x, cdma);
578
579 cdma->slots_free = 0;
580 cdma->slots_used = 0;
581 cdma->first_get = cdma->push_buffer.pos;
582
583 trace_host1x_cdma_begin(dev_name(job->channel->dev));
584 return 0;
585}
586
587/*
588 * Push two words into a push buffer slot
589 * Blocks as necessary if the push buffer is full.
590 */
591void host1x_cdma_push(struct host1x_cdma *cdma, u32 op1, u32 op2)
592{
593 struct push_buffer *pb = &cdma->push_buffer;
594 u32 slots_free = cdma->slots_free;
595
596 if (host1x_debug_trace_cmdbuf)
597 trace_host1x_cdma_push(dev_name(cdma_to_channel(cdma)->dev),
598 op1, op2);
599
600 if (slots_free == 0)
601 slots_free = host1x_cdma_wait_locked(cdma,
602 CDMA_EVENT_PUSH_BUFFER_SPACE);
603
604 cdma->slots_free = slots_free - 1;
605 cdma->slots_used++;
606 host1x_pushbuffer_push(pb, op1, op2);
607}
608
609/*
610 * Push four words into two consecutive push buffer slots. Note that extra
611 * care needs to be taken not to split the two slots across the end of the
612 * push buffer. Otherwise the RESTART opcode at the end of the push buffer
613 * that ensures processing will restart at the beginning will break up the
614 * four words.
615 *
616 * Blocks as necessary if the push buffer is full.
617 */
618void host1x_cdma_push_wide(struct host1x_cdma *cdma, u32 op1, u32 op2,
619 u32 op3, u32 op4)
620{
621 struct host1x_channel *channel = cdma_to_channel(cdma);
622 struct host1x *host1x = cdma_to_host1x(cdma);
623 struct push_buffer *pb = &cdma->push_buffer;
624 unsigned int space, needed = 2, extra = 0;
625
626 if (host1x_debug_trace_cmdbuf)
627 trace_host1x_cdma_push_wide(dev_name(channel->dev), op1, op2,
628 op3, op4);
629
630 /* compute number of extra slots needed for padding */
631 if (pb->pos + 16 > pb->size) {
632 extra = (pb->size - pb->pos) / 8;
633 needed += extra;
634 }
635
636 host1x_cdma_wait_pushbuffer_space(host1x, cdma, needed);
637 space = host1x_pushbuffer_space(pb);
638
639 cdma->slots_free = space - needed;
640 cdma->slots_used += needed;
641
642 if (extra > 0) {
643 /*
644 * If there isn't enough space at the tail of the pushbuffer,
645 * insert a RESTART(0) here to go back to the beginning.
646 * The code above adjusted the indexes appropriately.
647 */
648 host1x_pushbuffer_push(pb, (0x5 << 28), 0xdead0000);
649 }
650
651 host1x_pushbuffer_push(pb, op1, op2);
652 host1x_pushbuffer_push(pb, op3, op4);
653}
654
655/*
656 * End a cdma submit
657 * Kick off DMA, add job to the sync queue, and a number of slots to be freed
658 * from the pushbuffer. The handles for a submit must all be pinned at the same
659 * time, but they can be unpinned in smaller chunks.
660 */
661void host1x_cdma_end(struct host1x_cdma *cdma,
662 struct host1x_job *job)
663{
664 struct host1x *host1x = cdma_to_host1x(cdma);
665 bool idle = list_empty(&cdma->sync_queue);
666
667 host1x_hw_cdma_flush(host1x, cdma);
668
669 job->first_get = cdma->first_get;
670 job->num_slots = cdma->slots_used;
671 host1x_job_get(job);
672 list_add_tail(&job->list, &cdma->sync_queue);
673
674 /* start timer on idle -> active transitions */
675 if (job->timeout && idle)
676 cdma_start_timer_locked(cdma, job);
677
678 trace_host1x_cdma_end(dev_name(job->channel->dev));
679 mutex_unlock(&cdma->lock);
680}
681
682/*
683 * Update cdma state according to current sync point values
684 */
685void host1x_cdma_update(struct host1x_cdma *cdma)
686{
687 schedule_work(&cdma->update_work);
688}