projects/linux·blob master · refs

1512 lines · 44.0 KiB

1// SPDX-License-Identifier: GPL-2.0-only OR MIT
2/* Copyright (c) 2023 Imagination Technologies Ltd. */
3
4#include <drm/drm_managed.h>
5#include <drm/gpu_scheduler.h>
6#include <linux/overflow.h>
7
8#include "pvr_cccb.h"
9#include "pvr_context.h"
10#include "pvr_device.h"
11#include "pvr_drv.h"
12#include "pvr_job.h"
13#include "pvr_queue.h"
14#include "pvr_trace.h"
15#include "pvr_vm.h"
16
17#include "pvr_rogue_fwif_client.h"
18
19#define MAX_DEADLINE_MS 30000
20#define SCHED_TIMEOUT_PERIOD (2 * MAX_DEADLINE_MS)
21
22#define CTX_COMPUTE_CCCB_SIZE_LOG2 15
23#define CTX_FRAG_CCCB_SIZE_LOG2 15
24#define CTX_GEOM_CCCB_SIZE_LOG2 15
25#define CTX_TRANSFER_CCCB_SIZE_LOG2 15
26
27static int get_xfer_ctx_state_size(struct pvr_device *pvr_dev)
28{
29 u32 num_isp_store_registers;
30
31 if (PVR_HAS_FEATURE(pvr_dev, xe_memory_hierarchy)) {
32 num_isp_store_registers = 1;
33 } else {
34 int err;
35
36 err = PVR_FEATURE_VALUE(pvr_dev, num_isp_ipp_pipes, &num_isp_store_registers);
37 if (WARN_ON(err))
38 return err;
39 }
40
41 return struct_size_t(struct rogue_fwif_frag_ctx_state,
42 frag_reg_isp_store, num_isp_store_registers);
43}
44
45static int get_frag_ctx_state_size(struct pvr_device *pvr_dev)
46{
47 u32 num_isp_store_registers;
48 int err;
49
50 if (PVR_HAS_FEATURE(pvr_dev, xe_memory_hierarchy)) {
51 err = PVR_FEATURE_VALUE(pvr_dev, num_raster_pipes, &num_isp_store_registers);
52 if (WARN_ON(err))
53 return err;
54
55 if (PVR_HAS_FEATURE(pvr_dev, gpu_multicore_support)) {
56 u32 xpu_max_slaves;
57
58 err = PVR_FEATURE_VALUE(pvr_dev, xpu_max_slaves, &xpu_max_slaves);
59 if (WARN_ON(err))
60 return err;
61
62 num_isp_store_registers *= (1 + xpu_max_slaves);
63 }
64 } else {
65 err = PVR_FEATURE_VALUE(pvr_dev, num_isp_ipp_pipes, &num_isp_store_registers);
66 if (WARN_ON(err))
67 return err;
68 }
69
70 return struct_size_t(struct rogue_fwif_frag_ctx_state,
71 frag_reg_isp_store, num_isp_store_registers);
72}
73
74static int get_ctx_state_size(struct pvr_device *pvr_dev, enum drm_pvr_job_type type)
75{
76 switch (type) {
77 case DRM_PVR_JOB_TYPE_GEOMETRY:
78 return sizeof(struct rogue_fwif_geom_ctx_state);
79 case DRM_PVR_JOB_TYPE_FRAGMENT:
80 return get_frag_ctx_state_size(pvr_dev);
81 case DRM_PVR_JOB_TYPE_COMPUTE:
82 return sizeof(struct rogue_fwif_compute_ctx_state);
83 case DRM_PVR_JOB_TYPE_TRANSFER_FRAG:
84 return get_xfer_ctx_state_size(pvr_dev);
85 }
86
87 WARN(1, "Invalid queue type");
88 return -EINVAL;
89}
90
91static u32 get_ctx_offset(enum drm_pvr_job_type type)
92{
93 switch (type) {
94 case DRM_PVR_JOB_TYPE_GEOMETRY:
95 return offsetof(struct rogue_fwif_fwrendercontext, geom_context);
96 case DRM_PVR_JOB_TYPE_FRAGMENT:
97 return offsetof(struct rogue_fwif_fwrendercontext, frag_context);
98 case DRM_PVR_JOB_TYPE_COMPUTE:
99 return offsetof(struct rogue_fwif_fwcomputecontext, cdm_context);
100 case DRM_PVR_JOB_TYPE_TRANSFER_FRAG:
101 return offsetof(struct rogue_fwif_fwtransfercontext, tq_context);
102 }
103
104 return 0;
105}
106
107static const char *
108pvr_queue_fence_get_driver_name(struct dma_fence *f)
109{
110 return PVR_DRIVER_NAME;
111}
112
113static void pvr_queue_fence_release_work(struct work_struct *w)
114{
115 struct pvr_queue_fence *fence = container_of(w, struct pvr_queue_fence, release_work);
116
117 pvr_context_put(fence->queue->ctx);
118 dma_fence_free(&fence->base);
119}
120
121static void pvr_queue_fence_release(struct dma_fence *f)
122{
123 struct pvr_queue_fence *fence = container_of(f, struct pvr_queue_fence, base);
124 struct pvr_device *pvr_dev = fence->queue->ctx->pvr_dev;
125
126 queue_work(pvr_dev->sched_wq, &fence->release_work);
127}
128
129static const char *
130pvr_queue_job_fence_get_timeline_name(struct dma_fence *f)
131{
132 struct pvr_queue_fence *fence = container_of(f, struct pvr_queue_fence, base);
133
134 switch (fence->queue->type) {
135 case DRM_PVR_JOB_TYPE_GEOMETRY:
136 return "geometry";
137
138 case DRM_PVR_JOB_TYPE_FRAGMENT:
139 return "fragment";
140
141 case DRM_PVR_JOB_TYPE_COMPUTE:
142 return "compute";
143
144 case DRM_PVR_JOB_TYPE_TRANSFER_FRAG:
145 return "transfer";
146 }
147
148 WARN(1, "Invalid queue type");
149 return "invalid";
150}
151
152static const char *
153pvr_queue_cccb_fence_get_timeline_name(struct dma_fence *f)
154{
155 struct pvr_queue_fence *fence = container_of(f, struct pvr_queue_fence, base);
156
157 switch (fence->queue->type) {
158 case DRM_PVR_JOB_TYPE_GEOMETRY:
159 return "geometry-cccb";
160
161 case DRM_PVR_JOB_TYPE_FRAGMENT:
162 return "fragment-cccb";
163
164 case DRM_PVR_JOB_TYPE_COMPUTE:
165 return "compute-cccb";
166
167 case DRM_PVR_JOB_TYPE_TRANSFER_FRAG:
168 return "transfer-cccb";
169 }
170
171 WARN(1, "Invalid queue type");
172 return "invalid";
173}
174
175static const struct dma_fence_ops pvr_queue_job_fence_ops = {
176 .get_driver_name = pvr_queue_fence_get_driver_name,
177 .get_timeline_name = pvr_queue_job_fence_get_timeline_name,
178 .release = pvr_queue_fence_release,
179};
180
181/**
182 * pvr_queue_fence_is_ufo_backed() - Check if a dma_fence is backed by a UFO.
183 * @f: The dma_fence to check.
184 *
185 * Return:
186 * * true if the dma_fence is backed by a UFO, or
187 * * false otherwise.
188 */
189static inline bool
190pvr_queue_fence_is_ufo_backed(struct dma_fence *f)
191{
192 /*
193 * Currently the only dma_fence backed by a UFO object is the job fence,
194 * e.g. pvr_job::done_fence, wrapped by a pvr_queue_fence object.
195 */
196 return f && f->ops == &pvr_queue_job_fence_ops;
197}
198
199/**
200 * to_pvr_queue_job_fence() - Return a pvr_queue_fence object if the fence is
201 * already backed by a UFO.
202 * @f: The dma_fence to turn into a pvr_queue_fence.
203 *
204 * This could be called on:
205 * - a job fence directly, in which case it simply returns the containing pvr_queue_fence;
206 * - a drm_sched_fence's scheduled or finished fence, in which case it will first try to follow
207 * the parent pointer to find the job fence (note that the parent pointer is initialized
208 * only after the run_job() callback is called on the drm_sched_fence's owning job);
209 * - any other dma_fence, in which case it will return NULL.
210 *
211 * Return:
212 * * A non-NULL pvr_queue_fence object if the dma_fence is backed by a UFO, or
213 * * NULL otherwise.
214 */
215static struct pvr_queue_fence *
216to_pvr_queue_job_fence(struct dma_fence *f)
217{
218 struct drm_sched_fence *sched_fence = to_drm_sched_fence(f);
219
220 if (sched_fence)
221 f = sched_fence->parent;
222
223 if (pvr_queue_fence_is_ufo_backed(f))
224 return container_of(f, struct pvr_queue_fence, base);
225
226 return NULL;
227}
228
229static const struct dma_fence_ops pvr_queue_cccb_fence_ops = {
230 .get_driver_name = pvr_queue_fence_get_driver_name,
231 .get_timeline_name = pvr_queue_cccb_fence_get_timeline_name,
232 .release = pvr_queue_fence_release,
233};
234
235/**
236 * pvr_queue_fence_put() - Put wrapper for pvr_queue_fence objects.
237 * @f: The dma_fence object to put.
238 *
239 * If the pvr_queue_fence has been initialized, we call dma_fence_put(),
240 * otherwise we free the object with dma_fence_free(). This allows us
241 * to do the right thing before and after pvr_queue_fence_init() had been
242 * called.
243 */
244static void pvr_queue_fence_put(struct dma_fence *f)
245{
246 if (!f)
247 return;
248
249 if (WARN_ON(f->ops &&
250 f->ops != &pvr_queue_cccb_fence_ops &&
251 f->ops != &pvr_queue_job_fence_ops))
252 return;
253
254 /* If the fence hasn't been initialized yet, free the object directly. */
255 if (f->ops)
256 dma_fence_put(f);
257 else
258 dma_fence_free(f);
259}
260
261/**
262 * pvr_queue_fence_alloc() - Allocate a pvr_queue_fence fence object
263 *
264 * Call this function to allocate job CCCB and done fences. This only
265 * allocates the objects. Initialization happens when the underlying
266 * dma_fence object is to be returned to drm_sched (in prepare_job() or
267 * run_job()).
268 *
269 * Return:
270 * * A valid pointer if the allocation succeeds, or
271 * * NULL if the allocation fails.
272 */
273static struct dma_fence *
274pvr_queue_fence_alloc(void)
275{
276 struct pvr_queue_fence *fence;
277
278 fence = kzalloc_obj(*fence);
279 if (!fence)
280 return NULL;
281
282 return &fence->base;
283}
284
285/**
286 * pvr_queue_fence_init() - Initializes a pvr_queue_fence object.
287 * @f: The fence to initialize
288 * @queue: The queue this fence belongs to.
289 * @fence_ops: The fence operations.
290 * @fence_ctx: The fence context.
291 *
292 * Wrapper around dma_fence_init() that takes care of initializing the
293 * pvr_queue_fence::queue field too.
294 */
295static void
296pvr_queue_fence_init(struct dma_fence *f,
297 struct pvr_queue *queue,
298 const struct dma_fence_ops *fence_ops,
299 struct pvr_queue_fence_ctx *fence_ctx)
300{
301 struct pvr_queue_fence *fence = container_of(f, struct pvr_queue_fence, base);
302
303 pvr_context_get(queue->ctx);
304 fence->queue = queue;
305 INIT_WORK(&fence->release_work, pvr_queue_fence_release_work);
306 dma_fence_init(&fence->base, fence_ops,
307 &fence_ctx->lock, fence_ctx->id,
308 atomic_inc_return(&fence_ctx->seqno));
309}
310
311/**
312 * pvr_queue_cccb_fence_init() - Initializes a CCCB fence object.
313 * @fence: The fence to initialize.
314 * @queue: The queue this fence belongs to.
315 *
316 * Initializes a fence that can be used to wait for CCCB space.
317 *
318 * Should be called in the ::prepare_job() path, so the fence returned to
319 * drm_sched is valid.
320 */
321static void
322pvr_queue_cccb_fence_init(struct dma_fence *fence, struct pvr_queue *queue)
323{
324 pvr_queue_fence_init(fence, queue, &pvr_queue_cccb_fence_ops,
325 &queue->cccb_fence_ctx.base);
326}
327
328/**
329 * pvr_queue_job_fence_init() - Initializes a job done fence object.
330 * @fence: The fence to initialize.
331 * @queue: The queue this fence belongs to.
332 *
333 * Initializes a fence that will be signaled when the GPU is done executing
334 * a job.
335 *
336 * Should be called *before* the ::run_job() path, so the fence is initialised
337 * before being placed in the pending_list.
338 */
339static void
340pvr_queue_job_fence_init(struct dma_fence *fence, struct pvr_queue *queue)
341{
342 if (!fence->ops)
343 pvr_queue_fence_init(fence, queue, &pvr_queue_job_fence_ops,
344 &queue->job_fence_ctx);
345}
346
347/**
348 * pvr_queue_fence_ctx_init() - Queue fence context initialization.
349 * @fence_ctx: The context to initialize
350 */
351static void
352pvr_queue_fence_ctx_init(struct pvr_queue_fence_ctx *fence_ctx)
353{
354 spin_lock_init(&fence_ctx->lock);
355 fence_ctx->id = dma_fence_context_alloc(1);
356 atomic_set(&fence_ctx->seqno, 0);
357}
358
359static u32 ufo_cmds_size(u32 elem_count)
360{
361 /* We can pass at most ROGUE_FWIF_CCB_CMD_MAX_UFOS per UFO-related command. */
362 u32 full_cmd_count = elem_count / ROGUE_FWIF_CCB_CMD_MAX_UFOS;
363 u32 remaining_elems = elem_count % ROGUE_FWIF_CCB_CMD_MAX_UFOS;
364 u32 size = full_cmd_count *
365 pvr_cccb_get_size_of_cmd_with_hdr(ROGUE_FWIF_CCB_CMD_MAX_UFOS *
366 sizeof(struct rogue_fwif_ufo));
367
368 if (remaining_elems) {
369 size += pvr_cccb_get_size_of_cmd_with_hdr(remaining_elems *
370 sizeof(struct rogue_fwif_ufo));
371 }
372
373 return size;
374}
375
376static u32 job_cmds_size(struct pvr_job *job, u32 ufo_wait_count)
377{
378 /*
379 * One UFO command per native fence this job will be waiting on (unless any are
380 * signaled by the time the job is submitted), plus a command for the job itself,
381 * plus one UFO command for the fence signaling.
382 */
383 return ufo_cmds_size(ufo_wait_count) +
384 pvr_cccb_get_size_of_cmd_with_hdr(job->cmd_len) +
385 ufo_cmds_size(1);
386}
387
388static bool
389is_paired_job_fence(struct dma_fence *fence, struct pvr_job *job)
390{
391 /* This assumes "fence" is one of "job"'s drm_sched_job::dependencies */
392 return job->type == DRM_PVR_JOB_TYPE_FRAGMENT &&
393 job->paired_job &&
394 &job->paired_job->base.s_fence->scheduled == fence;
395}
396
397/**
398 * job_count_remaining_native_deps() - Count the number of non-signaled native dependencies.
399 * @job: Job to operate on.
400 *
401 * Returns: Number of non-signaled native deps remaining.
402 */
403static unsigned long job_count_remaining_native_deps(struct pvr_job *job)
404{
405 unsigned long remaining_count = 0;
406 struct dma_fence *fence = NULL;
407 unsigned long index;
408
409 xa_for_each(&job->base.dependencies, index, fence) {
410 struct pvr_queue_fence *jfence;
411
412 if (is_paired_job_fence(fence, job)) {
413 /*
414 * A fence between paired jobs won't resolve to a pvr_queue_fence (i.e.
415 * be backed by a UFO) until the jobs have been submitted, together.
416 * The submitting code will insert a partial render fence command for this.
417 */
418 WARN_ON(dma_fence_is_signaled(fence));
419 remaining_count++;
420 continue;
421 }
422
423 jfence = to_pvr_queue_job_fence(fence);
424 if (!jfence)
425 continue;
426
427 if (!dma_fence_is_signaled(&jfence->base))
428 remaining_count++;
429 }
430
431 return remaining_count;
432}
433
434/**
435 * pvr_queue_get_job_cccb_fence() - Get the CCCB fence attached to a job.
436 * @queue: The queue this job will be submitted to.
437 * @job: The job to get the CCCB fence on.
438 *
439 * The CCCB fence is a synchronization primitive allowing us to delay job
440 * submission until there's enough space in the CCCB to submit the job.
441 *
442 * Return:
443 * * NULL if there's enough space in the CCCB to submit this job, or
444 * * A valid dma_fence object otherwise.
445 */
446static struct dma_fence *
447pvr_queue_get_job_cccb_fence(struct pvr_queue *queue, struct pvr_job *job)
448{
449 struct pvr_queue_fence *cccb_fence;
450 unsigned int native_deps_remaining;
451
452 /* If the fence is NULL, that means we already checked that we had
453 * enough space in the cccb for our job.
454 */
455 if (!job->cccb_fence)
456 return NULL;
457
458 mutex_lock(&queue->cccb_fence_ctx.job_lock);
459
460 /* Count remaining native dependencies and check if the job fits in the CCCB. */
461 native_deps_remaining = job_count_remaining_native_deps(job);
462 if (pvr_cccb_cmdseq_fits(&queue->cccb, job_cmds_size(job, native_deps_remaining))) {
463 pvr_queue_fence_put(job->cccb_fence);
464 job->cccb_fence = NULL;
465 goto out_unlock;
466 }
467
468 /* There should be no job attached to the CCCB fence context:
469 * drm_sched_entity guarantees that jobs are submitted one at a time.
470 */
471 if (WARN_ON(queue->cccb_fence_ctx.job))
472 pvr_job_put(queue->cccb_fence_ctx.job);
473
474 queue->cccb_fence_ctx.job = pvr_job_get(job);
475
476 /* Initialize the fence before returning it. */
477 cccb_fence = container_of(job->cccb_fence, struct pvr_queue_fence, base);
478 if (!WARN_ON(cccb_fence->queue))
479 pvr_queue_cccb_fence_init(job->cccb_fence, queue);
480
481out_unlock:
482 mutex_unlock(&queue->cccb_fence_ctx.job_lock);
483
484 return dma_fence_get(job->cccb_fence);
485}
486
487/**
488 * pvr_queue_get_job_kccb_fence() - Get the KCCB fence attached to a job.
489 * @queue: The queue this job will be submitted to.
490 * @job: The job to get the KCCB fence on.
491 *
492 * The KCCB fence is a synchronization primitive allowing us to delay job
493 * submission until there's enough space in the KCCB to submit the job.
494 *
495 * Return:
496 * * NULL if there's enough space in the KCCB to submit this job, or
497 * * A valid dma_fence object otherwise.
498 */
499static struct dma_fence *
500pvr_queue_get_job_kccb_fence(struct pvr_queue *queue, struct pvr_job *job)
501{
502 struct pvr_device *pvr_dev = queue->ctx->pvr_dev;
503 struct dma_fence *kccb_fence = NULL;
504
505 /* If the fence is NULL, that means we already checked that we had
506 * enough space in the KCCB for our job.
507 */
508 if (!job->kccb_fence)
509 return NULL;
510
511 if (!WARN_ON(job->kccb_fence->ops)) {
512 kccb_fence = pvr_kccb_reserve_slot(pvr_dev, job->kccb_fence);
513 job->kccb_fence = NULL;
514 }
515
516 return kccb_fence;
517}
518
519static struct dma_fence *
520pvr_queue_get_paired_frag_job_dep(struct pvr_job *job)
521{
522 struct pvr_job *frag_job = job->type == DRM_PVR_JOB_TYPE_GEOMETRY ?
523 job->paired_job : NULL;
524 struct pvr_queue *frag_queue = frag_job ? frag_job->ctx->queues.fragment : NULL;
525 struct dma_fence *f;
526 unsigned long index;
527
528 if (!frag_job)
529 return NULL;
530
531 /* Have the geometry job wait on the paired fragment job's dependencies as well. */
532 xa_for_each(&frag_job->base.dependencies, index, f) {
533 /* Skip already signaled fences. */
534 if (dma_fence_is_signaled(f))
535 continue;
536
537 /*
538 * The paired job fence won't be signaled until both jobs have
539 * been submitted, so we can't wait on it to schedule them.
540 */
541 if (f == &job->base.s_fence->scheduled)
542 continue;
543
544 return dma_fence_get(f);
545 }
546
547 /* Initialize the paired fragment job's done_fence, so we can signal it. */
548 pvr_queue_job_fence_init(frag_job->done_fence, frag_queue);
549
550 return pvr_queue_get_job_cccb_fence(frag_queue, frag_job);
551}
552
553/**
554 * pvr_queue_prepare_job() - Return the next internal dependencies expressed as a dma_fence.
555 * @sched_job: The job to query the next internal dependency on
556 * @s_entity: The entity this job is queue on.
557 *
558 * After iterating over drm_sched_job::dependencies, drm_sched let the driver return
559 * its own internal dependencies. We use this function to return our internal dependencies.
560 */
561static struct dma_fence *
562pvr_queue_prepare_job(struct drm_sched_job *sched_job,
563 struct drm_sched_entity *s_entity)
564{
565 struct pvr_job *job = container_of(sched_job, struct pvr_job, base);
566 struct pvr_queue *queue = container_of(s_entity, struct pvr_queue, entity);
567 struct dma_fence *internal_dep = NULL;
568
569 if (job->type == DRM_PVR_JOB_TYPE_FRAGMENT && job->paired_job) {
570 /*
571 * This will be called on a paired fragment job after being submitted
572 * to the firmware as part of the paired geometry job's submission.
573 * We can tell if this is the case and bail early from whether run_job()
574 * has been called on the geometry job, which would issue a pm ref on
575 * this job as well.
576 */
577 if (job->has_pm_ref)
578 return NULL;
579 }
580
581 /*
582 * Initialize the done_fence, so we can signal it. This must be done
583 * here because otherwise by the time of run_job() the job will end up
584 * in the pending list without a valid fence.
585 */
586 pvr_queue_job_fence_init(job->done_fence, queue);
587
588 /* CCCB fence is used to make sure we have enough space in the CCCB to
589 * submit our commands.
590 */
591 internal_dep = pvr_queue_get_job_cccb_fence(queue, job);
592
593 /* KCCB fence is used to make sure we have a KCCB slot to queue our
594 * CMD_KICK.
595 */
596 if (!internal_dep)
597 internal_dep = pvr_queue_get_job_kccb_fence(queue, job);
598
599 /* Any extra internal dependency should be added here, using the following
600 * pattern:
601 *
602 * if (!internal_dep)
603 * internal_dep = pvr_queue_get_job_xxxx_fence(queue, job);
604 */
605
606 /* The paired job fence should come last, when everything else is ready. */
607 if (!internal_dep)
608 internal_dep = pvr_queue_get_paired_frag_job_dep(job);
609
610 return internal_dep;
611}
612
613/**
614 * pvr_queue_update_active_state_locked() - Update the queue active state.
615 * @queue: Queue to update the state on.
616 *
617 * Locked version of pvr_queue_update_active_state(). Must be called with
618 * pvr_device::queue::lock held.
619 */
620static void pvr_queue_update_active_state_locked(struct pvr_queue *queue)
621{
622 struct pvr_device *pvr_dev = queue->ctx->pvr_dev;
623
624 lockdep_assert_held(&pvr_dev->queues.lock);
625
626 /* The queue is temporary out of any list when it's being reset,
627 * we don't want a call to pvr_queue_update_active_state_locked()
628 * to re-insert it behind our back.
629 */
630 if (list_empty(&queue->node))
631 return;
632
633 if (!atomic_read(&queue->in_flight_job_count))
634 list_move_tail(&queue->node, &pvr_dev->queues.idle);
635 else
636 list_move_tail(&queue->node, &pvr_dev->queues.active);
637}
638
639/**
640 * pvr_queue_update_active_state() - Update the queue active state.
641 * @queue: Queue to update the state on.
642 *
643 * Active state is based on the in_flight_job_count value.
644 *
645 * Updating the active state implies moving the queue in or out of the
646 * active queue list, which also defines whether the queue is checked
647 * or not when a FW event is received.
648 *
649 * This function should be called any time a job is submitted or it done
650 * fence is signaled.
651 */
652static void pvr_queue_update_active_state(struct pvr_queue *queue)
653{
654 struct pvr_device *pvr_dev = queue->ctx->pvr_dev;
655
656 mutex_lock(&pvr_dev->queues.lock);
657 pvr_queue_update_active_state_locked(queue);
658 mutex_unlock(&pvr_dev->queues.lock);
659}
660
661static void pvr_queue_submit_job_to_cccb(struct pvr_job *job)
662{
663 struct pvr_queue *queue = container_of(job->base.sched, struct pvr_queue, scheduler);
664 struct rogue_fwif_ufo ufos[ROGUE_FWIF_CCB_CMD_MAX_UFOS];
665 struct pvr_cccb *cccb = &queue->cccb;
666 struct pvr_queue_fence *jfence;
667 struct dma_fence *fence;
668 unsigned long index;
669 u32 ufo_count = 0;
670
671 /* We need to add the queue to the active list before updating the CCCB,
672 * otherwise we might miss the FW event informing us that something
673 * happened on this queue.
674 */
675 atomic_inc(&queue->in_flight_job_count);
676 pvr_queue_update_active_state(queue);
677
678 xa_for_each(&job->base.dependencies, index, fence) {
679 jfence = to_pvr_queue_job_fence(fence);
680 if (!jfence)
681 continue;
682
683 /* Some dependencies might have been signaled since prepare_job() */
684 if (dma_fence_is_signaled(&jfence->base))
685 continue;
686
687 pvr_fw_object_get_fw_addr(jfence->queue->timeline_ufo.fw_obj,
688 &ufos[ufo_count].addr);
689 ufos[ufo_count++].value = jfence->base.seqno;
690
691 if (ufo_count == ARRAY_SIZE(ufos)) {
692 pvr_cccb_write_command_with_header(cccb, ROGUE_FWIF_CCB_CMD_TYPE_FENCE_PR,
693 sizeof(ufos), ufos, 0, 0);
694 ufo_count = 0;
695 }
696 }
697
698 if (job->type == DRM_PVR_JOB_TYPE_FRAGMENT && job->paired_job) {
699 /*
700 * The loop above will only process dependencies backed by a UFO i.e. with
701 * a valid parent fence assigned, but the paired job dependency won't have
702 * one until both jobs have been submitted. Access the parent fence directly
703 * here instead, submitting it last as partial render fence.
704 */
705 jfence = to_pvr_queue_job_fence(job->paired_job->done_fence);
706 if (!WARN_ON(!jfence)) {
707 pvr_fw_object_get_fw_addr(jfence->queue->timeline_ufo.fw_obj,
708 &ufos[ufo_count].addr);
709 ufos[ufo_count++].value = job->paired_job->done_fence->seqno;
710 }
711 }
712
713 if (ufo_count) {
714 pvr_cccb_write_command_with_header(cccb, ROGUE_FWIF_CCB_CMD_TYPE_FENCE_PR,
715 sizeof(ufos[0]) * ufo_count, ufos, 0, 0);
716 }
717
718 if (job->type == DRM_PVR_JOB_TYPE_GEOMETRY && job->paired_job) {
719 struct rogue_fwif_cmd_geom *cmd = job->cmd;
720
721 /* Reference value for the partial render test is the current queue fence
722 * seqno minus one.
723 */
724 pvr_fw_object_get_fw_addr(queue->timeline_ufo.fw_obj,
725 &cmd->partial_render_geom_frag_fence.addr);
726 cmd->partial_render_geom_frag_fence.value = job->done_fence->seqno - 1;
727 }
728
729 /* Submit job to FW */
730 pvr_cccb_write_command_with_header(cccb, job->fw_ccb_cmd_type, job->cmd_len, job->cmd,
731 job->id, job->id);
732
733 /* Update command to signal the job fence. */
734 pvr_fw_object_get_fw_addr(queue->timeline_ufo.fw_obj, &ufos[0].addr);
735 ufos[0].value = job->done_fence->seqno;
736 pvr_cccb_write_command_with_header(cccb, ROGUE_FWIF_CCB_CMD_TYPE_UPDATE,
737 sizeof(ufos[0]), ufos, 0, 0);
738}
739
740/**
741 * pvr_queue_run_job() - Submit a job to the FW.
742 * @sched_job: The job to submit.
743 *
744 * This function is called when all non-native dependencies have been met and
745 * when the commands resulting from this job are guaranteed to fit in the CCCB.
746 */
747static struct dma_fence *pvr_queue_run_job(struct drm_sched_job *sched_job)
748{
749 struct pvr_job *job = container_of(sched_job, struct pvr_job, base);
750 struct pvr_device *pvr_dev = job->pvr_dev;
751 int err;
752
753 /* The fragment job is issued along the geometry job when we use combined
754 * geom+frag kicks. When we get there, we should simply return the
755 * done_fence that's been initialized earlier.
756 */
757 if (job->paired_job && job->type == DRM_PVR_JOB_TYPE_FRAGMENT &&
758 job->done_fence->ops) {
759 return dma_fence_get(job->done_fence);
760 }
761
762 /* The only kind of jobs that can be paired are geometry and fragment, and
763 * we bail out early if we see a fragment job that's paired with a geometry job.
764 * Paired jobs must also target the same context and point to the same HWRT.
765 */
766 if (WARN_ON(job->paired_job &&
767 (job->type != DRM_PVR_JOB_TYPE_GEOMETRY ||
768 job->paired_job->type != DRM_PVR_JOB_TYPE_FRAGMENT ||
769 job->hwrt != job->paired_job->hwrt ||
770 job->ctx != job->paired_job->ctx)))
771 return ERR_PTR(-EINVAL);
772
773 err = pvr_job_get_pm_ref(job);
774 if (WARN_ON(err))
775 return ERR_PTR(err);
776
777 if (job->paired_job) {
778 err = pvr_job_get_pm_ref(job->paired_job);
779 if (WARN_ON(err))
780 return ERR_PTR(err);
781 }
782
783 /* Submit our job to the CCCB */
784 pvr_queue_submit_job_to_cccb(job);
785
786 if (job->paired_job) {
787 struct pvr_job *geom_job = job;
788 struct pvr_job *frag_job = job->paired_job;
789 struct pvr_queue *geom_queue = job->ctx->queues.geometry;
790 struct pvr_queue *frag_queue = job->ctx->queues.fragment;
791
792 /* Submit the fragment job along the geometry job and send a combined kick. */
793 pvr_queue_submit_job_to_cccb(frag_job);
794 pvr_cccb_send_kccb_combined_kick(pvr_dev,
795 &geom_queue->cccb, &frag_queue->cccb,
796 pvr_context_get_fw_addr(geom_job->ctx) +
797 geom_queue->ctx_offset,
798 pvr_context_get_fw_addr(frag_job->ctx) +
799 frag_queue->ctx_offset,
800 job->hwrt,
801 frag_job->fw_ccb_cmd_type ==
802 ROGUE_FWIF_CCB_CMD_TYPE_FRAG_PR);
803
804 trace_pvr_job_submit_fw(geom_job);
805 trace_pvr_job_submit_fw(frag_job);
806 } else {
807 struct pvr_queue *queue = container_of(job->base.sched,
808 struct pvr_queue, scheduler);
809
810 pvr_cccb_send_kccb_kick(pvr_dev, &queue->cccb,
811 pvr_context_get_fw_addr(job->ctx) + queue->ctx_offset,
812 job->hwrt);
813
814 trace_pvr_job_submit_fw(job);
815 }
816
817 return dma_fence_get(job->done_fence);
818}
819
820static void pvr_queue_stop(struct pvr_queue *queue, struct pvr_job *bad_job)
821{
822 drm_sched_stop(&queue->scheduler, bad_job ? &bad_job->base : NULL);
823}
824
825static void pvr_queue_start(struct pvr_queue *queue)
826{
827 struct pvr_job *job;
828
829 /* Make sure we CPU-signal the UFO object, so other queues don't get
830 * blocked waiting on it.
831 */
832 *queue->timeline_ufo.value = atomic_read(&queue->job_fence_ctx.seqno);
833
834 list_for_each_entry(job, &queue->scheduler.pending_list, base.list) {
835 if (dma_fence_is_signaled(job->done_fence)) {
836 /* Jobs might have completed after drm_sched_stop() was called.
837 * In that case, re-assign the parent field to the done_fence.
838 */
839 WARN_ON(job->base.s_fence->parent);
840 job->base.s_fence->parent = dma_fence_get(job->done_fence);
841 } else {
842 /* If we had unfinished jobs, flag the entity as guilty so no
843 * new job can be submitted.
844 */
845 atomic_set(&queue->ctx->faulty, 1);
846 }
847 }
848
849 drm_sched_start(&queue->scheduler, 0);
850}
851
852/**
853 * pvr_queue_timedout_job() - Handle a job timeout event.
854 * @s_job: The job this timeout occurred on.
855 *
856 * FIXME: We don't do anything here to unblock the situation, we just stop+start
857 * the scheduler, and re-assign parent fences in the middle.
858 *
859 * Return:
860 * *%DRM_GPU_SCHED_STAT_NO_HANG if the job fence has already been
861 * signaled, or
862 * *%DRM_GPU_SCHED_STAT_RESET otherwise.
863 */
864static enum drm_gpu_sched_stat
865pvr_queue_timedout_job(struct drm_sched_job *s_job)
866{
867 struct drm_gpu_scheduler *sched = s_job->sched;
868 struct pvr_queue *queue = container_of(sched, struct pvr_queue, scheduler);
869 struct pvr_device *pvr_dev = queue->ctx->pvr_dev;
870 struct pvr_job *job;
871 u32 job_count = 0;
872
873 if (dma_fence_is_signaled(s_job->s_fence->parent))
874 return DRM_GPU_SCHED_STAT_NO_HANG;
875
876 dev_err(sched->dev, "Job timeout\n");
877
878 /* Before we stop the scheduler, make sure the queue is out of any list, so
879 * any call to pvr_queue_update_active_state_locked() that might happen
880 * until the scheduler is really stopped doesn't end up re-inserting the
881 * queue in the active list. This would cause
882 * pvr_queue_signal_done_fences() and drm_sched_stop() to race with each
883 * other when accessing the pending_list, since drm_sched_stop() doesn't
884 * grab the job_list_lock when modifying the list (it's assuming the
885 * only other accessor is the scheduler, and it's safe to not grab the
886 * lock since it's stopped).
887 */
888 mutex_lock(&pvr_dev->queues.lock);
889 list_del_init(&queue->node);
890 mutex_unlock(&pvr_dev->queues.lock);
891
892 drm_sched_stop(sched, s_job);
893
894 /* Re-assign job parent fences. */
895 list_for_each_entry(job, &sched->pending_list, base.list) {
896 job->base.s_fence->parent = dma_fence_get(job->done_fence);
897 job_count++;
898 }
899 WARN_ON(atomic_read(&queue->in_flight_job_count) != job_count);
900
901 /* Re-insert the queue in the proper list, and kick a queue processing
902 * operation if there were jobs pending.
903 */
904 mutex_lock(&pvr_dev->queues.lock);
905 if (!job_count) {
906 list_move_tail(&queue->node, &pvr_dev->queues.idle);
907 } else {
908 atomic_set(&queue->in_flight_job_count, job_count);
909 list_move_tail(&queue->node, &pvr_dev->queues.active);
910 pvr_queue_process(queue);
911 }
912 mutex_unlock(&pvr_dev->queues.lock);
913
914 drm_sched_start(sched, 0);
915
916 return DRM_GPU_SCHED_STAT_RESET;
917}
918
919/**
920 * pvr_queue_free_job() - Release the reference the scheduler had on a job object.
921 * @sched_job: Job object to free.
922 */
923static void pvr_queue_free_job(struct drm_sched_job *sched_job)
924{
925 struct pvr_job *job = container_of(sched_job, struct pvr_job, base);
926
927 drm_sched_job_cleanup(sched_job);
928
929 if (job->type == DRM_PVR_JOB_TYPE_FRAGMENT && job->paired_job)
930 pvr_job_put(job->paired_job);
931
932 job->paired_job = NULL;
933 pvr_job_put(job);
934}
935
936static const struct drm_sched_backend_ops pvr_queue_sched_ops = {
937 .prepare_job = pvr_queue_prepare_job,
938 .run_job = pvr_queue_run_job,
939 .timedout_job = pvr_queue_timedout_job,
940 .free_job = pvr_queue_free_job,
941};
942
943/**
944 * pvr_queue_fence_is_native() - Check if a dma_fence is native to this driver.
945 * @f: Fence to test.
946 *
947 * Check if the fence we're being passed is a drm_sched_fence that is coming from this driver.
948 *
949 * It may be a UFO-backed fence i.e. a fence that can be signaled or waited upon FW-side,
950 * such as pvr_job::done_fence objects that are backed by the timeline UFO attached to the queue
951 * they are pushed to.
952 */
953bool pvr_queue_fence_is_native(struct dma_fence *f)
954{
955 struct drm_sched_fence *sched_fence = f ? to_drm_sched_fence(f) : NULL;
956
957 if (sched_fence &&
958 sched_fence->sched->ops == &pvr_queue_sched_ops)
959 return true;
960
961 return pvr_queue_fence_is_ufo_backed(f);
962}
963
964/**
965 * pvr_queue_signal_done_fences() - Signal done fences.
966 * @queue: Queue to check.
967 *
968 * Signal done fences of jobs whose seqno is less than the current value of
969 * the UFO object attached to the queue.
970 */
971static void
972pvr_queue_signal_done_fences(struct pvr_queue *queue)
973{
974 struct pvr_job *job, *tmp_job;
975 u32 cur_seqno;
976
977 spin_lock(&queue->scheduler.job_list_lock);
978 cur_seqno = *queue->timeline_ufo.value;
979 list_for_each_entry_safe(job, tmp_job, &queue->scheduler.pending_list, base.list) {
980 if ((int)(cur_seqno - lower_32_bits(job->done_fence->seqno)) < 0)
981 break;
982
983 if (!dma_fence_is_signaled(job->done_fence)) {
984 dma_fence_signal(job->done_fence);
985 pvr_job_release_pm_ref(job);
986 atomic_dec(&queue->in_flight_job_count);
987 }
988 }
989 spin_unlock(&queue->scheduler.job_list_lock);
990}
991
992/**
993 * pvr_queue_check_job_waiting_for_cccb_space() - Check if a job waiting for CCCB space
994 * can be unblocked and pushed to the CCCB.
995 * @queue: Queue to check
996 *
997 * If we have a job waiting for CCCB, and this job now fits in the CCCB, we signal
998 * its CCCB fence, which should kick drm_sched.
999 */
1000static void
1001pvr_queue_check_job_waiting_for_cccb_space(struct pvr_queue *queue)
1002{
1003 struct pvr_queue_fence *cccb_fence;
1004 u32 native_deps_remaining;
1005 struct pvr_job *job;
1006
1007 mutex_lock(&queue->cccb_fence_ctx.job_lock);
1008 job = queue->cccb_fence_ctx.job;
1009 if (!job)
1010 goto out_unlock;
1011
1012 /* If we have a job attached to the CCCB fence context, its CCCB fence
1013 * shouldn't be NULL.
1014 */
1015 if (WARN_ON(!job->cccb_fence)) {
1016 job = NULL;
1017 goto out_unlock;
1018 }
1019
1020 /* If we get there, CCCB fence has to be initialized. */
1021 cccb_fence = container_of(job->cccb_fence, struct pvr_queue_fence, base);
1022 if (WARN_ON(!cccb_fence->queue)) {
1023 job = NULL;
1024 goto out_unlock;
1025 }
1026
1027 /* Evict signaled dependencies before checking for CCCB space.
1028 * If the job fits, signal the CCCB fence, this should unblock
1029 * the drm_sched_entity.
1030 */
1031 native_deps_remaining = job_count_remaining_native_deps(job);
1032 if (!pvr_cccb_cmdseq_fits(&queue->cccb, job_cmds_size(job, native_deps_remaining))) {
1033 job = NULL;
1034 goto out_unlock;
1035 }
1036
1037 dma_fence_signal(job->cccb_fence);
1038 pvr_queue_fence_put(job->cccb_fence);
1039 job->cccb_fence = NULL;
1040 queue->cccb_fence_ctx.job = NULL;
1041
1042out_unlock:
1043 mutex_unlock(&queue->cccb_fence_ctx.job_lock);
1044
1045 pvr_job_put(job);
1046}
1047
1048/**
1049 * pvr_queue_process() - Process events that happened on a queue.
1050 * @queue: Queue to check
1051 *
1052 * Signal job fences and check if jobs waiting for CCCB space can be unblocked.
1053 */
1054void pvr_queue_process(struct pvr_queue *queue)
1055{
1056 lockdep_assert_held(&queue->ctx->pvr_dev->queues.lock);
1057
1058 pvr_queue_check_job_waiting_for_cccb_space(queue);
1059 pvr_queue_signal_done_fences(queue);
1060 pvr_queue_update_active_state_locked(queue);
1061}
1062
1063static u32 get_dm_type(struct pvr_queue *queue)
1064{
1065 switch (queue->type) {
1066 case DRM_PVR_JOB_TYPE_GEOMETRY:
1067 return PVR_FWIF_DM_GEOM;
1068 case DRM_PVR_JOB_TYPE_TRANSFER_FRAG:
1069 case DRM_PVR_JOB_TYPE_FRAGMENT:
1070 return PVR_FWIF_DM_FRAG;
1071 case DRM_PVR_JOB_TYPE_COMPUTE:
1072 return PVR_FWIF_DM_CDM;
1073 }
1074
1075 return ~0;
1076}
1077
1078/**
1079 * init_fw_context() - Initializes the queue part of a FW context.
1080 * @queue: Queue object to initialize the FW context for.
1081 * @fw_ctx_map: The FW context CPU mapping.
1082 *
1083 * FW contexts are containing various states, one of them being a per-queue state
1084 * that needs to be initialized for each queue being exposed by a context. This
1085 * function takes care of that.
1086 */
1087static void init_fw_context(struct pvr_queue *queue, void *fw_ctx_map)
1088{
1089 struct pvr_context *ctx = queue->ctx;
1090 struct pvr_fw_object *fw_mem_ctx_obj = pvr_vm_get_fw_mem_context(ctx->vm_ctx);
1091 struct rogue_fwif_fwcommoncontext *cctx_fw;
1092 struct pvr_cccb *cccb = &queue->cccb;
1093
1094 cctx_fw = fw_ctx_map + queue->ctx_offset;
1095 cctx_fw->ccbctl_fw_addr = cccb->ctrl_fw_addr;
1096 cctx_fw->ccb_fw_addr = cccb->cccb_fw_addr;
1097
1098 cctx_fw->dm = get_dm_type(queue);
1099 cctx_fw->priority = ctx->priority;
1100 cctx_fw->priority_seq_num = 0;
1101 cctx_fw->max_deadline_ms = MAX_DEADLINE_MS;
1102 cctx_fw->pid = task_tgid_nr(current);
1103 cctx_fw->server_common_context_id = ctx->ctx_id;
1104
1105 pvr_fw_object_get_fw_addr(fw_mem_ctx_obj, &cctx_fw->fw_mem_context_fw_addr);
1106
1107 pvr_fw_object_get_fw_addr(queue->reg_state_obj, &cctx_fw->context_state_addr);
1108}
1109
1110/**
1111 * pvr_queue_cleanup_fw_context() - Wait for the FW context to be idle and clean it up.
1112 * @queue: Queue on FW context to clean up.
1113 *
1114 * Return:
1115 * * 0 on success,
1116 * * Any error returned by pvr_fw_structure_cleanup() otherwise.
1117 */
1118static int pvr_queue_cleanup_fw_context(struct pvr_queue *queue)
1119{
1120 if (!queue->ctx->fw_obj)
1121 return 0;
1122
1123 return pvr_fw_structure_cleanup(queue->ctx->pvr_dev,
1124 ROGUE_FWIF_CLEANUP_FWCOMMONCONTEXT,
1125 queue->ctx->fw_obj, queue->ctx_offset);
1126}
1127
1128/**
1129 * pvr_queue_job_init() - Initialize queue related fields in a pvr_job object.
1130 * @job: The job to initialize.
1131 * @drm_client_id: drm_file.client_id submitting the job
1132 *
1133 * Bind the job to a queue and allocate memory to guarantee pvr_queue_job_arm()
1134 * and pvr_queue_job_push() can't fail. We also make sure the context type is
1135 * valid and the job can fit in the CCCB.
1136 *
1137 * Return:
1138 * * 0 on success, or
1139 * * An error code if something failed.
1140 */
1141int pvr_queue_job_init(struct pvr_job *job, u64 drm_client_id)
1142{
1143 /* Fragment jobs need at least one native fence wait on the geometry job fence. */
1144 u32 min_native_dep_count = job->type == DRM_PVR_JOB_TYPE_FRAGMENT ? 1 : 0;
1145 struct pvr_queue *queue;
1146 int err;
1147
1148 if (atomic_read(&job->ctx->faulty))
1149 return -EIO;
1150
1151 queue = pvr_context_get_queue_for_job(job->ctx, job->type);
1152 if (!queue)
1153 return -EINVAL;
1154
1155 if (!pvr_cccb_cmdseq_can_fit(&queue->cccb, job_cmds_size(job, min_native_dep_count)))
1156 return -E2BIG;
1157
1158 err = drm_sched_job_init(&job->base, &queue->entity, 1, THIS_MODULE, drm_client_id);
1159 if (err)
1160 return err;
1161
1162 job->cccb_fence = pvr_queue_fence_alloc();
1163 job->kccb_fence = pvr_kccb_fence_alloc();
1164 job->done_fence = pvr_queue_fence_alloc();
1165 if (!job->cccb_fence || !job->kccb_fence || !job->done_fence)
1166 return -ENOMEM;
1167
1168 return 0;
1169}
1170
1171/**
1172 * pvr_queue_job_arm() - Arm a job object.
1173 * @job: The job to arm.
1174 *
1175 * Initializes fences and return the drm_sched finished fence so it can
1176 * be exposed to the outside world. Once this function is called, you should
1177 * make sure the job is pushed using pvr_queue_job_push(), or guarantee that
1178 * no one grabbed a reference to the returned fence. The latter can happen if
1179 * we do multi-job submission, and something failed when creating/initializing
1180 * a job. In that case, we know the fence didn't leave the driver, and we
1181 * can thus guarantee nobody will wait on an dead fence object.
1182 *
1183 * Return:
1184 * * A dma_fence object.
1185 */
1186struct dma_fence *pvr_queue_job_arm(struct pvr_job *job)
1187{
1188 drm_sched_job_arm(&job->base);
1189
1190 return &job->base.s_fence->finished;
1191}
1192
1193/**
1194 * pvr_queue_job_cleanup() - Cleanup fence/scheduler related fields in the job object.
1195 * @job: The job to cleanup.
1196 *
1197 * Should be called in the job release path.
1198 */
1199void pvr_queue_job_cleanup(struct pvr_job *job)
1200{
1201 pvr_queue_fence_put(job->done_fence);
1202 pvr_queue_fence_put(job->cccb_fence);
1203 pvr_kccb_fence_put(job->kccb_fence);
1204
1205 if (job->base.s_fence)
1206 drm_sched_job_cleanup(&job->base);
1207
1208 trace_pvr_job_done(job);
1209}
1210
1211/**
1212 * pvr_queue_job_push() - Push a job to its queue.
1213 * @job: The job to push.
1214 *
1215 * Must be called after pvr_queue_job_init() and after all dependencies
1216 * have been added to the job. This will effectively queue the job to
1217 * the drm_sched_entity attached to the queue. We grab a reference on
1218 * the job object, so the caller is free to drop its reference when it's
1219 * done accessing the job object.
1220 */
1221void pvr_queue_job_push(struct pvr_job *job)
1222{
1223 struct pvr_queue *queue = container_of(job->base.sched, struct pvr_queue, scheduler);
1224
1225 /* Keep track of the last queued job scheduled fence for combined submit. */
1226 dma_fence_put(queue->last_queued_job_scheduled_fence);
1227 queue->last_queued_job_scheduled_fence = dma_fence_get(&job->base.s_fence->scheduled);
1228
1229 pvr_job_get(job);
1230 drm_sched_entity_push_job(&job->base);
1231}
1232
1233static void reg_state_init(void *cpu_ptr, void *priv)
1234{
1235 struct pvr_queue *queue = priv;
1236
1237 if (queue->type == DRM_PVR_JOB_TYPE_GEOMETRY) {
1238 struct rogue_fwif_geom_ctx_state *geom_ctx_state_fw = cpu_ptr;
1239
1240 geom_ctx_state_fw->geom_core[0].geom_reg_vdm_call_stack_pointer_init =
1241 queue->callstack_addr;
1242 }
1243}
1244
1245/**
1246 * pvr_queue_create() - Create a queue object.
1247 * @ctx: The context this queue will be attached to.
1248 * @type: The type of jobs being pushed to this queue.
1249 * @args: The arguments passed to the context creation function.
1250 * @fw_ctx_map: CPU mapping of the FW context object.
1251 *
1252 * Create a queue object that will be used to queue and track jobs.
1253 *
1254 * Return:
1255 * * A valid pointer to a pvr_queue object, or
1256 * * An error pointer if the creation/initialization failed.
1257 */
1258struct pvr_queue *pvr_queue_create(struct pvr_context *ctx,
1259 enum drm_pvr_job_type type,
1260 struct drm_pvr_ioctl_create_context_args *args,
1261 void *fw_ctx_map)
1262{
1263 static const struct {
1264 u32 cccb_size;
1265 const char *name;
1266 } props[] = {
1267 [DRM_PVR_JOB_TYPE_GEOMETRY] = {
1268 .cccb_size = CTX_GEOM_CCCB_SIZE_LOG2,
1269 .name = "geometry",
1270 },
1271 [DRM_PVR_JOB_TYPE_FRAGMENT] = {
1272 .cccb_size = CTX_FRAG_CCCB_SIZE_LOG2,
1273 .name = "fragment"
1274 },
1275 [DRM_PVR_JOB_TYPE_COMPUTE] = {
1276 .cccb_size = CTX_COMPUTE_CCCB_SIZE_LOG2,
1277 .name = "compute"
1278 },
1279 [DRM_PVR_JOB_TYPE_TRANSFER_FRAG] = {
1280 .cccb_size = CTX_TRANSFER_CCCB_SIZE_LOG2,
1281 .name = "transfer_frag"
1282 },
1283 };
1284 struct pvr_device *pvr_dev = ctx->pvr_dev;
1285 const struct drm_sched_init_args sched_args = {
1286 .ops = &pvr_queue_sched_ops,
1287 .submit_wq = pvr_dev->sched_wq,
1288 .num_rqs = 1,
1289 .credit_limit = 64 * 1024,
1290 .hang_limit = 1,
1291 .timeout = msecs_to_jiffies(SCHED_TIMEOUT_PERIOD),
1292 .timeout_wq = pvr_dev->sched_wq,
1293 .name = "pvr-queue",
1294 .dev = pvr_dev->base.dev,
1295 };
1296 struct drm_gpu_scheduler *sched;
1297 struct pvr_queue *queue;
1298 int ctx_state_size, err;
1299 void *cpu_map;
1300
1301 if (WARN_ON(type >= sizeof(props)))
1302 return ERR_PTR(-EINVAL);
1303
1304 switch (ctx->type) {
1305 case DRM_PVR_CTX_TYPE_RENDER:
1306 if (type != DRM_PVR_JOB_TYPE_GEOMETRY &&
1307 type != DRM_PVR_JOB_TYPE_FRAGMENT)
1308 return ERR_PTR(-EINVAL);
1309 break;
1310 case DRM_PVR_CTX_TYPE_COMPUTE:
1311 if (type != DRM_PVR_JOB_TYPE_COMPUTE)
1312 return ERR_PTR(-EINVAL);
1313 break;
1314 case DRM_PVR_CTX_TYPE_TRANSFER_FRAG:
1315 if (type != DRM_PVR_JOB_TYPE_TRANSFER_FRAG)
1316 return ERR_PTR(-EINVAL);
1317 break;
1318 default:
1319 return ERR_PTR(-EINVAL);
1320 }
1321
1322 ctx_state_size = get_ctx_state_size(pvr_dev, type);
1323 if (ctx_state_size < 0)
1324 return ERR_PTR(ctx_state_size);
1325
1326 queue = kzalloc_obj(*queue);
1327 if (!queue)
1328 return ERR_PTR(-ENOMEM);
1329
1330 queue->type = type;
1331 queue->ctx_offset = get_ctx_offset(type);
1332 queue->ctx = ctx;
1333 queue->callstack_addr = args->callstack_addr;
1334 sched = &queue->scheduler;
1335 INIT_LIST_HEAD(&queue->node);
1336 mutex_init(&queue->cccb_fence_ctx.job_lock);
1337 pvr_queue_fence_ctx_init(&queue->cccb_fence_ctx.base);
1338 pvr_queue_fence_ctx_init(&queue->job_fence_ctx);
1339
1340 err = pvr_cccb_init(pvr_dev, &queue->cccb, props[type].cccb_size, props[type].name);
1341 if (err)
1342 goto err_free_queue;
1343
1344 err = pvr_fw_object_create(pvr_dev, ctx_state_size,
1345 PVR_BO_FW_FLAGS_DEVICE_UNCACHED,
1346 reg_state_init, queue, &queue->reg_state_obj);
1347 if (err)
1348 goto err_cccb_fini;
1349
1350 init_fw_context(queue, fw_ctx_map);
1351
1352 if (type != DRM_PVR_JOB_TYPE_GEOMETRY && type != DRM_PVR_JOB_TYPE_FRAGMENT &&
1353 args->callstack_addr) {
1354 err = -EINVAL;
1355 goto err_release_reg_state;
1356 }
1357
1358 cpu_map = pvr_fw_object_create_and_map(pvr_dev, sizeof(*queue->timeline_ufo.value),
1359 PVR_BO_FW_FLAGS_DEVICE_UNCACHED,
1360 NULL, NULL, &queue->timeline_ufo.fw_obj);
1361 if (IS_ERR(cpu_map)) {
1362 err = PTR_ERR(cpu_map);
1363 goto err_release_reg_state;
1364 }
1365
1366 queue->timeline_ufo.value = cpu_map;
1367
1368 err = drm_sched_init(&queue->scheduler, &sched_args);
1369 if (err)
1370 goto err_release_ufo;
1371
1372 err = drm_sched_entity_init(&queue->entity,
1373 DRM_SCHED_PRIORITY_KERNEL,
1374 &sched, 1, &ctx->faulty);
1375 if (err)
1376 goto err_sched_fini;
1377
1378 mutex_lock(&pvr_dev->queues.lock);
1379 list_add_tail(&queue->node, &pvr_dev->queues.idle);
1380 mutex_unlock(&pvr_dev->queues.lock);
1381
1382 return queue;
1383
1384err_sched_fini:
1385 drm_sched_fini(&queue->scheduler);
1386
1387err_release_ufo:
1388 pvr_fw_object_unmap_and_destroy(queue->timeline_ufo.fw_obj);
1389
1390err_release_reg_state:
1391 pvr_fw_object_destroy(queue->reg_state_obj);
1392
1393err_cccb_fini:
1394 pvr_cccb_fini(&queue->cccb);
1395
1396err_free_queue:
1397 mutex_destroy(&queue->cccb_fence_ctx.job_lock);
1398 kfree(queue);
1399
1400 return ERR_PTR(err);
1401}
1402
1403void pvr_queue_device_pre_reset(struct pvr_device *pvr_dev)
1404{
1405 struct pvr_queue *queue;
1406
1407 mutex_lock(&pvr_dev->queues.lock);
1408 list_for_each_entry(queue, &pvr_dev->queues.idle, node)
1409 pvr_queue_stop(queue, NULL);
1410 list_for_each_entry(queue, &pvr_dev->queues.active, node)
1411 pvr_queue_stop(queue, NULL);
1412 mutex_unlock(&pvr_dev->queues.lock);
1413}
1414
1415void pvr_queue_device_post_reset(struct pvr_device *pvr_dev)
1416{
1417 struct pvr_queue *queue;
1418
1419 mutex_lock(&pvr_dev->queues.lock);
1420 list_for_each_entry(queue, &pvr_dev->queues.active, node)
1421 pvr_queue_start(queue);
1422 list_for_each_entry(queue, &pvr_dev->queues.idle, node)
1423 pvr_queue_start(queue);
1424 mutex_unlock(&pvr_dev->queues.lock);
1425}
1426
1427/**
1428 * pvr_queue_kill() - Kill a queue.
1429 * @queue: The queue to kill.
1430 *
1431 * Kill the queue so no new jobs can be pushed. Should be called when the
1432 * context handle is destroyed. The queue object might last longer if jobs
1433 * are still in flight and holding a reference to the context this queue
1434 * belongs to.
1435 */
1436void pvr_queue_kill(struct pvr_queue *queue)
1437{
1438 drm_sched_entity_destroy(&queue->entity);
1439 dma_fence_put(queue->last_queued_job_scheduled_fence);
1440 queue->last_queued_job_scheduled_fence = NULL;
1441}
1442
1443/**
1444 * pvr_queue_destroy() - Destroy a queue.
1445 * @queue: The queue to destroy.
1446 * @cleanup_queue_entity: Whether to cleanup the queue entity.
1447 *
1448 * Cleanup the queue and free the resources attached to it. Should be
1449 * called from the context release function.
1450 */
1451void pvr_queue_destroy(struct pvr_queue *queue, bool cleanup_queue_entity)
1452{
1453 if (!queue)
1454 return;
1455
1456 mutex_lock(&queue->ctx->pvr_dev->queues.lock);
1457 list_del_init(&queue->node);
1458 mutex_unlock(&queue->ctx->pvr_dev->queues.lock);
1459
1460 drm_sched_fini(&queue->scheduler);
1461 if (cleanup_queue_entity)
1462 drm_sched_entity_fini(&queue->entity);
1463
1464 if (WARN_ON(queue->last_queued_job_scheduled_fence))
1465 dma_fence_put(queue->last_queued_job_scheduled_fence);
1466
1467 pvr_queue_cleanup_fw_context(queue);
1468
1469 pvr_fw_object_unmap_and_destroy(queue->timeline_ufo.fw_obj);
1470 pvr_fw_object_destroy(queue->reg_state_obj);
1471 pvr_cccb_fini(&queue->cccb);
1472 mutex_destroy(&queue->cccb_fence_ctx.job_lock);
1473 kfree(queue);
1474}
1475
1476/**
1477 * pvr_queue_device_init() - Device-level initialization of queue related fields.
1478 * @pvr_dev: The device to initialize.
1479 *
1480 * Initializes all fields related to queue management in pvr_device.
1481 *
1482 * Return:
1483 * * 0 on success, or
1484 * * An error code on failure.
1485 */
1486int pvr_queue_device_init(struct pvr_device *pvr_dev)
1487{
1488 int err;
1489
1490 INIT_LIST_HEAD(&pvr_dev->queues.active);
1491 INIT_LIST_HEAD(&pvr_dev->queues.idle);
1492 err = drmm_mutex_init(from_pvr_device(pvr_dev), &pvr_dev->queues.lock);
1493 if (err)
1494 return err;
1495
1496 pvr_dev->sched_wq = alloc_workqueue("powervr-sched", WQ_UNBOUND, 0);
1497 if (!pvr_dev->sched_wq)
1498 return -ENOMEM;
1499
1500 return 0;
1501}
1502
1503/**
1504 * pvr_queue_device_fini() - Device-level cleanup of queue related fields.
1505 * @pvr_dev: The device to cleanup.
1506 *
1507 * Cleanup/free all queue-related resources attached to a pvr_device object.
1508 */
1509void pvr_queue_device_fini(struct pvr_device *pvr_dev)
1510{
1511 destroy_workqueue(pvr_dev->sched_wq);
1512}