projects/linux·blob master · refs

1873 lines · 51.1 KiB

1// SPDX-License-Identifier: MIT
2/*
3 * Copyright © 2021 Intel Corporation
4 */
5
6#include "xe_exec_queue.h"
7
8#include <linux/nospec.h>
9
10#include <drm/drm_device.h>
11#include <drm/drm_drv.h>
12#include <drm/drm_file.h>
13#include <drm/drm_syncobj.h>
14#include <uapi/drm/xe_drm.h>
15
16#include "xe_bo.h"
17#include "xe_dep_scheduler.h"
18#include "xe_device.h"
19#include "xe_gt.h"
20#include "xe_gt_sriov_pf.h"
21#include "xe_gt_sriov_vf.h"
22#include "xe_hw_engine_class_sysfs.h"
23#include "xe_hw_engine_group.h"
24#include "xe_irq.h"
25#include "xe_lrc.h"
26#include "xe_macros.h"
27#include "xe_migrate.h"
28#include "xe_pm.h"
29#include "xe_trace.h"
30#include "xe_vm.h"
31#include "xe_pxp.h"
32
33/**
34 * DOC: Execution Queue
35 *
36 * An Execution queue is an interface for the HW context of execution.
37 * The user creates an execution queue, submits the GPU jobs through those
38 * queues and in the end destroys them.
39 *
40 * Execution queues can also be created by XeKMD itself for driver internal
41 * operations like object migration etc.
42 *
43 * An execution queue is associated with a specified HW engine or a group of
44 * engines (belonging to the same tile and engine class) and any GPU job
45 * submitted on the queue will be run on one of these engines.
46 *
47 * An execution queue is tied to an address space (VM). It holds a reference
48 * of the associated VM and the underlying Logical Ring Context/s (LRC/s)
49 * until the queue is destroyed.
50 *
51 * The execution queue sits on top of the submission backend. It opaquely
52 * handles the GuC and Execlist backends whichever the platform uses, and
53 * the ring operations the different engine classes support.
54 */
55
56/**
57 * DOC: Multi Queue Group
58 *
59 * Multi Queue Group is another mode of execution supported by the compute
60 * and blitter copy command streamers (CCS and BCS, respectively). It is
61 * an enhancement of the existing hardware architecture and leverages the
62 * same submission model. It enables support for efficient, parallel
63 * execution of multiple queues within a single shared context. The multi
64 * queue group functionality is only supported with GuC submission backend.
65 * All the queues of a group must use the same address space (VM).
66 *
67 * The DRM_XE_EXEC_QUEUE_SET_PROPERTY_MULTI_QUEUE execution queue property
68 * supports creating a multi queue group and adding queues to a queue group.
69 *
70 * The XE_EXEC_QUEUE_CREATE ioctl call with above property with value field
71 * set to DRM_XE_MULTI_GROUP_CREATE, will create a new multi queue group with
72 * the queue being created as the primary queue (aka q0) of the group. To add
73 * secondary queues to the group, they need to be created with the above
74 * property with id of the primary queue as the value. The properties of
75 * the primary queue (like priority, time slice) applies to the whole group.
76 * So, these properties can't be set for secondary queues of a group.
77 *
78 * The hardware does not support removing a queue from a multi-queue group.
79 * However, queues can be dynamically added to the group. A group can have
80 * up to 64 queues. To support this, XeKMD holds references to LRCs of the
81 * queues even after the queues are destroyed by the user until the whole
82 * group is destroyed. The secondary queues hold a reference to the primary
83 * queue thus preventing the group from being destroyed when user destroys
84 * the primary queue. Once the primary queue is destroyed, secondary queues
85 * can't be added to the queue group and new job submissions on existing
86 * secondary queues are not allowed.
87 *
88 * The queues of a multi queue group can set their priority within the group
89 * through the DRM_XE_EXEC_QUEUE_SET_PROPERTY_MULTI_QUEUE_PRIORITY property.
90 * This multi queue priority can also be set dynamically through the
91 * XE_EXEC_QUEUE_SET_PROPERTY ioctl. This is the only other property
92 * supported by the secondary queues of a multi queue group, other than
93 * DRM_XE_EXEC_QUEUE_SET_PROPERTY_MULTI_QUEUE.
94 *
95 * When GuC reports an error on any of the queues of a multi queue group,
96 * the queue cleanup mechanism is invoked for all the queues of the group
97 * as hardware cannot make progress on the multi queue context.
98 *
99 * Refer :ref:`multi-queue-group-guc-interface` for multi queue group GuC
100 * interface.
101 */
102
103enum xe_exec_queue_sched_prop {
104 XE_EXEC_QUEUE_JOB_TIMEOUT = 0,
105 XE_EXEC_QUEUE_TIMESLICE = 1,
106 XE_EXEC_QUEUE_PREEMPT_TIMEOUT = 2,
107 XE_EXEC_QUEUE_SCHED_PROP_MAX = 3,
108};
109
110static int exec_queue_user_extensions(struct xe_device *xe, struct xe_exec_queue *q,
111 u64 extensions);
112
113static void xe_exec_queue_group_cleanup(struct xe_exec_queue *q)
114{
115 struct xe_exec_queue_group *group = q->multi_queue.group;
116 struct xe_lrc *lrc;
117 unsigned long idx;
118
119 if (xe_exec_queue_is_multi_queue_secondary(q)) {
120 /*
121 * Put pairs with get from xe_exec_queue_lookup() call
122 * in xe_exec_queue_group_validate().
123 */
124 xe_exec_queue_put(xe_exec_queue_multi_queue_primary(q));
125 return;
126 }
127
128 if (!group)
129 return;
130
131 /* Primary queue cleanup */
132 xa_for_each(&group->xa, idx, lrc)
133 xe_lrc_put(lrc);
134
135 xa_destroy(&group->xa);
136 mutex_destroy(&group->list_lock);
137 xe_bo_unpin_map_no_vm(group->cgp_bo);
138 kfree(group);
139}
140
141static void __xe_exec_queue_free(struct xe_exec_queue *q)
142{
143 int i;
144
145 for (i = 0; i < XE_EXEC_QUEUE_TLB_INVAL_COUNT; ++i)
146 if (q->tlb_inval[i].dep_scheduler)
147 xe_dep_scheduler_fini(q->tlb_inval[i].dep_scheduler);
148
149 if (xe_exec_queue_uses_pxp(q))
150 xe_pxp_exec_queue_remove(gt_to_xe(q->gt)->pxp, q);
151
152 if (xe_exec_queue_is_multi_queue(q))
153 xe_exec_queue_group_cleanup(q);
154
155 if (q->vm) {
156 xe_vm_remove_exec_queue(q->vm, q);
157 xe_vm_put(q->vm);
158 }
159
160 if (q->xef)
161 xe_file_put(q->xef);
162
163 kvfree(q->replay_state);
164 kfree(q);
165}
166
167static int alloc_dep_schedulers(struct xe_device *xe, struct xe_exec_queue *q)
168{
169 struct xe_tile *tile = gt_to_tile(q->gt);
170 int i;
171
172 for (i = 0; i < XE_EXEC_QUEUE_TLB_INVAL_COUNT; ++i) {
173 struct xe_dep_scheduler *dep_scheduler;
174 struct xe_gt *gt;
175 struct workqueue_struct *wq;
176
177 if (i == XE_EXEC_QUEUE_TLB_INVAL_PRIMARY_GT)
178 gt = tile->primary_gt;
179 else
180 gt = tile->media_gt;
181
182 if (!gt)
183 continue;
184
185 wq = gt->tlb_inval.job_wq;
186
187#define MAX_TLB_INVAL_JOBS 16 /* Picking a reasonable value */
188 dep_scheduler = xe_dep_scheduler_create(xe, wq, q->name,
189 MAX_TLB_INVAL_JOBS);
190 if (IS_ERR(dep_scheduler))
191 return PTR_ERR(dep_scheduler);
192
193 q->tlb_inval[i].dep_scheduler = dep_scheduler;
194 }
195#undef MAX_TLB_INVAL_JOBS
196
197 return 0;
198}
199
200static struct xe_exec_queue *__xe_exec_queue_alloc(struct xe_device *xe,
201 struct xe_vm *vm,
202 u32 logical_mask,
203 u16 width, struct xe_hw_engine *hwe,
204 u32 flags, u64 extensions)
205{
206 struct xe_exec_queue *q;
207 struct xe_gt *gt = hwe->gt;
208 int err;
209
210 /* only kernel queues can be permanent */
211 XE_WARN_ON((flags & EXEC_QUEUE_FLAG_PERMANENT) && !(flags & EXEC_QUEUE_FLAG_KERNEL));
212
213 q = kzalloc_flex(*q, lrc, width);
214 if (!q)
215 return ERR_PTR(-ENOMEM);
216
217 kref_init(&q->refcount);
218 q->flags = flags;
219 q->hwe = hwe;
220 q->gt = gt;
221 q->class = hwe->class;
222 q->width = width;
223 q->msix_vec = XE_IRQ_DEFAULT_MSIX;
224 q->logical_mask = logical_mask;
225 q->fence_irq = &gt->fence_irq[hwe->class];
226 q->ring_ops = gt->ring_ops[hwe->class];
227 q->ops = gt->exec_queue_ops;
228 INIT_LIST_HEAD(&q->lr.link);
229 INIT_LIST_HEAD(&q->vm_exec_queue_link);
230 INIT_LIST_HEAD(&q->multi_gt_link);
231 INIT_LIST_HEAD(&q->hw_engine_group_link);
232 INIT_LIST_HEAD(&q->pxp.link);
233 spin_lock_init(&q->multi_queue.lock);
234 spin_lock_init(&q->lrc_lookup_lock);
235 q->multi_queue.priority = XE_MULTI_QUEUE_PRIORITY_NORMAL;
236
237 q->sched_props.timeslice_us = hwe->eclass->sched_props.timeslice_us;
238 q->sched_props.preempt_timeout_us =
239 hwe->eclass->sched_props.preempt_timeout_us;
240 q->sched_props.job_timeout_ms =
241 hwe->eclass->sched_props.job_timeout_ms;
242 if (q->flags & EXEC_QUEUE_FLAG_KERNEL &&
243 q->flags & EXEC_QUEUE_FLAG_HIGH_PRIORITY)
244 q->sched_props.priority = XE_EXEC_QUEUE_PRIORITY_KERNEL;
245 else
246 q->sched_props.priority = XE_EXEC_QUEUE_PRIORITY_NORMAL;
247
248 if (q->flags & (EXEC_QUEUE_FLAG_MIGRATE | EXEC_QUEUE_FLAG_VM)) {
249 err = alloc_dep_schedulers(xe, q);
250 if (err) {
251 __xe_exec_queue_free(q);
252 return ERR_PTR(err);
253 }
254 }
255
256 if (vm)
257 q->vm = xe_vm_get(vm);
258
259 if (extensions) {
260 /*
261 * may set q->usm, must come before xe_lrc_create(),
262 * may overwrite q->sched_props, must come before q->ops->init()
263 */
264 err = exec_queue_user_extensions(xe, q, extensions);
265 if (err) {
266 __xe_exec_queue_free(q);
267 return ERR_PTR(err);
268 }
269 }
270
271 return q;
272}
273
274static void xe_exec_queue_set_lrc(struct xe_exec_queue *q, struct xe_lrc *lrc, u16 idx)
275{
276 xe_assert(gt_to_xe(q->gt), idx < q->width);
277
278 scoped_guard(spinlock, &q->lrc_lookup_lock) {
279 q->lrc[idx] = lrc;
280 if (xe_exec_queue_is_multi_queue(q))
281 q->lrc[idx]->multi_queue.primary_lrc =
282 q->multi_queue.group->primary->lrc[0];
283 }
284}
285
286/**
287 * xe_exec_queue_get_lrc() - Get the LRC from exec queue.
288 * @q: The exec queue instance.
289 * @idx: Index within multi-LRC array.
290 *
291 * Retrieves LRC of given index for the exec queue under lock
292 * and takes reference.
293 *
294 * Return: Pointer to LRC on success, error on failure, NULL on
295 * lookup failure.
296 */
297struct xe_lrc *xe_exec_queue_get_lrc(struct xe_exec_queue *q, u16 idx)
298{
299 struct xe_lrc *lrc;
300
301 xe_assert(gt_to_xe(q->gt), idx < q->width);
302
303 scoped_guard(spinlock, &q->lrc_lookup_lock) {
304 lrc = q->lrc[idx];
305 if (lrc)
306 xe_lrc_get(lrc);
307 }
308
309 return lrc;
310}
311
312/**
313 * xe_exec_queue_lrc() - Get the LRC from exec queue.
314 * @q: The exec queue instance.
315 *
316 * Retrieves the primary LRC for the exec queue. Note that this function
317 * returns only the first LRC instance, even when multiple parallel LRCs
318 * are configured. This function does not increment reference count,
319 * so the reference can be just forgotten after use.
320 *
321 * Return: Pointer to LRC on success, error on failure
322 */
323struct xe_lrc *xe_exec_queue_lrc(struct xe_exec_queue *q)
324{
325 return q->lrc[0];
326}
327
328static void __xe_exec_queue_fini(struct xe_exec_queue *q)
329{
330 int i;
331
332 q->ops->fini(q);
333
334 for (i = 0; i < q->width; ++i)
335 xe_lrc_put(q->lrc[i]);
336}
337
338static int __xe_exec_queue_init(struct xe_exec_queue *q, u32 exec_queue_flags)
339{
340 int i, err;
341 u32 flags = 0;
342
343 /*
344 * PXP workloads executing on RCS or CCS must run in isolation (i.e. no
345 * other workload can use the EUs at the same time). On MTL this is done
346 * by setting the RUNALONE bit in the LRC, while starting on Xe2 there
347 * is a dedicated bit for it.
348 */
349 if (xe_exec_queue_uses_pxp(q) &&
350 (q->class == XE_ENGINE_CLASS_RENDER || q->class == XE_ENGINE_CLASS_COMPUTE)) {
351 if (GRAPHICS_VER(gt_to_xe(q->gt)) >= 20)
352 flags |= XE_LRC_CREATE_PXP;
353 else
354 flags |= XE_LRC_CREATE_RUNALONE;
355 }
356
357 if (!(exec_queue_flags & EXEC_QUEUE_FLAG_KERNEL))
358 flags |= XE_LRC_CREATE_USER_CTX;
359
360 if (q->flags & EXEC_QUEUE_FLAG_DISABLE_STATE_CACHE_PERF_FIX)
361 flags |= XE_LRC_DISABLE_STATE_CACHE_PERF_FIX;
362
363 err = q->ops->init(q);
364 if (err)
365 return err;
366
367 /*
368 * This must occur after q->ops->init to avoid race conditions during VF
369 * post-migration recovery, as the fixups for the LRC GGTT addresses
370 * depend on the queue being present in the backend tracking structure.
371 *
372 * In addition to above, we must wait on inflight GGTT changes to avoid
373 * writing out stale values here. Such wait provides a solid solution
374 * (without a race) only if the function can detect migration instantly
375 * from the moment vCPU resumes execution.
376 */
377 for (i = 0; i < q->width; ++i) {
378 struct xe_lrc *__lrc = NULL;
379 int marker;
380
381 do {
382 struct xe_lrc *lrc;
383
384 marker = xe_gt_sriov_vf_wait_valid_ggtt(q->gt);
385
386 lrc = xe_lrc_create(q->hwe, q->vm, q->replay_state,
387 xe_lrc_ring_size(), q->msix_vec, flags);
388 if (IS_ERR(lrc)) {
389 err = PTR_ERR(lrc);
390 goto err_lrc;
391 }
392
393 xe_exec_queue_set_lrc(q, lrc, i);
394
395 if (__lrc)
396 xe_lrc_put(__lrc);
397 __lrc = lrc;
398
399 } while (marker != xe_vf_migration_fixups_complete_count(q->gt));
400 }
401
402 return 0;
403
404err_lrc:
405 __xe_exec_queue_fini(q);
406 return err;
407}
408
409/**
410 * xe_exec_queue_create() - Create an exec queue
411 * @xe: Xe device
412 * @vm: VM for the exec queue
413 * @logical_mask: Logical mask of HW engines
414 * @width: Width of the exec queue (number of LRCs)
415 * @hwe: Hardware engine
416 * @flags: Exec queue creation flags
417 * @extensions: Extensions for exec queue creation
418 *
419 * Create an exec queue (allocate and initialize) with the specified parameters
420 *
421 * Return: Pointer to the created exec queue on success, ERR_PTR on failure
422 */
423struct xe_exec_queue *xe_exec_queue_create(struct xe_device *xe, struct xe_vm *vm,
424 u32 logical_mask, u16 width,
425 struct xe_hw_engine *hwe, u32 flags,
426 u64 extensions)
427{
428 struct xe_exec_queue *q;
429 int err;
430
431 /* VMs for GSCCS queues (and only those) must have the XE_VM_FLAG_GSC flag */
432 xe_assert(xe, !vm || (!!(vm->flags & XE_VM_FLAG_GSC) == !!(hwe->engine_id == XE_HW_ENGINE_GSCCS0)));
433
434 q = __xe_exec_queue_alloc(xe, vm, logical_mask, width, hwe, flags,
435 extensions);
436 if (IS_ERR(q))
437 return q;
438
439 err = __xe_exec_queue_init(q, flags);
440 if (err)
441 goto err_post_alloc;
442
443 /*
444 * We can only add the queue to the PXP list after the init is complete,
445 * because the PXP termination can call exec_queue_kill and that will
446 * go bad if the queue is only half-initialized. This means that we
447 * can't do it when we handle the PXP extension in __xe_exec_queue_alloc
448 * and we need to do it here instead.
449 */
450 if (xe_exec_queue_uses_pxp(q)) {
451 err = xe_pxp_exec_queue_add(xe->pxp, q);
452 if (err)
453 goto err_post_init;
454 }
455
456 return q;
457
458err_post_init:
459 __xe_exec_queue_fini(q);
460err_post_alloc:
461 __xe_exec_queue_free(q);
462 return ERR_PTR(err);
463}
464ALLOW_ERROR_INJECTION(xe_exec_queue_create, ERRNO);
465
466/**
467 * xe_exec_queue_create_class() - Create an exec queue for a specific engine class
468 * @xe: Xe device
469 * @gt: GT for the exec queue
470 * @vm: VM for the exec queue
471 * @class: Engine class
472 * @flags: Exec queue creation flags
473 * @extensions: Extensions for exec queue creation
474 *
475 * Create an exec queue for the specified engine class.
476 *
477 * Return: Pointer to the created exec queue on success, ERR_PTR on failure
478 */
479struct xe_exec_queue *xe_exec_queue_create_class(struct xe_device *xe, struct xe_gt *gt,
480 struct xe_vm *vm,
481 enum xe_engine_class class,
482 u32 flags, u64 extensions)
483{
484 struct xe_hw_engine *hwe, *hwe0 = NULL;
485 enum xe_hw_engine_id id;
486 u32 logical_mask = 0;
487
488 for_each_hw_engine(hwe, gt, id) {
489 if (xe_hw_engine_is_reserved(hwe))
490 continue;
491
492 if (hwe->class == class) {
493 logical_mask |= BIT(hwe->logical_instance);
494 if (!hwe0)
495 hwe0 = hwe;
496 }
497 }
498
499 if (!logical_mask)
500 return ERR_PTR(-ENODEV);
501
502 return xe_exec_queue_create(xe, vm, logical_mask, 1, hwe0, flags, extensions);
503}
504
505/**
506 * xe_exec_queue_create_bind() - Create bind exec queue.
507 * @xe: Xe device.
508 * @tile: tile which bind exec queue belongs to.
509 * @flags: exec queue creation flags
510 * @user_vm: The user VM which this exec queue belongs to
511 * @extensions: exec queue creation extensions
512 *
513 * Normalize bind exec queue creation. Bind exec queue is tied to migration VM
514 * for access to physical memory required for page table programming. On a
515 * faulting devices the reserved copy engine instance must be used to avoid
516 * deadlocking (user binds cannot get stuck behind faults as kernel binds which
517 * resolve faults depend on user binds). On non-faulting devices any copy engine
518 * can be used.
519 *
520 * Returns exec queue on success, ERR_PTR on failure
521 */
522struct xe_exec_queue *xe_exec_queue_create_bind(struct xe_device *xe,
523 struct xe_tile *tile,
524 struct xe_vm *user_vm,
525 u32 flags, u64 extensions)
526{
527 struct xe_gt *gt = tile->primary_gt;
528 struct xe_exec_queue *q;
529 struct xe_vm *migrate_vm;
530
531 migrate_vm = xe_migrate_get_vm(tile->migrate);
532 if (xe->info.has_usm) {
533 struct xe_hw_engine *hwe = gt->usm.paging_hwe0;
534
535 if (!hwe) {
536 xe_vm_put(migrate_vm);
537 return ERR_PTR(-EINVAL);
538 }
539
540 q = xe_exec_queue_create(xe, migrate_vm,
541 BIT(hwe->logical_instance), 1, hwe,
542 flags, extensions);
543 } else {
544 q = xe_exec_queue_create_class(xe, gt, migrate_vm,
545 XE_ENGINE_CLASS_COPY, flags,
546 extensions);
547 }
548 xe_vm_put(migrate_vm);
549
550 if (!IS_ERR(q)) {
551 int err = drm_syncobj_create(&q->ufence_syncobj,
552 DRM_SYNCOBJ_CREATE_SIGNALED,
553 NULL);
554 if (err) {
555 xe_exec_queue_put(q);
556 return ERR_PTR(err);
557 }
558
559 if (user_vm)
560 q->user_vm = xe_vm_get(user_vm);
561 }
562
563 return q;
564}
565ALLOW_ERROR_INJECTION(xe_exec_queue_create_bind, ERRNO);
566
567/**
568 * xe_exec_queue_destroy() - Destroy an exec queue
569 * @ref: Reference count of the exec queue
570 *
571 * Called when the last reference to the exec queue is dropped.
572 * Cleans up all resources associated with the exec queue.
573 * This function should not be called directly; use xe_exec_queue_put() instead.
574 */
575void xe_exec_queue_destroy(struct kref *ref)
576{
577 struct xe_exec_queue *q = container_of(ref, struct xe_exec_queue, refcount);
578 struct xe_exec_queue *eq, *next;
579 int i;
580
581 xe_assert(gt_to_xe(q->gt), atomic_read(&q->job_cnt) == 0);
582
583 if (q->ufence_syncobj)
584 drm_syncobj_put(q->ufence_syncobj);
585
586 if (xe_exec_queue_uses_pxp(q))
587 xe_pxp_exec_queue_remove(gt_to_xe(q->gt)->pxp, q);
588
589 xe_exec_queue_last_fence_put_unlocked(q);
590 for_each_tlb_inval(i)
591 xe_exec_queue_tlb_inval_last_fence_put_unlocked(q, i);
592
593 if (!(q->flags & EXEC_QUEUE_FLAG_BIND_ENGINE_CHILD)) {
594 list_for_each_entry_safe(eq, next, &q->multi_gt_list,
595 multi_gt_link)
596 xe_exec_queue_put(eq);
597 }
598
599 if (q->user_vm) {
600 xe_vm_put(q->user_vm);
601 q->user_vm = NULL;
602 }
603
604 q->ops->destroy(q);
605}
606
607/**
608 * xe_exec_queue_fini() - Finalize an exec queue
609 * @q: The exec queue
610 *
611 * Finalizes the exec queue by updating run ticks, releasing LRC references,
612 * and freeing the queue structure. This is called after the queue has been
613 * destroyed and all references have been dropped.
614 */
615void xe_exec_queue_fini(struct xe_exec_queue *q)
616{
617 /*
618 * Before releasing our ref to lrc and xef, accumulate our run ticks
619 * and wakeup any waiters.
620 */
621 xe_exec_queue_update_run_ticks(q);
622 if (q->xef && atomic_dec_and_test(&q->xef->exec_queue.pending_removal))
623 wake_up_var(&q->xef->exec_queue.pending_removal);
624
625 __xe_exec_queue_fini(q);
626 __xe_exec_queue_free(q);
627}
628
629/**
630 * xe_exec_queue_assign_name() - Assign a name to an exec queue
631 * @q: The exec queue
632 * @instance: Instance number for the engine
633 *
634 * Assigns a human-readable name to the exec queue based on its engine class
635 * and instance number (e.g., "rcs0", "vcs1", "bcs2").
636 */
637void xe_exec_queue_assign_name(struct xe_exec_queue *q, u32 instance)
638{
639 switch (q->class) {
640 case XE_ENGINE_CLASS_RENDER:
641 snprintf(q->name, sizeof(q->name), "rcs%d", instance);
642 break;
643 case XE_ENGINE_CLASS_VIDEO_DECODE:
644 snprintf(q->name, sizeof(q->name), "vcs%d", instance);
645 break;
646 case XE_ENGINE_CLASS_VIDEO_ENHANCE:
647 snprintf(q->name, sizeof(q->name), "vecs%d", instance);
648 break;
649 case XE_ENGINE_CLASS_COPY:
650 snprintf(q->name, sizeof(q->name), "bcs%d", instance);
651 break;
652 case XE_ENGINE_CLASS_COMPUTE:
653 snprintf(q->name, sizeof(q->name), "ccs%d", instance);
654 break;
655 case XE_ENGINE_CLASS_OTHER:
656 snprintf(q->name, sizeof(q->name), "gsccs%d", instance);
657 break;
658 default:
659 XE_WARN_ON(q->class);
660 }
661}
662
663/**
664 * xe_exec_queue_lookup() - Look up an exec queue by ID
665 * @xef: Xe file private data
666 * @id: Exec queue ID
667 *
668 * Looks up an exec queue by its ID and increments its reference count.
669 *
670 * Return: Pointer to the exec queue if found, NULL otherwise
671 */
672struct xe_exec_queue *xe_exec_queue_lookup(struct xe_file *xef, u32 id)
673{
674 struct xe_exec_queue *q;
675
676 mutex_lock(&xef->exec_queue.lock);
677 q = xa_load(&xef->exec_queue.xa, id);
678 if (q)
679 xe_exec_queue_get(q);
680 mutex_unlock(&xef->exec_queue.lock);
681
682 return q;
683}
684
685/**
686 * xe_exec_queue_device_get_max_priority() - Get maximum priority for an exec queues
687 * @xe: Xe device
688 *
689 * Returns the maximum priority level that can be assigned to an exec queues.
690 *
691 * Return: Maximum priority level (HIGH if CAP_SYS_NICE, NORMAL otherwise)
692 */
693enum xe_exec_queue_priority
694xe_exec_queue_device_get_max_priority(struct xe_device *xe)
695{
696 return capable(CAP_SYS_NICE) ? XE_EXEC_QUEUE_PRIORITY_HIGH :
697 XE_EXEC_QUEUE_PRIORITY_NORMAL;
698}
699
700static int exec_queue_set_priority(struct xe_device *xe, struct xe_exec_queue *q,
701 u64 value)
702{
703 if (XE_IOCTL_DBG(xe, value > XE_EXEC_QUEUE_PRIORITY_HIGH))
704 return -EINVAL;
705
706 if (XE_IOCTL_DBG(xe, value > xe_exec_queue_device_get_max_priority(xe)))
707 return -EPERM;
708
709 q->sched_props.priority = value;
710 return 0;
711}
712
713static bool xe_exec_queue_enforce_schedule_limit(void)
714{
715#if IS_ENABLED(CONFIG_DRM_XE_ENABLE_SCHEDTIMEOUT_LIMIT)
716 return true;
717#else
718 return !capable(CAP_SYS_NICE);
719#endif
720}
721
722static void
723xe_exec_queue_get_prop_minmax(struct xe_hw_engine_class_intf *eclass,
724 enum xe_exec_queue_sched_prop prop,
725 u32 *min, u32 *max)
726{
727 switch (prop) {
728 case XE_EXEC_QUEUE_JOB_TIMEOUT:
729 *min = eclass->sched_props.job_timeout_min;
730 *max = eclass->sched_props.job_timeout_max;
731 break;
732 case XE_EXEC_QUEUE_TIMESLICE:
733 *min = eclass->sched_props.timeslice_min;
734 *max = eclass->sched_props.timeslice_max;
735 break;
736 case XE_EXEC_QUEUE_PREEMPT_TIMEOUT:
737 *min = eclass->sched_props.preempt_timeout_min;
738 *max = eclass->sched_props.preempt_timeout_max;
739 break;
740 default:
741 break;
742 }
743#if IS_ENABLED(CONFIG_DRM_XE_ENABLE_SCHEDTIMEOUT_LIMIT)
744 if (capable(CAP_SYS_NICE)) {
745 switch (prop) {
746 case XE_EXEC_QUEUE_JOB_TIMEOUT:
747 *min = XE_HW_ENGINE_JOB_TIMEOUT_MIN;
748 *max = XE_HW_ENGINE_JOB_TIMEOUT_MAX;
749 break;
750 case XE_EXEC_QUEUE_TIMESLICE:
751 *min = XE_HW_ENGINE_TIMESLICE_MIN;
752 *max = XE_HW_ENGINE_TIMESLICE_MAX;
753 break;
754 case XE_EXEC_QUEUE_PREEMPT_TIMEOUT:
755 *min = XE_HW_ENGINE_PREEMPT_TIMEOUT_MIN;
756 *max = XE_HW_ENGINE_PREEMPT_TIMEOUT_MAX;
757 break;
758 default:
759 break;
760 }
761 }
762#endif
763}
764
765static int exec_queue_set_timeslice(struct xe_device *xe, struct xe_exec_queue *q,
766 u64 value)
767{
768 u32 min = 0, max = 0;
769
770 xe_exec_queue_get_prop_minmax(q->hwe->eclass,
771 XE_EXEC_QUEUE_TIMESLICE, &min, &max);
772
773 if (xe_exec_queue_enforce_schedule_limit() &&
774 !xe_hw_engine_timeout_in_range(value, min, max))
775 return -EINVAL;
776
777 q->sched_props.timeslice_us = value;
778 return 0;
779}
780
781static int
782exec_queue_set_pxp_type(struct xe_device *xe, struct xe_exec_queue *q, u64 value)
783{
784 if (value == DRM_XE_PXP_TYPE_NONE)
785 return 0;
786
787 /* we only support HWDRM sessions right now */
788 if (XE_IOCTL_DBG(xe, value != DRM_XE_PXP_TYPE_HWDRM))
789 return -EINVAL;
790
791 if (!xe_pxp_is_enabled(xe->pxp))
792 return -ENODEV;
793
794 return xe_pxp_exec_queue_set_type(xe->pxp, q, DRM_XE_PXP_TYPE_HWDRM);
795}
796
797static int exec_queue_set_hang_replay_state(struct xe_device *xe,
798 struct xe_exec_queue *q,
799 u64 value)
800{
801 size_t size = xe_gt_lrc_hang_replay_size(q->gt, q->class);
802 u64 __user *address = u64_to_user_ptr(value);
803 void *ptr;
804
805 if (q->replay_state)
806 return -EINVAL;
807
808 ptr = vmemdup_user(address, size);
809 if (XE_IOCTL_DBG(xe, IS_ERR(ptr)))
810 return PTR_ERR(ptr);
811
812 q->replay_state = ptr;
813
814 return 0;
815}
816
817static int xe_exec_queue_group_init(struct xe_device *xe, struct xe_exec_queue *q)
818{
819 struct xe_tile *tile = gt_to_tile(q->gt);
820 struct xe_exec_queue_group *group;
821 struct xe_bo *bo;
822
823 group = kzalloc_obj(*group);
824 if (!group)
825 return -ENOMEM;
826
827 bo = xe_bo_create_pin_map_novm(xe, tile, SZ_4K, ttm_bo_type_kernel,
828 XE_BO_FLAG_VRAM_IF_DGFX(tile) |
829 XE_BO_FLAG_PINNED_LATE_RESTORE |
830 XE_BO_FLAG_FORCE_USER_VRAM |
831 XE_BO_FLAG_GGTT_INVALIDATE |
832 XE_BO_FLAG_GGTT, false);
833 if (IS_ERR(bo)) {
834 drm_err(&xe->drm, "CGP bo allocation for queue group failed: %ld\n",
835 PTR_ERR(bo));
836 kfree(group);
837 return PTR_ERR(bo);
838 }
839
840 xe_map_memset(xe, &bo->vmap, 0, 0, SZ_4K);
841
842 group->primary = q;
843 group->cgp_bo = bo;
844 INIT_LIST_HEAD(&group->list);
845 spin_lock_init(&group->suspend_lock);
846 xa_init_flags(&group->xa, XA_FLAGS_ALLOC1);
847 mutex_init(&group->list_lock);
848 q->multi_queue.group = group;
849
850 /* group->list_lock is used in submission backend */
851 if (IS_ENABLED(CONFIG_LOCKDEP)) {
852 fs_reclaim_acquire(GFP_KERNEL);
853 might_lock(&group->list_lock);
854 fs_reclaim_release(GFP_KERNEL);
855 }
856
857 return 0;
858}
859
860static int xe_exec_queue_group_validate(struct xe_device *xe, struct xe_exec_queue *q,
861 u32 primary_id)
862{
863 struct xe_exec_queue_group *group;
864 struct xe_exec_queue *primary;
865 int ret;
866
867 /*
868 * Get from below xe_exec_queue_lookup() pairs with put
869 * in xe_exec_queue_group_cleanup().
870 */
871 primary = xe_exec_queue_lookup(q->vm->xef, primary_id);
872 if (XE_IOCTL_DBG(xe, !primary))
873 return -ENOENT;
874
875 if (XE_IOCTL_DBG(xe, !xe_exec_queue_is_multi_queue_primary(primary)) ||
876 XE_IOCTL_DBG(xe, q->vm != primary->vm) ||
877 XE_IOCTL_DBG(xe, q->logical_mask != primary->logical_mask)) {
878 ret = -EINVAL;
879 goto put_primary;
880 }
881
882 group = primary->multi_queue.group;
883 q->multi_queue.valid = true;
884 q->multi_queue.group = group;
885
886 return 0;
887put_primary:
888 xe_exec_queue_put(primary);
889 return ret;
890}
891
892#define XE_MAX_GROUP_SIZE 64
893static int xe_exec_queue_group_add(struct xe_device *xe, struct xe_exec_queue *q)
894{
895 struct xe_exec_queue_group *group = q->multi_queue.group;
896 u32 pos;
897 int err;
898
899 xe_assert(xe, xe_exec_queue_is_multi_queue_secondary(q));
900
901 /* Primary queue holds a reference to LRCs of all secondary queues */
902 err = xa_alloc(&group->xa, &pos, xe_lrc_get(q->lrc[0]),
903 XA_LIMIT(1, XE_MAX_GROUP_SIZE - 1), GFP_KERNEL);
904 if (XE_IOCTL_DBG(xe, err)) {
905 xe_lrc_put(q->lrc[0]);
906
907 /* It is invalid if queue group limit is exceeded */
908 if (err == -EBUSY)
909 err = -EINVAL;
910
911 return err;
912 }
913
914 q->multi_queue.pos = pos;
915 q->lrc[0]->multi_queue.pos = pos;
916
917 return 0;
918}
919
920static void xe_exec_queue_group_delete(struct xe_device *xe, struct xe_exec_queue *q)
921{
922 struct xe_exec_queue_group *group = q->multi_queue.group;
923 struct xe_lrc *lrc;
924
925 xe_assert(xe, xe_exec_queue_is_multi_queue_secondary(q));
926
927 lrc = xa_erase(&group->xa, q->multi_queue.pos);
928 xe_assert(xe, lrc);
929 xe_lrc_put(lrc);
930}
931
932static int exec_queue_set_multi_group(struct xe_device *xe, struct xe_exec_queue *q,
933 u64 value)
934{
935 if (XE_IOCTL_DBG(xe, !xe_gt_supports_multi_queue(q->gt, q->class)))
936 return -ENODEV;
937
938 if (XE_IOCTL_DBG(xe, !xe_device_uc_enabled(xe)))
939 return -EOPNOTSUPP;
940
941 if (XE_IOCTL_DBG(xe, !q->vm->xef))
942 return -EINVAL;
943
944 if (XE_IOCTL_DBG(xe, xe_exec_queue_is_parallel(q)))
945 return -EINVAL;
946
947 if (XE_IOCTL_DBG(xe, xe_exec_queue_is_multi_queue(q)))
948 return -EINVAL;
949
950 if (value & DRM_XE_MULTI_GROUP_CREATE) {
951 if (XE_IOCTL_DBG(xe, value & ~DRM_XE_MULTI_GROUP_CREATE))
952 return -EINVAL;
953
954 q->multi_queue.valid = true;
955 q->multi_queue.is_primary = true;
956 q->multi_queue.pos = 0;
957 return 0;
958 }
959
960 /* While adding secondary queues, the upper 32 bits must be 0 */
961 if (XE_IOCTL_DBG(xe, value & (~0ull << 32)))
962 return -EINVAL;
963
964 return xe_exec_queue_group_validate(xe, q, value);
965}
966
967static int exec_queue_set_multi_queue_priority(struct xe_device *xe, struct xe_exec_queue *q,
968 u64 value)
969{
970 if (XE_IOCTL_DBG(xe, value > XE_MULTI_QUEUE_PRIORITY_HIGH))
971 return -EINVAL;
972
973 /* For queue creation time (!q->xef) setting, just store the priority value */
974 if (!q->xef) {
975 q->multi_queue.priority = value;
976 return 0;
977 }
978
979 if (!xe_exec_queue_is_multi_queue(q))
980 return -EINVAL;
981
982 return q->ops->set_multi_queue_priority(q, value);
983}
984
985static int exec_queue_set_state_cache_perf_fix(struct xe_device *xe, struct xe_exec_queue *q,
986 u64 value)
987{
988 if (XE_IOCTL_DBG(xe, q->class != XE_ENGINE_CLASS_RENDER))
989 return -EOPNOTSUPP;
990
991 q->flags |= value != 0 ? EXEC_QUEUE_FLAG_DISABLE_STATE_CACHE_PERF_FIX : 0;
992
993 return 0;
994}
995
996typedef int (*xe_exec_queue_set_property_fn)(struct xe_device *xe,
997 struct xe_exec_queue *q,
998 u64 value);
999
1000static const xe_exec_queue_set_property_fn exec_queue_set_property_funcs[] = {
1001 [DRM_XE_EXEC_QUEUE_SET_PROPERTY_PRIORITY] = exec_queue_set_priority,
1002 [DRM_XE_EXEC_QUEUE_SET_PROPERTY_TIMESLICE] = exec_queue_set_timeslice,
1003 [DRM_XE_EXEC_QUEUE_SET_PROPERTY_PXP_TYPE] = exec_queue_set_pxp_type,
1004 [DRM_XE_EXEC_QUEUE_SET_HANG_REPLAY_STATE] = exec_queue_set_hang_replay_state,
1005 [DRM_XE_EXEC_QUEUE_SET_PROPERTY_MULTI_GROUP] = exec_queue_set_multi_group,
1006 [DRM_XE_EXEC_QUEUE_SET_PROPERTY_MULTI_QUEUE_PRIORITY] =
1007 exec_queue_set_multi_queue_priority,
1008 [DRM_XE_EXEC_QUEUE_SET_DISABLE_STATE_CACHE_PERF_FIX] =
1009 exec_queue_set_state_cache_perf_fix,
1010};
1011
1012/**
1013 * xe_exec_queue_set_property_ioctl() - Set a property on an exec queue
1014 * @dev: DRM device
1015 * @data: IOCTL data
1016 * @file: DRM file
1017 *
1018 * Allows setting properties on an existing exec queue. Currently only
1019 * supports setting multi-queue priority.
1020 *
1021 * Return: 0 on success, negative error code on failure
1022 */
1023int xe_exec_queue_set_property_ioctl(struct drm_device *dev, void *data,
1024 struct drm_file *file)
1025{
1026 struct xe_device *xe = to_xe_device(dev);
1027 struct xe_file *xef = to_xe_file(file);
1028 struct drm_xe_exec_queue_set_property *args = data;
1029 struct xe_exec_queue *q;
1030 int ret;
1031 u32 idx;
1032
1033 if (XE_IOCTL_DBG(xe, args->reserved[0] || args->reserved[1]))
1034 return -EINVAL;
1035
1036 if (XE_IOCTL_DBG(xe, args->property !=
1037 DRM_XE_EXEC_QUEUE_SET_PROPERTY_MULTI_QUEUE_PRIORITY))
1038 return -EINVAL;
1039
1040 q = xe_exec_queue_lookup(xef, args->exec_queue_id);
1041 if (XE_IOCTL_DBG(xe, !q))
1042 return -ENOENT;
1043
1044 idx = array_index_nospec(args->property,
1045 ARRAY_SIZE(exec_queue_set_property_funcs));
1046 ret = exec_queue_set_property_funcs[idx](xe, q, args->value);
1047 if (XE_IOCTL_DBG(xe, ret))
1048 goto err_post_lookup;
1049
1050 xe_exec_queue_put(q);
1051 return 0;
1052
1053 err_post_lookup:
1054 xe_exec_queue_put(q);
1055 return ret;
1056}
1057
1058static int exec_queue_user_ext_check(struct xe_exec_queue *q, u64 properties)
1059{
1060 struct xe_device *xe = gt_to_xe(q->gt);
1061 u64 secondary_queue_valid_props = BIT_ULL(DRM_XE_EXEC_QUEUE_SET_PROPERTY_MULTI_GROUP) |
1062 BIT_ULL(DRM_XE_EXEC_QUEUE_SET_PROPERTY_MULTI_QUEUE_PRIORITY);
1063
1064 /*
1065 * Only MULTI_QUEUE_PRIORITY property is valid for secondary queues of a
1066 * multi-queue group.
1067 */
1068 if (xe_exec_queue_is_multi_queue_secondary(q) &&
1069 properties & ~secondary_queue_valid_props)
1070 return -EINVAL;
1071
1072 /*
1073 * HWDRM is the only supported PXP type today. It is display related and
1074 * hence can't work with multi-queue. Reject the combination. The secondary
1075 * queue path above already rejects any PXP property, so this also covers
1076 * the multi-queue primary which would otherwise allow it.
1077 */
1078 if (XE_IOCTL_DBG(xe, (properties & BIT_ULL(DRM_XE_EXEC_QUEUE_SET_PROPERTY_MULTI_GROUP)) &&
1079 (properties & BIT_ULL(DRM_XE_EXEC_QUEUE_SET_PROPERTY_PXP_TYPE))))
1080 return -EINVAL;
1081
1082 return 0;
1083}
1084
1085static int exec_queue_user_ext_check_final(struct xe_exec_queue *q, u64 properties)
1086{
1087 /* MULTI_QUEUE_PRIORITY only applies to multi-queue group queues */
1088 if ((properties & BIT_ULL(DRM_XE_EXEC_QUEUE_SET_PROPERTY_MULTI_QUEUE_PRIORITY)) &&
1089 !(properties & BIT_ULL(DRM_XE_EXEC_QUEUE_SET_PROPERTY_MULTI_GROUP)))
1090 return -EINVAL;
1091
1092 return 0;
1093}
1094
1095static int exec_queue_user_ext_set_property(struct xe_device *xe,
1096 struct xe_exec_queue *q,
1097 u64 extension, u64 *properties)
1098{
1099 u64 __user *address = u64_to_user_ptr(extension);
1100 struct drm_xe_ext_set_property ext;
1101 int err;
1102 u32 idx;
1103
1104 err = copy_from_user(&ext, address, sizeof(ext));
1105 if (XE_IOCTL_DBG(xe, err))
1106 return -EFAULT;
1107
1108 if (XE_IOCTL_DBG(xe, ext.property >=
1109 ARRAY_SIZE(exec_queue_set_property_funcs)) ||
1110 XE_IOCTL_DBG(xe, ext.pad) ||
1111 XE_IOCTL_DBG(xe, ext.property != DRM_XE_EXEC_QUEUE_SET_PROPERTY_PRIORITY &&
1112 ext.property != DRM_XE_EXEC_QUEUE_SET_PROPERTY_TIMESLICE &&
1113 ext.property != DRM_XE_EXEC_QUEUE_SET_PROPERTY_PXP_TYPE &&
1114 ext.property != DRM_XE_EXEC_QUEUE_SET_HANG_REPLAY_STATE &&
1115 ext.property != DRM_XE_EXEC_QUEUE_SET_PROPERTY_MULTI_GROUP &&
1116 ext.property != DRM_XE_EXEC_QUEUE_SET_PROPERTY_MULTI_QUEUE_PRIORITY &&
1117 ext.property != DRM_XE_EXEC_QUEUE_SET_DISABLE_STATE_CACHE_PERF_FIX))
1118 return -EINVAL;
1119
1120 idx = array_index_nospec(ext.property, ARRAY_SIZE(exec_queue_set_property_funcs));
1121 if (!exec_queue_set_property_funcs[idx])
1122 return -EINVAL;
1123
1124 *properties |= BIT_ULL(idx);
1125 err = exec_queue_user_ext_check(q, *properties);
1126 if (XE_IOCTL_DBG(xe, err))
1127 return err;
1128
1129 return exec_queue_set_property_funcs[idx](xe, q, ext.value);
1130}
1131
1132typedef int (*xe_exec_queue_user_extension_fn)(struct xe_device *xe,
1133 struct xe_exec_queue *q,
1134 u64 extension, u64 *properties);
1135
1136static const xe_exec_queue_user_extension_fn exec_queue_user_extension_funcs[] = {
1137 [DRM_XE_EXEC_QUEUE_EXTENSION_SET_PROPERTY] = exec_queue_user_ext_set_property,
1138};
1139
1140#define MAX_USER_EXTENSIONS 16
1141static int __exec_queue_user_extensions(struct xe_device *xe, struct xe_exec_queue *q,
1142 u64 extensions, int ext_number, u64 *properties)
1143{
1144 u64 __user *address = u64_to_user_ptr(extensions);
1145 struct drm_xe_user_extension ext;
1146 int err;
1147 u32 idx;
1148
1149 if (XE_IOCTL_DBG(xe, ext_number >= MAX_USER_EXTENSIONS))
1150 return -E2BIG;
1151
1152 err = copy_from_user(&ext, address, sizeof(ext));
1153 if (XE_IOCTL_DBG(xe, err))
1154 return -EFAULT;
1155
1156 if (XE_IOCTL_DBG(xe, ext.pad) ||
1157 XE_IOCTL_DBG(xe, ext.name >=
1158 ARRAY_SIZE(exec_queue_user_extension_funcs)))
1159 return -EINVAL;
1160
1161 idx = array_index_nospec(ext.name,
1162 ARRAY_SIZE(exec_queue_user_extension_funcs));
1163 err = exec_queue_user_extension_funcs[idx](xe, q, extensions, properties);
1164 if (XE_IOCTL_DBG(xe, err))
1165 return err;
1166
1167 if (ext.next_extension)
1168 return __exec_queue_user_extensions(xe, q, ext.next_extension,
1169 ++ext_number, properties);
1170
1171 return 0;
1172}
1173
1174static int exec_queue_user_extensions(struct xe_device *xe, struct xe_exec_queue *q,
1175 u64 extensions)
1176{
1177 u64 properties = 0;
1178 int err;
1179
1180 err = __exec_queue_user_extensions(xe, q, extensions, 0, &properties);
1181 if (XE_IOCTL_DBG(xe, err))
1182 return err;
1183
1184 err = exec_queue_user_ext_check_final(q, properties);
1185 if (XE_IOCTL_DBG(xe, err))
1186 return err;
1187
1188 if (xe_exec_queue_is_multi_queue_primary(q)) {
1189 err = xe_exec_queue_group_init(xe, q);
1190 if (XE_IOCTL_DBG(xe, err))
1191 return err;
1192 }
1193
1194 return 0;
1195}
1196
1197static u32 calc_validate_logical_mask(struct xe_device *xe,
1198 struct drm_xe_engine_class_instance *eci,
1199 u16 width, u16 num_placements)
1200{
1201 int len = width * num_placements;
1202 int i, j, n;
1203 u16 class;
1204 u16 gt_id;
1205 u32 return_mask = 0, prev_mask;
1206
1207 if (XE_IOCTL_DBG(xe, !xe_device_uc_enabled(xe) &&
1208 len > 1))
1209 return 0;
1210
1211 for (i = 0; i < width; ++i) {
1212 u32 current_mask = 0;
1213
1214 for (j = 0; j < num_placements; ++j) {
1215 struct xe_hw_engine *hwe;
1216
1217 n = j * width + i;
1218
1219 hwe = xe_hw_engine_lookup(xe, eci[n]);
1220 if (XE_IOCTL_DBG(xe, !hwe))
1221 return 0;
1222
1223 if (XE_IOCTL_DBG(xe, xe_hw_engine_is_reserved(hwe)))
1224 return 0;
1225
1226 if (XE_IOCTL_DBG(xe, n && eci[n].gt_id != gt_id) ||
1227 XE_IOCTL_DBG(xe, n && eci[n].engine_class != class))
1228 return 0;
1229
1230 class = eci[n].engine_class;
1231 gt_id = eci[n].gt_id;
1232
1233 if (width == 1 || !i)
1234 return_mask |= BIT(eci[n].engine_instance);
1235 current_mask |= BIT(eci[n].engine_instance);
1236 }
1237
1238 /* Parallel submissions must be logically contiguous */
1239 if (i && XE_IOCTL_DBG(xe, current_mask != prev_mask << 1))
1240 return 0;
1241
1242 prev_mask = current_mask;
1243 }
1244
1245 return return_mask;
1246}
1247
1248static bool has_sched_groups(struct xe_gt *gt)
1249{
1250 if (IS_SRIOV_PF(gt_to_xe(gt)) && xe_gt_sriov_pf_sched_groups_enabled(gt))
1251 return true;
1252
1253 if (IS_SRIOV_VF(gt_to_xe(gt)) && xe_gt_sriov_vf_sched_groups_enabled(gt))
1254 return true;
1255
1256 return false;
1257}
1258
1259/**
1260 * xe_exec_queue_create_ioctl() - Create an exec queue via IOCTL
1261 * @dev: DRM device
1262 * @data: IOCTL data
1263 * @file: DRM file
1264 *
1265 * Creates a new exec queue based on user-provided parameters. Supports
1266 * creating VM bind queues, regular exec queues, multi-lrc exec queues
1267 * and multi-queue groups.
1268 *
1269 * Return: 0 on success with exec_queue_id filled in, negative error code on failure
1270 */
1271int xe_exec_queue_create_ioctl(struct drm_device *dev, void *data,
1272 struct drm_file *file)
1273{
1274 struct xe_device *xe = to_xe_device(dev);
1275 struct xe_file *xef = to_xe_file(file);
1276 struct drm_xe_exec_queue_create *args = data;
1277 struct drm_xe_engine_class_instance eci[XE_HW_ENGINE_MAX_INSTANCE];
1278 struct drm_xe_engine_class_instance __user *user_eci =
1279 u64_to_user_ptr(args->instances);
1280 struct xe_hw_engine *hwe;
1281 struct xe_vm *vm;
1282 struct xe_tile *tile;
1283 struct xe_exec_queue *q = NULL;
1284 u32 logical_mask;
1285 u32 flags = 0;
1286 u32 id;
1287 u32 len;
1288 int err;
1289
1290 if (XE_IOCTL_DBG(xe, args->flags & ~DRM_XE_EXEC_QUEUE_LOW_LATENCY_HINT) ||
1291 XE_IOCTL_DBG(xe, args->reserved[0] || args->reserved[1]))
1292 return -EINVAL;
1293
1294 len = args->width * args->num_placements;
1295 if (XE_IOCTL_DBG(xe, !len || len > XE_HW_ENGINE_MAX_INSTANCE))
1296 return -EINVAL;
1297
1298 err = copy_from_user(eci, user_eci,
1299 sizeof(struct drm_xe_engine_class_instance) * len);
1300 if (XE_IOCTL_DBG(xe, err))
1301 return -EFAULT;
1302
1303 if (XE_IOCTL_DBG(xe, !xe_device_get_gt(xe, eci[0].gt_id)))
1304 return -EINVAL;
1305
1306 if (args->flags & DRM_XE_EXEC_QUEUE_LOW_LATENCY_HINT)
1307 flags |= EXEC_QUEUE_FLAG_LOW_LATENCY;
1308
1309 if (eci[0].engine_class == DRM_XE_ENGINE_CLASS_VM_BIND) {
1310 if (XE_IOCTL_DBG(xe, args->width != 1) ||
1311 XE_IOCTL_DBG(xe, args->num_placements != 1) ||
1312 XE_IOCTL_DBG(xe, eci[0].engine_instance != 0))
1313 return -EINVAL;
1314
1315 vm = xe_vm_lookup(xef, args->vm_id);
1316 if (XE_IOCTL_DBG(xe, !vm))
1317 return -ENOENT;
1318
1319 err = down_read_interruptible(&vm->lock);
1320 if (err) {
1321 xe_vm_put(vm);
1322 return err;
1323 }
1324
1325 if (XE_IOCTL_DBG(xe, xe_vm_is_closed_or_banned(vm))) {
1326 up_read(&vm->lock);
1327 xe_vm_put(vm);
1328 return -ENOENT;
1329 }
1330
1331 for_each_tile(tile, xe, id) {
1332 struct xe_exec_queue *new;
1333
1334 flags |= EXEC_QUEUE_FLAG_VM;
1335 if (id)
1336 flags |= EXEC_QUEUE_FLAG_BIND_ENGINE_CHILD;
1337
1338 new = xe_exec_queue_create_bind(xe, tile, vm, flags,
1339 args->extensions);
1340 if (IS_ERR(new)) {
1341 up_read(&vm->lock);
1342 xe_vm_put(vm);
1343 err = PTR_ERR(new);
1344 if (q)
1345 goto put_exec_queue;
1346 return err;
1347 }
1348 if (id == 0)
1349 q = new;
1350 else
1351 list_add_tail(&new->multi_gt_list,
1352 &q->multi_gt_link);
1353 }
1354 up_read(&vm->lock);
1355 xe_vm_put(vm);
1356 } else {
1357 logical_mask = calc_validate_logical_mask(xe, eci,
1358 args->width,
1359 args->num_placements);
1360 if (XE_IOCTL_DBG(xe, !logical_mask))
1361 return -EINVAL;
1362
1363 hwe = xe_hw_engine_lookup(xe, eci[0]);
1364 if (XE_IOCTL_DBG(xe, !hwe))
1365 return -EINVAL;
1366
1367 /* multi-lrc is only supported on select engine classes */
1368 if (XE_IOCTL_DBG(xe, args->width > 1 &&
1369 !(xe->info.multi_lrc_mask & BIT(hwe->class))))
1370 return -EOPNOTSUPP;
1371
1372 vm = xe_vm_lookup(xef, args->vm_id);
1373 if (XE_IOCTL_DBG(xe, !vm))
1374 return -ENOENT;
1375
1376 err = down_read_interruptible(&vm->lock);
1377 if (err) {
1378 xe_vm_put(vm);
1379 return err;
1380 }
1381
1382 if (XE_IOCTL_DBG(xe, xe_vm_is_closed_or_banned(vm))) {
1383 up_read(&vm->lock);
1384 xe_vm_put(vm);
1385 return -ENOENT;
1386 }
1387
1388 /* SRIOV sched groups are not compatible with multi-lrc */
1389 if (XE_IOCTL_DBG(xe, args->width > 1 && has_sched_groups(hwe->gt))) {
1390 up_read(&vm->lock);
1391 xe_vm_put(vm);
1392 return -EINVAL;
1393 }
1394
1395 q = xe_exec_queue_create(xe, vm, logical_mask,
1396 args->width, hwe, flags,
1397 args->extensions);
1398 up_read(&vm->lock);
1399 xe_vm_put(vm);
1400 if (IS_ERR(q))
1401 return PTR_ERR(q);
1402
1403 if (xe_exec_queue_is_multi_queue_secondary(q)) {
1404 err = xe_exec_queue_group_add(xe, q);
1405 if (XE_IOCTL_DBG(xe, err))
1406 goto put_exec_queue;
1407 }
1408
1409 if (xe_vm_in_preempt_fence_mode(vm)) {
1410 q->lr.context = dma_fence_context_alloc(1);
1411
1412 err = xe_vm_add_compute_exec_queue(vm, q);
1413 if (XE_IOCTL_DBG(xe, err))
1414 goto delete_queue_group;
1415 }
1416
1417 if (q->vm && q->hwe->hw_engine_group) {
1418 err = xe_hw_engine_group_add_exec_queue(q->hwe->hw_engine_group, q);
1419 if (err)
1420 goto kill_exec_queue;
1421 }
1422 }
1423
1424 q->xef = xe_file_get(xef);
1425 if (eci[0].engine_class != DRM_XE_ENGINE_CLASS_VM_BIND)
1426 xe_vm_add_exec_queue(vm, q);
1427
1428 /* user id alloc must always be last in ioctl to prevent UAF */
1429 err = xa_alloc(&xef->exec_queue.xa, &id, q, xa_limit_32b, GFP_KERNEL);
1430 if (err)
1431 goto del_hw_engine_group;
1432
1433 args->exec_queue_id = id;
1434
1435 return 0;
1436
1437del_hw_engine_group:
1438 if (q->vm && q->hwe && q->hwe->hw_engine_group)
1439 xe_hw_engine_group_del_exec_queue(q->hwe->hw_engine_group, q);
1440kill_exec_queue:
1441 xe_exec_queue_kill(q);
1442delete_queue_group:
1443 if (xe_exec_queue_is_multi_queue_secondary(q))
1444 xe_exec_queue_group_delete(xe, q);
1445put_exec_queue:
1446 xe_exec_queue_put(q);
1447 return err;
1448}
1449
1450/**
1451 * xe_exec_queue_get_property_ioctl() - Get a property from an exec queue
1452 * @dev: DRM device
1453 * @data: IOCTL data
1454 * @file: DRM file
1455 *
1456 * Retrieves property values from an existing exec queue. Currently supports
1457 * getting the ban/reset status.
1458 *
1459 * Return: 0 on success with value filled in, negative error code on failure
1460 */
1461int xe_exec_queue_get_property_ioctl(struct drm_device *dev, void *data,
1462 struct drm_file *file)
1463{
1464 struct xe_device *xe = to_xe_device(dev);
1465 struct xe_file *xef = to_xe_file(file);
1466 struct drm_xe_exec_queue_get_property *args = data;
1467 struct xe_exec_queue *q;
1468 int ret;
1469
1470 if (XE_IOCTL_DBG(xe, args->reserved[0] || args->reserved[1]))
1471 return -EINVAL;
1472
1473 q = xe_exec_queue_lookup(xef, args->exec_queue_id);
1474 if (XE_IOCTL_DBG(xe, !q))
1475 return -ENOENT;
1476
1477 switch (args->property) {
1478 case DRM_XE_EXEC_QUEUE_GET_PROPERTY_BAN:
1479 args->value = q->ops->reset_status(q);
1480 ret = 0;
1481 break;
1482 default:
1483 ret = -EINVAL;
1484 }
1485
1486 xe_exec_queue_put(q);
1487
1488 return ret;
1489}
1490
1491/**
1492 * xe_exec_queue_is_lr() - Whether an exec_queue is long-running
1493 * @q: The exec_queue
1494 *
1495 * Return: True if the exec_queue is long-running, false otherwise.
1496 */
1497bool xe_exec_queue_is_lr(struct xe_exec_queue *q)
1498{
1499 return q->vm && xe_vm_in_lr_mode(q->vm) &&
1500 !(q->flags & EXEC_QUEUE_FLAG_VM);
1501}
1502
1503/**
1504 * xe_exec_queue_is_idle() - Whether an exec_queue is idle.
1505 * @q: The exec_queue
1506 *
1507 * FIXME: Need to determine what to use as the short-lived
1508 * timeline lock for the exec_queues, so that the return value
1509 * of this function becomes more than just an advisory
1510 * snapshot in time. The timeline lock must protect the
1511 * seqno from racing submissions on the same exec_queue.
1512 * Typically vm->resv, but user-created timeline locks use the migrate vm
1513 * and never grabs the migrate vm->resv so we have a race there.
1514 *
1515 * Return: True if the exec_queue is idle, false otherwise.
1516 */
1517bool xe_exec_queue_is_idle(struct xe_exec_queue *q)
1518{
1519 if (xe_exec_queue_is_parallel(q)) {
1520 int i;
1521
1522 for (i = 0; i < q->width; ++i) {
1523 if (xe_lrc_seqno(q->lrc[i]) !=
1524 q->lrc[i]->fence_ctx.next_seqno - 1)
1525 return false;
1526 }
1527
1528 return true;
1529 }
1530
1531 return xe_lrc_seqno(q->lrc[0]) ==
1532 q->lrc[0]->fence_ctx.next_seqno - 1;
1533}
1534
1535/**
1536 * xe_exec_queue_update_run_ticks() - Update run time in ticks for this exec queue
1537 * from hw
1538 * @q: The exec queue
1539 *
1540 * Update the timestamp saved by HW for this exec queue and save run ticks
1541 * calculated by using the delta from last update.
1542 */
1543void xe_exec_queue_update_run_ticks(struct xe_exec_queue *q)
1544{
1545 struct xe_device *xe = gt_to_xe(q->gt);
1546 struct xe_lrc *lrc;
1547 u64 old_ts, new_ts;
1548 int idx;
1549
1550 /*
1551 * Jobs that are executed by kernel doesn't have a corresponding xe_file
1552 * and thus are not accounted.
1553 */
1554 if (!q->xef)
1555 return;
1556
1557 /* Synchronize with unbind while holding the xe file open */
1558 if (!drm_dev_enter(&xe->drm, &idx))
1559 return;
1560 /*
1561 * Only sample the first LRC. For parallel submission, all of them are
1562 * scheduled together and we compensate that below by multiplying by
1563 * width - this may introduce errors if that premise is not true and
1564 * they don't exit 100% aligned. On the other hand, looping through
1565 * the LRCs and reading them in different time could also introduce
1566 * errors.
1567 */
1568 lrc = q->lrc[0];
1569 new_ts = xe_lrc_update_timestamp(lrc, &old_ts);
1570 q->xef->run_ticks[q->class] += (new_ts - old_ts) * q->width;
1571
1572 drm_dev_exit(idx);
1573}
1574
1575/**
1576 * xe_exec_queue_kill - permanently stop all execution from an exec queue
1577 * @q: The exec queue
1578 *
1579 * This function permanently stops all activity on an exec queue. If the queue
1580 * is actively executing on the HW, it will be kicked off the engine; any
1581 * pending jobs are discarded and all future submissions are rejected.
1582 * This function is safe to call multiple times.
1583 */
1584void xe_exec_queue_kill(struct xe_exec_queue *q)
1585{
1586 struct xe_exec_queue *eq = q, *next;
1587
1588 list_for_each_entry_safe(eq, next, &eq->multi_gt_list,
1589 multi_gt_link) {
1590 q->ops->kill(eq);
1591 xe_vm_remove_compute_exec_queue(q->vm, eq);
1592 }
1593
1594 q->ops->kill(q);
1595 xe_vm_remove_compute_exec_queue(q->vm, q);
1596}
1597
1598/**
1599 * xe_exec_queue_destroy_ioctl() - Destroy an exec queue via IOCTL
1600 * @dev: DRM device
1601 * @data: IOCTL data
1602 * @file: DRM file
1603 *
1604 * Destroys an existing exec queue and releases its reference.
1605 *
1606 * Return: 0 on success, negative error code on failure
1607 */
1608int xe_exec_queue_destroy_ioctl(struct drm_device *dev, void *data,
1609 struct drm_file *file)
1610{
1611 struct xe_device *xe = to_xe_device(dev);
1612 struct xe_file *xef = to_xe_file(file);
1613 struct drm_xe_exec_queue_destroy *args = data;
1614 struct xe_exec_queue *q;
1615
1616 if (XE_IOCTL_DBG(xe, args->pad) ||
1617 XE_IOCTL_DBG(xe, args->reserved[0] || args->reserved[1]))
1618 return -EINVAL;
1619
1620 mutex_lock(&xef->exec_queue.lock);
1621 q = xa_erase(&xef->exec_queue.xa, args->exec_queue_id);
1622 if (q)
1623 atomic_inc(&xef->exec_queue.pending_removal);
1624 mutex_unlock(&xef->exec_queue.lock);
1625
1626 if (XE_IOCTL_DBG(xe, !q))
1627 return -ENOENT;
1628
1629 if (q->vm && q->hwe->hw_engine_group)
1630 xe_hw_engine_group_del_exec_queue(q->hwe->hw_engine_group, q);
1631
1632 xe_exec_queue_kill(q);
1633
1634 trace_xe_exec_queue_close(q);
1635 xe_exec_queue_put(q);
1636
1637 return 0;
1638}
1639
1640static void xe_exec_queue_last_fence_lockdep_assert(struct xe_exec_queue *q,
1641 struct xe_vm *vm)
1642{
1643 if (q->flags & EXEC_QUEUE_FLAG_MIGRATE) {
1644 xe_migrate_job_lock_assert(q);
1645 } else if (q->flags & EXEC_QUEUE_FLAG_VM) {
1646 lockdep_assert_held(&vm->lock);
1647 } else {
1648 xe_vm_assert_held(vm);
1649 lockdep_assert_held(&q->hwe->hw_engine_group->mode_sem);
1650 }
1651}
1652
1653/**
1654 * xe_exec_queue_last_fence_put() - Drop ref to last fence
1655 * @q: The exec queue
1656 * @vm: The VM the engine does a bind or exec for
1657 */
1658void xe_exec_queue_last_fence_put(struct xe_exec_queue *q, struct xe_vm *vm)
1659{
1660 xe_exec_queue_last_fence_lockdep_assert(q, vm);
1661
1662 xe_exec_queue_last_fence_put_unlocked(q);
1663}
1664
1665/**
1666 * xe_exec_queue_last_fence_put_unlocked() - Drop ref to last fence unlocked
1667 * @q: The exec queue
1668 *
1669 * Only safe to be called from xe_exec_queue_destroy().
1670 */
1671void xe_exec_queue_last_fence_put_unlocked(struct xe_exec_queue *q)
1672{
1673 if (q->last_fence) {
1674 dma_fence_put(q->last_fence);
1675 q->last_fence = NULL;
1676 }
1677}
1678
1679/**
1680 * xe_exec_queue_last_fence_get() - Get last fence
1681 * @q: The exec queue
1682 * @vm: The VM the engine does a bind or exec for
1683 *
1684 * Get last fence, takes a ref
1685 *
1686 * Returns: last fence if not signaled, dma fence stub if signaled
1687 */
1688struct dma_fence *xe_exec_queue_last_fence_get(struct xe_exec_queue *q,
1689 struct xe_vm *vm)
1690{
1691 struct dma_fence *fence;
1692
1693 xe_exec_queue_last_fence_lockdep_assert(q, vm);
1694
1695 if (q->last_fence &&
1696 test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &q->last_fence->flags))
1697 xe_exec_queue_last_fence_put(q, vm);
1698
1699 fence = q->last_fence ? q->last_fence : dma_fence_get_stub();
1700 dma_fence_get(fence);
1701 return fence;
1702}
1703
1704/**
1705 * xe_exec_queue_last_fence_get_for_resume() - Get last fence
1706 * @q: The exec queue
1707 * @vm: The VM the engine does a bind or exec for
1708 *
1709 * Get last fence, takes a ref. Only safe to be called in the context of
1710 * resuming the hw engine group's long-running exec queue, when the group
1711 * semaphore is held.
1712 *
1713 * Returns: last fence if not signaled, dma fence stub if signaled
1714 */
1715struct dma_fence *xe_exec_queue_last_fence_get_for_resume(struct xe_exec_queue *q,
1716 struct xe_vm *vm)
1717{
1718 struct dma_fence *fence;
1719
1720 lockdep_assert_held_write(&q->hwe->hw_engine_group->mode_sem);
1721
1722 if (q->last_fence &&
1723 test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &q->last_fence->flags))
1724 xe_exec_queue_last_fence_put_unlocked(q);
1725
1726 fence = q->last_fence ? q->last_fence : dma_fence_get_stub();
1727 dma_fence_get(fence);
1728 return fence;
1729}
1730
1731/**
1732 * xe_exec_queue_last_fence_set() - Set last fence
1733 * @q: The exec queue
1734 * @vm: The VM the engine does a bind or exec for
1735 * @fence: The fence
1736 *
1737 * Set the last fence for the engine. Increases reference count for fence, when
1738 * closing engine xe_exec_queue_last_fence_put should be called.
1739 */
1740void xe_exec_queue_last_fence_set(struct xe_exec_queue *q, struct xe_vm *vm,
1741 struct dma_fence *fence)
1742{
1743 xe_exec_queue_last_fence_lockdep_assert(q, vm);
1744 xe_assert(vm->xe, !dma_fence_is_container(fence));
1745
1746 xe_exec_queue_last_fence_put(q, vm);
1747 q->last_fence = dma_fence_get(fence);
1748}
1749
1750/**
1751 * xe_exec_queue_tlb_inval_last_fence_put() - Drop ref to last TLB invalidation fence
1752 * @q: The exec queue
1753 * @vm: The VM the engine does a bind for
1754 * @type: Either primary or media GT
1755 */
1756void xe_exec_queue_tlb_inval_last_fence_put(struct xe_exec_queue *q,
1757 struct xe_vm *vm,
1758 unsigned int type)
1759{
1760 xe_exec_queue_last_fence_lockdep_assert(q, vm);
1761 xe_assert(vm->xe, type == XE_EXEC_QUEUE_TLB_INVAL_MEDIA_GT ||
1762 type == XE_EXEC_QUEUE_TLB_INVAL_PRIMARY_GT);
1763
1764 xe_exec_queue_tlb_inval_last_fence_put_unlocked(q, type);
1765}
1766
1767/**
1768 * xe_exec_queue_tlb_inval_last_fence_put_unlocked() - Drop ref to last TLB
1769 * invalidation fence unlocked
1770 * @q: The exec queue
1771 * @type: Either primary or media GT
1772 *
1773 * Only safe to be called from xe_exec_queue_destroy().
1774 */
1775void xe_exec_queue_tlb_inval_last_fence_put_unlocked(struct xe_exec_queue *q,
1776 unsigned int type)
1777{
1778 xe_assert(gt_to_xe(q->gt), type == XE_EXEC_QUEUE_TLB_INVAL_MEDIA_GT ||
1779 type == XE_EXEC_QUEUE_TLB_INVAL_PRIMARY_GT);
1780
1781 dma_fence_put(q->tlb_inval[type].last_fence);
1782 q->tlb_inval[type].last_fence = NULL;
1783}
1784
1785/**
1786 * xe_exec_queue_tlb_inval_last_fence_get() - Get last fence for TLB invalidation
1787 * @q: The exec queue
1788 * @vm: The VM the engine does a bind for
1789 * @type: Either primary or media GT
1790 *
1791 * Get last fence, takes a ref
1792 *
1793 * Returns: last fence if not signaled, dma fence stub if signaled
1794 */
1795struct dma_fence *xe_exec_queue_tlb_inval_last_fence_get(struct xe_exec_queue *q,
1796 struct xe_vm *vm,
1797 unsigned int type)
1798{
1799 struct dma_fence *fence;
1800
1801 xe_exec_queue_last_fence_lockdep_assert(q, vm);
1802 xe_assert(vm->xe, type == XE_EXEC_QUEUE_TLB_INVAL_MEDIA_GT ||
1803 type == XE_EXEC_QUEUE_TLB_INVAL_PRIMARY_GT);
1804 xe_assert(vm->xe, q->flags & (EXEC_QUEUE_FLAG_VM |
1805 EXEC_QUEUE_FLAG_MIGRATE));
1806
1807 if (q->tlb_inval[type].last_fence &&
1808 test_bit(DMA_FENCE_FLAG_SIGNALED_BIT,
1809 &q->tlb_inval[type].last_fence->flags))
1810 xe_exec_queue_tlb_inval_last_fence_put(q, vm, type);
1811
1812 fence = q->tlb_inval[type].last_fence ?: dma_fence_get_stub();
1813 dma_fence_get(fence);
1814 return fence;
1815}
1816
1817/**
1818 * xe_exec_queue_tlb_inval_last_fence_set() - Set last fence for TLB invalidation
1819 * @q: The exec queue
1820 * @vm: The VM the engine does a bind for
1821 * @fence: The fence
1822 * @type: Either primary or media GT
1823 *
1824 * Set the last fence for the tlb invalidation type on the queue. Increases
1825 * reference count for fence, when closing queue
1826 * xe_exec_queue_tlb_inval_last_fence_put should be called.
1827 */
1828void xe_exec_queue_tlb_inval_last_fence_set(struct xe_exec_queue *q,
1829 struct xe_vm *vm,
1830 struct dma_fence *fence,
1831 unsigned int type)
1832{
1833 xe_exec_queue_last_fence_lockdep_assert(q, vm);
1834 xe_assert(vm->xe, type == XE_EXEC_QUEUE_TLB_INVAL_MEDIA_GT ||
1835 type == XE_EXEC_QUEUE_TLB_INVAL_PRIMARY_GT);
1836 xe_assert(vm->xe, q->flags & (EXEC_QUEUE_FLAG_VM |
1837 EXEC_QUEUE_FLAG_MIGRATE));
1838 xe_assert(vm->xe, !dma_fence_is_container(fence));
1839
1840 xe_exec_queue_tlb_inval_last_fence_put(q, vm, type);
1841 q->tlb_inval[type].last_fence = dma_fence_get(fence);
1842}
1843
1844/**
1845 * xe_exec_queue_contexts_hwsp_rebase - Re-compute GGTT references
1846 * within all LRCs of a queue.
1847 * @q: the &xe_exec_queue struct instance containing target LRCs
1848 * @scratch: scratch buffer to be used as temporary storage
1849 *
1850 * Returns: zero on success, negative error code on failure
1851 */
1852int xe_exec_queue_contexts_hwsp_rebase(struct xe_exec_queue *q, void *scratch)
1853{
1854 int i;
1855 int err = 0;
1856
1857 for (i = 0; i < q->width; ++i) {
1858 struct xe_lrc *lrc;
1859
1860 lrc = xe_exec_queue_get_lrc(q, i);
1861 if (!lrc)
1862 continue;
1863
1864 xe_lrc_update_memirq_regs_with_address(lrc, q->hwe, scratch);
1865 xe_lrc_update_hwctx_regs_with_address(lrc);
1866 err = xe_lrc_setup_wa_bb_with_scratch(lrc, q->hwe, scratch);
1867 xe_lrc_put(lrc);
1868 if (err)
1869 break;
1870 }
1871
1872 return err;
1873}