projects/linux·blob master · refs

379 lines · 11.0 KiB

1// SPDX-License-Identifier: MIT
2/*
3 * Copyright © 2022 Intel Corporation
4 */
5
6#include "xe_exec.h"
7
8#include <drm/drm_device.h>
9#include <drm/drm_exec.h>
10#include <drm/drm_file.h>
11#include <uapi/drm/xe_drm.h>
12#include <linux/delay.h>
13
14#include "xe_device.h"
15#include "xe_exec_queue.h"
16#include "xe_hw_engine_group.h"
17#include "xe_macros.h"
18#include "xe_pm.h"
19#include "xe_ring_ops_types.h"
20#include "xe_sched_job.h"
21#include "xe_sync.h"
22#include "xe_svm.h"
23#include "xe_trace.h"
24#include "xe_vm.h"
25
26/**
27 * DOC: Execbuf (User GPU command submission)
28 *
29 * Execs have historically been rather complicated in DRM drivers (at least in
30 * the i915) because a few things:
31 *
32 * - Passing in a list BO which are read / written to creating implicit syncs
33 * - Binding at exec time
34 * - Flow controlling the ring at exec time
35 *
36 * In Xe we avoid all of this complication by not allowing a BO list to be
37 * passed into an exec, using the dma-buf implicit sync uAPI, have binds as
38 * separate operations, and using the DRM scheduler to flow control the ring.
39 * Let's deep dive on each of these.
40 *
41 * We can get away from a BO list by forcing the user to use in / out fences on
42 * every exec rather than the kernel tracking dependencies of BO (e.g. if the
43 * user knows an exec writes to a BO and reads from the BO in the next exec, it
44 * is the user's responsibility to pass in / out fence between the two execs).
45 *
46 * We do not allow a user to trigger a bind at exec time rather we have a VM
47 * bind IOCTL which uses the same in / out fence interface as exec. In that
48 * sense, a VM bind is basically the same operation as an exec from the user
49 * perspective. e.g. If an exec depends on a VM bind use the in / out fence
50 * interface (struct drm_xe_sync) to synchronize like syncing between two
51 * dependent execs.
52 *
53 * Although a user cannot trigger a bind, we still have to rebind userptrs in
54 * the VM that have been invalidated since the last exec, likewise we also have
55 * to rebind BOs that have been evicted by the kernel. We schedule these rebinds
56 * behind any pending kernel operations on any external BOs in VM or any BOs
57 * private to the VM. This is accomplished by the rebinds waiting on BOs
58 * DMA_RESV_USAGE_KERNEL slot (kernel ops) and kernel ops waiting on all BOs
59 * slots (inflight execs are in the DMA_RESV_USAGE_BOOKKEEP for private BOs and
60 * for external BOs).
61 *
62 * Rebinds / dma-resv usage applies to non-compute mode VMs only as for compute
63 * mode VMs we use preempt fences and a rebind worker (TODO: add link).
64 *
65 * There is no need to flow control the ring in the exec as we write the ring at
66 * submission time and set the DRM scheduler max job limit SIZE_OF_RING /
67 * MAX_JOB_SIZE. The DRM scheduler will then hold all jobs until space in the
68 * ring is available.
69 *
70 * All of this results in a rather simple exec implementation.
71 *
72 * Flow
73 * ~~~~
74 *
75 * .. code-block::
76 *
77 * Parse input arguments
78 * Wait for any async VM bind passed as in-fences to start
79 * <----------------------------------------------------------------------|
80 * Lock global VM lock in read mode |
81 * Pin userptrs (also finds userptr invalidated since last exec) |
82 * Lock exec (VM dma-resv lock, external BOs dma-resv locks) |
83 * Validate BOs that have been evicted |
84 * Create job |
85 * Rebind invalidated userptrs + evicted BOs (non-compute-mode) |
86 * Add rebind fence dependency to job |
87 * Add job VM dma-resv bookkeeping slot (non-compute mode) |
88 * Add job to external BOs dma-resv write slots (non-compute mode) |
89 * Check if any userptrs invalidated since pin ------ Drop locks ---------|
90 * Install in / out fences for job
91 * Submit job
92 * Unlock all
93 */
94
95/*
96 * Add validation and rebinding to the drm_exec locking loop, since both can
97 * trigger eviction which may require sleeping dma_resv locks.
98 */
99static int xe_exec_fn(struct drm_gpuvm_exec *vm_exec)
100{
101 struct xe_vm *vm = container_of(vm_exec->vm, struct xe_vm, gpuvm);
102 int ret;
103
104 /* The fence slot added here is intended for the exec sched job. */
105 xe_vm_set_validation_exec(vm, &vm_exec->exec);
106 ret = xe_vm_validate_rebind(vm, &vm_exec->exec, 1);
107 xe_vm_set_validation_exec(vm, NULL);
108 return ret;
109}
110
111int xe_exec_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
112{
113 struct xe_device *xe = to_xe_device(dev);
114 struct xe_file *xef = to_xe_file(file);
115 struct drm_xe_exec *args = data;
116 struct drm_xe_sync __user *syncs_user = u64_to_user_ptr(args->syncs);
117 u64 __user *addresses_user = u64_to_user_ptr(args->address);
118 struct xe_exec_queue *q;
119 struct xe_sync_entry *syncs = NULL;
120 u64 addresses[XE_HW_ENGINE_MAX_INSTANCE];
121 struct drm_gpuvm_exec vm_exec = {.extra.fn = xe_exec_fn};
122 struct drm_exec *exec = &vm_exec.exec;
123 u32 i, num_syncs, num_in_sync = 0, num_ufence = 0;
124 struct xe_validation_ctx ctx;
125 struct xe_sched_job *job;
126 struct xe_vm *vm;
127 bool write_locked;
128 int err = 0;
129 struct xe_hw_engine_group *group;
130 enum xe_hw_engine_group_execution_mode mode, previous_mode;
131
132 if (XE_IOCTL_DBG(xe, args->extensions) ||
133 XE_IOCTL_DBG(xe, args->pad[0] || args->pad[1] || args->pad[2]) ||
134 XE_IOCTL_DBG(xe, args->reserved[0] || args->reserved[1]) ||
135 XE_IOCTL_DBG(xe, args->num_syncs > DRM_XE_MAX_SYNCS))
136 return -EINVAL;
137
138 q = xe_exec_queue_lookup(xef, args->exec_queue_id);
139 if (XE_IOCTL_DBG(xe, !q))
140 return -ENOENT;
141
142 if (XE_IOCTL_DBG(xe, q->flags & EXEC_QUEUE_FLAG_VM)) {
143 err = -EINVAL;
144 goto err_exec_queue;
145 }
146
147 if (XE_IOCTL_DBG(xe, args->num_batch_buffer &&
148 q->width != args->num_batch_buffer)) {
149 err = -EINVAL;
150 goto err_exec_queue;
151 }
152
153 if (XE_IOCTL_DBG(xe, q->ops->reset_status(q))) {
154 err = -ECANCELED;
155 goto err_exec_queue;
156 }
157
158 if (atomic_read(&q->job_cnt) >= XE_MAX_JOB_COUNT_PER_EXEC_QUEUE) {
159 trace_xe_exec_queue_reach_max_job_count(q, XE_MAX_JOB_COUNT_PER_EXEC_QUEUE);
160 err = -EAGAIN;
161 goto err_exec_queue;
162 }
163
164 if (args->num_syncs) {
165 syncs = kzalloc_objs(*syncs, args->num_syncs);
166 if (!syncs) {
167 err = -ENOMEM;
168 goto err_exec_queue;
169 }
170 }
171
172 vm = q->vm;
173
174 for (num_syncs = 0; num_syncs < args->num_syncs; num_syncs++) {
175 err = xe_sync_entry_parse(xe, xef, &syncs[num_syncs],
176 &syncs_user[num_syncs], NULL, 0,
177 SYNC_PARSE_FLAG_EXEC |
178 (xe_vm_in_lr_mode(vm) ?
179 SYNC_PARSE_FLAG_LR_MODE : 0));
180 if (err)
181 goto err_syncs;
182
183 if (xe_sync_is_ufence(&syncs[num_syncs]))
184 num_ufence++;
185
186 if (!num_in_sync && xe_sync_needs_wait(&syncs[num_syncs]))
187 num_in_sync++;
188 }
189
190 if (XE_IOCTL_DBG(xe, num_ufence > 1)) {
191 err = -EINVAL;
192 goto err_syncs;
193 }
194
195 if (args->num_batch_buffer && xe_exec_queue_is_parallel(q)) {
196 err = copy_from_user(addresses, addresses_user,
197 sizeof(u64) * q->width);
198 if (err) {
199 err = -EFAULT;
200 goto err_syncs;
201 }
202 }
203
204 group = q->hwe->hw_engine_group;
205 mode = xe_hw_engine_group_find_exec_mode(q);
206
207 if (mode == EXEC_MODE_DMA_FENCE) {
208 err = xe_hw_engine_group_get_mode(group, mode, &previous_mode,
209 syncs, num_in_sync ?
210 num_syncs : 0);
211 if (err)
212 goto err_syncs;
213 }
214
215retry:
216 if (!xe_vm_in_lr_mode(vm) && xe_vm_userptr_check_repin(vm)) {
217 err = down_write_killable(&vm->lock);
218 write_locked = true;
219 } else {
220 /* We don't allow execs while the VM is in error state */
221 err = down_read_interruptible(&vm->lock);
222 write_locked = false;
223 }
224 if (err)
225 goto err_hw_exec_mode;
226
227 if (write_locked) {
228 err = xe_vm_userptr_pin(vm);
229 downgrade_write(&vm->lock);
230 write_locked = false;
231 if (err)
232 goto err_unlock_list;
233 }
234
235 if (!args->num_batch_buffer) {
236 err = xe_vm_lock(vm, true);
237 if (err)
238 goto err_unlock_list;
239
240 if (!xe_vm_in_lr_mode(vm)) {
241 struct dma_fence *fence;
242
243 fence = xe_sync_in_fence_get(syncs, num_syncs, q, vm);
244 if (IS_ERR(fence)) {
245 err = PTR_ERR(fence);
246 xe_vm_unlock(vm);
247 goto err_unlock_list;
248 }
249 for (i = 0; i < num_syncs; i++)
250 xe_sync_entry_signal(&syncs[i], fence);
251 xe_exec_queue_last_fence_set(q, vm, fence);
252 dma_fence_put(fence);
253 }
254
255 xe_vm_unlock(vm);
256 goto err_unlock_list;
257 }
258
259 /*
260 * It's OK to block interruptible here with the vm lock held, since
261 * on task freezing during suspend / hibernate, the call will
262 * return -ERESTARTSYS and the IOCTL will be rerun.
263 */
264 err = xe_pm_block_on_suspend(xe);
265 if (err)
266 goto err_unlock_list;
267
268 if (!xe_vm_in_lr_mode(vm)) {
269 vm_exec.vm = &vm->gpuvm;
270 vm_exec.flags = DRM_EXEC_INTERRUPTIBLE_WAIT;
271 err = xe_validation_exec_lock(&ctx, &vm_exec, &xe->val);
272 if (err)
273 goto err_unlock_list;
274 }
275
276 if (xe_vm_is_closed_or_banned(q->vm)) {
277 drm_warn(&xe->drm, "Trying to schedule after vm is closed or banned\n");
278 err = -ECANCELED;
279 goto err_exec;
280 }
281
282 if (xe_exec_queue_uses_pxp(q)) {
283 err = xe_vm_validate_protected(q->vm);
284 if (err)
285 goto err_exec;
286 }
287
288 job = xe_sched_job_create(q, xe_exec_queue_is_parallel(q) ?
289 addresses : &args->address);
290 if (IS_ERR(job)) {
291 err = PTR_ERR(job);
292 goto err_exec;
293 }
294
295 /*
296 * Wait behind rebinds and any kernel operations (evictions, defrag
297 * moves, ...) on the VM and all external BOs. The VM's private BOs
298 * carry their kernel ops in the VM dma-resv KERNEL slot, while each
299 * external BO carries them in its own dma-resv KERNEL slot; both are
300 * covered by iterating every object locked by the exec, mirroring the
301 * drm_gpuvm_resv_add_fence() below.
302 */
303 if (!xe_vm_in_lr_mode(vm)) {
304 struct drm_gem_object *obj;
305
306 drm_exec_for_each_locked_object(exec, obj) {
307 err = xe_sched_job_add_deps(job, obj->resv,
308 DMA_RESV_USAGE_KERNEL);
309 if (err)
310 goto err_put_job;
311 }
312 }
313
314 for (i = 0; i < num_syncs && !err; i++)
315 err = xe_sync_entry_add_deps(&syncs[i], job);
316 if (err)
317 goto err_put_job;
318
319 if (!xe_vm_in_lr_mode(vm)) {
320 err = xe_svm_notifier_lock_interruptible(vm);
321 if (err)
322 goto err_put_job;
323
324 err = __xe_vm_userptr_needs_repin(vm);
325 if (err)
326 goto err_repin;
327 }
328
329 /*
330 * Point of no return, if we error after this point just set an error on
331 * the job and let the DRM scheduler / backend clean up the job.
332 */
333 xe_sched_job_arm(job);
334 if (!xe_vm_in_lr_mode(vm))
335 drm_gpuvm_resv_add_fence(&vm->gpuvm, exec, &job->drm.s_fence->finished,
336 DMA_RESV_USAGE_BOOKKEEP,
337 DMA_RESV_USAGE_BOOKKEEP);
338
339 for (i = 0; i < num_syncs; i++) {
340 xe_sync_entry_signal(&syncs[i], &job->drm.s_fence->finished);
341 xe_sched_job_init_user_fence(job, &syncs[i]);
342 }
343
344 if (!xe_vm_in_lr_mode(vm))
345 xe_exec_queue_last_fence_set(q, vm, &job->drm.s_fence->finished);
346 xe_sched_job_push(job);
347 xe_vm_reactivate_rebind(vm);
348
349 if (!err && !xe_vm_in_lr_mode(vm))
350 xe_vm_bump_lru(vm, exec);
351
352 if (mode == EXEC_MODE_LR)
353 xe_hw_engine_group_resume_faulting_lr_jobs(group);
354
355err_repin:
356 if (!xe_vm_in_lr_mode(vm))
357 xe_svm_notifier_unlock(vm);
358err_put_job:
359 if (err)
360 xe_sched_job_put(job);
361err_exec:
362 if (!xe_vm_in_lr_mode(vm))
363 xe_validation_ctx_fini(&ctx);
364err_unlock_list:
365 up_read(&vm->lock);
366 if (err == -EAGAIN)
367 goto retry;
368err_hw_exec_mode:
369 if (mode == EXEC_MODE_DMA_FENCE)
370 xe_hw_engine_group_put(group);
371err_syncs:
372 while (num_syncs--)
373 xe_sync_entry_cleanup(&syncs[num_syncs]);
374 kfree(syncs);
375err_exec_queue:
376 xe_exec_queue_put(q);
377
378 return err;
379}