projects/linux·blob master · refs

446 lines · 10.6 KiB

1// SPDX-License-Identifier: MIT
2/*
3 * Copyright © 2021 Intel Corporation
4 */
5
6#include "xe_sync.h"
7
8#include <linux/dma-fence-array.h>
9#include <linux/kthread.h>
10#include <linux/sched/mm.h>
11#include <linux/uaccess.h>
12
13#include <drm/drm_print.h>
14#include <drm/drm_syncobj.h>
15#include <uapi/drm/xe_drm.h>
16
17#include "xe_device.h"
18#include "xe_exec_queue.h"
19#include "xe_macros.h"
20#include "xe_sched_job_types.h"
21
22struct xe_user_fence {
23 struct xe_device *xe;
24 struct kref refcount;
25 struct dma_fence_cb cb;
26 struct work_struct worker;
27 struct mm_struct *mm;
28 u64 __user *addr;
29 u64 value;
30 int signalled;
31};
32
33static void user_fence_destroy(struct kref *kref)
34{
35 struct xe_user_fence *ufence = container_of(kref, struct xe_user_fence,
36 refcount);
37
38 mmdrop(ufence->mm);
39 kfree(ufence);
40}
41
42static void user_fence_get(struct xe_user_fence *ufence)
43{
44 kref_get(&ufence->refcount);
45}
46
47static void user_fence_put(struct xe_user_fence *ufence)
48{
49 kref_put(&ufence->refcount, user_fence_destroy);
50}
51
52static struct xe_user_fence *user_fence_create(struct xe_device *xe, u64 addr,
53 u64 value)
54{
55 struct xe_user_fence *ufence;
56 u64 __user *ptr = u64_to_user_ptr(addr);
57 u64 __maybe_unused prefetch_val;
58
59 if (get_user(prefetch_val, ptr))
60 return ERR_PTR(-EFAULT);
61
62 ufence = kzalloc_obj(*ufence);
63 if (!ufence)
64 return ERR_PTR(-ENOMEM);
65
66 ufence->xe = xe;
67 kref_init(&ufence->refcount);
68 ufence->addr = ptr;
69 ufence->value = value;
70 ufence->mm = current->mm;
71 mmgrab(ufence->mm);
72
73 return ufence;
74}
75
76static void user_fence_worker(struct work_struct *w)
77{
78 struct xe_user_fence *ufence = container_of(w, struct xe_user_fence, worker);
79
80 WRITE_ONCE(ufence->signalled, 1);
81 if (mmget_not_zero(ufence->mm)) {
82 kthread_use_mm(ufence->mm);
83 if (copy_to_user(ufence->addr, &ufence->value, sizeof(ufence->value)))
84 XE_WARN_ON("Copy to user failed");
85 kthread_unuse_mm(ufence->mm);
86 mmput(ufence->mm);
87 } else {
88 drm_dbg(&ufence->xe->drm, "mmget_not_zero() failed, ufence wasn't signaled\n");
89 }
90
91 /*
92 * Wake up waiters only after updating the ufence state, allowing the UMD
93 * to safely reuse the same ufence without encountering -EBUSY errors.
94 */
95 wake_up_all(&ufence->xe->ufence_wq);
96 user_fence_put(ufence);
97}
98
99static void kick_ufence(struct xe_user_fence *ufence, struct dma_fence *fence)
100{
101 INIT_WORK(&ufence->worker, user_fence_worker);
102 queue_work(ufence->xe->ordered_wq, &ufence->worker);
103 dma_fence_put(fence);
104}
105
106static void user_fence_cb(struct dma_fence *fence, struct dma_fence_cb *cb)
107{
108 struct xe_user_fence *ufence = container_of(cb, struct xe_user_fence, cb);
109
110 kick_ufence(ufence, fence);
111}
112
113int xe_sync_entry_parse(struct xe_device *xe, struct xe_file *xef,
114 struct xe_sync_entry *sync,
115 struct drm_xe_sync __user *sync_user,
116 struct drm_syncobj *ufence_syncobj,
117 u64 ufence_timeline_value,
118 unsigned int flags)
119{
120 struct drm_xe_sync sync_in;
121 int err;
122 bool exec = flags & SYNC_PARSE_FLAG_EXEC;
123 bool in_lr_mode = flags & SYNC_PARSE_FLAG_LR_MODE;
124 bool disallow_user_fence = flags & SYNC_PARSE_FLAG_DISALLOW_USER_FENCE;
125 bool signal;
126
127 if (copy_from_user(&sync_in, sync_user, sizeof(*sync_user)))
128 return -EFAULT;
129
130 if (XE_IOCTL_DBG(xe, sync_in.flags & ~DRM_XE_SYNC_FLAG_SIGNAL) ||
131 XE_IOCTL_DBG(xe, sync_in.reserved[0] || sync_in.reserved[1]))
132 return -EINVAL;
133
134 signal = sync_in.flags & DRM_XE_SYNC_FLAG_SIGNAL;
135 switch (sync_in.type) {
136 case DRM_XE_SYNC_TYPE_SYNCOBJ:
137 if (XE_IOCTL_DBG(xe, in_lr_mode && signal))
138 return -EOPNOTSUPP;
139
140 if (XE_IOCTL_DBG(xe, upper_32_bits(sync_in.addr)))
141 return -EINVAL;
142
143 sync->syncobj = drm_syncobj_find(xef->drm, sync_in.handle);
144 if (XE_IOCTL_DBG(xe, !sync->syncobj))
145 return -ENOENT;
146
147 if (!signal) {
148 sync->fence = drm_syncobj_fence_get(sync->syncobj);
149 if (XE_IOCTL_DBG(xe, !sync->fence)) {
150 err = -EINVAL;
151 goto free_sync;
152 }
153 }
154 break;
155
156 case DRM_XE_SYNC_TYPE_TIMELINE_SYNCOBJ:
157 if (XE_IOCTL_DBG(xe, in_lr_mode && signal))
158 return -EOPNOTSUPP;
159
160 if (XE_IOCTL_DBG(xe, upper_32_bits(sync_in.addr)))
161 return -EINVAL;
162
163 if (XE_IOCTL_DBG(xe, sync_in.timeline_value == 0))
164 return -EINVAL;
165
166 sync->syncobj = drm_syncobj_find(xef->drm, sync_in.handle);
167 if (XE_IOCTL_DBG(xe, !sync->syncobj))
168 return -ENOENT;
169
170 if (signal) {
171 sync->chain_fence = dma_fence_chain_alloc();
172 if (!sync->chain_fence) {
173 err = -ENOMEM;
174 goto free_sync;
175 }
176 } else {
177 sync->fence = drm_syncobj_fence_get(sync->syncobj);
178 if (XE_IOCTL_DBG(xe, !sync->fence)) {
179 err = -EINVAL;
180 goto free_sync;
181 }
182
183 err = dma_fence_chain_find_seqno(&sync->fence,
184 sync_in.timeline_value);
185 if (err)
186 goto free_sync;
187 }
188 break;
189
190 case DRM_XE_SYNC_TYPE_USER_FENCE:
191 if (XE_IOCTL_DBG(xe, disallow_user_fence))
192 return -EOPNOTSUPP;
193
194 if (XE_IOCTL_DBG(xe, !signal))
195 return -EOPNOTSUPP;
196
197 if (XE_IOCTL_DBG(xe, sync_in.addr & 0x7))
198 return -EINVAL;
199
200 if (exec) {
201 sync->addr = sync_in.addr;
202 } else {
203 sync->ufence_timeline_value = ufence_timeline_value;
204 sync->ufence = user_fence_create(xe, sync_in.addr,
205 sync_in.timeline_value);
206 if (XE_IOCTL_DBG(xe, IS_ERR(sync->ufence)))
207 return PTR_ERR(sync->ufence);
208 sync->ufence_chain_fence = dma_fence_chain_alloc();
209 if (!sync->ufence_chain_fence) {
210 err = -ENOMEM;
211 goto free_sync;
212 }
213 sync->ufence_syncobj = ufence_syncobj;
214 }
215
216 break;
217
218 default:
219 return -EINVAL;
220 }
221
222 sync->type = sync_in.type;
223 sync->flags = sync_in.flags;
224 sync->timeline_value = sync_in.timeline_value;
225
226 return 0;
227
228free_sync:
229 xe_sync_entry_cleanup(sync);
230 return err;
231}
232ALLOW_ERROR_INJECTION(xe_sync_entry_parse, ERRNO);
233
234int xe_sync_entry_add_deps(struct xe_sync_entry *sync, struct xe_sched_job *job)
235{
236 if (sync->fence)
237 return drm_sched_job_add_dependency(&job->drm,
238 dma_fence_get(sync->fence));
239
240 return 0;
241}
242
243/**
244 * xe_sync_entry_wait() - Wait on in-sync
245 * @sync: Sync object
246 *
247 * If the sync is in an in-sync, wait on the sync to signal.
248 *
249 * Return: 0 on success, -ERESTARTSYS on failure (interruption)
250 */
251int xe_sync_entry_wait(struct xe_sync_entry *sync)
252{
253 return xe_sync_needs_wait(sync) ?
254 dma_fence_wait(sync->fence, true) : 0;
255}
256
257/**
258 * xe_sync_needs_wait() - Sync needs a wait (input dma-fence not signaled)
259 * @sync: Sync object
260 *
261 * Return: True if sync needs a wait, False otherwise
262 */
263bool xe_sync_needs_wait(struct xe_sync_entry *sync)
264{
265 return sync->fence &&
266 !test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &sync->fence->flags);
267}
268
269void xe_sync_entry_signal(struct xe_sync_entry *sync, struct dma_fence *fence)
270{
271 if (!(sync->flags & DRM_XE_SYNC_FLAG_SIGNAL))
272 return;
273
274 if (sync->chain_fence) {
275 drm_syncobj_add_point(sync->syncobj, sync->chain_fence,
276 fence, sync->timeline_value);
277 /*
278 * The chain's ownership is transferred to the
279 * timeline.
280 */
281 sync->chain_fence = NULL;
282 } else if (sync->syncobj) {
283 drm_syncobj_replace_fence(sync->syncobj, fence);
284 } else if (sync->ufence) {
285 int err;
286
287 drm_syncobj_add_point(sync->ufence_syncobj,
288 sync->ufence_chain_fence,
289 fence, sync->ufence_timeline_value);
290 sync->ufence_chain_fence = NULL;
291
292 fence = drm_syncobj_fence_get(sync->ufence_syncobj);
293 user_fence_get(sync->ufence);
294 err = dma_fence_add_callback(fence, &sync->ufence->cb,
295 user_fence_cb);
296 if (err == -ENOENT) {
297 kick_ufence(sync->ufence, fence);
298 } else if (err) {
299 XE_WARN_ON("failed to add user fence");
300 user_fence_put(sync->ufence);
301 dma_fence_put(fence);
302 }
303 }
304}
305
306void xe_sync_entry_cleanup(struct xe_sync_entry *sync)
307{
308 if (sync->syncobj)
309 drm_syncobj_put(sync->syncobj);
310 dma_fence_put(sync->fence);
311 dma_fence_chain_free(sync->chain_fence);
312 dma_fence_chain_free(sync->ufence_chain_fence);
313 if (!IS_ERR_OR_NULL(sync->ufence))
314 user_fence_put(sync->ufence);
315}
316
317/**
318 * xe_sync_in_fence_get() - Get a fence from syncs, exec queue, and VM
319 * @sync: input syncs
320 * @num_sync: number of syncs
321 * @q: exec queue
322 * @vm: VM
323 *
324 * Get a fence from syncs, exec queue, and VM. If syncs contain in-fences create
325 * and return a composite fence of all in-fences + last fence. If no in-fences
326 * return last fence on input exec queue. Caller must drop reference to
327 * returned fence.
328 *
329 * Return: fence on success, ERR_PTR(-ENOMEM) on failure
330 */
331struct dma_fence *
332xe_sync_in_fence_get(struct xe_sync_entry *sync, int num_sync,
333 struct xe_exec_queue *q, struct xe_vm *vm)
334{
335 struct dma_fence **fences = NULL;
336 struct dma_fence_array *cf = NULL;
337 struct dma_fence *fence;
338 int i, num_fence = 0, current_fence = 0;
339
340 lockdep_assert_held(&vm->lock);
341
342 /* Reject in fences */
343 for (i = 0; i < num_sync; ++i)
344 if (sync[i].fence)
345 return ERR_PTR(-EOPNOTSUPP);
346
347 if (q->flags & EXEC_QUEUE_FLAG_VM) {
348 struct xe_exec_queue *__q;
349 struct xe_tile *tile;
350 u8 id;
351
352 for_each_tile(tile, vm->xe, id) {
353 num_fence++;
354 for_each_tlb_inval(i)
355 num_fence++;
356 }
357
358 fences = kmalloc_objs(*fences, num_fence);
359 if (!fences)
360 return ERR_PTR(-ENOMEM);
361
362 fences[current_fence++] =
363 xe_exec_queue_last_fence_get(q, vm);
364 for_each_tlb_inval(i)
365 fences[current_fence++] =
366 xe_exec_queue_tlb_inval_last_fence_get(q, vm, i);
367 list_for_each_entry(__q, &q->multi_gt_list,
368 multi_gt_link) {
369 fences[current_fence++] =
370 xe_exec_queue_last_fence_get(__q, vm);
371 for_each_tlb_inval(i)
372 fences[current_fence++] =
373 xe_exec_queue_tlb_inval_last_fence_get(__q, vm, i);
374 }
375
376 xe_assert(vm->xe, current_fence == num_fence);
377 cf = dma_fence_array_create(num_fence, fences,
378 dma_fence_context_alloc(1),
379 1);
380 if (!cf)
381 goto err_out;
382
383 return &cf->base;
384 }
385
386 fence = xe_exec_queue_last_fence_get(q, vm);
387 return fence;
388
389err_out:
390 while (current_fence)
391 dma_fence_put(fences[--current_fence]);
392 kfree(fences);
393
394 return ERR_PTR(-ENOMEM);
395}
396
397/**
398 * __xe_sync_ufence_get() - Get user fence from user fence
399 * @ufence: input user fence
400 *
401 * Get a user fence reference from user fence
402 *
403 * Return: xe_user_fence pointer with reference
404 */
405struct xe_user_fence *__xe_sync_ufence_get(struct xe_user_fence *ufence)
406{
407 user_fence_get(ufence);
408
409 return ufence;
410}
411
412/**
413 * xe_sync_ufence_get() - Get user fence from sync
414 * @sync: input sync
415 *
416 * Get a user fence reference from sync.
417 *
418 * Return: xe_user_fence pointer with reference
419 */
420struct xe_user_fence *xe_sync_ufence_get(struct xe_sync_entry *sync)
421{
422 user_fence_get(sync->ufence);
423
424 return sync->ufence;
425}
426
427/**
428 * xe_sync_ufence_put() - Put user fence reference
429 * @ufence: user fence reference
430 *
431 */
432void xe_sync_ufence_put(struct xe_user_fence *ufence)
433{
434 user_fence_put(ufence);
435}
436
437/**
438 * xe_sync_ufence_get_status() - Get user fence status
439 * @ufence: user fence
440 *
441 * Return: 1 if signalled, 0 not signalled, <0 on error
442 */
443int xe_sync_ufence_get_status(struct xe_user_fence *ufence)
444{
445 return READ_ONCE(ufence->signalled);
446}