projects/linux·blob master · refs

799 lines · 27.1 KiB

1/* SPDX-License-Identifier: GPL-2.0-only */
2/*
3 * Fence mechanism for dma-buf to allow for asynchronous dma access
4 *
5 * Copyright (C) 2012 Canonical Ltd
6 * Copyright (C) 2012 Texas Instruments
7 *
8 * Authors:
9 * Rob Clark <[email protected]>
10 * Maarten Lankhorst <[email protected]>
11 */
12
13#ifndef __LINUX_DMA_FENCE_H
14#define __LINUX_DMA_FENCE_H
15
16#include <linux/err.h>
17#include <linux/wait.h>
18#include <linux/list.h>
19#include <linux/bitops.h>
20#include <linux/kref.h>
21#include <linux/sched.h>
22#include <linux/printk.h>
23#include <linux/rcupdate.h>
24#include <linux/timekeeping.h>
25
26struct dma_fence;
27struct dma_fence_ops;
28struct dma_fence_cb;
29struct seq_file;
30
31/**
32 * struct dma_fence - software synchronization primitive
33 * @refcount: refcount for this fence
34 * @ops: dma_fence_ops associated with this fence
35 * @rcu: used for releasing fence with kfree_rcu
36 * @cb_list: list of all callbacks to call
37 * @extern_lock: external spin_lock_irqsave used for locking (deprecated)
38 * @inline_lock: alternative internal spin_lock_irqsave used for locking
39 * @context: execution context this fence belongs to, returned by
40 * dma_fence_context_alloc()
41 * @seqno: the sequence number of this fence inside the execution context,
42 * can be compared to decide which fence would be signaled later.
43 * @flags: A mask of DMA_FENCE_FLAG_* defined below
44 * @timestamp: Timestamp when the fence was signaled.
45 * @error: Optional, only valid if < 0, must be set before calling
46 * dma_fence_signal, indicates that the fence has completed with an error.
47 *
48 * the flags member must be manipulated and read using the appropriate
49 * atomic ops (bit_*), so taking the spinlock will not be needed most
50 * of the time.
51 *
52 * DMA_FENCE_FLAG_INITIALIZED_BIT - fence was initialized
53 * DMA_FENCE_FLAG_INLINE_LOCK_BIT - use inline spinlock instead of external one
54 * DMA_FENCE_FLAG_SIGNALED_BIT - fence is already signaled
55 * DMA_FENCE_FLAG_TIMESTAMP_BIT - timestamp recorded for fence signaling
56 * DMA_FENCE_FLAG_ENABLE_SIGNAL_BIT - enable_signaling might have been called
57 * DMA_FENCE_FLAG_USER_BITS - start of the unused bits, can be used by the
58 * implementer of the fence for its own purposes. Can be used in different
59 * ways by different fence implementers, so do not rely on this.
60 *
61 * Since atomic bitops are used, this is not guaranteed to be the case.
62 * Particularly, if the bit was set, but dma_fence_signal was called right
63 * before this bit was set, it would have been able to set the
64 * DMA_FENCE_FLAG_SIGNALED_BIT, before enable_signaling was called.
65 * Adding a check for DMA_FENCE_FLAG_SIGNALED_BIT after setting
66 * DMA_FENCE_FLAG_ENABLE_SIGNAL_BIT closes this race, and makes sure that
67 * after dma_fence_signal was called, any enable_signaling call will have either
68 * been completed, or never called at all.
69 */
70struct dma_fence {
71 union {
72 spinlock_t *extern_lock;
73 spinlock_t inline_lock;
74 };
75 const struct dma_fence_ops __rcu *ops;
76 /*
77 * We clear the callback list on kref_put so that by the time we
78 * release the fence it is unused. No one should be adding to the
79 * cb_list that they don't themselves hold a reference for.
80 *
81 * The lifetime of the timestamp is similarly tied to both the
82 * rcu freelist and the cb_list. The timestamp is only set upon
83 * signaling while simultaneously notifying the cb_list. Ergo, we
84 * only use either the cb_list of timestamp. Upon destruction,
85 * neither are accessible, and so we can use the rcu. This means
86 * that the cb_list is *only* valid until the signal bit is set,
87 * and to read either you *must* hold a reference to the fence,
88 * and not just the rcu_read_lock.
89 *
90 * Listed in chronological order.
91 */
92 union {
93 struct list_head cb_list;
94 /* @cb_list replaced by @timestamp on dma_fence_signal() */
95 ktime_t timestamp;
96 /* @timestamp replaced by @rcu on dma_fence_release() */
97 struct rcu_head rcu;
98 };
99 u64 context;
100 u64 seqno;
101 unsigned long flags;
102 struct kref refcount;
103 int error;
104};
105
106enum dma_fence_flag_bits {
107 DMA_FENCE_FLAG_INITIALIZED_BIT,
108 DMA_FENCE_FLAG_INLINE_LOCK_BIT,
109 DMA_FENCE_FLAG_SEQNO64_BIT,
110 DMA_FENCE_FLAG_SIGNALED_BIT,
111 DMA_FENCE_FLAG_TIMESTAMP_BIT,
112 DMA_FENCE_FLAG_ENABLE_SIGNAL_BIT,
113 DMA_FENCE_FLAG_USER_BITS, /* must always be last member */
114};
115
116typedef void (*dma_fence_func_t)(struct dma_fence *fence,
117 struct dma_fence_cb *cb);
118
119/**
120 * struct dma_fence_cb - callback for dma_fence_add_callback()
121 * @node: used by dma_fence_add_callback() to append this struct to fence::cb_list
122 * @func: dma_fence_func_t to call
123 *
124 * This struct will be initialized by dma_fence_add_callback(), additional
125 * data can be passed along by embedding dma_fence_cb in another struct.
126 */
127struct dma_fence_cb {
128 struct list_head node;
129 dma_fence_func_t func;
130};
131
132/**
133 * struct dma_fence_ops - operations implemented for fence
134 *
135 */
136struct dma_fence_ops {
137 /**
138 * @get_driver_name:
139 *
140 * Returns the driver name. This is a callback to allow drivers to
141 * compute the name at runtime, without having it to store permanently
142 * for each fence, or build a cache of some sort.
143 *
144 * The returned string is RCU protected and can be freed after the fence
145 * signaled and a RCU grace period passed.
146 *
147 * This callback is mandatory.
148 */
149 const char * (*get_driver_name)(struct dma_fence *fence);
150
151 /**
152 * @get_timeline_name:
153 *
154 * Return the name of the context this fence belongs to. This is a
155 * callback to allow drivers to compute the name at runtime, without
156 * having it to store permanently for each fence, or build a cache of
157 * some sort.
158 *
159 * The returned string is RCU protected and can be freed after the fence
160 * signaled and a RCU grace period passed.
161 *
162 * This callback is mandatory.
163 */
164 const char * (*get_timeline_name)(struct dma_fence *fence);
165
166 /**
167 * @enable_signaling:
168 *
169 * Enable software signaling of fence.
170 *
171 * For fence implementations that have the capability for hw->hw
172 * signaling, they can implement this op to enable the necessary
173 * interrupts, or insert commands into cmdstream, etc, to avoid these
174 * costly operations for the common case where only hw->hw
175 * synchronization is required. This is called in the first
176 * dma_fence_wait() or dma_fence_add_callback() path to let the fence
177 * implementation know that there is another driver waiting on the
178 * signal (ie. hw->sw case).
179 *
180 * This is called with irq's disabled, so only spinlocks which disable
181 * IRQ's can be used in the code outside of this callback.
182 *
183 * A return value of false indicates the fence already passed,
184 * or some failure occurred that made it impossible to enable
185 * signaling. True indicates successful enabling.
186 *
187 * &dma_fence.error may be set in enable_signaling, but only when false
188 * is returned.
189 *
190 * Since many implementations can call dma_fence_signal() even when before
191 * @enable_signaling has been called there's a race window, where the
192 * dma_fence_signal() might result in the final fence reference being
193 * released and its memory freed. To avoid this, implementations of this
194 * callback should grab their own reference using dma_fence_get(), to be
195 * released when the fence is signalled (through e.g. the interrupt
196 * handler).
197 *
198 * This callback is optional. If this callback is not present, then the
199 * driver must always have signaling enabled.
200 */
201 bool (*enable_signaling)(struct dma_fence *fence);
202
203 /**
204 * @signaled:
205 *
206 * Peek whether the fence is signaled, as a fastpath optimization for
207 * e.g. dma_fence_wait() or dma_fence_add_callback(). Note that this
208 * callback does not need to make any guarantees beyond that a fence
209 * once indicates as signalled must always return true from this
210 * callback. This callback may return false even if the fence has
211 * completed already, in this case information hasn't propogated throug
212 * the system yet. See also dma_fence_is_signaled().
213 *
214 * May set &dma_fence.error if returning true.
215 *
216 * This callback is optional.
217 */
218 bool (*signaled)(struct dma_fence *fence);
219
220 /**
221 * @wait:
222 *
223 * Custom wait implementation, defaults to dma_fence_default_wait() if
224 * not set.
225 *
226 * Deprecated and should not be used by new implementations. Only used
227 * by existing implementations which need special handling for their
228 * hardware reset procedure.
229 *
230 * Must return -ERESTARTSYS if the wait is intr = true and the wait was
231 * interrupted, and remaining jiffies if fence has signaled, or 0 if wait
232 * timed out. Can also return other error values on custom implementations,
233 * which should be treated as if the fence is signaled. For example a hardware
234 * lockup could be reported like that.
235 *
236 * Implementing this callback prevents the fence from detaching after
237 * signaling and so it is necessary for the module providing the
238 * dma_fence_ops to stay loaded as long as the dma_fence exists.
239 */
240 signed long (*wait)(struct dma_fence *fence,
241 bool intr, signed long timeout);
242
243 /**
244 * @release:
245 *
246 * Called on destruction of fence to release additional resources.
247 * Can be called from irq context. This callback is optional. If it is
248 * NULL, then dma_fence_free() is instead called as the default
249 * implementation.
250 *
251 * Implementing this callback prevents the fence from detaching after
252 * signaling and so it is necessary for the module providing the
253 * dma_fence_ops to stay loaded as long as the dma_fence exists.
254 *
255 * If the callback is implemented the memory backing the dma_fence
256 * object must be freed RCU safe.
257 */
258 void (*release)(struct dma_fence *fence);
259
260 /**
261 * @set_deadline:
262 *
263 * Callback to allow a fence waiter to inform the fence signaler of
264 * an upcoming deadline, such as vblank, by which point the waiter
265 * would prefer the fence to be signaled by. This is intended to
266 * give feedback to the fence signaler to aid in power management
267 * decisions, such as boosting GPU frequency.
268 *
269 * This is called without &dma_fence.lock held, it can be called
270 * multiple times and from any context. Locking is up to the callee
271 * if it has some state to manage. If multiple deadlines are set,
272 * the expectation is to track the soonest one. If the deadline is
273 * before the current time, it should be interpreted as an immediate
274 * deadline.
275 *
276 * This callback is optional.
277 */
278 void (*set_deadline)(struct dma_fence *fence, ktime_t deadline);
279};
280
281void dma_fence_init(struct dma_fence *fence, const struct dma_fence_ops *ops,
282 spinlock_t *lock, u64 context, u64 seqno);
283
284void dma_fence_init64(struct dma_fence *fence, const struct dma_fence_ops *ops,
285 spinlock_t *lock, u64 context, u64 seqno);
286
287void dma_fence_release(struct kref *kref);
288void dma_fence_free(struct dma_fence *fence);
289void dma_fence_describe(struct dma_fence *fence, struct seq_file *seq);
290
291/**
292 * dma_fence_was_initialized - test if fence was initialized
293 * @fence: fence to test
294 *
295 * Return: True if fence was ever initialized, false otherwise. Works correctly
296 * only when memory backing the fence structure is zero initialized on
297 * allocation.
298 */
299static inline bool dma_fence_was_initialized(struct dma_fence *fence)
300{
301 return fence && test_bit(DMA_FENCE_FLAG_INITIALIZED_BIT, &fence->flags);
302}
303
304/**
305 * dma_fence_put - decreases refcount of the fence
306 * @fence: fence to reduce refcount of
307 */
308static inline void dma_fence_put(struct dma_fence *fence)
309{
310 if (fence)
311 kref_put(&fence->refcount, dma_fence_release);
312}
313
314/**
315 * dma_fence_get - increases refcount of the fence
316 * @fence: fence to increase refcount of
317 *
318 * Returns the same fence, with refcount increased by 1.
319 */
320static inline struct dma_fence *dma_fence_get(struct dma_fence *fence)
321{
322 if (fence)
323 kref_get(&fence->refcount);
324 return fence;
325}
326
327/**
328 * dma_fence_get_rcu - get a fence from a dma_resv_list with
329 * rcu read lock
330 * @fence: fence to increase refcount of
331 *
332 * Function returns NULL if no refcount could be obtained, or the fence.
333 */
334static inline struct dma_fence *dma_fence_get_rcu(struct dma_fence *fence)
335{
336 if (kref_get_unless_zero(&fence->refcount))
337 return fence;
338 else
339 return NULL;
340}
341
342/**
343 * dma_fence_get_rcu_safe - acquire a reference to an RCU tracked fence
344 * @fencep: pointer to fence to increase refcount of
345 *
346 * Function returns NULL if no refcount could be obtained, or the fence.
347 * This function handles acquiring a reference to a fence that may be
348 * reallocated within the RCU grace period (such as with SLAB_TYPESAFE_BY_RCU),
349 * so long as the caller is using RCU on the pointer to the fence.
350 *
351 * An alternative mechanism is to employ a seqlock to protect a bunch of
352 * fences, such as used by struct dma_resv. When using a seqlock,
353 * the seqlock must be taken before and checked after a reference to the
354 * fence is acquired (as shown here).
355 *
356 * The caller is required to hold the RCU read lock.
357 */
358static inline struct dma_fence *
359dma_fence_get_rcu_safe(struct dma_fence __rcu **fencep)
360{
361 do {
362 struct dma_fence *fence;
363
364 fence = rcu_dereference(*fencep);
365 if (!fence)
366 return NULL;
367
368 if (!dma_fence_get_rcu(fence))
369 continue;
370
371 /* The atomic_inc_not_zero() inside dma_fence_get_rcu()
372 * provides a full memory barrier upon success (such as now).
373 * This is paired with the write barrier from assigning
374 * to the __rcu protected fence pointer so that if that
375 * pointer still matches the current fence, we know we
376 * have successfully acquire a reference to it. If it no
377 * longer matches, we are holding a reference to some other
378 * reallocated pointer. This is possible if the allocator
379 * is using a freelist like SLAB_TYPESAFE_BY_RCU where the
380 * fence remains valid for the RCU grace period, but it
381 * may be reallocated. When using such allocators, we are
382 * responsible for ensuring the reference we get is to
383 * the right fence, as below.
384 */
385 if (fence == rcu_access_pointer(*fencep))
386 return rcu_pointer_handoff(fence);
387
388 dma_fence_put(fence);
389 } while (1);
390}
391
392/**
393 * dma_fence_spinlock - return pointer to the spinlock protecting the fence
394 * @fence: the fence to get the lock from
395 *
396 * Return either the pointer to the embedded or the external spin lock.
397 */
398static inline spinlock_t *dma_fence_spinlock(struct dma_fence *fence)
399{
400 return test_bit(DMA_FENCE_FLAG_INLINE_LOCK_BIT, &fence->flags) ?
401 &fence->inline_lock : fence->extern_lock;
402}
403
404/**
405 * dma_fence_lock_irqsave - irqsave lock the fence
406 * @fence: the fence to lock
407 * @flags: where to store the CPU flags.
408 *
409 * Lock the fence, preventing it from changing to the signaled state.
410 */
411#define dma_fence_lock_irqsave(fence, flags) \
412 spin_lock_irqsave(dma_fence_spinlock(fence), flags)
413
414/**
415 * dma_fence_unlock_irqrestore - unlock the fence and irqrestore
416 * @fence: the fence to unlock
417 * @flags: the CPU flags to restore
418 *
419 * Unlock the fence, allowing it to change its state to signaled again.
420 */
421#define dma_fence_unlock_irqrestore(fence, flags) \
422 spin_unlock_irqrestore(dma_fence_spinlock(fence), flags)
423
424/**
425 * dma_fence_assert_held - lockdep assertion that fence is locked
426 * @fence: the fence which should be locked
427 */
428#define dma_fence_assert_held(fence) \
429 lockdep_assert_held(dma_fence_spinlock(fence));
430
431#ifdef CONFIG_LOCKDEP
432bool dma_fence_begin_signalling(void);
433void dma_fence_end_signalling(bool cookie);
434void __dma_fence_might_wait(void);
435#else
436static inline bool dma_fence_begin_signalling(void)
437{
438 return true;
439}
440static inline void dma_fence_end_signalling(bool cookie) {}
441static inline void __dma_fence_might_wait(void) {}
442#endif
443
444void dma_fence_signal(struct dma_fence *fence);
445bool dma_fence_check_and_signal(struct dma_fence *fence);
446bool dma_fence_check_and_signal_locked(struct dma_fence *fence);
447void dma_fence_signal_locked(struct dma_fence *fence);
448void dma_fence_signal_timestamp(struct dma_fence *fence, ktime_t timestamp);
449void dma_fence_signal_timestamp_locked(struct dma_fence *fence, ktime_t timestamp);
450signed long dma_fence_default_wait(struct dma_fence *fence,
451 bool intr, signed long timeout);
452int dma_fence_add_callback(struct dma_fence *fence,
453 struct dma_fence_cb *cb,
454 dma_fence_func_t func);
455bool dma_fence_remove_callback(struct dma_fence *fence,
456 struct dma_fence_cb *cb);
457void dma_fence_enable_signaling(struct dma_fence *fence);
458
459/**
460 * DOC: Safe external access to driver provided object members
461 *
462 * All data not stored directly in the dma-fence object, such as the
463 * &dma_fence.lock and memory potentially accessed by functions in the
464 * &dma_fence.ops table, MUST NOT be accessed after the fence has been signalled
465 * because after that point drivers are allowed to free it.
466 *
467 * All code accessing that data via the dma-fence API (or directly, which is
468 * discouraged), MUST make sure to contain the complete access within a
469 * &rcu_read_lock and &rcu_read_unlock pair.
470 *
471 * Some dma-fence API handles this automatically, while other, as for example
472 * &dma_fence_driver_name and &dma_fence_timeline_name, leave that
473 * responsibility to the caller.
474 *
475 * To enable this scheme to work drivers MUST ensure a RCU grace period elapses
476 * between signalling the fence and freeing the said data.
477 *
478 */
479const char __rcu *dma_fence_driver_name(struct dma_fence *fence);
480const char __rcu *dma_fence_timeline_name(struct dma_fence *fence);
481
482/*
483 * dma_fence_test_signaled_flag - Only check whether a fence is signaled yet.
484 * @fence: the fence to check
485 *
486 * This function just checks whether @fence is signaled, without interacting
487 * with the fence in any way. The user must, therefore, ensure through other
488 * means that fences get signaled eventually.
489 *
490 * This function uses test_bit(), which is thread-safe. Naturally, this function
491 * should be used opportunistically; a fence could get signaled at any moment
492 * after the check is done.
493 *
494 * Return: true if signaled, false otherwise.
495 */
496static inline bool
497dma_fence_test_signaled_flag(struct dma_fence *fence)
498{
499 return test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags);
500}
501
502/**
503 * dma_fence_is_signaled_locked - Return an indication if the fence
504 * is signaled yet.
505 * @fence: the fence to check
506 *
507 * Returns true if the fence was already signaled, false if not. Since this
508 * function doesn't enable signaling, it is not guaranteed to ever return
509 * true if dma_fence_add_callback(), dma_fence_wait() or
510 * dma_fence_enable_sw_signaling() haven't been called before.
511 *
512 * This function requires &dma_fence.lock to be held.
513 *
514 * See also dma_fence_is_signaled().
515 */
516static inline bool
517dma_fence_is_signaled_locked(struct dma_fence *fence)
518{
519 const struct dma_fence_ops *ops;
520
521 if (dma_fence_test_signaled_flag(fence))
522 return true;
523
524 rcu_read_lock();
525 ops = rcu_dereference(fence->ops);
526 if (ops && ops->signaled && ops->signaled(fence)) {
527 rcu_read_unlock();
528 dma_fence_signal_locked(fence);
529 return true;
530 }
531 rcu_read_unlock();
532
533 return false;
534}
535
536/**
537 * dma_fence_is_signaled - Return an indication if the fence is signaled yet.
538 * @fence: the fence to check
539 *
540 * Returns true if the fence was already signaled, false if not. Since this
541 * function doesn't enable signaling, it is not guaranteed to ever return
542 * true if dma_fence_add_callback(), dma_fence_wait() or
543 * dma_fence_enable_signaling() haven't been called before.
544 *
545 * It's recommended for seqno fences to call dma_fence_signal when the
546 * operation is complete, it makes it possible to prevent issues from
547 * wraparound between time of issue and time of use by checking the return
548 * value of this function before calling hardware-specific wait instructions.
549 *
550 * See also dma_fence_is_signaled_locked().
551 */
552static inline bool
553dma_fence_is_signaled(struct dma_fence *fence)
554{
555 const struct dma_fence_ops *ops;
556
557 if (dma_fence_test_signaled_flag(fence))
558 return true;
559
560 rcu_read_lock();
561 ops = rcu_dereference(fence->ops);
562 if (ops && ops->signaled && ops->signaled(fence)) {
563 rcu_read_unlock();
564 dma_fence_signal(fence);
565 return true;
566 }
567 rcu_read_unlock();
568
569 return false;
570}
571
572/**
573 * __dma_fence_is_later - return if f1 is chronologically later than f2
574 * @fence: fence in whose context to do the comparison
575 * @f1: the first fence's seqno
576 * @f2: the second fence's seqno from the same context
577 *
578 * Returns true if f1 is chronologically later than f2. Both fences must be
579 * from the same context, since a seqno is not common across contexts.
580 */
581static inline bool __dma_fence_is_later(struct dma_fence *fence, u64 f1, u64 f2)
582{
583 /* This is for backward compatibility with drivers which can only handle
584 * 32bit sequence numbers. Use a 64bit compare when the driver says to
585 * do so.
586 */
587 if (test_bit(DMA_FENCE_FLAG_SEQNO64_BIT, &fence->flags))
588 return f1 > f2;
589
590 return (int)(lower_32_bits(f1) - lower_32_bits(f2)) > 0;
591}
592
593/**
594 * dma_fence_is_later - return if f1 is chronologically later than f2
595 * @f1: the first fence from the same context
596 * @f2: the second fence from the same context
597 *
598 * Returns true if f1 is chronologically later than f2. Both fences must be
599 * from the same context, since a seqno is not re-used across contexts.
600 */
601static inline bool dma_fence_is_later(struct dma_fence *f1,
602 struct dma_fence *f2)
603{
604 if (WARN_ON(f1->context != f2->context))
605 return false;
606
607 return __dma_fence_is_later(f1, f1->seqno, f2->seqno);
608}
609
610/**
611 * dma_fence_is_later_or_same - return true if f1 is later or same as f2
612 * @f1: the first fence from the same context
613 * @f2: the second fence from the same context
614 *
615 * Returns true if f1 is chronologically later than f2 or the same fence. Both
616 * fences must be from the same context, since a seqno is not re-used across
617 * contexts.
618 */
619static inline bool dma_fence_is_later_or_same(struct dma_fence *f1,
620 struct dma_fence *f2)
621{
622 return f1 == f2 || dma_fence_is_later(f1, f2);
623}
624
625/**
626 * dma_fence_later - return the chronologically later fence
627 * @f1: the first fence from the same context
628 * @f2: the second fence from the same context
629 *
630 * Returns NULL if both fences are signaled, otherwise the fence that would be
631 * signaled last. Both fences must be from the same context, since a seqno is
632 * not re-used across contexts.
633 */
634static inline struct dma_fence *dma_fence_later(struct dma_fence *f1,
635 struct dma_fence *f2)
636{
637 if (WARN_ON(f1->context != f2->context))
638 return NULL;
639
640 /*
641 * Can't check just DMA_FENCE_FLAG_SIGNALED_BIT here, it may never
642 * have been set if enable_signaling wasn't called, and enabling that
643 * here is overkill.
644 */
645 if (dma_fence_is_later(f1, f2))
646 return dma_fence_is_signaled(f1) ? NULL : f1;
647 else
648 return dma_fence_is_signaled(f2) ? NULL : f2;
649}
650
651/**
652 * dma_fence_get_status_locked - returns the status upon completion
653 * @fence: the dma_fence to query
654 *
655 * Drivers can supply an optional error status condition before they signal
656 * the fence (to indicate whether the fence was completed due to an error
657 * rather than success). The value of the status condition is only valid
658 * if the fence has been signaled, dma_fence_get_status_locked() first checks
659 * the signal state before reporting the error status.
660 *
661 * Returns 0 if the fence has not yet been signaled, 1 if the fence has
662 * been signaled without an error condition, or a negative error code
663 * if the fence has been completed in err.
664 */
665static inline int dma_fence_get_status_locked(struct dma_fence *fence)
666{
667 if (dma_fence_is_signaled_locked(fence))
668 return fence->error ?: 1;
669 else
670 return 0;
671}
672
673int dma_fence_get_status(struct dma_fence *fence);
674
675/**
676 * dma_fence_set_error - flag an error condition on the fence
677 * @fence: the dma_fence
678 * @error: the error to store
679 *
680 * Drivers can supply an optional error status condition before they signal
681 * the fence, to indicate that the fence was completed due to an error
682 * rather than success. This must be set before signaling (so that the value
683 * is visible before any waiters on the signal callback are woken). This
684 * helper exists to help catching erroneous setting of #dma_fence.error.
685 *
686 * Examples of error codes which drivers should use:
687 *
688 * * %-ENODATA This operation produced no data, no other operation affected.
689 * * %-ECANCELED All operations from the same context have been canceled.
690 * * %-ETIME Operation caused a timeout and potentially device reset.
691 */
692static inline void dma_fence_set_error(struct dma_fence *fence,
693 int error)
694{
695 WARN_ON(test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags));
696 WARN_ON(error >= 0 || error < -MAX_ERRNO);
697
698 fence->error = error;
699}
700
701/**
702 * dma_fence_timestamp - helper to get the completion timestamp of a fence
703 * @fence: fence to get the timestamp from.
704 *
705 * After a fence is signaled the timestamp is updated with the signaling time,
706 * but setting the timestamp can race with tasks waiting for the signaling. This
707 * helper busy waits for the correct timestamp to appear.
708 */
709static inline ktime_t dma_fence_timestamp(struct dma_fence *fence)
710{
711 if (WARN_ON(!test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags)))
712 return ktime_get();
713
714 while (!test_bit(DMA_FENCE_FLAG_TIMESTAMP_BIT, &fence->flags))
715 cpu_relax();
716
717 return fence->timestamp;
718}
719
720signed long dma_fence_wait_timeout(struct dma_fence *,
721 bool intr, signed long timeout);
722signed long dma_fence_wait_any_timeout(struct dma_fence **fences,
723 uint32_t count,
724 bool intr, signed long timeout,
725 uint32_t *idx);
726
727/**
728 * dma_fence_wait - sleep until the fence gets signaled
729 * @fence: the fence to wait on
730 * @intr: if true, do an interruptible wait
731 *
732 * This function will return -ERESTARTSYS if interrupted by a signal,
733 * or 0 if the fence was signaled. Other error values may be
734 * returned on custom implementations.
735 *
736 * Performs a synchronous wait on this fence. It is assumed the caller
737 * directly or indirectly holds a reference to the fence, otherwise the
738 * fence might be freed before return, resulting in undefined behavior.
739 *
740 * See also dma_fence_wait_timeout() and dma_fence_wait_any_timeout().
741 */
742static inline signed long dma_fence_wait(struct dma_fence *fence, bool intr)
743{
744 signed long ret;
745
746 /* Since dma_fence_wait_timeout cannot timeout with
747 * MAX_SCHEDULE_TIMEOUT, only valid return values are
748 * -ERESTARTSYS and MAX_SCHEDULE_TIMEOUT.
749 */
750 ret = dma_fence_wait_timeout(fence, intr, MAX_SCHEDULE_TIMEOUT);
751
752 return ret < 0 ? ret : 0;
753}
754
755void dma_fence_set_deadline(struct dma_fence *fence, ktime_t deadline);
756
757struct dma_fence *dma_fence_get_stub(void);
758struct dma_fence *dma_fence_allocate_private_stub(ktime_t timestamp);
759u64 dma_fence_context_alloc(unsigned num);
760
761extern const struct dma_fence_ops dma_fence_array_ops;
762extern const struct dma_fence_ops dma_fence_chain_ops;
763
764/**
765 * dma_fence_is_array - check if a fence is from the array subclass
766 * @fence: the fence to test
767 *
768 * Return true if it is a dma_fence_array and false otherwise.
769 */
770static inline bool dma_fence_is_array(struct dma_fence *fence)
771{
772 return rcu_access_pointer(fence->ops) == &dma_fence_array_ops;
773}
774
775/**
776 * dma_fence_is_chain - check if a fence is from the chain subclass
777 * @fence: the fence to test
778 *
779 * Return true if it is a dma_fence_chain and false otherwise.
780 */
781static inline bool dma_fence_is_chain(struct dma_fence *fence)
782{
783 return rcu_access_pointer(fence->ops) == &dma_fence_chain_ops;
784}
785
786/**
787 * dma_fence_is_container - check if a fence is a container for other fences
788 * @fence: the fence to test
789 *
790 * Return true if this fence is a container for other fences, false otherwise.
791 * This is important since we can't build up large fence structure or otherwise
792 * we run into recursion during operation on those fences.
793 */
794static inline bool dma_fence_is_container(struct dma_fence *fence)
795{
796 return dma_fence_is_array(fence) || dma_fence_is_chain(fence);
797}
798
799#endif /* __LINUX_DMA_FENCE_H */