| 1 |
|
| 2 | #ifndef IOU_REQ_REF_H
|
| 3 | #define IOU_REQ_REF_H
|
| 4 |
|
| 5 | #include <linux/atomic.h>
|
| 6 | #include <linux/io_uring_types.h>
|
| 7 |
|
| 8 | |
| 9 | * Shamelessly stolen from the mm implementation of page reference checking,
|
| 10 | * see commit f958d7b528b1 for details.
|
| 11 |
|
| 12 | #define req_ref_zero_or_close_to_overflow(req) \
|
| 13 | ((unsigned int) atomic_read(&(req->refs)) + 127u <= 127u)
|
| 14 |
|
| 15 | static inline bool req_ref_inc_not_zero(struct io_kiocb *req)
|
| 16 | {
|
| 17 | WARN_ON_ONCE(!(req->flags & REQ_F_REFCOUNT));
|
| 18 | return atomic_inc_not_zero(&req->refs);
|
| 19 | }
|
| 20 |
|
| 21 | static inline bool req_ref_put_and_test_atomic(struct io_kiocb *req)
|
| 22 | {
|
| 23 | WARN_ON_ONCE(!(data_race(req->flags) & REQ_F_REFCOUNT));
|
| 24 | WARN_ON_ONCE(req_ref_zero_or_close_to_overflow(req));
|
| 25 | return atomic_dec_and_test(&req->refs);
|
| 26 | }
|
| 27 |
|
| 28 | static inline bool req_ref_put_and_test(struct io_kiocb *req)
|
| 29 | {
|
| 30 | if (likely(!(req->flags & REQ_F_REFCOUNT)))
|
| 31 | return true;
|
| 32 |
|
| 33 | WARN_ON_ONCE(req_ref_zero_or_close_to_overflow(req));
|
| 34 | return atomic_dec_and_test(&req->refs);
|
| 35 | }
|
| 36 |
|
| 37 | static inline void req_ref_get(struct io_kiocb *req)
|
| 38 | {
|
| 39 | WARN_ON_ONCE(!(req->flags & REQ_F_REFCOUNT));
|
| 40 | WARN_ON_ONCE(req_ref_zero_or_close_to_overflow(req));
|
| 41 | atomic_inc(&req->refs);
|
| 42 | }
|
| 43 |
|
| 44 | static inline void req_ref_put(struct io_kiocb *req)
|
| 45 | {
|
| 46 | WARN_ON_ONCE(!(req->flags & REQ_F_REFCOUNT));
|
| 47 | WARN_ON_ONCE(req_ref_zero_or_close_to_overflow(req));
|
| 48 | atomic_dec(&req->refs);
|
| 49 | }
|
| 50 |
|
| 51 | static inline void __io_req_set_refcount(struct io_kiocb *req, int nr)
|
| 52 | {
|
| 53 | if (!(req->flags & REQ_F_REFCOUNT)) {
|
| 54 | req->flags |= REQ_F_REFCOUNT;
|
| 55 | atomic_set(&req->refs, nr);
|
| 56 | }
|
| 57 | }
|
| 58 |
|
| 59 | static inline void io_req_set_refcount(struct io_kiocb *req)
|
| 60 | {
|
| 61 | __io_req_set_refcount(req, 1);
|
| 62 | }
|
| 63 | #endif
|