projects/linux·blob master · refs

275 lines · 6.1 KiB

1/* SPDX-License-Identifier: GPL-2.0 */
2#include <linux/mutex.h>
3#include <linux/bpf.h>
4#include <linux/bpf_verifier.h>
5
6#include "io_uring.h"
7#include "register.h"
8#include "loop.h"
9#include "memmap.h"
10#include "bpf-ops.h"
11
12static DEFINE_MUTEX(io_bpf_ctrl_mutex);
13static const struct btf_type *loop_params_type;
14
15__bpf_kfunc_start_defs();
16
17__bpf_kfunc int bpf_io_uring_submit_sqes(struct iou_ctx *loop_ctx, u32 nr)
18{
19 struct io_ring_ctx *ctx = io_loop_demangle_ctx(loop_ctx);
20
21 return io_submit_sqes(ctx, nr);
22}
23
24__bpf_kfunc
25__u8 *bpf_io_uring_get_region(struct iou_ctx *loop_ctx, __u32 region_id,
26 const size_t rdwr_buf_size)
27{
28 struct io_ring_ctx *ctx = io_loop_demangle_ctx(loop_ctx);
29 struct io_mapped_region *r;
30
31 lockdep_assert_held(&ctx->uring_lock);
32
33 switch (region_id) {
34 case IOU_REGION_MEM:
35 r = &ctx->param_region;
36 break;
37 case IOU_REGION_CQ:
38 r = &ctx->ring_region;
39 break;
40 case IOU_REGION_SQ:
41 r = &ctx->sq_region;
42 break;
43 default:
44 return NULL;
45 }
46
47 if (unlikely(rdwr_buf_size > io_region_size(r)))
48 return NULL;
49 return io_region_get_ptr(r);
50}
51
52__bpf_kfunc_end_defs();
53
54BTF_KFUNCS_START(io_uring_kfunc_set)
55BTF_ID_FLAGS(func, bpf_io_uring_submit_sqes, KF_SLEEPABLE);
56BTF_ID_FLAGS(func, bpf_io_uring_get_region, KF_RET_NULL);
57BTF_KFUNCS_END(io_uring_kfunc_set)
58
59static const struct btf_kfunc_id_set bpf_io_uring_kfunc_set = {
60 .owner = THIS_MODULE,
61 .set = &io_uring_kfunc_set,
62};
63
64static int io_bpf_ops__loop_step(struct iou_ctx *ctx,
65 struct iou_loop_params *lp)
66{
67 return IOU_LOOP_STOP;
68}
69
70static struct io_uring_bpf_ops io_bpf_ops_stubs = {
71 .loop_step = io_bpf_ops__loop_step,
72};
73
74static bool bpf_io_is_valid_access(int off, int size,
75 enum bpf_access_type type,
76 const struct bpf_prog *prog,
77 struct bpf_insn_access_aux *info)
78{
79 if (type != BPF_READ)
80 return false;
81 if (off < 0 || off >= sizeof(__u64) * MAX_BPF_FUNC_ARGS)
82 return false;
83 if (off % size != 0)
84 return false;
85
86 return btf_ctx_access(off, size, type, prog, info);
87}
88
89static int bpf_io_btf_struct_access(struct bpf_verifier_log *log,
90 const struct bpf_reg_state *reg, int off,
91 int size)
92{
93 const struct btf_type *t = btf_type_by_id(reg->btf, reg->btf_id);
94
95 if (t == loop_params_type) {
96 if (off + size <= offsetofend(struct iou_loop_params, cq_wait_idx))
97 return SCALAR_VALUE;
98 }
99
100 return -EACCES;
101}
102
103static const struct bpf_verifier_ops bpf_io_verifier_ops = {
104 .get_func_proto = bpf_base_func_proto,
105 .is_valid_access = bpf_io_is_valid_access,
106 .btf_struct_access = bpf_io_btf_struct_access,
107};
108
109static const struct btf_type *
110io_lookup_struct_type(struct btf *btf, const char *name)
111{
112 s32 type_id;
113
114 type_id = btf_find_by_name_kind(btf, name, BTF_KIND_STRUCT);
115 if (type_id < 0)
116 return NULL;
117 return btf_type_by_id(btf, type_id);
118}
119
120static int bpf_io_init(struct btf *btf)
121{
122 int ret;
123
124 loop_params_type = io_lookup_struct_type(btf, "iou_loop_params");
125 if (!loop_params_type) {
126 pr_err("io_uring: Failed to locate iou_loop_params\n");
127 return -EINVAL;
128 }
129
130 ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS,
131 &bpf_io_uring_kfunc_set);
132 if (ret) {
133 pr_err("io_uring: Failed to register kfuncs (%d)\n", ret);
134 return ret;
135 }
136 return 0;
137}
138
139static int bpf_io_check_member(const struct btf_type *t,
140 const struct btf_member *member,
141 const struct bpf_prog *prog)
142{
143 return 0;
144}
145
146static int bpf_io_init_member(const struct btf_type *t,
147 const struct btf_member *member,
148 void *kdata, const void *udata)
149{
150 u32 moff = __btf_member_bit_offset(t, member) / 8;
151 const struct io_uring_bpf_ops *uops = udata;
152 struct io_uring_bpf_ops *ops = kdata;
153
154 switch (moff) {
155 case offsetof(struct io_uring_bpf_ops, ring_fd):
156 ops->ring_fd = uops->ring_fd;
157 return 1;
158 }
159 return 0;
160}
161
162static int io_install_bpf(struct io_ring_ctx *ctx, struct io_uring_bpf_ops *ops)
163{
164 if (ctx->flags & (IORING_SETUP_SQPOLL | IORING_SETUP_IOPOLL))
165 return -EOPNOTSUPP;
166 if (!(ctx->flags & IORING_SETUP_DEFER_TASKRUN))
167 return -EOPNOTSUPP;
168
169 if (ctx->bpf_ops)
170 return -EBUSY;
171 if (ops->priv)
172 return -EBUSY;
173 if (WARN_ON_ONCE(!ops->loop_step))
174 return -EINVAL;
175
176 ops->priv = ctx;
177 ctx->bpf_ops = ops;
178 ctx->loop_step = ops->loop_step;
179 return 0;
180}
181
182static int bpf_io_reg(void *kdata, struct bpf_link *link)
183{
184 struct io_uring_bpf_ops *ops = kdata;
185 struct io_ring_ctx *ctx;
186 struct file *file;
187 int ret = -EBUSY;
188
189 file = io_uring_ctx_get_file(ops->ring_fd, false);
190 if (IS_ERR(file))
191 return PTR_ERR(file);
192 ctx = file->private_data;
193
194 scoped_guard(mutex, &io_bpf_ctrl_mutex) {
195 guard(mutex)(&ctx->uring_lock);
196 ret = io_install_bpf(ctx, ops);
197 }
198
199 fput(file);
200 return ret;
201}
202
203static void io_eject_bpf(struct io_ring_ctx *ctx)
204{
205 struct io_uring_bpf_ops *ops = ctx->bpf_ops;
206
207 if (WARN_ON_ONCE(!ops))
208 return;
209 if (WARN_ON_ONCE(ops->priv != ctx))
210 return;
211
212 ops->priv = NULL;
213 ctx->bpf_ops = NULL;
214 ctx->loop_step = NULL;
215}
216
217static void bpf_io_unreg(void *kdata, struct bpf_link *link)
218{
219 struct io_uring_bpf_ops *ops = kdata;
220 struct io_ring_ctx *ctx;
221
222 guard(mutex)(&io_bpf_ctrl_mutex);
223 ctx = ops->priv;
224 if (ctx) {
225 guard(mutex)(&ctx->uring_lock);
226 if (WARN_ON_ONCE(ctx->bpf_ops != ops))
227 return;
228
229 io_eject_bpf(ctx);
230 }
231}
232
233void io_unregister_bpf_ops(struct io_ring_ctx *ctx)
234{
235 /*
236 * ->bpf_ops is write protected by io_bpf_ctrl_mutex and uring_lock,
237 * and read protected by either. Try to avoid taking the global lock
238 * for rings that never had any bpf installed.
239 */
240 scoped_guard(mutex, &ctx->uring_lock) {
241 if (!ctx->bpf_ops)
242 return;
243 }
244
245 guard(mutex)(&io_bpf_ctrl_mutex);
246 guard(mutex)(&ctx->uring_lock);
247 if (ctx->bpf_ops)
248 io_eject_bpf(ctx);
249}
250
251static struct bpf_struct_ops bpf_ring_ops = {
252 .verifier_ops = &bpf_io_verifier_ops,
253 .reg = bpf_io_reg,
254 .unreg = bpf_io_unreg,
255 .check_member = bpf_io_check_member,
256 .init_member = bpf_io_init_member,
257 .init = bpf_io_init,
258 .cfi_stubs = &io_bpf_ops_stubs,
259 .name = "io_uring_bpf_ops",
260 .owner = THIS_MODULE,
261};
262
263static int __init io_uring_bpf_init(void)
264{
265 int ret;
266
267 ret = register_bpf_struct_ops(&bpf_ring_ops, io_uring_bpf_ops);
268 if (ret) {
269 pr_err("io_uring: Failed to register struct_ops (%d)\n", ret);
270 return ret;
271 }
272
273 return 0;
274}
275__initcall(io_uring_bpf_init);