projects/linux·blob uaccess-cleanups · refs

402 lines · 9.3 KiB

1// SPDX-License-Identifier: GPL-2.0-only
2/* Copyright (c) 2025 Meta Platforms, Inc. and affiliates. */
3
4#include <linux/bpf.h>
5#include <linux/filter.h>
6#include <linux/bpf_mem_alloc.h>
7#include <linux/gfp.h>
8#include <linux/memory.h>
9#include <linux/mutex.h>
10
11static void bpf_stream_elem_init(struct bpf_stream_elem *elem, int len)
12{
13 init_llist_node(&elem->node);
14 elem->total_len = len;
15 elem->consumed_len = 0;
16}
17
18static struct bpf_stream_elem *bpf_stream_elem_alloc(int len)
19{
20 const int max_len = ARRAY_SIZE((struct bpf_bprintf_buffers){}.buf);
21 struct bpf_stream_elem *elem;
22 size_t alloc_size;
23
24 /*
25 * Length denotes the amount of data to be written as part of stream element,
26 * thus includes '\0' byte. We're capped by how much bpf_bprintf_buffers can
27 * accomodate, therefore deny allocations that won't fit into them.
28 */
29 if (len < 0 || len > max_len)
30 return NULL;
31
32 alloc_size = offsetof(struct bpf_stream_elem, str[len]);
33 elem = kmalloc_nolock(alloc_size, __GFP_ZERO, -1);
34 if (!elem)
35 return NULL;
36
37 bpf_stream_elem_init(elem, len);
38
39 return elem;
40}
41
42static int __bpf_stream_push_str(struct llist_head *log, const char *str, int len)
43{
44 struct bpf_stream_elem *elem = NULL;
45
46 /*
47 * Allocate a bpf_prog_stream_elem and push it to the bpf_prog_stream
48 * log, elements will be popped at once and reversed to print the log.
49 */
50 elem = bpf_stream_elem_alloc(len);
51 if (!elem)
52 return -ENOMEM;
53
54 memcpy(elem->str, str, len);
55 llist_add(&elem->node, log);
56
57 return 0;
58}
59
60static int bpf_stream_consume_capacity(struct bpf_stream *stream, int len)
61{
62 if (atomic_read(&stream->capacity) >= BPF_STREAM_MAX_CAPACITY)
63 return -ENOSPC;
64 if (atomic_add_return(len, &stream->capacity) >= BPF_STREAM_MAX_CAPACITY) {
65 atomic_sub(len, &stream->capacity);
66 return -ENOSPC;
67 }
68 return 0;
69}
70
71static void bpf_stream_release_capacity(struct bpf_stream *stream, struct bpf_stream_elem *elem)
72{
73 int len = elem->total_len;
74
75 atomic_sub(len, &stream->capacity);
76}
77
78static int bpf_stream_push_str(struct bpf_stream *stream, const char *str, int len)
79{
80 int ret = bpf_stream_consume_capacity(stream, len);
81
82 return ret ?: __bpf_stream_push_str(&stream->log, str, len);
83}
84
85static struct bpf_stream *bpf_stream_get(enum bpf_stream_id stream_id, struct bpf_prog_aux *aux)
86{
87 if (stream_id != BPF_STDOUT && stream_id != BPF_STDERR)
88 return NULL;
89 return &aux->stream[stream_id - 1];
90}
91
92static void bpf_stream_free_elem(struct bpf_stream_elem *elem)
93{
94 kfree_nolock(elem);
95}
96
97static void bpf_stream_free_list(struct llist_node *list)
98{
99 struct bpf_stream_elem *elem, *tmp;
100
101 llist_for_each_entry_safe(elem, tmp, list, node)
102 bpf_stream_free_elem(elem);
103}
104
105static struct llist_node *bpf_stream_backlog_peek(struct bpf_stream *stream)
106{
107 return stream->backlog_head;
108}
109
110static struct llist_node *bpf_stream_backlog_pop(struct bpf_stream *stream)
111{
112 struct llist_node *node;
113
114 node = stream->backlog_head;
115 if (stream->backlog_head == stream->backlog_tail)
116 stream->backlog_head = stream->backlog_tail = NULL;
117 else
118 stream->backlog_head = node->next;
119 return node;
120}
121
122static void bpf_stream_backlog_fill(struct bpf_stream *stream)
123{
124 struct llist_node *head, *tail;
125
126 if (llist_empty(&stream->log))
127 return;
128 tail = llist_del_all(&stream->log);
129 if (!tail)
130 return;
131 head = llist_reverse_order(tail);
132
133 if (!stream->backlog_head) {
134 stream->backlog_head = head;
135 stream->backlog_tail = tail;
136 } else {
137 stream->backlog_tail->next = head;
138 stream->backlog_tail = tail;
139 }
140
141 return;
142}
143
144static bool bpf_stream_consume_elem(struct bpf_stream_elem *elem, int *len)
145{
146 int rem = elem->total_len - elem->consumed_len;
147 int used = min(rem, *len);
148
149 elem->consumed_len += used;
150 *len -= used;
151
152 return elem->consumed_len == elem->total_len;
153}
154
155static int bpf_stream_read(struct bpf_stream *stream, void __user *buf, int len)
156{
157 int rem_len = len, cons_len, ret = 0;
158 struct bpf_stream_elem *elem = NULL;
159 struct llist_node *node;
160
161 mutex_lock(&stream->lock);
162
163 while (rem_len) {
164 int pos = len - rem_len;
165 bool cont;
166
167 node = bpf_stream_backlog_peek(stream);
168 if (!node) {
169 bpf_stream_backlog_fill(stream);
170 node = bpf_stream_backlog_peek(stream);
171 }
172 if (!node)
173 break;
174 elem = container_of(node, typeof(*elem), node);
175
176 cons_len = elem->consumed_len;
177 cont = bpf_stream_consume_elem(elem, &rem_len) == false;
178
179 ret = copy_to_user(buf + pos, elem->str + cons_len,
180 elem->consumed_len - cons_len);
181 /* Restore in case of error. */
182 if (ret) {
183 ret = -EFAULT;
184 elem->consumed_len = cons_len;
185 break;
186 }
187
188 if (cont)
189 continue;
190 bpf_stream_backlog_pop(stream);
191 bpf_stream_release_capacity(stream, elem);
192 bpf_stream_free_elem(elem);
193 }
194
195 mutex_unlock(&stream->lock);
196 return ret ? ret : len - rem_len;
197}
198
199int bpf_prog_stream_read(struct bpf_prog *prog, enum bpf_stream_id stream_id, void __user *buf, int len)
200{
201 struct bpf_stream *stream;
202
203 stream = bpf_stream_get(stream_id, prog->aux);
204 if (!stream)
205 return -ENOENT;
206 return bpf_stream_read(stream, buf, len);
207}
208
209__bpf_kfunc_start_defs();
210
211/*
212 * Avoid using enum bpf_stream_id so that kfunc users don't have to pull in the
213 * enum in headers.
214 */
215__bpf_kfunc int bpf_stream_vprintk(int stream_id, const char *fmt__str, const void *args,
216 u32 len__sz, struct bpf_prog_aux *aux)
217{
218 struct bpf_bprintf_data data = {
219 .get_bin_args = true,
220 .get_buf = true,
221 };
222 u32 fmt_size = strlen(fmt__str) + 1;
223 struct bpf_stream *stream;
224 u32 data_len = len__sz;
225 int ret, num_args;
226
227 stream = bpf_stream_get(stream_id, aux);
228 if (!stream)
229 return -ENOENT;
230
231 if (data_len & 7 || data_len > MAX_BPRINTF_VARARGS * 8 ||
232 (data_len && !args))
233 return -EINVAL;
234 num_args = data_len / 8;
235
236 ret = bpf_bprintf_prepare(fmt__str, fmt_size, args, num_args, &data);
237 if (ret < 0)
238 return ret;
239
240 ret = bstr_printf(data.buf, MAX_BPRINTF_BUF, fmt__str, data.bin_args);
241 /* Exclude NULL byte during push. */
242 ret = bpf_stream_push_str(stream, data.buf, ret);
243 bpf_bprintf_cleanup(&data);
244
245 return ret;
246}
247
248/* Directly trigger a stack dump from the program. */
249__bpf_kfunc int bpf_stream_print_stack(int stream_id, struct bpf_prog_aux *aux)
250{
251 struct bpf_stream_stage ss;
252 struct bpf_prog *prog;
253
254 /* Make sure the stream ID is valid. */
255 if (!bpf_stream_get(stream_id, aux))
256 return -ENOENT;
257
258 prog = aux->main_prog_aux->prog;
259
260 bpf_stream_stage(ss, prog, stream_id, ({
261 bpf_stream_dump_stack(ss);
262 }));
263
264 return 0;
265}
266
267__bpf_kfunc_end_defs();
268
269/* Added kfunc to common_btf_ids */
270
271void bpf_prog_stream_init(struct bpf_prog *prog)
272{
273 int i;
274
275 for (i = 0; i < ARRAY_SIZE(prog->aux->stream); i++) {
276 atomic_set(&prog->aux->stream[i].capacity, 0);
277 init_llist_head(&prog->aux->stream[i].log);
278 mutex_init(&prog->aux->stream[i].lock);
279 prog->aux->stream[i].backlog_head = NULL;
280 prog->aux->stream[i].backlog_tail = NULL;
281 }
282}
283
284void bpf_prog_stream_free(struct bpf_prog *prog)
285{
286 struct llist_node *list;
287 int i;
288
289 for (i = 0; i < ARRAY_SIZE(prog->aux->stream); i++) {
290 list = llist_del_all(&prog->aux->stream[i].log);
291 bpf_stream_free_list(list);
292 bpf_stream_free_list(prog->aux->stream[i].backlog_head);
293 }
294}
295
296void bpf_stream_stage_init(struct bpf_stream_stage *ss)
297{
298 init_llist_head(&ss->log);
299 ss->len = 0;
300}
301
302void bpf_stream_stage_free(struct bpf_stream_stage *ss)
303{
304 struct llist_node *node;
305
306 node = llist_del_all(&ss->log);
307 bpf_stream_free_list(node);
308}
309
310int bpf_stream_stage_printk(struct bpf_stream_stage *ss, const char *fmt, ...)
311{
312 struct bpf_bprintf_buffers *buf;
313 va_list args;
314 int ret;
315
316 if (bpf_try_get_buffers(&buf))
317 return -EBUSY;
318
319 va_start(args, fmt);
320 ret = vsnprintf(buf->buf, ARRAY_SIZE(buf->buf), fmt, args);
321 va_end(args);
322 ss->len += ret;
323 /* Exclude NULL byte during push. */
324 ret = __bpf_stream_push_str(&ss->log, buf->buf, ret);
325 bpf_put_buffers();
326 return ret;
327}
328
329int bpf_stream_stage_commit(struct bpf_stream_stage *ss, struct bpf_prog *prog,
330 enum bpf_stream_id stream_id)
331{
332 struct llist_node *list, *head, *tail;
333 struct bpf_stream *stream;
334 int ret;
335
336 stream = bpf_stream_get(stream_id, prog->aux);
337 if (!stream)
338 return -EINVAL;
339
340 ret = bpf_stream_consume_capacity(stream, ss->len);
341 if (ret)
342 return ret;
343
344 list = llist_del_all(&ss->log);
345 head = tail = list;
346
347 if (!list)
348 return 0;
349 while (llist_next(list)) {
350 tail = llist_next(list);
351 list = tail;
352 }
353 llist_add_batch(head, tail, &stream->log);
354 return 0;
355}
356
357struct dump_stack_ctx {
358 struct bpf_stream_stage *ss;
359 int err;
360};
361
362static bool dump_stack_cb(void *cookie, u64 ip, u64 sp, u64 bp)
363{
364 struct dump_stack_ctx *ctxp = cookie;
365 const char *file = "", *line = "";
366 struct bpf_prog *prog;
367 int num, ret;
368
369 rcu_read_lock();
370 prog = bpf_prog_ksym_find(ip);
371 rcu_read_unlock();
372 if (prog) {
373 ret = bpf_prog_get_file_line(prog, ip, &file, &line, &num);
374 if (ret < 0)
375 goto end;
376 ctxp->err = bpf_stream_stage_printk(ctxp->ss, "%pS\n %s @ %s:%d\n",
377 (void *)(long)ip, line, file, num);
378 return !ctxp->err;
379 }
380end:
381 ctxp->err = bpf_stream_stage_printk(ctxp->ss, "%pS\n", (void *)(long)ip);
382 return !ctxp->err;
383}
384
385int bpf_stream_stage_dump_stack(struct bpf_stream_stage *ss)
386{
387 struct dump_stack_ctx ctx = { .ss = ss };
388 int ret;
389
390 ret = bpf_stream_stage_printk(ss, "CPU: %d UID: %d PID: %d Comm: %s\n",
391 raw_smp_processor_id(), __kuid_val(current_real_cred()->euid),
392 current->pid, current->comm);
393 if (ret)
394 return ret;
395 ret = bpf_stream_stage_printk(ss, "Call trace:\n");
396 if (ret)
397 return ret;
398 arch_bpf_stack_walk(dump_stack_cb, &ctx);
399 if (ctx.err)
400 return ctx.err;
401 return bpf_stream_stage_printk(ss, "\n");
402}