linux - kernel/bpf/diagnostics.c

2359 lines · 63.9 KiB

1	// SPDX-License-Identifier: GPL-2.0-only
2	// Copyright (c) 2026 Meta Platforms, Inc. and affiliates.
3
4	#include <linux/bpf.h>
5	#include <linux/bpf_verifier.h>
6	#include <linux/btf.h>
7	#include <linux/ctype.h>
8	#include <linux/kernel.h>
9	#include <linux/list.h>
10	#include <linux/seq_buf.h>
11	#include <linux/overflow.h>
12	#include <linux/slab.h>
13	#include <linux/stdarg.h>
14	#include <linux/string.h>
15
16	#include "disasm.h"
17	#include "diagnostics.h"
18
19	#define REGISTER_TYPE_SAFETY "Register Type Safety"
20	#define MEMORY_SAFETY "Memory Safety"
21	#define RESOURCE_LIFETIME_SAFETY "Resource Lifetime Safety"
22	#define CALL_TYPE_SAFETY "Call Type Safety"
23	#define EXECUTION_CONTEXT_SAFETY "Execution Context Safety"
24	#define PROGRAM_STRUCTURE "Program Structure"
25	#define POLICY "Policy"
26
27	#define BPF_DIAG_TEXT_WIDTH 100
28	#define BPF_DIAG_TEXT_INDENT " "
29	#define BPF_DIAG_CONTEXT 2
30	#define BPF_DIAG_CONTEXT_CNT (1 + BPF_DIAG_CONTEXT * 2)
31	#define BPF_DIAG_HISTORY_RENDER_MAX 64
32	#define BPF_DIAG_SOURCE_LANE_WIDTH 88
33	#define BPF_DIAG_TAB_WIDTH 8
34	#define BPF_DIAG_FMT_CHUNK_SIZE (PAGE_SIZE - sizeof(struct diag_fmt_chunk))
35	#define BPF_DIAG_FMT_BUF_SIZE 256
36	#define BPF_DIAG_EVENT_LOG_MAX_SIZE (64U << 20)
37	#define DISASM_LINE_LEN 160
38
39	enum bpf_diag_mod_target_kind {
40	BPF_DIAG_MOD_TARGET_NONE,
41	BPF_DIAG_MOD_TARGET_REG,
42	BPF_DIAG_MOD_TARGET_STACK_ARG,
43	BPF_DIAG_MOD_TARGET_STACK_SLOT,
44	BPF_DIAG_MOD_TARGET_STACK_RANGE,
45	};
46
47	struct bpf_diag_mod_target {
48	u32 frame_id;
49	union {
50	struct {
51	s16 min_off;
52	s16 max_off;
53	} range;
54	u16 spi;
55	u8 regno;
56	u8 stack_arg;
57	};
58	u8 frameno;
59	u8 kind;
60	};
61
62	static struct bpf_diag_mod_target diag_reg_target(u32 frame_id, u8 frameno, u8 regno)
63	{
64	return (struct bpf_diag_mod_target){
65	.frame_id = frame_id,
66	.frameno = frameno,
67	.kind = BPF_DIAG_MOD_TARGET_REG,
68	.regno = regno,
69	};
70	}
71
72	static struct bpf_diag_mod_target diag_stack_arg_target(u32 frame_id, u8 frameno, u8 slot)
73	{
74	return (struct bpf_diag_mod_target){
75	.frame_id = frame_id,
76	.frameno = frameno,
77	.kind = BPF_DIAG_MOD_TARGET_STACK_ARG,
78	.stack_arg = slot,
79	};
80	}
81
82	static struct bpf_diag_mod_target diag_stack_slot_target(u32 frame_id, u8 frameno, u16 spi)
83	{
84	return (struct bpf_diag_mod_target){
85	.frame_id = frame_id,
86	.frameno = frameno,
87	.kind = BPF_DIAG_MOD_TARGET_STACK_SLOT,
88	.spi = spi,
89	};
90	}
91
92	static struct bpf_diag_mod_target diag_stack_range_target(u32 frame_id, u8 frameno,
93	s16 min_off, s16 max_off)
94	{
95	return (struct bpf_diag_mod_target){
96	.frame_id = frame_id,
97	.frameno = frameno,
98	.kind = BPF_DIAG_MOD_TARGET_STACK_RANGE,
99	.range.min_off = min_off,
100	.range.max_off = max_off,
101	};
102	}
103
104	struct bpf_diag_reg_snapshot {
105	u32 type;
106	u32 btf_id;
107	const struct bpf_map *map_ptr;
108	const struct btf *btf;
109	struct tnum var_off;
110	struct cnum64 r64;
111	};
112
113	enum bpf_diag_history_kind {
114	BPF_DIAG_HISTORY_BRANCH,
115	BPF_DIAG_HISTORY_MOD,
116	BPF_DIAG_HISTORY_REF_ACQUIRE,
117	BPF_DIAG_HISTORY_REF_RELEASE,
118	BPF_DIAG_HISTORY_CONTEXT,
119	};
120
121	struct bpf_diag_history_event {
122	u32 insn_idx : 24;
123	u32 kind : 8;
124	u8 in_lineage : 1;
125	union {
126	struct {
127	bool cond_true;
128	} branch;
129	struct {
130	struct bpf_diag_mod_target target;
131	struct bpf_diag_mod_target origin;
132	struct bpf_diag_reg_snapshot old, new;
133	u8 reason;
134	bool origin_valid;
135	} mod;
136	struct {
137	u32 ref_id;
138	} ref;
139	struct {
140	u32 depth;
141	u8 kind;
142	bool enter;
143	} ctx;
144	};
145	};
146
147	enum bpf_diag_history_scope {
148	BPF_DIAG_HISTORY_SCOPE_REG,
149	BPF_DIAG_HISTORY_SCOPE_STACK_ARG,
150	BPF_DIAG_HISTORY_SCOPE_REF,
151	BPF_DIAG_HISTORY_SCOPE_CONTEXT,
152	};
153
154	struct bpf_diag_history_opts {
155	enum bpf_diag_history_scope scope;
156	u32 frame_id;
157	u32 frameno;
158	int regno;
159	int stack_arg_slot;
160	u32 ref_id;
161	enum bpf_diag_context_kind ctx_kind;
162	u32 ctx_depth;
163	};
164
165	static void diag_print_history(struct bpf_verifier_env *env,
166	const struct bpf_diag_history_opts *opts);
167	static bool diag_target_matches(const struct bpf_diag_mod_target *event_target,
168	const struct bpf_diag_mod_target *target);
169	static const char *diag_context_name(enum bpf_diag_context_kind kind);
170	struct disasm_line {
171	char text[DISASM_LINE_LEN];
172	int idx;
173	bool valid;
174	};
175
176	struct disasm_ctx {
177	struct bpf_verifier_env *env;
178	struct seq_buf seq;
179	};
180
181	struct diag_fmt_chunk {
182	struct list_head node;
183	struct seq_buf seq;
184	char data[];
185	};
186
187	struct diag_fmt_mark {
188	struct diag_fmt_chunk *chunk;
189	size_t len;
190	};
191
192	struct bpf_diag_log {
193	struct bpf_diag_history_event *events;
194	/* Sequence number of the oldest retained event on the active path. */
195	u64 first_seq;
196	u32 cnt;
197	u32 cap;
198	u32 head;
199	bool growth_failed;
200	};
201
202	struct bpf_diag_scratch {
203	struct bpf_linfo_source source_lines[BPF_DIAG_CONTEXT_CNT];
204	struct disasm_line disasm_lines[BPF_DIAG_CONTEXT_CNT];
205	};
206
207	struct bpf_diag_mod_scope {
208	struct bpf_reg_state target_reg_snapshot;
209	struct bpf_diag_mod_target target;
210	struct bpf_diag_mod_target origin;
211	enum bpf_diag_mod_reason reason;
212	u32 insn_idx;
213	bool active;
214	bool origin_valid;
215	};
216
217	struct bpf_diag {
218	struct bpf_diag_log log;
219	struct bpf_diag_scratch scratch;
220	struct list_head fmt_chunks;
221	struct bpf_diag_mod_scope mod;
222	u32 frame_id_gen;
223	};
224
225	bool bpf_diag_enabled(const struct bpf_verifier_env *env)
226	{
227	return env->log.level & BPF_LOG_LEVEL;
228	}
229
230	static void diag_write(struct bpf_verifier_env env, const char fmt, ...) __printf(2, 3);
231
232	int bpf_diag_init(struct bpf_verifier_env *env)
233	{
234	if (!bpf_diag_enabled(env))
235	return 0;
236
237	env->diag = kzalloc_obj(struct bpf_diag, GFP_KERNEL_ACCOUNT);
238	if (!env->diag)
239	return -ENOMEM;
240
241	INIT_LIST_HEAD(&env->diag->fmt_chunks);
242	return 0;
243	}
244
245	void bpf_diag_init_frame(struct bpf_verifier_env env, struct bpf_func_state state)
246	{
247	if (env->diag)
248	state->diag_frame_id = ++env->diag->frame_id_gen;
249	}
250
251	static char diag_fmt_alloc(struct bpf_verifier_env env, size_t size)
252	{
253	struct bpf_diag *diag = env->diag;
254	struct diag_fmt_chunk *chunk;
255	size_t capacity, available;
256	char *buf;
257
258	if (!diag \|\| !size \|\| size > INT_MAX)
259	return NULL;
260
261	if (!list_empty(&diag->fmt_chunks)) {
262	chunk = list_last_entry(&diag->fmt_chunks, struct diag_fmt_chunk, node);
263	available = seq_buf_get_buf(&chunk->seq, &buf);
264	if (available >= size)
265	goto commit;
266	}
267
268	capacity = max_t(size_t, BPF_DIAG_FMT_CHUNK_SIZE, size);
269	chunk = kmalloc_flex(*chunk, data, capacity, GFP_KERNEL_ACCOUNT);
270	if (!chunk)
271	return NULL;
272
273	seq_buf_init(&chunk->seq, chunk->data, capacity);
274	list_add_tail(&chunk->node, &diag->fmt_chunks);
275	available = seq_buf_get_buf(&chunk->seq, &buf);
276	if (WARN_ON_ONCE(available < size))
277	return NULL;
278
279	commit:
280	seq_buf_commit(&chunk->seq, size);
281	return buf;
282	}
283
284	char bpf_diag_fmt_buf(struct bpf_verifier_env env, size_t size)
285	{
286	char *buf;
287
288	buf = diag_fmt_alloc(env, size);
289	if (buf)
290	buf[0] = '\0';
291	return buf;
292	}
293
294	const char bpf_diag_vfmt(struct bpf_verifier_env env, const char *fmt, va_list args)
295	{
296	va_list copy;
297	char *buf;
298	int len;
299
300	va_copy(copy, args);
301	len = vsnprintf(NULL, 0, fmt, copy);
302	va_end(copy);
303	if (len < 0 \|\| len == INT_MAX)
304	return "";
305
306	buf = diag_fmt_alloc(env, len + 1);
307	if (buf)
308	vsnprintf(buf, len + 1, fmt, args);
309	return buf ?: "";
310	}
311
312	const char bpf_diag_fmt(struct bpf_verifier_env env, const char *fmt, ...)
313	{
314	const char *buf;
315	va_list args;
316
317	va_start(args, fmt);
318	buf = bpf_diag_vfmt(env, fmt, args);
319	va_end(args);
320	return buf;
321	}
322
323	static struct diag_fmt_mark diag_fmt_save(struct bpf_verifier_env *env)
324	{
325	struct bpf_diag *diag = env->diag;
326	struct diag_fmt_mark mark = {};
327
328	if (!diag \|\| list_empty(&diag->fmt_chunks))
329	return mark;
330
331	mark.chunk = list_last_entry(&diag->fmt_chunks, struct diag_fmt_chunk, node);
332	mark.len = mark.chunk->seq.len;
333	return mark;
334	}
335
336	static void diag_fmt_restore(struct bpf_verifier_env *env, struct diag_fmt_mark mark)
337	{
338	struct bpf_diag *diag = env->diag;
339	struct diag_fmt_chunk *chunk;
340
341	if (!diag)
342	return;
343
344	while (!list_empty(&diag->fmt_chunks)) {
345	chunk = list_last_entry(&diag->fmt_chunks, struct diag_fmt_chunk, node);
346	if (chunk == mark.chunk)
347	break;
348	list_del(&chunk->node);
349	kfree(chunk);
350	}
351
352	if (mark.chunk) {
353	mark.chunk->seq.len = mark.len;
354	seq_buf_str(&mark.chunk->seq);
355	}
356	}
357
358	void bpf_diag_free(struct bpf_verifier_env *env)
359	{
360	struct bpf_diag *diag = env->diag;
361
362	if (!diag)
363	return;
364
365	diag_fmt_restore(env, (struct diag_fmt_mark){});
366	kvfree(diag->log.events);
367	kfree(diag);
368	env->diag = NULL;
369	}
370
371	static void diag_write(struct bpf_verifier_env env, const char fmt, ...)
372	{
373	va_list args;
374
375	if (!bpf_diag_enabled(env))
376	return;
377
378	va_start(args, fmt);
379	bpf_verifier_vlog(&env->log, fmt, args);
380	va_end(args);
381	}
382
383	static u64 log_end(const struct bpf_diag_log *log)
384	{
385	return log->first_seq + log->cnt;
386	}
387
388	static u32 log_pos(const struct bpf_diag_log *log, u32 idx)
389	{
390	u32 pos = log->head + idx;
391
392	return pos < log->cap ? pos : pos - log->cap;
393	}
394
395	u64 bpf_diag_event_log_save(struct bpf_verifier_env *env)
396	{
397	struct bpf_diag *diag = env->diag;
398
399	return diag ? log_end(&diag->log) : 0;
400	}
401
402	void bpf_diag_event_log_restore(struct bpf_verifier_env *env, u64 log_pos)
403	{
404	struct bpf_diag *diag = env->diag;
405	struct bpf_diag_log *log;
406	u64 end_seq;
407
408	if (!diag)
409	return;
410
411	log = &diag->log;
412	end_seq = log_end(log);
413	if (WARN_ON_ONCE(log_pos > end_seq))
414	log_pos = end_seq;
415
416	/*
417	* A deep abandoned path may have rotated away the shared prefix. In
418	* that case, restart with an empty retained suffix and remember that
419	* every event before the restored mark is unavailable.
420	*/
421	if (log_pos <= log->first_seq) {
422	log->first_seq = log_pos;
423	log->head = 0;
424	log->cnt = 0;
425	return;
426	}
427
428	log->cnt = log_pos - log->first_seq;
429	}
430
431	u32 bpf_diag_irq_depth(const struct bpf_verifier_state *state)
432	{
433	u32 depth = 0;
434	int i;
435
436	for (i = 0; i < state->acquired_refs; i++) {
437	if (state->refs[i].type == REF_TYPE_IRQ)
438	depth++;
439	}
440
441	return depth;
442	}
443
444	static void diag_append_history(struct bpf_verifier_env *env,
445	const struct bpf_diag_history_event *event)
446	{
447	struct bpf_diag_history_event *events;
448	struct bpf_diag *diag = env->diag;
449	struct bpf_diag_log *log;
450	u32 cap, max_events;
451
452	if (!diag)
453	return;
454	log = &diag->log;
455
456	if (log->cnt < log->cap) {
457	log->events[log_pos(log, log->cnt++)] = *event;
458	return;
459	}
460
461	max_events = BPF_DIAG_EVENT_LOG_MAX_SIZE / sizeof(*events);
462	if (log->growth_failed \|\| log->cap == max_events)
463	goto rotate;
464
465	cap = min(log->cap ? log->cap * 2 : 64, max_events);
466	events = kvrealloc(log->events, array_size(cap, sizeof(*events)), GFP_KERNEL_ACCOUNT);
467	if (!events) {
468	log->growth_failed = true;
469	goto rotate;
470	}
471	log->events = events;
472	log->cap = cap;
473	log->events[log->cnt++] = *event;
474	return;
475
476	rotate:
477	if (log->cap) {
478	log->events[log->head++] = *event;
479	if (log->head == log->cap)
480	log->head = 0;
481	}
482	log->first_seq++;
483	}
484
485	static void diag_print_wrapped_prefixed(struct bpf_verifier_env env, const char first_prefix,
486	const char next_prefix, const char text)
487	{
488	const char *prefix = first_prefix;
489
490	while (*text) {
491	const char *line = text;
492	int prefix_len = strlen(prefix);
493	int text_width = BPF_DIAG_TEXT_WIDTH - prefix_len;
494	int len = 0, last_space = -1;
495
496	if (text_width < 1)
497	text_width = 1;
498
499	while (line[len] && line[len] != '\n' && len < text_width) {
500	if (line[len] == ' ')
501	last_space = len;
502	len++;
503	}
504
505	if (line[len] && line[len] != '\n' && line[len] != ' ' && last_space > 0)
506	len = last_space;
507
508	diag_write(env, "%s%.*s\n", prefix, len, line);
509
510	text = line + len;
511	while (*text == ' ')
512	text++;
513	if (*text == '\n')
514	text++;
515
516	prefix = next_prefix;
517	}
518	}
519
520	const char bpf_diag_fmt_btf_type(struct bpf_verifier_env env, const struct btf *btf, u32 type_id)
521	{
522	char *buf = bpf_diag_fmt_buf(env, BPF_DIAG_FMT_BUF_SIZE);
523	size_t len;
524	int ret;
525
526	if (!buf)
527	return "";
528
529	buf[0] = '\0';
530	ret = btf_type_name_to_buf(btf, type_id, buf, BPF_DIAG_FMT_BUF_SIZE);
531	if (ret < 0 \|\| !buf[0]) {
532	scnprintf(buf, BPF_DIAG_FMT_BUF_SIZE, "BTF type ID %u", type_id);
533	return buf;
534	}
535
536	len = strlen(buf);
537	if (len && buf[len - 1] == '{')
538	buf[len - 1] = '\0';
539	return buf;
540	}
541
542	static void diag_vprint_indented(struct bpf_verifier_env env, const char fmt, va_list args)
543	__printf(2, 0);
544
545	static void diag_vprint_indented(struct bpf_verifier_env env, const char fmt, va_list args)
546	{
547	char *buf;
548
549	if (!bpf_diag_enabled(env))
550	return;
551
552	buf = kvasprintf(GFP_KERNEL_ACCOUNT, fmt, args);
553	if (!buf) {
554	diag_write(env, "%s<failed to allocate diagnostic text>\n", BPF_DIAG_TEXT_INDENT);
555	return;
556	}
557
558	diag_print_wrapped_prefixed(env, BPF_DIAG_TEXT_INDENT, BPF_DIAG_TEXT_INDENT, buf);
559	kfree(buf);
560	}
561
562	static int diag_line_width(unsigned int line)
563	{
564	int width = 1;
565
566	while (line >= 10) {
567	line /= 10;
568	width++;
569	}
570
571	return width;
572	}
573
574	static int diag_line_indent(const char *line)
575	{
576	int indent = 0;
577
578	while (line == ' ' \|\| line == '\t') {
579	if (*line == '\t')
580	indent = round_up(indent + 1, BPF_DIAG_TAB_WIDTH);
581	else
582	indent++;
583	line++;
584	}
585
586	return indent;
587	}
588
589	static void disasm_print(void private_data, const char fmt, ...) __printf(2, 3);
590
591	static void disasm_print(void private_data, const char fmt, ...)
592	{
593	struct disasm_ctx *ctx = private_data;
594	va_list args;
595
596	va_start(args, fmt);
597	seq_buf_vprintf(&ctx->seq, fmt, args);
598	va_end(args);
599	}
600
601	static const char disasm_kfunc_name(void private_data, const struct bpf_insn *insn)
602	{
603	struct disasm_ctx *ctx = private_data;
604
605	return bpf_disasm_kfunc_name(ctx->env, insn);
606	}
607
608	static void format_disasm_line(struct bpf_verifier_env *env, int insn_idx,
609	struct disasm_line *line)
610	{
611	struct disasm_ctx ctx = { .env = env };
612	struct bpf_insn *insn;
613	const struct bpf_insn_cbs cbs = {
614	.cb_call = disasm_kfunc_name,
615	.cb_print = disasm_print,
616	.private_data = &ctx,
617	};
618
619	line->idx = insn_idx;
620	line->valid = false;
621	seq_buf_init(&ctx.seq, line->text, sizeof(line->text));
622
623	if (insn_idx < 0 \|\| insn_idx >= env->prog->len)
624	return;
625
626	if (insn_idx > 0 && bpf_is_ldimm64(&env->prog->insnsi[insn_idx - 1]))
627	return;
628
629	insn = &env->prog->insnsi[insn_idx];
630	if (bpf_is_ldimm64(insn) && insn_idx + 1 >= env->prog->len)
631	return;
632
633	print_bpf_insn(&cbs, insn, env->allow_ptr_leaks);
634	seq_buf_str(&ctx.seq);
635	ctx.seq.len = strnlen(line->text, sizeof(line->text));
636	while (ctx.seq.len && line->text[ctx.seq.len - 1] == '\n')
637	seq_buf_pop(&ctx.seq);
638	seq_buf_str(&ctx.seq);
639
640	line->valid = true;
641	}
642
643	static void diag_format_source_text(char buf, size_t size, const char line, int width)
644	{
645	int col = 0, len = 0;
646
647	if (!size)
648	return;
649	if (width <= 0) {
650	buf[0] = '\0';
651	return;
652	}
653
654	line = line ?: "...";
655	while (*line && col < width && len + 1 < size) {
656	if (*line == '\t') {
657	int next = round_up(col + 1, BPF_DIAG_TAB_WIDTH);
658
659	while (col < next && col < width && len + 1 < size) {
660	buf[len++] = ' ';
661	col++;
662	}
663	line++;
664	continue;
665	}
666
667	buf[len++] = *line++;
668	col++;
669	}
670
671	if (*line) {
672	int ellipsis_len = min(3, width);
673
674	while (len > 0 && col > width - ellipsis_len) {
675	len--;
676	col--;
677	}
678	while (ellipsis_len-- && len + 1 < size)
679	buf[len++] = '.';
680	}
681
682	buf[len] = '\0';
683	}
684
685	static void diag_format_source_lane(char buf, size_t size, const char source_prefix,
686	int source_line_width, int line_num, const char *line)
687	{
688	int len, text_width;
689
690	if (line_num <= 0) {
691	buf[0] = '\0';
692	return;
693	}
694
695	len = scnprintf(buf, size, "%s%*d \| ", source_prefix, source_line_width, line_num);
696	text_width = BPF_DIAG_SOURCE_LANE_WIDTH - len;
697	diag_format_source_text(buf + len, size - len, line, text_width);
698	}
699
700	static void bpf_diag_header(struct bpf_verifier_env env, const char category,
701	const char *problem)
702	{
703	char first;
704
705	if (!bpf_diag_enabled(env))
706	return;
707
708	category = category ?: "Verifier Error";
709	problem = problem ?: "";
710
711	if (!problem[0]) {
712	diag_write(env, "\nVerification failed: %s\n", category);
713	return;
714	}
715
716	first = toupper(problem[0]);
717	diag_write(env, "\nVerification failed: %s: %c%s\n", category, first, problem + 1);
718	}
719
720	static void diag_reason(struct bpf_verifier_env env, const char fmt, ...) __printf(2, 3);
721	static void diag_suggestion(struct bpf_verifier_env env, const char fmt, ...)
722	__printf(2, 3);
723
724	static void diag_section(struct bpf_verifier_env env, const char title)
725	{
726	if (!bpf_diag_enabled(env))
727	return;
728
729	diag_write(env, "\n%s:\n", title);
730	}
731
732	static void diag_reason(struct bpf_verifier_env env, const char fmt, ...)
733	{
734	va_list args;
735
736	if (!bpf_diag_enabled(env))
737	return;
738
739	diag_section(env, "Reason");
740
741	va_start(args, fmt);
742	diag_vprint_indented(env, fmt, args);
743	va_end(args);
744	}
745
746	static void diag_suggestion(struct bpf_verifier_env env, const char fmt, ...)
747	{
748	va_list args;
749
750	if (!bpf_diag_enabled(env))
751	return;
752
753	diag_section(env, "Suggestion");
754
755	va_start(args, fmt);
756	diag_vprint_indented(env, fmt, args);
757	va_end(args);
758	diag_write(env, "\n");
759	}
760
761	static void diag_print_source_annotation(struct bpf_verifier_env *env, int line_width, int indent,
762	const char label, const char msg)
763	{
764	const char first_prefix, next_prefix, *text;
765
766	indent = min_t(int, indent, max_t(int, 0, BPF_DIAG_SOURCE_LANE_WIDTH - line_width - 8));
767	text = bpf_diag_fmt(env, "%s: %s", label, msg);
768	first_prefix = bpf_diag_fmt(env, " %s \| %s^-- ", line_width + 4, "", indent, "");
769	next_prefix = bpf_diag_fmt(env, " %s \| %s ", line_width + 4, "", indent, "");
770
771	diag_print_wrapped_prefixed(env, first_prefix, next_prefix, text);
772	}
773
774	static void diag_print_insn_context(struct bpf_verifier_env *env, u32 insn_idx,
775	struct disasm_line *disasm_lines)
776	{
777	int insn_width = diag_line_width(env->prog->len ? env->prog->len - 1 : 0);
778	int i;
779
780	for (i = 0; i < BPF_DIAG_CONTEXT_CNT; i++) {
781	int row = i - BPF_DIAG_CONTEXT;
782
783	format_disasm_line(env, insn_idx + row, &disasm_lines[i]);
784	}
785
786	diag_write(env, " Instruction context:\n");
787	for (i = 0; i < BPF_DIAG_CONTEXT_CNT; i++) {
788	struct disasm_line *line = &disasm_lines[i];
789
790	if (line->valid)
791	diag_write(env, " %s%*d \| %s\n",
792	line->idx == insn_idx ? ">>> " : " ",
793	insn_width, line->idx, line->text);
794	}
795	}
796
797	static void bpf_diag_source(struct bpf_verifier_env env, u32 insn_idx, const char label,
798	const char *fmt, ...)
799	{
800	struct bpf_diag_scratch *scratch;
801	struct bpf_linfo_source *source_lines;
802	struct disasm_line *disasm_lines;
803	struct bpf_linfo_source src = {};
804	struct diag_fmt_mark mark;
805	const struct bpf_line_info *linfo;
806	const struct bpf_subprog_info *subprog;
807	struct btf *btf = env->prog->aux->btf;
808	char *source_lane;
809	const char *msg;
810	const char *func;
811	int start_line, end_line, width, indent, subprogno, linfo_start, linfo_end, i;
812	va_list args;
813
814	if (!bpf_diag_enabled(env))
815	return;
816	if (!env->diag)
817	return;
818
819	mark = diag_fmt_save(env);
820	label = label ?: "note";
821	scratch = &env->diag->scratch;
822	source_lines = scratch->source_lines;
823	disasm_lines = scratch->disasm_lines;
824	memset(source_lines, 0, sizeof(scratch->source_lines));
825	memset(disasm_lines, 0, sizeof(scratch->disasm_lines));
826
827	va_start(args, fmt);
828	msg = bpf_diag_vfmt(env, fmt, args);
829	va_end(args);
830	if (!*msg)
831	msg = "<failed to allocate diagnostic text>";
832
833	linfo = bpf_find_linfo(env->prog, insn_idx);
834	if (btf && linfo)
835	bpf_get_linfo_source(btf, linfo, &src);
836	if (!src.file \|\| !*src.file) {
837	diag_write(env, " insn %u\n", insn_idx);
838	goto out_annotation;
839	}
840
841	subprog = bpf_find_containing_subprog(env, insn_idx);
842	subprogno = subprog ? subprog - env->subprog_info : -ENOENT;
843	func = subprogno >= 0 ? bpf_subprog_name(env, subprogno) : NULL;
844	if (func && *func)
845	diag_write(env, " %s @ %s:%d:%d\n", func, src.file, src.line_num, src.line_col);
846	else
847	diag_write(env, " %s:%d:%d\n", src.file, src.line_num, src.line_col);
848	if (!src.line \|\| !*src.line)
849	goto out_annotation;
850
851	start_line = src.line_num - BPF_DIAG_CONTEXT;
852	end_line = src.line_num + BPF_DIAG_CONTEXT;
853	width = diag_line_width(end_line);
854	indent = diag_line_indent(src.line);
855	for (i = 0; i < BPF_DIAG_CONTEXT_CNT; i++)
856	source_lines[i].line_num = start_line + i;
857
858	linfo = env->prog->aux->linfo;
859	linfo_start = subprog ? subprog->linfo_idx : 0;
860	linfo_end = subprogno >= 0 && subprogno + 1 < env->subprog_cnt ?
861	env->subprog_info[subprogno + 1].linfo_idx : env->prog->aux->nr_linfo;
862	for (i = linfo_start; i < linfo_end; i++) {
863	struct bpf_linfo_source line_src;
864	int idx;
865
866	bpf_get_linfo_source(btf, &linfo[i], &line_src);
867	if (line_src.file_name_off != src.file_name_off \|\|
868	line_src.line_num < start_line \|\| line_src.line_num > end_line \|\|
869	!line_src.line \|\| !*line_src.line)
870	continue;
871
872	idx = line_src.line_num - start_line;
873	if (!source_lines[idx].line)
874	source_lines[idx] = line_src;
875	}
876
877	diag_write(env, " Source context:\n");
878	source_lane = bpf_diag_fmt_buf(env, BPF_DIAG_FMT_BUF_SIZE);
879	if (!source_lane)
880	goto out_restore;
881	for (i = 0; i < BPF_DIAG_CONTEXT_CNT; i++) {
882	const char *source_prefix;
883
884	source_prefix = source_lines[i].line_num == src.line_num ? ">>> " : " ";
885	diag_format_source_lane(source_lane, BPF_DIAG_FMT_BUF_SIZE, source_prefix, width,
886	source_lines[i].line_num, source_lines[i].line);
887	diag_write(env, " %s\n", source_lane);
888	if (source_lines[i].line_num == src.line_num)
889	diag_print_source_annotation(env, width, indent, label, msg);
890	}
891	diag_print_insn_context(env, insn_idx, disasm_lines);
892	goto out_restore;
893
894	out_annotation:
895	diag_print_source_annotation(env, 0, 0, label, msg);
896	diag_print_insn_context(env, insn_idx, disasm_lines);
897	out_restore:
898	diag_fmt_restore(env, mark);
899	}
900
901	static const struct bpf_func_state diag_current_frame(const struct bpf_verifier_env env)
902	{
903	return env->cur_state->frame[env->cur_state->curframe];
904	}
905
906	void bpf_diag_register_type(struct bpf_verifier_env *env, u32 insn_idx, int regno,
907	const char problem, const char reason, const char *suggestion)
908	{
909	const struct bpf_func_state *frame = diag_current_frame(env);
910	struct bpf_diag_history_opts opts = {
911	.scope = BPF_DIAG_HISTORY_SCOPE_REG,
912	.frame_id = frame->diag_frame_id,
913	.frameno = frame->frameno,
914	.regno = regno,
915	};
916
917	bpf_diag_header(env, REGISTER_TYPE_SAFETY, problem);
918	diag_reason(env, "%s", reason);
919
920	diag_section(env, "At");
921	bpf_diag_source(env, insn_idx, "error", "%s", problem);
922
923	if (regno >= 0)
924	diag_print_history(env, &opts);
925
926	diag_suggestion(env, "%s", suggestion);
927	}
928
929	const char bpf_diag_reg_type_plain(struct bpf_verifier_env env, enum bpf_reg_type type)
930	{
931	switch (base_type(type)) {
932	case NOT_INIT:
933	return "an uninitialized value";
934	case SCALAR_VALUE:
935	return "an integer scalar";
936	case PTR_TO_CTX:
937	return "a context pointer";
938	case PTR_TO_STACK:
939	return "a stack pointer";
940	case PTR_TO_MAP_VALUE:
941	if (type_may_be_null(type))
942	return "a nullable map value pointer";
943	return "a map value pointer";
944	case PTR_TO_MEM:
945	if (type_may_be_null(type))
946	return "a nullable memory pointer";
947	return "a memory pointer";
948	case PTR_TO_BTF_ID:
949	if (type_may_be_null(type))
950	return "a nullable kernel object pointer";
951	if (type_is_non_owning_ref(type))
952	return "a borrowed allocated object pointer";
953	if (type_is_ptr_alloc_obj(type))
954	return "an owned allocated object pointer";
955	if (type_flag(type) & PTR_UNTRUSTED)
956	return "an untrusted kernel object pointer";
957	return "a kernel object pointer";
958	default:
959	return reg_type_str(env, type);
960	}
961	}
962
963	static const char *diag_arg_ordinal(int argno)
964	{
965	switch (argno) {
966	case 1:
967	return "first";
968	case 2:
969	return "second";
970	case 3:
971	return "third";
972	case 4:
973	return "fourth";
974	case 5:
975	return "fifth";
976	case 6:
977	return "sixth";
978	case 7:
979	return "seventh";
980	case 8:
981	return "eighth";
982	case 9:
983	return "ninth";
984	case 10:
985	return "tenth";
986	case 11:
987	return "eleventh";
988	case 12:
989	return "twelfth";
990	default:
991	return NULL;
992	}
993	}
994
995	void bpf_diag_call_type(struct bpf_verifier_env *env, u32 insn_idx, int argno, int regno,
996	int stack_arg_slot, const char call_name, const char arg_name,
997	const char reason, const char suggestion)
998	{
999	const struct bpf_func_state *frame = diag_current_frame(env);
1000	struct bpf_diag_history_opts opts = {
1001	.frame_id = frame->diag_frame_id,
1002	.frameno = frame->frameno,
1003	};
1004	const char *ordinal = diag_arg_ordinal(argno);
1005	const char *arg_desc;
1006	bool print_history = true;
1007
1008	if (regno >= 0) {
1009	opts.scope = BPF_DIAG_HISTORY_SCOPE_REG;
1010	opts.regno = regno;
1011	} else if (stack_arg_slot >= 0) {
1012	opts.scope = BPF_DIAG_HISTORY_SCOPE_STACK_ARG;
1013	opts.stack_arg_slot = stack_arg_slot;
1014	} else {
1015	print_history = false;
1016	}
1017
1018	if (ordinal && arg_name)
1019	arg_desc = bpf_diag_fmt(env, "%s argument (%s)", ordinal, arg_name);
1020	else if (ordinal)
1021	arg_desc = bpf_diag_fmt(env, "%s argument", ordinal);
1022	else if (arg_name)
1023	arg_desc = bpf_diag_fmt(env, "argument %s", arg_name);
1024	else
1025	arg_desc = "argument";
1026
1027	bpf_diag_header(env, CALL_TYPE_SAFETY, "invalid call argument");
1028	diag_reason(env, "The %s to %s does not satisfy the verifier contract: %s.",
1029	arg_desc, call_name, reason);
1030
1031	diag_section(env, "At");
1032	bpf_diag_source(env, insn_idx, "error", "invalid %s for %s", arg_desc, call_name);
1033
1034	if (print_history)
1035	diag_print_history(env, &opts);
1036
1037	diag_suggestion(env, "%s", suggestion);
1038	}
1039
1040	static const char *diag_context_constraint(enum bpf_diag_context_kind kind)
1041	{
1042	switch (kind) {
1043	case BPF_DIAG_CONTEXT_RCU:
1044	return "RCU read-side critical sections cannot call operations that may sleep";
1045	case BPF_DIAG_CONTEXT_PREEMPT:
1046	return "preemption-disabled code cannot call operations that may sleep";
1047	case BPF_DIAG_CONTEXT_IRQ:
1048	return "IRQ-disabled code cannot call operations that may sleep";
1049	case BPF_DIAG_CONTEXT_LOCK:
1050	return "code holding a BPF spin lock cannot call operations that may sleep";
1051	case BPF_DIAG_CONTEXT_NONE:
1052	default:
1053	return NULL;
1054	}
1055	}
1056
1057	static const char diag_active_context(struct bpf_verifier_env env, u32 depth,
1058	const char *context)
1059	{
1060	if (depth == 1)
1061	return bpf_diag_fmt(env, "an active %s (depth 1)", context);
1062	return bpf_diag_fmt(env, "%u active %ss (depth %u)", depth, context, depth);
1063	}
1064
1065	static u32 diag_context_depth(struct bpf_verifier_env *env, enum bpf_diag_context_kind kind)
1066	{
1067	switch (kind) {
1068	case BPF_DIAG_CONTEXT_RCU:
1069	return env->cur_state->active_rcu_locks;
1070	case BPF_DIAG_CONTEXT_PREEMPT:
1071	return env->cur_state->active_preempt_locks;
1072	case BPF_DIAG_CONTEXT_IRQ:
1073	return bpf_diag_irq_depth(env->cur_state);
1074	case BPF_DIAG_CONTEXT_LOCK:
1075	return env->cur_state->active_locks;
1076	case BPF_DIAG_CONTEXT_NONE:
1077	default:
1078	return 0;
1079	}
1080	}
1081
1082	void bpf_diag_ctx_forbidden(struct bpf_verifier_env *env, u32 insn_idx,
1083	const char operation, const char suggestion)
1084	{
1085	struct bpf_diag_history_opts opts;
1086	enum bpf_diag_context_kind ctx_kind;
1087	const char constraint, context;
1088	u32 depth;
1089
1090	if (env->cur_state->active_rcu_locks)
1091	ctx_kind = BPF_DIAG_CONTEXT_RCU;
1092	else if (env->cur_state->active_preempt_locks)
1093	ctx_kind = BPF_DIAG_CONTEXT_PREEMPT;
1094	else if (env->cur_state->active_irq_id)
1095	ctx_kind = BPF_DIAG_CONTEXT_IRQ;
1096	else if (env->cur_state->active_locks)
1097	ctx_kind = BPF_DIAG_CONTEXT_LOCK;
1098	else
1099	ctx_kind = BPF_DIAG_CONTEXT_NONE;
1100
1101	depth = diag_context_depth(env, ctx_kind);
1102	opts = (struct bpf_diag_history_opts) {
1103	.scope = BPF_DIAG_HISTORY_SCOPE_CONTEXT,
1104	.ctx_kind = ctx_kind,
1105	.ctx_depth = depth,
1106	};
1107	constraint = diag_context_constraint(ctx_kind);
1108	context = diag_context_name(ctx_kind);
1109
1110	bpf_diag_header(env, EXECUTION_CONTEXT_SAFETY,
1111	"operation is not allowed in this context");
1112	if (constraint) {
1113	if (depth) {
1114	diag_reason(
1115	env, "The operation %s cannot be used in %s because %s. This path is still inside %s.",
1116	operation, context, constraint, diag_active_context(env, depth, context));
1117	} else {
1118	diag_reason(env, "The operation %s cannot be used in %s because %s.",
1119	operation, context, constraint);
1120	}
1121	} else {
1122	diag_reason(env, "The operation %s cannot be used in %s.", operation,
1123	context);
1124	}
1125
1126	diag_section(env, "At");
1127	bpf_diag_source(env, insn_idx, "error", "%s is not allowed in %s", operation,
1128	context);
1129
1130	if (ctx_kind != BPF_DIAG_CONTEXT_NONE)
1131	diag_print_history(env, &opts);
1132
1133	diag_suggestion(env, "%s", suggestion);
1134	}
1135
1136	void bpf_diag_ctx_active(struct bpf_verifier_env env, u32 insn_idx, const char operation,
1137	enum bpf_diag_context_kind ctx_kind, const char *suggestion)
1138	{
1139	u32 depth = diag_context_depth(env, ctx_kind);
1140	struct bpf_diag_history_opts opts = {
1141	.scope = BPF_DIAG_HISTORY_SCOPE_CONTEXT,
1142	.ctx_kind = ctx_kind,
1143	.ctx_depth = depth,
1144	};
1145	const char *context = diag_context_name(ctx_kind);
1146
1147	bpf_diag_header(env, EXECUTION_CONTEXT_SAFETY,
1148	"operation is not allowed in this context");
1149	diag_reason(
1150	env, "The operation %s cannot be used while this path is still inside %s. Leave the region before this operation.",
1151	operation, diag_active_context(env, depth, context));
1152
1153	diag_section(env, "At");
1154	bpf_diag_source(env, insn_idx, "error", "%s is not allowed before leaving %s",
1155	operation, context);
1156
1157	diag_print_history(env, &opts);
1158
1159	diag_suggestion(env, "%s", suggestion);
1160	}
1161
1162	void bpf_diag_ctx_required(struct bpf_verifier_env env, u32 insn_idx, const char operation,
1163	enum bpf_diag_context_kind ctx_kind, const char *suggestion)
1164	{
1165	const char *context = diag_context_name(ctx_kind);
1166
1167	bpf_diag_header(env, EXECUTION_CONTEXT_SAFETY, "required context is not active");
1168	diag_reason(env, "The operation %s requires an active %s, but this path is outside one.",
1169	operation, context);
1170
1171	diag_section(env, "At");
1172	bpf_diag_source(env, insn_idx, "error", "%s requires %s", operation, context);
1173
1174	diag_suggestion(env, "%s", suggestion);
1175	}
1176
1177	void bpf_diag_ctx_underflow(struct bpf_verifier_env *env, u32 insn_idx,
1178	const char *operation, enum bpf_diag_context_kind ctx_kind,
1179	const char *suggestion)
1180	{
1181	struct bpf_diag_history_opts opts = {
1182	.scope = BPF_DIAG_HISTORY_SCOPE_CONTEXT,
1183	.ctx_kind = ctx_kind,
1184	};
1185	const char *context = diag_context_name(ctx_kind);
1186
1187	bpf_diag_header(env, EXECUTION_CONTEXT_SAFETY, "unmatched context exit");
1188	diag_reason(
1189	env, "The operation %s tries to leave %s, but this path has no active %s to leave. The current depth is 0.",
1190	operation, context, context);
1191
1192	diag_section(env, "At");
1193	bpf_diag_source(env, insn_idx, "error", "%s has no matching enter on this path",
1194	operation);
1195
1196	diag_print_history(env, &opts);
1197
1198	diag_suggestion(env, "%s", suggestion);
1199	}
1200
1201	void bpf_diag_program_structure(struct bpf_verifier_env *env, u32 insn_idx,
1202	const char problem, const char suggestion,
1203	const char *reason_fmt, ...)
1204	{
1205	va_list args;
1206
1207	bpf_diag_header(env, PROGRAM_STRUCTURE, problem);
1208	diag_section(env, "Reason");
1209
1210	va_start(args, reason_fmt);
1211	diag_vprint_indented(env, reason_fmt, args);
1212	va_end(args);
1213
1214	diag_section(env, "At");
1215	bpf_diag_source(env, insn_idx, "error", "%s", problem);
1216
1217	diag_suggestion(env, "%s", suggestion);
1218	}
1219
1220	void bpf_diag_policy(struct bpf_verifier_env env, u32 insn_idx, const char operation,
1221	const char reason, const char suggestion)
1222	{
1223	bpf_diag_header(env, POLICY, "operation is not allowed");
1224	diag_reason(env, "The %s is not allowed: %s.", operation, reason);
1225
1226	diag_section(env, "At");
1227	bpf_diag_source(env, insn_idx, "error", "policy check failed for %s", operation);
1228
1229	diag_suggestion(env, "%s", suggestion);
1230	}
1231
1232	void bpf_diag_invalid_deref(struct bpf_verifier_env *env, u32 insn_idx, int regno,
1233	const char reg_name, const struct bpf_reg_state reg,
1234	enum bpf_diag_invalid_deref_kind kind, s64 offset)
1235	{
1236	const struct bpf_func_state *frame = diag_current_frame(env);
1237	struct bpf_diag_history_opts opts = {
1238	.scope = BPF_DIAG_HISTORY_SCOPE_REG,
1239	.frame_id = frame->diag_frame_id,
1240	.frameno = frame->frameno,
1241	.regno = regno,
1242	};
1243	const char *type_name = bpf_diag_reg_type_plain(env, reg->type);
1244
1245	bpf_diag_header(env, REGISTER_TYPE_SAFETY, "invalid dereference");
1246
1247	switch (kind) {
1248	case BPF_DIAG_DEREF_SCALAR:
1249	diag_reason(env, "%s is an integer scalar here, not a pointer to memory.",
1250	reg_name);
1251	break;
1252	case BPF_DIAG_DEREF_NULLABLE_PTR:
1253	diag_reason(
1254	env, "%s may be NULL here (%s). The program could dereference NULL on this path, so the verifier cannot prove this access is safe.",
1255	reg_name, type_name);
1256	break;
1257	case BPF_DIAG_DEREF_MODIFIED_PTR:
1258	diag_reason(
1259	env, "%s has offset %lld here, but this pointer type must be dereferenced in its original form.",
1260	reg_name, offset);
1261	break;
1262	case BPF_DIAG_DEREF_INVALID_PTR:
1263	default:
1264	diag_reason(
1265	env, "%s has type %s here, which is not valid for this memory access.",
1266	reg_name, type_name);
1267	break;
1268	}
1269
1270	diag_section(env, "At");
1271	if (kind == BPF_DIAG_DEREF_MODIFIED_PTR)
1272	bpf_diag_source(env, insn_idx, "error",
1273	"dereference requires the original %s pointer", type_name);
1274	else
1275	bpf_diag_source(env, insn_idx, "error", "invalid dereference of %s (%s)",
1276	reg_name, type_name);
1277
1278	if (regno >= 0)
1279	diag_print_history(env, &opts);
1280
1281	switch (kind) {
1282	case BPF_DIAG_DEREF_NULLABLE_PTR:
1283	diag_suggestion(
1284	env, "Add a NULL check before the access and dereference the pointer only on the non-NULL path.");
1285	break;
1286	case BPF_DIAG_DEREF_MODIFIED_PTR:
1287	diag_suggestion(
1288	env, "Preserve the original pointer in another register, or use only offsets this pointer type permits before dereferencing it.");
1289	break;
1290	case BPF_DIAG_DEREF_SCALAR:
1291	case BPF_DIAG_DEREF_INVALID_PTR:
1292	default:
1293	diag_suggestion(
1294	env, "Preserve a pointer-valued register where needed, or reload and revalidate the pointer after scalar arithmetic, helper calls, or other operations that can invalidate it.");
1295	break;
1296	}
1297	}
1298
1299	void bpf_diag_unreadable_reg(struct bpf_verifier_env *env, u32 insn_idx, int regno)
1300	{
1301	const struct bpf_func_state *frame = diag_current_frame(env);
1302	struct bpf_diag_history_opts opts = {
1303	.scope = BPF_DIAG_HISTORY_SCOPE_REG,
1304	.frame_id = frame->diag_frame_id,
1305	.frameno = frame->frameno,
1306	.regno = regno,
1307	};
1308	const struct bpf_diag_log *log = env->diag ? &env->diag->log : NULL;
1309	struct bpf_diag_mod_target target;
1310	bool invalidated = false;
1311	int i;
1312
1313	target = diag_reg_target(opts.frame_id, opts.frameno, regno);
1314	for (i = log ? log->cnt : 0; i > 0; i--) {
1315	const struct bpf_diag_history_event *event;
1316
1317	event = &log->events[log_pos(log, i - 1)];
1318
1319	if (event->kind != BPF_DIAG_HISTORY_MOD \|\|
1320	!diag_target_matches(&event->mod.target, &target))
1321	continue;
1322	invalidated = event->mod.new.type == NOT_INIT;
1323	break;
1324	}
1325
1326	bpf_diag_header(env, REGISTER_TYPE_SAFETY, "unreadable register");
1327	if (invalidated)
1328	diag_reason(
1329	env, "R%d is not readable here. A previous operation invalidated this register, so the verifier cannot use it as an input.",
1330	regno);
1331	else if (log && !log->first_seq)
1332	diag_reason(env,
1333	"R%d has never been initialized on this path, so the verifier cannot use it as an input.",
1334	regno);
1335	else
1336	diag_reason(
1337	env, "R%d is not readable here. It may never have been initialized, or an earlier operation may have invalidated it.",
1338	regno);
1339
1340	diag_section(env, "At");
1341	bpf_diag_source(env, insn_idx, "error", "R%d is not readable", regno);
1342
1343	if (regno >= 0)
1344	diag_print_history(env, &opts);
1345
1346	if (invalidated)
1347	diag_suggestion(
1348	env, "Avoid using the register after it is invalidated, or initialize it again before this instruction.");
1349	else if (log && !log->first_seq)
1350	diag_suggestion(env, "Initialize R%d on every path before this instruction.", regno);
1351	else
1352	diag_suggestion(
1353	env, "Initialize the register on every path, or initialize it again after any operation that invalidates it.");
1354	}
1355
1356	static int diag_stack_argno(u8 slot)
1357	{
1358	return MAX_BPF_FUNC_REG_ARGS + slot + 1;
1359	}
1360
1361	static void diag_format_stack_arg(char buf, size_t size, u8 slot, const char arg_name)
1362	{
1363	int argno = diag_stack_argno(slot);
1364	const char *ordinal = diag_arg_ordinal(argno);
1365
1366	if (ordinal && arg_name)
1367	scnprintf(buf, size, "outgoing stack argument %u (%s argument, %s)", slot + 1,
1368	ordinal, arg_name);
1369	else if (ordinal)
1370	scnprintf(buf, size, "outgoing stack argument %u (%s argument)", slot + 1, ordinal);
1371	else if (arg_name)
1372	scnprintf(buf, size, "outgoing stack argument %u (%s)", slot + 1, arg_name);
1373	else
1374	scnprintf(buf, size, "outgoing stack argument %u", slot + 1);
1375	}
1376
1377	void bpf_diag_stack_arg_uninit(struct bpf_verifier_env *env, u32 insn_idx, int nargs,
1378	int stack_arg_slot, const char *callee_name,
1379	const char *arg_name)
1380	{
1381	const struct bpf_func_state *frame = diag_current_frame(env);
1382	struct bpf_diag_history_opts opts = {
1383	.scope = BPF_DIAG_HISTORY_SCOPE_STACK_ARG,
1384	.frame_id = frame->diag_frame_id,
1385	.frameno = frame->frameno,
1386	.stack_arg_slot = stack_arg_slot,
1387	};
1388	const char *arg_buf;
1389
1390	arg_buf = bpf_diag_fmt_buf(env, BPF_DIAG_FMT_BUF_SIZE);
1391	if (arg_buf)
1392	diag_format_stack_arg((char *)arg_buf, BPF_DIAG_FMT_BUF_SIZE, stack_arg_slot,
1393	arg_name);
1394	else
1395	arg_buf = "";
1396	bpf_diag_header(env, REGISTER_TYPE_SAFETY, "missing stack argument");
1397	if (callee_name && *callee_name)
1398	diag_reason(
1399	env, "Function %s expects %d arguments, but %s is not initialized at this call.",
1400	callee_name, nargs, arg_buf);
1401	else
1402	diag_reason(
1403	env, "The callee expects %d arguments, but %s is not initialized at this call.",
1404	nargs, arg_buf);
1405
1406	diag_section(env, "At");
1407	bpf_diag_source(env, insn_idx, "error", "%s is not initialized", arg_buf);
1408
1409	if (stack_arg_slot >= 0)
1410	diag_print_history(env, &opts);
1411
1412	diag_suggestion(
1413	env, "Write the outgoing stack argument after any operation that may invalidate stored pointer values, and before making this call.");
1414	}
1415
1416	void bpf_diag_memory(struct bpf_verifier_env env, u32 insn_idx, const char problem,
1417	const char reason, const char suggestion)
1418	{
1419	bpf_diag_header(env, MEMORY_SAFETY, problem);
1420	diag_reason(env, "%s", reason);
1421
1422	diag_section(env, "At");
1423	bpf_diag_source(env, insn_idx, "error", "%s", problem);
1424
1425	diag_suggestion(env, "%s", suggestion);
1426	}
1427
1428	void bpf_diag_record_branch(struct bpf_verifier_env *env, u32 insn_idx, bool cond_true)
1429	{
1430	struct bpf_diag_history_event event = {
1431	.insn_idx = insn_idx,
1432	.kind = BPF_DIAG_HISTORY_BRANCH,
1433	.branch = {
1434	.cond_true = cond_true,
1435	},
1436	};
1437
1438	diag_append_history(env, &event);
1439	}
1440
1441	static void diag_snapshot_reg(struct bpf_diag_reg_snapshot *snapshot,
1442	const struct bpf_reg_state *reg)
1443	{
1444	snapshot->type = reg->type;
1445	if (type_is_map_ptr(reg->type))
1446	snapshot->map_ptr = reg->map_ptr;
1447	if (base_type(reg->type) == PTR_TO_BTF_ID && reg->btf && reg->btf_id) {
1448	snapshot->btf_id = reg->btf_id;
1449	snapshot->btf = reg->btf;
1450	}
1451	snapshot->var_off = reg->var_off;
1452	snapshot->r64 = reg->r64;
1453	}
1454
1455	static bool diag_mod_insn_origin(struct bpf_verifier_env *env, u32 insn_idx,
1456	const struct bpf_diag_mod_target *target,
1457	struct bpf_diag_mod_target *origin)
1458	{
1459	const struct bpf_insn *insn = &env->prog->insnsi[insn_idx];
1460	u8 class = BPF_CLASS(insn->code);
1461	const struct bpf_func_state *state;
1462
1463	if (target->kind == BPF_DIAG_MOD_TARGET_REG && (class == BPF_ALU \|\| class == BPF_ALU64) &&
1464	BPF_OP(insn->code) == BPF_MOV && BPF_SRC(insn->code) == BPF_X) {
1465	*origin = diag_reg_target(target->frame_id, target->frameno, insn->src_reg);
1466	return true;
1467	}
1468
1469	if ((target->kind != BPF_DIAG_MOD_TARGET_STACK_ARG &&
1470	target->kind != BPF_DIAG_MOD_TARGET_STACK_SLOT) \|\|
1471	class != BPF_STX)
1472	return false;
1473
1474	state = env->cur_state->frame[env->cur_state->curframe];
1475	*origin = diag_reg_target(state->diag_frame_id, state->frameno, insn->src_reg);
1476	return true;
1477	}
1478
1479	static bool diag_mod_keeps_lineage(struct bpf_verifier_env *env,
1480	const struct bpf_diag_history_event *event)
1481	{
1482	const struct bpf_insn *insn;
1483	u8 class;
1484
1485	if (event->mod.reason != BPF_DIAG_MOD_WRITE \|\|
1486	event->mod.target.kind != BPF_DIAG_MOD_TARGET_REG)
1487	return false;
1488
1489	insn = &env->prog->insnsi[event->insn_idx];
1490	class = BPF_CLASS(insn->code);
1491	if (class != BPF_ALU && class != BPF_ALU64)
1492	return false;
1493
1494	switch (BPF_OP(insn->code)) {
1495	case BPF_ADD:
1496	case BPF_SUB:
1497	case BPF_MUL:
1498	case BPF_OR:
1499	case BPF_AND:
1500	case BPF_LSH:
1501	case BPF_RSH:
1502	case BPF_ARSH:
1503	case BPF_XOR:
1504	case BPF_NEG:
1505	case BPF_END:
1506	return true;
1507	default:
1508	return false;
1509	}
1510	}
1511
1512	static void diag_record_mod(struct bpf_verifier_env *env, u32 insn_idx,
1513	struct bpf_diag_mod_target target,
1514	enum bpf_diag_mod_reason reason,
1515	const struct bpf_reg_state *old_reg,
1516	const struct bpf_reg_state *new_reg,
1517	const struct bpf_diag_mod_target *origin)
1518	{
1519	struct bpf_diag_history_event event = {
1520	.insn_idx = insn_idx,
1521	.kind = BPF_DIAG_HISTORY_MOD,
1522	.mod = {
1523	.target = target,
1524	.reason = reason,
1525	},
1526	};
1527
1528	if (old_reg)
1529	diag_snapshot_reg(&event.mod.old, old_reg);
1530	if (new_reg)
1531	diag_snapshot_reg(&event.mod.new, new_reg);
1532	if (origin) {
1533	event.mod.origin = *origin;
1534	event.mod.origin_valid = true;
1535	} else if (diag_mod_insn_origin(env, insn_idx, &target, &event.mod.origin)) {
1536	event.mod.origin_valid = true;
1537	}
1538	if (old_reg && new_reg &&
1539	(reason == BPF_DIAG_MOD_WRITE \|\| reason == BPF_DIAG_MOD_SPILL) &&
1540	!memcmp(&event.mod.old, &event.mod.new, sizeof(event.mod.old)) &&
1541	!event.mod.origin_valid &&
1542	diag_mod_keeps_lineage(env, &event))
1543	return;
1544
1545	diag_append_history(env, &event);
1546	}
1547
1548	static struct bpf_reg_state target_to_reg(struct bpf_verifier_env env,
1549	const struct bpf_diag_mod_target *target)
1550	{
1551	struct bpf_verifier_state *vstate = env->cur_state;
1552	struct bpf_func_state *state;
1553
1554	state = target->frameno <= vstate->curframe ? vstate->frame[target->frameno] : NULL;
1555
1556	if (!state)
1557	return NULL;
1558	if (state->diag_frame_id != target->frame_id)
1559	return NULL;
1560
1561	switch (target->kind) {
1562	case BPF_DIAG_MOD_TARGET_REG:
1563	if (target->regno >= MAX_BPF_REG)
1564	return NULL;
1565	return &state->regs[target->regno];
1566	case BPF_DIAG_MOD_TARGET_STACK_ARG:
1567	if (target->stack_arg >= state->out_stack_arg_cnt)
1568	return NULL;
1569	return &state->stack_arg_regs[target->stack_arg];
1570	case BPF_DIAG_MOD_TARGET_STACK_SLOT:
1571	if (target->spi >= state->allocated_stack / BPF_REG_SIZE)
1572	return NULL;
1573	return &state->stack[target->spi].spilled_ptr;
1574	default:
1575	return NULL;
1576	}
1577	}
1578
1579	static bool reg_to_target(struct bpf_verifier_env env, const struct bpf_reg_state reg,
1580	struct bpf_diag_mod_target *target)
1581	{
1582	struct bpf_verifier_state *vstate = env->cur_state;
1583	unsigned long addr = (unsigned long)reg;
1584	int frame;
1585
1586	for (frame = 0; frame <= vstate->curframe; frame++) {
1587	struct bpf_func_state *state = vstate->frame[frame];
1588	unsigned long start, end;
1589	u32 nslots = state->allocated_stack / BPF_REG_SIZE;
1590	int spi;
1591
1592	start = (unsigned long)state->regs;
1593	end = (unsigned long)(state->regs + MAX_BPF_REG);
1594	if (addr >= start && addr < end) {
1595	*target = diag_reg_target(state->diag_frame_id, state->frameno,
1596	reg - state->regs);
1597	return true;
1598	}
1599
1600	start = (unsigned long)state->stack_arg_regs;
1601	end = (unsigned long)(state->stack_arg_regs + state->out_stack_arg_cnt);
1602	if (state->out_stack_arg_cnt && addr >= start && addr < end) {
1603	*target = diag_stack_arg_target(state->diag_frame_id, state->frameno,
1604	reg - state->stack_arg_regs);
1605	return true;
1606	}
1607
1608	start = (unsigned long)state->stack;
1609	end = (unsigned long)(state->stack + nslots);
1610	if (nslots && addr >= start && addr < end) {
1611	spi = ((const char )reg - (const char )state->stack) /
1612	sizeof(*state->stack);
1613	*target = diag_stack_slot_target(state->diag_frame_id, state->frameno, spi);
1614	return true;
1615	}
1616	}
1617	return false;
1618	}
1619
1620	void bpf_diag_mod_begin(struct bpf_verifier_env env, const struct bpf_reg_state reg,
1621	const struct bpf_reg_state *origin, enum bpf_diag_mod_reason reason)
1622	{
1623	struct bpf_diag *diag = env->diag;
1624
1625	if (!diag)
1626	return;
1627	diag->mod.active = reg_to_target(env, reg, &diag->mod.target);
1628	if (!diag->mod.active)
1629	return;
1630	diag->mod.target_reg_snapshot = *reg;
1631	diag->mod.insn_idx = env->insn_idx;
1632	diag->mod.reason = reason;
1633	diag->mod.origin_valid = origin && reg_to_target(env, origin, &diag->mod.origin);
1634	}
1635
1636	void bpf_diag_mod_end(struct bpf_verifier_env *env)
1637	{
1638	struct bpf_diag *diag = env->diag;
1639	const struct bpf_reg_state *new_reg;
1640
1641	if (!diag \|\| !diag->mod.active)
1642	return;
1643	diag->mod.active = false;
1644	/*
1645	* Resolve the target again because the enclosing function state's stack
1646	* may have been reallocated while the modification was in progress.
1647	*/
1648	new_reg = target_to_reg(env, &diag->mod.target);
1649	if (!new_reg)
1650	return;
1651	diag_record_mod(env, diag->mod.insn_idx, diag->mod.target, diag->mod.reason,
1652	&diag->mod.target_reg_snapshot, new_reg,
1653	diag->mod.origin_valid ? &diag->mod.origin : NULL);
1654	}
1655
1656	void bpf_diag_record_scrub(struct bpf_verifier_env env, const struct bpf_reg_state reg,
1657	enum bpf_diag_mod_reason reason)
1658	{
1659	struct bpf_diag_mod_target target;
1660
1661	if (!env->diag \|\| reg->type == NOT_INIT \|\| !reg_to_target(env, reg, &target))
1662	return;
1663	diag_record_mod(env, env->insn_idx, target, reason, reg, NULL, NULL);
1664	}
1665
1666	void bpf_diag_record_scrub_stack(struct bpf_verifier_env *env,
1667	const struct bpf_func_state *state, s16 min_off, s16 max_off,
1668	enum bpf_diag_mod_reason reason)
1669	{
1670	diag_record_mod(env, env->insn_idx,
1671	diag_stack_range_target(state->diag_frame_id, state->frameno, min_off, max_off),
1672	reason, NULL, NULL, NULL);
1673	}
1674
1675	static void diag_record_ref(struct bpf_verifier_env *env, u32 insn_idx, u8 kind, u32 ref_id)
1676	{
1677	struct bpf_diag_history_event event = {
1678	.insn_idx = insn_idx,
1679	.kind = kind,
1680	.ref = {
1681	.ref_id = ref_id,
1682	},
1683	};
1684
1685	diag_append_history(env, &event);
1686	}
1687
1688	void bpf_diag_record_ref_acquire(struct bpf_verifier_env *env, u32 insn_idx, u32 ref_id)
1689	{
1690	diag_record_ref(env, insn_idx, BPF_DIAG_HISTORY_REF_ACQUIRE, ref_id);
1691	}
1692
1693	void bpf_diag_record_ref_release(struct bpf_verifier_env *env, u32 insn_idx, u32 ref_id)
1694	{
1695	diag_record_ref(env, insn_idx, BPF_DIAG_HISTORY_REF_RELEASE, ref_id);
1696	}
1697
1698	void bpf_diag_record_context(struct bpf_verifier_env *env, u32 insn_idx,
1699	enum bpf_diag_context_kind ctx_kind, bool enter, u32 depth)
1700	{
1701	/*
1702	* Keep leave events so context rendering can stop at a depth-zero exit
1703	* and show nested-region depth accurately for the active path.
1704	*/
1705	struct bpf_diag_history_event event = {
1706	.insn_idx = insn_idx,
1707	.kind = BPF_DIAG_HISTORY_CONTEXT,
1708	.ctx = {
1709	.kind = ctx_kind,
1710	.enter = enter,
1711	.depth = depth,
1712	},
1713	};
1714
1715	diag_append_history(env, &event);
1716	}
1717
1718	static int diag_history_context_start_idx(const struct bpf_diag_log *log,
1719	const struct bpf_diag_history_opts *opts)
1720	{
1721	int i;
1722
1723	if (!opts->ctx_depth)
1724	return 0;
1725
1726	/* Find the most recent outermost entry, or a depth-zero exit. */
1727	for (i = log->cnt; i > 0; i--) {
1728	const struct bpf_diag_history_event *event;
1729
1730	event = &log->events[log_pos(log, i - 1)];
1731
1732	if (event->kind != BPF_DIAG_HISTORY_CONTEXT \|\| event->ctx.kind != opts->ctx_kind)
1733	continue;
1734
1735	if (event->ctx.enter && event->ctx.depth == 1)
1736	return i - 1;
1737	if (!event->ctx.enter && event->ctx.depth == 0)
1738	return 0;
1739	}
1740
1741	return 0;
1742	}
1743
1744	struct bpf_diag_history_filter {
1745	const struct bpf_diag_history_opts *opts;
1746	u32 lineage_start;
1747	bool lineage_valid;
1748	};
1749
1750	static bool diag_target_matches(const struct bpf_diag_mod_target *event_target,
1751	const struct bpf_diag_mod_target *target)
1752	{
1753	int slot_off;
1754
1755	if (event_target->frame_id != target->frame_id \|\| event_target->frameno != target->frameno)
1756	return false;
1757
1758	if (event_target->kind == BPF_DIAG_MOD_TARGET_STACK_RANGE &&
1759	target->kind == BPF_DIAG_MOD_TARGET_STACK_SLOT) {
1760	slot_off = -(target->spi + 1) * BPF_REG_SIZE;
1761	return event_target->range.min_off < slot_off + BPF_REG_SIZE &&
1762	event_target->range.max_off > slot_off;
1763	}
1764
1765	if (event_target->kind != target->kind)
1766	return false;
1767
1768	switch (target->kind) {
1769	case BPF_DIAG_MOD_TARGET_REG:
1770	return event_target->regno == target->regno;
1771	case BPF_DIAG_MOD_TARGET_STACK_ARG:
1772	return event_target->stack_arg == target->stack_arg;
1773	case BPF_DIAG_MOD_TARGET_STACK_SLOT:
1774	return event_target->spi == target->spi;
1775	default:
1776	return false;
1777	}
1778	}
1779
1780	static void diag_build_lineage(struct bpf_verifier_env env, struct bpf_diag_log log,
1781	struct bpf_diag_history_filter *filter)
1782	{
1783	const struct bpf_diag_history_opts *opts = filter->opts;
1784	struct bpf_diag_mod_target target;
1785	int i;
1786
1787	for (i = 0; i < log->cnt; i++)
1788	log->events[log_pos(log, i)].in_lineage = false;
1789
1790	if (opts->scope == BPF_DIAG_HISTORY_SCOPE_REG)
1791	target = diag_reg_target(opts->frame_id, opts->frameno, opts->regno);
1792	else if (opts->scope == BPF_DIAG_HISTORY_SCOPE_STACK_ARG)
1793	target = diag_stack_arg_target(opts->frame_id, opts->frameno,
1794	opts->stack_arg_slot);
1795	else
1796	return;
1797
1798	/*
1799	* Find the nearest mutation of the active target. A fill or spill changes
1800	* the target to its origin, so the same walk follows register/stack
1801	* lineage recursively until it reaches the write that created the value.
1802	*/
1803	for (i = log->cnt; i > 0; i--) {
1804	struct bpf_diag_history_event *event;
1805
1806	event = &log->events[log_pos(log, i - 1)];
1807	if (event->kind != BPF_DIAG_HISTORY_MOD \|\|
1808	!diag_target_matches(&event->mod.target, &target))
1809	continue;
1810
1811	event->in_lineage = true;
1812	filter->lineage_start = i - 1;
1813	filter->lineage_valid = true;
1814
1815	if (event->mod.origin_valid) {
1816	target = event->mod.origin;
1817	continue;
1818	}
1819	if (event->mod.reason != BPF_DIAG_MOD_WRITE &&
1820	event->mod.reason != BPF_DIAG_MOD_SPILL)
1821	continue;
1822	if (diag_mod_keeps_lineage(env, event))
1823	continue;
1824	break;
1825	}
1826	}
1827
1828	static int diag_history_start_idx(const struct bpf_diag_log *log,
1829	const struct bpf_diag_history_filter *filter)
1830	{
1831	const struct bpf_diag_history_opts *opts = filter->opts;
1832	int i;
1833
1834	if (opts->scope == BPF_DIAG_HISTORY_SCOPE_CONTEXT)
1835	return diag_history_context_start_idx(log, opts);
1836	if (filter->lineage_valid)
1837	return filter->lineage_start;
1838	if (opts->scope != BPF_DIAG_HISTORY_SCOPE_REF)
1839	return 0;
1840
1841	for (i = log->cnt; i > 0; i--) {
1842	const struct bpf_diag_history_event *event;
1843
1844	event = &log->events[log_pos(log, i - 1)];
1845	if (event->kind == BPF_DIAG_HISTORY_REF_ACQUIRE &&
1846	event->ref.ref_id == opts->ref_id)
1847	return i - 1;
1848	}
1849
1850	return 0;
1851	}
1852
1853	static bool diag_history_event_visible(const struct bpf_diag_history_event *event,
1854	const struct bpf_diag_history_filter *filter)
1855	{
1856	const struct bpf_diag_history_opts *opts = filter->opts;
1857
1858	switch (event->kind) {
1859	case BPF_DIAG_HISTORY_BRANCH:
1860	return true;
1861	case BPF_DIAG_HISTORY_MOD:
1862	return filter->lineage_valid && event->in_lineage;
1863	case BPF_DIAG_HISTORY_REF_ACQUIRE:
1864	case BPF_DIAG_HISTORY_REF_RELEASE:
1865	return opts->scope == BPF_DIAG_HISTORY_SCOPE_REF &&
1866	event->ref.ref_id == opts->ref_id;
1867	case BPF_DIAG_HISTORY_CONTEXT:
1868	return opts->scope == BPF_DIAG_HISTORY_SCOPE_CONTEXT &&
1869	event->ctx.kind == opts->ctx_kind;
1870	default:
1871	return false;
1872	}
1873	}
1874
1875	static const char *diag_s64_bound_name(s64 value)
1876	{
1877	if (value == S64_MIN)
1878	return "S64_MIN";
1879	if (value == S64_MAX)
1880	return "S64_MAX";
1881	return NULL;
1882	}
1883
1884	static const char *diag_u64_bound_name(u64 value)
1885	{
1886	if (value == U64_MAX)
1887	return "U64_MAX";
1888	return NULL;
1889	}
1890
1891	static const char diag_s64_str(struct bpf_verifier_env env, s64 value)
1892	{
1893	return diag_s64_bound_name(value) ?: bpf_diag_fmt(env, "%lld", value);
1894	}
1895
1896	static const char diag_u64_str(struct bpf_verifier_env env, u64 value)
1897	{
1898	return diag_u64_bound_name(value) ?: bpf_diag_fmt(env, "%llu", value);
1899	}
1900
1901	static bool diag_cnum64_unknown(struct cnum64 range)
1902	{
1903	return cnum64_smin(range) == S64_MIN && cnum64_smax(range) == S64_MAX &&
1904	cnum64_umin(range) == 0 && cnum64_umax(range) == U64_MAX;
1905	}
1906
1907	static bool diag_snapshot_unknown(const struct bpf_diag_reg_snapshot *snapshot)
1908	{
1909	return tnum_is_unknown(snapshot->var_off) && diag_cnum64_unknown(snapshot->r64);
1910	}
1911
1912	static const char diag_scalar_range(struct bpf_verifier_env env, struct cnum64 range)
1913	{
1914	return bpf_diag_fmt(env, "signed range [%s, %s], unsigned range [%s, %s]",
1915	diag_s64_str(env, cnum64_smin(range)),
1916	diag_s64_str(env, cnum64_smax(range)),
1917	diag_u64_str(env, cnum64_umin(range)),
1918	diag_u64_str(env, cnum64_umax(range)));
1919	}
1920
1921	const char bpf_diag_fmt_s64_sum(struct bpf_verifier_env env, s64 value, int addend)
1922	{
1923	s64 sum;
1924
1925	if (check_add_overflow(value, (s64)addend, &sum))
1926	return bpf_diag_fmt(env, "%lld plus %d (%s)", value, addend,
1927	addend < 0 ? "below S64_MIN" : "above S64_MAX");
1928
1929	return bpf_diag_fmt(env, "%lld", sum);
1930	}
1931
1932	static const char diag_access_offset(struct bpf_verifier_env env, int off,
1933	const struct bpf_reg_state *reg)
1934	{
1935	if (tnum_is_const(reg->var_off))
1936	return bpf_diag_fmt(env, "constant %s",
1937	bpf_diag_fmt_s64_sum(env, (s64)reg->var_off.value, off));
1938
1939	if (tnum_is_unknown(reg->var_off) && diag_cnum64_unknown(reg->r64))
1940	return bpf_diag_fmt(env, "unbounded");
1941
1942	if (off)
1943	return bpf_diag_fmt(env,
1944	"variable: known bits %#llx, unknown mask %#llx, plus fixed offset %d; %s",
1945	(u64)reg->var_off.value, reg->var_off.mask, off,
1946	diag_scalar_range(env, reg->r64));
1947	return bpf_diag_fmt(env, "variable: known bits %#llx, unknown mask %#llx; %s",
1948	(u64)reg->var_off.value, reg->var_off.mask,
1949	diag_scalar_range(env, reg->r64));
1950	}
1951
1952	void bpf_diag_mem_bounds(struct bpf_verifier_env *env, u32 insn_idx, int regno,
1953	const char reg_name, const char type_name, const char *proof,
1954	int off, int size, u32 mem_size, const struct bpf_reg_state *reg)
1955	{
1956	const struct bpf_func_state *frame = diag_current_frame(env);
1957	struct bpf_diag_history_opts opts = {
1958	.scope = BPF_DIAG_HISTORY_SCOPE_REG,
1959	.frame_id = frame->diag_frame_id,
1960	.frameno = frame->frameno,
1961	.regno = regno,
1962	};
1963	const char *offset_desc;
1964
1965	if (!bpf_diag_enabled(env))
1966	return;
1967
1968	offset_desc = diag_access_offset(env, off, reg);
1969
1970	bpf_diag_header(env, MEMORY_SAFETY, "access outside bounds");
1971	diag_reason(
1972	env, "The verifier cannot prove offset + access_size <= object_size. Here, %s. %s is %s; offset is %s; access_size is %d; object_size is %u.",
1973	proof, reg_name, type_name, offset_desc, size, mem_size);
1974
1975	diag_section(env, "At");
1976	bpf_diag_source(env, insn_idx, "error", "access may be outside object bounds");
1977
1978	if (regno >= 0)
1979	diag_print_history(env, &opts);
1980
1981	diag_suggestion(
1982	env, "Add or adjust a bounds check that proves offset + access_size stays within the object.");
1983	}
1984
1985	static const char diag_lock_name(const struct bpf_reference_state lock)
1986	{
1987	switch (lock->type) {
1988	case REF_TYPE_LOCK:
1989	return "bpf_spin_lock";
1990	case REF_TYPE_RES_LOCK:
1991	return "resource spin lock";
1992	case REF_TYPE_RES_LOCK_IRQ:
1993	return "IRQ-saving resource spin lock";
1994	default:
1995	return "lock";
1996	}
1997	}
1998
1999	static void diag_res_report(struct bpf_verifier_env env, u32 insn_idx, const char problem,
2000	const char *reason)
2001	{
2002	bpf_diag_header(env, RESOURCE_LIFETIME_SAFETY, problem);
2003	diag_reason(env, "%s", reason);
2004
2005	diag_section(env, "At");
2006	bpf_diag_source(env, insn_idx, "error", "%s", problem);
2007	}
2008
2009	void bpf_diag_res(struct bpf_verifier_env env, u32 insn_idx, const char problem,
2010	const char reason, const char suggestion)
2011	{
2012	diag_res_report(env, insn_idx, problem, reason);
2013	diag_suggestion(env, "%s", suggestion);
2014	}
2015
2016	void bpf_diag_lock(struct bpf_verifier_env env, u32 insn_idx, const char problem,
2017	const char reason, const char suggestion,
2018	const struct bpf_reference_state *active_lock)
2019	{
2020	diag_res_report(env, insn_idx, problem, reason);
2021
2022	if (active_lock) {
2023	diag_section(env, "Active lock");
2024	bpf_diag_source(env, active_lock->insn_idx, "acquired",
2025	"active %s has verifier identity %d",
2026	diag_lock_name(active_lock), active_lock->id);
2027	}
2028
2029	diag_suggestion(env, "%s", suggestion);
2030	}
2031
2032	void bpf_diag_irq(struct bpf_verifier_env env, u32 insn_idx, const char problem,
2033	const char reason, const char suggestion, u32 depth)
2034	{
2035	struct bpf_diag_history_opts opts = {
2036	.scope = BPF_DIAG_HISTORY_SCOPE_CONTEXT,
2037	.ctx_kind = BPF_DIAG_CONTEXT_IRQ,
2038	.ctx_depth = depth,
2039	};
2040
2041	bpf_diag_header(env, RESOURCE_LIFETIME_SAFETY, problem);
2042	diag_reason(env, "%s", reason);
2043
2044	diag_section(env, "At");
2045	bpf_diag_source(env, insn_idx, "error", "%s", problem);
2046
2047	if (depth)
2048	diag_print_history(env, &opts);
2049
2050	diag_suggestion(env, "%s", suggestion);
2051	}
2052
2053	void bpf_diag_leak(struct bpf_verifier_env *env, u32 ref_id, u32 alloc_insn, u32 fail_insn)
2054	{
2055	struct bpf_diag_history_opts opts = {
2056	.scope = BPF_DIAG_HISTORY_SCOPE_REF,
2057	.ref_id = ref_id,
2058	};
2059
2060	bpf_diag_header(env, RESOURCE_LIFETIME_SAFETY, "unreleased resource");
2061	diag_reason(
2062	env, "Owned resource (id=%u) was acquired at instruction %u and still needs to be released before this exit path.",
2063	ref_id, alloc_insn);
2064
2065	diag_section(env, "At");
2066	bpf_diag_source(env, fail_insn, "error",
2067	"owned resource (id=%u) still needs release", ref_id);
2068
2069	diag_print_history(env, &opts);
2070
2071	diag_suggestion(
2072	env, "Release or transfer ownership of the acquired resource on every path before the program exits.");
2073	}
2074
2075	static const char diag_var_offset(struct bpf_verifier_env env,
2076	const struct bpf_diag_reg_snapshot *snapshot)
2077	{
2078	if (tnum_is_const(snapshot->var_off))
2079	return bpf_diag_fmt(env, "at offset %lld", (s64)snapshot->var_off.value);
2080
2081	if (diag_snapshot_unknown(snapshot))
2082	return bpf_diag_fmt(env, "with unknown offset");
2083
2084	return bpf_diag_fmt(env,
2085	"with variable offset: known bits %#llx, unknown mask %#llx, %s",
2086	snapshot->var_off.value, snapshot->var_off.mask,
2087	diag_scalar_range(env, snapshot->r64));
2088	}
2089
2090	static const char diag_reg_map_name(const struct bpf_map map)
2091	{
2092	if (!map \|\| !map->name[0])
2093	return NULL;
2094
2095	return map->name;
2096	}
2097
2098	static const char diag_reg_snapshot(struct bpf_verifier_env env,
2099	const struct bpf_diag_reg_snapshot *snapshot)
2100	{
2101	const char *type_name = reg_type_str(env, snapshot->type);
2102	const char *offset = diag_var_offset(env, snapshot);
2103	const char *btf = snapshot->btf && snapshot->btf_id ?
2104	bpf_diag_fmt_btf_type(env, snapshot->btf, snapshot->btf_id) : NULL;
2105	const char *map_name;
2106
2107	if (snapshot->type == SCALAR_VALUE) {
2108	if (tnum_is_const(snapshot->var_off))
2109	return bpf_diag_fmt(env, "integer scalar value %lld",
2110	(s64)snapshot->var_off.value);
2111	if (diag_snapshot_unknown(snapshot))
2112	return bpf_diag_fmt(env, "integer scalar with unknown value");
2113	if (cnum64_is_const(snapshot->r64))
2114	return bpf_diag_fmt(env, "integer scalar value %lld",
2115	cnum64_smin(snapshot->r64));
2116	return bpf_diag_fmt(env, "integer scalar with %s",
2117	diag_scalar_range(env, snapshot->r64));
2118	}
2119
2120	if (snapshot->type == NOT_INIT)
2121	return bpf_diag_fmt(env, "uninitialized value");
2122
2123	if (base_type(snapshot->type) == PTR_TO_CTX)
2124	return bpf_diag_fmt(env, "context pointer %s", offset);
2125
2126	if (base_type(snapshot->type) == PTR_TO_STACK)
2127	return bpf_diag_fmt(env, "stack pointer %s", offset);
2128
2129	if (base_type(snapshot->type) == PTR_TO_MAP_VALUE) {
2130	const char *kind = type_may_be_null(snapshot->type) ? "nullable map value" :
2131	"map value";
2132
2133	map_name = diag_reg_map_name(snapshot->map_ptr);
2134	if (map_name)
2135	return bpf_diag_fmt(env, "%s from %s %s", kind, map_name, offset);
2136	return bpf_diag_fmt(env, "%s %s", kind, offset);
2137	}
2138
2139	if (base_type(snapshot->type) == CONST_PTR_TO_MAP) {
2140	map_name = diag_reg_map_name(snapshot->map_ptr);
2141	if (map_name)
2142	return bpf_diag_fmt(env, "map pointer for map %s", map_name);
2143	return bpf_diag_fmt(env, "map pointer");
2144	}
2145
2146	if (type_is_non_owning_ref(snapshot->type)) {
2147	if (btf)
2148	return bpf_diag_fmt(env, "borrowed allocated object pointer type=%s", btf);
2149	return bpf_diag_fmt(env, "borrowed allocated object pointer");
2150	}
2151
2152	if (type_is_ptr_alloc_obj(snapshot->type)) {
2153	if (btf)
2154	return bpf_diag_fmt(env, "owned allocated object pointer type=%s", btf);
2155	return bpf_diag_fmt(env, "owned allocated object pointer");
2156	}
2157
2158	if (base_type(snapshot->type) == PTR_TO_BTF_ID && btf)
2159	return bpf_diag_fmt(env, "%s type=%s %s", type_name, btf, offset);
2160
2161	return bpf_diag_fmt(env, "%s %s", type_name, offset);
2162	}
2163
2164	static const char diag_mod_target_desc(struct bpf_verifier_env env,
2165	const struct bpf_diag_mod_target *target)
2166	{
2167	switch (target->kind) {
2168	case BPF_DIAG_MOD_TARGET_REG:
2169	return bpf_diag_fmt(env, "R%u", target->regno);
2170	case BPF_DIAG_MOD_TARGET_STACK_ARG:
2171	return bpf_diag_fmt(env, "(R11-%u)", (target->stack_arg + 1) BPF_REG_SIZE);
2172	case BPF_DIAG_MOD_TARGET_STACK_SLOT:
2173	return bpf_diag_fmt(env, "stack slot fp%d", -(target->spi + 1) * BPF_REG_SIZE);
2174	default:
2175	return "value";
2176	}
2177	}
2178
2179	static void diag_print_mod(struct bpf_verifier_env env, const struct bpf_diag_history_event event)
2180	{
2181	const struct bpf_diag_mod_target *target = &event->mod.target;
2182	const char target_desc, reason = NULL, old, new;
2183	const char *label = "update";
2184
2185	if (target->kind == BPF_DIAG_MOD_TARGET_STACK_RANGE) {
2186	bpf_diag_source(
2187	env, event->insn_idx, "invalidated",
2188	"variable-offset stack write may affect bytes fp%d through fp%d",
2189	target->range.min_off, target->range.max_off - 1);
2190	return;
2191	}
2192
2193	old = diag_reg_snapshot(env, &event->mod.old);
2194	new = diag_reg_snapshot(env, &event->mod.new);
2195	target_desc = diag_mod_target_desc(env, target);
2196
2197	switch (event->mod.reason) {
2198	case BPF_DIAG_MOD_REF_RELEASE:
2199	reason = target->kind == BPF_DIAG_MOD_TARGET_REG ? "resource release invalidated "
2200	"this pointer" :
2201	"resource release invalidated "
2202	"this value";
2203	break;
2204	case BPF_DIAG_MOD_PKT_DATA_CHANGE:
2205	reason = "packet data may have moved";
2206	break;
2207	case BPF_DIAG_MOD_NON_OWN_REF:
2208	reason = "leaving the protected region invalidated this borrowed pointer";
2209	break;
2210	case BPF_DIAG_MOD_CALLER_SAVED:
2211	reason = target->kind == BPF_DIAG_MOD_TARGET_STACK_ARG ?
2212	"call invalidated this outgoing stack argument" :
2213	"call invalidated this caller-saved register";
2214	break;
2215	case BPF_DIAG_MOD_WRITE:
2216	if (target->kind == BPF_DIAG_MOD_TARGET_STACK_SLOT)
2217	reason = "a later stack write overwrote this spilled value";
2218	break;
2219	case BPF_DIAG_MOD_SPILL:
2220	label = "spilled";
2221	break;
2222	case BPF_DIAG_MOD_VAR_WRITE:
2223	default:
2224	break;
2225	}
2226
2227	if (reason) {
2228	bpf_diag_source(env, event->insn_idx, "invalidated",
2229	"%s: %s; previous value was %s", target_desc, reason, old);
2230	return;
2231	}
2232
2233	bpf_diag_source(env, event->insn_idx, label, "%s changed from %s to %s", target_desc,
2234	old, new);
2235	}
2236
2237	static void diag_print_ref_event(struct bpf_verifier_env *env,
2238	const struct bpf_diag_history_event *event)
2239	{
2240	const char *label;
2241
2242	label = event->kind == BPF_DIAG_HISTORY_REF_ACQUIRE ? "acquired" : "released";
2243	bpf_diag_source(env, event->insn_idx, label, "owned resource (id=%u)",
2244	event->ref.ref_id);
2245	}
2246
2247	static const char *diag_context_name(enum bpf_diag_context_kind kind)
2248	{
2249	switch (kind) {
2250	case BPF_DIAG_CONTEXT_RCU:
2251	return "RCU read lock region";
2252	case BPF_DIAG_CONTEXT_PREEMPT:
2253	return "non-preemptible region";
2254	case BPF_DIAG_CONTEXT_IRQ:
2255	return "IRQ-disabled region";
2256	case BPF_DIAG_CONTEXT_LOCK:
2257	return "lock region";
2258	case BPF_DIAG_CONTEXT_NONE:
2259	default:
2260	return "non-sleepable program";
2261	}
2262	}
2263
2264	static void diag_print_context_event(struct bpf_verifier_env *env,
2265	const struct bpf_diag_history_event *event)
2266	{
2267	bpf_diag_source(env, event->insn_idx, "context", "%s %s; depth is now %u",
2268	event->ctx.enter ? "entered" : "left",
2269	diag_context_name(event->ctx.kind), event->ctx.depth);
2270	}
2271
2272	static void diag_print_history(struct bpf_verifier_env *env,
2273	const struct bpf_diag_history_opts *opts)
2274	{
2275	const struct bpf_diag_history_event *event;
2276	struct bpf_diag_history_filter filter = {
2277	.opts = opts,
2278	};
2279	struct bpf_diag_log *log;
2280	struct diag_fmt_mark mark;
2281	bool first = true;
2282	int start_idx;
2283	u32 i, visible_cnt = 0, visible_idx = 0;
2284
2285	if (!bpf_diag_enabled(env))
2286	return;
2287
2288	if (!env->diag)
2289	return;
2290	log = &env->diag->log;
2291
2292	diag_build_lineage(env, log, &filter);
2293
2294	start_idx = diag_history_start_idx(log, &filter);
2295	for (i = start_idx; i < log->cnt; i++) {
2296	event = &log->events[log_pos(log, i)];
2297	if (diag_history_event_visible(event, &filter))
2298	visible_cnt++;
2299	}
2300
2301	if (!visible_cnt && !log->first_seq && opts->scope == BPF_DIAG_HISTORY_SCOPE_STACK_ARG)
2302	return;
2303
2304	diag_section(env, "Causal path");
2305	mark = diag_fmt_save(env);
2306	for (i = start_idx; i < log->cnt; i++) {
2307	event = &log->events[log_pos(log, i)];
2308	if (!diag_history_event_visible(event, &filter))
2309	continue;
2310
2311	diag_fmt_restore(env, mark);
2312	if (visible_cnt > BPF_DIAG_HISTORY_RENDER_MAX &&
2313	visible_idx >= BPF_DIAG_HISTORY_RENDER_MAX / 2 &&
2314	visible_idx < visible_cnt - BPF_DIAG_HISTORY_RENDER_MAX / 2) {
2315	if (visible_idx++ != BPF_DIAG_HISTORY_RENDER_MAX / 2)
2316	continue;
2317	if (!first)
2318	diag_write(env, "\n");
2319	first = false;
2320	diag_write(env, " %u intermediate causal-history events omitted\n",
2321	visible_cnt - BPF_DIAG_HISTORY_RENDER_MAX);
2322	continue;
2323	}
2324	visible_idx++;
2325
2326	if (!first)
2327	diag_write(env, "\n");
2328	first = false;
2329
2330	switch (event->kind) {
2331	case BPF_DIAG_HISTORY_BRANCH:
2332	bpf_diag_source(env, event->insn_idx, "branch",
2333	"took the %s branch of this conditional, goto %s",
2334	event->branch.cond_true ? "true" : "false",
2335	event->branch.cond_true ? "followed" : "not followed");
2336	break;
2337	case BPF_DIAG_HISTORY_MOD:
2338	diag_print_mod(env, event);
2339	break;
2340	case BPF_DIAG_HISTORY_REF_ACQUIRE:
2341	case BPF_DIAG_HISTORY_REF_RELEASE:
2342	diag_print_ref_event(env, event);
2343	break;
2344	case BPF_DIAG_HISTORY_CONTEXT:
2345	diag_print_context_event(env, event);
2346	break;
2347	default:
2348	break;
2349	}
2350	}
2351
2352	if (!visible_cnt)
2353	diag_write(env, " no retained diagnostic events on this path\n");
2354	if (log->first_seq)
2355	diag_write(env, " %llu older causal-history event%s not retained because diagnostic "
2356	"event storage reached capacity\n",
2357	log->first_seq, log->first_seq == 1 ? "" : "s");
2358	diag_fmt_restore(env, mark);
2359	}