projects/linux·blob master · refs

1233 lines · 41.9 KiB

1// SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause)
2/* Copyright (c) 2021 Facebook */
3#include <stdio.h>
4#include <stdlib.h>
5#include <string.h>
6#include <errno.h>
7#include <asm/byteorder.h>
8#include <linux/filter.h>
9#include <sys/param.h>
10#include "btf.h"
11#include "bpf.h"
12#include "libbpf.h"
13#include "libbpf_internal.h"
14#include "hashmap.h"
15#include "bpf_gen_internal.h"
16#include "skel_internal.h"
17
18#define MAX_USED_MAPS 64
19#define MAX_USED_PROGS 32
20#define MAX_KFUNC_DESCS 256
21#define MAX_FD_ARRAY_SZ (MAX_USED_MAPS + MAX_KFUNC_DESCS)
22
23/* The following structure describes the stack layout of the loader program.
24 * In addition R6 contains the pointer to context.
25 * R7 contains the result of the last sys_bpf command (typically error or FD).
26 * R9 contains the result of the last sys_close command.
27 *
28 * Naming convention:
29 * ctx - bpf program context
30 * stack - bpf program stack
31 * blob - bpf_attr-s, strings, insns, map data.
32 * All the bytes that loader prog will use for read/write.
33 */
34struct loader_stack {
35 __u32 btf_fd;
36 __u32 inner_map_fd;
37 __u32 prog_fd[MAX_USED_PROGS];
38};
39
40#define stack_off(field) \
41 (__s16)(-sizeof(struct loader_stack) + offsetof(struct loader_stack, field))
42
43#define attr_field(attr, field) (attr + offsetof(union bpf_attr, field))
44
45static int blob_fd_array_off(struct bpf_gen *gen, int index)
46{
47 return gen->fd_array + index * sizeof(int);
48}
49
50static int realloc_insn_buf(struct bpf_gen *gen, __u32 size)
51{
52 size_t off = gen->insn_cur - gen->insn_start;
53 void *insn_start;
54
55 if (gen->error)
56 return gen->error;
57 if (size > INT32_MAX || off + size > INT32_MAX) {
58 gen->error = -ERANGE;
59 return -ERANGE;
60 }
61 insn_start = realloc(gen->insn_start, off + size);
62 if (!insn_start) {
63 gen->error = -ENOMEM;
64 free(gen->insn_start);
65 gen->insn_start = NULL;
66 gen->insn_cur = NULL;
67 return -ENOMEM;
68 }
69 gen->insn_start = insn_start;
70 gen->insn_cur = insn_start + off;
71 return 0;
72}
73
74static int realloc_data_buf(struct bpf_gen *gen, __u32 size)
75{
76 size_t off = gen->data_cur - gen->data_start;
77 void *data_start;
78
79 if (gen->error)
80 return gen->error;
81 if (size > INT32_MAX || off + size > INT32_MAX) {
82 gen->error = -ERANGE;
83 return -ERANGE;
84 }
85 data_start = realloc(gen->data_start, off + size);
86 if (!data_start) {
87 gen->error = -ENOMEM;
88 free(gen->data_start);
89 gen->data_start = NULL;
90 gen->data_cur = NULL;
91 return -ENOMEM;
92 }
93 gen->data_start = data_start;
94 gen->data_cur = data_start + off;
95 return 0;
96}
97
98static void emit(struct bpf_gen *gen, struct bpf_insn insn)
99{
100 if (realloc_insn_buf(gen, sizeof(insn)))
101 return;
102 memcpy(gen->insn_cur, &insn, sizeof(insn));
103 gen->insn_cur += sizeof(insn);
104}
105
106static void emit2(struct bpf_gen *gen, struct bpf_insn insn1, struct bpf_insn insn2)
107{
108 emit(gen, insn1);
109 emit(gen, insn2);
110}
111
112static int add_data(struct bpf_gen *gen, const void *data, __u32 size);
113static void emit_sys_close_blob(struct bpf_gen *gen, int blob_off);
114
115void bpf_gen__init(struct bpf_gen *gen, int log_level, int nr_progs, int nr_maps)
116{
117 size_t stack_sz = sizeof(struct loader_stack), nr_progs_sz;
118 int i;
119
120 gen->fd_array = add_data(gen, NULL, MAX_FD_ARRAY_SZ * sizeof(int));
121 gen->log_level = log_level;
122 /* save ctx pointer into R6 */
123 emit(gen, BPF_MOV64_REG(BPF_REG_6, BPF_REG_1));
124
125 /* bzero stack */
126 emit(gen, BPF_MOV64_REG(BPF_REG_1, BPF_REG_10));
127 emit(gen, BPF_ALU64_IMM(BPF_ADD, BPF_REG_1, -stack_sz));
128 emit(gen, BPF_MOV64_IMM(BPF_REG_2, stack_sz));
129 emit(gen, BPF_MOV64_IMM(BPF_REG_3, 0));
130 emit(gen, BPF_EMIT_CALL(BPF_FUNC_probe_read_kernel));
131
132 /* amount of stack actually used, only used to calculate iterations, not stack offset */
133 nr_progs_sz = offsetof(struct loader_stack, prog_fd[nr_progs]);
134 /* jump over cleanup code */
135 emit(gen, BPF_JMP_IMM(BPF_JA, 0, 0,
136 /* size of cleanup code below (including map fd cleanup) */
137 (nr_progs_sz / 4) * 3 + 2 +
138 /* 6 insns for emit_sys_close_blob,
139 * 6 insns for debug_regs in emit_sys_close_blob
140 */
141 nr_maps * (6 + (gen->log_level ? 6 : 0))));
142
143 /* remember the label where all error branches will jump to */
144 gen->cleanup_label = gen->insn_cur - gen->insn_start;
145 /* emit cleanup code: close all temp FDs */
146 for (i = 0; i < nr_progs_sz; i += 4) {
147 emit(gen, BPF_LDX_MEM(BPF_W, BPF_REG_1, BPF_REG_10, -stack_sz + i));
148 emit(gen, BPF_JMP_IMM(BPF_JSLE, BPF_REG_1, 0, 1));
149 emit(gen, BPF_EMIT_CALL(BPF_FUNC_sys_close));
150 }
151 for (i = 0; i < nr_maps; i++)
152 emit_sys_close_blob(gen, blob_fd_array_off(gen, i));
153 /* R7 contains the error code from sys_bpf. Copy it into R0 and exit. */
154 emit(gen, BPF_MOV64_REG(BPF_REG_0, BPF_REG_7));
155 emit(gen, BPF_EXIT_INSN());
156}
157
158static int add_data(struct bpf_gen *gen, const void *data, __u32 size)
159{
160 __u64 zero = 0;
161 __u32 size8;
162 void *prev;
163
164 if (size > INT32_MAX) {
165 gen->error = -ERANGE;
166 return 0;
167 }
168 size8 = roundup(size, 8);
169
170 if (realloc_data_buf(gen, size8))
171 return 0;
172 prev = gen->data_cur;
173 if (data) {
174 memcpy(gen->data_cur, data, size);
175 memcpy(gen->data_cur + size, &zero, size8 - size);
176 } else {
177 memset(gen->data_cur, 0, size8);
178 }
179 gen->data_cur += size8;
180 return prev - gen->data_start;
181}
182
183/* Get index for map_fd/btf_fd slot in reserved fd_array, or in data relative
184 * to start of fd_array. Caller can decide if it is usable or not.
185 */
186static int add_map_fd(struct bpf_gen *gen)
187{
188 if (gen->nr_maps == MAX_USED_MAPS) {
189 pr_warn("Total maps exceeds %d\n", MAX_USED_MAPS);
190 gen->error = -E2BIG;
191 return 0;
192 }
193 return gen->nr_maps++;
194}
195
196static int add_kfunc_btf_fd(struct bpf_gen *gen)
197{
198 int cur;
199
200 if (gen->nr_fd_array == MAX_KFUNC_DESCS) {
201 cur = add_data(gen, NULL, sizeof(int));
202 return (cur - gen->fd_array) / sizeof(int);
203 }
204 return MAX_USED_MAPS + gen->nr_fd_array++;
205}
206
207static int insn_bytes_to_bpf_size(__u32 sz)
208{
209 switch (sz) {
210 case 8: return BPF_DW;
211 case 4: return BPF_W;
212 case 2: return BPF_H;
213 case 1: return BPF_B;
214 default: return -1;
215 }
216}
217
218/* *(u64 *)(blob + off) = (u64)(void *)(blob + data) */
219static void emit_rel_store(struct bpf_gen *gen, int off, int data)
220{
221 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_0, BPF_PSEUDO_MAP_IDX_VALUE,
222 0, 0, 0, data));
223 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE,
224 0, 0, 0, off));
225 emit(gen, BPF_STX_MEM(BPF_DW, BPF_REG_1, BPF_REG_0, 0));
226}
227
228static void move_blob2blob(struct bpf_gen *gen, int off, int size, int blob_off)
229{
230 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_2, BPF_PSEUDO_MAP_IDX_VALUE,
231 0, 0, 0, blob_off));
232 emit(gen, BPF_LDX_MEM(insn_bytes_to_bpf_size(size), BPF_REG_0, BPF_REG_2, 0));
233 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE,
234 0, 0, 0, off));
235 emit(gen, BPF_STX_MEM(insn_bytes_to_bpf_size(size), BPF_REG_1, BPF_REG_0, 0));
236}
237
238static void move_blob2ctx(struct bpf_gen *gen, int ctx_off, int size, int blob_off)
239{
240 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE,
241 0, 0, 0, blob_off));
242 emit(gen, BPF_LDX_MEM(insn_bytes_to_bpf_size(size), BPF_REG_0, BPF_REG_1, 0));
243 emit(gen, BPF_STX_MEM(insn_bytes_to_bpf_size(size), BPF_REG_6, BPF_REG_0, ctx_off));
244}
245
246static void move_ctx2blob(struct bpf_gen *gen, int off, int size, int ctx_off,
247 bool check_non_zero)
248{
249 emit(gen, BPF_LDX_MEM(insn_bytes_to_bpf_size(size), BPF_REG_0, BPF_REG_6, ctx_off));
250 if (check_non_zero)
251 /* If value in ctx is zero don't update the blob.
252 * For example: when ctx->map.max_entries == 0, keep default max_entries from bpf.c
253 */
254 emit(gen, BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 0, 3));
255 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE,
256 0, 0, 0, off));
257 emit(gen, BPF_STX_MEM(insn_bytes_to_bpf_size(size), BPF_REG_1, BPF_REG_0, 0));
258}
259
260static void move_stack2blob(struct bpf_gen *gen, int off, int size, int stack_off)
261{
262 emit(gen, BPF_LDX_MEM(insn_bytes_to_bpf_size(size), BPF_REG_0, BPF_REG_10, stack_off));
263 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE,
264 0, 0, 0, off));
265 emit(gen, BPF_STX_MEM(insn_bytes_to_bpf_size(size), BPF_REG_1, BPF_REG_0, 0));
266}
267
268static void move_stack2ctx(struct bpf_gen *gen, int ctx_off, int size, int stack_off)
269{
270 emit(gen, BPF_LDX_MEM(insn_bytes_to_bpf_size(size), BPF_REG_0, BPF_REG_10, stack_off));
271 emit(gen, BPF_STX_MEM(insn_bytes_to_bpf_size(size), BPF_REG_6, BPF_REG_0, ctx_off));
272}
273
274static void emit_sys_bpf(struct bpf_gen *gen, int cmd, int attr, int attr_size)
275{
276 emit(gen, BPF_MOV64_IMM(BPF_REG_1, cmd));
277 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_2, BPF_PSEUDO_MAP_IDX_VALUE,
278 0, 0, 0, attr));
279 emit(gen, BPF_MOV64_IMM(BPF_REG_3, attr_size));
280 emit(gen, BPF_EMIT_CALL(BPF_FUNC_sys_bpf));
281 /* remember the result in R7 */
282 emit(gen, BPF_MOV64_REG(BPF_REG_7, BPF_REG_0));
283}
284
285static bool is_simm16(__s64 value)
286{
287 return value == (__s64)(__s16)value;
288}
289
290static void emit_check_err(struct bpf_gen *gen)
291{
292 __s64 off = -(gen->insn_cur - gen->insn_start - gen->cleanup_label) / 8 - 1;
293
294 /* R7 contains result of last sys_bpf command.
295 * if (R7 < 0) goto cleanup;
296 */
297 if (is_simm16(off)) {
298 emit(gen, BPF_JMP_IMM(BPF_JSLT, BPF_REG_7, 0, off));
299 } else {
300 gen->error = -ERANGE;
301 }
302}
303
304/* reg1 and reg2 should not be R1 - R5. They can be R0, R6 - R10 */
305static void emit_debug(struct bpf_gen *gen, int reg1, int reg2,
306 const char *fmt, va_list args)
307{
308 char buf[1024];
309 int addr, len, ret;
310
311 if (!gen->log_level)
312 return;
313 ret = vsnprintf(buf, sizeof(buf), fmt, args);
314 if (ret < 1024 - 7 && reg1 >= 0 && reg2 < 0)
315 /* The special case to accommodate common debug_ret():
316 * to avoid specifying BPF_REG_7 and adding " r=%%d" to
317 * prints explicitly.
318 */
319 strcat(buf, " r=%d");
320 len = strlen(buf) + 1;
321 addr = add_data(gen, buf, len);
322
323 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE,
324 0, 0, 0, addr));
325 emit(gen, BPF_MOV64_IMM(BPF_REG_2, len));
326 if (reg1 >= 0)
327 emit(gen, BPF_MOV64_REG(BPF_REG_3, reg1));
328 if (reg2 >= 0)
329 emit(gen, BPF_MOV64_REG(BPF_REG_4, reg2));
330 emit(gen, BPF_EMIT_CALL(BPF_FUNC_trace_printk));
331}
332
333static void debug_regs(struct bpf_gen *gen, int reg1, int reg2, const char *fmt, ...)
334{
335 va_list args;
336
337 va_start(args, fmt);
338 emit_debug(gen, reg1, reg2, fmt, args);
339 va_end(args);
340}
341
342static void debug_ret(struct bpf_gen *gen, const char *fmt, ...)
343{
344 va_list args;
345
346 va_start(args, fmt);
347 emit_debug(gen, BPF_REG_7, -1, fmt, args);
348 va_end(args);
349}
350
351static void __emit_sys_close(struct bpf_gen *gen)
352{
353 emit(gen, BPF_JMP_IMM(BPF_JSLE, BPF_REG_1, 0,
354 /* 2 is the number of the following insns
355 * * 6 is additional insns in debug_regs
356 */
357 2 + (gen->log_level ? 6 : 0)));
358 emit(gen, BPF_MOV64_REG(BPF_REG_9, BPF_REG_1));
359 emit(gen, BPF_EMIT_CALL(BPF_FUNC_sys_close));
360 debug_regs(gen, BPF_REG_9, BPF_REG_0, "close(%%d) = %%d");
361}
362
363static void emit_sys_close_stack(struct bpf_gen *gen, int stack_off)
364{
365 emit(gen, BPF_LDX_MEM(BPF_W, BPF_REG_1, BPF_REG_10, stack_off));
366 __emit_sys_close(gen);
367}
368
369static void emit_sys_close_blob(struct bpf_gen *gen, int blob_off)
370{
371 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_0, BPF_PSEUDO_MAP_IDX_VALUE,
372 0, 0, 0, blob_off));
373 emit(gen, BPF_LDX_MEM(BPF_W, BPF_REG_1, BPF_REG_0, 0));
374 __emit_sys_close(gen);
375}
376
377int bpf_gen__finish(struct bpf_gen *gen, int nr_progs, int nr_maps)
378{
379 int i;
380
381 if (nr_progs < gen->nr_progs || nr_maps != gen->nr_maps) {
382 pr_warn("nr_progs %d/%u nr_maps %d/%u mismatch\n",
383 nr_progs, gen->nr_progs, nr_maps, gen->nr_maps);
384 gen->error = -EFAULT;
385 return gen->error;
386 }
387 emit_sys_close_stack(gen, stack_off(btf_fd));
388 for (i = 0; i < gen->nr_progs; i++)
389 move_stack2ctx(gen,
390 sizeof(struct bpf_loader_ctx) +
391 sizeof(struct bpf_map_desc) * gen->nr_maps +
392 sizeof(struct bpf_prog_desc) * i +
393 offsetof(struct bpf_prog_desc, prog_fd), 4,
394 stack_off(prog_fd[i]));
395 for (i = 0; i < gen->nr_maps; i++)
396 move_blob2ctx(gen,
397 sizeof(struct bpf_loader_ctx) +
398 sizeof(struct bpf_map_desc) * i +
399 offsetof(struct bpf_map_desc, map_fd), 4,
400 blob_fd_array_off(gen, i));
401 emit(gen, BPF_MOV64_IMM(BPF_REG_0, 0));
402 emit(gen, BPF_EXIT_INSN());
403 if (!gen->error) {
404 struct gen_loader_opts *opts = gen->opts;
405
406 opts->insns = gen->insn_start;
407 opts->insns_sz = gen->insn_cur - gen->insn_start;
408 opts->data = gen->data_start;
409 opts->data_sz = gen->data_cur - gen->data_start;
410
411 /* use target endianness for embedded loader */
412 if (gen->swapped_endian) {
413 struct bpf_insn *insn = (struct bpf_insn *)opts->insns;
414 int insn_cnt = opts->insns_sz / sizeof(struct bpf_insn);
415
416 for (i = 0; i < insn_cnt; i++)
417 bpf_insn_bswap(insn++);
418 }
419 }
420 pr_debug("gen: finish %s\n", errstr(gen->error));
421 return gen->error;
422}
423
424void bpf_gen__free(struct bpf_gen *gen)
425{
426 if (!gen)
427 return;
428 free(gen->data_start);
429 free(gen->insn_start);
430 free(gen);
431}
432
433/*
434 * Fields of bpf_attr are set to values in native byte-order before being
435 * written to the target-bound data blob, and may need endian conversion.
436 * This macro allows providing the correct value in situ more simply than
437 * writing a separate converter for *all fields* of *all records* included
438 * in union bpf_attr. Note that sizeof(rval) should match the assignment
439 * target to avoid runtime problems.
440 */
441#define tgt_endian(rval) ({ \
442 typeof(rval) _val = (rval); \
443 if (gen->swapped_endian) { \
444 switch (sizeof(_val)) { \
445 case 1: break; \
446 case 2: _val = bswap_16(_val); break; \
447 case 4: _val = bswap_32(_val); break; \
448 case 8: _val = bswap_64(_val); break; \
449 default: pr_warn("unsupported bswap size!\n"); \
450 } \
451 } \
452 _val; \
453})
454
455void bpf_gen__load_btf(struct bpf_gen *gen, const void *btf_raw_data,
456 __u32 btf_raw_size)
457{
458 int attr_size = offsetofend(union bpf_attr, btf_log_level);
459 int btf_data, btf_load_attr;
460 union bpf_attr attr;
461
462 memset(&attr, 0, attr_size);
463 btf_data = add_data(gen, btf_raw_data, btf_raw_size);
464
465 attr.btf_size = tgt_endian(btf_raw_size);
466 btf_load_attr = add_data(gen, &attr, attr_size);
467 pr_debug("gen: load_btf: off %d size %u, attr: off %d size %d\n",
468 btf_data, btf_raw_size, btf_load_attr, attr_size);
469
470 /* populate union bpf_attr with user provided log details */
471 move_ctx2blob(gen, attr_field(btf_load_attr, btf_log_level), 4,
472 offsetof(struct bpf_loader_ctx, log_level), false);
473 move_ctx2blob(gen, attr_field(btf_load_attr, btf_log_size), 4,
474 offsetof(struct bpf_loader_ctx, log_size), false);
475 move_ctx2blob(gen, attr_field(btf_load_attr, btf_log_buf), 8,
476 offsetof(struct bpf_loader_ctx, log_buf), false);
477 /* populate union bpf_attr with a pointer to the BTF data */
478 emit_rel_store(gen, attr_field(btf_load_attr, btf), btf_data);
479 /* emit BTF_LOAD command */
480 emit_sys_bpf(gen, BPF_BTF_LOAD, btf_load_attr, attr_size);
481 debug_ret(gen, "btf_load size %d", btf_raw_size);
482 emit_check_err(gen);
483 /* remember btf_fd in the stack, if successful */
484 emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_10, BPF_REG_7, stack_off(btf_fd)));
485}
486
487void bpf_gen__map_create(struct bpf_gen *gen,
488 enum bpf_map_type map_type,
489 const char *map_name,
490 __u32 key_size, __u32 value_size, __u32 max_entries,
491 struct bpf_map_create_opts *map_attr, int map_idx)
492{
493 int attr_size = offsetofend(union bpf_attr, map_extra);
494 bool close_inner_map_fd = false;
495 int map_create_attr, idx;
496 union bpf_attr attr;
497
498 memset(&attr, 0, attr_size);
499 attr.map_type = tgt_endian(map_type);
500 attr.key_size = tgt_endian(key_size);
501 attr.value_size = tgt_endian(value_size);
502 attr.map_flags = tgt_endian(map_attr->map_flags);
503 attr.map_extra = tgt_endian(map_attr->map_extra);
504 if (map_name)
505 libbpf_strlcpy(attr.map_name, map_name, sizeof(attr.map_name));
506 attr.numa_node = tgt_endian(map_attr->numa_node);
507 attr.map_ifindex = tgt_endian(map_attr->map_ifindex);
508 attr.max_entries = tgt_endian(max_entries);
509 attr.btf_key_type_id = tgt_endian(map_attr->btf_key_type_id);
510 attr.btf_value_type_id = tgt_endian(map_attr->btf_value_type_id);
511
512 map_create_attr = add_data(gen, &attr, attr_size);
513 pr_debug("gen: map_create: %s idx %d type %u value_type_id %u, attr: off %d size %d\n",
514 map_name, map_idx, map_type, map_attr->btf_value_type_id,
515 map_create_attr, attr_size);
516
517 if (map_attr->btf_value_type_id)
518 /* populate union bpf_attr with btf_fd saved in the stack earlier */
519 move_stack2blob(gen, attr_field(map_create_attr, btf_fd), 4,
520 stack_off(btf_fd));
521 switch (map_type) {
522 case BPF_MAP_TYPE_ARRAY_OF_MAPS:
523 case BPF_MAP_TYPE_HASH_OF_MAPS:
524 move_stack2blob(gen, attr_field(map_create_attr, inner_map_fd), 4,
525 stack_off(inner_map_fd));
526 close_inner_map_fd = true;
527 break;
528 default:
529 break;
530 }
531
532 /*
533 * Conditionally update max_entries from the host-supplied loader
534 * ctx. This sizes the map at runtime, but for a signed loader
535 * (gen_hash) it would let an untrusted host re-dimension the
536 * program's maps, outside what the signature attests to: the
537 * metadata blob is covered by the program signature and verified
538 * by the kernel at load time. Keep the signer-provided max_entries
539 * baked into the blob in that case.
540 */
541 if (map_idx >= 0 && !OPTS_GET(gen->opts, gen_hash, false))
542 move_ctx2blob(gen, attr_field(map_create_attr, max_entries), 4,
543 sizeof(struct bpf_loader_ctx) +
544 sizeof(struct bpf_map_desc) * map_idx +
545 offsetof(struct bpf_map_desc, max_entries),
546 true /* check that max_entries != 0 */);
547
548 /* emit MAP_CREATE command */
549 emit_sys_bpf(gen, BPF_MAP_CREATE, map_create_attr, attr_size);
550 debug_ret(gen, "map_create %s idx %d type %d value_size %d value_btf_id %d",
551 map_name, map_idx, map_type, value_size,
552 map_attr->btf_value_type_id);
553 emit_check_err(gen);
554 /* remember map_fd in the stack, if successful */
555 if (map_idx < 0) {
556 /* This bpf_gen__map_create() function is called with map_idx >= 0
557 * for all maps that libbpf loading logic tracks.
558 * It's called with -1 to create an inner map.
559 */
560 emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_10, BPF_REG_7,
561 stack_off(inner_map_fd)));
562 } else if (map_idx != gen->nr_maps) {
563 gen->error = -EDOM; /* internal bug */
564 return;
565 } else {
566 /* add_map_fd does gen->nr_maps++ */
567 idx = add_map_fd(gen);
568 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE,
569 0, 0, 0, blob_fd_array_off(gen, idx)));
570 emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_1, BPF_REG_7, 0));
571 }
572 if (close_inner_map_fd)
573 emit_sys_close_stack(gen, stack_off(inner_map_fd));
574}
575
576void bpf_gen__record_attach_target(struct bpf_gen *gen, const char *attach_name,
577 enum bpf_attach_type type)
578{
579 const char *prefix;
580 int kind, ret;
581
582 btf_get_kernel_prefix_kind(type, &prefix, &kind);
583 gen->attach_kind = kind;
584 ret = snprintf(gen->attach_target, sizeof(gen->attach_target), "%s%s",
585 prefix, attach_name);
586 if (ret >= sizeof(gen->attach_target))
587 gen->error = -ENOSPC;
588}
589
590static void emit_find_attach_target(struct bpf_gen *gen)
591{
592 int name, len = strlen(gen->attach_target) + 1;
593
594 pr_debug("gen: find_attach_tgt %s %d\n", gen->attach_target, gen->attach_kind);
595 name = add_data(gen, gen->attach_target, len);
596
597 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE,
598 0, 0, 0, name));
599 emit(gen, BPF_MOV64_IMM(BPF_REG_2, len));
600 emit(gen, BPF_MOV64_IMM(BPF_REG_3, gen->attach_kind));
601 emit(gen, BPF_MOV64_IMM(BPF_REG_4, 0));
602 emit(gen, BPF_EMIT_CALL(BPF_FUNC_btf_find_by_name_kind));
603 emit(gen, BPF_MOV64_REG(BPF_REG_7, BPF_REG_0));
604 debug_ret(gen, "find_by_name_kind(%s,%d)",
605 gen->attach_target, gen->attach_kind);
606 emit_check_err(gen);
607 /* if successful, btf_id is in lower 32-bit of R7 and
608 * btf_obj_fd is in upper 32-bit
609 */
610}
611
612void bpf_gen__record_extern(struct bpf_gen *gen, const char *name, bool is_weak,
613 bool is_typeless, bool is_ld64, int kind, int insn_idx)
614{
615 struct ksym_relo_desc *relo;
616
617 relo = libbpf_reallocarray(gen->relos, gen->relo_cnt + 1, sizeof(*relo));
618 if (!relo) {
619 gen->error = -ENOMEM;
620 return;
621 }
622 gen->relos = relo;
623 relo += gen->relo_cnt;
624 relo->name = name;
625 relo->is_weak = is_weak;
626 relo->is_typeless = is_typeless;
627 relo->is_ld64 = is_ld64;
628 relo->kind = kind;
629 relo->insn_idx = insn_idx;
630 gen->relo_cnt++;
631}
632
633/* returns existing ksym_desc with ref incremented, or inserts a new one */
634static struct ksym_desc *get_ksym_desc(struct bpf_gen *gen, struct ksym_relo_desc *relo)
635{
636 struct ksym_desc *kdesc;
637 int i;
638
639 for (i = 0; i < gen->nr_ksyms; i++) {
640 kdesc = &gen->ksyms[i];
641 if (kdesc->kind == relo->kind && kdesc->is_ld64 == relo->is_ld64 &&
642 !strcmp(kdesc->name, relo->name)) {
643 kdesc->ref++;
644 return kdesc;
645 }
646 }
647 kdesc = libbpf_reallocarray(gen->ksyms, gen->nr_ksyms + 1, sizeof(*kdesc));
648 if (!kdesc) {
649 gen->error = -ENOMEM;
650 return NULL;
651 }
652 gen->ksyms = kdesc;
653 kdesc = &gen->ksyms[gen->nr_ksyms++];
654 kdesc->name = relo->name;
655 kdesc->kind = relo->kind;
656 kdesc->ref = 1;
657 kdesc->off = 0;
658 kdesc->insn = 0;
659 kdesc->is_ld64 = relo->is_ld64;
660 return kdesc;
661}
662
663/* Overwrites BPF_REG_{0, 1, 2, 3, 4, 7}
664 * Returns result in BPF_REG_7
665 */
666static void emit_bpf_find_by_name_kind(struct bpf_gen *gen, struct ksym_relo_desc *relo)
667{
668 int name_off, len = strlen(relo->name) + 1;
669
670 name_off = add_data(gen, relo->name, len);
671 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE,
672 0, 0, 0, name_off));
673 emit(gen, BPF_MOV64_IMM(BPF_REG_2, len));
674 emit(gen, BPF_MOV64_IMM(BPF_REG_3, relo->kind));
675 emit(gen, BPF_MOV64_IMM(BPF_REG_4, 0));
676 emit(gen, BPF_EMIT_CALL(BPF_FUNC_btf_find_by_name_kind));
677 emit(gen, BPF_MOV64_REG(BPF_REG_7, BPF_REG_0));
678 debug_ret(gen, "find_by_name_kind(%s,%d)", relo->name, relo->kind);
679}
680
681/* Overwrites BPF_REG_{0, 1, 2, 3, 4, 7}
682 * Returns result in BPF_REG_7
683 * Returns u64 symbol addr in BPF_REG_9
684 */
685static void emit_bpf_kallsyms_lookup_name(struct bpf_gen *gen, struct ksym_relo_desc *relo)
686{
687 int name_off, len = strlen(relo->name) + 1, res_off;
688
689 name_off = add_data(gen, relo->name, len);
690 res_off = add_data(gen, NULL, 8); /* res is u64 */
691 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE,
692 0, 0, 0, name_off));
693 emit(gen, BPF_MOV64_IMM(BPF_REG_2, len));
694 emit(gen, BPF_MOV64_IMM(BPF_REG_3, 0));
695 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_4, BPF_PSEUDO_MAP_IDX_VALUE,
696 0, 0, 0, res_off));
697 emit(gen, BPF_MOV64_REG(BPF_REG_7, BPF_REG_4));
698 emit(gen, BPF_EMIT_CALL(BPF_FUNC_kallsyms_lookup_name));
699 emit(gen, BPF_LDX_MEM(BPF_DW, BPF_REG_9, BPF_REG_7, 0));
700 emit(gen, BPF_MOV64_REG(BPF_REG_7, BPF_REG_0));
701 debug_ret(gen, "kallsyms_lookup_name(%s,%d)", relo->name, relo->kind);
702}
703
704/* Expects:
705 * BPF_REG_8 - pointer to instruction
706 *
707 * We need to reuse BTF fd for same symbol otherwise each relocation takes a new
708 * index, while kernel limits total kfunc BTFs to 256. For duplicate symbols,
709 * this would mean a new BTF fd index for each entry. By pairing symbol name
710 * with index, we get the insn->imm, insn->off pairing that kernel uses for
711 * kfunc_tab, which becomes the effective limit even though all of them may
712 * share same index in fd_array (such that kfunc_btf_tab has 1 element).
713 */
714static void emit_relo_kfunc_btf(struct bpf_gen *gen, struct ksym_relo_desc *relo, int insn)
715{
716 struct ksym_desc *kdesc;
717 int btf_fd_idx;
718
719 kdesc = get_ksym_desc(gen, relo);
720 if (!kdesc)
721 return;
722 /* try to copy from existing bpf_insn */
723 if (kdesc->ref > 1) {
724 move_blob2blob(gen, insn + offsetof(struct bpf_insn, imm), 4,
725 kdesc->insn + offsetof(struct bpf_insn, imm));
726 move_blob2blob(gen, insn + offsetof(struct bpf_insn, off), 2,
727 kdesc->insn + offsetof(struct bpf_insn, off));
728 goto log;
729 }
730 /* remember insn offset, so we can copy BTF ID and FD later */
731 kdesc->insn = insn;
732 emit_bpf_find_by_name_kind(gen, relo);
733 if (!relo->is_weak)
734 emit_check_err(gen);
735 /* get index in fd_array to store BTF FD at */
736 btf_fd_idx = add_kfunc_btf_fd(gen);
737 if (btf_fd_idx > INT16_MAX) {
738 pr_warn("BTF fd off %d for kfunc %s exceeds INT16_MAX, cannot process relocation\n",
739 btf_fd_idx, relo->name);
740 gen->error = -E2BIG;
741 return;
742 }
743 kdesc->off = btf_fd_idx;
744 /* jump to success case */
745 emit(gen, BPF_JMP_IMM(BPF_JSGE, BPF_REG_7, 0, 3));
746 /* set value for imm, off as 0 */
747 emit(gen, BPF_ST_MEM(BPF_W, BPF_REG_8, offsetof(struct bpf_insn, imm), 0));
748 emit(gen, BPF_ST_MEM(BPF_H, BPF_REG_8, offsetof(struct bpf_insn, off), 0));
749 /* skip success case for ret < 0 */
750 emit(gen, BPF_JMP_IMM(BPF_JA, 0, 0, 10));
751 /* store btf_id into insn[insn_idx].imm */
752 emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_8, BPF_REG_7, offsetof(struct bpf_insn, imm)));
753 /* obtain fd in BPF_REG_9 */
754 emit(gen, BPF_MOV64_REG(BPF_REG_9, BPF_REG_7));
755 emit(gen, BPF_ALU64_IMM(BPF_RSH, BPF_REG_9, 32));
756 /* load fd_array slot pointer */
757 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_0, BPF_PSEUDO_MAP_IDX_VALUE,
758 0, 0, 0, blob_fd_array_off(gen, btf_fd_idx)));
759 /* store BTF fd in slot, 0 for vmlinux */
760 emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_0, BPF_REG_9, 0));
761 /* jump to insn[insn_idx].off store if fd denotes module BTF */
762 emit(gen, BPF_JMP_IMM(BPF_JNE, BPF_REG_9, 0, 2));
763 /* set the default value for off */
764 emit(gen, BPF_ST_MEM(BPF_H, BPF_REG_8, offsetof(struct bpf_insn, off), 0));
765 /* skip BTF fd store for vmlinux BTF */
766 emit(gen, BPF_JMP_IMM(BPF_JA, 0, 0, 1));
767 /* store index into insn[insn_idx].off */
768 emit(gen, BPF_ST_MEM(BPF_H, BPF_REG_8, offsetof(struct bpf_insn, off), btf_fd_idx));
769log:
770 if (!gen->log_level)
771 return;
772 emit(gen, BPF_LDX_MEM(BPF_W, BPF_REG_7, BPF_REG_8,
773 offsetof(struct bpf_insn, imm)));
774 emit(gen, BPF_LDX_MEM(BPF_H, BPF_REG_9, BPF_REG_8,
775 offsetof(struct bpf_insn, off)));
776 debug_regs(gen, BPF_REG_7, BPF_REG_9, " func (%s:count=%d): imm: %%d, off: %%d",
777 relo->name, kdesc->ref);
778 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_0, BPF_PSEUDO_MAP_IDX_VALUE,
779 0, 0, 0, blob_fd_array_off(gen, kdesc->off)));
780 emit(gen, BPF_LDX_MEM(BPF_W, BPF_REG_9, BPF_REG_0, 0));
781 debug_regs(gen, BPF_REG_9, -1, " func (%s:count=%d): btf_fd",
782 relo->name, kdesc->ref);
783}
784
785static void emit_ksym_relo_log(struct bpf_gen *gen, struct ksym_relo_desc *relo,
786 int ref)
787{
788 if (!gen->log_level)
789 return;
790 emit(gen, BPF_LDX_MEM(BPF_W, BPF_REG_7, BPF_REG_8,
791 offsetof(struct bpf_insn, imm)));
792 emit(gen, BPF_LDX_MEM(BPF_H, BPF_REG_9, BPF_REG_8, sizeof(struct bpf_insn) +
793 offsetof(struct bpf_insn, imm)));
794 debug_regs(gen, BPF_REG_7, BPF_REG_9, " var t=%d w=%d (%s:count=%d): imm[0]: %%d, imm[1]: %%d",
795 relo->is_typeless, relo->is_weak, relo->name, ref);
796 emit(gen, BPF_LDX_MEM(BPF_B, BPF_REG_9, BPF_REG_8, offsetofend(struct bpf_insn, code)));
797 debug_regs(gen, BPF_REG_9, -1, " var t=%d w=%d (%s:count=%d): insn.reg",
798 relo->is_typeless, relo->is_weak, relo->name, ref);
799}
800
801/* Expects:
802 * BPF_REG_8 - pointer to instruction
803 */
804static void emit_relo_ksym_typeless(struct bpf_gen *gen,
805 struct ksym_relo_desc *relo, int insn)
806{
807 struct ksym_desc *kdesc;
808
809 kdesc = get_ksym_desc(gen, relo);
810 if (!kdesc)
811 return;
812 /* try to copy from existing ldimm64 insn */
813 if (kdesc->ref > 1) {
814 move_blob2blob(gen, insn + offsetof(struct bpf_insn, imm), 4,
815 kdesc->insn + offsetof(struct bpf_insn, imm));
816 move_blob2blob(gen, insn + sizeof(struct bpf_insn) + offsetof(struct bpf_insn, imm), 4,
817 kdesc->insn + sizeof(struct bpf_insn) + offsetof(struct bpf_insn, imm));
818 goto log;
819 }
820 /* remember insn offset, so we can copy ksym addr later */
821 kdesc->insn = insn;
822 /* skip typeless ksym_desc in fd closing loop in cleanup_relos */
823 kdesc->typeless = true;
824 emit_bpf_kallsyms_lookup_name(gen, relo);
825 emit(gen, BPF_JMP_IMM(BPF_JEQ, BPF_REG_7, -ENOENT, 1));
826 emit_check_err(gen);
827 /* store lower half of addr into insn[insn_idx].imm */
828 emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_8, BPF_REG_9, offsetof(struct bpf_insn, imm)));
829 /* store upper half of addr into insn[insn_idx + 1].imm */
830 emit(gen, BPF_ALU64_IMM(BPF_RSH, BPF_REG_9, 32));
831 emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_8, BPF_REG_9,
832 sizeof(struct bpf_insn) + offsetof(struct bpf_insn, imm)));
833log:
834 emit_ksym_relo_log(gen, relo, kdesc->ref);
835}
836
837static __u32 src_reg_mask(struct bpf_gen *gen)
838{
839#if defined(__LITTLE_ENDIAN_BITFIELD) /* src_reg,dst_reg,... */
840 return gen->swapped_endian ? 0xf0 : 0x0f;
841#elif defined(__BIG_ENDIAN_BITFIELD) /* dst_reg,src_reg,... */
842 return gen->swapped_endian ? 0x0f : 0xf0;
843#else
844#error "Unsupported bit endianness, cannot proceed"
845#endif
846}
847
848/* Expects:
849 * BPF_REG_8 - pointer to instruction
850 */
851static void emit_relo_ksym_btf(struct bpf_gen *gen, struct ksym_relo_desc *relo, int insn)
852{
853 struct ksym_desc *kdesc;
854 __u32 reg_mask;
855
856 kdesc = get_ksym_desc(gen, relo);
857 if (!kdesc)
858 return;
859 /* try to copy from existing ldimm64 insn */
860 if (kdesc->ref > 1) {
861 move_blob2blob(gen, insn + sizeof(struct bpf_insn) + offsetof(struct bpf_insn, imm), 4,
862 kdesc->insn + sizeof(struct bpf_insn) + offsetof(struct bpf_insn, imm));
863 move_blob2blob(gen, insn + offsetof(struct bpf_insn, imm), 4,
864 kdesc->insn + offsetof(struct bpf_insn, imm));
865 /* jump over src_reg adjustment if imm (btf_id) is not 0, reuse BPF_REG_0 from move_blob2blob
866 * If btf_id is zero, clear BPF_PSEUDO_BTF_ID flag in src_reg of ld_imm64 insn
867 */
868 emit(gen, BPF_JMP_IMM(BPF_JNE, BPF_REG_0, 0, 3));
869 goto clear_src_reg;
870 }
871 /* remember insn offset, so we can copy BTF ID and FD later */
872 kdesc->insn = insn;
873 emit_bpf_find_by_name_kind(gen, relo);
874 if (!relo->is_weak)
875 emit_check_err(gen);
876 /* jump to success case */
877 emit(gen, BPF_JMP_IMM(BPF_JSGE, BPF_REG_7, 0, 3));
878 /* set values for insn[insn_idx].imm, insn[insn_idx + 1].imm as 0 */
879 emit(gen, BPF_ST_MEM(BPF_W, BPF_REG_8, offsetof(struct bpf_insn, imm), 0));
880 emit(gen, BPF_ST_MEM(BPF_W, BPF_REG_8, sizeof(struct bpf_insn) + offsetof(struct bpf_insn, imm), 0));
881 /* skip success case for ret < 0 */
882 emit(gen, BPF_JMP_IMM(BPF_JA, 0, 0, 4));
883 /* store btf_id into insn[insn_idx].imm */
884 emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_8, BPF_REG_7, offsetof(struct bpf_insn, imm)));
885 /* store btf_obj_fd into insn[insn_idx + 1].imm */
886 emit(gen, BPF_ALU64_IMM(BPF_RSH, BPF_REG_7, 32));
887 emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_8, BPF_REG_7,
888 sizeof(struct bpf_insn) + offsetof(struct bpf_insn, imm)));
889 /* skip src_reg adjustment */
890 emit(gen, BPF_JMP_IMM(BPF_JA, 0, 0, 3));
891clear_src_reg:
892 /* clear bpf_object__relocate_data's src_reg assignment, otherwise we get a verifier failure */
893 reg_mask = src_reg_mask(gen);
894 emit(gen, BPF_LDX_MEM(BPF_B, BPF_REG_9, BPF_REG_8, offsetofend(struct bpf_insn, code)));
895 emit(gen, BPF_ALU32_IMM(BPF_AND, BPF_REG_9, reg_mask));
896 emit(gen, BPF_STX_MEM(BPF_B, BPF_REG_8, BPF_REG_9, offsetofend(struct bpf_insn, code)));
897
898 emit_ksym_relo_log(gen, relo, kdesc->ref);
899}
900
901void bpf_gen__record_relo_core(struct bpf_gen *gen,
902 const struct bpf_core_relo *core_relo)
903{
904 struct bpf_core_relo *relos;
905
906 relos = libbpf_reallocarray(gen->core_relos, gen->core_relo_cnt + 1, sizeof(*relos));
907 if (!relos) {
908 gen->error = -ENOMEM;
909 return;
910 }
911 gen->core_relos = relos;
912 relos += gen->core_relo_cnt;
913 memcpy(relos, core_relo, sizeof(*relos));
914 gen->core_relo_cnt++;
915}
916
917static void emit_relo(struct bpf_gen *gen, struct ksym_relo_desc *relo, int insns)
918{
919 int insn;
920
921 pr_debug("gen: emit_relo (%d): %s at %d %s\n",
922 relo->kind, relo->name, relo->insn_idx, relo->is_ld64 ? "ld64" : "call");
923 insn = insns + sizeof(struct bpf_insn) * relo->insn_idx;
924 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_8, BPF_PSEUDO_MAP_IDX_VALUE, 0, 0, 0, insn));
925 if (relo->is_ld64) {
926 if (relo->is_typeless)
927 emit_relo_ksym_typeless(gen, relo, insn);
928 else
929 emit_relo_ksym_btf(gen, relo, insn);
930 } else {
931 emit_relo_kfunc_btf(gen, relo, insn);
932 }
933}
934
935static void emit_relos(struct bpf_gen *gen, int insns)
936{
937 int i;
938
939 for (i = 0; i < gen->relo_cnt; i++)
940 emit_relo(gen, gen->relos + i, insns);
941}
942
943static void cleanup_core_relo(struct bpf_gen *gen)
944{
945 if (!gen->core_relo_cnt)
946 return;
947 free(gen->core_relos);
948 gen->core_relo_cnt = 0;
949 gen->core_relos = NULL;
950}
951
952static void cleanup_relos(struct bpf_gen *gen, int insns)
953{
954 struct ksym_desc *kdesc;
955 int i, insn;
956
957 for (i = 0; i < gen->nr_ksyms; i++) {
958 kdesc = &gen->ksyms[i];
959 /* only close fds for typed ksyms and kfuncs */
960 if (kdesc->is_ld64 && !kdesc->typeless) {
961 /* close fd recorded in insn[insn_idx + 1].imm */
962 insn = kdesc->insn;
963 insn += sizeof(struct bpf_insn) + offsetof(struct bpf_insn, imm);
964 emit_sys_close_blob(gen, insn);
965 } else if (!kdesc->is_ld64) {
966 emit_sys_close_blob(gen, blob_fd_array_off(gen, kdesc->off));
967 if (kdesc->off < MAX_FD_ARRAY_SZ)
968 gen->nr_fd_array--;
969 }
970 }
971 if (gen->nr_ksyms) {
972 free(gen->ksyms);
973 gen->nr_ksyms = 0;
974 gen->ksyms = NULL;
975 }
976 if (gen->relo_cnt) {
977 free(gen->relos);
978 gen->relo_cnt = 0;
979 gen->relos = NULL;
980 }
981 cleanup_core_relo(gen);
982}
983
984/* Convert func, line, and core relo info blobs to target endianness */
985static void info_blob_bswap(struct bpf_gen *gen, int func_info, int line_info,
986 int core_relos, struct bpf_prog_load_opts *load_attr)
987{
988 struct bpf_func_info *fi = gen->data_start + func_info;
989 struct bpf_line_info *li = gen->data_start + line_info;
990 struct bpf_core_relo *cr = gen->data_start + core_relos;
991 int i;
992
993 for (i = 0; i < load_attr->func_info_cnt; i++)
994 bpf_func_info_bswap(fi++);
995
996 for (i = 0; i < load_attr->line_info_cnt; i++)
997 bpf_line_info_bswap(li++);
998
999 for (i = 0; i < gen->core_relo_cnt; i++)
1000 bpf_core_relo_bswap(cr++);
1001}
1002
1003void bpf_gen__prog_load(struct bpf_gen *gen,
1004 enum bpf_prog_type prog_type, const char *prog_name,
1005 const char *license, struct bpf_insn *insns, size_t insn_cnt,
1006 struct bpf_prog_load_opts *load_attr, int prog_idx)
1007{
1008 int func_info_tot_sz = load_attr->func_info_cnt *
1009 load_attr->func_info_rec_size;
1010 int line_info_tot_sz = load_attr->line_info_cnt *
1011 load_attr->line_info_rec_size;
1012 int core_relo_tot_sz = gen->core_relo_cnt *
1013 sizeof(struct bpf_core_relo);
1014 int prog_load_attr, license_off, insns_off, func_info, line_info, core_relos;
1015 int attr_size = offsetofend(union bpf_attr, core_relo_rec_size);
1016 union bpf_attr attr;
1017
1018 memset(&attr, 0, attr_size);
1019 /* add license string to blob of bytes */
1020 license_off = add_data(gen, license, strlen(license) + 1);
1021 /* add insns to blob of bytes */
1022 insns_off = add_data(gen, insns, insn_cnt * sizeof(struct bpf_insn));
1023 pr_debug("gen: prog_load: prog_idx %d type %u insn off %d insns_cnt %zu license off %d\n",
1024 prog_idx, prog_type, insns_off, insn_cnt, license_off);
1025
1026 /* convert blob insns to target endianness */
1027 if (gen->swapped_endian && !gen->error) {
1028 struct bpf_insn *insn = gen->data_start + insns_off;
1029 int i;
1030
1031 for (i = 0; i < insn_cnt; i++, insn++)
1032 bpf_insn_bswap(insn);
1033 }
1034
1035 attr.prog_type = tgt_endian(prog_type);
1036 attr.expected_attach_type = tgt_endian(load_attr->expected_attach_type);
1037 attr.attach_btf_id = tgt_endian(load_attr->attach_btf_id);
1038 attr.prog_ifindex = tgt_endian(load_attr->prog_ifindex);
1039 attr.kern_version = 0;
1040 attr.insn_cnt = tgt_endian((__u32)insn_cnt);
1041 attr.prog_flags = tgt_endian(load_attr->prog_flags);
1042
1043 attr.func_info_rec_size = tgt_endian(load_attr->func_info_rec_size);
1044 attr.func_info_cnt = tgt_endian(load_attr->func_info_cnt);
1045 func_info = add_data(gen, load_attr->func_info, func_info_tot_sz);
1046 pr_debug("gen: prog_load: func_info: off %d cnt %u rec size %u\n",
1047 func_info, load_attr->func_info_cnt,
1048 load_attr->func_info_rec_size);
1049
1050 attr.line_info_rec_size = tgt_endian(load_attr->line_info_rec_size);
1051 attr.line_info_cnt = tgt_endian(load_attr->line_info_cnt);
1052 line_info = add_data(gen, load_attr->line_info, line_info_tot_sz);
1053 pr_debug("gen: prog_load: line_info: off %d cnt %u rec size %u\n",
1054 line_info, load_attr->line_info_cnt,
1055 load_attr->line_info_rec_size);
1056
1057 attr.core_relo_rec_size = tgt_endian((__u32)sizeof(struct bpf_core_relo));
1058 attr.core_relo_cnt = tgt_endian(gen->core_relo_cnt);
1059 core_relos = add_data(gen, gen->core_relos, core_relo_tot_sz);
1060 pr_debug("gen: prog_load: core_relos: off %d cnt %d rec size %zu\n",
1061 core_relos, gen->core_relo_cnt,
1062 sizeof(struct bpf_core_relo));
1063
1064 /* convert all info blobs to target endianness */
1065 if (gen->swapped_endian && !gen->error)
1066 info_blob_bswap(gen, func_info, line_info, core_relos, load_attr);
1067
1068 libbpf_strlcpy(attr.prog_name, prog_name, sizeof(attr.prog_name));
1069 prog_load_attr = add_data(gen, &attr, attr_size);
1070 pr_debug("gen: prog_load: attr: off %d size %d\n",
1071 prog_load_attr, attr_size);
1072
1073 /* populate union bpf_attr with a pointer to license */
1074 emit_rel_store(gen, attr_field(prog_load_attr, license), license_off);
1075
1076 /* populate union bpf_attr with a pointer to instructions */
1077 emit_rel_store(gen, attr_field(prog_load_attr, insns), insns_off);
1078
1079 /* populate union bpf_attr with a pointer to func_info */
1080 emit_rel_store(gen, attr_field(prog_load_attr, func_info), func_info);
1081
1082 /* populate union bpf_attr with a pointer to line_info */
1083 emit_rel_store(gen, attr_field(prog_load_attr, line_info), line_info);
1084
1085 /* populate union bpf_attr with a pointer to core_relos */
1086 emit_rel_store(gen, attr_field(prog_load_attr, core_relos), core_relos);
1087
1088 /* populate union bpf_attr fd_array with a pointer to data where map_fds are saved */
1089 emit_rel_store(gen, attr_field(prog_load_attr, fd_array), gen->fd_array);
1090
1091 /* populate union bpf_attr with user provided log details */
1092 move_ctx2blob(gen, attr_field(prog_load_attr, log_level), 4,
1093 offsetof(struct bpf_loader_ctx, log_level), false);
1094 move_ctx2blob(gen, attr_field(prog_load_attr, log_size), 4,
1095 offsetof(struct bpf_loader_ctx, log_size), false);
1096 move_ctx2blob(gen, attr_field(prog_load_attr, log_buf), 8,
1097 offsetof(struct bpf_loader_ctx, log_buf), false);
1098 /* populate union bpf_attr with btf_fd saved in the stack earlier */
1099 move_stack2blob(gen, attr_field(prog_load_attr, prog_btf_fd), 4,
1100 stack_off(btf_fd));
1101 if (gen->attach_kind) {
1102 emit_find_attach_target(gen);
1103 /* populate union bpf_attr with btf_id and btf_obj_fd found by helper */
1104 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_0, BPF_PSEUDO_MAP_IDX_VALUE,
1105 0, 0, 0, prog_load_attr));
1106 emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_0, BPF_REG_7,
1107 offsetof(union bpf_attr, attach_btf_id)));
1108 emit(gen, BPF_ALU64_IMM(BPF_RSH, BPF_REG_7, 32));
1109 emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_0, BPF_REG_7,
1110 offsetof(union bpf_attr, attach_btf_obj_fd)));
1111 }
1112 emit_relos(gen, insns_off);
1113 /* emit PROG_LOAD command */
1114 emit_sys_bpf(gen, BPF_PROG_LOAD, prog_load_attr, attr_size);
1115 debug_ret(gen, "prog_load %s insn_cnt %d", attr.prog_name, attr.insn_cnt);
1116 /* successful or not, close btf module FDs used in extern ksyms and attach_btf_obj_fd */
1117 cleanup_relos(gen, insns_off);
1118 if (gen->attach_kind) {
1119 emit_sys_close_blob(gen,
1120 attr_field(prog_load_attr, attach_btf_obj_fd));
1121 gen->attach_kind = 0;
1122 }
1123 emit_check_err(gen);
1124 /* remember prog_fd in the stack, if successful */
1125 emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_10, BPF_REG_7,
1126 stack_off(prog_fd[gen->nr_progs])));
1127 gen->nr_progs++;
1128}
1129
1130void bpf_gen__map_update_elem(struct bpf_gen *gen, int map_idx, void *pvalue,
1131 __u32 value_size, __u64 flags)
1132{
1133 int attr_size = offsetofend(union bpf_attr, flags);
1134 int map_update_attr, value, key;
1135 union bpf_attr attr;
1136 int zero = 0;
1137
1138 memset(&attr, 0, attr_size);
1139 attr.flags = tgt_endian(flags);
1140
1141 value = add_data(gen, pvalue, value_size);
1142 key = add_data(gen, &zero, sizeof(zero));
1143
1144 /*
1145 * if (map_desc[map_idx].initial_value) {
1146 * if (ctx->flags & BPF_SKEL_KERNEL)
1147 * bpf_probe_read_kernel(value, value_size, initial_value);
1148 * else
1149 * bpf_copy_from_user(value, value_size, initial_value);
1150 * }
1151 *
1152 * The runtime initial_value comes from the host-supplied loader
1153 * ctx and would overwrite the blob value that the program signature
1154 * covers and the kernel verifies at load time. For a signed loader
1155 * (gen_hash) the attested blob value must be authoritative, so skip
1156 * the override and leave the signed value in place.
1157 */
1158 if (!OPTS_GET(gen->opts, gen_hash, false)) {
1159 emit(gen, BPF_LDX_MEM(BPF_DW, BPF_REG_3, BPF_REG_6,
1160 sizeof(struct bpf_loader_ctx) +
1161 sizeof(struct bpf_map_desc) * map_idx +
1162 offsetof(struct bpf_map_desc, initial_value)));
1163 emit(gen, BPF_JMP_IMM(BPF_JEQ, BPF_REG_3, 0, 8));
1164 emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE,
1165 0, 0, 0, value));
1166 emit(gen, BPF_MOV64_IMM(BPF_REG_2, value_size));
1167 emit(gen, BPF_LDX_MEM(BPF_W, BPF_REG_0, BPF_REG_6,
1168 offsetof(struct bpf_loader_ctx, flags)));
1169 emit(gen, BPF_JMP_IMM(BPF_JSET, BPF_REG_0, BPF_SKEL_KERNEL, 2));
1170 emit(gen, BPF_EMIT_CALL(BPF_FUNC_copy_from_user));
1171 emit(gen, BPF_JMP_IMM(BPF_JA, 0, 0, 1));
1172 emit(gen, BPF_EMIT_CALL(BPF_FUNC_probe_read_kernel));
1173 }
1174
1175 map_update_attr = add_data(gen, &attr, attr_size);
1176 pr_debug("gen: map_update_elem: idx %d, value: off %d size %u, attr: off %d size %d\n",
1177 map_idx, value, value_size, map_update_attr, attr_size);
1178 move_blob2blob(gen, attr_field(map_update_attr, map_fd), 4,
1179 blob_fd_array_off(gen, map_idx));
1180 emit_rel_store(gen, attr_field(map_update_attr, key), key);
1181 emit_rel_store(gen, attr_field(map_update_attr, value), value);
1182 /* emit MAP_UPDATE_ELEM command */
1183 emit_sys_bpf(gen, BPF_MAP_UPDATE_ELEM, map_update_attr, attr_size);
1184 debug_ret(gen, "update_elem idx %d value_size %d", map_idx, value_size);
1185 emit_check_err(gen);
1186}
1187
1188void bpf_gen__populate_outer_map(struct bpf_gen *gen, int outer_map_idx, int slot,
1189 int inner_map_idx)
1190{
1191 int attr_size = offsetofend(union bpf_attr, flags);
1192 int map_update_attr, key;
1193 union bpf_attr attr;
1194 int tgt_slot;
1195
1196 memset(&attr, 0, attr_size);
1197
1198 tgt_slot = tgt_endian(slot);
1199 key = add_data(gen, &tgt_slot, sizeof(tgt_slot));
1200
1201 map_update_attr = add_data(gen, &attr, attr_size);
1202 pr_debug("gen: populate_outer_map: outer %d key %d inner %d, attr: off %d size %d\n",
1203 outer_map_idx, slot, inner_map_idx, map_update_attr, attr_size);
1204 move_blob2blob(gen, attr_field(map_update_attr, map_fd), 4,
1205 blob_fd_array_off(gen, outer_map_idx));
1206 emit_rel_store(gen, attr_field(map_update_attr, key), key);
1207 emit_rel_store(gen, attr_field(map_update_attr, value),
1208 blob_fd_array_off(gen, inner_map_idx));
1209
1210 /* emit MAP_UPDATE_ELEM command */
1211 emit_sys_bpf(gen, BPF_MAP_UPDATE_ELEM, map_update_attr, attr_size);
1212 debug_ret(gen, "populate_outer_map outer %d key %d inner %d",
1213 outer_map_idx, slot, inner_map_idx);
1214 emit_check_err(gen);
1215}
1216
1217void bpf_gen__map_freeze(struct bpf_gen *gen, int map_idx)
1218{
1219 int attr_size = offsetofend(union bpf_attr, map_fd);
1220 int map_freeze_attr;
1221 union bpf_attr attr;
1222
1223 memset(&attr, 0, attr_size);
1224 map_freeze_attr = add_data(gen, &attr, attr_size);
1225 pr_debug("gen: map_freeze: idx %d, attr: off %d size %d\n",
1226 map_idx, map_freeze_attr, attr_size);
1227 move_blob2blob(gen, attr_field(map_freeze_attr, map_fd), 4,
1228 blob_fd_array_off(gen, map_idx));
1229 /* emit MAP_FREEZE command */
1230 emit_sys_bpf(gen, BPF_MAP_FREEZE, map_freeze_attr, attr_size);
1231 debug_ret(gen, "map_freeze");
1232 emit_check_err(gen);
1233}