projects/linux·blob master · refs

304 lines · 7.2 KiB

1// SPDX-License-Identifier: GPL-2.0-only
2/* Copyright (c) 2025 Isovalent */
3
4#include <linux/bpf.h>
5
6struct bpf_insn_array {
7 struct bpf_map map;
8 atomic_t used;
9 long *ips;
10 DECLARE_FLEX_ARRAY(struct bpf_insn_array_value, values);
11};
12
13#define cast_insn_array(MAP_PTR) \
14 container_of((MAP_PTR), struct bpf_insn_array, map)
15
16#define INSN_DELETED ((u32)-1)
17
18static inline u64 insn_array_alloc_size(u32 max_entries)
19{
20 const u64 base_size = sizeof(struct bpf_insn_array);
21 const u64 entry_size = sizeof(struct bpf_insn_array_value);
22
23 return base_size + max_entries * (entry_size + sizeof(long));
24}
25
26static int insn_array_alloc_check(union bpf_attr *attr)
27{
28 u32 value_size = sizeof(struct bpf_insn_array_value);
29
30 if (attr->max_entries == 0 || attr->key_size != 4 ||
31 attr->value_size != value_size || attr->map_flags != 0)
32 return -EINVAL;
33
34 return 0;
35}
36
37static void insn_array_free(struct bpf_map *map)
38{
39 struct bpf_insn_array *insn_array = cast_insn_array(map);
40
41 bpf_map_area_free(insn_array);
42}
43
44static struct bpf_map *insn_array_alloc(union bpf_attr *attr)
45{
46 u64 size = insn_array_alloc_size(attr->max_entries);
47 struct bpf_insn_array *insn_array;
48
49 insn_array = bpf_map_area_alloc(size, NUMA_NO_NODE);
50 if (!insn_array)
51 return ERR_PTR(-ENOMEM);
52
53 /* ips are allocated right after the insn_array->values[] array */
54 insn_array->ips = (void *)&insn_array->values[attr->max_entries];
55
56 bpf_map_init_from_attr(&insn_array->map, attr);
57
58 /* BPF programs aren't allowed to write to the map */
59 insn_array->map.map_flags |= BPF_F_RDONLY_PROG;
60
61 return &insn_array->map;
62}
63
64static void *insn_array_lookup_elem(struct bpf_map *map, void *key)
65{
66 struct bpf_insn_array *insn_array = cast_insn_array(map);
67 u32 index = *(u32 *)key;
68
69 if (unlikely(index >= insn_array->map.max_entries))
70 return NULL;
71
72 return &insn_array->values[index];
73}
74
75static long insn_array_update_elem(struct bpf_map *map, void *key, void *value, u64 map_flags)
76{
77 struct bpf_insn_array *insn_array = cast_insn_array(map);
78 u32 index = *(u32 *)key;
79 struct bpf_insn_array_value val = {};
80
81 if (unlikely(index >= insn_array->map.max_entries))
82 return -E2BIG;
83
84 if (unlikely(map_flags & BPF_NOEXIST))
85 return -EEXIST;
86
87 copy_map_value(map, &val, value);
88 if (val.jitted_off || val.xlated_off)
89 return -EINVAL;
90
91 insn_array->values[index].orig_off = val.orig_off;
92
93 return 0;
94}
95
96static long insn_array_delete_elem(struct bpf_map *map, void *key)
97{
98 return -EINVAL;
99}
100
101static int insn_array_check_btf(struct bpf_map *map,
102 const struct btf *btf,
103 const struct btf_type *key_type,
104 const struct btf_type *value_type)
105{
106 if (!btf_type_is_i32(key_type))
107 return -EINVAL;
108
109 if (!btf_type_is_i64(value_type))
110 return -EINVAL;
111
112 return 0;
113}
114
115static u64 insn_array_mem_usage(const struct bpf_map *map)
116{
117 return insn_array_alloc_size(map->max_entries);
118}
119
120static int insn_array_map_direct_value_addr(const struct bpf_map *map, u64 *imm, u32 off)
121{
122 struct bpf_insn_array *insn_array = cast_insn_array(map);
123
124 if ((off % sizeof(long)) != 0 ||
125 (off / sizeof(long)) >= map->max_entries)
126 return -EACCES;
127
128 /* from BPF's point of view, this map is a jump table */
129 *imm = (unsigned long)insn_array->ips;
130
131 return 0;
132}
133
134BTF_ID_LIST_SINGLE(insn_array_btf_ids, struct, bpf_insn_array)
135
136const struct bpf_map_ops insn_array_map_ops = {
137 .map_alloc_check = insn_array_alloc_check,
138 .map_alloc = insn_array_alloc,
139 .map_free = insn_array_free,
140 .map_get_next_key = bpf_array_get_next_key,
141 .map_lookup_elem = insn_array_lookup_elem,
142 .map_update_elem = insn_array_update_elem,
143 .map_delete_elem = insn_array_delete_elem,
144 .map_check_btf = insn_array_check_btf,
145 .map_mem_usage = insn_array_mem_usage,
146 .map_direct_value_addr = insn_array_map_direct_value_addr,
147 .map_btf_id = &insn_array_btf_ids[0],
148};
149
150static inline bool is_frozen(struct bpf_map *map)
151{
152 guard(mutex)(&map->freeze_mutex);
153
154 return map->frozen;
155}
156
157static bool is_insn_array(const struct bpf_map *map)
158{
159 return map->map_type == BPF_MAP_TYPE_INSN_ARRAY;
160}
161
162static inline bool valid_offsets(const struct bpf_insn_array *insn_array,
163 const struct bpf_prog *prog)
164{
165 u32 off;
166 int i;
167
168 for (i = 0; i < insn_array->map.max_entries; i++) {
169 off = insn_array->values[i].orig_off;
170
171 if (off >= prog->len)
172 return false;
173
174 if (off > 0) {
175 if (prog->insnsi[off-1].code == (BPF_LD | BPF_DW | BPF_IMM))
176 return false;
177 }
178 }
179
180 return true;
181}
182
183int bpf_insn_array_init(struct bpf_map *map, const struct bpf_prog *prog)
184{
185 struct bpf_insn_array *insn_array = cast_insn_array(map);
186 struct bpf_insn_array_value *values = insn_array->values;
187 int i;
188
189 if (!is_frozen(map))
190 return -EINVAL;
191
192 if (!valid_offsets(insn_array, prog))
193 return -EINVAL;
194
195 /*
196 * There can be only one program using the map
197 */
198 if (atomic_xchg(&insn_array->used, 1))
199 return -EBUSY;
200
201 /*
202 * Reset all the map indexes to the original values. This is needed,
203 * e.g., when a replay of verification with different log level should
204 * be performed.
205 */
206 for (i = 0; i < map->max_entries; i++)
207 values[i].xlated_off = values[i].orig_off;
208
209 return 0;
210}
211
212int bpf_insn_array_ready(struct bpf_map *map)
213{
214 struct bpf_insn_array *insn_array = cast_insn_array(map);
215 int i;
216
217 for (i = 0; i < map->max_entries; i++) {
218 if (insn_array->values[i].xlated_off == INSN_DELETED)
219 continue;
220 if (!insn_array->ips[i])
221 return -EFAULT;
222 }
223
224 return 0;
225}
226
227void bpf_insn_array_release(struct bpf_map *map)
228{
229 struct bpf_insn_array *insn_array = cast_insn_array(map);
230
231 atomic_set(&insn_array->used, 0);
232}
233
234void bpf_insn_array_adjust(struct bpf_map *map, u32 off, u32 len)
235{
236 struct bpf_insn_array *insn_array = cast_insn_array(map);
237 int i;
238
239 if (len <= 1)
240 return;
241
242 for (i = 0; i < map->max_entries; i++) {
243 if (insn_array->values[i].xlated_off <= off)
244 continue;
245 if (insn_array->values[i].xlated_off == INSN_DELETED)
246 continue;
247 insn_array->values[i].xlated_off += len - 1;
248 }
249}
250
251void bpf_insn_array_adjust_after_remove(struct bpf_map *map, u32 off, u32 len)
252{
253 struct bpf_insn_array *insn_array = cast_insn_array(map);
254 int i;
255
256 for (i = 0; i < map->max_entries; i++) {
257 if (insn_array->values[i].xlated_off < off)
258 continue;
259 if (insn_array->values[i].xlated_off == INSN_DELETED)
260 continue;
261 if (insn_array->values[i].xlated_off < off + len)
262 insn_array->values[i].xlated_off = INSN_DELETED;
263 else
264 insn_array->values[i].xlated_off -= len;
265 }
266}
267
268/*
269 * This function is called by JITs. The image is the real program
270 * image, the offsets array set up the xlated -> jitted mapping.
271 * The offsets[xlated] offset should point to the beginning of
272 * the jitted instruction.
273 */
274void bpf_prog_update_insn_ptrs(struct bpf_prog *prog, u32 *offsets, void *image)
275{
276 struct bpf_insn_array *insn_array;
277 struct bpf_map *map;
278 u32 xlated_off;
279 int i, j;
280
281 if (!offsets || !image)
282 return;
283
284 for (i = 0; i < prog->aux->used_map_cnt; i++) {
285 map = prog->aux->used_maps[i];
286 if (!is_insn_array(map))
287 continue;
288
289 insn_array = cast_insn_array(map);
290 for (j = 0; j < map->max_entries; j++) {
291 xlated_off = insn_array->values[j].xlated_off;
292 if (xlated_off == INSN_DELETED)
293 continue;
294 if (xlated_off < prog->aux->subprog_start)
295 continue;
296 xlated_off -= prog->aux->subprog_start;
297 if (xlated_off >= prog->len)
298 continue;
299
300 insn_array->values[j].jitted_off = offsets[xlated_off];
301 insn_array->ips[j] = (long)(image + offsets[xlated_off]);
302 }
303 }
304}