| 1 | |
| 2 | |
| 3 | * Copyright (c) 2016 Facebook
|
| 4 |
|
| 5 | #include <linux/bpf.h>
|
| 6 | #include <linux/btf.h>
|
| 7 | #include <linux/jhash.h>
|
| 8 | #include <linux/filter.h>
|
| 9 | #include <linux/rculist_nulls.h>
|
| 10 | #include <linux/rcupdate_wait.h>
|
| 11 | #include <linux/random.h>
|
| 12 | #include <linux/rhashtable.h>
|
| 13 | #include <uapi/linux/btf.h>
|
| 14 | #include <linux/rcupdate_trace.h>
|
| 15 | #include <linux/btf_ids.h>
|
| 16 | #include "percpu_freelist.h"
|
| 17 | #include "bpf_lru_list.h"
|
| 18 | #include "map_in_map.h"
|
| 19 | #include <linux/bpf_mem_alloc.h>
|
| 20 | #include <asm/rqspinlock.h>
|
| 21 |
|
| 22 | #define HTAB_CREATE_FLAG_MASK \
|
| 23 | (BPF_F_NO_PREALLOC | BPF_F_NO_COMMON_LRU | BPF_F_NUMA_NODE | \
|
| 24 | BPF_F_ACCESS_MASK | BPF_F_ZERO_SEED)
|
| 25 |
|
| 26 | #define BATCH_OPS(_name) \
|
| 27 | .map_lookup_batch = \
|
| 28 | _name##_map_lookup_batch, \
|
| 29 | .map_lookup_and_delete_batch = \
|
| 30 | _name##_map_lookup_and_delete_batch, \
|
| 31 | .map_update_batch = \
|
| 32 | generic_map_update_batch, \
|
| 33 | .map_delete_batch = \
|
| 34 | generic_map_delete_batch
|
| 35 |
|
| 36 | |
| 37 | * The bucket lock has two protection scopes:
|
| 38 | *
|
| 39 | * 1) Serializing concurrent operations from BPF programs on different
|
| 40 | * CPUs
|
| 41 | *
|
| 42 | * 2) Serializing concurrent operations from BPF programs and sys_bpf()
|
| 43 | *
|
| 44 | * BPF programs can execute in any context including perf, kprobes and
|
| 45 | * tracing. As there are almost no limits where perf, kprobes and tracing
|
| 46 | * can be invoked from the lock operations need to be protected against
|
| 47 | * deadlocks. Deadlocks can be caused by recursion and by an invocation in
|
| 48 | * the lock held section when functions which acquire this lock are invoked
|
| 49 | * from sys_bpf(). BPF recursion is prevented by incrementing the per CPU
|
| 50 | * variable bpf_prog_active, which prevents BPF programs attached to perf
|
| 51 | * events, kprobes and tracing to be invoked before the prior invocation
|
| 52 | * from one of these contexts completed. sys_bpf() uses the same mechanism
|
| 53 | * by pinning the task to the current CPU and incrementing the recursion
|
| 54 | * protection across the map operation.
|
| 55 | *
|
| 56 | * This has subtle implications on PREEMPT_RT. PREEMPT_RT forbids certain
|
| 57 | * operations like memory allocations (even with GFP_ATOMIC) from atomic
|
| 58 | * contexts. This is required because even with GFP_ATOMIC the memory
|
| 59 | * allocator calls into code paths which acquire locks with long held lock
|
| 60 | * sections. To ensure the deterministic behaviour these locks are regular
|
| 61 | * spinlocks, which are converted to 'sleepable' spinlocks on RT. The only
|
| 62 | * true atomic contexts on an RT kernel are the low level hardware
|
| 63 | * handling, scheduling, low level interrupt handling, NMIs etc. None of
|
| 64 | * these contexts should ever do memory allocations.
|
| 65 | *
|
| 66 | * As regular device interrupt handlers and soft interrupts are forced into
|
| 67 | * thread context, the existing code which does
|
| 68 | * spin_lock*(); alloc(GFP_ATOMIC); spin_unlock*();
|
| 69 | * just works.
|
| 70 | *
|
| 71 | * In theory the BPF locks could be converted to regular spinlocks as well,
|
| 72 | * but the bucket locks and percpu_freelist locks can be taken from
|
| 73 | * arbitrary contexts (perf, kprobes, tracepoints) which are required to be
|
| 74 | * atomic contexts even on RT. Before the introduction of bpf_mem_alloc,
|
| 75 | * it is only safe to use raw spinlock for preallocated hash map on a RT kernel,
|
| 76 | * because there is no memory allocation within the lock held sections. However
|
| 77 | * after hash map was fully converted to use bpf_mem_alloc, there will be
|
| 78 | * non-synchronous memory allocation for non-preallocated hash map, so it is
|
| 79 | * safe to always use raw spinlock for bucket lock.
|
| 80 |
|
| 81 | struct bucket {
|
| 82 | struct hlist_nulls_head head;
|
| 83 | rqspinlock_t raw_lock;
|
| 84 | };
|
| 85 |
|
| 86 | struct bpf_htab {
|
| 87 | struct bpf_map map;
|
| 88 | struct bpf_mem_alloc ma;
|
| 89 | struct bpf_mem_alloc pcpu_ma;
|
| 90 | struct bucket *buckets;
|
| 91 | void *elems;
|
| 92 | union {
|
| 93 | struct pcpu_freelist freelist;
|
| 94 | struct bpf_lru lru;
|
| 95 | };
|
| 96 | struct htab_elem *__percpu *extra_elems;
|
| 97 | |
| 98 | * in either pcount or count
|
| 99 |
|
| 100 | struct percpu_counter pcount;
|
| 101 | atomic_t count;
|
| 102 | bool use_percpu_counter;
|
| 103 | u32 n_buckets;
|
| 104 | u32 elem_size;
|
| 105 | u32 hashrnd;
|
| 106 | };
|
| 107 |
|
| 108 |
|
| 109 | struct htab_elem {
|
| 110 | union {
|
| 111 | struct hlist_nulls_node hash_node;
|
| 112 | struct {
|
| 113 | void *padding;
|
| 114 | union {
|
| 115 | struct pcpu_freelist_node fnode;
|
| 116 | struct htab_elem *batch_flink;
|
| 117 | };
|
| 118 | };
|
| 119 | };
|
| 120 | union {
|
| 121 |
|
| 122 | void *ptr_to_pptr;
|
| 123 | struct bpf_lru_node lru_node;
|
| 124 | };
|
| 125 | u32 hash;
|
| 126 | char key[] __aligned(8);
|
| 127 | };
|
| 128 |
|
| 129 | struct htab_btf_record {
|
| 130 | struct btf_record *record;
|
| 131 | struct btf *btf;
|
| 132 | u32 key_size;
|
| 133 | };
|
| 134 |
|
| 135 | static inline bool htab_is_prealloc(const struct bpf_htab *htab)
|
| 136 | {
|
| 137 | return !(htab->map.map_flags & BPF_F_NO_PREALLOC);
|
| 138 | }
|
| 139 |
|
| 140 | static void htab_init_buckets(struct bpf_htab *htab)
|
| 141 | {
|
| 142 | unsigned int i;
|
| 143 |
|
| 144 | for (i = 0; i < htab->n_buckets; i++) {
|
| 145 | INIT_HLIST_NULLS_HEAD(&htab->buckets[i].head, i);
|
| 146 | raw_res_spin_lock_init(&htab->buckets[i].raw_lock);
|
| 147 | cond_resched();
|
| 148 | }
|
| 149 | }
|
| 150 |
|
| 151 | static inline int htab_lock_bucket(struct bucket *b, unsigned long *pflags)
|
| 152 | {
|
| 153 | unsigned long flags;
|
| 154 | int ret;
|
| 155 |
|
| 156 | ret = raw_res_spin_lock_irqsave(&b->raw_lock, flags);
|
| 157 | if (ret)
|
| 158 | return ret;
|
| 159 | *pflags = flags;
|
| 160 | return 0;
|
| 161 | }
|
| 162 |
|
| 163 | static inline void htab_unlock_bucket(struct bucket *b, unsigned long flags)
|
| 164 | {
|
| 165 | raw_res_spin_unlock_irqrestore(&b->raw_lock, flags);
|
| 166 | }
|
| 167 |
|
| 168 | static bool htab_lru_map_delete_node(void *arg, struct bpf_lru_node *node);
|
| 169 |
|
| 170 | static bool htab_is_lru(const struct bpf_htab *htab)
|
| 171 | {
|
| 172 | return htab->map.map_type == BPF_MAP_TYPE_LRU_HASH ||
|
| 173 | htab->map.map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH;
|
| 174 | }
|
| 175 |
|
| 176 | static bool htab_is_percpu(const struct bpf_htab *htab)
|
| 177 | {
|
| 178 | return htab->map.map_type == BPF_MAP_TYPE_PERCPU_HASH ||
|
| 179 | htab->map.map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH;
|
| 180 | }
|
| 181 |
|
| 182 | static inline bool is_fd_htab(const struct bpf_htab *htab)
|
| 183 | {
|
| 184 | return htab->map.map_type == BPF_MAP_TYPE_HASH_OF_MAPS;
|
| 185 | }
|
| 186 |
|
| 187 | static inline void *htab_elem_value(struct htab_elem *l, u32 key_size)
|
| 188 | {
|
| 189 | return l->key + round_up(key_size, 8);
|
| 190 | }
|
| 191 |
|
| 192 | static inline void htab_elem_set_ptr(struct htab_elem *l, u32 key_size,
|
| 193 | void __percpu *pptr)
|
| 194 | {
|
| 195 | *(void __percpu **)htab_elem_value(l, key_size) = pptr;
|
| 196 | }
|
| 197 |
|
| 198 | static inline void __percpu *htab_elem_get_ptr(struct htab_elem *l, u32 key_size)
|
| 199 | {
|
| 200 | return *(void __percpu **)htab_elem_value(l, key_size);
|
| 201 | }
|
| 202 |
|
| 203 | static void *fd_htab_map_get_ptr(const struct bpf_map *map, struct htab_elem *l)
|
| 204 | {
|
| 205 | return *(void **)htab_elem_value(l, map->key_size);
|
| 206 | }
|
| 207 |
|
| 208 | static struct htab_elem *get_htab_elem(struct bpf_htab *htab, int i)
|
| 209 | {
|
| 210 | return (struct htab_elem *) (htab->elems + i * (u64)htab->elem_size);
|
| 211 | }
|
| 212 |
|
| 213 | |
| 214 | * extra elem. LRU itself can remove the least used element, so
|
| 215 | * there is no need for an extra elem during map_update.
|
| 216 |
|
| 217 | static bool htab_has_extra_elems(struct bpf_htab *htab)
|
| 218 | {
|
| 219 | return !htab_is_percpu(htab) && !htab_is_lru(htab) && !is_fd_htab(htab);
|
| 220 | }
|
| 221 |
|
| 222 | static void htab_free_prealloced_internal_structs(struct bpf_htab *htab)
|
| 223 | {
|
| 224 | u32 num_entries = htab->map.max_entries;
|
| 225 | int i;
|
| 226 |
|
| 227 | if (htab_has_extra_elems(htab))
|
| 228 | num_entries += num_possible_cpus();
|
| 229 |
|
| 230 | for (i = 0; i < num_entries; i++) {
|
| 231 | struct htab_elem *elem;
|
| 232 |
|
| 233 | elem = get_htab_elem(htab, i);
|
| 234 | bpf_map_free_internal_structs(&htab->map,
|
| 235 | htab_elem_value(elem, htab->map.key_size));
|
| 236 | cond_resched();
|
| 237 | }
|
| 238 | }
|
| 239 |
|
| 240 | static void htab_free_prealloced_fields(struct bpf_htab *htab)
|
| 241 | {
|
| 242 | u32 num_entries = htab->map.max_entries;
|
| 243 | int i;
|
| 244 |
|
| 245 | if (IS_ERR_OR_NULL(htab->map.record))
|
| 246 | return;
|
| 247 | |
| 248 | * Preallocated maps do not have a bpf_mem_alloc destructor, so fully
|
| 249 | * destroy every element, including the extra elements.
|
| 250 |
|
| 251 | if (htab_has_extra_elems(htab))
|
| 252 | num_entries += num_possible_cpus();
|
| 253 | for (i = 0; i < num_entries; i++) {
|
| 254 | struct htab_elem *elem;
|
| 255 |
|
| 256 | elem = get_htab_elem(htab, i);
|
| 257 | if (htab_is_percpu(htab)) {
|
| 258 | void __percpu *pptr = htab_elem_get_ptr(elem, htab->map.key_size);
|
| 259 | int cpu;
|
| 260 |
|
| 261 | for_each_possible_cpu(cpu) {
|
| 262 | bpf_obj_free_fields(htab->map.record, per_cpu_ptr(pptr, cpu));
|
| 263 | cond_resched();
|
| 264 | }
|
| 265 | } else {
|
| 266 | bpf_obj_free_fields(htab->map.record,
|
| 267 | htab_elem_value(elem, htab->map.key_size));
|
| 268 | cond_resched();
|
| 269 | }
|
| 270 | cond_resched();
|
| 271 | }
|
| 272 | }
|
| 273 |
|
| 274 | static void htab_free_elems(struct bpf_htab *htab)
|
| 275 | {
|
| 276 | int i;
|
| 277 |
|
| 278 | if (!htab_is_percpu(htab))
|
| 279 | goto free_elems;
|
| 280 |
|
| 281 | for (i = 0; i < htab->map.max_entries; i++) {
|
| 282 | void __percpu *pptr;
|
| 283 |
|
| 284 | pptr = htab_elem_get_ptr(get_htab_elem(htab, i),
|
| 285 | htab->map.key_size);
|
| 286 | free_percpu(pptr);
|
| 287 | cond_resched();
|
| 288 | }
|
| 289 | free_elems:
|
| 290 | bpf_map_area_free(htab->elems);
|
| 291 | }
|
| 292 |
|
| 293 | |
| 294 | * (bucket_lock). If both locks need to be acquired together, the lock
|
| 295 | * order is always lru_lock -> bucket_lock and this only happens in
|
| 296 | * bpf_lru_list.c logic. For example, certain code path of
|
| 297 | * bpf_lru_pop_free(), which is called by function prealloc_lru_pop(),
|
| 298 | * will acquire lru_lock first followed by acquiring bucket_lock.
|
| 299 | *
|
| 300 | * In hashtab.c, to avoid deadlock, lock acquisition of
|
| 301 | * bucket_lock followed by lru_lock is not allowed. In such cases,
|
| 302 | * bucket_lock needs to be released first before acquiring lru_lock.
|
| 303 |
|
| 304 | static struct htab_elem *prealloc_lru_pop(struct bpf_htab *htab, void *key,
|
| 305 | u32 hash)
|
| 306 | {
|
| 307 | struct bpf_lru_node *node = bpf_lru_pop_free(&htab->lru, hash);
|
| 308 | struct htab_elem *l;
|
| 309 |
|
| 310 | if (node) {
|
| 311 | bpf_map_inc_elem_count(&htab->map);
|
| 312 | l = container_of(node, struct htab_elem, lru_node);
|
| 313 | memcpy(l->key, key, htab->map.key_size);
|
| 314 | return l;
|
| 315 | }
|
| 316 |
|
| 317 | return NULL;
|
| 318 | }
|
| 319 |
|
| 320 | static int prealloc_init(struct bpf_htab *htab)
|
| 321 | {
|
| 322 | u32 num_entries = htab->map.max_entries;
|
| 323 | int err = -ENOMEM, i;
|
| 324 |
|
| 325 | if (htab_has_extra_elems(htab))
|
| 326 | num_entries += num_possible_cpus();
|
| 327 |
|
| 328 | htab->elems = bpf_map_area_alloc((u64)htab->elem_size * num_entries,
|
| 329 | htab->map.numa_node);
|
| 330 | if (!htab->elems)
|
| 331 | return -ENOMEM;
|
| 332 |
|
| 333 | if (!htab_is_percpu(htab))
|
| 334 | goto skip_percpu_elems;
|
| 335 |
|
| 336 | for (i = 0; i < num_entries; i++) {
|
| 337 | u32 size = round_up(htab->map.value_size, 8);
|
| 338 | void __percpu *pptr;
|
| 339 |
|
| 340 | pptr = bpf_map_alloc_percpu(&htab->map, size, 8,
|
| 341 | GFP_USER | __GFP_NOWARN);
|
| 342 | if (!pptr)
|
| 343 | goto free_elems;
|
| 344 | htab_elem_set_ptr(get_htab_elem(htab, i), htab->map.key_size,
|
| 345 | pptr);
|
| 346 | cond_resched();
|
| 347 | }
|
| 348 |
|
| 349 | skip_percpu_elems:
|
| 350 | if (htab_is_lru(htab))
|
| 351 | err = bpf_lru_init(&htab->lru,
|
| 352 | htab->map.map_flags & BPF_F_NO_COMMON_LRU,
|
| 353 | offsetof(struct htab_elem, hash) -
|
| 354 | offsetof(struct htab_elem, lru_node),
|
| 355 | htab_lru_map_delete_node,
|
| 356 | htab);
|
| 357 | else
|
| 358 | err = pcpu_freelist_init(&htab->freelist);
|
| 359 |
|
| 360 | if (err)
|
| 361 | goto free_elems;
|
| 362 |
|
| 363 | if (htab_is_lru(htab))
|
| 364 | bpf_lru_populate(&htab->lru, htab->elems,
|
| 365 | offsetof(struct htab_elem, lru_node),
|
| 366 | htab->elem_size, num_entries);
|
| 367 | else
|
| 368 | pcpu_freelist_populate(&htab->freelist,
|
| 369 | htab->elems + offsetof(struct htab_elem, fnode),
|
| 370 | htab->elem_size, num_entries);
|
| 371 |
|
| 372 | return 0;
|
| 373 |
|
| 374 | free_elems:
|
| 375 | htab_free_elems(htab);
|
| 376 | return err;
|
| 377 | }
|
| 378 |
|
| 379 | static void prealloc_destroy(struct bpf_htab *htab)
|
| 380 | {
|
| 381 | htab_free_elems(htab);
|
| 382 |
|
| 383 | if (htab_is_lru(htab))
|
| 384 | bpf_lru_destroy(&htab->lru);
|
| 385 | else
|
| 386 | pcpu_freelist_destroy(&htab->freelist);
|
| 387 | }
|
| 388 |
|
| 389 | static int alloc_extra_elems(struct bpf_htab *htab)
|
| 390 | {
|
| 391 | struct htab_elem *__percpu *pptr, *l_new;
|
| 392 | struct pcpu_freelist_node *l;
|
| 393 | int cpu;
|
| 394 |
|
| 395 | pptr = bpf_map_alloc_percpu(&htab->map, sizeof(struct htab_elem *), 8,
|
| 396 | GFP_USER | __GFP_NOWARN);
|
| 397 | if (!pptr)
|
| 398 | return -ENOMEM;
|
| 399 |
|
| 400 | for_each_possible_cpu(cpu) {
|
| 401 | l = pcpu_freelist_pop(&htab->freelist);
|
| 402 | |
| 403 | * preallocated extra num_possible_cpus elements
|
| 404 |
|
| 405 | l_new = container_of(l, struct htab_elem, fnode);
|
| 406 | *per_cpu_ptr(pptr, cpu) = l_new;
|
| 407 | }
|
| 408 | htab->extra_elems = pptr;
|
| 409 | return 0;
|
| 410 | }
|
| 411 |
|
| 412 |
|
| 413 | static int htab_map_alloc_check(union bpf_attr *attr)
|
| 414 | {
|
| 415 | bool percpu = (attr->map_type == BPF_MAP_TYPE_PERCPU_HASH ||
|
| 416 | attr->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH);
|
| 417 | bool lru = (attr->map_type == BPF_MAP_TYPE_LRU_HASH ||
|
| 418 | attr->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH);
|
| 419 | |
| 420 | * it is different from BPF_MAP_TYPE_PERCPU_HASH where
|
| 421 | * the map's value itself is percpu. percpu_lru has
|
| 422 | * nothing to do with the map's value.
|
| 423 |
|
| 424 | bool percpu_lru = (attr->map_flags & BPF_F_NO_COMMON_LRU);
|
| 425 | bool prealloc = !(attr->map_flags & BPF_F_NO_PREALLOC);
|
| 426 | bool zero_seed = (attr->map_flags & BPF_F_ZERO_SEED);
|
| 427 | int numa_node = bpf_map_attr_numa_node(attr);
|
| 428 |
|
| 429 | BUILD_BUG_ON(offsetof(struct htab_elem, fnode.next) !=
|
| 430 | offsetof(struct htab_elem, hash_node.pprev));
|
| 431 |
|
| 432 | if (zero_seed && !capable(CAP_SYS_ADMIN))
|
| 433 |
|
| 434 | return -EPERM;
|
| 435 |
|
| 436 | if (attr->map_flags & ~HTAB_CREATE_FLAG_MASK ||
|
| 437 | !bpf_map_flags_access_ok(attr->map_flags))
|
| 438 | return -EINVAL;
|
| 439 |
|
| 440 | if (!lru && percpu_lru)
|
| 441 | return -EINVAL;
|
| 442 |
|
| 443 | if (lru && !prealloc)
|
| 444 | return -ENOTSUPP;
|
| 445 |
|
| 446 | if (numa_node != NUMA_NO_NODE && (percpu || percpu_lru))
|
| 447 | return -EINVAL;
|
| 448 |
|
| 449 | |
| 450 | * value_size == 0 may be allowed in the future to use map as a set
|
| 451 |
|
| 452 | if (attr->max_entries == 0 || attr->key_size == 0 ||
|
| 453 | attr->value_size == 0)
|
| 454 | return -EINVAL;
|
| 455 |
|
| 456 | if ((u64)attr->key_size + attr->value_size >= KMALLOC_MAX_SIZE -
|
| 457 | sizeof(struct htab_elem))
|
| 458 | |
| 459 | * able to access the elements via bpf syscall. This check
|
| 460 | * also makes sure that the elem_size doesn't overflow and it's
|
| 461 | * kmalloc-able later in htab_map_update_elem()
|
| 462 |
|
| 463 | return -E2BIG;
|
| 464 |
|
| 465 | if (percpu && round_up(attr->value_size, 8) > PCPU_MIN_UNIT_SIZE)
|
| 466 | return -E2BIG;
|
| 467 |
|
| 468 | return 0;
|
| 469 | }
|
| 470 |
|
| 471 | static void htab_mem_dtor(void *obj, void *ctx)
|
| 472 | {
|
| 473 | struct htab_btf_record *hrec = ctx;
|
| 474 | struct htab_elem *elem = obj;
|
| 475 | void *map_value;
|
| 476 |
|
| 477 | if (IS_ERR_OR_NULL(hrec->record))
|
| 478 | return;
|
| 479 |
|
| 480 | map_value = htab_elem_value(elem, hrec->key_size);
|
| 481 | bpf_obj_free_fields(hrec->record, map_value);
|
| 482 | }
|
| 483 |
|
| 484 | static void htab_pcpu_mem_dtor(void *obj, void *ctx)
|
| 485 | {
|
| 486 | void __percpu *pptr = *(void __percpu **)obj;
|
| 487 | struct htab_btf_record *hrec = ctx;
|
| 488 | int cpu;
|
| 489 |
|
| 490 | if (IS_ERR_OR_NULL(hrec->record))
|
| 491 | return;
|
| 492 |
|
| 493 | for_each_possible_cpu(cpu)
|
| 494 | bpf_obj_free_fields(hrec->record, per_cpu_ptr(pptr, cpu));
|
| 495 | }
|
| 496 |
|
| 497 | static void htab_dtor_ctx_free(void *ctx)
|
| 498 | {
|
| 499 | struct htab_btf_record *hrec = ctx;
|
| 500 |
|
| 501 | |
| 502 | * The duplicated record still points into the map BTF, so free it
|
| 503 | * before dropping the reference that keeps that BTF alive.
|
| 504 |
|
| 505 | btf_record_free(hrec->record);
|
| 506 | btf_put(hrec->btf);
|
| 507 | kfree(hrec);
|
| 508 | }
|
| 509 |
|
| 510 | static int bpf_ma_set_dtor(struct bpf_map *map, struct bpf_mem_alloc *ma,
|
| 511 | void (*dtor)(void *, void *))
|
| 512 | {
|
| 513 | struct htab_btf_record *hrec;
|
| 514 | int err;
|
| 515 |
|
| 516 |
|
| 517 | if (IS_ERR_OR_NULL(map->record))
|
| 518 | return 0;
|
| 519 |
|
| 520 | hrec = kzalloc_obj(*hrec);
|
| 521 | if (!hrec)
|
| 522 | return -ENOMEM;
|
| 523 | hrec->key_size = map->key_size;
|
| 524 | hrec->record = btf_record_dup(map->record);
|
| 525 | if (IS_ERR(hrec->record)) {
|
| 526 | err = PTR_ERR(hrec->record);
|
| 527 | kfree(hrec);
|
| 528 | return err;
|
| 529 | }
|
| 530 | |
| 531 | * btf_record_dup() only acquires kernel and module BTF. Fields whose
|
| 532 | * types live in the map BTF keep pointing into it: kptrs to local
|
| 533 | * types refer to map->btf, and graph roots carry a value record owned
|
| 534 | * by its struct meta table. The context can outlive the map when the
|
| 535 | * allocator defers its teardown, so hold a reference of our own.
|
| 536 |
|
| 537 | hrec->btf = map->btf;
|
| 538 | btf_get(hrec->btf);
|
| 539 | bpf_mem_alloc_set_dtor(ma, dtor, htab_dtor_ctx_free, hrec);
|
| 540 | return 0;
|
| 541 | }
|
| 542 |
|
| 543 | static int htab_map_check_btf(struct bpf_map *map, const struct btf *btf,
|
| 544 | const struct btf_type *key_type, const struct btf_type *value_type)
|
| 545 | {
|
| 546 | struct bpf_htab *htab = container_of(map, struct bpf_htab, map);
|
| 547 |
|
| 548 | if (btf_type_is_void(key_type))
|
| 549 | return -EINVAL;
|
| 550 |
|
| 551 | if (htab_is_prealloc(htab))
|
| 552 | return 0;
|
| 553 | |
| 554 | * We must set the dtor using this callback, as map's BTF record is not
|
| 555 | * populated in htab_map_alloc(), so it will always appear as NULL.
|
| 556 |
|
| 557 | if (htab_is_percpu(htab))
|
| 558 | return bpf_ma_set_dtor(map, &htab->pcpu_ma, htab_pcpu_mem_dtor);
|
| 559 | else
|
| 560 | return bpf_ma_set_dtor(map, &htab->ma, htab_mem_dtor);
|
| 561 | }
|
| 562 |
|
| 563 | static struct bpf_map *htab_map_alloc(union bpf_attr *attr)
|
| 564 | {
|
| 565 | bool percpu = (attr->map_type == BPF_MAP_TYPE_PERCPU_HASH ||
|
| 566 | attr->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH);
|
| 567 | |
| 568 | * it is different from BPF_MAP_TYPE_PERCPU_HASH where
|
| 569 | * the map's value itself is percpu. percpu_lru has
|
| 570 | * nothing to do with the map's value.
|
| 571 |
|
| 572 | bool percpu_lru = (attr->map_flags & BPF_F_NO_COMMON_LRU);
|
| 573 | bool prealloc = !(attr->map_flags & BPF_F_NO_PREALLOC);
|
| 574 | struct bpf_htab *htab;
|
| 575 | int err;
|
| 576 |
|
| 577 | htab = bpf_map_area_alloc(sizeof(*htab), NUMA_NO_NODE);
|
| 578 | if (!htab)
|
| 579 | return ERR_PTR(-ENOMEM);
|
| 580 |
|
| 581 | bpf_map_init_from_attr(&htab->map, attr);
|
| 582 |
|
| 583 | if (percpu_lru) {
|
| 584 | |
| 585 | * since we are at it, make each lru list has the same
|
| 586 | * number of elements.
|
| 587 |
|
| 588 | htab->map.max_entries = roundup(attr->max_entries,
|
| 589 | num_possible_cpus());
|
| 590 | if (htab->map.max_entries < attr->max_entries)
|
| 591 | htab->map.max_entries = rounddown(attr->max_entries,
|
| 592 | num_possible_cpus());
|
| 593 | }
|
| 594 |
|
| 595 | |
| 596 | * into UB on 32-bit arches, so check that first
|
| 597 |
|
| 598 | err = -E2BIG;
|
| 599 | if (htab->map.max_entries > 1UL << 31)
|
| 600 | goto free_htab;
|
| 601 |
|
| 602 | htab->n_buckets = roundup_pow_of_two(htab->map.max_entries);
|
| 603 |
|
| 604 | htab->elem_size = sizeof(struct htab_elem) +
|
| 605 | round_up(htab->map.key_size, 8);
|
| 606 | if (percpu)
|
| 607 | htab->elem_size += sizeof(void *);
|
| 608 | else
|
| 609 | htab->elem_size += round_up(htab->map.value_size, 8);
|
| 610 |
|
| 611 |
|
| 612 | if (htab->n_buckets > U32_MAX / sizeof(struct bucket))
|
| 613 | goto free_htab;
|
| 614 |
|
| 615 | err = bpf_map_init_elem_count(&htab->map);
|
| 616 | if (err)
|
| 617 | goto free_htab;
|
| 618 |
|
| 619 | err = -ENOMEM;
|
| 620 | htab->buckets = bpf_map_area_alloc(htab->n_buckets *
|
| 621 | sizeof(struct bucket),
|
| 622 | htab->map.numa_node);
|
| 623 | if (!htab->buckets)
|
| 624 | goto free_elem_count;
|
| 625 |
|
| 626 | if (htab->map.map_flags & BPF_F_ZERO_SEED)
|
| 627 | htab->hashrnd = 0;
|
| 628 | else
|
| 629 | htab->hashrnd = get_random_u32();
|
| 630 |
|
| 631 | htab_init_buckets(htab);
|
| 632 |
|
| 633 | |
| 634 | * and __percpu_counter_compare() needs
|
| 635 | * htab->max_entries - cur_number_of_elems to be more than batch * num_online_cpus()
|
| 636 | * for percpu_counter to be faster than atomic_t. In practice the average bpf
|
| 637 | * hash map size is 10k, which means that a system with 64 cpus will fill
|
| 638 | * hashmap to 20% of 10k before percpu_counter becomes ineffective. Therefore
|
| 639 | * define our own batch count as 32 then 10k hash map can be filled up to 80%:
|
| 640 | * 10k - 8k > 32 _batch_ * 64 _cpus_
|
| 641 | * and __percpu_counter_compare() will still be fast. At that point hash map
|
| 642 | * collisions will dominate its performance anyway. Assume that hash map filled
|
| 643 | * to 50+% isn't going to be O(1) and use the following formula to choose
|
| 644 | * between percpu_counter and atomic_t.
|
| 645 |
|
| 646 | #define PERCPU_COUNTER_BATCH 32
|
| 647 | if (attr->max_entries / 2 > num_online_cpus() * PERCPU_COUNTER_BATCH)
|
| 648 | htab->use_percpu_counter = true;
|
| 649 |
|
| 650 | if (htab->use_percpu_counter) {
|
| 651 | err = percpu_counter_init(&htab->pcount, 0, GFP_KERNEL);
|
| 652 | if (err)
|
| 653 | goto free_map_locked;
|
| 654 | }
|
| 655 |
|
| 656 | if (prealloc) {
|
| 657 | err = prealloc_init(htab);
|
| 658 | if (err)
|
| 659 | goto free_map_locked;
|
| 660 |
|
| 661 | if (htab_has_extra_elems(htab)) {
|
| 662 | err = alloc_extra_elems(htab);
|
| 663 | if (err)
|
| 664 | goto free_prealloc;
|
| 665 | }
|
| 666 | } else {
|
| 667 | err = bpf_mem_alloc_init(&htab->ma, htab->elem_size, false);
|
| 668 | if (err)
|
| 669 | goto free_map_locked;
|
| 670 | if (percpu) {
|
| 671 | err = bpf_mem_alloc_init(&htab->pcpu_ma,
|
| 672 | round_up(htab->map.value_size, 8), true);
|
| 673 | if (err)
|
| 674 | goto free_map_locked;
|
| 675 | }
|
| 676 | }
|
| 677 |
|
| 678 | return &htab->map;
|
| 679 |
|
| 680 | free_prealloc:
|
| 681 | prealloc_destroy(htab);
|
| 682 | free_map_locked:
|
| 683 | if (htab->use_percpu_counter)
|
| 684 | percpu_counter_destroy(&htab->pcount);
|
| 685 | bpf_map_area_free(htab->buckets);
|
| 686 | bpf_mem_alloc_destroy(&htab->pcpu_ma);
|
| 687 | bpf_mem_alloc_destroy(&htab->ma);
|
| 688 | free_elem_count:
|
| 689 | bpf_map_free_elem_count(&htab->map);
|
| 690 | free_htab:
|
| 691 | bpf_map_area_free(htab);
|
| 692 | return ERR_PTR(err);
|
| 693 | }
|
| 694 |
|
| 695 | static inline u32 htab_map_hash(const void *key, u32 key_len, u32 hashrnd)
|
| 696 | {
|
| 697 | if (likely(key_len % 4 == 0))
|
| 698 | return jhash2(key, key_len / 4, hashrnd);
|
| 699 | return jhash(key, key_len, hashrnd);
|
| 700 | }
|
| 701 |
|
| 702 | static inline struct bucket *__select_bucket(struct bpf_htab *htab, u32 hash)
|
| 703 | {
|
| 704 | return &htab->buckets[hash & (htab->n_buckets - 1)];
|
| 705 | }
|
| 706 |
|
| 707 | static inline struct hlist_nulls_head *select_bucket(struct bpf_htab *htab, u32 hash)
|
| 708 | {
|
| 709 | return &__select_bucket(htab, hash)->head;
|
| 710 | }
|
| 711 |
|
| 712 |
|
| 713 | static struct htab_elem *lookup_elem_raw(struct hlist_nulls_head *head, u32 hash,
|
| 714 | void *key, u32 key_size)
|
| 715 | {
|
| 716 | struct hlist_nulls_node *n;
|
| 717 | struct htab_elem *l;
|
| 718 |
|
| 719 | hlist_nulls_for_each_entry_rcu(l, n, head, hash_node)
|
| 720 | if (l->hash == hash && !memcmp(&l->key, key, key_size))
|
| 721 | return l;
|
| 722 |
|
| 723 | return NULL;
|
| 724 | }
|
| 725 |
|
| 726 | |
| 727 | * the unlikely event when elements moved from one bucket into another
|
| 728 | * while link list is being walked
|
| 729 |
|
| 730 | static struct htab_elem *lookup_nulls_elem_raw(struct hlist_nulls_head *head,
|
| 731 | u32 hash, void *key,
|
| 732 | u32 key_size, u32 n_buckets)
|
| 733 | {
|
| 734 | struct hlist_nulls_node *n;
|
| 735 | struct htab_elem *l;
|
| 736 |
|
| 737 | again:
|
| 738 | hlist_nulls_for_each_entry_rcu(l, n, head, hash_node)
|
| 739 | if (l->hash == hash && !memcmp(&l->key, key, key_size))
|
| 740 | return l;
|
| 741 |
|
| 742 | if (unlikely(get_nulls_value(n) != (hash & (n_buckets - 1))))
|
| 743 | goto again;
|
| 744 |
|
| 745 | return NULL;
|
| 746 | }
|
| 747 |
|
| 748 | |
| 749 | * arguments have to match bpf_map_lookup_elem() exactly.
|
| 750 | * The return value is adjusted by BPF instructions
|
| 751 | * in htab_map_gen_lookup().
|
| 752 |
|
| 753 | static void *__htab_map_lookup_elem(struct bpf_map *map, void *key)
|
| 754 | {
|
| 755 | struct bpf_htab *htab = container_of(map, struct bpf_htab, map);
|
| 756 | struct hlist_nulls_head *head;
|
| 757 | struct htab_elem *l;
|
| 758 | u32 hash, key_size;
|
| 759 |
|
| 760 | WARN_ON_ONCE(!bpf_rcu_lock_held());
|
| 761 |
|
| 762 | key_size = map->key_size;
|
| 763 |
|
| 764 | hash = htab_map_hash(key, key_size, htab->hashrnd);
|
| 765 |
|
| 766 | head = select_bucket(htab, hash);
|
| 767 |
|
| 768 | l = lookup_nulls_elem_raw(head, hash, key, key_size, htab->n_buckets);
|
| 769 |
|
| 770 | return l;
|
| 771 | }
|
| 772 |
|
| 773 | static void *htab_map_lookup_elem(struct bpf_map *map, void *key)
|
| 774 | {
|
| 775 | struct htab_elem *l = __htab_map_lookup_elem(map, key);
|
| 776 |
|
| 777 | if (l)
|
| 778 | return htab_elem_value(l, map->key_size);
|
| 779 |
|
| 780 | return NULL;
|
| 781 | }
|
| 782 |
|
| 783 | |
| 784 | * Instead of:
|
| 785 | * bpf_prog
|
| 786 | * bpf_map_lookup_elem
|
| 787 | * map->ops->map_lookup_elem
|
| 788 | * htab_map_lookup_elem
|
| 789 | * __htab_map_lookup_elem
|
| 790 | * do:
|
| 791 | * bpf_prog
|
| 792 | * __htab_map_lookup_elem
|
| 793 |
|
| 794 | static int htab_map_gen_lookup(struct bpf_map *map, struct bpf_insn *insn_buf)
|
| 795 | {
|
| 796 | struct bpf_insn *insn = insn_buf;
|
| 797 | const int ret = BPF_REG_0;
|
| 798 |
|
| 799 | BUILD_BUG_ON(!__same_type(&__htab_map_lookup_elem,
|
| 800 | (void *(*)(struct bpf_map *map, void *key))NULL));
|
| 801 | *insn++ = BPF_EMIT_CALL(__htab_map_lookup_elem);
|
| 802 | *insn++ = BPF_JMP_IMM(BPF_JEQ, ret, 0, 1);
|
| 803 | *insn++ = BPF_ALU64_IMM(BPF_ADD, ret,
|
| 804 | offsetof(struct htab_elem, key) +
|
| 805 | round_up(map->key_size, 8));
|
| 806 | return insn - insn_buf;
|
| 807 | }
|
| 808 |
|
| 809 | static __always_inline void *__htab_lru_map_lookup_elem(struct bpf_map *map,
|
| 810 | void *key, const bool mark)
|
| 811 | {
|
| 812 | struct htab_elem *l = __htab_map_lookup_elem(map, key);
|
| 813 |
|
| 814 | if (l) {
|
| 815 | if (mark)
|
| 816 | bpf_lru_node_set_ref(&l->lru_node);
|
| 817 | return htab_elem_value(l, map->key_size);
|
| 818 | }
|
| 819 |
|
| 820 | return NULL;
|
| 821 | }
|
| 822 |
|
| 823 | static void *htab_lru_map_lookup_elem(struct bpf_map *map, void *key)
|
| 824 | {
|
| 825 | return __htab_lru_map_lookup_elem(map, key, true);
|
| 826 | }
|
| 827 |
|
| 828 | static void *htab_lru_map_lookup_elem_sys(struct bpf_map *map, void *key)
|
| 829 | {
|
| 830 | return __htab_lru_map_lookup_elem(map, key, false);
|
| 831 | }
|
| 832 |
|
| 833 | static int htab_lru_map_gen_lookup(struct bpf_map *map,
|
| 834 | struct bpf_insn *insn_buf)
|
| 835 | {
|
| 836 | struct bpf_insn *insn = insn_buf;
|
| 837 | const int ret = BPF_REG_0;
|
| 838 | const int ref_reg = BPF_REG_1;
|
| 839 |
|
| 840 | BUILD_BUG_ON(!__same_type(&__htab_map_lookup_elem,
|
| 841 | (void *(*)(struct bpf_map *map, void *key))NULL));
|
| 842 | *insn++ = BPF_EMIT_CALL(__htab_map_lookup_elem);
|
| 843 | *insn++ = BPF_JMP_IMM(BPF_JEQ, ret, 0, 4);
|
| 844 | *insn++ = BPF_LDX_MEM(BPF_B, ref_reg, ret,
|
| 845 | offsetof(struct htab_elem, lru_node) +
|
| 846 | offsetof(struct bpf_lru_node, ref));
|
| 847 | *insn++ = BPF_JMP_IMM(BPF_JNE, ref_reg, 0, 1);
|
| 848 | *insn++ = BPF_ST_MEM(BPF_B, ret,
|
| 849 | offsetof(struct htab_elem, lru_node) +
|
| 850 | offsetof(struct bpf_lru_node, ref),
|
| 851 | 1);
|
| 852 | *insn++ = BPF_ALU64_IMM(BPF_ADD, ret,
|
| 853 | offsetof(struct htab_elem, key) +
|
| 854 | round_up(map->key_size, 8));
|
| 855 | return insn - insn_buf;
|
| 856 | }
|
| 857 |
|
| 858 | static void check_and_cancel_fields(struct bpf_htab *htab,
|
| 859 | struct htab_elem *elem)
|
| 860 | {
|
| 861 | if (IS_ERR_OR_NULL(htab->map.record))
|
| 862 | return;
|
| 863 |
|
| 864 | if (htab_is_percpu(htab)) {
|
| 865 | void __percpu *pptr = htab_elem_get_ptr(elem, htab->map.key_size);
|
| 866 | int cpu;
|
| 867 |
|
| 868 | for_each_possible_cpu(cpu)
|
| 869 | bpf_obj_cancel_fields(&htab->map, per_cpu_ptr(pptr, cpu));
|
| 870 | } else {
|
| 871 | void *map_value = htab_elem_value(elem, htab->map.key_size);
|
| 872 |
|
| 873 | bpf_obj_cancel_fields(&htab->map, map_value);
|
| 874 | }
|
| 875 | }
|
| 876 |
|
| 877 | |
| 878 | * older elements from the htab.
|
| 879 |
|
| 880 | static bool htab_lru_map_delete_node(void *arg, struct bpf_lru_node *node)
|
| 881 | {
|
| 882 | struct bpf_htab *htab = arg;
|
| 883 | struct htab_elem *l = NULL, *tgt_l;
|
| 884 | struct hlist_nulls_head *head;
|
| 885 | struct hlist_nulls_node *n;
|
| 886 | unsigned long flags;
|
| 887 | struct bucket *b;
|
| 888 | int ret;
|
| 889 |
|
| 890 | tgt_l = container_of(node, struct htab_elem, lru_node);
|
| 891 | b = __select_bucket(htab, tgt_l->hash);
|
| 892 | head = &b->head;
|
| 893 |
|
| 894 | ret = htab_lock_bucket(b, &flags);
|
| 895 | if (ret)
|
| 896 | return false;
|
| 897 |
|
| 898 | hlist_nulls_for_each_entry_rcu(l, n, head, hash_node)
|
| 899 | if (l == tgt_l) {
|
| 900 | hlist_nulls_del_rcu(&l->hash_node);
|
| 901 | bpf_map_dec_elem_count(&htab->map);
|
| 902 | break;
|
| 903 | }
|
| 904 |
|
| 905 | htab_unlock_bucket(b, flags);
|
| 906 |
|
| 907 | if (l == tgt_l)
|
| 908 | check_and_cancel_fields(htab, l);
|
| 909 | return l == tgt_l;
|
| 910 | }
|
| 911 |
|
| 912 |
|
| 913 | static int htab_map_get_next_key(struct bpf_map *map, void *key, void *next_key)
|
| 914 | {
|
| 915 | struct bpf_htab *htab = container_of(map, struct bpf_htab, map);
|
| 916 | struct hlist_nulls_head *head;
|
| 917 | struct htab_elem *l, *next_l;
|
| 918 | u32 hash, key_size;
|
| 919 | int i = 0;
|
| 920 |
|
| 921 | WARN_ON_ONCE(!rcu_read_lock_held());
|
| 922 |
|
| 923 | key_size = map->key_size;
|
| 924 |
|
| 925 | if (!key)
|
| 926 | goto find_first_elem;
|
| 927 |
|
| 928 | hash = htab_map_hash(key, key_size, htab->hashrnd);
|
| 929 |
|
| 930 | head = select_bucket(htab, hash);
|
| 931 |
|
| 932 |
|
| 933 | l = lookup_nulls_elem_raw(head, hash, key, key_size, htab->n_buckets);
|
| 934 |
|
| 935 | if (!l)
|
| 936 | goto find_first_elem;
|
| 937 |
|
| 938 |
|
| 939 | next_l = hlist_nulls_entry_safe(rcu_dereference_raw(hlist_nulls_next_rcu(&l->hash_node)),
|
| 940 | struct htab_elem, hash_node);
|
| 941 |
|
| 942 | if (next_l) {
|
| 943 |
|
| 944 | memcpy(next_key, next_l->key, key_size);
|
| 945 | return 0;
|
| 946 | }
|
| 947 |
|
| 948 |
|
| 949 | i = hash & (htab->n_buckets - 1);
|
| 950 | i++;
|
| 951 |
|
| 952 | find_first_elem:
|
| 953 |
|
| 954 | for (; i < htab->n_buckets; i++) {
|
| 955 | head = select_bucket(htab, i);
|
| 956 |
|
| 957 |
|
| 958 | next_l = hlist_nulls_entry_safe(rcu_dereference_raw(hlist_nulls_first_rcu(head)),
|
| 959 | struct htab_elem, hash_node);
|
| 960 | if (next_l) {
|
| 961 |
|
| 962 | memcpy(next_key, next_l->key, key_size);
|
| 963 | return 0;
|
| 964 | }
|
| 965 | }
|
| 966 |
|
| 967 |
|
| 968 | return -ENOENT;
|
| 969 | }
|
| 970 |
|
| 971 | static void htab_elem_free(struct bpf_htab *htab, struct htab_elem *l)
|
| 972 | {
|
| 973 | check_and_cancel_fields(htab, l);
|
| 974 |
|
| 975 | if (htab->map.map_type == BPF_MAP_TYPE_PERCPU_HASH)
|
| 976 | bpf_mem_cache_free(&htab->pcpu_ma, l->ptr_to_pptr);
|
| 977 | bpf_mem_cache_free(&htab->ma, l);
|
| 978 | }
|
| 979 |
|
| 980 | static void htab_put_fd_value(struct bpf_htab *htab, struct htab_elem *l)
|
| 981 | {
|
| 982 | struct bpf_map *map = &htab->map;
|
| 983 | void *ptr;
|
| 984 |
|
| 985 | if (map->ops->map_fd_put_ptr) {
|
| 986 | ptr = fd_htab_map_get_ptr(map, l);
|
| 987 | map->ops->map_fd_put_ptr(map, ptr, true);
|
| 988 | }
|
| 989 | }
|
| 990 |
|
| 991 | static bool is_map_full(struct bpf_htab *htab)
|
| 992 | {
|
| 993 | if (htab->use_percpu_counter)
|
| 994 | return __percpu_counter_compare(&htab->pcount, htab->map.max_entries,
|
| 995 | PERCPU_COUNTER_BATCH) >= 0;
|
| 996 | return atomic_read(&htab->count) >= htab->map.max_entries;
|
| 997 | }
|
| 998 |
|
| 999 | static void inc_elem_count(struct bpf_htab *htab)
|
| 1000 | {
|
| 1001 | bpf_map_inc_elem_count(&htab->map);
|
| 1002 |
|
| 1003 | if (htab->use_percpu_counter)
|
| 1004 | percpu_counter_add_batch(&htab->pcount, 1, PERCPU_COUNTER_BATCH);
|
| 1005 | else
|
| 1006 | atomic_inc(&htab->count);
|
| 1007 | }
|
| 1008 |
|
| 1009 | static void dec_elem_count(struct bpf_htab *htab)
|
| 1010 | {
|
| 1011 | bpf_map_dec_elem_count(&htab->map);
|
| 1012 |
|
| 1013 | if (htab->use_percpu_counter)
|
| 1014 | percpu_counter_add_batch(&htab->pcount, -1, PERCPU_COUNTER_BATCH);
|
| 1015 | else
|
| 1016 | atomic_dec(&htab->count);
|
| 1017 | }
|
| 1018 |
|
| 1019 | static void free_htab_elem(struct bpf_htab *htab, struct htab_elem *l)
|
| 1020 | {
|
| 1021 | htab_put_fd_value(htab, l);
|
| 1022 |
|
| 1023 | if (htab_is_prealloc(htab)) {
|
| 1024 | bpf_map_dec_elem_count(&htab->map);
|
| 1025 | check_and_cancel_fields(htab, l);
|
| 1026 | pcpu_freelist_push(&htab->freelist, &l->fnode);
|
| 1027 | } else {
|
| 1028 | dec_elem_count(htab);
|
| 1029 | htab_elem_free(htab, l);
|
| 1030 | }
|
| 1031 | }
|
| 1032 |
|
| 1033 | static void pcpu_copy_value(struct bpf_htab *htab, void __percpu *pptr,
|
| 1034 | void *value, bool onallcpus, u64 map_flags)
|
| 1035 | {
|
| 1036 | void *ptr;
|
| 1037 |
|
| 1038 | if (!onallcpus) {
|
| 1039 |
|
| 1040 | ptr = this_cpu_ptr(pptr);
|
| 1041 | copy_map_value(&htab->map, ptr, value);
|
| 1042 | bpf_obj_cancel_fields(&htab->map, ptr);
|
| 1043 | } else {
|
| 1044 | u32 size = round_up(htab->map.value_size, 8);
|
| 1045 | void *val;
|
| 1046 | int cpu, off = 0;
|
| 1047 |
|
| 1048 | if (map_flags & BPF_F_CPU) {
|
| 1049 | cpu = map_flags >> 32;
|
| 1050 | ptr = per_cpu_ptr(pptr, cpu);
|
| 1051 | copy_map_value(&htab->map, ptr, value);
|
| 1052 | bpf_obj_cancel_fields(&htab->map, ptr);
|
| 1053 | return;
|
| 1054 | }
|
| 1055 |
|
| 1056 | for_each_possible_cpu(cpu) {
|
| 1057 | ptr = per_cpu_ptr(pptr, cpu);
|
| 1058 | val = (map_flags & BPF_F_ALL_CPUS) ? value : value + off;
|
| 1059 | copy_map_value(&htab->map, ptr, val);
|
| 1060 | bpf_obj_cancel_fields(&htab->map, ptr);
|
| 1061 | off += size;
|
| 1062 | }
|
| 1063 | }
|
| 1064 | }
|
| 1065 |
|
| 1066 | static void pcpu_init_value(struct bpf_htab *htab, void __percpu *pptr,
|
| 1067 | void *value, bool onallcpus, u64 map_flags)
|
| 1068 | {
|
| 1069 | |
| 1070 | * values for other cpus. Otherwise, bpf program has no way to ensure
|
| 1071 | * known initial values for cpus other than current one
|
| 1072 | * (onallcpus=false always when coming from bpf prog,
|
| 1073 | * map_flags & BPF_F_CPU when coming from syscall but setting
|
| 1074 | * only one cpu).
|
| 1075 |
|
| 1076 | if (!onallcpus || (map_flags & BPF_F_CPU)) {
|
| 1077 | int init_cpu = (map_flags & BPF_F_CPU) ? map_flags >> 32 :
|
| 1078 | raw_smp_processor_id();
|
| 1079 | int cpu;
|
| 1080 |
|
| 1081 | for_each_possible_cpu(cpu) {
|
| 1082 | if (cpu == init_cpu)
|
| 1083 | copy_map_value(&htab->map, per_cpu_ptr(pptr, cpu), value);
|
| 1084 | else
|
| 1085 | zero_map_value(&htab->map, per_cpu_ptr(pptr, cpu));
|
| 1086 | }
|
| 1087 | } else {
|
| 1088 | pcpu_copy_value(htab, pptr, value, onallcpus, map_flags);
|
| 1089 | }
|
| 1090 | }
|
| 1091 |
|
| 1092 | static bool fd_htab_map_needs_adjust(const struct bpf_htab *htab)
|
| 1093 | {
|
| 1094 | return is_fd_htab(htab) && BITS_PER_LONG == 64;
|
| 1095 | }
|
| 1096 |
|
| 1097 | static struct htab_elem *alloc_htab_elem(struct bpf_htab *htab, void *key,
|
| 1098 | void *value, u32 key_size, u32 hash,
|
| 1099 | bool percpu, bool onallcpus,
|
| 1100 | struct htab_elem *old_elem, u64 map_flags)
|
| 1101 | {
|
| 1102 | u32 size = htab->map.value_size;
|
| 1103 | bool prealloc = htab_is_prealloc(htab);
|
| 1104 | struct htab_elem *l_new, **pl_new;
|
| 1105 | void __percpu *pptr;
|
| 1106 |
|
| 1107 | if (prealloc) {
|
| 1108 | if (old_elem) {
|
| 1109 | |
| 1110 | * use per-cpu extra elems to avoid freelist_pop/push
|
| 1111 |
|
| 1112 | pl_new = this_cpu_ptr(htab->extra_elems);
|
| 1113 | l_new = *pl_new;
|
| 1114 | *pl_new = old_elem;
|
| 1115 | } else {
|
| 1116 | struct pcpu_freelist_node *l;
|
| 1117 |
|
| 1118 | l = __pcpu_freelist_pop(&htab->freelist);
|
| 1119 | if (!l)
|
| 1120 | return ERR_PTR(-E2BIG);
|
| 1121 | l_new = container_of(l, struct htab_elem, fnode);
|
| 1122 | bpf_map_inc_elem_count(&htab->map);
|
| 1123 | }
|
| 1124 | } else {
|
| 1125 | if (is_map_full(htab))
|
| 1126 | if (!old_elem)
|
| 1127 | |
| 1128 | * old element, it's ok to allocate, since
|
| 1129 | * old element will be freed immediately.
|
| 1130 | * Otherwise return an error
|
| 1131 |
|
| 1132 | return ERR_PTR(-E2BIG);
|
| 1133 | inc_elem_count(htab);
|
| 1134 | l_new = bpf_mem_cache_alloc(&htab->ma);
|
| 1135 | if (!l_new) {
|
| 1136 | l_new = ERR_PTR(-ENOMEM);
|
| 1137 | goto dec_count;
|
| 1138 | }
|
| 1139 | }
|
| 1140 |
|
| 1141 | memcpy(l_new->key, key, key_size);
|
| 1142 | if (percpu) {
|
| 1143 | if (prealloc) {
|
| 1144 | pptr = htab_elem_get_ptr(l_new, key_size);
|
| 1145 | } else {
|
| 1146 |
|
| 1147 | void *ptr = bpf_mem_cache_alloc(&htab->pcpu_ma);
|
| 1148 |
|
| 1149 | if (!ptr) {
|
| 1150 | bpf_mem_cache_free(&htab->ma, l_new);
|
| 1151 | l_new = ERR_PTR(-ENOMEM);
|
| 1152 | goto dec_count;
|
| 1153 | }
|
| 1154 | l_new->ptr_to_pptr = ptr;
|
| 1155 | pptr = *(void __percpu **)ptr;
|
| 1156 | }
|
| 1157 |
|
| 1158 | pcpu_init_value(htab, pptr, value, onallcpus, map_flags);
|
| 1159 |
|
| 1160 | if (!prealloc)
|
| 1161 | htab_elem_set_ptr(l_new, key_size, pptr);
|
| 1162 | } else if (fd_htab_map_needs_adjust(htab)) {
|
| 1163 | size = round_up(size, 8);
|
| 1164 | memcpy(htab_elem_value(l_new, key_size), value, size);
|
| 1165 | } else if (map_flags & BPF_F_LOCK) {
|
| 1166 | copy_map_value_locked(&htab->map,
|
| 1167 | htab_elem_value(l_new, key_size),
|
| 1168 | value, false);
|
| 1169 | } else {
|
| 1170 | copy_map_value(&htab->map, htab_elem_value(l_new, key_size), value);
|
| 1171 | }
|
| 1172 |
|
| 1173 | l_new->hash = hash;
|
| 1174 | return l_new;
|
| 1175 | dec_count:
|
| 1176 | dec_elem_count(htab);
|
| 1177 | return l_new;
|
| 1178 | }
|
| 1179 |
|
| 1180 | static int check_flags(struct bpf_htab *htab, struct htab_elem *l_old,
|
| 1181 | u64 map_flags)
|
| 1182 | {
|
| 1183 | if (l_old && (map_flags & ~BPF_F_LOCK) == BPF_NOEXIST)
|
| 1184 |
|
| 1185 | return -EEXIST;
|
| 1186 |
|
| 1187 | if (!l_old && (map_flags & ~BPF_F_LOCK) == BPF_EXIST)
|
| 1188 |
|
| 1189 | return -ENOENT;
|
| 1190 |
|
| 1191 | return 0;
|
| 1192 | }
|
| 1193 |
|
| 1194 |
|
| 1195 | static long htab_map_update_elem(struct bpf_map *map, void *key, void *value,
|
| 1196 | u64 map_flags)
|
| 1197 | {
|
| 1198 | struct bpf_htab *htab = container_of(map, struct bpf_htab, map);
|
| 1199 | struct htab_elem *l_new, *l_old;
|
| 1200 | struct hlist_nulls_head *head;
|
| 1201 | unsigned long flags;
|
| 1202 | struct bucket *b;
|
| 1203 | u32 key_size, hash;
|
| 1204 | int ret;
|
| 1205 |
|
| 1206 | if (unlikely((map_flags & ~BPF_F_LOCK) > BPF_EXIST))
|
| 1207 |
|
| 1208 | return -EINVAL;
|
| 1209 |
|
| 1210 | WARN_ON_ONCE(!bpf_rcu_lock_held());
|
| 1211 |
|
| 1212 | key_size = map->key_size;
|
| 1213 |
|
| 1214 | hash = htab_map_hash(key, key_size, htab->hashrnd);
|
| 1215 |
|
| 1216 | b = __select_bucket(htab, hash);
|
| 1217 | head = &b->head;
|
| 1218 |
|
| 1219 | if (unlikely(map_flags & BPF_F_LOCK)) {
|
| 1220 | if (unlikely(!btf_record_has_field(map->record, BPF_SPIN_LOCK)))
|
| 1221 | return -EINVAL;
|
| 1222 |
|
| 1223 | l_old = lookup_nulls_elem_raw(head, hash, key, key_size,
|
| 1224 | htab->n_buckets);
|
| 1225 | ret = check_flags(htab, l_old, map_flags);
|
| 1226 | if (ret)
|
| 1227 | return ret;
|
| 1228 | if (l_old) {
|
| 1229 |
|
| 1230 | copy_map_value_locked(map,
|
| 1231 | htab_elem_value(l_old, key_size),
|
| 1232 | value, false);
|
| 1233 | return 0;
|
| 1234 | }
|
| 1235 | |
| 1236 | * 99.9% chance that the element won't be found,
|
| 1237 | * but second lookup under lock has to be done.
|
| 1238 |
|
| 1239 | }
|
| 1240 |
|
| 1241 | ret = htab_lock_bucket(b, &flags);
|
| 1242 | if (ret)
|
| 1243 | return ret;
|
| 1244 |
|
| 1245 | l_old = lookup_elem_raw(head, hash, key, key_size);
|
| 1246 |
|
| 1247 | ret = check_flags(htab, l_old, map_flags);
|
| 1248 | if (ret)
|
| 1249 | goto err;
|
| 1250 |
|
| 1251 | if (unlikely(l_old && (map_flags & BPF_F_LOCK))) {
|
| 1252 | |
| 1253 | * but second lookup with the bucket lock found it.
|
| 1254 | * This case is highly unlikely, but has to be dealt with:
|
| 1255 | * grab the element lock in addition to the bucket lock
|
| 1256 | * and update element in place
|
| 1257 |
|
| 1258 | copy_map_value_locked(map,
|
| 1259 | htab_elem_value(l_old, key_size),
|
| 1260 | value, false);
|
| 1261 | ret = 0;
|
| 1262 | goto err;
|
| 1263 | }
|
| 1264 |
|
| 1265 | l_new = alloc_htab_elem(htab, key, value, key_size, hash, false, false,
|
| 1266 | l_old, map_flags);
|
| 1267 | if (IS_ERR(l_new)) {
|
| 1268 |
|
| 1269 | ret = PTR_ERR(l_new);
|
| 1270 | goto err;
|
| 1271 | }
|
| 1272 |
|
| 1273 | |
| 1274 | * concurrent search will find it before old elem
|
| 1275 |
|
| 1276 | hlist_nulls_add_head_rcu(&l_new->hash_node, head);
|
| 1277 | if (l_old) {
|
| 1278 | hlist_nulls_del_rcu(&l_old->hash_node);
|
| 1279 |
|
| 1280 | |
| 1281 | * its reusable special fields before it is available for reuse.
|
| 1282 |
|
| 1283 | if (htab_is_prealloc(htab))
|
| 1284 | check_and_cancel_fields(htab, l_old);
|
| 1285 | }
|
| 1286 | htab_unlock_bucket(b, flags);
|
| 1287 | if (l_old && !htab_is_prealloc(htab))
|
| 1288 | free_htab_elem(htab, l_old);
|
| 1289 | return 0;
|
| 1290 | err:
|
| 1291 | htab_unlock_bucket(b, flags);
|
| 1292 | return ret;
|
| 1293 | }
|
| 1294 |
|
| 1295 | static void htab_lru_push_free(struct bpf_htab *htab, struct htab_elem *elem)
|
| 1296 | {
|
| 1297 | check_and_cancel_fields(htab, elem);
|
| 1298 | bpf_map_dec_elem_count(&htab->map);
|
| 1299 | bpf_lru_push_free(&htab->lru, &elem->lru_node);
|
| 1300 | }
|
| 1301 |
|
| 1302 | static long htab_lru_map_update_elem(struct bpf_map *map, void *key, void *value,
|
| 1303 | u64 map_flags)
|
| 1304 | {
|
| 1305 | struct bpf_htab *htab = container_of(map, struct bpf_htab, map);
|
| 1306 | struct htab_elem *l_new, *l_old = NULL;
|
| 1307 | struct hlist_nulls_head *head;
|
| 1308 | unsigned long flags;
|
| 1309 | struct bucket *b;
|
| 1310 | u32 key_size, hash;
|
| 1311 | int ret;
|
| 1312 |
|
| 1313 | if (unlikely(map_flags > BPF_EXIST))
|
| 1314 |
|
| 1315 | return -EINVAL;
|
| 1316 |
|
| 1317 | WARN_ON_ONCE(!bpf_rcu_lock_held());
|
| 1318 |
|
| 1319 | key_size = map->key_size;
|
| 1320 |
|
| 1321 | hash = htab_map_hash(key, key_size, htab->hashrnd);
|
| 1322 |
|
| 1323 | b = __select_bucket(htab, hash);
|
| 1324 | head = &b->head;
|
| 1325 |
|
| 1326 | |
| 1327 | * spinlock because getting free nodes from LRU may need
|
| 1328 | * to remove older elements from htab and this removal
|
| 1329 | * operation will need a bucket lock.
|
| 1330 |
|
| 1331 | l_new = prealloc_lru_pop(htab, key, hash);
|
| 1332 | if (!l_new)
|
| 1333 | return -ENOMEM;
|
| 1334 | copy_map_value(&htab->map, htab_elem_value(l_new, map->key_size), value);
|
| 1335 |
|
| 1336 | ret = htab_lock_bucket(b, &flags);
|
| 1337 | if (ret)
|
| 1338 | goto err_lock_bucket;
|
| 1339 |
|
| 1340 | l_old = lookup_elem_raw(head, hash, key, key_size);
|
| 1341 |
|
| 1342 | ret = check_flags(htab, l_old, map_flags);
|
| 1343 | if (ret)
|
| 1344 | goto err;
|
| 1345 |
|
| 1346 | |
| 1347 | * concurrent search will find it before old elem
|
| 1348 |
|
| 1349 | hlist_nulls_add_head_rcu(&l_new->hash_node, head);
|
| 1350 | if (l_old) {
|
| 1351 | bpf_lru_node_set_ref(&l_new->lru_node);
|
| 1352 | hlist_nulls_del_rcu(&l_old->hash_node);
|
| 1353 | }
|
| 1354 | ret = 0;
|
| 1355 |
|
| 1356 | err:
|
| 1357 | htab_unlock_bucket(b, flags);
|
| 1358 |
|
| 1359 | err_lock_bucket:
|
| 1360 | if (ret)
|
| 1361 | htab_lru_push_free(htab, l_new);
|
| 1362 | else if (l_old)
|
| 1363 | htab_lru_push_free(htab, l_old);
|
| 1364 |
|
| 1365 | return ret;
|
| 1366 | }
|
| 1367 |
|
| 1368 | static int htab_map_check_update_flags(bool onallcpus, u64 map_flags)
|
| 1369 | {
|
| 1370 | if (unlikely(!onallcpus && map_flags > BPF_EXIST))
|
| 1371 | return -EINVAL;
|
| 1372 | if (unlikely(onallcpus && ((map_flags & BPF_F_LOCK) || (u32)map_flags > BPF_F_ALL_CPUS)))
|
| 1373 | return -EINVAL;
|
| 1374 | return 0;
|
| 1375 | }
|
| 1376 |
|
| 1377 | static long htab_map_update_elem_in_place(struct bpf_map *map, void *key,
|
| 1378 | void *value, u64 map_flags,
|
| 1379 | bool percpu, bool onallcpus)
|
| 1380 | {
|
| 1381 | struct bpf_htab *htab = container_of(map, struct bpf_htab, map);
|
| 1382 | struct htab_elem *l_new, *l_old;
|
| 1383 | struct hlist_nulls_head *head;
|
| 1384 | void *old_map_ptr = NULL;
|
| 1385 | unsigned long flags;
|
| 1386 | struct bucket *b;
|
| 1387 | u32 key_size, hash;
|
| 1388 | int ret;
|
| 1389 |
|
| 1390 | ret = htab_map_check_update_flags(onallcpus, map_flags);
|
| 1391 | if (unlikely(ret))
|
| 1392 | return ret;
|
| 1393 |
|
| 1394 | WARN_ON_ONCE(!bpf_rcu_lock_held());
|
| 1395 |
|
| 1396 | key_size = map->key_size;
|
| 1397 |
|
| 1398 | hash = htab_map_hash(key, key_size, htab->hashrnd);
|
| 1399 |
|
| 1400 | b = __select_bucket(htab, hash);
|
| 1401 | head = &b->head;
|
| 1402 |
|
| 1403 | ret = htab_lock_bucket(b, &flags);
|
| 1404 | if (ret)
|
| 1405 | return ret;
|
| 1406 |
|
| 1407 | l_old = lookup_elem_raw(head, hash, key, key_size);
|
| 1408 |
|
| 1409 | ret = check_flags(htab, l_old, map_flags);
|
| 1410 | if (ret)
|
| 1411 | goto err;
|
| 1412 |
|
| 1413 | if (l_old) {
|
| 1414 |
|
| 1415 | if (percpu) {
|
| 1416 | pcpu_copy_value(htab, htab_elem_get_ptr(l_old, key_size),
|
| 1417 | value, onallcpus, map_flags);
|
| 1418 | } else {
|
| 1419 | void **inner_map_pptr = htab_elem_value(l_old, key_size);
|
| 1420 |
|
| 1421 | old_map_ptr = *inner_map_pptr;
|
| 1422 | WRITE_ONCE(*inner_map_pptr, *(void **)value);
|
| 1423 | }
|
| 1424 | } else {
|
| 1425 | l_new = alloc_htab_elem(htab, key, value, key_size,
|
| 1426 | hash, percpu, onallcpus, NULL, map_flags);
|
| 1427 | if (IS_ERR(l_new)) {
|
| 1428 | ret = PTR_ERR(l_new);
|
| 1429 | goto err;
|
| 1430 | }
|
| 1431 | hlist_nulls_add_head_rcu(&l_new->hash_node, head);
|
| 1432 | }
|
| 1433 | err:
|
| 1434 | htab_unlock_bucket(b, flags);
|
| 1435 | if (old_map_ptr)
|
| 1436 | map->ops->map_fd_put_ptr(map, old_map_ptr, true);
|
| 1437 | return ret;
|
| 1438 | }
|
| 1439 |
|
| 1440 | static long __htab_lru_percpu_map_update_elem(struct bpf_map *map, void *key,
|
| 1441 | void *value, u64 map_flags,
|
| 1442 | bool onallcpus)
|
| 1443 | {
|
| 1444 | struct bpf_htab *htab = container_of(map, struct bpf_htab, map);
|
| 1445 | struct htab_elem *l_new = NULL, *l_old;
|
| 1446 | struct hlist_nulls_head *head;
|
| 1447 | unsigned long flags;
|
| 1448 | struct bucket *b;
|
| 1449 | u32 key_size, hash;
|
| 1450 | int ret;
|
| 1451 |
|
| 1452 | ret = htab_map_check_update_flags(onallcpus, map_flags);
|
| 1453 | if (unlikely(ret))
|
| 1454 | return ret;
|
| 1455 |
|
| 1456 | WARN_ON_ONCE(!bpf_rcu_lock_held());
|
| 1457 |
|
| 1458 | key_size = map->key_size;
|
| 1459 |
|
| 1460 | hash = htab_map_hash(key, key_size, htab->hashrnd);
|
| 1461 |
|
| 1462 | b = __select_bucket(htab, hash);
|
| 1463 | head = &b->head;
|
| 1464 |
|
| 1465 | |
| 1466 | * spinlock because LRU's elem alloc may need
|
| 1467 | * to remove older elem from htab and this removal
|
| 1468 | * operation will need a bucket lock.
|
| 1469 |
|
| 1470 | if (map_flags != BPF_EXIST) {
|
| 1471 | l_new = prealloc_lru_pop(htab, key, hash);
|
| 1472 | if (!l_new)
|
| 1473 | return -ENOMEM;
|
| 1474 | }
|
| 1475 |
|
| 1476 | ret = htab_lock_bucket(b, &flags);
|
| 1477 | if (ret)
|
| 1478 | goto err_lock_bucket;
|
| 1479 |
|
| 1480 | l_old = lookup_elem_raw(head, hash, key, key_size);
|
| 1481 |
|
| 1482 | ret = check_flags(htab, l_old, map_flags);
|
| 1483 | if (ret)
|
| 1484 | goto err;
|
| 1485 |
|
| 1486 | if (l_old) {
|
| 1487 | bpf_lru_node_set_ref(&l_old->lru_node);
|
| 1488 |
|
| 1489 |
|
| 1490 | pcpu_copy_value(htab, htab_elem_get_ptr(l_old, key_size),
|
| 1491 | value, onallcpus, map_flags);
|
| 1492 | } else {
|
| 1493 | pcpu_init_value(htab, htab_elem_get_ptr(l_new, key_size),
|
| 1494 | value, onallcpus, map_flags);
|
| 1495 | hlist_nulls_add_head_rcu(&l_new->hash_node, head);
|
| 1496 | l_new = NULL;
|
| 1497 | }
|
| 1498 | ret = 0;
|
| 1499 | err:
|
| 1500 | htab_unlock_bucket(b, flags);
|
| 1501 | err_lock_bucket:
|
| 1502 | if (l_new) {
|
| 1503 | bpf_map_dec_elem_count(&htab->map);
|
| 1504 | bpf_lru_push_free(&htab->lru, &l_new->lru_node);
|
| 1505 | }
|
| 1506 | return ret;
|
| 1507 | }
|
| 1508 |
|
| 1509 | static long htab_percpu_map_update_elem(struct bpf_map *map, void *key,
|
| 1510 | void *value, u64 map_flags)
|
| 1511 | {
|
| 1512 | return htab_map_update_elem_in_place(map, key, value, map_flags, true, false);
|
| 1513 | }
|
| 1514 |
|
| 1515 | static long htab_lru_percpu_map_update_elem(struct bpf_map *map, void *key,
|
| 1516 | void *value, u64 map_flags)
|
| 1517 | {
|
| 1518 | return __htab_lru_percpu_map_update_elem(map, key, value, map_flags,
|
| 1519 | false);
|
| 1520 | }
|
| 1521 |
|
| 1522 |
|
| 1523 | static long htab_map_delete_elem(struct bpf_map *map, void *key)
|
| 1524 | {
|
| 1525 | struct bpf_htab *htab = container_of(map, struct bpf_htab, map);
|
| 1526 | struct hlist_nulls_head *head;
|
| 1527 | struct bucket *b;
|
| 1528 | struct htab_elem *l;
|
| 1529 | unsigned long flags;
|
| 1530 | u32 hash, key_size;
|
| 1531 | int ret;
|
| 1532 |
|
| 1533 | WARN_ON_ONCE(!bpf_rcu_lock_held());
|
| 1534 |
|
| 1535 | key_size = map->key_size;
|
| 1536 |
|
| 1537 | hash = htab_map_hash(key, key_size, htab->hashrnd);
|
| 1538 | b = __select_bucket(htab, hash);
|
| 1539 | head = &b->head;
|
| 1540 |
|
| 1541 | ret = htab_lock_bucket(b, &flags);
|
| 1542 | if (ret)
|
| 1543 | return ret;
|
| 1544 |
|
| 1545 | l = lookup_elem_raw(head, hash, key, key_size);
|
| 1546 | if (l)
|
| 1547 | hlist_nulls_del_rcu(&l->hash_node);
|
| 1548 | else
|
| 1549 | ret = -ENOENT;
|
| 1550 |
|
| 1551 | htab_unlock_bucket(b, flags);
|
| 1552 |
|
| 1553 | if (l)
|
| 1554 | free_htab_elem(htab, l);
|
| 1555 | return ret;
|
| 1556 | }
|
| 1557 |
|
| 1558 | static long htab_lru_map_delete_elem(struct bpf_map *map, void *key)
|
| 1559 | {
|
| 1560 | struct bpf_htab *htab = container_of(map, struct bpf_htab, map);
|
| 1561 | struct hlist_nulls_head *head;
|
| 1562 | struct bucket *b;
|
| 1563 | struct htab_elem *l;
|
| 1564 | unsigned long flags;
|
| 1565 | u32 hash, key_size;
|
| 1566 | int ret;
|
| 1567 |
|
| 1568 | WARN_ON_ONCE(!bpf_rcu_lock_held());
|
| 1569 |
|
| 1570 | key_size = map->key_size;
|
| 1571 |
|
| 1572 | hash = htab_map_hash(key, key_size, htab->hashrnd);
|
| 1573 | b = __select_bucket(htab, hash);
|
| 1574 | head = &b->head;
|
| 1575 |
|
| 1576 | ret = htab_lock_bucket(b, &flags);
|
| 1577 | if (ret)
|
| 1578 | return ret;
|
| 1579 |
|
| 1580 | l = lookup_elem_raw(head, hash, key, key_size);
|
| 1581 |
|
| 1582 | if (l)
|
| 1583 | hlist_nulls_del_rcu(&l->hash_node);
|
| 1584 | else
|
| 1585 | ret = -ENOENT;
|
| 1586 |
|
| 1587 | htab_unlock_bucket(b, flags);
|
| 1588 | if (l)
|
| 1589 | htab_lru_push_free(htab, l);
|
| 1590 | return ret;
|
| 1591 | }
|
| 1592 |
|
| 1593 | static void delete_all_elements(struct bpf_htab *htab)
|
| 1594 | {
|
| 1595 | int i;
|
| 1596 |
|
| 1597 | |
| 1598 | * therefore, it is OK to invoke bpf_mem_cache_free() directly.
|
| 1599 |
|
| 1600 | for (i = 0; i < htab->n_buckets; i++) {
|
| 1601 | struct hlist_nulls_head *head = select_bucket(htab, i);
|
| 1602 | struct hlist_nulls_node *n;
|
| 1603 | struct htab_elem *l;
|
| 1604 |
|
| 1605 | hlist_nulls_for_each_entry_safe(l, n, head, hash_node) {
|
| 1606 | hlist_nulls_del_rcu(&l->hash_node);
|
| 1607 | htab_elem_free(htab, l);
|
| 1608 | }
|
| 1609 | cond_resched();
|
| 1610 | }
|
| 1611 | }
|
| 1612 |
|
| 1613 | static void htab_free_malloced_internal_structs(struct bpf_htab *htab)
|
| 1614 | {
|
| 1615 | int i;
|
| 1616 |
|
| 1617 | rcu_read_lock();
|
| 1618 | for (i = 0; i < htab->n_buckets; i++) {
|
| 1619 | struct hlist_nulls_head *head = select_bucket(htab, i);
|
| 1620 | struct hlist_nulls_node *n;
|
| 1621 | struct htab_elem *l;
|
| 1622 |
|
| 1623 | hlist_nulls_for_each_entry(l, n, head, hash_node) {
|
| 1624 |
|
| 1625 | bpf_map_free_internal_structs(&htab->map,
|
| 1626 | htab_elem_value(l, htab->map.key_size));
|
| 1627 | }
|
| 1628 | cond_resched_rcu();
|
| 1629 | }
|
| 1630 | rcu_read_unlock();
|
| 1631 | }
|
| 1632 |
|
| 1633 | static void htab_map_free_internal_structs(struct bpf_map *map)
|
| 1634 | {
|
| 1635 | struct bpf_htab *htab = container_of(map, struct bpf_htab, map);
|
| 1636 |
|
| 1637 |
|
| 1638 | if (!bpf_map_has_internal_structs(map))
|
| 1639 | return;
|
| 1640 |
|
| 1641 | if (htab_is_prealloc(htab))
|
| 1642 | htab_free_prealloced_internal_structs(htab);
|
| 1643 | else
|
| 1644 | htab_free_malloced_internal_structs(htab);
|
| 1645 | }
|
| 1646 |
|
| 1647 |
|
| 1648 | static void htab_map_free(struct bpf_map *map)
|
| 1649 | {
|
| 1650 | struct bpf_htab *htab = container_of(map, struct bpf_htab, map);
|
| 1651 |
|
| 1652 | |
| 1653 | * bpf_free_used_maps() is called after bpf prog is no longer executing.
|
| 1654 | * There is no need to synchronize_rcu() here to protect map elements.
|
| 1655 |
|
| 1656 |
|
| 1657 | |
| 1658 | * underneath and is responsible for waiting for callbacks to finish
|
| 1659 | * during bpf_mem_alloc_destroy().
|
| 1660 |
|
| 1661 | if (!htab_is_prealloc(htab)) {
|
| 1662 | delete_all_elements(htab);
|
| 1663 | } else {
|
| 1664 | htab_free_prealloced_fields(htab);
|
| 1665 | prealloc_destroy(htab);
|
| 1666 | }
|
| 1667 |
|
| 1668 | bpf_map_free_elem_count(map);
|
| 1669 | free_percpu(htab->extra_elems);
|
| 1670 | bpf_map_area_free(htab->buckets);
|
| 1671 | bpf_mem_alloc_destroy(&htab->pcpu_ma);
|
| 1672 | bpf_mem_alloc_destroy(&htab->ma);
|
| 1673 | if (htab->use_percpu_counter)
|
| 1674 | percpu_counter_destroy(&htab->pcount);
|
| 1675 | bpf_map_area_free(htab);
|
| 1676 | }
|
| 1677 |
|
| 1678 | static void htab_map_seq_show_elem(struct bpf_map *map, void *key,
|
| 1679 | struct seq_file *m)
|
| 1680 | {
|
| 1681 | void *value;
|
| 1682 |
|
| 1683 | rcu_read_lock();
|
| 1684 |
|
| 1685 | value = htab_map_lookup_elem(map, key);
|
| 1686 | if (!value) {
|
| 1687 | rcu_read_unlock();
|
| 1688 | return;
|
| 1689 | }
|
| 1690 |
|
| 1691 | btf_type_seq_show(map->btf, map->btf_key_type_id, key, m);
|
| 1692 | seq_puts(m, ": ");
|
| 1693 | btf_type_seq_show(map->btf, map->btf_value_type_id, value, m);
|
| 1694 | seq_putc(m, '\n');
|
| 1695 |
|
| 1696 | rcu_read_unlock();
|
| 1697 | }
|
| 1698 |
|
| 1699 | static int __htab_map_lookup_and_delete_elem(struct bpf_map *map, void *key,
|
| 1700 | void *value, bool is_lru_map,
|
| 1701 | bool is_percpu, u64 flags)
|
| 1702 | {
|
| 1703 | struct bpf_htab *htab = container_of(map, struct bpf_htab, map);
|
| 1704 | struct hlist_nulls_head *head;
|
| 1705 | unsigned long bflags;
|
| 1706 | struct htab_elem *l;
|
| 1707 | u32 hash, key_size;
|
| 1708 | struct bucket *b;
|
| 1709 | int ret;
|
| 1710 |
|
| 1711 | key_size = map->key_size;
|
| 1712 |
|
| 1713 | hash = htab_map_hash(key, key_size, htab->hashrnd);
|
| 1714 | b = __select_bucket(htab, hash);
|
| 1715 | head = &b->head;
|
| 1716 |
|
| 1717 | ret = htab_lock_bucket(b, &bflags);
|
| 1718 | if (ret)
|
| 1719 | return ret;
|
| 1720 |
|
| 1721 | l = lookup_elem_raw(head, hash, key, key_size);
|
| 1722 | if (!l) {
|
| 1723 | ret = -ENOENT;
|
| 1724 | goto out_unlock;
|
| 1725 | }
|
| 1726 |
|
| 1727 | if (is_percpu) {
|
| 1728 | u32 roundup_value_size = round_up(map->value_size, 8);
|
| 1729 | void __percpu *pptr;
|
| 1730 | int off = 0, cpu;
|
| 1731 |
|
| 1732 | pptr = htab_elem_get_ptr(l, key_size);
|
| 1733 | for_each_possible_cpu(cpu) {
|
| 1734 | copy_map_value_long(&htab->map, value + off, per_cpu_ptr(pptr, cpu));
|
| 1735 | check_and_init_map_value(&htab->map, value + off);
|
| 1736 | off += roundup_value_size;
|
| 1737 | }
|
| 1738 | } else {
|
| 1739 | void *src = htab_elem_value(l, map->key_size);
|
| 1740 |
|
| 1741 | if (flags & BPF_F_LOCK)
|
| 1742 | copy_map_value_locked(map, value, src, true);
|
| 1743 | else
|
| 1744 | copy_map_value(map, value, src);
|
| 1745 |
|
| 1746 | check_and_init_map_value(map, value);
|
| 1747 | }
|
| 1748 | hlist_nulls_del_rcu(&l->hash_node);
|
| 1749 |
|
| 1750 | out_unlock:
|
| 1751 | htab_unlock_bucket(b, bflags);
|
| 1752 |
|
| 1753 | if (l) {
|
| 1754 | if (is_lru_map)
|
| 1755 | htab_lru_push_free(htab, l);
|
| 1756 | else
|
| 1757 | free_htab_elem(htab, l);
|
| 1758 | }
|
| 1759 |
|
| 1760 | return ret;
|
| 1761 | }
|
| 1762 |
|
| 1763 | static int htab_map_lookup_and_delete_elem(struct bpf_map *map, void *key,
|
| 1764 | void *value, u64 flags)
|
| 1765 | {
|
| 1766 | return __htab_map_lookup_and_delete_elem(map, key, value, false, false,
|
| 1767 | flags);
|
| 1768 | }
|
| 1769 |
|
| 1770 | static int htab_percpu_map_lookup_and_delete_elem(struct bpf_map *map,
|
| 1771 | void *key, void *value,
|
| 1772 | u64 flags)
|
| 1773 | {
|
| 1774 | return __htab_map_lookup_and_delete_elem(map, key, value, false, true,
|
| 1775 | flags);
|
| 1776 | }
|
| 1777 |
|
| 1778 | static int htab_lru_map_lookup_and_delete_elem(struct bpf_map *map, void *key,
|
| 1779 | void *value, u64 flags)
|
| 1780 | {
|
| 1781 | return __htab_map_lookup_and_delete_elem(map, key, value, true, false,
|
| 1782 | flags);
|
| 1783 | }
|
| 1784 |
|
| 1785 | static int htab_lru_percpu_map_lookup_and_delete_elem(struct bpf_map *map,
|
| 1786 | void *key, void *value,
|
| 1787 | u64 flags)
|
| 1788 | {
|
| 1789 | return __htab_map_lookup_and_delete_elem(map, key, value, true, true,
|
| 1790 | flags);
|
| 1791 | }
|
| 1792 |
|
| 1793 | |
| 1794 | * Max consecutive empty buckets to walk in one RCU +
|
| 1795 | * instrumentation-disabled section before rescheduling.
|
| 1796 |
|
| 1797 | #define HTAB_BATCH_EMPTY_RESCHED 64
|
| 1798 |
|
| 1799 | static int
|
| 1800 | __htab_map_lookup_and_delete_batch(struct bpf_map *map,
|
| 1801 | const union bpf_attr *attr,
|
| 1802 | union bpf_attr __user *uattr,
|
| 1803 | bool do_delete, bool is_lru_map,
|
| 1804 | bool is_percpu)
|
| 1805 | {
|
| 1806 | struct bpf_htab *htab = container_of(map, struct bpf_htab, map);
|
| 1807 | void *keys = NULL, *values = NULL, *value, *dst_key, *dst_val;
|
| 1808 | void __user *uvalues = u64_to_user_ptr(attr->batch.values);
|
| 1809 | void __user *ukeys = u64_to_user_ptr(attr->batch.keys);
|
| 1810 | void __user *ubatch = u64_to_user_ptr(attr->batch.in_batch);
|
| 1811 | u32 batch, max_count, size, bucket_size, map_id;
|
| 1812 | u64 elem_map_flags, map_flags, allowed_flags;
|
| 1813 | u32 bucket_cnt, total, key_size, value_size;
|
| 1814 | struct htab_elem *node_to_free = NULL;
|
| 1815 | struct hlist_nulls_head *head;
|
| 1816 | struct hlist_nulls_node *n;
|
| 1817 | unsigned long flags = 0;
|
| 1818 | bool locked = false;
|
| 1819 | struct htab_elem *l;
|
| 1820 | u32 empty_cnt = 0;
|
| 1821 | struct bucket *b;
|
| 1822 | int ret = 0;
|
| 1823 |
|
| 1824 | elem_map_flags = attr->batch.elem_flags;
|
| 1825 | allowed_flags = BPF_F_LOCK;
|
| 1826 | if (!do_delete && is_percpu)
|
| 1827 | allowed_flags |= BPF_F_CPU;
|
| 1828 | ret = bpf_map_check_op_flags(map, elem_map_flags, allowed_flags);
|
| 1829 | if (ret)
|
| 1830 | return ret;
|
| 1831 |
|
| 1832 | map_flags = attr->batch.flags;
|
| 1833 | if (map_flags)
|
| 1834 | return -EINVAL;
|
| 1835 |
|
| 1836 | max_count = attr->batch.count;
|
| 1837 | if (!max_count)
|
| 1838 | return 0;
|
| 1839 |
|
| 1840 | if (put_user(0, &uattr->batch.count))
|
| 1841 | return -EFAULT;
|
| 1842 |
|
| 1843 | batch = 0;
|
| 1844 | if (ubatch && copy_from_user(&batch, ubatch, sizeof(batch)))
|
| 1845 | return -EFAULT;
|
| 1846 |
|
| 1847 | if (batch >= htab->n_buckets)
|
| 1848 | return -ENOENT;
|
| 1849 |
|
| 1850 | key_size = htab->map.key_size;
|
| 1851 | value_size = htab->map.value_size;
|
| 1852 | size = round_up(value_size, 8);
|
| 1853 | if (is_percpu && !(elem_map_flags & BPF_F_CPU))
|
| 1854 | value_size = size * num_possible_cpus();
|
| 1855 | total = 0;
|
| 1856 | |
| 1857 | * 1000, it was observed that a bucket can have up to 5 entries.
|
| 1858 |
|
| 1859 | bucket_size = 5;
|
| 1860 |
|
| 1861 | alloc:
|
| 1862 | |
| 1863 | * the rcu_read_lock. Allocate enough space here.
|
| 1864 |
|
| 1865 | keys = kvmalloc_array(key_size, bucket_size, GFP_USER | __GFP_NOWARN);
|
| 1866 | values = kvmalloc_array(value_size, bucket_size, GFP_USER | __GFP_NOWARN);
|
| 1867 | if (!keys || !values) {
|
| 1868 | ret = -ENOMEM;
|
| 1869 | goto after_loop;
|
| 1870 | }
|
| 1871 |
|
| 1872 | again:
|
| 1873 | bpf_disable_instrumentation();
|
| 1874 | rcu_read_lock();
|
| 1875 | again_nocopy:
|
| 1876 | dst_key = keys;
|
| 1877 | dst_val = values;
|
| 1878 | b = &htab->buckets[batch];
|
| 1879 | head = &b->head;
|
| 1880 |
|
| 1881 | if (locked) {
|
| 1882 | ret = htab_lock_bucket(b, &flags);
|
| 1883 | if (ret) {
|
| 1884 | rcu_read_unlock();
|
| 1885 | bpf_enable_instrumentation();
|
| 1886 | goto after_loop;
|
| 1887 | }
|
| 1888 | }
|
| 1889 |
|
| 1890 | bucket_cnt = 0;
|
| 1891 | hlist_nulls_for_each_entry_rcu(l, n, head, hash_node)
|
| 1892 | bucket_cnt++;
|
| 1893 |
|
| 1894 | if (bucket_cnt && !locked) {
|
| 1895 | locked = true;
|
| 1896 | goto again_nocopy;
|
| 1897 | }
|
| 1898 |
|
| 1899 | if (bucket_cnt > (max_count - total)) {
|
| 1900 | if (total == 0)
|
| 1901 | ret = -ENOSPC;
|
| 1902 | |
| 1903 | * that the locked was grabbed, so release it.
|
| 1904 |
|
| 1905 | htab_unlock_bucket(b, flags);
|
| 1906 | rcu_read_unlock();
|
| 1907 | bpf_enable_instrumentation();
|
| 1908 | goto after_loop;
|
| 1909 | }
|
| 1910 |
|
| 1911 | if (bucket_cnt > bucket_size) {
|
| 1912 | bucket_size = bucket_cnt;
|
| 1913 | |
| 1914 | * that the locked was grabbed, so release it.
|
| 1915 |
|
| 1916 | htab_unlock_bucket(b, flags);
|
| 1917 | rcu_read_unlock();
|
| 1918 | bpf_enable_instrumentation();
|
| 1919 | kvfree(keys);
|
| 1920 | kvfree(values);
|
| 1921 | goto alloc;
|
| 1922 | }
|
| 1923 |
|
| 1924 |
|
| 1925 | if (!locked)
|
| 1926 | goto next_batch;
|
| 1927 |
|
| 1928 | hlist_nulls_for_each_entry_safe(l, n, head, hash_node) {
|
| 1929 | memcpy(dst_key, l->key, key_size);
|
| 1930 |
|
| 1931 | if (is_percpu) {
|
| 1932 | int off = 0, cpu;
|
| 1933 | void __percpu *pptr;
|
| 1934 |
|
| 1935 | pptr = htab_elem_get_ptr(l, map->key_size);
|
| 1936 | if (elem_map_flags & BPF_F_CPU) {
|
| 1937 | cpu = elem_map_flags >> 32;
|
| 1938 | copy_map_value(&htab->map, dst_val, per_cpu_ptr(pptr, cpu));
|
| 1939 | check_and_init_map_value(&htab->map, dst_val);
|
| 1940 | } else {
|
| 1941 | for_each_possible_cpu(cpu) {
|
| 1942 | copy_map_value_long(&htab->map, dst_val + off,
|
| 1943 | per_cpu_ptr(pptr, cpu));
|
| 1944 | check_and_init_map_value(&htab->map, dst_val + off);
|
| 1945 | off += size;
|
| 1946 | }
|
| 1947 | }
|
| 1948 | } else {
|
| 1949 | value = htab_elem_value(l, key_size);
|
| 1950 | if (is_fd_htab(htab)) {
|
| 1951 | struct bpf_map **inner_map = value;
|
| 1952 |
|
| 1953 |
|
| 1954 | map_id = map->ops->map_fd_sys_lookup_elem(*inner_map);
|
| 1955 | value = &map_id;
|
| 1956 | }
|
| 1957 |
|
| 1958 | if (elem_map_flags & BPF_F_LOCK)
|
| 1959 | copy_map_value_locked(map, dst_val, value,
|
| 1960 | true);
|
| 1961 | else
|
| 1962 | copy_map_value(map, dst_val, value);
|
| 1963 |
|
| 1964 | check_and_init_map_value(map, dst_val);
|
| 1965 | }
|
| 1966 | if (do_delete) {
|
| 1967 | hlist_nulls_del_rcu(&l->hash_node);
|
| 1968 |
|
| 1969 | |
| 1970 | * may cause deadlock. See comments in function
|
| 1971 | * prealloc_lru_pop(). Let us do bpf_lru_push_free()
|
| 1972 | * after releasing the bucket lock.
|
| 1973 | *
|
| 1974 | * For htab of maps, htab_put_fd_value() in
|
| 1975 | * free_htab_elem() may acquire a spinlock with bucket
|
| 1976 | * lock being held and it violates the lock rule, so
|
| 1977 | * invoke free_htab_elem() after unlock as well.
|
| 1978 |
|
| 1979 | l->batch_flink = node_to_free;
|
| 1980 | node_to_free = l;
|
| 1981 | }
|
| 1982 | dst_key += key_size;
|
| 1983 | dst_val += value_size;
|
| 1984 | }
|
| 1985 |
|
| 1986 | htab_unlock_bucket(b, flags);
|
| 1987 | locked = false;
|
| 1988 |
|
| 1989 | while (node_to_free) {
|
| 1990 | l = node_to_free;
|
| 1991 | node_to_free = node_to_free->batch_flink;
|
| 1992 | if (is_lru_map)
|
| 1993 | htab_lru_push_free(htab, l);
|
| 1994 | else
|
| 1995 | free_htab_elem(htab, l);
|
| 1996 | }
|
| 1997 |
|
| 1998 | next_batch:
|
| 1999 | |
| 2000 | * If we are not copying data, we can go to next bucket and avoid
|
| 2001 | * unlocking the rcu. Bound the walk though: after
|
| 2002 | * HTAB_BATCH_EMPTY_RESCHED consecutive empty buckets, fully exit
|
| 2003 | * the critical section (no locks are held here) and reschedule.
|
| 2004 |
|
| 2005 | if (!bucket_cnt && (batch + 1 < htab->n_buckets)) {
|
| 2006 | batch++;
|
| 2007 | if (++empty_cnt < HTAB_BATCH_EMPTY_RESCHED)
|
| 2008 | goto again_nocopy;
|
| 2009 | empty_cnt = 0;
|
| 2010 | rcu_read_unlock();
|
| 2011 | bpf_enable_instrumentation();
|
| 2012 | cond_resched_tasks_rcu_qs();
|
| 2013 | goto again;
|
| 2014 | }
|
| 2015 |
|
| 2016 | rcu_read_unlock();
|
| 2017 | bpf_enable_instrumentation();
|
| 2018 | if (bucket_cnt && (copy_to_user(ukeys + (size_t)total * key_size, keys,
|
| 2019 | (size_t)key_size * bucket_cnt) ||
|
| 2020 | copy_to_user(uvalues + (size_t)total * value_size, values,
|
| 2021 | (size_t)value_size * bucket_cnt))) {
|
| 2022 | ret = -EFAULT;
|
| 2023 | goto after_loop;
|
| 2024 | }
|
| 2025 |
|
| 2026 | total += bucket_cnt;
|
| 2027 | empty_cnt = 0;
|
| 2028 | batch++;
|
| 2029 | if (batch >= htab->n_buckets) {
|
| 2030 | ret = -ENOENT;
|
| 2031 | goto after_loop;
|
| 2032 | }
|
| 2033 | cond_resched_tasks_rcu_qs();
|
| 2034 | goto again;
|
| 2035 |
|
| 2036 | after_loop:
|
| 2037 | if (ret == -EFAULT)
|
| 2038 | goto out;
|
| 2039 |
|
| 2040 |
|
| 2041 | ubatch = u64_to_user_ptr(attr->batch.out_batch);
|
| 2042 | if (copy_to_user(ubatch, &batch, sizeof(batch)) ||
|
| 2043 | put_user(total, &uattr->batch.count))
|
| 2044 | ret = -EFAULT;
|
| 2045 |
|
| 2046 | out:
|
| 2047 | kvfree(keys);
|
| 2048 | kvfree(values);
|
| 2049 | return ret;
|
| 2050 | }
|
| 2051 |
|
| 2052 | static int
|
| 2053 | htab_percpu_map_lookup_batch(struct bpf_map *map, const union bpf_attr *attr,
|
| 2054 | union bpf_attr __user *uattr)
|
| 2055 | {
|
| 2056 | return __htab_map_lookup_and_delete_batch(map, attr, uattr, false,
|
| 2057 | false, true);
|
| 2058 | }
|
| 2059 |
|
| 2060 | static int
|
| 2061 | htab_percpu_map_lookup_and_delete_batch(struct bpf_map *map,
|
| 2062 | const union bpf_attr *attr,
|
| 2063 | union bpf_attr __user *uattr)
|
| 2064 | {
|
| 2065 | return __htab_map_lookup_and_delete_batch(map, attr, uattr, true,
|
| 2066 | false, true);
|
| 2067 | }
|
| 2068 |
|
| 2069 | static int
|
| 2070 | htab_map_lookup_batch(struct bpf_map *map, const union bpf_attr *attr,
|
| 2071 | union bpf_attr __user *uattr)
|
| 2072 | {
|
| 2073 | return __htab_map_lookup_and_delete_batch(map, attr, uattr, false,
|
| 2074 | false, false);
|
| 2075 | }
|
| 2076 |
|
| 2077 | static int
|
| 2078 | htab_map_lookup_and_delete_batch(struct bpf_map *map,
|
| 2079 | const union bpf_attr *attr,
|
| 2080 | union bpf_attr __user *uattr)
|
| 2081 | {
|
| 2082 | return __htab_map_lookup_and_delete_batch(map, attr, uattr, true,
|
| 2083 | false, false);
|
| 2084 | }
|
| 2085 |
|
| 2086 | static int
|
| 2087 | htab_lru_percpu_map_lookup_batch(struct bpf_map *map,
|
| 2088 | const union bpf_attr *attr,
|
| 2089 | union bpf_attr __user *uattr)
|
| 2090 | {
|
| 2091 | return __htab_map_lookup_and_delete_batch(map, attr, uattr, false,
|
| 2092 | true, true);
|
| 2093 | }
|
| 2094 |
|
| 2095 | static int
|
| 2096 | htab_lru_percpu_map_lookup_and_delete_batch(struct bpf_map *map,
|
| 2097 | const union bpf_attr *attr,
|
| 2098 | union bpf_attr __user *uattr)
|
| 2099 | {
|
| 2100 | return __htab_map_lookup_and_delete_batch(map, attr, uattr, true,
|
| 2101 | true, true);
|
| 2102 | }
|
| 2103 |
|
| 2104 | static int
|
| 2105 | htab_lru_map_lookup_batch(struct bpf_map *map, const union bpf_attr *attr,
|
| 2106 | union bpf_attr __user *uattr)
|
| 2107 | {
|
| 2108 | return __htab_map_lookup_and_delete_batch(map, attr, uattr, false,
|
| 2109 | true, false);
|
| 2110 | }
|
| 2111 |
|
| 2112 | static int
|
| 2113 | htab_lru_map_lookup_and_delete_batch(struct bpf_map *map,
|
| 2114 | const union bpf_attr *attr,
|
| 2115 | union bpf_attr __user *uattr)
|
| 2116 | {
|
| 2117 | return __htab_map_lookup_and_delete_batch(map, attr, uattr, true,
|
| 2118 | true, false);
|
| 2119 | }
|
| 2120 |
|
| 2121 | struct bpf_iter_seq_hash_map_info {
|
| 2122 | struct bpf_map *map;
|
| 2123 | struct bpf_htab *htab;
|
| 2124 | void *percpu_value_buf; |
| 2125 | u32 bucket_id;
|
| 2126 | u32 skip_elems;
|
| 2127 | };
|
| 2128 |
|
| 2129 | static struct htab_elem *
|
| 2130 | bpf_hash_map_seq_find_next(struct bpf_iter_seq_hash_map_info *info,
|
| 2131 | struct htab_elem *prev_elem)
|
| 2132 | {
|
| 2133 | const struct bpf_htab *htab = info->htab;
|
| 2134 | u32 skip_elems = info->skip_elems;
|
| 2135 | u32 bucket_id = info->bucket_id;
|
| 2136 | struct hlist_nulls_head *head;
|
| 2137 | struct hlist_nulls_node *n;
|
| 2138 | struct htab_elem *elem;
|
| 2139 | struct bucket *b;
|
| 2140 | u32 i, count;
|
| 2141 |
|
| 2142 | if (bucket_id >= htab->n_buckets)
|
| 2143 | return NULL;
|
| 2144 |
|
| 2145 |
|
| 2146 | if (prev_elem) {
|
| 2147 | |
| 2148 | * and we won't skip elements.
|
| 2149 |
|
| 2150 | n = rcu_dereference_raw(hlist_nulls_next_rcu(&prev_elem->hash_node));
|
| 2151 | elem = hlist_nulls_entry_safe(n, struct htab_elem, hash_node);
|
| 2152 | if (elem)
|
| 2153 | return elem;
|
| 2154 |
|
| 2155 |
|
| 2156 | b = &htab->buckets[bucket_id++];
|
| 2157 | rcu_read_unlock();
|
| 2158 | skip_elems = 0;
|
| 2159 | }
|
| 2160 |
|
| 2161 | for (i = bucket_id; i < htab->n_buckets; i++) {
|
| 2162 | b = &htab->buckets[i];
|
| 2163 | rcu_read_lock();
|
| 2164 |
|
| 2165 | count = 0;
|
| 2166 | head = &b->head;
|
| 2167 | hlist_nulls_for_each_entry_rcu(elem, n, head, hash_node) {
|
| 2168 | if (count >= skip_elems) {
|
| 2169 | info->bucket_id = i;
|
| 2170 | info->skip_elems = count;
|
| 2171 | return elem;
|
| 2172 | }
|
| 2173 | count++;
|
| 2174 | }
|
| 2175 |
|
| 2176 | rcu_read_unlock();
|
| 2177 | skip_elems = 0;
|
| 2178 | }
|
| 2179 |
|
| 2180 | info->bucket_id = i;
|
| 2181 | info->skip_elems = 0;
|
| 2182 | return NULL;
|
| 2183 | }
|
| 2184 |
|
| 2185 | static void *bpf_hash_map_seq_start(struct seq_file *seq, loff_t *pos)
|
| 2186 | {
|
| 2187 | struct bpf_iter_seq_hash_map_info *info = seq->private;
|
| 2188 | struct htab_elem *elem;
|
| 2189 |
|
| 2190 | elem = bpf_hash_map_seq_find_next(info, NULL);
|
| 2191 | if (!elem)
|
| 2192 | return NULL;
|
| 2193 |
|
| 2194 | if (*pos == 0)
|
| 2195 | ++*pos;
|
| 2196 | return elem;
|
| 2197 | }
|
| 2198 |
|
| 2199 | static void *bpf_hash_map_seq_next(struct seq_file *seq, void *v, loff_t *pos)
|
| 2200 | {
|
| 2201 | struct bpf_iter_seq_hash_map_info *info = seq->private;
|
| 2202 |
|
| 2203 | ++*pos;
|
| 2204 | ++info->skip_elems;
|
| 2205 | return bpf_hash_map_seq_find_next(info, v);
|
| 2206 | }
|
| 2207 |
|
| 2208 | static int __bpf_hash_map_seq_show(struct seq_file *seq, struct htab_elem *elem)
|
| 2209 | {
|
| 2210 | struct bpf_iter_seq_hash_map_info *info = seq->private;
|
| 2211 | struct bpf_iter__bpf_map_elem ctx = {};
|
| 2212 | struct bpf_map *map = info->map;
|
| 2213 | struct bpf_iter_meta meta;
|
| 2214 | int ret = 0, off = 0, cpu;
|
| 2215 | u32 roundup_value_size;
|
| 2216 | struct bpf_prog *prog;
|
| 2217 | void __percpu *pptr;
|
| 2218 |
|
| 2219 | meta.seq = seq;
|
| 2220 | prog = bpf_iter_get_info(&meta, elem == NULL);
|
| 2221 | if (prog) {
|
| 2222 | ctx.meta = &meta;
|
| 2223 | ctx.map = info->map;
|
| 2224 | if (elem) {
|
| 2225 | ctx.key = elem->key;
|
| 2226 | if (!info->percpu_value_buf) {
|
| 2227 | ctx.value = htab_elem_value(elem, map->key_size);
|
| 2228 | } else {
|
| 2229 | roundup_value_size = round_up(map->value_size, 8);
|
| 2230 | pptr = htab_elem_get_ptr(elem, map->key_size);
|
| 2231 | for_each_possible_cpu(cpu) {
|
| 2232 | copy_map_value_long(map, info->percpu_value_buf + off,
|
| 2233 | per_cpu_ptr(pptr, cpu));
|
| 2234 | check_and_init_map_value(map, info->percpu_value_buf + off);
|
| 2235 | off += roundup_value_size;
|
| 2236 | }
|
| 2237 | ctx.value = info->percpu_value_buf;
|
| 2238 | }
|
| 2239 | }
|
| 2240 | ret = bpf_iter_run_prog(prog, &ctx);
|
| 2241 | }
|
| 2242 |
|
| 2243 | return ret;
|
| 2244 | }
|
| 2245 |
|
| 2246 | static int bpf_hash_map_seq_show(struct seq_file *seq, void *v)
|
| 2247 | {
|
| 2248 | return __bpf_hash_map_seq_show(seq, v);
|
| 2249 | }
|
| 2250 |
|
| 2251 | static void bpf_hash_map_seq_stop(struct seq_file *seq, void *v)
|
| 2252 | {
|
| 2253 | if (!v)
|
| 2254 | (void)__bpf_hash_map_seq_show(seq, NULL);
|
| 2255 | else
|
| 2256 | rcu_read_unlock();
|
| 2257 | }
|
| 2258 |
|
| 2259 | static int bpf_iter_init_hash_map(void *priv_data,
|
| 2260 | struct bpf_iter_aux_info *aux)
|
| 2261 | {
|
| 2262 | struct bpf_iter_seq_hash_map_info *seq_info = priv_data;
|
| 2263 | struct bpf_map *map = aux->map;
|
| 2264 | void *value_buf;
|
| 2265 | u32 buf_size;
|
| 2266 |
|
| 2267 | if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH ||
|
| 2268 | map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH) {
|
| 2269 | buf_size = round_up(map->value_size, 8) * num_possible_cpus();
|
| 2270 | value_buf = kmalloc(buf_size, GFP_USER | __GFP_NOWARN);
|
| 2271 | if (!value_buf)
|
| 2272 | return -ENOMEM;
|
| 2273 |
|
| 2274 | seq_info->percpu_value_buf = value_buf;
|
| 2275 | }
|
| 2276 |
|
| 2277 | bpf_map_inc_with_uref(map);
|
| 2278 | seq_info->map = map;
|
| 2279 | seq_info->htab = container_of(map, struct bpf_htab, map);
|
| 2280 | return 0;
|
| 2281 | }
|
| 2282 |
|
| 2283 | static void bpf_iter_fini_hash_map(void *priv_data)
|
| 2284 | {
|
| 2285 | struct bpf_iter_seq_hash_map_info *seq_info = priv_data;
|
| 2286 |
|
| 2287 | bpf_map_put_with_uref(seq_info->map);
|
| 2288 | kfree(seq_info->percpu_value_buf);
|
| 2289 | }
|
| 2290 |
|
| 2291 | static const struct seq_operations bpf_hash_map_seq_ops = {
|
| 2292 | .start = bpf_hash_map_seq_start,
|
| 2293 | .next = bpf_hash_map_seq_next,
|
| 2294 | .stop = bpf_hash_map_seq_stop,
|
| 2295 | .show = bpf_hash_map_seq_show,
|
| 2296 | };
|
| 2297 |
|
| 2298 | static const struct bpf_iter_seq_info iter_seq_info = {
|
| 2299 | .seq_ops = &bpf_hash_map_seq_ops,
|
| 2300 | .init_seq_private = bpf_iter_init_hash_map,
|
| 2301 | .fini_seq_private = bpf_iter_fini_hash_map,
|
| 2302 | .seq_priv_size = sizeof(struct bpf_iter_seq_hash_map_info),
|
| 2303 | };
|
| 2304 |
|
| 2305 | static long bpf_for_each_hash_elem(struct bpf_map *map, bpf_callback_t callback_fn,
|
| 2306 | void *callback_ctx, u64 flags)
|
| 2307 | {
|
| 2308 | struct bpf_htab *htab = container_of(map, struct bpf_htab, map);
|
| 2309 | struct hlist_nulls_head *head;
|
| 2310 | struct hlist_nulls_node *n;
|
| 2311 | struct htab_elem *elem;
|
| 2312 | int i, num_elems = 0;
|
| 2313 | void __percpu *pptr;
|
| 2314 | struct bucket *b;
|
| 2315 | void *key, *val;
|
| 2316 | bool is_percpu;
|
| 2317 | u64 ret = 0;
|
| 2318 |
|
| 2319 | cant_migrate();
|
| 2320 |
|
| 2321 | if (flags != 0)
|
| 2322 | return -EINVAL;
|
| 2323 |
|
| 2324 | is_percpu = htab_is_percpu(htab);
|
| 2325 |
|
| 2326 | |
| 2327 | * the same as the one seen by the bpf program with
|
| 2328 | * bpf_map_lookup_elem().
|
| 2329 |
|
| 2330 | for (i = 0; i < htab->n_buckets; i++) {
|
| 2331 | b = &htab->buckets[i];
|
| 2332 | rcu_read_lock();
|
| 2333 | head = &b->head;
|
| 2334 | hlist_nulls_for_each_entry_safe(elem, n, head, hash_node) {
|
| 2335 | key = elem->key;
|
| 2336 | if (is_percpu) {
|
| 2337 |
|
| 2338 | pptr = htab_elem_get_ptr(elem, map->key_size);
|
| 2339 | val = this_cpu_ptr(pptr);
|
| 2340 | } else {
|
| 2341 | val = htab_elem_value(elem, map->key_size);
|
| 2342 | }
|
| 2343 | num_elems++;
|
| 2344 | ret = callback_fn((u64)(long)map, (u64)(long)key,
|
| 2345 | (u64)(long)val, (u64)(long)callback_ctx, 0);
|
| 2346 |
|
| 2347 | if (ret) {
|
| 2348 | rcu_read_unlock();
|
| 2349 | goto out;
|
| 2350 | }
|
| 2351 | }
|
| 2352 | rcu_read_unlock();
|
| 2353 | }
|
| 2354 | out:
|
| 2355 | return num_elems;
|
| 2356 | }
|
| 2357 |
|
| 2358 | static u64 htab_map_mem_usage(const struct bpf_map *map)
|
| 2359 | {
|
| 2360 | struct bpf_htab *htab = container_of(map, struct bpf_htab, map);
|
| 2361 | u32 value_size = round_up(htab->map.value_size, 8);
|
| 2362 | bool prealloc = htab_is_prealloc(htab);
|
| 2363 | bool percpu = htab_is_percpu(htab);
|
| 2364 | bool lru = htab_is_lru(htab);
|
| 2365 | u64 num_entries, usage;
|
| 2366 |
|
| 2367 | usage = sizeof(struct bpf_htab) +
|
| 2368 | sizeof(struct bucket) * htab->n_buckets;
|
| 2369 |
|
| 2370 | if (prealloc) {
|
| 2371 | num_entries = map->max_entries;
|
| 2372 | if (htab_has_extra_elems(htab))
|
| 2373 | num_entries += num_possible_cpus();
|
| 2374 |
|
| 2375 | usage += htab->elem_size * num_entries;
|
| 2376 |
|
| 2377 | if (percpu)
|
| 2378 | usage += value_size * num_possible_cpus() * num_entries;
|
| 2379 | else if (!lru)
|
| 2380 | usage += sizeof(struct htab_elem *) * num_possible_cpus();
|
| 2381 | } else {
|
| 2382 | #define LLIST_NODE_SZ sizeof(struct llist_node)
|
| 2383 |
|
| 2384 | num_entries = htab->use_percpu_counter ?
|
| 2385 | percpu_counter_sum(&htab->pcount) :
|
| 2386 | atomic_read(&htab->count);
|
| 2387 | usage += (htab->elem_size + LLIST_NODE_SZ) * num_entries;
|
| 2388 | if (percpu) {
|
| 2389 | usage += (LLIST_NODE_SZ + sizeof(void *)) * num_entries;
|
| 2390 | usage += value_size * num_possible_cpus() * num_entries;
|
| 2391 | }
|
| 2392 | }
|
| 2393 | return usage;
|
| 2394 | }
|
| 2395 |
|
| 2396 | BTF_ID_LIST_SINGLE(htab_map_btf_ids, struct, bpf_htab)
|
| 2397 | const struct bpf_map_ops htab_map_ops = {
|
| 2398 | .map_meta_equal = bpf_map_meta_equal,
|
| 2399 | .map_alloc_check = htab_map_alloc_check,
|
| 2400 | .map_alloc = htab_map_alloc,
|
| 2401 | .map_free = htab_map_free,
|
| 2402 | .map_get_next_key = htab_map_get_next_key,
|
| 2403 | .map_release_uref = htab_map_free_internal_structs,
|
| 2404 | .map_lookup_elem = htab_map_lookup_elem,
|
| 2405 | .map_lookup_and_delete_elem = htab_map_lookup_and_delete_elem,
|
| 2406 | .map_update_elem = htab_map_update_elem,
|
| 2407 | .map_delete_elem = htab_map_delete_elem,
|
| 2408 | .map_gen_lookup = htab_map_gen_lookup,
|
| 2409 | .map_seq_show_elem = htab_map_seq_show_elem,
|
| 2410 | .map_set_for_each_callback_args = map_set_for_each_callback_args,
|
| 2411 | .map_for_each_callback = bpf_for_each_hash_elem,
|
| 2412 | .map_check_btf = htab_map_check_btf,
|
| 2413 | .map_mem_usage = htab_map_mem_usage,
|
| 2414 | BATCH_OPS(htab),
|
| 2415 | .map_btf_id = &htab_map_btf_ids[0],
|
| 2416 | .iter_seq_info = &iter_seq_info,
|
| 2417 | };
|
| 2418 |
|
| 2419 | const struct bpf_map_ops htab_lru_map_ops = {
|
| 2420 | .map_meta_equal = bpf_map_meta_equal,
|
| 2421 | .map_alloc_check = htab_map_alloc_check,
|
| 2422 | .map_alloc = htab_map_alloc,
|
| 2423 | .map_free = htab_map_free,
|
| 2424 | .map_get_next_key = htab_map_get_next_key,
|
| 2425 | .map_release_uref = htab_map_free_internal_structs,
|
| 2426 | .map_lookup_elem = htab_lru_map_lookup_elem,
|
| 2427 | .map_lookup_and_delete_elem = htab_lru_map_lookup_and_delete_elem,
|
| 2428 | .map_lookup_elem_sys_only = htab_lru_map_lookup_elem_sys,
|
| 2429 | .map_update_elem = htab_lru_map_update_elem,
|
| 2430 | .map_delete_elem = htab_lru_map_delete_elem,
|
| 2431 | .map_gen_lookup = htab_lru_map_gen_lookup,
|
| 2432 | .map_seq_show_elem = htab_map_seq_show_elem,
|
| 2433 | .map_set_for_each_callback_args = map_set_for_each_callback_args,
|
| 2434 | .map_for_each_callback = bpf_for_each_hash_elem,
|
| 2435 | .map_check_btf = htab_map_check_btf,
|
| 2436 | .map_mem_usage = htab_map_mem_usage,
|
| 2437 | BATCH_OPS(htab_lru),
|
| 2438 | .map_btf_id = &htab_map_btf_ids[0],
|
| 2439 | .iter_seq_info = &iter_seq_info,
|
| 2440 | };
|
| 2441 |
|
| 2442 |
|
| 2443 | static void *htab_percpu_map_lookup_elem(struct bpf_map *map, void *key)
|
| 2444 | {
|
| 2445 | struct htab_elem *l = __htab_map_lookup_elem(map, key);
|
| 2446 |
|
| 2447 | if (l)
|
| 2448 | return this_cpu_ptr(htab_elem_get_ptr(l, map->key_size));
|
| 2449 | else
|
| 2450 | return NULL;
|
| 2451 | }
|
| 2452 |
|
| 2453 |
|
| 2454 | static int htab_percpu_map_gen_lookup(struct bpf_map *map, struct bpf_insn *insn_buf)
|
| 2455 | {
|
| 2456 | struct bpf_insn *insn = insn_buf;
|
| 2457 |
|
| 2458 | if (!bpf_jit_supports_percpu_insn())
|
| 2459 | return -EOPNOTSUPP;
|
| 2460 |
|
| 2461 | BUILD_BUG_ON(!__same_type(&__htab_map_lookup_elem,
|
| 2462 | (void *(*)(struct bpf_map *map, void *key))NULL));
|
| 2463 | *insn++ = BPF_EMIT_CALL(__htab_map_lookup_elem);
|
| 2464 | *insn++ = BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 0, 3);
|
| 2465 | *insn++ = BPF_ALU64_IMM(BPF_ADD, BPF_REG_0,
|
| 2466 | offsetof(struct htab_elem, key) + roundup(map->key_size, 8));
|
| 2467 | *insn++ = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_0, 0);
|
| 2468 | *insn++ = BPF_MOV64_PERCPU_REG(BPF_REG_0, BPF_REG_0);
|
| 2469 |
|
| 2470 | return insn - insn_buf;
|
| 2471 | }
|
| 2472 |
|
| 2473 | static void *htab_percpu_map_lookup_percpu_elem(struct bpf_map *map, void *key, u32 cpu)
|
| 2474 | {
|
| 2475 | struct htab_elem *l;
|
| 2476 |
|
| 2477 | if (cpu >= nr_cpu_ids)
|
| 2478 | return NULL;
|
| 2479 |
|
| 2480 | l = __htab_map_lookup_elem(map, key);
|
| 2481 | if (l)
|
| 2482 | return per_cpu_ptr(htab_elem_get_ptr(l, map->key_size), cpu);
|
| 2483 | else
|
| 2484 | return NULL;
|
| 2485 | }
|
| 2486 |
|
| 2487 | static void *htab_lru_percpu_map_lookup_elem(struct bpf_map *map, void *key)
|
| 2488 | {
|
| 2489 | struct htab_elem *l = __htab_map_lookup_elem(map, key);
|
| 2490 |
|
| 2491 | if (l) {
|
| 2492 | bpf_lru_node_set_ref(&l->lru_node);
|
| 2493 | return this_cpu_ptr(htab_elem_get_ptr(l, map->key_size));
|
| 2494 | }
|
| 2495 |
|
| 2496 | return NULL;
|
| 2497 | }
|
| 2498 |
|
| 2499 | static void *htab_lru_percpu_map_lookup_percpu_elem(struct bpf_map *map, void *key, u32 cpu)
|
| 2500 | {
|
| 2501 | struct htab_elem *l;
|
| 2502 |
|
| 2503 | if (cpu >= nr_cpu_ids)
|
| 2504 | return NULL;
|
| 2505 |
|
| 2506 | l = __htab_map_lookup_elem(map, key);
|
| 2507 | if (l) {
|
| 2508 | bpf_lru_node_set_ref(&l->lru_node);
|
| 2509 | return per_cpu_ptr(htab_elem_get_ptr(l, map->key_size), cpu);
|
| 2510 | }
|
| 2511 |
|
| 2512 | return NULL;
|
| 2513 | }
|
| 2514 |
|
| 2515 | int bpf_percpu_hash_copy(struct bpf_map *map, void *key, void *value, u64 map_flags)
|
| 2516 | {
|
| 2517 | struct htab_elem *l;
|
| 2518 | void __percpu *pptr;
|
| 2519 | int ret = -ENOENT;
|
| 2520 | int cpu, off = 0;
|
| 2521 | u32 size;
|
| 2522 |
|
| 2523 | |
| 2524 | * access 'value_size' of them, so copying rounded areas
|
| 2525 | * will not leak any kernel data
|
| 2526 |
|
| 2527 | size = round_up(map->value_size, 8);
|
| 2528 | rcu_read_lock();
|
| 2529 | l = __htab_map_lookup_elem(map, key);
|
| 2530 | if (!l)
|
| 2531 | goto out;
|
| 2532 | ret = 0;
|
| 2533 | |
| 2534 | * eviction heuristics when user space does a map walk.
|
| 2535 |
|
| 2536 | pptr = htab_elem_get_ptr(l, map->key_size);
|
| 2537 | if (map_flags & BPF_F_CPU) {
|
| 2538 | cpu = map_flags >> 32;
|
| 2539 | copy_map_value(map, value, per_cpu_ptr(pptr, cpu));
|
| 2540 | check_and_init_map_value(map, value);
|
| 2541 | goto out;
|
| 2542 | }
|
| 2543 | for_each_possible_cpu(cpu) {
|
| 2544 | copy_map_value_long(map, value + off, per_cpu_ptr(pptr, cpu));
|
| 2545 | check_and_init_map_value(map, value + off);
|
| 2546 | off += size;
|
| 2547 | }
|
| 2548 | out:
|
| 2549 | rcu_read_unlock();
|
| 2550 | return ret;
|
| 2551 | }
|
| 2552 |
|
| 2553 | int bpf_percpu_hash_update(struct bpf_map *map, void *key, void *value,
|
| 2554 | u64 map_flags)
|
| 2555 | {
|
| 2556 | struct bpf_htab *htab = container_of(map, struct bpf_htab, map);
|
| 2557 | int ret;
|
| 2558 |
|
| 2559 | rcu_read_lock();
|
| 2560 | if (htab_is_lru(htab))
|
| 2561 | ret = __htab_lru_percpu_map_update_elem(map, key, value,
|
| 2562 | map_flags, true);
|
| 2563 | else
|
| 2564 | ret = htab_map_update_elem_in_place(map, key, value, map_flags,
|
| 2565 | true, true);
|
| 2566 | rcu_read_unlock();
|
| 2567 |
|
| 2568 | return ret;
|
| 2569 | }
|
| 2570 |
|
| 2571 | static void htab_percpu_map_seq_show_elem(struct bpf_map *map, void *key,
|
| 2572 | struct seq_file *m)
|
| 2573 | {
|
| 2574 | struct htab_elem *l;
|
| 2575 | void __percpu *pptr;
|
| 2576 | int cpu;
|
| 2577 |
|
| 2578 | rcu_read_lock();
|
| 2579 |
|
| 2580 | l = __htab_map_lookup_elem(map, key);
|
| 2581 | if (!l) {
|
| 2582 | rcu_read_unlock();
|
| 2583 | return;
|
| 2584 | }
|
| 2585 |
|
| 2586 | btf_type_seq_show(map->btf, map->btf_key_type_id, key, m);
|
| 2587 | seq_puts(m, ": {\n");
|
| 2588 | pptr = htab_elem_get_ptr(l, map->key_size);
|
| 2589 | for_each_possible_cpu(cpu) {
|
| 2590 | seq_printf(m, "\tcpu%d: ", cpu);
|
| 2591 | btf_type_seq_show(map->btf, map->btf_value_type_id,
|
| 2592 | per_cpu_ptr(pptr, cpu), m);
|
| 2593 | seq_putc(m, '\n');
|
| 2594 | }
|
| 2595 | seq_puts(m, "}\n");
|
| 2596 |
|
| 2597 | rcu_read_unlock();
|
| 2598 | }
|
| 2599 |
|
| 2600 | const struct bpf_map_ops htab_percpu_map_ops = {
|
| 2601 | .map_meta_equal = bpf_map_meta_equal,
|
| 2602 | .map_alloc_check = htab_map_alloc_check,
|
| 2603 | .map_alloc = htab_map_alloc,
|
| 2604 | .map_free = htab_map_free,
|
| 2605 | .map_get_next_key = htab_map_get_next_key,
|
| 2606 | .map_lookup_elem = htab_percpu_map_lookup_elem,
|
| 2607 | .map_gen_lookup = htab_percpu_map_gen_lookup,
|
| 2608 | .map_lookup_and_delete_elem = htab_percpu_map_lookup_and_delete_elem,
|
| 2609 | .map_update_elem = htab_percpu_map_update_elem,
|
| 2610 | .map_delete_elem = htab_map_delete_elem,
|
| 2611 | .map_lookup_percpu_elem = htab_percpu_map_lookup_percpu_elem,
|
| 2612 | .map_seq_show_elem = htab_percpu_map_seq_show_elem,
|
| 2613 | .map_set_for_each_callback_args = map_set_for_each_callback_args,
|
| 2614 | .map_for_each_callback = bpf_for_each_hash_elem,
|
| 2615 | .map_check_btf = htab_map_check_btf,
|
| 2616 | .map_mem_usage = htab_map_mem_usage,
|
| 2617 | BATCH_OPS(htab_percpu),
|
| 2618 | .map_btf_id = &htab_map_btf_ids[0],
|
| 2619 | .iter_seq_info = &iter_seq_info,
|
| 2620 | };
|
| 2621 |
|
| 2622 | const struct bpf_map_ops htab_lru_percpu_map_ops = {
|
| 2623 | .map_meta_equal = bpf_map_meta_equal,
|
| 2624 | .map_alloc_check = htab_map_alloc_check,
|
| 2625 | .map_alloc = htab_map_alloc,
|
| 2626 | .map_free = htab_map_free,
|
| 2627 | .map_get_next_key = htab_map_get_next_key,
|
| 2628 | .map_lookup_elem = htab_lru_percpu_map_lookup_elem,
|
| 2629 | .map_lookup_and_delete_elem = htab_lru_percpu_map_lookup_and_delete_elem,
|
| 2630 | .map_update_elem = htab_lru_percpu_map_update_elem,
|
| 2631 | .map_delete_elem = htab_lru_map_delete_elem,
|
| 2632 | .map_lookup_percpu_elem = htab_lru_percpu_map_lookup_percpu_elem,
|
| 2633 | .map_seq_show_elem = htab_percpu_map_seq_show_elem,
|
| 2634 | .map_set_for_each_callback_args = map_set_for_each_callback_args,
|
| 2635 | .map_for_each_callback = bpf_for_each_hash_elem,
|
| 2636 | .map_check_btf = htab_map_check_btf,
|
| 2637 | .map_mem_usage = htab_map_mem_usage,
|
| 2638 | BATCH_OPS(htab_lru_percpu),
|
| 2639 | .map_btf_id = &htab_map_btf_ids[0],
|
| 2640 | .iter_seq_info = &iter_seq_info,
|
| 2641 | };
|
| 2642 |
|
| 2643 | static int fd_htab_map_alloc_check(union bpf_attr *attr)
|
| 2644 | {
|
| 2645 | if (attr->value_size != sizeof(u32))
|
| 2646 | return -EINVAL;
|
| 2647 | return htab_map_alloc_check(attr);
|
| 2648 | }
|
| 2649 |
|
| 2650 | static void fd_htab_map_free(struct bpf_map *map)
|
| 2651 | {
|
| 2652 | struct bpf_htab *htab = container_of(map, struct bpf_htab, map);
|
| 2653 | struct hlist_nulls_node *n;
|
| 2654 | struct hlist_nulls_head *head;
|
| 2655 | struct htab_elem *l;
|
| 2656 | int i;
|
| 2657 |
|
| 2658 | for (i = 0; i < htab->n_buckets; i++) {
|
| 2659 | head = select_bucket(htab, i);
|
| 2660 |
|
| 2661 | hlist_nulls_for_each_entry_safe(l, n, head, hash_node) {
|
| 2662 | void *ptr = fd_htab_map_get_ptr(map, l);
|
| 2663 |
|
| 2664 | map->ops->map_fd_put_ptr(map, ptr, false);
|
| 2665 | }
|
| 2666 | }
|
| 2667 |
|
| 2668 | htab_map_free(map);
|
| 2669 | }
|
| 2670 |
|
| 2671 |
|
| 2672 | int bpf_fd_htab_map_lookup_elem(struct bpf_map *map, void *key, u32 *value)
|
| 2673 | {
|
| 2674 | void **ptr;
|
| 2675 | int ret = 0;
|
| 2676 |
|
| 2677 | if (!map->ops->map_fd_sys_lookup_elem)
|
| 2678 | return -ENOTSUPP;
|
| 2679 |
|
| 2680 | rcu_read_lock();
|
| 2681 | ptr = htab_map_lookup_elem(map, key);
|
| 2682 | if (ptr)
|
| 2683 | *value = map->ops->map_fd_sys_lookup_elem(READ_ONCE(*ptr));
|
| 2684 | else
|
| 2685 | ret = -ENOENT;
|
| 2686 | rcu_read_unlock();
|
| 2687 |
|
| 2688 | return ret;
|
| 2689 | }
|
| 2690 |
|
| 2691 |
|
| 2692 | int bpf_fd_htab_map_update_elem(struct bpf_map *map, struct file *map_file,
|
| 2693 | void *key, void *value, u64 map_flags)
|
| 2694 | {
|
| 2695 | void *ptr;
|
| 2696 | int ret;
|
| 2697 |
|
| 2698 | ptr = map->ops->map_fd_get_ptr(map, map_file, *(int *)value);
|
| 2699 | if (IS_ERR(ptr))
|
| 2700 | return PTR_ERR(ptr);
|
| 2701 |
|
| 2702 | |
| 2703 | * htab map, and the following rcu_read_lock() is only used to avoid
|
| 2704 | * the WARN_ON_ONCE in htab_map_update_elem_in_place().
|
| 2705 |
|
| 2706 | rcu_read_lock();
|
| 2707 | ret = htab_map_update_elem_in_place(map, key, &ptr, map_flags, false, false);
|
| 2708 | rcu_read_unlock();
|
| 2709 | if (ret)
|
| 2710 | map->ops->map_fd_put_ptr(map, ptr, false);
|
| 2711 |
|
| 2712 | return ret;
|
| 2713 | }
|
| 2714 |
|
| 2715 | static struct bpf_map *htab_of_map_alloc(union bpf_attr *attr)
|
| 2716 | {
|
| 2717 | struct bpf_map *map, *inner_map_meta;
|
| 2718 |
|
| 2719 | inner_map_meta = bpf_map_meta_alloc(attr->inner_map_fd);
|
| 2720 | if (IS_ERR(inner_map_meta))
|
| 2721 | return inner_map_meta;
|
| 2722 |
|
| 2723 | map = htab_map_alloc(attr);
|
| 2724 | if (IS_ERR(map)) {
|
| 2725 | bpf_map_meta_free(inner_map_meta);
|
| 2726 | return map;
|
| 2727 | }
|
| 2728 |
|
| 2729 | map->inner_map_meta = inner_map_meta;
|
| 2730 |
|
| 2731 | return map;
|
| 2732 | }
|
| 2733 |
|
| 2734 | static void *htab_of_map_lookup_elem(struct bpf_map *map, void *key)
|
| 2735 | {
|
| 2736 | struct bpf_map **inner_map = htab_map_lookup_elem(map, key);
|
| 2737 |
|
| 2738 | if (!inner_map)
|
| 2739 | return NULL;
|
| 2740 |
|
| 2741 | return READ_ONCE(*inner_map);
|
| 2742 | }
|
| 2743 |
|
| 2744 | static int htab_of_map_gen_lookup(struct bpf_map *map,
|
| 2745 | struct bpf_insn *insn_buf)
|
| 2746 | {
|
| 2747 | struct bpf_insn *insn = insn_buf;
|
| 2748 | const int ret = BPF_REG_0;
|
| 2749 |
|
| 2750 | BUILD_BUG_ON(!__same_type(&__htab_map_lookup_elem,
|
| 2751 | (void *(*)(struct bpf_map *map, void *key))NULL));
|
| 2752 | *insn++ = BPF_EMIT_CALL(__htab_map_lookup_elem);
|
| 2753 | *insn++ = BPF_JMP_IMM(BPF_JEQ, ret, 0, 2);
|
| 2754 | *insn++ = BPF_ALU64_IMM(BPF_ADD, ret,
|
| 2755 | offsetof(struct htab_elem, key) +
|
| 2756 | round_up(map->key_size, 8));
|
| 2757 | *insn++ = BPF_LDX_MEM(BPF_DW, ret, ret, 0);
|
| 2758 |
|
| 2759 | return insn - insn_buf;
|
| 2760 | }
|
| 2761 |
|
| 2762 | static void htab_of_map_free(struct bpf_map *map)
|
| 2763 | {
|
| 2764 | bpf_map_meta_free(map->inner_map_meta);
|
| 2765 | fd_htab_map_free(map);
|
| 2766 | }
|
| 2767 |
|
| 2768 | const struct bpf_map_ops htab_of_maps_map_ops = {
|
| 2769 | .map_alloc_check = fd_htab_map_alloc_check,
|
| 2770 | .map_alloc = htab_of_map_alloc,
|
| 2771 | .map_free = htab_of_map_free,
|
| 2772 | .map_get_next_key = htab_map_get_next_key,
|
| 2773 | .map_lookup_elem = htab_of_map_lookup_elem,
|
| 2774 | .map_delete_elem = htab_map_delete_elem,
|
| 2775 | .map_fd_get_ptr = bpf_map_fd_get_ptr,
|
| 2776 | .map_fd_put_ptr = bpf_map_fd_put_ptr,
|
| 2777 | .map_fd_sys_lookup_elem = bpf_map_fd_sys_lookup_elem,
|
| 2778 | .map_gen_lookup = htab_of_map_gen_lookup,
|
| 2779 | .map_check_btf = map_check_no_btf,
|
| 2780 | .map_mem_usage = htab_map_mem_usage,
|
| 2781 | BATCH_OPS(htab),
|
| 2782 | .map_btf_id = &htab_map_btf_ids[0],
|
| 2783 | };
|
| 2784 |
|
| 2785 | struct rhtab_elem {
|
| 2786 | struct rhash_head node;
|
| 2787 |
|
| 2788 | u8 data[] __aligned(8);
|
| 2789 | };
|
| 2790 |
|
| 2791 | struct bpf_rhtab {
|
| 2792 | struct bpf_map map;
|
| 2793 | struct rhashtable ht;
|
| 2794 | struct bpf_mem_alloc ma;
|
| 2795 | u32 elem_size;
|
| 2796 | bool freeing_internal;
|
| 2797 | };
|
| 2798 |
|
| 2799 | static const struct rhashtable_params rhtab_params = {
|
| 2800 | .head_offset = offsetof(struct rhtab_elem, node),
|
| 2801 | .key_offset = offsetof(struct rhtab_elem, data),
|
| 2802 | };
|
| 2803 |
|
| 2804 | static inline void *rhtab_elem_value(struct rhtab_elem *l, u32 key_size)
|
| 2805 | {
|
| 2806 | return l->data + round_up(key_size, 8);
|
| 2807 | }
|
| 2808 |
|
| 2809 |
|
| 2810 | static __always_inline int rhtab_key_cmp_long(struct rhashtable_compare_arg *arg,
|
| 2811 | const void *ptr)
|
| 2812 | {
|
| 2813 | const unsigned long key1 = *(const unsigned long *)arg->key;
|
| 2814 | const struct rhtab_elem *key2 = ptr;
|
| 2815 |
|
| 2816 | return key1 != *(const unsigned long *)key2->data;
|
| 2817 | }
|
| 2818 |
|
| 2819 | static __always_inline u32 rhtab_hashfn_long(const void *data, u32 len, u32 seed)
|
| 2820 | {
|
| 2821 | u64 k = *(const unsigned long *)data;
|
| 2822 |
|
| 2823 | return (u32)(k ^ (k >> 32)) ^ seed;
|
| 2824 | }
|
| 2825 |
|
| 2826 | static const struct rhashtable_params rhtab_params_long = {
|
| 2827 | .head_offset = offsetof(struct rhtab_elem, node),
|
| 2828 | .key_offset = offsetof(struct rhtab_elem, data),
|
| 2829 | .key_len = sizeof(long),
|
| 2830 | .hashfn = rhtab_hashfn_long,
|
| 2831 | .obj_cmpfn = rhtab_key_cmp_long,
|
| 2832 | };
|
| 2833 |
|
| 2834 | static struct bpf_map *rhtab_map_alloc(union bpf_attr *attr)
|
| 2835 | {
|
| 2836 | struct rhashtable_params params;
|
| 2837 | struct bpf_rhtab *rhtab;
|
| 2838 | int err = 0;
|
| 2839 |
|
| 2840 | rhtab = bpf_map_area_alloc(sizeof(*rhtab), NUMA_NO_NODE);
|
| 2841 | if (!rhtab)
|
| 2842 | return ERR_PTR(-ENOMEM);
|
| 2843 |
|
| 2844 | bpf_map_init_from_attr(&rhtab->map, attr);
|
| 2845 |
|
| 2846 | if (rhtab->map.max_entries > 1UL << 31) {
|
| 2847 | err = -E2BIG;
|
| 2848 | goto free_rhtab;
|
| 2849 | }
|
| 2850 |
|
| 2851 | rhtab->elem_size = sizeof(struct rhtab_elem) + round_up(rhtab->map.key_size, 8) +
|
| 2852 | round_up(rhtab->map.value_size, 8);
|
| 2853 |
|
| 2854 | params = rhtab_params;
|
| 2855 | params.key_len = rhtab->map.key_size;
|
| 2856 | params.nelem_hint = (u32)attr->map_extra;
|
| 2857 | params.automatic_shrinking = true;
|
| 2858 |
|
| 2859 | if (rhtab->map.key_size == sizeof(long)) {
|
| 2860 | params.hashfn = rhtab_hashfn_long;
|
| 2861 | params.obj_cmpfn = rhtab_key_cmp_long;
|
| 2862 | }
|
| 2863 |
|
| 2864 | err = rhashtable_init(&rhtab->ht, ¶ms);
|
| 2865 | if (err)
|
| 2866 | goto free_rhtab;
|
| 2867 |
|
| 2868 |
|
| 2869 | rhtab->ht.max_elems = rhtab->map.max_entries;
|
| 2870 |
|
| 2871 | err = bpf_mem_alloc_init(&rhtab->ma, rhtab->elem_size, false);
|
| 2872 | if (err)
|
| 2873 | goto destroy_rhtab;
|
| 2874 |
|
| 2875 | return &rhtab->map;
|
| 2876 |
|
| 2877 | destroy_rhtab:
|
| 2878 | rhashtable_destroy(&rhtab->ht);
|
| 2879 | free_rhtab:
|
| 2880 | bpf_map_area_free(rhtab);
|
| 2881 | return ERR_PTR(err);
|
| 2882 | }
|
| 2883 |
|
| 2884 | static int rhtab_map_alloc_check(union bpf_attr *attr)
|
| 2885 | {
|
| 2886 | if (!(attr->map_flags & BPF_F_NO_PREALLOC))
|
| 2887 | return -EINVAL;
|
| 2888 |
|
| 2889 | if (attr->map_flags & BPF_F_ZERO_SEED)
|
| 2890 | return -EINVAL;
|
| 2891 |
|
| 2892 | if (attr->key_size > U16_MAX)
|
| 2893 | return -E2BIG;
|
| 2894 |
|
| 2895 | if (attr->map_extra >> 32)
|
| 2896 | return -EINVAL;
|
| 2897 |
|
| 2898 | if ((u32)attr->map_extra > U16_MAX)
|
| 2899 | return -E2BIG;
|
| 2900 |
|
| 2901 | if ((u32)attr->map_extra > attr->max_entries)
|
| 2902 | return -EINVAL;
|
| 2903 |
|
| 2904 | return htab_map_alloc_check(attr);
|
| 2905 | }
|
| 2906 |
|
| 2907 | static void rhtab_mem_dtor(void *obj, void *ctx)
|
| 2908 | {
|
| 2909 | struct htab_btf_record *hrec = ctx;
|
| 2910 | struct rhtab_elem *elem = obj;
|
| 2911 |
|
| 2912 | if (IS_ERR_OR_NULL(hrec->record))
|
| 2913 | return;
|
| 2914 |
|
| 2915 | bpf_obj_free_fields(hrec->record,
|
| 2916 | rhtab_elem_value(elem, hrec->key_size));
|
| 2917 | }
|
| 2918 |
|
| 2919 | static void rhtab_free_elem(void *ptr, void *arg)
|
| 2920 | {
|
| 2921 | struct bpf_rhtab *rhtab = arg;
|
| 2922 | struct rhtab_elem *elem = ptr;
|
| 2923 |
|
| 2924 | bpf_map_free_internal_structs(&rhtab->map, rhtab_elem_value(elem, rhtab->map.key_size));
|
| 2925 | bpf_mem_cache_free_rcu(&rhtab->ma, elem);
|
| 2926 | }
|
| 2927 |
|
| 2928 | static void rhtab_map_free(struct bpf_map *map)
|
| 2929 | {
|
| 2930 | struct bpf_rhtab *rhtab = container_of(map, struct bpf_rhtab, map);
|
| 2931 |
|
| 2932 | rhashtable_free_and_destroy(&rhtab->ht, rhtab_free_elem, rhtab);
|
| 2933 | bpf_mem_alloc_destroy(&rhtab->ma);
|
| 2934 | bpf_map_area_free(rhtab);
|
| 2935 | }
|
| 2936 |
|
| 2937 | static void *rhtab_lookup_elem(struct bpf_map *map, void *key)
|
| 2938 | {
|
| 2939 | struct bpf_rhtab *rhtab = container_of(map, struct bpf_rhtab, map);
|
| 2940 |
|
| 2941 |
|
| 2942 | guard(rcu)();
|
| 2943 |
|
| 2944 | if (map->key_size == sizeof(long))
|
| 2945 | return rhashtable_lookup_likely(&rhtab->ht, key, rhtab_params_long);
|
| 2946 |
|
| 2947 | return rhashtable_lookup_likely(&rhtab->ht, key, rhtab_params);
|
| 2948 | }
|
| 2949 |
|
| 2950 | static void *rhtab_map_lookup_elem(struct bpf_map *map, void *key) __must_hold(RCU)
|
| 2951 | {
|
| 2952 | struct rhtab_elem *l;
|
| 2953 |
|
| 2954 | l = rhtab_lookup_elem(map, key);
|
| 2955 | return l ? rhtab_elem_value(l, map->key_size) : NULL;
|
| 2956 | }
|
| 2957 |
|
| 2958 | static void rhtab_read_elem_value(struct bpf_map *map, void *dst, struct rhtab_elem *elem,
|
| 2959 | u64 flags)
|
| 2960 | {
|
| 2961 | void *src = rhtab_elem_value(elem, map->key_size);
|
| 2962 |
|
| 2963 | if (flags & BPF_F_LOCK)
|
| 2964 | copy_map_value_locked(map, dst, src, true);
|
| 2965 | else
|
| 2966 | copy_map_value(map, dst, src);
|
| 2967 | }
|
| 2968 |
|
| 2969 | static int rhtab_delete_elem(struct bpf_rhtab *rhtab, struct rhtab_elem *elem, void *copy,
|
| 2970 | u64 flags)
|
| 2971 | {
|
| 2972 | int err;
|
| 2973 |
|
| 2974 | |
| 2975 | * disable_instrumentation() mitigates the deadlock for programs running in NMI context.
|
| 2976 | * rhashtable locks bucket with local_irq_save(). Only NMI programs may reenter
|
| 2977 | * rhashtable code, bpf_disable_instrumentation() disables programs running in NMI, except
|
| 2978 | * raw tracepoints, which we don't have in rhashtable.
|
| 2979 |
|
| 2980 | bpf_disable_instrumentation();
|
| 2981 |
|
| 2982 | if (rhtab->map.key_size == sizeof(long))
|
| 2983 | err = rhashtable_remove_fast(&rhtab->ht, &elem->node, rhtab_params_long);
|
| 2984 | else
|
| 2985 | err = rhashtable_remove_fast(&rhtab->ht, &elem->node, rhtab_params);
|
| 2986 |
|
| 2987 | bpf_enable_instrumentation();
|
| 2988 |
|
| 2989 | if (err)
|
| 2990 | return err;
|
| 2991 |
|
| 2992 | if (copy) {
|
| 2993 | rhtab_read_elem_value(&rhtab->map, copy, elem, flags);
|
| 2994 | check_and_init_map_value(&rhtab->map, copy);
|
| 2995 | }
|
| 2996 | bpf_obj_cancel_fields(&rhtab->map,
|
| 2997 | rhtab_elem_value(elem, rhtab->map.key_size));
|
| 2998 | bpf_mem_cache_free_rcu(&rhtab->ma, elem);
|
| 2999 | return 0;
|
| 3000 | }
|
| 3001 |
|
| 3002 | static long rhtab_map_delete_elem(struct bpf_map *map, void *key)
|
| 3003 | {
|
| 3004 | struct bpf_rhtab *rhtab = container_of(map, struct bpf_rhtab, map);
|
| 3005 | struct rhtab_elem *elem;
|
| 3006 |
|
| 3007 | guard(rcu)();
|
| 3008 |
|
| 3009 | elem = rhtab_lookup_elem(map, key);
|
| 3010 | if (!elem)
|
| 3011 | return -ENOENT;
|
| 3012 |
|
| 3013 | return rhtab_delete_elem(rhtab, elem, NULL, 0);
|
| 3014 | }
|
| 3015 |
|
| 3016 | static int rhtab_map_lookup_and_delete_elem(struct bpf_map *map, void *key, void *value, u64 flags)
|
| 3017 | {
|
| 3018 | struct bpf_rhtab *rhtab = container_of(map, struct bpf_rhtab, map);
|
| 3019 | struct rhtab_elem *elem;
|
| 3020 | int err;
|
| 3021 |
|
| 3022 | err = bpf_map_check_op_flags(map, flags, BPF_F_LOCK);
|
| 3023 | if (err)
|
| 3024 | return err;
|
| 3025 |
|
| 3026 | guard(rcu)();
|
| 3027 |
|
| 3028 | elem = rhtab_lookup_elem(map, key);
|
| 3029 | if (!elem)
|
| 3030 | return -ENOENT;
|
| 3031 |
|
| 3032 | return rhtab_delete_elem(rhtab, elem, value, flags);
|
| 3033 | }
|
| 3034 |
|
| 3035 | static long rhtab_map_update_existing(struct bpf_map *map, struct rhtab_elem *elem, void *value,
|
| 3036 | u64 map_flags)
|
| 3037 | {
|
| 3038 | void *old_val = rhtab_elem_value(elem, map->key_size);
|
| 3039 |
|
| 3040 | if (map_flags & BPF_NOEXIST)
|
| 3041 | return -EEXIST;
|
| 3042 |
|
| 3043 | if (map_flags & BPF_F_LOCK)
|
| 3044 | copy_map_value_locked(map, old_val, value, false);
|
| 3045 | else
|
| 3046 | copy_map_value(map, old_val, value);
|
| 3047 |
|
| 3048 | |
| 3049 | * Torn reads: a concurrent reader without BPF_F_LOCK may observe
|
| 3050 | * the value mid-copy. Callers requiring consistent reads must use
|
| 3051 | * BPF_F_LOCK, matching arraymap semantics.
|
| 3052 | *
|
| 3053 | * copy_map_value() skips special-field offsets, so old timers/
|
| 3054 | * kptrs/etc. still sit in the slot. Cancel them after the copy
|
| 3055 | * to match arraymap's update semantics.
|
| 3056 |
|
| 3057 | bpf_obj_cancel_fields(map, old_val);
|
| 3058 | return 0;
|
| 3059 | }
|
| 3060 |
|
| 3061 | static long rhtab_map_update_elem(struct bpf_map *map, void *key, void *value, u64 map_flags)
|
| 3062 | {
|
| 3063 | struct bpf_rhtab *rhtab = container_of(map, struct bpf_rhtab, map);
|
| 3064 | struct rhtab_elem *elem, *tmp;
|
| 3065 |
|
| 3066 | if (unlikely((map_flags & ~BPF_F_LOCK) > BPF_EXIST))
|
| 3067 | return -EINVAL;
|
| 3068 |
|
| 3069 | if ((map_flags & BPF_F_LOCK) && !btf_record_has_field(map->record, BPF_SPIN_LOCK))
|
| 3070 | return -EINVAL;
|
| 3071 |
|
| 3072 | guard(rcu)();
|
| 3073 | elem = rhtab_lookup_elem(map, key);
|
| 3074 | if (elem)
|
| 3075 | return rhtab_map_update_existing(map, elem, value, map_flags);
|
| 3076 |
|
| 3077 | if (map_flags & BPF_EXIST)
|
| 3078 | return -ENOENT;
|
| 3079 |
|
| 3080 | |
| 3081 | * Reject new insertions while map_release_uref cleanup walks the
|
| 3082 | * table. Without this, new elements could keep triggering rehash
|
| 3083 | * and prevent the walk from terminating.
|
| 3084 |
|
| 3085 | if (READ_ONCE(rhtab->freeing_internal))
|
| 3086 | return -EBUSY;
|
| 3087 |
|
| 3088 |
|
| 3089 | if (atomic_read(&rhtab->ht.nelems) >= map->max_entries)
|
| 3090 | return -E2BIG;
|
| 3091 |
|
| 3092 | elem = bpf_mem_cache_alloc(&rhtab->ma);
|
| 3093 | if (!elem)
|
| 3094 | return -ENOMEM;
|
| 3095 |
|
| 3096 | memcpy(elem->data, key, map->key_size);
|
| 3097 | copy_map_value(map, rhtab_elem_value(elem, map->key_size), value);
|
| 3098 |
|
| 3099 |
|
| 3100 | bpf_disable_instrumentation();
|
| 3101 |
|
| 3102 | if (map->key_size == sizeof(long))
|
| 3103 | tmp = rhashtable_lookup_get_insert_fast(&rhtab->ht, &elem->node, rhtab_params_long);
|
| 3104 | else
|
| 3105 | tmp = rhashtable_lookup_get_insert_fast(&rhtab->ht, &elem->node, rhtab_params);
|
| 3106 |
|
| 3107 | bpf_enable_instrumentation();
|
| 3108 |
|
| 3109 | if (tmp) {
|
| 3110 | bpf_mem_cache_free(&rhtab->ma, elem);
|
| 3111 | if (IS_ERR(tmp))
|
| 3112 | return PTR_ERR(tmp);
|
| 3113 |
|
| 3114 | return rhtab_map_update_existing(map, tmp, value, map_flags);
|
| 3115 | }
|
| 3116 |
|
| 3117 | return 0;
|
| 3118 | }
|
| 3119 |
|
| 3120 | static int rhtab_map_gen_lookup(struct bpf_map *map, struct bpf_insn *insn_buf)
|
| 3121 | {
|
| 3122 | struct bpf_insn *insn = insn_buf;
|
| 3123 | const int ret = BPF_REG_0;
|
| 3124 |
|
| 3125 | BUILD_BUG_ON(!__same_type(&rhtab_lookup_elem,
|
| 3126 | (void *(*)(struct bpf_map *map, void *key)) NULL));
|
| 3127 | *insn++ = BPF_EMIT_CALL(rhtab_lookup_elem);
|
| 3128 | *insn++ = BPF_JMP_IMM(BPF_JEQ, ret, 0, 1);
|
| 3129 | *insn++ = BPF_ALU64_IMM(BPF_ADD, ret,
|
| 3130 | offsetof(struct rhtab_elem, data) + round_up(map->key_size, 8));
|
| 3131 |
|
| 3132 | return insn - insn_buf;
|
| 3133 | }
|
| 3134 |
|
| 3135 | static int rhtab_map_check_btf(struct bpf_map *map, const struct btf *btf,
|
| 3136 | const struct btf_type *key_type,
|
| 3137 | const struct btf_type *value_type)
|
| 3138 | {
|
| 3139 | struct bpf_rhtab *rhtab = container_of(map, struct bpf_rhtab, map);
|
| 3140 |
|
| 3141 | if (btf_type_is_void(key_type))
|
| 3142 | return -EINVAL;
|
| 3143 |
|
| 3144 | return bpf_ma_set_dtor(map, &rhtab->ma, rhtab_mem_dtor);
|
| 3145 | }
|
| 3146 |
|
| 3147 | static void rhtab_map_free_internal_structs(struct bpf_map *map)
|
| 3148 | {
|
| 3149 | struct bpf_rhtab *rhtab = container_of(map, struct bpf_rhtab, map);
|
| 3150 | struct rhashtable_iter iter;
|
| 3151 | struct rhtab_elem *elem;
|
| 3152 |
|
| 3153 | if (!bpf_map_has_internal_structs(map))
|
| 3154 | return;
|
| 3155 |
|
| 3156 | |
| 3157 | * Block new insertions. Once observed, no new growth is triggered,
|
| 3158 | * so any in-flight rehash will drain and the walker is guaranteed
|
| 3159 | * to stop returning -EAGAIN. Treat -EAGAIN as "rehash in progress,
|
| 3160 | * retry"; do not wait for the worker.
|
| 3161 |
|
| 3162 | WRITE_ONCE(rhtab->freeing_internal, true);
|
| 3163 |
|
| 3164 | rhashtable_walk_enter(&rhtab->ht, &iter);
|
| 3165 | rhashtable_walk_start(&iter);
|
| 3166 |
|
| 3167 | while ((elem = rhashtable_walk_next(&iter))) {
|
| 3168 | if (IS_ERR(elem)) {
|
| 3169 | if (PTR_ERR(elem) == -EAGAIN)
|
| 3170 | continue;
|
| 3171 | break;
|
| 3172 | }
|
| 3173 |
|
| 3174 | bpf_map_free_internal_structs(map, rhtab_elem_value(elem, map->key_size));
|
| 3175 |
|
| 3176 | if (need_resched()) {
|
| 3177 | rhashtable_walk_stop(&iter);
|
| 3178 | cond_resched();
|
| 3179 | rhashtable_walk_start(&iter);
|
| 3180 | }
|
| 3181 | }
|
| 3182 |
|
| 3183 | rhashtable_walk_stop(&iter);
|
| 3184 | rhashtable_walk_exit(&iter);
|
| 3185 | WRITE_ONCE(rhtab->freeing_internal, false);
|
| 3186 | }
|
| 3187 |
|
| 3188 | static int rhtab_map_get_next_key(struct bpf_map *map, void *key, void *next_key)
|
| 3189 | __must_hold_shared(RCU)
|
| 3190 | {
|
| 3191 | struct bpf_rhtab *rhtab = container_of(map, struct bpf_rhtab, map);
|
| 3192 | struct rhtab_elem *elem;
|
| 3193 |
|
| 3194 | elem = rhashtable_next_key(&rhtab->ht, key);
|
| 3195 |
|
| 3196 |
|
| 3197 | if (PTR_ERR(elem) == -ENOENT)
|
| 3198 | elem = rhashtable_next_key(&rhtab->ht, NULL);
|
| 3199 |
|
| 3200 | if (IS_ERR(elem))
|
| 3201 | return PTR_ERR(elem);
|
| 3202 | if (!elem)
|
| 3203 | return -ENOENT;
|
| 3204 |
|
| 3205 | memcpy(next_key, elem->data, map->key_size);
|
| 3206 | return 0;
|
| 3207 | }
|
| 3208 |
|
| 3209 | static void rhtab_map_seq_show_elem(struct bpf_map *map, void *key, struct seq_file *m)
|
| 3210 | {
|
| 3211 | void *value;
|
| 3212 |
|
| 3213 |
|
| 3214 | guard(rcu)();
|
| 3215 |
|
| 3216 | value = rhtab_map_lookup_elem(map, key);
|
| 3217 | if (!value)
|
| 3218 | return;
|
| 3219 |
|
| 3220 | btf_type_seq_show(map->btf, map->btf_key_type_id, key, m);
|
| 3221 | seq_puts(m, ": ");
|
| 3222 | btf_type_seq_show(map->btf, map->btf_value_type_id, value, m);
|
| 3223 | seq_putc(m, '\n');
|
| 3224 | }
|
| 3225 |
|
| 3226 | static long bpf_each_rhash_elem(struct bpf_map *map, bpf_callback_t callback_fn,
|
| 3227 | void *callback_ctx, u64 flags)
|
| 3228 | {
|
| 3229 | struct bpf_rhtab *rhtab = container_of(map, struct bpf_rhtab, map);
|
| 3230 | void *prev_key = NULL;
|
| 3231 | struct rhtab_elem *elem;
|
| 3232 | int num_elems = 0;
|
| 3233 | u64 ret = 0;
|
| 3234 |
|
| 3235 | cant_migrate();
|
| 3236 |
|
| 3237 | if (flags != 0)
|
| 3238 | return -EINVAL;
|
| 3239 |
|
| 3240 | rcu_read_lock();
|
| 3241 | |
| 3242 | * Best-effort iteration: if rhashtable is concurrently resized or
|
| 3243 | * elements are deleted/inserted, there may be missed or duplicate
|
| 3244 | * elements visited.
|
| 3245 |
|
| 3246 | while ((elem = rhashtable_next_key(&rhtab->ht, prev_key))) {
|
| 3247 | if (IS_ERR(elem))
|
| 3248 | break;
|
| 3249 | num_elems++;
|
| 3250 | ret = callback_fn((u64)(long)map,
|
| 3251 | (u64)(long)elem->data,
|
| 3252 | (u64)(long)rhtab_elem_value(elem, map->key_size),
|
| 3253 | (u64)(long)callback_ctx, 0);
|
| 3254 | if (ret)
|
| 3255 | break;
|
| 3256 |
|
| 3257 | prev_key = elem->data;
|
| 3258 | }
|
| 3259 | rcu_read_unlock();
|
| 3260 |
|
| 3261 | return num_elems;
|
| 3262 | }
|
| 3263 |
|
| 3264 | static u64 rhtab_map_mem_usage(const struct bpf_map *map)
|
| 3265 | {
|
| 3266 | struct bpf_rhtab *rhtab = container_of(map, struct bpf_rhtab, map);
|
| 3267 | u64 num_entries;
|
| 3268 |
|
| 3269 |
|
| 3270 | num_entries = atomic_read(&rhtab->ht.nelems);
|
| 3271 | return sizeof(struct bpf_rhtab) + rhtab->elem_size * num_entries;
|
| 3272 | }
|
| 3273 |
|
| 3274 | static int __rhtab_map_lookup_and_delete_batch(struct bpf_map *map,
|
| 3275 | const union bpf_attr *attr,
|
| 3276 | union bpf_attr __user *uattr,
|
| 3277 | bool do_delete)
|
| 3278 | {
|
| 3279 | struct bpf_rhtab *rhtab = container_of(map, struct bpf_rhtab, map);
|
| 3280 | void __user *uvalues = u64_to_user_ptr(attr->batch.values);
|
| 3281 | void __user *ukeys = u64_to_user_ptr(attr->batch.keys);
|
| 3282 | void __user *ubatch = u64_to_user_ptr(attr->batch.in_batch);
|
| 3283 | void *cursor = NULL, *keys = NULL, *values = NULL, *dst_key, *dst_val;
|
| 3284 | struct rhtab_elem **del_elems = NULL;
|
| 3285 | u32 max_count, total, key_size, value_size, i;
|
| 3286 | bool has_next_cursor = false;
|
| 3287 | struct rhtab_elem *elem;
|
| 3288 | u64 elem_map_flags, map_flags;
|
| 3289 | int ret = 0;
|
| 3290 |
|
| 3291 | elem_map_flags = attr->batch.elem_flags;
|
| 3292 | ret = bpf_map_check_op_flags(map, elem_map_flags, BPF_F_LOCK);
|
| 3293 | if (ret)
|
| 3294 | return ret;
|
| 3295 |
|
| 3296 | map_flags = attr->batch.flags;
|
| 3297 | if (map_flags)
|
| 3298 | return -EINVAL;
|
| 3299 |
|
| 3300 | max_count = attr->batch.count;
|
| 3301 | if (!max_count)
|
| 3302 | return 0;
|
| 3303 |
|
| 3304 | if (put_user(0, &uattr->batch.count))
|
| 3305 | return -EFAULT;
|
| 3306 |
|
| 3307 | key_size = map->key_size;
|
| 3308 | value_size = map->value_size;
|
| 3309 |
|
| 3310 | keys = kvmalloc_array(max_count, key_size, GFP_USER | __GFP_NOWARN);
|
| 3311 | values = kvmalloc_array(max_count, value_size, GFP_USER | __GFP_NOWARN);
|
| 3312 | if (do_delete)
|
| 3313 | del_elems = kvmalloc_array(max_count, sizeof(void *),
|
| 3314 | GFP_USER | __GFP_NOWARN);
|
| 3315 | cursor = kmalloc(key_size, GFP_USER | __GFP_NOWARN);
|
| 3316 |
|
| 3317 | if (!keys || !values || !cursor || (do_delete && !del_elems)) {
|
| 3318 | ret = -ENOMEM;
|
| 3319 | goto free;
|
| 3320 | }
|
| 3321 |
|
| 3322 | if (ubatch && copy_from_user(cursor, ubatch, key_size)) {
|
| 3323 | ret = -EFAULT;
|
| 3324 | goto free;
|
| 3325 | }
|
| 3326 |
|
| 3327 | dst_key = keys;
|
| 3328 | dst_val = values;
|
| 3329 | total = 0;
|
| 3330 |
|
| 3331 | rcu_read_lock();
|
| 3332 |
|
| 3333 | |
| 3334 | * Cursor stores the key of the next-to-process element (stashed by
|
| 3335 | * the previous batch). Look it up directly so the element is included
|
| 3336 | * here rather than skipped by next_key(). If the cursor was deleted
|
| 3337 | * concurrently (or by the previous do_delete batch), return -EAGAIN
|
| 3338 | * so userspace can distinguish a lost cursor from end-of-iteration
|
| 3339 | * (-ENOENT) and restart from a NULL cursor.
|
| 3340 |
|
| 3341 | if (ubatch) {
|
| 3342 | elem = rhtab_lookup_elem(map, cursor);
|
| 3343 | if (!elem) {
|
| 3344 | rcu_read_unlock();
|
| 3345 | ret = -EAGAIN;
|
| 3346 | goto free;
|
| 3347 | }
|
| 3348 | } else {
|
| 3349 | elem = rhashtable_next_key(&rhtab->ht, NULL);
|
| 3350 | }
|
| 3351 |
|
| 3352 | while (elem && !IS_ERR(elem) && total < max_count) {
|
| 3353 | memcpy(dst_key, elem->data, key_size);
|
| 3354 | rhtab_read_elem_value(map, dst_val, elem, elem_map_flags);
|
| 3355 | check_and_init_map_value(map, dst_val);
|
| 3356 |
|
| 3357 | if (do_delete)
|
| 3358 | del_elems[total] = elem;
|
| 3359 |
|
| 3360 | elem = rhashtable_next_key(&rhtab->ht, dst_key);
|
| 3361 | dst_key += key_size;
|
| 3362 | dst_val += value_size;
|
| 3363 | total++;
|
| 3364 |
|
| 3365 |
|
| 3366 | if (need_resched())
|
| 3367 | break;
|
| 3368 | }
|
| 3369 |
|
| 3370 | if (elem && !IS_ERR(elem)) {
|
| 3371 |
|
| 3372 | memcpy(cursor, elem->data, key_size);
|
| 3373 | has_next_cursor = true;
|
| 3374 | }
|
| 3375 |
|
| 3376 | if (do_delete) {
|
| 3377 | migrate_disable();
|
| 3378 | for (i = 0; i < total; i++)
|
| 3379 | rhtab_delete_elem(rhtab, del_elems[i], NULL, 0);
|
| 3380 | migrate_enable();
|
| 3381 | }
|
| 3382 |
|
| 3383 | rcu_read_unlock();
|
| 3384 |
|
| 3385 | if (total == 0) {
|
| 3386 | ret = -ENOENT;
|
| 3387 | goto free;
|
| 3388 | }
|
| 3389 |
|
| 3390 |
|
| 3391 | if (!has_next_cursor)
|
| 3392 | ret = -ENOENT;
|
| 3393 |
|
| 3394 | if (copy_to_user(ukeys, keys, (size_t)total * key_size) ||
|
| 3395 | copy_to_user(uvalues, values, (size_t)total * value_size) ||
|
| 3396 | put_user(total, &uattr->batch.count) ||
|
| 3397 | (has_next_cursor &&
|
| 3398 | copy_to_user(u64_to_user_ptr(attr->batch.out_batch),
|
| 3399 | cursor, key_size))) {
|
| 3400 | ret = -EFAULT;
|
| 3401 | goto free;
|
| 3402 | }
|
| 3403 |
|
| 3404 | free:
|
| 3405 | kfree(cursor);
|
| 3406 | kvfree(keys);
|
| 3407 | kvfree(values);
|
| 3408 | kvfree(del_elems);
|
| 3409 | return ret;
|
| 3410 | }
|
| 3411 |
|
| 3412 | static int rhtab_map_lookup_batch(struct bpf_map *map, const union bpf_attr *attr,
|
| 3413 | union bpf_attr __user *uattr)
|
| 3414 | {
|
| 3415 | return __rhtab_map_lookup_and_delete_batch(map, attr, uattr, false);
|
| 3416 | }
|
| 3417 |
|
| 3418 | static int rhtab_map_lookup_and_delete_batch(struct bpf_map *map, const union bpf_attr *attr,
|
| 3419 | union bpf_attr __user *uattr)
|
| 3420 | {
|
| 3421 | return __rhtab_map_lookup_and_delete_batch(map, attr, uattr, true);
|
| 3422 | }
|
| 3423 |
|
| 3424 | struct bpf_iter_seq_rhash_map_info {
|
| 3425 | struct bpf_map *map;
|
| 3426 | struct bpf_rhtab *rhtab;
|
| 3427 | struct rhashtable_iter iter;
|
| 3428 | };
|
| 3429 |
|
| 3430 | static void *bpf_rhash_map_seq_start(struct seq_file *seq, loff_t *pos)
|
| 3431 | __acquires(RCU)
|
| 3432 | {
|
| 3433 | struct bpf_iter_seq_rhash_map_info *info = seq->private;
|
| 3434 | struct rhtab_elem *elem;
|
| 3435 |
|
| 3436 | rhashtable_walk_start(&info->iter);
|
| 3437 | |
| 3438 | * Re-deliver the element returned by walk_next() at the end of the
|
| 3439 | * previous read() — bpf_seq_read may have stopped before show()
|
| 3440 | * consumed it. Rehash rewinds the walker; retry on -EAGAIN.
|
| 3441 |
|
| 3442 | do {
|
| 3443 | elem = rhashtable_walk_peek(&info->iter);
|
| 3444 | } while (PTR_ERR(elem) == -EAGAIN);
|
| 3445 |
|
| 3446 | if (IS_ERR(elem))
|
| 3447 | return NULL;
|
| 3448 |
|
| 3449 | if (elem && *pos == 0)
|
| 3450 | ++*pos;
|
| 3451 | return elem;
|
| 3452 | }
|
| 3453 |
|
| 3454 | static void *bpf_rhash_map_seq_next(struct seq_file *seq, void *v, loff_t *pos)
|
| 3455 | {
|
| 3456 | struct bpf_iter_seq_rhash_map_info *info = seq->private;
|
| 3457 | struct rhtab_elem *elem;
|
| 3458 |
|
| 3459 | ++*pos;
|
| 3460 |
|
| 3461 |
|
| 3462 | do {
|
| 3463 | elem = rhashtable_walk_next(&info->iter);
|
| 3464 | } while (PTR_ERR(elem) == -EAGAIN);
|
| 3465 |
|
| 3466 | if (IS_ERR(elem))
|
| 3467 | return NULL;
|
| 3468 | return elem;
|
| 3469 | }
|
| 3470 |
|
| 3471 | static int __bpf_rhash_map_seq_show(struct seq_file *seq,
|
| 3472 | struct rhtab_elem *elem)
|
| 3473 | {
|
| 3474 | struct bpf_iter_seq_rhash_map_info *info = seq->private;
|
| 3475 | struct bpf_iter__bpf_map_elem ctx = {};
|
| 3476 | struct bpf_iter_meta meta;
|
| 3477 | struct bpf_prog *prog;
|
| 3478 | int ret = 0;
|
| 3479 |
|
| 3480 | meta.seq = seq;
|
| 3481 | prog = bpf_iter_get_info(&meta, elem == NULL);
|
| 3482 | if (prog) {
|
| 3483 | ctx.meta = &meta;
|
| 3484 | ctx.map = info->map;
|
| 3485 | if (elem) {
|
| 3486 | ctx.key = elem->data;
|
| 3487 | ctx.value = rhtab_elem_value(elem, info->map->key_size);
|
| 3488 | }
|
| 3489 | ret = bpf_iter_run_prog(prog, &ctx);
|
| 3490 | }
|
| 3491 |
|
| 3492 | return ret;
|
| 3493 | }
|
| 3494 |
|
| 3495 | static int bpf_rhash_map_seq_show(struct seq_file *seq, void *v)
|
| 3496 | {
|
| 3497 | return __bpf_rhash_map_seq_show(seq, v);
|
| 3498 | }
|
| 3499 |
|
| 3500 | static void bpf_rhash_map_seq_stop(struct seq_file *seq, void *v)
|
| 3501 | __releases(RCU)
|
| 3502 | {
|
| 3503 | struct bpf_iter_seq_rhash_map_info *info = seq->private;
|
| 3504 |
|
| 3505 | if (!v)
|
| 3506 | (void)__bpf_rhash_map_seq_show(seq, NULL);
|
| 3507 |
|
| 3508 | rhashtable_walk_stop(&info->iter);
|
| 3509 | }
|
| 3510 |
|
| 3511 | static int bpf_iter_init_rhash_map(void *priv_data, struct bpf_iter_aux_info *aux)
|
| 3512 | {
|
| 3513 | struct bpf_iter_seq_rhash_map_info *info = priv_data;
|
| 3514 | struct bpf_map *map = aux->map;
|
| 3515 |
|
| 3516 | bpf_map_inc_with_uref(map);
|
| 3517 | info->map = map;
|
| 3518 | info->rhtab = container_of(map, struct bpf_rhtab, map);
|
| 3519 | rhashtable_walk_enter(&info->rhtab->ht, &info->iter);
|
| 3520 | return 0;
|
| 3521 | }
|
| 3522 |
|
| 3523 | static void bpf_iter_fini_rhash_map(void *priv_data)
|
| 3524 | {
|
| 3525 | struct bpf_iter_seq_rhash_map_info *info = priv_data;
|
| 3526 |
|
| 3527 | rhashtable_walk_exit(&info->iter);
|
| 3528 | bpf_map_put_with_uref(info->map);
|
| 3529 | }
|
| 3530 |
|
| 3531 | static const struct seq_operations bpf_rhash_map_seq_ops = {
|
| 3532 | .start = bpf_rhash_map_seq_start,
|
| 3533 | .next = bpf_rhash_map_seq_next,
|
| 3534 | .stop = bpf_rhash_map_seq_stop,
|
| 3535 | .show = bpf_rhash_map_seq_show,
|
| 3536 | };
|
| 3537 |
|
| 3538 | static const struct bpf_iter_seq_info rhash_iter_seq_info = {
|
| 3539 | .seq_ops = &bpf_rhash_map_seq_ops,
|
| 3540 | .init_seq_private = bpf_iter_init_rhash_map,
|
| 3541 | .fini_seq_private = bpf_iter_fini_rhash_map,
|
| 3542 | .seq_priv_size = sizeof(struct bpf_iter_seq_rhash_map_info),
|
| 3543 | };
|
| 3544 |
|
| 3545 | BTF_ID_LIST_SINGLE(rhtab_map_btf_ids, struct, bpf_rhtab)
|
| 3546 | const struct bpf_map_ops rhtab_map_ops = {
|
| 3547 | .map_meta_equal = bpf_map_meta_equal,
|
| 3548 | .map_alloc_check = rhtab_map_alloc_check,
|
| 3549 | .map_alloc = rhtab_map_alloc,
|
| 3550 | .map_free = rhtab_map_free,
|
| 3551 | .map_get_next_key = rhtab_map_get_next_key,
|
| 3552 | .map_release_uref = rhtab_map_free_internal_structs,
|
| 3553 | .map_check_btf = rhtab_map_check_btf,
|
| 3554 | .map_lookup_elem = rhtab_map_lookup_elem,
|
| 3555 | .map_lookup_and_delete_elem = rhtab_map_lookup_and_delete_elem,
|
| 3556 | .map_update_elem = rhtab_map_update_elem,
|
| 3557 | .map_delete_elem = rhtab_map_delete_elem,
|
| 3558 | .map_gen_lookup = rhtab_map_gen_lookup,
|
| 3559 | .map_seq_show_elem = rhtab_map_seq_show_elem,
|
| 3560 | .map_set_for_each_callback_args = map_set_for_each_callback_args,
|
| 3561 | .map_for_each_callback = bpf_each_rhash_elem,
|
| 3562 | .map_mem_usage = rhtab_map_mem_usage,
|
| 3563 | BATCH_OPS(rhtab),
|
| 3564 | .map_btf_id = &rhtab_map_btf_ids[0],
|
| 3565 | .iter_seq_info = &rhash_iter_seq_info,
|
| 3566 | };
|