| 1 | |
| 2 | |
| 3 |
|
| 4 | #include <linux/bpf.h>
|
| 5 | #include <linux/bpf_trace.h>
|
| 6 | #include <linux/bpf_lirc.h>
|
| 7 | #include <linux/btf.h>
|
| 8 | #include <linux/syscalls.h>
|
| 9 | #include <linux/slab.h>
|
| 10 | #include <linux/sched/signal.h>
|
| 11 | #include <linux/vmalloc.h>
|
| 12 | #include <linux/mmzone.h>
|
| 13 | #include <linux/anon_inodes.h>
|
| 14 | #include <linux/fdtable.h>
|
| 15 | #include <linux/file.h>
|
| 16 | #include <linux/fs.h>
|
| 17 | #include <linux/license.h>
|
| 18 | #include <linux/filter.h>
|
| 19 | #include <linux/version.h>
|
| 20 | #include <linux/kernel.h>
|
| 21 | #include <linux/idr.h>
|
| 22 | #include <linux/cred.h>
|
| 23 | #include <linux/timekeeping.h>
|
| 24 | #include <linux/ctype.h>
|
| 25 | #include <linux/nospec.h>
|
| 26 |
|
| 27 | #define IS_FD_ARRAY(map) ((map)->map_type == BPF_MAP_TYPE_PROG_ARRAY || \
|
| 28 | (map)->map_type == BPF_MAP_TYPE_PERF_EVENT_ARRAY || \
|
| 29 | (map)->map_type == BPF_MAP_TYPE_CGROUP_ARRAY || \
|
| 30 | (map)->map_type == BPF_MAP_TYPE_ARRAY_OF_MAPS)
|
| 31 | #define IS_FD_HASH(map) ((map)->map_type == BPF_MAP_TYPE_HASH_OF_MAPS)
|
| 32 | #define IS_FD_MAP(map) (IS_FD_ARRAY(map) || IS_FD_HASH(map))
|
| 33 |
|
| 34 | #define BPF_OBJ_FLAG_MASK (BPF_F_RDONLY | BPF_F_WRONLY)
|
| 35 |
|
| 36 | DEFINE_PER_CPU(int, bpf_prog_active);
|
| 37 | static DEFINE_IDR(prog_idr);
|
| 38 | static DEFINE_SPINLOCK(prog_idr_lock);
|
| 39 | static DEFINE_IDR(map_idr);
|
| 40 | static DEFINE_SPINLOCK(map_idr_lock);
|
| 41 |
|
| 42 | int sysctl_unprivileged_bpf_disabled __read_mostly;
|
| 43 |
|
| 44 | static const struct bpf_map_ops * const bpf_map_types[] = {
|
| 45 | #define BPF_PROG_TYPE(_id, _ops)
|
| 46 | #define BPF_MAP_TYPE(_id, _ops) \
|
| 47 | [_id] = &_ops,
|
| 48 | #include <linux/bpf_types.h>
|
| 49 | #undef BPF_PROG_TYPE
|
| 50 | #undef BPF_MAP_TYPE
|
| 51 | };
|
| 52 |
|
| 53 | |
| 54 | * If we're handed a bigger struct than we know of, ensure all the unknown bits
|
| 55 | * are 0 - i.e. new user-space does not rely on any kernel feature extensions
|
| 56 | * we don't know about yet.
|
| 57 | *
|
| 58 | * There is a ToCToU between this function call and the following
|
| 59 | * copy_from_user() call. However, this is not a concern since this function is
|
| 60 | * meant to be a future-proofing of bits.
|
| 61 |
|
| 62 | int bpf_check_uarg_tail_zero(void __user *uaddr,
|
| 63 | size_t expected_size,
|
| 64 | size_t actual_size)
|
| 65 | {
|
| 66 | unsigned char __user *addr;
|
| 67 | unsigned char __user *end;
|
| 68 | unsigned char val;
|
| 69 | int err;
|
| 70 |
|
| 71 | if (unlikely(actual_size > PAGE_SIZE))
|
| 72 | return -E2BIG;
|
| 73 |
|
| 74 | if (unlikely(!access_ok(uaddr, actual_size)))
|
| 75 | return -EFAULT;
|
| 76 |
|
| 77 | if (actual_size <= expected_size)
|
| 78 | return 0;
|
| 79 |
|
| 80 | addr = uaddr + expected_size;
|
| 81 | end = uaddr + actual_size;
|
| 82 |
|
| 83 | for (; addr < end; addr++) {
|
| 84 | err = get_user(val, addr);
|
| 85 | if (err)
|
| 86 | return err;
|
| 87 | if (val)
|
| 88 | return -E2BIG;
|
| 89 | }
|
| 90 |
|
| 91 | return 0;
|
| 92 | }
|
| 93 |
|
| 94 | const struct bpf_map_ops bpf_map_offload_ops = {
|
| 95 | .map_alloc = bpf_map_offload_map_alloc,
|
| 96 | .map_free = bpf_map_offload_map_free,
|
| 97 | .map_check_btf = map_check_no_btf,
|
| 98 | };
|
| 99 |
|
| 100 | static struct bpf_map *find_and_alloc_map(union bpf_attr *attr)
|
| 101 | {
|
| 102 | const struct bpf_map_ops *ops;
|
| 103 | u32 type = attr->map_type;
|
| 104 | struct bpf_map *map;
|
| 105 | int err;
|
| 106 |
|
| 107 | if (type >= ARRAY_SIZE(bpf_map_types))
|
| 108 | return ERR_PTR(-EINVAL);
|
| 109 | type = array_index_nospec(type, ARRAY_SIZE(bpf_map_types));
|
| 110 | ops = bpf_map_types[type];
|
| 111 | if (!ops)
|
| 112 | return ERR_PTR(-EINVAL);
|
| 113 |
|
| 114 | if (ops->map_alloc_check) {
|
| 115 | err = ops->map_alloc_check(attr);
|
| 116 | if (err)
|
| 117 | return ERR_PTR(err);
|
| 118 | }
|
| 119 | if (attr->map_ifindex)
|
| 120 | ops = &bpf_map_offload_ops;
|
| 121 | map = ops->map_alloc(attr);
|
| 122 | if (IS_ERR(map))
|
| 123 | return map;
|
| 124 | map->ops = ops;
|
| 125 | map->map_type = type;
|
| 126 | return map;
|
| 127 | }
|
| 128 |
|
| 129 | void *bpf_map_area_alloc(size_t size, int numa_node)
|
| 130 | {
|
| 131 | |
| 132 | * under memory pressure, therefore we set __GFP_NORETRY to kmalloc,
|
| 133 | * which is used for lower order allocation requests.
|
| 134 | *
|
| 135 | * It has been observed that higher order allocation requests done by
|
| 136 | * vmalloc with __GFP_NORETRY being set might fail due to not trying
|
| 137 | * to reclaim memory from the page cache, thus we set
|
| 138 | * __GFP_RETRY_MAYFAIL to avoid such situations.
|
| 139 |
|
| 140 |
|
| 141 | const gfp_t flags = __GFP_NOWARN | __GFP_ZERO;
|
| 142 | void *area;
|
| 143 |
|
| 144 | if (size <= (PAGE_SIZE << PAGE_ALLOC_COSTLY_ORDER)) {
|
| 145 | area = kmalloc_node(size, GFP_USER | __GFP_NORETRY | flags,
|
| 146 | numa_node);
|
| 147 | if (area != NULL)
|
| 148 | return area;
|
| 149 | }
|
| 150 |
|
| 151 | return __vmalloc_node_flags_caller(size, numa_node,
|
| 152 | GFP_KERNEL | __GFP_RETRY_MAYFAIL |
|
| 153 | flags, __builtin_return_address(0));
|
| 154 | }
|
| 155 |
|
| 156 | void bpf_map_area_free(void *area)
|
| 157 | {
|
| 158 | kvfree(area);
|
| 159 | }
|
| 160 |
|
| 161 | static u32 bpf_map_flags_retain_permanent(u32 flags)
|
| 162 | {
|
| 163 | |
| 164 | * rather to the map fd instead, so they have no meaning upon
|
| 165 | * map object inspection since multiple file descriptors with
|
| 166 | * different (access) properties can exist here. Thus, given
|
| 167 | * this has zero meaning for the map itself, lets clear these
|
| 168 | * from here.
|
| 169 |
|
| 170 | return flags & ~(BPF_F_RDONLY | BPF_F_WRONLY);
|
| 171 | }
|
| 172 |
|
| 173 | void bpf_map_init_from_attr(struct bpf_map *map, union bpf_attr *attr)
|
| 174 | {
|
| 175 | map->map_type = attr->map_type;
|
| 176 | map->key_size = attr->key_size;
|
| 177 | map->value_size = attr->value_size;
|
| 178 | map->max_entries = attr->max_entries;
|
| 179 | map->map_flags = bpf_map_flags_retain_permanent(attr->map_flags);
|
| 180 | map->numa_node = bpf_map_attr_numa_node(attr);
|
| 181 | }
|
| 182 |
|
| 183 | static int bpf_charge_memlock(struct user_struct *user, u32 pages)
|
| 184 | {
|
| 185 | unsigned long memlock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;
|
| 186 |
|
| 187 | if (atomic_long_add_return(pages, &user->locked_vm) > memlock_limit) {
|
| 188 | atomic_long_sub(pages, &user->locked_vm);
|
| 189 | return -EPERM;
|
| 190 | }
|
| 191 | return 0;
|
| 192 | }
|
| 193 |
|
| 194 | static void bpf_uncharge_memlock(struct user_struct *user, u32 pages)
|
| 195 | {
|
| 196 | if (user)
|
| 197 | atomic_long_sub(pages, &user->locked_vm);
|
| 198 | }
|
| 199 |
|
| 200 | int bpf_map_charge_init(struct bpf_map_memory *mem, size_t size)
|
| 201 | {
|
| 202 | u32 pages = round_up(size, PAGE_SIZE) >> PAGE_SHIFT;
|
| 203 | struct user_struct *user;
|
| 204 | int ret;
|
| 205 |
|
| 206 | if (size >= U32_MAX - PAGE_SIZE)
|
| 207 | return -E2BIG;
|
| 208 |
|
| 209 | user = get_current_user();
|
| 210 | ret = bpf_charge_memlock(user, pages);
|
| 211 | if (ret) {
|
| 212 | free_uid(user);
|
| 213 | return ret;
|
| 214 | }
|
| 215 |
|
| 216 | mem->pages = pages;
|
| 217 | mem->user = user;
|
| 218 |
|
| 219 | return 0;
|
| 220 | }
|
| 221 |
|
| 222 | void bpf_map_charge_finish(struct bpf_map_memory *mem)
|
| 223 | {
|
| 224 | bpf_uncharge_memlock(mem->user, mem->pages);
|
| 225 | free_uid(mem->user);
|
| 226 | }
|
| 227 |
|
| 228 | void bpf_map_charge_move(struct bpf_map_memory *dst,
|
| 229 | struct bpf_map_memory *src)
|
| 230 | {
|
| 231 | *dst = *src;
|
| 232 |
|
| 233 |
|
| 234 | memset(src, 0, sizeof(struct bpf_map_memory));
|
| 235 | }
|
| 236 |
|
| 237 | int bpf_map_charge_memlock(struct bpf_map *map, u32 pages)
|
| 238 | {
|
| 239 | int ret;
|
| 240 |
|
| 241 | ret = bpf_charge_memlock(map->memory.user, pages);
|
| 242 | if (ret)
|
| 243 | return ret;
|
| 244 | map->memory.pages += pages;
|
| 245 | return ret;
|
| 246 | }
|
| 247 |
|
| 248 | void bpf_map_uncharge_memlock(struct bpf_map *map, u32 pages)
|
| 249 | {
|
| 250 | bpf_uncharge_memlock(map->memory.user, pages);
|
| 251 | map->memory.pages -= pages;
|
| 252 | }
|
| 253 |
|
| 254 | static int bpf_map_alloc_id(struct bpf_map *map)
|
| 255 | {
|
| 256 | int id;
|
| 257 |
|
| 258 | idr_preload(GFP_KERNEL);
|
| 259 | spin_lock_bh(&map_idr_lock);
|
| 260 | id = idr_alloc_cyclic(&map_idr, map, 1, INT_MAX, GFP_ATOMIC);
|
| 261 | if (id > 0)
|
| 262 | map->id = id;
|
| 263 | spin_unlock_bh(&map_idr_lock);
|
| 264 | idr_preload_end();
|
| 265 |
|
| 266 | if (WARN_ON_ONCE(!id))
|
| 267 | return -ENOSPC;
|
| 268 |
|
| 269 | return id > 0 ? 0 : id;
|
| 270 | }
|
| 271 |
|
| 272 | void bpf_map_free_id(struct bpf_map *map, bool do_idr_lock)
|
| 273 | {
|
| 274 | unsigned long flags;
|
| 275 |
|
| 276 | |
| 277 | * disappears - even if someone holds an fd to them they are unusable,
|
| 278 | * the memory is gone, all ops will fail; they are simply waiting for
|
| 279 | * refcnt to drop to be freed.
|
| 280 |
|
| 281 | if (!map->id)
|
| 282 | return;
|
| 283 |
|
| 284 | if (do_idr_lock)
|
| 285 | spin_lock_irqsave(&map_idr_lock, flags);
|
| 286 | else
|
| 287 | __acquire(&map_idr_lock);
|
| 288 |
|
| 289 | idr_remove(&map_idr, map->id);
|
| 290 | map->id = 0;
|
| 291 |
|
| 292 | if (do_idr_lock)
|
| 293 | spin_unlock_irqrestore(&map_idr_lock, flags);
|
| 294 | else
|
| 295 | __release(&map_idr_lock);
|
| 296 | }
|
| 297 |
|
| 298 |
|
| 299 | static void bpf_map_free_deferred(struct work_struct *work)
|
| 300 | {
|
| 301 | struct bpf_map *map = container_of(work, struct bpf_map, work);
|
| 302 | struct bpf_map_memory mem;
|
| 303 |
|
| 304 | bpf_map_charge_move(&mem, &map->memory);
|
| 305 | security_bpf_map_free(map);
|
| 306 |
|
| 307 | map->ops->map_free(map);
|
| 308 | bpf_map_charge_finish(&mem);
|
| 309 | }
|
| 310 |
|
| 311 | static void bpf_map_put_uref(struct bpf_map *map)
|
| 312 | {
|
| 313 | if (atomic_dec_and_test(&map->usercnt)) {
|
| 314 | if (map->ops->map_release_uref)
|
| 315 | map->ops->map_release_uref(map);
|
| 316 | }
|
| 317 | }
|
| 318 |
|
| 319 | |
| 320 | * (unrelying map implementation ops->map_free() might sleep)
|
| 321 |
|
| 322 | static void __bpf_map_put(struct bpf_map *map, bool do_idr_lock)
|
| 323 | {
|
| 324 | if (atomic_dec_and_test(&map->refcnt)) {
|
| 325 |
|
| 326 | bpf_map_free_id(map, do_idr_lock);
|
| 327 | btf_put(map->btf);
|
| 328 | INIT_WORK(&map->work, bpf_map_free_deferred);
|
| 329 | schedule_work(&map->work);
|
| 330 | }
|
| 331 | }
|
| 332 |
|
| 333 | void bpf_map_put(struct bpf_map *map)
|
| 334 | {
|
| 335 | __bpf_map_put(map, true);
|
| 336 | }
|
| 337 | EXPORT_SYMBOL_GPL(bpf_map_put);
|
| 338 |
|
| 339 | void bpf_map_put_with_uref(struct bpf_map *map)
|
| 340 | {
|
| 341 | bpf_map_put_uref(map);
|
| 342 | bpf_map_put(map);
|
| 343 | }
|
| 344 |
|
| 345 | static int bpf_map_release(struct inode *inode, struct file *filp)
|
| 346 | {
|
| 347 | struct bpf_map *map = filp->private_data;
|
| 348 |
|
| 349 | if (map->ops->map_release)
|
| 350 | map->ops->map_release(map, filp);
|
| 351 |
|
| 352 | bpf_map_put_with_uref(map);
|
| 353 | return 0;
|
| 354 | }
|
| 355 |
|
| 356 | static fmode_t map_get_sys_perms(struct bpf_map *map, struct fd f)
|
| 357 | {
|
| 358 | fmode_t mode = f.file->f_mode;
|
| 359 |
|
| 360 | |
| 361 | * map permissions facing syscall side.
|
| 362 |
|
| 363 | if (READ_ONCE(map->frozen))
|
| 364 | mode &= ~FMODE_CAN_WRITE;
|
| 365 | return mode;
|
| 366 | }
|
| 367 |
|
| 368 | #ifdef CONFIG_PROC_FS
|
| 369 | static void bpf_map_show_fdinfo(struct seq_file *m, struct file *filp)
|
| 370 | {
|
| 371 | const struct bpf_map *map = filp->private_data;
|
| 372 | const struct bpf_array *array;
|
| 373 | u32 owner_prog_type = 0;
|
| 374 | u32 owner_jited = 0;
|
| 375 |
|
| 376 | if (map->map_type == BPF_MAP_TYPE_PROG_ARRAY) {
|
| 377 | array = container_of(map, struct bpf_array, map);
|
| 378 | owner_prog_type = array->owner_prog_type;
|
| 379 | owner_jited = array->owner_jited;
|
| 380 | }
|
| 381 |
|
| 382 | seq_printf(m,
|
| 383 | "map_type:\t%u\n"
|
| 384 | "key_size:\t%u\n"
|
| 385 | "value_size:\t%u\n"
|
| 386 | "max_entries:\t%u\n"
|
| 387 | "map_flags:\t%#x\n"
|
| 388 | "memlock:\t%llu\n"
|
| 389 | "map_id:\t%u\n"
|
| 390 | "frozen:\t%u\n",
|
| 391 | map->map_type,
|
| 392 | map->key_size,
|
| 393 | map->value_size,
|
| 394 | map->max_entries,
|
| 395 | map->map_flags,
|
| 396 | map->memory.pages * 1ULL << PAGE_SHIFT,
|
| 397 | map->id,
|
| 398 | READ_ONCE(map->frozen));
|
| 399 |
|
| 400 | if (owner_prog_type) {
|
| 401 | seq_printf(m, "owner_prog_type:\t%u\n",
|
| 402 | owner_prog_type);
|
| 403 | seq_printf(m, "owner_jited:\t%u\n",
|
| 404 | owner_jited);
|
| 405 | }
|
| 406 | }
|
| 407 | #endif
|
| 408 |
|
| 409 | static ssize_t bpf_dummy_read(struct file *filp, char __user *buf, size_t siz,
|
| 410 | loff_t *ppos)
|
| 411 | {
|
| 412 | |
| 413 | * f_mode with FMODE_CAN_READ.
|
| 414 |
|
| 415 | return -EINVAL;
|
| 416 | }
|
| 417 |
|
| 418 | static ssize_t bpf_dummy_write(struct file *filp, const char __user *buf,
|
| 419 | size_t siz, loff_t *ppos)
|
| 420 | {
|
| 421 | |
| 422 | * f_mode with FMODE_CAN_WRITE.
|
| 423 |
|
| 424 | return -EINVAL;
|
| 425 | }
|
| 426 |
|
| 427 | const struct file_operations bpf_map_fops = {
|
| 428 | #ifdef CONFIG_PROC_FS
|
| 429 | .show_fdinfo = bpf_map_show_fdinfo,
|
| 430 | #endif
|
| 431 | .release = bpf_map_release,
|
| 432 | .read = bpf_dummy_read,
|
| 433 | .write = bpf_dummy_write,
|
| 434 | };
|
| 435 |
|
| 436 | int bpf_map_new_fd(struct bpf_map *map, int flags)
|
| 437 | {
|
| 438 | int ret;
|
| 439 |
|
| 440 | ret = security_bpf_map(map, OPEN_FMODE(flags));
|
| 441 | if (ret < 0)
|
| 442 | return ret;
|
| 443 |
|
| 444 | return anon_inode_getfd("bpf-map", &bpf_map_fops, map,
|
| 445 | flags | O_CLOEXEC);
|
| 446 | }
|
| 447 |
|
| 448 | int bpf_get_file_flag(int flags)
|
| 449 | {
|
| 450 | if ((flags & BPF_F_RDONLY) && (flags & BPF_F_WRONLY))
|
| 451 | return -EINVAL;
|
| 452 | if (flags & BPF_F_RDONLY)
|
| 453 | return O_RDONLY;
|
| 454 | if (flags & BPF_F_WRONLY)
|
| 455 | return O_WRONLY;
|
| 456 | return O_RDWR;
|
| 457 | }
|
| 458 |
|
| 459 |
|
| 460 | #define CHECK_ATTR(CMD) \
|
| 461 | memchr_inv((void *) &attr->CMD##_LAST_FIELD + \
|
| 462 | sizeof(attr->CMD##_LAST_FIELD), 0, \
|
| 463 | sizeof(*attr) - \
|
| 464 | offsetof(union bpf_attr, CMD##_LAST_FIELD) - \
|
| 465 | sizeof(attr->CMD##_LAST_FIELD)) != NULL
|
| 466 |
|
| 467 | |
| 468 | * Return 0 on success and < 0 on error.
|
| 469 |
|
| 470 | static int bpf_obj_name_cpy(char *dst, const char *src)
|
| 471 | {
|
| 472 | const char *end = src + BPF_OBJ_NAME_LEN;
|
| 473 |
|
| 474 | memset(dst, 0, BPF_OBJ_NAME_LEN);
|
| 475 |
|
| 476 | while (src < end && *src) {
|
| 477 | if (!isalnum(*src) &&
|
| 478 | *src != '_' && *src != '.')
|
| 479 | return -EINVAL;
|
| 480 | *dst++ = *src++;
|
| 481 | }
|
| 482 |
|
| 483 |
|
| 484 | if (src == end)
|
| 485 | return -EINVAL;
|
| 486 |
|
| 487 | return 0;
|
| 488 | }
|
| 489 |
|
| 490 | int map_check_no_btf(const struct bpf_map *map,
|
| 491 | const struct btf *btf,
|
| 492 | const struct btf_type *key_type,
|
| 493 | const struct btf_type *value_type)
|
| 494 | {
|
| 495 | return -ENOTSUPP;
|
| 496 | }
|
| 497 |
|
| 498 | static int map_check_btf(struct bpf_map *map, const struct btf *btf,
|
| 499 | u32 btf_key_id, u32 btf_value_id)
|
| 500 | {
|
| 501 | const struct btf_type *key_type, *value_type;
|
| 502 | u32 key_size, value_size;
|
| 503 | int ret = 0;
|
| 504 |
|
| 505 |
|
| 506 | if (btf_key_id) {
|
| 507 | key_type = btf_type_id_size(btf, &btf_key_id, &key_size);
|
| 508 | if (!key_type || key_size != map->key_size)
|
| 509 | return -EINVAL;
|
| 510 | } else {
|
| 511 | key_type = btf_type_by_id(btf, 0);
|
| 512 | if (!map->ops->map_check_btf)
|
| 513 | return -EINVAL;
|
| 514 | }
|
| 515 |
|
| 516 | value_type = btf_type_id_size(btf, &btf_value_id, &value_size);
|
| 517 | if (!value_type || value_size != map->value_size)
|
| 518 | return -EINVAL;
|
| 519 |
|
| 520 | map->spin_lock_off = btf_find_spin_lock(btf, value_type);
|
| 521 |
|
| 522 | if (map_value_has_spin_lock(map)) {
|
| 523 | if (map->map_flags & BPF_F_RDONLY_PROG)
|
| 524 | return -EACCES;
|
| 525 | if (map->map_type != BPF_MAP_TYPE_HASH &&
|
| 526 | map->map_type != BPF_MAP_TYPE_ARRAY &&
|
| 527 | map->map_type != BPF_MAP_TYPE_CGROUP_STORAGE &&
|
| 528 | map->map_type != BPF_MAP_TYPE_SK_STORAGE)
|
| 529 | return -ENOTSUPP;
|
| 530 | if (map->spin_lock_off + sizeof(struct bpf_spin_lock) >
|
| 531 | map->value_size) {
|
| 532 | WARN_ONCE(1,
|
| 533 | "verifier bug spin_lock_off %d value_size %d\n",
|
| 534 | map->spin_lock_off, map->value_size);
|
| 535 | return -EFAULT;
|
| 536 | }
|
| 537 | }
|
| 538 |
|
| 539 | if (map->ops->map_check_btf)
|
| 540 | ret = map->ops->map_check_btf(map, btf, key_type, value_type);
|
| 541 |
|
| 542 | return ret;
|
| 543 | }
|
| 544 |
|
| 545 | #define BPF_MAP_CREATE_LAST_FIELD btf_value_type_id
|
| 546 |
|
| 547 | static int map_create(union bpf_attr *attr)
|
| 548 | {
|
| 549 | int numa_node = bpf_map_attr_numa_node(attr);
|
| 550 | struct bpf_map_memory mem;
|
| 551 | struct bpf_map *map;
|
| 552 | int f_flags;
|
| 553 | int err;
|
| 554 |
|
| 555 | err = CHECK_ATTR(BPF_MAP_CREATE);
|
| 556 | if (err)
|
| 557 | return -EINVAL;
|
| 558 |
|
| 559 | f_flags = bpf_get_file_flag(attr->map_flags);
|
| 560 | if (f_flags < 0)
|
| 561 | return f_flags;
|
| 562 |
|
| 563 | if (numa_node != NUMA_NO_NODE &&
|
| 564 | ((unsigned int)numa_node >= nr_node_ids ||
|
| 565 | !node_online(numa_node)))
|
| 566 | return -EINVAL;
|
| 567 |
|
| 568 |
|
| 569 | map = find_and_alloc_map(attr);
|
| 570 | if (IS_ERR(map))
|
| 571 | return PTR_ERR(map);
|
| 572 |
|
| 573 | err = bpf_obj_name_cpy(map->name, attr->map_name);
|
| 574 | if (err)
|
| 575 | goto free_map;
|
| 576 |
|
| 577 | atomic_set(&map->refcnt, 1);
|
| 578 | atomic_set(&map->usercnt, 1);
|
| 579 |
|
| 580 | if (attr->btf_key_type_id || attr->btf_value_type_id) {
|
| 581 | struct btf *btf;
|
| 582 |
|
| 583 | if (!attr->btf_value_type_id) {
|
| 584 | err = -EINVAL;
|
| 585 | goto free_map;
|
| 586 | }
|
| 587 |
|
| 588 | btf = btf_get_by_fd(attr->btf_fd);
|
| 589 | if (IS_ERR(btf)) {
|
| 590 | err = PTR_ERR(btf);
|
| 591 | goto free_map;
|
| 592 | }
|
| 593 |
|
| 594 | err = map_check_btf(map, btf, attr->btf_key_type_id,
|
| 595 | attr->btf_value_type_id);
|
| 596 | if (err) {
|
| 597 | btf_put(btf);
|
| 598 | goto free_map;
|
| 599 | }
|
| 600 |
|
| 601 | map->btf = btf;
|
| 602 | map->btf_key_type_id = attr->btf_key_type_id;
|
| 603 | map->btf_value_type_id = attr->btf_value_type_id;
|
| 604 | } else {
|
| 605 | map->spin_lock_off = -EINVAL;
|
| 606 | }
|
| 607 |
|
| 608 | err = security_bpf_map_alloc(map);
|
| 609 | if (err)
|
| 610 | goto free_map;
|
| 611 |
|
| 612 | err = bpf_map_alloc_id(map);
|
| 613 | if (err)
|
| 614 | goto free_map_sec;
|
| 615 |
|
| 616 | err = bpf_map_new_fd(map, f_flags);
|
| 617 | if (err < 0) {
|
| 618 | |
| 619 | * bpf_map_put_with_uref() is needed because the above
|
| 620 | * bpf_map_alloc_id() has published the map
|
| 621 | * to the userspace and the userspace may
|
| 622 | * have refcnt-ed it through BPF_MAP_GET_FD_BY_ID.
|
| 623 |
|
| 624 | bpf_map_put_with_uref(map);
|
| 625 | return err;
|
| 626 | }
|
| 627 |
|
| 628 | return err;
|
| 629 |
|
| 630 | free_map_sec:
|
| 631 | security_bpf_map_free(map);
|
| 632 | free_map:
|
| 633 | btf_put(map->btf);
|
| 634 | bpf_map_charge_move(&mem, &map->memory);
|
| 635 | map->ops->map_free(map);
|
| 636 | bpf_map_charge_finish(&mem);
|
| 637 | return err;
|
| 638 | }
|
| 639 |
|
| 640 | |
| 641 | * On success caller should complete fd access with matching fdput()
|
| 642 |
|
| 643 | struct bpf_map *__bpf_map_get(struct fd f)
|
| 644 | {
|
| 645 | if (!f.file)
|
| 646 | return ERR_PTR(-EBADF);
|
| 647 | if (f.file->f_op != &bpf_map_fops) {
|
| 648 | fdput(f);
|
| 649 | return ERR_PTR(-EINVAL);
|
| 650 | }
|
| 651 |
|
| 652 | return f.file->private_data;
|
| 653 | }
|
| 654 |
|
| 655 |
|
| 656 | #define BPF_MAX_REFCNT 32768
|
| 657 |
|
| 658 | struct bpf_map *bpf_map_inc(struct bpf_map *map, bool uref)
|
| 659 | {
|
| 660 | if (atomic_inc_return(&map->refcnt) > BPF_MAX_REFCNT) {
|
| 661 | atomic_dec(&map->refcnt);
|
| 662 | return ERR_PTR(-EBUSY);
|
| 663 | }
|
| 664 | if (uref)
|
| 665 | atomic_inc(&map->usercnt);
|
| 666 | return map;
|
| 667 | }
|
| 668 | EXPORT_SYMBOL_GPL(bpf_map_inc);
|
| 669 |
|
| 670 | struct bpf_map *bpf_map_get_with_uref(u32 ufd)
|
| 671 | {
|
| 672 | struct fd f = fdget(ufd);
|
| 673 | struct bpf_map *map;
|
| 674 |
|
| 675 | map = __bpf_map_get(f);
|
| 676 | if (IS_ERR(map))
|
| 677 | return map;
|
| 678 |
|
| 679 | map = bpf_map_inc(map, true);
|
| 680 | fdput(f);
|
| 681 |
|
| 682 | return map;
|
| 683 | }
|
| 684 |
|
| 685 |
|
| 686 | static struct bpf_map *bpf_map_inc_not_zero(struct bpf_map *map,
|
| 687 | bool uref)
|
| 688 | {
|
| 689 | int refold;
|
| 690 |
|
| 691 | refold = atomic_fetch_add_unless(&map->refcnt, 1, 0);
|
| 692 |
|
| 693 | if (refold >= BPF_MAX_REFCNT) {
|
| 694 | __bpf_map_put(map, false);
|
| 695 | return ERR_PTR(-EBUSY);
|
| 696 | }
|
| 697 |
|
| 698 | if (!refold)
|
| 699 | return ERR_PTR(-ENOENT);
|
| 700 |
|
| 701 | if (uref)
|
| 702 | atomic_inc(&map->usercnt);
|
| 703 |
|
| 704 | return map;
|
| 705 | }
|
| 706 |
|
| 707 | int __weak bpf_stackmap_copy(struct bpf_map *map, void *key, void *value)
|
| 708 | {
|
| 709 | return -ENOTSUPP;
|
| 710 | }
|
| 711 |
|
| 712 | static void *__bpf_copy_key(void __user *ukey, u64 key_size)
|
| 713 | {
|
| 714 | if (key_size)
|
| 715 | return memdup_user(ukey, key_size);
|
| 716 |
|
| 717 | if (ukey)
|
| 718 | return ERR_PTR(-EINVAL);
|
| 719 |
|
| 720 | return NULL;
|
| 721 | }
|
| 722 |
|
| 723 |
|
| 724 | #define BPF_MAP_LOOKUP_ELEM_LAST_FIELD flags
|
| 725 |
|
| 726 | static int map_lookup_elem(union bpf_attr *attr)
|
| 727 | {
|
| 728 | void __user *ukey = u64_to_user_ptr(attr->key);
|
| 729 | void __user *uvalue = u64_to_user_ptr(attr->value);
|
| 730 | int ufd = attr->map_fd;
|
| 731 | struct bpf_map *map;
|
| 732 | void *key, *value, *ptr;
|
| 733 | u32 value_size;
|
| 734 | struct fd f;
|
| 735 | int err;
|
| 736 |
|
| 737 | if (CHECK_ATTR(BPF_MAP_LOOKUP_ELEM))
|
| 738 | return -EINVAL;
|
| 739 |
|
| 740 | if (attr->flags & ~BPF_F_LOCK)
|
| 741 | return -EINVAL;
|
| 742 |
|
| 743 | f = fdget(ufd);
|
| 744 | map = __bpf_map_get(f);
|
| 745 | if (IS_ERR(map))
|
| 746 | return PTR_ERR(map);
|
| 747 | if (!(map_get_sys_perms(map, f) & FMODE_CAN_READ)) {
|
| 748 | err = -EPERM;
|
| 749 | goto err_put;
|
| 750 | }
|
| 751 |
|
| 752 | if ((attr->flags & BPF_F_LOCK) &&
|
| 753 | !map_value_has_spin_lock(map)) {
|
| 754 | err = -EINVAL;
|
| 755 | goto err_put;
|
| 756 | }
|
| 757 |
|
| 758 | key = __bpf_copy_key(ukey, map->key_size);
|
| 759 | if (IS_ERR(key)) {
|
| 760 | err = PTR_ERR(key);
|
| 761 | goto err_put;
|
| 762 | }
|
| 763 |
|
| 764 | if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH ||
|
| 765 | map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH ||
|
| 766 | map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY ||
|
| 767 | map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE)
|
| 768 | value_size = round_up(map->value_size, 8) * num_possible_cpus();
|
| 769 | else if (IS_FD_MAP(map))
|
| 770 | value_size = sizeof(u32);
|
| 771 | else
|
| 772 | value_size = map->value_size;
|
| 773 |
|
| 774 | err = -ENOMEM;
|
| 775 | value = kmalloc(value_size, GFP_USER | __GFP_NOWARN);
|
| 776 | if (!value)
|
| 777 | goto free_key;
|
| 778 |
|
| 779 | if (bpf_map_is_dev_bound(map)) {
|
| 780 | err = bpf_map_offload_lookup_elem(map, key, value);
|
| 781 | goto done;
|
| 782 | }
|
| 783 |
|
| 784 | preempt_disable();
|
| 785 | this_cpu_inc(bpf_prog_active);
|
| 786 | if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH ||
|
| 787 | map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH) {
|
| 788 | err = bpf_percpu_hash_copy(map, key, value);
|
| 789 | } else if (map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY) {
|
| 790 | err = bpf_percpu_array_copy(map, key, value);
|
| 791 | } else if (map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE) {
|
| 792 | err = bpf_percpu_cgroup_storage_copy(map, key, value);
|
| 793 | } else if (map->map_type == BPF_MAP_TYPE_STACK_TRACE) {
|
| 794 | err = bpf_stackmap_copy(map, key, value);
|
| 795 | } else if (IS_FD_ARRAY(map)) {
|
| 796 | err = bpf_fd_array_map_lookup_elem(map, key, value);
|
| 797 | } else if (IS_FD_HASH(map)) {
|
| 798 | err = bpf_fd_htab_map_lookup_elem(map, key, value);
|
| 799 | } else if (map->map_type == BPF_MAP_TYPE_REUSEPORT_SOCKARRAY) {
|
| 800 | err = bpf_fd_reuseport_array_lookup_elem(map, key, value);
|
| 801 | } else if (map->map_type == BPF_MAP_TYPE_QUEUE ||
|
| 802 | map->map_type == BPF_MAP_TYPE_STACK) {
|
| 803 | err = map->ops->map_peek_elem(map, value);
|
| 804 | } else {
|
| 805 | rcu_read_lock();
|
| 806 | if (map->ops->map_lookup_elem_sys_only)
|
| 807 | ptr = map->ops->map_lookup_elem_sys_only(map, key);
|
| 808 | else
|
| 809 | ptr = map->ops->map_lookup_elem(map, key);
|
| 810 | if (IS_ERR(ptr)) {
|
| 811 | err = PTR_ERR(ptr);
|
| 812 | } else if (!ptr) {
|
| 813 | err = -ENOENT;
|
| 814 | } else {
|
| 815 | err = 0;
|
| 816 | if (attr->flags & BPF_F_LOCK)
|
| 817 |
|
| 818 | copy_map_value_locked(map, value, ptr, true);
|
| 819 | else
|
| 820 | copy_map_value(map, value, ptr);
|
| 821 |
|
| 822 | check_and_init_map_lock(map, value);
|
| 823 | }
|
| 824 | rcu_read_unlock();
|
| 825 | }
|
| 826 | this_cpu_dec(bpf_prog_active);
|
| 827 | preempt_enable();
|
| 828 |
|
| 829 | done:
|
| 830 | if (err)
|
| 831 | goto free_value;
|
| 832 |
|
| 833 | err = -EFAULT;
|
| 834 | if (copy_to_user(uvalue, value, value_size) != 0)
|
| 835 | goto free_value;
|
| 836 |
|
| 837 | err = 0;
|
| 838 |
|
| 839 | free_value:
|
| 840 | kfree(value);
|
| 841 | free_key:
|
| 842 | kfree(key);
|
| 843 | err_put:
|
| 844 | fdput(f);
|
| 845 | return err;
|
| 846 | }
|
| 847 |
|
| 848 | static void maybe_wait_bpf_programs(struct bpf_map *map)
|
| 849 | {
|
| 850 | |
| 851 | * userspace, when we return to it, knows that all programs
|
| 852 | * that could be running use the new map value.
|
| 853 |
|
| 854 | if (map->map_type == BPF_MAP_TYPE_HASH_OF_MAPS ||
|
| 855 | map->map_type == BPF_MAP_TYPE_ARRAY_OF_MAPS)
|
| 856 | synchronize_rcu();
|
| 857 | }
|
| 858 |
|
| 859 | #define BPF_MAP_UPDATE_ELEM_LAST_FIELD flags
|
| 860 |
|
| 861 | static int map_update_elem(union bpf_attr *attr)
|
| 862 | {
|
| 863 | void __user *ukey = u64_to_user_ptr(attr->key);
|
| 864 | void __user *uvalue = u64_to_user_ptr(attr->value);
|
| 865 | int ufd = attr->map_fd;
|
| 866 | struct bpf_map *map;
|
| 867 | void *key, *value;
|
| 868 | u32 value_size;
|
| 869 | struct fd f;
|
| 870 | int err;
|
| 871 |
|
| 872 | if (CHECK_ATTR(BPF_MAP_UPDATE_ELEM))
|
| 873 | return -EINVAL;
|
| 874 |
|
| 875 | f = fdget(ufd);
|
| 876 | map = __bpf_map_get(f);
|
| 877 | if (IS_ERR(map))
|
| 878 | return PTR_ERR(map);
|
| 879 | if (!(map_get_sys_perms(map, f) & FMODE_CAN_WRITE)) {
|
| 880 | err = -EPERM;
|
| 881 | goto err_put;
|
| 882 | }
|
| 883 |
|
| 884 | if ((attr->flags & BPF_F_LOCK) &&
|
| 885 | !map_value_has_spin_lock(map)) {
|
| 886 | err = -EINVAL;
|
| 887 | goto err_put;
|
| 888 | }
|
| 889 |
|
| 890 | key = __bpf_copy_key(ukey, map->key_size);
|
| 891 | if (IS_ERR(key)) {
|
| 892 | err = PTR_ERR(key);
|
| 893 | goto err_put;
|
| 894 | }
|
| 895 |
|
| 896 | if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH ||
|
| 897 | map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH ||
|
| 898 | map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY ||
|
| 899 | map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE)
|
| 900 | value_size = round_up(map->value_size, 8) * num_possible_cpus();
|
| 901 | else
|
| 902 | value_size = map->value_size;
|
| 903 |
|
| 904 | err = -ENOMEM;
|
| 905 | value = kmalloc(value_size, GFP_USER | __GFP_NOWARN);
|
| 906 | if (!value)
|
| 907 | goto free_key;
|
| 908 |
|
| 909 | err = -EFAULT;
|
| 910 | if (copy_from_user(value, uvalue, value_size) != 0)
|
| 911 | goto free_value;
|
| 912 |
|
| 913 |
|
| 914 | if (bpf_map_is_dev_bound(map)) {
|
| 915 | err = bpf_map_offload_update_elem(map, key, value, attr->flags);
|
| 916 | goto out;
|
| 917 | } else if (map->map_type == BPF_MAP_TYPE_CPUMAP ||
|
| 918 | map->map_type == BPF_MAP_TYPE_SOCKHASH ||
|
| 919 | map->map_type == BPF_MAP_TYPE_SOCKMAP) {
|
| 920 | err = map->ops->map_update_elem(map, key, value, attr->flags);
|
| 921 | goto out;
|
| 922 | }
|
| 923 |
|
| 924 | |
| 925 | * inside bpf map update or delete otherwise deadlocks are possible
|
| 926 |
|
| 927 | preempt_disable();
|
| 928 | __this_cpu_inc(bpf_prog_active);
|
| 929 | if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH ||
|
| 930 | map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH) {
|
| 931 | err = bpf_percpu_hash_update(map, key, value, attr->flags);
|
| 932 | } else if (map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY) {
|
| 933 | err = bpf_percpu_array_update(map, key, value, attr->flags);
|
| 934 | } else if (map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE) {
|
| 935 | err = bpf_percpu_cgroup_storage_update(map, key, value,
|
| 936 | attr->flags);
|
| 937 | } else if (IS_FD_ARRAY(map)) {
|
| 938 | rcu_read_lock();
|
| 939 | err = bpf_fd_array_map_update_elem(map, f.file, key, value,
|
| 940 | attr->flags);
|
| 941 | rcu_read_unlock();
|
| 942 | } else if (map->map_type == BPF_MAP_TYPE_HASH_OF_MAPS) {
|
| 943 | rcu_read_lock();
|
| 944 | err = bpf_fd_htab_map_update_elem(map, f.file, key, value,
|
| 945 | attr->flags);
|
| 946 | rcu_read_unlock();
|
| 947 | } else if (map->map_type == BPF_MAP_TYPE_REUSEPORT_SOCKARRAY) {
|
| 948 |
|
| 949 | err = bpf_fd_reuseport_array_update_elem(map, key, value,
|
| 950 | attr->flags);
|
| 951 | } else if (map->map_type == BPF_MAP_TYPE_QUEUE ||
|
| 952 | map->map_type == BPF_MAP_TYPE_STACK) {
|
| 953 | err = map->ops->map_push_elem(map, value, attr->flags);
|
| 954 | } else {
|
| 955 | rcu_read_lock();
|
| 956 | err = map->ops->map_update_elem(map, key, value, attr->flags);
|
| 957 | rcu_read_unlock();
|
| 958 | }
|
| 959 | __this_cpu_dec(bpf_prog_active);
|
| 960 | preempt_enable();
|
| 961 | maybe_wait_bpf_programs(map);
|
| 962 | out:
|
| 963 | free_value:
|
| 964 | kfree(value);
|
| 965 | free_key:
|
| 966 | kfree(key);
|
| 967 | err_put:
|
| 968 | fdput(f);
|
| 969 | return err;
|
| 970 | }
|
| 971 |
|
| 972 | #define BPF_MAP_DELETE_ELEM_LAST_FIELD key
|
| 973 |
|
| 974 | static int map_delete_elem(union bpf_attr *attr)
|
| 975 | {
|
| 976 | void __user *ukey = u64_to_user_ptr(attr->key);
|
| 977 | int ufd = attr->map_fd;
|
| 978 | struct bpf_map *map;
|
| 979 | struct fd f;
|
| 980 | void *key;
|
| 981 | int err;
|
| 982 |
|
| 983 | if (CHECK_ATTR(BPF_MAP_DELETE_ELEM))
|
| 984 | return -EINVAL;
|
| 985 |
|
| 986 | f = fdget(ufd);
|
| 987 | map = __bpf_map_get(f);
|
| 988 | if (IS_ERR(map))
|
| 989 | return PTR_ERR(map);
|
| 990 | if (!(map_get_sys_perms(map, f) & FMODE_CAN_WRITE)) {
|
| 991 | err = -EPERM;
|
| 992 | goto err_put;
|
| 993 | }
|
| 994 |
|
| 995 | key = __bpf_copy_key(ukey, map->key_size);
|
| 996 | if (IS_ERR(key)) {
|
| 997 | err = PTR_ERR(key);
|
| 998 | goto err_put;
|
| 999 | }
|
| 1000 |
|
| 1001 | if (bpf_map_is_dev_bound(map)) {
|
| 1002 | err = bpf_map_offload_delete_elem(map, key);
|
| 1003 | goto out;
|
| 1004 | }
|
| 1005 |
|
| 1006 | preempt_disable();
|
| 1007 | __this_cpu_inc(bpf_prog_active);
|
| 1008 | rcu_read_lock();
|
| 1009 | err = map->ops->map_delete_elem(map, key);
|
| 1010 | rcu_read_unlock();
|
| 1011 | __this_cpu_dec(bpf_prog_active);
|
| 1012 | preempt_enable();
|
| 1013 | maybe_wait_bpf_programs(map);
|
| 1014 | out:
|
| 1015 | kfree(key);
|
| 1016 | err_put:
|
| 1017 | fdput(f);
|
| 1018 | return err;
|
| 1019 | }
|
| 1020 |
|
| 1021 |
|
| 1022 | #define BPF_MAP_GET_NEXT_KEY_LAST_FIELD next_key
|
| 1023 |
|
| 1024 | static int map_get_next_key(union bpf_attr *attr)
|
| 1025 | {
|
| 1026 | void __user *ukey = u64_to_user_ptr(attr->key);
|
| 1027 | void __user *unext_key = u64_to_user_ptr(attr->next_key);
|
| 1028 | int ufd = attr->map_fd;
|
| 1029 | struct bpf_map *map;
|
| 1030 | void *key, *next_key;
|
| 1031 | struct fd f;
|
| 1032 | int err;
|
| 1033 |
|
| 1034 | if (CHECK_ATTR(BPF_MAP_GET_NEXT_KEY))
|
| 1035 | return -EINVAL;
|
| 1036 |
|
| 1037 | f = fdget(ufd);
|
| 1038 | map = __bpf_map_get(f);
|
| 1039 | if (IS_ERR(map))
|
| 1040 | return PTR_ERR(map);
|
| 1041 | if (!(map_get_sys_perms(map, f) & FMODE_CAN_READ)) {
|
| 1042 | err = -EPERM;
|
| 1043 | goto err_put;
|
| 1044 | }
|
| 1045 |
|
| 1046 | if (ukey) {
|
| 1047 | key = __bpf_copy_key(ukey, map->key_size);
|
| 1048 | if (IS_ERR(key)) {
|
| 1049 | err = PTR_ERR(key);
|
| 1050 | goto err_put;
|
| 1051 | }
|
| 1052 | } else {
|
| 1053 | key = NULL;
|
| 1054 | }
|
| 1055 |
|
| 1056 | err = -ENOMEM;
|
| 1057 | next_key = kmalloc(map->key_size, GFP_USER);
|
| 1058 | if (!next_key)
|
| 1059 | goto free_key;
|
| 1060 |
|
| 1061 | if (bpf_map_is_dev_bound(map)) {
|
| 1062 | err = bpf_map_offload_get_next_key(map, key, next_key);
|
| 1063 | goto out;
|
| 1064 | }
|
| 1065 |
|
| 1066 | rcu_read_lock();
|
| 1067 | err = map->ops->map_get_next_key(map, key, next_key);
|
| 1068 | rcu_read_unlock();
|
| 1069 | out:
|
| 1070 | if (err)
|
| 1071 | goto free_next_key;
|
| 1072 |
|
| 1073 | err = -EFAULT;
|
| 1074 | if (copy_to_user(unext_key, next_key, map->key_size) != 0)
|
| 1075 | goto free_next_key;
|
| 1076 |
|
| 1077 | err = 0;
|
| 1078 |
|
| 1079 | free_next_key:
|
| 1080 | kfree(next_key);
|
| 1081 | free_key:
|
| 1082 | kfree(key);
|
| 1083 | err_put:
|
| 1084 | fdput(f);
|
| 1085 | return err;
|
| 1086 | }
|
| 1087 |
|
| 1088 | #define BPF_MAP_LOOKUP_AND_DELETE_ELEM_LAST_FIELD value
|
| 1089 |
|
| 1090 | static int map_lookup_and_delete_elem(union bpf_attr *attr)
|
| 1091 | {
|
| 1092 | void __user *ukey = u64_to_user_ptr(attr->key);
|
| 1093 | void __user *uvalue = u64_to_user_ptr(attr->value);
|
| 1094 | int ufd = attr->map_fd;
|
| 1095 | struct bpf_map *map;
|
| 1096 | void *key, *value;
|
| 1097 | u32 value_size;
|
| 1098 | struct fd f;
|
| 1099 | int err;
|
| 1100 |
|
| 1101 | if (CHECK_ATTR(BPF_MAP_LOOKUP_AND_DELETE_ELEM))
|
| 1102 | return -EINVAL;
|
| 1103 |
|
| 1104 | f = fdget(ufd);
|
| 1105 | map = __bpf_map_get(f);
|
| 1106 | if (IS_ERR(map))
|
| 1107 | return PTR_ERR(map);
|
| 1108 | if (!(map_get_sys_perms(map, f) & FMODE_CAN_WRITE)) {
|
| 1109 | err = -EPERM;
|
| 1110 | goto err_put;
|
| 1111 | }
|
| 1112 |
|
| 1113 | key = __bpf_copy_key(ukey, map->key_size);
|
| 1114 | if (IS_ERR(key)) {
|
| 1115 | err = PTR_ERR(key);
|
| 1116 | goto err_put;
|
| 1117 | }
|
| 1118 |
|
| 1119 | value_size = map->value_size;
|
| 1120 |
|
| 1121 | err = -ENOMEM;
|
| 1122 | value = kmalloc(value_size, GFP_USER | __GFP_NOWARN);
|
| 1123 | if (!value)
|
| 1124 | goto free_key;
|
| 1125 |
|
| 1126 | if (map->map_type == BPF_MAP_TYPE_QUEUE ||
|
| 1127 | map->map_type == BPF_MAP_TYPE_STACK) {
|
| 1128 | err = map->ops->map_pop_elem(map, value);
|
| 1129 | } else {
|
| 1130 | err = -ENOTSUPP;
|
| 1131 | }
|
| 1132 |
|
| 1133 | if (err)
|
| 1134 | goto free_value;
|
| 1135 |
|
| 1136 | if (copy_to_user(uvalue, value, value_size) != 0)
|
| 1137 | goto free_value;
|
| 1138 |
|
| 1139 | err = 0;
|
| 1140 |
|
| 1141 | free_value:
|
| 1142 | kfree(value);
|
| 1143 | free_key:
|
| 1144 | kfree(key);
|
| 1145 | err_put:
|
| 1146 | fdput(f);
|
| 1147 | return err;
|
| 1148 | }
|
| 1149 |
|
| 1150 | #define BPF_MAP_FREEZE_LAST_FIELD map_fd
|
| 1151 |
|
| 1152 | static int map_freeze(const union bpf_attr *attr)
|
| 1153 | {
|
| 1154 | int err = 0, ufd = attr->map_fd;
|
| 1155 | struct bpf_map *map;
|
| 1156 | struct fd f;
|
| 1157 |
|
| 1158 | if (CHECK_ATTR(BPF_MAP_FREEZE))
|
| 1159 | return -EINVAL;
|
| 1160 |
|
| 1161 | f = fdget(ufd);
|
| 1162 | map = __bpf_map_get(f);
|
| 1163 | if (IS_ERR(map))
|
| 1164 | return PTR_ERR(map);
|
| 1165 | if (READ_ONCE(map->frozen)) {
|
| 1166 | err = -EBUSY;
|
| 1167 | goto err_put;
|
| 1168 | }
|
| 1169 | if (!capable(CAP_SYS_ADMIN)) {
|
| 1170 | err = -EPERM;
|
| 1171 | goto err_put;
|
| 1172 | }
|
| 1173 |
|
| 1174 | WRITE_ONCE(map->frozen, true);
|
| 1175 | err_put:
|
| 1176 | fdput(f);
|
| 1177 | return err;
|
| 1178 | }
|
| 1179 |
|
| 1180 | static const struct bpf_prog_ops * const bpf_prog_types[] = {
|
| 1181 | #define BPF_PROG_TYPE(_id, _name) \
|
| 1182 | [_id] = & _name ## _prog_ops,
|
| 1183 | #define BPF_MAP_TYPE(_id, _ops)
|
| 1184 | #include <linux/bpf_types.h>
|
| 1185 | #undef BPF_PROG_TYPE
|
| 1186 | #undef BPF_MAP_TYPE
|
| 1187 | };
|
| 1188 |
|
| 1189 | static int find_prog_type(enum bpf_prog_type type, struct bpf_prog *prog)
|
| 1190 | {
|
| 1191 | const struct bpf_prog_ops *ops;
|
| 1192 |
|
| 1193 | if (type >= ARRAY_SIZE(bpf_prog_types))
|
| 1194 | return -EINVAL;
|
| 1195 | type = array_index_nospec(type, ARRAY_SIZE(bpf_prog_types));
|
| 1196 | ops = bpf_prog_types[type];
|
| 1197 | if (!ops)
|
| 1198 | return -EINVAL;
|
| 1199 |
|
| 1200 | if (!bpf_prog_is_dev_bound(prog->aux))
|
| 1201 | prog->aux->ops = ops;
|
| 1202 | else
|
| 1203 | prog->aux->ops = &bpf_offload_prog_ops;
|
| 1204 | prog->type = type;
|
| 1205 | return 0;
|
| 1206 | }
|
| 1207 |
|
| 1208 |
|
| 1209 | static void free_used_maps(struct bpf_prog_aux *aux)
|
| 1210 | {
|
| 1211 | enum bpf_cgroup_storage_type stype;
|
| 1212 | int i;
|
| 1213 |
|
| 1214 | for_each_cgroup_storage_type(stype) {
|
| 1215 | if (!aux->cgroup_storage[stype])
|
| 1216 | continue;
|
| 1217 | bpf_cgroup_storage_release(aux->prog,
|
| 1218 | aux->cgroup_storage[stype]);
|
| 1219 | }
|
| 1220 |
|
| 1221 | for (i = 0; i < aux->used_map_cnt; i++)
|
| 1222 | bpf_map_put(aux->used_maps[i]);
|
| 1223 |
|
| 1224 | kfree(aux->used_maps);
|
| 1225 | }
|
| 1226 |
|
| 1227 | int __bpf_prog_charge(struct user_struct *user, u32 pages)
|
| 1228 | {
|
| 1229 | unsigned long memlock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;
|
| 1230 | unsigned long user_bufs;
|
| 1231 |
|
| 1232 | if (user) {
|
| 1233 | user_bufs = atomic_long_add_return(pages, &user->locked_vm);
|
| 1234 | if (user_bufs > memlock_limit) {
|
| 1235 | atomic_long_sub(pages, &user->locked_vm);
|
| 1236 | return -EPERM;
|
| 1237 | }
|
| 1238 | }
|
| 1239 |
|
| 1240 | return 0;
|
| 1241 | }
|
| 1242 |
|
| 1243 | void __bpf_prog_uncharge(struct user_struct *user, u32 pages)
|
| 1244 | {
|
| 1245 | if (user)
|
| 1246 | atomic_long_sub(pages, &user->locked_vm);
|
| 1247 | }
|
| 1248 |
|
| 1249 | static int bpf_prog_charge_memlock(struct bpf_prog *prog)
|
| 1250 | {
|
| 1251 | struct user_struct *user = get_current_user();
|
| 1252 | int ret;
|
| 1253 |
|
| 1254 | ret = __bpf_prog_charge(user, prog->pages);
|
| 1255 | if (ret) {
|
| 1256 | free_uid(user);
|
| 1257 | return ret;
|
| 1258 | }
|
| 1259 |
|
| 1260 | prog->aux->user = user;
|
| 1261 | return 0;
|
| 1262 | }
|
| 1263 |
|
| 1264 | static void bpf_prog_uncharge_memlock(struct bpf_prog *prog)
|
| 1265 | {
|
| 1266 | struct user_struct *user = prog->aux->user;
|
| 1267 |
|
| 1268 | __bpf_prog_uncharge(user, prog->pages);
|
| 1269 | free_uid(user);
|
| 1270 | }
|
| 1271 |
|
| 1272 | static int bpf_prog_alloc_id(struct bpf_prog *prog)
|
| 1273 | {
|
| 1274 | int id;
|
| 1275 |
|
| 1276 | idr_preload(GFP_KERNEL);
|
| 1277 | spin_lock_bh(&prog_idr_lock);
|
| 1278 | id = idr_alloc_cyclic(&prog_idr, prog, 1, INT_MAX, GFP_ATOMIC);
|
| 1279 | if (id > 0)
|
| 1280 | prog->aux->id = id;
|
| 1281 | spin_unlock_bh(&prog_idr_lock);
|
| 1282 | idr_preload_end();
|
| 1283 |
|
| 1284 |
|
| 1285 | if (WARN_ON_ONCE(!id))
|
| 1286 | return -ENOSPC;
|
| 1287 |
|
| 1288 | return id > 0 ? 0 : id;
|
| 1289 | }
|
| 1290 |
|
| 1291 | void bpf_prog_free_id(struct bpf_prog *prog, bool do_idr_lock)
|
| 1292 | {
|
| 1293 | |
| 1294 | * Offloaded programs are removed from the store when their device
|
| 1295 | * disappears - even if someone grabs an fd to them they are unusable,
|
| 1296 | * simply waiting for refcnt to drop to be freed.
|
| 1297 |
|
| 1298 | if (!prog->aux->id)
|
| 1299 | return;
|
| 1300 |
|
| 1301 | if (do_idr_lock)
|
| 1302 | spin_lock_bh(&prog_idr_lock);
|
| 1303 | else
|
| 1304 | __acquire(&prog_idr_lock);
|
| 1305 |
|
| 1306 | idr_remove(&prog_idr, prog->aux->id);
|
| 1307 | prog->aux->id = 0;
|
| 1308 |
|
| 1309 | if (do_idr_lock)
|
| 1310 | spin_unlock_bh(&prog_idr_lock);
|
| 1311 | else
|
| 1312 | __release(&prog_idr_lock);
|
| 1313 | }
|
| 1314 |
|
| 1315 | static void __bpf_prog_put_rcu(struct rcu_head *rcu)
|
| 1316 | {
|
| 1317 | struct bpf_prog_aux *aux = container_of(rcu, struct bpf_prog_aux, rcu);
|
| 1318 |
|
| 1319 | free_used_maps(aux);
|
| 1320 | bpf_prog_uncharge_memlock(aux->prog);
|
| 1321 | security_bpf_prog_free(aux);
|
| 1322 | bpf_prog_free(aux->prog);
|
| 1323 | }
|
| 1324 |
|
| 1325 | static void __bpf_prog_put(struct bpf_prog *prog, bool do_idr_lock)
|
| 1326 | {
|
| 1327 | if (atomic_dec_and_test(&prog->aux->refcnt)) {
|
| 1328 | perf_event_bpf_event(prog, PERF_BPF_EVENT_PROG_UNLOAD, 0);
|
| 1329 |
|
| 1330 | bpf_prog_free_id(prog, do_idr_lock);
|
| 1331 | bpf_prog_kallsyms_del_all(prog);
|
| 1332 | btf_put(prog->aux->btf);
|
| 1333 | kvfree(prog->aux->func_info);
|
| 1334 | bpf_prog_free_linfo(prog);
|
| 1335 |
|
| 1336 | call_rcu(&prog->aux->rcu, __bpf_prog_put_rcu);
|
| 1337 | }
|
| 1338 | }
|
| 1339 |
|
| 1340 | void bpf_prog_put(struct bpf_prog *prog)
|
| 1341 | {
|
| 1342 | __bpf_prog_put(prog, true);
|
| 1343 | }
|
| 1344 | EXPORT_SYMBOL_GPL(bpf_prog_put);
|
| 1345 |
|
| 1346 | static int bpf_prog_release(struct inode *inode, struct file *filp)
|
| 1347 | {
|
| 1348 | struct bpf_prog *prog = filp->private_data;
|
| 1349 |
|
| 1350 | bpf_prog_put(prog);
|
| 1351 | return 0;
|
| 1352 | }
|
| 1353 |
|
| 1354 | static void bpf_prog_get_stats(const struct bpf_prog *prog,
|
| 1355 | struct bpf_prog_stats *stats)
|
| 1356 | {
|
| 1357 | u64 nsecs = 0, cnt = 0;
|
| 1358 | int cpu;
|
| 1359 |
|
| 1360 | for_each_possible_cpu(cpu) {
|
| 1361 | const struct bpf_prog_stats *st;
|
| 1362 | unsigned int start;
|
| 1363 | u64 tnsecs, tcnt;
|
| 1364 |
|
| 1365 | st = per_cpu_ptr(prog->aux->stats, cpu);
|
| 1366 | do {
|
| 1367 | start = u64_stats_fetch_begin_irq(&st->syncp);
|
| 1368 | tnsecs = st->nsecs;
|
| 1369 | tcnt = st->cnt;
|
| 1370 | } while (u64_stats_fetch_retry_irq(&st->syncp, start));
|
| 1371 | nsecs += tnsecs;
|
| 1372 | cnt += tcnt;
|
| 1373 | }
|
| 1374 | stats->nsecs = nsecs;
|
| 1375 | stats->cnt = cnt;
|
| 1376 | }
|
| 1377 |
|
| 1378 | #ifdef CONFIG_PROC_FS
|
| 1379 | static void bpf_prog_show_fdinfo(struct seq_file *m, struct file *filp)
|
| 1380 | {
|
| 1381 | const struct bpf_prog *prog = filp->private_data;
|
| 1382 | char prog_tag[sizeof(prog->tag) * 2 + 1] = { };
|
| 1383 | struct bpf_prog_stats stats;
|
| 1384 |
|
| 1385 | bpf_prog_get_stats(prog, &stats);
|
| 1386 | bin2hex(prog_tag, prog->tag, sizeof(prog->tag));
|
| 1387 | seq_printf(m,
|
| 1388 | "prog_type:\t%u\n"
|
| 1389 | "prog_jited:\t%u\n"
|
| 1390 | "prog_tag:\t%s\n"
|
| 1391 | "memlock:\t%llu\n"
|
| 1392 | "prog_id:\t%u\n"
|
| 1393 | "run_time_ns:\t%llu\n"
|
| 1394 | "run_cnt:\t%llu\n",
|
| 1395 | prog->type,
|
| 1396 | prog->jited,
|
| 1397 | prog_tag,
|
| 1398 | prog->pages * 1ULL << PAGE_SHIFT,
|
| 1399 | prog->aux->id,
|
| 1400 | stats.nsecs,
|
| 1401 | stats.cnt);
|
| 1402 | }
|
| 1403 | #endif
|
| 1404 |
|
| 1405 | const struct file_operations bpf_prog_fops = {
|
| 1406 | #ifdef CONFIG_PROC_FS
|
| 1407 | .show_fdinfo = bpf_prog_show_fdinfo,
|
| 1408 | #endif
|
| 1409 | .release = bpf_prog_release,
|
| 1410 | .read = bpf_dummy_read,
|
| 1411 | .write = bpf_dummy_write,
|
| 1412 | };
|
| 1413 |
|
| 1414 | int bpf_prog_new_fd(struct bpf_prog *prog)
|
| 1415 | {
|
| 1416 | int ret;
|
| 1417 |
|
| 1418 | ret = security_bpf_prog(prog);
|
| 1419 | if (ret < 0)
|
| 1420 | return ret;
|
| 1421 |
|
| 1422 | return anon_inode_getfd("bpf-prog", &bpf_prog_fops, prog,
|
| 1423 | O_RDWR | O_CLOEXEC);
|
| 1424 | }
|
| 1425 |
|
| 1426 | static struct bpf_prog *____bpf_prog_get(struct fd f)
|
| 1427 | {
|
| 1428 | if (!f.file)
|
| 1429 | return ERR_PTR(-EBADF);
|
| 1430 | if (f.file->f_op != &bpf_prog_fops) {
|
| 1431 | fdput(f);
|
| 1432 | return ERR_PTR(-EINVAL);
|
| 1433 | }
|
| 1434 |
|
| 1435 | return f.file->private_data;
|
| 1436 | }
|
| 1437 |
|
| 1438 | struct bpf_prog *bpf_prog_add(struct bpf_prog *prog, int i)
|
| 1439 | {
|
| 1440 | if (atomic_add_return(i, &prog->aux->refcnt) > BPF_MAX_REFCNT) {
|
| 1441 | atomic_sub(i, &prog->aux->refcnt);
|
| 1442 | return ERR_PTR(-EBUSY);
|
| 1443 | }
|
| 1444 | return prog;
|
| 1445 | }
|
| 1446 | EXPORT_SYMBOL_GPL(bpf_prog_add);
|
| 1447 |
|
| 1448 | void bpf_prog_sub(struct bpf_prog *prog, int i)
|
| 1449 | {
|
| 1450 | |
| 1451 | * error path. We still know that another entity in our call
|
| 1452 | * path holds a reference to the program, thus atomic_sub() can
|
| 1453 | * be safely used in such cases!
|
| 1454 |
|
| 1455 | WARN_ON(atomic_sub_return(i, &prog->aux->refcnt) == 0);
|
| 1456 | }
|
| 1457 | EXPORT_SYMBOL_GPL(bpf_prog_sub);
|
| 1458 |
|
| 1459 | struct bpf_prog *bpf_prog_inc(struct bpf_prog *prog)
|
| 1460 | {
|
| 1461 | return bpf_prog_add(prog, 1);
|
| 1462 | }
|
| 1463 | EXPORT_SYMBOL_GPL(bpf_prog_inc);
|
| 1464 |
|
| 1465 |
|
| 1466 | struct bpf_prog *bpf_prog_inc_not_zero(struct bpf_prog *prog)
|
| 1467 | {
|
| 1468 | int refold;
|
| 1469 |
|
| 1470 | refold = atomic_fetch_add_unless(&prog->aux->refcnt, 1, 0);
|
| 1471 |
|
| 1472 | if (refold >= BPF_MAX_REFCNT) {
|
| 1473 | __bpf_prog_put(prog, false);
|
| 1474 | return ERR_PTR(-EBUSY);
|
| 1475 | }
|
| 1476 |
|
| 1477 | if (!refold)
|
| 1478 | return ERR_PTR(-ENOENT);
|
| 1479 |
|
| 1480 | return prog;
|
| 1481 | }
|
| 1482 | EXPORT_SYMBOL_GPL(bpf_prog_inc_not_zero);
|
| 1483 |
|
| 1484 | bool bpf_prog_get_ok(struct bpf_prog *prog,
|
| 1485 | enum bpf_prog_type *attach_type, bool attach_drv)
|
| 1486 | {
|
| 1487 |
|
| 1488 | if (!attach_type)
|
| 1489 | return true;
|
| 1490 |
|
| 1491 | if (prog->type != *attach_type)
|
| 1492 | return false;
|
| 1493 | if (bpf_prog_is_dev_bound(prog->aux) && !attach_drv)
|
| 1494 | return false;
|
| 1495 |
|
| 1496 | return true;
|
| 1497 | }
|
| 1498 |
|
| 1499 | static struct bpf_prog *__bpf_prog_get(u32 ufd, enum bpf_prog_type *attach_type,
|
| 1500 | bool attach_drv)
|
| 1501 | {
|
| 1502 | struct fd f = fdget(ufd);
|
| 1503 | struct bpf_prog *prog;
|
| 1504 |
|
| 1505 | prog = ____bpf_prog_get(f);
|
| 1506 | if (IS_ERR(prog))
|
| 1507 | return prog;
|
| 1508 | if (!bpf_prog_get_ok(prog, attach_type, attach_drv)) {
|
| 1509 | prog = ERR_PTR(-EINVAL);
|
| 1510 | goto out;
|
| 1511 | }
|
| 1512 |
|
| 1513 | prog = bpf_prog_inc(prog);
|
| 1514 | out:
|
| 1515 | fdput(f);
|
| 1516 | return prog;
|
| 1517 | }
|
| 1518 |
|
| 1519 | struct bpf_prog *bpf_prog_get(u32 ufd)
|
| 1520 | {
|
| 1521 | return __bpf_prog_get(ufd, NULL, false);
|
| 1522 | }
|
| 1523 |
|
| 1524 | struct bpf_prog *bpf_prog_get_type_dev(u32 ufd, enum bpf_prog_type type,
|
| 1525 | bool attach_drv)
|
| 1526 | {
|
| 1527 | return __bpf_prog_get(ufd, &type, attach_drv);
|
| 1528 | }
|
| 1529 | EXPORT_SYMBOL_GPL(bpf_prog_get_type_dev);
|
| 1530 |
|
| 1531 | |
| 1532 | * expected_attach_type. Later for some of them specifying expected_attach_type
|
| 1533 | * at load time became required so that program could be validated properly.
|
| 1534 | * Programs of types that are allowed to be loaded both w/ and w/o (for
|
| 1535 | * backward compatibility) expected_attach_type, should have the default attach
|
| 1536 | * type assigned to expected_attach_type for the latter case, so that it can be
|
| 1537 | * validated later at attach time.
|
| 1538 | *
|
| 1539 | * bpf_prog_load_fixup_attach_type() sets expected_attach_type in @attr if
|
| 1540 | * prog type requires it but has some attach types that have to be backward
|
| 1541 | * compatible.
|
| 1542 |
|
| 1543 | static void bpf_prog_load_fixup_attach_type(union bpf_attr *attr)
|
| 1544 | {
|
| 1545 | switch (attr->prog_type) {
|
| 1546 | case BPF_PROG_TYPE_CGROUP_SOCK:
|
| 1547 | |
| 1548 | * exist so checking for non-zero is the way to go here.
|
| 1549 |
|
| 1550 | if (!attr->expected_attach_type)
|
| 1551 | attr->expected_attach_type =
|
| 1552 | BPF_CGROUP_INET_SOCK_CREATE;
|
| 1553 | break;
|
| 1554 | }
|
| 1555 | }
|
| 1556 |
|
| 1557 | static int
|
| 1558 | bpf_prog_load_check_attach_type(enum bpf_prog_type prog_type,
|
| 1559 | enum bpf_attach_type expected_attach_type)
|
| 1560 | {
|
| 1561 | switch (prog_type) {
|
| 1562 | case BPF_PROG_TYPE_CGROUP_SOCK:
|
| 1563 | switch (expected_attach_type) {
|
| 1564 | case BPF_CGROUP_INET_SOCK_CREATE:
|
| 1565 | case BPF_CGROUP_INET4_POST_BIND:
|
| 1566 | case BPF_CGROUP_INET6_POST_BIND:
|
| 1567 | return 0;
|
| 1568 | default:
|
| 1569 | return -EINVAL;
|
| 1570 | }
|
| 1571 | case BPF_PROG_TYPE_CGROUP_SOCK_ADDR:
|
| 1572 | switch (expected_attach_type) {
|
| 1573 | case BPF_CGROUP_INET4_BIND:
|
| 1574 | case BPF_CGROUP_INET6_BIND:
|
| 1575 | case BPF_CGROUP_INET4_CONNECT:
|
| 1576 | case BPF_CGROUP_INET6_CONNECT:
|
| 1577 | case BPF_CGROUP_UDP4_SENDMSG:
|
| 1578 | case BPF_CGROUP_UDP6_SENDMSG:
|
| 1579 | case BPF_CGROUP_UDP4_RECVMSG:
|
| 1580 | case BPF_CGROUP_UDP6_RECVMSG:
|
| 1581 | return 0;
|
| 1582 | default:
|
| 1583 | return -EINVAL;
|
| 1584 | }
|
| 1585 | case BPF_PROG_TYPE_CGROUP_SKB:
|
| 1586 | switch (expected_attach_type) {
|
| 1587 | case BPF_CGROUP_INET_INGRESS:
|
| 1588 | case BPF_CGROUP_INET_EGRESS:
|
| 1589 | return 0;
|
| 1590 | default:
|
| 1591 | return -EINVAL;
|
| 1592 | }
|
| 1593 | case BPF_PROG_TYPE_CGROUP_SOCKOPT:
|
| 1594 | switch (expected_attach_type) {
|
| 1595 | case BPF_CGROUP_SETSOCKOPT:
|
| 1596 | case BPF_CGROUP_GETSOCKOPT:
|
| 1597 | return 0;
|
| 1598 | default:
|
| 1599 | return -EINVAL;
|
| 1600 | }
|
| 1601 | default:
|
| 1602 | return 0;
|
| 1603 | }
|
| 1604 | }
|
| 1605 |
|
| 1606 |
|
| 1607 | #define BPF_PROG_LOAD_LAST_FIELD line_info_cnt
|
| 1608 |
|
| 1609 | static int bpf_prog_load(union bpf_attr *attr, union bpf_attr __user *uattr)
|
| 1610 | {
|
| 1611 | enum bpf_prog_type type = attr->prog_type;
|
| 1612 | struct bpf_prog *prog;
|
| 1613 | int err;
|
| 1614 | char license[128];
|
| 1615 | bool is_gpl;
|
| 1616 |
|
| 1617 | if (CHECK_ATTR(BPF_PROG_LOAD))
|
| 1618 | return -EINVAL;
|
| 1619 |
|
| 1620 | if (attr->prog_flags & ~(BPF_F_STRICT_ALIGNMENT |
|
| 1621 | BPF_F_ANY_ALIGNMENT |
|
| 1622 | BPF_F_TEST_RND_HI32))
|
| 1623 | return -EINVAL;
|
| 1624 |
|
| 1625 | if (!IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS) &&
|
| 1626 | (attr->prog_flags & BPF_F_ANY_ALIGNMENT) &&
|
| 1627 | !capable(CAP_SYS_ADMIN))
|
| 1628 | return -EPERM;
|
| 1629 |
|
| 1630 |
|
| 1631 | if (strncpy_from_user(license, u64_to_user_ptr(attr->license),
|
| 1632 | sizeof(license) - 1) < 0)
|
| 1633 | return -EFAULT;
|
| 1634 | license[sizeof(license) - 1] = 0;
|
| 1635 |
|
| 1636 |
|
| 1637 | is_gpl = license_is_gpl_compatible(license);
|
| 1638 |
|
| 1639 | if (attr->insn_cnt == 0 ||
|
| 1640 | attr->insn_cnt > (capable(CAP_SYS_ADMIN) ? BPF_COMPLEXITY_LIMIT_INSNS : BPF_MAXINSNS))
|
| 1641 | return -E2BIG;
|
| 1642 | if (type != BPF_PROG_TYPE_SOCKET_FILTER &&
|
| 1643 | type != BPF_PROG_TYPE_CGROUP_SKB &&
|
| 1644 | !capable(CAP_SYS_ADMIN))
|
| 1645 | return -EPERM;
|
| 1646 |
|
| 1647 | bpf_prog_load_fixup_attach_type(attr);
|
| 1648 | if (bpf_prog_load_check_attach_type(type, attr->expected_attach_type))
|
| 1649 | return -EINVAL;
|
| 1650 |
|
| 1651 |
|
| 1652 | prog = bpf_prog_alloc(bpf_prog_size(attr->insn_cnt), GFP_USER);
|
| 1653 | if (!prog)
|
| 1654 | return -ENOMEM;
|
| 1655 |
|
| 1656 | prog->expected_attach_type = attr->expected_attach_type;
|
| 1657 |
|
| 1658 | prog->aux->offload_requested = !!attr->prog_ifindex;
|
| 1659 |
|
| 1660 | err = security_bpf_prog_alloc(prog->aux);
|
| 1661 | if (err)
|
| 1662 | goto free_prog_nouncharge;
|
| 1663 |
|
| 1664 | err = bpf_prog_charge_memlock(prog);
|
| 1665 | if (err)
|
| 1666 | goto free_prog_sec;
|
| 1667 |
|
| 1668 | prog->len = attr->insn_cnt;
|
| 1669 |
|
| 1670 | err = -EFAULT;
|
| 1671 | if (copy_from_user(prog->insns, u64_to_user_ptr(attr->insns),
|
| 1672 | bpf_prog_insn_size(prog)) != 0)
|
| 1673 | goto free_prog;
|
| 1674 |
|
| 1675 | prog->orig_prog = NULL;
|
| 1676 | prog->jited = 0;
|
| 1677 |
|
| 1678 | atomic_set(&prog->aux->refcnt, 1);
|
| 1679 | prog->gpl_compatible = is_gpl ? 1 : 0;
|
| 1680 |
|
| 1681 | if (bpf_prog_is_dev_bound(prog->aux)) {
|
| 1682 | err = bpf_prog_offload_init(prog, attr);
|
| 1683 | if (err)
|
| 1684 | goto free_prog;
|
| 1685 | }
|
| 1686 |
|
| 1687 |
|
| 1688 | err = find_prog_type(type, prog);
|
| 1689 | if (err < 0)
|
| 1690 | goto free_prog;
|
| 1691 |
|
| 1692 | prog->aux->load_time = ktime_get_boottime_ns();
|
| 1693 | err = bpf_obj_name_cpy(prog->aux->name, attr->prog_name);
|
| 1694 | if (err)
|
| 1695 | goto free_prog;
|
| 1696 |
|
| 1697 |
|
| 1698 | err = bpf_check(&prog, attr, uattr);
|
| 1699 | if (err < 0)
|
| 1700 | goto free_used_maps;
|
| 1701 |
|
| 1702 | prog = bpf_prog_select_runtime(prog, &err);
|
| 1703 | if (err < 0)
|
| 1704 | goto free_used_maps;
|
| 1705 |
|
| 1706 | err = bpf_prog_alloc_id(prog);
|
| 1707 | if (err)
|
| 1708 | goto free_used_maps;
|
| 1709 |
|
| 1710 | |
| 1711 | * effectively publicly exposed. However, retrieving via
|
| 1712 | * bpf_prog_get_fd_by_id() will take another reference,
|
| 1713 | * therefore it cannot be gone underneath us.
|
| 1714 | *
|
| 1715 | * Only for the time /after/ successful bpf_prog_new_fd()
|
| 1716 | * and before returning to userspace, we might just hold
|
| 1717 | * one reference and any parallel close on that fd could
|
| 1718 | * rip everything out. Hence, below notifications must
|
| 1719 | * happen before bpf_prog_new_fd().
|
| 1720 | *
|
| 1721 | * Also, any failure handling from this point onwards must
|
| 1722 | * be using bpf_prog_put() given the program is exposed.
|
| 1723 |
|
| 1724 | bpf_prog_kallsyms_add(prog);
|
| 1725 | perf_event_bpf_event(prog, PERF_BPF_EVENT_PROG_LOAD, 0);
|
| 1726 |
|
| 1727 | err = bpf_prog_new_fd(prog);
|
| 1728 | if (err < 0)
|
| 1729 | bpf_prog_put(prog);
|
| 1730 | return err;
|
| 1731 |
|
| 1732 | free_used_maps:
|
| 1733 | bpf_prog_free_linfo(prog);
|
| 1734 | kvfree(prog->aux->func_info);
|
| 1735 | btf_put(prog->aux->btf);
|
| 1736 | bpf_prog_kallsyms_del_subprogs(prog);
|
| 1737 | free_used_maps(prog->aux);
|
| 1738 | free_prog:
|
| 1739 | bpf_prog_uncharge_memlock(prog);
|
| 1740 | free_prog_sec:
|
| 1741 | security_bpf_prog_free(prog->aux);
|
| 1742 | free_prog_nouncharge:
|
| 1743 | bpf_prog_free(prog);
|
| 1744 | return err;
|
| 1745 | }
|
| 1746 |
|
| 1747 | #define BPF_OBJ_LAST_FIELD file_flags
|
| 1748 |
|
| 1749 | static int bpf_obj_pin(const union bpf_attr *attr)
|
| 1750 | {
|
| 1751 | if (CHECK_ATTR(BPF_OBJ) || attr->file_flags != 0)
|
| 1752 | return -EINVAL;
|
| 1753 |
|
| 1754 | return bpf_obj_pin_user(attr->bpf_fd, u64_to_user_ptr(attr->pathname));
|
| 1755 | }
|
| 1756 |
|
| 1757 | static int bpf_obj_get(const union bpf_attr *attr)
|
| 1758 | {
|
| 1759 | if (CHECK_ATTR(BPF_OBJ) || attr->bpf_fd != 0 ||
|
| 1760 | attr->file_flags & ~BPF_OBJ_FLAG_MASK)
|
| 1761 | return -EINVAL;
|
| 1762 |
|
| 1763 | return bpf_obj_get_user(u64_to_user_ptr(attr->pathname),
|
| 1764 | attr->file_flags);
|
| 1765 | }
|
| 1766 |
|
| 1767 | struct bpf_raw_tracepoint {
|
| 1768 | struct bpf_raw_event_map *btp;
|
| 1769 | struct bpf_prog *prog;
|
| 1770 | };
|
| 1771 |
|
| 1772 | static int bpf_raw_tracepoint_release(struct inode *inode, struct file *filp)
|
| 1773 | {
|
| 1774 | struct bpf_raw_tracepoint *raw_tp = filp->private_data;
|
| 1775 |
|
| 1776 | if (raw_tp->prog) {
|
| 1777 | bpf_probe_unregister(raw_tp->btp, raw_tp->prog);
|
| 1778 | bpf_prog_put(raw_tp->prog);
|
| 1779 | }
|
| 1780 | bpf_put_raw_tracepoint(raw_tp->btp);
|
| 1781 | kfree(raw_tp);
|
| 1782 | return 0;
|
| 1783 | }
|
| 1784 |
|
| 1785 | static const struct file_operations bpf_raw_tp_fops = {
|
| 1786 | .release = bpf_raw_tracepoint_release,
|
| 1787 | .read = bpf_dummy_read,
|
| 1788 | .write = bpf_dummy_write,
|
| 1789 | };
|
| 1790 |
|
| 1791 | #define BPF_RAW_TRACEPOINT_OPEN_LAST_FIELD raw_tracepoint.prog_fd
|
| 1792 |
|
| 1793 | static int bpf_raw_tracepoint_open(const union bpf_attr *attr)
|
| 1794 | {
|
| 1795 | struct bpf_raw_tracepoint *raw_tp;
|
| 1796 | struct bpf_raw_event_map *btp;
|
| 1797 | struct bpf_prog *prog;
|
| 1798 | char tp_name[128];
|
| 1799 | int tp_fd, err;
|
| 1800 |
|
| 1801 | if (strncpy_from_user(tp_name, u64_to_user_ptr(attr->raw_tracepoint.name),
|
| 1802 | sizeof(tp_name) - 1) < 0)
|
| 1803 | return -EFAULT;
|
| 1804 | tp_name[sizeof(tp_name) - 1] = 0;
|
| 1805 |
|
| 1806 | btp = bpf_get_raw_tracepoint(tp_name);
|
| 1807 | if (!btp)
|
| 1808 | return -ENOENT;
|
| 1809 |
|
| 1810 | raw_tp = kzalloc(sizeof(*raw_tp), GFP_USER);
|
| 1811 | if (!raw_tp) {
|
| 1812 | err = -ENOMEM;
|
| 1813 | goto out_put_btp;
|
| 1814 | }
|
| 1815 | raw_tp->btp = btp;
|
| 1816 |
|
| 1817 | prog = bpf_prog_get(attr->raw_tracepoint.prog_fd);
|
| 1818 | if (IS_ERR(prog)) {
|
| 1819 | err = PTR_ERR(prog);
|
| 1820 | goto out_free_tp;
|
| 1821 | }
|
| 1822 | if (prog->type != BPF_PROG_TYPE_RAW_TRACEPOINT &&
|
| 1823 | prog->type != BPF_PROG_TYPE_RAW_TRACEPOINT_WRITABLE) {
|
| 1824 | err = -EINVAL;
|
| 1825 | goto out_put_prog;
|
| 1826 | }
|
| 1827 |
|
| 1828 | err = bpf_probe_register(raw_tp->btp, prog);
|
| 1829 | if (err)
|
| 1830 | goto out_put_prog;
|
| 1831 |
|
| 1832 | raw_tp->prog = prog;
|
| 1833 | tp_fd = anon_inode_getfd("bpf-raw-tracepoint", &bpf_raw_tp_fops, raw_tp,
|
| 1834 | O_CLOEXEC);
|
| 1835 | if (tp_fd < 0) {
|
| 1836 | bpf_probe_unregister(raw_tp->btp, prog);
|
| 1837 | err = tp_fd;
|
| 1838 | goto out_put_prog;
|
| 1839 | }
|
| 1840 | return tp_fd;
|
| 1841 |
|
| 1842 | out_put_prog:
|
| 1843 | bpf_prog_put(prog);
|
| 1844 | out_free_tp:
|
| 1845 | kfree(raw_tp);
|
| 1846 | out_put_btp:
|
| 1847 | bpf_put_raw_tracepoint(btp);
|
| 1848 | return err;
|
| 1849 | }
|
| 1850 |
|
| 1851 | static int bpf_prog_attach_check_attach_type(const struct bpf_prog *prog,
|
| 1852 | enum bpf_attach_type attach_type)
|
| 1853 | {
|
| 1854 | switch (prog->type) {
|
| 1855 | case BPF_PROG_TYPE_CGROUP_SOCK:
|
| 1856 | case BPF_PROG_TYPE_CGROUP_SOCK_ADDR:
|
| 1857 | case BPF_PROG_TYPE_CGROUP_SOCKOPT:
|
| 1858 | return attach_type == prog->expected_attach_type ? 0 : -EINVAL;
|
| 1859 | case BPF_PROG_TYPE_CGROUP_SKB:
|
| 1860 | return prog->enforce_expected_attach_type &&
|
| 1861 | prog->expected_attach_type != attach_type ?
|
| 1862 | -EINVAL : 0;
|
| 1863 | default:
|
| 1864 | return 0;
|
| 1865 | }
|
| 1866 | }
|
| 1867 |
|
| 1868 | #define BPF_PROG_ATTACH_LAST_FIELD attach_flags
|
| 1869 |
|
| 1870 | #define BPF_F_ATTACH_MASK \
|
| 1871 | (BPF_F_ALLOW_OVERRIDE | BPF_F_ALLOW_MULTI)
|
| 1872 |
|
| 1873 | static int bpf_prog_attach(const union bpf_attr *attr)
|
| 1874 | {
|
| 1875 | enum bpf_prog_type ptype;
|
| 1876 | struct bpf_prog *prog;
|
| 1877 | int ret;
|
| 1878 |
|
| 1879 | if (!capable(CAP_NET_ADMIN))
|
| 1880 | return -EPERM;
|
| 1881 |
|
| 1882 | if (CHECK_ATTR(BPF_PROG_ATTACH))
|
| 1883 | return -EINVAL;
|
| 1884 |
|
| 1885 | if (attr->attach_flags & ~BPF_F_ATTACH_MASK)
|
| 1886 | return -EINVAL;
|
| 1887 |
|
| 1888 | switch (attr->attach_type) {
|
| 1889 | case BPF_CGROUP_INET_INGRESS:
|
| 1890 | case BPF_CGROUP_INET_EGRESS:
|
| 1891 | ptype = BPF_PROG_TYPE_CGROUP_SKB;
|
| 1892 | break;
|
| 1893 | case BPF_CGROUP_INET_SOCK_CREATE:
|
| 1894 | case BPF_CGROUP_INET4_POST_BIND:
|
| 1895 | case BPF_CGROUP_INET6_POST_BIND:
|
| 1896 | ptype = BPF_PROG_TYPE_CGROUP_SOCK;
|
| 1897 | break;
|
| 1898 | case BPF_CGROUP_INET4_BIND:
|
| 1899 | case BPF_CGROUP_INET6_BIND:
|
| 1900 | case BPF_CGROUP_INET4_CONNECT:
|
| 1901 | case BPF_CGROUP_INET6_CONNECT:
|
| 1902 | case BPF_CGROUP_UDP4_SENDMSG:
|
| 1903 | case BPF_CGROUP_UDP6_SENDMSG:
|
| 1904 | case BPF_CGROUP_UDP4_RECVMSG:
|
| 1905 | case BPF_CGROUP_UDP6_RECVMSG:
|
| 1906 | ptype = BPF_PROG_TYPE_CGROUP_SOCK_ADDR;
|
| 1907 | break;
|
| 1908 | case BPF_CGROUP_SOCK_OPS:
|
| 1909 | ptype = BPF_PROG_TYPE_SOCK_OPS;
|
| 1910 | break;
|
| 1911 | case BPF_CGROUP_DEVICE:
|
| 1912 | ptype = BPF_PROG_TYPE_CGROUP_DEVICE;
|
| 1913 | break;
|
| 1914 | case BPF_SK_MSG_VERDICT:
|
| 1915 | ptype = BPF_PROG_TYPE_SK_MSG;
|
| 1916 | break;
|
| 1917 | case BPF_SK_SKB_STREAM_PARSER:
|
| 1918 | case BPF_SK_SKB_STREAM_VERDICT:
|
| 1919 | ptype = BPF_PROG_TYPE_SK_SKB;
|
| 1920 | break;
|
| 1921 | case BPF_LIRC_MODE2:
|
| 1922 | ptype = BPF_PROG_TYPE_LIRC_MODE2;
|
| 1923 | break;
|
| 1924 | case BPF_FLOW_DISSECTOR:
|
| 1925 | ptype = BPF_PROG_TYPE_FLOW_DISSECTOR;
|
| 1926 | break;
|
| 1927 | case BPF_CGROUP_SYSCTL:
|
| 1928 | ptype = BPF_PROG_TYPE_CGROUP_SYSCTL;
|
| 1929 | break;
|
| 1930 | case BPF_CGROUP_GETSOCKOPT:
|
| 1931 | case BPF_CGROUP_SETSOCKOPT:
|
| 1932 | ptype = BPF_PROG_TYPE_CGROUP_SOCKOPT;
|
| 1933 | break;
|
| 1934 | default:
|
| 1935 | return -EINVAL;
|
| 1936 | }
|
| 1937 |
|
| 1938 | prog = bpf_prog_get_type(attr->attach_bpf_fd, ptype);
|
| 1939 | if (IS_ERR(prog))
|
| 1940 | return PTR_ERR(prog);
|
| 1941 |
|
| 1942 | if (bpf_prog_attach_check_attach_type(prog, attr->attach_type)) {
|
| 1943 | bpf_prog_put(prog);
|
| 1944 | return -EINVAL;
|
| 1945 | }
|
| 1946 |
|
| 1947 | switch (ptype) {
|
| 1948 | case BPF_PROG_TYPE_SK_SKB:
|
| 1949 | case BPF_PROG_TYPE_SK_MSG:
|
| 1950 | ret = sock_map_get_from_fd(attr, prog);
|
| 1951 | break;
|
| 1952 | case BPF_PROG_TYPE_LIRC_MODE2:
|
| 1953 | ret = lirc_prog_attach(attr, prog);
|
| 1954 | break;
|
| 1955 | case BPF_PROG_TYPE_FLOW_DISSECTOR:
|
| 1956 | ret = skb_flow_dissector_bpf_prog_attach(attr, prog);
|
| 1957 | break;
|
| 1958 | default:
|
| 1959 | ret = cgroup_bpf_prog_attach(attr, ptype, prog);
|
| 1960 | }
|
| 1961 |
|
| 1962 | if (ret)
|
| 1963 | bpf_prog_put(prog);
|
| 1964 | return ret;
|
| 1965 | }
|
| 1966 |
|
| 1967 | #define BPF_PROG_DETACH_LAST_FIELD attach_type
|
| 1968 |
|
| 1969 | static int bpf_prog_detach(const union bpf_attr *attr)
|
| 1970 | {
|
| 1971 | enum bpf_prog_type ptype;
|
| 1972 |
|
| 1973 | if (!capable(CAP_NET_ADMIN))
|
| 1974 | return -EPERM;
|
| 1975 |
|
| 1976 | if (CHECK_ATTR(BPF_PROG_DETACH))
|
| 1977 | return -EINVAL;
|
| 1978 |
|
| 1979 | switch (attr->attach_type) {
|
| 1980 | case BPF_CGROUP_INET_INGRESS:
|
| 1981 | case BPF_CGROUP_INET_EGRESS:
|
| 1982 | ptype = BPF_PROG_TYPE_CGROUP_SKB;
|
| 1983 | break;
|
| 1984 | case BPF_CGROUP_INET_SOCK_CREATE:
|
| 1985 | case BPF_CGROUP_INET4_POST_BIND:
|
| 1986 | case BPF_CGROUP_INET6_POST_BIND:
|
| 1987 | ptype = BPF_PROG_TYPE_CGROUP_SOCK;
|
| 1988 | break;
|
| 1989 | case BPF_CGROUP_INET4_BIND:
|
| 1990 | case BPF_CGROUP_INET6_BIND:
|
| 1991 | case BPF_CGROUP_INET4_CONNECT:
|
| 1992 | case BPF_CGROUP_INET6_CONNECT:
|
| 1993 | case BPF_CGROUP_UDP4_SENDMSG:
|
| 1994 | case BPF_CGROUP_UDP6_SENDMSG:
|
| 1995 | case BPF_CGROUP_UDP4_RECVMSG:
|
| 1996 | case BPF_CGROUP_UDP6_RECVMSG:
|
| 1997 | ptype = BPF_PROG_TYPE_CGROUP_SOCK_ADDR;
|
| 1998 | break;
|
| 1999 | case BPF_CGROUP_SOCK_OPS:
|
| 2000 | ptype = BPF_PROG_TYPE_SOCK_OPS;
|
| 2001 | break;
|
| 2002 | case BPF_CGROUP_DEVICE:
|
| 2003 | ptype = BPF_PROG_TYPE_CGROUP_DEVICE;
|
| 2004 | break;
|
| 2005 | case BPF_SK_MSG_VERDICT:
|
| 2006 | return sock_map_get_from_fd(attr, NULL);
|
| 2007 | case BPF_SK_SKB_STREAM_PARSER:
|
| 2008 | case BPF_SK_SKB_STREAM_VERDICT:
|
| 2009 | return sock_map_get_from_fd(attr, NULL);
|
| 2010 | case BPF_LIRC_MODE2:
|
| 2011 | return lirc_prog_detach(attr);
|
| 2012 | case BPF_FLOW_DISSECTOR:
|
| 2013 | return skb_flow_dissector_bpf_prog_detach(attr);
|
| 2014 | case BPF_CGROUP_SYSCTL:
|
| 2015 | ptype = BPF_PROG_TYPE_CGROUP_SYSCTL;
|
| 2016 | break;
|
| 2017 | case BPF_CGROUP_GETSOCKOPT:
|
| 2018 | case BPF_CGROUP_SETSOCKOPT:
|
| 2019 | ptype = BPF_PROG_TYPE_CGROUP_SOCKOPT;
|
| 2020 | break;
|
| 2021 | default:
|
| 2022 | return -EINVAL;
|
| 2023 | }
|
| 2024 |
|
| 2025 | return cgroup_bpf_prog_detach(attr, ptype);
|
| 2026 | }
|
| 2027 |
|
| 2028 | #define BPF_PROG_QUERY_LAST_FIELD query.prog_cnt
|
| 2029 |
|
| 2030 | static int bpf_prog_query(const union bpf_attr *attr,
|
| 2031 | union bpf_attr __user *uattr)
|
| 2032 | {
|
| 2033 | if (!capable(CAP_NET_ADMIN))
|
| 2034 | return -EPERM;
|
| 2035 | if (CHECK_ATTR(BPF_PROG_QUERY))
|
| 2036 | return -EINVAL;
|
| 2037 | if (attr->query.query_flags & ~BPF_F_QUERY_EFFECTIVE)
|
| 2038 | return -EINVAL;
|
| 2039 |
|
| 2040 | switch (attr->query.attach_type) {
|
| 2041 | case BPF_CGROUP_INET_INGRESS:
|
| 2042 | case BPF_CGROUP_INET_EGRESS:
|
| 2043 | case BPF_CGROUP_INET_SOCK_CREATE:
|
| 2044 | case BPF_CGROUP_INET4_BIND:
|
| 2045 | case BPF_CGROUP_INET6_BIND:
|
| 2046 | case BPF_CGROUP_INET4_POST_BIND:
|
| 2047 | case BPF_CGROUP_INET6_POST_BIND:
|
| 2048 | case BPF_CGROUP_INET4_CONNECT:
|
| 2049 | case BPF_CGROUP_INET6_CONNECT:
|
| 2050 | case BPF_CGROUP_UDP4_SENDMSG:
|
| 2051 | case BPF_CGROUP_UDP6_SENDMSG:
|
| 2052 | case BPF_CGROUP_UDP4_RECVMSG:
|
| 2053 | case BPF_CGROUP_UDP6_RECVMSG:
|
| 2054 | case BPF_CGROUP_SOCK_OPS:
|
| 2055 | case BPF_CGROUP_DEVICE:
|
| 2056 | case BPF_CGROUP_SYSCTL:
|
| 2057 | case BPF_CGROUP_GETSOCKOPT:
|
| 2058 | case BPF_CGROUP_SETSOCKOPT:
|
| 2059 | break;
|
| 2060 | case BPF_LIRC_MODE2:
|
| 2061 | return lirc_prog_query(attr, uattr);
|
| 2062 | case BPF_FLOW_DISSECTOR:
|
| 2063 | return skb_flow_dissector_prog_query(attr, uattr);
|
| 2064 | default:
|
| 2065 | return -EINVAL;
|
| 2066 | }
|
| 2067 |
|
| 2068 | return cgroup_bpf_prog_query(attr, uattr);
|
| 2069 | }
|
| 2070 |
|
| 2071 | #define BPF_PROG_TEST_RUN_LAST_FIELD test.ctx_out
|
| 2072 |
|
| 2073 | static int bpf_prog_test_run(const union bpf_attr *attr,
|
| 2074 | union bpf_attr __user *uattr)
|
| 2075 | {
|
| 2076 | struct bpf_prog *prog;
|
| 2077 | int ret = -ENOTSUPP;
|
| 2078 |
|
| 2079 | if (!capable(CAP_SYS_ADMIN))
|
| 2080 | return -EPERM;
|
| 2081 | if (CHECK_ATTR(BPF_PROG_TEST_RUN))
|
| 2082 | return -EINVAL;
|
| 2083 |
|
| 2084 | if ((attr->test.ctx_size_in && !attr->test.ctx_in) ||
|
| 2085 | (!attr->test.ctx_size_in && attr->test.ctx_in))
|
| 2086 | return -EINVAL;
|
| 2087 |
|
| 2088 | if ((attr->test.ctx_size_out && !attr->test.ctx_out) ||
|
| 2089 | (!attr->test.ctx_size_out && attr->test.ctx_out))
|
| 2090 | return -EINVAL;
|
| 2091 |
|
| 2092 | prog = bpf_prog_get(attr->test.prog_fd);
|
| 2093 | if (IS_ERR(prog))
|
| 2094 | return PTR_ERR(prog);
|
| 2095 |
|
| 2096 | if (prog->aux->ops->test_run)
|
| 2097 | ret = prog->aux->ops->test_run(prog, attr, uattr);
|
| 2098 |
|
| 2099 | bpf_prog_put(prog);
|
| 2100 | return ret;
|
| 2101 | }
|
| 2102 |
|
| 2103 | #define BPF_OBJ_GET_NEXT_ID_LAST_FIELD next_id
|
| 2104 |
|
| 2105 | static int bpf_obj_get_next_id(const union bpf_attr *attr,
|
| 2106 | union bpf_attr __user *uattr,
|
| 2107 | struct idr *idr,
|
| 2108 | spinlock_t *lock)
|
| 2109 | {
|
| 2110 | u32 next_id = attr->start_id;
|
| 2111 | int err = 0;
|
| 2112 |
|
| 2113 | if (CHECK_ATTR(BPF_OBJ_GET_NEXT_ID) || next_id >= INT_MAX)
|
| 2114 | return -EINVAL;
|
| 2115 |
|
| 2116 | if (!capable(CAP_SYS_ADMIN))
|
| 2117 | return -EPERM;
|
| 2118 |
|
| 2119 | next_id++;
|
| 2120 | spin_lock_bh(lock);
|
| 2121 | if (!idr_get_next(idr, &next_id))
|
| 2122 | err = -ENOENT;
|
| 2123 | spin_unlock_bh(lock);
|
| 2124 |
|
| 2125 | if (!err)
|
| 2126 | err = put_user(next_id, &uattr->next_id);
|
| 2127 |
|
| 2128 | return err;
|
| 2129 | }
|
| 2130 |
|
| 2131 | #define BPF_PROG_GET_FD_BY_ID_LAST_FIELD prog_id
|
| 2132 |
|
| 2133 | static int bpf_prog_get_fd_by_id(const union bpf_attr *attr)
|
| 2134 | {
|
| 2135 | struct bpf_prog *prog;
|
| 2136 | u32 id = attr->prog_id;
|
| 2137 | int fd;
|
| 2138 |
|
| 2139 | if (CHECK_ATTR(BPF_PROG_GET_FD_BY_ID))
|
| 2140 | return -EINVAL;
|
| 2141 |
|
| 2142 | if (!capable(CAP_SYS_ADMIN))
|
| 2143 | return -EPERM;
|
| 2144 |
|
| 2145 | spin_lock_bh(&prog_idr_lock);
|
| 2146 | prog = idr_find(&prog_idr, id);
|
| 2147 | if (prog)
|
| 2148 | prog = bpf_prog_inc_not_zero(prog);
|
| 2149 | else
|
| 2150 | prog = ERR_PTR(-ENOENT);
|
| 2151 | spin_unlock_bh(&prog_idr_lock);
|
| 2152 |
|
| 2153 | if (IS_ERR(prog))
|
| 2154 | return PTR_ERR(prog);
|
| 2155 |
|
| 2156 | fd = bpf_prog_new_fd(prog);
|
| 2157 | if (fd < 0)
|
| 2158 | bpf_prog_put(prog);
|
| 2159 |
|
| 2160 | return fd;
|
| 2161 | }
|
| 2162 |
|
| 2163 | #define BPF_MAP_GET_FD_BY_ID_LAST_FIELD open_flags
|
| 2164 |
|
| 2165 | static int bpf_map_get_fd_by_id(const union bpf_attr *attr)
|
| 2166 | {
|
| 2167 | struct bpf_map *map;
|
| 2168 | u32 id = attr->map_id;
|
| 2169 | int f_flags;
|
| 2170 | int fd;
|
| 2171 |
|
| 2172 | if (CHECK_ATTR(BPF_MAP_GET_FD_BY_ID) ||
|
| 2173 | attr->open_flags & ~BPF_OBJ_FLAG_MASK)
|
| 2174 | return -EINVAL;
|
| 2175 |
|
| 2176 | if (!capable(CAP_SYS_ADMIN))
|
| 2177 | return -EPERM;
|
| 2178 |
|
| 2179 | f_flags = bpf_get_file_flag(attr->open_flags);
|
| 2180 | if (f_flags < 0)
|
| 2181 | return f_flags;
|
| 2182 |
|
| 2183 | spin_lock_bh(&map_idr_lock);
|
| 2184 | map = idr_find(&map_idr, id);
|
| 2185 | if (map)
|
| 2186 | map = bpf_map_inc_not_zero(map, true);
|
| 2187 | else
|
| 2188 | map = ERR_PTR(-ENOENT);
|
| 2189 | spin_unlock_bh(&map_idr_lock);
|
| 2190 |
|
| 2191 | if (IS_ERR(map))
|
| 2192 | return PTR_ERR(map);
|
| 2193 |
|
| 2194 | fd = bpf_map_new_fd(map, f_flags);
|
| 2195 | if (fd < 0)
|
| 2196 | bpf_map_put_with_uref(map);
|
| 2197 |
|
| 2198 | return fd;
|
| 2199 | }
|
| 2200 |
|
| 2201 | static const struct bpf_map *bpf_map_from_imm(const struct bpf_prog *prog,
|
| 2202 | unsigned long addr, u32 *off,
|
| 2203 | u32 *type)
|
| 2204 | {
|
| 2205 | const struct bpf_map *map;
|
| 2206 | int i;
|
| 2207 |
|
| 2208 | for (i = 0, *off = 0; i < prog->aux->used_map_cnt; i++) {
|
| 2209 | map = prog->aux->used_maps[i];
|
| 2210 | if (map == (void *)addr) {
|
| 2211 | *type = BPF_PSEUDO_MAP_FD;
|
| 2212 | return map;
|
| 2213 | }
|
| 2214 | if (!map->ops->map_direct_value_meta)
|
| 2215 | continue;
|
| 2216 | if (!map->ops->map_direct_value_meta(map, addr, off)) {
|
| 2217 | *type = BPF_PSEUDO_MAP_VALUE;
|
| 2218 | return map;
|
| 2219 | }
|
| 2220 | }
|
| 2221 |
|
| 2222 | return NULL;
|
| 2223 | }
|
| 2224 |
|
| 2225 | static struct bpf_insn *bpf_insn_prepare_dump(const struct bpf_prog *prog)
|
| 2226 | {
|
| 2227 | const struct bpf_map *map;
|
| 2228 | struct bpf_insn *insns;
|
| 2229 | u32 off, type;
|
| 2230 | u64 imm;
|
| 2231 | int i;
|
| 2232 |
|
| 2233 | insns = kmemdup(prog->insnsi, bpf_prog_insn_size(prog),
|
| 2234 | GFP_USER);
|
| 2235 | if (!insns)
|
| 2236 | return insns;
|
| 2237 |
|
| 2238 | for (i = 0; i < prog->len; i++) {
|
| 2239 | if (insns[i].code == (BPF_JMP | BPF_TAIL_CALL)) {
|
| 2240 | insns[i].code = BPF_JMP | BPF_CALL;
|
| 2241 | insns[i].imm = BPF_FUNC_tail_call;
|
| 2242 |
|
| 2243 | }
|
| 2244 | if (insns[i].code == (BPF_JMP | BPF_CALL) ||
|
| 2245 | insns[i].code == (BPF_JMP | BPF_CALL_ARGS)) {
|
| 2246 | if (insns[i].code == (BPF_JMP | BPF_CALL_ARGS))
|
| 2247 | insns[i].code = BPF_JMP | BPF_CALL;
|
| 2248 | if (!bpf_dump_raw_ok())
|
| 2249 | insns[i].imm = 0;
|
| 2250 | continue;
|
| 2251 | }
|
| 2252 |
|
| 2253 | if (insns[i].code != (BPF_LD | BPF_IMM | BPF_DW))
|
| 2254 | continue;
|
| 2255 |
|
| 2256 | imm = ((u64)insns[i + 1].imm << 32) | (u32)insns[i].imm;
|
| 2257 | map = bpf_map_from_imm(prog, imm, &off, &type);
|
| 2258 | if (map) {
|
| 2259 | insns[i].src_reg = type;
|
| 2260 | insns[i].imm = map->id;
|
| 2261 | insns[i + 1].imm = off;
|
| 2262 | continue;
|
| 2263 | }
|
| 2264 | }
|
| 2265 |
|
| 2266 | return insns;
|
| 2267 | }
|
| 2268 |
|
| 2269 | static int set_info_rec_size(struct bpf_prog_info *info)
|
| 2270 | {
|
| 2271 | |
| 2272 | * Ensure info.*_rec_size is the same as kernel expected size
|
| 2273 | *
|
| 2274 | * or
|
| 2275 | *
|
| 2276 | * Only allow zero *_rec_size if both _rec_size and _cnt are
|
| 2277 | * zero. In this case, the kernel will set the expected
|
| 2278 | * _rec_size back to the info.
|
| 2279 |
|
| 2280 |
|
| 2281 | if ((info->nr_func_info || info->func_info_rec_size) &&
|
| 2282 | info->func_info_rec_size != sizeof(struct bpf_func_info))
|
| 2283 | return -EINVAL;
|
| 2284 |
|
| 2285 | if ((info->nr_line_info || info->line_info_rec_size) &&
|
| 2286 | info->line_info_rec_size != sizeof(struct bpf_line_info))
|
| 2287 | return -EINVAL;
|
| 2288 |
|
| 2289 | if ((info->nr_jited_line_info || info->jited_line_info_rec_size) &&
|
| 2290 | info->jited_line_info_rec_size != sizeof(__u64))
|
| 2291 | return -EINVAL;
|
| 2292 |
|
| 2293 | info->func_info_rec_size = sizeof(struct bpf_func_info);
|
| 2294 | info->line_info_rec_size = sizeof(struct bpf_line_info);
|
| 2295 | info->jited_line_info_rec_size = sizeof(__u64);
|
| 2296 |
|
| 2297 | return 0;
|
| 2298 | }
|
| 2299 |
|
| 2300 | static int bpf_prog_get_info_by_fd(struct bpf_prog *prog,
|
| 2301 | const union bpf_attr *attr,
|
| 2302 | union bpf_attr __user *uattr)
|
| 2303 | {
|
| 2304 | struct bpf_prog_info __user *uinfo = u64_to_user_ptr(attr->info.info);
|
| 2305 | struct bpf_prog_info info = {};
|
| 2306 | u32 info_len = attr->info.info_len;
|
| 2307 | struct bpf_prog_stats stats;
|
| 2308 | char __user *uinsns;
|
| 2309 | u32 ulen;
|
| 2310 | int err;
|
| 2311 |
|
| 2312 | err = bpf_check_uarg_tail_zero(uinfo, sizeof(info), info_len);
|
| 2313 | if (err)
|
| 2314 | return err;
|
| 2315 | info_len = min_t(u32, sizeof(info), info_len);
|
| 2316 |
|
| 2317 | if (copy_from_user(&info, uinfo, info_len))
|
| 2318 | return -EFAULT;
|
| 2319 |
|
| 2320 | info.type = prog->type;
|
| 2321 | info.id = prog->aux->id;
|
| 2322 | info.load_time = prog->aux->load_time;
|
| 2323 | info.created_by_uid = from_kuid_munged(current_user_ns(),
|
| 2324 | prog->aux->user->uid);
|
| 2325 | info.gpl_compatible = prog->gpl_compatible;
|
| 2326 |
|
| 2327 | memcpy(info.tag, prog->tag, sizeof(prog->tag));
|
| 2328 | memcpy(info.name, prog->aux->name, sizeof(prog->aux->name));
|
| 2329 |
|
| 2330 | ulen = info.nr_map_ids;
|
| 2331 | info.nr_map_ids = prog->aux->used_map_cnt;
|
| 2332 | ulen = min_t(u32, info.nr_map_ids, ulen);
|
| 2333 | if (ulen) {
|
| 2334 | u32 __user *user_map_ids = u64_to_user_ptr(info.map_ids);
|
| 2335 | u32 i;
|
| 2336 |
|
| 2337 | for (i = 0; i < ulen; i++)
|
| 2338 | if (put_user(prog->aux->used_maps[i]->id,
|
| 2339 | &user_map_ids[i]))
|
| 2340 | return -EFAULT;
|
| 2341 | }
|
| 2342 |
|
| 2343 | err = set_info_rec_size(&info);
|
| 2344 | if (err)
|
| 2345 | return err;
|
| 2346 |
|
| 2347 | bpf_prog_get_stats(prog, &stats);
|
| 2348 | info.run_time_ns = stats.nsecs;
|
| 2349 | info.run_cnt = stats.cnt;
|
| 2350 |
|
| 2351 | if (!capable(CAP_SYS_ADMIN)) {
|
| 2352 | info.jited_prog_len = 0;
|
| 2353 | info.xlated_prog_len = 0;
|
| 2354 | info.nr_jited_ksyms = 0;
|
| 2355 | info.nr_jited_func_lens = 0;
|
| 2356 | info.nr_func_info = 0;
|
| 2357 | info.nr_line_info = 0;
|
| 2358 | info.nr_jited_line_info = 0;
|
| 2359 | goto done;
|
| 2360 | }
|
| 2361 |
|
| 2362 | ulen = info.xlated_prog_len;
|
| 2363 | info.xlated_prog_len = bpf_prog_insn_size(prog);
|
| 2364 | if (info.xlated_prog_len && ulen) {
|
| 2365 | struct bpf_insn *insns_sanitized;
|
| 2366 | bool fault;
|
| 2367 |
|
| 2368 | if (prog->blinded && !bpf_dump_raw_ok()) {
|
| 2369 | info.xlated_prog_insns = 0;
|
| 2370 | goto done;
|
| 2371 | }
|
| 2372 | insns_sanitized = bpf_insn_prepare_dump(prog);
|
| 2373 | if (!insns_sanitized)
|
| 2374 | return -ENOMEM;
|
| 2375 | uinsns = u64_to_user_ptr(info.xlated_prog_insns);
|
| 2376 | ulen = min_t(u32, info.xlated_prog_len, ulen);
|
| 2377 | fault = copy_to_user(uinsns, insns_sanitized, ulen);
|
| 2378 | kfree(insns_sanitized);
|
| 2379 | if (fault)
|
| 2380 | return -EFAULT;
|
| 2381 | }
|
| 2382 |
|
| 2383 | if (bpf_prog_is_dev_bound(prog->aux)) {
|
| 2384 | err = bpf_prog_offload_info_fill(&info, prog);
|
| 2385 | if (err)
|
| 2386 | return err;
|
| 2387 | goto done;
|
| 2388 | }
|
| 2389 |
|
| 2390 | |
| 2391 | * bpf_prog_offload_info_fill() is the place to fill similar fields
|
| 2392 | * for offload.
|
| 2393 |
|
| 2394 | ulen = info.jited_prog_len;
|
| 2395 | if (prog->aux->func_cnt) {
|
| 2396 | u32 i;
|
| 2397 |
|
| 2398 | info.jited_prog_len = 0;
|
| 2399 | for (i = 0; i < prog->aux->func_cnt; i++)
|
| 2400 | info.jited_prog_len += prog->aux->func[i]->jited_len;
|
| 2401 | } else {
|
| 2402 | info.jited_prog_len = prog->jited_len;
|
| 2403 | }
|
| 2404 |
|
| 2405 | if (info.jited_prog_len && ulen) {
|
| 2406 | if (bpf_dump_raw_ok()) {
|
| 2407 | uinsns = u64_to_user_ptr(info.jited_prog_insns);
|
| 2408 | ulen = min_t(u32, info.jited_prog_len, ulen);
|
| 2409 |
|
| 2410 | |
| 2411 | * instructions for all the functions
|
| 2412 |
|
| 2413 | if (prog->aux->func_cnt) {
|
| 2414 | u32 len, free, i;
|
| 2415 | u8 *img;
|
| 2416 |
|
| 2417 | free = ulen;
|
| 2418 | for (i = 0; i < prog->aux->func_cnt; i++) {
|
| 2419 | len = prog->aux->func[i]->jited_len;
|
| 2420 | len = min_t(u32, len, free);
|
| 2421 | img = (u8 *) prog->aux->func[i]->bpf_func;
|
| 2422 | if (copy_to_user(uinsns, img, len))
|
| 2423 | return -EFAULT;
|
| 2424 | uinsns += len;
|
| 2425 | free -= len;
|
| 2426 | if (!free)
|
| 2427 | break;
|
| 2428 | }
|
| 2429 | } else {
|
| 2430 | if (copy_to_user(uinsns, prog->bpf_func, ulen))
|
| 2431 | return -EFAULT;
|
| 2432 | }
|
| 2433 | } else {
|
| 2434 | info.jited_prog_insns = 0;
|
| 2435 | }
|
| 2436 | }
|
| 2437 |
|
| 2438 | ulen = info.nr_jited_ksyms;
|
| 2439 | info.nr_jited_ksyms = prog->aux->func_cnt ? : 1;
|
| 2440 | if (ulen) {
|
| 2441 | if (bpf_dump_raw_ok()) {
|
| 2442 | unsigned long ksym_addr;
|
| 2443 | u64 __user *user_ksyms;
|
| 2444 | u32 i;
|
| 2445 |
|
| 2446 | |
| 2447 | * corresponding to each function
|
| 2448 |
|
| 2449 | ulen = min_t(u32, info.nr_jited_ksyms, ulen);
|
| 2450 | user_ksyms = u64_to_user_ptr(info.jited_ksyms);
|
| 2451 | if (prog->aux->func_cnt) {
|
| 2452 | for (i = 0; i < ulen; i++) {
|
| 2453 | ksym_addr = (unsigned long)
|
| 2454 | prog->aux->func[i]->bpf_func;
|
| 2455 | if (put_user((u64) ksym_addr,
|
| 2456 | &user_ksyms[i]))
|
| 2457 | return -EFAULT;
|
| 2458 | }
|
| 2459 | } else {
|
| 2460 | ksym_addr = (unsigned long) prog->bpf_func;
|
| 2461 | if (put_user((u64) ksym_addr, &user_ksyms[0]))
|
| 2462 | return -EFAULT;
|
| 2463 | }
|
| 2464 | } else {
|
| 2465 | info.jited_ksyms = 0;
|
| 2466 | }
|
| 2467 | }
|
| 2468 |
|
| 2469 | ulen = info.nr_jited_func_lens;
|
| 2470 | info.nr_jited_func_lens = prog->aux->func_cnt ? : 1;
|
| 2471 | if (ulen) {
|
| 2472 | if (bpf_dump_raw_ok()) {
|
| 2473 | u32 __user *user_lens;
|
| 2474 | u32 func_len, i;
|
| 2475 |
|
| 2476 |
|
| 2477 | ulen = min_t(u32, info.nr_jited_func_lens, ulen);
|
| 2478 | user_lens = u64_to_user_ptr(info.jited_func_lens);
|
| 2479 | if (prog->aux->func_cnt) {
|
| 2480 | for (i = 0; i < ulen; i++) {
|
| 2481 | func_len =
|
| 2482 | prog->aux->func[i]->jited_len;
|
| 2483 | if (put_user(func_len, &user_lens[i]))
|
| 2484 | return -EFAULT;
|
| 2485 | }
|
| 2486 | } else {
|
| 2487 | func_len = prog->jited_len;
|
| 2488 | if (put_user(func_len, &user_lens[0]))
|
| 2489 | return -EFAULT;
|
| 2490 | }
|
| 2491 | } else {
|
| 2492 | info.jited_func_lens = 0;
|
| 2493 | }
|
| 2494 | }
|
| 2495 |
|
| 2496 | if (prog->aux->btf)
|
| 2497 | info.btf_id = btf_id(prog->aux->btf);
|
| 2498 |
|
| 2499 | ulen = info.nr_func_info;
|
| 2500 | info.nr_func_info = prog->aux->func_info_cnt;
|
| 2501 | if (info.nr_func_info && ulen) {
|
| 2502 | char __user *user_finfo;
|
| 2503 |
|
| 2504 | user_finfo = u64_to_user_ptr(info.func_info);
|
| 2505 | ulen = min_t(u32, info.nr_func_info, ulen);
|
| 2506 | if (copy_to_user(user_finfo, prog->aux->func_info,
|
| 2507 | info.func_info_rec_size * ulen))
|
| 2508 | return -EFAULT;
|
| 2509 | }
|
| 2510 |
|
| 2511 | ulen = info.nr_line_info;
|
| 2512 | info.nr_line_info = prog->aux->nr_linfo;
|
| 2513 | if (info.nr_line_info && ulen) {
|
| 2514 | __u8 __user *user_linfo;
|
| 2515 |
|
| 2516 | user_linfo = u64_to_user_ptr(info.line_info);
|
| 2517 | ulen = min_t(u32, info.nr_line_info, ulen);
|
| 2518 | if (copy_to_user(user_linfo, prog->aux->linfo,
|
| 2519 | info.line_info_rec_size * ulen))
|
| 2520 | return -EFAULT;
|
| 2521 | }
|
| 2522 |
|
| 2523 | ulen = info.nr_jited_line_info;
|
| 2524 | if (prog->aux->jited_linfo)
|
| 2525 | info.nr_jited_line_info = prog->aux->nr_linfo;
|
| 2526 | else
|
| 2527 | info.nr_jited_line_info = 0;
|
| 2528 | if (info.nr_jited_line_info && ulen) {
|
| 2529 | if (bpf_dump_raw_ok()) {
|
| 2530 | __u64 __user *user_linfo;
|
| 2531 | u32 i;
|
| 2532 |
|
| 2533 | user_linfo = u64_to_user_ptr(info.jited_line_info);
|
| 2534 | ulen = min_t(u32, info.nr_jited_line_info, ulen);
|
| 2535 | for (i = 0; i < ulen; i++) {
|
| 2536 | if (put_user((__u64)(long)prog->aux->jited_linfo[i],
|
| 2537 | &user_linfo[i]))
|
| 2538 | return -EFAULT;
|
| 2539 | }
|
| 2540 | } else {
|
| 2541 | info.jited_line_info = 0;
|
| 2542 | }
|
| 2543 | }
|
| 2544 |
|
| 2545 | ulen = info.nr_prog_tags;
|
| 2546 | info.nr_prog_tags = prog->aux->func_cnt ? : 1;
|
| 2547 | if (ulen) {
|
| 2548 | __u8 __user (*user_prog_tags)[BPF_TAG_SIZE];
|
| 2549 | u32 i;
|
| 2550 |
|
| 2551 | user_prog_tags = u64_to_user_ptr(info.prog_tags);
|
| 2552 | ulen = min_t(u32, info.nr_prog_tags, ulen);
|
| 2553 | if (prog->aux->func_cnt) {
|
| 2554 | for (i = 0; i < ulen; i++) {
|
| 2555 | if (copy_to_user(user_prog_tags[i],
|
| 2556 | prog->aux->func[i]->tag,
|
| 2557 | BPF_TAG_SIZE))
|
| 2558 | return -EFAULT;
|
| 2559 | }
|
| 2560 | } else {
|
| 2561 | if (copy_to_user(user_prog_tags[0],
|
| 2562 | prog->tag, BPF_TAG_SIZE))
|
| 2563 | return -EFAULT;
|
| 2564 | }
|
| 2565 | }
|
| 2566 |
|
| 2567 | done:
|
| 2568 | if (copy_to_user(uinfo, &info, info_len) ||
|
| 2569 | put_user(info_len, &uattr->info.info_len))
|
| 2570 | return -EFAULT;
|
| 2571 |
|
| 2572 | return 0;
|
| 2573 | }
|
| 2574 |
|
| 2575 | static int bpf_map_get_info_by_fd(struct bpf_map *map,
|
| 2576 | const union bpf_attr *attr,
|
| 2577 | union bpf_attr __user *uattr)
|
| 2578 | {
|
| 2579 | struct bpf_map_info __user *uinfo = u64_to_user_ptr(attr->info.info);
|
| 2580 | struct bpf_map_info info = {};
|
| 2581 | u32 info_len = attr->info.info_len;
|
| 2582 | int err;
|
| 2583 |
|
| 2584 | err = bpf_check_uarg_tail_zero(uinfo, sizeof(info), info_len);
|
| 2585 | if (err)
|
| 2586 | return err;
|
| 2587 | info_len = min_t(u32, sizeof(info), info_len);
|
| 2588 |
|
| 2589 | info.type = map->map_type;
|
| 2590 | info.id = map->id;
|
| 2591 | info.key_size = map->key_size;
|
| 2592 | info.value_size = map->value_size;
|
| 2593 | info.max_entries = map->max_entries;
|
| 2594 | info.map_flags = map->map_flags;
|
| 2595 | memcpy(info.name, map->name, sizeof(map->name));
|
| 2596 |
|
| 2597 | if (map->btf) {
|
| 2598 | info.btf_id = btf_id(map->btf);
|
| 2599 | info.btf_key_type_id = map->btf_key_type_id;
|
| 2600 | info.btf_value_type_id = map->btf_value_type_id;
|
| 2601 | }
|
| 2602 |
|
| 2603 | if (bpf_map_is_dev_bound(map)) {
|
| 2604 | err = bpf_map_offload_info_fill(&info, map);
|
| 2605 | if (err)
|
| 2606 | return err;
|
| 2607 | }
|
| 2608 |
|
| 2609 | if (copy_to_user(uinfo, &info, info_len) ||
|
| 2610 | put_user(info_len, &uattr->info.info_len))
|
| 2611 | return -EFAULT;
|
| 2612 |
|
| 2613 | return 0;
|
| 2614 | }
|
| 2615 |
|
| 2616 | static int bpf_btf_get_info_by_fd(struct btf *btf,
|
| 2617 | const union bpf_attr *attr,
|
| 2618 | union bpf_attr __user *uattr)
|
| 2619 | {
|
| 2620 | struct bpf_btf_info __user *uinfo = u64_to_user_ptr(attr->info.info);
|
| 2621 | u32 info_len = attr->info.info_len;
|
| 2622 | int err;
|
| 2623 |
|
| 2624 | err = bpf_check_uarg_tail_zero(uinfo, sizeof(*uinfo), info_len);
|
| 2625 | if (err)
|
| 2626 | return err;
|
| 2627 |
|
| 2628 | return btf_get_info_by_fd(btf, attr, uattr);
|
| 2629 | }
|
| 2630 |
|
| 2631 | #define BPF_OBJ_GET_INFO_BY_FD_LAST_FIELD info.info
|
| 2632 |
|
| 2633 | static int bpf_obj_get_info_by_fd(const union bpf_attr *attr,
|
| 2634 | union bpf_attr __user *uattr)
|
| 2635 | {
|
| 2636 | int ufd = attr->info.bpf_fd;
|
| 2637 | struct fd f;
|
| 2638 | int err;
|
| 2639 |
|
| 2640 | if (CHECK_ATTR(BPF_OBJ_GET_INFO_BY_FD))
|
| 2641 | return -EINVAL;
|
| 2642 |
|
| 2643 | f = fdget(ufd);
|
| 2644 | if (!f.file)
|
| 2645 | return -EBADFD;
|
| 2646 |
|
| 2647 | if (f.file->f_op == &bpf_prog_fops)
|
| 2648 | err = bpf_prog_get_info_by_fd(f.file->private_data, attr,
|
| 2649 | uattr);
|
| 2650 | else if (f.file->f_op == &bpf_map_fops)
|
| 2651 | err = bpf_map_get_info_by_fd(f.file->private_data, attr,
|
| 2652 | uattr);
|
| 2653 | else if (f.file->f_op == &btf_fops)
|
| 2654 | err = bpf_btf_get_info_by_fd(f.file->private_data, attr, uattr);
|
| 2655 | else
|
| 2656 | err = -EINVAL;
|
| 2657 |
|
| 2658 | fdput(f);
|
| 2659 | return err;
|
| 2660 | }
|
| 2661 |
|
| 2662 | #define BPF_BTF_LOAD_LAST_FIELD btf_log_level
|
| 2663 |
|
| 2664 | static int bpf_btf_load(const union bpf_attr *attr)
|
| 2665 | {
|
| 2666 | if (CHECK_ATTR(BPF_BTF_LOAD))
|
| 2667 | return -EINVAL;
|
| 2668 |
|
| 2669 | if (!capable(CAP_SYS_ADMIN))
|
| 2670 | return -EPERM;
|
| 2671 |
|
| 2672 | return btf_new_fd(attr);
|
| 2673 | }
|
| 2674 |
|
| 2675 | #define BPF_BTF_GET_FD_BY_ID_LAST_FIELD btf_id
|
| 2676 |
|
| 2677 | static int bpf_btf_get_fd_by_id(const union bpf_attr *attr)
|
| 2678 | {
|
| 2679 | if (CHECK_ATTR(BPF_BTF_GET_FD_BY_ID))
|
| 2680 | return -EINVAL;
|
| 2681 |
|
| 2682 | if (!capable(CAP_SYS_ADMIN))
|
| 2683 | return -EPERM;
|
| 2684 |
|
| 2685 | return btf_get_fd_by_id(attr->btf_id);
|
| 2686 | }
|
| 2687 |
|
| 2688 | static int bpf_task_fd_query_copy(const union bpf_attr *attr,
|
| 2689 | union bpf_attr __user *uattr,
|
| 2690 | u32 prog_id, u32 fd_type,
|
| 2691 | const char *buf, u64 probe_offset,
|
| 2692 | u64 probe_addr)
|
| 2693 | {
|
| 2694 | char __user *ubuf = u64_to_user_ptr(attr->task_fd_query.buf);
|
| 2695 | u32 len = buf ? strlen(buf) : 0, input_len;
|
| 2696 | int err = 0;
|
| 2697 |
|
| 2698 | if (put_user(len, &uattr->task_fd_query.buf_len))
|
| 2699 | return -EFAULT;
|
| 2700 | input_len = attr->task_fd_query.buf_len;
|
| 2701 | if (input_len && ubuf) {
|
| 2702 | if (!len) {
|
| 2703 |
|
| 2704 | char zero = '\0';
|
| 2705 |
|
| 2706 | if (put_user(zero, ubuf))
|
| 2707 | return -EFAULT;
|
| 2708 | } else if (input_len >= len + 1) {
|
| 2709 |
|
| 2710 | if (copy_to_user(ubuf, buf, len + 1))
|
| 2711 | return -EFAULT;
|
| 2712 | } else {
|
| 2713 | |
| 2714 | * do a partial copy with NULL terminator.
|
| 2715 |
|
| 2716 | char zero = '\0';
|
| 2717 |
|
| 2718 | err = -ENOSPC;
|
| 2719 | if (copy_to_user(ubuf, buf, input_len - 1))
|
| 2720 | return -EFAULT;
|
| 2721 | if (put_user(zero, ubuf + input_len - 1))
|
| 2722 | return -EFAULT;
|
| 2723 | }
|
| 2724 | }
|
| 2725 |
|
| 2726 | if (put_user(prog_id, &uattr->task_fd_query.prog_id) ||
|
| 2727 | put_user(fd_type, &uattr->task_fd_query.fd_type) ||
|
| 2728 | put_user(probe_offset, &uattr->task_fd_query.probe_offset) ||
|
| 2729 | put_user(probe_addr, &uattr->task_fd_query.probe_addr))
|
| 2730 | return -EFAULT;
|
| 2731 |
|
| 2732 | return err;
|
| 2733 | }
|
| 2734 |
|
| 2735 | #define BPF_TASK_FD_QUERY_LAST_FIELD task_fd_query.probe_addr
|
| 2736 |
|
| 2737 | static int bpf_task_fd_query(const union bpf_attr *attr,
|
| 2738 | union bpf_attr __user *uattr)
|
| 2739 | {
|
| 2740 | pid_t pid = attr->task_fd_query.pid;
|
| 2741 | u32 fd = attr->task_fd_query.fd;
|
| 2742 | const struct perf_event *event;
|
| 2743 | struct files_struct *files;
|
| 2744 | struct task_struct *task;
|
| 2745 | struct file *file;
|
| 2746 | int err;
|
| 2747 |
|
| 2748 | if (CHECK_ATTR(BPF_TASK_FD_QUERY))
|
| 2749 | return -EINVAL;
|
| 2750 |
|
| 2751 | if (!capable(CAP_SYS_ADMIN))
|
| 2752 | return -EPERM;
|
| 2753 |
|
| 2754 | if (attr->task_fd_query.flags != 0)
|
| 2755 | return -EINVAL;
|
| 2756 |
|
| 2757 | task = get_pid_task(find_vpid(pid), PIDTYPE_PID);
|
| 2758 | if (!task)
|
| 2759 | return -ENOENT;
|
| 2760 |
|
| 2761 | files = get_files_struct(task);
|
| 2762 | put_task_struct(task);
|
| 2763 | if (!files)
|
| 2764 | return -ENOENT;
|
| 2765 |
|
| 2766 | err = 0;
|
| 2767 | spin_lock(&files->file_lock);
|
| 2768 | file = fcheck_files(files, fd);
|
| 2769 | if (!file)
|
| 2770 | err = -EBADF;
|
| 2771 | else
|
| 2772 | get_file(file);
|
| 2773 | spin_unlock(&files->file_lock);
|
| 2774 | put_files_struct(files);
|
| 2775 |
|
| 2776 | if (err)
|
| 2777 | goto out;
|
| 2778 |
|
| 2779 | if (file->f_op == &bpf_raw_tp_fops) {
|
| 2780 | struct bpf_raw_tracepoint *raw_tp = file->private_data;
|
| 2781 | struct bpf_raw_event_map *btp = raw_tp->btp;
|
| 2782 |
|
| 2783 | err = bpf_task_fd_query_copy(attr, uattr,
|
| 2784 | raw_tp->prog->aux->id,
|
| 2785 | BPF_FD_TYPE_RAW_TRACEPOINT,
|
| 2786 | btp->tp->name, 0, 0);
|
| 2787 | goto put_file;
|
| 2788 | }
|
| 2789 |
|
| 2790 | event = perf_get_event(file);
|
| 2791 | if (!IS_ERR(event)) {
|
| 2792 | u64 probe_offset, probe_addr;
|
| 2793 | u32 prog_id, fd_type;
|
| 2794 | const char *buf;
|
| 2795 |
|
| 2796 | err = bpf_get_perf_event_info(event, &prog_id, &fd_type,
|
| 2797 | &buf, &probe_offset,
|
| 2798 | &probe_addr);
|
| 2799 | if (!err)
|
| 2800 | err = bpf_task_fd_query_copy(attr, uattr, prog_id,
|
| 2801 | fd_type, buf,
|
| 2802 | probe_offset,
|
| 2803 | probe_addr);
|
| 2804 | goto put_file;
|
| 2805 | }
|
| 2806 |
|
| 2807 | err = -ENOTSUPP;
|
| 2808 | put_file:
|
| 2809 | fput(file);
|
| 2810 | out:
|
| 2811 | return err;
|
| 2812 | }
|
| 2813 |
|
| 2814 | SYSCALL_DEFINE3(bpf, int, cmd, union bpf_attr __user *, uattr, unsigned int, size)
|
| 2815 | {
|
| 2816 | union bpf_attr attr = {};
|
| 2817 | int err;
|
| 2818 |
|
| 2819 | if (sysctl_unprivileged_bpf_disabled && !capable(CAP_SYS_ADMIN))
|
| 2820 | return -EPERM;
|
| 2821 |
|
| 2822 | err = bpf_check_uarg_tail_zero(uattr, sizeof(attr), size);
|
| 2823 | if (err)
|
| 2824 | return err;
|
| 2825 | size = min_t(u32, size, sizeof(attr));
|
| 2826 |
|
| 2827 |
|
| 2828 | if (copy_from_user(&attr, uattr, size) != 0)
|
| 2829 | return -EFAULT;
|
| 2830 |
|
| 2831 | err = security_bpf(cmd, &attr, size);
|
| 2832 | if (err < 0)
|
| 2833 | return err;
|
| 2834 |
|
| 2835 | switch (cmd) {
|
| 2836 | case BPF_MAP_CREATE:
|
| 2837 | err = map_create(&attr);
|
| 2838 | break;
|
| 2839 | case BPF_MAP_LOOKUP_ELEM:
|
| 2840 | err = map_lookup_elem(&attr);
|
| 2841 | break;
|
| 2842 | case BPF_MAP_UPDATE_ELEM:
|
| 2843 | err = map_update_elem(&attr);
|
| 2844 | break;
|
| 2845 | case BPF_MAP_DELETE_ELEM:
|
| 2846 | err = map_delete_elem(&attr);
|
| 2847 | break;
|
| 2848 | case BPF_MAP_GET_NEXT_KEY:
|
| 2849 | err = map_get_next_key(&attr);
|
| 2850 | break;
|
| 2851 | case BPF_MAP_FREEZE:
|
| 2852 | err = map_freeze(&attr);
|
| 2853 | break;
|
| 2854 | case BPF_PROG_LOAD:
|
| 2855 | err = bpf_prog_load(&attr, uattr);
|
| 2856 | break;
|
| 2857 | case BPF_OBJ_PIN:
|
| 2858 | err = bpf_obj_pin(&attr);
|
| 2859 | break;
|
| 2860 | case BPF_OBJ_GET:
|
| 2861 | err = bpf_obj_get(&attr);
|
| 2862 | break;
|
| 2863 | case BPF_PROG_ATTACH:
|
| 2864 | err = bpf_prog_attach(&attr);
|
| 2865 | break;
|
| 2866 | case BPF_PROG_DETACH:
|
| 2867 | err = bpf_prog_detach(&attr);
|
| 2868 | break;
|
| 2869 | case BPF_PROG_QUERY:
|
| 2870 | err = bpf_prog_query(&attr, uattr);
|
| 2871 | break;
|
| 2872 | case BPF_PROG_TEST_RUN:
|
| 2873 | err = bpf_prog_test_run(&attr, uattr);
|
| 2874 | break;
|
| 2875 | case BPF_PROG_GET_NEXT_ID:
|
| 2876 | err = bpf_obj_get_next_id(&attr, uattr,
|
| 2877 | &prog_idr, &prog_idr_lock);
|
| 2878 | break;
|
| 2879 | case BPF_MAP_GET_NEXT_ID:
|
| 2880 | err = bpf_obj_get_next_id(&attr, uattr,
|
| 2881 | &map_idr, &map_idr_lock);
|
| 2882 | break;
|
| 2883 | case BPF_PROG_GET_FD_BY_ID:
|
| 2884 | err = bpf_prog_get_fd_by_id(&attr);
|
| 2885 | break;
|
| 2886 | case BPF_MAP_GET_FD_BY_ID:
|
| 2887 | err = bpf_map_get_fd_by_id(&attr);
|
| 2888 | break;
|
| 2889 | case BPF_OBJ_GET_INFO_BY_FD:
|
| 2890 | err = bpf_obj_get_info_by_fd(&attr, uattr);
|
| 2891 | break;
|
| 2892 | case BPF_RAW_TRACEPOINT_OPEN:
|
| 2893 | err = bpf_raw_tracepoint_open(&attr);
|
| 2894 | break;
|
| 2895 | case BPF_BTF_LOAD:
|
| 2896 | err = bpf_btf_load(&attr);
|
| 2897 | break;
|
| 2898 | case BPF_BTF_GET_FD_BY_ID:
|
| 2899 | err = bpf_btf_get_fd_by_id(&attr);
|
| 2900 | break;
|
| 2901 | case BPF_TASK_FD_QUERY:
|
| 2902 | err = bpf_task_fd_query(&attr, uattr);
|
| 2903 | break;
|
| 2904 | case BPF_MAP_LOOKUP_AND_DELETE_ELEM:
|
| 2905 | err = map_lookup_and_delete_elem(&attr);
|
| 2906 | break;
|
| 2907 | default:
|
| 2908 | err = -EINVAL;
|
| 2909 | break;
|
| 2910 | }
|
| 2911 |
|
| 2912 | return err;
|
| 2913 | }
|