| 1 | |
| 2 | |
| 3 |
|
| 4 | #include <linux/bpf.h>
|
| 5 | #include <linux/jhash.h>
|
| 6 | #include <linux/filter.h>
|
| 7 | #include <linux/kernel.h>
|
| 8 | #include <linux/stacktrace.h>
|
| 9 | #include <linux/perf_event.h>
|
| 10 | #include <linux/btf_ids.h>
|
| 11 | #include <linux/buildid.h>
|
| 12 | #include "percpu_freelist.h"
|
| 13 | #include "mmap_unlock_work.h"
|
| 14 |
|
| 15 | #define STACK_CREATE_FLAG_MASK \
|
| 16 | (BPF_F_NUMA_NODE | BPF_F_RDONLY | BPF_F_WRONLY | \
|
| 17 | BPF_F_STACK_BUILD_ID)
|
| 18 |
|
| 19 | struct stack_map_bucket {
|
| 20 | struct pcpu_freelist_node fnode;
|
| 21 | u32 hash;
|
| 22 | u32 nr;
|
| 23 | u64 data[];
|
| 24 | };
|
| 25 |
|
| 26 | struct bpf_stack_map {
|
| 27 | struct bpf_map map;
|
| 28 | void *elems;
|
| 29 | struct pcpu_freelist freelist;
|
| 30 | u32 n_buckets;
|
| 31 | struct stack_map_bucket *buckets[];
|
| 32 | };
|
| 33 |
|
| 34 | static inline bool stack_map_use_build_id(struct bpf_map *map)
|
| 35 | {
|
| 36 | return (map->map_flags & BPF_F_STACK_BUILD_ID);
|
| 37 | }
|
| 38 |
|
| 39 | static inline int stack_map_data_size(struct bpf_map *map)
|
| 40 | {
|
| 41 | return stack_map_use_build_id(map) ?
|
| 42 | sizeof(struct bpf_stack_build_id) : sizeof(u64);
|
| 43 | }
|
| 44 |
|
| 45 | static int prealloc_elems_and_freelist(struct bpf_stack_map *smap)
|
| 46 | {
|
| 47 | u64 elem_size = sizeof(struct stack_map_bucket) +
|
| 48 | (u64)smap->map.value_size;
|
| 49 | int err;
|
| 50 |
|
| 51 | smap->elems = bpf_map_area_alloc(elem_size * smap->map.max_entries,
|
| 52 | smap->map.numa_node);
|
| 53 | if (!smap->elems)
|
| 54 | return -ENOMEM;
|
| 55 |
|
| 56 | err = pcpu_freelist_init(&smap->freelist);
|
| 57 | if (err)
|
| 58 | goto free_elems;
|
| 59 |
|
| 60 | pcpu_freelist_populate(&smap->freelist, smap->elems, elem_size,
|
| 61 | smap->map.max_entries);
|
| 62 | return 0;
|
| 63 |
|
| 64 | free_elems:
|
| 65 | bpf_map_area_free(smap->elems);
|
| 66 | return err;
|
| 67 | }
|
| 68 |
|
| 69 |
|
| 70 | static struct bpf_map *stack_map_alloc(union bpf_attr *attr)
|
| 71 | {
|
| 72 | u32 value_size = attr->value_size;
|
| 73 | struct bpf_stack_map *smap;
|
| 74 | u64 cost, n_buckets;
|
| 75 | int err;
|
| 76 |
|
| 77 | if (attr->map_flags & ~STACK_CREATE_FLAG_MASK)
|
| 78 | return ERR_PTR(-EINVAL);
|
| 79 |
|
| 80 |
|
| 81 | if (attr->max_entries == 0 || attr->key_size != 4 ||
|
| 82 | value_size < 8 || value_size % 8)
|
| 83 | return ERR_PTR(-EINVAL);
|
| 84 |
|
| 85 | BUILD_BUG_ON(sizeof(struct bpf_stack_build_id) % sizeof(u64));
|
| 86 | if (attr->map_flags & BPF_F_STACK_BUILD_ID) {
|
| 87 | if (value_size % sizeof(struct bpf_stack_build_id) ||
|
| 88 | value_size / sizeof(struct bpf_stack_build_id)
|
| 89 | > sysctl_perf_event_max_stack)
|
| 90 | return ERR_PTR(-EINVAL);
|
| 91 | } else if (value_size / 8 > sysctl_perf_event_max_stack)
|
| 92 | return ERR_PTR(-EINVAL);
|
| 93 |
|
| 94 |
|
| 95 | n_buckets = roundup_pow_of_two(attr->max_entries);
|
| 96 | if (!n_buckets)
|
| 97 | return ERR_PTR(-E2BIG);
|
| 98 |
|
| 99 | cost = n_buckets * sizeof(struct stack_map_bucket *) + sizeof(*smap);
|
| 100 | smap = bpf_map_area_alloc(cost, bpf_map_attr_numa_node(attr));
|
| 101 | if (!smap)
|
| 102 | return ERR_PTR(-ENOMEM);
|
| 103 |
|
| 104 | bpf_map_init_from_attr(&smap->map, attr);
|
| 105 | smap->n_buckets = n_buckets;
|
| 106 |
|
| 107 | err = get_callchain_buffers(sysctl_perf_event_max_stack);
|
| 108 | if (err)
|
| 109 | goto free_smap;
|
| 110 |
|
| 111 | err = prealloc_elems_and_freelist(smap);
|
| 112 | if (err)
|
| 113 | goto put_buffers;
|
| 114 |
|
| 115 | return &smap->map;
|
| 116 |
|
| 117 | put_buffers:
|
| 118 | put_callchain_buffers();
|
| 119 | free_smap:
|
| 120 | bpf_map_area_free(smap);
|
| 121 | return ERR_PTR(err);
|
| 122 | }
|
| 123 |
|
| 124 | static void stack_map_get_build_id_offset(struct bpf_stack_build_id *id_offs,
|
| 125 | u64 *ips, u32 trace_nr, bool user)
|
| 126 | {
|
| 127 | int i;
|
| 128 | struct mmap_unlock_irq_work *work = NULL;
|
| 129 | bool irq_work_busy = bpf_mmap_unlock_get_irq_work(&work);
|
| 130 | struct vm_area_struct *vma, *prev_vma = NULL;
|
| 131 | const char *prev_build_id;
|
| 132 |
|
| 133 | |
| 134 | * fallback is used for kernel stack (!user) on a stackmap with
|
| 135 | * build_id.
|
| 136 |
|
| 137 | if (!user || !current || !current->mm || irq_work_busy ||
|
| 138 | !mmap_read_trylock(current->mm)) {
|
| 139 |
|
| 140 | for (i = 0; i < trace_nr; i++) {
|
| 141 | id_offs[i].status = BPF_STACK_BUILD_ID_IP;
|
| 142 | id_offs[i].ip = ips[i];
|
| 143 | memset(id_offs[i].build_id, 0, BUILD_ID_SIZE_MAX);
|
| 144 | }
|
| 145 | return;
|
| 146 | }
|
| 147 |
|
| 148 | for (i = 0; i < trace_nr; i++) {
|
| 149 | if (range_in_vma(prev_vma, ips[i], ips[i])) {
|
| 150 | vma = prev_vma;
|
| 151 | memcpy(id_offs[i].build_id, prev_build_id,
|
| 152 | BUILD_ID_SIZE_MAX);
|
| 153 | goto build_id_valid;
|
| 154 | }
|
| 155 | vma = find_vma(current->mm, ips[i]);
|
| 156 | if (!vma || build_id_parse(vma, id_offs[i].build_id, NULL)) {
|
| 157 |
|
| 158 | id_offs[i].status = BPF_STACK_BUILD_ID_IP;
|
| 159 | id_offs[i].ip = ips[i];
|
| 160 | memset(id_offs[i].build_id, 0, BUILD_ID_SIZE_MAX);
|
| 161 | continue;
|
| 162 | }
|
| 163 | build_id_valid:
|
| 164 | id_offs[i].offset = (vma->vm_pgoff << PAGE_SHIFT) + ips[i]
|
| 165 | - vma->vm_start;
|
| 166 | id_offs[i].status = BPF_STACK_BUILD_ID_VALID;
|
| 167 | prev_vma = vma;
|
| 168 | prev_build_id = id_offs[i].build_id;
|
| 169 | }
|
| 170 | bpf_mmap_unlock_mm(work, current->mm);
|
| 171 | }
|
| 172 |
|
| 173 | static struct perf_callchain_entry *
|
| 174 | get_callchain_entry_for_task(struct task_struct *task, u32 max_depth)
|
| 175 | {
|
| 176 | #ifdef CONFIG_STACKTRACE
|
| 177 | struct perf_callchain_entry *entry;
|
| 178 | int rctx;
|
| 179 |
|
| 180 | entry = get_callchain_entry(&rctx);
|
| 181 |
|
| 182 | if (!entry)
|
| 183 | return NULL;
|
| 184 |
|
| 185 | entry->nr = stack_trace_save_tsk(task, (unsigned long *)entry->ip,
|
| 186 | max_depth, 0);
|
| 187 |
|
| 188 | |
| 189 | * perf_callchain_entry uses u64 array. For 32-bit systems, it is
|
| 190 | * necessary to fix this mismatch.
|
| 191 |
|
| 192 | if (__BITS_PER_LONG != 64) {
|
| 193 | unsigned long *from = (unsigned long *) entry->ip;
|
| 194 | u64 *to = entry->ip;
|
| 195 | int i;
|
| 196 |
|
| 197 |
|
| 198 | for (i = entry->nr - 1; i >= 0; i--)
|
| 199 | to[i] = (u64)(from[i]);
|
| 200 | }
|
| 201 |
|
| 202 | put_callchain_entry(rctx);
|
| 203 |
|
| 204 | return entry;
|
| 205 | #else
|
| 206 | return NULL;
|
| 207 | #endif
|
| 208 | }
|
| 209 |
|
| 210 | static long __bpf_get_stackid(struct bpf_map *map,
|
| 211 | struct perf_callchain_entry *trace, u64 flags)
|
| 212 | {
|
| 213 | struct bpf_stack_map *smap = container_of(map, struct bpf_stack_map, map);
|
| 214 | struct stack_map_bucket *bucket, *new_bucket, *old_bucket;
|
| 215 | u32 skip = flags & BPF_F_SKIP_FIELD_MASK;
|
| 216 | u32 hash, id, trace_nr, trace_len;
|
| 217 | bool user = flags & BPF_F_USER_STACK;
|
| 218 | u64 *ips;
|
| 219 | bool hash_matches;
|
| 220 |
|
| 221 | if (trace->nr <= skip)
|
| 222 |
|
| 223 | return -EFAULT;
|
| 224 |
|
| 225 | trace_nr = trace->nr - skip;
|
| 226 | trace_len = trace_nr * sizeof(u64);
|
| 227 | ips = trace->ip + skip;
|
| 228 | hash = jhash2((u32 *)ips, trace_len / sizeof(u32), 0);
|
| 229 | id = hash & (smap->n_buckets - 1);
|
| 230 | bucket = READ_ONCE(smap->buckets[id]);
|
| 231 |
|
| 232 | hash_matches = bucket && bucket->hash == hash;
|
| 233 |
|
| 234 | if (hash_matches && flags & BPF_F_FAST_STACK_CMP)
|
| 235 | return id;
|
| 236 |
|
| 237 | if (stack_map_use_build_id(map)) {
|
| 238 |
|
| 239 | new_bucket = (struct stack_map_bucket *)
|
| 240 | pcpu_freelist_pop(&smap->freelist);
|
| 241 | if (unlikely(!new_bucket))
|
| 242 | return -ENOMEM;
|
| 243 | new_bucket->nr = trace_nr;
|
| 244 | stack_map_get_build_id_offset(
|
| 245 | (struct bpf_stack_build_id *)new_bucket->data,
|
| 246 | ips, trace_nr, user);
|
| 247 | trace_len = trace_nr * sizeof(struct bpf_stack_build_id);
|
| 248 | if (hash_matches && bucket->nr == trace_nr &&
|
| 249 | memcmp(bucket->data, new_bucket->data, trace_len) == 0) {
|
| 250 | pcpu_freelist_push(&smap->freelist, &new_bucket->fnode);
|
| 251 | return id;
|
| 252 | }
|
| 253 | if (bucket && !(flags & BPF_F_REUSE_STACKID)) {
|
| 254 | pcpu_freelist_push(&smap->freelist, &new_bucket->fnode);
|
| 255 | return -EEXIST;
|
| 256 | }
|
| 257 | } else {
|
| 258 | if (hash_matches && bucket->nr == trace_nr &&
|
| 259 | memcmp(bucket->data, ips, trace_len) == 0)
|
| 260 | return id;
|
| 261 | if (bucket && !(flags & BPF_F_REUSE_STACKID))
|
| 262 | return -EEXIST;
|
| 263 |
|
| 264 | new_bucket = (struct stack_map_bucket *)
|
| 265 | pcpu_freelist_pop(&smap->freelist);
|
| 266 | if (unlikely(!new_bucket))
|
| 267 | return -ENOMEM;
|
| 268 | memcpy(new_bucket->data, ips, trace_len);
|
| 269 | }
|
| 270 |
|
| 271 | new_bucket->hash = hash;
|
| 272 | new_bucket->nr = trace_nr;
|
| 273 |
|
| 274 | old_bucket = xchg(&smap->buckets[id], new_bucket);
|
| 275 | if (old_bucket)
|
| 276 | pcpu_freelist_push(&smap->freelist, &old_bucket->fnode);
|
| 277 | return id;
|
| 278 | }
|
| 279 |
|
| 280 | BPF_CALL_3(bpf_get_stackid, struct pt_regs *, regs, struct bpf_map *, map,
|
| 281 | u64, flags)
|
| 282 | {
|
| 283 | u32 max_depth = map->value_size / stack_map_data_size(map);
|
| 284 | u32 skip = flags & BPF_F_SKIP_FIELD_MASK;
|
| 285 | bool user = flags & BPF_F_USER_STACK;
|
| 286 | struct perf_callchain_entry *trace;
|
| 287 | bool kernel = !user;
|
| 288 |
|
| 289 | if (unlikely(flags & ~(BPF_F_SKIP_FIELD_MASK | BPF_F_USER_STACK |
|
| 290 | BPF_F_FAST_STACK_CMP | BPF_F_REUSE_STACKID)))
|
| 291 | return -EINVAL;
|
| 292 |
|
| 293 | max_depth += skip;
|
| 294 | if (max_depth > sysctl_perf_event_max_stack)
|
| 295 | max_depth = sysctl_perf_event_max_stack;
|
| 296 |
|
| 297 | trace = get_perf_callchain(regs, 0, kernel, user, max_depth,
|
| 298 | false, false);
|
| 299 |
|
| 300 | if (unlikely(!trace))
|
| 301 |
|
| 302 | return -EFAULT;
|
| 303 |
|
| 304 | return __bpf_get_stackid(map, trace, flags);
|
| 305 | }
|
| 306 |
|
| 307 | const struct bpf_func_proto bpf_get_stackid_proto = {
|
| 308 | .func = bpf_get_stackid,
|
| 309 | .gpl_only = true,
|
| 310 | .ret_type = RET_INTEGER,
|
| 311 | .arg1_type = ARG_PTR_TO_CTX,
|
| 312 | .arg2_type = ARG_CONST_MAP_PTR,
|
| 313 | .arg3_type = ARG_ANYTHING,
|
| 314 | };
|
| 315 |
|
| 316 | static __u64 count_kernel_ip(struct perf_callchain_entry *trace)
|
| 317 | {
|
| 318 | __u64 nr_kernel = 0;
|
| 319 |
|
| 320 | while (nr_kernel < trace->nr) {
|
| 321 | if (trace->ip[nr_kernel] == PERF_CONTEXT_USER)
|
| 322 | break;
|
| 323 | nr_kernel++;
|
| 324 | }
|
| 325 | return nr_kernel;
|
| 326 | }
|
| 327 |
|
| 328 | BPF_CALL_3(bpf_get_stackid_pe, struct bpf_perf_event_data_kern *, ctx,
|
| 329 | struct bpf_map *, map, u64, flags)
|
| 330 | {
|
| 331 | struct perf_event *event = ctx->event;
|
| 332 | struct perf_callchain_entry *trace;
|
| 333 | bool kernel, user;
|
| 334 | __u64 nr_kernel;
|
| 335 | int ret;
|
| 336 |
|
| 337 |
|
| 338 | if (!(event->attr.sample_type & PERF_SAMPLE_CALLCHAIN))
|
| 339 | return bpf_get_stackid((unsigned long)(ctx->regs),
|
| 340 | (unsigned long) map, flags, 0, 0);
|
| 341 |
|
| 342 | if (unlikely(flags & ~(BPF_F_SKIP_FIELD_MASK | BPF_F_USER_STACK |
|
| 343 | BPF_F_FAST_STACK_CMP | BPF_F_REUSE_STACKID)))
|
| 344 | return -EINVAL;
|
| 345 |
|
| 346 | user = flags & BPF_F_USER_STACK;
|
| 347 | kernel = !user;
|
| 348 |
|
| 349 | trace = ctx->data->callchain;
|
| 350 | if (unlikely(!trace))
|
| 351 | return -EFAULT;
|
| 352 |
|
| 353 | nr_kernel = count_kernel_ip(trace);
|
| 354 |
|
| 355 | if (kernel) {
|
| 356 | __u64 nr = trace->nr;
|
| 357 |
|
| 358 | trace->nr = nr_kernel;
|
| 359 | ret = __bpf_get_stackid(map, trace, flags);
|
| 360 |
|
| 361 |
|
| 362 | trace->nr = nr;
|
| 363 | } else {
|
| 364 | u64 skip = flags & BPF_F_SKIP_FIELD_MASK;
|
| 365 |
|
| 366 | skip += nr_kernel;
|
| 367 | if (skip > BPF_F_SKIP_FIELD_MASK)
|
| 368 | return -EFAULT;
|
| 369 |
|
| 370 | flags = (flags & ~BPF_F_SKIP_FIELD_MASK) | skip;
|
| 371 | ret = __bpf_get_stackid(map, trace, flags);
|
| 372 | }
|
| 373 | return ret;
|
| 374 | }
|
| 375 |
|
| 376 | const struct bpf_func_proto bpf_get_stackid_proto_pe = {
|
| 377 | .func = bpf_get_stackid_pe,
|
| 378 | .gpl_only = false,
|
| 379 | .ret_type = RET_INTEGER,
|
| 380 | .arg1_type = ARG_PTR_TO_CTX,
|
| 381 | .arg2_type = ARG_CONST_MAP_PTR,
|
| 382 | .arg3_type = ARG_ANYTHING,
|
| 383 | };
|
| 384 |
|
| 385 | static long __bpf_get_stack(struct pt_regs *regs, struct task_struct *task,
|
| 386 | struct perf_callchain_entry *trace_in,
|
| 387 | void *buf, u32 size, u64 flags)
|
| 388 | {
|
| 389 | u32 trace_nr, copy_len, elem_size, num_elem, max_depth;
|
| 390 | bool user_build_id = flags & BPF_F_USER_BUILD_ID;
|
| 391 | u32 skip = flags & BPF_F_SKIP_FIELD_MASK;
|
| 392 | bool user = flags & BPF_F_USER_STACK;
|
| 393 | struct perf_callchain_entry *trace;
|
| 394 | bool kernel = !user;
|
| 395 | int err = -EINVAL;
|
| 396 | u64 *ips;
|
| 397 |
|
| 398 | if (unlikely(flags & ~(BPF_F_SKIP_FIELD_MASK | BPF_F_USER_STACK |
|
| 399 | BPF_F_USER_BUILD_ID)))
|
| 400 | goto clear;
|
| 401 | if (kernel && user_build_id)
|
| 402 | goto clear;
|
| 403 |
|
| 404 | elem_size = (user && user_build_id) ? sizeof(struct bpf_stack_build_id)
|
| 405 | : sizeof(u64);
|
| 406 | if (unlikely(size % elem_size))
|
| 407 | goto clear;
|
| 408 |
|
| 409 |
|
| 410 | if (task && user && !user_mode(regs))
|
| 411 | goto err_fault;
|
| 412 |
|
| 413 | num_elem = size / elem_size;
|
| 414 | max_depth = num_elem + skip;
|
| 415 | if (sysctl_perf_event_max_stack < max_depth)
|
| 416 | max_depth = sysctl_perf_event_max_stack;
|
| 417 |
|
| 418 | if (trace_in)
|
| 419 | trace = trace_in;
|
| 420 | else if (kernel && task)
|
| 421 | trace = get_callchain_entry_for_task(task, max_depth);
|
| 422 | else
|
| 423 | trace = get_perf_callchain(regs, 0, kernel, user, max_depth,
|
| 424 | false, false);
|
| 425 | if (unlikely(!trace))
|
| 426 | goto err_fault;
|
| 427 |
|
| 428 | if (trace->nr < skip)
|
| 429 | goto err_fault;
|
| 430 |
|
| 431 | trace_nr = trace->nr - skip;
|
| 432 | trace_nr = (trace_nr <= num_elem) ? trace_nr : num_elem;
|
| 433 | copy_len = trace_nr * elem_size;
|
| 434 |
|
| 435 | ips = trace->ip + skip;
|
| 436 | if (user && user_build_id)
|
| 437 | stack_map_get_build_id_offset(buf, ips, trace_nr, user);
|
| 438 | else
|
| 439 | memcpy(buf, ips, copy_len);
|
| 440 |
|
| 441 | if (size > copy_len)
|
| 442 | memset(buf + copy_len, 0, size - copy_len);
|
| 443 | return copy_len;
|
| 444 |
|
| 445 | err_fault:
|
| 446 | err = -EFAULT;
|
| 447 | clear:
|
| 448 | memset(buf, 0, size);
|
| 449 | return err;
|
| 450 | }
|
| 451 |
|
| 452 | BPF_CALL_4(bpf_get_stack, struct pt_regs *, regs, void *, buf, u32, size,
|
| 453 | u64, flags)
|
| 454 | {
|
| 455 | return __bpf_get_stack(regs, NULL, NULL, buf, size, flags);
|
| 456 | }
|
| 457 |
|
| 458 | const struct bpf_func_proto bpf_get_stack_proto = {
|
| 459 | .func = bpf_get_stack,
|
| 460 | .gpl_only = true,
|
| 461 | .ret_type = RET_INTEGER,
|
| 462 | .arg1_type = ARG_PTR_TO_CTX,
|
| 463 | .arg2_type = ARG_PTR_TO_UNINIT_MEM,
|
| 464 | .arg3_type = ARG_CONST_SIZE_OR_ZERO,
|
| 465 | .arg4_type = ARG_ANYTHING,
|
| 466 | };
|
| 467 |
|
| 468 | BPF_CALL_4(bpf_get_task_stack, struct task_struct *, task, void *, buf,
|
| 469 | u32, size, u64, flags)
|
| 470 | {
|
| 471 | struct pt_regs *regs;
|
| 472 | long res = -EINVAL;
|
| 473 |
|
| 474 | if (!try_get_task_stack(task))
|
| 475 | return -EFAULT;
|
| 476 |
|
| 477 | regs = task_pt_regs(task);
|
| 478 | if (regs)
|
| 479 | res = __bpf_get_stack(regs, task, NULL, buf, size, flags);
|
| 480 | put_task_stack(task);
|
| 481 |
|
| 482 | return res;
|
| 483 | }
|
| 484 |
|
| 485 | const struct bpf_func_proto bpf_get_task_stack_proto = {
|
| 486 | .func = bpf_get_task_stack,
|
| 487 | .gpl_only = false,
|
| 488 | .ret_type = RET_INTEGER,
|
| 489 | .arg1_type = ARG_PTR_TO_BTF_ID,
|
| 490 | .arg1_btf_id = &btf_tracing_ids[BTF_TRACING_TYPE_TASK],
|
| 491 | .arg2_type = ARG_PTR_TO_UNINIT_MEM,
|
| 492 | .arg3_type = ARG_CONST_SIZE_OR_ZERO,
|
| 493 | .arg4_type = ARG_ANYTHING,
|
| 494 | };
|
| 495 |
|
| 496 | BPF_CALL_4(bpf_get_stack_pe, struct bpf_perf_event_data_kern *, ctx,
|
| 497 | void *, buf, u32, size, u64, flags)
|
| 498 | {
|
| 499 | struct pt_regs *regs = (struct pt_regs *)(ctx->regs);
|
| 500 | struct perf_event *event = ctx->event;
|
| 501 | struct perf_callchain_entry *trace;
|
| 502 | bool kernel, user;
|
| 503 | int err = -EINVAL;
|
| 504 | __u64 nr_kernel;
|
| 505 |
|
| 506 | if (!(event->attr.sample_type & PERF_SAMPLE_CALLCHAIN))
|
| 507 | return __bpf_get_stack(regs, NULL, NULL, buf, size, flags);
|
| 508 |
|
| 509 | if (unlikely(flags & ~(BPF_F_SKIP_FIELD_MASK | BPF_F_USER_STACK |
|
| 510 | BPF_F_USER_BUILD_ID)))
|
| 511 | goto clear;
|
| 512 |
|
| 513 | user = flags & BPF_F_USER_STACK;
|
| 514 | kernel = !user;
|
| 515 |
|
| 516 | err = -EFAULT;
|
| 517 | trace = ctx->data->callchain;
|
| 518 | if (unlikely(!trace))
|
| 519 | goto clear;
|
| 520 |
|
| 521 | nr_kernel = count_kernel_ip(trace);
|
| 522 |
|
| 523 | if (kernel) {
|
| 524 | __u64 nr = trace->nr;
|
| 525 |
|
| 526 | trace->nr = nr_kernel;
|
| 527 | err = __bpf_get_stack(regs, NULL, trace, buf, size, flags);
|
| 528 |
|
| 529 |
|
| 530 | trace->nr = nr;
|
| 531 | } else {
|
| 532 | u64 skip = flags & BPF_F_SKIP_FIELD_MASK;
|
| 533 |
|
| 534 | skip += nr_kernel;
|
| 535 | if (skip > BPF_F_SKIP_FIELD_MASK)
|
| 536 | goto clear;
|
| 537 |
|
| 538 | flags = (flags & ~BPF_F_SKIP_FIELD_MASK) | skip;
|
| 539 | err = __bpf_get_stack(regs, NULL, trace, buf, size, flags);
|
| 540 | }
|
| 541 | return err;
|
| 542 |
|
| 543 | clear:
|
| 544 | memset(buf, 0, size);
|
| 545 | return err;
|
| 546 |
|
| 547 | }
|
| 548 |
|
| 549 | const struct bpf_func_proto bpf_get_stack_proto_pe = {
|
| 550 | .func = bpf_get_stack_pe,
|
| 551 | .gpl_only = true,
|
| 552 | .ret_type = RET_INTEGER,
|
| 553 | .arg1_type = ARG_PTR_TO_CTX,
|
| 554 | .arg2_type = ARG_PTR_TO_UNINIT_MEM,
|
| 555 | .arg3_type = ARG_CONST_SIZE_OR_ZERO,
|
| 556 | .arg4_type = ARG_ANYTHING,
|
| 557 | };
|
| 558 |
|
| 559 |
|
| 560 | static void *stack_map_lookup_elem(struct bpf_map *map, void *key)
|
| 561 | {
|
| 562 | return ERR_PTR(-EOPNOTSUPP);
|
| 563 | }
|
| 564 |
|
| 565 |
|
| 566 | int bpf_stackmap_copy(struct bpf_map *map, void *key, void *value)
|
| 567 | {
|
| 568 | struct bpf_stack_map *smap = container_of(map, struct bpf_stack_map, map);
|
| 569 | struct stack_map_bucket *bucket, *old_bucket;
|
| 570 | u32 id = *(u32 *)key, trace_len;
|
| 571 |
|
| 572 | if (unlikely(id >= smap->n_buckets))
|
| 573 | return -ENOENT;
|
| 574 |
|
| 575 | bucket = xchg(&smap->buckets[id], NULL);
|
| 576 | if (!bucket)
|
| 577 | return -ENOENT;
|
| 578 |
|
| 579 | trace_len = bucket->nr * stack_map_data_size(map);
|
| 580 | memcpy(value, bucket->data, trace_len);
|
| 581 | memset(value + trace_len, 0, map->value_size - trace_len);
|
| 582 |
|
| 583 | old_bucket = xchg(&smap->buckets[id], bucket);
|
| 584 | if (old_bucket)
|
| 585 | pcpu_freelist_push(&smap->freelist, &old_bucket->fnode);
|
| 586 | return 0;
|
| 587 | }
|
| 588 |
|
| 589 | static int stack_map_get_next_key(struct bpf_map *map, void *key,
|
| 590 | void *next_key)
|
| 591 | {
|
| 592 | struct bpf_stack_map *smap = container_of(map,
|
| 593 | struct bpf_stack_map, map);
|
| 594 | u32 id;
|
| 595 |
|
| 596 | WARN_ON_ONCE(!rcu_read_lock_held());
|
| 597 |
|
| 598 | if (!key) {
|
| 599 | id = 0;
|
| 600 | } else {
|
| 601 | id = *(u32 *)key;
|
| 602 | if (id >= smap->n_buckets || !smap->buckets[id])
|
| 603 | id = 0;
|
| 604 | else
|
| 605 | id++;
|
| 606 | }
|
| 607 |
|
| 608 | while (id < smap->n_buckets && !smap->buckets[id])
|
| 609 | id++;
|
| 610 |
|
| 611 | if (id >= smap->n_buckets)
|
| 612 | return -ENOENT;
|
| 613 |
|
| 614 | *(u32 *)next_key = id;
|
| 615 | return 0;
|
| 616 | }
|
| 617 |
|
| 618 | static long stack_map_update_elem(struct bpf_map *map, void *key, void *value,
|
| 619 | u64 map_flags)
|
| 620 | {
|
| 621 | return -EINVAL;
|
| 622 | }
|
| 623 |
|
| 624 |
|
| 625 | static long stack_map_delete_elem(struct bpf_map *map, void *key)
|
| 626 | {
|
| 627 | struct bpf_stack_map *smap = container_of(map, struct bpf_stack_map, map);
|
| 628 | struct stack_map_bucket *old_bucket;
|
| 629 | u32 id = *(u32 *)key;
|
| 630 |
|
| 631 | if (unlikely(id >= smap->n_buckets))
|
| 632 | return -E2BIG;
|
| 633 |
|
| 634 | old_bucket = xchg(&smap->buckets[id], NULL);
|
| 635 | if (old_bucket) {
|
| 636 | pcpu_freelist_push(&smap->freelist, &old_bucket->fnode);
|
| 637 | return 0;
|
| 638 | } else {
|
| 639 | return -ENOENT;
|
| 640 | }
|
| 641 | }
|
| 642 |
|
| 643 |
|
| 644 | static void stack_map_free(struct bpf_map *map)
|
| 645 | {
|
| 646 | struct bpf_stack_map *smap = container_of(map, struct bpf_stack_map, map);
|
| 647 |
|
| 648 | bpf_map_area_free(smap->elems);
|
| 649 | pcpu_freelist_destroy(&smap->freelist);
|
| 650 | bpf_map_area_free(smap);
|
| 651 | put_callchain_buffers();
|
| 652 | }
|
| 653 |
|
| 654 | static u64 stack_map_mem_usage(const struct bpf_map *map)
|
| 655 | {
|
| 656 | struct bpf_stack_map *smap = container_of(map, struct bpf_stack_map, map);
|
| 657 | u64 value_size = map->value_size;
|
| 658 | u64 n_buckets = smap->n_buckets;
|
| 659 | u64 enties = map->max_entries;
|
| 660 | u64 usage = sizeof(*smap);
|
| 661 |
|
| 662 | usage += n_buckets * sizeof(struct stack_map_bucket *);
|
| 663 | usage += enties * (sizeof(struct stack_map_bucket) + value_size);
|
| 664 | return usage;
|
| 665 | }
|
| 666 |
|
| 667 | BTF_ID_LIST_SINGLE(stack_trace_map_btf_ids, struct, bpf_stack_map)
|
| 668 | const struct bpf_map_ops stack_trace_map_ops = {
|
| 669 | .map_meta_equal = bpf_map_meta_equal,
|
| 670 | .map_alloc = stack_map_alloc,
|
| 671 | .map_free = stack_map_free,
|
| 672 | .map_get_next_key = stack_map_get_next_key,
|
| 673 | .map_lookup_elem = stack_map_lookup_elem,
|
| 674 | .map_update_elem = stack_map_update_elem,
|
| 675 | .map_delete_elem = stack_map_delete_elem,
|
| 676 | .map_check_btf = map_check_no_btf,
|
| 677 | .map_mem_usage = stack_map_mem_usage,
|
| 678 | .map_btf_id = &stack_trace_map_btf_ids[0],
|
| 679 | };
|