projects/linux·blob v7.2-rc5 · refs

1050 lines · 28.0 KiB

1// SPDX-License-Identifier: GPL-2.0-only
2/* Copyright (c) 2016 Facebook
3 */
4#include <linux/bpf.h>
5#include <linux/jhash.h>
6#include <linux/filter.h>
7#include <linux/kernel.h>
8#include <linux/stacktrace.h>
9#include <linux/perf_event.h>
10#include <linux/btf_ids.h>
11#include <linux/buildid.h>
12#include <linux/mmap_lock.h>
13#include "percpu_freelist.h"
14#include "mmap_unlock_work.h"
15
16#define STACK_CREATE_FLAG_MASK \
17 (BPF_F_NUMA_NODE | BPF_F_RDONLY | BPF_F_WRONLY | \
18 BPF_F_STACK_BUILD_ID)
19
20struct stack_map_bucket {
21 struct pcpu_freelist_node fnode;
22 u32 hash;
23 u32 nr;
24 u64 data[];
25};
26
27struct bpf_stack_map {
28 struct bpf_map map;
29 void *elems;
30 struct pcpu_freelist freelist;
31 u32 n_buckets;
32 struct stack_map_bucket *buckets[] __counted_by(n_buckets);
33};
34
35static inline bool stack_map_use_build_id(struct bpf_map *map)
36{
37 return (map->map_flags & BPF_F_STACK_BUILD_ID);
38}
39
40static inline int stack_map_data_size(struct bpf_map *map)
41{
42 return stack_map_use_build_id(map) ?
43 sizeof(struct bpf_stack_build_id) : sizeof(u64);
44}
45
46/**
47 * stack_map_calculate_max_depth - Calculate maximum allowed stack trace depth
48 * @size: Size of the buffer/map value in bytes
49 * @elem_size: Size of each stack trace element
50 * @flags: BPF stack trace flags (BPF_F_USER_STACK, BPF_F_USER_BUILD_ID, ...)
51 *
52 * Return: Maximum number of stack trace entries that can be safely stored
53 */
54static u32 stack_map_calculate_max_depth(u32 size, u32 elem_size, u64 flags)
55{
56 u32 skip = flags & BPF_F_SKIP_FIELD_MASK;
57 u32 max_depth;
58 u32 curr_sysctl_max_stack = READ_ONCE(sysctl_perf_event_max_stack);
59
60 max_depth = size / elem_size;
61 max_depth += skip;
62 if (max_depth > curr_sysctl_max_stack)
63 return curr_sysctl_max_stack;
64
65 return max_depth;
66}
67
68static int prealloc_elems_and_freelist(struct bpf_stack_map *smap)
69{
70 u64 elem_size = sizeof(struct stack_map_bucket) +
71 (u64)smap->map.value_size;
72 int err;
73
74 smap->elems = bpf_map_area_alloc(elem_size * smap->map.max_entries,
75 smap->map.numa_node);
76 if (!smap->elems)
77 return -ENOMEM;
78
79 err = pcpu_freelist_init(&smap->freelist);
80 if (err)
81 goto free_elems;
82
83 pcpu_freelist_populate(&smap->freelist, smap->elems, elem_size,
84 smap->map.max_entries);
85 return 0;
86
87free_elems:
88 bpf_map_area_free(smap->elems);
89 return err;
90}
91
92/* Called from syscall */
93static struct bpf_map *stack_map_alloc(union bpf_attr *attr)
94{
95 u32 value_size = attr->value_size;
96 struct bpf_stack_map *smap;
97 u64 cost, n_buckets;
98 int err;
99
100 if (attr->map_flags & ~STACK_CREATE_FLAG_MASK)
101 return ERR_PTR(-EINVAL);
102
103 /* check sanity of attributes */
104 if (attr->max_entries == 0 || attr->key_size != 4 ||
105 value_size < 8 || value_size % 8)
106 return ERR_PTR(-EINVAL);
107
108 BUILD_BUG_ON(sizeof(struct bpf_stack_build_id) % sizeof(u64));
109 if (attr->map_flags & BPF_F_STACK_BUILD_ID) {
110 if (value_size % sizeof(struct bpf_stack_build_id) ||
111 value_size / sizeof(struct bpf_stack_build_id)
112 > sysctl_perf_event_max_stack)
113 return ERR_PTR(-EINVAL);
114 } else if (value_size / 8 > sysctl_perf_event_max_stack)
115 return ERR_PTR(-EINVAL);
116
117 /* hash table size must be power of 2; roundup_pow_of_two() can overflow
118 * into UB on 32-bit arches, so check that first
119 */
120 if (attr->max_entries > 1UL << 31)
121 return ERR_PTR(-E2BIG);
122
123 n_buckets = roundup_pow_of_two(attr->max_entries);
124
125 cost = n_buckets * sizeof(struct stack_map_bucket *) + sizeof(*smap);
126 smap = bpf_map_area_alloc(cost, bpf_map_attr_numa_node(attr));
127 if (!smap)
128 return ERR_PTR(-ENOMEM);
129
130 bpf_map_init_from_attr(&smap->map, attr);
131 smap->n_buckets = n_buckets;
132
133 err = get_callchain_buffers(sysctl_perf_event_max_stack);
134 if (err)
135 goto free_smap;
136
137 err = prealloc_elems_and_freelist(smap);
138 if (err)
139 goto put_buffers;
140
141 return &smap->map;
142
143put_buffers:
144 put_callchain_buffers();
145free_smap:
146 bpf_map_area_free(smap);
147 return ERR_PTR(err);
148}
149
150static int fetch_build_id(struct vm_area_struct *vma, unsigned char *build_id, bool may_fault)
151{
152 return may_fault ? build_id_parse(vma, build_id, NULL)
153 : build_id_parse_nofault(vma, build_id, NULL);
154}
155
156static inline void stack_map_build_id_set_ip(struct bpf_stack_build_id *id)
157{
158 id->status = BPF_STACK_BUILD_ID_IP;
159 memset(id->build_id, 0, BUILD_ID_SIZE_MAX);
160}
161
162static inline u64 stack_map_build_id_offset(unsigned long vm_pgoff,
163 unsigned long vm_start, u64 ip)
164{
165 return (vm_pgoff << PAGE_SHIFT) + ip - vm_start;
166}
167
168static inline void stack_map_build_id_set_valid(struct bpf_stack_build_id *id,
169 u64 offset,
170 const unsigned char *build_id)
171{
172 id->status = BPF_STACK_BUILD_ID_VALID;
173 id->offset = offset;
174 if (id->build_id != build_id)
175 memcpy(id->build_id, build_id, BUILD_ID_SIZE_MAX);
176}
177
178/*
179 * A cached VMA lookup result. The range [vm_start, vm_end) is always set.
180 * vm_pgoff, file, build_id are set only when the build ID was resolved.
181 * Zero vm_end marks the slot empty. build_id aliases the id_offs[] entry.
182 */
183struct stack_map_cached_vma {
184 unsigned long vm_start;
185 unsigned long vm_end;
186 unsigned long vm_pgoff;
187 struct file *file; /* pinned in the sleepable path; NULL otherwise */
188 const unsigned char *build_id;
189};
190
191/*
192 * Per stack_map_get_build_id_offset() call cache of the last VMA with a build ID
193 * resolved and the last VMA with no usable build ID. Adjacent stack frames tend
194 * to land in the same VMA or the same backing file, so caching the last result
195 * of each kind lets us skip unnecessary VMA lookups and build ID parse calls.
196 * Keeping the two slots independent means a build-ID-less VMA doesn't evict the
197 * last resolved build ID.
198 */
199struct stack_map_build_id_cache {
200 struct stack_map_cached_vma resolved;
201 struct stack_map_cached_vma unresolved;
202};
203
204/*
205 * Fill @id from a cached range covering @ip. On a hit this writes @id (resolved
206 * range -> build ID + offset, unresolved range -> raw ip) and returns 0; on a
207 * miss it leaves @id untouched and returns -ENOENT.
208 */
209static int stack_map_build_id_set_from_cache(struct stack_map_build_id_cache *cache,
210 struct bpf_stack_build_id *id, u64 ip)
211{
212 unsigned long vm_start, vm_end, vm_pgoff;
213 u64 offset;
214
215 vm_start = cache->resolved.vm_start;
216 vm_end = cache->resolved.vm_end;
217 if (vm_end && ip >= vm_start && ip < vm_end) {
218 vm_pgoff = cache->resolved.vm_pgoff;
219 offset = stack_map_build_id_offset(vm_pgoff, vm_start, ip);
220 stack_map_build_id_set_valid(id, offset, cache->resolved.build_id);
221 return 0;
222 }
223
224 vm_start = cache->unresolved.vm_start;
225 vm_end = cache->unresolved.vm_end;
226 if (vm_end && ip >= vm_start && ip < vm_end) {
227 stack_map_build_id_set_ip(id);
228 return 0;
229 }
230
231 return -ENOENT;
232}
233
234/*
235 * Record @vma's build ID as the last resolved one. @file is the pinned backing
236 * file in the sleepable path (released when evicted), or NULL otherwise.
237 */
238static void stack_map_build_id_cache_set_resolved(struct stack_map_build_id_cache *cache,
239 struct file *file,
240 const unsigned char *build_id,
241 unsigned long vm_start,
242 unsigned long vm_end,
243 unsigned long vm_pgoff)
244{
245 if (cache->resolved.file)
246 fput(cache->resolved.file);
247 cache->resolved = (struct stack_map_cached_vma){
248 .vm_start = vm_start,
249 .vm_end = vm_end,
250 .vm_pgoff = vm_pgoff,
251 .file = file,
252 .build_id = build_id,
253 };
254}
255
256/* Record [vm_start, vm_end) as a range with no usable build ID. */
257static void stack_map_build_id_cache_set_unresolved(struct stack_map_build_id_cache *cache,
258 unsigned long vm_start,
259 unsigned long vm_end)
260{
261 cache->unresolved = (struct stack_map_cached_vma){
262 .vm_start = vm_start,
263 .vm_end = vm_end,
264 };
265}
266
267struct stack_map_vma_lock {
268 struct vm_area_struct *vma;
269 struct mm_struct *mm;
270};
271
272/*
273 * Acquire a stable read-side reference on the VMA covering @ip.
274 *
275 * With CONFIG_PER_VMA_LOCK=y this returns a VMA with its per-VMA read
276 * lock held and mmap_lock dropped, so the caller may sleep.
277 *
278 * With CONFIG_PER_VMA_LOCK=n it returns a VMA with mmap_lock still
279 * held; the caller must snapshot any fields it needs and pin vm_file
280 * with get_file() before stack_map_unlock_vma() drops mmap_lock, as
281 * the VMA may be split, merged, or freed after that.
282 *
283 * Returns NULL on failure, in which case no lock is held.
284 */
285static struct vm_area_struct *
286stack_map_lock_vma(struct stack_map_vma_lock *lock, unsigned long ip)
287{
288 struct mm_struct *mm = lock->mm;
289 struct vm_area_struct *vma;
290
291 /* noop under !CONFIG_PER_VMA_LOCK */
292 vma = lock_vma_under_rcu(mm, ip);
293 if (vma) {
294 lock->vma = vma;
295 return vma;
296 }
297
298 /*
299 * Taking mmap_read_lock() is unsafe here, because the caller BPF
300 * program might already hold it, causing a deadlock.
301 */
302 if (!mmap_read_trylock(mm))
303 return NULL;
304
305 vma = vma_lookup(mm, ip);
306 if (!vma) {
307 mmap_read_unlock(mm);
308 return NULL;
309 }
310
311#ifdef CONFIG_PER_VMA_LOCK
312 if (!vma_start_read_locked(vma)) {
313 mmap_read_unlock(mm);
314 return NULL;
315 }
316 mmap_read_unlock(mm);
317#endif
318
319 lock->vma = vma;
320 return vma;
321}
322
323static void stack_map_unlock_vma(struct stack_map_vma_lock *lock)
324{
325#ifdef CONFIG_PER_VMA_LOCK
326 vma_end_read(lock->vma);
327#else
328 mmap_read_unlock(lock->mm);
329#endif
330 lock->vma = NULL;
331}
332
333static void stack_map_get_build_id_offset_sleepable(struct bpf_stack_build_id *id_offs,
334 u32 trace_nr)
335{
336 struct stack_map_vma_lock lock = { .mm = current->mm };
337 struct stack_map_build_id_cache cache = {};
338 struct stack_map_cached_vma *res = &cache.resolved;
339 unsigned long vm_pgoff, vm_start, vm_end;
340 struct vm_area_struct *vma;
341 struct file *file;
342 u64 offset;
343 u64 ip;
344
345 for (u32 i = 0; i < trace_nr; i++) {
346 ip = READ_ONCE(id_offs[i].ip);
347
348 if (!stack_map_build_id_set_from_cache(&cache, &id_offs[i], ip))
349 continue;
350
351 vma = stack_map_lock_vma(&lock, ip);
352 if (!vma) {
353 stack_map_build_id_set_ip(&id_offs[i]);
354 continue;
355 }
356
357 vm_pgoff = vma->vm_pgoff;
358 vm_start = vma->vm_start;
359 vm_end = vma->vm_end;
360
361 if (vma_is_anonymous(vma) || !vma->vm_file) {
362 stack_map_unlock_vma(&lock);
363 stack_map_build_id_set_ip(&id_offs[i]);
364 stack_map_build_id_cache_set_unresolved(&cache, vm_start, vm_end);
365 continue;
366 }
367
368 file = vma->vm_file;
369 offset = stack_map_build_id_offset(vm_pgoff, vm_start, ip);
370
371 /*
372 * Same backing file as the last resolved VMA (another mapping
373 * of the same ELF binary): reuse its build_id without re-parsing.
374 */
375 if (file == res->file) {
376 stack_map_unlock_vma(&lock);
377 stack_map_build_id_set_valid(&id_offs[i], offset, res->build_id);
378 res->vm_start = vm_start;
379 res->vm_end = vm_end;
380 res->vm_pgoff = vm_pgoff;
381 continue;
382 }
383
384 file = get_file(file);
385 stack_map_unlock_vma(&lock);
386
387 /* build_id_parse_file() may block on filesystem reads */
388 if (build_id_parse_file(file, id_offs[i].build_id, NULL)) {
389 stack_map_build_id_set_ip(&id_offs[i]);
390 fput(file);
391 stack_map_build_id_cache_set_unresolved(&cache, vm_start, vm_end);
392 continue;
393 }
394
395 stack_map_build_id_set_valid(&id_offs[i], offset, id_offs[i].build_id);
396 stack_map_build_id_cache_set_resolved(&cache, file, id_offs[i].build_id,
397 vm_start, vm_end, vm_pgoff);
398 }
399
400 if (res->file)
401 fput(res->file);
402}
403
404/*
405 * Expects all id_offs[i].ip values to be set to correct initial IPs.
406 * They will be subsequently:
407 * - either adjusted in place to a file offset, if build ID fetching
408 * succeeds; in this case id_offs[i].build_id is set to correct build ID,
409 * and id_offs[i].status is set to BPF_STACK_BUILD_ID_VALID;
410 * - or IP will be kept intact, if build ID fetching failed; in this case
411 * id_offs[i].build_id is zeroed out and id_offs[i].status is set to
412 * BPF_STACK_BUILD_ID_IP.
413 */
414static void stack_map_get_build_id_offset(struct bpf_stack_build_id *id_offs,
415 u32 trace_nr, bool user, bool may_fault)
416{
417 struct mmap_unlock_irq_work *work = NULL;
418 bool irq_work_busy = bpf_mmap_unlock_get_irq_work(&work);
419 bool has_user_ctx = user && current && current->mm;
420 struct stack_map_build_id_cache cache = {};
421 struct vm_area_struct *vma;
422 int i;
423
424 if (may_fault && has_user_ctx) {
425 stack_map_get_build_id_offset_sleepable(id_offs, trace_nr);
426 return;
427 }
428
429 /* If the irq_work is in use, fall back to report ips. Same
430 * fallback is used for kernel stack (!user) on a stackmap with
431 * build_id.
432 */
433 if (!has_user_ctx || irq_work_busy || !mmap_read_trylock(current->mm)) {
434 /* cannot access current->mm, fall back to ips */
435 for (i = 0; i < trace_nr; i++)
436 stack_map_build_id_set_ip(&id_offs[i]);
437 return;
438 }
439
440 for (i = 0; i < trace_nr; i++) {
441 u64 ip = READ_ONCE(id_offs[i].ip);
442
443 if (!stack_map_build_id_set_from_cache(&cache, &id_offs[i], ip))
444 continue;
445
446 vma = find_vma(current->mm, ip);
447 if (!vma || vma_is_anonymous(vma) ||
448 fetch_build_id(vma, id_offs[i].build_id, may_fault)) {
449 /* per entry fall back to ips; cache build-ID-less range */
450 stack_map_build_id_set_ip(&id_offs[i]);
451 if (vma)
452 stack_map_build_id_cache_set_unresolved(&cache,
453 vma->vm_start, vma->vm_end);
454 continue;
455 }
456 /*
457 * mmap_lock is held for the whole loop, so the cached VMA
458 * fields stay valid; no file pinning is needed here.
459 */
460 stack_map_build_id_set_valid(&id_offs[i],
461 stack_map_build_id_offset(vma->vm_pgoff, vma->vm_start, ip),
462 id_offs[i].build_id);
463 stack_map_build_id_cache_set_resolved(&cache, NULL, id_offs[i].build_id,
464 vma->vm_start, vma->vm_end,
465 vma->vm_pgoff);
466 }
467 bpf_mmap_unlock_mm(work, current->mm);
468}
469
470static struct perf_callchain_entry *
471get_callchain_entry_for_task(struct task_struct *task, u32 max_depth)
472{
473#ifdef CONFIG_STACKTRACE
474 struct perf_callchain_entry *entry;
475 int rctx;
476
477 entry = get_callchain_entry(&rctx);
478
479 if (!entry)
480 return NULL;
481
482 entry->nr = stack_trace_save_tsk(task, (unsigned long *)entry->ip,
483 max_depth, 0);
484
485 /* stack_trace_save_tsk() works on unsigned long array, while
486 * perf_callchain_entry uses u64 array. For 32-bit systems, it is
487 * necessary to fix this mismatch.
488 */
489 if (__BITS_PER_LONG != 64) {
490 unsigned long *from = (unsigned long *) entry->ip;
491 u64 *to = entry->ip;
492 int i;
493
494 /* copy data from the end to avoid using extra buffer */
495 for (i = entry->nr - 1; i >= 0; i--)
496 to[i] = (u64)(from[i]);
497 }
498
499 put_callchain_entry(rctx);
500
501 return entry;
502#else /* CONFIG_STACKTRACE */
503 return NULL;
504#endif
505}
506
507static long __bpf_get_stackid(struct bpf_map *map,
508 struct perf_callchain_entry *trace, u64 flags)
509{
510 struct bpf_stack_map *smap = container_of(map, struct bpf_stack_map, map);
511 struct stack_map_bucket *bucket, *new_bucket, *old_bucket;
512 u32 hash, id, trace_nr, trace_len, i, max_depth;
513 u32 skip = flags & BPF_F_SKIP_FIELD_MASK;
514 bool user = flags & BPF_F_USER_STACK;
515 u64 *ips;
516 bool hash_matches;
517
518 if (trace->nr <= skip)
519 /* skipping more than usable stack trace */
520 return -EFAULT;
521
522 max_depth = stack_map_calculate_max_depth(map->value_size, stack_map_data_size(map), flags);
523 trace_nr = min_t(u32, trace->nr - skip, max_depth - skip);
524 trace_len = trace_nr * sizeof(u64);
525 ips = trace->ip + skip;
526 hash = jhash2((u32 *)ips, trace_len / sizeof(u32), 0);
527 id = hash & (smap->n_buckets - 1);
528 bucket = READ_ONCE(smap->buckets[id]);
529
530 hash_matches = bucket && bucket->hash == hash;
531 /* fast cmp */
532 if (hash_matches && flags & BPF_F_FAST_STACK_CMP)
533 return id;
534
535 if (stack_map_use_build_id(map)) {
536 struct bpf_stack_build_id *id_offs;
537
538 /* for build_id+offset, pop a bucket before slow cmp */
539 new_bucket = (struct stack_map_bucket *)
540 pcpu_freelist_pop(&smap->freelist);
541 if (unlikely(!new_bucket))
542 return -ENOMEM;
543 new_bucket->nr = trace_nr;
544 id_offs = (struct bpf_stack_build_id *)new_bucket->data;
545 for (i = 0; i < trace_nr; i++)
546 id_offs[i].ip = ips[i];
547 stack_map_get_build_id_offset(id_offs, trace_nr, user, false /* !may_fault */);
548 trace_len = trace_nr * sizeof(struct bpf_stack_build_id);
549 if (hash_matches && bucket->nr == trace_nr &&
550 memcmp(bucket->data, new_bucket->data, trace_len) == 0) {
551 pcpu_freelist_push(&smap->freelist, &new_bucket->fnode);
552 return id;
553 }
554 if (bucket && !(flags & BPF_F_REUSE_STACKID)) {
555 pcpu_freelist_push(&smap->freelist, &new_bucket->fnode);
556 return -EEXIST;
557 }
558 } else {
559 if (hash_matches && bucket->nr == trace_nr &&
560 memcmp(bucket->data, ips, trace_len) == 0)
561 return id;
562 if (bucket && !(flags & BPF_F_REUSE_STACKID))
563 return -EEXIST;
564
565 new_bucket = (struct stack_map_bucket *)
566 pcpu_freelist_pop(&smap->freelist);
567 if (unlikely(!new_bucket))
568 return -ENOMEM;
569 memcpy(new_bucket->data, ips, trace_len);
570 }
571
572 new_bucket->hash = hash;
573 new_bucket->nr = trace_nr;
574
575 old_bucket = xchg(&smap->buckets[id], new_bucket);
576 if (old_bucket)
577 pcpu_freelist_push(&smap->freelist, &old_bucket->fnode);
578 return id;
579}
580
581BPF_CALL_3(bpf_get_stackid, struct pt_regs *, regs, struct bpf_map *, map,
582 u64, flags)
583{
584 u32 elem_size = stack_map_data_size(map);
585 bool user = flags & BPF_F_USER_STACK;
586 struct perf_callchain_entry *trace;
587 bool kernel = !user;
588 u32 max_depth;
589
590 if (unlikely(flags & ~(BPF_F_SKIP_FIELD_MASK | BPF_F_USER_STACK |
591 BPF_F_FAST_STACK_CMP | BPF_F_REUSE_STACKID)))
592 return -EINVAL;
593
594 max_depth = stack_map_calculate_max_depth(map->value_size, elem_size, flags);
595 trace = get_perf_callchain(regs, kernel, user, max_depth,
596 false, false, 0);
597
598 if (unlikely(!trace))
599 /* couldn't fetch the stack trace */
600 return -EFAULT;
601
602 return __bpf_get_stackid(map, trace, flags);
603}
604
605const struct bpf_func_proto bpf_get_stackid_proto = {
606 .func = bpf_get_stackid,
607 .gpl_only = true,
608 .ret_type = RET_INTEGER,
609 .arg1_type = ARG_PTR_TO_CTX,
610 .arg2_type = ARG_CONST_MAP_PTR,
611 .arg3_type = ARG_ANYTHING,
612};
613
614static __u64 count_kernel_ip(struct perf_callchain_entry *trace)
615{
616 __u64 nr_kernel = 0;
617
618 while (nr_kernel < trace->nr) {
619 if (trace->ip[nr_kernel] == PERF_CONTEXT_USER)
620 break;
621 nr_kernel++;
622 }
623 return nr_kernel;
624}
625
626BPF_CALL_3(bpf_get_stackid_pe, struct bpf_perf_event_data_kern *, ctx,
627 struct bpf_map *, map, u64, flags)
628{
629 struct perf_event *event = ctx->event;
630 struct perf_callchain_entry *trace;
631 bool kernel, user;
632 __u64 nr_kernel;
633 int ret;
634
635 /* perf_sample_data doesn't have callchain, use bpf_get_stackid */
636 if (!(event->attr.sample_type & PERF_SAMPLE_CALLCHAIN))
637 return bpf_get_stackid((unsigned long)(ctx->regs),
638 (unsigned long) map, flags, 0, 0);
639
640 if (unlikely(flags & ~(BPF_F_SKIP_FIELD_MASK | BPF_F_USER_STACK |
641 BPF_F_FAST_STACK_CMP | BPF_F_REUSE_STACKID)))
642 return -EINVAL;
643
644 user = flags & BPF_F_USER_STACK;
645 kernel = !user;
646
647 trace = ctx->data->callchain;
648 if (unlikely(!trace))
649 return -EFAULT;
650
651 nr_kernel = count_kernel_ip(trace);
652 __u64 nr = trace->nr; /* save original */
653
654 if (kernel) {
655 trace->nr = nr_kernel;
656 ret = __bpf_get_stackid(map, trace, flags);
657 } else { /* user */
658 u64 skip = flags & BPF_F_SKIP_FIELD_MASK;
659
660 skip += nr_kernel;
661 if (skip > BPF_F_SKIP_FIELD_MASK)
662 return -EFAULT;
663
664 flags = (flags & ~BPF_F_SKIP_FIELD_MASK) | skip;
665 ret = __bpf_get_stackid(map, trace, flags);
666 }
667
668 /* restore nr */
669 trace->nr = nr;
670
671 return ret;
672}
673
674const struct bpf_func_proto bpf_get_stackid_proto_pe = {
675 .func = bpf_get_stackid_pe,
676 .gpl_only = false,
677 .ret_type = RET_INTEGER,
678 .arg1_type = ARG_PTR_TO_CTX,
679 .arg2_type = ARG_CONST_MAP_PTR,
680 .arg3_type = ARG_ANYTHING,
681};
682
683static long __bpf_get_stack(struct pt_regs *regs, struct task_struct *task,
684 struct perf_callchain_entry *trace_in,
685 void *buf, u32 size, u64 flags, bool may_fault)
686{
687 u32 trace_nr, copy_len, elem_size, max_depth;
688 bool user_build_id = flags & BPF_F_USER_BUILD_ID;
689 bool crosstask = task && task != current;
690 u32 skip = flags & BPF_F_SKIP_FIELD_MASK;
691 bool user = flags & BPF_F_USER_STACK;
692 struct perf_callchain_entry *trace;
693 bool kernel = !user;
694 int err = -EINVAL;
695 u64 *ips;
696
697 if (unlikely(flags & ~(BPF_F_SKIP_FIELD_MASK | BPF_F_USER_STACK |
698 BPF_F_USER_BUILD_ID)))
699 goto clear;
700 if (kernel && user_build_id)
701 goto clear;
702
703 elem_size = user_build_id ? sizeof(struct bpf_stack_build_id) : sizeof(u64);
704 if (unlikely(size % elem_size))
705 goto clear;
706
707 /* cannot get valid user stack for task without user_mode regs */
708 if (task && user && !user_mode(regs))
709 goto err_fault;
710
711 /* get_perf_callchain does not support crosstask user stack walking
712 * but returns an empty stack instead of NULL.
713 */
714 if (crosstask && user) {
715 err = -EOPNOTSUPP;
716 goto clear;
717 }
718
719 max_depth = stack_map_calculate_max_depth(size, elem_size, flags);
720
721 if (may_fault)
722 rcu_read_lock(); /* need RCU for perf's callchain below */
723
724 if (trace_in) {
725 trace = trace_in;
726 trace->nr = min_t(u32, trace->nr, max_depth);
727 } else if (kernel && task) {
728 trace = get_callchain_entry_for_task(task, max_depth);
729 } else {
730 trace = get_perf_callchain(regs, kernel, user, max_depth,
731 crosstask, false, 0);
732 }
733
734 if (unlikely(!trace) || trace->nr < skip) {
735 if (may_fault)
736 rcu_read_unlock();
737 goto err_fault;
738 }
739
740 trace_nr = trace->nr - skip;
741 copy_len = trace_nr * elem_size;
742
743 ips = trace->ip + skip;
744 if (user_build_id) {
745 struct bpf_stack_build_id *id_offs = buf;
746 u32 i;
747
748 for (i = 0; i < trace_nr; i++)
749 id_offs[i].ip = ips[i];
750 } else {
751 memcpy(buf, ips, copy_len);
752 }
753
754 /* trace/ips should not be dereferenced after this point */
755 if (may_fault)
756 rcu_read_unlock();
757
758 if (user_build_id)
759 stack_map_get_build_id_offset(buf, trace_nr, user, may_fault);
760
761 if (size > copy_len)
762 memset(buf + copy_len, 0, size - copy_len);
763 return copy_len;
764
765err_fault:
766 err = -EFAULT;
767clear:
768 memset(buf, 0, size);
769 return err;
770}
771
772BPF_CALL_4(bpf_get_stack, struct pt_regs *, regs, void *, buf, u32, size,
773 u64, flags)
774{
775 return __bpf_get_stack(regs, NULL, NULL, buf, size, flags, false /* !may_fault */);
776}
777
778const struct bpf_func_proto bpf_get_stack_proto = {
779 .func = bpf_get_stack,
780 .gpl_only = true,
781 .ret_type = RET_INTEGER,
782 .arg1_type = ARG_PTR_TO_CTX,
783 .arg2_type = ARG_PTR_TO_UNINIT_MEM,
784 .arg3_type = ARG_CONST_SIZE_OR_ZERO,
785 .arg4_type = ARG_ANYTHING,
786};
787
788BPF_CALL_4(bpf_get_stack_sleepable, struct pt_regs *, regs, void *, buf, u32, size,
789 u64, flags)
790{
791 return __bpf_get_stack(regs, NULL, NULL, buf, size, flags, true /* may_fault */);
792}
793
794const struct bpf_func_proto bpf_get_stack_sleepable_proto = {
795 .func = bpf_get_stack_sleepable,
796 .gpl_only = true,
797 .ret_type = RET_INTEGER,
798 .arg1_type = ARG_PTR_TO_CTX,
799 .arg2_type = ARG_PTR_TO_UNINIT_MEM,
800 .arg3_type = ARG_CONST_SIZE_OR_ZERO,
801 .arg4_type = ARG_ANYTHING,
802};
803
804static long __bpf_get_task_stack(struct task_struct *task, void *buf, u32 size,
805 u64 flags, bool may_fault)
806{
807 struct pt_regs *regs;
808 long res = -EINVAL;
809
810 if (!try_get_task_stack(task))
811 return -EFAULT;
812
813 regs = task_pt_regs(task);
814 if (regs)
815 res = __bpf_get_stack(regs, task, NULL, buf, size, flags, may_fault);
816 put_task_stack(task);
817
818 return res;
819}
820
821BPF_CALL_4(bpf_get_task_stack, struct task_struct *, task, void *, buf,
822 u32, size, u64, flags)
823{
824 return __bpf_get_task_stack(task, buf, size, flags, false /* !may_fault */);
825}
826
827const struct bpf_func_proto bpf_get_task_stack_proto = {
828 .func = bpf_get_task_stack,
829 .gpl_only = false,
830 .ret_type = RET_INTEGER,
831 .arg1_type = ARG_PTR_TO_BTF_ID,
832 .arg1_btf_id = &btf_tracing_ids[BTF_TRACING_TYPE_TASK],
833 .arg2_type = ARG_PTR_TO_UNINIT_MEM,
834 .arg3_type = ARG_CONST_SIZE_OR_ZERO,
835 .arg4_type = ARG_ANYTHING,
836};
837
838BPF_CALL_4(bpf_get_task_stack_sleepable, struct task_struct *, task, void *, buf,
839 u32, size, u64, flags)
840{
841 return __bpf_get_task_stack(task, buf, size, flags, true /* !may_fault */);
842}
843
844const struct bpf_func_proto bpf_get_task_stack_sleepable_proto = {
845 .func = bpf_get_task_stack_sleepable,
846 .gpl_only = false,
847 .ret_type = RET_INTEGER,
848 .arg1_type = ARG_PTR_TO_BTF_ID,
849 .arg1_btf_id = &btf_tracing_ids[BTF_TRACING_TYPE_TASK],
850 .arg2_type = ARG_PTR_TO_UNINIT_MEM,
851 .arg3_type = ARG_CONST_SIZE_OR_ZERO,
852 .arg4_type = ARG_ANYTHING,
853};
854
855BPF_CALL_4(bpf_get_stack_pe, struct bpf_perf_event_data_kern *, ctx,
856 void *, buf, u32, size, u64, flags)
857{
858 struct pt_regs *regs = (struct pt_regs *)(ctx->regs);
859 struct perf_event *event = ctx->event;
860 struct perf_callchain_entry *trace;
861 bool kernel, user;
862 int err = -EINVAL;
863 __u64 nr_kernel;
864
865 if (!(event->attr.sample_type & PERF_SAMPLE_CALLCHAIN))
866 return __bpf_get_stack(regs, NULL, NULL, buf, size, flags, false /* !may_fault */);
867
868 if (unlikely(flags & ~(BPF_F_SKIP_FIELD_MASK | BPF_F_USER_STACK |
869 BPF_F_USER_BUILD_ID)))
870 goto clear;
871
872 user = flags & BPF_F_USER_STACK;
873 kernel = !user;
874
875 err = -EFAULT;
876 trace = ctx->data->callchain;
877 if (unlikely(!trace))
878 goto clear;
879
880 nr_kernel = count_kernel_ip(trace);
881
882 if (kernel) {
883 __u64 nr = trace->nr;
884
885 trace->nr = nr_kernel;
886 err = __bpf_get_stack(regs, NULL, trace, buf, size, flags, false /* !may_fault */);
887
888 /* restore nr */
889 trace->nr = nr;
890 } else { /* user */
891 u64 skip = flags & BPF_F_SKIP_FIELD_MASK;
892
893 skip += nr_kernel;
894 if (skip > BPF_F_SKIP_FIELD_MASK)
895 goto clear;
896
897 flags = (flags & ~BPF_F_SKIP_FIELD_MASK) | skip;
898 err = __bpf_get_stack(regs, NULL, trace, buf, size, flags, false /* !may_fault */);
899 }
900 return err;
901
902clear:
903 memset(buf, 0, size);
904 return err;
905
906}
907
908const struct bpf_func_proto bpf_get_stack_proto_pe = {
909 .func = bpf_get_stack_pe,
910 .gpl_only = true,
911 .ret_type = RET_INTEGER,
912 .arg1_type = ARG_PTR_TO_CTX,
913 .arg2_type = ARG_PTR_TO_UNINIT_MEM,
914 .arg3_type = ARG_CONST_SIZE_OR_ZERO,
915 .arg4_type = ARG_ANYTHING,
916};
917
918/* Called from eBPF program */
919static void *stack_map_lookup_elem(struct bpf_map *map, void *key)
920{
921 return ERR_PTR(-EOPNOTSUPP);
922}
923
924/* Called from syscall */
925static int stack_map_lookup_and_delete_elem(struct bpf_map *map, void *key,
926 void *value, u64 flags)
927{
928 return bpf_stackmap_extract(map, key, value, true);
929}
930
931/* Called from syscall */
932int bpf_stackmap_extract(struct bpf_map *map, void *key, void *value,
933 bool delete)
934{
935 struct bpf_stack_map *smap = container_of(map, struct bpf_stack_map, map);
936 struct stack_map_bucket *bucket, *old_bucket;
937 u32 id = *(u32 *)key, trace_len;
938
939 if (unlikely(id >= smap->n_buckets))
940 return -ENOENT;
941
942 bucket = xchg(&smap->buckets[id], NULL);
943 if (!bucket)
944 return -ENOENT;
945
946 trace_len = bucket->nr * stack_map_data_size(map);
947 memcpy(value, bucket->data, trace_len);
948 memset(value + trace_len, 0, map->value_size - trace_len);
949
950 if (delete)
951 old_bucket = bucket;
952 else
953 old_bucket = xchg(&smap->buckets[id], bucket);
954 if (old_bucket)
955 pcpu_freelist_push(&smap->freelist, &old_bucket->fnode);
956 return 0;
957}
958
959static int stack_map_get_next_key(struct bpf_map *map, void *key,
960 void *next_key)
961{
962 struct bpf_stack_map *smap = container_of(map,
963 struct bpf_stack_map, map);
964 u32 id;
965
966 WARN_ON_ONCE(!rcu_read_lock_held());
967
968 if (!key) {
969 id = 0;
970 } else {
971 id = *(u32 *)key;
972 if (id >= smap->n_buckets || !smap->buckets[id])
973 id = 0;
974 else
975 id++;
976 }
977
978 while (id < smap->n_buckets && !smap->buckets[id])
979 id++;
980
981 if (id >= smap->n_buckets)
982 return -ENOENT;
983
984 *(u32 *)next_key = id;
985 return 0;
986}
987
988static long stack_map_update_elem(struct bpf_map *map, void *key, void *value,
989 u64 map_flags)
990{
991 return -EINVAL;
992}
993
994/* Called from syscall or from eBPF program */
995static long stack_map_delete_elem(struct bpf_map *map, void *key)
996{
997 struct bpf_stack_map *smap = container_of(map, struct bpf_stack_map, map);
998 struct stack_map_bucket *old_bucket;
999 u32 id = *(u32 *)key;
1000
1001 if (unlikely(id >= smap->n_buckets))
1002 return -E2BIG;
1003
1004 old_bucket = xchg(&smap->buckets[id], NULL);
1005 if (old_bucket) {
1006 pcpu_freelist_push(&smap->freelist, &old_bucket->fnode);
1007 return 0;
1008 } else {
1009 return -ENOENT;
1010 }
1011}
1012
1013/* Called when map->refcnt goes to zero, either from workqueue or from syscall */
1014static void stack_map_free(struct bpf_map *map)
1015{
1016 struct bpf_stack_map *smap = container_of(map, struct bpf_stack_map, map);
1017
1018 bpf_map_area_free(smap->elems);
1019 pcpu_freelist_destroy(&smap->freelist);
1020 bpf_map_area_free(smap);
1021 put_callchain_buffers();
1022}
1023
1024static u64 stack_map_mem_usage(const struct bpf_map *map)
1025{
1026 struct bpf_stack_map *smap = container_of(map, struct bpf_stack_map, map);
1027 u64 value_size = map->value_size;
1028 u64 n_buckets = smap->n_buckets;
1029 u64 enties = map->max_entries;
1030 u64 usage = sizeof(*smap);
1031
1032 usage += n_buckets * sizeof(struct stack_map_bucket *);
1033 usage += enties * (sizeof(struct stack_map_bucket) + value_size);
1034 return usage;
1035}
1036
1037BTF_ID_LIST_SINGLE(stack_trace_map_btf_ids, struct, bpf_stack_map)
1038const struct bpf_map_ops stack_trace_map_ops = {
1039 .map_meta_equal = bpf_map_meta_equal,
1040 .map_alloc = stack_map_alloc,
1041 .map_free = stack_map_free,
1042 .map_get_next_key = stack_map_get_next_key,
1043 .map_lookup_elem = stack_map_lookup_elem,
1044 .map_lookup_and_delete_elem = stack_map_lookup_and_delete_elem,
1045 .map_update_elem = stack_map_update_elem,
1046 .map_delete_elem = stack_map_delete_elem,
1047 .map_check_btf = map_check_no_btf,
1048 .map_mem_usage = stack_map_mem_usage,
1049 .map_btf_id = &stack_trace_map_btf_ids[0],
1050};