projects/linux·blob master · refs

1714 lines · 39.4 KiB

1// SPDX-License-Identifier: GPL-2.0
2/*
3 * uprobes-based tracing events
4 *
5 * Copyright (C) IBM Corporation, 2010-2012
6 * Author: Srikar Dronamraju <[email protected]>
7 */
8#define pr_fmt(fmt) "trace_uprobe: " fmt
9
10#include <linux/bpf-cgroup.h>
11#include <linux/cleanup.h>
12#include <linux/ctype.h>
13#include <linux/filter.h>
14#include <linux/module.h>
15#include <linux/namei.h>
16#include <linux/percpu.h>
17#include <linux/rculist.h>
18#include <linux/security.h>
19#include <linux/string.h>
20#include <linux/uaccess.h>
21#include <linux/uprobes.h>
22
23#include "trace.h"
24#include "trace_dynevent.h"
25#include "trace_probe.h"
26#include "trace_probe_tmpl.h"
27
28#define UPROBE_EVENT_SYSTEM "uprobes"
29
30struct uprobe_trace_entry_head {
31 struct trace_entry ent;
32 unsigned long vaddr[];
33};
34
35#define SIZEOF_TRACE_ENTRY(is_return) \
36 (sizeof(struct uprobe_trace_entry_head) + \
37 sizeof(unsigned long) * (is_return ? 2 : 1))
38
39#define DATAOF_TRACE_ENTRY(entry, is_return) \
40 ((void*)(entry) + SIZEOF_TRACE_ENTRY(is_return))
41
42static int trace_uprobe_create(const char *raw_command);
43static int trace_uprobe_show(struct seq_file *m, struct dyn_event *ev);
44static int trace_uprobe_release(struct dyn_event *ev);
45static bool trace_uprobe_is_busy(struct dyn_event *ev);
46static bool trace_uprobe_match(const char *system, const char *event,
47 int argc, const char **argv, struct dyn_event *ev);
48
49static struct dyn_event_operations trace_uprobe_ops = {
50 .create = trace_uprobe_create,
51 .show = trace_uprobe_show,
52 .is_busy = trace_uprobe_is_busy,
53 .free = trace_uprobe_release,
54 .match = trace_uprobe_match,
55};
56
57/*
58 * uprobe event core functions
59 */
60struct trace_uprobe {
61 struct dyn_event devent;
62 struct uprobe_consumer consumer;
63 struct path path;
64 char *filename;
65 struct uprobe *uprobe;
66 unsigned long offset;
67 unsigned long ref_ctr_offset;
68 unsigned long __percpu *nhits;
69 struct trace_probe tp;
70};
71
72static bool is_trace_uprobe(struct dyn_event *ev)
73{
74 return ev->ops == &trace_uprobe_ops;
75}
76
77static struct trace_uprobe *to_trace_uprobe(struct dyn_event *ev)
78{
79 return container_of(ev, struct trace_uprobe, devent);
80}
81
82/**
83 * for_each_trace_uprobe - iterate over the trace_uprobe list
84 * @pos: the struct trace_uprobe * for each entry
85 * @dpos: the struct dyn_event * to use as a loop cursor
86 */
87#define for_each_trace_uprobe(pos, dpos) \
88 for_each_dyn_event(dpos) \
89 if (is_trace_uprobe(dpos) && (pos = to_trace_uprobe(dpos)))
90
91static int register_uprobe_event(struct trace_uprobe *tu);
92static int unregister_uprobe_event(struct trace_uprobe *tu);
93
94static int uprobe_dispatcher(struct uprobe_consumer *con, struct pt_regs *regs,
95 __u64 *data);
96static int uretprobe_dispatcher(struct uprobe_consumer *con,
97 unsigned long func, struct pt_regs *regs,
98 __u64 *data);
99
100#ifdef CONFIG_STACK_GROWSUP
101static unsigned long adjust_stack_addr(unsigned long addr, unsigned int n)
102{
103 return addr - (n * sizeof(long));
104}
105#else
106static unsigned long adjust_stack_addr(unsigned long addr, unsigned int n)
107{
108 return addr + (n * sizeof(long));
109}
110#endif
111
112static unsigned long get_user_stack_nth(struct pt_regs *regs, unsigned int n)
113{
114 unsigned long ret;
115 unsigned long addr = user_stack_pointer(regs);
116
117 addr = adjust_stack_addr(addr, n);
118
119 if (copy_from_user(&ret, (void __force __user *) addr, sizeof(ret)))
120 return 0;
121
122 return ret;
123}
124
125/*
126 * Uprobes-specific fetch functions
127 */
128static nokprobe_inline int
129probe_mem_read(void *dest, void *src, size_t size)
130{
131 void __user *vaddr = (void __force __user *)src;
132
133 return copy_from_user(dest, vaddr, size) ? -EFAULT : 0;
134}
135
136static nokprobe_inline int
137probe_mem_read_user(void *dest, void *src, size_t size)
138{
139 return probe_mem_read(dest, src, size);
140}
141
142/*
143 * Fetch a null-terminated string. Caller MUST set *(u32 *)dest with max
144 * length and relative data location.
145 */
146static nokprobe_inline int
147fetch_store_string(unsigned long addr, void *dest, void *base)
148{
149 long ret;
150 u32 loc = *(u32 *)dest;
151 int maxlen = get_loc_len(loc);
152 u8 *dst = get_loc_data(dest, base);
153 void __user *src = (void __force __user *) addr;
154
155 if (unlikely(!maxlen))
156 return -ENOMEM;
157
158 if (addr == FETCH_TOKEN_COMM)
159 ret = strscpy(dst, current->comm, maxlen);
160 else
161 ret = strncpy_from_user(dst, src, maxlen);
162 if (ret >= 0) {
163 if (ret == maxlen)
164 dst[ret - 1] = '\0';
165 else
166 /*
167 * Include the terminating null byte. In this case it
168 * was copied by strncpy_from_user but not accounted
169 * for in ret.
170 */
171 ret++;
172 *(u32 *)dest = make_data_loc(ret, (void *)dst - base);
173 } else
174 *(u32 *)dest = make_data_loc(0, (void *)dst - base);
175
176 return ret;
177}
178
179static nokprobe_inline int
180fetch_store_string_user(unsigned long addr, void *dest, void *base)
181{
182 return fetch_store_string(addr, dest, base);
183}
184
185/* Return the length of string -- including null terminal byte */
186static nokprobe_inline int
187fetch_store_strlen(unsigned long addr)
188{
189 int len;
190 void __user *vaddr = (void __force __user *) addr;
191
192 if (addr == FETCH_TOKEN_COMM)
193 len = strlen(current->comm) + 1;
194 else
195 len = strnlen_user(vaddr, MAX_STRING_SIZE);
196
197 return (len > MAX_STRING_SIZE) ? 0 : len;
198}
199
200static nokprobe_inline int
201fetch_store_strlen_user(unsigned long addr)
202{
203 return fetch_store_strlen(addr);
204}
205
206static unsigned long translate_user_vaddr(unsigned long file_offset)
207{
208 unsigned long base_addr;
209 struct uprobe_dispatch_data *udd;
210
211 udd = (void *) current->utask->vaddr;
212
213 base_addr = udd->bp_addr - udd->tu->offset;
214 return base_addr + file_offset;
215}
216
217/* Note that we don't verify it, since the code does not come from user space */
218static int
219process_fetch_insn(struct fetch_insn *code, void *rec, void *edata,
220 void *dest, void *base)
221{
222 struct pt_regs *regs = rec;
223 unsigned long val;
224 int ret;
225
226 /* 1st stage: get value from context */
227 switch (code->op) {
228 case FETCH_OP_REG:
229 val = regs_get_register(regs, code->param);
230 break;
231 case FETCH_OP_STACK:
232 val = get_user_stack_nth(regs, code->param);
233 break;
234 case FETCH_OP_STACKP:
235 val = user_stack_pointer(regs);
236 break;
237 case FETCH_OP_RETVAL:
238 val = regs_return_value(regs);
239 break;
240 case FETCH_OP_COMM:
241 val = FETCH_TOKEN_COMM;
242 break;
243 case FETCH_OP_FOFFS:
244 val = translate_user_vaddr(code->immediate);
245 break;
246 default:
247 ret = process_common_fetch_insn(code, &val);
248 if (ret < 0)
249 return ret;
250 }
251 code++;
252
253 return process_fetch_insn_bottom(code, val, dest, base);
254}
255NOKPROBE_SYMBOL(process_fetch_insn)
256
257static inline void init_trace_uprobe_filter(struct trace_uprobe_filter *filter)
258{
259 rwlock_init(&filter->rwlock);
260 filter->nr_systemwide = 0;
261 INIT_LIST_HEAD(&filter->perf_events);
262}
263
264static inline bool uprobe_filter_is_empty(struct trace_uprobe_filter *filter)
265{
266 return !filter->nr_systemwide && list_empty(&filter->perf_events);
267}
268
269static inline bool is_ret_probe(struct trace_uprobe *tu)
270{
271 return tu->consumer.ret_handler != NULL;
272}
273
274static bool trace_uprobe_is_busy(struct dyn_event *ev)
275{
276 struct trace_uprobe *tu = to_trace_uprobe(ev);
277
278 return trace_probe_is_enabled(&tu->tp);
279}
280
281static bool trace_uprobe_match_command_head(struct trace_uprobe *tu,
282 int argc, const char **argv)
283{
284 char buf[64];
285 int len;
286
287 if (!argc)
288 return true;
289
290 len = strlen(tu->filename);
291 if (strncmp(tu->filename, argv[0], len) || argv[0][len] != ':')
292 return false;
293
294 if (tu->ref_ctr_offset == 0)
295 snprintf(buf, sizeof(buf), "0x%0*lx",
296 (int)(sizeof(void *) * 2), tu->offset);
297 else
298 snprintf(buf, sizeof(buf), "0x%0*lx(0x%lx)",
299 (int)(sizeof(void *) * 2), tu->offset,
300 tu->ref_ctr_offset);
301 if (strcmp(buf, &argv[0][len + 1]))
302 return false;
303
304 argc--; argv++;
305
306 return trace_probe_match_command_args(&tu->tp, argc, argv);
307}
308
309static bool trace_uprobe_match(const char *system, const char *event,
310 int argc, const char **argv, struct dyn_event *ev)
311{
312 struct trace_uprobe *tu = to_trace_uprobe(ev);
313
314 return (event[0] == '\0' ||
315 strcmp(trace_probe_name(&tu->tp), event) == 0) &&
316 (!system || strcmp(trace_probe_group_name(&tu->tp), system) == 0) &&
317 trace_uprobe_match_command_head(tu, argc, argv);
318}
319
320static nokprobe_inline struct trace_uprobe *
321trace_uprobe_primary_from_call(struct trace_event_call *call)
322{
323 struct trace_probe *tp;
324
325 tp = trace_probe_primary_from_call(call);
326 if (WARN_ON_ONCE(!tp))
327 return NULL;
328
329 return container_of(tp, struct trace_uprobe, tp);
330}
331
332/*
333 * Allocate new trace_uprobe and initialize it (including uprobes).
334 */
335static struct trace_uprobe *
336alloc_trace_uprobe(const char *group, const char *event, int nargs, bool is_ret)
337{
338 struct trace_uprobe *tu;
339 int ret;
340
341 tu = kzalloc_flex(*tu, tp.args, nargs);
342 if (!tu)
343 return ERR_PTR(-ENOMEM);
344
345 tu->nhits = alloc_percpu(unsigned long);
346 if (!tu->nhits) {
347 ret = -ENOMEM;
348 goto error;
349 }
350
351 ret = trace_probe_init(&tu->tp, event, group, true, nargs);
352 if (ret < 0)
353 goto error;
354
355 dyn_event_init(&tu->devent, &trace_uprobe_ops);
356 tu->consumer.handler = uprobe_dispatcher;
357 if (is_ret)
358 tu->consumer.ret_handler = uretprobe_dispatcher;
359 init_trace_uprobe_filter(tu->tp.event->filter);
360 return tu;
361
362error:
363 free_percpu(tu->nhits);
364 kfree(tu);
365
366 return ERR_PTR(ret);
367}
368
369static void free_trace_uprobe(struct trace_uprobe *tu)
370{
371 if (IS_ERR_OR_NULL(tu))
372 return;
373
374 path_put(&tu->path);
375 trace_probe_cleanup(&tu->tp);
376 kfree(tu->filename);
377 free_percpu(tu->nhits);
378 kfree(tu);
379}
380
381static struct trace_uprobe *find_probe_event(const char *event, const char *group)
382{
383 struct dyn_event *pos;
384 struct trace_uprobe *tu;
385
386 for_each_trace_uprobe(tu, pos)
387 if (strcmp(trace_probe_name(&tu->tp), event) == 0 &&
388 strcmp(trace_probe_group_name(&tu->tp), group) == 0)
389 return tu;
390
391 return NULL;
392}
393
394/* Unregister a trace_uprobe and probe_event */
395static int unregister_trace_uprobe(struct trace_uprobe *tu)
396{
397 int ret;
398
399 if (trace_probe_has_sibling(&tu->tp))
400 goto unreg;
401
402 /* If there's a reference to the dynamic event */
403 if (trace_event_dyn_busy(trace_probe_event_call(&tu->tp)))
404 return -EBUSY;
405
406 ret = unregister_uprobe_event(tu);
407 if (ret)
408 return ret;
409
410unreg:
411 dyn_event_remove(&tu->devent);
412 trace_probe_unlink(&tu->tp);
413 free_trace_uprobe(tu);
414 return 0;
415}
416
417static bool trace_uprobe_has_same_uprobe(struct trace_uprobe *orig,
418 struct trace_uprobe *comp)
419{
420 struct trace_probe_event *tpe = orig->tp.event;
421 struct inode *comp_inode = d_real_inode(comp->path.dentry);
422 int i;
423
424 list_for_each_entry(orig, &tpe->probes, tp.list) {
425 if (comp_inode != d_real_inode(orig->path.dentry) ||
426 comp->offset != orig->offset)
427 continue;
428
429 /*
430 * trace_probe_compare_arg_type() ensured that nr_args and
431 * each argument name and type are same. Let's compare comm.
432 */
433 for (i = 0; i < orig->tp.nr_args; i++) {
434 if (strcmp(orig->tp.args[i].comm,
435 comp->tp.args[i].comm))
436 break;
437 }
438
439 if (i == orig->tp.nr_args)
440 return true;
441 }
442
443 return false;
444}
445
446static int append_trace_uprobe(struct trace_uprobe *tu, struct trace_uprobe *to)
447{
448 int ret;
449
450 ret = trace_probe_compare_arg_type(&tu->tp, &to->tp);
451 if (ret) {
452 /* Note that argument starts index = 2 */
453 trace_probe_log_set_index(ret + 1);
454 trace_probe_log_err(0, DIFF_ARG_TYPE);
455 return -EEXIST;
456 }
457 if (trace_uprobe_has_same_uprobe(to, tu)) {
458 trace_probe_log_set_index(0);
459 trace_probe_log_err(0, SAME_PROBE);
460 return -EEXIST;
461 }
462
463 /* Append to existing event */
464 ret = trace_probe_append(&tu->tp, &to->tp);
465 if (!ret)
466 dyn_event_add(&tu->devent, trace_probe_event_call(&tu->tp));
467
468 return ret;
469}
470
471/*
472 * Uprobe with multiple reference counter is not allowed. i.e.
473 * If inode and offset matches, reference counter offset *must*
474 * match as well. Though, there is one exception: If user is
475 * replacing old trace_uprobe with new one(same group/event),
476 * then we allow same uprobe with new reference counter as far
477 * as the new one does not conflict with any other existing
478 * ones.
479 */
480static int validate_ref_ctr_offset(struct trace_uprobe *new)
481{
482 struct dyn_event *pos;
483 struct trace_uprobe *tmp;
484 struct inode *new_inode = d_real_inode(new->path.dentry);
485
486 for_each_trace_uprobe(tmp, pos) {
487 if (new_inode == d_real_inode(tmp->path.dentry) &&
488 new->offset == tmp->offset &&
489 new->ref_ctr_offset != tmp->ref_ctr_offset) {
490 pr_warn("Reference counter offset mismatch.");
491 return -EINVAL;
492 }
493 }
494 return 0;
495}
496
497/* Register a trace_uprobe and probe_event */
498static int register_trace_uprobe(struct trace_uprobe *tu)
499{
500 struct trace_uprobe *old_tu;
501 int ret;
502
503 guard(mutex)(&event_mutex);
504
505 ret = validate_ref_ctr_offset(tu);
506 if (ret)
507 return ret;
508
509 /* register as an event */
510 old_tu = find_probe_event(trace_probe_name(&tu->tp),
511 trace_probe_group_name(&tu->tp));
512 if (old_tu) {
513 if (is_ret_probe(tu) != is_ret_probe(old_tu)) {
514 trace_probe_log_set_index(0);
515 trace_probe_log_err(0, DIFF_PROBE_TYPE);
516 return -EEXIST;
517 }
518 return append_trace_uprobe(tu, old_tu);
519 }
520
521 ret = register_uprobe_event(tu);
522 if (ret) {
523 if (ret == -EEXIST) {
524 trace_probe_log_set_index(0);
525 trace_probe_log_err(0, EVENT_EXIST);
526 } else
527 pr_warn("Failed to register probe event(%d)\n", ret);
528 return ret;
529 }
530
531 dyn_event_add(&tu->devent, trace_probe_event_call(&tu->tp));
532
533 return ret;
534}
535
536DEFINE_FREE(free_trace_uprobe, struct trace_uprobe *, free_trace_uprobe(_T))
537
538/*
539 * Argument syntax:
540 * - Add uprobe: p|r[:[GRP/][EVENT]] PATH:OFFSET[%return][(REF)] [FETCHARGS]
541 */
542static int __trace_uprobe_create(int argc, const char **argv)
543{
544 struct traceprobe_parse_context *ctx __free(traceprobe_parse_context) = NULL;
545 struct trace_uprobe *tu __free(free_trace_uprobe) = NULL;
546 const char *trlog __free(trace_probe_log_clear) = NULL;
547 const char *event = NULL, *group = UPROBE_EVENT_SYSTEM;
548 struct path path __free(path_put) = {};
549 unsigned long offset, ref_ctr_offset;
550 char *filename __free(kfree) = NULL;
551 char *arg, *rctr, *rctr_end, *tmp;
552 char *gbuf __free(kfree) = NULL;
553 char *buf __free(kfree) = NULL;
554 enum probe_print_type ptype;
555 bool is_return = false;
556 int i, ret;
557
558 ref_ctr_offset = 0;
559
560 switch (argv[0][0]) {
561 case 'r':
562 is_return = true;
563 break;
564 case 'p':
565 break;
566 default:
567 return -ECANCELED;
568 }
569
570 if (argc < 2)
571 return -ECANCELED;
572
573 trlog = trace_probe_log_init("trace_uprobe", argc, argv);
574
575 if (argc - 2 > MAX_TRACE_ARGS) {
576 trace_probe_log_set_index(2);
577 trace_probe_log_err(0, TOO_MANY_ARGS);
578 return -E2BIG;
579 }
580
581 if (argv[0][1] == ':')
582 event = &argv[0][2];
583
584 if (!strchr(argv[1], '/'))
585 return -ECANCELED;
586
587 filename = kstrdup(argv[1], GFP_KERNEL);
588 if (!filename)
589 return -ENOMEM;
590
591 /* Find the last occurrence, in case the path contains ':' too. */
592 arg = strrchr(filename, ':');
593 if (!arg || !isdigit(arg[1]))
594 return -ECANCELED;
595
596 trace_probe_log_set_index(1); /* filename is the 2nd argument */
597
598 *arg++ = '\0';
599 ret = kern_path(filename, LOOKUP_FOLLOW, &path);
600 if (ret) {
601 trace_probe_log_err(0, FILE_NOT_FOUND);
602 return ret;
603 }
604 if (!d_is_reg(path.dentry)) {
605 trace_probe_log_err(0, NO_REGULAR_FILE);
606 return -EINVAL;
607 }
608
609 /* Parse reference counter offset if specified. */
610 rctr = strchr(arg, '(');
611 if (rctr) {
612 rctr_end = strchr(rctr, ')');
613 if (!rctr_end) {
614 rctr_end = rctr + strlen(rctr);
615 trace_probe_log_err(rctr_end - filename,
616 REFCNT_OPEN_BRACE);
617 return -EINVAL;
618 } else if (rctr_end[1] != '\0') {
619 trace_probe_log_err(rctr_end + 1 - filename,
620 BAD_REFCNT_SUFFIX);
621 return -EINVAL;
622 }
623
624 *rctr++ = '\0';
625 *rctr_end = '\0';
626 ret = kstrtoul(rctr, 0, &ref_ctr_offset);
627 if (ret) {
628 trace_probe_log_err(rctr - filename, BAD_REFCNT);
629 return ret;
630 }
631 }
632
633 /* Check if there is %return suffix */
634 tmp = strchr(arg, '%');
635 if (tmp) {
636 if (!strcmp(tmp, "%return")) {
637 *tmp = '\0';
638 is_return = true;
639 } else {
640 trace_probe_log_err(tmp - filename, BAD_ADDR_SUFFIX);
641 return -EINVAL;
642 }
643 }
644
645 /* Parse uprobe offset. */
646 ret = kstrtoul(arg, 0, &offset);
647 if (ret) {
648 trace_probe_log_err(arg - filename, BAD_UPROBE_OFFS);
649 return ret;
650 }
651
652 /* setup a probe */
653 trace_probe_log_set_index(0);
654 if (event) {
655 gbuf = kmalloc(MAX_EVENT_NAME_LEN, GFP_KERNEL);
656 if (!gbuf)
657 return -ENOMEM;
658
659 ret = traceprobe_parse_event_name(&event, &group, gbuf,
660 event - argv[0]);
661 if (ret)
662 return ret;
663 }
664
665 if (!event) {
666 char *tail;
667 char *ptr;
668
669 tail = kstrdup(kbasename(filename), GFP_KERNEL);
670 if (!tail)
671 return -ENOMEM;
672
673 ptr = strpbrk(tail, ".-_");
674 if (ptr)
675 *ptr = '\0';
676
677 buf = kmalloc(MAX_EVENT_NAME_LEN, GFP_KERNEL);
678 if (!buf)
679 return -ENOMEM;
680 snprintf(buf, MAX_EVENT_NAME_LEN, "%c_%s_0x%lx", 'p', tail, offset);
681 event = buf;
682 kfree(tail);
683 }
684
685 argc -= 2;
686 argv += 2;
687
688 tu = alloc_trace_uprobe(group, event, argc, is_return);
689 if (IS_ERR(tu)) {
690 ret = PTR_ERR(tu);
691 /* This must return -ENOMEM otherwise there is a bug */
692 WARN_ON_ONCE(ret != -ENOMEM);
693 return ret;
694 }
695 tu->offset = offset;
696 tu->ref_ctr_offset = ref_ctr_offset;
697 tu->path = path;
698 /* Clear @path so that it will not freed by path_put() */
699 memset(&path, 0, sizeof(path));
700 tu->filename = no_free_ptr(filename);
701
702 ctx = kzalloc_obj(*ctx);
703 if (!ctx)
704 return -ENOMEM;
705 ctx->flags = (is_return ? TPARG_FL_RETURN : 0) | TPARG_FL_USER;
706
707 /* parse arguments */
708 for (i = 0; i < argc; i++) {
709 trace_probe_log_set_index(i + 2);
710 ret = traceprobe_parse_probe_arg(&tu->tp, i, argv[i], ctx);
711 if (ret)
712 return ret;
713 }
714
715 ptype = is_ret_probe(tu) ? PROBE_PRINT_RETURN : PROBE_PRINT_NORMAL;
716 ret = traceprobe_set_print_fmt(&tu->tp, ptype);
717 if (ret < 0)
718 return ret;
719
720 ret = register_trace_uprobe(tu);
721 if (!ret)
722 tu = NULL;
723
724 return ret;
725}
726
727int trace_uprobe_create(const char *raw_command)
728{
729 return trace_probe_create(raw_command, __trace_uprobe_create);
730}
731
732static int create_or_delete_trace_uprobe(const char *raw_command)
733{
734 int ret;
735
736 if (raw_command[0] == '-')
737 return dyn_event_release(raw_command, &trace_uprobe_ops);
738
739 ret = dyn_event_create(raw_command, &trace_uprobe_ops);
740 return ret == -ECANCELED ? -EINVAL : ret;
741}
742
743static int trace_uprobe_release(struct dyn_event *ev)
744{
745 struct trace_uprobe *tu = to_trace_uprobe(ev);
746
747 return unregister_trace_uprobe(tu);
748}
749
750/* Probes listing interfaces */
751static int trace_uprobe_show(struct seq_file *m, struct dyn_event *ev)
752{
753 struct trace_uprobe *tu = to_trace_uprobe(ev);
754 char c = is_ret_probe(tu) ? 'r' : 'p';
755 int i;
756
757 seq_printf(m, "%c:%s/%s %s:0x%0*lx", c, trace_probe_group_name(&tu->tp),
758 trace_probe_name(&tu->tp), tu->filename,
759 (int)(sizeof(void *) * 2), tu->offset);
760
761 if (tu->ref_ctr_offset)
762 seq_printf(m, "(0x%lx)", tu->ref_ctr_offset);
763
764 for (i = 0; i < tu->tp.nr_args; i++)
765 seq_printf(m, " %s=%s", tu->tp.args[i].name, tu->tp.args[i].comm);
766
767 seq_putc(m, '\n');
768
769 trace_probe_dump_args(m, &tu->tp);
770
771 return 0;
772}
773
774static int probes_seq_show(struct seq_file *m, void *v)
775{
776 struct dyn_event *ev = v;
777
778 if (!is_trace_uprobe(ev))
779 return 0;
780
781 return trace_uprobe_show(m, ev);
782}
783
784static const struct seq_operations probes_seq_op = {
785 .start = dyn_event_seq_start,
786 .next = dyn_event_seq_next,
787 .stop = dyn_event_seq_stop,
788 .show = probes_seq_show
789};
790
791static int probes_open(struct inode *inode, struct file *file)
792{
793 int ret;
794
795 ret = security_locked_down(LOCKDOWN_TRACEFS);
796 if (ret)
797 return ret;
798
799 if ((file->f_mode & FMODE_WRITE) && (file->f_flags & O_TRUNC)) {
800 ret = dyn_events_release_all(&trace_uprobe_ops);
801 if (ret)
802 return ret;
803 }
804
805 return seq_open(file, &probes_seq_op);
806}
807
808static ssize_t probes_write(struct file *file, const char __user *buffer,
809 size_t count, loff_t *ppos)
810{
811 return trace_parse_run_command(file, buffer, count, ppos,
812 create_or_delete_trace_uprobe);
813}
814
815static const struct file_operations uprobe_events_ops = {
816 .owner = THIS_MODULE,
817 .open = probes_open,
818 .read = seq_read,
819 .llseek = seq_lseek,
820 .release = seq_release,
821 .write = probes_write,
822};
823
824/* Probes profiling interfaces */
825static int probes_profile_seq_show(struct seq_file *m, void *v)
826{
827 struct dyn_event *ev = v;
828 struct trace_uprobe *tu;
829 unsigned long nhits;
830 int cpu;
831
832 if (!is_trace_uprobe(ev))
833 return 0;
834
835 tu = to_trace_uprobe(ev);
836
837 nhits = 0;
838 for_each_possible_cpu(cpu) {
839 nhits += per_cpu(*tu->nhits, cpu);
840 }
841
842 seq_printf(m, " %s %-44s %15lu\n", tu->filename,
843 trace_probe_name(&tu->tp), nhits);
844 return 0;
845}
846
847static const struct seq_operations profile_seq_op = {
848 .start = dyn_event_seq_start,
849 .next = dyn_event_seq_next,
850 .stop = dyn_event_seq_stop,
851 .show = probes_profile_seq_show
852};
853
854static int profile_open(struct inode *inode, struct file *file)
855{
856 int ret;
857
858 ret = security_locked_down(LOCKDOWN_TRACEFS);
859 if (ret)
860 return ret;
861
862 return seq_open(file, &profile_seq_op);
863}
864
865static const struct file_operations uprobe_profile_ops = {
866 .owner = THIS_MODULE,
867 .open = profile_open,
868 .read = seq_read,
869 .llseek = seq_lseek,
870 .release = seq_release,
871};
872
873struct uprobe_cpu_buffer {
874 struct mutex mutex;
875 void *buf;
876 int dsize;
877};
878static struct uprobe_cpu_buffer __percpu *uprobe_cpu_buffer;
879static int uprobe_buffer_refcnt;
880#define MAX_UCB_BUFFER_SIZE PAGE_SIZE
881
882static int uprobe_buffer_init(void)
883{
884 int cpu, err_cpu;
885
886 uprobe_cpu_buffer = alloc_percpu(struct uprobe_cpu_buffer);
887 if (uprobe_cpu_buffer == NULL)
888 return -ENOMEM;
889
890 for_each_possible_cpu(cpu) {
891 struct page *p = alloc_pages_node(cpu_to_node(cpu),
892 GFP_KERNEL, 0);
893 if (p == NULL) {
894 err_cpu = cpu;
895 goto err;
896 }
897 per_cpu_ptr(uprobe_cpu_buffer, cpu)->buf = page_address(p);
898 mutex_init(&per_cpu_ptr(uprobe_cpu_buffer, cpu)->mutex);
899 }
900
901 return 0;
902
903err:
904 for_each_possible_cpu(cpu) {
905 if (cpu == err_cpu)
906 break;
907 free_page((unsigned long)per_cpu_ptr(uprobe_cpu_buffer, cpu)->buf);
908 }
909
910 free_percpu(uprobe_cpu_buffer);
911 return -ENOMEM;
912}
913
914static int uprobe_buffer_enable(void)
915{
916 int ret = 0;
917
918 lockdep_assert_held(&event_mutex);
919
920 if (uprobe_buffer_refcnt++ == 0) {
921 ret = uprobe_buffer_init();
922 if (ret < 0)
923 uprobe_buffer_refcnt--;
924 }
925
926 return ret;
927}
928
929static void uprobe_buffer_disable(void)
930{
931 int cpu;
932
933 lockdep_assert_held(&event_mutex);
934
935 if (--uprobe_buffer_refcnt == 0) {
936 for_each_possible_cpu(cpu)
937 free_page((unsigned long)per_cpu_ptr(uprobe_cpu_buffer,
938 cpu)->buf);
939
940 free_percpu(uprobe_cpu_buffer);
941 uprobe_cpu_buffer = NULL;
942 }
943}
944
945static struct uprobe_cpu_buffer *uprobe_buffer_get(void)
946{
947 struct uprobe_cpu_buffer *ucb;
948 int cpu;
949
950 cpu = raw_smp_processor_id();
951 ucb = per_cpu_ptr(uprobe_cpu_buffer, cpu);
952
953 /*
954 * Use per-cpu buffers for fastest access, but we might migrate
955 * so the mutex makes sure we have sole access to it.
956 */
957 mutex_lock(&ucb->mutex);
958
959 return ucb;
960}
961
962static void uprobe_buffer_put(struct uprobe_cpu_buffer *ucb)
963{
964 if (!ucb)
965 return;
966 mutex_unlock(&ucb->mutex);
967}
968
969static struct uprobe_cpu_buffer *prepare_uprobe_buffer(struct trace_uprobe *tu,
970 struct pt_regs *regs,
971 struct uprobe_cpu_buffer **ucbp)
972{
973 struct uprobe_cpu_buffer *ucb;
974 int dsize, esize;
975
976 if (*ucbp)
977 return *ucbp;
978
979 esize = SIZEOF_TRACE_ENTRY(is_ret_probe(tu));
980 dsize = __get_data_size(&tu->tp, regs, NULL);
981
982 ucb = uprobe_buffer_get();
983 ucb->dsize = tu->tp.size + dsize;
984
985 BUILD_BUG_ON(MAX_UCB_BUFFER_SIZE < MAX_PROBE_EVENT_SIZE);
986 if (WARN_ON_ONCE(ucb->dsize > MAX_UCB_BUFFER_SIZE)) {
987 ucb->dsize = MAX_UCB_BUFFER_SIZE;
988 dsize = MAX_UCB_BUFFER_SIZE - tu->tp.size;
989 }
990
991 store_trace_args(ucb->buf, &tu->tp, regs, NULL, esize, dsize);
992
993 *ucbp = ucb;
994 return ucb;
995}
996
997static void __uprobe_trace_func(struct trace_uprobe *tu,
998 unsigned long func, struct pt_regs *regs,
999 struct uprobe_cpu_buffer *ucb,
1000 struct trace_event_file *trace_file)
1001{
1002 struct uprobe_trace_entry_head *entry;
1003 struct trace_event_buffer fbuffer;
1004 void *data;
1005 int size, esize;
1006 struct trace_event_call *call = trace_probe_event_call(&tu->tp);
1007
1008 WARN_ON(call != trace_file->event_call);
1009
1010 if (trace_trigger_soft_disabled(trace_file))
1011 return;
1012
1013 esize = SIZEOF_TRACE_ENTRY(is_ret_probe(tu));
1014 size = esize + ucb->dsize;
1015 entry = trace_event_buffer_reserve(&fbuffer, trace_file, size);
1016 if (!entry)
1017 return;
1018
1019 if (is_ret_probe(tu)) {
1020 entry->vaddr[0] = func;
1021 entry->vaddr[1] = instruction_pointer(regs);
1022 data = DATAOF_TRACE_ENTRY(entry, true);
1023 } else {
1024 entry->vaddr[0] = instruction_pointer(regs);
1025 data = DATAOF_TRACE_ENTRY(entry, false);
1026 }
1027
1028 memcpy(data, ucb->buf, ucb->dsize);
1029
1030 trace_event_buffer_commit(&fbuffer);
1031}
1032
1033/* uprobe handler */
1034static int uprobe_trace_func(struct trace_uprobe *tu, struct pt_regs *regs,
1035 struct uprobe_cpu_buffer **ucbp)
1036{
1037 struct event_file_link *link;
1038 struct uprobe_cpu_buffer *ucb;
1039
1040 if (is_ret_probe(tu))
1041 return 0;
1042
1043 ucb = prepare_uprobe_buffer(tu, regs, ucbp);
1044
1045 rcu_read_lock();
1046 trace_probe_for_each_link_rcu(link, &tu->tp)
1047 __uprobe_trace_func(tu, 0, regs, ucb, link->file);
1048 rcu_read_unlock();
1049
1050 return 0;
1051}
1052
1053static void uretprobe_trace_func(struct trace_uprobe *tu, unsigned long func,
1054 struct pt_regs *regs,
1055 struct uprobe_cpu_buffer **ucbp)
1056{
1057 struct event_file_link *link;
1058 struct uprobe_cpu_buffer *ucb;
1059
1060 ucb = prepare_uprobe_buffer(tu, regs, ucbp);
1061
1062 rcu_read_lock();
1063 trace_probe_for_each_link_rcu(link, &tu->tp)
1064 __uprobe_trace_func(tu, func, regs, ucb, link->file);
1065 rcu_read_unlock();
1066}
1067
1068/* Event entry printers */
1069static enum print_line_t
1070print_uprobe_event(struct trace_iterator *iter, int flags, struct trace_event *event)
1071{
1072 struct uprobe_trace_entry_head *entry;
1073 struct trace_seq *s = &iter->seq;
1074 struct trace_uprobe *tu;
1075 u8 *data;
1076
1077 entry = (struct uprobe_trace_entry_head *)iter->ent;
1078 tu = trace_uprobe_primary_from_call(
1079 container_of(event, struct trace_event_call, event));
1080 if (unlikely(!tu))
1081 goto out;
1082
1083 if (is_ret_probe(tu)) {
1084 trace_seq_printf(s, "%s: (0x%lx <- 0x%lx)",
1085 trace_probe_name(&tu->tp),
1086 entry->vaddr[1], entry->vaddr[0]);
1087 data = DATAOF_TRACE_ENTRY(entry, true);
1088 } else {
1089 trace_seq_printf(s, "%s: (0x%lx)",
1090 trace_probe_name(&tu->tp),
1091 entry->vaddr[0]);
1092 data = DATAOF_TRACE_ENTRY(entry, false);
1093 }
1094
1095 if (trace_probe_print_args(s, tu->tp.args, tu->tp.nr_args, data, entry) < 0)
1096 goto out;
1097
1098 trace_seq_putc(s, '\n');
1099
1100 out:
1101 return trace_handle_return(s);
1102}
1103
1104typedef bool (*filter_func_t)(struct uprobe_consumer *self, struct mm_struct *mm);
1105
1106static int trace_uprobe_enable(struct trace_uprobe *tu, filter_func_t filter)
1107{
1108 struct inode *inode = d_real_inode(tu->path.dentry);
1109 struct uprobe *uprobe;
1110
1111 tu->consumer.filter = filter;
1112 uprobe = uprobe_register(inode, tu->offset, tu->ref_ctr_offset, &tu->consumer);
1113 if (IS_ERR(uprobe))
1114 return PTR_ERR(uprobe);
1115
1116 tu->uprobe = uprobe;
1117 return 0;
1118}
1119
1120static void __probe_event_disable(struct trace_probe *tp)
1121{
1122 struct trace_uprobe *tu;
1123 bool sync = false;
1124
1125 tu = container_of(tp, struct trace_uprobe, tp);
1126 WARN_ON(!uprobe_filter_is_empty(tu->tp.event->filter));
1127
1128 list_for_each_entry(tu, trace_probe_probe_list(tp), tp.list) {
1129 if (!tu->uprobe)
1130 continue;
1131
1132 uprobe_unregister_nosync(tu->uprobe, &tu->consumer);
1133 sync = true;
1134 tu->uprobe = NULL;
1135 }
1136 if (sync)
1137 uprobe_unregister_sync();
1138}
1139
1140static int probe_event_enable(struct trace_event_call *call,
1141 struct trace_event_file *file, filter_func_t filter)
1142{
1143 struct trace_probe *tp;
1144 struct trace_uprobe *tu;
1145 bool enabled;
1146 int ret;
1147
1148 tp = trace_probe_primary_from_call(call);
1149 if (WARN_ON_ONCE(!tp))
1150 return -ENODEV;
1151 enabled = trace_probe_is_enabled(tp);
1152
1153 /* This may also change "enabled" state */
1154 if (file) {
1155 if (trace_probe_test_flag(tp, TP_FLAG_PROFILE))
1156 return -EINTR;
1157
1158 ret = trace_probe_add_file(tp, file);
1159 if (ret < 0)
1160 return ret;
1161 } else {
1162 if (trace_probe_test_flag(tp, TP_FLAG_TRACE))
1163 return -EINTR;
1164
1165 trace_probe_set_flag(tp, TP_FLAG_PROFILE);
1166 }
1167
1168 tu = container_of(tp, struct trace_uprobe, tp);
1169 WARN_ON(!uprobe_filter_is_empty(tu->tp.event->filter));
1170
1171 if (enabled)
1172 return 0;
1173
1174 ret = uprobe_buffer_enable();
1175 if (ret)
1176 goto err_flags;
1177
1178 list_for_each_entry(tu, trace_probe_probe_list(tp), tp.list) {
1179 ret = trace_uprobe_enable(tu, filter);
1180 if (ret) {
1181 __probe_event_disable(tp);
1182 goto err_buffer;
1183 }
1184 }
1185
1186 return 0;
1187
1188 err_buffer:
1189 uprobe_buffer_disable();
1190
1191 err_flags:
1192 if (file)
1193 trace_probe_remove_file(tp, file);
1194 else
1195 trace_probe_clear_flag(tp, TP_FLAG_PROFILE);
1196
1197 return ret;
1198}
1199
1200static void probe_event_disable(struct trace_event_call *call,
1201 struct trace_event_file *file)
1202{
1203 struct trace_probe *tp;
1204
1205 tp = trace_probe_primary_from_call(call);
1206 if (WARN_ON_ONCE(!tp))
1207 return;
1208
1209 if (!trace_probe_is_enabled(tp))
1210 return;
1211
1212 if (file) {
1213 if (trace_probe_remove_file(tp, file) < 0)
1214 return;
1215
1216 if (trace_probe_is_enabled(tp))
1217 return;
1218 } else
1219 trace_probe_clear_flag(tp, TP_FLAG_PROFILE);
1220
1221 __probe_event_disable(tp);
1222 uprobe_buffer_disable();
1223}
1224
1225static int uprobe_event_define_fields(struct trace_event_call *event_call)
1226{
1227 int ret, size;
1228 struct uprobe_trace_entry_head field;
1229 struct trace_uprobe *tu;
1230
1231 tu = trace_uprobe_primary_from_call(event_call);
1232 if (unlikely(!tu))
1233 return -ENODEV;
1234
1235 if (is_ret_probe(tu)) {
1236 DEFINE_FIELD(unsigned long, vaddr[0], FIELD_STRING_FUNC, 0);
1237 DEFINE_FIELD(unsigned long, vaddr[1], FIELD_STRING_RETIP, 0);
1238 size = SIZEOF_TRACE_ENTRY(true);
1239 } else {
1240 DEFINE_FIELD(unsigned long, vaddr[0], FIELD_STRING_IP, 0);
1241 size = SIZEOF_TRACE_ENTRY(false);
1242 }
1243
1244 return traceprobe_define_arg_fields(event_call, size, &tu->tp);
1245}
1246
1247#ifdef CONFIG_PERF_EVENTS
1248static bool
1249__uprobe_perf_filter(struct trace_uprobe_filter *filter, struct mm_struct *mm)
1250{
1251 struct perf_event *event;
1252
1253 list_for_each_entry(event, &filter->perf_events, hw.tp_list) {
1254 if (event->hw.target->mm == mm)
1255 return true;
1256 }
1257
1258 return false;
1259}
1260
1261static inline bool
1262trace_uprobe_filter_event(struct trace_uprobe_filter *filter,
1263 struct perf_event *event)
1264{
1265 return __uprobe_perf_filter(filter, event->hw.target->mm);
1266}
1267
1268static bool trace_uprobe_filter_remove(struct trace_uprobe_filter *filter,
1269 struct perf_event *event)
1270{
1271 bool done;
1272
1273 write_lock(&filter->rwlock);
1274 if (event->hw.target) {
1275 list_del(&event->hw.tp_list);
1276 done = filter->nr_systemwide ||
1277 (event->hw.target->flags & PF_EXITING) ||
1278 trace_uprobe_filter_event(filter, event);
1279 } else {
1280 filter->nr_systemwide--;
1281 done = filter->nr_systemwide;
1282 }
1283 write_unlock(&filter->rwlock);
1284
1285 return done;
1286}
1287
1288/* This returns true if the filter always covers target mm */
1289static bool trace_uprobe_filter_add(struct trace_uprobe_filter *filter,
1290 struct perf_event *event)
1291{
1292 bool done;
1293
1294 write_lock(&filter->rwlock);
1295 if (event->hw.target) {
1296 /*
1297 * event->parent != NULL means copy_process(), we can avoid
1298 * uprobe_apply(). current->mm must be probed and we can rely
1299 * on dup_mmap() which preserves the already installed bp's.
1300 *
1301 * attr.enable_on_exec means that exec/mmap will install the
1302 * breakpoints we need.
1303 */
1304 done = filter->nr_systemwide ||
1305 event->parent || event->attr.enable_on_exec ||
1306 trace_uprobe_filter_event(filter, event);
1307 list_add(&event->hw.tp_list, &filter->perf_events);
1308 } else {
1309 done = filter->nr_systemwide;
1310 filter->nr_systemwide++;
1311 }
1312 write_unlock(&filter->rwlock);
1313
1314 return done;
1315}
1316
1317static int uprobe_perf_close(struct trace_event_call *call,
1318 struct perf_event *event)
1319{
1320 struct trace_probe *tp;
1321 struct trace_uprobe *tu;
1322 int ret = 0;
1323
1324 tp = trace_probe_primary_from_call(call);
1325 if (WARN_ON_ONCE(!tp))
1326 return -ENODEV;
1327
1328 tu = container_of(tp, struct trace_uprobe, tp);
1329 if (trace_uprobe_filter_remove(tu->tp.event->filter, event))
1330 return 0;
1331
1332 list_for_each_entry(tu, trace_probe_probe_list(tp), tp.list) {
1333 ret = uprobe_apply(tu->uprobe, &tu->consumer, false);
1334 if (ret)
1335 break;
1336 }
1337
1338 return ret;
1339}
1340
1341static int uprobe_perf_open(struct trace_event_call *call,
1342 struct perf_event *event)
1343{
1344 struct trace_probe *tp;
1345 struct trace_uprobe *tu;
1346 int err = 0;
1347
1348 tp = trace_probe_primary_from_call(call);
1349 if (WARN_ON_ONCE(!tp))
1350 return -ENODEV;
1351
1352 tu = container_of(tp, struct trace_uprobe, tp);
1353 if (trace_uprobe_filter_add(tu->tp.event->filter, event))
1354 return 0;
1355
1356 list_for_each_entry(tu, trace_probe_probe_list(tp), tp.list) {
1357 err = uprobe_apply(tu->uprobe, &tu->consumer, true);
1358 if (err) {
1359 uprobe_perf_close(call, event);
1360 break;
1361 }
1362 }
1363
1364 return err;
1365}
1366
1367static bool uprobe_perf_filter(struct uprobe_consumer *uc, struct mm_struct *mm)
1368{
1369 struct trace_uprobe_filter *filter;
1370 struct trace_uprobe *tu;
1371 int ret;
1372
1373 tu = container_of(uc, struct trace_uprobe, consumer);
1374 filter = tu->tp.event->filter;
1375
1376 /*
1377 * speculative short-circuiting check to avoid unnecessarily taking
1378 * filter->rwlock below, if the uprobe has system-wide consumer
1379 */
1380 if (READ_ONCE(filter->nr_systemwide))
1381 return true;
1382
1383 read_lock(&filter->rwlock);
1384 ret = __uprobe_perf_filter(filter, mm);
1385 read_unlock(&filter->rwlock);
1386
1387 return ret;
1388}
1389
1390static void __uprobe_perf_func(struct trace_uprobe *tu,
1391 unsigned long func, struct pt_regs *regs,
1392 struct uprobe_cpu_buffer **ucbp)
1393{
1394 struct trace_event_call *call = trace_probe_event_call(&tu->tp);
1395 struct uprobe_trace_entry_head *entry;
1396 struct uprobe_cpu_buffer *ucb;
1397 struct hlist_head *head;
1398 void *data;
1399 int size, esize;
1400 int rctx;
1401
1402#ifdef CONFIG_BPF_EVENTS
1403 if (bpf_prog_array_valid(call)) {
1404 const struct bpf_prog_array *array;
1405 u32 ret;
1406
1407 rcu_read_lock_trace();
1408 array = rcu_dereference_check(call->prog_array, rcu_read_lock_trace_held());
1409 ret = bpf_prog_run_array_uprobe(array, regs, bpf_prog_run);
1410 rcu_read_unlock_trace();
1411 if (!ret)
1412 return;
1413 }
1414#endif /* CONFIG_BPF_EVENTS */
1415
1416 esize = SIZEOF_TRACE_ENTRY(is_ret_probe(tu));
1417
1418 ucb = prepare_uprobe_buffer(tu, regs, ucbp);
1419 size = esize + ucb->dsize;
1420 size = ALIGN(size + sizeof(u32), sizeof(u64)) - sizeof(u32);
1421 if (WARN_ONCE(size > PERF_MAX_TRACE_SIZE, "profile buffer not large enough"))
1422 return;
1423
1424 preempt_disable();
1425 head = this_cpu_ptr(call->perf_events);
1426 if (hlist_empty(head))
1427 goto out;
1428
1429 entry = perf_trace_buf_alloc(size, NULL, &rctx);
1430 if (!entry)
1431 goto out;
1432
1433 if (is_ret_probe(tu)) {
1434 entry->vaddr[0] = func;
1435 entry->vaddr[1] = instruction_pointer(regs);
1436 data = DATAOF_TRACE_ENTRY(entry, true);
1437 } else {
1438 entry->vaddr[0] = instruction_pointer(regs);
1439 data = DATAOF_TRACE_ENTRY(entry, false);
1440 }
1441
1442 memcpy(data, ucb->buf, ucb->dsize);
1443
1444 if (size - esize > ucb->dsize)
1445 memset(data + ucb->dsize, 0, size - esize - ucb->dsize);
1446
1447 perf_trace_buf_submit(entry, size, rctx, call->event.type, 1, regs,
1448 head, NULL);
1449 out:
1450 preempt_enable();
1451}
1452
1453/* uprobe profile handler */
1454static int uprobe_perf_func(struct trace_uprobe *tu, struct pt_regs *regs,
1455 struct uprobe_cpu_buffer **ucbp)
1456{
1457 if (!uprobe_perf_filter(&tu->consumer, current->mm))
1458 return UPROBE_HANDLER_REMOVE;
1459
1460 if (!is_ret_probe(tu))
1461 __uprobe_perf_func(tu, 0, regs, ucbp);
1462 return 0;
1463}
1464
1465static void uretprobe_perf_func(struct trace_uprobe *tu, unsigned long func,
1466 struct pt_regs *regs,
1467 struct uprobe_cpu_buffer **ucbp)
1468{
1469 __uprobe_perf_func(tu, func, regs, ucbp);
1470}
1471
1472int bpf_get_uprobe_info(const struct perf_event *event, u32 *fd_type,
1473 const char **filename, u64 *probe_offset,
1474 u64 *probe_addr, bool perf_type_tracepoint)
1475{
1476 const char *pevent = trace_event_name(event->tp_event);
1477 const char *group = event->tp_event->class->system;
1478 struct trace_uprobe *tu;
1479
1480 if (perf_type_tracepoint)
1481 tu = find_probe_event(pevent, group);
1482 else
1483 tu = trace_uprobe_primary_from_call(event->tp_event);
1484 if (!tu)
1485 return -EINVAL;
1486
1487 *fd_type = is_ret_probe(tu) ? BPF_FD_TYPE_URETPROBE
1488 : BPF_FD_TYPE_UPROBE;
1489 *filename = tu->filename;
1490 *probe_offset = tu->offset;
1491 *probe_addr = tu->ref_ctr_offset;
1492 return 0;
1493}
1494#endif /* CONFIG_PERF_EVENTS */
1495
1496static int
1497trace_uprobe_register(struct trace_event_call *event, enum trace_reg type,
1498 void *data)
1499{
1500 struct trace_event_file *file = data;
1501
1502 switch (type) {
1503 case TRACE_REG_REGISTER:
1504 return probe_event_enable(event, file, NULL);
1505
1506 case TRACE_REG_UNREGISTER:
1507 probe_event_disable(event, file);
1508 return 0;
1509
1510#ifdef CONFIG_PERF_EVENTS
1511 case TRACE_REG_PERF_REGISTER:
1512 return probe_event_enable(event, NULL, uprobe_perf_filter);
1513
1514 case TRACE_REG_PERF_UNREGISTER:
1515 probe_event_disable(event, NULL);
1516 return 0;
1517
1518 case TRACE_REG_PERF_OPEN:
1519 return uprobe_perf_open(event, data);
1520
1521 case TRACE_REG_PERF_CLOSE:
1522 return uprobe_perf_close(event, data);
1523
1524#endif
1525 default:
1526 return 0;
1527 }
1528}
1529
1530static int uprobe_dispatcher(struct uprobe_consumer *con, struct pt_regs *regs,
1531 __u64 *data)
1532{
1533 struct trace_uprobe *tu;
1534 struct uprobe_dispatch_data udd;
1535 struct uprobe_cpu_buffer *ucb = NULL;
1536 unsigned int flags;
1537 int ret = 0;
1538
1539 tu = container_of(con, struct trace_uprobe, consumer);
1540
1541 this_cpu_inc(*tu->nhits);
1542
1543 udd.tu = tu;
1544 udd.bp_addr = instruction_pointer(regs);
1545
1546 current->utask->vaddr = (unsigned long) &udd;
1547
1548 if (WARN_ON_ONCE(!uprobe_cpu_buffer))
1549 return 0;
1550
1551 flags = trace_probe_load_flag(&tu->tp);
1552 if (flags & TP_FLAG_TRACE)
1553 ret |= uprobe_trace_func(tu, regs, &ucb);
1554
1555#ifdef CONFIG_PERF_EVENTS
1556 if (flags & TP_FLAG_PROFILE)
1557 ret |= uprobe_perf_func(tu, regs, &ucb);
1558#endif
1559 uprobe_buffer_put(ucb);
1560 return ret;
1561}
1562
1563static int uretprobe_dispatcher(struct uprobe_consumer *con,
1564 unsigned long func, struct pt_regs *regs,
1565 __u64 *data)
1566{
1567 struct trace_uprobe *tu;
1568 struct uprobe_dispatch_data udd;
1569 struct uprobe_cpu_buffer *ucb = NULL;
1570 unsigned int flags;
1571
1572 tu = container_of(con, struct trace_uprobe, consumer);
1573
1574 udd.tu = tu;
1575 udd.bp_addr = func;
1576
1577 current->utask->vaddr = (unsigned long) &udd;
1578
1579 if (WARN_ON_ONCE(!uprobe_cpu_buffer))
1580 return 0;
1581
1582 flags = trace_probe_load_flag(&tu->tp);
1583 if (flags & TP_FLAG_TRACE)
1584 uretprobe_trace_func(tu, func, regs, &ucb);
1585
1586#ifdef CONFIG_PERF_EVENTS
1587 if (flags & TP_FLAG_PROFILE)
1588 uretprobe_perf_func(tu, func, regs, &ucb);
1589#endif
1590 uprobe_buffer_put(ucb);
1591 return 0;
1592}
1593
1594static struct trace_event_functions uprobe_funcs = {
1595 .trace = print_uprobe_event
1596};
1597
1598static struct trace_event_fields uprobe_fields_array[] = {
1599 { .type = TRACE_FUNCTION_TYPE,
1600 .define_fields = uprobe_event_define_fields },
1601 {}
1602};
1603
1604static inline void init_trace_event_call(struct trace_uprobe *tu)
1605{
1606 struct trace_event_call *call = trace_probe_event_call(&tu->tp);
1607 call->event.funcs = &uprobe_funcs;
1608 call->class->fields_array = uprobe_fields_array;
1609
1610 call->flags = TRACE_EVENT_FL_UPROBE | TRACE_EVENT_FL_CAP_ANY;
1611 call->class->reg = trace_uprobe_register;
1612}
1613
1614static int register_uprobe_event(struct trace_uprobe *tu)
1615{
1616 init_trace_event_call(tu);
1617
1618 return trace_probe_register_event_call(&tu->tp);
1619}
1620
1621static int unregister_uprobe_event(struct trace_uprobe *tu)
1622{
1623 return trace_probe_unregister_event_call(&tu->tp);
1624}
1625
1626#ifdef CONFIG_PERF_EVENTS
1627struct trace_event_call *
1628create_local_trace_uprobe(char *name, unsigned long offs,
1629 unsigned long ref_ctr_offset, bool is_return)
1630{
1631 enum probe_print_type ptype;
1632 struct trace_uprobe *tu;
1633 struct path path;
1634 int ret;
1635
1636 ret = kern_path(name, LOOKUP_FOLLOW, &path);
1637 if (ret)
1638 return ERR_PTR(ret);
1639
1640 if (!d_is_reg(path.dentry)) {
1641 path_put(&path);
1642 return ERR_PTR(-EINVAL);
1643 }
1644
1645 /*
1646 * local trace_kprobes are not added to dyn_event, so they are never
1647 * searched in find_trace_kprobe(). Therefore, there is no concern of
1648 * duplicated name "DUMMY_EVENT" here.
1649 */
1650 tu = alloc_trace_uprobe(UPROBE_EVENT_SYSTEM, "DUMMY_EVENT", 0,
1651 is_return);
1652
1653 if (IS_ERR(tu)) {
1654 pr_info("Failed to allocate trace_uprobe.(%d)\n",
1655 (int)PTR_ERR(tu));
1656 path_put(&path);
1657 return ERR_CAST(tu);
1658 }
1659
1660 tu->offset = offs;
1661 tu->path = path;
1662 tu->ref_ctr_offset = ref_ctr_offset;
1663 tu->filename = kstrdup(name, GFP_KERNEL);
1664 if (!tu->filename) {
1665 ret = -ENOMEM;
1666 goto error;
1667 }
1668
1669 init_trace_event_call(tu);
1670
1671 ptype = is_ret_probe(tu) ? PROBE_PRINT_RETURN : PROBE_PRINT_NORMAL;
1672 if (traceprobe_set_print_fmt(&tu->tp, ptype) < 0) {
1673 ret = -ENOMEM;
1674 goto error;
1675 }
1676
1677 return trace_probe_event_call(&tu->tp);
1678error:
1679 free_trace_uprobe(tu);
1680 return ERR_PTR(ret);
1681}
1682
1683void destroy_local_trace_uprobe(struct trace_event_call *event_call)
1684{
1685 struct trace_uprobe *tu;
1686
1687 tu = trace_uprobe_primary_from_call(event_call);
1688
1689 free_trace_uprobe(tu);
1690}
1691#endif /* CONFIG_PERF_EVENTS */
1692
1693/* Make a trace interface for controlling probe points */
1694static __init int init_uprobe_trace(void)
1695{
1696 int ret;
1697
1698 ret = dyn_event_register(&trace_uprobe_ops);
1699 if (ret)
1700 return ret;
1701
1702 ret = tracing_init_dentry();
1703 if (ret)
1704 return 0;
1705
1706 trace_create_file("uprobe_events", TRACE_MODE_WRITE, NULL,
1707 NULL, &uprobe_events_ops);
1708 /* Profile interface */
1709 trace_create_file("uprobe_profile", TRACE_MODE_READ, NULL,
1710 NULL, &uprobe_profile_ops);
1711 return 0;
1712}
1713
1714fs_initcall(init_uprobe_trace);