projects/linux·blob master · refs

2986 lines · 71.9 KiB

1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Common code for probe-based Dynamic events.
4 *
5 * This code was copied from kernel/trace/trace_kprobe.c written by
6 * Masami Hiramatsu <[email protected]>
7 *
8 * Updates to make this generic:
9 * Copyright (C) IBM Corporation, 2010-2011
10 * Author: Srikar Dronamraju
11 */
12#define pr_fmt(fmt) "trace_probe: " fmt
13
14#include <linux/bpf.h>
15#include <linux/fs.h>
16
17#include "trace_btf.h"
18#include "trace_probe.h"
19
20#undef C
21#define C(a, b) b
22
23static const char *trace_probe_err_text[] = { ERRORS };
24
25static const char *reserved_field_names[] = {
26 "common_type",
27 "common_flags",
28 "common_preempt_count",
29 "common_pid",
30 "common_tgid",
31 FIELD_STRING_IP,
32 FIELD_STRING_RETIP,
33 FIELD_STRING_FUNC,
34};
35
36/* Printing in basic type function template */
37#define DEFINE_BASIC_PRINT_TYPE_FUNC(tname, type, fmt) \
38int PRINT_TYPE_FUNC_NAME(tname)(struct trace_seq *s, void *data, void *ent)\
39{ \
40 trace_seq_printf(s, fmt, *(type *)data); \
41 return !trace_seq_has_overflowed(s); \
42} \
43const char PRINT_TYPE_FMT_NAME(tname)[] = fmt;
44
45DEFINE_BASIC_PRINT_TYPE_FUNC(u8, u8, "%u")
46DEFINE_BASIC_PRINT_TYPE_FUNC(u16, u16, "%u")
47DEFINE_BASIC_PRINT_TYPE_FUNC(u32, u32, "%u")
48DEFINE_BASIC_PRINT_TYPE_FUNC(u64, u64, "%Lu")
49DEFINE_BASIC_PRINT_TYPE_FUNC(s8, s8, "%d")
50DEFINE_BASIC_PRINT_TYPE_FUNC(s16, s16, "%d")
51DEFINE_BASIC_PRINT_TYPE_FUNC(s32, s32, "%d")
52DEFINE_BASIC_PRINT_TYPE_FUNC(s64, s64, "%Ld")
53DEFINE_BASIC_PRINT_TYPE_FUNC(x8, u8, "0x%x")
54DEFINE_BASIC_PRINT_TYPE_FUNC(x16, u16, "0x%x")
55DEFINE_BASIC_PRINT_TYPE_FUNC(x32, u32, "0x%x")
56DEFINE_BASIC_PRINT_TYPE_FUNC(x64, u64, "0x%Lx")
57DEFINE_BASIC_PRINT_TYPE_FUNC(char, u8, "'%c'")
58
59int PRINT_TYPE_FUNC_NAME(symbol)(struct trace_seq *s, void *data, void *ent)
60{
61 trace_seq_printf(s, "%pS", (void *)*(unsigned long *)data);
62 return !trace_seq_has_overflowed(s);
63}
64const char PRINT_TYPE_FMT_NAME(symbol)[] = "%pS";
65
66/* Print type function for string type */
67int PRINT_TYPE_FUNC_NAME(string)(struct trace_seq *s, void *data, void *ent)
68{
69 int len = *(u32 *)data >> 16;
70
71 if (!len)
72 trace_seq_puts(s, FAULT_STRING);
73 else
74 trace_seq_printf(s, "\"%s\"",
75 (const char *)get_loc_data(data, ent));
76 return !trace_seq_has_overflowed(s);
77}
78
79const char PRINT_TYPE_FMT_NAME(string)[] = "\\\"%s\\\"";
80
81/* Fetch type information table */
82static const struct fetch_type probe_fetch_types[] = {
83 /* Special types */
84 __ASSIGN_FETCH_TYPE("string", string, string, sizeof(u32), 1, 1,
85 "__data_loc char[]"),
86 __ASSIGN_FETCH_TYPE("ustring", string, string, sizeof(u32), 1, 1,
87 "__data_loc char[]"),
88 __ASSIGN_FETCH_TYPE("symstr", string, string, sizeof(u32), 1, 1,
89 "__data_loc char[]"),
90 /* Basic types */
91 ASSIGN_FETCH_TYPE(u8, u8, 0),
92 ASSIGN_FETCH_TYPE(u16, u16, 0),
93 ASSIGN_FETCH_TYPE(u32, u32, 0),
94 ASSIGN_FETCH_TYPE(u64, u64, 0),
95 ASSIGN_FETCH_TYPE(s8, u8, 1),
96 ASSIGN_FETCH_TYPE(s16, u16, 1),
97 ASSIGN_FETCH_TYPE(s32, u32, 1),
98 ASSIGN_FETCH_TYPE(s64, u64, 1),
99 ASSIGN_FETCH_TYPE_ALIAS(x8, u8, u8, 0),
100 ASSIGN_FETCH_TYPE_ALIAS(x16, u16, u16, 0),
101 ASSIGN_FETCH_TYPE_ALIAS(x32, u32, u32, 0),
102 ASSIGN_FETCH_TYPE_ALIAS(x64, u64, u64, 0),
103 ASSIGN_FETCH_TYPE_ALIAS(char, u8, u8, 0),
104 ASSIGN_FETCH_TYPE_ALIAS(symbol, ADDR_FETCH_TYPE, ADDR_FETCH_TYPE, 0),
105
106 ASSIGN_FETCH_TYPE_END
107};
108
109static const struct fetch_type *find_fetch_type(const char *type, unsigned long flags)
110{
111 int i;
112
113 /* Reject the symbol/symstr for uprobes */
114 if (type && (flags & TPARG_FL_USER) &&
115 (!strcmp(type, "symbol") || !strcmp(type, "symstr")))
116 return NULL;
117
118 if (!type)
119 type = DEFAULT_FETCH_TYPE_STR;
120
121 /* Special case: bitfield */
122 if (*type == 'b') {
123 unsigned long bs;
124
125 type = strchr(type, '/');
126 if (!type)
127 goto fail;
128
129 type++;
130 if (kstrtoul(type, 0, &bs))
131 goto fail;
132
133 switch (bs) {
134 case 8:
135 return find_fetch_type("u8", flags);
136 case 16:
137 return find_fetch_type("u16", flags);
138 case 32:
139 return find_fetch_type("u32", flags);
140 case 64:
141 return find_fetch_type("u64", flags);
142 default:
143 goto fail;
144 }
145 }
146
147 for (i = 0; probe_fetch_types[i].name; i++) {
148 if (strcmp(type, probe_fetch_types[i].name) == 0)
149 return &probe_fetch_types[i];
150 }
151
152fail:
153 return NULL;
154}
155
156static struct trace_probe_log trace_probe_log;
157extern struct mutex dyn_event_ops_mutex;
158
159const char *trace_probe_log_init(const char *subsystem, int argc, const char **argv)
160{
161 lockdep_assert_held(&dyn_event_ops_mutex);
162
163 trace_probe_log.subsystem = subsystem;
164 trace_probe_log.argc = argc;
165 trace_probe_log.argv = argv;
166 trace_probe_log.index = 0;
167 return subsystem;
168}
169
170void trace_probe_log_clear(void)
171{
172 lockdep_assert_held(&dyn_event_ops_mutex);
173
174 memset(&trace_probe_log, 0, sizeof(trace_probe_log));
175}
176
177void trace_probe_log_set_index(int index)
178{
179 lockdep_assert_held(&dyn_event_ops_mutex);
180
181 trace_probe_log.index = index;
182}
183
184void __trace_probe_log_err(int offset, int err_type)
185{
186 char *command, *p;
187 int i, len = 0, pos = 0;
188
189 lockdep_assert_held(&dyn_event_ops_mutex);
190
191 if (!trace_probe_log.argv || !trace_probe_log.argc)
192 return;
193
194 /* Recalculate the length and allocate buffer */
195 for (i = 0; i < trace_probe_log.argc; i++) {
196 if (i == trace_probe_log.index)
197 pos = len;
198 len += strlen(trace_probe_log.argv[i]) + 1;
199 }
200 command = kzalloc(len, GFP_KERNEL);
201 if (!command)
202 return;
203
204 if (trace_probe_log.index >= trace_probe_log.argc) {
205 /**
206 * Set the error position is next to the last arg + space.
207 * Note that len includes the terminal null and the cursor
208 * appears at pos + 1.
209 */
210 pos = len;
211 offset = 0;
212 }
213
214 /* And make a command string from argv array */
215 p = command;
216 for (i = 0; i < trace_probe_log.argc; i++) {
217 len = strlen(trace_probe_log.argv[i]);
218 memcpy(p, trace_probe_log.argv[i], len);
219 p[len] = ' ';
220 p += len + 1;
221 }
222 *(p - 1) = '\0';
223
224 tracing_log_err(NULL, trace_probe_log.subsystem, command,
225 trace_probe_err_text, err_type, pos + offset);
226
227 kfree(command);
228}
229
230/* Split symbol and offset. */
231int traceprobe_split_symbol_offset(char *symbol, long *offset)
232{
233 char *tmp;
234 int ret;
235
236 if (!offset)
237 return -EINVAL;
238
239 tmp = strpbrk(symbol, "+-");
240 if (tmp) {
241 ret = kstrtol(tmp, 0, offset);
242 if (ret)
243 return ret;
244 *tmp = '\0';
245 } else
246 *offset = 0;
247
248 return 0;
249}
250
251/**
252 * traceprobe_parse_event_name() - Parse a string into group and event names
253 * @pevent: A pointer to the string to be parsed.
254 * @pgroup: A pointer to the group name.
255 * @buf: A buffer to store the parsed group name.
256 * @offset: The offset of the string in the original user command, for logging.
257 *
258 * This parses a string with the format `[GROUP/][EVENT]` or `[GROUP.][EVENT]`
259 * (either GROUP or EVENT or both must be specified).
260 * Since the parsed group name is stored in @buf, the caller must ensure @buf
261 * is at least MAX_EVENT_NAME_LEN bytes.
262 *
263 * Return: 0 on success, or -EINVAL on failure.
264 *
265 * If success, *@pevent is updated to point to the event name part of the
266 * original string, or NULL if there is no event name.
267 * Also, *@pgroup is updated to point to the parsed group which is stored
268 * in @buf, or NULL if there is no group name.
269 */
270int traceprobe_parse_event_name(const char **pevent, const char **pgroup,
271 char *buf, int offset)
272{
273 const char *slash, *event = *pevent;
274 int len;
275
276 slash = strchr(event, '/');
277 if (!slash)
278 slash = strchr(event, '.');
279
280 if (slash) {
281 if (slash == event) {
282 trace_probe_log_err(offset, NO_GROUP_NAME);
283 return -EINVAL;
284 }
285 if (slash - event + 1 > MAX_EVENT_NAME_LEN) {
286 trace_probe_log_err(offset, GROUP_TOO_LONG);
287 return -EINVAL;
288 }
289 strscpy(buf, event, slash - event + 1);
290 if (!is_good_system_name(buf)) {
291 trace_probe_log_err(offset, BAD_GROUP_NAME);
292 return -EINVAL;
293 }
294 *pgroup = buf;
295 *pevent = slash + 1;
296 offset += slash - event + 1;
297 event = *pevent;
298 }
299 len = strlen(event);
300 if (len == 0) {
301 if (slash) {
302 *pevent = NULL;
303 return 0;
304 }
305 trace_probe_log_err(offset, NO_EVENT_NAME);
306 return -EINVAL;
307 } else if (len >= MAX_EVENT_NAME_LEN) {
308 trace_probe_log_err(offset, EVENT_TOO_LONG);
309 return -EINVAL;
310 }
311 if (!is_good_name(event)) {
312 trace_probe_log_err(offset, BAD_EVENT_NAME);
313 return -EINVAL;
314 }
315 return 0;
316}
317
318static int parse_trace_event_arg(char *arg, struct fetch_insn *code,
319 struct traceprobe_parse_context *ctx)
320{
321 struct ftrace_event_field *field;
322 struct list_head *head;
323
324 head = trace_get_fields(ctx->event);
325 list_for_each_entry(field, head, link) {
326 if (!strcmp(arg, field->name)) {
327 code->op = FETCH_OP_TP_ARG;
328 code->data = field;
329 return 0;
330 }
331 }
332 return -ENOENT;
333}
334
335static int parse_trace_event(char *arg, struct fetch_insn *code,
336 struct traceprobe_parse_context *ctx)
337{
338 int ret;
339
340 if (code->data)
341 return -EFAULT;
342 ret = parse_trace_event_arg(arg, code, ctx);
343 if (!ret)
344 return 0;
345 return -EINVAL;
346}
347
348/* this_cpu_* parser */
349#define THIS_CPU_PTR_PREFIX "this_cpu_ptr("
350#define THIS_CPU_READ_PREFIX "this_cpu_read("
351#define THIS_CPU_PTR_LEN (sizeof(THIS_CPU_PTR_PREFIX) - 1)
352#define THIS_CPU_READ_LEN (sizeof(THIS_CPU_READ_PREFIX) - 1)
353
354static int
355parse_probe_arg(char *arg, const struct fetch_type *type,
356 struct fetch_insn **pcode, struct fetch_insn *end,
357 struct traceprobe_parse_context *ctx);
358
359static int parse_this_cpu(char *arg, struct traceprobe_parse_context *ctx)
360{
361 bool is_read = false;
362 char *tmp;
363
364 /*
365 * This is only for kernel probes, excluding eprobe, because per-cpu
366 * pointer should not be recorded by events.
367 */
368 if (!(ctx->flags & TPARG_FL_KERNEL) ||
369 (ctx->flags & TPARG_FL_TEVENT)) {
370 trace_probe_log_err(ctx->offset, NOSUP_PERCPU);
371 return -EINVAL;
372 }
373 if (str_has_prefix(arg, THIS_CPU_PTR_PREFIX)) {
374 arg += THIS_CPU_PTR_LEN;
375 ctx->offset += THIS_CPU_PTR_LEN;
376 } else if (str_has_prefix(arg, THIS_CPU_READ_PREFIX)) {
377 arg += THIS_CPU_READ_LEN;
378 ctx->offset += THIS_CPU_READ_LEN;
379 is_read = true;
380 } else {
381 trace_probe_log_err(ctx->offset, BAD_FETCH_ARG);
382 return -EINVAL;
383 }
384
385 tmp = strrchr(arg, ')');
386 if (!tmp) {
387 trace_probe_log_err(ctx->offset + strlen(arg),
388 DEREF_OPEN_BRACE);
389 return -EINVAL;
390 }
391 *tmp = '\0';
392
393 ctx->stack[ctx->depth].type = STATE_DEREF;
394 ctx->stack[ctx->depth].deref.deref = FETCH_OP_CPU_PTR;
395 ctx->stack[ctx->depth].deref.offset = 0;
396 ctx->stack[ctx->depth].deref.cur_offs = ctx->offset;
397 ctx->stack[ctx->depth].deref.inner_arg = arg;
398 ctx->stack[ctx->depth].deref.is_cpu_read = is_read;
399 return 0;
400}
401
402#ifdef CONFIG_PROBE_EVENTS_BTF_ARGS
403
404static u32 btf_type_int(const struct btf_type *t)
405{
406 return *(u32 *)(t + 1);
407}
408
409static bool btf_type_is_char_ptr(struct btf *btf, const struct btf_type *type)
410{
411 const struct btf_type *real_type;
412 u32 intdata;
413
414 real_type = btf_type_skip_modifiers(btf, type->type, NULL);
415 if (!real_type)
416 return false;
417
418 if (BTF_INFO_KIND(real_type->info) != BTF_KIND_INT)
419 return false;
420
421 intdata = btf_type_int(real_type);
422 return !(BTF_INT_ENCODING(intdata) & BTF_INT_SIGNED)
423 && BTF_INT_BITS(intdata) == 8;
424}
425
426static bool btf_type_is_char_array(struct btf *btf, const struct btf_type *type)
427{
428 const struct btf_type *real_type;
429 const struct btf_array *array;
430 u32 intdata;
431
432 if (BTF_INFO_KIND(type->info) != BTF_KIND_ARRAY)
433 return false;
434
435 array = (const struct btf_array *)(type + 1);
436
437 real_type = btf_type_skip_modifiers(btf, array->type, NULL);
438
439 intdata = btf_type_int(real_type);
440 return !(BTF_INT_ENCODING(intdata) & BTF_INT_SIGNED)
441 && BTF_INT_BITS(intdata) == 8;
442}
443
444static struct btf *ctx_btf(struct traceprobe_parse_context *ctx)
445{
446 return ctx->struct_btf ? : ctx->btf;
447}
448
449static int check_prepare_btf_string_fetch(char *typename,
450 struct fetch_insn **pcode,
451 struct traceprobe_parse_context *ctx)
452{
453 struct btf *btf = ctx_btf(ctx);
454
455 if (!btf || !ctx->last_type)
456 return 0;
457
458 /* char [] does not need any change. */
459 if (btf_type_is_char_array(btf, ctx->last_type))
460 return 0;
461
462 /* char * requires dereference the pointer. */
463 if (btf_type_is_char_ptr(btf, ctx->last_type)) {
464 struct fetch_insn *code = *pcode + 1;
465
466 if (code->op == FETCH_OP_END) {
467 trace_probe_log_err(ctx->offset, TOO_MANY_OPS);
468 return -E2BIG;
469 }
470 if (typename[0] == 'u')
471 code->op = FETCH_OP_UDEREF;
472 else
473 code->op = FETCH_OP_DEREF;
474 code->offset = 0;
475 *pcode = code;
476 return 0;
477 }
478 /* Other types are not available for string */
479 trace_probe_log_err(ctx->offset, BAD_TYPE4STR);
480 return -EINVAL;
481}
482
483static const char *fetch_type_from_btf_type(struct btf *btf,
484 const struct btf_type *type,
485 struct traceprobe_parse_context *ctx)
486{
487 u32 intdata;
488
489 /* TODO: const char * could be converted as a string */
490 switch (BTF_INFO_KIND(type->info)) {
491 case BTF_KIND_ENUM:
492 /* enum is "int", so convert to "s32" */
493 return "s32";
494 case BTF_KIND_ENUM64:
495 return "s64";
496 case BTF_KIND_PTR:
497 /* pointer will be converted to "x??" */
498 return IS_ENABLED(CONFIG_64BIT) ? "x64" : "x32";
499 case BTF_KIND_INT:
500 intdata = btf_type_int(type);
501 if (BTF_INT_ENCODING(intdata) & BTF_INT_SIGNED) {
502 switch (BTF_INT_BITS(intdata)) {
503 case 8:
504 return "s8";
505 case 16:
506 return "s16";
507 case 32:
508 return "s32";
509 case 64:
510 return "s64";
511 }
512 } else { /* unsigned */
513 switch (BTF_INT_BITS(intdata)) {
514 case 8:
515 return "u8";
516 case 16:
517 return "u16";
518 case 32:
519 return "u32";
520 case 64:
521 return "u64";
522 }
523 /* bitfield, size is encoded in the type */
524 ctx->last_bitsize = BTF_INT_BITS(intdata);
525 ctx->last_bitoffs += BTF_INT_OFFSET(intdata);
526 return "u64";
527 }
528 }
529 /* TODO: support other types */
530
531 return NULL;
532}
533
534static int query_btf_context(struct traceprobe_parse_context *ctx)
535{
536 const struct btf_param *param;
537 const struct btf_type *type;
538 struct btf *btf;
539 s32 nr;
540
541 if (ctx->btf)
542 return 0;
543
544 if (!ctx->funcname)
545 return -EINVAL;
546
547 type = btf_find_func_proto(ctx->funcname, &btf);
548 if (!type)
549 return -ENOENT;
550
551 ctx->btf = btf;
552 ctx->proto = type;
553
554 /* ctx->params is optional, since func(void) will not have params. */
555 nr = 0;
556 param = btf_get_func_param(type, &nr);
557 if (!IS_ERR_OR_NULL(param)) {
558 /* Hide the first 'data' argument of tracepoint */
559 if (ctx->flags & TPARG_FL_TPOINT) {
560 nr--;
561 param++;
562 }
563 }
564
565 if (nr > 0) {
566 ctx->nr_params = nr;
567 ctx->params = param;
568 } else {
569 ctx->nr_params = 0;
570 ctx->params = NULL;
571 }
572
573 return 0;
574}
575
576static void clear_struct_btf(struct traceprobe_parse_context *ctx)
577{
578 if (ctx->struct_btf) {
579 btf_put(ctx->struct_btf);
580 ctx->struct_btf = NULL;
581 ctx->last_struct = NULL;
582 }
583}
584
585static void clear_btf_context(struct traceprobe_parse_context *ctx)
586{
587 if (ctx->btf) {
588 btf_put(ctx->btf);
589 ctx->btf = NULL;
590 ctx->proto = NULL;
591 ctx->params = NULL;
592 ctx->nr_params = 0;
593 }
594}
595
596/* Return 1 if the field separator is arrow operator ('->') */
597static int split_next_field(char *varname, char **next_field,
598 struct traceprobe_parse_context *ctx)
599{
600 char *field;
601 int ret = 0;
602
603 field = strpbrk(varname, ".-");
604 if (field) {
605 if (field[0] == '-' && field[1] == '>') {
606 field[0] = '\0';
607 field += 2;
608 ret = 1;
609 } else if (field[0] == '.') {
610 field[0] = '\0';
611 field += 1;
612 } else {
613 trace_probe_log_err(ctx->offset + field - varname, BAD_HYPHEN);
614 return -EINVAL;
615 }
616 *next_field = field;
617 }
618
619 return ret;
620}
621
622/* Inner loop for solving dot operator ('.'). Return bit-offset of the given field */
623static int get_bitoffset_of_field(char **pfieldname, const struct btf_type **ptype,
624 struct traceprobe_parse_context *ctx)
625{
626 const struct btf_type *type = *ptype;
627 const struct btf_member *field;
628 const struct btf_type *mtype;
629 struct btf *btf = ctx_btf(ctx);
630 char *fieldname = *pfieldname;
631 int bitoffs = 0;
632 u32 anon_offs;
633 char *next;
634 int is_ptr;
635
636 do {
637 next = NULL;
638 is_ptr = split_next_field(fieldname, &next, ctx);
639 if (is_ptr < 0)
640 return is_ptr;
641
642 anon_offs = 0;
643 field = btf_find_struct_member(btf, type, fieldname,
644 &anon_offs, &mtype);
645 if (IS_ERR(field)) {
646 trace_probe_log_err(ctx->offset, BAD_BTF_TID);
647 return PTR_ERR(field);
648 }
649 if (!field) {
650 trace_probe_log_err(ctx->offset, NO_BTF_FIELD);
651 return -ENOENT;
652 }
653 /* Add anonymous structure/union offset */
654 bitoffs += anon_offs;
655
656 /* Accumulate the bit-offsets of the dot-connected fields */
657 if (btf_type_kflag(mtype)) {
658 bitoffs += BTF_MEMBER_BIT_OFFSET(field->offset);
659 ctx->last_bitsize = BTF_MEMBER_BITFIELD_SIZE(field->offset);
660 } else {
661 bitoffs += field->offset;
662 ctx->last_bitsize = 0;
663 }
664
665 type = btf_type_skip_modifiers(btf, field->type, NULL);
666 if (!type) {
667 trace_probe_log_err(ctx->offset, BAD_BTF_TID);
668 return -EINVAL;
669 }
670
671 if (next)
672 ctx->offset += next - fieldname;
673 fieldname = next;
674 } while (!is_ptr && fieldname);
675
676 *pfieldname = fieldname;
677 *ptype = type;
678
679 return bitoffs;
680}
681/*
682 * Parse the field of data structure. The @type must be a pointer type
683 * pointing the target data structure type.
684 */
685static int parse_btf_field(char *fieldname, const struct btf_type *type,
686 struct fetch_insn **pcode, struct fetch_insn *end,
687 struct traceprobe_parse_context *ctx)
688{
689 struct fetch_insn *code = *pcode;
690 struct btf *btf = ctx_btf(ctx);
691 bool is_first_field = true;
692 int bitoffs;
693
694 do {
695 /* For the first field of typecast, @type will be the target structure type. */
696 if (!(is_first_field && ctx->struct_btf)) {
697 /* Outer loop for solving arrow operator ('->') */
698 if (BTF_INFO_KIND(type->info) != BTF_KIND_PTR) {
699 trace_probe_log_err(ctx->offset, NO_PTR_STRCT);
700 return -EINVAL;
701 }
702
703 /* Convert a struct pointer type to a struct type */
704 type = btf_type_skip_modifiers(btf, type->type, NULL);
705 if (!type) {
706 trace_probe_log_err(ctx->offset, BAD_BTF_TID);
707 return -EINVAL;
708 }
709 }
710
711 bitoffs = get_bitoffset_of_field(&fieldname, &type, ctx);
712 if (bitoffs < 0)
713 return bitoffs;
714 if (++code == end) {
715 trace_probe_log_err(ctx->offset, TOO_MANY_OPS);
716 return -EINVAL;
717 }
718 code->op = FETCH_OP_DEREF; /* TODO: user deref support */
719 code->offset = bitoffs / 8;
720 if (is_first_field && ctx->struct_btf) {
721 /* The first field can be typecasted with field option. */
722 code->offset -= ctx->prefix_byteoffs;
723 }
724 *pcode = code;
725
726 ctx->last_bitoffs = bitoffs % 8;
727 ctx->last_type = type;
728 is_first_field = false;
729 } while (fieldname);
730
731 return 0;
732}
733
734static int __store_entry_arg(struct trace_probe *tp, int argnum);
735
736static int parse_btf_arg(char *varname,
737 struct fetch_insn **pcode, struct fetch_insn *end,
738 struct traceprobe_parse_context *ctx)
739{
740 struct fetch_insn *code = *pcode;
741 const struct btf_param *params;
742 const struct btf_type *type;
743 char *field = NULL;
744 int i, is_ptr, ret;
745 u32 tid;
746
747 /* Note: field is not separated at this point, so check prefix. */
748 if (!str_has_prefix(varname, "$current") &&
749 !ctx->funcname && !(ctx->flags & TPARG_FL_TEVENT))
750 return -EINVAL;
751
752 is_ptr = split_next_field(varname, &field, ctx);
753 if (is_ptr < 0)
754 return is_ptr;
755 if (!is_ptr && field) {
756 /* dot-connected field on an argument is not supported. */
757 trace_probe_log_err(ctx->offset + field - varname,
758 NOSUP_DAT_ARG);
759 return -EOPNOTSUPP;
760 }
761
762 if (!strcmp(varname, "$current")) {
763 code->op = FETCH_OP_CURRENT;
764 /* If no typecast is specified for $current, use task_struct by default */
765 ret = bpf_find_btf_id("task_struct", BTF_KIND_STRUCT, &ctx->struct_btf);
766 if (ret < 0) {
767 trace_probe_log_err(ctx->offset, NO_BTF_ENTRY);
768 return -ENOENT;
769 }
770 tid = (u32)ret;
771 type = ctx->last_struct =
772 btf_type_skip_modifiers(ctx->struct_btf, tid, NULL);
773 goto found_type;
774 }
775
776 if (ctx->flags & TPARG_FL_RETURN && !strcmp(varname, "$retval")) {
777 code->op = FETCH_OP_RETVAL;
778 /* Check whether the function return type is not void, even with typecast. */
779 if (query_btf_context(ctx) == 0) {
780 if (ctx->proto->type == 0) {
781 trace_probe_log_err(ctx->offset, NO_RETVAL);
782 return -ENOENT;
783 }
784 tid = ctx->proto->type;
785 goto found;
786 }
787 if (field) {
788 trace_probe_log_err(ctx->offset + field - varname,
789 NO_BTF_ENTRY);
790 return -ENOENT;
791 }
792 return 0;
793 }
794
795 if (!ctx->btf) {
796 ret = query_btf_context(ctx);
797 if (ret < 0 || ctx->nr_params == 0) {
798 trace_probe_log_err(ctx->offset, NO_BTF_ENTRY);
799 return -ENOENT;
800 }
801 }
802 params = ctx->params;
803
804 for (i = 0; i < ctx->nr_params; i++) {
805 const char *name = btf_name_by_offset(ctx->btf, params[i].name_off);
806
807 if (name && !strcmp(name, varname)) {
808 if (tparg_is_function_entry(ctx->flags)) {
809 code->op = FETCH_OP_ARG;
810 if (ctx->flags & TPARG_FL_TPOINT)
811 code->param = i + 1;
812 else
813 code->param = i;
814 } else if (tparg_is_function_return(ctx->flags)) {
815 code->op = FETCH_OP_EDATA;
816 ret = __store_entry_arg(ctx->tp, i);
817 if (ret < 0) {
818 /* internal error */
819 return ret;
820 }
821 code->offset = ret;
822 }
823 tid = params[i].type;
824 goto found;
825 }
826 }
827 trace_probe_log_err(ctx->offset, NO_BTFARG);
828 return -ENOENT;
829
830found:
831 type = btf_type_skip_modifiers(ctx->btf, tid, NULL);
832found_type:
833 if (!type) {
834 trace_probe_log_err(ctx->offset, BAD_BTF_TID);
835 return -EINVAL;
836 }
837 /* Initialize the last type information */
838 ctx->last_type = type;
839 ctx->last_bitoffs = 0;
840 ctx->last_bitsize = 0;
841 if (field) {
842 ctx->offset += field - varname;
843 return parse_btf_field(field, type, pcode, end, ctx);
844 }
845 return 0;
846}
847
848static const struct fetch_type *find_fetch_type_from_btf_type(
849 struct traceprobe_parse_context *ctx)
850{
851 struct btf *btf = ctx_btf(ctx);
852 const char *typestr = NULL;
853
854 if (btf && ctx->last_type)
855 typestr = fetch_type_from_btf_type(btf, ctx->last_type, ctx);
856
857 return find_fetch_type(typestr, ctx->flags);
858}
859
860static int parse_btf_bitfield(struct fetch_insn **pcode,
861 struct traceprobe_parse_context *ctx)
862{
863 struct fetch_insn *code = *pcode;
864
865 if ((ctx->last_bitsize % 8 == 0) && ctx->last_bitoffs == 0)
866 return 0;
867
868 code++;
869 if (code->op != FETCH_OP_NOP) {
870 trace_probe_log_err(ctx->offset, TOO_MANY_OPS);
871 return -EINVAL;
872 }
873 *pcode = code;
874
875 code->op = FETCH_OP_MOD_BF;
876 code->lshift = 64 - (ctx->last_bitsize + ctx->last_bitoffs);
877 code->rshift = 64 - ctx->last_bitsize;
878 code->basesize = 64 / 8;
879 return 0;
880}
881
882static int query_btf_struct(const char *sname, struct traceprobe_parse_context *ctx)
883{
884 struct btf *btf = NULL;
885 int id;
886
887 /* A struct_btf should only be used by a single argument */
888 if (WARN_ON_ONCE(ctx->struct_btf)) {
889 btf_put(ctx->struct_btf);
890 ctx->struct_btf = NULL;
891 }
892
893 id = bpf_find_btf_id(sname, BTF_KIND_STRUCT, &btf);
894 if (id < 0)
895 return id;
896 ctx->struct_btf = btf;
897 ctx->last_struct = btf_type_by_id(ctx->struct_btf, id);
898 return 0;
899}
900
901static int parse_btf_casttype(char *casttype, struct traceprobe_parse_context *ctx)
902{
903 char *field;
904 int ret;
905
906 /* Field option - evaluated later. */
907 field = strchr(casttype, ',');
908 if (field)
909 *field++ = '\0';
910
911 ret = query_btf_struct(casttype, ctx);
912 if (ret < 0) {
913 trace_probe_log_err(ctx->offset, NO_PTR_STRCT);
914 return -EINVAL;
915 }
916
917 if (field) {
918 struct btf_type *type = (struct btf_type *)ctx->last_struct;
919
920 ctx->offset += field - casttype;
921 ret = get_bitoffset_of_field(&field, &ctx->last_struct, ctx);
922 if (ret < 0)
923 return ret;
924 if (ret % 8) {
925 trace_probe_log_err(ctx->offset, TYPECAST_NOT_ALIGNED);
926 return -EINVAL;
927 }
928 if (field != NULL) {
929 /* this means @field skips an arrow operator ("->"). */
930 trace_probe_log_err(ctx->offset - 2, TYPECAST_BAD_ARROW);
931 return -EINVAL;
932 }
933 ctx->prefix_byteoffs = ret / 8;
934 /* Restore the original struct type (overwritten by get_bitoffset_of_field) */
935 ctx->last_struct = type;
936 }
937
938 return ret;
939}
940
941/* Find the matching closing parenthesis for a given opening parenthesis. */
942static char *find_matched_close_paren(char *s)
943{
944 char *p = s;
945 int count = 0;
946
947 while (*p) {
948 if (*p == '(')
949 count++;
950 else if (*p == ')') {
951 if (--count == 0)
952 return p;
953 }
954 p++;
955 }
956 return NULL;
957}
958
959static int handle_typecast(char *arg, struct traceprobe_parse_context *ctx)
960{
961 int orig_offset = ctx->offset;
962 char *close;
963 char *tmp;
964 char *fieldname;
965
966 if (!(tparg_is_event_probe(ctx->flags) ||
967 tparg_is_function_entry(ctx->flags) ||
968 tparg_is_function_return(ctx->flags))) {
969 trace_probe_log_err(ctx->offset, NOSUP_BTFARG);
970 return -EOPNOTSUPP;
971 }
972
973 /*
974 * Always consider the token after typecast as a nested call
975 * For example: (STRUCT)VAR->FIELD and (STRUCT)(VAR)->FIELD are same.
976 * VAR is solved in the nested call.
977 */
978 tmp = strchr(arg, ')');
979 if (!tmp) {
980 trace_probe_log_err(ctx->offset + strlen(arg),
981 DEREF_OPEN_BRACE);
982 return -EINVAL;
983 }
984 *tmp++ = '\0';
985
986 ctx->offset += tmp - arg;
987 if (*tmp == '(') {
988 close = find_matched_close_paren(tmp);
989
990 if (!close) {
991 trace_probe_log_err(ctx->offset, DEREF_OPEN_BRACE);
992 return -EINVAL;
993 }
994 /* We expect a field access for typecast */
995 if (close[1] != '-' || close[2] != '>') {
996 trace_probe_log_err(ctx->offset + close - tmp + 1,
997 TYPECAST_REQ_FIELD);
998 return -EINVAL;
999 }
1000 /* Skip '(' */
1001 ctx->offset += 1;
1002 tmp++;
1003 } else if (*tmp == '+' || *tmp == '-' ||
1004 str_has_prefix(tmp, THIS_CPU_PTR_PREFIX) ||
1005 str_has_prefix(tmp, THIS_CPU_READ_PREFIX)) {
1006 /* Dereference can have another field access inside it. */
1007 char *open = strchr(tmp + 1, '(');
1008
1009 if (!open) {
1010 trace_probe_log_err(ctx->offset,
1011 DEREF_NEED_BRACE);
1012 return -EINVAL;
1013 }
1014 close = find_matched_close_paren(open);
1015 if (!close) {
1016 trace_probe_log_err(ctx->offset + strlen(tmp),
1017 DEREF_OPEN_BRACE);
1018 return -EINVAL;
1019 }
1020 close++;
1021 /* We expect a field access for typecast */
1022 if (close[0] != '-' || close[1] != '>') {
1023 trace_probe_log_err(ctx->offset + close - tmp,
1024 TYPECAST_REQ_FIELD);
1025 return -EINVAL;
1026 }
1027 } else {
1028 if (tmp[0] == '@') {
1029 /* @sym+offset is not allowed without parenthesized */
1030 close = strpbrk(tmp, "+-");
1031 if (close && isdigit(close[1])) {
1032 trace_probe_log_err(ctx->offset,
1033 TYPECAST_SYM_OFFSET);
1034 return -EINVAL;
1035 }
1036 }
1037 /* Inner variable name */
1038 close = strchr(tmp, '-');
1039 if (!close || close[1] != '>') {
1040 trace_probe_log_err(ctx->offset + strlen(tmp),
1041 TYPECAST_REQ_FIELD);
1042 return -EINVAL;
1043 }
1044 }
1045 *close = '\0';
1046
1047 /* Let fieldname point the field name. */
1048 if (close[1] == '-')
1049 fieldname = close + 3; /* Skip "->" after closing parenthesis */
1050 else
1051 fieldname = close + 2; /* Skip ">" after inner variable name */
1052
1053 ctx->stack[ctx->depth].type = STATE_TYPECAST;
1054 ctx->stack[ctx->depth].typecast.casttype = arg + 1;
1055 ctx->stack[ctx->depth].typecast.fieldname = fieldname;
1056 ctx->stack[ctx->depth].typecast.orig_offset = orig_offset;
1057 ctx->stack[ctx->depth].typecast.field_offset_diff = fieldname - arg;
1058 ctx->stack[ctx->depth].typecast.inner_arg = tmp;
1059 return 0;
1060}
1061
1062#else /* !CONFIG_PROBE_EVENTS_BTF_ARGS */
1063
1064static void clear_struct_btf(struct traceprobe_parse_context *ctx)
1065{
1066 ctx->struct_btf = NULL;
1067}
1068
1069static void clear_btf_context(struct traceprobe_parse_context *ctx)
1070{
1071 ctx->btf = NULL;
1072}
1073
1074static int query_btf_context(struct traceprobe_parse_context *ctx)
1075{
1076 return -EOPNOTSUPP;
1077}
1078
1079static int parse_btf_arg(char *varname,
1080 struct fetch_insn **pcode, struct fetch_insn *end,
1081 struct traceprobe_parse_context *ctx)
1082{
1083 trace_probe_log_err(ctx->offset, NOSUP_BTFARG);
1084 return -EOPNOTSUPP;
1085}
1086
1087static int parse_btf_bitfield(struct fetch_insn **pcode,
1088 struct traceprobe_parse_context *ctx)
1089{
1090 trace_probe_log_err(ctx->offset, NOSUP_BTFARG);
1091 return -EOPNOTSUPP;
1092}
1093
1094#define find_fetch_type_from_btf_type(ctx) \
1095 find_fetch_type(NULL, ctx->flags)
1096
1097static int check_prepare_btf_string_fetch(char *typename,
1098 struct fetch_insn **pcode,
1099 struct traceprobe_parse_context *ctx)
1100{
1101 return 0;
1102}
1103
1104static int parse_btf_casttype(char *casttype,
1105 struct traceprobe_parse_context *ctx)
1106{
1107 return -EOPNOTSUPP;
1108}
1109
1110static int parse_btf_field(char *fieldname, const struct btf_type *type,
1111 struct fetch_insn **pcode, struct fetch_insn *end,
1112 struct traceprobe_parse_context *ctx)
1113{
1114 return -EOPNOTSUPP;
1115}
1116
1117static int handle_typecast(char *arg, struct traceprobe_parse_context *ctx)
1118{
1119 trace_probe_log_err(ctx->offset, NOSUP_BTFARG);
1120 return -EOPNOTSUPP;
1121}
1122
1123#endif /* CONFIG_PROBE_EVENTS_BTF_ARGS */
1124
1125#ifdef CONFIG_HAVE_FUNCTION_ARG_ACCESS_API
1126
1127static void store_entry_arg_at(struct fetch_insn *code, int argnum, int offset)
1128{
1129 code[0].op = FETCH_OP_ARG;
1130 code[0].param = argnum;
1131 code[1].op = FETCH_OP_ST_EDATA;
1132 code[1].offset = offset;
1133}
1134
1135static int get_entry_arg_max_offset(struct probe_entry_arg *earg)
1136{
1137 int i, max_offset = 0;
1138
1139 /*
1140 * earg->code[] array has an operation sequence which is run in
1141 * the entry handler.
1142 * The sequence stopped by FETCH_OP_END and each data stored in
1143 * the entry data buffer by FETCH_OP_ST_EDATA. The FETCH_OP_ST_EDATA
1144 * stores the data at the data buffer + its offset, and all data are
1145 * "unsigned long" size. The offset must be increased when a data is
1146 * stored. Thus we need to find the last FETCH_OP_ST_EDATA in the
1147 * code array.
1148 */
1149 for (i = 0; i < earg->size - 1 && earg->code[i].op != FETCH_OP_END; i++) {
1150 if (earg->code[i].op == FETCH_OP_ST_EDATA)
1151 if (earg->code[i].offset > max_offset)
1152 max_offset = earg->code[i].offset;
1153 }
1154 return max_offset;
1155}
1156
1157/*
1158 * Add the entry code to store the 'argnum'th parameter and return the offset
1159 * in the entry data buffer where the data will be stored.
1160 */
1161static int __store_entry_arg(struct trace_probe *tp, int argnum)
1162{
1163 struct probe_entry_arg *earg = tp->entry_arg;
1164 int i, offset, last_offset = 0;
1165
1166 if (!earg) {
1167 earg = kzalloc_flex(*earg, code, 2 * tp->nr_args + 1);
1168 if (!earg)
1169 return -ENOMEM;
1170 earg->size = 2 * tp->nr_args + 1;
1171 /* Fill the code buffer with 'end' to simplify it */
1172 for (i = 0; i < earg->size; i++)
1173 earg->code[i].op = FETCH_OP_END;
1174 tp->entry_arg = earg;
1175 store_entry_arg_at(earg->code, argnum, 0);
1176 return 0;
1177 }
1178
1179 /*
1180 * NOTE: if anyone change the following rule, please rewrite this.
1181 * The entry code array is filled with the pair of
1182 *
1183 * [FETCH_OP_ARG(argnum)]
1184 * [FETCH_OP_ST_EDATA(offset of entry data buffer)]
1185 *
1186 * and the rest of entries are filled with [FETCH_OP_END].
1187 * The offset should be incremented, thus the last pair should
1188 * have the largest offset.
1189 */
1190
1191 /* Search the offset for the sprcified argnum. */
1192 for (i = 0; i < earg->size - 1 && earg->code[i].op != FETCH_OP_END; i += 2) {
1193 if (WARN_ON_ONCE(earg->code[i].op != FETCH_OP_ARG))
1194 return -EINVAL;
1195
1196 if (earg->code[i].param != argnum)
1197 continue;
1198
1199 if (WARN_ON_ONCE(earg->code[i + 1].op != FETCH_OP_ST_EDATA))
1200 return -EINVAL;
1201
1202 return earg->code[i + 1].offset;
1203 }
1204 /* Not found, append new entry if possible. */
1205 if (i >= earg->size - 1)
1206 return -ENOSPC;
1207
1208 /* The last entry must have the largest offset. */
1209 if (i != 0) {
1210 if (WARN_ON_ONCE(earg->code[i - 1].op != FETCH_OP_ST_EDATA))
1211 return -EINVAL;
1212 last_offset = earg->code[i - 1].offset;
1213 }
1214
1215 offset = last_offset + sizeof(unsigned long);
1216 store_entry_arg_at(&earg->code[i], argnum, offset);
1217 return offset;
1218}
1219
1220int traceprobe_get_entry_data_size(struct trace_probe *tp)
1221{
1222 struct probe_entry_arg *earg = tp->entry_arg;
1223
1224 if (!earg)
1225 return 0;
1226
1227 return get_entry_arg_max_offset(earg) + sizeof(unsigned long);
1228}
1229
1230void store_trace_entry_data(void *edata, struct trace_probe *tp, struct pt_regs *regs)
1231{
1232 struct probe_entry_arg *earg = tp->entry_arg;
1233 unsigned long val = 0;
1234 int i;
1235
1236 if (!earg)
1237 return;
1238
1239 for (i = 0; i < earg->size; i++) {
1240 struct fetch_insn *code = &earg->code[i];
1241
1242 switch (code->op) {
1243 case FETCH_OP_ARG:
1244 val = regs_get_kernel_argument(regs, code->param);
1245 break;
1246 case FETCH_OP_ST_EDATA:
1247 *(unsigned long *)((u8 *)edata + code->offset) = val;
1248 break;
1249 case FETCH_OP_END:
1250 goto end;
1251 default:
1252 break;
1253 }
1254 }
1255end:
1256 return;
1257}
1258NOKPROBE_SYMBOL(store_trace_entry_data)
1259#endif
1260
1261#define PARAM_MAX_STACK (THREAD_SIZE / sizeof(unsigned long))
1262
1263static int parse_probe_var_retval(char *orig_arg,
1264 struct fetch_insn **pcode,
1265 struct fetch_insn *end,
1266 struct traceprobe_parse_context *ctx)
1267{
1268 struct fetch_insn *code = *pcode;
1269
1270 if (!(ctx->flags & TPARG_FL_RETURN)) {
1271 trace_probe_log_err(ctx->offset, RETVAL_ON_PROBE);
1272 return -EINVAL;
1273 }
1274 if (!(ctx->flags & TPARG_FL_KERNEL) ||
1275 !IS_ENABLED(CONFIG_PROBE_EVENTS_BTF_ARGS)) {
1276 code->op = FETCH_OP_RETVAL;
1277 return 0;
1278 }
1279 return parse_btf_arg(orig_arg, pcode, end, ctx);
1280}
1281
1282static int parse_probe_var_stack(char *arg, int len, struct fetch_insn *code,
1283 struct traceprobe_parse_context *ctx)
1284{
1285 unsigned long param;
1286 int ret;
1287
1288 if (arg[len] == '\0') {
1289 code->op = FETCH_OP_STACKP;
1290 return 0;
1291 }
1292
1293 if (isdigit(arg[len])) {
1294 ret = kstrtoul(arg + len, 10, &param);
1295 if (ret) {
1296 trace_probe_log_err(ctx->offset, BAD_VAR);
1297 return ret;
1298 }
1299
1300 if ((ctx->flags & TPARG_FL_KERNEL) &&
1301 param > PARAM_MAX_STACK) {
1302 trace_probe_log_err(ctx->offset, BAD_STACK_NUM);
1303 return -EINVAL;
1304 }
1305 code->op = FETCH_OP_STACK;
1306 code->param = (unsigned int)param;
1307 return 0;
1308 }
1309
1310 trace_probe_log_err(ctx->offset, BAD_VAR);
1311 return -EINVAL;
1312}
1313
1314static int parse_probe_var_current(char *orig_arg, char *arg,
1315 struct fetch_insn **pcode,
1316 struct fetch_insn *end,
1317 struct traceprobe_parse_context *ctx)
1318{
1319 struct fetch_insn *code = *pcode;
1320
1321 /* $current is only supported by kernel probe. */
1322 if (!(ctx->flags & TPARG_FL_KERNEL)) {
1323 trace_probe_log_err(ctx->offset, BAD_VAR);
1324 return -EINVAL;
1325 }
1326 arg += strlen("current");
1327 if (*arg == '-' && IS_ENABLED(CONFIG_PROBE_EVENTS_BTF_ARGS))
1328 return parse_btf_arg(orig_arg, pcode, end, ctx);
1329
1330 if (*arg != '\0') {
1331 trace_probe_log_err(ctx->offset, BAD_VAR);
1332 return -EINVAL;
1333 }
1334
1335 code->op = FETCH_OP_CURRENT;
1336 return 0;
1337}
1338
1339#ifdef CONFIG_HAVE_FUNCTION_ARG_ACCESS_API
1340static int parse_probe_var_arg(char *arg, int len, struct fetch_insn *code,
1341 struct traceprobe_parse_context *ctx)
1342{
1343 unsigned long param;
1344 int ret;
1345
1346 ret = kstrtoul(arg + len, 10, &param);
1347 if (ret) {
1348 trace_probe_log_err(ctx->offset, BAD_VAR);
1349 return ret;
1350 }
1351
1352 if (!param || param > PARAM_MAX_STACK) {
1353 trace_probe_log_err(ctx->offset, BAD_ARG_NUM);
1354 return -EINVAL;
1355 }
1356 param--; /* argN starts from 1, but internal arg[N] starts from 0 */
1357
1358 if (tparg_is_function_entry(ctx->flags)) {
1359 code->op = FETCH_OP_ARG;
1360 code->param = (unsigned int)param;
1361 /*
1362 * The tracepoint probe will probe a stub function, and the
1363 * first parameter of the stub is a dummy and should be ignored.
1364 */
1365 if (ctx->flags & TPARG_FL_TPOINT)
1366 code->param++;
1367 } else if (tparg_is_function_return(ctx->flags)) {
1368 /* function entry argument access from return probe */
1369 ret = __store_entry_arg(ctx->tp, param);
1370 if (ret < 0) /* This error should be an internal error */
1371 return ret;
1372
1373 code->op = FETCH_OP_EDATA;
1374 code->offset = ret;
1375 } else {
1376 trace_probe_log_err(ctx->offset, NOFENTRY_ARGS);
1377 return -EINVAL;
1378 }
1379 return 0;
1380}
1381#else
1382static int parse_probe_var_arg(char *arg, int len, struct fetch_insn *code,
1383 struct traceprobe_parse_context *ctx)
1384{
1385 trace_probe_log_err(ctx->offset, BAD_VAR);
1386 return -EINVAL;
1387}
1388#endif
1389
1390/* Parse $vars. @orig_arg points '$', which syncs to @ctx->offset */
1391static int parse_probe_vars(char *orig_arg, const struct fetch_type *t,
1392 struct fetch_insn **pcode,
1393 struct fetch_insn *end,
1394 struct traceprobe_parse_context *ctx)
1395{
1396 struct fetch_insn *code = *pcode;
1397 char *arg = orig_arg + 1;
1398 int len, ret;
1399
1400 if (ctx->flags & TPARG_FL_TEVENT) {
1401 ret = parse_trace_event(arg, code, ctx);
1402 if (!ret)
1403 return 0;
1404 }
1405
1406 if (strcmp(arg, "comm") == 0 || strcmp(arg, "COMM") == 0) {
1407 code->op = FETCH_OP_COMM;
1408 return 0;
1409 }
1410
1411 /* eprobe only support event fields or '$comm'. */
1412 if (ctx->flags & TPARG_FL_TEVENT)
1413 goto inval;
1414
1415 if (str_has_prefix(arg, "retval"))
1416 return parse_probe_var_retval(orig_arg, pcode, end, ctx);
1417
1418 len = str_has_prefix(arg, "stack");
1419 if (len)
1420 return parse_probe_var_stack(arg, len, code, ctx);
1421
1422 /* $current returns the address of the current task_struct. */
1423 if (str_has_prefix(arg, "current"))
1424 return parse_probe_var_current(orig_arg, arg, pcode, end, ctx);
1425
1426 len = str_has_prefix(arg, "arg");
1427 if (len)
1428 return parse_probe_var_arg(arg, len, code, ctx);
1429
1430inval:
1431 trace_probe_log_err(ctx->offset, BAD_VAR);
1432 return -EINVAL;
1433}
1434
1435static int str_to_immediate(char *str, unsigned long *imm)
1436{
1437 if (isdigit(str[0]))
1438 return kstrtoul(str, 0, imm);
1439 else if (str[0] == '-')
1440 return kstrtol(str, 0, (long *)imm);
1441 else if (str[0] == '+')
1442 return kstrtol(str + 1, 0, (long *)imm);
1443 return -EINVAL;
1444}
1445
1446static int __parse_imm_string(char *str, char **pbuf, int offs)
1447{
1448 size_t len = strlen(str);
1449
1450 if (!len || str[len - 1] != '"') {
1451 trace_probe_log_err(offs + len, IMMSTR_NO_CLOSE);
1452 return -EINVAL;
1453 }
1454 *pbuf = kstrndup(str, len - 1, GFP_KERNEL);
1455 if (!*pbuf)
1456 return -ENOMEM;
1457 return 0;
1458}
1459
1460static int parse_probe_arg_register(char *arg, struct fetch_insn *code,
1461 struct traceprobe_parse_context *ctx)
1462{
1463 int ret;
1464
1465 if (ctx->flags & (TPARG_FL_TEVENT | TPARG_FL_FPROBE)) {
1466 /* eprobe and fprobe do not handle registers */
1467 trace_probe_log_err(ctx->offset, BAD_VAR);
1468 return -EINVAL;
1469 }
1470 ret = regs_query_register_offset(arg + 1);
1471 if (ret >= 0) {
1472 code->op = FETCH_OP_REG;
1473 code->param = (unsigned int)ret;
1474 return 0;
1475 }
1476 trace_probe_log_err(ctx->offset, BAD_REG_NAME);
1477 return -EINVAL;
1478}
1479
1480static int parse_probe_arg_mem_symbol(char *arg, struct fetch_insn **pcode,
1481 struct fetch_insn *end,
1482 struct traceprobe_parse_context *ctx)
1483{
1484 struct fetch_insn *code = *pcode;
1485 unsigned long param;
1486 long offset = 0;
1487 int ret;
1488
1489 if (isdigit(arg[1])) {
1490 ret = kstrtoul(arg + 1, 0, &param);
1491 if (ret) {
1492 trace_probe_log_err(ctx->offset, BAD_MEM_ADDR);
1493 return ret;
1494 }
1495 /* load address */
1496 code->op = FETCH_OP_IMM;
1497 code->immediate = param;
1498 } else if (arg[1] == '+') {
1499 /* Kernel probes do not support file offsets */
1500 if (ctx->flags & TPARG_FL_KERNEL) {
1501 trace_probe_log_err(ctx->offset, FILE_ON_KPROBE);
1502 return -EINVAL;
1503 }
1504 ret = kstrtol(arg + 2, 0, &offset);
1505 if (ret) {
1506 trace_probe_log_err(ctx->offset, BAD_FILE_OFFS);
1507 return ret;
1508 }
1509
1510 code->op = FETCH_OP_FOFFS;
1511 code->immediate = (unsigned long)offset;
1512 offset = 0;
1513 } else {
1514 /* uprobes don't support symbols */
1515 if (!(ctx->flags & TPARG_FL_KERNEL)) {
1516 trace_probe_log_err(ctx->offset, SYM_ON_UPROBE);
1517 return -EINVAL;
1518 }
1519 /* Preserve symbol for updating */
1520 code->op = FETCH_NOP_SYMBOL;
1521 code->data = kstrdup(arg + 1, GFP_KERNEL);
1522 if (!code->data)
1523 return -ENOMEM;
1524 if (++code == end) {
1525 trace_probe_log_err(ctx->offset, TOO_MANY_OPS);
1526 return -EINVAL;
1527 }
1528 code->op = FETCH_OP_IMM;
1529 code->immediate = 0;
1530 }
1531 /* These are fetching from memory */
1532 if (++code == end) {
1533 trace_probe_log_err(ctx->offset, TOO_MANY_OPS);
1534 return -EINVAL;
1535 }
1536 *pcode = code;
1537 code->op = FETCH_OP_DEREF;
1538 code->offset = offset;
1539 return 0;
1540}
1541
1542static int parse_probe_arg_deref(char *arg, struct traceprobe_parse_context *ctx)
1543{
1544 int deref = FETCH_OP_DEREF;
1545 long offset = 0;
1546 char *tmp;
1547 int ret;
1548
1549 if (arg[1] == 'u') {
1550 deref = FETCH_OP_UDEREF;
1551 arg[1] = arg[0];
1552 arg++;
1553 }
1554 if (arg[0] == '+')
1555 arg++; /* Skip '+', because kstrtol() rejects it. */
1556 tmp = strchr(arg, '(');
1557 if (!tmp) {
1558 trace_probe_log_err(ctx->offset, DEREF_NEED_BRACE);
1559 return -EINVAL;
1560 }
1561 *tmp = '\0';
1562 ret = kstrtol(arg, 0, &offset);
1563 if (ret) {
1564 trace_probe_log_err(ctx->offset, BAD_DEREF_OFFS);
1565 return ret;
1566 }
1567 ctx->offset += (tmp + 1 - arg) + (arg[0] != '-' ? 1 : 0);
1568 arg = tmp + 1;
1569
1570 tmp = strrchr(arg, ')');
1571 if (!tmp) {
1572 trace_probe_log_err(ctx->offset + strlen(arg),
1573 DEREF_OPEN_BRACE);
1574 return -EINVAL;
1575 }
1576 *tmp = '\0';
1577
1578 ctx->stack[ctx->depth].type = STATE_DEREF;
1579 ctx->stack[ctx->depth].deref.deref = deref;
1580 ctx->stack[ctx->depth].deref.offset = offset;
1581 ctx->stack[ctx->depth].deref.cur_offs = ctx->offset;
1582 ctx->stack[ctx->depth].deref.inner_arg = arg;
1583 ctx->stack[ctx->depth].deref.is_cpu_read = false;
1584 return 0;
1585}
1586
1587static int parse_probe_arg_imm(char *arg, struct fetch_insn *code,
1588 struct traceprobe_parse_context *ctx)
1589{
1590 char *tmp;
1591 int ret;
1592
1593 if (arg[1] == '"') { /* Immediate string */
1594 ret = __parse_imm_string(arg + 2, &tmp, ctx->offset + 2);
1595 if (ret)
1596 return ret;
1597 code->op = FETCH_OP_IMMSTR;
1598 code->data = tmp;
1599 } else {
1600 ret = str_to_immediate(arg + 1, &code->immediate);
1601 if (ret) {
1602 trace_probe_log_err(ctx->offset + 1, BAD_IMM);
1603 return ret;
1604 }
1605 code->op = FETCH_OP_IMM;
1606 }
1607 return 0;
1608}
1609
1610static int parse_probe_arg_default(char *arg, struct fetch_insn **pcode,
1611 struct fetch_insn *end,
1612 struct traceprobe_parse_context *ctx)
1613{
1614 int ret;
1615
1616 if (isalpha(arg[0]) || arg[0] == '_') {
1617 /* BTF variable or event field */
1618 if (ctx->flags & TPARG_FL_TEVENT) {
1619 ret = parse_trace_event(arg, *pcode, ctx);
1620 if (ret < 0) {
1621 trace_probe_log_err(ctx->offset, NO_EVENT_FIELD);
1622 return -EINVAL;
1623 }
1624 return 0;
1625 }
1626 if (!tparg_is_function_entry(ctx->flags) &&
1627 !tparg_is_function_return(ctx->flags)) {
1628 trace_probe_log_err(ctx->offset, NOSUP_BTFARG);
1629 return -EINVAL;
1630 }
1631 return parse_btf_arg(arg, pcode, end, ctx);
1632 }
1633
1634 return 0;
1635}
1636
1637static int parse_probe_arg_nested(char **parg, struct traceprobe_parse_context *ctx)
1638{
1639 char *arg = *parg;
1640 int ret;
1641
1642 while (true) {
1643 /* Determine if this is a nested argument */
1644 if (arg[0] != '+' && arg[0] != '-' && arg[0] != '(' &&
1645 !str_has_prefix(arg, THIS_CPU_PTR_PREFIX) &&
1646 !str_has_prefix(arg, THIS_CPU_READ_PREFIX))
1647 break;
1648
1649 /* If nested, check the maximum depth limit */
1650 if (ctx->depth >= TRACEPROBE_MAX_NESTED_LEVEL) {
1651 trace_probe_log_err(ctx->offset, TOO_MANY_NESTED);
1652 return -E2BIG;
1653 }
1654
1655 /* Perform the actual parsing subroutine calls */
1656 switch (arg[0]) {
1657 case '+':
1658 case '-':
1659 ret = parse_probe_arg_deref(arg, ctx);
1660 if (ret)
1661 return ret;
1662 arg = ctx->stack[ctx->depth].deref.inner_arg;
1663 break;
1664 case '(':
1665 ret = handle_typecast(arg, ctx);
1666 if (ret)
1667 return ret;
1668 arg = ctx->stack[ctx->depth].typecast.inner_arg;
1669 break;
1670 default:
1671 ret = parse_this_cpu(arg, ctx);
1672 if (ret)
1673 return ret;
1674 arg = ctx->stack[ctx->depth].deref.inner_arg;
1675 break;
1676 }
1677 ctx->depth++;
1678 }
1679
1680 *parg = arg;
1681 return 0;
1682}
1683
1684static int parse_probe_arg_leaf(char *arg, const struct fetch_type *type,
1685 struct fetch_insn **pcode, struct fetch_insn *end,
1686 struct traceprobe_parse_context *ctx)
1687{
1688 struct fetch_insn *code = *pcode;
1689 int ret;
1690
1691 switch (arg[0]) {
1692 case '$':
1693 ret = parse_probe_vars(arg, type, pcode, end, ctx);
1694 break;
1695 case '%': /* named register */
1696 ret = parse_probe_arg_register(arg, code, ctx);
1697 break;
1698 case '@': /* memory, file-offset or symbol */
1699 ret = parse_probe_arg_mem_symbol(arg, pcode, end, ctx);
1700 break;
1701 case '\\': /* Immediate value */
1702 ret = parse_probe_arg_imm(arg, code, ctx);
1703 break;
1704 default:
1705 ret = parse_probe_arg_default(arg, pcode, end, ctx);
1706 break;
1707 }
1708
1709 if (ret)
1710 return ret;
1711
1712 if (code->op == FETCH_OP_NOP) {
1713 /* Parsed, but do not find fetch method */
1714 trace_probe_log_err(ctx->offset, BAD_FETCH_ARG);
1715 return -EINVAL;
1716 }
1717
1718 return 0;
1719}
1720
1721static int unwind_parse_states(struct fetch_insn **pcode, struct fetch_insn *end,
1722 struct traceprobe_parse_context *ctx)
1723{
1724 struct parse_state *state;
1725 struct fetch_insn *code;
1726 int ret;
1727
1728 while (ctx->depth > 0) {
1729 ctx->depth--;
1730 state = &ctx->stack[ctx->depth];
1731
1732 if (state->type == STATE_DEREF) {
1733 code = *pcode;
1734 ctx->offset = state->deref.cur_offs;
1735 if (code->op == FETCH_OP_COMM || code->op == FETCH_OP_IMMSTR) {
1736 trace_probe_log_err(ctx->offset, COMM_CANT_DEREF);
1737 return -EINVAL;
1738 }
1739
1740 if (!(state->deref.deref == FETCH_OP_CPU_PTR &&
1741 *state->deref.inner_arg == '@')) {
1742 code++;
1743 if (code == end) {
1744 trace_probe_log_err(ctx->offset, TOO_MANY_OPS);
1745 return -EINVAL;
1746 }
1747 }
1748 *pcode = code;
1749
1750 code->op = state->deref.deref;
1751 code->offset = state->deref.offset;
1752 ctx->last_type = NULL;
1753
1754 if (state->deref.is_cpu_read) {
1755 code = *pcode;
1756 code++;
1757 if (code == end) {
1758 trace_probe_log_err(ctx->offset, TOO_MANY_OPS);
1759 return -EINVAL;
1760 }
1761 code->op = FETCH_OP_DEREF;
1762 code->offset = 0;
1763 *pcode = code;
1764 }
1765 } else if (state->type == STATE_TYPECAST) {
1766 clear_struct_btf(ctx);
1767
1768 /* resolve the typecast struct name */
1769 ctx->offset = state->typecast.orig_offset + 1; /* for the '(' */
1770 ret = parse_btf_casttype(state->typecast.casttype, ctx);
1771 if (ret < 0)
1772 return ret;
1773
1774 ctx->offset = state->typecast.orig_offset +
1775 state->typecast.field_offset_diff;
1776 ret = parse_btf_field(state->typecast.fieldname,
1777 ctx->last_struct, pcode,
1778 end, ctx);
1779 ctx->prefix_byteoffs = 0;
1780 if (ret < 0)
1781 return ret;
1782 }
1783 }
1784
1785 return 0;
1786}
1787
1788/* Loop-based (non-recursive) argument parser */
1789static int
1790parse_probe_arg(char *arg, const struct fetch_type *type,
1791 struct fetch_insn **pcode, struct fetch_insn *end,
1792 struct traceprobe_parse_context *ctx)
1793{
1794 int ret;
1795
1796 ctx->depth = 0;
1797
1798 ret = parse_probe_arg_nested(&arg, ctx);
1799 if (ret)
1800 return ret;
1801
1802 ret = parse_probe_arg_leaf(arg, type, pcode, end, ctx);
1803 if (ret)
1804 return ret;
1805
1806 return unwind_parse_states(pcode, end, ctx);
1807}
1808
1809/* Bitfield type needs to be parsed into a fetch function */
1810static int __parse_bitfield_probe_arg(const char *bf,
1811 const struct fetch_type *t,
1812 struct fetch_insn **pcode)
1813{
1814 struct fetch_insn *code = *pcode;
1815 unsigned long bw, bo;
1816 char *tail;
1817
1818 if (*bf != 'b')
1819 return 0;
1820
1821 bw = simple_strtoul(bf + 1, &tail, 0); /* Use simple one */
1822
1823 if (bw == 0 || *tail != '@')
1824 return -EINVAL;
1825
1826 bf = tail + 1;
1827 bo = simple_strtoul(bf, &tail, 0);
1828
1829 if (tail == bf || *tail != '/')
1830 return -EINVAL;
1831 code++;
1832 if (code->op != FETCH_OP_NOP)
1833 return -EINVAL;
1834 *pcode = code;
1835
1836 code->op = FETCH_OP_MOD_BF;
1837 code->lshift = BYTES_TO_BITS(t->size) - (bw + bo);
1838 code->rshift = BYTES_TO_BITS(t->size) - bw;
1839 code->basesize = t->size;
1840
1841 return (BYTES_TO_BITS(t->size) < (bw + bo)) ? -EINVAL : 0;
1842}
1843
1844/* Split type part from @arg and return it. */
1845static char *parse_probe_arg_type(char *arg, struct probe_arg *parg,
1846 struct traceprobe_parse_context *ctx)
1847{
1848 char *t = NULL, *t2, *t3;
1849 int offs;
1850
1851 t = strchr(arg, ':');
1852 if (t) {
1853 *t++ = '\0';
1854 t2 = strchr(t, '[');
1855 if (t2) {
1856 *t2++ = '\0';
1857 t3 = strchr(t2, ']');
1858 if (!t3) {
1859 offs = t2 + strlen(t2) - arg;
1860
1861 trace_probe_log_err(ctx->offset + offs,
1862 ARRAY_NO_CLOSE);
1863 return ERR_PTR(-EINVAL);
1864 } else if (t3[1] != '\0') {
1865 trace_probe_log_err(ctx->offset + t3 + 1 - arg,
1866 BAD_ARRAY_SUFFIX);
1867 return ERR_PTR(-EINVAL);
1868 }
1869 *t3 = '\0';
1870 if (kstrtouint(t2, 0, &parg->count) || !parg->count) {
1871 trace_probe_log_err(ctx->offset + t2 - arg,
1872 BAD_ARRAY_NUM);
1873 return ERR_PTR(-EINVAL);
1874 }
1875 if (parg->count > MAX_ARRAY_LEN) {
1876 trace_probe_log_err(ctx->offset + t2 - arg,
1877 ARRAY_TOO_BIG);
1878 return ERR_PTR(-EINVAL);
1879 }
1880 }
1881 }
1882 offs = t ? t - arg : 0;
1883
1884 /*
1885 * Since $comm and immediate string can not be dereferenced,
1886 * we can find those by strcmp. But ignore for eprobes.
1887 */
1888 if (!(ctx->flags & TPARG_FL_TEVENT) &&
1889 (strcmp(arg, "$comm") == 0 || strcmp(arg, "$COMM") == 0 ||
1890 strncmp(arg, "\\\"", 2) == 0)) {
1891 /* The type of $comm must be "string", and not an array type. */
1892 if (parg->count || (t && strcmp(t, "string"))) {
1893 trace_probe_log_err(ctx->offset + offs, NEED_STRING_TYPE);
1894 return ERR_PTR(-EINVAL);
1895 }
1896 parg->type = find_fetch_type("string", ctx->flags);
1897 } else
1898 parg->type = find_fetch_type(t, ctx->flags);
1899
1900 if (!parg->type) {
1901 trace_probe_log_err(ctx->offset + offs, BAD_TYPE);
1902 return ERR_PTR(-EINVAL);
1903 }
1904
1905 return t;
1906}
1907
1908/* After parsing, adjust the fetch_insn according to the probe_arg */
1909static int finalize_fetch_insn(struct fetch_insn *code,
1910 struct probe_arg *parg,
1911 char *type,
1912 int type_offset,
1913 struct traceprobe_parse_context *ctx)
1914{
1915 struct fetch_insn *scode;
1916 int ret;
1917
1918 /* Store operation */
1919 if (parg->type->is_string) {
1920 /* Check bad combination of the type and the last fetch_insn. */
1921 if (!strcmp(parg->type->name, "symstr")) {
1922 if (code->op != FETCH_OP_REG && code->op != FETCH_OP_STACK &&
1923 code->op != FETCH_OP_RETVAL && code->op != FETCH_OP_ARG &&
1924 code->op != FETCH_OP_DEREF && code->op != FETCH_OP_TP_ARG) {
1925 trace_probe_log_err(ctx->offset + type_offset,
1926 BAD_SYMSTRING);
1927 return -EINVAL;
1928 }
1929 } else {
1930 if (code->op != FETCH_OP_DEREF && code->op != FETCH_OP_UDEREF &&
1931 code->op != FETCH_OP_IMM && code->op != FETCH_OP_COMM &&
1932 code->op != FETCH_OP_IMMSTR && code->op != FETCH_OP_TP_ARG) {
1933 trace_probe_log_err(ctx->offset + type_offset,
1934 BAD_STRING);
1935 return -EINVAL;
1936 }
1937 }
1938
1939 if (!strcmp(parg->type->name, "symstr") ||
1940 (code->op == FETCH_OP_IMM || code->op == FETCH_OP_COMM ||
1941 code->op == FETCH_OP_IMMSTR) || code->op == FETCH_OP_TP_ARG ||
1942 parg->count) {
1943 /*
1944 * IMM, DATA and COMM is pointing actual address, those
1945 * must be kept, and if parg->count != 0, this is an
1946 * array of string pointers instead of string address
1947 * itself.
1948 * For the symstr, it doesn't need to dereference, thus
1949 * it just get the value.
1950 */
1951 code++;
1952 if (code->op != FETCH_OP_NOP) {
1953 trace_probe_log_err(ctx->offset, TOO_MANY_OPS);
1954 return -EINVAL;
1955 }
1956 }
1957
1958 /* If op == DEREF, replace it with STRING */
1959 if (!strcmp(parg->type->name, "ustring") ||
1960 code->op == FETCH_OP_UDEREF)
1961 code->op = FETCH_OP_ST_USTRING;
1962 else if (!strcmp(parg->type->name, "symstr"))
1963 code->op = FETCH_OP_ST_SYMSTR;
1964 else
1965 code->op = FETCH_OP_ST_STRING;
1966 code->size = parg->type->size;
1967 parg->dynamic = true;
1968 } else if (code->op == FETCH_OP_DEREF) {
1969 code->op = FETCH_OP_ST_MEM;
1970 code->size = parg->type->size;
1971 } else if (code->op == FETCH_OP_UDEREF) {
1972 code->op = FETCH_OP_ST_UMEM;
1973 code->size = parg->type->size;
1974 } else {
1975 code++;
1976 if (code->op != FETCH_OP_NOP) {
1977 trace_probe_log_err(ctx->offset, TOO_MANY_OPS);
1978 return -E2BIG;
1979 }
1980 code->op = FETCH_OP_ST_RAW;
1981 code->size = parg->type->size;
1982 }
1983
1984 /* Save storing fetch_insn. */
1985 scode = code;
1986
1987 /* Modify operation */
1988 if (type != NULL) {
1989 /* Bitfield needs a special fetch_insn. */
1990 ret = __parse_bitfield_probe_arg(type, parg->type, &code);
1991 if (ret) {
1992 trace_probe_log_err(ctx->offset + type_offset, BAD_BITFIELD);
1993 return ret;
1994 }
1995 } else if (IS_ENABLED(CONFIG_PROBE_EVENTS_BTF_ARGS) &&
1996 ctx->last_type) {
1997 /* If user not specified the type, try parsing BTF bitfield. */
1998 ret = parse_btf_bitfield(&code, ctx);
1999 if (ret)
2000 return ret;
2001 }
2002
2003 /* Loop(Array) operation */
2004 if (parg->count) {
2005 if (scode->op != FETCH_OP_ST_MEM &&
2006 scode->op != FETCH_OP_ST_STRING &&
2007 scode->op != FETCH_OP_ST_USTRING) {
2008 trace_probe_log_err(ctx->offset + type_offset, BAD_STRING);
2009 return -EINVAL;
2010 }
2011 code++;
2012 if (code->op != FETCH_OP_NOP) {
2013 trace_probe_log_err(ctx->offset, TOO_MANY_OPS);
2014 return -E2BIG;
2015 }
2016 code->op = FETCH_OP_LP_ARRAY;
2017 code->param = parg->count;
2018 }
2019
2020 /* Finalize the fetch_insn array. */
2021 code++;
2022 code->op = FETCH_OP_END;
2023
2024 return 0;
2025}
2026
2027/* String length checking wrapper */
2028static int traceprobe_parse_probe_arg_body(const char *argv, ssize_t *size,
2029 struct probe_arg *parg,
2030 struct traceprobe_parse_context *ctx)
2031{
2032 struct fetch_insn *code, *tmp = NULL;
2033 char *type, *arg __free(kfree) = NULL;
2034 int ret, len;
2035
2036 len = strlen(argv);
2037 if (len > MAX_ARGSTR_LEN) {
2038 trace_probe_log_err(ctx->offset, ARG_TOO_LONG);
2039 return -E2BIG;
2040 } else if (len == 0) {
2041 trace_probe_log_err(ctx->offset, NO_ARG_BODY);
2042 return -EINVAL;
2043 }
2044
2045 arg = kstrdup(argv, GFP_KERNEL);
2046 if (!arg)
2047 return -ENOMEM;
2048
2049 parg->comm = kstrdup(arg, GFP_KERNEL);
2050 if (!parg->comm)
2051 return -ENOMEM;
2052
2053 type = parse_probe_arg_type(arg, parg, ctx);
2054 if (IS_ERR(type))
2055 return PTR_ERR(type);
2056
2057 code = tmp = kzalloc_objs(*code, FETCH_INSN_MAX);
2058 if (!code)
2059 return -ENOMEM;
2060 code[FETCH_INSN_MAX - 1].op = FETCH_OP_END;
2061
2062 ctx->last_type = NULL;
2063 ret = parse_probe_arg(arg, parg->type, &code, &code[FETCH_INSN_MAX - 1],
2064 ctx);
2065 if (ret < 0)
2066 goto fail;
2067 /* Update storing type if BTF is available */
2068 if (IS_ENABLED(CONFIG_PROBE_EVENTS_BTF_ARGS) &&
2069 ctx->last_type) {
2070 if (!type) {
2071 parg->type = find_fetch_type_from_btf_type(ctx);
2072 } else if (strstr(type, "string")) {
2073 ret = check_prepare_btf_string_fetch(type, &code, ctx);
2074 if (ret)
2075 goto fail;
2076 }
2077 }
2078 parg->offset = *size;
2079 *size += parg->type->size * (parg->count ?: 1);
2080
2081 if (*size > MAX_PROBE_EVENT_SIZE) {
2082 ret = -E2BIG;
2083 trace_probe_log_err(ctx->offset, EVENT_TOO_BIG);
2084 goto fail;
2085 }
2086
2087 if (parg->count) {
2088 len = strlen(parg->type->fmttype) + 6;
2089 parg->fmt = kmalloc(len, GFP_KERNEL);
2090 if (!parg->fmt) {
2091 ret = -ENOMEM;
2092 goto fail;
2093 }
2094 snprintf(parg->fmt, len, "%s[%d]", parg->type->fmttype,
2095 parg->count);
2096 }
2097
2098 ret = finalize_fetch_insn(code, parg, type, type ? type - arg : 0, ctx);
2099 if (ret < 0)
2100 goto fail;
2101
2102 for (; code < tmp + FETCH_INSN_MAX; code++)
2103 if (code->op == FETCH_OP_END)
2104 break;
2105 /* Shrink down the code buffer */
2106 parg->code = kzalloc_objs(*code, code - tmp + 1);
2107 if (!parg->code)
2108 ret = -ENOMEM;
2109 else
2110 memcpy(parg->code, tmp, sizeof(*code) * (code - tmp + 1));
2111
2112fail:
2113 if (ret < 0) {
2114 for (code = tmp; code < tmp + FETCH_INSN_MAX; code++)
2115 if (code->op == FETCH_NOP_SYMBOL ||
2116 code->op == FETCH_OP_IMMSTR)
2117 kfree(code->data);
2118 }
2119 kfree(tmp);
2120
2121 /* struct_btf should not be passed to other arguments */
2122 clear_struct_btf(ctx);
2123
2124 return ret;
2125}
2126
2127/* Return 1 if name is reserved or already used by another argument */
2128static int traceprobe_conflict_field_name(const char *name,
2129 struct probe_arg *args, int narg)
2130{
2131 int i;
2132
2133 for (i = 0; i < ARRAY_SIZE(reserved_field_names); i++)
2134 if (strcmp(reserved_field_names[i], name) == 0)
2135 return 1;
2136
2137 for (i = 0; i < narg; i++)
2138 if (strcmp(args[i].name, name) == 0)
2139 return 1;
2140
2141 return 0;
2142}
2143
2144static char *generate_probe_arg_name(const char *arg, int idx)
2145{
2146 char *name = NULL;
2147 const char *end;
2148
2149 /*
2150 * If argument name is omitted, try arg as a name (BTF variable)
2151 * or "argN".
2152 */
2153 if (IS_ENABLED(CONFIG_PROBE_EVENTS_BTF_ARGS)) {
2154 end = strchr(arg, ':');
2155 if (!end)
2156 end = arg + strlen(arg);
2157
2158 name = kmemdup_nul(arg, end - arg, GFP_KERNEL);
2159 if (!name || !is_good_name(name)) {
2160 kfree(name);
2161 name = NULL;
2162 }
2163 }
2164
2165 if (!name)
2166 name = kasprintf(GFP_KERNEL, "arg%d", idx + 1);
2167
2168 return name;
2169}
2170
2171int traceprobe_parse_probe_arg(struct trace_probe *tp, int i, const char *arg,
2172 struct traceprobe_parse_context *ctx)
2173{
2174 struct probe_arg *parg = &tp->args[i];
2175 const char *body;
2176
2177 ctx->tp = tp;
2178 body = strchr(arg, '=');
2179 if (body) {
2180 if (body - arg > MAX_ARG_NAME_LEN) {
2181 trace_probe_log_err(0, ARG_NAME_TOO_LONG);
2182 return -EINVAL;
2183 } else if (body == arg) {
2184 trace_probe_log_err(0, NO_ARG_NAME);
2185 return -EINVAL;
2186 }
2187 parg->name = kmemdup_nul(arg, body - arg, GFP_KERNEL);
2188 body++;
2189 } else {
2190 parg->name = generate_probe_arg_name(arg, i);
2191 body = arg;
2192 }
2193 if (!parg->name)
2194 return -ENOMEM;
2195
2196 if (!is_good_name(parg->name)) {
2197 trace_probe_log_err(0, BAD_ARG_NAME);
2198 return -EINVAL;
2199 }
2200 if (traceprobe_conflict_field_name(parg->name, tp->args, i)) {
2201 trace_probe_log_err(0, USED_ARG_NAME);
2202 return -EINVAL;
2203 }
2204 ctx->offset = body - arg;
2205 /* Parse fetch argument */
2206 return traceprobe_parse_probe_arg_body(body, &tp->size, parg, ctx);
2207}
2208
2209void traceprobe_free_probe_arg(struct probe_arg *arg)
2210{
2211 struct fetch_insn *code = arg->code;
2212
2213 while (code && code->op != FETCH_OP_END) {
2214 if (code->op == FETCH_NOP_SYMBOL ||
2215 code->op == FETCH_OP_IMMSTR)
2216 kfree(code->data);
2217 code++;
2218 }
2219 kfree(arg->code);
2220 kfree(arg->name);
2221 kfree(arg->comm);
2222 kfree(arg->fmt);
2223}
2224
2225static int argv_has_var_arg(int argc, const char *argv[], int *args_idx,
2226 struct traceprobe_parse_context *ctx)
2227{
2228 int i, found = 0;
2229
2230 for (i = 0; i < argc; i++)
2231 if (str_has_prefix(argv[i], "$arg")) {
2232 trace_probe_log_set_index(i + 2);
2233
2234 if (!tparg_is_function_entry(ctx->flags) &&
2235 !tparg_is_function_return(ctx->flags)) {
2236 trace_probe_log_err(0, NOFENTRY_ARGS);
2237 return -EINVAL;
2238 }
2239
2240 if (isdigit(argv[i][4])) {
2241 found = 1;
2242 continue;
2243 }
2244
2245 if (argv[i][4] != '*') {
2246 trace_probe_log_err(0, BAD_VAR);
2247 return -EINVAL;
2248 }
2249
2250 if (*args_idx >= 0 && *args_idx < argc) {
2251 trace_probe_log_err(0, DOUBLE_ARGS);
2252 return -EINVAL;
2253 }
2254 found = 1;
2255 *args_idx = i;
2256 }
2257
2258 return found;
2259}
2260
2261static int sprint_nth_btf_arg(int idx, const char *type,
2262 char *buf, int bufsize,
2263 struct traceprobe_parse_context *ctx)
2264{
2265 const char *name;
2266 int ret;
2267
2268 if (idx >= ctx->nr_params) {
2269 trace_probe_log_err(0, NO_BTFARG);
2270 return -ENOENT;
2271 }
2272 name = btf_name_by_offset(ctx->btf, ctx->params[idx].name_off);
2273 if (!name) {
2274 trace_probe_log_err(0, NO_BTF_ENTRY);
2275 return -ENOENT;
2276 }
2277 ret = snprintf(buf, bufsize, "%s%s", name, type);
2278 if (ret >= bufsize) {
2279 trace_probe_log_err(0, ARGS_2LONG);
2280 return -E2BIG;
2281 }
2282 return ret;
2283}
2284
2285/* Return new_argv which must be freed after use */
2286const char **traceprobe_expand_meta_args(int argc, const char *argv[],
2287 int *new_argc, char *buf, int bufsize,
2288 struct traceprobe_parse_context *ctx)
2289{
2290 int i, j, n, used, ret, args_idx = -1;
2291 const char **new_argv __free(kfree) = NULL;
2292
2293 ret = argv_has_var_arg(argc, argv, &args_idx, ctx);
2294 if (ret < 0)
2295 return ERR_PTR(ret);
2296
2297 if (!ret) {
2298 *new_argc = argc;
2299 return NULL;
2300 }
2301
2302 ret = query_btf_context(ctx);
2303 if (ret < 0 || ctx->nr_params == 0) {
2304 if (args_idx != -1) {
2305 /* $arg* requires BTF info */
2306 trace_probe_log_err(0, NOSUP_BTFARG);
2307 return ERR_PTR(-EOPNOTSUPP);
2308 }
2309 *new_argc = argc;
2310 return NULL;
2311 }
2312
2313 if (args_idx >= 0)
2314 *new_argc = argc + ctx->nr_params - 1;
2315 else
2316 *new_argc = argc;
2317
2318 new_argv = kcalloc(*new_argc, sizeof(char *), GFP_KERNEL);
2319 if (!new_argv)
2320 return ERR_PTR(-ENOMEM);
2321
2322 used = 0;
2323 for (i = 0, j = 0; i < argc; i++) {
2324 trace_probe_log_set_index(i + 2);
2325 if (i == args_idx) {
2326 for (n = 0; n < ctx->nr_params; n++) {
2327 ret = sprint_nth_btf_arg(n, "", buf + used,
2328 bufsize - used, ctx);
2329 if (ret < 0)
2330 return ERR_PTR(ret);
2331
2332 new_argv[j++] = buf + used;
2333 used += ret + 1;
2334 }
2335 continue;
2336 }
2337
2338 if (str_has_prefix(argv[i], "$arg")) {
2339 char *type = NULL;
2340
2341 n = simple_strtoul(argv[i] + 4, &type, 10);
2342 if (type && !(*type == ':' || *type == '\0')) {
2343 trace_probe_log_err(0, BAD_VAR);
2344 return ERR_PTR(-ENOENT);
2345 }
2346 /* Note: $argN starts from $arg1, so $arg0 is invalid. */
2347 if (n == 0) {
2348 trace_probe_log_err(0, BAD_ARG_NUM);
2349 return ERR_PTR(-EINVAL);
2350 }
2351 ret = sprint_nth_btf_arg(n - 1, type, buf + used,
2352 bufsize - used, ctx);
2353 if (ret < 0)
2354 return ERR_PTR(ret);
2355 new_argv[j++] = buf + used;
2356 used += ret + 1;
2357 } else
2358 new_argv[j++] = argv[i];
2359 }
2360
2361 return_ptr(new_argv);
2362}
2363
2364/* @buf: *buf must be equal to NULL. Caller must to free *buf */
2365int traceprobe_expand_dentry_args(int argc, const char *argv[], char **buf)
2366{
2367 int i, used, ret;
2368 const int bufsize = MAX_DENTRY_ARGS_LEN;
2369 char *tmpbuf __free(kfree) = NULL;
2370
2371 if (*buf)
2372 return -EINVAL;
2373
2374 used = 0;
2375 for (i = 0; i < argc; i++) {
2376 char *tmp __free(kfree) = NULL;
2377 char *equal;
2378 size_t arg_len;
2379
2380 if (!glob_match("*:%p[dD]", argv[i]))
2381 continue;
2382
2383 if (!tmpbuf) {
2384 tmpbuf = kmalloc(bufsize, GFP_KERNEL);
2385 if (!tmpbuf)
2386 return -ENOMEM;
2387 }
2388
2389 tmp = kstrdup(argv[i], GFP_KERNEL);
2390 if (!tmp)
2391 return -ENOMEM;
2392
2393 equal = strchr(tmp, '=');
2394 if (equal)
2395 *equal = '\0';
2396 arg_len = strlen(argv[i]);
2397 tmp[arg_len - 4] = '\0';
2398 if (argv[i][arg_len - 1] == 'd')
2399 ret = snprintf(tmpbuf + used, bufsize - used,
2400 "%s%s+0x0(+0x%zx(%s)):string",
2401 equal ? tmp : "", equal ? "=" : "",
2402 offsetof(struct dentry, d_name.name),
2403 equal ? equal + 1 : tmp);
2404 else
2405 ret = snprintf(tmpbuf + used, bufsize - used,
2406 "%s%s+0x0(+0x%zx(+0x%zx(%s))):string",
2407 equal ? tmp : "", equal ? "=" : "",
2408 offsetof(struct dentry, d_name.name),
2409 offsetof(struct file, f_path.dentry),
2410 equal ? equal + 1 : tmp);
2411
2412 if (ret >= bufsize - used)
2413 return -ENOMEM;
2414 argv[i] = tmpbuf + used;
2415 used += ret + 1;
2416 }
2417
2418 *buf = no_free_ptr(tmpbuf);
2419 return 0;
2420}
2421
2422void traceprobe_finish_parse(struct traceprobe_parse_context *ctx)
2423{
2424 clear_btf_context(ctx);
2425}
2426
2427int traceprobe_update_arg(struct probe_arg *arg)
2428{
2429 struct fetch_insn *code = arg->code;
2430 long offset;
2431 char *tmp;
2432 char c;
2433 int ret = 0;
2434
2435 while (code && code->op != FETCH_OP_END) {
2436 if (code->op == FETCH_NOP_SYMBOL) {
2437 if (code[1].op != FETCH_OP_IMM)
2438 return -EINVAL;
2439
2440 tmp = strpbrk(code->data, "+-");
2441 if (tmp)
2442 c = *tmp;
2443 ret = traceprobe_split_symbol_offset(code->data,
2444 &offset);
2445 if (ret)
2446 return ret;
2447
2448 code[1].immediate =
2449 (unsigned long)kallsyms_lookup_name(code->data);
2450 if (tmp)
2451 *tmp = c;
2452 if (!code[1].immediate)
2453 return -ENOENT;
2454 code[1].immediate += offset;
2455 }
2456 code++;
2457 }
2458 return 0;
2459}
2460
2461/* When len=0, we just calculate the needed length */
2462#define LEN_OR_ZERO (len > pos ? len - pos : 0)
2463static int __set_print_fmt(struct trace_probe *tp, char *buf, int len,
2464 enum probe_print_type ptype)
2465{
2466 struct probe_arg *parg;
2467 int i, j;
2468 int pos = 0;
2469 const char *fmt, *arg;
2470
2471 switch (ptype) {
2472 case PROBE_PRINT_NORMAL:
2473 fmt = "(%lx)";
2474 arg = ", REC->" FIELD_STRING_IP;
2475 break;
2476 case PROBE_PRINT_RETURN:
2477 fmt = "(%lx <- %lx)";
2478 arg = ", REC->" FIELD_STRING_FUNC ", REC->" FIELD_STRING_RETIP;
2479 break;
2480 case PROBE_PRINT_EVENT:
2481 fmt = "";
2482 arg = "";
2483 break;
2484 default:
2485 WARN_ON_ONCE(1);
2486 return 0;
2487 }
2488
2489 pos += snprintf(buf + pos, LEN_OR_ZERO, "\"%s", fmt);
2490
2491 for (i = 0; i < tp->nr_args; i++) {
2492 parg = tp->args + i;
2493 pos += snprintf(buf + pos, LEN_OR_ZERO, " %s=", parg->name);
2494 if (parg->count) {
2495 pos += snprintf(buf + pos, LEN_OR_ZERO, "{%s",
2496 parg->type->fmt);
2497 for (j = 1; j < parg->count; j++)
2498 pos += snprintf(buf + pos, LEN_OR_ZERO, ",%s",
2499 parg->type->fmt);
2500 pos += snprintf(buf + pos, LEN_OR_ZERO, "}");
2501 } else
2502 pos += snprintf(buf + pos, LEN_OR_ZERO, "%s",
2503 parg->type->fmt);
2504 }
2505
2506 pos += snprintf(buf + pos, LEN_OR_ZERO, "\"%s", arg);
2507
2508 for (i = 0; i < tp->nr_args; i++) {
2509 parg = tp->args + i;
2510 if (parg->count) {
2511 if (parg->type->is_string)
2512 fmt = ", __get_str(%s[%d])";
2513 else
2514 fmt = ", REC->%s[%d]";
2515 for (j = 0; j < parg->count; j++)
2516 pos += snprintf(buf + pos, LEN_OR_ZERO,
2517 fmt, parg->name, j);
2518 } else {
2519 if (parg->type->is_string)
2520 fmt = ", __get_str(%s)";
2521 else
2522 fmt = ", REC->%s";
2523 pos += snprintf(buf + pos, LEN_OR_ZERO,
2524 fmt, parg->name);
2525 }
2526 }
2527
2528 /* return the length of print_fmt */
2529 return pos;
2530}
2531#undef LEN_OR_ZERO
2532
2533int traceprobe_set_print_fmt(struct trace_probe *tp, enum probe_print_type ptype)
2534{
2535 struct trace_event_call *call = trace_probe_event_call(tp);
2536 int len;
2537 char *print_fmt;
2538
2539 /* First: called with 0 length to calculate the needed length */
2540 len = __set_print_fmt(tp, NULL, 0, ptype);
2541 print_fmt = kmalloc(len + 1, GFP_KERNEL);
2542 if (!print_fmt)
2543 return -ENOMEM;
2544
2545 /* Second: actually write the @print_fmt */
2546 __set_print_fmt(tp, print_fmt, len + 1, ptype);
2547 call->print_fmt = print_fmt;
2548
2549 return 0;
2550}
2551
2552int traceprobe_define_arg_fields(struct trace_event_call *event_call,
2553 size_t offset, struct trace_probe *tp)
2554{
2555 struct trace_probe_event *tpe = trace_probe_event_from_call(event_call);
2556 int ret, i;
2557
2558 /*
2559 * A field created by trace_define_field() only stores the name and
2560 * type pointers, it does not copy the strings. Here they point into
2561 * the probe_arg of @tp, which is freed when @tp is removed. For an
2562 * event with multiple probes attached, the field list is defined
2563 * once by the first probe but kept alive by the surviving siblings,
2564 * so removing that first probe would leave the fields referencing
2565 * freed memory. Duplicate the strings and anchor the copies on the
2566 * trace_probe_event, which lives as long as the field list itself.
2567 *
2568 * event_define_fields() ignores the return value of this hook, so
2569 * if a previous attempt failed before creating any field, it may
2570 * call here again. Release duplicates left behind by such an
2571 * attempt before starting over.
2572 */
2573 for (i = 0; i < tpe->nr_field_strings; i++)
2574 kfree(tpe->field_strings[i]);
2575 kfree(tpe->field_strings);
2576 tpe->field_strings = NULL;
2577 tpe->nr_field_strings = 0;
2578
2579 if (tp->nr_args) {
2580 tpe->field_strings = kcalloc(tp->nr_args * 2, sizeof(char *),
2581 GFP_KERNEL);
2582 if (!tpe->field_strings)
2583 return -ENOMEM;
2584 }
2585
2586 /* Set argument names as fields */
2587 for (i = 0; i < tp->nr_args; i++) {
2588 struct probe_arg *parg = &tp->args[i];
2589 const char *fmt = parg->type->fmttype;
2590 int size = parg->type->size;
2591 char *name, *type;
2592
2593 if (parg->fmt)
2594 fmt = parg->fmt;
2595 if (parg->count)
2596 size *= parg->count;
2597
2598 name = kstrdup(parg->name, GFP_KERNEL);
2599 type = kstrdup(fmt, GFP_KERNEL);
2600 if (!name || !type) {
2601 kfree(name);
2602 kfree(type);
2603 return -ENOMEM;
2604 }
2605 tpe->field_strings[tpe->nr_field_strings++] = name;
2606 tpe->field_strings[tpe->nr_field_strings++] = type;
2607
2608 ret = trace_define_field(event_call, type, name,
2609 offset + parg->offset, size,
2610 parg->type->is_signed,
2611 FILTER_OTHER);
2612 if (ret)
2613 return ret;
2614 }
2615 return 0;
2616}
2617
2618static void trace_probe_event_free(struct trace_probe_event *tpe)
2619{
2620 int i;
2621
2622 for (i = 0; i < tpe->nr_field_strings; i++)
2623 kfree(tpe->field_strings[i]);
2624 kfree(tpe->field_strings);
2625 kfree(tpe->class.system);
2626 kfree(tpe->call.name);
2627 kfree(tpe->call.print_fmt);
2628 kfree(tpe);
2629}
2630
2631int trace_probe_append(struct trace_probe *tp, struct trace_probe *to)
2632{
2633 if (trace_probe_has_sibling(tp))
2634 return -EBUSY;
2635
2636 list_del_init(&tp->list);
2637 trace_probe_event_free(tp->event);
2638
2639 tp->event = to->event;
2640 list_add_tail(&tp->list, trace_probe_probe_list(to));
2641
2642 return 0;
2643}
2644
2645void trace_probe_unlink(struct trace_probe *tp)
2646{
2647 list_del_init(&tp->list);
2648 if (list_empty(trace_probe_probe_list(tp)))
2649 trace_probe_event_free(tp->event);
2650 tp->event = NULL;
2651}
2652
2653void trace_probe_cleanup(struct trace_probe *tp)
2654{
2655 int i;
2656
2657 for (i = 0; i < tp->nr_args; i++)
2658 traceprobe_free_probe_arg(&tp->args[i]);
2659
2660 if (tp->entry_arg) {
2661 kfree(tp->entry_arg);
2662 tp->entry_arg = NULL;
2663 }
2664
2665 if (tp->event)
2666 trace_probe_unlink(tp);
2667}
2668
2669int trace_probe_init(struct trace_probe *tp, const char *event,
2670 const char *group, bool alloc_filter, int nargs)
2671{
2672 struct trace_event_call *call;
2673 size_t size = sizeof(struct trace_probe_event);
2674 int ret = 0;
2675
2676 if (!event || !group)
2677 return -EINVAL;
2678
2679 if (alloc_filter)
2680 size += sizeof(struct trace_uprobe_filter);
2681
2682 tp->event = kzalloc(size, GFP_KERNEL);
2683 if (!tp->event)
2684 return -ENOMEM;
2685
2686 INIT_LIST_HEAD(&tp->event->files);
2687 INIT_LIST_HEAD(&tp->event->class.fields);
2688 INIT_LIST_HEAD(&tp->event->probes);
2689 INIT_LIST_HEAD(&tp->list);
2690 list_add(&tp->list, &tp->event->probes);
2691
2692 call = trace_probe_event_call(tp);
2693 call->class = &tp->event->class;
2694 call->name = kstrdup(event, GFP_KERNEL);
2695 if (!call->name) {
2696 ret = -ENOMEM;
2697 goto error;
2698 }
2699
2700 tp->event->class.system = kstrdup(group, GFP_KERNEL);
2701 if (!tp->event->class.system) {
2702 ret = -ENOMEM;
2703 goto error;
2704 }
2705
2706 tp->nr_args = nargs;
2707 /* Make sure pointers in args[] are NULL */
2708 if (nargs)
2709 memset(tp->args, 0, sizeof(tp->args[0]) * nargs);
2710
2711 return 0;
2712
2713error:
2714 trace_probe_cleanup(tp);
2715 return ret;
2716}
2717
2718static struct trace_event_call *
2719find_trace_event_call(const char *system, const char *event_name)
2720{
2721 struct trace_event_call *tp_event;
2722 const char *name;
2723
2724 list_for_each_entry(tp_event, &ftrace_events, list) {
2725 if (!tp_event->class->system ||
2726 strcmp(system, tp_event->class->system))
2727 continue;
2728 name = trace_event_name(tp_event);
2729 if (!name || strcmp(event_name, name))
2730 continue;
2731 return tp_event;
2732 }
2733
2734 return NULL;
2735}
2736
2737int trace_probe_register_event_call(struct trace_probe *tp)
2738{
2739 struct trace_event_call *call = trace_probe_event_call(tp);
2740 int ret;
2741
2742 lockdep_assert_held(&event_mutex);
2743
2744 if (find_trace_event_call(trace_probe_group_name(tp),
2745 trace_probe_name(tp)))
2746 return -EEXIST;
2747
2748 ret = register_trace_event(&call->event);
2749 if (!ret)
2750 return -ENODEV;
2751
2752 ret = trace_add_event_call(call);
2753 if (ret)
2754 unregister_trace_event(&call->event);
2755
2756 return ret;
2757}
2758
2759int trace_probe_add_file(struct trace_probe *tp, struct trace_event_file *file)
2760{
2761 struct event_file_link *link;
2762
2763 link = kmalloc_obj(*link);
2764 if (!link)
2765 return -ENOMEM;
2766
2767 link->file = file;
2768 INIT_LIST_HEAD(&link->list);
2769 list_add_tail_rcu(&link->list, &tp->event->files);
2770 trace_probe_set_flag(tp, TP_FLAG_TRACE);
2771 return 0;
2772}
2773
2774struct event_file_link *trace_probe_get_file_link(struct trace_probe *tp,
2775 struct trace_event_file *file)
2776{
2777 struct event_file_link *link;
2778
2779 trace_probe_for_each_link(link, tp) {
2780 if (link->file == file)
2781 return link;
2782 }
2783
2784 return NULL;
2785}
2786
2787int trace_probe_remove_file(struct trace_probe *tp,
2788 struct trace_event_file *file)
2789{
2790 struct event_file_link *link;
2791
2792 link = trace_probe_get_file_link(tp, file);
2793 if (!link)
2794 return -ENOENT;
2795
2796 list_del_rcu(&link->list);
2797 kvfree_rcu_mightsleep(link);
2798
2799 if (list_empty(&tp->event->files))
2800 trace_probe_clear_flag(tp, TP_FLAG_TRACE);
2801
2802 return 0;
2803}
2804
2805/*
2806 * Return the smallest index of different type argument (start from 1).
2807 * If all argument types and name are same, return 0.
2808 */
2809int trace_probe_compare_arg_type(struct trace_probe *a, struct trace_probe *b)
2810{
2811 int i;
2812
2813 /* In case of more arguments */
2814 if (a->nr_args < b->nr_args)
2815 return a->nr_args + 1;
2816 if (a->nr_args > b->nr_args)
2817 return b->nr_args + 1;
2818
2819 for (i = 0; i < a->nr_args; i++) {
2820 if ((a->args[i].type != b->args[i].type) ||
2821 (a->args[i].count != b->args[i].count) ||
2822 strcmp(a->args[i].name, b->args[i].name))
2823 return i + 1;
2824 }
2825
2826 return 0;
2827}
2828
2829bool trace_probe_match_command_args(struct trace_probe *tp,
2830 int argc, const char **argv)
2831{
2832 int i;
2833
2834 if (tp->nr_args < argc)
2835 return false;
2836
2837 for (i = 0; i < argc; i++) {
2838 int len = strlen(tp->args[i].name);
2839
2840 if (strncmp(argv[i], tp->args[i].name, len) ||
2841 argv[i][len] != '=' ||
2842 strcmp(argv[i] + len + 1, tp->args[i].comm))
2843 return false;
2844 }
2845 return true;
2846}
2847
2848int trace_probe_create(const char *raw_command, int (*createfn)(int, const char **))
2849{
2850 int argc = 0, ret = 0;
2851 char **argv;
2852
2853 argv = argv_split(GFP_KERNEL, raw_command, &argc);
2854 if (!argv)
2855 return -ENOMEM;
2856
2857 if (argc)
2858 ret = createfn(argc, (const char **)argv);
2859
2860 argv_free(argv);
2861
2862 return ret;
2863}
2864
2865int trace_probe_print_args(struct trace_seq *s, struct probe_arg *args, int nr_args,
2866 u8 *data, void *field)
2867{
2868 void *p;
2869 int i, j;
2870
2871 for (i = 0; i < nr_args; i++) {
2872 struct probe_arg *a = args + i;
2873
2874 trace_seq_printf(s, " %s=", a->name);
2875 if (likely(!a->count)) {
2876 if (!a->type->print(s, data + a->offset, field))
2877 return -ENOMEM;
2878 continue;
2879 }
2880 trace_seq_putc(s, '{');
2881 p = data + a->offset;
2882 for (j = 0; j < a->count; j++) {
2883 if (!a->type->print(s, p, field))
2884 return -ENOMEM;
2885 trace_seq_putc(s, j == a->count - 1 ? '}' : ',');
2886 p += a->type->size;
2887 }
2888 }
2889 return 0;
2890}
2891
2892#ifdef CONFIG_PROBE_EVENTS_DUMP_FETCHARG
2893
2894struct fetch_op_decode {
2895 const char *name;
2896 void (*decode)(struct seq_file *m, struct fetch_insn *insn);
2897};
2898
2899static const struct fetch_op_decode fetch_op_decode[];
2900
2901static void fetcharg_decode_none(struct seq_file *m, struct fetch_insn *insn)
2902{
2903 seq_puts(m, fetch_op_decode[insn->op].name);
2904}
2905
2906static void fetcharg_decode_param(struct seq_file *m, struct fetch_insn *insn)
2907{
2908 seq_printf(m, "%s(%u)", fetch_op_decode[insn->op].name, insn->param);
2909}
2910
2911static void fetcharg_decode_imm(struct seq_file *m, struct fetch_insn *insn)
2912{
2913 seq_printf(m, "%s(0x%lx)", fetch_op_decode[insn->op].name, insn->immediate);
2914}
2915
2916static void fetcharg_decode_string(struct seq_file *m, struct fetch_insn *insn)
2917{
2918 seq_printf(m, "%s(%s)", fetch_op_decode[insn->op].name, (char *)insn->data);
2919}
2920
2921static void fetcharg_decode_symbol(struct seq_file *m, struct fetch_insn *insn)
2922{
2923 seq_printf(m, "%s(%s)", fetch_op_decode[insn->op].name, (char *)insn->data);
2924}
2925
2926static void fetcharg_decode_offset(struct seq_file *m, struct fetch_insn *insn)
2927{
2928 seq_printf(m, "%s(offset=%d)", fetch_op_decode[insn->op].name, insn->offset);
2929}
2930
2931static void fetcharg_decode_store(struct seq_file *m, struct fetch_insn *insn)
2932{
2933 if (insn->op == FETCH_OP_ST_RAW)
2934 seq_printf(m, "%s(size=%u)", fetch_op_decode[insn->op].name, insn->size);
2935 else
2936 seq_printf(m, "%s(offset=%d,size=%u)", fetch_op_decode[insn->op].name,
2937 insn->offset, insn->size);
2938}
2939
2940static void fetcharg_decode_bf(struct seq_file *m, struct fetch_insn *insn)
2941{
2942 seq_printf(m, "%s(basesize=%u,lshift=%u,rshift=%u)",
2943 fetch_op_decode[insn->op].name, insn->basesize, insn->lshift, insn->rshift);
2944}
2945
2946static void fetcharg_decode_tp_arg(struct seq_file *m, struct fetch_insn *insn)
2947{
2948 struct ftrace_event_field *field = insn->data;
2949
2950 seq_printf(m, "%s(%s)", fetch_op_decode[insn->op].name, field->name);
2951}
2952
2953#define FETCH_OP(opname, decode_fn) \
2954 [FETCH_OP_##opname] = { .name = #opname, .decode = fetcharg_decode_##decode_fn }
2955
2956static const struct fetch_op_decode fetch_op_decode[] = FETCH_OP_LIST;
2957#undef FETCH_OP
2958
2959static void trace_probe_dump_arg(struct seq_file *m, struct probe_arg *parg)
2960{
2961 int i;
2962
2963 seq_printf(m, "# %s: ", parg->name);
2964 for (i = 0; i < FETCH_INSN_MAX; i++) {
2965 struct fetch_insn *insn = parg->code + i;
2966
2967 if (insn->op >= ARRAY_SIZE(fetch_op_decode) || !fetch_op_decode[insn->op].decode)
2968 seq_printf(m, "unknown(%d)", insn->op);
2969 else
2970 fetch_op_decode[insn->op].decode(m, insn);
2971
2972 if (insn->op == FETCH_OP_END)
2973 break;
2974 seq_puts(m, " -> ");
2975 }
2976 seq_putc(m, '\n');
2977}
2978
2979void trace_probe_dump_args(struct seq_file *m, struct trace_probe *tp)
2980{
2981 int i;
2982
2983 for (i = 0; i < tp->nr_args; i++)
2984 trace_probe_dump_arg(m, &tp->args[i]);
2985}
2986#endif /* CONFIG_PROBE_EVENTS_DUMP_FETCHARG */