projects/linux·blob master · refs

5245 lines · 122.6 KiB

1// SPDX-License-Identifier: GPL-2.0
2/*
3 * event tracer
4 *
5 * Copyright (C) 2008 Red Hat Inc, Steven Rostedt <[email protected]>
6 *
7 * - Added format output of fields of the trace point.
8 * This was based off of work by Tom Zanussi <[email protected]>.
9 *
10 */
11
12#define pr_fmt(fmt) fmt
13
14#include <linux/workqueue.h>
15#include <linux/security.h>
16#include <linux/spinlock.h>
17#include <linux/seq_buf.h>
18#include <linux/kthread.h>
19#include <linux/tracefs.h>
20#include <linux/uaccess.h>
21#include <linux/module.h>
22#include <linux/ctype.h>
23#include <linux/sort.h>
24#include <linux/slab.h>
25#include <linux/delay.h>
26#include <linux/btf.h>
27
28#include <trace/events/sched.h>
29#include <trace/syscall.h>
30
31#include <asm/setup.h>
32
33#include "trace_output.h"
34
35#undef TRACE_SYSTEM
36#define TRACE_SYSTEM "TRACE_SYSTEM"
37
38DEFINE_MUTEX(event_mutex);
39
40LIST_HEAD(ftrace_events);
41static LIST_HEAD(ftrace_generic_fields);
42static LIST_HEAD(ftrace_common_fields);
43static bool eventdir_initialized;
44
45static LIST_HEAD(module_strings);
46
47struct module_string {
48 struct list_head next;
49 struct module *module;
50 char *str;
51};
52
53#define GFP_TRACE (GFP_KERNEL | __GFP_ZERO)
54
55static struct kmem_cache *field_cachep;
56static struct kmem_cache *file_cachep;
57
58static inline int system_refcount(struct event_subsystem *system)
59{
60 return system->ref_count;
61}
62
63static int system_refcount_inc(struct event_subsystem *system)
64{
65 return system->ref_count++;
66}
67
68static int system_refcount_dec(struct event_subsystem *system)
69{
70 return --system->ref_count;
71}
72
73/* Double loops, do not use break, only goto's work */
74#define do_for_each_event_file(tr, file) \
75 list_for_each_entry(tr, &ftrace_trace_arrays, list) { \
76 list_for_each_entry(file, &tr->events, list)
77
78#define do_for_each_event_file_safe(tr, file) \
79 list_for_each_entry(tr, &ftrace_trace_arrays, list) { \
80 struct trace_event_file *___n; \
81 list_for_each_entry_safe(file, ___n, &tr->events, list)
82
83#define while_for_each_event_file() \
84 }
85
86static struct ftrace_event_field *
87__find_event_field(struct list_head *head, const char *name)
88{
89 struct ftrace_event_field *field;
90
91 list_for_each_entry(field, head, link) {
92 if (!strcmp(field->name, name))
93 return field;
94 }
95
96 return NULL;
97}
98
99struct ftrace_event_field *
100trace_find_event_field(struct trace_event_call *call, char *name)
101{
102 struct ftrace_event_field *field;
103 struct list_head *head;
104
105 head = trace_get_fields(call);
106 field = __find_event_field(head, name);
107 if (field)
108 return field;
109
110 field = __find_event_field(&ftrace_generic_fields, name);
111 if (field)
112 return field;
113
114 return __find_event_field(&ftrace_common_fields, name);
115}
116
117static int __trace_define_field(struct list_head *head, const char *type,
118 const char *name, int offset, int size,
119 int is_signed, int filter_type, int len,
120 int need_test)
121{
122 struct ftrace_event_field *field;
123
124 field = kmem_cache_alloc(field_cachep, GFP_TRACE);
125 if (!field)
126 return -ENOMEM;
127
128 field->name = name;
129 field->type = type;
130
131 if (filter_type == FILTER_OTHER)
132 field->filter_type = filter_assign_type(type);
133 else
134 field->filter_type = filter_type;
135
136 field->offset = offset;
137 field->size = size;
138 field->is_signed = is_signed;
139 field->needs_test = need_test;
140 field->len = len;
141
142 list_add(&field->link, head);
143
144 return 0;
145}
146
147int trace_define_field(struct trace_event_call *call, const char *type,
148 const char *name, int offset, int size, int is_signed,
149 int filter_type)
150{
151 struct list_head *head;
152
153 if (WARN_ON(!call->class))
154 return 0;
155
156 head = trace_get_fields(call);
157 return __trace_define_field(head, type, name, offset, size,
158 is_signed, filter_type, 0, 0);
159}
160EXPORT_SYMBOL_GPL(trace_define_field);
161
162static int trace_define_field_ext(struct trace_event_call *call, const char *type,
163 const char *name, int offset, int size, int is_signed,
164 int filter_type, int len, int need_test)
165{
166 struct list_head *head;
167
168 if (WARN_ON(!call->class))
169 return 0;
170
171 head = trace_get_fields(call);
172 return __trace_define_field(head, type, name, offset, size,
173 is_signed, filter_type, len, need_test);
174}
175
176#define __generic_field(type, item, filter_type) \
177 ret = __trace_define_field(&ftrace_generic_fields, #type, \
178 #item, 0, 0, is_signed_type(type), \
179 filter_type, 0, 0); \
180 if (ret) \
181 return ret;
182
183#define __common_field(type, item) \
184 ret = __trace_define_field(&ftrace_common_fields, #type, \
185 "common_" #item, \
186 offsetof(typeof(ent), item), \
187 sizeof(ent.item), \
188 is_signed_type(type), FILTER_OTHER, \
189 0, 0); \
190 if (ret) \
191 return ret;
192
193static int trace_define_generic_fields(void)
194{
195 int ret;
196
197 __generic_field(int, CPU, FILTER_CPU);
198 __generic_field(int, cpu, FILTER_CPU);
199 __generic_field(int, common_cpu, FILTER_CPU);
200 __generic_field(char *, COMM, FILTER_COMM);
201 __generic_field(char *, comm, FILTER_COMM);
202 __generic_field(char *, stacktrace, FILTER_STACKTRACE);
203 __generic_field(char *, STACKTRACE, FILTER_STACKTRACE);
204
205 return ret;
206}
207
208static int trace_define_common_fields(void)
209{
210 int ret;
211 struct trace_entry ent;
212
213 __common_field(unsigned short, type);
214 __common_field(unsigned char, flags);
215 /* Holds both preempt_count and migrate_disable */
216 __common_field(unsigned char, preempt_count);
217 __common_field(int, pid);
218
219 return ret;
220}
221
222static void trace_destroy_fields(struct trace_event_call *call)
223{
224 struct ftrace_event_field *field, *next;
225 struct list_head *head;
226
227 head = trace_get_fields(call);
228 list_for_each_entry_safe(field, next, head, link) {
229 list_del(&field->link);
230 kmem_cache_free(field_cachep, field);
231 }
232}
233
234/*
235 * run-time version of trace_event_get_offsets_<call>() that returns the last
236 * accessible offset of trace fields excluding __dynamic_array bytes
237 */
238int trace_event_get_offsets(struct trace_event_call *call)
239{
240 struct ftrace_event_field *tail;
241 struct list_head *head;
242
243 head = trace_get_fields(call);
244 /*
245 * head->next points to the last field with the largest offset,
246 * since it was added last by trace_define_field()
247 */
248 tail = list_first_entry(head, struct ftrace_event_field, link);
249 return tail->offset + tail->size;
250}
251
252
253static struct trace_event_fields *find_event_field(const char *fmt,
254 struct trace_event_call *call)
255{
256 struct trace_event_fields *field = call->class->fields_array;
257 const char *p = fmt;
258 int len;
259
260 if (!(len = str_has_prefix(fmt, "REC->")))
261 return NULL;
262 fmt += len;
263 for (p = fmt; *p; p++) {
264 if (!isalnum(*p) && *p != '_')
265 break;
266 }
267 len = p - fmt;
268
269 for (; field->type; field++) {
270 if (strncmp(field->name, fmt, len) || field->name[len])
271 continue;
272
273 return field;
274 }
275 return NULL;
276}
277
278/*
279 * Check if the referenced field is an array and return true,
280 * as arrays are OK to dereference.
281 */
282static bool test_field(const char *fmt, struct trace_event_call *call)
283{
284 struct trace_event_fields *field;
285
286 field = find_event_field(fmt, call);
287 if (!field)
288 return false;
289
290 /* This is an array and is OK to dereference. */
291 return strchr(field->type, '[') != NULL;
292}
293
294/* Look for a string within an argument */
295static bool find_print_string(const char *arg, const char *str, const char *end)
296{
297 const char *r;
298
299 r = strstr(arg, str);
300 return r && r < end;
301}
302
303/* Return true if the argument pointer is safe */
304static bool process_pointer(const char *fmt, int len, struct trace_event_call *call)
305{
306 const char *r, *e, *a;
307
308 e = fmt + len;
309
310 /* Find the REC-> in the argument */
311 r = strstr(fmt, "REC->");
312 if (r && r < e) {
313 /*
314 * Addresses of events on the buffer, or an array on the buffer is
315 * OK to dereference. There's ways to fool this, but
316 * this is to catch common mistakes, not malicious code.
317 */
318 a = strchr(fmt, '&');
319 if ((a && (a < r)) || test_field(r, call))
320 return true;
321 } else if (find_print_string(fmt, "__get_dynamic_array(", e)) {
322 return true;
323 } else if (find_print_string(fmt, "__get_rel_dynamic_array(", e)) {
324 return true;
325 } else if (find_print_string(fmt, "__get_dynamic_array_len(", e)) {
326 return true;
327 } else if (find_print_string(fmt, "__get_rel_dynamic_array_len(", e)) {
328 return true;
329 } else if (find_print_string(fmt, "__get_sockaddr(", e)) {
330 return true;
331 } else if (find_print_string(fmt, "__get_rel_sockaddr(", e)) {
332 return true;
333 }
334 return false;
335}
336
337/* Return true if the string is safe */
338static bool process_string(const char *fmt, int len, struct trace_event_call *call)
339{
340 struct trace_event_fields *field;
341 const char *r, *e, *s;
342
343 e = fmt + len;
344
345 /*
346 * There are several helper functions that return strings.
347 * If the argument contains a function, then assume its field is valid.
348 * It is considered that the argument has a function if it has:
349 * alphanumeric or '_' before a parenthesis.
350 */
351 s = fmt;
352 do {
353 r = strstr(s, "(");
354 if (!r || r >= e)
355 break;
356 for (int i = 1; r - i >= s; i++) {
357 char ch = *(r - i);
358 if (isspace(ch))
359 continue;
360 if (isalnum(ch) || ch == '_')
361 return true;
362 /* Anything else, this isn't a function */
363 break;
364 }
365 /* A function could be wrapped in parenthesis, try the next one */
366 s = r + 1;
367 } while (s < e);
368
369 /*
370 * Check for arrays. If the argument has: foo[REC->val]
371 * then it is very likely that foo is an array of strings
372 * that are safe to use.
373 */
374 r = strstr(s, "[");
375 if (r && r < e) {
376 r = strstr(r, "REC->");
377 if (r && r < e)
378 return true;
379 }
380
381 /*
382 * If there's any strings in the argument consider this arg OK as it
383 * could be: REC->field ? "foo" : "bar" and we don't want to get into
384 * verifying that logic here.
385 */
386 if (find_print_string(fmt, "\"", e))
387 return true;
388
389 /* Dereferenced strings are also valid like any other pointer */
390 if (process_pointer(fmt, len, call))
391 return true;
392
393 /* Make sure the field is found */
394 field = find_event_field(fmt, call);
395 if (!field)
396 return false;
397
398 /* Test this field's string before printing the event */
399 call->flags |= TRACE_EVENT_FL_TEST_STR;
400 field->needs_test = 1;
401
402 return true;
403}
404
405static void test_double_dereference(const char *str, int len,
406 struct trace_event_call *call)
407{
408 const char *ptr;
409 const char *end = str + len;
410
411 ptr = strstr(str, "REC->");
412
413 while (ptr && ptr < end) {
414
415 ptr += 5;
416 for (; ptr < end; ptr++) {
417 if (ptr[0] == '-' && ptr[1] == '>') {
418 pr_warn("TRACE EVENT ERROR: Event %s has double dereference in TP_printk: %.*s\n",
419 trace_event_name(call), len, str);
420 WARN_ONCE(1, "Event %s has double dereference in TP_printk: %.*s\n",
421 trace_event_name(call), len, str);
422 return;
423 }
424 if (!isalnum(*ptr) && *ptr != '_')
425 break;
426 }
427
428 ptr = strstr(ptr, "REC->");
429 }
430}
431
432static void handle_dereference_arg(const char *arg_str, u64 string_flags, int len,
433 u64 *dereference_flags, int arg,
434 struct trace_event_call *call)
435{
436 if (string_flags & (1ULL << arg)) {
437 if (process_string(arg_str, len, call))
438 *dereference_flags &= ~(1ULL << arg);
439 } else if (process_pointer(arg_str, len, call))
440 *dereference_flags &= ~(1ULL << arg);
441 else
442 pr_warn("TRACE EVENT ERROR: Bad dereference argument: '%.*s'\n",
443 len, arg_str);
444}
445
446/*
447 * Examine the print fmt of the event looking for unsafe dereference
448 * pointers using %p* that could be recorded in the trace event and
449 * much later referenced after the pointer was freed. Dereferencing
450 * pointers are OK, if it is dereferenced into the event itself.
451 */
452static void test_event_printk(struct trace_event_call *call)
453{
454 u64 dereference_flags = 0;
455 u64 string_flags = 0;
456 bool first = true;
457 const char *fmt;
458 int parens = 0;
459 char in_quote = 0;
460 int start_arg = 0;
461 int arg = 0;
462 int i, e;
463
464 fmt = call->print_fmt;
465
466 if (!fmt)
467 return;
468
469 for (i = 0; fmt[i]; i++) {
470 switch (fmt[i]) {
471 case '\\':
472 i++;
473 if (!fmt[i])
474 return;
475 continue;
476 case '"':
477 case '\'':
478 /*
479 * The print fmt starts with a string that
480 * is processed first to find %p* usage,
481 * then after the first string, the print fmt
482 * contains arguments that are used to check
483 * if the dereferenced %p* usage is safe.
484 */
485 if (first) {
486 if (fmt[i] == '\'')
487 continue;
488 if (in_quote) {
489 arg = 0;
490 first = false;
491 }
492 }
493 if (in_quote) {
494 if (in_quote == fmt[i])
495 in_quote = 0;
496 } else {
497 in_quote = fmt[i];
498 }
499 continue;
500 case '%':
501 if (!first || !in_quote)
502 continue;
503 i++;
504 if (!fmt[i])
505 return;
506 switch (fmt[i]) {
507 case '%':
508 continue;
509 case 'p':
510 do_pointer:
511 /* Find dereferencing fields */
512 switch (fmt[i + 1]) {
513 case 'B': case 'R': case 'r':
514 case 'b': case 'M': case 'm':
515 case 'I': case 'i': case 'E':
516 case 'U': case 'V': case 'N':
517 case 'a': case 'd': case 'D':
518 case 'g': case 't': case 'C':
519 case 'O': case 'f':
520 if (WARN_ONCE(arg == 63,
521 "Too many args for event: %s",
522 trace_event_name(call)))
523 return;
524 dereference_flags |= 1ULL << arg;
525 }
526 break;
527 default:
528 {
529 bool star = false;
530 int j;
531
532 /* Increment arg if %*s exists. */
533 for (j = 0; fmt[i + j]; j++) {
534 if (isdigit(fmt[i + j]) ||
535 fmt[i + j] == '.')
536 continue;
537 if (fmt[i + j] == '*') {
538 star = true;
539 /* Handle %*pbl case */
540 if (!j && fmt[i + 1] == 'p') {
541 arg++;
542 i++;
543 goto do_pointer;
544 }
545 continue;
546 }
547 if ((fmt[i + j] == 's')) {
548 if (star)
549 arg++;
550 if (WARN_ONCE(arg == 63,
551 "Too many args for event: %s",
552 trace_event_name(call)))
553 return;
554 dereference_flags |= 1ULL << arg;
555 string_flags |= 1ULL << arg;
556 }
557 break;
558 }
559 break;
560 } /* default */
561
562 } /* switch */
563 arg++;
564 continue;
565 case '(':
566 if (in_quote)
567 continue;
568 parens++;
569 continue;
570 case ')':
571 if (in_quote)
572 continue;
573 parens--;
574 if (WARN_ONCE(parens < 0,
575 "Paren mismatch for event: %s\narg='%s'\n%*s",
576 trace_event_name(call),
577 fmt + start_arg,
578 (i - start_arg) + 5, "^"))
579 return;
580 continue;
581 case ',':
582 if (in_quote || parens)
583 continue;
584 e = i;
585 i++;
586 while (isspace(fmt[i]))
587 i++;
588
589 /*
590 * If start_arg is zero, then this is the start of the
591 * first argument. The processing of the argument happens
592 * when the end of the argument is found, as it needs to
593 * handle parenthesis and such.
594 */
595 if (!start_arg) {
596 start_arg = i;
597 /* Balance out the i++ in the for loop */
598 i--;
599 continue;
600 }
601
602 test_double_dereference(fmt + start_arg, e - start_arg, call);
603
604 if (dereference_flags & (1ULL << arg)) {
605 handle_dereference_arg(fmt + start_arg, string_flags,
606 e - start_arg,
607 &dereference_flags, arg, call);
608 }
609
610 start_arg = i;
611 arg++;
612 /* Balance out the i++ in the for loop */
613 i--;
614 }
615 }
616
617 test_double_dereference(fmt + start_arg, i - start_arg, call);
618
619 if (dereference_flags & (1ULL << arg)) {
620 handle_dereference_arg(fmt + start_arg, string_flags,
621 i - start_arg,
622 &dereference_flags, arg, call);
623 }
624
625 /*
626 * If you triggered the below warning, the trace event reported
627 * uses an unsafe dereference pointer %p*. As the data stored
628 * at the trace event time may no longer exist when the trace
629 * event is printed, dereferencing to the original source is
630 * unsafe. The source of the dereference must be copied into the
631 * event itself, and the dereference must access the copy instead.
632 */
633 if (WARN_ON_ONCE(dereference_flags)) {
634 arg = 1;
635 while (!(dereference_flags & 1)) {
636 dereference_flags >>= 1;
637 arg++;
638 }
639 pr_warn("event %s has unsafe dereference of argument %d\n",
640 trace_event_name(call), arg);
641 pr_warn("print_fmt: %s\n", fmt);
642 }
643}
644
645int trace_event_raw_init(struct trace_event_call *call)
646{
647 int id;
648
649 id = register_trace_event(&call->event);
650 if (!id)
651 return -ENODEV;
652
653 test_event_printk(call);
654
655 return 0;
656}
657EXPORT_SYMBOL_GPL(trace_event_raw_init);
658
659bool trace_event_ignore_this_pid(struct trace_event_file *trace_file)
660{
661 struct trace_array *tr = trace_file->tr;
662 struct trace_pid_list *no_pid_list;
663 struct trace_pid_list *pid_list;
664
665 pid_list = rcu_dereference_raw(tr->filtered_pids);
666 no_pid_list = rcu_dereference_raw(tr->filtered_no_pids);
667
668 if (!pid_list && !no_pid_list)
669 return false;
670
671 /*
672 * This is recorded at every sched_switch for this task.
673 * Thus, even if the task migrates the ignore value will be the same.
674 */
675 return this_cpu_read(tr->array_buffer.data->ignore_pid) != 0;
676}
677EXPORT_SYMBOL_GPL(trace_event_ignore_this_pid);
678
679/**
680 * trace_event_buffer_reserve - reserve space on the ring buffer for an event
681 * @fbuffer: information about how to save the event
682 * @trace_file: the instance file descriptor for the event
683 * @len: The length of the event
684 *
685 * The @fbuffer has information about the ring buffer and data will
686 * be added to it to be used by the call to trace_event_buffer_commit().
687 * The @trace_file is the desrciptor with information about the status
688 * of the given event for a specific trace_array instance.
689 * The @len is the length of data to save for the event.
690 *
691 * Returns a pointer to the data on the ring buffer or NULL if the
692 * event was not reserved (event was filtered, too big, or the buffer
693 * simply was disabled for write).
694 */
695void *trace_event_buffer_reserve(struct trace_event_buffer *fbuffer,
696 struct trace_event_file *trace_file,
697 unsigned long len)
698{
699 struct trace_event_call *event_call = trace_file->event_call;
700
701 if ((trace_file->flags & EVENT_FILE_FL_PID_FILTER) &&
702 trace_event_ignore_this_pid(trace_file))
703 return NULL;
704
705 /*
706 * If CONFIG_PREEMPTION is enabled, then the tracepoint itself disables
707 * preemption (adding one to the preempt_count). Since we are
708 * interested in the preempt_count at the time the tracepoint was
709 * hit, we need to subtract one to offset the increment.
710 */
711 fbuffer->trace_ctx = tracing_gen_ctx_dec();
712 fbuffer->trace_file = trace_file;
713
714 fbuffer->event =
715 trace_event_buffer_lock_reserve(&fbuffer->buffer, trace_file,
716 event_call->event.type, len,
717 fbuffer->trace_ctx);
718 if (!fbuffer->event)
719 return NULL;
720
721 fbuffer->regs = NULL;
722 fbuffer->entry = ring_buffer_event_data(fbuffer->event);
723 return fbuffer->entry;
724}
725EXPORT_SYMBOL_GPL(trace_event_buffer_reserve);
726
727int trace_event_reg(struct trace_event_call *call,
728 enum trace_reg type, void *data)
729{
730 struct trace_event_file *file = data;
731
732 WARN_ON(!(call->flags & TRACE_EVENT_FL_TRACEPOINT));
733 switch (type) {
734 case TRACE_REG_REGISTER:
735 return tracepoint_probe_register(call->tp,
736 call->class->probe,
737 file);
738 case TRACE_REG_UNREGISTER:
739 tracepoint_probe_unregister(call->tp,
740 call->class->probe,
741 file);
742 return 0;
743
744#ifdef CONFIG_PERF_EVENTS
745 case TRACE_REG_PERF_REGISTER:
746 if (!call->class->perf_probe)
747 return -ENODEV;
748 return tracepoint_probe_register(call->tp,
749 call->class->perf_probe,
750 call);
751 case TRACE_REG_PERF_UNREGISTER:
752 tracepoint_probe_unregister(call->tp,
753 call->class->perf_probe,
754 call);
755 return 0;
756 case TRACE_REG_PERF_OPEN:
757 case TRACE_REG_PERF_CLOSE:
758 case TRACE_REG_PERF_ADD:
759 case TRACE_REG_PERF_DEL:
760 return 0;
761#endif
762 }
763 return 0;
764}
765EXPORT_SYMBOL_GPL(trace_event_reg);
766
767void trace_event_enable_cmd_record(bool enable)
768{
769 struct trace_event_file *file;
770 struct trace_array *tr;
771
772 lockdep_assert_held(&event_mutex);
773
774 do_for_each_event_file(tr, file) {
775
776 if (!(file->flags & EVENT_FILE_FL_ENABLED))
777 continue;
778
779 if (enable) {
780 tracing_start_cmdline_record();
781 set_bit(EVENT_FILE_FL_RECORDED_CMD_BIT, &file->flags);
782 } else {
783 tracing_stop_cmdline_record();
784 clear_bit(EVENT_FILE_FL_RECORDED_CMD_BIT, &file->flags);
785 }
786 } while_for_each_event_file();
787}
788
789void trace_event_enable_tgid_record(bool enable)
790{
791 struct trace_event_file *file;
792 struct trace_array *tr;
793
794 lockdep_assert_held(&event_mutex);
795
796 do_for_each_event_file(tr, file) {
797 if (!(file->flags & EVENT_FILE_FL_ENABLED))
798 continue;
799
800 if (enable) {
801 tracing_start_tgid_record();
802 set_bit(EVENT_FILE_FL_RECORDED_TGID_BIT, &file->flags);
803 } else {
804 tracing_stop_tgid_record();
805 clear_bit(EVENT_FILE_FL_RECORDED_TGID_BIT,
806 &file->flags);
807 }
808 } while_for_each_event_file();
809}
810
811static int __ftrace_event_enable_disable(struct trace_event_file *file,
812 int enable, int soft_disable)
813{
814 struct trace_event_call *call = file->event_call;
815 struct trace_array *tr = file->tr;
816 bool soft_mode = atomic_read(&file->sm_ref) != 0;
817 int ret = 0;
818 int disable;
819
820 switch (enable) {
821 case 0:
822 /*
823 * When soft_disable is set and enable is cleared, the sm_ref
824 * reference counter is decremented. If it reaches 0, we want
825 * to clear the SOFT_DISABLED flag but leave the event in the
826 * state that it was. That is, if the event was enabled and
827 * SOFT_DISABLED isn't set, then do nothing. But if SOFT_DISABLED
828 * is set we do not want the event to be enabled before we
829 * clear the bit.
830 *
831 * When soft_disable is not set but the soft_mode is,
832 * we do nothing. Do not disable the tracepoint, otherwise
833 * "soft enable"s (clearing the SOFT_DISABLED bit) won't work.
834 */
835 if (soft_disable) {
836 if (atomic_dec_return(&file->sm_ref) > 0)
837 break;
838 disable = file->flags & EVENT_FILE_FL_SOFT_DISABLED;
839 soft_mode = false;
840 /* Disable use of trace_buffered_event */
841 trace_buffered_event_disable();
842 } else
843 disable = !soft_mode;
844
845 if (disable && (file->flags & EVENT_FILE_FL_ENABLED)) {
846 clear_bit(EVENT_FILE_FL_ENABLED_BIT, &file->flags);
847 if (file->flags & EVENT_FILE_FL_RECORDED_CMD) {
848 tracing_stop_cmdline_record();
849 clear_bit(EVENT_FILE_FL_RECORDED_CMD_BIT, &file->flags);
850 }
851
852 if (file->flags & EVENT_FILE_FL_RECORDED_TGID) {
853 tracing_stop_tgid_record();
854 clear_bit(EVENT_FILE_FL_RECORDED_TGID_BIT, &file->flags);
855 }
856
857 ret = call->class->reg(call, TRACE_REG_UNREGISTER, file);
858
859 WARN_ON_ONCE(ret);
860 }
861 /* If in soft mode, just set the SOFT_DISABLE_BIT, else clear it */
862 if (soft_mode)
863 set_bit(EVENT_FILE_FL_SOFT_DISABLED_BIT, &file->flags);
864 else
865 clear_bit(EVENT_FILE_FL_SOFT_DISABLED_BIT, &file->flags);
866 break;
867 case 1:
868 /*
869 * When soft_disable is set and enable is set, we want to
870 * register the tracepoint for the event, but leave the event
871 * as is. That means, if the event was already enabled, we do
872 * nothing. If the event is disabled, we set SOFT_DISABLED
873 * before enabling the event tracepoint, so it still seems
874 * to be disabled.
875 */
876 if (!soft_disable)
877 clear_bit(EVENT_FILE_FL_SOFT_DISABLED_BIT, &file->flags);
878 else {
879 if (atomic_inc_return(&file->sm_ref) > 1)
880 break;
881 /* Enable use of trace_buffered_event */
882 trace_buffered_event_enable();
883 }
884
885 if (!(file->flags & EVENT_FILE_FL_ENABLED)) {
886 bool cmd = false, tgid = false;
887
888 /* Keep the event disabled, when going to soft mode. */
889 if (soft_disable)
890 set_bit(EVENT_FILE_FL_SOFT_DISABLED_BIT, &file->flags);
891
892 if (tr->trace_flags & TRACE_ITER(RECORD_CMD)) {
893 cmd = true;
894 tracing_start_cmdline_record();
895 set_bit(EVENT_FILE_FL_RECORDED_CMD_BIT, &file->flags);
896 }
897
898 if (tr->trace_flags & TRACE_ITER(RECORD_TGID)) {
899 tgid = true;
900 tracing_start_tgid_record();
901 set_bit(EVENT_FILE_FL_RECORDED_TGID_BIT, &file->flags);
902 }
903
904 ret = call->class->reg(call, TRACE_REG_REGISTER, file);
905 if (ret) {
906 if (cmd)
907 tracing_stop_cmdline_record();
908 if (tgid)
909 tracing_stop_tgid_record();
910 pr_info("event trace: Could not enable event "
911 "%s\n", trace_event_name(call));
912 break;
913 }
914 set_bit(EVENT_FILE_FL_ENABLED_BIT, &file->flags);
915
916 /* WAS_ENABLED gets set but never cleared. */
917 set_bit(EVENT_FILE_FL_WAS_ENABLED_BIT, &file->flags);
918 }
919 break;
920 }
921
922 return ret;
923}
924
925int trace_event_enable_disable(struct trace_event_file *file,
926 int enable, int soft_disable)
927{
928 return __ftrace_event_enable_disable(file, enable, soft_disable);
929}
930
931static int ftrace_event_enable_disable(struct trace_event_file *file,
932 int enable)
933{
934 return __ftrace_event_enable_disable(file, enable, 0);
935}
936
937#ifdef CONFIG_MODULES
938struct event_mod_load {
939 struct list_head list;
940 char *module;
941 char *match;
942 char *system;
943 char *event;
944};
945
946static void free_event_mod(struct event_mod_load *event_mod)
947{
948 list_del(&event_mod->list);
949 kfree(event_mod->module);
950 kfree(event_mod->match);
951 kfree(event_mod->system);
952 kfree(event_mod->event);
953 kfree(event_mod);
954}
955
956static void clear_mod_events(struct trace_array *tr)
957{
958 struct event_mod_load *event_mod, *n;
959
960 list_for_each_entry_safe(event_mod, n, &tr->mod_events, list) {
961 free_event_mod(event_mod);
962 }
963}
964
965static int remove_cache_mod(struct trace_array *tr, const char *mod,
966 const char *match, const char *system, const char *event)
967{
968 struct event_mod_load *event_mod, *n;
969 int ret = -EINVAL;
970
971 list_for_each_entry_safe(event_mod, n, &tr->mod_events, list) {
972 if (strcmp(event_mod->module, mod) != 0)
973 continue;
974
975 if (match && (!event_mod->match || strcmp(event_mod->match, match) != 0))
976 continue;
977
978 if (system &&
979 (!event_mod->system || strcmp(event_mod->system, system) != 0))
980 continue;
981
982 if (event &&
983 (!event_mod->event || strcmp(event_mod->event, event) != 0))
984 continue;
985
986 free_event_mod(event_mod);
987 ret = 0;
988 }
989
990 return ret;
991}
992
993static int cache_mod(struct trace_array *tr, const char *mod, int set,
994 const char *match, const char *system, const char *event)
995{
996 struct event_mod_load *event_mod;
997
998 /* If the module exists, then this just failed to find an event */
999 if (module_exists(mod))
1000 return -EINVAL;
1001
1002 /* See if this is to remove a cached filter */
1003 if (!set)
1004 return remove_cache_mod(tr, mod, match, system, event);
1005
1006 event_mod = kzalloc_obj(*event_mod);
1007 if (!event_mod)
1008 return -ENOMEM;
1009
1010 INIT_LIST_HEAD(&event_mod->list);
1011 event_mod->module = kstrdup(mod, GFP_KERNEL);
1012 if (!event_mod->module)
1013 goto out_free;
1014
1015 if (match) {
1016 event_mod->match = kstrdup(match, GFP_KERNEL);
1017 if (!event_mod->match)
1018 goto out_free;
1019 }
1020
1021 if (system) {
1022 event_mod->system = kstrdup(system, GFP_KERNEL);
1023 if (!event_mod->system)
1024 goto out_free;
1025 }
1026
1027 if (event) {
1028 event_mod->event = kstrdup(event, GFP_KERNEL);
1029 if (!event_mod->event)
1030 goto out_free;
1031 }
1032
1033 list_add(&event_mod->list, &tr->mod_events);
1034
1035 return 0;
1036
1037 out_free:
1038 free_event_mod(event_mod);
1039
1040 return -ENOMEM;
1041}
1042#else /* CONFIG_MODULES */
1043static inline void clear_mod_events(struct trace_array *tr) { }
1044static int cache_mod(struct trace_array *tr, const char *mod, int set,
1045 const char *match, const char *system, const char *event)
1046{
1047 return -EINVAL;
1048}
1049#endif
1050
1051static void ftrace_clear_events(struct trace_array *tr)
1052{
1053 struct trace_event_file *file;
1054
1055 mutex_lock(&event_mutex);
1056 list_for_each_entry(file, &tr->events, list) {
1057 ftrace_event_enable_disable(file, 0);
1058 }
1059 clear_mod_events(tr);
1060 mutex_unlock(&event_mutex);
1061}
1062
1063static void
1064event_filter_pid_sched_process_exit(void *data, struct task_struct *task)
1065{
1066 struct trace_pid_list *pid_list;
1067 struct trace_array *tr = data;
1068
1069 guard(preempt)();
1070 pid_list = rcu_dereference_raw(tr->filtered_pids);
1071 trace_filter_add_remove_task(pid_list, NULL, task);
1072
1073 pid_list = rcu_dereference_raw(tr->filtered_no_pids);
1074 trace_filter_add_remove_task(pid_list, NULL, task);
1075}
1076
1077static void
1078event_filter_pid_sched_process_fork(void *data,
1079 struct task_struct *self,
1080 struct task_struct *task)
1081{
1082 struct trace_pid_list *pid_list;
1083 struct trace_array *tr = data;
1084
1085 guard(preempt)();
1086 pid_list = rcu_dereference_sched(tr->filtered_pids);
1087 trace_filter_add_remove_task(pid_list, self, task);
1088
1089 pid_list = rcu_dereference_sched(tr->filtered_no_pids);
1090 trace_filter_add_remove_task(pid_list, self, task);
1091}
1092
1093void trace_event_follow_fork(struct trace_array *tr, bool enable)
1094{
1095 if (enable) {
1096 register_trace_prio_sched_process_fork(event_filter_pid_sched_process_fork,
1097 tr, INT_MIN);
1098 register_trace_prio_sched_process_free(event_filter_pid_sched_process_exit,
1099 tr, INT_MAX);
1100 } else {
1101 unregister_trace_sched_process_fork(event_filter_pid_sched_process_fork,
1102 tr);
1103 unregister_trace_sched_process_free(event_filter_pid_sched_process_exit,
1104 tr);
1105 }
1106}
1107
1108static void
1109event_filter_pid_sched_switch_probe_pre(void *data, bool preempt,
1110 struct task_struct *prev,
1111 struct task_struct *next,
1112 unsigned int prev_state)
1113{
1114 struct trace_array *tr = data;
1115 struct trace_pid_list *no_pid_list;
1116 struct trace_pid_list *pid_list;
1117 bool ret;
1118
1119 pid_list = rcu_dereference_sched(tr->filtered_pids);
1120 no_pid_list = rcu_dereference_sched(tr->filtered_no_pids);
1121
1122 /*
1123 * Sched switch is funny, as we only want to ignore it
1124 * in the notrace case if both prev and next should be ignored.
1125 */
1126 ret = trace_ignore_this_task(NULL, no_pid_list, prev) &&
1127 trace_ignore_this_task(NULL, no_pid_list, next);
1128
1129 this_cpu_write(tr->array_buffer.data->ignore_pid, ret ||
1130 (trace_ignore_this_task(pid_list, NULL, prev) &&
1131 trace_ignore_this_task(pid_list, NULL, next)));
1132}
1133
1134static void
1135event_filter_pid_sched_switch_probe_post(void *data, bool preempt,
1136 struct task_struct *prev,
1137 struct task_struct *next,
1138 unsigned int prev_state)
1139{
1140 struct trace_array *tr = data;
1141 struct trace_pid_list *no_pid_list;
1142 struct trace_pid_list *pid_list;
1143
1144 pid_list = rcu_dereference_sched(tr->filtered_pids);
1145 no_pid_list = rcu_dereference_sched(tr->filtered_no_pids);
1146
1147 this_cpu_write(tr->array_buffer.data->ignore_pid,
1148 trace_ignore_this_task(pid_list, no_pid_list, next));
1149}
1150
1151static void
1152event_filter_pid_sched_wakeup_probe_pre(void *data, struct task_struct *task)
1153{
1154 struct trace_array *tr = data;
1155 struct trace_pid_list *no_pid_list;
1156 struct trace_pid_list *pid_list;
1157
1158 /* Nothing to do if we are already tracing */
1159 if (!this_cpu_read(tr->array_buffer.data->ignore_pid))
1160 return;
1161
1162 pid_list = rcu_dereference_sched(tr->filtered_pids);
1163 no_pid_list = rcu_dereference_sched(tr->filtered_no_pids);
1164
1165 this_cpu_write(tr->array_buffer.data->ignore_pid,
1166 trace_ignore_this_task(pid_list, no_pid_list, task));
1167}
1168
1169static void
1170event_filter_pid_sched_wakeup_probe_post(void *data, struct task_struct *task)
1171{
1172 struct trace_array *tr = data;
1173 struct trace_pid_list *no_pid_list;
1174 struct trace_pid_list *pid_list;
1175
1176 /* Nothing to do if we are not tracing */
1177 if (this_cpu_read(tr->array_buffer.data->ignore_pid))
1178 return;
1179
1180 pid_list = rcu_dereference_sched(tr->filtered_pids);
1181 no_pid_list = rcu_dereference_sched(tr->filtered_no_pids);
1182
1183 /* Set tracing if current is enabled */
1184 this_cpu_write(tr->array_buffer.data->ignore_pid,
1185 trace_ignore_this_task(pid_list, no_pid_list, current));
1186}
1187
1188static void unregister_pid_events(struct trace_array *tr)
1189{
1190 unregister_trace_sched_switch(event_filter_pid_sched_switch_probe_pre, tr);
1191 unregister_trace_sched_switch(event_filter_pid_sched_switch_probe_post, tr);
1192
1193 unregister_trace_sched_wakeup(event_filter_pid_sched_wakeup_probe_pre, tr);
1194 unregister_trace_sched_wakeup(event_filter_pid_sched_wakeup_probe_post, tr);
1195
1196 unregister_trace_sched_wakeup_new(event_filter_pid_sched_wakeup_probe_pre, tr);
1197 unregister_trace_sched_wakeup_new(event_filter_pid_sched_wakeup_probe_post, tr);
1198
1199 unregister_trace_sched_waking(event_filter_pid_sched_wakeup_probe_pre, tr);
1200 unregister_trace_sched_waking(event_filter_pid_sched_wakeup_probe_post, tr);
1201}
1202
1203static void __ftrace_clear_event_pids(struct trace_array *tr, int type)
1204{
1205 struct trace_pid_list *pid_list;
1206 struct trace_pid_list *no_pid_list;
1207 struct trace_event_file *file;
1208 int cpu;
1209
1210 pid_list = rcu_dereference_protected(tr->filtered_pids,
1211 lockdep_is_held(&event_mutex));
1212 no_pid_list = rcu_dereference_protected(tr->filtered_no_pids,
1213 lockdep_is_held(&event_mutex));
1214
1215 /* Make sure there's something to do */
1216 if (!pid_type_enabled(type, pid_list, no_pid_list))
1217 return;
1218
1219 if (!still_need_pid_events(type, pid_list, no_pid_list)) {
1220 unregister_pid_events(tr);
1221
1222 list_for_each_entry(file, &tr->events, list) {
1223 clear_bit(EVENT_FILE_FL_PID_FILTER_BIT, &file->flags);
1224 }
1225
1226 for_each_possible_cpu(cpu)
1227 per_cpu_ptr(tr->array_buffer.data, cpu)->ignore_pid = false;
1228 }
1229
1230 if (type & TRACE_PIDS)
1231 rcu_assign_pointer(tr->filtered_pids, NULL);
1232
1233 if (type & TRACE_NO_PIDS)
1234 rcu_assign_pointer(tr->filtered_no_pids, NULL);
1235
1236 /* Wait till all users are no longer using pid filtering */
1237 tracepoint_synchronize_unregister();
1238
1239 if ((type & TRACE_PIDS) && pid_list)
1240 trace_pid_list_free(pid_list);
1241
1242 if ((type & TRACE_NO_PIDS) && no_pid_list)
1243 trace_pid_list_free(no_pid_list);
1244}
1245
1246static void ftrace_clear_event_pids(struct trace_array *tr, int type)
1247{
1248 mutex_lock(&event_mutex);
1249 __ftrace_clear_event_pids(tr, type);
1250 mutex_unlock(&event_mutex);
1251}
1252
1253static void __put_system(struct event_subsystem *system)
1254{
1255 struct event_filter *filter = system->filter;
1256
1257 WARN_ON_ONCE(system_refcount(system) == 0);
1258 if (system_refcount_dec(system))
1259 return;
1260
1261 list_del(&system->list);
1262
1263 if (filter) {
1264 kfree(filter->filter_string);
1265 kfree(filter);
1266 }
1267 kfree_const(system->name);
1268 kfree(system);
1269}
1270
1271static void __get_system(struct event_subsystem *system)
1272{
1273 WARN_ON_ONCE(system_refcount(system) == 0);
1274 system_refcount_inc(system);
1275}
1276
1277static void __get_system_dir(struct trace_subsystem_dir *dir)
1278{
1279 WARN_ON_ONCE(dir->ref_count == 0);
1280 dir->ref_count++;
1281 __get_system(dir->subsystem);
1282}
1283
1284static void __put_system_dir(struct trace_subsystem_dir *dir)
1285{
1286 WARN_ON_ONCE(dir->ref_count == 0);
1287 /* If the subsystem is about to be freed, the dir must be too */
1288 WARN_ON_ONCE(system_refcount(dir->subsystem) == 1 && dir->ref_count != 1);
1289
1290 __put_system(dir->subsystem);
1291 if (!--dir->ref_count)
1292 kfree(dir);
1293}
1294
1295static void put_system(struct trace_subsystem_dir *dir)
1296{
1297 mutex_lock(&event_mutex);
1298 __put_system_dir(dir);
1299 mutex_unlock(&event_mutex);
1300}
1301
1302static void remove_subsystem(struct trace_subsystem_dir *dir)
1303{
1304 if (!dir)
1305 return;
1306
1307 if (!--dir->nr_events) {
1308 eventfs_remove_dir(dir->ei);
1309 list_del(&dir->list);
1310 __put_system_dir(dir);
1311 }
1312}
1313
1314void event_file_get(struct trace_event_file *file)
1315{
1316 refcount_inc(&file->ref);
1317}
1318
1319void event_file_put(struct trace_event_file *file)
1320{
1321 if (WARN_ON_ONCE(!refcount_read(&file->ref))) {
1322 if (file->flags & EVENT_FILE_FL_FREED)
1323 kmem_cache_free(file_cachep, file);
1324 return;
1325 }
1326
1327 if (refcount_dec_and_test(&file->ref)) {
1328 /* Count should only go to zero when it is freed */
1329 if (WARN_ON_ONCE(!(file->flags & EVENT_FILE_FL_FREED)))
1330 return;
1331 kmem_cache_free(file_cachep, file);
1332 }
1333}
1334
1335static void remove_event_file_dir(struct trace_event_file *file)
1336{
1337 eventfs_remove_dir(file->ei);
1338 list_del(&file->list);
1339 remove_subsystem(file->system);
1340 free_event_filter(file->filter);
1341 file->flags |= EVENT_FILE_FL_FREED;
1342 event_file_put(file);
1343
1344 /* Wake up hist poll waiters to notice the EVENT_FILE_FL_FREED flag. */
1345 hist_poll_wakeup();
1346}
1347
1348/*
1349 * __ftrace_set_clr_event(NULL, NULL, NULL, set) will set/unset all events.
1350 */
1351static int
1352__ftrace_set_clr_event_nolock(struct trace_array *tr, const char *match,
1353 const char *sub, const char *event, int set,
1354 const char *mod)
1355{
1356 struct trace_event_file *file;
1357 struct trace_event_call *call;
1358 char *module __free(kfree) = NULL;
1359 const char *name;
1360 int ret = -EINVAL;
1361 int eret = 0;
1362
1363 if (mod) {
1364 char *p;
1365
1366 module = kstrdup(mod, GFP_KERNEL);
1367 if (!module)
1368 return -ENOMEM;
1369
1370 /* Replace all '-' with '_' as that's what modules do */
1371 for (p = strchr(module, '-'); p; p = strchr(p + 1, '-'))
1372 *p = '_';
1373 }
1374
1375 list_for_each_entry(file, &tr->events, list) {
1376
1377 call = file->event_call;
1378
1379 /* If a module is specified, skip events that are not that module */
1380 if (module &&
1381 ((call->flags & TRACE_EVENT_FL_DYNAMIC) ||
1382 !call->module || strcmp(module_name(call->module), module)))
1383 continue;
1384
1385 name = trace_event_name(call);
1386
1387 if (!name || !call->class || !call->class->reg)
1388 continue;
1389
1390 if (call->flags & TRACE_EVENT_FL_IGNORE_ENABLE)
1391 continue;
1392
1393 if (match &&
1394 strcmp(match, name) != 0 &&
1395 strcmp(match, call->class->system) != 0)
1396 continue;
1397
1398 if (sub && strcmp(sub, call->class->system) != 0)
1399 continue;
1400
1401 if (event && strcmp(event, name) != 0)
1402 continue;
1403
1404 ret = ftrace_event_enable_disable(file, set);
1405
1406 /*
1407 * Save the first error and return that. Some events
1408 * may still have been enabled, but let the user
1409 * know that something went wrong.
1410 */
1411 if (ret && !eret)
1412 eret = ret;
1413
1414 ret = eret;
1415 }
1416
1417 /*
1418 * If this is a module setting and nothing was found,
1419 * check if the module was loaded. If it wasn't cache it.
1420 */
1421 if (module && ret == -EINVAL && !eret)
1422 ret = cache_mod(tr, module, set, match, sub, event);
1423
1424 return ret;
1425}
1426
1427static int __ftrace_set_clr_event(struct trace_array *tr, const char *match,
1428 const char *sub, const char *event, int set,
1429 const char *mod)
1430{
1431 int ret;
1432
1433 if (trace_array_is_readonly(tr))
1434 return -EACCES;
1435
1436 mutex_lock(&event_mutex);
1437 ret = __ftrace_set_clr_event_nolock(tr, match, sub, event, set, mod);
1438 mutex_unlock(&event_mutex);
1439
1440 return ret;
1441}
1442
1443int ftrace_set_clr_event(struct trace_array *tr, char *buf, int set)
1444{
1445 char *event = NULL, *sub = NULL, *match, *mod;
1446 int ret;
1447
1448 if (!tr)
1449 return -ENOENT;
1450
1451 /* Modules events can be appended with :mod:<module> */
1452 mod = strstr(buf, ":mod:");
1453 if (mod) {
1454 *mod = '\0';
1455 /* move to the module name */
1456 mod += 5;
1457 }
1458
1459 /*
1460 * The buf format can be <subsystem>:<event-name>
1461 * *:<event-name> means any event by that name.
1462 * :<event-name> is the same.
1463 *
1464 * <subsystem>:* means all events in that subsystem
1465 * <subsystem>: means the same.
1466 *
1467 * <name> (no ':') means all events in a subsystem with
1468 * the name <name> or any event that matches <name>
1469 */
1470
1471 match = strsep(&buf, ":");
1472 if (buf) {
1473 sub = match;
1474 event = buf;
1475 match = NULL;
1476
1477 if (!strlen(sub) || strcmp(sub, "*") == 0)
1478 sub = NULL;
1479 if (!strlen(event) || strcmp(event, "*") == 0)
1480 event = NULL;
1481 } else if (mod) {
1482 /* Allow wildcard for no length or star */
1483 if (!strlen(match) || strcmp(match, "*") == 0)
1484 match = NULL;
1485 }
1486
1487 ret = __ftrace_set_clr_event(tr, match, sub, event, set, mod);
1488
1489 /* Put back the colon to allow this to be called again */
1490 if (buf)
1491 *(buf - 1) = ':';
1492 if (mod)
1493 *(mod - 5) = ':';
1494
1495 return ret;
1496}
1497
1498/**
1499 * trace_set_clr_event - enable or disable an event
1500 * @system: system name to match (NULL for any system)
1501 * @event: event name to match (NULL for all events, within system)
1502 * @set: 1 to enable, 0 to disable
1503 *
1504 * This is a way for other parts of the kernel to enable or disable
1505 * event recording.
1506 *
1507 * Returns 0 on success, -EINVAL if the parameters do not match any
1508 * registered events.
1509 */
1510int trace_set_clr_event(const char *system, const char *event, int set)
1511{
1512 struct trace_array *tr = top_trace_array();
1513
1514 if (!tr)
1515 return -ENODEV;
1516
1517 return __ftrace_set_clr_event(tr, NULL, system, event, set, NULL);
1518}
1519EXPORT_SYMBOL_GPL(trace_set_clr_event);
1520
1521/**
1522 * trace_array_set_clr_event - enable or disable an event for a trace array.
1523 * @tr: concerned trace array.
1524 * @system: system name to match (NULL for any system)
1525 * @event: event name to match (NULL for all events, within system)
1526 * @enable: true to enable, false to disable
1527 *
1528 * This is a way for other parts of the kernel to enable or disable
1529 * event recording.
1530 *
1531 * Returns 0 on success, -EINVAL if the parameters do not match any
1532 * registered events.
1533 */
1534int trace_array_set_clr_event(struct trace_array *tr, const char *system,
1535 const char *event, bool enable)
1536{
1537 int set;
1538
1539 if (!tr)
1540 return -ENOENT;
1541
1542 set = (enable == true) ? 1 : 0;
1543 return __ftrace_set_clr_event(tr, NULL, system, event, set, NULL);
1544}
1545EXPORT_SYMBOL_GPL(trace_array_set_clr_event);
1546
1547/* 128 should be much more than enough */
1548#define EVENT_BUF_SIZE 127
1549
1550static ssize_t
1551ftrace_event_write(struct file *file, const char __user *ubuf,
1552 size_t cnt, loff_t *ppos)
1553{
1554 struct trace_parser parser;
1555 struct seq_file *m = file->private_data;
1556 struct trace_array *tr = m->private;
1557 ssize_t read, ret;
1558
1559 if (!cnt)
1560 return 0;
1561
1562 ret = tracing_update_buffers(tr);
1563 if (ret < 0)
1564 return ret;
1565
1566 if (trace_parser_get_init(&parser, EVENT_BUF_SIZE + 1))
1567 return -ENOMEM;
1568
1569 read = trace_get_user(&parser, ubuf, cnt, ppos);
1570
1571 if (read >= 0 && trace_parser_loaded((&parser))) {
1572 int set = 1;
1573
1574 if (*parser.buffer == '!')
1575 set = 0;
1576
1577 ret = ftrace_set_clr_event(tr, parser.buffer + !set, set);
1578 if (ret)
1579 goto out_put;
1580 }
1581
1582 ret = read;
1583
1584 out_put:
1585 trace_parser_put(&parser);
1586
1587 return ret;
1588}
1589
1590static void *
1591t_next(struct seq_file *m, void *v, loff_t *pos)
1592{
1593 struct trace_event_file *file = v;
1594 struct trace_event_call *call;
1595 struct trace_array *tr = m->private;
1596
1597 (*pos)++;
1598
1599 list_for_each_entry_continue(file, &tr->events, list) {
1600 call = file->event_call;
1601 /*
1602 * The ftrace subsystem is for showing formats only.
1603 * They can not be enabled or disabled via the event files.
1604 */
1605 if (call->class && call->class->reg &&
1606 !(call->flags & TRACE_EVENT_FL_IGNORE_ENABLE))
1607 return file;
1608 }
1609
1610 return NULL;
1611}
1612
1613static void *t_start(struct seq_file *m, loff_t *pos)
1614{
1615 struct trace_event_file *file;
1616 struct trace_array *tr = m->private;
1617 loff_t l;
1618
1619 mutex_lock(&event_mutex);
1620
1621 file = list_entry(&tr->events, struct trace_event_file, list);
1622 for (l = 0; l <= *pos; ) {
1623 file = t_next(m, file, &l);
1624 if (!file)
1625 break;
1626 }
1627 return file;
1628}
1629
1630enum set_event_iter_type {
1631 SET_EVENT_FILE,
1632 SET_EVENT_MOD,
1633};
1634
1635struct set_event_iter {
1636 enum set_event_iter_type type;
1637 union {
1638 struct trace_event_file *file;
1639 struct event_mod_load *event_mod;
1640 };
1641};
1642
1643static void *
1644s_next(struct seq_file *m, void *v, loff_t *pos)
1645{
1646 struct set_event_iter *iter = v;
1647 struct trace_event_file *file;
1648 struct trace_array *tr = m->private;
1649
1650 (*pos)++;
1651
1652 if (iter->type == SET_EVENT_FILE) {
1653 file = iter->file;
1654 list_for_each_entry_continue(file, &tr->events, list) {
1655 if (file->flags & EVENT_FILE_FL_ENABLED) {
1656 iter->file = file;
1657 return iter;
1658 }
1659 }
1660#ifdef CONFIG_MODULES
1661 iter->type = SET_EVENT_MOD;
1662 iter->event_mod = list_entry(&tr->mod_events, struct event_mod_load, list);
1663#endif
1664 }
1665
1666#ifdef CONFIG_MODULES
1667 list_for_each_entry_continue(iter->event_mod, &tr->mod_events, list)
1668 return iter;
1669#endif
1670
1671 /*
1672 * The iter is allocated in s_start() and passed via the 'v'
1673 * parameter. To stop the iterator, NULL must be returned. But
1674 * the return value is what the 'v' parameter in s_stop() receives
1675 * and frees. Free iter here as it will no longer be used.
1676 */
1677 kfree(iter);
1678 return NULL;
1679}
1680
1681static void *s_start(struct seq_file *m, loff_t *pos)
1682{
1683 struct trace_array *tr = m->private;
1684 struct set_event_iter *iter;
1685 loff_t l;
1686
1687 iter = kzalloc_obj(*iter);
1688 mutex_lock(&event_mutex);
1689 if (!iter)
1690 return NULL;
1691
1692 iter->type = SET_EVENT_FILE;
1693 iter->file = list_entry(&tr->events, struct trace_event_file, list);
1694
1695 for (l = 0; l <= *pos; ) {
1696 iter = s_next(m, iter, &l);
1697 if (!iter)
1698 break;
1699 }
1700 return iter;
1701}
1702
1703static int t_show(struct seq_file *m, void *v)
1704{
1705 struct trace_event_file *file = v;
1706 struct trace_event_call *call = file->event_call;
1707
1708 if (strcmp(call->class->system, TRACE_SYSTEM) != 0)
1709 seq_printf(m, "%s:", call->class->system);
1710 seq_printf(m, "%s\n", trace_event_name(call));
1711
1712 return 0;
1713}
1714
1715static void t_stop(struct seq_file *m, void *p)
1716{
1717 mutex_unlock(&event_mutex);
1718}
1719
1720static int get_call_len(struct trace_event_call *call)
1721{
1722 int len;
1723
1724 /* Get the length of "<system>:<event>" */
1725 len = strlen(call->class->system) + 1;
1726 len += strlen(trace_event_name(call));
1727
1728 /* Set the index to 32 bytes to separate event from data */
1729 return len >= 32 ? 1 : 32 - len;
1730}
1731
1732/**
1733 * t_show_filters - seq_file callback to display active event filters
1734 * @m: The seq_file interface for formatted output
1735 * @v: The current trace_event_file being iterated
1736 *
1737 * Identifies and prints active filters for the current event file in the
1738 * iteration. If a filter is applied to the current event and, if so,
1739 * prints the system name, event name, and the filter string.
1740 */
1741static int t_show_filters(struct seq_file *m, void *v)
1742{
1743 struct trace_event_file *file = v;
1744 struct trace_event_call *call = file->event_call;
1745 struct event_filter *filter;
1746 int len;
1747
1748 guard(rcu)();
1749 filter = rcu_dereference(file->filter);
1750 if (!filter || !filter->filter_string)
1751 return 0;
1752
1753 len = get_call_len(call);
1754
1755 seq_printf(m, "%s:%s%*s%s\n", call->class->system,
1756 trace_event_name(call), len, "", filter->filter_string);
1757
1758 return 0;
1759}
1760
1761/**
1762 * t_show_triggers - seq_file callback to display active event triggers
1763 * @m: The seq_file interface for formatted output
1764 * @v: The current trace_event_file being iterated
1765 *
1766 * Iterates through the trigger list of the current event file and prints
1767 * each active trigger's configuration using its associated print
1768 * operation.
1769 */
1770static int t_show_triggers(struct seq_file *m, void *v)
1771{
1772 struct trace_event_file *file = v;
1773 struct trace_event_call *call = file->event_call;
1774 struct event_trigger_data *data;
1775 int len;
1776
1777 /*
1778 * The event_mutex is held by t_start(), protecting the
1779 * file->triggers list traversal.
1780 */
1781 if (list_empty(&file->triggers))
1782 return 0;
1783
1784 len = get_call_len(call);
1785
1786 list_for_each_entry_rcu(data, &file->triggers, list) {
1787 seq_printf(m, "%s:%s%*s", call->class->system,
1788 trace_event_name(call), len, "");
1789
1790 data->cmd_ops->print(m, data);
1791 }
1792
1793 return 0;
1794}
1795
1796#ifdef CONFIG_MODULES
1797static int s_show(struct seq_file *m, void *v)
1798{
1799 struct set_event_iter *iter = v;
1800 const char *system;
1801 const char *event;
1802
1803 if (iter->type == SET_EVENT_FILE)
1804 return t_show(m, iter->file);
1805
1806 /* When match is set, system and event are not */
1807 if (iter->event_mod->match) {
1808 seq_printf(m, "%s:mod:%s\n", iter->event_mod->match,
1809 iter->event_mod->module);
1810 return 0;
1811 }
1812
1813 system = iter->event_mod->system ? : "*";
1814 event = iter->event_mod->event ? : "*";
1815
1816 seq_printf(m, "%s:%s:mod:%s\n", system, event, iter->event_mod->module);
1817
1818 return 0;
1819}
1820#else /* CONFIG_MODULES */
1821static int s_show(struct seq_file *m, void *v)
1822{
1823 struct set_event_iter *iter = v;
1824
1825 return t_show(m, iter->file);
1826}
1827#endif
1828
1829static void s_stop(struct seq_file *m, void *v)
1830{
1831 kfree(v);
1832 t_stop(m, NULL);
1833}
1834
1835static void *
1836__next(struct seq_file *m, void *v, loff_t *pos, int type)
1837{
1838 struct trace_array *tr = m->private;
1839 struct trace_pid_list *pid_list;
1840
1841 if (type == TRACE_PIDS)
1842 pid_list = rcu_dereference_sched(tr->filtered_pids);
1843 else
1844 pid_list = rcu_dereference_sched(tr->filtered_no_pids);
1845
1846 return trace_pid_next(pid_list, v, pos);
1847}
1848
1849static void *
1850p_next(struct seq_file *m, void *v, loff_t *pos)
1851{
1852 return __next(m, v, pos, TRACE_PIDS);
1853}
1854
1855static void *
1856np_next(struct seq_file *m, void *v, loff_t *pos)
1857{
1858 return __next(m, v, pos, TRACE_NO_PIDS);
1859}
1860
1861static void *__start(struct seq_file *m, loff_t *pos, int type)
1862 __acquires(RCU)
1863{
1864 struct trace_pid_list *pid_list;
1865 struct trace_array *tr = m->private;
1866
1867 /*
1868 * Grab the mutex, to keep calls to p_next() having the same
1869 * tr->filtered_pids as p_start() has.
1870 * If we just passed the tr->filtered_pids around, then RCU would
1871 * have been enough, but doing that makes things more complex.
1872 */
1873 mutex_lock(&event_mutex);
1874 rcu_read_lock_sched();
1875
1876 if (type == TRACE_PIDS)
1877 pid_list = rcu_dereference_sched(tr->filtered_pids);
1878 else
1879 pid_list = rcu_dereference_sched(tr->filtered_no_pids);
1880
1881 if (!pid_list)
1882 return NULL;
1883
1884 return trace_pid_start(pid_list, pos);
1885}
1886
1887static void *p_start(struct seq_file *m, loff_t *pos)
1888 __acquires(RCU)
1889{
1890 return __start(m, pos, TRACE_PIDS);
1891}
1892
1893static void *np_start(struct seq_file *m, loff_t *pos)
1894 __acquires(RCU)
1895{
1896 return __start(m, pos, TRACE_NO_PIDS);
1897}
1898
1899static void p_stop(struct seq_file *m, void *p)
1900 __releases(RCU)
1901{
1902 rcu_read_unlock_sched();
1903 mutex_unlock(&event_mutex);
1904}
1905
1906static ssize_t
1907event_enable_read(struct file *filp, char __user *ubuf, size_t cnt,
1908 loff_t *ppos)
1909{
1910 struct trace_event_file *file;
1911 unsigned long flags;
1912 char buf[4] = "0";
1913
1914 mutex_lock(&event_mutex);
1915 file = event_file_file(filp);
1916 if (likely(file))
1917 flags = file->flags;
1918 mutex_unlock(&event_mutex);
1919
1920 if (!file)
1921 return -ENODEV;
1922
1923 if (flags & EVENT_FILE_FL_ENABLED &&
1924 !(flags & EVENT_FILE_FL_SOFT_DISABLED))
1925 strcpy(buf, "1");
1926
1927 if (atomic_read(&file->sm_ref) != 0)
1928 strcat(buf, "*");
1929
1930 strcat(buf, "\n");
1931
1932 return simple_read_from_buffer(ubuf, cnt, ppos, buf, strlen(buf));
1933}
1934
1935static ssize_t
1936event_enable_write(struct file *filp, const char __user *ubuf, size_t cnt,
1937 loff_t *ppos)
1938{
1939 struct trace_event_file *file;
1940 unsigned long val;
1941 int ret;
1942
1943 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
1944 if (ret)
1945 return ret;
1946
1947 guard(mutex)(&event_mutex);
1948
1949 switch (val) {
1950 case 0:
1951 case 1:
1952 file = event_file_file(filp);
1953 if (!file)
1954 return -ENODEV;
1955 ret = tracing_update_buffers(file->tr);
1956 if (ret < 0)
1957 return ret;
1958 ret = ftrace_event_enable_disable(file, val);
1959 if (ret < 0)
1960 return ret;
1961 break;
1962
1963 default:
1964 return -EINVAL;
1965 }
1966
1967 *ppos += cnt;
1968
1969 return cnt;
1970}
1971
1972/*
1973 * Returns:
1974 * 0 : no events exist?
1975 * 1 : all events are disabled
1976 * 2 : all events are enabled
1977 * 3 : some events are enabled and some are enabled
1978 */
1979int trace_events_enabled(struct trace_array *tr, const char *system)
1980{
1981 struct trace_event_call *call;
1982 struct trace_event_file *file;
1983 int set = 0;
1984
1985 guard(mutex)(&event_mutex);
1986
1987 list_for_each_entry(file, &tr->events, list) {
1988 call = file->event_call;
1989 if ((call->flags & TRACE_EVENT_FL_IGNORE_ENABLE) ||
1990 !trace_event_name(call) || !call->class || !call->class->reg)
1991 continue;
1992
1993 if (system && strcmp(call->class->system, system) != 0)
1994 continue;
1995
1996 /*
1997 * We need to find out if all the events are set
1998 * or if all events or cleared, or if we have
1999 * a mixture.
2000 */
2001 set |= (1 << !!(file->flags & EVENT_FILE_FL_ENABLED));
2002
2003 /*
2004 * If we have a mixture, no need to look further.
2005 */
2006 if (set == 3)
2007 break;
2008 }
2009
2010 return set;
2011}
2012
2013static ssize_t
2014system_enable_read(struct file *filp, char __user *ubuf, size_t cnt,
2015 loff_t *ppos)
2016{
2017 const char set_to_char[4] = { '?', '0', '1', 'X' };
2018 struct trace_subsystem_dir *dir = filp->private_data;
2019 struct event_subsystem *system = dir->subsystem;
2020 struct trace_array *tr = dir->tr;
2021 char buf[2];
2022 int set;
2023 int ret;
2024
2025 set = trace_events_enabled(tr, system ? system->name : NULL);
2026
2027 buf[0] = set_to_char[set];
2028 buf[1] = '\n';
2029
2030 ret = simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
2031
2032 return ret;
2033}
2034
2035static ssize_t
2036system_enable_write(struct file *filp, const char __user *ubuf, size_t cnt,
2037 loff_t *ppos)
2038{
2039 struct trace_subsystem_dir *dir = filp->private_data;
2040 struct event_subsystem *system = dir->subsystem;
2041 const char *name = NULL;
2042 unsigned long val;
2043 ssize_t ret;
2044
2045 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
2046 if (ret)
2047 return ret;
2048
2049 ret = tracing_update_buffers(dir->tr);
2050 if (ret < 0)
2051 return ret;
2052
2053 if (val != 0 && val != 1)
2054 return -EINVAL;
2055
2056 /*
2057 * Opening of "enable" adds a ref count to system,
2058 * so the name is safe to use.
2059 */
2060 if (system)
2061 name = system->name;
2062
2063 ret = __ftrace_set_clr_event(dir->tr, NULL, name, NULL, val, NULL);
2064 if (ret)
2065 goto out;
2066
2067 ret = cnt;
2068
2069out:
2070 *ppos += cnt;
2071
2072 return ret;
2073}
2074
2075enum {
2076 FORMAT_HEADER = 1,
2077 FORMAT_FIELD_SEPERATOR = 2,
2078 FORMAT_PRINTFMT = 3,
2079};
2080
2081static void *f_next(struct seq_file *m, void *v, loff_t *pos)
2082{
2083 struct trace_event_file *file = event_file_data(m->private);
2084 struct trace_event_call *call = file->event_call;
2085 struct list_head *common_head = &ftrace_common_fields;
2086 struct list_head *head = trace_get_fields(call);
2087 struct list_head *node = v;
2088
2089 (*pos)++;
2090
2091 switch ((unsigned long)v) {
2092 case FORMAT_HEADER:
2093 node = common_head;
2094 break;
2095
2096 case FORMAT_FIELD_SEPERATOR:
2097 node = head;
2098 break;
2099
2100 case FORMAT_PRINTFMT:
2101 /* all done */
2102 return NULL;
2103 }
2104
2105 node = node->prev;
2106 if (node == common_head)
2107 return (void *)FORMAT_FIELD_SEPERATOR;
2108 else if (node == head)
2109 return (void *)FORMAT_PRINTFMT;
2110 else
2111 return node;
2112}
2113
2114static int f_show(struct seq_file *m, void *v)
2115{
2116 struct trace_event_file *file = event_file_data(m->private);
2117 struct trace_event_call *call = file->event_call;
2118 struct ftrace_event_field *field;
2119 const char *array_descriptor;
2120
2121 switch ((unsigned long)v) {
2122 case FORMAT_HEADER:
2123 seq_printf(m, "name: %s\n", trace_event_name(call));
2124 seq_printf(m, "ID: %d\n", call->event.type);
2125 seq_puts(m, "format:\n");
2126 return 0;
2127
2128 case FORMAT_FIELD_SEPERATOR:
2129 seq_putc(m, '\n');
2130 return 0;
2131
2132 case FORMAT_PRINTFMT:
2133 seq_printf(m, "\nprint fmt: %s\n",
2134 call->print_fmt);
2135 return 0;
2136 }
2137
2138 field = list_entry(v, struct ftrace_event_field, link);
2139 /*
2140 * Smartly shows the array type(except dynamic array).
2141 * Normal:
2142 * field:TYPE VAR
2143 * If TYPE := TYPE[LEN], it is shown:
2144 * field:TYPE VAR[LEN]
2145 */
2146 array_descriptor = strchr(field->type, '[');
2147
2148 if (str_has_prefix(field->type, "__data_loc"))
2149 array_descriptor = NULL;
2150
2151 if (!array_descriptor)
2152 seq_printf(m, "\tfield:%s %s;\toffset:%u;\tsize:%u;\tsigned:%d;\n",
2153 field->type, field->name, field->offset,
2154 field->size, !!field->is_signed);
2155 else if (field->len)
2156 seq_printf(m, "\tfield:%.*s %s[%d];\toffset:%u;\tsize:%u;\tsigned:%d;\n",
2157 (int)(array_descriptor - field->type),
2158 field->type, field->name,
2159 field->len, field->offset,
2160 field->size, !!field->is_signed);
2161 else
2162 seq_printf(m, "\tfield:%.*s %s[];\toffset:%u;\tsize:%u;\tsigned:%d;\n",
2163 (int)(array_descriptor - field->type),
2164 field->type, field->name,
2165 field->offset, field->size, !!field->is_signed);
2166
2167 return 0;
2168}
2169
2170static void *f_start(struct seq_file *m, loff_t *pos)
2171{
2172 struct trace_event_file *file;
2173 void *p = (void *)FORMAT_HEADER;
2174 loff_t l = 0;
2175
2176 /* ->stop() is called even if ->start() fails */
2177 mutex_lock(&event_mutex);
2178 file = event_file_file(m->private);
2179 if (!file)
2180 return ERR_PTR(-ENODEV);
2181
2182 while (l < *pos && p)
2183 p = f_next(m, p, &l);
2184
2185 return p;
2186}
2187
2188static void f_stop(struct seq_file *m, void *p)
2189{
2190 mutex_unlock(&event_mutex);
2191}
2192
2193static const struct seq_operations trace_format_seq_ops = {
2194 .start = f_start,
2195 .next = f_next,
2196 .stop = f_stop,
2197 .show = f_show,
2198};
2199
2200static int trace_format_open(struct inode *inode, struct file *file)
2201{
2202 struct seq_file *m;
2203 int ret;
2204
2205 /* Do we want to hide event format files on tracefs lockdown? */
2206
2207 ret = seq_open(file, &trace_format_seq_ops);
2208 if (ret < 0)
2209 return ret;
2210
2211 m = file->private_data;
2212 m->private = file;
2213
2214 return 0;
2215}
2216
2217#ifdef CONFIG_PERF_EVENTS
2218static ssize_t
2219event_id_read(struct file *filp, char __user *ubuf, size_t cnt, loff_t *ppos)
2220{
2221 /* id is directly in i_private and available for inode's lifetime. */
2222 int id = (long)file_inode(filp)->i_private;
2223 char buf[32];
2224 int len;
2225
2226 WARN_ON(!id);
2227
2228 len = sprintf(buf, "%d\n", id);
2229
2230 return simple_read_from_buffer(ubuf, cnt, ppos, buf, len);
2231}
2232#endif
2233
2234#ifdef CONFIG_BPF_EVENTS
2235static ssize_t
2236event_btf_ids_read(struct file *filp, char __user *ubuf, size_t cnt, loff_t *ppos)
2237{
2238 struct trace_event_file *file;
2239 struct trace_event_call *call;
2240 const struct btf_type *t;
2241 struct module *mod = NULL;
2242 u32 raw_id = 0, tp_id = 0, obj_id = 0;
2243 const u32 *ids;
2244 struct btf *btf;
2245 char buf[128];
2246 int len;
2247
2248 /* Module unload could free call->class and ids[] mid-read. */
2249 scoped_guard(mutex, &event_mutex) {
2250 file = event_file_file(filp);
2251 if (!file)
2252 return -ENODEV;
2253
2254 call = file->event_call;
2255 ids = call->class->btf_ids;
2256 if (!ids)
2257 return -ENOENT;
2258 if (!(call->flags & TRACE_EVENT_FL_DYNAMIC))
2259 mod = (struct module *)call->module;
2260
2261 btf = btf_get_module_btf(mod);
2262 if (IS_ERR_OR_NULL(btf))
2263 return -ENOENT;
2264
2265 /* Module-local ids in ids[] need base+local relocation. */
2266 tp_id = btf_relocate_id(btf, ids[1]);
2267
2268 /*
2269 * Without FL_TRACEPOINT the dispatcher is shared (e.g. all
2270 * per-syscall events fan out from __bpf_trace_sys_enter), so
2271 * raw_btf_id has no per-event attach point — report 0.
2272 */
2273 if (call->flags & TRACE_EVENT_FL_TRACEPOINT) {
2274 t = btf_type_by_id(btf, btf_relocate_id(btf, ids[0]));
2275 raw_id = t ? t->type : 0;
2276 }
2277 obj_id = btf_obj_id(btf);
2278 btf_put(btf);
2279 }
2280
2281 len = scnprintf(buf, sizeof(buf),
2282 "btf_obj_id: %u\nraw_btf_id: %u\ntp_btf_id: %u\n",
2283 obj_id, raw_id, tp_id);
2284
2285 return simple_read_from_buffer(ubuf, cnt, ppos, buf, len);
2286}
2287#endif
2288
2289static ssize_t
2290event_filter_read(struct file *filp, char __user *ubuf, size_t cnt,
2291 loff_t *ppos)
2292{
2293 struct trace_event_file *file;
2294 struct trace_seq *s;
2295 int r = -ENODEV;
2296
2297 if (*ppos)
2298 return 0;
2299
2300 s = kmalloc_obj(*s);
2301
2302 if (!s)
2303 return -ENOMEM;
2304
2305 trace_seq_init(s);
2306
2307 mutex_lock(&event_mutex);
2308 file = event_file_file(filp);
2309 if (file)
2310 print_event_filter(file, s);
2311 mutex_unlock(&event_mutex);
2312
2313 if (file)
2314 r = simple_read_from_buffer(ubuf, cnt, ppos,
2315 s->buffer, trace_seq_used(s));
2316
2317 kfree(s);
2318
2319 return r;
2320}
2321
2322static ssize_t
2323event_filter_write(struct file *filp, const char __user *ubuf, size_t cnt,
2324 loff_t *ppos)
2325{
2326 struct trace_event_file *file;
2327 char *buf;
2328 int err = -ENODEV;
2329
2330 if (cnt >= PAGE_SIZE)
2331 return -EINVAL;
2332
2333 buf = memdup_user_nul(ubuf, cnt);
2334 if (IS_ERR(buf))
2335 return PTR_ERR(buf);
2336
2337 mutex_lock(&event_mutex);
2338 file = event_file_file(filp);
2339 if (file)
2340 err = apply_event_filter(file, buf);
2341 mutex_unlock(&event_mutex);
2342
2343 kfree(buf);
2344 if (err < 0)
2345 return err;
2346
2347 *ppos += cnt;
2348
2349 return cnt;
2350}
2351
2352static LIST_HEAD(event_subsystems);
2353
2354static int subsystem_open(struct inode *inode, struct file *filp)
2355{
2356 struct trace_subsystem_dir *dir = NULL, *iter_dir;
2357 struct trace_array *tr = NULL, *iter_tr;
2358 struct event_subsystem *system = NULL;
2359 int ret;
2360
2361 if (unlikely(tracing_disabled))
2362 return -ENODEV;
2363
2364 /* Make sure the system still exists */
2365 mutex_lock(&event_mutex);
2366 mutex_lock(&trace_types_lock);
2367 list_for_each_entry(iter_tr, &ftrace_trace_arrays, list) {
2368 list_for_each_entry(iter_dir, &iter_tr->systems, list) {
2369 if (iter_dir == inode->i_private) {
2370 /* Don't open systems with no events */
2371 tr = iter_tr;
2372 dir = iter_dir;
2373 if (dir->nr_events) {
2374 __get_system_dir(dir);
2375 system = dir->subsystem;
2376 }
2377 goto exit_loop;
2378 }
2379 }
2380 }
2381 exit_loop:
2382 mutex_unlock(&trace_types_lock);
2383 mutex_unlock(&event_mutex);
2384
2385 if (!system)
2386 return -ENODEV;
2387
2388 /* Still need to increment the ref count of the system */
2389 if (trace_array_get(tr) < 0) {
2390 put_system(dir);
2391 return -ENODEV;
2392 }
2393
2394 ret = tracing_open_generic(inode, filp);
2395 if (ret < 0) {
2396 trace_array_put(tr);
2397 put_system(dir);
2398 }
2399
2400 return ret;
2401}
2402
2403static int system_tr_open(struct inode *inode, struct file *filp)
2404{
2405 struct trace_subsystem_dir *dir;
2406 struct trace_array *tr = inode->i_private;
2407 int ret;
2408
2409 /* Make a temporary dir that has no system but points to tr */
2410 dir = kzalloc_obj(*dir);
2411 if (!dir)
2412 return -ENOMEM;
2413
2414 ret = tracing_open_generic_tr(inode, filp);
2415 if (ret < 0) {
2416 kfree(dir);
2417 return ret;
2418 }
2419 dir->tr = tr;
2420 filp->private_data = dir;
2421
2422 return 0;
2423}
2424
2425static int subsystem_release(struct inode *inode, struct file *file)
2426{
2427 struct trace_subsystem_dir *dir = file->private_data;
2428
2429 trace_array_put(dir->tr);
2430
2431 /*
2432 * If dir->subsystem is NULL, then this is a temporary
2433 * descriptor that was made for a trace_array to enable
2434 * all subsystems.
2435 */
2436 if (dir->subsystem)
2437 put_system(dir);
2438 else
2439 kfree(dir);
2440
2441 return 0;
2442}
2443
2444static ssize_t
2445subsystem_filter_read(struct file *filp, char __user *ubuf, size_t cnt,
2446 loff_t *ppos)
2447{
2448 struct trace_subsystem_dir *dir = filp->private_data;
2449 struct event_subsystem *system = dir->subsystem;
2450 struct trace_seq *s;
2451 int r;
2452
2453 if (*ppos)
2454 return 0;
2455
2456 s = kmalloc_obj(*s);
2457 if (!s)
2458 return -ENOMEM;
2459
2460 trace_seq_init(s);
2461
2462 print_subsystem_event_filter(system, s);
2463 r = simple_read_from_buffer(ubuf, cnt, ppos,
2464 s->buffer, trace_seq_used(s));
2465
2466 kfree(s);
2467
2468 return r;
2469}
2470
2471static ssize_t
2472subsystem_filter_write(struct file *filp, const char __user *ubuf, size_t cnt,
2473 loff_t *ppos)
2474{
2475 struct trace_subsystem_dir *dir = filp->private_data;
2476 char *buf;
2477 int err;
2478
2479 if (cnt >= PAGE_SIZE)
2480 return -EINVAL;
2481
2482 buf = memdup_user_nul(ubuf, cnt);
2483 if (IS_ERR(buf))
2484 return PTR_ERR(buf);
2485
2486 err = apply_subsystem_event_filter(dir, buf);
2487 kfree(buf);
2488 if (err < 0)
2489 return err;
2490
2491 *ppos += cnt;
2492
2493 return cnt;
2494}
2495
2496static ssize_t
2497show_header_page_file(struct file *filp, char __user *ubuf, size_t cnt, loff_t *ppos)
2498{
2499 struct trace_array *tr = filp->private_data;
2500 struct trace_seq *s;
2501 int r;
2502
2503 if (*ppos)
2504 return 0;
2505
2506 s = kmalloc_obj(*s);
2507 if (!s)
2508 return -ENOMEM;
2509
2510 trace_seq_init(s);
2511
2512 ring_buffer_print_page_header(tr->array_buffer.buffer, s);
2513 r = simple_read_from_buffer(ubuf, cnt, ppos,
2514 s->buffer, trace_seq_used(s));
2515
2516 kfree(s);
2517
2518 return r;
2519}
2520
2521static ssize_t
2522show_header_event_file(struct file *filp, char __user *ubuf, size_t cnt, loff_t *ppos)
2523{
2524 struct trace_seq *s;
2525 int r;
2526
2527 if (*ppos)
2528 return 0;
2529
2530 s = kmalloc_obj(*s);
2531 if (!s)
2532 return -ENOMEM;
2533
2534 trace_seq_init(s);
2535
2536 ring_buffer_print_entry_header(s);
2537 r = simple_read_from_buffer(ubuf, cnt, ppos,
2538 s->buffer, trace_seq_used(s));
2539
2540 kfree(s);
2541
2542 return r;
2543}
2544
2545static void ignore_task_cpu(void *data)
2546{
2547 struct trace_array *tr = data;
2548 struct trace_pid_list *pid_list;
2549 struct trace_pid_list *no_pid_list;
2550
2551 /*
2552 * This function is called by on_each_cpu() while the
2553 * event_mutex is held.
2554 */
2555 pid_list = rcu_dereference_protected(tr->filtered_pids,
2556 mutex_is_locked(&event_mutex));
2557 no_pid_list = rcu_dereference_protected(tr->filtered_no_pids,
2558 mutex_is_locked(&event_mutex));
2559
2560 this_cpu_write(tr->array_buffer.data->ignore_pid,
2561 trace_ignore_this_task(pid_list, no_pid_list, current));
2562}
2563
2564static void register_pid_events(struct trace_array *tr)
2565{
2566 /*
2567 * Register a probe that is called before all other probes
2568 * to set ignore_pid if next or prev do not match.
2569 * Register a probe this is called after all other probes
2570 * to only keep ignore_pid set if next pid matches.
2571 */
2572 register_trace_prio_sched_switch(event_filter_pid_sched_switch_probe_pre,
2573 tr, INT_MAX);
2574 register_trace_prio_sched_switch(event_filter_pid_sched_switch_probe_post,
2575 tr, 0);
2576
2577 register_trace_prio_sched_wakeup(event_filter_pid_sched_wakeup_probe_pre,
2578 tr, INT_MAX);
2579 register_trace_prio_sched_wakeup(event_filter_pid_sched_wakeup_probe_post,
2580 tr, 0);
2581
2582 register_trace_prio_sched_wakeup_new(event_filter_pid_sched_wakeup_probe_pre,
2583 tr, INT_MAX);
2584 register_trace_prio_sched_wakeup_new(event_filter_pid_sched_wakeup_probe_post,
2585 tr, 0);
2586
2587 register_trace_prio_sched_waking(event_filter_pid_sched_wakeup_probe_pre,
2588 tr, INT_MAX);
2589 register_trace_prio_sched_waking(event_filter_pid_sched_wakeup_probe_post,
2590 tr, 0);
2591}
2592
2593static ssize_t
2594event_pid_write(struct file *filp, const char __user *ubuf,
2595 size_t cnt, loff_t *ppos, int type)
2596{
2597 struct seq_file *m = filp->private_data;
2598 struct trace_array *tr = m->private;
2599 struct trace_pid_list *filtered_pids = NULL;
2600 struct trace_pid_list *other_pids = NULL;
2601 struct trace_pid_list *pid_list;
2602 struct trace_event_file *file;
2603 ssize_t ret;
2604
2605 if (!cnt)
2606 return 0;
2607
2608 ret = tracing_update_buffers(tr);
2609 if (ret < 0)
2610 return ret;
2611
2612 guard(mutex)(&event_mutex);
2613
2614 if (type == TRACE_PIDS) {
2615 filtered_pids = rcu_dereference_protected(tr->filtered_pids,
2616 lockdep_is_held(&event_mutex));
2617 other_pids = rcu_dereference_protected(tr->filtered_no_pids,
2618 lockdep_is_held(&event_mutex));
2619 } else {
2620 filtered_pids = rcu_dereference_protected(tr->filtered_no_pids,
2621 lockdep_is_held(&event_mutex));
2622 other_pids = rcu_dereference_protected(tr->filtered_pids,
2623 lockdep_is_held(&event_mutex));
2624 }
2625
2626 ret = trace_pid_write(filtered_pids, &pid_list, ubuf, cnt);
2627 if (ret < 0)
2628 return ret;
2629
2630 if (type == TRACE_PIDS)
2631 rcu_assign_pointer(tr->filtered_pids, pid_list);
2632 else
2633 rcu_assign_pointer(tr->filtered_no_pids, pid_list);
2634
2635 list_for_each_entry(file, &tr->events, list) {
2636 set_bit(EVENT_FILE_FL_PID_FILTER_BIT, &file->flags);
2637 }
2638
2639 if (filtered_pids) {
2640 tracepoint_synchronize_unregister();
2641 trace_pid_list_free(filtered_pids);
2642 } else if (pid_list && !other_pids) {
2643 register_pid_events(tr);
2644 }
2645
2646 /*
2647 * Ignoring of pids is done at task switch. But we have to
2648 * check for those tasks that are currently running.
2649 * Always do this in case a pid was appended or removed.
2650 */
2651 on_each_cpu(ignore_task_cpu, tr, 1);
2652
2653 *ppos += ret;
2654
2655 return ret;
2656}
2657
2658static ssize_t
2659ftrace_event_pid_write(struct file *filp, const char __user *ubuf,
2660 size_t cnt, loff_t *ppos)
2661{
2662 return event_pid_write(filp, ubuf, cnt, ppos, TRACE_PIDS);
2663}
2664
2665static ssize_t
2666ftrace_event_npid_write(struct file *filp, const char __user *ubuf,
2667 size_t cnt, loff_t *ppos)
2668{
2669 return event_pid_write(filp, ubuf, cnt, ppos, TRACE_NO_PIDS);
2670}
2671
2672static int ftrace_event_avail_open(struct inode *inode, struct file *file);
2673static int ftrace_event_set_open(struct inode *inode, struct file *file);
2674static int ftrace_event_show_filters_open(struct inode *inode, struct file *file);
2675static int ftrace_event_show_triggers_open(struct inode *inode, struct file *file);
2676static int ftrace_event_set_pid_open(struct inode *inode, struct file *file);
2677static int ftrace_event_set_npid_open(struct inode *inode, struct file *file);
2678static int ftrace_event_release(struct inode *inode, struct file *file);
2679
2680static const struct seq_operations show_event_seq_ops = {
2681 .start = t_start,
2682 .next = t_next,
2683 .show = t_show,
2684 .stop = t_stop,
2685};
2686
2687static const struct seq_operations show_set_event_seq_ops = {
2688 .start = s_start,
2689 .next = s_next,
2690 .show = s_show,
2691 .stop = s_stop,
2692};
2693
2694static const struct seq_operations show_show_event_filters_seq_ops = {
2695 .start = t_start,
2696 .next = t_next,
2697 .show = t_show_filters,
2698 .stop = t_stop,
2699};
2700
2701static const struct seq_operations show_show_event_triggers_seq_ops = {
2702 .start = t_start,
2703 .next = t_next,
2704 .show = t_show_triggers,
2705 .stop = t_stop,
2706};
2707
2708static const struct seq_operations show_set_pid_seq_ops = {
2709 .start = p_start,
2710 .next = p_next,
2711 .show = trace_pid_show,
2712 .stop = p_stop,
2713};
2714
2715static const struct seq_operations show_set_no_pid_seq_ops = {
2716 .start = np_start,
2717 .next = np_next,
2718 .show = trace_pid_show,
2719 .stop = p_stop,
2720};
2721
2722static const struct file_operations ftrace_avail_fops = {
2723 .open = ftrace_event_avail_open,
2724 .read = seq_read,
2725 .llseek = seq_lseek,
2726 .release = seq_release,
2727};
2728
2729static const struct file_operations ftrace_set_event_fops = {
2730 .open = ftrace_event_set_open,
2731 .read = seq_read,
2732 .write = ftrace_event_write,
2733 .llseek = seq_lseek,
2734 .release = ftrace_event_release,
2735};
2736
2737static const struct file_operations ftrace_show_event_filters_fops = {
2738 .open = ftrace_event_show_filters_open,
2739 .read = seq_read,
2740 .llseek = seq_lseek,
2741 .release = ftrace_event_release,
2742};
2743
2744static const struct file_operations ftrace_show_event_triggers_fops = {
2745 .open = ftrace_event_show_triggers_open,
2746 .read = seq_read,
2747 .llseek = seq_lseek,
2748 .release = ftrace_event_release,
2749};
2750
2751static const struct file_operations ftrace_set_event_pid_fops = {
2752 .open = ftrace_event_set_pid_open,
2753 .read = seq_read,
2754 .write = ftrace_event_pid_write,
2755 .llseek = seq_lseek,
2756 .release = ftrace_event_release,
2757};
2758
2759static const struct file_operations ftrace_set_event_notrace_pid_fops = {
2760 .open = ftrace_event_set_npid_open,
2761 .read = seq_read,
2762 .write = ftrace_event_npid_write,
2763 .llseek = seq_lseek,
2764 .release = ftrace_event_release,
2765};
2766
2767static const struct file_operations ftrace_enable_fops = {
2768 .open = tracing_open_file_tr,
2769 .read = event_enable_read,
2770 .write = event_enable_write,
2771 .release = tracing_release_file_tr,
2772 .llseek = default_llseek,
2773};
2774
2775static const struct file_operations ftrace_event_format_fops = {
2776 .open = trace_format_open,
2777 .read = seq_read,
2778 .llseek = seq_lseek,
2779 .release = seq_release,
2780};
2781
2782#ifdef CONFIG_PERF_EVENTS
2783static const struct file_operations ftrace_event_id_fops = {
2784 .read = event_id_read,
2785 .llseek = default_llseek,
2786};
2787#endif
2788
2789#ifdef CONFIG_BPF_EVENTS
2790static const struct file_operations ftrace_event_btf_ids_fops = {
2791 .read = event_btf_ids_read,
2792 .llseek = default_llseek,
2793};
2794#endif
2795
2796static const struct file_operations ftrace_event_filter_fops = {
2797 .open = tracing_open_file_tr,
2798 .read = event_filter_read,
2799 .write = event_filter_write,
2800 .release = tracing_release_file_tr,
2801 .llseek = default_llseek,
2802};
2803
2804static const struct file_operations ftrace_subsystem_filter_fops = {
2805 .open = subsystem_open,
2806 .read = subsystem_filter_read,
2807 .write = subsystem_filter_write,
2808 .llseek = default_llseek,
2809 .release = subsystem_release,
2810};
2811
2812static const struct file_operations ftrace_system_enable_fops = {
2813 .open = subsystem_open,
2814 .read = system_enable_read,
2815 .write = system_enable_write,
2816 .llseek = default_llseek,
2817 .release = subsystem_release,
2818};
2819
2820static const struct file_operations ftrace_tr_enable_fops = {
2821 .open = system_tr_open,
2822 .read = system_enable_read,
2823 .write = system_enable_write,
2824 .llseek = default_llseek,
2825 .release = subsystem_release,
2826};
2827
2828static const struct file_operations ftrace_show_header_page_fops = {
2829 .open = tracing_open_generic_tr,
2830 .read = show_header_page_file,
2831 .llseek = default_llseek,
2832 .release = tracing_release_generic_tr,
2833};
2834
2835static const struct file_operations ftrace_show_header_event_fops = {
2836 .open = tracing_open_generic_tr,
2837 .read = show_header_event_file,
2838 .llseek = default_llseek,
2839 .release = tracing_release_generic_tr,
2840};
2841
2842static int
2843ftrace_event_open(struct inode *inode, struct file *file,
2844 const struct seq_operations *seq_ops)
2845{
2846 struct seq_file *m;
2847 int ret;
2848
2849 ret = security_locked_down(LOCKDOWN_TRACEFS);
2850 if (ret)
2851 return ret;
2852
2853 ret = seq_open(file, seq_ops);
2854 if (ret < 0)
2855 return ret;
2856 m = file->private_data;
2857 /* copy tr over to seq ops */
2858 m->private = inode->i_private;
2859
2860 return ret;
2861}
2862
2863static int ftrace_event_release(struct inode *inode, struct file *file)
2864{
2865 struct trace_array *tr = inode->i_private;
2866
2867 trace_array_put(tr);
2868
2869 return seq_release(inode, file);
2870}
2871
2872static int
2873ftrace_event_avail_open(struct inode *inode, struct file *file)
2874{
2875 const struct seq_operations *seq_ops = &show_event_seq_ops;
2876
2877 /* Checks for tracefs lockdown */
2878 return ftrace_event_open(inode, file, seq_ops);
2879}
2880
2881static int
2882ftrace_event_set_open(struct inode *inode, struct file *file)
2883{
2884 const struct seq_operations *seq_ops = &show_set_event_seq_ops;
2885 struct trace_array *tr = inode->i_private;
2886 int ret;
2887
2888 ret = tracing_check_open_get_tr(tr);
2889 if (ret)
2890 return ret;
2891
2892 if ((file->f_mode & FMODE_WRITE) &&
2893 (file->f_flags & O_TRUNC))
2894 ftrace_clear_events(tr);
2895
2896 ret = ftrace_event_open(inode, file, seq_ops);
2897 if (ret < 0)
2898 trace_array_put(tr);
2899 return ret;
2900}
2901
2902/**
2903 * ftrace_event_show_filters_open - open interface for set_event_filters
2904 * @inode: The inode of the file
2905 * @file: The file being opened
2906 *
2907 * Connects the set_event_filters file to the sequence operations
2908 * required to iterate over and display active event filters.
2909 */
2910static int
2911ftrace_event_show_filters_open(struct inode *inode, struct file *file)
2912{
2913 struct trace_array *tr = inode->i_private;
2914 int ret;
2915
2916 ret = tracing_check_open_get_tr(tr);
2917 if (ret)
2918 return ret;
2919
2920 ret = ftrace_event_open(inode, file, &show_show_event_filters_seq_ops);
2921 if (ret < 0)
2922 trace_array_put(tr);
2923 return ret;
2924}
2925
2926/**
2927 * ftrace_event_show_triggers_open - open interface for show_event_triggers
2928 * @inode: The inode of the file
2929 * @file: The file being opened
2930 *
2931 * Connects the show_event_triggers file to the sequence operations
2932 * required to iterate over and display active event triggers.
2933 */
2934static int
2935ftrace_event_show_triggers_open(struct inode *inode, struct file *file)
2936{
2937 struct trace_array *tr = inode->i_private;
2938 int ret;
2939
2940 ret = tracing_check_open_get_tr(tr);
2941 if (ret)
2942 return ret;
2943
2944 ret = ftrace_event_open(inode, file, &show_show_event_triggers_seq_ops);
2945 if (ret < 0)
2946 trace_array_put(tr);
2947 return ret;
2948}
2949
2950static int
2951ftrace_event_set_pid_open(struct inode *inode, struct file *file)
2952{
2953 const struct seq_operations *seq_ops = &show_set_pid_seq_ops;
2954 struct trace_array *tr = inode->i_private;
2955 int ret;
2956
2957 ret = tracing_check_open_get_tr(tr);
2958 if (ret)
2959 return ret;
2960
2961 if ((file->f_mode & FMODE_WRITE) &&
2962 (file->f_flags & O_TRUNC))
2963 ftrace_clear_event_pids(tr, TRACE_PIDS);
2964
2965 ret = ftrace_event_open(inode, file, seq_ops);
2966 if (ret < 0)
2967 trace_array_put(tr);
2968 return ret;
2969}
2970
2971static int
2972ftrace_event_set_npid_open(struct inode *inode, struct file *file)
2973{
2974 const struct seq_operations *seq_ops = &show_set_no_pid_seq_ops;
2975 struct trace_array *tr = inode->i_private;
2976 int ret;
2977
2978 ret = tracing_check_open_get_tr(tr);
2979 if (ret)
2980 return ret;
2981
2982 if ((file->f_mode & FMODE_WRITE) &&
2983 (file->f_flags & O_TRUNC))
2984 ftrace_clear_event_pids(tr, TRACE_NO_PIDS);
2985
2986 ret = ftrace_event_open(inode, file, seq_ops);
2987 if (ret < 0)
2988 trace_array_put(tr);
2989 return ret;
2990}
2991
2992static struct event_subsystem *
2993create_new_subsystem(const char *name)
2994{
2995 struct event_subsystem *system;
2996
2997 /* need to create new entry */
2998 system = kmalloc_obj(*system);
2999 if (!system)
3000 return NULL;
3001
3002 system->ref_count = 1;
3003
3004 /* Only allocate if dynamic (kprobes and modules) */
3005 system->name = kstrdup_const(name, GFP_KERNEL);
3006 if (!system->name)
3007 goto out_free;
3008
3009 system->filter = kzalloc_obj(struct event_filter);
3010 if (!system->filter)
3011 goto out_free;
3012
3013 list_add(&system->list, &event_subsystems);
3014
3015 return system;
3016
3017 out_free:
3018 kfree_const(system->name);
3019 kfree(system);
3020 return NULL;
3021}
3022
3023static int system_callback(const char *name, umode_t *mode, void **data,
3024 const struct file_operations **fops)
3025{
3026 if (strcmp(name, "filter") == 0)
3027 *fops = &ftrace_subsystem_filter_fops;
3028
3029 else if (strcmp(name, "enable") == 0)
3030 *fops = &ftrace_system_enable_fops;
3031
3032 else
3033 return 0;
3034
3035 *mode = TRACE_MODE_WRITE;
3036 return 1;
3037}
3038
3039static struct eventfs_inode *
3040event_subsystem_dir(struct trace_array *tr, const char *name,
3041 struct trace_event_file *file, struct eventfs_inode *parent)
3042{
3043 struct event_subsystem *system, *iter;
3044 struct trace_subsystem_dir *dir;
3045 struct eventfs_inode *ei;
3046 int nr_entries;
3047 static struct eventfs_entry system_entries[] = {
3048 {
3049 .name = "filter",
3050 .callback = system_callback,
3051 },
3052 {
3053 .name = "enable",
3054 .callback = system_callback,
3055 }
3056 };
3057
3058 /* First see if we did not already create this dir */
3059 list_for_each_entry(dir, &tr->systems, list) {
3060 system = dir->subsystem;
3061 if (strcmp(system->name, name) == 0) {
3062 dir->nr_events++;
3063 file->system = dir;
3064 return dir->ei;
3065 }
3066 }
3067
3068 /* Now see if the system itself exists. */
3069 system = NULL;
3070 list_for_each_entry(iter, &event_subsystems, list) {
3071 if (strcmp(iter->name, name) == 0) {
3072 system = iter;
3073 break;
3074 }
3075 }
3076
3077 dir = kmalloc_obj(*dir);
3078 if (!dir)
3079 goto out_fail;
3080
3081 if (!system) {
3082 system = create_new_subsystem(name);
3083 if (!system)
3084 goto out_free;
3085 } else
3086 __get_system(system);
3087
3088 /* ftrace only has directories no files, readonly instance too. */
3089 if (strcmp(name, "ftrace") == 0 || trace_array_is_readonly(tr))
3090 nr_entries = 0;
3091 else
3092 nr_entries = ARRAY_SIZE(system_entries);
3093
3094 ei = eventfs_create_dir(name, parent, system_entries, nr_entries, dir);
3095 if (IS_ERR(ei)) {
3096 pr_warn("Failed to create system directory %s\n", name);
3097 __put_system(system);
3098 goto out_free;
3099 }
3100
3101 dir->ei = ei;
3102 dir->tr = tr;
3103 dir->ref_count = 1;
3104 dir->nr_events = 1;
3105 dir->subsystem = system;
3106 file->system = dir;
3107
3108 list_add(&dir->list, &tr->systems);
3109
3110 return dir->ei;
3111
3112 out_free:
3113 kfree(dir);
3114 out_fail:
3115 /* Only print this message if failed on memory allocation */
3116 if (!dir || !system)
3117 pr_warn("No memory to create event subsystem %s\n", name);
3118 return NULL;
3119}
3120
3121static int
3122event_define_fields(struct trace_event_call *call)
3123{
3124 struct list_head *head;
3125 int ret = 0;
3126
3127 /*
3128 * Other events may have the same class. Only update
3129 * the fields if they are not already defined.
3130 */
3131 head = trace_get_fields(call);
3132 if (list_empty(head)) {
3133 struct trace_event_fields *field = call->class->fields_array;
3134 unsigned int offset = sizeof(struct trace_entry);
3135
3136 for (; field->type; field++) {
3137 if (field->type == TRACE_FUNCTION_TYPE) {
3138 field->define_fields(call);
3139 break;
3140 }
3141
3142 offset = ALIGN(offset, field->align);
3143 ret = trace_define_field_ext(call, field->type, field->name,
3144 offset, field->size,
3145 field->is_signed, field->filter_type,
3146 field->len, field->needs_test);
3147 if (WARN_ON_ONCE(ret)) {
3148 pr_err("error code is %d\n", ret);
3149 break;
3150 }
3151
3152 offset += field->size;
3153 }
3154 }
3155
3156 return ret;
3157}
3158
3159static int event_callback(const char *name, umode_t *mode, void **data,
3160 const struct file_operations **fops)
3161{
3162 struct trace_event_file *file = *data;
3163 struct trace_event_call *call = file->event_call;
3164
3165 if (strcmp(name, "format") == 0) {
3166 *mode = TRACE_MODE_READ;
3167 *fops = &ftrace_event_format_fops;
3168 return 1;
3169 }
3170
3171 /*
3172 * Only event directories that can be enabled should have
3173 * triggers or filters, with the exception of the "print"
3174 * event that can have a "trigger" file.
3175 */
3176 if (!(call->flags & TRACE_EVENT_FL_IGNORE_ENABLE)) {
3177 if (call->class->reg && strcmp(name, "enable") == 0) {
3178 *mode = TRACE_MODE_WRITE;
3179 *fops = &ftrace_enable_fops;
3180 return 1;
3181 }
3182
3183 if (strcmp(name, "filter") == 0) {
3184 *mode = TRACE_MODE_WRITE;
3185 *fops = &ftrace_event_filter_fops;
3186 return 1;
3187 }
3188 }
3189
3190 if (!(call->flags & TRACE_EVENT_FL_IGNORE_ENABLE) ||
3191 strcmp(trace_event_name(call), "print") == 0) {
3192 if (strcmp(name, "trigger") == 0) {
3193 *mode = TRACE_MODE_WRITE;
3194 *fops = &event_trigger_fops;
3195 return 1;
3196 }
3197 }
3198
3199#ifdef CONFIG_PERF_EVENTS
3200 if (call->event.type && call->class->reg &&
3201 strcmp(name, "id") == 0) {
3202 *mode = TRACE_MODE_READ;
3203 *data = (void *)(long)call->event.type;
3204 *fops = &ftrace_event_id_fops;
3205 return 1;
3206 }
3207#endif
3208
3209#ifdef CONFIG_BPF_EVENTS
3210 if (call->class->btf_ids && strcmp(name, "btf_ids") == 0) {
3211 *mode = TRACE_MODE_READ;
3212 *fops = &ftrace_event_btf_ids_fops;
3213 return 1;
3214 }
3215#endif
3216
3217#ifdef CONFIG_HIST_TRIGGERS
3218 if (strcmp(name, "hist") == 0) {
3219 *mode = TRACE_MODE_READ;
3220 *fops = &event_hist_fops;
3221 return 1;
3222 }
3223#endif
3224#ifdef CONFIG_HIST_TRIGGERS_DEBUG
3225 if (strcmp(name, "hist_debug") == 0) {
3226 *mode = TRACE_MODE_READ;
3227 *fops = &event_hist_debug_fops;
3228 return 1;
3229 }
3230#endif
3231#ifdef CONFIG_TRACE_EVENT_INJECT
3232 if (call->event.type && call->class->reg &&
3233 strcmp(name, "inject") == 0) {
3234 *mode = 0200;
3235 *fops = &event_inject_fops;
3236 return 1;
3237 }
3238#endif
3239 return 0;
3240}
3241
3242/* The file is incremented on creation and freeing the enable file decrements it */
3243static void event_release(const char *name, void *data)
3244{
3245 struct trace_event_file *file = data;
3246
3247 event_file_put(file);
3248}
3249
3250static int
3251event_create_dir(struct eventfs_inode *parent, struct trace_event_file *file)
3252{
3253 struct trace_event_call *call = file->event_call;
3254 struct trace_array *tr = file->tr;
3255 struct eventfs_inode *e_events;
3256 struct eventfs_inode *ei;
3257 const char *name;
3258 int nr_entries;
3259 int ret;
3260 static struct eventfs_entry event_entries[] = {
3261 {
3262 .name = "format",
3263 .callback = event_callback,
3264 },
3265#ifdef CONFIG_PERF_EVENTS
3266 {
3267 .name = "id",
3268 .callback = event_callback,
3269 },
3270#endif
3271#ifdef CONFIG_BPF_EVENTS
3272 {
3273 .name = "btf_ids",
3274 .callback = event_callback,
3275 },
3276#endif
3277#define NR_RO_EVENT_ENTRIES (1 + IS_ENABLED(CONFIG_PERF_EVENTS) + \
3278 IS_ENABLED(CONFIG_BPF_EVENTS))
3279/* Readonly files must be above this line and counted by NR_RO_EVENT_ENTRIES. */
3280 {
3281 .name = "enable",
3282 .callback = event_callback,
3283 .release = event_release,
3284 },
3285 {
3286 .name = "filter",
3287 .callback = event_callback,
3288 },
3289 {
3290 .name = "trigger",
3291 .callback = event_callback,
3292 },
3293#ifdef CONFIG_HIST_TRIGGERS
3294 {
3295 .name = "hist",
3296 .callback = event_callback,
3297 },
3298#endif
3299#ifdef CONFIG_HIST_TRIGGERS_DEBUG
3300 {
3301 .name = "hist_debug",
3302 .callback = event_callback,
3303 },
3304#endif
3305#ifdef CONFIG_TRACE_EVENT_INJECT
3306 {
3307 .name = "inject",
3308 .callback = event_callback,
3309 },
3310#endif
3311 };
3312
3313 /*
3314 * If the trace point header did not define TRACE_SYSTEM
3315 * then the system would be called "TRACE_SYSTEM". This should
3316 * never happen.
3317 */
3318 if (WARN_ON_ONCE(strcmp(call->class->system, TRACE_SYSTEM) == 0))
3319 return -ENODEV;
3320
3321 ret = event_define_fields(call);
3322 if (ret < 0) {
3323 pr_warn("Could not initialize trace point events/%s\n",
3324 trace_event_name(call));
3325 return ret;
3326 }
3327
3328 e_events = event_subsystem_dir(tr, call->class->system, file, parent);
3329 if (!e_events)
3330 return -ENOMEM;
3331
3332 if (trace_array_is_readonly(tr))
3333 nr_entries = NR_RO_EVENT_ENTRIES;
3334 else
3335 nr_entries = ARRAY_SIZE(event_entries);
3336
3337 name = trace_event_name(call);
3338 ei = eventfs_create_dir(name, e_events, event_entries, nr_entries, file);
3339 if (IS_ERR(ei)) {
3340 pr_warn("Could not create tracefs '%s' directory\n", name);
3341 return -1;
3342 }
3343
3344 file->ei = ei;
3345
3346 /* Gets decremented on freeing of the "enable" file */
3347 event_file_get(file);
3348
3349 return 0;
3350}
3351
3352static void remove_event_from_tracers(struct trace_event_call *call)
3353{
3354 struct trace_event_file *file;
3355 struct trace_array *tr;
3356
3357 do_for_each_event_file_safe(tr, file) {
3358 if (file->event_call != call)
3359 continue;
3360
3361 remove_event_file_dir(file);
3362 /*
3363 * The do_for_each_event_file_safe() is
3364 * a double loop. After finding the call for this
3365 * trace_array, we use break to jump to the next
3366 * trace_array.
3367 */
3368 break;
3369 } while_for_each_event_file();
3370}
3371
3372static void event_remove(struct trace_event_call *call)
3373{
3374 struct trace_array *tr;
3375 struct trace_event_file *file;
3376
3377 do_for_each_event_file(tr, file) {
3378 if (file->event_call != call)
3379 continue;
3380
3381 if (file->flags & EVENT_FILE_FL_WAS_ENABLED)
3382 tr->clear_trace = true;
3383
3384 ftrace_event_enable_disable(file, 0);
3385 /*
3386 * The do_for_each_event_file() is
3387 * a double loop. After finding the call for this
3388 * trace_array, we use break to jump to the next
3389 * trace_array.
3390 */
3391 break;
3392 } while_for_each_event_file();
3393
3394 if (call->event.funcs)
3395 __unregister_trace_event(&call->event);
3396 remove_event_from_tracers(call);
3397 list_del(&call->list);
3398}
3399
3400static int event_init(struct trace_event_call *call)
3401{
3402 int ret = 0;
3403 const char *name;
3404
3405 name = trace_event_name(call);
3406 if (WARN_ON(!name))
3407 return -EINVAL;
3408
3409 if (call->class->raw_init) {
3410 ret = call->class->raw_init(call);
3411 if (ret < 0 && ret != -ENOSYS)
3412 pr_warn("Could not initialize trace events/%s\n", name);
3413 }
3414
3415 return ret;
3416}
3417
3418static int
3419__register_event(struct trace_event_call *call, struct module *mod)
3420{
3421 int ret;
3422
3423 ret = event_init(call);
3424 if (ret < 0)
3425 return ret;
3426
3427 down_write(&trace_event_sem);
3428 list_add(&call->list, &ftrace_events);
3429 up_write(&trace_event_sem);
3430
3431 if (call->flags & TRACE_EVENT_FL_DYNAMIC)
3432 atomic_set(&call->refcnt, 0);
3433 else
3434 call->module = mod;
3435
3436 return 0;
3437}
3438
3439static char *eval_replace(char *ptr, struct trace_eval_map *map, int len)
3440{
3441 int rlen;
3442 int elen;
3443
3444 /* Find the length of the eval value as a string */
3445 elen = snprintf(ptr, 0, "%ld", map->eval_value);
3446 /* Make sure there's enough room to replace the string with the value */
3447 if (len < elen)
3448 return NULL;
3449
3450 snprintf(ptr, elen + 1, "%ld", map->eval_value);
3451
3452 /* Get the rest of the string of ptr */
3453 rlen = strlen(ptr + len);
3454 memmove(ptr + elen, ptr + len, rlen);
3455 /* Make sure we end the new string */
3456 ptr[elen + rlen] = 0;
3457
3458 return ptr + elen;
3459}
3460
3461static void update_event_printk(struct trace_event_call *call,
3462 struct trace_eval_map *map)
3463{
3464 char *ptr;
3465 int quote = 0;
3466 int len = strlen(map->eval_string);
3467
3468 for (ptr = call->print_fmt; *ptr; ptr++) {
3469 if (*ptr == '\\') {
3470 ptr++;
3471 /* paranoid */
3472 if (!*ptr)
3473 break;
3474 continue;
3475 }
3476 if (*ptr == '"') {
3477 quote ^= 1;
3478 continue;
3479 }
3480 if (quote)
3481 continue;
3482 if (isdigit(*ptr)) {
3483 /* skip numbers */
3484 do {
3485 ptr++;
3486 /* Check for alpha chars like ULL */
3487 } while (isalnum(*ptr));
3488 if (!*ptr)
3489 break;
3490 /*
3491 * A number must have some kind of delimiter after
3492 * it, and we can ignore that too.
3493 */
3494 continue;
3495 }
3496 if (isalpha(*ptr) || *ptr == '_') {
3497 if (strncmp(map->eval_string, ptr, len) == 0 &&
3498 !isalnum(ptr[len]) && ptr[len] != '_') {
3499 ptr = eval_replace(ptr, map, len);
3500 /* enum/sizeof string smaller than value */
3501 if (WARN_ON_ONCE(!ptr))
3502 return;
3503 /*
3504 * No need to decrement here, as eval_replace()
3505 * returns the pointer to the character passed
3506 * the eval, and two evals can not be placed
3507 * back to back without something in between.
3508 * We can skip that something in between.
3509 */
3510 continue;
3511 }
3512 skip_more:
3513 do {
3514 ptr++;
3515 } while (isalnum(*ptr) || *ptr == '_');
3516 if (!*ptr)
3517 break;
3518 /*
3519 * If what comes after this variable is a '.' or
3520 * '->' then we can continue to ignore that string.
3521 */
3522 if (*ptr == '.' || (ptr[0] == '-' && ptr[1] == '>')) {
3523 ptr += *ptr == '.' ? 1 : 2;
3524 if (!*ptr)
3525 break;
3526 goto skip_more;
3527 }
3528 /*
3529 * Once again, we can skip the delimiter that came
3530 * after the string.
3531 */
3532 continue;
3533 }
3534 }
3535}
3536
3537static void add_str_to_module(struct module *module, char *str)
3538{
3539 struct module_string *modstr;
3540
3541 modstr = kmalloc_obj(*modstr);
3542
3543 /*
3544 * If we failed to allocate memory here, then we'll just
3545 * let the str memory leak when the module is removed.
3546 * If this fails to allocate, there's worse problems than
3547 * a leaked string on module removal.
3548 */
3549 if (WARN_ON_ONCE(!modstr))
3550 return;
3551
3552 modstr->module = module;
3553 modstr->str = str;
3554
3555 list_add(&modstr->next, &module_strings);
3556}
3557
3558#define ATTRIBUTE_STR "__attribute__("
3559#define ATTRIBUTE_STR_LEN (sizeof(ATTRIBUTE_STR) - 1)
3560
3561/* Remove all __attribute__() from @type. Return allocated string or @type. */
3562static char *sanitize_field_type(const char *type)
3563{
3564 char *attr, *tmp, *next, *ret = (char *)type;
3565 int depth;
3566
3567 next = (char *)type;
3568 while ((attr = strstr(next, ATTRIBUTE_STR))) {
3569 /* Retry if "__attribute__(" is a part of another word. */
3570 if (attr != next && !isspace(attr[-1])) {
3571 next = attr + ATTRIBUTE_STR_LEN;
3572 continue;
3573 }
3574
3575 if (ret == type) {
3576 ret = kstrdup(type, GFP_KERNEL);
3577 if (WARN_ON_ONCE(!ret))
3578 return NULL;
3579 attr = ret + (attr - type);
3580 }
3581
3582 /* the ATTRIBUTE_STR already has the first '(' */
3583 depth = 1;
3584 next = attr + ATTRIBUTE_STR_LEN;
3585 do {
3586 tmp = strpbrk(next, "()");
3587 /* There is unbalanced parentheses */
3588 if (WARN_ON_ONCE(!tmp)) {
3589 kfree(ret);
3590 return (char *)type;
3591 }
3592
3593 if (*tmp == '(')
3594 depth++;
3595 else
3596 depth--;
3597 next = tmp + 1;
3598 } while (depth > 0);
3599 next = skip_spaces(next);
3600 strcpy(attr, next);
3601 next = attr;
3602 }
3603 return ret;
3604}
3605
3606static char *find_replacable_eval(const char *type, const char *eval_string,
3607 int len)
3608{
3609 char *ptr;
3610
3611 if (!eval_string)
3612 return NULL;
3613
3614 ptr = strchr(type, '[');
3615 if (!ptr)
3616 return NULL;
3617 ptr++;
3618
3619 if (!isalpha(*ptr) && *ptr != '_')
3620 return NULL;
3621
3622 if (strncmp(eval_string, ptr, len) != 0)
3623 return NULL;
3624
3625 return ptr;
3626}
3627
3628static void update_event_fields(struct trace_event_call *call,
3629 struct trace_eval_map *map)
3630{
3631 struct ftrace_event_field *field;
3632 const char *eval_string = NULL;
3633 struct list_head *head;
3634 int len = 0;
3635 char *ptr;
3636 char *str;
3637
3638 /* Dynamic events should never have field maps */
3639 if (call->flags & TRACE_EVENT_FL_DYNAMIC)
3640 return;
3641
3642 if (map) {
3643 eval_string = map->eval_string;
3644 len = strlen(map->eval_string);
3645 }
3646
3647 head = trace_get_fields(call);
3648 list_for_each_entry(field, head, link) {
3649 str = sanitize_field_type(field->type);
3650 if (!str)
3651 return;
3652
3653 ptr = find_replacable_eval(str, eval_string, len);
3654 if (ptr) {
3655 if (str == field->type) {
3656 str = kstrdup(field->type, GFP_KERNEL);
3657 if (WARN_ON_ONCE(!str))
3658 return;
3659 ptr = str + (ptr - field->type);
3660 }
3661
3662 ptr = eval_replace(ptr, map, len);
3663 /* enum/sizeof string smaller than value */
3664 if (WARN_ON_ONCE(!ptr)) {
3665 kfree(str);
3666 continue;
3667 }
3668 }
3669
3670 if (str == field->type)
3671 continue;
3672 /*
3673 * If the event is part of a module, then we need to free the string
3674 * when the module is removed. Otherwise, it will stay allocated
3675 * until a reboot.
3676 */
3677 if (call->module)
3678 add_str_to_module(call->module, str);
3679
3680 field->type = str;
3681 if (field->filter_type == FILTER_OTHER)
3682 field->filter_type = filter_assign_type(field->type);
3683 }
3684}
3685
3686/* Update all events for replacing eval and sanitizing */
3687void trace_event_update_all(struct trace_eval_map **map, int len, struct module *mod)
3688{
3689 struct trace_event_call *call, *p;
3690 const char *last_system = NULL;
3691 bool first = false;
3692 bool updated;
3693 int last_i;
3694 int i;
3695
3696 mutex_lock(&event_mutex);
3697 down_write(&trace_event_sem);
3698 list_for_each_entry_safe(call, p, &ftrace_events, list) {
3699
3700 if (mod && call->module != mod)
3701 continue;
3702
3703 /* events are usually grouped together with systems */
3704 if (!last_system || call->class->system != last_system) {
3705 first = true;
3706 last_i = 0;
3707 last_system = call->class->system;
3708 }
3709
3710 updated = false;
3711 /*
3712 * Since calls are grouped by systems, the likelihood that the
3713 * next call in the iteration belongs to the same system as the
3714 * previous call is high. As an optimization, we skip searching
3715 * for a map[] that matches the call's system if the last call
3716 * was from the same system. That's what last_i is for. If the
3717 * call has the same system as the previous call, then last_i
3718 * will be the index of the first map[] that has a matching
3719 * system.
3720 */
3721 for (i = last_i; i < len; i++) {
3722 if (call->class->system == map[i]->system) {
3723 /* Save the first system if need be */
3724 if (first) {
3725 last_i = i;
3726 first = false;
3727 }
3728 update_event_printk(call, map[i]);
3729 update_event_fields(call, map[i]);
3730 updated = true;
3731 }
3732 }
3733 /* If not updated yet, update field for sanitizing. */
3734 if (!updated)
3735 update_event_fields(call, NULL);
3736 cond_resched();
3737 }
3738 up_write(&trace_event_sem);
3739 mutex_unlock(&event_mutex);
3740}
3741
3742static bool event_in_systems(struct trace_event_call *call,
3743 const char *systems)
3744{
3745 const char *system;
3746 const char *p;
3747
3748 if (!systems)
3749 return true;
3750
3751 system = call->class->system;
3752 p = strstr(systems, system);
3753 if (!p)
3754 return false;
3755
3756 if (p != systems && !isspace(*(p - 1)) && *(p - 1) != ',')
3757 return false;
3758
3759 p += strlen(system);
3760 return !*p || isspace(*p) || *p == ',';
3761}
3762
3763#ifdef CONFIG_HIST_TRIGGERS
3764/*
3765 * Wake up waiter on the hist_poll_wq from irq_work because the hist trigger
3766 * may happen in any context.
3767 */
3768static void hist_poll_event_irq_work(struct irq_work *work)
3769{
3770 wake_up_all(&hist_poll_wq);
3771}
3772
3773DEFINE_IRQ_WORK(hist_poll_work, hist_poll_event_irq_work);
3774DECLARE_WAIT_QUEUE_HEAD(hist_poll_wq);
3775#endif
3776
3777static struct trace_event_file *
3778trace_create_new_event(struct trace_event_call *call,
3779 struct trace_array *tr)
3780{
3781 struct trace_pid_list *no_pid_list;
3782 struct trace_pid_list *pid_list;
3783 struct trace_event_file *file;
3784 unsigned int first;
3785
3786 if (!event_in_systems(call, tr->system_names))
3787 return NULL;
3788
3789 file = kmem_cache_alloc(file_cachep, GFP_TRACE);
3790 if (!file)
3791 return ERR_PTR(-ENOMEM);
3792
3793 pid_list = rcu_dereference_protected(tr->filtered_pids,
3794 lockdep_is_held(&event_mutex));
3795 no_pid_list = rcu_dereference_protected(tr->filtered_no_pids,
3796 lockdep_is_held(&event_mutex));
3797
3798 if (!trace_pid_list_first(pid_list, &first) ||
3799 !trace_pid_list_first(no_pid_list, &first))
3800 file->flags |= EVENT_FILE_FL_PID_FILTER;
3801
3802 file->event_call = call;
3803 file->tr = tr;
3804 atomic_set(&file->sm_ref, 0);
3805 atomic_set(&file->tm_ref, 0);
3806 INIT_LIST_HEAD(&file->triggers);
3807 list_add(&file->list, &tr->events);
3808 refcount_set(&file->ref, 1);
3809
3810 return file;
3811}
3812
3813#define MAX_BOOT_TRIGGERS 32
3814
3815static struct boot_triggers {
3816 const char *event;
3817 char *trigger;
3818} bootup_triggers[MAX_BOOT_TRIGGERS];
3819
3820static char bootup_trigger_buf[COMMAND_LINE_SIZE];
3821static int boot_trigger_buf_len;
3822static int nr_boot_triggers;
3823
3824static __init int setup_trace_triggers(char *str)
3825{
3826 char *trigger;
3827 char *buf;
3828 int len = boot_trigger_buf_len;
3829 int i;
3830
3831 if (len >= COMMAND_LINE_SIZE)
3832 return 1;
3833
3834 strscpy(bootup_trigger_buf + len, str, COMMAND_LINE_SIZE - len);
3835 trace_set_ring_buffer_expanded(NULL);
3836 disable_tracing_selftest("running event triggers");
3837
3838 buf = bootup_trigger_buf + len;
3839 boot_trigger_buf_len += strlen(buf) + 1;
3840
3841 for (i = nr_boot_triggers; i < MAX_BOOT_TRIGGERS; i++) {
3842 trigger = strsep(&buf, ",");
3843 if (!trigger)
3844 break;
3845 bootup_triggers[i].event = strsep(&trigger, ".");
3846 bootup_triggers[i].trigger = trigger;
3847 if (!bootup_triggers[i].trigger)
3848 break;
3849 }
3850
3851 nr_boot_triggers = i;
3852 return 1;
3853}
3854__setup("trace_trigger=", setup_trace_triggers);
3855
3856/* Add an event to a trace directory */
3857static int
3858__trace_add_new_event(struct trace_event_call *call, struct trace_array *tr)
3859{
3860 struct trace_event_file *file;
3861
3862 file = trace_create_new_event(call, tr);
3863 /*
3864 * trace_create_new_event() returns ERR_PTR(-ENOMEM) if failed
3865 * allocation, or NULL if the event is not part of the tr->system_names.
3866 * When the event is not part of the tr->system_names, return zero, not
3867 * an error.
3868 */
3869 if (!file)
3870 return 0;
3871
3872 if (IS_ERR(file))
3873 return PTR_ERR(file);
3874
3875 if (eventdir_initialized)
3876 return event_create_dir(tr->event_dir, file);
3877 else
3878 return event_define_fields(call);
3879}
3880
3881static void trace_early_triggers(struct trace_event_file *file, const char *name)
3882{
3883 int ret;
3884 int i;
3885
3886 for (i = 0; i < nr_boot_triggers; i++) {
3887 if (strcmp(name, bootup_triggers[i].event))
3888 continue;
3889 mutex_lock(&event_mutex);
3890 ret = trigger_process_regex(file, bootup_triggers[i].trigger);
3891 mutex_unlock(&event_mutex);
3892 if (ret)
3893 pr_err("Failed to register trigger '%s' on event %s\n",
3894 bootup_triggers[i].trigger,
3895 bootup_triggers[i].event);
3896 }
3897}
3898
3899/*
3900 * Just create a descriptor for early init. A descriptor is required
3901 * for enabling events at boot. We want to enable events before
3902 * the filesystem is initialized.
3903 */
3904static int
3905__trace_early_add_new_event(struct trace_event_call *call,
3906 struct trace_array *tr)
3907{
3908 struct trace_event_file *file;
3909 int ret;
3910
3911 file = trace_create_new_event(call, tr);
3912 /*
3913 * trace_create_new_event() returns ERR_PTR(-ENOMEM) if failed
3914 * allocation, or NULL if the event is not part of the tr->system_names.
3915 * When the event is not part of the tr->system_names, return zero, not
3916 * an error.
3917 */
3918 if (!file)
3919 return 0;
3920
3921 if (IS_ERR(file))
3922 return PTR_ERR(file);
3923
3924 ret = event_define_fields(call);
3925 if (ret)
3926 return ret;
3927
3928 trace_early_triggers(file, trace_event_name(call));
3929
3930 return 0;
3931}
3932
3933struct ftrace_module_file_ops;
3934static void __add_event_to_tracers(struct trace_event_call *call);
3935
3936/* Add an additional event_call dynamically */
3937int trace_add_event_call(struct trace_event_call *call)
3938{
3939 int ret;
3940 lockdep_assert_held(&event_mutex);
3941
3942 guard(mutex)(&trace_types_lock);
3943
3944 ret = __register_event(call, NULL);
3945 if (ret < 0)
3946 return ret;
3947
3948 __add_event_to_tracers(call);
3949 return ret;
3950}
3951EXPORT_SYMBOL_GPL(trace_add_event_call);
3952
3953/*
3954 * Must be called under locking of trace_types_lock, event_mutex and
3955 * trace_event_sem.
3956 */
3957static void __trace_remove_event_call(struct trace_event_call *call)
3958{
3959 event_remove(call);
3960 trace_destroy_fields(call);
3961}
3962
3963static int probe_remove_event_call(struct trace_event_call *call)
3964{
3965 struct trace_array *tr;
3966 struct trace_event_file *file;
3967
3968#ifdef CONFIG_PERF_EVENTS
3969 if (call->perf_refcount)
3970 return -EBUSY;
3971#endif
3972 do_for_each_event_file(tr, file) {
3973 if (file->event_call != call)
3974 continue;
3975 /*
3976 * We can't rely on ftrace_event_enable_disable(enable => 0)
3977 * we are going to do, soft mode can suppress
3978 * TRACE_REG_UNREGISTER.
3979 */
3980 if (file->flags & EVENT_FILE_FL_ENABLED)
3981 goto busy;
3982
3983 if (file->flags & EVENT_FILE_FL_WAS_ENABLED)
3984 tr->clear_trace = true;
3985 /*
3986 * The do_for_each_event_file_safe() is
3987 * a double loop. After finding the call for this
3988 * trace_array, we use break to jump to the next
3989 * trace_array.
3990 */
3991 break;
3992 } while_for_each_event_file();
3993
3994 __trace_remove_event_call(call);
3995
3996 return 0;
3997 busy:
3998 /* No need to clear the trace now */
3999 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
4000 tr->clear_trace = false;
4001 }
4002 return -EBUSY;
4003}
4004
4005/* Remove an event_call */
4006int trace_remove_event_call(struct trace_event_call *call)
4007{
4008 int ret;
4009
4010 lockdep_assert_held(&event_mutex);
4011
4012 mutex_lock(&trace_types_lock);
4013 down_write(&trace_event_sem);
4014 ret = probe_remove_event_call(call);
4015 up_write(&trace_event_sem);
4016 mutex_unlock(&trace_types_lock);
4017
4018 return ret;
4019}
4020EXPORT_SYMBOL_GPL(trace_remove_event_call);
4021
4022#define for_each_event(event, start, end) \
4023 for (event = start; \
4024 (unsigned long)event < (unsigned long)end; \
4025 event++)
4026
4027#ifdef CONFIG_MODULES
4028static void update_mod_cache(struct trace_array *tr, struct module *mod)
4029{
4030 struct event_mod_load *event_mod, *n;
4031
4032 list_for_each_entry_safe(event_mod, n, &tr->mod_events, list) {
4033 if (strcmp(event_mod->module, mod->name) != 0)
4034 continue;
4035
4036 __ftrace_set_clr_event_nolock(tr, event_mod->match,
4037 event_mod->system,
4038 event_mod->event, 1, mod->name);
4039 free_event_mod(event_mod);
4040 }
4041}
4042
4043static void update_cache_events(struct module *mod)
4044{
4045 struct trace_array *tr;
4046
4047 list_for_each_entry(tr, &ftrace_trace_arrays, list)
4048 update_mod_cache(tr, mod);
4049}
4050
4051static void trace_module_add_events(struct module *mod)
4052{
4053 struct trace_event_call **call, **start, **end;
4054
4055 if (!mod->num_trace_events)
4056 return;
4057
4058 /* Don't add infrastructure for mods without tracepoints */
4059 if (trace_module_has_bad_taint(mod)) {
4060 pr_err("%s: module has bad taint, not creating trace events\n",
4061 mod->name);
4062 return;
4063 }
4064
4065 start = mod->trace_events;
4066 end = mod->trace_events + mod->num_trace_events;
4067
4068 for_each_event(call, start, end) {
4069 if (!__register_event(*call, mod))
4070 __add_event_to_tracers(*call);
4071 }
4072
4073 update_cache_events(mod);
4074}
4075
4076static void trace_module_remove_events(struct module *mod)
4077{
4078 struct trace_event_call *call, *p;
4079 struct module_string *modstr, *m;
4080
4081 down_write(&trace_event_sem);
4082 list_for_each_entry_safe(call, p, &ftrace_events, list) {
4083 if ((call->flags & TRACE_EVENT_FL_DYNAMIC) || !call->module)
4084 continue;
4085 if (call->module == mod)
4086 __trace_remove_event_call(call);
4087 }
4088 /* Check for any strings allocated for this module */
4089 list_for_each_entry_safe(modstr, m, &module_strings, next) {
4090 if (modstr->module != mod)
4091 continue;
4092 list_del(&modstr->next);
4093 kfree(modstr->str);
4094 kfree(modstr);
4095 }
4096 up_write(&trace_event_sem);
4097
4098 /*
4099 * It is safest to reset the ring buffer if the module being unloaded
4100 * registered any events that were used. The only worry is if
4101 * a new module gets loaded, and takes on the same id as the events
4102 * of this module. When printing out the buffer, traced events left
4103 * over from this module may be passed to the new module events and
4104 * unexpected results may occur.
4105 */
4106 tracing_reset_all_online_cpus_unlocked();
4107}
4108
4109static int trace_module_notify(struct notifier_block *self,
4110 unsigned long val, void *data)
4111{
4112 struct module *mod = data;
4113
4114 mutex_lock(&event_mutex);
4115 mutex_lock(&trace_types_lock);
4116 switch (val) {
4117 case MODULE_STATE_COMING:
4118 trace_module_add_events(mod);
4119 break;
4120 case MODULE_STATE_GOING:
4121 trace_module_remove_events(mod);
4122 break;
4123 }
4124 mutex_unlock(&trace_types_lock);
4125 mutex_unlock(&event_mutex);
4126
4127 return NOTIFY_OK;
4128}
4129
4130static struct notifier_block trace_module_nb = {
4131 .notifier_call = trace_module_notify,
4132 .priority = 1, /* higher than trace.c module notify */
4133};
4134#endif /* CONFIG_MODULES */
4135
4136/* Create a new event directory structure for a trace directory. */
4137static void
4138__trace_add_event_dirs(struct trace_array *tr)
4139{
4140 struct trace_event_call *call;
4141 int ret;
4142
4143 lockdep_assert_held(&trace_event_sem);
4144
4145 list_for_each_entry(call, &ftrace_events, list) {
4146 ret = __trace_add_new_event(call, tr);
4147 if (ret < 0)
4148 pr_warn("Could not create directory for event %s\n",
4149 trace_event_name(call));
4150 }
4151}
4152
4153/* Returns any file that matches the system and event */
4154struct trace_event_file *
4155__find_event_file(struct trace_array *tr, const char *system, const char *event)
4156{
4157 struct trace_event_file *file;
4158 struct trace_event_call *call;
4159 const char *name;
4160
4161 list_for_each_entry(file, &tr->events, list) {
4162
4163 call = file->event_call;
4164 name = trace_event_name(call);
4165
4166 if (!name || !call->class)
4167 continue;
4168
4169 if (strcmp(event, name) == 0 &&
4170 strcmp(system, call->class->system) == 0)
4171 return file;
4172 }
4173 return NULL;
4174}
4175
4176/* Returns valid trace event files that match system and event */
4177struct trace_event_file *
4178find_event_file(struct trace_array *tr, const char *system, const char *event)
4179{
4180 struct trace_event_file *file;
4181
4182 file = __find_event_file(tr, system, event);
4183 if (!file || !file->event_call->class->reg ||
4184 file->event_call->flags & TRACE_EVENT_FL_IGNORE_ENABLE)
4185 return NULL;
4186
4187 return file;
4188}
4189
4190/**
4191 * trace_get_event_file - Find and return a trace event file
4192 * @instance: The name of the trace instance containing the event
4193 * @system: The name of the system containing the event
4194 * @event: The name of the event
4195 *
4196 * Return a trace event file given the trace instance name, trace
4197 * system, and trace event name. If the instance name is NULL, it
4198 * refers to the top-level trace array.
4199 *
4200 * This function will look it up and return it if found, after calling
4201 * trace_array_get() to prevent the instance from going away, and
4202 * increment the event's module refcount to prevent it from being
4203 * removed.
4204 *
4205 * To release the file, call trace_put_event_file(), which will call
4206 * trace_array_put() and decrement the event's module refcount.
4207 *
4208 * Return: The trace event on success, ERR_PTR otherwise.
4209 */
4210struct trace_event_file *trace_get_event_file(const char *instance,
4211 const char *system,
4212 const char *event)
4213{
4214 struct trace_array *tr = top_trace_array();
4215 struct trace_event_file *file = NULL;
4216 int ret = -EINVAL;
4217
4218 if (instance) {
4219 tr = trace_array_find_get(instance);
4220 if (!tr)
4221 return ERR_PTR(-ENOENT);
4222 } else {
4223 ret = trace_array_get(tr);
4224 if (ret)
4225 return ERR_PTR(ret);
4226 }
4227
4228 guard(mutex)(&event_mutex);
4229
4230 file = find_event_file(tr, system, event);
4231 if (!file) {
4232 trace_array_put(tr);
4233 return ERR_PTR(-EINVAL);
4234 }
4235
4236 /* Don't let event modules unload while in use */
4237 ret = trace_event_try_get_ref(file->event_call);
4238 if (!ret) {
4239 trace_array_put(tr);
4240 return ERR_PTR(-EBUSY);
4241 }
4242
4243 return file;
4244}
4245EXPORT_SYMBOL_GPL(trace_get_event_file);
4246
4247/**
4248 * trace_put_event_file - Release a file from trace_get_event_file()
4249 * @file: The trace event file
4250 *
4251 * If a file was retrieved using trace_get_event_file(), this should
4252 * be called when it's no longer needed. It will cancel the previous
4253 * trace_array_get() called by that function, and decrement the
4254 * event's module refcount.
4255 */
4256void trace_put_event_file(struct trace_event_file *file)
4257{
4258 mutex_lock(&event_mutex);
4259 trace_event_put_ref(file->event_call);
4260 mutex_unlock(&event_mutex);
4261
4262 trace_array_put(file->tr);
4263}
4264EXPORT_SYMBOL_GPL(trace_put_event_file);
4265
4266#ifdef CONFIG_DYNAMIC_FTRACE
4267struct event_probe_data {
4268 struct trace_event_file *file;
4269 unsigned long count;
4270 int ref;
4271 bool enable;
4272};
4273
4274static void update_event_probe(struct event_probe_data *data)
4275{
4276 if (data->enable)
4277 clear_bit(EVENT_FILE_FL_SOFT_DISABLED_BIT, &data->file->flags);
4278 else
4279 set_bit(EVENT_FILE_FL_SOFT_DISABLED_BIT, &data->file->flags);
4280}
4281
4282static void
4283event_enable_probe(unsigned long ip, unsigned long parent_ip,
4284 struct trace_array *tr, struct ftrace_probe_ops *ops,
4285 void *data)
4286{
4287 struct ftrace_func_mapper *mapper = data;
4288 struct event_probe_data *edata;
4289 void **pdata;
4290
4291 pdata = ftrace_func_mapper_find_ip(mapper, ip);
4292 if (!pdata || !*pdata)
4293 return;
4294
4295 edata = *pdata;
4296 update_event_probe(edata);
4297}
4298
4299static void
4300event_enable_count_probe(unsigned long ip, unsigned long parent_ip,
4301 struct trace_array *tr, struct ftrace_probe_ops *ops,
4302 void *data)
4303{
4304 struct ftrace_func_mapper *mapper = data;
4305 struct event_probe_data *edata;
4306 void **pdata;
4307
4308 pdata = ftrace_func_mapper_find_ip(mapper, ip);
4309 if (!pdata || !*pdata)
4310 return;
4311
4312 edata = *pdata;
4313
4314 if (!edata->count)
4315 return;
4316
4317 /* Skip if the event is in a state we want to switch to */
4318 if (edata->enable == !(edata->file->flags & EVENT_FILE_FL_SOFT_DISABLED))
4319 return;
4320
4321 if (edata->count != -1)
4322 (edata->count)--;
4323
4324 update_event_probe(edata);
4325}
4326
4327static int
4328event_enable_print(struct seq_file *m, unsigned long ip,
4329 struct ftrace_probe_ops *ops, void *data)
4330{
4331 struct ftrace_func_mapper *mapper = data;
4332 struct event_probe_data *edata;
4333 void **pdata;
4334
4335 pdata = ftrace_func_mapper_find_ip(mapper, ip);
4336
4337 if (WARN_ON_ONCE(!pdata || !*pdata))
4338 return 0;
4339
4340 edata = *pdata;
4341
4342 seq_printf(m, "%ps:", (void *)ip);
4343
4344 seq_printf(m, "%s:%s:%s",
4345 edata->enable ? ENABLE_EVENT_STR : DISABLE_EVENT_STR,
4346 edata->file->event_call->class->system,
4347 trace_event_name(edata->file->event_call));
4348
4349 if (edata->count == -1)
4350 seq_puts(m, ":unlimited\n");
4351 else
4352 seq_printf(m, ":count=%ld\n", edata->count);
4353
4354 return 0;
4355}
4356
4357static int
4358event_enable_init(struct ftrace_probe_ops *ops, struct trace_array *tr,
4359 unsigned long ip, void *init_data, void **data)
4360{
4361 struct ftrace_func_mapper *mapper = *data;
4362 struct event_probe_data *edata = init_data;
4363 int ret;
4364
4365 if (!mapper) {
4366 mapper = allocate_ftrace_func_mapper();
4367 if (!mapper)
4368 return -ENODEV;
4369 *data = mapper;
4370 }
4371
4372 ret = ftrace_func_mapper_add_ip(mapper, ip, edata);
4373 if (ret < 0)
4374 return ret;
4375
4376 edata->ref++;
4377
4378 return 0;
4379}
4380
4381static int free_probe_data(void *data)
4382{
4383 struct event_probe_data *edata = data;
4384
4385 edata->ref--;
4386 if (!edata->ref) {
4387 /* Remove soft mode */
4388 __ftrace_event_enable_disable(edata->file, 0, 1);
4389 trace_event_put_ref(edata->file->event_call);
4390 kfree(edata);
4391 }
4392 return 0;
4393}
4394
4395static void
4396event_enable_free(struct ftrace_probe_ops *ops, struct trace_array *tr,
4397 unsigned long ip, void *data)
4398{
4399 struct ftrace_func_mapper *mapper = data;
4400 struct event_probe_data *edata;
4401
4402 if (!ip) {
4403 if (!mapper)
4404 return;
4405 free_ftrace_func_mapper(mapper, free_probe_data);
4406 return;
4407 }
4408
4409 edata = ftrace_func_mapper_remove_ip(mapper, ip);
4410
4411 if (WARN_ON_ONCE(!edata))
4412 return;
4413
4414 if (WARN_ON_ONCE(edata->ref <= 0))
4415 return;
4416
4417 free_probe_data(edata);
4418}
4419
4420static struct ftrace_probe_ops event_enable_probe_ops = {
4421 .func = event_enable_probe,
4422 .print = event_enable_print,
4423 .init = event_enable_init,
4424 .free = event_enable_free,
4425};
4426
4427static struct ftrace_probe_ops event_enable_count_probe_ops = {
4428 .func = event_enable_count_probe,
4429 .print = event_enable_print,
4430 .init = event_enable_init,
4431 .free = event_enable_free,
4432};
4433
4434static struct ftrace_probe_ops event_disable_probe_ops = {
4435 .func = event_enable_probe,
4436 .print = event_enable_print,
4437 .init = event_enable_init,
4438 .free = event_enable_free,
4439};
4440
4441static struct ftrace_probe_ops event_disable_count_probe_ops = {
4442 .func = event_enable_count_probe,
4443 .print = event_enable_print,
4444 .init = event_enable_init,
4445 .free = event_enable_free,
4446};
4447
4448static int
4449event_enable_func(struct trace_array *tr, struct ftrace_hash *hash,
4450 char *glob, char *cmd, char *param, int enabled)
4451{
4452 struct trace_event_file *file;
4453 struct ftrace_probe_ops *ops;
4454 struct event_probe_data *data;
4455 unsigned long count = -1;
4456 const char *system;
4457 const char *event;
4458 char *number;
4459 bool enable;
4460 int ret;
4461
4462 if (!tr)
4463 return -ENODEV;
4464
4465 /* hash funcs only work with set_ftrace_filter */
4466 if (!enabled || !param)
4467 return -EINVAL;
4468
4469 system = strsep(&param, ":");
4470 if (!param)
4471 return -EINVAL;
4472
4473 event = strsep(&param, ":");
4474
4475 guard(mutex)(&event_mutex);
4476
4477 file = find_event_file(tr, system, event);
4478 if (!file)
4479 return -EINVAL;
4480
4481 enable = strcmp(cmd, ENABLE_EVENT_STR) == 0;
4482
4483 if (enable)
4484 ops = param ? &event_enable_count_probe_ops : &event_enable_probe_ops;
4485 else
4486 ops = param ? &event_disable_count_probe_ops : &event_disable_probe_ops;
4487
4488 if (glob[0] == '!')
4489 return unregister_ftrace_function_probe_func(glob+1, tr, ops);
4490
4491 if (param) {
4492 number = strsep(&param, ":");
4493
4494 if (!strlen(number))
4495 return -EINVAL;
4496
4497 /*
4498 * We use the callback data field (which is a pointer)
4499 * as our counter.
4500 */
4501 ret = kstrtoul(number, 0, &count);
4502 if (ret)
4503 return ret;
4504 }
4505
4506 /* Don't let event modules unload while probe registered */
4507 ret = trace_event_try_get_ref(file->event_call);
4508 if (!ret)
4509 return -EBUSY;
4510
4511 ret = __ftrace_event_enable_disable(file, 1, 1);
4512 if (ret < 0)
4513 goto out_put;
4514
4515 ret = -ENOMEM;
4516 data = kzalloc_obj(*data);
4517 if (!data)
4518 goto out_put;
4519
4520 data->enable = enable;
4521 data->count = count;
4522 data->file = file;
4523
4524 ret = register_ftrace_function_probe(glob, tr, ops, data);
4525 /*
4526 * The above returns on success the # of functions enabled,
4527 * but if it didn't find any functions it returns zero.
4528 * Consider no functions a failure too.
4529 */
4530
4531 /* Just return zero, not the number of enabled functions */
4532 if (ret > 0)
4533 return 0;
4534
4535 kfree(data);
4536
4537 if (!ret)
4538 ret = -ENOENT;
4539
4540 __ftrace_event_enable_disable(file, 0, 1);
4541 out_put:
4542 trace_event_put_ref(file->event_call);
4543 return ret;
4544}
4545
4546static struct ftrace_func_command event_enable_cmd = {
4547 .name = ENABLE_EVENT_STR,
4548 .func = event_enable_func,
4549};
4550
4551static struct ftrace_func_command event_disable_cmd = {
4552 .name = DISABLE_EVENT_STR,
4553 .func = event_enable_func,
4554};
4555
4556static __init int register_event_cmds(void)
4557{
4558 int ret;
4559
4560 ret = register_ftrace_command(&event_enable_cmd);
4561 if (WARN_ON(ret < 0))
4562 return ret;
4563 ret = register_ftrace_command(&event_disable_cmd);
4564 if (WARN_ON(ret < 0))
4565 unregister_ftrace_command(&event_enable_cmd);
4566 return ret;
4567}
4568#else
4569static inline int register_event_cmds(void) { return 0; }
4570#endif /* CONFIG_DYNAMIC_FTRACE */
4571
4572/*
4573 * The top level array and trace arrays created by boot-time tracing
4574 * have already had its trace_event_file descriptors created in order
4575 * to allow for early events to be recorded.
4576 * This function is called after the tracefs has been initialized,
4577 * and we now have to create the files associated to the events.
4578 */
4579static void __trace_early_add_event_dirs(struct trace_array *tr)
4580{
4581 struct trace_event_file *file;
4582 int ret;
4583
4584
4585 list_for_each_entry(file, &tr->events, list) {
4586 ret = event_create_dir(tr->event_dir, file);
4587 if (ret < 0)
4588 pr_warn("Could not create directory for event %s\n",
4589 trace_event_name(file->event_call));
4590 }
4591}
4592
4593/*
4594 * For early boot up, the top trace array and the trace arrays created
4595 * by boot-time tracing require to have a list of events that can be
4596 * enabled. This must be done before the filesystem is set up in order
4597 * to allow events to be traced early.
4598 */
4599void __trace_early_add_events(struct trace_array *tr)
4600{
4601 struct trace_event_call *call;
4602 int ret;
4603
4604 list_for_each_entry(call, &ftrace_events, list) {
4605 /* Early boot up should not have any modules loaded */
4606 if (!(call->flags & TRACE_EVENT_FL_DYNAMIC) &&
4607 WARN_ON_ONCE(call->module))
4608 continue;
4609
4610 ret = __trace_early_add_new_event(call, tr);
4611 if (ret < 0)
4612 pr_warn("Could not create early event %s\n",
4613 trace_event_name(call));
4614 }
4615}
4616
4617/* Remove the event directory structure for a trace directory. */
4618static void
4619__trace_remove_event_dirs(struct trace_array *tr)
4620{
4621 struct trace_event_file *file, *next;
4622
4623 list_for_each_entry_safe(file, next, &tr->events, list)
4624 remove_event_file_dir(file);
4625}
4626
4627static void __add_event_to_tracers(struct trace_event_call *call)
4628{
4629 struct trace_array *tr;
4630
4631 list_for_each_entry(tr, &ftrace_trace_arrays, list)
4632 __trace_add_new_event(call, tr);
4633}
4634
4635extern struct trace_event_call *__start_ftrace_events[];
4636extern struct trace_event_call *__stop_ftrace_events[];
4637
4638static char bootup_event_buf[COMMAND_LINE_SIZE] __initdata;
4639static struct seq_buf bootup_event_seq __initdata = {
4640 .buffer = bootup_event_buf,
4641 .size = sizeof(bootup_event_buf),
4642};
4643
4644static __init int setup_trace_event(char *str)
4645{
4646 if (seq_buf_used(&bootup_event_seq) > 0)
4647 seq_buf_puts(&bootup_event_seq, ",");
4648
4649 seq_buf_puts(&bootup_event_seq, str);
4650
4651 if (seq_buf_has_overflowed(&bootup_event_seq))
4652 return -ENOMEM;
4653
4654 trace_set_ring_buffer_expanded(NULL);
4655 disable_tracing_selftest("running event tracing");
4656
4657 return 1;
4658}
4659__setup("trace_event=", setup_trace_event);
4660
4661static int events_callback(const char *name, umode_t *mode, void **data,
4662 const struct file_operations **fops)
4663{
4664 if (strcmp(name, "enable") == 0) {
4665 *mode = TRACE_MODE_WRITE;
4666 *fops = &ftrace_tr_enable_fops;
4667 return 1;
4668 }
4669
4670 if (strcmp(name, "header_page") == 0) {
4671 *mode = TRACE_MODE_READ;
4672 *fops = &ftrace_show_header_page_fops;
4673
4674 } else if (strcmp(name, "header_event") == 0) {
4675 *mode = TRACE_MODE_READ;
4676 *fops = &ftrace_show_header_event_fops;
4677 } else
4678 return 0;
4679
4680 return 1;
4681}
4682
4683/* Expects to have event_mutex held when called */
4684static int
4685create_event_toplevel_files(struct dentry *parent, struct trace_array *tr)
4686{
4687 struct eventfs_inode *e_events;
4688 struct dentry *entry;
4689 int nr_entries;
4690 static struct eventfs_entry events_entries[] = {
4691 {
4692 .name = "header_page",
4693 .callback = events_callback,
4694 },
4695 {
4696 .name = "header_event",
4697 .callback = events_callback,
4698 },
4699#define NR_RO_TOP_ENTRIES 2
4700/* Readonly files must be above this line and counted by NR_RO_TOP_ENTRIES. */
4701 {
4702 .name = "enable",
4703 .callback = events_callback,
4704 },
4705 };
4706
4707 if (!trace_array_is_readonly(tr)) {
4708 entry = trace_create_file("set_event", TRACE_MODE_WRITE, parent,
4709 tr, &ftrace_set_event_fops);
4710 if (!entry)
4711 return -ENOMEM;
4712
4713 /* There are not as crucial, just warn if they are not created */
4714 trace_create_file("show_event_filters", TRACE_MODE_READ, parent, tr,
4715 &ftrace_show_event_filters_fops);
4716
4717 trace_create_file("show_event_triggers", TRACE_MODE_READ, parent, tr,
4718 &ftrace_show_event_triggers_fops);
4719
4720 trace_create_file("set_event_pid", TRACE_MODE_WRITE, parent,
4721 tr, &ftrace_set_event_pid_fops);
4722
4723 trace_create_file("set_event_notrace_pid",
4724 TRACE_MODE_WRITE, parent, tr,
4725 &ftrace_set_event_notrace_pid_fops);
4726 nr_entries = ARRAY_SIZE(events_entries);
4727 } else {
4728 nr_entries = NR_RO_TOP_ENTRIES;
4729 }
4730
4731 e_events = eventfs_create_events_dir("events", parent, events_entries,
4732 nr_entries, tr);
4733 if (IS_ERR(e_events)) {
4734 pr_warn("Could not create tracefs 'events' directory\n");
4735 return -ENOMEM;
4736 }
4737
4738 tr->event_dir = e_events;
4739
4740 return 0;
4741}
4742
4743/**
4744 * event_trace_add_tracer - add a instance of a trace_array to events
4745 * @parent: The parent dentry to place the files/directories for events in
4746 * @tr: The trace array associated with these events
4747 *
4748 * When a new instance is created, it needs to set up its events
4749 * directory, as well as other files associated with events. It also
4750 * creates the event hierarchy in the @parent/events directory.
4751 *
4752 * Returns 0 on success.
4753 *
4754 * Must be called with event_mutex held.
4755 */
4756int event_trace_add_tracer(struct dentry *parent, struct trace_array *tr)
4757{
4758 int ret;
4759
4760 lockdep_assert_held(&event_mutex);
4761
4762 ret = create_event_toplevel_files(parent, tr);
4763 if (ret)
4764 goto out;
4765
4766 down_write(&trace_event_sem);
4767 /* If tr already has the event list, it is initialized in early boot. */
4768 if (unlikely(!list_empty(&tr->events)))
4769 __trace_early_add_event_dirs(tr);
4770 else
4771 __trace_add_event_dirs(tr);
4772 up_write(&trace_event_sem);
4773
4774 out:
4775 return ret;
4776}
4777
4778/*
4779 * The top trace array already had its file descriptors created.
4780 * Now the files themselves need to be created.
4781 */
4782static __init int
4783early_event_add_tracer(struct dentry *parent, struct trace_array *tr)
4784{
4785 int ret;
4786
4787 guard(mutex)(&event_mutex);
4788
4789 ret = create_event_toplevel_files(parent, tr);
4790 if (ret)
4791 return ret;
4792
4793 down_write(&trace_event_sem);
4794 __trace_early_add_event_dirs(tr);
4795 up_write(&trace_event_sem);
4796
4797 return 0;
4798}
4799
4800/* Must be called with event_mutex held */
4801int event_trace_del_tracer(struct trace_array *tr)
4802{
4803 lockdep_assert_held(&event_mutex);
4804
4805 /* Disable any event triggers and associated soft-disabled events */
4806 clear_event_triggers(tr);
4807
4808 /* Clear the pid list */
4809 __ftrace_clear_event_pids(tr, TRACE_PIDS | TRACE_NO_PIDS);
4810
4811 /* Disable any running events */
4812 __ftrace_set_clr_event_nolock(tr, NULL, NULL, NULL, 0, NULL);
4813
4814 /* Make sure no more events are being executed */
4815 tracepoint_synchronize_unregister();
4816
4817 down_write(&trace_event_sem);
4818 __trace_remove_event_dirs(tr);
4819 eventfs_remove_events_dir(tr->event_dir);
4820 up_write(&trace_event_sem);
4821
4822 tr->event_dir = NULL;
4823
4824 return 0;
4825}
4826
4827static __init int event_trace_memsetup(void)
4828{
4829 field_cachep = KMEM_CACHE(ftrace_event_field, SLAB_PANIC);
4830 file_cachep = KMEM_CACHE(trace_event_file, SLAB_PANIC);
4831 return 0;
4832}
4833
4834/*
4835 * Helper function to enable or disable a comma-separated list of events
4836 * from the bootup buffer.
4837 */
4838static __init void __early_set_events(struct trace_array *tr, char *buf, bool enable)
4839{
4840 char *token;
4841
4842 while ((token = strsep(&buf, ","))) {
4843 if (*token) {
4844 if (enable) {
4845 if (ftrace_set_clr_event(tr, token, 1))
4846 pr_warn("Failed to enable trace event: %s\n", token);
4847 } else {
4848 ftrace_set_clr_event(tr, token, 0);
4849 }
4850 }
4851
4852 /* Put back the comma to allow this to be called again */
4853 if (buf)
4854 *(buf - 1) = ',';
4855 }
4856}
4857
4858/**
4859 * early_enable_events - enable events from the bootup buffer
4860 * @tr: The trace array to enable the events in
4861 * @buf: The buffer containing the comma separated list of events
4862 * @disable_first: If true, disable all events in @buf before enabling them
4863 *
4864 * This function enables events from the bootup buffer. If @disable_first
4865 * is true, it will first disable all events in the buffer before enabling
4866 * them.
4867 *
4868 * For syscall events, which rely on a global refcount to register the
4869 * SYSCALL_WORK_SYSCALL_TRACEPOINT flag (especially for pid 1), we must
4870 * ensure the refcount hits zero before re-enabling them. A simple
4871 * "disable then enable" per-event is not enough if multiple syscalls are
4872 * used, as the refcount will stay above zero. Thus, we need a two-phase
4873 * approach: disable all, then enable all.
4874 */
4875__init void
4876early_enable_events(struct trace_array *tr, char *buf, bool disable_first)
4877{
4878 if (disable_first)
4879 __early_set_events(tr, buf, false);
4880
4881 __early_set_events(tr, buf, true);
4882}
4883
4884static __init int event_trace_enable(void)
4885{
4886 struct trace_array *tr = top_trace_array();
4887 struct trace_event_call **iter, *call;
4888 int ret;
4889
4890 if (!tr)
4891 return -ENODEV;
4892
4893 for_each_event(iter, __start_ftrace_events, __stop_ftrace_events) {
4894
4895 call = *iter;
4896 ret = event_init(call);
4897 if (!ret)
4898 list_add(&call->list, &ftrace_events);
4899 }
4900
4901 register_trigger_cmds();
4902
4903 /*
4904 * We need the top trace array to have a working set of trace
4905 * points at early init, before the debug files and directories
4906 * are created. Create the file entries now, and attach them
4907 * to the actual file dentries later.
4908 */
4909 __trace_early_add_events(tr);
4910
4911 seq_buf_str(&bootup_event_seq);
4912 early_enable_events(tr, bootup_event_buf, false);
4913
4914 trace_printk_start_comm();
4915
4916 register_event_cmds();
4917
4918
4919 return 0;
4920}
4921
4922/*
4923 * event_trace_enable() is called from trace_event_init() first to
4924 * initialize events and perhaps start any events that are on the
4925 * command line. Unfortunately, there are some events that will not
4926 * start this early, like the system call tracepoints that need
4927 * to set the %SYSCALL_WORK_SYSCALL_TRACEPOINT flag of pid 1. But
4928 * event_trace_enable() is called before pid 1 starts, and this flag
4929 * is never set, making the syscall tracepoint never get reached, but
4930 * the event is enabled regardless (and not doing anything).
4931 */
4932static __init int event_trace_enable_again(void)
4933{
4934 struct trace_array *tr;
4935
4936 tr = top_trace_array();
4937 if (!tr)
4938 return -ENODEV;
4939
4940 seq_buf_str(&bootup_event_seq);
4941 early_enable_events(tr, bootup_event_buf, true);
4942
4943 return 0;
4944}
4945
4946early_initcall(event_trace_enable_again);
4947
4948/* Init fields which doesn't related to the tracefs */
4949static __init int event_trace_init_fields(void)
4950{
4951 if (trace_define_generic_fields())
4952 pr_warn("tracing: Failed to allocated generic fields");
4953
4954 if (trace_define_common_fields())
4955 pr_warn("tracing: Failed to allocate common fields");
4956
4957 return 0;
4958}
4959
4960__init int event_trace_init(void)
4961{
4962 struct trace_array *tr;
4963 int ret;
4964
4965 tr = top_trace_array();
4966 if (!tr)
4967 return -ENODEV;
4968
4969 trace_create_file("available_events", TRACE_MODE_READ,
4970 NULL, tr, &ftrace_avail_fops);
4971
4972 ret = early_event_add_tracer(NULL, tr);
4973 if (ret)
4974 return ret;
4975
4976#ifdef CONFIG_MODULES
4977 ret = register_module_notifier(&trace_module_nb);
4978 if (ret)
4979 pr_warn("Failed to register trace events module notifier\n");
4980#endif
4981
4982 eventdir_initialized = true;
4983
4984 return 0;
4985}
4986
4987void __init trace_event_init(void)
4988{
4989 event_trace_memsetup();
4990 init_ftrace_syscalls();
4991 event_trace_enable();
4992 event_trace_init_fields();
4993}
4994
4995#ifdef CONFIG_EVENT_TRACE_STARTUP_TEST
4996
4997static DEFINE_SPINLOCK(test_spinlock);
4998static DEFINE_SPINLOCK(test_spinlock_irq);
4999static DEFINE_MUTEX(test_mutex);
5000
5001static __init void test_work(struct work_struct *dummy)
5002{
5003 spin_lock(&test_spinlock);
5004 spin_lock_irq(&test_spinlock_irq);
5005 udelay(1);
5006 spin_unlock_irq(&test_spinlock_irq);
5007 spin_unlock(&test_spinlock);
5008
5009 mutex_lock(&test_mutex);
5010 msleep(1);
5011 mutex_unlock(&test_mutex);
5012}
5013
5014static __init int event_test_thread(void *unused)
5015{
5016 void *test_malloc;
5017
5018 test_malloc = kmalloc(1234, GFP_KERNEL);
5019 if (!test_malloc)
5020 pr_info("failed to kmalloc\n");
5021
5022 schedule_on_each_cpu(test_work);
5023
5024 kfree(test_malloc);
5025
5026 set_current_state(TASK_INTERRUPTIBLE);
5027 while (!kthread_should_stop()) {
5028 schedule();
5029 set_current_state(TASK_INTERRUPTIBLE);
5030 }
5031 __set_current_state(TASK_RUNNING);
5032
5033 return 0;
5034}
5035
5036/*
5037 * Do various things that may trigger events.
5038 */
5039static __init void event_test_stuff(void)
5040{
5041 struct task_struct *test_thread;
5042
5043 test_thread = kthread_run(event_test_thread, NULL, "test-events");
5044 if (WARN_ON(IS_ERR(test_thread)))
5045 return;
5046 msleep(1);
5047 kthread_stop(test_thread);
5048}
5049
5050/*
5051 * For every trace event defined, we will test each trace point separately,
5052 * and then by groups, and finally all trace points.
5053 */
5054static __init void event_trace_self_tests(void)
5055{
5056 struct trace_subsystem_dir *dir;
5057 struct trace_event_file *file;
5058 struct trace_event_call *call;
5059 struct event_subsystem *system;
5060 struct trace_array *tr;
5061 int ret;
5062
5063 tr = top_trace_array();
5064 if (!tr)
5065 return;
5066
5067 pr_info("Running tests on trace events:\n");
5068
5069 list_for_each_entry(file, &tr->events, list) {
5070
5071 call = file->event_call;
5072
5073 /* Only test those that have a probe */
5074 if (!call->class || !call->class->probe)
5075 continue;
5076
5077/*
5078 * Testing syscall events here is pretty useless, but
5079 * we still do it if configured. But this is time consuming.
5080 * What we really need is a user thread to perform the
5081 * syscalls as we test.
5082 */
5083#ifndef CONFIG_EVENT_TRACE_TEST_SYSCALLS
5084 if (call->class->system &&
5085 strcmp(call->class->system, "syscalls") == 0)
5086 continue;
5087#endif
5088
5089 pr_info("Testing event %s: ", trace_event_name(call));
5090
5091 /*
5092 * If an event is already enabled, someone is using
5093 * it and the self test should not be on.
5094 */
5095 if (file->flags & EVENT_FILE_FL_ENABLED) {
5096 pr_warn("Enabled event during self test!\n");
5097 WARN_ON_ONCE(1);
5098 continue;
5099 }
5100
5101 ftrace_event_enable_disable(file, 1);
5102 event_test_stuff();
5103 ftrace_event_enable_disable(file, 0);
5104
5105 pr_cont("OK\n");
5106 }
5107
5108 /* Now test at the sub system level */
5109
5110 pr_info("Running tests on trace event systems:\n");
5111
5112 list_for_each_entry(dir, &tr->systems, list) {
5113
5114 system = dir->subsystem;
5115
5116 /* the ftrace system is special, skip it */
5117 if (strcmp(system->name, "ftrace") == 0)
5118 continue;
5119
5120 pr_info("Testing event system %s: ", system->name);
5121
5122 ret = __ftrace_set_clr_event(tr, NULL, system->name, NULL, 1, NULL);
5123 if (WARN_ON_ONCE(ret)) {
5124 pr_warn("error enabling system %s\n",
5125 system->name);
5126 continue;
5127 }
5128
5129 event_test_stuff();
5130
5131 ret = __ftrace_set_clr_event(tr, NULL, system->name, NULL, 0, NULL);
5132 if (WARN_ON_ONCE(ret)) {
5133 pr_warn("error disabling system %s\n",
5134 system->name);
5135 continue;
5136 }
5137
5138 pr_cont("OK\n");
5139 }
5140
5141 /* Test with all events enabled */
5142
5143 pr_info("Running tests on all trace events:\n");
5144 pr_info("Testing all events: ");
5145
5146 ret = __ftrace_set_clr_event(tr, NULL, NULL, NULL, 1, NULL);
5147 if (WARN_ON_ONCE(ret)) {
5148 pr_warn("error enabling all events\n");
5149 return;
5150 }
5151
5152 event_test_stuff();
5153
5154 /* reset sysname */
5155 ret = __ftrace_set_clr_event(tr, NULL, NULL, NULL, 0, NULL);
5156 if (WARN_ON_ONCE(ret)) {
5157 pr_warn("error disabling all events\n");
5158 return;
5159 }
5160
5161 pr_cont("OK\n");
5162}
5163
5164#ifdef CONFIG_FUNCTION_TRACER
5165
5166static DEFINE_PER_CPU(atomic_t, ftrace_test_event_disable);
5167
5168static struct trace_event_file event_trace_file __initdata;
5169
5170static void __init
5171function_test_events_call(unsigned long ip, unsigned long parent_ip,
5172 struct ftrace_ops *op, struct ftrace_regs *regs)
5173{
5174 struct trace_buffer *buffer;
5175 struct ring_buffer_event *event;
5176 struct ftrace_entry *entry;
5177 unsigned int trace_ctx;
5178 long disabled;
5179 int cpu;
5180
5181 trace_ctx = tracing_gen_ctx();
5182 preempt_disable_notrace();
5183 cpu = raw_smp_processor_id();
5184 disabled = atomic_inc_return(&per_cpu(ftrace_test_event_disable, cpu));
5185
5186 if (disabled != 1)
5187 goto out;
5188
5189 event = trace_event_buffer_lock_reserve(&buffer, &event_trace_file,
5190 TRACE_FN, sizeof(*entry),
5191 trace_ctx);
5192 if (!event)
5193 goto out;
5194 entry = ring_buffer_event_data(event);
5195 entry->ip = ip;
5196 entry->parent_ip = parent_ip;
5197
5198 event_trigger_unlock_commit(&event_trace_file, buffer, event,
5199 entry, trace_ctx);
5200 out:
5201 atomic_dec(&per_cpu(ftrace_test_event_disable, cpu));
5202 preempt_enable_notrace();
5203}
5204
5205static struct ftrace_ops trace_ops __initdata =
5206{
5207 .func = function_test_events_call,
5208};
5209
5210static __init void event_trace_self_test_with_function(void)
5211{
5212 int ret;
5213
5214 event_trace_file.tr = top_trace_array();
5215 if (WARN_ON(!event_trace_file.tr))
5216 return;
5217
5218 ret = register_ftrace_function(&trace_ops);
5219 if (WARN_ON(ret < 0)) {
5220 pr_info("Failed to enable function tracer for event tests\n");
5221 return;
5222 }
5223 pr_info("Running tests again, along with the function tracer\n");
5224 event_trace_self_tests();
5225 unregister_ftrace_function(&trace_ops);
5226}
5227#else
5228static __init void event_trace_self_test_with_function(void)
5229{
5230}
5231#endif
5232
5233static __init int event_trace_self_tests_init(void)
5234{
5235 if (!tracing_selftest_disabled) {
5236 event_trace_self_tests();
5237 event_trace_self_test_with_function();
5238 }
5239
5240 return 0;
5241}
5242
5243late_initcall(event_trace_self_tests_init);
5244
5245#endif