projects/linux·blob master · refs

546 lines · 16.1 KiB

1/* SPDX-License-Identifier: GPL-2.0 */
2/*
3 * Stage 1 of the trace events.
4 *
5 * Override the macros in the event tracepoint header <trace/events/XXX.h>
6 * to include the following:
7 *
8 * struct trace_event_raw_<call> {
9 * struct trace_entry ent;
10 * <type> <item>;
11 * <type2> <item2>[<len>];
12 * [...]
13 * };
14 *
15 * The <type> <item> is created by the __field(type, item) macro or
16 * the __array(type2, item2, len) macro.
17 * We simply do "type item;", and that will create the fields
18 * in the structure.
19 */
20
21#include <linux/trace_events.h>
22#include <linux/btf_ids.h>
23
24#ifndef TRACE_SYSTEM_VAR
25#define TRACE_SYSTEM_VAR TRACE_SYSTEM
26#endif
27
28#include "stages/init.h"
29
30/*
31 * DECLARE_EVENT_CLASS can be used to add a generic function
32 * handlers for events. That is, if all events have the same
33 * parameters and just have distinct trace points.
34 * Each tracepoint can be defined with DEFINE_EVENT and that
35 * will map the DECLARE_EVENT_CLASS to the tracepoint.
36 *
37 * TRACE_EVENT is a one to one mapping between tracepoint and template.
38 */
39#undef TRACE_EVENT
40#define TRACE_EVENT(name, proto, args, tstruct, assign, print) \
41 DECLARE_EVENT_CLASS(name, \
42 PARAMS(proto), \
43 PARAMS(args), \
44 PARAMS(tstruct), \
45 PARAMS(assign), \
46 PARAMS(print)); \
47 DEFINE_EVENT(name, name, PARAMS(proto), PARAMS(args));
48
49#undef TRACE_EVENT_SYSCALL
50#define TRACE_EVENT_SYSCALL(name, proto, args, tstruct, assign, print, reg, unreg) \
51 DECLARE_EVENT_SYSCALL_CLASS(name, \
52 PARAMS(proto), \
53 PARAMS(args), \
54 PARAMS(tstruct), \
55 PARAMS(assign), \
56 PARAMS(print)); \
57 DEFINE_EVENT(name, name, PARAMS(proto), PARAMS(args));
58
59#include "stages/stage1_struct_define.h"
60
61#undef DECLARE_EVENT_CLASS
62#define DECLARE_EVENT_CLASS(name, proto, args, tstruct, assign, print) \
63 struct trace_event_raw_##name { \
64 struct trace_entry ent; \
65 tstruct \
66 char __data[]; \
67 }; \
68 \
69 static struct trace_event_class event_class_##name;
70
71#undef DECLARE_EVENT_SYSCALL_CLASS
72#define DECLARE_EVENT_SYSCALL_CLASS DECLARE_EVENT_CLASS
73
74#undef DEFINE_EVENT
75#define DEFINE_EVENT(template, name, proto, args) \
76 static struct trace_event_call __used \
77 __attribute__((__aligned__(4))) event_##name
78
79#undef DEFINE_EVENT_FN
80#define DEFINE_EVENT_FN(template, name, proto, args, reg, unreg) \
81 DEFINE_EVENT(template, name, PARAMS(proto), PARAMS(args))
82
83#undef DEFINE_EVENT_PRINT
84#define DEFINE_EVENT_PRINT(template, name, proto, args, print) \
85 DEFINE_EVENT(template, name, PARAMS(proto), PARAMS(args))
86
87/* Callbacks are meaningless to ftrace. */
88#undef TRACE_EVENT_FN
89#define TRACE_EVENT_FN(name, proto, args, tstruct, \
90 assign, print, reg, unreg) \
91 TRACE_EVENT(name, PARAMS(proto), PARAMS(args), \
92 PARAMS(tstruct), PARAMS(assign), PARAMS(print)) \
93
94#undef TRACE_EVENT_FN_COND
95#define TRACE_EVENT_FN_COND(name, proto, args, cond, tstruct, \
96 assign, print, reg, unreg) \
97 TRACE_EVENT_CONDITION(name, PARAMS(proto), PARAMS(args), PARAMS(cond), \
98 PARAMS(tstruct), PARAMS(assign), PARAMS(print)) \
99
100#undef TRACE_EVENT_FLAGS
101#define TRACE_EVENT_FLAGS(name, value) \
102 __TRACE_EVENT_FLAGS(name, value)
103
104#undef TRACE_EVENT_PERF_PERM
105#define TRACE_EVENT_PERF_PERM(name, expr...) \
106 __TRACE_EVENT_PERF_PERM(name, expr)
107
108#include TRACE_INCLUDE(TRACE_INCLUDE_FILE)
109
110/*
111 * Stage 2 of the trace events.
112 *
113 * Include the following:
114 *
115 * struct trace_event_data_offsets_<call> {
116 * u32 <item1>;
117 * u32 <item2>;
118 * [...]
119 * };
120 *
121 * The __dynamic_array() macro will create each u32 <item>, this is
122 * to keep the offset of each array from the beginning of the event.
123 * The size of an array is also encoded, in the higher 16 bits of <item>.
124 */
125
126#include "stages/stage2_data_offsets.h"
127
128#undef DECLARE_EVENT_CLASS
129#define DECLARE_EVENT_CLASS(call, proto, args, tstruct, assign, print) \
130 struct trace_event_data_offsets_##call { \
131 tstruct; \
132 };
133
134#undef DECLARE_EVENT_SYSCALL_CLASS
135#define DECLARE_EVENT_SYSCALL_CLASS DECLARE_EVENT_CLASS
136
137#undef DEFINE_EVENT
138#define DEFINE_EVENT(template, name, proto, args)
139
140#undef DEFINE_EVENT_PRINT
141#define DEFINE_EVENT_PRINT(template, name, proto, args, print)
142
143#undef TRACE_EVENT_FLAGS
144#define TRACE_EVENT_FLAGS(event, flag)
145
146#undef TRACE_EVENT_PERF_PERM
147#define TRACE_EVENT_PERF_PERM(event, expr...)
148
149#include TRACE_INCLUDE(TRACE_INCLUDE_FILE)
150
151/*
152 * Stage 3 of the trace events.
153 *
154 * Override the macros in the event tracepoint header <trace/events/XXX.h>
155 * to include the following:
156 *
157 * enum print_line_t
158 * trace_raw_output_<call>(struct trace_iterator *iter, int flags)
159 * {
160 * struct trace_seq *s = &iter->seq;
161 * struct trace_event_raw_<call> *field; <-- defined in stage 1
162 * struct trace_seq *p = &iter->tmp_seq;
163 *
164 * -------(for event)-------
165 *
166 * struct trace_entry *entry;
167 *
168 * entry = iter->ent;
169 *
170 * if (entry->type != event_<call>->event.type) {
171 * WARN_ON_ONCE(1);
172 * return TRACE_TYPE_UNHANDLED;
173 * }
174 *
175 * field = (typeof(field))entry;
176 *
177 * trace_seq_init(p);
178 * return trace_output_call(iter, <call>, <TP_printk> "\n");
179 *
180 * ------(or, for event class)------
181 *
182 * int ret;
183 *
184 * field = (typeof(field))iter->ent;
185 *
186 * ret = trace_raw_output_prep(iter, trace_event);
187 * if (ret != TRACE_TYPE_HANDLED)
188 * return ret;
189 *
190 * trace_event_printf(iter, <TP_printk> "\n");
191 *
192 * return trace_handle_return(s);
193 * -------
194 * }
195 *
196 * This is the method used to print the raw event to the trace
197 * output format. Note, this is not needed if the data is read
198 * in binary.
199 */
200
201#include "stages/stage3_trace_output.h"
202
203#undef DECLARE_EVENT_CLASS
204#define DECLARE_EVENT_CLASS(call, proto, args, tstruct, assign, print) \
205static notrace enum print_line_t \
206trace_raw_output_##call(struct trace_iterator *iter, int flags, \
207 struct trace_event *trace_event) \
208{ \
209 struct trace_seq *s = &iter->seq; \
210 struct trace_seq __maybe_unused *p = &iter->tmp_seq; \
211 struct trace_event_raw_##call *field; \
212 int ret; \
213 \
214 field = (typeof(field))iter->ent; \
215 \
216 ret = trace_raw_output_prep(iter, trace_event); \
217 if (ret != TRACE_TYPE_HANDLED) \
218 return ret; \
219 \
220 trace_event_printf(iter, print); \
221 \
222 return trace_handle_return(s); \
223} \
224static struct trace_event_functions trace_event_type_funcs_##call = { \
225 .trace = trace_raw_output_##call, \
226};
227
228#undef DECLARE_EVENT_SYSCALL_CLASS
229#define DECLARE_EVENT_SYSCALL_CLASS DECLARE_EVENT_CLASS
230
231#undef DEFINE_EVENT_PRINT
232#define DEFINE_EVENT_PRINT(template, call, proto, args, print) \
233static notrace enum print_line_t \
234trace_raw_output_##call(struct trace_iterator *iter, int flags, \
235 struct trace_event *event) \
236{ \
237 struct trace_event_raw_##template *field; \
238 struct trace_entry *entry; \
239 struct trace_seq *p = &iter->tmp_seq; \
240 \
241 entry = iter->ent; \
242 \
243 if (entry->type != event_##call.event.type) { \
244 WARN_ON_ONCE(1); \
245 return TRACE_TYPE_UNHANDLED; \
246 } \
247 \
248 field = (typeof(field))entry; \
249 \
250 trace_seq_init(p); \
251 return trace_output_call(iter, #call, print); \
252} \
253static struct trace_event_functions trace_event_type_funcs_##call = { \
254 .trace = trace_raw_output_##call, \
255};
256
257#include TRACE_INCLUDE(TRACE_INCLUDE_FILE)
258
259#include "stages/stage4_event_fields.h"
260
261#undef DECLARE_EVENT_CLASS
262#define DECLARE_EVENT_CLASS(call, proto, args, tstruct, func, print) \
263static struct trace_event_fields trace_event_fields_##call[] = { \
264 tstruct \
265 {} };
266
267#undef DECLARE_EVENT_SYSCALL_CLASS
268#define DECLARE_EVENT_SYSCALL_CLASS DECLARE_EVENT_CLASS
269
270#undef DEFINE_EVENT_PRINT
271#define DEFINE_EVENT_PRINT(template, name, proto, args, print)
272
273#include TRACE_INCLUDE(TRACE_INCLUDE_FILE)
274
275#include "stages/stage5_get_offsets.h"
276
277#undef DECLARE_EVENT_CLASS
278#define DECLARE_EVENT_CLASS(call, proto, args, tstruct, assign, print) \
279static inline notrace int trace_event_get_offsets_##call( \
280 struct trace_event_data_offsets_##call *__data_offsets, proto) \
281{ \
282 int __data_size = 0; \
283 int __maybe_unused __item_length; \
284 struct trace_event_raw_##call __maybe_unused *entry; \
285 \
286 tstruct; \
287 \
288 return __data_size; \
289}
290
291#undef DECLARE_EVENT_SYSCALL_CLASS
292#define DECLARE_EVENT_SYSCALL_CLASS DECLARE_EVENT_CLASS
293
294#include TRACE_INCLUDE(TRACE_INCLUDE_FILE)
295
296/*
297 * Stage 4 of the trace events.
298 *
299 * Override the macros in the event tracepoint header <trace/events/XXX.h>
300 * to include the following:
301 *
302 * For those macros defined with TRACE_EVENT:
303 *
304 * static struct trace_event_call event_<call>;
305 *
306 * static void trace_event_raw_event_<call>(void *__data, proto)
307 * {
308 * struct trace_event_file *trace_file = __data;
309 * struct trace_event_call *event_call = trace_file->event_call;
310 * struct trace_event_data_offsets_<call> __maybe_unused __data_offsets;
311 * unsigned long eflags = trace_file->flags;
312 * enum event_trigger_type __tt = ETT_NONE;
313 * struct ring_buffer_event *event;
314 * struct trace_event_raw_<call> *entry; <-- defined in stage 1
315 * struct trace_buffer *buffer;
316 * unsigned long irq_flags;
317 * int __data_size;
318 * int pc;
319 *
320 * if (!(eflags & EVENT_FILE_FL_TRIGGER_COND)) {
321 * if (eflags & EVENT_FILE_FL_TRIGGER_MODE)
322 * event_triggers_call(trace_file, NULL);
323 * if (eflags & EVENT_FILE_FL_SOFT_DISABLED)
324 * return;
325 * }
326 *
327 * local_save_flags(irq_flags);
328 * pc = preempt_count();
329 *
330 * __data_size = trace_event_get_offsets_<call>(&__data_offsets, args);
331 *
332 * event = trace_event_buffer_lock_reserve(&buffer, trace_file,
333 * event_<call>->event.type,
334 * sizeof(*entry) + __data_size,
335 * irq_flags, pc);
336 * if (!event)
337 * return;
338 * entry = ring_buffer_event_data(event);
339 *
340 * { <assign>; } <-- Here we assign the entries by the __field and
341 * __array macros.
342 *
343 * if (eflags & EVENT_FILE_FL_TRIGGER_COND)
344 * __tt = event_triggers_call(trace_file, entry);
345 *
346 * if (test_bit(EVENT_FILE_FL_SOFT_DISABLED_BIT,
347 * &trace_file->flags))
348 * ring_buffer_discard_commit(buffer, event);
349 * else if (!filter_check_discard(trace_file, entry, buffer, event))
350 * trace_buffer_unlock_commit(buffer, event, irq_flags, pc);
351 *
352 * if (__tt)
353 * event_triggers_post_call(trace_file, __tt);
354 * }
355 *
356 * static struct trace_event ftrace_event_type_<call> = {
357 * .trace = trace_raw_output_<call>, <-- stage 2
358 * };
359 *
360 * static char print_fmt_<call>[] = <TP_printk>;
361 *
362 * static struct trace_event_class __used event_class_<template> = {
363 * .system = "<system>",
364 * .fields_array = trace_event_fields_<call>,
365 * .fields = LIST_HEAD_INIT(event_class_##call.fields),
366 * .raw_init = trace_event_raw_init,
367 * .probe = trace_event_raw_event_##call,
368 * .reg = trace_event_reg,
369 * };
370 *
371 * static struct trace_event_call event_<call> = {
372 * .class = event_class_<template>,
373 * {
374 * .tp = &__tracepoint_<call>,
375 * },
376 * .event = &ftrace_event_type_<call>,
377 * .print_fmt = print_fmt_<call>,
378 * .flags = TRACE_EVENT_FL_TRACEPOINT,
379 * };
380 * // its only safe to use pointers when doing linker tricks to
381 * // create an array.
382 * static struct trace_event_call __used
383 * __section("_ftrace_events") *__event_<call> = &event_<call>;
384 *
385 */
386
387#ifdef CONFIG_PERF_EVENTS
388
389#define _TRACE_PERF_PROTO(call, proto) \
390 static notrace void \
391 perf_trace_##call(void *__data, proto);
392
393#define _TRACE_PERF_INIT(call) \
394 .perf_probe = perf_trace_##call,
395
396#else
397#define _TRACE_PERF_PROTO(call, proto)
398#define _TRACE_PERF_INIT(call)
399#endif /* CONFIG_PERF_EVENTS */
400
401#if defined(CONFIG_BPF_EVENTS) && defined(CONFIG_DEBUG_INFO_BTF)
402/*
403 * Per-template BTF id list, populated at link time by resolve_btfids:
404 * [0] FUNC __bpf_trace_<call> (the BPF dispatcher)
405 * [1] STRUCT trace_event_raw_<call> (the ring-buffer record)
406 * Exposed via the events/<sys>/<name>/btf_ids tracefs file.
407 */
408#define _TRACE_BTF_IDS_DECLARE(call) \
409 BTF_ID_LIST(__bpf_trace_btf_ids_##call) \
410 BTF_ID(func, __bpf_trace_##call) \
411 BTF_ID(struct, trace_event_raw_##call)
412
413#define _TRACE_BTF_IDS_INIT(call) \
414 .btf_ids = __bpf_trace_btf_ids_##call,
415
416#else
417#define _TRACE_BTF_IDS_DECLARE(call)
418#define _TRACE_BTF_IDS_INIT(call)
419#endif /* CONFIG_BPF_EVENTS && CONFIG_DEBUG_INFO_BTF */
420
421#include "stages/stage6_event_callback.h"
422
423
424#undef __DECLARE_EVENT_CLASS
425#define __DECLARE_EVENT_CLASS(call, proto, args, tstruct, assign, print) \
426static notrace void \
427do_trace_event_raw_event_##call(void *__data, proto) \
428{ \
429 struct trace_event_file *trace_file = __data; \
430 struct trace_event_data_offsets_##call __maybe_unused __data_offsets;\
431 struct trace_event_buffer fbuffer; \
432 struct trace_event_raw_##call *entry; \
433 int __data_size; \
434 \
435 if (trace_trigger_soft_disabled(trace_file)) \
436 return; \
437 \
438 __data_size = trace_event_get_offsets_##call(&__data_offsets, args); \
439 \
440 entry = trace_event_buffer_reserve(&fbuffer, trace_file, \
441 sizeof(*entry) + __data_size); \
442 \
443 if (!entry) \
444 return; \
445 \
446 tstruct \
447 \
448 { assign; } \
449 \
450 trace_event_buffer_commit(&fbuffer); \
451}
452
453#undef DECLARE_EVENT_CLASS
454#define DECLARE_EVENT_CLASS(call, proto, args, tstruct, assign, print) \
455__DECLARE_EVENT_CLASS(call, PARAMS(proto), PARAMS(args), PARAMS(tstruct), \
456 PARAMS(assign), PARAMS(print)) \
457static notrace void \
458trace_event_raw_event_##call(void *__data, proto) \
459{ \
460 guard(preempt_notrace)(); \
461 do_trace_event_raw_event_##call(__data, args); \
462}
463
464#undef DECLARE_EVENT_SYSCALL_CLASS
465#define DECLARE_EVENT_SYSCALL_CLASS(call, proto, args, tstruct, assign, print) \
466__DECLARE_EVENT_CLASS(call, PARAMS(proto), PARAMS(args), PARAMS(tstruct), \
467 PARAMS(assign), PARAMS(print)) \
468static notrace void \
469trace_event_raw_event_##call(void *__data, proto) \
470{ \
471 might_fault(); \
472 guard(preempt_notrace)(); \
473 do_trace_event_raw_event_##call(__data, args); \
474}
475
476/*
477 * The ftrace_test_probe is compiled out, it is only here as a build time check
478 * to make sure that if the tracepoint handling changes, the ftrace probe will
479 * fail to compile unless it too is updated.
480 */
481
482#undef DEFINE_EVENT
483#define DEFINE_EVENT(template, call, proto, args) \
484static inline void ftrace_test_probe_##call(void) \
485{ \
486 check_trace_callback_type_##call(trace_event_raw_event_##template); \
487}
488
489#include TRACE_INCLUDE(TRACE_INCLUDE_FILE)
490
491#undef __DECLARE_EVENT_CLASS
492
493#include "stages/stage7_class_define.h"
494
495#undef DECLARE_EVENT_CLASS
496#define DECLARE_EVENT_CLASS(call, proto, args, tstruct, assign, print) \
497_TRACE_PERF_PROTO(call, PARAMS(proto)); \
498_TRACE_BTF_IDS_DECLARE(call) \
499static char print_fmt_##call[] = print; \
500static struct trace_event_class __used __refdata event_class_##call = { \
501 .system = TRACE_SYSTEM_STRING, \
502 .fields_array = trace_event_fields_##call, \
503 .fields = LIST_HEAD_INIT(event_class_##call.fields),\
504 .raw_init = trace_event_raw_init, \
505 .probe = trace_event_raw_event_##call, \
506 .reg = trace_event_reg, \
507 _TRACE_PERF_INIT(call) \
508 _TRACE_BTF_IDS_INIT(call) \
509};
510
511#undef DECLARE_EVENT_SYSCALL_CLASS
512#define DECLARE_EVENT_SYSCALL_CLASS DECLARE_EVENT_CLASS
513
514#undef DEFINE_EVENT
515#define DEFINE_EVENT(template, call, proto, args) \
516 \
517static struct trace_event_call __used event_##call = { \
518 .class = &event_class_##template, \
519 { \
520 .tp = &__tracepoint_##call, \
521 }, \
522 .event.funcs = &trace_event_type_funcs_##template, \
523 .print_fmt = print_fmt_##template, \
524 .flags = TRACE_EVENT_FL_TRACEPOINT, \
525}; \
526static struct trace_event_call __used \
527__section("_ftrace_events") *__event_##call = &event_##call
528
529#undef DEFINE_EVENT_PRINT
530#define DEFINE_EVENT_PRINT(template, call, proto, args, print) \
531 \
532static char print_fmt_##call[] = print; \
533 \
534static struct trace_event_call __used event_##call = { \
535 .class = &event_class_##template, \
536 { \
537 .tp = &__tracepoint_##call, \
538 }, \
539 .event.funcs = &trace_event_type_funcs_##call, \
540 .print_fmt = print_fmt_##call, \
541 .flags = TRACE_EVENT_FL_TRACEPOINT, \
542}; \
543static struct trace_event_call __used \
544__section("_ftrace_events") *__event_##call = &event_##call
545
546#include TRACE_INCLUDE(TRACE_INCLUDE_FILE)