linux - kernel/trace/trace_events

7101 lines · 174.0 KiB

1	// SPDX-License-Identifier: GPL-2.0
2	/*
3	* trace_events_hist - trace event hist triggers
4	*
5	* Copyright (C) 2015 Tom Zanussi <[email protected]>
6	*/
7
8	#include <linux/module.h>
9	#include <linux/kallsyms.h>
10	#include <linux/security.h>
11	#include <linux/seq_buf.h>
12	#include <linux/mutex.h>
13	#include <linux/slab.h>
14	#include <linux/stacktrace.h>
15	#include <linux/rculist.h>
16	#include <linux/tracefs.h>
17
18	/* for gfp flag names */
19	#include <linux/trace_events.h>
20	#include <trace/events/mmflags.h>
21
22	#include "tracing_map.h"
23	#include "trace_synth.h"
24
25	#define ERRORS \
26	C(NONE, "No error"), \
27	C(DUPLICATE_VAR, "Variable already defined"), \
28	C(VAR_NOT_UNIQUE, "Variable name not unique, need to use fully qualified name (subsys.event.var) for variable"), \
29	C(TOO_MANY_VARS, "Too many variables defined"), \
30	C(MALFORMED_ASSIGNMENT, "Malformed assignment"), \
31	C(NAMED_MISMATCH, "Named hist trigger doesn't match existing named trigger (includes variables)"), \
32	C(TRIGGER_EEXIST, "Hist trigger already exists"), \
33	C(TRIGGER_ENOENT_CLEAR, "Can't clear or continue a nonexistent hist trigger"), \
34	C(SET_CLOCK_FAIL, "Couldn't set trace_clock"), \
35	C(BAD_FIELD_MODIFIER, "Invalid field modifier"), \
36	C(TOO_MANY_SUBEXPR, "Too many subexpressions (3 max)"), \
37	C(TIMESTAMP_MISMATCH, "Timestamp units in expression don't match"), \
38	C(TOO_MANY_FIELD_VARS, "Too many field variables defined"), \
39	C(EVENT_FILE_NOT_FOUND, "Event file not found"), \
40	C(HIST_NOT_FOUND, "Matching event histogram not found"), \
41	C(HIST_CREATE_FAIL, "Couldn't create histogram for field"), \
42	C(SYNTH_VAR_NOT_FOUND, "Couldn't find synthetic variable"), \
43	C(SYNTH_EVENT_NOT_FOUND,"Couldn't find synthetic event"), \
44	C(SYNTH_TYPE_MISMATCH, "Param type doesn't match synthetic event field type"), \
45	C(SYNTH_COUNT_MISMATCH, "Param count doesn't match synthetic event field count"), \
46	C(FIELD_VAR_PARSE_FAIL, "Couldn't parse field variable"), \
47	C(VAR_CREATE_FIND_FAIL, "Couldn't create or find variable"), \
48	C(ONX_NOT_VAR, "For onmax(x) or onchange(x), x must be a variable"), \
49	C(ONX_VAR_NOT_FOUND, "Couldn't find onmax or onchange variable"), \
50	C(ONX_VAR_CREATE_FAIL, "Couldn't create onmax or onchange variable"), \
51	C(FIELD_VAR_CREATE_FAIL,"Couldn't create field variable"), \
52	C(TOO_MANY_PARAMS, "Too many action params"), \
53	C(PARAM_NOT_FOUND, "Couldn't find param"), \
54	C(INVALID_PARAM, "Invalid action param"), \
55	C(ACTION_NOT_FOUND, "No action found"), \
56	C(NO_SAVE_PARAMS, "No params found for save()"), \
57	C(TOO_MANY_SAVE_ACTIONS,"Can't have more than one save() action per hist"), \
58	C(ACTION_MISMATCH, "Handler doesn't support action"), \
59	C(NO_CLOSING_PAREN, "No closing paren found"), \
60	C(SUBSYS_NOT_FOUND, "Missing subsystem"), \
61	C(INVALID_SUBSYS_EVENT, "Invalid subsystem or event name"), \
62	C(INVALID_REF_KEY, "Using variable references in keys not supported"), \
63	C(VAR_NOT_FOUND, "Couldn't find variable"), \
64	C(FIELD_NOT_FOUND, "Couldn't find field"), \
65	C(EMPTY_ASSIGNMENT, "Empty assignment"), \
66	C(INVALID_SORT_MODIFIER,"Invalid sort modifier"), \
67	C(EMPTY_SORT_FIELD, "Empty sort field"), \
68	C(TOO_MANY_SORT_FIELDS, "Too many sort fields (Max = 2)"), \
69	C(INVALID_SORT_FIELD, "Sort field must be a key or a val"), \
70	C(INVALID_STR_OPERAND, "String type can not be an operand in expression"), \
71	C(EXPECT_NUMBER, "Expecting numeric literal"), \
72	C(UNARY_MINUS_SUBEXPR, "Unary minus not supported in sub-expressions"), \
73	C(DIVISION_BY_ZERO, "Division by zero"), \
74	C(NEED_NOHC_VAL, "Non-hitcount value is required for 'nohitcount'"),
75
76	#undef C
77	#define C(a, b) HIST_ERR_##a
78
79	enum { ERRORS };
80
81	#undef C
82	#define C(a, b) b
83
84	static const char *err_text[] = { ERRORS };
85
86	struct hist_field;
87
88	typedef u64 (hist_field_fn_t) (struct hist_field field,
89	struct tracing_map_elt *elt,
90	struct trace_buffer *buffer,
91	struct ring_buffer_event *rbe,
92	void *event);
93
94	#define HIST_FIELD_OPERANDS_MAX 2
95	#define HIST_FIELDS_MAX (TRACING_MAP_FIELDS_MAX + TRACING_MAP_VARS_MAX)
96	#define HIST_ACTIONS_MAX 8
97	#define HIST_DIV_SHIFT 20 /* For optimizing division by constants */
98
99	enum field_op_id {
100	FIELD_OP_NONE,
101	FIELD_OP_PLUS,
102	FIELD_OP_MINUS,
103	FIELD_OP_UNARY_MINUS,
104	FIELD_OP_DIV,
105	FIELD_OP_MULT,
106	};
107
108	#define FIELD_FUNCS \
109	C(NOP, "nop"), \
110	C(VAR_REF, "var_ref"), \
111	C(COUNTER, "counter"), \
112	C(CONST, "const"), \
113	C(LOG2, "log2"), \
114	C(BUCKET, "bucket"), \
115	C(TIMESTAMP, "timestamp"), \
116	C(CPU, "cpu"), \
117	C(COMM, "comm"), \
118	C(STRING, "string"), \
119	C(DYNSTRING, "dynstring"), \
120	C(RELDYNSTRING, "reldynstring"), \
121	C(PSTRING, "pstring"), \
122	C(S64, "s64"), \
123	C(U64, "u64"), \
124	C(S32, "s32"), \
125	C(U32, "u32"), \
126	C(S16, "s16"), \
127	C(U16, "u16"), \
128	C(S8, "s8"), \
129	C(U8, "u8"), \
130	C(UMINUS, "uminus"), \
131	C(MINUS, "minus"), \
132	C(PLUS, "plus"), \
133	C(DIV, "div"), \
134	C(MULT, "mult"), \
135	C(DIV_POWER2, "div_power2"), \
136	C(DIV_NOT_POWER2, "div_not_power2"), \
137	C(DIV_MULT_SHIFT, "div_mult_shift"), \
138	C(EXECNAME, "execname"), \
139	C(STACK, "stack"),
140
141	#undef C
142	#define C(a, b) HIST_FIELD_FN_##a
143
144	enum hist_field_fn {
145	FIELD_FUNCS
146	};
147
148	/*
149	* A hist_var (histogram variable) contains variable information for
150	* hist_fields having the HIST_FIELD_FL_VAR or HIST_FIELD_FL_VAR_REF
151	* flag set. A hist_var has a variable name e.g. ts0, and is
152	* associated with a given histogram trigger, as specified by
153	* hist_data. The hist_var idx is the unique index assigned to the
154	* variable by the hist trigger's tracing_map. The idx is what is
155	* used to set a variable's value and, by a variable reference, to
156	* retrieve it.
157	*/
158	struct hist_var {
159	char *name;
160	struct hist_trigger_data *hist_data;
161	unsigned int idx;
162	};
163
164	struct hist_field {
165	struct ftrace_event_field *field;
166	unsigned long flags;
167	unsigned long buckets;
168	const char *type;
169	struct hist_field *operands[HIST_FIELD_OPERANDS_MAX];
170	struct hist_trigger_data *hist_data;
171	enum hist_field_fn fn_num;
172	unsigned int size;
173	unsigned int offset;
174	unsigned int is_signed;
175
176	/*
177	* Variable fields contain variable-specific info in var.
178	*/
179	struct hist_var var;
180	enum field_op_id operator;
181	char *system;
182	char *event_name;
183
184	/*
185	* The name field is used for EXPR and VAR_REF fields. VAR
186	* fields contain the variable name in var.name.
187	*/
188	char *name;
189
190	/*
191	* When a histogram trigger is hit, if it has any references
192	* to variables, the values of those variables are collected
193	* into a var_ref_vals array by resolve_var_refs(). The
194	* current value of each variable is read from the tracing_map
195	* using the hist field's hist_var.idx and entered into the
196	* var_ref_idx entry i.e. var_ref_vals[var_ref_idx].
197	*/
198	unsigned int var_ref_idx;
199	bool read_once;
200
201	unsigned int var_str_idx;
202
203	/* Numeric literals are represented as u64 */
204	u64 constant;
205	/* Used to optimize division by constants */
206	u64 div_multiplier;
207	};
208
209	static u64 hist_fn_call(struct hist_field *hist_field,
210	struct tracing_map_elt *elt,
211	struct trace_buffer *buffer,
212	struct ring_buffer_event *rbe,
213	void *event);
214
215	static u64 hist_field_const(struct hist_field *field,
216	struct tracing_map_elt *elt,
217	struct trace_buffer *buffer,
218	struct ring_buffer_event *rbe,
219	void *event)
220	{
221	return field->constant;
222	}
223
224	static u64 hist_field_counter(struct hist_field *field,
225	struct tracing_map_elt *elt,
226	struct trace_buffer *buffer,
227	struct ring_buffer_event *rbe,
228	void *event)
229	{
230	return 1;
231	}
232
233	static u64 hist_field_string(struct hist_field *hist_field,
234	struct tracing_map_elt *elt,
235	struct trace_buffer *buffer,
236	struct ring_buffer_event *rbe,
237	void *event)
238	{
239	char addr = (char )(event + hist_field->field->offset);
240
241	return (u64)(unsigned long)addr;
242	}
243
244	static u64 hist_field_dynstring(struct hist_field *hist_field,
245	struct tracing_map_elt *elt,
246	struct trace_buffer *buffer,
247	struct ring_buffer_event *rbe,
248	void *event)
249	{
250	u32 str_item = (u32 )(event + hist_field->field->offset);
251	int str_loc = str_item & 0xffff;
252	char addr = (char )(event + str_loc);
253
254	return (u64)(unsigned long)addr;
255	}
256
257	static u64 hist_field_reldynstring(struct hist_field *hist_field,
258	struct tracing_map_elt *elt,
259	struct trace_buffer *buffer,
260	struct ring_buffer_event *rbe,
261	void *event)
262	{
263	u32 *item = event + hist_field->field->offset;
264	u32 str_item = *item;
265	int str_loc = str_item & 0xffff;
266	char addr = (char )&item[1] + str_loc;
267
268	return (u64)(unsigned long)addr;
269	}
270
271	static u64 hist_field_pstring(struct hist_field *hist_field,
272	struct tracing_map_elt *elt,
273	struct trace_buffer *buffer,
274	struct ring_buffer_event *rbe,
275	void *event)
276	{
277	char addr = (char )(event + hist_field->field->offset);
278
279	return (u64)(unsigned long)*addr;
280	}
281
282	static u64 hist_field_log2(struct hist_field *hist_field,
283	struct tracing_map_elt *elt,
284	struct trace_buffer *buffer,
285	struct ring_buffer_event *rbe,
286	void *event)
287	{
288	struct hist_field *operand = hist_field->operands[0];
289
290	u64 val = hist_fn_call(operand, elt, buffer, rbe, event);
291
292	return order_base_2(val);
293	}
294
295	static u64 hist_field_bucket(struct hist_field *hist_field,
296	struct tracing_map_elt *elt,
297	struct trace_buffer *buffer,
298	struct ring_buffer_event *rbe,
299	void *event)
300	{
301	struct hist_field *operand = hist_field->operands[0];
302	unsigned long buckets = hist_field->buckets;
303
304	u64 val = hist_fn_call(operand, elt, buffer, rbe, event);
305
306	if (WARN_ON_ONCE(!buckets))
307	return val;
308
309	if (val >= LONG_MAX)
310	val = div64_ul(val, buckets);
311	else
312	val = (u64)((unsigned long)val / buckets);
313	return val * buckets;
314	}
315
316	static u64 hist_field_plus(struct hist_field *hist_field,
317	struct tracing_map_elt *elt,
318	struct trace_buffer *buffer,
319	struct ring_buffer_event *rbe,
320	void *event)
321	{
322	struct hist_field *operand1 = hist_field->operands[0];
323	struct hist_field *operand2 = hist_field->operands[1];
324
325	u64 val1 = hist_fn_call(operand1, elt, buffer, rbe, event);
326	u64 val2 = hist_fn_call(operand2, elt, buffer, rbe, event);
327
328	return val1 + val2;
329	}
330
331	static u64 hist_field_minus(struct hist_field *hist_field,
332	struct tracing_map_elt *elt,
333	struct trace_buffer *buffer,
334	struct ring_buffer_event *rbe,
335	void *event)
336	{
337	struct hist_field *operand1 = hist_field->operands[0];
338	struct hist_field *operand2 = hist_field->operands[1];
339
340	u64 val1 = hist_fn_call(operand1, elt, buffer, rbe, event);
341	u64 val2 = hist_fn_call(operand2, elt, buffer, rbe, event);
342
343	return val1 - val2;
344	}
345
346	static u64 hist_field_div(struct hist_field *hist_field,
347	struct tracing_map_elt *elt,
348	struct trace_buffer *buffer,
349	struct ring_buffer_event *rbe,
350	void *event)
351	{
352	struct hist_field *operand1 = hist_field->operands[0];
353	struct hist_field *operand2 = hist_field->operands[1];
354
355	u64 val1 = hist_fn_call(operand1, elt, buffer, rbe, event);
356	u64 val2 = hist_fn_call(operand2, elt, buffer, rbe, event);
357
358	/* Return -1 for the undefined case */
359	if (!val2)
360	return -1;
361
362	/* Use shift if the divisor is a power of 2 */
363	if (!(val2 & (val2 - 1)))
364	return val1 >> __ffs64(val2);
365
366	return div64_u64(val1, val2);
367	}
368
369	static u64 div_by_power_of_two(struct hist_field *hist_field,
370	struct tracing_map_elt *elt,
371	struct trace_buffer *buffer,
372	struct ring_buffer_event *rbe,
373	void *event)
374	{
375	struct hist_field *operand1 = hist_field->operands[0];
376	struct hist_field *operand2 = hist_field->operands[1];
377
378	u64 val1 = hist_fn_call(operand1, elt, buffer, rbe, event);
379
380	return val1 >> __ffs64(operand2->constant);
381	}
382
383	static u64 div_by_not_power_of_two(struct hist_field *hist_field,
384	struct tracing_map_elt *elt,
385	struct trace_buffer *buffer,
386	struct ring_buffer_event *rbe,
387	void *event)
388	{
389	struct hist_field *operand1 = hist_field->operands[0];
390	struct hist_field *operand2 = hist_field->operands[1];
391
392	u64 val1 = hist_fn_call(operand1, elt, buffer, rbe, event);
393
394	return div64_u64(val1, operand2->constant);
395	}
396
397	static u64 div_by_mult_and_shift(struct hist_field *hist_field,
398	struct tracing_map_elt *elt,
399	struct trace_buffer *buffer,
400	struct ring_buffer_event *rbe,
401	void *event)
402	{
403	struct hist_field *operand1 = hist_field->operands[0];
404	struct hist_field *operand2 = hist_field->operands[1];
405
406	u64 val1 = hist_fn_call(operand1, elt, buffer, rbe, event);
407
408	/*
409	* If the divisor is a constant, do a multiplication and shift instead.
410	*
411	* Choose Z = some power of 2. If Y <= Z, then:
412	* X / Y = (X * (Z / Y)) / Z
413	*
414	* (Z / Y) is a constant (mult) which is calculated at parse time, so:
415	* X / Y = (X * mult) / Z
416	*
417	* The division by Z can be replaced by a shift since Z is a power of 2:
418	* X / Y = (X * mult) >> HIST_DIV_SHIFT
419	*
420	* As long, as X < Z the results will not be off by more than 1.
421	*/
422	if (val1 < (1 << HIST_DIV_SHIFT)) {
423	u64 mult = operand2->div_multiplier;
424
425	return (val1 * mult + ((1 << HIST_DIV_SHIFT) - 1)) >> HIST_DIV_SHIFT;
426	}
427
428	return div64_u64(val1, operand2->constant);
429	}
430
431	static u64 hist_field_mult(struct hist_field *hist_field,
432	struct tracing_map_elt *elt,
433	struct trace_buffer *buffer,
434	struct ring_buffer_event *rbe,
435	void *event)
436	{
437	struct hist_field *operand1 = hist_field->operands[0];
438	struct hist_field *operand2 = hist_field->operands[1];
439
440	u64 val1 = hist_fn_call(operand1, elt, buffer, rbe, event);
441	u64 val2 = hist_fn_call(operand2, elt, buffer, rbe, event);
442
443	return val1 * val2;
444	}
445
446	static u64 hist_field_unary_minus(struct hist_field *hist_field,
447	struct tracing_map_elt *elt,
448	struct trace_buffer *buffer,
449	struct ring_buffer_event *rbe,
450	void *event)
451	{
452	struct hist_field *operand = hist_field->operands[0];
453
454	s64 sval = (s64)hist_fn_call(operand, elt, buffer, rbe, event);
455	u64 val = (u64)-sval;
456
457	return val;
458	}
459
460	#define DEFINE_HIST_FIELD_FN(type) \
461	static u64 hist_field_##type(struct hist_field *hist_field, \
462	struct tracing_map_elt *elt, \
463	struct trace_buffer *buffer, \
464	struct ring_buffer_event *rbe, \
465	void *event) \
466	{ \
467	type addr = (type )(event + hist_field->field->offset); \
468	\
469	return (u64)(unsigned long)*addr; \
470	}
471
472	DEFINE_HIST_FIELD_FN(s64);
473	DEFINE_HIST_FIELD_FN(u64);
474	DEFINE_HIST_FIELD_FN(s32);
475	DEFINE_HIST_FIELD_FN(u32);
476	DEFINE_HIST_FIELD_FN(s16);
477	DEFINE_HIST_FIELD_FN(u16);
478	DEFINE_HIST_FIELD_FN(s8);
479	DEFINE_HIST_FIELD_FN(u8);
480
481	#define for_each_hist_field(i, hist_data) \
482	for ((i) = 0; (i) < (hist_data)->n_fields; (i)++)
483
484	#define for_each_hist_val_field(i, hist_data) \
485	for ((i) = 0; (i) < (hist_data)->n_vals; (i)++)
486
487	#define for_each_hist_key_field(i, hist_data) \
488	for ((i) = (hist_data)->n_vals; (i) < (hist_data)->n_fields; (i)++)
489
490	#define HITCOUNT_IDX 0
491	#define HIST_KEY_SIZE_MAX (MAX_FILTER_STR_VAL + HIST_STACKTRACE_SIZE)
492
493	enum hist_field_flags {
494	HIST_FIELD_FL_HITCOUNT = 1 << 0,
495	HIST_FIELD_FL_KEY = 1 << 1,
496	HIST_FIELD_FL_STRING = 1 << 2,
497	HIST_FIELD_FL_HEX = 1 << 3,
498	HIST_FIELD_FL_SYM = 1 << 4,
499	HIST_FIELD_FL_SYM_OFFSET = 1 << 5,
500	HIST_FIELD_FL_EXECNAME = 1 << 6,
501	HIST_FIELD_FL_SYSCALL = 1 << 7,
502	HIST_FIELD_FL_STACKTRACE = 1 << 8,
503	HIST_FIELD_FL_LOG2 = 1 << 9,
504	HIST_FIELD_FL_TIMESTAMP = 1 << 10,
505	HIST_FIELD_FL_TIMESTAMP_USECS = 1 << 11,
506	HIST_FIELD_FL_VAR = 1 << 12,
507	HIST_FIELD_FL_EXPR = 1 << 13,
508	HIST_FIELD_FL_VAR_REF = 1 << 14,
509	HIST_FIELD_FL_CPU = 1 << 15,
510	HIST_FIELD_FL_ALIAS = 1 << 16,
511	HIST_FIELD_FL_BUCKET = 1 << 17,
512	HIST_FIELD_FL_CONST = 1 << 18,
513	HIST_FIELD_FL_PERCENT = 1 << 19,
514	HIST_FIELD_FL_GRAPH = 1 << 20,
515	HIST_FIELD_FL_COMM = 1 << 21,
516	};
517
518	struct var_defs {
519	unsigned int n_vars;
520	char *name[TRACING_MAP_VARS_MAX];
521	char *expr[TRACING_MAP_VARS_MAX];
522	};
523
524	struct hist_trigger_attrs {
525	char *keys_str;
526	char *vals_str;
527	char *sort_key_str;
528	char *name;
529	char *clock;
530	bool pause;
531	bool cont;
532	bool clear;
533	bool ts_in_usecs;
534	bool no_hitcount;
535	unsigned int map_bits;
536
537	char *assignment_str[TRACING_MAP_VARS_MAX];
538	unsigned int n_assignments;
539
540	char *action_str[HIST_ACTIONS_MAX];
541	unsigned int n_actions;
542
543	struct var_defs var_defs;
544	};
545
546	struct field_var {
547	struct hist_field *var;
548	struct hist_field *val;
549	};
550
551	struct field_var_hist {
552	struct hist_trigger_data *hist_data;
553	char *cmd;
554	};
555
556	struct hist_trigger_data {
557	struct hist_field *fields[HIST_FIELDS_MAX];
558	unsigned int n_vals;
559	unsigned int n_keys;
560	unsigned int n_fields;
561	unsigned int n_vars;
562	unsigned int n_var_str;
563	unsigned int key_size;
564	struct tracing_map_sort_key sort_keys[TRACING_MAP_SORT_KEYS_MAX];
565	unsigned int n_sort_keys;
566	struct trace_event_file *event_file;
567	struct hist_trigger_attrs *attrs;
568	struct tracing_map *map;
569	bool enable_timestamps;
570	bool remove;
571	struct hist_field *var_refs[TRACING_MAP_VARS_MAX];
572	unsigned int n_var_refs;
573
574	struct action_data *actions[HIST_ACTIONS_MAX];
575	unsigned int n_actions;
576
577	struct field_var *field_vars[SYNTH_FIELDS_MAX];
578	unsigned int n_field_vars;
579	unsigned int n_field_var_str;
580	struct field_var_hist *field_var_hists[SYNTH_FIELDS_MAX];
581	unsigned int n_field_var_hists;
582
583	struct field_var *save_vars[SYNTH_FIELDS_MAX];
584	unsigned int n_save_vars;
585	unsigned int n_save_var_str;
586	};
587
588	struct action_data;
589
590	typedef void (action_fn_t) (struct hist_trigger_data hist_data,
591	struct tracing_map_elt *elt,
592	struct trace_buffer buffer, void rec,
593	struct ring_buffer_event rbe, void key,
594	struct action_data data, u64 var_ref_vals);
595
596	typedef bool (*check_track_val_fn_t) (u64 track_val, u64 var_val);
597
598	enum handler_id {
599	HANDLER_ONMATCH = 1,
600	HANDLER_ONMAX,
601	HANDLER_ONCHANGE,
602	};
603
604	enum action_id {
605	ACTION_SAVE = 1,
606	ACTION_TRACE,
607	ACTION_SNAPSHOT,
608	};
609
610	struct action_data {
611	enum handler_id handler;
612	enum action_id action;
613	char *action_name;
614	action_fn_t fn;
615
616	unsigned int n_params;
617	char *params[SYNTH_FIELDS_MAX];
618
619	/*
620	* When a histogram trigger is hit, the values of any
621	* references to variables, including variables being passed
622	* as parameters to synthetic events, are collected into a
623	* var_ref_vals array. This var_ref_idx array is an array of
624	* indices into the var_ref_vals array, one for each synthetic
625	* event param, and is passed to the synthetic event
626	* invocation.
627	*/
628	unsigned int var_ref_idx[SYNTH_FIELDS_MAX];
629	struct synth_event *synth_event;
630	bool use_trace_keyword;
631	char *synth_event_name;
632
633	union {
634	struct {
635	char *event;
636	char *event_system;
637	} match_data;
638
639	struct {
640	/*
641	* var_str contains the $-unstripped variable
642	* name referenced by var_ref, and used when
643	* printing the action. Because var_ref
644	* creation is deferred to create_actions(),
645	* we need a per-action way to save it until
646	* then, thus var_str.
647	*/
648	char *var_str;
649
650	/*
651	* var_ref refers to the variable being
652	* tracked e.g onmax($var).
653	*/
654	struct hist_field *var_ref;
655
656	/*
657	* track_var contains the 'invisible' tracking
658	* variable created to keep the current
659	* e.g. max value.
660	*/
661	struct hist_field *track_var;
662
663	check_track_val_fn_t check_val;
664	action_fn_t save_data;
665	} track_data;
666	};
667	};
668
669	struct track_data {
670	u64 track_val;
671	bool updated;
672
673	unsigned int key_len;
674	void *key;
675	struct tracing_map_elt elt;
676
677	struct action_data *action_data;
678	struct hist_trigger_data *hist_data;
679	};
680
681	struct hist_elt_data {
682	char *comm;
683	u64 *var_ref_vals;
684	int n_field_var_str;
685	char *field_var_str[] __counted_by(n_field_var_str);
686	};
687
688	struct snapshot_context {
689	struct tracing_map_elt *elt;
690	void *key;
691	};
692
693	/*
694	* Returns the specific division function to use if the divisor
695	* is constant. This avoids extra branches when the trigger is hit.
696	*/
697	static enum hist_field_fn hist_field_get_div_fn(struct hist_field *divisor)
698	{
699	u64 div = divisor->constant;
700
701	if (!(div & (div - 1)))
702	return HIST_FIELD_FN_DIV_POWER2;
703
704	/* If the divisor is too large, do a regular division */
705	if (div > (1 << HIST_DIV_SHIFT))
706	return HIST_FIELD_FN_DIV_NOT_POWER2;
707
708	divisor->div_multiplier = div64_u64((u64)(1 << HIST_DIV_SHIFT), div);
709	return HIST_FIELD_FN_DIV_MULT_SHIFT;
710	}
711
712	static void track_data_free(struct track_data *track_data)
713	{
714	struct hist_elt_data *elt_data;
715
716	if (!track_data)
717	return;
718
719	kfree(track_data->key);
720
721	elt_data = track_data->elt.private_data;
722	if (elt_data) {
723	kfree(elt_data->comm);
724	kfree(elt_data);
725	}
726
727	kfree(track_data);
728	}
729
730	static struct track_data *track_data_alloc(unsigned int key_len,
731	struct action_data *action_data,
732	struct hist_trigger_data *hist_data)
733	{
734	struct track_data data = kzalloc_obj(data);
735	struct hist_elt_data *elt_data;
736
737	if (!data)
738	return ERR_PTR(-ENOMEM);
739
740	data->key = kzalloc(key_len, GFP_KERNEL);
741	if (!data->key) {
742	track_data_free(data);
743	return ERR_PTR(-ENOMEM);
744	}
745
746	data->key_len = key_len;
747	data->action_data = action_data;
748	data->hist_data = hist_data;
749
750	elt_data = kzalloc_obj(*elt_data);
751	if (!elt_data) {
752	track_data_free(data);
753	return ERR_PTR(-ENOMEM);
754	}
755
756	data->elt.private_data = elt_data;
757
758	elt_data->comm = kzalloc(TASK_COMM_LEN, GFP_KERNEL);
759	if (!elt_data->comm) {
760	track_data_free(data);
761	return ERR_PTR(-ENOMEM);
762	}
763
764	return data;
765	}
766
767	#define HIST_PREFIX "hist:"
768
769	static char *last_cmd;
770	static char last_cmd_loc[MAX_FILTER_STR_VAL];
771
772	static int errpos(char *str)
773	{
774	if (!str \|\| !last_cmd)
775	return 0;
776
777	return err_pos(last_cmd, str);
778	}
779
780	static void last_cmd_set(struct trace_event_file file, char str)
781	{
782	const char system = NULL, name = NULL;
783	struct trace_event_call *call;
784
785	if (!str)
786	return;
787
788	kfree(last_cmd);
789
790	last_cmd = kasprintf(GFP_KERNEL, HIST_PREFIX "%s", str);
791	if (!last_cmd)
792	return;
793
794	if (file) {
795	call = file->event_call;
796	system = call->class->system;
797	if (system) {
798	name = trace_event_name(call);
799	if (!name)
800	system = NULL;
801	}
802	}
803
804	if (system)
805	snprintf(last_cmd_loc, MAX_FILTER_STR_VAL, HIST_PREFIX "%s:%s", system, name);
806	}
807
808	static void hist_err(struct trace_array *tr, u8 err_type, u16 err_pos)
809	{
810	if (!last_cmd)
811	return;
812
813	tracing_log_err(tr, last_cmd_loc, last_cmd, err_text,
814	err_type, err_pos);
815	}
816
817	static void hist_err_clear(void)
818	{
819	if (last_cmd)
820	last_cmd[0] = '\0';
821	last_cmd_loc[0] = '\0';
822	}
823
824	typedef void (synth_probe_func_t) (void __data, u64 *var_ref_vals,
825	unsigned int *var_ref_idx);
826
827	static inline void trace_synth(struct synth_event event, u64 var_ref_vals,
828	unsigned int *var_ref_idx)
829	{
830	struct tracepoint *tp = event->tp;
831
832	if (unlikely(static_key_enabled(&tp->key))) {
833	struct tracepoint_func *probe_func_ptr;
834	synth_probe_func_t probe_func;
835	void *__data;
836
837	if (!(cpu_online(raw_smp_processor_id())))
838	return;
839
840	probe_func_ptr = rcu_dereference_sched((tp)->funcs);
841	if (probe_func_ptr) {
842	do {
843	probe_func = probe_func_ptr->func;
844	__data = probe_func_ptr->data;
845	probe_func(__data, var_ref_vals, var_ref_idx);
846	} while ((++probe_func_ptr)->func);
847	}
848	}
849	}
850
851	static void action_trace(struct hist_trigger_data *hist_data,
852	struct tracing_map_elt *elt,
853	struct trace_buffer buffer, void rec,
854	struct ring_buffer_event rbe, void key,
855	struct action_data data, u64 var_ref_vals)
856	{
857	struct synth_event *event = data->synth_event;
858
859	trace_synth(event, var_ref_vals, data->var_ref_idx);
860	}
861
862	struct hist_var_data {
863	struct list_head list;
864	struct hist_trigger_data *hist_data;
865	};
866
867	static u64 hist_field_timestamp(struct hist_field *hist_field,
868	struct tracing_map_elt *elt,
869	struct trace_buffer *buffer,
870	struct ring_buffer_event *rbe,
871	void *event)
872	{
873	struct hist_trigger_data *hist_data = hist_field->hist_data;
874	struct trace_array *tr = hist_data->event_file->tr;
875
876	u64 ts = tracing_event_time_stamp(buffer, rbe);
877
878	if (hist_data->attrs->ts_in_usecs && trace_clock_in_ns(tr))
879	ts = ns2usecs(ts);
880
881	return ts;
882	}
883
884	static u64 hist_field_cpu(struct hist_field *hist_field,
885	struct tracing_map_elt *elt,
886	struct trace_buffer *buffer,
887	struct ring_buffer_event *rbe,
888	void *event)
889	{
890	int cpu = smp_processor_id();
891
892	return cpu;
893	}
894
895	static u64 hist_field_comm(struct hist_field *hist_field,
896	struct tracing_map_elt *elt,
897	struct trace_buffer *buffer,
898	struct ring_buffer_event *rbe,
899	void *event)
900	{
901	return (u64)(unsigned long)current->comm;
902	}
903
904	/**
905	* check_field_for_var_ref - Check if a VAR_REF field references a variable
906	* @hist_field: The VAR_REF field to check
907	* @var_data: The hist trigger that owns the variable
908	* @var_idx: The trigger variable identifier
909	*
910	* Check the given VAR_REF field to see whether or not it references
911	* the given variable associated with the given trigger.
912	*
913	* Return: The VAR_REF field if it does reference the variable, NULL if not
914	*/
915	static struct hist_field *
916	check_field_for_var_ref(struct hist_field *hist_field,
917	struct hist_trigger_data *var_data,
918	unsigned int var_idx)
919	{
920	WARN_ON(!(hist_field && hist_field->flags & HIST_FIELD_FL_VAR_REF));
921
922	if (hist_field && hist_field->var.idx == var_idx &&
923	hist_field->var.hist_data == var_data)
924	return hist_field;
925
926	return NULL;
927	}
928
929	/**
930	* find_var_ref - Check if a trigger has a reference to a trigger variable
931	* @hist_data: The hist trigger that might have a reference to the variable
932	* @var_data: The hist trigger that owns the variable
933	* @var_idx: The trigger variable identifier
934	*
935	* Check the list of var_refs[] on the first hist trigger to see
936	* whether any of them are references to the variable on the second
937	* trigger.
938	*
939	* Return: The VAR_REF field referencing the variable if so, NULL if not
940	*/
941	static struct hist_field find_var_ref(struct hist_trigger_data hist_data,
942	struct hist_trigger_data *var_data,
943	unsigned int var_idx)
944	{
945	struct hist_field *hist_field;
946	unsigned int i;
947
948	for (i = 0; i < hist_data->n_var_refs; i++) {
949	hist_field = hist_data->var_refs[i];
950	if (check_field_for_var_ref(hist_field, var_data, var_idx))
951	return hist_field;
952	}
953
954	return NULL;
955	}
956
957	/**
958	* find_any_var_ref - Check if there is a reference to a given trigger variable
959	* @hist_data: The hist trigger
960	* @var_idx: The trigger variable identifier
961	*
962	* Check to see whether the given variable is currently referenced by
963	* any other trigger.
964	*
965	* The trigger the variable is defined on is explicitly excluded - the
966	* assumption being that a self-reference doesn't prevent a trigger
967	* from being removed.
968	*
969	* Return: The VAR_REF field referencing the variable if so, NULL if not
970	*/
971	static struct hist_field find_any_var_ref(struct hist_trigger_data hist_data,
972	unsigned int var_idx)
973	{
974	struct trace_array *tr = hist_data->event_file->tr;
975	struct hist_field *found = NULL;
976	struct hist_var_data *var_data;
977
978	list_for_each_entry(var_data, &tr->hist_vars, list) {
979	if (var_data->hist_data == hist_data)
980	continue;
981	found = find_var_ref(var_data->hist_data, hist_data, var_idx);
982	if (found)
983	break;
984	}
985
986	return found;
987	}
988
989	/**
990	* check_var_refs - Check if there is a reference to any of trigger's variables
991	* @hist_data: The hist trigger
992	*
993	* A trigger can define one or more variables. If any one of them is
994	* currently referenced by any other trigger, this function will
995	* determine that.
996	*
997	* Typically used to determine whether or not a trigger can be removed
998	* - if there are any references to a trigger's variables, it cannot.
999	*
1000	* Return: True if there is a reference to any of trigger's variables
1001	*/
1002	static bool check_var_refs(struct hist_trigger_data *hist_data)
1003	{
1004	struct hist_field *field;
1005	bool found = false;
1006	int i;
1007
1008	for_each_hist_field(i, hist_data) {
1009	field = hist_data->fields[i];
1010	if (field && field->flags & HIST_FIELD_FL_VAR) {
1011	if (find_any_var_ref(hist_data, field->var.idx)) {
1012	found = true;
1013	break;
1014	}
1015	}
1016	}
1017
1018	return found;
1019	}
1020
1021	static struct hist_var_data find_hist_vars(struct hist_trigger_data hist_data)
1022	{
1023	struct trace_array *tr = hist_data->event_file->tr;
1024	struct hist_var_data var_data, found = NULL;
1025
1026	list_for_each_entry(var_data, &tr->hist_vars, list) {
1027	if (var_data->hist_data == hist_data) {
1028	found = var_data;
1029	break;
1030	}
1031	}
1032
1033	return found;
1034	}
1035
1036	static bool field_has_hist_vars(struct hist_field *hist_field,
1037	unsigned int level)
1038	{
1039	int i;
1040
1041	if (level > 3)
1042	return false;
1043
1044	if (!hist_field)
1045	return false;
1046
1047	if (hist_field->flags & HIST_FIELD_FL_VAR \|\|
1048	hist_field->flags & HIST_FIELD_FL_VAR_REF)
1049	return true;
1050
1051	for (i = 0; i < HIST_FIELD_OPERANDS_MAX; i++) {
1052	struct hist_field *operand;
1053
1054	operand = hist_field->operands[i];
1055	if (field_has_hist_vars(operand, level + 1))
1056	return true;
1057	}
1058
1059	return false;
1060	}
1061
1062	static bool has_hist_vars(struct hist_trigger_data *hist_data)
1063	{
1064	struct hist_field *hist_field;
1065	int i;
1066
1067	for_each_hist_field(i, hist_data) {
1068	hist_field = hist_data->fields[i];
1069	if (field_has_hist_vars(hist_field, 0))
1070	return true;
1071	}
1072
1073	return false;
1074	}
1075
1076	static int save_hist_vars(struct hist_trigger_data *hist_data)
1077	{
1078	struct trace_array *tr = hist_data->event_file->tr;
1079	struct hist_var_data *var_data;
1080
1081	var_data = find_hist_vars(hist_data);
1082	if (var_data)
1083	return 0;
1084
1085	if (tracing_check_open_get_tr(tr))
1086	return -ENODEV;
1087
1088	var_data = kzalloc_obj(*var_data);
1089	if (!var_data) {
1090	trace_array_put(tr);
1091	return -ENOMEM;
1092	}
1093
1094	var_data->hist_data = hist_data;
1095	list_add(&var_data->list, &tr->hist_vars);
1096
1097	return 0;
1098	}
1099
1100	static void remove_hist_vars(struct hist_trigger_data *hist_data)
1101	{
1102	struct trace_array *tr = hist_data->event_file->tr;
1103	struct hist_var_data *var_data;
1104
1105	var_data = find_hist_vars(hist_data);
1106	if (!var_data)
1107	return;
1108
1109	if (WARN_ON(check_var_refs(hist_data)))
1110	return;
1111
1112	list_del(&var_data->list);
1113
1114	kfree(var_data);
1115
1116	trace_array_put(tr);
1117	}
1118
1119	static struct hist_field find_var_field(struct hist_trigger_data hist_data,
1120	const char *var_name)
1121	{
1122	struct hist_field hist_field, found = NULL;
1123	int i;
1124
1125	for_each_hist_field(i, hist_data) {
1126	hist_field = hist_data->fields[i];
1127	if (hist_field && hist_field->flags & HIST_FIELD_FL_VAR &&
1128	strcmp(hist_field->var.name, var_name) == 0) {
1129	found = hist_field;
1130	break;
1131	}
1132	}
1133
1134	return found;
1135	}
1136
1137	static struct hist_field find_var(struct hist_trigger_data hist_data,
1138	struct trace_event_file *file,
1139	const char *var_name)
1140	{
1141	struct hist_trigger_data *test_data;
1142	struct event_trigger_data *test;
1143	struct hist_field *hist_field;
1144
1145	lockdep_assert_held(&event_mutex);
1146
1147	hist_field = find_var_field(hist_data, var_name);
1148	if (hist_field)
1149	return hist_field;
1150
1151	list_for_each_entry(test, &file->triggers, list) {
1152	if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
1153	test_data = test->private_data;
1154	hist_field = find_var_field(test_data, var_name);
1155	if (hist_field)
1156	return hist_field;
1157	}
1158	}
1159
1160	return NULL;
1161	}
1162
1163	static struct trace_event_file find_var_file(struct trace_array tr,
1164	char *system,
1165	char *event_name,
1166	char *var_name)
1167	{
1168	struct hist_trigger_data *var_hist_data;
1169	struct hist_var_data *var_data;
1170	struct trace_event_file file, found = NULL;
1171
1172	if (system)
1173	return find_event_file(tr, system, event_name);
1174
1175	list_for_each_entry(var_data, &tr->hist_vars, list) {
1176	var_hist_data = var_data->hist_data;
1177	file = var_hist_data->event_file;
1178	if (file == found)
1179	continue;
1180
1181	if (find_var_field(var_hist_data, var_name)) {
1182	if (found) {
1183	hist_err(tr, HIST_ERR_VAR_NOT_UNIQUE, errpos(var_name));
1184	return NULL;
1185	}
1186
1187	found = file;
1188	}
1189	}
1190
1191	return found;
1192	}
1193
1194	static struct hist_field find_file_var(struct trace_event_file file,
1195	const char *var_name)
1196	{
1197	struct hist_trigger_data *test_data;
1198	struct event_trigger_data *test;
1199	struct hist_field *hist_field;
1200
1201	lockdep_assert_held(&event_mutex);
1202
1203	list_for_each_entry(test, &file->triggers, list) {
1204	if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
1205	test_data = test->private_data;
1206	hist_field = find_var_field(test_data, var_name);
1207	if (hist_field)
1208	return hist_field;
1209	}
1210	}
1211
1212	return NULL;
1213	}
1214
1215	static struct hist_field *
1216	find_match_var(struct hist_trigger_data hist_data, char var_name)
1217	{
1218	struct trace_array *tr = hist_data->event_file->tr;
1219	struct hist_field hist_field, found = NULL;
1220	struct trace_event_file *file;
1221	unsigned int i;
1222
1223	for (i = 0; i < hist_data->n_actions; i++) {
1224	struct action_data *data = hist_data->actions[i];
1225
1226	if (data->handler == HANDLER_ONMATCH) {
1227	char *system = data->match_data.event_system;
1228	char *event_name = data->match_data.event;
1229
1230	file = find_var_file(tr, system, event_name, var_name);
1231	if (!file)
1232	continue;
1233	hist_field = find_file_var(file, var_name);
1234	if (hist_field) {
1235	if (found) {
1236	hist_err(tr, HIST_ERR_VAR_NOT_UNIQUE,
1237	errpos(var_name));
1238	return ERR_PTR(-EINVAL);
1239	}
1240
1241	found = hist_field;
1242	}
1243	}
1244	}
1245	return found;
1246	}
1247
1248	static struct hist_field find_event_var(struct hist_trigger_data hist_data,
1249	char *system,
1250	char *event_name,
1251	char *var_name)
1252	{
1253	struct trace_array *tr = hist_data->event_file->tr;
1254	struct hist_field *hist_field = NULL;
1255	struct trace_event_file *file;
1256
1257	if (!system \|\| !event_name) {
1258	hist_field = find_match_var(hist_data, var_name);
1259	if (IS_ERR(hist_field))
1260	return NULL;
1261	if (hist_field)
1262	return hist_field;
1263	}
1264
1265	file = find_var_file(tr, system, event_name, var_name);
1266	if (!file)
1267	return NULL;
1268
1269	hist_field = find_file_var(file, var_name);
1270
1271	return hist_field;
1272	}
1273
1274	static u64 hist_field_var_ref(struct hist_field *hist_field,
1275	struct tracing_map_elt *elt,
1276	struct trace_buffer *buffer,
1277	struct ring_buffer_event *rbe,
1278	void *event)
1279	{
1280	struct hist_elt_data *elt_data;
1281	u64 var_val = 0;
1282
1283	if (WARN_ON_ONCE(!elt))
1284	return var_val;
1285
1286	elt_data = elt->private_data;
1287	var_val = elt_data->var_ref_vals[hist_field->var_ref_idx];
1288
1289	return var_val;
1290	}
1291
1292	static bool resolve_var_refs(struct hist_trigger_data hist_data, void key,
1293	u64 *var_ref_vals, bool self)
1294	{
1295	struct hist_trigger_data *var_data;
1296	struct tracing_map_elt *var_elt;
1297	struct hist_field *hist_field;
1298	unsigned int i, var_idx;
1299	bool resolved = true;
1300	u64 var_val = 0;
1301
1302	for (i = 0; i < hist_data->n_var_refs; i++) {
1303	hist_field = hist_data->var_refs[i];
1304	var_idx = hist_field->var.idx;
1305	var_data = hist_field->var.hist_data;
1306
1307	if (var_data == NULL) {
1308	resolved = false;
1309	break;
1310	}
1311
1312	if ((self && var_data != hist_data) \|\|
1313	(!self && var_data == hist_data))
1314	continue;
1315
1316	var_elt = tracing_map_lookup(var_data->map, key);
1317	if (!var_elt) {
1318	resolved = false;
1319	break;
1320	}
1321
1322	if (!tracing_map_var_set(var_elt, var_idx)) {
1323	resolved = false;
1324	break;
1325	}
1326
1327	if (self \|\| !hist_field->read_once)
1328	var_val = tracing_map_read_var(var_elt, var_idx);
1329	else
1330	var_val = tracing_map_read_var_once(var_elt, var_idx);
1331
1332	var_ref_vals[i] = var_val;
1333	}
1334
1335	return resolved;
1336	}
1337
1338	static const char hist_field_name(struct hist_field field,
1339	unsigned int level)
1340	{
1341	const char *field_name = "";
1342
1343	if (WARN_ON_ONCE(!field))
1344	return field_name;
1345
1346	if (level > 1)
1347	return field_name;
1348
1349	if (field->field)
1350	field_name = field->field->name;
1351	else if (field->flags & HIST_FIELD_FL_LOG2 \|\|
1352	field->flags & HIST_FIELD_FL_ALIAS \|\|
1353	field->flags & HIST_FIELD_FL_BUCKET)
1354	field_name = hist_field_name(field->operands[0], ++level);
1355	else if (field->flags & HIST_FIELD_FL_CPU)
1356	field_name = "common_cpu";
1357	else if (field->flags & HIST_FIELD_FL_COMM)
1358	field_name = "common_comm";
1359	else if (field->flags & HIST_FIELD_FL_EXPR \|\|
1360	field->flags & HIST_FIELD_FL_VAR_REF) {
1361	if (field->system) {
1362	static char full_name[MAX_FILTER_STR_VAL];
1363	static char *fmt;
1364	int len;
1365
1366	fmt = field->flags & HIST_FIELD_FL_VAR_REF ? "%s.%s.$%s" : "%s.%s.%s";
1367
1368	len = snprintf(full_name, sizeof(full_name), fmt,
1369	field->system, field->event_name,
1370	field->name);
1371	if (len < sizeof(full_name))
1372	field_name = full_name;
1373	} else
1374	field_name = field->name;
1375	} else if (field->flags & HIST_FIELD_FL_TIMESTAMP)
1376	field_name = "common_timestamp";
1377	else if (field->flags & HIST_FIELD_FL_STACKTRACE) {
1378	field_name = "common_stacktrace";
1379	} else if (field->flags & HIST_FIELD_FL_HITCOUNT)
1380	field_name = "hitcount";
1381
1382	if (field_name == NULL)
1383	field_name = "";
1384
1385	return field_name;
1386	}
1387
1388	static enum hist_field_fn select_value_fn(int field_size, int field_is_signed)
1389	{
1390	switch (field_size) {
1391	case 8:
1392	if (field_is_signed)
1393	return HIST_FIELD_FN_S64;
1394	else
1395	return HIST_FIELD_FN_U64;
1396	case 4:
1397	if (field_is_signed)
1398	return HIST_FIELD_FN_S32;
1399	else
1400	return HIST_FIELD_FN_U32;
1401	case 2:
1402	if (field_is_signed)
1403	return HIST_FIELD_FN_S16;
1404	else
1405	return HIST_FIELD_FN_U16;
1406	case 1:
1407	if (field_is_signed)
1408	return HIST_FIELD_FN_S8;
1409	else
1410	return HIST_FIELD_FN_U8;
1411	}
1412
1413	return HIST_FIELD_FN_NOP;
1414	}
1415
1416	static int parse_map_size(char *str)
1417	{
1418	unsigned long size, map_bits;
1419	int ret;
1420
1421	ret = kstrtoul(str, 0, &size);
1422	if (ret)
1423	goto out;
1424
1425	map_bits = ilog2(roundup_pow_of_two(size));
1426	if (map_bits < TRACING_MAP_BITS_MIN \|\|
1427	map_bits > TRACING_MAP_BITS_MAX)
1428	ret = -EINVAL;
1429	else
1430	ret = map_bits;
1431	out:
1432	return ret;
1433	}
1434
1435	static void destroy_hist_trigger_attrs(struct hist_trigger_attrs *attrs)
1436	{
1437	unsigned int i;
1438
1439	if (!attrs)
1440	return;
1441
1442	for (i = 0; i < attrs->n_assignments; i++)
1443	kfree(attrs->assignment_str[i]);
1444
1445	for (i = 0; i < attrs->n_actions; i++)
1446	kfree(attrs->action_str[i]);
1447
1448	kfree(attrs->name);
1449	kfree(attrs->sort_key_str);
1450	kfree(attrs->keys_str);
1451	kfree(attrs->vals_str);
1452	kfree(attrs->clock);
1453	kfree(attrs);
1454	}
1455
1456	static int parse_action(char str, struct hist_trigger_attrs attrs)
1457	{
1458	int ret = -EINVAL;
1459
1460	if (attrs->n_actions >= HIST_ACTIONS_MAX)
1461	return ret;
1462
1463	if ((str_has_prefix(str, "onmatch(")) \|\|
1464	(str_has_prefix(str, "onmax(")) \|\|
1465	(str_has_prefix(str, "onchange("))) {
1466	attrs->action_str[attrs->n_actions] = kstrdup(str, GFP_KERNEL);
1467	if (!attrs->action_str[attrs->n_actions]) {
1468	ret = -ENOMEM;
1469	return ret;
1470	}
1471	attrs->n_actions++;
1472	ret = 0;
1473	}
1474	return ret;
1475	}
1476
1477	static int parse_assignment(struct trace_array *tr,
1478	char str, struct hist_trigger_attrs attrs)
1479	{
1480	int len, ret = 0;
1481
1482	if ((len = str_has_prefix(str, "key=")) \|\|
1483	(len = str_has_prefix(str, "keys="))) {
1484	attrs->keys_str = kstrdup(str + len, GFP_KERNEL);
1485	if (!attrs->keys_str) {
1486	ret = -ENOMEM;
1487	goto out;
1488	}
1489	} else if ((len = str_has_prefix(str, "val=")) \|\|
1490	(len = str_has_prefix(str, "vals=")) \|\|
1491	(len = str_has_prefix(str, "values="))) {
1492	attrs->vals_str = kstrdup(str + len, GFP_KERNEL);
1493	if (!attrs->vals_str) {
1494	ret = -ENOMEM;
1495	goto out;
1496	}
1497	} else if ((len = str_has_prefix(str, "sort="))) {
1498	attrs->sort_key_str = kstrdup(str + len, GFP_KERNEL);
1499	if (!attrs->sort_key_str) {
1500	ret = -ENOMEM;
1501	goto out;
1502	}
1503	} else if (str_has_prefix(str, "name=")) {
1504	attrs->name = kstrdup(str, GFP_KERNEL);
1505	if (!attrs->name) {
1506	ret = -ENOMEM;
1507	goto out;
1508	}
1509	} else if ((len = str_has_prefix(str, "clock="))) {
1510	str += len;
1511
1512	str = strstrip(str);
1513	attrs->clock = kstrdup(str, GFP_KERNEL);
1514	if (!attrs->clock) {
1515	ret = -ENOMEM;
1516	goto out;
1517	}
1518	} else if ((len = str_has_prefix(str, "size="))) {
1519	int map_bits = parse_map_size(str + len);
1520
1521	if (map_bits < 0) {
1522	ret = map_bits;
1523	goto out;
1524	}
1525	attrs->map_bits = map_bits;
1526	} else {
1527	char *assignment;
1528
1529	if (attrs->n_assignments == TRACING_MAP_VARS_MAX) {
1530	hist_err(tr, HIST_ERR_TOO_MANY_VARS, errpos(str));
1531	ret = -EINVAL;
1532	goto out;
1533	}
1534
1535	assignment = kstrdup(str, GFP_KERNEL);
1536	if (!assignment) {
1537	ret = -ENOMEM;
1538	goto out;
1539	}
1540
1541	attrs->assignment_str[attrs->n_assignments++] = assignment;
1542	}
1543	out:
1544	return ret;
1545	}
1546
1547	static struct hist_trigger_attrs *
1548	parse_hist_trigger_attrs(struct trace_array tr, char trigger_str)
1549	{
1550	struct hist_trigger_attrs *attrs;
1551	int ret = 0;
1552
1553	attrs = kzalloc_obj(*attrs);
1554	if (!attrs)
1555	return ERR_PTR(-ENOMEM);
1556
1557	while (trigger_str) {
1558	char *str = strsep(&trigger_str, ":");
1559	char *rhs;
1560
1561	rhs = strchr(str, '=');
1562	if (rhs) {
1563	if (!strlen(++rhs)) {
1564	ret = -EINVAL;
1565	hist_err(tr, HIST_ERR_EMPTY_ASSIGNMENT, errpos(str));
1566	goto free;
1567	}
1568	ret = parse_assignment(tr, str, attrs);
1569	if (ret)
1570	goto free;
1571	} else if (strcmp(str, "nohitcount") == 0 \|\|
1572	strcmp(str, "NOHC") == 0)
1573	attrs->no_hitcount = true;
1574	else if (strcmp(str, "pause") == 0)
1575	attrs->pause = true;
1576	else if ((strcmp(str, "cont") == 0) \|\|
1577	(strcmp(str, "continue") == 0))
1578	attrs->cont = true;
1579	else if (strcmp(str, "clear") == 0)
1580	attrs->clear = true;
1581	else {
1582	ret = parse_action(str, attrs);
1583	if (ret)
1584	goto free;
1585	}
1586	}
1587
1588	if (!attrs->keys_str) {
1589	ret = -EINVAL;
1590	goto free;
1591	}
1592
1593	if (!attrs->clock) {
1594	attrs->clock = kstrdup("global", GFP_KERNEL);
1595	if (!attrs->clock) {
1596	ret = -ENOMEM;
1597	goto free;
1598	}
1599	}
1600
1601	return attrs;
1602	free:
1603	destroy_hist_trigger_attrs(attrs);
1604
1605	return ERR_PTR(ret);
1606	}
1607
1608	static inline void save_comm(char comm, struct task_struct task)
1609	{
1610	if (!task->pid) {
1611	strcpy(comm, "<idle>");
1612	return;
1613	}
1614
1615	if (WARN_ON_ONCE(task->pid < 0)) {
1616	strcpy(comm, "<XXX>");
1617	return;
1618	}
1619
1620	strscpy(comm, task->comm, TASK_COMM_LEN);
1621	}
1622
1623	static void hist_elt_data_free(struct hist_elt_data *elt_data)
1624	{
1625	unsigned int i;
1626
1627	for (i = 0; i < elt_data->n_field_var_str; i++)
1628	kfree(elt_data->field_var_str[i]);
1629
1630	kfree(elt_data->comm);
1631	kfree(elt_data);
1632	}
1633
1634	static void hist_trigger_elt_data_free(struct tracing_map_elt *elt)
1635	{
1636	struct hist_elt_data *elt_data = elt->private_data;
1637
1638	hist_elt_data_free(elt_data);
1639	}
1640
1641	static int hist_trigger_elt_data_alloc(struct tracing_map_elt *elt)
1642	{
1643	struct hist_trigger_data *hist_data = elt->map->private_data;
1644	unsigned int size = TASK_COMM_LEN;
1645	struct hist_elt_data *elt_data;
1646	struct hist_field *hist_field;
1647	unsigned int i, n_str;
1648
1649	BUILD_BUG_ON(STR_VAR_LEN_MAX & (sizeof(u64) - 1));
1650
1651	n_str = hist_data->n_field_var_str + hist_data->n_save_var_str +
1652	hist_data->n_var_str;
1653	if (n_str > SYNTH_FIELDS_MAX)
1654	return -EINVAL;
1655
1656	elt_data = kzalloc_flex(*elt_data, field_var_str, n_str);
1657	if (!elt_data)
1658	return -ENOMEM;
1659
1660	elt_data->n_field_var_str = n_str;
1661
1662	for_each_hist_field(i, hist_data) {
1663	hist_field = hist_data->fields[i];
1664
1665	if (hist_field->flags & HIST_FIELD_FL_EXECNAME) {
1666	elt_data->comm = kzalloc(size, GFP_KERNEL);
1667	if (!elt_data->comm) {
1668	kfree(elt_data);
1669	return -ENOMEM;
1670	}
1671	break;
1672	}
1673	}
1674
1675	size = STR_VAR_LEN_MAX;
1676
1677	for (i = 0; i < n_str; i++) {
1678	elt_data->field_var_str[i] = kzalloc(size, GFP_KERNEL);
1679	if (!elt_data->field_var_str[i]) {
1680	hist_elt_data_free(elt_data);
1681	return -ENOMEM;
1682	}
1683	}
1684
1685	elt->private_data = elt_data;
1686
1687	return 0;
1688	}
1689
1690	static void hist_trigger_elt_data_init(struct tracing_map_elt *elt)
1691	{
1692	struct hist_elt_data *elt_data = elt->private_data;
1693
1694	if (elt_data->comm)
1695	save_comm(elt_data->comm, current);
1696	}
1697
1698	static const struct tracing_map_ops hist_trigger_elt_data_ops = {
1699	.elt_alloc = hist_trigger_elt_data_alloc,
1700	.elt_free = hist_trigger_elt_data_free,
1701	.elt_init = hist_trigger_elt_data_init,
1702	};
1703
1704	static const char get_hist_field_flags(struct hist_field hist_field)
1705	{
1706	const char *flags_str = NULL;
1707
1708	if (hist_field->flags & HIST_FIELD_FL_HEX)
1709	flags_str = "hex";
1710	else if (hist_field->flags & HIST_FIELD_FL_SYM)
1711	flags_str = "sym";
1712	else if (hist_field->flags & HIST_FIELD_FL_SYM_OFFSET)
1713	flags_str = "sym-offset";
1714	else if (hist_field->flags & HIST_FIELD_FL_EXECNAME)
1715	flags_str = "execname";
1716	else if (hist_field->flags & HIST_FIELD_FL_SYSCALL)
1717	flags_str = "syscall";
1718	else if (hist_field->flags & HIST_FIELD_FL_LOG2)
1719	flags_str = "log2";
1720	else if (hist_field->flags & HIST_FIELD_FL_BUCKET)
1721	flags_str = "buckets";
1722	else if (hist_field->flags & HIST_FIELD_FL_TIMESTAMP_USECS)
1723	flags_str = "usecs";
1724	else if (hist_field->flags & HIST_FIELD_FL_PERCENT)
1725	flags_str = "percent";
1726	else if (hist_field->flags & HIST_FIELD_FL_GRAPH)
1727	flags_str = "graph";
1728	else if (hist_field->flags & HIST_FIELD_FL_STACKTRACE)
1729	flags_str = "stacktrace";
1730
1731	return flags_str;
1732	}
1733
1734	static bool expr_field_str(struct hist_field field, struct seq_buf s)
1735	{
1736	const char *field_name;
1737
1738	if (field->flags & HIST_FIELD_FL_VAR_REF) {
1739	if (!field->system)
1740	seq_buf_putc(s, '$');
1741	} else if (field->flags & HIST_FIELD_FL_CONST)
1742	seq_buf_printf(s, "%llu", field->constant);
1743
1744	field_name = hist_field_name(field, 0);
1745	if (!field_name)
1746	return false;
1747
1748	seq_buf_puts(s, field_name);
1749
1750	if (field->flags && !(field->flags & HIST_FIELD_FL_VAR_REF)) {
1751	const char *flags_str = get_hist_field_flags(field);
1752
1753	if (flags_str)
1754	seq_buf_printf(s, ".%s", flags_str);
1755	}
1756
1757	return !seq_buf_has_overflowed(s);
1758	}
1759
1760	static char expr_str(struct hist_field field, unsigned int level)
1761	{
1762	char *expr __free(kfree) = NULL;
1763	struct seq_buf s;
1764
1765	if (level > 1)
1766	return ERR_PTR(-EINVAL);
1767
1768	expr = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
1769	if (!expr)
1770	return ERR_PTR(-ENOMEM);
1771
1772	seq_buf_init(&s, expr, MAX_FILTER_STR_VAL);
1773
1774	if (!field->operands[0]) {
1775	if (!expr_field_str(field, &s))
1776	return ERR_PTR(-E2BIG);
1777
1778	return_ptr(expr);
1779	}
1780
1781	if (field->operator == FIELD_OP_UNARY_MINUS) {
1782	char *subexpr;
1783
1784	subexpr = expr_str(field->operands[0], ++level);
1785	if (IS_ERR(subexpr))
1786	return subexpr;
1787
1788	seq_buf_printf(&s, "-(%s)", subexpr);
1789	kfree(subexpr);
1790
1791	if (seq_buf_has_overflowed(&s))
1792	return ERR_PTR(-E2BIG);
1793
1794	return_ptr(expr);
1795	}
1796
1797	if (!expr_field_str(field->operands[0], &s))
1798	return ERR_PTR(-E2BIG);
1799
1800	switch (field->operator) {
1801	case FIELD_OP_MINUS:
1802	seq_buf_putc(&s, '-');
1803	break;
1804	case FIELD_OP_PLUS:
1805	seq_buf_putc(&s, '+');
1806	break;
1807	case FIELD_OP_DIV:
1808	seq_buf_putc(&s, '/');
1809	break;
1810	case FIELD_OP_MULT:
1811	seq_buf_putc(&s, '*');
1812	break;
1813	default:
1814	return ERR_PTR(-EINVAL);
1815	}
1816
1817	if (seq_buf_has_overflowed(&s) \|\|
1818	!expr_field_str(field->operands[1], &s))
1819	return ERR_PTR(-E2BIG);
1820
1821	return_ptr(expr);
1822	}
1823
1824	/*
1825	* If field_op != FIELD_OP_NONE, *sep points to the root operator
1826	* of the expression tree to be evaluated.
1827	*/
1828	static int contains_operator(char str, char *sep)
1829	{
1830	enum field_op_id field_op = FIELD_OP_NONE;
1831	char minus_op, plus_op, div_op, mult_op;
1832
1833
1834	/*
1835	* Report the last occurrence of the operators first, so that the
1836	* expression is evaluated left to right. This is important since
1837	* subtraction and division are not associative.
1838	*
1839	* e.g
1840	* 64/8/4/2 is 1, i.e 64/8/4/2 = ((64/8)/4)/2
1841	* 14-7-5-2 is 0, i.e 14-7-5-2 = ((14-7)-5)-2
1842	*/
1843
1844	/*
1845	* First, find lower precedence addition and subtraction
1846	* since the expression will be evaluated recursively.
1847	*/
1848	minus_op = strrchr(str, '-');
1849	if (minus_op) {
1850	/*
1851	* Unary minus is not supported in sub-expressions. If
1852	* present, it is always the next root operator.
1853	*/
1854	if (minus_op == str) {
1855	field_op = FIELD_OP_UNARY_MINUS;
1856	goto out;
1857	}
1858
1859	field_op = FIELD_OP_MINUS;
1860	}
1861
1862	plus_op = strrchr(str, '+');
1863	if (plus_op \|\| minus_op) {
1864	/*
1865	* For operators of the same precedence use to rightmost as the
1866	* root, so that the expression is evaluated left to right.
1867	*/
1868	if (plus_op > minus_op)
1869	field_op = FIELD_OP_PLUS;
1870	goto out;
1871	}
1872
1873	/*
1874	* Multiplication and division have higher precedence than addition and
1875	* subtraction.
1876	*/
1877	div_op = strrchr(str, '/');
1878	if (div_op)
1879	field_op = FIELD_OP_DIV;
1880
1881	mult_op = strrchr(str, '*');
1882	/*
1883	* For operators of the same precedence use to rightmost as the
1884	* root, so that the expression is evaluated left to right.
1885	*/
1886	if (mult_op > div_op)
1887	field_op = FIELD_OP_MULT;
1888
1889	out:
1890	if (sep) {
1891	switch (field_op) {
1892	case FIELD_OP_UNARY_MINUS:
1893	case FIELD_OP_MINUS:
1894	*sep = minus_op;
1895	break;
1896	case FIELD_OP_PLUS:
1897	*sep = plus_op;
1898	break;
1899	case FIELD_OP_DIV:
1900	*sep = div_op;
1901	break;
1902	case FIELD_OP_MULT:
1903	*sep = mult_op;
1904	break;
1905	case FIELD_OP_NONE:
1906	default:
1907	*sep = NULL;
1908	break;
1909	}
1910	}
1911
1912	return field_op;
1913	}
1914
1915	static void __destroy_hist_field(struct hist_field *hist_field)
1916	{
1917	kfree(hist_field->var.name);
1918	kfree(hist_field->name);
1919
1920	/* Can likely be a const */
1921	kfree_const(hist_field->type);
1922
1923	kfree(hist_field->system);
1924	kfree(hist_field->event_name);
1925
1926	kfree(hist_field);
1927	}
1928
1929	static void destroy_hist_field(struct hist_field *hist_field,
1930	unsigned int level)
1931	{
1932	unsigned int i;
1933
1934	if (level > 3)
1935	return;
1936
1937	if (!hist_field)
1938	return;
1939
1940	if (hist_field->flags & HIST_FIELD_FL_VAR_REF)
1941	return; /* var refs will be destroyed separately */
1942
1943	for (i = 0; i < HIST_FIELD_OPERANDS_MAX; i++)
1944	destroy_hist_field(hist_field->operands[i], level + 1);
1945
1946	__destroy_hist_field(hist_field);
1947	}
1948
1949	static struct hist_field create_hist_field(struct hist_trigger_data hist_data,
1950	struct ftrace_event_field *field,
1951	unsigned long flags,
1952	char *var_name)
1953	{
1954	struct hist_field *hist_field;
1955
1956	if (field && is_function_field(field))
1957	return NULL;
1958
1959	hist_field = kzalloc_obj(struct hist_field);
1960	if (!hist_field)
1961	return NULL;
1962
1963	hist_field->hist_data = hist_data;
1964
1965	if (flags & HIST_FIELD_FL_EXPR \|\| flags & HIST_FIELD_FL_ALIAS)
1966	goto out; /* caller will populate */
1967
1968	if (flags & HIST_FIELD_FL_VAR_REF) {
1969	hist_field->fn_num = HIST_FIELD_FN_VAR_REF;
1970	goto out;
1971	}
1972
1973	if (flags & HIST_FIELD_FL_HITCOUNT) {
1974	hist_field->fn_num = HIST_FIELD_FN_COUNTER;
1975	hist_field->size = sizeof(u64);
1976	hist_field->type = "u64";
1977	goto out;
1978	}
1979
1980	if (flags & HIST_FIELD_FL_CONST) {
1981	hist_field->fn_num = HIST_FIELD_FN_CONST;
1982	hist_field->size = sizeof(u64);
1983	hist_field->type = "u64";
1984	goto out;
1985	}
1986
1987	if (flags & HIST_FIELD_FL_STACKTRACE) {
1988	if (field)
1989	hist_field->fn_num = HIST_FIELD_FN_STACK;
1990	else
1991	hist_field->fn_num = HIST_FIELD_FN_NOP;
1992	hist_field->size = HIST_STACKTRACE_SIZE;
1993	hist_field->type = kstrdup_const("unsigned long[]", GFP_KERNEL);
1994	if (!hist_field->type)
1995	goto free;
1996	goto out;
1997	}
1998
1999	if (flags & (HIST_FIELD_FL_LOG2 \| HIST_FIELD_FL_BUCKET)) {
2000	unsigned long fl = flags & ~(HIST_FIELD_FL_LOG2 \| HIST_FIELD_FL_BUCKET);
2001	hist_field->fn_num = flags & HIST_FIELD_FL_LOG2 ? HIST_FIELD_FN_LOG2 :
2002	HIST_FIELD_FN_BUCKET;
2003	hist_field->operands[0] = create_hist_field(hist_data, field, fl, NULL);
2004	if (!hist_field->operands[0])
2005	goto free;
2006	hist_field->size = hist_field->operands[0]->size;
2007	hist_field->type = kstrdup_const(hist_field->operands[0]->type, GFP_KERNEL);
2008	if (!hist_field->type)
2009	goto free;
2010	goto out;
2011	}
2012
2013	if (flags & HIST_FIELD_FL_TIMESTAMP) {
2014	hist_field->fn_num = HIST_FIELD_FN_TIMESTAMP;
2015	hist_field->size = sizeof(u64);
2016	hist_field->type = "u64";
2017	goto out;
2018	}
2019
2020	if (flags & HIST_FIELD_FL_CPU) {
2021	hist_field->fn_num = HIST_FIELD_FN_CPU;
2022	hist_field->size = sizeof(int);
2023	hist_field->type = "unsigned int";
2024	goto out;
2025	}
2026
2027	if (flags & HIST_FIELD_FL_COMM) {
2028	hist_field->fn_num = HIST_FIELD_FN_COMM;
2029	hist_field->size = MAX_FILTER_STR_VAL;
2030	hist_field->type = "char[]";
2031	goto out;
2032	}
2033
2034	if (WARN_ON_ONCE(!field))
2035	goto out;
2036
2037	/* Pointers to strings are just pointers and dangerous to dereference */
2038	if (is_string_field(field) &&
2039	(field->filter_type != FILTER_PTR_STRING)) {
2040	flags \|= HIST_FIELD_FL_STRING;
2041
2042	hist_field->size = MAX_FILTER_STR_VAL;
2043	hist_field->type = kstrdup_const(field->type, GFP_KERNEL);
2044	if (!hist_field->type)
2045	goto free;
2046
2047	if (field->filter_type == FILTER_STATIC_STRING) {
2048	hist_field->fn_num = HIST_FIELD_FN_STRING;
2049	hist_field->size = field->size;
2050	} else if (field->filter_type == FILTER_DYN_STRING) {
2051	hist_field->fn_num = HIST_FIELD_FN_DYNSTRING;
2052	} else if (field->filter_type == FILTER_RDYN_STRING)
2053	hist_field->fn_num = HIST_FIELD_FN_RELDYNSTRING;
2054	else
2055	hist_field->fn_num = HIST_FIELD_FN_PSTRING;
2056	} else if (field->filter_type == FILTER_STACKTRACE) {
2057	flags \|= HIST_FIELD_FL_STACKTRACE;
2058
2059	hist_field->size = MAX_FILTER_STR_VAL;
2060	hist_field->type = kstrdup_const(field->type, GFP_KERNEL);
2061	if (!hist_field->type)
2062	goto free;
2063
2064	hist_field->fn_num = HIST_FIELD_FN_STACK;
2065	} else {
2066	hist_field->size = field->size;
2067	hist_field->is_signed = field->is_signed;
2068	hist_field->type = kstrdup_const(field->type, GFP_KERNEL);
2069	if (!hist_field->type)
2070	goto free;
2071
2072	hist_field->fn_num = select_value_fn(field->size,
2073	field->is_signed);
2074	if (hist_field->fn_num == HIST_FIELD_FN_NOP) {
2075	destroy_hist_field(hist_field, 0);
2076	return NULL;
2077	}
2078	}
2079	out:
2080	hist_field->field = field;
2081	hist_field->flags = flags;
2082
2083	if (var_name) {
2084	hist_field->var.name = kstrdup(var_name, GFP_KERNEL);
2085	if (!hist_field->var.name)
2086	goto free;
2087	}
2088
2089	return hist_field;
2090	free:
2091	destroy_hist_field(hist_field, 0);
2092	return NULL;
2093	}
2094
2095	static void destroy_hist_fields(struct hist_trigger_data *hist_data)
2096	{
2097	unsigned int i;
2098
2099	for (i = 0; i < HIST_FIELDS_MAX; i++) {
2100	if (hist_data->fields[i]) {
2101	destroy_hist_field(hist_data->fields[i], 0);
2102	hist_data->fields[i] = NULL;
2103	}
2104	}
2105
2106	for (i = 0; i < hist_data->n_var_refs; i++) {
2107	WARN_ON(!(hist_data->var_refs[i]->flags & HIST_FIELD_FL_VAR_REF));
2108	__destroy_hist_field(hist_data->var_refs[i]);
2109	hist_data->var_refs[i] = NULL;
2110	}
2111	}
2112
2113	static int init_var_ref(struct hist_field *ref_field,
2114	struct hist_field *var_field,
2115	char system, char event_name)
2116	{
2117	int err = 0;
2118
2119	ref_field->var.idx = var_field->var.idx;
2120	ref_field->var.hist_data = var_field->hist_data;
2121	ref_field->size = var_field->size;
2122	ref_field->is_signed = var_field->is_signed;
2123	ref_field->flags \|= var_field->flags &
2124	(HIST_FIELD_FL_TIMESTAMP \| HIST_FIELD_FL_TIMESTAMP_USECS \|
2125	HIST_FIELD_FL_STACKTRACE);
2126
2127	if (system) {
2128	ref_field->system = kstrdup(system, GFP_KERNEL);
2129	if (!ref_field->system)
2130	return -ENOMEM;
2131	}
2132
2133	if (event_name) {
2134	ref_field->event_name = kstrdup(event_name, GFP_KERNEL);
2135	if (!ref_field->event_name) {
2136	err = -ENOMEM;
2137	goto free;
2138	}
2139	}
2140
2141	if (var_field->var.name) {
2142	ref_field->name = kstrdup(var_field->var.name, GFP_KERNEL);
2143	if (!ref_field->name) {
2144	err = -ENOMEM;
2145	goto free;
2146	}
2147	} else if (var_field->name) {
2148	ref_field->name = kstrdup(var_field->name, GFP_KERNEL);
2149	if (!ref_field->name) {
2150	err = -ENOMEM;
2151	goto free;
2152	}
2153	}
2154
2155	ref_field->type = kstrdup_const(var_field->type, GFP_KERNEL);
2156	if (!ref_field->type) {
2157	err = -ENOMEM;
2158	goto free;
2159	}
2160	out:
2161	return err;
2162	free:
2163	kfree(ref_field->system);
2164	ref_field->system = NULL;
2165	kfree(ref_field->event_name);
2166	ref_field->event_name = NULL;
2167	kfree(ref_field->name);
2168	ref_field->name = NULL;
2169
2170	goto out;
2171	}
2172
2173	static int find_var_ref_idx(struct hist_trigger_data *hist_data,
2174	struct hist_field *var_field)
2175	{
2176	struct hist_field *ref_field;
2177	int i;
2178
2179	for (i = 0; i < hist_data->n_var_refs; i++) {
2180	ref_field = hist_data->var_refs[i];
2181	if (ref_field->var.idx == var_field->var.idx &&
2182	ref_field->var.hist_data == var_field->hist_data)
2183	return i;
2184	}
2185
2186	return -ENOENT;
2187	}
2188
2189	/**
2190	* create_var_ref - Create a variable reference and attach it to trigger
2191	* @hist_data: The trigger that will be referencing the variable
2192	* @var_field: The VAR field to create a reference to
2193	* @system: The optional system string
2194	* @event_name: The optional event_name string
2195	*
2196	* Given a variable hist_field, create a VAR_REF hist_field that
2197	* represents a reference to it.
2198	*
2199	* This function also adds the reference to the trigger that
2200	* now references the variable.
2201	*
2202	* Return: The VAR_REF field if successful, NULL if not
2203	*/
2204	static struct hist_field create_var_ref(struct hist_trigger_data hist_data,
2205	struct hist_field *var_field,
2206	char system, char event_name)
2207	{
2208	unsigned long flags = HIST_FIELD_FL_VAR_REF;
2209	struct hist_field *ref_field;
2210	int i;
2211
2212	/* Check if the variable already exists */
2213	for (i = 0; i < hist_data->n_var_refs; i++) {
2214	ref_field = hist_data->var_refs[i];
2215	if (ref_field->var.idx == var_field->var.idx &&
2216	ref_field->var.hist_data == var_field->hist_data)
2217	return ref_field;
2218	}
2219	/* Sanity check to avoid out-of-bound write on 'hist_data->var_refs' */
2220	if (hist_data->n_var_refs >= TRACING_MAP_VARS_MAX)
2221	return NULL;
2222	ref_field = create_hist_field(var_field->hist_data, NULL, flags, NULL);
2223	if (ref_field) {
2224	if (init_var_ref(ref_field, var_field, system, event_name)) {
2225	__destroy_hist_field(ref_field);
2226	return NULL;
2227	}
2228
2229	hist_data->var_refs[hist_data->n_var_refs] = ref_field;
2230	ref_field->var_ref_idx = hist_data->n_var_refs++;
2231	}
2232
2233	return ref_field;
2234	}
2235
2236	static bool is_var_ref(char *var_name)
2237	{
2238	if (!var_name \|\| strlen(var_name) < 2 \|\| var_name[0] != '$')
2239	return false;
2240
2241	return true;
2242	}
2243
2244	static char field_name_from_var(struct hist_trigger_data hist_data,
2245	char *var_name)
2246	{
2247	char name, field;
2248	unsigned int i;
2249
2250	for (i = 0; i < hist_data->attrs->var_defs.n_vars; i++) {
2251	name = hist_data->attrs->var_defs.name[i];
2252
2253	if (strcmp(var_name, name) == 0) {
2254	field = hist_data->attrs->var_defs.expr[i];
2255	if (contains_operator(field, NULL) \|\| is_var_ref(field))
2256	continue;
2257	return field;
2258	}
2259	}
2260
2261	return NULL;
2262	}
2263
2264	static char local_field_var_ref(struct hist_trigger_data hist_data,
2265	char system, char event_name,
2266	char *var_name)
2267	{
2268	struct trace_event_call *call;
2269
2270	if (system && event_name) {
2271	call = hist_data->event_file->event_call;
2272
2273	if (strcmp(system, call->class->system) != 0)
2274	return NULL;
2275
2276	if (strcmp(event_name, trace_event_name(call)) != 0)
2277	return NULL;
2278	}
2279
2280	if (!!system != !!event_name)
2281	return NULL;
2282
2283	if (!is_var_ref(var_name))
2284	return NULL;
2285
2286	var_name++;
2287
2288	return field_name_from_var(hist_data, var_name);
2289	}
2290
2291	static struct hist_field parse_var_ref(struct hist_trigger_data hist_data,
2292	char system, char event_name,
2293	char *var_name)
2294	{
2295	struct hist_field var_field = NULL, ref_field = NULL;
2296	struct trace_array *tr = hist_data->event_file->tr;
2297
2298	if (!is_var_ref(var_name))
2299	return NULL;
2300
2301	var_name++;
2302
2303	var_field = find_event_var(hist_data, system, event_name, var_name);
2304	if (var_field)
2305	ref_field = create_var_ref(hist_data, var_field,
2306	system, event_name);
2307
2308	if (!ref_field)
2309	hist_err(tr, HIST_ERR_VAR_NOT_FOUND, errpos(var_name));
2310
2311	return ref_field;
2312	}
2313
2314	static struct ftrace_event_field *
2315	parse_field(struct hist_trigger_data hist_data, struct trace_event_file file,
2316	char field_str, unsigned long flags, unsigned long *buckets)
2317	{
2318	struct ftrace_event_field *field = NULL;
2319	char field_name, modifier, *str;
2320	struct trace_array *tr = file->tr;
2321	bool stack_modifier = false;
2322
2323	modifier = str = kstrdup(field_str, GFP_KERNEL);
2324	if (!modifier)
2325	return ERR_PTR(-ENOMEM);
2326
2327	field_name = strsep(&modifier, ".");
2328	if (modifier) {
2329	if (strcmp(modifier, "hex") == 0)
2330	*flags \|= HIST_FIELD_FL_HEX;
2331	else if (strcmp(modifier, "sym") == 0)
2332	*flags \|= HIST_FIELD_FL_SYM;
2333	/*
2334	* 'sym-offset' occurrences in the trigger string are modified
2335	* to 'symXoffset' to simplify arithmetic expression parsing.
2336	*/
2337	else if (strcmp(modifier, "symXoffset") == 0)
2338	*flags \|= HIST_FIELD_FL_SYM_OFFSET;
2339	else if ((strcmp(modifier, "execname") == 0) &&
2340	(strcmp(field_name, "common_pid") == 0))
2341	*flags \|= HIST_FIELD_FL_EXECNAME;
2342	else if (strcmp(modifier, "syscall") == 0)
2343	*flags \|= HIST_FIELD_FL_SYSCALL;
2344	else if (strcmp(modifier, "stacktrace") == 0) {
2345	*flags \|= HIST_FIELD_FL_STACKTRACE;
2346	stack_modifier = true;
2347	} else if (strcmp(modifier, "log2") == 0)
2348	*flags \|= HIST_FIELD_FL_LOG2;
2349	else if (strcmp(modifier, "usecs") == 0)
2350	*flags \|= HIST_FIELD_FL_TIMESTAMP_USECS;
2351	else if (strncmp(modifier, "bucket", 6) == 0) {
2352	int ret;
2353
2354	modifier += 6;
2355
2356	if (*modifier == 's')
2357	modifier++;
2358	if (*modifier != '=')
2359	goto error;
2360	modifier++;
2361	ret = kstrtoul(modifier, 0, buckets);
2362	if (ret \|\| !(*buckets))
2363	goto error;
2364	*flags \|= HIST_FIELD_FL_BUCKET;
2365	} else if (strncmp(modifier, "percent", 7) == 0) {
2366	if (*flags & (HIST_FIELD_FL_VAR \| HIST_FIELD_FL_KEY))
2367	goto error;
2368	*flags \|= HIST_FIELD_FL_PERCENT;
2369	} else if (strncmp(modifier, "graph", 5) == 0) {
2370	if (*flags & (HIST_FIELD_FL_VAR \| HIST_FIELD_FL_KEY))
2371	goto error;
2372	*flags \|= HIST_FIELD_FL_GRAPH;
2373	} else {
2374	error:
2375	hist_err(tr, HIST_ERR_BAD_FIELD_MODIFIER, errpos(modifier));
2376	field = ERR_PTR(-EINVAL);
2377	goto out;
2378	}
2379	}
2380
2381	if (strcmp(field_name, "common_timestamp") == 0) {
2382	*flags \|= HIST_FIELD_FL_TIMESTAMP;
2383	hist_data->enable_timestamps = true;
2384	if (*flags & HIST_FIELD_FL_TIMESTAMP_USECS)
2385	hist_data->attrs->ts_in_usecs = true;
2386	} else if (strcmp(field_name, "common_stacktrace") == 0) {
2387	*flags \|= HIST_FIELD_FL_STACKTRACE;
2388	} else if (strcmp(field_name, "common_cpu") == 0) {
2389	*flags \|= HIST_FIELD_FL_CPU;
2390	} else if (strcmp(field_name, "common_comm") == 0) {
2391	*flags \|= HIST_FIELD_FL_COMM \| HIST_FIELD_FL_STRING;
2392	} else if (strcmp(field_name, "hitcount") == 0)
2393	*flags \|= HIST_FIELD_FL_HITCOUNT;
2394	else {
2395	field = trace_find_event_field(file->event_call, field_name);
2396	if (!field \|\| !field->size) {
2397	/*
2398	* For backward compatibility, if field_name
2399	* was "cpu" or "stacktrace", then we treat this
2400	* the same as common_cpu and common_stacktrace
2401	* respectively. This also works for "CPU", and
2402	* "STACKTRACE".
2403	*/
2404	if (field && field->filter_type == FILTER_CPU) {
2405	*flags \|= HIST_FIELD_FL_CPU;
2406	} else if (field && field->filter_type == FILTER_STACKTRACE) {
2407	*flags \|= HIST_FIELD_FL_STACKTRACE;
2408	field = NULL;
2409	} else if (field && field->filter_type == FILTER_COMM) {
2410	*flags \|= HIST_FIELD_FL_COMM \| HIST_FIELD_FL_STRING;
2411	} else {
2412	hist_err(tr, HIST_ERR_FIELD_NOT_FOUND,
2413	errpos(field_name));
2414	field = ERR_PTR(-EINVAL);
2415	goto out;
2416	}
2417	}
2418	}
2419
2420	if (stack_modifier &&
2421	(!field \|\| field->filter_type != FILTER_STACKTRACE)) {
2422	hist_err(tr, HIST_ERR_BAD_FIELD_MODIFIER, errpos(field_str));
2423	field = ERR_PTR(-EINVAL);
2424	}
2425	out:
2426	kfree(str);
2427
2428	return field;
2429	}
2430
2431	static struct hist_field create_alias(struct hist_trigger_data hist_data,
2432	struct hist_field *var_ref,
2433	char *var_name)
2434	{
2435	struct hist_field *alias = NULL;
2436	unsigned long flags = HIST_FIELD_FL_ALIAS \| HIST_FIELD_FL_VAR;
2437
2438	alias = create_hist_field(hist_data, NULL, flags, var_name);
2439	if (!alias)
2440	return NULL;
2441
2442	alias->fn_num = var_ref->fn_num;
2443	alias->operands[0] = var_ref;
2444
2445	if (init_var_ref(alias, var_ref, var_ref->system, var_ref->event_name)) {
2446	destroy_hist_field(alias, 0);
2447	return NULL;
2448	}
2449
2450	alias->var_ref_idx = var_ref->var_ref_idx;
2451
2452	return alias;
2453	}
2454
2455	static struct hist_field parse_const(struct hist_trigger_data hist_data,
2456	char str, char var_name,
2457	unsigned long *flags)
2458	{
2459	struct trace_array *tr = hist_data->event_file->tr;
2460	struct hist_field *field = NULL;
2461	u64 constant;
2462
2463	if (kstrtoull(str, 0, &constant)) {
2464	hist_err(tr, HIST_ERR_EXPECT_NUMBER, errpos(str));
2465	return NULL;
2466	}
2467
2468	*flags \|= HIST_FIELD_FL_CONST;
2469	field = create_hist_field(hist_data, NULL, *flags, var_name);
2470	if (!field)
2471	return NULL;
2472
2473	field->constant = constant;
2474
2475	return field;
2476	}
2477
2478	static struct hist_field parse_atom(struct hist_trigger_data hist_data,
2479	struct trace_event_file file, char str,
2480	unsigned long flags, char var_name)
2481	{
2482	char s, ref_system = NULL, ref_event = NULL, ref_var = str;
2483	struct ftrace_event_field *field = NULL;
2484	struct hist_field *hist_field = NULL;
2485	unsigned long buckets = 0;
2486	int ret = 0;
2487
2488	if (isdigit(str[0])) {
2489	hist_field = parse_const(hist_data, str, var_name, flags);
2490	if (!hist_field) {
2491	ret = -EINVAL;
2492	goto out;
2493	}
2494	return hist_field;
2495	}
2496
2497	s = strchr(str, '.');
2498	if (s) {
2499	s = strchr(++s, '.');
2500	if (s) {
2501	ref_system = strsep(&str, ".");
2502	if (!str) {
2503	ret = -EINVAL;
2504	goto out;
2505	}
2506	ref_event = strsep(&str, ".");
2507	if (!str) {
2508	ret = -EINVAL;
2509	goto out;
2510	}
2511	ref_var = str;
2512	}
2513	}
2514
2515	s = local_field_var_ref(hist_data, ref_system, ref_event, ref_var);
2516	if (!s) {
2517	hist_field = parse_var_ref(hist_data, ref_system,
2518	ref_event, ref_var);
2519	if (hist_field) {
2520	if (var_name) {
2521	hist_field = create_alias(hist_data, hist_field, var_name);
2522	if (!hist_field) {
2523	ret = -ENOMEM;
2524	goto out;
2525	}
2526	}
2527	return hist_field;
2528	}
2529	} else
2530	str = s;
2531
2532	field = parse_field(hist_data, file, str, flags, &buckets);
2533	if (IS_ERR(field)) {
2534	ret = PTR_ERR(field);
2535	goto out;
2536	}
2537
2538	hist_field = create_hist_field(hist_data, field, *flags, var_name);
2539	if (!hist_field) {
2540	ret = -ENOMEM;
2541	goto out;
2542	}
2543	hist_field->buckets = buckets;
2544
2545	return hist_field;
2546	out:
2547	return ERR_PTR(ret);
2548	}
2549
2550	static struct hist_field parse_expr(struct hist_trigger_data hist_data,
2551	struct trace_event_file *file,
2552	char *str, unsigned long flags,
2553	char var_name, unsigned int n_subexprs);
2554
2555	static struct hist_field parse_unary(struct hist_trigger_data hist_data,
2556	struct trace_event_file *file,
2557	char *str, unsigned long flags,
2558	char var_name, unsigned int n_subexprs)
2559	{
2560	struct hist_field operand1, expr = NULL;
2561	unsigned long operand_flags;
2562	int ret = 0;
2563	char *s;
2564
2565	/* Unary minus operator, increment n_subexprs */
2566	++*n_subexprs;
2567
2568	/* we support only -(xxx) i.e. explicit parens required */
2569
2570	if (*n_subexprs > 3) {
2571	hist_err(file->tr, HIST_ERR_TOO_MANY_SUBEXPR, errpos(str));
2572	ret = -EINVAL;
2573	goto free;
2574	}
2575
2576	str++; /* skip leading '-' */
2577
2578	s = strchr(str, '(');
2579	if (s)
2580	str++;
2581	else {
2582	ret = -EINVAL;
2583	goto free;
2584	}
2585
2586	s = strrchr(str, ')');
2587	if (s) {
2588	/* unary minus not supported in sub-expressions */
2589	if (*(s+1) != '\0') {
2590	hist_err(file->tr, HIST_ERR_UNARY_MINUS_SUBEXPR,
2591	errpos(str));
2592	ret = -EINVAL;
2593	goto free;
2594	}
2595	*s = '\0';
2596	}
2597	else {
2598	ret = -EINVAL; /* no closing ')' */
2599	goto free;
2600	}
2601
2602	flags \|= HIST_FIELD_FL_EXPR;
2603	expr = create_hist_field(hist_data, NULL, flags, var_name);
2604	if (!expr) {
2605	ret = -ENOMEM;
2606	goto free;
2607	}
2608
2609	operand_flags = 0;
2610	operand1 = parse_expr(hist_data, file, str, operand_flags, NULL, n_subexprs);
2611	if (IS_ERR(operand1)) {
2612	ret = PTR_ERR(operand1);
2613	goto free;
2614	}
2615	if (operand1->flags & HIST_FIELD_FL_STRING) {
2616	/* String type can not be the operand of unary operator. */
2617	hist_err(file->tr, HIST_ERR_INVALID_STR_OPERAND, errpos(str));
2618	destroy_hist_field(operand1, 0);
2619	ret = -EINVAL;
2620	goto free;
2621	}
2622
2623	expr->flags \|= operand1->flags &
2624	(HIST_FIELD_FL_TIMESTAMP \| HIST_FIELD_FL_TIMESTAMP_USECS);
2625	expr->fn_num = HIST_FIELD_FN_UMINUS;
2626	expr->operands[0] = operand1;
2627	expr->size = operand1->size;
2628	expr->is_signed = operand1->is_signed;
2629	expr->operator = FIELD_OP_UNARY_MINUS;
2630	expr->name = expr_str(expr, 0);
2631	if (IS_ERR(expr->name)) {
2632	ret = PTR_ERR(expr->name);
2633	expr->name = NULL;
2634	goto free;
2635	}
2636	expr->type = kstrdup_const(operand1->type, GFP_KERNEL);
2637	if (!expr->type) {
2638	ret = -ENOMEM;
2639	goto free;
2640	}
2641
2642	return expr;
2643	free:
2644	destroy_hist_field(expr, 0);
2645	return ERR_PTR(ret);
2646	}
2647
2648	/*
2649	* If the operands are var refs, return pointers the
2650	* variable(s) referenced in var1 and var2, else NULL.
2651	*/
2652	static int check_expr_operands(struct trace_array *tr,
2653	struct hist_field *operand1,
2654	struct hist_field *operand2,
2655	struct hist_field **var1,
2656	struct hist_field **var2)
2657	{
2658	unsigned long operand1_flags = operand1->flags;
2659	unsigned long operand2_flags = operand2->flags;
2660
2661	if ((operand1_flags & HIST_FIELD_FL_VAR_REF) \|\|
2662	(operand1_flags & HIST_FIELD_FL_ALIAS)) {
2663	struct hist_field *var;
2664
2665	var = find_var_field(operand1->var.hist_data, operand1->name);
2666	if (!var)
2667	return -EINVAL;
2668	operand1_flags = var->flags;
2669	*var1 = var;
2670	}
2671
2672	if ((operand2_flags & HIST_FIELD_FL_VAR_REF) \|\|
2673	(operand2_flags & HIST_FIELD_FL_ALIAS)) {
2674	struct hist_field *var;
2675
2676	var = find_var_field(operand2->var.hist_data, operand2->name);
2677	if (!var)
2678	return -EINVAL;
2679	operand2_flags = var->flags;
2680	*var2 = var;
2681	}
2682
2683	if ((operand1_flags & HIST_FIELD_FL_TIMESTAMP_USECS) !=
2684	(operand2_flags & HIST_FIELD_FL_TIMESTAMP_USECS)) {
2685	hist_err(tr, HIST_ERR_TIMESTAMP_MISMATCH, 0);
2686	return -EINVAL;
2687	}
2688
2689	return 0;
2690	}
2691
2692	static struct hist_field parse_expr(struct hist_trigger_data hist_data,
2693	struct trace_event_file *file,
2694	char *str, unsigned long flags,
2695	char var_name, unsigned int n_subexprs)
2696	{
2697	struct hist_field operand1 = NULL, operand2 = NULL, *expr = NULL;
2698	struct hist_field var1 = NULL, var2 = NULL;
2699	unsigned long operand_flags, operand2_flags;
2700	int field_op, ret = -EINVAL;
2701	char sep, operand1_str;
2702	enum hist_field_fn op_fn;
2703	bool combine_consts;
2704
2705	if (*n_subexprs > 3) {
2706	hist_err(file->tr, HIST_ERR_TOO_MANY_SUBEXPR, errpos(str));
2707	return ERR_PTR(-EINVAL);
2708	}
2709
2710	field_op = contains_operator(str, &sep);
2711
2712	if (field_op == FIELD_OP_NONE)
2713	return parse_atom(hist_data, file, str, &flags, var_name);
2714
2715	if (field_op == FIELD_OP_UNARY_MINUS)
2716	return parse_unary(hist_data, file, str, flags, var_name, n_subexprs);
2717
2718	/* Binary operator found, increment n_subexprs */
2719	++*n_subexprs;
2720
2721	/* Split the expression string at the root operator */
2722	if (!sep)
2723	return ERR_PTR(-EINVAL);
2724
2725	*sep = '\0';
2726	operand1_str = str;
2727	str = sep+1;
2728
2729	/* Binary operator requires both operands */
2730	if (operand1_str == '\0' \|\| str == '\0')
2731	return ERR_PTR(-EINVAL);
2732
2733	operand_flags = 0;
2734
2735	/* LHS of string is an expression e.g. a+b in a+b+c */
2736	operand1 = parse_expr(hist_data, file, operand1_str, operand_flags, NULL, n_subexprs);
2737	if (IS_ERR(operand1))
2738	return ERR_CAST(operand1);
2739
2740	if (operand1->flags & HIST_FIELD_FL_STRING) {
2741	hist_err(file->tr, HIST_ERR_INVALID_STR_OPERAND, errpos(operand1_str));
2742	ret = -EINVAL;
2743	goto free_op1;
2744	}
2745
2746	/* RHS of string is another expression e.g. c in a+b+c */
2747	operand_flags = 0;
2748	operand2 = parse_expr(hist_data, file, str, operand_flags, NULL, n_subexprs);
2749	if (IS_ERR(operand2)) {
2750	ret = PTR_ERR(operand2);
2751	goto free_op1;
2752	}
2753	if (operand2->flags & HIST_FIELD_FL_STRING) {
2754	hist_err(file->tr, HIST_ERR_INVALID_STR_OPERAND, errpos(str));
2755	ret = -EINVAL;
2756	goto free_operands;
2757	}
2758
2759	switch (field_op) {
2760	case FIELD_OP_MINUS:
2761	op_fn = HIST_FIELD_FN_MINUS;
2762	break;
2763	case FIELD_OP_PLUS:
2764	op_fn = HIST_FIELD_FN_PLUS;
2765	break;
2766	case FIELD_OP_DIV:
2767	op_fn = HIST_FIELD_FN_DIV;
2768	break;
2769	case FIELD_OP_MULT:
2770	op_fn = HIST_FIELD_FN_MULT;
2771	break;
2772	default:
2773	ret = -EINVAL;
2774	goto free_operands;
2775	}
2776
2777	ret = check_expr_operands(file->tr, operand1, operand2, &var1, &var2);
2778	if (ret)
2779	goto free_operands;
2780
2781	operand_flags = var1 ? var1->flags : operand1->flags;
2782	operand2_flags = var2 ? var2->flags : operand2->flags;
2783
2784	/*
2785	* If both operands are constant, the expression can be
2786	* collapsed to a single constant.
2787	*/
2788	combine_consts = operand_flags & operand2_flags & HIST_FIELD_FL_CONST;
2789
2790	flags \|= combine_consts ? HIST_FIELD_FL_CONST : HIST_FIELD_FL_EXPR;
2791
2792	flags \|= operand1->flags &
2793	(HIST_FIELD_FL_TIMESTAMP \| HIST_FIELD_FL_TIMESTAMP_USECS);
2794
2795	expr = create_hist_field(hist_data, NULL, flags, var_name);
2796	if (!expr) {
2797	ret = -ENOMEM;
2798	goto free_operands;
2799	}
2800
2801	operand1->read_once = true;
2802	operand2->read_once = true;
2803
2804	/* The operands are now owned and free'd by 'expr' */
2805	expr->operands[0] = operand1;
2806	expr->operands[1] = operand2;
2807
2808	if (field_op == FIELD_OP_DIV &&
2809	operand2_flags & HIST_FIELD_FL_CONST) {
2810	u64 divisor = var2 ? var2->constant : operand2->constant;
2811
2812	if (!divisor) {
2813	hist_err(file->tr, HIST_ERR_DIVISION_BY_ZERO, errpos(str));
2814	ret = -EDOM;
2815	goto free_expr;
2816	}
2817
2818	/*
2819	* Copy the divisor here so we don't have to look it up
2820	* later if this is a var ref
2821	*/
2822	operand2->constant = divisor;
2823	op_fn = hist_field_get_div_fn(operand2);
2824	}
2825
2826	expr->fn_num = op_fn;
2827
2828	if (combine_consts) {
2829	if (var1)
2830	expr->operands[0] = var1;
2831	if (var2)
2832	expr->operands[1] = var2;
2833
2834	expr->constant = hist_fn_call(expr, NULL, NULL, NULL, NULL);
2835	expr->fn_num = HIST_FIELD_FN_CONST;
2836
2837	expr->operands[0] = NULL;
2838	expr->operands[1] = NULL;
2839
2840	/*
2841	* var refs won't be destroyed immediately
2842	* See: destroy_hist_field()
2843	*/
2844	destroy_hist_field(operand2, 0);
2845	destroy_hist_field(operand1, 0);
2846
2847	expr->name = expr_str(expr, 0);
2848	if (IS_ERR(expr->name)) {
2849	ret = PTR_ERR(expr->name);
2850	expr->name = NULL;
2851	goto free_expr;
2852	}
2853	} else {
2854	/* The operand sizes should be the same, so just pick one */
2855	expr->size = operand1->size;
2856	expr->is_signed = operand1->is_signed;
2857
2858	expr->operator = field_op;
2859	expr->type = kstrdup_const(operand1->type, GFP_KERNEL);
2860	if (!expr->type) {
2861	ret = -ENOMEM;
2862	goto free_expr;
2863	}
2864
2865	expr->name = expr_str(expr, 0);
2866	if (IS_ERR(expr->name)) {
2867	ret = PTR_ERR(expr->name);
2868	expr->name = NULL;
2869	goto free_expr;
2870	}
2871	}
2872
2873	return expr;
2874
2875	free_operands:
2876	destroy_hist_field(operand2, 0);
2877	free_op1:
2878	destroy_hist_field(operand1, 0);
2879	return ERR_PTR(ret);
2880
2881	free_expr:
2882	destroy_hist_field(expr, 0);
2883	return ERR_PTR(ret);
2884	}
2885
2886	static char find_trigger_filter(struct hist_trigger_data hist_data,
2887	struct trace_event_file *file)
2888	{
2889	struct event_trigger_data *test;
2890
2891	lockdep_assert_held(&event_mutex);
2892
2893	list_for_each_entry(test, &file->triggers, list) {
2894	if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
2895	if (test->private_data == hist_data)
2896	return test->filter_str;
2897	}
2898	}
2899
2900	return NULL;
2901	}
2902
2903	static struct event_command trigger_hist_cmd;
2904	static int event_hist_trigger_parse(struct event_command *cmd_ops,
2905	struct trace_event_file *file,
2906	char glob, char cmd,
2907	char *param_and_filter);
2908
2909	static bool compatible_keys(struct hist_trigger_data *target_hist_data,
2910	struct hist_trigger_data *hist_data,
2911	unsigned int n_keys)
2912	{
2913	struct hist_field target_hist_field, hist_field;
2914	unsigned int n, i, j;
2915
2916	if (hist_data->n_fields - hist_data->n_vals != n_keys)
2917	return false;
2918
2919	i = hist_data->n_vals;
2920	j = target_hist_data->n_vals;
2921
2922	for (n = 0; n < n_keys; n++) {
2923	hist_field = hist_data->fields[i + n];
2924	target_hist_field = target_hist_data->fields[j + n];
2925
2926	if (strcmp(hist_field->type, target_hist_field->type) != 0)
2927	return false;
2928	if (hist_field->size != target_hist_field->size)
2929	return false;
2930	if (hist_field->is_signed != target_hist_field->is_signed)
2931	return false;
2932	}
2933
2934	return true;
2935	}
2936
2937	static struct hist_trigger_data *
2938	find_compatible_hist(struct hist_trigger_data *target_hist_data,
2939	struct trace_event_file *file)
2940	{
2941	struct hist_trigger_data *hist_data;
2942	struct event_trigger_data *test;
2943	unsigned int n_keys;
2944
2945	lockdep_assert_held(&event_mutex);
2946
2947	n_keys = target_hist_data->n_fields - target_hist_data->n_vals;
2948
2949	list_for_each_entry(test, &file->triggers, list) {
2950	if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
2951	hist_data = test->private_data;
2952
2953	if (compatible_keys(target_hist_data, hist_data, n_keys))
2954	return hist_data;
2955	}
2956	}
2957
2958	return NULL;
2959	}
2960
2961	static struct trace_event_file event_file(struct trace_array tr,
2962	char system, char event_name)
2963	{
2964	struct trace_event_file *file;
2965
2966	file = __find_event_file(tr, system, event_name);
2967	if (!file)
2968	return ERR_PTR(-EINVAL);
2969
2970	return file;
2971	}
2972
2973	static struct hist_field *
2974	find_synthetic_field_var(struct hist_trigger_data *target_hist_data,
2975	char system, char event_name, char *field_name)
2976	{
2977	struct hist_field *event_var;
2978	char *synthetic_name;
2979	struct seq_buf s;
2980
2981	synthetic_name = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
2982	if (!synthetic_name)
2983	return ERR_PTR(-ENOMEM);
2984
2985	seq_buf_init(&s, synthetic_name, MAX_FILTER_STR_VAL);
2986	seq_buf_printf(&s, "synthetic_%s", field_name);
2987
2988	/* Terminate synthetic_name with a NUL. */
2989	seq_buf_str(&s);
2990
2991	if (seq_buf_has_overflowed(&s)) {
2992	kfree(synthetic_name);
2993	return ERR_PTR(-E2BIG);
2994	}
2995
2996	event_var = find_event_var(target_hist_data, system, event_name, synthetic_name);
2997
2998	kfree(synthetic_name);
2999
3000	return event_var;
3001	}
3002
3003	/**
3004	* create_field_var_hist - Automatically create a histogram and var for a field
3005	* @target_hist_data: The target hist trigger
3006	* @subsys_name: Optional subsystem name
3007	* @event_name: Optional event name
3008	* @field_name: The name of the field (and the resulting variable)
3009	*
3010	* Hist trigger actions fetch data from variables, not directly from
3011	* events. However, for convenience, users are allowed to directly
3012	* specify an event field in an action, which will be automatically
3013	* converted into a variable on their behalf.
3014	*
3015	* If a user specifies a field on an event that isn't the event the
3016	* histogram currently being defined (the target event histogram), the
3017	* only way that can be accomplished is if a new hist trigger is
3018	* created and the field variable defined on that.
3019	*
3020	* This function creates a new histogram compatible with the target
3021	* event (meaning a histogram with the same key as the target
3022	* histogram), and creates a variable for the specified field, but
3023	* with 'synthetic_' prepended to the variable name in order to avoid
3024	* collision with normal field variables.
3025	*
3026	* Return: The variable created for the field.
3027	*/
3028	static struct hist_field *
3029	create_field_var_hist(struct hist_trigger_data *target_hist_data,
3030	char subsys_name, char event_name, char *field_name)
3031	{
3032	struct trace_array *tr = target_hist_data->event_file->tr;
3033	struct hist_trigger_data *hist_data;
3034	unsigned int i, n, first = true;
3035	struct field_var_hist *var_hist;
3036	struct trace_event_file *file;
3037	struct hist_field *key_field;
3038	struct hist_field *event_var;
3039	char *saved_filter;
3040	struct seq_buf s;
3041	char *cmd;
3042	int ret;
3043
3044	if (target_hist_data->n_field_var_hists >= SYNTH_FIELDS_MAX) {
3045	hist_err(tr, HIST_ERR_TOO_MANY_FIELD_VARS, errpos(field_name));
3046	return ERR_PTR(-EINVAL);
3047	}
3048
3049	file = event_file(tr, subsys_name, event_name);
3050
3051	if (IS_ERR(file)) {
3052	hist_err(tr, HIST_ERR_EVENT_FILE_NOT_FOUND, errpos(field_name));
3053	ret = PTR_ERR(file);
3054	return ERR_PTR(ret);
3055	}
3056
3057	/*
3058	* Look for a histogram compatible with target. We'll use the
3059	* found histogram specification to create a new matching
3060	* histogram with our variable on it. target_hist_data is not
3061	* yet a registered histogram so we can't use that.
3062	*/
3063	hist_data = find_compatible_hist(target_hist_data, file);
3064	if (!hist_data) {
3065	hist_err(tr, HIST_ERR_HIST_NOT_FOUND, errpos(field_name));
3066	return ERR_PTR(-EINVAL);
3067	}
3068
3069	/* See if a synthetic field variable has already been created */
3070	event_var = find_synthetic_field_var(target_hist_data, subsys_name,
3071	event_name, field_name);
3072	if (!IS_ERR_OR_NULL(event_var))
3073	return event_var;
3074
3075	var_hist = kzalloc_obj(*var_hist);
3076	if (!var_hist)
3077	return ERR_PTR(-ENOMEM);
3078
3079	cmd = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
3080	if (!cmd) {
3081	kfree(var_hist);
3082	return ERR_PTR(-ENOMEM);
3083	}
3084
3085	seq_buf_init(&s, cmd, MAX_FILTER_STR_VAL);
3086
3087	/* Use the same keys as the compatible histogram */
3088	seq_buf_puts(&s, "keys=");
3089
3090	for_each_hist_key_field(i, hist_data) {
3091	key_field = hist_data->fields[i];
3092	if (!first)
3093	seq_buf_putc(&s, ',');
3094	expr_field_str(key_field, &s);
3095	first = false;
3096	}
3097
3098	/* Create the synthetic field variable specification */
3099	seq_buf_printf(&s, ":synthetic_%s=%s", field_name, field_name);
3100
3101	/* Use the same filter as the compatible histogram */
3102	saved_filter = find_trigger_filter(hist_data, file);
3103	if (saved_filter)
3104	seq_buf_printf(&s, " if %s", saved_filter);
3105
3106	/* Terminate cmd with a NUL. */
3107	seq_buf_str(&s);
3108
3109	if (seq_buf_has_overflowed(&s)) {
3110	kfree(cmd);
3111	kfree(var_hist);
3112	return ERR_PTR(-E2BIG);
3113	}
3114
3115	var_hist->cmd = kstrdup(cmd, GFP_KERNEL);
3116	if (!var_hist->cmd) {
3117	kfree(cmd);
3118	kfree(var_hist);
3119	return ERR_PTR(-ENOMEM);
3120	}
3121
3122	/* Save the compatible histogram information */
3123	var_hist->hist_data = hist_data;
3124
3125	/* Create the new histogram with our variable */
3126	ret = event_hist_trigger_parse(&trigger_hist_cmd, file,
3127	"", "hist", cmd);
3128	if (ret) {
3129	kfree(cmd);
3130	kfree(var_hist->cmd);
3131	kfree(var_hist);
3132	hist_err(tr, HIST_ERR_HIST_CREATE_FAIL, errpos(field_name));
3133	return ERR_PTR(ret);
3134	}
3135
3136	kfree(cmd);
3137
3138	/* If we can't find the variable, something went wrong */
3139	event_var = find_synthetic_field_var(target_hist_data, subsys_name,
3140	event_name, field_name);
3141	if (IS_ERR_OR_NULL(event_var)) {
3142	kfree(var_hist->cmd);
3143	kfree(var_hist);
3144	hist_err(tr, HIST_ERR_SYNTH_VAR_NOT_FOUND, errpos(field_name));
3145	return ERR_PTR(-EINVAL);
3146	}
3147
3148	n = target_hist_data->n_field_var_hists;
3149	target_hist_data->field_var_hists[n] = var_hist;
3150	target_hist_data->n_field_var_hists++;
3151
3152	return event_var;
3153	}
3154
3155	static struct hist_field *
3156	find_target_event_var(struct hist_trigger_data *hist_data,
3157	char subsys_name, char event_name, char *var_name)
3158	{
3159	struct trace_event_file *file = hist_data->event_file;
3160	struct hist_field *hist_field = NULL;
3161
3162	if (subsys_name) {
3163	struct trace_event_call *call;
3164
3165	if (!event_name)
3166	return NULL;
3167
3168	call = file->event_call;
3169
3170	if (strcmp(subsys_name, call->class->system) != 0)
3171	return NULL;
3172
3173	if (strcmp(event_name, trace_event_name(call)) != 0)
3174	return NULL;
3175	}
3176
3177	hist_field = find_var_field(hist_data, var_name);
3178
3179	return hist_field;
3180	}
3181
3182	static inline void __update_field_vars(struct tracing_map_elt *elt,
3183	struct trace_buffer *buffer,
3184	struct ring_buffer_event *rbe,
3185	void *rec,
3186	struct field_var **field_vars,
3187	unsigned int n_field_vars,
3188	unsigned int field_var_str_start)
3189	{
3190	struct hist_elt_data *elt_data = elt->private_data;
3191	unsigned int i, j, var_idx;
3192	u64 var_val;
3193
3194	/* Make sure stacktrace can fit in the string variable length */
3195	BUILD_BUG_ON((HIST_STACKTRACE_DEPTH + 1) * sizeof(long) > STR_VAR_LEN_MAX);
3196
3197	for (i = 0, j = field_var_str_start; i < n_field_vars; i++) {
3198	struct field_var *field_var = field_vars[i];
3199	struct hist_field *var = field_var->var;
3200	struct hist_field *val = field_var->val;
3201
3202	var_val = hist_fn_call(val, elt, buffer, rbe, rec);
3203	var_idx = var->var.idx;
3204
3205	if (val->flags & (HIST_FIELD_FL_STRING \|
3206	HIST_FIELD_FL_STACKTRACE)) {
3207	char *str = elt_data->field_var_str[j++];
3208	char val_str = (char )(uintptr_t)var_val;
3209	unsigned int size;
3210
3211	if (val->flags & HIST_FIELD_FL_STRING) {
3212	size = min(val->size, STR_VAR_LEN_MAX);
3213	strscpy(str, val_str, size);
3214	} else {
3215	char *stack_start = str + sizeof(unsigned long);
3216	int e;
3217
3218	e = stack_trace_save((void *)stack_start,
3219	HIST_STACKTRACE_DEPTH,
3220	HIST_STACKTRACE_SKIP);
3221	if (e < HIST_STACKTRACE_DEPTH - 1)
3222	((unsigned long *)stack_start)[e] = 0;
3223	((unsigned long )str) = e;
3224	}
3225	var_val = (u64)(uintptr_t)str;
3226	}
3227	tracing_map_set_var(elt, var_idx, var_val);
3228	}
3229	}
3230
3231	static void update_field_vars(struct hist_trigger_data *hist_data,
3232	struct tracing_map_elt *elt,
3233	struct trace_buffer *buffer,
3234	struct ring_buffer_event *rbe,
3235	void *rec)
3236	{
3237	__update_field_vars(elt, buffer, rbe, rec, hist_data->field_vars,
3238	hist_data->n_field_vars, 0);
3239	}
3240
3241	static void save_track_data_vars(struct hist_trigger_data *hist_data,
3242	struct tracing_map_elt *elt,
3243	struct trace_buffer buffer, void rec,
3244	struct ring_buffer_event rbe, void key,
3245	struct action_data data, u64 var_ref_vals)
3246	{
3247	__update_field_vars(elt, buffer, rbe, rec, hist_data->save_vars,
3248	hist_data->n_save_vars, hist_data->n_field_var_str);
3249	}
3250
3251	static struct hist_field create_var(struct hist_trigger_data hist_data,
3252	struct trace_event_file *file,
3253	char name, int size, const char type)
3254	{
3255	struct hist_field *var;
3256	int idx;
3257
3258	if (find_var(hist_data, file, name) && !hist_data->remove) {
3259	var = ERR_PTR(-EINVAL);
3260	goto out;
3261	}
3262
3263	var = kzalloc_obj(struct hist_field);
3264	if (!var) {
3265	var = ERR_PTR(-ENOMEM);
3266	goto out;
3267	}
3268
3269	idx = tracing_map_add_var(hist_data->map);
3270	if (idx < 0) {
3271	kfree(var);
3272	var = ERR_PTR(-EINVAL);
3273	goto out;
3274	}
3275
3276	var->flags = HIST_FIELD_FL_VAR;
3277	var->var.idx = idx;
3278	var->var.hist_data = var->hist_data = hist_data;
3279	var->size = size;
3280	var->var.name = kstrdup(name, GFP_KERNEL);
3281	var->type = kstrdup_const(type, GFP_KERNEL);
3282	if (!var->var.name \|\| !var->type) {
3283	kfree_const(var->type);
3284	kfree(var->var.name);
3285	kfree(var);
3286	var = ERR_PTR(-ENOMEM);
3287	}
3288	out:
3289	return var;
3290	}
3291
3292	static struct field_var create_field_var(struct hist_trigger_data hist_data,
3293	struct trace_event_file *file,
3294	char *field_name)
3295	{
3296	struct hist_field val = NULL, var = NULL;
3297	unsigned long flags = HIST_FIELD_FL_VAR;
3298	struct trace_array *tr = file->tr;
3299	struct field_var *field_var;
3300	int ret = 0;
3301
3302	if (hist_data->n_field_vars >= SYNTH_FIELDS_MAX) {
3303	hist_err(tr, HIST_ERR_TOO_MANY_FIELD_VARS, errpos(field_name));
3304	ret = -EINVAL;
3305	goto err;
3306	}
3307
3308	val = parse_atom(hist_data, file, field_name, &flags, NULL);
3309	if (IS_ERR(val)) {
3310	hist_err(tr, HIST_ERR_FIELD_VAR_PARSE_FAIL, errpos(field_name));
3311	ret = PTR_ERR(val);
3312	goto err;
3313	}
3314
3315	var = create_var(hist_data, file, field_name, val->size, val->type);
3316	if (IS_ERR(var)) {
3317	hist_err(tr, HIST_ERR_VAR_CREATE_FIND_FAIL, errpos(field_name));
3318	destroy_hist_field(val, 0);
3319	ret = PTR_ERR(var);
3320	goto err;
3321	}
3322	var->flags \|= val->flags & HIST_FIELD_FL_STACKTRACE;
3323
3324	field_var = kzalloc_obj(struct field_var);
3325	if (!field_var) {
3326	destroy_hist_field(val, 0);
3327	kfree_const(var->type);
3328	kfree(var->var.name);
3329	kfree(var);
3330	ret = -ENOMEM;
3331	goto err;
3332	}
3333
3334	field_var->var = var;
3335	field_var->val = val;
3336	out:
3337	return field_var;
3338	err:
3339	field_var = ERR_PTR(ret);
3340	goto out;
3341	}
3342
3343	/**
3344	* create_target_field_var - Automatically create a variable for a field
3345	* @target_hist_data: The target hist trigger
3346	* @subsys_name: Optional subsystem name
3347	* @event_name: Optional event name
3348	* @var_name: The name of the field (and the resulting variable)
3349	*
3350	* Hist trigger actions fetch data from variables, not directly from
3351	* events. However, for convenience, users are allowed to directly
3352	* specify an event field in an action, which will be automatically
3353	* converted into a variable on their behalf.
3354	*
3355	* This function creates a field variable with the name var_name on
3356	* the hist trigger currently being defined on the target event. If
3357	* subsys_name and event_name are specified, this function simply
3358	* verifies that they do in fact match the target event subsystem and
3359	* event name.
3360	*
3361	* Return: The variable created for the field.
3362	*/
3363	static struct field_var *
3364	create_target_field_var(struct hist_trigger_data *target_hist_data,
3365	char subsys_name, char event_name, char *var_name)
3366	{
3367	struct trace_event_file *file = target_hist_data->event_file;
3368
3369	if (subsys_name) {
3370	struct trace_event_call *call;
3371
3372	if (!event_name)
3373	return NULL;
3374
3375	call = file->event_call;
3376
3377	if (strcmp(subsys_name, call->class->system) != 0)
3378	return NULL;
3379
3380	if (strcmp(event_name, trace_event_name(call)) != 0)
3381	return NULL;
3382	}
3383
3384	return create_field_var(target_hist_data, file, var_name);
3385	}
3386
3387	static bool check_track_val_max(u64 track_val, u64 var_val)
3388	{
3389	if (var_val <= track_val)
3390	return false;
3391
3392	return true;
3393	}
3394
3395	static bool check_track_val_changed(u64 track_val, u64 var_val)
3396	{
3397	if (var_val == track_val)
3398	return false;
3399
3400	return true;
3401	}
3402
3403	static u64 get_track_val(struct hist_trigger_data *hist_data,
3404	struct tracing_map_elt *elt,
3405	struct action_data *data)
3406	{
3407	unsigned int track_var_idx = data->track_data.track_var->var.idx;
3408	u64 track_val;
3409
3410	track_val = tracing_map_read_var(elt, track_var_idx);
3411
3412	return track_val;
3413	}
3414
3415	static void save_track_val(struct hist_trigger_data *hist_data,
3416	struct tracing_map_elt *elt,
3417	struct action_data *data, u64 var_val)
3418	{
3419	unsigned int track_var_idx = data->track_data.track_var->var.idx;
3420
3421	tracing_map_set_var(elt, track_var_idx, var_val);
3422	}
3423
3424	static void save_track_data(struct hist_trigger_data *hist_data,
3425	struct tracing_map_elt *elt,
3426	struct trace_buffer buffer, void rec,
3427	struct ring_buffer_event rbe, void key,
3428	struct action_data data, u64 var_ref_vals)
3429	{
3430	if (data->track_data.save_data)
3431	data->track_data.save_data(hist_data, elt, buffer, rec, rbe,
3432	key, data, var_ref_vals);
3433	}
3434
3435	static bool check_track_val(struct tracing_map_elt *elt,
3436	struct action_data *data,
3437	u64 var_val)
3438	{
3439	struct hist_trigger_data *hist_data;
3440	u64 track_val;
3441
3442	hist_data = data->track_data.track_var->hist_data;
3443	track_val = get_track_val(hist_data, elt, data);
3444
3445	return data->track_data.check_val(track_val, var_val);
3446	}
3447
3448	#ifdef CONFIG_TRACER_SNAPSHOT
3449	static bool cond_snapshot_update(struct trace_array tr, void cond_data)
3450	{
3451	/* called with tr->max_lock held */
3452	struct track_data *track_data = tr->cond_snapshot->cond_data;
3453	struct hist_elt_data elt_data, track_elt_data;
3454	struct snapshot_context *context = cond_data;
3455	struct action_data *action;
3456	u64 track_val;
3457
3458	if (!track_data)
3459	return false;
3460
3461	action = track_data->action_data;
3462
3463	track_val = get_track_val(track_data->hist_data, context->elt,
3464	track_data->action_data);
3465
3466	if (!action->track_data.check_val(track_data->track_val, track_val))
3467	return false;
3468
3469	track_data->track_val = track_val;
3470	memcpy(track_data->key, context->key, track_data->key_len);
3471
3472	elt_data = context->elt->private_data;
3473	track_elt_data = track_data->elt.private_data;
3474	if (elt_data->comm)
3475	strscpy(track_elt_data->comm, elt_data->comm, TASK_COMM_LEN);
3476
3477	track_data->updated = true;
3478
3479	return true;
3480	}
3481
3482	static void save_track_data_snapshot(struct hist_trigger_data *hist_data,
3483	struct tracing_map_elt *elt,
3484	struct trace_buffer buffer, void rec,
3485	struct ring_buffer_event rbe, void key,
3486	struct action_data *data,
3487	u64 *var_ref_vals)
3488	{
3489	struct trace_event_file *file = hist_data->event_file;
3490	struct snapshot_context context;
3491
3492	context.elt = elt;
3493	context.key = key;
3494
3495	tracing_snapshot_cond(file->tr, &context);
3496	}
3497
3498	static void hist_trigger_print_key(struct seq_file *m,
3499	struct hist_trigger_data *hist_data,
3500	void *key,
3501	struct tracing_map_elt *elt);
3502
3503	static struct action_data snapshot_action(struct hist_trigger_data hist_data)
3504	{
3505	unsigned int i;
3506
3507	if (!hist_data->n_actions)
3508	return NULL;
3509
3510	for (i = 0; i < hist_data->n_actions; i++) {
3511	struct action_data *data = hist_data->actions[i];
3512
3513	if (data->action == ACTION_SNAPSHOT)
3514	return data;
3515	}
3516
3517	return NULL;
3518	}
3519
3520	static void track_data_snapshot_print(struct seq_file *m,
3521	struct hist_trigger_data *hist_data)
3522	{
3523	struct trace_event_file *file = hist_data->event_file;
3524	struct track_data *track_data;
3525	struct action_data *action;
3526
3527	track_data = tracing_cond_snapshot_data(file->tr);
3528	if (!track_data)
3529	return;
3530
3531	if (!track_data->updated)
3532	return;
3533
3534	action = snapshot_action(hist_data);
3535	if (!action)
3536	return;
3537
3538	seq_puts(m, "\nSnapshot taken (see tracing/snapshot). Details:\n");
3539	seq_printf(m, "\ttriggering value { %s(%s) }: %10llu",
3540	action->handler == HANDLER_ONMAX ? "onmax" : "onchange",
3541	action->track_data.var_str, track_data->track_val);
3542
3543	seq_puts(m, "\ttriggered by event with key: ");
3544	hist_trigger_print_key(m, hist_data, track_data->key, &track_data->elt);
3545	seq_putc(m, '\n');
3546	}
3547	#else
3548	static bool cond_snapshot_update(struct trace_array tr, void cond_data)
3549	{
3550	return false;
3551	}
3552	static void save_track_data_snapshot(struct hist_trigger_data *hist_data,
3553	struct tracing_map_elt *elt,
3554	struct trace_buffer buffer, void rec,
3555	struct ring_buffer_event rbe, void key,
3556	struct action_data *data,
3557	u64 *var_ref_vals) {}
3558	static void track_data_snapshot_print(struct seq_file *m,
3559	struct hist_trigger_data *hist_data) {}
3560	#endif /* CONFIG_TRACER_SNAPSHOT */
3561
3562	static void track_data_print(struct seq_file *m,
3563	struct hist_trigger_data *hist_data,
3564	struct tracing_map_elt *elt,
3565	struct action_data *data)
3566	{
3567	u64 track_val = get_track_val(hist_data, elt, data);
3568	unsigned int i, save_var_idx;
3569
3570	if (data->handler == HANDLER_ONMAX)
3571	seq_printf(m, "\n\tmax: %10llu", track_val);
3572	else if (data->handler == HANDLER_ONCHANGE)
3573	seq_printf(m, "\n\tchanged: %10llu", track_val);
3574
3575	if (data->action == ACTION_SNAPSHOT)
3576	return;
3577
3578	for (i = 0; i < hist_data->n_save_vars; i++) {
3579	struct hist_field *save_val = hist_data->save_vars[i]->val;
3580	struct hist_field *save_var = hist_data->save_vars[i]->var;
3581	u64 val;
3582
3583	save_var_idx = save_var->var.idx;
3584
3585	val = tracing_map_read_var(elt, save_var_idx);
3586
3587	if (save_val->flags & HIST_FIELD_FL_STRING) {
3588	seq_printf(m, " %s: %-32s", save_var->var.name,
3589	(char *)(uintptr_t)(val));
3590	} else
3591	seq_printf(m, " %s: %10llu", save_var->var.name, val);
3592	}
3593	}
3594
3595	static void ontrack_action(struct hist_trigger_data *hist_data,
3596	struct tracing_map_elt *elt,
3597	struct trace_buffer buffer, void rec,
3598	struct ring_buffer_event rbe, void key,
3599	struct action_data data, u64 var_ref_vals)
3600	{
3601	u64 var_val = var_ref_vals[data->track_data.var_ref->var_ref_idx];
3602
3603	if (check_track_val(elt, data, var_val)) {
3604	save_track_val(hist_data, elt, data, var_val);
3605	save_track_data(hist_data, elt, buffer, rec, rbe,
3606	key, data, var_ref_vals);
3607	}
3608	}
3609
3610	static void action_data_destroy(struct action_data *data)
3611	{
3612	unsigned int i;
3613
3614	lockdep_assert_held(&event_mutex);
3615
3616	kfree(data->action_name);
3617
3618	for (i = 0; i < data->n_params; i++)
3619	kfree(data->params[i]);
3620
3621	if (data->synth_event)
3622	data->synth_event->ref--;
3623
3624	kfree(data->synth_event_name);
3625
3626	kfree(data);
3627	}
3628
3629	static void track_data_destroy(struct hist_trigger_data *hist_data,
3630	struct action_data *data)
3631	{
3632	struct trace_event_file *file = hist_data->event_file;
3633
3634	destroy_hist_field(data->track_data.track_var, 0);
3635
3636	if (data->action == ACTION_SNAPSHOT) {
3637	struct track_data *track_data;
3638
3639	track_data = tracing_cond_snapshot_data(file->tr);
3640	if (track_data && track_data->hist_data == hist_data) {
3641	tracing_snapshot_cond_disable(file->tr);
3642	track_data_free(track_data);
3643	}
3644	}
3645
3646	kfree(data->track_data.var_str);
3647
3648	action_data_destroy(data);
3649	}
3650
3651	static int action_create(struct hist_trigger_data *hist_data,
3652	struct action_data *data);
3653
3654	static int track_data_create(struct hist_trigger_data *hist_data,
3655	struct action_data *data)
3656	{
3657	struct hist_field var_field, ref_field, *track_var = NULL;
3658	struct trace_event_file *file = hist_data->event_file;
3659	struct trace_array *tr = file->tr;
3660	char *track_data_var_str;
3661	int ret = 0;
3662
3663	track_data_var_str = data->track_data.var_str;
3664	if (track_data_var_str[0] != '$') {
3665	hist_err(tr, HIST_ERR_ONX_NOT_VAR, errpos(track_data_var_str));
3666	return -EINVAL;
3667	}
3668	track_data_var_str++;
3669
3670	var_field = find_target_event_var(hist_data, NULL, NULL, track_data_var_str);
3671	if (!var_field) {
3672	hist_err(tr, HIST_ERR_ONX_VAR_NOT_FOUND, errpos(track_data_var_str));
3673	return -EINVAL;
3674	}
3675
3676	ref_field = create_var_ref(hist_data, var_field, NULL, NULL);
3677	if (!ref_field)
3678	return -ENOMEM;
3679
3680	data->track_data.var_ref = ref_field;
3681
3682	if (data->handler == HANDLER_ONMAX)
3683	track_var = create_var(hist_data, file, "__max", sizeof(u64), "u64");
3684	if (IS_ERR(track_var)) {
3685	hist_err(tr, HIST_ERR_ONX_VAR_CREATE_FAIL, 0);
3686	ret = PTR_ERR(track_var);
3687	goto out;
3688	}
3689
3690	if (data->handler == HANDLER_ONCHANGE)
3691	track_var = create_var(hist_data, file, "__change", sizeof(u64), "u64");
3692	if (IS_ERR(track_var)) {
3693	hist_err(tr, HIST_ERR_ONX_VAR_CREATE_FAIL, 0);
3694	ret = PTR_ERR(track_var);
3695	goto out;
3696	}
3697	data->track_data.track_var = track_var;
3698
3699	ret = action_create(hist_data, data);
3700	out:
3701	return ret;
3702	}
3703
3704	static int parse_action_params(struct trace_array tr, char params,
3705	struct action_data *data)
3706	{
3707	char param, saved_param;
3708	bool first_param = true;
3709	int ret = 0;
3710
3711	while (params) {
3712	if (data->n_params >= SYNTH_FIELDS_MAX) {
3713	hist_err(tr, HIST_ERR_TOO_MANY_PARAMS, 0);
3714	ret = -EINVAL;
3715	goto out;
3716	}
3717
3718	param = strsep(&params, ",");
3719	if (!param) {
3720	hist_err(tr, HIST_ERR_PARAM_NOT_FOUND, 0);
3721	ret = -EINVAL;
3722	goto out;
3723	}
3724
3725	param = strstrip(param);
3726	if (strlen(param) < 2) {
3727	hist_err(tr, HIST_ERR_INVALID_PARAM, errpos(param));
3728	ret = -EINVAL;
3729	goto out;
3730	}
3731
3732	saved_param = kstrdup(param, GFP_KERNEL);
3733	if (!saved_param) {
3734	ret = -ENOMEM;
3735	goto out;
3736	}
3737
3738	if (first_param && data->use_trace_keyword) {
3739	data->synth_event_name = saved_param;
3740	first_param = false;
3741	continue;
3742	}
3743	first_param = false;
3744
3745	data->params[data->n_params++] = saved_param;
3746	}
3747	out:
3748	return ret;
3749	}
3750
3751	static int action_parse(struct trace_array tr, char str, struct action_data *data,
3752	enum handler_id handler)
3753	{
3754	char *action_name;
3755	int ret = 0;
3756
3757	strsep(&str, ".");
3758	if (!str) {
3759	hist_err(tr, HIST_ERR_ACTION_NOT_FOUND, 0);
3760	ret = -EINVAL;
3761	goto out;
3762	}
3763
3764	action_name = strsep(&str, "(");
3765	if (!action_name \|\| !str) {
3766	hist_err(tr, HIST_ERR_ACTION_NOT_FOUND, 0);
3767	ret = -EINVAL;
3768	goto out;
3769	}
3770
3771	if (str_has_prefix(action_name, "save")) {
3772	char *params = strsep(&str, ")");
3773
3774	if (!params) {
3775	hist_err(tr, HIST_ERR_NO_SAVE_PARAMS, 0);
3776	ret = -EINVAL;
3777	goto out;
3778	}
3779
3780	ret = parse_action_params(tr, params, data);
3781	if (ret)
3782	goto out;
3783
3784	if (handler == HANDLER_ONMAX)
3785	data->track_data.check_val = check_track_val_max;
3786	else if (handler == HANDLER_ONCHANGE)
3787	data->track_data.check_val = check_track_val_changed;
3788	else {
3789	hist_err(tr, HIST_ERR_ACTION_MISMATCH, errpos(action_name));
3790	ret = -EINVAL;
3791	goto out;
3792	}
3793
3794	data->track_data.save_data = save_track_data_vars;
3795	data->fn = ontrack_action;
3796	data->action = ACTION_SAVE;
3797	} else if (str_has_prefix(action_name, "snapshot")) {
3798	char *params = strsep(&str, ")");
3799
3800	if (!str) {
3801	hist_err(tr, HIST_ERR_NO_CLOSING_PAREN, errpos(params));
3802	ret = -EINVAL;
3803	goto out;
3804	}
3805
3806	if (handler == HANDLER_ONMAX)
3807	data->track_data.check_val = check_track_val_max;
3808	else if (handler == HANDLER_ONCHANGE)
3809	data->track_data.check_val = check_track_val_changed;
3810	else {
3811	hist_err(tr, HIST_ERR_ACTION_MISMATCH, errpos(action_name));
3812	ret = -EINVAL;
3813	goto out;
3814	}
3815
3816	data->track_data.save_data = save_track_data_snapshot;
3817	data->fn = ontrack_action;
3818	data->action = ACTION_SNAPSHOT;
3819	} else {
3820	char *params = strsep(&str, ")");
3821
3822	if (str_has_prefix(action_name, "trace"))
3823	data->use_trace_keyword = true;
3824
3825	if (params) {
3826	ret = parse_action_params(tr, params, data);
3827	if (ret)
3828	goto out;
3829	}
3830
3831	if (handler == HANDLER_ONMAX)
3832	data->track_data.check_val = check_track_val_max;
3833	else if (handler == HANDLER_ONCHANGE)
3834	data->track_data.check_val = check_track_val_changed;
3835
3836	if (handler != HANDLER_ONMATCH) {
3837	data->track_data.save_data = action_trace;
3838	data->fn = ontrack_action;
3839	} else
3840	data->fn = action_trace;
3841
3842	data->action = ACTION_TRACE;
3843	}
3844
3845	data->action_name = kstrdup(action_name, GFP_KERNEL);
3846	if (!data->action_name) {
3847	ret = -ENOMEM;
3848	goto out;
3849	}
3850
3851	data->handler = handler;
3852	out:
3853	return ret;
3854	}
3855
3856	static struct action_data track_data_parse(struct hist_trigger_data hist_data,
3857	char *str, enum handler_id handler)
3858	{
3859	struct action_data *data;
3860	int ret = -EINVAL;
3861	char *var_str;
3862
3863	data = kzalloc_obj(*data);
3864	if (!data)
3865	return ERR_PTR(-ENOMEM);
3866
3867	var_str = strsep(&str, ")");
3868	if (!var_str \|\| !str) {
3869	ret = -EINVAL;
3870	goto free;
3871	}
3872
3873	data->track_data.var_str = kstrdup(var_str, GFP_KERNEL);
3874	if (!data->track_data.var_str) {
3875	ret = -ENOMEM;
3876	goto free;
3877	}
3878
3879	ret = action_parse(hist_data->event_file->tr, str, data, handler);
3880	if (ret)
3881	goto free;
3882	out:
3883	return data;
3884	free:
3885	track_data_destroy(hist_data, data);
3886	data = ERR_PTR(ret);
3887	goto out;
3888	}
3889
3890	static void onmatch_destroy(struct action_data *data)
3891	{
3892	kfree(data->match_data.event);
3893	kfree(data->match_data.event_system);
3894
3895	action_data_destroy(data);
3896	}
3897
3898	static void destroy_field_var(struct field_var *field_var)
3899	{
3900	if (!field_var)
3901	return;
3902
3903	destroy_hist_field(field_var->var, 0);
3904	destroy_hist_field(field_var->val, 0);
3905
3906	kfree(field_var);
3907	}
3908
3909	static void destroy_field_vars(struct hist_trigger_data *hist_data)
3910	{
3911	unsigned int i;
3912
3913	for (i = 0; i < hist_data->n_field_vars; i++)
3914	destroy_field_var(hist_data->field_vars[i]);
3915
3916	for (i = 0; i < hist_data->n_save_vars; i++)
3917	destroy_field_var(hist_data->save_vars[i]);
3918	}
3919
3920	static void save_field_var(struct hist_trigger_data *hist_data,
3921	struct field_var *field_var)
3922	{
3923	hist_data->field_vars[hist_data->n_field_vars++] = field_var;
3924
3925	/* Stack traces are saved in the string storage too */
3926	if (field_var->val->flags & (HIST_FIELD_FL_STRING \| HIST_FIELD_FL_STACKTRACE))
3927	hist_data->n_field_var_str++;
3928	}
3929
3930
3931	static int check_synth_field(struct synth_event *event,
3932	struct hist_field *hist_field,
3933	unsigned int field_pos)
3934	{
3935	struct synth_field *field;
3936
3937	if (field_pos >= event->n_fields)
3938	return -EINVAL;
3939
3940	field = event->fields[field_pos];
3941
3942	/*
3943	* A dynamic string synth field can accept static or
3944	* dynamic. A static string synth field can only accept a
3945	* same-sized static string, which is checked for later.
3946	*/
3947	if (strstr(hist_field->type, "char[") && field->is_string
3948	&& field->is_dynamic)
3949	return 0;
3950
3951	if (!!(hist_field->flags & HIST_FIELD_FL_STACKTRACE) != field->is_stack)
3952	return -EINVAL;
3953
3954	if (field->is_stack)
3955	return 0;
3956
3957	if (strcmp(field->type, hist_field->type) != 0) {
3958	if (field->size != hist_field->size \|\|
3959	(!field->is_string && field->is_signed != hist_field->is_signed))
3960	return -EINVAL;
3961	}
3962
3963	return 0;
3964	}
3965
3966	static struct hist_field *
3967	trace_action_find_var(struct hist_trigger_data *hist_data,
3968	struct action_data *data,
3969	char system, char event, char *var)
3970	{
3971	struct trace_array *tr = hist_data->event_file->tr;
3972	struct hist_field *hist_field;
3973
3974	var++; /* skip '$' */
3975
3976	hist_field = find_target_event_var(hist_data, system, event, var);
3977	if (!hist_field) {
3978	if (!system && data->handler == HANDLER_ONMATCH) {
3979	system = data->match_data.event_system;
3980	event = data->match_data.event;
3981	}
3982
3983	hist_field = find_event_var(hist_data, system, event, var);
3984	}
3985
3986	if (!hist_field)
3987	hist_err(tr, HIST_ERR_PARAM_NOT_FOUND, errpos(var));
3988
3989	return hist_field;
3990	}
3991
3992	static struct hist_field *
3993	trace_action_create_field_var(struct hist_trigger_data *hist_data,
3994	struct action_data data, char system,
3995	char event, char var)
3996	{
3997	struct hist_field *hist_field = NULL;
3998	struct field_var *field_var;
3999
4000	/*
4001	* First try to create a field var on the target event (the
4002	* currently being defined). This will create a variable for
4003	* unqualified fields on the target event, or if qualified,
4004	* target fields that have qualified names matching the target.
4005	*/
4006	field_var = create_target_field_var(hist_data, system, event, var);
4007
4008	if (field_var && !IS_ERR(field_var)) {
4009	save_field_var(hist_data, field_var);
4010	hist_field = field_var->var;
4011	} else {
4012	field_var = NULL;
4013	/*
4014	* If no explicit system.event is specified, default to
4015	* looking for fields on the onmatch(system.event.xxx)
4016	* event.
4017	*/
4018	if (!system && data->handler == HANDLER_ONMATCH) {
4019	system = data->match_data.event_system;
4020	event = data->match_data.event;
4021	}
4022
4023	if (!event)
4024	goto free;
4025	/*
4026	* At this point, we're looking at a field on another
4027	* event. Because we can't modify a hist trigger on
4028	* another event to add a variable for a field, we need
4029	* to create a new trigger on that event and create the
4030	* variable at the same time.
4031	*/
4032	hist_field = create_field_var_hist(hist_data, system, event, var);
4033	if (IS_ERR(hist_field))
4034	goto free;
4035	}
4036	out:
4037	return hist_field;
4038	free:
4039	destroy_field_var(field_var);
4040	hist_field = NULL;
4041	goto out;
4042	}
4043
4044	static int trace_action_create(struct hist_trigger_data *hist_data,
4045	struct action_data *data)
4046	{
4047	struct trace_array *tr = hist_data->event_file->tr;
4048	char event_name, param, *system = NULL;
4049	struct hist_field hist_field, var_ref;
4050	unsigned int i;
4051	unsigned int field_pos = 0;
4052	struct synth_event *event;
4053	char *synth_event_name;
4054	int var_ref_idx, ret = 0;
4055
4056	lockdep_assert_held(&event_mutex);
4057
4058	/* Sanity check to avoid out-of-bound write on 'data->var_ref_idx' */
4059	if (data->n_params > SYNTH_FIELDS_MAX)
4060	return -EINVAL;
4061
4062	if (data->use_trace_keyword)
4063	synth_event_name = data->synth_event_name;
4064	else
4065	synth_event_name = data->action_name;
4066
4067	event = find_synth_event(synth_event_name);
4068	if (!event) {
4069	hist_err(tr, HIST_ERR_SYNTH_EVENT_NOT_FOUND, errpos(synth_event_name));
4070	return -EINVAL;
4071	}
4072
4073	event->ref++;
4074
4075	for (i = 0; i < data->n_params; i++) {
4076	char *p;
4077
4078	p = param = kstrdup(data->params[i], GFP_KERNEL);
4079	if (!param) {
4080	ret = -ENOMEM;
4081	goto err;
4082	}
4083
4084	system = strsep(&param, ".");
4085	if (!param) {
4086	param = (char *)system;
4087	system = event_name = NULL;
4088	} else {
4089	event_name = strsep(&param, ".");
4090	if (!param) {
4091	kfree(p);
4092	ret = -EINVAL;
4093	goto err;
4094	}
4095	}
4096
4097	if (param[0] == '$')
4098	hist_field = trace_action_find_var(hist_data, data,
4099	system, event_name,
4100	param);
4101	else
4102	hist_field = trace_action_create_field_var(hist_data,
4103	data,
4104	system,
4105	event_name,
4106	param);
4107
4108	if (!hist_field) {
4109	kfree(p);
4110	ret = -EINVAL;
4111	goto err;
4112	}
4113
4114	if (check_synth_field(event, hist_field, field_pos) == 0) {
4115	var_ref = create_var_ref(hist_data, hist_field,
4116	system, event_name);
4117	if (!var_ref) {
4118	kfree(p);
4119	ret = -ENOMEM;
4120	goto err;
4121	}
4122
4123	var_ref_idx = find_var_ref_idx(hist_data, var_ref);
4124	if (WARN_ON(var_ref_idx < 0)) {
4125	kfree(p);
4126	ret = var_ref_idx;
4127	goto err;
4128	}
4129
4130	data->var_ref_idx[i] = var_ref_idx;
4131
4132	field_pos++;
4133	kfree(p);
4134	continue;
4135	}
4136
4137	hist_err(tr, HIST_ERR_SYNTH_TYPE_MISMATCH, errpos(param));
4138	kfree(p);
4139	ret = -EINVAL;
4140	goto err;
4141	}
4142
4143	if (field_pos != event->n_fields) {
4144	hist_err(tr, HIST_ERR_SYNTH_COUNT_MISMATCH, errpos(event->name));
4145	ret = -EINVAL;
4146	goto err;
4147	}
4148
4149	data->synth_event = event;
4150	out:
4151	return ret;
4152	err:
4153	event->ref--;
4154
4155	goto out;
4156	}
4157
4158	static int action_create(struct hist_trigger_data *hist_data,
4159	struct action_data *data)
4160	{
4161	struct trace_event_file *file = hist_data->event_file;
4162	struct trace_array *tr = file->tr;
4163	struct track_data *track_data;
4164	struct field_var *field_var;
4165	unsigned int i;
4166	char *param;
4167	int ret = 0;
4168
4169	if (data->action == ACTION_TRACE)
4170	return trace_action_create(hist_data, data);
4171
4172	if (data->action == ACTION_SNAPSHOT) {
4173	track_data = track_data_alloc(hist_data->key_size, data, hist_data);
4174	if (IS_ERR(track_data)) {
4175	ret = PTR_ERR(track_data);
4176	goto out;
4177	}
4178
4179	ret = tracing_snapshot_cond_enable(file->tr, track_data,
4180	cond_snapshot_update);
4181	if (ret)
4182	track_data_free(track_data);
4183
4184	goto out;
4185	}
4186
4187	if (data->action == ACTION_SAVE) {
4188	if (hist_data->n_save_vars) {
4189	ret = -EEXIST;
4190	hist_err(tr, HIST_ERR_TOO_MANY_SAVE_ACTIONS, 0);
4191	goto out;
4192	}
4193
4194	for (i = 0; i < data->n_params; i++) {
4195	param = kstrdup(data->params[i], GFP_KERNEL);
4196	if (!param) {
4197	ret = -ENOMEM;
4198	goto out;
4199	}
4200
4201	field_var = create_target_field_var(hist_data, NULL, NULL, param);
4202	if (IS_ERR(field_var)) {
4203	hist_err(tr, HIST_ERR_FIELD_VAR_CREATE_FAIL,
4204	errpos(param));
4205	ret = PTR_ERR(field_var);
4206	kfree(param);
4207	goto out;
4208	}
4209
4210	hist_data->save_vars[hist_data->n_save_vars++] = field_var;
4211	if (field_var->val->flags &
4212	(HIST_FIELD_FL_STRING \| HIST_FIELD_FL_STACKTRACE))
4213	hist_data->n_save_var_str++;
4214	kfree(param);
4215	}
4216	}
4217	out:
4218	return ret;
4219	}
4220
4221	static int onmatch_create(struct hist_trigger_data *hist_data,
4222	struct action_data *data)
4223	{
4224	return action_create(hist_data, data);
4225	}
4226
4227	static struct action_data onmatch_parse(struct trace_array tr, char *str)
4228	{
4229	char match_event, match_event_system;
4230	struct action_data *data;
4231	int ret = -EINVAL;
4232
4233	data = kzalloc_obj(*data);
4234	if (!data)
4235	return ERR_PTR(-ENOMEM);
4236
4237	match_event = strsep(&str, ")");
4238	if (!match_event \|\| !str) {
4239	hist_err(tr, HIST_ERR_NO_CLOSING_PAREN, errpos(match_event));
4240	goto free;
4241	}
4242
4243	match_event_system = strsep(&match_event, ".");
4244	if (!match_event) {
4245	hist_err(tr, HIST_ERR_SUBSYS_NOT_FOUND, errpos(match_event_system));
4246	goto free;
4247	}
4248
4249	if (IS_ERR(event_file(tr, match_event_system, match_event))) {
4250	hist_err(tr, HIST_ERR_INVALID_SUBSYS_EVENT, errpos(match_event));
4251	goto free;
4252	}
4253
4254	data->match_data.event = kstrdup(match_event, GFP_KERNEL);
4255	if (!data->match_data.event) {
4256	ret = -ENOMEM;
4257	goto free;
4258	}
4259
4260	data->match_data.event_system = kstrdup(match_event_system, GFP_KERNEL);
4261	if (!data->match_data.event_system) {
4262	ret = -ENOMEM;
4263	goto free;
4264	}
4265
4266	ret = action_parse(tr, str, data, HANDLER_ONMATCH);
4267	if (ret)
4268	goto free;
4269	out:
4270	return data;
4271	free:
4272	onmatch_destroy(data);
4273	data = ERR_PTR(ret);
4274	goto out;
4275	}
4276
4277	static int create_hitcount_val(struct hist_trigger_data *hist_data)
4278	{
4279	hist_data->fields[HITCOUNT_IDX] =
4280	create_hist_field(hist_data, NULL, HIST_FIELD_FL_HITCOUNT, NULL);
4281	if (!hist_data->fields[HITCOUNT_IDX])
4282	return -ENOMEM;
4283
4284	hist_data->n_vals++;
4285	hist_data->n_fields++;
4286
4287	if (WARN_ON(hist_data->n_vals > TRACING_MAP_VALS_MAX))
4288	return -EINVAL;
4289
4290	return 0;
4291	}
4292
4293	static int __create_val_field(struct hist_trigger_data *hist_data,
4294	unsigned int val_idx,
4295	struct trace_event_file *file,
4296	char var_name, char field_str,
4297	unsigned long flags)
4298	{
4299	struct hist_field *hist_field;
4300	int ret = 0, n_subexprs = 0;
4301
4302	hist_field = parse_expr(hist_data, file, field_str, flags, var_name, &n_subexprs);
4303	if (IS_ERR(hist_field)) {
4304	ret = PTR_ERR(hist_field);
4305	goto out;
4306	}
4307
4308	/* values and variables should not have some modifiers */
4309	if (hist_field->flags & HIST_FIELD_FL_VAR) {
4310	/* Variable */
4311	if (hist_field->flags & (HIST_FIELD_FL_GRAPH \| HIST_FIELD_FL_PERCENT \|
4312	HIST_FIELD_FL_BUCKET \| HIST_FIELD_FL_LOG2))
4313	goto err;
4314	} else {
4315	/* Value */
4316	if (hist_field->flags & (HIST_FIELD_FL_BUCKET \| HIST_FIELD_FL_LOG2 \|
4317	HIST_FIELD_FL_SYM \| HIST_FIELD_FL_SYM_OFFSET \|
4318	HIST_FIELD_FL_SYSCALL \| HIST_FIELD_FL_STACKTRACE))
4319	goto err;
4320	}
4321
4322	hist_data->fields[val_idx] = hist_field;
4323
4324	++hist_data->n_vals;
4325	++hist_data->n_fields;
4326
4327	if (WARN_ON(hist_data->n_vals > TRACING_MAP_VALS_MAX + TRACING_MAP_VARS_MAX))
4328	ret = -EINVAL;
4329	out:
4330	return ret;
4331	err:
4332	hist_err(file->tr, HIST_ERR_BAD_FIELD_MODIFIER, errpos(field_str));
4333	destroy_hist_field(hist_field, 0);
4334	return -EINVAL;
4335	}
4336
4337	static int create_val_field(struct hist_trigger_data *hist_data,
4338	unsigned int val_idx,
4339	struct trace_event_file *file,
4340	char *field_str)
4341	{
4342	if (WARN_ON(val_idx >= TRACING_MAP_VALS_MAX))
4343	return -EINVAL;
4344
4345	return __create_val_field(hist_data, val_idx, file, NULL, field_str, 0);
4346	}
4347
4348	static const char no_comm[] = "(no comm)";
4349
4350	static u64 hist_field_execname(struct hist_field *hist_field,
4351	struct tracing_map_elt *elt,
4352	struct trace_buffer *buffer,
4353	struct ring_buffer_event *rbe,
4354	void *event)
4355	{
4356	struct hist_elt_data *elt_data;
4357
4358	if (WARN_ON_ONCE(!elt))
4359	return (u64)(unsigned long)no_comm;
4360
4361	elt_data = elt->private_data;
4362
4363	if (WARN_ON_ONCE(!elt_data->comm))
4364	return (u64)(unsigned long)no_comm;
4365
4366	return (u64)(unsigned long)(elt_data->comm);
4367	}
4368
4369	static u64 hist_field_stack(struct hist_field *hist_field,
4370	struct tracing_map_elt *elt,
4371	struct trace_buffer *buffer,
4372	struct ring_buffer_event *rbe,
4373	void *event)
4374	{
4375	u32 str_item = (u32 )(event + hist_field->field->offset);
4376	int str_loc = str_item & 0xffff;
4377	char addr = (char )(event + str_loc);
4378
4379	return (u64)(unsigned long)addr;
4380	}
4381
4382	static u64 hist_fn_call(struct hist_field *hist_field,
4383	struct tracing_map_elt *elt,
4384	struct trace_buffer *buffer,
4385	struct ring_buffer_event *rbe,
4386	void *event)
4387	{
4388	switch (hist_field->fn_num) {
4389	case HIST_FIELD_FN_VAR_REF:
4390	return hist_field_var_ref(hist_field, elt, buffer, rbe, event);
4391	case HIST_FIELD_FN_COUNTER:
4392	return hist_field_counter(hist_field, elt, buffer, rbe, event);
4393	case HIST_FIELD_FN_CONST:
4394	return hist_field_const(hist_field, elt, buffer, rbe, event);
4395	case HIST_FIELD_FN_LOG2:
4396	return hist_field_log2(hist_field, elt, buffer, rbe, event);
4397	case HIST_FIELD_FN_BUCKET:
4398	return hist_field_bucket(hist_field, elt, buffer, rbe, event);
4399	case HIST_FIELD_FN_TIMESTAMP:
4400	return hist_field_timestamp(hist_field, elt, buffer, rbe, event);
4401	case HIST_FIELD_FN_CPU:
4402	return hist_field_cpu(hist_field, elt, buffer, rbe, event);
4403	case HIST_FIELD_FN_COMM:
4404	return hist_field_comm(hist_field, elt, buffer, rbe, event);
4405	case HIST_FIELD_FN_STRING:
4406	return hist_field_string(hist_field, elt, buffer, rbe, event);
4407	case HIST_FIELD_FN_DYNSTRING:
4408	return hist_field_dynstring(hist_field, elt, buffer, rbe, event);
4409	case HIST_FIELD_FN_RELDYNSTRING:
4410	return hist_field_reldynstring(hist_field, elt, buffer, rbe, event);
4411	case HIST_FIELD_FN_PSTRING:
4412	return hist_field_pstring(hist_field, elt, buffer, rbe, event);
4413	case HIST_FIELD_FN_S64:
4414	return hist_field_s64(hist_field, elt, buffer, rbe, event);
4415	case HIST_FIELD_FN_U64:
4416	return hist_field_u64(hist_field, elt, buffer, rbe, event);
4417	case HIST_FIELD_FN_S32:
4418	return hist_field_s32(hist_field, elt, buffer, rbe, event);
4419	case HIST_FIELD_FN_U32:
4420	return hist_field_u32(hist_field, elt, buffer, rbe, event);
4421	case HIST_FIELD_FN_S16:
4422	return hist_field_s16(hist_field, elt, buffer, rbe, event);
4423	case HIST_FIELD_FN_U16:
4424	return hist_field_u16(hist_field, elt, buffer, rbe, event);
4425	case HIST_FIELD_FN_S8:
4426	return hist_field_s8(hist_field, elt, buffer, rbe, event);
4427	case HIST_FIELD_FN_U8:
4428	return hist_field_u8(hist_field, elt, buffer, rbe, event);
4429	case HIST_FIELD_FN_UMINUS:
4430	return hist_field_unary_minus(hist_field, elt, buffer, rbe, event);
4431	case HIST_FIELD_FN_MINUS:
4432	return hist_field_minus(hist_field, elt, buffer, rbe, event);
4433	case HIST_FIELD_FN_PLUS:
4434	return hist_field_plus(hist_field, elt, buffer, rbe, event);
4435	case HIST_FIELD_FN_DIV:
4436	return hist_field_div(hist_field, elt, buffer, rbe, event);
4437	case HIST_FIELD_FN_MULT:
4438	return hist_field_mult(hist_field, elt, buffer, rbe, event);
4439	case HIST_FIELD_FN_DIV_POWER2:
4440	return div_by_power_of_two(hist_field, elt, buffer, rbe, event);
4441	case HIST_FIELD_FN_DIV_NOT_POWER2:
4442	return div_by_not_power_of_two(hist_field, elt, buffer, rbe, event);
4443	case HIST_FIELD_FN_DIV_MULT_SHIFT:
4444	return div_by_mult_and_shift(hist_field, elt, buffer, rbe, event);
4445	case HIST_FIELD_FN_EXECNAME:
4446	return hist_field_execname(hist_field, elt, buffer, rbe, event);
4447	case HIST_FIELD_FN_STACK:
4448	return hist_field_stack(hist_field, elt, buffer, rbe, event);
4449	default:
4450	return 0;
4451	}
4452	}
4453
4454	/* Convert a var that points to common_pid.execname to a string */
4455	static void update_var_execname(struct hist_field *hist_field)
4456	{
4457	hist_field->flags = HIST_FIELD_FL_STRING \| HIST_FIELD_FL_VAR \|
4458	HIST_FIELD_FL_EXECNAME;
4459	hist_field->size = MAX_FILTER_STR_VAL;
4460	hist_field->is_signed = 0;
4461
4462	kfree_const(hist_field->type);
4463	hist_field->type = "char[]";
4464
4465	hist_field->fn_num = HIST_FIELD_FN_EXECNAME;
4466	}
4467
4468	static int create_var_field(struct hist_trigger_data *hist_data,
4469	unsigned int val_idx,
4470	struct trace_event_file *file,
4471	char var_name, char expr_str)
4472	{
4473	struct trace_array *tr = hist_data->event_file->tr;
4474	unsigned long flags = 0;
4475	int ret;
4476
4477	if (WARN_ON(val_idx >= TRACING_MAP_VALS_MAX + TRACING_MAP_VARS_MAX))
4478	return -EINVAL;
4479
4480	if (find_var(hist_data, file, var_name) && !hist_data->remove) {
4481	hist_err(tr, HIST_ERR_DUPLICATE_VAR, errpos(var_name));
4482	return -EINVAL;
4483	}
4484
4485	flags \|= HIST_FIELD_FL_VAR;
4486	hist_data->n_vars++;
4487	if (WARN_ON(hist_data->n_vars > TRACING_MAP_VARS_MAX))
4488	return -EINVAL;
4489
4490	ret = __create_val_field(hist_data, val_idx, file, var_name, expr_str, flags);
4491
4492	if (!ret && hist_data->fields[val_idx]->flags & HIST_FIELD_FL_EXECNAME)
4493	update_var_execname(hist_data->fields[val_idx]);
4494
4495	if (!ret && !(hist_data->fields[val_idx]->flags & HIST_FIELD_FL_ALIAS) &&
4496	hist_data->fields[val_idx]->flags &
4497	(HIST_FIELD_FL_STRING \| HIST_FIELD_FL_STACKTRACE))
4498	hist_data->fields[val_idx]->var_str_idx = hist_data->n_var_str++;
4499
4500	return ret;
4501	}
4502
4503	static int create_val_fields(struct hist_trigger_data *hist_data,
4504	struct trace_event_file *file)
4505	{
4506	unsigned int i, j = 1, n_hitcount = 0;
4507	char fields_str, field_str;
4508	int ret;
4509
4510	ret = create_hitcount_val(hist_data);
4511	if (ret)
4512	goto out;
4513
4514	fields_str = hist_data->attrs->vals_str;
4515	if (!fields_str)
4516	goto out;
4517
4518	for (i = 0, j = 1; i < TRACING_MAP_VALS_MAX &&
4519	j < TRACING_MAP_VALS_MAX; i++) {
4520	field_str = strsep(&fields_str, ",");
4521	if (!field_str)
4522	break;
4523
4524	if (strcmp(field_str, "hitcount") == 0) {
4525	if (!n_hitcount++)
4526	continue;
4527	}
4528
4529	ret = create_val_field(hist_data, j++, file, field_str);
4530	if (ret)
4531	goto out;
4532	}
4533
4534	if (fields_str && (strcmp(fields_str, "hitcount") != 0))
4535	ret = -EINVAL;
4536	out:
4537	/* There is only raw hitcount but nohitcount suppresses it. */
4538	if (j == 1 && hist_data->attrs->no_hitcount) {
4539	hist_err(hist_data->event_file->tr, HIST_ERR_NEED_NOHC_VAL, 0);
4540	ret = -ENOENT;
4541	}
4542
4543	return ret;
4544	}
4545
4546	static int create_key_field(struct hist_trigger_data *hist_data,
4547	unsigned int key_idx,
4548	unsigned int key_offset,
4549	struct trace_event_file *file,
4550	char *field_str)
4551	{
4552	struct trace_array *tr = hist_data->event_file->tr;
4553	struct hist_field *hist_field = NULL;
4554	unsigned long flags = 0;
4555	unsigned int key_size;
4556	int ret = 0, n_subexprs = 0;
4557
4558	if (WARN_ON(key_idx >= HIST_FIELDS_MAX))
4559	return -EINVAL;
4560
4561	flags \|= HIST_FIELD_FL_KEY;
4562
4563	if (strcmp(field_str, "stacktrace") == 0) {
4564	flags \|= HIST_FIELD_FL_STACKTRACE;
4565	key_size = sizeof(unsigned long) * HIST_STACKTRACE_DEPTH;
4566	hist_field = create_hist_field(hist_data, NULL, flags, NULL);
4567	} else {
4568	hist_field = parse_expr(hist_data, file, field_str, flags,
4569	NULL, &n_subexprs);
4570	if (IS_ERR(hist_field)) {
4571	ret = PTR_ERR(hist_field);
4572	goto out;
4573	}
4574
4575	if (field_has_hist_vars(hist_field, 0)) {
4576	hist_err(tr, HIST_ERR_INVALID_REF_KEY, errpos(field_str));
4577	destroy_hist_field(hist_field, 0);
4578	ret = -EINVAL;
4579	goto out;
4580	}
4581
4582	key_size = hist_field->size;
4583	}
4584
4585	hist_data->fields[key_idx] = hist_field;
4586
4587	key_size = ALIGN(key_size, sizeof(u64));
4588	hist_data->fields[key_idx]->size = key_size;
4589	hist_data->fields[key_idx]->offset = key_offset;
4590
4591	hist_data->key_size += key_size;
4592
4593	if (hist_data->key_size > HIST_KEY_SIZE_MAX) {
4594	ret = -EINVAL;
4595	goto out;
4596	}
4597
4598	hist_data->n_keys++;
4599	hist_data->n_fields++;
4600
4601	if (WARN_ON(hist_data->n_keys > TRACING_MAP_KEYS_MAX))
4602	return -EINVAL;
4603
4604	ret = key_size;
4605	out:
4606	return ret;
4607	}
4608
4609	static int create_key_fields(struct hist_trigger_data *hist_data,
4610	struct trace_event_file *file)
4611	{
4612	unsigned int i, key_offset = 0, n_vals = hist_data->n_vals;
4613	char fields_str, field_str;
4614	int ret = -EINVAL;
4615
4616	fields_str = hist_data->attrs->keys_str;
4617	if (!fields_str)
4618	goto out;
4619
4620	for (i = n_vals; i < n_vals + TRACING_MAP_KEYS_MAX; i++) {
4621	field_str = strsep(&fields_str, ",");
4622	if (!field_str)
4623	break;
4624	ret = create_key_field(hist_data, i, key_offset,
4625	file, field_str);
4626	if (ret < 0)
4627	goto out;
4628	key_offset += ret;
4629	}
4630	if (fields_str) {
4631	ret = -EINVAL;
4632	goto out;
4633	}
4634	ret = 0;
4635	out:
4636	return ret;
4637	}
4638
4639	static int create_var_fields(struct hist_trigger_data *hist_data,
4640	struct trace_event_file *file)
4641	{
4642	unsigned int i, j = hist_data->n_vals;
4643	int ret = 0;
4644
4645	unsigned int n_vars = hist_data->attrs->var_defs.n_vars;
4646
4647	for (i = 0; i < n_vars; i++) {
4648	char *var_name = hist_data->attrs->var_defs.name[i];
4649	char *expr = hist_data->attrs->var_defs.expr[i];
4650
4651	ret = create_var_field(hist_data, j++, file, var_name, expr);
4652	if (ret)
4653	goto out;
4654	}
4655	out:
4656	return ret;
4657	}
4658
4659	static void free_var_defs(struct hist_trigger_data *hist_data)
4660	{
4661	unsigned int i;
4662
4663	for (i = 0; i < hist_data->attrs->var_defs.n_vars; i++) {
4664	kfree(hist_data->attrs->var_defs.name[i]);
4665	kfree(hist_data->attrs->var_defs.expr[i]);
4666	}
4667
4668	hist_data->attrs->var_defs.n_vars = 0;
4669	}
4670
4671	static int parse_var_defs(struct hist_trigger_data *hist_data)
4672	{
4673	struct trace_array *tr = hist_data->event_file->tr;
4674	char s, str, var_name, field_str;
4675	unsigned int i, j, n_vars = 0;
4676	int ret = 0;
4677
4678	for (i = 0; i < hist_data->attrs->n_assignments; i++) {
4679	str = hist_data->attrs->assignment_str[i];
4680	for (j = 0; j < TRACING_MAP_VARS_MAX; j++) {
4681	field_str = strsep(&str, ",");
4682	if (!field_str)
4683	break;
4684
4685	var_name = strsep(&field_str, "=");
4686	if (!var_name \|\| !field_str) {
4687	hist_err(tr, HIST_ERR_MALFORMED_ASSIGNMENT,
4688	errpos(var_name));
4689	ret = -EINVAL;
4690	goto free;
4691	}
4692
4693	if (n_vars == TRACING_MAP_VARS_MAX) {
4694	hist_err(tr, HIST_ERR_TOO_MANY_VARS, errpos(var_name));
4695	ret = -EINVAL;
4696	goto free;
4697	}
4698
4699	s = kstrdup(var_name, GFP_KERNEL);
4700	if (!s) {
4701	ret = -ENOMEM;
4702	goto free;
4703	}
4704	hist_data->attrs->var_defs.name[n_vars] = s;
4705
4706	s = kstrdup(field_str, GFP_KERNEL);
4707	if (!s) {
4708	kfree(hist_data->attrs->var_defs.name[n_vars]);
4709	hist_data->attrs->var_defs.name[n_vars] = NULL;
4710	ret = -ENOMEM;
4711	goto free;
4712	}
4713	hist_data->attrs->var_defs.expr[n_vars++] = s;
4714
4715	hist_data->attrs->var_defs.n_vars = n_vars;
4716	}
4717	}
4718
4719	return ret;
4720	free:
4721	free_var_defs(hist_data);
4722
4723	return ret;
4724	}
4725
4726	static int create_hist_fields(struct hist_trigger_data *hist_data,
4727	struct trace_event_file *file)
4728	{
4729	int ret;
4730
4731	ret = parse_var_defs(hist_data);
4732	if (ret)
4733	return ret;
4734
4735	ret = create_val_fields(hist_data, file);
4736	if (ret)
4737	goto out;
4738
4739	ret = create_var_fields(hist_data, file);
4740	if (ret)
4741	goto out;
4742
4743	ret = create_key_fields(hist_data, file);
4744
4745	out:
4746	free_var_defs(hist_data);
4747
4748	return ret;
4749	}
4750
4751	static int is_descending(struct trace_array tr, const char str)
4752	{
4753	if (!str)
4754	return 0;
4755
4756	if (strcmp(str, "descending") == 0)
4757	return 1;
4758
4759	if (strcmp(str, "ascending") == 0)
4760	return 0;
4761
4762	hist_err(tr, HIST_ERR_INVALID_SORT_MODIFIER, errpos((char *)str));
4763
4764	return -EINVAL;
4765	}
4766
4767	static int create_sort_keys(struct hist_trigger_data *hist_data)
4768	{
4769	struct trace_array *tr = hist_data->event_file->tr;
4770	char *fields_str = hist_data->attrs->sort_key_str;
4771	struct tracing_map_sort_key *sort_key;
4772	int descending, ret = 0;
4773	unsigned int i, j, k;
4774
4775	hist_data->n_sort_keys = 1; /* we always have at least one, hitcount */
4776
4777	if (!fields_str)
4778	goto out;
4779
4780	for (i = 0; i < TRACING_MAP_SORT_KEYS_MAX; i++) {
4781	struct hist_field *hist_field;
4782	char field_str, field_name;
4783	const char *test_name;
4784
4785	sort_key = &hist_data->sort_keys[i];
4786
4787	field_str = strsep(&fields_str, ",");
4788	if (!field_str)
4789	break;
4790
4791	if (!*field_str) {
4792	ret = -EINVAL;
4793	hist_err(tr, HIST_ERR_EMPTY_SORT_FIELD, errpos("sort="));
4794	break;
4795	}
4796
4797	if ((i == TRACING_MAP_SORT_KEYS_MAX - 1) && fields_str) {
4798	hist_err(tr, HIST_ERR_TOO_MANY_SORT_FIELDS, errpos("sort="));
4799	ret = -EINVAL;
4800	break;
4801	}
4802
4803	field_name = strsep(&field_str, ".");
4804	if (!field_name \|\| !*field_name) {
4805	ret = -EINVAL;
4806	hist_err(tr, HIST_ERR_EMPTY_SORT_FIELD, errpos("sort="));
4807	break;
4808	}
4809
4810	if (strcmp(field_name, "hitcount") == 0) {
4811	descending = is_descending(tr, field_str);
4812	if (descending < 0) {
4813	ret = descending;
4814	break;
4815	}
4816	sort_key->descending = descending;
4817	continue;
4818	}
4819
4820	for (j = 1, k = 1; j < hist_data->n_fields; j++) {
4821	unsigned int idx;
4822
4823	hist_field = hist_data->fields[j];
4824	if (hist_field->flags & HIST_FIELD_FL_VAR)
4825	continue;
4826
4827	idx = k++;
4828
4829	test_name = hist_field_name(hist_field, 0);
4830
4831	if (strcmp(field_name, test_name) == 0) {
4832	sort_key->field_idx = idx;
4833	descending = is_descending(tr, field_str);
4834	if (descending < 0) {
4835	ret = descending;
4836	goto out;
4837	}
4838	sort_key->descending = descending;
4839	break;
4840	}
4841	}
4842	if (j == hist_data->n_fields) {
4843	ret = -EINVAL;
4844	hist_err(tr, HIST_ERR_INVALID_SORT_FIELD, errpos(field_name));
4845	break;
4846	}
4847	}
4848
4849	hist_data->n_sort_keys = i;
4850	out:
4851	return ret;
4852	}
4853
4854	static void destroy_actions(struct hist_trigger_data *hist_data)
4855	{
4856	unsigned int i;
4857
4858	for (i = 0; i < hist_data->n_actions; i++) {
4859	struct action_data *data = hist_data->actions[i];
4860
4861	if (data->handler == HANDLER_ONMATCH)
4862	onmatch_destroy(data);
4863	else if (data->handler == HANDLER_ONMAX \|\|
4864	data->handler == HANDLER_ONCHANGE)
4865	track_data_destroy(hist_data, data);
4866	else
4867	kfree(data);
4868	}
4869	}
4870
4871	static int parse_actions(struct hist_trigger_data *hist_data)
4872	{
4873	struct trace_array *tr = hist_data->event_file->tr;
4874	struct action_data *data;
4875	unsigned int i;
4876	int ret = 0;
4877	char *str;
4878	int len;
4879
4880	for (i = 0; i < hist_data->attrs->n_actions; i++) {
4881	enum handler_id hid = 0;
4882	char *action_str;
4883
4884	str = hist_data->attrs->action_str[i];
4885
4886	if ((len = str_has_prefix(str, "onmatch(")))
4887	hid = HANDLER_ONMATCH;
4888	else if ((len = str_has_prefix(str, "onmax(")))
4889	hid = HANDLER_ONMAX;
4890	else if ((len = str_has_prefix(str, "onchange(")))
4891	hid = HANDLER_ONCHANGE;
4892
4893	action_str = str + len;
4894
4895	switch (hid) {
4896	case HANDLER_ONMATCH:
4897	data = onmatch_parse(tr, action_str);
4898	break;
4899	case HANDLER_ONMAX:
4900	case HANDLER_ONCHANGE:
4901	data = track_data_parse(hist_data, action_str, hid);
4902	break;
4903	default:
4904	data = ERR_PTR(-EINVAL);
4905	break;
4906	}
4907
4908	if (IS_ERR(data)) {
4909	ret = PTR_ERR(data);
4910	break;
4911	}
4912
4913	hist_data->actions[hist_data->n_actions++] = data;
4914	}
4915
4916	return ret;
4917	}
4918
4919	static int create_actions(struct hist_trigger_data *hist_data)
4920	{
4921	struct action_data *data;
4922	unsigned int i;
4923	int ret = 0;
4924
4925	for (i = 0; i < hist_data->attrs->n_actions; i++) {
4926	data = hist_data->actions[i];
4927
4928	if (data->handler == HANDLER_ONMATCH) {
4929	ret = onmatch_create(hist_data, data);
4930	if (ret)
4931	break;
4932	} else if (data->handler == HANDLER_ONMAX \|\|
4933	data->handler == HANDLER_ONCHANGE) {
4934	ret = track_data_create(hist_data, data);
4935	if (ret)
4936	break;
4937	} else {
4938	ret = -EINVAL;
4939	break;
4940	}
4941	}
4942
4943	return ret;
4944	}
4945
4946	static void print_actions(struct seq_file *m,
4947	struct hist_trigger_data *hist_data,
4948	struct tracing_map_elt *elt)
4949	{
4950	unsigned int i;
4951
4952	for (i = 0; i < hist_data->n_actions; i++) {
4953	struct action_data *data = hist_data->actions[i];
4954
4955	if (data->action == ACTION_SNAPSHOT)
4956	continue;
4957
4958	if (data->handler == HANDLER_ONMAX \|\|
4959	data->handler == HANDLER_ONCHANGE)
4960	track_data_print(m, hist_data, elt, data);
4961	}
4962	}
4963
4964	static void print_action_spec(struct seq_file *m,
4965	struct hist_trigger_data *hist_data,
4966	struct action_data *data)
4967	{
4968	unsigned int i;
4969
4970	if (data->action == ACTION_SAVE) {
4971	for (i = 0; i < hist_data->n_save_vars; i++) {
4972	seq_printf(m, "%s", hist_data->save_vars[i]->var->var.name);
4973	if (i < hist_data->n_save_vars - 1)
4974	seq_puts(m, ",");
4975	}
4976	} else if (data->action == ACTION_TRACE) {
4977	if (data->use_trace_keyword)
4978	seq_printf(m, "%s", data->synth_event_name);
4979	for (i = 0; i < data->n_params; i++) {
4980	if (i \|\| data->use_trace_keyword)
4981	seq_puts(m, ",");
4982	seq_printf(m, "%s", data->params[i]);
4983	}
4984	}
4985	}
4986
4987	static void print_track_data_spec(struct seq_file *m,
4988	struct hist_trigger_data *hist_data,
4989	struct action_data *data)
4990	{
4991	if (data->handler == HANDLER_ONMAX)
4992	seq_puts(m, ":onmax(");
4993	else if (data->handler == HANDLER_ONCHANGE)
4994	seq_puts(m, ":onchange(");
4995	seq_printf(m, "%s", data->track_data.var_str);
4996	seq_printf(m, ").%s(", data->action_name);
4997
4998	print_action_spec(m, hist_data, data);
4999
5000	seq_puts(m, ")");
5001	}
5002
5003	static void print_onmatch_spec(struct seq_file *m,
5004	struct hist_trigger_data *hist_data,
5005	struct action_data *data)
5006	{
5007	seq_printf(m, ":onmatch(%s.%s).", data->match_data.event_system,
5008	data->match_data.event);
5009
5010	seq_printf(m, "%s(", data->action_name);
5011
5012	print_action_spec(m, hist_data, data);
5013
5014	seq_puts(m, ")");
5015	}
5016
5017	static bool actions_match(struct hist_trigger_data *hist_data,
5018	struct hist_trigger_data *hist_data_test)
5019	{
5020	unsigned int i, j;
5021
5022	if (hist_data->n_actions != hist_data_test->n_actions)
5023	return false;
5024
5025	for (i = 0; i < hist_data->n_actions; i++) {
5026	struct action_data *data = hist_data->actions[i];
5027	struct action_data *data_test = hist_data_test->actions[i];
5028	char action_name, action_name_test;
5029
5030	if (data->handler != data_test->handler)
5031	return false;
5032	if (data->action != data_test->action)
5033	return false;
5034
5035	if (data->n_params != data_test->n_params)
5036	return false;
5037
5038	for (j = 0; j < data->n_params; j++) {
5039	if (strcmp(data->params[j], data_test->params[j]) != 0)
5040	return false;
5041	}
5042
5043	if (data->use_trace_keyword)
5044	action_name = data->synth_event_name;
5045	else
5046	action_name = data->action_name;
5047
5048	if (data_test->use_trace_keyword)
5049	action_name_test = data_test->synth_event_name;
5050	else
5051	action_name_test = data_test->action_name;
5052
5053	if (strcmp(action_name, action_name_test) != 0)
5054	return false;
5055
5056	if (data->handler == HANDLER_ONMATCH) {
5057	if (strcmp(data->match_data.event_system,
5058	data_test->match_data.event_system) != 0)
5059	return false;
5060	if (strcmp(data->match_data.event,
5061	data_test->match_data.event) != 0)
5062	return false;
5063	} else if (data->handler == HANDLER_ONMAX \|\|
5064	data->handler == HANDLER_ONCHANGE) {
5065	if (strcmp(data->track_data.var_str,
5066	data_test->track_data.var_str) != 0)
5067	return false;
5068	}
5069	}
5070
5071	return true;
5072	}
5073
5074
5075	static void print_actions_spec(struct seq_file *m,
5076	struct hist_trigger_data *hist_data)
5077	{
5078	unsigned int i;
5079
5080	for (i = 0; i < hist_data->n_actions; i++) {
5081	struct action_data *data = hist_data->actions[i];
5082
5083	if (data->handler == HANDLER_ONMATCH)
5084	print_onmatch_spec(m, hist_data, data);
5085	else if (data->handler == HANDLER_ONMAX \|\|
5086	data->handler == HANDLER_ONCHANGE)
5087	print_track_data_spec(m, hist_data, data);
5088	}
5089	}
5090
5091	static void destroy_field_var_hists(struct hist_trigger_data *hist_data)
5092	{
5093	unsigned int i;
5094
5095	for (i = 0; i < hist_data->n_field_var_hists; i++) {
5096	kfree(hist_data->field_var_hists[i]->cmd);
5097	kfree(hist_data->field_var_hists[i]);
5098	}
5099	}
5100
5101	static void destroy_hist_data(struct hist_trigger_data *hist_data)
5102	{
5103	if (!hist_data)
5104	return;
5105
5106	destroy_hist_trigger_attrs(hist_data->attrs);
5107	destroy_hist_fields(hist_data);
5108	tracing_map_destroy(hist_data->map);
5109
5110	destroy_actions(hist_data);
5111	destroy_field_vars(hist_data);
5112	destroy_field_var_hists(hist_data);
5113
5114	kfree(hist_data);
5115	}
5116
5117	static int create_tracing_map_fields(struct hist_trigger_data *hist_data)
5118	{
5119	struct tracing_map *map = hist_data->map;
5120	struct ftrace_event_field *field;
5121	struct hist_field *hist_field;
5122	int i, idx = 0;
5123
5124	for_each_hist_field(i, hist_data) {
5125	hist_field = hist_data->fields[i];
5126	if (hist_field->flags & HIST_FIELD_FL_KEY) {
5127	tracing_map_cmp_fn_t cmp_fn;
5128
5129	field = hist_field->field;
5130
5131	if (hist_field->flags & HIST_FIELD_FL_STACKTRACE)
5132	cmp_fn = tracing_map_cmp_none;
5133	else if (!field \|\| hist_field->flags & HIST_FIELD_FL_CPU)
5134	cmp_fn = tracing_map_cmp_num(hist_field->size,
5135	hist_field->is_signed);
5136	else if (is_string_field(field))
5137	cmp_fn = tracing_map_cmp_string;
5138	else
5139	cmp_fn = tracing_map_cmp_num(field->size,
5140	field->is_signed);
5141	idx = tracing_map_add_key_field(map,
5142	hist_field->offset,
5143	cmp_fn);
5144	} else if (!(hist_field->flags & HIST_FIELD_FL_VAR))
5145	idx = tracing_map_add_sum_field(map);
5146
5147	if (idx < 0)
5148	return idx;
5149
5150	if (hist_field->flags & HIST_FIELD_FL_VAR) {
5151	idx = tracing_map_add_var(map);
5152	if (idx < 0)
5153	return idx;
5154	hist_field->var.idx = idx;
5155	hist_field->var.hist_data = hist_data;
5156	}
5157	}
5158
5159	return 0;
5160	}
5161
5162	static struct hist_trigger_data *
5163	create_hist_data(unsigned int map_bits,
5164	struct hist_trigger_attrs *attrs,
5165	struct trace_event_file *file,
5166	bool remove)
5167	{
5168	const struct tracing_map_ops *map_ops = NULL;
5169	struct hist_trigger_data *hist_data;
5170	int ret = 0;
5171
5172	hist_data = kzalloc_obj(*hist_data);
5173	if (!hist_data)
5174	return ERR_PTR(-ENOMEM);
5175
5176	hist_data->attrs = attrs;
5177	hist_data->remove = remove;
5178	hist_data->event_file = file;
5179
5180	ret = parse_actions(hist_data);
5181	if (ret)
5182	goto free;
5183
5184	ret = create_hist_fields(hist_data, file);
5185	if (ret)
5186	goto free;
5187
5188	ret = create_sort_keys(hist_data);
5189	if (ret)
5190	goto free;
5191
5192	map_ops = &hist_trigger_elt_data_ops;
5193
5194	hist_data->map = tracing_map_create(map_bits, hist_data->key_size,
5195	map_ops, hist_data);
5196	if (IS_ERR(hist_data->map)) {
5197	ret = PTR_ERR(hist_data->map);
5198	hist_data->map = NULL;
5199	goto free;
5200	}
5201
5202	ret = create_tracing_map_fields(hist_data);
5203	if (ret)
5204	goto free;
5205	out:
5206	return hist_data;
5207	free:
5208	hist_data->attrs = NULL;
5209
5210	destroy_hist_data(hist_data);
5211
5212	hist_data = ERR_PTR(ret);
5213
5214	goto out;
5215	}
5216
5217	static void hist_trigger_elt_update(struct hist_trigger_data *hist_data,
5218	struct tracing_map_elt *elt,
5219	struct trace_buffer buffer, void rec,
5220	struct ring_buffer_event *rbe,
5221	u64 *var_ref_vals)
5222	{
5223	struct hist_elt_data *elt_data;
5224	struct hist_field *hist_field;
5225	unsigned int i, var_idx;
5226	u64 hist_val;
5227
5228	elt_data = elt->private_data;
5229	elt_data->var_ref_vals = var_ref_vals;
5230
5231	for_each_hist_val_field(i, hist_data) {
5232	hist_field = hist_data->fields[i];
5233	hist_val = hist_fn_call(hist_field, elt, buffer, rbe, rec);
5234	if (hist_field->flags & HIST_FIELD_FL_VAR) {
5235	var_idx = hist_field->var.idx;
5236
5237	if (!(hist_field->flags & HIST_FIELD_FL_ALIAS) &&
5238	hist_field->flags &
5239	(HIST_FIELD_FL_STRING \| HIST_FIELD_FL_STACKTRACE)) {
5240	unsigned int str_start, var_str_idx, idx;
5241	char str, val_str;
5242	unsigned int size;
5243
5244	str_start = hist_data->n_field_var_str +
5245	hist_data->n_save_var_str;
5246	var_str_idx = hist_field->var_str_idx;
5247	idx = str_start + var_str_idx;
5248
5249	str = elt_data->field_var_str[idx];
5250	val_str = (char *)(uintptr_t)hist_val;
5251
5252	if (hist_field->flags & HIST_FIELD_FL_STRING) {
5253	size = min(hist_field->size, STR_VAR_LEN_MAX);
5254	strscpy(str, val_str, size);
5255	} else {
5256	char *stack_start = str + sizeof(unsigned long);
5257	int e;
5258
5259	e = stack_trace_save((void *)stack_start,
5260	HIST_STACKTRACE_DEPTH,
5261	HIST_STACKTRACE_SKIP);
5262	if (e < HIST_STACKTRACE_DEPTH - 1)
5263	((unsigned long *)stack_start)[e] = 0;
5264	((unsigned long )str) = e;
5265	}
5266	hist_val = (u64)(uintptr_t)str;
5267	}
5268	tracing_map_set_var(elt, var_idx, hist_val);
5269	continue;
5270	}
5271	tracing_map_update_sum(elt, i, hist_val);
5272	}
5273
5274	for_each_hist_key_field(i, hist_data) {
5275	hist_field = hist_data->fields[i];
5276	if (hist_field->flags & HIST_FIELD_FL_VAR) {
5277	hist_val = hist_fn_call(hist_field, elt, buffer, rbe, rec);
5278	var_idx = hist_field->var.idx;
5279	tracing_map_set_var(elt, var_idx, hist_val);
5280	}
5281	}
5282
5283	update_field_vars(hist_data, elt, buffer, rbe, rec);
5284	}
5285
5286	static inline void add_to_key(char compound_key, void key,
5287	struct hist_field key_field, void rec)
5288	{
5289	size_t size = key_field->size;
5290
5291	if (key_field->flags & HIST_FIELD_FL_STRING) {
5292
5293	if (key_field->flags & HIST_FIELD_FL_COMM) {
5294	size = strlen((char *)key);
5295	} else {
5296	struct ftrace_event_field *field;
5297
5298	field = key_field->field;
5299	if (field->filter_type == FILTER_DYN_STRING \|\|
5300	field->filter_type == FILTER_RDYN_STRING)
5301	size = (u32 )(rec + field->offset) >> 16;
5302	else if (field->filter_type == FILTER_STATIC_STRING)
5303	size = field->size;
5304	}
5305
5306	/* ensure NULL-termination */
5307	if (size > key_field->size - 1)
5308	size = key_field->size - 1;
5309	}
5310	memcpy(compound_key + key_field->offset, key, size);
5311	}
5312
5313	static void
5314	hist_trigger_actions(struct hist_trigger_data *hist_data,
5315	struct tracing_map_elt *elt,
5316	struct trace_buffer buffer, void rec,
5317	struct ring_buffer_event rbe, void key,
5318	u64 *var_ref_vals)
5319	{
5320	struct action_data *data;
5321	unsigned int i;
5322
5323	for (i = 0; i < hist_data->n_actions; i++) {
5324	data = hist_data->actions[i];
5325	data->fn(hist_data, elt, buffer, rec, rbe, key, data, var_ref_vals);
5326	}
5327	}
5328
5329	/*
5330	* The hist_pad structure is used to save information to create
5331	* a histogram from the histogram trigger. It's too big to store
5332	* on the stack, so when the histogram trigger is initialized
5333	* a percpu array of 4 hist_pad structures is allocated.
5334	* This will cover every context from normal, softirq, irq and NMI
5335	* in the very unlikely event that a trigger happens at each of
5336	* these contexts and interrupts a currently active trigger.
5337	*/
5338	struct hist_pad {
5339	unsigned long entries[HIST_STACKTRACE_DEPTH];
5340	u64 var_ref_vals[TRACING_MAP_VARS_MAX];
5341	char compound_key[HIST_KEY_SIZE_MAX];
5342	};
5343
5344	static struct hist_pad __percpu *hist_pads;
5345	static DEFINE_PER_CPU(int, hist_pad_cnt);
5346	static refcount_t hist_pad_ref;
5347
5348	/* One hist_pad for every context (normal, softirq, irq, NMI) */
5349	#define MAX_HIST_CNT 4
5350
5351	static int alloc_hist_pad(void)
5352	{
5353	lockdep_assert_held(&event_mutex);
5354
5355	if (refcount_read(&hist_pad_ref)) {
5356	refcount_inc(&hist_pad_ref);
5357	return 0;
5358	}
5359
5360	hist_pads = __alloc_percpu(sizeof(struct hist_pad) * MAX_HIST_CNT,
5361	__alignof__(struct hist_pad));
5362	if (!hist_pads)
5363	return -ENOMEM;
5364
5365	refcount_set(&hist_pad_ref, 1);
5366	return 0;
5367	}
5368
5369	static void free_hist_pad(void)
5370	{
5371	lockdep_assert_held(&event_mutex);
5372
5373	if (!refcount_dec_and_test(&hist_pad_ref))
5374	return;
5375
5376	free_percpu(hist_pads);
5377	hist_pads = NULL;
5378	}
5379
5380	static struct hist_pad *get_hist_pad(void)
5381	{
5382	struct hist_pad *hist_pad;
5383	int cnt;
5384
5385	if (WARN_ON_ONCE(!hist_pads))
5386	return NULL;
5387
5388	preempt_disable();
5389
5390	hist_pad = per_cpu_ptr(hist_pads, smp_processor_id());
5391
5392	if (this_cpu_read(hist_pad_cnt) == MAX_HIST_CNT) {
5393	preempt_enable();
5394	return NULL;
5395	}
5396
5397	cnt = this_cpu_inc_return(hist_pad_cnt) - 1;
5398
5399	return &hist_pad[cnt];
5400	}
5401
5402	static void put_hist_pad(void)
5403	{
5404	this_cpu_dec(hist_pad_cnt);
5405	preempt_enable();
5406	}
5407
5408	static void event_hist_trigger(struct event_trigger_data *data,
5409	struct trace_buffer buffer, void rec,
5410	struct ring_buffer_event *rbe)
5411	{
5412	struct hist_trigger_data *hist_data = data->private_data;
5413	bool use_compound_key = (hist_data->n_keys > 1);
5414	struct tracing_map_elt *elt = NULL;
5415	struct hist_field *key_field;
5416	struct hist_pad *hist_pad;
5417	u64 field_contents;
5418	void *key = NULL;
5419	unsigned int i;
5420
5421	if (unlikely(!rbe))
5422	return;
5423
5424	hist_pad = get_hist_pad();
5425	if (!hist_pad)
5426	return;
5427
5428	memset(hist_pad->compound_key, 0, hist_data->key_size);
5429
5430	for_each_hist_key_field(i, hist_data) {
5431	key_field = hist_data->fields[i];
5432
5433	if (key_field->flags & HIST_FIELD_FL_STACKTRACE) {
5434	unsigned long *entries = hist_pad->entries;
5435
5436	memset(entries, 0, HIST_STACKTRACE_SIZE);
5437	if (key_field->field) {
5438	unsigned long *stack, n_entries;
5439
5440	field_contents = hist_fn_call(key_field, elt, buffer, rbe, rec);
5441	stack = (unsigned long *)(long)field_contents;
5442	n_entries = *stack;
5443	memcpy(entries, ++stack, n_entries * sizeof(unsigned long));
5444	} else {
5445	stack_trace_save(entries, HIST_STACKTRACE_DEPTH,
5446	HIST_STACKTRACE_SKIP);
5447	}
5448	key = entries;
5449	} else {
5450	field_contents = hist_fn_call(key_field, elt, buffer, rbe, rec);
5451	if (key_field->flags & HIST_FIELD_FL_STRING) {
5452	key = (void *)(unsigned long)field_contents;
5453	use_compound_key = true;
5454	} else
5455	key = (void *)&field_contents;
5456	}
5457
5458	if (use_compound_key)
5459	add_to_key(hist_pad->compound_key, key, key_field, rec);
5460	}
5461
5462	if (use_compound_key)
5463	key = hist_pad->compound_key;
5464
5465	if (hist_data->n_var_refs &&
5466	!resolve_var_refs(hist_data, key, hist_pad->var_ref_vals, false))
5467	goto out;
5468
5469	elt = tracing_map_insert(hist_data->map, key);
5470	if (!elt)
5471	goto out;
5472
5473	hist_trigger_elt_update(hist_data, elt, buffer, rec, rbe, hist_pad->var_ref_vals);
5474
5475	if (resolve_var_refs(hist_data, key, hist_pad->var_ref_vals, true)) {
5476	hist_trigger_actions(hist_data, elt, buffer, rec, rbe,
5477	key, hist_pad->var_ref_vals);
5478	}
5479
5480	hist_poll_wakeup();
5481
5482	out:
5483	put_hist_pad();
5484	}
5485
5486	static void hist_trigger_stacktrace_print(struct seq_file *m,
5487	unsigned long *stacktrace_entries,
5488	unsigned int max_entries)
5489	{
5490	unsigned int spaces = 8;
5491	unsigned int i;
5492
5493	for (i = 0; i < max_entries; i++) {
5494	if (!stacktrace_entries[i])
5495	return;
5496
5497	seq_printf(m, "%*c", 1 + spaces, ' ');
5498	seq_printf(m, "%pS\n", (void*)stacktrace_entries[i]);
5499	}
5500	}
5501
5502	static void hist_trigger_print_key(struct seq_file *m,
5503	struct hist_trigger_data *hist_data,
5504	void *key,
5505	struct tracing_map_elt *elt)
5506	{
5507	struct hist_field *key_field;
5508	bool multiline = false;
5509	const char *field_name;
5510	unsigned int i;
5511	u64 uval;
5512
5513	seq_puts(m, "{ ");
5514
5515	for_each_hist_key_field(i, hist_data) {
5516	key_field = hist_data->fields[i];
5517
5518	if (i > hist_data->n_vals)
5519	seq_puts(m, ", ");
5520
5521	field_name = hist_field_name(key_field, 0);
5522
5523	if (key_field->flags & HIST_FIELD_FL_HEX) {
5524	uval = (u64 )(key + key_field->offset);
5525	seq_printf(m, "%s: %llx", field_name, uval);
5526	} else if (key_field->flags & HIST_FIELD_FL_SYM) {
5527	uval = (u64 )(key + key_field->offset);
5528	seq_printf(m, "%s: [%llx] %-45ps", field_name,
5529	uval, (void *)(uintptr_t)uval);
5530	} else if (key_field->flags & HIST_FIELD_FL_SYM_OFFSET) {
5531	uval = (u64 )(key + key_field->offset);
5532	seq_printf(m, "%s: [%llx] %-55pS", field_name,
5533	uval, (void *)(uintptr_t)uval);
5534	} else if (key_field->flags & HIST_FIELD_FL_EXECNAME) {
5535	struct hist_elt_data *elt_data = elt->private_data;
5536	char *comm;
5537
5538	if (WARN_ON_ONCE(!elt_data))
5539	return;
5540
5541	comm = elt_data->comm;
5542
5543	uval = (u64 )(key + key_field->offset);
5544	seq_printf(m, "%s: %-16s[%10llu]", field_name,
5545	comm, uval);
5546	} else if (key_field->flags & HIST_FIELD_FL_SYSCALL) {
5547	const char *syscall_name;
5548
5549	uval = (u64 )(key + key_field->offset);
5550	syscall_name = get_syscall_name(uval);
5551	if (!syscall_name)
5552	syscall_name = "unknown_syscall";
5553
5554	seq_printf(m, "%s: %-30s[%3llu]", field_name,
5555	syscall_name, uval);
5556	} else if (key_field->flags & HIST_FIELD_FL_STACKTRACE) {
5557	if (key_field->field)
5558	seq_printf(m, "%s.stacktrace", key_field->field->name);
5559	else
5560	seq_puts(m, "common_stacktrace:\n");
5561	hist_trigger_stacktrace_print(m,
5562	key + key_field->offset,
5563	HIST_STACKTRACE_DEPTH);
5564	multiline = true;
5565	} else if (key_field->flags & HIST_FIELD_FL_LOG2) {
5566	seq_printf(m, "%s: ~ 2^%-2llu", field_name,
5567	(u64 )(key + key_field->offset));
5568	} else if (key_field->flags & HIST_FIELD_FL_BUCKET) {
5569	unsigned long buckets = key_field->buckets;
5570	uval = (u64 )(key + key_field->offset);
5571	seq_printf(m, "%s: ~ %llu-%llu", field_name,
5572	uval, uval + buckets -1);
5573	} else if (key_field->flags & HIST_FIELD_FL_STRING) {
5574	seq_printf(m, "%s: %-50s", field_name,
5575	(char *)(key + key_field->offset));
5576	} else {
5577	uval = (u64 )(key + key_field->offset);
5578	seq_printf(m, "%s: %10llu", field_name, uval);
5579	}
5580	}
5581
5582	if (!multiline)
5583	seq_puts(m, " ");
5584
5585	seq_puts(m, "}");
5586	}
5587
5588	/* Get the 100 times of the percentage of @val in @total */
5589	static inline unsigned int __get_percentage(u64 val, u64 total)
5590	{
5591	if (!total)
5592	goto div0;
5593
5594	if (val < (U64_MAX / 10000))
5595	return (unsigned int)div64_ul(val * 10000, total);
5596
5597	total = div64_u64(total, 10000);
5598	if (!total)
5599	goto div0;
5600
5601	return (unsigned int)div64_ul(val, total);
5602	div0:
5603	return val ? UINT_MAX : 0;
5604	}
5605
5606	#define BAR_CHAR '#'
5607
5608	static inline const char __fill_bar_str(char buf, int size, u64 val, u64 max)
5609	{
5610	unsigned int len = __get_percentage(val, max);
5611	int i;
5612
5613	if (len == UINT_MAX) {
5614	snprintf(buf, size, "[ERROR]");
5615	return buf;
5616	}
5617
5618	len = len * size / 10000;
5619	for (i = 0; i < len && i < size; i++)
5620	buf[i] = BAR_CHAR;
5621	while (i < size)
5622	buf[i++] = ' ';
5623	buf[size] = '\0';
5624
5625	return buf;
5626	}
5627
5628	struct hist_val_stat {
5629	u64 max;
5630	u64 total;
5631	};
5632
5633	static void hist_trigger_print_val(struct seq_file *m, unsigned int idx,
5634	const char *field_name, unsigned long flags,
5635	struct hist_val_stat *stats,
5636	struct tracing_map_elt *elt)
5637	{
5638	u64 val = tracing_map_read_sum(elt, idx);
5639	unsigned int pc;
5640	char bar[21];
5641
5642	if (flags & HIST_FIELD_FL_PERCENT) {
5643	pc = __get_percentage(val, stats[idx].total);
5644	if (pc == UINT_MAX)
5645	seq_printf(m, " %s (%%):[ERROR]", field_name);
5646	else
5647	seq_printf(m, " %s (%%): %3u.%02u", field_name,
5648	pc / 100, pc % 100);
5649	} else if (flags & HIST_FIELD_FL_GRAPH) {
5650	seq_printf(m, " %s: %20s", field_name,
5651	__fill_bar_str(bar, 20, val, stats[idx].max));
5652	} else if (flags & HIST_FIELD_FL_HEX) {
5653	seq_printf(m, " %s: %10llx", field_name, val);
5654	} else {
5655	seq_printf(m, " %s: %10llu", field_name, val);
5656	}
5657	}
5658
5659	static void hist_trigger_entry_print(struct seq_file *m,
5660	struct hist_trigger_data *hist_data,
5661	struct hist_val_stat *stats,
5662	void *key,
5663	struct tracing_map_elt *elt)
5664	{
5665	const char *field_name;
5666	unsigned int i = HITCOUNT_IDX;
5667	unsigned long flags;
5668
5669	hist_trigger_print_key(m, hist_data, key, elt);
5670
5671	/* At first, show the raw hitcount if !nohitcount */
5672	if (!hist_data->attrs->no_hitcount)
5673	hist_trigger_print_val(m, i, "hitcount", 0, stats, elt);
5674
5675	for (i = 1; i < hist_data->n_vals; i++) {
5676	field_name = hist_field_name(hist_data->fields[i], 0);
5677	flags = hist_data->fields[i]->flags;
5678	if (flags & HIST_FIELD_FL_VAR \|\| flags & HIST_FIELD_FL_EXPR)
5679	continue;
5680
5681	seq_puts(m, " ");
5682	hist_trigger_print_val(m, i, field_name, flags, stats, elt);
5683	}
5684
5685	print_actions(m, hist_data, elt);
5686
5687	seq_puts(m, "\n");
5688	}
5689
5690	static int print_entries(struct seq_file *m,
5691	struct hist_trigger_data *hist_data)
5692	{
5693	struct tracing_map_sort_entry **sort_entries = NULL;
5694	struct tracing_map *map = hist_data->map;
5695	int i, j, n_entries, ret;
5696	struct hist_val_stat *stats = NULL;
5697	u64 val;
5698
5699	n_entries = tracing_map_sort_entries(map, hist_data->sort_keys,
5700	hist_data->n_sort_keys,
5701	&sort_entries);
5702	if (n_entries < 0)
5703	return n_entries;
5704
5705	ret = n_entries;
5706
5707	/* Calculate the max and the total for each field if needed. */
5708	for (j = 0; j < hist_data->n_vals; j++) {
5709	if (!(hist_data->fields[j]->flags &
5710	(HIST_FIELD_FL_PERCENT \| HIST_FIELD_FL_GRAPH)))
5711	continue;
5712	if (!stats) {
5713	stats = kzalloc_objs(*stats, hist_data->n_vals);
5714	if (!stats) {
5715	ret = -ENOMEM;
5716	goto out;
5717	}
5718	}
5719	for (i = 0; i < n_entries; i++) {
5720	val = tracing_map_read_sum(sort_entries[i]->elt, j);
5721	stats[j].total += val;
5722	if (stats[j].max < val)
5723	stats[j].max = val;
5724	}
5725	}
5726
5727	for (i = 0; i < n_entries; i++)
5728	hist_trigger_entry_print(m, hist_data, stats,
5729	sort_entries[i]->key,
5730	sort_entries[i]->elt);
5731
5732	kfree(stats);
5733	out:
5734	tracing_map_destroy_sort_entries(sort_entries, n_entries);
5735
5736	return ret;
5737	}
5738
5739	static void hist_trigger_show(struct seq_file *m,
5740	struct event_trigger_data *data, int n)
5741	{
5742	struct hist_trigger_data *hist_data;
5743	int n_entries;
5744
5745	if (n > 0)
5746	seq_puts(m, "\n\n");
5747
5748	seq_puts(m, "# event histogram\n#\n# trigger info: ");
5749	data->cmd_ops->print(m, data);
5750	seq_puts(m, "#\n\n");
5751
5752	hist_data = data->private_data;
5753	n_entries = print_entries(m, hist_data);
5754	if (n_entries < 0)
5755	n_entries = 0;
5756
5757	track_data_snapshot_print(m, hist_data);
5758
5759	seq_printf(m, "\nTotals:\n Hits: %llu\n Entries: %u\n Dropped: %llu\n",
5760	(u64)atomic64_read(&hist_data->map->hits),
5761	n_entries, (u64)atomic64_read(&hist_data->map->drops));
5762	}
5763
5764	struct hist_file_data {
5765	struct file *file;
5766	u64 last_read;
5767	u64 last_act;
5768	};
5769
5770	static u64 get_hist_hit_count(struct trace_event_file *event_file)
5771	{
5772	struct hist_trigger_data *hist_data;
5773	struct event_trigger_data *data;
5774	u64 ret = 0;
5775
5776	list_for_each_entry(data, &event_file->triggers, list) {
5777	if (data->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5778	hist_data = data->private_data;
5779	ret += atomic64_read(&hist_data->map->hits);
5780	}
5781	}
5782	return ret;
5783	}
5784
5785	static int hist_show(struct seq_file m, void v)
5786	{
5787	struct hist_file_data *hist_file = m->private;
5788	struct event_trigger_data *data;
5789	struct trace_event_file *event_file;
5790	int n = 0;
5791
5792	guard(mutex)(&event_mutex);
5793
5794	event_file = event_file_file(hist_file->file);
5795	if (unlikely(!event_file))
5796	return -ENODEV;
5797
5798	list_for_each_entry(data, &event_file->triggers, list) {
5799	if (data->cmd_ops->trigger_type == ETT_EVENT_HIST)
5800	hist_trigger_show(m, data, n++);
5801	}
5802	hist_file->last_read = get_hist_hit_count(event_file);
5803	/*
5804	* Update last_act too so that poll()/POLLPRI can wait for the next
5805	* event after any syscall on hist file.
5806	*/
5807	hist_file->last_act = hist_file->last_read;
5808
5809	return 0;
5810	}
5811
5812	static __poll_t event_hist_poll(struct file file, struct poll_table_struct wait)
5813	{
5814	struct trace_event_file *event_file;
5815	struct seq_file *m = file->private_data;
5816	struct hist_file_data *hist_file = m->private;
5817	__poll_t ret = 0;
5818	u64 cnt;
5819
5820	guard(mutex)(&event_mutex);
5821
5822	event_file = event_file_file(file);
5823	if (!event_file)
5824	return EPOLLERR;
5825
5826	hist_poll_wait(file, wait);
5827
5828	cnt = get_hist_hit_count(event_file);
5829	if (hist_file->last_read != cnt)
5830	ret \|= EPOLLIN \| EPOLLRDNORM;
5831	if (hist_file->last_act != cnt) {
5832	hist_file->last_act = cnt;
5833	ret \|= EPOLLPRI;
5834	}
5835
5836	return ret;
5837	}
5838
5839	static int event_hist_release(struct inode inode, struct file file)
5840	{
5841	struct seq_file *m = file->private_data;
5842	struct hist_file_data *hist_file = m->private;
5843
5844	kfree(hist_file);
5845	return tracing_single_release_file_tr(inode, file);
5846	}
5847
5848	static int event_hist_open(struct inode inode, struct file file)
5849	{
5850	struct trace_event_file *event_file;
5851	struct hist_file_data *hist_file;
5852	int ret;
5853
5854	ret = tracing_open_file_tr(inode, file);
5855	if (ret)
5856	return ret;
5857
5858	guard(mutex)(&event_mutex);
5859
5860	event_file = event_file_file(file);
5861	if (!event_file) {
5862	ret = -ENODEV;
5863	goto err;
5864	}
5865
5866	hist_file = kzalloc_obj(*hist_file);
5867	if (!hist_file) {
5868	ret = -ENOMEM;
5869	goto err;
5870	}
5871
5872	hist_file->file = file;
5873	hist_file->last_act = get_hist_hit_count(event_file);
5874
5875	ret = single_open(file, hist_show, hist_file);
5876	if (ret) {
5877	kfree(hist_file);
5878	goto err;
5879	}
5880
5881	return 0;
5882	err:
5883	tracing_release_file_tr(inode, file);
5884	return ret;
5885	}
5886
5887	const struct file_operations event_hist_fops = {
5888	.open = event_hist_open,
5889	.read = seq_read,
5890	.llseek = seq_lseek,
5891	.release = event_hist_release,
5892	.poll = event_hist_poll,
5893	};
5894
5895	#ifdef CONFIG_HIST_TRIGGERS_DEBUG
5896
5897	#undef C
5898	#define C(a, b) b
5899
5900	static const char * const field_funcs[] = { FIELD_FUNCS };
5901
5902	static void hist_field_debug_show_flags(struct seq_file *m,
5903	unsigned long flags)
5904	{
5905	seq_puts(m, " flags:\n");
5906
5907	if (flags & HIST_FIELD_FL_KEY)
5908	seq_puts(m, " HIST_FIELD_FL_KEY\n");
5909	else if (flags & HIST_FIELD_FL_HITCOUNT)
5910	seq_puts(m, " VAL: HIST_FIELD_FL_HITCOUNT\n");
5911	else if (flags & HIST_FIELD_FL_VAR)
5912	seq_puts(m, " HIST_FIELD_FL_VAR\n");
5913	else if (flags & HIST_FIELD_FL_VAR_REF)
5914	seq_puts(m, " HIST_FIELD_FL_VAR_REF\n");
5915	else
5916	seq_puts(m, " VAL: normal u64 value\n");
5917
5918	if (flags & HIST_FIELD_FL_ALIAS)
5919	seq_puts(m, " HIST_FIELD_FL_ALIAS\n");
5920	else if (flags & HIST_FIELD_FL_CONST)
5921	seq_puts(m, " HIST_FIELD_FL_CONST\n");
5922	}
5923
5924	static int hist_field_debug_show(struct seq_file *m,
5925	struct hist_field *field, unsigned long flags)
5926	{
5927	if ((field->flags & flags) != flags) {
5928	seq_printf(m, "ERROR: bad flags - %lx\n", flags);
5929	return -EINVAL;
5930	}
5931
5932	hist_field_debug_show_flags(m, field->flags);
5933	if (field->field)
5934	seq_printf(m, " ftrace_event_field name: %s\n",
5935	field->field->name);
5936
5937	if (field->flags & HIST_FIELD_FL_VAR) {
5938	seq_printf(m, " var.name: %s\n", field->var.name);
5939	seq_printf(m, " var.idx (into tracing_map_elt.vars[]): %u\n",
5940	field->var.idx);
5941	}
5942
5943	if (field->flags & HIST_FIELD_FL_CONST)
5944	seq_printf(m, " constant: %llu\n", field->constant);
5945
5946	if (field->flags & HIST_FIELD_FL_ALIAS)
5947	seq_printf(m, " var_ref_idx (into hist_data->var_refs[]): %u\n",
5948	field->var_ref_idx);
5949
5950	if (field->flags & HIST_FIELD_FL_VAR_REF) {
5951	seq_printf(m, " name: %s\n", field->name);
5952	seq_printf(m, " var.idx (into tracing_map_elt.vars[]): %u\n",
5953	field->var.idx);
5954	seq_printf(m, " var.hist_data: %p\n", field->var.hist_data);
5955	seq_printf(m, " var_ref_idx (into hist_data->var_refs[]): %u\n",
5956	field->var_ref_idx);
5957	if (field->system)
5958	seq_printf(m, " system: %s\n", field->system);
5959	if (field->event_name)
5960	seq_printf(m, " event_name: %s\n", field->event_name);
5961	}
5962
5963	seq_printf(m, " type: %s\n", field->type);
5964	seq_printf(m, " size: %u\n", field->size);
5965	seq_printf(m, " is_signed: %u\n", field->is_signed);
5966	seq_printf(m, " function: hist_field_%s()\n", field_funcs[field->fn_num]);
5967
5968	return 0;
5969	}
5970
5971	static int field_var_debug_show(struct seq_file *m,
5972	struct field_var *field_var, unsigned int i,
5973	bool save_vars)
5974	{
5975	const char *vars_name = save_vars ? "save_vars" : "field_vars";
5976	struct hist_field *field;
5977	int ret = 0;
5978
5979	seq_printf(m, "\n hist_data->%s[%d]:\n", vars_name, i);
5980
5981	field = field_var->var;
5982
5983	seq_printf(m, "\n %s[%d].var:\n", vars_name, i);
5984
5985	hist_field_debug_show_flags(m, field->flags);
5986	seq_printf(m, " var.name: %s\n", field->var.name);
5987	seq_printf(m, " var.idx (into tracing_map_elt.vars[]): %u\n",
5988	field->var.idx);
5989
5990	field = field_var->val;
5991
5992	seq_printf(m, "\n %s[%d].val:\n", vars_name, i);
5993	if (field->field)
5994	seq_printf(m, " ftrace_event_field name: %s\n",
5995	field->field->name);
5996	else {
5997	ret = -EINVAL;
5998	goto out;
5999	}
6000
6001	seq_printf(m, " type: %s\n", field->type);
6002	seq_printf(m, " size: %u\n", field->size);
6003	seq_printf(m, " is_signed: %u\n", field->is_signed);
6004	out:
6005	return ret;
6006	}
6007
6008	static int hist_action_debug_show(struct seq_file *m,
6009	struct action_data *data, int i)
6010	{
6011	int ret = 0;
6012
6013	if (data->handler == HANDLER_ONMAX \|\|
6014	data->handler == HANDLER_ONCHANGE) {
6015	seq_printf(m, "\n hist_data->actions[%d].track_data.var_ref:\n", i);
6016	ret = hist_field_debug_show(m, data->track_data.var_ref,
6017	HIST_FIELD_FL_VAR_REF);
6018	if (ret)
6019	goto out;
6020
6021	seq_printf(m, "\n hist_data->actions[%d].track_data.track_var:\n", i);
6022	ret = hist_field_debug_show(m, data->track_data.track_var,
6023	HIST_FIELD_FL_VAR);
6024	if (ret)
6025	goto out;
6026	}
6027
6028	if (data->handler == HANDLER_ONMATCH) {
6029	seq_printf(m, "\n hist_data->actions[%d].match_data.event_system: %s\n",
6030	i, data->match_data.event_system);
6031	seq_printf(m, " hist_data->actions[%d].match_data.event: %s\n",
6032	i, data->match_data.event);
6033	}
6034	out:
6035	return ret;
6036	}
6037
6038	static int hist_actions_debug_show(struct seq_file *m,
6039	struct hist_trigger_data *hist_data)
6040	{
6041	int i, ret = 0;
6042
6043	if (hist_data->n_actions)
6044	seq_puts(m, "\n action tracking variables (for onmax()/onchange()/onmatch()):\n");
6045
6046	for (i = 0; i < hist_data->n_actions; i++) {
6047	struct action_data *action = hist_data->actions[i];
6048
6049	ret = hist_action_debug_show(m, action, i);
6050	if (ret)
6051	goto out;
6052	}
6053
6054	if (hist_data->n_save_vars)
6055	seq_puts(m, "\n save action variables (save() params):\n");
6056
6057	for (i = 0; i < hist_data->n_save_vars; i++) {
6058	ret = field_var_debug_show(m, hist_data->save_vars[i], i, true);
6059	if (ret)
6060	goto out;
6061	}
6062	out:
6063	return ret;
6064	}
6065
6066	static void hist_trigger_debug_show(struct seq_file *m,
6067	struct event_trigger_data *data, int n)
6068	{
6069	struct hist_trigger_data *hist_data;
6070	int i, ret;
6071
6072	if (n > 0)
6073	seq_puts(m, "\n\n");
6074
6075	seq_puts(m, "# event histogram\n#\n# trigger info: ");
6076	data->cmd_ops->print(m, data);
6077	seq_puts(m, "#\n\n");
6078
6079	hist_data = data->private_data;
6080
6081	seq_printf(m, "hist_data: %p\n\n", hist_data);
6082	seq_printf(m, " n_vals: %u\n", hist_data->n_vals);
6083	seq_printf(m, " n_keys: %u\n", hist_data->n_keys);
6084	seq_printf(m, " n_fields: %u\n", hist_data->n_fields);
6085
6086	seq_puts(m, "\n val fields:\n\n");
6087
6088	seq_puts(m, " hist_data->fields[0]:\n");
6089	ret = hist_field_debug_show(m, hist_data->fields[0],
6090	HIST_FIELD_FL_HITCOUNT);
6091	if (ret)
6092	return;
6093
6094	for (i = 1; i < hist_data->n_vals; i++) {
6095	seq_printf(m, "\n hist_data->fields[%d]:\n", i);
6096	ret = hist_field_debug_show(m, hist_data->fields[i], 0);
6097	if (ret)
6098	return;
6099	}
6100
6101	seq_puts(m, "\n key fields:\n");
6102
6103	for (i = hist_data->n_vals; i < hist_data->n_fields; i++) {
6104	seq_printf(m, "\n hist_data->fields[%d]:\n", i);
6105	ret = hist_field_debug_show(m, hist_data->fields[i],
6106	HIST_FIELD_FL_KEY);
6107	if (ret)
6108	return;
6109	}
6110
6111	if (hist_data->n_var_refs)
6112	seq_puts(m, "\n variable reference fields:\n");
6113
6114	for (i = 0; i < hist_data->n_var_refs; i++) {
6115	seq_printf(m, "\n hist_data->var_refs[%d]:\n", i);
6116	ret = hist_field_debug_show(m, hist_data->var_refs[i],
6117	HIST_FIELD_FL_VAR_REF);
6118	if (ret)
6119	return;
6120	}
6121
6122	if (hist_data->n_field_vars)
6123	seq_puts(m, "\n field variables:\n");
6124
6125	for (i = 0; i < hist_data->n_field_vars; i++) {
6126	ret = field_var_debug_show(m, hist_data->field_vars[i], i, false);
6127	if (ret)
6128	return;
6129	}
6130
6131	ret = hist_actions_debug_show(m, hist_data);
6132	if (ret)
6133	return;
6134	}
6135
6136	static int hist_debug_show(struct seq_file m, void v)
6137	{
6138	struct event_trigger_data *data;
6139	struct trace_event_file *event_file;
6140	int n = 0;
6141
6142	guard(mutex)(&event_mutex);
6143
6144	event_file = event_file_file(m->private);
6145	if (unlikely(!event_file))
6146	return -ENODEV;
6147
6148	list_for_each_entry(data, &event_file->triggers, list) {
6149	if (data->cmd_ops->trigger_type == ETT_EVENT_HIST)
6150	hist_trigger_debug_show(m, data, n++);
6151	}
6152	return 0;
6153	}
6154
6155	static int event_hist_debug_open(struct inode inode, struct file file)
6156	{
6157	int ret;
6158
6159	ret = tracing_open_file_tr(inode, file);
6160	if (ret)
6161	return ret;
6162
6163	ret = single_open(file, hist_debug_show, file);
6164	if (ret)
6165	tracing_release_file_tr(inode, file);
6166	return ret;
6167	}
6168
6169	const struct file_operations event_hist_debug_fops = {
6170	.open = event_hist_debug_open,
6171	.read = seq_read,
6172	.llseek = seq_lseek,
6173	.release = tracing_single_release_file_tr,
6174	};
6175	#endif
6176
6177	static void hist_field_print(struct seq_file m, struct hist_field hist_field)
6178	{
6179	const char *field_name = hist_field_name(hist_field, 0);
6180
6181	if (hist_field->var.name)
6182	seq_printf(m, "%s=", hist_field->var.name);
6183
6184	if (hist_field->flags & HIST_FIELD_FL_CPU)
6185	seq_puts(m, "common_cpu");
6186	if (hist_field->flags & HIST_FIELD_FL_COMM)
6187	seq_puts(m, "common_comm");
6188	else if (hist_field->flags & HIST_FIELD_FL_CONST)
6189	seq_printf(m, "%llu", hist_field->constant);
6190	else if (field_name) {
6191	if (hist_field->flags & HIST_FIELD_FL_VAR_REF \|\|
6192	hist_field->flags & HIST_FIELD_FL_ALIAS)
6193	if (!hist_field->system)
6194	seq_putc(m, '$');
6195	seq_printf(m, "%s", field_name);
6196	} else if (hist_field->flags & HIST_FIELD_FL_TIMESTAMP)
6197	seq_puts(m, "common_timestamp");
6198
6199	if (hist_field->flags) {
6200	if (!(hist_field->flags & HIST_FIELD_FL_VAR_REF) &&
6201	!(hist_field->flags & HIST_FIELD_FL_EXPR) &&
6202	!(hist_field->flags & HIST_FIELD_FL_STACKTRACE)) {
6203	const char *flags = get_hist_field_flags(hist_field);
6204
6205	if (flags)
6206	seq_printf(m, ".%s", flags);
6207	}
6208	}
6209	if (hist_field->buckets)
6210	seq_printf(m, "=%ld", hist_field->buckets);
6211	}
6212
6213	static int event_hist_trigger_print(struct seq_file *m,
6214	struct event_trigger_data *data)
6215	{
6216	struct hist_trigger_data *hist_data = data->private_data;
6217	struct hist_field *field;
6218	bool have_var = false;
6219	bool show_val = false;
6220	unsigned int i;
6221
6222	seq_puts(m, HIST_PREFIX);
6223
6224	if (data->name)
6225	seq_printf(m, "%s:", data->name);
6226
6227	seq_puts(m, "keys=");
6228
6229	for_each_hist_key_field(i, hist_data) {
6230	field = hist_data->fields[i];
6231
6232	if (i > hist_data->n_vals)
6233	seq_puts(m, ",");
6234
6235	if (field->flags & HIST_FIELD_FL_STACKTRACE) {
6236	if (field->field)
6237	seq_printf(m, "%s.stacktrace", field->field->name);
6238	else
6239	seq_puts(m, "common_stacktrace");
6240	} else
6241	hist_field_print(m, field);
6242	}
6243
6244	seq_puts(m, ":vals=");
6245
6246	for_each_hist_val_field(i, hist_data) {
6247	field = hist_data->fields[i];
6248	if (field->flags & HIST_FIELD_FL_VAR) {
6249	have_var = true;
6250	continue;
6251	}
6252
6253	if (i == HITCOUNT_IDX) {
6254	if (hist_data->attrs->no_hitcount)
6255	continue;
6256	seq_puts(m, "hitcount");
6257	} else {
6258	if (show_val)
6259	seq_puts(m, ",");
6260	hist_field_print(m, field);
6261	}
6262	show_val = true;
6263	}
6264
6265	if (have_var) {
6266	unsigned int n = 0;
6267
6268	seq_puts(m, ":");
6269
6270	for_each_hist_val_field(i, hist_data) {
6271	field = hist_data->fields[i];
6272
6273	if (field->flags & HIST_FIELD_FL_VAR) {
6274	if (n++)
6275	seq_puts(m, ",");
6276	hist_field_print(m, field);
6277	}
6278	}
6279	}
6280
6281	seq_puts(m, ":sort=");
6282
6283	for (i = 0; i < hist_data->n_sort_keys; i++) {
6284	struct tracing_map_sort_key *sort_key;
6285	unsigned int idx, first_key_idx;
6286
6287	/* skip VAR vals */
6288	first_key_idx = hist_data->n_vals - hist_data->n_vars;
6289
6290	sort_key = &hist_data->sort_keys[i];
6291	idx = sort_key->field_idx;
6292
6293	if (WARN_ON(idx >= HIST_FIELDS_MAX))
6294	return -EINVAL;
6295
6296	if (i > 0)
6297	seq_puts(m, ",");
6298
6299	if (idx == HITCOUNT_IDX)
6300	seq_puts(m, "hitcount");
6301	else {
6302	if (idx >= first_key_idx)
6303	idx += hist_data->n_vars;
6304	hist_field_print(m, hist_data->fields[idx]);
6305	}
6306
6307	if (sort_key->descending)
6308	seq_puts(m, ".descending");
6309	}
6310	seq_printf(m, ":size=%u", (1 << hist_data->map->map_bits));
6311	if (hist_data->enable_timestamps)
6312	seq_printf(m, ":clock=%s", hist_data->attrs->clock);
6313	if (hist_data->attrs->no_hitcount)
6314	seq_puts(m, ":nohitcount");
6315
6316	print_actions_spec(m, hist_data);
6317
6318	if (data->filter_str)
6319	seq_printf(m, " if %s", data->filter_str);
6320
6321	if (data->paused)
6322	seq_puts(m, " [paused]");
6323	else
6324	seq_puts(m, " [active]");
6325
6326	seq_putc(m, '\n');
6327
6328	return 0;
6329	}
6330
6331	static int event_hist_trigger_init(struct event_trigger_data *data)
6332	{
6333	struct hist_trigger_data *hist_data = data->private_data;
6334
6335	if (alloc_hist_pad() < 0)
6336	return -ENOMEM;
6337
6338	if (!data->ref && hist_data->attrs->name)
6339	save_named_trigger(hist_data->attrs->name, data);
6340
6341	data->ref++;
6342
6343	return 0;
6344	}
6345
6346	static void unregister_field_var_hists(struct hist_trigger_data *hist_data)
6347	{
6348	struct trace_event_file *file;
6349	unsigned int i;
6350	char *cmd;
6351	int ret;
6352
6353	for (i = 0; i < hist_data->n_field_var_hists; i++) {
6354	file = hist_data->field_var_hists[i]->hist_data->event_file;
6355	cmd = hist_data->field_var_hists[i]->cmd;
6356	ret = event_hist_trigger_parse(&trigger_hist_cmd, file,
6357	"!hist", "hist", cmd);
6358	WARN_ON_ONCE(ret < 0);
6359	}
6360	}
6361
6362	static void event_hist_trigger_free(struct event_trigger_data *data)
6363	{
6364	struct hist_trigger_data *hist_data = data->private_data;
6365
6366	if (WARN_ON_ONCE(data->ref <= 0))
6367	return;
6368
6369	data->ref--;
6370	if (!data->ref) {
6371	if (data->name)
6372	del_named_trigger(data);
6373
6374	trigger_data_free(data);
6375
6376	tracepoint_synchronize_unregister();
6377	remove_hist_vars(hist_data);
6378
6379	unregister_field_var_hists(hist_data);
6380
6381	destroy_hist_data(hist_data);
6382	}
6383	free_hist_pad();
6384	}
6385
6386	static int event_hist_trigger_named_init(struct event_trigger_data *data)
6387	{
6388	int ret;
6389
6390	ret = event_hist_trigger_init(data->named_data);
6391	if (ret < 0) {
6392	kfree(data->cmd_ops);
6393	data->cmd_ops = &trigger_hist_cmd;
6394	return ret;
6395	}
6396
6397	data->ref++;
6398
6399	save_named_trigger(data->named_data->name, data);
6400
6401	return 0;
6402	}
6403
6404	static void event_hist_trigger_named_free(struct event_trigger_data *data)
6405	{
6406	if (WARN_ON_ONCE(data->ref <= 0))
6407	return;
6408
6409	event_hist_trigger_free(data->named_data);
6410
6411	data->ref--;
6412	if (!data->ref) {
6413	struct event_command *cmd_ops = data->cmd_ops;
6414
6415	del_named_trigger(data);
6416	trigger_data_free(data);
6417	tracepoint_synchronize_unregister();
6418	kfree(cmd_ops);
6419	}
6420	}
6421
6422	static void hist_clear(struct event_trigger_data *data)
6423	{
6424	struct hist_trigger_data *hist_data = data->private_data;
6425
6426	if (data->name)
6427	pause_named_trigger(data);
6428
6429	tracepoint_synchronize_unregister();
6430
6431	tracing_map_clear(hist_data->map);
6432
6433	if (data->name)
6434	unpause_named_trigger(data);
6435	}
6436
6437	static bool compatible_field(struct ftrace_event_field *field,
6438	struct ftrace_event_field *test_field)
6439	{
6440	if (field == test_field)
6441	return true;
6442	if (field == NULL \|\| test_field == NULL)
6443	return false;
6444	if (strcmp(field->name, test_field->name) != 0)
6445	return false;
6446	if (strcmp(field->type, test_field->type) != 0)
6447	return false;
6448	if (field->size != test_field->size)
6449	return false;
6450	if (field->is_signed != test_field->is_signed)
6451	return false;
6452
6453	return true;
6454	}
6455
6456	static bool hist_trigger_match(struct event_trigger_data *data,
6457	struct event_trigger_data *data_test,
6458	struct event_trigger_data *named_data,
6459	bool ignore_filter)
6460	{
6461	struct tracing_map_sort_key sort_key, sort_key_test;
6462	struct hist_trigger_data hist_data, hist_data_test;
6463	struct hist_field key_field, key_field_test;
6464	unsigned int i;
6465
6466	if (named_data && (named_data != data_test) &&
6467	(named_data != data_test->named_data))
6468	return false;
6469
6470	if (!named_data && is_named_trigger(data_test))
6471	return false;
6472
6473	hist_data = data->private_data;
6474	hist_data_test = data_test->private_data;
6475
6476	if (hist_data->n_vals != hist_data_test->n_vals \|\|
6477	hist_data->n_fields != hist_data_test->n_fields \|\|
6478	hist_data->n_sort_keys != hist_data_test->n_sort_keys)
6479	return false;
6480
6481	if (!ignore_filter) {
6482	if ((data->filter_str && !data_test->filter_str) \|\|
6483	(!data->filter_str && data_test->filter_str))
6484	return false;
6485	}
6486
6487	for_each_hist_field(i, hist_data) {
6488	key_field = hist_data->fields[i];
6489	key_field_test = hist_data_test->fields[i];
6490
6491	if (key_field->flags != key_field_test->flags)
6492	return false;
6493	if (!compatible_field(key_field->field, key_field_test->field))
6494	return false;
6495	if (key_field->offset != key_field_test->offset)
6496	return false;
6497	if (key_field->size != key_field_test->size)
6498	return false;
6499	if (key_field->is_signed != key_field_test->is_signed)
6500	return false;
6501	if (!!key_field->var.name != !!key_field_test->var.name)
6502	return false;
6503	if (key_field->var.name &&
6504	strcmp(key_field->var.name, key_field_test->var.name) != 0)
6505	return false;
6506	}
6507
6508	for (i = 0; i < hist_data->n_sort_keys; i++) {
6509	sort_key = &hist_data->sort_keys[i];
6510	sort_key_test = &hist_data_test->sort_keys[i];
6511
6512	if (sort_key->field_idx != sort_key_test->field_idx \|\|
6513	sort_key->descending != sort_key_test->descending)
6514	return false;
6515	}
6516
6517	if (!ignore_filter && data->filter_str &&
6518	(strcmp(data->filter_str, data_test->filter_str) != 0))
6519	return false;
6520
6521	if (!actions_match(hist_data, hist_data_test))
6522	return false;
6523
6524	return true;
6525	}
6526
6527	static bool existing_hist_update_only(char *glob,
6528	struct event_trigger_data *data,
6529	struct trace_event_file *file)
6530	{
6531	struct hist_trigger_data *hist_data = data->private_data;
6532	struct event_trigger_data test, named_data = NULL;
6533	bool updated = false;
6534
6535	if (!hist_data->attrs->pause && !hist_data->attrs->cont &&
6536	!hist_data->attrs->clear)
6537	goto out;
6538
6539	if (hist_data->attrs->name) {
6540	named_data = find_named_trigger(hist_data->attrs->name);
6541	if (named_data) {
6542	if (!hist_trigger_match(data, named_data, named_data,
6543	true))
6544	goto out;
6545	}
6546	}
6547
6548	if (hist_data->attrs->name && !named_data)
6549	goto out;
6550
6551	list_for_each_entry(test, &file->triggers, list) {
6552	if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6553	if (!hist_trigger_match(data, test, named_data, false))
6554	continue;
6555	if (hist_data->attrs->pause)
6556	test->paused = true;
6557	else if (hist_data->attrs->cont)
6558	test->paused = false;
6559	else if (hist_data->attrs->clear)
6560	hist_clear(test);
6561	updated = true;
6562	goto out;
6563	}
6564	}
6565	out:
6566	return updated;
6567	}
6568
6569	/*
6570	* Set or disable using the per CPU trace_buffer_event when possible.
6571	*/
6572	static int tracing_set_filter_buffering(struct trace_array *tr, bool set)
6573	{
6574	guard(mutex)(&trace_types_lock);
6575
6576	if (set && tr->no_filter_buffering_ref++)
6577	return 0;
6578
6579	if (!set) {
6580	if (WARN_ON_ONCE(!tr->no_filter_buffering_ref))
6581	return -EINVAL;
6582
6583	--tr->no_filter_buffering_ref;
6584	}
6585
6586	return 0;
6587	}
6588
6589	static int hist_register_trigger(char *glob,
6590	struct event_trigger_data *data,
6591	struct trace_event_file *file)
6592	{
6593	struct hist_trigger_data *hist_data = data->private_data;
6594	struct event_trigger_data test, named_data = NULL;
6595	struct trace_array *tr = file->tr;
6596	int ret = 0;
6597
6598	if (hist_data->attrs->name) {
6599	named_data = find_named_trigger(hist_data->attrs->name);
6600	if (named_data) {
6601	if (!hist_trigger_match(data, named_data, named_data,
6602	true)) {
6603	hist_err(tr, HIST_ERR_NAMED_MISMATCH, errpos(hist_data->attrs->name));
6604	ret = -EINVAL;
6605	goto out;
6606	}
6607	}
6608	}
6609
6610	if (hist_data->attrs->name && !named_data)
6611	goto new;
6612
6613	lockdep_assert_held(&event_mutex);
6614
6615	list_for_each_entry(test, &file->triggers, list) {
6616	if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6617	if (hist_trigger_match(data, test, named_data, false)) {
6618	hist_err(tr, HIST_ERR_TRIGGER_EEXIST, 0);
6619	ret = -EEXIST;
6620	goto out;
6621	}
6622	}
6623	}
6624	new:
6625	if (hist_data->attrs->cont \|\| hist_data->attrs->clear) {
6626	hist_err(tr, HIST_ERR_TRIGGER_ENOENT_CLEAR, 0);
6627	ret = -ENOENT;
6628	goto out;
6629	}
6630
6631	if (hist_data->attrs->pause)
6632	data->paused = true;
6633
6634	if (named_data) {
6635	struct event_command *cmd_ops;
6636
6637	data->private_data = named_data->private_data;
6638	set_named_trigger_data(data, named_data);
6639	/* Copy the command ops and update some of the functions */
6640	cmd_ops = kmalloc_obj(*cmd_ops);
6641	if (!cmd_ops) {
6642	ret = -ENOMEM;
6643	goto out;
6644	}
6645	cmd_ops = data->cmd_ops;
6646	cmd_ops->init = event_hist_trigger_named_init;
6647	cmd_ops->free = event_hist_trigger_named_free;
6648	data->cmd_ops = cmd_ops;
6649	}
6650
6651	if (hist_data->enable_timestamps) {
6652	char *clock = hist_data->attrs->clock;
6653
6654	ret = tracing_set_clock(file->tr, hist_data->attrs->clock);
6655	if (ret) {
6656	hist_err(tr, HIST_ERR_SET_CLOCK_FAIL, errpos(clock));
6657	goto out;
6658	}
6659
6660	tracing_set_filter_buffering(file->tr, true);
6661	}
6662
6663	if (data->cmd_ops->init) {
6664	ret = data->cmd_ops->init(data);
6665	if (ret < 0) {
6666	if (hist_data->enable_timestamps)
6667	tracing_set_filter_buffering(file->tr, false);
6668	goto out;
6669	}
6670	}
6671
6672	if (named_data) {
6673	remove_hist_vars(hist_data);
6674	destroy_hist_data(hist_data);
6675	}
6676	out:
6677	return ret;
6678	}
6679
6680	static int hist_trigger_enable(struct event_trigger_data *data,
6681	struct trace_event_file *file)
6682	{
6683	int ret = 0;
6684
6685	list_add_tail_rcu(&data->list, &file->triggers);
6686
6687	update_cond_flag(file);
6688
6689	/*
6690	* On failure the caller undoes the registration, and
6691	* hist_unregister_trigger() can only find the trigger here.
6692	*/
6693	if (trace_event_trigger_enable_disable(file, 1) < 0)
6694	ret--;
6695
6696	return ret;
6697	}
6698
6699	static bool have_hist_trigger_match(struct event_trigger_data *data,
6700	struct trace_event_file *file)
6701	{
6702	struct hist_trigger_data *hist_data = data->private_data;
6703	struct event_trigger_data test, named_data = NULL;
6704	bool match = false;
6705
6706	lockdep_assert_held(&event_mutex);
6707
6708	if (hist_data->attrs->name)
6709	named_data = find_named_trigger(hist_data->attrs->name);
6710
6711	list_for_each_entry(test, &file->triggers, list) {
6712	if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6713	if (hist_trigger_match(data, test, named_data, false)) {
6714	match = true;
6715	break;
6716	}
6717	}
6718	}
6719
6720	return match;
6721	}
6722
6723	static bool hist_trigger_check_refs(struct event_trigger_data *data,
6724	struct trace_event_file *file)
6725	{
6726	struct hist_trigger_data *hist_data = data->private_data;
6727	struct event_trigger_data test, named_data = NULL;
6728
6729	lockdep_assert_held(&event_mutex);
6730
6731	if (hist_data->attrs->name)
6732	named_data = find_named_trigger(hist_data->attrs->name);
6733
6734	list_for_each_entry(test, &file->triggers, list) {
6735	if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6736	if (!hist_trigger_match(data, test, named_data, false))
6737	continue;
6738	hist_data = test->private_data;
6739	if (check_var_refs(hist_data))
6740	return true;
6741	break;
6742	}
6743	}
6744
6745	return false;
6746	}
6747
6748	static void hist_unregister_trigger(char *glob,
6749	struct event_trigger_data *data,
6750	struct trace_event_file *file)
6751	{
6752	struct event_trigger_data test = NULL, iter, *named_data = NULL;
6753	struct hist_trigger_data *hist_data = data->private_data;
6754
6755	lockdep_assert_held(&event_mutex);
6756
6757	if (hist_data->attrs->name)
6758	named_data = find_named_trigger(hist_data->attrs->name);
6759
6760	list_for_each_entry(iter, &file->triggers, list) {
6761	if (iter->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6762	if (!hist_trigger_match(data, iter, named_data, false))
6763	continue;
6764	test = iter;
6765	list_del_rcu(&test->list);
6766	trace_event_trigger_enable_disable(file, 0);
6767	update_cond_flag(file);
6768	break;
6769	}
6770	}
6771
6772	if (hist_data->enable_timestamps) {
6773	if (!hist_data->remove \|\| test)
6774	tracing_set_filter_buffering(file->tr, false);
6775	}
6776
6777	if (test && test->cmd_ops->free)
6778	test->cmd_ops->free(test);
6779	}
6780
6781	static bool hist_file_check_refs(struct trace_event_file *file)
6782	{
6783	struct hist_trigger_data *hist_data;
6784	struct event_trigger_data *test;
6785
6786	lockdep_assert_held(&event_mutex);
6787
6788	list_for_each_entry(test, &file->triggers, list) {
6789	if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6790	hist_data = test->private_data;
6791	if (check_var_refs(hist_data))
6792	return true;
6793	}
6794	}
6795
6796	return false;
6797	}
6798
6799	static void hist_unreg_all(struct trace_event_file *file)
6800	{
6801	struct event_trigger_data test, n;
6802	struct hist_trigger_data *hist_data;
6803	struct synth_event *se;
6804	const char *se_name;
6805
6806	lockdep_assert_held(&event_mutex);
6807
6808	if (hist_file_check_refs(file))
6809	return;
6810
6811	list_for_each_entry_safe(test, n, &file->triggers, list) {
6812	if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6813	hist_data = test->private_data;
6814	list_del_rcu(&test->list);
6815	trace_event_trigger_enable_disable(file, 0);
6816
6817	se_name = trace_event_name(file->event_call);
6818	se = find_synth_event(se_name);
6819	if (se)
6820	se->ref--;
6821
6822	update_cond_flag(file);
6823	if (hist_data->enable_timestamps)
6824	tracing_set_filter_buffering(file->tr, false);
6825	if (test->cmd_ops->free)
6826	test->cmd_ops->free(test);
6827	}
6828	}
6829	}
6830
6831	static int event_hist_trigger_parse(struct event_command *cmd_ops,
6832	struct trace_event_file *file,
6833	char glob, char cmd,
6834	char *param_and_filter)
6835	{
6836	unsigned int hist_trigger_bits = TRACING_MAP_BITS_DEFAULT;
6837	struct event_trigger_data *trigger_data;
6838	struct hist_trigger_attrs *attrs;
6839	struct hist_trigger_data *hist_data;
6840	char param, filter, p, start;
6841	struct synth_event *se;
6842	const char *se_name;
6843	bool remove;
6844	int ret = 0;
6845
6846	lockdep_assert_held(&event_mutex);
6847
6848	if (WARN_ON(!glob))
6849	return -EINVAL;
6850
6851	if (glob[0]) {
6852	hist_err_clear();
6853	last_cmd_set(file, param_and_filter);
6854	}
6855
6856	remove = event_trigger_check_remove(glob);
6857
6858	if (event_trigger_empty_param(param_and_filter))
6859	return -EINVAL;
6860
6861	/*
6862	* separate the trigger from the filter (k:v [if filter])
6863	* allowing for whitespace in the trigger
6864	*/
6865	p = param = param_and_filter;
6866	do {
6867	p = strstr(p, "if");
6868	if (!p)
6869	break;
6870	if (p == param_and_filter)
6871	return -EINVAL;
6872	if ((p - 1) != ' ' && (p - 1) != '\t') {
6873	p++;
6874	continue;
6875	}
6876	if (p >= param_and_filter + strlen(param_and_filter) - (sizeof("if") - 1) - 1)
6877	return -EINVAL;
6878	if ((p + sizeof("if") - 1) != ' ' && (p + sizeof("if") - 1) != '\t') {
6879	p++;
6880	continue;
6881	}
6882	break;
6883	} while (1);
6884
6885	if (!p)
6886	filter = NULL;
6887	else {
6888	*(p - 1) = '\0';
6889	filter = strstrip(p);
6890	param = strstrip(param);
6891	}
6892
6893	/*
6894	* To simplify arithmetic expression parsing, replace occurrences of
6895	* '.sym-offset' modifier with '.symXoffset'
6896	*/
6897	start = strstr(param, ".sym-offset");
6898	while (start) {
6899	*(start + 4) = 'X';
6900	start = strstr(start + 11, ".sym-offset");
6901	}
6902
6903	attrs = parse_hist_trigger_attrs(file->tr, param);
6904	if (IS_ERR(attrs))
6905	return PTR_ERR(attrs);
6906
6907	if (attrs->map_bits)
6908	hist_trigger_bits = attrs->map_bits;
6909
6910	hist_data = create_hist_data(hist_trigger_bits, attrs, file, remove);
6911	if (IS_ERR(hist_data)) {
6912	destroy_hist_trigger_attrs(attrs);
6913	return PTR_ERR(hist_data);
6914	}
6915
6916	trigger_data = trigger_data_alloc(cmd_ops, cmd, param, hist_data);
6917	if (!trigger_data) {
6918	ret = -ENOMEM;
6919	goto out_free;
6920	}
6921
6922	ret = event_trigger_set_filter(cmd_ops, file, filter, trigger_data);
6923	if (ret < 0)
6924	goto out_free;
6925
6926	if (remove) {
6927	if (!have_hist_trigger_match(trigger_data, file))
6928	goto out_free;
6929
6930	if (hist_trigger_check_refs(trigger_data, file)) {
6931	ret = -EBUSY;
6932	goto out_free;
6933	}
6934
6935	event_trigger_unregister(cmd_ops, file, glob+1, trigger_data);
6936	se_name = trace_event_name(file->event_call);
6937	se = find_synth_event(se_name);
6938	if (se)
6939	se->ref--;
6940	ret = 0;
6941	goto out_free;
6942	}
6943
6944	if (existing_hist_update_only(glob, trigger_data, file))
6945	goto out_free;
6946
6947	if (!get_named_trigger_data(trigger_data)) {
6948
6949	ret = create_actions(hist_data);
6950	if (ret)
6951	goto out_free;
6952
6953	if (has_hist_vars(hist_data) \|\| hist_data->n_var_refs) {
6954	ret = save_hist_vars(hist_data);
6955	if (ret)
6956	goto out_free;
6957	}
6958
6959	ret = tracing_map_init(hist_data->map);
6960	if (ret)
6961	goto out_free;
6962	}
6963
6964	ret = event_trigger_register(cmd_ops, file, glob, trigger_data);
6965	if (ret < 0)
6966	goto out_free;
6967
6968	ret = hist_trigger_enable(trigger_data, file);
6969	if (ret)
6970	goto out_unreg;
6971
6972	se_name = trace_event_name(file->event_call);
6973	se = find_synth_event(se_name);
6974	if (se)
6975	se->ref++;
6976	out:
6977	if (ret == 0 && glob[0])
6978	hist_err_clear();
6979
6980	return ret;
6981	out_unreg:
6982	event_trigger_unregister(cmd_ops, file, glob+1, trigger_data);
6983	/* The unregister frees trigger_data, skip out_free */
6984	goto out;
6985	out_free:
6986	remove_hist_vars(hist_data);
6987
6988	trigger_data_free(trigger_data);
6989
6990	destroy_hist_data(hist_data);
6991	goto out;
6992	}
6993
6994	static struct event_command trigger_hist_cmd = {
6995	.name = "hist",
6996	.trigger_type = ETT_EVENT_HIST,
6997	.flags = EVENT_CMD_FL_NEEDS_REC,
6998	.parse = event_hist_trigger_parse,
6999	.reg = hist_register_trigger,
7000	.unreg = hist_unregister_trigger,
7001	.unreg_all = hist_unreg_all,
7002	.set_filter = set_trigger_filter,
7003	.trigger = event_hist_trigger,
7004	.print = event_hist_trigger_print,
7005	.init = event_hist_trigger_init,
7006	.free = event_hist_trigger_free,
7007	};
7008
7009	__init int register_trigger_hist_cmd(void)
7010	{
7011	int ret;
7012
7013	ret = register_event_command(&trigger_hist_cmd);
7014	WARN_ON(ret < 0);
7015
7016	return ret;
7017	}
7018
7019	static void
7020	hist_enable_trigger(struct event_trigger_data *data,
7021	struct trace_buffer buffer, void rec,
7022	struct ring_buffer_event *event)
7023	{
7024	struct enable_trigger_data *enable_data = data->private_data;
7025	struct event_trigger_data *test;
7026
7027	list_for_each_entry_rcu(test, &enable_data->file->triggers, list,
7028	lockdep_is_held(&event_mutex)) {
7029	if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
7030	if (enable_data->enable)
7031	test->paused = false;
7032	else
7033	test->paused = true;
7034	}
7035	}
7036	}
7037
7038	static void hist_enable_unreg_all(struct trace_event_file *file)
7039	{
7040	struct event_trigger_data test, n;
7041
7042	list_for_each_entry_safe(test, n, &file->triggers, list) {
7043	if (test->cmd_ops->trigger_type == ETT_HIST_ENABLE) {
7044	list_del_rcu(&test->list);
7045	update_cond_flag(file);
7046	trace_event_trigger_enable_disable(file, 0);
7047	if (test->cmd_ops->free)
7048	test->cmd_ops->free(test);
7049	}
7050	}
7051	}
7052
7053	static struct event_command trigger_hist_enable_cmd = {
7054	.name = ENABLE_HIST_STR,
7055	.trigger_type = ETT_HIST_ENABLE,
7056	.parse = event_enable_trigger_parse,
7057	.reg = event_enable_register_trigger,
7058	.unreg = event_enable_unregister_trigger,
7059	.unreg_all = hist_enable_unreg_all,
7060	.set_filter = set_trigger_filter,
7061	.trigger = hist_enable_trigger,
7062	.count_func = event_trigger_count,
7063	.print = event_enable_trigger_print,
7064	.init = event_trigger_init,
7065	.free = event_enable_trigger_free,
7066	};
7067
7068	static struct event_command trigger_hist_disable_cmd = {
7069	.name = DISABLE_HIST_STR,
7070	.trigger_type = ETT_HIST_ENABLE,
7071	.parse = event_enable_trigger_parse,
7072	.reg = event_enable_register_trigger,
7073	.unreg = event_enable_unregister_trigger,
7074	.unreg_all = hist_enable_unreg_all,
7075	.set_filter = set_trigger_filter,
7076	.trigger = hist_enable_trigger,
7077	.count_func = event_trigger_count,
7078	.print = event_enable_trigger_print,
7079	.init = event_trigger_init,
7080	.free = event_enable_trigger_free,
7081	};
7082
7083	static __init void unregister_trigger_hist_enable_disable_cmds(void)
7084	{
7085	unregister_event_command(&trigger_hist_enable_cmd);
7086	unregister_event_command(&trigger_hist_disable_cmd);
7087	}
7088
7089	__init int register_trigger_hist_enable_disable_cmds(void)
7090	{
7091	int ret;
7092
7093	ret = register_event_command(&trigger_hist_enable_cmd);
7094	if (WARN_ON(ret < 0))
7095	return ret;
7096	ret = register_event_command(&trigger_hist_disable_cmd);
7097	if (WARN_ON(ret < 0))
7098	unregister_trigger_hist_enable_disable_cmds();
7099
7100	return ret;
7101	}