projects/linux·blob master · refs

2975 lines · 68.5 KiB

1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Copyright (c) 2021, Microsoft Corporation.
4 *
5 * Authors:
6 * Beau Belgrave <[email protected]>
7 */
8
9#include <linux/bitmap.h>
10#include <linux/cdev.h>
11#include <linux/hashtable.h>
12#include <linux/list.h>
13#include <linux/io.h>
14#include <linux/uio.h>
15#include <linux/ioctl.h>
16#include <linux/jhash.h>
17#include <linux/refcount.h>
18#include <linux/trace_events.h>
19#include <linux/tracefs.h>
20#include <linux/types.h>
21#include <linux/uaccess.h>
22#include <linux/highmem.h>
23#include <linux/init.h>
24#include <linux/user_events.h>
25#include "trace_dynevent.h"
26#include "trace_output.h"
27#include "trace.h"
28
29#define USER_EVENTS_PREFIX_LEN (sizeof(USER_EVENTS_PREFIX)-1)
30
31#define FIELD_DEPTH_TYPE 0
32#define FIELD_DEPTH_NAME 1
33#define FIELD_DEPTH_SIZE 2
34
35/* Limit how long of an event name plus args within the subsystem. */
36#define MAX_EVENT_DESC 512
37#define EVENT_NAME(user_event) ((user_event)->reg_name)
38#define EVENT_TP_NAME(user_event) ((user_event)->tracepoint.name)
39#define MAX_FIELD_ARRAY_SIZE 1024
40
41/*
42 * Internal bits (kernel side only) to keep track of connected probes:
43 * These are used when status is requested in text form about an event. These
44 * bits are compared against an internal byte on the event to determine which
45 * probes to print out to the user.
46 *
47 * These do not reflect the mapped bytes between the user and kernel space.
48 */
49#define EVENT_STATUS_FTRACE BIT(0)
50#define EVENT_STATUS_PERF BIT(1)
51#define EVENT_STATUS_OTHER BIT(7)
52
53/*
54 * Stores the system name, tables, and locks for a group of events. This
55 * allows isolation for events by various means.
56 */
57struct user_event_group {
58 char *system_name;
59 char *system_multi_name;
60 struct hlist_node node;
61 struct mutex reg_mutex;
62 DECLARE_HASHTABLE(register_table, 8);
63 /* ID that moves forward within the group for multi-event names */
64 u64 multi_id;
65};
66
67/* Group for init_user_ns mapping, top-most group */
68static struct user_event_group *init_group;
69
70/* Max allowed events for the whole system */
71static unsigned int max_user_events = 32768;
72
73/* Current number of events on the whole system */
74static unsigned int current_user_events;
75
76/*
77 * Stores per-event properties, as users register events
78 * within a file a user_event might be created if it does not
79 * already exist. These are globally used and their lifetime
80 * is tied to the refcnt member. These cannot go away until the
81 * refcnt reaches one.
82 */
83struct user_event {
84 struct user_event_group *group;
85 char *reg_name;
86 struct tracepoint tracepoint;
87 struct trace_event_call call;
88 struct trace_event_class class;
89 struct dyn_event devent;
90 struct hlist_node node;
91 struct list_head fields;
92 struct list_head validators;
93 struct work_struct put_work;
94 refcount_t refcnt;
95 int min_size;
96 int reg_flags;
97 char status;
98};
99
100/*
101 * Stores per-mm/event properties that enable an address to be
102 * updated properly for each task. As tasks are forked, we use
103 * these to track enablement sites that are tied to an event.
104 */
105struct user_event_enabler {
106 struct list_head mm_enablers_link;
107 struct user_event *event;
108 unsigned long addr;
109
110 /* Track enable bit, flags, etc. Aligned for bitops. */
111 unsigned long values;
112
113 /* Defer the event put and enabler free past an RCU grace period. */
114 struct rcu_work put_rwork;
115};
116
117/* Bits 0-5 are for the bit to update upon enable/disable (0-63 allowed) */
118#define ENABLE_VAL_BIT_MASK 0x3F
119
120/* Bit 6 is for faulting status of enablement */
121#define ENABLE_VAL_FAULTING_BIT 6
122
123/* Bit 7 is for freeing status of enablement */
124#define ENABLE_VAL_FREEING_BIT 7
125
126/* Bit 8 is for marking 32-bit on 64-bit */
127#define ENABLE_VAL_32_ON_64_BIT 8
128
129#define ENABLE_VAL_COMPAT_MASK (1 << ENABLE_VAL_32_ON_64_BIT)
130
131/* Only duplicate the bit and compat values */
132#define ENABLE_VAL_DUP_MASK (ENABLE_VAL_BIT_MASK | ENABLE_VAL_COMPAT_MASK)
133
134#define ENABLE_BITOPS(e) (&(e)->values)
135
136#define ENABLE_BIT(e) ((int)((e)->values & ENABLE_VAL_BIT_MASK))
137
138#define EVENT_MULTI_FORMAT(f) ((f) & USER_EVENT_REG_MULTI_FORMAT)
139
140/* Used for asynchronous faulting in of pages */
141struct user_event_enabler_fault {
142 struct work_struct work;
143 struct user_event_mm *mm;
144 struct user_event_enabler *enabler;
145 int attempt;
146};
147
148static struct kmem_cache *fault_cache;
149
150/* Global list of memory descriptors using user_events */
151static LIST_HEAD(user_event_mms);
152static DEFINE_SPINLOCK(user_event_mms_lock);
153
154/*
155 * Stores per-file events references, as users register events
156 * within a file this structure is modified and freed via RCU.
157 * The lifetime of this struct is tied to the lifetime of the file.
158 * These are not shared and only accessible by the file that created it.
159 */
160struct user_event_refs {
161 struct rcu_head rcu;
162 int count;
163 struct user_event *events[];
164};
165
166struct user_event_file_info {
167 struct user_event_group *group;
168 struct user_event_refs *refs;
169};
170
171#define VALIDATOR_ENSURE_NULL (1 << 0)
172#define VALIDATOR_REL (1 << 1)
173
174struct user_event_validator {
175 struct list_head user_event_link;
176 int offset;
177 int flags;
178};
179
180static inline void align_addr_bit(unsigned long *addr, int *bit,
181 unsigned long *flags)
182{
183 if (IS_ALIGNED(*addr, sizeof(long))) {
184#ifdef __BIG_ENDIAN
185 /* 32 bit on BE 64 bit requires a 32 bit offset when aligned. */
186 if (test_bit(ENABLE_VAL_32_ON_64_BIT, flags))
187 *bit += 32;
188#endif
189 return;
190 }
191
192 *addr = ALIGN_DOWN(*addr, sizeof(long));
193
194 /*
195 * We only support 32 and 64 bit values. The only time we need
196 * to align is a 32 bit value on a 64 bit kernel, which on LE
197 * is always 32 bits, and on BE requires no change when unaligned.
198 */
199#ifdef __LITTLE_ENDIAN
200 *bit += 32;
201#endif
202}
203
204typedef void (*user_event_func_t) (struct user_event *user, struct iov_iter *i,
205 void *tpdata, bool *faulted);
206
207static int user_event_parse(struct user_event_group *group, char *name,
208 char *args, char *flags,
209 struct user_event **newuser, int reg_flags);
210
211static struct user_event_mm *user_event_mm_get(struct user_event_mm *mm);
212static struct user_event_mm *user_event_mm_get_all(struct user_event *user);
213static void user_event_mm_put(struct user_event_mm *mm);
214static int destroy_user_event(struct user_event *user);
215static bool user_fields_match(struct user_event *user, int argc,
216 const char **argv);
217
218static u32 user_event_key(char *name)
219{
220 return jhash(name, strlen(name), 0);
221}
222
223static bool user_event_capable(u16 reg_flags)
224{
225 /* Persistent events require CAP_PERFMON / CAP_SYS_ADMIN */
226 if (reg_flags & USER_EVENT_REG_PERSIST) {
227 if (!perfmon_capable())
228 return false;
229 }
230
231 return true;
232}
233
234static struct user_event *user_event_get(struct user_event *user)
235{
236 refcount_inc(&user->refcnt);
237
238 return user;
239}
240
241static void delayed_destroy_user_event(struct work_struct *work)
242{
243 struct user_event *user = container_of(
244 work, struct user_event, put_work);
245
246 mutex_lock(&event_mutex);
247
248 if (!refcount_dec_and_test(&user->refcnt))
249 goto out;
250
251 if (destroy_user_event(user)) {
252 /*
253 * The only reason this would fail here is if we cannot
254 * update the visibility of the event. In this case the
255 * event stays in the hashtable, waiting for someone to
256 * attempt to delete it later.
257 */
258 pr_warn("user_events: Unable to delete event\n");
259 refcount_set(&user->refcnt, 1);
260 }
261out:
262 mutex_unlock(&event_mutex);
263}
264
265static void user_event_put(struct user_event *user, bool locked)
266{
267 bool delete;
268
269 if (unlikely(!user))
270 return;
271
272 /*
273 * When the event is not enabled for auto-delete there will always
274 * be at least 1 reference to the event. During the event creation
275 * we initially set the refcnt to 2 to achieve this. In those cases
276 * the caller must acquire event_mutex and after decrement check if
277 * the refcnt is 1, meaning this is the last reference. When auto
278 * delete is enabled, there will only be 1 ref, IE: refcnt will be
279 * only set to 1 during creation to allow the below checks to go
280 * through upon the last put. The last put must always be done with
281 * the event mutex held.
282 */
283 if (!locked) {
284 lockdep_assert_not_held(&event_mutex);
285 delete = refcount_dec_and_mutex_lock(&user->refcnt, &event_mutex);
286 } else {
287 lockdep_assert_held(&event_mutex);
288 delete = refcount_dec_and_test(&user->refcnt);
289 }
290
291 if (!delete)
292 return;
293
294 /*
295 * We now have the event_mutex in all cases, which ensures that
296 * no new references will be taken until event_mutex is released.
297 * New references come through find_user_event(), which requires
298 * the event_mutex to be held.
299 */
300
301 if (user->reg_flags & USER_EVENT_REG_PERSIST) {
302 /* We should not get here when persist flag is set */
303 pr_alert("BUG: Auto-delete engaged on persistent event\n");
304 goto out;
305 }
306
307 /*
308 * Unfortunately we have to attempt the actual destroy in a work
309 * queue. This is because not all cases handle a trace_event_call
310 * being removed within the class->reg() operation for unregister.
311 */
312 INIT_WORK(&user->put_work, delayed_destroy_user_event);
313
314 /*
315 * Since the event is still in the hashtable, we have to re-inc
316 * the ref count to 1. This count will be decremented and checked
317 * in the work queue to ensure it's still the last ref. This is
318 * needed because a user-process could register the same event in
319 * between the time of event_mutex release and the work queue
320 * running the delayed destroy. If we removed the item now from
321 * the hashtable, this would result in a timing window where a
322 * user process would fail a register because the trace_event_call
323 * register would fail in the tracing layers.
324 */
325 refcount_set(&user->refcnt, 1);
326
327 if (WARN_ON_ONCE(!schedule_work(&user->put_work))) {
328 /*
329 * If we fail we must wait for an admin to attempt delete or
330 * another register/close of the event, whichever is first.
331 */
332 pr_warn("user_events: Unable to queue delayed destroy\n");
333 }
334out:
335 /* Ensure if we didn't have event_mutex before we unlock it */
336 if (!locked)
337 mutex_unlock(&event_mutex);
338}
339
340static void user_event_group_destroy(struct user_event_group *group)
341{
342 kfree(group->system_name);
343 kfree(group->system_multi_name);
344 kfree(group);
345}
346
347static char *user_event_group_system_name(void)
348{
349 char *system_name;
350 int len = sizeof(USER_EVENTS_SYSTEM) + 1;
351
352 system_name = kmalloc(len, GFP_KERNEL);
353
354 if (!system_name)
355 return NULL;
356
357 snprintf(system_name, len, "%s", USER_EVENTS_SYSTEM);
358
359 return system_name;
360}
361
362static char *user_event_group_system_multi_name(void)
363{
364 return kstrdup(USER_EVENTS_MULTI_SYSTEM, GFP_KERNEL);
365}
366
367static struct user_event_group *current_user_event_group(void)
368{
369 return init_group;
370}
371
372static struct user_event_group *user_event_group_create(void)
373{
374 struct user_event_group *group;
375
376 group = kzalloc_obj(*group);
377
378 if (!group)
379 return NULL;
380
381 group->system_name = user_event_group_system_name();
382
383 if (!group->system_name)
384 goto error;
385
386 group->system_multi_name = user_event_group_system_multi_name();
387
388 if (!group->system_multi_name)
389 goto error;
390
391 mutex_init(&group->reg_mutex);
392 hash_init(group->register_table);
393
394 return group;
395error:
396 if (group)
397 user_event_group_destroy(group);
398
399 return NULL;
400};
401
402static void delayed_user_event_enabler_put(struct work_struct *work)
403{
404 struct user_event_enabler *enabler = container_of(to_rcu_work(work),
405 struct user_event_enabler, put_rwork);
406
407 /* No longer tracking the event via the enabler */
408 user_event_put(enabler->event, false);
409
410 /* Run from queue_rcu_work(), the RCU grace period has elapsed */
411 kfree(enabler);
412}
413
414static void user_event_enabler_destroy(struct user_event_enabler *enabler)
415{
416 list_del_rcu(&enabler->mm_enablers_link);
417
418 /*
419 * The enabler is removed from an RCU-traversed list
420 * (user_event_mm_dup() walks mm->enablers under rcu_read_lock() only),
421 * and readers there dereference enabler->event and take a new ref on
422 * it. Both the put of that event reference and the free of the enabler
423 * therefore have to wait for a grace period so no reader can be looking
424 * at the enabler or racing the last put of its event.
425 *
426 * The put itself must not run in RCU context: when it drops the last
427 * reference user_event_put() takes event_mutex, which cannot be taken
428 * from a softirq/RCU callback. Defer both to a work item scheduled
429 * after a grace period via queue_rcu_work().
430 */
431 INIT_RCU_WORK(&enabler->put_rwork, delayed_user_event_enabler_put);
432 queue_rcu_work(system_percpu_wq, &enabler->put_rwork);
433}
434
435static int user_event_mm_fault_in(struct user_event_mm *mm, unsigned long uaddr,
436 int attempt)
437{
438 bool unlocked;
439 int ret;
440
441 /*
442 * Normally this is low, ensure that it cannot be taken advantage of by
443 * bad user processes to cause excessive looping.
444 */
445 if (attempt > 10)
446 return -EFAULT;
447
448 mmap_read_lock(mm->mm);
449
450 /* Ensure MM has tasks, cannot use after exit_mm() */
451 if (refcount_read(&mm->tasks) == 0) {
452 ret = -ENOENT;
453 goto out;
454 }
455
456 ret = fixup_user_fault(mm->mm, uaddr, FAULT_FLAG_WRITE | FAULT_FLAG_REMOTE,
457 &unlocked);
458out:
459 mmap_read_unlock(mm->mm);
460
461 return ret;
462}
463
464static int user_event_enabler_write(struct user_event_mm *mm,
465 struct user_event_enabler *enabler,
466 bool fixup_fault, int *attempt);
467
468static void user_event_enabler_fault_fixup(struct work_struct *work)
469{
470 struct user_event_enabler_fault *fault = container_of(
471 work, struct user_event_enabler_fault, work);
472 struct user_event_enabler *enabler = fault->enabler;
473 struct user_event_mm *mm = fault->mm;
474 unsigned long uaddr = enabler->addr;
475 int attempt = fault->attempt;
476 int ret;
477
478 ret = user_event_mm_fault_in(mm, uaddr, attempt);
479
480 if (ret && ret != -ENOENT) {
481 struct user_event *user = enabler->event;
482
483 pr_warn("user_events: Fault for mm: 0x%p @ 0x%llx event: %s\n",
484 mm->mm, (unsigned long long)uaddr, EVENT_NAME(user));
485 }
486
487 /* Prevent state changes from racing */
488 mutex_lock(&event_mutex);
489
490 /* User asked for enabler to be removed during fault */
491 if (test_bit(ENABLE_VAL_FREEING_BIT, ENABLE_BITOPS(enabler))) {
492 user_event_enabler_destroy(enabler);
493 goto out;
494 }
495
496 /*
497 * If we managed to get the page, re-issue the write. We do not
498 * want to get into a possible infinite loop, which is why we only
499 * attempt again directly if the page came in. If we couldn't get
500 * the page here, then we will try again the next time the event is
501 * enabled/disabled.
502 */
503 clear_bit(ENABLE_VAL_FAULTING_BIT, ENABLE_BITOPS(enabler));
504
505 if (!ret) {
506 mmap_read_lock(mm->mm);
507 user_event_enabler_write(mm, enabler, true, &attempt);
508 mmap_read_unlock(mm->mm);
509 }
510out:
511 mutex_unlock(&event_mutex);
512
513 /* In all cases we no longer need the mm or fault */
514 user_event_mm_put(mm);
515 kmem_cache_free(fault_cache, fault);
516}
517
518static bool user_event_enabler_queue_fault(struct user_event_mm *mm,
519 struct user_event_enabler *enabler,
520 int attempt)
521{
522 struct user_event_enabler_fault *fault;
523
524 fault = kmem_cache_zalloc(fault_cache, GFP_NOWAIT);
525
526 if (!fault)
527 return false;
528
529 INIT_WORK(&fault->work, user_event_enabler_fault_fixup);
530 fault->mm = user_event_mm_get(mm);
531 fault->enabler = enabler;
532 fault->attempt = attempt;
533
534 /* Don't try to queue in again while we have a pending fault */
535 set_bit(ENABLE_VAL_FAULTING_BIT, ENABLE_BITOPS(enabler));
536
537 if (!schedule_work(&fault->work)) {
538 /* Allow another attempt later */
539 clear_bit(ENABLE_VAL_FAULTING_BIT, ENABLE_BITOPS(enabler));
540
541 user_event_mm_put(mm);
542 kmem_cache_free(fault_cache, fault);
543
544 return false;
545 }
546
547 return true;
548}
549
550static int user_event_enabler_write(struct user_event_mm *mm,
551 struct user_event_enabler *enabler,
552 bool fixup_fault, int *attempt)
553{
554 unsigned long uaddr = enabler->addr;
555 unsigned long *ptr;
556 struct page *page;
557 void *kaddr;
558 int bit = ENABLE_BIT(enabler);
559 int ret;
560
561 lockdep_assert_held(&event_mutex);
562 mmap_assert_locked(mm->mm);
563
564 *attempt += 1;
565
566 /* Ensure MM has tasks, cannot use after exit_mm() */
567 if (refcount_read(&mm->tasks) == 0)
568 return -ENOENT;
569
570 if (unlikely(test_bit(ENABLE_VAL_FAULTING_BIT, ENABLE_BITOPS(enabler)) ||
571 test_bit(ENABLE_VAL_FREEING_BIT, ENABLE_BITOPS(enabler))))
572 return -EBUSY;
573
574 align_addr_bit(&uaddr, &bit, ENABLE_BITOPS(enabler));
575
576 ret = pin_user_pages_remote(mm->mm, uaddr, 1, FOLL_WRITE | FOLL_NOFAULT,
577 &page, NULL);
578
579 if (unlikely(ret <= 0)) {
580 if (!fixup_fault)
581 return -EFAULT;
582
583 if (!user_event_enabler_queue_fault(mm, enabler, *attempt))
584 pr_warn("user_events: Unable to queue fault handler\n");
585
586 return -EFAULT;
587 }
588
589 kaddr = kmap_local_page(page);
590 ptr = kaddr + (uaddr & ~PAGE_MASK);
591
592 /* Update bit atomically, user tracers must be atomic as well */
593 if (enabler->event && enabler->event->status)
594 set_bit(bit, ptr);
595 else
596 clear_bit(bit, ptr);
597
598 kunmap_local(kaddr);
599 unpin_user_pages_dirty_lock(&page, 1, true);
600
601 return 0;
602}
603
604static bool user_event_enabler_exists(struct user_event_mm *mm,
605 unsigned long uaddr, unsigned char bit)
606{
607 struct user_event_enabler *enabler;
608
609 list_for_each_entry(enabler, &mm->enablers, mm_enablers_link) {
610 if (enabler->addr == uaddr && ENABLE_BIT(enabler) == bit)
611 return true;
612 }
613
614 return false;
615}
616
617static void user_event_enabler_update(struct user_event *user)
618{
619 struct user_event_enabler *enabler;
620 struct user_event_mm *next;
621 struct user_event_mm *mm;
622 int attempt;
623
624 lockdep_assert_held(&event_mutex);
625
626 /*
627 * We need to build a one-shot list of all the mms that have an
628 * enabler for the user_event passed in. This list is only valid
629 * while holding the event_mutex. The only reason for this is due
630 * to the global mm list being RCU protected and we use methods
631 * which can wait (mmap_read_lock and pin_user_pages_remote).
632 *
633 * NOTE: user_event_mm_get_all() increments the ref count of each
634 * mm that is added to the list to prevent removal timing windows.
635 * We must always put each mm after they are used, which may wait.
636 */
637 mm = user_event_mm_get_all(user);
638
639 while (mm) {
640 next = mm->next;
641 mmap_read_lock(mm->mm);
642
643 list_for_each_entry(enabler, &mm->enablers, mm_enablers_link) {
644 if (enabler->event == user) {
645 attempt = 0;
646 user_event_enabler_write(mm, enabler, true, &attempt);
647 }
648 }
649
650 mmap_read_unlock(mm->mm);
651 user_event_mm_put(mm);
652 mm = next;
653 }
654}
655
656static bool user_event_enabler_dup(struct user_event_enabler *orig,
657 struct user_event_mm *mm)
658{
659 struct user_event_enabler *enabler;
660
661 /* Skip pending frees */
662 if (unlikely(test_bit(ENABLE_VAL_FREEING_BIT, ENABLE_BITOPS(orig))))
663 return true;
664
665 enabler = kzalloc_obj(*enabler, GFP_NOWAIT | __GFP_ACCOUNT);
666
667 if (!enabler)
668 return false;
669
670 enabler->event = user_event_get(orig->event);
671 enabler->addr = orig->addr;
672
673 /* Only dup part of value (ignore future flags, etc) */
674 enabler->values = orig->values & ENABLE_VAL_DUP_MASK;
675
676 /* Enablers not exposed yet, RCU not required */
677 list_add(&enabler->mm_enablers_link, &mm->enablers);
678
679 return true;
680}
681
682static struct user_event_mm *user_event_mm_get(struct user_event_mm *mm)
683{
684 refcount_inc(&mm->refcnt);
685
686 return mm;
687}
688
689static struct user_event_mm *user_event_mm_get_all(struct user_event *user)
690{
691 struct user_event_mm *found = NULL;
692 struct user_event_enabler *enabler;
693 struct user_event_mm *mm;
694
695 /*
696 * We use the mm->next field to build a one-shot list from the global
697 * RCU protected list. To build this list the event_mutex must be held.
698 * This lets us build a list without requiring allocs that could fail
699 * when user based events are most wanted for diagnostics.
700 */
701 lockdep_assert_held(&event_mutex);
702
703 /*
704 * We do not want to block fork/exec while enablements are being
705 * updated, so we use RCU to walk the current tasks that have used
706 * user_events ABI for 1 or more events. Each enabler found in each
707 * task that matches the event being updated has a write to reflect
708 * the kernel state back into the process. Waits/faults must not occur
709 * during this. So we scan the list under RCU for all the mm that have
710 * the event within it. This is needed because mm_read_lock() can wait.
711 * Each user mm returned has a ref inc to handle remove RCU races.
712 */
713 rcu_read_lock();
714
715 list_for_each_entry_rcu(mm, &user_event_mms, mms_link) {
716 list_for_each_entry_rcu(enabler, &mm->enablers, mm_enablers_link) {
717 if (enabler->event == user) {
718 mm->next = found;
719 found = user_event_mm_get(mm);
720 break;
721 }
722 }
723 }
724
725 rcu_read_unlock();
726
727 return found;
728}
729
730static struct user_event_mm *user_event_mm_alloc(struct task_struct *t)
731{
732 struct user_event_mm *user_mm;
733
734 user_mm = kzalloc_obj(*user_mm, GFP_KERNEL_ACCOUNT);
735
736 if (!user_mm)
737 return NULL;
738
739 user_mm->mm = t->mm;
740 INIT_LIST_HEAD(&user_mm->enablers);
741 refcount_set(&user_mm->refcnt, 1);
742 refcount_set(&user_mm->tasks, 1);
743
744 /*
745 * The lifetime of the memory descriptor can slightly outlast
746 * the task lifetime if a ref to the user_event_mm is taken
747 * between list_del_rcu() and call_rcu(). Therefore we need
748 * to take a reference to it to ensure it can live this long
749 * under this corner case. This can also occur in clones that
750 * outlast the parent.
751 */
752 mmgrab(user_mm->mm);
753
754 return user_mm;
755}
756
757static void user_event_mm_attach(struct user_event_mm *user_mm, struct task_struct *t)
758{
759 unsigned long flags;
760
761 spin_lock_irqsave(&user_event_mms_lock, flags);
762 list_add_rcu(&user_mm->mms_link, &user_event_mms);
763 spin_unlock_irqrestore(&user_event_mms_lock, flags);
764
765 t->user_event_mm = user_mm;
766}
767
768static struct user_event_mm *current_user_event_mm(void)
769{
770 struct user_event_mm *user_mm = current->user_event_mm;
771
772 if (user_mm)
773 goto inc;
774
775 user_mm = user_event_mm_alloc(current);
776
777 if (!user_mm)
778 goto error;
779
780 user_event_mm_attach(user_mm, current);
781inc:
782 refcount_inc(&user_mm->refcnt);
783error:
784 return user_mm;
785}
786
787static void user_event_mm_destroy(struct user_event_mm *mm)
788{
789 struct user_event_enabler *enabler, *next;
790
791 list_for_each_entry_safe(enabler, next, &mm->enablers, mm_enablers_link)
792 user_event_enabler_destroy(enabler);
793
794 mmdrop(mm->mm);
795 kfree(mm);
796}
797
798static void user_event_mm_put(struct user_event_mm *mm)
799{
800 if (mm && refcount_dec_and_test(&mm->refcnt))
801 user_event_mm_destroy(mm);
802}
803
804static void delayed_user_event_mm_put(struct work_struct *work)
805{
806 struct user_event_mm *mm;
807
808 mm = container_of(to_rcu_work(work), struct user_event_mm, put_rwork);
809 user_event_mm_put(mm);
810}
811
812void user_event_mm_remove(struct task_struct *t)
813{
814 struct user_event_mm *mm;
815 unsigned long flags;
816
817 might_sleep();
818
819 mm = t->user_event_mm;
820 t->user_event_mm = NULL;
821
822 /* Clone will increment the tasks, only remove if last clone */
823 if (!refcount_dec_and_test(&mm->tasks))
824 return;
825
826 /* Remove the mm from the list, so it can no longer be enabled */
827 spin_lock_irqsave(&user_event_mms_lock, flags);
828 list_del_rcu(&mm->mms_link);
829 spin_unlock_irqrestore(&user_event_mms_lock, flags);
830
831 /*
832 * We need to wait for currently occurring writes to stop within
833 * the mm. This is required since exit_mm() snaps the current rss
834 * stats and clears them. On the final mmdrop(), check_mm() will
835 * report a bug if these increment.
836 *
837 * All writes/pins are done under mmap_read lock, take the write
838 * lock to ensure in-progress faults have completed. Faults that
839 * are pending but yet to run will check the task count and skip
840 * the fault since the mm is going away.
841 */
842 mmap_write_lock(mm->mm);
843 mmap_write_unlock(mm->mm);
844
845 /*
846 * Put for mm must be done after RCU delay to handle new refs in
847 * between the list_del_rcu() and now. This ensures any get refs
848 * during rcu_read_lock() are accounted for during list removal.
849 *
850 * CPU A | CPU B
851 * ---------------------------------------------------------------
852 * user_event_mm_remove() | rcu_read_lock();
853 * list_del_rcu() | list_for_each_entry_rcu();
854 * call_rcu() | refcount_inc();
855 * . | rcu_read_unlock();
856 * schedule_work() | .
857 * user_event_mm_put() | .
858 *
859 * mmdrop() cannot be called in the softirq context of call_rcu()
860 * so we use a work queue after call_rcu() to run within.
861 */
862 INIT_RCU_WORK(&mm->put_rwork, delayed_user_event_mm_put);
863 queue_rcu_work(system_percpu_wq, &mm->put_rwork);
864}
865
866void user_event_mm_dup(struct task_struct *t, struct user_event_mm *old_mm)
867{
868 struct user_event_mm *mm = user_event_mm_alloc(t);
869 struct user_event_enabler *enabler;
870
871 /* On failure, do not free parent's copy */
872 t->user_event_mm = NULL;
873
874 if (!mm)
875 return;
876
877 rcu_read_lock();
878
879 list_for_each_entry_rcu(enabler, &old_mm->enablers, mm_enablers_link) {
880 if (!user_event_enabler_dup(enabler, mm))
881 goto error;
882 }
883
884 rcu_read_unlock();
885
886 user_event_mm_attach(mm, t);
887 return;
888error:
889 rcu_read_unlock();
890 user_event_mm_destroy(mm);
891}
892
893static bool current_user_event_enabler_exists(unsigned long uaddr,
894 unsigned char bit)
895{
896 struct user_event_mm *user_mm = current_user_event_mm();
897 bool exists;
898
899 if (!user_mm)
900 return false;
901
902 exists = user_event_enabler_exists(user_mm, uaddr, bit);
903
904 user_event_mm_put(user_mm);
905
906 return exists;
907}
908
909static struct user_event_enabler
910*user_event_enabler_create(struct user_reg *reg, struct user_event *user,
911 int *write_result)
912{
913 struct user_event_enabler *enabler;
914 struct user_event_mm *user_mm;
915 unsigned long uaddr = (unsigned long)reg->enable_addr;
916 int attempt = 0;
917
918 user_mm = current_user_event_mm();
919
920 if (!user_mm)
921 return NULL;
922
923 enabler = kzalloc_obj(*enabler, GFP_KERNEL_ACCOUNT);
924
925 if (!enabler)
926 goto out;
927
928 enabler->event = user;
929 enabler->addr = uaddr;
930 enabler->values = reg->enable_bit;
931
932#if BITS_PER_LONG >= 64
933 if (reg->enable_size == 4)
934 set_bit(ENABLE_VAL_32_ON_64_BIT, ENABLE_BITOPS(enabler));
935#endif
936
937retry:
938 /* Prevents state changes from racing with new enablers */
939 mutex_lock(&event_mutex);
940
941 /* Attempt to reflect the current state within the process */
942 mmap_read_lock(user_mm->mm);
943 *write_result = user_event_enabler_write(user_mm, enabler, false,
944 &attempt);
945 mmap_read_unlock(user_mm->mm);
946
947 /*
948 * If the write works, then we will track the enabler. A ref to the
949 * underlying user_event is held by the enabler to prevent it going
950 * away while the enabler is still in use by a process. The ref is
951 * removed when the enabler is destroyed. This means a event cannot
952 * be forcefully deleted from the system until all tasks using it
953 * exit or run exec(), which includes forks and clones.
954 */
955 if (!*write_result) {
956 user_event_get(user);
957 list_add_rcu(&enabler->mm_enablers_link, &user_mm->enablers);
958 }
959
960 mutex_unlock(&event_mutex);
961
962 if (*write_result) {
963 /* Attempt to fault-in and retry if it worked */
964 if (!user_event_mm_fault_in(user_mm, uaddr, attempt))
965 goto retry;
966
967 kfree(enabler);
968 enabler = NULL;
969 }
970out:
971 user_event_mm_put(user_mm);
972
973 return enabler;
974}
975
976static __always_inline __must_check
977bool user_event_last_ref(struct user_event *user)
978{
979 int last = 0;
980
981 if (user->reg_flags & USER_EVENT_REG_PERSIST)
982 last = 1;
983
984 return refcount_read(&user->refcnt) == last;
985}
986
987static __always_inline __must_check
988size_t copy_nofault(void *addr, size_t bytes, struct iov_iter *i)
989{
990 size_t ret;
991
992 pagefault_disable();
993
994 ret = copy_from_iter_nocache(addr, bytes, i);
995
996 pagefault_enable();
997
998 return ret;
999}
1000
1001static struct list_head *user_event_get_fields(struct trace_event_call *call)
1002{
1003 struct user_event *user = (struct user_event *)call->data;
1004
1005 return &user->fields;
1006}
1007
1008/*
1009 * Parses a register command for user_events
1010 * Format: event_name[:FLAG1[,FLAG2...]] [field1[;field2...]]
1011 *
1012 * Example event named 'test' with a 20 char 'msg' field with an unsigned int
1013 * 'id' field after:
1014 * test char[20] msg;unsigned int id
1015 *
1016 * NOTE: Offsets are from the user data perspective, they are not from the
1017 * trace_entry/buffer perspective. We automatically add the common properties
1018 * sizes to the offset for the user.
1019 *
1020 * Upon success user_event has its ref count increased by 1.
1021 */
1022static int user_event_parse_cmd(struct user_event_group *group,
1023 char *raw_command, struct user_event **newuser,
1024 int reg_flags)
1025{
1026 char *name = raw_command;
1027 char *args = strpbrk(name, " ");
1028 char *flags;
1029
1030 if (args)
1031 *args++ = '\0';
1032
1033 flags = strpbrk(name, ":");
1034
1035 if (flags)
1036 *flags++ = '\0';
1037
1038 return user_event_parse(group, name, args, flags, newuser, reg_flags);
1039}
1040
1041static int user_field_array_size(const char *type)
1042{
1043 const char *start = strchr(type, '[');
1044 char val[8];
1045 char *bracket;
1046 int size = 0;
1047
1048 if (start == NULL)
1049 return -EINVAL;
1050
1051 if (strscpy(val, start + 1, sizeof(val)) <= 0)
1052 return -EINVAL;
1053
1054 bracket = strchr(val, ']');
1055
1056 if (!bracket)
1057 return -EINVAL;
1058
1059 *bracket = '\0';
1060
1061 if (kstrtouint(val, 0, &size))
1062 return -EINVAL;
1063
1064 if (size > MAX_FIELD_ARRAY_SIZE)
1065 return -EINVAL;
1066
1067 return size;
1068}
1069
1070static int user_field_size(const char *type)
1071{
1072 /* long is not allowed from a user, since it's ambiguous in size */
1073 if (strcmp(type, "s64") == 0)
1074 return sizeof(s64);
1075 if (strcmp(type, "u64") == 0)
1076 return sizeof(u64);
1077 if (strcmp(type, "s32") == 0)
1078 return sizeof(s32);
1079 if (strcmp(type, "u32") == 0)
1080 return sizeof(u32);
1081 if (strcmp(type, "int") == 0)
1082 return sizeof(int);
1083 if (strcmp(type, "unsigned int") == 0)
1084 return sizeof(unsigned int);
1085 if (strcmp(type, "s16") == 0)
1086 return sizeof(s16);
1087 if (strcmp(type, "u16") == 0)
1088 return sizeof(u16);
1089 if (strcmp(type, "short") == 0)
1090 return sizeof(short);
1091 if (strcmp(type, "unsigned short") == 0)
1092 return sizeof(unsigned short);
1093 if (strcmp(type, "s8") == 0)
1094 return sizeof(s8);
1095 if (strcmp(type, "u8") == 0)
1096 return sizeof(u8);
1097 if (strcmp(type, "char") == 0)
1098 return sizeof(char);
1099 if (strcmp(type, "unsigned char") == 0)
1100 return sizeof(unsigned char);
1101 if (str_has_prefix(type, "char["))
1102 return user_field_array_size(type);
1103 if (str_has_prefix(type, "unsigned char["))
1104 return user_field_array_size(type);
1105 if (str_has_prefix(type, "__data_loc "))
1106 return sizeof(u32);
1107 if (str_has_prefix(type, "__rel_loc "))
1108 return sizeof(u32);
1109
1110 /* Unknown basic type, error */
1111 return -EINVAL;
1112}
1113
1114static void user_event_destroy_validators(struct user_event *user)
1115{
1116 struct user_event_validator *validator, *next;
1117 struct list_head *head = &user->validators;
1118
1119 list_for_each_entry_safe(validator, next, head, user_event_link) {
1120 list_del(&validator->user_event_link);
1121 kfree(validator);
1122 }
1123}
1124
1125static void user_event_destroy_fields(struct list_head *head)
1126{
1127 struct ftrace_event_field *field, *next;
1128
1129 list_for_each_entry_safe(field, next, head, link) {
1130 list_del(&field->link);
1131 kfree(field);
1132 }
1133}
1134
1135static int user_event_add_field(struct user_event *user, const char *type,
1136 const char *name, int offset, int size,
1137 int is_signed, int filter_type)
1138{
1139 struct user_event_validator *validator;
1140 struct ftrace_event_field *field;
1141 int validator_flags = 0;
1142
1143 field = kmalloc_obj(*field, GFP_KERNEL_ACCOUNT);
1144
1145 if (!field)
1146 return -ENOMEM;
1147
1148 if (str_has_prefix(type, "__data_loc "))
1149 goto add_validator;
1150
1151 if (str_has_prefix(type, "__rel_loc ")) {
1152 validator_flags |= VALIDATOR_REL;
1153 goto add_validator;
1154 }
1155
1156 goto add_field;
1157
1158add_validator:
1159 if (strstr(type, "char") != NULL)
1160 validator_flags |= VALIDATOR_ENSURE_NULL;
1161
1162 validator = kmalloc_obj(*validator, GFP_KERNEL_ACCOUNT);
1163
1164 if (!validator) {
1165 kfree(field);
1166 return -ENOMEM;
1167 }
1168
1169 validator->flags = validator_flags;
1170 validator->offset = offset;
1171
1172 /* Want sequential access when validating */
1173 list_add_tail(&validator->user_event_link, &user->validators);
1174
1175add_field:
1176 field->type = type;
1177 field->name = name;
1178 field->offset = offset;
1179 field->size = size;
1180 field->is_signed = is_signed;
1181 field->filter_type = filter_type;
1182
1183 if (filter_type == FILTER_OTHER)
1184 field->filter_type = filter_assign_type(type);
1185
1186 list_add(&field->link, &user->fields);
1187
1188 /*
1189 * Min size from user writes that are required, this does not include
1190 * the size of trace_entry (common fields).
1191 */
1192 user->min_size = (offset + size) - sizeof(struct trace_entry);
1193
1194 return 0;
1195}
1196
1197/*
1198 * Parses the values of a field within the description
1199 * Format: type name [size]
1200 */
1201static int user_event_parse_field(char *field, struct user_event *user,
1202 u32 *offset)
1203{
1204 char *part, *type, *name;
1205 u32 depth = 0, saved_offset = *offset;
1206 int len, size = -EINVAL;
1207 bool is_struct = false;
1208
1209 field = skip_spaces(field);
1210
1211 if (*field == '\0')
1212 return 0;
1213
1214 /* Handle types that have a space within */
1215 len = str_has_prefix(field, "unsigned ");
1216 if (len)
1217 goto skip_next;
1218
1219 len = str_has_prefix(field, "struct ");
1220 if (len) {
1221 is_struct = true;
1222 goto skip_next;
1223 }
1224
1225 len = str_has_prefix(field, "__data_loc unsigned ");
1226 if (len)
1227 goto skip_next;
1228
1229 len = str_has_prefix(field, "__data_loc ");
1230 if (len)
1231 goto skip_next;
1232
1233 len = str_has_prefix(field, "__rel_loc unsigned ");
1234 if (len)
1235 goto skip_next;
1236
1237 len = str_has_prefix(field, "__rel_loc ");
1238 if (len)
1239 goto skip_next;
1240
1241 goto parse;
1242skip_next:
1243 type = field;
1244 field = strpbrk(field + len, " ");
1245
1246 if (field == NULL)
1247 return -EINVAL;
1248
1249 *field++ = '\0';
1250 depth++;
1251parse:
1252 name = NULL;
1253
1254 while ((part = strsep(&field, " ")) != NULL) {
1255 switch (depth++) {
1256 case FIELD_DEPTH_TYPE:
1257 type = part;
1258 break;
1259 case FIELD_DEPTH_NAME:
1260 name = part;
1261 break;
1262 case FIELD_DEPTH_SIZE:
1263 if (!is_struct)
1264 return -EINVAL;
1265
1266 if (kstrtou32(part, 10, &size))
1267 return -EINVAL;
1268 break;
1269 default:
1270 return -EINVAL;
1271 }
1272 }
1273
1274 if (depth < FIELD_DEPTH_SIZE || !name)
1275 return -EINVAL;
1276
1277 if (depth == FIELD_DEPTH_SIZE)
1278 size = user_field_size(type);
1279
1280 if (size == 0)
1281 return -EINVAL;
1282
1283 if (size < 0)
1284 return size;
1285
1286 *offset = saved_offset + size;
1287
1288 return user_event_add_field(user, type, name, saved_offset, size,
1289 type[0] != 'u', FILTER_OTHER);
1290}
1291
1292static int user_event_parse_fields(struct user_event *user, char *args)
1293{
1294 char *field;
1295 u32 offset = sizeof(struct trace_entry);
1296 int ret = -EINVAL;
1297
1298 if (args == NULL)
1299 return 0;
1300
1301 while ((field = strsep(&args, ";")) != NULL) {
1302 ret = user_event_parse_field(field, user, &offset);
1303
1304 if (ret)
1305 break;
1306 }
1307
1308 return ret;
1309}
1310
1311static struct trace_event_fields user_event_fields_array[1];
1312
1313static const char *user_field_format(const char *type)
1314{
1315 if (strcmp(type, "s64") == 0)
1316 return "%lld";
1317 if (strcmp(type, "u64") == 0)
1318 return "%llu";
1319 if (strcmp(type, "s32") == 0)
1320 return "%d";
1321 if (strcmp(type, "u32") == 0)
1322 return "%u";
1323 if (strcmp(type, "int") == 0)
1324 return "%d";
1325 if (strcmp(type, "unsigned int") == 0)
1326 return "%u";
1327 if (strcmp(type, "s16") == 0)
1328 return "%d";
1329 if (strcmp(type, "u16") == 0)
1330 return "%u";
1331 if (strcmp(type, "short") == 0)
1332 return "%d";
1333 if (strcmp(type, "unsigned short") == 0)
1334 return "%u";
1335 if (strcmp(type, "s8") == 0)
1336 return "%d";
1337 if (strcmp(type, "u8") == 0)
1338 return "%u";
1339 if (strcmp(type, "char") == 0)
1340 return "%d";
1341 if (strcmp(type, "unsigned char") == 0)
1342 return "%u";
1343 if (strstr(type, "char[") != NULL)
1344 return "%s";
1345
1346 /* Unknown, likely struct, allowed treat as 64-bit */
1347 return "%llu";
1348}
1349
1350static bool user_field_is_dyn_string(const char *type, const char **str_func)
1351{
1352 if (str_has_prefix(type, "__data_loc ")) {
1353 *str_func = "__get_str";
1354 goto check;
1355 }
1356
1357 if (str_has_prefix(type, "__rel_loc ")) {
1358 *str_func = "__get_rel_str";
1359 goto check;
1360 }
1361
1362 return false;
1363check:
1364 return strstr(type, "char") != NULL;
1365}
1366
1367#define LEN_OR_ZERO (len ? len - pos : 0)
1368static int user_dyn_field_set_string(int argc, const char **argv, int *iout,
1369 char *buf, int len, bool *colon)
1370{
1371 int pos = 0, i = *iout;
1372
1373 *colon = false;
1374
1375 for (; i < argc; ++i) {
1376 if (i != *iout)
1377 pos += snprintf(buf + pos, LEN_OR_ZERO, " ");
1378
1379 pos += snprintf(buf + pos, LEN_OR_ZERO, "%s", argv[i]);
1380
1381 if (strchr(argv[i], ';')) {
1382 ++i;
1383 *colon = true;
1384 break;
1385 }
1386 }
1387
1388 /* Actual set, advance i */
1389 if (len != 0)
1390 *iout = i;
1391
1392 return pos + 1;
1393}
1394
1395static int user_field_set_string(struct ftrace_event_field *field,
1396 char *buf, int len, bool colon)
1397{
1398 int pos = 0;
1399
1400 pos += snprintf(buf + pos, LEN_OR_ZERO, "%s", field->type);
1401 pos += snprintf(buf + pos, LEN_OR_ZERO, " ");
1402 pos += snprintf(buf + pos, LEN_OR_ZERO, "%s", field->name);
1403
1404 if (str_has_prefix(field->type, "struct "))
1405 pos += snprintf(buf + pos, LEN_OR_ZERO, " %d", field->size);
1406
1407 if (colon)
1408 pos += snprintf(buf + pos, LEN_OR_ZERO, ";");
1409
1410 return pos + 1;
1411}
1412
1413static int user_event_set_print_fmt(struct user_event *user, char *buf, int len)
1414{
1415 struct ftrace_event_field *field;
1416 struct list_head *head = &user->fields;
1417 int pos = 0, depth = 0;
1418 const char *str_func;
1419
1420 pos += snprintf(buf + pos, LEN_OR_ZERO, "\"");
1421
1422 list_for_each_entry_reverse(field, head, link) {
1423 if (depth != 0)
1424 pos += snprintf(buf + pos, LEN_OR_ZERO, " ");
1425
1426 pos += snprintf(buf + pos, LEN_OR_ZERO, "%s=%s",
1427 field->name, user_field_format(field->type));
1428
1429 depth++;
1430 }
1431
1432 pos += snprintf(buf + pos, LEN_OR_ZERO, "\"");
1433
1434 list_for_each_entry_reverse(field, head, link) {
1435 if (user_field_is_dyn_string(field->type, &str_func))
1436 pos += snprintf(buf + pos, LEN_OR_ZERO,
1437 ", %s(%s)", str_func, field->name);
1438 else
1439 pos += snprintf(buf + pos, LEN_OR_ZERO,
1440 ", REC->%s", field->name);
1441 }
1442
1443 return pos + 1;
1444}
1445#undef LEN_OR_ZERO
1446
1447static int user_event_create_print_fmt(struct user_event *user)
1448{
1449 char *print_fmt;
1450 int len;
1451
1452 len = user_event_set_print_fmt(user, NULL, 0);
1453
1454 print_fmt = kmalloc(len, GFP_KERNEL_ACCOUNT);
1455
1456 if (!print_fmt)
1457 return -ENOMEM;
1458
1459 user_event_set_print_fmt(user, print_fmt, len);
1460
1461 user->call.print_fmt = print_fmt;
1462
1463 return 0;
1464}
1465
1466static enum print_line_t user_event_print_trace(struct trace_iterator *iter,
1467 int flags,
1468 struct trace_event *event)
1469{
1470 return print_event_fields(iter, event);
1471}
1472
1473static struct trace_event_functions user_event_funcs = {
1474 .trace = user_event_print_trace,
1475};
1476
1477static int user_event_set_call_visible(struct user_event *user, bool visible)
1478{
1479 CLASS(prepare_creds, cred)();
1480 if (!cred)
1481 return -ENOMEM;
1482
1483 /*
1484 * While by default tracefs is locked down, systems can be configured
1485 * to allow user_event files to be less locked down. The extreme case
1486 * being "other" has read/write access to user_events_data/status.
1487 *
1488 * When not locked down, processes may not have permissions to
1489 * add/remove calls themselves to tracefs. We need to temporarily
1490 * switch to root file permission to allow for this scenario.
1491 */
1492 cred->fsuid = GLOBAL_ROOT_UID;
1493
1494 scoped_with_creds(cred) {
1495 if (visible)
1496 return trace_add_event_call(&user->call);
1497
1498 return trace_remove_event_call(&user->call);
1499 }
1500}
1501
1502static int destroy_user_event(struct user_event *user)
1503{
1504 LIST_HEAD(fields);
1505 int ret = 0;
1506
1507 lockdep_assert_held(&event_mutex);
1508
1509 /*
1510 * Detach the fields before removing the call. Removing the event
1511 * frees the field list memory (trace_destroy_fields() is run on
1512 * successful removal and kmem_cache_free()s the fields), but the
1513 * fields here are allocated and owned by user_events. Destroy
1514 * them separately once removal has succeeded.
1515 */
1516 list_splice_init(&user->fields, &fields);
1517
1518 ret = user_event_set_call_visible(user, false);
1519
1520 if (ret) {
1521 /*
1522 * Removal failed and the event stays registered, recover
1523 * the fields so it is left in a consistent state.
1524 */
1525 list_splice(&fields, &user->fields);
1526 return ret;
1527 }
1528
1529 user_event_destroy_fields(&fields);
1530
1531 dyn_event_remove(&user->devent);
1532 hash_del(&user->node);
1533
1534 user_event_destroy_validators(user);
1535
1536 /* If we have different names, both must be freed */
1537 if (EVENT_NAME(user) != EVENT_TP_NAME(user))
1538 kfree(EVENT_TP_NAME(user));
1539
1540 kfree(user->call.print_fmt);
1541 kfree(EVENT_NAME(user));
1542 kfree(user);
1543
1544 if (current_user_events > 0)
1545 current_user_events--;
1546 else
1547 pr_alert("BUG: Bad current_user_events\n");
1548
1549 return ret;
1550}
1551
1552static struct user_event *find_user_event(struct user_event_group *group,
1553 char *name, int argc, const char **argv,
1554 u32 flags, u32 *outkey)
1555{
1556 struct user_event *user;
1557 u32 key = user_event_key(name);
1558
1559 *outkey = key;
1560
1561 hash_for_each_possible(group->register_table, user, node, key) {
1562 /*
1563 * Single-format events shouldn't return multi-format
1564 * events. Callers expect the underlying tracepoint to match
1565 * the name exactly in these cases. Only check like-formats.
1566 */
1567 if (EVENT_MULTI_FORMAT(flags) != EVENT_MULTI_FORMAT(user->reg_flags))
1568 continue;
1569
1570 if (strcmp(EVENT_NAME(user), name))
1571 continue;
1572
1573 if (user_fields_match(user, argc, argv))
1574 return user_event_get(user);
1575
1576 /* Scan others if this is a multi-format event */
1577 if (EVENT_MULTI_FORMAT(flags))
1578 continue;
1579
1580 return ERR_PTR(-EADDRINUSE);
1581 }
1582
1583 return NULL;
1584}
1585
1586static int user_event_validate(struct user_event *user, void *data, int len)
1587{
1588 struct list_head *head = &user->validators;
1589 struct user_event_validator *validator;
1590 void *pos, *end = data + len;
1591 u32 loc, offset, size;
1592
1593 list_for_each_entry(validator, head, user_event_link) {
1594 pos = data + validator->offset;
1595
1596 /* Already done min_size check, no bounds check here */
1597 loc = *(u32 *)pos;
1598 offset = loc & 0xffff;
1599 size = loc >> 16;
1600
1601 if (likely(validator->flags & VALIDATOR_REL))
1602 pos += offset + sizeof(loc);
1603 else
1604 pos = data + offset;
1605
1606 pos += size;
1607
1608 if (unlikely(pos > end))
1609 return -EFAULT;
1610
1611 if (likely(validator->flags & VALIDATOR_ENSURE_NULL))
1612 if (unlikely(*(char *)(pos - 1) != '\0'))
1613 return -EFAULT;
1614 }
1615
1616 return 0;
1617}
1618
1619/*
1620 * Writes the user supplied payload out to a trace file.
1621 */
1622static void user_event_ftrace(struct user_event *user, struct iov_iter *i,
1623 void *tpdata, bool *faulted)
1624{
1625 struct trace_event_file *file;
1626 struct trace_entry *entry;
1627 struct trace_event_buffer event_buffer;
1628 size_t size = sizeof(*entry) + i->count;
1629
1630 file = (struct trace_event_file *)tpdata;
1631
1632 if (!file ||
1633 !(file->flags & EVENT_FILE_FL_ENABLED) ||
1634 trace_trigger_soft_disabled(file))
1635 return;
1636
1637 /* Allocates and fills trace_entry, + 1 of this is data payload */
1638 entry = trace_event_buffer_reserve(&event_buffer, file, size);
1639
1640 if (unlikely(!entry))
1641 return;
1642
1643 if (unlikely(i->count != 0 && !copy_nofault(entry + 1, i->count, i)))
1644 goto discard;
1645
1646 if (!list_empty(&user->validators) &&
1647 unlikely(user_event_validate(user, entry, size)))
1648 goto discard;
1649
1650 trace_event_buffer_commit(&event_buffer);
1651
1652 return;
1653discard:
1654 *faulted = true;
1655 __trace_event_discard_commit(event_buffer.buffer,
1656 event_buffer.event);
1657}
1658
1659#ifdef CONFIG_PERF_EVENTS
1660/*
1661 * Writes the user supplied payload out to perf ring buffer.
1662 */
1663static void user_event_perf(struct user_event *user, struct iov_iter *i,
1664 void *tpdata, bool *faulted)
1665{
1666 struct hlist_head *perf_head;
1667
1668 perf_head = this_cpu_ptr(user->call.perf_events);
1669
1670 if (perf_head && !hlist_empty(perf_head)) {
1671 struct trace_entry *perf_entry;
1672 struct pt_regs *regs;
1673 size_t size = sizeof(*perf_entry) + i->count;
1674 int context;
1675
1676 perf_entry = perf_trace_buf_alloc(ALIGN(size, 8),
1677 &regs, &context);
1678
1679 if (unlikely(!perf_entry))
1680 return;
1681
1682 perf_fetch_caller_regs(regs);
1683
1684 if (unlikely(i->count != 0 && !copy_nofault(perf_entry + 1, i->count, i)))
1685 goto discard;
1686
1687 if (!list_empty(&user->validators) &&
1688 unlikely(user_event_validate(user, perf_entry, size)))
1689 goto discard;
1690
1691 perf_trace_buf_submit(perf_entry, size, context,
1692 user->call.event.type, 1, regs,
1693 perf_head, NULL);
1694
1695 return;
1696discard:
1697 *faulted = true;
1698 perf_swevent_put_recursion_context(context);
1699 }
1700}
1701#endif
1702
1703/*
1704 * Update the enabled bit among all user processes.
1705 */
1706static void update_enable_bit_for(struct user_event *user)
1707{
1708 struct tracepoint *tp = &user->tracepoint;
1709 char status = 0;
1710
1711 if (static_key_enabled(&tp->key)) {
1712 struct tracepoint_func *probe_func_ptr;
1713 user_event_func_t probe_func;
1714
1715 rcu_read_lock_sched();
1716
1717 probe_func_ptr = rcu_dereference_sched(tp->funcs);
1718
1719 if (probe_func_ptr) {
1720 do {
1721 probe_func = probe_func_ptr->func;
1722
1723 if (probe_func == user_event_ftrace)
1724 status |= EVENT_STATUS_FTRACE;
1725#ifdef CONFIG_PERF_EVENTS
1726 else if (probe_func == user_event_perf)
1727 status |= EVENT_STATUS_PERF;
1728#endif
1729 else
1730 status |= EVENT_STATUS_OTHER;
1731 } while ((++probe_func_ptr)->func);
1732 }
1733
1734 rcu_read_unlock_sched();
1735 }
1736
1737 user->status = status;
1738
1739 user_event_enabler_update(user);
1740}
1741
1742/*
1743 * Register callback for our events from tracing sub-systems.
1744 */
1745static int user_event_reg(struct trace_event_call *call,
1746 enum trace_reg type,
1747 void *data)
1748{
1749 struct user_event *user = (struct user_event *)call->data;
1750 int ret = 0;
1751
1752 if (!user)
1753 return -ENOENT;
1754
1755 switch (type) {
1756 case TRACE_REG_REGISTER:
1757 ret = tracepoint_probe_register(call->tp,
1758 call->class->probe,
1759 data);
1760 if (!ret)
1761 goto inc;
1762 break;
1763
1764 case TRACE_REG_UNREGISTER:
1765 tracepoint_probe_unregister(call->tp,
1766 call->class->probe,
1767 data);
1768 goto dec;
1769
1770#ifdef CONFIG_PERF_EVENTS
1771 case TRACE_REG_PERF_REGISTER:
1772 ret = tracepoint_probe_register(call->tp,
1773 call->class->perf_probe,
1774 data);
1775 if (!ret)
1776 goto inc;
1777 break;
1778
1779 case TRACE_REG_PERF_UNREGISTER:
1780 tracepoint_probe_unregister(call->tp,
1781 call->class->perf_probe,
1782 data);
1783 goto dec;
1784
1785 case TRACE_REG_PERF_OPEN:
1786 case TRACE_REG_PERF_CLOSE:
1787 case TRACE_REG_PERF_ADD:
1788 case TRACE_REG_PERF_DEL:
1789 break;
1790#endif
1791 }
1792
1793 return ret;
1794inc:
1795 user_event_get(user);
1796 update_enable_bit_for(user);
1797 return 0;
1798dec:
1799 update_enable_bit_for(user);
1800 user_event_put(user, true);
1801 return 0;
1802}
1803
1804static int user_event_create(const char *raw_command)
1805{
1806 struct user_event_group *group;
1807 struct user_event *user;
1808 char *name;
1809 int ret;
1810
1811 if (!str_has_prefix(raw_command, USER_EVENTS_PREFIX))
1812 return -ECANCELED;
1813
1814 raw_command += USER_EVENTS_PREFIX_LEN;
1815 raw_command = skip_spaces(raw_command);
1816
1817 name = kstrdup(raw_command, GFP_KERNEL_ACCOUNT);
1818
1819 if (!name)
1820 return -ENOMEM;
1821
1822 group = current_user_event_group();
1823
1824 if (!group) {
1825 kfree(name);
1826 return -ENOENT;
1827 }
1828
1829 mutex_lock(&group->reg_mutex);
1830
1831 /* Dyn events persist, otherwise they would cleanup immediately */
1832 ret = user_event_parse_cmd(group, name, &user, USER_EVENT_REG_PERSIST);
1833
1834 if (!ret)
1835 user_event_put(user, false);
1836
1837 mutex_unlock(&group->reg_mutex);
1838
1839 if (ret)
1840 kfree(name);
1841
1842 return ret;
1843}
1844
1845static int user_event_show(struct seq_file *m, struct dyn_event *ev)
1846{
1847 struct user_event *user = container_of(ev, struct user_event, devent);
1848 struct ftrace_event_field *field;
1849 struct list_head *head;
1850 int depth = 0;
1851
1852 seq_printf(m, "%s%s", USER_EVENTS_PREFIX, EVENT_NAME(user));
1853
1854 head = trace_get_fields(&user->call);
1855
1856 list_for_each_entry_reverse(field, head, link) {
1857 if (depth == 0)
1858 seq_putc(m, ' ');
1859 else
1860 seq_puts(m, "; ");
1861
1862 seq_printf(m, "%s %s", field->type, field->name);
1863
1864 if (str_has_prefix(field->type, "struct "))
1865 seq_printf(m, " %d", field->size);
1866
1867 depth++;
1868 }
1869
1870 seq_putc(m, '\n');
1871
1872 return 0;
1873}
1874
1875static bool user_event_is_busy(struct dyn_event *ev)
1876{
1877 struct user_event *user = container_of(ev, struct user_event, devent);
1878
1879 return !user_event_last_ref(user);
1880}
1881
1882static int user_event_free(struct dyn_event *ev)
1883{
1884 struct user_event *user = container_of(ev, struct user_event, devent);
1885
1886 if (!user_event_last_ref(user))
1887 return -EBUSY;
1888
1889 if (!user_event_capable(user->reg_flags))
1890 return -EPERM;
1891
1892 return destroy_user_event(user);
1893}
1894
1895static bool user_field_match(struct ftrace_event_field *field, int argc,
1896 const char **argv, int *iout)
1897{
1898 char *field_name = NULL, *dyn_field_name = NULL;
1899 bool colon = false, match = false;
1900 int dyn_len, len;
1901
1902 if (*iout >= argc)
1903 return false;
1904
1905 dyn_len = user_dyn_field_set_string(argc, argv, iout, dyn_field_name,
1906 0, &colon);
1907
1908 len = user_field_set_string(field, field_name, 0, colon);
1909
1910 if (dyn_len != len)
1911 return false;
1912
1913 dyn_field_name = kmalloc(dyn_len, GFP_KERNEL);
1914 field_name = kmalloc(len, GFP_KERNEL);
1915
1916 if (!dyn_field_name || !field_name)
1917 goto out;
1918
1919 user_dyn_field_set_string(argc, argv, iout, dyn_field_name,
1920 dyn_len, &colon);
1921
1922 user_field_set_string(field, field_name, len, colon);
1923
1924 match = strcmp(dyn_field_name, field_name) == 0;
1925out:
1926 kfree(dyn_field_name);
1927 kfree(field_name);
1928
1929 return match;
1930}
1931
1932static bool user_fields_match(struct user_event *user, int argc,
1933 const char **argv)
1934{
1935 struct ftrace_event_field *field;
1936 struct list_head *head = &user->fields;
1937 int i = 0;
1938
1939 if (argc == 0)
1940 return list_empty(head);
1941
1942 list_for_each_entry_reverse(field, head, link) {
1943 if (!user_field_match(field, argc, argv, &i))
1944 return false;
1945 }
1946
1947 if (i != argc)
1948 return false;
1949
1950 return true;
1951}
1952
1953static bool user_event_match(const char *system, const char *event,
1954 int argc, const char **argv, struct dyn_event *ev)
1955{
1956 struct user_event *user = container_of(ev, struct user_event, devent);
1957 bool match;
1958
1959 match = strcmp(EVENT_NAME(user), event) == 0;
1960
1961 if (match && system) {
1962 match = strcmp(system, user->group->system_name) == 0 ||
1963 strcmp(system, user->group->system_multi_name) == 0;
1964 }
1965
1966 if (match)
1967 match = user_fields_match(user, argc, argv);
1968
1969 return match;
1970}
1971
1972static struct dyn_event_operations user_event_dops = {
1973 .create = user_event_create,
1974 .show = user_event_show,
1975 .is_busy = user_event_is_busy,
1976 .free = user_event_free,
1977 .match = user_event_match,
1978};
1979
1980static int user_event_trace_register(struct user_event *user)
1981{
1982 int ret;
1983
1984 ret = register_trace_event(&user->call.event);
1985
1986 if (!ret)
1987 return -ENODEV;
1988
1989 ret = user_event_set_call_visible(user, true);
1990
1991 if (ret)
1992 unregister_trace_event(&user->call.event);
1993
1994 return ret;
1995}
1996
1997static int user_event_set_tp_name(struct user_event *user)
1998{
1999 lockdep_assert_held(&user->group->reg_mutex);
2000
2001 if (EVENT_MULTI_FORMAT(user->reg_flags)) {
2002 char *multi_name;
2003
2004 multi_name = kasprintf(GFP_KERNEL_ACCOUNT, "%s.%llx",
2005 user->reg_name, user->group->multi_id);
2006
2007 if (!multi_name)
2008 return -ENOMEM;
2009
2010 user->call.name = multi_name;
2011 user->tracepoint.name = multi_name;
2012
2013 /* Inc to ensure unique multi-event name next time */
2014 user->group->multi_id++;
2015 } else {
2016 /* Non Multi-format uses register name */
2017 user->call.name = user->reg_name;
2018 user->tracepoint.name = user->reg_name;
2019 }
2020
2021 return 0;
2022}
2023
2024/*
2025 * Counts how many ';' without a trailing space are in the args.
2026 */
2027static int count_semis_no_space(char *args)
2028{
2029 int count = 0;
2030
2031 while ((args = strchr(args, ';'))) {
2032 args++;
2033
2034 if (!isspace(*args))
2035 count++;
2036 }
2037
2038 return count;
2039}
2040
2041/*
2042 * Copies the arguments while ensuring all ';' have a trailing space.
2043 */
2044static char *insert_space_after_semis(char *args, int count)
2045{
2046 char *fixed, *pos;
2047 int len;
2048
2049 len = strlen(args) + count;
2050 fixed = kmalloc(len + 1, GFP_KERNEL);
2051
2052 if (!fixed)
2053 return NULL;
2054
2055 pos = fixed;
2056
2057 /* Insert a space after ';' if there is no trailing space. */
2058 while (*args) {
2059 *pos = *args++;
2060
2061 if (*pos++ == ';' && !isspace(*args))
2062 *pos++ = ' ';
2063 }
2064
2065 *pos = '\0';
2066
2067 return fixed;
2068}
2069
2070static char **user_event_argv_split(char *args, int *argc)
2071{
2072 char **split;
2073 char *fixed;
2074 int count;
2075
2076 /* Count how many ';' without a trailing space */
2077 count = count_semis_no_space(args);
2078
2079 /* No fixup is required */
2080 if (!count)
2081 return argv_split(GFP_KERNEL, args, argc);
2082
2083 /* We must fixup 'field;field' to 'field; field' */
2084 fixed = insert_space_after_semis(args, count);
2085
2086 if (!fixed)
2087 return NULL;
2088
2089 /* We do a normal split afterwards */
2090 split = argv_split(GFP_KERNEL, fixed, argc);
2091
2092 /* We can free since argv_split makes a copy */
2093 kfree(fixed);
2094
2095 return split;
2096}
2097
2098/*
2099 * Parses the event name, arguments and flags then registers if successful.
2100 * The name buffer lifetime is owned by this method for success cases only.
2101 * Upon success the returned user_event has its ref count increased by 1.
2102 */
2103static int user_event_parse(struct user_event_group *group, char *name,
2104 char *args, char *flags,
2105 struct user_event **newuser, int reg_flags)
2106{
2107 struct user_event *user;
2108 char **argv = NULL;
2109 int argc = 0;
2110 int ret;
2111 u32 key;
2112
2113 /* Currently don't support any text based flags */
2114 if (flags != NULL)
2115 return -EINVAL;
2116
2117 if (!user_event_capable(reg_flags))
2118 return -EPERM;
2119
2120 if (args) {
2121 argv = user_event_argv_split(args, &argc);
2122
2123 if (!argv)
2124 return -ENOMEM;
2125 }
2126
2127 /* Prevent dyn_event from racing */
2128 mutex_lock(&event_mutex);
2129 user = find_user_event(group, name, argc, (const char **)argv,
2130 reg_flags, &key);
2131 mutex_unlock(&event_mutex);
2132
2133 if (argv)
2134 argv_free(argv);
2135
2136 if (IS_ERR(user))
2137 return PTR_ERR(user);
2138
2139 if (user) {
2140 *newuser = user;
2141 /*
2142 * Name is allocated by caller, free it since it already exists.
2143 * Caller only worries about failure cases for freeing.
2144 */
2145 kfree(name);
2146
2147 return 0;
2148 }
2149
2150 user = kzalloc_obj(*user, GFP_KERNEL_ACCOUNT);
2151
2152 if (!user)
2153 return -ENOMEM;
2154
2155 INIT_LIST_HEAD(&user->class.fields);
2156 INIT_LIST_HEAD(&user->fields);
2157 INIT_LIST_HEAD(&user->validators);
2158
2159 user->group = group;
2160 user->reg_name = name;
2161 user->reg_flags = reg_flags;
2162
2163 ret = user_event_set_tp_name(user);
2164
2165 if (ret)
2166 goto put_user;
2167
2168 ret = user_event_parse_fields(user, args);
2169
2170 if (ret)
2171 goto put_user;
2172
2173 ret = user_event_create_print_fmt(user);
2174
2175 if (ret)
2176 goto put_user;
2177
2178 user->call.data = user;
2179 user->call.class = &user->class;
2180 user->call.flags = TRACE_EVENT_FL_TRACEPOINT;
2181 user->call.tp = &user->tracepoint;
2182 user->call.event.funcs = &user_event_funcs;
2183
2184 if (EVENT_MULTI_FORMAT(user->reg_flags))
2185 user->class.system = group->system_multi_name;
2186 else
2187 user->class.system = group->system_name;
2188
2189 user->class.fields_array = user_event_fields_array;
2190 user->class.get_fields = user_event_get_fields;
2191 user->class.reg = user_event_reg;
2192 user->class.probe = user_event_ftrace;
2193#ifdef CONFIG_PERF_EVENTS
2194 user->class.perf_probe = user_event_perf;
2195#endif
2196
2197 mutex_lock(&event_mutex);
2198
2199 if (current_user_events >= max_user_events) {
2200 ret = -EMFILE;
2201 goto put_user_lock;
2202 }
2203
2204 ret = user_event_trace_register(user);
2205
2206 if (ret)
2207 goto put_user_lock;
2208
2209 if (user->reg_flags & USER_EVENT_REG_PERSIST) {
2210 /* Ensure we track self ref and caller ref (2) */
2211 refcount_set(&user->refcnt, 2);
2212 } else {
2213 /* Ensure we track only caller ref (1) */
2214 refcount_set(&user->refcnt, 1);
2215 }
2216
2217 dyn_event_init(&user->devent, &user_event_dops);
2218 dyn_event_add(&user->devent, &user->call);
2219 hash_add(group->register_table, &user->node, key);
2220 current_user_events++;
2221
2222 mutex_unlock(&event_mutex);
2223
2224 *newuser = user;
2225 return 0;
2226put_user_lock:
2227 mutex_unlock(&event_mutex);
2228put_user:
2229 user_event_destroy_fields(&user->fields);
2230 user_event_destroy_validators(user);
2231 kfree(user->call.print_fmt);
2232
2233 /* Caller frees reg_name on error, but not multi-name */
2234 if (EVENT_NAME(user) != EVENT_TP_NAME(user))
2235 kfree(EVENT_TP_NAME(user));
2236
2237 kfree(user);
2238 return ret;
2239}
2240
2241/*
2242 * Deletes previously created events if they are no longer being used.
2243 */
2244static int delete_user_event(struct user_event_group *group, char *name)
2245{
2246 struct user_event *user;
2247 struct hlist_node *tmp;
2248 u32 key = user_event_key(name);
2249 int ret = -ENOENT;
2250
2251 /* Attempt to delete all event(s) with the name passed in */
2252 hash_for_each_possible_safe(group->register_table, user, tmp, node, key) {
2253 if (strcmp(EVENT_NAME(user), name))
2254 continue;
2255
2256 if (!user_event_last_ref(user))
2257 return -EBUSY;
2258
2259 if (!user_event_capable(user->reg_flags))
2260 return -EPERM;
2261
2262 ret = destroy_user_event(user);
2263
2264 if (ret)
2265 goto out;
2266 }
2267out:
2268 return ret;
2269}
2270
2271/*
2272 * Validates the user payload and writes via iterator.
2273 */
2274static ssize_t user_events_write_core(struct file *file, struct iov_iter *i)
2275{
2276 struct user_event_file_info *info = file->private_data;
2277 struct user_event_refs *refs;
2278 struct user_event *user = NULL;
2279 struct tracepoint *tp;
2280 ssize_t ret = i->count;
2281 int idx;
2282
2283 if (unlikely(copy_from_iter(&idx, sizeof(idx), i) != sizeof(idx)))
2284 return -EFAULT;
2285
2286 if (idx < 0)
2287 return -EINVAL;
2288
2289 rcu_read_lock_sched();
2290
2291 refs = rcu_dereference_sched(info->refs);
2292
2293 /*
2294 * The refs->events array is protected by RCU, and new items may be
2295 * added. But the user retrieved from indexing into the events array
2296 * shall be immutable while the file is opened.
2297 */
2298 if (likely(refs && idx < refs->count))
2299 user = refs->events[idx];
2300
2301 rcu_read_unlock_sched();
2302
2303 if (unlikely(user == NULL))
2304 return -ENOENT;
2305
2306 if (unlikely(i->count < user->min_size))
2307 return -EINVAL;
2308
2309 tp = &user->tracepoint;
2310
2311 /*
2312 * It's possible key.enabled disables after this check, however
2313 * we don't mind if a few events are included in this condition.
2314 */
2315 if (likely(static_key_enabled(&tp->key))) {
2316 struct tracepoint_func *probe_func_ptr;
2317 user_event_func_t probe_func;
2318 struct iov_iter copy;
2319 void *tpdata;
2320 bool faulted;
2321
2322 if (unlikely(fault_in_iov_iter_readable(i, i->count)))
2323 return -EFAULT;
2324
2325 faulted = false;
2326
2327 rcu_read_lock_sched();
2328
2329 probe_func_ptr = rcu_dereference_sched(tp->funcs);
2330
2331 if (probe_func_ptr) {
2332 do {
2333 copy = *i;
2334 probe_func = probe_func_ptr->func;
2335 tpdata = probe_func_ptr->data;
2336 probe_func(user, &copy, tpdata, &faulted);
2337 } while ((++probe_func_ptr)->func);
2338 }
2339
2340 rcu_read_unlock_sched();
2341
2342 if (unlikely(faulted))
2343 return -EFAULT;
2344 } else
2345 return -EBADF;
2346
2347 return ret;
2348}
2349
2350static int user_events_open(struct inode *node, struct file *file)
2351{
2352 struct user_event_group *group;
2353 struct user_event_file_info *info;
2354
2355 group = current_user_event_group();
2356
2357 if (!group)
2358 return -ENOENT;
2359
2360 info = kzalloc_obj(*info, GFP_KERNEL_ACCOUNT);
2361
2362 if (!info)
2363 return -ENOMEM;
2364
2365 info->group = group;
2366
2367 file->private_data = info;
2368
2369 return 0;
2370}
2371
2372static ssize_t user_events_write(struct file *file, const char __user *ubuf,
2373 size_t count, loff_t *ppos)
2374{
2375 struct iov_iter i;
2376
2377 if (unlikely(*ppos != 0))
2378 return -EFAULT;
2379
2380 if (unlikely(import_ubuf(ITER_SOURCE, (char __user *)ubuf, count, &i)))
2381 return -EFAULT;
2382
2383 return user_events_write_core(file, &i);
2384}
2385
2386static ssize_t user_events_write_iter(struct kiocb *kp, struct iov_iter *i)
2387{
2388 return user_events_write_core(kp->ki_filp, i);
2389}
2390
2391static int user_events_ref_add(struct user_event_file_info *info,
2392 struct user_event *user)
2393{
2394 struct user_event_group *group = info->group;
2395 struct user_event_refs *refs, *new_refs;
2396 int i, size, count = 0;
2397
2398 refs = rcu_dereference_protected(info->refs,
2399 lockdep_is_held(&group->reg_mutex));
2400
2401 if (refs) {
2402 count = refs->count;
2403
2404 for (i = 0; i < count; ++i)
2405 if (refs->events[i] == user)
2406 return i;
2407 }
2408
2409 size = struct_size(refs, events, count + 1);
2410
2411 new_refs = kzalloc(size, GFP_KERNEL_ACCOUNT);
2412
2413 if (!new_refs)
2414 return -ENOMEM;
2415
2416 new_refs->count = count + 1;
2417
2418 for (i = 0; i < count; ++i)
2419 new_refs->events[i] = refs->events[i];
2420
2421 new_refs->events[i] = user_event_get(user);
2422
2423 rcu_assign_pointer(info->refs, new_refs);
2424
2425 if (refs)
2426 kfree_rcu(refs, rcu);
2427
2428 return i;
2429}
2430
2431static long user_reg_get(struct user_reg __user *ureg, struct user_reg *kreg)
2432{
2433 u32 size;
2434 long ret;
2435
2436 ret = get_user(size, &ureg->size);
2437
2438 if (ret)
2439 return ret;
2440
2441 if (size > PAGE_SIZE)
2442 return -E2BIG;
2443
2444 if (size < offsetofend(struct user_reg, write_index))
2445 return -EINVAL;
2446
2447 ret = copy_struct_from_user(kreg, sizeof(*kreg), ureg, size);
2448
2449 if (ret)
2450 return ret;
2451
2452 /* Ensure only valid flags */
2453 if (kreg->flags & ~(USER_EVENT_REG_MAX-1))
2454 return -EINVAL;
2455
2456 /* Ensure supported size */
2457 switch (kreg->enable_size) {
2458 case 4:
2459 /* 32-bit */
2460 break;
2461#if BITS_PER_LONG >= 64
2462 case 8:
2463 /* 64-bit */
2464 break;
2465#endif
2466 default:
2467 return -EINVAL;
2468 }
2469
2470 /* Ensure natural alignment */
2471 if (kreg->enable_addr % kreg->enable_size)
2472 return -EINVAL;
2473
2474 /* Ensure bit range for size */
2475 if (kreg->enable_bit > (kreg->enable_size * BITS_PER_BYTE) - 1)
2476 return -EINVAL;
2477
2478 /* Ensure accessible */
2479 if (!access_ok((const void __user *)(uintptr_t)kreg->enable_addr,
2480 kreg->enable_size))
2481 return -EFAULT;
2482
2483 kreg->size = size;
2484
2485 return 0;
2486}
2487
2488/*
2489 * Registers a user_event on behalf of a user process.
2490 */
2491static long user_events_ioctl_reg(struct user_event_file_info *info,
2492 unsigned long uarg)
2493{
2494 struct user_reg __user *ureg = (struct user_reg __user *)uarg;
2495 struct user_reg reg;
2496 struct user_event *user;
2497 struct user_event_enabler *enabler;
2498 char *name;
2499 long ret;
2500 int write_result;
2501
2502 ret = user_reg_get(ureg, &reg);
2503
2504 if (ret)
2505 return ret;
2506
2507 /*
2508 * Prevent users from using the same address and bit multiple times
2509 * within the same mm address space. This can cause unexpected behavior
2510 * for user processes that is far easier to debug if this is explicitly
2511 * an error upon registering.
2512 */
2513 if (current_user_event_enabler_exists((unsigned long)reg.enable_addr,
2514 reg.enable_bit))
2515 return -EADDRINUSE;
2516
2517 name = strndup_user((const char __user *)(uintptr_t)reg.name_args,
2518 MAX_EVENT_DESC);
2519
2520 if (IS_ERR(name)) {
2521 ret = PTR_ERR(name);
2522 return ret;
2523 }
2524
2525 ret = user_event_parse_cmd(info->group, name, &user, reg.flags);
2526
2527 if (ret) {
2528 kfree(name);
2529 return ret;
2530 }
2531
2532 ret = user_events_ref_add(info, user);
2533
2534 /* No longer need parse ref, ref_add either worked or not */
2535 user_event_put(user, false);
2536
2537 /* Positive number is index and valid */
2538 if (ret < 0)
2539 return ret;
2540
2541 /*
2542 * user_events_ref_add succeeded:
2543 * At this point we have a user_event, it's lifetime is bound by the
2544 * reference count, not this file. If anything fails, the user_event
2545 * still has a reference until the file is released. During release
2546 * any remaining references (from user_events_ref_add) are decremented.
2547 *
2548 * Attempt to create an enabler, which too has a lifetime tied in the
2549 * same way for the event. Once the task that caused the enabler to be
2550 * created exits or issues exec() then the enablers it has created
2551 * will be destroyed and the ref to the event will be decremented.
2552 */
2553 enabler = user_event_enabler_create(&reg, user, &write_result);
2554
2555 if (!enabler)
2556 return -ENOMEM;
2557
2558 /* Write failed/faulted, give error back to caller */
2559 if (write_result)
2560 return write_result;
2561
2562 put_user((u32)ret, &ureg->write_index);
2563
2564 return 0;
2565}
2566
2567/*
2568 * Deletes a user_event on behalf of a user process.
2569 */
2570static long user_events_ioctl_del(struct user_event_file_info *info,
2571 unsigned long uarg)
2572{
2573 void __user *ubuf = (void __user *)uarg;
2574 char *name;
2575 long ret;
2576
2577 name = strndup_user(ubuf, MAX_EVENT_DESC);
2578
2579 if (IS_ERR(name))
2580 return PTR_ERR(name);
2581
2582 /* event_mutex prevents dyn_event from racing */
2583 mutex_lock(&event_mutex);
2584 ret = delete_user_event(info->group, name);
2585 mutex_unlock(&event_mutex);
2586
2587 kfree(name);
2588
2589 return ret;
2590}
2591
2592static long user_unreg_get(struct user_unreg __user *ureg,
2593 struct user_unreg *kreg)
2594{
2595 u32 size;
2596 long ret;
2597
2598 ret = get_user(size, &ureg->size);
2599
2600 if (ret)
2601 return ret;
2602
2603 if (size > PAGE_SIZE)
2604 return -E2BIG;
2605
2606 if (size < offsetofend(struct user_unreg, disable_addr))
2607 return -EINVAL;
2608
2609 ret = copy_struct_from_user(kreg, sizeof(*kreg), ureg, size);
2610
2611 /* Ensure no reserved values, since we don't support any yet */
2612 if (kreg->__reserved || kreg->__reserved2)
2613 return -EINVAL;
2614
2615 return ret;
2616}
2617
2618static int user_event_mm_clear_bit(struct user_event_mm *user_mm,
2619 unsigned long uaddr, unsigned char bit,
2620 unsigned long flags)
2621{
2622 struct user_event_enabler enabler;
2623 int result;
2624 int attempt = 0;
2625
2626 memset(&enabler, 0, sizeof(enabler));
2627 enabler.addr = uaddr;
2628 enabler.values = bit | flags;
2629retry:
2630 /* Prevents state changes from racing with new enablers */
2631 mutex_lock(&event_mutex);
2632
2633 /* Force the bit to be cleared, since no event is attached */
2634 mmap_read_lock(user_mm->mm);
2635 result = user_event_enabler_write(user_mm, &enabler, false, &attempt);
2636 mmap_read_unlock(user_mm->mm);
2637
2638 mutex_unlock(&event_mutex);
2639
2640 if (result) {
2641 /* Attempt to fault-in and retry if it worked */
2642 if (!user_event_mm_fault_in(user_mm, uaddr, attempt))
2643 goto retry;
2644 }
2645
2646 return result;
2647}
2648
2649/*
2650 * Unregisters an enablement address/bit within a task/user mm.
2651 */
2652static long user_events_ioctl_unreg(unsigned long uarg)
2653{
2654 struct user_unreg __user *ureg = (struct user_unreg __user *)uarg;
2655 struct user_event_mm *mm = current->user_event_mm;
2656 struct user_event_enabler *enabler, *next;
2657 struct user_unreg reg;
2658 unsigned long flags;
2659 long ret;
2660
2661 ret = user_unreg_get(ureg, &reg);
2662
2663 if (ret)
2664 return ret;
2665
2666 if (!mm)
2667 return -ENOENT;
2668
2669 flags = 0;
2670 ret = -ENOENT;
2671
2672 /*
2673 * Flags freeing and faulting are used to indicate if the enabler is in
2674 * use at all. When faulting is set a page-fault is occurring asyncly.
2675 * During async fault if freeing is set, the enabler will be destroyed.
2676 * If no async fault is happening, we can destroy it now since we hold
2677 * the event_mutex during these checks.
2678 */
2679 mutex_lock(&event_mutex);
2680
2681 list_for_each_entry_safe(enabler, next, &mm->enablers, mm_enablers_link) {
2682 if (enabler->addr == reg.disable_addr &&
2683 ENABLE_BIT(enabler) == reg.disable_bit) {
2684 set_bit(ENABLE_VAL_FREEING_BIT, ENABLE_BITOPS(enabler));
2685
2686 /* We must keep compat flags for the clear */
2687 flags |= enabler->values & ENABLE_VAL_COMPAT_MASK;
2688
2689 if (!test_bit(ENABLE_VAL_FAULTING_BIT, ENABLE_BITOPS(enabler)))
2690 user_event_enabler_destroy(enabler);
2691
2692 /* Removed at least one */
2693 ret = 0;
2694 }
2695 }
2696
2697 mutex_unlock(&event_mutex);
2698
2699 /* Ensure bit is now cleared for user, regardless of event status */
2700 if (!ret)
2701 ret = user_event_mm_clear_bit(mm, reg.disable_addr,
2702 reg.disable_bit, flags);
2703
2704 return ret;
2705}
2706
2707/*
2708 * Handles the ioctl from user mode to register or alter operations.
2709 */
2710static long user_events_ioctl(struct file *file, unsigned int cmd,
2711 unsigned long uarg)
2712{
2713 struct user_event_file_info *info = file->private_data;
2714 struct user_event_group *group = info->group;
2715 long ret = -ENOTTY;
2716
2717 switch (cmd) {
2718 case DIAG_IOCSREG:
2719 mutex_lock(&group->reg_mutex);
2720 ret = user_events_ioctl_reg(info, uarg);
2721 mutex_unlock(&group->reg_mutex);
2722 break;
2723
2724 case DIAG_IOCSDEL:
2725 mutex_lock(&group->reg_mutex);
2726 ret = user_events_ioctl_del(info, uarg);
2727 mutex_unlock(&group->reg_mutex);
2728 break;
2729
2730 case DIAG_IOCSUNREG:
2731 mutex_lock(&group->reg_mutex);
2732 ret = user_events_ioctl_unreg(uarg);
2733 mutex_unlock(&group->reg_mutex);
2734 break;
2735 }
2736
2737 return ret;
2738}
2739
2740/*
2741 * Handles the final close of the file from user mode.
2742 */
2743static int user_events_release(struct inode *node, struct file *file)
2744{
2745 struct user_event_file_info *info = file->private_data;
2746 struct user_event_group *group;
2747 struct user_event_refs *refs;
2748 int i;
2749
2750 if (!info)
2751 return -EINVAL;
2752
2753 group = info->group;
2754
2755 /*
2756 * Ensure refs cannot change under any situation by taking the
2757 * register mutex during the final freeing of the references.
2758 */
2759 mutex_lock(&group->reg_mutex);
2760
2761 refs = info->refs;
2762
2763 if (!refs)
2764 goto out;
2765
2766 /*
2767 * The lifetime of refs has reached an end, it's tied to this file.
2768 * The underlying user_events are ref counted, and cannot be freed.
2769 * After this decrement, the user_events may be freed elsewhere.
2770 */
2771 for (i = 0; i < refs->count; ++i)
2772 user_event_put(refs->events[i], false);
2773
2774out:
2775 file->private_data = NULL;
2776
2777 mutex_unlock(&group->reg_mutex);
2778
2779 kfree(refs);
2780 kfree(info);
2781
2782 return 0;
2783}
2784
2785static const struct file_operations user_data_fops = {
2786 .open = user_events_open,
2787 .write = user_events_write,
2788 .write_iter = user_events_write_iter,
2789 .unlocked_ioctl = user_events_ioctl,
2790 .release = user_events_release,
2791};
2792
2793static void *user_seq_start(struct seq_file *m, loff_t *pos)
2794{
2795 if (*pos)
2796 return NULL;
2797
2798 return (void *)1;
2799}
2800
2801static void *user_seq_next(struct seq_file *m, void *p, loff_t *pos)
2802{
2803 ++*pos;
2804 return NULL;
2805}
2806
2807static void user_seq_stop(struct seq_file *m, void *p)
2808{
2809}
2810
2811static int user_seq_show(struct seq_file *m, void *p)
2812{
2813 struct user_event_group *group = m->private;
2814 struct user_event *user;
2815 char status;
2816 int i, active = 0, busy = 0;
2817
2818 if (!group)
2819 return -EINVAL;
2820
2821 mutex_lock(&group->reg_mutex);
2822
2823 hash_for_each(group->register_table, i, user, node) {
2824 status = user->status;
2825
2826 seq_puts(m, EVENT_TP_NAME(user));
2827
2828 if (status != 0) {
2829 seq_puts(m, " # Used by");
2830 if (status & EVENT_STATUS_FTRACE)
2831 seq_puts(m, " ftrace");
2832 if (status & EVENT_STATUS_PERF)
2833 seq_puts(m, " perf");
2834 if (status & EVENT_STATUS_OTHER)
2835 seq_puts(m, " other");
2836 busy++;
2837 }
2838
2839 seq_putc(m, '\n');
2840 active++;
2841 }
2842
2843 mutex_unlock(&group->reg_mutex);
2844
2845 seq_putc(m, '\n');
2846 seq_printf(m, "Active: %d\n", active);
2847 seq_printf(m, "Busy: %d\n", busy);
2848
2849 return 0;
2850}
2851
2852static const struct seq_operations user_seq_ops = {
2853 .start = user_seq_start,
2854 .next = user_seq_next,
2855 .stop = user_seq_stop,
2856 .show = user_seq_show,
2857};
2858
2859static int user_status_open(struct inode *node, struct file *file)
2860{
2861 struct user_event_group *group;
2862 int ret;
2863
2864 group = current_user_event_group();
2865
2866 if (!group)
2867 return -ENOENT;
2868
2869 ret = seq_open(file, &user_seq_ops);
2870
2871 if (!ret) {
2872 /* Chain group to seq_file */
2873 struct seq_file *m = file->private_data;
2874
2875 m->private = group;
2876 }
2877
2878 return ret;
2879}
2880
2881static const struct file_operations user_status_fops = {
2882 .open = user_status_open,
2883 .read = seq_read,
2884 .llseek = seq_lseek,
2885 .release = seq_release,
2886};
2887
2888/*
2889 * Creates a set of tracefs files to allow user mode interactions.
2890 */
2891static int create_user_tracefs(void)
2892{
2893 struct dentry *edata, *emmap;
2894
2895 edata = tracefs_create_file("user_events_data", TRACE_MODE_WRITE,
2896 NULL, NULL, &user_data_fops);
2897
2898 if (!edata) {
2899 pr_warn("Could not create tracefs 'user_events_data' entry\n");
2900 goto err;
2901 }
2902
2903 emmap = tracefs_create_file("user_events_status", TRACE_MODE_READ,
2904 NULL, NULL, &user_status_fops);
2905
2906 if (!emmap) {
2907 tracefs_remove(edata);
2908 pr_warn("Could not create tracefs 'user_events_mmap' entry\n");
2909 goto err;
2910 }
2911
2912 return 0;
2913err:
2914 return -ENODEV;
2915}
2916
2917static int set_max_user_events_sysctl(const struct ctl_table *table, int write,
2918 void *buffer, size_t *lenp, loff_t *ppos)
2919{
2920 int ret;
2921
2922 mutex_lock(&event_mutex);
2923
2924 ret = proc_douintvec(table, write, buffer, lenp, ppos);
2925
2926 mutex_unlock(&event_mutex);
2927
2928 return ret;
2929}
2930
2931static const struct ctl_table user_event_sysctls[] = {
2932 {
2933 .procname = "user_events_max",
2934 .data = &max_user_events,
2935 .maxlen = sizeof(unsigned int),
2936 .mode = 0644,
2937 .proc_handler = set_max_user_events_sysctl,
2938 },
2939};
2940
2941static int __init trace_events_user_init(void)
2942{
2943 int ret;
2944
2945 fault_cache = KMEM_CACHE(user_event_enabler_fault, 0);
2946
2947 if (!fault_cache)
2948 return -ENOMEM;
2949
2950 init_group = user_event_group_create();
2951
2952 if (!init_group) {
2953 kmem_cache_destroy(fault_cache);
2954 return -ENOMEM;
2955 }
2956
2957 ret = create_user_tracefs();
2958
2959 if (ret) {
2960 pr_warn("user_events could not register with tracefs\n");
2961 user_event_group_destroy(init_group);
2962 kmem_cache_destroy(fault_cache);
2963 init_group = NULL;
2964 return ret;
2965 }
2966
2967 if (dyn_event_register(&user_event_dops))
2968 pr_warn("user_events could not register with dyn_events\n");
2969
2970 register_sysctl_init("kernel", user_event_sysctls);
2971
2972 return 0;
2973}
2974
2975fs_initcall(trace_events_user_init);