projects/linux·blob master · refs

1392 lines · 31.2 KiB

1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Copyright (C) 2025 - Google LLC
4 * Author: Vincent Donnefort <[email protected]>
5 */
6
7#include <linux/kstrtox.h>
8#include <linux/lockdep.h>
9#include <linux/mutex.h>
10#include <linux/tracefs.h>
11#include <linux/trace_remote.h>
12#include <linux/trace_seq.h>
13#include <linux/types.h>
14
15#include "trace.h"
16
17#define TRACEFS_DIR "remotes"
18#define TRACEFS_MODE_WRITE 0640
19#define TRACEFS_MODE_READ 0440
20
21enum tri_type {
22 TRI_CONSUMING,
23 TRI_NONCONSUMING,
24};
25
26struct trace_remote_iterator {
27 struct trace_remote *remote;
28 struct trace_seq seq;
29 struct delayed_work poll_work;
30 unsigned long lost_events;
31 u64 ts;
32 struct ring_buffer_iter *rb_iter;
33 struct ring_buffer_iter **rb_iters;
34 struct remote_event_hdr *evt;
35 int cpu;
36 int evt_cpu;
37 loff_t pos;
38 enum tri_type type;
39};
40
41struct trace_remote {
42 struct trace_remote_callbacks *cbs;
43 void *priv;
44 struct trace_buffer *trace_buffer;
45 struct trace_buffer_desc *trace_buffer_desc;
46 struct dentry *dentry;
47 struct eventfs_inode *eventfs;
48 struct remote_event *events;
49 unsigned long nr_events;
50 unsigned long trace_buffer_size;
51 struct ring_buffer_remote rb_remote;
52 struct mutex lock;
53 struct rw_semaphore reader_lock;
54 struct rw_semaphore *pcpu_reader_locks;
55 unsigned int nr_readers;
56 unsigned int poll_ms;
57 bool tracing_on;
58};
59
60static bool trace_remote_loaded(struct trace_remote *remote)
61{
62 return !!remote->trace_buffer;
63}
64
65static int trace_remote_load(struct trace_remote *remote)
66{
67 struct ring_buffer_remote *rb_remote = &remote->rb_remote;
68 struct trace_buffer_desc *desc;
69
70 lockdep_assert_held(&remote->lock);
71
72 if (trace_remote_loaded(remote))
73 return 0;
74
75 desc = remote->cbs->load_trace_buffer(remote->trace_buffer_size, remote->priv);
76 if (IS_ERR(desc))
77 return PTR_ERR(desc);
78
79 rb_remote->desc = desc;
80 rb_remote->swap_reader_page = remote->cbs->swap_reader_page;
81 rb_remote->priv = remote->priv;
82 rb_remote->reset = remote->cbs->reset;
83 remote->trace_buffer = ring_buffer_alloc_remote(rb_remote);
84 if (!remote->trace_buffer) {
85 remote->cbs->unload_trace_buffer(desc, remote->priv);
86 return -ENOMEM;
87 }
88
89 remote->trace_buffer_desc = desc;
90
91 return 0;
92}
93
94static void trace_remote_try_unload(struct trace_remote *remote)
95{
96 lockdep_assert_held(&remote->lock);
97
98 if (!trace_remote_loaded(remote))
99 return;
100
101 /* The buffer is being read or writable */
102 if (remote->nr_readers || remote->tracing_on)
103 return;
104
105 /* The buffer has readable data */
106 if (!ring_buffer_empty(remote->trace_buffer))
107 return;
108
109 ring_buffer_free(remote->trace_buffer);
110 remote->trace_buffer = NULL;
111 remote->cbs->unload_trace_buffer(remote->trace_buffer_desc, remote->priv);
112}
113
114static int trace_remote_enable_tracing(struct trace_remote *remote)
115{
116 int ret;
117
118 lockdep_assert_held(&remote->lock);
119
120 if (remote->tracing_on)
121 return 0;
122
123 ret = trace_remote_load(remote);
124 if (ret)
125 return ret;
126
127 ret = remote->cbs->enable_tracing(true, remote->priv);
128 if (ret) {
129 trace_remote_try_unload(remote);
130 return ret;
131 }
132
133 remote->tracing_on = true;
134
135 return 0;
136}
137
138static int trace_remote_disable_tracing(struct trace_remote *remote)
139{
140 int ret;
141
142 lockdep_assert_held(&remote->lock);
143
144 if (!remote->tracing_on)
145 return 0;
146
147 ret = remote->cbs->enable_tracing(false, remote->priv);
148 if (ret)
149 return ret;
150
151 ring_buffer_poll_remote(remote->trace_buffer, RING_BUFFER_ALL_CPUS);
152 remote->tracing_on = false;
153 trace_remote_try_unload(remote);
154
155 return 0;
156}
157
158static void trace_remote_reset(struct trace_remote *remote, int cpu)
159{
160 lockdep_assert_held(&remote->lock);
161
162 if (!trace_remote_loaded(remote))
163 return;
164
165 if (cpu == RING_BUFFER_ALL_CPUS)
166 ring_buffer_reset(remote->trace_buffer);
167 else
168 ring_buffer_reset_cpu(remote->trace_buffer, cpu);
169
170 trace_remote_try_unload(remote);
171}
172
173static ssize_t
174tracing_on_write(struct file *filp, const char __user *ubuf, size_t cnt, loff_t *ppos)
175{
176 struct seq_file *seq = filp->private_data;
177 struct trace_remote *remote = seq->private;
178 unsigned long val;
179 int ret;
180
181 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
182 if (ret)
183 return ret;
184
185 guard(mutex)(&remote->lock);
186
187 ret = val ? trace_remote_enable_tracing(remote) : trace_remote_disable_tracing(remote);
188 if (ret)
189 return ret;
190
191 return cnt;
192}
193static int tracing_on_show(struct seq_file *s, void *unused)
194{
195 struct trace_remote *remote = s->private;
196
197 seq_printf(s, "%d\n", remote->tracing_on);
198
199 return 0;
200}
201DEFINE_SHOW_STORE_ATTRIBUTE(tracing_on);
202
203static ssize_t buffer_size_kb_write(struct file *filp, const char __user *ubuf, size_t cnt,
204 loff_t *ppos)
205{
206 struct seq_file *seq = filp->private_data;
207 struct trace_remote *remote = seq->private;
208 unsigned long val;
209 int ret;
210
211 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
212 if (ret)
213 return ret;
214
215 /* KiB to Bytes */
216 if (!val || check_shl_overflow(val, 10, &val))
217 return -EINVAL;
218
219 guard(mutex)(&remote->lock);
220
221 if (trace_remote_loaded(remote))
222 return -EBUSY;
223
224 remote->trace_buffer_size = val;
225
226 return cnt;
227}
228
229static int buffer_size_kb_show(struct seq_file *s, void *unused)
230{
231 struct trace_remote *remote = s->private;
232
233 seq_printf(s, "%lu (%s)\n", remote->trace_buffer_size >> 10,
234 trace_remote_loaded(remote) ? "loaded" : "unloaded");
235
236 return 0;
237}
238DEFINE_SHOW_STORE_ATTRIBUTE(buffer_size_kb);
239
240static int trace_remote_get(struct trace_remote *remote, int cpu)
241{
242 int ret;
243
244 if (remote->nr_readers == UINT_MAX)
245 return -EBUSY;
246
247 ret = trace_remote_load(remote);
248 if (ret)
249 return ret;
250
251 if (cpu != RING_BUFFER_ALL_CPUS && !remote->pcpu_reader_locks) {
252 int lock_cpu;
253
254 remote->pcpu_reader_locks = kzalloc_objs(*remote->pcpu_reader_locks,
255 nr_cpu_ids);
256 if (!remote->pcpu_reader_locks) {
257 trace_remote_try_unload(remote);
258 return -ENOMEM;
259 }
260
261 for_each_possible_cpu(lock_cpu)
262 init_rwsem(&remote->pcpu_reader_locks[lock_cpu]);
263 }
264
265 remote->nr_readers++;
266
267 return 0;
268}
269
270static void trace_remote_put(struct trace_remote *remote)
271{
272 if (WARN_ON(!remote->nr_readers))
273 return;
274
275 remote->nr_readers--;
276 if (remote->nr_readers)
277 return;
278
279 kfree(remote->pcpu_reader_locks);
280 remote->pcpu_reader_locks = NULL;
281
282 trace_remote_try_unload(remote);
283}
284
285static bool trace_remote_has_cpu(struct trace_remote *remote, int cpu)
286{
287 if (cpu == RING_BUFFER_ALL_CPUS)
288 return true;
289
290 return ring_buffer_poll_remote(remote->trace_buffer, cpu) == 0;
291}
292
293static void __poll_remote(struct work_struct *work)
294{
295 struct delayed_work *dwork = to_delayed_work(work);
296 struct trace_remote_iterator *iter;
297
298 iter = container_of(dwork, struct trace_remote_iterator, poll_work);
299 ring_buffer_poll_remote(iter->remote->trace_buffer, iter->cpu);
300 schedule_delayed_work((struct delayed_work *)work,
301 msecs_to_jiffies(iter->remote->poll_ms));
302}
303
304static void __free_ring_buffer_iter(struct trace_remote_iterator *iter, int cpu)
305{
306 if (cpu != RING_BUFFER_ALL_CPUS) {
307 ring_buffer_read_finish(iter->rb_iter);
308 return;
309 }
310
311 for_each_possible_cpu(cpu) {
312 if (iter->rb_iters[cpu])
313 ring_buffer_read_finish(iter->rb_iters[cpu]);
314 }
315
316 kfree(iter->rb_iters);
317}
318
319static int __alloc_ring_buffer_iter(struct trace_remote_iterator *iter, int cpu)
320{
321 if (cpu != RING_BUFFER_ALL_CPUS) {
322 iter->rb_iter = ring_buffer_read_start(iter->remote->trace_buffer, cpu, GFP_KERNEL);
323
324 return iter->rb_iter ? 0 : -ENOMEM;
325 }
326
327 iter->rb_iters = kzalloc_objs(*iter->rb_iters, nr_cpu_ids);
328 if (!iter->rb_iters)
329 return -ENOMEM;
330
331 for_each_possible_cpu(cpu) {
332 iter->rb_iters[cpu] = ring_buffer_read_start(iter->remote->trace_buffer, cpu,
333 GFP_KERNEL);
334 if (!iter->rb_iters[cpu]) {
335 /* This CPU isn't part of trace_buffer. Skip it */
336 if (!trace_remote_has_cpu(iter->remote, cpu))
337 continue;
338
339 __free_ring_buffer_iter(iter, RING_BUFFER_ALL_CPUS);
340 return -ENOMEM;
341 }
342 }
343
344 return 0;
345}
346
347static struct trace_remote_iterator
348*trace_remote_iter(struct trace_remote *remote, int cpu, enum tri_type type)
349{
350 struct trace_remote_iterator *iter = NULL;
351 int ret;
352
353 lockdep_assert_held(&remote->lock);
354
355 if (type == TRI_NONCONSUMING && !trace_remote_loaded(remote))
356 return NULL;
357
358 ret = trace_remote_get(remote, cpu);
359 if (ret)
360 return ERR_PTR(ret);
361
362 if (!trace_remote_has_cpu(remote, cpu)) {
363 ret = -ENODEV;
364 goto err;
365 }
366
367 iter = kzalloc_obj(*iter);
368 if (iter) {
369 iter->remote = remote;
370 iter->cpu = cpu;
371 iter->type = type;
372 trace_seq_init(&iter->seq);
373
374 switch (type) {
375 case TRI_CONSUMING:
376 ring_buffer_poll_remote(remote->trace_buffer, cpu);
377 INIT_DELAYED_WORK(&iter->poll_work, __poll_remote);
378 schedule_delayed_work(&iter->poll_work, msecs_to_jiffies(remote->poll_ms));
379 break;
380 case TRI_NONCONSUMING:
381 ret = __alloc_ring_buffer_iter(iter, cpu);
382 break;
383 }
384
385 if (ret)
386 goto err;
387
388 return iter;
389 }
390 ret = -ENOMEM;
391
392err:
393 kfree(iter);
394 trace_remote_put(remote);
395
396 return ERR_PTR(ret);
397}
398
399static void trace_remote_iter_free(struct trace_remote_iterator *iter)
400{
401 struct trace_remote *remote;
402
403 if (!iter)
404 return;
405
406 remote = iter->remote;
407
408 lockdep_assert_held(&remote->lock);
409
410 switch (iter->type) {
411 case TRI_CONSUMING:
412 cancel_delayed_work_sync(&iter->poll_work);
413 break;
414 case TRI_NONCONSUMING:
415 __free_ring_buffer_iter(iter, iter->cpu);
416 break;
417 }
418
419 kfree(iter);
420 trace_remote_put(remote);
421}
422
423static void trace_remote_iter_read_start(struct trace_remote_iterator *iter)
424{
425 struct trace_remote *remote = iter->remote;
426 int cpu = iter->cpu;
427
428 /* Acquire global reader lock */
429 if (cpu == RING_BUFFER_ALL_CPUS && iter->type == TRI_CONSUMING)
430 down_write(&remote->reader_lock);
431 else
432 down_read(&remote->reader_lock);
433
434 if (cpu == RING_BUFFER_ALL_CPUS)
435 return;
436
437 /*
438 * No need for the remote lock here, iter holds a reference on
439 * remote->nr_readers
440 */
441
442 /* Get the per-CPU one */
443 if (WARN_ON_ONCE(!remote->pcpu_reader_locks))
444 return;
445
446 if (iter->type == TRI_CONSUMING)
447 down_write(&remote->pcpu_reader_locks[cpu]);
448 else
449 down_read(&remote->pcpu_reader_locks[cpu]);
450}
451
452static void trace_remote_iter_read_finished(struct trace_remote_iterator *iter)
453{
454 struct trace_remote *remote = iter->remote;
455 int cpu = iter->cpu;
456
457 /* Release per-CPU reader lock */
458 if (cpu != RING_BUFFER_ALL_CPUS) {
459 /*
460 * No need for the remote lock here, iter holds a reference on
461 * remote->nr_readers
462 */
463 if (iter->type == TRI_CONSUMING)
464 up_write(&remote->pcpu_reader_locks[cpu]);
465 else
466 up_read(&remote->pcpu_reader_locks[cpu]);
467 }
468
469 /* Release global reader lock */
470 if (cpu == RING_BUFFER_ALL_CPUS && iter->type == TRI_CONSUMING)
471 up_write(&remote->reader_lock);
472 else
473 up_read(&remote->reader_lock);
474}
475
476static struct ring_buffer_iter *__get_rb_iter(struct trace_remote_iterator *iter, int cpu)
477{
478 return iter->cpu != RING_BUFFER_ALL_CPUS ? iter->rb_iter : iter->rb_iters[cpu];
479}
480
481static struct ring_buffer_event *
482__peek_event(struct trace_remote_iterator *iter, int cpu, u64 *ts, unsigned long *lost_events)
483{
484 struct ring_buffer_event *rb_evt;
485 struct ring_buffer_iter *rb_iter;
486
487 switch (iter->type) {
488 case TRI_CONSUMING:
489 return ring_buffer_peek(iter->remote->trace_buffer, cpu, ts, lost_events);
490 case TRI_NONCONSUMING:
491 rb_iter = __get_rb_iter(iter, cpu);
492 if (!rb_iter)
493 return NULL;
494
495 rb_evt = ring_buffer_iter_peek(rb_iter, ts);
496 if (!rb_evt)
497 return NULL;
498
499 *lost_events = ring_buffer_iter_dropped(rb_iter);
500
501 return rb_evt;
502 }
503
504 return NULL;
505}
506
507static bool trace_remote_iter_read_event(struct trace_remote_iterator *iter)
508{
509 struct trace_buffer *trace_buffer = iter->remote->trace_buffer;
510 struct ring_buffer_event *rb_evt;
511 int cpu = iter->cpu;
512
513 if (cpu != RING_BUFFER_ALL_CPUS) {
514 if (ring_buffer_empty_cpu(trace_buffer, cpu))
515 return false;
516
517 rb_evt = __peek_event(iter, cpu, &iter->ts, &iter->lost_events);
518 if (!rb_evt)
519 return false;
520
521 iter->evt_cpu = cpu;
522 iter->evt = ring_buffer_event_data(rb_evt);
523 return true;
524 }
525
526 iter->ts = U64_MAX;
527 for_each_possible_cpu(cpu) {
528 unsigned long lost_events;
529 u64 ts;
530
531 if (ring_buffer_empty_cpu(trace_buffer, cpu))
532 continue;
533
534 rb_evt = __peek_event(iter, cpu, &ts, &lost_events);
535 if (!rb_evt)
536 continue;
537
538 if (ts >= iter->ts)
539 continue;
540
541 iter->ts = ts;
542 iter->evt_cpu = cpu;
543 iter->evt = ring_buffer_event_data(rb_evt);
544 iter->lost_events = lost_events;
545 }
546
547 return iter->ts != U64_MAX;
548}
549
550static void trace_remote_iter_move(struct trace_remote_iterator *iter)
551{
552 struct trace_buffer *trace_buffer = iter->remote->trace_buffer;
553
554 switch (iter->type) {
555 case TRI_CONSUMING:
556 ring_buffer_consume(trace_buffer, iter->evt_cpu, NULL, NULL);
557 break;
558 case TRI_NONCONSUMING:
559 ring_buffer_iter_advance(__get_rb_iter(iter, iter->evt_cpu));
560 break;
561 }
562}
563
564static struct remote_event *trace_remote_find_event(struct trace_remote *remote, unsigned short id);
565
566static int trace_remote_iter_print_event(struct trace_remote_iterator *iter)
567{
568 struct remote_event *evt;
569 unsigned long usecs_rem;
570 u64 ts = iter->ts;
571
572 if (iter->lost_events)
573 trace_seq_printf(&iter->seq, "CPU:%d [LOST %lu EVENTS]\n",
574 iter->evt_cpu, iter->lost_events);
575
576 do_div(ts, 1000);
577 usecs_rem = do_div(ts, USEC_PER_SEC);
578
579 trace_seq_printf(&iter->seq, "[%03d]\t%5llu.%06lu: ", iter->evt_cpu,
580 ts, usecs_rem);
581
582 evt = trace_remote_find_event(iter->remote, iter->evt->id);
583 if (!evt)
584 trace_seq_printf(&iter->seq, "UNKNOWN id=%d\n", iter->evt->id);
585 else
586 evt->print(iter->evt, &iter->seq);
587
588 return trace_seq_has_overflowed(&iter->seq) ? -EOVERFLOW : 0;
589}
590
591static int trace_pipe_open(struct inode *inode, struct file *filp)
592{
593 struct trace_remote *remote = inode->i_private;
594 struct trace_remote_iterator *iter;
595 int cpu = tracing_get_cpu(inode);
596
597 guard(mutex)(&remote->lock);
598
599 iter = trace_remote_iter(remote, cpu, TRI_CONSUMING);
600 if (IS_ERR(iter))
601 return PTR_ERR(iter);
602
603 filp->private_data = iter;
604
605 return 0;
606}
607
608static int trace_pipe_release(struct inode *inode, struct file *filp)
609{
610 struct trace_remote_iterator *iter = filp->private_data;
611 struct trace_remote *remote = iter->remote;
612
613 guard(mutex)(&remote->lock);
614
615 trace_remote_iter_free(iter);
616
617 return 0;
618}
619
620static ssize_t trace_pipe_read(struct file *filp, char __user *ubuf, size_t cnt, loff_t *ppos)
621{
622 struct trace_remote_iterator *iter = filp->private_data;
623 struct trace_buffer *trace_buffer = iter->remote->trace_buffer;
624 int ret;
625
626copy_to_user:
627 ret = trace_seq_to_user(&iter->seq, ubuf, cnt);
628 if (ret != -EBUSY)
629 return ret;
630
631 trace_seq_init(&iter->seq);
632
633 ret = ring_buffer_wait(trace_buffer, iter->cpu, 0, NULL, NULL);
634 if (ret < 0)
635 return ret;
636
637 trace_remote_iter_read_start(iter);
638
639 while (trace_remote_iter_read_event(iter)) {
640 int prev_len = iter->seq.seq.len;
641
642 if (trace_remote_iter_print_event(iter)) {
643 iter->seq.seq.len = prev_len;
644 break;
645 }
646
647 trace_remote_iter_move(iter);
648 }
649
650 trace_remote_iter_read_finished(iter);
651
652 goto copy_to_user;
653}
654
655static const struct file_operations trace_pipe_fops = {
656 .open = trace_pipe_open,
657 .read = trace_pipe_read,
658 .release = trace_pipe_release,
659};
660
661static void *trace_next(struct seq_file *m, void *v, loff_t *pos)
662{
663 struct trace_remote_iterator *iter = m->private;
664
665 ++*pos;
666
667 if (!iter || !trace_remote_iter_read_event(iter))
668 return NULL;
669
670 trace_remote_iter_move(iter);
671 iter->pos++;
672
673 return iter;
674}
675
676static void *trace_start(struct seq_file *m, loff_t *pos)
677{
678 struct trace_remote_iterator *iter = m->private;
679 loff_t i;
680
681 if (!iter)
682 return NULL;
683
684 trace_remote_iter_read_start(iter);
685
686 if (!*pos) {
687 iter->pos = -1;
688 return trace_next(m, NULL, &i);
689 }
690
691 i = iter->pos;
692 while (i < *pos) {
693 iter = trace_next(m, NULL, &i);
694 if (!iter)
695 return NULL;
696 }
697
698 return iter;
699}
700
701static int trace_show(struct seq_file *m, void *v)
702{
703 struct trace_remote_iterator *iter = v;
704
705 trace_seq_init(&iter->seq);
706
707 if (trace_remote_iter_print_event(iter)) {
708 seq_printf(m, "[EVENT %d PRINT TOO BIG]\n", iter->evt->id);
709 return 0;
710 }
711
712 return trace_print_seq(m, &iter->seq);
713}
714
715static void trace_stop(struct seq_file *m, void *v)
716{
717 struct trace_remote_iterator *iter = m->private;
718
719 if (iter)
720 trace_remote_iter_read_finished(iter);
721}
722
723static const struct seq_operations trace_sops = {
724 .start = trace_start,
725 .next = trace_next,
726 .show = trace_show,
727 .stop = trace_stop,
728};
729
730static int trace_open(struct inode *inode, struct file *filp)
731{
732 struct trace_remote *remote = inode->i_private;
733 struct trace_remote_iterator *iter = NULL;
734 int cpu = tracing_get_cpu(inode);
735 int ret;
736
737 if (!(filp->f_mode & FMODE_READ))
738 return 0;
739
740 guard(mutex)(&remote->lock);
741
742 iter = trace_remote_iter(remote, cpu, TRI_NONCONSUMING);
743 if (IS_ERR(iter))
744 return PTR_ERR(iter);
745
746 ret = seq_open(filp, &trace_sops);
747 if (ret) {
748 trace_remote_iter_free(iter);
749 return ret;
750 }
751
752 ((struct seq_file *)filp->private_data)->private = (void *)iter;
753
754 return 0;
755}
756
757static int trace_release(struct inode *inode, struct file *filp)
758{
759 struct trace_remote_iterator *iter;
760
761 if (!(filp->f_mode & FMODE_READ))
762 return 0;
763
764 iter = ((struct seq_file *)filp->private_data)->private;
765 seq_release(inode, filp);
766
767 if (!iter)
768 return 0;
769
770 guard(mutex)(&iter->remote->lock);
771
772 trace_remote_iter_free(iter);
773
774 return 0;
775}
776
777static ssize_t trace_write(struct file *filp, const char __user *ubuf, size_t cnt, loff_t *ppos)
778{
779 struct inode *inode = file_inode(filp);
780 struct trace_remote *remote = inode->i_private;
781 int cpu = tracing_get_cpu(inode);
782
783 guard(mutex)(&remote->lock);
784
785 trace_remote_reset(remote, cpu);
786
787 return cnt;
788}
789
790static const struct file_operations trace_fops = {
791 .open = trace_open,
792 .write = trace_write,
793 .read = seq_read,
794 .read_iter = seq_read_iter,
795 .release = trace_release,
796};
797
798static int trace_remote_init_tracefs(const char *name, struct trace_remote *remote)
799{
800 struct dentry *remote_d, *percpu_d, *d;
801 static struct dentry *root;
802 static DEFINE_MUTEX(lock);
803 bool root_inited = false;
804 int cpu;
805
806 guard(mutex)(&lock);
807
808 if (!root) {
809 root = tracefs_create_dir(TRACEFS_DIR, NULL);
810 if (!root) {
811 pr_err("Failed to create tracefs dir "TRACEFS_DIR"\n");
812 return -ENOMEM;
813 }
814 root_inited = true;
815 }
816
817 remote_d = tracefs_create_dir(name, root);
818 if (!remote_d) {
819 pr_err("Failed to create tracefs dir "TRACEFS_DIR"%s/\n", name);
820 goto err;
821 }
822
823 d = trace_create_file("tracing_on", TRACEFS_MODE_WRITE, remote_d, remote, &tracing_on_fops);
824 if (!d)
825 goto err;
826
827 d = trace_create_file("buffer_size_kb", TRACEFS_MODE_WRITE, remote_d, remote,
828 &buffer_size_kb_fops);
829 if (!d)
830 goto err;
831
832 d = trace_create_file("trace_pipe", TRACEFS_MODE_READ, remote_d, remote, &trace_pipe_fops);
833 if (!d)
834 goto err;
835
836 d = trace_create_file("trace", TRACEFS_MODE_WRITE, remote_d, remote, &trace_fops);
837 if (!d)
838 goto err;
839
840 percpu_d = tracefs_create_dir("per_cpu", remote_d);
841 if (!percpu_d) {
842 pr_err("Failed to create tracefs dir "TRACEFS_DIR"%s/per_cpu/\n", name);
843 goto err;
844 }
845
846 for_each_possible_cpu(cpu) {
847 struct dentry *cpu_d;
848 char cpu_name[16];
849
850 snprintf(cpu_name, sizeof(cpu_name), "cpu%d", cpu);
851 cpu_d = tracefs_create_dir(cpu_name, percpu_d);
852 if (!cpu_d) {
853 pr_err("Failed to create tracefs dir "TRACEFS_DIR"%s/percpu/cpu%d\n",
854 name, cpu);
855 goto err;
856 }
857
858 d = trace_create_cpu_file("trace_pipe", TRACEFS_MODE_READ, cpu_d, remote, cpu,
859 &trace_pipe_fops);
860 if (!d)
861 goto err;
862
863 d = trace_create_cpu_file("trace", TRACEFS_MODE_WRITE, cpu_d, remote, cpu,
864 &trace_fops);
865 if (!d)
866 goto err;
867 }
868
869 remote->dentry = remote_d;
870
871 return 0;
872
873err:
874 if (root_inited) {
875 tracefs_remove(root);
876 root = NULL;
877 } else {
878 tracefs_remove(remote_d);
879 }
880
881 return -ENOMEM;
882}
883
884static int trace_remote_register_events(const char *remote_name, struct trace_remote *remote,
885 struct remote_event *events, size_t nr_events);
886
887/**
888 * trace_remote_register() - Register a Tracefs remote
889 * @name: Name of the remote, used for the Tracefs remotes/ directory.
890 * @cbs: Set of callbacks used to control the remote.
891 * @priv: Private data, passed to each callback from @cbs.
892 * @events: Array of events. &remote_event.name and &remote_event.id must be
893 * filled by the caller.
894 * @nr_events: Number of events in the @events array.
895 *
896 * A trace remote is an entity, outside of the kernel (most likely firmware or
897 * hypervisor) capable of writing events into a Tracefs compatible ring-buffer.
898 * The kernel would then act as a reader.
899 *
900 * The registered remote will be found under the Tracefs directory
901 * remotes/<name>.
902 *
903 * Return: 0 on success, negative error code on failure.
904 */
905int trace_remote_register(const char *name, struct trace_remote_callbacks *cbs, void *priv,
906 struct remote_event *events, size_t nr_events)
907{
908 struct trace_remote *remote;
909 int ret;
910
911 remote = kzalloc_obj(*remote);
912 if (!remote)
913 return -ENOMEM;
914
915 remote->cbs = cbs;
916 remote->priv = priv;
917 remote->trace_buffer_size = 7 << 10;
918 remote->poll_ms = 100;
919 mutex_init(&remote->lock);
920 init_rwsem(&remote->reader_lock);
921
922 if (trace_remote_init_tracefs(name, remote)) {
923 kfree(remote);
924 return -ENOMEM;
925 }
926
927 ret = trace_remote_register_events(name, remote, events, nr_events);
928 if (ret) {
929 pr_err("Failed to register events for trace remote '%s' (%d)\n",
930 name, ret);
931 return ret;
932 }
933
934 ret = cbs->init ? cbs->init(remote->dentry, priv) : 0;
935 if (ret)
936 pr_err("Init failed for trace remote '%s' (%d)\n", name, ret);
937
938 return ret;
939}
940EXPORT_SYMBOL_GPL(trace_remote_register);
941
942/**
943 * trace_remote_free_buffer() - Free trace buffer allocated with trace_remote_alloc_buffer()
944 * @desc: Descriptor of the per-CPU ring-buffers, originally filled by
945 * trace_remote_alloc_buffer()
946 *
947 * Most likely called from &trace_remote_callbacks.unload_trace_buffer.
948 */
949void trace_remote_free_buffer(struct trace_buffer_desc *desc)
950{
951 struct ring_buffer_desc *rb_desc;
952 int cpu;
953
954 for_each_ring_buffer_desc(rb_desc, cpu, desc) {
955 unsigned int id;
956
957 free_page(rb_desc->meta_va);
958
959 for (id = 0; id < rb_desc->nr_page_va; id++)
960 free_page(rb_desc->page_va[id]);
961 }
962}
963EXPORT_SYMBOL_GPL(trace_remote_free_buffer);
964
965/**
966 * trace_remote_alloc_buffer() - Dynamically allocate a trace buffer
967 * @desc: Uninitialized trace_buffer_desc
968 * @desc_size: Size of the trace_buffer_desc. Must be at least equal to
969 * trace_buffer_desc_size()
970 * @buffer_size: Size in bytes of each per-CPU ring-buffer
971 * @cpumask: CPUs to allocate a ring-buffer for
972 *
973 * Helper to dynamically allocate a set of pages (enough to cover @buffer_size)
974 * for each CPU from @cpumask and fill @desc. Most likely called from
975 * &trace_remote_callbacks.load_trace_buffer.
976 *
977 * Return: 0 on success, negative error code on failure.
978 */
979int trace_remote_alloc_buffer(struct trace_buffer_desc *desc, size_t desc_size, size_t buffer_size,
980 const struct cpumask *cpumask)
981{
982 size_t min_desc_size = trace_buffer_desc_size(buffer_size, cpumask_weight(cpumask));
983 struct ring_buffer_desc *rb_desc;
984 int cpu, ret = -ENOMEM;
985 unsigned int nr_pages;
986
987 if (min_desc_size == SIZE_MAX)
988 return -E2BIG;
989
990 if (desc_size < min_desc_size)
991 return -EINVAL;
992
993 desc->nr_cpus = 0;
994 desc->struct_len = min_desc_size;
995
996 rb_desc = __first_ring_buffer_desc(desc);
997 nr_pages = __calc_nr_pages_ring_buffer_desc(buffer_size);
998
999 for_each_cpu(cpu, cpumask) {
1000 unsigned int id;
1001
1002 rb_desc->cpu = cpu;
1003 rb_desc->nr_page_va = 0;
1004 rb_desc->meta_va = (unsigned long)__get_free_page(GFP_KERNEL);
1005 if (!rb_desc->meta_va)
1006 goto err;
1007
1008 desc->nr_cpus++;
1009
1010 for (id = 0; id < nr_pages; id++) {
1011 rb_desc->nr_page_va++;
1012 rb_desc->page_va[id] = (unsigned long)__get_free_page(GFP_KERNEL);
1013 if (!rb_desc->page_va[id])
1014 goto err;
1015 }
1016 rb_desc = __next_ring_buffer_desc(rb_desc);
1017 }
1018
1019 return 0;
1020
1021err:
1022 trace_remote_free_buffer(desc);
1023 return ret;
1024}
1025EXPORT_SYMBOL_GPL(trace_remote_alloc_buffer);
1026
1027static int
1028trace_remote_enable_event(struct trace_remote *remote, struct remote_event *evt, bool enable)
1029{
1030 int ret;
1031
1032 lockdep_assert_held(&remote->lock);
1033
1034 if (evt->enabled == enable)
1035 return 0;
1036
1037 ret = remote->cbs->enable_event(evt->id, enable, remote->priv);
1038 if (ret)
1039 return ret;
1040
1041 evt->enabled = enable;
1042
1043 return 0;
1044}
1045
1046static int remote_event_enable_show(struct seq_file *s, void *unused)
1047{
1048 struct remote_event *evt = s->private;
1049
1050 seq_printf(s, "%d\n", evt->enabled);
1051
1052 return 0;
1053}
1054
1055static ssize_t remote_event_enable_write(struct file *filp, const char __user *ubuf,
1056 size_t count, loff_t *ppos)
1057{
1058 struct seq_file *seq = filp->private_data;
1059 struct remote_event *evt = seq->private;
1060 struct trace_remote *remote = evt->remote;
1061 u8 enable;
1062 int ret;
1063
1064 ret = kstrtou8_from_user(ubuf, count, 10, &enable);
1065 if (ret)
1066 return ret;
1067
1068 guard(mutex)(&remote->lock);
1069
1070 ret = trace_remote_enable_event(remote, evt, enable);
1071 if (ret)
1072 return ret;
1073
1074 return count;
1075}
1076DEFINE_SHOW_STORE_ATTRIBUTE(remote_event_enable);
1077
1078static int remote_event_id_show(struct seq_file *s, void *unused)
1079{
1080 struct remote_event *evt = s->private;
1081
1082 seq_printf(s, "%d\n", evt->id);
1083
1084 return 0;
1085}
1086DEFINE_SHOW_ATTRIBUTE(remote_event_id);
1087
1088static int remote_event_format_show(struct seq_file *s, void *unused)
1089{
1090 size_t offset = sizeof(struct remote_event_hdr);
1091 struct remote_event *evt = s->private;
1092 struct trace_event_fields *field;
1093
1094 seq_printf(s, "name: %s\n", evt->name);
1095 seq_printf(s, "ID: %d\n", evt->id);
1096 seq_puts(s,
1097 "format:\n\tfield:unsigned short common_type;\toffset:0;\tsize:2;\tsigned:0;\n\n");
1098
1099 field = &evt->fields[0];
1100 while (field->name) {
1101 seq_printf(s, "\tfield:%s %s;\toffset:%zu;\tsize:%u;\tsigned:%d;\n",
1102 field->type, field->name, offset, field->size,
1103 field->is_signed);
1104 offset += field->size;
1105 field++;
1106 }
1107
1108 if (field != &evt->fields[0])
1109 seq_puts(s, "\n");
1110
1111 seq_printf(s, "print fmt: %s\n", evt->print_fmt);
1112
1113 return 0;
1114}
1115DEFINE_SHOW_ATTRIBUTE(remote_event_format);
1116
1117static int remote_event_callback(const char *name, umode_t *mode, void **data,
1118 const struct file_operations **fops)
1119{
1120 if (!strcmp(name, "enable")) {
1121 *mode = TRACEFS_MODE_WRITE;
1122 *fops = &remote_event_enable_fops;
1123 return 1;
1124 }
1125
1126 if (!strcmp(name, "id")) {
1127 *mode = TRACEFS_MODE_READ;
1128 *fops = &remote_event_id_fops;
1129 return 1;
1130 }
1131
1132 if (!strcmp(name, "format")) {
1133 *mode = TRACEFS_MODE_READ;
1134 *fops = &remote_event_format_fops;
1135 return 1;
1136 }
1137
1138 return 0;
1139}
1140
1141static ssize_t remote_events_dir_enable_write(struct file *filp, const char __user *ubuf,
1142 size_t count, loff_t *ppos)
1143{
1144 struct trace_remote *remote = file_inode(filp)->i_private;
1145 int i, ret;
1146 u8 enable;
1147
1148 ret = kstrtou8_from_user(ubuf, count, 10, &enable);
1149 if (ret)
1150 return ret;
1151
1152 guard(mutex)(&remote->lock);
1153
1154 for (i = 0; i < remote->nr_events; i++) {
1155 struct remote_event *evt = &remote->events[i];
1156 int eret;
1157
1158 eret = trace_remote_enable_event(remote, evt, enable);
1159 /*
1160 * Save the first error and return that. Some events
1161 * may still have been enabled, but let the user
1162 * know that something went wrong.
1163 */
1164 if (!ret && eret)
1165 ret = eret;
1166 }
1167
1168 if (ret)
1169 return ret;
1170
1171 return count;
1172}
1173
1174static ssize_t remote_events_dir_enable_read(struct file *filp, char __user *ubuf, size_t cnt,
1175 loff_t *ppos)
1176{
1177 struct trace_remote *remote = file_inode(filp)->i_private;
1178 const char enabled_char[] = {'0', '1', 'X'};
1179 char enabled_str[] = " \n";
1180 int i, enabled = -1;
1181
1182 guard(mutex)(&remote->lock);
1183
1184 for (i = 0; i < remote->nr_events; i++) {
1185 struct remote_event *evt = &remote->events[i];
1186
1187 if (enabled == -1) {
1188 enabled = evt->enabled;
1189 } else if (enabled != evt->enabled) {
1190 enabled = 2;
1191 break;
1192 }
1193 }
1194
1195 enabled_str[0] = enabled_char[enabled == -1 ? 0 : enabled];
1196
1197 return simple_read_from_buffer(ubuf, cnt, ppos, enabled_str, 2);
1198}
1199
1200static const struct file_operations remote_events_dir_enable_fops = {
1201 .write = remote_events_dir_enable_write,
1202 .read = remote_events_dir_enable_read,
1203};
1204
1205static ssize_t
1206remote_events_dir_header_page_read(struct file *filp, char __user *ubuf, size_t cnt, loff_t *ppos)
1207{
1208 struct trace_seq *s;
1209 int ret;
1210
1211 s = kmalloc_obj(*s);
1212 if (!s)
1213 return -ENOMEM;
1214
1215 trace_seq_init(s);
1216
1217 ring_buffer_print_page_header(NULL, s);
1218 ret = simple_read_from_buffer(ubuf, cnt, ppos, s->buffer, trace_seq_used(s));
1219 kfree(s);
1220
1221 return ret;
1222}
1223
1224static const struct file_operations remote_events_dir_header_page_fops = {
1225 .read = remote_events_dir_header_page_read,
1226};
1227
1228static ssize_t
1229remote_events_dir_header_event_read(struct file *filp, char __user *ubuf, size_t cnt, loff_t *ppos)
1230{
1231 struct trace_seq *s;
1232 int ret;
1233
1234 s = kmalloc_obj(*s);
1235 if (!s)
1236 return -ENOMEM;
1237
1238 trace_seq_init(s);
1239
1240 ring_buffer_print_entry_header(s);
1241 ret = simple_read_from_buffer(ubuf, cnt, ppos, s->buffer, trace_seq_used(s));
1242 kfree(s);
1243
1244 return ret;
1245}
1246
1247static const struct file_operations remote_events_dir_header_event_fops = {
1248 .read = remote_events_dir_header_event_read,
1249};
1250
1251static int remote_events_dir_callback(const char *name, umode_t *mode, void **data,
1252 const struct file_operations **fops)
1253{
1254 if (!strcmp(name, "enable")) {
1255 *mode = TRACEFS_MODE_WRITE;
1256 *fops = &remote_events_dir_enable_fops;
1257 return 1;
1258 }
1259
1260 if (!strcmp(name, "header_page")) {
1261 *mode = TRACEFS_MODE_READ;
1262 *fops = &remote_events_dir_header_page_fops;
1263 return 1;
1264 }
1265
1266 if (!strcmp(name, "header_event")) {
1267 *mode = TRACEFS_MODE_READ;
1268 *fops = &remote_events_dir_header_event_fops;
1269 return 1;
1270 }
1271
1272 return 0;
1273}
1274
1275static int trace_remote_init_eventfs(const char *remote_name, struct trace_remote *remote,
1276 struct remote_event *evt)
1277{
1278 struct eventfs_inode *eventfs = remote->eventfs;
1279 static struct eventfs_entry dir_entries[] = {
1280 {
1281 .name = "enable",
1282 .callback = remote_events_dir_callback,
1283 }, {
1284 .name = "header_page",
1285 .callback = remote_events_dir_callback,
1286 }, {
1287 .name = "header_event",
1288 .callback = remote_events_dir_callback,
1289 }
1290 };
1291 static struct eventfs_entry entries[] = {
1292 {
1293 .name = "enable",
1294 .callback = remote_event_callback,
1295 }, {
1296 .name = "id",
1297 .callback = remote_event_callback,
1298 }, {
1299 .name = "format",
1300 .callback = remote_event_callback,
1301 }
1302 };
1303 bool eventfs_create = false;
1304
1305 if (!eventfs) {
1306 eventfs = eventfs_create_events_dir("events", remote->dentry, dir_entries,
1307 ARRAY_SIZE(dir_entries), remote);
1308 if (IS_ERR(eventfs))
1309 return PTR_ERR(eventfs);
1310
1311 /*
1312 * Create similar hierarchy as local events even if a single system is supported at
1313 * the moment
1314 */
1315 eventfs = eventfs_create_dir(remote_name, eventfs, NULL, 0, NULL);
1316 if (IS_ERR(eventfs))
1317 return PTR_ERR(eventfs);
1318
1319 remote->eventfs = eventfs;
1320 eventfs_create = true;
1321 }
1322
1323 eventfs = eventfs_create_dir(evt->name, eventfs, entries, ARRAY_SIZE(entries), evt);
1324 if (IS_ERR(eventfs)) {
1325 if (eventfs_create) {
1326 eventfs_remove_events_dir(remote->eventfs);
1327 remote->eventfs = NULL;
1328 }
1329 return PTR_ERR(eventfs);
1330 }
1331
1332 return 0;
1333}
1334
1335static int trace_remote_attach_events(struct trace_remote *remote, struct remote_event *events,
1336 size_t nr_events)
1337{
1338 int i;
1339
1340 for (i = 0; i < nr_events; i++) {
1341 struct remote_event *evt = &events[i];
1342
1343 if (evt->remote)
1344 return -EEXIST;
1345
1346 evt->remote = remote;
1347
1348 /* We need events to be sorted for efficient lookup */
1349 if (i && evt->id <= events[i - 1].id)
1350 return -EINVAL;
1351 }
1352
1353 remote->events = events;
1354 remote->nr_events = nr_events;
1355
1356 return 0;
1357}
1358
1359static int trace_remote_register_events(const char *remote_name, struct trace_remote *remote,
1360 struct remote_event *events, size_t nr_events)
1361{
1362 int i, ret;
1363
1364 ret = trace_remote_attach_events(remote, events, nr_events);
1365 if (ret)
1366 return ret;
1367
1368 for (i = 0; i < nr_events; i++) {
1369 struct remote_event *evt = &events[i];
1370
1371 ret = trace_remote_init_eventfs(remote_name, remote, evt);
1372 if (ret)
1373 pr_warn("Failed to init eventfs for event '%s' (%d)",
1374 evt->name, ret);
1375 }
1376
1377 return 0;
1378}
1379
1380static int __cmp_events(const void *key, const void *data)
1381{
1382 const struct remote_event *evt = data;
1383 int id = (int)((long)key);
1384
1385 return id - (int)evt->id;
1386}
1387
1388static struct remote_event *trace_remote_find_event(struct trace_remote *remote, unsigned short id)
1389{
1390 return bsearch((const void *)(unsigned long)id, remote->events, remote->nr_events,
1391 sizeof(*remote->events), __cmp_events);
1392}