projects/linux·blob master · refs

229 lines · 8.3 KiB

1/* SPDX-License-Identifier: GPL-2.0+ */
2/*
3 * Read-Copy Update mechanism for mutual exclusion, adapted for tracing.
4 *
5 * Copyright (C) 2020 Paul E. McKenney.
6 */
7
8#ifndef __LINUX_RCUPDATE_TRACE_H
9#define __LINUX_RCUPDATE_TRACE_H
10
11#include <linux/sched.h>
12#include <linux/rcupdate.h>
13#include <linux/cleanup.h>
14
15#ifdef CONFIG_TASKS_TRACE_RCU
16extern struct srcu_struct rcu_tasks_trace_srcu_struct;
17#endif // #ifdef CONFIG_TASKS_TRACE_RCU
18
19#if defined(CONFIG_DEBUG_LOCK_ALLOC) && defined(CONFIG_TASKS_TRACE_RCU)
20
21static inline int rcu_read_lock_trace_held(void)
22{
23 return srcu_read_lock_held(&rcu_tasks_trace_srcu_struct);
24}
25
26#else // #if defined(CONFIG_DEBUG_LOCK_ALLOC) && defined(CONFIG_TASKS_TRACE_RCU)
27
28static inline int rcu_read_lock_trace_held(void)
29{
30 return 1;
31}
32
33#endif // #else // #if defined(CONFIG_DEBUG_LOCK_ALLOC) && defined(CONFIG_TASKS_TRACE_RCU)
34
35#ifdef CONFIG_TASKS_TRACE_RCU
36
37/**
38 * rcu_read_lock_tasks_trace - mark beginning of RCU-trace read-side critical section
39 *
40 * When synchronize_rcu_tasks_trace() is invoked by one task, then that
41 * task is guaranteed to block until all other tasks exit their read-side
42 * critical sections. Similarly, if call_rcu_trace() is invoked on one
43 * task while other tasks are within RCU read-side critical sections,
44 * invocation of the corresponding RCU callback is deferred until after
45 * the all the other tasks exit their critical sections.
46 *
47 * For more details, please see the documentation for
48 * srcu_read_lock_fast(). For a description of how implicit RCU
49 * readers provide the needed ordering for architectures defining the
50 * ARCH_WANTS_NO_INSTR Kconfig option (and thus promising never to trace
51 * code where RCU is not watching), please see the __srcu_read_lock_fast()
52 * (non-kerneldoc) header comment. Otherwise, the smp_mb() below provided
53 * the needed ordering.
54 */
55static inline struct srcu_ctr __percpu *rcu_read_lock_tasks_trace(void)
56{
57 struct srcu_ctr __percpu *ret = __srcu_read_lock_fast(&rcu_tasks_trace_srcu_struct);
58
59 rcu_try_lock_acquire(&rcu_tasks_trace_srcu_struct.dep_map);
60 if (!IS_ENABLED(CONFIG_TASKS_TRACE_RCU_NO_MB))
61 smp_mb(); // Provide ordering on noinstr-incomplete architectures.
62 return ret;
63}
64
65/**
66 * rcu_read_unlock_tasks_trace - mark end of RCU-trace read-side critical section
67 * @scp: return value from corresponding rcu_read_lock_tasks_trace().
68 *
69 * Pairs with the preceding call to rcu_read_lock_tasks_trace() that
70 * returned the value passed in via scp.
71 *
72 * For more details, please see the documentation for rcu_read_unlock().
73 * For memory-ordering information, please see the header comment for the
74 * rcu_read_lock_tasks_trace() function.
75 */
76static inline void rcu_read_unlock_tasks_trace(struct srcu_ctr __percpu *scp)
77{
78 if (!IS_ENABLED(CONFIG_TASKS_TRACE_RCU_NO_MB))
79 smp_mb(); // Provide ordering on noinstr-incomplete architectures.
80 __srcu_read_unlock_fast(&rcu_tasks_trace_srcu_struct, scp);
81 srcu_lock_release(&rcu_tasks_trace_srcu_struct.dep_map);
82}
83
84/**
85 * rcu_read_lock_trace - mark beginning of RCU-trace read-side critical section
86 *
87 * When synchronize_rcu_tasks_trace() is invoked by one task, then that
88 * task is guaranteed to block until all other tasks exit their read-side
89 * critical sections. Similarly, if call_rcu_trace() is invoked on one
90 * task while other tasks are within RCU read-side critical sections,
91 * invocation of the corresponding RCU callback is deferred until after
92 * the all the other tasks exit their critical sections.
93 *
94 * For more details, please see the documentation for rcu_read_lock().
95 */
96static inline void rcu_read_lock_trace(void)
97{
98 int n;
99 struct task_struct *t = current;
100
101 rcu_try_lock_acquire(&rcu_tasks_trace_srcu_struct.dep_map);
102 n = READ_ONCE(t->trc_reader_nesting);
103 WRITE_ONCE(t->trc_reader_nesting, n + 1);
104 if (n) {
105 // In case we interrupted a Tasks Trace RCU reader.
106 return;
107 }
108 barrier(); // nesting before scp to protect against interrupt handler.
109 t->trc_reader_scp = __srcu_read_lock_fast(&rcu_tasks_trace_srcu_struct);
110 if (!IS_ENABLED(CONFIG_TASKS_TRACE_RCU_NO_MB))
111 smp_mb(); // Placeholder for more selective ordering
112}
113
114/**
115 * rcu_read_unlock_trace - mark end of RCU-trace read-side critical section
116 *
117 * Pairs with a preceding call to rcu_read_lock_trace(), and nesting is
118 * allowed. Invoking a rcu_read_unlock_trace() when there is no matching
119 * rcu_read_lock_trace() is verboten, and will result in lockdep complaints.
120 *
121 * For more details, please see the documentation for rcu_read_unlock().
122 */
123static inline void rcu_read_unlock_trace(void)
124{
125 int n;
126 struct srcu_ctr __percpu *scp;
127 struct task_struct *t = current;
128
129 n = READ_ONCE(t->trc_reader_nesting) - 1;
130 if (n) {
131 WRITE_ONCE(t->trc_reader_nesting, n);
132 } else {
133 scp = t->trc_reader_scp; // Compiler cannot hoist load due to data raciness.
134 barrier(); // scp before nesting to protect against interrupt handler.
135 WRITE_ONCE(t->trc_reader_nesting, n);
136 if (!IS_ENABLED(CONFIG_TASKS_TRACE_RCU_NO_MB))
137 smp_mb(); // Placeholder for more selective ordering
138 __srcu_read_unlock_fast(&rcu_tasks_trace_srcu_struct, scp);
139 }
140 srcu_lock_release(&rcu_tasks_trace_srcu_struct.dep_map);
141}
142
143/**
144 * call_rcu_tasks_trace() - Queue a callback trace task-based grace period
145 * @rhp: structure to be used for queueing the RCU updates.
146 * @func: actual callback function to be invoked after the grace period
147 *
148 * The callback function will be invoked some time after a trace rcu-tasks
149 * grace period elapses, in other words after all currently executing
150 * trace rcu-tasks read-side critical sections have completed. These
151 * read-side critical sections are delimited by calls to rcu_read_lock_trace()
152 * and rcu_read_unlock_trace().
153 *
154 * See the description of call_rcu() for more detailed information on
155 * memory ordering guarantees.
156 */
157static inline void call_rcu_tasks_trace(struct rcu_head *rhp, rcu_callback_t func)
158{
159 call_srcu(&rcu_tasks_trace_srcu_struct, rhp, func);
160}
161
162/**
163 * synchronize_rcu_tasks_trace - wait for a trace rcu-tasks grace period
164 *
165 * Control will return to the caller some time after a trace rcu-tasks
166 * grace period has elapsed, in other words after all currently executing
167 * trace rcu-tasks read-side critical sections have elapsed. These read-side
168 * critical sections are delimited by calls to rcu_read_lock_trace()
169 * and rcu_read_unlock_trace().
170 *
171 * This is a very specialized primitive, intended only for a few uses in
172 * tracing and other situations requiring manipulation of function preambles
173 * and profiling hooks. The synchronize_rcu_tasks_trace() function is not
174 * (yet) intended for heavy use from multiple CPUs.
175 *
176 * See the description of synchronize_rcu() for more detailed information
177 * on memory ordering guarantees.
178 */
179static inline void synchronize_rcu_tasks_trace(void)
180{
181 synchronize_srcu(&rcu_tasks_trace_srcu_struct);
182}
183
184/**
185 * rcu_barrier_tasks_trace - Wait for in-flight call_rcu_tasks_trace() callbacks.
186 *
187 * Note that rcu_barrier_tasks_trace() is not obligated to actually wait,
188 * for example, if there are no pending callbacks.
189 */
190static inline void rcu_barrier_tasks_trace(void)
191{
192 srcu_barrier(&rcu_tasks_trace_srcu_struct);
193}
194
195/**
196 * rcu_tasks_trace_expedite_current - Expedite the current Tasks Trace RCU grace period
197 *
198 * Cause the current Tasks Trace RCU grace period to become expedited.
199 * The grace period following the current one might also be expedited.
200 * If there is no current grace period, one might be created. If the
201 * current grace period is currently sleeping, that sleep will complete
202 * before expediting will take effect.
203 */
204static inline void rcu_tasks_trace_expedite_current(void)
205{
206 srcu_expedite_current(&rcu_tasks_trace_srcu_struct);
207}
208
209unsigned long rcu_tasks_trace_batches_completed(void);
210
211// Placeholders to enable stepwise transition.
212void __init rcu_tasks_trace_suppress_unused(void);
213
214#else
215static inline unsigned long rcu_tasks_trace_batches_completed(void) { return 0; }
216/*
217 * The BPF JIT forms these addresses even when it doesn't call these
218 * functions, so provide definitions that result in runtime errors.
219 */
220static inline void call_rcu_tasks_trace(struct rcu_head *rhp, rcu_callback_t func) { BUG(); }
221static inline void rcu_read_lock_trace(void) { BUG(); }
222static inline void rcu_read_unlock_trace(void) { BUG(); }
223#endif /* #ifdef CONFIG_TASKS_TRACE_RCU */
224
225DEFINE_LOCK_GUARD_0(rcu_tasks_trace,
226 rcu_read_lock_trace(),
227 rcu_read_unlock_trace())
228
229#endif /* __LINUX_RCUPDATE_TRACE_H */