projects/linux·blob master · refs

1772 lines · 53.3 KiB

1/* SPDX-License-Identifier: GPL-2.0+ */
2/*
3 * Read-Copy Update mechanism for mutual exclusion (tree-based version)
4 * Internal non-public definitions that provide either classic
5 * or preemptible semantics.
6 *
7 * Copyright Red Hat, 2009
8 * Copyright IBM Corporation, 2009
9 * Copyright SUSE, 2021
10 *
11 * Author: Ingo Molnar <[email protected]>
12 * Paul E. McKenney <[email protected]>
13 * Frederic Weisbecker <[email protected]>
14 */
15
16#ifdef CONFIG_RCU_NOCB_CPU
17static cpumask_var_t rcu_nocb_mask; /* CPUs to have callbacks offloaded. */
18static bool __read_mostly rcu_nocb_poll; /* Offload kthread are to poll. */
19
20static inline bool rcu_current_is_nocb_kthread(struct rcu_data *rdp)
21{
22 /* Race on early boot between thread creation and assignment */
23 if (!rdp->nocb_cb_kthread || !rdp->nocb_gp_kthread)
24 return true;
25
26 if (current == rdp->nocb_cb_kthread || current == rdp->nocb_gp_kthread)
27 if (in_task())
28 return true;
29 return false;
30}
31
32/*
33 * Offload callback processing from the boot-time-specified set of CPUs
34 * specified by rcu_nocb_mask. For the CPUs in the set, there are kthreads
35 * created that pull the callbacks from the corresponding CPU, wait for
36 * a grace period to elapse, and invoke the callbacks. These kthreads
37 * are organized into GP kthreads, which manage incoming callbacks, wait for
38 * grace periods, and awaken CB kthreads, and the CB kthreads, which only
39 * invoke callbacks. Each GP kthread invokes its own CBs. The no-CBs CPUs
40 * do a wake_up() on their GP kthread when they insert a callback into any
41 * empty list, unless the rcu_nocb_poll boot parameter has been specified,
42 * in which case each kthread actively polls its CPU. (Which isn't so great
43 * for energy efficiency, but which does reduce RCU's overhead on that CPU.)
44 *
45 * This is intended to be used in conjunction with Frederic Weisbecker's
46 * adaptive-idle work, which would seriously reduce OS jitter on CPUs
47 * running CPU-bound user-mode computations.
48 *
49 * Offloading of callbacks can also be used as an energy-efficiency
50 * measure because CPUs with no RCU callbacks queued are more aggressive
51 * about entering dyntick-idle mode.
52 */
53
54
55/*
56 * Parse the boot-time rcu_nocb_mask CPU list from the kernel parameters.
57 * If the list is invalid, a warning is emitted and all CPUs are offloaded.
58 */
59static int __init rcu_nocb_setup(char *str)
60{
61 alloc_bootmem_cpumask_var(&rcu_nocb_mask);
62 if (*str == '=') {
63 if (cpulist_parse(++str, rcu_nocb_mask)) {
64 pr_warn("rcu_nocbs= bad CPU range, all CPUs set\n");
65 cpumask_setall(rcu_nocb_mask);
66 }
67 }
68 rcu_state.nocb_is_setup = true;
69 return 1;
70}
71__setup("rcu_nocbs", rcu_nocb_setup);
72
73static int __init parse_rcu_nocb_poll(char *arg)
74{
75 rcu_nocb_poll = true;
76 return 1;
77}
78__setup("rcu_nocb_poll", parse_rcu_nocb_poll);
79
80/*
81 * Don't bother bypassing ->cblist if the call_rcu() rate is low.
82 * After all, the main point of bypassing is to avoid lock contention
83 * on ->nocb_lock, which only can happen at high call_rcu() rates.
84 */
85static int nocb_nobypass_lim_per_jiffy = 16 * 1000 / HZ;
86module_param(nocb_nobypass_lim_per_jiffy, int, 0);
87
88/*
89 * Acquire the specified rcu_data structure's ->nocb_bypass_lock. If the
90 * lock isn't immediately available, perform minimal sanity check.
91 */
92static void rcu_nocb_bypass_lock(struct rcu_data *rdp)
93 __acquires(&rdp->nocb_bypass_lock)
94{
95 lockdep_assert_irqs_disabled();
96 if (raw_spin_trylock(&rdp->nocb_bypass_lock))
97 return;
98 /*
99 * Contention expected only when local enqueue collide with
100 * remote flush from kthreads.
101 */
102 WARN_ON_ONCE(smp_processor_id() != rdp->cpu);
103 raw_spin_lock(&rdp->nocb_bypass_lock);
104}
105
106/*
107 * Conditionally acquire the specified rcu_data structure's
108 * ->nocb_bypass_lock.
109 */
110static bool rcu_nocb_bypass_trylock(struct rcu_data *rdp)
111{
112 lockdep_assert_irqs_disabled();
113 return raw_spin_trylock(&rdp->nocb_bypass_lock);
114}
115
116/*
117 * Release the specified rcu_data structure's ->nocb_bypass_lock.
118 */
119static void rcu_nocb_bypass_unlock(struct rcu_data *rdp)
120 __releases(&rdp->nocb_bypass_lock)
121{
122 lockdep_assert_irqs_disabled();
123 raw_spin_unlock(&rdp->nocb_bypass_lock);
124}
125
126/*
127 * Acquire the specified rcu_data structure's ->nocb_lock, but only
128 * if it corresponds to a no-CBs CPU.
129 */
130static void rcu_nocb_lock(struct rcu_data *rdp)
131{
132 lockdep_assert_irqs_disabled();
133 if (!rcu_rdp_is_offloaded(rdp))
134 return;
135 raw_spin_lock(&rdp->nocb_lock);
136}
137
138/*
139 * Release the specified rcu_data structure's ->nocb_lock, but only
140 * if it corresponds to a no-CBs CPU.
141 */
142static void rcu_nocb_unlock(struct rcu_data *rdp)
143{
144 if (rcu_rdp_is_offloaded(rdp)) {
145 lockdep_assert_irqs_disabled();
146 raw_spin_unlock(&rdp->nocb_lock);
147 }
148}
149
150/*
151 * Release the specified rcu_data structure's ->nocb_lock and restore
152 * interrupts, but only if it corresponds to a no-CBs CPU.
153 */
154static void rcu_nocb_unlock_irqrestore(struct rcu_data *rdp,
155 unsigned long flags)
156{
157 if (rcu_rdp_is_offloaded(rdp)) {
158 lockdep_assert_irqs_disabled();
159 raw_spin_unlock_irqrestore(&rdp->nocb_lock, flags);
160 } else {
161 local_irq_restore(flags);
162 }
163}
164
165/* Lockdep check that ->cblist may be safely accessed. */
166static void rcu_lockdep_assert_cblist_protected(struct rcu_data *rdp)
167{
168 lockdep_assert_irqs_disabled();
169 if (rcu_rdp_is_offloaded(rdp))
170 lockdep_assert_held(&rdp->nocb_lock);
171}
172
173static void rcu_nocb_cleanup_wake(struct swait_queue_head *sq)
174{
175 if (swait_active(sq))
176 swake_up_all(sq);
177}
178
179/*
180 * Wake up any no-CBs CPUs' kthreads that were waiting on the just-ended
181 * grace period.
182 */
183static void rcu_nocb_gp_cleanup(struct swait_queue_head *sq)
184{
185 /*
186 * swait_active() can be checked first because of the following
187 * ordering, which pairs the smp_mb() in rcu_gp_cleanup() against
188 * the implicit barrier in prepare_to_swait()/set_current_state()
189 * on the nocb_gp_wait() side:
190 *
191 * rcu_gp_cleanup() nocb_gp_wait()
192 * --------------- --------------
193 * WRITE_ONCE(root->gp_seq, new_gp_seq); swait_event_interruptible_exclusive(sq)
194 * smp_mb() prepare_to_swait()
195 * if swait_active(sq) list_add_tail(...)
196 * swake_up_all(sq) set_current_state()
197 * smp_mb()
198 * if (poll_state_synchronize_rcu_full())
199 * ...
200 */
201 rcu_nocb_cleanup_wake(sq);
202}
203
204static struct swait_queue_head *rcu_nocb_gp_get(struct rcu_node *rnp)
205{
206 return &rnp->nocb_gp_wq[rcu_seq_ctr(rnp->gp_seq) & 0x1];
207}
208
209static void rcu_init_one_nocb(struct rcu_node *rnp)
210{
211 init_swait_queue_head(&rnp->nocb_gp_wq[0]);
212 init_swait_queue_head(&rnp->nocb_gp_wq[1]);
213}
214
215/*
216 * Wake NOCB rcuog kthreads on a leaf node so that they can advance
217 * callbacks that were waiting for the just-completed expedited GP.
218 *
219 * The rcuog kthread waiting for a grace period sleeps on the per-leaf-node
220 * ->nocb_gp_wq[] (not on its rdp_gp's ->nocb_gp_wq, which only signals that
221 * new callbacks have shown up), so this is the queue that must be woken.
222 * Both the even and odd waitqueues are woken because the expedited sequence
223 * does not share parity with the normal ->gp_seq the waiter indexed with.
224 */
225static void rcu_nocb_exp_cleanup(struct rcu_node *rnp)
226{
227 /*
228 * swait_active() can be checked first because of the following
229 * ordering, which pairs the smp_mb() in rcu_exp_wait_wake() against
230 * the implicit barrier in prepare_to_swait()/set_current_state()
231 * on the nocb_gp_wait() side:
232 *
233 * rcu_exp_wait_wake() nocb_gp_wait()
234 * --------------- --------------
235 * rcu_seq_end(&rcu_state.expedited_sequence); swait_event_interruptible_exclusive(sq)
236 * smp_mb() prepare_to_swait()
237 * if swait_active(sq) list_add_tail(...)
238 * swake_up_all(sq) set_current_state()
239 * smp_mb()
240 * if (poll_state_synchronize_rcu_full())
241 * ...
242 */
243 rcu_nocb_cleanup_wake(&rnp->nocb_gp_wq[0]);
244 rcu_nocb_cleanup_wake(&rnp->nocb_gp_wq[1]);
245}
246
247/* Clear any pending deferred wakeup timer (nocb_gp_lock must be held). */
248static void nocb_defer_wakeup_cancel(struct rcu_data *rdp_gp)
249{
250 if (rdp_gp->nocb_defer_wakeup > RCU_NOCB_WAKE_NOT) {
251 WRITE_ONCE(rdp_gp->nocb_defer_wakeup, RCU_NOCB_WAKE_NOT);
252 timer_delete(&rdp_gp->nocb_timer);
253 }
254}
255
256static bool __wake_nocb_gp(struct rcu_data *rdp_gp,
257 struct rcu_data *rdp,
258 unsigned long flags)
259 __releases(rdp_gp->nocb_gp_lock)
260{
261 bool needwake = false;
262
263 if (!READ_ONCE(rdp_gp->nocb_gp_kthread)) {
264 raw_spin_unlock_irqrestore(&rdp_gp->nocb_gp_lock, flags);
265 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu,
266 TPS("AlreadyAwake"));
267 return false;
268 }
269
270 nocb_defer_wakeup_cancel(rdp_gp);
271
272 if (READ_ONCE(rdp_gp->nocb_gp_sleep)) {
273 WRITE_ONCE(rdp_gp->nocb_gp_sleep, false);
274 needwake = true;
275 }
276 raw_spin_unlock_irqrestore(&rdp_gp->nocb_gp_lock, flags);
277 if (needwake) {
278 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("DoWake"));
279 swake_up_one(&rdp_gp->nocb_gp_wq);
280 }
281
282 return needwake;
283}
284
285/*
286 * Kick the GP kthread for this NOCB group.
287 */
288static bool wake_nocb_gp(struct rcu_data *rdp)
289{
290 unsigned long flags;
291 struct rcu_data *rdp_gp = rdp->nocb_gp_rdp;
292
293 raw_spin_lock_irqsave(&rdp_gp->nocb_gp_lock, flags);
294 return __wake_nocb_gp(rdp_gp, rdp, flags);
295}
296
297#ifdef CONFIG_RCU_LAZY
298/*
299 * LAZY_FLUSH_JIFFIES decides the maximum amount of time that
300 * can elapse before lazy callbacks are flushed. Lazy callbacks
301 * could be flushed much earlier for a number of other reasons
302 * however, LAZY_FLUSH_JIFFIES will ensure no lazy callbacks are
303 * left unsubmitted to RCU after those many jiffies.
304 */
305#define LAZY_FLUSH_JIFFIES (10 * HZ)
306static unsigned long jiffies_lazy_flush = LAZY_FLUSH_JIFFIES;
307
308// To be called only from test code.
309void rcu_set_jiffies_lazy_flush(unsigned long jif)
310{
311 jiffies_lazy_flush = jif;
312}
313EXPORT_SYMBOL(rcu_set_jiffies_lazy_flush);
314
315unsigned long rcu_get_jiffies_lazy_flush(void)
316{
317 return jiffies_lazy_flush;
318}
319EXPORT_SYMBOL(rcu_get_jiffies_lazy_flush);
320#endif
321
322/*
323 * Arrange to wake the GP kthread for this NOCB group at some future
324 * time when it is safe to do so.
325 */
326static void wake_nocb_gp_defer(struct rcu_data *rdp, int waketype,
327 const char *reason)
328{
329 unsigned long flags;
330 struct rcu_data *rdp_gp = rdp->nocb_gp_rdp;
331
332 raw_spin_lock_irqsave(&rdp_gp->nocb_gp_lock, flags);
333
334 /*
335 * Bypass wakeup overrides previous deferments. In case of
336 * callback storms, no need to wake up too early.
337 */
338 if (waketype == RCU_NOCB_WAKE_LAZY &&
339 rdp_gp->nocb_defer_wakeup == RCU_NOCB_WAKE_NOT) {
340 mod_timer(&rdp_gp->nocb_timer, jiffies + rcu_get_jiffies_lazy_flush());
341 WRITE_ONCE(rdp_gp->nocb_defer_wakeup, waketype);
342 } else if (waketype == RCU_NOCB_WAKE_BYPASS) {
343 mod_timer(&rdp_gp->nocb_timer, jiffies + 2);
344 WRITE_ONCE(rdp_gp->nocb_defer_wakeup, waketype);
345 } else {
346 if (rdp_gp->nocb_defer_wakeup < RCU_NOCB_WAKE)
347 mod_timer(&rdp_gp->nocb_timer, jiffies + 1);
348 if (rdp_gp->nocb_defer_wakeup < waketype)
349 WRITE_ONCE(rdp_gp->nocb_defer_wakeup, waketype);
350 }
351
352 raw_spin_unlock_irqrestore(&rdp_gp->nocb_gp_lock, flags);
353
354 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, reason);
355}
356
357/*
358 * Flush the ->nocb_bypass queue into ->cblist, enqueuing rhp if non-NULL.
359 * However, if there is a callback to be enqueued and if ->nocb_bypass
360 * proves to be initially empty, just return false because the no-CB GP
361 * kthread may need to be awakened in this case.
362 *
363 * Return true if there was something to be flushed and it succeeded, otherwise
364 * false.
365 *
366 * Note that this function always returns true if rhp is NULL.
367 */
368static bool rcu_nocb_do_flush_bypass(struct rcu_data *rdp, struct rcu_head *rhp_in,
369 unsigned long j, bool lazy)
370{
371 struct rcu_cblist rcl;
372 struct rcu_head *rhp = rhp_in;
373
374 WARN_ON_ONCE(!rcu_rdp_is_offloaded(rdp));
375 rcu_lockdep_assert_cblist_protected(rdp);
376 lockdep_assert_held(&rdp->nocb_bypass_lock);
377 if (rhp && !rcu_cblist_n_cbs(&rdp->nocb_bypass)) {
378 raw_spin_unlock(&rdp->nocb_bypass_lock);
379 return false;
380 }
381 /* Note: ->cblist.len already accounts for ->nocb_bypass contents. */
382 if (rhp)
383 rcu_segcblist_inc_len(&rdp->cblist); /* Must precede enqueue. */
384
385 /*
386 * If the new CB requested was a lazy one, queue it onto the main
387 * ->cblist so that we can take advantage of the grace-period that will
388 * happen regardless. But queue it onto the bypass list first so that
389 * the lazy CB is ordered with the existing CBs in the bypass list.
390 */
391 if (lazy && rhp) {
392 rcu_cblist_enqueue(&rdp->nocb_bypass, rhp);
393 rhp = NULL;
394 }
395 rcu_cblist_flush_enqueue(&rcl, &rdp->nocb_bypass, rhp);
396 WRITE_ONCE(rdp->lazy_len, 0);
397
398 rcu_segcblist_insert_pend_cbs(&rdp->cblist, &rcl);
399 WRITE_ONCE(rdp->nocb_bypass_first, j);
400 rcu_nocb_bypass_unlock(rdp);
401 return true;
402}
403
404/*
405 * Flush the ->nocb_bypass queue into ->cblist, enqueuing rhp if non-NULL.
406 * However, if there is a callback to be enqueued and if ->nocb_bypass
407 * proves to be initially empty, just return false because the no-CB GP
408 * kthread may need to be awakened in this case.
409 *
410 * Note that this function always returns true if rhp is NULL.
411 */
412static bool rcu_nocb_flush_bypass(struct rcu_data *rdp, struct rcu_head *rhp,
413 unsigned long j, bool lazy)
414{
415 if (!rcu_rdp_is_offloaded(rdp))
416 return true;
417 rcu_lockdep_assert_cblist_protected(rdp);
418 rcu_nocb_bypass_lock(rdp);
419 return rcu_nocb_do_flush_bypass(rdp, rhp, j, lazy);
420}
421
422/*
423 * If the ->nocb_bypass_lock is immediately available, flush the
424 * ->nocb_bypass queue into ->cblist.
425 */
426static void rcu_nocb_try_flush_bypass(struct rcu_data *rdp, unsigned long j)
427{
428 rcu_lockdep_assert_cblist_protected(rdp);
429 if (!rcu_rdp_is_offloaded(rdp) ||
430 !rcu_nocb_bypass_trylock(rdp))
431 return;
432 WARN_ON_ONCE(!rcu_nocb_do_flush_bypass(rdp, NULL, j, false));
433}
434
435/*
436 * Determine if the bypass queue needs to be flushed based on time and size.
437 * For lazy-only bypass queues, use the lazy flush timeout; otherwise flush
438 * based on jiffy advancement. The flush_faster controls flush aggressiveness.
439 */
440static bool nocb_bypass_needs_flush(struct rcu_data *rdp, long bypass_ncbs,
441 long lazy_ncbs, unsigned long j,
442 bool flush_faster)
443{
444 bool bypass_is_lazy;
445 unsigned long bypass_first;
446 unsigned long flush_timeout;
447 long qhimark_thresh;
448
449 if (!bypass_ncbs)
450 return false;
451
452 qhimark_thresh = flush_faster ? qhimark : 2 * qhimark;
453 if (bypass_ncbs >= qhimark_thresh)
454 return true;
455
456 bypass_first = READ_ONCE(rdp->nocb_bypass_first);
457 bypass_is_lazy = (bypass_ncbs == lazy_ncbs);
458
459 if (bypass_is_lazy)
460 flush_timeout = rcu_get_jiffies_lazy_flush();
461 else
462 flush_timeout = flush_faster ? 0 : 1;
463
464 return time_after(j, bypass_first + flush_timeout);
465}
466
467/*
468 * See whether it is appropriate to use the ->nocb_bypass list in order
469 * to control contention on ->nocb_lock. A limited number of direct
470 * enqueues are permitted into ->cblist per jiffy. If ->nocb_bypass
471 * is non-empty, further callbacks must be placed into ->nocb_bypass,
472 * otherwise rcu_barrier() breaks. Use rcu_nocb_flush_bypass() to switch
473 * back to direct use of ->cblist. However, ->nocb_bypass should not be
474 * used if ->cblist is empty, because otherwise callbacks can be stranded
475 * on ->nocb_bypass because we cannot count on the current CPU ever again
476 * invoking call_rcu(). The general rule is that if ->nocb_bypass is
477 * non-empty, the corresponding no-CBs grace-period kthread must not be
478 * in an indefinite sleep state.
479 *
480 * Finally, it is not permitted to use the bypass during early boot,
481 * as doing so would confuse the auto-initialization code. Besides
482 * which, there is no point in worrying about lock contention while
483 * there is only one CPU in operation.
484 */
485static bool rcu_nocb_try_bypass(struct rcu_data *rdp, struct rcu_head *rhp,
486 bool *was_alldone, unsigned long flags,
487 bool lazy)
488{
489 unsigned long c;
490 struct rcu_gp_seq cur_gp_seq;
491 unsigned long j = jiffies;
492 long ncbs = rcu_cblist_n_cbs(&rdp->nocb_bypass);
493 long lazy_len = READ_ONCE(rdp->lazy_len);
494 bool bypass_is_lazy = (ncbs == lazy_len);
495
496 lockdep_assert_irqs_disabled();
497
498 // Pure softirq/rcuc based processing: no bypassing, no
499 // locking.
500 if (!rcu_rdp_is_offloaded(rdp)) {
501 *was_alldone = !rcu_segcblist_pend_cbs(&rdp->cblist);
502 return false;
503 }
504
505 // Don't use ->nocb_bypass during early boot.
506 if (rcu_scheduler_active != RCU_SCHEDULER_RUNNING) {
507 rcu_nocb_lock(rdp);
508 WARN_ON_ONCE(rcu_cblist_n_cbs(&rdp->nocb_bypass));
509 *was_alldone = !rcu_segcblist_pend_cbs(&rdp->cblist);
510 return false;
511 }
512
513 // If we have advanced to a new jiffy, reset counts to allow
514 // moving back from ->nocb_bypass to ->cblist.
515 if (j == rdp->nocb_nobypass_last) {
516 c = rdp->nocb_nobypass_count + 1;
517 } else {
518 WRITE_ONCE(rdp->nocb_nobypass_last, j);
519 c = rdp->nocb_nobypass_count - nocb_nobypass_lim_per_jiffy;
520 if (ULONG_CMP_LT(rdp->nocb_nobypass_count,
521 nocb_nobypass_lim_per_jiffy))
522 c = 0;
523 else if (c > nocb_nobypass_lim_per_jiffy)
524 c = nocb_nobypass_lim_per_jiffy;
525 }
526 WRITE_ONCE(rdp->nocb_nobypass_count, c);
527
528 // If there hasn't yet been all that many ->cblist enqueues
529 // this jiffy, tell the caller to enqueue onto ->cblist. But flush
530 // ->nocb_bypass first.
531 // Lazy CBs throttle this back and do immediate bypass queuing.
532 if (rdp->nocb_nobypass_count < nocb_nobypass_lim_per_jiffy && !lazy) {
533 rcu_nocb_lock(rdp);
534 *was_alldone = !rcu_segcblist_pend_cbs(&rdp->cblist);
535 if (*was_alldone)
536 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu,
537 TPS("FirstQ"));
538
539 WARN_ON_ONCE(!rcu_nocb_flush_bypass(rdp, NULL, j, false));
540 WARN_ON_ONCE(rcu_cblist_n_cbs(&rdp->nocb_bypass));
541 return false; // Caller must enqueue the callback.
542 }
543
544 // If ->nocb_bypass has been used too long or is too full,
545 // flush ->nocb_bypass to ->cblist.
546 if (nocb_bypass_needs_flush(rdp, ncbs, lazy_len, j, true)) {
547 rcu_nocb_lock(rdp);
548 *was_alldone = !rcu_segcblist_pend_cbs(&rdp->cblist);
549
550 if (!rcu_nocb_flush_bypass(rdp, rhp, j, lazy)) {
551 if (*was_alldone)
552 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu,
553 TPS("FirstQ"));
554 WARN_ON_ONCE(rcu_cblist_n_cbs(&rdp->nocb_bypass));
555 return false; // Caller must enqueue the callback.
556 }
557 if (j != rdp->nocb_gp_adv_time &&
558 rcu_segcblist_nextgp(&rdp->cblist, &cur_gp_seq) &&
559 poll_state_synchronize_rcu_full(&cur_gp_seq)) {
560 rcu_advance_cbs_nowake(rdp->mynode, rdp);
561 rdp->nocb_gp_adv_time = j;
562 }
563
564 // The flush succeeded and we moved CBs into the regular list.
565 // Don't wait for the wake up timer as it may be too far ahead.
566 // Wake up the GP thread now instead, if the cblist was empty.
567 __call_rcu_nocb_wake(rdp, *was_alldone, flags);
568
569 return true; // Callback already enqueued.
570 }
571
572 // We need to use the bypass.
573 rcu_nocb_bypass_lock(rdp);
574 ncbs = rcu_cblist_n_cbs(&rdp->nocb_bypass);
575 rcu_segcblist_inc_len(&rdp->cblist); /* Must precede enqueue. */
576 rcu_cblist_enqueue(&rdp->nocb_bypass, rhp);
577
578 if (lazy)
579 WRITE_ONCE(rdp->lazy_len, rdp->lazy_len + 1);
580
581 if (!ncbs) {
582 WRITE_ONCE(rdp->nocb_bypass_first, j);
583 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("FirstBQ"));
584 }
585 rcu_nocb_bypass_unlock(rdp);
586
587 // A wake up of the grace period kthread or timer adjustment
588 // needs to be done only if:
589 // 1. Bypass list was fully empty before (this is the first
590 // bypass list entry), or:
591 // 2. Both of these conditions are met:
592 // a. The bypass list previously had only lazy CBs, and:
593 // b. The new CB is non-lazy.
594 if (!ncbs || (bypass_is_lazy && !lazy)) {
595 // No-CBs GP kthread might be indefinitely asleep, if so, wake.
596 rcu_nocb_lock(rdp); // Rare during call_rcu() flood.
597 if (!rcu_segcblist_pend_cbs(&rdp->cblist)) {
598 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu,
599 TPS("FirstBQwake"));
600 __call_rcu_nocb_wake(rdp, true, flags);
601 } else {
602 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu,
603 TPS("FirstBQnoWake"));
604 rcu_nocb_unlock(rdp);
605 }
606 }
607 return true; // Callback already enqueued.
608}
609
610/*
611 * Awaken the no-CBs grace-period kthread if needed due to it legitimately
612 * being asleep.
613 */
614static void __call_rcu_nocb_wake(struct rcu_data *rdp, bool was_alldone,
615 unsigned long flags)
616 __releases(rdp->nocb_lock)
617{
618 long bypass_len;
619 long lazy_len;
620 long len;
621 struct task_struct *t;
622
623 // If we are being polled or there is no kthread, just leave.
624 t = READ_ONCE(rdp->nocb_gp_kthread);
625 if (rcu_nocb_poll || !t) {
626 rcu_nocb_unlock(rdp);
627 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu,
628 TPS("WakeNotPoll"));
629 return;
630 }
631 // Need to actually to a wakeup.
632 len = rcu_segcblist_n_cbs(&rdp->cblist);
633 bypass_len = rcu_cblist_n_cbs(&rdp->nocb_bypass);
634 lazy_len = READ_ONCE(rdp->lazy_len);
635 if (was_alldone) {
636 rdp->qlen_last_fqs_check = len;
637 rcu_nocb_unlock(rdp);
638 // Only lazy CBs in bypass list
639 if (lazy_len && bypass_len == lazy_len) {
640 wake_nocb_gp_defer(rdp, RCU_NOCB_WAKE_LAZY,
641 TPS("WakeLazy"));
642 } else if (!irqs_disabled_flags(flags)) {
643 /* ... if queue was empty ... */
644 wake_nocb_gp(rdp);
645 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu,
646 TPS("WakeEmpty"));
647 } else {
648 wake_nocb_gp_defer(rdp, RCU_NOCB_WAKE,
649 TPS("WakeEmptyIsDeferred"));
650 }
651
652 return;
653 }
654
655 rcu_nocb_unlock(rdp);
656 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("WakeNot"));
657}
658
659static void call_rcu_nocb(struct rcu_data *rdp, struct rcu_head *head,
660 unsigned long flags, bool lazy)
661{
662 bool was_alldone;
663
664 if (!rcu_nocb_try_bypass(rdp, head, &was_alldone, flags, lazy)) {
665 /* Not enqueued on bypass but locked, do regular enqueue */
666 rcutree_enqueue(rdp, head);
667 __call_rcu_nocb_wake(rdp, was_alldone, flags); /* unlocks */
668 }
669}
670
671static void nocb_gp_toggle_rdp(struct rcu_data *rdp_gp, struct rcu_data *rdp)
672{
673 struct rcu_segcblist *cblist = &rdp->cblist;
674 unsigned long flags;
675
676 /*
677 * Locking orders future de-offloaded callbacks enqueue against previous
678 * handling of this rdp. Ie: Make sure rcuog is done with this rdp before
679 * deoffloaded callbacks can be enqueued.
680 */
681 raw_spin_lock_irqsave(&rdp->nocb_lock, flags);
682 if (!rcu_segcblist_test_flags(cblist, SEGCBLIST_OFFLOADED)) {
683 /*
684 * Offloading. Set our flag and notify the offload worker.
685 * We will handle this rdp until it ever gets de-offloaded.
686 */
687 list_add_tail(&rdp->nocb_entry_rdp, &rdp_gp->nocb_head_rdp);
688 rcu_segcblist_set_flags(cblist, SEGCBLIST_OFFLOADED);
689 } else {
690 /*
691 * De-offloading. Clear our flag and notify the de-offload worker.
692 * We will ignore this rdp until it ever gets re-offloaded.
693 */
694 list_del(&rdp->nocb_entry_rdp);
695 rcu_segcblist_clear_flags(cblist, SEGCBLIST_OFFLOADED);
696 }
697 raw_spin_unlock_irqrestore(&rdp->nocb_lock, flags);
698}
699
700static void nocb_gp_sleep(struct rcu_data *my_rdp, int cpu)
701{
702 trace_rcu_nocb_wake(rcu_state.name, cpu, TPS("Sleep"));
703 swait_event_interruptible_exclusive(my_rdp->nocb_gp_wq,
704 !READ_ONCE(my_rdp->nocb_gp_sleep));
705 trace_rcu_nocb_wake(rcu_state.name, cpu, TPS("EndSleep"));
706}
707
708/*
709 * No-CBs GP kthreads come here to wait for additional callbacks to show up
710 * or for grace periods to end.
711 */
712static noinline_for_stack void nocb_gp_wait(struct rcu_data *my_rdp)
713{
714 bool bypass = false;
715 int __maybe_unused cpu = my_rdp->cpu;
716 unsigned long flags;
717 bool gotcbs = false;
718 unsigned long j = jiffies;
719 bool lazy = false;
720 bool needwait_gp = false; // This prevents actual uninitialized use.
721 bool needwake;
722 bool needwake_gp;
723 struct rcu_data *rdp, *rdp_toggling = NULL;
724 struct rcu_node *rnp;
725 struct rcu_gp_seq wait_gp_seq = {0}; // Suppress "use uninitialized" warning.
726 bool wasempty = false;
727
728 /*
729 * Each pass through the following loop checks for CBs and for the
730 * nearest grace period (if any) to wait for next. The CB kthreads
731 * and the global grace-period kthread are awakened if needed.
732 */
733 WARN_ON_ONCE(my_rdp->nocb_gp_rdp != my_rdp);
734 /*
735 * An rcu_data structure is removed from the list after its
736 * CPU is de-offloaded and added to the list before that CPU is
737 * (re-)offloaded. If the following loop happens to be referencing
738 * that rcu_data structure during the time that the corresponding
739 * CPU is de-offloaded and then immediately re-offloaded, this
740 * loop's rdp pointer will be carried to the end of the list by
741 * the resulting pair of list operations. This can cause the loop
742 * to skip over some of the rcu_data structures that were supposed
743 * to have been scanned. Fortunately a new iteration through the
744 * entire loop is forced after a given CPU's rcu_data structure
745 * is added to the list, so the skipped-over rcu_data structures
746 * won't be ignored for long.
747 */
748 list_for_each_entry(rdp, &my_rdp->nocb_head_rdp, nocb_entry_rdp) {
749 struct rcu_gp_seq cur_gp_seq;
750 long bypass_ncbs;
751 bool flush_bypass = false;
752 long lazy_ncbs;
753
754 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("Check"));
755 rcu_nocb_lock_irqsave(rdp, flags);
756 lockdep_assert_held(&rdp->nocb_lock);
757 bypass_ncbs = rcu_cblist_n_cbs(&rdp->nocb_bypass);
758 lazy_ncbs = READ_ONCE(rdp->lazy_len);
759
760 flush_bypass = nocb_bypass_needs_flush(rdp, bypass_ncbs, lazy_ncbs, j, false);
761 if (!flush_bypass && !bypass_ncbs && rcu_segcblist_empty(&rdp->cblist)) {
762 rcu_nocb_unlock_irqrestore(rdp, flags);
763 continue; /* No callbacks here, try next. */
764 }
765
766 if (flush_bypass) {
767 // Bypass full or old, so flush it.
768 (void)rcu_nocb_try_flush_bypass(rdp, j);
769 bypass_ncbs = rcu_cblist_n_cbs(&rdp->nocb_bypass);
770 lazy_ncbs = READ_ONCE(rdp->lazy_len);
771 }
772
773 if (bypass_ncbs) {
774 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu,
775 bypass_ncbs == lazy_ncbs ? TPS("Lazy") : TPS("Bypass"));
776 if (bypass_ncbs == lazy_ncbs)
777 lazy = true;
778 else
779 bypass = true;
780 }
781 rnp = rdp->mynode;
782
783 // Advance callbacks if helpful and low contention.
784 needwake_gp = false;
785 if (!rcu_segcblist_restempty(&rdp->cblist,
786 RCU_NEXT_READY_TAIL) ||
787 (rcu_segcblist_nextgp(&rdp->cblist, &cur_gp_seq) &&
788 poll_state_synchronize_rcu_full(&cur_gp_seq))) {
789 raw_spin_lock_rcu_node(rnp); /* irqs disabled. */
790 needwake_gp = rcu_advance_cbs(rnp, rdp);
791 wasempty = rcu_segcblist_restempty(&rdp->cblist,
792 RCU_NEXT_READY_TAIL);
793 raw_spin_unlock_rcu_node(rnp); /* irqs disabled. */
794 }
795 WARN_ON_ONCE(wasempty &&
796 !rcu_segcblist_restempty(&rdp->cblist,
797 RCU_NEXT_READY_TAIL));
798 // Need to wait on some grace period?
799 if (rcu_segcblist_nextgp(&rdp->cblist, &cur_gp_seq)) {
800 /*
801 * Track the earliest pending normal and expedited GP
802 * across the group so the wait below can be released by
803 * whichever completes first.
804 */
805 if (!needwait_gp || ULONG_CMP_LT(cur_gp_seq.norm, wait_gp_seq.norm))
806 wait_gp_seq.norm = cur_gp_seq.norm;
807 if (!needwait_gp || ULONG_CMP_LT(cur_gp_seq.exp, wait_gp_seq.exp))
808 wait_gp_seq.exp = cur_gp_seq.exp;
809 needwait_gp = true;
810 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu,
811 TPS("NeedWaitGP"));
812 }
813 if (rcu_segcblist_ready_cbs(&rdp->cblist)) {
814 needwake = rdp->nocb_cb_sleep;
815 WRITE_ONCE(rdp->nocb_cb_sleep, false);
816 } else {
817 needwake = false;
818 }
819 rcu_nocb_unlock_irqrestore(rdp, flags);
820 if (needwake) {
821 swake_up_one(&rdp->nocb_cb_wq);
822 gotcbs = true;
823 }
824 if (needwake_gp)
825 rcu_gp_kthread_wake();
826 }
827
828 my_rdp->nocb_gp_bypass = bypass;
829 my_rdp->nocb_gp_gp = needwait_gp;
830 if (needwait_gp)
831 my_rdp->nocb_gp_seq = wait_gp_seq;
832
833 // At least one child with non-empty ->nocb_bypass, so set
834 // timer in order to avoid stranding its callbacks.
835 if (!rcu_nocb_poll) {
836 // If bypass list only has lazy CBs. Add a deferred lazy wake up.
837 if (lazy && !bypass) {
838 wake_nocb_gp_defer(my_rdp, RCU_NOCB_WAKE_LAZY,
839 TPS("WakeLazyIsDeferred"));
840 // Otherwise add a deferred bypass wake up.
841 } else if (bypass) {
842 wake_nocb_gp_defer(my_rdp, RCU_NOCB_WAKE_BYPASS,
843 TPS("WakeBypassIsDeferred"));
844 }
845 }
846
847 if (rcu_nocb_poll) {
848 /* Polling, so trace if first poll in the series. */
849 if (gotcbs)
850 trace_rcu_nocb_wake(rcu_state.name, cpu, TPS("Poll"));
851 if (list_empty(&my_rdp->nocb_head_rdp)) {
852 raw_spin_lock_irqsave(&my_rdp->nocb_gp_lock, flags);
853 if (!my_rdp->nocb_toggling_rdp)
854 WRITE_ONCE(my_rdp->nocb_gp_sleep, true);
855 raw_spin_unlock_irqrestore(&my_rdp->nocb_gp_lock, flags);
856 /* Wait for any offloading rdp */
857 nocb_gp_sleep(my_rdp, cpu);
858 } else {
859 schedule_timeout_idle(1);
860 }
861 } else if (!needwait_gp) {
862 /* Wait for callbacks to appear. */
863 nocb_gp_sleep(my_rdp, cpu);
864 } else {
865 rnp = my_rdp->mynode;
866 trace_rcu_this_gp(rnp, wait_gp_seq.norm, TPS("StartWait"));
867 swait_event_interruptible_exclusive(
868 rnp->nocb_gp_wq[rcu_seq_ctr(wait_gp_seq.norm) & 0x1],
869 poll_state_synchronize_rcu_full(&wait_gp_seq) ||
870 !READ_ONCE(my_rdp->nocb_gp_sleep));
871 trace_rcu_this_gp(rnp, wait_gp_seq.norm, TPS("EndWait"));
872 }
873
874 if (!rcu_nocb_poll) {
875 raw_spin_lock_irqsave(&my_rdp->nocb_gp_lock, flags);
876 // (De-)queue an rdp to/from the group if its nocb state is changing
877 rdp_toggling = my_rdp->nocb_toggling_rdp;
878 if (rdp_toggling)
879 my_rdp->nocb_toggling_rdp = NULL;
880
881 nocb_defer_wakeup_cancel(my_rdp);
882 WRITE_ONCE(my_rdp->nocb_gp_sleep, true);
883 raw_spin_unlock_irqrestore(&my_rdp->nocb_gp_lock, flags);
884 } else {
885 rdp_toggling = READ_ONCE(my_rdp->nocb_toggling_rdp);
886 if (rdp_toggling) {
887 /*
888 * Paranoid locking to make sure nocb_toggling_rdp is well
889 * reset *before* we (re)set SEGCBLIST_KTHREAD_GP or we could
890 * race with another round of nocb toggling for this rdp.
891 * Nocb locking should prevent from that already but we stick
892 * to paranoia, especially in rare path.
893 */
894 raw_spin_lock_irqsave(&my_rdp->nocb_gp_lock, flags);
895 my_rdp->nocb_toggling_rdp = NULL;
896 raw_spin_unlock_irqrestore(&my_rdp->nocb_gp_lock, flags);
897 }
898 }
899
900 if (rdp_toggling) {
901 nocb_gp_toggle_rdp(my_rdp, rdp_toggling);
902 swake_up_one(&rdp_toggling->nocb_state_wq);
903 }
904
905 my_rdp->nocb_gp_seq.norm = -1;
906 my_rdp->nocb_gp_seq.exp = -1;
907 WARN_ON(signal_pending(current));
908}
909
910/*
911 * No-CBs grace-period-wait kthread. There is one of these per group
912 * of CPUs, but only once at least one CPU in that group has come online
913 * at least once since boot. This kthread checks for newly posted
914 * callbacks from any of the CPUs it is responsible for, waits for a
915 * grace period, then awakens all of the rcu_nocb_cb_kthread() instances
916 * that then have callback-invocation work to do.
917 */
918static int rcu_nocb_gp_kthread(void *arg)
919{
920 struct rcu_data *rdp = arg;
921
922 for (;;) {
923 WRITE_ONCE(rdp->nocb_gp_loops, rdp->nocb_gp_loops + 1);
924 nocb_gp_wait(rdp);
925 cond_resched_tasks_rcu_qs();
926 }
927 return 0;
928}
929
930static inline bool nocb_cb_wait_cond(struct rcu_data *rdp)
931{
932 return !READ_ONCE(rdp->nocb_cb_sleep) || kthread_should_park();
933}
934
935/*
936 * Invoke any ready callbacks from the corresponding no-CBs CPU,
937 * then, if there are no more, wait for more to appear.
938 */
939static void nocb_cb_wait(struct rcu_data *rdp)
940{
941 struct rcu_segcblist *cblist = &rdp->cblist;
942 struct rcu_gp_seq cur_gp_seq;
943 unsigned long flags;
944 bool needwake_gp = false;
945 struct rcu_node *rnp = rdp->mynode;
946
947 swait_event_interruptible_exclusive(rdp->nocb_cb_wq,
948 nocb_cb_wait_cond(rdp));
949 if (kthread_should_park()) {
950 /*
951 * kthread_park() must be preceded by an rcu_barrier().
952 * But yet another rcu_barrier() might have sneaked in between
953 * the barrier callback execution and the callbacks counter
954 * decrement.
955 */
956 if (rdp->nocb_cb_sleep) {
957 rcu_nocb_lock_irqsave(rdp, flags);
958 WARN_ON_ONCE(rcu_segcblist_n_cbs(&rdp->cblist));
959 rcu_nocb_unlock_irqrestore(rdp, flags);
960 kthread_parkme();
961 }
962 } else if (READ_ONCE(rdp->nocb_cb_sleep)) {
963 WARN_ON(signal_pending(current));
964 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("WokeEmpty"));
965 }
966
967 WARN_ON_ONCE(!rcu_rdp_is_offloaded(rdp));
968
969 local_irq_save(flags);
970 rcu_momentary_eqs();
971 local_irq_restore(flags);
972 /*
973 * Disable BH to provide the expected environment. Also, when
974 * transitioning to/from NOCB mode, a self-requeuing callback might
975 * be invoked from softirq. A short grace period could cause both
976 * instances of this callback would execute concurrently.
977 */
978 local_bh_disable();
979 rcu_do_batch(rdp);
980 local_bh_enable();
981 lockdep_assert_irqs_enabled();
982 rcu_nocb_lock_irqsave(rdp, flags);
983 if (rcu_segcblist_nextgp(cblist, &cur_gp_seq) &&
984 poll_state_synchronize_rcu_full(&cur_gp_seq) &&
985 raw_spin_trylock_rcu_node(rnp)) { /* irqs already disabled. */
986 needwake_gp = rcu_advance_cbs(rdp->mynode, rdp);
987 raw_spin_unlock_rcu_node(rnp); /* irqs remain disabled. */
988 }
989
990 if (!rcu_segcblist_ready_cbs(cblist)) {
991 WRITE_ONCE(rdp->nocb_cb_sleep, true);
992 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("CBSleep"));
993 } else {
994 WRITE_ONCE(rdp->nocb_cb_sleep, false);
995 }
996
997 rcu_nocb_unlock_irqrestore(rdp, flags);
998 if (needwake_gp)
999 rcu_gp_kthread_wake();
1000}
1001
1002/*
1003 * Per-rcu_data kthread, but only for no-CBs CPUs. Repeatedly invoke
1004 * nocb_cb_wait() to do the dirty work.
1005 */
1006static int rcu_nocb_cb_kthread(void *arg)
1007{
1008 struct rcu_data *rdp = arg;
1009
1010 // Each pass through this loop does one callback batch, and,
1011 // if there are no more ready callbacks, waits for them.
1012 for (;;) {
1013 nocb_cb_wait(rdp);
1014 cond_resched_tasks_rcu_qs();
1015 }
1016 return 0;
1017}
1018
1019/* Is a deferred wakeup of rcu_nocb_kthread() required? */
1020static int rcu_nocb_need_deferred_wakeup(struct rcu_data *rdp, int level)
1021{
1022 return READ_ONCE(rdp->nocb_defer_wakeup) >= level;
1023}
1024
1025/* Do a deferred wakeup of rcu_nocb_kthread(). */
1026static bool do_nocb_deferred_wakeup_common(struct rcu_data *rdp_gp,
1027 struct rcu_data *rdp, int level,
1028 unsigned long flags)
1029 __releases(rdp_gp->nocb_gp_lock)
1030{
1031 int ret;
1032
1033 if (!rcu_nocb_need_deferred_wakeup(rdp_gp, level)) {
1034 raw_spin_unlock_irqrestore(&rdp_gp->nocb_gp_lock, flags);
1035 return false;
1036 }
1037
1038 ret = __wake_nocb_gp(rdp_gp, rdp, flags);
1039 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("DeferredWake"));
1040
1041 return ret;
1042}
1043
1044/* Do a deferred wakeup of rcu_nocb_kthread() from a timer handler. */
1045static void do_nocb_deferred_wakeup_timer(struct timer_list *t)
1046{
1047 unsigned long flags;
1048 struct rcu_data *rdp = timer_container_of(rdp, t, nocb_timer);
1049
1050 WARN_ON_ONCE(rdp->nocb_gp_rdp != rdp);
1051 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("Timer"));
1052
1053 raw_spin_lock_irqsave(&rdp->nocb_gp_lock, flags);
1054 do_nocb_deferred_wakeup_common(rdp, rdp, RCU_NOCB_WAKE_BYPASS, flags);
1055}
1056
1057/*
1058 * Do a deferred wakeup of rcu_nocb_kthread() from fastpath.
1059 * This means we do an inexact common-case check. Note that if
1060 * we miss, ->nocb_timer will eventually clean things up.
1061 */
1062static bool do_nocb_deferred_wakeup(struct rcu_data *rdp)
1063{
1064 unsigned long flags;
1065 struct rcu_data *rdp_gp = rdp->nocb_gp_rdp;
1066
1067 if (!rdp_gp || !rcu_nocb_need_deferred_wakeup(rdp_gp, RCU_NOCB_WAKE))
1068 return false;
1069
1070 raw_spin_lock_irqsave(&rdp_gp->nocb_gp_lock, flags);
1071 return do_nocb_deferred_wakeup_common(rdp_gp, rdp, RCU_NOCB_WAKE, flags);
1072}
1073
1074void rcu_nocb_flush_deferred_wakeup(void)
1075{
1076 do_nocb_deferred_wakeup(this_cpu_ptr(&rcu_data));
1077}
1078EXPORT_SYMBOL_GPL(rcu_nocb_flush_deferred_wakeup);
1079
1080static int rcu_nocb_queue_toggle_rdp(struct rcu_data *rdp)
1081{
1082 struct rcu_data *rdp_gp = rdp->nocb_gp_rdp;
1083 bool wake_gp = false;
1084 unsigned long flags;
1085
1086 raw_spin_lock_irqsave(&rdp_gp->nocb_gp_lock, flags);
1087 // Queue this rdp for add/del to/from the list to iterate on rcuog
1088 WRITE_ONCE(rdp_gp->nocb_toggling_rdp, rdp);
1089 if (rdp_gp->nocb_gp_sleep) {
1090 rdp_gp->nocb_gp_sleep = false;
1091 wake_gp = true;
1092 }
1093 raw_spin_unlock_irqrestore(&rdp_gp->nocb_gp_lock, flags);
1094
1095 return wake_gp;
1096}
1097
1098static bool rcu_nocb_rdp_deoffload_wait_cond(struct rcu_data *rdp)
1099{
1100 unsigned long flags;
1101 bool ret;
1102
1103 /*
1104 * Locking makes sure rcuog is done handling this rdp before deoffloaded
1105 * enqueue can happen. Also it keeps the SEGCBLIST_OFFLOADED flag stable
1106 * while the ->nocb_lock is held.
1107 */
1108 raw_spin_lock_irqsave(&rdp->nocb_lock, flags);
1109 ret = !rcu_segcblist_test_flags(&rdp->cblist, SEGCBLIST_OFFLOADED);
1110 raw_spin_unlock_irqrestore(&rdp->nocb_lock, flags);
1111
1112 return ret;
1113}
1114
1115static int rcu_nocb_rdp_deoffload(struct rcu_data *rdp)
1116{
1117 unsigned long flags;
1118 int wake_gp;
1119 struct rcu_data *rdp_gp = rdp->nocb_gp_rdp;
1120
1121 /* CPU must be offline, unless it's early boot */
1122 WARN_ON_ONCE(cpu_online(rdp->cpu) && rdp->cpu != raw_smp_processor_id());
1123
1124 pr_info("De-offloading %d\n", rdp->cpu);
1125
1126 /* Flush all callbacks from segcblist and bypass */
1127 rcu_barrier();
1128
1129 /*
1130 * Make sure the rcuoc kthread isn't in the middle of a nocb locked
1131 * sequence while offloading is deactivated, along with nocb locking.
1132 */
1133 if (rdp->nocb_cb_kthread)
1134 kthread_park(rdp->nocb_cb_kthread);
1135
1136 rcu_nocb_lock_irqsave(rdp, flags);
1137 WARN_ON_ONCE(rcu_cblist_n_cbs(&rdp->nocb_bypass));
1138 WARN_ON_ONCE(rcu_segcblist_n_cbs(&rdp->cblist));
1139 rcu_nocb_unlock_irqrestore(rdp, flags);
1140
1141 wake_gp = rcu_nocb_queue_toggle_rdp(rdp);
1142
1143 mutex_lock(&rdp_gp->nocb_gp_kthread_mutex);
1144
1145 if (rdp_gp->nocb_gp_kthread) {
1146 if (wake_gp)
1147 wake_up_process(rdp_gp->nocb_gp_kthread);
1148
1149 swait_event_exclusive(rdp->nocb_state_wq,
1150 rcu_nocb_rdp_deoffload_wait_cond(rdp));
1151 } else {
1152 /*
1153 * No kthread to clear the flags for us or remove the rdp from the nocb list
1154 * to iterate. Do it here instead. Locking doesn't look stricly necessary
1155 * but we stick to paranoia in this rare path.
1156 */
1157 raw_spin_lock_irqsave(&rdp->nocb_lock, flags);
1158 rcu_segcblist_clear_flags(&rdp->cblist, SEGCBLIST_OFFLOADED);
1159 raw_spin_unlock_irqrestore(&rdp->nocb_lock, flags);
1160
1161 list_del(&rdp->nocb_entry_rdp);
1162 }
1163
1164 mutex_unlock(&rdp_gp->nocb_gp_kthread_mutex);
1165
1166 return 0;
1167}
1168
1169static bool rcu_nocb_rdp_offload_wait_cond(struct rcu_data *rdp)
1170{
1171 unsigned long flags;
1172 bool ret;
1173
1174 raw_spin_lock_irqsave(&rdp->nocb_lock, flags);
1175 ret = rcu_segcblist_test_flags(&rdp->cblist, SEGCBLIST_OFFLOADED);
1176 raw_spin_unlock_irqrestore(&rdp->nocb_lock, flags);
1177
1178 return ret;
1179}
1180
1181static int rcu_nocb_rdp_offload(struct rcu_data *rdp)
1182{
1183 int wake_gp;
1184
1185 WARN_ON_ONCE(cpu_online(rdp->cpu));
1186 /*
1187 * For now we only support re-offload, ie: the rdp must have been
1188 * offloaded on boot first.
1189 */
1190 if (!rdp->nocb_gp_rdp)
1191 return -EINVAL;
1192
1193 if (WARN_ON_ONCE(!rdp->nocb_gp_kthread))
1194 return -EINVAL;
1195
1196 pr_info("Offloading %d\n", rdp->cpu);
1197
1198 WARN_ON_ONCE(rcu_cblist_n_cbs(&rdp->nocb_bypass));
1199 WARN_ON_ONCE(rcu_segcblist_n_cbs(&rdp->cblist));
1200
1201 wake_gp = rcu_nocb_queue_toggle_rdp(rdp);
1202 if (wake_gp)
1203 wake_up_process(rdp->nocb_gp_kthread);
1204
1205 swait_event_exclusive(rdp->nocb_state_wq,
1206 rcu_nocb_rdp_offload_wait_cond(rdp));
1207
1208 kthread_unpark(rdp->nocb_cb_kthread);
1209
1210 return 0;
1211}
1212
1213/* Common helper for CPU offload/deoffload operations. */
1214static int rcu_nocb_cpu_toggle_offload(int cpu, bool offload)
1215{
1216 struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu);
1217 int ret = 0;
1218
1219 cpus_read_lock();
1220 mutex_lock(&rcu_state.nocb_mutex);
1221
1222 /* Already in desired state, nothing to do. */
1223 if (rcu_rdp_is_offloaded(rdp) == offload)
1224 goto out_unlock;
1225
1226 if (cpu_online(cpu)) {
1227 pr_info("NOCB: Cannot CB-%soffload online CPU %d\n",
1228 offload ? "" : "de", rdp->cpu);
1229 ret = -EINVAL;
1230 goto out_unlock;
1231 }
1232
1233 if (offload) {
1234 ret = rcu_nocb_rdp_offload(rdp);
1235 if (!ret)
1236 cpumask_set_cpu(cpu, rcu_nocb_mask);
1237 } else {
1238 ret = rcu_nocb_rdp_deoffload(rdp);
1239 if (!ret)
1240 cpumask_clear_cpu(cpu, rcu_nocb_mask);
1241 }
1242
1243out_unlock:
1244 mutex_unlock(&rcu_state.nocb_mutex);
1245 cpus_read_unlock();
1246 return ret;
1247}
1248
1249int rcu_nocb_cpu_deoffload(int cpu)
1250{
1251 return rcu_nocb_cpu_toggle_offload(cpu, false /* de-offload */);
1252}
1253EXPORT_SYMBOL_GPL(rcu_nocb_cpu_deoffload);
1254
1255int rcu_nocb_cpu_offload(int cpu)
1256{
1257 return rcu_nocb_cpu_toggle_offload(cpu, true /* offload */);
1258}
1259EXPORT_SYMBOL_GPL(rcu_nocb_cpu_offload);
1260
1261#ifdef CONFIG_RCU_LAZY
1262static unsigned long
1263lazy_rcu_shrink_count(struct shrinker *shrink, struct shrink_control *sc)
1264{
1265 int cpu;
1266 unsigned long count = 0;
1267
1268 if (WARN_ON_ONCE(!cpumask_available(rcu_nocb_mask)))
1269 return 0;
1270
1271 /* Protect rcu_nocb_mask against concurrent (de-)offloading. */
1272 if (!mutex_trylock(&rcu_state.nocb_mutex))
1273 return 0;
1274
1275 /* Snapshot count of all CPUs */
1276 for_each_cpu(cpu, rcu_nocb_mask) {
1277 struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu);
1278
1279 count += READ_ONCE(rdp->lazy_len);
1280 }
1281
1282 mutex_unlock(&rcu_state.nocb_mutex);
1283
1284 return count ? count : SHRINK_EMPTY;
1285}
1286
1287static unsigned long
1288lazy_rcu_shrink_scan(struct shrinker *shrink, struct shrink_control *sc)
1289{
1290 int cpu;
1291 unsigned long flags;
1292 unsigned long count = 0;
1293
1294 if (WARN_ON_ONCE(!cpumask_available(rcu_nocb_mask)))
1295 return 0;
1296 /*
1297 * Protect against concurrent (de-)offloading. Otherwise nocb locking
1298 * may be ignored or imbalanced.
1299 */
1300 if (!mutex_trylock(&rcu_state.nocb_mutex)) {
1301 /*
1302 * But really don't insist if nocb_mutex is contended since we
1303 * can't guarantee that it will never engage in a dependency
1304 * chain involving memory allocation. The lock is seldom contended
1305 * anyway.
1306 */
1307 return 0;
1308 }
1309
1310 /* Snapshot count of all CPUs */
1311 for_each_cpu(cpu, rcu_nocb_mask) {
1312 struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu);
1313 int _count;
1314
1315 if (WARN_ON_ONCE(!rcu_rdp_is_offloaded(rdp)))
1316 continue;
1317
1318 if (!READ_ONCE(rdp->lazy_len))
1319 continue;
1320
1321 rcu_nocb_lock_irqsave(rdp, flags);
1322 /*
1323 * Recheck under the nocb lock. Since we are not holding the bypass
1324 * lock we may still race with increments from the enqueuer but still
1325 * we know for sure if there is at least one lazy callback.
1326 */
1327 _count = READ_ONCE(rdp->lazy_len);
1328 if (!_count) {
1329 rcu_nocb_unlock_irqrestore(rdp, flags);
1330 continue;
1331 }
1332 rcu_nocb_try_flush_bypass(rdp, jiffies);
1333 rcu_nocb_unlock_irqrestore(rdp, flags);
1334 wake_nocb_gp(rdp);
1335 sc->nr_to_scan -= _count;
1336 count += _count;
1337 if (sc->nr_to_scan <= 0)
1338 break;
1339 }
1340
1341 mutex_unlock(&rcu_state.nocb_mutex);
1342
1343 return count ? count : SHRINK_STOP;
1344}
1345#endif // #ifdef CONFIG_RCU_LAZY
1346
1347void __init rcu_init_nohz(void)
1348{
1349 int cpu;
1350 struct rcu_data *rdp;
1351 const struct cpumask *cpumask = NULL;
1352 struct shrinker * __maybe_unused lazy_rcu_shrinker;
1353
1354#if defined(CONFIG_NO_HZ_FULL)
1355 if (tick_nohz_full_running && !cpumask_empty(tick_nohz_full_mask))
1356 cpumask = tick_nohz_full_mask;
1357#endif
1358
1359 if (IS_ENABLED(CONFIG_RCU_NOCB_CPU_DEFAULT_ALL) &&
1360 !rcu_state.nocb_is_setup && !cpumask)
1361 cpumask = cpu_possible_mask;
1362
1363 if (cpumask) {
1364 if (!cpumask_available(rcu_nocb_mask)) {
1365 if (!zalloc_cpumask_var(&rcu_nocb_mask, GFP_KERNEL)) {
1366 pr_info("rcu_nocb_mask allocation failed, callback offloading disabled.\n");
1367 return;
1368 }
1369 }
1370
1371 cpumask_or(rcu_nocb_mask, rcu_nocb_mask, cpumask);
1372 rcu_state.nocb_is_setup = true;
1373 }
1374
1375 if (!rcu_state.nocb_is_setup)
1376 return;
1377
1378#ifdef CONFIG_RCU_LAZY
1379 lazy_rcu_shrinker = shrinker_alloc(0, "rcu-lazy");
1380 if (!lazy_rcu_shrinker) {
1381 pr_err("Failed to allocate lazy_rcu shrinker!\n");
1382 } else {
1383 lazy_rcu_shrinker->count_objects = lazy_rcu_shrink_count;
1384 lazy_rcu_shrinker->scan_objects = lazy_rcu_shrink_scan;
1385
1386 shrinker_register(lazy_rcu_shrinker);
1387 }
1388#endif // #ifdef CONFIG_RCU_LAZY
1389
1390 if (!cpumask_subset(rcu_nocb_mask, cpu_possible_mask)) {
1391 pr_info("\tNote: kernel parameter 'rcu_nocbs=', 'nohz_full', or 'isolcpus=' contains nonexistent CPUs.\n");
1392 cpumask_and(rcu_nocb_mask, cpu_possible_mask,
1393 rcu_nocb_mask);
1394 }
1395 if (cpumask_empty(rcu_nocb_mask))
1396 pr_info("\tOffload RCU callbacks from CPUs: (none).\n");
1397 else
1398 pr_info("\tOffload RCU callbacks from CPUs: %*pbl.\n",
1399 cpumask_pr_args(rcu_nocb_mask));
1400 if (rcu_nocb_poll)
1401 pr_info("\tPoll for callbacks from no-CBs CPUs.\n");
1402
1403 for_each_cpu(cpu, rcu_nocb_mask) {
1404 rdp = per_cpu_ptr(&rcu_data, cpu);
1405 if (rcu_segcblist_empty(&rdp->cblist))
1406 rcu_segcblist_init(&rdp->cblist);
1407 rcu_segcblist_set_flags(&rdp->cblist, SEGCBLIST_OFFLOADED);
1408 }
1409 rcu_organize_nocb_kthreads();
1410}
1411
1412/* Initialize per-rcu_data variables for no-CBs CPUs. */
1413static void __init rcu_boot_init_nocb_percpu_data(struct rcu_data *rdp)
1414{
1415 init_swait_queue_head(&rdp->nocb_cb_wq);
1416 init_swait_queue_head(&rdp->nocb_gp_wq);
1417 init_swait_queue_head(&rdp->nocb_state_wq);
1418 raw_spin_lock_init(&rdp->nocb_lock);
1419 raw_spin_lock_init(&rdp->nocb_bypass_lock);
1420 raw_spin_lock_init(&rdp->nocb_gp_lock);
1421 timer_setup(&rdp->nocb_timer, do_nocb_deferred_wakeup_timer, 0);
1422 rcu_cblist_init(&rdp->nocb_bypass);
1423 WRITE_ONCE(rdp->lazy_len, 0);
1424 mutex_init(&rdp->nocb_gp_kthread_mutex);
1425}
1426
1427/*
1428 * If the specified CPU is a no-CBs CPU that does not already have its
1429 * rcuo CB kthread, spawn it. Additionally, if the rcuo GP kthread
1430 * for this CPU's group has not yet been created, spawn it as well.
1431 */
1432static void rcu_spawn_cpu_nocb_kthread(int cpu)
1433{
1434 struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu);
1435 struct rcu_data *rdp_gp;
1436 struct task_struct *t;
1437 struct sched_param sp;
1438
1439 if (!rcu_scheduler_fully_active || !rcu_state.nocb_is_setup)
1440 return;
1441
1442 /* If there already is an rcuo kthread, then nothing to do. */
1443 if (rdp->nocb_cb_kthread)
1444 return;
1445
1446 /* If we didn't spawn the GP kthread first, reorganize! */
1447 sp.sched_priority = kthread_prio;
1448 rdp_gp = rdp->nocb_gp_rdp;
1449 mutex_lock(&rdp_gp->nocb_gp_kthread_mutex);
1450 if (!rdp_gp->nocb_gp_kthread) {
1451 t = kthread_run(rcu_nocb_gp_kthread, rdp_gp,
1452 "rcuog/%d", rdp_gp->cpu);
1453 if (WARN_ONCE(IS_ERR(t), "%s: Could not start rcuo GP kthread, OOM is now expected behavior\n", __func__)) {
1454 mutex_unlock(&rdp_gp->nocb_gp_kthread_mutex);
1455 goto err;
1456 }
1457 WRITE_ONCE(rdp_gp->nocb_gp_kthread, t);
1458 if (kthread_prio)
1459 sched_setscheduler_nocheck(t, SCHED_FIFO, &sp);
1460 }
1461 mutex_unlock(&rdp_gp->nocb_gp_kthread_mutex);
1462
1463 /* Spawn the kthread for this CPU. */
1464 t = kthread_create(rcu_nocb_cb_kthread, rdp,
1465 "rcuo%c/%d", rcu_state.abbr, cpu);
1466 if (WARN_ONCE(IS_ERR(t), "%s: Could not start rcuo CB kthread, OOM is now expected behavior\n", __func__))
1467 goto err;
1468
1469 if (rcu_rdp_is_offloaded(rdp))
1470 wake_up_process(t);
1471 else
1472 kthread_park(t);
1473
1474 if (IS_ENABLED(CONFIG_RCU_NOCB_CPU_CB_BOOST) && kthread_prio)
1475 sched_setscheduler_nocheck(t, SCHED_FIFO, &sp);
1476
1477 WRITE_ONCE(rdp->nocb_cb_kthread, t);
1478 WRITE_ONCE(rdp->nocb_gp_kthread, rdp_gp->nocb_gp_kthread);
1479 return;
1480
1481err:
1482 /*
1483 * No need to protect against concurrent rcu_barrier()
1484 * because the number of callbacks should be 0 for a non-boot CPU,
1485 * therefore rcu_barrier() shouldn't even try to grab the nocb_lock.
1486 * But hold nocb_mutex to avoid nocb_lock imbalance from shrinker.
1487 */
1488 WARN_ON_ONCE(system_state > SYSTEM_BOOTING && rcu_segcblist_n_cbs(&rdp->cblist));
1489 mutex_lock(&rcu_state.nocb_mutex);
1490 if (rcu_rdp_is_offloaded(rdp)) {
1491 rcu_nocb_rdp_deoffload(rdp);
1492 cpumask_clear_cpu(cpu, rcu_nocb_mask);
1493 }
1494 mutex_unlock(&rcu_state.nocb_mutex);
1495}
1496
1497/* How many CB CPU IDs per GP kthread? Default of -1 for sqrt(nr_cpu_ids). */
1498static int rcu_nocb_gp_stride = -1;
1499module_param(rcu_nocb_gp_stride, int, 0444);
1500
1501/*
1502 * Initialize GP-CB relationships for all no-CBs CPU.
1503 */
1504static void __init rcu_organize_nocb_kthreads(void)
1505{
1506 int cpu;
1507 bool firsttime = true;
1508 bool gotnocbs = false;
1509 bool gotnocbscbs = true;
1510 int ls = rcu_nocb_gp_stride;
1511 int nl = 0; /* Next GP kthread. */
1512 struct rcu_data *rdp;
1513 struct rcu_data *rdp_gp = NULL; /* Suppress misguided gcc warn. */
1514
1515 if (!cpumask_available(rcu_nocb_mask))
1516 return;
1517 if (ls == -1) {
1518 ls = nr_cpu_ids / int_sqrt(nr_cpu_ids);
1519 rcu_nocb_gp_stride = ls;
1520 }
1521
1522 /*
1523 * Each pass through this loop sets up one rcu_data structure.
1524 * Should the corresponding CPU come online in the future, then
1525 * we will spawn the needed set of rcu_nocb_kthread() kthreads.
1526 */
1527 for_each_possible_cpu(cpu) {
1528 rdp = per_cpu_ptr(&rcu_data, cpu);
1529 if (rdp->cpu >= nl) {
1530 /* New GP kthread, set up for CBs & next GP. */
1531 gotnocbs = true;
1532 nl = DIV_ROUND_UP(rdp->cpu + 1, ls) * ls;
1533 rdp_gp = rdp;
1534 INIT_LIST_HEAD(&rdp->nocb_head_rdp);
1535 if (dump_tree) {
1536 if (!firsttime)
1537 pr_cont("%s\n", gotnocbscbs
1538 ? "" : " (self only)");
1539 gotnocbscbs = false;
1540 firsttime = false;
1541 pr_alert("%s: No-CB GP kthread CPU %d:",
1542 __func__, cpu);
1543 }
1544 } else {
1545 /* Another CB kthread, link to previous GP kthread. */
1546 gotnocbscbs = true;
1547 if (dump_tree)
1548 pr_cont(" %d", cpu);
1549 }
1550 rdp->nocb_gp_rdp = rdp_gp;
1551 if (cpumask_test_cpu(cpu, rcu_nocb_mask))
1552 list_add_tail(&rdp->nocb_entry_rdp, &rdp_gp->nocb_head_rdp);
1553 }
1554 if (gotnocbs && dump_tree)
1555 pr_cont("%s\n", gotnocbscbs ? "" : " (self only)");
1556}
1557
1558/*
1559 * Bind the current task to the offloaded CPUs. If there are no offloaded
1560 * CPUs, leave the task unbound. Splat if the bind attempt fails.
1561 */
1562void rcu_bind_current_to_nocb(void)
1563{
1564 if (cpumask_available(rcu_nocb_mask) && !cpumask_empty(rcu_nocb_mask))
1565 WARN_ON(sched_setaffinity(current->pid, rcu_nocb_mask));
1566}
1567EXPORT_SYMBOL_GPL(rcu_bind_current_to_nocb);
1568
1569// The ->on_cpu field is available only in CONFIG_SMP=y, so...
1570#ifdef CONFIG_SMP
1571static char *show_rcu_should_be_on_cpu(struct task_struct *tsp)
1572{
1573 return tsp && task_is_running(tsp) && !tsp->on_cpu ? "!" : "";
1574}
1575#else // #ifdef CONFIG_SMP
1576static char *show_rcu_should_be_on_cpu(struct task_struct *tsp)
1577{
1578 return "";
1579}
1580#endif // #else #ifdef CONFIG_SMP
1581
1582/*
1583 * Dump out nocb grace-period kthread state for the specified rcu_data
1584 * structure.
1585 */
1586static void show_rcu_nocb_gp_state(struct rcu_data *rdp)
1587{
1588 struct rcu_node *rnp = rdp->mynode;
1589
1590 pr_info("nocb GP %d %c%c%c%c%c %c[%c%c] %c%c:%ld/%ld rnp %d:%d %lu %c CPU %d%s\n",
1591 rdp->cpu,
1592 "kK"[!!rdp->nocb_gp_kthread],
1593 "lL"[raw_spin_is_locked(&rdp->nocb_gp_lock)],
1594 "dD"[!!rdp->nocb_defer_wakeup],
1595 "tT"[timer_pending(&rdp->nocb_timer)],
1596 "sS"[!!rdp->nocb_gp_sleep],
1597 ".W"[swait_active(&rdp->nocb_gp_wq)],
1598 ".W"[swait_active(&rnp->nocb_gp_wq[0])],
1599 ".W"[swait_active(&rnp->nocb_gp_wq[1])],
1600 ".B"[!!rdp->nocb_gp_bypass],
1601 ".G"[!!rdp->nocb_gp_gp],
1602 (long)rdp->nocb_gp_seq.norm,
1603 (long)rdp->nocb_gp_seq.exp,
1604 rnp->grplo, rnp->grphi, READ_ONCE(rdp->nocb_gp_loops),
1605 rdp->nocb_gp_kthread ? task_state_to_char(rdp->nocb_gp_kthread) : '.',
1606 rdp->nocb_gp_kthread ? (int)task_cpu(rdp->nocb_gp_kthread) : -1,
1607 show_rcu_should_be_on_cpu(rdp->nocb_gp_kthread));
1608}
1609
1610/* Dump out nocb kthread state for the specified rcu_data structure. */
1611static void show_rcu_nocb_state(struct rcu_data *rdp)
1612{
1613 char bufd[22];
1614 char bufw[64];
1615 char bufr[64];
1616 char bufn[22];
1617 char bufb[22];
1618 struct rcu_data *nocb_next_rdp;
1619 struct rcu_segcblist *rsclp = &rdp->cblist;
1620 bool waslocked;
1621 bool wassleep;
1622
1623 if (rdp->nocb_gp_rdp == rdp)
1624 show_rcu_nocb_gp_state(rdp);
1625
1626 if (!rcu_segcblist_is_offloaded(&rdp->cblist))
1627 return;
1628
1629 nocb_next_rdp = list_next_or_null_rcu(&rdp->nocb_gp_rdp->nocb_head_rdp,
1630 &rdp->nocb_entry_rdp,
1631 typeof(*rdp),
1632 nocb_entry_rdp);
1633
1634 sprintf(bufd, "%ld", rsclp->seglen[RCU_DONE_TAIL]);
1635 sprintf(bufw, "%ld(%ld/%ld)", rsclp->seglen[RCU_WAIT_TAIL],
1636 rsclp->gp_seq[RCU_WAIT_TAIL].norm,
1637 rsclp->gp_seq[RCU_WAIT_TAIL].exp);
1638 sprintf(bufr, "%ld(%ld/%ld)", rsclp->seglen[RCU_NEXT_READY_TAIL],
1639 rsclp->gp_seq[RCU_NEXT_READY_TAIL].norm,
1640 rsclp->gp_seq[RCU_NEXT_READY_TAIL].exp);
1641 sprintf(bufn, "%ld", rsclp->seglen[RCU_NEXT_TAIL]);
1642 sprintf(bufb, "%ld", rcu_cblist_n_cbs(&rdp->nocb_bypass));
1643 pr_info(" CB %d^%d->%d %c%c%c%c%c F%ld L%ld C%d %c%s%c%s%c%s%c%s%c%s q%ld %c CPU %d%s\n",
1644 rdp->cpu, rdp->nocb_gp_rdp->cpu,
1645 nocb_next_rdp ? nocb_next_rdp->cpu : -1,
1646 "kK"[!!rdp->nocb_cb_kthread],
1647 "bB"[raw_spin_is_locked(&rdp->nocb_bypass_lock)],
1648 "lL"[raw_spin_is_locked(&rdp->nocb_lock)],
1649 "sS"[!!rdp->nocb_cb_sleep],
1650 ".W"[swait_active(&rdp->nocb_cb_wq)],
1651 jiffies - rdp->nocb_bypass_first,
1652 jiffies - rdp->nocb_nobypass_last,
1653 rdp->nocb_nobypass_count,
1654 ".D"[rcu_segcblist_ready_cbs(rsclp)],
1655 rcu_segcblist_segempty(rsclp, RCU_DONE_TAIL) ? "" : bufd,
1656 ".W"[!rcu_segcblist_segempty(rsclp, RCU_WAIT_TAIL)],
1657 rcu_segcblist_segempty(rsclp, RCU_WAIT_TAIL) ? "" : bufw,
1658 ".R"[!rcu_segcblist_segempty(rsclp, RCU_NEXT_READY_TAIL)],
1659 rcu_segcblist_segempty(rsclp, RCU_NEXT_READY_TAIL) ? "" : bufr,
1660 ".N"[!rcu_segcblist_segempty(rsclp, RCU_NEXT_TAIL)],
1661 rcu_segcblist_segempty(rsclp, RCU_NEXT_TAIL) ? "" : bufn,
1662 ".B"[!!rcu_cblist_n_cbs(&rdp->nocb_bypass)],
1663 !rcu_cblist_n_cbs(&rdp->nocb_bypass) ? "" : bufb,
1664 rcu_segcblist_n_cbs(&rdp->cblist),
1665 rdp->nocb_cb_kthread ? task_state_to_char(rdp->nocb_cb_kthread) : '.',
1666 rdp->nocb_cb_kthread ? (int)task_cpu(rdp->nocb_cb_kthread) : -1,
1667 show_rcu_should_be_on_cpu(rdp->nocb_cb_kthread));
1668
1669 /* It is OK for GP kthreads to have GP state. */
1670 if (rdp->nocb_gp_rdp == rdp)
1671 return;
1672
1673 waslocked = raw_spin_is_locked(&rdp->nocb_gp_lock);
1674 wassleep = swait_active(&rdp->nocb_gp_wq);
1675 if (!rdp->nocb_gp_sleep && !waslocked && !wassleep)
1676 return; /* Nothing untoward. */
1677
1678 pr_info(" nocb GP activity on CB-only CPU!!! %c%c%c %c\n",
1679 "lL"[waslocked],
1680 "dD"[!!rdp->nocb_defer_wakeup],
1681 "sS"[!!rdp->nocb_gp_sleep],
1682 ".W"[wassleep]);
1683}
1684
1685#else /* #ifdef CONFIG_RCU_NOCB_CPU */
1686
1687/* No ->nocb_lock to acquire. */
1688static void rcu_nocb_lock(struct rcu_data *rdp)
1689{
1690}
1691
1692/* No ->nocb_lock to release. */
1693static void rcu_nocb_unlock(struct rcu_data *rdp)
1694{
1695}
1696
1697/* No ->nocb_lock to release. */
1698static void rcu_nocb_unlock_irqrestore(struct rcu_data *rdp,
1699 unsigned long flags)
1700{
1701 local_irq_restore(flags);
1702}
1703
1704/* Lockdep check that ->cblist may be safely accessed. */
1705static void rcu_lockdep_assert_cblist_protected(struct rcu_data *rdp)
1706{
1707 lockdep_assert_irqs_disabled();
1708}
1709
1710static void rcu_nocb_gp_cleanup(struct swait_queue_head *sq)
1711{
1712}
1713
1714static struct swait_queue_head *rcu_nocb_gp_get(struct rcu_node *rnp)
1715{
1716 return NULL;
1717}
1718
1719static void rcu_init_one_nocb(struct rcu_node *rnp)
1720{
1721}
1722
1723static void rcu_nocb_exp_cleanup(struct rcu_node *rnp)
1724{
1725}
1726
1727static bool wake_nocb_gp(struct rcu_data *rdp)
1728{
1729 return false;
1730}
1731
1732static bool rcu_nocb_flush_bypass(struct rcu_data *rdp, struct rcu_head *rhp,
1733 unsigned long j, bool lazy)
1734{
1735 return true;
1736}
1737
1738static void call_rcu_nocb(struct rcu_data *rdp, struct rcu_head *head,
1739 unsigned long flags, bool lazy)
1740{
1741 WARN_ON_ONCE(1); /* Should be dead code! */
1742}
1743
1744static void __call_rcu_nocb_wake(struct rcu_data *rdp, bool was_empty,
1745 unsigned long flags)
1746{
1747 WARN_ON_ONCE(1); /* Should be dead code! */
1748}
1749
1750static void __init rcu_boot_init_nocb_percpu_data(struct rcu_data *rdp)
1751{
1752}
1753
1754static int rcu_nocb_need_deferred_wakeup(struct rcu_data *rdp, int level)
1755{
1756 return false;
1757}
1758
1759static bool do_nocb_deferred_wakeup(struct rcu_data *rdp)
1760{
1761 return false;
1762}
1763
1764static void rcu_spawn_cpu_nocb_kthread(int cpu)
1765{
1766}
1767
1768static void show_rcu_nocb_state(struct rcu_data *rdp)
1769{
1770}
1771
1772#endif /* #else #ifdef CONFIG_RCU_NOCB_CPU */