projects/linux·blob master · refs

1256 lines · 34.0 KiB

1// SPDX-License-Identifier: GPL-2.0
2/*
3 * This file contains functions which emulate a local clock-event
4 * device via a broadcast event source.
5 *
6 * Copyright(C) 2005-2006, Linutronix GmbH, Thomas Gleixner <[email protected]>
7 * Copyright(C) 2005-2007, Red Hat, Inc., Ingo Molnar
8 * Copyright(C) 2006-2007, Timesys Corp., Thomas Gleixner
9 */
10#include <linux/cpu.h>
11#include <linux/err.h>
12#include <linux/hrtimer.h>
13#include <linux/interrupt.h>
14#include <linux/percpu.h>
15#include <linux/profile.h>
16#include <linux/sched.h>
17#include <linux/smp.h>
18#include <linux/module.h>
19
20#include "tick-internal.h"
21
22/*
23 * Broadcast support for broken x86 hardware, where the local apic
24 * timer stops in C3 state.
25 */
26
27static struct tick_device tick_broadcast_device;
28static cpumask_var_t tick_broadcast_mask __cpumask_var_read_mostly;
29static cpumask_var_t tick_broadcast_on __cpumask_var_read_mostly;
30static cpumask_var_t tmpmask __cpumask_var_read_mostly;
31static int tick_broadcast_forced;
32
33static __cacheline_aligned_in_smp DEFINE_RAW_SPINLOCK(tick_broadcast_lock);
34
35#ifdef CONFIG_TICK_ONESHOT
36static DEFINE_PER_CPU(struct clock_event_device *, tick_oneshot_wakeup_device);
37
38static void tick_broadcast_setup_oneshot(struct clock_event_device *bc, bool from_periodic);
39static void tick_broadcast_clear_oneshot(int cpu);
40static void tick_resume_broadcast_oneshot(struct clock_event_device *bc);
41# ifdef CONFIG_HOTPLUG_CPU
42static void tick_broadcast_oneshot_offline(unsigned int cpu);
43# endif
44#else
45static inline void
46tick_broadcast_setup_oneshot(struct clock_event_device *bc, bool from_periodic) { BUG(); }
47static inline void tick_broadcast_clear_oneshot(int cpu) { }
48static inline void tick_resume_broadcast_oneshot(struct clock_event_device *bc) { }
49# ifdef CONFIG_HOTPLUG_CPU
50static inline void tick_broadcast_oneshot_offline(unsigned int cpu) { }
51# endif
52#endif
53
54/*
55 * Debugging: see timer_list.c
56 */
57struct tick_device *tick_get_broadcast_device(void)
58{
59 return &tick_broadcast_device;
60}
61
62struct cpumask *tick_get_broadcast_mask(void)
63{
64 return tick_broadcast_mask;
65}
66
67static struct clock_event_device *tick_get_oneshot_wakeup_device(int cpu);
68
69const struct clock_event_device *tick_get_wakeup_device(int cpu)
70{
71 return tick_get_oneshot_wakeup_device(cpu);
72}
73
74/*
75 * Start the device in periodic mode
76 */
77static void tick_broadcast_start_periodic(struct clock_event_device *bc)
78{
79 if (bc) {
80 bc->next_event_forced = 0;
81 tick_setup_periodic(bc, 1);
82 }
83}
84
85/*
86 * Check, if the device can be utilized as broadcast device:
87 */
88static bool tick_check_broadcast_device(struct clock_event_device *curdev,
89 struct clock_event_device *newdev)
90{
91 if ((newdev->features & CLOCK_EVT_FEAT_DUMMY) ||
92 (newdev->features & CLOCK_EVT_FEAT_PERCPU) ||
93 (newdev->features & CLOCK_EVT_FEAT_C3STOP))
94 return false;
95
96 if (tick_broadcast_device.mode == TICKDEV_MODE_ONESHOT &&
97 !(newdev->features & CLOCK_EVT_FEAT_ONESHOT))
98 return false;
99
100 return !curdev || newdev->rating > curdev->rating;
101}
102
103#ifdef CONFIG_TICK_ONESHOT
104static struct clock_event_device *tick_get_oneshot_wakeup_device(int cpu)
105{
106 return per_cpu(tick_oneshot_wakeup_device, cpu);
107}
108
109static void tick_oneshot_wakeup_handler(struct clock_event_device *wd)
110{
111 wd->next_event_forced = 0;
112 /*
113 * If we woke up early and the tick was reprogrammed in the
114 * meantime then this may be spurious but harmless.
115 */
116 tick_receive_broadcast();
117}
118
119static bool tick_set_oneshot_wakeup_device(struct clock_event_device *newdev,
120 int cpu)
121{
122 struct clock_event_device *curdev = tick_get_oneshot_wakeup_device(cpu);
123
124 if (!newdev)
125 goto set_device;
126
127 if ((newdev->features & CLOCK_EVT_FEAT_DUMMY) ||
128 (newdev->features & CLOCK_EVT_FEAT_C3STOP))
129 return false;
130
131 if (!(newdev->features & CLOCK_EVT_FEAT_PERCPU) ||
132 !(newdev->features & CLOCK_EVT_FEAT_ONESHOT))
133 return false;
134
135 if (!cpumask_equal(newdev->cpumask, cpumask_of(cpu)))
136 return false;
137
138 if (curdev && newdev->rating <= curdev->rating)
139 return false;
140
141 if (!try_module_get(newdev->owner))
142 return false;
143
144 newdev->event_handler = tick_oneshot_wakeup_handler;
145set_device:
146 clockevents_exchange_device(curdev, newdev);
147 per_cpu(tick_oneshot_wakeup_device, cpu) = newdev;
148 return true;
149}
150#else
151static struct clock_event_device *tick_get_oneshot_wakeup_device(int cpu)
152{
153 return NULL;
154}
155
156static bool tick_set_oneshot_wakeup_device(struct clock_event_device *newdev,
157 int cpu)
158{
159 return false;
160}
161#endif
162
163/*
164 * Conditionally install/replace broadcast device
165 */
166void tick_install_broadcast_device(struct clock_event_device *dev, int cpu)
167{
168 struct clock_event_device *cur;
169
170 scoped_guard(raw_spinlock_irqsave, &tick_broadcast_lock) {
171
172 if (tick_set_oneshot_wakeup_device(dev, cpu))
173 return;
174
175 cur = tick_broadcast_device.evtdev;
176 if (!tick_check_broadcast_device(cur, dev))
177 return;
178
179 if (!try_module_get(dev->owner))
180 return;
181
182 __clockevents_exchange_device(cur, dev);
183 if (cur)
184 cur->event_handler = clockevents_handle_noop;
185 WRITE_ONCE(tick_broadcast_device.evtdev, dev);
186 if (!cpumask_empty(tick_broadcast_mask))
187 tick_broadcast_start_periodic(dev);
188 }
189
190 /* Module release must be outside of the lock */
191 if (cur)
192 module_put(cur->owner);
193
194 if (!(dev->features & CLOCK_EVT_FEAT_ONESHOT))
195 return;
196
197 /*
198 * If the system already runs in oneshot mode, switch the newly
199 * registered broadcast device to oneshot mode explicitly.
200 */
201 if (tick_broadcast_oneshot_active()) {
202 tick_broadcast_switch_to_oneshot();
203 return;
204 }
205
206 /*
207 * Inform all cpus about this. We might be in a situation
208 * where we did not switch to oneshot mode because the per cpu
209 * devices are affected by CLOCK_EVT_FEAT_C3STOP and the lack
210 * of a oneshot capable broadcast device. Without that
211 * notification the systems stays stuck in periodic mode
212 * forever.
213 */
214 tick_clock_notify();
215}
216
217/*
218 * Check, if the device is the broadcast device
219 */
220int tick_is_broadcast_device(struct clock_event_device *dev)
221{
222 return (dev && tick_broadcast_device.evtdev == dev);
223}
224
225int tick_broadcast_update_freq(struct clock_event_device *dev, u32 freq)
226{
227 int ret = -ENODEV;
228
229 if (tick_is_broadcast_device(dev)) {
230 raw_spin_lock(&tick_broadcast_lock);
231 ret = __clockevents_update_freq(dev, freq);
232 raw_spin_unlock(&tick_broadcast_lock);
233 }
234 return ret;
235}
236
237
238static void err_broadcast(const struct cpumask *mask)
239{
240 pr_crit_once("Failed to broadcast timer tick. Some CPUs may be unresponsive.\n");
241}
242
243static void tick_device_setup_broadcast_func(struct clock_event_device *dev)
244{
245 if (!dev->broadcast)
246 dev->broadcast = tick_broadcast;
247 if (!dev->broadcast) {
248 pr_warn_once("%s depends on broadcast, but no broadcast function available\n",
249 dev->name);
250 dev->broadcast = err_broadcast;
251 }
252}
253
254/*
255 * Check, if the device is dysfunctional and a placeholder, which
256 * needs to be handled by the broadcast device.
257 */
258int tick_device_uses_broadcast(struct clock_event_device *dev, int cpu)
259{
260 struct clock_event_device *bc = tick_broadcast_device.evtdev;
261 unsigned long flags;
262 int ret = 0;
263
264 raw_spin_lock_irqsave(&tick_broadcast_lock, flags);
265
266 /*
267 * Devices might be registered with both periodic and oneshot
268 * mode disabled. This signals, that the device needs to be
269 * operated from the broadcast device and is a placeholder for
270 * the cpu local device.
271 */
272 if (!tick_device_is_functional(dev)) {
273 dev->event_handler = tick_handle_periodic;
274 tick_device_setup_broadcast_func(dev);
275 cpumask_set_cpu(cpu, tick_broadcast_mask);
276 if (tick_broadcast_device.mode == TICKDEV_MODE_PERIODIC)
277 tick_broadcast_start_periodic(bc);
278 else
279 tick_broadcast_setup_oneshot(bc, false);
280 ret = 1;
281 } else {
282 /*
283 * Clear the broadcast bit for this cpu if the
284 * device is not power state affected.
285 */
286 if (!(dev->features & CLOCK_EVT_FEAT_C3STOP))
287 cpumask_clear_cpu(cpu, tick_broadcast_mask);
288 else
289 tick_device_setup_broadcast_func(dev);
290
291 /*
292 * Clear the broadcast bit if the CPU is not in
293 * periodic broadcast on state.
294 */
295 if (!cpumask_test_cpu(cpu, tick_broadcast_on))
296 cpumask_clear_cpu(cpu, tick_broadcast_mask);
297
298 switch (tick_broadcast_device.mode) {
299 case TICKDEV_MODE_ONESHOT:
300 /*
301 * If the system is in oneshot mode we can
302 * unconditionally clear the oneshot mask bit,
303 * because the CPU is running and therefore
304 * not in an idle state which causes the power
305 * state affected device to stop. Let the
306 * caller initialize the device.
307 */
308 tick_broadcast_clear_oneshot(cpu);
309 ret = 0;
310 break;
311
312 case TICKDEV_MODE_PERIODIC:
313 /*
314 * If the system is in periodic mode, check
315 * whether the broadcast device can be
316 * switched off now.
317 */
318 if (cpumask_empty(tick_broadcast_mask) && bc)
319 clockevents_shutdown(bc);
320 /*
321 * If we kept the cpu in the broadcast mask,
322 * tell the caller to leave the per cpu device
323 * in shutdown state. The periodic interrupt
324 * is delivered by the broadcast device, if
325 * the broadcast device exists and is not
326 * hrtimer based.
327 */
328 if (bc && !(bc->features & CLOCK_EVT_FEAT_HRTIMER))
329 ret = cpumask_test_cpu(cpu, tick_broadcast_mask);
330 break;
331 default:
332 break;
333 }
334 }
335 raw_spin_unlock_irqrestore(&tick_broadcast_lock, flags);
336 return ret;
337}
338
339int tick_receive_broadcast(void)
340{
341 struct tick_device *td = this_cpu_ptr(&tick_cpu_device);
342 struct clock_event_device *evt = td->evtdev;
343
344 if (!evt)
345 return -ENODEV;
346
347 if (!evt->event_handler)
348 return -EINVAL;
349
350 evt->event_handler(evt);
351 return 0;
352}
353
354/*
355 * Broadcast the event to the cpus, which are set in the mask (mangled).
356 */
357static bool tick_do_broadcast(struct cpumask *mask)
358{
359 int cpu = smp_processor_id();
360 struct tick_device *td;
361 bool local = false;
362
363 /*
364 * Check, if the current cpu is in the mask
365 */
366 if (cpumask_test_cpu(cpu, mask)) {
367 struct clock_event_device *bc = tick_broadcast_device.evtdev;
368
369 cpumask_clear_cpu(cpu, mask);
370 /*
371 * We only run the local handler, if the broadcast
372 * device is not hrtimer based. Otherwise we run into
373 * a hrtimer recursion.
374 *
375 * local timer_interrupt()
376 * local_handler()
377 * expire_hrtimers()
378 * bc_handler()
379 * local_handler()
380 * expire_hrtimers()
381 */
382 local = !(bc->features & CLOCK_EVT_FEAT_HRTIMER);
383 }
384
385 if (!cpumask_empty(mask)) {
386 /*
387 * It might be necessary to actually check whether the devices
388 * have different broadcast functions. For now, just use the
389 * one of the first device. This works as long as we have this
390 * misfeature only on x86 (lapic)
391 */
392 td = &per_cpu(tick_cpu_device, cpumask_first(mask));
393 td->evtdev->broadcast(mask);
394 }
395 return local;
396}
397
398/*
399 * Periodic broadcast:
400 * - invoke the broadcast handlers
401 */
402static bool tick_do_periodic_broadcast(void)
403{
404 cpumask_and(tmpmask, cpu_online_mask, tick_broadcast_mask);
405 return tick_do_broadcast(tmpmask);
406}
407
408/*
409 * Event handler for periodic broadcast ticks
410 */
411static void tick_handle_periodic_broadcast(struct clock_event_device *dev)
412{
413 struct tick_device *td = this_cpu_ptr(&tick_cpu_device);
414 bool bc_local;
415
416 raw_spin_lock(&tick_broadcast_lock);
417 tick_broadcast_device.evtdev->next_event_forced = 0;
418
419 /* Handle spurious interrupts gracefully */
420 if (clockevent_state_shutdown(tick_broadcast_device.evtdev)) {
421 raw_spin_unlock(&tick_broadcast_lock);
422 return;
423 }
424
425 bc_local = tick_do_periodic_broadcast();
426
427 if (clockevent_state_oneshot(dev)) {
428 ktime_t next = ktime_add_ns(dev->next_event, TICK_NSEC);
429
430 clockevents_program_event(dev, next, true);
431 }
432 raw_spin_unlock(&tick_broadcast_lock);
433
434 /*
435 * We run the handler of the local cpu after dropping
436 * tick_broadcast_lock because the handler might deadlock when
437 * trying to switch to oneshot mode.
438 */
439 if (bc_local)
440 td->evtdev->event_handler(td->evtdev);
441}
442
443/**
444 * tick_broadcast_control - Enable/disable or force broadcast mode
445 * @mode: The selected broadcast mode
446 *
447 * Called when the system enters a state where affected tick devices
448 * might stop. Note: TICK_BROADCAST_FORCE cannot be undone.
449 */
450void tick_broadcast_control(enum tick_broadcast_mode mode)
451{
452 struct clock_event_device *bc, *dev;
453 struct tick_device *td;
454 int cpu, bc_stopped;
455 unsigned long flags;
456
457 /* Protects also the local clockevent device. */
458 raw_spin_lock_irqsave(&tick_broadcast_lock, flags);
459 td = this_cpu_ptr(&tick_cpu_device);
460 dev = td->evtdev;
461
462 /*
463 * Is the device not affected by the powerstate ?
464 */
465 if (!dev || !(dev->features & CLOCK_EVT_FEAT_C3STOP))
466 goto out;
467
468 if (!tick_device_is_functional(dev))
469 goto out;
470
471 cpu = smp_processor_id();
472 bc = tick_broadcast_device.evtdev;
473 bc_stopped = cpumask_empty(tick_broadcast_mask);
474
475 switch (mode) {
476 case TICK_BROADCAST_FORCE:
477 tick_broadcast_forced = 1;
478 fallthrough;
479 case TICK_BROADCAST_ON:
480 cpumask_set_cpu(cpu, tick_broadcast_on);
481 if (!cpumask_test_and_set_cpu(cpu, tick_broadcast_mask)) {
482 /*
483 * Only shutdown the cpu local device, if:
484 *
485 * - the broadcast device exists
486 * - the broadcast device is not a hrtimer based one
487 * - the broadcast device is in periodic mode to
488 * avoid a hiccup during switch to oneshot mode
489 */
490 if (bc && !(bc->features & CLOCK_EVT_FEAT_HRTIMER) &&
491 tick_broadcast_device.mode == TICKDEV_MODE_PERIODIC)
492 clockevents_shutdown(dev);
493 }
494 break;
495
496 case TICK_BROADCAST_OFF:
497 if (tick_broadcast_forced)
498 break;
499 cpumask_clear_cpu(cpu, tick_broadcast_on);
500 if (cpumask_test_and_clear_cpu(cpu, tick_broadcast_mask)) {
501 if (tick_broadcast_device.mode ==
502 TICKDEV_MODE_PERIODIC)
503 tick_setup_periodic(dev, 0);
504 }
505 break;
506 }
507
508 if (bc) {
509 if (cpumask_empty(tick_broadcast_mask)) {
510 if (!bc_stopped)
511 clockevents_shutdown(bc);
512 } else if (bc_stopped) {
513 if (tick_broadcast_device.mode == TICKDEV_MODE_PERIODIC)
514 tick_broadcast_start_periodic(bc);
515 else
516 tick_broadcast_setup_oneshot(bc, false);
517 }
518 }
519out:
520 raw_spin_unlock_irqrestore(&tick_broadcast_lock, flags);
521}
522EXPORT_SYMBOL_GPL(tick_broadcast_control);
523
524/*
525 * Set the periodic handler depending on broadcast on/off
526 */
527void tick_set_periodic_handler(struct clock_event_device *dev, int broadcast)
528{
529 if (!broadcast)
530 dev->event_handler = tick_handle_periodic;
531 else
532 dev->event_handler = tick_handle_periodic_broadcast;
533}
534
535#ifdef CONFIG_HOTPLUG_CPU
536static void tick_shutdown_broadcast(void)
537{
538 struct clock_event_device *bc = tick_broadcast_device.evtdev;
539
540 if (tick_broadcast_device.mode == TICKDEV_MODE_PERIODIC) {
541 if (bc && cpumask_empty(tick_broadcast_mask))
542 clockevents_shutdown(bc);
543 }
544}
545
546/*
547 * Remove a CPU from broadcasting
548 */
549void tick_broadcast_offline(unsigned int cpu)
550{
551 raw_spin_lock(&tick_broadcast_lock);
552 cpumask_clear_cpu(cpu, tick_broadcast_mask);
553 cpumask_clear_cpu(cpu, tick_broadcast_on);
554 tick_broadcast_oneshot_offline(cpu);
555 tick_shutdown_broadcast();
556 raw_spin_unlock(&tick_broadcast_lock);
557}
558
559#endif
560
561void tick_suspend_broadcast(void)
562{
563 struct clock_event_device *bc;
564 unsigned long flags;
565
566 raw_spin_lock_irqsave(&tick_broadcast_lock, flags);
567
568 bc = tick_broadcast_device.evtdev;
569 if (bc)
570 clockevents_shutdown(bc);
571
572 raw_spin_unlock_irqrestore(&tick_broadcast_lock, flags);
573}
574
575/*
576 * This is called from tick_resume_local() on a resuming CPU. That's
577 * called from the core resume function, tick_unfreeze() and the magic XEN
578 * resume hackery.
579 *
580 * In none of these cases the broadcast device mode can change and the
581 * bit of the resuming CPU in the broadcast mask is safe as well.
582 */
583bool tick_resume_check_broadcast(void)
584{
585 if (tick_broadcast_device.mode == TICKDEV_MODE_ONESHOT)
586 return false;
587 else
588 return cpumask_test_cpu(smp_processor_id(), tick_broadcast_mask);
589}
590
591void tick_resume_broadcast(void)
592{
593 struct clock_event_device *bc;
594 unsigned long flags;
595
596 raw_spin_lock_irqsave(&tick_broadcast_lock, flags);
597
598 bc = tick_broadcast_device.evtdev;
599
600 if (bc) {
601 clockevents_tick_resume(bc);
602
603 switch (tick_broadcast_device.mode) {
604 case TICKDEV_MODE_PERIODIC:
605 if (!cpumask_empty(tick_broadcast_mask))
606 tick_broadcast_start_periodic(bc);
607 break;
608 case TICKDEV_MODE_ONESHOT:
609 if (!cpumask_empty(tick_broadcast_mask))
610 tick_resume_broadcast_oneshot(bc);
611 break;
612 }
613 }
614 raw_spin_unlock_irqrestore(&tick_broadcast_lock, flags);
615}
616
617#ifdef CONFIG_TICK_ONESHOT
618
619static cpumask_var_t tick_broadcast_oneshot_mask __cpumask_var_read_mostly;
620static cpumask_var_t tick_broadcast_pending_mask __cpumask_var_read_mostly;
621static cpumask_var_t tick_broadcast_force_mask __cpumask_var_read_mostly;
622
623/*
624 * Exposed for debugging: see timer_list.c
625 */
626struct cpumask *tick_get_broadcast_oneshot_mask(void)
627{
628 return tick_broadcast_oneshot_mask;
629}
630
631/*
632 * Called before going idle with interrupts disabled. Checks whether a
633 * broadcast event from the other core is about to happen. We detected
634 * that in tick_broadcast_oneshot_control(). The callsite can use this
635 * to avoid a deep idle transition as we are about to get the
636 * broadcast IPI right away.
637 */
638noinstr int tick_check_broadcast_expired(void)
639{
640#ifdef _ASM_GENERIC_BITOPS_INSTRUMENTED_NON_ATOMIC_H
641 return arch_test_bit(smp_processor_id(), cpumask_bits(tick_broadcast_force_mask));
642#else
643 return cpumask_test_cpu(smp_processor_id(), tick_broadcast_force_mask);
644#endif
645}
646
647/*
648 * Set broadcast interrupt affinity
649 */
650static void tick_broadcast_set_affinity(struct clock_event_device *bc,
651 const struct cpumask *cpumask)
652{
653 if (!(bc->features & CLOCK_EVT_FEAT_DYNIRQ))
654 return;
655
656 if (cpumask_equal(bc->cpumask, cpumask))
657 return;
658
659 bc->cpumask = cpumask;
660 irq_set_affinity(bc->irq, bc->cpumask);
661}
662
663static void tick_broadcast_set_event(struct clock_event_device *bc, int cpu,
664 ktime_t expires)
665{
666 if (!clockevent_state_oneshot(bc))
667 clockevents_switch_state(bc, CLOCK_EVT_STATE_ONESHOT);
668
669 clockevents_program_event(bc, expires, 1);
670 tick_broadcast_set_affinity(bc, cpumask_of(cpu));
671}
672
673static void tick_resume_broadcast_oneshot(struct clock_event_device *bc)
674{
675 clockevents_switch_state(bc, CLOCK_EVT_STATE_ONESHOT);
676}
677
678/*
679 * Called from irq_enter() when idle was interrupted to reenable the
680 * per cpu device.
681 */
682void tick_check_oneshot_broadcast_this_cpu(void)
683{
684 if (cpumask_test_cpu(smp_processor_id(), tick_broadcast_oneshot_mask)) {
685 struct tick_device *td = this_cpu_ptr(&tick_cpu_device);
686
687 /*
688 * We might be in the middle of switching over from
689 * periodic to oneshot. If the CPU has not yet
690 * switched over, leave the device alone.
691 */
692 if (td->mode == TICKDEV_MODE_ONESHOT) {
693 clockevents_switch_state(td->evtdev,
694 CLOCK_EVT_STATE_ONESHOT);
695 }
696 }
697}
698
699/*
700 * Handle oneshot mode broadcasting
701 */
702static void tick_handle_oneshot_broadcast(struct clock_event_device *dev)
703{
704 struct tick_device *td;
705 ktime_t now, next_event;
706 int cpu, next_cpu = 0;
707 bool bc_local;
708
709 raw_spin_lock(&tick_broadcast_lock);
710 dev->next_event = KTIME_MAX;
711 tick_broadcast_device.evtdev->next_event_forced = 0;
712 next_event = KTIME_MAX;
713 cpumask_clear(tmpmask);
714 now = ktime_get();
715 /* Find all expired events */
716 for_each_cpu(cpu, tick_broadcast_oneshot_mask) {
717 /*
718 * Required for !SMP because for_each_cpu() reports
719 * unconditionally CPU0 as set on UP kernels.
720 */
721 if (!IS_ENABLED(CONFIG_SMP) &&
722 cpumask_empty(tick_broadcast_oneshot_mask))
723 break;
724
725 td = &per_cpu(tick_cpu_device, cpu);
726 if (td->evtdev->next_event <= now) {
727 cpumask_set_cpu(cpu, tmpmask);
728 /*
729 * Mark the remote cpu in the pending mask, so
730 * it can avoid reprogramming the cpu local
731 * timer in tick_broadcast_oneshot_control().
732 */
733 cpumask_set_cpu(cpu, tick_broadcast_pending_mask);
734 } else if (td->evtdev->next_event < next_event) {
735 next_event = td->evtdev->next_event;
736 next_cpu = cpu;
737 }
738 }
739
740 /*
741 * Remove the current cpu from the pending mask. The event is
742 * delivered immediately in tick_do_broadcast() !
743 */
744 cpumask_clear_cpu(smp_processor_id(), tick_broadcast_pending_mask);
745
746 /* Take care of enforced broadcast requests */
747 cpumask_or(tmpmask, tmpmask, tick_broadcast_force_mask);
748 cpumask_clear(tick_broadcast_force_mask);
749
750 /*
751 * Sanity check. Catch the case where we try to broadcast to
752 * offline cpus.
753 */
754 if (WARN_ON_ONCE(!cpumask_subset(tmpmask, cpu_online_mask)))
755 cpumask_and(tmpmask, tmpmask, cpu_online_mask);
756
757 /*
758 * Wakeup the cpus which have an expired event.
759 */
760 bc_local = tick_do_broadcast(tmpmask);
761
762 /*
763 * Two reasons for reprogram:
764 *
765 * - The global event did not expire any CPU local
766 * events. This happens in dyntick mode, as the maximum PIT
767 * delta is quite small.
768 *
769 * - There are pending events on sleeping CPUs which were not
770 * in the event mask
771 */
772 if (next_event != KTIME_MAX)
773 tick_broadcast_set_event(dev, next_cpu, next_event);
774
775 raw_spin_unlock(&tick_broadcast_lock);
776
777 if (bc_local) {
778 td = this_cpu_ptr(&tick_cpu_device);
779 td->evtdev->event_handler(td->evtdev);
780 }
781}
782
783static int broadcast_needs_cpu(struct clock_event_device *bc, int cpu)
784{
785 if (!(bc->features & CLOCK_EVT_FEAT_HRTIMER))
786 return 0;
787 if (bc->next_event == KTIME_MAX)
788 return 0;
789 return bc->bound_on == cpu ? -EBUSY : 0;
790}
791
792static void broadcast_shutdown_local(struct clock_event_device *bc,
793 struct clock_event_device *dev)
794{
795 /*
796 * For hrtimer based broadcasting we cannot shutdown the cpu
797 * local device if our own event is the first one to expire or
798 * if we own the broadcast timer.
799 */
800 if (bc->features & CLOCK_EVT_FEAT_HRTIMER) {
801 if (broadcast_needs_cpu(bc, smp_processor_id()))
802 return;
803 if (dev->next_event < bc->next_event)
804 return;
805 }
806 clockevents_switch_state(dev, CLOCK_EVT_STATE_SHUTDOWN);
807}
808
809static int ___tick_broadcast_oneshot_control(enum tick_broadcast_state state,
810 struct tick_device *td,
811 int cpu)
812{
813 struct clock_event_device *bc, *dev = td->evtdev;
814 int ret = 0;
815 ktime_t now;
816
817 raw_spin_lock(&tick_broadcast_lock);
818 bc = tick_broadcast_device.evtdev;
819
820 if (state == TICK_BROADCAST_ENTER) {
821 /*
822 * If the current CPU owns the hrtimer broadcast
823 * mechanism, it cannot go deep idle and we do not add
824 * the CPU to the broadcast mask. We don't have to go
825 * through the EXIT path as the local timer is not
826 * shutdown.
827 */
828 ret = broadcast_needs_cpu(bc, cpu);
829 if (ret)
830 goto out;
831
832 /*
833 * If the broadcast device is in periodic mode, we
834 * return.
835 */
836 if (tick_broadcast_device.mode == TICKDEV_MODE_PERIODIC) {
837 /* If it is a hrtimer based broadcast, return busy */
838 if (bc->features & CLOCK_EVT_FEAT_HRTIMER)
839 ret = -EBUSY;
840 goto out;
841 }
842
843 if (!cpumask_test_and_set_cpu(cpu, tick_broadcast_oneshot_mask)) {
844 WARN_ON_ONCE(cpumask_test_cpu(cpu, tick_broadcast_pending_mask));
845
846 /* Conditionally shut down the local timer. */
847 broadcast_shutdown_local(bc, dev);
848
849 /*
850 * We only reprogram the broadcast timer if we
851 * did not mark ourself in the force mask and
852 * if the cpu local event is earlier than the
853 * broadcast event. If the current CPU is in
854 * the force mask, then we are going to be
855 * woken by the IPI right away; we return
856 * busy, so the CPU does not try to go deep
857 * idle.
858 */
859 if (cpumask_test_cpu(cpu, tick_broadcast_force_mask)) {
860 ret = -EBUSY;
861 } else if (dev->next_event < bc->next_event) {
862 tick_broadcast_set_event(bc, cpu, dev->next_event);
863 /*
864 * In case of hrtimer broadcasts the
865 * programming might have moved the
866 * timer to this cpu. If yes, remove
867 * us from the broadcast mask and
868 * return busy.
869 */
870 ret = broadcast_needs_cpu(bc, cpu);
871 if (ret) {
872 cpumask_clear_cpu(cpu,
873 tick_broadcast_oneshot_mask);
874 }
875 }
876 }
877 } else {
878 if (cpumask_test_and_clear_cpu(cpu, tick_broadcast_oneshot_mask)) {
879 clockevents_switch_state(dev, CLOCK_EVT_STATE_ONESHOT);
880 /*
881 * The cpu which was handling the broadcast
882 * timer marked this cpu in the broadcast
883 * pending mask and fired the broadcast
884 * IPI. So we are going to handle the expired
885 * event anyway via the broadcast IPI
886 * handler. No need to reprogram the timer
887 * with an already expired event.
888 */
889 if (cpumask_test_and_clear_cpu(cpu,
890 tick_broadcast_pending_mask))
891 goto out;
892
893 /*
894 * Bail out if there is no next event.
895 */
896 if (dev->next_event == KTIME_MAX)
897 goto out;
898 /*
899 * If the pending bit is not set, then we are
900 * either the CPU handling the broadcast
901 * interrupt or we got woken by something else.
902 *
903 * We are no longer in the broadcast mask, so
904 * if the cpu local expiry time is already
905 * reached, we would reprogram the cpu local
906 * timer with an already expired event.
907 *
908 * This can lead to a ping-pong when we return
909 * to idle and therefore rearm the broadcast
910 * timer before the cpu local timer was able
911 * to fire. This happens because the forced
912 * reprogramming makes sure that the event
913 * will happen in the future and depending on
914 * the min_delta setting this might be far
915 * enough out that the ping-pong starts.
916 *
917 * If the cpu local next_event has expired
918 * then we know that the broadcast timer
919 * next_event has expired as well and
920 * broadcast is about to be handled. So we
921 * avoid reprogramming and enforce that the
922 * broadcast handler, which did not run yet,
923 * will invoke the cpu local handler.
924 *
925 * We cannot call the handler directly from
926 * here, because we might be in a NOHZ phase
927 * and we did not go through the irq_enter()
928 * nohz fixups.
929 */
930 now = ktime_get();
931 if (dev->next_event <= now) {
932 cpumask_set_cpu(cpu, tick_broadcast_force_mask);
933 goto out;
934 }
935 /*
936 * We got woken by something else. Reprogram
937 * the cpu local timer device.
938 */
939 tick_program_event(dev->next_event, 1);
940 }
941 }
942out:
943 raw_spin_unlock(&tick_broadcast_lock);
944 return ret;
945}
946
947static int tick_oneshot_wakeup_control(enum tick_broadcast_state state,
948 struct tick_device *td,
949 int cpu)
950{
951 struct clock_event_device *dev, *wd;
952
953 dev = td->evtdev;
954 if (td->mode != TICKDEV_MODE_ONESHOT)
955 return -EINVAL;
956
957 wd = tick_get_oneshot_wakeup_device(cpu);
958 if (!wd)
959 return -ENODEV;
960
961 switch (state) {
962 case TICK_BROADCAST_ENTER:
963 clockevents_switch_state(dev, CLOCK_EVT_STATE_ONESHOT_STOPPED);
964 clockevents_switch_state(wd, CLOCK_EVT_STATE_ONESHOT);
965 clockevents_program_event(wd, dev->next_event, 1);
966 break;
967 case TICK_BROADCAST_EXIT:
968 /* We may have transitioned to oneshot mode while idle */
969 if (clockevent_get_state(wd) != CLOCK_EVT_STATE_ONESHOT)
970 return -ENODEV;
971 }
972
973 return 0;
974}
975
976int __tick_broadcast_oneshot_control(enum tick_broadcast_state state)
977{
978 struct tick_device *td = this_cpu_ptr(&tick_cpu_device);
979 int cpu = smp_processor_id();
980
981 if (!tick_oneshot_wakeup_control(state, td, cpu))
982 return 0;
983
984 if (tick_broadcast_device.evtdev)
985 return ___tick_broadcast_oneshot_control(state, td, cpu);
986
987 /*
988 * If there is no broadcast or wakeup device, tell the caller not
989 * to go into deep idle.
990 */
991 return -EBUSY;
992}
993
994/*
995 * Reset the one shot broadcast for a cpu
996 *
997 * Called with tick_broadcast_lock held
998 */
999static void tick_broadcast_clear_oneshot(int cpu)
1000{
1001 cpumask_clear_cpu(cpu, tick_broadcast_oneshot_mask);
1002 cpumask_clear_cpu(cpu, tick_broadcast_pending_mask);
1003}
1004
1005static void tick_broadcast_init_next_event(struct cpumask *mask,
1006 ktime_t expires)
1007{
1008 struct tick_device *td;
1009 int cpu;
1010
1011 for_each_cpu(cpu, mask) {
1012 td = &per_cpu(tick_cpu_device, cpu);
1013 if (td->evtdev)
1014 td->evtdev->next_event = expires;
1015 }
1016}
1017
1018static inline ktime_t tick_get_next_period(void)
1019{
1020 ktime_t next;
1021
1022 /*
1023 * Protect against concurrent updates (store /load tearing on
1024 * 32bit). It does not matter if the time is already in the
1025 * past. The broadcast device which is about to be programmed will
1026 * fire in any case.
1027 */
1028 raw_spin_lock(&jiffies_lock);
1029 next = tick_next_period;
1030 raw_spin_unlock(&jiffies_lock);
1031 return next;
1032}
1033
1034/**
1035 * tick_broadcast_setup_oneshot - setup the broadcast device
1036 * @bc: the broadcast device
1037 * @from_periodic: true if called from periodic mode
1038 */
1039static void tick_broadcast_setup_oneshot(struct clock_event_device *bc,
1040 bool from_periodic)
1041{
1042 int cpu = smp_processor_id();
1043 ktime_t nexttick = 0;
1044
1045 if (!bc)
1046 return;
1047
1048 /*
1049 * When the broadcast device was switched to oneshot by the first
1050 * CPU handling the NOHZ change, the other CPUs will reach this
1051 * code via hrtimer_run_queues() -> tick_check_oneshot_change()
1052 * too. Set up the broadcast device only once!
1053 */
1054 if (bc->event_handler == tick_handle_oneshot_broadcast) {
1055 /*
1056 * The CPU which switched from periodic to oneshot mode
1057 * set the broadcast oneshot bit for all other CPUs which
1058 * are in the general (periodic) broadcast mask to ensure
1059 * that CPUs which wait for the periodic broadcast are
1060 * woken up.
1061 *
1062 * Clear the bit for the local CPU as the set bit would
1063 * prevent the first tick_broadcast_enter() after this CPU
1064 * switched to oneshot state to program the broadcast
1065 * device.
1066 *
1067 * This code can also be reached via tick_broadcast_control(),
1068 * but this cannot avoid the tick_broadcast_clear_oneshot()
1069 * as that would break the periodic to oneshot transition of
1070 * secondary CPUs. But that's harmless as the below only
1071 * clears already cleared bits.
1072 */
1073 tick_broadcast_clear_oneshot(cpu);
1074 return;
1075 }
1076
1077
1078 bc->event_handler = tick_handle_oneshot_broadcast;
1079 bc->next_event_forced = 0;
1080 bc->next_event = KTIME_MAX;
1081
1082 /*
1083 * When the tick mode is switched from periodic to oneshot it must
1084 * be ensured that CPUs which are waiting for periodic broadcast
1085 * get their wake-up at the next tick. This is achieved by ORing
1086 * tick_broadcast_mask into tick_broadcast_oneshot_mask.
1087 *
1088 * For other callers, e.g. broadcast device replacement,
1089 * tick_broadcast_oneshot_mask must not be touched as this would
1090 * set bits for CPUs which are already NOHZ, but not idle. Their
1091 * next tick_broadcast_enter() would observe the bit set and fail
1092 * to update the expiry time and the broadcast event device.
1093 */
1094 if (from_periodic) {
1095 cpumask_copy(tmpmask, tick_broadcast_mask);
1096 /* Remove the local CPU as it is obviously not idle */
1097 cpumask_clear_cpu(cpu, tmpmask);
1098 cpumask_or(tick_broadcast_oneshot_mask, tick_broadcast_oneshot_mask, tmpmask);
1099
1100 /*
1101 * Ensure that the oneshot broadcast handler will wake the
1102 * CPUs which are still waiting for periodic broadcast.
1103 */
1104 nexttick = tick_get_next_period();
1105 tick_broadcast_init_next_event(tmpmask, nexttick);
1106
1107 /*
1108 * If the underlying broadcast clock event device is
1109 * already in oneshot state, then there is nothing to do.
1110 * The device was already armed for the next tick
1111 * in tick_handle_broadcast_periodic()
1112 */
1113 if (clockevent_state_oneshot(bc))
1114 return;
1115 }
1116
1117 /*
1118 * When switching from periodic to oneshot mode arm the broadcast
1119 * device for the next tick.
1120 *
1121 * If the broadcast device has been replaced in oneshot mode and
1122 * the oneshot broadcast mask is not empty, then arm it to expire
1123 * immediately in order to reevaluate the next expiring timer.
1124 * @nexttick is 0 and therefore in the past which will cause the
1125 * clockevent code to force an event.
1126 *
1127 * For both cases the programming can be avoided when the oneshot
1128 * broadcast mask is empty.
1129 *
1130 * tick_broadcast_set_event() implicitly switches the broadcast
1131 * device to oneshot state.
1132 */
1133 if (!cpumask_empty(tick_broadcast_oneshot_mask))
1134 tick_broadcast_set_event(bc, cpu, nexttick);
1135}
1136
1137/*
1138 * Select oneshot operating mode for the broadcast device
1139 */
1140void tick_broadcast_switch_to_oneshot(void)
1141{
1142 struct clock_event_device *bc;
1143 enum tick_device_mode oldmode;
1144 unsigned long flags;
1145
1146 raw_spin_lock_irqsave(&tick_broadcast_lock, flags);
1147
1148 oldmode = tick_broadcast_device.mode;
1149 tick_broadcast_device.mode = TICKDEV_MODE_ONESHOT;
1150 bc = tick_broadcast_device.evtdev;
1151 if (bc)
1152 tick_broadcast_setup_oneshot(bc, oldmode == TICKDEV_MODE_PERIODIC);
1153
1154 raw_spin_unlock_irqrestore(&tick_broadcast_lock, flags);
1155}
1156
1157#ifdef CONFIG_HOTPLUG_CPU
1158void hotplug_cpu__broadcast_tick_pull(int deadcpu)
1159{
1160 struct clock_event_device *bc;
1161 unsigned long flags;
1162
1163 raw_spin_lock_irqsave(&tick_broadcast_lock, flags);
1164 bc = tick_broadcast_device.evtdev;
1165
1166 if (bc && broadcast_needs_cpu(bc, deadcpu)) {
1167 /*
1168 * If the broadcast force bit of the current CPU is set,
1169 * then the current CPU has not yet reprogrammed the local
1170 * timer device to avoid a ping-pong race. See
1171 * ___tick_broadcast_oneshot_control().
1172 *
1173 * If the broadcast device is hrtimer based then
1174 * programming the broadcast event below does not have any
1175 * effect because the local clockevent device is not
1176 * running and not programmed because the broadcast event
1177 * is not earlier than the pending event of the local clock
1178 * event device. As a consequence all CPUs waiting for a
1179 * broadcast event are stuck forever.
1180 *
1181 * Detect this condition and reprogram the cpu local timer
1182 * device to avoid the starvation.
1183 */
1184 if (tick_check_broadcast_expired()) {
1185 struct tick_device *td = this_cpu_ptr(&tick_cpu_device);
1186
1187 cpumask_clear_cpu(smp_processor_id(), tick_broadcast_force_mask);
1188 tick_program_event(td->evtdev->next_event, 1);
1189 }
1190
1191 /* This moves the broadcast assignment to this CPU: */
1192 bc->next_event_forced = 0;
1193 clockevents_program_event(bc, bc->next_event, 1);
1194 }
1195 raw_spin_unlock_irqrestore(&tick_broadcast_lock, flags);
1196}
1197
1198/*
1199 * Remove a dying CPU from broadcasting
1200 */
1201static void tick_broadcast_oneshot_offline(unsigned int cpu)
1202{
1203 if (tick_get_oneshot_wakeup_device(cpu))
1204 tick_set_oneshot_wakeup_device(NULL, cpu);
1205
1206 /*
1207 * Clear the broadcast masks for the dead cpu, but do not stop
1208 * the broadcast device!
1209 */
1210 cpumask_clear_cpu(cpu, tick_broadcast_oneshot_mask);
1211 cpumask_clear_cpu(cpu, tick_broadcast_pending_mask);
1212 cpumask_clear_cpu(cpu, tick_broadcast_force_mask);
1213}
1214#endif
1215
1216/*
1217 * Check, whether the broadcast device is in one shot mode
1218 */
1219int tick_broadcast_oneshot_active(void)
1220{
1221 return tick_broadcast_device.mode == TICKDEV_MODE_ONESHOT;
1222}
1223
1224/*
1225 * Check whether the broadcast device supports oneshot.
1226 */
1227bool tick_broadcast_oneshot_available(void)
1228{
1229 struct clock_event_device *bc = READ_ONCE(tick_broadcast_device.evtdev);
1230
1231 return bc ? bc->features & CLOCK_EVT_FEAT_ONESHOT : false;
1232}
1233
1234#else
1235int __tick_broadcast_oneshot_control(enum tick_broadcast_state state)
1236{
1237 struct clock_event_device *bc = READ_ONCE(tick_broadcast_device.evtdev);
1238
1239 if (!bc || (bc->features & CLOCK_EVT_FEAT_HRTIMER))
1240 return -EBUSY;
1241
1242 return 0;
1243}
1244#endif
1245
1246void __init tick_broadcast_init(void)
1247{
1248 zalloc_cpumask_var(&tick_broadcast_mask, GFP_NOWAIT);
1249 zalloc_cpumask_var(&tick_broadcast_on, GFP_NOWAIT);
1250 zalloc_cpumask_var(&tmpmask, GFP_NOWAIT);
1251#ifdef CONFIG_TICK_ONESHOT
1252 zalloc_cpumask_var(&tick_broadcast_oneshot_mask, GFP_NOWAIT);
1253 zalloc_cpumask_var(&tick_broadcast_pending_mask, GFP_NOWAIT);
1254 zalloc_cpumask_var(&tick_broadcast_force_mask, GFP_NOWAIT);
1255#endif
1256}