projects/linux·blob v7.3-rc2 · refs

2639 lines · 76.8 KiB

1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Copyright(C) 2005-2006, Linutronix GmbH, Thomas Gleixner <[email protected]>
4 * Copyright(C) 2005-2007, Red Hat, Inc., Ingo Molnar
5 * Copyright(C) 2006-2007 Timesys Corp., Thomas Gleixner
6 *
7 * High-resolution kernel timers
8 *
9 * In contrast to the low-resolution timeout API, aka timer wheel,
10 * hrtimers provide finer resolution and accuracy depending on system
11 * configuration and capabilities.
12 *
13 * Started by: Thomas Gleixner and Ingo Molnar
14 *
15 * Credits:
16 * Based on the original timer wheel code
17 *
18 * Help, testing, suggestions, bugfixes, improvements were
19 * provided by:
20 *
21 * George Anzinger, Andrew Morton, Steven Rostedt, Roman Zippel
22 * et. al.
23 */
24
25#include <linux/cpu.h>
26#include <linux/export.h>
27#include <linux/percpu.h>
28#include <linux/hrtimer.h>
29#include <linux/hrtimer_bases.h>
30#include <linux/notifier.h>
31#include <linux/syscalls.h>
32#include <linux/interrupt.h>
33#include <linux/tick.h>
34#include <linux/err.h>
35#include <linux/debugobjects.h>
36#include <linux/sched/signal.h>
37#include <linux/sched/sysctl.h>
38#include <linux/sched/rt.h>
39#include <linux/sched/deadline.h>
40#include <linux/sched/nohz.h>
41#include <linux/sched/debug.h>
42#include <linux/sched/isolation.h>
43#include <linux/timer.h>
44#include <linux/freezer.h>
45#include <linux/compat.h>
46
47#include <linux/uaccess.h>
48
49#include <trace/events/timer.h>
50
51#include "tick-internal.h"
52
53/*
54 * Constants to set the queued state of the timer (INACTIVE, ENQUEUED)
55 *
56 * The callback state is kept separate in the CPU base because having it in
57 * the timer would required touching the timer after the callback, which
58 * makes it impossible to free the timer from the callback function.
59 *
60 * Therefore we track the callback state in:
61 *
62 * timer->base->cpu_base->running == timer
63 *
64 * On SMP it is possible to have a "callback function running and enqueued"
65 * status. It happens for example when a posix timer expired and the callback
66 * queued a signal. Between dropping the lock which protects the posix timer
67 * and reacquiring the base lock of the hrtimer, another CPU can deliver the
68 * signal and rearm the timer.
69 *
70 * All state transitions are protected by cpu_base->lock.
71 */
72#define HRTIMER_STATE_INACTIVE false
73#define HRTIMER_STATE_ENQUEUED true
74
75/*
76 * The resolution of the clocks. The resolution value is returned in
77 * the clock_getres() system call to give application programmers an
78 * idea of the (in)accuracy of timers. Timer values are rounded up to
79 * this resolution values.
80 */
81#define HIGH_RES_NSEC 1
82
83/*
84 * Masks for selecting the soft and hard context timers from
85 * cpu_base->active
86 */
87#define MASK_SHIFT (HRTIMER_BASE_MONOTONIC_SOFT)
88#define HRTIMER_ACTIVE_HARD ((1U << MASK_SHIFT) - 1)
89#define HRTIMER_ACTIVE_SOFT (HRTIMER_ACTIVE_HARD << MASK_SHIFT)
90#define HRTIMER_ACTIVE_ALL (HRTIMER_ACTIVE_SOFT | HRTIMER_ACTIVE_HARD)
91
92static void retrigger_next_event(void *arg);
93static ktime_t __hrtimer_cb_get_time(clockid_t clock_id);
94
95/*
96 * The timer bases:
97 *
98 * There are more clockids than hrtimer bases. Thus, we index
99 * into the timer bases by the hrtimer_base_type enum. When trying
100 * to reach a base using a clockid, hrtimer_clockid_to_base()
101 * is used to convert from clockid to the proper hrtimer_base_type.
102 */
103
104#define BASE_INIT(idx, cid) \
105 [idx] = { .index = idx, .clockid = cid }
106
107DEFINE_PER_CPU(struct hrtimer_cpu_base, hrtimer_bases) =
108{
109 .lock = __RAW_SPIN_LOCK_UNLOCKED(hrtimer_bases.lock),
110 .clock_base = {
111 BASE_INIT(HRTIMER_BASE_MONOTONIC, CLOCK_MONOTONIC),
112 BASE_INIT(HRTIMER_BASE_REALTIME, CLOCK_REALTIME),
113 BASE_INIT(HRTIMER_BASE_BOOTTIME, CLOCK_BOOTTIME),
114 BASE_INIT(HRTIMER_BASE_TAI, CLOCK_TAI),
115 BASE_INIT(HRTIMER_BASE_MONOTONIC_SOFT, CLOCK_MONOTONIC),
116 BASE_INIT(HRTIMER_BASE_REALTIME_SOFT, CLOCK_REALTIME),
117 BASE_INIT(HRTIMER_BASE_BOOTTIME_SOFT, CLOCK_BOOTTIME),
118 BASE_INIT(HRTIMER_BASE_TAI_SOFT, CLOCK_TAI),
119 },
120 .csd = CSD_INIT(retrigger_next_event, NULL)
121};
122
123static inline bool hrtimer_base_is_online(struct hrtimer_cpu_base *base)
124{
125 if (!IS_ENABLED(CONFIG_HOTPLUG_CPU))
126 return true;
127 else
128 return likely(base->online);
129}
130
131#ifdef CONFIG_HIGH_RES_TIMERS
132DEFINE_STATIC_KEY_FALSE(hrtimer_highres_enabled_key);
133
134static void hrtimer_hres_workfn(struct work_struct *work)
135{
136 static_branch_enable(&hrtimer_highres_enabled_key);
137}
138
139static DECLARE_WORK(hrtimer_hres_work, hrtimer_hres_workfn);
140
141static inline void hrtimer_schedule_hres_work(void)
142{
143 if (!hrtimer_highres_enabled())
144 schedule_work(&hrtimer_hres_work);
145}
146#else
147static inline void hrtimer_schedule_hres_work(void) { }
148#endif
149
150/*
151 * Functions and macros which are different for UP/SMP systems are kept in a
152 * single place
153 */
154#ifdef CONFIG_SMP
155/*
156 * We require the migration_base for lock_hrtimer_base()/switch_hrtimer_base()
157 * such that hrtimer_callback_running() can unconditionally dereference
158 * timer->base->cpu_base
159 */
160static struct hrtimer_cpu_base migration_cpu_base = {
161 .clock_base = {
162 [0] = {
163 .cpu_base = &migration_cpu_base,
164 .seq = SEQCNT_RAW_SPINLOCK_ZERO(migration_cpu_base.seq,
165 &migration_cpu_base.lock),
166 },
167 },
168};
169
170#define migration_base migration_cpu_base.clock_base[0]
171
172/*
173 * We are using hashed locking: holding per_cpu(hrtimer_bases)[n].lock
174 * means that all timers which are tied to this base via timer->base are
175 * locked, and the base itself is locked too.
176 *
177 * So __run_timers/migrate_timers can safely modify all timers which could
178 * be found on the lists/queues.
179 *
180 * When the timer's base is locked, and the timer removed from list, it is
181 * possible to set timer->base = &migration_base and drop the lock: the timer
182 * remains locked.
183 */
184static struct hrtimer_clock_base *lock_hrtimer_base(const struct hrtimer *timer,
185 unsigned long *flags)
186 __acquires(&timer->base->lock)
187{
188 for (;;) {
189 struct hrtimer_clock_base *base = READ_ONCE(timer->base);
190
191 if (likely(base != &migration_base)) {
192 raw_spin_lock_irqsave(&base->cpu_base->lock, *flags);
193 if (likely(base == timer->base))
194 return base;
195 /* The timer has migrated to another CPU: */
196 raw_spin_unlock_irqrestore(&base->cpu_base->lock, *flags);
197 }
198 cpu_relax();
199 }
200}
201
202/*
203 * Check if the elected target is suitable considering its next
204 * event and the hotplug state of the current CPU.
205 *
206 * If the elected target is remote and its next event is after the timer
207 * to queue, then a remote reprogram is necessary. However there is no
208 * guarantee the IPI handling the operation would arrive in time to meet
209 * the high resolution deadline. In this case the local CPU becomes a
210 * preferred target, unless it is offline.
211 *
212 * High and low resolution modes are handled the same way for simplicity.
213 *
214 * Called with cpu_base->lock of target cpu held.
215 */
216static bool hrtimer_suitable_target(struct hrtimer *timer, struct hrtimer_clock_base *new_base,
217 struct hrtimer_cpu_base *new_cpu_base,
218 struct hrtimer_cpu_base *this_cpu_base)
219{
220 ktime_t expires;
221
222 /*
223 * The local CPU clockevent can be reprogrammed. Also get_target_base()
224 * guarantees it is online.
225 */
226 if (new_cpu_base == this_cpu_base)
227 return true;
228
229 /*
230 * The offline local CPU can't be the default target if the
231 * next remote target event is after this timer. Keep the
232 * elected new base. An IPI will be issued to reprogram
233 * it as a last resort.
234 */
235 if (!hrtimer_base_is_online(this_cpu_base))
236 return true;
237
238 expires = ktime_sub(hrtimer_get_expires(timer), new_base->offset);
239
240 return expires >= new_base->cpu_base->expires_next;
241}
242
243static inline struct hrtimer_cpu_base *get_target_base(struct hrtimer_cpu_base *base, bool pinned)
244{
245 if (!hrtimer_base_is_online(base)) {
246 int cpu = cpumask_any_and(cpu_online_mask, housekeeping_cpumask(HK_TYPE_TIMER));
247
248 return &per_cpu(hrtimer_bases, cpu);
249 }
250
251#if defined(CONFIG_SMP) && defined(CONFIG_NO_HZ_COMMON)
252 if (static_branch_likely(&timers_migration_enabled) && !pinned)
253 return &per_cpu(hrtimer_bases, get_nohz_timer_target());
254#endif
255 return base;
256}
257
258/*
259 * We switch the timer base to a power-optimized selected CPU target,
260 * if:
261 * - NO_HZ_COMMON is enabled
262 * - timer migration is enabled
263 * - the timer callback is not running
264 * - the timer is not the first expiring timer on the new target
265 *
266 * If one of the above requirements is not fulfilled we move the timer
267 * to the current CPU or leave it on the previously assigned CPU if
268 * the timer callback is currently running.
269 */
270static inline struct hrtimer_clock_base *
271switch_hrtimer_base(struct hrtimer *timer, struct hrtimer_clock_base *base, bool pinned)
272{
273 struct hrtimer_cpu_base *new_cpu_base, *this_cpu_base;
274 struct hrtimer_clock_base *new_base;
275 int basenum = base->index;
276
277 this_cpu_base = this_cpu_ptr(&hrtimer_bases);
278 new_cpu_base = get_target_base(this_cpu_base, pinned);
279again:
280 new_base = &new_cpu_base->clock_base[basenum];
281
282 if (base != new_base) {
283 /*
284 * We are trying to move timer to new_base. However we can't
285 * change timer's base while it is running, so we keep it on
286 * the same CPU. No hassle vs. reprogramming the event source
287 * in the high resolution case. The remote CPU will take care
288 * of this when the timer function has completed. There is no
289 * conflict as we hold the lock until the timer is enqueued.
290 */
291 if (unlikely(hrtimer_callback_running(timer)))
292 return base;
293
294 /* See the comment in lock_hrtimer_base() */
295 WRITE_ONCE(timer->base, &migration_base);
296 raw_spin_unlock(&base->cpu_base->lock);
297 raw_spin_lock(&new_base->cpu_base->lock);
298
299 if (!hrtimer_suitable_target(timer, new_base, new_cpu_base, this_cpu_base)) {
300 raw_spin_unlock(&new_base->cpu_base->lock);
301 raw_spin_lock(&base->cpu_base->lock);
302 new_cpu_base = this_cpu_base;
303 WRITE_ONCE(timer->base, base);
304 goto again;
305 }
306 WRITE_ONCE(timer->base, new_base);
307 } else {
308 if (!hrtimer_suitable_target(timer, new_base, new_cpu_base, this_cpu_base)) {
309 new_cpu_base = this_cpu_base;
310 goto again;
311 }
312 }
313 return new_base;
314}
315
316#else /* CONFIG_SMP */
317
318static inline struct hrtimer_clock_base *lock_hrtimer_base(const struct hrtimer *timer,
319 unsigned long *flags)
320 __acquires(&timer->base->cpu_base->lock)
321{
322 struct hrtimer_clock_base *base = timer->base;
323
324 raw_spin_lock_irqsave(&base->cpu_base->lock, *flags);
325 return base;
326}
327
328# define switch_hrtimer_base(t, b, p) (b)
329
330#endif /* !CONFIG_SMP */
331
332/*
333 * Functions for the union type storage format of ktime_t which are
334 * too large for inlining:
335 */
336#if BITS_PER_LONG < 64
337/*
338 * Divide a ktime value by a nanosecond value
339 */
340s64 __ktime_divns(const ktime_t kt, s64 div)
341{
342 int sft = 0;
343 s64 dclc;
344 u64 tmp;
345
346 dclc = ktime_to_ns(kt);
347 tmp = dclc < 0 ? -dclc : dclc;
348
349 /* Make sure the divisor is less than 2^32: */
350 while (div >> 32) {
351 sft++;
352 div >>= 1;
353 }
354 tmp >>= sft;
355 do_div(tmp, (u32) div);
356 return dclc < 0 ? -tmp : tmp;
357}
358EXPORT_SYMBOL_GPL(__ktime_divns);
359#endif /* BITS_PER_LONG < 64 */
360
361/*
362 * Add two ktime values and do a safety check for overflow:
363 */
364ktime_t ktime_add_safe(const ktime_t lhs, const ktime_t rhs)
365{
366 ktime_t res = ktime_add_unsafe(lhs, rhs);
367
368 /*
369 * We use KTIME_SEC_MAX here, the maximum timeout which we can
370 * return to user space in a timespec:
371 */
372 if (res < 0 || res < lhs || res < rhs)
373 res = ktime_set(KTIME_SEC_MAX, 0);
374
375 return res;
376}
377
378EXPORT_SYMBOL_GPL(ktime_add_safe);
379
380#ifdef CONFIG_DEBUG_OBJECTS_TIMERS
381
382static const struct debug_obj_descr hrtimer_debug_descr;
383
384static void *hrtimer_debug_hint(void *addr)
385{
386 return ACCESS_PRIVATE((struct hrtimer *)addr, function);
387}
388
389/*
390 * fixup_init is called when:
391 * - an active object is initialized
392 */
393static bool hrtimer_fixup_init(void *addr, enum debug_obj_state state)
394{
395 struct hrtimer *timer = addr;
396
397 switch (state) {
398 case ODEBUG_STATE_ACTIVE:
399 hrtimer_cancel(timer);
400 debug_object_init(timer, &hrtimer_debug_descr);
401 return true;
402 default:
403 return false;
404 }
405}
406
407/*
408 * fixup_activate is called when:
409 * - an active object is activated
410 * - an unknown non-static object is activated
411 */
412static bool hrtimer_fixup_activate(void *addr, enum debug_obj_state state)
413{
414 switch (state) {
415 case ODEBUG_STATE_ACTIVE:
416 WARN_ON(1);
417 fallthrough;
418 default:
419 return false;
420 }
421}
422
423/*
424 * fixup_free is called when:
425 * - an active object is freed
426 */
427static bool hrtimer_fixup_free(void *addr, enum debug_obj_state state)
428{
429 struct hrtimer *timer = addr;
430
431 switch (state) {
432 case ODEBUG_STATE_ACTIVE:
433 hrtimer_cancel(timer);
434 debug_object_free(timer, &hrtimer_debug_descr);
435 return true;
436 default:
437 return false;
438 }
439}
440
441/* Stub timer callback for improperly used timers. */
442static enum hrtimer_restart stub_timer(struct hrtimer *unused)
443{
444 WARN_ON_ONCE(1);
445 return HRTIMER_NORESTART;
446}
447
448/*
449 * hrtimer_fixup_assert_init is called when:
450 * - an untracked/uninit-ed object is found
451 */
452static bool hrtimer_fixup_assert_init(void *addr, enum debug_obj_state state)
453{
454 struct hrtimer *timer = addr;
455
456 switch (state) {
457 case ODEBUG_STATE_NOTAVAILABLE:
458 hrtimer_setup(timer, stub_timer, CLOCK_MONOTONIC, 0);
459 return true;
460 default:
461 return false;
462 }
463}
464
465static const struct debug_obj_descr hrtimer_debug_descr = {
466 .name = "hrtimer",
467 .debug_hint = hrtimer_debug_hint,
468 .fixup_init = hrtimer_fixup_init,
469 .fixup_activate = hrtimer_fixup_activate,
470 .fixup_free = hrtimer_fixup_free,
471 .fixup_assert_init = hrtimer_fixup_assert_init,
472};
473
474static inline void debug_hrtimer_init(struct hrtimer *timer)
475{
476 debug_object_init(timer, &hrtimer_debug_descr);
477}
478
479static inline void debug_hrtimer_init_on_stack(struct hrtimer *timer)
480{
481 debug_object_init_on_stack(timer, &hrtimer_debug_descr);
482}
483
484static inline void debug_hrtimer_activate(struct hrtimer *timer, enum hrtimer_mode mode)
485{
486 debug_object_activate(timer, &hrtimer_debug_descr);
487}
488
489static inline void debug_hrtimer_deactivate(struct hrtimer *timer)
490{
491 debug_object_deactivate(timer, &hrtimer_debug_descr);
492}
493
494static inline void debug_hrtimer_assert_init(struct hrtimer *timer)
495{
496 debug_object_assert_init(timer, &hrtimer_debug_descr);
497}
498
499void destroy_hrtimer_on_stack(struct hrtimer *timer)
500{
501 debug_object_free(timer, &hrtimer_debug_descr);
502}
503EXPORT_SYMBOL_GPL(destroy_hrtimer_on_stack);
504
505#else
506
507static inline void debug_hrtimer_init(struct hrtimer *timer) { }
508static inline void debug_hrtimer_init_on_stack(struct hrtimer *timer) { }
509static inline void debug_hrtimer_activate(struct hrtimer *timer, enum hrtimer_mode mode) { }
510static inline void debug_hrtimer_deactivate(struct hrtimer *timer) { }
511static inline void debug_hrtimer_assert_init(struct hrtimer *timer) { }
512#endif
513
514static inline void debug_setup(struct hrtimer *timer, clockid_t clockid, enum hrtimer_mode mode)
515{
516 debug_hrtimer_init(timer);
517 trace_hrtimer_setup(timer, clockid, mode);
518}
519
520static inline void debug_setup_on_stack(struct hrtimer *timer, clockid_t clockid,
521 enum hrtimer_mode mode)
522{
523 debug_hrtimer_init_on_stack(timer);
524 trace_hrtimer_setup(timer, clockid, mode);
525}
526
527static inline void debug_activate(struct hrtimer *timer, enum hrtimer_mode mode, bool was_armed)
528{
529 debug_hrtimer_activate(timer, mode);
530 trace_hrtimer_start(timer, mode, was_armed);
531}
532
533#define for_each_active_base(base, cpu_base, active) \
534 for (unsigned int idx = ffs(active); idx--; idx = ffs((active))) \
535 for (bool done = false; !done; active &= ~(1U << idx)) \
536 for (base = &cpu_base->clock_base[idx]; !done; done = true)
537
538#define hrtimer_from_timerqueue_node(_n) container_of_const(_n, struct hrtimer, node)
539
540#if defined(CONFIG_NO_HZ_COMMON)
541/*
542 * Same as hrtimer_bases_next_event() below, but skips the excluded timer and
543 * does not update cpu_base->next_timer/expires.
544 */
545static ktime_t hrtimer_bases_next_event_without(struct hrtimer_cpu_base *cpu_base,
546 const struct hrtimer *exclude,
547 unsigned int active, ktime_t expires_next)
548{
549 struct hrtimer_clock_base *base;
550 ktime_t expires;
551
552 lockdep_assert_held(&cpu_base->lock);
553
554 for_each_active_base(base, cpu_base, active) {
555 expires = ktime_sub(base->expires_next, base->offset);
556 if (expires >= expires_next)
557 continue;
558
559 /*
560 * If the excluded timer is the first on this base evaluate the
561 * next timer.
562 */
563 struct timerqueue_linked_node *node = timerqueue_linked_first(&base->active);
564
565 if (unlikely(&exclude->node == node)) {
566 node = timerqueue_linked_next(node);
567 if (!node)
568 continue;
569 expires = ktime_sub(node->expires, base->offset);
570 if (expires >= expires_next)
571 continue;
572 }
573 expires_next = expires;
574 }
575 /* If base->offset changed, the result might be negative */
576 return max(expires_next, 0);
577}
578#endif
579
580static __always_inline struct hrtimer *clock_base_next_timer(struct hrtimer_clock_base *base)
581{
582 struct timerqueue_linked_node *next = timerqueue_linked_first(&base->active);
583
584 return hrtimer_from_timerqueue_node(next);
585}
586
587/* Find the base with the earliest expiry */
588static void hrtimer_bases_first(struct hrtimer_cpu_base *cpu_base,unsigned int active,
589 ktime_t *expires_next, struct hrtimer **next_timer)
590{
591 struct hrtimer_clock_base *base;
592 ktime_t expires;
593
594 for_each_active_base(base, cpu_base, active) {
595 expires = ktime_sub(base->expires_next, base->offset);
596 if (expires < *expires_next) {
597 *expires_next = expires;
598 *next_timer = clock_base_next_timer(base);
599 }
600 }
601}
602
603/*
604 * Recomputes cpu_base::*next_timer and returns the earliest expires_next
605 * but does not set cpu_base::*expires_next, that is done by
606 * hrtimer[_force]_reprogram and hrtimer_interrupt only. When updating
607 * cpu_base::*expires_next right away, reprogramming logic would no longer
608 * work.
609 *
610 * When a softirq is pending, we can ignore the HRTIMER_ACTIVE_SOFT bases,
611 * those timers will get run whenever the softirq gets handled, at the end of
612 * hrtimer_run_softirq(), hrtimer_update_softirq_timer() will re-add these bases.
613 *
614 * Therefore softirq values are those from the HRTIMER_ACTIVE_SOFT clock bases.
615 * The !softirq values are the minima across HRTIMER_ACTIVE_ALL, unless an actual
616 * softirq is pending, in which case they're the minima of HRTIMER_ACTIVE_HARD.
617 *
618 * @active_mask must be one of:
619 * - HRTIMER_ACTIVE_ALL,
620 * - HRTIMER_ACTIVE_SOFT, or
621 * - HRTIMER_ACTIVE_HARD.
622 */
623static ktime_t __hrtimer_get_next_event(struct hrtimer_cpu_base *cpu_base, unsigned int active_mask)
624{
625 struct hrtimer *next_timer = NULL;
626 ktime_t expires_next = KTIME_MAX;
627 unsigned int active;
628
629 lockdep_assert_held(&cpu_base->lock);
630
631 if (!cpu_base->softirq_activated && (active_mask & HRTIMER_ACTIVE_SOFT)) {
632 active = cpu_base->active_bases & HRTIMER_ACTIVE_SOFT;
633 if (active)
634 hrtimer_bases_first(cpu_base, active, &expires_next, &next_timer);
635 cpu_base->softirq_next_timer = next_timer;
636 }
637
638 if (active_mask & HRTIMER_ACTIVE_HARD) {
639 active = cpu_base->active_bases & HRTIMER_ACTIVE_HARD;
640 if (active)
641 hrtimer_bases_first(cpu_base, active, &expires_next, &next_timer);
642 cpu_base->next_timer = next_timer;
643 }
644 return max(expires_next, 0);
645}
646
647static ktime_t hrtimer_update_next_event(struct hrtimer_cpu_base *cpu_base)
648{
649 ktime_t expires_next, soft = KTIME_MAX;
650
651 /*
652 * If the soft interrupt has already been activated, ignore the
653 * soft bases. They will be handled in the already raised soft
654 * interrupt.
655 */
656 if (!cpu_base->softirq_activated) {
657 soft = __hrtimer_get_next_event(cpu_base, HRTIMER_ACTIVE_SOFT);
658 /*
659 * Update the soft expiry time. clock_settime() might have
660 * affected it.
661 */
662 cpu_base->softirq_expires_next = soft;
663 }
664
665 expires_next = __hrtimer_get_next_event(cpu_base, HRTIMER_ACTIVE_HARD);
666 /*
667 * If a softirq timer is expiring first, update cpu_base->next_timer
668 * and program the hardware with the soft expiry time.
669 */
670 if (expires_next > soft) {
671 cpu_base->next_timer = cpu_base->softirq_next_timer;
672 expires_next = soft;
673 }
674
675 return expires_next;
676}
677
678static inline ktime_t hrtimer_update_base(struct hrtimer_cpu_base *base)
679{
680 lockdep_assert_held(&base->lock);
681
682 ktime_t *offs_real = &base->clock_base[HRTIMER_BASE_REALTIME].offset;
683 ktime_t *offs_boot = &base->clock_base[HRTIMER_BASE_BOOTTIME].offset;
684 ktime_t *offs_tai = &base->clock_base[HRTIMER_BASE_TAI].offset;
685
686 ktime_t now = ktime_get_update_offsets_now(&base->clock_was_set_seq, offs_real,
687 offs_boot, offs_tai);
688
689 base->clock_base[HRTIMER_BASE_REALTIME_SOFT].offset = *offs_real;
690 base->clock_base[HRTIMER_BASE_BOOTTIME_SOFT].offset = *offs_boot;
691 base->clock_base[HRTIMER_BASE_TAI_SOFT].offset = *offs_tai;
692
693 return now;
694}
695
696/*
697 * Is the high resolution mode active in the CPU base. This cannot use the
698 * static key as the CPUs are switched to high resolution mode
699 * asynchronously.
700 */
701static inline int hrtimer_hres_active(struct hrtimer_cpu_base *cpu_base)
702{
703 return IS_ENABLED(CONFIG_HIGH_RES_TIMERS) ?
704 cpu_base->hres_active : 0;
705}
706
707static inline void hrtimer_rearm_event(ktime_t expires_next, bool deferred)
708{
709 trace_hrtimer_rearm(expires_next, deferred);
710 tick_program_event(expires_next, 1);
711}
712
713static void __hrtimer_reprogram(struct hrtimer_cpu_base *cpu_base,
714 ktime_t expires_next)
715{
716 cpu_base->expires_next = expires_next;
717
718 /*
719 * If hres is not active, hardware does not have to be
720 * reprogrammed yet.
721 *
722 * If a hang was detected in the last timer interrupt then we
723 * leave the hang delay active in the hardware. We want the
724 * system to make progress. That also prevents the following
725 * scenario:
726 * T1 expires 50ms from now
727 * T2 expires 5s from now
728 *
729 * T1 is removed, so this code is called and would reprogram
730 * the hardware to 5s from now. Any hrtimer_start after that
731 * will not reprogram the hardware due to hang_detected being
732 * set. So we'd effectively block all timers until the T2 event
733 * fires.
734 */
735 if (!hrtimer_hres_active(cpu_base) || cpu_base->hang_detected)
736 return;
737
738 hrtimer_rearm_event(expires_next, false);
739}
740
741/* Reprogram the event source with a evaluation of all clock bases */
742static void hrtimer_force_reprogram(struct hrtimer_cpu_base *cpu_base, bool skip_equal)
743{
744 ktime_t expires_next = hrtimer_update_next_event(cpu_base);
745
746 if (skip_equal && expires_next == cpu_base->expires_next)
747 return;
748
749 __hrtimer_reprogram(cpu_base, expires_next);
750}
751
752/* High resolution timer related functions */
753#ifdef CONFIG_HIGH_RES_TIMERS
754
755/* High resolution timer enabled ? */
756static bool hrtimer_hres_enabled __read_mostly = true;
757unsigned int hrtimer_resolution __read_mostly = LOW_RES_NSEC;
758EXPORT_SYMBOL_GPL(hrtimer_resolution);
759
760/* Enable / Disable high resolution mode */
761static int __init setup_hrtimer_hres(char *str)
762{
763 return (kstrtobool(str, &hrtimer_hres_enabled) == 0);
764}
765__setup("highres=", setup_hrtimer_hres);
766
767/* hrtimer_high_res_enabled - query, if the highres mode is enabled */
768static inline bool hrtimer_is_hres_enabled(void)
769{
770 return hrtimer_hres_enabled;
771}
772
773/* Switch to high resolution mode */
774static void hrtimer_switch_to_hres(void)
775{
776 struct hrtimer_cpu_base *base = this_cpu_ptr(&hrtimer_bases);
777
778 if (tick_init_highres()) {
779 pr_warn("Could not switch to high resolution mode on CPU %u\n", base->cpu);
780 return;
781 }
782 base->hres_active = true;
783 hrtimer_resolution = HIGH_RES_NSEC;
784
785 tick_setup_sched_timer(true);
786 /* "Retrigger" the interrupt to get things going */
787 retrigger_next_event(NULL);
788 hrtimer_schedule_hres_work();
789}
790
791#else
792
793static inline bool hrtimer_is_hres_enabled(void) { return 0; }
794static inline void hrtimer_switch_to_hres(void) { }
795
796#endif /* CONFIG_HIGH_RES_TIMERS */
797
798/*
799 * Retrigger next event is called after clock was set with interrupts
800 * disabled through an SMP function call or directly from low level
801 * resume code.
802 *
803 * This is only invoked when:
804 * - CONFIG_HIGH_RES_TIMERS is enabled.
805 * - CONFIG_NO_HZ_COMMON is enabled
806 *
807 * For the other cases this function is empty and because the call sites
808 * are optimized out it vanishes as well, i.e. no need for lots of
809 * #ifdeffery.
810 */
811static void retrigger_next_event(void *arg)
812{
813 struct hrtimer_cpu_base *base = this_cpu_ptr(&hrtimer_bases);
814
815 /*
816 * When high resolution mode or nohz is active, then the offsets of
817 * CLOCK_REALTIME/TAI/BOOTTIME have to be updated. Otherwise the
818 * next tick will take care of that.
819 *
820 * If high resolution mode is active then the next expiring timer
821 * must be reevaluated and the clock event device reprogrammed if
822 * necessary.
823 *
824 * In the NOHZ case the update of the offset and the reevaluation
825 * of the next expiring timer is enough. The return from the SMP
826 * function call will take care of the reprogramming in case the
827 * CPU was in a NOHZ idle sleep.
828 *
829 * In periodic low resolution mode, the next softirq expiration
830 * must also be updated.
831 */
832 guard(raw_spinlock)(&base->lock);
833 hrtimer_update_base(base);
834 if (hrtimer_hres_active(base))
835 hrtimer_force_reprogram(base, /* skip_equal */ false);
836 else
837 hrtimer_update_next_event(base);
838}
839
840/*
841 * When a timer is enqueued and expires earlier than the already enqueued
842 * timers, we have to check, whether it expires earlier than the timer for
843 * which the clock event device was armed.
844 *
845 * Called with interrupts disabled and base->cpu_base.lock held
846 */
847static void hrtimer_reprogram(struct hrtimer *timer, bool reprogram)
848{
849 struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases);
850 struct hrtimer_clock_base *base = timer->base;
851 ktime_t expires = hrtimer_get_expires(timer);
852
853 WARN_ON_ONCE(expires < 0);
854
855 expires = ktime_sub(expires, base->offset);
856 /*
857 * CLOCK_REALTIME timer might be requested with an absolute
858 * expiry time which is less than base->offset. Set it to 0.
859 */
860 if (expires < 0)
861 expires = 0;
862
863 if (timer->is_soft) {
864 /*
865 * soft hrtimer could be started on a remote CPU. In this
866 * case softirq_expires_next needs to be updated on the
867 * remote CPU. The soft hrtimer will not expire before the
868 * first hard hrtimer on the remote CPU -
869 * hrtimer_check_target() prevents this case.
870 */
871 struct hrtimer_cpu_base *timer_cpu_base = base->cpu_base;
872
873 if (timer_cpu_base->softirq_activated)
874 return;
875
876 if (!ktime_before(expires, timer_cpu_base->softirq_expires_next))
877 return;
878
879 timer_cpu_base->softirq_next_timer = timer;
880 timer_cpu_base->softirq_expires_next = expires;
881
882 if (!ktime_before(expires, timer_cpu_base->expires_next) || !reprogram)
883 return;
884 }
885
886 /*
887 * If the timer is not on the current cpu, we cannot reprogram
888 * the other cpus clock event device.
889 */
890 if (base->cpu_base != cpu_base)
891 return;
892
893 if (expires >= cpu_base->expires_next)
894 return;
895
896 /* If a deferred rearm is pending skip reprogramming the device */
897 if (cpu_base->deferred_rearm)
898 return;
899
900 cpu_base->next_timer = timer;
901
902 __hrtimer_reprogram(cpu_base, expires);
903}
904
905static bool update_needs_ipi(struct hrtimer_cpu_base *cpu_base, unsigned int active)
906{
907 struct hrtimer_clock_base *base;
908 ktime_t expires;
909 u32 seq;
910
911 /*
912 * Update the base offsets unconditionally so the following
913 * checks whether the SMP function call is required works.
914 *
915 * The update is safe even when the remote CPU is in the hrtimer
916 * interrupt or the hrtimer soft interrupt and expiring affected
917 * bases. Either it will see the update before handling a base or
918 * it will see it when it finishes the processing and reevaluates
919 * the next expiring timer.
920 */
921 seq = cpu_base->clock_was_set_seq;
922 hrtimer_update_base(cpu_base);
923
924 /*
925 * If the sequence did not change over the update then the
926 * remote CPU already handled it.
927 */
928 if (seq == cpu_base->clock_was_set_seq)
929 return false;
930
931 /* If a deferred rearm is pending the remote CPU will take care of it */
932 if (cpu_base->deferred_rearm) {
933 cpu_base->deferred_needs_update = true;
934 return false;
935 }
936
937 /*
938 * Walk the affected clock bases and check whether the first expiring
939 * timer in a clock base is moving ahead of the first expiring timer of
940 * @cpu_base. If so, the IPI must be invoked because per CPU clock
941 * event devices cannot be remotely reprogrammed.
942 */
943 active &= cpu_base->active_bases;
944
945 for_each_active_base(base, cpu_base, active) {
946 struct timerqueue_linked_node *next;
947
948 next = timerqueue_linked_first(&base->active);
949 expires = ktime_sub(next->expires, base->offset);
950 if (expires < cpu_base->expires_next)
951 return true;
952
953 /* Extra check for softirq clock bases */
954 if (base->index < HRTIMER_BASE_MONOTONIC_SOFT)
955 continue;
956 if (cpu_base->softirq_activated)
957 continue;
958 if (expires < cpu_base->softirq_expires_next)
959 return true;
960 }
961 return false;
962}
963
964/*
965 * Clock was set. This might affect CLOCK_REALTIME, CLOCK_TAI and
966 * CLOCK_BOOTTIME (for late sleep time injection).
967 *
968 * This requires to update the offsets for these clocks
969 * vs. CLOCK_MONOTONIC. When high resolution timers are enabled, then this
970 * also requires to eventually reprogram the per CPU clock event devices
971 * when the change moves an affected timer ahead of the first expiring
972 * timer on that CPU. Obviously remote per CPU clock event devices cannot
973 * be reprogrammed. The other reason why an IPI has to be sent is when the
974 * system is in !HIGH_RES and NOHZ mode. The NOHZ mode updates the offsets
975 * in the tick, which obviously might be stopped, so this has to bring out
976 * the remote CPU which might sleep in idle to get this sorted.
977 */
978void clock_was_set(unsigned int bases)
979{
980 cpumask_var_t mask;
981
982 if (!hrtimer_highres_enabled() && !tick_nohz_is_active())
983 goto out_timerfd;
984
985 if (!zalloc_cpumask_var(&mask, GFP_KERNEL)) {
986 on_each_cpu(retrigger_next_event, NULL, 1);
987 goto out_timerfd;
988 }
989
990 /* Avoid interrupting CPUs if possible */
991 scoped_guard(cpus_read_lock) {
992 int cpu;
993
994 for_each_online_cpu(cpu) {
995 struct hrtimer_cpu_base *cpu_base = &per_cpu(hrtimer_bases, cpu);
996
997 guard(raw_spinlock_irqsave)(&cpu_base->lock);
998 if (update_needs_ipi(cpu_base, bases))
999 cpumask_set_cpu(cpu, mask);
1000 }
1001 scoped_guard(preempt)
1002 smp_call_function_many(mask, retrigger_next_event, NULL, 1);
1003 }
1004 free_cpumask_var(mask);
1005
1006out_timerfd:
1007 timerfd_clock_was_set();
1008}
1009
1010static void clock_was_set_work(struct work_struct *work)
1011{
1012 clock_was_set(CLOCK_SET_WALL);
1013}
1014
1015static DECLARE_WORK(hrtimer_work, clock_was_set_work);
1016
1017/*
1018 * Called from timekeeping code to reprogram the hrtimer interrupt device
1019 * on all cpus and to notify timerfd.
1020 */
1021void clock_was_set_delayed(void)
1022{
1023 schedule_work(&hrtimer_work);
1024}
1025
1026/*
1027 * Called during resume either directly from via timekeeping_resume()
1028 * or in the case of s2idle from tick_unfreeze() to ensure that the
1029 * hrtimers are up to date.
1030 */
1031void hrtimers_resume_local(void)
1032{
1033 lockdep_assert_irqs_disabled();
1034 /* Retrigger on the local CPU */
1035 retrigger_next_event(NULL);
1036}
1037
1038/* Counterpart to lock_hrtimer_base above */
1039static inline void unlock_hrtimer_base(const struct hrtimer *timer, unsigned long *flags)
1040 __releases(&timer->base->cpu_base->lock)
1041{
1042 raw_spin_unlock_irqrestore(&timer->base->cpu_base->lock, *flags);
1043}
1044
1045/**
1046 * hrtimer_update_function - Update the timer's callback function
1047 * @timer: Timer to update
1048 * @function: New callback function
1049 *
1050 * Only safe to call if the timer is not enqueued. Can be called in the callback function if the
1051 * timer is not enqueued at the same time (see the comments above HRTIMER_STATE_ENQUEUED).
1052 */
1053void hrtimer_update_function(struct hrtimer *timer,
1054 enum hrtimer_restart (*function)(struct hrtimer *))
1055{
1056#ifdef CONFIG_PROVE_LOCKING
1057 guard(raw_spinlock_irqsave)(&timer->base->cpu_base->lock);
1058
1059 if (WARN_ON_ONCE(hrtimer_is_queued(timer)))
1060 return;
1061
1062 if (WARN_ON_ONCE(!function))
1063 return;
1064#endif
1065 ACCESS_PRIVATE(timer, function) = function;
1066}
1067EXPORT_SYMBOL_GPL(hrtimer_update_function);
1068
1069/**
1070 * hrtimer_forward() - forward the timer expiry
1071 * @timer: hrtimer to forward
1072 * @now: forward past this time
1073 * @interval: the interval to forward
1074 *
1075 * Forward the timer expiry so it will expire in the future.
1076 *
1077 * .. note::
1078 * This only updates the timer expiry value and does not requeue the timer.
1079 *
1080 * There is also a variant of this function: hrtimer_forward_now().
1081 *
1082 * Context: Can be safely called from the callback function of @timer. If called
1083 * from other contexts @timer must neither be enqueued nor running the
1084 * callback and the caller needs to take care of serialization.
1085 *
1086 * Return: The number of overruns are returned.
1087 */
1088u64 hrtimer_forward(struct hrtimer *timer, ktime_t now, ktime_t interval)
1089{
1090 ktime_t delta;
1091 u64 orun = 1;
1092
1093 delta = ktime_sub(now, hrtimer_get_expires(timer));
1094
1095 if (delta < 0)
1096 return 0;
1097
1098 if (WARN_ON(timer->is_queued))
1099 return 0;
1100
1101 if (interval < hrtimer_resolution)
1102 interval = hrtimer_resolution;
1103
1104 if (unlikely(delta >= interval)) {
1105 s64 incr = ktime_to_ns(interval);
1106
1107 orun = ktime_divns(delta, incr);
1108 hrtimer_add_expires_ns(timer, incr * orun);
1109 if (hrtimer_get_expires(timer) > now)
1110 return orun;
1111 /*
1112 * This (and the ktime_add() below) is the
1113 * correction for exact:
1114 */
1115 orun++;
1116 }
1117 hrtimer_add_expires(timer, interval);
1118
1119 return orun;
1120}
1121EXPORT_SYMBOL_GPL(hrtimer_forward);
1122
1123/*
1124 * enqueue_hrtimer - internal function to (re)start a timer
1125 *
1126 * The timer is inserted in expiry order. Insertion into the
1127 * red black tree is O(log(n)).
1128 *
1129 * Returns true when the new timer is the leftmost timer in the tree.
1130 */
1131static bool enqueue_hrtimer(struct hrtimer *timer, struct hrtimer_clock_base *base,
1132 enum hrtimer_mode mode, bool was_armed)
1133{
1134 lockdep_assert_held(&base->cpu_base->lock);
1135
1136 debug_activate(timer, mode, was_armed);
1137 WARN_ON_ONCE(!base->cpu_base->online);
1138
1139 base->cpu_base->active_bases |= 1 << base->index;
1140
1141 /* Pairs with the lockless read in hrtimer_is_queued() */
1142 WRITE_ONCE(timer->is_queued, HRTIMER_STATE_ENQUEUED);
1143
1144 if (!timerqueue_linked_add(&base->active, &timer->node))
1145 return false;
1146
1147 base->expires_next = hrtimer_get_expires(timer);
1148 return true;
1149}
1150
1151static inline void base_update_next_timer(struct hrtimer_clock_base *base)
1152{
1153 struct timerqueue_linked_node *next = timerqueue_linked_first(&base->active);
1154
1155 base->expires_next = next ? next->expires : KTIME_MAX;
1156}
1157
1158/*
1159 * __remove_hrtimer - internal function to remove a timer
1160 *
1161 * High resolution timer mode reprograms the clock event device when the
1162 * timer is the one which expires next. The caller can disable this by setting
1163 * reprogram to zero. This is useful, when the context does a reprogramming
1164 * anyway (e.g. timer interrupt)
1165 */
1166static void __remove_hrtimer(struct hrtimer *timer, struct hrtimer_clock_base *base,
1167 bool newstate, bool reprogram)
1168{
1169 struct hrtimer_cpu_base *cpu_base = base->cpu_base;
1170 bool was_first;
1171
1172 lockdep_assert_held(&cpu_base->lock);
1173
1174 if (!timer->is_queued)
1175 return;
1176
1177 /* Pairs with the lockless read in hrtimer_is_queued() */
1178 WRITE_ONCE(timer->is_queued, newstate);
1179
1180 was_first = !timerqueue_linked_prev(&timer->node);
1181
1182 if (!timerqueue_linked_del(&base->active, &timer->node))
1183 cpu_base->active_bases &= ~(1 << base->index);
1184
1185 /* Nothing to update if this was not the first timer in the base */
1186 if (!was_first)
1187 return;
1188
1189 base_update_next_timer(base);
1190
1191 /*
1192 * If reprogram is false don't update cpu_base->next_timer and do not
1193 * touch the clock event device.
1194 *
1195 * This happens when removing the first timer on a remote CPU, which
1196 * will be handled by the remote CPU's interrupt. It also happens when
1197 * a local timer is removed to be immediately restarted. That's handled
1198 * at the call site.
1199 */
1200 if (!reprogram || timer != cpu_base->next_timer || timer->is_lazy)
1201 return;
1202
1203 if (cpu_base->deferred_rearm)
1204 cpu_base->deferred_needs_update = true;
1205 else
1206 hrtimer_force_reprogram(cpu_base, /* skip_equal */ true);
1207}
1208
1209static inline bool remove_hrtimer(struct hrtimer *timer, struct hrtimer_clock_base *base,
1210 bool newstate)
1211{
1212 lockdep_assert_held(&base->cpu_base->lock);
1213
1214 if (timer->is_queued) {
1215 bool reprogram;
1216
1217 debug_hrtimer_deactivate(timer);
1218
1219 /*
1220 * Remove the timer and force reprogramming when high
1221 * resolution mode is active and the timer is on the current
1222 * CPU. If we remove a timer on another CPU, reprogramming is
1223 * skipped. The interrupt event on this CPU is fired and
1224 * reprogramming happens in the interrupt handler. This is a
1225 * rare case and less expensive than a smp call.
1226 */
1227 reprogram = base->cpu_base == this_cpu_ptr(&hrtimer_bases);
1228
1229 __remove_hrtimer(timer, base, newstate, reprogram);
1230 return true;
1231 }
1232 return false;
1233}
1234
1235/*
1236 * Update in place has to retrieve the expiry times of the neighbour nodes
1237 * if they exist. That is cache line neutral because the dequeue/enqueue
1238 * operation is going to need the same cache lines. But there is a big win
1239 * when the dequeue/enqueue can be avoided because the RB tree does not
1240 * have to be rebalanced twice.
1241 */
1242static inline bool
1243hrtimer_can_update_in_place(struct hrtimer *timer, struct hrtimer_clock_base *base, ktime_t expires)
1244{
1245 struct timerqueue_linked_node *next = timerqueue_linked_next(&timer->node);
1246 struct timerqueue_linked_node *prev = timerqueue_linked_prev(&timer->node);
1247
1248 /* If the new expiry goes behind the next timer, requeue is required */
1249 if (next && expires > next->expires)
1250 return false;
1251
1252 /* If this is the first timer, update in place */
1253 if (!prev)
1254 return true;
1255
1256 /* Update in place when it does not go ahead of the previous one */
1257 return expires >= prev->expires;
1258}
1259
1260static inline bool
1261remove_and_enqueue_same_base(struct hrtimer *timer, struct hrtimer_clock_base *base,
1262 const enum hrtimer_mode mode, ktime_t expires, u64 delta_ns)
1263{
1264 bool was_first = false;
1265
1266 /* Remove it from the timer queue if active */
1267 if (timer->is_queued) {
1268 was_first = !timerqueue_linked_prev(&timer->node);
1269
1270 /* Try to update in place to avoid the de/enqueue dance */
1271 if (hrtimer_can_update_in_place(timer, base, expires)) {
1272 hrtimer_set_expires_range_ns(timer, expires, delta_ns);
1273 trace_hrtimer_start(timer, mode, true);
1274 if (was_first)
1275 base->expires_next = expires;
1276 return was_first;
1277 }
1278
1279 debug_hrtimer_deactivate(timer);
1280 timerqueue_linked_del(&base->active, &timer->node);
1281 }
1282
1283 /* Set the new expiry time */
1284 hrtimer_set_expires_range_ns(timer, expires, delta_ns);
1285
1286 debug_activate(timer, mode, timer->is_queued);
1287 base->cpu_base->active_bases |= 1 << base->index;
1288
1289 /* Pairs with the lockless read in hrtimer_is_queued() */
1290 WRITE_ONCE(timer->is_queued, HRTIMER_STATE_ENQUEUED);
1291
1292 /* If it's the first expiring timer now or again, update base */
1293 if (timerqueue_linked_add(&base->active, &timer->node)) {
1294 base->expires_next = expires;
1295 return true;
1296 }
1297
1298 if (was_first)
1299 base_update_next_timer(base);
1300
1301 return false;
1302}
1303
1304static inline ktime_t hrtimer_update_lowres(struct hrtimer *timer, ktime_t tim,
1305 const enum hrtimer_mode mode)
1306{
1307#ifdef CONFIG_TIME_LOW_RES
1308 /*
1309 * CONFIG_TIME_LOW_RES indicates that the system has no way to return
1310 * granular time values. For relative timers we add hrtimer_resolution
1311 * (i.e. one jiffy) to prevent short timeouts.
1312 */
1313 timer->is_rel = mode & HRTIMER_MODE_REL;
1314 if (timer->is_rel)
1315 tim = ktime_add_safe(tim, hrtimer_resolution);
1316#endif
1317 return tim;
1318}
1319
1320static void hrtimer_update_softirq_timer(struct hrtimer_cpu_base *cpu_base, bool reprogram)
1321{
1322 ktime_t expires = __hrtimer_get_next_event(cpu_base, HRTIMER_ACTIVE_SOFT);
1323
1324 /*
1325 * Reprogramming needs to be triggered, even if the next soft
1326 * hrtimer expires at the same time as the next hard
1327 * hrtimer. cpu_base->softirq_expires_next needs to be updated!
1328 */
1329 if (expires == KTIME_MAX)
1330 return;
1331
1332 /*
1333 * cpu_base->next_timer is recomputed by __hrtimer_get_next_event()
1334 * cpu_base->expires_next is only set by hrtimer_reprogram()
1335 */
1336 hrtimer_reprogram(cpu_base->softirq_next_timer, reprogram);
1337}
1338
1339#if defined(CONFIG_SMP) && defined(CONFIG_NO_HZ_COMMON)
1340static __always_inline bool hrtimer_prefer_local(bool is_local, bool is_first, bool is_pinned)
1341{
1342 if (static_branch_likely(&timers_migration_enabled)) {
1343 /*
1344 * If it is local and the first expiring timer keep it on the local
1345 * CPU to optimize reprogramming of the clockevent device. Also
1346 * avoid switch_hrtimer_base() overhead when local and pinned.
1347 */
1348 if (!is_local)
1349 return false;
1350 if (is_first || is_pinned)
1351 return true;
1352
1353 /* Honour the NOHZ full restrictions */
1354 if (!housekeeping_cpu(smp_processor_id(), HK_TYPE_KERNEL_NOISE))
1355 return false;
1356
1357 /*
1358 * If the tick is not stopped or need_resched() is set, then
1359 * there is no point in moving the timer somewhere else.
1360 */
1361 return !tick_nohz_tick_stopped() || need_resched();
1362 }
1363 return is_local;
1364}
1365#else
1366static __always_inline bool hrtimer_prefer_local(bool is_local, bool is_first, bool is_pinned)
1367{
1368 return is_local;
1369}
1370#endif
1371
1372static inline bool hrtimer_keep_base(struct hrtimer *timer, bool is_local, bool is_first,
1373 bool is_pinned)
1374{
1375 /* If the timer is running the callback it has to stay on its CPU base. */
1376 if (unlikely(timer->base->running == timer))
1377 return true;
1378
1379 return hrtimer_prefer_local(is_local, is_first, is_pinned);
1380}
1381
1382enum {
1383 HRTIMER_REPROGRAM_NONE,
1384 HRTIMER_REPROGRAM,
1385 HRTIMER_REPROGRAM_FORCE,
1386};
1387
1388static int __hrtimer_start_range_ns(struct hrtimer *timer, ktime_t tim, u64 delta_ns,
1389 const enum hrtimer_mode mode, struct hrtimer_clock_base *base)
1390{
1391 struct hrtimer_cpu_base *this_cpu_base = this_cpu_ptr(&hrtimer_bases);
1392 bool is_pinned, first, was_first, keep_base = false;
1393 struct hrtimer_cpu_base *cpu_base = base->cpu_base;
1394
1395 was_first = cpu_base->next_timer == timer;
1396 is_pinned = !!(mode & HRTIMER_MODE_PINNED);
1397
1398 /*
1399 * Don't keep it local if this enqueue happens on a unplugged CPU
1400 * after hrtimer_cpu_dying() has been invoked.
1401 */
1402 if (likely(this_cpu_base->online)) {
1403 bool is_local = cpu_base == this_cpu_base;
1404
1405 keep_base = hrtimer_keep_base(timer, is_local, was_first, is_pinned);
1406 }
1407
1408 /* Calculate absolute expiry time for relative timers */
1409 if (mode & HRTIMER_MODE_REL)
1410 tim = ktime_add_safe(tim, __hrtimer_cb_get_time(base->clockid));
1411 /* Compensate for low resolution granularity */
1412 tim = hrtimer_update_lowres(timer, tim, mode);
1413
1414 /*
1415 * Remove an active timer from the queue. In case it is not queued
1416 * on the current CPU, make sure that remove_hrtimer() updates the
1417 * remote data correctly.
1418 *
1419 * If it's on the current CPU and the first expiring timer, then
1420 * skip reprogramming, keep the timer local and enforce
1421 * reprogramming later if it was the first expiring timer. This
1422 * avoids programming the underlying clock event twice (once at
1423 * removal and once after enqueue).
1424 *
1425 * @keep_base is also true if the timer callback is running on a
1426 * remote CPU and for local pinned timers.
1427 */
1428 if (likely(keep_base)) {
1429 first = remove_and_enqueue_same_base(timer, base, mode, tim, delta_ns);
1430 } else {
1431 /* Keep the ENQUEUED state in case it is queued */
1432 bool was_armed = remove_hrtimer(timer, base, HRTIMER_STATE_ENQUEUED);
1433
1434 hrtimer_set_expires_range_ns(timer, tim, delta_ns);
1435
1436 /* Switch the timer base, if necessary: */
1437 base = switch_hrtimer_base(timer, base, is_pinned);
1438 cpu_base = base->cpu_base;
1439
1440 first = enqueue_hrtimer(timer, base, mode, was_armed);
1441 }
1442
1443 /* If a deferred rearm is pending skip reprogramming the device */
1444 if (cpu_base->deferred_rearm) {
1445 cpu_base->deferred_needs_update = true;
1446 return HRTIMER_REPROGRAM_NONE;
1447 }
1448
1449 if (!was_first || cpu_base != this_cpu_base) {
1450 /*
1451 * If the current CPU base is online, then the timer is never
1452 * queued on a remote CPU if it would be the first expiring
1453 * timer there unless the timer callback is currently executed
1454 * on the remote CPU. In the latter case the remote CPU will
1455 * re-evaluate the first expiring timer after completing the
1456 * callbacks.
1457 */
1458 if (likely(hrtimer_base_is_online(this_cpu_base)))
1459 return first ? HRTIMER_REPROGRAM : HRTIMER_REPROGRAM_NONE;
1460
1461 /*
1462 * Timer was enqueued remote because the current base is
1463 * already offline. If the timer is the first to expire,
1464 * kick the remote CPU to reprogram the clock event.
1465 */
1466 if (first)
1467 smp_call_function_single_async(cpu_base->cpu, &cpu_base->csd);
1468 return HRTIMER_REPROGRAM_NONE;
1469 }
1470
1471 /*
1472 * Special case for the HRTICK timer. It is frequently rearmed and most
1473 * of the time moves the expiry into the future. That's expensive in
1474 * virtual machines and it's better to take the pointless already armed
1475 * interrupt than reprogramming the hardware on every context switch.
1476 *
1477 * If the new expiry is before the armed time, then reprogramming is
1478 * required.
1479 */
1480 if (timer->is_lazy) {
1481 if (cpu_base->expires_next <= hrtimer_get_expires(timer))
1482 return HRTIMER_REPROGRAM_NONE;
1483 }
1484
1485 /*
1486 * Timer was the first expiring timer and forced to stay on the
1487 * current CPU to avoid reprogramming on removal and enqueue. Force
1488 * reprogram the hardware by evaluating the new first expiring
1489 * timer.
1490 */
1491 return HRTIMER_REPROGRAM_FORCE;
1492}
1493
1494static int hrtimer_start_range_ns_common(struct hrtimer *timer, ktime_t tim,
1495 u64 delta_ns, const enum hrtimer_mode mode,
1496 struct hrtimer_clock_base *base)
1497{
1498 /*
1499 * Check whether the HRTIMER_MODE_SOFT bit and hrtimer.is_soft
1500 * match on CONFIG_PREEMPT_RT = n. With PREEMPT_RT check the hard
1501 * expiry mode because unmarked timers are moved to softirq expiry.
1502 */
1503 if (!IS_ENABLED(CONFIG_PREEMPT_RT))
1504 WARN_ON_ONCE(!(mode & HRTIMER_MODE_SOFT) ^ !timer->is_soft);
1505 else
1506 WARN_ON_ONCE(!(mode & HRTIMER_MODE_HARD) ^ !timer->is_hard);
1507
1508 return __hrtimer_start_range_ns(timer, tim, delta_ns, mode, base);
1509}
1510
1511/**
1512 * hrtimer_start_range_ns - (re)start an hrtimer
1513 * @timer: the timer to be added
1514 * @tim: expiry time
1515 * @delta_ns: "slack" range for the timer
1516 * @mode: timer mode: absolute (HRTIMER_MODE_ABS) or
1517 * relative (HRTIMER_MODE_REL), and pinned (HRTIMER_MODE_PINNED);
1518 * softirq based mode is considered for debug purpose only!
1519 */
1520void hrtimer_start_range_ns(struct hrtimer *timer, ktime_t tim, u64 delta_ns,
1521 const enum hrtimer_mode mode)
1522{
1523 struct hrtimer_clock_base *base;
1524 unsigned long flags;
1525
1526 debug_hrtimer_assert_init(timer);
1527
1528 base = lock_hrtimer_base(timer, &flags);
1529
1530 switch (hrtimer_start_range_ns_common(timer, tim, delta_ns, mode, base)) {
1531 case HRTIMER_REPROGRAM:
1532 hrtimer_reprogram(timer, true);
1533 break;
1534 case HRTIMER_REPROGRAM_FORCE:
1535 hrtimer_force_reprogram(timer->base->cpu_base, 1);
1536 break;
1537 case HRTIMER_REPROGRAM_NONE:
1538 break;
1539 }
1540
1541 unlock_hrtimer_base(timer, &flags);
1542}
1543EXPORT_SYMBOL_GPL(hrtimer_start_range_ns);
1544
1545static inline bool hrtimer_check_user_timer(struct hrtimer *timer)
1546{
1547 struct hrtimer_cpu_base *cpu_base = timer->base->cpu_base;
1548 ktime_t expires;
1549
1550 /*
1551 * This uses soft expires because that's the user provided
1552 * expiry time, while expires can be further in the past
1553 * due to a slack value added to the user expiry time.
1554 */
1555 expires = hrtimer_get_softexpires(timer);
1556
1557 /* Convert to monotonic */
1558 expires = ktime_sub(expires, timer->base->offset);
1559
1560 /*
1561 * Check whether this timer will end up as the first expiring timer in
1562 * the CPU base. If not, no further checks required as it's then
1563 * guaranteed to expire in the future.
1564 */
1565 if (expires >= cpu_base->expires_next)
1566 return true;
1567
1568 /* Validate that the expiry time is in the future. */
1569 if (expires > ktime_get())
1570 return true;
1571
1572 debug_hrtimer_deactivate(timer);
1573 __remove_hrtimer(timer, timer->base, HRTIMER_STATE_INACTIVE, false);
1574 trace_hrtimer_start_expired(timer);
1575 return false;
1576}
1577
1578/**
1579 * hrtimer_start_range_ns_user - (re)start an user controlled hrtimer
1580 * @timer: the timer to be added
1581 * @tim: expiry time
1582 * @delta_ns: "slack" range for the timer
1583 * @mode: timer mode: absolute (HRTIMER_MODE_ABS) or
1584 * relative (HRTIMER_MODE_REL), and pinned (HRTIMER_MODE_PINNED);
1585 * softirq based mode is considered for debug purpose only!
1586 *
1587 * Returns: True when the timer was queued, false if it was already expired
1588 *
1589 * This function cannot invoke the timer callback for expired timers as it might
1590 * be called under a lock which the timer callback needs to acquire. So the
1591 * caller has to handle that case.
1592 */
1593bool hrtimer_start_range_ns_user(struct hrtimer *timer, ktime_t tim,
1594 u64 delta_ns, const enum hrtimer_mode mode)
1595{
1596 struct hrtimer_clock_base *base;
1597 unsigned long flags;
1598 bool ret = true;
1599
1600 debug_hrtimer_assert_init(timer);
1601
1602 base = lock_hrtimer_base(timer, &flags);
1603
1604 switch (hrtimer_start_range_ns_common(timer, tim, delta_ns, mode, base)) {
1605 case HRTIMER_REPROGRAM:
1606 ret = hrtimer_check_user_timer(timer);
1607 if (ret)
1608 hrtimer_reprogram(timer, true);
1609 break;
1610 case HRTIMER_REPROGRAM_FORCE:
1611 ret = hrtimer_check_user_timer(timer);
1612 /*
1613 * The base must always be reevaluated, independent of the
1614 * result above because the timer was the first pending timer.
1615 */
1616 hrtimer_force_reprogram(timer->base->cpu_base, 1);
1617 break;
1618 case HRTIMER_REPROGRAM_NONE:
1619 break;
1620 }
1621
1622 unlock_hrtimer_base(timer, &flags);
1623 return ret;
1624}
1625EXPORT_SYMBOL_GPL(hrtimer_start_range_ns_user);
1626
1627/**
1628 * hrtimer_try_to_cancel - try to deactivate a timer
1629 * @timer: hrtimer to stop
1630 *
1631 * Returns:
1632 *
1633 * * 0 when the timer was not active
1634 * * 1 when the timer was active
1635 * * -1 when the timer is currently executing the callback function and
1636 * cannot be stopped
1637 */
1638int hrtimer_try_to_cancel(struct hrtimer *timer)
1639{
1640 struct hrtimer_clock_base *base;
1641 unsigned long flags;
1642 int ret = -1;
1643
1644 /*
1645 * Check lockless first. If the timer is not active (neither
1646 * enqueued nor running the callback, nothing to do here. The
1647 * base lock does not serialize against a concurrent enqueue,
1648 * so we can avoid taking it.
1649 */
1650 if (!hrtimer_active(timer))
1651 return 0;
1652
1653 base = lock_hrtimer_base(timer, &flags);
1654
1655 if (!hrtimer_callback_running(timer)) {
1656 ret = remove_hrtimer(timer, base, HRTIMER_STATE_INACTIVE);
1657 if (ret)
1658 trace_hrtimer_cancel(timer);
1659 }
1660
1661 unlock_hrtimer_base(timer, &flags);
1662
1663 return ret;
1664
1665}
1666EXPORT_SYMBOL_GPL(hrtimer_try_to_cancel);
1667
1668#ifdef CONFIG_PREEMPT_RT
1669static void hrtimer_cpu_base_init_expiry_lock(struct hrtimer_cpu_base *base)
1670{
1671 spin_lock_init(&base->softirq_expiry_lock);
1672}
1673
1674static void hrtimer_cpu_base_lock_expiry(struct hrtimer_cpu_base *base)
1675 __acquires(&base->softirq_expiry_lock)
1676{
1677 spin_lock(&base->softirq_expiry_lock);
1678}
1679
1680static void hrtimer_cpu_base_unlock_expiry(struct hrtimer_cpu_base *base)
1681 __releases(&base->softirq_expiry_lock)
1682{
1683 spin_unlock(&base->softirq_expiry_lock);
1684}
1685
1686/*
1687 * The counterpart to hrtimer_cancel_wait_running().
1688 *
1689 * If there is a waiter for cpu_base->expiry_lock, then it was waiting for
1690 * the timer callback to finish. Drop expiry_lock and reacquire it. That
1691 * allows the waiter to acquire the lock and make progress.
1692 */
1693static void hrtimer_sync_wait_running(struct hrtimer_cpu_base *cpu_base, unsigned long flags)
1694{
1695 if (atomic_read(&cpu_base->timer_waiters)) {
1696 raw_spin_unlock_irqrestore(&cpu_base->lock, flags);
1697 spin_unlock(&cpu_base->softirq_expiry_lock);
1698 spin_lock(&cpu_base->softirq_expiry_lock);
1699 raw_spin_lock_irq(&cpu_base->lock);
1700 }
1701}
1702
1703#ifdef CONFIG_SMP
1704static __always_inline bool is_migration_base(struct hrtimer_clock_base *base)
1705{
1706 return base == &migration_base;
1707}
1708#else
1709static __always_inline bool is_migration_base(struct hrtimer_clock_base *base)
1710{
1711 return false;
1712}
1713#endif
1714
1715/*
1716 * This function is called on PREEMPT_RT kernels when the fast path
1717 * deletion of a timer failed because the timer callback function was
1718 * running.
1719 *
1720 * This prevents priority inversion: if the soft irq thread is preempted
1721 * in the middle of a timer callback, then calling hrtimer_cancel() can
1722 * lead to two issues:
1723 *
1724 * - If the caller is on a remote CPU then it has to spin wait for the timer
1725 * handler to complete. This can result in unbound priority inversion.
1726 *
1727 * - If the caller originates from the task which preempted the timer
1728 * handler on the same CPU, then spin waiting for the timer handler to
1729 * complete is never going to end.
1730 */
1731void hrtimer_cancel_wait_running(const struct hrtimer *timer)
1732{
1733 /* Lockless read. Prevent the compiler from reloading it below */
1734 struct hrtimer_clock_base *base = READ_ONCE(timer->base);
1735
1736 /*
1737 * Just relax if the timer expires in hard interrupt context or if
1738 * it is currently on the migration base.
1739 */
1740 if (!timer->is_soft || is_migration_base(base)) {
1741 cpu_relax();
1742 return;
1743 }
1744
1745 /*
1746 * Mark the base as contended and grab the expiry lock, which is
1747 * held by the softirq across the timer callback. Drop the lock
1748 * immediately so the softirq can expire the next timer. In theory
1749 * the timer could already be running again, but that's more than
1750 * unlikely and just causes another wait loop.
1751 */
1752 atomic_inc(&base->cpu_base->timer_waiters);
1753 spin_lock_bh(&base->cpu_base->softirq_expiry_lock);
1754 atomic_dec(&base->cpu_base->timer_waiters);
1755 spin_unlock_bh(&base->cpu_base->softirq_expiry_lock);
1756}
1757#else
1758static inline void hrtimer_cpu_base_init_expiry_lock(struct hrtimer_cpu_base *base) { }
1759static inline void hrtimer_cpu_base_lock_expiry(struct hrtimer_cpu_base *base) { }
1760static inline void hrtimer_cpu_base_unlock_expiry(struct hrtimer_cpu_base *base) { }
1761static inline void hrtimer_sync_wait_running(struct hrtimer_cpu_base *base, unsigned long fl) { }
1762#endif
1763
1764/**
1765 * hrtimer_cancel - cancel a timer and wait for the handler to finish.
1766 * @timer: the timer to be cancelled
1767 *
1768 * Returns:
1769 * 0 when the timer was not active
1770 * 1 when the timer was active
1771 */
1772int hrtimer_cancel(struct hrtimer *timer)
1773{
1774 int ret;
1775
1776 do {
1777 ret = hrtimer_try_to_cancel(timer);
1778
1779 if (ret < 0)
1780 hrtimer_cancel_wait_running(timer);
1781 } while (ret < 0);
1782 return ret;
1783}
1784EXPORT_SYMBOL_GPL(hrtimer_cancel);
1785
1786/**
1787 * __hrtimer_get_remaining - get remaining time for the timer
1788 * @timer: the timer to read
1789 * @adjust: adjust relative timers when CONFIG_TIME_LOW_RES=y
1790 */
1791ktime_t __hrtimer_get_remaining(const struct hrtimer *timer, bool adjust)
1792{
1793 unsigned long flags;
1794 ktime_t rem;
1795
1796 lock_hrtimer_base(timer, &flags);
1797 if (IS_ENABLED(CONFIG_TIME_LOW_RES) && adjust)
1798 rem = hrtimer_expires_remaining_adjusted(timer);
1799 else
1800 rem = hrtimer_expires_remaining(timer);
1801 unlock_hrtimer_base(timer, &flags);
1802
1803 return rem;
1804}
1805EXPORT_SYMBOL_GPL(__hrtimer_get_remaining);
1806
1807#ifdef CONFIG_NO_HZ_COMMON
1808/**
1809 * hrtimer_get_next_event - get the time until next expiry event
1810 *
1811 * Returns the next expiry time or KTIME_MAX if no timer is pending.
1812 */
1813ktime_t hrtimer_get_next_event(void)
1814{
1815 struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases);
1816
1817 /*
1818 * When HRES is active cmp_next_hrtimer_event() expects KTIME_MAX.
1819 *
1820 * cpu_base->hres_active is written only by the local CPU in
1821 * hrtimer_switch_to_hres() from hard interrupt context and in
1822 * hrtimers_cpu_starting() during CPU bring-up, and all callers reach
1823 * this with interrupts disabled on the same CPU, so an unlocked read is
1824 * stable without holding the lock.
1825 */
1826 if (hrtimer_hres_active(cpu_base))
1827 return KTIME_MAX;
1828
1829 guard(raw_spinlock_irqsave)(&cpu_base->lock);
1830 return __hrtimer_get_next_event(cpu_base, HRTIMER_ACTIVE_ALL);
1831}
1832
1833/**
1834 * hrtimer_next_event_without - time until next expiry event w/o one timer
1835 * @exclude: timer to exclude
1836 *
1837 * Returns the next expiry time over all timers except for the @exclude one or
1838 * KTIME_MAX if none of them is pending.
1839 */
1840ktime_t hrtimer_next_event_without(const struct hrtimer *exclude)
1841{
1842 struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases);
1843 ktime_t expires = KTIME_MAX;
1844 unsigned int active;
1845
1846 guard(raw_spinlock_irqsave)(&cpu_base->lock);
1847 if (!hrtimer_hres_active(cpu_base))
1848 return expires;
1849
1850 active = cpu_base->active_bases & HRTIMER_ACTIVE_SOFT;
1851 if (active && !cpu_base->softirq_activated)
1852 expires = hrtimer_bases_next_event_without(cpu_base, exclude, active, KTIME_MAX);
1853
1854 active = cpu_base->active_bases & HRTIMER_ACTIVE_HARD;
1855 if (!active)
1856 return expires;
1857 return hrtimer_bases_next_event_without(cpu_base, exclude, active, expires);
1858}
1859#endif
1860
1861static inline int hrtimer_clockid_to_base(clockid_t clock_id)
1862{
1863 switch (clock_id) {
1864 case CLOCK_MONOTONIC:
1865 return HRTIMER_BASE_MONOTONIC;
1866 case CLOCK_REALTIME:
1867 return HRTIMER_BASE_REALTIME;
1868 case CLOCK_BOOTTIME:
1869 return HRTIMER_BASE_BOOTTIME;
1870 case CLOCK_TAI:
1871 return HRTIMER_BASE_TAI;
1872 default:
1873 WARN(1, "Invalid clockid %d. Using MONOTONIC\n", clock_id);
1874 return HRTIMER_BASE_MONOTONIC;
1875 }
1876}
1877
1878static ktime_t __hrtimer_cb_get_time(clockid_t clock_id)
1879{
1880 switch (clock_id) {
1881 case CLOCK_MONOTONIC:
1882 return ktime_get();
1883 case CLOCK_REALTIME:
1884 return ktime_get_real();
1885 case CLOCK_BOOTTIME:
1886 return ktime_get_boottime();
1887 case CLOCK_TAI:
1888 return ktime_get_clocktai();
1889 default:
1890 WARN(1, "Invalid clockid %d. Using MONOTONIC\n", clock_id);
1891 return ktime_get();
1892 }
1893}
1894
1895ktime_t hrtimer_cb_get_time(const struct hrtimer *timer)
1896{
1897 return __hrtimer_cb_get_time(timer->base->clockid);
1898}
1899EXPORT_SYMBOL_GPL(hrtimer_cb_get_time);
1900
1901static void __hrtimer_setup(struct hrtimer *timer, enum hrtimer_restart (*fn)(struct hrtimer *),
1902 clockid_t clock_id, enum hrtimer_mode mode)
1903{
1904 bool softtimer = !!(mode & HRTIMER_MODE_SOFT);
1905 struct hrtimer_cpu_base *cpu_base;
1906 int base;
1907
1908 /*
1909 * On PREEMPT_RT enabled kernels hrtimers which are not explicitly
1910 * marked for hard interrupt expiry mode are moved into soft
1911 * interrupt context for latency reasons and because the callbacks
1912 * can invoke functions which might sleep on RT, e.g. spin_lock().
1913 */
1914 if (IS_ENABLED(CONFIG_PREEMPT_RT) && !(mode & HRTIMER_MODE_HARD))
1915 softtimer = true;
1916
1917 memset(timer, 0, sizeof(struct hrtimer));
1918
1919 cpu_base = raw_cpu_ptr(&hrtimer_bases);
1920
1921 /*
1922 * POSIX magic: Relative CLOCK_REALTIME timers are not affected by
1923 * clock modifications, so they needs to become CLOCK_MONOTONIC to
1924 * ensure POSIX compliance.
1925 */
1926 if (clock_id == CLOCK_REALTIME && mode & HRTIMER_MODE_REL)
1927 clock_id = CLOCK_MONOTONIC;
1928
1929 base = softtimer ? HRTIMER_MAX_CLOCK_BASES / 2 : 0;
1930 base += hrtimer_clockid_to_base(clock_id);
1931 timer->is_soft = softtimer;
1932 timer->is_hard = !!(mode & HRTIMER_MODE_HARD);
1933 timer->is_lazy = !!(mode & HRTIMER_MODE_LAZY_REARM);
1934 timer->base = &cpu_base->clock_base[base];
1935 timerqueue_linked_init(&timer->node);
1936
1937 if (WARN_ON_ONCE(!fn))
1938 ACCESS_PRIVATE(timer, function) = hrtimer_dummy_timeout;
1939 else
1940 ACCESS_PRIVATE(timer, function) = fn;
1941}
1942
1943/**
1944 * hrtimer_setup - initialize a timer to the given clock
1945 * @timer: the timer to be initialized
1946 * @function: the callback function
1947 * @clock_id: the clock to be used
1948 * @mode: The modes which are relevant for initialization:
1949 * HRTIMER_MODE_ABS, HRTIMER_MODE_REL, HRTIMER_MODE_ABS_SOFT,
1950 * HRTIMER_MODE_REL_SOFT
1951 *
1952 * The PINNED variants of the above can be handed in,
1953 * but the PINNED bit is ignored as pinning happens
1954 * when the hrtimer is started
1955 */
1956void hrtimer_setup(struct hrtimer *timer, enum hrtimer_restart (*function)(struct hrtimer *),
1957 clockid_t clock_id, enum hrtimer_mode mode)
1958{
1959 debug_setup(timer, clock_id, mode);
1960 __hrtimer_setup(timer, function, clock_id, mode);
1961}
1962EXPORT_SYMBOL_GPL(hrtimer_setup);
1963
1964/**
1965 * hrtimer_setup_on_stack - initialize a timer on stack memory
1966 * @timer: The timer to be initialized
1967 * @function: the callback function
1968 * @clock_id: The clock to be used
1969 * @mode: The timer mode
1970 *
1971 * Similar to hrtimer_setup(), except that this one must be used if struct hrtimer is in stack
1972 * memory.
1973 */
1974void hrtimer_setup_on_stack(struct hrtimer *timer,
1975 enum hrtimer_restart (*function)(struct hrtimer *),
1976 clockid_t clock_id, enum hrtimer_mode mode)
1977{
1978 debug_setup_on_stack(timer, clock_id, mode);
1979 __hrtimer_setup(timer, function, clock_id, mode);
1980}
1981EXPORT_SYMBOL_GPL(hrtimer_setup_on_stack);
1982
1983/*
1984 * A timer is active, when it is enqueued into the rbtree or the
1985 * callback function is running or it's in the state of being migrated
1986 * to another cpu.
1987 *
1988 * It is important for this function to not return a false negative.
1989 */
1990bool hrtimer_active(const struct hrtimer *timer)
1991{
1992 struct hrtimer_clock_base *base;
1993 unsigned int seq;
1994
1995 do {
1996 base = READ_ONCE(timer->base);
1997 seq = raw_read_seqcount_begin(&base->seq);
1998
1999 if (timer->is_queued || base->running == timer)
2000 return true;
2001
2002 } while (read_seqcount_retry(&base->seq, seq) || base != READ_ONCE(timer->base));
2003
2004 return false;
2005}
2006EXPORT_SYMBOL_GPL(hrtimer_active);
2007
2008/*
2009 * The write_seqcount_barrier()s in __run_hrtimer() split the thing into 3
2010 * distinct sections:
2011 *
2012 * - queued: the timer is queued
2013 * - callback: the timer is being ran
2014 * - post: the timer is inactive or (re)queued
2015 *
2016 * On the read side we ensure we observe timer->is_queued and cpu_base->running
2017 * from the same section, if anything changed while we looked at it, we retry.
2018 * This includes timer->base changing because sequence numbers alone are
2019 * insufficient for that.
2020 *
2021 * The sequence numbers are required because otherwise we could still observe
2022 * a false negative if the read side got smeared over multiple consecutive
2023 * __run_hrtimer() invocations.
2024 */
2025static void __run_hrtimer(struct hrtimer_cpu_base *cpu_base, struct hrtimer_clock_base *base,
2026 struct hrtimer *timer, ktime_t now, unsigned long flags)
2027 __must_hold(&cpu_base->lock)
2028{
2029 enum hrtimer_restart (*fn)(struct hrtimer *);
2030 bool expires_in_hardirq;
2031 int restart;
2032
2033 lockdep_assert_held(&cpu_base->lock);
2034
2035 debug_hrtimer_deactivate(timer);
2036 base->running = timer;
2037
2038 /*
2039 * Separate the ->running assignment from the ->is_queued assignment.
2040 *
2041 * As with a regular write barrier, this ensures the read side in
2042 * hrtimer_active() cannot observe base->running == NULL &&
2043 * timer->is_queued == INACTIVE.
2044 */
2045 raw_write_seqcount_barrier(&base->seq);
2046
2047 __remove_hrtimer(timer, base, HRTIMER_STATE_INACTIVE, false);
2048 fn = ACCESS_PRIVATE(timer, function);
2049
2050 /*
2051 * Clear the 'is relative' flag for the TIME_LOW_RES case. If the
2052 * timer is restarted with a period then it becomes an absolute
2053 * timer. If its not restarted it does not matter.
2054 */
2055 if (IS_ENABLED(CONFIG_TIME_LOW_RES))
2056 timer->is_rel = false;
2057
2058 /*
2059 * The timer is marked as running in the CPU base, so it is
2060 * protected against migration to a different CPU even if the lock
2061 * is dropped.
2062 */
2063 raw_spin_unlock_irqrestore(&cpu_base->lock, flags);
2064 trace_hrtimer_expire_entry(timer, now);
2065 expires_in_hardirq = lockdep_hrtimer_enter(timer);
2066
2067 restart = fn(timer);
2068
2069 lockdep_hrtimer_exit(expires_in_hardirq);
2070 trace_hrtimer_expire_exit(timer);
2071 raw_spin_lock_irq(&cpu_base->lock);
2072
2073 /*
2074 * Note: We clear the running state after enqueue_hrtimer and
2075 * we do not reprogram the event hardware. Happens either in
2076 * hrtimer_start_range_ns() or in hrtimer_interrupt()
2077 *
2078 * Note: Because we dropped the cpu_base->lock above,
2079 * hrtimer_start_range_ns() can have popped in and enqueued the timer
2080 * for us already.
2081 */
2082 if (restart == HRTIMER_RESTART && !timer->is_queued)
2083 enqueue_hrtimer(timer, base, HRTIMER_MODE_ABS, false);
2084
2085 /*
2086 * Separate the ->running assignment from the ->is_queued assignment.
2087 *
2088 * As with a regular write barrier, this ensures the read side in
2089 * hrtimer_active() cannot observe base->running.timer == NULL &&
2090 * timer->is_queued == INACTIVE.
2091 */
2092 raw_write_seqcount_barrier(&base->seq);
2093
2094 WARN_ON_ONCE(base->running != timer);
2095 base->running = NULL;
2096}
2097
2098static void __hrtimer_run_queues(struct hrtimer_cpu_base *cpu_base, ktime_t now,
2099 unsigned long flags, unsigned int active_mask)
2100{
2101 unsigned int active = cpu_base->active_bases & active_mask;
2102 struct hrtimer_clock_base *base;
2103
2104 for_each_active_base(base, cpu_base, active) {
2105 ktime_t basenow = ktime_add(now, base->offset);
2106 struct hrtimer *timer;
2107
2108 while ((timer = clock_base_next_timer(base))) {
2109 /*
2110 * The immediate goal for using the softexpires is
2111 * minimizing wakeups, not running timers at the
2112 * earliest interrupt after their soft expiration.
2113 * This allows us to avoid using a Priority Search
2114 * Tree, which can answer a stabbing query for
2115 * overlapping intervals and instead use the simple
2116 * BST we already have.
2117 * We don't add extra wakeups by delaying timers that
2118 * are right-of a not yet expired timer, because that
2119 * timer will have to trigger a wakeup anyway.
2120 */
2121 if (basenow < hrtimer_get_softexpires(timer))
2122 break;
2123
2124 __run_hrtimer(cpu_base, base, timer, basenow, flags);
2125 if (active_mask == HRTIMER_ACTIVE_SOFT)
2126 hrtimer_sync_wait_running(cpu_base, flags);
2127 }
2128 }
2129}
2130
2131static __latent_entropy void hrtimer_run_softirq(void)
2132{
2133 struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases);
2134 unsigned long flags;
2135 ktime_t now;
2136
2137 hrtimer_cpu_base_lock_expiry(cpu_base);
2138 raw_spin_lock_irqsave(&cpu_base->lock, flags);
2139
2140 now = hrtimer_update_base(cpu_base);
2141 __hrtimer_run_queues(cpu_base, now, flags, HRTIMER_ACTIVE_SOFT);
2142
2143 cpu_base->softirq_activated = false;
2144 hrtimer_update_softirq_timer(cpu_base, true);
2145
2146 raw_spin_unlock_irqrestore(&cpu_base->lock, flags);
2147 hrtimer_cpu_base_unlock_expiry(cpu_base);
2148}
2149
2150#ifdef CONFIG_HIGH_RES_TIMERS
2151
2152/*
2153 * Very similar to hrtimer_force_reprogram(), except it deals with
2154 * deferred_rearm and hang_detected.
2155 */
2156static void hrtimer_rearm(struct hrtimer_cpu_base *cpu_base, ktime_t expires_next, bool deferred)
2157{
2158 cpu_base->expires_next = expires_next;
2159 cpu_base->deferred_rearm = false;
2160
2161 if (unlikely(cpu_base->hang_detected)) {
2162 /*
2163 * Give the system a chance to do something else than looping
2164 * on hrtimer interrupts.
2165 */
2166 expires_next = ktime_add_ns(ktime_get(),
2167 min(100 * NSEC_PER_MSEC, cpu_base->max_hang_time));
2168 }
2169 hrtimer_rearm_event(expires_next, deferred);
2170}
2171
2172#ifdef CONFIG_HRTIMER_REARM_DEFERRED
2173void __hrtimer_rearm_deferred(void)
2174{
2175 struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases);
2176 ktime_t expires_next;
2177
2178 if (!cpu_base->deferred_rearm)
2179 return;
2180
2181 guard(raw_spinlock)(&cpu_base->lock);
2182 if (cpu_base->deferred_needs_update) {
2183 hrtimer_update_base(cpu_base);
2184 expires_next = hrtimer_update_next_event(cpu_base);
2185 } else {
2186 /* No timer added/removed. Use the cached value */
2187 expires_next = cpu_base->deferred_expires_next;
2188 }
2189 hrtimer_rearm(cpu_base, expires_next, true);
2190}
2191
2192static __always_inline void
2193hrtimer_interrupt_rearm(struct hrtimer_cpu_base *cpu_base, ktime_t expires_next)
2194{
2195 /* hrtimer_interrupt() just re-evaluated the first expiring timer */
2196 cpu_base->deferred_needs_update = false;
2197 /* Cache the expiry time */
2198 cpu_base->deferred_expires_next = expires_next;
2199 set_thread_flag(TIF_HRTIMER_REARM);
2200}
2201#else /* CONFIG_HRTIMER_REARM_DEFERRED */
2202static __always_inline void
2203hrtimer_interrupt_rearm(struct hrtimer_cpu_base *cpu_base, ktime_t expires_next)
2204{
2205 hrtimer_rearm(cpu_base, expires_next, false);
2206}
2207#endif /* !CONFIG_HRTIMER_REARM_DEFERRED */
2208
2209/*
2210 * High resolution timer interrupt
2211 * Called with interrupts disabled
2212 */
2213void hrtimer_interrupt(struct clock_event_device *dev)
2214{
2215 struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases);
2216 ktime_t expires_next, now, entry_time, delta;
2217 unsigned long flags;
2218 int retries = 0;
2219
2220 BUG_ON(!cpu_base->hres_active);
2221 cpu_base->nr_events++;
2222 dev->next_event = KTIME_MAX;
2223 dev->next_event_forced = 0;
2224
2225 raw_spin_lock_irqsave(&cpu_base->lock, flags);
2226 entry_time = now = hrtimer_update_base(cpu_base);
2227retry:
2228 cpu_base->deferred_rearm = true;
2229 /*
2230 * Set expires_next to KTIME_MAX, which prevents that remote CPUs queue
2231 * timers while __hrtimer_run_queues() is expiring the clock bases.
2232 * Timers which are re/enqueued on the local CPU are not affected by
2233 * this.
2234 */
2235 cpu_base->expires_next = KTIME_MAX;
2236
2237 if (!ktime_before(now, cpu_base->softirq_expires_next)) {
2238 cpu_base->softirq_expires_next = KTIME_MAX;
2239 cpu_base->softirq_activated = true;
2240 raise_timer_softirq(HRTIMER_SOFTIRQ);
2241 }
2242
2243 __hrtimer_run_queues(cpu_base, now, flags, HRTIMER_ACTIVE_HARD);
2244
2245 /*
2246 * The next timer was already expired due to:
2247 * - tracing
2248 * - long lasting callbacks
2249 * - being scheduled away when running in a VM
2250 *
2251 * We need to prevent that we loop forever in the hrtiner interrupt
2252 * routine. We give it 3 attempts to avoid overreacting on some
2253 * spurious event.
2254 */
2255 now = hrtimer_update_base(cpu_base);
2256 expires_next = hrtimer_update_next_event(cpu_base);
2257 cpu_base->hang_detected = false;
2258 if (expires_next < now) {
2259 if (++retries < 3) {
2260 cpu_base->nr_retries++;
2261 goto retry;
2262 }
2263
2264 delta = ktime_sub(now, entry_time);
2265 cpu_base->max_hang_time = max_t(unsigned int, cpu_base->max_hang_time, delta);
2266 cpu_base->nr_hangs++;
2267 cpu_base->hang_detected = true;
2268 }
2269
2270 hrtimer_interrupt_rearm(cpu_base, expires_next);
2271 raw_spin_unlock_irqrestore(&cpu_base->lock, flags);
2272}
2273
2274#endif /* !CONFIG_HIGH_RES_TIMERS */
2275
2276/*
2277 * Called from run_local_timers in hardirq context every jiffy
2278 */
2279void hrtimer_run_queues(void)
2280{
2281 struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases);
2282 unsigned long flags;
2283 ktime_t now;
2284
2285 if (hrtimer_hres_active(cpu_base))
2286 return;
2287
2288 /*
2289 * This _is_ ugly: We have to check periodically, whether we
2290 * can switch to highres and / or nohz mode. The clocksource
2291 * switch happens with xtime_lock held. Notification from
2292 * there only sets the check bit in the tick_oneshot code,
2293 * otherwise we might deadlock vs. xtime_lock.
2294 */
2295 if (tick_check_oneshot_change(!hrtimer_is_hres_enabled())) {
2296 hrtimer_switch_to_hres();
2297 return;
2298 }
2299
2300 raw_spin_lock_irqsave(&cpu_base->lock, flags);
2301 now = hrtimer_update_base(cpu_base);
2302
2303 if (!ktime_before(now, cpu_base->softirq_expires_next)) {
2304 cpu_base->softirq_expires_next = KTIME_MAX;
2305 cpu_base->softirq_activated = true;
2306 raise_timer_softirq(HRTIMER_SOFTIRQ);
2307 }
2308
2309 __hrtimer_run_queues(cpu_base, now, flags, HRTIMER_ACTIVE_HARD);
2310 raw_spin_unlock_irqrestore(&cpu_base->lock, flags);
2311}
2312
2313/*
2314 * Sleep related functions:
2315 */
2316static enum hrtimer_restart hrtimer_wakeup(struct hrtimer *timer)
2317{
2318 struct hrtimer_sleeper *t = container_of(timer, struct hrtimer_sleeper, timer);
2319 struct task_struct *task = t->task;
2320
2321 t->task = NULL;
2322 if (task)
2323 wake_up_process(task);
2324
2325 return HRTIMER_NORESTART;
2326}
2327
2328/**
2329 * hrtimer_sleeper_start_expires - Start a hrtimer sleeper timer
2330 * @sl: sleeper to be started
2331 * @mode: timer mode abs/rel
2332 *
2333 * Wrapper around hrtimer_start_expires() for hrtimer_sleeper based timers
2334 * to allow PREEMPT_RT to tweak the delivery mode (soft/hardirq context)
2335 */
2336void hrtimer_sleeper_start_expires(struct hrtimer_sleeper *sl, enum hrtimer_mode mode)
2337{
2338 /*
2339 * Make the enqueue delivery mode check work on RT. If the sleeper
2340 * was initialized for hard interrupt delivery, force the mode bit.
2341 * This is a special case for hrtimer_sleepers because
2342 * __hrtimer_setup_sleeper() determines the delivery mode on RT so the
2343 * fiddling with this decision is avoided at the call sites.
2344 */
2345 if (IS_ENABLED(CONFIG_PREEMPT_RT) && sl->timer.is_hard)
2346 mode |= HRTIMER_MODE_HARD;
2347
2348 /* If already expired, clear the task pointer and set current state to running */
2349 if (!hrtimer_start_expires_user(&sl->timer, mode)) {
2350 sl->task = NULL;
2351 __set_current_state(TASK_RUNNING);
2352 }
2353}
2354EXPORT_SYMBOL_GPL(hrtimer_sleeper_start_expires);
2355
2356static void __hrtimer_setup_sleeper(struct hrtimer_sleeper *sl, clockid_t clock_id,
2357 enum hrtimer_mode mode)
2358{
2359 /*
2360 * On PREEMPT_RT enabled kernels hrtimers which are not explicitly
2361 * marked for hard interrupt expiry mode are moved into soft
2362 * interrupt context either for latency reasons or because the
2363 * hrtimer callback takes regular spinlocks or invokes other
2364 * functions which are not suitable for hard interrupt context on
2365 * PREEMPT_RT.
2366 *
2367 * The hrtimer_sleeper callback is RT compatible in hard interrupt
2368 * context, but there is a latency concern: Untrusted userspace can
2369 * spawn many threads which arm timers for the same expiry time on
2370 * the same CPU. That causes a latency spike due to the wakeup of
2371 * a gazillion threads.
2372 *
2373 * OTOH, privileged real-time user space applications rely on the
2374 * low latency of hard interrupt wakeups. If the current task is in
2375 * a real-time scheduling class, mark the mode for hard interrupt
2376 * expiry.
2377 */
2378 if (IS_ENABLED(CONFIG_PREEMPT_RT)) {
2379 if (rt_or_dl_task_policy(current) && !(mode & HRTIMER_MODE_SOFT))
2380 mode |= HRTIMER_MODE_HARD;
2381 }
2382
2383 __hrtimer_setup(&sl->timer, hrtimer_wakeup, clock_id, mode);
2384 sl->task = current;
2385}
2386
2387/**
2388 * hrtimer_setup_sleeper_on_stack - initialize a sleeper in stack memory
2389 * @sl: sleeper to be initialized
2390 * @clock_id: the clock to be used
2391 * @mode: timer mode abs/rel
2392 */
2393void hrtimer_setup_sleeper_on_stack(struct hrtimer_sleeper *sl, clockid_t clock_id,
2394 enum hrtimer_mode mode)
2395{
2396 debug_setup_on_stack(&sl->timer, clock_id, mode);
2397 __hrtimer_setup_sleeper(sl, clock_id, mode);
2398}
2399EXPORT_SYMBOL_GPL(hrtimer_setup_sleeper_on_stack);
2400
2401int nanosleep_copyout(struct restart_block *restart, struct timespec64 *ts)
2402{
2403 switch(restart->nanosleep.type) {
2404#ifdef CONFIG_COMPAT_32BIT_TIME
2405 case TT_COMPAT:
2406 if (put_old_timespec32(ts, restart->nanosleep.compat_rmtp))
2407 return -EFAULT;
2408 break;
2409#endif
2410 case TT_NATIVE:
2411 if (put_timespec64(ts, restart->nanosleep.rmtp))
2412 return -EFAULT;
2413 break;
2414 default:
2415 BUG();
2416 }
2417 return -ERESTART_RESTARTBLOCK;
2418}
2419
2420static int __sched do_nanosleep(struct hrtimer_sleeper *t, enum hrtimer_mode mode)
2421{
2422 struct restart_block *restart;
2423
2424 do {
2425 set_current_state(TASK_INTERRUPTIBLE|TASK_FREEZABLE);
2426 hrtimer_sleeper_start_expires(t, mode);
2427
2428 if (likely(t->task))
2429 schedule();
2430
2431 hrtimer_cancel(&t->timer);
2432 mode = HRTIMER_MODE_ABS;
2433
2434 } while (t->task && !signal_pending(current));
2435
2436 __set_current_state(TASK_RUNNING);
2437
2438 if (!t->task)
2439 return 0;
2440
2441 restart = &current->restart_block;
2442 if (restart->nanosleep.type != TT_NONE) {
2443 ktime_t rem = hrtimer_expires_remaining(&t->timer);
2444 struct timespec64 rmt;
2445
2446 if (rem <= 0)
2447 return 0;
2448 rmt = ktime_to_timespec64(rem);
2449
2450 return nanosleep_copyout(restart, &rmt);
2451 }
2452 return -ERESTART_RESTARTBLOCK;
2453}
2454
2455static long __sched hrtimer_nanosleep_restart(struct restart_block *restart)
2456{
2457 struct hrtimer_sleeper t;
2458 int ret;
2459
2460 hrtimer_setup_sleeper_on_stack(&t, restart->nanosleep.clockid, HRTIMER_MODE_ABS);
2461 hrtimer_set_expires(&t.timer, restart->nanosleep.expires);
2462 ret = do_nanosleep(&t, HRTIMER_MODE_ABS);
2463 destroy_hrtimer_on_stack(&t.timer);
2464 return ret;
2465}
2466
2467long hrtimer_nanosleep(ktime_t rqtp, const enum hrtimer_mode mode, const clockid_t clockid)
2468{
2469 struct restart_block *restart;
2470 struct hrtimer_sleeper t;
2471 int ret;
2472
2473 hrtimer_setup_sleeper_on_stack(&t, clockid, mode);
2474 hrtimer_set_expires_range_ns(&t.timer, rqtp, current->timer_slack_ns);
2475 ret = do_nanosleep(&t, mode);
2476 if (ret != -ERESTART_RESTARTBLOCK)
2477 goto out;
2478
2479 /* Absolute timers do not update the rmtp value and restart: */
2480 if (mode == HRTIMER_MODE_ABS) {
2481 ret = -ERESTARTNOHAND;
2482 goto out;
2483 }
2484
2485 restart = &current->restart_block;
2486 restart->nanosleep.clockid = t.timer.base->clockid;
2487 restart->nanosleep.expires = hrtimer_get_expires(&t.timer);
2488 set_restart_fn(restart, hrtimer_nanosleep_restart);
2489out:
2490 destroy_hrtimer_on_stack(&t.timer);
2491 return ret;
2492}
2493
2494#ifdef CONFIG_64BIT
2495
2496SYSCALL_DEFINE2(nanosleep, struct __kernel_timespec __user *, rqtp,
2497 struct __kernel_timespec __user *, rmtp)
2498{
2499 struct timespec64 tu;
2500
2501 if (get_timespec64(&tu, rqtp))
2502 return -EFAULT;
2503
2504 if (!timespec64_valid(&tu))
2505 return -EINVAL;
2506
2507 current->restart_block.fn = do_no_restart_syscall;
2508 current->restart_block.nanosleep.type = rmtp ? TT_NATIVE : TT_NONE;
2509 current->restart_block.nanosleep.rmtp = rmtp;
2510 return hrtimer_nanosleep(timespec64_to_ktime(tu), HRTIMER_MODE_REL, CLOCK_MONOTONIC);
2511}
2512
2513#endif
2514
2515#ifdef CONFIG_COMPAT_32BIT_TIME
2516
2517SYSCALL_DEFINE2(nanosleep_time32, struct old_timespec32 __user *, rqtp,
2518 struct old_timespec32 __user *, rmtp)
2519{
2520 struct timespec64 tu;
2521
2522 if (get_old_timespec32(&tu, rqtp))
2523 return -EFAULT;
2524
2525 if (!timespec64_valid(&tu))
2526 return -EINVAL;
2527
2528 current->restart_block.fn = do_no_restart_syscall;
2529 current->restart_block.nanosleep.type = rmtp ? TT_COMPAT : TT_NONE;
2530 current->restart_block.nanosleep.compat_rmtp = rmtp;
2531 return hrtimer_nanosleep(timespec64_to_ktime(tu), HRTIMER_MODE_REL, CLOCK_MONOTONIC);
2532}
2533#endif
2534
2535/*
2536 * Functions related to boot-time initialization:
2537 */
2538int hrtimers_prepare_cpu(unsigned int cpu)
2539{
2540 struct hrtimer_cpu_base *cpu_base = &per_cpu(hrtimer_bases, cpu);
2541
2542 for (int i = 0; i < HRTIMER_MAX_CLOCK_BASES; i++) {
2543 struct hrtimer_clock_base *clock_b = &cpu_base->clock_base[i];
2544
2545 clock_b->cpu_base = cpu_base;
2546 seqcount_raw_spinlock_init(&clock_b->seq, &cpu_base->lock);
2547 timerqueue_linked_init_head(&clock_b->active);
2548 }
2549
2550 cpu_base->cpu = cpu;
2551 hrtimer_cpu_base_init_expiry_lock(cpu_base);
2552 return 0;
2553}
2554
2555int hrtimers_cpu_starting(unsigned int cpu)
2556{
2557 struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases);
2558
2559 /* Clear out any left over state from a CPU down operation */
2560 cpu_base->active_bases = 0;
2561 cpu_base->hres_active = false;
2562 cpu_base->hang_detected = false;
2563 cpu_base->next_timer = NULL;
2564 cpu_base->softirq_next_timer = NULL;
2565 cpu_base->expires_next = KTIME_MAX;
2566 cpu_base->softirq_expires_next = KTIME_MAX;
2567 cpu_base->softirq_activated = false;
2568 cpu_base->online = true;
2569 return 0;
2570}
2571
2572#ifdef CONFIG_HOTPLUG_CPU
2573
2574static void migrate_hrtimer_list(struct hrtimer_clock_base *old_base,
2575 struct hrtimer_clock_base *new_base)
2576{
2577 struct timerqueue_linked_node *node;
2578 struct hrtimer *timer;
2579
2580 while ((node = timerqueue_linked_first(&old_base->active))) {
2581 timer = hrtimer_from_timerqueue_node(node);
2582 BUG_ON(hrtimer_callback_running(timer));
2583 debug_hrtimer_deactivate(timer);
2584
2585 /*
2586 * Mark it as ENQUEUED not INACTIVE otherwise the
2587 * timer could be seen as !active and just vanish away
2588 * under us on another CPU
2589 */
2590 __remove_hrtimer(timer, old_base, HRTIMER_STATE_ENQUEUED, false);
2591 timer->base = new_base;
2592 /*
2593 * Enqueue the timers on the new cpu. This does not
2594 * reprogram the event device in case the timer
2595 * expires before the earliest on this CPU, but we run
2596 * hrtimer_interrupt after we migrated everything to
2597 * sort out already expired timers and reprogram the
2598 * event device.
2599 */
2600 enqueue_hrtimer(timer, new_base, HRTIMER_MODE_ABS, true);
2601 }
2602}
2603
2604int hrtimers_cpu_dying(unsigned int dying_cpu)
2605{
2606 int ncpu = cpumask_any_and(cpu_active_mask, housekeeping_cpumask(HK_TYPE_TIMER));
2607 struct hrtimer_cpu_base *old_base, *new_base;
2608
2609 old_base = this_cpu_ptr(&hrtimer_bases);
2610 new_base = &per_cpu(hrtimer_bases, ncpu);
2611
2612 /*
2613 * The caller is globally serialized and nobody else
2614 * takes two locks at once, deadlock is not possible.
2615 */
2616 raw_spin_lock(&old_base->lock);
2617 raw_spin_lock_nested(&new_base->lock, SINGLE_DEPTH_NESTING);
2618
2619 for (int i = 0; i < HRTIMER_MAX_CLOCK_BASES; i++)
2620 migrate_hrtimer_list(&old_base->clock_base[i], &new_base->clock_base[i]);
2621
2622 /* Tell the other CPU to retrigger the next event */
2623 smp_call_function_single(ncpu, retrigger_next_event, NULL, 0);
2624
2625 raw_spin_unlock(&new_base->lock);
2626 old_base->online = false;
2627 raw_spin_unlock(&old_base->lock);
2628
2629 return 0;
2630}
2631
2632#endif /* CONFIG_HOTPLUG_CPU */
2633
2634void __init hrtimers_init(void)
2635{
2636 hrtimers_prepare_cpu(smp_processor_id());
2637 hrtimers_cpu_starting(smp_processor_id());
2638 open_softirq(HRTIMER_SOFTIRQ, hrtimer_run_softirq);
2639}