projects/linux·blob v7.0 · refs

511 lines · 12.8 KiB

1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * sched_clock() for unstable CPU clocks
4 *
5 * Copyright (C) 2008 Red Hat, Inc., Peter Zijlstra
6 *
7 * Updates and enhancements:
8 * Copyright (C) 2008 Red Hat, Inc. Steven Rostedt <[email protected]>
9 *
10 * Based on code by:
11 * Ingo Molnar <[email protected]>
12 * Guillaume Chazarain <[email protected]>
13 *
14 *
15 * What this file implements:
16 *
17 * cpu_clock(i) provides a fast (execution time) high resolution
18 * clock with bounded drift between CPUs. The value of cpu_clock(i)
19 * is monotonic for constant i. The timestamp returned is in nanoseconds.
20 *
21 * ######################### BIG FAT WARNING ##########################
22 * # when comparing cpu_clock(i) to cpu_clock(j) for i != j, time can #
23 * # go backwards !! #
24 * ####################################################################
25 *
26 * There is no strict promise about the base, although it tends to start
27 * at 0 on boot (but people really shouldn't rely on that).
28 *
29 * cpu_clock(i) -- can be used from any context, including NMI.
30 * local_clock() -- is cpu_clock() on the current CPU.
31 *
32 * sched_clock_cpu(i)
33 *
34 * How it is implemented:
35 *
36 * The implementation either uses sched_clock() when
37 * !CONFIG_HAVE_UNSTABLE_SCHED_CLOCK, which means in that case the
38 * sched_clock() is assumed to provide these properties (mostly it means
39 * the architecture provides a globally synchronized highres time source).
40 *
41 * Otherwise it tries to create a semi stable clock from a mixture of other
42 * clocks, including:
43 *
44 * - GTOD (clock monotonic)
45 * - sched_clock()
46 * - explicit idle events
47 *
48 * We use GTOD as base and use sched_clock() deltas to improve resolution. The
49 * deltas are filtered to provide monotonicity and keeping it within an
50 * expected window.
51 *
52 * Furthermore, explicit sleep and wakeup hooks allow us to account for time
53 * that is otherwise invisible (TSC gets stopped).
54 *
55 */
56
57#include <linux/sched/clock.h>
58#include "sched.h"
59
60/*
61 * Scheduler clock - returns current time in nanosec units.
62 * This is default implementation.
63 * Architectures and sub-architectures can override this.
64 */
65notrace unsigned long long __weak sched_clock(void)
66{
67 return (unsigned long long)(jiffies - INITIAL_JIFFIES)
68 * (NSEC_PER_SEC / HZ);
69}
70EXPORT_SYMBOL_GPL(sched_clock);
71
72static DEFINE_STATIC_KEY_FALSE(sched_clock_running);
73
74#ifdef CONFIG_HAVE_UNSTABLE_SCHED_CLOCK
75/*
76 * We must start with !__sched_clock_stable because the unstable -> stable
77 * transition is accurate, while the stable -> unstable transition is not.
78 *
79 * Similarly we start with __sched_clock_stable_early, thereby assuming we
80 * will become stable, such that there's only a single 1 -> 0 transition.
81 */
82static DEFINE_STATIC_KEY_FALSE(__sched_clock_stable);
83static int __sched_clock_stable_early = 1;
84
85/*
86 * We want: ktime_get_ns() + __gtod_offset == sched_clock() + __sched_clock_offset
87 */
88__read_mostly u64 __sched_clock_offset;
89static __read_mostly u64 __gtod_offset;
90
91struct sched_clock_data {
92 u64 tick_raw;
93 u64 tick_gtod;
94 u64 clock;
95};
96
97static DEFINE_PER_CPU_SHARED_ALIGNED(struct sched_clock_data, sched_clock_data);
98
99static __always_inline struct sched_clock_data *this_scd(void)
100{
101 return this_cpu_ptr(&sched_clock_data);
102}
103
104notrace static inline struct sched_clock_data *cpu_sdc(int cpu)
105{
106 return &per_cpu(sched_clock_data, cpu);
107}
108
109notrace int sched_clock_stable(void)
110{
111 return static_branch_likely(&__sched_clock_stable);
112}
113
114notrace static void __scd_stamp(struct sched_clock_data *scd)
115{
116 scd->tick_gtod = ktime_get_ns();
117 scd->tick_raw = sched_clock();
118}
119
120notrace static void __set_sched_clock_stable(void)
121{
122 struct sched_clock_data *scd;
123
124 /*
125 * Since we're still unstable and the tick is already running, we have
126 * to disable IRQs in order to get a consistent scd->tick* reading.
127 */
128 local_irq_disable();
129 scd = this_scd();
130 /*
131 * Attempt to make the (initial) unstable->stable transition continuous.
132 */
133 __sched_clock_offset = (scd->tick_gtod + __gtod_offset) - (scd->tick_raw);
134 local_irq_enable();
135
136 printk(KERN_INFO "sched_clock: Marking stable (%lld, %lld)->(%lld, %lld)\n",
137 scd->tick_gtod, __gtod_offset,
138 scd->tick_raw, __sched_clock_offset);
139
140 static_branch_enable(&__sched_clock_stable);
141 tick_dep_clear(TICK_DEP_BIT_CLOCK_UNSTABLE);
142}
143
144/*
145 * If we ever get here, we're screwed, because we found out -- typically after
146 * the fact -- that TSC wasn't good. This means all our clocksources (including
147 * ktime) could have reported wrong values.
148 *
149 * What we do here is an attempt to fix up and continue sort of where we left
150 * off in a coherent manner.
151 *
152 * The only way to fully avoid random clock jumps is to boot with:
153 * "tsc=unstable".
154 */
155notrace static void __sched_clock_work(struct work_struct *work)
156{
157 struct sched_clock_data *scd;
158 int cpu;
159
160 /* take a current timestamp and set 'now' */
161 preempt_disable();
162 scd = this_scd();
163 __scd_stamp(scd);
164 scd->clock = scd->tick_gtod + __gtod_offset;
165 preempt_enable();
166
167 /* clone to all CPUs */
168 for_each_possible_cpu(cpu)
169 per_cpu(sched_clock_data, cpu) = *scd;
170
171 printk(KERN_WARNING "TSC found unstable after boot, most likely due to broken BIOS. Use 'tsc=unstable'.\n");
172 printk(KERN_INFO "sched_clock: Marking unstable (%lld, %lld)<-(%lld, %lld)\n",
173 scd->tick_gtod, __gtod_offset,
174 scd->tick_raw, __sched_clock_offset);
175
176 disable_sched_clock_irqtime();
177 static_branch_disable(&__sched_clock_stable);
178}
179
180static DECLARE_WORK(sched_clock_work, __sched_clock_work);
181
182notrace static void __clear_sched_clock_stable(void)
183{
184 if (!sched_clock_stable())
185 return;
186
187 tick_dep_set(TICK_DEP_BIT_CLOCK_UNSTABLE);
188 schedule_work(&sched_clock_work);
189}
190
191notrace void clear_sched_clock_stable(void)
192{
193 __sched_clock_stable_early = 0;
194
195 smp_mb(); /* matches sched_clock_init_late() */
196
197 if (static_key_count(&sched_clock_running.key) == 2)
198 __clear_sched_clock_stable();
199}
200
201notrace static void __sched_clock_gtod_offset(void)
202{
203 struct sched_clock_data *scd = this_scd();
204
205 __scd_stamp(scd);
206 __gtod_offset = (scd->tick_raw + __sched_clock_offset) - scd->tick_gtod;
207}
208
209void __init sched_clock_init(void)
210{
211 /*
212 * Set __gtod_offset such that once we mark sched_clock_running,
213 * sched_clock_tick() continues where sched_clock() left off.
214 *
215 * Even if TSC is buggered, we're still UP at this point so it
216 * can't really be out of sync.
217 */
218 local_irq_disable();
219 __sched_clock_gtod_offset();
220 local_irq_enable();
221
222 static_branch_inc(&sched_clock_running);
223}
224/*
225 * We run this as late_initcall() such that it runs after all built-in drivers,
226 * notably: acpi_processor and intel_idle, which can mark the TSC as unstable.
227 */
228static int __init sched_clock_init_late(void)
229{
230 static_branch_inc(&sched_clock_running);
231 /*
232 * Ensure that it is impossible to not do a static_key update.
233 *
234 * Either {set,clear}_sched_clock_stable() must see sched_clock_running
235 * and do the update, or we must see their __sched_clock_stable_early
236 * and do the update, or both.
237 */
238 smp_mb(); /* matches {set,clear}_sched_clock_stable() */
239
240 if (__sched_clock_stable_early)
241 __set_sched_clock_stable();
242 else
243 disable_sched_clock_irqtime(); /* disable if clock unstable. */
244
245 return 0;
246}
247late_initcall(sched_clock_init_late);
248
249/*
250 * min, max except they take wrapping into account
251 */
252
253static __always_inline u64 wrap_min(u64 x, u64 y)
254{
255 return (s64)(x - y) < 0 ? x : y;
256}
257
258static __always_inline u64 wrap_max(u64 x, u64 y)
259{
260 return (s64)(x - y) > 0 ? x : y;
261}
262
263/*
264 * update the percpu scd from the raw @now value
265 *
266 * - filter out backward motion
267 * - use the GTOD tick value to create a window to filter crazy TSC values
268 */
269static __always_inline u64 sched_clock_local(struct sched_clock_data *scd)
270{
271 u64 now, clock, old_clock, min_clock, max_clock, gtod;
272 s64 delta;
273
274again:
275 now = sched_clock_noinstr();
276 delta = now - scd->tick_raw;
277 if (unlikely(delta < 0))
278 delta = 0;
279
280 old_clock = scd->clock;
281
282 /*
283 * scd->clock = clamp(scd->tick_gtod + delta,
284 * max(scd->tick_gtod, scd->clock),
285 * scd->tick_gtod + TICK_NSEC);
286 */
287
288 gtod = scd->tick_gtod + __gtod_offset;
289 clock = gtod + delta;
290 min_clock = wrap_max(gtod, old_clock);
291 max_clock = wrap_max(old_clock, gtod + TICK_NSEC);
292
293 clock = wrap_max(clock, min_clock);
294 clock = wrap_min(clock, max_clock);
295
296 if (!raw_try_cmpxchg64(&scd->clock, &old_clock, clock))
297 goto again;
298
299 return clock;
300}
301
302noinstr u64 local_clock_noinstr(void)
303{
304 u64 clock;
305
306 if (static_branch_likely(&__sched_clock_stable))
307 return sched_clock_noinstr() + __sched_clock_offset;
308
309 if (!static_branch_likely(&sched_clock_running))
310 return sched_clock_noinstr();
311
312 clock = sched_clock_local(this_scd());
313
314 return clock;
315}
316
317u64 local_clock(void)
318{
319 u64 now;
320 preempt_disable_notrace();
321 now = local_clock_noinstr();
322 preempt_enable_notrace();
323 return now;
324}
325EXPORT_SYMBOL_GPL(local_clock);
326
327static notrace u64 sched_clock_remote(struct sched_clock_data *scd)
328{
329 struct sched_clock_data *my_scd = this_scd();
330 u64 this_clock, remote_clock;
331 u64 *ptr, old_val, val;
332
333#if BITS_PER_LONG != 64
334again:
335 /*
336 * Careful here: The local and the remote clock values need to
337 * be read out atomic as we need to compare the values and
338 * then update either the local or the remote side. So the
339 * cmpxchg64 below only protects one readout.
340 *
341 * We must reread via sched_clock_local() in the retry case on
342 * 32-bit kernels as an NMI could use sched_clock_local() via the
343 * tracer and hit between the readout of
344 * the low 32-bit and the high 32-bit portion.
345 */
346 this_clock = sched_clock_local(my_scd);
347 /*
348 * We must enforce atomic readout on 32-bit, otherwise the
349 * update on the remote CPU can hit in between the readout of
350 * the low 32-bit and the high 32-bit portion.
351 */
352 remote_clock = cmpxchg64(&scd->clock, 0, 0);
353#else
354 /*
355 * On 64-bit kernels the read of [my]scd->clock is atomic versus the
356 * update, so we can avoid the above 32-bit dance.
357 */
358 sched_clock_local(my_scd);
359again:
360 this_clock = my_scd->clock;
361 remote_clock = scd->clock;
362#endif
363
364 /*
365 * Use the opportunity that we have both locks
366 * taken to couple the two clocks: we take the
367 * larger time as the latest time for both
368 * runqueues. (this creates monotonic movement)
369 */
370 if (likely((s64)(remote_clock - this_clock) < 0)) {
371 ptr = &scd->clock;
372 old_val = remote_clock;
373 val = this_clock;
374 } else {
375 /*
376 * Should be rare, but possible:
377 */
378 ptr = &my_scd->clock;
379 old_val = this_clock;
380 val = remote_clock;
381 }
382
383 if (!try_cmpxchg64(ptr, &old_val, val))
384 goto again;
385
386 return val;
387}
388
389/*
390 * Similar to cpu_clock(), but requires local IRQs to be disabled.
391 *
392 * See cpu_clock().
393 */
394notrace u64 sched_clock_cpu(int cpu)
395{
396 struct sched_clock_data *scd;
397 u64 clock;
398
399 if (sched_clock_stable())
400 return sched_clock() + __sched_clock_offset;
401
402 if (!static_branch_likely(&sched_clock_running))
403 return sched_clock();
404
405 preempt_disable_notrace();
406 scd = cpu_sdc(cpu);
407
408 if (cpu != smp_processor_id())
409 clock = sched_clock_remote(scd);
410 else
411 clock = sched_clock_local(scd);
412 preempt_enable_notrace();
413
414 return clock;
415}
416EXPORT_SYMBOL_GPL(sched_clock_cpu);
417
418notrace void sched_clock_tick(void)
419{
420 struct sched_clock_data *scd;
421
422 if (sched_clock_stable())
423 return;
424
425 if (!static_branch_likely(&sched_clock_running))
426 return;
427
428 lockdep_assert_irqs_disabled();
429
430 scd = this_scd();
431 __scd_stamp(scd);
432 sched_clock_local(scd);
433}
434
435notrace void sched_clock_tick_stable(void)
436{
437 if (!sched_clock_stable())
438 return;
439
440 /*
441 * Called under watchdog_lock.
442 *
443 * The watchdog just found this TSC to (still) be stable, so now is a
444 * good moment to update our __gtod_offset. Because once we find the
445 * TSC to be unstable, any computation will be computing crap.
446 */
447 local_irq_disable();
448 __sched_clock_gtod_offset();
449 local_irq_enable();
450}
451
452/*
453 * We are going deep-idle (IRQs are disabled):
454 */
455notrace void sched_clock_idle_sleep_event(void)
456{
457 sched_clock_cpu(smp_processor_id());
458}
459EXPORT_SYMBOL_GPL(sched_clock_idle_sleep_event);
460
461/*
462 * We just idled; resync with ktime.
463 */
464notrace void sched_clock_idle_wakeup_event(void)
465{
466 unsigned long flags;
467
468 if (sched_clock_stable())
469 return;
470
471 if (unlikely(timekeeping_suspended))
472 return;
473
474 local_irq_save(flags);
475 sched_clock_tick();
476 local_irq_restore(flags);
477}
478EXPORT_SYMBOL_GPL(sched_clock_idle_wakeup_event);
479
480#else /* !CONFIG_HAVE_UNSTABLE_SCHED_CLOCK: */
481
482void __init sched_clock_init(void)
483{
484 static_branch_inc(&sched_clock_running);
485 local_irq_disable();
486 generic_sched_clock_init();
487 local_irq_enable();
488}
489
490notrace u64 sched_clock_cpu(int cpu)
491{
492 if (!static_branch_likely(&sched_clock_running))
493 return 0;
494
495 return sched_clock();
496}
497
498#endif /* !CONFIG_HAVE_UNSTABLE_SCHED_CLOCK */
499
500/*
501 * Running clock - returns the time that has elapsed while a guest has been
502 * running.
503 * On a guest this value should be local_clock minus the time the guest was
504 * suspended by the hypervisor (for any reason).
505 * On bare metal this function should return the same as local_clock.
506 * Architectures and sub-architectures can override this.
507 */
508notrace u64 __weak running_clock(void)
509{
510 return local_clock();
511}