projects/linux·blob v4.13 · refs

470 lines · 11.6 KiB

1/*
2 * sched_clock for unstable cpu clocks
3 *
4 * Copyright (C) 2008 Red Hat, Inc., Peter Zijlstra
5 *
6 * Updates and enhancements:
7 * Copyright (C) 2008 Red Hat, Inc. Steven Rostedt <[email protected]>
8 *
9 * Based on code by:
10 * Ingo Molnar <[email protected]>
11 * Guillaume Chazarain <[email protected]>
12 *
13 *
14 * What:
15 *
16 * cpu_clock(i) provides a fast (execution time) high resolution
17 * clock with bounded drift between CPUs. The value of cpu_clock(i)
18 * is monotonic for constant i. The timestamp returned is in nanoseconds.
19 *
20 * ######################### BIG FAT WARNING ##########################
21 * # when comparing cpu_clock(i) to cpu_clock(j) for i != j, time can #
22 * # go backwards !! #
23 * ####################################################################
24 *
25 * There is no strict promise about the base, although it tends to start
26 * at 0 on boot (but people really shouldn't rely on that).
27 *
28 * cpu_clock(i) -- can be used from any context, including NMI.
29 * local_clock() -- is cpu_clock() on the current cpu.
30 *
31 * sched_clock_cpu(i)
32 *
33 * How:
34 *
35 * The implementation either uses sched_clock() when
36 * !CONFIG_HAVE_UNSTABLE_SCHED_CLOCK, which means in that case the
37 * sched_clock() is assumed to provide these properties (mostly it means
38 * the architecture provides a globally synchronized highres time source).
39 *
40 * Otherwise it tries to create a semi stable clock from a mixture of other
41 * clocks, including:
42 *
43 * - GTOD (clock monotomic)
44 * - sched_clock()
45 * - explicit idle events
46 *
47 * We use GTOD as base and use sched_clock() deltas to improve resolution. The
48 * deltas are filtered to provide monotonicity and keeping it within an
49 * expected window.
50 *
51 * Furthermore, explicit sleep and wakeup hooks allow us to account for time
52 * that is otherwise invisible (TSC gets stopped).
53 *
54 */
55#include <linux/spinlock.h>
56#include <linux/hardirq.h>
57#include <linux/export.h>
58#include <linux/percpu.h>
59#include <linux/ktime.h>
60#include <linux/sched.h>
61#include <linux/nmi.h>
62#include <linux/sched/clock.h>
63#include <linux/static_key.h>
64#include <linux/workqueue.h>
65#include <linux/compiler.h>
66#include <linux/tick.h>
67#include <linux/init.h>
68
69/*
70 * Scheduler clock - returns current time in nanosec units.
71 * This is default implementation.
72 * Architectures and sub-architectures can override this.
73 */
74unsigned long long __weak sched_clock(void)
75{
76 return (unsigned long long)(jiffies - INITIAL_JIFFIES)
77 * (NSEC_PER_SEC / HZ);
78}
79EXPORT_SYMBOL_GPL(sched_clock);
80
81__read_mostly int sched_clock_running;
82
83void sched_clock_init(void)
84{
85 sched_clock_running = 1;
86}
87
88#ifdef CONFIG_HAVE_UNSTABLE_SCHED_CLOCK
89/*
90 * We must start with !__sched_clock_stable because the unstable -> stable
91 * transition is accurate, while the stable -> unstable transition is not.
92 *
93 * Similarly we start with __sched_clock_stable_early, thereby assuming we
94 * will become stable, such that there's only a single 1 -> 0 transition.
95 */
96static DEFINE_STATIC_KEY_FALSE(__sched_clock_stable);
97static int __sched_clock_stable_early = 1;
98
99/*
100 * We want: ktime_get_ns() + __gtod_offset == sched_clock() + __sched_clock_offset
101 */
102__read_mostly u64 __sched_clock_offset;
103static __read_mostly u64 __gtod_offset;
104
105struct sched_clock_data {
106 u64 tick_raw;
107 u64 tick_gtod;
108 u64 clock;
109};
110
111static DEFINE_PER_CPU_SHARED_ALIGNED(struct sched_clock_data, sched_clock_data);
112
113static inline struct sched_clock_data *this_scd(void)
114{
115 return this_cpu_ptr(&sched_clock_data);
116}
117
118static inline struct sched_clock_data *cpu_sdc(int cpu)
119{
120 return &per_cpu(sched_clock_data, cpu);
121}
122
123int sched_clock_stable(void)
124{
125 return static_branch_likely(&__sched_clock_stable);
126}
127
128static void __scd_stamp(struct sched_clock_data *scd)
129{
130 scd->tick_gtod = ktime_get_ns();
131 scd->tick_raw = sched_clock();
132}
133
134static void __set_sched_clock_stable(void)
135{
136 struct sched_clock_data *scd;
137
138 /*
139 * Since we're still unstable and the tick is already running, we have
140 * to disable IRQs in order to get a consistent scd->tick* reading.
141 */
142 local_irq_disable();
143 scd = this_scd();
144 /*
145 * Attempt to make the (initial) unstable->stable transition continuous.
146 */
147 __sched_clock_offset = (scd->tick_gtod + __gtod_offset) - (scd->tick_raw);
148 local_irq_enable();
149
150 printk(KERN_INFO "sched_clock: Marking stable (%lld, %lld)->(%lld, %lld)\n",
151 scd->tick_gtod, __gtod_offset,
152 scd->tick_raw, __sched_clock_offset);
153
154 static_branch_enable(&__sched_clock_stable);
155 tick_dep_clear(TICK_DEP_BIT_CLOCK_UNSTABLE);
156}
157
158/*
159 * If we ever get here, we're screwed, because we found out -- typically after
160 * the fact -- that TSC wasn't good. This means all our clocksources (including
161 * ktime) could have reported wrong values.
162 *
163 * What we do here is an attempt to fix up and continue sort of where we left
164 * off in a coherent manner.
165 *
166 * The only way to fully avoid random clock jumps is to boot with:
167 * "tsc=unstable".
168 */
169static void __sched_clock_work(struct work_struct *work)
170{
171 struct sched_clock_data *scd;
172 int cpu;
173
174 /* take a current timestamp and set 'now' */
175 preempt_disable();
176 scd = this_scd();
177 __scd_stamp(scd);
178 scd->clock = scd->tick_gtod + __gtod_offset;
179 preempt_enable();
180
181 /* clone to all CPUs */
182 for_each_possible_cpu(cpu)
183 per_cpu(sched_clock_data, cpu) = *scd;
184
185 printk(KERN_WARNING "TSC found unstable after boot, most likely due to broken BIOS. Use 'tsc=unstable'.\n");
186 printk(KERN_INFO "sched_clock: Marking unstable (%lld, %lld)<-(%lld, %lld)\n",
187 scd->tick_gtod, __gtod_offset,
188 scd->tick_raw, __sched_clock_offset);
189
190 static_branch_disable(&__sched_clock_stable);
191}
192
193static DECLARE_WORK(sched_clock_work, __sched_clock_work);
194
195static void __clear_sched_clock_stable(void)
196{
197 if (!sched_clock_stable())
198 return;
199
200 tick_dep_set(TICK_DEP_BIT_CLOCK_UNSTABLE);
201 schedule_work(&sched_clock_work);
202}
203
204void clear_sched_clock_stable(void)
205{
206 __sched_clock_stable_early = 0;
207
208 smp_mb(); /* matches sched_clock_init_late() */
209
210 if (sched_clock_running == 2)
211 __clear_sched_clock_stable();
212}
213
214/*
215 * We run this as late_initcall() such that it runs after all built-in drivers,
216 * notably: acpi_processor and intel_idle, which can mark the TSC as unstable.
217 */
218static int __init sched_clock_init_late(void)
219{
220 sched_clock_running = 2;
221 /*
222 * Ensure that it is impossible to not do a static_key update.
223 *
224 * Either {set,clear}_sched_clock_stable() must see sched_clock_running
225 * and do the update, or we must see their __sched_clock_stable_early
226 * and do the update, or both.
227 */
228 smp_mb(); /* matches {set,clear}_sched_clock_stable() */
229
230 if (__sched_clock_stable_early)
231 __set_sched_clock_stable();
232
233 return 0;
234}
235late_initcall(sched_clock_init_late);
236
237/*
238 * min, max except they take wrapping into account
239 */
240
241static inline u64 wrap_min(u64 x, u64 y)
242{
243 return (s64)(x - y) < 0 ? x : y;
244}
245
246static inline u64 wrap_max(u64 x, u64 y)
247{
248 return (s64)(x - y) > 0 ? x : y;
249}
250
251/*
252 * update the percpu scd from the raw @now value
253 *
254 * - filter out backward motion
255 * - use the GTOD tick value to create a window to filter crazy TSC values
256 */
257static u64 sched_clock_local(struct sched_clock_data *scd)
258{
259 u64 now, clock, old_clock, min_clock, max_clock, gtod;
260 s64 delta;
261
262again:
263 now = sched_clock();
264 delta = now - scd->tick_raw;
265 if (unlikely(delta < 0))
266 delta = 0;
267
268 old_clock = scd->clock;
269
270 /*
271 * scd->clock = clamp(scd->tick_gtod + delta,
272 * max(scd->tick_gtod, scd->clock),
273 * scd->tick_gtod + TICK_NSEC);
274 */
275
276 gtod = scd->tick_gtod + __gtod_offset;
277 clock = gtod + delta;
278 min_clock = wrap_max(gtod, old_clock);
279 max_clock = wrap_max(old_clock, gtod + TICK_NSEC);
280
281 clock = wrap_max(clock, min_clock);
282 clock = wrap_min(clock, max_clock);
283
284 if (cmpxchg64(&scd->clock, old_clock, clock) != old_clock)
285 goto again;
286
287 return clock;
288}
289
290static u64 sched_clock_remote(struct sched_clock_data *scd)
291{
292 struct sched_clock_data *my_scd = this_scd();
293 u64 this_clock, remote_clock;
294 u64 *ptr, old_val, val;
295
296#if BITS_PER_LONG != 64
297again:
298 /*
299 * Careful here: The local and the remote clock values need to
300 * be read out atomic as we need to compare the values and
301 * then update either the local or the remote side. So the
302 * cmpxchg64 below only protects one readout.
303 *
304 * We must reread via sched_clock_local() in the retry case on
305 * 32bit as an NMI could use sched_clock_local() via the
306 * tracer and hit between the readout of
307 * the low32bit and the high 32bit portion.
308 */
309 this_clock = sched_clock_local(my_scd);
310 /*
311 * We must enforce atomic readout on 32bit, otherwise the
312 * update on the remote cpu can hit inbetween the readout of
313 * the low32bit and the high 32bit portion.
314 */
315 remote_clock = cmpxchg64(&scd->clock, 0, 0);
316#else
317 /*
318 * On 64bit the read of [my]scd->clock is atomic versus the
319 * update, so we can avoid the above 32bit dance.
320 */
321 sched_clock_local(my_scd);
322again:
323 this_clock = my_scd->clock;
324 remote_clock = scd->clock;
325#endif
326
327 /*
328 * Use the opportunity that we have both locks
329 * taken to couple the two clocks: we take the
330 * larger time as the latest time for both
331 * runqueues. (this creates monotonic movement)
332 */
333 if (likely((s64)(remote_clock - this_clock) < 0)) {
334 ptr = &scd->clock;
335 old_val = remote_clock;
336 val = this_clock;
337 } else {
338 /*
339 * Should be rare, but possible:
340 */
341 ptr = &my_scd->clock;
342 old_val = this_clock;
343 val = remote_clock;
344 }
345
346 if (cmpxchg64(ptr, old_val, val) != old_val)
347 goto again;
348
349 return val;
350}
351
352/*
353 * Similar to cpu_clock(), but requires local IRQs to be disabled.
354 *
355 * See cpu_clock().
356 */
357u64 sched_clock_cpu(int cpu)
358{
359 struct sched_clock_data *scd;
360 u64 clock;
361
362 if (sched_clock_stable())
363 return sched_clock() + __sched_clock_offset;
364
365 if (unlikely(!sched_clock_running))
366 return 0ull;
367
368 preempt_disable_notrace();
369 scd = cpu_sdc(cpu);
370
371 if (cpu != smp_processor_id())
372 clock = sched_clock_remote(scd);
373 else
374 clock = sched_clock_local(scd);
375 preempt_enable_notrace();
376
377 return clock;
378}
379EXPORT_SYMBOL_GPL(sched_clock_cpu);
380
381void sched_clock_tick(void)
382{
383 struct sched_clock_data *scd;
384
385 if (sched_clock_stable())
386 return;
387
388 if (unlikely(!sched_clock_running))
389 return;
390
391 WARN_ON_ONCE(!irqs_disabled());
392
393 scd = this_scd();
394 __scd_stamp(scd);
395 sched_clock_local(scd);
396}
397
398void sched_clock_tick_stable(void)
399{
400 u64 gtod, clock;
401
402 if (!sched_clock_stable())
403 return;
404
405 /*
406 * Called under watchdog_lock.
407 *
408 * The watchdog just found this TSC to (still) be stable, so now is a
409 * good moment to update our __gtod_offset. Because once we find the
410 * TSC to be unstable, any computation will be computing crap.
411 */
412 local_irq_disable();
413 gtod = ktime_get_ns();
414 clock = sched_clock();
415 __gtod_offset = (clock + __sched_clock_offset) - gtod;
416 local_irq_enable();
417}
418
419/*
420 * We are going deep-idle (irqs are disabled):
421 */
422void sched_clock_idle_sleep_event(void)
423{
424 sched_clock_cpu(smp_processor_id());
425}
426EXPORT_SYMBOL_GPL(sched_clock_idle_sleep_event);
427
428/*
429 * We just idled; resync with ktime.
430 */
431void sched_clock_idle_wakeup_event(void)
432{
433 unsigned long flags;
434
435 if (sched_clock_stable())
436 return;
437
438 if (unlikely(timekeeping_suspended))
439 return;
440
441 local_irq_save(flags);
442 sched_clock_tick();
443 local_irq_restore(flags);
444}
445EXPORT_SYMBOL_GPL(sched_clock_idle_wakeup_event);
446
447#else /* CONFIG_HAVE_UNSTABLE_SCHED_CLOCK */
448
449u64 sched_clock_cpu(int cpu)
450{
451 if (unlikely(!sched_clock_running))
452 return 0;
453
454 return sched_clock();
455}
456
457#endif /* CONFIG_HAVE_UNSTABLE_SCHED_CLOCK */
458
459/*
460 * Running clock - returns the time that has elapsed while a guest has been
461 * running.
462 * On a guest this value should be local_clock minus the time the guest was
463 * suspended by the hypervisor (for any reason).
464 * On bare metal this function should return the same as local_clock.
465 * Architectures and sub-architectures can override this.
466 */
467u64 __weak running_clock(void)
468{
469 return local_clock();
470}