projects/linux·blob master · refs

513 lines · 11.7 KiB

1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Generic userspace implementations of gettimeofday() and similar.
4 */
5#include <vdso/auxclock.h>
6#include <vdso/clocksource.h>
7#include <vdso/datapage.h>
8#include <vdso/helpers.h>
9#include <vdso/ktime.h>
10#include <vdso/limits.h>
11#include <vdso/math64.h>
12#include <vdso/time32.h>
13#include <vdso/time64.h>
14
15#include <uapi/linux/unistd.h>
16
17/*
18 * The generic vDSO implementation requires that gettimeofday.h
19 * provides:
20 * - __arch_get_hw_counter(): to get the hw counter based on the
21 * clock_mode.
22 * - gettimeofday_fallback(): fallback for gettimeofday.
23 * - clock_gettime_fallback(): fallback for clock_gettime.
24 * - clock_getres_fallback(): fallback for clock_getres.
25 */
26#include <asm/vdso/gettimeofday.h>
27
28#include <linux/build_bug.h>
29
30/* Bring in default accessors */
31#include <vdso/vsyscall.h>
32
33#ifndef vdso_calc_ns
34
35#ifdef VDSO_DELTA_NOMASK
36# define VDSO_DELTA_MASK(vd) ULLONG_MAX
37#else
38# define VDSO_DELTA_MASK(vd) (vd->mask)
39#endif
40
41#ifdef CONFIG_GENERIC_VDSO_OVERFLOW_PROTECT
42static __always_inline bool vdso_delta_ok(const struct vdso_clock *vc, u64 delta)
43{
44 return delta < vc->max_cycles;
45}
46#else
47static __always_inline bool vdso_delta_ok(const struct vdso_clock *vc, u64 delta)
48{
49 return true;
50}
51#endif
52
53#ifndef vdso_shift_ns
54static __always_inline u64 vdso_shift_ns(u64 ns, u32 shift)
55{
56 return ns >> shift;
57}
58#endif
59
60/*
61 * Default implementation which works for all sane clocksources. That
62 * obviously excludes x86/TSC.
63 */
64static __always_inline u64 vdso_calc_ns(const struct vdso_clock *vc, u64 cycles, u64 base)
65{
66 u64 delta = (cycles - vc->cycle_last) & VDSO_DELTA_MASK(vc);
67
68 if (likely(vdso_delta_ok(vc, delta)))
69 return vdso_shift_ns((delta * vc->mult) + base, vc->shift);
70
71 return mul_u64_u32_add_u64_shr(delta, vc->mult, base, vc->shift);
72}
73#endif /* vdso_calc_ns */
74
75#ifndef __arch_vdso_hres_capable
76static inline bool __arch_vdso_hres_capable(void)
77{
78 return true;
79}
80#endif
81
82#ifndef vdso_clocksource_ok
83static inline bool vdso_clocksource_ok(const struct vdso_clock *vc)
84{
85 return vc->clock_mode != VDSO_CLOCKMODE_NONE;
86}
87#endif
88
89#ifndef vdso_cycles_ok
90static inline bool vdso_cycles_ok(u64 cycles)
91{
92 return true;
93}
94#endif
95
96static __always_inline bool vdso_clockid_valid(clockid_t clock)
97{
98 /* Check for negative values or invalid clocks */
99 return likely((u32) clock <= CLOCK_AUX_LAST);
100}
101
102/*
103 * Must not be invoked within the sequence read section as a race inside
104 * that loop could result in __iter_div_u64_rem() being extremely slow.
105 */
106static __always_inline void vdso_set_timespec(struct __kernel_timespec *ts, u64 sec, u64 ns)
107{
108 ts->tv_sec = sec + __iter_div_u64_rem(ns, NSEC_PER_SEC, &ns);
109 ts->tv_nsec = ns;
110}
111
112static __always_inline
113bool vdso_get_timestamp(const struct vdso_time_data *vd, const struct vdso_clock *vc,
114 unsigned int clkidx, u64 *sec, u64 *ns)
115{
116 const struct vdso_timestamp *vdso_ts = &vc->basetime[clkidx];
117 u64 cycles;
118
119 if (unlikely(!vdso_clocksource_ok(vc)))
120 return false;
121
122 cycles = __arch_get_hw_counter(vc->clock_mode, vd);
123 if (unlikely(!vdso_cycles_ok(cycles)))
124 return false;
125
126 *ns = vdso_calc_ns(vc, cycles, vdso_ts->nsec);
127 *sec = vdso_ts->sec;
128
129 return true;
130}
131
132static __always_inline
133const struct vdso_time_data *vdso_timens_data(const struct vdso_time_data *vd)
134{
135 return (void *)vd + PAGE_SIZE;
136}
137
138static __always_inline
139bool do_hres_timens(const struct vdso_time_data *vdns, const struct vdso_clock *vcns,
140 clockid_t clk, struct __kernel_timespec *ts)
141{
142 const struct vdso_time_data *vd = vdso_timens_data(vdns);
143 const struct timens_offset *offs = &vcns->offset[clk];
144 const struct vdso_clock *vc = vd->clock_data;
145 u32 seq;
146 s64 sec;
147 u64 ns;
148
149 if (clk != CLOCK_MONOTONIC_RAW)
150 vc = &vc[CS_HRES_COARSE];
151 else
152 vc = &vc[CS_RAW];
153
154 do {
155 seq = vdso_read_begin(vc);
156
157 if (!vdso_get_timestamp(vd, vc, clk, &sec, &ns))
158 return false;
159 } while (vdso_read_retry(vc, seq));
160
161 /* Add the namespace offset */
162 sec += offs->sec;
163 ns += offs->nsec;
164
165 vdso_set_timespec(ts, sec, ns);
166
167 return true;
168}
169
170static __always_inline
171bool do_hres(const struct vdso_time_data *vd, const struct vdso_clock *vc,
172 clockid_t clk, struct __kernel_timespec *ts)
173{
174 u64 sec, ns;
175 u32 seq;
176
177 /* Allows to compile the high resolution parts out */
178 if (!__arch_vdso_hres_capable())
179 return false;
180
181 do {
182 if (vdso_read_begin_timens(vc, &seq))
183 return do_hres_timens(vd, vc, clk, ts);
184
185 if (!vdso_get_timestamp(vd, vc, clk, &sec, &ns))
186 return false;
187 } while (vdso_read_retry(vc, seq));
188
189 vdso_set_timespec(ts, sec, ns);
190
191 return true;
192}
193
194static __always_inline
195bool do_coarse_timens(const struct vdso_time_data *vdns, const struct vdso_clock *vcns,
196 clockid_t clk, struct __kernel_timespec *ts)
197{
198 const struct vdso_time_data *vd = vdso_timens_data(vdns);
199 const struct timens_offset *offs = &vcns->offset[clk];
200 const struct vdso_clock *vc = vd->clock_data;
201 const struct vdso_timestamp *vdso_ts;
202 u64 nsec;
203 s64 sec;
204 s32 seq;
205
206 vdso_ts = &vc->basetime[clk];
207
208 do {
209 seq = vdso_read_begin(vc);
210 sec = vdso_ts->sec;
211 nsec = vdso_ts->nsec;
212 } while (vdso_read_retry(vc, seq));
213
214 /* Add the namespace offset */
215 sec += offs->sec;
216 nsec += offs->nsec;
217
218 vdso_set_timespec(ts, sec, nsec);
219
220 return true;
221}
222
223static __always_inline
224bool do_coarse(const struct vdso_time_data *vd, const struct vdso_clock *vc,
225 clockid_t clk, struct __kernel_timespec *ts)
226{
227 const struct vdso_timestamp *vdso_ts = &vc->basetime[clk];
228 u32 seq;
229
230 do {
231 if (vdso_read_begin_timens(vc, &seq))
232 return do_coarse_timens(vd, vc, clk, ts);
233
234 ts->tv_sec = vdso_ts->sec;
235 ts->tv_nsec = vdso_ts->nsec;
236 } while (vdso_read_retry(vc, seq));
237
238 return true;
239}
240
241static __always_inline
242bool do_aux(const struct vdso_time_data *vd, clockid_t clock, struct __kernel_timespec *ts)
243{
244 const struct vdso_clock *vc;
245 u32 seq, idx;
246 u64 sec, ns;
247
248 if (!IS_ENABLED(CONFIG_POSIX_AUX_CLOCKS))
249 return false;
250
251 idx = clock - CLOCK_AUX;
252 vc = &vd->aux_clock_data[idx];
253
254 do {
255 while (vdso_read_begin_timens(vc, &seq)) {
256 /* Re-read from the real time data page, reload seq by looping */
257 vd = vdso_timens_data(vd);
258 vc = &vd->aux_clock_data[idx];
259 }
260
261 /* Auxclock disabled? */
262 if (vc->clock_mode == VDSO_CLOCKMODE_NONE)
263 return false;
264
265 if (!vdso_get_timestamp(vd, vc, VDSO_BASE_AUX, &sec, &ns))
266 return false;
267 } while (vdso_read_retry(vc, seq));
268
269 vdso_set_timespec(ts, sec, ns);
270
271 return true;
272}
273
274static __always_inline bool
275__cvdso_clock_gettime_common(const struct vdso_time_data *vd, clockid_t clock,
276 struct __kernel_timespec *ts)
277{
278 const struct vdso_clock *vc = vd->clock_data;
279 u32 msk;
280
281 if (!vdso_clockid_valid(clock))
282 return false;
283
284 /*
285 * Convert the clockid to a bitmask and use it to check which
286 * clocks are handled in the VDSO directly.
287 */
288 msk = 1U << clock;
289 if (likely(msk & VDSO_HRES))
290 vc = &vc[CS_HRES_COARSE];
291 else if (msk & VDSO_COARSE)
292 return do_coarse(vd, &vc[CS_HRES_COARSE], clock, ts);
293 else if (msk & VDSO_RAW)
294 vc = &vc[CS_RAW];
295 else if (msk & VDSO_AUX)
296 return do_aux(vd, clock, ts);
297 else
298 return false;
299
300 return do_hres(vd, vc, clock, ts);
301}
302
303static int
304__cvdso_clock_gettime_data(const struct vdso_time_data *vd, clockid_t clock,
305 struct __kernel_timespec *ts)
306{
307 bool ok;
308
309 ok = __cvdso_clock_gettime_common(vd, clock, ts);
310
311 if (unlikely(!ok))
312 return clock_gettime_fallback(clock, ts);
313 return 0;
314}
315
316static __maybe_unused int
317__cvdso_clock_gettime(clockid_t clock, struct __kernel_timespec *ts)
318{
319 return __cvdso_clock_gettime_data(__arch_get_vdso_u_time_data(), clock, ts);
320}
321
322#ifdef BUILD_VDSO32
323static int
324__cvdso_clock_gettime32_data(const struct vdso_time_data *vd, clockid_t clock,
325 struct old_timespec32 *res)
326{
327 struct __kernel_timespec ts;
328 bool ok;
329
330 BUILD_BUG_ON(!IS_ENABLED(CONFIG_COMPAT_32BIT_TIME));
331
332 ok = __cvdso_clock_gettime_common(vd, clock, &ts);
333
334 if (unlikely(!ok))
335 return clock_gettime32_fallback(clock, res);
336
337 /* For ok == true */
338 res->tv_sec = ts.tv_sec;
339 res->tv_nsec = ts.tv_nsec;
340
341 return 0;
342}
343
344static __maybe_unused int
345__cvdso_clock_gettime32(clockid_t clock, struct old_timespec32 *res)
346{
347 return __cvdso_clock_gettime32_data(__arch_get_vdso_u_time_data(), clock, res);
348}
349#endif /* BUILD_VDSO32 */
350
351static int
352__cvdso_gettimeofday_data(const struct vdso_time_data *vd,
353 struct __kernel_old_timeval *tv, struct timezone *tz)
354{
355 const struct vdso_clock *vc = vd->clock_data;
356
357#ifndef __NR_gettimeofday
358 BUILD_BUG();
359#endif
360
361 BUILD_BUG_ON(sizeof(tv->tv_sec) != 8 && !IS_ENABLED(CONFIG_COMPAT_32BIT_TIME));
362
363 if (likely(tv != NULL)) {
364 struct __kernel_timespec ts;
365
366 if (!do_hres(vd, &vc[CS_HRES_COARSE], CLOCK_REALTIME, &ts))
367 return gettimeofday_fallback(tv, tz);
368
369 tv->tv_sec = ts.tv_sec;
370 tv->tv_usec = (u32)ts.tv_nsec / NSEC_PER_USEC;
371 }
372
373 if (unlikely(tz != NULL)) {
374 if (vdso_is_timens_clock(vc))
375 vd = vdso_timens_data(vd);
376
377 tz->tz_minuteswest = vd[CS_HRES_COARSE].tz_minuteswest;
378 tz->tz_dsttime = vd[CS_HRES_COARSE].tz_dsttime;
379 }
380
381 return 0;
382}
383
384static __maybe_unused int
385__cvdso_gettimeofday(struct __kernel_old_timeval *tv, struct timezone *tz)
386{
387 return __cvdso_gettimeofday_data(__arch_get_vdso_u_time_data(), tv, tz);
388}
389
390#ifdef VDSO_HAS_TIME
391static __kernel_old_time_t
392__cvdso_time_data(const struct vdso_time_data *vd, __kernel_old_time_t *time)
393{
394 const struct vdso_clock *vc = vd->clock_data;
395 __kernel_old_time_t t;
396
397#ifndef __NR_time
398 BUILD_BUG();
399#endif
400
401 BUILD_BUG_ON(sizeof(*time) != 8 && !IS_ENABLED(CONFIG_COMPAT_32BIT_TIME));
402
403 if (vdso_is_timens_clock(vc)) {
404 vd = vdso_timens_data(vd);
405 vc = vd->clock_data;
406 }
407
408 t = READ_ONCE(vc[CS_HRES_COARSE].basetime[CLOCK_REALTIME].sec);
409
410 if (time)
411 *time = t;
412
413 return t;
414}
415
416static __maybe_unused __kernel_old_time_t __cvdso_time(__kernel_old_time_t *time)
417{
418 return __cvdso_time_data(__arch_get_vdso_u_time_data(), time);
419}
420#endif /* VDSO_HAS_TIME */
421
422#ifdef VDSO_HAS_CLOCK_GETRES
423static __always_inline
424bool __cvdso_clock_getres_common(const struct vdso_time_data *vd, clockid_t clock,
425 struct __kernel_timespec *res)
426{
427 const struct vdso_clock *vc = vd->clock_data;
428 u32 msk;
429 u64 ns;
430
431 if (!vdso_clockid_valid(clock))
432 return false;
433
434 if (vdso_is_timens_clock(vc))
435 vd = vdso_timens_data(vd);
436
437 /*
438 * Convert the clockid to a bitmask and use it to check which
439 * clocks are handled in the VDSO directly.
440 */
441 msk = 1U << clock;
442 if (msk & (VDSO_HRES | VDSO_RAW)) {
443 /*
444 * Preserves the behaviour of posix_get_hrtimer_res().
445 */
446 ns = READ_ONCE(vd->hrtimer_res);
447 } else if (msk & VDSO_COARSE) {
448 /*
449 * Preserves the behaviour of posix_get_coarse_res().
450 */
451 ns = LOW_RES_NSEC;
452 } else if (msk & VDSO_AUX) {
453 ns = aux_clock_resolution_ns();
454 } else {
455 return false;
456 }
457
458 if (likely(res)) {
459 res->tv_sec = 0;
460 res->tv_nsec = ns;
461 }
462 return true;
463}
464
465static
466int __cvdso_clock_getres_data(const struct vdso_time_data *vd, clockid_t clock,
467 struct __kernel_timespec *res)
468{
469 bool ok;
470
471 ok = __cvdso_clock_getres_common(vd, clock, res);
472
473 if (unlikely(!ok))
474 return clock_getres_fallback(clock, res);
475 return 0;
476}
477
478static __maybe_unused
479int __cvdso_clock_getres(clockid_t clock, struct __kernel_timespec *res)
480{
481 return __cvdso_clock_getres_data(__arch_get_vdso_u_time_data(), clock, res);
482}
483
484#ifdef BUILD_VDSO32
485static int
486__cvdso_clock_getres_time32_data(const struct vdso_time_data *vd, clockid_t clock,
487 struct old_timespec32 *res)
488{
489 struct __kernel_timespec ts;
490 bool ok;
491
492 BUILD_BUG_ON(!IS_ENABLED(CONFIG_COMPAT_32BIT_TIME));
493
494 ok = __cvdso_clock_getres_common(vd, clock, &ts);
495
496 if (unlikely(!ok))
497 return clock_getres32_fallback(clock, res);
498
499 if (likely(res)) {
500 res->tv_sec = ts.tv_sec;
501 res->tv_nsec = ts.tv_nsec;
502 }
503 return 0;
504}
505
506static __maybe_unused int
507__cvdso_clock_getres_time32(clockid_t clock, struct old_timespec32 *res)
508{
509 return __cvdso_clock_getres_time32_data(__arch_get_vdso_u_time_data(),
510 clock, res);
511}
512#endif /* BUILD_VDSO32 */
513#endif /* VDSO_HAS_CLOCK_GETRES */