projects/linux·blob assoc-array-fixes · refs

1352 lines · 41.8 KiB

1// SPDX-License-Identifier: GPL-2.0
2/*
3 * NTP state machine interfaces and logic.
4 *
5 * This code was mainly moved from kernel/timer.c and kernel/time.c
6 * Please see those files for relevant copyright info and historical
7 * changelogs.
8 */
9#include <linux/capability.h>
10#include <linux/clocksource.h>
11#include <linux/workqueue.h>
12#include <linux/hrtimer.h>
13#include <linux/jiffies.h>
14#include <linux/math64.h>
15#include <linux/timex.h>
16#include <linux/time.h>
17#include <linux/mm.h>
18#include <linux/module.h>
19#include <linux/rtc.h>
20#include <linux/audit.h>
21#include <linux/timekeeper_internal.h>
22
23#include "ntp_internal.h"
24#include "timekeeping_internal.h"
25
26/**
27 * struct ntp_data - Structure holding all NTP related state
28 * @tick_usec: USER_HZ period in microseconds
29 * @tick_length: Tick length in ns << NTP_SCALE_SHIFT
30 * @time_state: State of the clock synchronization
31 * @time_status: Clock status bits
32 * @time_offset: Time adjustment in nanoseconds
33 * @skew_delta: Per-tick phase slew rate for the coming second, in
34 * @time_offset units (shifted-ns / HZ). Set by
35 * second_overflow().
36 * @time_constant: PLL time constant
37 * @time_maxerror: Maximum error in microseconds holding the NTP sync distance
38 * (NTP dispersion + delay / 2)
39 * @time_esterror: Estimated error in microseconds holding NTP dispersion
40 * @time_freq: Frequency offset scaled nsecs/secs
41 * @time_reftime: Time at last adjustment in seconds
42 * @time_adjust: Adjustment value
43 * @time_adjust_frac: Sub-microsecond remainder of @time_adjust being
44 * delivered, in ns << NTP_SCALE_SHIFT (not divided by HZ).
45 * @ntp_tick_adj: Constant boot-param configurable NTP tick adjustment (upscaled)
46 * @cs_tick_adj: Fixed per-second adjustment compensating for the difference
47 * between the nominal NTP interval and the real time taken
48 * by the clocksource's integer @cycle_interval (upscaled).
49 * Set by the timekeeping core via ntp_clear().
50 * @ntp_next_leap_sec: Second value of the next pending leapsecond, or TIME64_MAX if no leap
51 *
52 * @pps_valid: PPS signal watchdog counter
53 * @pps_tf: PPS phase median filter
54 * @pps_jitter: PPS current jitter in nanoseconds
55 * @pps_fbase: PPS beginning of the last freq interval
56 * @pps_shift: PPS current interval duration in seconds (shift value)
57 * @pps_intcnt: PPS interval counter
58 * @pps_freq: PPS frequency offset in scaled ns/s
59 * @pps_stabil: PPS current stability in scaled ns/s
60 * @pps_calcnt: PPS monitor: calibration intervals
61 * @pps_jitcnt: PPS monitor: jitter limit exceeded
62 * @pps_stbcnt: PPS monitor: stability limit exceeded
63 * @pps_errcnt: PPS monitor: calibration errors
64 *
65 * Protected by the timekeeping locks.
66 */
67struct ntp_data {
68 unsigned long tick_usec;
69 u64 tick_length;
70 int time_state;
71 int time_status;
72 s64 time_offset;
73 s64 skew_delta;
74 long time_constant;
75 long time_maxerror;
76 long time_esterror;
77 s64 time_freq;
78 time64_t time_reftime;
79 long time_adjust;
80 s64 time_adjust_frac;
81 s64 ntp_tick_adj;
82 s64 cs_tick_adj;
83 time64_t ntp_next_leap_sec;
84#ifdef CONFIG_NTP_PPS
85 int pps_valid;
86 long pps_tf[3];
87 long pps_jitter;
88 struct timespec64 pps_fbase;
89 int pps_shift;
90 int pps_intcnt;
91 s64 pps_freq;
92 long pps_stabil;
93 long pps_calcnt;
94 long pps_jitcnt;
95 long pps_stbcnt;
96 long pps_errcnt;
97#endif
98};
99
100static struct ntp_data tk_ntp_data[TIMEKEEPERS_MAX] = {
101 [ 0 ... TIMEKEEPERS_MAX - 1 ] = {
102 .tick_usec = USER_TICK_USEC,
103 .time_state = TIME_OK,
104 .time_status = STA_UNSYNC,
105 .time_constant = 2,
106 .time_maxerror = NTP_PHASE_LIMIT,
107 .time_esterror = NTP_PHASE_LIMIT,
108 .ntp_next_leap_sec = TIME64_MAX,
109 },
110};
111
112#define SECS_PER_DAY 86400
113#define MAX_TICKADJ 500LL /* usecs */
114/* One microsecond of phase, in plain shifted-ns (ns << NTP_SCALE_SHIFT) */
115#define ONE_US_NS ((s64)NSEC_PER_USEC << NTP_SCALE_SHIFT)
116/* Per-tick MAX_TICKADJ slew, in plain shifted-ns */
117#define MAX_TICKADJ_SCALED \
118 (((MAX_TICKADJ * NSEC_PER_USEC) << NTP_SCALE_SHIFT) / NTP_INTERVAL_FREQ)
119#define MAX_TAI_OFFSET 100000
120
121#ifdef CONFIG_NTP_PPS
122
123/*
124 * The following variables are used when a pulse-per-second (PPS) signal
125 * is available. They establish the engineering parameters of the clock
126 * discipline loop when controlled by the PPS signal.
127 */
128#define PPS_VALID 10 /* PPS signal watchdog max (s) */
129#define PPS_POPCORN 4 /* popcorn spike threshold (shift) */
130#define PPS_INTMIN 2 /* min freq interval (s) (shift) */
131#define PPS_INTMAX 8 /* max freq interval (s) (shift) */
132#define PPS_INTCOUNT 4 /* number of consecutive good intervals to
133 increase pps_shift or consecutive bad
134 intervals to decrease it */
135#define PPS_MAXWANDER 100000 /* max PPS freq wander (ns/s) */
136
137/*
138 * PPS kernel consumer compensates the whole phase error immediately.
139 * Otherwise, reduce the offset by a fixed factor times the time constant.
140 */
141static inline s64 ntp_offset_chunk(struct ntp_data *ntpdata, s64 offset)
142{
143 if (ntpdata->time_status & STA_PPSTIME && ntpdata->time_status & STA_PPSSIGNAL)
144 return offset;
145 else
146 return shift_right(offset, SHIFT_PLL + ntpdata->time_constant);
147}
148
149static inline void pps_reset_freq_interval(struct ntp_data *ntpdata)
150{
151 /* The PPS calibration interval may end surprisingly early */
152 ntpdata->pps_shift = PPS_INTMIN;
153 ntpdata->pps_intcnt = 0;
154}
155
156/**
157 * pps_clear - Clears the PPS state variables
158 * @ntpdata: Pointer to ntp data
159 */
160static inline void pps_clear(struct ntp_data *ntpdata)
161{
162 pps_reset_freq_interval(ntpdata);
163 ntpdata->pps_tf[0] = 0;
164 ntpdata->pps_tf[1] = 0;
165 ntpdata->pps_tf[2] = 0;
166 ntpdata->pps_fbase.tv_sec = ntpdata->pps_fbase.tv_nsec = 0;
167 ntpdata->pps_freq = 0;
168}
169
170/*
171 * Decrease pps_valid to indicate that another second has passed since the
172 * last PPS signal. When it reaches 0, indicate that PPS signal is missing.
173 */
174static inline void pps_dec_valid(struct ntp_data *ntpdata)
175{
176 if (ntpdata->pps_valid > 0) {
177 ntpdata->pps_valid--;
178 } else {
179 ntpdata->time_status &= ~(STA_PPSSIGNAL | STA_PPSJITTER |
180 STA_PPSWANDER | STA_PPSERROR);
181 pps_clear(ntpdata);
182 }
183}
184
185static inline void pps_set_freq(struct ntp_data *ntpdata)
186{
187 ntpdata->pps_freq = ntpdata->time_freq;
188}
189
190static inline bool is_error_status(int status)
191{
192 return (status & (STA_UNSYNC|STA_CLOCKERR))
193 /*
194 * PPS signal lost when either PPS time or PPS frequency
195 * synchronization requested
196 */
197 || ((status & (STA_PPSFREQ|STA_PPSTIME))
198 && !(status & STA_PPSSIGNAL))
199 /*
200 * PPS jitter exceeded when PPS time synchronization
201 * requested
202 */
203 || ((status & (STA_PPSTIME|STA_PPSJITTER))
204 == (STA_PPSTIME|STA_PPSJITTER))
205 /*
206 * PPS wander exceeded or calibration error when PPS
207 * frequency synchronization requested
208 */
209 || ((status & STA_PPSFREQ)
210 && (status & (STA_PPSWANDER|STA_PPSERROR)));
211}
212
213static inline void pps_fill_timex(struct ntp_data *ntpdata, struct __kernel_timex *txc)
214{
215 txc->ppsfreq = shift_right((ntpdata->pps_freq >> PPM_SCALE_INV_SHIFT) *
216 PPM_SCALE_INV, NTP_SCALE_SHIFT);
217 txc->jitter = ntpdata->pps_jitter;
218 if (!(ntpdata->time_status & STA_NANO))
219 txc->jitter = ntpdata->pps_jitter / NSEC_PER_USEC;
220 txc->shift = ntpdata->pps_shift;
221 txc->stabil = ntpdata->pps_stabil;
222 txc->jitcnt = ntpdata->pps_jitcnt;
223 txc->calcnt = ntpdata->pps_calcnt;
224 txc->errcnt = ntpdata->pps_errcnt;
225 txc->stbcnt = ntpdata->pps_stbcnt;
226}
227
228#else /* !CONFIG_NTP_PPS */
229
230static inline s64 ntp_offset_chunk(struct ntp_data *ntpdata, s64 offset)
231{
232 return shift_right(offset, SHIFT_PLL + ntpdata->time_constant);
233}
234
235static inline void pps_reset_freq_interval(struct ntp_data *ntpdata) {}
236static inline void pps_clear(struct ntp_data *ntpdata) {}
237static inline void pps_dec_valid(struct ntp_data *ntpdata) {}
238static inline void pps_set_freq(struct ntp_data *ntpdata) {}
239
240static inline bool is_error_status(int status)
241{
242 return status & (STA_UNSYNC|STA_CLOCKERR);
243}
244
245static inline void pps_fill_timex(struct ntp_data *ntpdata, struct __kernel_timex *txc)
246{
247 /* PPS is not implemented, so these are zero */
248 txc->ppsfreq = 0;
249 txc->jitter = 0;
250 txc->shift = 0;
251 txc->stabil = 0;
252 txc->jitcnt = 0;
253 txc->calcnt = 0;
254 txc->errcnt = 0;
255 txc->stbcnt = 0;
256}
257
258#endif /* CONFIG_NTP_PPS */
259
260/*
261 * Update tick_length based on tick_usec, ntp_tick_adj and time_freq:
262 */
263static void ntp_update_frequency(struct ntp_data *ntpdata)
264{
265 u64 second_length, new_base, tick_usec = (u64)ntpdata->tick_usec;
266
267 second_length = (u64)(tick_usec * NSEC_PER_USEC * USER_HZ) << NTP_SCALE_SHIFT;
268
269 second_length += ntpdata->ntp_tick_adj;
270 second_length += ntpdata->cs_tick_adj;
271 second_length += ntpdata->time_freq;
272
273 new_base = div_u64(second_length, NTP_INTERVAL_FREQ);
274
275 /*
276 * Don't wait for the next second_overflow, apply the change to the
277 * tick length immediately:
278 */
279 ntpdata->tick_length = new_base;
280}
281
282static inline s64 ntp_update_offset_fll(struct ntp_data *ntpdata, s64 offset64, long secs)
283{
284 ntpdata->time_status &= ~STA_MODE;
285
286 if (secs < MINSEC)
287 return 0;
288
289 if (!(ntpdata->time_status & STA_FLL) && (secs <= MAXSEC))
290 return 0;
291
292 ntpdata->time_status |= STA_MODE;
293
294 return div64_long(offset64 << (NTP_SCALE_SHIFT - SHIFT_FLL), secs);
295}
296
297static void ntp_update_offset(struct ntp_data *ntpdata, long offset)
298{
299 s64 freq_adj, offset64;
300 long secs, real_secs;
301
302 if (!(ntpdata->time_status & STA_PLL))
303 return;
304
305 if (!(ntpdata->time_status & STA_NANO)) {
306 /* Make sure the multiplication below won't overflow */
307 offset = clamp(offset, -USEC_PER_SEC, USEC_PER_SEC);
308 offset *= NSEC_PER_USEC;
309 }
310
311 /* Scale the phase adjustment and clamp to the operating range. */
312 offset = clamp(offset, -MAXPHASE, MAXPHASE);
313
314 /*
315 * Select how the frequency is to be controlled
316 * and in which mode (PLL or FLL).
317 */
318 real_secs = ktime_get_ntp_seconds(ntpdata - tk_ntp_data);
319 secs = (long)(real_secs - ntpdata->time_reftime);
320 if (unlikely(ntpdata->time_status & STA_FREQHOLD))
321 secs = 0;
322
323 ntpdata->time_reftime = real_secs;
324
325 offset64 = offset;
326 freq_adj = ntp_update_offset_fll(ntpdata, offset64, secs);
327
328 /*
329 * Clamp update interval to reduce PLL gain with low
330 * sampling rate (e.g. intermittent network connection)
331 * to avoid instability.
332 */
333 if (unlikely(secs > 1 << (SHIFT_PLL + 1 + ntpdata->time_constant)))
334 secs = 1 << (SHIFT_PLL + 1 + ntpdata->time_constant);
335
336 freq_adj += (offset64 * secs) <<
337 (NTP_SCALE_SHIFT - 2 * (SHIFT_PLL + 2 + ntpdata->time_constant));
338
339 freq_adj = min(freq_adj + ntpdata->time_freq, MAXFREQ_SCALED);
340
341 ntpdata->time_freq = max(freq_adj, -MAXFREQ_SCALED);
342
343 ntpdata->time_offset = div_s64(offset64 << NTP_SCALE_SHIFT, NTP_INTERVAL_FREQ);
344}
345
346static void __ntp_clear(struct ntp_data *ntpdata)
347{
348 /* Stop active adjtime() */
349 ntpdata->time_adjust = 0;
350 ntpdata->time_adjust_frac = 0;
351 ntpdata->time_status |= STA_UNSYNC;
352 ntpdata->time_maxerror = NTP_PHASE_LIMIT;
353 ntpdata->time_esterror = NTP_PHASE_LIMIT;
354
355 ntp_update_frequency(ntpdata);
356
357 ntpdata->time_offset = 0;
358 ntpdata->skew_delta = 0;
359
360 ntpdata->ntp_next_leap_sec = TIME64_MAX;
361 /* Clear PPS state variables */
362 pps_clear(ntpdata);
363}
364
365/**
366 * ntp_clear - Clear NTP state and set the clocksource quantisation adjustment
367 * @tkid: Timekeeper ID
368 * @cs_tick_adj: Per-second adjustment in ns << NTP_SCALE_SHIFT
369 *
370 * The timekeeping core uses an integer number of cycles (@cycle_interval)
371 * per NTP interval, so the real time that interval represents differs from
372 * the nominal NTP_INTERVAL_LENGTH by up to half a counter period. Folding
373 * this fixed offset into @cs_tick_adj makes it an explicit part of the NTP
374 * tick_length computation in ntp.c, instead of being applied during
375 * timekeeping accumulation where the NTP code never saw it. Like
376 * @ntp_tick_adj it stays internal to the kernel; userspace still sees the
377 * nominal tick via adjtimex. NTP retains its full symmetric ±MAXFREQ range
378 * around the corrected base rate.
379 *
380 * Called whenever the clocksource is (re)configured, which is also when the
381 * rest of the NTP state must be cleared, so the two are done together.
382 */
383void ntp_clear(unsigned int tkid, s64 cs_tick_adj)
384{
385 tk_ntp_data[tkid].cs_tick_adj = cs_tick_adj;
386 __ntp_clear(&tk_ntp_data[tkid]);
387}
388
389
390u64 ntp_tick_length(unsigned int tkid)
391{
392 return tk_ntp_data[tkid].tick_length;
393}
394
395s64 ntp_get_skew_delta(unsigned int tkid)
396{
397 return tk_ntp_data[tkid].skew_delta;
398}
399
400/* Sign of @x as +1 or -1 (zero counts as positive; callers pass nonzero). */
401static inline int signof(s64 x)
402{
403 return x < 0 ? -1 : 1;
404}
405
406static s64 ntp_drain_time_offset(unsigned int tkid, s64 amount)
407{
408 struct ntp_data *ntpdata = &tk_ntp_data[tkid];
409
410 /* Only drain if amount and time_offset have the same sign */
411 if (!amount || signof(amount) != signof(ntpdata->time_offset))
412 return amount;
413
414 /* Clamp: don't overshoot zero */
415 if (abs(amount) > abs(ntpdata->time_offset)) {
416 s64 undrained = amount - ntpdata->time_offset;
417
418 ntpdata->time_offset = 0;
419 return undrained;
420 }
421
422 ntpdata->time_offset -= amount;
423 return 0;
424}
425
426/*
427 * Drain the legacy adjtime() correction (time_adjust) as it is delivered.
428 *
429 * @amount is the total intentional per-tick skew for this accumulation
430 * (skew_delta << shift), in time_offset units (shifted_ns / HZ); it covers
431 * both the exponential time_offset slew and the linear adjtime slew. This
432 * function claims only the adjtime share — capped at the MAX_TICKADJ rate —
433 * and returns the remainder for ntp_drain_time_offset().
434 *
435 * time_adjust is in whole µs. The sub-µs remainder being delivered lives in
436 * time_adjust_frac (plain shifted-ns, i.e. ns << NTP_SCALE_SHIFT -- unlike
437 * time_offset these are NOT pre-divided by HZ); we top it up by borrowing
438 * whole microseconds from time_adjust as the drain consumes it.
439 */
440static s64 ntp_drain_time_adjust(unsigned int tkid, s64 amount, unsigned int shift)
441{
442 struct ntp_data *ntpdata = &tk_ntp_data[tkid];
443 /* Sign reference: time_adjust if any whole us remain, else the drawer */
444 s64 ref = ntpdata->time_adjust ? (s64)ntpdata->time_adjust
445 : ntpdata->time_adjust_frac;
446 s64 deliver, deficit, claimed;
447
448 if (!amount || !ref || signof(amount) != signof(ref))
449 return amount;
450
451 /*
452 * Phase to deliver this accumulation, in plain shifted-ns. The drain
453 * @amount is in ÷HZ units, so multiply by HZ first, then clamp to the
454 * MAX_TICKADJ rate (MAX_TICKADJ_SCALED is the per-tick slew in
455 * shifted-ns). Multiply-then-clamp avoids an s64 divide for the cap.
456 */
457 deliver = min(abs(amount) * NTP_INTERVAL_FREQ,
458 (s64)MAX_TICKADJ_SCALED << shift);
459
460 /* Top up the sub-µs drawer from whole-µs time_adjust as needed */
461 deficit = deliver - abs(ntpdata->time_adjust_frac);
462 if (deficit > 0 && ntpdata->time_adjust) {
463 long borrow = div64_u64(deficit + ONE_US_NS - 1, ONE_US_NS);
464
465 if (ntpdata->time_adjust > 0) {
466 borrow = min(borrow, ntpdata->time_adjust);
467 ntpdata->time_adjust -= borrow;
468 ntpdata->time_adjust_frac += (s64)borrow * ONE_US_NS;
469 } else {
470 /* Clamp without negating time_adjust (UB for LONG_MIN) */
471 if (ntpdata->time_adjust > -borrow)
472 borrow = -ntpdata->time_adjust;
473 ntpdata->time_adjust += borrow;
474 ntpdata->time_adjust_frac -= (s64)borrow * ONE_US_NS;
475 }
476 }
477
478 /* Never deliver more than the drawer holds */
479 deliver = min(deliver, abs(ntpdata->time_adjust_frac));
480 if (ntpdata->time_adjust_frac > 0)
481 ntpdata->time_adjust_frac -= deliver;
482 else
483 ntpdata->time_adjust_frac += deliver;
484
485 /* Return the unclaimed remainder in ÷HZ drain units for time_offset */
486 claimed = div_s64(deliver, NTP_INTERVAL_FREQ);
487 return amount - signof(amount) * claimed;
488}
489
490/*
491 * Drain one accumulation's worth of intentional skew as it is delivered.
492 *
493 * @amount is the total intentional per-tick skew for this accumulation
494 * (skew_delta << shift), in time_offset units (shifted_ns / HZ). The
495 * adjtime() linear share is taken from time_adjust first (capped at the
496 * MAX_TICKADJ rate, hence @shift), then the exponential remainder from
497 * time_offset. Returns the amount actually claimed (same ÷HZ units).
498 */
499s64 ntp_drain_skew(unsigned int tkid, s64 amount, unsigned int shift)
500{
501 s64 unclaimed = ntp_drain_time_adjust(tkid, amount, shift);
502
503 unclaimed = ntp_drain_time_offset(tkid, unclaimed);
504
505 /*
506 * Return the amount actually drained from the intentional
507 * phase offset in time_offset and/or time_adjust.
508 */
509 return amount - unclaimed;
510}
511
512/*
513 * time_offset (drained exponentially) and time_adjust (drained linearly at the
514 * MAX_TICKADJ rate) can be asked to slew the clock in opposite directions.
515 * second_overflow() only folds their *net* into skew_delta, so the cancelling
516 * part would never be drained from either tracker via the per-tick code -- and
517 * if they cancel exactly, skew_delta is zero and neither converges at all.
518 *
519 * Settle that cancelling phase directly between the two here. No clock motion
520 * results (the opposing slews annihilate), but both move toward zero so neither
521 * stalls. @amount is the phase to take off time_offset, in its (÷HZ) units and
522 * with its sign; the same real magnitude comes off time_adjust in the opposite
523 * direction. Clamped so neither tracker is driven past zero.
524 */
525static void ntp_transfer_offset_adjust(struct ntp_data *ntpdata, s64 amount)
526{
527 s64 frac_delta, carry;
528
529 /*
530 * Don't drain time_offset past zero. @amount shares its sign and is
531 * normally bounded below it by ntp_offset_chunk(), but the ±1 skew_delta
532 * floor for a tiny time_offset can exceed it, so clamp.
533 */
534 if (abs(amount) > abs(ntpdata->time_offset))
535 amount = ntpdata->time_offset;
536 if (!amount)
537 return;
538
539 /*
540 * Remove the matching phase from time_adjust, in plain shifted-ns. No
541 * clamp against time_adjust's zero is needed: @amount is bounded by the
542 * adjtime chunk, which second_overflow() never lets exceed time_adjust's
543 * own pending phase, so this cannot overshoot.
544 */
545 frac_delta = amount * NTP_INTERVAL_FREQ;
546
547 ntpdata->time_offset -= amount;
548
549 /* Add the matching phase to time_adjust, carrying whole µs (O(1)). */
550 ntpdata->time_adjust_frac += frac_delta;
551 if (ntpdata->time_adjust_frac >= ONE_US_NS ||
552 ntpdata->time_adjust_frac <= -ONE_US_NS) {
553 carry = div64_s64(ntpdata->time_adjust_frac, ONE_US_NS);
554 ntpdata->time_adjust += carry;
555 ntpdata->time_adjust_frac -= carry * ONE_US_NS;
556 }
557
558 /*
559 * Keep time_adjust and its sub-µs remainder the same sign. The
560 * truncating carry above can leave them opposed (e.g. +4 µs paired
561 * with -250 ns), and ntp_drain_time_adjust() treats abs(time_adjust_frac)
562 * as same-direction drawer capacity -- an opposing remainder there makes
563 * it over-deliver phase that was never removed from the pile. Borrow or
564 * repay a single whole µs to realign; the total phase is unchanged.
565 */
566 if (ntpdata->time_adjust > 0 && ntpdata->time_adjust_frac < 0) {
567 ntpdata->time_adjust--;
568 ntpdata->time_adjust_frac += ONE_US_NS;
569 } else if (ntpdata->time_adjust < 0 && ntpdata->time_adjust_frac > 0) {
570 ntpdata->time_adjust++;
571 ntpdata->time_adjust_frac -= ONE_US_NS;
572 }
573}
574
575/**
576 * ntp_get_next_leap - Returns the next leapsecond in CLOCK_REALTIME ktime_t
577 * @tkid: Timekeeper ID
578 *
579 * Returns: For @tkid == TIMEKEEPER_CORE this provides the time of the next
580 * leap second against CLOCK_REALTIME in a ktime_t format if a
581 * leap second is pending. KTIME_MAX otherwise.
582 */
583ktime_t ntp_get_next_leap(unsigned int tkid)
584{
585 struct ntp_data *ntpdata = &tk_ntp_data[TIMEKEEPER_CORE];
586
587 if (tkid != TIMEKEEPER_CORE)
588 return KTIME_MAX;
589
590 if ((ntpdata->time_state == TIME_INS) && (ntpdata->time_status & STA_INS))
591 return ktime_set(ntpdata->ntp_next_leap_sec, 0);
592
593 return KTIME_MAX;
594}
595
596/*
597 * This routine handles the overflow of the microsecond field
598 *
599 * The tricky bits of code to handle the accurate clock support
600 * were provided by Dave Mills ([email protected]) of NTP fame.
601 * They were originally developed for SUN and DEC kernels.
602 * All the kudos should go to Dave for this stuff.
603 *
604 * Also handles leap second processing, and returns leap offset
605 */
606int second_overflow(unsigned int tkid, time64_t secs)
607{
608 struct ntp_data *ntpdata = &tk_ntp_data[tkid];
609 int leap = 0;
610 s32 rem;
611
612 /*
613 * Leap second processing. If in leap-insert state at the end of the
614 * day, the system clock is set back one second; if in leap-delete
615 * state, the system clock is set ahead one second.
616 */
617 switch (ntpdata->time_state) {
618 case TIME_OK:
619 if (ntpdata->time_status & STA_INS) {
620 ntpdata->time_state = TIME_INS;
621 div_s64_rem(secs, SECS_PER_DAY, &rem);
622 ntpdata->ntp_next_leap_sec = secs + SECS_PER_DAY - rem;
623 } else if (ntpdata->time_status & STA_DEL) {
624 ntpdata->time_state = TIME_DEL;
625 div_s64_rem(secs + 1, SECS_PER_DAY, &rem);
626 ntpdata->ntp_next_leap_sec = secs + SECS_PER_DAY - rem;
627 }
628 break;
629 case TIME_INS:
630 if (!(ntpdata->time_status & STA_INS)) {
631 ntpdata->ntp_next_leap_sec = TIME64_MAX;
632 ntpdata->time_state = TIME_OK;
633 } else if (secs == ntpdata->ntp_next_leap_sec) {
634 leap = -1;
635 ntpdata->time_state = TIME_OOP;
636 pr_notice("Clock: inserting leap second 23:59:60 UTC\n");
637 }
638 break;
639 case TIME_DEL:
640 if (!(ntpdata->time_status & STA_DEL)) {
641 ntpdata->ntp_next_leap_sec = TIME64_MAX;
642 ntpdata->time_state = TIME_OK;
643 } else if (secs == ntpdata->ntp_next_leap_sec) {
644 leap = 1;
645 ntpdata->ntp_next_leap_sec = TIME64_MAX;
646 ntpdata->time_state = TIME_WAIT;
647 pr_notice("Clock: deleting leap second 23:59:59 UTC\n");
648 }
649 break;
650 case TIME_OOP:
651 ntpdata->ntp_next_leap_sec = TIME64_MAX;
652 ntpdata->time_state = TIME_WAIT;
653 break;
654 case TIME_WAIT:
655 if (!(ntpdata->time_status & (STA_INS | STA_DEL)))
656 ntpdata->time_state = TIME_OK;
657 break;
658 }
659
660 /* Bump the maxerror field */
661 ntpdata->time_maxerror += MAXFREQ / NSEC_PER_USEC;
662 if (ntpdata->time_maxerror > NTP_PHASE_LIMIT) {
663 ntpdata->time_maxerror = NTP_PHASE_LIMIT;
664 ntpdata->time_status |= STA_UNSYNC;
665 }
666
667 /* Compute the phase adjustment for the next second */
668
669 /* Check PPS signal */
670 pps_dec_valid(ntpdata);
671
672 /*
673 * Set the per-tick skew rate for the next second. This is in
674 * the same units as time_offset: (ns << NTP_SCALE_SHIFT) / HZ.
675 * If the result is so low that the skew imparted would round
676 * to zero, pass the bare minimum ±1 to ensure that it *does*
677 * actually drain completely to zero. It won't overshoot because
678 * logarithmic_accumulation() only drains what it can from
679 * time_offset or time_adjust, and the rest ends up in ntp_error
680 * which drives the selection of 'mult' immediately each tick.
681 */
682 if (ntpdata->time_offset || ntpdata->time_adjust ||
683 ntpdata->time_adjust_frac) {
684 s64 off_chunk = ntp_offset_chunk(ntpdata, ntpdata->time_offset);
685 s64 adj_chunk = 0, net;
686
687 /*
688 * Once the exponential chunk rounds to zero, deliver the last
689 * remaining offset this second so it converges to zero instead
690 * of stalling just above it.
691 */
692 if (!off_chunk)
693 off_chunk = ntpdata->time_offset;
694
695 if (ntpdata->time_adjust || ntpdata->time_adjust_frac) {
696 s64 adj;
697
698 if (ntpdata->time_adjust >= MAX_TICKADJ)
699 adj = MAX_TICKADJ * ONE_US_NS;
700 else if (ntpdata->time_adjust <= -MAX_TICKADJ)
701 adj = -MAX_TICKADJ * ONE_US_NS;
702 else
703 adj = ntpdata->time_adjust * ONE_US_NS +
704 ntpdata->time_adjust_frac;
705
706 adj_chunk = div_s64(adj, NTP_INTERVAL_FREQ);
707 if (!adj_chunk)
708 adj_chunk = signof(ntpdata->time_adjust_frac);
709 }
710
711 /*
712 * If the two slews oppose, only their net would drive the
713 * per-tick drain, so the cancelling part would never drain from
714 * either tracker and an exact cancellation would stall both.
715 * Settle that overlap directly between them (no clock motion).
716 */
717 if (off_chunk && adj_chunk && signof(off_chunk) != signof(adj_chunk)) {
718 s64 conflict = min(abs(off_chunk), abs(adj_chunk));
719
720 ntp_transfer_offset_adjust(ntpdata, signof(off_chunk) * conflict);
721 }
722
723 /* Net is what the clock delivers; reduce to per-tick, then floor. */
724 net = off_chunk + adj_chunk;
725 ntpdata->skew_delta = div_s64(net, NTP_INTERVAL_FREQ);
726 if (!ntpdata->skew_delta && net)
727 ntpdata->skew_delta = signof(net);
728 } else {
729 ntpdata->skew_delta = 0;
730 }
731
732 return leap;
733}
734
735#if defined(CONFIG_GENERIC_CMOS_UPDATE) || defined(CONFIG_RTC_SYSTOHC)
736static void sync_hw_clock(struct work_struct *work);
737static DECLARE_WORK(sync_work, sync_hw_clock);
738static struct hrtimer sync_hrtimer;
739#define SYNC_PERIOD_NS (11ULL * 60 * NSEC_PER_SEC)
740
741static enum hrtimer_restart sync_timer_callback(struct hrtimer *timer)
742{
743 queue_work(system_freezable_power_efficient_wq, &sync_work);
744
745 return HRTIMER_NORESTART;
746}
747
748static void sched_sync_hw_clock(unsigned long offset_nsec, bool retry)
749{
750 ktime_t exp = ktime_set(ktime_get_real_seconds(), 0);
751
752 if (retry)
753 exp = ktime_add_ns(exp, 2ULL * NSEC_PER_SEC - offset_nsec);
754 else
755 exp = ktime_add_ns(exp, SYNC_PERIOD_NS - offset_nsec);
756
757 hrtimer_start(&sync_hrtimer, exp, HRTIMER_MODE_ABS);
758}
759
760/*
761 * Check whether @now is correct versus the required time to update the RTC
762 * and calculate the value which needs to be written to the RTC so that the
763 * next seconds increment of the RTC after the write is aligned with the next
764 * seconds increment of clock REALTIME.
765 *
766 * tsched t1 write(t2.tv_sec - 1sec)) t2 RTC increments seconds
767 *
768 * t2.tv_nsec == 0
769 * tsched = t2 - set_offset_nsec
770 * newval = t2 - NSEC_PER_SEC
771 *
772 * ==> neval = tsched + set_offset_nsec - NSEC_PER_SEC
773 *
774 * As the execution of this code is not guaranteed to happen exactly at
775 * tsched this allows it to happen within a fuzzy region:
776 *
777 * abs(now - tsched) < FUZZ
778 *
779 * If @now is not inside the allowed window the function returns false.
780 */
781static inline bool rtc_tv_nsec_ok(unsigned long set_offset_nsec,
782 struct timespec64 *to_set,
783 const struct timespec64 *now)
784{
785 /* Allowed error in tv_nsec, arbitrarily set to 5 jiffies in ns. */
786 const unsigned long TIME_SET_NSEC_FUZZ = TICK_NSEC * 5;
787 struct timespec64 delay = {.tv_sec = -1,
788 .tv_nsec = set_offset_nsec};
789
790 *to_set = timespec64_add(*now, delay);
791
792 if (to_set->tv_nsec < TIME_SET_NSEC_FUZZ) {
793 to_set->tv_nsec = 0;
794 return true;
795 }
796
797 if (to_set->tv_nsec > NSEC_PER_SEC - TIME_SET_NSEC_FUZZ) {
798 to_set->tv_sec++;
799 to_set->tv_nsec = 0;
800 return true;
801 }
802 return false;
803}
804
805#ifdef CONFIG_GENERIC_CMOS_UPDATE
806int __weak update_persistent_clock64(struct timespec64 now64)
807{
808 return -ENODEV;
809}
810#else
811static inline int update_persistent_clock64(struct timespec64 now64)
812{
813 return -ENODEV;
814}
815#endif
816
817#ifdef CONFIG_RTC_SYSTOHC
818/* Save NTP synchronized time to the RTC */
819static int update_rtc(struct timespec64 *to_set, unsigned long *offset_nsec)
820{
821 struct rtc_device *rtc;
822 struct rtc_time tm;
823 int err = -ENODEV;
824
825 rtc = rtc_class_open(CONFIG_RTC_SYSTOHC_DEVICE);
826 if (!rtc)
827 return -ENODEV;
828
829 if (!rtc->ops || !rtc->ops->set_time)
830 goto out_close;
831
832 /* First call might not have the correct offset */
833 if (*offset_nsec == rtc->set_offset_nsec) {
834 rtc_time64_to_tm(to_set->tv_sec, &tm);
835 err = rtc_set_time(rtc, &tm);
836 } else {
837 /* Store the update offset and let the caller try again */
838 *offset_nsec = rtc->set_offset_nsec;
839 err = -EAGAIN;
840 }
841out_close:
842 rtc_class_close(rtc);
843 return err;
844}
845#else
846static inline int update_rtc(struct timespec64 *to_set, unsigned long *offset_nsec)
847{
848 return -ENODEV;
849}
850#endif
851
852/**
853 * ntp_synced - Tells whether the NTP status is not UNSYNC
854 * Returns: true if not UNSYNC, false otherwise
855 */
856static inline bool ntp_synced(void)
857{
858 return !(tk_ntp_data[TIMEKEEPER_CORE].time_status & STA_UNSYNC);
859}
860
861/*
862 * If we have an externally synchronized Linux clock, then update RTC clock
863 * accordingly every ~11 minutes. Generally RTCs can only store second
864 * precision, but many RTCs will adjust the phase of their second tick to
865 * match the moment of update. This infrastructure arranges to call to the RTC
866 * set at the correct moment to phase synchronize the RTC second tick over
867 * with the kernel clock.
868 */
869static void sync_hw_clock(struct work_struct *work)
870{
871 /*
872 * The default synchronization offset is 500ms for the deprecated
873 * update_persistent_clock64() under the assumption that it uses
874 * the infamous CMOS clock (MC146818).
875 */
876 static unsigned long offset_nsec = NSEC_PER_SEC / 2;
877 struct timespec64 now, to_set;
878 int res = -EAGAIN;
879
880 /*
881 * Don't update if STA_UNSYNC is set and if ntp_notify_cmos_timer()
882 * managed to schedule the work between the timer firing and the
883 * work being able to rearm the timer. Wait for the timer to expire.
884 */
885 if (!ntp_synced() || hrtimer_is_queued(&sync_hrtimer))
886 return;
887
888 ktime_get_real_ts64(&now);
889 /* If @now is not in the allowed window, try again */
890 if (!rtc_tv_nsec_ok(offset_nsec, &to_set, &now))
891 goto rearm;
892
893 /* Take timezone adjusted RTCs into account */
894 if (persistent_clock_is_local)
895 to_set.tv_sec -= (sys_tz.tz_minuteswest * 60);
896
897 /* Try the legacy RTC first. */
898 res = update_persistent_clock64(to_set);
899 if (res != -ENODEV)
900 goto rearm;
901
902 /* Try the RTC class */
903 res = update_rtc(&to_set, &offset_nsec);
904 if (res == -ENODEV)
905 return;
906rearm:
907 sched_sync_hw_clock(offset_nsec, res != 0);
908}
909
910void ntp_notify_cmos_timer(bool offset_set)
911{
912 /*
913 * If the time jumped (using ADJ_SETOFFSET) cancels sync timer,
914 * which may have been running if the time was synchronized
915 * prior to the ADJ_SETOFFSET call.
916 */
917 if (offset_set)
918 hrtimer_cancel(&sync_hrtimer);
919
920 /*
921 * When the work is currently executed but has not yet the timer
922 * rearmed this queues the work immediately again. No big issue,
923 * just a pointless work scheduled.
924 */
925 if (ntp_synced() && !hrtimer_is_queued(&sync_hrtimer))
926 queue_work(system_freezable_power_efficient_wq, &sync_work);
927}
928
929static void __init ntp_init_cmos_sync(void)
930{
931 hrtimer_setup(&sync_hrtimer, sync_timer_callback, CLOCK_REALTIME, HRTIMER_MODE_ABS);
932}
933#else /* CONFIG_GENERIC_CMOS_UPDATE) || defined(CONFIG_RTC_SYSTOHC) */
934static inline void __init ntp_init_cmos_sync(void) { }
935#endif /* !CONFIG_GENERIC_CMOS_UPDATE) || defined(CONFIG_RTC_SYSTOHC) */
936
937/*
938 * Propagate a new txc->status value into the NTP state:
939 */
940static inline void process_adj_status(struct ntp_data *ntpdata, const struct __kernel_timex *txc)
941{
942 if ((ntpdata->time_status & STA_PLL) && !(txc->status & STA_PLL)) {
943 ntpdata->time_state = TIME_OK;
944 ntpdata->time_status = STA_UNSYNC;
945 ntpdata->ntp_next_leap_sec = TIME64_MAX;
946 /* Restart PPS frequency calibration */
947 pps_reset_freq_interval(ntpdata);
948 }
949
950 /*
951 * If we turn on PLL adjustments then reset the
952 * reference time to current time.
953 */
954 if (!(ntpdata->time_status & STA_PLL) && (txc->status & STA_PLL))
955 ntpdata->time_reftime = ktime_get_ntp_seconds(ntpdata - tk_ntp_data);
956
957 /* only set allowed bits */
958 ntpdata->time_status &= STA_RONLY;
959 ntpdata->time_status |= txc->status & ~STA_RONLY;
960}
961
962static inline void process_adjtimex_modes(struct ntp_data *ntpdata, const struct __kernel_timex *txc,
963 s32 *time_tai)
964{
965 if (txc->modes & ADJ_STATUS)
966 process_adj_status(ntpdata, txc);
967
968 if (txc->modes & ADJ_NANO)
969 ntpdata->time_status |= STA_NANO;
970
971 if (txc->modes & ADJ_MICRO)
972 ntpdata->time_status &= ~STA_NANO;
973
974 if (txc->modes & ADJ_FREQUENCY) {
975 ntpdata->time_freq = txc->freq * PPM_SCALE;
976 ntpdata->time_freq = min(ntpdata->time_freq, MAXFREQ_SCALED);
977 ntpdata->time_freq = max(ntpdata->time_freq, -MAXFREQ_SCALED);
978 /* Update pps_freq */
979 pps_set_freq(ntpdata);
980 }
981
982 if (txc->modes & ADJ_MAXERROR)
983 ntpdata->time_maxerror = clamp(txc->maxerror, 0, NTP_PHASE_LIMIT);
984
985 if (txc->modes & ADJ_ESTERROR)
986 ntpdata->time_esterror = clamp(txc->esterror, 0, NTP_PHASE_LIMIT);
987
988 if (txc->modes & ADJ_TIMECONST) {
989 ntpdata->time_constant = clamp(txc->constant, 0, MAXTC);
990 if (!(ntpdata->time_status & STA_NANO))
991 ntpdata->time_constant += 4;
992 ntpdata->time_constant = clamp(ntpdata->time_constant, 0, MAXTC);
993 }
994
995 if (txc->modes & ADJ_TAI && txc->constant >= 0 && txc->constant <= MAX_TAI_OFFSET)
996 *time_tai = txc->constant;
997
998 if (txc->modes & ADJ_OFFSET)
999 ntp_update_offset(ntpdata, txc->offset);
1000
1001 if (txc->modes & ADJ_TICK)
1002 ntpdata->tick_usec = txc->tick;
1003
1004 if (txc->modes & (ADJ_TICK|ADJ_FREQUENCY|ADJ_OFFSET))
1005 ntp_update_frequency(ntpdata);
1006}
1007
1008/*
1009 * adjtimex() mainly allows reading (and writing, if superuser) of
1010 * kernel time-keeping variables. used by xntpd.
1011 */
1012int ntp_adjtimex(unsigned int tkid, struct __kernel_timex *txc, const struct timespec64 *ts,
1013 s32 *time_tai, struct audit_ntp_data *ad)
1014{
1015 struct ntp_data *ntpdata = &tk_ntp_data[tkid];
1016 int result;
1017
1018 if (txc->modes & ADJ_ADJTIME) {
1019 long save_adjust = ntpdata->time_adjust;
1020
1021 if (!(txc->modes & ADJ_OFFSET_READONLY)) {
1022 /* adjtime() is independent from ntp_adjtime() */
1023 ntpdata->time_adjust = txc->offset;
1024 ntpdata->time_adjust_frac = 0;
1025 ntp_update_frequency(ntpdata);
1026
1027 audit_ntp_set_old(ad, AUDIT_NTP_ADJUST, save_adjust);
1028 audit_ntp_set_new(ad, AUDIT_NTP_ADJUST, ntpdata->time_adjust);
1029 }
1030 txc->offset = save_adjust;
1031 } else {
1032 /* If there are input parameters, then process them: */
1033 if (txc->modes) {
1034 audit_ntp_set_old(ad, AUDIT_NTP_OFFSET, ntpdata->time_offset);
1035 audit_ntp_set_old(ad, AUDIT_NTP_FREQ, ntpdata->time_freq);
1036 audit_ntp_set_old(ad, AUDIT_NTP_STATUS, ntpdata->time_status);
1037 audit_ntp_set_old(ad, AUDIT_NTP_TAI, *time_tai);
1038 audit_ntp_set_old(ad, AUDIT_NTP_TICK, ntpdata->tick_usec);
1039
1040 process_adjtimex_modes(ntpdata, txc, time_tai);
1041
1042 audit_ntp_set_new(ad, AUDIT_NTP_OFFSET, ntpdata->time_offset);
1043 audit_ntp_set_new(ad, AUDIT_NTP_FREQ, ntpdata->time_freq);
1044 audit_ntp_set_new(ad, AUDIT_NTP_STATUS, ntpdata->time_status);
1045 audit_ntp_set_new(ad, AUDIT_NTP_TAI, *time_tai);
1046 audit_ntp_set_new(ad, AUDIT_NTP_TICK, ntpdata->tick_usec);
1047 }
1048
1049 txc->offset = shift_right(ntpdata->time_offset * NTP_INTERVAL_FREQ, NTP_SCALE_SHIFT);
1050 if (!(ntpdata->time_status & STA_NANO))
1051 txc->offset = div_s64(txc->offset, NSEC_PER_USEC);
1052 }
1053
1054 result = ntpdata->time_state;
1055 if (is_error_status(ntpdata->time_status))
1056 result = TIME_ERROR;
1057
1058 txc->freq = shift_right((ntpdata->time_freq >> PPM_SCALE_INV_SHIFT) *
1059 PPM_SCALE_INV, NTP_SCALE_SHIFT);
1060 txc->maxerror = ntpdata->time_maxerror;
1061 txc->esterror = ntpdata->time_esterror;
1062 txc->status = ntpdata->time_status;
1063 txc->constant = ntpdata->time_constant;
1064 txc->precision = 1;
1065 txc->tolerance = MAXFREQ_SCALED / PPM_SCALE;
1066 txc->tick = ntpdata->tick_usec;
1067 txc->tai = *time_tai;
1068
1069 /* Fill PPS status fields */
1070 pps_fill_timex(ntpdata, txc);
1071
1072 txc->time.tv_sec = ts->tv_sec;
1073 txc->time.tv_usec = ts->tv_nsec;
1074 if (!(ntpdata->time_status & STA_NANO))
1075 txc->time.tv_usec = ts->tv_nsec / NSEC_PER_USEC;
1076
1077 /* Handle leapsec adjustments */
1078 if (unlikely(ts->tv_sec >= ntpdata->ntp_next_leap_sec)) {
1079 if ((ntpdata->time_state == TIME_INS) && (ntpdata->time_status & STA_INS)) {
1080 result = TIME_OOP;
1081 txc->tai++;
1082 txc->time.tv_sec--;
1083 }
1084 if ((ntpdata->time_state == TIME_DEL) && (ntpdata->time_status & STA_DEL)) {
1085 result = TIME_WAIT;
1086 txc->tai--;
1087 txc->time.tv_sec++;
1088 }
1089 if ((ntpdata->time_state == TIME_OOP) && (ts->tv_sec == ntpdata->ntp_next_leap_sec))
1090 result = TIME_WAIT;
1091 }
1092
1093 return result;
1094}
1095
1096#ifdef CONFIG_NTP_PPS
1097
1098/*
1099 * struct pps_normtime is basically a struct timespec, but it is
1100 * semantically different (and it is the reason why it was invented):
1101 * pps_normtime.nsec has a range of ( -NSEC_PER_SEC / 2, NSEC_PER_SEC / 2 ]
1102 * while timespec.tv_nsec has a range of [0, NSEC_PER_SEC)
1103 */
1104struct pps_normtime {
1105 s64 sec; /* seconds */
1106 long nsec; /* nanoseconds */
1107};
1108
1109/*
1110 * Normalize the timestamp so that nsec is in the
1111 * [ -NSEC_PER_SEC / 2, NSEC_PER_SEC / 2 ] interval
1112 */
1113static inline struct pps_normtime pps_normalize_ts(struct timespec64 ts)
1114{
1115 struct pps_normtime norm = {
1116 .sec = ts.tv_sec,
1117 .nsec = ts.tv_nsec
1118 };
1119
1120 if (norm.nsec > (NSEC_PER_SEC >> 1)) {
1121 norm.nsec -= NSEC_PER_SEC;
1122 norm.sec++;
1123 }
1124
1125 return norm;
1126}
1127
1128/* Get current phase correction and jitter */
1129static inline long pps_phase_filter_get(struct ntp_data *ntpdata, long *jitter)
1130{
1131 *jitter = ntpdata->pps_tf[0] - ntpdata->pps_tf[1];
1132 if (*jitter < 0)
1133 *jitter = -*jitter;
1134
1135 /* TODO: test various filters */
1136 return ntpdata->pps_tf[0];
1137}
1138
1139/* Add the sample to the phase filter */
1140static inline void pps_phase_filter_add(struct ntp_data *ntpdata, long err)
1141{
1142 ntpdata->pps_tf[2] = ntpdata->pps_tf[1];
1143 ntpdata->pps_tf[1] = ntpdata->pps_tf[0];
1144 ntpdata->pps_tf[0] = err;
1145}
1146
1147/*
1148 * Decrease frequency calibration interval length. It is halved after four
1149 * consecutive unstable intervals.
1150 */
1151static inline void pps_dec_freq_interval(struct ntp_data *ntpdata)
1152{
1153 if (--ntpdata->pps_intcnt <= -PPS_INTCOUNT) {
1154 ntpdata->pps_intcnt = -PPS_INTCOUNT;
1155 if (ntpdata->pps_shift > PPS_INTMIN) {
1156 ntpdata->pps_shift--;
1157 ntpdata->pps_intcnt = 0;
1158 }
1159 }
1160}
1161
1162/*
1163 * Increase frequency calibration interval length. It is doubled after
1164 * four consecutive stable intervals.
1165 */
1166static inline void pps_inc_freq_interval(struct ntp_data *ntpdata)
1167{
1168 if (++ntpdata->pps_intcnt >= PPS_INTCOUNT) {
1169 ntpdata->pps_intcnt = PPS_INTCOUNT;
1170 if (ntpdata->pps_shift < PPS_INTMAX) {
1171 ntpdata->pps_shift++;
1172 ntpdata->pps_intcnt = 0;
1173 }
1174 }
1175}
1176
1177/*
1178 * Update clock frequency based on MONOTONIC_RAW clock PPS signal
1179 * timestamps
1180 *
1181 * At the end of the calibration interval the difference between the
1182 * first and last MONOTONIC_RAW clock timestamps divided by the length
1183 * of the interval becomes the frequency update. If the interval was
1184 * too long, the data are discarded.
1185 * Returns the difference between old and new frequency values.
1186 */
1187static long hardpps_update_freq(struct ntp_data *ntpdata, struct pps_normtime freq_norm)
1188{
1189 long delta, delta_mod;
1190 s64 ftemp;
1191
1192 /* Check if the frequency interval was too long */
1193 if (freq_norm.sec > (2 << ntpdata->pps_shift)) {
1194 ntpdata->time_status |= STA_PPSERROR;
1195 ntpdata->pps_errcnt++;
1196 pps_dec_freq_interval(ntpdata);
1197 printk_deferred(KERN_ERR "hardpps: PPSERROR: interval too long - %lld s\n",
1198 freq_norm.sec);
1199 return 0;
1200 }
1201
1202 /*
1203 * Here the raw frequency offset and wander (stability) is
1204 * calculated. If the wander is less than the wander threshold the
1205 * interval is increased; otherwise it is decreased.
1206 */
1207 ftemp = div_s64(((s64)(-freq_norm.nsec)) << NTP_SCALE_SHIFT,
1208 freq_norm.sec);
1209 delta = shift_right(ftemp - ntpdata->pps_freq, NTP_SCALE_SHIFT);
1210 ntpdata->pps_freq = ftemp;
1211 if (delta > PPS_MAXWANDER || delta < -PPS_MAXWANDER) {
1212 printk_deferred(KERN_WARNING "hardpps: PPSWANDER: change=%ld\n", delta);
1213 ntpdata->time_status |= STA_PPSWANDER;
1214 ntpdata->pps_stbcnt++;
1215 pps_dec_freq_interval(ntpdata);
1216 } else {
1217 /* Good sample */
1218 pps_inc_freq_interval(ntpdata);
1219 }
1220
1221 /*
1222 * The stability metric is calculated as the average of recent
1223 * frequency changes, but is used only for performance monitoring
1224 */
1225 delta_mod = delta;
1226 if (delta_mod < 0)
1227 delta_mod = -delta_mod;
1228 ntpdata->pps_stabil += (div_s64(((s64)delta_mod) << (NTP_SCALE_SHIFT - SHIFT_USEC),
1229 NSEC_PER_USEC) - ntpdata->pps_stabil) >> PPS_INTMIN;
1230
1231 /* If enabled, the system clock frequency is updated */
1232 if ((ntpdata->time_status & STA_PPSFREQ) && !(ntpdata->time_status & STA_FREQHOLD)) {
1233 ntpdata->time_freq = ntpdata->pps_freq;
1234 ntp_update_frequency(ntpdata);
1235 }
1236
1237 return delta;
1238}
1239
1240/* Correct REALTIME clock phase error against PPS signal */
1241static void hardpps_update_phase(struct ntp_data *ntpdata, long error)
1242{
1243 long correction = -error;
1244 long jitter;
1245
1246 /* Add the sample to the median filter */
1247 pps_phase_filter_add(ntpdata, correction);
1248 correction = pps_phase_filter_get(ntpdata, &jitter);
1249
1250 /*
1251 * Nominal jitter is due to PPS signal noise. If it exceeds the
1252 * threshold, the sample is discarded; otherwise, if so enabled,
1253 * the time offset is updated.
1254 */
1255 if (jitter > (ntpdata->pps_jitter << PPS_POPCORN)) {
1256 printk_deferred(KERN_WARNING "hardpps: PPSJITTER: jitter=%ld, limit=%ld\n",
1257 jitter, (ntpdata->pps_jitter << PPS_POPCORN));
1258 ntpdata->time_status |= STA_PPSJITTER;
1259 ntpdata->pps_jitcnt++;
1260 } else if (ntpdata->time_status & STA_PPSTIME) {
1261 /* Correct the time using the phase offset */
1262 ntpdata->time_offset = div_s64(((s64)correction) << NTP_SCALE_SHIFT,
1263 NTP_INTERVAL_FREQ);
1264 /* Cancel running adjtime() */
1265 ntpdata->time_adjust = 0;
1266 ntpdata->time_adjust_frac = 0;
1267 }
1268 /* Update jitter */
1269 ntpdata->pps_jitter += (jitter - ntpdata->pps_jitter) >> PPS_INTMIN;
1270}
1271
1272/*
1273 * __hardpps() - discipline CPU clock oscillator to external PPS signal
1274 *
1275 * This routine is called at each PPS signal arrival in order to
1276 * discipline the CPU clock oscillator to the PPS signal. It takes two
1277 * parameters: REALTIME and MONOTONIC_RAW clock timestamps. The former
1278 * is used to correct clock phase error and the latter is used to
1279 * correct the frequency.
1280 *
1281 * This code is based on David Mills's reference nanokernel
1282 * implementation. It was mostly rewritten but keeps the same idea.
1283 */
1284void __hardpps(const struct timespec64 *phase_ts, const struct timespec64 *raw_ts)
1285{
1286 struct ntp_data *ntpdata = &tk_ntp_data[TIMEKEEPER_CORE];
1287 struct pps_normtime pts_norm, freq_norm;
1288
1289 pts_norm = pps_normalize_ts(*phase_ts);
1290
1291 /* Clear the error bits, they will be set again if needed */
1292 ntpdata->time_status &= ~(STA_PPSJITTER | STA_PPSWANDER | STA_PPSERROR);
1293
1294 /* indicate signal presence */
1295 ntpdata->time_status |= STA_PPSSIGNAL;
1296 ntpdata->pps_valid = PPS_VALID;
1297
1298 /*
1299 * When called for the first time, just start the frequency
1300 * interval
1301 */
1302 if (unlikely(ntpdata->pps_fbase.tv_sec == 0)) {
1303 ntpdata->pps_fbase = *raw_ts;
1304 return;
1305 }
1306
1307 /* Ok, now we have a base for frequency calculation */
1308 freq_norm = pps_normalize_ts(timespec64_sub(*raw_ts, ntpdata->pps_fbase));
1309
1310 /*
1311 * Check that the signal is in the range
1312 * [1s - MAXFREQ us, 1s + MAXFREQ us], otherwise reject it
1313 */
1314 if ((freq_norm.sec == 0) || (freq_norm.nsec > MAXFREQ * freq_norm.sec) ||
1315 (freq_norm.nsec < -MAXFREQ * freq_norm.sec)) {
1316 ntpdata->time_status |= STA_PPSJITTER;
1317 /* Restart the frequency calibration interval */
1318 ntpdata->pps_fbase = *raw_ts;
1319 printk_deferred(KERN_ERR "hardpps: PPSJITTER: bad pulse\n");
1320 return;
1321 }
1322
1323 /* Signal is ok. Check if the current frequency interval is finished */
1324 if (freq_norm.sec >= (1 << ntpdata->pps_shift)) {
1325 ntpdata->pps_calcnt++;
1326 /* Restart the frequency calibration interval */
1327 ntpdata->pps_fbase = *raw_ts;
1328 hardpps_update_freq(ntpdata, freq_norm);
1329 }
1330
1331 hardpps_update_phase(ntpdata, pts_norm.nsec);
1332
1333}
1334#endif /* CONFIG_NTP_PPS */
1335
1336static int __init ntp_tick_adj_setup(char *str)
1337{
1338 int rc = kstrtos64(str, 0, &tk_ntp_data[TIMEKEEPER_CORE].ntp_tick_adj);
1339 if (rc)
1340 return rc;
1341
1342 tk_ntp_data[TIMEKEEPER_CORE].ntp_tick_adj <<= NTP_SCALE_SHIFT;
1343 return 1;
1344}
1345__setup("ntp_tick_adj=", ntp_tick_adj_setup);
1346
1347void __init ntp_init(void)
1348{
1349 for (int id = 0; id < TIMEKEEPERS_MAX; id++)
1350 __ntp_clear(tk_ntp_data + id);
1351 ntp_init_cmos_sync();
1352}