projects/linux·blob master · refs

318 lines · 8.6 KiB

1// SPDX-License-Identifier: GPL-2.0
2/* calibrate.c: default delay calibration
3 *
4 * Excised from init/main.c
5 * Copyright (C) 1991, 1992 Linus Torvalds
6 */
7
8#include <linux/delay.h>
9#include <linux/init.h>
10#include <linux/jiffies.h>
11#include <linux/kstrtox.h>
12#include <linux/percpu.h>
13#include <linux/printk.h>
14#include <linux/smp.h>
15#include <linux/stddef.h>
16
17unsigned long lpj_fine;
18unsigned long preset_lpj;
19
20static int __init lpj_setup(char *str)
21{
22 return kstrtoul(str, 0, &preset_lpj) == 0;
23}
24
25__setup("lpj=", lpj_setup);
26
27#ifdef CONFIG_ARCH_HAS_DELAY_TIMER
28
29/* This routine uses the delay_read_timer() routine and gets the
30 * loops per jiffy directly, instead of guessing it using delay().
31 * Also, this code tries to handle non-maskable asynchronous events
32 * (like SMIs)
33 */
34#define DELAY_CALIBRATION_TICKS ((HZ < 100) ? 1 : (HZ/100))
35#define MAX_DIRECT_CALIBRATION_RETRIES 5
36
37static unsigned long calibrate_delay_direct(void)
38{
39 unsigned long pre_start, start, post_start;
40 unsigned long pre_end, end, post_end;
41 unsigned long start_jiffies;
42 unsigned long timer_rate_min, timer_rate_max;
43 unsigned long good_timer_sum = 0;
44 unsigned long good_timer_count = 0;
45 unsigned long measured_times[MAX_DIRECT_CALIBRATION_RETRIES];
46 int max = -1; /* index of measured_times with max/min values or not set */
47 int min = -1;
48 int i;
49
50 if (!delay_read_timer(&pre_start))
51 return 0;
52
53 /*
54 * A simple loop like
55 * while ( jiffies < start_jiffies+1)
56 * start = delay_read_timer();
57 * will not do. As we don't really know whether jiffy switch
58 * happened first or timer_value was read first. And some asynchronous
59 * event can happen between these two events introducing errors in lpj.
60 *
61 * So, we do
62 * 1. pre_start <- When we are sure that jiffy switch hasn't happened
63 * 2. check jiffy switch
64 * 3. start <- timer value before or after jiffy switch
65 * 4. post_start <- When we are sure that jiffy switch has happened
66 *
67 * Note, we don't know anything about order of 2 and 3.
68 * Now, by looking at post_start and pre_start difference, we can
69 * check whether any asynchronous event happened or not
70 */
71
72 for (i = 0; i < MAX_DIRECT_CALIBRATION_RETRIES; i++) {
73 pre_start = 0;
74 delay_read_timer(&start);
75 start_jiffies = jiffies;
76 while (time_before_eq(jiffies, start_jiffies + 1)) {
77 pre_start = start;
78 delay_read_timer(&start);
79 }
80 delay_read_timer(&post_start);
81
82 pre_end = 0;
83 end = post_start;
84 while (time_before_eq(jiffies, start_jiffies + 1 +
85 DELAY_CALIBRATION_TICKS)) {
86 pre_end = end;
87 delay_read_timer(&end);
88 }
89 delay_read_timer(&post_end);
90
91 timer_rate_max = (post_end - pre_start) /
92 DELAY_CALIBRATION_TICKS;
93 timer_rate_min = (pre_end - post_start) /
94 DELAY_CALIBRATION_TICKS;
95
96 /*
97 * If the upper limit and lower limit of the timer_rate is
98 * >= 12.5% apart, redo calibration.
99 */
100 if (start >= post_end)
101 printk(KERN_NOTICE "calibrate_delay_direct() ignoring "
102 "timer_rate as we had a TSC wrap around"
103 " start=%lu >=post_end=%lu\n",
104 start, post_end);
105 if (start < post_end && pre_start != 0 && pre_end != 0 &&
106 (timer_rate_max - timer_rate_min) < (timer_rate_max >> 3)) {
107 good_timer_count++;
108 good_timer_sum += timer_rate_max;
109 measured_times[i] = timer_rate_max;
110 if (max < 0 || timer_rate_max > measured_times[max])
111 max = i;
112 if (min < 0 || timer_rate_max < measured_times[min])
113 min = i;
114 } else
115 measured_times[i] = 0;
116
117 }
118
119 /*
120 * Find the maximum & minimum - if they differ too much throw out the
121 * one with the largest difference from the mean and try again...
122 */
123 while (good_timer_count > 1) {
124 unsigned long estimate;
125 unsigned long maxdiff;
126
127 /* compute the estimate */
128 estimate = (good_timer_sum/good_timer_count);
129 maxdiff = estimate >> 3;
130
131 /* if range is within 12% let's take it */
132 if ((measured_times[max] - measured_times[min]) < maxdiff)
133 return estimate;
134
135 /* ok - drop the worse value and try again... */
136 good_timer_sum = 0;
137 good_timer_count = 0;
138 if ((measured_times[max] - estimate) <
139 (estimate - measured_times[min])) {
140 printk(KERN_NOTICE "calibrate_delay_direct() dropping "
141 "min bogoMips estimate %d = %lu\n",
142 min, measured_times[min]);
143 measured_times[min] = 0;
144 min = max;
145 } else {
146 printk(KERN_NOTICE "calibrate_delay_direct() dropping "
147 "max bogoMips estimate %d = %lu\n",
148 max, measured_times[max]);
149 measured_times[max] = 0;
150 max = min;
151 }
152
153 for (i = 0; i < MAX_DIRECT_CALIBRATION_RETRIES; i++) {
154 if (measured_times[i] == 0)
155 continue;
156 good_timer_count++;
157 good_timer_sum += measured_times[i];
158 if (measured_times[i] < measured_times[min])
159 min = i;
160 if (measured_times[i] > measured_times[max])
161 max = i;
162 }
163
164 }
165
166 printk(KERN_NOTICE "calibrate_delay_direct() failed to get a good "
167 "estimate for loops_per_jiffy.\nProbably due to long platform "
168 "interrupts. Consider using \"lpj=\" boot option.\n");
169 return 0;
170}
171#else
172static unsigned long calibrate_delay_direct(void)
173{
174 return 0;
175}
176#endif
177
178/*
179 * This is the number of bits of precision for the loops_per_jiffy. Each
180 * time we refine our estimate after the first takes 1.5/HZ seconds, so try
181 * to start with a good estimate.
182 * For the boot cpu we can skip the delay calibration and assign it a value
183 * calculated based on the timer frequency.
184 * For the rest of the CPUs we cannot assume that the timer frequency is same as
185 * the cpu frequency, hence do the calibration for those.
186 */
187#define LPS_PREC 8
188
189static unsigned long calibrate_delay_converge(void)
190{
191 /* First stage - slowly accelerate to find initial bounds */
192 unsigned long lpj, lpj_base, ticks, loopadd, loopadd_base, chop_limit;
193 int trials = 0, band = 0, trial_in_band = 0;
194
195 lpj = (1<<12);
196
197 /* wait for "start of" clock tick */
198 ticks = jiffies;
199 while (ticks == jiffies)
200 ; /* nothing */
201 /* Go .. */
202 ticks = jiffies;
203 do {
204 if (++trial_in_band == (1<<band)) {
205 ++band;
206 trial_in_band = 0;
207 }
208 __delay(lpj * band);
209 trials += band;
210 } while (ticks == jiffies);
211 /*
212 * We overshot, so retreat to a clear underestimate. Then estimate
213 * the largest likely undershoot. This defines our chop bounds.
214 */
215 trials -= band;
216 loopadd_base = lpj * band;
217 lpj_base = lpj * trials;
218
219recalibrate:
220 lpj = lpj_base;
221 loopadd = loopadd_base;
222
223 /*
224 * Do a binary approximation to get lpj set to
225 * equal one clock (up to LPS_PREC bits)
226 */
227 chop_limit = lpj >> LPS_PREC;
228 while (loopadd > chop_limit) {
229 lpj += loopadd;
230 ticks = jiffies;
231 while (ticks == jiffies)
232 ; /* nothing */
233 ticks = jiffies;
234 __delay(lpj);
235 if (jiffies != ticks) /* longer than 1 tick */
236 lpj -= loopadd;
237 loopadd >>= 1;
238 }
239 /*
240 * If we incremented every single time possible, presume we've
241 * massively underestimated initially, and retry with a higher
242 * start, and larger range. (Only seen on x86_64, due to SMIs)
243 */
244 if (lpj + loopadd * 2 == lpj_base + loopadd_base * 2) {
245 lpj_base = lpj;
246 loopadd_base <<= 2;
247 goto recalibrate;
248 }
249
250 return lpj;
251}
252
253static DEFINE_PER_CPU(unsigned long, cpu_loops_per_jiffy) = { 0 };
254
255/*
256 * Check if cpu calibration delay is already known. For example,
257 * some processors with multi-core sockets may have all cores
258 * with the same calibration delay.
259 *
260 * Architectures should override this function if a faster calibration
261 * method is available.
262 */
263unsigned long __attribute__((weak)) calibrate_delay_is_known(void)
264{
265 return 0;
266}
267
268/*
269 * Indicate the cpu delay calibration is done. This can be used by
270 * architectures to stop accepting delay timer registrations after this point.
271 */
272
273void __attribute__((weak)) calibration_delay_done(void)
274{
275}
276
277void calibrate_delay(void)
278{
279 unsigned long lpj;
280 static bool printed;
281 int this_cpu = smp_processor_id();
282
283 if (per_cpu(cpu_loops_per_jiffy, this_cpu)) {
284 lpj = per_cpu(cpu_loops_per_jiffy, this_cpu);
285 if (!printed)
286 pr_info("Calibrating delay loop (skipped) "
287 "already calibrated this CPU");
288 } else if (preset_lpj) {
289 lpj = preset_lpj;
290 if (!printed)
291 pr_info("Calibrating delay loop (skipped) "
292 "preset value.. ");
293 } else if ((!printed) && lpj_fine) {
294 lpj = lpj_fine;
295 pr_info("Calibrating delay loop (skipped), "
296 "value calculated using timer frequency.. ");
297 } else if ((lpj = calibrate_delay_is_known())) {
298 ;
299 } else if ((lpj = calibrate_delay_direct()) != 0) {
300 if (!printed)
301 pr_info("Calibrating delay using timer "
302 "specific routine.. ");
303 } else {
304 if (!printed)
305 pr_info("Calibrating delay loop... ");
306 lpj = calibrate_delay_converge();
307 }
308 per_cpu(cpu_loops_per_jiffy, this_cpu) = lpj;
309 if (!printed)
310 pr_cont("%lu.%02lu BogoMIPS (lpj=%lu)\n",
311 lpj/(500000/HZ),
312 (lpj/(5000/HZ)) % 100, lpj);
313
314 loops_per_jiffy = lpj;
315 printed = true;
316
317 calibration_delay_done();
318}