projects/linux·blob master · refs

2992 lines · 77.3 KiB

1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * intel_idle.c - native hardware idle loop for modern Intel processors
4 *
5 * Copyright (c) 2013 - 2020, Intel Corporation.
6 * Len Brown <[email protected]>
7 * Rafael J. Wysocki <[email protected]>
8 */
9
10/*
11 * intel_idle is a cpuidle driver that loads on all Intel CPUs with MWAIT
12 * in lieu of the legacy ACPI processor_idle driver. The intent is to
13 * make Linux more efficient on these processors, as intel_idle knows
14 * more than ACPI, as well as make Linux more immune to ACPI BIOS bugs.
15 */
16
17/*
18 * Design Assumptions
19 *
20 * All CPUs have same idle states as boot CPU
21 *
22 * Chipset BM_STS (bus master status) bit is a NOP
23 * for preventing entry into deep C-states
24 *
25 * CPU will flush caches as needed when entering a C-state via MWAIT
26 * (in contrast to entering ACPI C3, in which case the WBINVD
27 * instruction needs to be executed to flush the caches)
28 */
29
30/*
31 * Known limitations
32 *
33 * ACPI has a .suspend hack to turn off deep c-statees during suspend
34 * to avoid complications with the lapic timer workaround.
35 * Have not seen issues with suspend, but may need same workaround here.
36 *
37 */
38
39/* un-comment DEBUG to enable pr_debug() statements */
40/* #define DEBUG */
41
42#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
43
44#include <linux/acpi.h>
45#include <linux/kernel.h>
46#include <linux/cpuidle.h>
47#include <linux/tick.h>
48#include <linux/time64.h>
49#include <trace/events/power.h>
50#include <linux/sched.h>
51#include <linux/sched/smt.h>
52#include <linux/mutex.h>
53#include <linux/notifier.h>
54#include <linux/cpu.h>
55#include <linux/moduleparam.h>
56#include <linux/pm_qos.h>
57#include <linux/sysfs.h>
58#include <asm/cpuid/api.h>
59#include <asm/cpu_device_id.h>
60#include <asm/intel-family.h>
61#include <asm/mwait.h>
62#include <asm/spec-ctrl.h>
63#include <asm/msr.h>
64#include <asm/tsc.h>
65#include <asm/fpu/api.h>
66#include <asm/smp.h>
67
68static struct cpuidle_driver intel_idle_driver = {
69 .name = "intel_idle",
70 .owner = THIS_MODULE,
71};
72/* intel_idle.max_cstate=0 disables driver */
73static int max_cstate = CPUIDLE_STATE_MAX - 1;
74static unsigned int disabled_states_mask __read_mostly;
75static bool force_irq_on __read_mostly;
76static bool ibrs_off __read_mostly;
77
78/* The maximum allowed length for the 'table' module parameter */
79#define MAX_CMDLINE_TABLE_LEN 256
80/* Maximum allowed C-state latency */
81#define MAX_CMDLINE_LATENCY_US (5 * USEC_PER_MSEC)
82/* Maximum allowed C-state target residency */
83#define MAX_CMDLINE_RESIDENCY_US (100 * USEC_PER_MSEC)
84
85/* The Package C-State Limit bits in MSR_PKG_CST_CONFIG_CONTROL */
86#define SKX_PKG_CST_LIMIT_MASK GENMASK(2, 0)
87/* PC6 is enabled when Package C-State Limit >= this value */
88#define SKX_PKG_CST_LIMIT_PC6 2
89
90static char cmdline_table_str[MAX_CMDLINE_TABLE_LEN] __read_mostly;
91
92static struct cpuidle_device __percpu *intel_idle_cpuidle_devices;
93
94static unsigned long auto_demotion_disable_flags;
95
96static enum {
97 C1E_PROMOTION_PRESERVE,
98 C1E_PROMOTION_ENABLE,
99 C1E_PROMOTION_DISABLE
100} c1e_promotion = C1E_PROMOTION_PRESERVE;
101
102struct idle_cpu {
103 struct cpuidle_state *state_table;
104
105 /*
106 * Hardware C-state auto-demotion may not always be optimal.
107 * Indicate which enable bits to clear here.
108 */
109 unsigned long auto_demotion_disable_flags;
110 bool disable_promotion_to_c1e;
111 bool c1_demotion_supported;
112 bool use_acpi;
113};
114
115static bool c1_demotion_supported;
116static DEFINE_MUTEX(c1_demotion_mutex);
117
118static struct device *sysfs_root __initdata;
119
120static const struct idle_cpu *icpu __initdata;
121static struct cpuidle_state *cpuidle_state_table __initdata;
122
123/* C-states data from the 'intel_idle.table' cmdline parameter */
124static struct cpuidle_state cmdline_states[CPUIDLE_STATE_MAX] __initdata;
125
126static unsigned int mwait_substates __initdata;
127
128/*
129 * Enable interrupts before entering the C-state. On some platforms and for
130 * some C-states, this may measurably decrease interrupt latency.
131 */
132#define CPUIDLE_FLAG_IRQ_ENABLE BIT(14)
133
134/*
135 * Enable this state by default even if the ACPI _CST does not list it.
136 */
137#define CPUIDLE_FLAG_ALWAYS_ENABLE BIT(15)
138
139/*
140 * Disable IBRS across idle (when KERNEL_IBRS), is exclusive vs IRQ_ENABLE
141 * above.
142 */
143#define CPUIDLE_FLAG_IBRS BIT(16)
144
145/*
146 * Initialize large xstate for the C6-state entrance.
147 */
148#define CPUIDLE_FLAG_INIT_XSTATE BIT(17)
149
150/*
151 * Ignore the sub-state when matching mwait hints between the ACPI _CST and
152 * custom tables.
153 */
154#define CPUIDLE_FLAG_PARTIAL_HINT_MATCH BIT(18)
155
156/*
157 * MWAIT takes an 8-bit "hint" in EAX "suggesting"
158 * the C-state (top nibble) and sub-state (bottom nibble)
159 * 0x00 means "MWAIT(C1)", 0x10 means "MWAIT(C2)" etc.
160 *
161 * We store the hint at the top of our "flags" for each state.
162 */
163#define flg2MWAIT(flags) (((flags) >> 24) & 0xFF)
164#define MWAIT2flg(eax) ((eax & 0xFF) << 24)
165
166static __always_inline int __intel_idle(struct cpuidle_device *dev,
167 struct cpuidle_driver *drv,
168 int index, bool irqoff)
169{
170 struct cpuidle_state *state = &drv->states[index];
171 unsigned int eax = flg2MWAIT(state->flags);
172 unsigned int ecx = 1*irqoff; /* break on interrupt flag */
173
174 mwait_idle_with_hints(eax, ecx);
175
176 return index;
177}
178
179/**
180 * intel_idle - Ask the processor to enter the given idle state.
181 * @dev: cpuidle device of the target CPU.
182 * @drv: cpuidle driver (assumed to point to intel_idle_driver).
183 * @index: Target idle state index.
184 *
185 * Use the MWAIT instruction to notify the processor that the CPU represented by
186 * @dev is idle and it can try to enter the idle state corresponding to @index.
187 *
188 * If the local APIC timer is not known to be reliable in the target idle state,
189 * enable one-shot tick broadcasting for the target CPU before executing MWAIT.
190 *
191 * Must be called under local_irq_disable().
192 */
193static __cpuidle int intel_idle(struct cpuidle_device *dev,
194 struct cpuidle_driver *drv, int index)
195{
196 return __intel_idle(dev, drv, index, true);
197}
198
199static __cpuidle int intel_idle_irq(struct cpuidle_device *dev,
200 struct cpuidle_driver *drv, int index)
201{
202 return __intel_idle(dev, drv, index, false);
203}
204
205static __cpuidle int intel_idle_ibrs(struct cpuidle_device *dev,
206 struct cpuidle_driver *drv, int index)
207{
208 bool smt_active = sched_smt_active();
209 u64 spec_ctrl = spec_ctrl_current();
210 int ret;
211
212 if (smt_active)
213 __update_spec_ctrl(0);
214
215 ret = __intel_idle(dev, drv, index, true);
216
217 if (smt_active)
218 __update_spec_ctrl(spec_ctrl);
219
220 return ret;
221}
222
223static __cpuidle int intel_idle_xstate(struct cpuidle_device *dev,
224 struct cpuidle_driver *drv, int index)
225{
226 fpu_idle_fpregs();
227 return __intel_idle(dev, drv, index, true);
228}
229
230/**
231 * intel_idle_s2idle - Ask the processor to enter the given idle state.
232 * @dev: cpuidle device of the target CPU.
233 * @drv: cpuidle driver (assumed to point to intel_idle_driver).
234 * @index: Target idle state index.
235 *
236 * Use the MWAIT instruction to notify the processor that the CPU represented by
237 * @dev is idle and it can try to enter the idle state corresponding to @index.
238 *
239 * Invoked as a suspend-to-idle callback routine with frozen user space, frozen
240 * scheduler tick and suspended scheduler clock on the target CPU.
241 */
242static __cpuidle int intel_idle_s2idle(struct cpuidle_device *dev,
243 struct cpuidle_driver *drv, int index)
244{
245 struct cpuidle_state *state = &drv->states[index];
246 unsigned int eax = flg2MWAIT(state->flags);
247 unsigned int ecx = 1; /* break on interrupt flag */
248
249 if (state->flags & CPUIDLE_FLAG_INIT_XSTATE)
250 fpu_idle_fpregs();
251
252 mwait_idle_with_hints(eax, ecx);
253
254 return 0;
255}
256
257static void intel_idle_enter_dead(struct cpuidle_device *dev, int index)
258{
259 struct cpuidle_driver *drv = cpuidle_get_cpu_driver(dev);
260 struct cpuidle_state *state = &drv->states[index];
261 unsigned long eax = flg2MWAIT(state->flags);
262
263 mwait_play_dead(eax);
264}
265
266/*
267 * States are indexed by the cstate number,
268 * which is also the index into the MWAIT hint array.
269 * Thus C0 is a dummy.
270 */
271static struct cpuidle_state nehalem_cstates[] __initdata = {
272 {
273 .name = "C1",
274 .desc = "MWAIT 0x00",
275 .flags = MWAIT2flg(0x00),
276 .exit_latency = 3,
277 .target_residency = 6,
278 .enter = intel_idle,
279 .enter_s2idle = intel_idle_s2idle, },
280 {
281 .name = "C1E",
282 .desc = "MWAIT 0x01",
283 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
284 .exit_latency = 10,
285 .target_residency = 20,
286 .enter = intel_idle,
287 .enter_s2idle = intel_idle_s2idle, },
288 {
289 .name = "C3",
290 .desc = "MWAIT 0x10",
291 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
292 .exit_latency = 20,
293 .target_residency = 80,
294 .enter = intel_idle,
295 .enter_s2idle = intel_idle_s2idle, },
296 {
297 .name = "C6",
298 .desc = "MWAIT 0x20",
299 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
300 .exit_latency = 200,
301 .target_residency = 800,
302 .enter = intel_idle,
303 .enter_s2idle = intel_idle_s2idle, },
304 {
305 .enter = NULL }
306};
307
308static struct cpuidle_state snb_cstates[] __initdata = {
309 {
310 .name = "C1",
311 .desc = "MWAIT 0x00",
312 .flags = MWAIT2flg(0x00),
313 .exit_latency = 2,
314 .target_residency = 2,
315 .enter = intel_idle,
316 .enter_s2idle = intel_idle_s2idle, },
317 {
318 .name = "C1E",
319 .desc = "MWAIT 0x01",
320 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
321 .exit_latency = 10,
322 .target_residency = 20,
323 .enter = intel_idle,
324 .enter_s2idle = intel_idle_s2idle, },
325 {
326 .name = "C3",
327 .desc = "MWAIT 0x10",
328 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
329 .exit_latency = 80,
330 .target_residency = 211,
331 .enter = intel_idle,
332 .enter_s2idle = intel_idle_s2idle, },
333 {
334 .name = "C6",
335 .desc = "MWAIT 0x20",
336 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
337 .exit_latency = 104,
338 .target_residency = 345,
339 .enter = intel_idle,
340 .enter_s2idle = intel_idle_s2idle, },
341 {
342 .name = "C7",
343 .desc = "MWAIT 0x30",
344 .flags = MWAIT2flg(0x30) | CPUIDLE_FLAG_TLB_FLUSHED,
345 .exit_latency = 109,
346 .target_residency = 345,
347 .enter = intel_idle,
348 .enter_s2idle = intel_idle_s2idle, },
349 {
350 .enter = NULL }
351};
352
353static struct cpuidle_state byt_cstates[] __initdata = {
354 {
355 .name = "C1",
356 .desc = "MWAIT 0x00",
357 .flags = MWAIT2flg(0x00),
358 .exit_latency = 1,
359 .target_residency = 1,
360 .enter = intel_idle,
361 .enter_s2idle = intel_idle_s2idle, },
362 {
363 .name = "C6N",
364 .desc = "MWAIT 0x58",
365 .flags = MWAIT2flg(0x58) | CPUIDLE_FLAG_TLB_FLUSHED,
366 .exit_latency = 300,
367 .target_residency = 275,
368 .enter = intel_idle,
369 .enter_s2idle = intel_idle_s2idle, },
370 {
371 .name = "C6S",
372 .desc = "MWAIT 0x52",
373 .flags = MWAIT2flg(0x52) | CPUIDLE_FLAG_TLB_FLUSHED,
374 .exit_latency = 500,
375 .target_residency = 560,
376 .enter = intel_idle,
377 .enter_s2idle = intel_idle_s2idle, },
378 {
379 .name = "C7",
380 .desc = "MWAIT 0x60",
381 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,
382 .exit_latency = 1200,
383 .target_residency = 4000,
384 .enter = intel_idle,
385 .enter_s2idle = intel_idle_s2idle, },
386 {
387 .name = "C7S",
388 .desc = "MWAIT 0x64",
389 .flags = MWAIT2flg(0x64) | CPUIDLE_FLAG_TLB_FLUSHED,
390 .exit_latency = 10000,
391 .target_residency = 20000,
392 .enter = intel_idle,
393 .enter_s2idle = intel_idle_s2idle, },
394 {
395 .enter = NULL }
396};
397
398static struct cpuidle_state cht_cstates[] __initdata = {
399 {
400 .name = "C1",
401 .desc = "MWAIT 0x00",
402 .flags = MWAIT2flg(0x00),
403 .exit_latency = 1,
404 .target_residency = 1,
405 .enter = intel_idle,
406 .enter_s2idle = intel_idle_s2idle, },
407 {
408 .name = "C6N",
409 .desc = "MWAIT 0x58",
410 .flags = MWAIT2flg(0x58) | CPUIDLE_FLAG_TLB_FLUSHED,
411 .exit_latency = 80,
412 .target_residency = 275,
413 .enter = intel_idle,
414 .enter_s2idle = intel_idle_s2idle, },
415 {
416 .name = "C6S",
417 .desc = "MWAIT 0x52",
418 .flags = MWAIT2flg(0x52) | CPUIDLE_FLAG_TLB_FLUSHED,
419 .exit_latency = 200,
420 .target_residency = 560,
421 .enter = intel_idle,
422 .enter_s2idle = intel_idle_s2idle, },
423 {
424 .name = "C7",
425 .desc = "MWAIT 0x60",
426 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,
427 .exit_latency = 1200,
428 .target_residency = 4000,
429 .enter = intel_idle,
430 .enter_s2idle = intel_idle_s2idle, },
431 {
432 .name = "C7S",
433 .desc = "MWAIT 0x64",
434 .flags = MWAIT2flg(0x64) | CPUIDLE_FLAG_TLB_FLUSHED,
435 .exit_latency = 10000,
436 .target_residency = 20000,
437 .enter = intel_idle,
438 .enter_s2idle = intel_idle_s2idle, },
439 {
440 .enter = NULL }
441};
442
443static struct cpuidle_state ivb_cstates[] __initdata = {
444 {
445 .name = "C1",
446 .desc = "MWAIT 0x00",
447 .flags = MWAIT2flg(0x00),
448 .exit_latency = 1,
449 .target_residency = 1,
450 .enter = intel_idle,
451 .enter_s2idle = intel_idle_s2idle, },
452 {
453 .name = "C1E",
454 .desc = "MWAIT 0x01",
455 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
456 .exit_latency = 10,
457 .target_residency = 20,
458 .enter = intel_idle,
459 .enter_s2idle = intel_idle_s2idle, },
460 {
461 .name = "C3",
462 .desc = "MWAIT 0x10",
463 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
464 .exit_latency = 59,
465 .target_residency = 156,
466 .enter = intel_idle,
467 .enter_s2idle = intel_idle_s2idle, },
468 {
469 .name = "C6",
470 .desc = "MWAIT 0x20",
471 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
472 .exit_latency = 80,
473 .target_residency = 300,
474 .enter = intel_idle,
475 .enter_s2idle = intel_idle_s2idle, },
476 {
477 .name = "C7",
478 .desc = "MWAIT 0x30",
479 .flags = MWAIT2flg(0x30) | CPUIDLE_FLAG_TLB_FLUSHED,
480 .exit_latency = 87,
481 .target_residency = 300,
482 .enter = intel_idle,
483 .enter_s2idle = intel_idle_s2idle, },
484 {
485 .enter = NULL }
486};
487
488static struct cpuidle_state ivt_cstates[] __initdata = {
489 {
490 .name = "C1",
491 .desc = "MWAIT 0x00",
492 .flags = MWAIT2flg(0x00),
493 .exit_latency = 1,
494 .target_residency = 1,
495 .enter = intel_idle,
496 .enter_s2idle = intel_idle_s2idle, },
497 {
498 .name = "C1E",
499 .desc = "MWAIT 0x01",
500 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
501 .exit_latency = 10,
502 .target_residency = 80,
503 .enter = intel_idle,
504 .enter_s2idle = intel_idle_s2idle, },
505 {
506 .name = "C3",
507 .desc = "MWAIT 0x10",
508 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
509 .exit_latency = 59,
510 .target_residency = 156,
511 .enter = intel_idle,
512 .enter_s2idle = intel_idle_s2idle, },
513 {
514 .name = "C6",
515 .desc = "MWAIT 0x20",
516 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
517 .exit_latency = 82,
518 .target_residency = 300,
519 .enter = intel_idle,
520 .enter_s2idle = intel_idle_s2idle, },
521 {
522 .enter = NULL }
523};
524
525static struct cpuidle_state ivt_cstates_4s[] __initdata = {
526 {
527 .name = "C1",
528 .desc = "MWAIT 0x00",
529 .flags = MWAIT2flg(0x00),
530 .exit_latency = 1,
531 .target_residency = 1,
532 .enter = intel_idle,
533 .enter_s2idle = intel_idle_s2idle, },
534 {
535 .name = "C1E",
536 .desc = "MWAIT 0x01",
537 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
538 .exit_latency = 10,
539 .target_residency = 250,
540 .enter = intel_idle,
541 .enter_s2idle = intel_idle_s2idle, },
542 {
543 .name = "C3",
544 .desc = "MWAIT 0x10",
545 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
546 .exit_latency = 59,
547 .target_residency = 300,
548 .enter = intel_idle,
549 .enter_s2idle = intel_idle_s2idle, },
550 {
551 .name = "C6",
552 .desc = "MWAIT 0x20",
553 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
554 .exit_latency = 84,
555 .target_residency = 400,
556 .enter = intel_idle,
557 .enter_s2idle = intel_idle_s2idle, },
558 {
559 .enter = NULL }
560};
561
562static struct cpuidle_state ivt_cstates_8s[] __initdata = {
563 {
564 .name = "C1",
565 .desc = "MWAIT 0x00",
566 .flags = MWAIT2flg(0x00),
567 .exit_latency = 1,
568 .target_residency = 1,
569 .enter = intel_idle,
570 .enter_s2idle = intel_idle_s2idle, },
571 {
572 .name = "C1E",
573 .desc = "MWAIT 0x01",
574 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
575 .exit_latency = 10,
576 .target_residency = 500,
577 .enter = intel_idle,
578 .enter_s2idle = intel_idle_s2idle, },
579 {
580 .name = "C3",
581 .desc = "MWAIT 0x10",
582 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
583 .exit_latency = 59,
584 .target_residency = 600,
585 .enter = intel_idle,
586 .enter_s2idle = intel_idle_s2idle, },
587 {
588 .name = "C6",
589 .desc = "MWAIT 0x20",
590 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
591 .exit_latency = 88,
592 .target_residency = 700,
593 .enter = intel_idle,
594 .enter_s2idle = intel_idle_s2idle, },
595 {
596 .enter = NULL }
597};
598
599static struct cpuidle_state hsw_cstates[] __initdata = {
600 {
601 .name = "C1",
602 .desc = "MWAIT 0x00",
603 .flags = MWAIT2flg(0x00),
604 .exit_latency = 2,
605 .target_residency = 2,
606 .enter = intel_idle,
607 .enter_s2idle = intel_idle_s2idle, },
608 {
609 .name = "C1E",
610 .desc = "MWAIT 0x01",
611 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
612 .exit_latency = 10,
613 .target_residency = 20,
614 .enter = intel_idle,
615 .enter_s2idle = intel_idle_s2idle, },
616 {
617 .name = "C3",
618 .desc = "MWAIT 0x10",
619 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
620 .exit_latency = 33,
621 .target_residency = 100,
622 .enter = intel_idle,
623 .enter_s2idle = intel_idle_s2idle, },
624 {
625 .name = "C6",
626 .desc = "MWAIT 0x20",
627 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
628 .exit_latency = 133,
629 .target_residency = 400,
630 .enter = intel_idle,
631 .enter_s2idle = intel_idle_s2idle, },
632 {
633 .name = "C7s",
634 .desc = "MWAIT 0x32",
635 .flags = MWAIT2flg(0x32) | CPUIDLE_FLAG_TLB_FLUSHED,
636 .exit_latency = 166,
637 .target_residency = 500,
638 .enter = intel_idle,
639 .enter_s2idle = intel_idle_s2idle, },
640 {
641 .name = "C8",
642 .desc = "MWAIT 0x40",
643 .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED,
644 .exit_latency = 300,
645 .target_residency = 900,
646 .enter = intel_idle,
647 .enter_s2idle = intel_idle_s2idle, },
648 {
649 .name = "C9",
650 .desc = "MWAIT 0x50",
651 .flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED,
652 .exit_latency = 600,
653 .target_residency = 1800,
654 .enter = intel_idle,
655 .enter_s2idle = intel_idle_s2idle, },
656 {
657 .name = "C10",
658 .desc = "MWAIT 0x60",
659 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,
660 .exit_latency = 2600,
661 .target_residency = 7700,
662 .enter = intel_idle,
663 .enter_s2idle = intel_idle_s2idle, },
664 {
665 .enter = NULL }
666};
667static struct cpuidle_state bdw_cstates[] __initdata = {
668 {
669 .name = "C1",
670 .desc = "MWAIT 0x00",
671 .flags = MWAIT2flg(0x00),
672 .exit_latency = 2,
673 .target_residency = 2,
674 .enter = intel_idle,
675 .enter_s2idle = intel_idle_s2idle, },
676 {
677 .name = "C1E",
678 .desc = "MWAIT 0x01",
679 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
680 .exit_latency = 10,
681 .target_residency = 20,
682 .enter = intel_idle,
683 .enter_s2idle = intel_idle_s2idle, },
684 {
685 .name = "C3",
686 .desc = "MWAIT 0x10",
687 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
688 .exit_latency = 40,
689 .target_residency = 100,
690 .enter = intel_idle,
691 .enter_s2idle = intel_idle_s2idle, },
692 {
693 .name = "C6",
694 .desc = "MWAIT 0x20",
695 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
696 .exit_latency = 133,
697 .target_residency = 400,
698 .enter = intel_idle,
699 .enter_s2idle = intel_idle_s2idle, },
700 {
701 .name = "C7s",
702 .desc = "MWAIT 0x32",
703 .flags = MWAIT2flg(0x32) | CPUIDLE_FLAG_TLB_FLUSHED,
704 .exit_latency = 166,
705 .target_residency = 500,
706 .enter = intel_idle,
707 .enter_s2idle = intel_idle_s2idle, },
708 {
709 .name = "C8",
710 .desc = "MWAIT 0x40",
711 .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED,
712 .exit_latency = 300,
713 .target_residency = 900,
714 .enter = intel_idle,
715 .enter_s2idle = intel_idle_s2idle, },
716 {
717 .name = "C9",
718 .desc = "MWAIT 0x50",
719 .flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED,
720 .exit_latency = 600,
721 .target_residency = 1800,
722 .enter = intel_idle,
723 .enter_s2idle = intel_idle_s2idle, },
724 {
725 .name = "C10",
726 .desc = "MWAIT 0x60",
727 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,
728 .exit_latency = 2600,
729 .target_residency = 7700,
730 .enter = intel_idle,
731 .enter_s2idle = intel_idle_s2idle, },
732 {
733 .enter = NULL }
734};
735
736static struct cpuidle_state skl_cstates[] __initdata = {
737 {
738 .name = "C1",
739 .desc = "MWAIT 0x00",
740 .flags = MWAIT2flg(0x00),
741 .exit_latency = 2,
742 .target_residency = 2,
743 .enter = intel_idle,
744 .enter_s2idle = intel_idle_s2idle, },
745 {
746 .name = "C1E",
747 .desc = "MWAIT 0x01",
748 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
749 .exit_latency = 10,
750 .target_residency = 20,
751 .enter = intel_idle,
752 .enter_s2idle = intel_idle_s2idle, },
753 {
754 .name = "C3",
755 .desc = "MWAIT 0x10",
756 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
757 .exit_latency = 70,
758 .target_residency = 100,
759 .enter = intel_idle,
760 .enter_s2idle = intel_idle_s2idle, },
761 {
762 .name = "C6",
763 .desc = "MWAIT 0x20",
764 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED | CPUIDLE_FLAG_IBRS,
765 .exit_latency = 85,
766 .target_residency = 200,
767 .enter = intel_idle,
768 .enter_s2idle = intel_idle_s2idle, },
769 {
770 .name = "C7s",
771 .desc = "MWAIT 0x33",
772 .flags = MWAIT2flg(0x33) | CPUIDLE_FLAG_TLB_FLUSHED | CPUIDLE_FLAG_IBRS,
773 .exit_latency = 124,
774 .target_residency = 800,
775 .enter = intel_idle,
776 .enter_s2idle = intel_idle_s2idle, },
777 {
778 .name = "C8",
779 .desc = "MWAIT 0x40",
780 .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED | CPUIDLE_FLAG_IBRS,
781 .exit_latency = 200,
782 .target_residency = 800,
783 .enter = intel_idle,
784 .enter_s2idle = intel_idle_s2idle, },
785 {
786 .name = "C9",
787 .desc = "MWAIT 0x50",
788 .flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED | CPUIDLE_FLAG_IBRS,
789 .exit_latency = 480,
790 .target_residency = 5000,
791 .enter = intel_idle,
792 .enter_s2idle = intel_idle_s2idle, },
793 {
794 .name = "C10",
795 .desc = "MWAIT 0x60",
796 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED | CPUIDLE_FLAG_IBRS,
797 .exit_latency = 890,
798 .target_residency = 5000,
799 .enter = intel_idle,
800 .enter_s2idle = intel_idle_s2idle, },
801 {
802 .enter = NULL }
803};
804
805static struct cpuidle_state skx_cstates[] __initdata = {
806 {
807 .name = "C1",
808 .desc = "MWAIT 0x00",
809 .flags = MWAIT2flg(0x00) | CPUIDLE_FLAG_IRQ_ENABLE,
810 .exit_latency = 2,
811 .target_residency = 2,
812 .enter = intel_idle,
813 .enter_s2idle = intel_idle_s2idle, },
814 {
815 .name = "C1E",
816 .desc = "MWAIT 0x01",
817 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
818 .exit_latency = 10,
819 .target_residency = 20,
820 .enter = intel_idle,
821 .enter_s2idle = intel_idle_s2idle, },
822 {
823 .name = "C6",
824 .desc = "MWAIT 0x20",
825 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED | CPUIDLE_FLAG_IBRS,
826 .exit_latency = 133,
827 .target_residency = 600,
828 .enter = intel_idle,
829 .enter_s2idle = intel_idle_s2idle, },
830 {
831 .enter = NULL }
832};
833
834static struct cpuidle_state icx_cstates[] __initdata = {
835 {
836 .name = "C1",
837 .desc = "MWAIT 0x00",
838 .flags = MWAIT2flg(0x00) | CPUIDLE_FLAG_IRQ_ENABLE,
839 .exit_latency = 1,
840 .target_residency = 1,
841 .enter = intel_idle,
842 .enter_s2idle = intel_idle_s2idle, },
843 {
844 .name = "C1E",
845 .desc = "MWAIT 0x01",
846 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
847 .exit_latency = 4,
848 .target_residency = 4,
849 .enter = intel_idle,
850 .enter_s2idle = intel_idle_s2idle, },
851 {
852 .name = "C6",
853 .desc = "MWAIT 0x20",
854 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
855 .exit_latency = 170,
856 .target_residency = 600,
857 .enter = intel_idle,
858 .enter_s2idle = intel_idle_s2idle, },
859 {
860 .enter = NULL }
861};
862
863/*
864 * On AlderLake C1 has to be disabled if C1E is enabled, and vice versa.
865 * C1E is enabled only if "C1E promotion" bit is set in MSR_IA32_POWER_CTL.
866 * But in this case there is effectively no C1, because C1 requests are
867 * promoted to C1E. If the "C1E promotion" bit is cleared, then both C1
868 * and C1E requests end up with C1, so there is effectively no C1E.
869 *
870 * By default we enable C1E and disable C1 by marking it with
871 * 'CPUIDLE_FLAG_UNUSABLE'.
872 */
873static struct cpuidle_state adl_cstates[] __initdata = {
874 {
875 .name = "C1",
876 .desc = "MWAIT 0x00",
877 .flags = MWAIT2flg(0x00) | CPUIDLE_FLAG_UNUSABLE,
878 .exit_latency = 1,
879 .target_residency = 1,
880 .enter = intel_idle,
881 .enter_s2idle = intel_idle_s2idle, },
882 {
883 .name = "C1E",
884 .desc = "MWAIT 0x01",
885 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
886 .exit_latency = 2,
887 .target_residency = 4,
888 .enter = intel_idle,
889 .enter_s2idle = intel_idle_s2idle, },
890 {
891 .name = "C6",
892 .desc = "MWAIT 0x20",
893 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
894 .exit_latency = 220,
895 .target_residency = 600,
896 .enter = intel_idle,
897 .enter_s2idle = intel_idle_s2idle, },
898 {
899 .name = "C8",
900 .desc = "MWAIT 0x40",
901 .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED,
902 .exit_latency = 280,
903 .target_residency = 800,
904 .enter = intel_idle,
905 .enter_s2idle = intel_idle_s2idle, },
906 {
907 .name = "C10",
908 .desc = "MWAIT 0x60",
909 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,
910 .exit_latency = 680,
911 .target_residency = 2000,
912 .enter = intel_idle,
913 .enter_s2idle = intel_idle_s2idle, },
914 {
915 .enter = NULL }
916};
917
918static struct cpuidle_state adl_l_cstates[] __initdata = {
919 {
920 .name = "C1",
921 .desc = "MWAIT 0x00",
922 .flags = MWAIT2flg(0x00) | CPUIDLE_FLAG_UNUSABLE,
923 .exit_latency = 1,
924 .target_residency = 1,
925 .enter = intel_idle,
926 .enter_s2idle = intel_idle_s2idle, },
927 {
928 .name = "C1E",
929 .desc = "MWAIT 0x01",
930 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
931 .exit_latency = 2,
932 .target_residency = 4,
933 .enter = intel_idle,
934 .enter_s2idle = intel_idle_s2idle, },
935 {
936 .name = "C6",
937 .desc = "MWAIT 0x20",
938 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
939 .exit_latency = 170,
940 .target_residency = 500,
941 .enter = intel_idle,
942 .enter_s2idle = intel_idle_s2idle, },
943 {
944 .name = "C8",
945 .desc = "MWAIT 0x40",
946 .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED,
947 .exit_latency = 200,
948 .target_residency = 600,
949 .enter = intel_idle,
950 .enter_s2idle = intel_idle_s2idle, },
951 {
952 .name = "C10",
953 .desc = "MWAIT 0x60",
954 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,
955 .exit_latency = 230,
956 .target_residency = 700,
957 .enter = intel_idle,
958 .enter_s2idle = intel_idle_s2idle, },
959 {
960 .enter = NULL }
961};
962
963static struct cpuidle_state mtl_l_cstates[] __initdata = {
964 {
965 .name = "C1E",
966 .desc = "MWAIT 0x01",
967 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
968 .exit_latency = 1,
969 .target_residency = 1,
970 .enter = intel_idle,
971 .enter_s2idle = intel_idle_s2idle, },
972 {
973 .name = "C6",
974 .desc = "MWAIT 0x20",
975 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
976 .exit_latency = 140,
977 .target_residency = 420,
978 .enter = intel_idle,
979 .enter_s2idle = intel_idle_s2idle, },
980 {
981 .name = "C10",
982 .desc = "MWAIT 0x60",
983 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,
984 .exit_latency = 310,
985 .target_residency = 930,
986 .enter = intel_idle,
987 .enter_s2idle = intel_idle_s2idle, },
988 {
989 .enter = NULL }
990};
991
992static struct cpuidle_state ptl_cstates[] __initdata = {
993 {
994 .name = "C1",
995 .desc = "MWAIT 0x00",
996 .flags = MWAIT2flg(0x00),
997 .exit_latency = 1,
998 .target_residency = 1,
999 .enter = &intel_idle,
1000 .enter_s2idle = intel_idle_s2idle, },
1001 {
1002 .name = "C1E",
1003 .desc = "MWAIT 0x01",
1004 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
1005 .exit_latency = 10,
1006 .target_residency = 10,
1007 .enter = &intel_idle,
1008 .enter_s2idle = intel_idle_s2idle, },
1009 {
1010 .name = "C6S",
1011 .desc = "MWAIT 0x21",
1012 .flags = MWAIT2flg(0x21) | CPUIDLE_FLAG_TLB_FLUSHED,
1013 .exit_latency = 300,
1014 .target_residency = 300,
1015 .enter = &intel_idle,
1016 .enter_s2idle = intel_idle_s2idle, },
1017 {
1018 .name = "C10",
1019 .desc = "MWAIT 0x60",
1020 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,
1021 .exit_latency = 370,
1022 .target_residency = 2500,
1023 .enter = &intel_idle,
1024 .enter_s2idle = intel_idle_s2idle, },
1025 {
1026 .enter = NULL }
1027};
1028
1029static struct cpuidle_state gmt_cstates[] __initdata = {
1030 {
1031 .name = "C1",
1032 .desc = "MWAIT 0x00",
1033 .flags = MWAIT2flg(0x00) | CPUIDLE_FLAG_UNUSABLE,
1034 .exit_latency = 1,
1035 .target_residency = 1,
1036 .enter = intel_idle,
1037 .enter_s2idle = intel_idle_s2idle, },
1038 {
1039 .name = "C1E",
1040 .desc = "MWAIT 0x01",
1041 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
1042 .exit_latency = 2,
1043 .target_residency = 4,
1044 .enter = intel_idle,
1045 .enter_s2idle = intel_idle_s2idle, },
1046 {
1047 .name = "C6",
1048 .desc = "MWAIT 0x20",
1049 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
1050 .exit_latency = 195,
1051 .target_residency = 585,
1052 .enter = intel_idle,
1053 .enter_s2idle = intel_idle_s2idle, },
1054 {
1055 .name = "C8",
1056 .desc = "MWAIT 0x40",
1057 .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED,
1058 .exit_latency = 260,
1059 .target_residency = 1040,
1060 .enter = intel_idle,
1061 .enter_s2idle = intel_idle_s2idle, },
1062 {
1063 .name = "C10",
1064 .desc = "MWAIT 0x60",
1065 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,
1066 .exit_latency = 660,
1067 .target_residency = 1980,
1068 .enter = intel_idle,
1069 .enter_s2idle = intel_idle_s2idle, },
1070 {
1071 .enter = NULL }
1072};
1073
1074static struct cpuidle_state spr_cstates[] __initdata = {
1075 {
1076 .name = "C1",
1077 .desc = "MWAIT 0x00",
1078 .flags = MWAIT2flg(0x00),
1079 .exit_latency = 1,
1080 .target_residency = 1,
1081 .enter = intel_idle,
1082 .enter_s2idle = intel_idle_s2idle, },
1083 {
1084 .name = "C1E",
1085 .desc = "MWAIT 0x01",
1086 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
1087 .exit_latency = 2,
1088 .target_residency = 4,
1089 .enter = intel_idle,
1090 .enter_s2idle = intel_idle_s2idle, },
1091 {
1092 .name = "C6",
1093 .desc = "MWAIT 0x20",
1094 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED |
1095 CPUIDLE_FLAG_INIT_XSTATE,
1096 .exit_latency = 290,
1097 .target_residency = 800,
1098 .enter = intel_idle,
1099 .enter_s2idle = intel_idle_s2idle, },
1100 {
1101 .enter = NULL }
1102};
1103
1104static struct cpuidle_state gnr_cstates[] __initdata = {
1105 {
1106 .name = "C1",
1107 .desc = "MWAIT 0x00",
1108 .flags = MWAIT2flg(0x00),
1109 .exit_latency = 1,
1110 .target_residency = 1,
1111 .enter = intel_idle,
1112 .enter_s2idle = intel_idle_s2idle, },
1113 {
1114 .name = "C1E",
1115 .desc = "MWAIT 0x01",
1116 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
1117 .exit_latency = 4,
1118 .target_residency = 4,
1119 .enter = intel_idle,
1120 .enter_s2idle = intel_idle_s2idle, },
1121 {
1122 .name = "C6",
1123 .desc = "MWAIT 0x20",
1124 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED |
1125 CPUIDLE_FLAG_INIT_XSTATE |
1126 CPUIDLE_FLAG_PARTIAL_HINT_MATCH,
1127 .exit_latency = 170,
1128 .target_residency = 650,
1129 .enter = intel_idle,
1130 .enter_s2idle = intel_idle_s2idle, },
1131 {
1132 .name = "C6P",
1133 .desc = "MWAIT 0x21",
1134 .flags = MWAIT2flg(0x21) | CPUIDLE_FLAG_TLB_FLUSHED |
1135 CPUIDLE_FLAG_INIT_XSTATE |
1136 CPUIDLE_FLAG_PARTIAL_HINT_MATCH,
1137 .exit_latency = 210,
1138 .target_residency = 1000,
1139 .enter = intel_idle,
1140 .enter_s2idle = intel_idle_s2idle, },
1141 {
1142 .enter = NULL }
1143};
1144
1145static struct cpuidle_state gnrd_cstates[] __initdata = {
1146 {
1147 .name = "C1",
1148 .desc = "MWAIT 0x00",
1149 .flags = MWAIT2flg(0x00),
1150 .exit_latency = 1,
1151 .target_residency = 1,
1152 .enter = intel_idle,
1153 .enter_s2idle = intel_idle_s2idle, },
1154 {
1155 .name = "C1E",
1156 .desc = "MWAIT 0x01",
1157 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
1158 .exit_latency = 4,
1159 .target_residency = 4,
1160 .enter = intel_idle,
1161 .enter_s2idle = intel_idle_s2idle, },
1162 {
1163 .name = "C6",
1164 .desc = "MWAIT 0x20",
1165 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED |
1166 CPUIDLE_FLAG_INIT_XSTATE |
1167 CPUIDLE_FLAG_PARTIAL_HINT_MATCH,
1168 .exit_latency = 220,
1169 .target_residency = 650,
1170 .enter = intel_idle,
1171 .enter_s2idle = intel_idle_s2idle, },
1172 {
1173 .name = "C6P",
1174 .desc = "MWAIT 0x21",
1175 .flags = MWAIT2flg(0x21) | CPUIDLE_FLAG_TLB_FLUSHED |
1176 CPUIDLE_FLAG_INIT_XSTATE |
1177 CPUIDLE_FLAG_PARTIAL_HINT_MATCH,
1178 .exit_latency = 240,
1179 .target_residency = 750,
1180 .enter = intel_idle,
1181 .enter_s2idle = intel_idle_s2idle, },
1182 {
1183 .enter = NULL }
1184};
1185
1186static struct cpuidle_state atom_cstates[] __initdata = {
1187 {
1188 .name = "C1E",
1189 .desc = "MWAIT 0x00",
1190 .flags = MWAIT2flg(0x00),
1191 .exit_latency = 10,
1192 .target_residency = 20,
1193 .enter = intel_idle,
1194 .enter_s2idle = intel_idle_s2idle, },
1195 {
1196 .name = "C2",
1197 .desc = "MWAIT 0x10",
1198 .flags = MWAIT2flg(0x10),
1199 .exit_latency = 20,
1200 .target_residency = 80,
1201 .enter = intel_idle,
1202 .enter_s2idle = intel_idle_s2idle, },
1203 {
1204 .name = "C4",
1205 .desc = "MWAIT 0x30",
1206 .flags = MWAIT2flg(0x30) | CPUIDLE_FLAG_TLB_FLUSHED,
1207 .exit_latency = 100,
1208 .target_residency = 400,
1209 .enter = intel_idle,
1210 .enter_s2idle = intel_idle_s2idle, },
1211 {
1212 .name = "C6",
1213 .desc = "MWAIT 0x52",
1214 .flags = MWAIT2flg(0x52) | CPUIDLE_FLAG_TLB_FLUSHED,
1215 .exit_latency = 140,
1216 .target_residency = 560,
1217 .enter = intel_idle,
1218 .enter_s2idle = intel_idle_s2idle, },
1219 {
1220 .enter = NULL }
1221};
1222static struct cpuidle_state tangier_cstates[] __initdata = {
1223 {
1224 .name = "C1",
1225 .desc = "MWAIT 0x00",
1226 .flags = MWAIT2flg(0x00),
1227 .exit_latency = 1,
1228 .target_residency = 4,
1229 .enter = intel_idle,
1230 .enter_s2idle = intel_idle_s2idle, },
1231 {
1232 .name = "C4",
1233 .desc = "MWAIT 0x30",
1234 .flags = MWAIT2flg(0x30) | CPUIDLE_FLAG_TLB_FLUSHED,
1235 .exit_latency = 100,
1236 .target_residency = 400,
1237 .enter = intel_idle,
1238 .enter_s2idle = intel_idle_s2idle, },
1239 {
1240 .name = "C6",
1241 .desc = "MWAIT 0x52",
1242 .flags = MWAIT2flg(0x52) | CPUIDLE_FLAG_TLB_FLUSHED,
1243 .exit_latency = 140,
1244 .target_residency = 560,
1245 .enter = intel_idle,
1246 .enter_s2idle = intel_idle_s2idle, },
1247 {
1248 .name = "C7",
1249 .desc = "MWAIT 0x60",
1250 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,
1251 .exit_latency = 1200,
1252 .target_residency = 4000,
1253 .enter = intel_idle,
1254 .enter_s2idle = intel_idle_s2idle, },
1255 {
1256 .name = "C9",
1257 .desc = "MWAIT 0x64",
1258 .flags = MWAIT2flg(0x64) | CPUIDLE_FLAG_TLB_FLUSHED,
1259 .exit_latency = 10000,
1260 .target_residency = 20000,
1261 .enter = intel_idle,
1262 .enter_s2idle = intel_idle_s2idle, },
1263 {
1264 .enter = NULL }
1265};
1266static struct cpuidle_state avn_cstates[] __initdata = {
1267 {
1268 .name = "C1",
1269 .desc = "MWAIT 0x00",
1270 .flags = MWAIT2flg(0x00),
1271 .exit_latency = 2,
1272 .target_residency = 2,
1273 .enter = intel_idle,
1274 .enter_s2idle = intel_idle_s2idle, },
1275 {
1276 .name = "C6",
1277 .desc = "MWAIT 0x51",
1278 .flags = MWAIT2flg(0x51) | CPUIDLE_FLAG_TLB_FLUSHED,
1279 .exit_latency = 15,
1280 .target_residency = 45,
1281 .enter = intel_idle,
1282 .enter_s2idle = intel_idle_s2idle, },
1283 {
1284 .enter = NULL }
1285};
1286static struct cpuidle_state knl_cstates[] __initdata = {
1287 {
1288 .name = "C1",
1289 .desc = "MWAIT 0x00",
1290 .flags = MWAIT2flg(0x00),
1291 .exit_latency = 1,
1292 .target_residency = 2,
1293 .enter = intel_idle,
1294 .enter_s2idle = intel_idle_s2idle },
1295 {
1296 .name = "C6",
1297 .desc = "MWAIT 0x10",
1298 .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
1299 .exit_latency = 120,
1300 .target_residency = 500,
1301 .enter = intel_idle,
1302 .enter_s2idle = intel_idle_s2idle },
1303 {
1304 .enter = NULL }
1305};
1306
1307static struct cpuidle_state bxt_cstates[] __initdata = {
1308 {
1309 .name = "C1",
1310 .desc = "MWAIT 0x00",
1311 .flags = MWAIT2flg(0x00),
1312 .exit_latency = 2,
1313 .target_residency = 2,
1314 .enter = intel_idle,
1315 .enter_s2idle = intel_idle_s2idle, },
1316 {
1317 .name = "C1E",
1318 .desc = "MWAIT 0x01",
1319 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
1320 .exit_latency = 10,
1321 .target_residency = 20,
1322 .enter = intel_idle,
1323 .enter_s2idle = intel_idle_s2idle, },
1324 {
1325 .name = "C6",
1326 .desc = "MWAIT 0x20",
1327 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
1328 .exit_latency = 133,
1329 .target_residency = 133,
1330 .enter = intel_idle,
1331 .enter_s2idle = intel_idle_s2idle, },
1332 {
1333 .name = "C7s",
1334 .desc = "MWAIT 0x31",
1335 .flags = MWAIT2flg(0x31) | CPUIDLE_FLAG_TLB_FLUSHED,
1336 .exit_latency = 155,
1337 .target_residency = 155,
1338 .enter = intel_idle,
1339 .enter_s2idle = intel_idle_s2idle, },
1340 {
1341 .name = "C8",
1342 .desc = "MWAIT 0x40",
1343 .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED,
1344 .exit_latency = 1000,
1345 .target_residency = 1000,
1346 .enter = intel_idle,
1347 .enter_s2idle = intel_idle_s2idle, },
1348 {
1349 .name = "C9",
1350 .desc = "MWAIT 0x50",
1351 .flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED,
1352 .exit_latency = 2000,
1353 .target_residency = 2000,
1354 .enter = intel_idle,
1355 .enter_s2idle = intel_idle_s2idle, },
1356 {
1357 .name = "C10",
1358 .desc = "MWAIT 0x60",
1359 .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,
1360 .exit_latency = 10000,
1361 .target_residency = 10000,
1362 .enter = intel_idle,
1363 .enter_s2idle = intel_idle_s2idle, },
1364 {
1365 .enter = NULL }
1366};
1367
1368static struct cpuidle_state dnv_cstates[] __initdata = {
1369 {
1370 .name = "C1",
1371 .desc = "MWAIT 0x00",
1372 .flags = MWAIT2flg(0x00),
1373 .exit_latency = 2,
1374 .target_residency = 2,
1375 .enter = intel_idle,
1376 .enter_s2idle = intel_idle_s2idle, },
1377 {
1378 .name = "C1E",
1379 .desc = "MWAIT 0x01",
1380 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
1381 .exit_latency = 10,
1382 .target_residency = 20,
1383 .enter = intel_idle,
1384 .enter_s2idle = intel_idle_s2idle, },
1385 {
1386 .name = "C6",
1387 .desc = "MWAIT 0x20",
1388 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
1389 .exit_latency = 50,
1390 .target_residency = 500,
1391 .enter = intel_idle,
1392 .enter_s2idle = intel_idle_s2idle, },
1393 {
1394 .enter = NULL }
1395};
1396
1397/*
1398 * Note, depending on HW and FW revision, SnowRidge SoC may or may not support
1399 * C6, and this is indicated in the CPUID mwait leaf.
1400 */
1401static struct cpuidle_state snr_cstates[] __initdata = {
1402 {
1403 .name = "C1",
1404 .desc = "MWAIT 0x00",
1405 .flags = MWAIT2flg(0x00),
1406 .exit_latency = 2,
1407 .target_residency = 2,
1408 .enter = intel_idle,
1409 .enter_s2idle = intel_idle_s2idle, },
1410 {
1411 .name = "C1E",
1412 .desc = "MWAIT 0x01",
1413 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
1414 .exit_latency = 15,
1415 .target_residency = 25,
1416 .enter = intel_idle,
1417 .enter_s2idle = intel_idle_s2idle, },
1418 {
1419 .name = "C6",
1420 .desc = "MWAIT 0x20",
1421 .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
1422 .exit_latency = 130,
1423 .target_residency = 500,
1424 .enter = intel_idle,
1425 .enter_s2idle = intel_idle_s2idle, },
1426 {
1427 .enter = NULL }
1428};
1429
1430static struct cpuidle_state grr_cstates[] __initdata = {
1431 {
1432 .name = "C1",
1433 .desc = "MWAIT 0x00",
1434 .flags = MWAIT2flg(0x00) | CPUIDLE_FLAG_ALWAYS_ENABLE,
1435 .exit_latency = 1,
1436 .target_residency = 1,
1437 .enter = intel_idle,
1438 .enter_s2idle = intel_idle_s2idle, },
1439 {
1440 .name = "C1E",
1441 .desc = "MWAIT 0x01",
1442 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
1443 .exit_latency = 2,
1444 .target_residency = 10,
1445 .enter = intel_idle,
1446 .enter_s2idle = intel_idle_s2idle, },
1447 {
1448 .name = "C6S",
1449 .desc = "MWAIT 0x22",
1450 .flags = MWAIT2flg(0x22) | CPUIDLE_FLAG_TLB_FLUSHED,
1451 .exit_latency = 140,
1452 .target_residency = 500,
1453 .enter = intel_idle,
1454 .enter_s2idle = intel_idle_s2idle, },
1455 {
1456 .enter = NULL }
1457};
1458
1459static struct cpuidle_state srf_cstates[] __initdata = {
1460 {
1461 .name = "C1",
1462 .desc = "MWAIT 0x00",
1463 .flags = MWAIT2flg(0x00) | CPUIDLE_FLAG_ALWAYS_ENABLE,
1464 .exit_latency = 1,
1465 .target_residency = 1,
1466 .enter = intel_idle,
1467 .enter_s2idle = intel_idle_s2idle, },
1468 {
1469 .name = "C1E",
1470 .desc = "MWAIT 0x01",
1471 .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
1472 .exit_latency = 2,
1473 .target_residency = 10,
1474 .enter = intel_idle,
1475 .enter_s2idle = intel_idle_s2idle, },
1476 {
1477 .name = "C6S",
1478 .desc = "MWAIT 0x22",
1479 .flags = MWAIT2flg(0x22) | CPUIDLE_FLAG_TLB_FLUSHED |
1480 CPUIDLE_FLAG_PARTIAL_HINT_MATCH,
1481 .exit_latency = 270,
1482 .target_residency = 700,
1483 .enter = intel_idle,
1484 .enter_s2idle = intel_idle_s2idle, },
1485 {
1486 .name = "C6SP",
1487 .desc = "MWAIT 0x23",
1488 .flags = MWAIT2flg(0x23) | CPUIDLE_FLAG_TLB_FLUSHED |
1489 CPUIDLE_FLAG_PARTIAL_HINT_MATCH,
1490 .exit_latency = 310,
1491 .target_residency = 900,
1492 .enter = intel_idle,
1493 .enter_s2idle = intel_idle_s2idle, },
1494 {
1495 .enter = NULL }
1496};
1497
1498static const struct idle_cpu idle_cpu_nehalem __initconst = {
1499 .state_table = nehalem_cstates,
1500 .auto_demotion_disable_flags = NHM_C1_AUTO_DEMOTE | NHM_C3_AUTO_DEMOTE,
1501 .disable_promotion_to_c1e = true,
1502};
1503
1504static const struct idle_cpu idle_cpu_nhx __initconst = {
1505 .state_table = nehalem_cstates,
1506 .auto_demotion_disable_flags = NHM_C1_AUTO_DEMOTE | NHM_C3_AUTO_DEMOTE,
1507 .disable_promotion_to_c1e = true,
1508 .use_acpi = true,
1509};
1510
1511static const struct idle_cpu idle_cpu_atom __initconst = {
1512 .state_table = atom_cstates,
1513};
1514
1515static const struct idle_cpu idle_cpu_tangier __initconst = {
1516 .state_table = tangier_cstates,
1517};
1518
1519static const struct idle_cpu idle_cpu_lincroft __initconst = {
1520 .state_table = atom_cstates,
1521 .auto_demotion_disable_flags = ATM_LNC_C6_AUTO_DEMOTE,
1522};
1523
1524static const struct idle_cpu idle_cpu_snb __initconst = {
1525 .state_table = snb_cstates,
1526 .disable_promotion_to_c1e = true,
1527};
1528
1529static const struct idle_cpu idle_cpu_snx __initconst = {
1530 .state_table = snb_cstates,
1531 .disable_promotion_to_c1e = true,
1532 .use_acpi = true,
1533};
1534
1535static const struct idle_cpu idle_cpu_byt __initconst = {
1536 .state_table = byt_cstates,
1537 .disable_promotion_to_c1e = true,
1538};
1539
1540static const struct idle_cpu idle_cpu_cht __initconst = {
1541 .state_table = cht_cstates,
1542 .disable_promotion_to_c1e = true,
1543};
1544
1545static const struct idle_cpu idle_cpu_ivb __initconst = {
1546 .state_table = ivb_cstates,
1547 .disable_promotion_to_c1e = true,
1548};
1549
1550static const struct idle_cpu idle_cpu_ivt __initconst = {
1551 .state_table = ivt_cstates,
1552 .disable_promotion_to_c1e = true,
1553 .use_acpi = true,
1554};
1555
1556static const struct idle_cpu idle_cpu_hsw __initconst = {
1557 .state_table = hsw_cstates,
1558 .disable_promotion_to_c1e = true,
1559};
1560
1561static const struct idle_cpu idle_cpu_hsx __initconst = {
1562 .state_table = hsw_cstates,
1563 .disable_promotion_to_c1e = true,
1564 .use_acpi = true,
1565};
1566
1567static const struct idle_cpu idle_cpu_bdw __initconst = {
1568 .state_table = bdw_cstates,
1569 .disable_promotion_to_c1e = true,
1570};
1571
1572static const struct idle_cpu idle_cpu_bdx __initconst = {
1573 .state_table = bdw_cstates,
1574 .disable_promotion_to_c1e = true,
1575 .use_acpi = true,
1576};
1577
1578static const struct idle_cpu idle_cpu_skl __initconst = {
1579 .state_table = skl_cstates,
1580 .disable_promotion_to_c1e = true,
1581};
1582
1583static const struct idle_cpu idle_cpu_skx __initconst = {
1584 .state_table = skx_cstates,
1585 .disable_promotion_to_c1e = true,
1586 .use_acpi = true,
1587};
1588
1589static const struct idle_cpu idle_cpu_icx __initconst = {
1590 .state_table = icx_cstates,
1591 .disable_promotion_to_c1e = true,
1592 .use_acpi = true,
1593};
1594
1595static const struct idle_cpu idle_cpu_adl __initconst = {
1596 .state_table = adl_cstates,
1597};
1598
1599static const struct idle_cpu idle_cpu_adl_l __initconst = {
1600 .state_table = adl_l_cstates,
1601};
1602
1603static const struct idle_cpu idle_cpu_mtl_l __initconst = {
1604 .state_table = mtl_l_cstates,
1605};
1606
1607static const struct idle_cpu idle_cpu_ptl __initconst = {
1608 .state_table = ptl_cstates,
1609};
1610
1611static const struct idle_cpu idle_cpu_gmt __initconst = {
1612 .state_table = gmt_cstates,
1613};
1614
1615static const struct idle_cpu idle_cpu_spr __initconst = {
1616 .state_table = spr_cstates,
1617 .disable_promotion_to_c1e = true,
1618 .c1_demotion_supported = true,
1619 .use_acpi = true,
1620};
1621
1622static const struct idle_cpu idle_cpu_gnr __initconst = {
1623 .state_table = gnr_cstates,
1624 .disable_promotion_to_c1e = true,
1625 .c1_demotion_supported = true,
1626 .use_acpi = true,
1627};
1628
1629static const struct idle_cpu idle_cpu_gnrd __initconst = {
1630 .state_table = gnrd_cstates,
1631 .disable_promotion_to_c1e = true,
1632 .c1_demotion_supported = true,
1633 .use_acpi = true,
1634};
1635
1636static const struct idle_cpu idle_cpu_avn __initconst = {
1637 .state_table = avn_cstates,
1638 .disable_promotion_to_c1e = true,
1639 .use_acpi = true,
1640};
1641
1642static const struct idle_cpu idle_cpu_knl __initconst = {
1643 .state_table = knl_cstates,
1644 .use_acpi = true,
1645};
1646
1647static const struct idle_cpu idle_cpu_bxt __initconst = {
1648 .state_table = bxt_cstates,
1649 .disable_promotion_to_c1e = true,
1650};
1651
1652static const struct idle_cpu idle_cpu_dnv __initconst = {
1653 .state_table = dnv_cstates,
1654 .disable_promotion_to_c1e = true,
1655 .use_acpi = true,
1656};
1657
1658static const struct idle_cpu idle_cpu_tmt __initconst = {
1659 .disable_promotion_to_c1e = true,
1660};
1661
1662static const struct idle_cpu idle_cpu_snr __initconst = {
1663 .state_table = snr_cstates,
1664 .disable_promotion_to_c1e = true,
1665 .use_acpi = true,
1666};
1667
1668static const struct idle_cpu idle_cpu_grr __initconst = {
1669 .state_table = grr_cstates,
1670 .disable_promotion_to_c1e = true,
1671 .c1_demotion_supported = true,
1672 .use_acpi = true,
1673};
1674
1675static const struct idle_cpu idle_cpu_srf __initconst = {
1676 .state_table = srf_cstates,
1677 .disable_promotion_to_c1e = true,
1678 .c1_demotion_supported = true,
1679 .use_acpi = true,
1680};
1681
1682static const struct x86_cpu_id intel_idle_ids[] __initconst = {
1683 X86_MATCH_VFM(INTEL_NEHALEM_EP, &idle_cpu_nhx),
1684 X86_MATCH_VFM(INTEL_NEHALEM, &idle_cpu_nehalem),
1685 X86_MATCH_VFM(INTEL_NEHALEM_G, &idle_cpu_nehalem),
1686 X86_MATCH_VFM(INTEL_WESTMERE, &idle_cpu_nehalem),
1687 X86_MATCH_VFM(INTEL_WESTMERE_EP, &idle_cpu_nhx),
1688 X86_MATCH_VFM(INTEL_NEHALEM_EX, &idle_cpu_nhx),
1689 X86_MATCH_VFM(INTEL_ATOM_BONNELL, &idle_cpu_atom),
1690 X86_MATCH_VFM(INTEL_ATOM_BONNELL_MID, &idle_cpu_lincroft),
1691 X86_MATCH_VFM(INTEL_WESTMERE_EX, &idle_cpu_nhx),
1692 X86_MATCH_VFM(INTEL_SANDYBRIDGE, &idle_cpu_snb),
1693 X86_MATCH_VFM(INTEL_SANDYBRIDGE_X, &idle_cpu_snx),
1694 X86_MATCH_VFM(INTEL_ATOM_SALTWELL, &idle_cpu_atom),
1695 X86_MATCH_VFM(INTEL_ATOM_SILVERMONT, &idle_cpu_byt),
1696 X86_MATCH_VFM(INTEL_ATOM_SILVERMONT_MID, &idle_cpu_tangier),
1697 X86_MATCH_VFM(INTEL_ATOM_AIRMONT, &idle_cpu_cht),
1698 X86_MATCH_VFM(INTEL_IVYBRIDGE, &idle_cpu_ivb),
1699 X86_MATCH_VFM(INTEL_IVYBRIDGE_X, &idle_cpu_ivt),
1700 X86_MATCH_VFM(INTEL_HASWELL, &idle_cpu_hsw),
1701 X86_MATCH_VFM(INTEL_HASWELL_X, &idle_cpu_hsx),
1702 X86_MATCH_VFM(INTEL_HASWELL_L, &idle_cpu_hsw),
1703 X86_MATCH_VFM(INTEL_HASWELL_G, &idle_cpu_hsw),
1704 X86_MATCH_VFM(INTEL_ATOM_SILVERMONT_D, &idle_cpu_avn),
1705 X86_MATCH_VFM(INTEL_BROADWELL, &idle_cpu_bdw),
1706 X86_MATCH_VFM(INTEL_BROADWELL_G, &idle_cpu_bdw),
1707 X86_MATCH_VFM(INTEL_BROADWELL_X, &idle_cpu_bdx),
1708 X86_MATCH_VFM(INTEL_BROADWELL_D, &idle_cpu_bdx),
1709 X86_MATCH_VFM(INTEL_SKYLAKE_L, &idle_cpu_skl),
1710 X86_MATCH_VFM(INTEL_SKYLAKE, &idle_cpu_skl),
1711 X86_MATCH_VFM(INTEL_KABYLAKE_L, &idle_cpu_skl),
1712 X86_MATCH_VFM(INTEL_KABYLAKE, &idle_cpu_skl),
1713 X86_MATCH_VFM(INTEL_SKYLAKE_X, &idle_cpu_skx),
1714 X86_MATCH_VFM(INTEL_ICELAKE_X, &idle_cpu_icx),
1715 X86_MATCH_VFM(INTEL_ICELAKE_D, &idle_cpu_icx),
1716 X86_MATCH_VFM(INTEL_ALDERLAKE, &idle_cpu_adl),
1717 X86_MATCH_VFM(INTEL_ALDERLAKE_L, &idle_cpu_adl_l),
1718 X86_MATCH_VFM(INTEL_METEORLAKE_L, &idle_cpu_mtl_l),
1719 X86_MATCH_VFM(INTEL_PANTHERLAKE_L, &idle_cpu_ptl),
1720 X86_MATCH_VFM(INTEL_ATOM_GRACEMONT, &idle_cpu_gmt),
1721 X86_MATCH_VFM(INTEL_SAPPHIRERAPIDS_X, &idle_cpu_spr),
1722 X86_MATCH_VFM(INTEL_EMERALDRAPIDS_X, &idle_cpu_spr),
1723 X86_MATCH_VFM(INTEL_GRANITERAPIDS_X, &idle_cpu_gnr),
1724 X86_MATCH_VFM(INTEL_GRANITERAPIDS_D, &idle_cpu_gnrd),
1725 X86_MATCH_VFM(INTEL_XEON_PHI_KNL, &idle_cpu_knl),
1726 X86_MATCH_VFM(INTEL_XEON_PHI_KNM, &idle_cpu_knl),
1727 X86_MATCH_VFM(INTEL_ATOM_GOLDMONT, &idle_cpu_bxt),
1728 X86_MATCH_VFM(INTEL_ATOM_GOLDMONT_PLUS, &idle_cpu_bxt),
1729 X86_MATCH_VFM(INTEL_ATOM_GOLDMONT_D, &idle_cpu_dnv),
1730 X86_MATCH_VFM(INTEL_ATOM_TREMONT, &idle_cpu_tmt),
1731 X86_MATCH_VFM(INTEL_ATOM_TREMONT_L, &idle_cpu_tmt),
1732 X86_MATCH_VFM(INTEL_ATOM_TREMONT_D, &idle_cpu_snr),
1733 X86_MATCH_VFM(INTEL_ATOM_CRESTMONT, &idle_cpu_grr),
1734 X86_MATCH_VFM(INTEL_ATOM_CRESTMONT_X, &idle_cpu_srf),
1735 X86_MATCH_VFM(INTEL_ATOM_DARKMONT_X, &idle_cpu_srf),
1736 {}
1737};
1738
1739static const struct x86_cpu_id intel_mwait_ids[] __initconst = {
1740 X86_MATCH_VENDOR_FAM_FEATURE(INTEL, X86_FAMILY_ANY, X86_FEATURE_MWAIT, NULL),
1741 {}
1742};
1743
1744static bool __init intel_idle_max_cstate_reached(int cstate)
1745{
1746 if (cstate + 1 > max_cstate) {
1747 pr_info("max_cstate %d reached\n", max_cstate);
1748 return true;
1749 }
1750 return false;
1751}
1752
1753static bool __init intel_idle_state_needs_timer_stop(struct cpuidle_state *state)
1754{
1755 unsigned long eax = flg2MWAIT(state->flags);
1756
1757 if (boot_cpu_has(X86_FEATURE_ARAT))
1758 return false;
1759
1760 /*
1761 * Switch over to one-shot tick broadcast if the target C-state
1762 * is deeper than C1.
1763 */
1764 return !!((eax >> MWAIT_SUBSTATE_SIZE) & MWAIT_CSTATE_MASK);
1765}
1766
1767#ifdef CONFIG_ACPI_PROCESSOR_CSTATE
1768#include <acpi/processor.h>
1769
1770static bool no_acpi __read_mostly;
1771module_param(no_acpi, bool, 0444);
1772MODULE_PARM_DESC(no_acpi, "Do not use ACPI _CST for building the idle states list");
1773
1774static bool force_use_acpi __read_mostly; /* No effect if no_acpi is set. */
1775module_param_named(use_acpi, force_use_acpi, bool, 0444);
1776MODULE_PARM_DESC(use_acpi, "Use ACPI _CST for building the idle states list");
1777
1778static bool no_native __read_mostly; /* No effect if no_acpi is set. */
1779module_param_named(no_native, no_native, bool, 0444);
1780MODULE_PARM_DESC(no_native, "Ignore cpu specific (native) idle states in lieu of ACPI idle states");
1781
1782static struct acpi_processor_power acpi_state_table __initdata;
1783static bool acpi_lpi_available __initdata;
1784
1785/**
1786 * intel_idle_cst_usable - Check if the _CST information can be used.
1787 *
1788 * Check if all of the C-states listed by _CST in the max_cstate range are
1789 * ACPI_CSTATE_FFH, which means that they should be entered via MWAIT.
1790 */
1791static bool __init intel_idle_cst_usable(void)
1792{
1793 int cstate, limit;
1794
1795 limit = min_t(int, min_t(int, CPUIDLE_STATE_MAX, max_cstate + 1),
1796 acpi_state_table.count);
1797
1798 for (cstate = 1; cstate < limit; cstate++) {
1799 struct acpi_processor_cx *cx = &acpi_state_table.states[cstate];
1800
1801 if (cx->entry_method != ACPI_CSTATE_FFH)
1802 return false;
1803 }
1804
1805 return true;
1806}
1807
1808static bool __init intel_idle_acpi_extract_lpi_cstates(void)
1809{
1810 unsigned int cpu;
1811
1812 for_each_possible_cpu(cpu) {
1813 struct acpi_processor *pr;
1814
1815 pr = per_cpu(processors, cpu);
1816 if (!pr)
1817 continue;
1818
1819 if (acpi_processor_extract_lpi_info(pr->handle,
1820 &acpi_state_table, true))
1821 continue;
1822
1823 acpi_lpi_available = true;
1824 return true;
1825 }
1826
1827 pr_debug("No ACPI _LPI idle states\n");
1828 return false;
1829}
1830
1831static bool __init intel_idle_acpi_extract_cst_cstates(void)
1832{
1833 unsigned int cpu;
1834
1835 for_each_possible_cpu(cpu) {
1836 struct acpi_processor *pr;
1837
1838 pr = per_cpu(processors, cpu);
1839 if (!pr)
1840 continue;
1841
1842 if (acpi_processor_evaluate_cst(pr->handle, cpu, &acpi_state_table))
1843 continue;
1844
1845 acpi_state_table.count++;
1846
1847 if (!intel_idle_cst_usable())
1848 continue;
1849
1850 return true;
1851 }
1852
1853 pr_debug("ACPI _CST not found or not usable\n");
1854 return false;
1855}
1856
1857static bool __init intel_idle_acpi_extract_cstates(void)
1858{
1859 if (intel_idle_acpi_extract_lpi_cstates())
1860 return true;
1861
1862 if (intel_idle_acpi_extract_cst_cstates())
1863 return true;
1864
1865 return false;
1866}
1867
1868static bool __init intel_idle_acpi_probe(void)
1869{
1870 if (no_acpi) {
1871 pr_debug("Not allowed to use ACPI for C-states extraction\n");
1872 return false;
1873 }
1874
1875 if (intel_idle_acpi_extract_cstates() &&
1876 acpi_processor_claim_cst_control())
1877 return true;
1878
1879 acpi_state_table.count = 0;
1880 return false;
1881}
1882
1883static void __init intel_idle_complete_state_init(struct cpuidle_state *state)
1884{
1885 if (intel_idle_state_needs_timer_stop(state))
1886 state->flags |= CPUIDLE_FLAG_TIMER_STOP;
1887
1888 state->enter = intel_idle;
1889 state->enter_dead = intel_idle_enter_dead;
1890 state->enter_s2idle = intel_idle_s2idle;
1891}
1892
1893static void __init intel_idle_init_cstates_acpi_lpi(struct cpuidle_driver *drv)
1894{
1895 int index;
1896
1897 for (index = 0; index < acpi_state_table.count; index++) {
1898 struct acpi_lpi_state *lpi_state;
1899 struct cpuidle_state *state;
1900
1901 if (intel_idle_max_cstate_reached(index))
1902 break;
1903
1904 lpi_state = &acpi_state_table.lpi_states[index];
1905
1906 state = &drv->states[drv->state_count++];
1907
1908 scnprintf(state->name, CPUIDLE_NAME_LEN, "C%d_LPI", index + 1);
1909 strscpy(state->desc, lpi_state->desc, CPUIDLE_DESC_LEN);
1910 state->exit_latency = lpi_state->wake_latency;
1911 state->target_residency = lpi_state->min_residency;
1912 state->flags = MWAIT2flg(lpi_state->address);
1913 /*
1914 * Assume that entering any of the idle states extracted from
1915 * _LPI except for the first two will cause the TLB to be
1916 * flushed and let the core call leave_mm() for them upfront
1917 * to avoid unnecessary wakeups due to TLB shootdowns.
1918 */
1919 if (index > 1)
1920 state->flags |= CPUIDLE_FLAG_TLB_FLUSHED;
1921
1922 if (disabled_states_mask & BIT(index + 1))
1923 state->flags |= CPUIDLE_FLAG_OFF;
1924
1925 intel_idle_complete_state_init(state);
1926
1927 pr_info("%s: MWAIT hint 0x%x\n", state->name, flg2MWAIT(state->flags));
1928 }
1929
1930 /*
1931 * Assume the first idle state in the table to be C1 and if any deeper
1932 * idle states are exposed while X86_FEATURE_NONSTOP_TSC is unset, mark
1933 * the TSC as unstable.
1934 */
1935 if (index > 1 && !boot_cpu_has(X86_FEATURE_NONSTOP_TSC))
1936 mark_tsc_unstable("TSC halts in idle");
1937}
1938
1939static void __init intel_idle_init_cstates_acpi_cst(struct cpuidle_driver *drv)
1940{
1941 int cstate, limit = min_t(int, CPUIDLE_STATE_MAX, acpi_state_table.count);
1942
1943 /*
1944 * If limit > 0, intel_idle_cst_usable() has returned 'true', so all of
1945 * the interesting states are ACPI_CSTATE_FFH.
1946 */
1947 for (cstate = 1; cstate < limit; cstate++) {
1948 struct acpi_processor_cx *cx;
1949 struct cpuidle_state *state;
1950
1951 if (intel_idle_max_cstate_reached(cstate - 1))
1952 break;
1953
1954 cx = &acpi_state_table.states[cstate];
1955
1956 state = &drv->states[drv->state_count++];
1957
1958 snprintf(state->name, CPUIDLE_NAME_LEN, "C%d_ACPI", cstate);
1959 strscpy(state->desc, cx->desc, CPUIDLE_DESC_LEN);
1960 state->exit_latency = cx->latency;
1961 /*
1962 * For C1-type C-states use the same number for both the exit
1963 * latency and target residency, because that is the case for
1964 * C1 in the majority of the static C-states tables above.
1965 * For the other types of C-states, however, set the target
1966 * residency to 3 times the exit latency which should lead to
1967 * a reasonable balance between energy-efficiency and
1968 * performance in the majority of interesting cases.
1969 */
1970 state->target_residency = cx->latency;
1971 if (cx->type > ACPI_STATE_C1)
1972 state->target_residency *= 3;
1973
1974 state->flags = MWAIT2flg(cx->address);
1975 if (cx->type > ACPI_STATE_C2)
1976 state->flags |= CPUIDLE_FLAG_TLB_FLUSHED;
1977
1978 if (disabled_states_mask & BIT(cstate))
1979 state->flags |= CPUIDLE_FLAG_OFF;
1980
1981 if (cx->type > ACPI_STATE_C1 && !boot_cpu_has(X86_FEATURE_NONSTOP_TSC))
1982 mark_tsc_unstable("TSC halts in idle");
1983
1984 intel_idle_complete_state_init(state);
1985 }
1986}
1987
1988static void __init intel_idle_init_cstates_acpi(struct cpuidle_driver *drv)
1989{
1990 if (acpi_lpi_available)
1991 intel_idle_init_cstates_acpi_lpi(drv);
1992 else
1993 intel_idle_init_cstates_acpi_cst(drv);
1994}
1995
1996static bool __init intel_idle_acpi_hint_match(unsigned int flags, u32 acpi_hint,
1997 u32 table_hint)
1998{
1999 if (flags & CPUIDLE_FLAG_PARTIAL_HINT_MATCH) {
2000 acpi_hint &= ~MWAIT_SUBSTATE_MASK;
2001 table_hint &= ~MWAIT_SUBSTATE_MASK;
2002 }
2003 return acpi_hint == table_hint;
2004}
2005
2006static bool __init intel_idle_off_by_default_lpi(unsigned int flags, u32 mwait_hint)
2007{
2008 int index;
2009
2010 for (index = 0; index < acpi_state_table.count; index++) {
2011 u32 acpi_hint = acpi_state_table.lpi_states[index].address;
2012
2013 if (intel_idle_acpi_hint_match(flags, acpi_hint, mwait_hint))
2014 return false;
2015 }
2016 return true;
2017}
2018
2019static bool __init intel_idle_off_by_default_cst(unsigned int flags, u32 mwait_hint)
2020{
2021 int cstate, limit = min_t(int, CPUIDLE_STATE_MAX, acpi_state_table.count);
2022
2023 /*
2024 * If limit > 0, intel_idle_cst_usable() has returned 'true', so all of
2025 * the interesting states are ACPI_CSTATE_FFH.
2026 */
2027 for (cstate = 1; cstate < limit; cstate++) {
2028 u32 acpi_hint = acpi_state_table.states[cstate].address;
2029
2030 if (intel_idle_acpi_hint_match(flags, acpi_hint, mwait_hint))
2031 return false;
2032 }
2033 return true;
2034}
2035
2036static bool __init intel_idle_off_by_default(unsigned int flags, u32 mwait_hint)
2037{
2038 /*
2039 * If there is no C-states information in the ACPI tables, do not
2040 * disable any C-states by default.
2041 */
2042 if (!acpi_state_table.count)
2043 return false;
2044
2045 if (acpi_lpi_available)
2046 return intel_idle_off_by_default_lpi(flags, mwait_hint);
2047
2048 return intel_idle_off_by_default_cst(flags, mwait_hint);
2049}
2050
2051static inline bool ignore_native(void)
2052{
2053 return no_native && !no_acpi;
2054}
2055#else /* !CONFIG_ACPI_PROCESSOR_CSTATE */
2056#define force_use_acpi (false)
2057
2058static inline bool intel_idle_acpi_probe(void) { return false; }
2059static inline void intel_idle_init_cstates_acpi(struct cpuidle_driver *drv) { }
2060static inline bool intel_idle_off_by_default(unsigned int flags, u32 mwait_hint)
2061{
2062 return false;
2063}
2064static inline bool ignore_native(void) { return false; }
2065#endif /* !CONFIG_ACPI_PROCESSOR_CSTATE */
2066
2067/**
2068 * ivt_idle_state_table_update - Tune the idle states table for Ivy Town.
2069 *
2070 * Tune IVT multi-socket targets.
2071 * Assumption: num_sockets == (max_package_num + 1).
2072 */
2073static void __init ivt_idle_state_table_update(void)
2074{
2075 /* IVT uses a different table for 1-2, 3-4, and > 4 sockets */
2076 int cpu, package_num, num_sockets = 1;
2077
2078 for_each_online_cpu(cpu) {
2079 package_num = topology_physical_package_id(cpu);
2080 if (package_num + 1 > num_sockets) {
2081 num_sockets = package_num + 1;
2082
2083 if (num_sockets > 4) {
2084 cpuidle_state_table = ivt_cstates_8s;
2085 return;
2086 }
2087 }
2088 }
2089
2090 if (num_sockets > 2)
2091 cpuidle_state_table = ivt_cstates_4s;
2092
2093 /* else, 1 and 2 socket systems use default ivt_cstates */
2094}
2095
2096/**
2097 * irtl_2_usec - IRTL to microseconds conversion.
2098 * @irtl: IRTL MSR value.
2099 *
2100 * Translate the IRTL (Interrupt Response Time Limit) MSR value to microseconds.
2101 */
2102static unsigned long long __init irtl_2_usec(unsigned long long irtl)
2103{
2104 static const unsigned int irtl_ns_units[] __initconst = {
2105 1, 32, 1024, 32768, 1048576, 33554432, 0, 0
2106 };
2107 unsigned long long ns;
2108
2109 if (!irtl)
2110 return 0;
2111
2112 ns = irtl_ns_units[(irtl >> 10) & 0x7];
2113
2114 return div_u64((irtl & 0x3FF) * ns, NSEC_PER_USEC);
2115}
2116
2117/**
2118 * bxt_idle_state_table_update - Fix up the Broxton idle states table.
2119 *
2120 * On BXT, trust the IRTL (Interrupt Response Time Limit) MSR to show the
2121 * definitive maximum latency and use the same value for target_residency.
2122 */
2123static void __init bxt_idle_state_table_update(void)
2124{
2125 unsigned long long msr;
2126 unsigned int usec;
2127
2128 rdmsrq(MSR_PKGC6_IRTL, msr);
2129 usec = irtl_2_usec(msr);
2130 if (usec) {
2131 bxt_cstates[2].exit_latency = usec;
2132 bxt_cstates[2].target_residency = usec;
2133 }
2134
2135 rdmsrq(MSR_PKGC7_IRTL, msr);
2136 usec = irtl_2_usec(msr);
2137 if (usec) {
2138 bxt_cstates[3].exit_latency = usec;
2139 bxt_cstates[3].target_residency = usec;
2140 }
2141
2142 rdmsrq(MSR_PKGC8_IRTL, msr);
2143 usec = irtl_2_usec(msr);
2144 if (usec) {
2145 bxt_cstates[4].exit_latency = usec;
2146 bxt_cstates[4].target_residency = usec;
2147 }
2148
2149 rdmsrq(MSR_PKGC9_IRTL, msr);
2150 usec = irtl_2_usec(msr);
2151 if (usec) {
2152 bxt_cstates[5].exit_latency = usec;
2153 bxt_cstates[5].target_residency = usec;
2154 }
2155
2156 rdmsrq(MSR_PKGC10_IRTL, msr);
2157 usec = irtl_2_usec(msr);
2158 if (usec) {
2159 bxt_cstates[6].exit_latency = usec;
2160 bxt_cstates[6].target_residency = usec;
2161 }
2162
2163}
2164
2165/**
2166 * sklh_idle_state_table_update - Fix up the Sky Lake idle states table.
2167 *
2168 * On SKL-H (model 0x5e) skip C8 and C9 if C10 is enabled and SGX disabled.
2169 */
2170static void __init sklh_idle_state_table_update(void)
2171{
2172 unsigned long long msr;
2173 unsigned int eax, ebx, ecx, edx;
2174
2175
2176 /* if PC10 disabled via cmdline intel_idle.max_cstate=7 or shallower */
2177 if (max_cstate <= 7)
2178 return;
2179
2180 /* if PC10 not present in CPUID.MWAIT.EDX */
2181 if ((mwait_substates & (0xF << 28)) == 0)
2182 return;
2183
2184 rdmsrq(MSR_PKG_CST_CONFIG_CONTROL, msr);
2185
2186 /* PC10 is not enabled in PKG C-state limit */
2187 if ((msr & 0xF) != 8)
2188 return;
2189
2190 ecx = 0;
2191 cpuid(7, &eax, &ebx, &ecx, &edx);
2192
2193 /* if SGX is present */
2194 if (ebx & (1 << 2)) {
2195
2196 rdmsrq(MSR_IA32_FEAT_CTL, msr);
2197
2198 /* if SGX is enabled */
2199 if (msr & (1 << 18))
2200 return;
2201 }
2202
2203 skl_cstates[5].flags |= CPUIDLE_FLAG_UNUSABLE; /* C8-SKL */
2204 skl_cstates[6].flags |= CPUIDLE_FLAG_UNUSABLE; /* C9-SKL */
2205}
2206
2207/**
2208 * skx_is_pc6_disabled() - Check if PC6 is disabled in BIOS.
2209 *
2210 * Return: %true if PC6 is disabled, %false otherwise.
2211 */
2212static bool __init skx_is_pc6_disabled(void)
2213{
2214 u64 msr;
2215
2216 rdmsrq(MSR_PKG_CST_CONFIG_CONTROL, msr);
2217
2218 /*
2219 * 000b: C0/C1 (no package C-state support)
2220 * 001b: C2
2221 * 010b: C6 (non-retention)
2222 * 011b: C6 (retention)
2223 * 111b: No Package C state limits.
2224 */
2225 return (msr & SKX_PKG_CST_LIMIT_MASK) < SKX_PKG_CST_LIMIT_PC6;
2226}
2227
2228/**
2229 * skx_idle_state_table_update - Adjust the SKX/CLX idle states table.
2230 *
2231 * Adjust Sky Lake or Cascade Lake Xeon idle states if PC6 is disabled in BIOS.
2232 * Use the CC6 + PC0 latency and 3 times of that latency for target_residency.
2233 * This is consistent with how the intel_idle driver uses _CST to set the
2234 * target_residency.
2235 */
2236static void __init skx_idle_state_table_update(void)
2237{
2238 if (skx_is_pc6_disabled()) {
2239 skx_cstates[2].exit_latency = 92;
2240 skx_cstates[2].target_residency = 276;
2241 }
2242}
2243
2244/**
2245 * spr_idle_state_table_update - Adjust Sapphire Rapids Xeon idle states table.
2246 *
2247 * By default, the C6 state assumes the worst-case scenario of package C6.
2248 * However, if PC6 is disabled in BIOS, update the numbers to match core C6.
2249 */
2250static void __init spr_idle_state_table_update(void)
2251{
2252 if (skx_is_pc6_disabled()) {
2253 spr_cstates[2].exit_latency = 190;
2254 spr_cstates[2].target_residency = 600;
2255 }
2256}
2257
2258/**
2259 * drop_pc6_redundant_cstates() - Drop C-states redundant when PC6 is disabled.
2260 * @states: Idle states table to modify.
2261 *
2262 * When PC6 is disabled in BIOS, C-states that exist solely to enable PC6
2263 * entry (such as C6P or C6SP) become identical to shallower C-states like
2264 * C6, and are therefore redundant. Should be called only on systems with
2265 * multiple C6 flavors.
2266 */
2267static void __init drop_pc6_redundant_cstates(struct cpuidle_state *states)
2268{
2269 int count;
2270
2271 if (!skx_is_pc6_disabled())
2272 /* PC6 is not disabled, nothing to do */
2273 return;
2274
2275 for (count = 0; states[count].enter; count++)
2276 continue;
2277
2278 if (count < 2) {
2279 pr_debug("Too few idle states to drop PC6-redundant states\n");
2280 return;
2281 }
2282
2283 /*
2284 * Sanity check: At this point all platforms with multiple C6 flavors
2285 * use the CPUIDLE_FLAG_PARTIAL_HINT_MATCH flag. And the last state in
2286 * the table is the one that becomes redundant when PC6 is disabled.
2287 */
2288 if (!(states[count - 1].flags & CPUIDLE_FLAG_PARTIAL_HINT_MATCH)) {
2289 pr_debug("Can't drop PC6-redundant states: unexpected flags\n");
2290 return;
2291 }
2292
2293 /*
2294 * On all current platforms with multiple C6 flavors, there is only one
2295 * C-state that becomes redundant when PC6 is disabled. This state is
2296 * the last one in the table. Drop it by marking it with
2297 * CPUIDLE_FLAG_UNUSABLE so that cpuidle excludes it when registering
2298 * idle states.
2299 */
2300 pr_info("Dropping idle state %s because PC6 is disabled\n",
2301 states[count - 1].name);
2302 states[count - 1].flags |= CPUIDLE_FLAG_UNUSABLE;
2303}
2304
2305/**
2306 * byt_cht_auto_demotion_disable - Disable Bay/Cherry Trail auto-demotion.
2307 */
2308static void __init byt_cht_auto_demotion_disable(void)
2309{
2310 wrmsrq(MSR_CC6_DEMOTION_POLICY_CONFIG, 0);
2311 wrmsrq(MSR_MC6_DEMOTION_POLICY_CONFIG, 0);
2312}
2313
2314static bool __init intel_idle_verify_cstate(unsigned int mwait_hint)
2315{
2316 unsigned int mwait_cstate = (MWAIT_HINT2CSTATE(mwait_hint) + 1) &
2317 MWAIT_CSTATE_MASK;
2318 unsigned int num_substates = (mwait_substates >> mwait_cstate * 4) &
2319 MWAIT_SUBSTATE_MASK;
2320
2321 /* Ignore the C-state if there are NO sub-states in CPUID for it. */
2322 if (num_substates == 0)
2323 return false;
2324
2325 if (mwait_cstate > 2 && !boot_cpu_has(X86_FEATURE_NONSTOP_TSC))
2326 mark_tsc_unstable("TSC halts in idle states deeper than C2");
2327
2328 return true;
2329}
2330
2331static void state_update_enter_method(struct cpuidle_state *state, int cstate)
2332{
2333 if (state->flags & CPUIDLE_FLAG_INIT_XSTATE) {
2334 /*
2335 * Combining with XSTATE with IBRS or IRQ_ENABLE flags
2336 * is not currently supported but this driver.
2337 */
2338 WARN_ON_ONCE(state->flags & CPUIDLE_FLAG_IBRS);
2339 WARN_ON_ONCE(state->flags & CPUIDLE_FLAG_IRQ_ENABLE);
2340 state->enter = intel_idle_xstate;
2341 return;
2342 }
2343
2344 if (cpu_feature_enabled(X86_FEATURE_KERNEL_IBRS) &&
2345 ((state->flags & CPUIDLE_FLAG_IBRS) || ibrs_off)) {
2346 /*
2347 * IBRS mitigation requires that C-states are entered
2348 * with interrupts disabled.
2349 */
2350 if (ibrs_off && (state->flags & CPUIDLE_FLAG_IRQ_ENABLE))
2351 state->flags &= ~CPUIDLE_FLAG_IRQ_ENABLE;
2352 WARN_ON_ONCE(state->flags & CPUIDLE_FLAG_IRQ_ENABLE);
2353 state->enter = intel_idle_ibrs;
2354 return;
2355 }
2356
2357 if (state->flags & CPUIDLE_FLAG_IRQ_ENABLE) {
2358 state->enter = intel_idle_irq;
2359 return;
2360 }
2361
2362 if (force_irq_on) {
2363 pr_info("forced intel_idle_irq for state %d\n", cstate);
2364 state->enter = intel_idle_irq;
2365 }
2366}
2367
2368static void __init intel_idle_init_cstates_icpu(struct cpuidle_driver *drv)
2369{
2370 int cstate;
2371
2372 switch (boot_cpu_data.x86_vfm) {
2373 case INTEL_IVYBRIDGE_X:
2374 ivt_idle_state_table_update();
2375 break;
2376 case INTEL_ATOM_GOLDMONT:
2377 case INTEL_ATOM_GOLDMONT_PLUS:
2378 bxt_idle_state_table_update();
2379 break;
2380 case INTEL_SKYLAKE:
2381 sklh_idle_state_table_update();
2382 break;
2383 case INTEL_SKYLAKE_X:
2384 skx_idle_state_table_update();
2385 break;
2386 case INTEL_SAPPHIRERAPIDS_X:
2387 case INTEL_EMERALDRAPIDS_X:
2388 spr_idle_state_table_update();
2389 break;
2390 case INTEL_ATOM_SILVERMONT:
2391 case INTEL_ATOM_AIRMONT:
2392 byt_cht_auto_demotion_disable();
2393 break;
2394 case INTEL_GRANITERAPIDS_D:
2395 case INTEL_GRANITERAPIDS_X:
2396 case INTEL_ATOM_CRESTMONT_X:
2397 case INTEL_ATOM_DARKMONT_X:
2398 drop_pc6_redundant_cstates(cpuidle_state_table);
2399 break;
2400 }
2401
2402 for (cstate = 0; cstate < CPUIDLE_STATE_MAX; ++cstate) {
2403 struct cpuidle_state *state;
2404 unsigned int mwait_hint;
2405
2406 if (intel_idle_max_cstate_reached(cstate))
2407 break;
2408
2409 if (!cpuidle_state_table[cstate].enter &&
2410 !cpuidle_state_table[cstate].enter_s2idle)
2411 break;
2412
2413 if (!cpuidle_state_table[cstate].enter_dead)
2414 cpuidle_state_table[cstate].enter_dead = intel_idle_enter_dead;
2415
2416 /* If marked as unusable, skip this state. */
2417 if (cpuidle_state_table[cstate].flags & CPUIDLE_FLAG_UNUSABLE) {
2418 pr_debug("state %s is disabled\n",
2419 cpuidle_state_table[cstate].name);
2420 continue;
2421 }
2422
2423 mwait_hint = flg2MWAIT(cpuidle_state_table[cstate].flags);
2424 if (!intel_idle_verify_cstate(mwait_hint))
2425 continue;
2426
2427 /* Structure copy. */
2428 drv->states[drv->state_count] = cpuidle_state_table[cstate];
2429 state = &drv->states[drv->state_count];
2430
2431 state_update_enter_method(state, cstate);
2432
2433
2434 if ((disabled_states_mask & BIT(drv->state_count)) ||
2435 ((icpu->use_acpi || force_use_acpi) &&
2436 intel_idle_off_by_default(state->flags, mwait_hint) &&
2437 !(state->flags & CPUIDLE_FLAG_ALWAYS_ENABLE)))
2438 state->flags |= CPUIDLE_FLAG_OFF;
2439
2440 if (intel_idle_state_needs_timer_stop(state))
2441 state->flags |= CPUIDLE_FLAG_TIMER_STOP;
2442
2443 drv->state_count++;
2444 }
2445}
2446
2447/**
2448 * intel_idle_cpuidle_driver_init - Create the list of available idle states.
2449 * @drv: cpuidle driver structure to initialize.
2450 */
2451static void __init intel_idle_cpuidle_driver_init(struct cpuidle_driver *drv)
2452{
2453 cpuidle_poll_state_init(drv);
2454
2455 if (disabled_states_mask & BIT(0))
2456 drv->states[0].flags |= CPUIDLE_FLAG_OFF;
2457
2458 drv->state_count = 1;
2459
2460 if (icpu && icpu->state_table)
2461 intel_idle_init_cstates_icpu(drv);
2462 else
2463 intel_idle_init_cstates_acpi(drv);
2464}
2465
2466static void auto_demotion_disable(void)
2467{
2468 unsigned long long msr_bits;
2469
2470 rdmsrq(MSR_PKG_CST_CONFIG_CONTROL, msr_bits);
2471 msr_bits &= ~auto_demotion_disable_flags;
2472 wrmsrq(MSR_PKG_CST_CONFIG_CONTROL, msr_bits);
2473}
2474
2475static void c1e_promotion_enable(void)
2476{
2477 unsigned long long msr_bits;
2478
2479 rdmsrq(MSR_IA32_POWER_CTL, msr_bits);
2480 msr_bits |= 0x2;
2481 wrmsrq(MSR_IA32_POWER_CTL, msr_bits);
2482}
2483
2484static void c1e_promotion_disable(void)
2485{
2486 unsigned long long msr_bits;
2487
2488 rdmsrq(MSR_IA32_POWER_CTL, msr_bits);
2489 msr_bits &= ~0x2;
2490 wrmsrq(MSR_IA32_POWER_CTL, msr_bits);
2491}
2492
2493/**
2494 * intel_idle_cpu_init - Register the target CPU with the cpuidle core.
2495 * @cpu: CPU to initialize.
2496 *
2497 * Register a cpuidle device object for @cpu and update its MSRs in accordance
2498 * with the processor model flags.
2499 */
2500static int intel_idle_cpu_init(unsigned int cpu)
2501{
2502 struct cpuidle_device *dev;
2503
2504 dev = per_cpu_ptr(intel_idle_cpuidle_devices, cpu);
2505 dev->cpu = cpu;
2506
2507 if (cpuidle_register_device(dev)) {
2508 pr_debug("cpuidle_register_device %d failed!\n", cpu);
2509 return -EIO;
2510 }
2511
2512 if (auto_demotion_disable_flags)
2513 auto_demotion_disable();
2514
2515 if (c1e_promotion == C1E_PROMOTION_ENABLE)
2516 c1e_promotion_enable();
2517 else if (c1e_promotion == C1E_PROMOTION_DISABLE)
2518 c1e_promotion_disable();
2519
2520 return 0;
2521}
2522
2523static int intel_idle_cpu_online(unsigned int cpu)
2524{
2525 struct cpuidle_device *dev;
2526
2527 if (!boot_cpu_has(X86_FEATURE_ARAT))
2528 tick_broadcast_enable();
2529
2530 /*
2531 * Some systems can hotplug a cpu at runtime after
2532 * the kernel has booted, we have to initialize the
2533 * driver in this case
2534 */
2535 dev = per_cpu_ptr(intel_idle_cpuidle_devices, cpu);
2536 if (!dev->registered)
2537 return intel_idle_cpu_init(cpu);
2538
2539 return 0;
2540}
2541
2542/**
2543 * intel_idle_cpuidle_devices_uninit - Unregister all cpuidle devices.
2544 */
2545static void __init intel_idle_cpuidle_devices_uninit(void)
2546{
2547 int i;
2548
2549 for_each_online_cpu(i)
2550 cpuidle_unregister_device(per_cpu_ptr(intel_idle_cpuidle_devices, i));
2551}
2552
2553static void intel_c1_demotion_toggle(void *enable)
2554{
2555 unsigned long long msr_val;
2556
2557 rdmsrq(MSR_PKG_CST_CONFIG_CONTROL, msr_val);
2558 /*
2559 * Enable/disable C1 undemotion along with C1 demotion, as this is the
2560 * most sensible configuration in general.
2561 */
2562 if (enable)
2563 msr_val |= NHM_C1_AUTO_DEMOTE | SNB_C1_AUTO_UNDEMOTE;
2564 else
2565 msr_val &= ~(NHM_C1_AUTO_DEMOTE | SNB_C1_AUTO_UNDEMOTE);
2566 wrmsrq(MSR_PKG_CST_CONFIG_CONTROL, msr_val);
2567}
2568
2569static ssize_t intel_c1_demotion_store(struct device *dev,
2570 struct device_attribute *attr,
2571 const char *buf, size_t count)
2572{
2573 bool enable;
2574 int err;
2575
2576 err = kstrtobool(buf, &enable);
2577 if (err)
2578 return err;
2579
2580 mutex_lock(&c1_demotion_mutex);
2581 /* Enable/disable C1 demotion on all CPUs */
2582 on_each_cpu(intel_c1_demotion_toggle, (void *)enable, 1);
2583 mutex_unlock(&c1_demotion_mutex);
2584
2585 return count;
2586}
2587
2588static ssize_t intel_c1_demotion_show(struct device *dev,
2589 struct device_attribute *attr, char *buf)
2590{
2591 unsigned long long msr_val;
2592
2593 /*
2594 * Read the MSR value for a CPU and assume it is the same for all CPUs. Any other
2595 * configuration would be a BIOS bug.
2596 */
2597 rdmsrq(MSR_PKG_CST_CONFIG_CONTROL, msr_val);
2598 return sysfs_emit(buf, "%d\n", !!(msr_val & NHM_C1_AUTO_DEMOTE));
2599}
2600static DEVICE_ATTR_RW(intel_c1_demotion);
2601
2602static int __init intel_idle_sysfs_init(void)
2603{
2604 int err;
2605
2606 if (!c1_demotion_supported)
2607 return 0;
2608
2609 sysfs_root = bus_get_dev_root(&cpu_subsys);
2610 if (!sysfs_root)
2611 return 0;
2612
2613 err = sysfs_add_file_to_group(&sysfs_root->kobj,
2614 &dev_attr_intel_c1_demotion.attr,
2615 "cpuidle");
2616 if (err) {
2617 put_device(sysfs_root);
2618 return err;
2619 }
2620
2621 return 0;
2622}
2623
2624static void __init intel_idle_sysfs_uninit(void)
2625{
2626 if (!sysfs_root)
2627 return;
2628
2629 sysfs_remove_file_from_group(&sysfs_root->kobj,
2630 &dev_attr_intel_c1_demotion.attr,
2631 "cpuidle");
2632 put_device(sysfs_root);
2633}
2634
2635 /**
2636 * get_cmdline_field - Get the current field from a cmdline string.
2637 * @args: The cmdline string to get the current field from.
2638 * @field: Pointer to the current field upon return.
2639 * @sep: The fields separator character.
2640 *
2641 * Examples:
2642 * Input: args="C1:1:1,C1E:2:10", sep=':'
2643 * Output: field="C1", return "1:1,C1E:2:10"
2644 * Input: args="C1:1:1,C1E:2:10", sep=','
2645 * Output: field="C1:1:1", return "C1E:2:10"
2646 * Ipnut: args="::", sep=':'
2647 * Output: field="", return ":"
2648 *
2649 * Return: The continuation of the cmdline string after the field or NULL.
2650 */
2651static char *get_cmdline_field(char *args, char **field, char sep)
2652{
2653 unsigned int i;
2654
2655 for (i = 0; args[i] && !isspace(args[i]); i++) {
2656 if (args[i] == sep)
2657 break;
2658 }
2659
2660 *field = args;
2661
2662 if (args[i] != sep)
2663 return NULL;
2664
2665 args[i] = '\0';
2666 return args + i + 1;
2667}
2668
2669/**
2670 * validate_cmdline_cstate - Validate a C-state from cmdline.
2671 * @state: The C-state to validate.
2672 * @prev_state: The previous C-state in the table or NULL.
2673 *
2674 * Return: 0 if the C-state is valid or -EINVAL otherwise.
2675 */
2676static int validate_cmdline_cstate(struct cpuidle_state *state,
2677 struct cpuidle_state *prev_state)
2678{
2679 if (state->exit_latency == 0)
2680 /* Exit latency 0 can only be used for the POLL state */
2681 return -EINVAL;
2682
2683 if (state->exit_latency > MAX_CMDLINE_LATENCY_US)
2684 return -EINVAL;
2685
2686 if (state->target_residency > MAX_CMDLINE_RESIDENCY_US)
2687 return -EINVAL;
2688
2689 if (state->target_residency < state->exit_latency)
2690 return -EINVAL;
2691
2692 if (!prev_state)
2693 return 0;
2694
2695 if (state->exit_latency <= prev_state->exit_latency)
2696 return -EINVAL;
2697
2698 if (state->target_residency <= prev_state->target_residency)
2699 return -EINVAL;
2700
2701 return 0;
2702}
2703
2704/**
2705 * cmdline_table_adjust - Adjust the C-states table with data from cmdline.
2706 * @drv: cpuidle driver (assumed to point to intel_idle_driver).
2707 *
2708 * Adjust the C-states table with data from the 'intel_idle.table' module
2709 * parameter (if specified).
2710 */
2711static void __init cmdline_table_adjust(struct cpuidle_driver *drv)
2712{
2713 char *args = cmdline_table_str;
2714 struct cpuidle_state *state;
2715 int i;
2716
2717 if (args[0] == '\0')
2718 /* The 'intel_idle.table' module parameter was not specified */
2719 return;
2720
2721 /* Create a copy of the C-states table */
2722 for (i = 0; i < drv->state_count; i++)
2723 cmdline_states[i] = drv->states[i];
2724
2725 /*
2726 * Adjust the C-states table copy with data from the 'intel_idle.table'
2727 * module parameter.
2728 */
2729 while (args) {
2730 char *fields, *name, *val;
2731
2732 /*
2733 * Get the next C-state definition, which is expected to be
2734 * '<name>:<latency_us>:<target_residency_us>'. Treat "empty"
2735 * fields as unchanged. For example,
2736 * '<name>::<target_residency_us>' leaves the latency unchanged.
2737 */
2738 args = get_cmdline_field(args, &fields, ',');
2739
2740 /* name */
2741 fields = get_cmdline_field(fields, &name, ':');
2742 if (!fields)
2743 goto error;
2744
2745 if (!strcmp(name, "POLL")) {
2746 pr_err("Cannot adjust POLL\n");
2747 continue;
2748 }
2749
2750 /* Find the C-state by its name */
2751 state = NULL;
2752 for (i = 0; i < drv->state_count; i++) {
2753 if (!strcmp(name, drv->states[i].name)) {
2754 state = &cmdline_states[i];
2755 break;
2756 }
2757 }
2758
2759 if (!state) {
2760 pr_err("C-state '%s' was not found\n", name);
2761 continue;
2762 }
2763
2764 /* Latency */
2765 fields = get_cmdline_field(fields, &val, ':');
2766 if (!fields)
2767 goto error;
2768
2769 if (*val) {
2770 if (kstrtouint(val, 0, &state->exit_latency))
2771 goto error;
2772 }
2773
2774 /* Target residency */
2775 fields = get_cmdline_field(fields, &val, ':');
2776
2777 if (*val) {
2778 if (kstrtouint(val, 0, &state->target_residency))
2779 goto error;
2780 }
2781
2782 /*
2783 * Allow for 3 more fields, but ignore them. Helps to make
2784 * possible future extensions of the cmdline format backward
2785 * compatible.
2786 */
2787 for (i = 0; fields && i < 3; i++) {
2788 fields = get_cmdline_field(fields, &val, ':');
2789 if (!fields)
2790 break;
2791 }
2792
2793 if (fields) {
2794 pr_err("Too many fields for C-state '%s'\n", state->name);
2795 goto error;
2796 }
2797
2798 pr_info("C-state from cmdline: name=%s, latency=%u, residency=%u\n",
2799 state->name, state->exit_latency, state->target_residency);
2800 }
2801
2802 /* Validate the adjusted C-states, start with index 1 to skip POLL */
2803 for (i = 1; i < drv->state_count; i++) {
2804 struct cpuidle_state *prev_state;
2805
2806 state = &cmdline_states[i];
2807 prev_state = &cmdline_states[i - 1];
2808
2809 if (validate_cmdline_cstate(state, prev_state)) {
2810 pr_err("C-state '%s' validation failed\n", state->name);
2811 goto error;
2812 }
2813 }
2814
2815 /* Copy the adjusted C-states table back */
2816 for (i = 1; i < drv->state_count; i++)
2817 drv->states[i] = cmdline_states[i];
2818
2819 pr_info("Adjusted C-states with data from 'intel_idle.table'\n");
2820 return;
2821
2822error:
2823 pr_info("Failed to adjust C-states with data from 'intel_idle.table'\n");
2824}
2825
2826#define INTEL_IDLE_INIT_QOS 20
2827static struct pm_qos_request qos_req __initdata;
2828
2829static int __init intel_idle_init(void)
2830{
2831 const struct x86_cpu_id *id;
2832 unsigned int eax, ebx, ecx;
2833 int retval;
2834
2835 /* Do not load intel_idle at all for now if idle= is passed */
2836 if (boot_option_idle_override != IDLE_NO_OVERRIDE)
2837 return -ENODEV;
2838
2839 if (max_cstate == 0) {
2840 pr_debug("disabled\n");
2841 return -EPERM;
2842 }
2843
2844 id = x86_match_cpu(intel_idle_ids);
2845 if (id) {
2846 if (!boot_cpu_has(X86_FEATURE_MWAIT)) {
2847 pr_debug("Please enable MWAIT in BIOS SETUP\n");
2848 return -ENODEV;
2849 }
2850 } else {
2851 id = x86_match_cpu(intel_mwait_ids);
2852 if (!id)
2853 return -ENODEV;
2854 }
2855
2856 cpuid(CPUID_LEAF_MWAIT, &eax, &ebx, &ecx, &mwait_substates);
2857
2858 if (!(ecx & CPUID5_ECX_EXTENSIONS_SUPPORTED) ||
2859 !(ecx & CPUID5_ECX_INTERRUPT_BREAK) ||
2860 !mwait_substates)
2861 return -ENODEV;
2862
2863 pr_debug("MWAIT substates: 0x%x\n", mwait_substates);
2864
2865 icpu = (const struct idle_cpu *)id->driver_data;
2866 if (icpu && ignore_native()) {
2867 pr_debug("ignoring native CPU idle states\n");
2868 icpu = NULL;
2869 }
2870 if (icpu) {
2871 if (icpu->state_table)
2872 cpuidle_state_table = icpu->state_table;
2873 else if (!intel_idle_acpi_probe())
2874 return -ENODEV;
2875
2876 auto_demotion_disable_flags = icpu->auto_demotion_disable_flags;
2877 if (icpu->disable_promotion_to_c1e)
2878 c1e_promotion = C1E_PROMOTION_DISABLE;
2879 if (icpu->c1_demotion_supported)
2880 c1_demotion_supported = true;
2881 if (icpu->use_acpi || force_use_acpi)
2882 intel_idle_acpi_probe();
2883 } else if (!intel_idle_acpi_probe()) {
2884 return -ENODEV;
2885 }
2886
2887 intel_idle_cpuidle_devices = alloc_percpu(struct cpuidle_device);
2888 if (!intel_idle_cpuidle_devices)
2889 return -ENOMEM;
2890
2891 intel_idle_cpuidle_driver_init(&intel_idle_driver);
2892 cmdline_table_adjust(&intel_idle_driver);
2893
2894 retval = intel_idle_sysfs_init();
2895 if (retval)
2896 pr_warn("failed to initialized sysfs");
2897
2898 /*
2899 * Some platforms, in particular the Intel S1200BTL motherboard, have a
2900 * problem with using package idle states too early, so prevent that
2901 * from taking place until the device_initcall() phase is over.
2902 */
2903 cpu_latency_qos_add_request(&qos_req, INTEL_IDLE_INIT_QOS);
2904
2905 retval = cpuidle_register_driver(&intel_idle_driver);
2906 if (retval) {
2907 struct cpuidle_driver *drv = cpuidle_get_driver();
2908 printk(KERN_DEBUG pr_fmt("intel_idle yielding to %s\n"),
2909 drv ? drv->name : "none");
2910 goto init_driver_fail;
2911 }
2912
2913 retval = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "idle/intel:online",
2914 intel_idle_cpu_online, NULL);
2915 if (retval < 0)
2916 goto hp_setup_fail;
2917
2918 pr_debug("Local APIC timer is reliable in %s\n",
2919 boot_cpu_has(X86_FEATURE_ARAT) ? "all C-states" : "C1");
2920
2921 arch_cpu_rescan_dead_smt_siblings();
2922
2923 return 0;
2924
2925hp_setup_fail:
2926 intel_idle_cpuidle_devices_uninit();
2927 cpuidle_unregister_driver(&intel_idle_driver);
2928init_driver_fail:
2929 if (cpu_latency_qos_request_active((&qos_req)))
2930 cpu_latency_qos_remove_request(&qos_req);
2931
2932 intel_idle_sysfs_uninit();
2933 free_percpu(intel_idle_cpuidle_devices);
2934 return retval;
2935
2936}
2937subsys_initcall_sync(intel_idle_init);
2938
2939static int __init intel_idle_init_complete(void)
2940{
2941 if (cpu_latency_qos_request_active((&qos_req)))
2942 cpu_latency_qos_remove_request(&qos_req);
2943
2944 return 0;
2945}
2946device_initcall_sync(intel_idle_init_complete);
2947
2948/*
2949 * We are not really modular, but we used to support that. Meaning we also
2950 * support "intel_idle.max_cstate=..." at boot and also a read-only export of
2951 * it at /sys/module/intel_idle/parameters/max_cstate -- so using module_param
2952 * is the easiest way (currently) to continue doing that.
2953 */
2954module_param(max_cstate, int, 0444);
2955/*
2956 * The positions of the bits that are set in this number are the indices of the
2957 * idle states to be disabled by default (as reflected by the names of the
2958 * corresponding idle state directories in sysfs, "state0", "state1" ...
2959 * "state<i>" ..., where <i> is the index of the given state).
2960 */
2961module_param_named(states_off, disabled_states_mask, uint, 0444);
2962MODULE_PARM_DESC(states_off, "Mask of disabled idle states");
2963/*
2964 * Debugging option that forces the driver to enter all C-states with
2965 * interrupts enabled. Does not apply to C-states with
2966 * 'CPUIDLE_FLAG_INIT_XSTATE' and 'CPUIDLE_FLAG_IBRS' flags.
2967 */
2968module_param(force_irq_on, bool, 0444);
2969/*
2970 * Force the disabling of IBRS when X86_FEATURE_KERNEL_IBRS is on and
2971 * CPUIDLE_FLAG_IRQ_ENABLE isn't set.
2972 */
2973module_param(ibrs_off, bool, 0444);
2974MODULE_PARM_DESC(ibrs_off, "Disable IBRS when idle");
2975
2976/*
2977 * Define the C-states table from a user input string. Expected format is
2978 * 'name:latency:residency', where:
2979 * - name: The C-state name.
2980 * - latency: The C-state exit latency in us.
2981 * - residency: The C-state target residency in us.
2982 *
2983 * Multiple C-states can be defined by separating them with commas:
2984 * 'name1:latency1:residency1,name2:latency2:residency2'
2985 *
2986 * Example: intel_idle.table=C1:1:1,C1E:5:10,C6:100:600
2987 *
2988 * To leave latency or residency unchanged, use an empty field, for example:
2989 * 'C1:1:1,C1E::10' - leaves C1E latency unchanged.
2990 */
2991module_param_string(table, cmdline_table_str, MAX_CMDLINE_TABLE_LEN, 0444);
2992MODULE_PARM_DESC(table, "Build the C-states table from a user input string");