projects/linux·blob master · refs

1205 lines · 29.6 KiB

1// SPDX-License-Identifier: GPL-2.0-or-later
2/*
3 * processor_idle - idle state submodule to the ACPI processor driver
4 *
5 * Copyright (C) 2001, 2002 Andy Grover <[email protected]>
6 * Copyright (C) 2001, 2002 Paul Diefenbaugh <[email protected]>
7 * Copyright (C) 2004, 2005 Dominik Brodowski <[email protected]>
8 * Copyright (C) 2004 Anil S Keshavamurthy <[email protected]>
9 * - Added processor hotplug support
10 * Copyright (C) 2005 Venkatesh Pallipadi <[email protected]>
11 * - Added support for C3 on SMP
12 */
13#define pr_fmt(fmt) "ACPI: " fmt
14
15#include <linux/module.h>
16#include <linux/acpi.h>
17#include <linux/dmi.h>
18#include <linux/sched.h> /* need_resched() */
19#include <linux/tick.h>
20#include <linux/cpuidle.h>
21#include <linux/cpu.h>
22#include <linux/minmax.h>
23#include <linux/perf_event.h>
24#include <acpi/processor.h>
25#include <linux/context_tracking.h>
26
27#include "internal.h"
28
29/*
30 * Include the apic definitions for x86 to have the APIC timer related defines
31 * available also for UP (on SMP it gets magically included via linux/smp.h).
32 * asm/acpi.h is not an option, as it would require more include magic. Also
33 * creating an empty asm-ia64/apic.h would just trade pest vs. cholera.
34 */
35#ifdef CONFIG_X86
36#include <asm/apic.h>
37#include <asm/cpu.h>
38#endif
39
40#define ACPI_IDLE_STATE_START (IS_ENABLED(CONFIG_ARCH_HAS_CPU_RELAX) ? 1 : 0)
41
42static unsigned int max_cstate __read_mostly = ACPI_PROCESSOR_MAX_POWER;
43module_param(max_cstate, uint, 0400);
44static bool nocst __read_mostly;
45module_param(nocst, bool, 0400);
46static bool bm_check_disable __read_mostly;
47module_param(bm_check_disable, bool, 0400);
48
49static unsigned int latency_factor __read_mostly = 2;
50module_param(latency_factor, uint, 0644);
51
52static DEFINE_PER_CPU(struct cpuidle_device *, acpi_cpuidle_device);
53
54static struct cpuidle_driver acpi_idle_driver = {
55 .name = "acpi_idle",
56 .owner = THIS_MODULE,
57};
58
59#ifdef CONFIG_ACPI_PROCESSOR_CSTATE
60void acpi_idle_rescan_dead_smt_siblings(void)
61{
62 if (cpuidle_get_driver() == &acpi_idle_driver)
63 arch_cpu_rescan_dead_smt_siblings();
64}
65
66static
67DEFINE_PER_CPU(struct acpi_processor_cx * [CPUIDLE_STATE_MAX], acpi_cstate);
68
69static int disabled_by_idle_boot_param(void)
70{
71 return boot_option_idle_override == IDLE_POLL ||
72 boot_option_idle_override == IDLE_HALT;
73}
74
75/*
76 * IBM ThinkPad R40e crashes mysteriously when going into C2 or C3.
77 * For now disable this. Probably a bug somewhere else.
78 *
79 * To skip this limit, boot/load with a large max_cstate limit.
80 */
81static int set_max_cstate(const struct dmi_system_id *id)
82{
83 if (max_cstate > ACPI_PROCESSOR_MAX_POWER)
84 return 0;
85
86 pr_notice("%s detected - limiting to C%ld max_cstate."
87 " Override with \"processor.max_cstate=%d\"\n", id->ident,
88 (long)id->driver_data, ACPI_PROCESSOR_MAX_POWER + 1);
89
90 max_cstate = (long)id->driver_data;
91
92 return 0;
93}
94
95static const struct dmi_system_id processor_power_dmi_table[] = {
96 { set_max_cstate, "Clevo 5600D", {
97 DMI_MATCH(DMI_BIOS_VENDOR,"Phoenix Technologies LTD"),
98 DMI_MATCH(DMI_BIOS_VERSION,"SHE845M0.86C.0013.D.0302131307")},
99 (void *)2},
100 { set_max_cstate, "Pavilion zv5000", {
101 DMI_MATCH(DMI_SYS_VENDOR, "Hewlett-Packard"),
102 DMI_MATCH(DMI_PRODUCT_NAME,"Pavilion zv5000 (DS502A#ABA)")},
103 (void *)1},
104 { set_max_cstate, "Asus L8400B", {
105 DMI_MATCH(DMI_SYS_VENDOR, "ASUSTeK Computer Inc."),
106 DMI_MATCH(DMI_PRODUCT_NAME,"L8400B series Notebook PC")},
107 (void *)1},
108 {},
109};
110
111
112/*
113 * Callers should disable interrupts before the call and enable
114 * interrupts after return.
115 */
116static void __cpuidle acpi_safe_halt(void)
117{
118 if (!tif_need_resched()) {
119 raw_safe_halt();
120 raw_local_irq_disable();
121 }
122}
123
124#ifdef ARCH_APICTIMER_STOPS_ON_C3
125
126/*
127 * Some BIOS implementations switch to C3 in the published C2 state.
128 * This seems to be a common problem on AMD boxen, but other vendors
129 * are affected too. We pick the most conservative approach: we assume
130 * that the local APIC stops in both C2 and C3.
131 */
132static void lapic_timer_check_state(int state, struct acpi_processor *pr,
133 struct acpi_processor_cx *cx)
134{
135 struct acpi_processor_power *pwr = &pr->power;
136 u8 type = local_apic_timer_c2_ok ? ACPI_STATE_C3 : ACPI_STATE_C2;
137
138 if (cpu_has(&cpu_data(pr->id), X86_FEATURE_ARAT))
139 return;
140
141 if (boot_cpu_has_bug(X86_BUG_AMD_APIC_C1E))
142 type = ACPI_STATE_C1;
143
144 /*
145 * Check, if one of the previous states already marked the lapic
146 * unstable
147 */
148 if (pwr->timer_broadcast_on_state < state)
149 return;
150
151 if (cx->type >= type)
152 pr->power.timer_broadcast_on_state = state;
153}
154
155static void __lapic_timer_propagate_broadcast(void *arg)
156{
157 struct acpi_processor *pr = arg;
158
159 if (pr->power.timer_broadcast_on_state < INT_MAX)
160 tick_broadcast_enable();
161 else
162 tick_broadcast_disable();
163}
164
165static void lapic_timer_propagate_broadcast(struct acpi_processor *pr)
166{
167 smp_call_function_single(pr->id, __lapic_timer_propagate_broadcast,
168 (void *)pr, 1);
169}
170
171/* Power(C) State timer broadcast control */
172static bool lapic_timer_needs_broadcast(struct acpi_processor *pr,
173 struct acpi_processor_cx *cx)
174{
175 return cx - pr->power.states >= pr->power.timer_broadcast_on_state;
176}
177
178#else
179
180static void lapic_timer_check_state(int state, struct acpi_processor *pr,
181 struct acpi_processor_cx *cstate) { }
182static void lapic_timer_propagate_broadcast(struct acpi_processor *pr) { }
183
184static bool lapic_timer_needs_broadcast(struct acpi_processor *pr,
185 struct acpi_processor_cx *cx)
186{
187 return false;
188}
189
190#endif
191
192#if defined(CONFIG_X86)
193static void tsc_check_state(int state)
194{
195 switch (boot_cpu_data.x86_vendor) {
196 case X86_VENDOR_HYGON:
197 case X86_VENDOR_AMD:
198 case X86_VENDOR_INTEL:
199 case X86_VENDOR_CENTAUR:
200 case X86_VENDOR_ZHAOXIN:
201 /*
202 * AMD Fam10h TSC will tick in all
203 * C/P/S0/S1 states when this bit is set.
204 */
205 if (boot_cpu_has(X86_FEATURE_NONSTOP_TSC))
206 return;
207 fallthrough;
208 default:
209 /* TSC could halt in idle, so notify users */
210 if (state > ACPI_STATE_C1)
211 mark_tsc_unstable("TSC halts in idle");
212 }
213}
214#else
215static void tsc_check_state(int state) { return; }
216#endif
217
218static int acpi_processor_get_power_info_fadt(struct acpi_processor *pr)
219{
220
221 if (!pr->pblk)
222 return -ENODEV;
223
224 /* if info is obtained from pblk/fadt, type equals state */
225 pr->power.states[ACPI_STATE_C2].type = ACPI_STATE_C2;
226 pr->power.states[ACPI_STATE_C3].type = ACPI_STATE_C3;
227
228#ifndef CONFIG_HOTPLUG_CPU
229 /*
230 * Check for P_LVL2_UP flag before entering C2 and above on
231 * an SMP system.
232 */
233 if ((num_online_cpus() > 1) &&
234 !(acpi_gbl_FADT.flags & ACPI_FADT_C2_MP_SUPPORTED))
235 return -ENODEV;
236#endif
237
238 /* determine C2 and C3 address from pblk */
239 pr->power.states[ACPI_STATE_C2].address = pr->pblk + 4;
240 pr->power.states[ACPI_STATE_C3].address = pr->pblk + 5;
241
242 /* determine latencies from FADT */
243 pr->power.states[ACPI_STATE_C2].latency = acpi_gbl_FADT.c2_latency;
244 pr->power.states[ACPI_STATE_C3].latency = acpi_gbl_FADT.c3_latency;
245
246 /*
247 * FADT specified C2 latency must be less than or equal to
248 * 100 microseconds.
249 */
250 if (acpi_gbl_FADT.c2_latency > ACPI_PROCESSOR_MAX_C2_LATENCY) {
251 acpi_handle_debug(pr->handle, "C2 latency too large [%d]\n",
252 acpi_gbl_FADT.c2_latency);
253 /* invalidate C2 */
254 pr->power.states[ACPI_STATE_C2].address = 0;
255 }
256
257 /*
258 * FADT supplied C3 latency must be less than or equal to
259 * 1000 microseconds.
260 */
261 if (acpi_gbl_FADT.c3_latency > ACPI_PROCESSOR_MAX_C3_LATENCY) {
262 acpi_handle_debug(pr->handle, "C3 latency too large [%d]\n",
263 acpi_gbl_FADT.c3_latency);
264 /* invalidate C3 */
265 pr->power.states[ACPI_STATE_C3].address = 0;
266 }
267
268 acpi_handle_debug(pr->handle, "lvl2[0x%08x] lvl3[0x%08x]\n",
269 pr->power.states[ACPI_STATE_C2].address,
270 pr->power.states[ACPI_STATE_C3].address);
271
272 snprintf(pr->power.states[ACPI_STATE_C2].desc,
273 ACPI_CX_DESC_LEN, "ACPI P_LVL2 IOPORT 0x%x",
274 pr->power.states[ACPI_STATE_C2].address);
275 snprintf(pr->power.states[ACPI_STATE_C3].desc,
276 ACPI_CX_DESC_LEN, "ACPI P_LVL3 IOPORT 0x%x",
277 pr->power.states[ACPI_STATE_C3].address);
278
279 if (!pr->power.states[ACPI_STATE_C2].address &&
280 !pr->power.states[ACPI_STATE_C3].address)
281 return -ENODEV;
282
283 return 0;
284}
285
286static int acpi_processor_get_power_info_default(struct acpi_processor *pr)
287{
288 if (!pr->power.states[ACPI_STATE_C1].valid) {
289 /* set the first C-State to C1 */
290 /* all processors need to support C1 */
291 pr->power.states[ACPI_STATE_C1].type = ACPI_STATE_C1;
292 pr->power.states[ACPI_STATE_C1].valid = 1;
293 pr->power.states[ACPI_STATE_C1].entry_method = ACPI_CSTATE_HALT;
294
295 snprintf(pr->power.states[ACPI_STATE_C1].desc,
296 ACPI_CX_DESC_LEN, "ACPI HLT");
297 }
298 /* the C0 state only exists as a filler in our array */
299 pr->power.states[ACPI_STATE_C0].valid = 1;
300 return 0;
301}
302
303static int acpi_processor_get_power_info_cst(struct acpi_processor *pr)
304{
305 int ret;
306
307 if (nocst)
308 return -ENODEV;
309
310 ret = acpi_processor_evaluate_cst(pr->handle, pr->id, &pr->power);
311 if (ret)
312 return ret;
313
314 if (!pr->power.count)
315 return -EFAULT;
316
317 pr->flags.has_cst = 1;
318 return 0;
319}
320
321static void acpi_processor_power_verify_c3(struct acpi_processor *pr,
322 struct acpi_processor_cx *cx)
323{
324 static int bm_check_flag = -1;
325 static int bm_control_flag = -1;
326
327
328 if (!cx->address)
329 return;
330
331 /*
332 * PIIX4 Erratum #18: We don't support C3 when Type-F (fast)
333 * DMA transfers are used by any ISA device to avoid livelock.
334 * Note that we could disable Type-F DMA (as recommended by
335 * the erratum), but this is known to disrupt certain ISA
336 * devices thus we take the conservative approach.
337 */
338 if (errata.piix4.fdma) {
339 acpi_handle_debug(pr->handle,
340 "C3 not supported on PIIX4 with Type-F DMA\n");
341 return;
342 }
343
344 /* All the logic here assumes flags.bm_check is same across all CPUs */
345 if (bm_check_flag == -1) {
346 /* Determine whether bm_check is needed based on CPU */
347 acpi_processor_power_init_bm_check(&(pr->flags), pr->id);
348 bm_check_flag = pr->flags.bm_check;
349 bm_control_flag = pr->flags.bm_control;
350 } else {
351 pr->flags.bm_check = bm_check_flag;
352 pr->flags.bm_control = bm_control_flag;
353 }
354
355 if (pr->flags.bm_check) {
356 if (!pr->flags.bm_control) {
357 if (pr->flags.has_cst != 1) {
358 /* bus mastering control is necessary */
359 acpi_handle_debug(pr->handle,
360 "C3 support requires BM control\n");
361 return;
362 } else {
363 /* Here we enter C3 without bus mastering */
364 acpi_handle_debug(pr->handle,
365 "C3 support without BM control\n");
366 }
367 }
368 } else {
369 /*
370 * WBINVD should be set in fadt, for C3 state to be
371 * supported on when bm_check is not required.
372 */
373 if (!(acpi_gbl_FADT.flags & ACPI_FADT_WBINVD)) {
374 acpi_handle_debug(pr->handle,
375 "Cache invalidation should work properly"
376 " for C3 to be enabled on SMP systems\n");
377 return;
378 }
379 }
380
381 /*
382 * Otherwise we've met all of our C3 requirements.
383 * Normalize the C3 latency to expidite policy. Enable
384 * checking of bus mastering status (bm_check) so we can
385 * use this in our C3 policy
386 */
387 cx->valid = 1;
388
389 /*
390 * On older chipsets, BM_RLD needs to be set
391 * in order for Bus Master activity to wake the
392 * system from C3. Newer chipsets handle DMA
393 * during C3 automatically and BM_RLD is a NOP.
394 * In either case, the proper way to
395 * handle BM_RLD is to set it and leave it set.
396 */
397 acpi_write_bit_register(ACPI_BITREG_BUS_MASTER_RLD, 1);
398}
399
400static void acpi_cst_latency_sort(struct acpi_processor_cx *states, size_t length)
401{
402 int i, j, k;
403
404 for (i = 1; i < length; i++) {
405 if (!states[i].valid)
406 continue;
407
408 for (j = i - 1, k = i; j >= 0; j--) {
409 if (!states[j].valid)
410 continue;
411
412 if (states[j].latency > states[k].latency)
413 swap(states[j].latency, states[k].latency);
414
415 k = j;
416 }
417 }
418}
419
420static int acpi_processor_power_verify(struct acpi_processor *pr)
421{
422 unsigned int i;
423 unsigned int working = 0;
424 unsigned int last_latency = 0;
425 unsigned int last_type = 0;
426 bool buggy_latency = false;
427
428 pr->power.timer_broadcast_on_state = INT_MAX;
429
430 for (i = 1; i < ACPI_PROCESSOR_MAX_POWER && i <= max_cstate; i++) {
431 struct acpi_processor_cx *cx = &pr->power.states[i];
432
433 switch (cx->type) {
434 case ACPI_STATE_C1:
435 cx->valid = 1;
436 break;
437
438 case ACPI_STATE_C2:
439 if (!cx->address)
440 break;
441 cx->valid = 1;
442 break;
443
444 case ACPI_STATE_C3:
445 acpi_processor_power_verify_c3(pr, cx);
446 break;
447 }
448 if (!cx->valid)
449 continue;
450 if (cx->type >= last_type && cx->latency < last_latency)
451 buggy_latency = true;
452 last_latency = cx->latency;
453 last_type = cx->type;
454
455 lapic_timer_check_state(i, pr, cx);
456 tsc_check_state(cx->type);
457 working++;
458 }
459
460 if (buggy_latency) {
461 pr_notice("FW issue: working around C-state latencies out of order\n");
462 acpi_cst_latency_sort(&pr->power.states[1], max_cstate);
463 }
464
465 lapic_timer_propagate_broadcast(pr);
466
467 return working;
468}
469
470static int acpi_processor_get_cstate_info(struct acpi_processor *pr)
471{
472 int result;
473
474 /* NOTE: the idle thread may not be running while calling
475 * this function */
476
477 /* Zero initialize all the C-states info. */
478 memset(pr->power.states, 0, sizeof(pr->power.states));
479
480 result = acpi_processor_get_power_info_cst(pr);
481 if (result == -ENODEV)
482 result = acpi_processor_get_power_info_fadt(pr);
483
484 if (result)
485 return result;
486
487 acpi_processor_get_power_info_default(pr);
488
489 pr->power.count = acpi_processor_power_verify(pr);
490 pr->flags.power = 1;
491
492 return 0;
493}
494
495/**
496 * acpi_idle_bm_check - checks if bus master activity was detected
497 */
498static int acpi_idle_bm_check(void)
499{
500 u32 bm_status = 0;
501
502 if (bm_check_disable)
503 return 0;
504
505 acpi_read_bit_register(ACPI_BITREG_BUS_MASTER_STATUS, &bm_status);
506 if (bm_status)
507 acpi_write_bit_register(ACPI_BITREG_BUS_MASTER_STATUS, 1);
508 /*
509 * PIIX4 Erratum #18: Note that BM_STS doesn't always reflect
510 * the true state of bus mastering activity; forcing us to
511 * manually check the BMIDEA bit of each IDE channel.
512 */
513 else if (errata.piix4.bmisx) {
514 if ((inb_p(errata.piix4.bmisx + 0x02) & 0x01)
515 || (inb_p(errata.piix4.bmisx + 0x0A) & 0x01))
516 bm_status = 1;
517 }
518 return bm_status;
519}
520
521static __cpuidle void io_idle(unsigned long addr)
522{
523 /* IO port based C-state */
524 inb(addr);
525
526#ifdef CONFIG_X86
527 /* No delay is needed if we are in guest */
528 if (boot_cpu_has(X86_FEATURE_HYPERVISOR))
529 return;
530 /*
531 * Modern (>=Nehalem) Intel systems use ACPI via intel_idle,
532 * not this code. Assume that any Intel systems using this
533 * are ancient and may need the dummy wait. This also assumes
534 * that the motivating chipset issue was Intel-only.
535 */
536 if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL)
537 return;
538#endif
539 /*
540 * Dummy wait op - must do something useless after P_LVL2 read
541 * because chipsets cannot guarantee that STPCLK# signal gets
542 * asserted in time to freeze execution properly
543 *
544 * This workaround has been in place since the original ACPI
545 * implementation was merged, circa 2002.
546 *
547 * If a profile is pointing to this instruction, please first
548 * consider moving your system to a more modern idle
549 * mechanism.
550 */
551 inl(acpi_gbl_FADT.xpm_timer_block.address);
552}
553
554/**
555 * acpi_idle_do_entry - enter idle state using the appropriate method
556 * @cx: cstate data
557 *
558 * Caller disables interrupt before call and enables interrupt after return.
559 */
560static void __cpuidle acpi_idle_do_entry(struct acpi_processor_cx *cx)
561{
562 perf_lopwr_cb(true);
563
564 if (cx->entry_method == ACPI_CSTATE_FFH) {
565 /* Call into architectural FFH based C-state */
566 acpi_processor_ffh_cstate_enter(cx);
567 } else if (cx->entry_method == ACPI_CSTATE_HALT) {
568 acpi_safe_halt();
569 } else {
570 io_idle(cx->address);
571 }
572
573 perf_lopwr_cb(false);
574}
575
576/**
577 * acpi_idle_play_dead - enters an ACPI state for long-term idle (i.e. off-lining)
578 * @dev: the target CPU
579 * @index: the index of suggested state
580 */
581static void acpi_idle_play_dead(struct cpuidle_device *dev, int index)
582{
583 struct acpi_processor_cx *cx = per_cpu(acpi_cstate[index], dev->cpu);
584
585 ACPI_FLUSH_CPU_CACHE();
586
587 while (1) {
588
589 if (cx->entry_method == ACPI_CSTATE_HALT)
590 raw_safe_halt();
591 else if (cx->entry_method == ACPI_CSTATE_SYSTEMIO) {
592 io_idle(cx->address);
593 } else if (cx->entry_method == ACPI_CSTATE_FFH) {
594 acpi_processor_ffh_play_dead(cx);
595 } else
596 return;
597 }
598}
599
600static __always_inline bool acpi_idle_fallback_to_c1(struct acpi_processor *pr)
601{
602 return IS_ENABLED(CONFIG_HOTPLUG_CPU) && !pr->flags.has_cst &&
603 !(acpi_gbl_FADT.flags & ACPI_FADT_C2_MP_SUPPORTED);
604}
605
606static int c3_cpu_count;
607static DEFINE_RAW_SPINLOCK(c3_lock);
608
609/**
610 * acpi_idle_enter_bm - enters C3 with proper BM handling
611 * @drv: cpuidle driver
612 * @pr: Target processor
613 * @cx: Target state context
614 * @index: index of target state
615 */
616static int __cpuidle acpi_idle_enter_bm(struct cpuidle_driver *drv,
617 struct acpi_processor *pr,
618 struct acpi_processor_cx *cx,
619 int index)
620{
621 static struct acpi_processor_cx safe_cx = {
622 .entry_method = ACPI_CSTATE_HALT,
623 };
624
625 /*
626 * disable bus master
627 * bm_check implies we need ARB_DIS
628 * bm_control implies whether we can do ARB_DIS
629 *
630 * That leaves a case where bm_check is set and bm_control is not set.
631 * In that case we cannot do much, we enter C3 without doing anything.
632 */
633 bool dis_bm = pr->flags.bm_control;
634
635 instrumentation_begin();
636
637 /* If we can skip BM, demote to a safe state. */
638 if (!cx->bm_sts_skip && acpi_idle_bm_check()) {
639 dis_bm = false;
640 index = drv->safe_state_index;
641 if (index >= 0) {
642 cx = this_cpu_read(acpi_cstate[index]);
643 } else {
644 cx = &safe_cx;
645 index = -EBUSY;
646 }
647 }
648
649 if (dis_bm) {
650 raw_spin_lock(&c3_lock);
651 c3_cpu_count++;
652 /* Disable bus master arbitration when all CPUs are in C3 */
653 if (c3_cpu_count == num_online_cpus())
654 acpi_write_bit_register(ACPI_BITREG_ARB_DISABLE, 1);
655 raw_spin_unlock(&c3_lock);
656 }
657
658 ct_cpuidle_enter();
659
660 acpi_idle_do_entry(cx);
661
662 ct_cpuidle_exit();
663
664 /* Re-enable bus master arbitration */
665 if (dis_bm) {
666 raw_spin_lock(&c3_lock);
667 acpi_write_bit_register(ACPI_BITREG_ARB_DISABLE, 0);
668 c3_cpu_count--;
669 raw_spin_unlock(&c3_lock);
670 }
671
672 instrumentation_end();
673
674 return index;
675}
676
677static int __cpuidle acpi_idle_enter(struct cpuidle_device *dev,
678 struct cpuidle_driver *drv, int index)
679{
680 struct acpi_processor_cx *cx = per_cpu(acpi_cstate[index], dev->cpu);
681 struct acpi_processor *pr;
682
683 pr = __this_cpu_read(processors);
684 if (unlikely(!pr))
685 return -EINVAL;
686
687 if (cx->type != ACPI_STATE_C1) {
688 if (cx->type == ACPI_STATE_C3 && pr->flags.bm_check)
689 return acpi_idle_enter_bm(drv, pr, cx, index);
690
691 /* C2 to C1 demotion. */
692 if (acpi_idle_fallback_to_c1(pr) && num_online_cpus() > 1) {
693 index = ACPI_IDLE_STATE_START;
694 cx = per_cpu(acpi_cstate[index], dev->cpu);
695 }
696 }
697
698 if (cx->type == ACPI_STATE_C3)
699 ACPI_FLUSH_CPU_CACHE();
700
701 acpi_idle_do_entry(cx);
702
703 return index;
704}
705
706static int __cpuidle acpi_idle_enter_s2idle(struct cpuidle_device *dev,
707 struct cpuidle_driver *drv, int index)
708{
709 struct acpi_processor_cx *cx = per_cpu(acpi_cstate[index], dev->cpu);
710
711 if (cx->type == ACPI_STATE_C3) {
712 struct acpi_processor *pr = __this_cpu_read(processors);
713
714 if (unlikely(!pr))
715 return 0;
716
717 if (pr->flags.bm_check) {
718 u8 bm_sts_skip = cx->bm_sts_skip;
719
720 /* Don't check BM_STS, do an unconditional ARB_DIS for S2IDLE */
721 cx->bm_sts_skip = 1;
722 acpi_idle_enter_bm(drv, pr, cx, index);
723 cx->bm_sts_skip = bm_sts_skip;
724
725 return 0;
726 } else {
727 ACPI_FLUSH_CPU_CACHE();
728 }
729 }
730 acpi_idle_do_entry(cx);
731
732 return 0;
733}
734
735static void acpi_processor_setup_cpuidle_cx(struct acpi_processor *pr,
736 struct cpuidle_device *dev)
737{
738 int i, count = ACPI_IDLE_STATE_START;
739 struct acpi_processor_cx *cx;
740
741 if (max_cstate == 0)
742 max_cstate = 1;
743
744 for (i = 1; i < ACPI_PROCESSOR_MAX_POWER && i <= max_cstate; i++) {
745 cx = &pr->power.states[i];
746
747 if (!cx->valid)
748 continue;
749
750 per_cpu(acpi_cstate[count], dev->cpu) = cx;
751
752 count++;
753 if (count == CPUIDLE_STATE_MAX)
754 break;
755 }
756}
757
758static void acpi_processor_setup_cstates(struct acpi_processor *pr)
759{
760 int i, count;
761 struct acpi_processor_cx *cx;
762 struct cpuidle_state *state;
763 struct cpuidle_driver *drv = &acpi_idle_driver;
764
765 if (max_cstate == 0)
766 max_cstate = 1;
767
768 if (IS_ENABLED(CONFIG_ARCH_HAS_CPU_RELAX)) {
769 cpuidle_poll_state_init(drv);
770 count = 1;
771 } else {
772 count = 0;
773 }
774
775 for (i = 1; i < ACPI_PROCESSOR_MAX_POWER && i <= max_cstate; i++) {
776 cx = &pr->power.states[i];
777
778 if (!cx->valid)
779 continue;
780
781 state = &drv->states[count];
782 snprintf(state->name, CPUIDLE_NAME_LEN, "C%d", i);
783 strscpy(state->desc, cx->desc, CPUIDLE_DESC_LEN);
784 state->exit_latency = cx->latency;
785 state->target_residency = cx->latency * latency_factor;
786 state->enter = acpi_idle_enter;
787
788 state->flags = 0;
789
790 state->enter_dead = acpi_idle_play_dead;
791
792 if (cx->type == ACPI_STATE_C1 || cx->type == ACPI_STATE_C2)
793 drv->safe_state_index = count;
794
795 /*
796 * Halt-induced C1 is not good for ->enter_s2idle, because it
797 * re-enables interrupts on exit. Moreover, C1 is generally not
798 * particularly interesting from the suspend-to-idle angle, so
799 * avoid C1 and the situations in which we may need to fall back
800 * to it altogether.
801 */
802 if (cx->type != ACPI_STATE_C1 && !acpi_idle_fallback_to_c1(pr))
803 state->enter_s2idle = acpi_idle_enter_s2idle;
804
805 if (lapic_timer_needs_broadcast(pr, cx))
806 state->flags |= CPUIDLE_FLAG_TIMER_STOP;
807
808 if (cx->type == ACPI_STATE_C3) {
809 state->flags |= CPUIDLE_FLAG_TLB_FLUSHED;
810 if (pr->flags.bm_check)
811 state->flags |= CPUIDLE_FLAG_RCU_IDLE;
812 }
813
814 count++;
815 if (count == CPUIDLE_STATE_MAX)
816 break;
817 }
818
819 drv->state_count = count;
820}
821
822static inline void acpi_processor_update_max_cstate(void)
823{
824 dmi_check_system(processor_power_dmi_table);
825 max_cstate = acpi_processor_cstate_check(max_cstate);
826 if (max_cstate < ACPI_C_STATES_MAX)
827 pr_notice("processor limited to max C-state %d\n", max_cstate);
828
829 if (nocst)
830 return;
831
832 acpi_processor_claim_cst_control();
833}
834#else
835
836static inline int disabled_by_idle_boot_param(void) { return 0; }
837static inline void acpi_processor_update_max_cstate(void) { }
838static int acpi_processor_get_cstate_info(struct acpi_processor *pr)
839{
840 return -ENODEV;
841}
842
843static int acpi_processor_setup_cpuidle_cx(struct acpi_processor *pr,
844 struct cpuidle_device *dev)
845{
846 return -EINVAL;
847}
848
849static int acpi_processor_setup_cstates(struct acpi_processor *pr)
850{
851 return -EINVAL;
852}
853
854#endif /* CONFIG_ACPI_PROCESSOR_CSTATE */
855
856int __weak acpi_processor_ffh_lpi_probe(unsigned int cpu)
857{
858 return -EOPNOTSUPP;
859}
860
861static int acpi_processor_get_lpi_info(struct acpi_processor *pr)
862{
863 int ret;
864
865 /* make sure our architecture has support */
866 ret = acpi_processor_ffh_lpi_probe(pr->id);
867 if (ret == -EOPNOTSUPP)
868 return ret;
869
870 ret = acpi_processor_extract_lpi_info(pr->handle, &pr->power, false);
871 if (ret)
872 return ret;
873
874 /* Tell driver that _LPI is supported. */
875 pr->flags.has_lpi = 1;
876 pr->flags.power = 1;
877
878 return 0;
879}
880
881int __weak __cpuidle acpi_processor_ffh_lpi_enter(struct acpi_lpi_state *lpi)
882{
883 return -ENODEV;
884}
885
886/**
887 * acpi_idle_lpi_enter - enters an ACPI any LPI state
888 * @dev: the target CPU
889 * @drv: cpuidle driver containing cpuidle state info
890 * @index: index of target state
891 *
892 * Return: 0 for success or negative value for error
893 */
894static int __cpuidle acpi_idle_lpi_enter(struct cpuidle_device *dev,
895 struct cpuidle_driver *drv, int index)
896{
897 struct acpi_processor *pr;
898 struct acpi_lpi_state *lpi;
899
900 pr = __this_cpu_read(processors);
901
902 if (unlikely(!pr))
903 return -EINVAL;
904
905 lpi = &pr->power.lpi_states[index];
906 if (lpi->entry_method == ACPI_CSTATE_FFH)
907 return acpi_processor_ffh_lpi_enter(lpi);
908
909 return -EINVAL;
910}
911
912static void acpi_processor_setup_lpi_states(struct acpi_processor *pr)
913{
914 int i;
915 struct acpi_lpi_state *lpi;
916 struct cpuidle_state *state;
917 struct cpuidle_driver *drv = &acpi_idle_driver;
918
919 if (!pr->flags.has_lpi)
920 return;
921
922 for (i = 0; i < pr->power.count && i < CPUIDLE_STATE_MAX; i++) {
923 lpi = &pr->power.lpi_states[i];
924
925 state = &drv->states[i];
926 snprintf(state->name, CPUIDLE_NAME_LEN, "LPI-%d", i);
927 strscpy(state->desc, lpi->desc, CPUIDLE_DESC_LEN);
928 state->exit_latency = lpi->wake_latency;
929 state->target_residency = lpi->min_residency;
930 state->flags |= arch_get_idle_state_flags(lpi->arch_flags);
931 if (i != 0 && lpi->entry_method == ACPI_CSTATE_FFH)
932 state->flags |= CPUIDLE_FLAG_RCU_IDLE;
933 state->enter = acpi_idle_lpi_enter;
934 drv->safe_state_index = i;
935 }
936
937 drv->state_count = i;
938}
939
940/**
941 * acpi_processor_setup_cpuidle_states- prepares and configures cpuidle
942 * global state data i.e. idle routines
943 *
944 * @pr: the ACPI processor
945 */
946static void acpi_processor_setup_cpuidle_states(struct acpi_processor *pr)
947{
948 int i;
949 struct cpuidle_driver *drv = &acpi_idle_driver;
950
951 if (!pr->flags.power_setup_done || !pr->flags.power)
952 return;
953
954 drv->safe_state_index = -1;
955 for (i = ACPI_IDLE_STATE_START; i < CPUIDLE_STATE_MAX; i++) {
956 drv->states[i].name[0] = '\0';
957 drv->states[i].desc[0] = '\0';
958 }
959
960 if (pr->flags.has_lpi) {
961 acpi_processor_setup_lpi_states(pr);
962 return;
963 }
964
965 acpi_processor_setup_cstates(pr);
966}
967
968/**
969 * acpi_processor_setup_cpuidle_dev - configures CPUIDLE
970 * device i.e. per-cpu data
971 *
972 * @pr: the ACPI processor
973 * @dev : the cpuidle device
974 */
975static void acpi_processor_setup_cpuidle_dev(struct acpi_processor *pr,
976 struct cpuidle_device *dev)
977{
978 if (!pr->flags.power_setup_done || !pr->flags.power || !dev)
979 return;
980
981 dev->cpu = pr->id;
982 if (!pr->flags.has_lpi)
983 acpi_processor_setup_cpuidle_cx(pr, dev);
984}
985
986static int acpi_processor_get_power_info(struct acpi_processor *pr)
987{
988 int ret;
989
990 ret = acpi_processor_get_lpi_info(pr);
991 if (ret)
992 return acpi_processor_get_cstate_info(pr);
993
994 if (pr->flags.has_lpi) {
995 ret = acpi_processor_ffh_lpi_probe(pr->id);
996 if (ret)
997 pr_err("CPU%u: Invalid FFH LPI data\n", pr->id);
998 }
999
1000 return ret;
1001}
1002
1003int acpi_processor_hotplug(struct acpi_processor *pr)
1004{
1005 struct cpuidle_device *dev = per_cpu(acpi_cpuidle_device, pr->id);
1006 int ret = 0;
1007
1008 if (disabled_by_idle_boot_param())
1009 return 0;
1010
1011 if (!pr->flags.power_setup_done || !dev)
1012 return -ENODEV;
1013
1014 cpuidle_pause_and_lock();
1015 cpuidle_disable_device(dev);
1016 ret = acpi_processor_get_power_info(pr);
1017 if (!ret && pr->flags.power) {
1018 acpi_processor_setup_cpuidle_dev(pr, dev);
1019 ret = cpuidle_enable_device(dev);
1020 }
1021 cpuidle_resume_and_unlock();
1022
1023 return ret;
1024}
1025
1026int acpi_processor_power_state_has_changed(struct acpi_processor *pr)
1027{
1028 int cpu;
1029 struct acpi_processor *_pr;
1030 struct cpuidle_device *dev;
1031
1032 if (disabled_by_idle_boot_param())
1033 return 0;
1034
1035 if (!pr->flags.power_setup_done)
1036 return -ENODEV;
1037
1038 /*
1039 * FIXME: Design the ACPI notification to make it once per
1040 * system instead of once per-cpu. This condition is a hack
1041 * to make the code that updates C-States be called once.
1042 */
1043
1044 if (pr->id == 0 && cpuidle_get_driver() == &acpi_idle_driver) {
1045 /* Protect against cpu-hotplug */
1046 cpus_read_lock();
1047
1048 /* Unregister cpuidle device of all CPUs */
1049 cpuidle_pause_and_lock();
1050 for_each_possible_cpu(cpu) {
1051 dev = per_cpu(acpi_cpuidle_device, cpu);
1052 _pr = per_cpu(processors, cpu);
1053 if (!_pr || !_pr->flags.power || !dev)
1054 continue;
1055
1056 cpuidle_unregister_device_no_lock(dev);
1057 kfree(dev);
1058 _pr->flags.power = 0;
1059 }
1060 cpuidle_resume_and_unlock();
1061
1062 /*
1063 * Unregister ACPI idle driver, reinitialize ACPI idle states
1064 * and register ACPI idle driver again.
1065 */
1066 acpi_processor_unregister_idle_driver();
1067 acpi_processor_register_idle_driver();
1068
1069 /*
1070 * Reinitialize power information of all CPUs and re-register
1071 * all cpuidle devices. Now idle states is ok to use, can enable
1072 * cpuidle of each CPU safely one by one.
1073 */
1074 for_each_possible_cpu(cpu) {
1075 _pr = per_cpu(processors, cpu);
1076 if (!_pr)
1077 continue;
1078 acpi_processor_power_init(_pr);
1079 }
1080
1081 cpus_read_unlock();
1082 }
1083
1084 return 0;
1085}
1086
1087void acpi_processor_register_idle_driver(void)
1088{
1089 struct acpi_processor *pr;
1090 int ret = -ENODEV;
1091 int cpu;
1092
1093 /*
1094 * If a cpuidle driver is already registered, there is no need to
1095 * evaluate _CST or attempt to register the ACPI idle driver.
1096 */
1097 if (cpuidle_get_driver()) {
1098 pr_debug("cpuidle driver %pS already registered.\n", cpuidle_get_driver());
1099 return;
1100 }
1101
1102 acpi_processor_update_max_cstate();
1103
1104 /*
1105 * ACPI idle driver is used by all possible CPUs.
1106 * Use the processor power info of one in them to set up idle states.
1107 * Note that the existing idle handler will be used on platforms that
1108 * only support C1.
1109 */
1110 for_each_possible_cpu(cpu) {
1111 pr = per_cpu(processors, cpu);
1112 if (!pr)
1113 continue;
1114
1115 ret = acpi_processor_get_power_info(pr);
1116 if (!ret) {
1117 pr->flags.power_setup_done = 1;
1118 acpi_processor_setup_cpuidle_states(pr);
1119 break;
1120 }
1121 }
1122
1123 if (ret) {
1124 pr_debug("No ACPI power information from any CPUs.\n");
1125 return;
1126 }
1127
1128 ret = cpuidle_register_driver(&acpi_idle_driver);
1129 if (ret) {
1130 pr->flags.power_setup_done = 0;
1131 pr_debug("register %s failed.\n", acpi_idle_driver.name);
1132 return;
1133 }
1134 pr_debug("%s registered with cpuidle.\n", acpi_idle_driver.name);
1135}
1136
1137void acpi_processor_unregister_idle_driver(void)
1138{
1139 struct acpi_processor *pr;
1140 int cpu;
1141
1142 cpuidle_unregister_driver(&acpi_idle_driver);
1143 for_each_possible_cpu(cpu) {
1144 pr = per_cpu(processors, cpu);
1145 if (!pr)
1146 continue;
1147 pr->flags.power_setup_done = 0;
1148 }
1149}
1150
1151void acpi_processor_power_init(struct acpi_processor *pr)
1152{
1153 struct cpuidle_device *dev;
1154
1155 /*
1156 * The code below only works if the current cpuidle driver is the ACPI
1157 * idle driver.
1158 */
1159 if (cpuidle_get_driver() != &acpi_idle_driver)
1160 return;
1161
1162 if (disabled_by_idle_boot_param())
1163 return;
1164
1165 if (!acpi_processor_get_power_info(pr))
1166 pr->flags.power_setup_done = 1;
1167
1168 if (!pr->flags.power)
1169 return;
1170
1171 dev = kzalloc_obj(*dev);
1172 if (!dev)
1173 return;
1174
1175 per_cpu(acpi_cpuidle_device, pr->id) = dev;
1176
1177 acpi_processor_setup_cpuidle_dev(pr, dev);
1178
1179 /*
1180 * Register a cpuidle device for this CPU. The cpuidle driver using
1181 * this device is expected to be registered.
1182 */
1183 if (cpuidle_register_device(dev)) {
1184 per_cpu(acpi_cpuidle_device, pr->id) = NULL;
1185 pr->flags.power_setup_done = 0;
1186 kfree(dev);
1187 }
1188}
1189
1190void acpi_processor_power_exit(struct acpi_processor *pr)
1191{
1192 struct cpuidle_device *dev = per_cpu(acpi_cpuidle_device, pr->id);
1193
1194 if (disabled_by_idle_boot_param())
1195 return;
1196
1197 if (pr->flags.power) {
1198 cpuidle_unregister_device(dev);
1199 kfree(dev);
1200 }
1201
1202 pr->flags.power_setup_done = 0;
1203}
1204
1205MODULE_IMPORT_NS("ACPI_PROCESSOR_IDLE");