projects/linux·blob master · refs

1090 lines · 27.8 KiB

1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * CPPC (Collaborative Processor Performance Control) driver for
4 * interfacing with the CPUfreq layer and governors. See
5 * cppc_acpi.c for CPPC specific methods.
6 *
7 * (C) Copyright 2014, 2015 Linaro Ltd.
8 * Author: Ashwin Chaugule <[email protected]>
9 */
10
11#define pr_fmt(fmt) "CPPC Cpufreq:" fmt
12
13#include <linux/arch_topology.h>
14#include <linux/kernel.h>
15#include <linux/module.h>
16#include <linux/delay.h>
17#include <linux/cpu.h>
18#include <linux/cpufreq.h>
19#include <linux/irq_work.h>
20#include <linux/kthread.h>
21#include <linux/time.h>
22#include <linux/vmalloc.h>
23#include <uapi/linux/sched/types.h>
24
25#include <linux/unaligned.h>
26
27#include <acpi/cppc_acpi.h>
28
29static struct cpufreq_driver cppc_cpufreq_driver;
30
31#ifdef CONFIG_ACPI_CPPC_CPUFREQ_FIE
32static enum {
33 FIE_UNSET = -1,
34 FIE_ENABLED,
35 FIE_DISABLED
36} fie_disabled = FIE_UNSET;
37
38module_param(fie_disabled, int, 0444);
39MODULE_PARM_DESC(fie_disabled, "Disable Frequency Invariance Engine (FIE)");
40
41/* Frequency invariance support */
42struct cppc_freq_invariance {
43 int cpu;
44 struct irq_work irq_work;
45 struct kthread_work work;
46 struct cppc_perf_fb_ctrs prev_perf_fb_ctrs;
47 struct cppc_cpudata *cpu_data;
48};
49
50static DEFINE_PER_CPU(struct cppc_freq_invariance, cppc_freq_inv);
51static struct kthread_worker *kworker_fie;
52
53static int cppc_perf_from_fbctrs(u64 reference_perf,
54 struct cppc_perf_fb_ctrs *fb_ctrs_t0,
55 struct cppc_perf_fb_ctrs *fb_ctrs_t1);
56
57/**
58 * __cppc_scale_freq_tick - CPPC arch_freq_scale updater for frequency invariance
59 * @cppc_fi: per-cpu CPPC FIE data.
60 *
61 * The CPPC driver registers itself with the topology core to provide its own
62 * implementation (cppc_scale_freq_tick()) of topology_scale_freq_tick() which
63 * gets called by the scheduler on every tick.
64 *
65 * Note that the arch specific counters have higher priority than CPPC counters,
66 * if available, though the CPPC driver doesn't need to have any special
67 * handling for that.
68 */
69static void __cppc_scale_freq_tick(struct cppc_freq_invariance *cppc_fi)
70{
71 struct cppc_perf_fb_ctrs fb_ctrs = {0};
72 struct cppc_cpudata *cpu_data;
73 unsigned long local_freq_scale;
74 u64 perf, ref_perf;
75
76 cpu_data = cppc_fi->cpu_data;
77
78 if (cppc_get_perf_ctrs(cppc_fi->cpu, &fb_ctrs)) {
79 pr_warn("%s: failed to read perf counters\n", __func__);
80 return;
81 }
82
83 ref_perf = cpu_data->perf_caps.reference_perf;
84 perf = cppc_perf_from_fbctrs(ref_perf,
85 &cppc_fi->prev_perf_fb_ctrs, &fb_ctrs);
86 if (!perf)
87 return;
88
89 cppc_fi->prev_perf_fb_ctrs = fb_ctrs;
90
91 perf <<= SCHED_CAPACITY_SHIFT;
92 local_freq_scale = div64_u64(perf, cpu_data->perf_caps.highest_perf);
93
94 /* This can happen due to counter's overflow */
95 if (unlikely(local_freq_scale > 1024))
96 local_freq_scale = 1024;
97
98 per_cpu(arch_freq_scale, cppc_fi->cpu) = local_freq_scale;
99}
100
101static void cppc_scale_freq_tick(void)
102{
103 __cppc_scale_freq_tick(&per_cpu(cppc_freq_inv, smp_processor_id()));
104}
105
106static struct scale_freq_data cppc_sftd = {
107 .source = SCALE_FREQ_SOURCE_CPPC,
108 .set_freq_scale = cppc_scale_freq_tick,
109};
110
111static void cppc_scale_freq_workfn(struct kthread_work *work)
112{
113 struct cppc_freq_invariance *cppc_fi;
114
115 cppc_fi = container_of(work, struct cppc_freq_invariance, work);
116 __cppc_scale_freq_tick(cppc_fi);
117}
118
119static void cppc_irq_work(struct irq_work *irq_work)
120{
121 struct cppc_freq_invariance *cppc_fi;
122
123 cppc_fi = container_of(irq_work, struct cppc_freq_invariance, irq_work);
124 kthread_queue_work(kworker_fie, &cppc_fi->work);
125}
126
127/*
128 * Reading perf counters may sleep if the CPC regs are in PCC. Thus, we
129 * schedule an irq work in scale_freq_tick (since we reach here from hard-irq
130 * context), which then schedules a normal work item cppc_scale_freq_workfn()
131 * that updates the per_cpu arch_freq_scale variable based on the counter
132 * updates since the last tick.
133 */
134static void cppc_scale_freq_tick_pcc(void)
135{
136 struct cppc_freq_invariance *cppc_fi = &per_cpu(cppc_freq_inv, smp_processor_id());
137
138 /*
139 * cppc_get_perf_ctrs() can potentially sleep, call that from the right
140 * context.
141 */
142 irq_work_queue(&cppc_fi->irq_work);
143}
144
145static struct scale_freq_data cppc_sftd_pcc = {
146 .source = SCALE_FREQ_SOURCE_CPPC,
147 .set_freq_scale = cppc_scale_freq_tick_pcc,
148};
149
150static void cppc_cpufreq_cpu_fie_init(struct cpufreq_policy *policy)
151{
152 struct scale_freq_data *sftd = &cppc_sftd;
153 struct cppc_freq_invariance *cppc_fi;
154 int cpu, ret;
155
156 if (fie_disabled)
157 return;
158
159 for_each_cpu(cpu, policy->cpus) {
160 cppc_fi = &per_cpu(cppc_freq_inv, cpu);
161 cppc_fi->cpu = cpu;
162 cppc_fi->cpu_data = policy->driver_data;
163 if (cppc_perf_ctrs_in_pcc_cpu(cpu)) {
164 kthread_init_work(&cppc_fi->work, cppc_scale_freq_workfn);
165 init_irq_work(&cppc_fi->irq_work, cppc_irq_work);
166 sftd = &cppc_sftd_pcc;
167 }
168
169 ret = cppc_get_perf_ctrs(cpu, &cppc_fi->prev_perf_fb_ctrs);
170
171 /*
172 * Don't abort as the CPU was offline while the driver was
173 * getting registered.
174 */
175 if (ret && cpu_online(cpu)) {
176 pr_debug("%s: failed to read perf counters for cpu:%d: %d\n",
177 __func__, cpu, ret);
178 return;
179 }
180 }
181
182 /* Register for freq-invariance */
183 topology_set_scale_freq_source(sftd, policy->cpus);
184}
185
186/*
187 * We free all the resources on policy's removal and not on CPU removal as the
188 * irq-work are per-cpu and the hotplug core takes care of flushing the pending
189 * irq-works (hint: smpcfd_dying_cpu()) on CPU hotplug. Even if the kthread-work
190 * fires on another CPU after the concerned CPU is removed, it won't harm.
191 *
192 * We just need to make sure to remove them all on policy->exit().
193 */
194static void cppc_cpufreq_cpu_fie_exit(struct cpufreq_policy *policy)
195{
196 struct cppc_freq_invariance *cppc_fi;
197 int cpu;
198
199 if (fie_disabled)
200 return;
201
202 /* policy->cpus will be empty here, use related_cpus instead */
203 topology_clear_scale_freq_source(SCALE_FREQ_SOURCE_CPPC, policy->related_cpus);
204
205 for_each_cpu(cpu, policy->related_cpus) {
206 if (!cppc_perf_ctrs_in_pcc_cpu(cpu))
207 continue;
208 cppc_fi = &per_cpu(cppc_freq_inv, cpu);
209 irq_work_sync(&cppc_fi->irq_work);
210 kthread_cancel_work_sync(&cppc_fi->work);
211 }
212}
213
214static void cppc_fie_kworker_init(void)
215{
216 struct sched_attr attr = {
217 .size = sizeof(struct sched_attr),
218 .sched_policy = SCHED_DEADLINE,
219 .sched_nice = 0,
220 .sched_priority = 0,
221 /*
222 * Fake (unused) bandwidth; workaround to "fix"
223 * priority inheritance.
224 */
225 .sched_runtime = NSEC_PER_MSEC,
226 .sched_deadline = 10 * NSEC_PER_MSEC,
227 .sched_period = 10 * NSEC_PER_MSEC,
228 };
229 int ret;
230
231 kworker_fie = kthread_run_worker(0, "cppc_fie");
232 if (IS_ERR(kworker_fie)) {
233 pr_warn("%s: failed to create kworker_fie: %ld\n", __func__,
234 PTR_ERR(kworker_fie));
235 fie_disabled = FIE_DISABLED;
236 kworker_fie = NULL;
237 return;
238 }
239
240 ret = sched_setattr_nocheck(kworker_fie->task, &attr);
241 if (ret) {
242 pr_warn("%s: failed to set SCHED_DEADLINE: %d\n", __func__,
243 ret);
244 kthread_destroy_worker(kworker_fie);
245 fie_disabled = FIE_DISABLED;
246 kworker_fie = NULL;
247 }
248}
249
250static void __init cppc_freq_invariance_init(void)
251{
252 bool perf_ctrs_in_pcc = cppc_perf_ctrs_in_pcc();
253
254 if (fie_disabled == FIE_UNSET) {
255 if (perf_ctrs_in_pcc) {
256 pr_info("FIE not enabled on systems with registers in PCC\n");
257 fie_disabled = FIE_DISABLED;
258 } else {
259 fie_disabled = FIE_ENABLED;
260 }
261 }
262
263 if (fie_disabled || !perf_ctrs_in_pcc)
264 return;
265
266 cppc_fie_kworker_init();
267}
268
269static void cppc_freq_invariance_exit(void)
270{
271 if (kworker_fie)
272 kthread_destroy_worker(kworker_fie);
273}
274
275#else
276static inline void cppc_cpufreq_cpu_fie_init(struct cpufreq_policy *policy)
277{
278}
279
280static inline void cppc_cpufreq_cpu_fie_exit(struct cpufreq_policy *policy)
281{
282}
283
284static inline void cppc_freq_invariance_init(void)
285{
286}
287
288static inline void cppc_freq_invariance_exit(void)
289{
290}
291#endif /* CONFIG_ACPI_CPPC_CPUFREQ_FIE */
292
293static void cppc_cpufreq_get_perf_limits(struct cppc_cpudata *cpu_data,
294 struct cpufreq_policy *policy,
295 u32 *min_perf, u32 *max_perf)
296{
297 struct cppc_perf_caps *caps = &cpu_data->perf_caps;
298 unsigned int min_freq, max_freq;
299 u32 min, max;
300
301 min_freq = READ_ONCE(policy->min);
302 max_freq = READ_ONCE(policy->max);
303 if (unlikely(min_freq > max_freq))
304 min_freq = max_freq;
305
306 min = cppc_khz_to_perf(caps, min_freq);
307 max = cppc_khz_to_perf(caps, max_freq);
308
309 *min_perf = clamp_t(u32, min, caps->lowest_perf, caps->highest_perf);
310 *max_perf = clamp_t(u32, max, caps->lowest_perf, caps->highest_perf);
311}
312
313static void cppc_cpufreq_update_perf_limits(struct cppc_cpudata *cpu_data,
314 struct cpufreq_policy *policy)
315{
316 cppc_cpufreq_get_perf_limits(cpu_data, policy,
317 &cpu_data->perf_ctrls.min_perf,
318 &cpu_data->perf_ctrls.max_perf);
319}
320
321static int cppc_cpufreq_set_target(struct cpufreq_policy *policy,
322 unsigned int target_freq,
323 unsigned int relation)
324{
325 struct cppc_cpudata *cpu_data = policy->driver_data;
326 unsigned int cpu = policy->cpu;
327 struct cpufreq_freqs freqs;
328 int ret = 0;
329
330 cpu_data->perf_ctrls.desired_perf =
331 cppc_khz_to_perf(&cpu_data->perf_caps, target_freq);
332 cppc_cpufreq_update_perf_limits(cpu_data, policy);
333
334 freqs.old = policy->cur;
335 freqs.new = target_freq;
336
337 cpufreq_freq_transition_begin(policy, &freqs);
338 ret = cppc_set_perf(cpu, &cpu_data->perf_ctrls);
339 cpufreq_freq_transition_end(policy, &freqs, ret != 0);
340
341 if (ret)
342 pr_debug("Failed to set target on CPU:%d. ret:%d\n",
343 cpu, ret);
344
345 return ret;
346}
347
348static unsigned int cppc_cpufreq_fast_switch(struct cpufreq_policy *policy,
349 unsigned int target_freq)
350{
351 struct cppc_cpudata *cpu_data = policy->driver_data;
352 unsigned int cpu = policy->cpu;
353 u32 desired_perf;
354 int ret;
355
356 desired_perf = cppc_khz_to_perf(&cpu_data->perf_caps, target_freq);
357 cpu_data->perf_ctrls.desired_perf = desired_perf;
358 cppc_cpufreq_update_perf_limits(cpu_data, policy);
359
360 ret = cppc_set_perf(cpu, &cpu_data->perf_ctrls);
361 if (ret) {
362 pr_debug("Failed to set target on CPU:%d. ret:%d\n",
363 cpu, ret);
364 return 0;
365 }
366
367 return target_freq;
368}
369
370static int cppc_verify_policy(struct cpufreq_policy_data *policy)
371{
372 cpufreq_verify_within_cpu_limits(policy);
373 return 0;
374}
375
376static unsigned int __cppc_cpufreq_get_transition_delay_us(unsigned int cpu)
377{
378 int transition_latency_ns = cppc_get_transition_latency(cpu);
379
380 if (transition_latency_ns < 0)
381 return CPUFREQ_DEFAULT_TRANSITION_LATENCY_NS / NSEC_PER_USEC;
382
383 return transition_latency_ns / NSEC_PER_USEC;
384}
385
386/*
387 * The PCC subspace describes the rate at which platform can accept commands
388 * on the shared PCC channel (including READs which do not count towards freq
389 * transition requests), so ideally we need to use the PCC values as a fallback
390 * if we don't have a platform specific transition_delay_us
391 */
392#ifdef CONFIG_ARM64
393#include <asm/cputype.h>
394
395static unsigned int cppc_cpufreq_get_transition_delay_us(unsigned int cpu)
396{
397 unsigned long implementor = read_cpuid_implementor();
398 unsigned long part_num = read_cpuid_part_number();
399
400 switch (implementor) {
401 case ARM_CPU_IMP_QCOM:
402 switch (part_num) {
403 case QCOM_CPU_PART_FALKOR_V1:
404 case QCOM_CPU_PART_FALKOR:
405 return 10000;
406 }
407 }
408 return __cppc_cpufreq_get_transition_delay_us(cpu);
409}
410#else
411static unsigned int cppc_cpufreq_get_transition_delay_us(unsigned int cpu)
412{
413 return __cppc_cpufreq_get_transition_delay_us(cpu);
414}
415#endif
416
417#if defined(CONFIG_ARM64) && defined(CONFIG_ENERGY_MODEL)
418
419static DEFINE_PER_CPU(unsigned int, efficiency_class);
420
421/* Create an artificial performance state every CPPC_EM_CAP_STEP capacity unit. */
422#define CPPC_EM_CAP_STEP (20)
423/* Increase the cost value by CPPC_EM_COST_STEP every performance state. */
424#define CPPC_EM_COST_STEP (1)
425/* Add a cost gap correspnding to the energy of 4 CPUs. */
426#define CPPC_EM_COST_GAP (4 * SCHED_CAPACITY_SCALE * CPPC_EM_COST_STEP \
427 / CPPC_EM_CAP_STEP)
428
429static unsigned int get_perf_level_count(struct cpufreq_policy *policy)
430{
431 struct cppc_perf_caps *perf_caps;
432 unsigned int min_cap, max_cap;
433 struct cppc_cpudata *cpu_data;
434 int cpu = policy->cpu;
435
436 cpu_data = policy->driver_data;
437 perf_caps = &cpu_data->perf_caps;
438 max_cap = arch_scale_cpu_capacity(cpu);
439 min_cap = div_u64((u64)max_cap * perf_caps->lowest_perf,
440 perf_caps->highest_perf);
441 if ((min_cap == 0) || (max_cap < min_cap))
442 return 0;
443 return 1 + max_cap / CPPC_EM_CAP_STEP - min_cap / CPPC_EM_CAP_STEP;
444}
445
446/*
447 * The cost is defined as:
448 * cost = power * max_frequency / frequency
449 */
450static inline unsigned long compute_cost(int cpu, int step)
451{
452 return CPPC_EM_COST_GAP * per_cpu(efficiency_class, cpu) +
453 step * CPPC_EM_COST_STEP;
454}
455
456static int cppc_get_cpu_power(struct device *cpu_dev,
457 unsigned long *power, unsigned long *KHz)
458{
459 unsigned long perf_step, perf_prev, perf, perf_check;
460 unsigned int min_step, max_step, step, step_check;
461 unsigned long prev_freq = *KHz;
462 unsigned int min_cap, max_cap;
463 struct cpufreq_policy *policy;
464
465 struct cppc_perf_caps *perf_caps;
466 struct cppc_cpudata *cpu_data;
467
468 policy = cpufreq_cpu_get_raw(cpu_dev->id);
469 if (!policy)
470 return -EINVAL;
471
472 cpu_data = policy->driver_data;
473 perf_caps = &cpu_data->perf_caps;
474 max_cap = arch_scale_cpu_capacity(cpu_dev->id);
475 min_cap = div_u64((u64)max_cap * perf_caps->lowest_perf,
476 perf_caps->highest_perf);
477 perf_step = div_u64((u64)CPPC_EM_CAP_STEP * perf_caps->highest_perf,
478 max_cap);
479 min_step = min_cap / CPPC_EM_CAP_STEP;
480 max_step = max_cap / CPPC_EM_CAP_STEP;
481
482 perf_prev = cppc_khz_to_perf(perf_caps, *KHz);
483 step = perf_prev / perf_step;
484
485 if (step > max_step)
486 return -EINVAL;
487
488 if (min_step == max_step) {
489 step = max_step;
490 perf = perf_caps->highest_perf;
491 } else if (step < min_step) {
492 step = min_step;
493 perf = perf_caps->lowest_perf;
494 } else {
495 step++;
496 if (step == max_step)
497 perf = perf_caps->highest_perf;
498 else
499 perf = step * perf_step;
500 }
501
502 *KHz = cppc_perf_to_khz(perf_caps, perf);
503 perf_check = cppc_khz_to_perf(perf_caps, *KHz);
504 step_check = perf_check / perf_step;
505
506 /*
507 * To avoid bad integer approximation, check that new frequency value
508 * increased and that the new frequency will be converted to the
509 * desired step value.
510 */
511 while ((*KHz == prev_freq) || (step_check != step)) {
512 perf++;
513 *KHz = cppc_perf_to_khz(perf_caps, perf);
514 perf_check = cppc_khz_to_perf(perf_caps, *KHz);
515 step_check = perf_check / perf_step;
516 }
517
518 /*
519 * With an artificial EM, only the cost value is used. Still the power
520 * is populated such as 0 < power < EM_MAX_POWER. This allows to add
521 * more sense to the artificial performance states.
522 */
523 *power = compute_cost(cpu_dev->id, step);
524
525 return 0;
526}
527
528static int cppc_get_cpu_cost(struct device *cpu_dev, unsigned long KHz,
529 unsigned long *cost)
530{
531 unsigned long perf_step, perf_prev;
532 struct cppc_perf_caps *perf_caps;
533 struct cpufreq_policy *policy;
534 struct cppc_cpudata *cpu_data;
535 unsigned int max_cap;
536 int step;
537
538 policy = cpufreq_cpu_get_raw(cpu_dev->id);
539 if (!policy)
540 return -EINVAL;
541
542 cpu_data = policy->driver_data;
543 perf_caps = &cpu_data->perf_caps;
544 max_cap = arch_scale_cpu_capacity(cpu_dev->id);
545
546 perf_prev = cppc_khz_to_perf(perf_caps, KHz);
547 perf_step = CPPC_EM_CAP_STEP * perf_caps->highest_perf / max_cap;
548 step = perf_prev / perf_step;
549
550 *cost = compute_cost(cpu_dev->id, step);
551
552 return 0;
553}
554
555static void cppc_cpufreq_register_em(struct cpufreq_policy *policy)
556{
557 struct cppc_cpudata *cpu_data;
558 struct em_data_callback em_cb =
559 EM_ADV_DATA_CB(cppc_get_cpu_power, cppc_get_cpu_cost);
560
561 cpu_data = policy->driver_data;
562 em_dev_register_perf_domain(get_cpu_device(policy->cpu),
563 get_perf_level_count(policy), &em_cb,
564 cpu_data->shared_cpu_map, 0);
565}
566
567static void populate_efficiency_class(void)
568{
569 struct acpi_madt_generic_interrupt *gicc;
570 DECLARE_BITMAP(used_classes, 256) = {};
571 int class, cpu, index;
572
573 for_each_possible_cpu(cpu) {
574 gicc = acpi_cpu_get_madt_gicc(cpu);
575 class = gicc->efficiency_class;
576 bitmap_set(used_classes, class, 1);
577 }
578
579 if (bitmap_weight(used_classes, 256) <= 1) {
580 pr_debug("Efficiency classes are all equal (=%d). "
581 "No EM registered", class);
582 return;
583 }
584
585 /*
586 * Squeeze efficiency class values on [0:#efficiency_class-1].
587 * Values are per spec in [0:255].
588 */
589 index = 0;
590 for_each_set_bit(class, used_classes, 256) {
591 for_each_possible_cpu(cpu) {
592 gicc = acpi_cpu_get_madt_gicc(cpu);
593 if (gicc->efficiency_class == class)
594 per_cpu(efficiency_class, cpu) = index;
595 }
596 index++;
597 }
598 cppc_cpufreq_driver.register_em = cppc_cpufreq_register_em;
599}
600
601#else
602static void populate_efficiency_class(void)
603{
604}
605#endif
606
607static struct cppc_cpudata *cppc_cpufreq_get_cpu_data(unsigned int cpu)
608{
609 struct cppc_cpudata *cpu_data;
610 int ret;
611
612 cpu_data = kzalloc_obj(struct cppc_cpudata);
613 if (!cpu_data)
614 goto out;
615
616 if (!zalloc_cpumask_var(&cpu_data->shared_cpu_map, GFP_KERNEL))
617 goto free_cpu;
618
619 ret = acpi_get_psd_map(cpu, cpu_data);
620 if (ret) {
621 pr_debug("Err parsing CPU%d PSD data: ret:%d\n", cpu, ret);
622 goto free_mask;
623 }
624
625 ret = cppc_get_perf_caps(cpu, &cpu_data->perf_caps);
626 if (ret) {
627 pr_debug("Err reading CPU%d perf caps: ret:%d\n", cpu, ret);
628 goto free_mask;
629 }
630
631 ret = cppc_get_perf(cpu, &cpu_data->perf_ctrls);
632 if (ret) {
633 pr_debug("Err reading CPU%d perf ctrls: ret:%d\n", cpu, ret);
634 goto free_mask;
635 }
636
637 return cpu_data;
638
639free_mask:
640 free_cpumask_var(cpu_data->shared_cpu_map);
641free_cpu:
642 kfree(cpu_data);
643out:
644 return NULL;
645}
646
647static void cppc_cpufreq_put_cpu_data(struct cpufreq_policy *policy)
648{
649 struct cppc_cpudata *cpu_data = policy->driver_data;
650
651 free_cpumask_var(cpu_data->shared_cpu_map);
652 kfree(cpu_data);
653 policy->driver_data = NULL;
654}
655
656static int cppc_cpufreq_cpu_init(struct cpufreq_policy *policy)
657{
658 unsigned int cpu = policy->cpu;
659 struct cppc_cpudata *cpu_data;
660 struct cppc_perf_caps *caps;
661 int ret;
662
663 cpu_data = cppc_cpufreq_get_cpu_data(cpu);
664 if (!cpu_data) {
665 pr_err("Error in acquiring _CPC/_PSD data for CPU%d.\n", cpu);
666 return -ENODEV;
667 }
668 caps = &cpu_data->perf_caps;
669 policy->driver_data = cpu_data;
670
671 /*
672 * Set min to lowest nonlinear perf to avoid any efficiency penalty (see
673 * Section 8.4.7.1.1.5 of ACPI 6.1 spec)
674 */
675 policy->min = cppc_perf_to_khz(caps, caps->lowest_nonlinear_perf);
676
677 /*
678 * Set cpuinfo.min_freq to Lowest to make the full range of performance
679 * available if userspace wants to use any perf between lowest & lowest
680 * nonlinear perf
681 */
682 policy->cpuinfo.min_freq = cppc_perf_to_khz(caps, caps->lowest_perf);
683 policy->cpuinfo.max_freq = cppc_perf_to_khz(caps, policy->boost_enabled ?
684 caps->highest_perf : caps->nominal_perf);
685
686 policy->transition_delay_us = cppc_cpufreq_get_transition_delay_us(cpu);
687 policy->shared_type = cpu_data->shared_type;
688
689 switch (policy->shared_type) {
690 case CPUFREQ_SHARED_TYPE_HW:
691 case CPUFREQ_SHARED_TYPE_NONE:
692 /* Nothing to be done - we'll have a policy for each CPU */
693 break;
694 case CPUFREQ_SHARED_TYPE_ANY:
695 /*
696 * All CPUs in the domain will share a policy and all cpufreq
697 * operations will use a single cppc_cpudata structure stored
698 * in policy->driver_data.
699 */
700 cpumask_copy(policy->cpus, cpu_data->shared_cpu_map);
701 break;
702 default:
703 pr_debug("Unsupported CPU co-ord type: %d\n",
704 policy->shared_type);
705 ret = -EFAULT;
706 goto out;
707 }
708
709 policy->fast_switch_possible = cppc_allow_fast_switch(policy->cpus);
710 policy->dvfs_possible_from_any_cpu = true;
711
712 /*
713 * If 'highest_perf' is greater than 'nominal_perf', we assume CPU Boost
714 * is supported.
715 */
716 if (caps->highest_perf > caps->nominal_perf)
717 policy->boost_supported = true;
718
719 /* Set policy->cur to max now. The governors will adjust later. */
720 policy->cur = cppc_perf_to_khz(caps, caps->highest_perf);
721 cpu_data->perf_ctrls.desired_perf = caps->highest_perf;
722
723 ret = cppc_set_perf(cpu, &cpu_data->perf_ctrls);
724 if (ret) {
725 pr_debug("Err setting perf value:%d on CPU:%d. ret:%d\n",
726 caps->highest_perf, cpu, ret);
727 goto out;
728 }
729
730 cppc_cpufreq_cpu_fie_init(policy);
731 return 0;
732
733out:
734 cppc_cpufreq_put_cpu_data(policy);
735 return ret;
736}
737
738static void cppc_cpufreq_cpu_exit(struct cpufreq_policy *policy)
739{
740 struct cppc_cpudata *cpu_data = policy->driver_data;
741 struct cppc_perf_caps *caps = &cpu_data->perf_caps;
742 unsigned int cpu = policy->cpu;
743 int ret;
744
745 cppc_cpufreq_cpu_fie_exit(policy);
746
747 cpu_data->perf_ctrls.desired_perf = caps->lowest_perf;
748
749 ret = cppc_set_perf(cpu, &cpu_data->perf_ctrls);
750 if (ret)
751 pr_debug("Err setting perf value:%d on CPU:%d. ret:%d\n",
752 caps->lowest_perf, cpu, ret);
753
754 cppc_cpufreq_put_cpu_data(policy);
755}
756
757static inline u64 get_delta(u64 t1, u64 t0)
758{
759 if (t1 > t0 || t0 > ~(u32)0)
760 return t1 - t0;
761
762 return (u32)t1 - (u32)t0;
763}
764
765static int cppc_perf_from_fbctrs(u64 reference_perf,
766 struct cppc_perf_fb_ctrs *fb_ctrs_t0,
767 struct cppc_perf_fb_ctrs *fb_ctrs_t1)
768{
769 u64 delta_reference, delta_delivered;
770
771 delta_reference = get_delta(fb_ctrs_t1->reference,
772 fb_ctrs_t0->reference);
773 delta_delivered = get_delta(fb_ctrs_t1->delivered,
774 fb_ctrs_t0->delivered);
775
776 /*
777 * Avoid divide-by zero and unchanged feedback counters.
778 * Leave it for callers to handle.
779 */
780 if (!delta_reference || !delta_delivered)
781 return 0;
782
783 return (reference_perf * delta_delivered) / delta_reference;
784}
785
786static int cppc_get_perf_ctrs_sample(int cpu,
787 struct cppc_perf_fb_ctrs *fb_ctrs_t0,
788 struct cppc_perf_fb_ctrs *fb_ctrs_t1)
789{
790 int ret;
791
792 ret = cppc_get_perf_ctrs(cpu, fb_ctrs_t0);
793 if (ret)
794 return ret;
795
796 udelay(2); /* 2usec delay between sampling */
797
798 return cppc_get_perf_ctrs(cpu, fb_ctrs_t1);
799}
800
801static unsigned int cppc_cpufreq_get_rate(unsigned int cpu)
802{
803 struct cpufreq_policy *policy __free(put_cpufreq_policy) = cpufreq_cpu_get(cpu);
804 struct cppc_perf_fb_ctrs fb_ctrs_t0 = {0}, fb_ctrs_t1 = {0};
805 struct cppc_cpudata *cpu_data;
806 u64 delivered_perf, reference_perf;
807 int ret;
808
809 if (!policy)
810 return 0;
811
812 cpu_data = policy->driver_data;
813
814 ret = cppc_get_perf_ctrs_sample(cpu, &fb_ctrs_t0, &fb_ctrs_t1);
815 if (ret) {
816 if (ret == -EFAULT)
817 /* Any of the associated CPPC regs is 0. */
818 goto out_invalid_counters;
819 else
820 return 0;
821 }
822
823 reference_perf = cpu_data->perf_caps.reference_perf;
824 delivered_perf = cppc_perf_from_fbctrs(reference_perf,
825 &fb_ctrs_t0, &fb_ctrs_t1);
826 if (!delivered_perf)
827 goto out_invalid_counters;
828
829 return cppc_perf_to_khz(&cpu_data->perf_caps, delivered_perf);
830
831out_invalid_counters:
832 /*
833 * Feedback counters could be unchanged or 0 when a cpu enters a
834 * low-power idle state, e.g. clock-gated or power-gated.
835 * Use desired perf for reflecting frequency. Get the latest register
836 * value first as some platforms may update the actual delivered perf
837 * there; if failed, resort to the cached desired perf.
838 */
839 if (cppc_get_desired_perf(cpu, &delivered_perf) || !delivered_perf)
840 delivered_perf = cpu_data->perf_ctrls.desired_perf;
841
842 return cppc_perf_to_khz(&cpu_data->perf_caps, delivered_perf);
843}
844
845static int cppc_cpufreq_set_boost(struct cpufreq_policy *policy, int state)
846{
847 struct cppc_cpudata *cpu_data = policy->driver_data;
848 struct cppc_perf_caps *caps = &cpu_data->perf_caps;
849
850 if (state)
851 policy->cpuinfo.max_freq = cppc_perf_to_khz(caps, caps->highest_perf);
852 else
853 policy->cpuinfo.max_freq = cppc_perf_to_khz(caps, caps->nominal_perf);
854
855 return 0;
856}
857
858static ssize_t show_freqdomain_cpus(struct cpufreq_policy *policy, char *buf)
859{
860 struct cppc_cpudata *cpu_data = policy->driver_data;
861
862 return cpufreq_show_cpus(cpu_data->shared_cpu_map, buf);
863}
864
865static ssize_t show_auto_select(struct cpufreq_policy *policy, char *buf)
866{
867 bool val;
868 int ret;
869
870 ret = cppc_get_auto_sel(policy->cpu, &val);
871
872 /* show "<unsupported>" when this register is not supported by cpc */
873 if (ret == -EOPNOTSUPP)
874 return sysfs_emit(buf, "<unsupported>\n");
875
876 if (ret)
877 return ret;
878
879 return sysfs_emit(buf, "%d\n", val);
880}
881
882static ssize_t store_auto_select(struct cpufreq_policy *policy,
883 const char *buf, size_t count)
884{
885 struct cppc_cpudata *cpu_data = policy->driver_data;
886 bool val;
887 int ret;
888
889 ret = kstrtobool(buf, &val);
890 if (ret)
891 return ret;
892
893 ret = cppc_set_auto_sel(policy->cpu, val);
894 if (ret)
895 return ret;
896
897 cpu_data->perf_ctrls.auto_sel = val;
898
899 if (val) {
900 u32 old_min_perf = cpu_data->perf_ctrls.min_perf;
901 u32 old_max_perf = cpu_data->perf_ctrls.max_perf;
902
903 /*
904 * When enabling autonomous selection, program MIN_PERF and
905 * MAX_PERF from current policy limits so that the platform
906 * uses the correct performance bounds immediately.
907 */
908 cppc_cpufreq_update_perf_limits(cpu_data, policy);
909
910 ret = cppc_set_perf(policy->cpu, &cpu_data->perf_ctrls);
911 if (ret) {
912 cpu_data->perf_ctrls.min_perf = old_min_perf;
913 cpu_data->perf_ctrls.max_perf = old_max_perf;
914 cppc_set_auto_sel(policy->cpu, false);
915 cpu_data->perf_ctrls.auto_sel = false;
916 return ret;
917 }
918 }
919
920 return count;
921}
922
923static ssize_t cppc_cpufreq_sysfs_show_u64(unsigned int cpu,
924 int (*get_func)(int, u64 *),
925 char *buf)
926{
927 u64 val;
928 int ret = get_func((int)cpu, &val);
929
930 if (ret == -EOPNOTSUPP)
931 return sysfs_emit(buf, "<unsupported>\n");
932
933 if (ret)
934 return ret;
935
936 return sysfs_emit(buf, "%llu\n", val);
937}
938
939static ssize_t cppc_cpufreq_sysfs_store_u64(unsigned int cpu,
940 int (*set_func)(int, u64),
941 const char *buf, size_t count)
942{
943 u64 val;
944 int ret;
945
946 ret = kstrtou64(buf, 0, &val);
947 if (ret)
948 return ret;
949
950 ret = set_func((int)cpu, val);
951
952 return ret ? ret : count;
953}
954
955#define CPPC_CPUFREQ_ATTR_RW_U64(_name, _get_func, _set_func) \
956static ssize_t show_##_name(struct cpufreq_policy *policy, char *buf) \
957{ \
958 return cppc_cpufreq_sysfs_show_u64(policy->cpu, _get_func, buf);\
959} \
960static ssize_t store_##_name(struct cpufreq_policy *policy, \
961 const char *buf, size_t count) \
962{ \
963 return cppc_cpufreq_sysfs_store_u64(policy->cpu, _set_func, \
964 buf, count); \
965}
966
967CPPC_CPUFREQ_ATTR_RW_U64(auto_act_window, cppc_get_auto_act_window,
968 cppc_set_auto_act_window)
969
970static ssize_t
971show_energy_performance_preference_val(struct cpufreq_policy *policy, char *buf)
972{
973 return cppc_cpufreq_sysfs_show_u64(policy->cpu, cppc_get_epp_perf, buf);
974}
975
976static ssize_t
977store_energy_performance_preference_val(struct cpufreq_policy *policy,
978 const char *buf, size_t count)
979{
980 struct cppc_cpudata *cpu_data = policy->driver_data;
981 u64 val;
982 int ret;
983
984 ret = kstrtou64(buf, 0, &val);
985 if (ret)
986 return ret;
987
988 ret = cppc_set_epp(policy->cpu, val);
989 if (ret)
990 return ret;
991
992 cpu_data->perf_ctrls.energy_perf = val;
993
994 return count;
995}
996
997static int cppc_get_perf_limited_filtered(int cpu, u64 *perf_limited)
998{
999 struct cpufreq_policy *policy;
1000 struct cppc_cpudata *cpu_data;
1001 int ret;
1002
1003 ret = cppc_get_perf_limited(cpu, perf_limited);
1004 if (ret)
1005 return ret;
1006
1007 policy = cpufreq_cpu_get_raw(cpu);
1008 if (!policy)
1009 return -EINVAL;
1010
1011 cpu_data = policy->driver_data;
1012
1013 /*
1014 * Desired Excursion is ignored when autonomous selection is
1015 * enabled. Clear the bit to avoid exposing meaningless state
1016 * to userspace.
1017 */
1018 if (cpu_data && cpu_data->perf_ctrls.auto_sel)
1019 *perf_limited &= ~CPPC_PERF_LIMITED_DESIRED_EXCURSION;
1020
1021 return 0;
1022}
1023
1024CPPC_CPUFREQ_ATTR_RW_U64(perf_limited, cppc_get_perf_limited_filtered,
1025 cppc_set_perf_limited)
1026
1027cpufreq_freq_attr_ro(freqdomain_cpus);
1028cpufreq_freq_attr_rw(auto_select);
1029cpufreq_freq_attr_rw(auto_act_window);
1030cpufreq_freq_attr_rw(energy_performance_preference_val);
1031cpufreq_freq_attr_rw(perf_limited);
1032
1033static struct freq_attr *cppc_cpufreq_attr[] = {
1034 &freqdomain_cpus,
1035 &auto_select,
1036 &auto_act_window,
1037 &energy_performance_preference_val,
1038 &perf_limited,
1039 NULL,
1040};
1041
1042static struct cpufreq_driver cppc_cpufreq_driver = {
1043 .flags = CPUFREQ_CONST_LOOPS | CPUFREQ_NEED_UPDATE_LIMITS,
1044 .verify = cppc_verify_policy,
1045 .target = cppc_cpufreq_set_target,
1046 .get = cppc_cpufreq_get_rate,
1047 .fast_switch = cppc_cpufreq_fast_switch,
1048 .init = cppc_cpufreq_cpu_init,
1049 .exit = cppc_cpufreq_cpu_exit,
1050 .set_boost = cppc_cpufreq_set_boost,
1051 .attr = cppc_cpufreq_attr,
1052 .name = "cppc_cpufreq",
1053};
1054
1055static int __init cppc_cpufreq_init(void)
1056{
1057 int ret;
1058
1059 if (!acpi_cpc_valid())
1060 return -ENODEV;
1061
1062 cppc_freq_invariance_init();
1063 populate_efficiency_class();
1064
1065 ret = cpufreq_register_driver(&cppc_cpufreq_driver);
1066 if (ret)
1067 cppc_freq_invariance_exit();
1068
1069 return ret;
1070}
1071
1072static void __exit cppc_cpufreq_exit(void)
1073{
1074 cpufreq_unregister_driver(&cppc_cpufreq_driver);
1075 cppc_freq_invariance_exit();
1076}
1077
1078module_exit(cppc_cpufreq_exit);
1079MODULE_AUTHOR("Ashwin Chaugule");
1080MODULE_DESCRIPTION("CPUFreq driver based on the ACPI CPPC v5.0+ spec");
1081MODULE_LICENSE("GPL");
1082
1083late_initcall(cppc_cpufreq_init);
1084
1085static const struct acpi_device_id cppc_acpi_ids[] __used = {
1086 {ACPI_PROCESSOR_DEVICE_HID, },
1087 {}
1088};
1089
1090MODULE_DEVICE_TABLE(acpi, cppc_acpi_ids);