projects/linux·blob master · refs

3089 lines · 80.6 KiB

1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * linux/drivers/cpufreq/cpufreq.c
4 *
5 * Copyright (C) 2001 Russell King
6 * (C) 2002 - 2003 Dominik Brodowski <[email protected]>
7 * (C) 2013 Viresh Kumar <[email protected]>
8 *
9 * Oct 2005 - Ashok Raj <[email protected]>
10 * Added handling for CPU hotplug
11 * Feb 2006 - Jacob Shin <[email protected]>
12 * Fix handling for CPU hotplug -- affected CPUs
13 */
14
15#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
16
17#include <linux/cpu.h>
18#include <linux/cpufreq.h>
19#include <linux/cpu_cooling.h>
20#include <linux/delay.h>
21#include <linux/device.h>
22#include <linux/init.h>
23#include <linux/kernel_stat.h>
24#include <linux/module.h>
25#include <linux/mutex.h>
26#include <linux/pm_qos.h>
27#include <linux/slab.h>
28#include <linux/string_choices.h>
29#include <linux/suspend.h>
30#include <linux/syscore_ops.h>
31#include <linux/tick.h>
32#include <linux/units.h>
33#include <trace/events/power.h>
34
35static LIST_HEAD(cpufreq_policy_list);
36
37/* Macros to iterate over CPU policies */
38#define for_each_suitable_policy(__policy, __active) \
39 list_for_each_entry(__policy, &cpufreq_policy_list, policy_list) \
40 if ((__active) == !policy_is_inactive(__policy))
41
42#define for_each_active_policy(__policy) \
43 for_each_suitable_policy(__policy, true)
44#define for_each_inactive_policy(__policy) \
45 for_each_suitable_policy(__policy, false)
46
47/* Iterate over governors */
48static LIST_HEAD(cpufreq_governor_list);
49#define for_each_governor(__governor) \
50 list_for_each_entry(__governor, &cpufreq_governor_list, governor_list)
51
52static char default_governor[CPUFREQ_NAME_LEN];
53
54/*
55 * The "cpufreq driver" - the arch- or hardware-dependent low
56 * level driver of CPUFreq support, and its spinlock. This lock
57 * also protects the cpufreq_cpu_data array.
58 */
59static struct cpufreq_driver *cpufreq_driver;
60static DEFINE_PER_CPU(struct cpufreq_policy *, cpufreq_cpu_data);
61static DEFINE_RWLOCK(cpufreq_driver_lock);
62
63static DEFINE_STATIC_KEY_FALSE(cpufreq_freq_invariance);
64bool cpufreq_supports_freq_invariance(void)
65{
66 return static_branch_likely(&cpufreq_freq_invariance);
67}
68
69/* Flag to suspend/resume CPUFreq governors */
70static bool cpufreq_suspended;
71
72static inline bool has_target(void)
73{
74 return cpufreq_driver->target_index || cpufreq_driver->target;
75}
76
77bool has_target_index(void)
78{
79 return !!cpufreq_driver->target_index;
80}
81
82/* internal prototypes */
83static unsigned int __cpufreq_get(struct cpufreq_policy *policy);
84static int cpufreq_init_governor(struct cpufreq_policy *policy);
85static void cpufreq_exit_governor(struct cpufreq_policy *policy);
86static void cpufreq_governor_limits(struct cpufreq_policy *policy);
87static int cpufreq_set_policy(struct cpufreq_policy *policy,
88 struct cpufreq_governor *new_gov,
89 unsigned int new_pol);
90static bool cpufreq_boost_supported(void);
91static int cpufreq_boost_trigger_state(int state);
92
93/*
94 * Two notifier lists: the "policy" list is involved in the
95 * validation process for a new CPU frequency policy; the
96 * "transition" list for kernel code that needs to handle
97 * changes to devices when the CPU clock speed changes.
98 * The mutex locks both lists.
99 */
100static BLOCKING_NOTIFIER_HEAD(cpufreq_policy_notifier_list);
101SRCU_NOTIFIER_HEAD_STATIC(cpufreq_transition_notifier_list);
102
103static int off __read_mostly;
104static int cpufreq_disabled(void)
105{
106 return off;
107}
108void disable_cpufreq(void)
109{
110 off = 1;
111}
112EXPORT_SYMBOL_GPL(disable_cpufreq);
113
114static DEFINE_MUTEX(cpufreq_governor_mutex);
115
116bool have_governor_per_policy(void)
117{
118 return !!(cpufreq_driver->flags & CPUFREQ_HAVE_GOVERNOR_PER_POLICY);
119}
120EXPORT_SYMBOL_GPL(have_governor_per_policy);
121
122static struct kobject *cpufreq_global_kobject;
123
124struct kobject *get_governor_parent_kobj(struct cpufreq_policy *policy)
125{
126 if (have_governor_per_policy())
127 return &policy->kobj;
128 else
129 return cpufreq_global_kobject;
130}
131EXPORT_SYMBOL_GPL(get_governor_parent_kobj);
132
133u64 get_cpu_idle_time(unsigned int cpu, u64 *wall, int io_busy)
134{
135 u64 idle_time = get_cpu_idle_time_us(cpu, io_busy ? wall : NULL);
136
137 if (!io_busy)
138 idle_time += get_cpu_iowait_time_us(cpu, wall);
139
140 return idle_time;
141}
142EXPORT_SYMBOL_GPL(get_cpu_idle_time);
143
144/*
145 * This is a generic cpufreq init() routine which can be used by cpufreq
146 * drivers of SMP systems. It will do following:
147 * - validate & show freq table passed
148 * - set policies transition latency
149 * - policy->cpus with all possible CPUs
150 */
151void cpufreq_generic_init(struct cpufreq_policy *policy,
152 struct cpufreq_frequency_table *table,
153 unsigned int transition_latency)
154{
155 policy->freq_table = table;
156 policy->cpuinfo.transition_latency = transition_latency;
157
158 /*
159 * The driver only supports the SMP configuration where all processors
160 * share the clock and voltage and clock.
161 */
162 cpumask_setall(policy->cpus);
163}
164EXPORT_SYMBOL_GPL(cpufreq_generic_init);
165
166struct cpufreq_policy *cpufreq_cpu_get_raw(unsigned int cpu)
167{
168 struct cpufreq_policy *policy = per_cpu(cpufreq_cpu_data, cpu);
169
170 return policy && cpumask_test_cpu(cpu, policy->cpus) ? policy : NULL;
171}
172EXPORT_SYMBOL_GPL(cpufreq_cpu_get_raw);
173
174struct cpufreq_policy *cpufreq_cpu_policy(unsigned int cpu)
175{
176 return per_cpu(cpufreq_cpu_data, cpu);
177}
178EXPORT_SYMBOL_GPL(cpufreq_cpu_policy);
179
180unsigned int cpufreq_generic_get(unsigned int cpu)
181{
182 struct cpufreq_policy *policy = cpufreq_cpu_get_raw(cpu);
183
184 if (!policy || IS_ERR(policy->clk)) {
185 pr_err("%s: No %s associated to cpu: %d\n",
186 __func__, policy ? "clk" : "policy", cpu);
187 return 0;
188 }
189
190 return clk_get_rate(policy->clk) / 1000;
191}
192EXPORT_SYMBOL_GPL(cpufreq_generic_get);
193
194/**
195 * cpufreq_cpu_get - Return policy for a CPU and mark it as busy.
196 * @cpu: CPU to find the policy for.
197 *
198 * Call cpufreq_cpu_get_raw() to obtain a cpufreq policy for @cpu and increment
199 * the kobject reference counter of that policy. Return a valid policy on
200 * success or NULL on failure.
201 *
202 * The policy returned by this function has to be released with the help of
203 * cpufreq_cpu_put() to balance its kobject reference counter properly.
204 */
205struct cpufreq_policy *cpufreq_cpu_get(unsigned int cpu)
206{
207 struct cpufreq_policy *policy = NULL;
208 unsigned long flags;
209
210 if (WARN_ON(cpu >= nr_cpu_ids))
211 return NULL;
212
213 /* get the cpufreq driver */
214 read_lock_irqsave(&cpufreq_driver_lock, flags);
215
216 if (cpufreq_driver) {
217 /* get the CPU */
218 policy = cpufreq_cpu_get_raw(cpu);
219 if (policy)
220 kobject_get(&policy->kobj);
221 }
222
223 read_unlock_irqrestore(&cpufreq_driver_lock, flags);
224
225 return policy;
226}
227EXPORT_SYMBOL_GPL(cpufreq_cpu_get);
228
229/**
230 * cpufreq_cpu_put - Decrement kobject usage counter for cpufreq policy.
231 * @policy: cpufreq policy returned by cpufreq_cpu_get().
232 */
233void cpufreq_cpu_put(struct cpufreq_policy *policy)
234{
235 kobject_put(&policy->kobj);
236}
237EXPORT_SYMBOL_GPL(cpufreq_cpu_put);
238
239/*********************************************************************
240 * EXTERNALLY AFFECTING FREQUENCY CHANGES *
241 *********************************************************************/
242
243/**
244 * adjust_jiffies - Adjust the system "loops_per_jiffy".
245 * @val: CPUFREQ_PRECHANGE or CPUFREQ_POSTCHANGE.
246 * @ci: Frequency change information.
247 *
248 * This function alters the system "loops_per_jiffy" for the clock
249 * speed change. Note that loops_per_jiffy cannot be updated on SMP
250 * systems as each CPU might be scaled differently. So, use the arch
251 * per-CPU loops_per_jiffy value wherever possible.
252 */
253static void adjust_jiffies(unsigned long val, struct cpufreq_freqs *ci)
254{
255#ifndef CONFIG_SMP
256 static unsigned long l_p_j_ref;
257 static unsigned int l_p_j_ref_freq;
258
259 if (ci->flags & CPUFREQ_CONST_LOOPS)
260 return;
261
262 if (!l_p_j_ref_freq) {
263 l_p_j_ref = loops_per_jiffy;
264 l_p_j_ref_freq = ci->old;
265 pr_debug("saving %lu as reference value for loops_per_jiffy; freq is %u kHz\n",
266 l_p_j_ref, l_p_j_ref_freq);
267 }
268 if (val == CPUFREQ_POSTCHANGE && ci->old != ci->new) {
269 loops_per_jiffy = cpufreq_scale(l_p_j_ref, l_p_j_ref_freq,
270 ci->new);
271 pr_debug("scaling loops_per_jiffy to %lu for frequency %u kHz\n",
272 loops_per_jiffy, ci->new);
273 }
274#endif
275}
276
277/**
278 * cpufreq_notify_transition - Notify frequency transition and adjust jiffies.
279 * @policy: cpufreq policy to enable fast frequency switching for.
280 * @freqs: contain details of the frequency update.
281 * @state: set to CPUFREQ_PRECHANGE or CPUFREQ_POSTCHANGE.
282 *
283 * This function calls the transition notifiers and adjust_jiffies().
284 *
285 * It is called twice on all CPU frequency changes that have external effects.
286 */
287static void cpufreq_notify_transition(struct cpufreq_policy *policy,
288 struct cpufreq_freqs *freqs,
289 unsigned int state)
290{
291 int cpu;
292
293 BUG_ON(irqs_disabled());
294
295 if (cpufreq_disabled())
296 return;
297
298 freqs->policy = policy;
299 freqs->flags = cpufreq_driver->flags;
300 pr_debug("notification %u of frequency transition to %u kHz\n",
301 state, freqs->new);
302
303 switch (state) {
304 case CPUFREQ_PRECHANGE:
305 /*
306 * Detect if the driver reported a value as "old frequency"
307 * which is not equal to what the cpufreq core thinks is
308 * "old frequency".
309 */
310 if (policy->cur && policy->cur != freqs->old) {
311 pr_debug("Warning: CPU frequency is %u, cpufreq assumed %u kHz\n",
312 freqs->old, policy->cur);
313 freqs->old = policy->cur;
314 }
315
316 srcu_notifier_call_chain(&cpufreq_transition_notifier_list,
317 CPUFREQ_PRECHANGE, freqs);
318
319 adjust_jiffies(CPUFREQ_PRECHANGE, freqs);
320 break;
321
322 case CPUFREQ_POSTCHANGE:
323 adjust_jiffies(CPUFREQ_POSTCHANGE, freqs);
324 pr_debug("FREQ: %u - CPUs: %*pbl\n", freqs->new,
325 cpumask_pr_args(policy->cpus));
326
327 for_each_cpu(cpu, policy->cpus)
328 trace_cpu_frequency(freqs->new, cpu);
329
330 srcu_notifier_call_chain(&cpufreq_transition_notifier_list,
331 CPUFREQ_POSTCHANGE, freqs);
332
333 cpufreq_stats_record_transition(policy, freqs->new);
334 policy->cur = freqs->new;
335 }
336}
337
338/* Do post notifications when there are chances that transition has failed */
339static void cpufreq_notify_post_transition(struct cpufreq_policy *policy,
340 struct cpufreq_freqs *freqs, int transition_failed)
341{
342 cpufreq_notify_transition(policy, freqs, CPUFREQ_POSTCHANGE);
343 if (!transition_failed)
344 return;
345
346 swap(freqs->old, freqs->new);
347 cpufreq_notify_transition(policy, freqs, CPUFREQ_PRECHANGE);
348 cpufreq_notify_transition(policy, freqs, CPUFREQ_POSTCHANGE);
349}
350
351void cpufreq_freq_transition_begin(struct cpufreq_policy *policy,
352 struct cpufreq_freqs *freqs)
353{
354
355 /*
356 * Catch double invocations of _begin() which lead to self-deadlock.
357 * ASYNC_NOTIFICATION drivers are left out because the cpufreq core
358 * doesn't invoke _begin() on their behalf, and hence the chances of
359 * double invocations are very low. Moreover, there are scenarios
360 * where these checks can emit false-positive warnings in these
361 * drivers; so we avoid that by skipping them altogether.
362 */
363 WARN_ON(!(cpufreq_driver->flags & CPUFREQ_ASYNC_NOTIFICATION)
364 && current == policy->transition_task);
365
366wait:
367 wait_event(policy->transition_wait, !policy->transition_ongoing);
368
369 spin_lock(&policy->transition_lock);
370
371 if (unlikely(policy->transition_ongoing)) {
372 spin_unlock(&policy->transition_lock);
373 goto wait;
374 }
375
376 policy->transition_ongoing = true;
377 policy->transition_task = current;
378
379 spin_unlock(&policy->transition_lock);
380
381 cpufreq_notify_transition(policy, freqs, CPUFREQ_PRECHANGE);
382}
383EXPORT_SYMBOL_GPL(cpufreq_freq_transition_begin);
384
385void cpufreq_freq_transition_end(struct cpufreq_policy *policy,
386 struct cpufreq_freqs *freqs, int transition_failed)
387{
388 if (WARN_ON(!policy->transition_ongoing))
389 return;
390
391 cpufreq_notify_post_transition(policy, freqs, transition_failed);
392
393 arch_set_freq_scale(policy->related_cpus,
394 policy->cur,
395 arch_scale_freq_ref(policy->cpu));
396
397 spin_lock(&policy->transition_lock);
398 policy->transition_ongoing = false;
399 policy->transition_task = NULL;
400 spin_unlock(&policy->transition_lock);
401
402 wake_up(&policy->transition_wait);
403}
404EXPORT_SYMBOL_GPL(cpufreq_freq_transition_end);
405
406/*
407 * Fast frequency switching status count. Positive means "enabled", negative
408 * means "disabled" and 0 means "not decided yet".
409 */
410static int cpufreq_fast_switch_count;
411static DEFINE_MUTEX(cpufreq_fast_switch_lock);
412
413static void cpufreq_list_transition_notifiers(void)
414{
415 struct notifier_block *nb;
416
417 pr_info("Registered transition notifiers:\n");
418
419 mutex_lock(&cpufreq_transition_notifier_list.mutex);
420
421 for (nb = cpufreq_transition_notifier_list.head; nb; nb = nb->next)
422 pr_info("%pS\n", nb->notifier_call);
423
424 mutex_unlock(&cpufreq_transition_notifier_list.mutex);
425}
426
427/**
428 * cpufreq_enable_fast_switch - Enable fast frequency switching for policy.
429 * @policy: cpufreq policy to enable fast frequency switching for.
430 *
431 * Try to enable fast frequency switching for @policy.
432 *
433 * The attempt will fail if there is at least one transition notifier registered
434 * at this point, as fast frequency switching is quite fundamentally at odds
435 * with transition notifiers. Thus if successful, it will make registration of
436 * transition notifiers fail going forward.
437 */
438void cpufreq_enable_fast_switch(struct cpufreq_policy *policy)
439{
440 lockdep_assert_held(&policy->rwsem);
441
442 if (!policy->fast_switch_possible)
443 return;
444
445 mutex_lock(&cpufreq_fast_switch_lock);
446 if (cpufreq_fast_switch_count >= 0) {
447 cpufreq_fast_switch_count++;
448 policy->fast_switch_enabled = true;
449 } else {
450 pr_warn("CPU%u: Fast frequency switching not enabled\n",
451 policy->cpu);
452 cpufreq_list_transition_notifiers();
453 }
454 mutex_unlock(&cpufreq_fast_switch_lock);
455}
456EXPORT_SYMBOL_GPL(cpufreq_enable_fast_switch);
457
458/**
459 * cpufreq_disable_fast_switch - Disable fast frequency switching for policy.
460 * @policy: cpufreq policy to disable fast frequency switching for.
461 */
462void cpufreq_disable_fast_switch(struct cpufreq_policy *policy)
463{
464 mutex_lock(&cpufreq_fast_switch_lock);
465 if (policy->fast_switch_enabled) {
466 policy->fast_switch_enabled = false;
467 if (!WARN_ON(cpufreq_fast_switch_count <= 0))
468 cpufreq_fast_switch_count--;
469 }
470 mutex_unlock(&cpufreq_fast_switch_lock);
471}
472EXPORT_SYMBOL_GPL(cpufreq_disable_fast_switch);
473
474static unsigned int __resolve_freq(struct cpufreq_policy *policy,
475 unsigned int target_freq,
476 unsigned int min, unsigned int max,
477 unsigned int relation)
478{
479 unsigned int idx;
480
481 target_freq = clamp_val(target_freq, min, max);
482
483 if (!policy->freq_table)
484 return target_freq;
485
486 idx = cpufreq_frequency_table_target(policy, target_freq, min, max, relation);
487 policy->cached_resolved_idx = idx;
488 policy->cached_target_freq = target_freq;
489 return policy->freq_table[idx].frequency;
490}
491
492/**
493 * cpufreq_driver_resolve_freq - Map a target frequency to a driver-supported
494 * one.
495 * @policy: associated policy to interrogate
496 * @target_freq: target frequency to resolve.
497 *
498 * The target to driver frequency mapping is cached in the policy.
499 *
500 * Return: Lowest driver-supported frequency greater than or equal to the
501 * given target_freq, subject to policy (min/max) and driver limitations.
502 */
503unsigned int cpufreq_driver_resolve_freq(struct cpufreq_policy *policy,
504 unsigned int target_freq)
505{
506 unsigned int min = READ_ONCE(policy->min);
507 unsigned int max = READ_ONCE(policy->max);
508
509 /*
510 * If this function runs in parallel with cpufreq_set_policy(), it may
511 * read policy->min before the update and policy->max after the update
512 * or the other way around, so there is no ordering guarantee.
513 *
514 * Resolve this by always honoring the max (in case it comes from
515 * thermal throttling or similar).
516 */
517 if (unlikely(min > max))
518 min = max;
519
520 return __resolve_freq(policy, target_freq, min, max, CPUFREQ_RELATION_LE);
521}
522EXPORT_SYMBOL_GPL(cpufreq_driver_resolve_freq);
523
524unsigned int cpufreq_policy_transition_delay_us(struct cpufreq_policy *policy)
525{
526 unsigned int latency;
527
528 if (policy->transition_delay_us)
529 return policy->transition_delay_us;
530
531 latency = policy->cpuinfo.transition_latency / NSEC_PER_USEC;
532 if (latency)
533 /* Give a 50% breathing room between updates */
534 return latency + (latency >> 1);
535
536 return USEC_PER_MSEC;
537}
538EXPORT_SYMBOL_GPL(cpufreq_policy_transition_delay_us);
539
540/*********************************************************************
541 * SYSFS INTERFACE *
542 *********************************************************************/
543static ssize_t show_boost(struct kobject *kobj,
544 struct kobj_attribute *attr, char *buf)
545{
546 return sysfs_emit(buf, "%d\n", cpufreq_driver->boost_enabled);
547}
548
549static ssize_t store_boost(struct kobject *kobj, struct kobj_attribute *attr,
550 const char *buf, size_t count)
551{
552 bool enable;
553
554 if (kstrtobool(buf, &enable))
555 return -EINVAL;
556
557 if (cpufreq_boost_trigger_state(enable)) {
558 pr_err("%s: Cannot %s BOOST!\n",
559 __func__, str_enable_disable(enable));
560 return -EINVAL;
561 }
562
563 pr_debug("%s: cpufreq BOOST %s\n",
564 __func__, str_enabled_disabled(enable));
565
566 return count;
567}
568define_one_global_rw(boost);
569
570static ssize_t show_local_boost(struct cpufreq_policy *policy, char *buf)
571{
572 return sysfs_emit(buf, "%d\n", policy->boost_enabled);
573}
574
575static int policy_set_boost(struct cpufreq_policy *policy, bool enable)
576{
577 int ret;
578
579 if (policy->boost_enabled == enable)
580 return 0;
581
582 policy->boost_enabled = enable;
583
584 ret = cpufreq_driver->set_boost(policy, enable);
585 if (ret) {
586 policy->boost_enabled = !policy->boost_enabled;
587 return ret;
588 }
589
590 ret = freq_qos_update_request(&policy->boost_freq_req, policy->cpuinfo.max_freq);
591 if (ret < 0) {
592 policy->boost_enabled = !policy->boost_enabled;
593 cpufreq_driver->set_boost(policy, policy->boost_enabled);
594 return ret;
595 }
596
597 return 0;
598}
599
600static ssize_t store_local_boost(struct cpufreq_policy *policy,
601 const char *buf, size_t count)
602{
603 int ret;
604 bool enable;
605
606 if (kstrtobool(buf, &enable))
607 return -EINVAL;
608
609 if (!cpufreq_driver->boost_enabled)
610 return -EINVAL;
611
612 if (!policy->boost_supported)
613 return -EINVAL;
614
615 ret = policy_set_boost(policy, enable);
616 if (!ret)
617 return count;
618
619 return ret;
620}
621
622static struct freq_attr local_boost = __ATTR(boost, 0644, show_local_boost, store_local_boost);
623
624static struct cpufreq_governor *find_governor(const char *str_governor)
625{
626 struct cpufreq_governor *t;
627
628 for_each_governor(t)
629 if (!strncasecmp(str_governor, t->name, CPUFREQ_NAME_LEN))
630 return t;
631
632 return NULL;
633}
634
635static struct cpufreq_governor *get_governor(const char *str_governor)
636{
637 struct cpufreq_governor *t;
638
639 mutex_lock(&cpufreq_governor_mutex);
640 t = find_governor(str_governor);
641 if (!t)
642 goto unlock;
643
644 if (!try_module_get(t->owner))
645 t = NULL;
646
647unlock:
648 mutex_unlock(&cpufreq_governor_mutex);
649
650 return t;
651}
652
653static unsigned int cpufreq_parse_policy(char *str_governor)
654{
655 if (!strncasecmp(str_governor, "performance", strlen("performance")))
656 return CPUFREQ_POLICY_PERFORMANCE;
657
658 if (!strncasecmp(str_governor, "powersave", strlen("powersave")))
659 return CPUFREQ_POLICY_POWERSAVE;
660
661 return CPUFREQ_POLICY_UNKNOWN;
662}
663
664/**
665 * cpufreq_parse_governor - parse a governor string only for has_target()
666 * @str_governor: Governor name.
667 */
668static struct cpufreq_governor *cpufreq_parse_governor(char *str_governor)
669{
670 struct cpufreq_governor *t;
671
672 t = get_governor(str_governor);
673 if (t)
674 return t;
675
676 if (request_module("cpufreq_%s", str_governor))
677 return NULL;
678
679 return get_governor(str_governor);
680}
681
682/*
683 * cpufreq_per_cpu_attr_read() / show_##file_name() -
684 * print out cpufreq information
685 *
686 * Write out information from cpufreq_driver->policy[cpu]; object must be
687 * "unsigned int".
688 */
689
690#define show_one(file_name, object) \
691static ssize_t show_##file_name \
692(struct cpufreq_policy *policy, char *buf) \
693{ \
694 return sysfs_emit(buf, "%u\n", policy->object); \
695}
696
697show_one(cpuinfo_min_freq, cpuinfo.min_freq);
698show_one(cpuinfo_max_freq, cpuinfo.max_freq);
699show_one(cpuinfo_transition_latency, cpuinfo.transition_latency);
700show_one(scaling_min_freq, min);
701show_one(scaling_max_freq, max);
702
703__weak int arch_freq_get_on_cpu(int cpu)
704{
705 return -EOPNOTSUPP;
706}
707
708static inline bool cpufreq_avg_freq_supported(struct cpufreq_policy *policy)
709{
710 return arch_freq_get_on_cpu(policy->cpu) != -EOPNOTSUPP;
711}
712
713static ssize_t show_scaling_cur_freq(struct cpufreq_policy *policy, char *buf)
714{
715 ssize_t ret;
716 int freq;
717
718 freq = IS_ENABLED(CONFIG_CPUFREQ_ARCH_CUR_FREQ)
719 ? arch_freq_get_on_cpu(policy->cpu)
720 : 0;
721
722 if (freq > 0)
723 ret = sysfs_emit(buf, "%u\n", freq);
724 else if (cpufreq_driver->setpolicy && cpufreq_driver->get)
725 ret = sysfs_emit(buf, "%u\n", cpufreq_driver->get(policy->cpu));
726 else
727 ret = sysfs_emit(buf, "%u\n", policy->cur);
728 return ret;
729}
730
731/*
732 * cpufreq_per_cpu_attr_write() / store_##file_name() - sysfs write access
733 */
734#define store_one(file_name, object) \
735static ssize_t store_##file_name \
736(struct cpufreq_policy *policy, const char *buf, size_t count) \
737{ \
738 unsigned long val; \
739 int ret; \
740 \
741 ret = kstrtoul(buf, 0, &val); \
742 if (ret) \
743 return ret; \
744 \
745 ret = freq_qos_update_request(&policy->object##_freq_req, val); \
746 return ret >= 0 ? count : ret; \
747}
748
749store_one(scaling_min_freq, min);
750store_one(scaling_max_freq, max);
751
752/*
753 * show_cpuinfo_cur_freq - current CPU frequency as detected by hardware
754 */
755static ssize_t show_cpuinfo_cur_freq(struct cpufreq_policy *policy,
756 char *buf)
757{
758 unsigned int cur_freq = __cpufreq_get(policy);
759
760 if (cur_freq)
761 return sysfs_emit(buf, "%u\n", cur_freq);
762
763 return sysfs_emit(buf, "<unknown>\n");
764}
765
766/*
767 * show_cpuinfo_avg_freq - average CPU frequency as detected by hardware
768 */
769static ssize_t show_cpuinfo_avg_freq(struct cpufreq_policy *policy,
770 char *buf)
771{
772 int avg_freq = arch_freq_get_on_cpu(policy->cpu);
773
774 if (avg_freq > 0)
775 return sysfs_emit(buf, "%u\n", avg_freq);
776 return avg_freq != 0 ? avg_freq : -EINVAL;
777}
778
779/*
780 * show_scaling_governor - show the current policy for the specified CPU
781 */
782static ssize_t show_scaling_governor(struct cpufreq_policy *policy, char *buf)
783{
784 if (policy->policy == CPUFREQ_POLICY_POWERSAVE)
785 return sysfs_emit(buf, "powersave\n");
786 else if (policy->policy == CPUFREQ_POLICY_PERFORMANCE)
787 return sysfs_emit(buf, "performance\n");
788 else if (policy->governor)
789 return sysfs_emit(buf, "%s\n", policy->governor->name);
790 return -EINVAL;
791}
792
793/*
794 * store_scaling_governor - store policy for the specified CPU
795 */
796static ssize_t store_scaling_governor(struct cpufreq_policy *policy,
797 const char *buf, size_t count)
798{
799 char str_governor[CPUFREQ_NAME_LEN];
800 int ret;
801
802 ret = sscanf(buf, "%15s", str_governor);
803 if (ret != 1)
804 return -EINVAL;
805
806 if (cpufreq_driver->setpolicy) {
807 unsigned int new_pol;
808
809 new_pol = cpufreq_parse_policy(str_governor);
810 if (!new_pol)
811 return -EINVAL;
812
813 ret = cpufreq_set_policy(policy, NULL, new_pol);
814 } else {
815 struct cpufreq_governor *new_gov;
816
817 new_gov = cpufreq_parse_governor(str_governor);
818 if (!new_gov)
819 return -EINVAL;
820
821 ret = cpufreq_set_policy(policy, new_gov,
822 CPUFREQ_POLICY_UNKNOWN);
823
824 module_put(new_gov->owner);
825 }
826
827 return ret ? ret : count;
828}
829
830/*
831 * show_scaling_driver - show the cpufreq driver currently loaded
832 */
833static ssize_t show_scaling_driver(struct cpufreq_policy *policy, char *buf)
834{
835 return scnprintf(buf, CPUFREQ_NAME_PLEN, "%s\n", cpufreq_driver->name);
836}
837
838/*
839 * show_scaling_available_governors - show the available CPUfreq governors
840 */
841static ssize_t show_scaling_available_governors(struct cpufreq_policy *policy,
842 char *buf)
843{
844 ssize_t i = 0;
845 struct cpufreq_governor *t;
846
847 if (!has_target()) {
848 i += sysfs_emit(buf, "performance powersave");
849 goto out;
850 }
851
852 mutex_lock(&cpufreq_governor_mutex);
853 for_each_governor(t) {
854 if (i >= (ssize_t) ((PAGE_SIZE / sizeof(char))
855 - (CPUFREQ_NAME_LEN + 2)))
856 break;
857 i += sysfs_emit_at(buf, i, "%s ", t->name);
858 }
859 mutex_unlock(&cpufreq_governor_mutex);
860out:
861 i += sysfs_emit_at(buf, i, "\n");
862 return i;
863}
864
865ssize_t cpufreq_show_cpus(const struct cpumask *mask, char *buf)
866{
867 ssize_t i = 0;
868 unsigned int cpu;
869
870 for_each_cpu(cpu, mask) {
871 i += sysfs_emit_at(buf, i, "%u ", cpu);
872 if (i >= (PAGE_SIZE - 5))
873 break;
874 }
875
876 /* Remove the extra space at the end */
877 i--;
878
879 i += sysfs_emit_at(buf, i, "\n");
880 return i;
881}
882EXPORT_SYMBOL_GPL(cpufreq_show_cpus);
883
884/*
885 * show_related_cpus - show the CPUs affected by each transition even if
886 * hw coordination is in use
887 */
888static ssize_t show_related_cpus(struct cpufreq_policy *policy, char *buf)
889{
890 return cpufreq_show_cpus(policy->related_cpus, buf);
891}
892
893/*
894 * show_affected_cpus - show the CPUs affected by each transition
895 */
896static ssize_t show_affected_cpus(struct cpufreq_policy *policy, char *buf)
897{
898 return cpufreq_show_cpus(policy->cpus, buf);
899}
900
901static ssize_t store_scaling_setspeed(struct cpufreq_policy *policy,
902 const char *buf, size_t count)
903{
904 unsigned int freq = 0;
905 int ret;
906
907 if (!policy->governor || !policy->governor->store_setspeed)
908 return -EINVAL;
909
910 ret = kstrtouint(buf, 0, &freq);
911 if (ret)
912 return ret;
913
914 policy->governor->store_setspeed(policy, freq);
915
916 return count;
917}
918
919static ssize_t show_scaling_setspeed(struct cpufreq_policy *policy, char *buf)
920{
921 if (!policy->governor || !policy->governor->show_setspeed)
922 return sysfs_emit(buf, "<unsupported>\n");
923
924 return policy->governor->show_setspeed(policy, buf);
925}
926
927/*
928 * show_bios_limit - show the current cpufreq HW/BIOS limitation
929 */
930static ssize_t show_bios_limit(struct cpufreq_policy *policy, char *buf)
931{
932 unsigned int limit;
933 int ret;
934 ret = cpufreq_driver->bios_limit(policy->cpu, &limit);
935 if (!ret)
936 return sysfs_emit(buf, "%u\n", limit);
937 return sysfs_emit(buf, "%u\n", policy->cpuinfo.max_freq);
938}
939
940cpufreq_freq_attr_ro_perm(cpuinfo_cur_freq, 0400);
941cpufreq_freq_attr_ro(cpuinfo_avg_freq);
942cpufreq_freq_attr_ro(cpuinfo_min_freq);
943cpufreq_freq_attr_ro(cpuinfo_max_freq);
944cpufreq_freq_attr_ro(cpuinfo_transition_latency);
945cpufreq_freq_attr_ro(scaling_available_governors);
946cpufreq_freq_attr_ro(scaling_driver);
947cpufreq_freq_attr_ro(scaling_cur_freq);
948cpufreq_freq_attr_ro(bios_limit);
949cpufreq_freq_attr_ro(related_cpus);
950cpufreq_freq_attr_ro(affected_cpus);
951cpufreq_freq_attr_rw(scaling_min_freq);
952cpufreq_freq_attr_rw(scaling_max_freq);
953cpufreq_freq_attr_rw(scaling_governor);
954cpufreq_freq_attr_rw(scaling_setspeed);
955
956static struct attribute *cpufreq_attrs[] = {
957 &cpuinfo_min_freq.attr,
958 &cpuinfo_max_freq.attr,
959 &cpuinfo_transition_latency.attr,
960 &scaling_cur_freq.attr,
961 &scaling_min_freq.attr,
962 &scaling_max_freq.attr,
963 &affected_cpus.attr,
964 &related_cpus.attr,
965 &scaling_governor.attr,
966 &scaling_driver.attr,
967 &scaling_available_governors.attr,
968 &scaling_setspeed.attr,
969 NULL
970};
971ATTRIBUTE_GROUPS(cpufreq);
972
973#define to_policy(k) container_of(k, struct cpufreq_policy, kobj)
974#define to_attr(a) container_of(a, struct freq_attr, attr)
975
976static ssize_t show(struct kobject *kobj, struct attribute *attr, char *buf)
977{
978 struct cpufreq_policy *policy = to_policy(kobj);
979 struct freq_attr *fattr = to_attr(attr);
980
981 if (!fattr->show)
982 return -EIO;
983
984 guard(cpufreq_policy_read)(policy);
985
986 if (likely(!policy_is_inactive(policy)))
987 return fattr->show(policy, buf);
988
989 return -EBUSY;
990}
991
992static ssize_t store(struct kobject *kobj, struct attribute *attr,
993 const char *buf, size_t count)
994{
995 struct cpufreq_policy *policy = to_policy(kobj);
996 struct freq_attr *fattr = to_attr(attr);
997
998 if (!fattr->store)
999 return -EIO;
1000
1001 guard(cpufreq_policy_write)(policy);
1002
1003 if (likely(!policy_is_inactive(policy)))
1004 return fattr->store(policy, buf, count);
1005
1006 return -EBUSY;
1007}
1008
1009static void cpufreq_sysfs_release(struct kobject *kobj)
1010{
1011 struct cpufreq_policy *policy = to_policy(kobj);
1012 pr_debug("last reference is dropped\n");
1013 complete(&policy->kobj_unregister);
1014}
1015
1016static const struct sysfs_ops sysfs_ops = {
1017 .show = show,
1018 .store = store,
1019};
1020
1021static const struct kobj_type ktype_cpufreq = {
1022 .sysfs_ops = &sysfs_ops,
1023 .default_groups = cpufreq_groups,
1024 .release = cpufreq_sysfs_release,
1025};
1026
1027static void add_cpu_dev_symlink(struct cpufreq_policy *policy, unsigned int cpu,
1028 struct device *dev)
1029{
1030 if (unlikely(!dev))
1031 return;
1032
1033 if (cpumask_test_and_set_cpu(cpu, policy->real_cpus))
1034 return;
1035
1036 dev_dbg(dev, "%s: Adding symlink\n", __func__);
1037 if (sysfs_create_link(&dev->kobj, &policy->kobj, "cpufreq"))
1038 dev_err(dev, "cpufreq symlink creation failed\n");
1039}
1040
1041static void remove_cpu_dev_symlink(struct cpufreq_policy *policy, int cpu,
1042 struct device *dev)
1043{
1044 dev_dbg(dev, "%s: Removing symlink\n", __func__);
1045 sysfs_remove_link(&dev->kobj, "cpufreq");
1046 cpumask_clear_cpu(cpu, policy->real_cpus);
1047}
1048
1049static int cpufreq_add_dev_interface(struct cpufreq_policy *policy)
1050{
1051 struct freq_attr **drv_attr;
1052 int ret = 0;
1053
1054 /* Attributes that need freq_table */
1055 if (policy->freq_table) {
1056 ret = sysfs_create_file(&policy->kobj,
1057 &cpufreq_freq_attr_scaling_available_freqs.attr);
1058 if (ret)
1059 return ret;
1060
1061 if (cpufreq_boost_supported()) {
1062 ret = sysfs_create_file(&policy->kobj,
1063 &cpufreq_freq_attr_scaling_boost_freqs.attr);
1064 if (ret)
1065 return ret;
1066 }
1067 }
1068
1069 /* set up files for this cpu device */
1070 drv_attr = cpufreq_driver->attr;
1071 while (drv_attr && *drv_attr) {
1072 ret = sysfs_create_file(&policy->kobj, &((*drv_attr)->attr));
1073 if (ret)
1074 return ret;
1075 drv_attr++;
1076 }
1077 if (cpufreq_driver->get) {
1078 ret = sysfs_create_file(&policy->kobj, &cpuinfo_cur_freq.attr);
1079 if (ret)
1080 return ret;
1081 }
1082
1083 if (cpufreq_avg_freq_supported(policy)) {
1084 ret = sysfs_create_file(&policy->kobj, &cpuinfo_avg_freq.attr);
1085 if (ret)
1086 return ret;
1087 }
1088
1089 if (cpufreq_driver->bios_limit) {
1090 ret = sysfs_create_file(&policy->kobj, &bios_limit.attr);
1091 if (ret)
1092 return ret;
1093 }
1094
1095 if (cpufreq_boost_supported()) {
1096 ret = sysfs_create_file(&policy->kobj, &local_boost.attr);
1097 if (ret)
1098 return ret;
1099 }
1100
1101 return 0;
1102}
1103
1104static int cpufreq_init_policy(struct cpufreq_policy *policy)
1105{
1106 struct cpufreq_governor *gov = NULL;
1107 unsigned int pol = CPUFREQ_POLICY_UNKNOWN;
1108 int ret;
1109
1110 if (has_target()) {
1111 /* Update policy governor to the one used before hotplug. */
1112 if (policy->last_governor[0] != '\0')
1113 gov = get_governor(policy->last_governor);
1114 if (gov) {
1115 pr_debug("Restoring governor %s for cpu %d\n",
1116 gov->name, policy->cpu);
1117 } else {
1118 gov = get_governor(default_governor);
1119 }
1120
1121 if (!gov) {
1122 gov = cpufreq_default_governor();
1123 __module_get(gov->owner);
1124 }
1125
1126 } else {
1127
1128 /* Use the default policy if there is no last_policy. */
1129 if (policy->last_policy) {
1130 pol = policy->last_policy;
1131 } else {
1132 pol = cpufreq_parse_policy(default_governor);
1133 /*
1134 * In case the default governor is neither "performance"
1135 * nor "powersave", fall back to the initial policy
1136 * value set by the driver.
1137 */
1138 if (pol == CPUFREQ_POLICY_UNKNOWN)
1139 pol = policy->policy;
1140 }
1141 if (pol != CPUFREQ_POLICY_PERFORMANCE &&
1142 pol != CPUFREQ_POLICY_POWERSAVE)
1143 return -ENODATA;
1144 }
1145
1146 ret = cpufreq_set_policy(policy, gov, pol);
1147 if (gov)
1148 module_put(gov->owner);
1149
1150 return ret;
1151}
1152
1153static int cpufreq_add_policy_cpu(struct cpufreq_policy *policy, unsigned int cpu)
1154{
1155 int ret = 0;
1156
1157 /* Has this CPU been taken care of already? */
1158 if (cpumask_test_cpu(cpu, policy->cpus))
1159 return 0;
1160
1161 guard(cpufreq_policy_write)(policy);
1162
1163 if (has_target())
1164 cpufreq_stop_governor(policy);
1165
1166 cpumask_set_cpu(cpu, policy->cpus);
1167
1168 if (has_target()) {
1169 ret = cpufreq_start_governor(policy);
1170 if (ret)
1171 pr_err("%s: Failed to start governor\n", __func__);
1172 }
1173
1174 return ret;
1175}
1176
1177void refresh_frequency_limits(struct cpufreq_policy *policy)
1178{
1179 if (!policy_is_inactive(policy)) {
1180 pr_debug("updating policy for CPU %u\n", policy->cpu);
1181
1182 cpufreq_set_policy(policy, policy->governor, policy->policy);
1183 }
1184}
1185EXPORT_SYMBOL(refresh_frequency_limits);
1186
1187static void handle_update(struct work_struct *work)
1188{
1189 struct cpufreq_policy *policy =
1190 container_of(work, struct cpufreq_policy, update);
1191
1192 pr_debug("handle_update for cpu %u called\n", policy->cpu);
1193
1194 guard(cpufreq_policy_write)(policy);
1195
1196 refresh_frequency_limits(policy);
1197}
1198
1199static int cpufreq_notifier_min(struct notifier_block *nb, unsigned long freq,
1200 void *data)
1201{
1202 struct cpufreq_policy *policy = container_of(nb, struct cpufreq_policy, nb_min);
1203
1204 schedule_work(&policy->update);
1205 return 0;
1206}
1207
1208static int cpufreq_notifier_max(struct notifier_block *nb, unsigned long freq,
1209 void *data)
1210{
1211 struct cpufreq_policy *policy = container_of(nb, struct cpufreq_policy, nb_max);
1212
1213 schedule_work(&policy->update);
1214 return 0;
1215}
1216
1217static void cpufreq_policy_put_kobj(struct cpufreq_policy *policy)
1218{
1219 struct kobject *kobj;
1220 struct completion *cmp;
1221
1222 scoped_guard(cpufreq_policy_write, policy) {
1223 cpufreq_stats_free_table(policy);
1224 kobj = &policy->kobj;
1225 cmp = &policy->kobj_unregister;
1226 }
1227 kobject_put(kobj);
1228
1229 /*
1230 * We need to make sure that the underlying kobj is
1231 * actually not referenced anymore by anybody before we
1232 * proceed with unloading.
1233 */
1234 pr_debug("waiting for dropping of refcount\n");
1235 wait_for_completion(cmp);
1236 pr_debug("wait complete\n");
1237}
1238
1239static struct cpufreq_policy *cpufreq_policy_alloc(unsigned int cpu)
1240{
1241 struct cpufreq_policy *policy;
1242 struct device *dev = get_cpu_device(cpu);
1243 int ret;
1244
1245 if (!dev)
1246 return NULL;
1247
1248 policy = kzalloc_obj(*policy);
1249 if (!policy)
1250 return NULL;
1251
1252 if (!zalloc_cpumask_var(&policy->cpus, GFP_KERNEL))
1253 goto err_free_policy;
1254
1255 if (!zalloc_cpumask_var(&policy->related_cpus, GFP_KERNEL))
1256 goto err_free_cpumask;
1257
1258 if (!zalloc_cpumask_var(&policy->real_cpus, GFP_KERNEL))
1259 goto err_free_rcpumask;
1260
1261 init_rwsem(&policy->rwsem);
1262
1263 init_completion(&policy->kobj_unregister);
1264 ret = kobject_init_and_add(&policy->kobj, &ktype_cpufreq,
1265 cpufreq_global_kobject, "policy%u", cpu);
1266 if (ret) {
1267 dev_err(dev, "%s: failed to init policy->kobj: %d\n", __func__, ret);
1268 /*
1269 * The entire policy object will be freed below, but the extra
1270 * memory allocated for the kobject name needs to be freed by
1271 * releasing the kobject.
1272 */
1273 kobject_put(&policy->kobj);
1274 goto err_free_real_cpus;
1275 }
1276
1277 freq_constraints_init(&policy->constraints);
1278
1279 policy->nb_min.notifier_call = cpufreq_notifier_min;
1280 policy->nb_max.notifier_call = cpufreq_notifier_max;
1281
1282 ret = freq_qos_add_notifier(&policy->constraints, FREQ_QOS_MIN,
1283 &policy->nb_min);
1284 if (ret) {
1285 dev_err(dev, "Failed to register MIN QoS notifier: %d (CPU%u)\n",
1286 ret, cpu);
1287 goto err_kobj_remove;
1288 }
1289
1290 ret = freq_qos_add_notifier(&policy->constraints, FREQ_QOS_MAX,
1291 &policy->nb_max);
1292 if (ret) {
1293 dev_err(dev, "Failed to register MAX QoS notifier: %d (CPU%u)\n",
1294 ret, cpu);
1295 goto err_min_qos_notifier;
1296 }
1297
1298 INIT_LIST_HEAD(&policy->policy_list);
1299 spin_lock_init(&policy->transition_lock);
1300 init_waitqueue_head(&policy->transition_wait);
1301 INIT_WORK(&policy->update, handle_update);
1302
1303 return policy;
1304
1305err_min_qos_notifier:
1306 freq_qos_remove_notifier(&policy->constraints, FREQ_QOS_MIN,
1307 &policy->nb_min);
1308err_kobj_remove:
1309 cpufreq_policy_put_kobj(policy);
1310err_free_real_cpus:
1311 free_cpumask_var(policy->real_cpus);
1312err_free_rcpumask:
1313 free_cpumask_var(policy->related_cpus);
1314err_free_cpumask:
1315 free_cpumask_var(policy->cpus);
1316err_free_policy:
1317 kfree(policy);
1318
1319 return NULL;
1320}
1321
1322static void cpufreq_policy_free(struct cpufreq_policy *policy)
1323{
1324 unsigned long flags;
1325 int cpu;
1326
1327 /*
1328 * The callers must ensure the policy is inactive by now, to avoid any
1329 * races with show()/store() callbacks.
1330 */
1331 if (unlikely(!policy_is_inactive(policy)))
1332 pr_warn("%s: Freeing active policy\n", __func__);
1333
1334 /* Remove policy from list */
1335 write_lock_irqsave(&cpufreq_driver_lock, flags);
1336 list_del(&policy->policy_list);
1337
1338 for_each_cpu(cpu, policy->related_cpus)
1339 per_cpu(cpufreq_cpu_data, cpu) = NULL;
1340 write_unlock_irqrestore(&cpufreq_driver_lock, flags);
1341
1342 freq_qos_remove_notifier(&policy->constraints, FREQ_QOS_MAX,
1343 &policy->nb_max);
1344 freq_qos_remove_notifier(&policy->constraints, FREQ_QOS_MIN,
1345 &policy->nb_min);
1346
1347 /* Cancel any pending policy->update work before freeing the policy. */
1348 cancel_work_sync(&policy->update);
1349
1350 if (freq_qos_request_active(&policy->max_freq_req)) {
1351 /*
1352 * Remove max_freq_req after sending CPUFREQ_REMOVE_POLICY
1353 * notification, since CPUFREQ_CREATE_POLICY notification was
1354 * sent after adding max_freq_req earlier.
1355 */
1356 blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1357 CPUFREQ_REMOVE_POLICY, policy);
1358 freq_qos_remove_request(&policy->max_freq_req);
1359 }
1360
1361 if (freq_qos_request_active(&policy->min_freq_req))
1362 freq_qos_remove_request(&policy->min_freq_req);
1363 if (freq_qos_request_active(&policy->boost_freq_req))
1364 freq_qos_remove_request(&policy->boost_freq_req);
1365
1366 cpufreq_policy_put_kobj(policy);
1367 free_cpumask_var(policy->real_cpus);
1368 free_cpumask_var(policy->related_cpus);
1369 free_cpumask_var(policy->cpus);
1370 kfree(policy);
1371}
1372
1373static int cpufreq_policy_init_qos(struct cpufreq_policy *policy)
1374{
1375 unsigned int min_freq, max_freq;
1376 int ret;
1377
1378 /* Use policy->min/max set by the driver as QoS requests. */
1379 min_freq = max(FREQ_QOS_MIN_DEFAULT_VALUE, policy->min);
1380 if (policy->max)
1381 max_freq = min(FREQ_QOS_MAX_DEFAULT_VALUE, policy->max);
1382 else
1383 max_freq = FREQ_QOS_MAX_DEFAULT_VALUE;
1384
1385 if (policy->boost_supported) {
1386 ret = freq_qos_add_request(&policy->constraints,
1387 &policy->boost_freq_req,
1388 FREQ_QOS_MAX,
1389 policy->cpuinfo.max_freq);
1390 if (ret < 0)
1391 return ret;
1392 }
1393
1394 ret = freq_qos_add_request(&policy->constraints, &policy->min_freq_req,
1395 FREQ_QOS_MIN, min_freq);
1396 if (ret < 0)
1397 return ret;
1398
1399 ret = freq_qos_add_request(&policy->constraints, &policy->max_freq_req,
1400 FREQ_QOS_MAX, max_freq);
1401 if (ret < 0)
1402 return ret;
1403
1404 return 0;
1405}
1406
1407static int cpufreq_policy_online(struct cpufreq_policy *policy,
1408 unsigned int cpu, bool new_policy)
1409{
1410 unsigned long flags;
1411 unsigned int j;
1412 int ret;
1413
1414 guard(cpufreq_policy_write)(policy);
1415
1416 policy->cpu = cpu;
1417 policy->governor = NULL;
1418
1419 if (!new_policy && cpufreq_driver->online) {
1420 /* Recover policy->cpus using related_cpus */
1421 cpumask_copy(policy->cpus, policy->related_cpus);
1422
1423 ret = cpufreq_driver->online(policy);
1424 if (ret) {
1425 pr_debug("%s: %d: initialization failed\n", __func__,
1426 __LINE__);
1427 goto out_exit_policy;
1428 }
1429 } else {
1430 cpumask_copy(policy->cpus, cpumask_of(cpu));
1431
1432 /*
1433 * Call driver. From then on the cpufreq must be able
1434 * to accept all calls to ->verify and ->setpolicy for this CPU.
1435 */
1436 ret = cpufreq_driver->init(policy);
1437 if (ret) {
1438 pr_debug("%s: %d: initialization failed\n", __func__,
1439 __LINE__);
1440 goto out_clear_policy;
1441 }
1442
1443 /*
1444 * The initialization has succeeded and the policy is online.
1445 * If there is a problem with its frequency table, take it
1446 * offline and drop it.
1447 */
1448 ret = cpufreq_table_validate_and_sort(policy);
1449 if (ret)
1450 goto out_offline_policy;
1451
1452 if (new_policy) {
1453 ret = cpufreq_policy_init_qos(policy);
1454 if (ret < 0)
1455 goto out_offline_policy;
1456 }
1457
1458 /*
1459 * If the driver hasn't set policy->min/max, set them as they
1460 * are used for clamping frequency requests.
1461 */
1462 policy->min = policy->min ? policy->min : policy->cpuinfo.min_freq;
1463 policy->max = policy->max ? policy->max : policy->cpuinfo.max_freq;
1464
1465 /* related_cpus should at least include policy->cpus. */
1466 cpumask_copy(policy->related_cpus, policy->cpus);
1467 }
1468
1469 /*
1470 * affected cpus must always be the one, which are online. We aren't
1471 * managing offline cpus here.
1472 */
1473 cpumask_and(policy->cpus, policy->cpus, cpu_online_mask);
1474
1475 if (new_policy) {
1476 for_each_cpu(j, policy->related_cpus) {
1477 per_cpu(cpufreq_cpu_data, j) = policy;
1478 add_cpu_dev_symlink(policy, j, get_cpu_device(j));
1479 }
1480
1481 blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1482 CPUFREQ_CREATE_POLICY, policy);
1483 }
1484
1485 if (cpufreq_driver->get && has_target()) {
1486 policy->cur = cpufreq_driver->get(policy->cpu);
1487 if (!policy->cur) {
1488 ret = -EIO;
1489 pr_err("%s: ->get() failed\n", __func__);
1490 goto out_destroy_policy;
1491 }
1492 }
1493
1494 /*
1495 * Sometimes boot loaders set CPU frequency to a value outside of
1496 * frequency table present with cpufreq core. In such cases CPU might be
1497 * unstable if it has to run on that frequency for long duration of time
1498 * and so its better to set it to a frequency which is specified in
1499 * freq-table. This also makes cpufreq stats inconsistent as
1500 * cpufreq-stats would fail to register because current frequency of CPU
1501 * isn't found in freq-table.
1502 *
1503 * Because we don't want this change to effect boot process badly, we go
1504 * for the next freq which is >= policy->cur ('cur' must be set by now,
1505 * otherwise we will end up setting freq to lowest of the table as 'cur'
1506 * is initialized to zero).
1507 *
1508 * We are passing target-freq as "policy->cur - 1" otherwise
1509 * __cpufreq_driver_target() would simply fail, as policy->cur will be
1510 * equal to target-freq.
1511 */
1512 if ((cpufreq_driver->flags & CPUFREQ_NEED_INITIAL_FREQ_CHECK)
1513 && has_target()) {
1514 unsigned int old_freq = policy->cur;
1515
1516 /* Are we running at unknown frequency ? */
1517 ret = cpufreq_frequency_table_get_index(policy, old_freq);
1518 if (ret == -EINVAL) {
1519 ret = __cpufreq_driver_target(policy, old_freq - 1,
1520 CPUFREQ_RELATION_L);
1521
1522 /*
1523 * Reaching here after boot in a few seconds may not
1524 * mean that system will remain stable at "unknown"
1525 * frequency for longer duration. Hence, a BUG_ON().
1526 */
1527 BUG_ON(ret);
1528 pr_info("%s: CPU%d: Running at unlisted initial frequency: %u kHz, changing to: %u kHz\n",
1529 __func__, policy->cpu, old_freq, policy->cur);
1530 }
1531 }
1532
1533 if (new_policy) {
1534 ret = cpufreq_add_dev_interface(policy);
1535 if (ret)
1536 goto out_destroy_policy;
1537
1538 cpufreq_stats_create_table(policy);
1539
1540 write_lock_irqsave(&cpufreq_driver_lock, flags);
1541 list_add(&policy->policy_list, &cpufreq_policy_list);
1542 write_unlock_irqrestore(&cpufreq_driver_lock, flags);
1543
1544 /*
1545 * Register with the energy model before
1546 * em_rebuild_sched_domains() is called, which will result
1547 * in rebuilding of the sched domains, which should only be done
1548 * once the energy model is properly initialized for the policy
1549 * first.
1550 *
1551 * Also, this should be called before the policy is registered
1552 * with cooling framework.
1553 */
1554 if (cpufreq_driver->register_em)
1555 cpufreq_driver->register_em(policy);
1556 }
1557
1558 ret = cpufreq_init_policy(policy);
1559 if (ret) {
1560 pr_err("%s: Failed to initialize policy for cpu: %d (%d)\n",
1561 __func__, cpu, ret);
1562 goto out_destroy_policy;
1563 }
1564
1565 return 0;
1566
1567out_destroy_policy:
1568 for_each_cpu(j, policy->real_cpus)
1569 remove_cpu_dev_symlink(policy, j, get_cpu_device(j));
1570
1571out_offline_policy:
1572 if (cpufreq_driver->offline)
1573 cpufreq_driver->offline(policy);
1574
1575out_exit_policy:
1576 if (cpufreq_driver->exit)
1577 cpufreq_driver->exit(policy);
1578
1579out_clear_policy:
1580 cpumask_clear(policy->cpus);
1581
1582 return ret;
1583}
1584
1585static int cpufreq_online(unsigned int cpu)
1586{
1587 struct cpufreq_policy *policy;
1588 bool new_policy;
1589 int ret;
1590
1591 pr_debug("%s: bringing CPU%u online\n", __func__, cpu);
1592
1593 /* Check if this CPU already has a policy to manage it */
1594 policy = per_cpu(cpufreq_cpu_data, cpu);
1595 if (policy) {
1596 WARN_ON(!cpumask_test_cpu(cpu, policy->related_cpus));
1597 if (!policy_is_inactive(policy))
1598 return cpufreq_add_policy_cpu(policy, cpu);
1599
1600 /* This is the only online CPU for the policy. Start over. */
1601 new_policy = false;
1602 } else {
1603 new_policy = true;
1604 policy = cpufreq_policy_alloc(cpu);
1605 if (!policy)
1606 return -ENOMEM;
1607 }
1608
1609 ret = cpufreq_policy_online(policy, cpu, new_policy);
1610 if (ret) {
1611 cpufreq_policy_free(policy);
1612 return ret;
1613 }
1614
1615 kobject_uevent(&policy->kobj, KOBJ_ADD);
1616
1617 /* Callback for handling stuff after policy is ready */
1618 if (cpufreq_driver->ready)
1619 cpufreq_driver->ready(policy);
1620
1621 /* Register cpufreq cooling only for a new policy */
1622 if (new_policy && cpufreq_thermal_control_enabled(cpufreq_driver))
1623 policy->cdev = of_cpufreq_cooling_register(policy);
1624
1625 /*
1626 * Let the per-policy boost flag mirror the cpufreq_driver boost during
1627 * initialization for a new policy. For an existing policy, maintain the
1628 * previous boost value unless global boost is disabled.
1629 */
1630 if (cpufreq_driver->set_boost && policy->boost_supported &&
1631 (new_policy || !cpufreq_boost_enabled())) {
1632 ret = policy_set_boost(policy, cpufreq_boost_enabled());
1633 if (ret) {
1634 /* If the set_boost fails, the online operation is not affected */
1635 pr_info("%s: CPU%d: Cannot %s BOOST\n", __func__, policy->cpu,
1636 str_enable_disable(cpufreq_boost_enabled()));
1637 }
1638 }
1639
1640 pr_debug("initialization complete\n");
1641
1642 return 0;
1643}
1644
1645/**
1646 * cpufreq_add_dev - the cpufreq interface for a CPU device.
1647 * @dev: CPU device.
1648 * @sif: Subsystem interface structure pointer (not used)
1649 */
1650static int cpufreq_add_dev(struct device *dev, struct subsys_interface *sif)
1651{
1652 struct cpufreq_policy *policy;
1653 unsigned cpu = dev->id;
1654 int ret;
1655
1656 dev_dbg(dev, "%s: adding CPU%u\n", __func__, cpu);
1657
1658 if (cpu_online(cpu)) {
1659 ret = cpufreq_online(cpu);
1660 if (ret)
1661 return ret;
1662 }
1663
1664 /* Create sysfs link on CPU registration */
1665 policy = per_cpu(cpufreq_cpu_data, cpu);
1666 if (policy)
1667 add_cpu_dev_symlink(policy, cpu, dev);
1668
1669 return 0;
1670}
1671
1672static void __cpufreq_offline(unsigned int cpu, struct cpufreq_policy *policy)
1673{
1674 int ret;
1675
1676 if (has_target())
1677 cpufreq_stop_governor(policy);
1678
1679 cpumask_clear_cpu(cpu, policy->cpus);
1680
1681 if (!policy_is_inactive(policy)) {
1682 /* Nominate a new CPU if necessary. */
1683 if (cpu == policy->cpu)
1684 policy->cpu = cpumask_any(policy->cpus);
1685
1686 /* Start the governor again for the active policy. */
1687 if (has_target()) {
1688 ret = cpufreq_start_governor(policy);
1689 if (ret)
1690 pr_err("%s: Failed to start governor\n", __func__);
1691 }
1692
1693 return;
1694 }
1695
1696 if (has_target()) {
1697 strscpy(policy->last_governor, policy->governor->name,
1698 CPUFREQ_NAME_LEN);
1699 cpufreq_exit_governor(policy);
1700 } else {
1701 policy->last_policy = policy->policy;
1702 }
1703
1704 /*
1705 * Perform the ->offline() during light-weight tear-down, as
1706 * that allows fast recovery when the CPU comes back.
1707 */
1708 if (cpufreq_driver->offline) {
1709 cpufreq_driver->offline(policy);
1710 return;
1711 }
1712
1713 if (cpufreq_driver->exit)
1714 cpufreq_driver->exit(policy);
1715
1716 policy->freq_table = NULL;
1717}
1718
1719static int cpufreq_offline(unsigned int cpu)
1720{
1721 struct cpufreq_policy *policy;
1722
1723 pr_debug("%s: unregistering CPU %u\n", __func__, cpu);
1724
1725 policy = cpufreq_cpu_get_raw(cpu);
1726 if (!policy) {
1727 pr_debug("%s: No cpu_data found\n", __func__);
1728 return 0;
1729 }
1730
1731 guard(cpufreq_policy_write)(policy);
1732
1733 __cpufreq_offline(cpu, policy);
1734
1735 return 0;
1736}
1737
1738/*
1739 * cpufreq_remove_dev - remove a CPU device
1740 *
1741 * Removes the cpufreq interface for a CPU device.
1742 */
1743static void cpufreq_remove_dev(struct device *dev, struct subsys_interface *sif)
1744{
1745 unsigned int cpu = dev->id;
1746 struct cpufreq_policy *policy = per_cpu(cpufreq_cpu_data, cpu);
1747
1748 if (!policy)
1749 return;
1750
1751 scoped_guard(cpufreq_policy_write, policy) {
1752 if (cpu_online(cpu))
1753 __cpufreq_offline(cpu, policy);
1754
1755 remove_cpu_dev_symlink(policy, cpu, dev);
1756
1757 if (!cpumask_empty(policy->real_cpus))
1758 return;
1759
1760 /*
1761 * Unregister cpufreq cooling once all the CPUs of the policy
1762 * are removed.
1763 */
1764 if (cpufreq_thermal_control_enabled(cpufreq_driver)) {
1765 cpufreq_cooling_unregister(policy->cdev);
1766 policy->cdev = NULL;
1767 }
1768
1769 /* We did light-weight exit earlier, do full tear down now */
1770 if (cpufreq_driver->offline && cpufreq_driver->exit)
1771 cpufreq_driver->exit(policy);
1772 }
1773
1774 cpufreq_policy_free(policy);
1775}
1776
1777/**
1778 * cpufreq_out_of_sync - Fix up actual and saved CPU frequency difference.
1779 * @policy: Policy managing CPUs.
1780 * @new_freq: New CPU frequency.
1781 *
1782 * Adjust to the current frequency first and clean up later by either calling
1783 * cpufreq_update_policy(), or scheduling handle_update().
1784 */
1785static void cpufreq_out_of_sync(struct cpufreq_policy *policy,
1786 unsigned int new_freq)
1787{
1788 struct cpufreq_freqs freqs;
1789
1790 pr_debug("Warning: CPU frequency out of sync: cpufreq and timing core thinks of %u, is %u kHz\n",
1791 policy->cur, new_freq);
1792
1793 freqs.old = policy->cur;
1794 freqs.new = new_freq;
1795
1796 cpufreq_freq_transition_begin(policy, &freqs);
1797 cpufreq_freq_transition_end(policy, &freqs, 0);
1798}
1799
1800static unsigned int cpufreq_verify_current_freq(struct cpufreq_policy *policy, bool update)
1801{
1802 unsigned int new_freq;
1803
1804 if (!cpufreq_driver->get)
1805 return 0;
1806
1807 new_freq = cpufreq_driver->get(policy->cpu);
1808 if (!new_freq)
1809 return 0;
1810
1811 /*
1812 * If fast frequency switching is used with the given policy, the check
1813 * against policy->cur is pointless, so skip it in that case.
1814 */
1815 if (policy->fast_switch_enabled || !has_target())
1816 return new_freq;
1817
1818 if (policy->cur != new_freq) {
1819 /*
1820 * For some platforms, the frequency returned by hardware may be
1821 * slightly different from what is provided in the frequency
1822 * table, for example hardware may return 499 MHz instead of 500
1823 * MHz. In such cases it is better to avoid getting into
1824 * unnecessary frequency updates.
1825 */
1826 if (abs(policy->cur - new_freq) < KHZ_PER_MHZ)
1827 return policy->cur;
1828
1829 cpufreq_out_of_sync(policy, new_freq);
1830 if (update)
1831 schedule_work(&policy->update);
1832 }
1833
1834 return new_freq;
1835}
1836
1837/**
1838 * cpufreq_quick_get - get the CPU frequency (in kHz) from policy->cur
1839 * @cpu: CPU number
1840 *
1841 * This is the last known freq, without actually getting it from the driver.
1842 * Return value will be same as what is shown in scaling_cur_freq in sysfs.
1843 */
1844unsigned int cpufreq_quick_get(unsigned int cpu)
1845{
1846 unsigned long flags;
1847
1848 read_lock_irqsave(&cpufreq_driver_lock, flags);
1849
1850 if (cpufreq_driver && cpufreq_driver->setpolicy && cpufreq_driver->get) {
1851 unsigned int ret_freq = cpufreq_driver->get(cpu);
1852
1853 read_unlock_irqrestore(&cpufreq_driver_lock, flags);
1854
1855 return ret_freq;
1856 }
1857
1858 read_unlock_irqrestore(&cpufreq_driver_lock, flags);
1859
1860 struct cpufreq_policy *policy __free(put_cpufreq_policy) = cpufreq_cpu_get(cpu);
1861 if (policy)
1862 return policy->cur;
1863
1864 return 0;
1865}
1866EXPORT_SYMBOL(cpufreq_quick_get);
1867
1868/**
1869 * cpufreq_quick_get_max - get the max reported CPU frequency for this CPU
1870 * @cpu: CPU number
1871 *
1872 * Just return the max possible frequency for a given CPU.
1873 */
1874unsigned int cpufreq_quick_get_max(unsigned int cpu)
1875{
1876 struct cpufreq_policy *policy __free(put_cpufreq_policy) = cpufreq_cpu_get(cpu);
1877 if (policy)
1878 return policy->max;
1879
1880 return 0;
1881}
1882EXPORT_SYMBOL(cpufreq_quick_get_max);
1883
1884/**
1885 * cpufreq_get_hw_max_freq - get the max hardware frequency of the CPU
1886 * @cpu: CPU number
1887 *
1888 * The default return value is the max_freq field of cpuinfo.
1889 */
1890__weak unsigned int cpufreq_get_hw_max_freq(unsigned int cpu)
1891{
1892 struct cpufreq_policy *policy __free(put_cpufreq_policy) = cpufreq_cpu_get(cpu);
1893 if (policy)
1894 return policy->cpuinfo.max_freq;
1895
1896 return 0;
1897}
1898EXPORT_SYMBOL(cpufreq_get_hw_max_freq);
1899
1900static unsigned int __cpufreq_get(struct cpufreq_policy *policy)
1901{
1902 if (unlikely(policy_is_inactive(policy)))
1903 return 0;
1904
1905 return cpufreq_verify_current_freq(policy, true);
1906}
1907
1908/**
1909 * cpufreq_get - get the current CPU frequency (in kHz)
1910 * @cpu: CPU number
1911 *
1912 * Get the CPU current (static) CPU frequency
1913 */
1914unsigned int cpufreq_get(unsigned int cpu)
1915{
1916 struct cpufreq_policy *policy __free(put_cpufreq_policy) = cpufreq_cpu_get(cpu);
1917 if (!policy)
1918 return 0;
1919
1920 guard(cpufreq_policy_read)(policy);
1921
1922 return __cpufreq_get(policy);
1923}
1924EXPORT_SYMBOL(cpufreq_get);
1925
1926static struct subsys_interface cpufreq_interface = {
1927 .name = "cpufreq",
1928 .subsys = &cpu_subsys,
1929 .add_dev = cpufreq_add_dev,
1930 .remove_dev = cpufreq_remove_dev,
1931};
1932
1933/*
1934 * In case platform wants some specific frequency to be configured
1935 * during suspend..
1936 */
1937int cpufreq_generic_suspend(struct cpufreq_policy *policy)
1938{
1939 int ret;
1940
1941 if (!policy->suspend_freq) {
1942 pr_debug("%s: suspend_freq not defined\n", __func__);
1943 return 0;
1944 }
1945
1946 pr_debug("%s: Setting suspend-freq: %u\n", __func__,
1947 policy->suspend_freq);
1948
1949 ret = __cpufreq_driver_target(policy, policy->suspend_freq,
1950 CPUFREQ_RELATION_H);
1951 if (ret)
1952 pr_err("%s: unable to set suspend-freq: %u. err: %d\n",
1953 __func__, policy->suspend_freq, ret);
1954
1955 return ret;
1956}
1957EXPORT_SYMBOL(cpufreq_generic_suspend);
1958
1959/**
1960 * cpufreq_suspend() - Suspend CPUFreq governors.
1961 *
1962 * Called during system wide Suspend/Hibernate cycles for suspending governors
1963 * as some platforms can't change frequency after this point in suspend cycle.
1964 * Because some of the devices (like: i2c, regulators, etc) they use for
1965 * changing frequency are suspended quickly after this point.
1966 */
1967void cpufreq_suspend(void)
1968{
1969 struct cpufreq_policy *policy;
1970
1971 if (!cpufreq_driver)
1972 return;
1973
1974 cpus_read_lock();
1975 if (!has_target() && !cpufreq_driver->suspend)
1976 goto suspend;
1977
1978 pr_debug("%s: Suspending Governors\n", __func__);
1979
1980 for_each_active_policy(policy) {
1981 if (has_target()) {
1982 scoped_guard(cpufreq_policy_write, policy) {
1983 cpufreq_stop_governor(policy);
1984 }
1985 }
1986
1987 if (cpufreq_driver->suspend && cpufreq_driver->suspend(policy))
1988 pr_err("%s: Failed to suspend driver: %s\n", __func__,
1989 cpufreq_driver->name);
1990 }
1991
1992suspend:
1993 cpufreq_suspended = true;
1994 cpus_read_unlock();
1995}
1996
1997/**
1998 * cpufreq_resume() - Resume CPUFreq governors.
1999 *
2000 * Called during system wide Suspend/Hibernate cycle for resuming governors that
2001 * are suspended with cpufreq_suspend().
2002 */
2003void cpufreq_resume(void)
2004{
2005 struct cpufreq_policy *policy;
2006 int ret;
2007
2008 if (!cpufreq_driver)
2009 return;
2010
2011 if (unlikely(!cpufreq_suspended))
2012 return;
2013
2014 cpufreq_suspended = false;
2015
2016 if (!has_target() && !cpufreq_driver->resume)
2017 return;
2018
2019 pr_debug("%s: Resuming Governors\n", __func__);
2020
2021 for_each_active_policy(policy) {
2022 if (cpufreq_driver->resume && cpufreq_driver->resume(policy)) {
2023 pr_err("%s: Failed to resume driver: %s\n", __func__,
2024 cpufreq_driver->name);
2025 } else if (has_target()) {
2026 scoped_guard(cpufreq_policy_write, policy) {
2027 ret = cpufreq_start_governor(policy);
2028 }
2029
2030 if (ret)
2031 pr_err("%s: Failed to start governor for CPU%u's policy\n",
2032 __func__, policy->cpu);
2033 }
2034 }
2035}
2036
2037/**
2038 * cpufreq_driver_test_flags - Test cpufreq driver's flags against given ones.
2039 * @flags: Flags to test against the current cpufreq driver's flags.
2040 *
2041 * Assumes that the driver is there, so callers must ensure that this is the
2042 * case.
2043 */
2044bool cpufreq_driver_test_flags(u16 flags)
2045{
2046 return !!(cpufreq_driver->flags & flags);
2047}
2048
2049/**
2050 * cpufreq_get_current_driver - Return the current driver's name.
2051 *
2052 * Return the name string of the currently registered cpufreq driver or NULL if
2053 * none.
2054 */
2055const char *cpufreq_get_current_driver(void)
2056{
2057 if (cpufreq_driver)
2058 return cpufreq_driver->name;
2059
2060 return NULL;
2061}
2062EXPORT_SYMBOL_GPL(cpufreq_get_current_driver);
2063
2064/**
2065 * cpufreq_get_driver_data - Return current driver data.
2066 *
2067 * Return the private data of the currently registered cpufreq driver, or NULL
2068 * if no cpufreq driver has been registered.
2069 */
2070void *cpufreq_get_driver_data(void)
2071{
2072 if (cpufreq_driver)
2073 return cpufreq_driver->driver_data;
2074
2075 return NULL;
2076}
2077EXPORT_SYMBOL_GPL(cpufreq_get_driver_data);
2078
2079/*********************************************************************
2080 * NOTIFIER LISTS INTERFACE *
2081 *********************************************************************/
2082
2083/**
2084 * cpufreq_register_notifier - Register a notifier with cpufreq.
2085 * @nb: notifier function to register.
2086 * @list: CPUFREQ_TRANSITION_NOTIFIER or CPUFREQ_POLICY_NOTIFIER.
2087 *
2088 * Add a notifier to one of two lists: either a list of notifiers that run on
2089 * clock rate changes (once before and once after every transition), or a list
2090 * of notifiers that ron on cpufreq policy changes.
2091 *
2092 * This function may sleep and it has the same return values as
2093 * blocking_notifier_chain_register().
2094 */
2095int cpufreq_register_notifier(struct notifier_block *nb, unsigned int list)
2096{
2097 int ret;
2098
2099 if (cpufreq_disabled())
2100 return -EINVAL;
2101
2102 switch (list) {
2103 case CPUFREQ_TRANSITION_NOTIFIER:
2104 mutex_lock(&cpufreq_fast_switch_lock);
2105
2106 if (cpufreq_fast_switch_count > 0) {
2107 mutex_unlock(&cpufreq_fast_switch_lock);
2108 return -EBUSY;
2109 }
2110 ret = srcu_notifier_chain_register(
2111 &cpufreq_transition_notifier_list, nb);
2112 if (!ret)
2113 cpufreq_fast_switch_count--;
2114
2115 mutex_unlock(&cpufreq_fast_switch_lock);
2116 break;
2117 case CPUFREQ_POLICY_NOTIFIER:
2118 ret = blocking_notifier_chain_register(
2119 &cpufreq_policy_notifier_list, nb);
2120 break;
2121 default:
2122 ret = -EINVAL;
2123 }
2124
2125 return ret;
2126}
2127EXPORT_SYMBOL(cpufreq_register_notifier);
2128
2129/**
2130 * cpufreq_unregister_notifier - Unregister a notifier from cpufreq.
2131 * @nb: notifier block to be unregistered.
2132 * @list: CPUFREQ_TRANSITION_NOTIFIER or CPUFREQ_POLICY_NOTIFIER.
2133 *
2134 * Remove a notifier from one of the cpufreq notifier lists.
2135 *
2136 * This function may sleep and it has the same return values as
2137 * blocking_notifier_chain_unregister().
2138 */
2139int cpufreq_unregister_notifier(struct notifier_block *nb, unsigned int list)
2140{
2141 int ret;
2142
2143 if (cpufreq_disabled())
2144 return -EINVAL;
2145
2146 switch (list) {
2147 case CPUFREQ_TRANSITION_NOTIFIER:
2148 mutex_lock(&cpufreq_fast_switch_lock);
2149
2150 ret = srcu_notifier_chain_unregister(
2151 &cpufreq_transition_notifier_list, nb);
2152 if (!ret && !WARN_ON(cpufreq_fast_switch_count >= 0))
2153 cpufreq_fast_switch_count++;
2154
2155 mutex_unlock(&cpufreq_fast_switch_lock);
2156 break;
2157 case CPUFREQ_POLICY_NOTIFIER:
2158 ret = blocking_notifier_chain_unregister(
2159 &cpufreq_policy_notifier_list, nb);
2160 break;
2161 default:
2162 ret = -EINVAL;
2163 }
2164
2165 return ret;
2166}
2167EXPORT_SYMBOL(cpufreq_unregister_notifier);
2168
2169
2170/*********************************************************************
2171 * GOVERNORS *
2172 *********************************************************************/
2173
2174/**
2175 * cpufreq_driver_fast_switch - Carry out a fast CPU frequency switch.
2176 * @policy: cpufreq policy to switch the frequency for.
2177 * @target_freq: New frequency to set (may be approximate).
2178 *
2179 * Carry out a fast frequency switch without sleeping.
2180 *
2181 * The driver's ->fast_switch() callback invoked by this function must be
2182 * suitable for being called from within RCU-sched read-side critical sections
2183 * and it is expected to select the minimum available frequency greater than or
2184 * equal to @target_freq (CPUFREQ_RELATION_L).
2185 *
2186 * This function must not be called if policy->fast_switch_enabled is unset.
2187 *
2188 * Governors calling this function must guarantee that it will never be invoked
2189 * twice in parallel for the same policy and that it will never be called in
2190 * parallel with either ->target() or ->target_index() for the same policy.
2191 *
2192 * Returns the actual frequency set for the CPU.
2193 *
2194 * If 0 is returned by the driver's ->fast_switch() callback to indicate an
2195 * error condition, the hardware configuration must be preserved.
2196 */
2197unsigned int cpufreq_driver_fast_switch(struct cpufreq_policy *policy,
2198 unsigned int target_freq)
2199{
2200 unsigned int freq;
2201 int cpu;
2202
2203 target_freq = clamp_val(target_freq, policy->min, policy->max);
2204 freq = cpufreq_driver->fast_switch(policy, target_freq);
2205
2206 if (!freq)
2207 return 0;
2208
2209 policy->cur = freq;
2210 arch_set_freq_scale(policy->related_cpus, freq,
2211 arch_scale_freq_ref(policy->cpu));
2212 cpufreq_stats_record_transition(policy, freq);
2213
2214 if (trace_cpu_frequency_enabled()) {
2215 for_each_cpu(cpu, policy->cpus)
2216 trace_call__cpu_frequency(freq, cpu);
2217 }
2218
2219 return freq;
2220}
2221EXPORT_SYMBOL_GPL(cpufreq_driver_fast_switch);
2222
2223/**
2224 * cpufreq_driver_adjust_perf - Adjust CPU performance level in one go.
2225 * @policy: cpufreq policy object of the target CPU.
2226 * @min_perf: Minimum (required) performance level (units of @capacity).
2227 * @target_perf: Target (desired) performance level (units of @capacity).
2228 * @max_perf: Maximum (allowed) performance level (units of @capacity).
2229 * @capacity: Capacity of the target CPU.
2230 *
2231 * Carry out a fast performance level adjustment for the CPU represented by
2232 * @policy without sleeping.
2233 *
2234 * The driver's ->adjust_perf() callback invoked by this function must be
2235 * suitable for calling from within RCU-sched read-side critical sections and
2236 * it is expected to program the processor to select suitable performance
2237 * levels between @min_perf and @max_perf inclusive and preferably close to
2238 * @target_perf going forward for the CPU represented by @policy.
2239 *
2240 * This function must not be called if policy->fast_switch_enabled is unset.
2241 *
2242 * Governors calling this function must guarantee that it will never be invoked
2243 * twice in parallel for the same CPU and that it will never be called in
2244 * parallel with either ->target() or ->target_index() or ->fast_switch() for
2245 * the same CPU.
2246 */
2247void cpufreq_driver_adjust_perf(struct cpufreq_policy *policy,
2248 unsigned long min_perf,
2249 unsigned long target_perf,
2250 unsigned long max_perf,
2251 unsigned long capacity)
2252{
2253 cpufreq_driver->adjust_perf(policy, min_perf, target_perf, max_perf, capacity);
2254}
2255
2256/**
2257 * cpufreq_driver_has_adjust_perf - Check "direct fast switch" callback.
2258 *
2259 * Return 'true' if the ->adjust_perf callback is present for the
2260 * current driver or 'false' otherwise.
2261 */
2262bool cpufreq_driver_has_adjust_perf(void)
2263{
2264 return !!cpufreq_driver->adjust_perf;
2265}
2266
2267/* Must set freqs->new to intermediate frequency */
2268static int __target_intermediate(struct cpufreq_policy *policy,
2269 struct cpufreq_freqs *freqs, int index)
2270{
2271 int ret;
2272
2273 freqs->new = cpufreq_driver->get_intermediate(policy, index);
2274
2275 /* We don't need to switch to intermediate freq */
2276 if (!freqs->new)
2277 return 0;
2278
2279 pr_debug("%s: cpu: %d, switching to intermediate freq: oldfreq: %u, intermediate freq: %u\n",
2280 __func__, policy->cpu, freqs->old, freqs->new);
2281
2282 cpufreq_freq_transition_begin(policy, freqs);
2283 ret = cpufreq_driver->target_intermediate(policy, index);
2284 cpufreq_freq_transition_end(policy, freqs, ret);
2285
2286 if (ret)
2287 pr_err("%s: Failed to change to intermediate frequency: %d\n",
2288 __func__, ret);
2289
2290 return ret;
2291}
2292
2293static int __target_index(struct cpufreq_policy *policy, int index)
2294{
2295 struct cpufreq_freqs freqs = {.old = policy->cur, .flags = 0};
2296 unsigned int restore_freq, intermediate_freq = 0;
2297 unsigned int newfreq = policy->freq_table[index].frequency;
2298 int retval = -EINVAL;
2299 bool notify;
2300
2301 if (newfreq == policy->cur)
2302 return 0;
2303
2304 /* Save last value to restore later on errors */
2305 restore_freq = policy->cur;
2306
2307 notify = !(cpufreq_driver->flags & CPUFREQ_ASYNC_NOTIFICATION);
2308 if (notify) {
2309 /* Handle switching to intermediate frequency */
2310 if (cpufreq_driver->get_intermediate) {
2311 retval = __target_intermediate(policy, &freqs, index);
2312 if (retval)
2313 return retval;
2314
2315 intermediate_freq = freqs.new;
2316 /* Set old freq to intermediate */
2317 if (intermediate_freq)
2318 freqs.old = freqs.new;
2319 }
2320
2321 freqs.new = newfreq;
2322 pr_debug("%s: cpu: %d, oldfreq: %u, new freq: %u\n",
2323 __func__, policy->cpu, freqs.old, freqs.new);
2324
2325 cpufreq_freq_transition_begin(policy, &freqs);
2326 }
2327
2328 retval = cpufreq_driver->target_index(policy, index);
2329 if (retval)
2330 pr_err("%s: Failed to change cpu frequency: %d\n", __func__,
2331 retval);
2332
2333 if (notify) {
2334 cpufreq_freq_transition_end(policy, &freqs, retval);
2335
2336 /*
2337 * Failed after setting to intermediate freq? Driver should have
2338 * reverted back to initial frequency and so should we. Check
2339 * here for intermediate_freq instead of get_intermediate, in
2340 * case we haven't switched to intermediate freq at all.
2341 */
2342 if (unlikely(retval && intermediate_freq)) {
2343 freqs.old = intermediate_freq;
2344 freqs.new = restore_freq;
2345 cpufreq_freq_transition_begin(policy, &freqs);
2346 cpufreq_freq_transition_end(policy, &freqs, 0);
2347 }
2348 }
2349
2350 return retval;
2351}
2352
2353int __cpufreq_driver_target(struct cpufreq_policy *policy,
2354 unsigned int target_freq,
2355 unsigned int relation)
2356{
2357 unsigned int old_target_freq = target_freq;
2358
2359 if (cpufreq_disabled())
2360 return -ENODEV;
2361
2362 target_freq = __resolve_freq(policy, target_freq, policy->min,
2363 policy->max, relation);
2364
2365 pr_debug("CPU %u: cur %u kHz -> target %u kHz (req %u kHz, rel %u)\n",
2366 policy->cpu, policy->cur, target_freq, old_target_freq, relation);
2367
2368 /*
2369 * This might look like a redundant call as we are checking it again
2370 * after finding index. But it is left intentionally for cases where
2371 * exactly same freq is called again and so we can save on few function
2372 * calls.
2373 */
2374 if (target_freq == policy->cur) {
2375 if (!(cpufreq_driver->flags & CPUFREQ_NEED_UPDATE_LIMITS) ||
2376 !policy->update_limits)
2377 return 0;
2378
2379 policy->update_limits = false;
2380 }
2381
2382 if (cpufreq_driver->target) {
2383 /*
2384 * If the driver hasn't setup a single inefficient frequency,
2385 * it's unlikely it knows how to decode CPUFREQ_RELATION_E.
2386 */
2387 if (!policy->efficiencies_available)
2388 relation &= ~CPUFREQ_RELATION_E;
2389
2390 return cpufreq_driver->target(policy, target_freq, relation);
2391 }
2392
2393 if (!cpufreq_driver->target_index)
2394 return -EINVAL;
2395
2396 return __target_index(policy, policy->cached_resolved_idx);
2397}
2398EXPORT_SYMBOL_GPL(__cpufreq_driver_target);
2399
2400int cpufreq_driver_target(struct cpufreq_policy *policy,
2401 unsigned int target_freq,
2402 unsigned int relation)
2403{
2404 guard(cpufreq_policy_write)(policy);
2405
2406 return __cpufreq_driver_target(policy, target_freq, relation);
2407}
2408EXPORT_SYMBOL_GPL(cpufreq_driver_target);
2409
2410__weak struct cpufreq_governor *cpufreq_fallback_governor(void)
2411{
2412 return NULL;
2413}
2414
2415static int cpufreq_init_governor(struct cpufreq_policy *policy)
2416{
2417 int ret;
2418
2419 /* Don't start any governor operations if we are entering suspend */
2420 if (cpufreq_suspended)
2421 return 0;
2422 /*
2423 * Governor might not be initiated here if ACPI _PPC changed
2424 * notification happened, so check it.
2425 */
2426 if (!policy->governor)
2427 return -EINVAL;
2428
2429 /* Platform doesn't want dynamic frequency switching ? */
2430 if (policy->governor->flags & CPUFREQ_GOV_DYNAMIC_SWITCHING &&
2431 cpufreq_driver->flags & CPUFREQ_NO_AUTO_DYNAMIC_SWITCHING) {
2432 struct cpufreq_governor *gov = cpufreq_fallback_governor();
2433
2434 if (gov) {
2435 pr_warn("Can't use %s governor as dynamic switching is disallowed. Fallback to %s governor\n",
2436 policy->governor->name, gov->name);
2437 policy->governor = gov;
2438 } else {
2439 return -EINVAL;
2440 }
2441 }
2442
2443 if (!try_module_get(policy->governor->owner))
2444 return -EINVAL;
2445
2446 pr_debug("%s: for CPU %u\n", __func__, policy->cpu);
2447
2448 if (policy->governor->init) {
2449 ret = policy->governor->init(policy);
2450 if (ret) {
2451 module_put(policy->governor->owner);
2452 return ret;
2453 }
2454 }
2455
2456 policy->strict_target = !!(policy->governor->flags & CPUFREQ_GOV_STRICT_TARGET);
2457
2458 return 0;
2459}
2460
2461static void cpufreq_exit_governor(struct cpufreq_policy *policy)
2462{
2463 if (cpufreq_suspended || !policy->governor)
2464 return;
2465
2466 pr_debug("%s: for CPU %u\n", __func__, policy->cpu);
2467
2468 if (policy->governor->exit)
2469 policy->governor->exit(policy);
2470
2471 module_put(policy->governor->owner);
2472}
2473
2474int cpufreq_start_governor(struct cpufreq_policy *policy)
2475{
2476 int ret;
2477
2478 if (cpufreq_suspended)
2479 return 0;
2480
2481 if (!policy->governor)
2482 return -EINVAL;
2483
2484 pr_debug("%s: for CPU %u\n", __func__, policy->cpu);
2485
2486 cpufreq_verify_current_freq(policy, false);
2487
2488 if (policy->governor->start) {
2489 ret = policy->governor->start(policy);
2490 if (ret)
2491 return ret;
2492 }
2493
2494 if (policy->governor->limits)
2495 policy->governor->limits(policy);
2496
2497 return 0;
2498}
2499
2500void cpufreq_stop_governor(struct cpufreq_policy *policy)
2501{
2502 if (cpufreq_suspended || !policy->governor)
2503 return;
2504
2505 pr_debug("%s: for CPU %u\n", __func__, policy->cpu);
2506
2507 if (policy->governor->stop)
2508 policy->governor->stop(policy);
2509}
2510
2511static void cpufreq_governor_limits(struct cpufreq_policy *policy)
2512{
2513 if (cpufreq_suspended || !policy->governor)
2514 return;
2515
2516 pr_debug("%s: for CPU %u\n", __func__, policy->cpu);
2517
2518 if (policy->governor->limits)
2519 policy->governor->limits(policy);
2520}
2521
2522int cpufreq_register_governor(struct cpufreq_governor *governor)
2523{
2524 int err;
2525
2526 if (!governor)
2527 return -EINVAL;
2528
2529 if (cpufreq_disabled())
2530 return -ENODEV;
2531
2532 mutex_lock(&cpufreq_governor_mutex);
2533
2534 err = -EBUSY;
2535 if (!find_governor(governor->name)) {
2536 err = 0;
2537 list_add(&governor->governor_list, &cpufreq_governor_list);
2538 }
2539
2540 mutex_unlock(&cpufreq_governor_mutex);
2541 return err;
2542}
2543EXPORT_SYMBOL_GPL(cpufreq_register_governor);
2544
2545void cpufreq_unregister_governor(struct cpufreq_governor *governor)
2546{
2547 struct cpufreq_policy *policy;
2548 unsigned long flags;
2549
2550 if (!governor)
2551 return;
2552
2553 if (cpufreq_disabled())
2554 return;
2555
2556 /* clear last_governor for all inactive policies */
2557 read_lock_irqsave(&cpufreq_driver_lock, flags);
2558 for_each_inactive_policy(policy) {
2559 if (!strcmp(policy->last_governor, governor->name)) {
2560 policy->governor = NULL;
2561 policy->last_governor[0] = '\0';
2562 }
2563 }
2564 read_unlock_irqrestore(&cpufreq_driver_lock, flags);
2565
2566 mutex_lock(&cpufreq_governor_mutex);
2567 list_del(&governor->governor_list);
2568 mutex_unlock(&cpufreq_governor_mutex);
2569}
2570EXPORT_SYMBOL_GPL(cpufreq_unregister_governor);
2571
2572
2573/*********************************************************************
2574 * POLICY INTERFACE *
2575 *********************************************************************/
2576
2577DEFINE_PER_CPU(unsigned long, cpufreq_pressure);
2578
2579/**
2580 * cpufreq_update_pressure() - Update cpufreq pressure for CPUs
2581 * @policy: cpufreq policy of the CPUs.
2582 *
2583 * Update the value of cpufreq pressure for all @cpus in the policy.
2584 */
2585static void cpufreq_update_pressure(struct cpufreq_policy *policy)
2586{
2587 unsigned long max_capacity, capped_freq, pressure;
2588 u32 max_freq;
2589 int cpu;
2590
2591 cpu = cpumask_first(policy->related_cpus);
2592 max_freq = arch_scale_freq_ref(cpu);
2593 if (!max_freq && cpufreq_driver->scale_freq_ref)
2594 max_freq = cpufreq_driver->scale_freq_ref(policy);
2595
2596 capped_freq = policy->max;
2597
2598 /*
2599 * Handle properly the boost frequencies, which should simply clean
2600 * the cpufreq pressure value.
2601 */
2602 if (max_freq <= capped_freq) {
2603 pressure = 0;
2604 } else {
2605 max_capacity = arch_scale_cpu_capacity(cpu);
2606 pressure = max_capacity -
2607 mult_frac(max_capacity, capped_freq, max_freq);
2608 }
2609
2610 for_each_cpu(cpu, policy->related_cpus)
2611 WRITE_ONCE(per_cpu(cpufreq_pressure, cpu), pressure);
2612}
2613
2614/**
2615 * cpufreq_set_policy - Modify cpufreq policy parameters.
2616 * @policy: Policy object to modify.
2617 * @new_gov: Policy governor pointer.
2618 * @new_pol: Policy value (for drivers with built-in governors).
2619 *
2620 * Invoke the cpufreq driver's ->verify() callback to sanity-check the frequency
2621 * limits to be set for the policy, update @policy with the verified limits
2622 * values and either invoke the driver's ->setpolicy() callback (if present) or
2623 * carry out a governor update for @policy. That is, run the current governor's
2624 * ->limits() callback (if @new_gov points to the same object as the one in
2625 * @policy) or replace the governor for @policy with @new_gov.
2626 *
2627 * The cpuinfo part of @policy is not updated by this function.
2628 */
2629static int cpufreq_set_policy(struct cpufreq_policy *policy,
2630 struct cpufreq_governor *new_gov,
2631 unsigned int new_pol)
2632{
2633 struct cpufreq_policy_data new_data;
2634 struct cpufreq_governor *old_gov;
2635 unsigned int freq;
2636 int ret;
2637
2638 memcpy(&new_data.cpuinfo, &policy->cpuinfo, sizeof(policy->cpuinfo));
2639 new_data.freq_table = policy->freq_table;
2640 new_data.cpu = policy->cpu;
2641 /*
2642 * PM QoS framework collects all the requests from users and provide us
2643 * the final aggregated value here.
2644 */
2645 new_data.min = freq_qos_read_value(&policy->constraints, FREQ_QOS_MIN);
2646 new_data.max = freq_qos_read_value(&policy->constraints, FREQ_QOS_MAX);
2647
2648 pr_debug("setting new policy for CPU %u: %u - %u kHz\n",
2649 new_data.cpu, new_data.min, new_data.max);
2650
2651 /*
2652 * Verify that the CPU speed can be set within these limits and make sure
2653 * that min <= max.
2654 */
2655 ret = cpufreq_driver->verify(&new_data);
2656 if (ret)
2657 return ret;
2658
2659 /*
2660 * Resolve policy min/max to available frequencies. It ensures
2661 * no frequency resolution will neither overshoot the requested maximum
2662 * nor undershoot the requested minimum.
2663 *
2664 * Avoid storing intermediate values in policy->max or policy->min and
2665 * compiler optimizations around them because they may be accessed
2666 * concurrently by cpufreq_driver_resolve_freq() during the update.
2667 */
2668 freq = __resolve_freq(policy, new_data.max, new_data.min, new_data.max,
2669 CPUFREQ_RELATION_H);
2670 if (freq != policy->max) {
2671 WRITE_ONCE(policy->max, freq);
2672 policy->update_limits = true;
2673 }
2674
2675 freq = __resolve_freq(policy, new_data.min, new_data.min, new_data.max,
2676 CPUFREQ_RELATION_L);
2677 freq = min(freq, policy->max);
2678 if (freq != policy->min) {
2679 WRITE_ONCE(policy->min, freq);
2680 policy->update_limits = true;
2681 }
2682
2683 trace_cpu_frequency_limits(policy);
2684
2685 cpufreq_update_pressure(policy);
2686
2687 policy->cached_target_freq = UINT_MAX;
2688
2689 pr_debug("new min and max freqs are %u - %u kHz\n",
2690 policy->min, policy->max);
2691
2692 if (cpufreq_driver->setpolicy) {
2693 policy->policy = new_pol;
2694 pr_debug("setting range\n");
2695 return cpufreq_driver->setpolicy(policy);
2696 }
2697
2698 if (new_gov == policy->governor) {
2699 pr_debug("governor limits update\n");
2700 cpufreq_governor_limits(policy);
2701 return 0;
2702 }
2703
2704 pr_debug("governor switch\n");
2705
2706 /* save old, working values */
2707 old_gov = policy->governor;
2708 /* end old governor */
2709 if (old_gov) {
2710 cpufreq_stop_governor(policy);
2711 cpufreq_exit_governor(policy);
2712 }
2713
2714 /* start new governor */
2715 policy->governor = new_gov;
2716 ret = cpufreq_init_governor(policy);
2717 if (!ret) {
2718 ret = cpufreq_start_governor(policy);
2719 if (!ret) {
2720 pr_debug("governor change\n");
2721 return 0;
2722 }
2723 cpufreq_exit_governor(policy);
2724 }
2725
2726 /* new governor failed, so re-start old one */
2727 pr_debug("starting governor %s failed\n", policy->governor->name);
2728 if (old_gov) {
2729 policy->governor = old_gov;
2730 if (cpufreq_init_governor(policy)) {
2731 policy->governor = NULL;
2732 } else if (cpufreq_start_governor(policy)) {
2733 cpufreq_exit_governor(policy);
2734 policy->governor = NULL;
2735 }
2736 }
2737
2738 return ret;
2739}
2740
2741static void cpufreq_policy_refresh(struct cpufreq_policy *policy)
2742{
2743 guard(cpufreq_policy_write)(policy);
2744
2745 /*
2746 * BIOS might change freq behind our back
2747 * -> ask driver for current freq and notify governors about a change
2748 */
2749 if (cpufreq_driver->get && has_target() &&
2750 (cpufreq_suspended || WARN_ON(!cpufreq_verify_current_freq(policy, false))))
2751 return;
2752
2753 refresh_frequency_limits(policy);
2754}
2755
2756/**
2757 * cpufreq_update_policy - Re-evaluate an existing cpufreq policy.
2758 * @cpu: CPU to re-evaluate the policy for.
2759 *
2760 * Update the current frequency for the cpufreq policy of @cpu and use
2761 * cpufreq_set_policy() to re-apply the min and max limits, which triggers the
2762 * evaluation of policy notifiers and the cpufreq driver's ->verify() callback
2763 * for the policy in question, among other things.
2764 */
2765void cpufreq_update_policy(unsigned int cpu)
2766{
2767 struct cpufreq_policy *policy __free(put_cpufreq_policy) = cpufreq_cpu_get(cpu);
2768 if (!policy)
2769 return;
2770
2771 cpufreq_policy_refresh(policy);
2772}
2773EXPORT_SYMBOL(cpufreq_update_policy);
2774
2775/**
2776 * cpufreq_update_limits - Update policy limits for a given CPU.
2777 * @cpu: CPU to update the policy limits for.
2778 *
2779 * Invoke the driver's ->update_limits callback if present or call
2780 * cpufreq_policy_refresh() for @cpu.
2781 */
2782void cpufreq_update_limits(unsigned int cpu)
2783{
2784 struct cpufreq_policy *policy __free(put_cpufreq_policy) = cpufreq_cpu_get(cpu);
2785 if (!policy)
2786 return;
2787
2788 if (cpufreq_driver->update_limits)
2789 cpufreq_driver->update_limits(policy);
2790 else
2791 cpufreq_policy_refresh(policy);
2792}
2793EXPORT_SYMBOL_GPL(cpufreq_update_limits);
2794
2795/*********************************************************************
2796 * BOOST *
2797 *********************************************************************/
2798int cpufreq_boost_set_sw(struct cpufreq_policy *policy, int state)
2799{
2800 int ret;
2801
2802 if (!policy->freq_table)
2803 return -ENXIO;
2804
2805 ret = cpufreq_frequency_table_cpuinfo(policy);
2806 if (ret)
2807 pr_err("%s: Policy frequency update failed\n", __func__);
2808
2809 return ret;
2810}
2811EXPORT_SYMBOL_GPL(cpufreq_boost_set_sw);
2812
2813static int cpufreq_boost_trigger_state(int state)
2814{
2815 struct cpufreq_policy *policy;
2816 unsigned long flags;
2817 int ret = -EOPNOTSUPP;
2818
2819 /*
2820 * Don't compare 'cpufreq_driver->boost_enabled' with 'state' here to
2821 * make sure all policies are in sync with global boost flag.
2822 */
2823
2824 write_lock_irqsave(&cpufreq_driver_lock, flags);
2825 cpufreq_driver->boost_enabled = state;
2826 write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2827
2828 cpus_read_lock();
2829 for_each_active_policy(policy) {
2830 if (!policy->boost_supported)
2831 continue;
2832
2833 ret = policy_set_boost(policy, state);
2834 if (unlikely(ret))
2835 break;
2836 }
2837
2838 cpus_read_unlock();
2839
2840 if (likely(!ret))
2841 return 0;
2842
2843 write_lock_irqsave(&cpufreq_driver_lock, flags);
2844 cpufreq_driver->boost_enabled = !state;
2845 write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2846
2847 pr_err("%s: Cannot %s BOOST\n",
2848 __func__, str_enable_disable(state));
2849
2850 return ret;
2851}
2852
2853static bool cpufreq_boost_supported(void)
2854{
2855 return cpufreq_driver->set_boost;
2856}
2857
2858static int create_boost_sysfs_file(void)
2859{
2860 int ret;
2861
2862 ret = sysfs_create_file(cpufreq_global_kobject, &boost.attr);
2863 if (ret)
2864 pr_err("%s: cannot register global BOOST sysfs file\n",
2865 __func__);
2866
2867 return ret;
2868}
2869
2870static void remove_boost_sysfs_file(void)
2871{
2872 if (cpufreq_boost_supported())
2873 sysfs_remove_file(cpufreq_global_kobject, &boost.attr);
2874}
2875
2876bool cpufreq_boost_enabled(void)
2877{
2878 return cpufreq_driver->boost_enabled;
2879}
2880EXPORT_SYMBOL_GPL(cpufreq_boost_enabled);
2881
2882/*********************************************************************
2883 * REGISTER / UNREGISTER CPUFREQ DRIVER *
2884 *********************************************************************/
2885static enum cpuhp_state hp_online;
2886
2887static int cpuhp_cpufreq_online(unsigned int cpu)
2888{
2889 cpufreq_online(cpu);
2890
2891 return 0;
2892}
2893
2894static int cpuhp_cpufreq_offline(unsigned int cpu)
2895{
2896 cpufreq_offline(cpu);
2897
2898 return 0;
2899}
2900
2901/**
2902 * cpufreq_register_driver - register a CPU Frequency driver
2903 * @driver_data: A struct cpufreq_driver containing the values#
2904 * submitted by the CPU Frequency driver.
2905 *
2906 * Registers a CPU Frequency driver to this core code. This code
2907 * returns zero on success, -EEXIST when another driver got here first
2908 * (and isn't unregistered in the meantime).
2909 *
2910 */
2911int cpufreq_register_driver(struct cpufreq_driver *driver_data)
2912{
2913 unsigned long flags;
2914 int ret;
2915
2916 if (cpufreq_disabled())
2917 return -ENODEV;
2918
2919 /*
2920 * The cpufreq core depends heavily on the availability of device
2921 * structure, make sure they are available before proceeding further.
2922 */
2923 if (!get_cpu_device(0))
2924 return -EPROBE_DEFER;
2925
2926 if (!driver_data || !driver_data->verify || !driver_data->init ||
2927 (driver_data->target_index && driver_data->target) ||
2928 (!!driver_data->setpolicy == (driver_data->target_index || driver_data->target)) ||
2929 (!driver_data->get_intermediate != !driver_data->target_intermediate) ||
2930 (!driver_data->online != !driver_data->offline) ||
2931 (driver_data->adjust_perf && !driver_data->fast_switch))
2932 return -EINVAL;
2933
2934 pr_debug("trying to register driver %s\n", driver_data->name);
2935
2936 /* Protect against concurrent CPU online/offline. */
2937 cpus_read_lock();
2938
2939 write_lock_irqsave(&cpufreq_driver_lock, flags);
2940 if (cpufreq_driver) {
2941 write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2942 ret = -EEXIST;
2943 goto out;
2944 }
2945 cpufreq_driver = driver_data;
2946 write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2947
2948 if (driver_data->setpolicy)
2949 driver_data->flags |= CPUFREQ_CONST_LOOPS;
2950
2951 if (cpufreq_boost_supported()) {
2952 ret = create_boost_sysfs_file();
2953 if (ret)
2954 goto err_null_driver;
2955 }
2956
2957 /*
2958 * Mark support for the scheduler's frequency invariance engine for
2959 * drivers that implement target(), target_index() or fast_switch().
2960 */
2961 if (!cpufreq_driver->setpolicy) {
2962 static_branch_enable_cpuslocked(&cpufreq_freq_invariance);
2963 pr_debug("cpufreq: supports frequency invariance\n");
2964 }
2965
2966 ret = subsys_interface_register(&cpufreq_interface);
2967 if (ret)
2968 goto err_boost_unreg;
2969
2970 if (unlikely(list_empty(&cpufreq_policy_list))) {
2971 /* if all ->init() calls failed, unregister */
2972 ret = -ENODEV;
2973 pr_debug("%s: No CPU initialized for driver %s\n", __func__,
2974 driver_data->name);
2975 goto err_if_unreg;
2976 }
2977
2978 ret = cpuhp_setup_state_nocalls_cpuslocked(CPUHP_AP_ONLINE_DYN,
2979 "cpufreq:online",
2980 cpuhp_cpufreq_online,
2981 cpuhp_cpufreq_offline);
2982 if (ret < 0)
2983 goto err_if_unreg;
2984 hp_online = ret;
2985 ret = 0;
2986
2987 pr_debug("driver %s up and running\n", driver_data->name);
2988 goto out;
2989
2990err_if_unreg:
2991 subsys_interface_unregister(&cpufreq_interface);
2992err_boost_unreg:
2993 if (!cpufreq_driver->setpolicy)
2994 static_branch_disable_cpuslocked(&cpufreq_freq_invariance);
2995 remove_boost_sysfs_file();
2996err_null_driver:
2997 write_lock_irqsave(&cpufreq_driver_lock, flags);
2998 cpufreq_driver = NULL;
2999 write_unlock_irqrestore(&cpufreq_driver_lock, flags);
3000out:
3001 cpus_read_unlock();
3002 return ret;
3003}
3004EXPORT_SYMBOL_GPL(cpufreq_register_driver);
3005
3006/*
3007 * cpufreq_unregister_driver - unregister the current CPUFreq driver
3008 *
3009 * Unregister the current CPUFreq driver. Only call this if you have
3010 * the right to do so, i.e. if you have succeeded in initialising before!
3011 * Returns zero if successful, and -EINVAL if the cpufreq_driver is
3012 * currently not initialised.
3013 */
3014void cpufreq_unregister_driver(struct cpufreq_driver *driver)
3015{
3016 unsigned long flags;
3017
3018 if (WARN_ON(!cpufreq_driver || (driver != cpufreq_driver)))
3019 return;
3020
3021 pr_debug("unregistering driver %s\n", driver->name);
3022
3023 /* Protect against concurrent cpu hotplug */
3024 cpus_read_lock();
3025 subsys_interface_unregister(&cpufreq_interface);
3026 remove_boost_sysfs_file();
3027 static_branch_disable_cpuslocked(&cpufreq_freq_invariance);
3028 cpuhp_remove_state_nocalls_cpuslocked(hp_online);
3029
3030 write_lock_irqsave(&cpufreq_driver_lock, flags);
3031
3032 cpufreq_driver = NULL;
3033
3034 write_unlock_irqrestore(&cpufreq_driver_lock, flags);
3035 cpus_read_unlock();
3036}
3037EXPORT_SYMBOL_GPL(cpufreq_unregister_driver);
3038
3039static int __init cpufreq_core_init(void)
3040{
3041 struct cpufreq_governor *gov = cpufreq_default_governor();
3042 struct device *dev_root;
3043
3044 if (cpufreq_disabled())
3045 return -ENODEV;
3046
3047 dev_root = bus_get_dev_root(&cpu_subsys);
3048 if (dev_root) {
3049 cpufreq_global_kobject = kobject_create_and_add("cpufreq", &dev_root->kobj);
3050 put_device(dev_root);
3051 }
3052 BUG_ON(!cpufreq_global_kobject);
3053
3054 if (!strlen(default_governor))
3055 strscpy(default_governor, gov->name, CPUFREQ_NAME_LEN);
3056
3057 return 0;
3058}
3059
3060static bool cpufreq_policy_is_good_for_eas(unsigned int cpu)
3061{
3062 struct cpufreq_policy *policy __free(put_cpufreq_policy) = cpufreq_cpu_get(cpu);
3063 if (!policy) {
3064 pr_debug("cpufreq policy not set for CPU: %d\n", cpu);
3065 return false;
3066 }
3067
3068 return sugov_is_governor(policy);
3069}
3070
3071bool cpufreq_ready_for_eas(const struct cpumask *cpu_mask)
3072{
3073 unsigned int cpu;
3074
3075 /* Do not attempt EAS if schedutil is not being used. */
3076 for_each_cpu(cpu, cpu_mask) {
3077 if (!cpufreq_policy_is_good_for_eas(cpu)) {
3078 pr_debug("rd %*pbl: schedutil is mandatory for EAS\n",
3079 cpumask_pr_args(cpu_mask));
3080 return false;
3081 }
3082 }
3083
3084 return true;
3085}
3086
3087module_param(off, int, 0444);
3088module_param_string(default_governor, default_governor, CPUFREQ_NAME_LEN, 0444);
3089core_initcall(cpufreq_core_init);