projects/linux·blob master · refs

3122 lines · 84.0 KiB

1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Generic OPP Interface
4 *
5 * Copyright (C) 2009-2010 Texas Instruments Incorporated.
6 * Nishanth Menon
7 * Romit Dasgupta
8 * Kevin Hilman
9 */
10
11#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
12
13#include <linux/clk.h>
14#include <linux/errno.h>
15#include <linux/err.h>
16#include <linux/device.h>
17#include <linux/export.h>
18#include <linux/pm_domain.h>
19#include <linux/regulator/consumer.h>
20#include <linux/slab.h>
21#include <linux/xarray.h>
22
23#include "opp.h"
24
25/*
26 * The root of the list of all opp-tables. All opp_table structures branch off
27 * from here, with each opp_table containing the list of opps it supports in
28 * various states of availability.
29 */
30LIST_HEAD(opp_tables);
31
32/* Lock to allow exclusive modification to the device and opp lists */
33DEFINE_MUTEX(opp_table_lock);
34/* Flag indicating that opp_tables list is being updated at the moment */
35static bool opp_tables_busy;
36
37/* OPP ID allocator */
38static DEFINE_XARRAY_ALLOC1(opp_configs);
39
40static bool _find_opp_dev(const struct device *dev, struct opp_table *opp_table)
41{
42 struct opp_device *opp_dev;
43
44 guard(mutex)(&opp_table->lock);
45
46 list_for_each_entry(opp_dev, &opp_table->dev_list, node)
47 if (opp_dev->dev == dev)
48 return true;
49
50 return false;
51}
52
53static struct opp_table *_find_opp_table_unlocked(struct device *dev)
54{
55 struct opp_table *opp_table;
56
57 list_for_each_entry(opp_table, &opp_tables, node) {
58 if (_find_opp_dev(dev, opp_table))
59 return dev_pm_opp_get_opp_table_ref(opp_table);
60 }
61
62 return ERR_PTR(-ENODEV);
63}
64
65/**
66 * _find_opp_table() - find opp_table struct using device pointer
67 * @dev: device pointer used to lookup OPP table
68 *
69 * Search OPP table for one containing matching device.
70 *
71 * Return: pointer to 'struct opp_table' if found, otherwise -ENODEV or
72 * -EINVAL based on type of error.
73 *
74 * The callers must call dev_pm_opp_put_opp_table() after the table is used.
75 */
76struct opp_table *_find_opp_table(struct device *dev)
77{
78 if (IS_ERR_OR_NULL(dev)) {
79 pr_err("%s: Invalid parameters\n", __func__);
80 return ERR_PTR(-EINVAL);
81 }
82
83 guard(mutex)(&opp_table_lock);
84 return _find_opp_table_unlocked(dev);
85}
86
87/*
88 * Returns true if multiple clocks aren't there, else returns false with WARN.
89 *
90 * We don't force clk_count == 1 here as there are users who don't have a clock
91 * representation in the OPP table and manage the clock configuration themselves
92 * in an platform specific way.
93 */
94static bool assert_single_clk(struct opp_table *opp_table,
95 unsigned int __always_unused index)
96{
97 return !WARN_ON(opp_table->clk_count > 1);
98}
99
100/*
101 * Returns true if clock table is large enough to contain the clock index.
102 */
103static bool assert_clk_index(struct opp_table *opp_table,
104 unsigned int index)
105{
106 return opp_table->clk_count > index;
107}
108
109/*
110 * Returns true if bandwidth table is large enough to contain the bandwidth index.
111 */
112static bool assert_bandwidth_index(struct opp_table *opp_table,
113 unsigned int index)
114{
115 return opp_table->path_count > index;
116}
117
118/**
119 * dev_pm_opp_get_bw() - Gets the bandwidth corresponding to an opp
120 * @opp: opp for which bandwidth has to be returned for
121 * @peak: select peak or average bandwidth
122 * @index: bandwidth index
123 *
124 * Return: bandwidth in kBps, else return 0
125 */
126unsigned long dev_pm_opp_get_bw(struct dev_pm_opp *opp, bool peak, int index)
127{
128 if (IS_ERR_OR_NULL(opp)) {
129 pr_err("%s: Invalid parameters\n", __func__);
130 return 0;
131 }
132
133 if (index >= opp->opp_table->path_count)
134 return 0;
135
136 if (!opp->bandwidth)
137 return 0;
138
139 return peak ? opp->bandwidth[index].peak : opp->bandwidth[index].avg;
140}
141EXPORT_SYMBOL_GPL(dev_pm_opp_get_bw);
142
143/**
144 * dev_pm_opp_get_voltage() - Gets the voltage corresponding to an opp
145 * @opp: opp for which voltage has to be returned for
146 *
147 * Return: voltage in micro volt corresponding to the opp, else
148 * return 0
149 *
150 * This is useful only for devices with single power supply.
151 */
152unsigned long dev_pm_opp_get_voltage(struct dev_pm_opp *opp)
153{
154 if (IS_ERR_OR_NULL(opp)) {
155 pr_err("%s: Invalid parameters\n", __func__);
156 return 0;
157 }
158
159 return opp->supplies[0].u_volt;
160}
161EXPORT_SYMBOL_GPL(dev_pm_opp_get_voltage);
162
163/**
164 * dev_pm_opp_get_supplies() - Gets the supply information corresponding to an opp
165 * @opp: opp for which voltage has to be returned for
166 * @supplies: Placeholder for copying the supply information.
167 *
168 * Return: negative error number on failure, 0 otherwise on success after
169 * setting @supplies.
170 *
171 * This can be used for devices with any number of power supplies. The caller
172 * must ensure the @supplies array must contain space for each regulator.
173 */
174int dev_pm_opp_get_supplies(struct dev_pm_opp *opp,
175 struct dev_pm_opp_supply *supplies)
176{
177 if (IS_ERR_OR_NULL(opp) || !supplies) {
178 pr_err("%s: Invalid parameters\n", __func__);
179 return -EINVAL;
180 }
181
182 memcpy(supplies, opp->supplies,
183 sizeof(*supplies) * opp->opp_table->regulator_count);
184 return 0;
185}
186EXPORT_SYMBOL_GPL(dev_pm_opp_get_supplies);
187
188/**
189 * dev_pm_opp_get_power() - Gets the power corresponding to an opp
190 * @opp: opp for which power has to be returned for
191 *
192 * Return: power in micro watt corresponding to the opp, else
193 * return 0
194 *
195 * This is useful only for devices with single power supply.
196 */
197unsigned long dev_pm_opp_get_power(struct dev_pm_opp *opp)
198{
199 unsigned long opp_power = 0;
200 int i;
201
202 if (IS_ERR_OR_NULL(opp)) {
203 pr_err("%s: Invalid parameters\n", __func__);
204 return 0;
205 }
206 for (i = 0; i < opp->opp_table->regulator_count; i++)
207 opp_power += opp->supplies[i].u_watt;
208
209 return opp_power;
210}
211EXPORT_SYMBOL_GPL(dev_pm_opp_get_power);
212
213/**
214 * dev_pm_opp_get_freq_indexed() - Gets the frequency corresponding to an
215 * available opp with specified index
216 * @opp: opp for which frequency has to be returned for
217 * @index: index of the frequency within the required opp
218 *
219 * Return: frequency in hertz corresponding to the opp with specified index,
220 * else return 0
221 */
222unsigned long dev_pm_opp_get_freq_indexed(struct dev_pm_opp *opp, u32 index)
223{
224 if (IS_ERR_OR_NULL(opp) || index >= opp->opp_table->clk_count) {
225 pr_err("%s: Invalid parameters\n", __func__);
226 return 0;
227 }
228
229 return opp->rates[index];
230}
231EXPORT_SYMBOL_GPL(dev_pm_opp_get_freq_indexed);
232
233/**
234 * dev_pm_opp_get_level() - Gets the level corresponding to an available opp
235 * @opp: opp for which level value has to be returned for
236 *
237 * Return: level read from device tree corresponding to the opp, else
238 * return U32_MAX.
239 */
240unsigned int dev_pm_opp_get_level(struct dev_pm_opp *opp)
241{
242 if (IS_ERR_OR_NULL(opp) || !opp->available) {
243 pr_err("%s: Invalid parameters\n", __func__);
244 return U32_MAX;
245 }
246
247 return opp->level;
248}
249EXPORT_SYMBOL_GPL(dev_pm_opp_get_level);
250
251/**
252 * dev_pm_opp_get_required_pstate() - Gets the required performance state
253 * corresponding to an available opp
254 * @opp: opp for which performance state has to be returned for
255 * @index: index of the required opp
256 *
257 * Return: performance state read from device tree corresponding to the
258 * required opp, else return U32_MAX.
259 */
260unsigned int dev_pm_opp_get_required_pstate(struct dev_pm_opp *opp,
261 unsigned int index)
262{
263 if (IS_ERR_OR_NULL(opp) || !opp->available ||
264 index >= opp->opp_table->required_opp_count) {
265 pr_err("%s: Invalid parameters\n", __func__);
266 return 0;
267 }
268
269 /* required-opps not fully initialized yet */
270 if (lazy_linking_pending(opp->opp_table))
271 return 0;
272
273 /* The required OPP table must belong to a genpd */
274 if (unlikely(!opp->opp_table->required_opp_tables[index]->is_genpd)) {
275 pr_err("%s: Performance state is only valid for genpds.\n", __func__);
276 return 0;
277 }
278
279 return opp->required_opps[index]->level;
280}
281EXPORT_SYMBOL_GPL(dev_pm_opp_get_required_pstate);
282
283/**
284 * dev_pm_opp_is_turbo() - Returns if opp is turbo OPP or not
285 * @opp: opp for which turbo mode is being verified
286 *
287 * Turbo OPPs are not for normal use, and can be enabled (under certain
288 * conditions) for short duration of times to finish high throughput work
289 * quickly. Running on them for longer times may overheat the chip.
290 *
291 * Return: true if opp is turbo opp, else false.
292 */
293bool dev_pm_opp_is_turbo(struct dev_pm_opp *opp)
294{
295 if (IS_ERR_OR_NULL(opp) || !opp->available) {
296 pr_err("%s: Invalid parameters\n", __func__);
297 return false;
298 }
299
300 return opp->turbo;
301}
302EXPORT_SYMBOL_GPL(dev_pm_opp_is_turbo);
303
304/**
305 * dev_pm_opp_get_max_clock_latency() - Get max clock latency in nanoseconds
306 * @dev: device for which we do this operation
307 *
308 * Return: This function returns the max clock latency in nanoseconds.
309 */
310unsigned long dev_pm_opp_get_max_clock_latency(struct device *dev)
311{
312 struct opp_table *opp_table __free(put_opp_table) =
313 _find_opp_table(dev);
314
315 if (IS_ERR(opp_table))
316 return 0;
317
318 return opp_table->clock_latency_ns_max;
319}
320EXPORT_SYMBOL_GPL(dev_pm_opp_get_max_clock_latency);
321
322/**
323 * dev_pm_opp_get_max_volt_latency() - Get max voltage latency in nanoseconds
324 * @dev: device for which we do this operation
325 *
326 * Return: This function returns the max voltage latency in nanoseconds.
327 */
328unsigned long dev_pm_opp_get_max_volt_latency(struct device *dev)
329{
330 struct dev_pm_opp *opp;
331 struct regulator *reg;
332 unsigned long latency_ns = 0;
333 int ret, i, count;
334 struct {
335 unsigned long min;
336 unsigned long max;
337 } *uV;
338
339 struct opp_table *opp_table __free(put_opp_table) =
340 _find_opp_table(dev);
341
342 if (IS_ERR(opp_table))
343 return 0;
344
345 /* Regulator may not be required for the device */
346 if (!opp_table->regulators)
347 return 0;
348
349 count = opp_table->regulator_count;
350
351 uV = kmalloc_objs(*uV, count);
352 if (!uV)
353 return 0;
354
355 scoped_guard(mutex, &opp_table->lock) {
356 for (i = 0; i < count; i++) {
357 uV[i].min = ~0;
358 uV[i].max = 0;
359
360 list_for_each_entry(opp, &opp_table->opp_list, node) {
361 if (!opp->available)
362 continue;
363
364 if (opp->supplies[i].u_volt_min < uV[i].min)
365 uV[i].min = opp->supplies[i].u_volt_min;
366 if (opp->supplies[i].u_volt_max > uV[i].max)
367 uV[i].max = opp->supplies[i].u_volt_max;
368 }
369 }
370 }
371
372 /*
373 * The caller needs to ensure that opp_table (and hence the regulator)
374 * isn't freed, while we are executing this routine.
375 */
376 for (i = 0; i < count; i++) {
377 reg = opp_table->regulators[i];
378 ret = regulator_set_voltage_time(reg, uV[i].min, uV[i].max);
379 if (ret > 0)
380 latency_ns += ret * 1000;
381 }
382
383 kfree(uV);
384
385 return latency_ns;
386}
387EXPORT_SYMBOL_GPL(dev_pm_opp_get_max_volt_latency);
388
389/**
390 * dev_pm_opp_get_max_transition_latency() - Get max transition latency in
391 * nanoseconds
392 * @dev: device for which we do this operation
393 *
394 * Return: This function returns the max transition latency, in nanoseconds, to
395 * switch from one OPP to other.
396 */
397unsigned long dev_pm_opp_get_max_transition_latency(struct device *dev)
398{
399 return dev_pm_opp_get_max_volt_latency(dev) +
400 dev_pm_opp_get_max_clock_latency(dev);
401}
402EXPORT_SYMBOL_GPL(dev_pm_opp_get_max_transition_latency);
403
404/**
405 * dev_pm_opp_get_suspend_opp_freq() - Get frequency of suspend opp in Hz
406 * @dev: device for which we do this operation
407 *
408 * Return: This function returns the frequency of the OPP marked as suspend_opp
409 * if one is available, else returns 0;
410 */
411unsigned long dev_pm_opp_get_suspend_opp_freq(struct device *dev)
412{
413 unsigned long freq = 0;
414
415 struct opp_table *opp_table __free(put_opp_table) =
416 _find_opp_table(dev);
417
418 if (IS_ERR(opp_table))
419 return 0;
420
421 if (opp_table->suspend_opp && opp_table->suspend_opp->available)
422 freq = dev_pm_opp_get_freq(opp_table->suspend_opp);
423
424 return freq;
425}
426EXPORT_SYMBOL_GPL(dev_pm_opp_get_suspend_opp_freq);
427
428int _get_opp_count(struct opp_table *opp_table)
429{
430 struct dev_pm_opp *opp;
431 int count = 0;
432
433 guard(mutex)(&opp_table->lock);
434
435 list_for_each_entry(opp, &opp_table->opp_list, node) {
436 if (opp->available)
437 count++;
438 }
439
440 return count;
441}
442
443/**
444 * dev_pm_opp_get_opp_count() - Get number of opps available in the opp table
445 * @dev: device for which we do this operation
446 *
447 * Return: This function returns the number of available opps if there are any,
448 * else returns 0 if none or the corresponding error value.
449 */
450int dev_pm_opp_get_opp_count(struct device *dev)
451{
452 struct opp_table *opp_table __free(put_opp_table) =
453 _find_opp_table(dev);
454
455 if (IS_ERR(opp_table)) {
456 dev_dbg(dev, "%s: OPP table not found (%pe)\n",
457 __func__, opp_table);
458 return PTR_ERR(opp_table);
459 }
460
461 return _get_opp_count(opp_table);
462}
463EXPORT_SYMBOL_GPL(dev_pm_opp_get_opp_count);
464
465/* Helpers to read keys */
466static unsigned long _read_freq(struct dev_pm_opp *opp, int index)
467{
468 return opp->rates[index];
469}
470
471static unsigned long _read_level(struct dev_pm_opp *opp, int index)
472{
473 return opp->level;
474}
475
476static unsigned long _read_bw(struct dev_pm_opp *opp, int index)
477{
478 return opp->bandwidth[index].peak;
479}
480
481static unsigned long _read_opp_key(struct dev_pm_opp *opp, int index,
482 struct dev_pm_opp_key *key)
483{
484 key->bw = opp->bandwidth ? opp->bandwidth[index].peak : 0;
485 key->freq = opp->rates[index];
486 key->level = opp->level;
487
488 return true;
489}
490
491/* Generic comparison helpers */
492static bool _compare_exact(struct dev_pm_opp **opp, struct dev_pm_opp *temp_opp,
493 unsigned long opp_key, unsigned long key)
494{
495 if (opp_key == key) {
496 *opp = temp_opp;
497 return true;
498 }
499
500 return false;
501}
502
503static bool _compare_ceil(struct dev_pm_opp **opp, struct dev_pm_opp *temp_opp,
504 unsigned long opp_key, unsigned long key)
505{
506 if (opp_key >= key) {
507 *opp = temp_opp;
508 return true;
509 }
510
511 return false;
512}
513
514static bool _compare_floor(struct dev_pm_opp **opp, struct dev_pm_opp *temp_opp,
515 unsigned long opp_key, unsigned long key)
516{
517 if (opp_key > key)
518 return true;
519
520 *opp = temp_opp;
521 return false;
522}
523
524static bool _compare_opp_key_exact(struct dev_pm_opp **opp,
525 struct dev_pm_opp *temp_opp, struct dev_pm_opp_key *opp_key,
526 struct dev_pm_opp_key *key)
527{
528 bool level_match = (key->level == OPP_LEVEL_UNSET || opp_key->level == key->level);
529 bool freq_match = (key->freq == 0 || opp_key->freq == key->freq);
530 bool bw_match = (key->bw == 0 || opp_key->bw == key->bw);
531
532 if (freq_match && level_match && bw_match) {
533 *opp = temp_opp;
534 return true;
535 }
536
537 return false;
538}
539
540/* Generic key finding helpers */
541static struct dev_pm_opp *_opp_table_find_key(struct opp_table *opp_table,
542 unsigned long *key, int index, bool available,
543 unsigned long (*read)(struct dev_pm_opp *opp, int index),
544 bool (*compare)(struct dev_pm_opp **opp, struct dev_pm_opp *temp_opp,
545 unsigned long opp_key, unsigned long key),
546 bool (*assert)(struct opp_table *opp_table, unsigned int index))
547{
548 struct dev_pm_opp *temp_opp, *opp = ERR_PTR(-ERANGE);
549
550 /* Assert that the requirement is met */
551 if (assert && !assert(opp_table, index))
552 return ERR_PTR(-EINVAL);
553
554 guard(mutex)(&opp_table->lock);
555
556 list_for_each_entry(temp_opp, &opp_table->opp_list, node) {
557 if (temp_opp->available == available) {
558 if (compare(&opp, temp_opp, read(temp_opp, index), *key))
559 break;
560 }
561 }
562
563 /* Increment the reference count of OPP */
564 if (!IS_ERR(opp)) {
565 *key = read(opp, index);
566 dev_pm_opp_get(opp);
567 }
568
569 return opp;
570}
571
572static struct dev_pm_opp *_opp_table_find_opp_key(struct opp_table *opp_table,
573 struct dev_pm_opp_key *key, bool available,
574 unsigned long (*read)(struct dev_pm_opp *opp, int index,
575 struct dev_pm_opp_key *key),
576 bool (*compare)(struct dev_pm_opp **opp, struct dev_pm_opp *temp_opp,
577 struct dev_pm_opp_key *opp_key, struct dev_pm_opp_key *key),
578 bool (*assert)(struct opp_table *opp_table, unsigned int index))
579{
580 struct dev_pm_opp *temp_opp, *opp = ERR_PTR(-ERANGE);
581 struct dev_pm_opp_key temp_key;
582
583 /* Assert that the requirement is met */
584 if (!assert(opp_table, 0))
585 return ERR_PTR(-EINVAL);
586
587 guard(mutex)(&opp_table->lock);
588
589 list_for_each_entry(temp_opp, &opp_table->opp_list, node) {
590 if (temp_opp->available == available) {
591 read(temp_opp, 0, &temp_key);
592 if (compare(&opp, temp_opp, &temp_key, key)) {
593 /* Increment the reference count of OPP */
594 dev_pm_opp_get(opp);
595 break;
596 }
597 }
598 }
599
600 return opp;
601}
602
603static struct dev_pm_opp *
604_find_key(struct device *dev, unsigned long *key, int index, bool available,
605 unsigned long (*read)(struct dev_pm_opp *opp, int index),
606 bool (*compare)(struct dev_pm_opp **opp, struct dev_pm_opp *temp_opp,
607 unsigned long opp_key, unsigned long key),
608 bool (*assert)(struct opp_table *opp_table, unsigned int index))
609{
610 struct opp_table *opp_table __free(put_opp_table) =
611 _find_opp_table(dev);
612
613 if (IS_ERR(opp_table)) {
614 dev_err(dev, "%s: OPP table not found (%pe)\n", __func__,
615 opp_table);
616 return ERR_CAST(opp_table);
617 }
618
619 return _opp_table_find_key(opp_table, key, index, available, read,
620 compare, assert);
621}
622
623static struct dev_pm_opp *_find_key_exact(struct device *dev,
624 unsigned long key, int index, bool available,
625 unsigned long (*read)(struct dev_pm_opp *opp, int index),
626 bool (*assert)(struct opp_table *opp_table, unsigned int index))
627{
628 /*
629 * The value of key will be updated here, but will be ignored as the
630 * caller doesn't need it.
631 */
632 return _find_key(dev, &key, index, available, read, _compare_exact,
633 assert);
634}
635
636static struct dev_pm_opp *_opp_table_find_key_ceil(struct opp_table *opp_table,
637 unsigned long *key, int index, bool available,
638 unsigned long (*read)(struct dev_pm_opp *opp, int index),
639 bool (*assert)(struct opp_table *opp_table, unsigned int index))
640{
641 return _opp_table_find_key(opp_table, key, index, available, read,
642 _compare_ceil, assert);
643}
644
645static struct dev_pm_opp *_find_key_ceil(struct device *dev, unsigned long *key,
646 int index, bool available,
647 unsigned long (*read)(struct dev_pm_opp *opp, int index),
648 bool (*assert)(struct opp_table *opp_table, unsigned int index))
649{
650 return _find_key(dev, key, index, available, read, _compare_ceil,
651 assert);
652}
653
654static struct dev_pm_opp *_find_key_floor(struct device *dev,
655 unsigned long *key, int index, bool available,
656 unsigned long (*read)(struct dev_pm_opp *opp, int index),
657 bool (*assert)(struct opp_table *opp_table, unsigned int index))
658{
659 return _find_key(dev, key, index, available, read, _compare_floor,
660 assert);
661}
662
663/**
664 * dev_pm_opp_find_freq_exact() - search for an exact frequency
665 * @dev: device for which we do this operation
666 * @freq: frequency to search for
667 * @available: true/false - match for available opp
668 *
669 * Return: Searches for exact match in the opp table and returns pointer to the
670 * matching opp if found, else returns ERR_PTR in case of error and should
671 * be handled using IS_ERR. Error return values can be:
672 * EINVAL: for bad pointer
673 * ERANGE: no match found for search
674 * ENODEV: if device not found in list of registered devices
675 *
676 * Note: available is a modifier for the search. if available=true, then the
677 * match is for exact matching frequency and is available in the stored OPP
678 * table. if false, the match is for exact frequency which is not available.
679 *
680 * This provides a mechanism to enable an opp which is not available currently
681 * or the opposite as well.
682 *
683 * The callers are required to call dev_pm_opp_put() for the returned OPP after
684 * use.
685 */
686struct dev_pm_opp *dev_pm_opp_find_freq_exact(struct device *dev,
687 unsigned long freq, bool available)
688{
689 return _find_key_exact(dev, freq, 0, available, _read_freq,
690 assert_single_clk);
691}
692EXPORT_SYMBOL_GPL(dev_pm_opp_find_freq_exact);
693
694/**
695 * dev_pm_opp_find_key_exact() - Search for an OPP with exact key set
696 * @dev: Device for which the OPP is being searched
697 * @key: OPP key set to match
698 * @available: true/false - match for available OPP
699 *
700 * Search for an exact match of the key set in the OPP table.
701 *
702 * Return: A matching opp on success, else ERR_PTR in case of error.
703 * Possible error values:
704 * EINVAL: for bad pointers
705 * ERANGE: no match found for search
706 * ENODEV: if device not found in list of registered devices
707 *
708 * Note: 'available' is a modifier for the search. If 'available' == true,
709 * then the match is for exact matching key and is available in the stored
710 * OPP table. If false, the match is for exact key which is not available.
711 *
712 * This provides a mechanism to enable an OPP which is not available currently
713 * or the opposite as well.
714 *
715 * The callers are required to call dev_pm_opp_put() for the returned OPP after
716 * use.
717 */
718struct dev_pm_opp *dev_pm_opp_find_key_exact(struct device *dev,
719 struct dev_pm_opp_key *key,
720 bool available)
721{
722 struct opp_table *opp_table __free(put_opp_table) = _find_opp_table(dev);
723
724 if (IS_ERR(opp_table)) {
725 dev_err(dev, "%s: OPP table not found (%pe)\n", __func__,
726 opp_table);
727 return ERR_CAST(opp_table);
728 }
729
730 return _opp_table_find_opp_key(opp_table, key, available,
731 _read_opp_key, _compare_opp_key_exact,
732 assert_single_clk);
733}
734EXPORT_SYMBOL_GPL(dev_pm_opp_find_key_exact);
735
736/**
737 * dev_pm_opp_find_freq_exact_indexed() - Search for an exact freq for the
738 * clock corresponding to the index
739 * @dev: Device for which we do this operation
740 * @freq: frequency to search for
741 * @index: Clock index
742 * @available: true/false - match for available opp
743 *
744 * Search for the matching exact OPP for the clock corresponding to the
745 * specified index from a starting freq for a device.
746 *
747 * Return: matching *opp , else returns ERR_PTR in case of error and should be
748 * handled using IS_ERR. Error return values can be:
749 * EINVAL: for bad pointer
750 * ERANGE: no match found for search
751 * ENODEV: if device not found in list of registered devices
752 *
753 * The callers are required to call dev_pm_opp_put() for the returned OPP after
754 * use.
755 */
756struct dev_pm_opp *
757dev_pm_opp_find_freq_exact_indexed(struct device *dev, unsigned long freq,
758 u32 index, bool available)
759{
760 return _find_key_exact(dev, freq, index, available, _read_freq,
761 assert_clk_index);
762}
763EXPORT_SYMBOL_GPL(dev_pm_opp_find_freq_exact_indexed);
764
765static noinline struct dev_pm_opp *_find_freq_ceil(struct opp_table *opp_table,
766 unsigned long *freq)
767{
768 return _opp_table_find_key_ceil(opp_table, freq, 0, true, _read_freq,
769 assert_single_clk);
770}
771
772/**
773 * dev_pm_opp_find_freq_ceil() - Search for an rounded ceil freq
774 * @dev: device for which we do this operation
775 * @freq: Start frequency
776 *
777 * Search for the matching ceil *available* OPP from a starting freq
778 * for a device.
779 *
780 * Return: matching *opp and refreshes *freq accordingly, else returns
781 * ERR_PTR in case of error and should be handled using IS_ERR. Error return
782 * values can be:
783 * EINVAL: for bad pointer
784 * ERANGE: no match found for search
785 * ENODEV: if device not found in list of registered devices
786 *
787 * The callers are required to call dev_pm_opp_put() for the returned OPP after
788 * use.
789 */
790struct dev_pm_opp *dev_pm_opp_find_freq_ceil(struct device *dev,
791 unsigned long *freq)
792{
793 return _find_key_ceil(dev, freq, 0, true, _read_freq, assert_single_clk);
794}
795EXPORT_SYMBOL_GPL(dev_pm_opp_find_freq_ceil);
796
797/**
798 * dev_pm_opp_find_freq_ceil_indexed() - Search for a rounded ceil freq for the
799 * clock corresponding to the index
800 * @dev: Device for which we do this operation
801 * @freq: Start frequency
802 * @index: Clock index
803 *
804 * Search for the matching ceil *available* OPP for the clock corresponding to
805 * the specified index from a starting freq for a device.
806 *
807 * Return: matching *opp and refreshes *freq accordingly, else returns
808 * ERR_PTR in case of error and should be handled using IS_ERR. Error return
809 * values can be:
810 * EINVAL: for bad pointer
811 * ERANGE: no match found for search
812 * ENODEV: if device not found in list of registered devices
813 *
814 * The callers are required to call dev_pm_opp_put() for the returned OPP after
815 * use.
816 */
817struct dev_pm_opp *
818dev_pm_opp_find_freq_ceil_indexed(struct device *dev, unsigned long *freq,
819 u32 index)
820{
821 return _find_key_ceil(dev, freq, index, true, _read_freq,
822 assert_clk_index);
823}
824EXPORT_SYMBOL_GPL(dev_pm_opp_find_freq_ceil_indexed);
825
826/**
827 * dev_pm_opp_find_freq_floor() - Search for a rounded floor freq
828 * @dev: device for which we do this operation
829 * @freq: Start frequency
830 *
831 * Search for the matching floor *available* OPP from a starting freq
832 * for a device.
833 *
834 * Return: matching *opp and refreshes *freq accordingly, else returns
835 * ERR_PTR in case of error and should be handled using IS_ERR. Error return
836 * values can be:
837 * EINVAL: for bad pointer
838 * ERANGE: no match found for search
839 * ENODEV: if device not found in list of registered devices
840 *
841 * The callers are required to call dev_pm_opp_put() for the returned OPP after
842 * use.
843 */
844struct dev_pm_opp *dev_pm_opp_find_freq_floor(struct device *dev,
845 unsigned long *freq)
846{
847 return _find_key_floor(dev, freq, 0, true, _read_freq, assert_single_clk);
848}
849EXPORT_SYMBOL_GPL(dev_pm_opp_find_freq_floor);
850
851/**
852 * dev_pm_opp_find_freq_floor_indexed() - Search for a rounded floor freq for the
853 * clock corresponding to the index
854 * @dev: Device for which we do this operation
855 * @freq: Start frequency
856 * @index: Clock index
857 *
858 * Search for the matching floor *available* OPP for the clock corresponding to
859 * the specified index from a starting freq for a device.
860 *
861 * Return: matching *opp and refreshes *freq accordingly, else returns
862 * ERR_PTR in case of error and should be handled using IS_ERR. Error return
863 * values can be:
864 * EINVAL: for bad pointer
865 * ERANGE: no match found for search
866 * ENODEV: if device not found in list of registered devices
867 *
868 * The callers are required to call dev_pm_opp_put() for the returned OPP after
869 * use.
870 */
871struct dev_pm_opp *
872dev_pm_opp_find_freq_floor_indexed(struct device *dev, unsigned long *freq,
873 u32 index)
874{
875 return _find_key_floor(dev, freq, index, true, _read_freq, assert_clk_index);
876}
877EXPORT_SYMBOL_GPL(dev_pm_opp_find_freq_floor_indexed);
878
879/**
880 * dev_pm_opp_find_level_exact() - search for an exact level
881 * @dev: device for which we do this operation
882 * @level: level to search for
883 *
884 * Return: Searches for exact match in the opp table and returns pointer to the
885 * matching opp if found, else returns ERR_PTR in case of error and should
886 * be handled using IS_ERR. Error return values can be:
887 * EINVAL: for bad pointer
888 * ERANGE: no match found for search
889 * ENODEV: if device not found in list of registered devices
890 *
891 * The callers are required to call dev_pm_opp_put() for the returned OPP after
892 * use.
893 */
894struct dev_pm_opp *dev_pm_opp_find_level_exact(struct device *dev,
895 unsigned int level)
896{
897 return _find_key_exact(dev, level, 0, true, _read_level, NULL);
898}
899EXPORT_SYMBOL_GPL(dev_pm_opp_find_level_exact);
900
901/**
902 * dev_pm_opp_find_level_ceil() - search for an rounded up level
903 * @dev: device for which we do this operation
904 * @level: level to search for
905 *
906 * Return: Searches for rounded up match in the opp table and returns pointer
907 * to the matching opp if found, else returns ERR_PTR in case of error and
908 * should be handled using IS_ERR. Error return values can be:
909 * EINVAL: for bad pointer
910 * ERANGE: no match found for search
911 * ENODEV: if device not found in list of registered devices
912 *
913 * The callers are required to call dev_pm_opp_put() for the returned OPP after
914 * use.
915 */
916struct dev_pm_opp *dev_pm_opp_find_level_ceil(struct device *dev,
917 unsigned int *level)
918{
919 unsigned long temp = *level;
920 struct dev_pm_opp *opp;
921
922 opp = _find_key_ceil(dev, &temp, 0, true, _read_level, NULL);
923 if (IS_ERR(opp))
924 return opp;
925
926 /* False match */
927 if (temp == OPP_LEVEL_UNSET) {
928 dev_err(dev, "%s: OPP levels aren't available\n", __func__);
929 dev_pm_opp_put(opp);
930 return ERR_PTR(-ENODEV);
931 }
932
933 *level = temp;
934 return opp;
935}
936EXPORT_SYMBOL_GPL(dev_pm_opp_find_level_ceil);
937
938/**
939 * dev_pm_opp_find_level_floor() - Search for a rounded floor level
940 * @dev: device for which we do this operation
941 * @level: Start level
942 *
943 * Search for the matching floor *available* OPP from a starting level
944 * for a device.
945 *
946 * Return: matching *opp and refreshes *level accordingly, else returns
947 * ERR_PTR in case of error and should be handled using IS_ERR. Error return
948 * values can be:
949 * EINVAL: for bad pointer
950 * ERANGE: no match found for search
951 * ENODEV: if device not found in list of registered devices
952 *
953 * The callers are required to call dev_pm_opp_put() for the returned OPP after
954 * use.
955 */
956struct dev_pm_opp *dev_pm_opp_find_level_floor(struct device *dev,
957 unsigned int *level)
958{
959 unsigned long temp = *level;
960 struct dev_pm_opp *opp;
961
962 opp = _find_key_floor(dev, &temp, 0, true, _read_level, NULL);
963 *level = temp;
964 return opp;
965}
966EXPORT_SYMBOL_GPL(dev_pm_opp_find_level_floor);
967
968/**
969 * dev_pm_opp_find_bw_ceil() - Search for a rounded ceil bandwidth
970 * @dev: device for which we do this operation
971 * @bw: start bandwidth
972 * @index: which bandwidth to compare, in case of OPPs with several values
973 *
974 * Search for the matching floor *available* OPP from a starting bandwidth
975 * for a device.
976 *
977 * Return: matching *opp and refreshes *bw accordingly, else returns
978 * ERR_PTR in case of error and should be handled using IS_ERR. Error return
979 * values can be:
980 * EINVAL: for bad pointer
981 * ERANGE: no match found for search
982 * ENODEV: if device not found in list of registered devices
983 *
984 * The callers are required to call dev_pm_opp_put() for the returned OPP after
985 * use.
986 */
987struct dev_pm_opp *dev_pm_opp_find_bw_ceil(struct device *dev, unsigned int *bw,
988 int index)
989{
990 unsigned long temp = *bw;
991 struct dev_pm_opp *opp;
992
993 opp = _find_key_ceil(dev, &temp, index, true, _read_bw,
994 assert_bandwidth_index);
995 *bw = temp;
996 return opp;
997}
998EXPORT_SYMBOL_GPL(dev_pm_opp_find_bw_ceil);
999
1000/**
1001 * dev_pm_opp_find_bw_floor() - Search for a rounded floor bandwidth
1002 * @dev: device for which we do this operation
1003 * @bw: start bandwidth
1004 * @index: which bandwidth to compare, in case of OPPs with several values
1005 *
1006 * Search for the matching floor *available* OPP from a starting bandwidth
1007 * for a device.
1008 *
1009 * Return: matching *opp and refreshes *bw accordingly, else returns
1010 * ERR_PTR in case of error and should be handled using IS_ERR. Error return
1011 * values can be:
1012 * EINVAL: for bad pointer
1013 * ERANGE: no match found for search
1014 * ENODEV: if device not found in list of registered devices
1015 *
1016 * The callers are required to call dev_pm_opp_put() for the returned OPP after
1017 * use.
1018 */
1019struct dev_pm_opp *dev_pm_opp_find_bw_floor(struct device *dev,
1020 unsigned int *bw, int index)
1021{
1022 unsigned long temp = *bw;
1023 struct dev_pm_opp *opp;
1024
1025 opp = _find_key_floor(dev, &temp, index, true, _read_bw,
1026 assert_bandwidth_index);
1027 *bw = temp;
1028 return opp;
1029}
1030EXPORT_SYMBOL_GPL(dev_pm_opp_find_bw_floor);
1031
1032static int _set_opp_voltage(struct device *dev, struct regulator *reg,
1033 struct dev_pm_opp_supply *supply)
1034{
1035 int ret;
1036
1037 /* Regulator not available for device */
1038 if (IS_ERR(reg)) {
1039 dev_dbg(dev, "%s: regulator not available: %pe\n", __func__,
1040 reg);
1041 return 0;
1042 }
1043
1044 dev_dbg(dev, "%s: voltages (mV): %lu %lu %lu\n", __func__,
1045 supply->u_volt_min, supply->u_volt, supply->u_volt_max);
1046
1047 ret = regulator_set_voltage_triplet(reg, supply->u_volt_min,
1048 supply->u_volt, supply->u_volt_max);
1049 if (ret)
1050 dev_err(dev, "%s: failed to set voltage (%lu %lu %lu mV): %d\n",
1051 __func__, supply->u_volt_min, supply->u_volt,
1052 supply->u_volt_max, ret);
1053
1054 return ret;
1055}
1056
1057static int
1058_opp_config_clk_single(struct device *dev, struct opp_table *opp_table,
1059 struct dev_pm_opp *opp, void *data, bool scaling_down)
1060{
1061 unsigned long *target = data;
1062 unsigned long freq;
1063 int ret;
1064
1065 /* One of target and opp must be available */
1066 if (target) {
1067 freq = *target;
1068 } else if (opp) {
1069 freq = opp->rates[0];
1070 } else {
1071 WARN_ON(1);
1072 return -EINVAL;
1073 }
1074
1075 ret = clk_set_rate(opp_table->clk, freq);
1076 if (ret) {
1077 dev_err(dev, "%s: failed to set clock rate: %d\n", __func__,
1078 ret);
1079 } else {
1080 opp_table->current_rate_single_clk = freq;
1081 }
1082
1083 return ret;
1084}
1085
1086/*
1087 * Simple implementation for configuring multiple clocks. Configure clocks in
1088 * the order in which they are present in the array while scaling up.
1089 */
1090int dev_pm_opp_config_clks_simple(struct device *dev,
1091 struct opp_table *opp_table, struct dev_pm_opp *opp, void *data,
1092 bool scaling_down)
1093{
1094 int ret, i;
1095
1096 if (scaling_down) {
1097 for (i = opp_table->clk_count - 1; i >= 0; i--) {
1098 ret = clk_set_rate(opp_table->clks[i], opp->rates[i]);
1099 if (ret) {
1100 dev_err(dev, "%s: failed to set clock rate: %d\n", __func__,
1101 ret);
1102 return ret;
1103 }
1104 }
1105 } else {
1106 for (i = 0; i < opp_table->clk_count; i++) {
1107 ret = clk_set_rate(opp_table->clks[i], opp->rates[i]);
1108 if (ret) {
1109 dev_err(dev, "%s: failed to set clock rate: %d\n", __func__,
1110 ret);
1111 return ret;
1112 }
1113 }
1114 }
1115
1116 return 0;
1117}
1118EXPORT_SYMBOL_GPL(dev_pm_opp_config_clks_simple);
1119
1120static int _opp_config_regulator_single(struct device *dev,
1121 struct dev_pm_opp *old_opp, struct dev_pm_opp *new_opp,
1122 struct regulator **regulators, unsigned int count)
1123{
1124 struct regulator *reg = regulators[0];
1125 int ret;
1126
1127 /* This function only supports single regulator per device */
1128 if (WARN_ON(count > 1)) {
1129 dev_err(dev, "multiple regulators are not supported\n");
1130 return -EINVAL;
1131 }
1132
1133 ret = _set_opp_voltage(dev, reg, new_opp->supplies);
1134 if (ret)
1135 return ret;
1136
1137 /*
1138 * Enable the regulator after setting its voltages, otherwise it breaks
1139 * some boot-enabled regulators.
1140 */
1141 if (unlikely(!new_opp->opp_table->enabled)) {
1142 ret = regulator_enable(reg);
1143 if (ret < 0)
1144 dev_warn(dev, "Failed to enable regulator: %d", ret);
1145 }
1146
1147 return 0;
1148}
1149
1150static int _set_opp_bw(const struct opp_table *opp_table,
1151 struct dev_pm_opp *opp, struct device *dev)
1152{
1153 u32 avg, peak;
1154 int i, ret;
1155
1156 if (!opp_table->paths)
1157 return 0;
1158
1159 for (i = 0; i < opp_table->path_count; i++) {
1160 if (!opp) {
1161 avg = 0;
1162 peak = 0;
1163 } else {
1164 avg = opp->bandwidth[i].avg;
1165 peak = opp->bandwidth[i].peak;
1166 }
1167 ret = icc_set_bw(opp_table->paths[i], avg, peak);
1168 if (ret) {
1169 dev_err(dev, "Failed to %s bandwidth[%d]: %d\n",
1170 opp ? "set" : "remove", i, ret);
1171 return ret;
1172 }
1173 }
1174
1175 return 0;
1176}
1177
1178static int _set_opp_level(struct device *dev, struct dev_pm_opp *opp)
1179{
1180 unsigned int level = 0;
1181 int ret = 0;
1182
1183 if (opp) {
1184 if (opp->level == OPP_LEVEL_UNSET)
1185 return 0;
1186
1187 level = opp->level;
1188 }
1189
1190 /* Request a new performance state through the device's PM domain. */
1191 ret = dev_pm_domain_set_performance_state(dev, level);
1192 if (ret)
1193 dev_err(dev, "Failed to set performance state %u (%d)\n", level,
1194 ret);
1195
1196 return ret;
1197}
1198
1199/* This is only called for PM domain for now */
1200static int _set_required_opps(struct device *dev, struct opp_table *opp_table,
1201 struct dev_pm_opp *opp, bool up)
1202{
1203 struct device **devs = opp_table->required_devs;
1204 struct dev_pm_opp *required_opp;
1205 int index, target, delta, ret;
1206
1207 if (!devs)
1208 return 0;
1209
1210 /* required-opps not fully initialized yet */
1211 if (lazy_linking_pending(opp_table))
1212 return -EBUSY;
1213
1214 /* Scaling up? Set required OPPs in normal order, else reverse */
1215 if (up) {
1216 index = 0;
1217 target = opp_table->required_opp_count;
1218 delta = 1;
1219 } else {
1220 index = opp_table->required_opp_count - 1;
1221 target = -1;
1222 delta = -1;
1223 }
1224
1225 while (index != target) {
1226 if (devs[index]) {
1227 required_opp = opp ? opp->required_opps[index] : NULL;
1228
1229 ret = _set_opp_level(devs[index], required_opp);
1230 if (ret)
1231 return ret;
1232 }
1233
1234 index += delta;
1235 }
1236
1237 return 0;
1238}
1239
1240static void _find_current_opp(struct device *dev, struct opp_table *opp_table)
1241{
1242 struct dev_pm_opp *opp = ERR_PTR(-ENODEV);
1243 unsigned long freq;
1244
1245 if (!IS_ERR(opp_table->clk)) {
1246 freq = clk_get_rate(opp_table->clk);
1247 opp = _find_freq_ceil(opp_table, &freq);
1248 }
1249
1250 /*
1251 * Unable to find the current OPP ? Pick the first from the list since
1252 * it is in ascending order, otherwise rest of the code will need to
1253 * make special checks to validate current_opp.
1254 */
1255 if (IS_ERR(opp)) {
1256 guard(mutex)(&opp_table->lock);
1257 opp = dev_pm_opp_get(list_first_entry(&opp_table->opp_list,
1258 struct dev_pm_opp, node));
1259 }
1260
1261 opp_table->current_opp = opp;
1262}
1263
1264static int _disable_opp_table(struct device *dev, struct opp_table *opp_table)
1265{
1266 int ret;
1267
1268 if (!opp_table->enabled)
1269 return 0;
1270
1271 /*
1272 * Some drivers need to support cases where some platforms may
1273 * have OPP table for the device, while others don't and
1274 * opp_set_rate() just needs to behave like clk_set_rate().
1275 */
1276 if (!_get_opp_count(opp_table))
1277 return 0;
1278
1279 ret = _set_opp_bw(opp_table, NULL, dev);
1280 if (ret)
1281 return ret;
1282
1283 if (opp_table->regulators)
1284 regulator_disable(opp_table->regulators[0]);
1285
1286 ret = _set_opp_level(dev, NULL);
1287 if (ret)
1288 goto out;
1289
1290 ret = _set_required_opps(dev, opp_table, NULL, false);
1291
1292out:
1293 opp_table->enabled = false;
1294 return ret;
1295}
1296
1297static int _set_opp(struct device *dev, struct opp_table *opp_table,
1298 struct dev_pm_opp *opp, void *clk_data, bool forced)
1299{
1300 struct dev_pm_opp *old_opp;
1301 int scaling_down, ret;
1302
1303 if (unlikely(!opp))
1304 return _disable_opp_table(dev, opp_table);
1305
1306 /* Find the currently set OPP if we don't know already */
1307 if (unlikely(!opp_table->current_opp))
1308 _find_current_opp(dev, opp_table);
1309
1310 old_opp = opp_table->current_opp;
1311
1312 /* Return early if nothing to do */
1313 if (!forced && old_opp == opp && opp_table->enabled) {
1314 dev_dbg_ratelimited(dev, "%s: OPPs are same, nothing to do\n", __func__);
1315 return 0;
1316 }
1317
1318 dev_dbg(dev, "%s: switching OPP: Freq %lu -> %lu Hz, Level %u -> %u, Bw %u -> %u\n",
1319 __func__, old_opp->rates[0], opp->rates[0], old_opp->level,
1320 opp->level, old_opp->bandwidth ? old_opp->bandwidth[0].peak : 0,
1321 opp->bandwidth ? opp->bandwidth[0].peak : 0);
1322
1323 scaling_down = _opp_compare_key(opp_table, old_opp, opp);
1324 if (scaling_down == -1)
1325 scaling_down = 0;
1326
1327 /* Scaling up? Configure required OPPs before frequency */
1328 if (!scaling_down) {
1329 ret = _set_required_opps(dev, opp_table, opp, true);
1330 if (ret) {
1331 dev_err(dev, "Failed to set required opps: %d\n", ret);
1332 return ret;
1333 }
1334
1335 ret = _set_opp_level(dev, opp);
1336 if (ret)
1337 return ret;
1338
1339 ret = _set_opp_bw(opp_table, opp, dev);
1340 if (ret) {
1341 dev_err(dev, "Failed to set bw: %d\n", ret);
1342 return ret;
1343 }
1344
1345 if (opp_table->config_regulators) {
1346 ret = opp_table->config_regulators(dev, old_opp, opp,
1347 opp_table->regulators,
1348 opp_table->regulator_count);
1349 if (ret) {
1350 dev_err(dev, "Failed to set regulator voltages: %d\n",
1351 ret);
1352 return ret;
1353 }
1354 }
1355 }
1356
1357 if (opp_table->config_clks) {
1358 ret = opp_table->config_clks(dev, opp_table, opp, clk_data, scaling_down);
1359 if (ret)
1360 return ret;
1361 }
1362
1363 /* Scaling down? Configure required OPPs after frequency */
1364 if (scaling_down) {
1365 if (opp_table->config_regulators) {
1366 ret = opp_table->config_regulators(dev, old_opp, opp,
1367 opp_table->regulators,
1368 opp_table->regulator_count);
1369 if (ret) {
1370 dev_err(dev, "Failed to set regulator voltages: %d\n",
1371 ret);
1372 return ret;
1373 }
1374 }
1375
1376 ret = _set_opp_bw(opp_table, opp, dev);
1377 if (ret) {
1378 dev_err(dev, "Failed to set bw: %d\n", ret);
1379 return ret;
1380 }
1381
1382 ret = _set_opp_level(dev, opp);
1383 if (ret)
1384 return ret;
1385
1386 ret = _set_required_opps(dev, opp_table, opp, false);
1387 if (ret) {
1388 dev_err(dev, "Failed to set required opps: %d\n", ret);
1389 return ret;
1390 }
1391 }
1392
1393 opp_table->enabled = true;
1394 dev_pm_opp_put(old_opp);
1395
1396 /* Make sure current_opp doesn't get freed */
1397 opp_table->current_opp = dev_pm_opp_get(opp);
1398
1399 return ret;
1400}
1401
1402/**
1403 * dev_pm_opp_set_rate() - Configure new OPP based on frequency
1404 * @dev: device for which we do this operation
1405 * @target_freq: frequency to achieve
1406 *
1407 * This configures the power-supplies to the levels specified by the OPP
1408 * corresponding to the target_freq, and programs the clock to a value <=
1409 * target_freq, as rounded by clk_round_rate(). Device wanting to run at fmax
1410 * provided by the opp, should have already rounded to the target OPP's
1411 * frequency.
1412 */
1413int dev_pm_opp_set_rate(struct device *dev, unsigned long target_freq)
1414{
1415 struct opp_table *opp_table __free(put_opp_table) =
1416 _find_opp_table(dev);
1417 struct dev_pm_opp *opp __free(put_opp) = NULL;
1418 unsigned long freq = 0, temp_freq;
1419 bool forced = false;
1420
1421 if (IS_ERR(opp_table)) {
1422 dev_err(dev, "%s: device's opp table doesn't exist\n", __func__);
1423 return PTR_ERR(opp_table);
1424 }
1425
1426 if (target_freq) {
1427 /*
1428 * For IO devices which require an OPP on some platforms/SoCs
1429 * while just needing to scale the clock on some others
1430 * we look for empty OPP tables with just a clock handle and
1431 * scale only the clk. This makes dev_pm_opp_set_rate()
1432 * equivalent to a clk_set_rate()
1433 */
1434 if (!_get_opp_count(opp_table)) {
1435 return opp_table->config_clks(dev, opp_table, NULL,
1436 &target_freq, false);
1437 }
1438
1439 freq = clk_round_rate(opp_table->clk, target_freq);
1440 if ((long)freq <= 0)
1441 freq = target_freq;
1442
1443 /*
1444 * The clock driver may support finer resolution of the
1445 * frequencies than the OPP table, don't update the frequency we
1446 * pass to clk_set_rate() here.
1447 */
1448 temp_freq = freq;
1449 opp = _find_freq_ceil(opp_table, &temp_freq);
1450 if (IS_ERR(opp)) {
1451 dev_err(dev, "%s: failed to find OPP for freq %lu (%pe)\n",
1452 __func__, freq, opp);
1453 return PTR_ERR(opp);
1454 }
1455
1456 /*
1457 * An OPP entry specifies the highest frequency at which other
1458 * properties of the OPP entry apply. Even if the new OPP is
1459 * same as the old one, we may still reach here for a different
1460 * value of the frequency. In such a case, do not abort but
1461 * configure the hardware to the desired frequency forcefully.
1462 */
1463 forced = opp_table->current_rate_single_clk != freq;
1464 }
1465
1466 return _set_opp(dev, opp_table, opp, &freq, forced);
1467}
1468EXPORT_SYMBOL_GPL(dev_pm_opp_set_rate);
1469
1470/**
1471 * dev_pm_opp_set_opp() - Configure device for OPP
1472 * @dev: device for which we do this operation
1473 * @opp: OPP to set to
1474 *
1475 * This configures the device based on the properties of the OPP passed to this
1476 * routine.
1477 *
1478 * Return: 0 on success, a negative error number otherwise.
1479 */
1480int dev_pm_opp_set_opp(struct device *dev, struct dev_pm_opp *opp)
1481{
1482 struct opp_table *opp_table __free(put_opp_table) =
1483 _find_opp_table(dev);
1484
1485 if (IS_ERR(opp_table)) {
1486 dev_err(dev, "%s: device opp doesn't exist\n", __func__);
1487 return PTR_ERR(opp_table);
1488 }
1489
1490 return _set_opp(dev, opp_table, opp, NULL, false);
1491}
1492EXPORT_SYMBOL_GPL(dev_pm_opp_set_opp);
1493
1494/* OPP-dev Helpers */
1495static void _remove_opp_dev(struct opp_device *opp_dev,
1496 struct opp_table *opp_table)
1497{
1498 opp_debug_unregister(opp_dev, opp_table);
1499 list_del(&opp_dev->node);
1500 kfree(opp_dev);
1501}
1502
1503struct opp_device *_add_opp_dev(const struct device *dev,
1504 struct opp_table *opp_table)
1505{
1506 struct opp_device *opp_dev;
1507
1508 opp_dev = kzalloc_obj(*opp_dev);
1509 if (!opp_dev)
1510 return NULL;
1511
1512 /* Initialize opp-dev */
1513 opp_dev->dev = dev;
1514
1515 scoped_guard(mutex, &opp_table->lock)
1516 list_add(&opp_dev->node, &opp_table->dev_list);
1517
1518 /* Create debugfs entries for the opp_table */
1519 opp_debug_register(opp_dev, opp_table);
1520
1521 return opp_dev;
1522}
1523
1524static struct opp_table *_allocate_opp_table(struct device *dev, int index)
1525{
1526 struct opp_table *opp_table;
1527 struct opp_device *opp_dev;
1528 int ret;
1529
1530 /*
1531 * Allocate a new OPP table. In the infrequent case where a new
1532 * device is needed to be added, we pay this penalty.
1533 */
1534 opp_table = kzalloc_obj(*opp_table);
1535 if (!opp_table)
1536 return ERR_PTR(-ENOMEM);
1537
1538 mutex_init(&opp_table->lock);
1539 INIT_LIST_HEAD(&opp_table->dev_list);
1540 INIT_LIST_HEAD(&opp_table->lazy);
1541
1542 opp_table->clk = ERR_PTR(-ENODEV);
1543
1544 /* Mark regulator count uninitialized */
1545 opp_table->regulator_count = -1;
1546
1547 opp_dev = _add_opp_dev(dev, opp_table);
1548 if (!opp_dev) {
1549 ret = -ENOMEM;
1550 goto err;
1551 }
1552
1553 _of_init_opp_table(opp_table, dev, index);
1554
1555 /* Find interconnect path(s) for the device */
1556 ret = dev_pm_opp_of_find_icc_paths(dev, opp_table);
1557 if (ret) {
1558 if (ret == -EPROBE_DEFER)
1559 goto remove_opp_dev;
1560
1561 dev_warn(dev, "%s: Error finding interconnect paths: %d\n",
1562 __func__, ret);
1563 }
1564
1565 BLOCKING_INIT_NOTIFIER_HEAD(&opp_table->head);
1566 INIT_LIST_HEAD(&opp_table->opp_list);
1567 kref_init(&opp_table->kref);
1568
1569 return opp_table;
1570
1571remove_opp_dev:
1572 _of_clear_opp_table(opp_table);
1573 _remove_opp_dev(opp_dev, opp_table);
1574 mutex_destroy(&opp_table->lock);
1575err:
1576 kfree(opp_table);
1577 return ERR_PTR(ret);
1578}
1579
1580static struct opp_table *_update_opp_table_clk(struct device *dev,
1581 struct opp_table *opp_table,
1582 bool getclk)
1583{
1584 int ret;
1585
1586 /*
1587 * Return early if we don't need to get clk or we have already done it
1588 * earlier.
1589 */
1590 if (!getclk || IS_ERR(opp_table) || !IS_ERR(opp_table->clk) ||
1591 opp_table->clks)
1592 return opp_table;
1593
1594 /*
1595 * There are few platforms which don't want the OPP core to manage
1596 * device's clock settings. In such cases neither the platform
1597 * provides the clks explicitly to us, nor the DT contains a valid
1598 * clk entry. The OPP nodes in DT may still contain "opp-hz" property
1599 * though, which we need to parse and allow the platform to find an
1600 * OPP based on freq later on.
1601 *
1602 * This is a simple solution to take care of such corner cases, i.e.
1603 * make the clk_count 1, which lets us allocate space for frequency
1604 * in opp->rates and also parse the entries in DT. Use
1605 * clk_get_optional() instead of clk_get() so opp_table->clk stays
1606 * NULL for such devices, instead of holding an ERR_PTR(-ENOENT) that
1607 * consumers must remember to special-case.
1608 */
1609 opp_table->clk = clk_get_optional(dev, NULL);
1610
1611 if (IS_ERR(opp_table->clk)) {
1612 ret = dev_err_probe(dev, PTR_ERR(opp_table->clk), "Couldn't find clock\n");
1613 dev_pm_opp_put_opp_table(opp_table);
1614 return ERR_PTR(ret);
1615 }
1616
1617 if (opp_table->clk)
1618 opp_table->config_clks = _opp_config_clk_single;
1619
1620 opp_table->clk_count = 1;
1621
1622 return opp_table;
1623}
1624
1625/*
1626 * We need to make sure that the OPP table for a device doesn't get added twice,
1627 * if this routine gets called in parallel with the same device pointer.
1628 *
1629 * The simplest way to enforce that is to perform everything (find existing
1630 * table and if not found, create a new one) under the opp_table_lock, so only
1631 * one creator gets access to the same. But that expands the critical section
1632 * under the lock and may end up causing circular dependencies with frameworks
1633 * like debugfs, interconnect or clock framework as they may be direct or
1634 * indirect users of OPP core.
1635 *
1636 * And for that reason we have to go for a bit tricky implementation here, which
1637 * uses the opp_tables_busy flag to indicate if another creator is in the middle
1638 * of adding an OPP table and others should wait for it to finish.
1639 */
1640struct opp_table *_add_opp_table_indexed(struct device *dev, int index,
1641 bool getclk)
1642{
1643 struct opp_table *opp_table;
1644
1645again:
1646 mutex_lock(&opp_table_lock);
1647
1648 opp_table = _find_opp_table_unlocked(dev);
1649 if (!IS_ERR(opp_table))
1650 goto unlock;
1651
1652 /*
1653 * The opp_tables list or an OPP table's dev_list is getting updated by
1654 * another user, wait for it to finish.
1655 */
1656 if (unlikely(opp_tables_busy)) {
1657 mutex_unlock(&opp_table_lock);
1658 cpu_relax();
1659 goto again;
1660 }
1661
1662 opp_tables_busy = true;
1663 opp_table = _managed_opp(dev, index);
1664
1665 /* Drop the lock to reduce the size of critical section */
1666 mutex_unlock(&opp_table_lock);
1667
1668 if (opp_table) {
1669 if (!_add_opp_dev(dev, opp_table)) {
1670 dev_pm_opp_put_opp_table(opp_table);
1671 opp_table = ERR_PTR(-ENOMEM);
1672 }
1673
1674 mutex_lock(&opp_table_lock);
1675 } else {
1676 opp_table = _allocate_opp_table(dev, index);
1677
1678 mutex_lock(&opp_table_lock);
1679 if (!IS_ERR(opp_table))
1680 list_add(&opp_table->node, &opp_tables);
1681 }
1682
1683 opp_tables_busy = false;
1684
1685unlock:
1686 mutex_unlock(&opp_table_lock);
1687
1688 return _update_opp_table_clk(dev, opp_table, getclk);
1689}
1690
1691static struct opp_table *_add_opp_table(struct device *dev, bool getclk)
1692{
1693 return _add_opp_table_indexed(dev, 0, getclk);
1694}
1695
1696struct opp_table *dev_pm_opp_get_opp_table(struct device *dev)
1697{
1698 return _find_opp_table(dev);
1699}
1700EXPORT_SYMBOL_GPL(dev_pm_opp_get_opp_table);
1701
1702static void _opp_table_kref_release(struct kref *kref)
1703{
1704 struct opp_table *opp_table = container_of(kref, struct opp_table, kref);
1705 struct opp_device *opp_dev, *temp;
1706 int i;
1707
1708 /* Drop the lock as soon as we can */
1709 list_del(&opp_table->node);
1710 mutex_unlock(&opp_table_lock);
1711
1712 if (opp_table->current_opp)
1713 dev_pm_opp_put(opp_table->current_opp);
1714
1715 _of_clear_opp_table(opp_table);
1716
1717 /* Release automatically acquired single clk */
1718 if (!IS_ERR(opp_table->clk))
1719 clk_put(opp_table->clk);
1720
1721 if (opp_table->paths) {
1722 for (i = 0; i < opp_table->path_count; i++)
1723 icc_put(opp_table->paths[i]);
1724 kfree(opp_table->paths);
1725 }
1726
1727 WARN_ON(!list_empty(&opp_table->opp_list));
1728
1729 list_for_each_entry_safe(opp_dev, temp, &opp_table->dev_list, node)
1730 _remove_opp_dev(opp_dev, opp_table);
1731
1732 mutex_destroy(&opp_table->lock);
1733 kfree(opp_table);
1734}
1735
1736struct opp_table *dev_pm_opp_get_opp_table_ref(struct opp_table *opp_table)
1737{
1738 kref_get(&opp_table->kref);
1739 return opp_table;
1740}
1741EXPORT_SYMBOL_GPL(dev_pm_opp_get_opp_table_ref);
1742
1743void dev_pm_opp_put_opp_table(struct opp_table *opp_table)
1744{
1745 kref_put_mutex(&opp_table->kref, _opp_table_kref_release,
1746 &opp_table_lock);
1747}
1748EXPORT_SYMBOL_GPL(dev_pm_opp_put_opp_table);
1749
1750void _opp_free(struct dev_pm_opp *opp)
1751{
1752 kfree(opp);
1753}
1754
1755static void _opp_kref_release(struct kref *kref)
1756{
1757 struct dev_pm_opp *opp = container_of(kref, struct dev_pm_opp, kref);
1758 struct opp_table *opp_table = opp->opp_table;
1759
1760 list_del(&opp->node);
1761 mutex_unlock(&opp_table->lock);
1762
1763 /*
1764 * Notify the changes in the availability of the operable
1765 * frequency/voltage list.
1766 */
1767 blocking_notifier_call_chain(&opp_table->head, OPP_EVENT_REMOVE, opp);
1768 _of_clear_opp(opp_table, opp);
1769 opp_debug_remove_one(opp);
1770 kfree(opp);
1771}
1772
1773struct dev_pm_opp *dev_pm_opp_get(struct dev_pm_opp *opp)
1774{
1775 kref_get(&opp->kref);
1776 return opp;
1777}
1778EXPORT_SYMBOL_GPL(dev_pm_opp_get);
1779
1780void dev_pm_opp_put(struct dev_pm_opp *opp)
1781{
1782 kref_put_mutex(&opp->kref, _opp_kref_release, &opp->opp_table->lock);
1783}
1784EXPORT_SYMBOL_GPL(dev_pm_opp_put);
1785
1786/**
1787 * dev_pm_opp_remove() - Remove an OPP from OPP table
1788 * @dev: device for which we do this operation
1789 * @freq: OPP to remove with matching 'freq'
1790 *
1791 * This function removes an opp from the opp table.
1792 */
1793void dev_pm_opp_remove(struct device *dev, unsigned long freq)
1794{
1795 struct dev_pm_opp *opp = NULL, *iter;
1796
1797 struct opp_table *opp_table __free(put_opp_table) =
1798 _find_opp_table(dev);
1799
1800 if (IS_ERR(opp_table))
1801 return;
1802
1803 if (!assert_single_clk(opp_table, 0))
1804 return;
1805
1806 scoped_guard(mutex, &opp_table->lock) {
1807 list_for_each_entry(iter, &opp_table->opp_list, node) {
1808 if (iter->rates[0] == freq) {
1809 opp = iter;
1810 break;
1811 }
1812 }
1813 }
1814
1815 if (opp) {
1816 dev_pm_opp_put(opp);
1817
1818 /* Drop the reference taken by dev_pm_opp_add() */
1819 dev_pm_opp_put_opp_table(opp_table);
1820 } else {
1821 dev_warn(dev, "%s: Couldn't find OPP with freq: %lu\n",
1822 __func__, freq);
1823 }
1824}
1825EXPORT_SYMBOL_GPL(dev_pm_opp_remove);
1826
1827static struct dev_pm_opp *_opp_get_next(struct opp_table *opp_table,
1828 bool dynamic)
1829{
1830 struct dev_pm_opp *opp;
1831
1832 guard(mutex)(&opp_table->lock);
1833
1834 list_for_each_entry(opp, &opp_table->opp_list, node) {
1835 /*
1836 * Refcount must be dropped only once for each OPP by OPP core,
1837 * do that with help of "removed" flag.
1838 */
1839 if (!opp->removed && dynamic == opp->dynamic)
1840 return opp;
1841 }
1842
1843 return NULL;
1844}
1845
1846/*
1847 * Can't call dev_pm_opp_put() from under the lock as debugfs removal needs to
1848 * happen lock less to avoid circular dependency issues. This routine must be
1849 * called without the opp_table->lock held.
1850 */
1851static void _opp_remove_all(struct opp_table *opp_table, bool dynamic)
1852{
1853 struct dev_pm_opp *opp;
1854
1855 while ((opp = _opp_get_next(opp_table, dynamic))) {
1856 opp->removed = true;
1857 dev_pm_opp_put(opp);
1858
1859 /* Drop the references taken by dev_pm_opp_add() */
1860 if (dynamic)
1861 dev_pm_opp_put_opp_table(opp_table);
1862 }
1863}
1864
1865bool _opp_remove_all_static(struct opp_table *opp_table)
1866{
1867 scoped_guard(mutex, &opp_table->lock) {
1868 if (!opp_table->parsed_static_opps)
1869 return false;
1870
1871 if (--opp_table->parsed_static_opps)
1872 return true;
1873 }
1874
1875 _opp_remove_all(opp_table, false);
1876 return true;
1877}
1878
1879/**
1880 * dev_pm_opp_remove_all_dynamic() - Remove all dynamically created OPPs
1881 * @dev: device for which we do this operation
1882 *
1883 * This function removes all dynamically created OPPs from the opp table.
1884 */
1885void dev_pm_opp_remove_all_dynamic(struct device *dev)
1886{
1887 struct opp_table *opp_table __free(put_opp_table) =
1888 _find_opp_table(dev);
1889
1890 if (IS_ERR(opp_table))
1891 return;
1892
1893 _opp_remove_all(opp_table, true);
1894}
1895EXPORT_SYMBOL_GPL(dev_pm_opp_remove_all_dynamic);
1896
1897struct dev_pm_opp *_opp_allocate(struct opp_table *opp_table)
1898{
1899 struct dev_pm_opp *opp;
1900 int supply_count, supply_size, icc_size, clk_size;
1901
1902 /* Allocate space for at least one supply */
1903 supply_count = opp_table->regulator_count > 0 ?
1904 opp_table->regulator_count : 1;
1905 supply_size = sizeof(*opp->supplies) * supply_count;
1906 clk_size = sizeof(*opp->rates) * opp_table->clk_count;
1907 icc_size = sizeof(*opp->bandwidth) * opp_table->path_count;
1908
1909 /* allocate new OPP node and supplies structures */
1910 opp = kzalloc(sizeof(*opp) + supply_size + clk_size + icc_size, GFP_KERNEL);
1911 if (!opp)
1912 return NULL;
1913
1914 /* Put the supplies, bw and clock at the end of the OPP structure */
1915 opp->supplies = (struct dev_pm_opp_supply *)(opp + 1);
1916
1917 opp->rates = (unsigned long *)(opp->supplies + supply_count);
1918
1919 if (icc_size)
1920 opp->bandwidth = (struct dev_pm_opp_icc_bw *)(opp->rates + opp_table->clk_count);
1921
1922 INIT_LIST_HEAD(&opp->node);
1923
1924 opp->level = OPP_LEVEL_UNSET;
1925
1926 return opp;
1927}
1928
1929static bool _opp_supported_by_regulators(struct dev_pm_opp *opp,
1930 struct opp_table *opp_table)
1931{
1932 struct regulator *reg;
1933 int i;
1934
1935 if (!opp_table->regulators)
1936 return true;
1937
1938 for (i = 0; i < opp_table->regulator_count; i++) {
1939 reg = opp_table->regulators[i];
1940
1941 if (!regulator_is_supported_voltage(reg,
1942 opp->supplies[i].u_volt_min,
1943 opp->supplies[i].u_volt_max)) {
1944 pr_warn("%s: OPP minuV: %lu maxuV: %lu, not supported by regulator\n",
1945 __func__, opp->supplies[i].u_volt_min,
1946 opp->supplies[i].u_volt_max);
1947 return false;
1948 }
1949 }
1950
1951 return true;
1952}
1953
1954static int _opp_compare_rate(struct opp_table *opp_table,
1955 struct dev_pm_opp *opp1, struct dev_pm_opp *opp2)
1956{
1957 int i;
1958
1959 for (i = 0; i < opp_table->clk_count; i++) {
1960 if (opp1->rates[i] != opp2->rates[i])
1961 return opp1->rates[i] < opp2->rates[i] ? -1 : 1;
1962 }
1963
1964 /* Same rates for both OPPs */
1965 return 0;
1966}
1967
1968static int _opp_compare_bw(struct opp_table *opp_table, struct dev_pm_opp *opp1,
1969 struct dev_pm_opp *opp2)
1970{
1971 int i;
1972
1973 for (i = 0; i < opp_table->path_count; i++) {
1974 if (opp1->bandwidth[i].peak != opp2->bandwidth[i].peak)
1975 return opp1->bandwidth[i].peak < opp2->bandwidth[i].peak ? -1 : 1;
1976 }
1977
1978 /* Same bw for both OPPs */
1979 return 0;
1980}
1981
1982/*
1983 * Returns
1984 * 0: opp1 == opp2
1985 * 1: opp1 > opp2
1986 * -1: opp1 < opp2
1987 */
1988int _opp_compare_key(struct opp_table *opp_table, struct dev_pm_opp *opp1,
1989 struct dev_pm_opp *opp2)
1990{
1991 int ret;
1992
1993 ret = _opp_compare_rate(opp_table, opp1, opp2);
1994 if (ret)
1995 return ret;
1996
1997 ret = _opp_compare_bw(opp_table, opp1, opp2);
1998 if (ret)
1999 return ret;
2000
2001 if (opp1->level != opp2->level)
2002 return opp1->level < opp2->level ? -1 : 1;
2003
2004 /* Duplicate OPPs */
2005 return 0;
2006}
2007
2008static int _opp_is_duplicate(struct device *dev, struct dev_pm_opp *new_opp,
2009 struct opp_table *opp_table,
2010 struct list_head **head)
2011{
2012 struct dev_pm_opp *opp;
2013 int opp_cmp;
2014
2015 /*
2016 * Insert new OPP in order of increasing frequency and discard if
2017 * already present.
2018 *
2019 * Need to use &opp_table->opp_list in the condition part of the 'for'
2020 * loop, don't replace it with head otherwise it will become an infinite
2021 * loop.
2022 */
2023 list_for_each_entry(opp, &opp_table->opp_list, node) {
2024 opp_cmp = _opp_compare_key(opp_table, new_opp, opp);
2025 if (opp_cmp > 0) {
2026 *head = &opp->node;
2027 continue;
2028 }
2029
2030 if (opp_cmp < 0)
2031 return 0;
2032
2033 /* Duplicate OPPs */
2034 dev_warn(dev, "%s: duplicate OPPs detected. Existing: freq: %lu, volt: %lu, enabled: %d. New: freq: %lu, volt: %lu, enabled: %d\n",
2035 __func__, opp->rates[0], opp->supplies[0].u_volt,
2036 opp->available, new_opp->rates[0],
2037 new_opp->supplies[0].u_volt, new_opp->available);
2038
2039 /* Should we compare voltages for all regulators here ? */
2040 return opp->available &&
2041 new_opp->supplies[0].u_volt == opp->supplies[0].u_volt ? -EBUSY : -EEXIST;
2042 }
2043
2044 return 0;
2045}
2046
2047void _required_opps_available(struct dev_pm_opp *opp, int count)
2048{
2049 int i;
2050
2051 for (i = 0; i < count; i++) {
2052 if (opp->required_opps[i]->available)
2053 continue;
2054
2055 opp->available = false;
2056 pr_warn("%s: OPP not supported by required OPP %pOF (%lu)\n",
2057 __func__, opp->required_opps[i]->np, opp->rates[0]);
2058 return;
2059 }
2060}
2061
2062/*
2063 * Returns:
2064 * 0: On success. And appropriate error message for duplicate OPPs.
2065 * -EBUSY: For OPP with same freq/volt and is available. The callers of
2066 * _opp_add() must return 0 if they receive -EBUSY from it. This is to make
2067 * sure we don't print error messages unnecessarily if different parts of
2068 * kernel try to initialize the OPP table.
2069 * -EEXIST: For OPP with same freq but different volt or is unavailable. This
2070 * should be considered an error by the callers of _opp_add().
2071 */
2072int _opp_add(struct device *dev, struct dev_pm_opp *new_opp,
2073 struct opp_table *opp_table)
2074{
2075 struct list_head *head;
2076 int ret;
2077
2078 scoped_guard(mutex, &opp_table->lock) {
2079 head = &opp_table->opp_list;
2080
2081 ret = _opp_is_duplicate(dev, new_opp, opp_table, &head);
2082 if (ret)
2083 return ret;
2084
2085 list_add(&new_opp->node, head);
2086 new_opp->opp_table = opp_table;
2087 kref_init(&new_opp->kref);
2088 }
2089
2090 opp_debug_create_one(new_opp, opp_table);
2091
2092 if (!_opp_supported_by_regulators(new_opp, opp_table)) {
2093 new_opp->available = false;
2094 dev_warn(dev, "%s: OPP not supported by regulators (%lu)\n",
2095 __func__, new_opp->rates[0]);
2096 }
2097
2098 /* required-opps not fully initialized yet */
2099 if (lazy_linking_pending(opp_table))
2100 return 0;
2101
2102 _required_opps_available(new_opp, opp_table->required_opp_count);
2103
2104 return 0;
2105}
2106
2107/**
2108 * _opp_add_v1() - Allocate a OPP based on v1 bindings.
2109 * @opp_table: OPP table
2110 * @dev: device for which we do this operation
2111 * @data: The OPP data for the OPP to add
2112 * @dynamic: Dynamically added OPPs.
2113 *
2114 * This function adds an opp definition to the opp table and returns status.
2115 * The opp is made available by default and it can be controlled using
2116 * dev_pm_opp_enable/disable functions and may be removed by dev_pm_opp_remove.
2117 *
2118 * NOTE: "dynamic" parameter impacts OPPs added by the dev_pm_opp_of_add_table
2119 * and freed by dev_pm_opp_of_remove_table.
2120 *
2121 * Return:
2122 * 0 On success OR
2123 * Duplicate OPPs (both freq and volt are same) and opp->available
2124 * -EEXIST Freq are same and volt are different OR
2125 * Duplicate OPPs (both freq and volt are same) and !opp->available
2126 * -ENOMEM Memory allocation failure
2127 */
2128int _opp_add_v1(struct opp_table *opp_table, struct device *dev,
2129 struct dev_pm_opp_data *data, bool dynamic)
2130{
2131 struct dev_pm_opp *new_opp;
2132 unsigned long tol, u_volt = data->u_volt;
2133 int ret;
2134
2135 if (!assert_single_clk(opp_table, 0))
2136 return -EINVAL;
2137
2138 new_opp = _opp_allocate(opp_table);
2139 if (!new_opp)
2140 return -ENOMEM;
2141
2142 /* populate the opp table */
2143 new_opp->rates[0] = data->freq;
2144 new_opp->level = data->level;
2145 new_opp->turbo = data->turbo;
2146 tol = u_volt * opp_table->voltage_tolerance_v1 / 100;
2147 new_opp->supplies[0].u_volt = u_volt;
2148 new_opp->supplies[0].u_volt_min = u_volt - tol;
2149 new_opp->supplies[0].u_volt_max = u_volt + tol;
2150 new_opp->available = true;
2151 new_opp->dynamic = dynamic;
2152
2153 ret = _opp_add(dev, new_opp, opp_table);
2154 if (ret) {
2155 /* Don't return error for duplicate OPPs */
2156 if (ret == -EBUSY)
2157 ret = 0;
2158 goto free_opp;
2159 }
2160
2161 /*
2162 * Notify the changes in the availability of the operable
2163 * frequency/voltage list.
2164 */
2165 blocking_notifier_call_chain(&opp_table->head, OPP_EVENT_ADD, new_opp);
2166 return 0;
2167
2168free_opp:
2169 _opp_free(new_opp);
2170
2171 return ret;
2172}
2173
2174/*
2175 * This is required only for the V2 bindings, and it enables a platform to
2176 * specify the hierarchy of versions it supports. OPP layer will then enable
2177 * OPPs, which are available for those versions, based on its 'opp-supported-hw'
2178 * property.
2179 */
2180static int _opp_set_supported_hw(struct opp_table *opp_table,
2181 const u32 *versions, unsigned int count)
2182{
2183 /* Another CPU that shares the OPP table has set the property ? */
2184 if (opp_table->supported_hw)
2185 return 0;
2186
2187 opp_table->supported_hw = kmemdup_array(versions, count,
2188 sizeof(*versions), GFP_KERNEL);
2189 if (!opp_table->supported_hw)
2190 return -ENOMEM;
2191
2192 opp_table->supported_hw_count = count;
2193
2194 return 0;
2195}
2196
2197static void _opp_put_supported_hw(struct opp_table *opp_table)
2198{
2199 if (opp_table->supported_hw) {
2200 kfree(opp_table->supported_hw);
2201 opp_table->supported_hw = NULL;
2202 opp_table->supported_hw_count = 0;
2203 }
2204}
2205
2206/*
2207 * This is required only for the V2 bindings, and it enables a platform to
2208 * specify the extn to be used for certain property names. The properties to
2209 * which the extension will apply are opp-microvolt and opp-microamp. OPP core
2210 * should postfix the property name with -<name> while looking for them.
2211 */
2212static int _opp_set_prop_name(struct opp_table *opp_table, const char *name)
2213{
2214 /* Another CPU that shares the OPP table has set the property ? */
2215 if (!opp_table->prop_name) {
2216 opp_table->prop_name = kstrdup(name, GFP_KERNEL);
2217 if (!opp_table->prop_name)
2218 return -ENOMEM;
2219 }
2220
2221 return 0;
2222}
2223
2224static void _opp_put_prop_name(struct opp_table *opp_table)
2225{
2226 if (opp_table->prop_name) {
2227 kfree(opp_table->prop_name);
2228 opp_table->prop_name = NULL;
2229 }
2230}
2231
2232/*
2233 * In order to support OPP switching, OPP layer needs to know the name of the
2234 * device's regulators, as the core would be required to switch voltages as
2235 * well.
2236 *
2237 * This must be called before any OPPs are initialized for the device.
2238 */
2239static int _opp_set_regulators(struct opp_table *opp_table, struct device *dev,
2240 const char * const names[])
2241{
2242 const char * const *temp = names;
2243 struct regulator *reg;
2244 int count = 0, ret, i;
2245
2246 /* Count number of regulators */
2247 while (*temp++)
2248 count++;
2249
2250 if (!count)
2251 return -EINVAL;
2252
2253 /* Another CPU that shares the OPP table has set the regulators ? */
2254 if (opp_table->regulators)
2255 return 0;
2256
2257 opp_table->regulators = kmalloc_objs(*opp_table->regulators, count);
2258 if (!opp_table->regulators)
2259 return -ENOMEM;
2260
2261 for (i = 0; i < count; i++) {
2262 reg = regulator_get_optional(dev, names[i]);
2263 if (IS_ERR(reg)) {
2264 ret = dev_err_probe(dev, PTR_ERR(reg),
2265 "%s: no regulator (%s) found\n",
2266 __func__, names[i]);
2267 goto free_regulators;
2268 }
2269
2270 opp_table->regulators[i] = reg;
2271 }
2272
2273 opp_table->regulator_count = count;
2274
2275 /* Set generic config_regulators() for single regulators here */
2276 if (count == 1)
2277 opp_table->config_regulators = _opp_config_regulator_single;
2278
2279 return 0;
2280
2281free_regulators:
2282 while (i != 0)
2283 regulator_put(opp_table->regulators[--i]);
2284
2285 kfree(opp_table->regulators);
2286 opp_table->regulators = NULL;
2287 opp_table->regulator_count = -1;
2288
2289 return ret;
2290}
2291
2292static void _opp_put_regulators(struct opp_table *opp_table)
2293{
2294 int i;
2295
2296 if (!opp_table->regulators)
2297 return;
2298
2299 if (opp_table->enabled) {
2300 for (i = opp_table->regulator_count - 1; i >= 0; i--)
2301 regulator_disable(opp_table->regulators[i]);
2302 }
2303
2304 for (i = opp_table->regulator_count - 1; i >= 0; i--)
2305 regulator_put(opp_table->regulators[i]);
2306
2307 kfree(opp_table->regulators);
2308 opp_table->regulators = NULL;
2309 opp_table->regulator_count = -1;
2310}
2311
2312static void _put_clks(struct opp_table *opp_table, int count)
2313{
2314 int i;
2315
2316 for (i = count - 1; i >= 0; i--)
2317 clk_put(opp_table->clks[i]);
2318
2319 kfree(opp_table->clks);
2320 opp_table->clks = NULL;
2321}
2322
2323/*
2324 * In order to support OPP switching, OPP layer needs to get pointers to the
2325 * clocks for the device. Simple cases work fine without using this routine
2326 * (i.e. by passing connection-id as NULL), but for a device with multiple
2327 * clocks available, the OPP core needs to know the exact names of the clks to
2328 * use.
2329 *
2330 * This must be called before any OPPs are initialized for the device.
2331 */
2332static int _opp_set_clknames(struct opp_table *opp_table, struct device *dev,
2333 const char * const names[],
2334 config_clks_t config_clks)
2335{
2336 const char * const *temp = names;
2337 int count = 0, ret, i;
2338 struct clk *clk;
2339
2340 /* Count number of clks */
2341 while (*temp++)
2342 count++;
2343
2344 /*
2345 * This is a special case where we have a single clock, whose connection
2346 * id name is NULL, i.e. first two entries are NULL in the array.
2347 */
2348 if (!count && !names[1])
2349 count = 1;
2350
2351 /* Fail early for invalid configurations */
2352 if (!count || (!config_clks && count > 1))
2353 return -EINVAL;
2354
2355 /* Another CPU that shares the OPP table has set the clkname ? */
2356 if (opp_table->clks)
2357 return 0;
2358
2359 opp_table->clks = kmalloc_objs(*opp_table->clks, count);
2360 if (!opp_table->clks)
2361 return -ENOMEM;
2362
2363 /* Find clks for the device */
2364 for (i = 0; i < count; i++) {
2365 clk = clk_get(dev, names[i]);
2366 if (IS_ERR(clk)) {
2367 ret = dev_err_probe(dev, PTR_ERR(clk),
2368 "%s: Couldn't find clock with name: %s\n",
2369 __func__, names[i]);
2370 goto free_clks;
2371 }
2372
2373 opp_table->clks[i] = clk;
2374 }
2375
2376 opp_table->clk_count = count;
2377 opp_table->config_clks = config_clks;
2378
2379 /* Set generic single clk set here */
2380 if (count == 1) {
2381 if (!opp_table->config_clks)
2382 opp_table->config_clks = _opp_config_clk_single;
2383
2384 /*
2385 * We could have just dropped the "clk" field and used "clks"
2386 * everywhere. Instead we kept the "clk" field around for
2387 * following reasons:
2388 *
2389 * - avoiding clks[0] everywhere else.
2390 * - not running single clk helpers for multiple clk usecase by
2391 * mistake.
2392 *
2393 * Since this is single-clk case, just update the clk pointer
2394 * too.
2395 */
2396 opp_table->clk = opp_table->clks[0];
2397 }
2398
2399 return 0;
2400
2401free_clks:
2402 _put_clks(opp_table, i);
2403 return ret;
2404}
2405
2406static void _opp_put_clknames(struct opp_table *opp_table)
2407{
2408 if (!opp_table->clks)
2409 return;
2410
2411 opp_table->config_clks = NULL;
2412 opp_table->clk = ERR_PTR(-ENODEV);
2413
2414 _put_clks(opp_table, opp_table->clk_count);
2415}
2416
2417/*
2418 * This is useful to support platforms with multiple regulators per device.
2419 *
2420 * This must be called before any OPPs are initialized for the device.
2421 */
2422static int _opp_set_config_regulators_helper(struct opp_table *opp_table,
2423 struct device *dev, config_regulators_t config_regulators)
2424{
2425 /* Another CPU that shares the OPP table has set the helper ? */
2426 if (!opp_table->config_regulators)
2427 opp_table->config_regulators = config_regulators;
2428
2429 return 0;
2430}
2431
2432static void _opp_put_config_regulators_helper(struct opp_table *opp_table)
2433{
2434 if (opp_table->config_regulators)
2435 opp_table->config_regulators = NULL;
2436}
2437
2438static int _opp_set_required_dev(struct opp_table *opp_table,
2439 struct device *dev,
2440 struct device *required_dev,
2441 unsigned int index)
2442{
2443 struct opp_table *required_table, *pd_table;
2444 struct device *gdev;
2445
2446 /* Genpd core takes care of propagation to parent genpd */
2447 if (opp_table->is_genpd) {
2448 dev_err(dev, "%s: Operation not supported for genpds\n", __func__);
2449 return -EOPNOTSUPP;
2450 }
2451
2452 if (index >= opp_table->required_opp_count) {
2453 dev_err(dev, "Required OPPs not available, can't set required devs\n");
2454 return -EINVAL;
2455 }
2456
2457 required_table = opp_table->required_opp_tables[index];
2458 if (IS_ERR(required_table)) {
2459 dev_err(dev, "Missing OPP table, unable to set the required devs\n");
2460 return -ENODEV;
2461 }
2462
2463 /*
2464 * The required_opp_tables parsing is not perfect, as the OPP core does
2465 * the parsing solely based on the DT node pointers. The core sets the
2466 * required_opp_tables entry to the first OPP table in the "opp_tables"
2467 * list, that matches with the node pointer.
2468 *
2469 * If the target DT OPP table is used by multiple devices and they all
2470 * create separate instances of 'struct opp_table' from it, then it is
2471 * possible that the required_opp_tables entry may be set to the
2472 * incorrect sibling device.
2473 *
2474 * Cross check it again and fix if required.
2475 */
2476 gdev = dev_to_genpd_dev(required_dev);
2477 if (IS_ERR(gdev))
2478 return PTR_ERR(gdev);
2479
2480 pd_table = _find_opp_table(gdev);
2481 if (!IS_ERR(pd_table)) {
2482 if (pd_table != required_table) {
2483 dev_pm_opp_put_opp_table(required_table);
2484 opp_table->required_opp_tables[index] = pd_table;
2485 } else {
2486 dev_pm_opp_put_opp_table(pd_table);
2487 }
2488 }
2489
2490 opp_table->required_devs[index] = required_dev;
2491 return 0;
2492}
2493
2494static void _opp_put_required_dev(struct opp_table *opp_table,
2495 unsigned int index)
2496{
2497 opp_table->required_devs[index] = NULL;
2498}
2499
2500static void _opp_clear_config(struct opp_config_data *data)
2501{
2502 if (data->flags & OPP_CONFIG_REQUIRED_DEV)
2503 _opp_put_required_dev(data->opp_table,
2504 data->required_dev_index);
2505 if (data->flags & OPP_CONFIG_REGULATOR)
2506 _opp_put_regulators(data->opp_table);
2507 if (data->flags & OPP_CONFIG_SUPPORTED_HW)
2508 _opp_put_supported_hw(data->opp_table);
2509 if (data->flags & OPP_CONFIG_REGULATOR_HELPER)
2510 _opp_put_config_regulators_helper(data->opp_table);
2511 if (data->flags & OPP_CONFIG_PROP_NAME)
2512 _opp_put_prop_name(data->opp_table);
2513 if (data->flags & OPP_CONFIG_CLK)
2514 _opp_put_clknames(data->opp_table);
2515
2516 dev_pm_opp_put_opp_table(data->opp_table);
2517 kfree(data);
2518}
2519
2520/**
2521 * dev_pm_opp_set_config() - Set OPP configuration for the device.
2522 * @dev: Device for which configuration is being set.
2523 * @config: OPP configuration.
2524 *
2525 * This allows all device OPP configurations to be performed at once.
2526 *
2527 * This must be called before any OPPs are initialized for the device. This may
2528 * be called multiple times for the same OPP table, for example once for each
2529 * CPU that share the same table. This must be balanced by the same number of
2530 * calls to dev_pm_opp_clear_config() in order to free the OPP table properly.
2531 *
2532 * This returns a token to the caller, which must be passed to
2533 * dev_pm_opp_clear_config() to free the resources later. The value of the
2534 * returned token will be >= 1 for success and negative for errors. The minimum
2535 * value of 1 is chosen here to make it easy for callers to manage the resource.
2536 */
2537int dev_pm_opp_set_config(struct device *dev, struct dev_pm_opp_config *config)
2538{
2539 struct opp_table *opp_table;
2540 struct opp_config_data *data;
2541 unsigned int id;
2542 int ret;
2543
2544 data = kmalloc_obj(*data);
2545 if (!data)
2546 return -ENOMEM;
2547
2548 opp_table = _add_opp_table(dev, false);
2549 if (IS_ERR(opp_table)) {
2550 kfree(data);
2551 return PTR_ERR(opp_table);
2552 }
2553
2554 data->opp_table = opp_table;
2555 data->flags = 0;
2556
2557 /* This should be called before OPPs are initialized */
2558 if (WARN_ON(!list_empty(&opp_table->opp_list))) {
2559 ret = -EBUSY;
2560 goto err;
2561 }
2562
2563 /* Configure clocks */
2564 if (config->clk_names) {
2565 ret = _opp_set_clknames(opp_table, dev, config->clk_names,
2566 config->config_clks);
2567 if (ret)
2568 goto err;
2569
2570 data->flags |= OPP_CONFIG_CLK;
2571 } else if (config->config_clks) {
2572 /* Don't allow config callback without clocks */
2573 ret = -EINVAL;
2574 goto err;
2575 }
2576
2577 /* Configure property names */
2578 if (config->prop_name) {
2579 ret = _opp_set_prop_name(opp_table, config->prop_name);
2580 if (ret)
2581 goto err;
2582
2583 data->flags |= OPP_CONFIG_PROP_NAME;
2584 }
2585
2586 /* Configure config_regulators helper */
2587 if (config->config_regulators) {
2588 ret = _opp_set_config_regulators_helper(opp_table, dev,
2589 config->config_regulators);
2590 if (ret)
2591 goto err;
2592
2593 data->flags |= OPP_CONFIG_REGULATOR_HELPER;
2594 }
2595
2596 /* Configure supported hardware */
2597 if (config->supported_hw) {
2598 ret = _opp_set_supported_hw(opp_table, config->supported_hw,
2599 config->supported_hw_count);
2600 if (ret)
2601 goto err;
2602
2603 data->flags |= OPP_CONFIG_SUPPORTED_HW;
2604 }
2605
2606 /* Configure supplies */
2607 if (config->regulator_names) {
2608 ret = _opp_set_regulators(opp_table, dev,
2609 config->regulator_names);
2610 if (ret)
2611 goto err;
2612
2613 data->flags |= OPP_CONFIG_REGULATOR;
2614 }
2615
2616 if (config->required_dev) {
2617 ret = _opp_set_required_dev(opp_table, dev,
2618 config->required_dev,
2619 config->required_dev_index);
2620 if (ret)
2621 goto err;
2622
2623 data->required_dev_index = config->required_dev_index;
2624 data->flags |= OPP_CONFIG_REQUIRED_DEV;
2625 }
2626
2627 ret = xa_alloc(&opp_configs, &id, data, XA_LIMIT(1, INT_MAX),
2628 GFP_KERNEL);
2629 if (ret)
2630 goto err;
2631
2632 return id;
2633
2634err:
2635 _opp_clear_config(data);
2636 return ret;
2637}
2638EXPORT_SYMBOL_GPL(dev_pm_opp_set_config);
2639
2640/**
2641 * dev_pm_opp_clear_config() - Releases resources blocked for OPP configuration.
2642 * @token: The token returned by dev_pm_opp_set_config() previously.
2643 *
2644 * This allows all device OPP configurations to be cleared at once. This must be
2645 * called once for each call made to dev_pm_opp_set_config(), in order to free
2646 * the OPPs properly.
2647 *
2648 * Currently the first call itself ends up freeing all the OPP configurations,
2649 * while the later ones only drop the OPP table reference. This works well for
2650 * now as we would never want to use an half initialized OPP table and want to
2651 * remove the configurations together.
2652 */
2653void dev_pm_opp_clear_config(int token)
2654{
2655 struct opp_config_data *data;
2656
2657 /*
2658 * This lets the callers call this unconditionally and keep their code
2659 * simple.
2660 */
2661 if (unlikely(token <= 0))
2662 return;
2663
2664 data = xa_erase(&opp_configs, token);
2665 if (WARN_ON(!data))
2666 return;
2667
2668 _opp_clear_config(data);
2669}
2670EXPORT_SYMBOL_GPL(dev_pm_opp_clear_config);
2671
2672static void devm_pm_opp_config_release(void *token)
2673{
2674 dev_pm_opp_clear_config((unsigned long)token);
2675}
2676
2677/**
2678 * devm_pm_opp_set_config() - Set OPP configuration for the device.
2679 * @dev: Device for which configuration is being set.
2680 * @config: OPP configuration.
2681 *
2682 * This allows all device OPP configurations to be performed at once.
2683 * This is a resource-managed variant of dev_pm_opp_set_config().
2684 *
2685 * Return: 0 on success and errorno otherwise.
2686 */
2687int devm_pm_opp_set_config(struct device *dev, struct dev_pm_opp_config *config)
2688{
2689 int token = dev_pm_opp_set_config(dev, config);
2690
2691 if (token < 0)
2692 return token;
2693
2694 return devm_add_action_or_reset(dev, devm_pm_opp_config_release,
2695 (void *) ((unsigned long) token));
2696}
2697EXPORT_SYMBOL_GPL(devm_pm_opp_set_config);
2698
2699/**
2700 * dev_pm_opp_xlate_required_opp() - Find required OPP for @src_table OPP.
2701 * @src_table: OPP table which has @dst_table as one of its required OPP table.
2702 * @dst_table: Required OPP table of the @src_table.
2703 * @src_opp: OPP from the @src_table.
2704 *
2705 * This function returns the OPP (present in @dst_table) pointed out by the
2706 * "required-opps" property of the @src_opp (present in @src_table).
2707 *
2708 * The callers are required to call dev_pm_opp_put() for the returned OPP after
2709 * use.
2710 *
2711 * Return: pointer to 'struct dev_pm_opp' on success and errorno otherwise.
2712 */
2713struct dev_pm_opp *dev_pm_opp_xlate_required_opp(struct opp_table *src_table,
2714 struct opp_table *dst_table,
2715 struct dev_pm_opp *src_opp)
2716{
2717 struct dev_pm_opp *opp, *dest_opp = ERR_PTR(-ENODEV);
2718 int i;
2719
2720 if (!src_table || !dst_table || !src_opp ||
2721 !src_table->required_opp_tables)
2722 return ERR_PTR(-EINVAL);
2723
2724 /* required-opps not fully initialized yet */
2725 if (lazy_linking_pending(src_table))
2726 return ERR_PTR(-EBUSY);
2727
2728 for (i = 0; i < src_table->required_opp_count; i++) {
2729 if (src_table->required_opp_tables[i] != dst_table)
2730 continue;
2731
2732 scoped_guard(mutex, &src_table->lock) {
2733 list_for_each_entry(opp, &src_table->opp_list, node) {
2734 if (opp == src_opp) {
2735 dest_opp = dev_pm_opp_get(opp->required_opps[i]);
2736 break;
2737 }
2738 }
2739 }
2740 break;
2741 }
2742
2743 if (IS_ERR(dest_opp)) {
2744 pr_err("%s: Couldn't find matching OPP (%p: %p)\n", __func__,
2745 src_table, dst_table);
2746 }
2747
2748 return dest_opp;
2749}
2750EXPORT_SYMBOL_GPL(dev_pm_opp_xlate_required_opp);
2751
2752/**
2753 * dev_pm_opp_xlate_performance_state() - Find required OPP's pstate for src_table.
2754 * @src_table: OPP table which has dst_table as one of its required OPP table.
2755 * @dst_table: Required OPP table of the src_table.
2756 * @pstate: Current performance state of the src_table.
2757 *
2758 * This Returns pstate of the OPP (present in @dst_table) pointed out by the
2759 * "required-opps" property of the OPP (present in @src_table) which has
2760 * performance state set to @pstate.
2761 *
2762 * Return: Zero or positive performance state on success, otherwise negative
2763 * value on errors.
2764 */
2765int dev_pm_opp_xlate_performance_state(struct opp_table *src_table,
2766 struct opp_table *dst_table,
2767 unsigned int pstate)
2768{
2769 struct dev_pm_opp *opp;
2770 int i;
2771
2772 /*
2773 * Normally the src_table will have the "required_opps" property set to
2774 * point to one of the OPPs in the dst_table, but in some cases the
2775 * genpd and its master have one to one mapping of performance states
2776 * and so none of them have the "required-opps" property set. Return the
2777 * pstate of the src_table as it is in such cases.
2778 */
2779 if (!src_table || !src_table->required_opp_count)
2780 return pstate;
2781
2782 /* Both OPP tables must belong to genpds */
2783 if (unlikely(!src_table->is_genpd || !dst_table->is_genpd)) {
2784 pr_err("%s: Performance state is only valid for genpds.\n", __func__);
2785 return -EINVAL;
2786 }
2787
2788 /* required-opps not fully initialized yet */
2789 if (lazy_linking_pending(src_table))
2790 return -EBUSY;
2791
2792 for (i = 0; i < src_table->required_opp_count; i++) {
2793 if (src_table->required_opp_tables[i]->np == dst_table->np)
2794 break;
2795 }
2796
2797 if (unlikely(i == src_table->required_opp_count)) {
2798 pr_err("%s: Couldn't find matching OPP table (%p: %p)\n",
2799 __func__, src_table, dst_table);
2800 return -EINVAL;
2801 }
2802
2803 guard(mutex)(&src_table->lock);
2804
2805 list_for_each_entry(opp, &src_table->opp_list, node) {
2806 if (opp->level == pstate)
2807 return opp->required_opps[i]->level;
2808 }
2809
2810 pr_err("%s: Couldn't find matching OPP (%p: %p)\n", __func__, src_table,
2811 dst_table);
2812
2813 return -EINVAL;
2814}
2815
2816/**
2817 * dev_pm_opp_add_dynamic() - Add an OPP table from a table definitions
2818 * @dev: The device for which we do this operation
2819 * @data: The OPP data for the OPP to add
2820 *
2821 * This function adds an opp definition to the opp table and returns status.
2822 * The opp is made available by default and it can be controlled using
2823 * dev_pm_opp_enable/disable functions.
2824 *
2825 * Return:
2826 * 0 On success OR
2827 * Duplicate OPPs (both freq and volt are same) and opp->available
2828 * -EEXIST Freq are same and volt are different OR
2829 * Duplicate OPPs (both freq and volt are same) and !opp->available
2830 * -ENOMEM Memory allocation failure
2831 */
2832int dev_pm_opp_add_dynamic(struct device *dev, struct dev_pm_opp_data *data)
2833{
2834 struct opp_table *opp_table;
2835 int ret;
2836
2837 opp_table = _add_opp_table(dev, true);
2838 if (IS_ERR(opp_table))
2839 return PTR_ERR(opp_table);
2840
2841 /* Fix regulator count for dynamic OPPs */
2842 opp_table->regulator_count = 1;
2843
2844 ret = _opp_add_v1(opp_table, dev, data, true);
2845 if (ret)
2846 dev_pm_opp_put_opp_table(opp_table);
2847
2848 return ret;
2849}
2850EXPORT_SYMBOL_GPL(dev_pm_opp_add_dynamic);
2851
2852/**
2853 * _opp_set_availability() - helper to set the availability of an opp
2854 * @dev: device for which we do this operation
2855 * @freq: OPP frequency to modify availability
2856 * @availability_req: availability status requested for this opp
2857 *
2858 * Set the availability of an OPP, opp_{enable,disable} share a common logic
2859 * which is isolated here.
2860 *
2861 * Return: -EINVAL for bad pointers, -ENOMEM if no memory available for the
2862 * copy operation, returns 0 if no modification was done OR modification was
2863 * successful.
2864 */
2865static int _opp_set_availability(struct device *dev, unsigned long freq,
2866 bool availability_req)
2867{
2868 /* Find the opp_table */
2869 struct opp_table *opp_table __free(put_opp_table) =
2870 _find_opp_table(dev);
2871 struct dev_pm_opp *opp __free(put_opp) = ERR_PTR(-ENODEV), *tmp_opp;
2872
2873 if (IS_ERR(opp_table)) {
2874 dev_warn(dev, "%s: Device OPP not found (%pe)\n", __func__,
2875 opp_table);
2876 return PTR_ERR(opp_table);
2877 }
2878
2879 if (!assert_single_clk(opp_table, 0))
2880 return -EINVAL;
2881
2882 scoped_guard(mutex, &opp_table->lock) {
2883 /* Do we have the frequency? */
2884 list_for_each_entry(tmp_opp, &opp_table->opp_list, node) {
2885 if (tmp_opp->rates[0] == freq) {
2886 opp = dev_pm_opp_get(tmp_opp);
2887
2888 /* Is update really needed? */
2889 if (opp->available == availability_req)
2890 return 0;
2891
2892 opp->available = availability_req;
2893 break;
2894 }
2895 }
2896 }
2897
2898 if (IS_ERR(opp))
2899 return PTR_ERR(opp);
2900
2901 /* Notify the change of the OPP availability */
2902 if (availability_req)
2903 blocking_notifier_call_chain(&opp_table->head, OPP_EVENT_ENABLE,
2904 opp);
2905 else
2906 blocking_notifier_call_chain(&opp_table->head,
2907 OPP_EVENT_DISABLE, opp);
2908
2909 return 0;
2910}
2911
2912/**
2913 * dev_pm_opp_adjust_voltage() - helper to change the voltage of an OPP
2914 * @dev: device for which we do this operation
2915 * @freq: OPP frequency to adjust voltage of
2916 * @u_volt: new OPP target voltage
2917 * @u_volt_min: new OPP min voltage
2918 * @u_volt_max: new OPP max voltage
2919 *
2920 * Return: -EINVAL for bad pointers, -ENOMEM if no memory available for the
2921 * copy operation, returns 0 if no modifcation was done OR modification was
2922 * successful.
2923 */
2924int dev_pm_opp_adjust_voltage(struct device *dev, unsigned long freq,
2925 unsigned long u_volt, unsigned long u_volt_min,
2926 unsigned long u_volt_max)
2927
2928{
2929 /* Find the opp_table */
2930 struct opp_table *opp_table __free(put_opp_table) =
2931 _find_opp_table(dev);
2932 struct dev_pm_opp *opp __free(put_opp) = ERR_PTR(-ENODEV), *tmp_opp;
2933 int r;
2934
2935 if (IS_ERR(opp_table)) {
2936 r = PTR_ERR(opp_table);
2937 dev_warn(dev, "%s: Device OPP not found (%d)\n", __func__, r);
2938 return r;
2939 }
2940
2941 if (!assert_single_clk(opp_table, 0))
2942 return -EINVAL;
2943
2944 scoped_guard(mutex, &opp_table->lock) {
2945 /* Do we have the frequency? */
2946 list_for_each_entry(tmp_opp, &opp_table->opp_list, node) {
2947 if (tmp_opp->rates[0] == freq) {
2948 opp = dev_pm_opp_get(tmp_opp);
2949
2950 /* Is update really needed? */
2951 if (opp->supplies->u_volt == u_volt)
2952 return 0;
2953
2954 opp->supplies->u_volt = u_volt;
2955 opp->supplies->u_volt_min = u_volt_min;
2956 opp->supplies->u_volt_max = u_volt_max;
2957
2958 break;
2959 }
2960 }
2961 }
2962
2963 if (IS_ERR(opp))
2964 return PTR_ERR(opp);
2965
2966 /* Notify the voltage change of the OPP */
2967 blocking_notifier_call_chain(&opp_table->head, OPP_EVENT_ADJUST_VOLTAGE,
2968 opp);
2969
2970 return 0;
2971}
2972EXPORT_SYMBOL_GPL(dev_pm_opp_adjust_voltage);
2973
2974/**
2975 * dev_pm_opp_sync_regulators() - Sync state of voltage regulators
2976 * @dev: device for which we do this operation
2977 *
2978 * Sync voltage state of the OPP table regulators.
2979 *
2980 * Return: 0 on success or a negative error value.
2981 */
2982int dev_pm_opp_sync_regulators(struct device *dev)
2983{
2984 struct regulator *reg;
2985 int ret, i;
2986
2987 /* Device may not have OPP table */
2988 struct opp_table *opp_table __free(put_opp_table) =
2989 _find_opp_table(dev);
2990
2991 if (IS_ERR(opp_table))
2992 return 0;
2993
2994 /* Regulator may not be required for the device */
2995 if (unlikely(!opp_table->regulators))
2996 return 0;
2997
2998 /* Nothing to sync if voltage wasn't changed */
2999 if (!opp_table->enabled)
3000 return 0;
3001
3002 for (i = 0; i < opp_table->regulator_count; i++) {
3003 reg = opp_table->regulators[i];
3004 ret = regulator_sync_voltage(reg);
3005 if (ret)
3006 return ret;
3007 }
3008
3009 return 0;
3010}
3011EXPORT_SYMBOL_GPL(dev_pm_opp_sync_regulators);
3012
3013/**
3014 * dev_pm_opp_enable() - Enable a specific OPP
3015 * @dev: device for which we do this operation
3016 * @freq: OPP frequency to enable
3017 *
3018 * Enables a provided opp. If the operation is valid, this returns 0, else the
3019 * corresponding error value. It is meant to be used for users an OPP available
3020 * after being temporarily made unavailable with dev_pm_opp_disable.
3021 *
3022 * Return: -EINVAL for bad pointers, -ENOMEM if no memory available for the
3023 * copy operation, returns 0 if no modification was done OR modification was
3024 * successful.
3025 */
3026int dev_pm_opp_enable(struct device *dev, unsigned long freq)
3027{
3028 return _opp_set_availability(dev, freq, true);
3029}
3030EXPORT_SYMBOL_GPL(dev_pm_opp_enable);
3031
3032/**
3033 * dev_pm_opp_disable() - Disable a specific OPP
3034 * @dev: device for which we do this operation
3035 * @freq: OPP frequency to disable
3036 *
3037 * Disables a provided opp. If the operation is valid, this returns
3038 * 0, else the corresponding error value. It is meant to be a temporary
3039 * control by users to make this OPP not available until the circumstances are
3040 * right to make it available again (with a call to dev_pm_opp_enable).
3041 *
3042 * Return: -EINVAL for bad pointers, -ENOMEM if no memory available for the
3043 * copy operation, returns 0 if no modification was done OR modification was
3044 * successful.
3045 */
3046int dev_pm_opp_disable(struct device *dev, unsigned long freq)
3047{
3048 return _opp_set_availability(dev, freq, false);
3049}
3050EXPORT_SYMBOL_GPL(dev_pm_opp_disable);
3051
3052/**
3053 * dev_pm_opp_register_notifier() - Register OPP notifier for the device
3054 * @dev: Device for which notifier needs to be registered
3055 * @nb: Notifier block to be registered
3056 *
3057 * Return: 0 on success or a negative error value.
3058 */
3059int dev_pm_opp_register_notifier(struct device *dev, struct notifier_block *nb)
3060{
3061 struct opp_table *opp_table __free(put_opp_table) =
3062 _find_opp_table(dev);
3063
3064 if (IS_ERR(opp_table))
3065 return PTR_ERR(opp_table);
3066
3067 return blocking_notifier_chain_register(&opp_table->head, nb);
3068}
3069EXPORT_SYMBOL(dev_pm_opp_register_notifier);
3070
3071/**
3072 * dev_pm_opp_unregister_notifier() - Unregister OPP notifier for the device
3073 * @dev: Device for which notifier needs to be unregistered
3074 * @nb: Notifier block to be unregistered
3075 *
3076 * Return: 0 on success or a negative error value.
3077 */
3078int dev_pm_opp_unregister_notifier(struct device *dev,
3079 struct notifier_block *nb)
3080{
3081 struct opp_table *opp_table __free(put_opp_table) =
3082 _find_opp_table(dev);
3083
3084 if (IS_ERR(opp_table))
3085 return PTR_ERR(opp_table);
3086
3087 return blocking_notifier_chain_unregister(&opp_table->head, nb);
3088}
3089EXPORT_SYMBOL(dev_pm_opp_unregister_notifier);
3090
3091/**
3092 * dev_pm_opp_remove_table() - Free all OPPs associated with the device
3093 * @dev: device pointer used to lookup OPP table.
3094 *
3095 * Free both OPPs created using static entries present in DT and the
3096 * dynamically added entries.
3097 */
3098void dev_pm_opp_remove_table(struct device *dev)
3099{
3100 /* Check for existing table for 'dev' */
3101 struct opp_table *opp_table __free(put_opp_table) =
3102 _find_opp_table(dev);
3103
3104 if (IS_ERR(opp_table)) {
3105 int error = PTR_ERR(opp_table);
3106
3107 if (error != -ENODEV)
3108 WARN(1, "%s: opp_table: %d\n",
3109 IS_ERR_OR_NULL(dev) ?
3110 "Invalid device" : dev_name(dev),
3111 error);
3112 return;
3113 }
3114
3115 /*
3116 * Drop the extra reference only if the OPP table was successfully added
3117 * with dev_pm_opp_of_add_table() earlier.
3118 **/
3119 if (_opp_remove_all_static(opp_table))
3120 dev_pm_opp_put_opp_table(opp_table);
3121}
3122EXPORT_SYMBOL_GPL(dev_pm_opp_remove_table);