projects/linux·blob master · refs

1273 lines · 29.3 KiB

1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Interconnect framework core driver
4 *
5 * Copyright (c) 2017-2019, Linaro Ltd.
6 * Author: Georgi Djakov <[email protected]>
7 */
8
9#include <linux/debugfs.h>
10#include <linux/device.h>
11#include <linux/idr.h>
12#include <linux/init.h>
13#include <linux/interconnect.h>
14#include <linux/interconnect-provider.h>
15#include <linux/list.h>
16#include <linux/mutex.h>
17#include <linux/slab.h>
18#include <linux/of.h>
19#include <linux/overflow.h>
20
21#include "internal.h"
22
23#define ICC_DYN_ID_START 100000
24
25#define CREATE_TRACE_POINTS
26#include "trace.h"
27
28static DEFINE_IDR(icc_idr);
29static LIST_HEAD(icc_providers);
30static int providers_count;
31static bool synced_state;
32static DEFINE_MUTEX(icc_lock);
33static DEFINE_MUTEX(icc_bw_lock);
34static struct dentry *icc_debugfs_dir;
35
36static void icc_summary_show_one(struct seq_file *s, struct icc_node *n)
37{
38 if (!n)
39 return;
40
41 seq_printf(s, "%-42s %12u %12u\n",
42 n->name, n->avg_bw, n->peak_bw);
43}
44
45static int icc_summary_show(struct seq_file *s, void *data)
46{
47 struct icc_provider *provider;
48
49 seq_puts(s, " node tag avg peak\n");
50 seq_puts(s, "--------------------------------------------------------------------\n");
51
52 mutex_lock(&icc_lock);
53
54 list_for_each_entry(provider, &icc_providers, provider_list) {
55 struct icc_node *n;
56
57 list_for_each_entry(n, &provider->nodes, node_list) {
58 struct icc_req *r;
59
60 icc_summary_show_one(s, n);
61 hlist_for_each_entry(r, &n->req_list, req_node) {
62 u32 avg_bw = 0, peak_bw = 0;
63
64 if (!r->dev)
65 continue;
66
67 if (r->enabled) {
68 avg_bw = r->avg_bw;
69 peak_bw = r->peak_bw;
70 }
71
72 seq_printf(s, " %-27s %12u %12u %12u\n",
73 dev_name(r->dev), r->tag, avg_bw, peak_bw);
74 }
75 }
76 }
77
78 mutex_unlock(&icc_lock);
79
80 return 0;
81}
82DEFINE_SHOW_ATTRIBUTE(icc_summary);
83
84static void icc_graph_show_link(struct seq_file *s, int level,
85 struct icc_node *n, struct icc_node *m)
86{
87 seq_printf(s, "%s\"%d:%s\" -> \"%d:%s\"\n",
88 level == 2 ? "\t\t" : "\t",
89 n->id, n->name, m->id, m->name);
90}
91
92static void icc_graph_show_node(struct seq_file *s, struct icc_node *n)
93{
94 seq_printf(s, "\t\t\"%d:%s\" [label=\"%d:%s",
95 n->id, n->name, n->id, n->name);
96 seq_printf(s, "\n\t\t\t|avg_bw=%ukBps", n->avg_bw);
97 seq_printf(s, "\n\t\t\t|peak_bw=%ukBps", n->peak_bw);
98 seq_puts(s, "\"]\n");
99}
100
101static int icc_graph_show(struct seq_file *s, void *data)
102{
103 struct icc_provider *provider;
104 struct icc_node *n;
105 int cluster_index = 0;
106 int i;
107
108 seq_puts(s, "digraph {\n\trankdir = LR\n\tnode [shape = record]\n");
109 mutex_lock(&icc_lock);
110
111 /* draw providers as cluster subgraphs */
112 cluster_index = 0;
113 list_for_each_entry(provider, &icc_providers, provider_list) {
114 seq_printf(s, "\tsubgraph cluster_%d {\n", ++cluster_index);
115 if (provider->dev)
116 seq_printf(s, "\t\tlabel = \"%s\"\n",
117 dev_name(provider->dev));
118
119 /* draw nodes */
120 list_for_each_entry(n, &provider->nodes, node_list)
121 icc_graph_show_node(s, n);
122
123 /* draw internal links */
124 list_for_each_entry(n, &provider->nodes, node_list)
125 for (i = 0; i < n->num_links; ++i)
126 if (n->provider == n->links[i]->provider)
127 icc_graph_show_link(s, 2, n,
128 n->links[i]);
129
130 seq_puts(s, "\t}\n");
131 }
132
133 /* draw external links */
134 list_for_each_entry(provider, &icc_providers, provider_list)
135 list_for_each_entry(n, &provider->nodes, node_list)
136 for (i = 0; i < n->num_links; ++i)
137 if (n->provider != n->links[i]->provider)
138 icc_graph_show_link(s, 1, n,
139 n->links[i]);
140
141 mutex_unlock(&icc_lock);
142 seq_puts(s, "}");
143
144 return 0;
145}
146DEFINE_SHOW_ATTRIBUTE(icc_graph);
147
148static struct icc_node *node_find(const int id)
149{
150 return idr_find(&icc_idr, id);
151}
152
153static struct icc_node *node_find_by_name(const char *name)
154{
155 struct icc_provider *provider;
156 struct icc_node *n;
157
158 list_for_each_entry(provider, &icc_providers, provider_list) {
159 list_for_each_entry(n, &provider->nodes, node_list) {
160 if (!strcmp(n->name, name))
161 return n;
162 }
163 }
164
165 return NULL;
166}
167
168static struct icc_path *path_init(struct device *dev, struct icc_node *dst,
169 ssize_t num_nodes)
170{
171 struct icc_node *node = dst;
172 struct icc_path *path;
173 int i;
174
175 path = kzalloc_flex(*path, reqs, num_nodes);
176 if (!path)
177 return ERR_PTR(-ENOMEM);
178
179 path->num_nodes = num_nodes;
180
181 mutex_lock(&icc_bw_lock);
182
183 for (i = num_nodes - 1; i >= 0; i--) {
184 node->provider->users++;
185 hlist_add_head(&path->reqs[i].req_node, &node->req_list);
186 path->reqs[i].node = node;
187 path->reqs[i].dev = dev;
188 path->reqs[i].enabled = true;
189 /* reference to previous node was saved during path traversal */
190 node = node->reverse;
191 }
192
193 mutex_unlock(&icc_bw_lock);
194
195 return path;
196}
197
198static struct icc_path *path_find(struct device *dev, struct icc_node *src,
199 struct icc_node *dst)
200{
201 struct icc_path *path = ERR_PTR(-EPROBE_DEFER);
202 struct icc_node *n, *node = NULL;
203 struct list_head traverse_list;
204 struct list_head edge_list;
205 struct list_head visited_list;
206 size_t i, depth = 1;
207 bool found = false;
208
209 INIT_LIST_HEAD(&traverse_list);
210 INIT_LIST_HEAD(&edge_list);
211 INIT_LIST_HEAD(&visited_list);
212
213 list_add(&src->search_list, &traverse_list);
214 src->reverse = NULL;
215
216 do {
217 list_for_each_entry_safe(node, n, &traverse_list, search_list) {
218 if (node == dst) {
219 found = true;
220 list_splice_init(&edge_list, &visited_list);
221 list_splice_init(&traverse_list, &visited_list);
222 break;
223 }
224 for (i = 0; i < node->num_links; i++) {
225 struct icc_node *tmp = node->links[i];
226
227 if (!tmp) {
228 path = ERR_PTR(-ENOENT);
229 goto out;
230 }
231
232 if (tmp->is_traversed)
233 continue;
234
235 tmp->is_traversed = true;
236 tmp->reverse = node;
237 list_add_tail(&tmp->search_list, &edge_list);
238 }
239 }
240
241 if (found)
242 break;
243
244 list_splice_init(&traverse_list, &visited_list);
245 list_splice_init(&edge_list, &traverse_list);
246
247 /* count the hops including the source */
248 depth++;
249
250 } while (!list_empty(&traverse_list));
251
252out:
253
254 /* reset the traversed state */
255 list_for_each_entry_reverse(n, &visited_list, search_list)
256 n->is_traversed = false;
257
258 if (found)
259 path = path_init(dev, dst, depth);
260
261 return path;
262}
263
264/*
265 * We want the path to honor all bandwidth requests, so the average and peak
266 * bandwidth requirements from each consumer are aggregated at each node.
267 * The aggregation is platform specific, so each platform can customize it by
268 * implementing its own aggregate() function.
269 */
270
271static int aggregate_requests(struct icc_node *node)
272{
273 struct icc_provider *p = node->provider;
274 struct icc_req *r;
275 u32 avg_bw, peak_bw;
276
277 node->avg_bw = 0;
278 node->peak_bw = 0;
279
280 if (p->pre_aggregate)
281 p->pre_aggregate(node);
282
283 hlist_for_each_entry(r, &node->req_list, req_node) {
284 if (r->enabled) {
285 avg_bw = r->avg_bw;
286 peak_bw = r->peak_bw;
287 } else {
288 avg_bw = 0;
289 peak_bw = 0;
290 }
291 p->aggregate(node, r->tag, avg_bw, peak_bw,
292 &node->avg_bw, &node->peak_bw);
293
294 /* during boot use the initial bandwidth as a floor value */
295 if (!synced_state) {
296 node->avg_bw = max(node->avg_bw, node->init_avg);
297 node->peak_bw = max(node->peak_bw, node->init_peak);
298 }
299 }
300
301 return 0;
302}
303
304static int apply_constraints(struct icc_path *path)
305{
306 struct icc_node *next, *prev = NULL;
307 struct icc_provider *p;
308 int ret = -EINVAL;
309 int i;
310
311 for (i = 0; i < path->num_nodes; i++) {
312 next = path->reqs[i].node;
313 p = next->provider;
314
315 /* both endpoints should be valid master-slave pairs */
316 if (!prev || (p != prev->provider && !p->inter_set)) {
317 prev = next;
318 continue;
319 }
320
321 /* set the constraints */
322 ret = p->set(prev, next);
323 if (ret)
324 goto out;
325
326 prev = next;
327 }
328out:
329 return ret;
330}
331
332int icc_std_aggregate(struct icc_node *node, u32 tag, u32 avg_bw,
333 u32 peak_bw, u32 *agg_avg, u32 *agg_peak)
334{
335 *agg_avg += avg_bw;
336 *agg_peak = max(*agg_peak, peak_bw);
337
338 return 0;
339}
340EXPORT_SYMBOL_GPL(icc_std_aggregate);
341
342/* of_icc_xlate_onecell() - Translate function using a single index.
343 * @spec: OF phandle args to map into an interconnect node.
344 * @data: private data (pointer to struct icc_onecell_data)
345 *
346 * This is a generic translate function that can be used to model simple
347 * interconnect providers that have one device tree node and provide
348 * multiple interconnect nodes. A single cell is used as an index into
349 * an array of icc nodes specified in the icc_onecell_data struct when
350 * registering the provider.
351 */
352struct icc_node *of_icc_xlate_onecell(const struct of_phandle_args *spec,
353 void *data)
354{
355 struct icc_onecell_data *icc_data = data;
356 unsigned int idx = spec->args[0];
357
358 if (idx >= icc_data->num_nodes) {
359 pr_err("%s: invalid index %u\n", __func__, idx);
360 return ERR_PTR(-EINVAL);
361 }
362
363 return icc_data->nodes[idx];
364}
365EXPORT_SYMBOL_GPL(of_icc_xlate_onecell);
366
367/**
368 * of_icc_get_from_provider() - Look-up interconnect node
369 * @spec: OF phandle args to use for look-up
370 *
371 * Looks for interconnect provider under the node specified by @spec and if
372 * found, uses xlate function of the provider to map phandle args to node.
373 *
374 * Returns a valid pointer to struct icc_node_data on success or ERR_PTR()
375 * on failure.
376 */
377struct icc_node_data *of_icc_get_from_provider(const struct of_phandle_args *spec)
378{
379 struct icc_node *node = ERR_PTR(-EPROBE_DEFER);
380 struct icc_node_data *data = NULL;
381 struct icc_provider *provider;
382
383 if (!spec)
384 return ERR_PTR(-EINVAL);
385
386 mutex_lock(&icc_lock);
387 list_for_each_entry(provider, &icc_providers, provider_list) {
388 if (device_match_of_node(provider->dev, spec->np)) {
389 if (provider->xlate_extended) {
390 data = provider->xlate_extended(spec, provider->data);
391 if (!IS_ERR(data)) {
392 node = data->node;
393 break;
394 }
395 } else {
396 node = provider->xlate(spec, provider->data);
397 if (!IS_ERR(node))
398 break;
399 }
400 }
401 }
402 mutex_unlock(&icc_lock);
403
404 if (!node)
405 return ERR_PTR(-EINVAL);
406
407 if (IS_ERR(node))
408 return ERR_CAST(node);
409
410 if (!data) {
411 data = kzalloc_obj(*data);
412 if (!data)
413 return ERR_PTR(-ENOMEM);
414 data->node = node;
415 }
416
417 return data;
418}
419EXPORT_SYMBOL_GPL(of_icc_get_from_provider);
420
421static void devm_icc_release(struct device *dev, void *res)
422{
423 icc_put(*(struct icc_path **)res);
424}
425
426struct icc_path *devm_of_icc_get(struct device *dev, const char *name)
427{
428 struct icc_path **ptr, *path;
429
430 ptr = devres_alloc(devm_icc_release, sizeof(*ptr), GFP_KERNEL);
431 if (!ptr)
432 return ERR_PTR(-ENOMEM);
433
434 path = of_icc_get(dev, name);
435 if (!IS_ERR_OR_NULL(path)) {
436 *ptr = path;
437 devres_add(dev, ptr);
438 } else {
439 devres_free(ptr);
440 }
441
442 return path;
443}
444EXPORT_SYMBOL_GPL(devm_of_icc_get);
445
446struct icc_path *devm_of_icc_get_by_index(struct device *dev, int idx)
447{
448 struct icc_path **ptr, *path;
449
450 ptr = devres_alloc(devm_icc_release, sizeof(*ptr), GFP_KERNEL);
451 if (!ptr)
452 return ERR_PTR(-ENOMEM);
453
454 path = of_icc_get_by_index(dev, idx);
455 if (!IS_ERR(path)) {
456 *ptr = path;
457 devres_add(dev, ptr);
458 } else {
459 devres_free(ptr);
460 }
461
462 return path;
463}
464EXPORT_SYMBOL_GPL(devm_of_icc_get_by_index);
465
466/**
467 * of_icc_get_by_index() - get a path handle from a DT node based on index
468 * @dev: device pointer for the consumer device
469 * @idx: interconnect path index
470 *
471 * This function will search for a path between two endpoints and return an
472 * icc_path handle on success. Use icc_put() to release constraints when they
473 * are not needed anymore.
474 * If the interconnect API is disabled, NULL is returned and the consumer
475 * drivers will still build. Drivers are free to handle this specifically,
476 * but they don't have to.
477 *
478 * Return: icc_path pointer on success or ERR_PTR() on error. NULL is returned
479 * when the API is disabled or the "interconnects" DT property is missing.
480 */
481struct icc_path *of_icc_get_by_index(struct device *dev, int idx)
482{
483 struct icc_path *path;
484 struct icc_node_data *src_data, *dst_data;
485 struct device_node *np;
486 struct of_phandle_args src_args, dst_args;
487 int ret;
488
489 if (!dev || !dev->of_node)
490 return ERR_PTR(-ENODEV);
491
492 np = dev->of_node;
493
494 /*
495 * When the consumer DT node do not have "interconnects" property
496 * return a NULL path to skip setting constraints.
497 */
498 if (!of_property_present(np, "interconnects"))
499 return NULL;
500
501 /*
502 * We use a combination of phandle and specifier for endpoint. For now
503 * lets support only global ids and extend this in the future if needed
504 * without breaking DT compatibility.
505 */
506 ret = of_parse_phandle_with_args(np, "interconnects",
507 "#interconnect-cells", idx * 2,
508 &src_args);
509 if (ret)
510 return ERR_PTR(ret);
511
512 of_node_put(src_args.np);
513
514 ret = of_parse_phandle_with_args(np, "interconnects",
515 "#interconnect-cells", idx * 2 + 1,
516 &dst_args);
517 if (ret)
518 return ERR_PTR(ret);
519
520 of_node_put(dst_args.np);
521
522 src_data = of_icc_get_from_provider(&src_args);
523
524 if (IS_ERR(src_data)) {
525 dev_err_probe(dev, PTR_ERR(src_data), "error finding src node\n");
526 return ERR_CAST(src_data);
527 }
528
529 dst_data = of_icc_get_from_provider(&dst_args);
530
531 if (IS_ERR(dst_data)) {
532 dev_err_probe(dev, PTR_ERR(dst_data), "error finding dst node\n");
533 kfree(src_data);
534 return ERR_CAST(dst_data);
535 }
536
537 mutex_lock(&icc_lock);
538 path = path_find(dev, src_data->node, dst_data->node);
539 mutex_unlock(&icc_lock);
540 if (IS_ERR(path)) {
541 dev_err(dev, "%s: invalid path=%ld\n", __func__, PTR_ERR(path));
542 goto free_icc_data;
543 }
544
545 if (src_data->tag && src_data->tag == dst_data->tag)
546 icc_set_tag(path, src_data->tag);
547
548 path->name = kasprintf(GFP_KERNEL, "%s-%s",
549 src_data->node->name, dst_data->node->name);
550 if (!path->name) {
551 icc_put(path);
552 path = ERR_PTR(-ENOMEM);
553 }
554
555free_icc_data:
556 kfree(src_data);
557 kfree(dst_data);
558 return path;
559}
560EXPORT_SYMBOL_GPL(of_icc_get_by_index);
561
562/**
563 * of_icc_get() - get a path handle from a DT node based on name
564 * @dev: device pointer for the consumer device
565 * @name: interconnect path name
566 *
567 * This function will search for a path between two endpoints and return an
568 * icc_path handle on success. Use icc_put() to release constraints when they
569 * are not needed anymore.
570 * If the interconnect API is disabled, NULL is returned and the consumer
571 * drivers will still build. Drivers are free to handle this specifically,
572 * but they don't have to.
573 *
574 * Return: icc_path pointer on success or ERR_PTR() on error. NULL is returned
575 * when the API is disabled or the "interconnects" DT property is missing.
576 */
577struct icc_path *of_icc_get(struct device *dev, const char *name)
578{
579 struct device_node *np;
580 int idx = 0;
581
582 if (!dev || !dev->of_node)
583 return ERR_PTR(-ENODEV);
584
585 np = dev->of_node;
586
587 /*
588 * When the consumer DT node do not have "interconnects" property
589 * return a NULL path to skip setting constraints.
590 */
591 if (!of_property_present(np, "interconnects"))
592 return NULL;
593
594 /*
595 * We use a combination of phandle and specifier for endpoint. For now
596 * lets support only global ids and extend this in the future if needed
597 * without breaking DT compatibility.
598 */
599 if (name) {
600 idx = of_property_match_string(np, "interconnect-names", name);
601 if (idx < 0)
602 return ERR_PTR(idx);
603 }
604
605 return of_icc_get_by_index(dev, idx);
606}
607EXPORT_SYMBOL_GPL(of_icc_get);
608
609/**
610 * icc_get() - get a path handle between two endpoints
611 * @dev: device pointer for the consumer device
612 * @src: source node name
613 * @dst: destination node name
614 *
615 * This function will search for a path between two endpoints and return an
616 * icc_path handle on success. Use icc_put() to release constraints when they
617 * are not needed anymore.
618 *
619 * Return: icc_path pointer on success or ERR_PTR() on error. NULL is returned
620 * when the API is disabled.
621 */
622struct icc_path *icc_get(struct device *dev, const char *src, const char *dst)
623{
624 struct icc_node *src_node, *dst_node;
625 struct icc_path *path = ERR_PTR(-EPROBE_DEFER);
626
627 mutex_lock(&icc_lock);
628
629 src_node = node_find_by_name(src);
630 if (!src_node) {
631 dev_err(dev, "%s: invalid src=%s\n", __func__, src);
632 goto out;
633 }
634
635 dst_node = node_find_by_name(dst);
636 if (!dst_node) {
637 dev_err(dev, "%s: invalid dst=%s\n", __func__, dst);
638 goto out;
639 }
640
641 path = path_find(dev, src_node, dst_node);
642 if (IS_ERR(path)) {
643 dev_err(dev, "%s: invalid path=%ld\n", __func__, PTR_ERR(path));
644 goto out;
645 }
646
647 path->name = kasprintf(GFP_KERNEL, "%s-%s", src_node->name, dst_node->name);
648 if (!path->name) {
649 mutex_unlock(&icc_lock);
650 icc_put(path);
651 return ERR_PTR(-ENOMEM);
652 }
653out:
654 mutex_unlock(&icc_lock);
655 return path;
656}
657
658/**
659 * icc_set_tag() - set an optional tag on a path
660 * @path: the path we want to tag
661 * @tag: the tag value
662 *
663 * This function allows consumers to append a tag to the requests associated
664 * with a path, so that a different aggregation could be done based on this tag.
665 */
666void icc_set_tag(struct icc_path *path, u32 tag)
667{
668 int i;
669
670 if (!path)
671 return;
672
673 mutex_lock(&icc_lock);
674
675 for (i = 0; i < path->num_nodes; i++)
676 path->reqs[i].tag = tag;
677
678 mutex_unlock(&icc_lock);
679}
680EXPORT_SYMBOL_GPL(icc_set_tag);
681
682/**
683 * icc_get_name() - Get name of the icc path
684 * @path: interconnect path
685 *
686 * This function is used by an interconnect consumer to get the name of the icc
687 * path.
688 *
689 * Returns a valid pointer on success, or NULL otherwise.
690 */
691const char *icc_get_name(struct icc_path *path)
692{
693 if (!path)
694 return NULL;
695
696 return path->name;
697}
698EXPORT_SYMBOL_GPL(icc_get_name);
699
700/**
701 * icc_set_bw() - set bandwidth constraints on an interconnect path
702 * @path: interconnect path
703 * @avg_bw: average bandwidth in kilobytes per second
704 * @peak_bw: peak bandwidth in kilobytes per second
705 *
706 * This function is used by an interconnect consumer to express its own needs
707 * in terms of bandwidth for a previously requested path between two endpoints.
708 * The requests are aggregated and each node is updated accordingly. The entire
709 * path is locked by a mutex to ensure that the set() is completed.
710 * The @path can be NULL when the "interconnects" DT properties is missing,
711 * which will mean that no constraints will be set.
712 *
713 * Returns 0 on success, or an appropriate error code otherwise.
714 */
715int icc_set_bw(struct icc_path *path, u32 avg_bw, u32 peak_bw)
716{
717 struct icc_node *node;
718 u32 old_avg, old_peak;
719 size_t i;
720 int ret;
721
722 if (!path)
723 return 0;
724
725 if (WARN_ON(IS_ERR(path) || !path->num_nodes))
726 return -EINVAL;
727
728 mutex_lock(&icc_bw_lock);
729
730 old_avg = path->reqs[0].avg_bw;
731 old_peak = path->reqs[0].peak_bw;
732
733 for (i = 0; i < path->num_nodes; i++) {
734 node = path->reqs[i].node;
735
736 /* update the consumer request for this path */
737 path->reqs[i].avg_bw = avg_bw;
738 path->reqs[i].peak_bw = peak_bw;
739
740 /* aggregate requests for this node */
741 aggregate_requests(node);
742
743 trace_icc_set_bw(path, node, i, avg_bw, peak_bw);
744 }
745
746 ret = apply_constraints(path);
747 if (ret) {
748 pr_debug("interconnect: error applying constraints (%d)\n",
749 ret);
750
751 for (i = 0; i < path->num_nodes; i++) {
752 node = path->reqs[i].node;
753 path->reqs[i].avg_bw = old_avg;
754 path->reqs[i].peak_bw = old_peak;
755 aggregate_requests(node);
756 }
757 apply_constraints(path);
758 }
759
760 mutex_unlock(&icc_bw_lock);
761
762 trace_icc_set_bw_end(path, ret);
763
764 return ret;
765}
766EXPORT_SYMBOL_GPL(icc_set_bw);
767
768static int __icc_enable(struct icc_path *path, bool enable)
769{
770 int i;
771
772 if (!path)
773 return 0;
774
775 if (WARN_ON(IS_ERR(path) || !path->num_nodes))
776 return -EINVAL;
777
778 mutex_lock(&icc_lock);
779
780 for (i = 0; i < path->num_nodes; i++)
781 path->reqs[i].enabled = enable;
782
783 mutex_unlock(&icc_lock);
784
785 return icc_set_bw(path, path->reqs[0].avg_bw,
786 path->reqs[0].peak_bw);
787}
788
789int icc_enable(struct icc_path *path)
790{
791 return __icc_enable(path, true);
792}
793EXPORT_SYMBOL_GPL(icc_enable);
794
795int icc_disable(struct icc_path *path)
796{
797 return __icc_enable(path, false);
798}
799EXPORT_SYMBOL_GPL(icc_disable);
800
801/**
802 * icc_put() - release the reference to the icc_path
803 * @path: interconnect path
804 *
805 * Use this function to release the constraints on a path when the path is
806 * no longer needed. The constraints will be re-aggregated.
807 */
808void icc_put(struct icc_path *path)
809{
810 struct icc_node *node;
811 size_t i;
812 int ret;
813
814 if (!path || WARN_ON(IS_ERR(path)))
815 return;
816
817 ret = icc_set_bw(path, 0, 0);
818 if (ret)
819 pr_err("%s: error (%d)\n", __func__, ret);
820
821 mutex_lock(&icc_lock);
822 mutex_lock(&icc_bw_lock);
823
824 for (i = 0; i < path->num_nodes; i++) {
825 node = path->reqs[i].node;
826 hlist_del(&path->reqs[i].req_node);
827 if (!WARN_ON(!node->provider->users))
828 node->provider->users--;
829 }
830
831 mutex_unlock(&icc_bw_lock);
832 mutex_unlock(&icc_lock);
833
834 kfree(path->name);
835 kfree(path);
836}
837EXPORT_SYMBOL_GPL(icc_put);
838
839static struct icc_node *icc_node_create_nolock(int id)
840{
841 struct icc_node *node;
842
843 if (id >= ICC_DYN_ID_START)
844 return ERR_PTR(-EINVAL);
845
846 /* check if node already exists */
847 node = node_find(id);
848 if (node)
849 return node;
850
851 node = kzalloc_obj(*node);
852 if (!node)
853 return ERR_PTR(-ENOMEM);
854
855 /* dynamic id allocation */
856 if (id == ICC_ALLOC_DYN_ID)
857 id = idr_alloc(&icc_idr, node, ICC_DYN_ID_START, 0, GFP_KERNEL);
858 else
859 id = idr_alloc(&icc_idr, node, id, id + 1, GFP_KERNEL);
860
861 if (id < 0) {
862 WARN(1, "%s: couldn't get idr\n", __func__);
863 kfree(node);
864 return ERR_PTR(id);
865 }
866
867 node->id = id;
868
869 return node;
870}
871
872/**
873 * icc_node_create_dyn() - create a node with dynamic id
874 *
875 * Return: icc_node pointer on success, or ERR_PTR() on error
876 */
877struct icc_node *icc_node_create_dyn(void)
878{
879 struct icc_node *node;
880
881 mutex_lock(&icc_lock);
882
883 node = icc_node_create_nolock(ICC_ALLOC_DYN_ID);
884
885 mutex_unlock(&icc_lock);
886
887 return node;
888}
889EXPORT_SYMBOL_GPL(icc_node_create_dyn);
890
891/**
892 * icc_node_create() - create a node
893 * @id: node id
894 *
895 * Return: icc_node pointer on success, or ERR_PTR() on error
896 */
897struct icc_node *icc_node_create(int id)
898{
899 struct icc_node *node;
900
901 mutex_lock(&icc_lock);
902
903 node = icc_node_create_nolock(id);
904
905 mutex_unlock(&icc_lock);
906
907 return node;
908}
909EXPORT_SYMBOL_GPL(icc_node_create);
910
911/**
912 * icc_node_destroy() - destroy a node
913 * @id: node id
914 */
915void icc_node_destroy(int id)
916{
917 struct icc_node *node;
918
919 mutex_lock(&icc_lock);
920
921 node = node_find(id);
922 if (node) {
923 idr_remove(&icc_idr, node->id);
924 WARN_ON(!hlist_empty(&node->req_list));
925 }
926
927 mutex_unlock(&icc_lock);
928
929 if (!node)
930 return;
931
932 kfree(node->links);
933 if (node->id >= ICC_DYN_ID_START)
934 kfree(node->name);
935 kfree(node);
936}
937EXPORT_SYMBOL_GPL(icc_node_destroy);
938
939/**
940 * icc_node_set_name() - set node name
941 * @node: node
942 * @provider: node provider
943 * @name: node name
944 *
945 * Return: 0 on success, or -ENOMEM on allocation failure
946 */
947int icc_node_set_name(struct icc_node *node, const struct icc_provider *provider, const char *name)
948{
949 if (node->id >= ICC_DYN_ID_START) {
950 node->name = kasprintf(GFP_KERNEL, "%s@%s", name,
951 dev_name(provider->dev));
952 if (!node->name)
953 return -ENOMEM;
954 } else {
955 node->name = name;
956 }
957
958 return 0;
959}
960EXPORT_SYMBOL_GPL(icc_node_set_name);
961
962/**
963 * icc_link_nodes() - create link between two nodes
964 * @src_node: source node
965 * @dst_node: destination node
966 *
967 * Create a link between two nodes. The nodes might belong to different
968 * interconnect providers and the @dst_node might not exist (if the
969 * provider driver has not probed yet). So just create the @dst_node
970 * and when the actual provider driver is probed, the rest of the node
971 * data is filled.
972 *
973 * Return: 0 on success, or an error code otherwise
974 */
975int icc_link_nodes(struct icc_node *src_node, struct icc_node **dst_node)
976{
977 struct icc_node **new;
978 int ret = 0;
979
980 if (!src_node->provider)
981 return -EINVAL;
982
983 mutex_lock(&icc_lock);
984
985 if (!*dst_node) {
986 *dst_node = icc_node_create_nolock(ICC_ALLOC_DYN_ID);
987
988 if (IS_ERR(*dst_node)) {
989 ret = PTR_ERR(*dst_node);
990 goto out;
991 }
992 }
993
994 new = krealloc(src_node->links,
995 (src_node->num_links + 1) * sizeof(*src_node->links),
996 GFP_KERNEL);
997 if (!new) {
998 ret = -ENOMEM;
999 goto out;
1000 }
1001
1002 src_node->links = new;
1003 src_node->links[src_node->num_links++] = *dst_node;
1004
1005out:
1006 mutex_unlock(&icc_lock);
1007
1008 return ret;
1009}
1010EXPORT_SYMBOL_GPL(icc_link_nodes);
1011
1012/**
1013 * icc_link_create() - create a link between two nodes
1014 * @node: source node id
1015 * @dst_id: destination node id
1016 *
1017 * Create a link between two nodes. The nodes might belong to different
1018 * interconnect providers and the @dst_id node might not exist (if the
1019 * provider driver has not probed yet). So just create the @dst_id node
1020 * and when the actual provider driver is probed, the rest of the node
1021 * data is filled.
1022 *
1023 * Return: 0 on success, or an error code otherwise
1024 */
1025int icc_link_create(struct icc_node *node, const int dst_id)
1026{
1027 struct icc_node *dst;
1028 struct icc_node **new;
1029 int ret = 0;
1030
1031 if (!node->provider)
1032 return -EINVAL;
1033
1034 mutex_lock(&icc_lock);
1035
1036 dst = node_find(dst_id);
1037 if (!dst) {
1038 dst = icc_node_create_nolock(dst_id);
1039
1040 if (IS_ERR(dst)) {
1041 ret = PTR_ERR(dst);
1042 goto out;
1043 }
1044 }
1045
1046 new = krealloc(node->links,
1047 (node->num_links + 1) * sizeof(*node->links),
1048 GFP_KERNEL);
1049 if (!new) {
1050 ret = -ENOMEM;
1051 goto out;
1052 }
1053
1054 node->links = new;
1055 node->links[node->num_links++] = dst;
1056
1057out:
1058 mutex_unlock(&icc_lock);
1059
1060 return ret;
1061}
1062EXPORT_SYMBOL_GPL(icc_link_create);
1063
1064/**
1065 * icc_node_add() - add interconnect node to interconnect provider
1066 * @node: pointer to the interconnect node
1067 * @provider: pointer to the interconnect provider
1068 */
1069void icc_node_add(struct icc_node *node, struct icc_provider *provider)
1070{
1071 if (WARN_ON(node->provider))
1072 return;
1073
1074 mutex_lock(&icc_lock);
1075 mutex_lock(&icc_bw_lock);
1076
1077 node->provider = provider;
1078 list_add_tail(&node->node_list, &provider->nodes);
1079
1080 /* get the initial bandwidth values and sync them with hardware */
1081 if (provider->get_bw) {
1082 provider->get_bw(node, &node->init_avg, &node->init_peak);
1083 } else {
1084 node->init_avg = INT_MAX;
1085 node->init_peak = INT_MAX;
1086 }
1087 node->avg_bw = node->init_avg;
1088 node->peak_bw = node->init_peak;
1089
1090 if (node->avg_bw || node->peak_bw) {
1091 if (provider->pre_aggregate)
1092 provider->pre_aggregate(node);
1093
1094 if (provider->aggregate)
1095 provider->aggregate(node, 0, node->init_avg, node->init_peak,
1096 &node->avg_bw, &node->peak_bw);
1097 if (provider->set)
1098 provider->set(node, node);
1099 }
1100
1101 node->avg_bw = 0;
1102 node->peak_bw = 0;
1103
1104 mutex_unlock(&icc_bw_lock);
1105 mutex_unlock(&icc_lock);
1106}
1107EXPORT_SYMBOL_GPL(icc_node_add);
1108
1109/**
1110 * icc_node_del() - delete interconnect node from interconnect provider
1111 * @node: pointer to the interconnect node
1112 */
1113void icc_node_del(struct icc_node *node)
1114{
1115 mutex_lock(&icc_lock);
1116
1117 list_del(&node->node_list);
1118
1119 mutex_unlock(&icc_lock);
1120}
1121EXPORT_SYMBOL_GPL(icc_node_del);
1122
1123/**
1124 * icc_nodes_remove() - remove all previously added nodes from provider
1125 * @provider: the interconnect provider we are removing nodes from
1126 *
1127 * Return: 0 on success, or an error code otherwise
1128 */
1129int icc_nodes_remove(struct icc_provider *provider)
1130{
1131 struct icc_node *n, *tmp;
1132
1133 if (WARN_ON(IS_ERR_OR_NULL(provider)))
1134 return -EINVAL;
1135
1136 list_for_each_entry_safe_reverse(n, tmp, &provider->nodes, node_list) {
1137 icc_node_del(n);
1138 icc_node_destroy(n->id);
1139 }
1140
1141 return 0;
1142}
1143EXPORT_SYMBOL_GPL(icc_nodes_remove);
1144
1145/**
1146 * icc_provider_init() - initialize a new interconnect provider
1147 * @provider: the interconnect provider to initialize
1148 *
1149 * Must be called before adding nodes to the provider.
1150 */
1151void icc_provider_init(struct icc_provider *provider)
1152{
1153 WARN_ON(!provider->set);
1154
1155 INIT_LIST_HEAD(&provider->nodes);
1156}
1157EXPORT_SYMBOL_GPL(icc_provider_init);
1158
1159/**
1160 * icc_provider_register() - register a new interconnect provider
1161 * @provider: the interconnect provider to register
1162 *
1163 * Return: 0 on success, or an error code otherwise
1164 */
1165int icc_provider_register(struct icc_provider *provider)
1166{
1167 if (WARN_ON(!provider->xlate && !provider->xlate_extended))
1168 return -EINVAL;
1169
1170 mutex_lock(&icc_lock);
1171 list_add_tail(&provider->provider_list, &icc_providers);
1172 mutex_unlock(&icc_lock);
1173
1174 dev_dbg(provider->dev, "interconnect provider registered\n");
1175
1176 return 0;
1177}
1178EXPORT_SYMBOL_GPL(icc_provider_register);
1179
1180/**
1181 * icc_provider_deregister() - deregister an interconnect provider
1182 * @provider: the interconnect provider to deregister
1183 */
1184void icc_provider_deregister(struct icc_provider *provider)
1185{
1186 mutex_lock(&icc_lock);
1187 WARN_ON(provider->users);
1188
1189 list_del(&provider->provider_list);
1190 mutex_unlock(&icc_lock);
1191}
1192EXPORT_SYMBOL_GPL(icc_provider_deregister);
1193
1194static const struct of_device_id __maybe_unused ignore_list[] = {
1195 { .compatible = "qcom,sc7180-ipa-virt" },
1196 { .compatible = "qcom,sc8180x-ipa-virt" },
1197 { .compatible = "qcom,sdx55-ipa-virt" },
1198 { .compatible = "qcom,sm8150-ipa-virt" },
1199 { .compatible = "qcom,sm8250-ipa-virt" },
1200 {}
1201};
1202
1203static int of_count_icc_providers(struct device_node *np)
1204{
1205 struct device_node *child;
1206 int count = 0;
1207
1208 for_each_available_child_of_node(np, child) {
1209 if (of_property_present(child, "#interconnect-cells") &&
1210 likely(!of_match_node(ignore_list, child)))
1211 count++;
1212 count += of_count_icc_providers(child);
1213 }
1214
1215 return count;
1216}
1217
1218void icc_sync_state(struct device *dev)
1219{
1220 struct icc_provider *p;
1221 struct icc_node *n;
1222 static int count;
1223
1224 count++;
1225
1226 if (count < providers_count)
1227 return;
1228
1229 mutex_lock(&icc_lock);
1230 mutex_lock(&icc_bw_lock);
1231 synced_state = true;
1232 list_for_each_entry(p, &icc_providers, provider_list) {
1233 dev_dbg(p->dev, "interconnect provider is in synced state\n");
1234 list_for_each_entry(n, &p->nodes, node_list) {
1235 if (n->init_avg || n->init_peak) {
1236 n->init_avg = 0;
1237 n->init_peak = 0;
1238 aggregate_requests(n);
1239 p->set(n, n);
1240 }
1241 }
1242 }
1243 mutex_unlock(&icc_bw_lock);
1244 mutex_unlock(&icc_lock);
1245}
1246EXPORT_SYMBOL_GPL(icc_sync_state);
1247
1248static int __init icc_init(void)
1249{
1250 struct device_node *root;
1251
1252 /* Teach lockdep about lock ordering wrt. shrinker: */
1253 fs_reclaim_acquire(GFP_KERNEL);
1254 might_lock(&icc_bw_lock);
1255 fs_reclaim_release(GFP_KERNEL);
1256
1257 root = of_find_node_by_path("/");
1258
1259 providers_count = of_count_icc_providers(root);
1260 of_node_put(root);
1261
1262 icc_debugfs_dir = debugfs_create_dir("interconnect", NULL);
1263 debugfs_create_file("interconnect_summary", 0444,
1264 icc_debugfs_dir, NULL, &icc_summary_fops);
1265 debugfs_create_file("interconnect_graph", 0444,
1266 icc_debugfs_dir, NULL, &icc_graph_fops);
1267
1268 icc_debugfs_client_init(icc_debugfs_dir);
1269
1270 return 0;
1271}
1272
1273device_initcall(icc_init);