linux - drivers/thunderbolt/tb.c

3439 lines · 89.2 KiB

1	// SPDX-License-Identifier: GPL-2.0
2	/*
3	* Thunderbolt driver - bus logic (NHI independent)
4	*
5	* Copyright (c) 2014 Andreas Noever <[email protected]>
6	* Copyright (C) 2019, Intel Corporation
7	*/
8
9	#include <linux/slab.h>
10	#include <linux/errno.h>
11	#include <linux/delay.h>
12	#include <linux/pm_runtime.h>
13	#include <linux/platform_data/x86/apple.h>
14
15	#include "tb.h"
16	#include "tb_regs.h"
17	#include "tunnel.h"
18
19	#define TB_TIMEOUT 100 /* ms */
20	#define TB_RELEASE_BW_TIMEOUT 10000 /* ms */
21
22	/*
23	* How many time bandwidth allocation request from graphics driver is
24	* retried if the DP tunnel is still activating.
25	*/
26	#define TB_BW_ALLOC_RETRIES 3
27
28	/*
29	* Minimum bandwidth (in Mb/s) that is needed in the single transmitter/receiver
30	* direction. This is 40G - 10% guard band bandwidth.
31	*/
32	#define TB_ASYM_MIN (40000 * 90 / 100)
33
34	/*
35	* Threshold bandwidth (in Mb/s) that is used to switch the links to
36	* asymmetric and back. This is selected as 45G which means when the
37	* request is higher than this, we switch the link to asymmetric, and
38	* when it is less than this we switch it back. The 45G is selected so
39	* that we still have 27G (of the total 72G) for bulk PCIe traffic when
40	* switching back to symmetric.
41	*/
42	#define TB_ASYM_THRESHOLD 45000
43
44	#define MAX_GROUPS 7 /* max Group_ID is 7 */
45
46	static unsigned int asym_threshold = TB_ASYM_THRESHOLD;
47	module_param_named(asym_threshold, asym_threshold, uint, 0444);
48	MODULE_PARM_DESC(asym_threshold,
49	"threshold (Mb/s) when to Gen 4 switch link symmetry. 0 disables. (default: "
50	__MODULE_STRING(TB_ASYM_THRESHOLD) ")");
51
52	/**
53	* struct tb_cm - Simple Thunderbolt connection manager
54	* @tunnel_list: List of active tunnels
55	* @dp_resources: List of available DP resources for DP tunneling
56	* @hotplug_active: tb_handle_hotplug will stop progressing plug
57	* events and exit if this is not set (it needs to
58	* acquire the lock one more time). Used to drain wq
59	* after cfg has been paused.
60	* @remove_work: Work used to remove any unplugged routers after
61	* runtime resume
62	* @groups: Bandwidth groups used in this domain.
63	*/
64	struct tb_cm {
65	struct list_head tunnel_list;
66	struct list_head dp_resources;
67	bool hotplug_active;
68	struct delayed_work remove_work;
69	struct tb_bandwidth_group groups[MAX_GROUPS];
70	};
71
72	static inline struct tb tcm_to_tb(struct tb_cm tcm)
73	{
74	return ((void *)tcm - sizeof(struct tb));
75	}
76
77	struct tb_hotplug_event {
78	struct delayed_work work;
79	struct tb *tb;
80	u64 route;
81	u8 port;
82	bool unplug;
83	int retry;
84	};
85
86	static void tb_scan_port(struct tb_port *port);
87	static void tb_handle_hotplug(struct work_struct *work);
88	static void tb_dp_resource_unavailable(struct tb tb, struct tb_port port,
89	const char *reason);
90	static void tb_queue_dp_bandwidth_request(struct tb *tb, u64 route, u8 port,
91	int retry, unsigned long delay);
92	static void tb_dp_tunnel_active(struct tb_tunnel tunnel, void data);
93
94	static void tb_queue_hotplug(struct tb *tb, u64 route, u8 port, bool unplug)
95	{
96	struct tb_hotplug_event *ev;
97
98	ev = kmalloc_obj(*ev);
99	if (!ev)
100	return;
101
102	ev->tb = tb_domain_get(tb);
103	ev->route = route;
104	ev->port = port;
105	ev->unplug = unplug;
106	INIT_DELAYED_WORK(&ev->work, tb_handle_hotplug);
107	queue_delayed_work(tb->wq, &ev->work, 0);
108	}
109
110	/* enumeration & hot plug handling */
111
112	static void tb_add_dp_resources(struct tb_switch *sw)
113	{
114	struct tb_cm *tcm = tb_priv(sw->tb);
115	struct tb_port *port;
116
117	tb_switch_for_each_port(sw, port) {
118	if (!tb_port_is_dpin(port))
119	continue;
120
121	if (!tb_switch_query_dp_resource(sw, port))
122	continue;
123
124	/*
125	* If DP IN on device router exist, position it at the
126	* beginning of the DP resources list, so that it is used
127	* before DP IN of the host router. This way external GPU(s)
128	* will be prioritized when pairing DP IN to a DP OUT.
129	*/
130	if (tb_route(sw))
131	list_add(&port->list, &tcm->dp_resources);
132	else
133	list_add_tail(&port->list, &tcm->dp_resources);
134
135	tb_port_dbg(port, "DP IN resource available\n");
136	}
137	}
138
139	static void tb_remove_dp_resources(struct tb_switch *sw)
140	{
141	struct tb_cm *tcm = tb_priv(sw->tb);
142	struct tb_port port, tmp;
143
144	/* Clear children resources first */
145	tb_switch_for_each_port(sw, port) {
146	if (tb_port_has_remote(port))
147	tb_remove_dp_resources(port->remote->sw);
148	}
149
150	list_for_each_entry_safe(port, tmp, &tcm->dp_resources, list) {
151	if (port->sw == sw) {
152	tb_port_dbg(port, "DP OUT resource unavailable\n");
153	list_del_init(&port->list);
154	}
155	}
156	}
157
158	static void tb_discover_dp_resource(struct tb tb, struct tb_port port)
159	{
160	struct tb_cm *tcm = tb_priv(tb);
161	struct tb_port *p;
162
163	list_for_each_entry(p, &tcm->dp_resources, list) {
164	if (p == port)
165	return;
166	}
167
168	tb_port_dbg(port, "DP %s resource available discovered\n",
169	tb_port_is_dpin(port) ? "IN" : "OUT");
170	list_add_tail(&port->list, &tcm->dp_resources);
171	}
172
173	static void tb_discover_dp_resources(struct tb *tb)
174	{
175	struct tb_cm *tcm = tb_priv(tb);
176	struct tb_tunnel *tunnel;
177
178	list_for_each_entry(tunnel, &tcm->tunnel_list, list) {
179	if (tb_tunnel_is_dp(tunnel))
180	tb_discover_dp_resource(tb, tunnel->dst_port);
181	}
182	}
183
184	/* Enables CL states up to host router */
185	static int tb_enable_clx(struct tb_switch *sw)
186	{
187	struct tb_cm *tcm = tb_priv(sw->tb);
188	unsigned int clx = TB_CL0S \| TB_CL1;
189	const struct tb_tunnel *tunnel;
190	int ret;
191
192	/*
193	* Currently only enable CLx for the first link. This is enough
194	* to allow the CPU to save energy at least on Intel hardware
195	* and makes it slightly simpler to implement. We may change
196	* this in the future to cover the whole topology if it turns
197	* out to be beneficial.
198	*/
199	while (sw && tb_switch_depth(sw) > 1)
200	sw = tb_switch_parent(sw);
201
202	if (!sw)
203	return 0;
204
205	if (tb_switch_depth(sw) != 1)
206	return 0;
207
208	/*
209	* If we are re-enabling then check if there is an active DMA
210	* tunnel and in that case bail out.
211	*/
212	list_for_each_entry(tunnel, &tcm->tunnel_list, list) {
213	if (tb_tunnel_is_dma(tunnel)) {
214	if (tb_tunnel_port_on_path(tunnel, tb_upstream_port(sw)))
215	return 0;
216	}
217	}
218
219	/*
220	* Initially try with CL2. If that's not supported by the
221	* topology try with CL0s and CL1 and then give up.
222	*/
223	ret = tb_switch_clx_enable(sw, clx \| TB_CL2);
224	if (ret == -EOPNOTSUPP)
225	ret = tb_switch_clx_enable(sw, clx);
226	return ret == -EOPNOTSUPP ? 0 : ret;
227	}
228
229	/*
230	* Disables CL states from @sw up to the host router.
231	*
232	* This can be used to figure out whether the link was setup by us or the
233	* boot firmware so we don't accidentally enable them if they were not
234	* enabled during discovery.
235	*/
236	static bool tb_disable_clx(struct tb_switch *sw)
237	{
238	bool disabled = false;
239
240	do {
241	int ret;
242
243	ret = tb_switch_clx_disable(sw);
244	if (ret > 0)
245	disabled = true;
246	else if (ret < 0)
247	tb_sw_warn(sw, "failed to disable CL states\n");
248
249	sw = tb_switch_parent(sw);
250	} while (sw);
251
252	return disabled;
253	}
254
255	static int tb_increase_switch_tmu_accuracy(struct device dev, void data)
256	{
257	struct tb_switch *sw;
258
259	sw = tb_to_switch(dev);
260	if (!sw)
261	return 0;
262
263	if (tb_switch_tmu_is_configured(sw, TB_SWITCH_TMU_MODE_LOWRES)) {
264	enum tb_switch_tmu_mode mode;
265	int ret;
266
267	if (tb_switch_clx_is_enabled(sw, TB_CL1))
268	mode = TB_SWITCH_TMU_MODE_HIFI_UNI;
269	else
270	mode = TB_SWITCH_TMU_MODE_HIFI_BI;
271
272	ret = tb_switch_tmu_configure(sw, mode);
273	if (ret)
274	return ret;
275
276	return tb_switch_tmu_enable(sw);
277	}
278
279	return 0;
280	}
281
282	static void tb_increase_tmu_accuracy(struct tb_tunnel *tunnel)
283	{
284	struct tb_switch *sw;
285
286	if (!tunnel)
287	return;
288
289	/*
290	* Once first DP tunnel is established we change the TMU
291	* accuracy of first depth child routers (and the host router)
292	* to the highest. This is needed for the DP tunneling to work
293	* but also allows CL0s.
294	*
295	* If both routers are v2 then we don't need to do anything as
296	* they are using enhanced TMU mode that allows all CLx.
297	*/
298	sw = tunnel->tb->root_switch;
299	device_for_each_child(&sw->dev, NULL, tb_increase_switch_tmu_accuracy);
300	}
301
302	static int tb_switch_tmu_hifi_uni_required(struct device dev, void not_used)
303	{
304	struct tb_switch *sw = tb_to_switch(dev);
305
306	if (sw && tb_switch_tmu_is_enabled(sw) &&
307	tb_switch_tmu_is_configured(sw, TB_SWITCH_TMU_MODE_HIFI_UNI))
308	return 1;
309
310	return device_for_each_child(dev, NULL,
311	tb_switch_tmu_hifi_uni_required);
312	}
313
314	static bool tb_tmu_hifi_uni_required(struct tb *tb)
315	{
316	return device_for_each_child(&tb->dev, NULL,
317	tb_switch_tmu_hifi_uni_required) == 1;
318	}
319
320	static int tb_enable_tmu(struct tb_switch *sw)
321	{
322	int ret;
323
324	/*
325	* If both routers at the end of the link are v2 we simply
326	* enable the enhanced uni-directional mode. That covers all
327	* the CL states. For v1 and before we need to use the normal
328	* rate to allow CL1 (when supported). Otherwise we keep the TMU
329	* running at the highest accuracy.
330	*/
331	ret = tb_switch_tmu_configure(sw,
332	TB_SWITCH_TMU_MODE_MEDRES_ENHANCED_UNI);
333	if (ret == -EOPNOTSUPP) {
334	if (tb_switch_clx_is_enabled(sw, TB_CL1)) {
335	/*
336	* Figure out uni-directional HiFi TMU requirements
337	* currently in the domain. If there are no
338	* uni-directional HiFi requirements we can put the TMU
339	* into LowRes mode.
340	*
341	* Deliberately skip bi-directional HiFi links
342	* as these work independently of other links
343	* (and they do not allow any CL states anyway).
344	*/
345	if (tb_tmu_hifi_uni_required(sw->tb))
346	ret = tb_switch_tmu_configure(sw,
347	TB_SWITCH_TMU_MODE_HIFI_UNI);
348	else
349	ret = tb_switch_tmu_configure(sw,
350	TB_SWITCH_TMU_MODE_LOWRES);
351	} else {
352	ret = tb_switch_tmu_configure(sw, TB_SWITCH_TMU_MODE_HIFI_BI);
353	}
354
355	/* If not supported, fallback to bi-directional HiFi */
356	if (ret == -EOPNOTSUPP)
357	ret = tb_switch_tmu_configure(sw, TB_SWITCH_TMU_MODE_HIFI_BI);
358	}
359	if (ret)
360	return ret;
361
362	/* If it is already enabled in correct mode, don't touch it */
363	if (tb_switch_tmu_is_enabled(sw))
364	return 0;
365
366	ret = tb_switch_tmu_disable(sw);
367	if (ret)
368	return ret;
369
370	ret = tb_switch_tmu_post_time(sw);
371	if (ret)
372	return ret;
373
374	return tb_switch_tmu_enable(sw);
375	}
376
377	static void tb_switch_discover_tunnels(struct tb_switch *sw,
378	struct list_head *list,
379	bool alloc_hopids)
380	{
381	struct tb *tb = sw->tb;
382	struct tb_port *port;
383
384	tb_switch_for_each_port(sw, port) {
385	struct tb_tunnel *tunnel = NULL;
386
387	switch (port->config.type) {
388	case TB_TYPE_DP_HDMI_IN:
389	tunnel = tb_tunnel_discover_dp(tb, port, alloc_hopids,
390	tb_dp_tunnel_active, tb);
391	tb_increase_tmu_accuracy(tunnel);
392	break;
393
394	case TB_TYPE_PCIE_DOWN:
395	tunnel = tb_tunnel_discover_pci(tb, port, alloc_hopids);
396	break;
397
398	case TB_TYPE_USB3_DOWN:
399	tunnel = tb_tunnel_discover_usb3(tb, port, alloc_hopids);
400	break;
401
402	default:
403	break;
404	}
405
406	if (tunnel)
407	list_add_tail(&tunnel->list, list);
408	}
409
410	tb_switch_for_each_port(sw, port) {
411	if (tb_port_has_remote(port)) {
412	tb_switch_discover_tunnels(port->remote->sw, list,
413	alloc_hopids);
414	}
415	}
416	}
417
418	static int tb_port_configure_xdomain(struct tb_port port, struct tb_xdomain xd)
419	{
420	if (tb_switch_is_usb4(port->sw))
421	return usb4_port_configure_xdomain(port, xd);
422	return tb_lc_configure_xdomain(port);
423	}
424
425	static void tb_port_unconfigure_xdomain(struct tb_port *port)
426	{
427	if (tb_switch_is_usb4(port->sw))
428	usb4_port_unconfigure_xdomain(port);
429	else
430	tb_lc_unconfigure_xdomain(port);
431	}
432
433	static void tb_scan_xdomain(struct tb_port *port)
434	{
435	struct tb_switch *sw = port->sw;
436	struct tb *tb = sw->tb;
437	struct tb_xdomain *xd;
438	u64 route;
439
440	if (!tb_is_xdomain_enabled())
441	return;
442
443	route = tb_downstream_route(port);
444	xd = tb_xdomain_find_by_route(tb, route);
445	if (xd) {
446	tb_xdomain_put(xd);
447	return;
448	}
449
450	xd = tb_xdomain_alloc(tb, &sw->dev, route, tb->root_switch->uuid,
451	NULL);
452	if (xd) {
453	tb_port_at(route, sw)->xdomain = xd;
454	tb_port_configure_xdomain(port, xd);
455	tb_xdomain_add(xd);
456	}
457	}
458
459	/*
460	* Returns the first inactive port on @sw.
461	*/
462	static struct tb_port tb_find_unused_port(struct tb_switch sw,
463	enum tb_port_type type)
464	{
465	struct tb_port *port;
466
467	tb_switch_for_each_port(sw, port) {
468	if (tb_is_upstream_port(port))
469	continue;
470	if (port->config.type != type)
471	continue;
472	if (!port->cap_adap)
473	continue;
474	if (tb_port_is_enabled(port))
475	continue;
476	return port;
477	}
478	return NULL;
479	}
480
481	static struct tb_port tb_find_usb3_down(struct tb_switch sw,
482	const struct tb_port *port)
483	{
484	struct tb_port *down;
485
486	down = usb4_switch_map_usb3_down(sw, port);
487	if (down && !tb_usb3_port_is_enabled(down))
488	return down;
489	return NULL;
490	}
491
492	static struct tb_tunnel tb_find_tunnel(struct tb tb, enum tb_tunnel_type type,
493	struct tb_port *src_port,
494	struct tb_port *dst_port)
495	{
496	struct tb_cm *tcm = tb_priv(tb);
497	struct tb_tunnel *tunnel;
498
499	list_for_each_entry(tunnel, &tcm->tunnel_list, list) {
500	if (tunnel->type == type &&
501	((src_port && src_port == tunnel->src_port) \|\|
502	(dst_port && dst_port == tunnel->dst_port))) {
503	return tunnel;
504	}
505	}
506
507	return NULL;
508	}
509
510	static struct tb_tunnel tb_find_first_usb3_tunnel(struct tb tb,
511	struct tb_port *src_port,
512	struct tb_port *dst_port)
513	{
514	struct tb_port port, usb3_down;
515	struct tb_switch *sw;
516
517	/* Pick the router that is deepest in the topology */
518	if (tb_port_path_direction_downstream(src_port, dst_port))
519	sw = dst_port->sw;
520	else
521	sw = src_port->sw;
522
523	/* Can't be the host router */
524	if (sw == tb->root_switch)
525	return NULL;
526
527	/* Find the downstream USB4 port that leads to this router */
528	port = tb_port_at(tb_route(sw), tb->root_switch);
529	/* Find the corresponding host router USB3 downstream port */
530	usb3_down = usb4_switch_map_usb3_down(tb->root_switch, port);
531	if (!usb3_down)
532	return NULL;
533
534	return tb_find_tunnel(tb, TB_TUNNEL_USB3, usb3_down, NULL);
535	}
536
537	/**
538	* tb_consumed_usb3_pcie_bandwidth() - Consumed USB3/PCIe bandwidth over a single link
539	* @tb: Domain structure
540	* @src_port: Source protocol adapter
541	* @dst_port: Destination protocol adapter
542	* @port: USB4 port the consumed bandwidth is calculated
543	* @consumed_up: Consumed upstream bandwidth (Mb/s)
544	* @consumed_down: Consumed downstream bandwidth (Mb/s)
545	*
546	* Calculates consumed USB3 and PCIe bandwidth at @port between path
547	* from @src_port to @dst_port. Does not take USB3 tunnel starting from
548	* @src_port and ending on @src_port into account because that bandwidth is
549	* already included in as part of the "first hop" USB3 tunnel.
550	*
551	* Return: %0 on success, negative errno otherwise.
552	*/
553	static int tb_consumed_usb3_pcie_bandwidth(struct tb *tb,
554	struct tb_port *src_port,
555	struct tb_port *dst_port,
556	struct tb_port *port,
557	int *consumed_up,
558	int *consumed_down)
559	{
560	int pci_consumed_up, pci_consumed_down;
561	struct tb_tunnel *tunnel;
562
563	consumed_up = consumed_down = 0;
564
565	tunnel = tb_find_first_usb3_tunnel(tb, src_port, dst_port);
566	if (tunnel && !tb_port_is_usb3_down(src_port) &&
567	!tb_port_is_usb3_up(dst_port)) {
568	int ret;
569
570	ret = tb_tunnel_consumed_bandwidth(tunnel, consumed_up,
571	consumed_down);
572	if (ret)
573	return ret;
574	}
575
576	/*
577	* If there is anything reserved for PCIe bulk traffic take it
578	* into account here too.
579	*/
580	if (tb_tunnel_reserved_pci(port, &pci_consumed_up, &pci_consumed_down)) {
581	*consumed_up += pci_consumed_up;
582	*consumed_down += pci_consumed_down;
583	}
584
585	return 0;
586	}
587
588	/**
589	* tb_consumed_dp_bandwidth() - Consumed DP bandwidth over a single link
590	* @tb: Domain structure
591	* @src_port: Source protocol adapter
592	* @dst_port: Destination protocol adapter
593	* @port: USB4 port the consumed bandwidth is calculated
594	* @consumed_up: Consumed upstream bandwidth (Mb/s)
595	* @consumed_down: Consumed downstream bandwidth (Mb/s)
596	*
597	* Calculates consumed DP bandwidth at @port between path from @src_port
598	* to @dst_port. Does not take tunnel starting from @src_port and ending
599	* from @src_port into account.
600	*
601	* If there is bandwidth reserved for any of the groups between
602	* @src_port and @dst_port (but not yet used) that is also taken into
603	* account in the returned consumed bandwidth.
604	*
605	* Return: %0 on success, negative errno otherwise.
606	*/
607	static int tb_consumed_dp_bandwidth(struct tb *tb,
608	struct tb_port *src_port,
609	struct tb_port *dst_port,
610	struct tb_port *port,
611	int *consumed_up,
612	int *consumed_down)
613	{
614	int group_reserved[MAX_GROUPS + 1] = {};
615	struct tb_cm *tcm = tb_priv(tb);
616	struct tb_tunnel *tunnel;
617	bool downstream;
618	int i, ret;
619
620	consumed_up = consumed_down = 0;
621
622	/*
623	* Find all DP tunnels that cross the port and reduce
624	* their consumed bandwidth from the available.
625	*/
626	list_for_each_entry(tunnel, &tcm->tunnel_list, list) {
627	const struct tb_bandwidth_group *group;
628	int dp_consumed_up, dp_consumed_down;
629
630	if (tb_tunnel_is_invalid(tunnel))
631	continue;
632
633	if (!tb_tunnel_is_dp(tunnel))
634	continue;
635
636	if (!tb_tunnel_port_on_path(tunnel, port))
637	continue;
638
639	/*
640	* Calculate what is reserved for groups crossing the
641	* same ports only once (as that is reserved for all the
642	* tunnels in the group).
643	*/
644	group = tunnel->src_port->group;
645	if (group && group->reserved && !group_reserved[group->index])
646	group_reserved[group->index] = group->reserved;
647
648	/*
649	* Ignore the DP tunnel between src_port and dst_port
650	* because it is the same tunnel and we may be
651	* re-calculating estimated bandwidth.
652	*/
653	if (tunnel->src_port == src_port &&
654	tunnel->dst_port == dst_port)
655	continue;
656
657	ret = tb_tunnel_consumed_bandwidth(tunnel, &dp_consumed_up,
658	&dp_consumed_down);
659	if (ret)
660	return ret;
661
662	*consumed_up += dp_consumed_up;
663	*consumed_down += dp_consumed_down;
664	}
665
666	downstream = tb_port_path_direction_downstream(src_port, dst_port);
667	for (i = 0; i < ARRAY_SIZE(group_reserved); i++) {
668	if (downstream)
669	*consumed_down += group_reserved[i];
670	else
671	*consumed_up += group_reserved[i];
672	}
673
674	return 0;
675	}
676
677	static bool tb_asym_supported(struct tb_port src_port, struct tb_port dst_port,
678	struct tb_port *port)
679	{
680	bool downstream = tb_port_path_direction_downstream(src_port, dst_port);
681	enum tb_link_width width;
682
683	if (tb_is_upstream_port(port))
684	width = downstream ? TB_LINK_WIDTH_ASYM_RX : TB_LINK_WIDTH_ASYM_TX;
685	else
686	width = downstream ? TB_LINK_WIDTH_ASYM_TX : TB_LINK_WIDTH_ASYM_RX;
687
688	return tb_port_width_supported(port, width);
689	}
690
691	/**
692	* tb_maximum_bandwidth() - Maximum bandwidth over a single link
693	* @tb: Domain structure
694	* @src_port: Source protocol adapter
695	* @dst_port: Destination protocol adapter
696	* @port: USB4 port the total bandwidth is calculated
697	* @max_up: Maximum upstream bandwidth (Mb/s)
698	* @max_down: Maximum downstream bandwidth (Mb/s)
699	* @include_asym: Include bandwidth if the link is switched from
700	* symmetric to asymmetric
701	*
702	* Returns maximum possible bandwidth in @max_up and @max_down over a
703	* single link at @port. If @include_asym is set then includes the
704	* additional banwdith if the links are transitioned into asymmetric to
705	* direction from @src_port to @dst_port.
706	*
707	* Return: %0 on success, negative errno otherwise.
708	*/
709	static int tb_maximum_bandwidth(struct tb tb, struct tb_port src_port,
710	struct tb_port dst_port, struct tb_port port,
711	int max_up, int max_down, bool include_asym)
712	{
713	bool downstream = tb_port_path_direction_downstream(src_port, dst_port);
714	int link_speed, link_width, up_bw, down_bw;
715
716	/*
717	* Can include asymmetric, only if it is actually supported by
718	* the lane adapter.
719	*/
720	if (!tb_asym_supported(src_port, dst_port, port))
721	include_asym = false;
722
723	if (tb_is_upstream_port(port)) {
724	link_speed = port->sw->link_speed;
725	/*
726	* sw->link_width is from upstream perspective so we use
727	* the opposite for downstream of the host router.
728	*/
729	if (port->sw->link_width == TB_LINK_WIDTH_ASYM_TX) {
730	up_bw = link_speed * 3 * 1000;
731	down_bw = link_speed * 1 * 1000;
732	} else if (port->sw->link_width == TB_LINK_WIDTH_ASYM_RX) {
733	up_bw = link_speed * 1 * 1000;
734	down_bw = link_speed * 3 * 1000;
735	} else if (include_asym) {
736	/*
737	* The link is symmetric at the moment but we
738	* can switch it to asymmetric as needed. Report
739	* this bandwidth as available (even though it
740	* is not yet enabled).
741	*/
742	if (downstream) {
743	up_bw = link_speed * 1 * 1000;
744	down_bw = link_speed * 3 * 1000;
745	} else {
746	up_bw = link_speed * 3 * 1000;
747	down_bw = link_speed * 1 * 1000;
748	}
749	} else {
750	up_bw = link_speed * port->sw->link_width * 1000;
751	down_bw = up_bw;
752	}
753	} else {
754	link_speed = tb_port_get_link_speed(port);
755	if (link_speed < 0)
756	return link_speed;
757
758	link_width = tb_port_get_link_width(port);
759	if (link_width < 0)
760	return link_width;
761
762	if (link_width == TB_LINK_WIDTH_ASYM_TX) {
763	up_bw = link_speed * 1 * 1000;
764	down_bw = link_speed * 3 * 1000;
765	} else if (link_width == TB_LINK_WIDTH_ASYM_RX) {
766	up_bw = link_speed * 3 * 1000;
767	down_bw = link_speed * 1 * 1000;
768	} else if (include_asym) {
769	/*
770	* The link is symmetric at the moment but we
771	* can switch it to asymmetric as needed. Report
772	* this bandwidth as available (even though it
773	* is not yet enabled).
774	*/
775	if (downstream) {
776	up_bw = link_speed * 1 * 1000;
777	down_bw = link_speed * 3 * 1000;
778	} else {
779	up_bw = link_speed * 3 * 1000;
780	down_bw = link_speed * 1 * 1000;
781	}
782	} else {
783	up_bw = link_speed * link_width * 1000;
784	down_bw = up_bw;
785	}
786	}
787
788	/* Leave 10% guard band */
789	*max_up = up_bw - up_bw / 10;
790	*max_down = down_bw - down_bw / 10;
791
792	tb_port_dbg(port, "link maximum bandwidth %d/%d Mb/s\n", max_up, max_down);
793	return 0;
794	}
795
796	/**
797	* tb_available_bandwidth() - Available bandwidth for tunneling
798	* @tb: Domain structure
799	* @src_port: Source protocol adapter
800	* @dst_port: Destination protocol adapter
801	* @available_up: Available bandwidth upstream (Mb/s)
802	* @available_down: Available bandwidth downstream (Mb/s)
803	* @include_asym: Include bandwidth if the link is switched from
804	* symmetric to asymmetric
805	*
806	* Calculates maximum available bandwidth for protocol tunneling between
807	* @src_port and @dst_port at the moment. This is minimum of maximum
808	* link bandwidth across all links reduced by currently consumed
809	* bandwidth on that link.
810	*
811	* If @include_asym is true then includes also bandwidth that can be
812	* added when the links are transitioned into asymmetric (but does not
813	* transition the links).
814	*
815	* Return: %0 on success, negative errno otherwise.
816	*/
817	static int tb_available_bandwidth(struct tb tb, struct tb_port src_port,
818	struct tb_port dst_port, int available_up,
819	int *available_down, bool include_asym)
820	{
821	struct tb_port *port;
822	int ret;
823
824	/* Maximum possible bandwidth asymmetric Gen 4 link is 120 Gb/s */
825	available_up = available_down = 120000;
826
827	/* Find the minimum available bandwidth over all links */
828	tb_for_each_port_on_path(src_port, dst_port, port) {
829	int max_up, max_down, consumed_up, consumed_down;
830
831	if (!tb_port_is_null(port))
832	continue;
833
834	ret = tb_maximum_bandwidth(tb, src_port, dst_port, port,
835	&max_up, &max_down, include_asym);
836	if (ret)
837	return ret;
838
839	ret = tb_consumed_usb3_pcie_bandwidth(tb, src_port, dst_port,
840	port, &consumed_up,
841	&consumed_down);
842	if (ret)
843	return ret;
844	max_up -= consumed_up;
845	max_down -= consumed_down;
846
847	ret = tb_consumed_dp_bandwidth(tb, src_port, dst_port, port,
848	&consumed_up, &consumed_down);
849	if (ret)
850	return ret;
851	max_up -= consumed_up;
852	max_down -= consumed_down;
853
854	if (max_up < *available_up)
855	*available_up = max_up;
856	if (max_down < *available_down)
857	*available_down = max_down;
858	}
859
860	if (*available_up < 0)
861	*available_up = 0;
862	if (*available_down < 0)
863	*available_down = 0;
864
865	return 0;
866	}
867
868	static int tb_release_unused_usb3_bandwidth(struct tb *tb,
869	struct tb_port *src_port,
870	struct tb_port *dst_port)
871	{
872	struct tb_tunnel *tunnel;
873
874	tunnel = tb_find_first_usb3_tunnel(tb, src_port, dst_port);
875	return tunnel ? tb_tunnel_release_unused_bandwidth(tunnel) : 0;
876	}
877
878	static void tb_reclaim_usb3_bandwidth(struct tb tb, struct tb_port src_port,
879	struct tb_port *dst_port)
880	{
881	int ret, available_up, available_down;
882	struct tb_tunnel *tunnel;
883
884	tunnel = tb_find_first_usb3_tunnel(tb, src_port, dst_port);
885	if (!tunnel)
886	return;
887
888	tb_tunnel_dbg(tunnel, "reclaiming unused bandwidth\n");
889
890	/*
891	* Calculate available bandwidth for the first hop USB3 tunnel.
892	* That determines the whole USB3 bandwidth for this branch.
893	*/
894	ret = tb_available_bandwidth(tb, tunnel->src_port, tunnel->dst_port,
895	&available_up, &available_down, false);
896	if (ret) {
897	tb_tunnel_warn(tunnel, "failed to calculate available bandwidth\n");
898	return;
899	}
900
901	tb_tunnel_dbg(tunnel, "available bandwidth %d/%d Mb/s\n", available_up,
902	available_down);
903
904	tb_tunnel_reclaim_available_bandwidth(tunnel, &available_up, &available_down);
905	}
906
907	static int tb_tunnel_usb3(struct tb tb, struct tb_switch sw)
908	{
909	struct tb_switch *parent = tb_switch_parent(sw);
910	int ret, available_up, available_down;
911	struct tb_port up, down, *port;
912	struct tb_cm *tcm = tb_priv(tb);
913	struct tb_tunnel *tunnel;
914
915	if (!tb_acpi_may_tunnel_usb3()) {
916	tb_dbg(tb, "USB3 tunneling disabled, not creating tunnel\n");
917	return 0;
918	}
919
920	up = tb_switch_find_port(sw, TB_TYPE_USB3_UP);
921	if (!up)
922	return 0;
923
924	if (!sw->link_usb4)
925	return 0;
926
927	/*
928	* Look up available down port. Since we are chaining it should
929	* be found right above this switch.
930	*/
931	port = tb_switch_downstream_port(sw);
932	down = tb_find_usb3_down(parent, port);
933	if (!down)
934	return 0;
935
936	if (tb_route(parent)) {
937	struct tb_port *parent_up;
938	/*
939	* Check first that the parent switch has its upstream USB3
940	* port enabled. Otherwise the chain is not complete and
941	* there is no point setting up a new tunnel.
942	*/
943	parent_up = tb_switch_find_port(parent, TB_TYPE_USB3_UP);
944	if (!parent_up \|\| !tb_port_is_enabled(parent_up))
945	return 0;
946
947	/* Make all unused bandwidth available for the new tunnel */
948	ret = tb_release_unused_usb3_bandwidth(tb, down, up);
949	if (ret)
950	return ret;
951	}
952
953	ret = tb_available_bandwidth(tb, down, up, &available_up, &available_down,
954	false);
955	if (ret)
956	goto err_reclaim;
957
958	tb_port_dbg(up, "available bandwidth for new USB3 tunnel %d/%d Mb/s\n",
959	available_up, available_down);
960
961	/*
962	* If the available bandwidth is less than 1.5 Gb/s notify
963	* userspace that the connected isochronous device may not work
964	* properly.
965	*/
966	if (available_up < 1500 \|\| available_down < 1500)
967	tb_tunnel_event(tb, TB_TUNNEL_LOW_BANDWIDTH, TB_TUNNEL_USB3,
968	down, up);
969
970	tunnel = tb_tunnel_alloc_usb3(tb, up, down, available_up,
971	available_down);
972	if (!tunnel) {
973	ret = -ENOMEM;
974	goto err_reclaim;
975	}
976
977	if (tb_tunnel_activate(tunnel)) {
978	tb_port_info(up,
979	"USB3 tunnel activation failed, aborting\n");
980	ret = -EIO;
981	goto err_free;
982	}
983
984	list_add_tail(&tunnel->list, &tcm->tunnel_list);
985	if (tb_route(parent))
986	tb_reclaim_usb3_bandwidth(tb, down, up);
987
988	return 0;
989
990	err_free:
991	tb_tunnel_put(tunnel);
992	err_reclaim:
993	if (tb_route(parent))
994	tb_reclaim_usb3_bandwidth(tb, down, up);
995
996	return ret;
997	}
998
999	static int tb_create_usb3_tunnels(struct tb_switch *sw)
1000	{
1001	struct tb_port *port;
1002	int ret;
1003
1004	if (!tb_acpi_may_tunnel_usb3())
1005	return 0;
1006
1007	if (tb_route(sw)) {
1008	ret = tb_tunnel_usb3(sw->tb, sw);
1009	if (ret)
1010	return ret;
1011	}
1012
1013	tb_switch_for_each_port(sw, port) {
1014	if (!tb_port_has_remote(port))
1015	continue;
1016	ret = tb_create_usb3_tunnels(port->remote->sw);
1017	if (ret)
1018	return ret;
1019	}
1020
1021	return 0;
1022	}
1023
1024	/**
1025	* tb_configure_asym() - Transition links to asymmetric if needed
1026	* @tb: Domain structure
1027	* @src_port: Source adapter to start the transition
1028	* @dst_port: Destination adapter
1029	* @requested_up: Additional bandwidth (Mb/s) required upstream
1030	* @requested_down: Additional bandwidth (Mb/s) required downstream
1031	*
1032	* Transition links between @src_port and @dst_port into asymmetric, with
1033	* three lanes in the direction from @src_port towards @dst_port and one lane
1034	* in the opposite direction, if the bandwidth requirements
1035	* (requested + currently consumed) on that link exceed @asym_threshold.
1036	*
1037	* Must be called with available >= requested over all links.
1038	*
1039	* Return: %0 on success, negative errno otherwise.
1040	*/
1041	static int tb_configure_asym(struct tb tb, struct tb_port src_port,
1042	struct tb_port *dst_port, int requested_up,
1043	int requested_down)
1044	{
1045	bool clx = false, clx_disabled = false, downstream;
1046	struct tb_switch *sw;
1047	struct tb_port *up;
1048	int ret = 0;
1049
1050	if (!asym_threshold)
1051	return 0;
1052
1053	downstream = tb_port_path_direction_downstream(src_port, dst_port);
1054	/* Pick up router deepest in the hierarchy */
1055	if (downstream)
1056	sw = dst_port->sw;
1057	else
1058	sw = src_port->sw;
1059
1060	tb_for_each_upstream_port_on_path(src_port, dst_port, up) {
1061	struct tb_port *down = tb_switch_downstream_port(up->sw);
1062	enum tb_link_width width_up, width_down;
1063	int consumed_up, consumed_down;
1064
1065	ret = tb_consumed_dp_bandwidth(tb, src_port, dst_port, up,
1066	&consumed_up, &consumed_down);
1067	if (ret)
1068	break;
1069
1070	if (downstream) {
1071	/*
1072	* Downstream so make sure upstream is within the 36G
1073	* (40G - guard band 10%), and the requested is above
1074	* what the threshold is.
1075	*/
1076	if (consumed_up + requested_up >= TB_ASYM_MIN) {
1077	ret = -ENOBUFS;
1078	break;
1079	}
1080	/* Does consumed + requested exceed the threshold */
1081	if (consumed_down + requested_down < asym_threshold)
1082	continue;
1083
1084	width_up = TB_LINK_WIDTH_ASYM_RX;
1085	width_down = TB_LINK_WIDTH_ASYM_TX;
1086	} else {
1087	/* Upstream, the opposite of above */
1088	if (consumed_down + requested_down >= TB_ASYM_MIN) {
1089	ret = -ENOBUFS;
1090	break;
1091	}
1092	if (consumed_up + requested_up < asym_threshold)
1093	continue;
1094
1095	width_up = TB_LINK_WIDTH_ASYM_TX;
1096	width_down = TB_LINK_WIDTH_ASYM_RX;
1097	}
1098
1099	if (up->sw->link_width == width_up)
1100	continue;
1101
1102	if (!tb_port_width_supported(up, width_up) \|\|
1103	!tb_port_width_supported(down, width_down))
1104	continue;
1105
1106	/*
1107	* Disable CL states before doing any transitions. We
1108	* delayed it until now that we know there is a real
1109	* transition taking place.
1110	*/
1111	if (!clx_disabled) {
1112	clx = tb_disable_clx(sw);
1113	clx_disabled = true;
1114	}
1115
1116	tb_sw_dbg(up->sw, "configuring asymmetric link\n");
1117
1118	/*
1119	* Here requested + consumed > threshold so we need to
1120	* transition the link into asymmetric now.
1121	*/
1122	ret = tb_switch_set_link_width(up->sw, width_up);
1123	if (ret) {
1124	tb_sw_warn(up->sw, "failed to set link width\n");
1125	break;
1126	}
1127	}
1128
1129	/* Re-enable CL states if they were previosly enabled */
1130	if (clx)
1131	tb_enable_clx(sw);
1132
1133	return ret;
1134	}
1135
1136	/**
1137	* tb_configure_sym() - Transition links to symmetric if possible
1138	* @tb: Domain structure
1139	* @src_port: Source adapter to start the transition
1140	* @dst_port: Destination adapter
1141	* @keep_asym: Keep asymmetric link if preferred
1142	*
1143	* Goes over each link from @src_port to @dst_port and tries to
1144	* transition the link to symmetric if the currently consumed bandwidth
1145	* allows and link asymmetric preference is ignored (if @keep_asym is %false).
1146	*
1147	* Return: %0 on success, negative errno otherwise.
1148	*/
1149	static int tb_configure_sym(struct tb tb, struct tb_port src_port,
1150	struct tb_port *dst_port, bool keep_asym)
1151	{
1152	bool clx = false, clx_disabled = false, downstream;
1153	struct tb_switch *sw;
1154	struct tb_port *up;
1155	int ret = 0;
1156
1157	if (!asym_threshold)
1158	return 0;
1159
1160	downstream = tb_port_path_direction_downstream(src_port, dst_port);
1161	/* Pick up router deepest in the hierarchy */
1162	if (downstream)
1163	sw = dst_port->sw;
1164	else
1165	sw = src_port->sw;
1166
1167	tb_for_each_upstream_port_on_path(src_port, dst_port, up) {
1168	int consumed_up, consumed_down;
1169
1170	/* Already symmetric */
1171	if (up->sw->link_width <= TB_LINK_WIDTH_DUAL)
1172	continue;
1173	/* Unplugged, no need to switch */
1174	if (up->sw->is_unplugged)
1175	continue;
1176
1177	ret = tb_consumed_dp_bandwidth(tb, src_port, dst_port, up,
1178	&consumed_up, &consumed_down);
1179	if (ret)
1180	break;
1181
1182	if (downstream) {
1183	/*
1184	* Downstream so we want the consumed_down < threshold.
1185	* Upstream traffic should be less than 36G (40G
1186	* guard band 10%) as the link was configured asymmetric
1187	* already.
1188	*/
1189	if (consumed_down >= asym_threshold)
1190	continue;
1191	} else {
1192	if (consumed_up >= asym_threshold)
1193	continue;
1194	}
1195
1196	if (up->sw->link_width == TB_LINK_WIDTH_DUAL)
1197	continue;
1198
1199	/*
1200	* Here consumed < threshold so we can transition the
1201	* link to symmetric.
1202	*
1203	* However, if the router prefers asymmetric link we
1204	* honor that (unless @keep_asym is %false).
1205	*/
1206	if (keep_asym &&
1207	up->sw->preferred_link_width > TB_LINK_WIDTH_DUAL) {
1208	tb_sw_dbg(up->sw, "keeping preferred asymmetric link\n");
1209	continue;
1210	}
1211
1212	/* Disable CL states before doing any transitions */
1213	if (!clx_disabled) {
1214	clx = tb_disable_clx(sw);
1215	clx_disabled = true;
1216	}
1217
1218	tb_sw_dbg(up->sw, "configuring symmetric link\n");
1219
1220	ret = tb_switch_set_link_width(up->sw, TB_LINK_WIDTH_DUAL);
1221	if (ret) {
1222	tb_sw_warn(up->sw, "failed to set link width\n");
1223	break;
1224	}
1225	}
1226
1227	/* Re-enable CL states if they were previosly enabled */
1228	if (clx)
1229	tb_enable_clx(sw);
1230
1231	return ret;
1232	}
1233
1234	static void tb_configure_link(struct tb_port down, struct tb_port up,
1235	struct tb_switch *sw)
1236	{
1237	struct tb *tb = sw->tb;
1238
1239	/* Link the routers using both links if available */
1240	down->remote = up;
1241	up->remote = down;
1242	if (down->dual_link_port && up->dual_link_port) {
1243	down->dual_link_port->remote = up->dual_link_port;
1244	up->dual_link_port->remote = down->dual_link_port;
1245	}
1246
1247	/*
1248	* Enable lane bonding if the link is currently two single lane
1249	* links.
1250	*/
1251	if (sw->link_width < TB_LINK_WIDTH_DUAL)
1252	tb_switch_set_link_width(sw, TB_LINK_WIDTH_DUAL);
1253
1254	/*
1255	* Device router that comes up as symmetric link is
1256	* connected deeper in the hierarchy, we transition the links
1257	* above into symmetric if bandwidth allows.
1258	*/
1259	if (tb_switch_depth(sw) > 1 &&
1260	tb_port_get_link_generation(up) >= 4 &&
1261	up->sw->link_width == TB_LINK_WIDTH_DUAL) {
1262	struct tb_port *host_port;
1263
1264	host_port = tb_port_at(tb_route(sw), tb->root_switch);
1265	tb_configure_sym(tb, host_port, up, false);
1266	}
1267
1268	/* Set the link configured */
1269	tb_switch_configure_link(sw);
1270	}
1271
1272	/*
1273	* tb_scan_switch() - scan for and initialize downstream switches
1274	*/
1275	static void tb_scan_switch(struct tb_switch *sw)
1276	{
1277	struct tb_port *port;
1278
1279	pm_runtime_get_sync(&sw->dev);
1280
1281	tb_switch_for_each_port(sw, port)
1282	tb_scan_port(port);
1283
1284	pm_runtime_mark_last_busy(&sw->dev);
1285	pm_runtime_put_autosuspend(&sw->dev);
1286	}
1287
1288	/*
1289	* tb_scan_port() - check for and initialize switches below port
1290	*/
1291	static void tb_scan_port(struct tb_port *port)
1292	{
1293	struct tb_cm *tcm = tb_priv(port->sw->tb);
1294	struct tb_port *upstream_port;
1295	bool discovery = false;
1296	struct tb_switch *sw;
1297
1298	if (tb_is_upstream_port(port))
1299	return;
1300
1301	if (tb_port_is_dpout(port) && tb_dp_port_hpd_is_active(port) == 1 &&
1302	!tb_dp_port_is_enabled(port)) {
1303	tb_port_dbg(port, "DP adapter HPD set, queuing hotplug\n");
1304	tb_queue_hotplug(port->sw->tb, tb_route(port->sw), port->port,
1305	false);
1306	return;
1307	}
1308
1309	if (port->config.type != TB_TYPE_PORT)
1310	return;
1311	if (port->dual_link_port && port->link_nr)
1312	return; /*
1313	* Downstream switch is reachable through two ports.
1314	* Only scan on the primary port (link_nr == 0).
1315	*/
1316
1317	if (port->usb4)
1318	pm_runtime_get_sync(&port->usb4->dev);
1319
1320	if (tb_wait_for_port(port, false) <= 0)
1321	goto out_rpm_put;
1322	if (port->remote) {
1323	tb_port_dbg(port, "port already has a remote\n");
1324	goto out_rpm_put;
1325	}
1326
1327	sw = tb_switch_alloc(port->sw->tb, &port->sw->dev,
1328	tb_downstream_route(port));
1329	if (IS_ERR(sw)) {
1330	/*
1331	* Make the downstream retimers available even if there
1332	* is no router connected.
1333	*/
1334	tb_retimer_scan(port, true);
1335
1336	/*
1337	* If there is an error accessing the connected switch
1338	* it may be connected to another domain. Also we allow
1339	* the other domain to be connected to a max depth switch.
1340	*/
1341	if (PTR_ERR(sw) == -EIO \|\| PTR_ERR(sw) == -EADDRNOTAVAIL)
1342	tb_scan_xdomain(port);
1343	goto out_rpm_put;
1344	}
1345
1346	if (tb_switch_configure(sw)) {
1347	tb_switch_put(sw);
1348	goto out_rpm_put;
1349	}
1350
1351	/*
1352	* If there was previously another domain connected remove it
1353	* first.
1354	*/
1355	if (port->xdomain) {
1356	tb_xdomain_remove(port->xdomain);
1357	tb_port_unconfigure_xdomain(port);
1358	port->xdomain = NULL;
1359	}
1360
1361	/*
1362	* Do not send uevents until we have discovered all existing
1363	* tunnels and know which switches were authorized already by
1364	* the boot firmware.
1365	*/
1366	if (!tcm->hotplug_active) {
1367	dev_set_uevent_suppress(&sw->dev, true);
1368	discovery = true;
1369	}
1370
1371	/*
1372	* At the moment Thunderbolt 2 and beyond (devices with LC) we
1373	* can support runtime PM.
1374	*/
1375	sw->rpm = sw->generation > 1;
1376
1377	if (tb_switch_add(sw)) {
1378	tb_switch_put(sw);
1379	goto out_rpm_put;
1380	}
1381
1382	upstream_port = tb_upstream_port(sw);
1383	tb_configure_link(port, upstream_port, sw);
1384
1385	/*
1386	* Scan for downstream retimers. We only scan them after the
1387	* router has been enumerated to avoid issues with certain
1388	* Pluggable devices that expect the host to enumerate them
1389	* within certain timeout.
1390	*/
1391	tb_retimer_scan(port, true);
1392
1393	/*
1394	* CL0s and CL1 are enabled and supported together.
1395	* Silently ignore CLx enabling in case CLx is not supported.
1396	*/
1397	if (discovery)
1398	tb_sw_dbg(sw, "discovery, not touching CL states\n");
1399	else if (tb_enable_clx(sw))
1400	tb_sw_warn(sw, "failed to enable CL states\n");
1401
1402	if (tb_enable_tmu(sw))
1403	tb_sw_warn(sw, "failed to enable TMU\n");
1404
1405	/*
1406	* Configuration valid needs to be set after the TMU has been
1407	* enabled for the upstream port of the router so we do it here.
1408	*/
1409	tb_switch_configuration_valid(sw);
1410
1411	/* Scan upstream retimers */
1412	tb_retimer_scan(upstream_port, true);
1413
1414	/*
1415	* Create USB 3.x tunnels only when the switch is plugged to the
1416	* domain. This is because we scan the domain also during discovery
1417	* and want to discover existing USB 3.x tunnels before we create
1418	* any new.
1419	*/
1420	if (tcm->hotplug_active && tb_tunnel_usb3(sw->tb, sw))
1421	tb_sw_warn(sw, "USB3 tunnel creation failed\n");
1422
1423	tb_add_dp_resources(sw);
1424	tb_scan_switch(sw);
1425
1426	out_rpm_put:
1427	if (port->usb4) {
1428	pm_runtime_mark_last_busy(&port->usb4->dev);
1429	pm_runtime_put_autosuspend(&port->usb4->dev);
1430	}
1431	}
1432
1433	static void
1434	tb_recalc_estimated_bandwidth_for_group(struct tb_bandwidth_group *group)
1435	{
1436	struct tb_tunnel *first_tunnel;
1437	struct tb *tb = group->tb;
1438	struct tb_port *in;
1439	int ret;
1440
1441	tb_dbg(tb, "re-calculating bandwidth estimation for group %u\n",
1442	group->index);
1443
1444	first_tunnel = NULL;
1445	list_for_each_entry(in, &group->ports, group_list) {
1446	int estimated_bw, estimated_up, estimated_down;
1447	struct tb_tunnel *tunnel;
1448	struct tb_port *out;
1449
1450	if (!usb4_dp_port_bandwidth_mode_enabled(in))
1451	continue;
1452
1453	tunnel = tb_find_tunnel(tb, TB_TUNNEL_DP, in, NULL);
1454	if (WARN_ON(!tunnel))
1455	break;
1456
1457	if (!first_tunnel) {
1458	/*
1459	* Since USB3 bandwidth is shared by all DP
1460	* tunnels under the host router USB4 port, even
1461	* if they do not begin from the host router, we
1462	* can release USB3 bandwidth just once and not
1463	* for each tunnel separately.
1464	*/
1465	first_tunnel = tunnel;
1466	ret = tb_release_unused_usb3_bandwidth(tb,
1467	first_tunnel->src_port, first_tunnel->dst_port);
1468	if (ret) {
1469	tb_tunnel_warn(tunnel,
1470	"failed to release unused bandwidth\n");
1471	break;
1472	}
1473	}
1474
1475	out = tunnel->dst_port;
1476	ret = tb_available_bandwidth(tb, in, out, &estimated_up,
1477	&estimated_down, true);
1478	if (ret) {
1479	tb_tunnel_warn(tunnel,
1480	"failed to re-calculate estimated bandwidth\n");
1481	break;
1482	}
1483
1484	/*
1485	* Estimated bandwidth includes:
1486	* - already allocated bandwidth for the DP tunnel
1487	* - available bandwidth along the path
1488	* - bandwidth allocated for USB 3.x but not used.
1489	*/
1490	if (tb_tunnel_direction_downstream(tunnel))
1491	estimated_bw = estimated_down;
1492	else
1493	estimated_bw = estimated_up;
1494
1495	/*
1496	* If there is reserved bandwidth for the group that is
1497	* not yet released we report that too.
1498	*/
1499	tb_tunnel_dbg(tunnel,
1500	"re-calculated estimated bandwidth %u (+ %u reserved) = %u Mb/s\n",
1501	estimated_bw, group->reserved,
1502	estimated_bw + group->reserved);
1503
1504	if (usb4_dp_port_set_estimated_bandwidth(in,
1505	estimated_bw + group->reserved))
1506	tb_tunnel_warn(tunnel,
1507	"failed to update estimated bandwidth\n");
1508	}
1509
1510	if (first_tunnel)
1511	tb_reclaim_usb3_bandwidth(tb, first_tunnel->src_port,
1512	first_tunnel->dst_port);
1513
1514	tb_dbg(tb, "bandwidth estimation for group %u done\n", group->index);
1515	}
1516
1517	static void tb_recalc_estimated_bandwidth(struct tb *tb)
1518	{
1519	struct tb_cm *tcm = tb_priv(tb);
1520	int i;
1521
1522	tb_dbg(tb, "bandwidth consumption changed, re-calculating estimated bandwidth\n");
1523
1524	for (i = 0; i < ARRAY_SIZE(tcm->groups); i++) {
1525	struct tb_bandwidth_group *group = &tcm->groups[i];
1526
1527	if (!list_empty(&group->ports))
1528	tb_recalc_estimated_bandwidth_for_group(group);
1529	}
1530
1531	tb_dbg(tb, "bandwidth re-calculation done\n");
1532	}
1533
1534	static bool __release_group_bandwidth(struct tb_bandwidth_group *group)
1535	{
1536	if (group->reserved) {
1537	tb_dbg(group->tb, "group %d released total %d Mb/s\n", group->index,
1538	group->reserved);
1539	group->reserved = 0;
1540	return true;
1541	}
1542	return false;
1543	}
1544
1545	static void __configure_group_sym(struct tb_bandwidth_group *group)
1546	{
1547	struct tb_tunnel *tunnel;
1548	struct tb_port *in;
1549
1550	if (list_empty(&group->ports))
1551	return;
1552
1553	/*
1554	* All the tunnels in the group go through the same USB4 links
1555	* so we find the first one here and pass the IN and OUT
1556	* adapters to tb_configure_sym() which now transitions the
1557	* links back to symmetric if bandwidth requirement < asym_threshold.
1558	*
1559	* We do this here to avoid unnecessary transitions (for example
1560	* if the graphics released bandwidth for other tunnel in the
1561	* same group).
1562	*/
1563	in = list_first_entry(&group->ports, struct tb_port, group_list);
1564	tunnel = tb_find_tunnel(group->tb, TB_TUNNEL_DP, in, NULL);
1565	if (tunnel)
1566	tb_configure_sym(group->tb, in, tunnel->dst_port, true);
1567	}
1568
1569	static void tb_bandwidth_group_release_work(struct work_struct *work)
1570	{
1571	struct tb_bandwidth_group *group =
1572	container_of(work, typeof(*group), release_work.work);
1573	struct tb *tb = group->tb;
1574
1575	mutex_lock(&tb->lock);
1576	if (__release_group_bandwidth(group))
1577	tb_recalc_estimated_bandwidth(tb);
1578	__configure_group_sym(group);
1579	mutex_unlock(&tb->lock);
1580	}
1581
1582	static void tb_init_bandwidth_groups(struct tb_cm *tcm)
1583	{
1584	int i;
1585
1586	for (i = 0; i < ARRAY_SIZE(tcm->groups); i++) {
1587	struct tb_bandwidth_group *group = &tcm->groups[i];
1588
1589	group->tb = tcm_to_tb(tcm);
1590	group->index = i + 1;
1591	INIT_LIST_HEAD(&group->ports);
1592	INIT_DELAYED_WORK(&group->release_work,
1593	tb_bandwidth_group_release_work);
1594	}
1595	}
1596
1597	static void tb_bandwidth_group_attach_port(struct tb_bandwidth_group *group,
1598	struct tb_port *in)
1599	{
1600	if (!group \|\| WARN_ON(in->group))
1601	return;
1602
1603	in->group = group;
1604	list_add_tail(&in->group_list, &group->ports);
1605
1606	tb_port_dbg(in, "attached to bandwidth group %d\n", group->index);
1607	}
1608
1609	static struct tb_bandwidth_group tb_find_free_bandwidth_group(struct tb_cm tcm)
1610	{
1611	int i;
1612
1613	for (i = 0; i < ARRAY_SIZE(tcm->groups); i++) {
1614	struct tb_bandwidth_group *group = &tcm->groups[i];
1615
1616	if (list_empty(&group->ports))
1617	return group;
1618	}
1619
1620	return NULL;
1621	}
1622
1623	static struct tb_bandwidth_group *
1624	tb_attach_bandwidth_group(struct tb_cm tcm, struct tb_port in,
1625	struct tb_port *out)
1626	{
1627	struct tb_bandwidth_group *group;
1628	struct tb_tunnel *tunnel;
1629
1630	/*
1631	* Find all DP tunnels that go through all the same USB4 links
1632	* as this one. Because we always setup tunnels the same way we
1633	* can just check for the routers at both ends of the tunnels
1634	* and if they are the same we have a match.
1635	*/
1636	list_for_each_entry(tunnel, &tcm->tunnel_list, list) {
1637	if (!tb_tunnel_is_dp(tunnel))
1638	continue;
1639
1640	if (tunnel->src_port->sw == in->sw &&
1641	tunnel->dst_port->sw == out->sw) {
1642	group = tunnel->src_port->group;
1643	if (group) {
1644	tb_bandwidth_group_attach_port(group, in);
1645	return group;
1646	}
1647	}
1648	}
1649
1650	/* Pick up next available group then */
1651	group = tb_find_free_bandwidth_group(tcm);
1652	if (group)
1653	tb_bandwidth_group_attach_port(group, in);
1654	else
1655	tb_port_warn(in, "no available bandwidth groups\n");
1656
1657	return group;
1658	}
1659
1660	static void tb_discover_bandwidth_group(struct tb_cm tcm, struct tb_port in,
1661	struct tb_port *out)
1662	{
1663	if (usb4_dp_port_bandwidth_mode_enabled(in)) {
1664	int index, i;
1665
1666	index = usb4_dp_port_group_id(in);
1667	for (i = 0; i < ARRAY_SIZE(tcm->groups); i++) {
1668	if (tcm->groups[i].index == index) {
1669	tb_bandwidth_group_attach_port(&tcm->groups[i], in);
1670	return;
1671	}
1672	}
1673	}
1674
1675	tb_attach_bandwidth_group(tcm, in, out);
1676	}
1677
1678	static void tb_detach_bandwidth_group(struct tb_port *in)
1679	{
1680	struct tb_bandwidth_group *group = in->group;
1681
1682	if (group) {
1683	in->group = NULL;
1684	list_del_init(&in->group_list);
1685
1686	tb_port_dbg(in, "detached from bandwidth group %d\n", group->index);
1687
1688	/* No more tunnels so release the reserved bandwidth if any */
1689	if (list_empty(&group->ports)) {
1690	cancel_delayed_work(&group->release_work);
1691	__release_group_bandwidth(group);
1692	}
1693	}
1694	}
1695
1696	static void tb_discover_tunnels(struct tb *tb)
1697	{
1698	struct tb_cm *tcm = tb_priv(tb);
1699	struct tb_tunnel *tunnel;
1700
1701	tb_switch_discover_tunnels(tb->root_switch, &tcm->tunnel_list, true);
1702
1703	list_for_each_entry(tunnel, &tcm->tunnel_list, list) {
1704	if (tb_tunnel_is_pci(tunnel)) {
1705	struct tb_switch *parent = tunnel->dst_port->sw;
1706
1707	while (parent != tunnel->src_port->sw) {
1708	parent->boot = true;
1709	parent = tb_switch_parent(parent);
1710	}
1711	} else if (tb_tunnel_is_dp(tunnel)) {
1712	struct tb_port *in = tunnel->src_port;
1713	struct tb_port *out = tunnel->dst_port;
1714
1715	/* Keep the domain from powering down */
1716	pm_runtime_get_sync(&in->sw->dev);
1717	pm_runtime_get_sync(&out->sw->dev);
1718
1719	tb_discover_bandwidth_group(tcm, in, out);
1720	}
1721	}
1722	}
1723
1724	static void tb_deactivate_and_free_tunnel(struct tb_tunnel *tunnel)
1725	{
1726	struct tb_port src_port, dst_port;
1727	struct tb *tb;
1728
1729	if (!tunnel)
1730	return;
1731
1732	tb_tunnel_deactivate(tunnel);
1733	list_del(&tunnel->list);
1734
1735	tb = tunnel->tb;
1736	src_port = tunnel->src_port;
1737	dst_port = tunnel->dst_port;
1738
1739	switch (tunnel->type) {
1740	case TB_TUNNEL_DP:
1741	tb_detach_bandwidth_group(src_port);
1742	/*
1743	* In case of DP tunnel make sure the DP IN resource is
1744	* deallocated properly.
1745	*/
1746	tb_switch_dealloc_dp_resource(src_port->sw, src_port);
1747	/*
1748	* If bandwidth on a link is < asym_threshold
1749	* transition the link to symmetric.
1750	*/
1751	tb_configure_sym(tb, src_port, dst_port, true);
1752	/* Now we can allow the domain to runtime suspend again */
1753	pm_runtime_mark_last_busy(&dst_port->sw->dev);
1754	pm_runtime_put_autosuspend(&dst_port->sw->dev);
1755	pm_runtime_mark_last_busy(&src_port->sw->dev);
1756	pm_runtime_put_autosuspend(&src_port->sw->dev);
1757	fallthrough;
1758
1759	case TB_TUNNEL_USB3:
1760	tb_reclaim_usb3_bandwidth(tb, src_port, dst_port);
1761	break;
1762
1763	default:
1764	/*
1765	* PCIe and DMA tunnels do not consume guaranteed
1766	* bandwidth.
1767	*/
1768	break;
1769	}
1770
1771	tb_tunnel_put(tunnel);
1772	}
1773
1774	/*
1775	* tb_free_invalid_tunnels() - destroy tunnels of devices that have gone away
1776	*/
1777	static void tb_free_invalid_tunnels(struct tb *tb)
1778	{
1779	struct tb_cm *tcm = tb_priv(tb);
1780	struct tb_tunnel *tunnel;
1781	struct tb_tunnel *n;
1782
1783	list_for_each_entry_safe(tunnel, n, &tcm->tunnel_list, list) {
1784	if (tb_tunnel_is_invalid(tunnel))
1785	tb_deactivate_and_free_tunnel(tunnel);
1786	}
1787	}
1788
1789	/*
1790	* tb_free_unplugged_children() - traverse hierarchy and free unplugged switches
1791	*/
1792	static void tb_free_unplugged_children(struct tb_switch *sw)
1793	{
1794	struct tb_port *port;
1795
1796	tb_switch_for_each_port(sw, port) {
1797	if (!tb_port_has_remote(port))
1798	continue;
1799
1800	if (port->remote->sw->is_unplugged) {
1801	tb_retimer_remove_all(port);
1802	tb_remove_dp_resources(port->remote->sw);
1803	tb_switch_unconfigure_link(port->remote->sw);
1804	tb_switch_set_link_width(port->remote->sw,
1805	TB_LINK_WIDTH_SINGLE);
1806	tb_switch_remove(port->remote->sw);
1807	port->remote = NULL;
1808	if (port->dual_link_port)
1809	port->dual_link_port->remote = NULL;
1810	} else {
1811	tb_free_unplugged_children(port->remote->sw);
1812	}
1813	}
1814	}
1815
1816	static struct tb_port tb_find_pcie_down(struct tb_switch sw,
1817	const struct tb_port *port)
1818	{
1819	struct tb_port *down = NULL;
1820
1821	/*
1822	* To keep plugging devices consistently in the same PCIe
1823	* hierarchy, do mapping here for switch downstream PCIe ports.
1824	*/
1825	if (tb_switch_is_usb4(sw)) {
1826	down = usb4_switch_map_pcie_down(sw, port);
1827	} else if (!tb_route(sw)) {
1828	int phy_port = tb_phy_port_from_link(port->port);
1829	int index;
1830
1831	/*
1832	* Hard-coded Thunderbolt port to PCIe down port mapping
1833	* per controller.
1834	*/
1835	if (tb_switch_is_cactus_ridge(sw) \|\|
1836	tb_switch_is_alpine_ridge(sw))
1837	index = !phy_port ? 6 : 7;
1838	else if (tb_switch_is_falcon_ridge(sw))
1839	index = !phy_port ? 6 : 8;
1840	else if (tb_switch_is_titan_ridge(sw))
1841	index = !phy_port ? 8 : 9;
1842	else
1843	goto out;
1844
1845	/* Validate the hard-coding */
1846	if (WARN_ON(index > sw->config.max_port_number))
1847	goto out;
1848
1849	down = &sw->ports[index];
1850	}
1851
1852	if (down) {
1853	if (WARN_ON(!tb_port_is_pcie_down(down)))
1854	goto out;
1855	if (tb_pci_port_is_enabled(down))
1856	goto out;
1857
1858	return down;
1859	}
1860
1861	out:
1862	return tb_find_unused_port(sw, TB_TYPE_PCIE_DOWN);
1863	}
1864
1865	static struct tb_port tb_find_dp_out(struct tb tb, struct tb_port *in)
1866	{
1867	struct tb_port host_port, port;
1868	struct tb_cm *tcm = tb_priv(tb);
1869
1870	host_port = tb_route(in->sw) ?
1871	tb_port_at(tb_route(in->sw), tb->root_switch) : NULL;
1872
1873	list_for_each_entry(port, &tcm->dp_resources, list) {
1874	if (!tb_port_is_dpout(port))
1875	continue;
1876
1877	if (tb_port_is_enabled(port)) {
1878	tb_port_dbg(port, "DP OUT in use\n");
1879	continue;
1880	}
1881
1882	/* Needs to be on different routers */
1883	if (in->sw == port->sw) {
1884	tb_port_dbg(port, "skipping DP OUT on same router\n");
1885	continue;
1886	}
1887
1888	tb_port_dbg(port, "DP OUT available\n");
1889
1890	/*
1891	* Keep the DP tunnel under the topology starting from
1892	* the same host router downstream port.
1893	*/
1894	if (host_port && tb_route(port->sw)) {
1895	struct tb_port *p;
1896
1897	p = tb_port_at(tb_route(port->sw), tb->root_switch);
1898	if (p != host_port)
1899	continue;
1900	}
1901
1902	return port;
1903	}
1904
1905	return NULL;
1906	}
1907
1908	static void tb_dp_tunnel_active(struct tb_tunnel tunnel, void data)
1909	{
1910	struct tb_port *in = tunnel->src_port;
1911	struct tb_port *out = tunnel->dst_port;
1912	struct tb *tb = data;
1913
1914	mutex_lock(&tb->lock);
1915
1916	/*
1917	* If the DPRX read was canceled the tunnel is already being torn
1918	* down by whoever canceled it. Do not touch the adapters here
1919	* because the routers may be gone by now.
1920	*/
1921	if (tunnel->dprx_canceled) {
1922	tb_tunnel_dbg(tunnel, "DPRX read canceled, not activating\n");
1923	mutex_unlock(&tb->lock);
1924	return;
1925	}
1926
1927	if (tb_tunnel_is_active(tunnel)) {
1928	int consumed_up, consumed_down, ret;
1929
1930	tb_tunnel_dbg(tunnel, "DPRX capabilities read completed\n");
1931
1932	/* If fail reading tunnel's consumed bandwidth, tear it down */
1933	ret = tb_tunnel_consumed_bandwidth(tunnel, &consumed_up,
1934	&consumed_down);
1935	if (ret) {
1936	tb_tunnel_warn(tunnel,
1937	"failed to read consumed bandwidth, tearing down\n");
1938	tb_deactivate_and_free_tunnel(tunnel);
1939	} else {
1940	tb_reclaim_usb3_bandwidth(tb, in, out);
1941	/*
1942	* Transition the links to asymmetric if the
1943	* consumption exceeds the threshold.
1944	*/
1945	tb_configure_asym(tb, in, out, consumed_up,
1946	consumed_down);
1947	/*
1948	* Update the domain with the new bandwidth
1949	* estimation.
1950	*/
1951	tb_recalc_estimated_bandwidth(tb);
1952	/*
1953	* In case DP tunnel exists, change host
1954	* router's 1st children TMU mode to HiFi for
1955	* CL0s to work.
1956	*/
1957	tb_increase_tmu_accuracy(tunnel);
1958	}
1959	} else {
1960	struct tb_port *in = tunnel->src_port;
1961
1962	/*
1963	* This tunnel failed to establish. This means DPRX
1964	* negotiation most likely did not complete which
1965	* happens either because there is no graphics driver
1966	* loaded or not all DP cables where connected to the
1967	* discrete router.
1968	*
1969	* In both cases we remove the DP IN adapter from the
1970	* available resources as it is not usable. This will
1971	* also tear down the tunnel and try to re-use the
1972	* released DP OUT.
1973	*
1974	* It will be added back only if there is hotplug for
1975	* the DP IN again.
1976	*/
1977	tb_tunnel_warn(tunnel, "not active, tearing down\n");
1978	tb_dp_resource_unavailable(tb, in, "DPRX negotiation failed");
1979	}
1980	mutex_unlock(&tb->lock);
1981	}
1982
1983	static void tb_tunnel_one_dp(struct tb tb, struct tb_port in,
1984	struct tb_port *out)
1985	{
1986	int available_up, available_down, ret, link_nr;
1987	struct tb_cm *tcm = tb_priv(tb);
1988	struct tb_tunnel *tunnel;
1989
1990	/*
1991	* This is only applicable to links that are not bonded (so
1992	* when Thunderbolt 1 hardware is involved somewhere in the
1993	* topology). For these try to share the DP bandwidth between
1994	* the two lanes.
1995	*/
1996	link_nr = 1;
1997	list_for_each_entry(tunnel, &tcm->tunnel_list, list) {
1998	if (tb_tunnel_is_dp(tunnel)) {
1999	link_nr = 0;
2000	break;
2001	}
2002	}
2003
2004	/*
2005	* DP stream needs the domain to be active so runtime resume
2006	* both ends of the tunnel.
2007	*
2008	* This should bring the routers in the middle active as well
2009	* and keeps the domain from runtime suspending while the DP
2010	* tunnel is active.
2011	*/
2012	pm_runtime_get_sync(&in->sw->dev);
2013	pm_runtime_get_sync(&out->sw->dev);
2014
2015	if (tb_switch_alloc_dp_resource(in->sw, in)) {
2016	tb_port_dbg(in, "no resource available for DP IN, not tunneling\n");
2017	goto err_rpm_put;
2018	}
2019
2020	if (!tb_attach_bandwidth_group(tcm, in, out))
2021	goto err_dealloc_dp;
2022
2023	/* Make all unused USB3 bandwidth available for the new DP tunnel */
2024	ret = tb_release_unused_usb3_bandwidth(tb, in, out);
2025	if (ret) {
2026	tb_warn(tb, "failed to release unused bandwidth\n");
2027	goto err_detach_group;
2028	}
2029
2030	ret = tb_available_bandwidth(tb, in, out, &available_up, &available_down,
2031	true);
2032	if (ret) {
2033	tb_tunnel_event(tb, TB_TUNNEL_NO_BANDWIDTH, TB_TUNNEL_DP, in, out);
2034	goto err_reclaim_usb;
2035	}
2036
2037	tb_dbg(tb, "available bandwidth for new DP tunnel %u/%u Mb/s\n",
2038	available_up, available_down);
2039
2040	tunnel = tb_tunnel_alloc_dp(tb, in, out, link_nr, available_up,
2041	available_down, tb_dp_tunnel_active, tb);
2042	if (!tunnel) {
2043	tb_port_dbg(out, "could not allocate DP tunnel\n");
2044	goto err_reclaim_usb;
2045	}
2046
2047	list_add_tail(&tunnel->list, &tcm->tunnel_list);
2048
2049	ret = tb_tunnel_activate(tunnel);
2050	if (ret && ret != -EINPROGRESS) {
2051	tb_port_info(out, "DP tunnel activation failed, aborting\n");
2052	list_del(&tunnel->list);
2053	goto err_free;
2054	}
2055
2056	return;
2057
2058	err_free:
2059	tb_tunnel_put(tunnel);
2060	err_reclaim_usb:
2061	tb_reclaim_usb3_bandwidth(tb, in, out);
2062	err_detach_group:
2063	tb_detach_bandwidth_group(in);
2064	err_dealloc_dp:
2065	tb_switch_dealloc_dp_resource(in->sw, in);
2066	err_rpm_put:
2067	pm_runtime_mark_last_busy(&out->sw->dev);
2068	pm_runtime_put_autosuspend(&out->sw->dev);
2069	pm_runtime_mark_last_busy(&in->sw->dev);
2070	pm_runtime_put_autosuspend(&in->sw->dev);
2071	}
2072
2073	static void tb_tunnel_dp(struct tb *tb)
2074	{
2075	struct tb_cm *tcm = tb_priv(tb);
2076	struct tb_port port, in, *out;
2077
2078	if (!tb_acpi_may_tunnel_dp()) {
2079	tb_dbg(tb, "DP tunneling disabled, not creating tunnel\n");
2080	return;
2081	}
2082
2083	/*
2084	* Find pair of inactive DP IN and DP OUT adapters and then
2085	* establish a DP tunnel between them.
2086	*/
2087	tb_dbg(tb, "looking for DP IN <-> DP OUT pairs:\n");
2088
2089	in = NULL;
2090	out = NULL;
2091	list_for_each_entry(port, &tcm->dp_resources, list) {
2092	if (!tb_port_is_dpin(port))
2093	continue;
2094
2095	if (tb_port_is_enabled(port)) {
2096	tb_port_dbg(port, "DP IN in use\n");
2097	continue;
2098	}
2099
2100	in = port;
2101	tb_port_dbg(in, "DP IN available\n");
2102
2103	out = tb_find_dp_out(tb, port);
2104	if (out)
2105	tb_tunnel_one_dp(tb, in, out);
2106	else
2107	tb_port_dbg(in, "no suitable DP OUT adapter available, not tunneling\n");
2108	}
2109
2110	if (!in)
2111	tb_dbg(tb, "no suitable DP IN adapter available, not tunneling\n");
2112	}
2113
2114	static void tb_enter_redrive(struct tb_port *port)
2115	{
2116	struct tb_switch *sw = port->sw;
2117
2118	if (!(sw->quirks & QUIRK_KEEP_POWER_IN_DP_REDRIVE))
2119	return;
2120
2121	/*
2122	* If we get hot-unplug for the DP IN port of the host router
2123	* and the DP resource is not available anymore it means there
2124	* is a monitor connected directly to the Type-C port and we are
2125	* in "redrive" mode. For this to work we cannot enter RTD3 so
2126	* we bump up the runtime PM reference count here.
2127	*/
2128	if (!tb_port_is_dpin(port))
2129	return;
2130	if (tb_route(sw))
2131	return;
2132	if (!tb_switch_query_dp_resource(sw, port)) {
2133	port->redrive = true;
2134	pm_runtime_get(&sw->dev);
2135	tb_port_dbg(port, "enter redrive mode, keeping powered\n");
2136	}
2137	}
2138
2139	static void tb_exit_redrive(struct tb_port *port)
2140	{
2141	struct tb_switch *sw = port->sw;
2142
2143	if (!(sw->quirks & QUIRK_KEEP_POWER_IN_DP_REDRIVE))
2144	return;
2145
2146	if (!tb_port_is_dpin(port))
2147	return;
2148	if (tb_route(sw))
2149	return;
2150	if (port->redrive && tb_switch_query_dp_resource(sw, port)) {
2151	port->redrive = false;
2152	pm_runtime_put(&sw->dev);
2153	tb_port_dbg(port, "exit redrive mode\n");
2154	}
2155	}
2156
2157	static void tb_switch_enter_redrive(struct tb_switch *sw)
2158	{
2159	struct tb_port *port;
2160
2161	tb_switch_for_each_port(sw, port)
2162	tb_enter_redrive(port);
2163	}
2164
2165	/*
2166	* Called during system and runtime suspend to forcefully exit redrive
2167	* mode without querying whether the resource is available.
2168	*/
2169	static void tb_switch_exit_redrive(struct tb_switch *sw)
2170	{
2171	struct tb_port *port;
2172
2173	if (!(sw->quirks & QUIRK_KEEP_POWER_IN_DP_REDRIVE))
2174	return;
2175
2176	tb_switch_for_each_port(sw, port) {
2177	if (!tb_port_is_dpin(port))
2178	continue;
2179
2180	if (port->redrive) {
2181	port->redrive = false;
2182	pm_runtime_put(&sw->dev);
2183	tb_port_dbg(port, "exit redrive mode\n");
2184	}
2185	}
2186	}
2187
2188	static void tb_dp_resource_unavailable(struct tb tb, struct tb_port port,
2189	const char *reason)
2190	{
2191	struct tb_port in, out;
2192	struct tb_tunnel *tunnel;
2193
2194	if (tb_port_is_dpin(port)) {
2195	tb_port_dbg(port, "DP IN resource unavailable: %s\n", reason);
2196	in = port;
2197	out = NULL;
2198	} else {
2199	tb_port_dbg(port, "DP OUT resource unavailable: %s\n", reason);
2200	in = NULL;
2201	out = port;
2202	}
2203
2204	tunnel = tb_find_tunnel(tb, TB_TUNNEL_DP, in, out);
2205	if (tunnel)
2206	tb_deactivate_and_free_tunnel(tunnel);
2207	else
2208	tb_enter_redrive(port);
2209	list_del_init(&port->list);
2210
2211	/*
2212	* See if there is another DP OUT port that can be used for
2213	* to create another tunnel.
2214	*/
2215	tb_recalc_estimated_bandwidth(tb);
2216	tb_tunnel_dp(tb);
2217	}
2218
2219	static void tb_dp_resource_available(struct tb tb, struct tb_port port)
2220	{
2221	struct tb_cm *tcm = tb_priv(tb);
2222	struct tb_port *p;
2223
2224	if (tb_port_is_enabled(port))
2225	return;
2226
2227	list_for_each_entry(p, &tcm->dp_resources, list) {
2228	if (p == port)
2229	return;
2230	}
2231
2232	tb_port_dbg(port, "DP %s resource available after hotplug\n",
2233	tb_port_is_dpin(port) ? "IN" : "OUT");
2234	list_add_tail(&port->list, &tcm->dp_resources);
2235	tb_exit_redrive(port);
2236
2237	/* Look for suitable DP IN <-> DP OUT pairs now */
2238	tb_tunnel_dp(tb);
2239	}
2240
2241	static void tb_disconnect_and_release_dp(struct tb *tb)
2242	{
2243	struct tb_cm *tcm = tb_priv(tb);
2244	struct tb_tunnel tunnel, n;
2245
2246	/*
2247	* Tear down all DP tunnels and release their resources. They
2248	* will be re-established after resume based on plug events.
2249	*/
2250	list_for_each_entry_safe_reverse(tunnel, n, &tcm->tunnel_list, list) {
2251	if (tb_tunnel_is_dp(tunnel))
2252	tb_deactivate_and_free_tunnel(tunnel);
2253	}
2254
2255	while (!list_empty(&tcm->dp_resources)) {
2256	struct tb_port *port;
2257
2258	port = list_first_entry(&tcm->dp_resources,
2259	struct tb_port, list);
2260	list_del_init(&port->list);
2261	}
2262	}
2263
2264	static int tb_disconnect_pci(struct tb tb, struct tb_switch sw)
2265	{
2266	struct tb_tunnel *tunnel;
2267	struct tb_port *up;
2268
2269	up = tb_switch_find_port(sw, TB_TYPE_PCIE_UP);
2270	if (WARN_ON(!up))
2271	return -ENODEV;
2272
2273	tunnel = tb_find_tunnel(tb, TB_TUNNEL_PCI, NULL, up);
2274	if (WARN_ON(!tunnel))
2275	return -ENODEV;
2276
2277	tb_switch_xhci_disconnect(sw);
2278
2279	tb_tunnel_deactivate(tunnel);
2280	list_del(&tunnel->list);
2281	tb_tunnel_put(tunnel);
2282	return 0;
2283	}
2284
2285	static int tb_tunnel_pci(struct tb tb, struct tb_switch sw)
2286	{
2287	struct tb_port up, down, *port;
2288	struct tb_cm *tcm = tb_priv(tb);
2289	struct tb_tunnel *tunnel;
2290
2291	up = tb_switch_find_port(sw, TB_TYPE_PCIE_UP);
2292	if (!up)
2293	return 0;
2294
2295	/*
2296	* Look up available down port. Since we are chaining it should
2297	* be found right above this switch.
2298	*/
2299	port = tb_switch_downstream_port(sw);
2300	down = tb_find_pcie_down(tb_switch_parent(sw), port);
2301	if (!down)
2302	return 0;
2303
2304	tunnel = tb_tunnel_alloc_pci(tb, up, down);
2305	if (!tunnel)
2306	return -ENOMEM;
2307
2308	if (tb_tunnel_activate(tunnel)) {
2309	tb_port_info(up,
2310	"PCIe tunnel activation failed, aborting\n");
2311	tb_tunnel_put(tunnel);
2312	return -EIO;
2313	}
2314
2315	/*
2316	* PCIe L1 is needed to enable CL0s for Titan Ridge so enable it
2317	* here.
2318	*/
2319	if (tb_switch_pcie_l1_enable(sw))
2320	tb_sw_warn(sw, "failed to enable PCIe L1 for Titan Ridge\n");
2321
2322	if (tb_switch_xhci_connect(sw))
2323	tb_sw_warn(sw, "failed to connect xHCI\n");
2324
2325	list_add_tail(&tunnel->list, &tcm->tunnel_list);
2326	return 0;
2327	}
2328
2329	static int tb_approve_xdomain_paths(struct tb tb, struct tb_xdomain xd,
2330	int transmit_path, int transmit_ring,
2331	int receive_path, int receive_ring)
2332	{
2333	struct tb_cm *tcm = tb_priv(tb);
2334	struct tb_port nhi_port, dst_port;
2335	struct tb_tunnel *tunnel;
2336	struct tb_switch *sw;
2337	int ret;
2338
2339	sw = tb_to_switch(xd->dev.parent);
2340	dst_port = tb_port_at(xd->route, sw);
2341	nhi_port = tb_switch_find_port(tb->root_switch, TB_TYPE_NHI);
2342
2343	mutex_lock(&tb->lock);
2344
2345	/*
2346	* When tunneling DMA paths the link should not enter CL states
2347	* so disable them now.
2348	*/
2349	tb_disable_clx(sw);
2350
2351	tunnel = tb_tunnel_alloc_dma(tb, nhi_port, dst_port, transmit_path,
2352	transmit_ring, receive_path, receive_ring);
2353	if (!tunnel) {
2354	ret = -ENOMEM;
2355	goto err_clx;
2356	}
2357
2358	if (tb_tunnel_activate(tunnel)) {
2359	tb_port_info(nhi_port,
2360	"DMA tunnel activation failed, aborting\n");
2361	ret = -EIO;
2362	goto err_free;
2363	}
2364
2365	list_add_tail(&tunnel->list, &tcm->tunnel_list);
2366	mutex_unlock(&tb->lock);
2367	return 0;
2368
2369	err_free:
2370	tb_tunnel_put(tunnel);
2371	err_clx:
2372	tb_enable_clx(sw);
2373	mutex_unlock(&tb->lock);
2374
2375	return ret;
2376	}
2377
2378	static void __tb_disconnect_xdomain_paths(struct tb tb, struct tb_xdomain xd,
2379	int transmit_path, int transmit_ring,
2380	int receive_path, int receive_ring)
2381	{
2382	struct tb_cm *tcm = tb_priv(tb);
2383	struct tb_port nhi_port, dst_port;
2384	struct tb_tunnel tunnel, n;
2385	struct tb_switch *sw;
2386
2387	sw = tb_to_switch(xd->dev.parent);
2388	dst_port = tb_port_at(xd->route, sw);
2389	nhi_port = tb_switch_find_port(tb->root_switch, TB_TYPE_NHI);
2390
2391	list_for_each_entry_safe(tunnel, n, &tcm->tunnel_list, list) {
2392	if (!tb_tunnel_is_dma(tunnel))
2393	continue;
2394	if (tunnel->src_port != nhi_port \|\| tunnel->dst_port != dst_port)
2395	continue;
2396
2397	if (tb_tunnel_match_dma(tunnel, transmit_path, transmit_ring,
2398	receive_path, receive_ring))
2399	tb_deactivate_and_free_tunnel(tunnel);
2400	}
2401
2402	/*
2403	* Try to re-enable CL states now, it is OK if this fails
2404	* because we may still have another DMA tunnel active through
2405	* the same host router USB4 downstream port.
2406	*/
2407	tb_enable_clx(sw);
2408	}
2409
2410	static int tb_disconnect_xdomain_paths(struct tb tb, struct tb_xdomain xd,
2411	int transmit_path, int transmit_ring,
2412	int receive_path, int receive_ring)
2413	{
2414	if (!xd->is_unplugged) {
2415	mutex_lock(&tb->lock);
2416	__tb_disconnect_xdomain_paths(tb, xd, transmit_path,
2417	transmit_ring, receive_path,
2418	receive_ring);
2419	mutex_unlock(&tb->lock);
2420	}
2421	return 0;
2422	}
2423
2424	/* hotplug handling */
2425
2426	/*
2427	* tb_handle_hotplug() - handle hotplug event
2428	*
2429	* Executes on tb->wq.
2430	*/
2431	static void tb_handle_hotplug(struct work_struct *work)
2432	{
2433	struct tb_hotplug_event ev = container_of(work, typeof(ev), work.work);
2434	struct tb *tb = ev->tb;
2435	struct tb_cm *tcm = tb_priv(tb);
2436	struct tb_switch *sw;
2437	struct tb_port *port;
2438
2439	/* Bring the domain back from sleep if it was suspended */
2440	pm_runtime_get_sync(&tb->dev);
2441
2442	mutex_lock(&tb->lock);
2443	if (!tcm->hotplug_active)
2444	goto out; /* during init, suspend or shutdown */
2445
2446	sw = tb_switch_find_by_route(tb, ev->route);
2447	if (!sw) {
2448	tb_warn(tb,
2449	"hotplug event from non existent switch %llx:%x (unplug: %d)\n",
2450	ev->route, ev->port, ev->unplug);
2451	goto out;
2452	}
2453	if (ev->port > sw->config.max_port_number) {
2454	tb_warn(tb,
2455	"hotplug event from non existent port %llx:%x (unplug: %d)\n",
2456	ev->route, ev->port, ev->unplug);
2457	goto put_sw;
2458	}
2459	port = &sw->ports[ev->port];
2460	if (tb_is_upstream_port(port)) {
2461	tb_dbg(tb, "hotplug event for upstream port %llx:%x (unplug: %d)\n",
2462	ev->route, ev->port, ev->unplug);
2463	goto put_sw;
2464	}
2465
2466	pm_runtime_get_sync(&sw->dev);
2467
2468	if (ev->unplug) {
2469	tb_retimer_remove_all(port);
2470
2471	if (tb_port_has_remote(port)) {
2472	tb_port_dbg(port, "switch unplugged\n");
2473	tb_sw_set_unplugged(port->remote->sw);
2474	tb_free_invalid_tunnels(tb);
2475	tb_remove_dp_resources(port->remote->sw);
2476	tb_switch_tmu_disable(port->remote->sw);
2477	tb_switch_unconfigure_link(port->remote->sw);
2478	tb_switch_set_link_width(port->remote->sw,
2479	TB_LINK_WIDTH_SINGLE);
2480	tb_switch_remove(port->remote->sw);
2481	port->remote = NULL;
2482	if (port->dual_link_port)
2483	port->dual_link_port->remote = NULL;
2484	/* Maybe we can create another DP tunnel */
2485	tb_recalc_estimated_bandwidth(tb);
2486	tb_tunnel_dp(tb);
2487	} else if (port->xdomain) {
2488	struct tb_xdomain *xd = tb_xdomain_get(port->xdomain);
2489
2490	tb_port_dbg(port, "xdomain unplugged\n");
2491	/*
2492	* Service drivers are unbound during
2493	* tb_xdomain_remove() so setting XDomain as
2494	* unplugged here prevents deadlock if they call
2495	* tb_xdomain_disable_paths(). We will tear down
2496	* all the tunnels below.
2497	*/
2498	xd->is_unplugged = true;
2499	tb_xdomain_remove(xd);
2500	port->xdomain = NULL;
2501	__tb_disconnect_xdomain_paths(tb, xd, -1, -1, -1, -1);
2502	tb_xdomain_put(xd);
2503	tb_port_unconfigure_xdomain(port);
2504	} else if (tb_port_is_dpout(port) \|\| tb_port_is_dpin(port)) {
2505	tb_dp_resource_unavailable(tb, port, "adapter unplug");
2506	} else if (!port->port) {
2507	tb_sw_dbg(sw, "xHCI disconnect request\n");
2508	tb_switch_xhci_disconnect(sw);
2509	} else {
2510	tb_port_dbg(port,
2511	"got unplug event for disconnected port, ignoring\n");
2512	}
2513	} else if (port->remote) {
2514	tb_port_dbg(port, "got plug event for connected port, ignoring\n");
2515	} else if (!port->port && sw->authorized) {
2516	tb_sw_dbg(sw, "xHCI connect request\n");
2517	tb_switch_xhci_connect(sw);
2518	} else {
2519	if (tb_port_is_null(port)) {
2520	tb_port_dbg(port, "hotplug: scanning\n");
2521	tb_scan_port(port);
2522	if (!port->remote)
2523	tb_port_dbg(port, "hotplug: no switch found\n");
2524	} else if (tb_port_is_dpout(port) \|\| tb_port_is_dpin(port)) {
2525	tb_dp_resource_available(tb, port);
2526	}
2527	}
2528
2529	pm_runtime_mark_last_busy(&sw->dev);
2530	pm_runtime_put_autosuspend(&sw->dev);
2531
2532	put_sw:
2533	tb_switch_put(sw);
2534	out:
2535	mutex_unlock(&tb->lock);
2536
2537	tb_domain_unregister_unplugged_xdomains(tb);
2538
2539	pm_runtime_mark_last_busy(&tb->dev);
2540	pm_runtime_put_autosuspend(&tb->dev);
2541
2542	/* Undo the refcount increased in tb_queue_hotplug() */
2543	tb_domain_put(tb);
2544
2545	kfree(ev);
2546	}
2547
2548	static int tb_alloc_dp_bandwidth(struct tb_tunnel tunnel, int requested_up,
2549	int *requested_down)
2550	{
2551	int allocated_up, allocated_down, available_up, available_down, ret;
2552	int requested_up_corrected, requested_down_corrected, granularity;
2553	int max_up, max_down, max_up_rounded, max_down_rounded;
2554	struct tb_bandwidth_group *group;
2555	struct tb *tb = tunnel->tb;
2556	struct tb_port in, out;
2557	bool downstream;
2558
2559	ret = tb_tunnel_allocated_bandwidth(tunnel, &allocated_up, &allocated_down);
2560	if (ret)
2561	return ret;
2562
2563	in = tunnel->src_port;
2564	out = tunnel->dst_port;
2565
2566	tb_tunnel_dbg(tunnel, "bandwidth allocated currently %d/%d Mb/s\n",
2567	allocated_up, allocated_down);
2568
2569	/*
2570	* If we get rounded up request from graphics side, say HBR2 x 4
2571	* that is 17500 instead of 17280 (this is because of the
2572	* granularity), we allow it too. Here the graphics has already
2573	* negotiated with the DPRX the maximum possible rates (which is
2574	* 17280 in this case).
2575	*
2576	* Since the link cannot go higher than 17280 we use that in our
2577	* calculations but the DP IN adapter Allocated BW write must be
2578	* the same value (17500) otherwise the adapter will mark it as
2579	* failed for graphics.
2580	*/
2581	ret = tb_tunnel_maximum_bandwidth(tunnel, &max_up, &max_down);
2582	if (ret)
2583	goto fail;
2584
2585	ret = usb4_dp_port_granularity(in);
2586	if (ret < 0)
2587	goto fail;
2588	granularity = ret;
2589
2590	max_up_rounded = roundup(max_up, granularity);
2591	max_down_rounded = roundup(max_down, granularity);
2592
2593	/*
2594	* This will "fix" the request down to the maximum supported
2595	* rate * lanes if it is at the maximum rounded up level.
2596	*/
2597	requested_up_corrected = *requested_up;
2598	if (requested_up_corrected == max_up_rounded)
2599	requested_up_corrected = max_up;
2600	else if (requested_up_corrected < 0)
2601	requested_up_corrected = 0;
2602	requested_down_corrected = *requested_down;
2603	if (requested_down_corrected == max_down_rounded)
2604	requested_down_corrected = max_down;
2605	else if (requested_down_corrected < 0)
2606	requested_down_corrected = 0;
2607
2608	tb_tunnel_dbg(tunnel, "corrected bandwidth request %d/%d Mb/s\n",
2609	requested_up_corrected, requested_down_corrected);
2610
2611	if ((*requested_up >= 0 && requested_up_corrected > max_up_rounded) \|\|
2612	(*requested_down >= 0 && requested_down_corrected > max_down_rounded)) {
2613	tb_tunnel_dbg(tunnel,
2614	"bandwidth request too high (%d/%d Mb/s > %d/%d Mb/s)\n",
2615	requested_up_corrected, requested_down_corrected,
2616	max_up_rounded, max_down_rounded);
2617	ret = -ENOBUFS;
2618	goto fail;
2619	}
2620
2621	downstream = tb_tunnel_direction_downstream(tunnel);
2622	group = in->group;
2623
2624	if ((*requested_up >= 0 && requested_up_corrected <= allocated_up) \|\|
2625	(*requested_down >= 0 && requested_down_corrected <= allocated_down)) {
2626	if (tunnel->bw_mode) {
2627	int reserved;
2628	/*
2629	* If requested bandwidth is less or equal than
2630	* what is currently allocated to that tunnel we
2631	* simply change the reservation of the tunnel
2632	* and add the released bandwidth for the group
2633	* for the next 10s. Then we release it for
2634	* others to use.
2635	*/
2636	if (downstream)
2637	reserved = allocated_down - *requested_down;
2638	else
2639	reserved = allocated_up - *requested_up;
2640
2641	if (reserved > 0) {
2642	group->reserved += reserved;
2643	tb_dbg(tb, "group %d reserved %d total %d Mb/s\n",
2644	group->index, reserved, group->reserved);
2645
2646	/*
2647	* If it was not already pending,
2648	* schedule release now. If it is then
2649	* postpone it for the next 10s (unless
2650	* it is already running in which case
2651	* the 10s already expired and we should
2652	* give the reserved back to others).
2653	*/
2654	mod_delayed_work(system_percpu_wq, &group->release_work,
2655	msecs_to_jiffies(TB_RELEASE_BW_TIMEOUT));
2656	}
2657	}
2658
2659	ret = tb_tunnel_alloc_bandwidth(tunnel, requested_up,
2660	requested_down);
2661	if (ret)
2662	goto fail;
2663
2664	return 0;
2665	}
2666
2667	/*
2668	* More bandwidth is requested. Release all the potential
2669	* bandwidth from USB3 first.
2670	*/
2671	ret = tb_release_unused_usb3_bandwidth(tb, in, out);
2672	if (ret)
2673	goto fail;
2674
2675	/*
2676	* Then go over all tunnels that cross the same USB4 ports (they
2677	* are also in the same group but we use the same function here
2678	* that we use with the normal bandwidth allocation).
2679	*/
2680	ret = tb_available_bandwidth(tb, in, out, &available_up, &available_down,
2681	true);
2682	if (ret)
2683	goto reclaim;
2684
2685	tb_tunnel_dbg(tunnel, "bandwidth available for allocation %d/%d (+ %u reserved) Mb/s\n",
2686	available_up, available_down, group->reserved);
2687
2688	if ((*requested_up >= 0 &&
2689	available_up + group->reserved >= requested_up_corrected) \|\|
2690	(*requested_down >= 0 &&
2691	available_down + group->reserved >= requested_down_corrected)) {
2692	int released = 0;
2693
2694	/*
2695	* If bandwidth on a link is >= asym_threshold
2696	* transition the link to asymmetric.
2697	*/
2698	ret = tb_configure_asym(tb, in, out, *requested_up,
2699	*requested_down);
2700	if (ret) {
2701	tb_configure_sym(tb, in, out, true);
2702	goto fail;
2703	}
2704
2705	ret = tb_tunnel_alloc_bandwidth(tunnel, requested_up,
2706	requested_down);
2707	if (ret) {
2708	tb_tunnel_warn(tunnel, "failed to allocate bandwidth\n");
2709	tb_configure_sym(tb, in, out, true);
2710	}
2711
2712	if (downstream) {
2713	if (*requested_down > available_down)
2714	released = *requested_down - available_down;
2715	} else {
2716	if (*requested_up > available_up)
2717	released = *requested_up - available_up;
2718	}
2719	if (released) {
2720	group->reserved -= released;
2721	tb_dbg(tb, "group %d released %d total %d Mb/s\n",
2722	group->index, released, group->reserved);
2723	}
2724	} else {
2725	ret = -ENOBUFS;
2726	}
2727
2728	reclaim:
2729	tb_reclaim_usb3_bandwidth(tb, in, out);
2730	fail:
2731	if (ret && ret != -ENODEV) {
2732	/*
2733	* Write back the same allocated (so no change), this
2734	* makes the DPTX request fail on graphics side.
2735	*/
2736	tb_tunnel_dbg(tunnel,
2737	"failing the request by rewriting allocated %d/%d Mb/s\n",
2738	allocated_up, allocated_down);
2739	tb_tunnel_alloc_bandwidth(tunnel, &allocated_up, &allocated_down);
2740	tb_tunnel_event(tb, TB_TUNNEL_NO_BANDWIDTH, TB_TUNNEL_DP, in, out);
2741	}
2742
2743	return ret;
2744	}
2745
2746	static void tb_handle_dp_bandwidth_request(struct work_struct *work)
2747	{
2748	struct tb_hotplug_event ev = container_of(work, typeof(ev), work.work);
2749	int requested_bw, requested_up, requested_down, ret;
2750	struct tb_tunnel *tunnel;
2751	struct tb *tb = ev->tb;
2752	struct tb_cm *tcm = tb_priv(tb);
2753	struct tb_switch *sw;
2754	struct tb_port *in;
2755
2756	pm_runtime_get_sync(&tb->dev);
2757
2758	mutex_lock(&tb->lock);
2759	if (!tcm->hotplug_active)
2760	goto unlock;
2761
2762	sw = tb_switch_find_by_route(tb, ev->route);
2763	if (!sw) {
2764	tb_warn(tb, "bandwidth request from non-existent router %llx\n",
2765	ev->route);
2766	goto unlock;
2767	}
2768
2769	in = &sw->ports[ev->port];
2770	if (!tb_port_is_dpin(in)) {
2771	tb_port_warn(in, "bandwidth request to non-DP IN adapter\n");
2772	goto put_sw;
2773	}
2774
2775	tb_port_dbg(in, "handling bandwidth allocation request, retry %d\n", ev->retry);
2776
2777	tunnel = tb_find_tunnel(tb, TB_TUNNEL_DP, in, NULL);
2778	if (!tunnel) {
2779	tb_port_warn(in, "failed to find tunnel\n");
2780	goto put_sw;
2781	}
2782
2783	if (!usb4_dp_port_bandwidth_mode_enabled(in)) {
2784	if (tunnel->bw_mode) {
2785	/*
2786	* Reset the tunnel back to use the legacy
2787	* allocation.
2788	*/
2789	tunnel->bw_mode = false;
2790	tb_port_dbg(in, "DPTX disabled bandwidth allocation mode\n");
2791	} else {
2792	tb_port_warn(in, "bandwidth allocation mode not enabled\n");
2793	}
2794	goto put_sw;
2795	}
2796
2797	ret = usb4_dp_port_requested_bandwidth(in);
2798	if (ret < 0) {
2799	if (ret == -ENODATA) {
2800	/*
2801	* There is no request active so this means the
2802	* BW allocation mode was enabled from graphics
2803	* side. At this point we know that the graphics
2804	* driver has read the DPRX capabilities so we
2805	* can offer better bandwidth estimation.
2806	*/
2807	tb_port_dbg(in, "DPTX enabled bandwidth allocation mode, updating estimated bandwidth\n");
2808	tb_recalc_estimated_bandwidth(tb);
2809	} else {
2810	tb_port_warn(in, "failed to read requested bandwidth\n");
2811	}
2812	goto put_sw;
2813	}
2814	requested_bw = ret;
2815
2816	tb_port_dbg(in, "requested bandwidth %d Mb/s\n", requested_bw);
2817
2818	if (tb_tunnel_direction_downstream(tunnel)) {
2819	requested_up = -1;
2820	requested_down = requested_bw;
2821	} else {
2822	requested_up = requested_bw;
2823	requested_down = -1;
2824	}
2825
2826	ret = tb_alloc_dp_bandwidth(tunnel, &requested_up, &requested_down);
2827	if (ret) {
2828	if (ret == -ENOBUFS) {
2829	tb_tunnel_warn(tunnel,
2830	"not enough bandwidth available\n");
2831	} else if (ret == -ENOTCONN) {
2832	tb_tunnel_dbg(tunnel, "not active yet\n");
2833	/*
2834	* We got bandwidth allocation request but the
2835	* tunnel is not yet active. This means that
2836	* tb_dp_tunnel_active() is not yet called for
2837	* this tunnel. Allow it some time and retry
2838	* this request a couple of times.
2839	*/
2840	if (ev->retry < TB_BW_ALLOC_RETRIES) {
2841	tb_tunnel_dbg(tunnel,
2842	"retrying bandwidth allocation request\n");
2843	tb_queue_dp_bandwidth_request(tb, ev->route,
2844	ev->port,
2845	ev->retry + 1,
2846	msecs_to_jiffies(50));
2847	} else {
2848	tb_tunnel_dbg(tunnel,
2849	"run out of retries, failing the request");
2850	}
2851	} else {
2852	tb_tunnel_warn(tunnel,
2853	"failed to change bandwidth allocation\n");
2854	}
2855	} else {
2856	tb_tunnel_dbg(tunnel,
2857	"bandwidth allocation changed to %d/%d Mb/s\n",
2858	requested_up, requested_down);
2859
2860	/* Update other clients about the allocation change */
2861	tb_recalc_estimated_bandwidth(tb);
2862
2863	tb_dbg(tb, "checking if more DP tunnels can be established now\n");
2864	tb_tunnel_dp(tb);
2865	}
2866
2867	put_sw:
2868	tb_switch_put(sw);
2869	unlock:
2870	mutex_unlock(&tb->lock);
2871
2872	pm_runtime_mark_last_busy(&tb->dev);
2873	pm_runtime_put_autosuspend(&tb->dev);
2874
2875	kfree(ev);
2876	}
2877
2878	static void tb_queue_dp_bandwidth_request(struct tb *tb, u64 route, u8 port,
2879	int retry, unsigned long delay)
2880	{
2881	struct tb_hotplug_event *ev;
2882
2883	ev = kmalloc_obj(*ev);
2884	if (!ev)
2885	return;
2886
2887	ev->tb = tb;
2888	ev->route = route;
2889	ev->port = port;
2890	ev->retry = retry;
2891	INIT_DELAYED_WORK(&ev->work, tb_handle_dp_bandwidth_request);
2892	queue_delayed_work(tb->wq, &ev->work, delay);
2893	}
2894
2895	static void tb_handle_notification(struct tb *tb, u64 route,
2896	const struct cfg_error_pkg *error)
2897	{
2898
2899	switch (error->error) {
2900	case TB_CFG_ERROR_PCIE_WAKE:
2901	case TB_CFG_ERROR_DP_CON_CHANGE:
2902	case TB_CFG_ERROR_DPTX_DISCOVERY:
2903	if (tb_cfg_ack_notification(tb->ctl, route, error))
2904	tb_warn(tb, "could not ack notification on %llx\n",
2905	route);
2906	break;
2907
2908	case TB_CFG_ERROR_DP_BW:
2909	if (tb_cfg_ack_notification(tb->ctl, route, error))
2910	tb_warn(tb, "could not ack notification on %llx\n",
2911	route);
2912	tb_queue_dp_bandwidth_request(tb, route, error->port, 0, 0);
2913	break;
2914
2915	default:
2916	/* Ignore for now */
2917	break;
2918	}
2919	}
2920
2921	/*
2922	* tb_schedule_hotplug_handler() - callback function for the control channel
2923	*
2924	* Delegates to tb_handle_hotplug.
2925	*/
2926	static void tb_handle_event(struct tb *tb, enum tb_cfg_pkg_type type,
2927	const void *buf, size_t size)
2928	{
2929	const struct cfg_event_pkg *pkg = buf;
2930	u64 route = tb_cfg_get_route(&pkg->header);
2931
2932	switch (type) {
2933	case TB_CFG_PKG_ERROR:
2934	tb_handle_notification(tb, route, (const struct cfg_error_pkg *)buf);
2935	return;
2936	case TB_CFG_PKG_EVENT:
2937	break;
2938	default:
2939	tb_warn(tb, "unexpected event %#x, ignoring\n", type);
2940	return;
2941	}
2942
2943	if (tb_cfg_ack_plug(tb->ctl, route, pkg->port, pkg->unplug)) {
2944	tb_warn(tb, "could not ack plug event on %llx:%x\n", route,
2945	pkg->port);
2946	}
2947
2948	tb_queue_hotplug(tb, route, pkg->port, pkg->unplug);
2949	}
2950
2951	static void tb_stop(struct tb *tb)
2952	{
2953	struct tb_cm *tcm = tb_priv(tb);
2954	struct tb_nhi *nhi = tb->nhi;
2955	struct tb_tunnel *tunnel;
2956	struct tb_port *port;
2957	struct tb_tunnel *n;
2958
2959	cancel_delayed_work(&tcm->remove_work);
2960	/* tunnels are only present after everything has been initialized */
2961	list_for_each_entry_safe(tunnel, n, &tcm->tunnel_list, list) {
2962	/*
2963	* DMA tunnels and DP tunnels which are not yet active require
2964	* the driver to be functional so we tear them down.
2965	* Other protocol tunnels can be left intact.
2966	*/
2967	if (tb_tunnel_is_dma(tunnel) \|\|
2968	(tb_tunnel_is_dp(tunnel) && !tb_tunnel_is_active(tunnel)))
2969	tb_tunnel_deactivate(tunnel);
2970	tb_tunnel_put(tunnel);
2971	}
2972	/*
2973	* Signal disconnect to connected devices before the router tree is
2974	* removed below. A Thunderbolt 3 device directly connected to a USB4
2975	* host otherwise never receives a disconnect indication, leaving
2976	* firmware to poll the dead link for up to ~60 s which on some
2977	* platforms turns the shutdown into a warm reset. Asserting
2978	* PORT_CS_19.DPR drives SBTX low (USB4 spec section 6.9) so the device
2979	* detects SBRX low and goes to Uninitialized Unplugged immediately.
2980	*/
2981	if (nhi->host_reset) {
2982	tb_switch_for_each_port(tb->root_switch, port) {
2983	if (!tb_port_is_null(port) \|\| !tb_port_has_remote(port))
2984	continue;
2985	if (tb_switch_is_usb4(port->remote->sw))
2986	continue;
2987	if (tb_port_reset(port))
2988	tb_port_dbg(port, "downstream port reset failed, continuing\n");
2989	}
2990	}
2991	tb_switch_remove(tb->root_switch);
2992	tb->root_switch = NULL;
2993	tcm->hotplug_active = false; /* signal tb_handle_hotplug to quit */
2994	}
2995
2996	static void tb_deinit(struct tb *tb)
2997	{
2998	struct tb_cm *tcm = tb_priv(tb);
2999	int i;
3000
3001	/* Cancel all the release bandwidth workers */
3002	for (i = 0; i < ARRAY_SIZE(tcm->groups); i++)
3003	cancel_delayed_work_sync(&tcm->groups[i].release_work);
3004	}
3005
3006	static int tb_scan_finalize_switch(struct device dev, void data)
3007	{
3008	if (tb_is_switch(dev)) {
3009	struct tb_switch *sw = tb_to_switch(dev);
3010
3011	/*
3012	* If we found that the switch was already setup by the
3013	* boot firmware, mark it as authorized now before we
3014	* send uevent to userspace.
3015	*/
3016	if (sw->boot)
3017	sw->authorized = 1;
3018
3019	dev_set_uevent_suppress(dev, false);
3020	kobject_uevent(&dev->kobj, KOBJ_ADD);
3021	device_for_each_child(dev, NULL, tb_scan_finalize_switch);
3022	}
3023
3024	return 0;
3025	}
3026
3027	static int tb_start(struct tb *tb, bool reset)
3028	{
3029	struct tb_cm *tcm = tb_priv(tb);
3030	bool discover = true;
3031	int ret;
3032
3033	tb->root_switch = tb_switch_alloc(tb, &tb->dev, 0);
3034	if (IS_ERR(tb->root_switch))
3035	return dev_err_probe(tb->nhi->dev, PTR_ERR(tb->root_switch),
3036	"failed to allocate host router\n");
3037
3038	/*
3039	* ICM firmware upgrade needs running firmware and in native
3040	* mode that is not available so disable firmware upgrade of the
3041	* root switch.
3042	*
3043	* However, USB4 routers support NVM firmware upgrade if they
3044	* implement the necessary router operations.
3045	*/
3046	tb->root_switch->no_nvm_upgrade = !tb_switch_is_usb4(tb->root_switch);
3047	/* All USB4 routers support runtime PM */
3048	tb->root_switch->rpm = tb_switch_is_usb4(tb->root_switch);
3049
3050	ret = tb_switch_configure(tb->root_switch);
3051	if (ret) {
3052	tb_switch_put(tb->root_switch);
3053	return dev_err_probe(tb->nhi->dev, ret, "failed to configure host router\n");
3054	}
3055
3056	/* Announce the switch to the world */
3057	ret = tb_switch_add(tb->root_switch);
3058	if (ret) {
3059	tb_switch_put(tb->root_switch);
3060	return dev_err_probe(tb->nhi->dev, ret, "failed to add host router\n");
3061	}
3062
3063	/*
3064	* To support highest CLx state, we set host router's TMU to
3065	* Normal mode.
3066	*/
3067	tb_switch_tmu_configure(tb->root_switch, TB_SWITCH_TMU_MODE_LOWRES);
3068	/* Enable TMU if it is off */
3069	tb_switch_tmu_enable(tb->root_switch);
3070
3071	/*
3072	* Boot firmware might have created tunnels of its own. Since we
3073	* cannot be sure they are usable for us, tear them down and
3074	* reset the ports to handle it as new hotplug for USB4 v1
3075	* routers (for USB4 v2 and beyond we already do host reset).
3076	*/
3077	if (reset && tb_switch_is_usb4(tb->root_switch)) {
3078	discover = false;
3079	if (usb4_switch_version(tb->root_switch) == 1)
3080	tb_switch_reset(tb->root_switch);
3081	}
3082
3083	if (discover) {
3084	/* Full scan to discover devices added before the driver was loaded. */
3085	tb_scan_switch(tb->root_switch);
3086	/* Find out tunnels created by the boot firmware */
3087	tb_discover_tunnels(tb);
3088	/* Add DP resources from the DP tunnels created by the boot firmware */
3089	tb_discover_dp_resources(tb);
3090	}
3091
3092	/*
3093	* If the boot firmware did not create USB 3.x tunnels create them
3094	* now for the whole topology.
3095	*/
3096	tb_create_usb3_tunnels(tb->root_switch);
3097	/* Add DP IN resources for the root switch */
3098	tb_add_dp_resources(tb->root_switch);
3099	tb_switch_enter_redrive(tb->root_switch);
3100	/* Make the discovered switches available to the userspace */
3101	device_for_each_child(&tb->root_switch->dev, NULL,
3102	tb_scan_finalize_switch);
3103
3104	/* Allow tb_handle_hotplug to progress events */
3105	tcm->hotplug_active = true;
3106	return 0;
3107	}
3108
3109	static int tb_suspend_noirq(struct tb *tb)
3110	{
3111	struct tb_cm *tcm = tb_priv(tb);
3112
3113	tb_dbg(tb, "suspending...\n");
3114	tb_disconnect_and_release_dp(tb);
3115	tb_switch_exit_redrive(tb->root_switch);
3116	tb_switch_suspend(tb->root_switch, false);
3117	tcm->hotplug_active = false; /* signal tb_handle_hotplug to quit */
3118	tb_dbg(tb, "suspend finished\n");
3119
3120	return 0;
3121	}
3122
3123	static void tb_restore_children(struct tb_switch *sw)
3124	{
3125	struct tb_port *port;
3126
3127	/* No need to restore if the router is already unplugged */
3128	if (sw->is_unplugged)
3129	return;
3130
3131	if (tb_enable_clx(sw))
3132	tb_sw_warn(sw, "failed to re-enable CL states\n");
3133
3134	if (tb_enable_tmu(sw))
3135	tb_sw_warn(sw, "failed to restore TMU configuration\n");
3136
3137	tb_switch_configuration_valid(sw);
3138
3139	tb_switch_for_each_port(sw, port) {
3140	if (!tb_port_has_remote(port) && !port->xdomain)
3141	continue;
3142
3143	if (port->remote) {
3144	tb_switch_set_link_width(port->remote->sw,
3145	port->remote->sw->link_width);
3146	tb_switch_configure_link(port->remote->sw);
3147
3148	tb_restore_children(port->remote->sw);
3149	} else if (port->xdomain) {
3150	tb_port_configure_xdomain(port, port->xdomain);
3151	}
3152	}
3153	}
3154
3155	static void tb_free_unplugged_xdomains(struct tb_switch *sw)
3156	{
3157	struct tb_port *port;
3158
3159	tb_switch_for_each_port(sw, port) {
3160	if (tb_is_upstream_port(port))
3161	continue;
3162	if (port->xdomain && port->xdomain->is_unplugged) {
3163	tb_retimer_remove_all(port);
3164	tb_xdomain_remove(port->xdomain);
3165	tb_port_unconfigure_xdomain(port);
3166	port->xdomain = NULL;
3167	} else if (port->remote) {
3168	tb_free_unplugged_xdomains(port->remote->sw);
3169	}
3170	}
3171	}
3172
3173	static int tb_resume_noirq(struct tb *tb)
3174	{
3175	struct tb_cm *tcm = tb_priv(tb);
3176	struct tb_tunnel tunnel, n;
3177	unsigned int usb3_delay = 0;
3178	LIST_HEAD(tunnels);
3179
3180	tb_dbg(tb, "resuming...\n");
3181
3182	/*
3183	* For non-USB4 hosts (Apple systems) remove any PCIe devices
3184	* the firmware might have setup.
3185	*/
3186	if (!tb_switch_is_usb4(tb->root_switch))
3187	tb_switch_reset(tb->root_switch);
3188
3189	tb_switch_resume(tb->root_switch, false);
3190	tb_free_invalid_tunnels(tb);
3191	tb_free_unplugged_children(tb->root_switch);
3192	tb_free_unplugged_xdomains(tb->root_switch);
3193	tb_restore_children(tb->root_switch);
3194
3195	/*
3196	* If we get here from suspend to disk the boot firmware or the
3197	* restore kernel might have created tunnels of its own. Since
3198	* we cannot be sure they are usable for us we find and tear
3199	* them down.
3200	*/
3201	tb_switch_discover_tunnels(tb->root_switch, &tunnels, false);
3202	list_for_each_entry_safe_reverse(tunnel, n, &tunnels, list) {
3203	if (tb_tunnel_is_usb3(tunnel))
3204	usb3_delay = 500;
3205	tb_tunnel_deactivate(tunnel);
3206	tb_tunnel_put(tunnel);
3207	}
3208
3209	/* Re-create our tunnels now */
3210	list_for_each_entry_safe(tunnel, n, &tcm->tunnel_list, list) {
3211	/* USB3 requires delay before it can be re-activated */
3212	if (tb_tunnel_is_usb3(tunnel)) {
3213	msleep(usb3_delay);
3214	/* Only need to do it once */
3215	usb3_delay = 0;
3216	}
3217	tb_tunnel_activate(tunnel);
3218	}
3219	if (!list_empty(&tcm->tunnel_list)) {
3220	/*
3221	* the pcie links need some time to get going.
3222	* 100ms works for me...
3223	*/
3224	tb_dbg(tb, "tunnels restarted, sleeping for 100ms\n");
3225	msleep(100);
3226	}
3227	tb_switch_enter_redrive(tb->root_switch);
3228	/* Allow tb_handle_hotplug to progress events */
3229	tcm->hotplug_active = true;
3230	tb_dbg(tb, "resume finished\n");
3231
3232	return 0;
3233	}
3234
3235	static int tb_freeze_noirq(struct tb *tb)
3236	{
3237	struct tb_cm *tcm = tb_priv(tb);
3238
3239	tcm->hotplug_active = false;
3240	return 0;
3241	}
3242
3243	static int tb_thaw_noirq(struct tb *tb)
3244	{
3245	struct tb_cm *tcm = tb_priv(tb);
3246
3247	tcm->hotplug_active = true;
3248	return 0;
3249	}
3250
3251	static void tb_complete(struct tb *tb)
3252	{
3253	/*
3254	* Unregister unplugged XDomains and if there is a case where
3255	* another domain is swapped in place of unplugged XDomain we
3256	* need to run another rescan.
3257	*/
3258	if (tb_domain_unregister_unplugged_xdomains(tb)) {
3259	scoped_guard(mutex, &tb->lock)
3260	tb_scan_switch(tb->root_switch);
3261	}
3262	}
3263
3264	static int tb_runtime_suspend(struct tb *tb)
3265	{
3266	struct tb_cm *tcm = tb_priv(tb);
3267
3268	mutex_lock(&tb->lock);
3269	/*
3270	* The below call only releases DP resources to allow exiting and
3271	* re-entering redrive mode.
3272	*/
3273	tb_disconnect_and_release_dp(tb);
3274	tb_switch_exit_redrive(tb->root_switch);
3275	tb_switch_suspend(tb->root_switch, true);
3276	tcm->hotplug_active = false;
3277	mutex_unlock(&tb->lock);
3278
3279	return 0;
3280	}
3281
3282	static void tb_remove_work(struct work_struct *work)
3283	{
3284	struct tb_cm *tcm = container_of(work, struct tb_cm, remove_work.work);
3285	struct tb *tb = tcm_to_tb(tcm);
3286
3287	mutex_lock(&tb->lock);
3288	if (tb->root_switch) {
3289	tb_free_unplugged_children(tb->root_switch);
3290	tb_free_unplugged_xdomains(tb->root_switch);
3291	}
3292	mutex_unlock(&tb->lock);
3293	}
3294
3295	static int tb_runtime_resume(struct tb *tb)
3296	{
3297	struct tb_cm *tcm = tb_priv(tb);
3298	struct tb_tunnel tunnel, n;
3299
3300	mutex_lock(&tb->lock);
3301	tb_switch_resume(tb->root_switch, true);
3302	tb_free_invalid_tunnels(tb);
3303	tb_restore_children(tb->root_switch);
3304	list_for_each_entry_safe(tunnel, n, &tcm->tunnel_list, list)
3305	tb_tunnel_activate(tunnel);
3306	tb_switch_enter_redrive(tb->root_switch);
3307	tcm->hotplug_active = true;
3308	mutex_unlock(&tb->lock);
3309
3310	/*
3311	* Schedule cleanup of any unplugged devices. Run this in a
3312	* separate thread to avoid possible deadlock if the device
3313	* removal runtime resumes the unplugged device.
3314	*/
3315	queue_delayed_work(tb->wq, &tcm->remove_work, msecs_to_jiffies(50));
3316	return 0;
3317	}
3318
3319	static const struct tb_cm_ops tb_cm_ops = {
3320	.start = tb_start,
3321	.stop = tb_stop,
3322	.deinit = tb_deinit,
3323	.suspend_noirq = tb_suspend_noirq,
3324	.resume_noirq = tb_resume_noirq,
3325	.freeze_noirq = tb_freeze_noirq,
3326	.thaw_noirq = tb_thaw_noirq,
3327	.complete = tb_complete,
3328	.runtime_suspend = tb_runtime_suspend,
3329	.runtime_resume = tb_runtime_resume,
3330	.handle_event = tb_handle_event,
3331	.disapprove_switch = tb_disconnect_pci,
3332	.approve_switch = tb_tunnel_pci,
3333	.approve_xdomain_paths = tb_approve_xdomain_paths,
3334	.disconnect_xdomain_paths = tb_disconnect_xdomain_paths,
3335	};
3336
3337	/*
3338	* During suspend the Thunderbolt controller is reset and all PCIe
3339	* tunnels are lost. The NHI driver will try to reestablish all tunnels
3340	* during resume. This adds device links between the tunneled PCIe
3341	* downstream ports and the NHI so that the device core will make sure
3342	* NHI is resumed first before the rest.
3343	*/
3344	static bool tb_apple_add_links(struct tb_nhi *nhi)
3345	{
3346	struct pci_dev *nhi_pdev = to_pci_dev(nhi->dev);
3347	struct pci_dev upstream, pdev;
3348	bool ret;
3349
3350	if (!x86_apple_machine)
3351	return false;
3352
3353	switch (nhi_pdev->device) {
3354	case PCI_DEVICE_ID_INTEL_LIGHT_RIDGE:
3355	case PCI_DEVICE_ID_INTEL_CACTUS_RIDGE_4C:
3356	case PCI_DEVICE_ID_INTEL_FALCON_RIDGE_2C_NHI:
3357	case PCI_DEVICE_ID_INTEL_FALCON_RIDGE_4C_NHI:
3358	break;
3359	default:
3360	return false;
3361	}
3362
3363	upstream = pci_upstream_bridge(nhi_pdev);
3364	while (upstream) {
3365	if (!pci_is_pcie(upstream))
3366	return false;
3367	if (pci_pcie_type(upstream) == PCI_EXP_TYPE_UPSTREAM)
3368	break;
3369	upstream = pci_upstream_bridge(upstream);
3370	}
3371
3372	if (!upstream)
3373	return false;
3374
3375	/*
3376	* For each hotplug downstream port, create add device link
3377	* back to NHI so that PCIe tunnels can be re-established after
3378	* sleep.
3379	*/
3380	ret = false;
3381	for_each_pci_bridge(pdev, upstream->subordinate) {
3382	const struct device_link *link;
3383
3384	if (!pci_is_pcie(pdev))
3385	continue;
3386	if (pci_pcie_type(pdev) != PCI_EXP_TYPE_DOWNSTREAM \|\|
3387	!pdev->is_pciehp)
3388	continue;
3389
3390	link = device_link_add(&pdev->dev, nhi->dev,
3391	DL_FLAG_AUTOREMOVE_SUPPLIER \|
3392	DL_FLAG_PM_RUNTIME);
3393	if (link) {
3394	dev_dbg(nhi->dev, "created link from %s\n",
3395	dev_name(&pdev->dev));
3396	ret = true;
3397	} else {
3398	dev_warn(nhi->dev, "device link creation from %s failed\n",
3399	dev_name(&pdev->dev));
3400	}
3401	}
3402
3403	return ret;
3404	}
3405
3406	struct tb tb_probe(struct tb_nhi nhi)
3407	{
3408	struct tb_cm *tcm;
3409	struct tb *tb;
3410
3411	tb = tb_domain_alloc(nhi, TB_TIMEOUT, sizeof(*tcm));
3412	if (!tb)
3413	return NULL;
3414
3415	if (tb_acpi_may_tunnel_pcie())
3416	tb->security_level = TB_SECURITY_USER;
3417	else
3418	tb->security_level = TB_SECURITY_NOPCIE;
3419
3420	tb->cm_ops = &tb_cm_ops;
3421
3422	tcm = tb_priv(tb);
3423	INIT_LIST_HEAD(&tcm->tunnel_list);
3424	INIT_LIST_HEAD(&tcm->dp_resources);
3425	INIT_DELAYED_WORK(&tcm->remove_work, tb_remove_work);
3426	tb_init_bandwidth_groups(tcm);
3427
3428	tb_dbg(tb, "using software connection manager\n");
3429
3430	/*
3431	* Device links are needed to make sure we establish tunnels
3432	* before the PCIe/USB stack is resumed so complain here if we
3433	* found them missing.
3434	*/
3435	if (!tb_apple_add_links(nhi) && !tb_acpi_add_links(nhi))
3436	tb_warn(tb, "device links to tunneled native ports are missing!\n");
3437
3438	return tb;
3439	}