projects/linux·blob master · refs

2607 lines · 65.7 KiB

1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Internal Thunderbolt Connection Manager. This is a firmware running on
4 * the Thunderbolt host controller performing most of the low-level
5 * handling.
6 *
7 * Copyright (C) 2017, Intel Corporation
8 * Authors: Michael Jamet <[email protected]>
9 * Mika Westerberg <[email protected]>
10 */
11
12#include <linux/delay.h>
13#include <linux/mutex.h>
14#include <linux/moduleparam.h>
15#include <linux/pci.h>
16#include <linux/pm_runtime.h>
17#include <linux/platform_data/x86/apple.h>
18#include <linux/sizes.h>
19#include <linux/slab.h>
20#include <linux/workqueue.h>
21
22#include "ctl.h"
23#include "nhi_regs.h"
24#include "tb.h"
25#include "tunnel.h"
26
27#define PCIE2CIO_CMD 0x30
28#define PCIE2CIO_CMD_TIMEOUT BIT(31)
29#define PCIE2CIO_CMD_START BIT(30)
30#define PCIE2CIO_CMD_WRITE BIT(21)
31#define PCIE2CIO_CMD_CS_MASK GENMASK(20, 19)
32#define PCIE2CIO_CMD_CS_SHIFT 19
33#define PCIE2CIO_CMD_PORT_MASK GENMASK(18, 13)
34#define PCIE2CIO_CMD_PORT_SHIFT 13
35
36#define PCIE2CIO_WRDATA 0x34
37#define PCIE2CIO_RDDATA 0x38
38
39#define PHY_PORT_CS1 0x37
40#define PHY_PORT_CS1_LINK_DISABLE BIT(14)
41#define PHY_PORT_CS1_LINK_STATE_MASK GENMASK(29, 26)
42#define PHY_PORT_CS1_LINK_STATE_SHIFT 26
43
44#define ICM_TIMEOUT 5000 /* ms */
45#define ICM_RETRIES 3
46#define ICM_APPROVE_TIMEOUT 10000 /* ms */
47#define ICM_MAX_LINK 4
48
49static bool start_icm;
50module_param(start_icm, bool, 0444);
51MODULE_PARM_DESC(start_icm, "start ICM firmware if it is not running (default: false)");
52
53/**
54 * struct usb4_switch_nvm_auth - Holds USB4 NVM_AUTH status
55 * @reply: Reply from ICM firmware is placed here
56 * @request: Request that is sent to ICM firmware
57 * @icm: Pointer to ICM private data
58 */
59struct usb4_switch_nvm_auth {
60 struct icm_usb4_switch_op_response reply;
61 struct icm_usb4_switch_op request;
62 struct icm *icm;
63};
64
65/**
66 * struct icm - Internal connection manager private data
67 * @request_lock: Makes sure only one message is send to ICM at time
68 * @rescan_work: Work used to rescan the surviving switches after resume
69 * @upstream_port: Pointer to the PCIe upstream port this host
70 * controller is connected. This is only set for systems
71 * where ICM needs to be started manually
72 * @vnd_cap: Vendor defined capability where PCIe2CIO mailbox resides
73 * (only set when @upstream_port is not %NULL)
74 * @safe_mode: ICM is in safe mode
75 * @max_boot_acl: Maximum number of preboot ACL entries (%0 if not supported)
76 * @rpm: Does the controller support runtime PM (RTD3)
77 * @can_upgrade_nvm: Can the NVM firmware be upgrade on this controller
78 * @proto_version: Firmware protocol version
79 * @last_nvm_auth: Last USB4 router NVM_AUTH result (or %NULL if not set)
80 * @veto: Is RTD3 veto in effect
81 * @is_supported: Checks if we can support ICM on this controller
82 * @cio_reset: Trigger CIO reset
83 * @get_mode: Read and return the ICM firmware mode (optional)
84 * @get_route: Find a route string for given switch
85 * @save_devices: Ask ICM to save devices to ACL when suspending (optional)
86 * @driver_ready: Send driver ready message to ICM
87 * @set_uuid: Set UUID for the root switch (optional)
88 * @device_connected: Handle device connected ICM message
89 * @device_disconnected: Handle device disconnected ICM message
90 * @xdomain_connected: Handle XDomain connected ICM message
91 * @xdomain_disconnected: Handle XDomain disconnected ICM message
92 * @rtd3_veto: Handle RTD3 veto notification ICM message
93 */
94struct icm {
95 struct mutex request_lock;
96 struct delayed_work rescan_work;
97 struct pci_dev *upstream_port;
98 int vnd_cap;
99 bool safe_mode;
100 size_t max_boot_acl;
101 bool rpm;
102 bool can_upgrade_nvm;
103 u8 proto_version;
104 struct usb4_switch_nvm_auth *last_nvm_auth;
105 bool veto;
106 bool (*is_supported)(struct tb *tb);
107 int (*cio_reset)(struct tb *tb);
108 int (*get_mode)(struct tb *tb);
109 int (*get_route)(struct tb *tb, u8 link, u8 depth, u64 *route);
110 void (*save_devices)(struct tb *tb);
111 int (*driver_ready)(struct tb *tb,
112 enum tb_security_level *security_level,
113 u8 *proto_version, size_t *nboot_acl, bool *rpm);
114 void (*set_uuid)(struct tb *tb);
115 void (*device_connected)(struct tb *tb,
116 const struct icm_pkg_header *hdr);
117 void (*device_disconnected)(struct tb *tb,
118 const struct icm_pkg_header *hdr);
119 void (*xdomain_connected)(struct tb *tb,
120 const struct icm_pkg_header *hdr);
121 void (*xdomain_disconnected)(struct tb *tb,
122 const struct icm_pkg_header *hdr);
123 void (*rtd3_veto)(struct tb *tb, const struct icm_pkg_header *hdr);
124};
125
126struct icm_notification {
127 struct work_struct work;
128 struct icm_pkg_header *pkg;
129 struct tb *tb;
130};
131
132struct ep_name_entry {
133 u8 len;
134 u8 type;
135 u8 data[];
136};
137
138#define EP_NAME_INTEL_VSS 0x10
139
140/* Intel Vendor specific structure */
141struct intel_vss {
142 u16 vendor;
143 u16 model;
144 u8 mc;
145 u8 flags;
146 u16 pci_devid;
147 u32 nvm_version;
148};
149
150#define INTEL_VSS_FLAGS_RTD3 BIT(0)
151
152static const struct intel_vss *parse_intel_vss(const void *ep_name, size_t size)
153{
154 const void *end = ep_name + size;
155
156 while (ep_name < end) {
157 const struct ep_name_entry *ep = ep_name;
158
159 if (!ep->len)
160 break;
161 if (ep_name + ep->len > end)
162 break;
163
164 if (ep->type == EP_NAME_INTEL_VSS)
165 return (const struct intel_vss *)ep->data;
166
167 ep_name += ep->len;
168 }
169
170 return NULL;
171}
172
173static bool intel_vss_is_rtd3(const void *ep_name, size_t size)
174{
175 const struct intel_vss *vss;
176
177 vss = parse_intel_vss(ep_name, size);
178 if (vss)
179 return !!(vss->flags & INTEL_VSS_FLAGS_RTD3);
180
181 return false;
182}
183
184static inline struct tb *icm_to_tb(struct icm *icm)
185{
186 return ((void *)icm - sizeof(struct tb));
187}
188
189static inline u8 phy_port_from_route(u64 route, u8 depth)
190{
191 u8 link;
192
193 link = depth ? route >> ((depth - 1) * 8) : route;
194 return tb_phy_port_from_link(link);
195}
196
197static inline u8 dual_link_from_link(u8 link)
198{
199 return link ? ((link - 1) ^ 0x01) + 1 : 0;
200}
201
202static inline u64 get_route(u32 route_hi, u32 route_lo)
203{
204 return (u64)route_hi << 32 | route_lo;
205}
206
207static inline u64 get_parent_route(u64 route)
208{
209 int depth = tb_route_length(route);
210 return depth ? route & ~(0xffULL << (depth - 1) * TB_ROUTE_SHIFT) : 0;
211}
212
213static int pci2cio_wait_completion(struct icm *icm, unsigned long timeout_msec)
214{
215 unsigned long end = jiffies + msecs_to_jiffies(timeout_msec);
216 u32 cmd;
217
218 do {
219 pci_read_config_dword(icm->upstream_port,
220 icm->vnd_cap + PCIE2CIO_CMD, &cmd);
221 if (!(cmd & PCIE2CIO_CMD_START)) {
222 if (cmd & PCIE2CIO_CMD_TIMEOUT)
223 break;
224 return 0;
225 }
226
227 msleep(50);
228 } while (time_before(jiffies, end));
229
230 return -ETIMEDOUT;
231}
232
233static int pcie2cio_read(struct icm *icm, enum tb_cfg_space cs,
234 unsigned int port, unsigned int index, u32 *data)
235{
236 struct pci_dev *pdev = icm->upstream_port;
237 int ret, vnd_cap = icm->vnd_cap;
238 u32 cmd;
239
240 cmd = index;
241 cmd |= (port << PCIE2CIO_CMD_PORT_SHIFT) & PCIE2CIO_CMD_PORT_MASK;
242 cmd |= (cs << PCIE2CIO_CMD_CS_SHIFT) & PCIE2CIO_CMD_CS_MASK;
243 cmd |= PCIE2CIO_CMD_START;
244 pci_write_config_dword(pdev, vnd_cap + PCIE2CIO_CMD, cmd);
245
246 ret = pci2cio_wait_completion(icm, 5000);
247 if (ret)
248 return ret;
249
250 pci_read_config_dword(pdev, vnd_cap + PCIE2CIO_RDDATA, data);
251 return 0;
252}
253
254static int pcie2cio_write(struct icm *icm, enum tb_cfg_space cs,
255 unsigned int port, unsigned int index, u32 data)
256{
257 struct pci_dev *pdev = icm->upstream_port;
258 int vnd_cap = icm->vnd_cap;
259 u32 cmd;
260
261 pci_write_config_dword(pdev, vnd_cap + PCIE2CIO_WRDATA, data);
262
263 cmd = index;
264 cmd |= (port << PCIE2CIO_CMD_PORT_SHIFT) & PCIE2CIO_CMD_PORT_MASK;
265 cmd |= (cs << PCIE2CIO_CMD_CS_SHIFT) & PCIE2CIO_CMD_CS_MASK;
266 cmd |= PCIE2CIO_CMD_WRITE | PCIE2CIO_CMD_START;
267 pci_write_config_dword(pdev, vnd_cap + PCIE2CIO_CMD, cmd);
268
269 return pci2cio_wait_completion(icm, 5000);
270}
271
272static bool icm_match(const struct tb_cfg_request *req,
273 const struct ctl_pkg *pkg)
274{
275 const struct icm_pkg_header *res_hdr = pkg->buffer;
276 const struct icm_pkg_header *req_hdr = req->request;
277
278 if (pkg->frame.eof != req->response_type)
279 return false;
280 if (res_hdr->code != req_hdr->code)
281 return false;
282
283 return true;
284}
285
286static bool icm_copy(struct tb_cfg_request *req, const struct ctl_pkg *pkg)
287{
288 const struct icm_pkg_header *hdr = pkg->buffer;
289
290 if (hdr->packet_id < req->npackets) {
291 size_t offset = hdr->packet_id * req->response_size;
292
293 memcpy(req->response + offset, pkg->buffer, req->response_size);
294 }
295
296 return hdr->packet_id == hdr->total_packets - 1;
297}
298
299static int icm_request(struct tb *tb, const void *request, size_t request_size,
300 void *response, size_t response_size, size_t npackets,
301 int retries, unsigned int timeout_msec)
302{
303 struct icm *icm = tb_priv(tb);
304
305 do {
306 struct tb_cfg_request *req;
307 struct tb_cfg_result res;
308
309 req = tb_cfg_request_alloc();
310 if (!req)
311 return -ENOMEM;
312
313 req->match = icm_match;
314 req->copy = icm_copy;
315 req->request = request;
316 req->request_size = request_size;
317 req->request_type = TB_CFG_PKG_ICM_CMD;
318 req->response = response;
319 req->npackets = npackets;
320 req->response_size = response_size;
321 req->response_type = TB_CFG_PKG_ICM_RESP;
322
323 mutex_lock(&icm->request_lock);
324 res = tb_cfg_request_sync(tb->ctl, req, timeout_msec);
325 mutex_unlock(&icm->request_lock);
326
327 tb_cfg_request_put(req);
328
329 if (res.err != -ETIMEDOUT)
330 return res.err == 1 ? -EIO : res.err;
331
332 usleep_range(20, 50);
333 } while (retries--);
334
335 return -ETIMEDOUT;
336}
337
338/*
339 * If rescan is queued to run (we are resuming), postpone it to give the
340 * firmware some more time to send device connected notifications for next
341 * devices in the chain.
342 */
343static void icm_postpone_rescan(struct tb *tb)
344{
345 struct icm *icm = tb_priv(tb);
346
347 if (delayed_work_pending(&icm->rescan_work))
348 mod_delayed_work(tb->wq, &icm->rescan_work,
349 msecs_to_jiffies(500));
350}
351
352static void icm_veto_begin(struct tb *tb)
353{
354 struct icm *icm = tb_priv(tb);
355
356 if (!icm->veto) {
357 icm->veto = true;
358 /* Keep the domain powered while veto is in effect */
359 pm_runtime_get(&tb->dev);
360 }
361}
362
363static void icm_veto_end(struct tb *tb)
364{
365 struct icm *icm = tb_priv(tb);
366
367 if (icm->veto) {
368 icm->veto = false;
369 /* Allow the domain suspend now */
370 pm_runtime_mark_last_busy(&tb->dev);
371 pm_runtime_put_autosuspend(&tb->dev);
372 }
373}
374
375static bool icm_firmware_running(const struct tb_nhi *nhi)
376{
377 u32 val;
378
379 val = ioread32(nhi->iobase + REG_FW_STS);
380 return !!(val & REG_FW_STS_ICM_EN);
381}
382
383static void icm_xdomain_activated(struct tb_xdomain *xd, bool activated)
384{
385 struct tb_port *nhi_port, *dst_port;
386 struct tb *tb = xd->tb;
387
388 nhi_port = tb_switch_find_port(tb->root_switch, TB_TYPE_NHI);
389 dst_port = tb_xdomain_downstream_port(xd);
390
391 if (activated)
392 tb_tunnel_event(tb, TB_TUNNEL_ACTIVATED, TB_TUNNEL_DMA,
393 nhi_port, dst_port);
394 else
395 tb_tunnel_event(tb, TB_TUNNEL_DEACTIVATED, TB_TUNNEL_DMA,
396 nhi_port, dst_port);
397}
398
399static void icm_dp_event(struct tb *tb)
400{
401 tb_tunnel_event(tb, TB_TUNNEL_CHANGED, TB_TUNNEL_DP, NULL, NULL);
402}
403
404static bool icm_fr_is_supported(struct tb *tb)
405{
406 return !x86_apple_machine;
407}
408
409static inline int icm_fr_get_switch_index(u32 port)
410{
411 int index;
412
413 if ((port & ICM_PORT_TYPE_MASK) != TB_TYPE_PORT)
414 return 0;
415
416 index = port >> ICM_PORT_INDEX_SHIFT;
417 return index != 0xff ? index : 0;
418}
419
420static int icm_fr_get_route(struct tb *tb, u8 link, u8 depth, u64 *route)
421{
422 struct icm_fr_pkg_get_topology_response *switches, *sw;
423 struct icm_fr_pkg_get_topology request = {
424 .hdr = { .code = ICM_GET_TOPOLOGY },
425 };
426 size_t npackets = ICM_GET_TOPOLOGY_PACKETS;
427 int ret, index;
428 u8 i;
429
430 switches = kzalloc_objs(*switches, npackets);
431 if (!switches)
432 return -ENOMEM;
433
434 ret = icm_request(tb, &request, sizeof(request), switches,
435 sizeof(*switches), npackets, ICM_RETRIES, ICM_TIMEOUT);
436 if (ret)
437 goto err_free;
438
439 sw = &switches[0];
440 index = icm_fr_get_switch_index(sw->ports[link]);
441 if (!index) {
442 ret = -ENODEV;
443 goto err_free;
444 }
445
446 sw = &switches[index];
447 for (i = 1; i < depth; i++) {
448 unsigned int j;
449
450 if (!(sw->first_data & ICM_SWITCH_USED)) {
451 ret = -ENODEV;
452 goto err_free;
453 }
454
455 for (j = 0; j < ARRAY_SIZE(sw->ports); j++) {
456 index = icm_fr_get_switch_index(sw->ports[j]);
457 if (index > sw->switch_index) {
458 sw = &switches[index];
459 break;
460 }
461 }
462 }
463
464 *route = get_route(sw->route_hi, sw->route_lo);
465
466err_free:
467 kfree(switches);
468 return ret;
469}
470
471static void icm_fr_save_devices(struct tb *tb)
472{
473 nhi_mailbox_cmd(tb->nhi, NHI_MAILBOX_SAVE_DEVS, 0);
474}
475
476static int
477icm_fr_driver_ready(struct tb *tb, enum tb_security_level *security_level,
478 u8 *proto_version, size_t *nboot_acl, bool *rpm)
479{
480 struct icm_fr_pkg_driver_ready_response reply;
481 struct icm_pkg_driver_ready request = {
482 .hdr.code = ICM_DRIVER_READY,
483 };
484 int ret;
485
486 memset(&reply, 0, sizeof(reply));
487 ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),
488 1, ICM_RETRIES, ICM_TIMEOUT);
489 if (ret)
490 return ret;
491
492 if (security_level)
493 *security_level = reply.security_level & ICM_FR_SLEVEL_MASK;
494
495 return 0;
496}
497
498static int icm_fr_approve_switch(struct tb *tb, struct tb_switch *sw)
499{
500 struct icm_fr_pkg_approve_device request;
501 struct icm_fr_pkg_approve_device reply;
502 int ret;
503
504 memset(&request, 0, sizeof(request));
505 memcpy(&request.ep_uuid, sw->uuid, sizeof(request.ep_uuid));
506 request.hdr.code = ICM_APPROVE_DEVICE;
507 request.connection_id = sw->connection_id;
508 request.connection_key = sw->connection_key;
509
510 memset(&reply, 0, sizeof(reply));
511 /* Use larger timeout as establishing tunnels can take some time */
512 ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),
513 1, ICM_RETRIES, ICM_APPROVE_TIMEOUT);
514 if (ret)
515 return ret;
516
517 if (reply.hdr.flags & ICM_FLAGS_ERROR) {
518 tb_warn(tb, "PCIe tunnel creation failed\n");
519 return -EIO;
520 }
521
522 return 0;
523}
524
525static int icm_fr_add_switch_key(struct tb *tb, struct tb_switch *sw)
526{
527 struct icm_fr_pkg_add_device_key request;
528 struct icm_fr_pkg_add_device_key_response reply;
529 int ret;
530
531 memset(&request, 0, sizeof(request));
532 memcpy(&request.ep_uuid, sw->uuid, sizeof(request.ep_uuid));
533 request.hdr.code = ICM_ADD_DEVICE_KEY;
534 request.connection_id = sw->connection_id;
535 request.connection_key = sw->connection_key;
536 memcpy(request.key, sw->key, TB_SWITCH_KEY_SIZE);
537
538 memset(&reply, 0, sizeof(reply));
539 ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),
540 1, ICM_RETRIES, ICM_TIMEOUT);
541 if (ret)
542 return ret;
543
544 if (reply.hdr.flags & ICM_FLAGS_ERROR) {
545 tb_warn(tb, "Adding key to switch failed\n");
546 return -EIO;
547 }
548
549 return 0;
550}
551
552static int icm_fr_challenge_switch_key(struct tb *tb, struct tb_switch *sw,
553 const u8 *challenge, u8 *response)
554{
555 struct icm_fr_pkg_challenge_device request;
556 struct icm_fr_pkg_challenge_device_response reply;
557 int ret;
558
559 memset(&request, 0, sizeof(request));
560 memcpy(&request.ep_uuid, sw->uuid, sizeof(request.ep_uuid));
561 request.hdr.code = ICM_CHALLENGE_DEVICE;
562 request.connection_id = sw->connection_id;
563 request.connection_key = sw->connection_key;
564 memcpy(request.challenge, challenge, TB_SWITCH_KEY_SIZE);
565
566 memset(&reply, 0, sizeof(reply));
567 ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),
568 1, ICM_RETRIES, ICM_TIMEOUT);
569 if (ret)
570 return ret;
571
572 if (reply.hdr.flags & ICM_FLAGS_ERROR)
573 return -EKEYREJECTED;
574 if (reply.hdr.flags & ICM_FLAGS_NO_KEY)
575 return -ENOKEY;
576
577 memcpy(response, reply.response, TB_SWITCH_KEY_SIZE);
578
579 return 0;
580}
581
582static int icm_fr_approve_xdomain_paths(struct tb *tb, struct tb_xdomain *xd,
583 int transmit_path, int transmit_ring,
584 int receive_path, int receive_ring)
585{
586 struct icm_fr_pkg_approve_xdomain_response reply;
587 struct icm_fr_pkg_approve_xdomain request;
588 int ret;
589
590 if (atomic_read(&xd->ntunnels) >= 1) {
591 tb_warn(tb, "only one tunnel is supported by the firmware\n");
592 return -EOPNOTSUPP;
593 }
594
595 memset(&request, 0, sizeof(request));
596 request.hdr.code = ICM_APPROVE_XDOMAIN;
597 request.link_info = xd->depth << ICM_LINK_INFO_DEPTH_SHIFT | xd->link;
598 memcpy(&request.remote_uuid, xd->remote_uuid, sizeof(*xd->remote_uuid));
599
600 request.transmit_path = transmit_path;
601 request.transmit_ring = transmit_ring;
602 request.receive_path = receive_path;
603 request.receive_ring = receive_ring;
604
605 memset(&reply, 0, sizeof(reply));
606 ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),
607 1, ICM_RETRIES, ICM_TIMEOUT);
608 if (ret)
609 return ret;
610
611 if (reply.hdr.flags & ICM_FLAGS_ERROR)
612 return -EIO;
613
614 icm_xdomain_activated(xd, true);
615 return 0;
616}
617
618static int icm_fr_disconnect_xdomain_paths(struct tb *tb, struct tb_xdomain *xd,
619 int transmit_path, int transmit_ring,
620 int receive_path, int receive_ring)
621{
622 u8 phy_port;
623 u8 cmd;
624
625 phy_port = tb_phy_port_from_link(xd->link);
626 if (phy_port == 0)
627 cmd = NHI_MAILBOX_DISCONNECT_PA;
628 else
629 cmd = NHI_MAILBOX_DISCONNECT_PB;
630
631 nhi_mailbox_cmd(tb->nhi, cmd, 1);
632 usleep_range(10, 50);
633 nhi_mailbox_cmd(tb->nhi, cmd, 2);
634
635 icm_xdomain_activated(xd, false);
636 return 0;
637}
638
639static struct tb_switch *alloc_switch(struct tb_switch *parent_sw, u64 route,
640 const uuid_t *uuid)
641{
642 struct tb *tb = parent_sw->tb;
643 struct tb_switch *sw;
644
645 sw = tb_switch_alloc(tb, &parent_sw->dev, route);
646 if (IS_ERR(sw)) {
647 tb_warn(tb, "failed to allocate switch at %llx\n", route);
648 return sw;
649 }
650
651 sw->uuid = kmemdup(uuid, sizeof(*uuid), GFP_KERNEL);
652 if (!sw->uuid) {
653 tb_switch_put(sw);
654 return ERR_PTR(-ENOMEM);
655 }
656
657 init_completion(&sw->rpm_complete);
658 return sw;
659}
660
661static int add_switch(struct tb_switch *parent_sw, struct tb_switch *sw)
662{
663 u64 route = tb_route(sw);
664 int ret;
665
666 /* Link the two switches now */
667 tb_port_at(route, parent_sw)->remote = tb_upstream_port(sw);
668 tb_upstream_port(sw)->remote = tb_port_at(route, parent_sw);
669
670 ret = tb_switch_add(sw);
671 if (ret)
672 tb_port_at(tb_route(sw), parent_sw)->remote = NULL;
673
674 return ret;
675}
676
677static void update_switch(struct tb_switch *sw, u64 route, u8 connection_id,
678 u8 connection_key, u8 link, u8 depth, bool boot)
679{
680 struct tb_switch *parent_sw = tb_switch_parent(sw);
681
682 /* Disconnect from parent */
683 tb_switch_downstream_port(sw)->remote = NULL;
684 /* Re-connect via updated port */
685 tb_port_at(route, parent_sw)->remote = tb_upstream_port(sw);
686
687 /* Update with the new addressing information */
688 sw->config.route_hi = upper_32_bits(route);
689 sw->config.route_lo = lower_32_bits(route);
690 sw->connection_id = connection_id;
691 sw->connection_key = connection_key;
692 sw->link = link;
693 sw->depth = depth;
694 sw->boot = boot;
695
696 /* This switch still exists */
697 sw->is_unplugged = false;
698
699 /* Runtime resume is now complete */
700 complete(&sw->rpm_complete);
701}
702
703static void remove_switch(struct tb_switch *sw)
704{
705 tb_switch_downstream_port(sw)->remote = NULL;
706 tb_switch_remove(sw);
707}
708
709static void add_xdomain(struct tb_switch *sw, u64 route,
710 const uuid_t *local_uuid, const uuid_t *remote_uuid,
711 u8 link, u8 depth)
712{
713 struct tb_xdomain *xd;
714
715 pm_runtime_get_sync(&sw->dev);
716
717 xd = tb_xdomain_alloc(sw->tb, &sw->dev, route, local_uuid, remote_uuid);
718 if (!xd)
719 goto out;
720
721 xd->link = link;
722 xd->depth = depth;
723
724 tb_port_at(route, sw)->xdomain = xd;
725
726 tb_xdomain_add(xd);
727
728out:
729 pm_runtime_mark_last_busy(&sw->dev);
730 pm_runtime_put_autosuspend(&sw->dev);
731}
732
733static void update_xdomain(struct tb_xdomain *xd, u64 route, u8 link)
734{
735 xd->link = link;
736 xd->route = route;
737 xd->is_unplugged = false;
738}
739
740static void remove_xdomain(struct tb_xdomain *xd)
741{
742 struct tb_switch *sw;
743
744 sw = tb_to_switch(xd->dev.parent);
745 tb_port_at(xd->route, sw)->xdomain = NULL;
746 xd->is_unplugged = true;
747 tb_xdomain_remove(xd);
748}
749
750static void
751icm_fr_device_connected(struct tb *tb, const struct icm_pkg_header *hdr)
752{
753 const struct icm_fr_event_device_connected *pkg =
754 (const struct icm_fr_event_device_connected *)hdr;
755 enum tb_security_level security_level;
756 struct tb_switch *sw, *parent_sw;
757 bool boot, dual_lane, speed_gen3;
758 struct icm *icm = tb_priv(tb);
759 bool authorized = false;
760 struct tb_xdomain *xd;
761 u8 link, depth;
762 u64 route;
763 int ret;
764
765 icm_postpone_rescan(tb);
766
767 link = pkg->link_info & ICM_LINK_INFO_LINK_MASK;
768 depth = (pkg->link_info & ICM_LINK_INFO_DEPTH_MASK) >>
769 ICM_LINK_INFO_DEPTH_SHIFT;
770 authorized = pkg->link_info & ICM_LINK_INFO_APPROVED;
771 security_level = (pkg->hdr.flags & ICM_FLAGS_SLEVEL_MASK) >>
772 ICM_FLAGS_SLEVEL_SHIFT;
773 boot = pkg->link_info & ICM_LINK_INFO_BOOT;
774 dual_lane = pkg->hdr.flags & ICM_FLAGS_DUAL_LANE;
775 speed_gen3 = pkg->hdr.flags & ICM_FLAGS_SPEED_GEN3;
776
777 if (pkg->link_info & ICM_LINK_INFO_REJECTED) {
778 tb_info(tb, "switch at %u.%u was rejected by ICM firmware because topology limit exceeded\n",
779 link, depth);
780 return;
781 }
782
783 sw = tb_switch_find_by_uuid(tb, &pkg->ep_uuid);
784 if (sw) {
785 u8 phy_port, sw_phy_port;
786
787 sw_phy_port = tb_phy_port_from_link(sw->link);
788 phy_port = tb_phy_port_from_link(link);
789
790 /*
791 * On resume ICM will send us connected events for the
792 * devices that still are present. However, that
793 * information might have changed for example by the
794 * fact that a switch on a dual-link connection might
795 * have been enumerated using the other link now. Make
796 * sure our bookkeeping matches that.
797 */
798 if (sw->depth == depth && sw_phy_port == phy_port &&
799 !!sw->authorized == authorized) {
800 /*
801 * It was enumerated through another link so update
802 * route string accordingly.
803 */
804 if (sw->link != link) {
805 ret = icm->get_route(tb, link, depth, &route);
806 if (ret) {
807 tb_err(tb, "failed to update route string for switch at %u.%u\n",
808 link, depth);
809 tb_switch_put(sw);
810 return;
811 }
812 } else {
813 route = tb_route(sw);
814 }
815
816 update_switch(sw, route, pkg->connection_id,
817 pkg->connection_key, link, depth, boot);
818 tb_switch_put(sw);
819 return;
820 }
821
822 /*
823 * User connected the same switch to another physical
824 * port or to another part of the topology. Remove the
825 * existing switch now before adding the new one.
826 */
827 remove_switch(sw);
828 tb_switch_put(sw);
829 }
830
831 /*
832 * If the switch was not found by UUID, look for a switch on
833 * same physical port (taking possible link aggregation into
834 * account) and depth. If we found one it is definitely a stale
835 * one so remove it first.
836 */
837 sw = tb_switch_find_by_link_depth(tb, link, depth);
838 if (!sw) {
839 u8 dual_link;
840
841 dual_link = dual_link_from_link(link);
842 if (dual_link)
843 sw = tb_switch_find_by_link_depth(tb, dual_link, depth);
844 }
845 if (sw) {
846 remove_switch(sw);
847 tb_switch_put(sw);
848 }
849
850 /* Remove existing XDomain connection if found */
851 xd = tb_xdomain_find_by_link_depth(tb, link, depth);
852 if (xd) {
853 remove_xdomain(xd);
854 tb_xdomain_put(xd);
855 }
856
857 parent_sw = tb_switch_find_by_link_depth(tb, link, depth - 1);
858 if (!parent_sw) {
859 tb_err(tb, "failed to find parent switch for %u.%u\n",
860 link, depth);
861 return;
862 }
863
864 ret = icm->get_route(tb, link, depth, &route);
865 if (ret) {
866 tb_err(tb, "failed to find route string for switch at %u.%u\n",
867 link, depth);
868 tb_switch_put(parent_sw);
869 return;
870 }
871
872 pm_runtime_get_sync(&parent_sw->dev);
873
874 sw = alloc_switch(parent_sw, route, &pkg->ep_uuid);
875 if (!IS_ERR(sw)) {
876 sw->connection_id = pkg->connection_id;
877 sw->connection_key = pkg->connection_key;
878 sw->link = link;
879 sw->depth = depth;
880 sw->authorized = authorized;
881 sw->security_level = security_level;
882 sw->boot = boot;
883 sw->link_speed = speed_gen3 ? 20 : 10;
884 sw->link_width = dual_lane ? TB_LINK_WIDTH_DUAL :
885 TB_LINK_WIDTH_SINGLE;
886 sw->rpm = intel_vss_is_rtd3(pkg->ep_name, sizeof(pkg->ep_name));
887
888 if (add_switch(parent_sw, sw))
889 tb_switch_put(sw);
890 }
891
892 pm_runtime_mark_last_busy(&parent_sw->dev);
893 pm_runtime_put_autosuspend(&parent_sw->dev);
894
895 tb_switch_put(parent_sw);
896}
897
898static void
899icm_fr_device_disconnected(struct tb *tb, const struct icm_pkg_header *hdr)
900{
901 const struct icm_fr_event_device_disconnected *pkg =
902 (const struct icm_fr_event_device_disconnected *)hdr;
903 struct tb_switch *sw;
904 u8 link, depth;
905
906 link = pkg->link_info & ICM_LINK_INFO_LINK_MASK;
907 depth = (pkg->link_info & ICM_LINK_INFO_DEPTH_MASK) >>
908 ICM_LINK_INFO_DEPTH_SHIFT;
909
910 if (link > ICM_MAX_LINK || depth > TB_SWITCH_MAX_DEPTH) {
911 tb_warn(tb, "invalid topology %u.%u, ignoring\n", link, depth);
912 return;
913 }
914
915 sw = tb_switch_find_by_link_depth(tb, link, depth);
916 if (!sw) {
917 tb_warn(tb, "no switch exists at %u.%u, ignoring\n", link,
918 depth);
919 return;
920 }
921
922 pm_runtime_get_sync(sw->dev.parent);
923
924 remove_switch(sw);
925
926 pm_runtime_mark_last_busy(sw->dev.parent);
927 pm_runtime_put_autosuspend(sw->dev.parent);
928
929 tb_switch_put(sw);
930}
931
932static void
933icm_fr_xdomain_connected(struct tb *tb, const struct icm_pkg_header *hdr)
934{
935 const struct icm_fr_event_xdomain_connected *pkg =
936 (const struct icm_fr_event_xdomain_connected *)hdr;
937 struct tb_xdomain *xd;
938 struct tb_switch *sw;
939 u8 link, depth;
940 u64 route;
941
942 link = pkg->link_info & ICM_LINK_INFO_LINK_MASK;
943 depth = (pkg->link_info & ICM_LINK_INFO_DEPTH_MASK) >>
944 ICM_LINK_INFO_DEPTH_SHIFT;
945
946 if (link > ICM_MAX_LINK || depth > TB_SWITCH_MAX_DEPTH) {
947 tb_warn(tb, "invalid topology %u.%u, ignoring\n", link, depth);
948 return;
949 }
950
951 route = get_route(pkg->local_route_hi, pkg->local_route_lo);
952
953 xd = tb_xdomain_find_by_uuid(tb, &pkg->remote_uuid);
954 if (xd) {
955 u8 xd_phy_port, phy_port;
956
957 xd_phy_port = phy_port_from_route(xd->route, xd->depth);
958 phy_port = phy_port_from_route(route, depth);
959
960 if (xd->depth == depth && xd_phy_port == phy_port) {
961 update_xdomain(xd, route, link);
962 tb_xdomain_put(xd);
963 return;
964 }
965
966 /*
967 * If we find an existing XDomain connection remove it
968 * now. We need to go through login handshake and
969 * everything anyway to be able to re-establish the
970 * connection.
971 */
972 remove_xdomain(xd);
973 tb_xdomain_put(xd);
974 }
975
976 /*
977 * Look if there already exists an XDomain in the same place
978 * as the new one and in that case remove it because it is
979 * most likely another host that got disconnected.
980 */
981 xd = tb_xdomain_find_by_link_depth(tb, link, depth);
982 if (!xd) {
983 u8 dual_link;
984
985 dual_link = dual_link_from_link(link);
986 if (dual_link)
987 xd = tb_xdomain_find_by_link_depth(tb, dual_link,
988 depth);
989 }
990 if (xd) {
991 remove_xdomain(xd);
992 tb_xdomain_put(xd);
993 }
994
995 /*
996 * If the user disconnected a switch during suspend and
997 * connected another host to the same port, remove the switch
998 * first.
999 */
1000 sw = tb_switch_find_by_route(tb, route);
1001 if (sw) {
1002 remove_switch(sw);
1003 tb_switch_put(sw);
1004 }
1005
1006 sw = tb_switch_find_by_link_depth(tb, link, depth);
1007 if (!sw) {
1008 tb_warn(tb, "no switch exists at %u.%u, ignoring\n", link,
1009 depth);
1010 return;
1011 }
1012
1013 add_xdomain(sw, route, &pkg->local_uuid, &pkg->remote_uuid, link,
1014 depth);
1015 tb_switch_put(sw);
1016}
1017
1018static void
1019icm_fr_xdomain_disconnected(struct tb *tb, const struct icm_pkg_header *hdr)
1020{
1021 const struct icm_fr_event_xdomain_disconnected *pkg =
1022 (const struct icm_fr_event_xdomain_disconnected *)hdr;
1023 struct tb_xdomain *xd;
1024
1025 /*
1026 * If the connection is through one or multiple devices, the
1027 * XDomain device is removed along with them so it is fine if we
1028 * cannot find it here.
1029 */
1030 xd = tb_xdomain_find_by_uuid(tb, &pkg->remote_uuid);
1031 if (xd) {
1032 remove_xdomain(xd);
1033 tb_xdomain_put(xd);
1034 }
1035}
1036
1037static int icm_tr_cio_reset(struct tb *tb)
1038{
1039 return pcie2cio_write(tb_priv(tb), TB_CFG_SWITCH, 0, 0x777, BIT(1));
1040}
1041
1042static int
1043icm_tr_driver_ready(struct tb *tb, enum tb_security_level *security_level,
1044 u8 *proto_version, size_t *nboot_acl, bool *rpm)
1045{
1046 struct icm_tr_pkg_driver_ready_response reply;
1047 struct icm_pkg_driver_ready request = {
1048 .hdr.code = ICM_DRIVER_READY,
1049 };
1050 int ret;
1051
1052 memset(&reply, 0, sizeof(reply));
1053 ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),
1054 1, 10, 250);
1055 if (ret)
1056 return ret;
1057
1058 if (security_level)
1059 *security_level = reply.info & ICM_TR_INFO_SLEVEL_MASK;
1060 if (proto_version)
1061 *proto_version = (reply.info & ICM_TR_INFO_PROTO_VERSION_MASK) >>
1062 ICM_TR_INFO_PROTO_VERSION_SHIFT;
1063 if (nboot_acl)
1064 *nboot_acl = (reply.info & ICM_TR_INFO_BOOT_ACL_MASK) >>
1065 ICM_TR_INFO_BOOT_ACL_SHIFT;
1066 if (rpm)
1067 *rpm = !!(reply.hdr.flags & ICM_TR_FLAGS_RTD3);
1068
1069 return 0;
1070}
1071
1072static int icm_tr_approve_switch(struct tb *tb, struct tb_switch *sw)
1073{
1074 struct icm_tr_pkg_approve_device request;
1075 struct icm_tr_pkg_approve_device reply;
1076 int ret;
1077
1078 memset(&request, 0, sizeof(request));
1079 memcpy(&request.ep_uuid, sw->uuid, sizeof(request.ep_uuid));
1080 request.hdr.code = ICM_APPROVE_DEVICE;
1081 request.route_lo = sw->config.route_lo;
1082 request.route_hi = sw->config.route_hi;
1083 request.connection_id = sw->connection_id;
1084
1085 memset(&reply, 0, sizeof(reply));
1086 ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),
1087 1, ICM_RETRIES, ICM_APPROVE_TIMEOUT);
1088 if (ret)
1089 return ret;
1090
1091 if (reply.hdr.flags & ICM_FLAGS_ERROR) {
1092 tb_warn(tb, "PCIe tunnel creation failed\n");
1093 return -EIO;
1094 }
1095
1096 return 0;
1097}
1098
1099static int icm_tr_add_switch_key(struct tb *tb, struct tb_switch *sw)
1100{
1101 struct icm_tr_pkg_add_device_key_response reply;
1102 struct icm_tr_pkg_add_device_key request;
1103 int ret;
1104
1105 memset(&request, 0, sizeof(request));
1106 memcpy(&request.ep_uuid, sw->uuid, sizeof(request.ep_uuid));
1107 request.hdr.code = ICM_ADD_DEVICE_KEY;
1108 request.route_lo = sw->config.route_lo;
1109 request.route_hi = sw->config.route_hi;
1110 request.connection_id = sw->connection_id;
1111 memcpy(request.key, sw->key, TB_SWITCH_KEY_SIZE);
1112
1113 memset(&reply, 0, sizeof(reply));
1114 ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),
1115 1, ICM_RETRIES, ICM_TIMEOUT);
1116 if (ret)
1117 return ret;
1118
1119 if (reply.hdr.flags & ICM_FLAGS_ERROR) {
1120 tb_warn(tb, "Adding key to switch failed\n");
1121 return -EIO;
1122 }
1123
1124 return 0;
1125}
1126
1127static int icm_tr_challenge_switch_key(struct tb *tb, struct tb_switch *sw,
1128 const u8 *challenge, u8 *response)
1129{
1130 struct icm_tr_pkg_challenge_device_response reply;
1131 struct icm_tr_pkg_challenge_device request;
1132 int ret;
1133
1134 memset(&request, 0, sizeof(request));
1135 memcpy(&request.ep_uuid, sw->uuid, sizeof(request.ep_uuid));
1136 request.hdr.code = ICM_CHALLENGE_DEVICE;
1137 request.route_lo = sw->config.route_lo;
1138 request.route_hi = sw->config.route_hi;
1139 request.connection_id = sw->connection_id;
1140 memcpy(request.challenge, challenge, TB_SWITCH_KEY_SIZE);
1141
1142 memset(&reply, 0, sizeof(reply));
1143 ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),
1144 1, ICM_RETRIES, ICM_TIMEOUT);
1145 if (ret)
1146 return ret;
1147
1148 if (reply.hdr.flags & ICM_FLAGS_ERROR)
1149 return -EKEYREJECTED;
1150 if (reply.hdr.flags & ICM_FLAGS_NO_KEY)
1151 return -ENOKEY;
1152
1153 memcpy(response, reply.response, TB_SWITCH_KEY_SIZE);
1154
1155 return 0;
1156}
1157
1158static int icm_tr_approve_xdomain_paths(struct tb *tb, struct tb_xdomain *xd,
1159 int transmit_path, int transmit_ring,
1160 int receive_path, int receive_ring)
1161{
1162 struct icm_tr_pkg_approve_xdomain_response reply;
1163 struct icm_tr_pkg_approve_xdomain request;
1164 int ret;
1165
1166 if (atomic_read(&xd->ntunnels) >= 1) {
1167 tb_warn(tb, "only one tunnel is supported by the firmware\n");
1168 return -EOPNOTSUPP;
1169 }
1170
1171 memset(&request, 0, sizeof(request));
1172 request.hdr.code = ICM_APPROVE_XDOMAIN;
1173 request.route_hi = upper_32_bits(xd->route);
1174 request.route_lo = lower_32_bits(xd->route);
1175 request.transmit_path = transmit_path;
1176 request.transmit_ring = transmit_ring;
1177 request.receive_path = receive_path;
1178 request.receive_ring = receive_ring;
1179 memcpy(&request.remote_uuid, xd->remote_uuid, sizeof(*xd->remote_uuid));
1180
1181 memset(&reply, 0, sizeof(reply));
1182 ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),
1183 1, ICM_RETRIES, ICM_TIMEOUT);
1184 if (ret)
1185 return ret;
1186
1187 if (reply.hdr.flags & ICM_FLAGS_ERROR)
1188 return -EIO;
1189
1190 icm_xdomain_activated(xd, true);
1191 return 0;
1192}
1193
1194static int icm_tr_xdomain_tear_down(struct tb *tb, struct tb_xdomain *xd,
1195 int stage)
1196{
1197 struct icm_tr_pkg_disconnect_xdomain_response reply;
1198 struct icm_tr_pkg_disconnect_xdomain request;
1199 int ret;
1200
1201 memset(&request, 0, sizeof(request));
1202 request.hdr.code = ICM_DISCONNECT_XDOMAIN;
1203 request.stage = stage;
1204 request.route_hi = upper_32_bits(xd->route);
1205 request.route_lo = lower_32_bits(xd->route);
1206 memcpy(&request.remote_uuid, xd->remote_uuid, sizeof(*xd->remote_uuid));
1207
1208 memset(&reply, 0, sizeof(reply));
1209 ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),
1210 1, ICM_RETRIES, ICM_TIMEOUT);
1211 if (ret)
1212 return ret;
1213
1214 if (reply.hdr.flags & ICM_FLAGS_ERROR)
1215 return -EIO;
1216
1217 return 0;
1218}
1219
1220static int icm_tr_disconnect_xdomain_paths(struct tb *tb, struct tb_xdomain *xd,
1221 int transmit_path, int transmit_ring,
1222 int receive_path, int receive_ring)
1223{
1224 int ret;
1225
1226 ret = icm_tr_xdomain_tear_down(tb, xd, 1);
1227 if (ret)
1228 return ret;
1229
1230 usleep_range(10, 50);
1231 ret = icm_tr_xdomain_tear_down(tb, xd, 2);
1232 if (ret)
1233 return ret;
1234
1235 icm_xdomain_activated(xd, false);
1236 return 0;
1237}
1238
1239static void
1240__icm_tr_device_connected(struct tb *tb, const struct icm_pkg_header *hdr,
1241 bool force_rtd3)
1242{
1243 const struct icm_tr_event_device_connected *pkg =
1244 (const struct icm_tr_event_device_connected *)hdr;
1245 bool authorized, boot, dual_lane, speed_gen3;
1246 enum tb_security_level security_level;
1247 struct tb_switch *sw, *parent_sw;
1248 struct tb_xdomain *xd;
1249 u64 route;
1250
1251 icm_postpone_rescan(tb);
1252
1253 /*
1254 * Currently we don't use the QoS information coming with the
1255 * device connected message so simply just ignore that extra
1256 * packet for now.
1257 */
1258 if (pkg->hdr.packet_id)
1259 return;
1260
1261 route = get_route(pkg->route_hi, pkg->route_lo);
1262 authorized = pkg->link_info & ICM_LINK_INFO_APPROVED;
1263 security_level = (pkg->hdr.flags & ICM_FLAGS_SLEVEL_MASK) >>
1264 ICM_FLAGS_SLEVEL_SHIFT;
1265 boot = pkg->link_info & ICM_LINK_INFO_BOOT;
1266 dual_lane = pkg->hdr.flags & ICM_FLAGS_DUAL_LANE;
1267 speed_gen3 = pkg->hdr.flags & ICM_FLAGS_SPEED_GEN3;
1268
1269 if (pkg->link_info & ICM_LINK_INFO_REJECTED) {
1270 tb_info(tb, "switch at %llx was rejected by ICM firmware because topology limit exceeded\n",
1271 route);
1272 return;
1273 }
1274
1275 sw = tb_switch_find_by_uuid(tb, &pkg->ep_uuid);
1276 if (sw) {
1277 /* Update the switch if it is still in the same place */
1278 if (tb_route(sw) == route && !!sw->authorized == authorized) {
1279 update_switch(sw, route, pkg->connection_id, 0, 0, 0,
1280 boot);
1281 tb_switch_put(sw);
1282 return;
1283 }
1284
1285 remove_switch(sw);
1286 tb_switch_put(sw);
1287 }
1288
1289 /* Another switch with the same address */
1290 sw = tb_switch_find_by_route(tb, route);
1291 if (sw) {
1292 remove_switch(sw);
1293 tb_switch_put(sw);
1294 }
1295
1296 /* XDomain connection with the same address */
1297 xd = tb_xdomain_find_by_route(tb, route);
1298 if (xd) {
1299 remove_xdomain(xd);
1300 tb_xdomain_put(xd);
1301 }
1302
1303 parent_sw = tb_switch_find_by_route(tb, get_parent_route(route));
1304 if (!parent_sw) {
1305 tb_err(tb, "failed to find parent switch for %llx\n", route);
1306 return;
1307 }
1308
1309 pm_runtime_get_sync(&parent_sw->dev);
1310
1311 sw = alloc_switch(parent_sw, route, &pkg->ep_uuid);
1312 if (!IS_ERR(sw)) {
1313 sw->connection_id = pkg->connection_id;
1314 sw->authorized = authorized;
1315 sw->security_level = security_level;
1316 sw->boot = boot;
1317 sw->link_speed = speed_gen3 ? 20 : 10;
1318 sw->link_width = dual_lane ? TB_LINK_WIDTH_DUAL :
1319 TB_LINK_WIDTH_SINGLE;
1320 sw->rpm = force_rtd3;
1321 if (!sw->rpm)
1322 sw->rpm = intel_vss_is_rtd3(pkg->ep_name,
1323 sizeof(pkg->ep_name));
1324
1325 if (add_switch(parent_sw, sw))
1326 tb_switch_put(sw);
1327 }
1328
1329 pm_runtime_mark_last_busy(&parent_sw->dev);
1330 pm_runtime_put_autosuspend(&parent_sw->dev);
1331
1332 tb_switch_put(parent_sw);
1333}
1334
1335static void
1336icm_tr_device_connected(struct tb *tb, const struct icm_pkg_header *hdr)
1337{
1338 __icm_tr_device_connected(tb, hdr, false);
1339}
1340
1341static void
1342icm_tr_device_disconnected(struct tb *tb, const struct icm_pkg_header *hdr)
1343{
1344 const struct icm_tr_event_device_disconnected *pkg =
1345 (const struct icm_tr_event_device_disconnected *)hdr;
1346 struct tb_switch *sw;
1347 u64 route;
1348
1349 route = get_route(pkg->route_hi, pkg->route_lo);
1350
1351 sw = tb_switch_find_by_route(tb, route);
1352 if (!sw) {
1353 tb_warn(tb, "no switch exists at %llx, ignoring\n", route);
1354 return;
1355 }
1356 pm_runtime_get_sync(sw->dev.parent);
1357
1358 remove_switch(sw);
1359
1360 pm_runtime_mark_last_busy(sw->dev.parent);
1361 pm_runtime_put_autosuspend(sw->dev.parent);
1362
1363 tb_switch_put(sw);
1364}
1365
1366static void
1367icm_tr_xdomain_connected(struct tb *tb, const struct icm_pkg_header *hdr)
1368{
1369 const struct icm_tr_event_xdomain_connected *pkg =
1370 (const struct icm_tr_event_xdomain_connected *)hdr;
1371 struct tb_xdomain *xd;
1372 struct tb_switch *sw;
1373 u64 route;
1374
1375 if (!tb->root_switch)
1376 return;
1377
1378 route = get_route(pkg->local_route_hi, pkg->local_route_lo);
1379
1380 xd = tb_xdomain_find_by_uuid(tb, &pkg->remote_uuid);
1381 if (xd) {
1382 if (xd->route == route) {
1383 update_xdomain(xd, route, 0);
1384 tb_xdomain_put(xd);
1385 return;
1386 }
1387
1388 remove_xdomain(xd);
1389 tb_xdomain_put(xd);
1390 }
1391
1392 /* An existing xdomain with the same address */
1393 xd = tb_xdomain_find_by_route(tb, route);
1394 if (xd) {
1395 remove_xdomain(xd);
1396 tb_xdomain_put(xd);
1397 }
1398
1399 /*
1400 * If the user disconnected a switch during suspend and
1401 * connected another host to the same port, remove the switch
1402 * first.
1403 */
1404 sw = tb_switch_find_by_route(tb, route);
1405 if (sw) {
1406 remove_switch(sw);
1407 tb_switch_put(sw);
1408 }
1409
1410 sw = tb_switch_find_by_route(tb, get_parent_route(route));
1411 if (!sw) {
1412 tb_warn(tb, "no switch exists at %llx, ignoring\n", route);
1413 return;
1414 }
1415
1416 add_xdomain(sw, route, &pkg->local_uuid, &pkg->remote_uuid, 0, 0);
1417 tb_switch_put(sw);
1418}
1419
1420static void
1421icm_tr_xdomain_disconnected(struct tb *tb, const struct icm_pkg_header *hdr)
1422{
1423 const struct icm_tr_event_xdomain_disconnected *pkg =
1424 (const struct icm_tr_event_xdomain_disconnected *)hdr;
1425 struct tb_xdomain *xd;
1426 u64 route;
1427
1428 route = get_route(pkg->route_hi, pkg->route_lo);
1429
1430 xd = tb_xdomain_find_by_route(tb, route);
1431 if (xd) {
1432 remove_xdomain(xd);
1433 tb_xdomain_put(xd);
1434 }
1435}
1436
1437static struct pci_dev *get_upstream_port(struct pci_dev *pdev)
1438{
1439 struct pci_dev *parent;
1440
1441 parent = pci_upstream_bridge(pdev);
1442 while (parent) {
1443 if (!pci_is_pcie(parent))
1444 return NULL;
1445 if (pci_pcie_type(parent) == PCI_EXP_TYPE_UPSTREAM)
1446 break;
1447 parent = pci_upstream_bridge(parent);
1448 }
1449
1450 if (!parent)
1451 return NULL;
1452
1453 switch (parent->device) {
1454 case PCI_DEVICE_ID_INTEL_ALPINE_RIDGE_2C_BRIDGE:
1455 case PCI_DEVICE_ID_INTEL_ALPINE_RIDGE_4C_BRIDGE:
1456 case PCI_DEVICE_ID_INTEL_ALPINE_RIDGE_LP_BRIDGE:
1457 case PCI_DEVICE_ID_INTEL_ALPINE_RIDGE_C_4C_BRIDGE:
1458 case PCI_DEVICE_ID_INTEL_ALPINE_RIDGE_C_2C_BRIDGE:
1459 case PCI_DEVICE_ID_INTEL_TITAN_RIDGE_2C_BRIDGE:
1460 case PCI_DEVICE_ID_INTEL_TITAN_RIDGE_4C_BRIDGE:
1461 return parent;
1462 }
1463
1464 return NULL;
1465}
1466
1467static bool icm_ar_is_supported(struct tb *tb)
1468{
1469 struct pci_dev *pdev = to_pci_dev(tb->nhi->dev);
1470 struct pci_dev *upstream_port;
1471 struct icm *icm = tb_priv(tb);
1472
1473 /*
1474 * Starting from Alpine Ridge we can use ICM on Apple machines
1475 * as well. We just need to reset and re-enable it first.
1476 * However, only start it if explicitly asked by the user.
1477 */
1478 if (icm_firmware_running(tb->nhi))
1479 return true;
1480 if (!start_icm)
1481 return false;
1482
1483 /*
1484 * Find the upstream PCIe port in case we need to do reset
1485 * through its vendor specific registers.
1486 */
1487 upstream_port = get_upstream_port(pdev);
1488 if (upstream_port) {
1489 int cap;
1490
1491 cap = pci_find_ext_capability(upstream_port,
1492 PCI_EXT_CAP_ID_VNDR);
1493 if (cap > 0) {
1494 icm->upstream_port = upstream_port;
1495 icm->vnd_cap = cap;
1496
1497 return true;
1498 }
1499 }
1500
1501 return false;
1502}
1503
1504static int icm_ar_cio_reset(struct tb *tb)
1505{
1506 return pcie2cio_write(tb_priv(tb), TB_CFG_SWITCH, 0, 0x50, BIT(9));
1507}
1508
1509static int icm_ar_get_mode(struct tb *tb)
1510{
1511 struct tb_nhi *nhi = tb->nhi;
1512 int retries = 60;
1513 u32 val;
1514
1515 do {
1516 val = ioread32(nhi->iobase + REG_FW_STS);
1517 if (val & REG_FW_STS_NVM_AUTH_DONE)
1518 break;
1519 msleep(50);
1520 } while (--retries);
1521
1522 if (!retries) {
1523 dev_err(nhi->dev, "ICM firmware not authenticated\n");
1524 return -ENODEV;
1525 }
1526
1527 return nhi_mailbox_mode(nhi);
1528}
1529
1530static int
1531icm_ar_driver_ready(struct tb *tb, enum tb_security_level *security_level,
1532 u8 *proto_version, size_t *nboot_acl, bool *rpm)
1533{
1534 struct icm_ar_pkg_driver_ready_response reply;
1535 struct icm_pkg_driver_ready request = {
1536 .hdr.code = ICM_DRIVER_READY,
1537 };
1538 int ret;
1539
1540 memset(&reply, 0, sizeof(reply));
1541 ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),
1542 1, ICM_RETRIES, ICM_TIMEOUT);
1543 if (ret)
1544 return ret;
1545
1546 if (security_level)
1547 *security_level = reply.info & ICM_AR_INFO_SLEVEL_MASK;
1548 if (nboot_acl && (reply.info & ICM_AR_INFO_BOOT_ACL_SUPPORTED))
1549 *nboot_acl = (reply.info & ICM_AR_INFO_BOOT_ACL_MASK) >>
1550 ICM_AR_INFO_BOOT_ACL_SHIFT;
1551 if (rpm)
1552 *rpm = !!(reply.hdr.flags & ICM_AR_FLAGS_RTD3);
1553
1554 return 0;
1555}
1556
1557static int icm_ar_get_route(struct tb *tb, u8 link, u8 depth, u64 *route)
1558{
1559 struct icm_ar_pkg_get_route_response reply;
1560 struct icm_ar_pkg_get_route request = {
1561 .hdr = { .code = ICM_GET_ROUTE },
1562 .link_info = depth << ICM_LINK_INFO_DEPTH_SHIFT | link,
1563 };
1564 int ret;
1565
1566 memset(&reply, 0, sizeof(reply));
1567 ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),
1568 1, ICM_RETRIES, ICM_TIMEOUT);
1569 if (ret)
1570 return ret;
1571
1572 if (reply.hdr.flags & ICM_FLAGS_ERROR)
1573 return -EIO;
1574
1575 *route = get_route(reply.route_hi, reply.route_lo);
1576 return 0;
1577}
1578
1579static int icm_ar_get_boot_acl(struct tb *tb, uuid_t *uuids, size_t nuuids)
1580{
1581 struct icm_ar_pkg_preboot_acl_response reply;
1582 struct icm_ar_pkg_preboot_acl request = {
1583 .hdr = { .code = ICM_PREBOOT_ACL },
1584 };
1585 int ret, i;
1586
1587 memset(&reply, 0, sizeof(reply));
1588 ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),
1589 1, ICM_RETRIES, ICM_TIMEOUT);
1590 if (ret)
1591 return ret;
1592
1593 if (reply.hdr.flags & ICM_FLAGS_ERROR)
1594 return -EIO;
1595
1596 for (i = 0; i < nuuids; i++) {
1597 u32 *uuid = (u32 *)&uuids[i];
1598
1599 uuid[0] = reply.acl[i].uuid_lo;
1600 uuid[1] = reply.acl[i].uuid_hi;
1601
1602 if (uuid[0] == 0xffffffff && uuid[1] == 0xffffffff) {
1603 /* Map empty entries to null UUID */
1604 uuid[0] = 0;
1605 uuid[1] = 0;
1606 } else if (uuid[0] != 0 || uuid[1] != 0) {
1607 /* Upper two DWs are always one's */
1608 uuid[2] = 0xffffffff;
1609 uuid[3] = 0xffffffff;
1610 }
1611 }
1612
1613 return ret;
1614}
1615
1616static int icm_ar_set_boot_acl(struct tb *tb, const uuid_t *uuids,
1617 size_t nuuids)
1618{
1619 struct icm_ar_pkg_preboot_acl_response reply;
1620 struct icm_ar_pkg_preboot_acl request = {
1621 .hdr = {
1622 .code = ICM_PREBOOT_ACL,
1623 .flags = ICM_FLAGS_WRITE,
1624 },
1625 };
1626 int ret, i;
1627
1628 for (i = 0; i < nuuids; i++) {
1629 const u32 *uuid = (const u32 *)&uuids[i];
1630
1631 if (uuid_is_null(&uuids[i])) {
1632 /*
1633 * Map null UUID to the empty (all one) entries
1634 * for ICM.
1635 */
1636 request.acl[i].uuid_lo = 0xffffffff;
1637 request.acl[i].uuid_hi = 0xffffffff;
1638 } else {
1639 /* Two high DWs need to be set to all one */
1640 if (uuid[2] != 0xffffffff || uuid[3] != 0xffffffff)
1641 return -EINVAL;
1642
1643 request.acl[i].uuid_lo = uuid[0];
1644 request.acl[i].uuid_hi = uuid[1];
1645 }
1646 }
1647
1648 memset(&reply, 0, sizeof(reply));
1649 ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),
1650 1, ICM_RETRIES, ICM_TIMEOUT);
1651 if (ret)
1652 return ret;
1653
1654 if (reply.hdr.flags & ICM_FLAGS_ERROR)
1655 return -EIO;
1656
1657 return 0;
1658}
1659
1660static int
1661icm_icl_driver_ready(struct tb *tb, enum tb_security_level *security_level,
1662 u8 *proto_version, size_t *nboot_acl, bool *rpm)
1663{
1664 struct icm_tr_pkg_driver_ready_response reply;
1665 struct icm_pkg_driver_ready request = {
1666 .hdr.code = ICM_DRIVER_READY,
1667 };
1668 int ret;
1669
1670 memset(&reply, 0, sizeof(reply));
1671 ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),
1672 1, ICM_RETRIES, 20000);
1673 if (ret)
1674 return ret;
1675
1676 if (proto_version)
1677 *proto_version = (reply.info & ICM_TR_INFO_PROTO_VERSION_MASK) >>
1678 ICM_TR_INFO_PROTO_VERSION_SHIFT;
1679
1680 /* Ice Lake always supports RTD3 */
1681 if (rpm)
1682 *rpm = true;
1683
1684 return 0;
1685}
1686
1687static void icm_icl_set_uuid(struct tb *tb)
1688{
1689 struct pci_dev *pdev = to_pci_dev(tb->nhi->dev);
1690 u32 uuid[4];
1691
1692 pci_read_config_dword(pdev, VS_CAP_10, &uuid[0]);
1693 pci_read_config_dword(pdev, VS_CAP_11, &uuid[1]);
1694 uuid[2] = 0xffffffff;
1695 uuid[3] = 0xffffffff;
1696
1697 tb->root_switch->uuid = kmemdup(uuid, sizeof(uuid), GFP_KERNEL);
1698}
1699
1700static void
1701icm_icl_device_connected(struct tb *tb, const struct icm_pkg_header *hdr)
1702{
1703 __icm_tr_device_connected(tb, hdr, true);
1704}
1705
1706static void icm_icl_rtd3_veto(struct tb *tb, const struct icm_pkg_header *hdr)
1707{
1708 const struct icm_icl_event_rtd3_veto *pkg =
1709 (const struct icm_icl_event_rtd3_veto *)hdr;
1710
1711 tb_dbg(tb, "ICM rtd3 veto=0x%08x\n", pkg->veto_reason);
1712
1713 if (pkg->veto_reason)
1714 icm_veto_begin(tb);
1715 else
1716 icm_veto_end(tb);
1717}
1718
1719static bool icm_tgl_is_supported(struct tb *tb)
1720{
1721 unsigned long end = jiffies + msecs_to_jiffies(10);
1722
1723 do {
1724 u32 val;
1725
1726 val = ioread32(tb->nhi->iobase + REG_FW_STS);
1727 if (val & REG_FW_STS_NVM_AUTH_DONE)
1728 return true;
1729 usleep_range(100, 500);
1730 } while (time_before(jiffies, end));
1731
1732 return false;
1733}
1734
1735static void icm_handle_notification(struct work_struct *work)
1736{
1737 struct icm_notification *n = container_of(work, typeof(*n), work);
1738 struct tb *tb = n->tb;
1739 struct icm *icm = tb_priv(tb);
1740
1741 mutex_lock(&tb->lock);
1742
1743 /*
1744 * When the domain is stopped we flush its workqueue but before
1745 * that the root switch is removed. In that case we should treat
1746 * the queued events as being canceled.
1747 */
1748 if (tb->root_switch) {
1749 switch (n->pkg->code) {
1750 case ICM_EVENT_DEVICE_CONNECTED:
1751 icm->device_connected(tb, n->pkg);
1752 break;
1753 case ICM_EVENT_DEVICE_DISCONNECTED:
1754 icm->device_disconnected(tb, n->pkg);
1755 break;
1756 case ICM_EVENT_XDOMAIN_CONNECTED:
1757 if (tb_is_xdomain_enabled())
1758 icm->xdomain_connected(tb, n->pkg);
1759 break;
1760 case ICM_EVENT_XDOMAIN_DISCONNECTED:
1761 if (tb_is_xdomain_enabled())
1762 icm->xdomain_disconnected(tb, n->pkg);
1763 break;
1764 case ICM_EVENT_DP_CONFIG_CHANGED:
1765 icm_dp_event(tb);
1766 break;
1767 case ICM_EVENT_RTD3_VETO:
1768 icm->rtd3_veto(tb, n->pkg);
1769 break;
1770 }
1771 }
1772
1773 mutex_unlock(&tb->lock);
1774
1775 kfree(n->pkg);
1776 kfree(n);
1777
1778 tb_domain_unregister_unplugged_xdomains(tb);
1779}
1780
1781static void icm_handle_event(struct tb *tb, enum tb_cfg_pkg_type type,
1782 const void *buf, size_t size)
1783{
1784 struct icm_notification *n;
1785
1786 n = kmalloc_obj(*n);
1787 if (!n)
1788 return;
1789
1790 n->pkg = kmemdup(buf, size, GFP_KERNEL);
1791 if (!n->pkg) {
1792 kfree(n);
1793 return;
1794 }
1795
1796 INIT_WORK(&n->work, icm_handle_notification);
1797 n->tb = tb;
1798
1799 queue_work(tb->wq, &n->work);
1800}
1801
1802static int
1803__icm_driver_ready(struct tb *tb, enum tb_security_level *security_level,
1804 u8 *proto_version, size_t *nboot_acl, bool *rpm)
1805{
1806 struct icm *icm = tb_priv(tb);
1807 unsigned int retries = 50;
1808 int ret;
1809
1810 ret = icm->driver_ready(tb, security_level, proto_version, nboot_acl,
1811 rpm);
1812 if (ret) {
1813 tb_err(tb, "failed to send driver ready to ICM\n");
1814 return ret;
1815 }
1816
1817 /*
1818 * Hold on here until the switch config space is accessible so
1819 * that we can read root switch config successfully.
1820 */
1821 do {
1822 struct tb_cfg_result res;
1823 u32 tmp;
1824
1825 res = tb_cfg_read_raw(tb->ctl, &tmp, 0, 0, TB_CFG_SWITCH,
1826 0, 1, 100);
1827 if (!res.err)
1828 return 0;
1829
1830 msleep(50);
1831 } while (--retries);
1832
1833 tb_err(tb, "failed to read root switch config space, giving up\n");
1834 return -ETIMEDOUT;
1835}
1836
1837static int icm_firmware_reset(struct tb *tb, struct tb_nhi *nhi)
1838{
1839 struct icm *icm = tb_priv(tb);
1840 u32 val;
1841
1842 if (!icm->upstream_port)
1843 return -ENODEV;
1844
1845 /* Put ARC to wait for CIO reset event to happen */
1846 val = ioread32(nhi->iobase + REG_FW_STS);
1847 val |= REG_FW_STS_CIO_RESET_REQ;
1848 iowrite32(val, nhi->iobase + REG_FW_STS);
1849
1850 /* Re-start ARC */
1851 val = ioread32(nhi->iobase + REG_FW_STS);
1852 val |= REG_FW_STS_ICM_EN_INVERT;
1853 val |= REG_FW_STS_ICM_EN_CPU;
1854 iowrite32(val, nhi->iobase + REG_FW_STS);
1855
1856 /* Trigger CIO reset now */
1857 return icm->cio_reset(tb);
1858}
1859
1860static int icm_firmware_start(struct tb *tb, struct tb_nhi *nhi)
1861{
1862 unsigned int retries = 10;
1863 int ret;
1864 u32 val;
1865
1866 /* Check if the ICM firmware is already running */
1867 if (icm_firmware_running(nhi))
1868 return 0;
1869
1870 dev_dbg(nhi->dev, "starting ICM firmware\n");
1871
1872 ret = icm_firmware_reset(tb, nhi);
1873 if (ret)
1874 return ret;
1875
1876 /* Wait until the ICM firmware tells us it is up and running */
1877 do {
1878 /* Check that the ICM firmware is running */
1879 val = ioread32(nhi->iobase + REG_FW_STS);
1880 if (val & REG_FW_STS_NVM_AUTH_DONE)
1881 return 0;
1882
1883 msleep(300);
1884 } while (--retries);
1885
1886 return -ETIMEDOUT;
1887}
1888
1889static int icm_reset_phy_port(struct tb *tb, int phy_port)
1890{
1891 struct icm *icm = tb_priv(tb);
1892 u32 state0, state1;
1893 int port0, port1;
1894 u32 val0, val1;
1895 int ret;
1896
1897 if (!icm->upstream_port)
1898 return 0;
1899
1900 if (phy_port) {
1901 port0 = 3;
1902 port1 = 4;
1903 } else {
1904 port0 = 1;
1905 port1 = 2;
1906 }
1907
1908 /*
1909 * Read link status of both null ports belonging to a single
1910 * physical port.
1911 */
1912 ret = pcie2cio_read(icm, TB_CFG_PORT, port0, PHY_PORT_CS1, &val0);
1913 if (ret)
1914 return ret;
1915 ret = pcie2cio_read(icm, TB_CFG_PORT, port1, PHY_PORT_CS1, &val1);
1916 if (ret)
1917 return ret;
1918
1919 state0 = val0 & PHY_PORT_CS1_LINK_STATE_MASK;
1920 state0 >>= PHY_PORT_CS1_LINK_STATE_SHIFT;
1921 state1 = val1 & PHY_PORT_CS1_LINK_STATE_MASK;
1922 state1 >>= PHY_PORT_CS1_LINK_STATE_SHIFT;
1923
1924 /* If they are both up we need to reset them now */
1925 if (state0 != TB_PORT_UP || state1 != TB_PORT_UP)
1926 return 0;
1927
1928 val0 |= PHY_PORT_CS1_LINK_DISABLE;
1929 ret = pcie2cio_write(icm, TB_CFG_PORT, port0, PHY_PORT_CS1, val0);
1930 if (ret)
1931 return ret;
1932
1933 val1 |= PHY_PORT_CS1_LINK_DISABLE;
1934 ret = pcie2cio_write(icm, TB_CFG_PORT, port1, PHY_PORT_CS1, val1);
1935 if (ret)
1936 return ret;
1937
1938 /* Wait a bit and then re-enable both ports */
1939 usleep_range(10, 100);
1940
1941 ret = pcie2cio_read(icm, TB_CFG_PORT, port0, PHY_PORT_CS1, &val0);
1942 if (ret)
1943 return ret;
1944 ret = pcie2cio_read(icm, TB_CFG_PORT, port1, PHY_PORT_CS1, &val1);
1945 if (ret)
1946 return ret;
1947
1948 val0 &= ~PHY_PORT_CS1_LINK_DISABLE;
1949 ret = pcie2cio_write(icm, TB_CFG_PORT, port0, PHY_PORT_CS1, val0);
1950 if (ret)
1951 return ret;
1952
1953 val1 &= ~PHY_PORT_CS1_LINK_DISABLE;
1954 return pcie2cio_write(icm, TB_CFG_PORT, port1, PHY_PORT_CS1, val1);
1955}
1956
1957static int icm_firmware_init(struct tb *tb)
1958{
1959 struct icm *icm = tb_priv(tb);
1960 struct tb_nhi *nhi = tb->nhi;
1961 int ret;
1962
1963 ret = icm_firmware_start(tb, nhi);
1964 if (ret) {
1965 dev_err(nhi->dev, "could not start ICM firmware\n");
1966 return ret;
1967 }
1968
1969 if (icm->get_mode) {
1970 ret = icm->get_mode(tb);
1971
1972 switch (ret) {
1973 case NHI_FW_SAFE_MODE:
1974 icm->safe_mode = true;
1975 break;
1976
1977 case NHI_FW_CM_MODE:
1978 /* Ask ICM to accept all Thunderbolt devices */
1979 nhi_mailbox_cmd(nhi, NHI_MAILBOX_ALLOW_ALL_DEVS, 0);
1980 break;
1981
1982 default:
1983 if (ret < 0)
1984 return ret;
1985
1986 tb_err(tb, "ICM firmware is in wrong mode: %u\n", ret);
1987 return -ENODEV;
1988 }
1989 }
1990
1991 /*
1992 * Reset both physical ports if there is anything connected to
1993 * them already.
1994 */
1995 ret = icm_reset_phy_port(tb, 0);
1996 if (ret)
1997 dev_warn(nhi->dev, "failed to reset links on port0\n");
1998 ret = icm_reset_phy_port(tb, 1);
1999 if (ret)
2000 dev_warn(nhi->dev, "failed to reset links on port1\n");
2001
2002 return 0;
2003}
2004
2005static int icm_driver_ready(struct tb *tb)
2006{
2007 struct icm *icm = tb_priv(tb);
2008 int ret;
2009
2010 ret = icm_firmware_init(tb);
2011 if (ret)
2012 return ret;
2013
2014 if (icm->safe_mode) {
2015 tb_info(tb, "Thunderbolt host controller is in safe mode.\n");
2016 tb_info(tb, "You need to update NVM firmware of the controller before it can be used.\n");
2017 tb_info(tb, "Use fwupd tool to apply update. Check Documentation/admin-guide/thunderbolt.rst for details.\n");
2018 return 0;
2019 }
2020
2021 ret = __icm_driver_ready(tb, &tb->security_level, &icm->proto_version,
2022 &tb->nboot_acl, &icm->rpm);
2023 if (ret)
2024 return ret;
2025
2026 /*
2027 * Make sure the number of supported preboot ACL matches what we
2028 * expect or disable the whole feature.
2029 */
2030 if (tb->nboot_acl > icm->max_boot_acl)
2031 tb->nboot_acl = 0;
2032
2033 if (icm->proto_version >= 3)
2034 tb_dbg(tb, "USB4 proxy operations supported\n");
2035
2036 return 0;
2037}
2038
2039static int icm_suspend(struct tb *tb)
2040{
2041 struct icm *icm = tb_priv(tb);
2042
2043 if (icm->save_devices)
2044 icm->save_devices(tb);
2045
2046 nhi_mailbox_cmd(tb->nhi, NHI_MAILBOX_DRV_UNLOADS, 0);
2047 return 0;
2048}
2049
2050/*
2051 * Mark all switches (except root switch) below this one unplugged. ICM
2052 * firmware will send us an updated list of switches after we have send
2053 * it driver ready command. If a switch is not in that list it will be
2054 * removed when we perform rescan.
2055 */
2056static void icm_unplug_children(struct tb_switch *sw)
2057{
2058 struct tb_port *port;
2059
2060 if (tb_route(sw))
2061 sw->is_unplugged = true;
2062
2063 tb_switch_for_each_port(sw, port) {
2064 if (port->xdomain)
2065 port->xdomain->is_unplugged = true;
2066 else if (tb_port_has_remote(port))
2067 icm_unplug_children(port->remote->sw);
2068 }
2069}
2070
2071static int complete_rpm(struct device *dev, void *data)
2072{
2073 struct tb_switch *sw = tb_to_switch(dev);
2074
2075 if (sw)
2076 complete(&sw->rpm_complete);
2077 return 0;
2078}
2079
2080static void remove_unplugged_switch(struct tb_switch *sw)
2081{
2082 struct device *parent = get_device(sw->dev.parent);
2083
2084 pm_runtime_get_sync(parent);
2085
2086 /*
2087 * Signal this and switches below for rpm_complete because
2088 * tb_switch_remove() calls pm_runtime_get_sync() that then waits
2089 * for it.
2090 */
2091 complete_rpm(&sw->dev, NULL);
2092 bus_for_each_dev(&tb_bus_type, &sw->dev, NULL, complete_rpm);
2093 tb_switch_remove(sw);
2094
2095 pm_runtime_mark_last_busy(parent);
2096 pm_runtime_put_autosuspend(parent);
2097
2098 put_device(parent);
2099}
2100
2101static void icm_free_unplugged_children(struct tb_switch *sw)
2102{
2103 struct tb_port *port;
2104
2105 tb_switch_for_each_port(sw, port) {
2106 if (port->xdomain && port->xdomain->is_unplugged) {
2107 tb_xdomain_remove(port->xdomain);
2108 port->xdomain = NULL;
2109 } else if (tb_port_has_remote(port)) {
2110 if (port->remote->sw->is_unplugged) {
2111 remove_unplugged_switch(port->remote->sw);
2112 port->remote = NULL;
2113 } else {
2114 icm_free_unplugged_children(port->remote->sw);
2115 }
2116 }
2117 }
2118}
2119
2120static void icm_rescan_work(struct work_struct *work)
2121{
2122 struct icm *icm = container_of(work, struct icm, rescan_work.work);
2123 struct tb *tb = icm_to_tb(icm);
2124
2125 mutex_lock(&tb->lock);
2126 if (tb->root_switch)
2127 icm_free_unplugged_children(tb->root_switch);
2128 mutex_unlock(&tb->lock);
2129
2130 tb_domain_unregister_unplugged_xdomains(tb);
2131}
2132
2133static void icm_complete(struct tb *tb)
2134{
2135 struct icm *icm = tb_priv(tb);
2136
2137 if (tb->nhi->going_away)
2138 return;
2139
2140 /*
2141 * If RTD3 was vetoed before we entered system suspend allow it
2142 * again now before driver ready is sent. Firmware sends a new RTD3
2143 * veto if it is still the case after we have sent it driver ready
2144 * command.
2145 */
2146 icm_veto_end(tb);
2147 icm_unplug_children(tb->root_switch);
2148
2149 /*
2150 * Now all existing children should be resumed, start events
2151 * from ICM to get updated status.
2152 */
2153 __icm_driver_ready(tb, NULL, NULL, NULL, NULL);
2154
2155 /*
2156 * We do not get notifications of devices that have been
2157 * unplugged during suspend so schedule rescan to clean them up
2158 * if any.
2159 */
2160 queue_delayed_work(tb->wq, &icm->rescan_work, msecs_to_jiffies(500));
2161}
2162
2163static int icm_runtime_suspend(struct tb *tb)
2164{
2165 nhi_mailbox_cmd(tb->nhi, NHI_MAILBOX_DRV_UNLOADS, 0);
2166 return 0;
2167}
2168
2169static int icm_runtime_suspend_switch(struct tb_switch *sw)
2170{
2171 if (tb_route(sw))
2172 reinit_completion(&sw->rpm_complete);
2173 return 0;
2174}
2175
2176static int icm_runtime_resume_switch(struct tb_switch *sw)
2177{
2178 if (tb_route(sw)) {
2179 if (!wait_for_completion_timeout(&sw->rpm_complete,
2180 msecs_to_jiffies(500))) {
2181 dev_dbg(&sw->dev, "runtime resuming timed out\n");
2182 }
2183 }
2184 return 0;
2185}
2186
2187static int icm_runtime_resume(struct tb *tb)
2188{
2189 /*
2190 * We can reuse the same resume functionality as with system
2191 * suspend.
2192 */
2193 icm_complete(tb);
2194 return 0;
2195}
2196
2197static int icm_start(struct tb *tb, bool not_used)
2198{
2199 struct icm *icm = tb_priv(tb);
2200 int ret;
2201
2202 if (icm->safe_mode)
2203 tb->root_switch = tb_switch_alloc_safe_mode(tb, &tb->dev, 0);
2204 else
2205 tb->root_switch = tb_switch_alloc(tb, &tb->dev, 0);
2206 if (IS_ERR(tb->root_switch))
2207 return PTR_ERR(tb->root_switch);
2208
2209 tb->root_switch->no_nvm_upgrade = !icm->can_upgrade_nvm;
2210 tb->root_switch->rpm = icm->rpm;
2211
2212 if (icm->set_uuid)
2213 icm->set_uuid(tb);
2214
2215 ret = tb_switch_add(tb->root_switch);
2216 if (ret) {
2217 tb_switch_put(tb->root_switch);
2218 tb->root_switch = NULL;
2219 }
2220
2221 return ret;
2222}
2223
2224static void icm_stop(struct tb *tb)
2225{
2226 struct icm *icm = tb_priv(tb);
2227
2228 cancel_delayed_work(&icm->rescan_work);
2229 tb_switch_remove(tb->root_switch);
2230 tb->root_switch = NULL;
2231 nhi_mailbox_cmd(tb->nhi, NHI_MAILBOX_DRV_UNLOADS, 0);
2232 kfree(icm->last_nvm_auth);
2233 icm->last_nvm_auth = NULL;
2234}
2235
2236static int icm_disconnect_pcie_paths(struct tb *tb)
2237{
2238 return nhi_mailbox_cmd(tb->nhi, NHI_MAILBOX_DISCONNECT_PCIE_PATHS, 0);
2239}
2240
2241static void icm_usb4_switch_nvm_auth_complete(void *data)
2242{
2243 struct usb4_switch_nvm_auth *auth = data;
2244 struct icm *icm = auth->icm;
2245 struct tb *tb = icm_to_tb(icm);
2246
2247 tb_dbg(tb, "NVM_AUTH response for %llx flags %#x status %#x\n",
2248 get_route(auth->reply.route_hi, auth->reply.route_lo),
2249 auth->reply.hdr.flags, auth->reply.status);
2250
2251 mutex_lock(&tb->lock);
2252 if (WARN_ON(icm->last_nvm_auth))
2253 kfree(icm->last_nvm_auth);
2254 icm->last_nvm_auth = auth;
2255 mutex_unlock(&tb->lock);
2256}
2257
2258static int icm_usb4_switch_nvm_authenticate(struct tb *tb, u64 route)
2259{
2260 struct usb4_switch_nvm_auth *auth;
2261 struct icm *icm = tb_priv(tb);
2262 struct tb_cfg_request *req;
2263 int ret;
2264
2265 auth = kzalloc_obj(*auth);
2266 if (!auth)
2267 return -ENOMEM;
2268
2269 auth->icm = icm;
2270 auth->request.hdr.code = ICM_USB4_SWITCH_OP;
2271 auth->request.route_hi = upper_32_bits(route);
2272 auth->request.route_lo = lower_32_bits(route);
2273 auth->request.opcode = USB4_SWITCH_OP_NVM_AUTH;
2274
2275 req = tb_cfg_request_alloc();
2276 if (!req) {
2277 ret = -ENOMEM;
2278 goto err_free_auth;
2279 }
2280
2281 req->match = icm_match;
2282 req->copy = icm_copy;
2283 req->request = &auth->request;
2284 req->request_size = sizeof(auth->request);
2285 req->request_type = TB_CFG_PKG_ICM_CMD;
2286 req->response = &auth->reply;
2287 req->npackets = 1;
2288 req->response_size = sizeof(auth->reply);
2289 req->response_type = TB_CFG_PKG_ICM_RESP;
2290
2291 tb_dbg(tb, "NVM_AUTH request for %llx\n", route);
2292
2293 mutex_lock(&icm->request_lock);
2294 ret = tb_cfg_request(tb->ctl, req, icm_usb4_switch_nvm_auth_complete,
2295 auth);
2296 mutex_unlock(&icm->request_lock);
2297
2298 tb_cfg_request_put(req);
2299 if (ret)
2300 goto err_free_auth;
2301 return 0;
2302
2303err_free_auth:
2304 kfree(auth);
2305 return ret;
2306}
2307
2308static int icm_usb4_switch_op(struct tb_switch *sw, u16 opcode, u32 *metadata,
2309 u8 *status, const void *tx_data, size_t tx_data_len,
2310 void *rx_data, size_t rx_data_len)
2311{
2312 struct icm_usb4_switch_op_response reply;
2313 struct icm_usb4_switch_op request;
2314 struct tb *tb = sw->tb;
2315 struct icm *icm = tb_priv(tb);
2316 u64 route = tb_route(sw);
2317 int ret;
2318
2319 /*
2320 * USB4 router operation proxy is supported in firmware if the
2321 * protocol version is 3 or higher.
2322 */
2323 if (icm->proto_version < 3)
2324 return -EOPNOTSUPP;
2325
2326 /*
2327 * NVM_AUTH is a special USB4 proxy operation that does not
2328 * return immediately so handle it separately.
2329 */
2330 if (opcode == USB4_SWITCH_OP_NVM_AUTH)
2331 return icm_usb4_switch_nvm_authenticate(tb, route);
2332
2333 memset(&request, 0, sizeof(request));
2334 request.hdr.code = ICM_USB4_SWITCH_OP;
2335 request.route_hi = upper_32_bits(route);
2336 request.route_lo = lower_32_bits(route);
2337 request.opcode = opcode;
2338 if (metadata)
2339 request.metadata = *metadata;
2340
2341 if (tx_data_len) {
2342 request.data_len_valid |= ICM_USB4_SWITCH_DATA_VALID;
2343 if (tx_data_len < ARRAY_SIZE(request.data))
2344 request.data_len_valid |=
2345 tx_data_len & ICM_USB4_SWITCH_DATA_LEN_MASK;
2346 memcpy(request.data, tx_data, tx_data_len * sizeof(u32));
2347 }
2348
2349 memset(&reply, 0, sizeof(reply));
2350 ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),
2351 1, ICM_RETRIES, ICM_TIMEOUT);
2352 if (ret)
2353 return ret;
2354
2355 if (reply.hdr.flags & ICM_FLAGS_ERROR)
2356 return -EIO;
2357
2358 if (status)
2359 *status = reply.status;
2360
2361 if (metadata)
2362 *metadata = reply.metadata;
2363
2364 if (rx_data_len)
2365 memcpy(rx_data, reply.data, rx_data_len * sizeof(u32));
2366
2367 return 0;
2368}
2369
2370static int icm_usb4_switch_nvm_authenticate_status(struct tb_switch *sw,
2371 u32 *status)
2372{
2373 struct usb4_switch_nvm_auth *auth;
2374 struct tb *tb = sw->tb;
2375 struct icm *icm = tb_priv(tb);
2376 int ret = 0;
2377
2378 if (icm->proto_version < 3)
2379 return -EOPNOTSUPP;
2380
2381 auth = icm->last_nvm_auth;
2382 icm->last_nvm_auth = NULL;
2383
2384 if (auth && auth->reply.route_hi == sw->config.route_hi &&
2385 auth->reply.route_lo == sw->config.route_lo) {
2386 tb_dbg(tb, "NVM_AUTH found for %llx flags %#x status %#x\n",
2387 tb_route(sw), auth->reply.hdr.flags, auth->reply.status);
2388 if (auth->reply.hdr.flags & ICM_FLAGS_ERROR)
2389 ret = -EIO;
2390 else
2391 *status = auth->reply.status;
2392 } else {
2393 *status = 0;
2394 }
2395
2396 kfree(auth);
2397 return ret;
2398}
2399
2400/* Falcon Ridge */
2401static const struct tb_cm_ops icm_fr_ops = {
2402 .driver_ready = icm_driver_ready,
2403 .start = icm_start,
2404 .stop = icm_stop,
2405 .suspend = icm_suspend,
2406 .complete = icm_complete,
2407 .handle_event = icm_handle_event,
2408 .approve_switch = icm_fr_approve_switch,
2409 .add_switch_key = icm_fr_add_switch_key,
2410 .challenge_switch_key = icm_fr_challenge_switch_key,
2411 .disconnect_pcie_paths = icm_disconnect_pcie_paths,
2412 .approve_xdomain_paths = icm_fr_approve_xdomain_paths,
2413 .disconnect_xdomain_paths = icm_fr_disconnect_xdomain_paths,
2414};
2415
2416/* Alpine Ridge */
2417static const struct tb_cm_ops icm_ar_ops = {
2418 .driver_ready = icm_driver_ready,
2419 .start = icm_start,
2420 .stop = icm_stop,
2421 .suspend = icm_suspend,
2422 .complete = icm_complete,
2423 .runtime_suspend = icm_runtime_suspend,
2424 .runtime_resume = icm_runtime_resume,
2425 .runtime_suspend_switch = icm_runtime_suspend_switch,
2426 .runtime_resume_switch = icm_runtime_resume_switch,
2427 .handle_event = icm_handle_event,
2428 .get_boot_acl = icm_ar_get_boot_acl,
2429 .set_boot_acl = icm_ar_set_boot_acl,
2430 .approve_switch = icm_fr_approve_switch,
2431 .add_switch_key = icm_fr_add_switch_key,
2432 .challenge_switch_key = icm_fr_challenge_switch_key,
2433 .disconnect_pcie_paths = icm_disconnect_pcie_paths,
2434 .approve_xdomain_paths = icm_fr_approve_xdomain_paths,
2435 .disconnect_xdomain_paths = icm_fr_disconnect_xdomain_paths,
2436};
2437
2438/* Titan Ridge */
2439static const struct tb_cm_ops icm_tr_ops = {
2440 .driver_ready = icm_driver_ready,
2441 .start = icm_start,
2442 .stop = icm_stop,
2443 .suspend = icm_suspend,
2444 .complete = icm_complete,
2445 .runtime_suspend = icm_runtime_suspend,
2446 .runtime_resume = icm_runtime_resume,
2447 .runtime_suspend_switch = icm_runtime_suspend_switch,
2448 .runtime_resume_switch = icm_runtime_resume_switch,
2449 .handle_event = icm_handle_event,
2450 .get_boot_acl = icm_ar_get_boot_acl,
2451 .set_boot_acl = icm_ar_set_boot_acl,
2452 .approve_switch = icm_tr_approve_switch,
2453 .add_switch_key = icm_tr_add_switch_key,
2454 .challenge_switch_key = icm_tr_challenge_switch_key,
2455 .disconnect_pcie_paths = icm_disconnect_pcie_paths,
2456 .approve_xdomain_paths = icm_tr_approve_xdomain_paths,
2457 .disconnect_xdomain_paths = icm_tr_disconnect_xdomain_paths,
2458 .usb4_switch_op = icm_usb4_switch_op,
2459 .usb4_switch_nvm_authenticate_status =
2460 icm_usb4_switch_nvm_authenticate_status,
2461};
2462
2463/* Ice Lake */
2464static const struct tb_cm_ops icm_icl_ops = {
2465 .driver_ready = icm_driver_ready,
2466 .start = icm_start,
2467 .stop = icm_stop,
2468 .complete = icm_complete,
2469 .runtime_suspend = icm_runtime_suspend,
2470 .runtime_resume = icm_runtime_resume,
2471 .handle_event = icm_handle_event,
2472 .approve_xdomain_paths = icm_tr_approve_xdomain_paths,
2473 .disconnect_xdomain_paths = icm_tr_disconnect_xdomain_paths,
2474 .usb4_switch_op = icm_usb4_switch_op,
2475 .usb4_switch_nvm_authenticate_status =
2476 icm_usb4_switch_nvm_authenticate_status,
2477};
2478
2479struct tb *icm_probe(struct tb_nhi *nhi)
2480{
2481 struct pci_dev *pdev = to_pci_dev(nhi->dev);
2482 struct icm *icm;
2483 struct tb *tb;
2484
2485 tb = tb_domain_alloc(nhi, ICM_TIMEOUT, sizeof(struct icm));
2486 if (!tb)
2487 return NULL;
2488
2489 icm = tb_priv(tb);
2490 INIT_DELAYED_WORK(&icm->rescan_work, icm_rescan_work);
2491 mutex_init(&icm->request_lock);
2492
2493 switch (pdev->device) {
2494 case PCI_DEVICE_ID_INTEL_FALCON_RIDGE_2C_NHI:
2495 case PCI_DEVICE_ID_INTEL_FALCON_RIDGE_4C_NHI:
2496 icm->can_upgrade_nvm = true;
2497 icm->is_supported = icm_fr_is_supported;
2498 icm->get_route = icm_fr_get_route;
2499 icm->save_devices = icm_fr_save_devices;
2500 icm->driver_ready = icm_fr_driver_ready;
2501 icm->device_connected = icm_fr_device_connected;
2502 icm->device_disconnected = icm_fr_device_disconnected;
2503 icm->xdomain_connected = icm_fr_xdomain_connected;
2504 icm->xdomain_disconnected = icm_fr_xdomain_disconnected;
2505 tb->cm_ops = &icm_fr_ops;
2506 break;
2507
2508 case PCI_DEVICE_ID_INTEL_ALPINE_RIDGE_2C_NHI:
2509 case PCI_DEVICE_ID_INTEL_ALPINE_RIDGE_4C_NHI:
2510 case PCI_DEVICE_ID_INTEL_ALPINE_RIDGE_LP_NHI:
2511 case PCI_DEVICE_ID_INTEL_ALPINE_RIDGE_C_4C_NHI:
2512 case PCI_DEVICE_ID_INTEL_ALPINE_RIDGE_C_2C_NHI:
2513 icm->max_boot_acl = ICM_AR_PREBOOT_ACL_ENTRIES;
2514 /*
2515 * NVM upgrade has not been tested on Apple systems and
2516 * they don't provide images publicly either. To be on
2517 * the safe side prevent root switch NVM upgrade on Macs
2518 * for now.
2519 */
2520 icm->can_upgrade_nvm = !x86_apple_machine;
2521 icm->is_supported = icm_ar_is_supported;
2522 icm->cio_reset = icm_ar_cio_reset;
2523 icm->get_mode = icm_ar_get_mode;
2524 icm->get_route = icm_ar_get_route;
2525 icm->save_devices = icm_fr_save_devices;
2526 icm->driver_ready = icm_ar_driver_ready;
2527 icm->device_connected = icm_fr_device_connected;
2528 icm->device_disconnected = icm_fr_device_disconnected;
2529 icm->xdomain_connected = icm_fr_xdomain_connected;
2530 icm->xdomain_disconnected = icm_fr_xdomain_disconnected;
2531 tb->cm_ops = &icm_ar_ops;
2532 break;
2533
2534 case PCI_DEVICE_ID_INTEL_TITAN_RIDGE_2C_NHI:
2535 case PCI_DEVICE_ID_INTEL_TITAN_RIDGE_4C_NHI:
2536 icm->max_boot_acl = ICM_AR_PREBOOT_ACL_ENTRIES;
2537 icm->can_upgrade_nvm = !x86_apple_machine;
2538 icm->is_supported = icm_ar_is_supported;
2539 icm->cio_reset = icm_tr_cio_reset;
2540 icm->get_mode = icm_ar_get_mode;
2541 icm->driver_ready = icm_tr_driver_ready;
2542 icm->device_connected = icm_tr_device_connected;
2543 icm->device_disconnected = icm_tr_device_disconnected;
2544 icm->xdomain_connected = icm_tr_xdomain_connected;
2545 icm->xdomain_disconnected = icm_tr_xdomain_disconnected;
2546 tb->cm_ops = &icm_tr_ops;
2547 break;
2548
2549 case PCI_DEVICE_ID_INTEL_ICL_NHI0:
2550 case PCI_DEVICE_ID_INTEL_ICL_NHI1:
2551 icm->is_supported = icm_fr_is_supported;
2552 icm->driver_ready = icm_icl_driver_ready;
2553 icm->set_uuid = icm_icl_set_uuid;
2554 icm->device_connected = icm_icl_device_connected;
2555 icm->device_disconnected = icm_tr_device_disconnected;
2556 icm->xdomain_connected = icm_tr_xdomain_connected;
2557 icm->xdomain_disconnected = icm_tr_xdomain_disconnected;
2558 icm->rtd3_veto = icm_icl_rtd3_veto;
2559 tb->cm_ops = &icm_icl_ops;
2560 break;
2561
2562 case PCI_DEVICE_ID_INTEL_TGL_NHI0:
2563 case PCI_DEVICE_ID_INTEL_TGL_NHI1:
2564 case PCI_DEVICE_ID_INTEL_TGL_H_NHI0:
2565 case PCI_DEVICE_ID_INTEL_TGL_H_NHI1:
2566 case PCI_DEVICE_ID_INTEL_ADL_NHI0:
2567 case PCI_DEVICE_ID_INTEL_ADL_NHI1:
2568 case PCI_DEVICE_ID_INTEL_RPL_NHI0:
2569 case PCI_DEVICE_ID_INTEL_RPL_NHI1:
2570 case PCI_DEVICE_ID_INTEL_MTL_M_NHI0:
2571 case PCI_DEVICE_ID_INTEL_MTL_P_NHI0:
2572 case PCI_DEVICE_ID_INTEL_MTL_P_NHI1:
2573 icm->is_supported = icm_tgl_is_supported;
2574 icm->driver_ready = icm_icl_driver_ready;
2575 icm->set_uuid = icm_icl_set_uuid;
2576 icm->device_connected = icm_icl_device_connected;
2577 icm->device_disconnected = icm_tr_device_disconnected;
2578 icm->xdomain_connected = icm_tr_xdomain_connected;
2579 icm->xdomain_disconnected = icm_tr_xdomain_disconnected;
2580 icm->rtd3_veto = icm_icl_rtd3_veto;
2581 tb->cm_ops = &icm_icl_ops;
2582 break;
2583
2584 case PCI_DEVICE_ID_INTEL_MAPLE_RIDGE_2C_NHI:
2585 case PCI_DEVICE_ID_INTEL_MAPLE_RIDGE_4C_NHI:
2586 icm->can_upgrade_nvm = true;
2587 icm->is_supported = icm_tgl_is_supported;
2588 icm->get_mode = icm_ar_get_mode;
2589 icm->driver_ready = icm_tr_driver_ready;
2590 icm->device_connected = icm_tr_device_connected;
2591 icm->device_disconnected = icm_tr_device_disconnected;
2592 icm->xdomain_connected = icm_tr_xdomain_connected;
2593 icm->xdomain_disconnected = icm_tr_xdomain_disconnected;
2594 tb->cm_ops = &icm_tr_ops;
2595 break;
2596 }
2597
2598 if (!icm->is_supported || !icm->is_supported(tb)) {
2599 dev_dbg(nhi->dev, "ICM not supported on this controller\n");
2600 tb_domain_put(tb);
2601 return NULL;
2602 }
2603
2604 tb_dbg(tb, "using firmware connection manager\n");
2605
2606 return tb;
2607}