projects/linux·blob master · refs

631 lines · 16.2 KiB

1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Thunderbolt driver - path/tunnel functionality
4 *
5 * Copyright (c) 2014 Andreas Noever <[email protected]>
6 * Copyright (C) 2019, Intel Corporation
7 */
8
9#include <linux/slab.h>
10#include <linux/errno.h>
11#include <linux/delay.h>
12#include <linux/ktime.h>
13
14#include "tb.h"
15
16static void tb_dump_hop(const struct tb_path_hop *hop, const struct tb_regs_hop *regs)
17{
18 const struct tb_port *port = hop->in_port;
19
20 tb_port_dbg(port, " In HopID: %d => Out port: %d Out HopID: %d\n",
21 hop->in_hop_index, regs->out_port, regs->next_hop);
22 tb_port_dbg(port, " Weight: %d Priority: %d Credits: %d Drop: %d PM: %d\n",
23 regs->weight, regs->priority, regs->initial_credits,
24 regs->drop_packages, regs->pmps);
25 tb_port_dbg(port, " Counter enabled: %d Counter index: %d\n",
26 regs->counter_enable, regs->counter);
27 tb_port_dbg(port, " Flow Control (In/Eg): %d/%d Shared Buffer (In/Eg): %d/%d\n",
28 regs->ingress_fc, regs->egress_fc,
29 regs->ingress_shared_buffer, regs->egress_shared_buffer);
30 tb_port_dbg(port, " Unknown1: %#x Unknown2: %#x Unknown3: %#x\n",
31 regs->unknown1, regs->unknown2, regs->unknown3);
32}
33
34static struct tb_port *tb_path_find_dst_port(struct tb_port *src, int src_hopid,
35 int dst_hopid)
36{
37 struct tb_port *port, *out_port = NULL;
38 struct tb_regs_hop hop;
39 struct tb_switch *sw;
40 int i, ret, hopid;
41
42 hopid = src_hopid;
43 port = src;
44
45 for (i = 0; port && i < TB_PATH_MAX_HOPS; i++) {
46 sw = port->sw;
47
48 ret = tb_port_read(port, &hop, TB_CFG_HOPS, 2 * hopid, 2);
49 if (ret) {
50 tb_port_warn(port, "failed to read path at %d\n", hopid);
51 return NULL;
52 }
53
54 if (!hop.enable)
55 return NULL;
56
57 out_port = &sw->ports[hop.out_port];
58 hopid = hop.next_hop;
59 port = out_port->remote;
60 }
61
62 return out_port && hopid == dst_hopid ? out_port : NULL;
63}
64
65static int tb_path_find_src_hopid(struct tb_port *src,
66 const struct tb_port *dst, int dst_hopid)
67{
68 struct tb_port *out;
69 int i;
70
71 for (i = TB_PATH_MIN_HOPID; i <= src->config.max_in_hop_id; i++) {
72 out = tb_path_find_dst_port(src, i, dst_hopid);
73 if (out == dst)
74 return i;
75 }
76
77 return 0;
78}
79
80/**
81 * tb_path_discover() - Discover a path
82 * @src: First input port of a path
83 * @src_hopid: Starting HopID of a path (%-1 if don't care)
84 * @dst: Expected destination port of the path (%NULL if don't care)
85 * @dst_hopid: HopID to the @dst (%-1 if don't care)
86 * @last: Last port is filled here if not %NULL
87 * @name: Name of the path
88 * @alloc_hopid: Allocate HopIDs for the ports
89 *
90 * Follows a path starting from @src and @src_hopid to the last output
91 * port of the path. Allocates HopIDs for the visited ports (if
92 * @alloc_hopid is true). Call tb_path_free() to release the path and
93 * allocated HopIDs when the path is not needed anymore.
94 *
95 * Note function discovers also incomplete paths so caller should check
96 * that the @dst port is the expected one. If it is not, the path can be
97 * cleaned up by calling tb_path_deactivate() before tb_path_free().
98 *
99 * Return: Pointer to &struct tb_path, %NULL in case of failure.
100 */
101struct tb_path *tb_path_discover(struct tb_port *src, int src_hopid,
102 struct tb_port *dst, int dst_hopid,
103 struct tb_port **last, const char *name,
104 bool alloc_hopid)
105{
106 struct tb_port *out_port;
107 struct tb_regs_hop hop;
108 struct tb_path *path;
109 struct tb_switch *sw;
110 struct tb_port *p;
111 size_t num_hops;
112 int ret, i, h;
113
114 if (src_hopid < 0 && dst) {
115 /*
116 * For incomplete paths the intermediate HopID can be
117 * different from the one used by the protocol adapter
118 * so in that case find a path that ends on @dst with
119 * matching @dst_hopid. That should give us the correct
120 * HopID for the @src.
121 */
122 src_hopid = tb_path_find_src_hopid(src, dst, dst_hopid);
123 if (!src_hopid)
124 return NULL;
125 }
126
127 p = src;
128 h = src_hopid;
129 num_hops = 0;
130
131 for (i = 0; p && i < TB_PATH_MAX_HOPS; i++) {
132 sw = p->sw;
133
134 ret = tb_port_read(p, &hop, TB_CFG_HOPS, 2 * h, 2);
135 if (ret) {
136 tb_port_warn(p, "failed to read path at %d\n", h);
137 return NULL;
138 }
139
140 /* If the hop is not enabled we got an incomplete path */
141 if (!hop.enable)
142 break;
143
144 out_port = &sw->ports[hop.out_port];
145 if (last)
146 *last = out_port;
147
148 h = hop.next_hop;
149 p = out_port->remote;
150 num_hops++;
151 }
152
153 path = kzalloc_flex(*path, hops, num_hops);
154 if (!path)
155 return NULL;
156
157 path->path_length = num_hops;
158
159 path->name = name;
160 path->tb = src->sw->tb;
161 path->activated = true;
162 path->alloc_hopid = alloc_hopid;
163
164 tb_dbg(path->tb, "discovering %s path starting from %llx:%u\n",
165 path->name, tb_route(src->sw), src->port);
166
167 p = src;
168 h = src_hopid;
169
170 for (i = 0; i < num_hops; i++) {
171 int next_hop;
172
173 sw = p->sw;
174
175 ret = tb_port_read(p, &hop, TB_CFG_HOPS, 2 * h, 2);
176 if (ret) {
177 tb_port_warn(p, "failed to read path at %d\n", h);
178 goto err;
179 }
180
181 if (alloc_hopid && tb_port_alloc_in_hopid(p, h, h) < 0)
182 goto err;
183
184 out_port = &sw->ports[hop.out_port];
185 next_hop = hop.next_hop;
186
187 if (alloc_hopid &&
188 tb_port_alloc_out_hopid(out_port, next_hop, next_hop) < 0) {
189 tb_port_release_in_hopid(p, h);
190 goto err;
191 }
192
193 path->hops[i].in_port = p;
194 path->hops[i].in_hop_index = h;
195 path->hops[i].in_counter_index = -1;
196 path->hops[i].out_port = out_port;
197 path->hops[i].next_hop_index = next_hop;
198
199 tb_dump_hop(&path->hops[i], &hop);
200
201 h = next_hop;
202 p = out_port->remote;
203 }
204
205 tb_dbg(path->tb, "path discovery complete\n");
206 return path;
207
208err:
209 tb_port_warn(src, "failed to discover path starting at HopID %d\n",
210 src_hopid);
211 tb_path_free(path);
212 return NULL;
213}
214
215/**
216 * tb_path_alloc() - allocate a thunderbolt path between two ports
217 * @tb: Domain pointer
218 * @src: Source port of the path
219 * @src_hopid: HopID used for the first ingress port in the path
220 * @dst: Destination port of the path
221 * @dst_hopid: HopID used for the last egress port in the path
222 * @link_nr: Preferred link if there are dual links on the path
223 * @name: Name of the path
224 *
225 * Creates path between two ports starting with given @src_hopid. Reserves
226 * HopIDs for each port (they can be different from @src_hopid depending on
227 * how many HopIDs each port already have reserved). If there are dual
228 * links on the path, prioritizes using @link_nr but takes into account
229 * that the lanes may be bonded.
230 *
231 * Return: Pointer to &struct tb_path, %NULL in case of failure.
232 */
233struct tb_path *tb_path_alloc(struct tb *tb, struct tb_port *src, int src_hopid,
234 struct tb_port *dst, int dst_hopid, int link_nr,
235 const char *name)
236{
237 struct tb_port *in_port, *out_port, *first_port, *last_port;
238 int in_hopid, out_hopid;
239 struct tb_path *path;
240 size_t num_hops;
241 int i, ret;
242
243 first_port = last_port = NULL;
244 i = 0;
245 tb_for_each_port_on_path(src, dst, in_port) {
246 if (!first_port)
247 first_port = in_port;
248 last_port = in_port;
249 i++;
250 }
251
252 /* Check that src and dst are reachable */
253 if (first_port != src || last_port != dst)
254 return NULL;
255
256 /* Each hop takes two ports */
257 num_hops = i / 2;
258
259 path = kzalloc_flex(*path, hops, num_hops);
260 if (!path)
261 return NULL;
262
263 path->path_length = num_hops;
264 path->alloc_hopid = true;
265
266 in_hopid = src_hopid;
267 out_port = NULL;
268
269 for (i = 0; i < num_hops; i++) {
270 in_port = tb_next_port_on_path(src, dst, out_port);
271 if (!in_port)
272 goto err;
273
274 /* When lanes are bonded primary link must be used */
275 if (!in_port->bonded && in_port->dual_link_port &&
276 in_port->link_nr != link_nr)
277 in_port = in_port->dual_link_port;
278
279 ret = tb_port_alloc_in_hopid(in_port, in_hopid, in_hopid);
280 if (ret < 0)
281 goto err;
282 in_hopid = ret;
283
284 out_port = tb_next_port_on_path(src, dst, in_port);
285 if (!out_port)
286 goto err;
287
288 /*
289 * Pick up right port when going from non-bonded to
290 * bonded or from bonded to non-bonded.
291 */
292 if (out_port->dual_link_port) {
293 if (!in_port->bonded && out_port->bonded &&
294 out_port->link_nr) {
295 /*
296 * Use primary link when going from
297 * non-bonded to bonded.
298 */
299 out_port = out_port->dual_link_port;
300 } else if (!out_port->bonded &&
301 out_port->link_nr != link_nr) {
302 /*
303 * If out port is not bonded follow
304 * link_nr.
305 */
306 out_port = out_port->dual_link_port;
307 }
308 }
309
310 if (i == num_hops - 1)
311 ret = tb_port_alloc_out_hopid(out_port, dst_hopid,
312 dst_hopid);
313 else
314 ret = tb_port_alloc_out_hopid(out_port, -1, -1);
315
316 if (ret < 0)
317 goto err;
318 out_hopid = ret;
319
320 path->hops[i].in_hop_index = in_hopid;
321 path->hops[i].in_port = in_port;
322 path->hops[i].in_counter_index = -1;
323 path->hops[i].out_port = out_port;
324 path->hops[i].next_hop_index = out_hopid;
325
326 in_hopid = out_hopid;
327 }
328
329 path->tb = tb;
330 path->name = name;
331
332 return path;
333
334err:
335 tb_path_free(path);
336 return NULL;
337}
338
339/**
340 * tb_path_free() - free a path
341 * @path: Path to free
342 *
343 * Frees a path. The path does not need to be deactivated.
344 */
345void tb_path_free(struct tb_path *path)
346{
347 if (path->alloc_hopid) {
348 int i;
349
350 for (i = 0; i < path->path_length; i++) {
351 const struct tb_path_hop *hop = &path->hops[i];
352
353 if (hop->in_port)
354 tb_port_release_in_hopid(hop->in_port,
355 hop->in_hop_index);
356 if (hop->out_port)
357 tb_port_release_out_hopid(hop->out_port,
358 hop->next_hop_index);
359 }
360 }
361
362 kfree(path);
363}
364
365static void __tb_path_deallocate_nfc(struct tb_path *path, int first_hop)
366{
367 int i, res;
368 for (i = first_hop; i < path->path_length; i++) {
369 res = tb_port_add_nfc_credits(path->hops[i].in_port,
370 -path->hops[i].nfc_credits);
371 if (res)
372 tb_port_warn(path->hops[i].in_port,
373 "nfc credits deallocation failed for hop %d\n",
374 i);
375 }
376}
377
378static int __tb_path_deactivate_hop(struct tb_port *port, int hop_index,
379 bool clear_fc)
380{
381 struct tb_regs_hop hop;
382 ktime_t timeout;
383 int ret;
384
385 /* Disable the path */
386 ret = tb_port_read(port, &hop, TB_CFG_HOPS, 2 * hop_index, 2);
387 if (ret)
388 return ret;
389
390 /* Already disabled */
391 if (!hop.enable)
392 return 0;
393
394 hop.enable = 0;
395
396 ret = tb_port_write(port, &hop, TB_CFG_HOPS, 2 * hop_index, 2);
397 if (ret)
398 return ret;
399
400 /* Wait until it is drained */
401 timeout = ktime_add_ms(ktime_get(), 500);
402 do {
403 ret = tb_port_read(port, &hop, TB_CFG_HOPS, 2 * hop_index, 2);
404 if (ret)
405 return ret;
406
407 if (!hop.pending) {
408 if (clear_fc) {
409 /*
410 * Clear flow control. Protocol adapters
411 * IFC and ISE bits are vendor defined
412 * in the USB4 spec so we clear them
413 * only for pre-USB4 adapters.
414 */
415 if (tb_port_is_null(port) ||
416 !tb_switch_is_usb4(port->sw)) {
417 hop.ingress_fc = 0;
418 hop.ingress_shared_buffer = 0;
419 }
420 hop.egress_fc = 0;
421 hop.egress_shared_buffer = 0;
422
423 return tb_port_write(port, &hop, TB_CFG_HOPS,
424 2 * hop_index, 2);
425 }
426
427 return 0;
428 }
429
430 usleep_range(10, 20);
431 } while (ktime_before(ktime_get(), timeout));
432
433 return -ETIMEDOUT;
434}
435
436/**
437 * tb_path_deactivate_hop() - Deactivate one path in path config space
438 * @port: Lane or protocol adapter
439 * @hop_index: HopID of the path to be cleared
440 *
441 * This deactivates or clears a single path config space entry at
442 * @hop_index.
443 *
444 * Return: %0 on success, negative errno otherwise.
445 */
446int tb_path_deactivate_hop(struct tb_port *port, int hop_index)
447{
448 return __tb_path_deactivate_hop(port, hop_index, true);
449}
450
451static void __tb_path_deactivate_hops(struct tb_path *path, int first_hop)
452{
453 int i, res;
454
455 for (i = first_hop; i < path->path_length; i++) {
456 res = __tb_path_deactivate_hop(path->hops[i].in_port,
457 path->hops[i].in_hop_index,
458 path->clear_fc);
459 if (res && res != -ENODEV)
460 tb_port_warn(path->hops[i].in_port,
461 "hop deactivation failed for hop %d, index %d\n",
462 i, path->hops[i].in_hop_index);
463 }
464}
465
466void tb_path_deactivate(struct tb_path *path)
467{
468 if (!path->activated) {
469 tb_WARN(path->tb, "trying to deactivate an inactive path\n");
470 return;
471 }
472 tb_dbg(path->tb,
473 "deactivating %s path from %llx:%u to %llx:%u\n",
474 path->name, tb_route(path->hops[0].in_port->sw),
475 path->hops[0].in_port->port,
476 tb_route(path->hops[path->path_length - 1].out_port->sw),
477 path->hops[path->path_length - 1].out_port->port);
478 __tb_path_deactivate_hops(path, 0);
479 __tb_path_deallocate_nfc(path, 0);
480 path->activated = false;
481}
482
483/**
484 * tb_path_activate() - activate a path
485 * @path: Path to activate
486 *
487 * Activate a path starting with the first hop and ending on the last hop. The
488 * caller must fill path->hops before calling tb_path_activate().
489 *
490 * Return: %0 on success, negative errno otherwise.
491 */
492int tb_path_activate(struct tb_path *path)
493{
494 int i, res;
495 enum tb_path_port out_mask, in_mask;
496 if (path->activated) {
497 tb_WARN(path->tb, "trying to activate already activated path\n");
498 return -EINVAL;
499 }
500
501 tb_dbg(path->tb,
502 "activating %s path from %llx:%u to %llx:%u\n",
503 path->name, tb_route(path->hops[0].in_port->sw),
504 path->hops[0].in_port->port,
505 tb_route(path->hops[path->path_length - 1].out_port->sw),
506 path->hops[path->path_length - 1].out_port->port);
507
508 /* Clear counters. */
509 for (i = path->path_length - 1; i >= 0; i--) {
510 if (path->hops[i].in_counter_index == -1)
511 continue;
512 res = tb_port_clear_counter(path->hops[i].in_port,
513 path->hops[i].in_counter_index);
514 if (res)
515 goto err;
516 }
517
518 /* Add non flow controlled credits. */
519 for (i = path->path_length - 1; i >= 0; i--) {
520 res = tb_port_add_nfc_credits(path->hops[i].in_port,
521 path->hops[i].nfc_credits);
522 if (res) {
523 __tb_path_deallocate_nfc(path, i);
524 goto err;
525 }
526 }
527
528 /* Activate hops. */
529 for (i = 0; i < path->path_length; i++) {
530 struct tb_regs_hop hop = { 0 };
531
532 /* If it is left active deactivate it first */
533 __tb_path_deactivate_hop(path->hops[i].in_port,
534 path->hops[i].in_hop_index, path->clear_fc);
535
536 /* Needed for USB4 routers, read path config space before write */
537 res = tb_port_read(path->hops[i].in_port, &hop, TB_CFG_HOPS,
538 2 * path->hops[i].in_hop_index, 2);
539 if (res)
540 goto err;
541
542 hop.next_hop = path->hops[i].next_hop_index;
543 hop.out_port = path->hops[i].out_port->port;
544 hop.pmps = path->hops[i].pm_support;
545 hop.unknown1 = 0;
546 hop.enable = 1;
547
548 out_mask = (i == path->path_length - 1) ?
549 TB_PATH_DESTINATION : TB_PATH_INTERNAL;
550 in_mask = (i == 0) ? TB_PATH_SOURCE : TB_PATH_INTERNAL;
551 hop.weight = path->weight;
552 hop.unknown2 = 0;
553 hop.priority = path->priority;
554 hop.drop_packages = path->drop_packages;
555 hop.counter = path->hops[i].in_counter_index;
556 hop.counter_enable = path->hops[i].in_counter_index != -1;
557 hop.egress_fc = path->egress_fc_enable & out_mask;
558 hop.egress_shared_buffer = path->egress_shared_buffer & out_mask;
559 /*
560 * Protocol adapters IFC and ISE bits, and Path Credits
561 * Allocated are vendor defined in the USB4 spec so we
562 * program them only for pre-USB4 and lane adapters.
563 */
564 if (tb_port_is_null(path->hops[i].in_port) ||
565 !tb_switch_is_usb4(path->hops[i].in_port->sw)) {
566 hop.initial_credits = path->hops[i].initial_credits;
567 hop.ingress_fc = path->ingress_fc_enable & in_mask;
568 hop.ingress_shared_buffer =
569 path->ingress_shared_buffer & in_mask;
570 }
571
572 hop.unknown3 = 0;
573
574 tb_port_dbg(path->hops[i].in_port, "Writing hop %d\n", i);
575 tb_dump_hop(&path->hops[i], &hop);
576 res = tb_port_write(path->hops[i].in_port, &hop, TB_CFG_HOPS,
577 2 * path->hops[i].in_hop_index, 2);
578 if (res) {
579 __tb_path_deactivate_hops(path, 0);
580 __tb_path_deallocate_nfc(path, 0);
581 goto err;
582 }
583 }
584 path->activated = true;
585 tb_dbg(path->tb, "%s path activation complete\n", path->name);
586 return 0;
587err:
588 tb_warn(path->tb, "%s path activation failed: %d\n", path->name, res);
589 return res;
590}
591
592/**
593 * tb_path_is_invalid() - check whether any ports on the path are invalid
594 * @path: Path to check
595 *
596 * Return: %true if the path is invalid, %false otherwise.
597 */
598bool tb_path_is_invalid(struct tb_path *path)
599{
600 int i = 0;
601 for (i = 0; i < path->path_length; i++) {
602 if (path->hops[i].in_port->sw->is_unplugged)
603 return true;
604 if (path->hops[i].out_port->sw->is_unplugged)
605 return true;
606 }
607 return false;
608}
609
610/**
611 * tb_path_port_on_path() - Does the path go through certain port
612 * @path: Path to check
613 * @port: Switch to check
614 *
615 * Goes over all hops on path and checks if @port is any of them.
616 * Direction does not matter.
617 *
618 * Return: %true if port is on the path, %false otherwise.
619 */
620bool tb_path_port_on_path(const struct tb_path *path, const struct tb_port *port)
621{
622 int i;
623
624 for (i = 0; i < path->path_length; i++) {
625 if (path->hops[i].in_port == port ||
626 path->hops[i].out_port == port)
627 return true;
628 }
629
630 return false;
631}