projects/linux·blob master · refs

1303 lines · 32.2 KiB

1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Thunderbolt driver - NHI driver
4 *
5 * The NHI (native host interface) is the device that allows us to send and
6 * receive frames from the thunderbolt bus.
7 *
8 * Copyright (c) 2014 Andreas Noever <[email protected]>
9 * Copyright (C) 2018, Intel Corporation
10 */
11
12#include <linux/pm_runtime.h>
13#include <linux/slab.h>
14#include <linux/errno.h>
15#include <linux/dma-mapping.h>
16#include <linux/interrupt.h>
17#include <linux/iommu.h>
18#include <linux/lockdep.h>
19#include <linux/module.h>
20#include <linux/delay.h>
21#include <linux/property.h>
22#include <linux/string_choices.h>
23#include <linux/string_helpers.h>
24
25#include "nhi.h"
26#include "nhi_regs.h"
27#include "tb.h"
28
29#define RING_TYPE(ring) ((ring)->is_tx ? "TX ring" : "RX ring")
30
31#define RING_FIRST_USABLE_HOPID 1
32/*
33 * Used with QUIRK_E2E to specify an unused HopID the Rx credits are
34 * transferred.
35 */
36#define RING_E2E_RESERVED_HOPID RING_FIRST_USABLE_HOPID
37
38#define NHI_MAILBOX_TIMEOUT 500 /* ms */
39
40static bool host_reset = true;
41module_param(host_reset, bool, 0444);
42MODULE_PARM_DESC(host_reset, "reset USB4 host router (default: true)");
43
44static int ring_interrupt_index(const struct tb_ring *ring)
45{
46 int bit = ring->hop;
47 if (!ring->is_tx)
48 bit += ring->nhi->hop_count;
49 return bit;
50}
51
52static void nhi_mask_interrupt(struct tb_nhi *nhi, int mask, int ring)
53{
54 if (nhi->quirks & QUIRK_AUTO_CLEAR_INT) {
55 u32 val;
56
57 val = ioread32(nhi->iobase + REG_RING_INTERRUPT_BASE + ring);
58 iowrite32(val & ~mask, nhi->iobase + REG_RING_INTERRUPT_BASE + ring);
59 } else {
60 iowrite32(mask, nhi->iobase + REG_RING_INTERRUPT_MASK_CLEAR_BASE + ring);
61 }
62}
63
64static void nhi_clear_interrupt(struct tb_nhi *nhi, int ring)
65{
66 if (nhi->quirks & QUIRK_AUTO_CLEAR_INT)
67 ioread32(nhi->iobase + REG_RING_NOTIFY_BASE + ring);
68 else
69 iowrite32(~0, nhi->iobase + REG_RING_INT_CLEAR + ring);
70}
71
72/*
73 * ring_interrupt_active() - activate/deactivate interrupts for a single ring
74 *
75 * ring->nhi->lock must be held.
76 */
77static void ring_interrupt_active(struct tb_ring *ring, bool active)
78{
79 int index = ring_interrupt_index(ring) / 32 * 4;
80 int reg = REG_RING_INTERRUPT_BASE + index;
81 int interrupt_bit = ring_interrupt_index(ring) & 31;
82 int mask = 1 << interrupt_bit;
83 u32 old, new;
84
85 if (ring->irq > 0) {
86 u32 step, shift, ivr, misc, itr;
87 void __iomem *ivr_base;
88 int auto_clear_bit;
89 int index;
90
91 if (ring->is_tx)
92 index = ring->hop;
93 else
94 index = ring->hop + ring->nhi->hop_count;
95
96 /*
97 * Intel routers support a bit that isn't part of
98 * the USB4 spec to ask the hardware to clear
99 * interrupt status bits automatically since
100 * we already know which interrupt was triggered.
101 *
102 * Other routers explicitly disable auto-clear
103 * to prevent conditions that may occur where two
104 * MSIX interrupts are simultaneously active and
105 * reading the register clears both of them.
106 */
107 misc = ioread32(ring->nhi->iobase + REG_DMA_MISC);
108 if (ring->nhi->quirks & QUIRK_AUTO_CLEAR_INT)
109 auto_clear_bit = REG_DMA_MISC_INT_AUTO_CLEAR;
110 else
111 auto_clear_bit = REG_DMA_MISC_DISABLE_AUTO_CLEAR;
112 if (!(misc & auto_clear_bit))
113 iowrite32(misc | auto_clear_bit,
114 ring->nhi->iobase + REG_DMA_MISC);
115
116 ivr_base = ring->nhi->iobase + REG_INT_VEC_ALLOC_BASE;
117 step = index / REG_INT_VEC_ALLOC_REGS * REG_INT_VEC_ALLOC_BITS;
118 shift = index % REG_INT_VEC_ALLOC_REGS * REG_INT_VEC_ALLOC_BITS;
119 ivr = ioread32(ivr_base + step);
120 ivr &= ~(REG_INT_VEC_ALLOC_MASK << shift);
121 if (active)
122 ivr |= ring->vector << shift;
123 iowrite32(ivr, ivr_base + step);
124
125 /* Throttling is specified in 256ns increments */
126 itr = DIV_ROUND_UP(ring->interval_nsec, 256);
127 itr &= REG_INT_THROTTLING_RATE_INTERVAL_MASK;
128 iowrite32(itr, ring->nhi->iobase + REG_INT_THROTTLING_RATE +
129 ring->vector * 4);
130 }
131
132 old = ioread32(ring->nhi->iobase + reg);
133 if (active)
134 new = old | mask;
135 else
136 new = old & ~mask;
137
138 dev_dbg(ring->nhi->dev,
139 "%s interrupt at register %#x bit %d (%#x -> %#x)\n",
140 active ? "enabling" : "disabling", reg, interrupt_bit, old, new);
141
142 if (new == old)
143 dev_WARN(ring->nhi->dev, "interrupt for %s %d is already %s\n",
144 RING_TYPE(ring), ring->hop,
145 str_enabled_disabled(active));
146
147 if (active)
148 iowrite32(new, ring->nhi->iobase + reg);
149 else
150 nhi_mask_interrupt(ring->nhi, mask, index);
151}
152
153/*
154 * nhi_disable_interrupts() - disable interrupts for all rings
155 *
156 * Use only during init and shutdown.
157 */
158void nhi_disable_interrupts(struct tb_nhi *nhi)
159{
160 int i = 0;
161 /* disable interrupts */
162 for (i = 0; i < RING_INTERRUPT_REG_COUNT(nhi); i++)
163 nhi_mask_interrupt(nhi, ~0, 4 * i);
164
165 /* clear interrupt status bits */
166 for (i = 0; i < RING_NOTIFY_REG_COUNT(nhi); i++)
167 nhi_clear_interrupt(nhi, 4 * i);
168}
169
170/* ring helper methods */
171
172static void __iomem *ring_desc_base(struct tb_ring *ring)
173{
174 void __iomem *io = ring->nhi->iobase;
175 io += ring->is_tx ? REG_TX_RING_BASE : REG_RX_RING_BASE;
176 io += ring->hop * 16;
177 return io;
178}
179
180static void __iomem *ring_options_base(struct tb_ring *ring)
181{
182 void __iomem *io = ring->nhi->iobase;
183 io += ring->is_tx ? REG_TX_OPTIONS_BASE : REG_RX_OPTIONS_BASE;
184 io += ring->hop * 32;
185 return io;
186}
187
188static void ring_iowrite_cons(struct tb_ring *ring, u16 cons)
189{
190 /*
191 * The other 16-bits in the register is read-only and writes to it
192 * are ignored by the hardware so we can save one ioread32() by
193 * filling the read-only bits with zeroes.
194 */
195 iowrite32(cons, ring_desc_base(ring) + 8);
196}
197
198static void ring_iowrite_prod(struct tb_ring *ring, u16 prod)
199{
200 /* See ring_iowrite_cons() above for explanation */
201 iowrite32(prod << 16, ring_desc_base(ring) + 8);
202}
203
204static void ring_iowrite32desc(struct tb_ring *ring, u32 value, u32 offset)
205{
206 iowrite32(value, ring_desc_base(ring) + offset);
207}
208
209static void ring_iowrite64desc(struct tb_ring *ring, u64 value, u32 offset)
210{
211 iowrite32(value, ring_desc_base(ring) + offset);
212 iowrite32(value >> 32, ring_desc_base(ring) + offset + 4);
213}
214
215static void ring_iowrite32options(struct tb_ring *ring, u32 value, u32 offset)
216{
217 iowrite32(value, ring_options_base(ring) + offset);
218}
219
220static bool ring_full(struct tb_ring *ring)
221{
222 return ((ring->head + 1) % ring->size) == ring->tail;
223}
224
225static bool ring_empty(struct tb_ring *ring)
226{
227 return ring->head == ring->tail;
228}
229
230/*
231 * ring_write_descriptors() - post frames from ring->queue to the controller
232 *
233 * ring->lock is held.
234 */
235static void ring_write_descriptors(struct tb_ring *ring)
236{
237 struct ring_frame *frame, *n;
238 struct ring_desc *descriptor;
239 u32 flags;
240
241 flags = RING_DESC_POSTED;
242 if (!(ring->flags & RING_FLAG_NO_INTERRUPT))
243 flags |= RING_DESC_INTERRUPT;
244
245 list_for_each_entry_safe(frame, n, &ring->queue, list) {
246 if (ring_full(ring))
247 break;
248 list_move_tail(&frame->list, &ring->in_flight);
249 descriptor = &ring->descriptors[ring->head];
250 descriptor->phys = frame->buffer_phy;
251 descriptor->time = 0;
252 descriptor->flags = flags;
253 if (ring->is_tx) {
254 descriptor->length = frame->size;
255 descriptor->eof = frame->eof;
256 descriptor->sof = frame->sof;
257 }
258 ring->head = (ring->head + 1) % ring->size;
259 if (ring->is_tx)
260 ring_iowrite_prod(ring, ring->head);
261 else
262 ring_iowrite_cons(ring, ring->head);
263 }
264}
265
266/*
267 * ring_work() - progress completed frames
268 *
269 * If the ring is shutting down then all frames are marked as canceled and
270 * their callbacks are invoked.
271 *
272 * Otherwise we collect all completed frame from the ring buffer, write new
273 * frame to the ring buffer and invoke the callbacks for the completed frames.
274 */
275static void ring_work(struct work_struct *work)
276{
277 struct tb_ring *ring = container_of(work, typeof(*ring), work);
278 struct ring_frame *frame;
279 bool canceled = false;
280 unsigned long flags;
281 LIST_HEAD(done);
282
283 spin_lock_irqsave(&ring->lock, flags);
284
285 if (!ring->running) {
286 /* Move all frames to done and mark them as canceled. */
287 list_splice_tail_init(&ring->in_flight, &done);
288 list_splice_tail_init(&ring->queue, &done);
289 canceled = true;
290 goto invoke_callback;
291 }
292
293 while (!ring_empty(ring)) {
294 if (!(ring->descriptors[ring->tail].flags
295 & RING_DESC_COMPLETED))
296 break;
297 frame = list_first_entry(&ring->in_flight, typeof(*frame),
298 list);
299 list_move_tail(&frame->list, &done);
300 if (!ring->is_tx) {
301 frame->size = ring->descriptors[ring->tail].length;
302 frame->eof = ring->descriptors[ring->tail].eof;
303 frame->sof = ring->descriptors[ring->tail].sof;
304 frame->flags = ring->descriptors[ring->tail].flags;
305 }
306 ring->tail = (ring->tail + 1) % ring->size;
307 }
308 ring_write_descriptors(ring);
309
310invoke_callback:
311 /* allow callbacks to schedule new work */
312 spin_unlock_irqrestore(&ring->lock, flags);
313 while (!list_empty(&done)) {
314 frame = list_first_entry(&done, typeof(*frame), list);
315 /*
316 * The callback may reenqueue or delete frame.
317 * Do not hold on to it.
318 */
319 list_del_init(&frame->list);
320 if (frame->callback)
321 frame->callback(ring, frame, canceled);
322 }
323
324 wake_up(&ring->wait);
325}
326
327int __tb_ring_enqueue(struct tb_ring *ring, struct ring_frame *frame)
328{
329 unsigned long flags;
330 int ret = 0;
331
332 spin_lock_irqsave(&ring->lock, flags);
333 if (ring->running) {
334 list_add_tail(&frame->list, &ring->queue);
335 ring_write_descriptors(ring);
336 } else {
337 ret = -ESHUTDOWN;
338 }
339 spin_unlock_irqrestore(&ring->lock, flags);
340 return ret;
341}
342EXPORT_SYMBOL_GPL(__tb_ring_enqueue);
343
344/**
345 * tb_ring_poll() - Poll one completed frame from the ring
346 * @ring: Ring to poll
347 *
348 * This function can be called when @start_poll callback of the @ring
349 * has been called or the ring is created with %RING_FLAG_NO_INTERRUPT.
350 * It will read one completed frame from the ring and return it to the
351 * caller.
352 *
353 * Return: Pointer to &struct ring_frame, %NULL if there is no more
354 * completed frames.
355 */
356struct ring_frame *tb_ring_poll(struct tb_ring *ring)
357{
358 struct ring_frame *frame = NULL;
359 unsigned long flags;
360
361 spin_lock_irqsave(&ring->lock, flags);
362 if (!ring->running)
363 goto unlock;
364 if (ring_empty(ring))
365 goto unlock;
366
367 if (ring->descriptors[ring->tail].flags & RING_DESC_COMPLETED) {
368 frame = list_first_entry(&ring->in_flight, typeof(*frame),
369 list);
370 list_del_init(&frame->list);
371
372 if (!ring->is_tx) {
373 frame->size = ring->descriptors[ring->tail].length;
374 frame->eof = ring->descriptors[ring->tail].eof;
375 frame->sof = ring->descriptors[ring->tail].sof;
376 frame->flags = ring->descriptors[ring->tail].flags;
377 }
378
379 ring->tail = (ring->tail + 1) % ring->size;
380 }
381
382unlock:
383 spin_unlock_irqrestore(&ring->lock, flags);
384 return frame;
385}
386EXPORT_SYMBOL_GPL(tb_ring_poll);
387
388static void __ring_interrupt_mask(struct tb_ring *ring, bool mask)
389{
390 int idx = ring_interrupt_index(ring);
391 int reg = REG_RING_INTERRUPT_BASE + idx / 32 * 4;
392 int bit = idx % 32;
393 u32 val;
394
395 val = ioread32(ring->nhi->iobase + reg);
396 if (mask)
397 val &= ~BIT(bit);
398 else
399 val |= BIT(bit);
400 iowrite32(val, ring->nhi->iobase + reg);
401}
402
403/* Both @nhi->lock and @ring->lock should be held */
404static void __ring_interrupt(struct tb_ring *ring)
405{
406 if (!ring->running)
407 return;
408
409 if (ring->start_poll) {
410 __ring_interrupt_mask(ring, true);
411 ring->start_poll(ring->poll_data);
412 } else {
413 schedule_work(&ring->work);
414 }
415}
416
417/**
418 * tb_ring_poll_complete() - Re-start interrupt for the ring
419 * @ring: Ring to re-start the interrupt
420 *
421 * This will re-start (unmask) the ring interrupt once the user is done
422 * with polling.
423 */
424void tb_ring_poll_complete(struct tb_ring *ring)
425{
426 unsigned long flags;
427
428 spin_lock_irqsave(&ring->nhi->lock, flags);
429 spin_lock(&ring->lock);
430 if (ring->start_poll)
431 __ring_interrupt_mask(ring, false);
432 spin_unlock(&ring->lock);
433 spin_unlock_irqrestore(&ring->nhi->lock, flags);
434}
435EXPORT_SYMBOL_GPL(tb_ring_poll_complete);
436
437static void ring_clear_msix(const struct tb_ring *ring)
438{
439 int bit;
440
441 if (ring->nhi->quirks & QUIRK_AUTO_CLEAR_INT)
442 return;
443
444 bit = ring_interrupt_index(ring) & 31;
445 if (ring->is_tx)
446 iowrite32(BIT(bit), ring->nhi->iobase + REG_RING_INT_CLEAR);
447 else
448 iowrite32(BIT(bit), ring->nhi->iobase + REG_RING_INT_CLEAR +
449 4 * (ring->nhi->hop_count / 32));
450}
451
452irqreturn_t ring_msix(int irq, void *data)
453{
454 struct tb_ring *ring = data;
455
456 spin_lock(&ring->nhi->lock);
457 ring_clear_msix(ring);
458 spin_lock(&ring->lock);
459 __ring_interrupt(ring);
460 spin_unlock(&ring->lock);
461 spin_unlock(&ring->nhi->lock);
462
463 return IRQ_HANDLED;
464}
465
466static int nhi_alloc_hop(struct tb_nhi *nhi, struct tb_ring *ring)
467{
468 unsigned int start_hop = RING_FIRST_USABLE_HOPID;
469 int ret = 0;
470
471 if (nhi->quirks & QUIRK_E2E) {
472 start_hop = RING_FIRST_USABLE_HOPID + 1;
473 if (ring->flags & RING_FLAG_E2E && !ring->is_tx) {
474 dev_dbg(nhi->dev, "quirking E2E TX HopID %u -> %u\n",
475 ring->e2e_tx_hop, RING_E2E_RESERVED_HOPID);
476 ring->e2e_tx_hop = RING_E2E_RESERVED_HOPID;
477 }
478 }
479
480 spin_lock_irq(&nhi->lock);
481
482 if (ring->hop < 0) {
483 unsigned int i;
484
485 /*
486 * Automatically allocate HopID from the non-reserved
487 * range 1 .. hop_count - 1.
488 */
489 for (i = start_hop; i < nhi->hop_count; i++) {
490 if (ring->is_tx) {
491 if (!nhi->tx_rings[i]) {
492 ring->hop = i;
493 break;
494 }
495 } else {
496 if (!nhi->rx_rings[i]) {
497 ring->hop = i;
498 break;
499 }
500 }
501 }
502 }
503
504 if (ring->hop > 0 && ring->hop < start_hop) {
505 dev_warn(nhi->dev, "invalid hop: %d\n", ring->hop);
506 ret = -EINVAL;
507 goto err_unlock;
508 }
509 if (ring->hop < 0 || ring->hop >= nhi->hop_count) {
510 dev_warn(nhi->dev, "invalid hop: %d\n", ring->hop);
511 ret = -EINVAL;
512 goto err_unlock;
513 }
514 if (ring->is_tx && nhi->tx_rings[ring->hop]) {
515 dev_warn(nhi->dev, "TX hop %d already allocated\n",
516 ring->hop);
517 ret = -EBUSY;
518 goto err_unlock;
519 }
520 if (!ring->is_tx && nhi->rx_rings[ring->hop]) {
521 dev_warn(nhi->dev, "RX hop %d already allocated\n",
522 ring->hop);
523 ret = -EBUSY;
524 goto err_unlock;
525 }
526
527 if (ring->is_tx)
528 nhi->tx_rings[ring->hop] = ring;
529 else
530 nhi->rx_rings[ring->hop] = ring;
531
532err_unlock:
533 spin_unlock_irq(&nhi->lock);
534
535 return ret;
536}
537
538static struct tb_ring *tb_ring_alloc(struct tb_nhi *nhi, u32 hop, int size,
539 bool transmit, unsigned int flags,
540 int e2e_tx_hop, u16 sof_mask, u16 eof_mask,
541 void (*start_poll)(void *),
542 void *poll_data)
543{
544 struct tb_ring *ring = NULL;
545
546 dev_dbg(nhi->dev, "allocating %s ring %d of size %d\n",
547 transmit ? "TX" : "RX", hop, size);
548
549 if ((flags & RING_FLAG_NO_INTERRUPT) && start_poll) {
550 dev_WARN(nhi->dev,
551 "start_poll() and NO_INTERRUPT cannot be used at the same time\n");
552 return NULL;
553 }
554
555 ring = kzalloc_obj(*ring);
556 if (!ring)
557 return NULL;
558
559 spin_lock_init(&ring->lock);
560 INIT_LIST_HEAD(&ring->queue);
561 INIT_LIST_HEAD(&ring->in_flight);
562 INIT_WORK(&ring->work, ring_work);
563 init_waitqueue_head(&ring->wait);
564 lockdep_register_key(&ring->lock_key);
565 lockdep_init_map(&ring->work.lockdep_map, "ring.work", &ring->lock_key, 0);
566
567 ring->nhi = nhi;
568 ring->hop = hop;
569 ring->is_tx = transmit;
570 ring->size = size;
571 ring->flags = flags;
572 ring->e2e_tx_hop = e2e_tx_hop;
573 ring->sof_mask = sof_mask;
574 ring->eof_mask = eof_mask;
575 ring->head = 0;
576 ring->tail = 0;
577 ring->running = false;
578 ring->start_poll = start_poll;
579 ring->poll_data = poll_data;
580
581 ring->descriptors = dma_alloc_coherent(ring->nhi->dev,
582 size * sizeof(*ring->descriptors),
583 &ring->descriptors_dma, GFP_KERNEL | __GFP_ZERO);
584 if (!ring->descriptors)
585 goto err_free_ring;
586
587 if (!(flags & RING_FLAG_NO_INTERRUPT) && nhi->ops->request_ring_irq) {
588 if (nhi->ops->request_ring_irq(ring, flags & RING_FLAG_NO_SUSPEND))
589 goto err_free_descs;
590 }
591
592 if (nhi_alloc_hop(nhi, ring))
593 goto err_release_msix;
594
595 return ring;
596
597err_release_msix:
598 if (nhi->ops->release_ring_irq)
599 nhi->ops->release_ring_irq(ring);
600err_free_descs:
601 dma_free_coherent(ring->nhi->dev,
602 ring->size * sizeof(*ring->descriptors),
603 ring->descriptors, ring->descriptors_dma);
604err_free_ring:
605 lockdep_unregister_key(&ring->lock_key);
606 kfree(ring);
607
608 return NULL;
609}
610
611/**
612 * tb_ring_alloc_tx() - Allocate DMA ring for transmit
613 * @nhi: Pointer to the NHI the ring is to be allocated
614 * @hop: HopID (ring) to allocate
615 * @size: Number of entries in the ring
616 * @flags: Flags for the ring
617 *
618 * Return: Pointer to &struct tb_ring, %NULL otherwise.
619 */
620struct tb_ring *tb_ring_alloc_tx(struct tb_nhi *nhi, int hop, int size,
621 unsigned int flags)
622{
623 return tb_ring_alloc(nhi, hop, size, true, flags, 0, 0, 0, NULL, NULL);
624}
625EXPORT_SYMBOL_GPL(tb_ring_alloc_tx);
626
627/**
628 * tb_ring_alloc_rx() - Allocate DMA ring for receive
629 * @nhi: Pointer to the NHI the ring is to be allocated
630 * @hop: HopID (ring) to allocate. Pass %-1 for automatic allocation.
631 * @size: Number of entries in the ring
632 * @flags: Flags for the ring
633 * @e2e_tx_hop: Transmit HopID when E2E is enabled in @flags
634 * @sof_mask: Mask of PDF values that start a frame
635 * @eof_mask: Mask of PDF values that end a frame
636 * @start_poll: If not %NULL the ring will call this function when an
637 * interrupt is triggered and masked, instead of callback
638 * in each Rx frame.
639 * @poll_data: Optional data passed to @start_poll
640 *
641 * Return: Pointer to &struct tb_ring, %NULL otherwise.
642 */
643struct tb_ring *tb_ring_alloc_rx(struct tb_nhi *nhi, int hop, int size,
644 unsigned int flags, int e2e_tx_hop,
645 u16 sof_mask, u16 eof_mask,
646 void (*start_poll)(void *), void *poll_data)
647{
648 return tb_ring_alloc(nhi, hop, size, false, flags, e2e_tx_hop, sof_mask, eof_mask,
649 start_poll, poll_data);
650}
651EXPORT_SYMBOL_GPL(tb_ring_alloc_rx);
652
653/**
654 * tb_ring_start() - enable a ring
655 * @ring: Ring to start
656 *
657 * Must not be invoked in parallel with tb_ring_stop().
658 */
659void tb_ring_start(struct tb_ring *ring)
660{
661 u16 frame_size;
662 u32 flags;
663
664 spin_lock_irq(&ring->nhi->lock);
665 spin_lock(&ring->lock);
666 if (ring->nhi->going_away)
667 goto err;
668 if (ring->running) {
669 dev_WARN(ring->nhi->dev, "ring already started\n");
670 goto err;
671 }
672 dev_dbg(ring->nhi->dev, "starting %s %d\n",
673 RING_TYPE(ring), ring->hop);
674
675 if (ring->flags & RING_FLAG_FRAME) {
676 /* Means 4096 */
677 frame_size = 0;
678 flags = RING_FLAG_ENABLE;
679 } else {
680 frame_size = TB_FRAME_SIZE;
681 flags = RING_FLAG_ENABLE | RING_FLAG_RAW;
682 }
683
684 ring_iowrite64desc(ring, ring->descriptors_dma, 0);
685 if (ring->is_tx) {
686 ring_iowrite32desc(ring, ring->size, 12);
687 ring_iowrite32options(ring, 0, 4);
688 ring_iowrite32options(ring, flags, 0);
689 } else {
690 u32 sof_eof_mask = ring->sof_mask << 16 | ring->eof_mask;
691
692 ring_iowrite32desc(ring, (frame_size << 16) | ring->size, 12);
693 ring_iowrite32options(ring, sof_eof_mask, 4);
694 ring_iowrite32options(ring, flags, 0);
695 }
696
697 /*
698 * Now that the ring valid bit is set we can configure E2E if
699 * enabled for the ring.
700 */
701 if (ring->flags & RING_FLAG_E2E) {
702 if (!ring->is_tx) {
703 u32 hop;
704
705 hop = ring->e2e_tx_hop << REG_RX_OPTIONS_E2E_HOP_SHIFT;
706 hop &= REG_RX_OPTIONS_E2E_HOP_MASK;
707 flags |= hop;
708
709 dev_dbg(ring->nhi->dev,
710 "enabling E2E for %s %d with TX HopID %d\n",
711 RING_TYPE(ring), ring->hop, ring->e2e_tx_hop);
712 } else {
713 dev_dbg(ring->nhi->dev, "enabling E2E for %s %d\n",
714 RING_TYPE(ring), ring->hop);
715 }
716
717 flags |= RING_FLAG_E2E_FLOW_CONTROL;
718 ring_iowrite32options(ring, flags, 0);
719 }
720
721 if (!(ring->flags & RING_FLAG_NO_INTERRUPT))
722 ring_interrupt_active(ring, true);
723 ring->running = true;
724err:
725 spin_unlock(&ring->lock);
726 spin_unlock_irq(&ring->nhi->lock);
727}
728EXPORT_SYMBOL_GPL(tb_ring_start);
729
730static bool tb_ring_empty(struct tb_ring *ring)
731{
732 guard(spinlock_irqsave)(&ring->lock);
733 return list_empty(&ring->in_flight);
734}
735
736/**
737 * tb_ring_flush() - Waits for a ring to be empty
738 * @ring: Ring to wait
739 * @timeout_msec: Timeout in ms how long to wait.
740 *
741 * This can be called before stopping a ring to make sure all the frames
742 * submitted prior have been completed.
743 *
744 * Return: %true if the ring is empty now, %false otherwise.
745 */
746bool tb_ring_flush(struct tb_ring *ring, unsigned int timeout_msec)
747{
748 if (!wait_event_timeout(ring->wait, tb_ring_empty(ring),
749 msecs_to_jiffies(timeout_msec)))
750 return false;
751 return tb_ring_empty(ring);
752}
753EXPORT_SYMBOL_GPL(tb_ring_flush);
754
755/**
756 * tb_ring_stop() - shutdown a ring
757 * @ring: Ring to stop
758 *
759 * Must not be invoked from a callback.
760 *
761 * This method will disable the ring. Further calls to
762 * tb_ring_tx/tb_ring_rx will return -ESHUTDOWN until ring_stop has been
763 * called.
764 *
765 * All enqueued frames will be canceled and their callbacks will be executed
766 * with frame->canceled set to true (on the callback thread). This method
767 * returns only after all callback invocations have finished.
768 */
769void tb_ring_stop(struct tb_ring *ring)
770{
771 spin_lock_irq(&ring->nhi->lock);
772 spin_lock(&ring->lock);
773 dev_dbg(ring->nhi->dev, "stopping %s %d\n",
774 RING_TYPE(ring), ring->hop);
775 if (ring->nhi->going_away)
776 goto err;
777 if (!ring->running) {
778 dev_WARN(ring->nhi->dev, "%s %d already stopped\n",
779 RING_TYPE(ring), ring->hop);
780 goto err;
781 }
782 if (!(ring->flags & RING_FLAG_NO_INTERRUPT))
783 ring_interrupt_active(ring, false);
784
785 ring_iowrite32options(ring, 0, 0);
786 ring_iowrite64desc(ring, 0, 0);
787 ring_iowrite32desc(ring, 0, 8);
788 ring_iowrite32desc(ring, 0, 12);
789 ring->head = 0;
790 ring->tail = 0;
791 ring->running = false;
792
793err:
794 spin_unlock(&ring->lock);
795 spin_unlock_irq(&ring->nhi->lock);
796
797 /*
798 * schedule ring->work to invoke callbacks on all remaining frames.
799 */
800 schedule_work(&ring->work);
801 flush_work(&ring->work);
802}
803EXPORT_SYMBOL_GPL(tb_ring_stop);
804
805/*
806 * tb_ring_free() - free ring
807 *
808 * When this method returns all invocations of ring->callback will have
809 * finished.
810 *
811 * Ring must be stopped.
812 *
813 * Must NOT be called from ring_frame->callback!
814 */
815void tb_ring_free(struct tb_ring *ring)
816{
817 struct tb_nhi *nhi = ring->nhi;
818
819 spin_lock_irq(&ring->nhi->lock);
820 /*
821 * Dissociate the ring from the NHI. This also ensures that
822 * nhi_interrupt_work cannot reschedule ring->work.
823 */
824 if (ring->is_tx)
825 ring->nhi->tx_rings[ring->hop] = NULL;
826 else
827 ring->nhi->rx_rings[ring->hop] = NULL;
828
829 if (ring->running) {
830 dev_WARN(ring->nhi->dev, "%s %d still running\n",
831 RING_TYPE(ring), ring->hop);
832 }
833 spin_unlock_irq(&ring->nhi->lock);
834
835 if (nhi->ops->release_ring_irq)
836 nhi->ops->release_ring_irq(ring);
837
838 dma_free_coherent(ring->nhi->dev,
839 ring->size * sizeof(*ring->descriptors),
840 ring->descriptors, ring->descriptors_dma);
841
842 ring->descriptors = NULL;
843 ring->descriptors_dma = 0;
844
845
846 dev_dbg(ring->nhi->dev, "freeing %s %d\n", RING_TYPE(ring),
847 ring->hop);
848
849 /*
850 * ring->work can no longer be scheduled (it is scheduled only
851 * by nhi_interrupt_work, ring_stop and ring_msix). Wait for it
852 * to finish before freeing the ring.
853 */
854 flush_work(&ring->work);
855 lockdep_unregister_key(&ring->lock_key);
856 kfree(ring);
857}
858EXPORT_SYMBOL_GPL(tb_ring_free);
859
860/**
861 * tb_ring_throttling() - Configure throttling for ring interrupt
862 * @ring: Ring to configure
863 * @interval_nsec: Interval counter for moderation (in ns), %0 disables
864 *
865 * Enables or disables ring interrupt throttling. The ring must be
866 * stopped for this to be called. Granularity is 256 ns.
867 *
868 * Return: %0 on success, negative errno otherwise.
869 */
870int tb_ring_throttling(struct tb_ring *ring, unsigned int interval_nsec)
871{
872 guard(spinlock_irqsave)(&ring->lock);
873 if (WARN_ON_ONCE(ring->running))
874 return -EBUSY;
875 ring->interval_nsec = interval_nsec;
876 return 0;
877}
878EXPORT_SYMBOL_GPL(tb_ring_throttling);
879
880/**
881 * nhi_mailbox_cmd() - Send a command through NHI mailbox
882 * @nhi: Pointer to the NHI structure
883 * @cmd: Command to send
884 * @data: Data to be send with the command
885 *
886 * Sends mailbox command to the firmware running on NHI.
887 *
888 * Return: %0 on success, negative errno otherwise.
889 */
890int nhi_mailbox_cmd(struct tb_nhi *nhi, enum nhi_mailbox_cmd cmd, u32 data)
891{
892 ktime_t timeout;
893 u32 val;
894
895 iowrite32(data, nhi->iobase + REG_INMAIL_DATA);
896
897 val = ioread32(nhi->iobase + REG_INMAIL_CMD);
898 val &= ~(REG_INMAIL_CMD_MASK | REG_INMAIL_ERROR);
899 val |= REG_INMAIL_OP_REQUEST | cmd;
900 iowrite32(val, nhi->iobase + REG_INMAIL_CMD);
901
902 timeout = ktime_add_ms(ktime_get(), NHI_MAILBOX_TIMEOUT);
903 do {
904 val = ioread32(nhi->iobase + REG_INMAIL_CMD);
905 if (!(val & REG_INMAIL_OP_REQUEST))
906 break;
907 usleep_range(10, 20);
908 } while (ktime_before(ktime_get(), timeout));
909
910 if (val & REG_INMAIL_OP_REQUEST)
911 return -ETIMEDOUT;
912 if (val & REG_INMAIL_ERROR)
913 return -EIO;
914
915 return 0;
916}
917
918/**
919 * nhi_mailbox_mode() - Return current firmware operation mode
920 * @nhi: Pointer to the NHI structure
921 *
922 * The function reads current firmware operation mode using NHI mailbox
923 * registers and returns it to the caller.
924 *
925 * Return: &enum nhi_fw_mode.
926 */
927enum nhi_fw_mode nhi_mailbox_mode(struct tb_nhi *nhi)
928{
929 u32 val;
930
931 val = ioread32(nhi->iobase + REG_OUTMAIL_CMD);
932 val &= REG_OUTMAIL_CMD_OPMODE_MASK;
933 val >>= REG_OUTMAIL_CMD_OPMODE_SHIFT;
934
935 return (enum nhi_fw_mode)val;
936}
937
938void nhi_interrupt_work(struct work_struct *work)
939{
940 struct tb_nhi *nhi = container_of(work, typeof(*nhi), interrupt_work);
941 int value = 0; /* Suppress uninitialized usage warning. */
942 int bit;
943 int hop = -1;
944 int type = 0; /* current interrupt type 0: TX, 1: RX, 2: RX overflow */
945 struct tb_ring *ring;
946
947 spin_lock_irq(&nhi->lock);
948
949 /*
950 * Starting at REG_RING_NOTIFY_BASE there are three status bitfields
951 * (TX, RX, RX overflow). We iterate over the bits and read a new
952 * dwords as required. The registers are cleared on read.
953 */
954 for (bit = 0; bit < 3 * nhi->hop_count; bit++) {
955 if (bit % 32 == 0)
956 value = ioread32(nhi->iobase
957 + REG_RING_NOTIFY_BASE
958 + 4 * (bit / 32));
959 if (++hop == nhi->hop_count) {
960 hop = 0;
961 type++;
962 }
963 if ((value & (1 << (bit % 32))) == 0)
964 continue;
965 if (type == 2) {
966 dev_warn(nhi->dev, "RX overflow for ring %d\n", hop);
967 continue;
968 }
969 if (type == 0)
970 ring = nhi->tx_rings[hop];
971 else
972 ring = nhi->rx_rings[hop];
973 if (ring == NULL) {
974 dev_warn(nhi->dev,
975 "got interrupt for inactive %s ring %d\n",
976 type ? "RX" : "TX",
977 hop);
978 continue;
979 }
980
981 spin_lock(&ring->lock);
982 __ring_interrupt(ring);
983 spin_unlock(&ring->lock);
984 }
985 spin_unlock_irq(&nhi->lock);
986}
987
988irqreturn_t nhi_msi(int irq, void *data)
989{
990 struct tb_nhi *nhi = data;
991 schedule_work(&nhi->interrupt_work);
992 return IRQ_HANDLED;
993}
994
995static int __nhi_suspend_noirq(struct device *dev, bool wakeup)
996{
997 struct tb *tb = dev_get_drvdata(dev);
998 struct tb_nhi *nhi = tb->nhi;
999 int ret;
1000
1001 ret = tb_domain_suspend_noirq(tb);
1002 if (ret)
1003 return ret;
1004
1005 if (nhi->ops->suspend_noirq) {
1006 ret = nhi->ops->suspend_noirq(tb->nhi, wakeup);
1007 if (ret)
1008 return ret;
1009 }
1010
1011 return 0;
1012}
1013
1014static int nhi_suspend_noirq(struct device *dev)
1015{
1016 return __nhi_suspend_noirq(dev, device_may_wakeup(dev));
1017}
1018
1019static int nhi_freeze_noirq(struct device *dev)
1020{
1021 struct tb *tb = dev_get_drvdata(dev);
1022
1023 return tb_domain_freeze_noirq(tb);
1024}
1025
1026static int nhi_thaw_noirq(struct device *dev)
1027{
1028 struct tb *tb = dev_get_drvdata(dev);
1029
1030 return tb_domain_thaw_noirq(tb);
1031}
1032
1033static bool nhi_wake_supported(struct device *dev)
1034{
1035 u8 val;
1036
1037 /*
1038 * If power rails are sustainable for wakeup from S4 this
1039 * property is set by the BIOS.
1040 */
1041 if (!device_property_read_u8(dev, "WAKE_SUPPORTED", &val))
1042 return !!val;
1043
1044 return true;
1045}
1046
1047static int nhi_poweroff_noirq(struct device *dev)
1048{
1049 bool wakeup;
1050
1051 wakeup = device_may_wakeup(dev) && nhi_wake_supported(dev);
1052 return __nhi_suspend_noirq(dev, wakeup);
1053}
1054
1055static int nhi_resume_noirq(struct device *dev)
1056{
1057 struct tb *tb = dev_get_drvdata(dev);
1058 struct tb_nhi *nhi = tb->nhi;
1059 int ret;
1060
1061 /*
1062 * Check that the device is still there. It may be that the user
1063 * unplugged last device which causes the host controller to go
1064 * away on PCs.
1065 */
1066 if ((nhi->ops->is_present && !nhi->ops->is_present(nhi))) {
1067 nhi->going_away = true;
1068 } else if (nhi->ops->resume_noirq) {
1069 ret = nhi->ops->resume_noirq(nhi);
1070 if (ret)
1071 return ret;
1072 }
1073
1074 return tb_domain_resume_noirq(tb);
1075}
1076
1077static int nhi_suspend(struct device *dev)
1078{
1079 struct tb *tb = dev_get_drvdata(dev);
1080
1081 return tb_domain_suspend(tb);
1082}
1083
1084static void nhi_complete(struct device *dev)
1085{
1086 struct tb *tb = dev_get_drvdata(dev);
1087
1088 /*
1089 * If we were runtime suspended when system suspend started,
1090 * schedule runtime resume now. It should bring the domain back
1091 * to functional state.
1092 */
1093 if (pm_runtime_suspended(dev))
1094 pm_runtime_resume(dev);
1095 else
1096 tb_domain_complete(tb);
1097}
1098
1099static int nhi_runtime_suspend(struct device *dev)
1100{
1101 struct tb *tb = dev_get_drvdata(dev);
1102 struct tb_nhi *nhi = tb->nhi;
1103 int ret;
1104
1105 ret = tb_domain_runtime_suspend(tb);
1106 if (ret)
1107 return ret;
1108
1109 if (nhi->ops->runtime_suspend) {
1110 ret = nhi->ops->runtime_suspend(tb->nhi);
1111 if (ret)
1112 return ret;
1113 }
1114 return 0;
1115}
1116
1117static int nhi_runtime_resume(struct device *dev)
1118{
1119 struct tb *tb = dev_get_drvdata(dev);
1120 struct tb_nhi *nhi = tb->nhi;
1121 int ret;
1122
1123 if (nhi->ops->runtime_resume) {
1124 ret = nhi->ops->runtime_resume(nhi);
1125 if (ret)
1126 return ret;
1127 }
1128
1129 return tb_domain_runtime_resume(tb);
1130}
1131
1132void nhi_shutdown(struct tb_nhi *nhi)
1133{
1134 int i;
1135
1136 dev_dbg(nhi->dev, "shutdown\n");
1137
1138 for (i = 0; i < nhi->hop_count; i++) {
1139 if (nhi->tx_rings[i])
1140 dev_WARN(nhi->dev,
1141 "TX ring %d is still active\n", i);
1142 if (nhi->rx_rings[i])
1143 dev_WARN(nhi->dev,
1144 "RX ring %d is still active\n", i);
1145 }
1146 nhi_disable_interrupts(nhi);
1147
1148 if (nhi->ops->shutdown)
1149 nhi->ops->shutdown(nhi);
1150}
1151
1152static void nhi_reset(struct tb_nhi *nhi)
1153{
1154 ktime_t timeout;
1155 u32 val;
1156
1157 val = ioread32(nhi->iobase + REG_CAPS);
1158 /* Reset only v2 and later routers */
1159 if (FIELD_GET(REG_CAPS_VERSION_MASK, val) < REG_CAPS_VERSION_2)
1160 return;
1161
1162 if (!host_reset) {
1163 dev_dbg(nhi->dev, "skipping host router reset\n");
1164 return;
1165 }
1166
1167 iowrite32(REG_RESET_HRR, nhi->iobase + REG_RESET);
1168 msleep(100);
1169
1170 timeout = ktime_add_ms(ktime_get(), 500);
1171 do {
1172 val = ioread32(nhi->iobase + REG_RESET);
1173 if (!(val & REG_RESET_HRR)) {
1174 dev_warn(nhi->dev, "host router reset successful\n");
1175 return;
1176 }
1177 usleep_range(10, 20);
1178 } while (ktime_before(ktime_get(), timeout));
1179
1180 dev_warn(nhi->dev, "timeout resetting host router\n");
1181}
1182
1183static struct tb *nhi_select_cm(struct tb_nhi *nhi)
1184{
1185 struct tb *tb;
1186
1187 /*
1188 * USB4 case is simple. If we got control of any of the
1189 * capabilities, we use software CM.
1190 */
1191 if (tb_acpi_is_native())
1192 return tb_probe(nhi);
1193
1194 /*
1195 * Either firmware based CM is running (we did not get control
1196 * from the firmware) or this is pre-USB4 PC so try first
1197 * firmware CM and then fallback to software CM.
1198 */
1199 tb = icm_probe(nhi);
1200 if (!tb)
1201 tb = tb_probe(nhi);
1202
1203 return tb;
1204}
1205
1206int nhi_probe(struct tb_nhi *nhi)
1207{
1208 struct device *dev = nhi->dev;
1209 struct tb *tb;
1210 int res;
1211
1212 if (!nhi->ops)
1213 return dev_err_probe(dev, -EINVAL, "NHI ops not set\n");
1214
1215 if (!nhi->ops->init_interrupts)
1216 return dev_err_probe(dev, -EINVAL, "missing required NHI ops\n");
1217
1218 nhi->hop_count = ioread32(nhi->iobase + REG_CAPS) & 0x3ff;
1219 dev_dbg(dev, "total paths: %d\n", nhi->hop_count);
1220
1221 nhi->tx_rings = devm_kcalloc(dev, nhi->hop_count,
1222 sizeof(*nhi->tx_rings), GFP_KERNEL);
1223 nhi->rx_rings = devm_kcalloc(dev, nhi->hop_count,
1224 sizeof(*nhi->rx_rings), GFP_KERNEL);
1225 if (!nhi->tx_rings || !nhi->rx_rings)
1226 return -ENOMEM;
1227
1228 nhi_reset(nhi);
1229
1230 /* In case someone left them on. */
1231 nhi_disable_interrupts(nhi);
1232
1233 res = nhi->ops->init_interrupts(nhi);
1234 if (res)
1235 return dev_err_probe(dev, res, "cannot enable interrupts, aborting\n");
1236
1237 spin_lock_init(&nhi->lock);
1238
1239 res = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64));
1240 if (res)
1241 return dev_err_probe(dev, res, "failed to set DMA mask\n");
1242
1243 if (nhi->ops->init) {
1244 res = nhi->ops->init(nhi);
1245 if (res)
1246 return dev_err_probe(dev, res, "NHI specific init failed\n");
1247 }
1248
1249 init_completion(&nhi->domain_released);
1250
1251 tb = nhi_select_cm(nhi);
1252 if (!tb)
1253 return dev_err_probe(dev, -ENODEV,
1254 "failed to determine connection manager, aborting\n");
1255
1256 dev_dbg(dev, "NHI initialized, starting thunderbolt\n");
1257
1258 nhi->host_reset = host_reset;
1259
1260 res = tb_domain_add(tb, host_reset);
1261 if (res) {
1262 /*
1263 * At this point the RX/TX rings might already have been
1264 * activated. Do a proper shutdown.
1265 */
1266 tb_domain_put(tb);
1267 wait_for_completion(&nhi->domain_released);
1268 nhi_shutdown(nhi);
1269 return dev_err_probe(dev, res, "failed to add domain\n");
1270 }
1271 dev_set_drvdata(dev, tb);
1272
1273 device_wakeup_enable(dev);
1274
1275 pm_runtime_allow(dev);
1276 pm_runtime_set_autosuspend_delay(dev, TB_AUTOSUSPEND_DELAY);
1277 pm_runtime_use_autosuspend(dev);
1278 pm_runtime_put_autosuspend(dev);
1279
1280 return 0;
1281}
1282
1283/*
1284 * The tunneled pci bridges are siblings of us. Use resume_noirq to reenable
1285 * the tunnels asap. A corresponding pci quirk blocks the downstream bridges
1286 * resume_noirq until we are done.
1287 */
1288const struct dev_pm_ops nhi_pm_ops = {
1289 .suspend_noirq = nhi_suspend_noirq,
1290 .resume_noirq = nhi_resume_noirq,
1291 .freeze_noirq = nhi_freeze_noirq, /*
1292 * we just disable hotplug, the
1293 * pci-tunnels stay alive.
1294 */
1295 .thaw_noirq = nhi_thaw_noirq,
1296 .restore_noirq = nhi_resume_noirq,
1297 .suspend = nhi_suspend,
1298 .poweroff_noirq = nhi_poweroff_noirq,
1299 .poweroff = nhi_suspend,
1300 .complete = nhi_complete,
1301 .runtime_suspend = nhi_runtime_suspend,
1302 .runtime_resume = nhi_runtime_resume,
1303};