projects/linux·blob master · refs

1437 lines · 37.0 KiB

1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Microchip Switchtec(tm) DMA Controller Driver
4 * Copyright (c) 2025, Kelvin Cao <[email protected]>
5 * Copyright (c) 2025, Microchip Corporation
6 */
7
8#include <linux/bitfield.h>
9#include <linux/circ_buf.h>
10#include <linux/dmaengine.h>
11#include <linux/module.h>
12#include <linux/pci.h>
13#include <linux/delay.h>
14#include <linux/iopoll.h>
15
16#include "dmaengine.h"
17
18MODULE_DESCRIPTION("Switchtec PCIe Switch DMA Engine");
19MODULE_LICENSE("GPL");
20MODULE_AUTHOR("Kelvin Cao");
21
22#define SWITCHTEC_DMAC_CHAN_CTRL_OFFSET 0x1000
23#define SWITCHTEC_DMAC_CHAN_CFG_STS_OFFSET 0x160000
24
25#define SWITCHTEC_DMA_CHAN_HW_REGS_SIZE 0x1000
26#define SWITCHTEC_DMA_CHAN_FW_REGS_SIZE 0x80
27
28#define SWITCHTEC_REG_CAP 0x80
29#define SWITCHTEC_REG_CHAN_CNT 0x84
30#define SWITCHTEC_REG_TAG_LIMIT 0x90
31#define SWITCHTEC_REG_CHAN_STS_VEC 0x94
32#define SWITCHTEC_REG_SE_BUF_CNT 0x98
33#define SWITCHTEC_REG_SE_BUF_BASE 0x9a
34
35#define SWITCHTEC_DESC_MAX_SIZE 0x100000
36
37#define SWITCHTEC_CHAN_CTRL_PAUSE BIT(0)
38#define SWITCHTEC_CHAN_CTRL_HALT BIT(1)
39#define SWITCHTEC_CHAN_CTRL_RESET BIT(2)
40#define SWITCHTEC_CHAN_CTRL_ERR_PAUSE BIT(3)
41
42#define SWITCHTEC_CHAN_STS_PAUSED BIT(9)
43#define SWITCHTEC_CHAN_STS_HALTED BIT(10)
44#define SWITCHTEC_CHAN_STS_PAUSED_MASK GENMASK(29, 13)
45
46#define SWITCHTEC_INVALID_HFID 0xffff
47
48#define SWITCHTEC_DMA_SQ_SIZE SZ_32K
49#define SWITCHTEC_DMA_CQ_SIZE SZ_32K
50
51#define SWITCHTEC_DMA_RING_SIZE SZ_32K
52
53static const char * const channel_status_str[] = {
54 [13] = "received a VDM with length error status",
55 [14] = "received a VDM or Cpl with Unsupported Request error status",
56 [15] = "received a VDM or Cpl with Completion Abort error status",
57 [16] = "received a VDM with ECRC error status",
58 [17] = "received a VDM with EP error status",
59 [18] = "received a VDM with Reserved Cpl error status",
60 [19] = "received only part of split SE CplD",
61 [20] = "the ISP_DMAC detected a Completion Time Out",
62 [21] = "received a Cpl with Unsupported Request status",
63 [22] = "received a Cpl with Completion Abort status",
64 [23] = "received a Cpl with a reserved status",
65 [24] = "received a TLP with ECRC error status in its metadata",
66 [25] = "received a TLP with the EP bit set in the header",
67 [26] = "the ISP_DMAC tried to process a SE with an invalid Connection ID",
68 [27] = "the ISP_DMAC tried to process a SE with an invalid Remote Host interrupt",
69 [28] = "a reserved opcode was detected in an SE",
70 [29] = "received a SE Cpl with error status",
71};
72
73struct chan_hw_regs {
74 u16 cq_head;
75 u16 rsvd1;
76 u16 sq_tail;
77 u16 rsvd2;
78 u8 ctrl;
79 u8 rsvd3[3];
80 u16 status;
81 u16 rsvd4;
82};
83
84#define PERF_BURST_SCALE_MASK GENMASK_U32(3, 2)
85#define PERF_MRRS_MASK GENMASK_U32(6, 4)
86#define PERF_INTERVAL_MASK GENMASK_U32(10, 8)
87#define PERF_BURST_SIZE_MASK GENMASK_U32(14, 12)
88#define PERF_ARB_WEIGHT_MASK GENMASK_U32(31, 24)
89
90#define SE_BUF_BASE_MASK GENMASK_U32(10, 2)
91#define SE_BUF_LEN_MASK GENMASK_U32(20, 12)
92#define SE_THRESH_MASK GENMASK_U32(31, 23)
93
94#define SWITCHTEC_CHAN_ENABLE BIT(1)
95
96struct chan_fw_regs {
97 u32 valid_en_se;
98 u32 cq_base_lo;
99 u32 cq_base_hi;
100 u16 cq_size;
101 u16 rsvd1;
102 u32 sq_base_lo;
103 u32 sq_base_hi;
104 u16 sq_size;
105 u16 rsvd2;
106 u32 int_vec;
107 u32 perf_cfg;
108 u32 rsvd3;
109 u32 perf_latency_selector;
110 u32 perf_fetched_se_cnt_lo;
111 u32 perf_fetched_se_cnt_hi;
112 u32 perf_byte_cnt_lo;
113 u32 perf_byte_cnt_hi;
114 u32 rsvd4;
115 u16 perf_se_pending;
116 u16 perf_se_buf_empty;
117 u32 perf_chan_idle;
118 u32 perf_lat_max;
119 u32 perf_lat_min;
120 u32 perf_lat_last;
121 u16 sq_current;
122 u16 sq_phase;
123 u16 cq_current;
124 u16 cq_phase;
125};
126
127struct switchtec_dma_chan {
128 struct switchtec_dma_dev *swdma_dev;
129 struct dma_chan dma_chan;
130 struct chan_hw_regs __iomem *mmio_chan_hw;
131 struct chan_fw_regs __iomem *mmio_chan_fw;
132
133 /* Serialize hardware control register access */
134 spinlock_t hw_ctrl_lock;
135
136 struct tasklet_struct desc_task;
137
138 /* Serialize descriptor preparation */
139 spinlock_t submit_lock;
140 bool ring_active;
141 int cid;
142
143 /* Serialize completion processing */
144 spinlock_t complete_lock;
145 bool comp_ring_active;
146
147 /* channel index and irq */
148 int index;
149 int irq;
150
151 /*
152 * In driver context, head is advanced by producer while
153 * tail is advanced by consumer.
154 */
155
156 /* the head and tail for both desc_ring and hw_sq */
157 int head;
158 int tail;
159 int phase_tag;
160 struct switchtec_dma_hw_se_desc *hw_sq;
161 dma_addr_t dma_addr_sq;
162
163 /* the tail for hw_cq */
164 int cq_tail;
165 struct switchtec_dma_hw_ce *hw_cq;
166 dma_addr_t dma_addr_cq;
167
168 struct list_head list;
169
170 struct switchtec_dma_desc *desc_ring[SWITCHTEC_DMA_RING_SIZE];
171};
172
173struct switchtec_dma_dev {
174 struct dma_device dma_dev;
175 struct pci_dev __rcu *pdev;
176 void __iomem *bar;
177
178 struct switchtec_dma_chan **swdma_chans;
179 int chan_cnt;
180 int chan_status_irq;
181};
182
183enum chan_op {
184 ENABLE_CHAN,
185 DISABLE_CHAN,
186};
187
188enum switchtec_dma_opcode {
189 SWITCHTEC_DMA_OPC_MEMCPY = 0,
190 SWITCHTEC_DMA_OPC_RDIMM = 0x1,
191 SWITCHTEC_DMA_OPC_WRIMM = 0x2,
192 SWITCHTEC_DMA_OPC_RHI = 0x6,
193 SWITCHTEC_DMA_OPC_NOP = 0x7,
194};
195
196struct switchtec_dma_hw_se_desc {
197 u8 opc;
198 u8 ctrl;
199 __le16 tlp_setting;
200 __le16 rsvd1;
201 __le16 cid;
202 __le32 byte_cnt;
203 __le32 addr_lo; /* SADDR_LO/WIADDR_LO */
204 __le32 addr_hi; /* SADDR_HI/WIADDR_HI */
205 __le32 daddr_lo;
206 __le32 daddr_hi;
207 __le16 dfid;
208 __le16 sfid;
209};
210
211#define SWITCHTEC_SE_DFM BIT(5)
212#define SWITCHTEC_SE_LIOF BIT(6)
213#define SWITCHTEC_SE_BRR BIT(7)
214#define SWITCHTEC_SE_CID_MASK GENMASK(15, 0)
215
216#define SWITCHTEC_CE_SC_LEN_ERR BIT(0)
217#define SWITCHTEC_CE_SC_UR BIT(1)
218#define SWITCHTEC_CE_SC_CA BIT(2)
219#define SWITCHTEC_CE_SC_RSVD_CPL BIT(3)
220#define SWITCHTEC_CE_SC_ECRC_ERR BIT(4)
221#define SWITCHTEC_CE_SC_EP_SET BIT(5)
222#define SWITCHTEC_CE_SC_D_RD_CTO BIT(8)
223#define SWITCHTEC_CE_SC_D_RIMM_UR BIT(9)
224#define SWITCHTEC_CE_SC_D_RIMM_CA BIT(10)
225#define SWITCHTEC_CE_SC_D_RIMM_RSVD_CPL BIT(11)
226#define SWITCHTEC_CE_SC_D_ECRC BIT(12)
227#define SWITCHTEC_CE_SC_D_EP_SET BIT(13)
228#define SWITCHTEC_CE_SC_D_BAD_CONNID BIT(14)
229#define SWITCHTEC_CE_SC_D_BAD_RHI_ADDR BIT(15)
230#define SWITCHTEC_CE_SC_D_INVD_CMD BIT(16)
231#define SWITCHTEC_CE_SC_MASK GENMASK(16, 0)
232
233struct switchtec_dma_hw_ce {
234 __le32 rdimm_cpl_dw0;
235 __le32 rdimm_cpl_dw1;
236 __le32 rsvd1;
237 __le32 cpl_byte_cnt;
238 __le16 sq_head;
239 __le16 rsvd2;
240 __le32 rsvd3;
241 __le32 sts_code;
242 __le16 cid;
243 __le16 phase_tag;
244};
245
246struct switchtec_dma_desc {
247 struct dma_async_tx_descriptor txd;
248 struct switchtec_dma_hw_se_desc *hw;
249 u32 orig_size;
250 bool completed;
251};
252
253static int wait_for_chan_status(struct chan_hw_regs __iomem *chan_hw, u32 mask,
254 bool set)
255{
256 u32 status;
257
258 return readl_poll_timeout_atomic(&chan_hw->status, status,
259 (set && (status & mask)) ||
260 (!set && !(status & mask)),
261 10, 100 * USEC_PER_MSEC);
262}
263
264static int halt_channel(struct switchtec_dma_chan *swdma_chan)
265{
266 struct chan_hw_regs __iomem *chan_hw = swdma_chan->mmio_chan_hw;
267 struct pci_dev *pdev;
268 int ret;
269
270 rcu_read_lock();
271 pdev = rcu_dereference(swdma_chan->swdma_dev->pdev);
272 if (!pdev) {
273 ret = -ENODEV;
274 goto unlock_and_exit;
275 }
276
277 spin_lock(&swdma_chan->hw_ctrl_lock);
278 writeb(SWITCHTEC_CHAN_CTRL_HALT, &chan_hw->ctrl);
279 ret = wait_for_chan_status(chan_hw, SWITCHTEC_CHAN_STS_HALTED, true);
280 spin_unlock(&swdma_chan->hw_ctrl_lock);
281
282unlock_and_exit:
283 rcu_read_unlock();
284 return ret;
285}
286
287static int unhalt_channel(struct switchtec_dma_chan *swdma_chan)
288{
289 struct chan_hw_regs __iomem *chan_hw = swdma_chan->mmio_chan_hw;
290 struct pci_dev *pdev;
291 u8 ctrl;
292 int ret;
293
294 rcu_read_lock();
295 pdev = rcu_dereference(swdma_chan->swdma_dev->pdev);
296 if (!pdev) {
297 ret = -ENODEV;
298 goto unlock_and_exit;
299 }
300
301 spin_lock(&swdma_chan->hw_ctrl_lock);
302 ctrl = readb(&chan_hw->ctrl);
303 ctrl &= ~SWITCHTEC_CHAN_CTRL_HALT;
304 writeb(ctrl, &chan_hw->ctrl);
305 ret = wait_for_chan_status(chan_hw, SWITCHTEC_CHAN_STS_HALTED, false);
306 spin_unlock(&swdma_chan->hw_ctrl_lock);
307
308unlock_and_exit:
309 rcu_read_unlock();
310 return ret;
311}
312
313static void flush_pci_write(struct chan_hw_regs __iomem *chan_hw)
314{
315 readl(&chan_hw->cq_head);
316}
317
318static int reset_channel(struct switchtec_dma_chan *swdma_chan)
319{
320 struct chan_hw_regs __iomem *chan_hw = swdma_chan->mmio_chan_hw;
321 struct pci_dev *pdev;
322
323 rcu_read_lock();
324 pdev = rcu_dereference(swdma_chan->swdma_dev->pdev);
325 if (!pdev) {
326 rcu_read_unlock();
327 return -ENODEV;
328 }
329
330 spin_lock(&swdma_chan->hw_ctrl_lock);
331 writel(SWITCHTEC_CHAN_CTRL_RESET | SWITCHTEC_CHAN_CTRL_ERR_PAUSE,
332 &chan_hw->ctrl);
333 flush_pci_write(chan_hw);
334
335 udelay(1000);
336
337 writel(SWITCHTEC_CHAN_CTRL_ERR_PAUSE, &chan_hw->ctrl);
338 spin_unlock(&swdma_chan->hw_ctrl_lock);
339 flush_pci_write(chan_hw);
340
341 rcu_read_unlock();
342 return 0;
343}
344
345static int pause_reset_channel(struct switchtec_dma_chan *swdma_chan)
346{
347 struct chan_hw_regs __iomem *chan_hw = swdma_chan->mmio_chan_hw;
348 struct pci_dev *pdev;
349
350 rcu_read_lock();
351 pdev = rcu_dereference(swdma_chan->swdma_dev->pdev);
352 if (!pdev) {
353 rcu_read_unlock();
354 return -ENODEV;
355 }
356
357 spin_lock(&swdma_chan->hw_ctrl_lock);
358 writeb(SWITCHTEC_CHAN_CTRL_PAUSE, &chan_hw->ctrl);
359 spin_unlock(&swdma_chan->hw_ctrl_lock);
360
361 flush_pci_write(chan_hw);
362
363 rcu_read_unlock();
364
365 /* wait 60ms to ensure no pending CEs */
366 mdelay(60);
367
368 return reset_channel(swdma_chan);
369}
370
371static int channel_op(struct switchtec_dma_chan *swdma_chan, int op)
372{
373 struct chan_fw_regs __iomem *chan_fw = swdma_chan->mmio_chan_fw;
374 struct pci_dev *pdev;
375 u32 valid_en_se;
376
377 rcu_read_lock();
378 pdev = rcu_dereference(swdma_chan->swdma_dev->pdev);
379 if (!pdev) {
380 rcu_read_unlock();
381 return -ENODEV;
382 }
383
384 valid_en_se = readl(&chan_fw->valid_en_se);
385 if (op == ENABLE_CHAN)
386 valid_en_se |= SWITCHTEC_CHAN_ENABLE;
387 else
388 valid_en_se &= ~SWITCHTEC_CHAN_ENABLE;
389
390 writel(valid_en_se, &chan_fw->valid_en_se);
391
392 rcu_read_unlock();
393 return 0;
394}
395
396static int enable_channel(struct switchtec_dma_chan *swdma_chan)
397{
398 return channel_op(swdma_chan, ENABLE_CHAN);
399}
400
401static int disable_channel(struct switchtec_dma_chan *swdma_chan)
402{
403 return channel_op(swdma_chan, DISABLE_CHAN);
404}
405
406static void
407switchtec_dma_cleanup_completed(struct switchtec_dma_chan *swdma_chan)
408{
409 struct device *chan_dev = &swdma_chan->dma_chan.dev->device;
410 struct switchtec_dma_desc *desc;
411 struct switchtec_dma_hw_ce *ce;
412 struct dmaengine_result res;
413 int tail, cid, se_idx, i;
414 __le16 phase_tag;
415 u32 sts_code;
416 __le32 *p;
417
418 do {
419 spin_lock_bh(&swdma_chan->complete_lock);
420 if (!swdma_chan->comp_ring_active) {
421 spin_unlock_bh(&swdma_chan->complete_lock);
422 break;
423 }
424
425 ce = &swdma_chan->hw_cq[swdma_chan->cq_tail];
426 /*
427 * phase_tag is updated by hardware, ensure the value is
428 * not from the cache
429 */
430 phase_tag = smp_load_acquire(&ce->phase_tag);
431 if (le16_to_cpu(phase_tag) == swdma_chan->phase_tag) {
432 spin_unlock_bh(&swdma_chan->complete_lock);
433 break;
434 }
435
436 cid = le16_to_cpu(ce->cid);
437 se_idx = cid & (SWITCHTEC_DMA_SQ_SIZE - 1);
438 desc = swdma_chan->desc_ring[se_idx];
439
440 tail = swdma_chan->tail;
441
442 res.residue = desc->orig_size - le32_to_cpu(ce->cpl_byte_cnt);
443
444 sts_code = le32_to_cpu(ce->sts_code);
445
446 if (!(sts_code & SWITCHTEC_CE_SC_MASK)) {
447 res.result = DMA_TRANS_NOERROR;
448 } else {
449 if (sts_code & SWITCHTEC_CE_SC_D_RD_CTO)
450 res.result = DMA_TRANS_READ_FAILED;
451 else
452 res.result = DMA_TRANS_WRITE_FAILED;
453
454 dev_err(chan_dev, "CID 0x%04x failed, SC 0x%08x\n", cid,
455 (u32)(sts_code & SWITCHTEC_CE_SC_MASK));
456
457 p = (__le32 *)ce;
458 for (i = 0; i < sizeof(*ce) / 4; i++) {
459 dev_err(chan_dev, "CE DW%d: 0x%08x\n", i,
460 le32_to_cpu(*p));
461 p++;
462 }
463 }
464
465 desc->completed = true;
466
467 swdma_chan->cq_tail++;
468 swdma_chan->cq_tail &= SWITCHTEC_DMA_CQ_SIZE - 1;
469
470 rcu_read_lock();
471 if (!rcu_dereference(swdma_chan->swdma_dev->pdev)) {
472 rcu_read_unlock();
473 spin_unlock_bh(&swdma_chan->complete_lock);
474 return;
475 }
476 writew(swdma_chan->cq_tail, &swdma_chan->mmio_chan_hw->cq_head);
477 rcu_read_unlock();
478
479 if (swdma_chan->cq_tail == 0)
480 swdma_chan->phase_tag = !swdma_chan->phase_tag;
481
482 /* Out of order CE */
483 if (se_idx != tail) {
484 spin_unlock_bh(&swdma_chan->complete_lock);
485 continue;
486 }
487
488 do {
489 dma_cookie_complete(&desc->txd);
490 dma_descriptor_unmap(&desc->txd);
491 dmaengine_desc_get_callback_invoke(&desc->txd, &res);
492 desc->txd.callback = NULL;
493 desc->txd.callback_result = NULL;
494 desc->completed = false;
495
496 tail++;
497 tail &= SWITCHTEC_DMA_SQ_SIZE - 1;
498
499 /*
500 * Ensure the desc updates are visible before updating
501 * the tail index
502 */
503 smp_store_release(&swdma_chan->tail, tail);
504 desc = swdma_chan->desc_ring[swdma_chan->tail];
505 if (!desc->completed)
506 break;
507 } while (CIRC_CNT(READ_ONCE(swdma_chan->head), swdma_chan->tail,
508 SWITCHTEC_DMA_SQ_SIZE));
509
510 spin_unlock_bh(&swdma_chan->complete_lock);
511 } while (1);
512}
513
514static void
515switchtec_dma_abort_desc(struct switchtec_dma_chan *swdma_chan, int force)
516{
517 struct switchtec_dma_desc *desc;
518 struct dmaengine_result res;
519
520 if (!force)
521 switchtec_dma_cleanup_completed(swdma_chan);
522
523 spin_lock_bh(&swdma_chan->complete_lock);
524
525 while (CIRC_CNT(swdma_chan->head, swdma_chan->tail,
526 SWITCHTEC_DMA_SQ_SIZE) >= 1) {
527 desc = swdma_chan->desc_ring[swdma_chan->tail];
528
529 res.residue = desc->orig_size;
530 res.result = DMA_TRANS_ABORTED;
531
532 dma_cookie_complete(&desc->txd);
533 dma_descriptor_unmap(&desc->txd);
534 if (!force)
535 dmaengine_desc_get_callback_invoke(&desc->txd, &res);
536 desc->txd.callback = NULL;
537 desc->txd.callback_result = NULL;
538
539 swdma_chan->tail++;
540 swdma_chan->tail &= SWITCHTEC_DMA_SQ_SIZE - 1;
541 }
542
543 spin_unlock_bh(&swdma_chan->complete_lock);
544}
545
546static void switchtec_dma_chan_stop(struct switchtec_dma_chan *swdma_chan)
547{
548 int rc;
549
550 rc = halt_channel(swdma_chan);
551 if (rc)
552 return;
553
554 rcu_read_lock();
555 if (!rcu_dereference(swdma_chan->swdma_dev->pdev)) {
556 rcu_read_unlock();
557 return;
558 }
559
560 writel(0, &swdma_chan->mmio_chan_fw->sq_base_lo);
561 writel(0, &swdma_chan->mmio_chan_fw->sq_base_hi);
562 writel(0, &swdma_chan->mmio_chan_fw->cq_base_lo);
563 writel(0, &swdma_chan->mmio_chan_fw->cq_base_hi);
564
565 rcu_read_unlock();
566}
567
568static int switchtec_dma_terminate_all(struct dma_chan *chan)
569{
570 struct switchtec_dma_chan *swdma_chan =
571 container_of(chan, struct switchtec_dma_chan, dma_chan);
572
573 spin_lock_bh(&swdma_chan->complete_lock);
574 swdma_chan->comp_ring_active = false;
575 spin_unlock_bh(&swdma_chan->complete_lock);
576
577 return pause_reset_channel(swdma_chan);
578}
579
580static void switchtec_dma_synchronize(struct dma_chan *chan)
581{
582 struct switchtec_dma_chan *swdma_chan =
583 container_of(chan, struct switchtec_dma_chan, dma_chan);
584
585 int rc;
586
587 switchtec_dma_abort_desc(swdma_chan, 1);
588
589 rc = enable_channel(swdma_chan);
590 if (rc)
591 return;
592
593 rc = reset_channel(swdma_chan);
594 if (rc)
595 return;
596
597 rc = unhalt_channel(swdma_chan);
598 if (rc)
599 return;
600
601 spin_lock_bh(&swdma_chan->submit_lock);
602 swdma_chan->head = 0;
603 spin_unlock_bh(&swdma_chan->submit_lock);
604
605 spin_lock_bh(&swdma_chan->complete_lock);
606 swdma_chan->comp_ring_active = true;
607 swdma_chan->phase_tag = 0;
608 swdma_chan->tail = 0;
609 swdma_chan->cq_tail = 0;
610 swdma_chan->cid = 0;
611 dma_cookie_init(chan);
612 spin_unlock_bh(&swdma_chan->complete_lock);
613}
614
615static struct dma_async_tx_descriptor *
616switchtec_dma_prep_desc(struct dma_chan *c, u16 dst_fid, dma_addr_t dma_dst,
617 u16 src_fid, dma_addr_t dma_src, u64 data,
618 size_t len, unsigned long flags)
619 __acquires(swdma_chan->submit_lock)
620{
621 struct switchtec_dma_chan *swdma_chan =
622 container_of(c, struct switchtec_dma_chan, dma_chan);
623 struct switchtec_dma_desc *desc;
624 int head, tail;
625
626 spin_lock_bh(&swdma_chan->submit_lock);
627
628 if (!swdma_chan->ring_active)
629 goto err_unlock;
630
631 tail = READ_ONCE(swdma_chan->tail);
632 head = swdma_chan->head;
633
634 if (!CIRC_SPACE(head, tail, SWITCHTEC_DMA_RING_SIZE))
635 goto err_unlock;
636
637 desc = swdma_chan->desc_ring[head];
638
639 if (src_fid != SWITCHTEC_INVALID_HFID &&
640 dst_fid != SWITCHTEC_INVALID_HFID)
641 desc->hw->ctrl |= SWITCHTEC_SE_DFM;
642
643 if (flags & DMA_PREP_INTERRUPT)
644 desc->hw->ctrl |= SWITCHTEC_SE_LIOF;
645
646 if (flags & DMA_PREP_FENCE)
647 desc->hw->ctrl |= SWITCHTEC_SE_BRR;
648
649 desc->txd.flags = flags;
650
651 desc->completed = false;
652 desc->hw->opc = SWITCHTEC_DMA_OPC_MEMCPY;
653 desc->hw->addr_lo = cpu_to_le32(lower_32_bits(dma_src));
654 desc->hw->addr_hi = cpu_to_le32(upper_32_bits(dma_src));
655 desc->hw->daddr_lo = cpu_to_le32(lower_32_bits(dma_dst));
656 desc->hw->daddr_hi = cpu_to_le32(upper_32_bits(dma_dst));
657 desc->hw->byte_cnt = cpu_to_le32(len);
658 desc->hw->tlp_setting = 0;
659 desc->hw->dfid = cpu_to_le16(dst_fid);
660 desc->hw->sfid = cpu_to_le16(src_fid);
661 swdma_chan->cid &= SWITCHTEC_SE_CID_MASK;
662 desc->hw->cid = cpu_to_le16(swdma_chan->cid++);
663 desc->orig_size = len;
664
665 /* return with the lock held, it will be released in tx_submit */
666
667 return &desc->txd;
668
669err_unlock:
670 /*
671 * Keep sparse happy by restoring an even lock count on
672 * this lock.
673 */
674 __acquire(swdma_chan->submit_lock);
675
676 spin_unlock_bh(&swdma_chan->submit_lock);
677 return NULL;
678}
679
680static struct dma_async_tx_descriptor *
681switchtec_dma_prep_memcpy(struct dma_chan *c, dma_addr_t dma_dst,
682 dma_addr_t dma_src, size_t len, unsigned long flags)
683 __acquires(swdma_chan->submit_lock)
684{
685 if (len > SWITCHTEC_DESC_MAX_SIZE) {
686 /*
687 * Keep sparse happy by restoring an even lock count on
688 * this lock.
689 */
690 __acquire(swdma_chan->submit_lock);
691 return NULL;
692 }
693
694 return switchtec_dma_prep_desc(c, SWITCHTEC_INVALID_HFID, dma_dst,
695 SWITCHTEC_INVALID_HFID, dma_src, 0, len,
696 flags);
697}
698
699static dma_cookie_t
700switchtec_dma_tx_submit(struct dma_async_tx_descriptor *desc)
701 __releases(swdma_chan->submit_lock)
702{
703 struct switchtec_dma_chan *swdma_chan =
704 container_of(desc->chan, struct switchtec_dma_chan, dma_chan);
705 dma_cookie_t cookie;
706 int head;
707
708 head = swdma_chan->head + 1;
709 head &= SWITCHTEC_DMA_RING_SIZE - 1;
710
711 /*
712 * Ensure the desc updates are visible before updating the head index
713 */
714 smp_store_release(&swdma_chan->head, head);
715
716 cookie = dma_cookie_assign(desc);
717
718 spin_unlock_bh(&swdma_chan->submit_lock);
719
720 return cookie;
721}
722
723static enum dma_status switchtec_dma_tx_status(struct dma_chan *chan,
724 dma_cookie_t cookie, struct dma_tx_state *txstate)
725{
726 struct switchtec_dma_chan *swdma_chan =
727 container_of(chan, struct switchtec_dma_chan, dma_chan);
728 enum dma_status ret;
729
730 ret = dma_cookie_status(chan, cookie, txstate);
731 if (ret == DMA_COMPLETE)
732 return ret;
733
734 /*
735 * For jobs where the interrupts are disabled, this is the only place
736 * to process the completions returned by the hardware. Callers that
737 * disable interrupts must call tx_status() to determine when a job
738 * is done, so it is safe to process completions here. If a job has
739 * interrupts enabled, then the completions will normally be processed
740 * in the tasklet that is triggered by the interrupt and tx_status()
741 * does not need to be called.
742 */
743 switchtec_dma_cleanup_completed(swdma_chan);
744
745 return dma_cookie_status(chan, cookie, txstate);
746}
747
748static void switchtec_dma_issue_pending(struct dma_chan *chan)
749{
750 struct switchtec_dma_chan *swdma_chan =
751 container_of(chan, struct switchtec_dma_chan, dma_chan);
752 struct switchtec_dma_dev *swdma_dev = swdma_chan->swdma_dev;
753
754 /*
755 * The sq_tail register is actually for the head of the
756 * submisssion queue. Chip has the opposite define of head/tail
757 * to the Linux kernel.
758 */
759
760 rcu_read_lock();
761 if (!rcu_dereference(swdma_dev->pdev)) {
762 rcu_read_unlock();
763 return;
764 }
765
766 spin_lock_bh(&swdma_chan->submit_lock);
767 writew(swdma_chan->head, &swdma_chan->mmio_chan_hw->sq_tail);
768 spin_unlock_bh(&swdma_chan->submit_lock);
769
770 rcu_read_unlock();
771}
772
773static int switchtec_dma_pause(struct dma_chan *chan)
774{
775 struct switchtec_dma_chan *swdma_chan =
776 container_of(chan, struct switchtec_dma_chan, dma_chan);
777 struct chan_hw_regs __iomem *chan_hw = swdma_chan->mmio_chan_hw;
778 struct pci_dev *pdev;
779 int ret;
780
781 rcu_read_lock();
782 pdev = rcu_dereference(swdma_chan->swdma_dev->pdev);
783 if (!pdev) {
784 ret = -ENODEV;
785 goto unlock_and_exit;
786 }
787
788 spin_lock(&swdma_chan->hw_ctrl_lock);
789 writeb(SWITCHTEC_CHAN_CTRL_PAUSE, &chan_hw->ctrl);
790 ret = wait_for_chan_status(chan_hw, SWITCHTEC_CHAN_STS_PAUSED, true);
791 spin_unlock(&swdma_chan->hw_ctrl_lock);
792
793unlock_and_exit:
794 rcu_read_unlock();
795 return ret;
796}
797
798static int switchtec_dma_resume(struct dma_chan *chan)
799{
800 struct switchtec_dma_chan *swdma_chan =
801 container_of(chan, struct switchtec_dma_chan, dma_chan);
802 struct chan_hw_regs __iomem *chan_hw = swdma_chan->mmio_chan_hw;
803 struct pci_dev *pdev;
804 int ret;
805
806 rcu_read_lock();
807 pdev = rcu_dereference(swdma_chan->swdma_dev->pdev);
808 if (!pdev) {
809 ret = -ENODEV;
810 goto unlock_and_exit;
811 }
812
813 spin_lock(&swdma_chan->hw_ctrl_lock);
814 writeb(0, &chan_hw->ctrl);
815 ret = wait_for_chan_status(chan_hw, SWITCHTEC_CHAN_STS_PAUSED, false);
816 spin_unlock(&swdma_chan->hw_ctrl_lock);
817
818unlock_and_exit:
819 rcu_read_unlock();
820 return ret;
821}
822
823static void switchtec_dma_desc_task(unsigned long data)
824{
825 struct switchtec_dma_chan *swdma_chan = (void *)data;
826
827 switchtec_dma_cleanup_completed(swdma_chan);
828}
829
830static irqreturn_t switchtec_dma_isr(int irq, void *chan)
831{
832 struct switchtec_dma_chan *swdma_chan = chan;
833
834 if (swdma_chan->comp_ring_active)
835 tasklet_schedule(&swdma_chan->desc_task);
836
837 return IRQ_HANDLED;
838}
839
840static irqreturn_t switchtec_dma_chan_status_isr(int irq, void *dma)
841{
842 struct switchtec_dma_dev *swdma_dev = dma;
843 struct dma_device *dma_dev = &swdma_dev->dma_dev;
844 struct switchtec_dma_chan *swdma_chan;
845 struct chan_hw_regs __iomem *chan_hw;
846 struct device *chan_dev;
847 struct dma_chan *chan;
848 u32 chan_status;
849 int bit;
850
851 list_for_each_entry(chan, &dma_dev->channels, device_node) {
852 swdma_chan = container_of(chan, struct switchtec_dma_chan,
853 dma_chan);
854 chan_dev = &swdma_chan->dma_chan.dev->device;
855 chan_hw = swdma_chan->mmio_chan_hw;
856
857 rcu_read_lock();
858 if (!rcu_dereference(swdma_dev->pdev)) {
859 rcu_read_unlock();
860 goto out;
861 }
862
863 chan_status = readl(&chan_hw->status);
864 chan_status &= SWITCHTEC_CHAN_STS_PAUSED_MASK;
865 rcu_read_unlock();
866
867 bit = ffs(chan_status);
868 if (!bit)
869 dev_dbg(chan_dev, "No pause bit set.\n");
870 else
871 dev_err(chan_dev, "Paused, %s\n",
872 channel_status_str[bit - 1]);
873 }
874
875out:
876 return IRQ_HANDLED;
877}
878
879static void switchtec_dma_free_desc(struct switchtec_dma_chan *swdma_chan)
880{
881 struct switchtec_dma_dev *swdma_dev = swdma_chan->swdma_dev;
882 size_t size;
883 int i;
884
885 size = SWITCHTEC_DMA_SQ_SIZE * sizeof(*swdma_chan->hw_sq);
886 if (swdma_chan->hw_sq)
887 dma_free_coherent(swdma_dev->dma_dev.dev, size,
888 swdma_chan->hw_sq, swdma_chan->dma_addr_sq);
889
890 size = SWITCHTEC_DMA_CQ_SIZE * sizeof(*swdma_chan->hw_cq);
891 if (swdma_chan->hw_cq)
892 dma_free_coherent(swdma_dev->dma_dev.dev, size,
893 swdma_chan->hw_cq, swdma_chan->dma_addr_cq);
894
895 for (i = 0; i < SWITCHTEC_DMA_RING_SIZE; i++)
896 kfree(swdma_chan->desc_ring[i]);
897}
898
899static int switchtec_dma_alloc_desc(struct switchtec_dma_chan *swdma_chan)
900{
901 struct switchtec_dma_dev *swdma_dev = swdma_chan->swdma_dev;
902 struct chan_fw_regs __iomem *chan_fw = swdma_chan->mmio_chan_fw;
903 struct switchtec_dma_desc *desc;
904 struct pci_dev *pdev;
905 size_t size;
906 int rc, i;
907
908 swdma_chan->head = 0;
909 swdma_chan->tail = 0;
910 swdma_chan->cq_tail = 0;
911
912 size = SWITCHTEC_DMA_SQ_SIZE * sizeof(*swdma_chan->hw_sq);
913 swdma_chan->hw_sq = dma_alloc_coherent(swdma_dev->dma_dev.dev, size,
914 &swdma_chan->dma_addr_sq,
915 GFP_NOWAIT);
916 if (!swdma_chan->hw_sq) {
917 rc = -ENOMEM;
918 goto free_and_exit;
919 }
920
921 size = SWITCHTEC_DMA_CQ_SIZE * sizeof(*swdma_chan->hw_cq);
922 swdma_chan->hw_cq = dma_alloc_coherent(swdma_dev->dma_dev.dev, size,
923 &swdma_chan->dma_addr_cq,
924 GFP_NOWAIT);
925 if (!swdma_chan->hw_cq) {
926 rc = -ENOMEM;
927 goto free_and_exit;
928 }
929
930 /* reset host phase tag */
931 swdma_chan->phase_tag = 0;
932
933 for (i = 0; i < SWITCHTEC_DMA_RING_SIZE; i++) {
934 desc = kzalloc_obj(*desc, GFP_NOWAIT);
935 if (!desc) {
936 rc = -ENOMEM;
937 goto free_and_exit;
938 }
939
940 dma_async_tx_descriptor_init(&desc->txd, &swdma_chan->dma_chan);
941 desc->txd.tx_submit = switchtec_dma_tx_submit;
942 desc->hw = &swdma_chan->hw_sq[i];
943 desc->completed = true;
944
945 swdma_chan->desc_ring[i] = desc;
946 }
947
948 rcu_read_lock();
949 pdev = rcu_dereference(swdma_dev->pdev);
950 if (!pdev) {
951 rcu_read_unlock();
952 rc = -ENODEV;
953 goto free_and_exit;
954 }
955
956 /* set sq/cq */
957 writel(lower_32_bits(swdma_chan->dma_addr_sq), &chan_fw->sq_base_lo);
958 writel(upper_32_bits(swdma_chan->dma_addr_sq), &chan_fw->sq_base_hi);
959 writel(lower_32_bits(swdma_chan->dma_addr_cq), &chan_fw->cq_base_lo);
960 writel(upper_32_bits(swdma_chan->dma_addr_cq), &chan_fw->cq_base_hi);
961
962 writew(SWITCHTEC_DMA_SQ_SIZE, &swdma_chan->mmio_chan_fw->sq_size);
963 writew(SWITCHTEC_DMA_CQ_SIZE, &swdma_chan->mmio_chan_fw->cq_size);
964
965 rcu_read_unlock();
966 return 0;
967
968free_and_exit:
969 switchtec_dma_free_desc(swdma_chan);
970 return rc;
971}
972
973static int switchtec_dma_alloc_chan_resources(struct dma_chan *chan)
974{
975 struct switchtec_dma_chan *swdma_chan =
976 container_of(chan, struct switchtec_dma_chan, dma_chan);
977 struct switchtec_dma_dev *swdma_dev = swdma_chan->swdma_dev;
978 u32 perf_cfg;
979 int rc;
980
981 rc = switchtec_dma_alloc_desc(swdma_chan);
982 if (rc)
983 return rc;
984
985 rc = enable_channel(swdma_chan);
986 if (rc)
987 return rc;
988
989 rc = reset_channel(swdma_chan);
990 if (rc)
991 return rc;
992
993 rc = unhalt_channel(swdma_chan);
994 if (rc)
995 return rc;
996
997 swdma_chan->ring_active = true;
998 swdma_chan->comp_ring_active = true;
999 swdma_chan->cid = 0;
1000
1001 dma_cookie_init(chan);
1002
1003 rcu_read_lock();
1004 if (!rcu_dereference(swdma_dev->pdev)) {
1005 rcu_read_unlock();
1006 return -ENODEV;
1007 }
1008
1009 perf_cfg = readl(&swdma_chan->mmio_chan_fw->perf_cfg);
1010 rcu_read_unlock();
1011
1012 dev_dbg(&chan->dev->device, "Burst Size: 0x%x\n",
1013 FIELD_GET(PERF_BURST_SIZE_MASK, perf_cfg));
1014
1015 dev_dbg(&chan->dev->device, "Burst Scale: 0x%x\n",
1016 FIELD_GET(PERF_BURST_SCALE_MASK, perf_cfg));
1017
1018 dev_dbg(&chan->dev->device, "Interval: 0x%x\n",
1019 FIELD_GET(PERF_INTERVAL_MASK, perf_cfg));
1020
1021 dev_dbg(&chan->dev->device, "Arb Weight: 0x%x\n",
1022 FIELD_GET(PERF_ARB_WEIGHT_MASK, perf_cfg));
1023
1024 dev_dbg(&chan->dev->device, "MRRS: 0x%x\n",
1025 FIELD_GET(PERF_MRRS_MASK, perf_cfg));
1026
1027 return SWITCHTEC_DMA_SQ_SIZE;
1028}
1029
1030static void switchtec_dma_free_chan_resources(struct dma_chan *chan)
1031{
1032 struct switchtec_dma_chan *swdma_chan =
1033 container_of(chan, struct switchtec_dma_chan, dma_chan);
1034
1035 spin_lock_bh(&swdma_chan->submit_lock);
1036 swdma_chan->ring_active = false;
1037 spin_unlock_bh(&swdma_chan->submit_lock);
1038
1039 spin_lock_bh(&swdma_chan->complete_lock);
1040 swdma_chan->comp_ring_active = false;
1041 spin_unlock_bh(&swdma_chan->complete_lock);
1042
1043 switchtec_dma_chan_stop(swdma_chan);
1044 switchtec_dma_abort_desc(swdma_chan, 0);
1045 switchtec_dma_free_desc(swdma_chan);
1046
1047 disable_channel(swdma_chan);
1048}
1049
1050static int switchtec_dma_chan_init(struct switchtec_dma_dev *swdma_dev,
1051 struct pci_dev *pdev, int i)
1052{
1053 struct dma_device *dma = &swdma_dev->dma_dev;
1054 struct switchtec_dma_chan *swdma_chan;
1055 u32 valid_en_se, thresh;
1056 int se_buf_len, irq, rc;
1057 struct dma_chan *chan;
1058
1059 swdma_chan = kzalloc_obj(*swdma_chan);
1060 if (!swdma_chan)
1061 return -ENOMEM;
1062
1063 swdma_chan->phase_tag = 0;
1064 swdma_chan->index = i;
1065 swdma_chan->swdma_dev = swdma_dev;
1066
1067 spin_lock_init(&swdma_chan->hw_ctrl_lock);
1068 spin_lock_init(&swdma_chan->submit_lock);
1069 spin_lock_init(&swdma_chan->complete_lock);
1070 tasklet_init(&swdma_chan->desc_task, switchtec_dma_desc_task,
1071 (unsigned long)swdma_chan);
1072
1073 swdma_chan->mmio_chan_fw =
1074 swdma_dev->bar + SWITCHTEC_DMAC_CHAN_CFG_STS_OFFSET +
1075 i * SWITCHTEC_DMA_CHAN_FW_REGS_SIZE;
1076 swdma_chan->mmio_chan_hw =
1077 swdma_dev->bar + SWITCHTEC_DMAC_CHAN_CTRL_OFFSET +
1078 i * SWITCHTEC_DMA_CHAN_HW_REGS_SIZE;
1079
1080 swdma_dev->swdma_chans[i] = swdma_chan;
1081
1082 rc = pause_reset_channel(swdma_chan);
1083 if (rc)
1084 goto free_and_exit;
1085
1086 /* init perf tuner */
1087 writel(FIELD_PREP(PERF_BURST_SCALE_MASK, 1) |
1088 FIELD_PREP(PERF_MRRS_MASK, 3) |
1089 FIELD_PREP(PERF_BURST_SIZE_MASK, 6) |
1090 FIELD_PREP(PERF_ARB_WEIGHT_MASK, 1),
1091 &swdma_chan->mmio_chan_fw->perf_cfg);
1092
1093 valid_en_se = readl(&swdma_chan->mmio_chan_fw->valid_en_se);
1094
1095 dev_dbg(&pdev->dev, "Channel %d: SE buffer base %d\n", i,
1096 FIELD_GET(SE_BUF_BASE_MASK, valid_en_se));
1097
1098 se_buf_len = FIELD_GET(SE_BUF_LEN_MASK, valid_en_se);
1099 dev_dbg(&pdev->dev, "Channel %d: SE buffer count %d\n", i, se_buf_len);
1100
1101 thresh = se_buf_len / 2;
1102 valid_en_se |= FIELD_PREP(SE_THRESH_MASK, thresh);
1103 writel(valid_en_se, &swdma_chan->mmio_chan_fw->valid_en_se);
1104
1105 /* request irqs */
1106 irq = readl(&swdma_chan->mmio_chan_fw->int_vec);
1107 dev_dbg(&pdev->dev, "Channel %d: CE irq vector %d\n", i, irq);
1108
1109 rc = pci_request_irq(pdev, irq, switchtec_dma_isr, NULL, swdma_chan,
1110 KBUILD_MODNAME);
1111 if (rc)
1112 goto free_and_exit;
1113
1114 swdma_chan->irq = irq;
1115
1116 chan = &swdma_chan->dma_chan;
1117 chan->device = dma;
1118 dma_cookie_init(chan);
1119
1120 list_add_tail(&chan->device_node, &dma->channels);
1121
1122 return 0;
1123
1124free_and_exit:
1125 kfree(swdma_chan);
1126 return rc;
1127}
1128
1129static int switchtec_dma_chan_free(struct pci_dev *pdev,
1130 struct switchtec_dma_chan *swdma_chan)
1131{
1132 spin_lock_bh(&swdma_chan->submit_lock);
1133 swdma_chan->ring_active = false;
1134 spin_unlock_bh(&swdma_chan->submit_lock);
1135
1136 spin_lock_bh(&swdma_chan->complete_lock);
1137 swdma_chan->comp_ring_active = false;
1138 spin_unlock_bh(&swdma_chan->complete_lock);
1139
1140 pci_free_irq(pdev, swdma_chan->irq, swdma_chan);
1141 tasklet_kill(&swdma_chan->desc_task);
1142
1143 switchtec_dma_chan_stop(swdma_chan);
1144
1145 return 0;
1146}
1147
1148static int switchtec_dma_chans_release(struct pci_dev *pdev,
1149 struct switchtec_dma_dev *swdma_dev)
1150{
1151 int i;
1152
1153 for (i = 0; i < swdma_dev->chan_cnt; i++)
1154 switchtec_dma_chan_free(pdev, swdma_dev->swdma_chans[i]);
1155
1156 return 0;
1157}
1158
1159static int switchtec_dma_chans_enumerate(struct switchtec_dma_dev *swdma_dev,
1160 struct pci_dev *pdev, int chan_cnt)
1161{
1162 struct dma_device *dma = &swdma_dev->dma_dev;
1163 int base, cnt, rc, i;
1164
1165 swdma_dev->swdma_chans = kzalloc_objs(*swdma_dev->swdma_chans, chan_cnt);
1166
1167 if (!swdma_dev->swdma_chans)
1168 return -ENOMEM;
1169
1170 base = readw(swdma_dev->bar + SWITCHTEC_REG_SE_BUF_BASE);
1171 cnt = readw(swdma_dev->bar + SWITCHTEC_REG_SE_BUF_CNT);
1172
1173 dev_dbg(&pdev->dev, "EP SE buffer base %d\n", base);
1174 dev_dbg(&pdev->dev, "EP SE buffer count %d\n", cnt);
1175
1176 INIT_LIST_HEAD(&dma->channels);
1177
1178 for (i = 0; i < chan_cnt; i++) {
1179 rc = switchtec_dma_chan_init(swdma_dev, pdev, i);
1180 if (rc) {
1181 dev_err(&pdev->dev, "Channel %d: init channel failed\n",
1182 i);
1183 chan_cnt = i;
1184 goto err_exit;
1185 }
1186 }
1187
1188 return chan_cnt;
1189
1190err_exit:
1191 for (i = 0; i < chan_cnt; i++)
1192 switchtec_dma_chan_free(pdev, swdma_dev->swdma_chans[i]);
1193
1194 kfree(swdma_dev->swdma_chans);
1195
1196 return rc;
1197}
1198
1199static void switchtec_dma_release(struct dma_device *dma_dev)
1200{
1201 struct switchtec_dma_dev *swdma_dev =
1202 container_of(dma_dev, struct switchtec_dma_dev, dma_dev);
1203 int i;
1204
1205 for (i = 0; i < swdma_dev->chan_cnt; i++)
1206 kfree(swdma_dev->swdma_chans[i]);
1207
1208 kfree(swdma_dev->swdma_chans);
1209
1210 put_device(dma_dev->dev);
1211 kfree(swdma_dev);
1212}
1213
1214static int switchtec_dma_create(struct pci_dev *pdev)
1215{
1216 struct switchtec_dma_dev *swdma_dev;
1217 int chan_cnt, nr_vecs, irq, rc;
1218 struct dma_device *dma;
1219 struct dma_chan *chan;
1220
1221 /*
1222 * Create the switchtec dma device
1223 */
1224 swdma_dev = kzalloc_obj(*swdma_dev);
1225 if (!swdma_dev)
1226 return -ENOMEM;
1227
1228 swdma_dev->bar = ioremap(pci_resource_start(pdev, 0),
1229 pci_resource_len(pdev, 0));
1230
1231 RCU_INIT_POINTER(swdma_dev->pdev, pdev);
1232
1233 nr_vecs = pci_msix_vec_count(pdev);
1234 rc = pci_alloc_irq_vectors(pdev, nr_vecs, nr_vecs, PCI_IRQ_MSIX);
1235 if (rc < 0)
1236 goto err_exit;
1237
1238 irq = readw(swdma_dev->bar + SWITCHTEC_REG_CHAN_STS_VEC);
1239 pci_dbg(pdev, "Channel pause irq vector %d\n", irq);
1240
1241 rc = pci_request_irq(pdev, irq, NULL, switchtec_dma_chan_status_isr,
1242 swdma_dev, KBUILD_MODNAME);
1243 if (rc)
1244 goto err_exit;
1245
1246 swdma_dev->chan_status_irq = irq;
1247
1248 chan_cnt = readl(swdma_dev->bar + SWITCHTEC_REG_CHAN_CNT);
1249 if (!chan_cnt) {
1250 pci_err(pdev, "No channel configured.\n");
1251 rc = -ENXIO;
1252 goto err_exit;
1253 }
1254
1255 chan_cnt = switchtec_dma_chans_enumerate(swdma_dev, pdev, chan_cnt);
1256 if (chan_cnt < 0) {
1257 pci_err(pdev, "Failed to enumerate dma channels: %d\n",
1258 chan_cnt);
1259 rc = -ENXIO;
1260 goto err_exit;
1261 }
1262
1263 swdma_dev->chan_cnt = chan_cnt;
1264
1265 dma = &swdma_dev->dma_dev;
1266 dma->copy_align = DMAENGINE_ALIGN_8_BYTES;
1267 dma_cap_set(DMA_MEMCPY, dma->cap_mask);
1268 dma_cap_set(DMA_PRIVATE, dma->cap_mask);
1269 dma->dev = get_device(&pdev->dev);
1270
1271 dma->device_alloc_chan_resources = switchtec_dma_alloc_chan_resources;
1272 dma->device_free_chan_resources = switchtec_dma_free_chan_resources;
1273 dma->device_prep_dma_memcpy = switchtec_dma_prep_memcpy;
1274 dma->device_tx_status = switchtec_dma_tx_status;
1275 dma->device_issue_pending = switchtec_dma_issue_pending;
1276 dma->device_pause = switchtec_dma_pause;
1277 dma->device_resume = switchtec_dma_resume;
1278 dma->device_terminate_all = switchtec_dma_terminate_all;
1279 dma->device_synchronize = switchtec_dma_synchronize;
1280 dma->device_release = switchtec_dma_release;
1281
1282 rc = dma_async_device_register(dma);
1283 if (rc) {
1284 pci_err(pdev, "Failed to register dma device: %d\n", rc);
1285 goto err_chans_release_exit;
1286 }
1287
1288 pci_dbg(pdev, "Channel count: %d\n", chan_cnt);
1289
1290 list_for_each_entry(chan, &dma->channels, device_node)
1291 pci_dbg(pdev, "%s\n", dma_chan_name(chan));
1292
1293 pci_set_drvdata(pdev, swdma_dev);
1294
1295 return 0;
1296
1297err_chans_release_exit:
1298 switchtec_dma_chans_release(pdev, swdma_dev);
1299
1300err_exit:
1301 if (swdma_dev->chan_status_irq)
1302 free_irq(swdma_dev->chan_status_irq, swdma_dev);
1303
1304 iounmap(swdma_dev->bar);
1305 kfree(swdma_dev);
1306 return rc;
1307}
1308
1309static int switchtec_dma_probe(struct pci_dev *pdev,
1310 const struct pci_device_id *id)
1311{
1312 int rc;
1313
1314 rc = pci_enable_device(pdev);
1315 if (rc)
1316 return rc;
1317
1318 dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
1319
1320 rc = pci_request_mem_regions(pdev, KBUILD_MODNAME);
1321 if (rc)
1322 goto err_disable;
1323
1324 pci_set_master(pdev);
1325
1326 rc = switchtec_dma_create(pdev);
1327 if (rc)
1328 goto err_free;
1329
1330 return 0;
1331
1332err_free:
1333 pci_free_irq_vectors(pdev);
1334 pci_release_mem_regions(pdev);
1335
1336err_disable:
1337 pci_disable_device(pdev);
1338
1339 return rc;
1340}
1341
1342static void switchtec_dma_remove(struct pci_dev *pdev)
1343{
1344 struct switchtec_dma_dev *swdma_dev = pci_get_drvdata(pdev);
1345
1346 switchtec_dma_chans_release(pdev, swdma_dev);
1347
1348 rcu_assign_pointer(swdma_dev->pdev, NULL);
1349 synchronize_rcu();
1350
1351 pci_free_irq(pdev, swdma_dev->chan_status_irq, swdma_dev);
1352
1353 pci_free_irq_vectors(pdev);
1354
1355 dma_async_device_unregister(&swdma_dev->dma_dev);
1356
1357 iounmap(swdma_dev->bar);
1358 pci_release_mem_regions(pdev);
1359 pci_disable_device(pdev);
1360}
1361
1362/*
1363 * Also use the class code to identify the devices, as some of the
1364 * device IDs are also used for other devices with other classes by
1365 * Microsemi.
1366 */
1367#define SW_ID(vendor_id, device_id) \
1368 { \
1369 .vendor = vendor_id, \
1370 .device = device_id, \
1371 .subvendor = PCI_ANY_ID, \
1372 .subdevice = PCI_ANY_ID, \
1373 .class = PCI_CLASS_SYSTEM_OTHER << 8, \
1374 .class_mask = 0xffffffff, \
1375 }
1376
1377static const struct pci_device_id switchtec_dma_pci_tbl[] = {
1378 SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x4000), /* PFX 100XG4 */
1379 SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x4084), /* PFX 84XG4 */
1380 SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x4068), /* PFX 68XG4 */
1381 SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x4052), /* PFX 52XG4 */
1382 SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x4036), /* PFX 36XG4 */
1383 SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x4028), /* PFX 28XG4 */
1384 SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x4100), /* PSX 100XG4 */
1385 SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x4184), /* PSX 84XG4 */
1386 SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x4168), /* PSX 68XG4 */
1387 SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x4152), /* PSX 52XG4 */
1388 SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x4136), /* PSX 36XG4 */
1389 SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x4128), /* PSX 28XG4 */
1390 SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x4352), /* PFXA 52XG4 */
1391 SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x4336), /* PFXA 36XG4 */
1392 SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x4328), /* PFXA 28XG4 */
1393 SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x4452), /* PSXA 52XG4 */
1394 SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x4436), /* PSXA 36XG4 */
1395 SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x4428), /* PSXA 28XG4 */
1396 SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x5000), /* PFX 100XG5 */
1397 SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x5084), /* PFX 84XG5 */
1398 SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x5068), /* PFX 68XG5 */
1399 SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x5052), /* PFX 52XG5 */
1400 SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x5036), /* PFX 36XG5 */
1401 SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x5028), /* PFX 28XG5 */
1402 SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x5100), /* PSX 100XG5 */
1403 SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x5184), /* PSX 84XG5 */
1404 SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x5168), /* PSX 68XG5 */
1405 SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x5152), /* PSX 52XG5 */
1406 SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x5136), /* PSX 36XG5 */
1407 SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x5128), /* PSX 28XG5 */
1408 SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x5300), /* PFXA 100XG5 */
1409 SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x5384), /* PFXA 84XG5 */
1410 SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x5368), /* PFXA 68XG5 */
1411 SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x5352), /* PFXA 52XG5 */
1412 SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x5336), /* PFXA 36XG5 */
1413 SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x5328), /* PFXA 28XG5 */
1414 SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x5400), /* PSXA 100XG5 */
1415 SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x5484), /* PSXA 84XG5 */
1416 SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x5468), /* PSXA 68XG5 */
1417 SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x5452), /* PSXA 52XG5 */
1418 SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x5436), /* PSXA 36XG5 */
1419 SW_ID(PCI_VENDOR_ID_MICROSEMI, 0x5428), /* PSXA 28XG5 */
1420 SW_ID(PCI_VENDOR_ID_EFAR, 0x1001), /* PCI1001 16XG4 */
1421 SW_ID(PCI_VENDOR_ID_EFAR, 0x1002), /* PCI1002 16XG4 */
1422 SW_ID(PCI_VENDOR_ID_EFAR, 0x1003), /* PCI1003 16XG4 */
1423 SW_ID(PCI_VENDOR_ID_EFAR, 0x1004), /* PCI1004 16XG4 */
1424 SW_ID(PCI_VENDOR_ID_EFAR, 0x1005), /* PCI1005 16XG4 */
1425 SW_ID(PCI_VENDOR_ID_EFAR, 0x1006), /* PCI1006 16XG4 */
1426 SW_ID(PCI_VENDOR_ID_EFAR, 0x1008), /* PCI1008 16XG4 */
1427 {0}
1428};
1429MODULE_DEVICE_TABLE(pci, switchtec_dma_pci_tbl);
1430
1431static struct pci_driver switchtec_dma_pci_driver = {
1432 .name = KBUILD_MODNAME,
1433 .id_table = switchtec_dma_pci_tbl,
1434 .probe = switchtec_dma_probe,
1435 .remove = switchtec_dma_remove,
1436};
1437module_pci_driver(switchtec_dma_pci_driver);