projects/linux·blob master · refs

1490 lines · 37.8 KiB

1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Host side test driver to test endpoint functionality
4 *
5 * Copyright (C) 2017 Texas Instruments
6 * Author: Kishon Vijay Abraham I <[email protected]>
7 */
8
9#include <linux/crc32.h>
10#include <linux/cleanup.h>
11#include <linux/delay.h>
12#include <linux/fs.h>
13#include <linux/io.h>
14#include <linux/interrupt.h>
15#include <linux/irq.h>
16#include <linux/miscdevice.h>
17#include <linux/module.h>
18#include <linux/mutex.h>
19#include <linux/random.h>
20#include <linux/slab.h>
21#include <linux/uaccess.h>
22#include <linux/pci.h>
23#include <linux/pci_ids.h>
24
25#include <linux/pci_regs.h>
26
27#include <uapi/linux/pcitest.h>
28
29#define DRV_MODULE_NAME "pci-endpoint-test"
30
31#define PCI_ENDPOINT_TEST_MAGIC 0x0
32
33#define PCI_ENDPOINT_TEST_COMMAND 0x4
34#define COMMAND_RAISE_INTX_IRQ BIT(0)
35#define COMMAND_RAISE_MSI_IRQ BIT(1)
36#define COMMAND_RAISE_MSIX_IRQ BIT(2)
37#define COMMAND_READ BIT(3)
38#define COMMAND_WRITE BIT(4)
39#define COMMAND_COPY BIT(5)
40#define COMMAND_ENABLE_DOORBELL BIT(6)
41#define COMMAND_DISABLE_DOORBELL BIT(7)
42#define COMMAND_BAR_SUBRANGE_SETUP BIT(8)
43#define COMMAND_BAR_SUBRANGE_CLEAR BIT(9)
44
45#define PCI_ENDPOINT_TEST_STATUS 0x8
46#define STATUS_READ_SUCCESS BIT(0)
47#define STATUS_READ_FAIL BIT(1)
48#define STATUS_WRITE_SUCCESS BIT(2)
49#define STATUS_WRITE_FAIL BIT(3)
50#define STATUS_COPY_SUCCESS BIT(4)
51#define STATUS_COPY_FAIL BIT(5)
52#define STATUS_IRQ_RAISED BIT(6)
53#define STATUS_SRC_ADDR_INVALID BIT(7)
54#define STATUS_DST_ADDR_INVALID BIT(8)
55#define STATUS_DOORBELL_SUCCESS BIT(9)
56#define STATUS_DOORBELL_ENABLE_SUCCESS BIT(10)
57#define STATUS_DOORBELL_ENABLE_FAIL BIT(11)
58#define STATUS_DOORBELL_DISABLE_SUCCESS BIT(12)
59#define STATUS_DOORBELL_DISABLE_FAIL BIT(13)
60#define STATUS_BAR_SUBRANGE_SETUP_SUCCESS BIT(14)
61#define STATUS_BAR_SUBRANGE_SETUP_FAIL BIT(15)
62#define STATUS_BAR_SUBRANGE_CLEAR_SUCCESS BIT(16)
63#define STATUS_BAR_SUBRANGE_CLEAR_FAIL BIT(17)
64#define STATUS_NO_RESOURCE BIT(18)
65
66#define PCI_ENDPOINT_TEST_LOWER_SRC_ADDR 0x0c
67#define PCI_ENDPOINT_TEST_UPPER_SRC_ADDR 0x10
68
69#define PCI_ENDPOINT_TEST_LOWER_DST_ADDR 0x14
70#define PCI_ENDPOINT_TEST_UPPER_DST_ADDR 0x18
71
72#define PCI_ENDPOINT_TEST_SIZE 0x1c
73#define PCI_ENDPOINT_TEST_CHECKSUM 0x20
74
75#define PCI_ENDPOINT_TEST_IRQ_TYPE 0x24
76#define PCI_ENDPOINT_TEST_IRQ_NUMBER 0x28
77
78#define PCI_ENDPOINT_TEST_FLAGS 0x2c
79
80#define FLAG_USE_DMA BIT(0)
81
82#define PCI_ENDPOINT_TEST_CAPS 0x30
83#define CAP_UNALIGNED_ACCESS BIT(0)
84#define CAP_MSI BIT(1)
85#define CAP_MSIX BIT(2)
86#define CAP_INTX BIT(3)
87#define CAP_SUBRANGE_MAPPING BIT(4)
88#define CAP_DYNAMIC_INBOUND_MAPPING BIT(5)
89#define CAP_BAR0_RESERVED BIT(6)
90#define CAP_BAR1_RESERVED BIT(7)
91#define CAP_BAR2_RESERVED BIT(8)
92#define CAP_BAR3_RESERVED BIT(9)
93#define CAP_BAR4_RESERVED BIT(10)
94#define CAP_BAR5_RESERVED BIT(11)
95
96#define PCI_ENDPOINT_TEST_DB_BAR 0x34
97#define PCI_ENDPOINT_TEST_DB_OFFSET 0x38
98#define PCI_ENDPOINT_TEST_DB_DATA 0x3c
99
100#define PCI_DEVICE_ID_TI_AM654 0xb00c
101#define PCI_DEVICE_ID_TI_J7200 0xb00f
102#define PCI_DEVICE_ID_TI_AM64 0xb010
103#define PCI_DEVICE_ID_TI_J721S2 0xb013
104#define PCI_DEVICE_ID_LS1088A 0x80c0
105#define PCI_DEVICE_ID_IMX8 0x0808
106
107#define is_am654_pci_dev(pdev) \
108 ((pdev)->device == PCI_DEVICE_ID_TI_AM654)
109
110#define PCI_DEVICE_ID_RENESAS_R8A774A1 0x0028
111#define PCI_DEVICE_ID_RENESAS_R8A774B1 0x002b
112#define PCI_DEVICE_ID_RENESAS_R8A774C0 0x002d
113#define PCI_DEVICE_ID_RENESAS_R8A774E1 0x0025
114#define PCI_DEVICE_ID_RENESAS_R8A779F0 0x0031
115
116#define PCI_DEVICE_ID_ROCKCHIP_RK3588 0x3588
117
118#define PCI_DEVICE_ID_NVIDIA_TEGRA194_EP 0x1ad4
119#define PCI_DEVICE_ID_NVIDIA_TEGRA234_EP 0x229b
120
121#define PCI_ENDPOINT_TEST_BAR_SUBRANGE_NSUB 2
122
123static DEFINE_IDA(pci_endpoint_test_ida);
124
125#define to_endpoint_test(priv) container_of((priv), struct pci_endpoint_test, \
126 miscdev)
127
128enum pci_barno {
129 BAR_0,
130 BAR_1,
131 BAR_2,
132 BAR_3,
133 BAR_4,
134 BAR_5,
135 NO_BAR = -1,
136};
137
138struct pci_endpoint_test {
139 struct pci_dev *pdev;
140 void __iomem *base;
141 void __iomem *bar[PCI_STD_NUM_BARS];
142 struct completion irq_raised;
143 int last_irq;
144 int num_irqs;
145 int irq_type;
146 /* mutex to protect the ioctls */
147 struct mutex mutex;
148 struct miscdevice miscdev;
149 enum pci_barno test_reg_bar;
150 size_t alignment;
151 u32 ep_caps;
152 const char *name;
153};
154
155struct pci_endpoint_test_data {
156 enum pci_barno test_reg_bar;
157 size_t alignment;
158};
159
160static inline u32 pci_endpoint_test_readl(struct pci_endpoint_test *test,
161 u32 offset)
162{
163 return readl(test->base + offset);
164}
165
166static inline void pci_endpoint_test_writel(struct pci_endpoint_test *test,
167 u32 offset, u32 value)
168{
169 writel(value, test->base + offset);
170}
171
172static irqreturn_t pci_endpoint_test_irqhandler(int irq, void *dev_id)
173{
174 struct pci_endpoint_test *test = dev_id;
175 u32 reg;
176
177 reg = pci_endpoint_test_readl(test, PCI_ENDPOINT_TEST_STATUS);
178 if (reg & STATUS_IRQ_RAISED) {
179 test->last_irq = irq;
180 complete(&test->irq_raised);
181 }
182
183 return IRQ_HANDLED;
184}
185
186static void pci_endpoint_test_free_irq_vectors(struct pci_endpoint_test *test)
187{
188 struct pci_dev *pdev = test->pdev;
189
190 pci_free_irq_vectors(pdev);
191 test->irq_type = PCITEST_IRQ_TYPE_UNDEFINED;
192}
193
194static int pci_endpoint_test_alloc_irq_vectors(struct pci_endpoint_test *test,
195 int type)
196{
197 int irq;
198 struct pci_dev *pdev = test->pdev;
199 struct device *dev = &pdev->dev;
200
201 switch (type) {
202 case PCITEST_IRQ_TYPE_INTX:
203 irq = pci_alloc_irq_vectors(pdev, 1, 1, PCI_IRQ_INTX);
204 if (irq < 0) {
205 dev_err(dev, "Failed to get Legacy interrupt\n");
206 return irq;
207 }
208
209 break;
210 case PCITEST_IRQ_TYPE_MSI:
211 irq = pci_alloc_irq_vectors(pdev, 1, 32, PCI_IRQ_MSI);
212 if (irq < 0) {
213 dev_err(dev, "Failed to get MSI interrupts\n");
214 return irq;
215 }
216
217 break;
218 case PCITEST_IRQ_TYPE_MSIX:
219 irq = pci_alloc_irq_vectors(pdev, 1, 2048, PCI_IRQ_MSIX);
220 if (irq < 0) {
221 dev_err(dev, "Failed to get MSI-X interrupts\n");
222 return irq;
223 }
224
225 break;
226 default:
227 dev_err(dev, "Invalid IRQ type selected\n");
228 return -EINVAL;
229 }
230
231 test->irq_type = type;
232 test->num_irqs = irq;
233
234 return 0;
235}
236
237static void pci_endpoint_test_release_irq(struct pci_endpoint_test *test)
238{
239 int i;
240 struct pci_dev *pdev = test->pdev;
241
242 for (i = 0; i < test->num_irqs; i++)
243 free_irq(pci_irq_vector(pdev, i), test);
244
245 test->num_irqs = 0;
246}
247
248static int pci_endpoint_test_request_irq(struct pci_endpoint_test *test)
249{
250 int i;
251 int ret;
252 struct pci_dev *pdev = test->pdev;
253 struct device *dev = &pdev->dev;
254
255 for (i = 0; i < test->num_irqs; i++) {
256 ret = request_irq(pci_irq_vector(pdev, i),
257 pci_endpoint_test_irqhandler, IRQF_SHARED,
258 test->name, test);
259 if (ret)
260 goto fail;
261 }
262
263 return 0;
264
265fail:
266 switch (test->irq_type) {
267 case PCITEST_IRQ_TYPE_INTX:
268 dev_err(dev, "Failed to request IRQ %d for Legacy\n",
269 pci_irq_vector(pdev, i));
270 break;
271 case PCITEST_IRQ_TYPE_MSI:
272 dev_err(dev, "Failed to request IRQ %d for MSI %d\n",
273 pci_irq_vector(pdev, i),
274 i + 1);
275 break;
276 case PCITEST_IRQ_TYPE_MSIX:
277 dev_err(dev, "Failed to request IRQ %d for MSI-X %d\n",
278 pci_irq_vector(pdev, i),
279 i + 1);
280 break;
281 }
282
283 test->num_irqs = i;
284 pci_endpoint_test_release_irq(test);
285
286 return ret;
287}
288
289static bool bar_is_reserved(struct pci_endpoint_test *test, enum pci_barno bar)
290{
291 return test->ep_caps & BIT(bar + __fls(CAP_BAR0_RESERVED));
292}
293
294static const u32 bar_test_pattern[] = {
295 0xA0A0A0A0,
296 0xA1A1A1A1,
297 0xA2A2A2A2,
298 0xA3A3A3A3,
299 0xA4A4A4A4,
300 0xA5A5A5A5,
301};
302
303static int pci_endpoint_test_bar_memcmp(struct pci_endpoint_test *test,
304 enum pci_barno barno,
305 resource_size_t offset, void *write_buf,
306 void *read_buf, int size)
307{
308 memset(write_buf, bar_test_pattern[barno], size);
309 memcpy_toio(test->bar[barno] + offset, write_buf, size);
310
311 memcpy_fromio(read_buf, test->bar[barno] + offset, size);
312
313 return memcmp(write_buf, read_buf, size);
314}
315
316static int pci_endpoint_test_bar(struct pci_endpoint_test *test,
317 enum pci_barno barno)
318{
319 resource_size_t bar_size, offset = 0;
320 void *write_buf __free(kfree) = NULL;
321 void *read_buf __free(kfree) = NULL;
322 struct pci_dev *pdev = test->pdev;
323 int buf_size;
324
325 bar_size = pci_resource_len(pdev, barno);
326 if (!bar_size)
327 return -ENODATA;
328
329 if (!test->bar[barno])
330 return -ENOMEM;
331
332 if (barno == test->test_reg_bar)
333 bar_size = 0x4;
334
335 /*
336 * Allocate a buffer of max size 1MB, and reuse that buffer while
337 * iterating over the whole BAR size (which might be much larger).
338 */
339 buf_size = min(SZ_1M, bar_size);
340
341 write_buf = kmalloc(buf_size, GFP_KERNEL);
342 if (!write_buf)
343 return -ENOMEM;
344
345 read_buf = kmalloc(buf_size, GFP_KERNEL);
346 if (!read_buf)
347 return -ENOMEM;
348
349 while (offset < bar_size) {
350 if (pci_endpoint_test_bar_memcmp(test, barno, offset, write_buf,
351 read_buf, buf_size))
352 return -EIO;
353 offset += buf_size;
354 }
355
356 return 0;
357}
358
359static u32 bar_test_pattern_with_offset(enum pci_barno barno, int offset)
360{
361 u32 val;
362
363 /* Keep the BAR pattern in the top byte. */
364 val = bar_test_pattern[barno] & 0xff000000;
365 /* Store the (partial) offset in the remaining bytes. */
366 val |= offset & 0x00ffffff;
367
368 return val;
369}
370
371static void pci_endpoint_test_bars_write_bar(struct pci_endpoint_test *test,
372 enum pci_barno barno)
373{
374 struct pci_dev *pdev = test->pdev;
375 int j, size;
376
377 size = pci_resource_len(pdev, barno);
378
379 if (barno == test->test_reg_bar)
380 size = 0x4;
381
382 for (j = 0; j < size; j += 4)
383 writel_relaxed(bar_test_pattern_with_offset(barno, j),
384 test->bar[barno] + j);
385}
386
387static int pci_endpoint_test_bars_read_bar(struct pci_endpoint_test *test,
388 enum pci_barno barno)
389{
390 struct pci_dev *pdev = test->pdev;
391 struct device *dev = &pdev->dev;
392 int j, size;
393 u32 val;
394
395 size = pci_resource_len(pdev, barno);
396
397 if (barno == test->test_reg_bar)
398 size = 0x4;
399
400 for (j = 0; j < size; j += 4) {
401 u32 expected = bar_test_pattern_with_offset(barno, j);
402
403 val = readl_relaxed(test->bar[barno] + j);
404 if (val != expected) {
405 dev_err(dev,
406 "BAR%d incorrect data at offset: %#x, got: %#x expected: %#x\n",
407 barno, j, val, expected);
408 return -EIO;
409 }
410 }
411
412 return 0;
413}
414
415static int pci_endpoint_test_bars(struct pci_endpoint_test *test)
416{
417 enum pci_barno bar;
418 int ret;
419
420 /* Write all BARs in order (without reading). */
421 for (bar = 0; bar < PCI_STD_NUM_BARS; bar++)
422 if (test->bar[bar] && !bar_is_reserved(test, bar))
423 pci_endpoint_test_bars_write_bar(test, bar);
424
425 /*
426 * Read all BARs in order (without writing).
427 * If there is an address translation issue on the EP, writing one BAR
428 * might have overwritten another BAR. Ensure that this is not the case.
429 * (Reading back the BAR directly after writing can not detect this.)
430 */
431 for (bar = 0; bar < PCI_STD_NUM_BARS; bar++) {
432 if (test->bar[bar] && !bar_is_reserved(test, bar)) {
433 ret = pci_endpoint_test_bars_read_bar(test, bar);
434 if (ret)
435 return ret;
436 }
437 }
438
439 return 0;
440}
441
442static u8 pci_endpoint_test_subrange_sig_byte(enum pci_barno barno,
443 unsigned int subno)
444{
445 return 0x50 + (barno * 8) + subno;
446}
447
448static u8 pci_endpoint_test_subrange_test_byte(enum pci_barno barno,
449 unsigned int subno)
450{
451 return 0xa0 + (barno * 8) + subno;
452}
453
454static int pci_endpoint_test_bar_subrange_cmd(struct pci_endpoint_test *test,
455 enum pci_barno barno, u32 command,
456 u32 ok_bit, u32 fail_bit)
457{
458 struct pci_dev *pdev = test->pdev;
459 struct device *dev = &pdev->dev;
460 int irq_type = test->irq_type;
461 u32 status;
462
463 if (irq_type < PCITEST_IRQ_TYPE_INTX ||
464 irq_type > PCITEST_IRQ_TYPE_MSIX) {
465 dev_err(dev, "Invalid IRQ type\n");
466 return -EINVAL;
467 }
468
469 reinit_completion(&test->irq_raised);
470
471 pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_STATUS, 0);
472 pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_IRQ_TYPE, irq_type);
473 pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_IRQ_NUMBER, 1);
474 /* Reuse SIZE as a command parameter: bar number. */
475 pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_SIZE, barno);
476 pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_COMMAND, command);
477
478 if (!wait_for_completion_timeout(&test->irq_raised,
479 msecs_to_jiffies(1000)))
480 return -ETIMEDOUT;
481
482 status = pci_endpoint_test_readl(test, PCI_ENDPOINT_TEST_STATUS);
483 if (status & fail_bit)
484 return (status & STATUS_NO_RESOURCE) ? -ENOSPC : -EIO;
485
486 if (!(status & ok_bit))
487 return -EIO;
488
489 return 0;
490}
491
492static int pci_endpoint_test_bar_subrange_setup(struct pci_endpoint_test *test,
493 enum pci_barno barno)
494{
495 return pci_endpoint_test_bar_subrange_cmd(test, barno,
496 COMMAND_BAR_SUBRANGE_SETUP,
497 STATUS_BAR_SUBRANGE_SETUP_SUCCESS,
498 STATUS_BAR_SUBRANGE_SETUP_FAIL);
499}
500
501static int pci_endpoint_test_bar_subrange_clear(struct pci_endpoint_test *test,
502 enum pci_barno barno)
503{
504 return pci_endpoint_test_bar_subrange_cmd(test, barno,
505 COMMAND_BAR_SUBRANGE_CLEAR,
506 STATUS_BAR_SUBRANGE_CLEAR_SUCCESS,
507 STATUS_BAR_SUBRANGE_CLEAR_FAIL);
508}
509
510static int pci_endpoint_test_bar_subrange(struct pci_endpoint_test *test,
511 enum pci_barno barno)
512{
513 u32 nsub = PCI_ENDPOINT_TEST_BAR_SUBRANGE_NSUB;
514 struct device *dev = &test->pdev->dev;
515 size_t sub_size, buf_size;
516 resource_size_t bar_size;
517 void __iomem *bar_addr;
518 void *read_buf = NULL;
519 int ret, clear_ret;
520 size_t off, chunk;
521 u32 i, exp, val;
522 u8 pattern;
523
524 if (!(test->ep_caps & CAP_SUBRANGE_MAPPING))
525 return -EOPNOTSUPP;
526
527 /*
528 * The test register BAR is not safe to reprogram and write/read
529 * over its full size. BAR_TEST already special-cases it to a tiny
530 * range. For subrange mapping tests, let's simply skip it.
531 */
532 if (barno == test->test_reg_bar)
533 return -EBUSY;
534
535 bar_size = pci_resource_len(test->pdev, barno);
536 if (!bar_size)
537 return -ENODATA;
538
539 bar_addr = test->bar[barno];
540 if (!bar_addr)
541 return -ENOMEM;
542
543 ret = pci_endpoint_test_bar_subrange_setup(test, barno);
544 if (ret)
545 return ret;
546
547 if (bar_size % nsub || bar_size / nsub > SIZE_MAX) {
548 ret = -EINVAL;
549 goto out_clear;
550 }
551
552 sub_size = bar_size / nsub;
553 if (sub_size < sizeof(u32)) {
554 ret = -EINVAL;
555 goto out_clear;
556 }
557
558 /* Limit the temporary buffer size */
559 buf_size = min_t(size_t, sub_size, SZ_1M);
560
561 read_buf = kmalloc(buf_size, GFP_KERNEL);
562 if (!read_buf) {
563 ret = -ENOMEM;
564 goto out_clear;
565 }
566
567 /*
568 * Step 1: verify EP-provided signature per subrange. This detects
569 * whether the EP actually applied the submap order.
570 */
571 for (i = 0; i < nsub; i++) {
572 exp = (u32)pci_endpoint_test_subrange_sig_byte(barno, i) *
573 0x01010101U;
574 val = ioread32(bar_addr + (i * sub_size));
575 if (val != exp) {
576 dev_err(dev,
577 "BAR%d subrange%u signature mismatch @%#zx: exp %#08x got %#08x\n",
578 barno, i, (size_t)i * sub_size, exp, val);
579 ret = -EIO;
580 goto out_clear;
581 }
582 val = ioread32(bar_addr + (i * sub_size) + sub_size - sizeof(u32));
583 if (val != exp) {
584 dev_err(dev,
585 "BAR%d subrange%u signature mismatch @%#zx: exp %#08x got %#08x\n",
586 barno, i,
587 ((size_t)i * sub_size) + sub_size - sizeof(u32),
588 exp, val);
589 ret = -EIO;
590 goto out_clear;
591 }
592 }
593
594 /* Step 2: write unique pattern per subrange (write all first). */
595 for (i = 0; i < nsub; i++) {
596 pattern = pci_endpoint_test_subrange_test_byte(barno, i);
597 memset_io(bar_addr + (i * sub_size), pattern, sub_size);
598 }
599
600 /* Step 3: read back and verify (read all after all writes). */
601 for (i = 0; i < nsub; i++) {
602 pattern = pci_endpoint_test_subrange_test_byte(barno, i);
603 for (off = 0; off < sub_size; off += chunk) {
604 void *bad;
605
606 chunk = min_t(size_t, buf_size, sub_size - off);
607 memcpy_fromio(read_buf, bar_addr + (i * sub_size) + off,
608 chunk);
609 bad = memchr_inv(read_buf, pattern, chunk);
610 if (bad) {
611 size_t bad_off = (u8 *)bad - (u8 *)read_buf;
612
613 dev_err(dev,
614 "BAR%d subrange%u data mismatch @%#zx (pattern %#02x)\n",
615 barno, i, (size_t)i * sub_size + off + bad_off,
616 pattern);
617 ret = -EIO;
618 goto out_clear;
619 }
620 }
621 }
622
623out_clear:
624 kfree(read_buf);
625 clear_ret = pci_endpoint_test_bar_subrange_clear(test, barno);
626 return ret ?: clear_ret;
627}
628
629static int pci_endpoint_test_intx_irq(struct pci_endpoint_test *test)
630{
631 u32 val;
632
633 pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_IRQ_TYPE,
634 PCITEST_IRQ_TYPE_INTX);
635 pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_IRQ_NUMBER, 0);
636 pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_COMMAND,
637 COMMAND_RAISE_INTX_IRQ);
638 val = wait_for_completion_timeout(&test->irq_raised,
639 msecs_to_jiffies(1000));
640 if (!val)
641 return -ETIMEDOUT;
642
643 return 0;
644}
645
646static int pci_endpoint_test_msi_irq(struct pci_endpoint_test *test,
647 u16 msi_num, bool msix)
648{
649 struct pci_dev *pdev = test->pdev;
650 u32 val;
651 int irq;
652
653 irq = pci_irq_vector(pdev, msi_num - 1);
654 if (irq < 0)
655 return irq;
656
657 pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_IRQ_TYPE,
658 msix ? PCITEST_IRQ_TYPE_MSIX :
659 PCITEST_IRQ_TYPE_MSI);
660 pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_IRQ_NUMBER, msi_num);
661 pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_COMMAND,
662 msix ? COMMAND_RAISE_MSIX_IRQ :
663 COMMAND_RAISE_MSI_IRQ);
664 val = wait_for_completion_timeout(&test->irq_raised,
665 msecs_to_jiffies(1000));
666 if (!val)
667 return -ETIMEDOUT;
668
669 if (irq != test->last_irq)
670 return -EIO;
671
672 return 0;
673}
674
675static int pci_endpoint_test_validate_xfer_params(struct device *dev,
676 struct pci_endpoint_test_xfer_param *param, size_t alignment)
677{
678 if (!param->size) {
679 dev_dbg(dev, "Data size is zero\n");
680 return -EINVAL;
681 }
682
683 if (param->size > SIZE_MAX - alignment) {
684 dev_dbg(dev, "Maximum transfer data size exceeded\n");
685 return -EINVAL;
686 }
687
688 return 0;
689}
690
691static int pci_endpoint_test_copy(struct pci_endpoint_test *test,
692 unsigned long arg)
693{
694 struct pci_endpoint_test_xfer_param param;
695 void *src_addr;
696 void *dst_addr;
697 u32 flags = 0;
698 bool use_dma;
699 size_t size;
700 dma_addr_t src_phys_addr;
701 dma_addr_t dst_phys_addr;
702 struct pci_dev *pdev = test->pdev;
703 struct device *dev = &pdev->dev;
704 void *orig_src_addr;
705 dma_addr_t orig_src_phys_addr;
706 void *orig_dst_addr;
707 dma_addr_t orig_dst_phys_addr;
708 size_t offset;
709 size_t alignment = test->alignment;
710 int irq_type = test->irq_type;
711 u32 src_crc32;
712 u32 dst_crc32;
713 int ret;
714
715 ret = copy_from_user(&param, (void __user *)arg, sizeof(param));
716 if (ret) {
717 dev_err(dev, "Failed to get transfer param\n");
718 return -EFAULT;
719 }
720
721 ret = pci_endpoint_test_validate_xfer_params(dev, &param, alignment);
722 if (ret)
723 return ret;
724
725 size = param.size;
726
727 use_dma = !!(param.flags & PCITEST_FLAGS_USE_DMA);
728 if (use_dma)
729 flags |= FLAG_USE_DMA;
730
731 if (irq_type < PCITEST_IRQ_TYPE_INTX ||
732 irq_type > PCITEST_IRQ_TYPE_MSIX) {
733 dev_err(dev, "Invalid IRQ type option\n");
734 return -EINVAL;
735 }
736
737 orig_src_addr = kzalloc(size + alignment, GFP_KERNEL);
738 if (!orig_src_addr) {
739 dev_err(dev, "Failed to allocate source buffer\n");
740 return -ENOMEM;
741 }
742
743 get_random_bytes(orig_src_addr, size + alignment);
744 orig_src_phys_addr = dma_map_single(dev, orig_src_addr,
745 size + alignment, DMA_TO_DEVICE);
746 ret = dma_mapping_error(dev, orig_src_phys_addr);
747 if (ret) {
748 dev_err(dev, "failed to map source buffer address\n");
749 goto err_src_phys_addr;
750 }
751
752 if (alignment && !IS_ALIGNED(orig_src_phys_addr, alignment)) {
753 src_phys_addr = PTR_ALIGN(orig_src_phys_addr, alignment);
754 offset = src_phys_addr - orig_src_phys_addr;
755 src_addr = orig_src_addr + offset;
756 } else {
757 src_phys_addr = orig_src_phys_addr;
758 src_addr = orig_src_addr;
759 }
760
761 pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_LOWER_SRC_ADDR,
762 lower_32_bits(src_phys_addr));
763
764 pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_UPPER_SRC_ADDR,
765 upper_32_bits(src_phys_addr));
766
767 src_crc32 = crc32_le(~0, src_addr, size);
768
769 orig_dst_addr = kzalloc(size + alignment, GFP_KERNEL);
770 if (!orig_dst_addr) {
771 dev_err(dev, "Failed to allocate destination address\n");
772 ret = -ENOMEM;
773 goto err_dst_addr;
774 }
775
776 orig_dst_phys_addr = dma_map_single(dev, orig_dst_addr,
777 size + alignment, DMA_FROM_DEVICE);
778 ret = dma_mapping_error(dev, orig_dst_phys_addr);
779 if (ret) {
780 dev_err(dev, "failed to map destination buffer address\n");
781 goto err_dst_phys_addr;
782 }
783
784 if (alignment && !IS_ALIGNED(orig_dst_phys_addr, alignment)) {
785 dst_phys_addr = PTR_ALIGN(orig_dst_phys_addr, alignment);
786 offset = dst_phys_addr - orig_dst_phys_addr;
787 dst_addr = orig_dst_addr + offset;
788 } else {
789 dst_phys_addr = orig_dst_phys_addr;
790 dst_addr = orig_dst_addr;
791 }
792
793 pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_LOWER_DST_ADDR,
794 lower_32_bits(dst_phys_addr));
795 pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_UPPER_DST_ADDR,
796 upper_32_bits(dst_phys_addr));
797
798 pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_SIZE,
799 size);
800
801 pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_FLAGS, flags);
802 pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_IRQ_TYPE, irq_type);
803 pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_IRQ_NUMBER, 1);
804 pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_COMMAND,
805 COMMAND_COPY);
806
807 wait_for_completion(&test->irq_raised);
808
809 dma_unmap_single(dev, orig_dst_phys_addr, size + alignment,
810 DMA_FROM_DEVICE);
811
812 dst_crc32 = crc32_le(~0, dst_addr, size);
813 if (dst_crc32 != src_crc32)
814 ret = -EIO;
815
816err_dst_phys_addr:
817 kfree(orig_dst_addr);
818
819err_dst_addr:
820 dma_unmap_single(dev, orig_src_phys_addr, size + alignment,
821 DMA_TO_DEVICE);
822
823err_src_phys_addr:
824 kfree(orig_src_addr);
825 return ret;
826}
827
828static int pci_endpoint_test_write(struct pci_endpoint_test *test,
829 unsigned long arg)
830{
831 struct pci_endpoint_test_xfer_param param;
832 u32 flags = 0;
833 bool use_dma;
834 u32 reg;
835 void *addr;
836 dma_addr_t phys_addr;
837 struct pci_dev *pdev = test->pdev;
838 struct device *dev = &pdev->dev;
839 void *orig_addr;
840 dma_addr_t orig_phys_addr;
841 size_t offset;
842 size_t alignment = test->alignment;
843 int irq_type = test->irq_type;
844 size_t size;
845 u32 crc32;
846 int ret;
847
848 ret = copy_from_user(&param, (void __user *)arg, sizeof(param));
849 if (ret) {
850 dev_err(dev, "Failed to get transfer param\n");
851 return -EFAULT;
852 }
853
854 ret = pci_endpoint_test_validate_xfer_params(dev, &param, alignment);
855 if (ret)
856 return ret;
857
858 size = param.size;
859
860 use_dma = !!(param.flags & PCITEST_FLAGS_USE_DMA);
861 if (use_dma)
862 flags |= FLAG_USE_DMA;
863
864 if (irq_type < PCITEST_IRQ_TYPE_INTX ||
865 irq_type > PCITEST_IRQ_TYPE_MSIX) {
866 dev_err(dev, "Invalid IRQ type option\n");
867 return -EINVAL;
868 }
869
870 orig_addr = kzalloc(size + alignment, GFP_KERNEL);
871 if (!orig_addr) {
872 dev_err(dev, "Failed to allocate address\n");
873 return -ENOMEM;
874 }
875
876 get_random_bytes(orig_addr, size + alignment);
877
878 orig_phys_addr = dma_map_single(dev, orig_addr, size + alignment,
879 DMA_TO_DEVICE);
880 ret = dma_mapping_error(dev, orig_phys_addr);
881 if (ret) {
882 dev_err(dev, "failed to map source buffer address\n");
883 goto err_phys_addr;
884 }
885
886 if (alignment && !IS_ALIGNED(orig_phys_addr, alignment)) {
887 phys_addr = PTR_ALIGN(orig_phys_addr, alignment);
888 offset = phys_addr - orig_phys_addr;
889 addr = orig_addr + offset;
890 } else {
891 phys_addr = orig_phys_addr;
892 addr = orig_addr;
893 }
894
895 crc32 = crc32_le(~0, addr, size);
896 pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_CHECKSUM,
897 crc32);
898
899 pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_LOWER_SRC_ADDR,
900 lower_32_bits(phys_addr));
901 pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_UPPER_SRC_ADDR,
902 upper_32_bits(phys_addr));
903
904 pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_SIZE, size);
905
906 pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_FLAGS, flags);
907 pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_IRQ_TYPE, irq_type);
908 pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_IRQ_NUMBER, 1);
909 pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_COMMAND,
910 COMMAND_READ);
911
912 wait_for_completion(&test->irq_raised);
913
914 reg = pci_endpoint_test_readl(test, PCI_ENDPOINT_TEST_STATUS);
915 if (!(reg & STATUS_READ_SUCCESS))
916 ret = -EIO;
917
918 dma_unmap_single(dev, orig_phys_addr, size + alignment,
919 DMA_TO_DEVICE);
920
921err_phys_addr:
922 kfree(orig_addr);
923 return ret;
924}
925
926static int pci_endpoint_test_read(struct pci_endpoint_test *test,
927 unsigned long arg)
928{
929 struct pci_endpoint_test_xfer_param param;
930 u32 flags = 0;
931 bool use_dma;
932 size_t size;
933 void *addr;
934 dma_addr_t phys_addr;
935 struct pci_dev *pdev = test->pdev;
936 struct device *dev = &pdev->dev;
937 void *orig_addr;
938 dma_addr_t orig_phys_addr;
939 size_t offset;
940 size_t alignment = test->alignment;
941 int irq_type = test->irq_type;
942 u32 crc32;
943 int ret;
944
945 ret = copy_from_user(&param, (void __user *)arg, sizeof(param));
946 if (ret) {
947 dev_err(dev, "Failed to get transfer param\n");
948 return -EFAULT;
949 }
950
951 ret = pci_endpoint_test_validate_xfer_params(dev, &param, alignment);
952 if (ret)
953 return ret;
954
955 size = param.size;
956
957 use_dma = !!(param.flags & PCITEST_FLAGS_USE_DMA);
958 if (use_dma)
959 flags |= FLAG_USE_DMA;
960
961 if (irq_type < PCITEST_IRQ_TYPE_INTX ||
962 irq_type > PCITEST_IRQ_TYPE_MSIX) {
963 dev_err(dev, "Invalid IRQ type option\n");
964 return -EINVAL;
965 }
966
967 orig_addr = kzalloc(size + alignment, GFP_KERNEL);
968 if (!orig_addr) {
969 dev_err(dev, "Failed to allocate destination address\n");
970 return -ENOMEM;
971 }
972
973 orig_phys_addr = dma_map_single(dev, orig_addr, size + alignment,
974 DMA_FROM_DEVICE);
975 ret = dma_mapping_error(dev, orig_phys_addr);
976 if (ret) {
977 dev_err(dev, "failed to map source buffer address\n");
978 goto err_phys_addr;
979 }
980
981 if (alignment && !IS_ALIGNED(orig_phys_addr, alignment)) {
982 phys_addr = PTR_ALIGN(orig_phys_addr, alignment);
983 offset = phys_addr - orig_phys_addr;
984 addr = orig_addr + offset;
985 } else {
986 phys_addr = orig_phys_addr;
987 addr = orig_addr;
988 }
989
990 pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_LOWER_DST_ADDR,
991 lower_32_bits(phys_addr));
992 pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_UPPER_DST_ADDR,
993 upper_32_bits(phys_addr));
994
995 pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_SIZE, size);
996
997 pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_FLAGS, flags);
998 pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_IRQ_TYPE, irq_type);
999 pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_IRQ_NUMBER, 1);
1000 pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_COMMAND,
1001 COMMAND_WRITE);
1002
1003 wait_for_completion(&test->irq_raised);
1004
1005 dma_unmap_single(dev, orig_phys_addr, size + alignment,
1006 DMA_FROM_DEVICE);
1007
1008 crc32 = crc32_le(~0, addr, size);
1009 if (crc32 != pci_endpoint_test_readl(test, PCI_ENDPOINT_TEST_CHECKSUM))
1010 ret = -EIO;
1011
1012err_phys_addr:
1013 kfree(orig_addr);
1014 return ret;
1015}
1016
1017static int pci_endpoint_test_clear_irq(struct pci_endpoint_test *test)
1018{
1019 pci_endpoint_test_release_irq(test);
1020 pci_endpoint_test_free_irq_vectors(test);
1021
1022 return 0;
1023}
1024
1025static int pci_endpoint_test_set_irq(struct pci_endpoint_test *test,
1026 int req_irq_type)
1027{
1028 struct pci_dev *pdev = test->pdev;
1029 struct device *dev = &pdev->dev;
1030 int ret;
1031
1032 if (req_irq_type < PCITEST_IRQ_TYPE_INTX ||
1033 req_irq_type > PCITEST_IRQ_TYPE_AUTO) {
1034 dev_err(dev, "Invalid IRQ type option\n");
1035 return -EINVAL;
1036 }
1037
1038 if (req_irq_type == PCITEST_IRQ_TYPE_AUTO) {
1039 if (test->ep_caps & CAP_MSI)
1040 req_irq_type = PCITEST_IRQ_TYPE_MSI;
1041 else if (test->ep_caps & CAP_MSIX)
1042 req_irq_type = PCITEST_IRQ_TYPE_MSIX;
1043 else if (test->ep_caps & CAP_INTX)
1044 req_irq_type = PCITEST_IRQ_TYPE_INTX;
1045 else
1046 /* fallback to MSI if no caps defined */
1047 req_irq_type = PCITEST_IRQ_TYPE_MSI;
1048 }
1049
1050 if (test->irq_type == req_irq_type)
1051 return 0;
1052
1053 pci_endpoint_test_release_irq(test);
1054 pci_endpoint_test_free_irq_vectors(test);
1055
1056 ret = pci_endpoint_test_alloc_irq_vectors(test, req_irq_type);
1057 if (ret)
1058 return ret;
1059
1060 ret = pci_endpoint_test_request_irq(test);
1061 if (ret) {
1062 pci_endpoint_test_free_irq_vectors(test);
1063 return ret;
1064 }
1065
1066 return 0;
1067}
1068
1069static int pci_endpoint_test_doorbell(struct pci_endpoint_test *test)
1070{
1071 struct pci_dev *pdev = test->pdev;
1072 struct device *dev = &pdev->dev;
1073 int irq_type = test->irq_type;
1074 int ret = 0;
1075 enum pci_barno bar;
1076 u32 data, status;
1077 u32 addr;
1078 int left;
1079
1080 if (!(test->ep_caps & CAP_DYNAMIC_INBOUND_MAPPING))
1081 return -EOPNOTSUPP;
1082
1083 if (irq_type < PCITEST_IRQ_TYPE_INTX ||
1084 irq_type > PCITEST_IRQ_TYPE_MSIX) {
1085 dev_err(dev, "Invalid IRQ type\n");
1086 return -EINVAL;
1087 }
1088
1089 pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_IRQ_TYPE, irq_type);
1090 pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_IRQ_NUMBER, 1);
1091 pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_COMMAND,
1092 COMMAND_ENABLE_DOORBELL);
1093
1094 left = wait_for_completion_timeout(&test->irq_raised, msecs_to_jiffies(1000));
1095
1096 status = pci_endpoint_test_readl(test, PCI_ENDPOINT_TEST_STATUS);
1097 if (!left || !(status & STATUS_DOORBELL_ENABLE_SUCCESS)) {
1098 dev_err(dev, "Failed to enable doorbell\n");
1099 return -EINVAL;
1100 }
1101
1102 data = pci_endpoint_test_readl(test, PCI_ENDPOINT_TEST_DB_DATA);
1103 addr = pci_endpoint_test_readl(test, PCI_ENDPOINT_TEST_DB_OFFSET);
1104
1105 pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_IRQ_TYPE, irq_type);
1106 pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_IRQ_NUMBER, 1);
1107
1108 pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_STATUS, 0);
1109
1110 bar = pci_endpoint_test_readl(test, PCI_ENDPOINT_TEST_DB_BAR);
1111 if (bar < BAR_0 || bar >= PCI_STD_NUM_BARS) {
1112 dev_err(dev, "BAR %d reported by endpoint out of range [0, %u]\n",
1113 bar, PCI_STD_NUM_BARS - 1);
1114 return -ERANGE;
1115 }
1116
1117 writel(data, test->bar[bar] + addr);
1118
1119 left = wait_for_completion_timeout(&test->irq_raised, msecs_to_jiffies(1000));
1120
1121 status = pci_endpoint_test_readl(test, PCI_ENDPOINT_TEST_STATUS);
1122
1123 if (!left || !(status & STATUS_DOORBELL_SUCCESS)) {
1124 dev_err(dev, "Failed to trigger doorbell in endpoint\n");
1125 /* Store error code, but continue to disable doorbell. */
1126 ret = -EINVAL;
1127 }
1128
1129 pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_COMMAND,
1130 COMMAND_DISABLE_DOORBELL);
1131
1132 wait_for_completion_timeout(&test->irq_raised, msecs_to_jiffies(1000));
1133
1134 status |= pci_endpoint_test_readl(test, PCI_ENDPOINT_TEST_STATUS);
1135
1136 if (!(status & STATUS_DOORBELL_DISABLE_SUCCESS)) {
1137 dev_err(dev, "Failed to disable doorbell\n");
1138 return -EINVAL;
1139 }
1140
1141 return ret;
1142}
1143
1144static long pci_endpoint_test_ioctl(struct file *file, unsigned int cmd,
1145 unsigned long arg)
1146{
1147 int ret = -EINVAL;
1148 enum pci_barno bar;
1149 struct pci_endpoint_test *test = to_endpoint_test(file->private_data);
1150 struct pci_dev *pdev = test->pdev;
1151
1152 mutex_lock(&test->mutex);
1153
1154 reinit_completion(&test->irq_raised);
1155 test->last_irq = -ENODATA;
1156
1157 switch (cmd) {
1158 case PCITEST_BAR:
1159 case PCITEST_BAR_SUBRANGE:
1160 bar = arg;
1161 if (bar <= NO_BAR || bar > BAR_5)
1162 goto ret;
1163 if (is_am654_pci_dev(pdev) && bar == BAR_0)
1164 goto ret;
1165
1166 if (bar_is_reserved(test, bar)) {
1167 ret = -ENOBUFS;
1168 goto ret;
1169 }
1170
1171 if (cmd == PCITEST_BAR)
1172 ret = pci_endpoint_test_bar(test, bar);
1173 else
1174 ret = pci_endpoint_test_bar_subrange(test, bar);
1175 break;
1176 case PCITEST_BARS:
1177 ret = pci_endpoint_test_bars(test);
1178 break;
1179 case PCITEST_INTX_IRQ:
1180 ret = pci_endpoint_test_intx_irq(test);
1181 break;
1182 case PCITEST_MSI:
1183 case PCITEST_MSIX:
1184 ret = pci_endpoint_test_msi_irq(test, arg, cmd == PCITEST_MSIX);
1185 break;
1186 case PCITEST_WRITE:
1187 ret = pci_endpoint_test_write(test, arg);
1188 break;
1189 case PCITEST_READ:
1190 ret = pci_endpoint_test_read(test, arg);
1191 break;
1192 case PCITEST_COPY:
1193 ret = pci_endpoint_test_copy(test, arg);
1194 break;
1195 case PCITEST_SET_IRQTYPE:
1196 ret = pci_endpoint_test_set_irq(test, arg);
1197 break;
1198 case PCITEST_GET_IRQTYPE:
1199 ret = test->irq_type;
1200 break;
1201 case PCITEST_CLEAR_IRQ:
1202 ret = pci_endpoint_test_clear_irq(test);
1203 break;
1204 case PCITEST_DOORBELL:
1205 ret = pci_endpoint_test_doorbell(test);
1206 break;
1207 }
1208
1209ret:
1210 mutex_unlock(&test->mutex);
1211 return ret;
1212}
1213
1214static const struct file_operations pci_endpoint_test_fops = {
1215 .owner = THIS_MODULE,
1216 .unlocked_ioctl = pci_endpoint_test_ioctl,
1217};
1218
1219static void pci_endpoint_test_get_capabilities(struct pci_endpoint_test *test)
1220{
1221 struct pci_dev *pdev = test->pdev;
1222 struct device *dev = &pdev->dev;
1223
1224 test->ep_caps = pci_endpoint_test_readl(test, PCI_ENDPOINT_TEST_CAPS);
1225 dev_dbg(dev, "PCI_ENDPOINT_TEST_CAPS: %#x\n", test->ep_caps);
1226
1227 /* CAP_UNALIGNED_ACCESS is set if the EP can do unaligned access */
1228 if (test->ep_caps & CAP_UNALIGNED_ACCESS)
1229 test->alignment = 0;
1230}
1231
1232static int pci_endpoint_test_probe(struct pci_dev *pdev,
1233 const struct pci_device_id *ent)
1234{
1235 int ret;
1236 int id;
1237 char name[29];
1238 enum pci_barno bar;
1239 void __iomem *base;
1240 struct device *dev = &pdev->dev;
1241 struct pci_endpoint_test *test;
1242 struct pci_endpoint_test_data *data;
1243 enum pci_barno test_reg_bar = BAR_0;
1244 struct miscdevice *misc_device;
1245
1246 if (pci_is_bridge(pdev))
1247 return -ENODEV;
1248
1249 test = devm_kzalloc(dev, sizeof(*test), GFP_KERNEL);
1250 if (!test)
1251 return -ENOMEM;
1252
1253 test->pdev = pdev;
1254 test->irq_type = PCITEST_IRQ_TYPE_UNDEFINED;
1255
1256 data = (struct pci_endpoint_test_data *)ent->driver_data;
1257 if (data) {
1258 test_reg_bar = data->test_reg_bar;
1259 test->test_reg_bar = test_reg_bar;
1260 test->alignment = data->alignment;
1261 }
1262
1263 init_completion(&test->irq_raised);
1264 mutex_init(&test->mutex);
1265
1266 dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(48));
1267
1268 ret = pci_enable_device(pdev);
1269 if (ret) {
1270 dev_err(dev, "Cannot enable PCI device\n");
1271 return ret;
1272 }
1273
1274 ret = pci_request_regions(pdev, DRV_MODULE_NAME);
1275 if (ret) {
1276 dev_err(dev, "Cannot obtain PCI resources\n");
1277 goto err_disable_pdev;
1278 }
1279
1280 pci_set_master(pdev);
1281
1282 for (bar = 0; bar < PCI_STD_NUM_BARS; bar++) {
1283 if (pci_resource_flags(pdev, bar) & IORESOURCE_MEM) {
1284 base = pci_ioremap_bar(pdev, bar);
1285 if (!base) {
1286 dev_err(dev, "Failed to read BAR%d\n", bar);
1287 WARN_ON(bar == test_reg_bar);
1288 }
1289 test->bar[bar] = base;
1290 }
1291 }
1292
1293 test->base = test->bar[test_reg_bar];
1294 if (!test->base) {
1295 ret = -ENOMEM;
1296 dev_err(dev, "Cannot perform PCI test without BAR%d\n",
1297 test_reg_bar);
1298 goto err_iounmap;
1299 }
1300
1301 pci_set_drvdata(pdev, test);
1302
1303 id = ida_alloc(&pci_endpoint_test_ida, GFP_KERNEL);
1304 if (id < 0) {
1305 ret = id;
1306 dev_err(dev, "Unable to get id\n");
1307 goto err_iounmap;
1308 }
1309
1310 snprintf(name, sizeof(name), DRV_MODULE_NAME ".%d", id);
1311 test->name = kstrdup(name, GFP_KERNEL);
1312 if (!test->name) {
1313 ret = -ENOMEM;
1314 goto err_ida_remove;
1315 }
1316
1317 pci_endpoint_test_get_capabilities(test);
1318
1319 misc_device = &test->miscdev;
1320 misc_device->minor = MISC_DYNAMIC_MINOR;
1321 misc_device->name = kstrdup(name, GFP_KERNEL);
1322 if (!misc_device->name) {
1323 ret = -ENOMEM;
1324 goto err_kfree_test_name;
1325 }
1326 misc_device->parent = &pdev->dev;
1327 misc_device->fops = &pci_endpoint_test_fops;
1328
1329 pci_save_state(pdev);
1330
1331 ret = misc_register(misc_device);
1332 if (ret) {
1333 dev_err(dev, "Failed to register device\n");
1334 goto err_kfree_name;
1335 }
1336
1337 return 0;
1338
1339err_kfree_name:
1340 kfree(misc_device->name);
1341
1342err_kfree_test_name:
1343 kfree(test->name);
1344
1345err_ida_remove:
1346 ida_free(&pci_endpoint_test_ida, id);
1347
1348err_iounmap:
1349 for (bar = 0; bar < PCI_STD_NUM_BARS; bar++) {
1350 if (test->bar[bar])
1351 pci_iounmap(pdev, test->bar[bar]);
1352 }
1353
1354 pci_release_regions(pdev);
1355
1356err_disable_pdev:
1357 pci_disable_device(pdev);
1358
1359 return ret;
1360}
1361
1362static void pci_endpoint_test_remove(struct pci_dev *pdev)
1363{
1364 int id;
1365 enum pci_barno bar;
1366 struct pci_endpoint_test *test = pci_get_drvdata(pdev);
1367 struct miscdevice *misc_device = &test->miscdev;
1368
1369 if (sscanf(misc_device->name, DRV_MODULE_NAME ".%d", &id) != 1)
1370 return;
1371 if (id < 0)
1372 return;
1373
1374 pci_endpoint_test_release_irq(test);
1375 pci_endpoint_test_free_irq_vectors(test);
1376
1377 misc_deregister(&test->miscdev);
1378 kfree(misc_device->name);
1379 kfree(test->name);
1380 ida_free(&pci_endpoint_test_ida, id);
1381 for (bar = 0; bar < PCI_STD_NUM_BARS; bar++) {
1382 if (test->bar[bar])
1383 pci_iounmap(pdev, test->bar[bar]);
1384 }
1385
1386 pci_release_regions(pdev);
1387 pci_disable_device(pdev);
1388}
1389
1390static const struct pci_endpoint_test_data default_data = {
1391 .test_reg_bar = BAR_0,
1392 .alignment = SZ_4K,
1393};
1394
1395static const struct pci_endpoint_test_data am654_data = {
1396 .test_reg_bar = BAR_2,
1397 .alignment = SZ_64K,
1398};
1399
1400static const struct pci_endpoint_test_data j721e_data = {
1401 .alignment = 256,
1402};
1403
1404static const struct pci_endpoint_test_data rk3588_data = {
1405 .alignment = SZ_64K,
1406};
1407
1408/*
1409 * If the controller's Vendor/Device ID are programmable, you may be able to
1410 * use one of the existing entries for testing instead of adding a new one.
1411 */
1412static const struct pci_device_id pci_endpoint_test_tbl[] = {
1413 { PCI_DEVICE(PCI_VENDOR_ID_TI, PCI_DEVICE_ID_TI_DRA74x),
1414 .driver_data = (kernel_ulong_t)&default_data,
1415 },
1416 { PCI_DEVICE(PCI_VENDOR_ID_TI, PCI_DEVICE_ID_TI_DRA72x),
1417 .driver_data = (kernel_ulong_t)&default_data,
1418 },
1419 { PCI_DEVICE(PCI_VENDOR_ID_FREESCALE, 0x81c0),
1420 .driver_data = (kernel_ulong_t)&default_data,
1421 },
1422 { PCI_DEVICE(PCI_VENDOR_ID_FREESCALE, PCI_DEVICE_ID_IMX8) },
1423 { PCI_DEVICE(PCI_VENDOR_ID_FREESCALE, PCI_DEVICE_ID_LS1088A),
1424 .driver_data = (kernel_ulong_t)&default_data,
1425 },
1426 { PCI_DEVICE_DATA(SYNOPSYS, EDDA, NULL) },
1427 { PCI_DEVICE(PCI_VENDOR_ID_TI, PCI_DEVICE_ID_TI_AM654),
1428 .driver_data = (kernel_ulong_t)&am654_data,
1429 },
1430 { PCI_DEVICE(PCI_VENDOR_ID_RENESAS, PCI_DEVICE_ID_RENESAS_R8A774A1) },
1431 { PCI_DEVICE(PCI_VENDOR_ID_RENESAS, PCI_DEVICE_ID_RENESAS_R8A774B1) },
1432 { PCI_DEVICE(PCI_VENDOR_ID_RENESAS, PCI_DEVICE_ID_RENESAS_R8A774C0) },
1433 { PCI_DEVICE(PCI_VENDOR_ID_RENESAS, PCI_DEVICE_ID_RENESAS_R8A774E1) },
1434 { PCI_DEVICE(PCI_VENDOR_ID_RENESAS, PCI_DEVICE_ID_RENESAS_R8A779F0),
1435 .driver_data = (kernel_ulong_t)&default_data,
1436 },
1437 { PCI_DEVICE(PCI_VENDOR_ID_TI, PCI_DEVICE_ID_TI_J721E),
1438 .driver_data = (kernel_ulong_t)&j721e_data,
1439 },
1440 { PCI_DEVICE(PCI_VENDOR_ID_TI, PCI_DEVICE_ID_TI_J7200),
1441 .driver_data = (kernel_ulong_t)&j721e_data,
1442 },
1443 { PCI_DEVICE(PCI_VENDOR_ID_TI, PCI_DEVICE_ID_TI_AM64),
1444 .driver_data = (kernel_ulong_t)&j721e_data,
1445 },
1446 { PCI_DEVICE(PCI_VENDOR_ID_TI, PCI_DEVICE_ID_TI_J721S2),
1447 .driver_data = (kernel_ulong_t)&j721e_data,
1448 },
1449 { PCI_DEVICE(PCI_VENDOR_ID_ROCKCHIP, PCI_DEVICE_ID_ROCKCHIP_RK3588),
1450 .driver_data = (kernel_ulong_t)&rk3588_data,
1451 },
1452 { PCI_DEVICE(PCI_VENDOR_ID_NVIDIA, PCI_DEVICE_ID_NVIDIA_TEGRA194_EP) },
1453 { PCI_DEVICE(PCI_VENDOR_ID_NVIDIA, PCI_DEVICE_ID_NVIDIA_TEGRA234_EP) },
1454 { }
1455};
1456MODULE_DEVICE_TABLE(pci, pci_endpoint_test_tbl);
1457
1458static pci_ers_result_t pci_endpoint_test_error_detected(struct pci_dev *pdev,
1459 pci_channel_state_t state)
1460{
1461 if (state == pci_channel_io_perm_failure)
1462 return PCI_ERS_RESULT_DISCONNECT;
1463
1464 return PCI_ERS_RESULT_NEED_RESET;
1465}
1466
1467static pci_ers_result_t pci_endpoint_test_slot_reset(struct pci_dev *pdev)
1468{
1469 pci_restore_state(pdev);
1470 return PCI_ERS_RESULT_RECOVERED;
1471}
1472
1473static const struct pci_error_handlers pci_endpoint_test_err_handler = {
1474 .error_detected = pci_endpoint_test_error_detected,
1475 .slot_reset = pci_endpoint_test_slot_reset,
1476};
1477
1478static struct pci_driver pci_endpoint_test_driver = {
1479 .name = DRV_MODULE_NAME,
1480 .id_table = pci_endpoint_test_tbl,
1481 .probe = pci_endpoint_test_probe,
1482 .remove = pci_endpoint_test_remove,
1483 .sriov_configure = pci_sriov_configure_simple,
1484 .err_handler = &pci_endpoint_test_err_handler,
1485};
1486module_pci_driver(pci_endpoint_test_driver);
1487
1488MODULE_DESCRIPTION("PCI ENDPOINT TEST HOST DRIVER");
1489MODULE_AUTHOR("Kishon Vijay Abraham I <[email protected]>");
1490MODULE_LICENSE("GPL v2");