projects/linux·blob master · refs

1378 lines · 35.7 KiB

1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * DMA Engine test module
4 *
5 * Copyright (C) 2007 Atmel Corporation
6 * Copyright (C) 2013 Intel Corporation
7 */
8#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
9
10#include <linux/err.h>
11#include <linux/delay.h>
12#include <linux/dma-mapping.h>
13#include <linux/dmaengine.h>
14#include <linux/freezer.h>
15#include <linux/init.h>
16#include <linux/kthread.h>
17#include <linux/sched/task.h>
18#include <linux/module.h>
19#include <linux/moduleparam.h>
20#include <linux/random.h>
21#include <linux/slab.h>
22#include <linux/wait.h>
23
24static bool nobounce;
25module_param(nobounce, bool, 0644);
26MODULE_PARM_DESC(nobounce, "Prevent using swiotlb buffer (default: use swiotlb buffer)");
27
28static unsigned int test_buf_size = 16384;
29module_param(test_buf_size, uint, 0644);
30MODULE_PARM_DESC(test_buf_size, "Size of the memcpy test buffer");
31
32static char test_device[32];
33module_param_string(device, test_device, sizeof(test_device), 0644);
34MODULE_PARM_DESC(device, "Bus ID of the DMA Engine to test (default: any)");
35
36static unsigned int threads_per_chan = 1;
37module_param(threads_per_chan, uint, 0644);
38MODULE_PARM_DESC(threads_per_chan,
39 "Number of threads to start per channel (default: 1)");
40
41static unsigned int max_channels;
42module_param(max_channels, uint, 0644);
43MODULE_PARM_DESC(max_channels,
44 "Maximum number of channels to use (default: all)");
45
46static unsigned int iterations;
47module_param(iterations, uint, 0644);
48MODULE_PARM_DESC(iterations,
49 "Iterations before stopping test (default: infinite)");
50
51static unsigned int dmatest;
52module_param(dmatest, uint, 0644);
53MODULE_PARM_DESC(dmatest,
54 "dmatest 0-memcpy 1-memset (default: 0)");
55
56static unsigned int xor_sources = 3;
57module_param(xor_sources, uint, 0644);
58MODULE_PARM_DESC(xor_sources,
59 "Number of xor source buffers (default: 3)");
60
61static unsigned int pq_sources = 3;
62module_param(pq_sources, uint, 0644);
63MODULE_PARM_DESC(pq_sources,
64 "Number of p+q source buffers (default: 3)");
65
66static int timeout = 3000;
67module_param(timeout, int, 0644);
68MODULE_PARM_DESC(timeout, "Transfer Timeout in msec (default: 3000), "
69 "Pass -1 for infinite timeout");
70
71static bool noverify;
72module_param(noverify, bool, 0644);
73MODULE_PARM_DESC(noverify, "Disable data verification (default: verify)");
74
75static bool norandom;
76module_param(norandom, bool, 0644);
77MODULE_PARM_DESC(norandom, "Disable random offset setup (default: random)");
78
79static bool verbose;
80module_param(verbose, bool, 0644);
81MODULE_PARM_DESC(verbose, "Enable \"success\" result messages (default: off)");
82
83static int alignment = -1;
84module_param(alignment, int, 0644);
85MODULE_PARM_DESC(alignment, "Custom data address alignment taken as 2^(alignment) (default: not used (-1))");
86
87static unsigned int transfer_size;
88module_param(transfer_size, uint, 0644);
89MODULE_PARM_DESC(transfer_size, "Optional custom transfer size in bytes (default: not used (0))");
90
91static bool polled;
92module_param(polled, bool, 0644);
93MODULE_PARM_DESC(polled, "Use polling for completion instead of interrupts");
94
95/**
96 * struct dmatest_params - test parameters.
97 * @nobounce: prevent using swiotlb buffer
98 * @buf_size: size of the memcpy test buffer
99 * @channel: bus ID of the channel to test
100 * @device: bus ID of the DMA Engine to test
101 * @threads_per_chan: number of threads to start per channel
102 * @max_channels: maximum number of channels to use
103 * @iterations: iterations before stopping test
104 * @xor_sources: number of xor source buffers
105 * @pq_sources: number of p+q source buffers
106 * @timeout: transfer timeout in msec, -1 for infinite timeout
107 * @noverify: disable data verification
108 * @norandom: disable random offset setup
109 * @alignment: custom data address alignment taken as 2^alignment
110 * @transfer_size: custom transfer size in bytes
111 * @polled: use polling for completion instead of interrupts
112 */
113struct dmatest_params {
114 bool nobounce;
115 unsigned int buf_size;
116 char channel[20];
117 char device[32];
118 unsigned int threads_per_chan;
119 unsigned int max_channels;
120 unsigned int iterations;
121 unsigned int xor_sources;
122 unsigned int pq_sources;
123 int timeout;
124 bool noverify;
125 bool norandom;
126 int alignment;
127 unsigned int transfer_size;
128 bool polled;
129};
130
131/**
132 * struct dmatest_info - test information.
133 * @params: test parameters
134 * @channels: channels under test
135 * @nr_channels: number of channels under test
136 * @lock: access protection to the fields of this structure
137 * @did_init: module has been initialized completely
138 * @last_error: test has faced configuration issues
139 */
140struct dmatest_info {
141 /* Test parameters */
142 struct dmatest_params params;
143
144 /* Internal state */
145 struct list_head channels;
146 unsigned int nr_channels;
147 int last_error;
148 struct mutex lock;
149 bool did_init;
150};
151
152static struct dmatest_info test_info = {
153 .channels = LIST_HEAD_INIT(test_info.channels),
154 .lock = __MUTEX_INITIALIZER(test_info.lock),
155};
156
157static int dmatest_run_set(const char *val, const struct kernel_param *kp);
158static int dmatest_run_get(char *val, const struct kernel_param *kp);
159static const struct kernel_param_ops run_ops = {
160 .set = dmatest_run_set,
161 .get = dmatest_run_get,
162};
163static bool dmatest_run;
164module_param_cb(run, &run_ops, &dmatest_run, 0644);
165MODULE_PARM_DESC(run, "Run the test (default: false)");
166
167static int dmatest_chan_set(const char *val, const struct kernel_param *kp);
168static int dmatest_chan_get(char *val, const struct kernel_param *kp);
169static const struct kernel_param_ops multi_chan_ops = {
170 .set = dmatest_chan_set,
171 .get = dmatest_chan_get,
172};
173
174static char test_channel[20];
175static struct kparam_string newchan_kps = {
176 .string = test_channel,
177 .maxlen = 20,
178};
179module_param_cb(channel, &multi_chan_ops, &newchan_kps, 0644);
180MODULE_PARM_DESC(channel, "Bus ID of the channel to test (default: any)");
181
182static int dmatest_test_list_get(char *val, const struct kernel_param *kp);
183static const struct kernel_param_ops test_list_ops = {
184 .get = dmatest_test_list_get,
185};
186module_param_cb(test_list, &test_list_ops, NULL, 0444);
187MODULE_PARM_DESC(test_list, "Print current test list");
188
189/* Maximum amount of mismatched bytes in buffer to print */
190#define MAX_ERROR_COUNT 32
191
192/*
193 * Initialization patterns. All bytes in the source buffer has bit 7
194 * set, all bytes in the destination buffer has bit 7 cleared.
195 *
196 * Bit 6 is set for all bytes which are to be copied by the DMA
197 * engine. Bit 5 is set for all bytes which are to be overwritten by
198 * the DMA engine.
199 *
200 * The remaining bits are the inverse of a counter which increments by
201 * one for each byte address.
202 */
203#define PATTERN_SRC 0x80
204#define PATTERN_DST 0x00
205#define PATTERN_COPY 0x40
206#define PATTERN_OVERWRITE 0x20
207#define PATTERN_COUNT_MASK 0x1f
208#define PATTERN_MEMSET_IDX 0x01
209
210/* Fixed point arithmetic ops */
211#define FIXPT_SHIFT 8
212#define FIXPNT_MASK 0xFF
213#define FIXPT_TO_INT(a) ((a) >> FIXPT_SHIFT)
214#define INT_TO_FIXPT(a) ((a) << FIXPT_SHIFT)
215#define FIXPT_GET_FRAC(a) ((((a) & FIXPNT_MASK) * 100) >> FIXPT_SHIFT)
216
217/* poor man's completion - we want to use wait_event_freezable() on it */
218struct dmatest_done {
219 bool done;
220 wait_queue_head_t *wait;
221};
222
223struct dmatest_data {
224 u8 **raw;
225 u8 **aligned;
226 gfp_t gfp_flags;
227 unsigned int cnt;
228 unsigned int off;
229};
230
231struct dmatest_thread {
232 struct list_head node;
233 struct dmatest_info *info;
234 struct task_struct *task;
235 struct dma_chan *chan;
236 struct dmatest_data src;
237 struct dmatest_data dst;
238 enum dma_transaction_type type;
239 wait_queue_head_t done_wait;
240 struct dmatest_done test_done;
241 bool done;
242 bool pending;
243};
244
245struct dmatest_chan {
246 struct list_head node;
247 struct dma_chan *chan;
248 struct list_head threads;
249};
250
251static DECLARE_WAIT_QUEUE_HEAD(thread_wait);
252static bool wait;
253
254static bool is_threaded_test_run(struct dmatest_info *info)
255{
256 struct dmatest_chan *dtc;
257
258 list_for_each_entry(dtc, &info->channels, node) {
259 struct dmatest_thread *thread;
260
261 list_for_each_entry(thread, &dtc->threads, node) {
262 if (!thread->done && !thread->pending)
263 return true;
264 }
265 }
266
267 return false;
268}
269
270static bool is_threaded_test_pending(struct dmatest_info *info)
271{
272 struct dmatest_chan *dtc;
273
274 list_for_each_entry(dtc, &info->channels, node) {
275 struct dmatest_thread *thread;
276
277 list_for_each_entry(thread, &dtc->threads, node) {
278 if (thread->pending)
279 return true;
280 }
281 }
282
283 return false;
284}
285
286static int dmatest_wait_get(char *val, const struct kernel_param *kp)
287{
288 struct dmatest_info *info = &test_info;
289 struct dmatest_params *params = &info->params;
290
291 if (params->iterations)
292 wait_event(thread_wait, !is_threaded_test_run(info));
293 wait = true;
294 return param_get_bool(val, kp);
295}
296
297static const struct kernel_param_ops wait_ops = {
298 .get = dmatest_wait_get,
299 .set = param_set_bool,
300};
301module_param_cb(wait, &wait_ops, &wait, 0444);
302MODULE_PARM_DESC(wait, "Wait for tests to complete (default: false)");
303
304static bool dmatest_match_channel(struct dmatest_params *params,
305 struct dma_chan *chan)
306{
307 if (params->channel[0] == '\0')
308 return true;
309 return strcmp(dma_chan_name(chan), params->channel) == 0;
310}
311
312static bool dmatest_match_device(struct dmatest_params *params,
313 struct dma_device *device)
314{
315 if (params->device[0] == '\0')
316 return true;
317 return strcmp(dev_name(device->dev), params->device) == 0;
318}
319
320static unsigned long dmatest_random(void)
321{
322 unsigned long buf;
323
324 get_random_bytes(&buf, sizeof(buf));
325 return buf;
326}
327
328static inline u8 gen_inv_idx(u8 index, bool is_memset)
329{
330 u8 val = is_memset ? PATTERN_MEMSET_IDX : index;
331
332 return ~val & PATTERN_COUNT_MASK;
333}
334
335static inline u8 gen_src_value(u8 index, bool is_memset)
336{
337 return PATTERN_SRC | gen_inv_idx(index, is_memset);
338}
339
340static inline u8 gen_dst_value(u8 index, bool is_memset)
341{
342 return PATTERN_DST | gen_inv_idx(index, is_memset);
343}
344
345static void dmatest_init_srcs(u8 **bufs, unsigned int start, unsigned int len,
346 unsigned int buf_size, bool is_memset)
347{
348 unsigned int i;
349 u8 *buf;
350
351 for (; (buf = *bufs); bufs++) {
352 for (i = 0; i < start; i++)
353 buf[i] = gen_src_value(i, is_memset);
354 for ( ; i < start + len; i++)
355 buf[i] = gen_src_value(i, is_memset) | PATTERN_COPY;
356 for ( ; i < buf_size; i++)
357 buf[i] = gen_src_value(i, is_memset);
358 buf++;
359 }
360}
361
362static void dmatest_init_dsts(u8 **bufs, unsigned int start, unsigned int len,
363 unsigned int buf_size, bool is_memset)
364{
365 unsigned int i;
366 u8 *buf;
367
368 for (; (buf = *bufs); bufs++) {
369 for (i = 0; i < start; i++)
370 buf[i] = gen_dst_value(i, is_memset);
371 for ( ; i < start + len; i++)
372 buf[i] = gen_dst_value(i, is_memset) |
373 PATTERN_OVERWRITE;
374 for ( ; i < buf_size; i++)
375 buf[i] = gen_dst_value(i, is_memset);
376 }
377}
378
379static void dmatest_mismatch(u8 actual, u8 pattern, unsigned int index,
380 unsigned int counter, bool is_srcbuf, bool is_memset)
381{
382 u8 diff = actual ^ pattern;
383 u8 expected = pattern | gen_inv_idx(counter, is_memset);
384 const char *thread_name = current->comm;
385
386 if (is_srcbuf)
387 pr_warn("%s: srcbuf[0x%x] overwritten! Expected %02x, got %02x\n",
388 thread_name, index, expected, actual);
389 else if ((pattern & PATTERN_COPY)
390 && (diff & (PATTERN_COPY | PATTERN_OVERWRITE)))
391 pr_warn("%s: dstbuf[0x%x] not copied! Expected %02x, got %02x\n",
392 thread_name, index, expected, actual);
393 else if (diff & PATTERN_SRC)
394 pr_warn("%s: dstbuf[0x%x] was copied! Expected %02x, got %02x\n",
395 thread_name, index, expected, actual);
396 else
397 pr_warn("%s: dstbuf[0x%x] mismatch! Expected %02x, got %02x\n",
398 thread_name, index, expected, actual);
399}
400
401static unsigned int dmatest_verify(u8 **bufs, unsigned int start,
402 unsigned int end, unsigned int counter, u8 pattern,
403 bool is_srcbuf, bool is_memset)
404{
405 unsigned int i;
406 unsigned int error_count = 0;
407 u8 actual;
408 u8 expected;
409 u8 *buf;
410 unsigned int counter_orig = counter;
411
412 for (; (buf = *bufs); bufs++) {
413 counter = counter_orig;
414 for (i = start; i < end; i++) {
415 actual = buf[i];
416 expected = pattern | gen_inv_idx(counter, is_memset);
417 if (actual != expected) {
418 if (error_count < MAX_ERROR_COUNT)
419 dmatest_mismatch(actual, pattern, i,
420 counter, is_srcbuf,
421 is_memset);
422 error_count++;
423 }
424 counter++;
425 }
426 }
427
428 if (error_count > MAX_ERROR_COUNT)
429 pr_warn("%s: %u errors suppressed\n",
430 current->comm, error_count - MAX_ERROR_COUNT);
431
432 return error_count;
433}
434
435
436static void dmatest_callback(void *arg)
437{
438 struct dmatest_done *done = arg;
439 struct dmatest_thread *thread =
440 container_of(done, struct dmatest_thread, test_done);
441 if (!thread->done) {
442 done->done = true;
443 wake_up_all(done->wait);
444 } else {
445 /*
446 * If thread->done, it means that this callback occurred
447 * after the parent thread has cleaned up. This can
448 * happen in the case that driver doesn't implement
449 * the terminate_all() functionality and a dma operation
450 * did not occur within the timeout period
451 */
452 WARN(1, "dmatest: Kernel memory may be corrupted!!\n");
453 }
454}
455
456static unsigned int min_odd(unsigned int x, unsigned int y)
457{
458 unsigned int val = min(x, y);
459
460 return val % 2 ? val : val - 1;
461}
462
463static void result(const char *err, unsigned int n, unsigned int src_off,
464 unsigned int dst_off, unsigned int len, unsigned long data)
465{
466 if (IS_ERR_VALUE(data)) {
467 pr_info("%s: result #%u: '%s' with src_off=0x%x dst_off=0x%x len=0x%x (%ld)\n",
468 current->comm, n, err, src_off, dst_off, len, data);
469 } else {
470 pr_info("%s: result #%u: '%s' with src_off=0x%x dst_off=0x%x len=0x%x (%lu)\n",
471 current->comm, n, err, src_off, dst_off, len, data);
472 }
473}
474
475static void dbg_result(const char *err, unsigned int n, unsigned int src_off,
476 unsigned int dst_off, unsigned int len,
477 unsigned long data)
478{
479 pr_debug("%s: result #%u: '%s' with src_off=0x%x dst_off=0x%x len=0x%x (%lu)\n",
480 current->comm, n, err, src_off, dst_off, len, data);
481}
482
483#define verbose_result(err, n, src_off, dst_off, len, data) ({ \
484 if (verbose) \
485 result(err, n, src_off, dst_off, len, data); \
486 else \
487 dbg_result(err, n, src_off, dst_off, len, data);\
488})
489
490static unsigned long long dmatest_persec(s64 runtime, unsigned int val)
491{
492 unsigned long long per_sec = 1000000;
493
494 if (runtime <= 0)
495 return 0;
496
497 /* drop precision until runtime is 32-bits */
498 while (runtime > UINT_MAX) {
499 runtime >>= 1;
500 per_sec <<= 1;
501 }
502
503 per_sec *= val;
504 per_sec = INT_TO_FIXPT(per_sec);
505 do_div(per_sec, (u32)runtime);
506
507 return per_sec;
508}
509
510static unsigned long long dmatest_KBs(s64 runtime, unsigned long long len)
511{
512 return FIXPT_TO_INT(dmatest_persec(runtime, len >> 10));
513}
514
515static void __dmatest_free_test_data(struct dmatest_data *d, unsigned int cnt)
516{
517 unsigned int i;
518
519 for (i = 0; i < cnt; i++)
520 kfree(d->raw[i]);
521
522 kfree(d->aligned);
523 kfree(d->raw);
524}
525
526static void dmatest_free_test_data(struct dmatest_data *d)
527{
528 __dmatest_free_test_data(d, d->cnt);
529}
530
531static int dmatest_alloc_test_data(struct dmatest_data *d,
532 unsigned int buf_size, u8 align)
533{
534 unsigned int i = 0;
535
536 d->raw = kcalloc(d->cnt + 1, sizeof(u8 *), GFP_KERNEL);
537 if (!d->raw)
538 return -ENOMEM;
539
540 d->aligned = kcalloc(d->cnt + 1, sizeof(u8 *), GFP_KERNEL);
541 if (!d->aligned)
542 goto err;
543
544 for (i = 0; i < d->cnt; i++) {
545 d->raw[i] = kmalloc(buf_size + align, d->gfp_flags);
546 if (!d->raw[i])
547 goto err;
548
549 /* align to alignment restriction */
550 if (align)
551 d->aligned[i] = PTR_ALIGN(d->raw[i], align);
552 else
553 d->aligned[i] = d->raw[i];
554 }
555
556 return 0;
557err:
558 __dmatest_free_test_data(d, i);
559 return -ENOMEM;
560}
561
562/*
563 * This function repeatedly tests DMA transfers of various lengths and
564 * offsets for a given operation type until it is told to exit by
565 * kthread_stop(). There may be multiple threads running this function
566 * in parallel for a single channel, and there may be multiple channels
567 * being tested in parallel.
568 *
569 * Before each test, the source and destination buffer is initialized
570 * with a known pattern. This pattern is different depending on
571 * whether it's in an area which is supposed to be copied or
572 * overwritten, and different in the source and destination buffers.
573 * So if the DMA engine doesn't copy exactly what we tell it to copy,
574 * we'll notice.
575 */
576static int dmatest_func(void *data)
577{
578 struct dmatest_thread *thread = data;
579 struct dmatest_done *done = &thread->test_done;
580 struct dmatest_info *info;
581 struct dmatest_params *params;
582 struct dma_chan *chan;
583 struct dma_device *dev;
584 struct device *dma_dev;
585 unsigned int error_count;
586 unsigned int failed_tests = 0;
587 unsigned int total_tests = 0;
588 dma_cookie_t cookie;
589 enum dma_status status;
590 enum dma_ctrl_flags flags;
591 u8 *pq_coefs = NULL;
592 int ret;
593 unsigned int buf_size;
594 struct dmatest_data *src;
595 struct dmatest_data *dst;
596 int i;
597 ktime_t ktime, start, diff;
598 ktime_t filltime = 0;
599 ktime_t comparetime = 0;
600 s64 runtime = 0;
601 unsigned long long total_len = 0;
602 unsigned long long iops = 0;
603 u8 align = 0;
604 bool is_memset = false;
605 dma_addr_t *srcs;
606 dma_addr_t *dma_pq;
607
608 set_freezable();
609
610 ret = -ENOMEM;
611
612 smp_rmb();
613 thread->pending = false;
614 info = thread->info;
615 params = &info->params;
616 chan = thread->chan;
617 dev = chan->device;
618 dma_dev = dmaengine_get_dma_device(chan);
619
620 src = &thread->src;
621 dst = &thread->dst;
622 if (thread->type == DMA_MEMCPY) {
623 align = params->alignment < 0 ? dev->copy_align :
624 params->alignment;
625 src->cnt = dst->cnt = 1;
626 } else if (thread->type == DMA_MEMSET) {
627 align = params->alignment < 0 ? dev->fill_align :
628 params->alignment;
629 src->cnt = dst->cnt = 1;
630 is_memset = true;
631 } else if (thread->type == DMA_XOR) {
632 /* force odd to ensure dst = src */
633 src->cnt = min_odd(params->xor_sources | 1, dev->max_xor);
634 dst->cnt = 1;
635 align = params->alignment < 0 ? dev->xor_align :
636 params->alignment;
637 } else if (thread->type == DMA_PQ) {
638 /* force odd to ensure dst = src */
639 src->cnt = min_odd(params->pq_sources | 1, dma_maxpq(dev, 0));
640 dst->cnt = 2;
641 align = params->alignment < 0 ? dev->pq_align :
642 params->alignment;
643
644 pq_coefs = kmalloc(params->pq_sources + 1, GFP_KERNEL);
645 if (!pq_coefs)
646 goto err_thread_type;
647
648 for (i = 0; i < src->cnt; i++)
649 pq_coefs[i] = 1;
650 } else
651 goto err_thread_type;
652
653 /* Check if buffer count fits into map count variable (u8) */
654 if ((src->cnt + dst->cnt) >= 255) {
655 pr_err("too many buffers (%d of 255 supported)\n",
656 src->cnt + dst->cnt);
657 goto err_free_coefs;
658 }
659
660 buf_size = params->buf_size;
661 if (1 << align > buf_size) {
662 pr_err("%u-byte buffer too small for %d-byte alignment\n",
663 buf_size, 1 << align);
664 goto err_free_coefs;
665 }
666
667 src->gfp_flags = GFP_KERNEL;
668 dst->gfp_flags = GFP_KERNEL;
669 if (params->nobounce) {
670 src->gfp_flags = GFP_DMA;
671 dst->gfp_flags = GFP_DMA;
672 }
673
674 if (dmatest_alloc_test_data(src, buf_size, align) < 0)
675 goto err_free_coefs;
676
677 if (dmatest_alloc_test_data(dst, buf_size, align) < 0)
678 goto err_src;
679
680 set_user_nice(current, 10);
681
682 srcs = kzalloc_objs(dma_addr_t, src->cnt);
683 if (!srcs)
684 goto err_dst;
685
686 dma_pq = kzalloc_objs(dma_addr_t, dst->cnt);
687 if (!dma_pq)
688 goto err_srcs_array;
689
690 /*
691 * src and dst buffers are freed by ourselves below
692 */
693 if (params->polled)
694 flags = DMA_CTRL_ACK;
695 else
696 flags = DMA_CTRL_ACK | DMA_PREP_INTERRUPT;
697
698 ktime = ktime_get();
699 while (!(kthread_should_stop() ||
700 (params->iterations && total_tests >= params->iterations))) {
701 struct dma_async_tx_descriptor *tx = NULL;
702 struct dmaengine_unmap_data *um;
703 dma_addr_t *dsts;
704 unsigned int len;
705
706 total_tests++;
707
708 if (params->transfer_size) {
709 if (params->transfer_size >= buf_size) {
710 pr_err("%u-byte transfer size must be lower than %u-buffer size\n",
711 params->transfer_size, buf_size);
712 break;
713 }
714 len = params->transfer_size;
715 } else if (params->norandom) {
716 len = buf_size;
717 } else {
718 len = dmatest_random() % buf_size + 1;
719 }
720
721 /* Do not alter transfer size explicitly defined by user */
722 if (!params->transfer_size) {
723 len = (len >> align) << align;
724 if (!len)
725 len = 1 << align;
726 }
727 total_len += len;
728
729 if (params->norandom) {
730 src->off = 0;
731 dst->off = 0;
732 } else {
733 src->off = dmatest_random() % (buf_size - len + 1);
734 dst->off = dmatest_random() % (buf_size - len + 1);
735
736 src->off = (src->off >> align) << align;
737 dst->off = (dst->off >> align) << align;
738 }
739
740 if (!params->noverify) {
741 start = ktime_get();
742 dmatest_init_srcs(src->aligned, src->off, len,
743 buf_size, is_memset);
744 dmatest_init_dsts(dst->aligned, dst->off, len,
745 buf_size, is_memset);
746
747 diff = ktime_sub(ktime_get(), start);
748 filltime = ktime_add(filltime, diff);
749 }
750
751 um = dmaengine_get_unmap_data(dma_dev, src->cnt + dst->cnt,
752 GFP_KERNEL);
753 if (!um) {
754 failed_tests++;
755 result("unmap data NULL", total_tests,
756 src->off, dst->off, len, ret);
757 continue;
758 }
759
760 um->len = buf_size;
761 for (i = 0; i < src->cnt; i++) {
762 void *buf = src->aligned[i];
763 struct page *pg = virt_to_page(buf);
764 unsigned long pg_off = offset_in_page(buf);
765
766 um->addr[i] = dma_map_page(dma_dev, pg, pg_off,
767 um->len, DMA_TO_DEVICE);
768 srcs[i] = um->addr[i] + src->off;
769 ret = dma_mapping_error(dma_dev, um->addr[i]);
770 if (ret) {
771 result("src mapping error", total_tests,
772 src->off, dst->off, len, ret);
773 goto error_unmap_continue;
774 }
775 um->to_cnt++;
776 }
777 /* map with DMA_BIDIRECTIONAL to force writeback/invalidate */
778 dsts = &um->addr[src->cnt];
779 for (i = 0; i < dst->cnt; i++) {
780 void *buf = dst->aligned[i];
781 struct page *pg = virt_to_page(buf);
782 unsigned long pg_off = offset_in_page(buf);
783
784 dsts[i] = dma_map_page(dma_dev, pg, pg_off, um->len,
785 DMA_BIDIRECTIONAL);
786 ret = dma_mapping_error(dma_dev, dsts[i]);
787 if (ret) {
788 result("dst mapping error", total_tests,
789 src->off, dst->off, len, ret);
790 goto error_unmap_continue;
791 }
792 um->bidi_cnt++;
793 }
794
795 if (thread->type == DMA_MEMCPY)
796 tx = dev->device_prep_dma_memcpy(chan,
797 dsts[0] + dst->off,
798 srcs[0], len, flags);
799 else if (thread->type == DMA_MEMSET)
800 tx = dev->device_prep_dma_memset(chan,
801 dsts[0] + dst->off,
802 *(src->aligned[0] + src->off),
803 len, flags);
804 else if (thread->type == DMA_XOR)
805 tx = dev->device_prep_dma_xor(chan,
806 dsts[0] + dst->off,
807 srcs, src->cnt,
808 len, flags);
809 else if (thread->type == DMA_PQ) {
810 for (i = 0; i < dst->cnt; i++)
811 dma_pq[i] = dsts[i] + dst->off;
812 tx = dev->device_prep_dma_pq(chan, dma_pq, srcs,
813 src->cnt, pq_coefs,
814 len, flags);
815 }
816
817 if (!tx) {
818 result("prep error", total_tests, src->off,
819 dst->off, len, ret);
820 msleep(100);
821 goto error_unmap_continue;
822 }
823
824 done->done = false;
825 if (!params->polled) {
826 tx->callback = dmatest_callback;
827 tx->callback_param = done;
828 }
829 cookie = tx->tx_submit(tx);
830
831 if (dma_submit_error(cookie)) {
832 result("submit error", total_tests, src->off,
833 dst->off, len, ret);
834 msleep(100);
835 goto error_unmap_continue;
836 }
837
838 if (params->polled) {
839 status = dma_sync_wait(chan, cookie);
840 dmaengine_terminate_sync(chan);
841 if (status == DMA_COMPLETE)
842 done->done = true;
843 } else {
844 dma_async_issue_pending(chan);
845
846 wait_event_freezable_timeout(thread->done_wait,
847 done->done,
848 msecs_to_jiffies(params->timeout));
849
850 status = dma_async_is_tx_complete(chan, cookie, NULL,
851 NULL);
852 }
853
854 if (!done->done) {
855 result("test timed out", total_tests, src->off, dst->off,
856 len, 0);
857 goto error_unmap_continue;
858 } else if (status != DMA_COMPLETE &&
859 !(dma_has_cap(DMA_COMPLETION_NO_ORDER,
860 dev->cap_mask) &&
861 status == DMA_OUT_OF_ORDER)) {
862 result(status == DMA_ERROR ?
863 "completion error status" :
864 "completion busy status", total_tests, src->off,
865 dst->off, len, ret);
866 goto error_unmap_continue;
867 }
868
869 dmaengine_unmap_put(um);
870
871 if (params->noverify) {
872 verbose_result("test passed", total_tests, src->off,
873 dst->off, len, 0);
874 continue;
875 }
876
877 start = ktime_get();
878 pr_debug("%s: verifying source buffer...\n", current->comm);
879 error_count = dmatest_verify(src->aligned, 0, src->off,
880 0, PATTERN_SRC, true, is_memset);
881 error_count += dmatest_verify(src->aligned, src->off,
882 src->off + len, src->off,
883 PATTERN_SRC | PATTERN_COPY, true, is_memset);
884 error_count += dmatest_verify(src->aligned, src->off + len,
885 buf_size, src->off + len,
886 PATTERN_SRC, true, is_memset);
887
888 pr_debug("%s: verifying dest buffer...\n", current->comm);
889 error_count += dmatest_verify(dst->aligned, 0, dst->off,
890 0, PATTERN_DST, false, is_memset);
891
892 error_count += dmatest_verify(dst->aligned, dst->off,
893 dst->off + len, src->off,
894 PATTERN_SRC | PATTERN_COPY, false, is_memset);
895
896 error_count += dmatest_verify(dst->aligned, dst->off + len,
897 buf_size, dst->off + len,
898 PATTERN_DST, false, is_memset);
899
900 diff = ktime_sub(ktime_get(), start);
901 comparetime = ktime_add(comparetime, diff);
902
903 if (error_count) {
904 result("data error", total_tests, src->off, dst->off,
905 len, error_count);
906 failed_tests++;
907 } else {
908 verbose_result("test passed", total_tests, src->off,
909 dst->off, len, 0);
910 }
911
912 continue;
913
914error_unmap_continue:
915 dmaengine_unmap_put(um);
916 failed_tests++;
917 }
918 ktime = ktime_sub(ktime_get(), ktime);
919 ktime = ktime_sub(ktime, comparetime);
920 ktime = ktime_sub(ktime, filltime);
921 runtime = ktime_to_us(ktime);
922
923 ret = 0;
924 kfree(dma_pq);
925err_srcs_array:
926 kfree(srcs);
927err_dst:
928 dmatest_free_test_data(dst);
929err_src:
930 dmatest_free_test_data(src);
931err_free_coefs:
932 kfree(pq_coefs);
933err_thread_type:
934 iops = dmatest_persec(runtime, total_tests);
935 pr_info("%s: summary %u tests, %u failures %llu.%02llu iops %llu KB/s (%d)\n",
936 current->comm, total_tests, failed_tests,
937 FIXPT_TO_INT(iops), FIXPT_GET_FRAC(iops),
938 dmatest_KBs(runtime, total_len), ret);
939
940 /* terminate all transfers on specified channels */
941 if (ret || failed_tests)
942 dmaengine_terminate_sync(chan);
943
944 thread->done = true;
945 wake_up(&thread_wait);
946
947 return ret;
948}
949
950static void dmatest_cleanup_channel(struct dmatest_chan *dtc)
951{
952 struct dmatest_thread *thread;
953 struct dmatest_thread *_thread;
954 int ret;
955
956 list_for_each_entry_safe(thread, _thread, &dtc->threads, node) {
957 ret = kthread_stop(thread->task);
958 pr_debug("thread %s exited with status %d\n",
959 thread->task->comm, ret);
960 list_del(&thread->node);
961 put_task_struct(thread->task);
962 kfree(thread);
963 }
964
965 /* terminate all transfers on specified channels */
966 dmaengine_terminate_sync(dtc->chan);
967
968 kfree(dtc);
969}
970
971static int dmatest_add_threads(struct dmatest_info *info,
972 struct dmatest_chan *dtc, enum dma_transaction_type type)
973{
974 struct dmatest_params *params = &info->params;
975 struct dmatest_thread *thread;
976 struct dma_chan *chan = dtc->chan;
977 char *op;
978 unsigned int i;
979
980 if (type == DMA_MEMCPY)
981 op = "copy";
982 else if (type == DMA_MEMSET)
983 op = "set";
984 else if (type == DMA_XOR)
985 op = "xor";
986 else if (type == DMA_PQ)
987 op = "pq";
988 else
989 return -EINVAL;
990
991 for (i = 0; i < params->threads_per_chan; i++) {
992 thread = kzalloc_obj(struct dmatest_thread);
993 if (!thread) {
994 pr_warn("No memory for %s-%s%u\n",
995 dma_chan_name(chan), op, i);
996 break;
997 }
998 thread->info = info;
999 thread->chan = dtc->chan;
1000 thread->type = type;
1001 thread->test_done.wait = &thread->done_wait;
1002 init_waitqueue_head(&thread->done_wait);
1003 smp_wmb();
1004 thread->task = kthread_create(dmatest_func, thread, "%s-%s%u",
1005 dma_chan_name(chan), op, i);
1006 if (IS_ERR(thread->task)) {
1007 pr_warn("Failed to create thread %s-%s%u\n",
1008 dma_chan_name(chan), op, i);
1009 kfree(thread);
1010 break;
1011 }
1012
1013 /* srcbuf and dstbuf are allocated by the thread itself */
1014 get_task_struct(thread->task);
1015 list_add_tail(&thread->node, &dtc->threads);
1016 thread->pending = true;
1017 }
1018
1019 return i;
1020}
1021
1022static int dmatest_add_channel(struct dmatest_info *info,
1023 struct dma_chan *chan)
1024{
1025 struct dmatest_chan *dtc;
1026 struct dma_device *dma_dev = chan->device;
1027 unsigned int thread_count = 0;
1028 int cnt;
1029
1030 dtc = kmalloc_obj(struct dmatest_chan);
1031 if (!dtc) {
1032 pr_warn("No memory for %s\n", dma_chan_name(chan));
1033 return -ENOMEM;
1034 }
1035
1036 dtc->chan = chan;
1037 INIT_LIST_HEAD(&dtc->threads);
1038
1039 if (dma_has_cap(DMA_COMPLETION_NO_ORDER, dma_dev->cap_mask) &&
1040 info->params.polled) {
1041 info->params.polled = false;
1042 pr_warn("DMA_COMPLETION_NO_ORDER, polled disabled\n");
1043 }
1044
1045 if (dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask)) {
1046 if (dmatest == 0) {
1047 cnt = dmatest_add_threads(info, dtc, DMA_MEMCPY);
1048 thread_count += cnt > 0 ? cnt : 0;
1049 }
1050 }
1051
1052 if (dma_has_cap(DMA_MEMSET, dma_dev->cap_mask)) {
1053 if (dmatest == 1) {
1054 cnt = dmatest_add_threads(info, dtc, DMA_MEMSET);
1055 thread_count += cnt > 0 ? cnt : 0;
1056 }
1057 }
1058
1059 if (dma_has_cap(DMA_XOR, dma_dev->cap_mask)) {
1060 cnt = dmatest_add_threads(info, dtc, DMA_XOR);
1061 thread_count += cnt > 0 ? cnt : 0;
1062 }
1063 if (dma_has_cap(DMA_PQ, dma_dev->cap_mask)) {
1064 cnt = dmatest_add_threads(info, dtc, DMA_PQ);
1065 thread_count += cnt > 0 ? cnt : 0;
1066 }
1067
1068 pr_info("Added %u threads using %s\n",
1069 thread_count, dma_chan_name(chan));
1070
1071 list_add_tail(&dtc->node, &info->channels);
1072 info->nr_channels++;
1073
1074 return 0;
1075}
1076
1077static bool filter(struct dma_chan *chan, void *param)
1078{
1079 return dmatest_match_channel(param, chan) && dmatest_match_device(param, chan->device);
1080}
1081
1082static void request_channels(struct dmatest_info *info,
1083 enum dma_transaction_type type)
1084{
1085 dma_cap_mask_t mask;
1086
1087 dma_cap_zero(mask);
1088 dma_cap_set(type, mask);
1089 for (;;) {
1090 struct dmatest_params *params = &info->params;
1091 struct dma_chan *chan;
1092
1093 chan = dma_request_channel(mask, filter, params);
1094 if (chan) {
1095 if (dmatest_add_channel(info, chan)) {
1096 dma_release_channel(chan);
1097 break; /* add_channel failed, punt */
1098 }
1099 } else
1100 break; /* no more channels available */
1101 if (params->max_channels &&
1102 info->nr_channels >= params->max_channels)
1103 break; /* we have all we need */
1104 }
1105}
1106
1107static void add_threaded_test(struct dmatest_info *info)
1108{
1109 struct dmatest_params *params = &info->params;
1110
1111 /* Copy test parameters */
1112 params->nobounce = nobounce;
1113 params->buf_size = test_buf_size;
1114 strscpy(params->channel, strim(test_channel), sizeof(params->channel));
1115 strscpy(params->device, strim(test_device), sizeof(params->device));
1116 params->threads_per_chan = threads_per_chan;
1117 params->max_channels = max_channels;
1118 params->iterations = iterations;
1119 params->xor_sources = xor_sources;
1120 params->pq_sources = pq_sources;
1121 params->timeout = timeout;
1122 params->noverify = noverify;
1123 params->norandom = norandom;
1124 params->alignment = alignment;
1125 params->transfer_size = transfer_size;
1126 params->polled = polled;
1127
1128 request_channels(info, DMA_MEMCPY);
1129 request_channels(info, DMA_MEMSET);
1130 request_channels(info, DMA_XOR);
1131 request_channels(info, DMA_PQ);
1132}
1133
1134static void run_pending_tests(struct dmatest_info *info)
1135{
1136 struct dmatest_chan *dtc;
1137 unsigned int thread_count = 0;
1138
1139 list_for_each_entry(dtc, &info->channels, node) {
1140 struct dmatest_thread *thread;
1141
1142 thread_count = 0;
1143 list_for_each_entry(thread, &dtc->threads, node) {
1144 wake_up_process(thread->task);
1145 thread_count++;
1146 }
1147 pr_info("Started %u threads using %s\n",
1148 thread_count, dma_chan_name(dtc->chan));
1149 }
1150}
1151
1152static void stop_threaded_test(struct dmatest_info *info)
1153{
1154 struct dmatest_chan *dtc, *_dtc;
1155 struct dma_chan *chan;
1156
1157 list_for_each_entry_safe(dtc, _dtc, &info->channels, node) {
1158 list_del(&dtc->node);
1159 chan = dtc->chan;
1160 dmatest_cleanup_channel(dtc);
1161 pr_debug("dropped channel %s\n", dma_chan_name(chan));
1162 dma_release_channel(chan);
1163 }
1164
1165 info->nr_channels = 0;
1166}
1167
1168static void start_threaded_tests(struct dmatest_info *info)
1169{
1170 /* we might be called early to set run=, defer running until all
1171 * parameters have been evaluated
1172 */
1173 if (!info->did_init)
1174 return;
1175
1176 run_pending_tests(info);
1177}
1178
1179static int dmatest_run_get(char *val, const struct kernel_param *kp)
1180{
1181 struct dmatest_info *info = &test_info;
1182
1183 mutex_lock(&info->lock);
1184 if (is_threaded_test_run(info)) {
1185 dmatest_run = true;
1186 } else {
1187 if (!is_threaded_test_pending(info))
1188 stop_threaded_test(info);
1189 dmatest_run = false;
1190 }
1191 mutex_unlock(&info->lock);
1192
1193 return param_get_bool(val, kp);
1194}
1195
1196static int dmatest_run_set(const char *val, const struct kernel_param *kp)
1197{
1198 struct dmatest_info *info = &test_info;
1199 int ret;
1200
1201 mutex_lock(&info->lock);
1202 ret = param_set_bool(val, kp);
1203 if (ret) {
1204 mutex_unlock(&info->lock);
1205 return ret;
1206 } else if (dmatest_run) {
1207 if (!is_threaded_test_pending(info)) {
1208 /*
1209 * We have nothing to run. This can be due to:
1210 */
1211 ret = info->last_error;
1212 if (ret) {
1213 /* 1) Misconfiguration */
1214 pr_err("Channel misconfigured, can't continue\n");
1215 mutex_unlock(&info->lock);
1216 return ret;
1217 } else {
1218 /* 2) We rely on defaults */
1219 pr_info("No channels configured, continue with any\n");
1220 if (!is_threaded_test_run(info))
1221 stop_threaded_test(info);
1222 add_threaded_test(info);
1223 }
1224 }
1225 start_threaded_tests(info);
1226 } else {
1227 stop_threaded_test(info);
1228 }
1229
1230 mutex_unlock(&info->lock);
1231
1232 return ret;
1233}
1234
1235static int dmatest_chan_set(const char *val, const struct kernel_param *kp)
1236{
1237 struct dmatest_info *info = &test_info;
1238 struct dmatest_chan *dtc;
1239 char chan_reset_val[20];
1240 int ret;
1241
1242 mutex_lock(&info->lock);
1243 ret = param_set_copystring(val, kp);
1244 if (ret) {
1245 mutex_unlock(&info->lock);
1246 return ret;
1247 }
1248 /*Clear any previously run threads */
1249 if (!is_threaded_test_run(info) && !is_threaded_test_pending(info))
1250 stop_threaded_test(info);
1251 /* Reject channels that are already registered */
1252 if (is_threaded_test_pending(info)) {
1253 list_for_each_entry(dtc, &info->channels, node) {
1254 if (strcmp(dma_chan_name(dtc->chan),
1255 strim(test_channel)) == 0) {
1256 dtc = list_last_entry(&info->channels,
1257 struct dmatest_chan,
1258 node);
1259 strscpy(chan_reset_val,
1260 dma_chan_name(dtc->chan),
1261 sizeof(chan_reset_val));
1262 ret = -EBUSY;
1263 goto add_chan_err;
1264 }
1265 }
1266 }
1267
1268 add_threaded_test(info);
1269
1270 /* Check if channel was added successfully */
1271 if (!list_empty(&info->channels)) {
1272 /*
1273 * if new channel was not successfully added, revert the
1274 * "test_channel" string to the name of the last successfully
1275 * added channel. exception for when users issues empty string
1276 * to channel parameter.
1277 */
1278 dtc = list_last_entry(&info->channels, struct dmatest_chan, node);
1279 if ((strcmp(dma_chan_name(dtc->chan), strim(test_channel)) != 0)
1280 && (strcmp("", strim(test_channel)) != 0)) {
1281 ret = -EINVAL;
1282 strscpy(chan_reset_val, dma_chan_name(dtc->chan),
1283 sizeof(chan_reset_val));
1284 goto add_chan_err;
1285 }
1286
1287 } else {
1288 /* Clear test_channel if no channels were added successfully */
1289 strscpy(chan_reset_val, "", sizeof(chan_reset_val));
1290 ret = -EBUSY;
1291 goto add_chan_err;
1292 }
1293
1294 info->last_error = ret;
1295 mutex_unlock(&info->lock);
1296
1297 return ret;
1298
1299add_chan_err:
1300 param_set_copystring(chan_reset_val, kp);
1301 info->last_error = ret;
1302 mutex_unlock(&info->lock);
1303
1304 return ret;
1305}
1306
1307static int dmatest_chan_get(char *val, const struct kernel_param *kp)
1308{
1309 struct dmatest_info *info = &test_info;
1310
1311 mutex_lock(&info->lock);
1312 if (!is_threaded_test_run(info) && !is_threaded_test_pending(info)) {
1313 stop_threaded_test(info);
1314 strscpy(test_channel, "", sizeof(test_channel));
1315 }
1316 mutex_unlock(&info->lock);
1317
1318 return param_get_string(val, kp);
1319}
1320
1321static int dmatest_test_list_get(char *val, const struct kernel_param *kp)
1322{
1323 struct dmatest_info *info = &test_info;
1324 struct dmatest_chan *dtc;
1325 unsigned int thread_count = 0;
1326
1327 list_for_each_entry(dtc, &info->channels, node) {
1328 struct dmatest_thread *thread;
1329
1330 thread_count = 0;
1331 list_for_each_entry(thread, &dtc->threads, node) {
1332 thread_count++;
1333 }
1334 pr_info("%u threads using %s\n",
1335 thread_count, dma_chan_name(dtc->chan));
1336 }
1337
1338 return 0;
1339}
1340
1341static int __init dmatest_init(void)
1342{
1343 struct dmatest_info *info = &test_info;
1344 struct dmatest_params *params = &info->params;
1345
1346 if (dmatest_run) {
1347 mutex_lock(&info->lock);
1348 add_threaded_test(info);
1349 run_pending_tests(info);
1350 mutex_unlock(&info->lock);
1351 }
1352
1353 if (params->iterations && wait)
1354 wait_event(thread_wait, !is_threaded_test_run(info));
1355
1356 /* module parameters are stable, inittime tests are started,
1357 * let userspace take over 'run' control
1358 */
1359 info->did_init = true;
1360
1361 return 0;
1362}
1363/* when compiled-in wait for drivers to load first */
1364late_initcall(dmatest_init);
1365
1366static void __exit dmatest_exit(void)
1367{
1368 struct dmatest_info *info = &test_info;
1369
1370 mutex_lock(&info->lock);
1371 stop_threaded_test(info);
1372 mutex_unlock(&info->lock);
1373}
1374module_exit(dmatest_exit);
1375
1376MODULE_AUTHOR("Haavard Skinnemoen (Atmel)");
1377MODULE_DESCRIPTION("DMA Engine test module");
1378MODULE_LICENSE("GPL v2");