projects/linux·blob master · refs

577 lines · 14.0 KiB

1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Copyright (C) 2020 HiSilicon Limited.
4 */
5
6#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
7
8#include <linux/cleanup.h>
9#include <linux/debugfs.h>
10#include <linux/delay.h>
11#include <linux/device.h>
12#include <linux/dma-mapping.h>
13#include <linux/kernel.h>
14#include <linux/kthread.h>
15#include <linux/math64.h>
16#include <linux/module.h>
17#include <linux/pci.h>
18#include <linux/platform_device.h>
19#include <linux/scatterlist.h>
20#include <linux/slab.h>
21#include <linux/timekeeping.h>
22#include <uapi/linux/map_benchmark.h>
23
24struct map_benchmark_data {
25 struct map_benchmark bparam;
26 struct device *dev;
27 struct dentry *debugfs;
28 enum dma_data_direction dir;
29 atomic64_t sum_map_100ns;
30 atomic64_t sum_unmap_100ns;
31 atomic64_t sum_sq_map;
32 atomic64_t sum_sq_unmap;
33 atomic64_t loops;
34};
35
36struct map_benchmark_ops {
37 void *(*prepare)(struct map_benchmark_data *map);
38 void (*unprepare)(void *mparam);
39 void (*initialize_data)(void *mparam);
40 int (*do_map)(void *mparam);
41 void (*do_unmap)(void *mparam);
42};
43
44struct dma_single_map_param {
45 struct device *dev;
46 dma_addr_t addr;
47 void *xbuf;
48 u32 npages;
49 u32 dma_dir;
50};
51
52static void *dma_single_map_benchmark_prepare(struct map_benchmark_data *map)
53{
54 struct dma_single_map_param *params __free(kfree) = kzalloc_obj(*params);
55 if (!params)
56 return NULL;
57
58 params->npages = map->bparam.granule;
59 params->dma_dir = map->bparam.dma_dir;
60 params->dev = map->dev;
61 params->xbuf = alloc_pages_exact(params->npages * PAGE_SIZE, GFP_KERNEL);
62 if (!params->xbuf)
63 return NULL;
64
65 return_ptr(params);
66}
67
68static void dma_single_map_benchmark_unprepare(void *mparam)
69{
70 struct dma_single_map_param *params = mparam;
71
72 free_pages_exact(params->xbuf, params->npages * PAGE_SIZE);
73 kfree(params);
74}
75
76static void dma_single_map_benchmark_initialize_data(void *mparam)
77{
78 struct dma_single_map_param *params = mparam;
79
80 /*
81 * for a non-coherent device, if we don't stain them in the
82 * cache, this will give an underestimate of the real-world
83 * overhead of BIDIRECTIONAL or TO_DEVICE mappings;
84 * 66 means everything goes well! 66 is lucky.
85 */
86 if (params->dma_dir != DMA_FROM_DEVICE)
87 memset(params->xbuf, 0x66, params->npages * PAGE_SIZE);
88}
89
90static int dma_single_map_benchmark_do_map(void *mparam)
91{
92 struct dma_single_map_param *params = mparam;
93
94 params->addr = dma_map_single(params->dev, params->xbuf,
95 params->npages * PAGE_SIZE, params->dma_dir);
96 if (unlikely(dma_mapping_error(params->dev, params->addr))) {
97 pr_err("dma_map_single failed on %s\n", dev_name(params->dev));
98 return -ENOMEM;
99 }
100
101 return 0;
102}
103
104static void dma_single_map_benchmark_do_unmap(void *mparam)
105{
106 struct dma_single_map_param *params = mparam;
107
108 dma_unmap_single(params->dev, params->addr,
109 params->npages * PAGE_SIZE, params->dma_dir);
110}
111
112static struct map_benchmark_ops dma_single_map_benchmark_ops = {
113 .prepare = dma_single_map_benchmark_prepare,
114 .unprepare = dma_single_map_benchmark_unprepare,
115 .initialize_data = dma_single_map_benchmark_initialize_data,
116 .do_map = dma_single_map_benchmark_do_map,
117 .do_unmap = dma_single_map_benchmark_do_unmap,
118};
119
120struct dma_sg_map_param {
121 struct sg_table sgt;
122 struct device *dev;
123 u32 npages;
124 u32 dma_dir;
125 void *buf[] __counted_by(npages);
126};
127
128static void *dma_sg_map_benchmark_prepare(struct map_benchmark_data *map)
129{
130 struct dma_sg_map_param *params;
131 struct scatterlist *sg;
132 u32 npages;
133 int i;
134
135 /*
136 * Set the number of scatterlist entries based on the granule.
137 * In SG mode, 'granule' represents the number of scatterlist entries.
138 * Each scatterlist entry corresponds to a single page.
139 */
140 npages = map->bparam.granule;
141
142 params = kzalloc_flex(*params, buf, npages);
143 if (!params)
144 return NULL;
145
146 params->npages = npages;
147 params->dma_dir = map->bparam.dma_dir;
148 params->dev = map->dev;
149
150 if (sg_alloc_table(&params->sgt, params->npages, GFP_KERNEL))
151 goto free_params;
152
153 for_each_sgtable_sg(&params->sgt, sg, i) {
154 params->buf[i] = (void *)__get_free_page(GFP_KERNEL);
155 if (!params->buf[i])
156 goto free_page;
157
158 sg_set_buf(sg, params->buf[i], PAGE_SIZE);
159 }
160
161 return params;
162
163free_page:
164 while (i-- > 0)
165 free_page((unsigned long)params->buf[i]);
166
167 sg_free_table(&params->sgt);
168free_params:
169 kfree(params);
170 return NULL;
171}
172
173static void dma_sg_map_benchmark_unprepare(void *mparam)
174{
175 struct dma_sg_map_param *params = mparam;
176 int i;
177
178 for (i = 0; i < params->npages; i++)
179 free_page((unsigned long)params->buf[i]);
180
181 sg_free_table(&params->sgt);
182
183 kfree(params);
184}
185
186static void dma_sg_map_benchmark_initialize_data(void *mparam)
187{
188 struct dma_sg_map_param *params = mparam;
189 struct scatterlist *sg;
190 int i = 0;
191
192 if (params->dma_dir == DMA_FROM_DEVICE)
193 return;
194
195 for_each_sgtable_sg(&params->sgt, sg, i)
196 memset(params->buf[i], 0x66, PAGE_SIZE);
197}
198
199static int dma_sg_map_benchmark_do_map(void *mparam)
200{
201 struct dma_sg_map_param *params = mparam;
202 int ret = 0;
203
204 int sg_mapped = dma_map_sg(params->dev, params->sgt.sgl,
205 params->npages, params->dma_dir);
206 if (!sg_mapped) {
207 pr_err("dma_map_sg failed on %s\n", dev_name(params->dev));
208 ret = -ENOMEM;
209 }
210
211 return ret;
212}
213
214static void dma_sg_map_benchmark_do_unmap(void *mparam)
215{
216 struct dma_sg_map_param *params = mparam;
217
218 dma_unmap_sg(params->dev, params->sgt.sgl, params->npages,
219 params->dma_dir);
220}
221
222static struct map_benchmark_ops dma_sg_map_benchmark_ops = {
223 .prepare = dma_sg_map_benchmark_prepare,
224 .unprepare = dma_sg_map_benchmark_unprepare,
225 .initialize_data = dma_sg_map_benchmark_initialize_data,
226 .do_map = dma_sg_map_benchmark_do_map,
227 .do_unmap = dma_sg_map_benchmark_do_unmap,
228};
229
230static struct map_benchmark_ops *dma_map_benchmark_ops[DMA_MAP_BENCH_MODE_MAX] = {
231 [DMA_MAP_BENCH_SINGLE_MODE] = &dma_single_map_benchmark_ops,
232 [DMA_MAP_BENCH_SG_MODE] = &dma_sg_map_benchmark_ops,
233};
234
235static int map_benchmark_thread(void *data)
236{
237 struct map_benchmark_data *map = data;
238 __u8 map_mode = map->bparam.map_mode;
239 int ret = 0;
240
241 struct map_benchmark_ops *mb_ops = dma_map_benchmark_ops[map_mode];
242 void *mparam = mb_ops->prepare(map);
243
244 if (!mparam)
245 return -ENOMEM;
246
247 while (!kthread_should_stop()) {
248 u64 map_100ns, unmap_100ns, map_sq, unmap_sq;
249 ktime_t map_stime, map_etime, unmap_stime, unmap_etime;
250 ktime_t map_delta, unmap_delta;
251
252 mb_ops->initialize_data(mparam);
253 map_stime = ktime_get();
254 ret = mb_ops->do_map(mparam);
255 if (ret)
256 goto out;
257
258 map_etime = ktime_get();
259 map_delta = ktime_sub(map_etime, map_stime);
260
261 /* Pretend DMA is transmitting */
262 ndelay(map->bparam.dma_trans_ns);
263
264 unmap_stime = ktime_get();
265 mb_ops->do_unmap(mparam);
266
267 unmap_etime = ktime_get();
268 unmap_delta = ktime_sub(unmap_etime, unmap_stime);
269
270 /* calculate sum and sum of squares */
271
272 map_100ns = div64_ul(map_delta, 100);
273 unmap_100ns = div64_ul(unmap_delta, 100);
274 map_sq = map_100ns * map_100ns;
275 unmap_sq = unmap_100ns * unmap_100ns;
276
277 atomic64_add(map_100ns, &map->sum_map_100ns);
278 atomic64_add(unmap_100ns, &map->sum_unmap_100ns);
279 atomic64_add(map_sq, &map->sum_sq_map);
280 atomic64_add(unmap_sq, &map->sum_sq_unmap);
281 atomic64_inc(&map->loops);
282
283 /*
284 * We may test for a long time so periodically check whether
285 * we need to schedule to avoid starving the others. Otherwise
286 * we may hangup the kernel in a non-preemptible kernel when
287 * the test kthreads number >= CPU number, the test kthreads
288 * will run endless on every CPU since the thread resposible
289 * for notifying the kthread stop (in do_map_benchmark())
290 * could not be scheduled.
291 *
292 * Note this may degrade the test concurrency since the test
293 * threads may need to share the CPU time with other load
294 * in the system. So it's recommended to run this benchmark
295 * on an idle system.
296 */
297 cond_resched();
298 }
299
300out:
301 mb_ops->unprepare(mparam);
302 return ret;
303}
304
305static int do_map_benchmark(struct map_benchmark_data *map)
306{
307 struct task_struct **tsk;
308 int threads = map->bparam.threads;
309 int node = map->bparam.node;
310 u64 loops;
311 int ret = 0;
312 int i;
313
314 tsk = kmalloc_objs(*tsk, threads);
315 if (!tsk)
316 return -ENOMEM;
317
318 get_device(map->dev);
319
320 for (i = 0; i < threads; i++) {
321 tsk[i] = kthread_create_on_node(map_benchmark_thread, map,
322 map->bparam.node, "dma-map-benchmark/%d", i);
323 if (IS_ERR(tsk[i])) {
324 pr_err("create dma_map thread failed\n");
325 ret = PTR_ERR(tsk[i]);
326 while (--i >= 0)
327 kthread_stop(tsk[i]);
328 goto out;
329 }
330
331 if (node != NUMA_NO_NODE)
332 kthread_bind_mask(tsk[i], cpumask_of_node(node));
333 }
334
335 /* clear the old value in the previous benchmark */
336 atomic64_set(&map->sum_map_100ns, 0);
337 atomic64_set(&map->sum_unmap_100ns, 0);
338 atomic64_set(&map->sum_sq_map, 0);
339 atomic64_set(&map->sum_sq_unmap, 0);
340 atomic64_set(&map->loops, 0);
341
342 for (i = 0; i < threads; i++) {
343 get_task_struct(tsk[i]);
344 wake_up_process(tsk[i]);
345 }
346
347 msleep_interruptible(map->bparam.seconds * 1000);
348
349 /* wait for the completion of all started benchmark threads */
350 for (i = 0; i < threads; i++) {
351 int kthread_ret = kthread_stop_put(tsk[i]);
352
353 if (kthread_ret)
354 ret = kthread_ret;
355 }
356
357 if (ret)
358 goto out;
359
360 loops = atomic64_read(&map->loops);
361 if (likely(loops > 0)) {
362 u64 map_variance, unmap_variance;
363 u64 sum_map = atomic64_read(&map->sum_map_100ns);
364 u64 sum_unmap = atomic64_read(&map->sum_unmap_100ns);
365 u64 sum_sq_map = atomic64_read(&map->sum_sq_map);
366 u64 sum_sq_unmap = atomic64_read(&map->sum_sq_unmap);
367
368 /* average latency */
369 map->bparam.avg_map_100ns = div64_u64(sum_map, loops);
370 map->bparam.avg_unmap_100ns = div64_u64(sum_unmap, loops);
371
372 /* standard deviation of latency */
373 map_variance = div64_u64(sum_sq_map, loops) -
374 map->bparam.avg_map_100ns *
375 map->bparam.avg_map_100ns;
376 unmap_variance = div64_u64(sum_sq_unmap, loops) -
377 map->bparam.avg_unmap_100ns *
378 map->bparam.avg_unmap_100ns;
379 map->bparam.map_stddev = int_sqrt64(map_variance);
380 map->bparam.unmap_stddev = int_sqrt64(unmap_variance);
381 }
382
383out:
384 put_device(map->dev);
385 kfree(tsk);
386 return ret;
387}
388
389static long map_benchmark_ioctl(struct file *file, unsigned int cmd,
390 unsigned long arg)
391{
392 struct map_benchmark_data *map = file->private_data;
393 void __user *argp = (void __user *)arg;
394 u64 old_dma_mask;
395 int ret;
396
397 if (copy_from_user(&map->bparam, argp, sizeof(map->bparam)))
398 return -EFAULT;
399
400 switch (cmd) {
401 case DMA_MAP_BENCHMARK:
402 if (map->bparam.map_mode < 0 ||
403 map->bparam.map_mode >= DMA_MAP_BENCH_MODE_MAX) {
404 pr_err("invalid map mode\n");
405 return -EINVAL;
406 }
407
408 if (map->bparam.threads == 0 ||
409 map->bparam.threads > DMA_MAP_MAX_THREADS) {
410 pr_err("invalid thread number\n");
411 return -EINVAL;
412 }
413
414 if (map->bparam.seconds == 0 ||
415 map->bparam.seconds > DMA_MAP_MAX_SECONDS) {
416 pr_err("invalid duration seconds\n");
417 return -EINVAL;
418 }
419
420 if (map->bparam.dma_trans_ns > DMA_MAP_MAX_TRANS_DELAY) {
421 pr_err("invalid transmission delay\n");
422 return -EINVAL;
423 }
424
425 if (map->bparam.node != NUMA_NO_NODE &&
426 (map->bparam.node < 0 || map->bparam.node >= MAX_NUMNODES ||
427 !node_possible(map->bparam.node))) {
428 pr_err("invalid numa node\n");
429 return -EINVAL;
430 }
431
432 if (map->bparam.granule < 1 || map->bparam.granule > 1024) {
433 pr_err("invalid granule size\n");
434 return -EINVAL;
435 }
436
437 switch (map->bparam.dma_dir) {
438 case DMA_MAP_BIDIRECTIONAL:
439 map->dir = DMA_BIDIRECTIONAL;
440 break;
441 case DMA_MAP_FROM_DEVICE:
442 map->dir = DMA_FROM_DEVICE;
443 break;
444 case DMA_MAP_TO_DEVICE:
445 map->dir = DMA_TO_DEVICE;
446 break;
447 default:
448 pr_err("invalid DMA direction\n");
449 return -EINVAL;
450 }
451
452 old_dma_mask = dma_get_mask(map->dev);
453
454 ret = dma_set_mask(map->dev,
455 DMA_BIT_MASK(map->bparam.dma_bits));
456 if (ret) {
457 pr_err("failed to set dma_mask on device %s\n",
458 dev_name(map->dev));
459 return -EINVAL;
460 }
461
462 ret = do_map_benchmark(map);
463
464 /*
465 * restore the original dma_mask as many devices' dma_mask are
466 * set by architectures, acpi, busses. When we bind them back
467 * to their original drivers, those drivers shouldn't see
468 * dma_mask changed by benchmark
469 */
470 dma_set_mask(map->dev, old_dma_mask);
471
472 if (ret)
473 return ret;
474 break;
475 default:
476 return -EINVAL;
477 }
478
479 if (copy_to_user(argp, &map->bparam, sizeof(map->bparam)))
480 return -EFAULT;
481
482 return ret;
483}
484
485static const struct file_operations map_benchmark_fops = {
486 .open = simple_open,
487 .unlocked_ioctl = map_benchmark_ioctl,
488};
489
490static void map_benchmark_remove_debugfs(void *data)
491{
492 struct map_benchmark_data *map = (struct map_benchmark_data *)data;
493
494 debugfs_remove(map->debugfs);
495}
496
497static int __map_benchmark_probe(struct device *dev)
498{
499 struct dentry *entry;
500 struct map_benchmark_data *map;
501 int ret;
502
503 map = devm_kzalloc(dev, sizeof(*map), GFP_KERNEL);
504 if (!map)
505 return -ENOMEM;
506 map->dev = dev;
507
508 ret = devm_add_action(dev, map_benchmark_remove_debugfs, map);
509 if (ret) {
510 pr_err("Can't add debugfs remove action\n");
511 return ret;
512 }
513
514 /*
515 * we only permit a device bound with this driver, 2nd probe
516 * will fail
517 */
518 entry = debugfs_create_file("dma_map_benchmark", 0600, NULL, map,
519 &map_benchmark_fops);
520 if (IS_ERR(entry))
521 return PTR_ERR(entry);
522 map->debugfs = entry;
523
524 return 0;
525}
526
527static int map_benchmark_platform_probe(struct platform_device *pdev)
528{
529 return __map_benchmark_probe(&pdev->dev);
530}
531
532static struct platform_driver map_benchmark_platform_driver = {
533 .driver = {
534 .name = "dma_map_benchmark",
535 },
536 .probe = map_benchmark_platform_probe,
537};
538
539static int
540map_benchmark_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
541{
542 return __map_benchmark_probe(&pdev->dev);
543}
544
545static struct pci_driver map_benchmark_pci_driver = {
546 .name = "dma_map_benchmark",
547 .probe = map_benchmark_pci_probe,
548};
549
550static int __init map_benchmark_init(void)
551{
552 int ret;
553
554 ret = pci_register_driver(&map_benchmark_pci_driver);
555 if (ret)
556 return ret;
557
558 ret = platform_driver_register(&map_benchmark_platform_driver);
559 if (ret) {
560 pci_unregister_driver(&map_benchmark_pci_driver);
561 return ret;
562 }
563
564 return 0;
565}
566
567static void __exit map_benchmark_cleanup(void)
568{
569 platform_driver_unregister(&map_benchmark_platform_driver);
570 pci_unregister_driver(&map_benchmark_pci_driver);
571}
572
573module_init(map_benchmark_init);
574module_exit(map_benchmark_cleanup);
575
576MODULE_AUTHOR("Barry Song <[email protected]>");
577MODULE_DESCRIPTION("dma_map benchmark driver");