projects/linux·blob master · refs

1656 lines · 40.8 KiB

1// SPDX-License-Identifier: GPL-2.0-or-later
2/*
3 * Copyright(c) 2004 - 2006 Intel Corporation. All rights reserved.
4 */
5
6/*
7 * This code implements the DMA subsystem. It provides a HW-neutral interface
8 * for other kernel code to use asynchronous memory copy capabilities,
9 * if present, and allows different HW DMA drivers to register as providing
10 * this capability.
11 *
12 * Due to the fact we are accelerating what is already a relatively fast
13 * operation, the code goes to great lengths to avoid additional overhead,
14 * such as locking.
15 *
16 * LOCKING:
17 *
18 * The subsystem keeps a global list of dma_device structs it is protected by a
19 * mutex, dma_list_mutex.
20 *
21 * A subsystem can get access to a channel by calling dmaengine_get() followed
22 * by dma_find_channel(), or if it has need for an exclusive channel it can call
23 * dma_request_channel(). Once a channel is allocated a reference is taken
24 * against its corresponding driver to disable removal.
25 *
26 * Each device has a channels list, which runs unlocked but is never modified
27 * once the device is registered, it's just setup by the driver.
28 *
29 * See Documentation/driver-api/dmaengine for more details
30 */
31
32#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
33
34#include <linux/acpi.h>
35#include <linux/acpi_dma.h>
36#include <linux/device.h>
37#include <linux/dma-mapping.h>
38#include <linux/dmaengine.h>
39#include <linux/hardirq.h>
40#include <linux/idr.h>
41#include <linux/init.h>
42#include <linux/jiffies.h>
43#include <linux/mempool.h>
44#include <linux/mm.h>
45#include <linux/module.h>
46#include <linux/mutex.h>
47#include <linux/numa.h>
48#include <linux/of.h>
49#include <linux/of_dma.h>
50#include <linux/percpu.h>
51#include <linux/platform_device.h>
52#include <linux/property.h>
53#include <linux/rculist.h>
54#include <linux/rcupdate.h>
55#include <linux/slab.h>
56#include <linux/spinlock.h>
57
58#include "dmaengine.h"
59
60static DEFINE_MUTEX(dma_list_mutex);
61static DEFINE_IDA(dma_ida);
62static LIST_HEAD(dma_device_list);
63static long dmaengine_ref_count;
64
65/* --- debugfs implementation --- */
66#ifdef CONFIG_DEBUG_FS
67#include <linux/debugfs.h>
68
69static struct dentry *rootdir;
70
71static void dmaengine_debug_register(struct dma_device *dma_dev)
72{
73 dma_dev->dbg_dev_root = debugfs_create_dir(dev_name(dma_dev->dev),
74 rootdir);
75 if (IS_ERR(dma_dev->dbg_dev_root))
76 dma_dev->dbg_dev_root = NULL;
77}
78
79static void dmaengine_debug_unregister(struct dma_device *dma_dev)
80{
81 debugfs_remove_recursive(dma_dev->dbg_dev_root);
82 dma_dev->dbg_dev_root = NULL;
83}
84
85static void dmaengine_dbg_summary_show(struct seq_file *s,
86 struct dma_device *dma_dev)
87{
88 struct dma_chan *chan;
89
90 list_for_each_entry(chan, &dma_dev->channels, device_node) {
91 if (chan->client_count) {
92 seq_printf(s, " %-13s| %s", dma_chan_name(chan),
93 chan->dbg_client_name ?: "in-use");
94
95 if (chan->router)
96 seq_printf(s, " (via router: %s)\n",
97 dev_name(chan->router->dev));
98 else
99 seq_puts(s, "\n");
100 }
101 }
102}
103
104static int dmaengine_summary_show(struct seq_file *s, void *data)
105{
106 struct dma_device *dma_dev = NULL;
107
108 mutex_lock(&dma_list_mutex);
109 list_for_each_entry(dma_dev, &dma_device_list, global_node) {
110 seq_printf(s, "dma%d (%s): number of channels: %u\n",
111 dma_dev->dev_id, dev_name(dma_dev->dev),
112 dma_dev->chancnt);
113
114 if (dma_dev->dbg_summary_show)
115 dma_dev->dbg_summary_show(s, dma_dev);
116 else
117 dmaengine_dbg_summary_show(s, dma_dev);
118
119 if (!list_is_last(&dma_dev->global_node, &dma_device_list))
120 seq_puts(s, "\n");
121 }
122 mutex_unlock(&dma_list_mutex);
123
124 return 0;
125}
126DEFINE_SHOW_ATTRIBUTE(dmaengine_summary);
127
128static void __init dmaengine_debugfs_init(void)
129{
130 rootdir = debugfs_create_dir("dmaengine", NULL);
131
132 /* /sys/kernel/debug/dmaengine/summary */
133 debugfs_create_file("summary", 0444, rootdir, NULL,
134 &dmaengine_summary_fops);
135}
136#else
137static inline void dmaengine_debugfs_init(void) { }
138static inline int dmaengine_debug_register(struct dma_device *dma_dev)
139{
140 return 0;
141}
142
143static inline void dmaengine_debug_unregister(struct dma_device *dma_dev) { }
144#endif /* DEBUG_FS */
145
146/* --- sysfs implementation --- */
147
148#define DMA_SLAVE_NAME "slave"
149
150/**
151 * dev_to_dma_chan - convert a device pointer to its sysfs container object
152 * @dev: device node
153 *
154 * Must be called under dma_list_mutex.
155 */
156static struct dma_chan *dev_to_dma_chan(struct device *dev)
157{
158 struct dma_chan_dev *chan_dev;
159
160 chan_dev = container_of(dev, typeof(*chan_dev), device);
161 return chan_dev->chan;
162}
163
164static ssize_t memcpy_count_show(struct device *dev,
165 struct device_attribute *attr, char *buf)
166{
167 struct dma_chan *chan;
168 unsigned long count = 0;
169 int i;
170 int err;
171
172 mutex_lock(&dma_list_mutex);
173 chan = dev_to_dma_chan(dev);
174 if (chan) {
175 for_each_possible_cpu(i)
176 count += per_cpu_ptr(chan->local, i)->memcpy_count;
177 err = sysfs_emit(buf, "%lu\n", count);
178 } else
179 err = -ENODEV;
180 mutex_unlock(&dma_list_mutex);
181
182 return err;
183}
184static DEVICE_ATTR_RO(memcpy_count);
185
186static ssize_t bytes_transferred_show(struct device *dev,
187 struct device_attribute *attr, char *buf)
188{
189 struct dma_chan *chan;
190 unsigned long count = 0;
191 int i;
192 int err;
193
194 mutex_lock(&dma_list_mutex);
195 chan = dev_to_dma_chan(dev);
196 if (chan) {
197 for_each_possible_cpu(i)
198 count += per_cpu_ptr(chan->local, i)->bytes_transferred;
199 err = sysfs_emit(buf, "%lu\n", count);
200 } else
201 err = -ENODEV;
202 mutex_unlock(&dma_list_mutex);
203
204 return err;
205}
206static DEVICE_ATTR_RO(bytes_transferred);
207
208static ssize_t in_use_show(struct device *dev, struct device_attribute *attr,
209 char *buf)
210{
211 struct dma_chan *chan;
212 int err;
213
214 mutex_lock(&dma_list_mutex);
215 chan = dev_to_dma_chan(dev);
216 if (chan)
217 err = sysfs_emit(buf, "%d\n", chan->client_count);
218 else
219 err = -ENODEV;
220 mutex_unlock(&dma_list_mutex);
221
222 return err;
223}
224static DEVICE_ATTR_RO(in_use);
225
226static struct attribute *dma_dev_attrs[] = {
227 &dev_attr_memcpy_count.attr,
228 &dev_attr_bytes_transferred.attr,
229 &dev_attr_in_use.attr,
230 NULL,
231};
232ATTRIBUTE_GROUPS(dma_dev);
233
234static void chan_dev_release(struct device *dev)
235{
236 struct dma_chan_dev *chan_dev;
237
238 chan_dev = container_of(dev, typeof(*chan_dev), device);
239 kfree(chan_dev);
240}
241
242static struct class dma_devclass = {
243 .name = "dma",
244 .dev_groups = dma_dev_groups,
245 .dev_release = chan_dev_release,
246};
247
248/* --- client and device registration --- */
249
250/* enable iteration over all operation types */
251static dma_cap_mask_t dma_cap_mask_all;
252
253/**
254 * struct dma_chan_tbl_ent - tracks channel allocations per core/operation
255 * @chan: associated channel for this entry
256 */
257struct dma_chan_tbl_ent {
258 struct dma_chan *chan;
259};
260
261/* percpu lookup table for memory-to-memory offload providers */
262static struct dma_chan_tbl_ent __percpu *channel_table[DMA_TX_TYPE_END];
263
264static int __init dma_channel_table_init(void)
265{
266 enum dma_transaction_type cap;
267 int err = 0;
268
269 bitmap_fill(dma_cap_mask_all.bits, DMA_TX_TYPE_END);
270
271 /* 'interrupt', 'private', and 'slave' are channel capabilities,
272 * but are not associated with an operation so they do not need
273 * an entry in the channel_table
274 */
275 clear_bit(DMA_INTERRUPT, dma_cap_mask_all.bits);
276 clear_bit(DMA_PRIVATE, dma_cap_mask_all.bits);
277 clear_bit(DMA_SLAVE, dma_cap_mask_all.bits);
278
279 for_each_dma_cap_mask(cap, dma_cap_mask_all) {
280 channel_table[cap] = alloc_percpu(struct dma_chan_tbl_ent);
281 if (!channel_table[cap]) {
282 err = -ENOMEM;
283 break;
284 }
285 }
286
287 if (err) {
288 pr_err("dmaengine dma_channel_table_init failure: %d\n", err);
289 for_each_dma_cap_mask(cap, dma_cap_mask_all)
290 free_percpu(channel_table[cap]);
291 }
292
293 return err;
294}
295arch_initcall(dma_channel_table_init);
296
297/**
298 * dma_chan_is_local - checks if the channel is in the same NUMA-node as the CPU
299 * @chan: DMA channel to test
300 * @cpu: CPU index which the channel should be close to
301 *
302 * Returns true if the channel is in the same NUMA-node as the CPU.
303 */
304static bool dma_chan_is_local(struct dma_chan *chan, int cpu)
305{
306 int node = dev_to_node(chan->device->dev);
307 return node == NUMA_NO_NODE ||
308 cpumask_test_cpu(cpu, cpumask_of_node(node));
309}
310
311/**
312 * min_chan - finds the channel with min count and in the same NUMA-node as the CPU
313 * @cap: capability to match
314 * @cpu: CPU index which the channel should be close to
315 *
316 * If some channels are close to the given CPU, the one with the lowest
317 * reference count is returned. Otherwise, CPU is ignored and only the
318 * reference count is taken into account.
319 *
320 * Must be called under dma_list_mutex.
321 */
322static struct dma_chan *min_chan(enum dma_transaction_type cap, int cpu)
323{
324 struct dma_device *device;
325 struct dma_chan *chan;
326 struct dma_chan *min = NULL;
327 struct dma_chan *localmin = NULL;
328
329 list_for_each_entry(device, &dma_device_list, global_node) {
330 if (!dma_has_cap(cap, device->cap_mask) ||
331 dma_has_cap(DMA_PRIVATE, device->cap_mask))
332 continue;
333 list_for_each_entry(chan, &device->channels, device_node) {
334 if (!chan->client_count)
335 continue;
336 if (!min || chan->table_count < min->table_count)
337 min = chan;
338
339 if (dma_chan_is_local(chan, cpu))
340 if (!localmin ||
341 chan->table_count < localmin->table_count)
342 localmin = chan;
343 }
344 }
345
346 chan = localmin ? localmin : min;
347
348 if (chan)
349 chan->table_count++;
350
351 return chan;
352}
353
354/**
355 * dma_channel_rebalance - redistribute the available channels
356 *
357 * Optimize for CPU isolation (each CPU gets a dedicated channel for an
358 * operation type) in the SMP case, and operation isolation (avoid
359 * multi-tasking channels) in the non-SMP case.
360 *
361 * Must be called under dma_list_mutex.
362 */
363static void dma_channel_rebalance(void)
364{
365 struct dma_chan *chan;
366 struct dma_device *device;
367 int cpu;
368 int cap;
369
370 /* undo the last distribution */
371 for_each_dma_cap_mask(cap, dma_cap_mask_all)
372 for_each_possible_cpu(cpu)
373 per_cpu_ptr(channel_table[cap], cpu)->chan = NULL;
374
375 list_for_each_entry(device, &dma_device_list, global_node) {
376 if (dma_has_cap(DMA_PRIVATE, device->cap_mask))
377 continue;
378 list_for_each_entry(chan, &device->channels, device_node)
379 chan->table_count = 0;
380 }
381
382 /* don't populate the channel_table if no clients are available */
383 if (!dmaengine_ref_count)
384 return;
385
386 /* redistribute available channels */
387 for_each_dma_cap_mask(cap, dma_cap_mask_all)
388 for_each_online_cpu(cpu) {
389 chan = min_chan(cap, cpu);
390 per_cpu_ptr(channel_table[cap], cpu)->chan = chan;
391 }
392}
393
394static int dma_device_satisfies_mask(struct dma_device *device,
395 const dma_cap_mask_t *want)
396{
397 dma_cap_mask_t has;
398
399 bitmap_and(has.bits, want->bits, device->cap_mask.bits,
400 DMA_TX_TYPE_END);
401 return bitmap_equal(want->bits, has.bits, DMA_TX_TYPE_END);
402}
403
404static struct module *dma_chan_to_owner(struct dma_chan *chan)
405{
406 return chan->device->owner;
407}
408
409/**
410 * balance_ref_count - catch up the channel reference count
411 * @chan: channel to balance ->client_count versus dmaengine_ref_count
412 *
413 * Must be called under dma_list_mutex.
414 */
415static void balance_ref_count(struct dma_chan *chan)
416{
417 struct module *owner = dma_chan_to_owner(chan);
418
419 while (chan->client_count < dmaengine_ref_count) {
420 __module_get(owner);
421 chan->client_count++;
422 }
423}
424
425static void dma_device_release(struct kref *ref)
426{
427 struct dma_device *device = container_of(ref, struct dma_device, ref);
428
429 list_del_rcu(&device->global_node);
430 dma_channel_rebalance();
431 synchronize_rcu();
432
433 if (device->device_release)
434 device->device_release(device);
435}
436
437static int __must_check dma_device_get(struct dma_device *device)
438{
439 lockdep_assert_held(&dma_list_mutex);
440 return kref_get_unless_zero(&device->ref);
441}
442
443static void dma_device_put(struct dma_device *device)
444{
445 lockdep_assert_held(&dma_list_mutex);
446 kref_put(&device->ref, dma_device_release);
447}
448
449/**
450 * dma_chan_get - try to grab a DMA channel's parent driver module
451 * @chan: channel to grab
452 *
453 * Must be called under dma_list_mutex.
454 */
455static int dma_chan_get(struct dma_chan *chan)
456{
457 struct module *owner = dma_chan_to_owner(chan);
458 int ret;
459
460 /* The channel is already in use, update client count */
461 if (chan->client_count) {
462 __module_get(owner);
463 chan->client_count++;
464 return 0;
465 }
466
467 if (!try_module_get(owner))
468 return -ENODEV;
469
470 if (!dma_device_get(chan->device)) {
471 ret = -ENODEV;
472 goto module_put_out;
473 }
474
475 /* allocate upon first client reference */
476 if (chan->device->device_alloc_chan_resources) {
477 ret = chan->device->device_alloc_chan_resources(chan);
478 if (ret < 0)
479 goto err_out;
480 }
481
482 chan->client_count++;
483
484 if (!dma_has_cap(DMA_PRIVATE, chan->device->cap_mask))
485 balance_ref_count(chan);
486
487 return 0;
488
489err_out:
490 dma_device_put(chan->device);
491module_put_out:
492 module_put(owner);
493 return ret;
494}
495
496/**
497 * dma_chan_put - drop a reference to a DMA channel's parent driver module
498 * @chan: channel to release
499 *
500 * Must be called under dma_list_mutex.
501 */
502static void dma_chan_put(struct dma_chan *chan)
503{
504 struct module *owner;
505
506 /* This channel is not in use, bail out */
507 if (!chan->client_count)
508 return;
509
510 owner = dma_chan_to_owner(chan);
511 chan->client_count--;
512
513 /* This channel is not in use anymore, free it */
514 if (!chan->client_count && chan->device->device_free_chan_resources) {
515 /* Make sure all operations have completed */
516 dmaengine_synchronize(chan);
517 chan->device->device_free_chan_resources(chan);
518 }
519
520 /* If the channel is used via a DMA request router, free the mapping */
521 if (chan->router && chan->router->route_free) {
522 chan->router->route_free(chan->router->dev, chan->route_data);
523 chan->router = NULL;
524 chan->route_data = NULL;
525 }
526
527 /* This channel is not in use anymore, drop the device ref */
528 if (!chan->client_count)
529 dma_device_put(chan->device);
530 module_put(owner);
531}
532
533enum dma_status dma_sync_wait(struct dma_chan *chan, dma_cookie_t cookie)
534{
535 enum dma_status status;
536 unsigned long dma_sync_wait_timeout = jiffies + msecs_to_jiffies(5000);
537
538 dma_async_issue_pending(chan);
539 do {
540 status = dma_async_is_tx_complete(chan, cookie, NULL, NULL);
541 if (time_after_eq(jiffies, dma_sync_wait_timeout)) {
542 dev_err(chan->device->dev, "%s: timeout!\n", __func__);
543 return DMA_ERROR;
544 }
545 if (status != DMA_IN_PROGRESS)
546 break;
547 cpu_relax();
548 } while (1);
549
550 return status;
551}
552EXPORT_SYMBOL(dma_sync_wait);
553
554/**
555 * dma_find_channel - find a channel to carry out the operation
556 * @tx_type: transaction type
557 */
558struct dma_chan *dma_find_channel(enum dma_transaction_type tx_type)
559{
560 return this_cpu_read(channel_table[tx_type]->chan);
561}
562EXPORT_SYMBOL(dma_find_channel);
563
564/**
565 * dma_issue_pending_all - flush all pending operations across all channels
566 */
567void dma_issue_pending_all(void)
568{
569 struct dma_device *device;
570 struct dma_chan *chan;
571
572 rcu_read_lock();
573 list_for_each_entry_rcu(device, &dma_device_list, global_node) {
574 if (dma_has_cap(DMA_PRIVATE, device->cap_mask))
575 continue;
576 list_for_each_entry(chan, &device->channels, device_node)
577 if (chan->client_count)
578 device->device_issue_pending(chan);
579 }
580 rcu_read_unlock();
581}
582EXPORT_SYMBOL(dma_issue_pending_all);
583
584int dma_get_slave_caps(struct dma_chan *chan, struct dma_slave_caps *caps)
585{
586 struct dma_device *device;
587
588 if (!chan || !caps)
589 return -EINVAL;
590
591 device = chan->device;
592
593 /* check if the channel supports slave transactions */
594 if (!(test_bit(DMA_SLAVE, device->cap_mask.bits) ||
595 test_bit(DMA_CYCLIC, device->cap_mask.bits)))
596 return -ENXIO;
597
598 /*
599 * Check whether it reports it uses the generic slave
600 * capabilities, if not, that means it doesn't support any
601 * kind of slave capabilities reporting.
602 */
603 if (!device->directions)
604 return -ENXIO;
605
606 caps->src_addr_widths = device->src_addr_widths;
607 caps->dst_addr_widths = device->dst_addr_widths;
608 caps->directions = device->directions;
609 caps->min_burst = device->min_burst;
610 caps->max_burst = device->max_burst;
611 caps->max_sg_burst = device->max_sg_burst;
612 caps->residue_granularity = device->residue_granularity;
613 caps->descriptor_reuse = device->descriptor_reuse;
614 caps->cmd_pause = !!device->device_pause;
615 caps->cmd_resume = !!device->device_resume;
616 caps->cmd_terminate = !!device->device_terminate_all;
617
618 /*
619 * DMA engine device might be configured with non-uniformly
620 * distributed slave capabilities per device channels. In this
621 * case the corresponding driver may provide the device_caps
622 * callback to override the generic capabilities with
623 * channel-specific ones.
624 */
625 if (device->device_caps)
626 device->device_caps(chan, caps);
627
628 return 0;
629}
630EXPORT_SYMBOL_GPL(dma_get_slave_caps);
631
632static struct dma_chan *private_candidate(const dma_cap_mask_t *mask,
633 struct dma_device *dev,
634 dma_filter_fn fn, void *fn_param)
635{
636 struct dma_chan *chan;
637
638 if (mask && !dma_device_satisfies_mask(dev, mask)) {
639 dev_dbg(dev->dev, "%s: wrong capabilities\n", __func__);
640 return NULL;
641 }
642 /* devices with multiple channels need special handling as we need to
643 * ensure that all channels are either private or public.
644 */
645 if (dev->chancnt > 1 && !dma_has_cap(DMA_PRIVATE, dev->cap_mask))
646 list_for_each_entry(chan, &dev->channels, device_node) {
647 /* some channels are already publicly allocated */
648 if (chan->client_count)
649 return NULL;
650 }
651
652 list_for_each_entry(chan, &dev->channels, device_node) {
653 if (chan->client_count) {
654 dev_dbg(dev->dev, "%s: %s busy\n",
655 __func__, dma_chan_name(chan));
656 continue;
657 }
658 if (fn && !fn(chan, fn_param)) {
659 dev_dbg(dev->dev, "%s: %s filter said false\n",
660 __func__, dma_chan_name(chan));
661 continue;
662 }
663 return chan;
664 }
665
666 return NULL;
667}
668
669static struct dma_chan *find_candidate(struct dma_device *device,
670 const dma_cap_mask_t *mask,
671 dma_filter_fn fn, void *fn_param)
672{
673 struct dma_chan *chan = private_candidate(mask, device, fn, fn_param);
674 int err;
675
676 if (chan) {
677 /* Found a suitable channel, try to grab, prep, and return it.
678 * We first set DMA_PRIVATE to disable balance_ref_count as this
679 * channel will not be published in the general-purpose
680 * allocator
681 */
682 dma_cap_set(DMA_PRIVATE, device->cap_mask);
683 device->privatecnt++;
684 err = dma_chan_get(chan);
685
686 if (err) {
687 if (err == -ENODEV) {
688 dev_dbg(device->dev, "%s: %s module removed\n",
689 __func__, dma_chan_name(chan));
690 list_del_rcu(&device->global_node);
691 } else
692 dev_dbg(device->dev,
693 "%s: failed to get %s: (%d)\n",
694 __func__, dma_chan_name(chan), err);
695
696 if (--device->privatecnt == 0)
697 dma_cap_clear(DMA_PRIVATE, device->cap_mask);
698
699 chan = ERR_PTR(err);
700 }
701 }
702
703 return chan ? chan : ERR_PTR(-EPROBE_DEFER);
704}
705
706/**
707 * dma_get_slave_channel - try to get specific channel exclusively
708 * @chan: target channel
709 */
710struct dma_chan *dma_get_slave_channel(struct dma_chan *chan)
711{
712 /* lock against __dma_request_channel */
713 mutex_lock(&dma_list_mutex);
714
715 if (chan->client_count == 0) {
716 struct dma_device *device = chan->device;
717 int err;
718
719 dma_cap_set(DMA_PRIVATE, device->cap_mask);
720 device->privatecnt++;
721 err = dma_chan_get(chan);
722 if (err) {
723 dev_dbg(chan->device->dev,
724 "%s: failed to get %s: (%d)\n",
725 __func__, dma_chan_name(chan), err);
726 chan = NULL;
727 if (--device->privatecnt == 0)
728 dma_cap_clear(DMA_PRIVATE, device->cap_mask);
729 }
730 } else
731 chan = NULL;
732
733 mutex_unlock(&dma_list_mutex);
734
735
736 return chan;
737}
738EXPORT_SYMBOL_GPL(dma_get_slave_channel);
739
740struct dma_chan *dma_get_any_slave_channel(struct dma_device *device)
741{
742 dma_cap_mask_t mask;
743 struct dma_chan *chan;
744
745 dma_cap_zero(mask);
746 dma_cap_set(DMA_SLAVE, mask);
747
748 /* lock against __dma_request_channel */
749 mutex_lock(&dma_list_mutex);
750
751 chan = find_candidate(device, &mask, NULL, NULL);
752
753 mutex_unlock(&dma_list_mutex);
754
755 return IS_ERR(chan) ? NULL : chan;
756}
757EXPORT_SYMBOL_GPL(dma_get_any_slave_channel);
758
759/**
760 * __dma_request_channel - try to allocate an exclusive channel
761 * @mask: capabilities that the channel must satisfy
762 * @fn: optional callback to disposition available channels
763 * @fn_param: opaque parameter to pass to dma_filter_fn()
764 * @np: device node to look for DMA channels
765 *
766 * Returns pointer to appropriate DMA channel on success or NULL.
767 */
768struct dma_chan *__dma_request_channel(const dma_cap_mask_t *mask,
769 dma_filter_fn fn, void *fn_param,
770 struct device_node *np)
771{
772 struct dma_device *device, *_d;
773 struct dma_chan *chan = NULL;
774
775 /* Find a channel */
776 mutex_lock(&dma_list_mutex);
777 list_for_each_entry_safe(device, _d, &dma_device_list, global_node) {
778 /* Finds a DMA controller with matching device node */
779 if (np && !device_match_of_node(device->dev, np))
780 continue;
781
782 chan = find_candidate(device, mask, fn, fn_param);
783 if (!IS_ERR(chan))
784 break;
785
786 chan = NULL;
787 }
788 mutex_unlock(&dma_list_mutex);
789
790 pr_debug("%s: %s (%s)\n",
791 __func__,
792 chan ? "success" : "fail",
793 chan ? dma_chan_name(chan) : NULL);
794
795 return chan;
796}
797EXPORT_SYMBOL_GPL(__dma_request_channel);
798
799static const struct dma_slave_map *dma_filter_match(struct dma_device *device,
800 const char *name,
801 struct device *dev)
802{
803 int i;
804
805 if (!device->filter.mapcnt)
806 return NULL;
807
808 for (i = 0; i < device->filter.mapcnt; i++) {
809 const struct dma_slave_map *map = &device->filter.map[i];
810
811 if (!strcmp(map->devname, dev_name(dev)) &&
812 !strcmp(map->slave, name))
813 return map;
814 }
815
816 return NULL;
817}
818
819/**
820 * dma_request_chan - try to allocate an exclusive slave channel
821 * @dev: pointer to client device structure
822 * @name: slave channel name
823 *
824 * Returns pointer to appropriate DMA channel on success or an error pointer.
825 */
826struct dma_chan *dma_request_chan(struct device *dev, const char *name)
827{
828 struct fwnode_handle *fwnode;
829 struct dma_device *d, *_d;
830 struct dma_chan *chan = NULL;
831
832 if (WARN_ON(!dev || !name))
833 return ERR_PTR(-EINVAL);
834
835 fwnode = dev_fwnode(dev);
836
837 if (is_of_node(fwnode))
838 chan = of_dma_request_slave_channel(to_of_node(fwnode), name);
839 else if (is_acpi_device_node(fwnode))
840 chan = acpi_dma_request_slave_chan_by_name(dev, name);
841
842 if (PTR_ERR(chan) == -EPROBE_DEFER)
843 return chan;
844
845 if (!IS_ERR_OR_NULL(chan))
846 goto found;
847
848 /* Try to find the channel via the DMA filter map(s) */
849 mutex_lock(&dma_list_mutex);
850 list_for_each_entry_safe(d, _d, &dma_device_list, global_node) {
851 dma_cap_mask_t mask;
852 const struct dma_slave_map *map = dma_filter_match(d, name, dev);
853
854 if (!map)
855 continue;
856
857 dma_cap_zero(mask);
858 dma_cap_set(DMA_SLAVE, mask);
859
860 chan = find_candidate(d, &mask, d->filter.fn, map->param);
861 if (!IS_ERR(chan))
862 break;
863 }
864 mutex_unlock(&dma_list_mutex);
865
866 if (IS_ERR(chan))
867 return chan;
868 if (!chan)
869 return ERR_PTR(-EPROBE_DEFER);
870
871found:
872#ifdef CONFIG_DEBUG_FS
873 chan->dbg_client_name = kasprintf(GFP_KERNEL, "%s:%s", dev_name(dev), name);
874 /* No functional issue if it fails, users are supposed to test before use */
875#endif
876
877 chan->name = kasprintf(GFP_KERNEL, "dma:%s", name);
878 if (!chan->name)
879 return chan;
880 chan->slave = dev;
881
882 if (sysfs_create_link(&chan->dev->device.kobj, &dev->kobj,
883 DMA_SLAVE_NAME))
884 dev_warn(dev, "Cannot create DMA %s symlink\n", DMA_SLAVE_NAME);
885 if (sysfs_create_link(&dev->kobj, &chan->dev->device.kobj, chan->name))
886 dev_warn(dev, "Cannot create DMA %s symlink\n", chan->name);
887
888 return chan;
889}
890EXPORT_SYMBOL_GPL(dma_request_chan);
891
892/**
893 * dma_request_chan_by_mask - allocate a channel satisfying certain capabilities
894 * @mask: capabilities that the channel must satisfy
895 *
896 * Returns pointer to appropriate DMA channel on success or an error pointer.
897 */
898struct dma_chan *dma_request_chan_by_mask(const dma_cap_mask_t *mask)
899{
900 struct dma_chan *chan;
901
902 if (!mask)
903 return ERR_PTR(-ENODEV);
904
905 chan = __dma_request_channel(mask, NULL, NULL, NULL);
906 if (!chan) {
907 mutex_lock(&dma_list_mutex);
908 if (list_empty(&dma_device_list))
909 chan = ERR_PTR(-EPROBE_DEFER);
910 else
911 chan = ERR_PTR(-ENODEV);
912 mutex_unlock(&dma_list_mutex);
913 }
914
915 return chan;
916}
917EXPORT_SYMBOL_GPL(dma_request_chan_by_mask);
918
919void dma_release_channel(struct dma_chan *chan)
920{
921 mutex_lock(&dma_list_mutex);
922 WARN_ONCE(chan->client_count != 1,
923 "chan reference count %d != 1\n", chan->client_count);
924 /* drop PRIVATE cap enabled by __dma_request_channel() */
925 if (--chan->device->privatecnt == 0)
926 dma_cap_clear(DMA_PRIVATE, chan->device->cap_mask);
927
928 dma_chan_put(chan);
929
930 if (chan->slave) {
931 sysfs_remove_link(&chan->dev->device.kobj, DMA_SLAVE_NAME);
932 sysfs_remove_link(&chan->slave->kobj, chan->name);
933 kfree(chan->name);
934 chan->name = NULL;
935 chan->slave = NULL;
936 }
937
938#ifdef CONFIG_DEBUG_FS
939 kfree(chan->dbg_client_name);
940 chan->dbg_client_name = NULL;
941#endif
942 mutex_unlock(&dma_list_mutex);
943}
944EXPORT_SYMBOL_GPL(dma_release_channel);
945
946static void dmaenginem_release_channel(void *chan)
947{
948 dma_release_channel(chan);
949}
950
951/**
952 * devm_dma_request_chan - try to allocate an exclusive slave channel
953 * @dev: pointer to client device structure
954 * @name: slave channel name
955 *
956 * Returns pointer to appropriate DMA channel on success or an error pointer.
957 *
958 * The operation is managed and will be undone on driver detach.
959 */
960
961struct dma_chan *devm_dma_request_chan(struct device *dev, const char *name)
962{
963 struct dma_chan *chan;
964 int ret;
965
966 chan = dma_request_chan(dev, name);
967 if (IS_ERR(chan))
968 return chan;
969
970 ret = devm_add_action_or_reset(dev, dmaenginem_release_channel, chan);
971 if (ret)
972 return ERR_PTR(ret);
973
974 return chan;
975}
976EXPORT_SYMBOL_GPL(devm_dma_request_chan);
977
978/**
979 * dmaengine_get - register interest in dma_channels
980 */
981void dmaengine_get(void)
982{
983 struct dma_device *device, *_d;
984 struct dma_chan *chan;
985 int err;
986
987 mutex_lock(&dma_list_mutex);
988 dmaengine_ref_count++;
989
990 /* try to grab channels */
991 list_for_each_entry_safe(device, _d, &dma_device_list, global_node) {
992 if (dma_has_cap(DMA_PRIVATE, device->cap_mask))
993 continue;
994 list_for_each_entry(chan, &device->channels, device_node) {
995 err = dma_chan_get(chan);
996 if (err == -ENODEV) {
997 /* module removed before we could use it */
998 list_del_rcu(&device->global_node);
999 break;
1000 } else if (err)
1001 dev_dbg(chan->device->dev,
1002 "%s: failed to get %s: (%d)\n",
1003 __func__, dma_chan_name(chan), err);
1004 }
1005 }
1006
1007 /* if this is the first reference and there were channels
1008 * waiting we need to rebalance to get those channels
1009 * incorporated into the channel table
1010 */
1011 if (dmaengine_ref_count == 1)
1012 dma_channel_rebalance();
1013 mutex_unlock(&dma_list_mutex);
1014}
1015EXPORT_SYMBOL(dmaengine_get);
1016
1017/**
1018 * dmaengine_put - let DMA drivers be removed when ref_count == 0
1019 */
1020void dmaengine_put(void)
1021{
1022 struct dma_device *device, *_d;
1023 struct dma_chan *chan;
1024
1025 mutex_lock(&dma_list_mutex);
1026 dmaengine_ref_count--;
1027 BUG_ON(dmaengine_ref_count < 0);
1028 /* drop channel references */
1029 list_for_each_entry_safe(device, _d, &dma_device_list, global_node) {
1030 if (dma_has_cap(DMA_PRIVATE, device->cap_mask))
1031 continue;
1032 list_for_each_entry(chan, &device->channels, device_node)
1033 dma_chan_put(chan);
1034 }
1035 mutex_unlock(&dma_list_mutex);
1036}
1037EXPORT_SYMBOL(dmaengine_put);
1038
1039static bool device_has_all_tx_types(struct dma_device *device)
1040{
1041 /* A device that satisfies this test has channels that will never cause
1042 * an async_tx channel switch event as all possible operation types can
1043 * be handled.
1044 */
1045 #ifdef CONFIG_ASYNC_TX_DMA
1046 if (!dma_has_cap(DMA_INTERRUPT, device->cap_mask))
1047 return false;
1048 #endif
1049
1050 #if IS_ENABLED(CONFIG_ASYNC_MEMCPY)
1051 if (!dma_has_cap(DMA_MEMCPY, device->cap_mask))
1052 return false;
1053 #endif
1054
1055 #if IS_ENABLED(CONFIG_ASYNC_XOR)
1056 if (!dma_has_cap(DMA_XOR, device->cap_mask))
1057 return false;
1058
1059 #ifndef CONFIG_ASYNC_TX_DISABLE_XOR_VAL_DMA
1060 if (!dma_has_cap(DMA_XOR_VAL, device->cap_mask))
1061 return false;
1062 #endif
1063 #endif
1064
1065 #if IS_ENABLED(CONFIG_ASYNC_PQ)
1066 if (!dma_has_cap(DMA_PQ, device->cap_mask))
1067 return false;
1068
1069 #ifndef CONFIG_ASYNC_TX_DISABLE_PQ_VAL_DMA
1070 if (!dma_has_cap(DMA_PQ_VAL, device->cap_mask))
1071 return false;
1072 #endif
1073 #endif
1074
1075 return true;
1076}
1077
1078static int get_dma_id(struct dma_device *device)
1079{
1080 int rc = ida_alloc(&dma_ida, GFP_KERNEL);
1081
1082 if (rc < 0)
1083 return rc;
1084 device->dev_id = rc;
1085 return 0;
1086}
1087
1088static int __dma_async_device_channel_register(struct dma_device *device,
1089 struct dma_chan *chan,
1090 const char *name)
1091{
1092 int rc;
1093
1094 chan->local = alloc_percpu(typeof(*chan->local));
1095 if (!chan->local)
1096 return -ENOMEM;
1097 chan->dev = kzalloc_obj(*chan->dev);
1098 if (!chan->dev) {
1099 rc = -ENOMEM;
1100 goto err_free_local;
1101 }
1102
1103 /*
1104 * When the chan_id is a negative value, we are dynamically adding
1105 * the channel. Otherwise we are static enumerating.
1106 */
1107 chan->chan_id = ida_alloc(&device->chan_ida, GFP_KERNEL);
1108 if (chan->chan_id < 0) {
1109 pr_err("%s: unable to alloc ida for chan: %d\n",
1110 __func__, chan->chan_id);
1111 rc = chan->chan_id;
1112 goto err_free_dev;
1113 }
1114
1115 chan->dev->device.class = &dma_devclass;
1116 chan->dev->device.parent = device->dev;
1117 chan->dev->chan = chan;
1118 chan->dev->dev_id = device->dev_id;
1119 spin_lock_init(&chan->lock);
1120
1121 if (!name)
1122 dev_set_name(&chan->dev->device, "dma%dchan%d", device->dev_id, chan->chan_id);
1123 else
1124 dev_set_name(&chan->dev->device, "%s", name);
1125 rc = device_register(&chan->dev->device);
1126 if (rc)
1127 goto err_out_ida;
1128 chan->client_count = 0;
1129 device->chancnt++;
1130
1131 return 0;
1132
1133 err_out_ida:
1134 ida_free(&device->chan_ida, chan->chan_id);
1135 err_free_dev:
1136 kfree(chan->dev);
1137 err_free_local:
1138 free_percpu(chan->local);
1139 chan->local = NULL;
1140 return rc;
1141}
1142
1143int dma_async_device_channel_register(struct dma_device *device,
1144 struct dma_chan *chan,
1145 const char *name)
1146{
1147 int rc;
1148
1149 rc = __dma_async_device_channel_register(device, chan, name);
1150 if (rc < 0)
1151 return rc;
1152
1153 dma_channel_rebalance();
1154 return 0;
1155}
1156EXPORT_SYMBOL_GPL(dma_async_device_channel_register);
1157
1158static void __dma_async_device_channel_unregister(struct dma_device *device,
1159 struct dma_chan *chan)
1160{
1161 if (chan->local == NULL)
1162 return;
1163
1164 WARN_ONCE(!device->device_release && chan->client_count,
1165 "%s called while %d clients hold a reference\n",
1166 __func__, chan->client_count);
1167 mutex_lock(&dma_list_mutex);
1168 device->chancnt--;
1169 chan->dev->chan = NULL;
1170 mutex_unlock(&dma_list_mutex);
1171 ida_free(&device->chan_ida, chan->chan_id);
1172 device_unregister(&chan->dev->device);
1173 free_percpu(chan->local);
1174}
1175
1176void dma_async_device_channel_unregister(struct dma_device *device,
1177 struct dma_chan *chan)
1178{
1179 __dma_async_device_channel_unregister(device, chan);
1180 dma_channel_rebalance();
1181}
1182EXPORT_SYMBOL_GPL(dma_async_device_channel_unregister);
1183
1184/**
1185 * dma_async_device_register - registers DMA devices found
1186 * @device: pointer to &struct dma_device
1187 *
1188 * After calling this routine the structure should not be freed except in the
1189 * device_release() callback which will be called after
1190 * dma_async_device_unregister() is called and no further references are taken.
1191 */
1192int dma_async_device_register(struct dma_device *device)
1193{
1194 int rc;
1195 struct dma_chan* chan;
1196
1197 if (!device)
1198 return -ENODEV;
1199
1200 /* validate device routines */
1201 if (!device->dev) {
1202 pr_err("DMAdevice must have dev\n");
1203 return -EIO;
1204 }
1205
1206 device->owner = device->dev->driver->owner;
1207
1208#define CHECK_CAP(_name, _type) \
1209{ \
1210 if (dma_has_cap(_type, device->cap_mask) && !device->device_prep_##_name) { \
1211 dev_err(device->dev, \
1212 "Device claims capability %s, but op is not defined\n", \
1213 __stringify(_type)); \
1214 return -EIO; \
1215 } \
1216}
1217
1218 CHECK_CAP(dma_memcpy, DMA_MEMCPY);
1219 CHECK_CAP(dma_xor, DMA_XOR);
1220 CHECK_CAP(dma_xor_val, DMA_XOR_VAL);
1221 CHECK_CAP(dma_pq, DMA_PQ);
1222 CHECK_CAP(dma_pq_val, DMA_PQ_VAL);
1223 CHECK_CAP(dma_memset, DMA_MEMSET);
1224 CHECK_CAP(dma_interrupt, DMA_INTERRUPT);
1225 CHECK_CAP(dma_cyclic, DMA_CYCLIC);
1226 CHECK_CAP(interleaved_dma, DMA_INTERLEAVE);
1227
1228#undef CHECK_CAP
1229
1230 if (!device->device_tx_status) {
1231 dev_err(device->dev, "Device tx_status is not defined\n");
1232 return -EIO;
1233 }
1234
1235
1236 if (!device->device_issue_pending) {
1237 dev_err(device->dev, "Device issue_pending is not defined\n");
1238 return -EIO;
1239 }
1240
1241 if (!device->device_release)
1242 dev_dbg(device->dev,
1243 "WARN: Device release is not defined so it is not safe to unbind this driver while in use\n");
1244
1245 kref_init(&device->ref);
1246
1247 /* note: this only matters in the
1248 * CONFIG_ASYNC_TX_ENABLE_CHANNEL_SWITCH=n case
1249 */
1250 if (device_has_all_tx_types(device))
1251 dma_cap_set(DMA_ASYNC_TX, device->cap_mask);
1252
1253 rc = get_dma_id(device);
1254 if (rc != 0)
1255 return rc;
1256
1257 ida_init(&device->chan_ida);
1258
1259 /* represent channels in sysfs. Probably want devs too */
1260 list_for_each_entry(chan, &device->channels, device_node) {
1261 rc = __dma_async_device_channel_register(device, chan, NULL);
1262 if (rc < 0)
1263 goto err_out;
1264 }
1265
1266 mutex_lock(&dma_list_mutex);
1267 /* take references on public channels */
1268 if (dmaengine_ref_count && !dma_has_cap(DMA_PRIVATE, device->cap_mask))
1269 list_for_each_entry(chan, &device->channels, device_node) {
1270 /* if clients are already waiting for channels we need
1271 * to take references on their behalf
1272 */
1273 if (dma_chan_get(chan) == -ENODEV) {
1274 /* note we can only get here for the first
1275 * channel as the remaining channels are
1276 * guaranteed to get a reference
1277 */
1278 rc = -ENODEV;
1279 mutex_unlock(&dma_list_mutex);
1280 goto err_out;
1281 }
1282 }
1283 list_add_tail_rcu(&device->global_node, &dma_device_list);
1284 if (dma_has_cap(DMA_PRIVATE, device->cap_mask))
1285 device->privatecnt++; /* Always private */
1286 dma_channel_rebalance();
1287 mutex_unlock(&dma_list_mutex);
1288
1289 dmaengine_debug_register(device);
1290
1291 return 0;
1292
1293err_out:
1294 /* if we never registered a channel just release the idr */
1295 if (!device->chancnt) {
1296 ida_free(&dma_ida, device->dev_id);
1297 return rc;
1298 }
1299
1300 list_for_each_entry(chan, &device->channels, device_node) {
1301 if (chan->local == NULL)
1302 continue;
1303 mutex_lock(&dma_list_mutex);
1304 chan->dev->chan = NULL;
1305 mutex_unlock(&dma_list_mutex);
1306 device_unregister(&chan->dev->device);
1307 free_percpu(chan->local);
1308 }
1309 return rc;
1310}
1311EXPORT_SYMBOL(dma_async_device_register);
1312
1313/**
1314 * dma_async_device_unregister - unregister a DMA device
1315 * @device: pointer to &struct dma_device
1316 *
1317 * This routine is called by dma driver exit routines, dmaengine holds module
1318 * references to prevent it being called while channels are in use.
1319 */
1320void dma_async_device_unregister(struct dma_device *device)
1321{
1322 struct dma_chan *chan, *n;
1323
1324 dmaengine_debug_unregister(device);
1325
1326 list_for_each_entry_safe(chan, n, &device->channels, device_node)
1327 __dma_async_device_channel_unregister(device, chan);
1328
1329 mutex_lock(&dma_list_mutex);
1330 /*
1331 * setting DMA_PRIVATE ensures the device being torn down will not
1332 * be used in the channel_table
1333 */
1334 dma_cap_set(DMA_PRIVATE, device->cap_mask);
1335 dma_channel_rebalance();
1336 ida_free(&dma_ida, device->dev_id);
1337 dma_device_put(device);
1338 mutex_unlock(&dma_list_mutex);
1339}
1340EXPORT_SYMBOL(dma_async_device_unregister);
1341
1342static void dmaenginem_async_device_unregister(void *device)
1343{
1344 dma_async_device_unregister(device);
1345}
1346
1347/**
1348 * dmaenginem_async_device_register - registers DMA devices found
1349 * @device: pointer to &struct dma_device
1350 *
1351 * The operation is managed and will be undone on driver detach.
1352 */
1353int dmaenginem_async_device_register(struct dma_device *device)
1354{
1355 int ret;
1356
1357 ret = dma_async_device_register(device);
1358 if (ret)
1359 return ret;
1360
1361 return devm_add_action_or_reset(device->dev, dmaenginem_async_device_unregister, device);
1362}
1363EXPORT_SYMBOL(dmaenginem_async_device_register);
1364
1365struct dmaengine_unmap_pool {
1366 struct kmem_cache *cache;
1367 const char *name;
1368 mempool_t *pool;
1369 size_t size;
1370};
1371
1372#define __UNMAP_POOL(x) { .size = x, .name = "dmaengine-unmap-" __stringify(x) }
1373static struct dmaengine_unmap_pool unmap_pool[] = {
1374 __UNMAP_POOL(2),
1375 #if IS_ENABLED(CONFIG_DMA_ENGINE_RAID)
1376 __UNMAP_POOL(16),
1377 __UNMAP_POOL(128),
1378 __UNMAP_POOL(256),
1379 #endif
1380};
1381
1382static struct dmaengine_unmap_pool *__get_unmap_pool(int nr)
1383{
1384 int order = get_count_order(nr);
1385
1386 switch (order) {
1387 case 0 ... 1:
1388 return &unmap_pool[0];
1389#if IS_ENABLED(CONFIG_DMA_ENGINE_RAID)
1390 case 2 ... 4:
1391 return &unmap_pool[1];
1392 case 5 ... 7:
1393 return &unmap_pool[2];
1394 case 8:
1395 return &unmap_pool[3];
1396#endif
1397 default:
1398 BUG();
1399 return NULL;
1400 }
1401}
1402
1403static void dmaengine_unmap(struct kref *kref)
1404{
1405 struct dmaengine_unmap_data *unmap = container_of(kref, typeof(*unmap), kref);
1406 struct device *dev = unmap->dev;
1407 int cnt, i;
1408
1409 cnt = unmap->to_cnt;
1410 for (i = 0; i < cnt; i++)
1411 dma_unmap_page(dev, unmap->addr[i], unmap->len,
1412 DMA_TO_DEVICE);
1413 cnt += unmap->from_cnt;
1414 for (; i < cnt; i++)
1415 dma_unmap_page(dev, unmap->addr[i], unmap->len,
1416 DMA_FROM_DEVICE);
1417 cnt += unmap->bidi_cnt;
1418 for (; i < cnt; i++) {
1419 if (unmap->addr[i] == 0)
1420 continue;
1421 dma_unmap_page(dev, unmap->addr[i], unmap->len,
1422 DMA_BIDIRECTIONAL);
1423 }
1424 cnt = unmap->map_cnt;
1425 mempool_free(unmap, __get_unmap_pool(cnt)->pool);
1426}
1427
1428void dmaengine_unmap_put(struct dmaengine_unmap_data *unmap)
1429{
1430 if (unmap)
1431 kref_put(&unmap->kref, dmaengine_unmap);
1432}
1433EXPORT_SYMBOL_GPL(dmaengine_unmap_put);
1434
1435static void dmaengine_destroy_unmap_pool(void)
1436{
1437 int i;
1438
1439 for (i = 0; i < ARRAY_SIZE(unmap_pool); i++) {
1440 struct dmaengine_unmap_pool *p = &unmap_pool[i];
1441
1442 mempool_destroy(p->pool);
1443 p->pool = NULL;
1444 kmem_cache_destroy(p->cache);
1445 p->cache = NULL;
1446 }
1447}
1448
1449static int __init dmaengine_init_unmap_pool(void)
1450{
1451 int i;
1452
1453 for (i = 0; i < ARRAY_SIZE(unmap_pool); i++) {
1454 struct dmaengine_unmap_pool *p = &unmap_pool[i];
1455 size_t size;
1456
1457 size = sizeof(struct dmaengine_unmap_data) +
1458 sizeof(dma_addr_t) * p->size;
1459
1460 p->cache = kmem_cache_create(p->name, size, 0,
1461 SLAB_HWCACHE_ALIGN, NULL);
1462 if (!p->cache)
1463 break;
1464 p->pool = mempool_create_slab_pool(1, p->cache);
1465 if (!p->pool)
1466 break;
1467 }
1468
1469 if (i == ARRAY_SIZE(unmap_pool))
1470 return 0;
1471
1472 dmaengine_destroy_unmap_pool();
1473 return -ENOMEM;
1474}
1475
1476struct dmaengine_unmap_data *
1477dmaengine_get_unmap_data(struct device *dev, int nr, gfp_t flags)
1478{
1479 struct dmaengine_unmap_data *unmap;
1480
1481 unmap = mempool_alloc(__get_unmap_pool(nr)->pool, flags);
1482 if (!unmap)
1483 return NULL;
1484
1485 memset(unmap, 0, sizeof(*unmap));
1486 kref_init(&unmap->kref);
1487 unmap->dev = dev;
1488 unmap->map_cnt = nr;
1489
1490 return unmap;
1491}
1492EXPORT_SYMBOL(dmaengine_get_unmap_data);
1493
1494void dma_async_tx_descriptor_init(struct dma_async_tx_descriptor *tx,
1495 struct dma_chan *chan)
1496{
1497 tx->chan = chan;
1498 #ifdef CONFIG_ASYNC_TX_ENABLE_CHANNEL_SWITCH
1499 spin_lock_init(&tx->lock);
1500 #endif
1501}
1502EXPORT_SYMBOL(dma_async_tx_descriptor_init);
1503
1504static inline int desc_check_and_set_metadata_mode(
1505 struct dma_async_tx_descriptor *desc, enum dma_desc_metadata_mode mode)
1506{
1507 /* Make sure that the metadata mode is not mixed */
1508 if (!desc->desc_metadata_mode) {
1509 if (dmaengine_is_metadata_mode_supported(desc->chan, mode))
1510 desc->desc_metadata_mode = mode;
1511 else
1512 return -ENOTSUPP;
1513 } else if (desc->desc_metadata_mode != mode) {
1514 return -EINVAL;
1515 }
1516
1517 return 0;
1518}
1519
1520int dmaengine_desc_attach_metadata(struct dma_async_tx_descriptor *desc,
1521 void *data, size_t len)
1522{
1523 int ret;
1524
1525 if (!desc)
1526 return -EINVAL;
1527
1528 ret = desc_check_and_set_metadata_mode(desc, DESC_METADATA_CLIENT);
1529 if (ret)
1530 return ret;
1531
1532 if (!desc->metadata_ops || !desc->metadata_ops->attach)
1533 return -ENOTSUPP;
1534
1535 return desc->metadata_ops->attach(desc, data, len);
1536}
1537EXPORT_SYMBOL_GPL(dmaengine_desc_attach_metadata);
1538
1539void *dmaengine_desc_get_metadata_ptr(struct dma_async_tx_descriptor *desc,
1540 size_t *payload_len, size_t *max_len)
1541{
1542 int ret;
1543
1544 if (!desc)
1545 return ERR_PTR(-EINVAL);
1546
1547 ret = desc_check_and_set_metadata_mode(desc, DESC_METADATA_ENGINE);
1548 if (ret)
1549 return ERR_PTR(ret);
1550
1551 if (!desc->metadata_ops || !desc->metadata_ops->get_ptr)
1552 return ERR_PTR(-ENOTSUPP);
1553
1554 return desc->metadata_ops->get_ptr(desc, payload_len, max_len);
1555}
1556EXPORT_SYMBOL_GPL(dmaengine_desc_get_metadata_ptr);
1557
1558int dmaengine_desc_set_metadata_len(struct dma_async_tx_descriptor *desc,
1559 size_t payload_len)
1560{
1561 int ret;
1562
1563 if (!desc)
1564 return -EINVAL;
1565
1566 ret = desc_check_and_set_metadata_mode(desc, DESC_METADATA_ENGINE);
1567 if (ret)
1568 return ret;
1569
1570 if (!desc->metadata_ops || !desc->metadata_ops->set_len)
1571 return -ENOTSUPP;
1572
1573 return desc->metadata_ops->set_len(desc, payload_len);
1574}
1575EXPORT_SYMBOL_GPL(dmaengine_desc_set_metadata_len);
1576
1577/**
1578 * dma_wait_for_async_tx - spin wait for a transaction to complete
1579 * @tx: in-flight transaction to wait on
1580 */
1581enum dma_status
1582dma_wait_for_async_tx(struct dma_async_tx_descriptor *tx)
1583{
1584 unsigned long dma_sync_wait_timeout = jiffies + msecs_to_jiffies(5000);
1585
1586 if (!tx)
1587 return DMA_COMPLETE;
1588
1589 while (tx->cookie == -EBUSY) {
1590 if (time_after_eq(jiffies, dma_sync_wait_timeout)) {
1591 dev_err(tx->chan->device->dev,
1592 "%s timeout waiting for descriptor submission\n",
1593 __func__);
1594 return DMA_ERROR;
1595 }
1596 cpu_relax();
1597 }
1598 return dma_sync_wait(tx->chan, tx->cookie);
1599}
1600EXPORT_SYMBOL_GPL(dma_wait_for_async_tx);
1601
1602/**
1603 * dma_run_dependencies - process dependent operations on the target channel
1604 * @tx: transaction with dependencies
1605 *
1606 * Helper routine for DMA drivers to process (start) dependent operations
1607 * on their target channel.
1608 */
1609void dma_run_dependencies(struct dma_async_tx_descriptor *tx)
1610{
1611 struct dma_async_tx_descriptor *dep = txd_next(tx);
1612 struct dma_async_tx_descriptor *dep_next;
1613 struct dma_chan *chan;
1614
1615 if (!dep)
1616 return;
1617
1618 /* we'll submit tx->next now, so clear the link */
1619 txd_clear_next(tx);
1620 chan = dep->chan;
1621
1622 /* keep submitting up until a channel switch is detected
1623 * in that case we will be called again as a result of
1624 * processing the interrupt from async_tx_channel_switch
1625 */
1626 for (; dep; dep = dep_next) {
1627 txd_lock(dep);
1628 txd_clear_parent(dep);
1629 dep_next = txd_next(dep);
1630 if (dep_next && dep_next->chan == chan)
1631 txd_clear_next(dep); /* ->next will be submitted */
1632 else
1633 dep_next = NULL; /* submit current dep and terminate */
1634 txd_unlock(dep);
1635
1636 dep->tx_submit(dep);
1637 }
1638
1639 chan->device->device_issue_pending(chan);
1640}
1641EXPORT_SYMBOL_GPL(dma_run_dependencies);
1642
1643static int __init dma_bus_init(void)
1644{
1645 int err = dmaengine_init_unmap_pool();
1646
1647 if (err)
1648 return err;
1649
1650 err = class_register(&dma_devclass);
1651 if (!err)
1652 dmaengine_debugfs_init();
1653
1654 return err;
1655}
1656arch_initcall(dma_bus_init);