projects/linux·blob master · refs

465 lines · 13.3 KiB

1// SPDX-License-Identifier: GPL-2.0-or-later
2/*
3 * Copyright 2014-2015 Analog Devices Inc.
4 * Author: Lars-Peter Clausen <[email protected]>
5 */
6
7#include <linux/slab.h>
8#include <linux/kernel.h>
9#include <linux/cleanup.h>
10#include <linux/dmaengine.h>
11#include <linux/dma-mapping.h>
12#include <linux/spinlock.h>
13#include <linux/err.h>
14#include <linux/module.h>
15
16#include <linux/iio/iio.h>
17#include <linux/iio/sysfs.h>
18#include <linux/iio/buffer.h>
19#include <linux/iio/buffer_impl.h>
20#include <linux/iio/buffer-dma.h>
21#include <linux/iio/buffer-dmaengine.h>
22
23/*
24 * The IIO DMAengine buffer combines the generic IIO DMA buffer infrastructure
25 * with the DMAengine framework. The generic IIO DMA buffer infrastructure is
26 * used to manage the buffer memory and implement the IIO buffer operations
27 * while the DMAengine framework is used to perform the DMA transfers. Combined
28 * this results in a device independent fully functional DMA buffer
29 * implementation that can be used by device drivers for peripherals which are
30 * connected to a DMA controller which has a DMAengine driver implementation.
31 */
32
33struct dmaengine_buffer {
34 struct iio_dma_buffer_queue queue;
35
36 struct dma_chan *chan;
37 struct list_head active;
38
39 size_t align;
40 size_t max_size;
41};
42
43static struct dmaengine_buffer *iio_buffer_to_dmaengine_buffer(struct iio_buffer *buffer)
44{
45 return container_of(buffer, struct dmaengine_buffer, queue.buffer);
46}
47
48static void iio_dmaengine_buffer_block_done(void *data,
49 const struct dmaengine_result *result)
50{
51 struct iio_dma_buffer_block *block = data;
52
53 scoped_guard(spinlock_irqsave, &block->queue->list_lock)
54 list_del(&block->head);
55 block->bytes_used -= result->residue;
56 iio_dma_buffer_block_done(block);
57}
58
59/*
60 * Cyclic transfers run until abort. Cap active cyclic blocks to one until there
61 * is a use case for more.
62 */
63static bool iio_dmaengine_buffer_has_active_cyclic(struct dmaengine_buffer *dmaengine_buffer)
64{
65 struct iio_dma_buffer_block *block;
66
67 guard(spinlock_irqsave)(&dmaengine_buffer->queue.list_lock);
68 list_for_each_entry(block, &dmaengine_buffer->active, head) {
69 if (block->cyclic)
70 return true;
71 }
72
73 return false;
74}
75
76static int iio_dmaengine_buffer_submit_block(struct iio_dma_buffer_queue *queue,
77 struct iio_dma_buffer_block *block)
78{
79 struct dmaengine_buffer *dmaengine_buffer =
80 iio_buffer_to_dmaengine_buffer(&queue->buffer);
81 struct dma_async_tx_descriptor *desc;
82 enum dma_transfer_direction dma_dir;
83 struct scatterlist *sgl;
84 struct dma_vec *vecs;
85 size_t max_size;
86 dma_cookie_t cookie;
87 size_t len_total;
88 unsigned int i;
89 int nents;
90
91 max_size = min(block->size, dmaengine_buffer->max_size);
92 max_size = round_down(max_size, dmaengine_buffer->align);
93
94 if (queue->buffer.direction == IIO_BUFFER_DIRECTION_IN)
95 dma_dir = DMA_DEV_TO_MEM;
96 else
97 dma_dir = DMA_MEM_TO_DEV;
98
99 if (block->cyclic && iio_dmaengine_buffer_has_active_cyclic(dmaengine_buffer)) {
100 dev_err(queue->dev, "cyclic DMA transfer already active\n");
101 return -EBUSY;
102 }
103
104 if (block->sg_table) {
105 unsigned long flags;
106
107 /*
108 * Only the first sgt->nents entries carry a valid
109 * sg_dma_address()/sg_dma_len() pair as mapping the table may
110 * have coalesced entries, in which case nents is smaller than
111 * orig_nents.
112 */
113 nents = block->sg_table->nents;
114
115 vecs = kmalloc_objs(*vecs, nents, GFP_ATOMIC);
116 if (!vecs)
117 return -ENOMEM;
118
119 len_total = block->bytes_used;
120
121 sgl = block->sg_table->sgl;
122 for (i = 0; i < nents && len_total; i++) {
123 vecs[i].addr = sg_dma_address(sgl);
124 vecs[i].len = min(sg_dma_len(sgl), len_total);
125 len_total -= vecs[i].len;
126
127 sgl = sg_next(sgl);
128 }
129
130 nents = i;
131
132 if (block->cyclic)
133 flags = DMA_PREP_REPEAT;
134 else
135 flags = DMA_PREP_INTERRUPT;
136
137 /*
138 * A new transfer may need to end an already active cyclic transfer
139 * before it can run, so always set the EOT flag.
140 */
141 flags |= DMA_PREP_LOAD_EOT;
142 desc = dmaengine_prep_peripheral_dma_vec(dmaengine_buffer->chan,
143 vecs, nents, dma_dir,
144 flags);
145 kfree(vecs);
146 } else {
147 max_size = min(block->size, dmaengine_buffer->max_size);
148 max_size = round_down(max_size, dmaengine_buffer->align);
149
150 if (queue->buffer.direction == IIO_BUFFER_DIRECTION_IN)
151 block->bytes_used = max_size;
152
153 if (!block->bytes_used || block->bytes_used > max_size)
154 return -EINVAL;
155
156 desc = dmaengine_prep_slave_single(dmaengine_buffer->chan,
157 block->phys_addr,
158 block->bytes_used,
159 dma_dir,
160 DMA_PREP_INTERRUPT);
161 }
162 if (!desc)
163 return -ENOMEM;
164
165 if (!block->cyclic) {
166 desc->callback_result = iio_dmaengine_buffer_block_done;
167 desc->callback_param = block;
168 }
169
170 cookie = dmaengine_submit(desc);
171 if (dma_submit_error(cookie))
172 return dma_submit_error(cookie);
173
174 scoped_guard(spinlock_irq, &dmaengine_buffer->queue.list_lock)
175 list_add_tail(&block->head, &dmaengine_buffer->active);
176
177 dma_async_issue_pending(dmaengine_buffer->chan);
178
179 return 0;
180}
181
182static void iio_dmaengine_buffer_abort(struct iio_dma_buffer_queue *queue)
183{
184 struct dmaengine_buffer *dmaengine_buffer =
185 iio_buffer_to_dmaengine_buffer(&queue->buffer);
186
187 dmaengine_terminate_sync(dmaengine_buffer->chan);
188 iio_dma_buffer_block_list_abort(queue, &dmaengine_buffer->active);
189}
190
191static void iio_dmaengine_buffer_release(struct iio_buffer *buf)
192{
193 struct dmaengine_buffer *dmaengine_buffer =
194 iio_buffer_to_dmaengine_buffer(buf);
195
196 iio_dma_buffer_release(&dmaengine_buffer->queue);
197 kfree(dmaengine_buffer);
198}
199
200static const struct iio_buffer_access_funcs iio_dmaengine_buffer_ops = {
201 .read = iio_dma_buffer_read,
202 .write = iio_dma_buffer_write,
203 .set_bytes_per_datum = iio_dma_buffer_set_bytes_per_datum,
204 .set_length = iio_dma_buffer_set_length,
205 .request_update = iio_dma_buffer_request_update,
206 .enable = iio_dma_buffer_enable,
207 .disable = iio_dma_buffer_disable,
208 .data_available = iio_dma_buffer_usage,
209 .space_available = iio_dma_buffer_usage,
210 .release = iio_dmaengine_buffer_release,
211
212 .enqueue_dmabuf = iio_dma_buffer_enqueue_dmabuf,
213 .attach_dmabuf = iio_dma_buffer_attach_dmabuf,
214 .detach_dmabuf = iio_dma_buffer_detach_dmabuf,
215
216 .lock_queue = iio_dma_buffer_lock_queue,
217 .unlock_queue = iio_dma_buffer_unlock_queue,
218
219 .get_dma_dev = iio_dma_buffer_get_dma_dev,
220
221 .modes = INDIO_BUFFER_HARDWARE,
222 .flags = INDIO_BUFFER_FLAG_FIXED_WATERMARK,
223};
224
225static const struct iio_dma_buffer_ops iio_dmaengine_default_ops = {
226 .submit = iio_dmaengine_buffer_submit_block,
227 .abort = iio_dmaengine_buffer_abort,
228};
229
230static ssize_t iio_dmaengine_buffer_get_length_align(struct device *dev,
231 struct device_attribute *attr, char *buf)
232{
233 struct iio_buffer *buffer = to_iio_dev_attr(attr)->buffer;
234 struct dmaengine_buffer *dmaengine_buffer =
235 iio_buffer_to_dmaengine_buffer(buffer);
236
237 return sysfs_emit(buf, "%zu\n", dmaengine_buffer->align);
238}
239
240static IIO_DEVICE_ATTR(length_align_bytes, 0444,
241 iio_dmaengine_buffer_get_length_align, NULL, 0);
242
243static const struct iio_dev_attr *iio_dmaengine_buffer_attrs[] = {
244 &iio_dev_attr_length_align_bytes,
245 NULL,
246};
247
248/**
249 * iio_dmaengine_buffer_alloc() - Allocate new buffer which uses DMAengine
250 * @chan: DMA channel.
251 *
252 * This allocates a new IIO buffer which internally uses the DMAengine framework
253 * to perform its transfers.
254 *
255 * Once done using the buffer iio_dmaengine_buffer_free() should be used to
256 * release it.
257 */
258static struct iio_buffer *iio_dmaengine_buffer_alloc(struct dma_chan *chan)
259{
260 struct dmaengine_buffer *dmaengine_buffer;
261 unsigned int width, src_width, dest_width;
262 struct dma_slave_caps caps;
263 int ret;
264
265 ret = dma_get_slave_caps(chan, &caps);
266 if (ret < 0)
267 return ERR_PTR(ret);
268
269 dmaengine_buffer = kzalloc_obj(*dmaengine_buffer);
270 if (!dmaengine_buffer)
271 return ERR_PTR(-ENOMEM);
272
273 /* Needs to be aligned to the maximum of the minimums */
274 if (caps.src_addr_widths)
275 src_width = __ffs(caps.src_addr_widths);
276 else
277 src_width = 1;
278 if (caps.dst_addr_widths)
279 dest_width = __ffs(caps.dst_addr_widths);
280 else
281 dest_width = 1;
282 width = max(src_width, dest_width);
283
284 INIT_LIST_HEAD(&dmaengine_buffer->active);
285 dmaengine_buffer->chan = chan;
286 dmaengine_buffer->align = width;
287 dmaengine_buffer->max_size = dma_get_max_seg_size(chan->device->dev);
288
289 iio_dma_buffer_init(&dmaengine_buffer->queue, chan->device->dev,
290 &iio_dmaengine_default_ops);
291
292 dmaengine_buffer->queue.buffer.attrs = iio_dmaengine_buffer_attrs;
293 dmaengine_buffer->queue.buffer.access = &iio_dmaengine_buffer_ops;
294
295 return &dmaengine_buffer->queue.buffer;
296}
297
298/**
299 * iio_dmaengine_buffer_free() - Free dmaengine buffer
300 * @buffer: Buffer to free
301 *
302 * Frees a buffer previously allocated with iio_dmaengine_buffer_alloc().
303 */
304static void iio_dmaengine_buffer_free(struct iio_buffer *buffer)
305{
306 struct dmaengine_buffer *dmaengine_buffer =
307 iio_buffer_to_dmaengine_buffer(buffer);
308
309 iio_dma_buffer_exit(&dmaengine_buffer->queue);
310 iio_buffer_put(buffer);
311}
312
313/**
314 * iio_dmaengine_buffer_teardown() - Releases DMA channel and frees buffer
315 * @buffer: Buffer to free
316 *
317 * Releases the DMA channel and frees the buffer previously setup with
318 * iio_dmaengine_buffer_setup_ext().
319 */
320void iio_dmaengine_buffer_teardown(struct iio_buffer *buffer)
321{
322 struct dmaengine_buffer *dmaengine_buffer =
323 iio_buffer_to_dmaengine_buffer(buffer);
324 struct dma_chan *chan = dmaengine_buffer->chan;
325
326 iio_dmaengine_buffer_free(buffer);
327 dma_release_channel(chan);
328}
329EXPORT_SYMBOL_NS_GPL(iio_dmaengine_buffer_teardown, "IIO_DMAENGINE_BUFFER");
330
331static struct iio_buffer
332*__iio_dmaengine_buffer_setup_ext(struct iio_dev *indio_dev,
333 struct dma_chan *chan,
334 enum iio_buffer_direction dir)
335{
336 struct iio_buffer *buffer;
337 int ret;
338
339 buffer = iio_dmaengine_buffer_alloc(chan);
340 if (IS_ERR(buffer))
341 return ERR_CAST(buffer);
342
343 indio_dev->modes |= INDIO_BUFFER_HARDWARE;
344
345 buffer->direction = dir;
346
347 ret = iio_device_attach_buffer(indio_dev, buffer);
348 if (ret) {
349 iio_dmaengine_buffer_free(buffer);
350 return ERR_PTR(ret);
351 }
352
353 return buffer;
354}
355
356/**
357 * iio_dmaengine_buffer_setup_ext() - Setup a DMA buffer for an IIO device
358 * @dev: DMA channel consumer device
359 * @indio_dev: IIO device to which to attach this buffer.
360 * @channel: DMA channel name, typically "rx".
361 * @dir: Direction of buffer (in or out)
362 *
363 * This allocates a new IIO buffer with devm_iio_dmaengine_buffer_alloc()
364 * and attaches it to an IIO device with iio_device_attach_buffer().
365 * It also appends the INDIO_BUFFER_HARDWARE mode to the supported modes of the
366 * IIO device.
367 *
368 * Once done using the buffer iio_dmaengine_buffer_teardown() should be used to
369 * release it.
370 */
371struct iio_buffer *iio_dmaengine_buffer_setup_ext(struct device *dev,
372 struct iio_dev *indio_dev,
373 const char *channel,
374 enum iio_buffer_direction dir)
375{
376 struct dma_chan *chan;
377 struct iio_buffer *buffer;
378
379 chan = dma_request_chan(dev, channel);
380 if (IS_ERR(chan))
381 return ERR_CAST(chan);
382
383 buffer = __iio_dmaengine_buffer_setup_ext(indio_dev, chan, dir);
384 if (IS_ERR(buffer))
385 dma_release_channel(chan);
386
387 return buffer;
388}
389EXPORT_SYMBOL_NS_GPL(iio_dmaengine_buffer_setup_ext, "IIO_DMAENGINE_BUFFER");
390
391static void devm_iio_dmaengine_buffer_teardown(void *buffer)
392{
393 iio_dmaengine_buffer_teardown(buffer);
394}
395
396/**
397 * devm_iio_dmaengine_buffer_setup_ext() - Setup a DMA buffer for an IIO device
398 * @dev: Device for devm ownership and DMA channel consumer device
399 * @indio_dev: IIO device to which to attach this buffer.
400 * @channel: DMA channel name, typically "rx".
401 * @dir: Direction of buffer (in or out)
402 *
403 * This allocates a new IIO buffer with devm_iio_dmaengine_buffer_alloc()
404 * and attaches it to an IIO device with iio_device_attach_buffer().
405 * It also appends the INDIO_BUFFER_HARDWARE mode to the supported modes of the
406 * IIO device.
407 */
408int devm_iio_dmaengine_buffer_setup_ext(struct device *dev,
409 struct iio_dev *indio_dev,
410 const char *channel,
411 enum iio_buffer_direction dir)
412{
413 struct iio_buffer *buffer;
414
415 buffer = iio_dmaengine_buffer_setup_ext(dev, indio_dev, channel, dir);
416 if (IS_ERR(buffer))
417 return PTR_ERR(buffer);
418
419 return devm_add_action_or_reset(dev, devm_iio_dmaengine_buffer_teardown,
420 buffer);
421}
422EXPORT_SYMBOL_NS_GPL(devm_iio_dmaengine_buffer_setup_ext, "IIO_DMAENGINE_BUFFER");
423
424static void devm_iio_dmaengine_buffer_free(void *buffer)
425{
426 iio_dmaengine_buffer_free(buffer);
427}
428
429/**
430 * devm_iio_dmaengine_buffer_setup_with_handle() - Setup a DMA buffer for an
431 * IIO device
432 * @dev: Device for devm ownership
433 * @indio_dev: IIO device to which to attach this buffer.
434 * @chan: DMA channel
435 * @dir: Direction of buffer (in or out)
436 *
437 * This allocates a new IIO buffer with devm_iio_dmaengine_buffer_alloc()
438 * and attaches it to an IIO device with iio_device_attach_buffer().
439 * It also appends the INDIO_BUFFER_HARDWARE mode to the supported modes of the
440 * IIO device.
441 *
442 * This is the same as devm_iio_dmaengine_buffer_setup_ext() except that the
443 * caller manages requesting and releasing the DMA channel handle.
444 */
445int devm_iio_dmaengine_buffer_setup_with_handle(struct device *dev,
446 struct iio_dev *indio_dev,
447 struct dma_chan *chan,
448 enum iio_buffer_direction dir)
449{
450 struct iio_buffer *buffer;
451
452 buffer = __iio_dmaengine_buffer_setup_ext(indio_dev, chan, dir);
453 if (IS_ERR(buffer))
454 return PTR_ERR(buffer);
455
456 return devm_add_action_or_reset(dev, devm_iio_dmaengine_buffer_free,
457 buffer);
458}
459EXPORT_SYMBOL_NS_GPL(devm_iio_dmaengine_buffer_setup_with_handle,
460 "IIO_DMAENGINE_BUFFER");
461
462MODULE_AUTHOR("Lars-Peter Clausen <[email protected]>");
463MODULE_DESCRIPTION("DMA buffer for the IIO framework");
464MODULE_LICENSE("GPL");
465MODULE_IMPORT_NS("IIO_DMA_BUFFER");