| 1 | |
| 2 | |
| 3 | * Freescale MPC85xx, MPC83xx DMA Engine support
|
| 4 | *
|
| 5 | * Copyright (C) 2007-2010 Freescale Semiconductor, Inc. All rights reserved.
|
| 6 | *
|
| 7 | * Author:
|
| 8 | * Zhang Wei <[email protected]>, Jul 2007
|
| 9 | * Ebony Zhu <[email protected]>, May 2007
|
| 10 | *
|
| 11 | * Description:
|
| 12 | * DMA engine driver for Freescale MPC8540 DMA controller, which is
|
| 13 | * also fit for MPC8560, MPC8555, MPC8548, MPC8641, and etc.
|
| 14 | * The support for MPC8349 DMA controller is also added.
|
| 15 | *
|
| 16 | * This driver instructs the DMA controller to issue the PCI Read Multiple
|
| 17 | * command for PCI read operations, instead of using the default PCI Read Line
|
| 18 | * command. Please be aware that this setting may result in read pre-fetching
|
| 19 | * on some platforms.
|
| 20 |
|
| 21 |
|
| 22 | #include <linux/init.h>
|
| 23 | #include <linux/module.h>
|
| 24 | #include <linux/pci.h>
|
| 25 | #include <linux/slab.h>
|
| 26 | #include <linux/interrupt.h>
|
| 27 | #include <linux/dmaengine.h>
|
| 28 | #include <linux/delay.h>
|
| 29 | #include <linux/dma-mapping.h>
|
| 30 | #include <linux/dmapool.h>
|
| 31 | #include <linux/of.h>
|
| 32 | #include <linux/of_address.h>
|
| 33 | #include <linux/of_irq.h>
|
| 34 | #include <linux/platform_device.h>
|
| 35 | #include <linux/fsldma.h>
|
| 36 | #include "dmaengine.h"
|
| 37 | #include "fsldma.h"
|
| 38 |
|
| 39 | #define chan_dbg(chan, fmt, arg...) \
|
| 40 | dev_dbg(chan->dev, "%s: " fmt, chan->name, ##arg)
|
| 41 | #define chan_err(chan, fmt, arg...) \
|
| 42 | dev_err(chan->dev, "%s: " fmt, chan->name, ##arg)
|
| 43 |
|
| 44 | static const char msg_ld_oom[] = "No free memory for link descriptor";
|
| 45 |
|
| 46 | |
| 47 | * Register Helpers
|
| 48 |
|
| 49 |
|
| 50 | static void set_sr(struct fsldma_chan *chan, u32 val)
|
| 51 | {
|
| 52 | FSL_DMA_OUT(chan, &chan->regs->sr, val, 32);
|
| 53 | }
|
| 54 |
|
| 55 | static u32 get_sr(struct fsldma_chan *chan)
|
| 56 | {
|
| 57 | return FSL_DMA_IN(chan, &chan->regs->sr, 32);
|
| 58 | }
|
| 59 |
|
| 60 | static void set_mr(struct fsldma_chan *chan, u32 val)
|
| 61 | {
|
| 62 | FSL_DMA_OUT(chan, &chan->regs->mr, val, 32);
|
| 63 | }
|
| 64 |
|
| 65 | static u32 get_mr(struct fsldma_chan *chan)
|
| 66 | {
|
| 67 | return FSL_DMA_IN(chan, &chan->regs->mr, 32);
|
| 68 | }
|
| 69 |
|
| 70 | static void set_cdar(struct fsldma_chan *chan, dma_addr_t addr)
|
| 71 | {
|
| 72 | FSL_DMA_OUT(chan, &chan->regs->cdar, addr | FSL_DMA_SNEN, 64);
|
| 73 | }
|
| 74 |
|
| 75 | static dma_addr_t get_cdar(struct fsldma_chan *chan)
|
| 76 | {
|
| 77 | return FSL_DMA_IN(chan, &chan->regs->cdar, 64) & ~FSL_DMA_SNEN;
|
| 78 | }
|
| 79 |
|
| 80 | static void set_bcr(struct fsldma_chan *chan, u32 val)
|
| 81 | {
|
| 82 | FSL_DMA_OUT(chan, &chan->regs->bcr, val, 32);
|
| 83 | }
|
| 84 |
|
| 85 | static u32 get_bcr(struct fsldma_chan *chan)
|
| 86 | {
|
| 87 | return FSL_DMA_IN(chan, &chan->regs->bcr, 32);
|
| 88 | }
|
| 89 |
|
| 90 | |
| 91 | * Descriptor Helpers
|
| 92 |
|
| 93 |
|
| 94 | static void set_desc_cnt(struct fsldma_chan *chan,
|
| 95 | struct fsl_dma_ld_hw *hw, u32 count)
|
| 96 | {
|
| 97 | hw->count = CPU_TO_DMA(chan, count, 32);
|
| 98 | }
|
| 99 |
|
| 100 | static void set_desc_src(struct fsldma_chan *chan,
|
| 101 | struct fsl_dma_ld_hw *hw, dma_addr_t src)
|
| 102 | {
|
| 103 | u64 snoop_bits;
|
| 104 |
|
| 105 | snoop_bits = ((chan->feature & FSL_DMA_IP_MASK) == FSL_DMA_IP_85XX)
|
| 106 | ? ((u64)FSL_DMA_SATR_SREADTYPE_SNOOP_READ << 32) : 0;
|
| 107 | hw->src_addr = CPU_TO_DMA(chan, snoop_bits | src, 64);
|
| 108 | }
|
| 109 |
|
| 110 | static void set_desc_dst(struct fsldma_chan *chan,
|
| 111 | struct fsl_dma_ld_hw *hw, dma_addr_t dst)
|
| 112 | {
|
| 113 | u64 snoop_bits;
|
| 114 |
|
| 115 | snoop_bits = ((chan->feature & FSL_DMA_IP_MASK) == FSL_DMA_IP_85XX)
|
| 116 | ? ((u64)FSL_DMA_DATR_DWRITETYPE_SNOOP_WRITE << 32) : 0;
|
| 117 | hw->dst_addr = CPU_TO_DMA(chan, snoop_bits | dst, 64);
|
| 118 | }
|
| 119 |
|
| 120 | static void set_desc_next(struct fsldma_chan *chan,
|
| 121 | struct fsl_dma_ld_hw *hw, dma_addr_t next)
|
| 122 | {
|
| 123 | u64 snoop_bits;
|
| 124 |
|
| 125 | snoop_bits = ((chan->feature & FSL_DMA_IP_MASK) == FSL_DMA_IP_83XX)
|
| 126 | ? FSL_DMA_SNEN : 0;
|
| 127 | hw->next_ln_addr = CPU_TO_DMA(chan, snoop_bits | next, 64);
|
| 128 | }
|
| 129 |
|
| 130 | static void set_ld_eol(struct fsldma_chan *chan, struct fsl_desc_sw *desc)
|
| 131 | {
|
| 132 | u64 snoop_bits;
|
| 133 |
|
| 134 | snoop_bits = ((chan->feature & FSL_DMA_IP_MASK) == FSL_DMA_IP_83XX)
|
| 135 | ? FSL_DMA_SNEN : 0;
|
| 136 |
|
| 137 | desc->hw.next_ln_addr = CPU_TO_DMA(chan,
|
| 138 | DMA_TO_CPU(chan, desc->hw.next_ln_addr, 64) | FSL_DMA_EOL
|
| 139 | | snoop_bits, 64);
|
| 140 | }
|
| 141 |
|
| 142 | |
| 143 | * DMA Engine Hardware Control Helpers
|
| 144 |
|
| 145 |
|
| 146 | static void dma_init(struct fsldma_chan *chan)
|
| 147 | {
|
| 148 |
|
| 149 | set_mr(chan, 0);
|
| 150 |
|
| 151 | switch (chan->feature & FSL_DMA_IP_MASK) {
|
| 152 | case FSL_DMA_IP_85XX:
|
| 153 | |
| 154 | * EIE - Error interrupt enable
|
| 155 | * EOLNIE - End of links interrupt enable
|
| 156 | * BWC - Bandwidth sharing among channels
|
| 157 |
|
| 158 | set_mr(chan, FSL_DMA_MR_BWC | FSL_DMA_MR_EIE
|
| 159 | | FSL_DMA_MR_EOLNIE);
|
| 160 | break;
|
| 161 | case FSL_DMA_IP_83XX:
|
| 162 | |
| 163 | * EOTIE - End-of-transfer interrupt enable
|
| 164 | * PRC_RM - PCI read multiple
|
| 165 |
|
| 166 | set_mr(chan, FSL_DMA_MR_EOTIE | FSL_DMA_MR_PRC_RM);
|
| 167 | break;
|
| 168 | }
|
| 169 | }
|
| 170 |
|
| 171 | static int dma_is_idle(struct fsldma_chan *chan)
|
| 172 | {
|
| 173 | u32 sr = get_sr(chan);
|
| 174 | return (!(sr & FSL_DMA_SR_CB)) || (sr & FSL_DMA_SR_CH);
|
| 175 | }
|
| 176 |
|
| 177 | |
| 178 | * Start the DMA controller
|
| 179 | *
|
| 180 | * Preconditions:
|
| 181 | * - the CDAR register must point to the start descriptor
|
| 182 | * - the MRn[CS] bit must be cleared
|
| 183 |
|
| 184 | static void dma_start(struct fsldma_chan *chan)
|
| 185 | {
|
| 186 | u32 mode;
|
| 187 |
|
| 188 | mode = get_mr(chan);
|
| 189 |
|
| 190 | if (chan->feature & FSL_DMA_CHAN_PAUSE_EXT) {
|
| 191 | set_bcr(chan, 0);
|
| 192 | mode |= FSL_DMA_MR_EMP_EN;
|
| 193 | } else {
|
| 194 | mode &= ~FSL_DMA_MR_EMP_EN;
|
| 195 | }
|
| 196 |
|
| 197 | if (chan->feature & FSL_DMA_CHAN_START_EXT) {
|
| 198 | mode |= FSL_DMA_MR_EMS_EN;
|
| 199 | } else {
|
| 200 | mode &= ~FSL_DMA_MR_EMS_EN;
|
| 201 | mode |= FSL_DMA_MR_CS;
|
| 202 | }
|
| 203 |
|
| 204 | set_mr(chan, mode);
|
| 205 | }
|
| 206 |
|
| 207 | static void dma_halt(struct fsldma_chan *chan)
|
| 208 | {
|
| 209 | u32 mode;
|
| 210 | int i;
|
| 211 |
|
| 212 |
|
| 213 | mode = get_mr(chan);
|
| 214 |
|
| 215 | |
| 216 | * The 85xx controller supports channel abort, which will stop
|
| 217 | * the current transfer. On 83xx, this bit is the transfer error
|
| 218 | * mask bit, which should not be changed.
|
| 219 |
|
| 220 | if ((chan->feature & FSL_DMA_IP_MASK) == FSL_DMA_IP_85XX) {
|
| 221 | mode |= FSL_DMA_MR_CA;
|
| 222 | set_mr(chan, mode);
|
| 223 |
|
| 224 | mode &= ~FSL_DMA_MR_CA;
|
| 225 | }
|
| 226 |
|
| 227 |
|
| 228 | mode &= ~(FSL_DMA_MR_CS | FSL_DMA_MR_EMS_EN);
|
| 229 | set_mr(chan, mode);
|
| 230 |
|
| 231 |
|
| 232 | for (i = 0; i < 100; i++) {
|
| 233 | if (dma_is_idle(chan))
|
| 234 | return;
|
| 235 |
|
| 236 | udelay(10);
|
| 237 | }
|
| 238 |
|
| 239 | if (!dma_is_idle(chan))
|
| 240 | chan_err(chan, "DMA halt timeout!\n");
|
| 241 | }
|
| 242 |
|
| 243 | |
| 244 | * fsl_chan_set_src_loop_size - Set source address hold transfer size
|
| 245 | * @chan : Freescale DMA channel
|
| 246 | * @size : Address loop size, 0 for disable loop
|
| 247 | *
|
| 248 | * The set source address hold transfer size. The source
|
| 249 | * address hold or loop transfer size is when the DMA transfer
|
| 250 | * data from source address (SA), if the loop size is 4, the DMA will
|
| 251 | * read data from SA, SA + 1, SA + 2, SA + 3, then loop back to SA,
|
| 252 | * SA + 1 ... and so on.
|
| 253 |
|
| 254 | static void fsl_chan_set_src_loop_size(struct fsldma_chan *chan, int size)
|
| 255 | {
|
| 256 | u32 mode;
|
| 257 |
|
| 258 | mode = get_mr(chan);
|
| 259 |
|
| 260 | switch (size) {
|
| 261 | case 0:
|
| 262 | mode &= ~FSL_DMA_MR_SAHE;
|
| 263 | break;
|
| 264 | case 1:
|
| 265 | case 2:
|
| 266 | case 4:
|
| 267 | case 8:
|
| 268 | mode &= ~FSL_DMA_MR_SAHTS_MASK;
|
| 269 | mode |= FSL_DMA_MR_SAHE | (__ilog2(size) << 14);
|
| 270 | break;
|
| 271 | }
|
| 272 |
|
| 273 | set_mr(chan, mode);
|
| 274 | }
|
| 275 |
|
| 276 | |
| 277 | * fsl_chan_set_dst_loop_size - Set destination address hold transfer size
|
| 278 | * @chan : Freescale DMA channel
|
| 279 | * @size : Address loop size, 0 for disable loop
|
| 280 | *
|
| 281 | * The set destination address hold transfer size. The destination
|
| 282 | * address hold or loop transfer size is when the DMA transfer
|
| 283 | * data to destination address (TA), if the loop size is 4, the DMA will
|
| 284 | * write data to TA, TA + 1, TA + 2, TA + 3, then loop back to TA,
|
| 285 | * TA + 1 ... and so on.
|
| 286 |
|
| 287 | static void fsl_chan_set_dst_loop_size(struct fsldma_chan *chan, int size)
|
| 288 | {
|
| 289 | u32 mode;
|
| 290 |
|
| 291 | mode = get_mr(chan);
|
| 292 |
|
| 293 | switch (size) {
|
| 294 | case 0:
|
| 295 | mode &= ~FSL_DMA_MR_DAHE;
|
| 296 | break;
|
| 297 | case 1:
|
| 298 | case 2:
|
| 299 | case 4:
|
| 300 | case 8:
|
| 301 | mode &= ~FSL_DMA_MR_DAHTS_MASK;
|
| 302 | mode |= FSL_DMA_MR_DAHE | (__ilog2(size) << 16);
|
| 303 | break;
|
| 304 | }
|
| 305 |
|
| 306 | set_mr(chan, mode);
|
| 307 | }
|
| 308 |
|
| 309 | |
| 310 | * fsl_chan_set_request_count - Set DMA Request Count for external control
|
| 311 | * @chan : Freescale DMA channel
|
| 312 | * @size : Number of bytes to transfer in a single request
|
| 313 | *
|
| 314 | * The Freescale DMA channel can be controlled by the external signal DREQ#.
|
| 315 | * The DMA request count is how many bytes are allowed to transfer before
|
| 316 | * pausing the channel, after which a new assertion of DREQ# resumes channel
|
| 317 | * operation.
|
| 318 | *
|
| 319 | * A size of 0 disables external pause control. The maximum size is 1024.
|
| 320 |
|
| 321 | static void fsl_chan_set_request_count(struct fsldma_chan *chan, int size)
|
| 322 | {
|
| 323 | u32 mode;
|
| 324 |
|
| 325 | BUG_ON(size > 1024);
|
| 326 |
|
| 327 | mode = get_mr(chan);
|
| 328 | mode &= ~FSL_DMA_MR_BWC_MASK;
|
| 329 | mode |= (__ilog2(size) << 24) & FSL_DMA_MR_BWC_MASK;
|
| 330 |
|
| 331 | set_mr(chan, mode);
|
| 332 | }
|
| 333 |
|
| 334 | |
| 335 | * fsl_chan_toggle_ext_pause - Toggle channel external pause status
|
| 336 | * @chan : Freescale DMA channel
|
| 337 | * @enable : 0 is disabled, 1 is enabled.
|
| 338 | *
|
| 339 | * The Freescale DMA channel can be controlled by the external signal DREQ#.
|
| 340 | * The DMA Request Count feature should be used in addition to this feature
|
| 341 | * to set the number of bytes to transfer before pausing the channel.
|
| 342 |
|
| 343 | static void fsl_chan_toggle_ext_pause(struct fsldma_chan *chan, int enable)
|
| 344 | {
|
| 345 | if (enable)
|
| 346 | chan->feature |= FSL_DMA_CHAN_PAUSE_EXT;
|
| 347 | else
|
| 348 | chan->feature &= ~FSL_DMA_CHAN_PAUSE_EXT;
|
| 349 | }
|
| 350 |
|
| 351 | |
| 352 | * fsl_chan_toggle_ext_start - Toggle channel external start status
|
| 353 | * @chan : Freescale DMA channel
|
| 354 | * @enable : 0 is disabled, 1 is enabled.
|
| 355 | *
|
| 356 | * If enable the external start, the channel can be started by an
|
| 357 | * external DMA start pin. So the dma_start() does not start the
|
| 358 | * transfer immediately. The DMA channel will wait for the
|
| 359 | * control pin asserted.
|
| 360 |
|
| 361 | static void fsl_chan_toggle_ext_start(struct fsldma_chan *chan, int enable)
|
| 362 | {
|
| 363 | if (enable)
|
| 364 | chan->feature |= FSL_DMA_CHAN_START_EXT;
|
| 365 | else
|
| 366 | chan->feature &= ~FSL_DMA_CHAN_START_EXT;
|
| 367 | }
|
| 368 |
|
| 369 | int fsl_dma_external_start(struct dma_chan *dchan, int enable)
|
| 370 | {
|
| 371 | struct fsldma_chan *chan;
|
| 372 |
|
| 373 | if (!dchan)
|
| 374 | return -EINVAL;
|
| 375 |
|
| 376 | chan = to_fsl_chan(dchan);
|
| 377 |
|
| 378 | fsl_chan_toggle_ext_start(chan, enable);
|
| 379 | return 0;
|
| 380 | }
|
| 381 | EXPORT_SYMBOL_GPL(fsl_dma_external_start);
|
| 382 |
|
| 383 | static void append_ld_queue(struct fsldma_chan *chan, struct fsl_desc_sw *desc)
|
| 384 | {
|
| 385 | struct fsl_desc_sw *tail = to_fsl_desc(chan->ld_pending.prev);
|
| 386 |
|
| 387 | if (list_empty(&chan->ld_pending))
|
| 388 | goto out_splice;
|
| 389 |
|
| 390 | |
| 391 | * Add the hardware descriptor to the chain of hardware descriptors
|
| 392 | * that already exists in memory.
|
| 393 | *
|
| 394 | * This will un-set the EOL bit of the existing transaction, and the
|
| 395 | * last link in this transaction will become the EOL descriptor.
|
| 396 |
|
| 397 | set_desc_next(chan, &tail->hw, desc->async_tx.phys);
|
| 398 |
|
| 399 | |
| 400 | * Add the software descriptor and all children to the list
|
| 401 | * of pending transactions
|
| 402 |
|
| 403 | out_splice:
|
| 404 | list_splice_tail_init(&desc->tx_list, &chan->ld_pending);
|
| 405 | }
|
| 406 |
|
| 407 | static dma_cookie_t fsl_dma_tx_submit(struct dma_async_tx_descriptor *tx)
|
| 408 | {
|
| 409 | struct fsldma_chan *chan = to_fsl_chan(tx->chan);
|
| 410 | struct fsl_desc_sw *desc = tx_to_fsl_desc(tx);
|
| 411 | struct fsl_desc_sw *child;
|
| 412 | dma_cookie_t cookie = -EINVAL;
|
| 413 |
|
| 414 | spin_lock_bh(&chan->desc_lock);
|
| 415 |
|
| 416 | #ifdef CONFIG_PM
|
| 417 | if (unlikely(chan->pm_state != RUNNING)) {
|
| 418 | chan_dbg(chan, "cannot submit due to suspend\n");
|
| 419 | spin_unlock_bh(&chan->desc_lock);
|
| 420 | return -1;
|
| 421 | }
|
| 422 | #endif
|
| 423 |
|
| 424 | |
| 425 | * assign cookies to all of the software descriptors
|
| 426 | * that make up this transaction
|
| 427 |
|
| 428 | list_for_each_entry(child, &desc->tx_list, node) {
|
| 429 | cookie = dma_cookie_assign(&child->async_tx);
|
| 430 | }
|
| 431 |
|
| 432 |
|
| 433 | append_ld_queue(chan, desc);
|
| 434 |
|
| 435 | spin_unlock_bh(&chan->desc_lock);
|
| 436 |
|
| 437 | return cookie;
|
| 438 | }
|
| 439 |
|
| 440 | |
| 441 | * fsl_dma_free_descriptor - Free descriptor from channel's DMA pool.
|
| 442 | * @chan : Freescale DMA channel
|
| 443 | * @desc: descriptor to be freed
|
| 444 |
|
| 445 | static void fsl_dma_free_descriptor(struct fsldma_chan *chan,
|
| 446 | struct fsl_desc_sw *desc)
|
| 447 | {
|
| 448 | list_del(&desc->node);
|
| 449 | chan_dbg(chan, "LD %p free\n", desc);
|
| 450 | dma_pool_free(chan->desc_pool, desc, desc->async_tx.phys);
|
| 451 | }
|
| 452 |
|
| 453 | |
| 454 | * fsl_dma_alloc_descriptor - Allocate descriptor from channel's DMA pool.
|
| 455 | * @chan : Freescale DMA channel
|
| 456 | *
|
| 457 | * Return - The descriptor allocated. NULL for failed.
|
| 458 |
|
| 459 | static struct fsl_desc_sw *fsl_dma_alloc_descriptor(struct fsldma_chan *chan)
|
| 460 | {
|
| 461 | struct fsl_desc_sw *desc;
|
| 462 | dma_addr_t pdesc;
|
| 463 |
|
| 464 | desc = dma_pool_zalloc(chan->desc_pool, GFP_ATOMIC, &pdesc);
|
| 465 | if (!desc) {
|
| 466 | chan_dbg(chan, "out of memory for link descriptor\n");
|
| 467 | return NULL;
|
| 468 | }
|
| 469 |
|
| 470 | INIT_LIST_HEAD(&desc->tx_list);
|
| 471 | dma_async_tx_descriptor_init(&desc->async_tx, &chan->common);
|
| 472 | desc->async_tx.tx_submit = fsl_dma_tx_submit;
|
| 473 | desc->async_tx.phys = pdesc;
|
| 474 |
|
| 475 | chan_dbg(chan, "LD %p allocated\n", desc);
|
| 476 |
|
| 477 | return desc;
|
| 478 | }
|
| 479 |
|
| 480 | |
| 481 | * fsldma_clean_completed_descriptor - free all descriptors which
|
| 482 | * has been completed and acked
|
| 483 | * @chan: Freescale DMA channel
|
| 484 | *
|
| 485 | * This function is used on all completed and acked descriptors.
|
| 486 | * All descriptors should only be freed in this function.
|
| 487 |
|
| 488 | static void fsldma_clean_completed_descriptor(struct fsldma_chan *chan)
|
| 489 | {
|
| 490 | struct fsl_desc_sw *desc, *_desc;
|
| 491 |
|
| 492 |
|
| 493 | list_for_each_entry_safe(desc, _desc, &chan->ld_completed, node)
|
| 494 | if (async_tx_test_ack(&desc->async_tx))
|
| 495 | fsl_dma_free_descriptor(chan, desc);
|
| 496 | }
|
| 497 |
|
| 498 | |
| 499 | * fsldma_run_tx_complete_actions - cleanup a single link descriptor
|
| 500 | * @chan: Freescale DMA channel
|
| 501 | * @desc: descriptor to cleanup and free
|
| 502 | * @cookie: Freescale DMA transaction identifier
|
| 503 | *
|
| 504 | * This function is used on a descriptor which has been executed by the DMA
|
| 505 | * controller. It will run any callbacks, submit any dependencies.
|
| 506 |
|
| 507 | static dma_cookie_t fsldma_run_tx_complete_actions(struct fsldma_chan *chan,
|
| 508 | struct fsl_desc_sw *desc, dma_cookie_t cookie)
|
| 509 | {
|
| 510 | struct dma_async_tx_descriptor *txd = &desc->async_tx;
|
| 511 | dma_cookie_t ret = cookie;
|
| 512 |
|
| 513 | BUG_ON(txd->cookie < 0);
|
| 514 |
|
| 515 | if (txd->cookie > 0) {
|
| 516 | ret = txd->cookie;
|
| 517 |
|
| 518 | dma_descriptor_unmap(txd);
|
| 519 |
|
| 520 | dmaengine_desc_get_callback_invoke(txd, NULL);
|
| 521 | }
|
| 522 |
|
| 523 |
|
| 524 | dma_run_dependencies(txd);
|
| 525 |
|
| 526 | return ret;
|
| 527 | }
|
| 528 |
|
| 529 | |
| 530 | * fsldma_clean_running_descriptor - move the completed descriptor from
|
| 531 | * ld_running to ld_completed
|
| 532 | * @chan: Freescale DMA channel
|
| 533 | * @desc: the descriptor which is completed
|
| 534 | *
|
| 535 | * Free the descriptor directly if acked by async_tx api, or move it to
|
| 536 | * queue ld_completed.
|
| 537 |
|
| 538 | static void fsldma_clean_running_descriptor(struct fsldma_chan *chan,
|
| 539 | struct fsl_desc_sw *desc)
|
| 540 | {
|
| 541 |
|
| 542 | list_del(&desc->node);
|
| 543 |
|
| 544 | |
| 545 | * the client is allowed to attach dependent operations
|
| 546 | * until 'ack' is set
|
| 547 |
|
| 548 | if (!async_tx_test_ack(&desc->async_tx)) {
|
| 549 | |
| 550 | * Move this descriptor to the list of descriptors which is
|
| 551 | * completed, but still awaiting the 'ack' bit to be set.
|
| 552 |
|
| 553 | list_add_tail(&desc->node, &chan->ld_completed);
|
| 554 | return;
|
| 555 | }
|
| 556 |
|
| 557 | dma_pool_free(chan->desc_pool, desc, desc->async_tx.phys);
|
| 558 | }
|
| 559 |
|
| 560 | |
| 561 | * fsl_chan_xfer_ld_queue - transfer any pending transactions
|
| 562 | * @chan : Freescale DMA channel
|
| 563 | *
|
| 564 | * HARDWARE STATE: idle
|
| 565 | * LOCKING: must hold chan->desc_lock
|
| 566 |
|
| 567 | static void fsl_chan_xfer_ld_queue(struct fsldma_chan *chan)
|
| 568 | {
|
| 569 | struct fsl_desc_sw *desc;
|
| 570 |
|
| 571 | |
| 572 | * If the list of pending descriptors is empty, then we
|
| 573 | * don't need to do any work at all
|
| 574 |
|
| 575 | if (list_empty(&chan->ld_pending)) {
|
| 576 | chan_dbg(chan, "no pending LDs\n");
|
| 577 | return;
|
| 578 | }
|
| 579 |
|
| 580 | |
| 581 | * The DMA controller is not idle, which means that the interrupt
|
| 582 | * handler will start any queued transactions when it runs after
|
| 583 | * this transaction finishes
|
| 584 |
|
| 585 | if (!chan->idle) {
|
| 586 | chan_dbg(chan, "DMA controller still busy\n");
|
| 587 | return;
|
| 588 | }
|
| 589 |
|
| 590 | |
| 591 | * If there are some link descriptors which have not been
|
| 592 | * transferred, we need to start the controller
|
| 593 |
|
| 594 |
|
| 595 | |
| 596 | * Move all elements from the queue of pending transactions
|
| 597 | * onto the list of running transactions
|
| 598 |
|
| 599 | chan_dbg(chan, "idle, starting controller\n");
|
| 600 | desc = list_first_entry(&chan->ld_pending, struct fsl_desc_sw, node);
|
| 601 | list_splice_tail_init(&chan->ld_pending, &chan->ld_running);
|
| 602 |
|
| 603 | |
| 604 | * The 85xx DMA controller doesn't clear the channel start bit
|
| 605 | * automatically at the end of a transfer. Therefore we must clear
|
| 606 | * it in software before starting the transfer.
|
| 607 |
|
| 608 | if ((chan->feature & FSL_DMA_IP_MASK) == FSL_DMA_IP_85XX) {
|
| 609 | u32 mode;
|
| 610 |
|
| 611 | mode = get_mr(chan);
|
| 612 | mode &= ~FSL_DMA_MR_CS;
|
| 613 | set_mr(chan, mode);
|
| 614 | }
|
| 615 |
|
| 616 | |
| 617 | * Program the descriptor's address into the DMA controller,
|
| 618 | * then start the DMA transaction
|
| 619 |
|
| 620 | set_cdar(chan, desc->async_tx.phys);
|
| 621 | get_cdar(chan);
|
| 622 |
|
| 623 | dma_start(chan);
|
| 624 | chan->idle = false;
|
| 625 | }
|
| 626 |
|
| 627 | |
| 628 | * fsldma_cleanup_descriptors - cleanup link descriptors which are completed
|
| 629 | * and move them to ld_completed to free until flag 'ack' is set
|
| 630 | * @chan: Freescale DMA channel
|
| 631 | *
|
| 632 | * This function is used on descriptors which have been executed by the DMA
|
| 633 | * controller. It will run any callbacks, submit any dependencies, then
|
| 634 | * free these descriptors if flag 'ack' is set.
|
| 635 |
|
| 636 | static void fsldma_cleanup_descriptors(struct fsldma_chan *chan)
|
| 637 | {
|
| 638 | struct fsl_desc_sw *desc, *_desc;
|
| 639 | dma_cookie_t cookie = 0;
|
| 640 | dma_addr_t curr_phys = get_cdar(chan);
|
| 641 | int seen_current = 0;
|
| 642 |
|
| 643 | fsldma_clean_completed_descriptor(chan);
|
| 644 |
|
| 645 |
|
| 646 | list_for_each_entry_safe(desc, _desc, &chan->ld_running, node) {
|
| 647 | |
| 648 | * do not advance past the current descriptor loaded into the
|
| 649 | * hardware channel, subsequent descriptors are either in
|
| 650 | * process or have not been submitted
|
| 651 |
|
| 652 | if (seen_current)
|
| 653 | break;
|
| 654 |
|
| 655 | |
| 656 | * stop the search if we reach the current descriptor and the
|
| 657 | * channel is busy
|
| 658 |
|
| 659 | if (desc->async_tx.phys == curr_phys) {
|
| 660 | seen_current = 1;
|
| 661 | if (!dma_is_idle(chan))
|
| 662 | break;
|
| 663 | }
|
| 664 |
|
| 665 | cookie = fsldma_run_tx_complete_actions(chan, desc, cookie);
|
| 666 |
|
| 667 | fsldma_clean_running_descriptor(chan, desc);
|
| 668 | }
|
| 669 |
|
| 670 | |
| 671 | * Start any pending transactions automatically
|
| 672 | *
|
| 673 | * In the ideal case, we keep the DMA controller busy while we go
|
| 674 | * ahead and free the descriptors below.
|
| 675 |
|
| 676 | fsl_chan_xfer_ld_queue(chan);
|
| 677 |
|
| 678 | if (cookie > 0)
|
| 679 | chan->common.completed_cookie = cookie;
|
| 680 | }
|
| 681 |
|
| 682 | |
| 683 | * fsl_dma_alloc_chan_resources - Allocate resources for DMA channel.
|
| 684 | * @chan : Freescale DMA channel
|
| 685 | *
|
| 686 | * This function will create a dma pool for descriptor allocation.
|
| 687 | *
|
| 688 | * Return - The number of descriptors allocated.
|
| 689 |
|
| 690 | static int fsl_dma_alloc_chan_resources(struct dma_chan *dchan)
|
| 691 | {
|
| 692 | struct fsldma_chan *chan = to_fsl_chan(dchan);
|
| 693 |
|
| 694 |
|
| 695 | if (chan->desc_pool)
|
| 696 | return 1;
|
| 697 |
|
| 698 | |
| 699 | * We need the descriptor to be aligned to 32bytes
|
| 700 | * for meeting FSL DMA specification requirement.
|
| 701 |
|
| 702 | chan->desc_pool = dma_pool_create(chan->name, chan->dev,
|
| 703 | sizeof(struct fsl_desc_sw),
|
| 704 | __alignof__(struct fsl_desc_sw), 0);
|
| 705 | if (!chan->desc_pool) {
|
| 706 | chan_err(chan, "unable to allocate descriptor pool\n");
|
| 707 | return -ENOMEM;
|
| 708 | }
|
| 709 |
|
| 710 |
|
| 711 | return 1;
|
| 712 | }
|
| 713 |
|
| 714 | |
| 715 | * fsldma_free_desc_list - Free all descriptors in a queue
|
| 716 | * @chan: Freescae DMA channel
|
| 717 | * @list: the list to free
|
| 718 | *
|
| 719 | * LOCKING: must hold chan->desc_lock
|
| 720 |
|
| 721 | static void fsldma_free_desc_list(struct fsldma_chan *chan,
|
| 722 | struct list_head *list)
|
| 723 | {
|
| 724 | struct fsl_desc_sw *desc, *_desc;
|
| 725 |
|
| 726 | list_for_each_entry_safe(desc, _desc, list, node)
|
| 727 | fsl_dma_free_descriptor(chan, desc);
|
| 728 | }
|
| 729 |
|
| 730 | static void fsldma_free_desc_list_reverse(struct fsldma_chan *chan,
|
| 731 | struct list_head *list)
|
| 732 | {
|
| 733 | struct fsl_desc_sw *desc, *_desc;
|
| 734 |
|
| 735 | list_for_each_entry_safe_reverse(desc, _desc, list, node)
|
| 736 | fsl_dma_free_descriptor(chan, desc);
|
| 737 | }
|
| 738 |
|
| 739 | |
| 740 | * fsl_dma_free_chan_resources - Free all resources of the channel.
|
| 741 | * @chan : Freescale DMA channel
|
| 742 |
|
| 743 | static void fsl_dma_free_chan_resources(struct dma_chan *dchan)
|
| 744 | {
|
| 745 | struct fsldma_chan *chan = to_fsl_chan(dchan);
|
| 746 |
|
| 747 | chan_dbg(chan, "free all channel resources\n");
|
| 748 | spin_lock_bh(&chan->desc_lock);
|
| 749 | fsldma_cleanup_descriptors(chan);
|
| 750 | fsldma_free_desc_list(chan, &chan->ld_pending);
|
| 751 | fsldma_free_desc_list(chan, &chan->ld_running);
|
| 752 | fsldma_free_desc_list(chan, &chan->ld_completed);
|
| 753 | spin_unlock_bh(&chan->desc_lock);
|
| 754 |
|
| 755 | dma_pool_destroy(chan->desc_pool);
|
| 756 | chan->desc_pool = NULL;
|
| 757 | }
|
| 758 |
|
| 759 | static struct dma_async_tx_descriptor *
|
| 760 | fsl_dma_prep_memcpy(struct dma_chan *dchan,
|
| 761 | dma_addr_t dma_dst, dma_addr_t dma_src,
|
| 762 | size_t len, unsigned long flags)
|
| 763 | {
|
| 764 | struct fsldma_chan *chan;
|
| 765 | struct fsl_desc_sw *first = NULL, *prev = NULL, *new;
|
| 766 | size_t copy;
|
| 767 |
|
| 768 | if (!dchan)
|
| 769 | return NULL;
|
| 770 |
|
| 771 | if (!len)
|
| 772 | return NULL;
|
| 773 |
|
| 774 | chan = to_fsl_chan(dchan);
|
| 775 |
|
| 776 | do {
|
| 777 |
|
| 778 |
|
| 779 | new = fsl_dma_alloc_descriptor(chan);
|
| 780 | if (!new) {
|
| 781 | chan_err(chan, "%s\n", msg_ld_oom);
|
| 782 | goto fail;
|
| 783 | }
|
| 784 |
|
| 785 | copy = min(len, (size_t)FSL_DMA_BCR_MAX_CNT);
|
| 786 |
|
| 787 | set_desc_cnt(chan, &new->hw, copy);
|
| 788 | set_desc_src(chan, &new->hw, dma_src);
|
| 789 | set_desc_dst(chan, &new->hw, dma_dst);
|
| 790 |
|
| 791 | if (!first)
|
| 792 | first = new;
|
| 793 | else
|
| 794 | set_desc_next(chan, &prev->hw, new->async_tx.phys);
|
| 795 |
|
| 796 | new->async_tx.cookie = 0;
|
| 797 | async_tx_ack(&new->async_tx);
|
| 798 |
|
| 799 | prev = new;
|
| 800 | len -= copy;
|
| 801 | dma_src += copy;
|
| 802 | dma_dst += copy;
|
| 803 |
|
| 804 |
|
| 805 | list_add_tail(&new->node, &first->tx_list);
|
| 806 | } while (len);
|
| 807 |
|
| 808 | new->async_tx.flags = flags;
|
| 809 | new->async_tx.cookie = -EBUSY;
|
| 810 |
|
| 811 |
|
| 812 | set_ld_eol(chan, new);
|
| 813 |
|
| 814 | return &first->async_tx;
|
| 815 |
|
| 816 | fail:
|
| 817 | if (!first)
|
| 818 | return NULL;
|
| 819 |
|
| 820 | fsldma_free_desc_list_reverse(chan, &first->tx_list);
|
| 821 | return NULL;
|
| 822 | }
|
| 823 |
|
| 824 | static int fsl_dma_device_terminate_all(struct dma_chan *dchan)
|
| 825 | {
|
| 826 | struct fsldma_chan *chan;
|
| 827 |
|
| 828 | if (!dchan)
|
| 829 | return -EINVAL;
|
| 830 |
|
| 831 | chan = to_fsl_chan(dchan);
|
| 832 |
|
| 833 | spin_lock_bh(&chan->desc_lock);
|
| 834 |
|
| 835 |
|
| 836 | dma_halt(chan);
|
| 837 |
|
| 838 |
|
| 839 | fsldma_free_desc_list(chan, &chan->ld_pending);
|
| 840 | fsldma_free_desc_list(chan, &chan->ld_running);
|
| 841 | fsldma_free_desc_list(chan, &chan->ld_completed);
|
| 842 | chan->idle = true;
|
| 843 |
|
| 844 | spin_unlock_bh(&chan->desc_lock);
|
| 845 | return 0;
|
| 846 | }
|
| 847 |
|
| 848 | static int fsl_dma_device_config(struct dma_chan *dchan,
|
| 849 | struct dma_slave_config *config)
|
| 850 | {
|
| 851 | struct fsldma_chan *chan;
|
| 852 | int size;
|
| 853 |
|
| 854 | if (!dchan)
|
| 855 | return -EINVAL;
|
| 856 |
|
| 857 | chan = to_fsl_chan(dchan);
|
| 858 |
|
| 859 |
|
| 860 | if (!chan->set_request_count)
|
| 861 | return -ENXIO;
|
| 862 |
|
| 863 |
|
| 864 | if (config->direction == DMA_MEM_TO_DEV)
|
| 865 | size = config->dst_addr_width * config->dst_maxburst;
|
| 866 | else
|
| 867 | size = config->src_addr_width * config->src_maxburst;
|
| 868 |
|
| 869 | chan->set_request_count(chan, size);
|
| 870 | return 0;
|
| 871 | }
|
| 872 |
|
| 873 |
|
| 874 | |
| 875 | * fsl_dma_memcpy_issue_pending - Issue the DMA start command
|
| 876 | * @chan : Freescale DMA channel
|
| 877 |
|
| 878 | static void fsl_dma_memcpy_issue_pending(struct dma_chan *dchan)
|
| 879 | {
|
| 880 | struct fsldma_chan *chan = to_fsl_chan(dchan);
|
| 881 |
|
| 882 | spin_lock_bh(&chan->desc_lock);
|
| 883 | fsl_chan_xfer_ld_queue(chan);
|
| 884 | spin_unlock_bh(&chan->desc_lock);
|
| 885 | }
|
| 886 |
|
| 887 | |
| 888 | * fsl_tx_status - Determine the DMA status
|
| 889 | * @chan : Freescale DMA channel
|
| 890 |
|
| 891 | static enum dma_status fsl_tx_status(struct dma_chan *dchan,
|
| 892 | dma_cookie_t cookie,
|
| 893 | struct dma_tx_state *txstate)
|
| 894 | {
|
| 895 | struct fsldma_chan *chan = to_fsl_chan(dchan);
|
| 896 | enum dma_status ret;
|
| 897 |
|
| 898 | ret = dma_cookie_status(dchan, cookie, txstate);
|
| 899 | if (ret == DMA_COMPLETE)
|
| 900 | return ret;
|
| 901 |
|
| 902 | spin_lock_bh(&chan->desc_lock);
|
| 903 | fsldma_cleanup_descriptors(chan);
|
| 904 | spin_unlock_bh(&chan->desc_lock);
|
| 905 |
|
| 906 | return dma_cookie_status(dchan, cookie, txstate);
|
| 907 | }
|
| 908 |
|
| 909 |
|
| 910 |
|
| 911 |
|
| 912 |
|
| 913 | static irqreturn_t fsldma_chan_irq(int irq, void *data)
|
| 914 | {
|
| 915 | struct fsldma_chan *chan = data;
|
| 916 | u32 stat;
|
| 917 |
|
| 918 |
|
| 919 | stat = get_sr(chan);
|
| 920 | set_sr(chan, stat);
|
| 921 | chan_dbg(chan, "irq: stat = 0x%x\n", stat);
|
| 922 |
|
| 923 |
|
| 924 | stat &= ~(FSL_DMA_SR_CB | FSL_DMA_SR_CH);
|
| 925 | if (!stat)
|
| 926 | return IRQ_NONE;
|
| 927 |
|
| 928 | if (stat & FSL_DMA_SR_TE)
|
| 929 | chan_err(chan, "Transfer Error!\n");
|
| 930 |
|
| 931 | |
| 932 | * Programming Error
|
| 933 | * The DMA_INTERRUPT async_tx is a NULL transfer, which will
|
| 934 | * trigger a PE interrupt.
|
| 935 |
|
| 936 | if (stat & FSL_DMA_SR_PE) {
|
| 937 | chan_dbg(chan, "irq: Programming Error INT\n");
|
| 938 | stat &= ~FSL_DMA_SR_PE;
|
| 939 | if (get_bcr(chan) != 0)
|
| 940 | chan_err(chan, "Programming Error!\n");
|
| 941 | }
|
| 942 |
|
| 943 | |
| 944 | * For MPC8349, EOCDI event need to update cookie
|
| 945 | * and start the next transfer if it exist.
|
| 946 |
|
| 947 | if (stat & FSL_DMA_SR_EOCDI) {
|
| 948 | chan_dbg(chan, "irq: End-of-Chain link INT\n");
|
| 949 | stat &= ~FSL_DMA_SR_EOCDI;
|
| 950 | }
|
| 951 |
|
| 952 | |
| 953 | * If it current transfer is the end-of-transfer,
|
| 954 | * we should clear the Channel Start bit for
|
| 955 | * prepare next transfer.
|
| 956 |
|
| 957 | if (stat & FSL_DMA_SR_EOLNI) {
|
| 958 | chan_dbg(chan, "irq: End-of-link INT\n");
|
| 959 | stat &= ~FSL_DMA_SR_EOLNI;
|
| 960 | }
|
| 961 |
|
| 962 |
|
| 963 | if (!dma_is_idle(chan))
|
| 964 | chan_err(chan, "irq: controller not idle!\n");
|
| 965 |
|
| 966 |
|
| 967 | if (stat)
|
| 968 | chan_err(chan, "irq: unhandled sr 0x%08x\n", stat);
|
| 969 |
|
| 970 | |
| 971 | * Schedule the tasklet to handle all cleanup of the current
|
| 972 | * transaction. It will start a new transaction if there is
|
| 973 | * one pending.
|
| 974 |
|
| 975 | tasklet_schedule(&chan->tasklet);
|
| 976 | chan_dbg(chan, "irq: Exit\n");
|
| 977 | return IRQ_HANDLED;
|
| 978 | }
|
| 979 |
|
| 980 | static void dma_do_tasklet(struct tasklet_struct *t)
|
| 981 | {
|
| 982 | struct fsldma_chan *chan = from_tasklet(chan, t, tasklet);
|
| 983 |
|
| 984 | chan_dbg(chan, "tasklet entry\n");
|
| 985 |
|
| 986 | spin_lock(&chan->desc_lock);
|
| 987 |
|
| 988 |
|
| 989 | chan->idle = true;
|
| 990 |
|
| 991 |
|
| 992 | fsldma_cleanup_descriptors(chan);
|
| 993 |
|
| 994 | spin_unlock(&chan->desc_lock);
|
| 995 |
|
| 996 | chan_dbg(chan, "tasklet exit\n");
|
| 997 | }
|
| 998 |
|
| 999 | static irqreturn_t fsldma_ctrl_irq(int irq, void *data)
|
| 1000 | {
|
| 1001 | struct fsldma_device *fdev = data;
|
| 1002 | struct fsldma_chan *chan;
|
| 1003 | unsigned int handled = 0;
|
| 1004 | u32 gsr, mask;
|
| 1005 | int i;
|
| 1006 |
|
| 1007 | gsr = (fdev->feature & FSL_DMA_BIG_ENDIAN) ? in_be32(fdev->regs)
|
| 1008 | : in_le32(fdev->regs);
|
| 1009 | mask = 0xff000000;
|
| 1010 | dev_dbg(fdev->dev, "IRQ: gsr 0x%.8x\n", gsr);
|
| 1011 |
|
| 1012 | for (i = 0; i < FSL_DMA_MAX_CHANS_PER_DEVICE; i++) {
|
| 1013 | chan = fdev->chan[i];
|
| 1014 | if (!chan)
|
| 1015 | continue;
|
| 1016 |
|
| 1017 | if (gsr & mask) {
|
| 1018 | dev_dbg(fdev->dev, "IRQ: chan %d\n", chan->id);
|
| 1019 | fsldma_chan_irq(irq, chan);
|
| 1020 | handled++;
|
| 1021 | }
|
| 1022 |
|
| 1023 | gsr &= ~mask;
|
| 1024 | mask >>= 8;
|
| 1025 | }
|
| 1026 |
|
| 1027 | return IRQ_RETVAL(handled);
|
| 1028 | }
|
| 1029 |
|
| 1030 | static void fsldma_free_irqs(struct fsldma_device *fdev)
|
| 1031 | {
|
| 1032 | struct fsldma_chan *chan;
|
| 1033 | int i;
|
| 1034 |
|
| 1035 | if (fdev->irq) {
|
| 1036 | dev_dbg(fdev->dev, "free per-controller IRQ\n");
|
| 1037 | free_irq(fdev->irq, fdev);
|
| 1038 | return;
|
| 1039 | }
|
| 1040 |
|
| 1041 | for (i = 0; i < FSL_DMA_MAX_CHANS_PER_DEVICE; i++) {
|
| 1042 | chan = fdev->chan[i];
|
| 1043 | if (chan && chan->irq) {
|
| 1044 | chan_dbg(chan, "free per-channel IRQ\n");
|
| 1045 | free_irq(chan->irq, chan);
|
| 1046 | }
|
| 1047 | }
|
| 1048 | }
|
| 1049 |
|
| 1050 | static int fsldma_request_irqs(struct fsldma_device *fdev)
|
| 1051 | {
|
| 1052 | struct fsldma_chan *chan;
|
| 1053 | int ret;
|
| 1054 | int i;
|
| 1055 |
|
| 1056 |
|
| 1057 | if (fdev->irq) {
|
| 1058 | dev_dbg(fdev->dev, "request per-controller IRQ\n");
|
| 1059 | ret = request_irq(fdev->irq, fsldma_ctrl_irq, IRQF_SHARED,
|
| 1060 | "fsldma-controller", fdev);
|
| 1061 | return ret;
|
| 1062 | }
|
| 1063 |
|
| 1064 |
|
| 1065 | for (i = 0; i < FSL_DMA_MAX_CHANS_PER_DEVICE; i++) {
|
| 1066 | chan = fdev->chan[i];
|
| 1067 | if (!chan)
|
| 1068 | continue;
|
| 1069 |
|
| 1070 | if (!chan->irq) {
|
| 1071 | chan_err(chan, "interrupts property missing in device tree\n");
|
| 1072 | ret = -ENODEV;
|
| 1073 | goto out_unwind;
|
| 1074 | }
|
| 1075 |
|
| 1076 | chan_dbg(chan, "request per-channel IRQ\n");
|
| 1077 | ret = request_irq(chan->irq, fsldma_chan_irq, IRQF_SHARED,
|
| 1078 | "fsldma-chan", chan);
|
| 1079 | if (ret) {
|
| 1080 | chan_err(chan, "unable to request per-channel IRQ\n");
|
| 1081 | goto out_unwind;
|
| 1082 | }
|
| 1083 | }
|
| 1084 |
|
| 1085 | return 0;
|
| 1086 |
|
| 1087 | out_unwind:
|
| 1088 | for (; i >= 0; i--) {
|
| 1089 | chan = fdev->chan[i];
|
| 1090 | if (!chan)
|
| 1091 | continue;
|
| 1092 |
|
| 1093 | if (!chan->irq)
|
| 1094 | continue;
|
| 1095 |
|
| 1096 | free_irq(chan->irq, chan);
|
| 1097 | }
|
| 1098 |
|
| 1099 | return ret;
|
| 1100 | }
|
| 1101 |
|
| 1102 |
|
| 1103 |
|
| 1104 |
|
| 1105 |
|
| 1106 | static int fsl_dma_chan_probe(struct fsldma_device *fdev,
|
| 1107 | struct device_node *node, u32 feature, const char *compatible)
|
| 1108 | {
|
| 1109 | struct fsldma_chan *chan;
|
| 1110 | struct resource res;
|
| 1111 | int err;
|
| 1112 |
|
| 1113 |
|
| 1114 | chan = kzalloc_obj(*chan);
|
| 1115 | if (!chan) {
|
| 1116 | err = -ENOMEM;
|
| 1117 | goto out_return;
|
| 1118 | }
|
| 1119 |
|
| 1120 |
|
| 1121 | chan->regs = of_iomap(node, 0);
|
| 1122 | if (!chan->regs) {
|
| 1123 | dev_err(fdev->dev, "unable to ioremap registers\n");
|
| 1124 | err = -ENOMEM;
|
| 1125 | goto out_free_chan;
|
| 1126 | }
|
| 1127 |
|
| 1128 | err = of_address_to_resource(node, 0, &res);
|
| 1129 | if (err) {
|
| 1130 | dev_err(fdev->dev, "unable to find 'reg' property\n");
|
| 1131 | goto out_iounmap_regs;
|
| 1132 | }
|
| 1133 |
|
| 1134 | chan->feature = feature;
|
| 1135 | if (!fdev->feature)
|
| 1136 | fdev->feature = chan->feature;
|
| 1137 |
|
| 1138 | |
| 1139 | * If the DMA device's feature is different than the feature
|
| 1140 | * of its channels, report the bug
|
| 1141 |
|
| 1142 | WARN_ON(fdev->feature != chan->feature);
|
| 1143 |
|
| 1144 | chan->dev = fdev->dev;
|
| 1145 | chan->id = (res.start & 0xfff) < 0x300 ?
|
| 1146 | ((res.start - 0x100) & 0xfff) >> 7 :
|
| 1147 | ((res.start - 0x200) & 0xfff) >> 7;
|
| 1148 | if (chan->id >= FSL_DMA_MAX_CHANS_PER_DEVICE) {
|
| 1149 | dev_err(fdev->dev, "too many channels for device\n");
|
| 1150 | err = -EINVAL;
|
| 1151 | goto out_iounmap_regs;
|
| 1152 | }
|
| 1153 |
|
| 1154 | fdev->chan[chan->id] = chan;
|
| 1155 | tasklet_setup(&chan->tasklet, dma_do_tasklet);
|
| 1156 | snprintf(chan->name, sizeof(chan->name), "chan%d", chan->id);
|
| 1157 |
|
| 1158 |
|
| 1159 | dma_init(chan);
|
| 1160 |
|
| 1161 |
|
| 1162 | set_cdar(chan, 0);
|
| 1163 |
|
| 1164 | switch (chan->feature & FSL_DMA_IP_MASK) {
|
| 1165 | case FSL_DMA_IP_85XX:
|
| 1166 | chan->toggle_ext_pause = fsl_chan_toggle_ext_pause;
|
| 1167 | fallthrough;
|
| 1168 | case FSL_DMA_IP_83XX:
|
| 1169 | chan->toggle_ext_start = fsl_chan_toggle_ext_start;
|
| 1170 | chan->set_src_loop_size = fsl_chan_set_src_loop_size;
|
| 1171 | chan->set_dst_loop_size = fsl_chan_set_dst_loop_size;
|
| 1172 | chan->set_request_count = fsl_chan_set_request_count;
|
| 1173 | }
|
| 1174 |
|
| 1175 | spin_lock_init(&chan->desc_lock);
|
| 1176 | INIT_LIST_HEAD(&chan->ld_pending);
|
| 1177 | INIT_LIST_HEAD(&chan->ld_running);
|
| 1178 | INIT_LIST_HEAD(&chan->ld_completed);
|
| 1179 | chan->idle = true;
|
| 1180 | #ifdef CONFIG_PM
|
| 1181 | chan->pm_state = RUNNING;
|
| 1182 | #endif
|
| 1183 |
|
| 1184 | chan->common.device = &fdev->common;
|
| 1185 | dma_cookie_init(&chan->common);
|
| 1186 |
|
| 1187 |
|
| 1188 | chan->irq = irq_of_parse_and_map(node, 0);
|
| 1189 |
|
| 1190 |
|
| 1191 | list_add_tail(&chan->common.device_node, &fdev->common.channels);
|
| 1192 |
|
| 1193 | dev_info(fdev->dev, "#%d (%s), irq %d\n", chan->id, compatible,
|
| 1194 | chan->irq ? chan->irq : fdev->irq);
|
| 1195 |
|
| 1196 | return 0;
|
| 1197 |
|
| 1198 | out_iounmap_regs:
|
| 1199 | iounmap(chan->regs);
|
| 1200 | out_free_chan:
|
| 1201 | kfree(chan);
|
| 1202 | out_return:
|
| 1203 | return err;
|
| 1204 | }
|
| 1205 |
|
| 1206 | static void fsl_dma_chan_remove(struct fsldma_chan *chan)
|
| 1207 | {
|
| 1208 | irq_dispose_mapping(chan->irq);
|
| 1209 | list_del(&chan->common.device_node);
|
| 1210 | iounmap(chan->regs);
|
| 1211 | kfree(chan);
|
| 1212 | }
|
| 1213 |
|
| 1214 | static int fsldma_of_probe(struct platform_device *op)
|
| 1215 | {
|
| 1216 | struct fsldma_device *fdev;
|
| 1217 | struct device_node *child;
|
| 1218 | unsigned int i;
|
| 1219 | int err;
|
| 1220 |
|
| 1221 | fdev = kzalloc_obj(*fdev);
|
| 1222 | if (!fdev) {
|
| 1223 | err = -ENOMEM;
|
| 1224 | goto out_return;
|
| 1225 | }
|
| 1226 |
|
| 1227 | fdev->dev = &op->dev;
|
| 1228 | INIT_LIST_HEAD(&fdev->common.channels);
|
| 1229 |
|
| 1230 | fdev->addr_bits = (long)device_get_match_data(fdev->dev);
|
| 1231 |
|
| 1232 |
|
| 1233 | fdev->regs = of_iomap(op->dev.of_node, 0);
|
| 1234 | if (!fdev->regs) {
|
| 1235 | dev_err(&op->dev, "unable to ioremap registers\n");
|
| 1236 | err = -ENOMEM;
|
| 1237 | goto out_free;
|
| 1238 | }
|
| 1239 |
|
| 1240 |
|
| 1241 | fdev->irq = irq_of_parse_and_map(op->dev.of_node, 0);
|
| 1242 |
|
| 1243 | dma_cap_set(DMA_MEMCPY, fdev->common.cap_mask);
|
| 1244 | dma_cap_set(DMA_SLAVE, fdev->common.cap_mask);
|
| 1245 | fdev->common.device_alloc_chan_resources = fsl_dma_alloc_chan_resources;
|
| 1246 | fdev->common.device_free_chan_resources = fsl_dma_free_chan_resources;
|
| 1247 | fdev->common.device_prep_dma_memcpy = fsl_dma_prep_memcpy;
|
| 1248 | fdev->common.device_tx_status = fsl_tx_status;
|
| 1249 | fdev->common.device_issue_pending = fsl_dma_memcpy_issue_pending;
|
| 1250 | fdev->common.device_config = fsl_dma_device_config;
|
| 1251 | fdev->common.device_terminate_all = fsl_dma_device_terminate_all;
|
| 1252 | fdev->common.dev = &op->dev;
|
| 1253 |
|
| 1254 | fdev->common.src_addr_widths = FSL_DMA_BUSWIDTHS;
|
| 1255 | fdev->common.dst_addr_widths = FSL_DMA_BUSWIDTHS;
|
| 1256 | fdev->common.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV);
|
| 1257 | fdev->common.residue_granularity = DMA_RESIDUE_GRANULARITY_DESCRIPTOR;
|
| 1258 |
|
| 1259 | dma_set_mask(&(op->dev), DMA_BIT_MASK(fdev->addr_bits));
|
| 1260 |
|
| 1261 | platform_set_drvdata(op, fdev);
|
| 1262 |
|
| 1263 | |
| 1264 | * We cannot use of_platform_bus_probe() because there is no
|
| 1265 | * of_platform_bus_remove(). Instead, we manually instantiate every DMA
|
| 1266 | * channel object.
|
| 1267 |
|
| 1268 | for_each_child_of_node(op->dev.of_node, child) {
|
| 1269 | if (of_device_is_compatible(child, "fsl,eloplus-dma-channel")) {
|
| 1270 | fsl_dma_chan_probe(fdev, child,
|
| 1271 | FSL_DMA_IP_85XX | FSL_DMA_BIG_ENDIAN,
|
| 1272 | "fsl,eloplus-dma-channel");
|
| 1273 | }
|
| 1274 |
|
| 1275 | if (of_device_is_compatible(child, "fsl,elo-dma-channel")) {
|
| 1276 | fsl_dma_chan_probe(fdev, child,
|
| 1277 | FSL_DMA_IP_83XX | FSL_DMA_LITTLE_ENDIAN,
|
| 1278 | "fsl,elo-dma-channel");
|
| 1279 | }
|
| 1280 | }
|
| 1281 |
|
| 1282 | |
| 1283 | * Hookup the IRQ handler(s)
|
| 1284 | *
|
| 1285 | * If we have a per-controller interrupt, we prefer that to the
|
| 1286 | * per-channel interrupts to reduce the number of shared interrupt
|
| 1287 | * handlers on the same IRQ line
|
| 1288 |
|
| 1289 | err = fsldma_request_irqs(fdev);
|
| 1290 | if (err) {
|
| 1291 | dev_err(fdev->dev, "unable to request IRQs\n");
|
| 1292 | goto out_free_fdev;
|
| 1293 | }
|
| 1294 |
|
| 1295 | dma_async_device_register(&fdev->common);
|
| 1296 | return 0;
|
| 1297 |
|
| 1298 | out_free_fdev:
|
| 1299 | for (i = 0; i < FSL_DMA_MAX_CHANS_PER_DEVICE; i++) {
|
| 1300 | if (fdev->chan[i])
|
| 1301 | fsl_dma_chan_remove(fdev->chan[i]);
|
| 1302 | }
|
| 1303 | irq_dispose_mapping(fdev->irq);
|
| 1304 | iounmap(fdev->regs);
|
| 1305 | out_free:
|
| 1306 | kfree(fdev);
|
| 1307 | out_return:
|
| 1308 | return err;
|
| 1309 | }
|
| 1310 |
|
| 1311 | static void fsldma_of_remove(struct platform_device *op)
|
| 1312 | {
|
| 1313 | struct fsldma_device *fdev;
|
| 1314 | unsigned int i;
|
| 1315 |
|
| 1316 | fdev = platform_get_drvdata(op);
|
| 1317 | dma_async_device_unregister(&fdev->common);
|
| 1318 |
|
| 1319 | fsldma_free_irqs(fdev);
|
| 1320 |
|
| 1321 | for (i = 0; i < FSL_DMA_MAX_CHANS_PER_DEVICE; i++) {
|
| 1322 | if (fdev->chan[i])
|
| 1323 | fsl_dma_chan_remove(fdev->chan[i]);
|
| 1324 | }
|
| 1325 | irq_dispose_mapping(fdev->irq);
|
| 1326 |
|
| 1327 | iounmap(fdev->regs);
|
| 1328 | kfree(fdev);
|
| 1329 | }
|
| 1330 |
|
| 1331 | #ifdef CONFIG_PM
|
| 1332 | static int fsldma_suspend_late(struct device *dev)
|
| 1333 | {
|
| 1334 | struct fsldma_device *fdev = dev_get_drvdata(dev);
|
| 1335 | struct fsldma_chan *chan;
|
| 1336 | int i;
|
| 1337 |
|
| 1338 | for (i = 0; i < FSL_DMA_MAX_CHANS_PER_DEVICE; i++) {
|
| 1339 | chan = fdev->chan[i];
|
| 1340 | if (!chan)
|
| 1341 | continue;
|
| 1342 |
|
| 1343 | spin_lock_bh(&chan->desc_lock);
|
| 1344 | if (unlikely(!chan->idle))
|
| 1345 | goto out;
|
| 1346 | chan->regs_save.mr = get_mr(chan);
|
| 1347 | chan->pm_state = SUSPENDED;
|
| 1348 | spin_unlock_bh(&chan->desc_lock);
|
| 1349 | }
|
| 1350 | return 0;
|
| 1351 |
|
| 1352 | out:
|
| 1353 | for (; i >= 0; i--) {
|
| 1354 | chan = fdev->chan[i];
|
| 1355 | if (!chan)
|
| 1356 | continue;
|
| 1357 | chan->pm_state = RUNNING;
|
| 1358 | spin_unlock_bh(&chan->desc_lock);
|
| 1359 | }
|
| 1360 | return -EBUSY;
|
| 1361 | }
|
| 1362 |
|
| 1363 | static int fsldma_resume_early(struct device *dev)
|
| 1364 | {
|
| 1365 | struct fsldma_device *fdev = dev_get_drvdata(dev);
|
| 1366 | struct fsldma_chan *chan;
|
| 1367 | u32 mode;
|
| 1368 | int i;
|
| 1369 |
|
| 1370 | for (i = 0; i < FSL_DMA_MAX_CHANS_PER_DEVICE; i++) {
|
| 1371 | chan = fdev->chan[i];
|
| 1372 | if (!chan)
|
| 1373 | continue;
|
| 1374 |
|
| 1375 | spin_lock_bh(&chan->desc_lock);
|
| 1376 | mode = chan->regs_save.mr
|
| 1377 | & ~FSL_DMA_MR_CS & ~FSL_DMA_MR_CC & ~FSL_DMA_MR_CA;
|
| 1378 | set_mr(chan, mode);
|
| 1379 | chan->pm_state = RUNNING;
|
| 1380 | spin_unlock_bh(&chan->desc_lock);
|
| 1381 | }
|
| 1382 |
|
| 1383 | return 0;
|
| 1384 | }
|
| 1385 |
|
| 1386 | static const struct dev_pm_ops fsldma_pm_ops = {
|
| 1387 | .suspend_late = fsldma_suspend_late,
|
| 1388 | .resume_early = fsldma_resume_early,
|
| 1389 | };
|
| 1390 | #endif
|
| 1391 |
|
| 1392 |
|
| 1393 | static const struct of_device_id fsldma_of_ids[] = {
|
| 1394 | {
|
| 1395 | .compatible = "fsl,elo3-dma",
|
| 1396 | .data = (void *)40,
|
| 1397 | },
|
| 1398 | {
|
| 1399 | .compatible = "fsl,eloplus-dma",
|
| 1400 | .data = (void *)36,
|
| 1401 | },
|
| 1402 | {
|
| 1403 | .compatible = "fsl,elo-dma",
|
| 1404 | .data = (void *)32,
|
| 1405 | },
|
| 1406 | {}
|
| 1407 | };
|
| 1408 | MODULE_DEVICE_TABLE(of, fsldma_of_ids);
|
| 1409 |
|
| 1410 | static struct platform_driver fsldma_of_driver = {
|
| 1411 | .driver = {
|
| 1412 | .name = "fsl-elo-dma",
|
| 1413 | .of_match_table = fsldma_of_ids,
|
| 1414 | #ifdef CONFIG_PM
|
| 1415 | .pm = &fsldma_pm_ops,
|
| 1416 | #endif
|
| 1417 | },
|
| 1418 | .probe = fsldma_of_probe,
|
| 1419 | .remove = fsldma_of_remove,
|
| 1420 | };
|
| 1421 |
|
| 1422 |
|
| 1423 |
|
| 1424 |
|
| 1425 |
|
| 1426 | static __init int fsldma_init(void)
|
| 1427 | {
|
| 1428 | pr_info("Freescale Elo series DMA driver\n");
|
| 1429 | return platform_driver_register(&fsldma_of_driver);
|
| 1430 | }
|
| 1431 |
|
| 1432 | static void __exit fsldma_exit(void)
|
| 1433 | {
|
| 1434 | platform_driver_unregister(&fsldma_of_driver);
|
| 1435 | }
|
| 1436 |
|
| 1437 | subsys_initcall(fsldma_init);
|
| 1438 | module_exit(fsldma_exit);
|
| 1439 |
|
| 1440 | MODULE_DESCRIPTION("Freescale Elo series DMA driver");
|
| 1441 | MODULE_LICENSE("GPL");
|