2 * TI EDMA DMA engine driver
4 * Copyright 2012 Texas Instruments
6 * This program is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU General Public License as
8 * published by the Free Software Foundation version 2.
10 * This program is distributed "as is" WITHOUT ANY WARRANTY of any
11 * kind, whether express or implied; without even the implied warranty
12 * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
16 #include <linux/dmaengine.h>
17 #include <linux/dma-mapping.h>
18 #include <linux/err.h>
19 #include <linux/init.h>
20 #include <linux/interrupt.h>
21 #include <linux/list.h>
22 #include <linux/module.h>
23 #include <linux/platform_device.h>
24 #include <linux/slab.h>
25 #include <linux/spinlock.h>
27 #include <linux/platform_data/edma.h>
29 #include "dmaengine.h"
33 * This will go away when the private EDMA API is folded
34 * into this driver and the platform device(s) are
35 * instantiated in the arch code. We can only get away
36 * with this simplification because DA8XX may not be built
37 * in the same kernel image with other DaVinci parts. This
38 * avoids having to sprinkle dmaengine driver platform devices
39 * and data throughout all the existing board files.
41 #ifdef CONFIG_ARCH_DAVINCI_DA8XX
47 #endif /* CONFIG_ARCH_DAVINCI_DA8XX */
50 * Max of 20 segments per channel to conserve PaRAM slots
51 * Also note that MAX_NR_SG should be atleast the no.of periods
52 * that are required for ASoC, otherwise DMA prep calls will
53 * fail. Today davinci-pcm is the only user of this driver and
54 * requires atleast 17 slots, so we setup the default to 20.
57 #define EDMA_MAX_SLOTS MAX_NR_SG
58 #define EDMA_DESCRIPTORS 16
63 struct edmacc_param param;
67 struct virt_dma_desc vdesc;
68 struct list_head node;
69 enum dma_transfer_direction direction;
73 struct edma_chan *echan;
77 * The following 4 elements are used for residue accounting.
79 * - processed_stat: the number of SG elements we have traversed
80 * so far to cover accounting. This is updated directly to processed
81 * during edma_callback and is always <= processed, because processed
82 * refers to the number of pending transfer (programmed to EDMA
83 * controller), where as processed_stat tracks number of transfers
84 * accounted for so far.
86 * - residue: The amount of bytes we have left to transfer for this desc
88 * - residue_stat: The residue in bytes of data we have covered
89 * so far for accounting. This is updated directly to residue
90 * during callbacks to keep it current.
92 * - sg_len: Tracks the length of the current intermediate transfer,
93 * this is required to update the residue during intermediate transfer
94 * completion callback.
101 struct edma_pset pset[0];
107 struct virt_dma_chan vchan;
108 struct list_head node;
109 struct edma_desc *edesc;
113 int slot[EDMA_MAX_SLOTS];
115 struct dma_slave_config cfg;
120 struct dma_device dma_slave;
121 struct edma_chan slave_chans[EDMA_CHANS];
126 static inline struct edma_cc *to_edma_cc(struct dma_device *d)
128 return container_of(d, struct edma_cc, dma_slave);
131 static inline struct edma_chan *to_edma_chan(struct dma_chan *c)
133 return container_of(c, struct edma_chan, vchan.chan);
136 static inline struct edma_desc
137 *to_edma_desc(struct dma_async_tx_descriptor *tx)
139 return container_of(tx, struct edma_desc, vdesc.tx);
142 static void edma_desc_free(struct virt_dma_desc *vdesc)
144 kfree(container_of(vdesc, struct edma_desc, vdesc));
147 /* Dispatch a queued descriptor to the controller (caller holds lock) */
148 static void edma_execute(struct edma_chan *echan)
150 struct virt_dma_desc *vdesc;
151 struct edma_desc *edesc;
152 struct device *dev = echan->vchan.chan.device->dev;
153 int i, j, left, nslots;
155 /* If either we processed all psets or we're still not started */
157 echan->edesc->pset_nr == echan->edesc->processed) {
159 vdesc = vchan_next_desc(&echan->vchan);
164 list_del(&vdesc->node);
165 echan->edesc = to_edma_desc(&vdesc->tx);
168 edesc = echan->edesc;
170 /* Find out how many left */
171 left = edesc->pset_nr - edesc->processed;
172 nslots = min(MAX_NR_SG, left);
175 /* Write descriptor PaRAM set(s) */
176 for (i = 0; i < nslots; i++) {
177 j = i + edesc->processed;
178 edma_write_slot(echan->slot[i], &edesc->pset[j].param);
179 edesc->sg_len += edesc->pset[j].len;
180 dev_vdbg(echan->vchan.chan.device->dev,
192 j, echan->ch_num, echan->slot[i],
193 edesc->pset[j].param.opt,
194 edesc->pset[j].param.src,
195 edesc->pset[j].param.dst,
196 edesc->pset[j].param.a_b_cnt,
197 edesc->pset[j].param.ccnt,
198 edesc->pset[j].param.src_dst_bidx,
199 edesc->pset[j].param.src_dst_cidx,
200 edesc->pset[j].param.link_bcntrld);
201 /* Link to the previous slot if not the last set */
202 if (i != (nslots - 1))
203 edma_link(echan->slot[i], echan->slot[i+1]);
206 edesc->processed += nslots;
209 * If this is either the last set in a set of SG-list transactions
210 * then setup a link to the dummy slot, this results in all future
211 * events being absorbed and that's OK because we're done
213 if (edesc->processed == edesc->pset_nr) {
215 edma_link(echan->slot[nslots-1], echan->slot[1]);
217 edma_link(echan->slot[nslots-1],
218 echan->ecc->dummy_slot);
221 if (edesc->processed <= MAX_NR_SG) {
222 dev_dbg(dev, "first transfer starting on channel %d\n",
224 edma_start(echan->ch_num);
226 dev_dbg(dev, "chan: %d: completed %d elements, resuming\n",
227 echan->ch_num, edesc->processed);
228 edma_resume(echan->ch_num);
232 * This happens due to setup times between intermediate transfers
233 * in long SG lists which have to be broken up into transfers of
237 dev_dbg(dev, "missed event on channel %d\n", echan->ch_num);
238 edma_clean_channel(echan->ch_num);
239 edma_stop(echan->ch_num);
240 edma_start(echan->ch_num);
241 edma_trigger_channel(echan->ch_num);
246 static int edma_terminate_all(struct edma_chan *echan)
251 spin_lock_irqsave(&echan->vchan.lock, flags);
254 * Stop DMA activity: we assume the callback will not be called
255 * after edma_dma() returns (even if it does, it will see
256 * echan->edesc is NULL and exit.)
260 edma_stop(echan->ch_num);
263 vchan_get_all_descriptors(&echan->vchan, &head);
264 spin_unlock_irqrestore(&echan->vchan.lock, flags);
265 vchan_dma_desc_free_list(&echan->vchan, &head);
270 static int edma_slave_config(struct edma_chan *echan,
271 struct dma_slave_config *cfg)
273 if (cfg->src_addr_width == DMA_SLAVE_BUSWIDTH_8_BYTES ||
274 cfg->dst_addr_width == DMA_SLAVE_BUSWIDTH_8_BYTES)
277 memcpy(&echan->cfg, cfg, sizeof(echan->cfg));
282 static int edma_dma_pause(struct edma_chan *echan)
284 /* Pause/Resume only allowed with cyclic mode */
285 if (!echan->edesc->cyclic)
288 edma_pause(echan->ch_num);
292 static int edma_dma_resume(struct edma_chan *echan)
294 /* Pause/Resume only allowed with cyclic mode */
295 if (!echan->edesc->cyclic)
298 edma_resume(echan->ch_num);
302 static int edma_control(struct dma_chan *chan, enum dma_ctrl_cmd cmd,
306 struct dma_slave_config *config;
307 struct edma_chan *echan = to_edma_chan(chan);
310 case DMA_TERMINATE_ALL:
311 edma_terminate_all(echan);
313 case DMA_SLAVE_CONFIG:
314 config = (struct dma_slave_config *)arg;
315 ret = edma_slave_config(echan, config);
318 ret = edma_dma_pause(echan);
322 ret = edma_dma_resume(echan);
333 * A PaRAM set configuration abstraction used by other modes
334 * @chan: Channel who's PaRAM set we're configuring
335 * @pset: PaRAM set to initialize and setup.
336 * @src_addr: Source address of the DMA
337 * @dst_addr: Destination address of the DMA
338 * @burst: In units of dev_width, how much to send
339 * @dev_width: How much is the dev_width
340 * @dma_length: Total length of the DMA transfer
341 * @direction: Direction of the transfer
343 static int edma_config_pset(struct dma_chan *chan, struct edma_pset *epset,
344 dma_addr_t src_addr, dma_addr_t dst_addr, u32 burst,
345 enum dma_slave_buswidth dev_width, unsigned int dma_length,
346 enum dma_transfer_direction direction)
348 struct edma_chan *echan = to_edma_chan(chan);
349 struct device *dev = chan->device->dev;
350 struct edmacc_param *param = &epset->param;
351 int acnt, bcnt, ccnt, cidx;
352 int src_bidx, dst_bidx, src_cidx, dst_cidx;
357 /* src/dst_maxburst == 0 is the same case as src/dst_maxburst == 1 */
361 * If the maxburst is equal to the fifo width, use
362 * A-synced transfers. This allows for large contiguous
363 * buffer transfers using only one PaRAM set.
367 * For the A-sync case, bcnt and ccnt are the remainder
368 * and quotient respectively of the division of:
369 * (dma_length / acnt) by (SZ_64K -1). This is so
370 * that in case bcnt over flows, we have ccnt to use.
371 * Note: In A-sync tranfer only, bcntrld is used, but it
372 * only applies for sg_dma_len(sg) >= SZ_64K.
373 * In this case, the best way adopted is- bccnt for the
374 * first frame will be the remainder below. Then for
375 * every successive frame, bcnt will be SZ_64K-1. This
376 * is assured as bcntrld = 0xffff in end of function.
379 ccnt = dma_length / acnt / (SZ_64K - 1);
380 bcnt = dma_length / acnt - ccnt * (SZ_64K - 1);
382 * If bcnt is non-zero, we have a remainder and hence an
383 * extra frame to transfer, so increment ccnt.
392 * If maxburst is greater than the fifo address_width,
393 * use AB-synced transfers where A count is the fifo
394 * address_width and B count is the maxburst. In this
395 * case, we are limited to transfers of C count frames
396 * of (address_width * maxburst) where C count is limited
397 * to SZ_64K-1. This places an upper bound on the length
398 * of an SG segment that can be handled.
402 ccnt = dma_length / (acnt * bcnt);
403 if (ccnt > (SZ_64K - 1)) {
404 dev_err(dev, "Exceeded max SG segment size\n");
410 epset->len = dma_length;
412 if (direction == DMA_MEM_TO_DEV) {
417 epset->addr = src_addr;
418 } else if (direction == DMA_DEV_TO_MEM) {
423 epset->addr = dst_addr;
424 } else if (direction == DMA_MEM_TO_MEM) {
430 dev_err(dev, "%s: direction not implemented yet\n", __func__);
434 param->opt = EDMA_TCC(EDMA_CHAN_SLOT(echan->ch_num));
435 /* Configure A or AB synchronized transfers */
437 param->opt |= SYNCDIM;
439 param->src = src_addr;
440 param->dst = dst_addr;
442 param->src_dst_bidx = (dst_bidx << 16) | src_bidx;
443 param->src_dst_cidx = (dst_cidx << 16) | src_cidx;
445 param->a_b_cnt = bcnt << 16 | acnt;
448 * Only time when (bcntrld) auto reload is required is for
449 * A-sync case, and in this case, a requirement of reload value
450 * of SZ_64K-1 only is assured. 'link' is initially set to NULL
451 * and then later will be populated by edma_execute.
453 param->link_bcntrld = 0xffffffff;
457 static struct dma_async_tx_descriptor *edma_prep_slave_sg(
458 struct dma_chan *chan, struct scatterlist *sgl,
459 unsigned int sg_len, enum dma_transfer_direction direction,
460 unsigned long tx_flags, void *context)
462 struct edma_chan *echan = to_edma_chan(chan);
463 struct device *dev = chan->device->dev;
464 struct edma_desc *edesc;
465 dma_addr_t src_addr = 0, dst_addr = 0;
466 enum dma_slave_buswidth dev_width;
468 struct scatterlist *sg;
471 if (unlikely(!echan || !sgl || !sg_len))
474 if (direction == DMA_DEV_TO_MEM) {
475 src_addr = echan->cfg.src_addr;
476 dev_width = echan->cfg.src_addr_width;
477 burst = echan->cfg.src_maxburst;
478 } else if (direction == DMA_MEM_TO_DEV) {
479 dst_addr = echan->cfg.dst_addr;
480 dev_width = echan->cfg.dst_addr_width;
481 burst = echan->cfg.dst_maxburst;
483 dev_err(dev, "%s: bad direction: %d\n", __func__, direction);
487 if (dev_width == DMA_SLAVE_BUSWIDTH_UNDEFINED) {
488 dev_err(dev, "%s: Undefined slave buswidth\n", __func__);
492 edesc = kzalloc(sizeof(*edesc) + sg_len *
493 sizeof(edesc->pset[0]), GFP_ATOMIC);
495 dev_err(dev, "%s: Failed to allocate a descriptor\n", __func__);
499 edesc->pset_nr = sg_len;
501 edesc->direction = direction;
502 edesc->echan = echan;
504 /* Allocate a PaRAM slot, if needed */
505 nslots = min_t(unsigned, MAX_NR_SG, sg_len);
507 for (i = 0; i < nslots; i++) {
508 if (echan->slot[i] < 0) {
510 edma_alloc_slot(EDMA_CTLR(echan->ch_num),
512 if (echan->slot[i] < 0) {
514 dev_err(dev, "%s: Failed to allocate slot\n",
521 /* Configure PaRAM sets for each SG */
522 for_each_sg(sgl, sg, sg_len, i) {
523 /* Get address for each SG */
524 if (direction == DMA_DEV_TO_MEM)
525 dst_addr = sg_dma_address(sg);
527 src_addr = sg_dma_address(sg);
529 ret = edma_config_pset(chan, &edesc->pset[i], src_addr,
530 dst_addr, burst, dev_width,
531 sg_dma_len(sg), direction);
538 edesc->residue += sg_dma_len(sg);
540 /* If this is the last in a current SG set of transactions,
541 enable interrupts so that next set is processed */
542 if (!((i+1) % MAX_NR_SG))
543 edesc->pset[i].param.opt |= TCINTEN;
545 /* If this is the last set, enable completion interrupt flag */
547 edesc->pset[i].param.opt |= TCINTEN;
549 edesc->residue_stat = edesc->residue;
551 return vchan_tx_prep(&echan->vchan, &edesc->vdesc, tx_flags);
554 struct dma_async_tx_descriptor *edma_prep_dma_memcpy(
555 struct dma_chan *chan, dma_addr_t dest, dma_addr_t src,
556 size_t len, unsigned long tx_flags)
559 struct edma_desc *edesc;
560 struct device *dev = chan->device->dev;
561 struct edma_chan *echan = to_edma_chan(chan);
563 if (unlikely(!echan || !len))
566 edesc = kzalloc(sizeof(*edesc) + sizeof(edesc->pset[0]), GFP_ATOMIC);
568 dev_dbg(dev, "Failed to allocate a descriptor\n");
574 ret = edma_config_pset(chan, &edesc->pset[0], src, dest, 1,
575 DMA_SLAVE_BUSWIDTH_4_BYTES, len, DMA_MEM_TO_MEM);
582 * Enable intermediate transfer chaining to re-trigger channel
583 * on completion of every TR, and enable transfer-completion
584 * interrupt on completion of the whole transfer.
586 edesc->pset[0].param.opt |= ITCCHEN;
587 edesc->pset[0].param.opt |= TCINTEN;
589 return vchan_tx_prep(&echan->vchan, &edesc->vdesc, tx_flags);
592 static struct dma_async_tx_descriptor *edma_prep_dma_cyclic(
593 struct dma_chan *chan, dma_addr_t buf_addr, size_t buf_len,
594 size_t period_len, enum dma_transfer_direction direction,
595 unsigned long tx_flags, void *context)
597 struct edma_chan *echan = to_edma_chan(chan);
598 struct device *dev = chan->device->dev;
599 struct edma_desc *edesc;
600 dma_addr_t src_addr, dst_addr;
601 enum dma_slave_buswidth dev_width;
605 if (unlikely(!echan || !buf_len || !period_len))
608 if (direction == DMA_DEV_TO_MEM) {
609 src_addr = echan->cfg.src_addr;
611 dev_width = echan->cfg.src_addr_width;
612 burst = echan->cfg.src_maxburst;
613 } else if (direction == DMA_MEM_TO_DEV) {
615 dst_addr = echan->cfg.dst_addr;
616 dev_width = echan->cfg.dst_addr_width;
617 burst = echan->cfg.dst_maxburst;
619 dev_err(dev, "%s: bad direction: %d\n", __func__, direction);
623 if (dev_width == DMA_SLAVE_BUSWIDTH_UNDEFINED) {
624 dev_err(dev, "%s: Undefined slave buswidth\n", __func__);
628 if (unlikely(buf_len % period_len)) {
629 dev_err(dev, "Period should be multiple of Buffer length\n");
633 nslots = (buf_len / period_len) + 1;
636 * Cyclic DMA users such as audio cannot tolerate delays introduced
637 * by cases where the number of periods is more than the maximum
638 * number of SGs the EDMA driver can handle at a time. For DMA types
639 * such as Slave SGs, such delays are tolerable and synchronized,
640 * but the synchronization is difficult to achieve with Cyclic and
641 * cannot be guaranteed, so we error out early.
643 if (nslots > MAX_NR_SG)
646 edesc = kzalloc(sizeof(*edesc) + nslots *
647 sizeof(edesc->pset[0]), GFP_ATOMIC);
649 dev_err(dev, "%s: Failed to allocate a descriptor\n", __func__);
654 edesc->pset_nr = nslots;
655 edesc->residue = edesc->residue_stat = buf_len;
656 edesc->direction = direction;
657 edesc->echan = echan;
659 dev_dbg(dev, "%s: channel=%d nslots=%d period_len=%zu buf_len=%zu\n",
660 __func__, echan->ch_num, nslots, period_len, buf_len);
662 for (i = 0; i < nslots; i++) {
663 /* Allocate a PaRAM slot, if needed */
664 if (echan->slot[i] < 0) {
666 edma_alloc_slot(EDMA_CTLR(echan->ch_num),
668 if (echan->slot[i] < 0) {
670 dev_err(dev, "%s: Failed to allocate slot\n",
676 if (i == nslots - 1) {
677 memcpy(&edesc->pset[i], &edesc->pset[0],
678 sizeof(edesc->pset[0]));
682 ret = edma_config_pset(chan, &edesc->pset[i], src_addr,
683 dst_addr, burst, dev_width, period_len,
690 if (direction == DMA_DEV_TO_MEM)
691 dst_addr += period_len;
693 src_addr += period_len;
695 dev_vdbg(dev, "%s: Configure period %d of buf:\n", __func__, i);
708 i, echan->ch_num, echan->slot[i],
709 edesc->pset[i].param.opt,
710 edesc->pset[i].param.src,
711 edesc->pset[i].param.dst,
712 edesc->pset[i].param.a_b_cnt,
713 edesc->pset[i].param.ccnt,
714 edesc->pset[i].param.src_dst_bidx,
715 edesc->pset[i].param.src_dst_cidx,
716 edesc->pset[i].param.link_bcntrld);
721 * Enable interrupts for every period because callback
722 * has to be called for every period.
724 edesc->pset[i].param.opt |= TCINTEN;
727 return vchan_tx_prep(&echan->vchan, &edesc->vdesc, tx_flags);
730 static void edma_callback(unsigned ch_num, u16 ch_status, void *data)
732 struct edma_chan *echan = data;
733 struct device *dev = echan->vchan.chan.device->dev;
734 struct edma_desc *edesc;
735 struct edmacc_param p;
737 edesc = echan->edesc;
739 /* Pause the channel for non-cyclic */
740 if (!edesc || (edesc && !edesc->cyclic))
741 edma_pause(echan->ch_num);
744 case EDMA_DMA_COMPLETE:
745 spin_lock(&echan->vchan.lock);
749 vchan_cyclic_callback(&edesc->vdesc);
750 } else if (edesc->processed == edesc->pset_nr) {
751 dev_dbg(dev, "Transfer complete, stopping channel %d\n", ch_num);
753 edma_stop(echan->ch_num);
754 vchan_cookie_complete(&edesc->vdesc);
757 dev_dbg(dev, "Intermediate transfer complete on channel %d\n", ch_num);
759 /* Update statistics for tx_status */
760 edesc->residue -= edesc->sg_len;
761 edesc->residue_stat = edesc->residue;
762 edesc->processed_stat = edesc->processed;
768 spin_unlock(&echan->vchan.lock);
771 case EDMA_DMA_CC_ERROR:
772 spin_lock(&echan->vchan.lock);
774 edma_read_slot(EDMA_CHAN_SLOT(echan->slot[0]), &p);
777 * Issue later based on missed flag which will be sure
779 * (1) we finished transmitting an intermediate slot and
780 * edma_execute is coming up.
781 * (2) or we finished current transfer and issue will
784 * Important note: issuing can be dangerous here and
785 * lead to some nasty recursion when we are in a NULL
786 * slot. So we avoid doing so and set the missed flag.
788 if (p.a_b_cnt == 0 && p.ccnt == 0) {
789 dev_dbg(dev, "Error occurred, looks like slot is null, just setting miss\n");
793 * The slot is already programmed but the event got
794 * missed, so its safe to issue it here.
796 dev_dbg(dev, "Error occurred but slot is non-null, TRIGGERING\n");
797 edma_clean_channel(echan->ch_num);
798 edma_stop(echan->ch_num);
799 edma_start(echan->ch_num);
800 edma_trigger_channel(echan->ch_num);
803 spin_unlock(&echan->vchan.lock);
811 /* Alloc channel resources */
812 static int edma_alloc_chan_resources(struct dma_chan *chan)
814 struct edma_chan *echan = to_edma_chan(chan);
815 struct device *dev = chan->device->dev;
820 a_ch_num = edma_alloc_channel(echan->ch_num, edma_callback,
821 chan, EVENTQ_DEFAULT);
828 if (a_ch_num != echan->ch_num) {
829 dev_err(dev, "failed to allocate requested channel %u:%u\n",
830 EDMA_CTLR(echan->ch_num),
831 EDMA_CHAN_SLOT(echan->ch_num));
836 echan->alloced = true;
837 echan->slot[0] = echan->ch_num;
839 dev_dbg(dev, "allocated channel %d for %u:%u\n", echan->ch_num,
840 EDMA_CTLR(echan->ch_num), EDMA_CHAN_SLOT(echan->ch_num));
845 edma_free_channel(a_ch_num);
850 /* Free channel resources */
851 static void edma_free_chan_resources(struct dma_chan *chan)
853 struct edma_chan *echan = to_edma_chan(chan);
854 struct device *dev = chan->device->dev;
857 /* Terminate transfers */
858 edma_stop(echan->ch_num);
860 vchan_free_chan_resources(&echan->vchan);
862 /* Free EDMA PaRAM slots */
863 for (i = 1; i < EDMA_MAX_SLOTS; i++) {
864 if (echan->slot[i] >= 0) {
865 edma_free_slot(echan->slot[i]);
870 /* Free EDMA channel */
871 if (echan->alloced) {
872 edma_free_channel(echan->ch_num);
873 echan->alloced = false;
876 dev_dbg(dev, "freeing channel for %u\n", echan->ch_num);
879 /* Send pending descriptor to hardware */
880 static void edma_issue_pending(struct dma_chan *chan)
882 struct edma_chan *echan = to_edma_chan(chan);
885 spin_lock_irqsave(&echan->vchan.lock, flags);
886 if (vchan_issue_pending(&echan->vchan) && !echan->edesc)
888 spin_unlock_irqrestore(&echan->vchan.lock, flags);
891 static u32 edma_residue(struct edma_desc *edesc)
893 bool dst = edesc->direction == DMA_DEV_TO_MEM;
894 struct edma_pset *pset = edesc->pset;
895 dma_addr_t done, pos;
899 * We always read the dst/src position from the first RamPar
900 * pset. That's the one which is active now.
902 pos = edma_get_position(edesc->echan->slot[0], dst);
905 * Cyclic is simple. Just subtract pset[0].addr from pos.
907 * We never update edesc->residue in the cyclic case, so we
908 * can tell the remaining room to the end of the circular
912 done = pos - pset->addr;
913 edesc->residue_stat = edesc->residue - done;
914 return edesc->residue_stat;
918 * For SG operation we catch up with the last processed
921 pset += edesc->processed_stat;
923 for (i = edesc->processed_stat; i < edesc->processed; i++, pset++) {
925 * If we are inside this pset address range, we know
926 * this is the active one. Get the current delta and
927 * stop walking the psets.
929 if (pos >= pset->addr && pos < pset->addr + pset->len)
930 return edesc->residue_stat - (pos - pset->addr);
932 /* Otherwise mark it done and update residue_stat. */
933 edesc->processed_stat++;
934 edesc->residue_stat -= pset->len;
936 return edesc->residue_stat;
939 /* Check request completion status */
940 static enum dma_status edma_tx_status(struct dma_chan *chan,
942 struct dma_tx_state *txstate)
944 struct edma_chan *echan = to_edma_chan(chan);
945 struct virt_dma_desc *vdesc;
949 ret = dma_cookie_status(chan, cookie, txstate);
950 if (ret == DMA_COMPLETE || !txstate)
953 spin_lock_irqsave(&echan->vchan.lock, flags);
954 if (echan->edesc && echan->edesc->vdesc.tx.cookie == cookie)
955 txstate->residue = edma_residue(echan->edesc);
956 else if ((vdesc = vchan_find_desc(&echan->vchan, cookie)))
957 txstate->residue = to_edma_desc(&vdesc->tx)->residue;
958 spin_unlock_irqrestore(&echan->vchan.lock, flags);
963 static void __init edma_chan_init(struct edma_cc *ecc,
964 struct dma_device *dma,
965 struct edma_chan *echans)
969 for (i = 0; i < EDMA_CHANS; i++) {
970 struct edma_chan *echan = &echans[i];
971 echan->ch_num = EDMA_CTLR_CHAN(ecc->ctlr, i);
973 echan->vchan.desc_free = edma_desc_free;
975 vchan_init(&echan->vchan, dma);
977 INIT_LIST_HEAD(&echan->node);
978 for (j = 0; j < EDMA_MAX_SLOTS; j++)
983 #define EDMA_DMA_BUSWIDTHS (BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | \
984 BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | \
985 BIT(DMA_SLAVE_BUSWIDTH_4_BYTES))
987 static int edma_dma_device_slave_caps(struct dma_chan *dchan,
988 struct dma_slave_caps *caps)
990 caps->src_addr_widths = EDMA_DMA_BUSWIDTHS;
991 caps->dstn_addr_widths = EDMA_DMA_BUSWIDTHS;
992 caps->directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV);
993 caps->cmd_pause = true;
994 caps->cmd_terminate = true;
995 caps->residue_granularity = DMA_RESIDUE_GRANULARITY_DESCRIPTOR;
1000 static void edma_dma_init(struct edma_cc *ecc, struct dma_device *dma,
1003 dma->device_prep_slave_sg = edma_prep_slave_sg;
1004 dma->device_prep_dma_cyclic = edma_prep_dma_cyclic;
1005 dma->device_prep_dma_memcpy = edma_prep_dma_memcpy;
1006 dma->device_alloc_chan_resources = edma_alloc_chan_resources;
1007 dma->device_free_chan_resources = edma_free_chan_resources;
1008 dma->device_issue_pending = edma_issue_pending;
1009 dma->device_tx_status = edma_tx_status;
1010 dma->device_control = edma_control;
1011 dma->device_slave_caps = edma_dma_device_slave_caps;
1015 * code using dma memcpy must make sure alignment of
1016 * length is at dma->copy_align boundary.
1018 dma->copy_align = DMA_SLAVE_BUSWIDTH_4_BYTES;
1020 INIT_LIST_HEAD(&dma->channels);
1023 static int edma_probe(struct platform_device *pdev)
1025 struct edma_cc *ecc;
1028 ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
1032 ecc = devm_kzalloc(&pdev->dev, sizeof(*ecc), GFP_KERNEL);
1034 dev_err(&pdev->dev, "Can't allocate controller\n");
1038 ecc->ctlr = pdev->id;
1039 ecc->dummy_slot = edma_alloc_slot(ecc->ctlr, EDMA_SLOT_ANY);
1040 if (ecc->dummy_slot < 0) {
1041 dev_err(&pdev->dev, "Can't allocate PaRAM dummy slot\n");
1045 dma_cap_zero(ecc->dma_slave.cap_mask);
1046 dma_cap_set(DMA_SLAVE, ecc->dma_slave.cap_mask);
1047 dma_cap_set(DMA_CYCLIC, ecc->dma_slave.cap_mask);
1048 dma_cap_set(DMA_MEMCPY, ecc->dma_slave.cap_mask);
1050 edma_dma_init(ecc, &ecc->dma_slave, &pdev->dev);
1052 edma_chan_init(ecc, &ecc->dma_slave, ecc->slave_chans);
1054 ret = dma_async_device_register(&ecc->dma_slave);
1058 platform_set_drvdata(pdev, ecc);
1060 dev_info(&pdev->dev, "TI EDMA DMA engine driver\n");
1065 edma_free_slot(ecc->dummy_slot);
1069 static int edma_remove(struct platform_device *pdev)
1071 struct device *dev = &pdev->dev;
1072 struct edma_cc *ecc = dev_get_drvdata(dev);
1074 dma_async_device_unregister(&ecc->dma_slave);
1075 edma_free_slot(ecc->dummy_slot);
1080 static struct platform_driver edma_driver = {
1081 .probe = edma_probe,
1082 .remove = edma_remove,
1084 .name = "edma-dma-engine",
1085 .owner = THIS_MODULE,
1089 bool edma_filter_fn(struct dma_chan *chan, void *param)
1091 if (chan->device->dev->driver == &edma_driver.driver) {
1092 struct edma_chan *echan = to_edma_chan(chan);
1093 unsigned ch_req = *(unsigned *)param;
1094 return ch_req == echan->ch_num;
1098 EXPORT_SYMBOL(edma_filter_fn);
1100 static struct platform_device *pdev0, *pdev1;
1102 static const struct platform_device_info edma_dev_info0 = {
1103 .name = "edma-dma-engine",
1105 .dma_mask = DMA_BIT_MASK(32),
1108 static const struct platform_device_info edma_dev_info1 = {
1109 .name = "edma-dma-engine",
1111 .dma_mask = DMA_BIT_MASK(32),
1114 static int edma_init(void)
1116 int ret = platform_driver_register(&edma_driver);
1119 pdev0 = platform_device_register_full(&edma_dev_info0);
1120 if (IS_ERR(pdev0)) {
1121 platform_driver_unregister(&edma_driver);
1122 ret = PTR_ERR(pdev0);
1127 if (EDMA_CTLRS == 2) {
1128 pdev1 = platform_device_register_full(&edma_dev_info1);
1129 if (IS_ERR(pdev1)) {
1130 platform_driver_unregister(&edma_driver);
1131 platform_device_unregister(pdev0);
1132 ret = PTR_ERR(pdev1);
1139 subsys_initcall(edma_init);
1141 static void __exit edma_exit(void)
1143 platform_device_unregister(pdev0);
1145 platform_device_unregister(pdev1);
1146 platform_driver_unregister(&edma_driver);
1148 module_exit(edma_exit);
1150 MODULE_AUTHOR("Matt Porter <matt.porter@linaro.org>");
1151 MODULE_DESCRIPTION("TI EDMA DMA engine driver");
1152 MODULE_LICENSE("GPL v2");