drivers/dma/dw_dmac.c

   1 /*
   2  * Core driver for the Synopsys DesignWare DMA Controller
   3  *
   4  * Copyright (C) 2007-2008 Atmel Corporation
   5  * Copyright (C) 2010-2011 ST Microelectronics
   6  *
   7  * This program is free software; you can redistribute it and/or modify
   8  * it under the terms of the GNU General Public License version 2 as
   9  * published by the Free Software Foundation.
  10  */
  11
  12 #include <linux/bitops.h>
  13 #include <linux/clk.h>
  14 #include <linux/delay.h>
  15 #include <linux/dmaengine.h>
  16 #include <linux/dma-mapping.h>
  17 #include <linux/init.h>
  18 #include <linux/interrupt.h>
  19 #include <linux/io.h>
  20 #include <linux/of.h>
  21 #include <linux/mm.h>
  22 #include <linux/module.h>
  23 #include <linux/platform_device.h>
  24 #include <linux/slab.h>
  25
  26 #include "dw_dmac_regs.h"
  27 #include "dmaengine.h"
  28
  29 /*
  30  * This supports the Synopsys "DesignWare AHB Central DMA Controller",
  31  * (DW_ahb_dmac) which is used with various AMBA 2.0 systems (not all
  32  * of which use ARM any more).  See the "Databook" from Synopsys for
  33  * information beyond what licensees probably provide.
  34  *
  35  * The driver has currently been tested only with the Atmel AT32AP7000,
  36  * which does not support descriptor writeback.
  37  */
  38
  39 static inline unsigned int dwc_get_dms(struct dw_dma_slave *slave)
  40 {
  41         return slave ? slave->dst_master : 0;
  42 }
  43
  44 static inline unsigned int dwc_get_sms(struct dw_dma_slave *slave)
  45 {
  46         return slave ? slave->src_master : 1;
  47 }
  48
  49 #define DWC_DEFAULT_CTLLO(_chan) ({                             \
  50                 struct dw_dma_slave *__slave = (_chan->private);        \
  51                 struct dw_dma_chan *_dwc = to_dw_dma_chan(_chan);       \
  52                 struct dma_slave_config *_sconfig = &_dwc->dma_sconfig; \
  53                 bool _is_slave = is_slave_direction(_dwc->direction);   \
  54                 int _dms = dwc_get_dms(__slave);                \
  55                 int _sms = dwc_get_sms(__slave);                \
  56                 u8 _smsize = _is_slave ? _sconfig->src_maxburst :       \
  57                         DW_DMA_MSIZE_16;                        \
  58                 u8 _dmsize = _is_slave ? _sconfig->dst_maxburst :       \
  59                         DW_DMA_MSIZE_16;                        \
  60                                                                 \
  61                 (DWC_CTLL_DST_MSIZE(_dmsize)                    \
  62                  | DWC_CTLL_SRC_MSIZE(_smsize)                  \
  63                  | DWC_CTLL_LLP_D_EN                            \
  64                  | DWC_CTLL_LLP_S_EN                            \
  65                  | DWC_CTLL_DMS(_dms)                           \
  66                  | DWC_CTLL_SMS(_sms));                         \
  67         })
  68
  69 /*
  70  * Number of descriptors to allocate for each channel. This should be
  71  * made configurable somehow; preferably, the clients (at least the
  72  * ones using slave transfers) should be able to give us a hint.
  73  */
  74 #define NR_DESCS_PER_CHANNEL    64
  75
  76 /*----------------------------------------------------------------------*/
  77
  78 /*
  79  * Because we're not relying on writeback from the controller (it may not
  80  * even be configured into the core!) we don't need to use dma_pool.  These
  81  * descriptors -- and associated data -- are cacheable.  We do need to make
  82  * sure their dcache entries are written back before handing them off to
  83  * the controller, though.
  84  */
  85
  86 static struct device *chan2dev(struct dma_chan *chan)
  87 {
  88         return &chan->dev->device;
  89 }
  90 static struct device *chan2parent(struct dma_chan *chan)
  91 {
  92         return chan->dev->device.parent;
  93 }
  94
  95 static struct dw_desc *dwc_first_active(struct dw_dma_chan *dwc)
  96 {
  97         return to_dw_desc(dwc->active_list.next);
  98 }
  99
 100 static struct dw_desc *dwc_desc_get(struct dw_dma_chan *dwc)
 101 {
 102         struct dw_desc *desc, *_desc;
 103         struct dw_desc *ret = NULL;
 104         unsigned int i = 0;
 105         unsigned long flags;
 106
 107         spin_lock_irqsave(&dwc->lock, flags);
 108         list_for_each_entry_safe(desc, _desc, &dwc->free_list, desc_node) {
 109                 i++;
 110                 if (async_tx_test_ack(&desc->txd)) {
 111                         list_del(&desc->desc_node);
 112                         ret = desc;
 113                         break;
 114                 }
 115                 dev_dbg(chan2dev(&dwc->chan), "desc %p not ACKed\n", desc);
 116         }
 117         spin_unlock_irqrestore(&dwc->lock, flags);
 118
 119         dev_vdbg(chan2dev(&dwc->chan), "scanned %u descriptors on freelist\n", i);
 120
 121         return ret;
 122 }
 123
 124 static void dwc_sync_desc_for_cpu(struct dw_dma_chan *dwc, struct dw_desc *desc)
 125 {
 126         struct dw_desc  *child;
 127
 128         list_for_each_entry(child, &desc->tx_list, desc_node)
 129                 dma_sync_single_for_cpu(chan2parent(&dwc->chan),
 130                                 child->txd.phys, sizeof(child->lli),
 131                                 DMA_TO_DEVICE);
 132         dma_sync_single_for_cpu(chan2parent(&dwc->chan),
 133                         desc->txd.phys, sizeof(desc->lli),
 134                         DMA_TO_DEVICE);
 135 }
 136
 137 /*
 138  * Move a descriptor, including any children, to the free list.
 139  * `desc' must not be on any lists.
 140  */
 141 static void dwc_desc_put(struct dw_dma_chan *dwc, struct dw_desc *desc)
 142 {
 143         unsigned long flags;
 144
 145         if (desc) {
 146                 struct dw_desc *child;
 147
 148                 dwc_sync_desc_for_cpu(dwc, desc);
 149
 150                 spin_lock_irqsave(&dwc->lock, flags);
 151                 list_for_each_entry(child, &desc->tx_list, desc_node)
 152                         dev_vdbg(chan2dev(&dwc->chan),
 153                                         "moving child desc %p to freelist\n",
 154                                         child);
 155                 list_splice_init(&desc->tx_list, &dwc->free_list);
 156                 dev_vdbg(chan2dev(&dwc->chan), "moving desc %p to freelist\n", desc);
 157                 list_add(&desc->desc_node, &dwc->free_list);
 158                 spin_unlock_irqrestore(&dwc->lock, flags);
 159         }
 160 }
 161
 162 static void dwc_initialize(struct dw_dma_chan *dwc)
 163 {
 164         struct dw_dma *dw = to_dw_dma(dwc->chan.device);
 165         struct dw_dma_slave *dws = dwc->chan.private;
 166         u32 cfghi = DWC_CFGH_FIFO_MODE;
 167         u32 cfglo = DWC_CFGL_CH_PRIOR(dwc->priority);
 168
 169         if (dwc->initialized == true)
 170                 return;
 171
 172         if (dws) {
 173                 /*
 174                  * We need controller-specific data to set up slave
 175                  * transfers.
 176                  */
 177                 BUG_ON(!dws->dma_dev || dws->dma_dev != dw->dma.dev);
 178
 179                 cfghi = dws->cfg_hi;
 180                 cfglo |= dws->cfg_lo & ~DWC_CFGL_CH_PRIOR_MASK;
 181         } else {
 182                 if (dwc->direction == DMA_MEM_TO_DEV)
 183                         cfghi = DWC_CFGH_DST_PER(dwc->dma_sconfig.slave_id);
 184                 else if (dwc->direction == DMA_DEV_TO_MEM)
 185                         cfghi = DWC_CFGH_SRC_PER(dwc->dma_sconfig.slave_id);
 186         }
 187
 188         channel_writel(dwc, CFG_LO, cfglo);
 189         channel_writel(dwc, CFG_HI, cfghi);
 190
 191         /* Enable interrupts */
 192         channel_set_bit(dw, MASK.XFER, dwc->mask);
 193         channel_set_bit(dw, MASK.ERROR, dwc->mask);
 194
 195         dwc->initialized = true;
 196 }
 197
 198 /*----------------------------------------------------------------------*/
 199
 200 static inline unsigned int dwc_fast_fls(unsigned long long v)
 201 {
 202         /*
 203          * We can be a lot more clever here, but this should take care
 204          * of the most common optimization.
 205          */
 206         if (!(v & 7))
 207                 return 3;
 208         else if (!(v & 3))
 209                 return 2;
 210         else if (!(v & 1))
 211                 return 1;
 212         return 0;
 213 }
 214
 215 static inline void dwc_dump_chan_regs(struct dw_dma_chan *dwc)
 216 {
 217         dev_err(chan2dev(&dwc->chan),
 218                 "  SAR: 0x%x DAR: 0x%x LLP: 0x%x CTL: 0x%x:%08x\n",
 219                 channel_readl(dwc, SAR),
 220                 channel_readl(dwc, DAR),
 221                 channel_readl(dwc, LLP),
 222                 channel_readl(dwc, CTL_HI),
 223                 channel_readl(dwc, CTL_LO));
 224 }
 225
 226 static inline void dwc_chan_disable(struct dw_dma *dw, struct dw_dma_chan *dwc)
 227 {
 228         channel_clear_bit(dw, CH_EN, dwc->mask);
 229         while (dma_readl(dw, CH_EN) & dwc->mask)
 230                 cpu_relax();
 231 }
 232
 233 /*----------------------------------------------------------------------*/
 234
 235 /* Perform single block transfer */
 236 static inline void dwc_do_single_block(struct dw_dma_chan *dwc,
 237                                        struct dw_desc *desc)
 238 {
 239         struct dw_dma   *dw = to_dw_dma(dwc->chan.device);
 240         u32             ctllo;
 241
 242         /* Software emulation of LLP mode relies on interrupts to continue
 243          * multi block transfer. */
 244         ctllo = desc->lli.ctllo | DWC_CTLL_INT_EN;
 245
 246         channel_writel(dwc, SAR, desc->lli.sar);
 247         channel_writel(dwc, DAR, desc->lli.dar);
 248         channel_writel(dwc, CTL_LO, ctllo);
 249         channel_writel(dwc, CTL_HI, desc->lli.ctlhi);
 250         channel_set_bit(dw, CH_EN, dwc->mask);
 251
 252         /* Move pointer to next descriptor */
 253         dwc->tx_node_active = dwc->tx_node_active->next;
 254 }
 255
 256 /* Called with dwc->lock held and bh disabled */
 257 static void dwc_dostart(struct dw_dma_chan *dwc, struct dw_desc *first)
 258 {
 259         struct dw_dma   *dw = to_dw_dma(dwc->chan.device);
 260         unsigned long   was_soft_llp;
 261
 262         /* ASSERT:  channel is idle */
 263         if (dma_readl(dw, CH_EN) & dwc->mask) {
 264                 dev_err(chan2dev(&dwc->chan),
 265                         "BUG: Attempted to start non-idle channel\n");
 266                 dwc_dump_chan_regs(dwc);
 267
 268                 /* The tasklet will hopefully advance the queue... */
 269                 return;
 270         }
 271
 272         if (dwc->nollp) {
 273                 was_soft_llp = test_and_set_bit(DW_DMA_IS_SOFT_LLP,
 274                                                 &dwc->flags);
 275                 if (was_soft_llp) {
 276                         dev_err(chan2dev(&dwc->chan),
 277                                 "BUG: Attempted to start new LLP transfer "
 278                                 "inside ongoing one\n");
 279                         return;
 280                 }
 281
 282                 dwc_initialize(dwc);
 283
 284                 dwc->tx_list = &first->tx_list;
 285                 dwc->tx_node_active = &first->tx_list;
 286
 287                 dwc_do_single_block(dwc, first);
 288
 289                 return;
 290         }
 291
 292         dwc_initialize(dwc);
 293
 294         channel_writel(dwc, LLP, first->txd.phys);
 295         channel_writel(dwc, CTL_LO,
 296                         DWC_CTLL_LLP_D_EN | DWC_CTLL_LLP_S_EN);
 297         channel_writel(dwc, CTL_HI, 0);
 298         channel_set_bit(dw, CH_EN, dwc->mask);
 299 }
 300
 301 /*----------------------------------------------------------------------*/
 302
 303 static void
 304 dwc_descriptor_complete(struct dw_dma_chan *dwc, struct dw_desc *desc,
 305                 bool callback_required)
 306 {
 307         dma_async_tx_callback           callback = NULL;
 308         void                            *param = NULL;
 309         struct dma_async_tx_descriptor  *txd = &desc->txd;
 310         struct dw_desc                  *child;
 311         unsigned long                   flags;
 312
 313         dev_vdbg(chan2dev(&dwc->chan), "descriptor %u complete\n", txd->cookie);
 314
 315         spin_lock_irqsave(&dwc->lock, flags);
 316         dma_cookie_complete(txd);
 317         if (callback_required) {
 318                 callback = txd->callback;
 319                 param = txd->callback_param;
 320         }
 321
 322         dwc_sync_desc_for_cpu(dwc, desc);
 323
 324         /* async_tx_ack */
 325         list_for_each_entry(child, &desc->tx_list, desc_node)
 326                 async_tx_ack(&child->txd);
 327         async_tx_ack(&desc->txd);
 328
 329         list_splice_init(&desc->tx_list, &dwc->free_list);
 330         list_move(&desc->desc_node, &dwc->free_list);
 331
 332         if (!is_slave_direction(dwc->direction)) {
 333                 struct device *parent = chan2parent(&dwc->chan);
 334                 if (!(txd->flags & DMA_COMPL_SKIP_DEST_UNMAP)) {
 335                         if (txd->flags & DMA_COMPL_DEST_UNMAP_SINGLE)
 336                                 dma_unmap_single(parent, desc->lli.dar,
 337                                                 desc->len, DMA_FROM_DEVICE);
 338                         else
 339                                 dma_unmap_page(parent, desc->lli.dar,
 340                                                 desc->len, DMA_FROM_DEVICE);
 341                 }
 342                 if (!(txd->flags & DMA_COMPL_SKIP_SRC_UNMAP)) {
 343                         if (txd->flags & DMA_COMPL_SRC_UNMAP_SINGLE)
 344                                 dma_unmap_single(parent, desc->lli.sar,
 345                                                 desc->len, DMA_TO_DEVICE);
 346                         else
 347                                 dma_unmap_page(parent, desc->lli.sar,
 348                                                 desc->len, DMA_TO_DEVICE);
 349                 }
 350         }
 351
 352         spin_unlock_irqrestore(&dwc->lock, flags);
 353
 354         if (callback)
 355                 callback(param);
 356 }
 357
 358 static void dwc_complete_all(struct dw_dma *dw, struct dw_dma_chan *dwc)
 359 {
 360         struct dw_desc *desc, *_desc;
 361         LIST_HEAD(list);
 362         unsigned long flags;
 363
 364         spin_lock_irqsave(&dwc->lock, flags);
 365         if (dma_readl(dw, CH_EN) & dwc->mask) {
 366                 dev_err(chan2dev(&dwc->chan),
 367                         "BUG: XFER bit set, but channel not idle!\n");
 368
 369                 /* Try to continue after resetting the channel... */
 370                 dwc_chan_disable(dw, dwc);
 371         }
 372
 373         /*
 374          * Submit queued descriptors ASAP, i.e. before we go through
 375          * the completed ones.
 376          */
 377         list_splice_init(&dwc->active_list, &list);
 378         if (!list_empty(&dwc->queue)) {
 379                 list_move(dwc->queue.next, &dwc->active_list);
 380                 dwc_dostart(dwc, dwc_first_active(dwc));
 381         }
 382
 383         spin_unlock_irqrestore(&dwc->lock, flags);
 384
 385         list_for_each_entry_safe(desc, _desc, &list, desc_node)
 386                 dwc_descriptor_complete(dwc, desc, true);
 387 }
 388
 389 static void dwc_scan_descriptors(struct dw_dma *dw, struct dw_dma_chan *dwc)
 390 {
 391         dma_addr_t llp;
 392         struct dw_desc *desc, *_desc;
 393         struct dw_desc *child;
 394         u32 status_xfer;
 395         unsigned long flags;
 396
 397         spin_lock_irqsave(&dwc->lock, flags);
 398         llp = channel_readl(dwc, LLP);
 399         status_xfer = dma_readl(dw, RAW.XFER);
 400
 401         if (status_xfer & dwc->mask) {
 402                 /* Everything we've submitted is done */
 403                 dma_writel(dw, CLEAR.XFER, dwc->mask);
 404                 spin_unlock_irqrestore(&dwc->lock, flags);
 405
 406                 dwc_complete_all(dw, dwc);
 407                 return;
 408         }
 409
 410         if (list_empty(&dwc->active_list)) {
 411                 spin_unlock_irqrestore(&dwc->lock, flags);
 412                 return;
 413         }
 414
 415         dev_vdbg(chan2dev(&dwc->chan), "%s: llp=0x%llx\n", __func__,
 416                         (unsigned long long)llp);
 417
 418         list_for_each_entry_safe(desc, _desc, &dwc->active_list, desc_node) {
 419                 /* check first descriptors addr */
 420                 if (desc->txd.phys == llp) {
 421                         spin_unlock_irqrestore(&dwc->lock, flags);
 422                         return;
 423                 }
 424
 425                 /* check first descriptors llp */
 426                 if (desc->lli.llp == llp) {
 427                         /* This one is currently in progress */
 428                         spin_unlock_irqrestore(&dwc->lock, flags);
 429                         return;
 430                 }
 431
 432                 list_for_each_entry(child, &desc->tx_list, desc_node)
 433                         if (child->lli.llp == llp) {
 434                                 /* Currently in progress */
 435                                 spin_unlock_irqrestore(&dwc->lock, flags);
 436                                 return;
 437                         }
 438
 439                 /*
 440                  * No descriptors so far seem to be in progress, i.e.
 441                  * this one must be done.
 442                  */
 443                 spin_unlock_irqrestore(&dwc->lock, flags);
 444                 dwc_descriptor_complete(dwc, desc, true);
 445                 spin_lock_irqsave(&dwc->lock, flags);
 446         }
 447
 448         dev_err(chan2dev(&dwc->chan),
 449                 "BUG: All descriptors done, but channel not idle!\n");
 450
 451         /* Try to continue after resetting the channel... */
 452         dwc_chan_disable(dw, dwc);
 453
 454         if (!list_empty(&dwc->queue)) {
 455                 list_move(dwc->queue.next, &dwc->active_list);
 456                 dwc_dostart(dwc, dwc_first_active(dwc));
 457         }
 458         spin_unlock_irqrestore(&dwc->lock, flags);
 459 }
 460
 461 static inline void dwc_dump_lli(struct dw_dma_chan *dwc, struct dw_lli *lli)
 462 {
 463         dev_crit(chan2dev(&dwc->chan), "  desc: s0x%x d0x%x l0x%x c0x%x:%x\n",
 464                  lli->sar, lli->dar, lli->llp, lli->ctlhi, lli->ctllo);
 465 }
 466
 467 static void dwc_handle_error(struct dw_dma *dw, struct dw_dma_chan *dwc)
 468 {
 469         struct dw_desc *bad_desc;
 470         struct dw_desc *child;
 471         unsigned long flags;
 472
 473         dwc_scan_descriptors(dw, dwc);
 474
 475         spin_lock_irqsave(&dwc->lock, flags);
 476
 477         /*
 478          * The descriptor currently at the head of the active list is
 479          * borked. Since we don't have any way to report errors, we'll
 480          * just have to scream loudly and try to carry on.
 481          */
 482         bad_desc = dwc_first_active(dwc);
 483         list_del_init(&bad_desc->desc_node);
 484         list_move(dwc->queue.next, dwc->active_list.prev);
 485
 486         /* Clear the error flag and try to restart the controller */
 487         dma_writel(dw, CLEAR.ERROR, dwc->mask);
 488         if (!list_empty(&dwc->active_list))
 489                 dwc_dostart(dwc, dwc_first_active(dwc));
 490
 491         /*
 492          * WARN may seem harsh, but since this only happens
 493          * when someone submits a bad physical address in a
 494          * descriptor, we should consider ourselves lucky that the
 495          * controller flagged an error instead of scribbling over
 496          * random memory locations.
 497          */
 498         dev_WARN(chan2dev(&dwc->chan), "Bad descriptor submitted for DMA!\n"
 499                                        "  cookie: %d\n", bad_desc->txd.cookie);
 500         dwc_dump_lli(dwc, &bad_desc->lli);
 501         list_for_each_entry(child, &bad_desc->tx_list, desc_node)
 502                 dwc_dump_lli(dwc, &child->lli);
 503
 504         spin_unlock_irqrestore(&dwc->lock, flags);
 505
 506         /* Pretend the descriptor completed successfully */
 507         dwc_descriptor_complete(dwc, bad_desc, true);
 508 }
 509
 510 /* --------------------- Cyclic DMA API extensions -------------------- */
 511
 512 inline dma_addr_t dw_dma_get_src_addr(struct dma_chan *chan)
 513 {
 514         struct dw_dma_chan *dwc = to_dw_dma_chan(chan);
 515         return channel_readl(dwc, SAR);
 516 }
 517 EXPORT_SYMBOL(dw_dma_get_src_addr);
 518
 519 inline dma_addr_t dw_dma_get_dst_addr(struct dma_chan *chan)
 520 {
 521         struct dw_dma_chan *dwc = to_dw_dma_chan(chan);
 522         return channel_readl(dwc, DAR);
 523 }
 524 EXPORT_SYMBOL(dw_dma_get_dst_addr);
 525
 526 /* called with dwc->lock held and all DMAC interrupts disabled */
 527 static void dwc_handle_cyclic(struct dw_dma *dw, struct dw_dma_chan *dwc,
 528                 u32 status_err, u32 status_xfer)
 529 {
 530         unsigned long flags;
 531
 532         if (dwc->mask) {
 533                 void (*callback)(void *param);
 534                 void *callback_param;
 535
 536                 dev_vdbg(chan2dev(&dwc->chan), "new cyclic period llp 0x%08x\n",
 537                                 channel_readl(dwc, LLP));
 538
 539                 callback = dwc->cdesc->period_callback;
 540                 callback_param = dwc->cdesc->period_callback_param;
 541
 542                 if (callback)
 543                         callback(callback_param);
 544         }
 545
 546         /*
 547          * Error and transfer complete are highly unlikely, and will most
 548          * likely be due to a configuration error by the user.
 549          */
 550         if (unlikely(status_err & dwc->mask) ||
 551                         unlikely(status_xfer & dwc->mask)) {
 552                 int i;
 553
 554                 dev_err(chan2dev(&dwc->chan), "cyclic DMA unexpected %s "
 555                                 "interrupt, stopping DMA transfer\n",
 556                                 status_xfer ? "xfer" : "error");
 557
 558                 spin_lock_irqsave(&dwc->lock, flags);
 559
 560                 dwc_dump_chan_regs(dwc);
 561
 562                 dwc_chan_disable(dw, dwc);
 563
 564                 /* make sure DMA does not restart by loading a new list */
 565                 channel_writel(dwc, LLP, 0);
 566                 channel_writel(dwc, CTL_LO, 0);
 567                 channel_writel(dwc, CTL_HI, 0);
 568
 569                 dma_writel(dw, CLEAR.ERROR, dwc->mask);
 570                 dma_writel(dw, CLEAR.XFER, dwc->mask);
 571
 572                 for (i = 0; i < dwc->cdesc->periods; i++)
 573                         dwc_dump_lli(dwc, &dwc->cdesc->desc[i]->lli);
 574
 575                 spin_unlock_irqrestore(&dwc->lock, flags);
 576         }
 577 }
 578
 579 /* ------------------------------------------------------------------------- */
 580
 581 static void dw_dma_tasklet(unsigned long data)
 582 {
 583         struct dw_dma *dw = (struct dw_dma *)data;
 584         struct dw_dma_chan *dwc;
 585         u32 status_xfer;
 586         u32 status_err;
 587         int i;
 588
 589         status_xfer = dma_readl(dw, RAW.XFER);
 590         status_err = dma_readl(dw, RAW.ERROR);
 591
 592         dev_vdbg(dw->dma.dev, "%s: status_err=%x\n", __func__, status_err);
 593
 594         for (i = 0; i < dw->dma.chancnt; i++) {
 595                 dwc = &dw->chan[i];
 596                 if (test_bit(DW_DMA_IS_CYCLIC, &dwc->flags))
 597                         dwc_handle_cyclic(dw, dwc, status_err, status_xfer);
 598                 else if (status_err & (1 << i))
 599                         dwc_handle_error(dw, dwc);
 600                 else if (status_xfer & (1 << i)) {
 601                         unsigned long flags;
 602
 603                         spin_lock_irqsave(&dwc->lock, flags);
 604                         if (test_bit(DW_DMA_IS_SOFT_LLP, &dwc->flags)) {
 605                                 if (dwc->tx_node_active != dwc->tx_list) {
 606                                         struct dw_desc *desc =
 607                                                 to_dw_desc(dwc->tx_node_active);
 608
 609                                         dma_writel(dw, CLEAR.XFER, dwc->mask);
 610
 611                                         dwc_do_single_block(dwc, desc);
 612
 613                                         spin_unlock_irqrestore(&dwc->lock, flags);
 614                                         continue;
 615                                 }
 616                                 /* we are done here */
 617                                 clear_bit(DW_DMA_IS_SOFT_LLP, &dwc->flags);
 618                         }
 619                         spin_unlock_irqrestore(&dwc->lock, flags);
 620
 621                         dwc_scan_descriptors(dw, dwc);
 622                 }
 623         }
 624
 625         /*
 626          * Re-enable interrupts.
 627          */
 628         channel_set_bit(dw, MASK.XFER, dw->all_chan_mask);
 629         channel_set_bit(dw, MASK.ERROR, dw->all_chan_mask);
 630 }
 631
 632 static irqreturn_t dw_dma_interrupt(int irq, void *dev_id)
 633 {
 634         struct dw_dma *dw = dev_id;
 635         u32 status;
 636
 637         dev_vdbg(dw->dma.dev, "%s: status=0x%x\n", __func__,
 638                         dma_readl(dw, STATUS_INT));
 639
 640         /*
 641          * Just disable the interrupts. We'll turn them back on in the
 642          * softirq handler.
 643          */
 644         channel_clear_bit(dw, MASK.XFER, dw->all_chan_mask);
 645         channel_clear_bit(dw, MASK.ERROR, dw->all_chan_mask);
 646
 647         status = dma_readl(dw, STATUS_INT);
 648         if (status) {
 649                 dev_err(dw->dma.dev,
 650                         "BUG: Unexpected interrupts pending: 0x%x\n",
 651                         status);
 652
 653                 /* Try to recover */
 654                 channel_clear_bit(dw, MASK.XFER, (1 << 8) - 1);
 655                 channel_clear_bit(dw, MASK.SRC_TRAN, (1 << 8) - 1);
 656                 channel_clear_bit(dw, MASK.DST_TRAN, (1 << 8) - 1);
 657                 channel_clear_bit(dw, MASK.ERROR, (1 << 8) - 1);
 658         }
 659
 660         tasklet_schedule(&dw->tasklet);
 661
 662         return IRQ_HANDLED;
 663 }
 664
 665 /*----------------------------------------------------------------------*/
 666
 667 static dma_cookie_t dwc_tx_submit(struct dma_async_tx_descriptor *tx)
 668 {
 669         struct dw_desc          *desc = txd_to_dw_desc(tx);
 670         struct dw_dma_chan      *dwc = to_dw_dma_chan(tx->chan);
 671         dma_cookie_t            cookie;
 672         unsigned long           flags;
 673
 674         spin_lock_irqsave(&dwc->lock, flags);
 675         cookie = dma_cookie_assign(tx);
 676
 677         /*
 678          * REVISIT: We should attempt to chain as many descriptors as
 679          * possible, perhaps even appending to those already submitted
 680          * for DMA. But this is hard to do in a race-free manner.
 681          */
 682         if (list_empty(&dwc->active_list)) {
 683                 dev_vdbg(chan2dev(tx->chan), "%s: started %u\n", __func__,
 684                                 desc->txd.cookie);
 685                 list_add_tail(&desc->desc_node, &dwc->active_list);
 686                 dwc_dostart(dwc, dwc_first_active(dwc));
 687         } else {
 688                 dev_vdbg(chan2dev(tx->chan), "%s: queued %u\n", __func__,
 689                                 desc->txd.cookie);
 690
 691                 list_add_tail(&desc->desc_node, &dwc->queue);
 692         }
 693
 694         spin_unlock_irqrestore(&dwc->lock, flags);
 695
 696         return cookie;
 697 }
 698
 699 static struct dma_async_tx_descriptor *
 700 dwc_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, dma_addr_t src,
 701                 size_t len, unsigned long flags)
 702 {
 703         struct dw_dma_chan      *dwc = to_dw_dma_chan(chan);
 704         struct dw_dma_slave     *dws = chan->private;
 705         struct dw_desc          *desc;
 706         struct dw_desc          *first;
 707         struct dw_desc          *prev;
 708         size_t                  xfer_count;
 709         size_t                  offset;
 710         unsigned int            src_width;
 711         unsigned int            dst_width;
 712         unsigned int            data_width;
 713         u32                     ctllo;
 714
 715         dev_vdbg(chan2dev(chan),
 716                         "%s: d0x%llx s0x%llx l0x%zx f0x%lx\n", __func__,
 717                         (unsigned long long)dest, (unsigned long long)src,
 718                         len, flags);
 719
 720         if (unlikely(!len)) {
 721                 dev_dbg(chan2dev(chan), "%s: length is zero!\n", __func__);
 722                 return NULL;
 723         }
 724
 725         dwc->direction = DMA_MEM_TO_MEM;
 726
 727         data_width = min_t(unsigned int, dwc->dw->data_width[dwc_get_sms(dws)],
 728                                          dwc->dw->data_width[dwc_get_dms(dws)]);
 729
 730         src_width = dst_width = min_t(unsigned int, data_width,
 731                                       dwc_fast_fls(src | dest | len));
 732
 733         ctllo = DWC_DEFAULT_CTLLO(chan)
 734                         | DWC_CTLL_DST_WIDTH(dst_width)
 735                         | DWC_CTLL_SRC_WIDTH(src_width)
 736                         | DWC_CTLL_DST_INC
 737                         | DWC_CTLL_SRC_INC
 738                         | DWC_CTLL_FC_M2M;
 739         prev = first = NULL;
 740
 741         for (offset = 0; offset < len; offset += xfer_count << src_width) {
 742                 xfer_count = min_t(size_t, (len - offset) >> src_width,
 743                                            dwc->block_size);
 744
 745                 desc = dwc_desc_get(dwc);
 746                 if (!desc)
 747                         goto err_desc_get;
 748
 749                 desc->lli.sar = src + offset;
 750                 desc->lli.dar = dest + offset;
 751                 desc->lli.ctllo = ctllo;
 752                 desc->lli.ctlhi = xfer_count;
 753
 754                 if (!first) {
 755                         first = desc;
 756                 } else {
 757                         prev->lli.llp = desc->txd.phys;
 758                         dma_sync_single_for_device(chan2parent(chan),
 759                                         prev->txd.phys, sizeof(prev->lli),
 760                                         DMA_TO_DEVICE);
 761                         list_add_tail(&desc->desc_node,
 762                                         &first->tx_list);
 763                 }
 764                 prev = desc;
 765         }
 766
 767
 768         if (flags & DMA_PREP_INTERRUPT)
 769                 /* Trigger interrupt after last block */
 770                 prev->lli.ctllo |= DWC_CTLL_INT_EN;
 771
 772         prev->lli.llp = 0;
 773         dma_sync_single_for_device(chan2parent(chan),
 774                         prev->txd.phys, sizeof(prev->lli),
 775                         DMA_TO_DEVICE);
 776
 777         first->txd.flags = flags;
 778         first->len = len;
 779
 780         return &first->txd;
 781
 782 err_desc_get:
 783         dwc_desc_put(dwc, first);
 784         return NULL;
 785 }
 786
 787 static struct dma_async_tx_descriptor *
 788 dwc_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,
 789                 unsigned int sg_len, enum dma_transfer_direction direction,
 790                 unsigned long flags, void *context)
 791 {
 792         struct dw_dma_chan      *dwc = to_dw_dma_chan(chan);
 793         struct dw_dma_slave     *dws = chan->private;
 794         struct dma_slave_config *sconfig = &dwc->dma_sconfig;
 795         struct dw_desc          *prev;
 796         struct dw_desc          *first;
 797         u32                     ctllo;
 798         dma_addr_t              reg;
 799         unsigned int            reg_width;
 800         unsigned int            mem_width;
 801         unsigned int            data_width;
 802         unsigned int            i;
 803         struct scatterlist      *sg;
 804         size_t                  total_len = 0;
 805
 806         dev_vdbg(chan2dev(chan), "%s\n", __func__);
 807
 808         if (unlikely(!is_slave_direction(direction) || !sg_len))
 809                 return NULL;
 810
 811         dwc->direction = direction;
 812
 813         prev = first = NULL;
 814
 815         switch (direction) {
 816         case DMA_MEM_TO_DEV:
 817                 reg_width = __fls(sconfig->dst_addr_width);
 818                 reg = sconfig->dst_addr;
 819                 ctllo = (DWC_DEFAULT_CTLLO(chan)
 820                                 | DWC_CTLL_DST_WIDTH(reg_width)
 821                                 | DWC_CTLL_DST_FIX
 822                                 | DWC_CTLL_SRC_INC);
 823
 824                 ctllo |= sconfig->device_fc ? DWC_CTLL_FC(DW_DMA_FC_P_M2P) :
 825                         DWC_CTLL_FC(DW_DMA_FC_D_M2P);
 826
 827                 data_width = dwc->dw->data_width[dwc_get_sms(dws)];
 828
 829                 for_each_sg(sgl, sg, sg_len, i) {
 830                         struct dw_desc  *desc;
 831                         u32             len, dlen, mem;
 832
 833                         mem = sg_dma_address(sg);
 834                         len = sg_dma_len(sg);
 835
 836                         mem_width = min_t(unsigned int,
 837                                           data_width, dwc_fast_fls(mem | len));
 838
 839 slave_sg_todev_fill_desc:
 840                         desc = dwc_desc_get(dwc);
 841                         if (!desc) {
 842                                 dev_err(chan2dev(chan),
 843                                         "not enough descriptors available\n");
 844                                 goto err_desc_get;
 845                         }
 846
 847                         desc->lli.sar = mem;
 848                         desc->lli.dar = reg;
 849                         desc->lli.ctllo = ctllo | DWC_CTLL_SRC_WIDTH(mem_width);
 850                         if ((len >> mem_width) > dwc->block_size) {
 851                                 dlen = dwc->block_size << mem_width;
 852                                 mem += dlen;
 853                                 len -= dlen;
 854                         } else {
 855                                 dlen = len;
 856                                 len = 0;
 857                         }
 858
 859                         desc->lli.ctlhi = dlen >> mem_width;
 860
 861                         if (!first) {
 862                                 first = desc;
 863                         } else {
 864                                 prev->lli.llp = desc->txd.phys;
 865                                 dma_sync_single_for_device(chan2parent(chan),
 866                                                 prev->txd.phys,
 867                                                 sizeof(prev->lli),
 868                                                 DMA_TO_DEVICE);
 869                                 list_add_tail(&desc->desc_node,
 870                                                 &first->tx_list);
 871                         }
 872                         prev = desc;
 873                         total_len += dlen;
 874
 875                         if (len)
 876                                 goto slave_sg_todev_fill_desc;
 877                 }
 878                 break;
 879         case DMA_DEV_TO_MEM:
 880                 reg_width = __fls(sconfig->src_addr_width);
 881                 reg = sconfig->src_addr;
 882                 ctllo = (DWC_DEFAULT_CTLLO(chan)
 883                                 | DWC_CTLL_SRC_WIDTH(reg_width)
 884                                 | DWC_CTLL_DST_INC
 885                                 | DWC_CTLL_SRC_FIX);
 886
 887                 ctllo |= sconfig->device_fc ? DWC_CTLL_FC(DW_DMA_FC_P_P2M) :
 888                         DWC_CTLL_FC(DW_DMA_FC_D_P2M);
 889
 890                 data_width = dwc->dw->data_width[dwc_get_dms(dws)];
 891
 892                 for_each_sg(sgl, sg, sg_len, i) {
 893                         struct dw_desc  *desc;
 894                         u32             len, dlen, mem;
 895
 896                         mem = sg_dma_address(sg);
 897                         len = sg_dma_len(sg);
 898
 899                         mem_width = min_t(unsigned int,
 900                                           data_width, dwc_fast_fls(mem | len));
 901
 902 slave_sg_fromdev_fill_desc:
 903                         desc = dwc_desc_get(dwc);
 904                         if (!desc) {
 905                                 dev_err(chan2dev(chan),
 906                                                 "not enough descriptors available\n");
 907                                 goto err_desc_get;
 908                         }
 909
 910                         desc->lli.sar = reg;
 911                         desc->lli.dar = mem;
 912                         desc->lli.ctllo = ctllo | DWC_CTLL_DST_WIDTH(mem_width);
 913                         if ((len >> reg_width) > dwc->block_size) {
 914                                 dlen = dwc->block_size << reg_width;
 915                                 mem += dlen;
 916                                 len -= dlen;
 917                         } else {
 918                                 dlen = len;
 919                                 len = 0;
 920                         }
 921                         desc->lli.ctlhi = dlen >> reg_width;
 922
 923                         if (!first) {
 924                                 first = desc;
 925                         } else {
 926                                 prev->lli.llp = desc->txd.phys;
 927                                 dma_sync_single_for_device(chan2parent(chan),
 928                                                 prev->txd.phys,
 929                                                 sizeof(prev->lli),
 930                                                 DMA_TO_DEVICE);
 931                                 list_add_tail(&desc->desc_node,
 932                                                 &first->tx_list);
 933                         }
 934                         prev = desc;
 935                         total_len += dlen;
 936
 937                         if (len)
 938                                 goto slave_sg_fromdev_fill_desc;
 939                 }
 940                 break;
 941         default:
 942                 return NULL;
 943         }
 944
 945         if (flags & DMA_PREP_INTERRUPT)
 946                 /* Trigger interrupt after last block */
 947                 prev->lli.ctllo |= DWC_CTLL_INT_EN;
 948
 949         prev->lli.llp = 0;
 950         dma_sync_single_for_device(chan2parent(chan),
 951                         prev->txd.phys, sizeof(prev->lli),
 952                         DMA_TO_DEVICE);
 953
 954         first->len = total_len;
 955
 956         return &first->txd;
 957
 958 err_desc_get:
 959         dwc_desc_put(dwc, first);
 960         return NULL;
 961 }
 962
 963 /*
 964  * Fix sconfig's burst size according to dw_dmac. We need to convert them as:
 965  * 1 -> 0, 4 -> 1, 8 -> 2, 16 -> 3.
 966  *
 967  * NOTE: burst size 2 is not supported by controller.
 968  *
 969  * This can be done by finding least significant bit set: n & (n - 1)
 970  */
 971 static inline void convert_burst(u32 *maxburst)
 972 {
 973         if (*maxburst > 1)
 974                 *maxburst = fls(*maxburst) - 2;
 975         else
 976                 *maxburst = 0;
 977 }
 978
 979 static int
 980 set_runtime_config(struct dma_chan *chan, struct dma_slave_config *sconfig)
 981 {
 982         struct dw_dma_chan *dwc = to_dw_dma_chan(chan);
 983
 984         /* Check if chan will be configured for slave transfers */
 985         if (!is_slave_direction(sconfig->direction))
 986                 return -EINVAL;
 987
 988         memcpy(&dwc->dma_sconfig, sconfig, sizeof(*sconfig));
 989         dwc->direction = sconfig->direction;
 990
 991         convert_burst(&dwc->dma_sconfig.src_maxburst);
 992         convert_burst(&dwc->dma_sconfig.dst_maxburst);
 993
 994         return 0;
 995 }
 996
 997 static inline void dwc_chan_pause(struct dw_dma_chan *dwc)
 998 {
 999         u32 cfglo = channel_readl(dwc, CFG_LO);
1000
1001         channel_writel(dwc, CFG_LO, cfglo | DWC_CFGL_CH_SUSP);
1002         while (!(channel_readl(dwc, CFG_LO) & DWC_CFGL_FIFO_EMPTY))
1003                 cpu_relax();
1004
1005         dwc->paused = true;
1006 }
1007
1008 static inline void dwc_chan_resume(struct dw_dma_chan *dwc)
1009 {
1010         u32 cfglo = channel_readl(dwc, CFG_LO);
1011
1012         channel_writel(dwc, CFG_LO, cfglo & ~DWC_CFGL_CH_SUSP);
1013
1014         dwc->paused = false;
1015 }
1016
1017 static int dwc_control(struct dma_chan *chan, enum dma_ctrl_cmd cmd,
1018                        unsigned long arg)
1019 {
1020         struct dw_dma_chan      *dwc = to_dw_dma_chan(chan);
1021         struct dw_dma           *dw = to_dw_dma(chan->device);
1022         struct dw_desc          *desc, *_desc;
1023         unsigned long           flags;
1024         LIST_HEAD(list);
1025
1026         if (cmd == DMA_PAUSE) {
1027                 spin_lock_irqsave(&dwc->lock, flags);
1028
1029                 dwc_chan_pause(dwc);
1030
1031                 spin_unlock_irqrestore(&dwc->lock, flags);
1032         } else if (cmd == DMA_RESUME) {
1033                 if (!dwc->paused)
1034                         return 0;
1035
1036                 spin_lock_irqsave(&dwc->lock, flags);
1037
1038                 dwc_chan_resume(dwc);
1039
1040                 spin_unlock_irqrestore(&dwc->lock, flags);
1041         } else if (cmd == DMA_TERMINATE_ALL) {
1042                 spin_lock_irqsave(&dwc->lock, flags);
1043
1044                 clear_bit(DW_DMA_IS_SOFT_LLP, &dwc->flags);
1045
1046                 dwc_chan_disable(dw, dwc);
1047
1048                 dwc->paused = false;
1049
1050                 /* active_list entries will end up before queued entries */
1051                 list_splice_init(&dwc->queue, &list);
1052                 list_splice_init(&dwc->active_list, &list);
1053
1054                 spin_unlock_irqrestore(&dwc->lock, flags);
1055
1056                 /* Flush all pending and queued descriptors */
1057                 list_for_each_entry_safe(desc, _desc, &list, desc_node)
1058                         dwc_descriptor_complete(dwc, desc, false);
1059         } else if (cmd == DMA_SLAVE_CONFIG) {
1060                 return set_runtime_config(chan, (struct dma_slave_config *)arg);
1061         } else {
1062                 return -ENXIO;
1063         }
1064
1065         return 0;
1066 }
1067
1068 static enum dma_status
1069 dwc_tx_status(struct dma_chan *chan,
1070               dma_cookie_t cookie,
1071               struct dma_tx_state *txstate)
1072 {
1073         struct dw_dma_chan      *dwc = to_dw_dma_chan(chan);
1074         enum dma_status         ret;
1075
1076         ret = dma_cookie_status(chan, cookie, txstate);
1077         if (ret != DMA_SUCCESS) {
1078                 dwc_scan_descriptors(to_dw_dma(chan->device), dwc);
1079
1080                 ret = dma_cookie_status(chan, cookie, txstate);
1081         }
1082
1083         if (ret != DMA_SUCCESS)
1084                 dma_set_residue(txstate, dwc_first_active(dwc)->len);
1085
1086         if (dwc->paused)
1087                 return DMA_PAUSED;
1088
1089         return ret;
1090 }
1091
1092 static void dwc_issue_pending(struct dma_chan *chan)
1093 {
1094         struct dw_dma_chan      *dwc = to_dw_dma_chan(chan);
1095
1096         if (!list_empty(&dwc->queue))
1097                 dwc_scan_descriptors(to_dw_dma(chan->device), dwc);
1098 }
1099
1100 static int dwc_alloc_chan_resources(struct dma_chan *chan)
1101 {
1102         struct dw_dma_chan      *dwc = to_dw_dma_chan(chan);
1103         struct dw_dma           *dw = to_dw_dma(chan->device);
1104         struct dw_desc          *desc;
1105         int                     i;
1106         unsigned long           flags;
1107         int                     ret;
1108
1109         dev_vdbg(chan2dev(chan), "%s\n", __func__);
1110
1111         /* ASSERT:  channel is idle */
1112         if (dma_readl(dw, CH_EN) & dwc->mask) {
1113                 dev_dbg(chan2dev(chan), "DMA channel not idle?\n");
1114                 return -EIO;
1115         }
1116
1117         dma_cookie_init(chan);
1118
1119         /*
1120          * NOTE: some controllers may have additional features that we
1121          * need to initialize here, like "scatter-gather" (which
1122          * doesn't mean what you think it means), and status writeback.
1123          */
1124
1125         spin_lock_irqsave(&dwc->lock, flags);
1126         i = dwc->descs_allocated;
1127         while (dwc->descs_allocated < NR_DESCS_PER_CHANNEL) {
1128                 spin_unlock_irqrestore(&dwc->lock, flags);
1129
1130                 desc = kzalloc(sizeof(struct dw_desc), GFP_KERNEL);
1131                 if (!desc)
1132                         goto err_desc_alloc;
1133
1134                 INIT_LIST_HEAD(&desc->tx_list);
1135                 dma_async_tx_descriptor_init(&desc->txd, chan);
1136                 desc->txd.tx_submit = dwc_tx_submit;
1137                 desc->txd.flags = DMA_CTRL_ACK;
1138                 desc->txd.phys = dma_map_single(chan2parent(chan), &desc->lli,
1139                                 sizeof(desc->lli), DMA_TO_DEVICE);
1140                 ret = dma_mapping_error(chan2parent(chan), desc->txd.phys);
1141                 if (ret)
1142                         goto err_desc_alloc;
1143
1144                 dwc_desc_put(dwc, desc);
1145
1146                 spin_lock_irqsave(&dwc->lock, flags);
1147                 i = ++dwc->descs_allocated;
1148         }
1149
1150         spin_unlock_irqrestore(&dwc->lock, flags);
1151
1152         dev_dbg(chan2dev(chan), "%s: allocated %d descriptors\n", __func__, i);
1153
1154         return i;
1155
1156 err_desc_alloc:
1157         kfree(desc);
1158
1159         dev_info(chan2dev(chan), "only allocated %d descriptors\n", i);
1160
1161         return i;
1162 }
1163
1164 static void dwc_free_chan_resources(struct dma_chan *chan)
1165 {
1166         struct dw_dma_chan      *dwc = to_dw_dma_chan(chan);
1167         struct dw_dma           *dw = to_dw_dma(chan->device);
1168         struct dw_desc          *desc, *_desc;
1169         unsigned long           flags;
1170         LIST_HEAD(list);
1171
1172         dev_dbg(chan2dev(chan), "%s: descs allocated=%u\n", __func__,
1173                         dwc->descs_allocated);
1174
1175         /* ASSERT:  channel is idle */
1176         BUG_ON(!list_empty(&dwc->active_list));
1177         BUG_ON(!list_empty(&dwc->queue));
1178         BUG_ON(dma_readl(to_dw_dma(chan->device), CH_EN) & dwc->mask);
1179
1180         spin_lock_irqsave(&dwc->lock, flags);
1181         list_splice_init(&dwc->free_list, &list);
1182         dwc->descs_allocated = 0;
1183         dwc->initialized = false;
1184
1185         /* Disable interrupts */
1186         channel_clear_bit(dw, MASK.XFER, dwc->mask);
1187         channel_clear_bit(dw, MASK.ERROR, dwc->mask);
1188
1189         spin_unlock_irqrestore(&dwc->lock, flags);
1190
1191         list_for_each_entry_safe(desc, _desc, &list, desc_node) {
1192                 dev_vdbg(chan2dev(chan), "  freeing descriptor %p\n", desc);
1193                 dma_unmap_single(chan2parent(chan), desc->txd.phys,
1194                                 sizeof(desc->lli), DMA_TO_DEVICE);
1195                 kfree(desc);
1196         }
1197
1198         dev_vdbg(chan2dev(chan), "%s: done\n", __func__);
1199 }
1200
1201 bool dw_dma_generic_filter(struct dma_chan *chan, void *param)
1202 {
1203         struct dw_dma *dw = to_dw_dma(chan->device);
1204         static struct dw_dma *last_dw;
1205         static char *last_bus_id;
1206         int i = -1;
1207
1208         /*
1209          * dmaengine framework calls this routine for all channels of all dma
1210          * controller, until true is returned. If 'param' bus_id is not
1211          * registered with a dma controller (dw), then there is no need of
1212          * running below function for all channels of dw.
1213          *
1214          * This block of code does this by saving the parameters of last
1215          * failure. If dw and param are same, i.e. trying on same dw with
1216          * different channel, return false.
1217          */
1218         if ((last_dw == dw) && (last_bus_id == param))
1219                 return false;
1220         /*
1221          * Return true:
1222          * - If dw_dma's platform data is not filled with slave info, then all
1223          *   dma controllers are fine for transfer.
1224          * - Or if param is NULL
1225          */
1226         if (!dw->sd || !param)
1227                 return true;
1228
1229         while (++i < dw->sd_count) {
1230                 if (!strcmp(dw->sd[i].bus_id, param)) {
1231                         chan->private = &dw->sd[i];
1232                         last_dw = NULL;
1233                         last_bus_id = NULL;
1234
1235                         return true;
1236                 }
1237         }
1238
1239         last_dw = dw;
1240         last_bus_id = param;
1241         return false;
1242 }
1243 EXPORT_SYMBOL(dw_dma_generic_filter);
1244
1245 /* --------------------- Cyclic DMA API extensions -------------------- */
1246
1247 /**
1248  * dw_dma_cyclic_start - start the cyclic DMA transfer
1249  * @chan: the DMA channel to start
1250  *
1251  * Must be called with soft interrupts disabled. Returns zero on success or
1252  * -errno on failure.
1253  */
1254 int dw_dma_cyclic_start(struct dma_chan *chan)
1255 {
1256         struct dw_dma_chan      *dwc = to_dw_dma_chan(chan);
1257         struct dw_dma           *dw = to_dw_dma(dwc->chan.device);
1258         unsigned long           flags;
1259
1260         if (!test_bit(DW_DMA_IS_CYCLIC, &dwc->flags)) {
1261                 dev_err(chan2dev(&dwc->chan), "missing prep for cyclic DMA\n");
1262                 return -ENODEV;
1263         }
1264
1265         spin_lock_irqsave(&dwc->lock, flags);
1266
1267         /* assert channel is idle */
1268         if (dma_readl(dw, CH_EN) & dwc->mask) {
1269                 dev_err(chan2dev(&dwc->chan),
1270                         "BUG: Attempted to start non-idle channel\n");
1271                 dwc_dump_chan_regs(dwc);
1272                 spin_unlock_irqrestore(&dwc->lock, flags);
1273                 return -EBUSY;
1274         }
1275
1276         dma_writel(dw, CLEAR.ERROR, dwc->mask);
1277         dma_writel(dw, CLEAR.XFER, dwc->mask);
1278
1279         /* setup DMAC channel registers */
1280         channel_writel(dwc, LLP, dwc->cdesc->desc[0]->txd.phys);
1281         channel_writel(dwc, CTL_LO, DWC_CTLL_LLP_D_EN | DWC_CTLL_LLP_S_EN);
1282         channel_writel(dwc, CTL_HI, 0);
1283
1284         channel_set_bit(dw, CH_EN, dwc->mask);
1285
1286         spin_unlock_irqrestore(&dwc->lock, flags);
1287
1288         return 0;
1289 }
1290 EXPORT_SYMBOL(dw_dma_cyclic_start);
1291
1292 /**
1293  * dw_dma_cyclic_stop - stop the cyclic DMA transfer
1294  * @chan: the DMA channel to stop
1295  *
1296  * Must be called with soft interrupts disabled.
1297  */
1298 void dw_dma_cyclic_stop(struct dma_chan *chan)
1299 {
1300         struct dw_dma_chan      *dwc = to_dw_dma_chan(chan);
1301         struct dw_dma           *dw = to_dw_dma(dwc->chan.device);
1302         unsigned long           flags;
1303
1304         spin_lock_irqsave(&dwc->lock, flags);
1305
1306         dwc_chan_disable(dw, dwc);
1307
1308         spin_unlock_irqrestore(&dwc->lock, flags);
1309 }
1310 EXPORT_SYMBOL(dw_dma_cyclic_stop);
1311
1312 /**
1313  * dw_dma_cyclic_prep - prepare the cyclic DMA transfer
1314  * @chan: the DMA channel to prepare
1315  * @buf_addr: physical DMA address where the buffer starts
1316  * @buf_len: total number of bytes for the entire buffer
1317  * @period_len: number of bytes for each period
1318  * @direction: transfer direction, to or from device
1319  *
1320  * Must be called before trying to start the transfer. Returns a valid struct
1321  * dw_cyclic_desc if successful or an ERR_PTR(-errno) if not successful.
1322  */
1323 struct dw_cyclic_desc *dw_dma_cyclic_prep(struct dma_chan *chan,
1324                 dma_addr_t buf_addr, size_t buf_len, size_t period_len,
1325                 enum dma_transfer_direction direction)
1326 {
1327         struct dw_dma_chan              *dwc = to_dw_dma_chan(chan);
1328         struct dma_slave_config         *sconfig = &dwc->dma_sconfig;
1329         struct dw_cyclic_desc           *cdesc;
1330         struct dw_cyclic_desc           *retval = NULL;
1331         struct dw_desc                  *desc;
1332         struct dw_desc                  *last = NULL;
1333         unsigned long                   was_cyclic;
1334         unsigned int                    reg_width;
1335         unsigned int                    periods;
1336         unsigned int                    i;
1337         unsigned long                   flags;
1338
1339         spin_lock_irqsave(&dwc->lock, flags);
1340         if (dwc->nollp) {
1341                 spin_unlock_irqrestore(&dwc->lock, flags);
1342                 dev_dbg(chan2dev(&dwc->chan),
1343                                 "channel doesn't support LLP transfers\n");
1344                 return ERR_PTR(-EINVAL);
1345         }
1346
1347         if (!list_empty(&dwc->queue) || !list_empty(&dwc->active_list)) {
1348                 spin_unlock_irqrestore(&dwc->lock, flags);
1349                 dev_dbg(chan2dev(&dwc->chan),
1350                                 "queue and/or active list are not empty\n");
1351                 return ERR_PTR(-EBUSY);
1352         }
1353
1354         was_cyclic = test_and_set_bit(DW_DMA_IS_CYCLIC, &dwc->flags);
1355         spin_unlock_irqrestore(&dwc->lock, flags);
1356         if (was_cyclic) {
1357                 dev_dbg(chan2dev(&dwc->chan),
1358                                 "channel already prepared for cyclic DMA\n");
1359                 return ERR_PTR(-EBUSY);
1360         }
1361
1362         retval = ERR_PTR(-EINVAL);
1363
1364         if (unlikely(!is_slave_direction(direction)))
1365                 goto out_err;
1366
1367         dwc->direction = direction;
1368
1369         if (direction == DMA_MEM_TO_DEV)
1370                 reg_width = __ffs(sconfig->dst_addr_width);
1371         else
1372                 reg_width = __ffs(sconfig->src_addr_width);
1373
1374         periods = buf_len / period_len;
1375
1376         /* Check for too big/unaligned periods and unaligned DMA buffer. */
1377         if (period_len > (dwc->block_size << reg_width))
1378                 goto out_err;
1379         if (unlikely(period_len & ((1 << reg_width) - 1)))
1380                 goto out_err;
1381         if (unlikely(buf_addr & ((1 << reg_width) - 1)))
1382                 goto out_err;
1383
1384         retval = ERR_PTR(-ENOMEM);
1385
1386         if (periods > NR_DESCS_PER_CHANNEL)
1387                 goto out_err;
1388
1389         cdesc = kzalloc(sizeof(struct dw_cyclic_desc), GFP_KERNEL);
1390         if (!cdesc)
1391                 goto out_err;
1392
1393         cdesc->desc = kzalloc(sizeof(struct dw_desc *) * periods, GFP_KERNEL);
1394         if (!cdesc->desc)
1395                 goto out_err_alloc;
1396
1397         for (i = 0; i < periods; i++) {
1398                 desc = dwc_desc_get(dwc);
1399                 if (!desc)
1400                         goto out_err_desc_get;
1401
1402                 switch (direction) {
1403                 case DMA_MEM_TO_DEV:
1404                         desc->lli.dar = sconfig->dst_addr;
1405                         desc->lli.sar = buf_addr + (period_len * i);
1406                         desc->lli.ctllo = (DWC_DEFAULT_CTLLO(chan)
1407                                         | DWC_CTLL_DST_WIDTH(reg_width)
1408                                         | DWC_CTLL_SRC_WIDTH(reg_width)
1409                                         | DWC_CTLL_DST_FIX
1410                                         | DWC_CTLL_SRC_INC
1411                                         | DWC_CTLL_INT_EN);
1412
1413                         desc->lli.ctllo |= sconfig->device_fc ?
1414                                 DWC_CTLL_FC(DW_DMA_FC_P_M2P) :
1415                                 DWC_CTLL_FC(DW_DMA_FC_D_M2P);
1416
1417                         break;
1418                 case DMA_DEV_TO_MEM:
1419                         desc->lli.dar = buf_addr + (period_len * i);
1420                         desc->lli.sar = sconfig->src_addr;
1421                         desc->lli.ctllo = (DWC_DEFAULT_CTLLO(chan)
1422                                         | DWC_CTLL_SRC_WIDTH(reg_width)
1423                                         | DWC_CTLL_DST_WIDTH(reg_width)
1424                                         | DWC_CTLL_DST_INC
1425                                         | DWC_CTLL_SRC_FIX
1426                                         | DWC_CTLL_INT_EN);
1427
1428                         desc->lli.ctllo |= sconfig->device_fc ?
1429                                 DWC_CTLL_FC(DW_DMA_FC_P_P2M) :
1430                                 DWC_CTLL_FC(DW_DMA_FC_D_P2M);
1431
1432                         break;
1433                 default:
1434                         break;
1435                 }
1436
1437                 desc->lli.ctlhi = (period_len >> reg_width);
1438                 cdesc->desc[i] = desc;
1439
1440                 if (last) {
1441                         last->lli.llp = desc->txd.phys;
1442                         dma_sync_single_for_device(chan2parent(chan),
1443                                         last->txd.phys, sizeof(last->lli),
1444                                         DMA_TO_DEVICE);
1445                 }
1446
1447                 last = desc;
1448         }
1449
1450         /* lets make a cyclic list */
1451         last->lli.llp = cdesc->desc[0]->txd.phys;
1452         dma_sync_single_for_device(chan2parent(chan), last->txd.phys,
1453                         sizeof(last->lli), DMA_TO_DEVICE);
1454
1455         dev_dbg(chan2dev(&dwc->chan), "cyclic prepared buf 0x%llx len %zu "
1456                         "period %zu periods %d\n", (unsigned long long)buf_addr,
1457                         buf_len, period_len, periods);
1458
1459         cdesc->periods = periods;
1460         dwc->cdesc = cdesc;
1461
1462         return cdesc;
1463
1464 out_err_desc_get:
1465         while (i--)
1466                 dwc_desc_put(dwc, cdesc->desc[i]);
1467 out_err_alloc:
1468         kfree(cdesc);
1469 out_err:
1470         clear_bit(DW_DMA_IS_CYCLIC, &dwc->flags);
1471         return (struct dw_cyclic_desc *)retval;
1472 }
1473 EXPORT_SYMBOL(dw_dma_cyclic_prep);
1474
1475 /**
1476  * dw_dma_cyclic_free - free a prepared cyclic DMA transfer
1477  * @chan: the DMA channel to free
1478  */
1479 void dw_dma_cyclic_free(struct dma_chan *chan)
1480 {
1481         struct dw_dma_chan      *dwc = to_dw_dma_chan(chan);
1482         struct dw_dma           *dw = to_dw_dma(dwc->chan.device);
1483         struct dw_cyclic_desc   *cdesc = dwc->cdesc;
1484         int                     i;
1485         unsigned long           flags;
1486
1487         dev_dbg(chan2dev(&dwc->chan), "%s\n", __func__);
1488
1489         if (!cdesc)
1490                 return;
1491
1492         spin_lock_irqsave(&dwc->lock, flags);
1493
1494         dwc_chan_disable(dw, dwc);
1495
1496         dma_writel(dw, CLEAR.ERROR, dwc->mask);
1497         dma_writel(dw, CLEAR.XFER, dwc->mask);
1498
1499         spin_unlock_irqrestore(&dwc->lock, flags);
1500
1501         for (i = 0; i < cdesc->periods; i++)
1502                 dwc_desc_put(dwc, cdesc->desc[i]);
1503
1504         kfree(cdesc->desc);
1505         kfree(cdesc);
1506
1507         clear_bit(DW_DMA_IS_CYCLIC, &dwc->flags);
1508 }
1509 EXPORT_SYMBOL(dw_dma_cyclic_free);
1510
1511 /*----------------------------------------------------------------------*/
1512
1513 static void dw_dma_off(struct dw_dma *dw)
1514 {
1515         int i;
1516
1517         dma_writel(dw, CFG, 0);
1518
1519         channel_clear_bit(dw, MASK.XFER, dw->all_chan_mask);
1520         channel_clear_bit(dw, MASK.SRC_TRAN, dw->all_chan_mask);
1521         channel_clear_bit(dw, MASK.DST_TRAN, dw->all_chan_mask);
1522         channel_clear_bit(dw, MASK.ERROR, dw->all_chan_mask);
1523
1524         while (dma_readl(dw, CFG) & DW_CFG_DMA_EN)
1525                 cpu_relax();
1526
1527         for (i = 0; i < dw->dma.chancnt; i++)
1528                 dw->chan[i].initialized = false;
1529 }
1530
1531 #ifdef CONFIG_OF
1532 static struct dw_dma_platform_data *
1533 dw_dma_parse_dt(struct platform_device *pdev)
1534 {
1535         struct device_node *sn, *cn, *np = pdev->dev.of_node;
1536         struct dw_dma_platform_data *pdata;
1537         struct dw_dma_slave *sd;
1538         u32 tmp, arr[4];
1539
1540         if (!np) {
1541                 dev_err(&pdev->dev, "Missing DT data\n");
1542                 return NULL;
1543         }
1544
1545         pdata = devm_kzalloc(&pdev->dev, sizeof(*pdata), GFP_KERNEL);
1546         if (!pdata)
1547                 return NULL;
1548
1549         if (of_property_read_u32(np, "nr_channels", &pdata->nr_channels))
1550                 return NULL;
1551
1552         if (of_property_read_bool(np, "is_private"))
1553                 pdata->is_private = true;
1554
1555         if (!of_property_read_u32(np, "chan_allocation_order", &tmp))
1556                 pdata->chan_allocation_order = (unsigned char)tmp;
1557
1558         if (!of_property_read_u32(np, "chan_priority", &tmp))
1559                 pdata->chan_priority = tmp;
1560
1561         if (!of_property_read_u32(np, "block_size", &tmp))
1562                 pdata->block_size = tmp;
1563
1564         if (!of_property_read_u32(np, "nr_masters", &tmp)) {
1565                 if (tmp > 4)
1566                         return NULL;
1567
1568                 pdata->nr_masters = tmp;
1569         }
1570
1571         if (!of_property_read_u32_array(np, "data_width", arr,
1572                                 pdata->nr_masters))
1573                 for (tmp = 0; tmp < pdata->nr_masters; tmp++)
1574                         pdata->data_width[tmp] = arr[tmp];
1575
1576         /* parse slave data */
1577         sn = of_find_node_by_name(np, "slave_info");
1578         if (!sn)
1579                 return pdata;
1580
1581         /* calculate number of slaves */
1582         tmp = of_get_child_count(sn);
1583         if (!tmp)
1584                 return NULL;
1585
1586         sd = devm_kzalloc(&pdev->dev, sizeof(*sd) * tmp, GFP_KERNEL);
1587         if (!sd)
1588                 return NULL;
1589
1590         pdata->sd = sd;
1591         pdata->sd_count = tmp;
1592
1593         for_each_child_of_node(sn, cn) {
1594                 sd->dma_dev = &pdev->dev;
1595                 of_property_read_string(cn, "bus_id", &sd->bus_id);
1596                 of_property_read_u32(cn, "cfg_hi", &sd->cfg_hi);
1597                 of_property_read_u32(cn, "cfg_lo", &sd->cfg_lo);
1598                 if (!of_property_read_u32(cn, "src_master", &tmp))
1599                         sd->src_master = tmp;
1600
1601                 if (!of_property_read_u32(cn, "dst_master", &tmp))
1602                         sd->dst_master = tmp;
1603                 sd++;
1604         }
1605
1606         return pdata;
1607 }
1608 #else
1609 static inline struct dw_dma_platform_data *
1610 dw_dma_parse_dt(struct platform_device *pdev)
1611 {
1612         return NULL;
1613 }
1614 #endif
1615
1616 static int dw_probe(struct platform_device *pdev)
1617 {
1618         struct dw_dma_platform_data *pdata;
1619         struct resource         *io;
1620         struct dw_dma           *dw;
1621         size_t                  size;
1622         void __iomem            *regs;
1623         bool                    autocfg;
1624         unsigned int            dw_params;
1625         unsigned int            nr_channels;
1626         unsigned int            max_blk_size = 0;
1627         int                     irq;
1628         int                     err;
1629         int                     i;
1630
1631         io = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1632         if (!io)
1633                 return -EINVAL;
1634
1635         irq = platform_get_irq(pdev, 0);
1636         if (irq < 0)
1637                 return irq;
1638
1639         regs = devm_request_and_ioremap(&pdev->dev, io);
1640         if (!regs)
1641                 return -EBUSY;
1642
1643         dw_params = dma_read_byaddr(regs, DW_PARAMS);
1644         autocfg = dw_params >> DW_PARAMS_EN & 0x1;
1645
1646         pdata = dev_get_platdata(&pdev->dev);
1647         if (!pdata)
1648                 pdata = dw_dma_parse_dt(pdev);
1649
1650         if (!pdata && autocfg) {
1651                 pdata = devm_kzalloc(&pdev->dev, sizeof(*pdata), GFP_KERNEL);
1652                 if (!pdata)
1653                         return -ENOMEM;
1654
1655                 /* Fill platform data with the default values */
1656                 pdata->is_private = true;
1657                 pdata->chan_allocation_order = CHAN_ALLOCATION_ASCENDING;
1658                 pdata->chan_priority = CHAN_PRIORITY_ASCENDING;
1659         } else if (!pdata || pdata->nr_channels > DW_DMA_MAX_NR_CHANNELS)
1660                 return -EINVAL;
1661
1662         if (autocfg)
1663                 nr_channels = (dw_params >> DW_PARAMS_NR_CHAN & 0x7) + 1;
1664         else
1665                 nr_channels = pdata->nr_channels;
1666
1667         size = sizeof(struct dw_dma) + nr_channels * sizeof(struct dw_dma_chan);
1668         dw = devm_kzalloc(&pdev->dev, size, GFP_KERNEL);
1669         if (!dw)
1670                 return -ENOMEM;
1671
1672         dw->clk = devm_clk_get(&pdev->dev, "hclk");
1673         if (IS_ERR(dw->clk))
1674                 return PTR_ERR(dw->clk);
1675         clk_prepare_enable(dw->clk);
1676
1677         dw->regs = regs;
1678         dw->sd = pdata->sd;
1679         dw->sd_count = pdata->sd_count;
1680
1681         /* get hardware configuration parameters */
1682         if (autocfg) {
1683                 max_blk_size = dma_readl(dw, MAX_BLK_SIZE);
1684
1685                 dw->nr_masters = (dw_params >> DW_PARAMS_NR_MASTER & 3) + 1;
1686                 for (i = 0; i < dw->nr_masters; i++) {
1687                         dw->data_width[i] =
1688                                 (dw_params >> DW_PARAMS_DATA_WIDTH(i) & 3) + 2;
1689                 }
1690         } else {
1691                 dw->nr_masters = pdata->nr_masters;
1692                 memcpy(dw->data_width, pdata->data_width, 4);
1693         }
1694
1695         /* Calculate all channel mask before DMA setup */
1696         dw->all_chan_mask = (1 << nr_channels) - 1;
1697
1698         /* force dma off, just in case */
1699         dw_dma_off(dw);
1700
1701         /* disable BLOCK interrupts as well */
1702         channel_clear_bit(dw, MASK.BLOCK, dw->all_chan_mask);
1703
1704         err = devm_request_irq(&pdev->dev, irq, dw_dma_interrupt, 0,
1705                                "dw_dmac", dw);
1706         if (err)
1707                 return err;
1708
1709         platform_set_drvdata(pdev, dw);
1710
1711         tasklet_init(&dw->tasklet, dw_dma_tasklet, (unsigned long)dw);
1712
1713         INIT_LIST_HEAD(&dw->dma.channels);
1714         for (i = 0; i < nr_channels; i++) {
1715                 struct dw_dma_chan      *dwc = &dw->chan[i];
1716                 int                     r = nr_channels - i - 1;
1717
1718                 dwc->chan.device = &dw->dma;
1719                 dma_cookie_init(&dwc->chan);
1720                 if (pdata->chan_allocation_order == CHAN_ALLOCATION_ASCENDING)
1721                         list_add_tail(&dwc->chan.device_node,
1722                                         &dw->dma.channels);
1723                 else
1724                         list_add(&dwc->chan.device_node, &dw->dma.channels);
1725
1726                 /* 7 is highest priority & 0 is lowest. */
1727                 if (pdata->chan_priority == CHAN_PRIORITY_ASCENDING)
1728                         dwc->priority = r;
1729                 else
1730                         dwc->priority = i;
1731
1732                 dwc->ch_regs = &__dw_regs(dw)->CHAN[i];
1733                 spin_lock_init(&dwc->lock);
1734                 dwc->mask = 1 << i;
1735
1736                 INIT_LIST_HEAD(&dwc->active_list);
1737                 INIT_LIST_HEAD(&dwc->queue);
1738                 INIT_LIST_HEAD(&dwc->free_list);
1739
1740                 channel_clear_bit(dw, CH_EN, dwc->mask);
1741
1742                 dwc->dw = dw;
1743                 dwc->direction = DMA_TRANS_NONE;
1744
1745                 /* hardware configuration */
1746                 if (autocfg) {
1747                         unsigned int dwc_params;
1748
1749                         dwc_params = dma_read_byaddr(regs + r * sizeof(u32),
1750                                                      DWC_PARAMS);
1751
1752                         /* Decode maximum block size for given channel. The
1753                          * stored 4 bit value represents blocks from 0x00 for 3
1754                          * up to 0x0a for 4095. */
1755                         dwc->block_size =
1756                                 (4 << ((max_blk_size >> 4 * i) & 0xf)) - 1;
1757                         dwc->nollp =
1758                                 (dwc_params >> DWC_PARAMS_MBLK_EN & 0x1) == 0;
1759                 } else {
1760                         dwc->block_size = pdata->block_size;
1761
1762                         /* Check if channel supports multi block transfer */
1763                         channel_writel(dwc, LLP, 0xfffffffc);
1764                         dwc->nollp =
1765                                 (channel_readl(dwc, LLP) & 0xfffffffc) == 0;
1766                         channel_writel(dwc, LLP, 0);
1767                 }
1768         }
1769
1770         /* Clear all interrupts on all channels. */
1771         dma_writel(dw, CLEAR.XFER, dw->all_chan_mask);
1772         dma_writel(dw, CLEAR.BLOCK, dw->all_chan_mask);
1773         dma_writel(dw, CLEAR.SRC_TRAN, dw->all_chan_mask);
1774         dma_writel(dw, CLEAR.DST_TRAN, dw->all_chan_mask);
1775         dma_writel(dw, CLEAR.ERROR, dw->all_chan_mask);
1776
1777         dma_cap_set(DMA_MEMCPY, dw->dma.cap_mask);
1778         dma_cap_set(DMA_SLAVE, dw->dma.cap_mask);
1779         if (pdata->is_private)
1780                 dma_cap_set(DMA_PRIVATE, dw->dma.cap_mask);
1781         dw->dma.dev = &pdev->dev;
1782         dw->dma.device_alloc_chan_resources = dwc_alloc_chan_resources;
1783         dw->dma.device_free_chan_resources = dwc_free_chan_resources;
1784
1785         dw->dma.device_prep_dma_memcpy = dwc_prep_dma_memcpy;
1786
1787         dw->dma.device_prep_slave_sg = dwc_prep_slave_sg;
1788         dw->dma.device_control = dwc_control;
1789
1790         dw->dma.device_tx_status = dwc_tx_status;
1791         dw->dma.device_issue_pending = dwc_issue_pending;
1792
1793         dma_writel(dw, CFG, DW_CFG_DMA_EN);
1794
1795         dev_info(&pdev->dev, "DesignWare DMA Controller, %d channels\n",
1796                  nr_channels);
1797
1798         dma_async_device_register(&dw->dma);
1799
1800         return 0;
1801 }
1802
1803 static int __devexit dw_remove(struct platform_device *pdev)
1804 {
1805         struct dw_dma           *dw = platform_get_drvdata(pdev);
1806         struct dw_dma_chan      *dwc, *_dwc;
1807
1808         dw_dma_off(dw);
1809         dma_async_device_unregister(&dw->dma);
1810
1811         tasklet_kill(&dw->tasklet);
1812
1813         list_for_each_entry_safe(dwc, _dwc, &dw->dma.channels,
1814                         chan.device_node) {
1815                 list_del(&dwc->chan.device_node);
1816                 channel_clear_bit(dw, CH_EN, dwc->mask);
1817         }
1818
1819         return 0;
1820 }
1821
1822 static void dw_shutdown(struct platform_device *pdev)
1823 {
1824         struct dw_dma   *dw = platform_get_drvdata(pdev);
1825
1826         dw_dma_off(dw);
1827         clk_disable_unprepare(dw->clk);
1828 }
1829
1830 static int dw_suspend_noirq(struct device *dev)
1831 {
1832         struct platform_device *pdev = to_platform_device(dev);
1833         struct dw_dma   *dw = platform_get_drvdata(pdev);
1834
1835         dw_dma_off(dw);
1836         clk_disable_unprepare(dw->clk);
1837
1838         return 0;
1839 }
1840
1841 static int dw_resume_noirq(struct device *dev)
1842 {
1843         struct platform_device *pdev = to_platform_device(dev);
1844         struct dw_dma   *dw = platform_get_drvdata(pdev);
1845
1846         clk_prepare_enable(dw->clk);
1847         dma_writel(dw, CFG, DW_CFG_DMA_EN);
1848
1849         return 0;
1850 }
1851
1852 static const struct dev_pm_ops dw_dev_pm_ops = {
1853         .suspend_noirq = dw_suspend_noirq,
1854         .resume_noirq = dw_resume_noirq,
1855         .freeze_noirq = dw_suspend_noirq,
1856         .thaw_noirq = dw_resume_noirq,
1857         .restore_noirq = dw_resume_noirq,
1858         .poweroff_noirq = dw_suspend_noirq,
1859 };
1860
1861 #ifdef CONFIG_OF
1862 static const struct of_device_id dw_dma_id_table[] = {
1863         { .compatible = "snps,dma-spear1340" },
1864         {}
1865 };
1866 MODULE_DEVICE_TABLE(of, dw_dma_id_table);
1867 #endif
1868
1869 static struct platform_driver dw_driver = {
1870         .probe          = dw_probe,
1871         .remove         = dw_remove,
1872         .shutdown       = dw_shutdown,
1873         .driver = {
1874                 .name   = "dw_dmac",
1875                 .pm     = &dw_dev_pm_ops,
1876                 .of_match_table = of_match_ptr(dw_dma_id_table),
1877         },
1878 };
1879
1880 static int __init dw_init(void)
1881 {
1882         return platform_driver_register(&dw_driver);
1883 }
1884 subsys_initcall(dw_init);
1885
1886 static void __exit dw_exit(void)
1887 {
1888         platform_driver_unregister(&dw_driver);
1889 }
1890 module_exit(dw_exit);
1891
1892 MODULE_LICENSE("GPL v2");
1893 MODULE_DESCRIPTION("Synopsys DesignWare DMA Controller driver");
1894 MODULE_AUTHOR("Haavard Skinnemoen (Atmel)");
1895 MODULE_AUTHOR("Viresh Kumar <viresh.linux@gmail.com>");