drivers/dma/ste_dma40.c

   1 /*
   2  * Copyright (C) Ericsson AB 2007-2008
   3  * Copyright (C) ST-Ericsson SA 2008-2010
   4  * Author: Per Forlin <per.forlin@stericsson.com> for ST-Ericsson
   5  * Author: Jonas Aaberg <jonas.aberg@stericsson.com> for ST-Ericsson
   6  * License terms: GNU General Public License (GPL) version 2
   7  */
   8
   9 #include <linux/kernel.h>
  10 #include <linux/slab.h>
  11 #include <linux/dmaengine.h>
  12 #include <linux/platform_device.h>
  13 #include <linux/clk.h>
  14 #include <linux/delay.h>
  15 #include <linux/err.h>
  16
  17 #include <plat/ste_dma40.h>
  18
  19 #include "ste_dma40_ll.h"
  20
  21 #define D40_NAME "dma40"
  22
  23 #define D40_PHY_CHAN -1
  24
  25 /* For masking out/in 2 bit channel positions */
  26 #define D40_CHAN_POS(chan)  (2 * (chan / 2))
  27 #define D40_CHAN_POS_MASK(chan) (0x3 << D40_CHAN_POS(chan))
  28
  29 /* Maximum iterations taken before giving up suspending a channel */
  30 #define D40_SUSPEND_MAX_IT 500
  31
  32 /* Hardware requirement on LCLA alignment */
  33 #define LCLA_ALIGNMENT 0x40000
  34
  35 /* Max number of links per event group */
  36 #define D40_LCLA_LINK_PER_EVENT_GRP 128
  37 #define D40_LCLA_END D40_LCLA_LINK_PER_EVENT_GRP
  38
  39 /* Attempts before giving up to trying to get pages that are aligned */
  40 #define MAX_LCLA_ALLOC_ATTEMPTS 256
  41
  42 /* Bit markings for allocation map */
  43 #define D40_ALLOC_FREE          (1 << 31)
  44 #define D40_ALLOC_PHY           (1 << 30)
  45 #define D40_ALLOC_LOG_FREE      0
  46
  47 /* Hardware designer of the block */
  48 #define D40_HW_DESIGNER 0x8
  49
  50 /**
  51  * enum 40_command - The different commands and/or statuses.
  52  *
  53  * @D40_DMA_STOP: DMA channel command STOP or status STOPPED,
  54  * @D40_DMA_RUN: The DMA channel is RUNNING of the command RUN.
  55  * @D40_DMA_SUSPEND_REQ: Request the DMA to SUSPEND as soon as possible.
  56  * @D40_DMA_SUSPENDED: The DMA channel is SUSPENDED.
  57  */
  58 enum d40_command {
  59         D40_DMA_STOP            = 0,
  60         D40_DMA_RUN             = 1,
  61         D40_DMA_SUSPEND_REQ     = 2,
  62         D40_DMA_SUSPENDED       = 3
  63 };
  64
  65 /**
  66  * struct d40_lli_pool - Structure for keeping LLIs in memory
  67  *
  68  * @base: Pointer to memory area when the pre_alloc_lli's are not large
  69  * enough, IE bigger than the most common case, 1 dst and 1 src. NULL if
  70  * pre_alloc_lli is used.
  71  * @size: The size in bytes of the memory at base or the size of pre_alloc_lli.
  72  * @pre_alloc_lli: Pre allocated area for the most common case of transfers,
  73  * one buffer to one buffer.
  74  */
  75 struct d40_lli_pool {
  76         void    *base;
  77         int      size;
  78         /* Space for dst and src, plus an extra for padding */
  79         u8       pre_alloc_lli[3 * sizeof(struct d40_phy_lli)];
  80 };
  81
  82 /**
  83  * struct d40_desc - A descriptor is one DMA job.
  84  *
  85  * @lli_phy: LLI settings for physical channel. Both src and dst=
  86  * points into the lli_pool, to base if lli_len > 1 or to pre_alloc_lli if
  87  * lli_len equals one.
  88  * @lli_log: Same as above but for logical channels.
  89  * @lli_pool: The pool with two entries pre-allocated.
  90  * @lli_len: Number of llis of current descriptor.
  91  * @lli_current: Number of transfered llis.
  92  * @lcla_alloc: Number of LCLA entries allocated.
  93  * @txd: DMA engine struct. Used for among other things for communication
  94  * during a transfer.
  95  * @node: List entry.
  96  * @is_in_client_list: true if the client owns this descriptor.
  97  * @is_hw_linked: true if this job will automatically be continued for
  98  * the previous one.
  99  *
 100  * This descriptor is used for both logical and physical transfers.
 101  */
 102 struct d40_desc {
 103         /* LLI physical */
 104         struct d40_phy_lli_bidir         lli_phy;
 105         /* LLI logical */
 106         struct d40_log_lli_bidir         lli_log;
 107
 108         struct d40_lli_pool              lli_pool;
 109         int                              lli_len;
 110         int                              lli_current;
 111         int                              lcla_alloc;
 112
 113         struct dma_async_tx_descriptor   txd;
 114         struct list_head                 node;
 115
 116         bool                             is_in_client_list;
 117         bool                             is_hw_linked;
 118 };
 119
 120 /**
 121  * struct d40_lcla_pool - LCLA pool settings and data.
 122  *
 123  * @base: The virtual address of LCLA. 18 bit aligned.
 124  * @base_unaligned: The orignal kmalloc pointer, if kmalloc is used.
 125  * This pointer is only there for clean-up on error.
 126  * @pages: The number of pages needed for all physical channels.
 127  * Only used later for clean-up on error
 128  * @lock: Lock to protect the content in this struct.
 129  * @alloc_map: big map over which LCLA entry is own by which job.
 130  */
 131 struct d40_lcla_pool {
 132         void            *base;
 133         void            *base_unaligned;
 134         int              pages;
 135         spinlock_t       lock;
 136         struct d40_desc **alloc_map;
 137 };
 138
 139 /**
 140  * struct d40_phy_res - struct for handling eventlines mapped to physical
 141  * channels.
 142  *
 143  * @lock: A lock protection this entity.
 144  * @num: The physical channel number of this entity.
 145  * @allocated_src: Bit mapped to show which src event line's are mapped to
 146  * this physical channel. Can also be free or physically allocated.
 147  * @allocated_dst: Same as for src but is dst.
 148  * allocated_dst and allocated_src uses the D40_ALLOC* defines as well as
 149  * event line number.
 150  */
 151 struct d40_phy_res {
 152         spinlock_t lock;
 153         int        num;
 154         u32        allocated_src;
 155         u32        allocated_dst;
 156 };
 157
 158 struct d40_base;
 159
 160 /**
 161  * struct d40_chan - Struct that describes a channel.
 162  *
 163  * @lock: A spinlock to protect this struct.
 164  * @log_num: The logical number, if any of this channel.
 165  * @completed: Starts with 1, after first interrupt it is set to dma engine's
 166  * current cookie.
 167  * @pending_tx: The number of pending transfers. Used between interrupt handler
 168  * and tasklet.
 169  * @busy: Set to true when transfer is ongoing on this channel.
 170  * @phy_chan: Pointer to physical channel which this instance runs on. If this
 171  * point is NULL, then the channel is not allocated.
 172  * @chan: DMA engine handle.
 173  * @tasklet: Tasklet that gets scheduled from interrupt context to complete a
 174  * transfer and call client callback.
 175  * @client: Cliented owned descriptor list.
 176  * @active: Active descriptor.
 177  * @queue: Queued jobs.
 178  * @dma_cfg: The client configuration of this dma channel.
 179  * @configured: whether the dma_cfg configuration is valid
 180  * @base: Pointer to the device instance struct.
 181  * @src_def_cfg: Default cfg register setting for src.
 182  * @dst_def_cfg: Default cfg register setting for dst.
 183  * @log_def: Default logical channel settings.
 184  * @lcla: Space for one dst src pair for logical channel transfers.
 185  * @lcpa: Pointer to dst and src lcpa settings.
 186  *
 187  * This struct can either "be" a logical or a physical channel.
 188  */
 189 struct d40_chan {
 190         spinlock_t                       lock;
 191         int                              log_num;
 192         /* ID of the most recent completed transfer */
 193         int                              completed;
 194         int                              pending_tx;
 195         bool                             busy;
 196         struct d40_phy_res              *phy_chan;
 197         struct dma_chan                  chan;
 198         struct tasklet_struct            tasklet;
 199         struct list_head                 client;
 200         struct list_head                 active;
 201         struct list_head                 queue;
 202         struct stedma40_chan_cfg         dma_cfg;
 203         bool                             configured;
 204         struct d40_base                 *base;
 205         /* Default register configurations */
 206         u32                              src_def_cfg;
 207         u32                              dst_def_cfg;
 208         struct d40_def_lcsp              log_def;
 209         struct d40_log_lli_full         *lcpa;
 210         /* Runtime reconfiguration */
 211         dma_addr_t                      runtime_addr;
 212         enum dma_data_direction         runtime_direction;
 213 };
 214
 215 /**
 216  * struct d40_base - The big global struct, one for each probe'd instance.
 217  *
 218  * @interrupt_lock: Lock used to make sure one interrupt is handle a time.
 219  * @execmd_lock: Lock for execute command usage since several channels share
 220  * the same physical register.
 221  * @dev: The device structure.
 222  * @virtbase: The virtual base address of the DMA's register.
 223  * @rev: silicon revision detected.
 224  * @clk: Pointer to the DMA clock structure.
 225  * @phy_start: Physical memory start of the DMA registers.
 226  * @phy_size: Size of the DMA register map.
 227  * @irq: The IRQ number.
 228  * @num_phy_chans: The number of physical channels. Read from HW. This
 229  * is the number of available channels for this driver, not counting "Secure
 230  * mode" allocated physical channels.
 231  * @num_log_chans: The number of logical channels. Calculated from
 232  * num_phy_chans.
 233  * @dma_both: dma_device channels that can do both memcpy and slave transfers.
 234  * @dma_slave: dma_device channels that can do only do slave transfers.
 235  * @dma_memcpy: dma_device channels that can do only do memcpy transfers.
 236  * @log_chans: Room for all possible logical channels in system.
 237  * @lookup_log_chans: Used to map interrupt number to logical channel. Points
 238  * to log_chans entries.
 239  * @lookup_phy_chans: Used to map interrupt number to physical channel. Points
 240  * to phy_chans entries.
 241  * @plat_data: Pointer to provided platform_data which is the driver
 242  * configuration.
 243  * @phy_res: Vector containing all physical channels.
 244  * @lcla_pool: lcla pool settings and data.
 245  * @lcpa_base: The virtual mapped address of LCPA.
 246  * @phy_lcpa: The physical address of the LCPA.
 247  * @lcpa_size: The size of the LCPA area.
 248  * @desc_slab: cache for descriptors.
 249  */
 250 struct d40_base {
 251         spinlock_t                       interrupt_lock;
 252         spinlock_t                       execmd_lock;
 253         struct device                    *dev;
 254         void __iomem                     *virtbase;
 255         u8                                rev:4;
 256         struct clk                       *clk;
 257         phys_addr_t                       phy_start;
 258         resource_size_t                   phy_size;
 259         int                               irq;
 260         int                               num_phy_chans;
 261         int                               num_log_chans;
 262         struct dma_device                 dma_both;
 263         struct dma_device                 dma_slave;
 264         struct dma_device                 dma_memcpy;
 265         struct d40_chan                  *phy_chans;
 266         struct d40_chan                  *log_chans;
 267         struct d40_chan                 **lookup_log_chans;
 268         struct d40_chan                 **lookup_phy_chans;
 269         struct stedma40_platform_data    *plat_data;
 270         /* Physical half channels */
 271         struct d40_phy_res               *phy_res;
 272         struct d40_lcla_pool              lcla_pool;
 273         void                             *lcpa_base;
 274         dma_addr_t                        phy_lcpa;
 275         resource_size_t                   lcpa_size;
 276         struct kmem_cache                *desc_slab;
 277 };
 278
 279 /**
 280  * struct d40_interrupt_lookup - lookup table for interrupt handler
 281  *
 282  * @src: Interrupt mask register.
 283  * @clr: Interrupt clear register.
 284  * @is_error: true if this is an error interrupt.
 285  * @offset: start delta in the lookup_log_chans in d40_base. If equals to
 286  * D40_PHY_CHAN, the lookup_phy_chans shall be used instead.
 287  */
 288 struct d40_interrupt_lookup {
 289         u32 src;
 290         u32 clr;
 291         bool is_error;
 292         int offset;
 293 };
 294
 295 /**
 296  * struct d40_reg_val - simple lookup struct
 297  *
 298  * @reg: The register.
 299  * @val: The value that belongs to the register in reg.
 300  */
 301 struct d40_reg_val {
 302         unsigned int reg;
 303         unsigned int val;
 304 };
 305
 306 static struct device *chan2dev(struct d40_chan *d40c)
 307 {
 308         return &d40c->chan.dev->device;
 309 }
 310
 311 static int d40_pool_lli_alloc(struct d40_desc *d40d,
 312                               int lli_len, bool is_log)
 313 {
 314         u32 align;
 315         void *base;
 316
 317         if (is_log)
 318                 align = sizeof(struct d40_log_lli);
 319         else
 320                 align = sizeof(struct d40_phy_lli);
 321
 322         if (lli_len == 1) {
 323                 base = d40d->lli_pool.pre_alloc_lli;
 324                 d40d->lli_pool.size = sizeof(d40d->lli_pool.pre_alloc_lli);
 325                 d40d->lli_pool.base = NULL;
 326         } else {
 327                 d40d->lli_pool.size = ALIGN(lli_len * 2 * align, align);
 328
 329                 base = kmalloc(d40d->lli_pool.size + align, GFP_NOWAIT);
 330                 d40d->lli_pool.base = base;
 331
 332                 if (d40d->lli_pool.base == NULL)
 333                         return -ENOMEM;
 334         }
 335
 336         if (is_log) {
 337                 d40d->lli_log.src = PTR_ALIGN((struct d40_log_lli *) base,
 338                                               align);
 339                 d40d->lli_log.dst = PTR_ALIGN(d40d->lli_log.src + lli_len,
 340                                               align);
 341         } else {
 342                 d40d->lli_phy.src = PTR_ALIGN((struct d40_phy_lli *)base,
 343                                               align);
 344                 d40d->lli_phy.dst = PTR_ALIGN(d40d->lli_phy.src + lli_len,
 345                                               align);
 346         }
 347
 348         return 0;
 349 }
 350
 351 static void d40_pool_lli_free(struct d40_desc *d40d)
 352 {
 353         kfree(d40d->lli_pool.base);
 354         d40d->lli_pool.base = NULL;
 355         d40d->lli_pool.size = 0;
 356         d40d->lli_log.src = NULL;
 357         d40d->lli_log.dst = NULL;
 358         d40d->lli_phy.src = NULL;
 359         d40d->lli_phy.dst = NULL;
 360 }
 361
 362 static int d40_lcla_alloc_one(struct d40_chan *d40c,
 363                               struct d40_desc *d40d)
 364 {
 365         unsigned long flags;
 366         int i;
 367         int ret = -EINVAL;
 368         int p;
 369
 370         spin_lock_irqsave(&d40c->base->lcla_pool.lock, flags);
 371
 372         p = d40c->phy_chan->num * D40_LCLA_LINK_PER_EVENT_GRP;
 373
 374         /*
 375          * Allocate both src and dst at the same time, therefore the half
 376          * start on 1 since 0 can't be used since zero is used as end marker.
 377          */
 378         for (i = 1 ; i < D40_LCLA_LINK_PER_EVENT_GRP / 2; i++) {
 379                 if (!d40c->base->lcla_pool.alloc_map[p + i]) {
 380                         d40c->base->lcla_pool.alloc_map[p + i] = d40d;
 381                         d40d->lcla_alloc++;
 382                         ret = i;
 383                         break;
 384                 }
 385         }
 386
 387         spin_unlock_irqrestore(&d40c->base->lcla_pool.lock, flags);
 388
 389         return ret;
 390 }
 391
 392 static int d40_lcla_free_all(struct d40_chan *d40c,
 393                              struct d40_desc *d40d)
 394 {
 395         unsigned long flags;
 396         int i;
 397         int ret = -EINVAL;
 398
 399         if (d40c->log_num == D40_PHY_CHAN)
 400                 return 0;
 401
 402         spin_lock_irqsave(&d40c->base->lcla_pool.lock, flags);
 403
 404         for (i = 1 ; i < D40_LCLA_LINK_PER_EVENT_GRP / 2; i++) {
 405                 if (d40c->base->lcla_pool.alloc_map[d40c->phy_chan->num *
 406                                                     D40_LCLA_LINK_PER_EVENT_GRP + i] == d40d) {
 407                         d40c->base->lcla_pool.alloc_map[d40c->phy_chan->num *
 408                                                         D40_LCLA_LINK_PER_EVENT_GRP + i] = NULL;
 409                         d40d->lcla_alloc--;
 410                         if (d40d->lcla_alloc == 0) {
 411                                 ret = 0;
 412                                 break;
 413                         }
 414                 }
 415         }
 416
 417         spin_unlock_irqrestore(&d40c->base->lcla_pool.lock, flags);
 418
 419         return ret;
 420
 421 }
 422
 423 static void d40_desc_remove(struct d40_desc *d40d)
 424 {
 425         list_del(&d40d->node);
 426 }
 427
 428 static struct d40_desc *d40_desc_get(struct d40_chan *d40c)
 429 {
 430         struct d40_desc *desc = NULL;
 431
 432         if (!list_empty(&d40c->client)) {
 433                 struct d40_desc *d;
 434                 struct d40_desc *_d;
 435
 436                 list_for_each_entry_safe(d, _d, &d40c->client, node)
 437                         if (async_tx_test_ack(&d->txd)) {
 438                                 d40_pool_lli_free(d);
 439                                 d40_desc_remove(d);
 440                                 desc = d;
 441                                 memset(desc, 0, sizeof(*desc));
 442                                 break;
 443                         }
 444         }
 445
 446         if (!desc)
 447                 desc = kmem_cache_zalloc(d40c->base->desc_slab, GFP_NOWAIT);
 448
 449         if (desc)
 450                 INIT_LIST_HEAD(&desc->node);
 451
 452         return desc;
 453 }
 454
 455 static void d40_desc_free(struct d40_chan *d40c, struct d40_desc *d40d)
 456 {
 457
 458         d40_lcla_free_all(d40c, d40d);
 459         kmem_cache_free(d40c->base->desc_slab, d40d);
 460 }
 461
 462 static void d40_desc_submit(struct d40_chan *d40c, struct d40_desc *desc)
 463 {
 464         list_add_tail(&desc->node, &d40c->active);
 465 }
 466
 467 static void d40_desc_load(struct d40_chan *d40c, struct d40_desc *d40d)
 468 {
 469         int curr_lcla = -EINVAL, next_lcla;
 470
 471         if (d40c->log_num == D40_PHY_CHAN) {
 472                 d40_phy_lli_write(d40c->base->virtbase,
 473                                   d40c->phy_chan->num,
 474                                   d40d->lli_phy.dst,
 475                                   d40d->lli_phy.src);
 476                 d40d->lli_current = d40d->lli_len;
 477         } else {
 478
 479                 if ((d40d->lli_len - d40d->lli_current) > 1)
 480                         curr_lcla = d40_lcla_alloc_one(d40c, d40d);
 481
 482                 d40_log_lli_lcpa_write(d40c->lcpa,
 483                                        &d40d->lli_log.dst[d40d->lli_current],
 484                                        &d40d->lli_log.src[d40d->lli_current],
 485                                        curr_lcla);
 486
 487                 d40d->lli_current++;
 488                 for (; d40d->lli_current < d40d->lli_len; d40d->lli_current++) {
 489                         struct d40_log_lli *lcla;
 490
 491                         if (d40d->lli_current + 1 < d40d->lli_len)
 492                                 next_lcla = d40_lcla_alloc_one(d40c, d40d);
 493                         else
 494                                 next_lcla = -EINVAL;
 495
 496                         lcla = d40c->base->lcla_pool.base +
 497                                 d40c->phy_chan->num * 1024 +
 498                                 8 * curr_lcla * 2;
 499
 500                         d40_log_lli_lcla_write(lcla,
 501                                                &d40d->lli_log.dst[d40d->lli_current],
 502                                                &d40d->lli_log.src[d40d->lli_current],
 503                                                next_lcla);
 504
 505                         (void) dma_map_single(d40c->base->dev, lcla,
 506                                               2 * sizeof(struct d40_log_lli),
 507                                               DMA_TO_DEVICE);
 508
 509                         curr_lcla = next_lcla;
 510
 511                         if (curr_lcla == -EINVAL) {
 512                                 d40d->lli_current++;
 513                                 break;
 514                         }
 515
 516                 }
 517         }
 518 }
 519
 520 static struct d40_desc *d40_first_active_get(struct d40_chan *d40c)
 521 {
 522         struct d40_desc *d;
 523
 524         if (list_empty(&d40c->active))
 525                 return NULL;
 526
 527         d = list_first_entry(&d40c->active,
 528                              struct d40_desc,
 529                              node);
 530         return d;
 531 }
 532
 533 static void d40_desc_queue(struct d40_chan *d40c, struct d40_desc *desc)
 534 {
 535         list_add_tail(&desc->node, &d40c->queue);
 536 }
 537
 538 static struct d40_desc *d40_first_queued(struct d40_chan *d40c)
 539 {
 540         struct d40_desc *d;
 541
 542         if (list_empty(&d40c->queue))
 543                 return NULL;
 544
 545         d = list_first_entry(&d40c->queue,
 546                              struct d40_desc,
 547                              node);
 548         return d;
 549 }
 550
 551 static struct d40_desc *d40_last_queued(struct d40_chan *d40c)
 552 {
 553         struct d40_desc *d;
 554
 555         if (list_empty(&d40c->queue))
 556                 return NULL;
 557         list_for_each_entry(d, &d40c->queue, node)
 558                 if (list_is_last(&d->node, &d40c->queue))
 559                         break;
 560         return d;
 561 }
 562
 563 static int d40_psize_2_burst_size(bool is_log, int psize)
 564 {
 565         if (is_log) {
 566                 if (psize == STEDMA40_PSIZE_LOG_1)
 567                         return 1;
 568         } else {
 569                 if (psize == STEDMA40_PSIZE_PHY_1)
 570                         return 1;
 571         }
 572
 573         return 2 << psize;
 574 }
 575
 576 /*
 577  * The dma only supports transmitting packages up to
 578  * STEDMA40_MAX_SEG_SIZE << data_width. Calculate the total number of
 579  * dma elements required to send the entire sg list
 580  */
 581 static int d40_size_2_dmalen(int size, u32 data_width1, u32 data_width2)
 582 {
 583         int dmalen;
 584         u32 max_w = max(data_width1, data_width2);
 585         u32 min_w = min(data_width1, data_width2);
 586         u32 seg_max = ALIGN(STEDMA40_MAX_SEG_SIZE << min_w, 1 << max_w);
 587
 588         if (seg_max > STEDMA40_MAX_SEG_SIZE)
 589                 seg_max -= (1 << max_w);
 590
 591         if (!IS_ALIGNED(size, 1 << max_w))
 592                 return -EINVAL;
 593
 594         if (size <= seg_max)
 595                 dmalen = 1;
 596         else {
 597                 dmalen = size / seg_max;
 598                 if (dmalen * seg_max < size)
 599                         dmalen++;
 600         }
 601         return dmalen;
 602 }
 603
 604 static int d40_sg_2_dmalen(struct scatterlist *sgl, int sg_len,
 605                            u32 data_width1, u32 data_width2)
 606 {
 607         struct scatterlist *sg;
 608         int i;
 609         int len = 0;
 610         int ret;
 611
 612         for_each_sg(sgl, sg, sg_len, i) {
 613                 ret = d40_size_2_dmalen(sg_dma_len(sg),
 614                                         data_width1, data_width2);
 615                 if (ret < 0)
 616                         return ret;
 617                 len += ret;
 618         }
 619         return len;
 620 }
 621
 622 /* Support functions for logical channels */
 623
 624 static int d40_channel_execute_command(struct d40_chan *d40c,
 625                                        enum d40_command command)
 626 {
 627         u32 status;
 628         int i;
 629         void __iomem *active_reg;
 630         int ret = 0;
 631         unsigned long flags;
 632         u32 wmask;
 633
 634         spin_lock_irqsave(&d40c->base->execmd_lock, flags);
 635
 636         if (d40c->phy_chan->num % 2 == 0)
 637                 active_reg = d40c->base->virtbase + D40_DREG_ACTIVE;
 638         else
 639                 active_reg = d40c->base->virtbase + D40_DREG_ACTIVO;
 640
 641         if (command == D40_DMA_SUSPEND_REQ) {
 642                 status = (readl(active_reg) &
 643                           D40_CHAN_POS_MASK(d40c->phy_chan->num)) >>
 644                         D40_CHAN_POS(d40c->phy_chan->num);
 645
 646                 if (status == D40_DMA_SUSPENDED || status == D40_DMA_STOP)
 647                         goto done;
 648         }
 649
 650         wmask = 0xffffffff & ~(D40_CHAN_POS_MASK(d40c->phy_chan->num));
 651         writel(wmask | (command << D40_CHAN_POS(d40c->phy_chan->num)),
 652                active_reg);
 653
 654         if (command == D40_DMA_SUSPEND_REQ) {
 655
 656                 for (i = 0 ; i < D40_SUSPEND_MAX_IT; i++) {
 657                         status = (readl(active_reg) &
 658                                   D40_CHAN_POS_MASK(d40c->phy_chan->num)) >>
 659                                 D40_CHAN_POS(d40c->phy_chan->num);
 660
 661                         cpu_relax();
 662                         /*
 663                          * Reduce the number of bus accesses while
 664                          * waiting for the DMA to suspend.
 665                          */
 666                         udelay(3);
 667
 668                         if (status == D40_DMA_STOP ||
 669                             status == D40_DMA_SUSPENDED)
 670                                 break;
 671                 }
 672
 673                 if (i == D40_SUSPEND_MAX_IT) {
 674                         dev_err(&d40c->chan.dev->device,
 675                                 "[%s]: unable to suspend the chl %d (log: %d) status %x\n",
 676                                 __func__, d40c->phy_chan->num, d40c->log_num,
 677                                 status);
 678                         dump_stack();
 679                         ret = -EBUSY;
 680                 }
 681
 682         }
 683 done:
 684         spin_unlock_irqrestore(&d40c->base->execmd_lock, flags);
 685         return ret;
 686 }
 687
 688 static void d40_term_all(struct d40_chan *d40c)
 689 {
 690         struct d40_desc *d40d;
 691
 692         /* Release active descriptors */
 693         while ((d40d = d40_first_active_get(d40c))) {
 694                 d40_desc_remove(d40d);
 695                 d40_desc_free(d40c, d40d);
 696         }
 697
 698         /* Release queued descriptors waiting for transfer */
 699         while ((d40d = d40_first_queued(d40c))) {
 700                 d40_desc_remove(d40d);
 701                 d40_desc_free(d40c, d40d);
 702         }
 703
 704
 705         d40c->pending_tx = 0;
 706         d40c->busy = false;
 707 }
 708
 709 static void __d40_config_set_event(struct d40_chan *d40c, bool enable,
 710                                    u32 event, int reg)
 711 {
 712         void __iomem *addr = d40c->base->virtbase + D40_DREG_PCBASE
 713                              + d40c->phy_chan->num * D40_DREG_PCDELTA + reg;
 714         int tries;
 715
 716         if (!enable) {
 717                 writel((D40_DEACTIVATE_EVENTLINE << D40_EVENTLINE_POS(event))
 718                        | ~D40_EVENTLINE_MASK(event), addr);
 719                 return;
 720         }
 721
 722         /*
 723          * The hardware sometimes doesn't register the enable when src and dst
 724          * event lines are active on the same logical channel.  Retry to ensure
 725          * it does.  Usually only one retry is sufficient.
 726          */
 727         tries = 100;
 728         while (--tries) {
 729                 writel((D40_ACTIVATE_EVENTLINE << D40_EVENTLINE_POS(event))
 730                        | ~D40_EVENTLINE_MASK(event), addr);
 731
 732                 if (readl(addr) & D40_EVENTLINE_MASK(event))
 733                         break;
 734         }
 735
 736         if (tries != 99)
 737                 dev_dbg(chan2dev(d40c),
 738                         "[%s] workaround enable S%cLNK (%d tries)\n",
 739                         __func__, reg == D40_CHAN_REG_SSLNK ? 'S' : 'D',
 740                         100 - tries);
 741
 742         WARN_ON(!tries);
 743 }
 744
 745 static void d40_config_set_event(struct d40_chan *d40c, bool do_enable)
 746 {
 747         unsigned long flags;
 748
 749         spin_lock_irqsave(&d40c->phy_chan->lock, flags);
 750
 751         /* Enable event line connected to device (or memcpy) */
 752         if ((d40c->dma_cfg.dir ==  STEDMA40_PERIPH_TO_MEM) ||
 753             (d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_PERIPH)) {
 754                 u32 event = D40_TYPE_TO_EVENT(d40c->dma_cfg.src_dev_type);
 755
 756                 __d40_config_set_event(d40c, do_enable, event,
 757                                        D40_CHAN_REG_SSLNK);
 758         }
 759
 760         if (d40c->dma_cfg.dir !=  STEDMA40_PERIPH_TO_MEM) {
 761                 u32 event = D40_TYPE_TO_EVENT(d40c->dma_cfg.dst_dev_type);
 762
 763                 __d40_config_set_event(d40c, do_enable, event,
 764                                        D40_CHAN_REG_SDLNK);
 765         }
 766
 767         spin_unlock_irqrestore(&d40c->phy_chan->lock, flags);
 768 }
 769
 770 static u32 d40_chan_has_events(struct d40_chan *d40c)
 771 {
 772         u32 val;
 773
 774         val = readl(d40c->base->virtbase + D40_DREG_PCBASE +
 775                     d40c->phy_chan->num * D40_DREG_PCDELTA +
 776                     D40_CHAN_REG_SSLNK);
 777
 778         val |= readl(d40c->base->virtbase + D40_DREG_PCBASE +
 779                      d40c->phy_chan->num * D40_DREG_PCDELTA +
 780                      D40_CHAN_REG_SDLNK);
 781         return val;
 782 }
 783
 784 static u32 d40_get_prmo(struct d40_chan *d40c)
 785 {
 786         static const unsigned int phy_map[] = {
 787                 [STEDMA40_PCHAN_BASIC_MODE]
 788                         = D40_DREG_PRMO_PCHAN_BASIC,
 789                 [STEDMA40_PCHAN_MODULO_MODE]
 790                         = D40_DREG_PRMO_PCHAN_MODULO,
 791                 [STEDMA40_PCHAN_DOUBLE_DST_MODE]
 792                         = D40_DREG_PRMO_PCHAN_DOUBLE_DST,
 793         };
 794         static const unsigned int log_map[] = {
 795                 [STEDMA40_LCHAN_SRC_PHY_DST_LOG]
 796                         = D40_DREG_PRMO_LCHAN_SRC_PHY_DST_LOG,
 797                 [STEDMA40_LCHAN_SRC_LOG_DST_PHY]
 798                         = D40_DREG_PRMO_LCHAN_SRC_LOG_DST_PHY,
 799                 [STEDMA40_LCHAN_SRC_LOG_DST_LOG]
 800                         = D40_DREG_PRMO_LCHAN_SRC_LOG_DST_LOG,
 801         };
 802
 803         if (d40c->log_num == D40_PHY_CHAN)
 804                 return phy_map[d40c->dma_cfg.mode_opt];
 805         else
 806                 return log_map[d40c->dma_cfg.mode_opt];
 807 }
 808
 809 static void d40_config_write(struct d40_chan *d40c)
 810 {
 811         u32 addr_base;
 812         u32 var;
 813
 814         /* Odd addresses are even addresses + 4 */
 815         addr_base = (d40c->phy_chan->num % 2) * 4;
 816         /* Setup channel mode to logical or physical */
 817         var = ((u32)(d40c->log_num != D40_PHY_CHAN) + 1) <<
 818                 D40_CHAN_POS(d40c->phy_chan->num);
 819         writel(var, d40c->base->virtbase + D40_DREG_PRMSE + addr_base);
 820
 821         /* Setup operational mode option register */
 822         var = d40_get_prmo(d40c) << D40_CHAN_POS(d40c->phy_chan->num);
 823
 824         writel(var, d40c->base->virtbase + D40_DREG_PRMOE + addr_base);
 825
 826         if (d40c->log_num != D40_PHY_CHAN) {
 827                 /* Set default config for CFG reg */
 828                 writel(d40c->src_def_cfg,
 829                        d40c->base->virtbase + D40_DREG_PCBASE +
 830                        d40c->phy_chan->num * D40_DREG_PCDELTA +
 831                        D40_CHAN_REG_SSCFG);
 832                 writel(d40c->dst_def_cfg,
 833                        d40c->base->virtbase + D40_DREG_PCBASE +
 834                        d40c->phy_chan->num * D40_DREG_PCDELTA +
 835                        D40_CHAN_REG_SDCFG);
 836
 837                 /* Set LIDX for lcla */
 838                 writel((d40c->phy_chan->num << D40_SREG_ELEM_LOG_LIDX_POS) &
 839                        D40_SREG_ELEM_LOG_LIDX_MASK,
 840                        d40c->base->virtbase + D40_DREG_PCBASE +
 841                        d40c->phy_chan->num * D40_DREG_PCDELTA +
 842                        D40_CHAN_REG_SDELT);
 843
 844                 writel((d40c->phy_chan->num << D40_SREG_ELEM_LOG_LIDX_POS) &
 845                        D40_SREG_ELEM_LOG_LIDX_MASK,
 846                        d40c->base->virtbase + D40_DREG_PCBASE +
 847                        d40c->phy_chan->num * D40_DREG_PCDELTA +
 848                        D40_CHAN_REG_SSELT);
 849
 850         }
 851 }
 852
 853 static u32 d40_residue(struct d40_chan *d40c)
 854 {
 855         u32 num_elt;
 856
 857         if (d40c->log_num != D40_PHY_CHAN)
 858                 num_elt = (readl(&d40c->lcpa->lcsp2) & D40_MEM_LCSP2_ECNT_MASK)
 859                         >> D40_MEM_LCSP2_ECNT_POS;
 860         else
 861                 num_elt = (readl(d40c->base->virtbase + D40_DREG_PCBASE +
 862                                  d40c->phy_chan->num * D40_DREG_PCDELTA +
 863                                  D40_CHAN_REG_SDELT) &
 864                            D40_SREG_ELEM_PHY_ECNT_MASK) >>
 865                         D40_SREG_ELEM_PHY_ECNT_POS;
 866         return num_elt * (1 << d40c->dma_cfg.dst_info.data_width);
 867 }
 868
 869 static bool d40_tx_is_linked(struct d40_chan *d40c)
 870 {
 871         bool is_link;
 872
 873         if (d40c->log_num != D40_PHY_CHAN)
 874                 is_link = readl(&d40c->lcpa->lcsp3) &  D40_MEM_LCSP3_DLOS_MASK;
 875         else
 876                 is_link = readl(d40c->base->virtbase + D40_DREG_PCBASE +
 877                                 d40c->phy_chan->num * D40_DREG_PCDELTA +
 878                                 D40_CHAN_REG_SDLNK) &
 879                         D40_SREG_LNK_PHYS_LNK_MASK;
 880         return is_link;
 881 }
 882
 883 static int d40_pause(struct dma_chan *chan)
 884 {
 885         struct d40_chan *d40c =
 886                 container_of(chan, struct d40_chan, chan);
 887         int res = 0;
 888         unsigned long flags;
 889
 890         if (!d40c->busy)
 891                 return 0;
 892
 893         spin_lock_irqsave(&d40c->lock, flags);
 894
 895         res = d40_channel_execute_command(d40c, D40_DMA_SUSPEND_REQ);
 896         if (res == 0) {
 897                 if (d40c->log_num != D40_PHY_CHAN) {
 898                         d40_config_set_event(d40c, false);
 899                         /* Resume the other logical channels if any */
 900                         if (d40_chan_has_events(d40c))
 901                                 res = d40_channel_execute_command(d40c,
 902                                                                   D40_DMA_RUN);
 903                 }
 904         }
 905
 906         spin_unlock_irqrestore(&d40c->lock, flags);
 907         return res;
 908 }
 909
 910 static int d40_resume(struct dma_chan *chan)
 911 {
 912         struct d40_chan *d40c =
 913                 container_of(chan, struct d40_chan, chan);
 914         int res = 0;
 915         unsigned long flags;
 916
 917         if (!d40c->busy)
 918                 return 0;
 919
 920         spin_lock_irqsave(&d40c->lock, flags);
 921
 922         if (d40c->base->rev == 0)
 923                 if (d40c->log_num != D40_PHY_CHAN) {
 924                         res = d40_channel_execute_command(d40c,
 925                                                           D40_DMA_SUSPEND_REQ);
 926                         goto no_suspend;
 927                 }
 928
 929         /* If bytes left to transfer or linked tx resume job */
 930         if (d40_residue(d40c) || d40_tx_is_linked(d40c)) {
 931
 932                 if (d40c->log_num != D40_PHY_CHAN)
 933                         d40_config_set_event(d40c, true);
 934
 935                 res = d40_channel_execute_command(d40c, D40_DMA_RUN);
 936         }
 937
 938 no_suspend:
 939         spin_unlock_irqrestore(&d40c->lock, flags);
 940         return res;
 941 }
 942
 943 static void d40_tx_submit_log(struct d40_chan *d40c, struct d40_desc *d40d)
 944 {
 945         /* TODO: Write */
 946 }
 947
 948 static void d40_tx_submit_phy(struct d40_chan *d40c, struct d40_desc *d40d)
 949 {
 950         struct d40_desc *d40d_prev = NULL;
 951         int i;
 952         u32 val;
 953
 954         if (!list_empty(&d40c->queue))
 955                 d40d_prev = d40_last_queued(d40c);
 956         else if (!list_empty(&d40c->active))
 957                 d40d_prev = d40_first_active_get(d40c);
 958
 959         if (!d40d_prev)
 960                 return;
 961
 962         /* Here we try to join this job with previous jobs */
 963         val = readl(d40c->base->virtbase + D40_DREG_PCBASE +
 964                     d40c->phy_chan->num * D40_DREG_PCDELTA +
 965                     D40_CHAN_REG_SSLNK);
 966
 967         /* Figure out which link we're currently transmitting */
 968         for (i = 0; i < d40d_prev->lli_len; i++)
 969                 if (val == d40d_prev->lli_phy.src[i].reg_lnk)
 970                         break;
 971
 972         val = readl(d40c->base->virtbase + D40_DREG_PCBASE +
 973                     d40c->phy_chan->num * D40_DREG_PCDELTA +
 974                     D40_CHAN_REG_SSELT) >> D40_SREG_ELEM_LOG_ECNT_POS;
 975
 976         if (i == (d40d_prev->lli_len - 1) && val > 0) {
 977                 /* Change the current one */
 978                 writel(virt_to_phys(d40d->lli_phy.src),
 979                        d40c->base->virtbase + D40_DREG_PCBASE +
 980                        d40c->phy_chan->num * D40_DREG_PCDELTA +
 981                        D40_CHAN_REG_SSLNK);
 982                 writel(virt_to_phys(d40d->lli_phy.dst),
 983                        d40c->base->virtbase + D40_DREG_PCBASE +
 984                        d40c->phy_chan->num * D40_DREG_PCDELTA +
 985                        D40_CHAN_REG_SDLNK);
 986
 987                 d40d->is_hw_linked = true;
 988
 989         } else if (i < d40d_prev->lli_len) {
 990                 (void) dma_unmap_single(d40c->base->dev,
 991                                         virt_to_phys(d40d_prev->lli_phy.src),
 992                                         d40d_prev->lli_pool.size,
 993                                         DMA_TO_DEVICE);
 994
 995                 /* Keep the settings */
 996                 val = d40d_prev->lli_phy.src[d40d_prev->lli_len - 1].reg_lnk &
 997                         ~D40_SREG_LNK_PHYS_LNK_MASK;
 998                 d40d_prev->lli_phy.src[d40d_prev->lli_len - 1].reg_lnk =
 999                         val | virt_to_phys(d40d->lli_phy.src);
1000
1001                 val = d40d_prev->lli_phy.dst[d40d_prev->lli_len - 1].reg_lnk &
1002                         ~D40_SREG_LNK_PHYS_LNK_MASK;
1003                 d40d_prev->lli_phy.dst[d40d_prev->lli_len - 1].reg_lnk =
1004                         val | virt_to_phys(d40d->lli_phy.dst);
1005
1006                 (void) dma_map_single(d40c->base->dev,
1007                                       d40d_prev->lli_phy.src,
1008                                       d40d_prev->lli_pool.size,
1009                                       DMA_TO_DEVICE);
1010                 d40d->is_hw_linked = true;
1011         }
1012 }
1013
1014 static dma_cookie_t d40_tx_submit(struct dma_async_tx_descriptor *tx)
1015 {
1016         struct d40_chan *d40c = container_of(tx->chan,
1017                                              struct d40_chan,
1018                                              chan);
1019         struct d40_desc *d40d = container_of(tx, struct d40_desc, txd);
1020         unsigned long flags;
1021
1022         (void) d40_pause(&d40c->chan);
1023
1024         spin_lock_irqsave(&d40c->lock, flags);
1025
1026         d40c->chan.cookie++;
1027
1028         if (d40c->chan.cookie < 0)
1029                 d40c->chan.cookie = 1;
1030
1031         d40d->txd.cookie = d40c->chan.cookie;
1032
1033         if (d40c->log_num == D40_PHY_CHAN)
1034                 d40_tx_submit_phy(d40c, d40d);
1035         else
1036                 d40_tx_submit_log(d40c, d40d);
1037
1038         d40_desc_queue(d40c, d40d);
1039
1040         spin_unlock_irqrestore(&d40c->lock, flags);
1041
1042         (void) d40_resume(&d40c->chan);
1043
1044         return tx->cookie;
1045 }
1046
1047 static int d40_start(struct d40_chan *d40c)
1048 {
1049         if (d40c->base->rev == 0) {
1050                 int err;
1051
1052                 if (d40c->log_num != D40_PHY_CHAN) {
1053                         err = d40_channel_execute_command(d40c,
1054                                                           D40_DMA_SUSPEND_REQ);
1055                         if (err)
1056                                 return err;
1057                 }
1058         }
1059
1060         if (d40c->log_num != D40_PHY_CHAN)
1061                 d40_config_set_event(d40c, true);
1062
1063         return d40_channel_execute_command(d40c, D40_DMA_RUN);
1064 }
1065
1066 static struct d40_desc *d40_queue_start(struct d40_chan *d40c)
1067 {
1068         struct d40_desc *d40d;
1069         int err;
1070
1071         /* Start queued jobs, if any */
1072         d40d = d40_first_queued(d40c);
1073
1074         if (d40d != NULL) {
1075                 d40c->busy = true;
1076
1077                 /* Remove from queue */
1078                 d40_desc_remove(d40d);
1079
1080                 /* Add to active queue */
1081                 d40_desc_submit(d40c, d40d);
1082
1083                 /*
1084                  * If this job is already linked in hw,
1085                  * do not submit it.
1086                  */
1087
1088                 if (!d40d->is_hw_linked) {
1089                         /* Initiate DMA job */
1090                         d40_desc_load(d40c, d40d);
1091
1092                         /* Start dma job */
1093                         err = d40_start(d40c);
1094
1095                         if (err)
1096                                 return NULL;
1097                 }
1098         }
1099
1100         return d40d;
1101 }
1102
1103 /* called from interrupt context */
1104 static void dma_tc_handle(struct d40_chan *d40c)
1105 {
1106         struct d40_desc *d40d;
1107
1108         /* Get first active entry from list */
1109         d40d = d40_first_active_get(d40c);
1110
1111         if (d40d == NULL)
1112                 return;
1113
1114         d40_lcla_free_all(d40c, d40d);
1115
1116         if (d40d->lli_current < d40d->lli_len) {
1117                 d40_desc_load(d40c, d40d);
1118                 /* Start dma job */
1119                 (void) d40_start(d40c);
1120                 return;
1121         }
1122
1123         if (d40_queue_start(d40c) == NULL)
1124                 d40c->busy = false;
1125
1126         d40c->pending_tx++;
1127         tasklet_schedule(&d40c->tasklet);
1128
1129 }
1130
1131 static void dma_tasklet(unsigned long data)
1132 {
1133         struct d40_chan *d40c = (struct d40_chan *) data;
1134         struct d40_desc *d40d;
1135         unsigned long flags;
1136         dma_async_tx_callback callback;
1137         void *callback_param;
1138
1139         spin_lock_irqsave(&d40c->lock, flags);
1140
1141         /* Get first active entry from list */
1142         d40d = d40_first_active_get(d40c);
1143
1144         if (d40d == NULL)
1145                 goto err;
1146
1147         d40c->completed = d40d->txd.cookie;
1148
1149         /*
1150          * If terminating a channel pending_tx is set to zero.
1151          * This prevents any finished active jobs to return to the client.
1152          */
1153         if (d40c->pending_tx == 0) {
1154                 spin_unlock_irqrestore(&d40c->lock, flags);
1155                 return;
1156         }
1157
1158         /* Callback to client */
1159         callback = d40d->txd.callback;
1160         callback_param = d40d->txd.callback_param;
1161
1162         if (async_tx_test_ack(&d40d->txd)) {
1163                 d40_pool_lli_free(d40d);
1164                 d40_desc_remove(d40d);
1165                 d40_desc_free(d40c, d40d);
1166         } else {
1167                 if (!d40d->is_in_client_list) {
1168                         d40_desc_remove(d40d);
1169                         d40_lcla_free_all(d40c, d40d);
1170                         list_add_tail(&d40d->node, &d40c->client);
1171                         d40d->is_in_client_list = true;
1172                 }
1173         }
1174
1175         d40c->pending_tx--;
1176
1177         if (d40c->pending_tx)
1178                 tasklet_schedule(&d40c->tasklet);
1179
1180         spin_unlock_irqrestore(&d40c->lock, flags);
1181
1182         if (callback && (d40d->txd.flags & DMA_PREP_INTERRUPT))
1183                 callback(callback_param);
1184
1185         return;
1186
1187  err:
1188         /* Rescue manouver if receiving double interrupts */
1189         if (d40c->pending_tx > 0)
1190                 d40c->pending_tx--;
1191         spin_unlock_irqrestore(&d40c->lock, flags);
1192 }
1193
1194 static irqreturn_t d40_handle_interrupt(int irq, void *data)
1195 {
1196         static const struct d40_interrupt_lookup il[] = {
1197                 {D40_DREG_LCTIS0, D40_DREG_LCICR0, false,  0},
1198                 {D40_DREG_LCTIS1, D40_DREG_LCICR1, false, 32},
1199                 {D40_DREG_LCTIS2, D40_DREG_LCICR2, false, 64},
1200                 {D40_DREG_LCTIS3, D40_DREG_LCICR3, false, 96},
1201                 {D40_DREG_LCEIS0, D40_DREG_LCICR0, true,   0},
1202                 {D40_DREG_LCEIS1, D40_DREG_LCICR1, true,  32},
1203                 {D40_DREG_LCEIS2, D40_DREG_LCICR2, true,  64},
1204                 {D40_DREG_LCEIS3, D40_DREG_LCICR3, true,  96},
1205                 {D40_DREG_PCTIS,  D40_DREG_PCICR,  false, D40_PHY_CHAN},
1206                 {D40_DREG_PCEIS,  D40_DREG_PCICR,  true,  D40_PHY_CHAN},
1207         };
1208
1209         int i;
1210         u32 regs[ARRAY_SIZE(il)];
1211         u32 idx;
1212         u32 row;
1213         long chan = -1;
1214         struct d40_chan *d40c;
1215         unsigned long flags;
1216         struct d40_base *base = data;
1217
1218         spin_lock_irqsave(&base->interrupt_lock, flags);
1219
1220         /* Read interrupt status of both logical and physical channels */
1221         for (i = 0; i < ARRAY_SIZE(il); i++)
1222                 regs[i] = readl(base->virtbase + il[i].src);
1223
1224         for (;;) {
1225
1226                 chan = find_next_bit((unsigned long *)regs,
1227                                      BITS_PER_LONG * ARRAY_SIZE(il), chan + 1);
1228
1229                 /* No more set bits found? */
1230                 if (chan == BITS_PER_LONG * ARRAY_SIZE(il))
1231                         break;
1232
1233                 row = chan / BITS_PER_LONG;
1234                 idx = chan & (BITS_PER_LONG - 1);
1235
1236                 /* ACK interrupt */
1237                 writel(1 << idx, base->virtbase + il[row].clr);
1238
1239                 if (il[row].offset == D40_PHY_CHAN)
1240                         d40c = base->lookup_phy_chans[idx];
1241                 else
1242                         d40c = base->lookup_log_chans[il[row].offset + idx];
1243                 spin_lock(&d40c->lock);
1244
1245                 if (!il[row].is_error)
1246                         dma_tc_handle(d40c);
1247                 else
1248                         dev_err(base->dev,
1249                                 "[%s] IRQ chan: %ld offset %d idx %d\n",
1250                                 __func__, chan, il[row].offset, idx);
1251
1252                 spin_unlock(&d40c->lock);
1253         }
1254
1255         spin_unlock_irqrestore(&base->interrupt_lock, flags);
1256
1257         return IRQ_HANDLED;
1258 }
1259
1260 static int d40_validate_conf(struct d40_chan *d40c,
1261                              struct stedma40_chan_cfg *conf)
1262 {
1263         int res = 0;
1264         u32 dst_event_group = D40_TYPE_TO_GROUP(conf->dst_dev_type);
1265         u32 src_event_group = D40_TYPE_TO_GROUP(conf->src_dev_type);
1266         bool is_log = conf->mode == STEDMA40_MODE_LOGICAL;
1267
1268         if (!conf->dir) {
1269                 dev_err(&d40c->chan.dev->device, "[%s] Invalid direction.\n",
1270                         __func__);
1271                 res = -EINVAL;
1272         }
1273
1274         if (conf->dst_dev_type != STEDMA40_DEV_DST_MEMORY &&
1275             d40c->base->plat_data->dev_tx[conf->dst_dev_type] == 0 &&
1276             d40c->runtime_addr == 0) {
1277
1278                 dev_err(&d40c->chan.dev->device,
1279                         "[%s] Invalid TX channel address (%d)\n",
1280                         __func__, conf->dst_dev_type);
1281                 res = -EINVAL;
1282         }
1283
1284         if (conf->src_dev_type != STEDMA40_DEV_SRC_MEMORY &&
1285             d40c->base->plat_data->dev_rx[conf->src_dev_type] == 0 &&
1286             d40c->runtime_addr == 0) {
1287                 dev_err(&d40c->chan.dev->device,
1288                         "[%s] Invalid RX channel address (%d)\n",
1289                         __func__, conf->src_dev_type);
1290                 res = -EINVAL;
1291         }
1292
1293         if (conf->dir == STEDMA40_MEM_TO_PERIPH &&
1294             dst_event_group == STEDMA40_DEV_DST_MEMORY) {
1295                 dev_err(&d40c->chan.dev->device, "[%s] Invalid dst\n",
1296                         __func__);
1297                 res = -EINVAL;
1298         }
1299
1300         if (conf->dir == STEDMA40_PERIPH_TO_MEM &&
1301             src_event_group == STEDMA40_DEV_SRC_MEMORY) {
1302                 dev_err(&d40c->chan.dev->device, "[%s] Invalid src\n",
1303                         __func__);
1304                 res = -EINVAL;
1305         }
1306
1307         if (src_event_group == STEDMA40_DEV_SRC_MEMORY &&
1308             dst_event_group == STEDMA40_DEV_DST_MEMORY && is_log) {
1309                 dev_err(&d40c->chan.dev->device,
1310                         "[%s] No event line\n", __func__);
1311                 res = -EINVAL;
1312         }
1313
1314         if (conf->dir == STEDMA40_PERIPH_TO_PERIPH &&
1315             (src_event_group != dst_event_group)) {
1316                 dev_err(&d40c->chan.dev->device,
1317                         "[%s] Invalid event group\n", __func__);
1318                 res = -EINVAL;
1319         }
1320
1321         if (conf->dir == STEDMA40_PERIPH_TO_PERIPH) {
1322                 /*
1323                  * DMAC HW supports it. Will be added to this driver,
1324                  * in case any dma client requires it.
1325                  */
1326                 dev_err(&d40c->chan.dev->device,
1327                         "[%s] periph to periph not supported\n",
1328                         __func__);
1329                 res = -EINVAL;
1330         }
1331
1332         if (d40_psize_2_burst_size(is_log, conf->src_info.psize) *
1333             (1 << conf->src_info.data_width) !=
1334             d40_psize_2_burst_size(is_log, conf->dst_info.psize) *
1335             (1 << conf->dst_info.data_width)) {
1336                 /*
1337                  * The DMAC hardware only supports
1338                  * src (burst x width) == dst (burst x width)
1339                  */
1340
1341                 dev_err(&d40c->chan.dev->device,
1342                         "[%s] src (burst x width) != dst (burst x width)\n",
1343                         __func__);
1344                 res = -EINVAL;
1345         }
1346
1347         return res;
1348 }
1349
1350 static bool d40_alloc_mask_set(struct d40_phy_res *phy, bool is_src,
1351                                int log_event_line, bool is_log)
1352 {
1353         unsigned long flags;
1354         spin_lock_irqsave(&phy->lock, flags);
1355         if (!is_log) {
1356                 /* Physical interrupts are masked per physical full channel */
1357                 if (phy->allocated_src == D40_ALLOC_FREE &&
1358                     phy->allocated_dst == D40_ALLOC_FREE) {
1359                         phy->allocated_dst = D40_ALLOC_PHY;
1360                         phy->allocated_src = D40_ALLOC_PHY;
1361                         goto found;
1362                 } else
1363                         goto not_found;
1364         }
1365
1366         /* Logical channel */
1367         if (is_src) {
1368                 if (phy->allocated_src == D40_ALLOC_PHY)
1369                         goto not_found;
1370
1371                 if (phy->allocated_src == D40_ALLOC_FREE)
1372                         phy->allocated_src = D40_ALLOC_LOG_FREE;
1373
1374                 if (!(phy->allocated_src & (1 << log_event_line))) {
1375                         phy->allocated_src |= 1 << log_event_line;
1376                         goto found;
1377                 } else
1378                         goto not_found;
1379         } else {
1380                 if (phy->allocated_dst == D40_ALLOC_PHY)
1381                         goto not_found;
1382
1383                 if (phy->allocated_dst == D40_ALLOC_FREE)
1384                         phy->allocated_dst = D40_ALLOC_LOG_FREE;
1385
1386                 if (!(phy->allocated_dst & (1 << log_event_line))) {
1387                         phy->allocated_dst |= 1 << log_event_line;
1388                         goto found;
1389                 } else
1390                         goto not_found;
1391         }
1392
1393 not_found:
1394         spin_unlock_irqrestore(&phy->lock, flags);
1395         return false;
1396 found:
1397         spin_unlock_irqrestore(&phy->lock, flags);
1398         return true;
1399 }
1400
1401 static bool d40_alloc_mask_free(struct d40_phy_res *phy, bool is_src,
1402                                int log_event_line)
1403 {
1404         unsigned long flags;
1405         bool is_free = false;
1406
1407         spin_lock_irqsave(&phy->lock, flags);
1408         if (!log_event_line) {
1409                 phy->allocated_dst = D40_ALLOC_FREE;
1410                 phy->allocated_src = D40_ALLOC_FREE;
1411                 is_free = true;
1412                 goto out;
1413         }
1414
1415         /* Logical channel */
1416         if (is_src) {
1417                 phy->allocated_src &= ~(1 << log_event_line);
1418                 if (phy->allocated_src == D40_ALLOC_LOG_FREE)
1419                         phy->allocated_src = D40_ALLOC_FREE;
1420         } else {
1421                 phy->allocated_dst &= ~(1 << log_event_line);
1422                 if (phy->allocated_dst == D40_ALLOC_LOG_FREE)
1423                         phy->allocated_dst = D40_ALLOC_FREE;
1424         }
1425
1426         is_free = ((phy->allocated_src | phy->allocated_dst) ==
1427                    D40_ALLOC_FREE);
1428
1429 out:
1430         spin_unlock_irqrestore(&phy->lock, flags);
1431
1432         return is_free;
1433 }
1434
1435 static int d40_allocate_channel(struct d40_chan *d40c)
1436 {
1437         int dev_type;
1438         int event_group;
1439         int event_line;
1440         struct d40_phy_res *phys;
1441         int i;
1442         int j;
1443         int log_num;
1444         bool is_src;
1445         bool is_log = d40c->dma_cfg.mode == STEDMA40_MODE_LOGICAL;
1446
1447         phys = d40c->base->phy_res;
1448
1449         if (d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_MEM) {
1450                 dev_type = d40c->dma_cfg.src_dev_type;
1451                 log_num = 2 * dev_type;
1452                 is_src = true;
1453         } else if (d40c->dma_cfg.dir == STEDMA40_MEM_TO_PERIPH ||
1454                    d40c->dma_cfg.dir == STEDMA40_MEM_TO_MEM) {
1455                 /* dst event lines are used for logical memcpy */
1456                 dev_type = d40c->dma_cfg.dst_dev_type;
1457                 log_num = 2 * dev_type + 1;
1458                 is_src = false;
1459         } else
1460                 return -EINVAL;
1461
1462         event_group = D40_TYPE_TO_GROUP(dev_type);
1463         event_line = D40_TYPE_TO_EVENT(dev_type);
1464
1465         if (!is_log) {
1466                 if (d40c->dma_cfg.dir == STEDMA40_MEM_TO_MEM) {
1467                         /* Find physical half channel */
1468                         for (i = 0; i < d40c->base->num_phy_chans; i++) {
1469
1470                                 if (d40_alloc_mask_set(&phys[i], is_src,
1471                                                        0, is_log))
1472                                         goto found_phy;
1473                         }
1474                 } else
1475                         for (j = 0; j < d40c->base->num_phy_chans; j += 8) {
1476                                 int phy_num = j  + event_group * 2;
1477                                 for (i = phy_num; i < phy_num + 2; i++) {
1478                                         if (d40_alloc_mask_set(&phys[i],
1479                                                                is_src,
1480                                                                0,
1481                                                                is_log))
1482                                                 goto found_phy;
1483                                 }
1484                         }
1485                 return -EINVAL;
1486 found_phy:
1487                 d40c->phy_chan = &phys[i];
1488                 d40c->log_num = D40_PHY_CHAN;
1489                 goto out;
1490         }
1491         if (dev_type == -1)
1492                 return -EINVAL;
1493
1494         /* Find logical channel */
1495         for (j = 0; j < d40c->base->num_phy_chans; j += 8) {
1496                 int phy_num = j + event_group * 2;
1497                 /*
1498                  * Spread logical channels across all available physical rather
1499                  * than pack every logical channel at the first available phy
1500                  * channels.
1501                  */
1502                 if (is_src) {
1503                         for (i = phy_num; i < phy_num + 2; i++) {
1504                                 if (d40_alloc_mask_set(&phys[i], is_src,
1505                                                        event_line, is_log))
1506                                         goto found_log;
1507                         }
1508                 } else {
1509                         for (i = phy_num + 1; i >= phy_num; i--) {
1510                                 if (d40_alloc_mask_set(&phys[i], is_src,
1511                                                        event_line, is_log))
1512                                         goto found_log;
1513                         }
1514                 }
1515         }
1516         return -EINVAL;
1517
1518 found_log:
1519         d40c->phy_chan = &phys[i];
1520         d40c->log_num = log_num;
1521 out:
1522
1523         if (is_log)
1524                 d40c->base->lookup_log_chans[d40c->log_num] = d40c;
1525         else
1526                 d40c->base->lookup_phy_chans[d40c->phy_chan->num] = d40c;
1527
1528         return 0;
1529
1530 }
1531
1532 static int d40_config_memcpy(struct d40_chan *d40c)
1533 {
1534         dma_cap_mask_t cap = d40c->chan.device->cap_mask;
1535
1536         if (dma_has_cap(DMA_MEMCPY, cap) && !dma_has_cap(DMA_SLAVE, cap)) {
1537                 d40c->dma_cfg = *d40c->base->plat_data->memcpy_conf_log;
1538                 d40c->dma_cfg.src_dev_type = STEDMA40_DEV_SRC_MEMORY;
1539                 d40c->dma_cfg.dst_dev_type = d40c->base->plat_data->
1540                         memcpy[d40c->chan.chan_id];
1541
1542         } else if (dma_has_cap(DMA_MEMCPY, cap) &&
1543                    dma_has_cap(DMA_SLAVE, cap)) {
1544                 d40c->dma_cfg = *d40c->base->plat_data->memcpy_conf_phy;
1545         } else {
1546                 dev_err(&d40c->chan.dev->device, "[%s] No memcpy\n",
1547                         __func__);
1548                 return -EINVAL;
1549         }
1550
1551         return 0;
1552 }
1553
1554
1555 static int d40_free_dma(struct d40_chan *d40c)
1556 {
1557
1558         int res = 0;
1559         u32 event;
1560         struct d40_phy_res *phy = d40c->phy_chan;
1561         bool is_src;
1562         struct d40_desc *d;
1563         struct d40_desc *_d;
1564
1565
1566         /* Terminate all queued and active transfers */
1567         d40_term_all(d40c);
1568
1569         /* Release client owned descriptors */
1570         if (!list_empty(&d40c->client))
1571                 list_for_each_entry_safe(d, _d, &d40c->client, node) {
1572                         d40_pool_lli_free(d);
1573                         d40_desc_remove(d);
1574                         d40_desc_free(d40c, d);
1575                 }
1576
1577         if (phy == NULL) {
1578                 dev_err(&d40c->chan.dev->device, "[%s] phy == null\n",
1579                         __func__);
1580                 return -EINVAL;
1581         }
1582
1583         if (phy->allocated_src == D40_ALLOC_FREE &&
1584             phy->allocated_dst == D40_ALLOC_FREE) {
1585                 dev_err(&d40c->chan.dev->device, "[%s] channel already free\n",
1586                         __func__);
1587                 return -EINVAL;
1588         }
1589
1590         if (d40c->dma_cfg.dir == STEDMA40_MEM_TO_PERIPH ||
1591             d40c->dma_cfg.dir == STEDMA40_MEM_TO_MEM) {
1592                 event = D40_TYPE_TO_EVENT(d40c->dma_cfg.dst_dev_type);
1593                 is_src = false;
1594         } else if (d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_MEM) {
1595                 event = D40_TYPE_TO_EVENT(d40c->dma_cfg.src_dev_type);
1596                 is_src = true;
1597         } else {
1598                 dev_err(&d40c->chan.dev->device,
1599                         "[%s] Unknown direction\n", __func__);
1600                 return -EINVAL;
1601         }
1602
1603         res = d40_channel_execute_command(d40c, D40_DMA_SUSPEND_REQ);
1604         if (res) {
1605                 dev_err(&d40c->chan.dev->device, "[%s] suspend failed\n",
1606                         __func__);
1607                 return res;
1608         }
1609
1610         if (d40c->log_num != D40_PHY_CHAN) {
1611                 /* Release logical channel, deactivate the event line */
1612
1613                 d40_config_set_event(d40c, false);
1614                 d40c->base->lookup_log_chans[d40c->log_num] = NULL;
1615
1616                 /*
1617                  * Check if there are more logical allocation
1618                  * on this phy channel.
1619                  */
1620                 if (!d40_alloc_mask_free(phy, is_src, event)) {
1621                         /* Resume the other logical channels if any */
1622                         if (d40_chan_has_events(d40c)) {
1623                                 res = d40_channel_execute_command(d40c,
1624                                                                   D40_DMA_RUN);
1625                                 if (res) {
1626                                         dev_err(&d40c->chan.dev->device,
1627                                                 "[%s] Executing RUN command\n",
1628                                                 __func__);
1629                                         return res;
1630                                 }
1631                         }
1632                         return 0;
1633                 }
1634         } else {
1635                 (void) d40_alloc_mask_free(phy, is_src, 0);
1636         }
1637
1638         /* Release physical channel */
1639         res = d40_channel_execute_command(d40c, D40_DMA_STOP);
1640         if (res) {
1641                 dev_err(&d40c->chan.dev->device,
1642                         "[%s] Failed to stop channel\n", __func__);
1643                 return res;
1644         }
1645         d40c->phy_chan = NULL;
1646         d40c->configured = false;
1647         d40c->base->lookup_phy_chans[phy->num] = NULL;
1648
1649         return 0;
1650 }
1651
1652 static bool d40_is_paused(struct d40_chan *d40c)
1653 {
1654         bool is_paused = false;
1655         unsigned long flags;
1656         void __iomem *active_reg;
1657         u32 status;
1658         u32 event;
1659
1660         spin_lock_irqsave(&d40c->lock, flags);
1661
1662         if (d40c->log_num == D40_PHY_CHAN) {
1663                 if (d40c->phy_chan->num % 2 == 0)
1664                         active_reg = d40c->base->virtbase + D40_DREG_ACTIVE;
1665                 else
1666                         active_reg = d40c->base->virtbase + D40_DREG_ACTIVO;
1667
1668                 status = (readl(active_reg) &
1669                           D40_CHAN_POS_MASK(d40c->phy_chan->num)) >>
1670                         D40_CHAN_POS(d40c->phy_chan->num);
1671                 if (status == D40_DMA_SUSPENDED || status == D40_DMA_STOP)
1672                         is_paused = true;
1673
1674                 goto _exit;
1675         }
1676
1677         if (d40c->dma_cfg.dir == STEDMA40_MEM_TO_PERIPH ||
1678             d40c->dma_cfg.dir == STEDMA40_MEM_TO_MEM) {
1679                 event = D40_TYPE_TO_EVENT(d40c->dma_cfg.dst_dev_type);
1680                 status = readl(d40c->base->virtbase + D40_DREG_PCBASE +
1681                                d40c->phy_chan->num * D40_DREG_PCDELTA +
1682                                D40_CHAN_REG_SDLNK);
1683         } else if (d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_MEM) {
1684                 event = D40_TYPE_TO_EVENT(d40c->dma_cfg.src_dev_type);
1685                 status = readl(d40c->base->virtbase + D40_DREG_PCBASE +
1686                                d40c->phy_chan->num * D40_DREG_PCDELTA +
1687                                D40_CHAN_REG_SSLNK);
1688         } else {
1689                 dev_err(&d40c->chan.dev->device,
1690                         "[%s] Unknown direction\n", __func__);
1691                 goto _exit;
1692         }
1693
1694         status = (status & D40_EVENTLINE_MASK(event)) >>
1695                 D40_EVENTLINE_POS(event);
1696
1697         if (status != D40_DMA_RUN)
1698                 is_paused = true;
1699 _exit:
1700         spin_unlock_irqrestore(&d40c->lock, flags);
1701         return is_paused;
1702
1703 }
1704
1705
1706 static u32 stedma40_residue(struct dma_chan *chan)
1707 {
1708         struct d40_chan *d40c =
1709                 container_of(chan, struct d40_chan, chan);
1710         u32 bytes_left;
1711         unsigned long flags;
1712
1713         spin_lock_irqsave(&d40c->lock, flags);
1714         bytes_left = d40_residue(d40c);
1715         spin_unlock_irqrestore(&d40c->lock, flags);
1716
1717         return bytes_left;
1718 }
1719
1720 struct dma_async_tx_descriptor *stedma40_memcpy_sg(struct dma_chan *chan,
1721                                                    struct scatterlist *sgl_dst,
1722                                                    struct scatterlist *sgl_src,
1723                                                    unsigned int sgl_len,
1724                                                    unsigned long dma_flags)
1725 {
1726         int res;
1727         struct d40_desc *d40d;
1728         struct d40_chan *d40c = container_of(chan, struct d40_chan,
1729                                              chan);
1730         unsigned long flags;
1731
1732         if (d40c->phy_chan == NULL) {
1733                 dev_err(&d40c->chan.dev->device,
1734                         "[%s] Unallocated channel.\n", __func__);
1735                 return ERR_PTR(-EINVAL);
1736         }
1737
1738         spin_lock_irqsave(&d40c->lock, flags);
1739         d40d = d40_desc_get(d40c);
1740
1741         if (d40d == NULL)
1742                 goto err;
1743
1744         d40d->lli_len = d40_sg_2_dmalen(sgl_dst, sgl_len,
1745                                         d40c->dma_cfg.src_info.data_width,
1746                                         d40c->dma_cfg.dst_info.data_width);
1747         if (d40d->lli_len < 0) {
1748                 dev_err(&d40c->chan.dev->device,
1749                         "[%s] Unaligned size\n", __func__);
1750                 goto err;
1751         }
1752
1753         d40d->lli_current = 0;
1754         d40d->txd.flags = dma_flags;
1755
1756         if (d40c->log_num != D40_PHY_CHAN) {
1757
1758                 if (d40_pool_lli_alloc(d40d, d40d->lli_len, true) < 0) {
1759                         dev_err(&d40c->chan.dev->device,
1760                                 "[%s] Out of memory\n", __func__);
1761                         goto err;
1762                 }
1763
1764                 (void) d40_log_sg_to_lli(sgl_src,
1765                                          sgl_len,
1766                                          d40d->lli_log.src,
1767                                          d40c->log_def.lcsp1,
1768                                          d40c->dma_cfg.src_info.data_width,
1769                                          d40c->dma_cfg.dst_info.data_width);
1770
1771                 (void) d40_log_sg_to_lli(sgl_dst,
1772                                          sgl_len,
1773                                          d40d->lli_log.dst,
1774                                          d40c->log_def.lcsp3,
1775                                          d40c->dma_cfg.dst_info.data_width,
1776                                          d40c->dma_cfg.src_info.data_width);
1777         } else {
1778                 if (d40_pool_lli_alloc(d40d, d40d->lli_len, false) < 0) {
1779                         dev_err(&d40c->chan.dev->device,
1780                                 "[%s] Out of memory\n", __func__);
1781                         goto err;
1782                 }
1783
1784                 res = d40_phy_sg_to_lli(sgl_src,
1785                                         sgl_len,
1786                                         0,
1787                                         d40d->lli_phy.src,
1788                                         virt_to_phys(d40d->lli_phy.src),
1789                                         d40c->src_def_cfg,
1790                                         d40c->dma_cfg.src_info.data_width,
1791                                         d40c->dma_cfg.dst_info.data_width,
1792                                         d40c->dma_cfg.src_info.psize);
1793
1794                 if (res < 0)
1795                         goto err;
1796
1797                 res = d40_phy_sg_to_lli(sgl_dst,
1798                                         sgl_len,
1799                                         0,
1800                                         d40d->lli_phy.dst,
1801                                         virt_to_phys(d40d->lli_phy.dst),
1802                                         d40c->dst_def_cfg,
1803                                         d40c->dma_cfg.dst_info.data_width,
1804                                         d40c->dma_cfg.src_info.data_width,
1805                                         d40c->dma_cfg.dst_info.psize);
1806
1807                 if (res < 0)
1808                         goto err;
1809
1810                 (void) dma_map_single(d40c->base->dev, d40d->lli_phy.src,
1811                                       d40d->lli_pool.size, DMA_TO_DEVICE);
1812         }
1813
1814         dma_async_tx_descriptor_init(&d40d->txd, chan);
1815
1816         d40d->txd.tx_submit = d40_tx_submit;
1817
1818         spin_unlock_irqrestore(&d40c->lock, flags);
1819
1820         return &d40d->txd;
1821 err:
1822         if (d40d)
1823                 d40_desc_free(d40c, d40d);
1824         spin_unlock_irqrestore(&d40c->lock, flags);
1825         return NULL;
1826 }
1827 EXPORT_SYMBOL(stedma40_memcpy_sg);
1828
1829 bool stedma40_filter(struct dma_chan *chan, void *data)
1830 {
1831         struct stedma40_chan_cfg *info = data;
1832         struct d40_chan *d40c =
1833                 container_of(chan, struct d40_chan, chan);
1834         int err;
1835
1836         if (data) {
1837                 err = d40_validate_conf(d40c, info);
1838                 if (!err)
1839                         d40c->dma_cfg = *info;
1840         } else
1841                 err = d40_config_memcpy(d40c);
1842
1843         if (!err)
1844                 d40c->configured = true;
1845
1846         return err == 0;
1847 }
1848 EXPORT_SYMBOL(stedma40_filter);
1849
1850 /* DMA ENGINE functions */
1851 static int d40_alloc_chan_resources(struct dma_chan *chan)
1852 {
1853         int err;
1854         unsigned long flags;
1855         struct d40_chan *d40c =
1856                 container_of(chan, struct d40_chan, chan);
1857         bool is_free_phy;
1858         spin_lock_irqsave(&d40c->lock, flags);
1859
1860         d40c->completed = chan->cookie = 1;
1861
1862         /* If no dma configuration is set use default configuration (memcpy) */
1863         if (!d40c->configured) {
1864                 err = d40_config_memcpy(d40c);
1865                 if (err) {
1866                         dev_err(&d40c->chan.dev->device,
1867                                 "[%s] Failed to configure memcpy channel\n",
1868                                 __func__);
1869                         goto fail;
1870                 }
1871         }
1872         is_free_phy = (d40c->phy_chan == NULL);
1873
1874         err = d40_allocate_channel(d40c);
1875         if (err) {
1876                 dev_err(&d40c->chan.dev->device,
1877                         "[%s] Failed to allocate channel\n", __func__);
1878                 goto fail;
1879         }
1880
1881         /* Fill in basic CFG register values */
1882         d40_phy_cfg(&d40c->dma_cfg, &d40c->src_def_cfg,
1883                     &d40c->dst_def_cfg, d40c->log_num != D40_PHY_CHAN);
1884
1885         if (d40c->log_num != D40_PHY_CHAN) {
1886                 d40_log_cfg(&d40c->dma_cfg,
1887                             &d40c->log_def.lcsp1, &d40c->log_def.lcsp3);
1888
1889                 if (d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_MEM)
1890                         d40c->lcpa = d40c->base->lcpa_base +
1891                           d40c->dma_cfg.src_dev_type * D40_LCPA_CHAN_SIZE;
1892                 else
1893                         d40c->lcpa = d40c->base->lcpa_base +
1894                           d40c->dma_cfg.dst_dev_type *
1895                           D40_LCPA_CHAN_SIZE + D40_LCPA_CHAN_DST_DELTA;
1896         }
1897
1898         /*
1899          * Only write channel configuration to the DMA if the physical
1900          * resource is free. In case of multiple logical channels
1901          * on the same physical resource, only the first write is necessary.
1902          */
1903         if (is_free_phy)
1904                 d40_config_write(d40c);
1905 fail:
1906         spin_unlock_irqrestore(&d40c->lock, flags);
1907         return err;
1908 }
1909
1910 static void d40_free_chan_resources(struct dma_chan *chan)
1911 {
1912         struct d40_chan *d40c =
1913                 container_of(chan, struct d40_chan, chan);
1914         int err;
1915         unsigned long flags;
1916
1917         if (d40c->phy_chan == NULL) {
1918                 dev_err(&d40c->chan.dev->device,
1919                         "[%s] Cannot free unallocated channel\n", __func__);
1920                 return;
1921         }
1922
1923
1924         spin_lock_irqsave(&d40c->lock, flags);
1925
1926         err = d40_free_dma(d40c);
1927
1928         if (err)
1929                 dev_err(&d40c->chan.dev->device,
1930                         "[%s] Failed to free channel\n", __func__);
1931         spin_unlock_irqrestore(&d40c->lock, flags);
1932 }
1933
1934 static struct dma_async_tx_descriptor *d40_prep_memcpy(struct dma_chan *chan,
1935                                                        dma_addr_t dst,
1936                                                        dma_addr_t src,
1937                                                        size_t size,
1938                                                        unsigned long dma_flags)
1939 {
1940         struct d40_desc *d40d;
1941         struct d40_chan *d40c = container_of(chan, struct d40_chan,
1942                                              chan);
1943         unsigned long flags;
1944
1945         if (d40c->phy_chan == NULL) {
1946                 dev_err(&d40c->chan.dev->device,
1947                         "[%s] Channel is not allocated.\n", __func__);
1948                 return ERR_PTR(-EINVAL);
1949         }
1950
1951         spin_lock_irqsave(&d40c->lock, flags);
1952         d40d = d40_desc_get(d40c);
1953
1954         if (d40d == NULL) {
1955                 dev_err(&d40c->chan.dev->device,
1956                         "[%s] Descriptor is NULL\n", __func__);
1957                 goto err;
1958         }
1959
1960         d40d->txd.flags = dma_flags;
1961         d40d->lli_len = d40_size_2_dmalen(size,
1962                                           d40c->dma_cfg.src_info.data_width,
1963                                           d40c->dma_cfg.dst_info.data_width);
1964         if (d40d->lli_len < 0) {
1965                 dev_err(&d40c->chan.dev->device,
1966                         "[%s] Unaligned size\n", __func__);
1967                 goto err;
1968         }
1969
1970
1971         dma_async_tx_descriptor_init(&d40d->txd, chan);
1972
1973         d40d->txd.tx_submit = d40_tx_submit;
1974
1975         if (d40c->log_num != D40_PHY_CHAN) {
1976
1977                 if (d40_pool_lli_alloc(d40d, d40d->lli_len, true) < 0) {
1978                         dev_err(&d40c->chan.dev->device,
1979                                 "[%s] Out of memory\n", __func__);
1980                         goto err;
1981                 }
1982                 d40d->lli_current = 0;
1983
1984                 if (d40_log_buf_to_lli(d40d->lli_log.src,
1985                                        src,
1986                                        size,
1987                                        d40c->log_def.lcsp1,
1988                                        d40c->dma_cfg.src_info.data_width,
1989                                        d40c->dma_cfg.dst_info.data_width,
1990                                        true) == NULL)
1991                         goto err;
1992
1993                 if (d40_log_buf_to_lli(d40d->lli_log.dst,
1994                                        dst,
1995                                        size,
1996                                        d40c->log_def.lcsp3,
1997                                        d40c->dma_cfg.dst_info.data_width,
1998                                        d40c->dma_cfg.src_info.data_width,
1999                                        true) == NULL)
2000                         goto err;
2001
2002         } else {
2003
2004                 if (d40_pool_lli_alloc(d40d, d40d->lli_len, false) < 0) {
2005                         dev_err(&d40c->chan.dev->device,
2006                                 "[%s] Out of memory\n", __func__);
2007                         goto err;
2008                 }
2009
2010                 if (d40_phy_buf_to_lli(d40d->lli_phy.src,
2011                                        src,
2012                                        size,
2013                                        d40c->dma_cfg.src_info.psize,
2014                                        0,
2015                                        d40c->src_def_cfg,
2016                                        true,
2017                                        d40c->dma_cfg.src_info.data_width,
2018                                        d40c->dma_cfg.dst_info.data_width,
2019                                        false) == NULL)
2020                         goto err;
2021
2022                 if (d40_phy_buf_to_lli(d40d->lli_phy.dst,
2023                                        dst,
2024                                        size,
2025                                        d40c->dma_cfg.dst_info.psize,
2026                                        0,
2027                                        d40c->dst_def_cfg,
2028                                        true,
2029                                        d40c->dma_cfg.dst_info.data_width,
2030                                        d40c->dma_cfg.src_info.data_width,
2031                                        false) == NULL)
2032                         goto err;
2033
2034                 (void) dma_map_single(d40c->base->dev, d40d->lli_phy.src,
2035                                       d40d->lli_pool.size, DMA_TO_DEVICE);
2036         }
2037
2038         spin_unlock_irqrestore(&d40c->lock, flags);
2039         return &d40d->txd;
2040
2041 err:
2042         if (d40d)
2043                 d40_desc_free(d40c, d40d);
2044         spin_unlock_irqrestore(&d40c->lock, flags);
2045         return NULL;
2046 }
2047
2048 static struct dma_async_tx_descriptor *
2049 d40_prep_sg(struct dma_chan *chan,
2050             struct scatterlist *dst_sg, unsigned int dst_nents,
2051             struct scatterlist *src_sg, unsigned int src_nents,
2052             unsigned long dma_flags)
2053 {
2054         if (dst_nents != src_nents)
2055                 return NULL;
2056
2057         return stedma40_memcpy_sg(chan, dst_sg, src_sg, dst_nents, dma_flags);
2058 }
2059
2060 static int d40_prep_slave_sg_log(struct d40_desc *d40d,
2061                                  struct d40_chan *d40c,
2062                                  struct scatterlist *sgl,
2063                                  unsigned int sg_len,
2064                                  enum dma_data_direction direction,
2065                                  unsigned long dma_flags)
2066 {
2067         dma_addr_t dev_addr = 0;
2068         int total_size;
2069
2070         d40d->lli_len = d40_sg_2_dmalen(sgl, sg_len,
2071                                         d40c->dma_cfg.src_info.data_width,
2072                                         d40c->dma_cfg.dst_info.data_width);
2073         if (d40d->lli_len < 0) {
2074                 dev_err(&d40c->chan.dev->device,
2075                         "[%s] Unaligned size\n", __func__);
2076                 return -EINVAL;
2077         }
2078
2079         if (d40_pool_lli_alloc(d40d, d40d->lli_len, true) < 0) {
2080                 dev_err(&d40c->chan.dev->device,
2081                         "[%s] Out of memory\n", __func__);
2082                 return -ENOMEM;
2083         }
2084
2085         d40d->lli_current = 0;
2086
2087         if (direction == DMA_FROM_DEVICE)
2088                 if (d40c->runtime_addr)
2089                         dev_addr = d40c->runtime_addr;
2090                 else
2091                         dev_addr = d40c->base->plat_data->dev_rx[d40c->dma_cfg.src_dev_type];
2092         else if (direction == DMA_TO_DEVICE)
2093                 if (d40c->runtime_addr)
2094                         dev_addr = d40c->runtime_addr;
2095                 else
2096                         dev_addr = d40c->base->plat_data->dev_tx[d40c->dma_cfg.dst_dev_type];
2097
2098         else
2099                 return -EINVAL;
2100
2101         total_size = d40_log_sg_to_dev(sgl, sg_len,
2102                                        &d40d->lli_log,
2103                                        &d40c->log_def,
2104                                        d40c->dma_cfg.src_info.data_width,
2105                                        d40c->dma_cfg.dst_info.data_width,
2106                                        direction,
2107                                        dev_addr);
2108
2109         if (total_size < 0)
2110                 return -EINVAL;
2111
2112         return 0;
2113 }
2114
2115 static int d40_prep_slave_sg_phy(struct d40_desc *d40d,
2116                                  struct d40_chan *d40c,
2117                                  struct scatterlist *sgl,
2118                                  unsigned int sgl_len,
2119                                  enum dma_data_direction direction,
2120                                  unsigned long dma_flags)
2121 {
2122         dma_addr_t src_dev_addr;
2123         dma_addr_t dst_dev_addr;
2124         int res;
2125
2126         d40d->lli_len = d40_sg_2_dmalen(sgl, sgl_len,
2127                                         d40c->dma_cfg.src_info.data_width,
2128                                         d40c->dma_cfg.dst_info.data_width);
2129         if (d40d->lli_len < 0) {
2130                 dev_err(&d40c->chan.dev->device,
2131                         "[%s] Unaligned size\n", __func__);
2132                 return -EINVAL;
2133         }
2134
2135         if (d40_pool_lli_alloc(d40d, d40d->lli_len, false) < 0) {
2136                 dev_err(&d40c->chan.dev->device,
2137                         "[%s] Out of memory\n", __func__);
2138                 return -ENOMEM;
2139         }
2140
2141         d40d->lli_current = 0;
2142
2143         if (direction == DMA_FROM_DEVICE) {
2144                 dst_dev_addr = 0;
2145                 if (d40c->runtime_addr)
2146                         src_dev_addr = d40c->runtime_addr;
2147                 else
2148                         src_dev_addr = d40c->base->plat_data->dev_rx[d40c->dma_cfg.src_dev_type];
2149         } else if (direction == DMA_TO_DEVICE) {
2150                 if (d40c->runtime_addr)
2151                         dst_dev_addr = d40c->runtime_addr;
2152                 else
2153                         dst_dev_addr = d40c->base->plat_data->dev_tx[d40c->dma_cfg.dst_dev_type];
2154                 src_dev_addr = 0;
2155         } else
2156                 return -EINVAL;
2157
2158         res = d40_phy_sg_to_lli(sgl,
2159                                 sgl_len,
2160                                 src_dev_addr,
2161                                 d40d->lli_phy.src,
2162                                 virt_to_phys(d40d->lli_phy.src),
2163                                 d40c->src_def_cfg,
2164                                 d40c->dma_cfg.src_info.data_width,
2165                                 d40c->dma_cfg.dst_info.data_width,
2166                                 d40c->dma_cfg.src_info.psize);
2167         if (res < 0)
2168                 return res;
2169
2170         res = d40_phy_sg_to_lli(sgl,
2171                                 sgl_len,
2172                                 dst_dev_addr,
2173                                 d40d->lli_phy.dst,
2174                                 virt_to_phys(d40d->lli_phy.dst),
2175                                 d40c->dst_def_cfg,
2176                                 d40c->dma_cfg.dst_info.data_width,
2177                                 d40c->dma_cfg.src_info.data_width,
2178                                 d40c->dma_cfg.dst_info.psize);
2179         if (res < 0)
2180                 return res;
2181
2182         (void) dma_map_single(d40c->base->dev, d40d->lli_phy.src,
2183                               d40d->lli_pool.size, DMA_TO_DEVICE);
2184         return 0;
2185 }
2186
2187 static struct dma_async_tx_descriptor *d40_prep_slave_sg(struct dma_chan *chan,
2188                                                          struct scatterlist *sgl,
2189                                                          unsigned int sg_len,
2190                                                          enum dma_data_direction direction,
2191                                                          unsigned long dma_flags)
2192 {
2193         struct d40_desc *d40d;
2194         struct d40_chan *d40c = container_of(chan, struct d40_chan,
2195                                              chan);
2196         unsigned long flags;
2197         int err;
2198
2199         if (d40c->phy_chan == NULL) {
2200                 dev_err(&d40c->chan.dev->device,
2201                         "[%s] Cannot prepare unallocated channel\n", __func__);
2202                 return ERR_PTR(-EINVAL);
2203         }
2204
2205         spin_lock_irqsave(&d40c->lock, flags);
2206         d40d = d40_desc_get(d40c);
2207
2208         if (d40d == NULL)
2209                 goto err;
2210
2211         if (d40c->log_num != D40_PHY_CHAN)
2212                 err = d40_prep_slave_sg_log(d40d, d40c, sgl, sg_len,
2213                                             direction, dma_flags);
2214         else
2215                 err = d40_prep_slave_sg_phy(d40d, d40c, sgl, sg_len,
2216                                             direction, dma_flags);
2217         if (err) {
2218                 dev_err(&d40c->chan.dev->device,
2219                         "[%s] Failed to prepare %s slave sg job: %d\n",
2220                         __func__,
2221                         d40c->log_num != D40_PHY_CHAN ? "log" : "phy", err);
2222                 goto err;
2223         }
2224
2225         d40d->txd.flags = dma_flags;
2226
2227         dma_async_tx_descriptor_init(&d40d->txd, chan);
2228
2229         d40d->txd.tx_submit = d40_tx_submit;
2230
2231         spin_unlock_irqrestore(&d40c->lock, flags);
2232         return &d40d->txd;
2233
2234 err:
2235         if (d40d)
2236                 d40_desc_free(d40c, d40d);
2237         spin_unlock_irqrestore(&d40c->lock, flags);
2238         return NULL;
2239 }
2240
2241 static enum dma_status d40_tx_status(struct dma_chan *chan,
2242                                      dma_cookie_t cookie,
2243                                      struct dma_tx_state *txstate)
2244 {
2245         struct d40_chan *d40c = container_of(chan, struct d40_chan, chan);
2246         dma_cookie_t last_used;
2247         dma_cookie_t last_complete;
2248         int ret;
2249
2250         if (d40c->phy_chan == NULL) {
2251                 dev_err(&d40c->chan.dev->device,
2252                         "[%s] Cannot read status of unallocated channel\n",
2253                         __func__);
2254                 return -EINVAL;
2255         }
2256
2257         last_complete = d40c->completed;
2258         last_used = chan->cookie;
2259
2260         if (d40_is_paused(d40c))
2261                 ret = DMA_PAUSED;
2262         else
2263                 ret = dma_async_is_complete(cookie, last_complete, last_used);
2264
2265         dma_set_tx_state(txstate, last_complete, last_used,
2266                          stedma40_residue(chan));
2267
2268         return ret;
2269 }
2270
2271 static void d40_issue_pending(struct dma_chan *chan)
2272 {
2273         struct d40_chan *d40c = container_of(chan, struct d40_chan, chan);
2274         unsigned long flags;
2275
2276         if (d40c->phy_chan == NULL) {
2277                 dev_err(&d40c->chan.dev->device,
2278                         "[%s] Channel is not allocated!\n", __func__);
2279                 return;
2280         }
2281
2282         spin_lock_irqsave(&d40c->lock, flags);
2283
2284         /* Busy means that pending jobs are already being processed */
2285         if (!d40c->busy)
2286                 (void) d40_queue_start(d40c);
2287
2288         spin_unlock_irqrestore(&d40c->lock, flags);
2289 }
2290
2291 /* Runtime reconfiguration extension */
2292 static void d40_set_runtime_config(struct dma_chan *chan,
2293                                struct dma_slave_config *config)
2294 {
2295         struct d40_chan *d40c = container_of(chan, struct d40_chan, chan);
2296         struct stedma40_chan_cfg *cfg = &d40c->dma_cfg;
2297         enum dma_slave_buswidth config_addr_width;
2298         dma_addr_t config_addr;
2299         u32 config_maxburst;
2300         enum stedma40_periph_data_width addr_width;
2301         int psize;
2302
2303         if (config->direction == DMA_FROM_DEVICE) {
2304                 dma_addr_t dev_addr_rx =
2305                         d40c->base->plat_data->dev_rx[cfg->src_dev_type];
2306
2307                 config_addr = config->src_addr;
2308                 if (dev_addr_rx)
2309                         dev_dbg(d40c->base->dev,
2310                                 "channel has a pre-wired RX address %08x "
2311                                 "overriding with %08x\n",
2312                                 dev_addr_rx, config_addr);
2313                 if (cfg->dir != STEDMA40_PERIPH_TO_MEM)
2314                         dev_dbg(d40c->base->dev,
2315                                 "channel was not configured for peripheral "
2316                                 "to memory transfer (%d) overriding\n",
2317                                 cfg->dir);
2318                 cfg->dir = STEDMA40_PERIPH_TO_MEM;
2319
2320                 config_addr_width = config->src_addr_width;
2321                 config_maxburst = config->src_maxburst;
2322
2323         } else if (config->direction == DMA_TO_DEVICE) {
2324                 dma_addr_t dev_addr_tx =
2325                         d40c->base->plat_data->dev_tx[cfg->dst_dev_type];
2326
2327                 config_addr = config->dst_addr;
2328                 if (dev_addr_tx)
2329                         dev_dbg(d40c->base->dev,
2330                                 "channel has a pre-wired TX address %08x "
2331                                 "overriding with %08x\n",
2332                                 dev_addr_tx, config_addr);
2333                 if (cfg->dir != STEDMA40_MEM_TO_PERIPH)
2334                         dev_dbg(d40c->base->dev,
2335                                 "channel was not configured for memory "
2336                                 "to peripheral transfer (%d) overriding\n",
2337                                 cfg->dir);
2338                 cfg->dir = STEDMA40_MEM_TO_PERIPH;
2339
2340                 config_addr_width = config->dst_addr_width;
2341                 config_maxburst = config->dst_maxburst;
2342
2343         } else {
2344                 dev_err(d40c->base->dev,
2345                         "unrecognized channel direction %d\n",
2346                         config->direction);
2347                 return;
2348         }
2349
2350         switch (config_addr_width) {
2351         case DMA_SLAVE_BUSWIDTH_1_BYTE:
2352                 addr_width = STEDMA40_BYTE_WIDTH;
2353                 break;
2354         case DMA_SLAVE_BUSWIDTH_2_BYTES:
2355                 addr_width = STEDMA40_HALFWORD_WIDTH;
2356                 break;
2357         case DMA_SLAVE_BUSWIDTH_4_BYTES:
2358                 addr_width = STEDMA40_WORD_WIDTH;
2359                 break;
2360         case DMA_SLAVE_BUSWIDTH_8_BYTES:
2361                 addr_width = STEDMA40_DOUBLEWORD_WIDTH;
2362                 break;
2363         default:
2364                 dev_err(d40c->base->dev,
2365                         "illegal peripheral address width "
2366                         "requested (%d)\n",
2367                         config->src_addr_width);
2368                 return;
2369         }
2370
2371         if (d40c->log_num != D40_PHY_CHAN) {
2372                 if (config_maxburst >= 16)
2373                         psize = STEDMA40_PSIZE_LOG_16;
2374                 else if (config_maxburst >= 8)
2375                         psize = STEDMA40_PSIZE_LOG_8;
2376                 else if (config_maxburst >= 4)
2377                         psize = STEDMA40_PSIZE_LOG_4;
2378                 else
2379                         psize = STEDMA40_PSIZE_LOG_1;
2380         } else {
2381                 if (config_maxburst >= 16)
2382                         psize = STEDMA40_PSIZE_PHY_16;
2383                 else if (config_maxburst >= 8)
2384                         psize = STEDMA40_PSIZE_PHY_8;
2385                 else if (config_maxburst >= 4)
2386                         psize = STEDMA40_PSIZE_PHY_4;
2387                 else if (config_maxburst >= 2)
2388                         psize = STEDMA40_PSIZE_PHY_2;
2389                 else
2390                         psize = STEDMA40_PSIZE_PHY_1;
2391         }
2392
2393         /* Set up all the endpoint configs */
2394         cfg->src_info.data_width = addr_width;
2395         cfg->src_info.psize = psize;
2396         cfg->src_info.big_endian = false;
2397         cfg->src_info.flow_ctrl = STEDMA40_NO_FLOW_CTRL;
2398         cfg->dst_info.data_width = addr_width;
2399         cfg->dst_info.psize = psize;
2400         cfg->dst_info.big_endian = false;
2401         cfg->dst_info.flow_ctrl = STEDMA40_NO_FLOW_CTRL;
2402
2403         /* Fill in register values */
2404         if (d40c->log_num != D40_PHY_CHAN)
2405                 d40_log_cfg(cfg, &d40c->log_def.lcsp1, &d40c->log_def.lcsp3);
2406         else
2407                 d40_phy_cfg(cfg, &d40c->src_def_cfg,
2408                             &d40c->dst_def_cfg, false);
2409
2410         /* These settings will take precedence later */
2411         d40c->runtime_addr = config_addr;
2412         d40c->runtime_direction = config->direction;
2413         dev_dbg(d40c->base->dev,
2414                 "configured channel %s for %s, data width %d, "
2415                 "maxburst %d bytes, LE, no flow control\n",
2416                 dma_chan_name(chan),
2417                 (config->direction == DMA_FROM_DEVICE) ? "RX" : "TX",
2418                 config_addr_width,
2419                 config_maxburst);
2420 }
2421
2422 static int d40_control(struct dma_chan *chan, enum dma_ctrl_cmd cmd,
2423                        unsigned long arg)
2424 {
2425         unsigned long flags;
2426         struct d40_chan *d40c = container_of(chan, struct d40_chan, chan);
2427
2428         if (d40c->phy_chan == NULL) {
2429                 dev_err(&d40c->chan.dev->device,
2430                         "[%s] Channel is not allocated!\n", __func__);
2431                 return -EINVAL;
2432         }
2433
2434         switch (cmd) {
2435         case DMA_TERMINATE_ALL:
2436                 spin_lock_irqsave(&d40c->lock, flags);
2437                 d40_term_all(d40c);
2438                 spin_unlock_irqrestore(&d40c->lock, flags);
2439                 return 0;
2440         case DMA_PAUSE:
2441                 return d40_pause(chan);
2442         case DMA_RESUME:
2443                 return d40_resume(chan);
2444         case DMA_SLAVE_CONFIG:
2445                 d40_set_runtime_config(chan,
2446                         (struct dma_slave_config *) arg);
2447                 return 0;
2448         default:
2449                 break;
2450         }
2451
2452         /* Other commands are unimplemented */
2453         return -ENXIO;
2454 }
2455
2456 /* Initialization functions */
2457
2458 static void __init d40_chan_init(struct d40_base *base, struct dma_device *dma,
2459                                  struct d40_chan *chans, int offset,
2460                                  int num_chans)
2461 {
2462         int i = 0;
2463         struct d40_chan *d40c;
2464
2465         INIT_LIST_HEAD(&dma->channels);
2466
2467         for (i = offset; i < offset + num_chans; i++) {
2468                 d40c = &chans[i];
2469                 d40c->base = base;
2470                 d40c->chan.device = dma;
2471
2472                 spin_lock_init(&d40c->lock);
2473
2474                 d40c->log_num = D40_PHY_CHAN;
2475
2476                 INIT_LIST_HEAD(&d40c->active);
2477                 INIT_LIST_HEAD(&d40c->queue);
2478                 INIT_LIST_HEAD(&d40c->client);
2479
2480                 tasklet_init(&d40c->tasklet, dma_tasklet,
2481                              (unsigned long) d40c);
2482
2483                 list_add_tail(&d40c->chan.device_node,
2484                               &dma->channels);
2485         }
2486 }
2487
2488 static int __init d40_dmaengine_init(struct d40_base *base,
2489                                      int num_reserved_chans)
2490 {
2491         int err ;
2492
2493         d40_chan_init(base, &base->dma_slave, base->log_chans,
2494                       0, base->num_log_chans);
2495
2496         dma_cap_zero(base->dma_slave.cap_mask);
2497         dma_cap_set(DMA_SLAVE, base->dma_slave.cap_mask);
2498
2499         base->dma_slave.device_alloc_chan_resources = d40_alloc_chan_resources;
2500         base->dma_slave.device_free_chan_resources = d40_free_chan_resources;
2501         base->dma_slave.device_prep_dma_memcpy = d40_prep_memcpy;
2502         base->dma_slave.device_prep_dma_sg = d40_prep_sg;
2503         base->dma_slave.device_prep_slave_sg = d40_prep_slave_sg;
2504         base->dma_slave.device_tx_status = d40_tx_status;
2505         base->dma_slave.device_issue_pending = d40_issue_pending;
2506         base->dma_slave.device_control = d40_control;
2507         base->dma_slave.dev = base->dev;
2508
2509         err = dma_async_device_register(&base->dma_slave);
2510
2511         if (err) {
2512                 dev_err(base->dev,
2513                         "[%s] Failed to register slave channels\n",
2514                         __func__);
2515                 goto failure1;
2516         }
2517
2518         d40_chan_init(base, &base->dma_memcpy, base->log_chans,
2519                       base->num_log_chans, base->plat_data->memcpy_len);
2520
2521         dma_cap_zero(base->dma_memcpy.cap_mask);
2522         dma_cap_set(DMA_MEMCPY, base->dma_memcpy.cap_mask);
2523         dma_cap_set(DMA_SG, base->dma_slave.cap_mask);
2524
2525         base->dma_memcpy.device_alloc_chan_resources = d40_alloc_chan_resources;
2526         base->dma_memcpy.device_free_chan_resources = d40_free_chan_resources;
2527         base->dma_memcpy.device_prep_dma_memcpy = d40_prep_memcpy;
2528         base->dma_slave.device_prep_dma_sg = d40_prep_sg;
2529         base->dma_memcpy.device_prep_slave_sg = d40_prep_slave_sg;
2530         base->dma_memcpy.device_tx_status = d40_tx_status;
2531         base->dma_memcpy.device_issue_pending = d40_issue_pending;
2532         base->dma_memcpy.device_control = d40_control;
2533         base->dma_memcpy.dev = base->dev;
2534         /*
2535          * This controller can only access address at even
2536          * 32bit boundaries, i.e. 2^2
2537          */
2538         base->dma_memcpy.copy_align = 2;
2539
2540         err = dma_async_device_register(&base->dma_memcpy);
2541
2542         if (err) {
2543                 dev_err(base->dev,
2544                         "[%s] Failed to regsiter memcpy only channels\n",
2545                         __func__);
2546                 goto failure2;
2547         }
2548
2549         d40_chan_init(base, &base->dma_both, base->phy_chans,
2550                       0, num_reserved_chans);
2551
2552         dma_cap_zero(base->dma_both.cap_mask);
2553         dma_cap_set(DMA_SLAVE, base->dma_both.cap_mask);
2554         dma_cap_set(DMA_MEMCPY, base->dma_both.cap_mask);
2555         dma_cap_set(DMA_SG, base->dma_slave.cap_mask);
2556
2557         base->dma_both.device_alloc_chan_resources = d40_alloc_chan_resources;
2558         base->dma_both.device_free_chan_resources = d40_free_chan_resources;
2559         base->dma_both.device_prep_dma_memcpy = d40_prep_memcpy;
2560         base->dma_slave.device_prep_dma_sg = d40_prep_sg;
2561         base->dma_both.device_prep_slave_sg = d40_prep_slave_sg;
2562         base->dma_both.device_tx_status = d40_tx_status;
2563         base->dma_both.device_issue_pending = d40_issue_pending;
2564         base->dma_both.device_control = d40_control;
2565         base->dma_both.dev = base->dev;
2566         base->dma_both.copy_align = 2;
2567         err = dma_async_device_register(&base->dma_both);
2568
2569         if (err) {
2570                 dev_err(base->dev,
2571                         "[%s] Failed to register logical and physical capable channels\n",
2572                         __func__);
2573                 goto failure3;
2574         }
2575         return 0;
2576 failure3:
2577         dma_async_device_unregister(&base->dma_memcpy);
2578 failure2:
2579         dma_async_device_unregister(&base->dma_slave);
2580 failure1:
2581         return err;
2582 }
2583
2584 /* Initialization functions. */
2585
2586 static int __init d40_phy_res_init(struct d40_base *base)
2587 {
2588         int i;
2589         int num_phy_chans_avail = 0;
2590         u32 val[2];
2591         int odd_even_bit = -2;
2592
2593         val[0] = readl(base->virtbase + D40_DREG_PRSME);
2594         val[1] = readl(base->virtbase + D40_DREG_PRSMO);
2595
2596         for (i = 0; i < base->num_phy_chans; i++) {
2597                 base->phy_res[i].num = i;
2598                 odd_even_bit += 2 * ((i % 2) == 0);
2599                 if (((val[i % 2] >> odd_even_bit) & 3) == 1) {
2600                         /* Mark security only channels as occupied */
2601                         base->phy_res[i].allocated_src = D40_ALLOC_PHY;
2602                         base->phy_res[i].allocated_dst = D40_ALLOC_PHY;
2603                 } else {
2604                         base->phy_res[i].allocated_src = D40_ALLOC_FREE;
2605                         base->phy_res[i].allocated_dst = D40_ALLOC_FREE;
2606                         num_phy_chans_avail++;
2607                 }
2608                 spin_lock_init(&base->phy_res[i].lock);
2609         }
2610
2611         /* Mark disabled channels as occupied */
2612         for (i = 0; base->plat_data->disabled_channels[i] != -1; i++) {
2613                 int chan = base->plat_data->disabled_channels[i];
2614
2615                 base->phy_res[chan].allocated_src = D40_ALLOC_PHY;
2616                 base->phy_res[chan].allocated_dst = D40_ALLOC_PHY;
2617                 num_phy_chans_avail--;
2618         }
2619
2620         dev_info(base->dev, "%d of %d physical DMA channels available\n",
2621                  num_phy_chans_avail, base->num_phy_chans);
2622
2623         /* Verify settings extended vs standard */
2624         val[0] = readl(base->virtbase + D40_DREG_PRTYP);
2625
2626         for (i = 0; i < base->num_phy_chans; i++) {
2627
2628                 if (base->phy_res[i].allocated_src == D40_ALLOC_FREE &&
2629                     (val[0] & 0x3) != 1)
2630                         dev_info(base->dev,
2631                                  "[%s] INFO: channel %d is misconfigured (%d)\n",
2632                                  __func__, i, val[0] & 0x3);
2633
2634                 val[0] = val[0] >> 2;
2635         }
2636
2637         return num_phy_chans_avail;
2638 }
2639
2640 static struct d40_base * __init d40_hw_detect_init(struct platform_device *pdev)
2641 {
2642         static const struct d40_reg_val dma_id_regs[] = {
2643                 /* Peripheral Id */
2644                 { .reg = D40_DREG_PERIPHID0, .val = 0x0040},
2645                 { .reg = D40_DREG_PERIPHID1, .val = 0x0000},
2646                 /*
2647                  * D40_DREG_PERIPHID2 Depends on HW revision:
2648                  *  MOP500/HREF ED has 0x0008,
2649                  *  ? has 0x0018,
2650                  *  HREF V1 has 0x0028
2651                  */
2652                 { .reg = D40_DREG_PERIPHID3, .val = 0x0000},
2653
2654                 /* PCell Id */
2655                 { .reg = D40_DREG_CELLID0, .val = 0x000d},
2656                 { .reg = D40_DREG_CELLID1, .val = 0x00f0},
2657                 { .reg = D40_DREG_CELLID2, .val = 0x0005},
2658                 { .reg = D40_DREG_CELLID3, .val = 0x00b1}
2659         };
2660         struct stedma40_platform_data *plat_data;
2661         struct clk *clk = NULL;
2662         void __iomem *virtbase = NULL;
2663         struct resource *res = NULL;
2664         struct d40_base *base = NULL;
2665         int num_log_chans = 0;
2666         int num_phy_chans;
2667         int i;
2668         u32 val;
2669         u32 rev;
2670
2671         clk = clk_get(&pdev->dev, NULL);
2672
2673         if (IS_ERR(clk)) {
2674                 dev_err(&pdev->dev, "[%s] No matching clock found\n",
2675                         __func__);
2676                 goto failure;
2677         }
2678
2679         clk_enable(clk);
2680
2681         /* Get IO for DMAC base address */
2682         res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "base");
2683         if (!res)
2684                 goto failure;
2685
2686         if (request_mem_region(res->start, resource_size(res),
2687                                D40_NAME " I/O base") == NULL)
2688                 goto failure;
2689
2690         virtbase = ioremap(res->start, resource_size(res));
2691         if (!virtbase)
2692                 goto failure;
2693
2694         /* HW version check */
2695         for (i = 0; i < ARRAY_SIZE(dma_id_regs); i++) {
2696                 if (dma_id_regs[i].val !=
2697                     readl(virtbase + dma_id_regs[i].reg)) {
2698                         dev_err(&pdev->dev,
2699                                 "[%s] Unknown hardware! Expected 0x%x at 0x%x but got 0x%x\n",
2700                                 __func__,
2701                                 dma_id_regs[i].val,
2702                                 dma_id_regs[i].reg,
2703                                 readl(virtbase + dma_id_regs[i].reg));
2704                         goto failure;
2705                 }
2706         }
2707
2708         /* Get silicon revision and designer */
2709         val = readl(virtbase + D40_DREG_PERIPHID2);
2710
2711         if ((val & D40_DREG_PERIPHID2_DESIGNER_MASK) !=
2712             D40_HW_DESIGNER) {
2713                 dev_err(&pdev->dev,
2714                         "[%s] Unknown designer! Got %x wanted %x\n",
2715                         __func__, val & D40_DREG_PERIPHID2_DESIGNER_MASK,
2716                         D40_HW_DESIGNER);
2717                 goto failure;
2718         }
2719
2720         rev = (val & D40_DREG_PERIPHID2_REV_MASK) >>
2721                 D40_DREG_PERIPHID2_REV_POS;
2722
2723         /* The number of physical channels on this HW */
2724         num_phy_chans = 4 * (readl(virtbase + D40_DREG_ICFG) & 0x7) + 4;
2725
2726         dev_info(&pdev->dev, "hardware revision: %d @ 0x%x\n",
2727                  rev, res->start);
2728
2729         plat_data = pdev->dev.platform_data;
2730
2731         /* Count the number of logical channels in use */
2732         for (i = 0; i < plat_data->dev_len; i++)
2733                 if (plat_data->dev_rx[i] != 0)
2734                         num_log_chans++;
2735
2736         for (i = 0; i < plat_data->dev_len; i++)
2737                 if (plat_data->dev_tx[i] != 0)
2738                         num_log_chans++;
2739
2740         base = kzalloc(ALIGN(sizeof(struct d40_base), 4) +
2741                        (num_phy_chans + num_log_chans + plat_data->memcpy_len) *
2742                        sizeof(struct d40_chan), GFP_KERNEL);
2743
2744         if (base == NULL) {
2745                 dev_err(&pdev->dev, "[%s] Out of memory\n", __func__);
2746                 goto failure;
2747         }
2748
2749         base->rev = rev;
2750         base->clk = clk;
2751         base->num_phy_chans = num_phy_chans;
2752         base->num_log_chans = num_log_chans;
2753         base->phy_start = res->start;
2754         base->phy_size = resource_size(res);
2755         base->virtbase = virtbase;
2756         base->plat_data = plat_data;
2757         base->dev = &pdev->dev;
2758         base->phy_chans = ((void *)base) + ALIGN(sizeof(struct d40_base), 4);
2759         base->log_chans = &base->phy_chans[num_phy_chans];
2760
2761         base->phy_res = kzalloc(num_phy_chans * sizeof(struct d40_phy_res),
2762                                 GFP_KERNEL);
2763         if (!base->phy_res)
2764                 goto failure;
2765
2766         base->lookup_phy_chans = kzalloc(num_phy_chans *
2767                                          sizeof(struct d40_chan *),
2768                                          GFP_KERNEL);
2769         if (!base->lookup_phy_chans)
2770                 goto failure;
2771
2772         if (num_log_chans + plat_data->memcpy_len) {
2773                 /*
2774                  * The max number of logical channels are event lines for all
2775                  * src devices and dst devices
2776                  */
2777                 base->lookup_log_chans = kzalloc(plat_data->dev_len * 2 *
2778                                                  sizeof(struct d40_chan *),
2779                                                  GFP_KERNEL);
2780                 if (!base->lookup_log_chans)
2781                         goto failure;
2782         }
2783
2784         base->lcla_pool.alloc_map = kzalloc(num_phy_chans *
2785                                             sizeof(struct d40_desc *) *
2786                                             D40_LCLA_LINK_PER_EVENT_GRP,
2787                                             GFP_KERNEL);
2788         if (!base->lcla_pool.alloc_map)
2789                 goto failure;
2790
2791         base->desc_slab = kmem_cache_create(D40_NAME, sizeof(struct d40_desc),
2792                                             0, SLAB_HWCACHE_ALIGN,
2793                                             NULL);
2794         if (base->desc_slab == NULL)
2795                 goto failure;
2796
2797         return base;
2798
2799 failure:
2800         if (!IS_ERR(clk)) {
2801                 clk_disable(clk);
2802                 clk_put(clk);
2803         }
2804         if (virtbase)
2805                 iounmap(virtbase);
2806         if (res)
2807                 release_mem_region(res->start,
2808                                    resource_size(res));
2809         if (virtbase)
2810                 iounmap(virtbase);
2811
2812         if (base) {
2813                 kfree(base->lcla_pool.alloc_map);
2814                 kfree(base->lookup_log_chans);
2815                 kfree(base->lookup_phy_chans);
2816                 kfree(base->phy_res);
2817                 kfree(base);
2818         }
2819
2820         return NULL;
2821 }
2822
2823 static void __init d40_hw_init(struct d40_base *base)
2824 {
2825
2826         static const struct d40_reg_val dma_init_reg[] = {
2827                 /* Clock every part of the DMA block from start */
2828                 { .reg = D40_DREG_GCC,    .val = 0x0000ff01},
2829
2830                 /* Interrupts on all logical channels */
2831                 { .reg = D40_DREG_LCMIS0, .val = 0xFFFFFFFF},
2832                 { .reg = D40_DREG_LCMIS1, .val = 0xFFFFFFFF},
2833                 { .reg = D40_DREG_LCMIS2, .val = 0xFFFFFFFF},
2834                 { .reg = D40_DREG_LCMIS3, .val = 0xFFFFFFFF},
2835                 { .reg = D40_DREG_LCICR0, .val = 0xFFFFFFFF},
2836                 { .reg = D40_DREG_LCICR1, .val = 0xFFFFFFFF},
2837                 { .reg = D40_DREG_LCICR2, .val = 0xFFFFFFFF},
2838                 { .reg = D40_DREG_LCICR3, .val = 0xFFFFFFFF},
2839                 { .reg = D40_DREG_LCTIS0, .val = 0xFFFFFFFF},
2840                 { .reg = D40_DREG_LCTIS1, .val = 0xFFFFFFFF},
2841                 { .reg = D40_DREG_LCTIS2, .val = 0xFFFFFFFF},
2842                 { .reg = D40_DREG_LCTIS3, .val = 0xFFFFFFFF}
2843         };
2844         int i;
2845         u32 prmseo[2] = {0, 0};
2846         u32 activeo[2] = {0xFFFFFFFF, 0xFFFFFFFF};
2847         u32 pcmis = 0;
2848         u32 pcicr = 0;
2849
2850         for (i = 0; i < ARRAY_SIZE(dma_init_reg); i++)
2851                 writel(dma_init_reg[i].val,
2852                        base->virtbase + dma_init_reg[i].reg);
2853
2854         /* Configure all our dma channels to default settings */
2855         for (i = 0; i < base->num_phy_chans; i++) {
2856
2857                 activeo[i % 2] = activeo[i % 2] << 2;
2858
2859                 if (base->phy_res[base->num_phy_chans - i - 1].allocated_src
2860                     == D40_ALLOC_PHY) {
2861                         activeo[i % 2] |= 3;
2862                         continue;
2863                 }
2864
2865                 /* Enable interrupt # */
2866                 pcmis = (pcmis << 1) | 1;
2867
2868                 /* Clear interrupt # */
2869                 pcicr = (pcicr << 1) | 1;
2870
2871                 /* Set channel to physical mode */
2872                 prmseo[i % 2] = prmseo[i % 2] << 2;
2873                 prmseo[i % 2] |= 1;
2874
2875         }
2876
2877         writel(prmseo[1], base->virtbase + D40_DREG_PRMSE);
2878         writel(prmseo[0], base->virtbase + D40_DREG_PRMSO);
2879         writel(activeo[1], base->virtbase + D40_DREG_ACTIVE);
2880         writel(activeo[0], base->virtbase + D40_DREG_ACTIVO);
2881
2882         /* Write which interrupt to enable */
2883         writel(pcmis, base->virtbase + D40_DREG_PCMIS);
2884
2885         /* Write which interrupt to clear */
2886         writel(pcicr, base->virtbase + D40_DREG_PCICR);
2887
2888 }
2889
2890 static int __init d40_lcla_allocate(struct d40_base *base)
2891 {
2892         unsigned long *page_list;
2893         int i, j;
2894         int ret = 0;
2895
2896         /*
2897          * This is somewhat ugly. We need 8192 bytes that are 18 bit aligned,
2898          * To full fill this hardware requirement without wasting 256 kb
2899          * we allocate pages until we get an aligned one.
2900          */
2901         page_list = kmalloc(sizeof(unsigned long) * MAX_LCLA_ALLOC_ATTEMPTS,
2902                             GFP_KERNEL);
2903
2904         if (!page_list) {
2905                 ret = -ENOMEM;
2906                 goto failure;
2907         }
2908
2909         /* Calculating how many pages that are required */
2910         base->lcla_pool.pages = SZ_1K * base->num_phy_chans / PAGE_SIZE;
2911
2912         for (i = 0; i < MAX_LCLA_ALLOC_ATTEMPTS; i++) {
2913                 page_list[i] = __get_free_pages(GFP_KERNEL,
2914                                                 base->lcla_pool.pages);
2915                 if (!page_list[i]) {
2916
2917                         dev_err(base->dev,
2918                                 "[%s] Failed to allocate %d pages.\n",
2919                                 __func__, base->lcla_pool.pages);
2920
2921                         for (j = 0; j < i; j++)
2922                                 free_pages(page_list[j], base->lcla_pool.pages);
2923                         goto failure;
2924                 }
2925
2926                 if ((virt_to_phys((void *)page_list[i]) &
2927                      (LCLA_ALIGNMENT - 1)) == 0)
2928                         break;
2929         }
2930
2931         for (j = 0; j < i; j++)
2932                 free_pages(page_list[j], base->lcla_pool.pages);
2933
2934         if (i < MAX_LCLA_ALLOC_ATTEMPTS) {
2935                 base->lcla_pool.base = (void *)page_list[i];
2936         } else {
2937                 /*
2938                  * After many attempts and no succees with finding the correct
2939                  * alignment, try with allocating a big buffer.
2940                  */
2941                 dev_warn(base->dev,
2942                          "[%s] Failed to get %d pages @ 18 bit align.\n",
2943                          __func__, base->lcla_pool.pages);
2944                 base->lcla_pool.base_unaligned = kmalloc(SZ_1K *
2945                                                          base->num_phy_chans +
2946                                                          LCLA_ALIGNMENT,
2947                                                          GFP_KERNEL);
2948                 if (!base->lcla_pool.base_unaligned) {
2949                         ret = -ENOMEM;
2950                         goto failure;
2951                 }
2952
2953                 base->lcla_pool.base = PTR_ALIGN(base->lcla_pool.base_unaligned,
2954                                                  LCLA_ALIGNMENT);
2955         }
2956
2957         writel(virt_to_phys(base->lcla_pool.base),
2958                base->virtbase + D40_DREG_LCLA);
2959 failure:
2960         kfree(page_list);
2961         return ret;
2962 }
2963
2964 static int __init d40_probe(struct platform_device *pdev)
2965 {
2966         int err;
2967         int ret = -ENOENT;
2968         struct d40_base *base;
2969         struct resource *res = NULL;
2970         int num_reserved_chans;
2971         u32 val;
2972
2973         base = d40_hw_detect_init(pdev);
2974
2975         if (!base)
2976                 goto failure;
2977
2978         num_reserved_chans = d40_phy_res_init(base);
2979
2980         platform_set_drvdata(pdev, base);
2981
2982         spin_lock_init(&base->interrupt_lock);
2983         spin_lock_init(&base->execmd_lock);
2984
2985         /* Get IO for logical channel parameter address */
2986         res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "lcpa");
2987         if (!res) {
2988                 ret = -ENOENT;
2989                 dev_err(&pdev->dev,
2990                         "[%s] No \"lcpa\" memory resource\n",
2991                         __func__);
2992                 goto failure;
2993         }
2994         base->lcpa_size = resource_size(res);
2995         base->phy_lcpa = res->start;
2996
2997         if (request_mem_region(res->start, resource_size(res),
2998                                D40_NAME " I/O lcpa") == NULL) {
2999                 ret = -EBUSY;
3000                 dev_err(&pdev->dev,
3001                         "[%s] Failed to request LCPA region 0x%x-0x%x\n",
3002                         __func__, res->start, res->end);
3003                 goto failure;
3004         }
3005
3006         /* We make use of ESRAM memory for this. */
3007         val = readl(base->virtbase + D40_DREG_LCPA);
3008         if (res->start != val && val != 0) {
3009                 dev_warn(&pdev->dev,
3010                          "[%s] Mismatch LCPA dma 0x%x, def 0x%x\n",
3011                          __func__, val, res->start);
3012         } else
3013                 writel(res->start, base->virtbase + D40_DREG_LCPA);
3014
3015         base->lcpa_base = ioremap(res->start, resource_size(res));
3016         if (!base->lcpa_base) {
3017                 ret = -ENOMEM;
3018                 dev_err(&pdev->dev,
3019                         "[%s] Failed to ioremap LCPA region\n",
3020                         __func__);
3021                 goto failure;
3022         }
3023
3024         ret = d40_lcla_allocate(base);
3025         if (ret) {
3026                 dev_err(&pdev->dev, "[%s] Failed to allocate LCLA area\n",
3027                         __func__);
3028                 goto failure;
3029         }
3030
3031         spin_lock_init(&base->lcla_pool.lock);
3032
3033         base->irq = platform_get_irq(pdev, 0);
3034
3035         ret = request_irq(base->irq, d40_handle_interrupt, 0, D40_NAME, base);
3036
3037         if (ret) {
3038                 dev_err(&pdev->dev, "[%s] No IRQ defined\n", __func__);
3039                 goto failure;
3040         }
3041
3042         err = d40_dmaengine_init(base, num_reserved_chans);
3043         if (err)
3044                 goto failure;
3045
3046         d40_hw_init(base);
3047
3048         dev_info(base->dev, "initialized\n");
3049         return 0;
3050
3051 failure:
3052         if (base) {
3053                 if (base->desc_slab)
3054                         kmem_cache_destroy(base->desc_slab);
3055                 if (base->virtbase)
3056                         iounmap(base->virtbase);
3057                 if (!base->lcla_pool.base_unaligned && base->lcla_pool.base)
3058                         free_pages((unsigned long)base->lcla_pool.base,
3059                                    base->lcla_pool.pages);
3060
3061                 kfree(base->lcla_pool.base_unaligned);
3062
3063                 if (base->phy_lcpa)
3064                         release_mem_region(base->phy_lcpa,
3065                                            base->lcpa_size);
3066                 if (base->phy_start)
3067                         release_mem_region(base->phy_start,
3068                                            base->phy_size);
3069                 if (base->clk) {
3070                         clk_disable(base->clk);
3071                         clk_put(base->clk);
3072                 }
3073
3074                 kfree(base->lcla_pool.alloc_map);
3075                 kfree(base->lookup_log_chans);
3076                 kfree(base->lookup_phy_chans);
3077                 kfree(base->phy_res);
3078                 kfree(base);
3079         }
3080
3081         dev_err(&pdev->dev, "[%s] probe failed\n", __func__);
3082         return ret;
3083 }
3084
3085 static struct platform_driver d40_driver = {
3086         .driver = {
3087                 .owner = THIS_MODULE,
3088                 .name  = D40_NAME,
3089         },
3090 };
3091
3092 static int __init stedma40_init(void)
3093 {
3094         return platform_driver_probe(&d40_driver, d40_probe);
3095 }
3096 arch_initcall(stedma40_init);