drivers/usb/dwc3/gadget.c

   1 /**
   2  * gadget.c - DesignWare USB3 DRD Controller Gadget Framework Link
   3  *
   4  * Copyright (C) 2010-2011 Texas Instruments Incorporated - http://www.ti.com
   5  *
   6  * Authors: Felipe Balbi <balbi@ti.com>,
   7  *          Sebastian Andrzej Siewior <bigeasy@linutronix.de>
   8  *
   9  * Redistribution and use in source and binary forms, with or without
  10  * modification, are permitted provided that the following conditions
  11  * are met:
  12  * 1. Redistributions of source code must retain the above copyright
  13  *    notice, this list of conditions, and the following disclaimer,
  14  *    without modification.
  15  * 2. Redistributions in binary form must reproduce the above copyright
  16  *    notice, this list of conditions and the following disclaimer in the
  17  *    documentation and/or other materials provided with the distribution.
  18  * 3. The names of the above-listed copyright holders may not be used
  19  *    to endorse or promote products derived from this software without
  20  *    specific prior written permission.
  21  *
  22  * ALTERNATIVELY, this software may be distributed under the terms of the
  23  * GNU General Public License ("GPL") version 2, as published by the Free
  24  * Software Foundation.
  25  *
  26  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
  27  * IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
  28  * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  29  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
  30  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
  31  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
  32  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
  33  * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
  34  * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
  35  * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
  36  * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  37  */
  38
  39 #include <linux/kernel.h>
  40 #include <linux/delay.h>
  41 #include <linux/slab.h>
  42 #include <linux/spinlock.h>
  43 #include <linux/platform_device.h>
  44 #include <linux/pm_runtime.h>
  45 #include <linux/interrupt.h>
  46 #include <linux/io.h>
  47 #include <linux/list.h>
  48 #include <linux/dma-mapping.h>
  49
  50 #include <linux/usb/ch9.h>
  51 #include <linux/usb/gadget.h>
  52
  53 #include "core.h"
  54 #include "gadget.h"
  55 #include "io.h"
  56
  57 /**
  58  * dwc3_gadget_set_test_mode - Enables USB2 Test Modes
  59  * @dwc: pointer to our context structure
  60  * @mode: the mode to set (J, K SE0 NAK, Force Enable)
  61  *
  62  * Caller should take care of locking. This function will
  63  * return 0 on success or -EINVAL if wrong Test Selector
  64  * is passed
  65  */
  66 int dwc3_gadget_set_test_mode(struct dwc3 *dwc, int mode)
  67 {
  68         u32             reg;
  69
  70         reg = dwc3_readl(dwc->regs, DWC3_DCTL);
  71         reg &= ~DWC3_DCTL_TSTCTRL_MASK;
  72
  73         switch (mode) {
  74         case TEST_J:
  75         case TEST_K:
  76         case TEST_SE0_NAK:
  77         case TEST_PACKET:
  78         case TEST_FORCE_EN:
  79                 reg |= mode << 1;
  80                 break;
  81         default:
  82                 return -EINVAL;
  83         }
  84
  85         dwc3_writel(dwc->regs, DWC3_DCTL, reg);
  86
  87         return 0;
  88 }
  89
  90 /**
  91  * dwc3_gadget_set_link_state - Sets USB Link to a particular State
  92  * @dwc: pointer to our context structure
  93  * @state: the state to put link into
  94  *
  95  * Caller should take care of locking. This function will
  96  * return 0 on success or -ETIMEDOUT.
  97  */
  98 int dwc3_gadget_set_link_state(struct dwc3 *dwc, enum dwc3_link_state state)
  99 {
 100         int             retries = 10000;
 101         u32             reg;
 102
 103         /*
 104          * Wait until device controller is ready. Only applies to 1.94a and
 105          * later RTL.
 106          */
 107         if (dwc->revision >= DWC3_REVISION_194A) {
 108                 while (--retries) {
 109                         reg = dwc3_readl(dwc->regs, DWC3_DSTS);
 110                         if (reg & DWC3_DSTS_DCNRD)
 111                                 udelay(5);
 112                         else
 113                                 break;
 114                 }
 115
 116                 if (retries <= 0)
 117                         return -ETIMEDOUT;
 118         }
 119
 120         reg = dwc3_readl(dwc->regs, DWC3_DCTL);
 121         reg &= ~DWC3_DCTL_ULSTCHNGREQ_MASK;
 122
 123         /* set requested state */
 124         reg |= DWC3_DCTL_ULSTCHNGREQ(state);
 125         dwc3_writel(dwc->regs, DWC3_DCTL, reg);
 126
 127         /*
 128          * The following code is racy when called from dwc3_gadget_wakeup,
 129          * and is not needed, at least on newer versions
 130          */
 131         if (dwc->revision >= DWC3_REVISION_194A)
 132                 return 0;
 133
 134         /* wait for a change in DSTS */
 135         retries = 10000;
 136         while (--retries) {
 137                 reg = dwc3_readl(dwc->regs, DWC3_DSTS);
 138
 139                 if (DWC3_DSTS_USBLNKST(reg) == state)
 140                         return 0;
 141
 142                 udelay(5);
 143         }
 144
 145         dev_vdbg(dwc->dev, "link state change request timed out\n");
 146
 147         return -ETIMEDOUT;
 148 }
 149
 150 /**
 151  * dwc3_gadget_resize_tx_fifos - reallocate fifo spaces for current use-case
 152  * @dwc: pointer to our context structure
 153  *
 154  * This function will a best effort FIFO allocation in order
 155  * to improve FIFO usage and throughput, while still allowing
 156  * us to enable as many endpoints as possible.
 157  *
 158  * Keep in mind that this operation will be highly dependent
 159  * on the configured size for RAM1 - which contains TxFifo -,
 160  * the amount of endpoints enabled on coreConsultant tool, and
 161  * the width of the Master Bus.
 162  *
 163  * In the ideal world, we would always be able to satisfy the
 164  * following equation:
 165  *
 166  * ((512 + 2 * MDWIDTH-Bytes) + (Number of IN Endpoints - 1) * \
 167  * (3 * (1024 + MDWIDTH-Bytes) + MDWIDTH-Bytes)) / MDWIDTH-Bytes
 168  *
 169  * Unfortunately, due to many variables that's not always the case.
 170  */
 171 int dwc3_gadget_resize_tx_fifos(struct dwc3 *dwc)
 172 {
 173         int             last_fifo_depth = 0;
 174         int             ram1_depth;
 175         int             fifo_size;
 176         int             mdwidth;
 177         int             num;
 178
 179         if (!dwc->needs_fifo_resize)
 180                 return 0;
 181
 182         ram1_depth = DWC3_RAM1_DEPTH(dwc->hwparams.hwparams7);
 183         mdwidth = DWC3_MDWIDTH(dwc->hwparams.hwparams0);
 184
 185         /* MDWIDTH is represented in bits, we need it in bytes */
 186         mdwidth >>= 3;
 187
 188         /*
 189          * FIXME For now we will only allocate 1 wMaxPacketSize space
 190          * for each enabled endpoint, later patches will come to
 191          * improve this algorithm so that we better use the internal
 192          * FIFO space
 193          */
 194         for (num = 0; num < DWC3_ENDPOINTS_NUM; num++) {
 195                 struct dwc3_ep  *dep = dwc->eps[num];
 196                 int             fifo_number = dep->number >> 1;
 197                 int             mult = 1;
 198                 int             tmp;
 199
 200                 if (!(dep->number & 1))
 201                         continue;
 202
 203                 if (!(dep->flags & DWC3_EP_ENABLED))
 204                         continue;
 205
 206                 if (usb_endpoint_xfer_bulk(dep->endpoint.desc)
 207                                 || usb_endpoint_xfer_isoc(dep->endpoint.desc))
 208                         mult = 3;
 209
 210                 /*
 211                  * REVISIT: the following assumes we will always have enough
 212                  * space available on the FIFO RAM for all possible use cases.
 213                  * Make sure that's true somehow and change FIFO allocation
 214                  * accordingly.
 215                  *
 216                  * If we have Bulk or Isochronous endpoints, we want
 217                  * them to be able to be very, very fast. So we're giving
 218                  * those endpoints a fifo_size which is enough for 3 full
 219                  * packets
 220                  */
 221                 tmp = mult * (dep->endpoint.maxpacket + mdwidth);
 222                 tmp += mdwidth;
 223
 224                 fifo_size = DIV_ROUND_UP(tmp, mdwidth);
 225
 226                 fifo_size |= (last_fifo_depth << 16);
 227
 228                 dev_vdbg(dwc->dev, "%s: Fifo Addr %04x Size %d\n",
 229                                 dep->name, last_fifo_depth, fifo_size & 0xffff);
 230
 231                 dwc3_writel(dwc->regs, DWC3_GTXFIFOSIZ(fifo_number),
 232                                 fifo_size);
 233
 234                 last_fifo_depth += (fifo_size & 0xffff);
 235         }
 236
 237         return 0;
 238 }
 239
 240 void dwc3_gadget_giveback(struct dwc3_ep *dep, struct dwc3_request *req,
 241                 int status)
 242 {
 243         struct dwc3                     *dwc = dep->dwc;
 244
 245         if (req->queued) {
 246                 if (req->request.num_mapped_sgs)
 247                         dep->busy_slot += req->request.num_mapped_sgs;
 248                 else
 249                         dep->busy_slot++;
 250
 251                 /*
 252                  * Skip LINK TRB. We can't use req->trb and check for
 253                  * DWC3_TRBCTL_LINK_TRB because it points the TRB we just
 254                  * completed (not the LINK TRB).
 255                  */
 256                 if (((dep->busy_slot & DWC3_TRB_MASK) == DWC3_TRB_NUM - 1) &&
 257                                 usb_endpoint_xfer_isoc(dep->endpoint.desc))
 258                         dep->busy_slot++;
 259         }
 260         list_del(&req->list);
 261         req->trb = NULL;
 262
 263         if (req->request.status == -EINPROGRESS)
 264                 req->request.status = status;
 265
 266         usb_gadget_unmap_request(&dwc->gadget, &req->request,
 267                         req->direction);
 268
 269         dev_dbg(dwc->dev, "request %p from %s completed %d/%d ===> %d\n",
 270                         req, dep->name, req->request.actual,
 271                         req->request.length, status);
 272
 273         spin_unlock(&dwc->lock);
 274         req->request.complete(&dep->endpoint, &req->request);
 275         spin_lock(&dwc->lock);
 276 }
 277
 278 static const char *dwc3_gadget_ep_cmd_string(u8 cmd)
 279 {
 280         switch (cmd) {
 281         case DWC3_DEPCMD_DEPSTARTCFG:
 282                 return "Start New Configuration";
 283         case DWC3_DEPCMD_ENDTRANSFER:
 284                 return "End Transfer";
 285         case DWC3_DEPCMD_UPDATETRANSFER:
 286                 return "Update Transfer";
 287         case DWC3_DEPCMD_STARTTRANSFER:
 288                 return "Start Transfer";
 289         case DWC3_DEPCMD_CLEARSTALL:
 290                 return "Clear Stall";
 291         case DWC3_DEPCMD_SETSTALL:
 292                 return "Set Stall";
 293         case DWC3_DEPCMD_GETEPSTATE:
 294                 return "Get Endpoint State";
 295         case DWC3_DEPCMD_SETTRANSFRESOURCE:
 296                 return "Set Endpoint Transfer Resource";
 297         case DWC3_DEPCMD_SETEPCONFIG:
 298                 return "Set Endpoint Configuration";
 299         default:
 300                 return "UNKNOWN command";
 301         }
 302 }
 303
 304 int dwc3_send_gadget_generic_command(struct dwc3 *dwc, int cmd, u32 param)
 305 {
 306         u32             timeout = 500;
 307         u32             reg;
 308
 309         dwc3_writel(dwc->regs, DWC3_DGCMDPAR, param);
 310         dwc3_writel(dwc->regs, DWC3_DGCMD, cmd | DWC3_DGCMD_CMDACT);
 311
 312         do {
 313                 reg = dwc3_readl(dwc->regs, DWC3_DGCMD);
 314                 if (!(reg & DWC3_DGCMD_CMDACT)) {
 315                         dev_vdbg(dwc->dev, "Command Complete --> %d\n",
 316                                         DWC3_DGCMD_STATUS(reg));
 317                         return 0;
 318                 }
 319
 320                 /*
 321                  * We can't sleep here, because it's also called from
 322                  * interrupt context.
 323                  */
 324                 timeout--;
 325                 if (!timeout)
 326                         return -ETIMEDOUT;
 327                 udelay(1);
 328         } while (1);
 329 }
 330
 331 int dwc3_send_gadget_ep_cmd(struct dwc3 *dwc, unsigned ep,
 332                 unsigned cmd, struct dwc3_gadget_ep_cmd_params *params)
 333 {
 334         struct dwc3_ep          *dep = dwc->eps[ep];
 335         u32                     timeout = 500;
 336         u32                     reg;
 337
 338         dev_vdbg(dwc->dev, "%s: cmd '%s' params %08x %08x %08x\n",
 339                         dep->name,
 340                         dwc3_gadget_ep_cmd_string(cmd), params->param0,
 341                         params->param1, params->param2);
 342
 343         dwc3_writel(dwc->regs, DWC3_DEPCMDPAR0(ep), params->param0);
 344         dwc3_writel(dwc->regs, DWC3_DEPCMDPAR1(ep), params->param1);
 345         dwc3_writel(dwc->regs, DWC3_DEPCMDPAR2(ep), params->param2);
 346
 347         dwc3_writel(dwc->regs, DWC3_DEPCMD(ep), cmd | DWC3_DEPCMD_CMDACT);
 348         do {
 349                 reg = dwc3_readl(dwc->regs, DWC3_DEPCMD(ep));
 350                 if (!(reg & DWC3_DEPCMD_CMDACT)) {
 351                         dev_vdbg(dwc->dev, "Command Complete --> %d\n",
 352                                         DWC3_DEPCMD_STATUS(reg));
 353                         return 0;
 354                 }
 355
 356                 /*
 357                  * We can't sleep here, because it is also called from
 358                  * interrupt context.
 359                  */
 360                 timeout--;
 361                 if (!timeout)
 362                         return -ETIMEDOUT;
 363
 364                 udelay(1);
 365         } while (1);
 366 }
 367
 368 static dma_addr_t dwc3_trb_dma_offset(struct dwc3_ep *dep,
 369                 struct dwc3_trb *trb)
 370 {
 371         u32             offset = (char *) trb - (char *) dep->trb_pool;
 372
 373         return dep->trb_pool_dma + offset;
 374 }
 375
 376 static int dwc3_alloc_trb_pool(struct dwc3_ep *dep)
 377 {
 378         struct dwc3             *dwc = dep->dwc;
 379
 380         if (dep->trb_pool)
 381                 return 0;
 382
 383         if (dep->number == 0 || dep->number == 1)
 384                 return 0;
 385
 386         dep->trb_pool = dma_alloc_coherent(dwc->dev,
 387                         sizeof(struct dwc3_trb) * DWC3_TRB_NUM,
 388                         &dep->trb_pool_dma, GFP_KERNEL);
 389         if (!dep->trb_pool) {
 390                 dev_err(dep->dwc->dev, "failed to allocate trb pool for %s\n",
 391                                 dep->name);
 392                 return -ENOMEM;
 393         }
 394
 395         return 0;
 396 }
 397
 398 static void dwc3_free_trb_pool(struct dwc3_ep *dep)
 399 {
 400         struct dwc3             *dwc = dep->dwc;
 401
 402         dma_free_coherent(dwc->dev, sizeof(struct dwc3_trb) * DWC3_TRB_NUM,
 403                         dep->trb_pool, dep->trb_pool_dma);
 404
 405         dep->trb_pool = NULL;
 406         dep->trb_pool_dma = 0;
 407 }
 408
 409 static int dwc3_gadget_start_config(struct dwc3 *dwc, struct dwc3_ep *dep)
 410 {
 411         struct dwc3_gadget_ep_cmd_params params;
 412         u32                     cmd;
 413
 414         memset(&params, 0x00, sizeof(params));
 415
 416         if (dep->number != 1) {
 417                 cmd = DWC3_DEPCMD_DEPSTARTCFG;
 418                 /* XferRscIdx == 0 for ep0 and 2 for the remaining */
 419                 if (dep->number > 1) {
 420                         if (dwc->start_config_issued)
 421                                 return 0;
 422                         dwc->start_config_issued = true;
 423                         cmd |= DWC3_DEPCMD_PARAM(2);
 424                 }
 425
 426                 return dwc3_send_gadget_ep_cmd(dwc, 0, cmd, &params);
 427         }
 428
 429         return 0;
 430 }
 431
 432 static int dwc3_gadget_set_ep_config(struct dwc3 *dwc, struct dwc3_ep *dep,
 433                 const struct usb_endpoint_descriptor *desc,
 434                 const struct usb_ss_ep_comp_descriptor *comp_desc,
 435                 bool ignore)
 436 {
 437         struct dwc3_gadget_ep_cmd_params params;
 438
 439         memset(&params, 0x00, sizeof(params));
 440
 441         params.param0 = DWC3_DEPCFG_EP_TYPE(usb_endpoint_type(desc))
 442                 | DWC3_DEPCFG_MAX_PACKET_SIZE(usb_endpoint_maxp(desc))
 443                 | DWC3_DEPCFG_BURST_SIZE(dep->endpoint.maxburst - 1);
 444
 445         if (ignore)
 446                 params.param0 |= DWC3_DEPCFG_IGN_SEQ_NUM;
 447
 448         params.param1 = DWC3_DEPCFG_XFER_COMPLETE_EN
 449                 | DWC3_DEPCFG_XFER_NOT_READY_EN;
 450
 451         if (usb_ss_max_streams(comp_desc) && usb_endpoint_xfer_bulk(desc)) {
 452                 params.param1 |= DWC3_DEPCFG_STREAM_CAPABLE
 453                         | DWC3_DEPCFG_STREAM_EVENT_EN;
 454                 dep->stream_capable = true;
 455         }
 456
 457         if (usb_endpoint_xfer_isoc(desc))
 458                 params.param1 |= DWC3_DEPCFG_XFER_IN_PROGRESS_EN;
 459
 460         /*
 461          * We are doing 1:1 mapping for endpoints, meaning
 462          * Physical Endpoints 2 maps to Logical Endpoint 2 and
 463          * so on. We consider the direction bit as part of the physical
 464          * endpoint number. So USB endpoint 0x81 is 0x03.
 465          */
 466         params.param1 |= DWC3_DEPCFG_EP_NUMBER(dep->number);
 467
 468         /*
 469          * We must use the lower 16 TX FIFOs even though
 470          * HW might have more
 471          */
 472         if (dep->direction)
 473                 params.param0 |= DWC3_DEPCFG_FIFO_NUMBER(dep->number >> 1);
 474
 475         if (desc->bInterval) {
 476                 params.param1 |= DWC3_DEPCFG_BINTERVAL_M1(desc->bInterval - 1);
 477                 dep->interval = 1 << (desc->bInterval - 1);
 478         }
 479
 480         return dwc3_send_gadget_ep_cmd(dwc, dep->number,
 481                         DWC3_DEPCMD_SETEPCONFIG, &params);
 482 }
 483
 484 static int dwc3_gadget_set_xfer_resource(struct dwc3 *dwc, struct dwc3_ep *dep)
 485 {
 486         struct dwc3_gadget_ep_cmd_params params;
 487
 488         memset(&params, 0x00, sizeof(params));
 489
 490         params.param0 = DWC3_DEPXFERCFG_NUM_XFER_RES(1);
 491
 492         return dwc3_send_gadget_ep_cmd(dwc, dep->number,
 493                         DWC3_DEPCMD_SETTRANSFRESOURCE, &params);
 494 }
 495
 496 /**
 497  * __dwc3_gadget_ep_enable - Initializes a HW endpoint
 498  * @dep: endpoint to be initialized
 499  * @desc: USB Endpoint Descriptor
 500  *
 501  * Caller should take care of locking
 502  */
 503 static int __dwc3_gadget_ep_enable(struct dwc3_ep *dep,
 504                 const struct usb_endpoint_descriptor *desc,
 505                 const struct usb_ss_ep_comp_descriptor *comp_desc,
 506                 bool ignore)
 507 {
 508         struct dwc3             *dwc = dep->dwc;
 509         u32                     reg;
 510         int                     ret = -ENOMEM;
 511
 512         if (!(dep->flags & DWC3_EP_ENABLED)) {
 513                 ret = dwc3_gadget_start_config(dwc, dep);
 514                 if (ret)
 515                         return ret;
 516         }
 517
 518         ret = dwc3_gadget_set_ep_config(dwc, dep, desc, comp_desc, ignore);
 519         if (ret)
 520                 return ret;
 521
 522         if (!(dep->flags & DWC3_EP_ENABLED)) {
 523                 struct dwc3_trb *trb_st_hw;
 524                 struct dwc3_trb *trb_link;
 525
 526                 ret = dwc3_gadget_set_xfer_resource(dwc, dep);
 527                 if (ret)
 528                         return ret;
 529
 530                 dep->endpoint.desc = desc;
 531                 dep->comp_desc = comp_desc;
 532                 dep->type = usb_endpoint_type(desc);
 533                 dep->flags |= DWC3_EP_ENABLED;
 534
 535                 reg = dwc3_readl(dwc->regs, DWC3_DALEPENA);
 536                 reg |= DWC3_DALEPENA_EP(dep->number);
 537                 dwc3_writel(dwc->regs, DWC3_DALEPENA, reg);
 538
 539                 if (!usb_endpoint_xfer_isoc(desc))
 540                         return 0;
 541
 542                 memset(&trb_link, 0, sizeof(trb_link));
 543
 544                 /* Link TRB for ISOC. The HWO bit is never reset */
 545                 trb_st_hw = &dep->trb_pool[0];
 546
 547                 trb_link = &dep->trb_pool[DWC3_TRB_NUM - 1];
 548
 549                 trb_link->bpl = lower_32_bits(dwc3_trb_dma_offset(dep, trb_st_hw));
 550                 trb_link->bph = upper_32_bits(dwc3_trb_dma_offset(dep, trb_st_hw));
 551                 trb_link->ctrl |= DWC3_TRBCTL_LINK_TRB;
 552                 trb_link->ctrl |= DWC3_TRB_CTRL_HWO;
 553         }
 554
 555         return 0;
 556 }
 557
 558 static void dwc3_stop_active_transfer(struct dwc3 *dwc, u32 epnum);
 559 static void dwc3_remove_requests(struct dwc3 *dwc, struct dwc3_ep *dep)
 560 {
 561         struct dwc3_request             *req;
 562
 563         if (!list_empty(&dep->req_queued)) {
 564                 dwc3_stop_active_transfer(dwc, dep->number);
 565
 566                 /*
 567                  * NOTICE: We are violating what the Databook says about the
 568                  * EndTransfer command. Ideally we would _always_ wait for the
 569                  * EndTransfer Command Completion IRQ, but that's causing too
 570                  * much trouble synchronizing between us and gadget driver.
 571                  *
 572                  * We have discussed this with the IP Provider and it was
 573                  * suggested to giveback all requests here, but give HW some
 574                  * extra time to synchronize with the interconnect. We're using
 575                  * an arbitraty 100us delay for that.
 576                  *
 577                  * Note also that a similar handling was tested by Synopsys
 578                  * (thanks a lot Paul) and nothing bad has come out of it.
 579                  * In short, what we're doing is:
 580                  *
 581                  * - Issue EndTransfer WITH CMDIOC bit set
 582                  * - Wait 100us
 583                  * - giveback all requests to gadget driver
 584                  */
 585                 udelay(100);
 586
 587                 while (!list_empty(&dep->req_queued)) {
 588                         req = next_request(&dep->req_queued);
 589
 590                         dwc3_gadget_giveback(dep, req, -ESHUTDOWN);
 591                 }
 592         }
 593
 594         while (!list_empty(&dep->request_list)) {
 595                 req = next_request(&dep->request_list);
 596
 597                 dwc3_gadget_giveback(dep, req, -ESHUTDOWN);
 598         }
 599 }
 600
 601 /**
 602  * __dwc3_gadget_ep_disable - Disables a HW endpoint
 603  * @dep: the endpoint to disable
 604  *
 605  * This function also removes requests which are currently processed ny the
 606  * hardware and those which are not yet scheduled.
 607  * Caller should take care of locking.
 608  */
 609 static int __dwc3_gadget_ep_disable(struct dwc3_ep *dep)
 610 {
 611         struct dwc3             *dwc = dep->dwc;
 612         u32                     reg;
 613
 614         dwc3_remove_requests(dwc, dep);
 615
 616         reg = dwc3_readl(dwc->regs, DWC3_DALEPENA);
 617         reg &= ~DWC3_DALEPENA_EP(dep->number);
 618         dwc3_writel(dwc->regs, DWC3_DALEPENA, reg);
 619
 620         dep->stream_capable = false;
 621         dep->endpoint.desc = NULL;
 622         dep->comp_desc = NULL;
 623         dep->type = 0;
 624         dep->flags = 0;
 625
 626         return 0;
 627 }
 628
 629 /* -------------------------------------------------------------------------- */
 630
 631 static int dwc3_gadget_ep0_enable(struct usb_ep *ep,
 632                 const struct usb_endpoint_descriptor *desc)
 633 {
 634         return -EINVAL;
 635 }
 636
 637 static int dwc3_gadget_ep0_disable(struct usb_ep *ep)
 638 {
 639         return -EINVAL;
 640 }
 641
 642 /* -------------------------------------------------------------------------- */
 643
 644 static int dwc3_gadget_ep_enable(struct usb_ep *ep,
 645                 const struct usb_endpoint_descriptor *desc)
 646 {
 647         struct dwc3_ep                  *dep;
 648         struct dwc3                     *dwc;
 649         unsigned long                   flags;
 650         int                             ret;
 651
 652         if (!ep || !desc || desc->bDescriptorType != USB_DT_ENDPOINT) {
 653                 pr_debug("dwc3: invalid parameters\n");
 654                 return -EINVAL;
 655         }
 656
 657         if (!desc->wMaxPacketSize) {
 658                 pr_debug("dwc3: missing wMaxPacketSize\n");
 659                 return -EINVAL;
 660         }
 661
 662         dep = to_dwc3_ep(ep);
 663         dwc = dep->dwc;
 664
 665         switch (usb_endpoint_type(desc)) {
 666         case USB_ENDPOINT_XFER_CONTROL:
 667                 strlcat(dep->name, "-control", sizeof(dep->name));
 668                 break;
 669         case USB_ENDPOINT_XFER_ISOC:
 670                 strlcat(dep->name, "-isoc", sizeof(dep->name));
 671                 break;
 672         case USB_ENDPOINT_XFER_BULK:
 673                 strlcat(dep->name, "-bulk", sizeof(dep->name));
 674                 break;
 675         case USB_ENDPOINT_XFER_INT:
 676                 strlcat(dep->name, "-int", sizeof(dep->name));
 677                 break;
 678         default:
 679                 dev_err(dwc->dev, "invalid endpoint transfer type\n");
 680         }
 681
 682         if (dep->flags & DWC3_EP_ENABLED) {
 683                 dev_WARN_ONCE(dwc->dev, true, "%s is already enabled\n",
 684                                 dep->name);
 685                 return 0;
 686         }
 687
 688         dev_vdbg(dwc->dev, "Enabling %s\n", dep->name);
 689
 690         spin_lock_irqsave(&dwc->lock, flags);
 691         ret = __dwc3_gadget_ep_enable(dep, desc, ep->comp_desc, false);
 692         spin_unlock_irqrestore(&dwc->lock, flags);
 693
 694         return ret;
 695 }
 696
 697 static int dwc3_gadget_ep_disable(struct usb_ep *ep)
 698 {
 699         struct dwc3_ep                  *dep;
 700         struct dwc3                     *dwc;
 701         unsigned long                   flags;
 702         int                             ret;
 703
 704         if (!ep) {
 705                 pr_debug("dwc3: invalid parameters\n");
 706                 return -EINVAL;
 707         }
 708
 709         dep = to_dwc3_ep(ep);
 710         dwc = dep->dwc;
 711
 712         if (!(dep->flags & DWC3_EP_ENABLED)) {
 713                 dev_WARN_ONCE(dwc->dev, true, "%s is already disabled\n",
 714                                 dep->name);
 715                 return 0;
 716         }
 717
 718         snprintf(dep->name, sizeof(dep->name), "ep%d%s",
 719                         dep->number >> 1,
 720                         (dep->number & 1) ? "in" : "out");
 721
 722         spin_lock_irqsave(&dwc->lock, flags);
 723         ret = __dwc3_gadget_ep_disable(dep);
 724         spin_unlock_irqrestore(&dwc->lock, flags);
 725
 726         return ret;
 727 }
 728
 729 static struct usb_request *dwc3_gadget_ep_alloc_request(struct usb_ep *ep,
 730         gfp_t gfp_flags)
 731 {
 732         struct dwc3_request             *req;
 733         struct dwc3_ep                  *dep = to_dwc3_ep(ep);
 734         struct dwc3                     *dwc = dep->dwc;
 735
 736         req = kzalloc(sizeof(*req), gfp_flags);
 737         if (!req) {
 738                 dev_err(dwc->dev, "not enough memory\n");
 739                 return NULL;
 740         }
 741
 742         req->epnum      = dep->number;
 743         req->dep        = dep;
 744
 745         return &req->request;
 746 }
 747
 748 static void dwc3_gadget_ep_free_request(struct usb_ep *ep,
 749                 struct usb_request *request)
 750 {
 751         struct dwc3_request             *req = to_dwc3_request(request);
 752
 753         kfree(req);
 754 }
 755
 756 /**
 757  * dwc3_prepare_one_trb - setup one TRB from one request
 758  * @dep: endpoint for which this request is prepared
 759  * @req: dwc3_request pointer
 760  */
 761 static void dwc3_prepare_one_trb(struct dwc3_ep *dep,
 762                 struct dwc3_request *req, dma_addr_t dma,
 763                 unsigned length, unsigned last, unsigned chain)
 764 {
 765         struct dwc3             *dwc = dep->dwc;
 766         struct dwc3_trb         *trb;
 767
 768         unsigned int            cur_slot;
 769
 770         dev_vdbg(dwc->dev, "%s: req %p dma %08llx length %d%s%s\n",
 771                         dep->name, req, (unsigned long long) dma,
 772                         length, last ? " last" : "",
 773                         chain ? " chain" : "");
 774
 775         trb = &dep->trb_pool[dep->free_slot & DWC3_TRB_MASK];
 776         cur_slot = dep->free_slot;
 777         dep->free_slot++;
 778
 779         /* Skip the LINK-TRB on ISOC */
 780         if (((cur_slot & DWC3_TRB_MASK) == DWC3_TRB_NUM - 1) &&
 781                         usb_endpoint_xfer_isoc(dep->endpoint.desc))
 782                 return;
 783
 784         if (!req->trb) {
 785                 dwc3_gadget_move_request_queued(req);
 786                 req->trb = trb;
 787                 req->trb_dma = dwc3_trb_dma_offset(dep, trb);
 788         }
 789
 790         trb->size = DWC3_TRB_SIZE_LENGTH(length);
 791         trb->bpl = lower_32_bits(dma);
 792         trb->bph = upper_32_bits(dma);
 793
 794         switch (usb_endpoint_type(dep->endpoint.desc)) {
 795         case USB_ENDPOINT_XFER_CONTROL:
 796                 trb->ctrl = DWC3_TRBCTL_CONTROL_SETUP;
 797                 break;
 798
 799         case USB_ENDPOINT_XFER_ISOC:
 800                 trb->ctrl = DWC3_TRBCTL_ISOCHRONOUS_FIRST;
 801
 802                 if (!req->request.no_interrupt)
 803                         trb->ctrl |= DWC3_TRB_CTRL_IOC;
 804                 break;
 805
 806         case USB_ENDPOINT_XFER_BULK:
 807         case USB_ENDPOINT_XFER_INT:
 808                 trb->ctrl = DWC3_TRBCTL_NORMAL;
 809                 break;
 810         default:
 811                 /*
 812                  * This is only possible with faulty memory because we
 813                  * checked it already :)
 814                  */
 815                 BUG();
 816         }
 817
 818         if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) {
 819                 trb->ctrl |= DWC3_TRB_CTRL_ISP_IMI;
 820                 trb->ctrl |= DWC3_TRB_CTRL_CSP;
 821         } else {
 822                 if (chain)
 823                         trb->ctrl |= DWC3_TRB_CTRL_CHN;
 824
 825                 if (last)
 826                         trb->ctrl |= DWC3_TRB_CTRL_LST;
 827         }
 828
 829         if (usb_endpoint_xfer_bulk(dep->endpoint.desc) && dep->stream_capable)
 830                 trb->ctrl |= DWC3_TRB_CTRL_SID_SOFN(req->request.stream_id);
 831
 832         trb->ctrl |= DWC3_TRB_CTRL_HWO;
 833 }
 834
 835 /*
 836  * dwc3_prepare_trbs - setup TRBs from requests
 837  * @dep: endpoint for which requests are being prepared
 838  * @starting: true if the endpoint is idle and no requests are queued.
 839  *
 840  * The function goes through the requests list and sets up TRBs for the
 841  * transfers. The function returns once there are no more TRBs available or
 842  * it runs out of requests.
 843  */
 844 static void dwc3_prepare_trbs(struct dwc3_ep *dep, bool starting)
 845 {
 846         struct dwc3_request     *req, *n;
 847         u32                     trbs_left;
 848         u32                     max;
 849         unsigned int            last_one = 0;
 850
 851         BUILD_BUG_ON_NOT_POWER_OF_2(DWC3_TRB_NUM);
 852
 853         /* the first request must not be queued */
 854         trbs_left = (dep->busy_slot - dep->free_slot) & DWC3_TRB_MASK;
 855
 856         /* Can't wrap around on a non-isoc EP since there's no link TRB */
 857         if (!usb_endpoint_xfer_isoc(dep->endpoint.desc)) {
 858                 max = DWC3_TRB_NUM - (dep->free_slot & DWC3_TRB_MASK);
 859                 if (trbs_left > max)
 860                         trbs_left = max;
 861         }
 862
 863         /*
 864          * If busy & slot are equal than it is either full or empty. If we are
 865          * starting to process requests then we are empty. Otherwise we are
 866          * full and don't do anything
 867          */
 868         if (!trbs_left) {
 869                 if (!starting)
 870                         return;
 871                 trbs_left = DWC3_TRB_NUM;
 872                 /*
 873                  * In case we start from scratch, we queue the ISOC requests
 874                  * starting from slot 1. This is done because we use ring
 875                  * buffer and have no LST bit to stop us. Instead, we place
 876                  * IOC bit every TRB_NUM/4. We try to avoid having an interrupt
 877                  * after the first request so we start at slot 1 and have
 878                  * 7 requests proceed before we hit the first IOC.
 879                  * Other transfer types don't use the ring buffer and are
 880                  * processed from the first TRB until the last one. Since we
 881                  * don't wrap around we have to start at the beginning.
 882                  */
 883                 if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) {
 884                         dep->busy_slot = 1;
 885                         dep->free_slot = 1;
 886                 } else {
 887                         dep->busy_slot = 0;
 888                         dep->free_slot = 0;
 889                 }
 890         }
 891
 892         /* The last TRB is a link TRB, not used for xfer */
 893         if ((trbs_left <= 1) && usb_endpoint_xfer_isoc(dep->endpoint.desc))
 894                 return;
 895
 896         list_for_each_entry_safe(req, n, &dep->request_list, list) {
 897                 unsigned        length;
 898                 dma_addr_t      dma;
 899
 900                 if (req->request.num_mapped_sgs > 0) {
 901                         struct usb_request *request = &req->request;
 902                         struct scatterlist *sg = request->sg;
 903                         struct scatterlist *s;
 904                         int             i;
 905
 906                         for_each_sg(sg, s, request->num_mapped_sgs, i) {
 907                                 unsigned chain = true;
 908
 909                                 length = sg_dma_len(s);
 910                                 dma = sg_dma_address(s);
 911
 912                                 if (i == (request->num_mapped_sgs - 1) ||
 913                                                 sg_is_last(s)) {
 914                                         last_one = true;
 915                                         chain = false;
 916                                 }
 917
 918                                 trbs_left--;
 919                                 if (!trbs_left)
 920                                         last_one = true;
 921
 922                                 if (last_one)
 923                                         chain = false;
 924
 925                                 dwc3_prepare_one_trb(dep, req, dma, length,
 926                                                 last_one, chain);
 927
 928                                 if (last_one)
 929                                         break;
 930                         }
 931                 } else {
 932                         dma = req->request.dma;
 933                         length = req->request.length;
 934                         trbs_left--;
 935
 936                         if (!trbs_left)
 937                                 last_one = 1;
 938
 939                         /* Is this the last request? */
 940                         if (list_is_last(&req->list, &dep->request_list))
 941                                 last_one = 1;
 942
 943                         dwc3_prepare_one_trb(dep, req, dma, length,
 944                                         last_one, false);
 945
 946                         if (last_one)
 947                                 break;
 948                 }
 949         }
 950 }
 951
 952 static int __dwc3_gadget_kick_transfer(struct dwc3_ep *dep, u16 cmd_param,
 953                 int start_new)
 954 {
 955         struct dwc3_gadget_ep_cmd_params params;
 956         struct dwc3_request             *req;
 957         struct dwc3                     *dwc = dep->dwc;
 958         int                             ret;
 959         u32                             cmd;
 960
 961         if (start_new && (dep->flags & DWC3_EP_BUSY)) {
 962                 dev_vdbg(dwc->dev, "%s: endpoint busy\n", dep->name);
 963                 return -EBUSY;
 964         }
 965         dep->flags &= ~DWC3_EP_PENDING_REQUEST;
 966
 967         /*
 968          * If we are getting here after a short-out-packet we don't enqueue any
 969          * new requests as we try to set the IOC bit only on the last request.
 970          */
 971         if (start_new) {
 972                 if (list_empty(&dep->req_queued))
 973                         dwc3_prepare_trbs(dep, start_new);
 974
 975                 /* req points to the first request which will be sent */
 976                 req = next_request(&dep->req_queued);
 977         } else {
 978                 dwc3_prepare_trbs(dep, start_new);
 979
 980                 /*
 981                  * req points to the first request where HWO changed from 0 to 1
 982                  */
 983                 req = next_request(&dep->req_queued);
 984         }
 985         if (!req) {
 986                 dep->flags |= DWC3_EP_PENDING_REQUEST;
 987                 return 0;
 988         }
 989
 990         memset(&params, 0, sizeof(params));
 991         params.param0 = upper_32_bits(req->trb_dma);
 992         params.param1 = lower_32_bits(req->trb_dma);
 993
 994         if (start_new)
 995                 cmd = DWC3_DEPCMD_STARTTRANSFER;
 996         else
 997                 cmd = DWC3_DEPCMD_UPDATETRANSFER;
 998
 999         cmd |= DWC3_DEPCMD_PARAM(cmd_param);
1000         ret = dwc3_send_gadget_ep_cmd(dwc, dep->number, cmd, &params);
1001         if (ret < 0) {
1002                 dev_dbg(dwc->dev, "failed to send STARTTRANSFER command\n");
1003
1004                 /*
1005                  * FIXME we need to iterate over the list of requests
1006                  * here and stop, unmap, free and del each of the linked
1007                  * requests instead of what we do now.
1008                  */
1009                 usb_gadget_unmap_request(&dwc->gadget, &req->request,
1010                                 req->direction);
1011                 list_del(&req->list);
1012                 return ret;
1013         }
1014
1015         dep->flags |= DWC3_EP_BUSY;
1016
1017         if (start_new) {
1018                 dep->resource_index = dwc3_gadget_ep_get_transfer_index(dwc,
1019                                 dep->number);
1020                 WARN_ON_ONCE(!dep->resource_index);
1021         }
1022
1023         return 0;
1024 }
1025
1026 static void __dwc3_gadget_start_isoc(struct dwc3 *dwc,
1027                 struct dwc3_ep *dep, u32 cur_uf)
1028 {
1029         u32 uf;
1030
1031         if (list_empty(&dep->request_list)) {
1032                 dev_vdbg(dwc->dev, "ISOC ep %s run out for requests.\n",
1033                         dep->name);
1034                 return;
1035         }
1036
1037         /* 4 micro frames in the future */
1038         uf = cur_uf + dep->interval * 4;
1039
1040         __dwc3_gadget_kick_transfer(dep, uf, 1);
1041 }
1042
1043 static void dwc3_gadget_start_isoc(struct dwc3 *dwc,
1044                 struct dwc3_ep *dep, const struct dwc3_event_depevt *event)
1045 {
1046         u32 cur_uf, mask;
1047
1048         mask = ~(dep->interval - 1);
1049         cur_uf = event->parameters & mask;
1050
1051         __dwc3_gadget_start_isoc(dwc, dep, cur_uf);
1052 }
1053
1054 static int __dwc3_gadget_ep_queue(struct dwc3_ep *dep, struct dwc3_request *req)
1055 {
1056         struct dwc3             *dwc = dep->dwc;
1057         int                     ret;
1058
1059         req->request.actual     = 0;
1060         req->request.status     = -EINPROGRESS;
1061         req->direction          = dep->direction;
1062         req->epnum              = dep->number;
1063
1064         /*
1065          * We only add to our list of requests now and
1066          * start consuming the list once we get XferNotReady
1067          * IRQ.
1068          *
1069          * That way, we avoid doing anything that we don't need
1070          * to do now and defer it until the point we receive a
1071          * particular token from the Host side.
1072          *
1073          * This will also avoid Host cancelling URBs due to too
1074          * many NAKs.
1075          */
1076         ret = usb_gadget_map_request(&dwc->gadget, &req->request,
1077                         dep->direction);
1078         if (ret)
1079                 return ret;
1080
1081         list_add_tail(&req->list, &dep->request_list);
1082
1083         /*
1084          * There are a few special cases:
1085          *
1086          * 1. XferNotReady with empty list of requests. We need to kick the
1087          *    transfer here in that situation, otherwise we will be NAKing
1088          *    forever. If we get XferNotReady before gadget driver has a
1089          *    chance to queue a request, we will ACK the IRQ but won't be
1090          *    able to receive the data until the next request is queued.
1091          *    The following code is handling exactly that.
1092          *
1093          */
1094         if (dep->flags & DWC3_EP_PENDING_REQUEST) {
1095                 ret = __dwc3_gadget_kick_transfer(dep, 0, true);
1096                 if (ret && ret != -EBUSY)
1097                         dev_dbg(dwc->dev, "%s: failed to kick transfers\n",
1098                                         dep->name);
1099         }
1100
1101         /*
1102          * 2. XferInProgress on Isoc EP with an active transfer. We need to
1103          *    kick the transfer here after queuing a request, otherwise the
1104          *    core may not see the modified TRB(s).
1105          */
1106         if (usb_endpoint_xfer_isoc(dep->endpoint.desc) &&
1107                         (dep->flags & DWC3_EP_BUSY) &&
1108                         !(dep->flags & DWC3_EP_MISSED_ISOC)) {
1109                 WARN_ON_ONCE(!dep->resource_index);
1110                 ret = __dwc3_gadget_kick_transfer(dep, dep->resource_index,
1111                                 false);
1112                 if (ret && ret != -EBUSY)
1113                         dev_dbg(dwc->dev, "%s: failed to kick transfers\n",
1114                                         dep->name);
1115         }
1116
1117         /*
1118          * 3. Missed ISOC Handling. We need to start isoc transfer on the saved
1119          * uframe number.
1120          */
1121         if (usb_endpoint_xfer_isoc(dep->endpoint.desc) &&
1122                 (dep->flags & DWC3_EP_MISSED_ISOC)) {
1123                         __dwc3_gadget_start_isoc(dwc, dep, dep->current_uf);
1124                         dep->flags &= ~DWC3_EP_MISSED_ISOC;
1125         }
1126
1127         return 0;
1128 }
1129
1130 static int dwc3_gadget_ep_queue(struct usb_ep *ep, struct usb_request *request,
1131         gfp_t gfp_flags)
1132 {
1133         struct dwc3_request             *req = to_dwc3_request(request);
1134         struct dwc3_ep                  *dep = to_dwc3_ep(ep);
1135         struct dwc3                     *dwc = dep->dwc;
1136
1137         unsigned long                   flags;
1138
1139         int                             ret;
1140
1141         if (!dep->endpoint.desc) {
1142                 dev_dbg(dwc->dev, "trying to queue request %p to disabled %s\n",
1143                                 request, ep->name);
1144                 return -ESHUTDOWN;
1145         }
1146
1147         dev_vdbg(dwc->dev, "queing request %p to %s length %d\n",
1148                         request, ep->name, request->length);
1149
1150         spin_lock_irqsave(&dwc->lock, flags);
1151         ret = __dwc3_gadget_ep_queue(dep, req);
1152         spin_unlock_irqrestore(&dwc->lock, flags);
1153
1154         return ret;
1155 }
1156
1157 static int dwc3_gadget_ep_dequeue(struct usb_ep *ep,
1158                 struct usb_request *request)
1159 {
1160         struct dwc3_request             *req = to_dwc3_request(request);
1161         struct dwc3_request             *r = NULL;
1162
1163         struct dwc3_ep                  *dep = to_dwc3_ep(ep);
1164         struct dwc3                     *dwc = dep->dwc;
1165
1166         unsigned long                   flags;
1167         int                             ret = 0;
1168
1169         spin_lock_irqsave(&dwc->lock, flags);
1170
1171         list_for_each_entry(r, &dep->request_list, list) {
1172                 if (r == req)
1173                         break;
1174         }
1175
1176         if (r != req) {
1177                 list_for_each_entry(r, &dep->req_queued, list) {
1178                         if (r == req)
1179                                 break;
1180                 }
1181                 if (r == req) {
1182                         /* wait until it is processed */
1183                         dwc3_stop_active_transfer(dwc, dep->number);
1184                         goto out1;
1185                 }
1186                 dev_err(dwc->dev, "request %p was not queued to %s\n",
1187                                 request, ep->name);
1188                 ret = -EINVAL;
1189                 goto out0;
1190         }
1191
1192 out1:
1193         /* giveback the request */
1194         dwc3_gadget_giveback(dep, req, -ECONNRESET);
1195
1196 out0:
1197         spin_unlock_irqrestore(&dwc->lock, flags);
1198
1199         return ret;
1200 }
1201
1202 int __dwc3_gadget_ep_set_halt(struct dwc3_ep *dep, int value)
1203 {
1204         struct dwc3_gadget_ep_cmd_params        params;
1205         struct dwc3                             *dwc = dep->dwc;
1206         int                                     ret;
1207
1208         memset(&params, 0x00, sizeof(params));
1209
1210         if (value) {
1211                 ret = dwc3_send_gadget_ep_cmd(dwc, dep->number,
1212                         DWC3_DEPCMD_SETSTALL, &params);
1213                 if (ret)
1214                         dev_err(dwc->dev, "failed to %s STALL on %s\n",
1215                                         value ? "set" : "clear",
1216                                         dep->name);
1217                 else
1218                         dep->flags |= DWC3_EP_STALL;
1219         } else {
1220                 if (dep->flags & DWC3_EP_WEDGE)
1221                         return 0;
1222
1223                 ret = dwc3_send_gadget_ep_cmd(dwc, dep->number,
1224                         DWC3_DEPCMD_CLEARSTALL, &params);
1225                 if (ret)
1226                         dev_err(dwc->dev, "failed to %s STALL on %s\n",
1227                                         value ? "set" : "clear",
1228                                         dep->name);
1229                 else
1230                         dep->flags &= ~DWC3_EP_STALL;
1231         }
1232
1233         return ret;
1234 }
1235
1236 static int dwc3_gadget_ep_set_halt(struct usb_ep *ep, int value)
1237 {
1238         struct dwc3_ep                  *dep = to_dwc3_ep(ep);
1239         struct dwc3                     *dwc = dep->dwc;
1240
1241         unsigned long                   flags;
1242
1243         int                             ret;
1244
1245         spin_lock_irqsave(&dwc->lock, flags);
1246
1247         if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) {
1248                 dev_err(dwc->dev, "%s is of Isochronous type\n", dep->name);
1249                 ret = -EINVAL;
1250                 goto out;
1251         }
1252
1253         ret = __dwc3_gadget_ep_set_halt(dep, value);
1254 out:
1255         spin_unlock_irqrestore(&dwc->lock, flags);
1256
1257         return ret;
1258 }
1259
1260 static int dwc3_gadget_ep_set_wedge(struct usb_ep *ep)
1261 {
1262         struct dwc3_ep                  *dep = to_dwc3_ep(ep);
1263         struct dwc3                     *dwc = dep->dwc;
1264         unsigned long                   flags;
1265
1266         spin_lock_irqsave(&dwc->lock, flags);
1267         dep->flags |= DWC3_EP_WEDGE;
1268         spin_unlock_irqrestore(&dwc->lock, flags);
1269
1270         if (dep->number == 0 || dep->number == 1)
1271                 return dwc3_gadget_ep0_set_halt(ep, 1);
1272         else
1273                 return dwc3_gadget_ep_set_halt(ep, 1);
1274 }
1275
1276 /* -------------------------------------------------------------------------- */
1277
1278 static struct usb_endpoint_descriptor dwc3_gadget_ep0_desc = {
1279         .bLength        = USB_DT_ENDPOINT_SIZE,
1280         .bDescriptorType = USB_DT_ENDPOINT,
1281         .bmAttributes   = USB_ENDPOINT_XFER_CONTROL,
1282 };
1283
1284 static const struct usb_ep_ops dwc3_gadget_ep0_ops = {
1285         .enable         = dwc3_gadget_ep0_enable,
1286         .disable        = dwc3_gadget_ep0_disable,
1287         .alloc_request  = dwc3_gadget_ep_alloc_request,
1288         .free_request   = dwc3_gadget_ep_free_request,
1289         .queue          = dwc3_gadget_ep0_queue,
1290         .dequeue        = dwc3_gadget_ep_dequeue,
1291         .set_halt       = dwc3_gadget_ep0_set_halt,
1292         .set_wedge      = dwc3_gadget_ep_set_wedge,
1293 };
1294
1295 static const struct usb_ep_ops dwc3_gadget_ep_ops = {
1296         .enable         = dwc3_gadget_ep_enable,
1297         .disable        = dwc3_gadget_ep_disable,
1298         .alloc_request  = dwc3_gadget_ep_alloc_request,
1299         .free_request   = dwc3_gadget_ep_free_request,
1300         .queue          = dwc3_gadget_ep_queue,
1301         .dequeue        = dwc3_gadget_ep_dequeue,
1302         .set_halt       = dwc3_gadget_ep_set_halt,
1303         .set_wedge      = dwc3_gadget_ep_set_wedge,
1304 };
1305
1306 /* -------------------------------------------------------------------------- */
1307
1308 static int dwc3_gadget_get_frame(struct usb_gadget *g)
1309 {
1310         struct dwc3             *dwc = gadget_to_dwc(g);
1311         u32                     reg;
1312
1313         reg = dwc3_readl(dwc->regs, DWC3_DSTS);
1314         return DWC3_DSTS_SOFFN(reg);
1315 }
1316
1317 static int dwc3_gadget_wakeup(struct usb_gadget *g)
1318 {
1319         struct dwc3             *dwc = gadget_to_dwc(g);
1320
1321         unsigned long           timeout;
1322         unsigned long           flags;
1323
1324         u32                     reg;
1325
1326         int                     ret = 0;
1327
1328         u8                      link_state;
1329         u8                      speed;
1330
1331         spin_lock_irqsave(&dwc->lock, flags);
1332
1333         /*
1334          * According to the Databook Remote wakeup request should
1335          * be issued only when the device is in early suspend state.
1336          *
1337          * We can check that via USB Link State bits in DSTS register.
1338          */
1339         reg = dwc3_readl(dwc->regs, DWC3_DSTS);
1340
1341         speed = reg & DWC3_DSTS_CONNECTSPD;
1342         if (speed == DWC3_DSTS_SUPERSPEED) {
1343                 dev_dbg(dwc->dev, "no wakeup on SuperSpeed\n");
1344                 ret = -EINVAL;
1345                 goto out;
1346         }
1347
1348         link_state = DWC3_DSTS_USBLNKST(reg);
1349
1350         switch (link_state) {
1351         case DWC3_LINK_STATE_RX_DET:    /* in HS, means Early Suspend */
1352         case DWC3_LINK_STATE_U3:        /* in HS, means SUSPEND */
1353                 break;
1354         default:
1355                 dev_dbg(dwc->dev, "can't wakeup from link state %d\n",
1356                                 link_state);
1357                 ret = -EINVAL;
1358                 goto out;
1359         }
1360
1361         ret = dwc3_gadget_set_link_state(dwc, DWC3_LINK_STATE_RECOV);
1362         if (ret < 0) {
1363                 dev_err(dwc->dev, "failed to put link in Recovery\n");
1364                 goto out;
1365         }
1366
1367         /* Recent versions do this automatically */
1368         if (dwc->revision < DWC3_REVISION_194A) {
1369                 /* write zeroes to Link Change Request */
1370                 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
1371                 reg &= ~DWC3_DCTL_ULSTCHNGREQ_MASK;
1372                 dwc3_writel(dwc->regs, DWC3_DCTL, reg);
1373         }
1374
1375         /* poll until Link State changes to ON */
1376         timeout = jiffies + msecs_to_jiffies(100);
1377
1378         while (!time_after(jiffies, timeout)) {
1379                 reg = dwc3_readl(dwc->regs, DWC3_DSTS);
1380
1381                 /* in HS, means ON */
1382                 if (DWC3_DSTS_USBLNKST(reg) == DWC3_LINK_STATE_U0)
1383                         break;
1384         }
1385
1386         if (DWC3_DSTS_USBLNKST(reg) != DWC3_LINK_STATE_U0) {
1387                 dev_err(dwc->dev, "failed to send remote wakeup\n");
1388                 ret = -EINVAL;
1389         }
1390
1391 out:
1392         spin_unlock_irqrestore(&dwc->lock, flags);
1393
1394         return ret;
1395 }
1396
1397 static int dwc3_gadget_set_selfpowered(struct usb_gadget *g,
1398                 int is_selfpowered)
1399 {
1400         struct dwc3             *dwc = gadget_to_dwc(g);
1401         unsigned long           flags;
1402
1403         spin_lock_irqsave(&dwc->lock, flags);
1404         dwc->is_selfpowered = !!is_selfpowered;
1405         spin_unlock_irqrestore(&dwc->lock, flags);
1406
1407         return 0;
1408 }
1409
1410 static int dwc3_gadget_run_stop(struct dwc3 *dwc, int is_on)
1411 {
1412         u32                     reg;
1413         u32                     timeout = 500;
1414
1415         reg = dwc3_readl(dwc->regs, DWC3_DCTL);
1416         if (is_on) {
1417                 if (dwc->revision <= DWC3_REVISION_187A) {
1418                         reg &= ~DWC3_DCTL_TRGTULST_MASK;
1419                         reg |= DWC3_DCTL_TRGTULST_RX_DET;
1420                 }
1421
1422                 if (dwc->revision >= DWC3_REVISION_194A)
1423                         reg &= ~DWC3_DCTL_KEEP_CONNECT;
1424                 reg |= DWC3_DCTL_RUN_STOP;
1425         } else {
1426                 reg &= ~DWC3_DCTL_RUN_STOP;
1427         }
1428
1429         dwc3_writel(dwc->regs, DWC3_DCTL, reg);
1430
1431         do {
1432                 reg = dwc3_readl(dwc->regs, DWC3_DSTS);
1433                 if (is_on) {
1434                         if (!(reg & DWC3_DSTS_DEVCTRLHLT))
1435                                 break;
1436                 } else {
1437                         if (reg & DWC3_DSTS_DEVCTRLHLT)
1438                                 break;
1439                 }
1440                 timeout--;
1441                 if (!timeout)
1442                         return -ETIMEDOUT;
1443                 udelay(1);
1444         } while (1);
1445
1446         dev_vdbg(dwc->dev, "gadget %s data soft-%s\n",
1447                         dwc->gadget_driver
1448                         ? dwc->gadget_driver->function : "no-function",
1449                         is_on ? "connect" : "disconnect");
1450
1451         return 0;
1452 }
1453
1454 static int dwc3_gadget_pullup(struct usb_gadget *g, int is_on)
1455 {
1456         struct dwc3             *dwc = gadget_to_dwc(g);
1457         unsigned long           flags;
1458         int                     ret;
1459
1460         is_on = !!is_on;
1461
1462         spin_lock_irqsave(&dwc->lock, flags);
1463         ret = dwc3_gadget_run_stop(dwc, is_on);
1464         spin_unlock_irqrestore(&dwc->lock, flags);
1465
1466         return ret;
1467 }
1468
1469 static int dwc3_gadget_start(struct usb_gadget *g,
1470                 struct usb_gadget_driver *driver)
1471 {
1472         struct dwc3             *dwc = gadget_to_dwc(g);
1473         struct dwc3_ep          *dep;
1474         unsigned long           flags;
1475         int                     ret = 0;
1476         u32                     reg;
1477
1478         spin_lock_irqsave(&dwc->lock, flags);
1479
1480         if (dwc->gadget_driver) {
1481                 dev_err(dwc->dev, "%s is already bound to %s\n",
1482                                 dwc->gadget.name,
1483                                 dwc->gadget_driver->driver.name);
1484                 ret = -EBUSY;
1485                 goto err0;
1486         }
1487
1488         dwc->gadget_driver      = driver;
1489         dwc->gadget.dev.driver  = &driver->driver;
1490
1491         reg = dwc3_readl(dwc->regs, DWC3_DCFG);
1492         reg &= ~(DWC3_DCFG_SPEED_MASK);
1493
1494         /**
1495          * WORKAROUND: DWC3 revision < 2.20a have an issue
1496          * which would cause metastability state on Run/Stop
1497          * bit if we try to force the IP to USB2-only mode.
1498          *
1499          * Because of that, we cannot configure the IP to any
1500          * speed other than the SuperSpeed
1501          *
1502          * Refers to:
1503          *
1504          * STAR#9000525659: Clock Domain Crossing on DCTL in
1505          * USB 2.0 Mode
1506          */
1507         if (dwc->revision < DWC3_REVISION_220A)
1508                 reg |= DWC3_DCFG_SUPERSPEED;
1509         else
1510                 reg |= dwc->maximum_speed;
1511         dwc3_writel(dwc->regs, DWC3_DCFG, reg);
1512
1513         dwc->start_config_issued = false;
1514
1515         /* Start with SuperSpeed Default */
1516         dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(512);
1517
1518         dep = dwc->eps[0];
1519         ret = __dwc3_gadget_ep_enable(dep, &dwc3_gadget_ep0_desc, NULL, false);
1520         if (ret) {
1521                 dev_err(dwc->dev, "failed to enable %s\n", dep->name);
1522                 goto err0;
1523         }
1524
1525         dep = dwc->eps[1];
1526         ret = __dwc3_gadget_ep_enable(dep, &dwc3_gadget_ep0_desc, NULL, false);
1527         if (ret) {
1528                 dev_err(dwc->dev, "failed to enable %s\n", dep->name);
1529                 goto err1;
1530         }
1531
1532         /* begin to receive SETUP packets */
1533         dwc->ep0state = EP0_SETUP_PHASE;
1534         dwc3_ep0_out_start(dwc);
1535
1536         spin_unlock_irqrestore(&dwc->lock, flags);
1537
1538         return 0;
1539
1540 err1:
1541         __dwc3_gadget_ep_disable(dwc->eps[0]);
1542
1543 err0:
1544         spin_unlock_irqrestore(&dwc->lock, flags);
1545
1546         return ret;
1547 }
1548
1549 static int dwc3_gadget_stop(struct usb_gadget *g,
1550                 struct usb_gadget_driver *driver)
1551 {
1552         struct dwc3             *dwc = gadget_to_dwc(g);
1553         unsigned long           flags;
1554
1555         spin_lock_irqsave(&dwc->lock, flags);
1556
1557         __dwc3_gadget_ep_disable(dwc->eps[0]);
1558         __dwc3_gadget_ep_disable(dwc->eps[1]);
1559
1560         dwc->gadget_driver      = NULL;
1561         dwc->gadget.dev.driver  = NULL;
1562
1563         spin_unlock_irqrestore(&dwc->lock, flags);
1564
1565         return 0;
1566 }
1567
1568 static const struct usb_gadget_ops dwc3_gadget_ops = {
1569         .get_frame              = dwc3_gadget_get_frame,
1570         .wakeup                 = dwc3_gadget_wakeup,
1571         .set_selfpowered        = dwc3_gadget_set_selfpowered,
1572         .pullup                 = dwc3_gadget_pullup,
1573         .udc_start              = dwc3_gadget_start,
1574         .udc_stop               = dwc3_gadget_stop,
1575 };
1576
1577 /* -------------------------------------------------------------------------- */
1578
1579 static int __devinit dwc3_gadget_init_endpoints(struct dwc3 *dwc)
1580 {
1581         struct dwc3_ep                  *dep;
1582         u8                              epnum;
1583
1584         INIT_LIST_HEAD(&dwc->gadget.ep_list);
1585
1586         for (epnum = 0; epnum < DWC3_ENDPOINTS_NUM; epnum++) {
1587                 dep = kzalloc(sizeof(*dep), GFP_KERNEL);
1588                 if (!dep) {
1589                         dev_err(dwc->dev, "can't allocate endpoint %d\n",
1590                                         epnum);
1591                         return -ENOMEM;
1592                 }
1593
1594                 dep->dwc = dwc;
1595                 dep->number = epnum;
1596                 dwc->eps[epnum] = dep;
1597
1598                 snprintf(dep->name, sizeof(dep->name), "ep%d%s", epnum >> 1,
1599                                 (epnum & 1) ? "in" : "out");
1600                 dep->endpoint.name = dep->name;
1601                 dep->direction = (epnum & 1);
1602
1603                 if (epnum == 0 || epnum == 1) {
1604                         dep->endpoint.maxpacket = 512;
1605                         dep->endpoint.ops = &dwc3_gadget_ep0_ops;
1606                         if (!epnum)
1607                                 dwc->gadget.ep0 = &dep->endpoint;
1608                 } else {
1609                         int             ret;
1610
1611                         dep->endpoint.maxpacket = 1024;
1612                         dep->endpoint.max_streams = 15;
1613                         dep->endpoint.ops = &dwc3_gadget_ep_ops;
1614                         list_add_tail(&dep->endpoint.ep_list,
1615                                         &dwc->gadget.ep_list);
1616
1617                         ret = dwc3_alloc_trb_pool(dep);
1618                         if (ret)
1619                                 return ret;
1620                 }
1621
1622                 INIT_LIST_HEAD(&dep->request_list);
1623                 INIT_LIST_HEAD(&dep->req_queued);
1624         }
1625
1626         return 0;
1627 }
1628
1629 static void dwc3_gadget_free_endpoints(struct dwc3 *dwc)
1630 {
1631         struct dwc3_ep                  *dep;
1632         u8                              epnum;
1633
1634         for (epnum = 0; epnum < DWC3_ENDPOINTS_NUM; epnum++) {
1635                 dep = dwc->eps[epnum];
1636                 dwc3_free_trb_pool(dep);
1637
1638                 if (epnum != 0 && epnum != 1)
1639                         list_del(&dep->endpoint.ep_list);
1640
1641                 kfree(dep);
1642         }
1643 }
1644
1645 static void dwc3_gadget_release(struct device *dev)
1646 {
1647         dev_dbg(dev, "%s\n", __func__);
1648 }
1649
1650 /* -------------------------------------------------------------------------- */
1651 static int dwc3_cleanup_done_reqs(struct dwc3 *dwc, struct dwc3_ep *dep,
1652                 const struct dwc3_event_depevt *event, int status)
1653 {
1654         struct dwc3_request     *req;
1655         struct dwc3_trb         *trb;
1656         unsigned int            count;
1657         unsigned int            s_pkt = 0;
1658         unsigned int            trb_status;
1659
1660         do {
1661                 req = next_request(&dep->req_queued);
1662                 if (!req) {
1663                         WARN_ON_ONCE(1);
1664                         return 1;
1665                 }
1666
1667                 trb = req->trb;
1668
1669                 if ((trb->ctrl & DWC3_TRB_CTRL_HWO) && status != -ESHUTDOWN)
1670                         /*
1671                          * We continue despite the error. There is not much we
1672                          * can do. If we don't clean it up we loop forever. If
1673                          * we skip the TRB then it gets overwritten after a
1674                          * while since we use them in a ring buffer. A BUG()
1675                          * would help. Lets hope that if this occurs, someone
1676                          * fixes the root cause instead of looking away :)
1677                          */
1678                         dev_err(dwc->dev, "%s's TRB (%p) still owned by HW\n",
1679                                         dep->name, req->trb);
1680                 count = trb->size & DWC3_TRB_SIZE_MASK;
1681
1682                 if (dep->direction) {
1683                         if (count) {
1684                                 trb_status = DWC3_TRB_SIZE_TRBSTS(trb->size);
1685                                 if (trb_status == DWC3_TRBSTS_MISSED_ISOC) {
1686                                         dev_dbg(dwc->dev, "incomplete IN transfer %s\n",
1687                                                         dep->name);
1688                                         dep->current_uf = event->parameters &
1689                                                 ~(dep->interval - 1);
1690                                         dep->flags |= DWC3_EP_MISSED_ISOC;
1691                                 } else {
1692                                         dev_err(dwc->dev, "incomplete IN transfer %s\n",
1693                                                         dep->name);
1694                                         status = -ECONNRESET;
1695                                 }
1696                         }
1697                 } else {
1698                         if (count && (event->status & DEPEVT_STATUS_SHORT))
1699                                 s_pkt = 1;
1700                 }
1701
1702                 /*
1703                  * We assume here we will always receive the entire data block
1704                  * which we should receive. Meaning, if we program RX to
1705                  * receive 4K but we receive only 2K, we assume that's all we
1706                  * should receive and we simply bounce the request back to the
1707                  * gadget driver for further processing.
1708                  */
1709                 req->request.actual += req->request.length - count;
1710                 dwc3_gadget_giveback(dep, req, status);
1711                 if (s_pkt)
1712                         break;
1713                 if ((event->status & DEPEVT_STATUS_LST) &&
1714                                 (trb->ctrl & (DWC3_TRB_CTRL_LST |
1715                                                 DWC3_TRB_CTRL_HWO)))
1716                         break;
1717                 if ((event->status & DEPEVT_STATUS_IOC) &&
1718                                 (trb->ctrl & DWC3_TRB_CTRL_IOC))
1719                         break;
1720         } while (1);
1721
1722         if ((event->status & DEPEVT_STATUS_IOC) &&
1723                         (trb->ctrl & DWC3_TRB_CTRL_IOC))
1724                 return 0;
1725         return 1;
1726 }
1727
1728 static void dwc3_endpoint_transfer_complete(struct dwc3 *dwc,
1729                 struct dwc3_ep *dep, const struct dwc3_event_depevt *event,
1730                 int start_new)
1731 {
1732         unsigned                status = 0;
1733         int                     clean_busy;
1734
1735         if (event->status & DEPEVT_STATUS_BUSERR)
1736                 status = -ECONNRESET;
1737
1738         clean_busy = dwc3_cleanup_done_reqs(dwc, dep, event, status);
1739         if (clean_busy)
1740                 dep->flags &= ~DWC3_EP_BUSY;
1741
1742         /*
1743          * WORKAROUND: This is the 2nd half of U1/U2 -> U0 workaround.
1744          * See dwc3_gadget_linksts_change_interrupt() for 1st half.
1745          */
1746         if (dwc->revision < DWC3_REVISION_183A) {
1747                 u32             reg;
1748                 int             i;
1749
1750                 for (i = 0; i < DWC3_ENDPOINTS_NUM; i++) {
1751                         dep = dwc->eps[i];
1752
1753                         if (!(dep->flags & DWC3_EP_ENABLED))
1754                                 continue;
1755
1756                         if (!list_empty(&dep->req_queued))
1757                                 return;
1758                 }
1759
1760                 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
1761                 reg |= dwc->u1u2;
1762                 dwc3_writel(dwc->regs, DWC3_DCTL, reg);
1763
1764                 dwc->u1u2 = 0;
1765         }
1766 }
1767
1768 static void dwc3_endpoint_interrupt(struct dwc3 *dwc,
1769                 const struct dwc3_event_depevt *event)
1770 {
1771         struct dwc3_ep          *dep;
1772         u8                      epnum = event->endpoint_number;
1773
1774         dep = dwc->eps[epnum];
1775
1776         if (!(dep->flags & DWC3_EP_ENABLED))
1777                 return;
1778
1779         dev_vdbg(dwc->dev, "%s: %s\n", dep->name,
1780                         dwc3_ep_event_string(event->endpoint_event));
1781
1782         if (epnum == 0 || epnum == 1) {
1783                 dwc3_ep0_interrupt(dwc, event);
1784                 return;
1785         }
1786
1787         switch (event->endpoint_event) {
1788         case DWC3_DEPEVT_XFERCOMPLETE:
1789                 dep->resource_index = 0;
1790
1791                 if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) {
1792                         dev_dbg(dwc->dev, "%s is an Isochronous endpoint\n",
1793                                         dep->name);
1794                         return;
1795                 }
1796
1797                 dwc3_endpoint_transfer_complete(dwc, dep, event, 1);
1798                 break;
1799         case DWC3_DEPEVT_XFERINPROGRESS:
1800                 if (!usb_endpoint_xfer_isoc(dep->endpoint.desc)) {
1801                         dev_dbg(dwc->dev, "%s is not an Isochronous endpoint\n",
1802                                         dep->name);
1803                         return;
1804                 }
1805
1806                 dwc3_endpoint_transfer_complete(dwc, dep, event, 0);
1807                 break;
1808         case DWC3_DEPEVT_XFERNOTREADY:
1809                 if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) {
1810                         dwc3_gadget_start_isoc(dwc, dep, event);
1811                 } else {
1812                         int ret;
1813
1814                         dev_vdbg(dwc->dev, "%s: reason %s\n",
1815                                         dep->name, event->status &
1816                                         DEPEVT_STATUS_TRANSFER_ACTIVE
1817                                         ? "Transfer Active"
1818                                         : "Transfer Not Active");
1819
1820                         ret = __dwc3_gadget_kick_transfer(dep, 0, 1);
1821                         if (!ret || ret == -EBUSY)
1822                                 return;
1823
1824                         dev_dbg(dwc->dev, "%s: failed to kick transfers\n",
1825                                         dep->name);
1826                 }
1827
1828                 break;
1829         case DWC3_DEPEVT_STREAMEVT:
1830                 if (!usb_endpoint_xfer_bulk(dep->endpoint.desc)) {
1831                         dev_err(dwc->dev, "Stream event for non-Bulk %s\n",
1832                                         dep->name);
1833                         return;
1834                 }
1835
1836                 switch (event->status) {
1837                 case DEPEVT_STREAMEVT_FOUND:
1838                         dev_vdbg(dwc->dev, "Stream %d found and started\n",
1839                                         event->parameters);
1840
1841                         break;
1842                 case DEPEVT_STREAMEVT_NOTFOUND:
1843                         /* FALLTHROUGH */
1844                 default:
1845                         dev_dbg(dwc->dev, "Couldn't find suitable stream\n");
1846                 }
1847                 break;
1848         case DWC3_DEPEVT_RXTXFIFOEVT:
1849                 dev_dbg(dwc->dev, "%s FIFO Overrun\n", dep->name);
1850                 break;
1851         case DWC3_DEPEVT_EPCMDCMPLT:
1852                 dev_vdbg(dwc->dev, "Endpoint Command Complete\n");
1853                 break;
1854         }
1855 }
1856
1857 static void dwc3_disconnect_gadget(struct dwc3 *dwc)
1858 {
1859         if (dwc->gadget_driver && dwc->gadget_driver->disconnect) {
1860                 spin_unlock(&dwc->lock);
1861                 dwc->gadget_driver->disconnect(&dwc->gadget);
1862                 spin_lock(&dwc->lock);
1863         }
1864 }
1865
1866 static void dwc3_stop_active_transfer(struct dwc3 *dwc, u32 epnum)
1867 {
1868         struct dwc3_ep *dep;
1869         struct dwc3_gadget_ep_cmd_params params;
1870         u32 cmd;
1871         int ret;
1872
1873         dep = dwc->eps[epnum];
1874
1875         if (!dep->resource_index)
1876                 return;
1877
1878         cmd = DWC3_DEPCMD_ENDTRANSFER;
1879         cmd |= DWC3_DEPCMD_HIPRI_FORCERM | DWC3_DEPCMD_CMDIOC;
1880         cmd |= DWC3_DEPCMD_PARAM(dep->resource_index);
1881         memset(&params, 0, sizeof(params));
1882         ret = dwc3_send_gadget_ep_cmd(dwc, dep->number, cmd, &params);
1883         WARN_ON_ONCE(ret);
1884         dep->resource_index = 0;
1885 }
1886
1887 static void dwc3_stop_active_transfers(struct dwc3 *dwc)
1888 {
1889         u32 epnum;
1890
1891         for (epnum = 2; epnum < DWC3_ENDPOINTS_NUM; epnum++) {
1892                 struct dwc3_ep *dep;
1893
1894                 dep = dwc->eps[epnum];
1895                 if (!(dep->flags & DWC3_EP_ENABLED))
1896                         continue;
1897
1898                 dwc3_remove_requests(dwc, dep);
1899         }
1900 }
1901
1902 static void dwc3_clear_stall_all_ep(struct dwc3 *dwc)
1903 {
1904         u32 epnum;
1905
1906         for (epnum = 1; epnum < DWC3_ENDPOINTS_NUM; epnum++) {
1907                 struct dwc3_ep *dep;
1908                 struct dwc3_gadget_ep_cmd_params params;
1909                 int ret;
1910
1911                 dep = dwc->eps[epnum];
1912
1913                 if (!(dep->flags & DWC3_EP_STALL))
1914                         continue;
1915
1916                 dep->flags &= ~DWC3_EP_STALL;
1917
1918                 memset(&params, 0, sizeof(params));
1919                 ret = dwc3_send_gadget_ep_cmd(dwc, dep->number,
1920                                 DWC3_DEPCMD_CLEARSTALL, &params);
1921                 WARN_ON_ONCE(ret);
1922         }
1923 }
1924
1925 static void dwc3_gadget_disconnect_interrupt(struct dwc3 *dwc)
1926 {
1927         int                     reg;
1928
1929         dev_vdbg(dwc->dev, "%s\n", __func__);
1930
1931         reg = dwc3_readl(dwc->regs, DWC3_DCTL);
1932         reg &= ~DWC3_DCTL_INITU1ENA;
1933         dwc3_writel(dwc->regs, DWC3_DCTL, reg);
1934
1935         reg &= ~DWC3_DCTL_INITU2ENA;
1936         dwc3_writel(dwc->regs, DWC3_DCTL, reg);
1937
1938         dwc3_disconnect_gadget(dwc);
1939         dwc->start_config_issued = false;
1940
1941         dwc->gadget.speed = USB_SPEED_UNKNOWN;
1942         dwc->setup_packet_pending = false;
1943 }
1944
1945 static void dwc3_gadget_usb3_phy_suspend(struct dwc3 *dwc, int suspend)
1946 {
1947         u32                     reg;
1948
1949         reg = dwc3_readl(dwc->regs, DWC3_GUSB3PIPECTL(0));
1950
1951         if (suspend)
1952                 reg |= DWC3_GUSB3PIPECTL_SUSPHY;
1953         else
1954                 reg &= ~DWC3_GUSB3PIPECTL_SUSPHY;
1955
1956         dwc3_writel(dwc->regs, DWC3_GUSB3PIPECTL(0), reg);
1957 }
1958
1959 static void dwc3_gadget_usb2_phy_suspend(struct dwc3 *dwc, int suspend)
1960 {
1961         u32                     reg;
1962
1963         reg = dwc3_readl(dwc->regs, DWC3_GUSB2PHYCFG(0));
1964
1965         if (suspend)
1966                 reg |= DWC3_GUSB2PHYCFG_SUSPHY;
1967         else
1968                 reg &= ~DWC3_GUSB2PHYCFG_SUSPHY;
1969
1970         dwc3_writel(dwc->regs, DWC3_GUSB2PHYCFG(0), reg);
1971 }
1972
1973 static void dwc3_gadget_reset_interrupt(struct dwc3 *dwc)
1974 {
1975         u32                     reg;
1976
1977         dev_vdbg(dwc->dev, "%s\n", __func__);
1978
1979         /*
1980          * WORKAROUND: DWC3 revisions <1.88a have an issue which
1981          * would cause a missing Disconnect Event if there's a
1982          * pending Setup Packet in the FIFO.
1983          *
1984          * There's no suggested workaround on the official Bug
1985          * report, which states that "unless the driver/application
1986          * is doing any special handling of a disconnect event,
1987          * there is no functional issue".
1988          *
1989          * Unfortunately, it turns out that we _do_ some special
1990          * handling of a disconnect event, namely complete all
1991          * pending transfers, notify gadget driver of the
1992          * disconnection, and so on.
1993          *
1994          * Our suggested workaround is to follow the Disconnect
1995          * Event steps here, instead, based on a setup_packet_pending
1996          * flag. Such flag gets set whenever we have a XferNotReady
1997          * event on EP0 and gets cleared on XferComplete for the
1998          * same endpoint.
1999          *
2000          * Refers to:
2001          *
2002          * STAR#9000466709: RTL: Device : Disconnect event not
2003          * generated if setup packet pending in FIFO
2004          */
2005         if (dwc->revision < DWC3_REVISION_188A) {
2006                 if (dwc->setup_packet_pending)
2007                         dwc3_gadget_disconnect_interrupt(dwc);
2008         }
2009
2010         /* after reset -> Default State */
2011         dwc->dev_state = DWC3_DEFAULT_STATE;
2012
2013         /* Recent versions support automatic phy suspend and don't need this */
2014         if (dwc->revision < DWC3_REVISION_194A) {
2015                 /* Resume PHYs */
2016                 dwc3_gadget_usb2_phy_suspend(dwc, false);
2017                 dwc3_gadget_usb3_phy_suspend(dwc, false);
2018         }
2019
2020         if (dwc->gadget.speed != USB_SPEED_UNKNOWN)
2021                 dwc3_disconnect_gadget(dwc);
2022
2023         reg = dwc3_readl(dwc->regs, DWC3_DCTL);
2024         reg &= ~DWC3_DCTL_TSTCTRL_MASK;
2025         dwc3_writel(dwc->regs, DWC3_DCTL, reg);
2026         dwc->test_mode = false;
2027
2028         dwc3_stop_active_transfers(dwc);
2029         dwc3_clear_stall_all_ep(dwc);
2030         dwc->start_config_issued = false;
2031
2032         /* Reset device address to zero */
2033         reg = dwc3_readl(dwc->regs, DWC3_DCFG);
2034         reg &= ~(DWC3_DCFG_DEVADDR_MASK);
2035         dwc3_writel(dwc->regs, DWC3_DCFG, reg);
2036 }
2037
2038 static void dwc3_update_ram_clk_sel(struct dwc3 *dwc, u32 speed)
2039 {
2040         u32 reg;
2041         u32 usb30_clock = DWC3_GCTL_CLK_BUS;
2042
2043         /*
2044          * We change the clock only at SS but I dunno why I would want to do
2045          * this. Maybe it becomes part of the power saving plan.
2046          */
2047
2048         if (speed != DWC3_DSTS_SUPERSPEED)
2049                 return;
2050
2051         /*
2052          * RAMClkSel is reset to 0 after USB reset, so it must be reprogrammed
2053          * each time on Connect Done.
2054          */
2055         if (!usb30_clock)
2056                 return;
2057
2058         reg = dwc3_readl(dwc->regs, DWC3_GCTL);
2059         reg |= DWC3_GCTL_RAMCLKSEL(usb30_clock);
2060         dwc3_writel(dwc->regs, DWC3_GCTL, reg);
2061 }
2062
2063 static void dwc3_gadget_phy_suspend(struct dwc3 *dwc, u8 speed)
2064 {
2065         switch (speed) {
2066         case USB_SPEED_SUPER:
2067                 dwc3_gadget_usb2_phy_suspend(dwc, true);
2068                 break;
2069         case USB_SPEED_HIGH:
2070         case USB_SPEED_FULL:
2071         case USB_SPEED_LOW:
2072                 dwc3_gadget_usb3_phy_suspend(dwc, true);
2073                 break;
2074         }
2075 }
2076
2077 static void dwc3_gadget_conndone_interrupt(struct dwc3 *dwc)
2078 {
2079         struct dwc3_gadget_ep_cmd_params params;
2080         struct dwc3_ep          *dep;
2081         int                     ret;
2082         u32                     reg;
2083         u8                      speed;
2084
2085         dev_vdbg(dwc->dev, "%s\n", __func__);
2086
2087         memset(&params, 0x00, sizeof(params));
2088
2089         reg = dwc3_readl(dwc->regs, DWC3_DSTS);
2090         speed = reg & DWC3_DSTS_CONNECTSPD;
2091         dwc->speed = speed;
2092
2093         dwc3_update_ram_clk_sel(dwc, speed);
2094
2095         switch (speed) {
2096         case DWC3_DCFG_SUPERSPEED:
2097                 /*
2098                  * WORKAROUND: DWC3 revisions <1.90a have an issue which
2099                  * would cause a missing USB3 Reset event.
2100                  *
2101                  * In such situations, we should force a USB3 Reset
2102                  * event by calling our dwc3_gadget_reset_interrupt()
2103                  * routine.
2104                  *
2105                  * Refers to:
2106                  *
2107                  * STAR#9000483510: RTL: SS : USB3 reset event may
2108                  * not be generated always when the link enters poll
2109                  */
2110                 if (dwc->revision < DWC3_REVISION_190A)
2111                         dwc3_gadget_reset_interrupt(dwc);
2112
2113                 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(512);
2114                 dwc->gadget.ep0->maxpacket = 512;
2115                 dwc->gadget.speed = USB_SPEED_SUPER;
2116                 break;
2117         case DWC3_DCFG_HIGHSPEED:
2118                 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(64);
2119                 dwc->gadget.ep0->maxpacket = 64;
2120                 dwc->gadget.speed = USB_SPEED_HIGH;
2121                 break;
2122         case DWC3_DCFG_FULLSPEED2:
2123         case DWC3_DCFG_FULLSPEED1:
2124                 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(64);
2125                 dwc->gadget.ep0->maxpacket = 64;
2126                 dwc->gadget.speed = USB_SPEED_FULL;
2127                 break;
2128         case DWC3_DCFG_LOWSPEED:
2129                 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(8);
2130                 dwc->gadget.ep0->maxpacket = 8;
2131                 dwc->gadget.speed = USB_SPEED_LOW;
2132                 break;
2133         }
2134
2135         /* Recent versions support automatic phy suspend and don't need this */
2136         if (dwc->revision < DWC3_REVISION_194A) {
2137                 /* Suspend unneeded PHY */
2138                 dwc3_gadget_phy_suspend(dwc, dwc->gadget.speed);
2139         }
2140
2141         dep = dwc->eps[0];
2142         ret = __dwc3_gadget_ep_enable(dep, &dwc3_gadget_ep0_desc, NULL, true);
2143         if (ret) {
2144                 dev_err(dwc->dev, "failed to enable %s\n", dep->name);
2145                 return;
2146         }
2147
2148         dep = dwc->eps[1];
2149         ret = __dwc3_gadget_ep_enable(dep, &dwc3_gadget_ep0_desc, NULL, true);
2150         if (ret) {
2151                 dev_err(dwc->dev, "failed to enable %s\n", dep->name);
2152                 return;
2153         }
2154
2155         /*
2156          * Configure PHY via GUSB3PIPECTLn if required.
2157          *
2158          * Update GTXFIFOSIZn
2159          *
2160          * In both cases reset values should be sufficient.
2161          */
2162 }
2163
2164 static void dwc3_gadget_wakeup_interrupt(struct dwc3 *dwc)
2165 {
2166         dev_vdbg(dwc->dev, "%s\n", __func__);
2167
2168         /*
2169          * TODO take core out of low power mode when that's
2170          * implemented.
2171          */
2172
2173         dwc->gadget_driver->resume(&dwc->gadget);
2174 }
2175
2176 static void dwc3_gadget_linksts_change_interrupt(struct dwc3 *dwc,
2177                 unsigned int evtinfo)
2178 {
2179         enum dwc3_link_state    next = evtinfo & DWC3_LINK_STATE_MASK;
2180
2181         /*
2182          * WORKAROUND: DWC3 Revisions <1.83a have an issue which, depending
2183          * on the link partner, the USB session might do multiple entry/exit
2184          * of low power states before a transfer takes place.
2185          *
2186          * Due to this problem, we might experience lower throughput. The
2187          * suggested workaround is to disable DCTL[12:9] bits if we're
2188          * transitioning from U1/U2 to U0 and enable those bits again
2189          * after a transfer completes and there are no pending transfers
2190          * on any of the enabled endpoints.
2191          *
2192          * This is the first half of that workaround.
2193          *
2194          * Refers to:
2195          *
2196          * STAR#9000446952: RTL: Device SS : if U1/U2 ->U0 takes >128us
2197          * core send LGO_Ux entering U0
2198          */
2199         if (dwc->revision < DWC3_REVISION_183A) {
2200                 if (next == DWC3_LINK_STATE_U0) {
2201                         u32     u1u2;
2202                         u32     reg;
2203
2204                         switch (dwc->link_state) {
2205                         case DWC3_LINK_STATE_U1:
2206                         case DWC3_LINK_STATE_U2:
2207                                 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
2208                                 u1u2 = reg & (DWC3_DCTL_INITU2ENA
2209                                                 | DWC3_DCTL_ACCEPTU2ENA
2210                                                 | DWC3_DCTL_INITU1ENA
2211                                                 | DWC3_DCTL_ACCEPTU1ENA);
2212
2213                                 if (!dwc->u1u2)
2214                                         dwc->u1u2 = reg & u1u2;
2215
2216                                 reg &= ~u1u2;
2217
2218                                 dwc3_writel(dwc->regs, DWC3_DCTL, reg);
2219                                 break;
2220                         default:
2221                                 /* do nothing */
2222                                 break;
2223                         }
2224                 }
2225         }
2226
2227         dwc->link_state = next;
2228
2229         dev_vdbg(dwc->dev, "%s link %d\n", __func__, dwc->link_state);
2230 }
2231
2232 static void dwc3_gadget_interrupt(struct dwc3 *dwc,
2233                 const struct dwc3_event_devt *event)
2234 {
2235         switch (event->type) {
2236         case DWC3_DEVICE_EVENT_DISCONNECT:
2237                 dwc3_gadget_disconnect_interrupt(dwc);
2238                 break;
2239         case DWC3_DEVICE_EVENT_RESET:
2240                 dwc3_gadget_reset_interrupt(dwc);
2241                 break;
2242         case DWC3_DEVICE_EVENT_CONNECT_DONE:
2243                 dwc3_gadget_conndone_interrupt(dwc);
2244                 break;
2245         case DWC3_DEVICE_EVENT_WAKEUP:
2246                 dwc3_gadget_wakeup_interrupt(dwc);
2247                 break;
2248         case DWC3_DEVICE_EVENT_LINK_STATUS_CHANGE:
2249                 dwc3_gadget_linksts_change_interrupt(dwc, event->event_info);
2250                 break;
2251         case DWC3_DEVICE_EVENT_EOPF:
2252                 dev_vdbg(dwc->dev, "End of Periodic Frame\n");
2253                 break;
2254         case DWC3_DEVICE_EVENT_SOF:
2255                 dev_vdbg(dwc->dev, "Start of Periodic Frame\n");
2256                 break;
2257         case DWC3_DEVICE_EVENT_ERRATIC_ERROR:
2258                 dev_vdbg(dwc->dev, "Erratic Error\n");
2259                 break;
2260         case DWC3_DEVICE_EVENT_CMD_CMPL:
2261                 dev_vdbg(dwc->dev, "Command Complete\n");
2262                 break;
2263         case DWC3_DEVICE_EVENT_OVERFLOW:
2264                 dev_vdbg(dwc->dev, "Overflow\n");
2265                 break;
2266         default:
2267                 dev_dbg(dwc->dev, "UNKNOWN IRQ %d\n", event->type);
2268         }
2269 }
2270
2271 static void dwc3_process_event_entry(struct dwc3 *dwc,
2272                 const union dwc3_event *event)
2273 {
2274         /* Endpoint IRQ, handle it and return early */
2275         if (event->type.is_devspec == 0) {
2276                 /* depevt */
2277                 return dwc3_endpoint_interrupt(dwc, &event->depevt);
2278         }
2279
2280         switch (event->type.type) {
2281         case DWC3_EVENT_TYPE_DEV:
2282                 dwc3_gadget_interrupt(dwc, &event->devt);
2283                 break;
2284         /* REVISIT what to do with Carkit and I2C events ? */
2285         default:
2286                 dev_err(dwc->dev, "UNKNOWN IRQ type %d\n", event->raw);
2287         }
2288 }
2289
2290 static irqreturn_t dwc3_process_event_buf(struct dwc3 *dwc, u32 buf)
2291 {
2292         struct dwc3_event_buffer *evt;
2293         int left;
2294         u32 count;
2295
2296         count = dwc3_readl(dwc->regs, DWC3_GEVNTCOUNT(buf));
2297         count &= DWC3_GEVNTCOUNT_MASK;
2298         if (!count)
2299                 return IRQ_NONE;
2300
2301         evt = dwc->ev_buffs[buf];
2302         left = count;
2303
2304         while (left > 0) {
2305                 union dwc3_event event;
2306
2307                 event.raw = *(u32 *) (evt->buf + evt->lpos);
2308
2309                 dwc3_process_event_entry(dwc, &event);
2310                 /*
2311                  * XXX we wrap around correctly to the next entry as almost all
2312                  * entries are 4 bytes in size. There is one entry which has 12
2313                  * bytes which is a regular entry followed by 8 bytes data. ATM
2314                  * I don't know how things are organized if were get next to the
2315                  * a boundary so I worry about that once we try to handle that.
2316                  */
2317                 evt->lpos = (evt->lpos + 4) % DWC3_EVENT_BUFFERS_SIZE;
2318                 left -= 4;
2319
2320                 dwc3_writel(dwc->regs, DWC3_GEVNTCOUNT(buf), 4);
2321         }
2322
2323         return IRQ_HANDLED;
2324 }
2325
2326 static irqreturn_t dwc3_interrupt(int irq, void *_dwc)
2327 {
2328         struct dwc3                     *dwc = _dwc;
2329         int                             i;
2330         irqreturn_t                     ret = IRQ_NONE;
2331
2332         spin_lock(&dwc->lock);
2333
2334         for (i = 0; i < dwc->num_event_buffers; i++) {
2335                 irqreturn_t status;
2336
2337                 status = dwc3_process_event_buf(dwc, i);
2338                 if (status == IRQ_HANDLED)
2339                         ret = status;
2340         }
2341
2342         spin_unlock(&dwc->lock);
2343
2344         return ret;
2345 }
2346
2347 /**
2348  * dwc3_gadget_init - Initializes gadget related registers
2349  * @dwc: pointer to our controller context structure
2350  *
2351  * Returns 0 on success otherwise negative errno.
2352  */
2353 int __devinit dwc3_gadget_init(struct dwc3 *dwc)
2354 {
2355         u32                                     reg;
2356         int                                     ret;
2357         int                                     irq;
2358
2359         dwc->ctrl_req = dma_alloc_coherent(dwc->dev, sizeof(*dwc->ctrl_req),
2360                         &dwc->ctrl_req_addr, GFP_KERNEL);
2361         if (!dwc->ctrl_req) {
2362                 dev_err(dwc->dev, "failed to allocate ctrl request\n");
2363                 ret = -ENOMEM;
2364                 goto err0;
2365         }
2366
2367         dwc->ep0_trb = dma_alloc_coherent(dwc->dev, sizeof(*dwc->ep0_trb),
2368                         &dwc->ep0_trb_addr, GFP_KERNEL);
2369         if (!dwc->ep0_trb) {
2370                 dev_err(dwc->dev, "failed to allocate ep0 trb\n");
2371                 ret = -ENOMEM;
2372                 goto err1;
2373         }
2374
2375         dwc->setup_buf = kzalloc(DWC3_EP0_BOUNCE_SIZE, GFP_KERNEL);
2376         if (!dwc->setup_buf) {
2377                 dev_err(dwc->dev, "failed to allocate setup buffer\n");
2378                 ret = -ENOMEM;
2379                 goto err2;
2380         }
2381
2382         dwc->ep0_bounce = dma_alloc_coherent(dwc->dev,
2383                         DWC3_EP0_BOUNCE_SIZE, &dwc->ep0_bounce_addr,
2384                         GFP_KERNEL);
2385         if (!dwc->ep0_bounce) {
2386                 dev_err(dwc->dev, "failed to allocate ep0 bounce buffer\n");
2387                 ret = -ENOMEM;
2388                 goto err3;
2389         }
2390
2391         dev_set_name(&dwc->gadget.dev, "gadget");
2392
2393         dwc->gadget.ops                 = &dwc3_gadget_ops;
2394         dwc->gadget.max_speed           = USB_SPEED_SUPER;
2395         dwc->gadget.speed               = USB_SPEED_UNKNOWN;
2396         dwc->gadget.dev.parent          = dwc->dev;
2397         dwc->gadget.sg_supported        = true;
2398
2399         dma_set_coherent_mask(&dwc->gadget.dev, dwc->dev->coherent_dma_mask);
2400
2401         dwc->gadget.dev.dma_parms       = dwc->dev->dma_parms;
2402         dwc->gadget.dev.dma_mask        = dwc->dev->dma_mask;
2403         dwc->gadget.dev.release         = dwc3_gadget_release;
2404         dwc->gadget.name                = "dwc3-gadget";
2405
2406         /*
2407          * REVISIT: Here we should clear all pending IRQs to be
2408          * sure we're starting from a well known location.
2409          */
2410
2411         ret = dwc3_gadget_init_endpoints(dwc);
2412         if (ret)
2413                 goto err4;
2414
2415         irq = platform_get_irq(to_platform_device(dwc->dev), 0);
2416
2417         ret = request_irq(irq, dwc3_interrupt, IRQF_SHARED,
2418                         "dwc3", dwc);
2419         if (ret) {
2420                 dev_err(dwc->dev, "failed to request irq #%d --> %d\n",
2421                                 irq, ret);
2422                 goto err5;
2423         }
2424
2425         reg = dwc3_readl(dwc->regs, DWC3_DCFG);
2426         reg |= DWC3_DCFG_LPM_CAP;
2427         dwc3_writel(dwc->regs, DWC3_DCFG, reg);
2428
2429         /* Enable all but Start and End of Frame IRQs */
2430         reg = (DWC3_DEVTEN_VNDRDEVTSTRCVEDEN |
2431                         DWC3_DEVTEN_EVNTOVERFLOWEN |
2432                         DWC3_DEVTEN_CMDCMPLTEN |
2433                         DWC3_DEVTEN_ERRTICERREN |
2434                         DWC3_DEVTEN_WKUPEVTEN |
2435                         DWC3_DEVTEN_ULSTCNGEN |
2436                         DWC3_DEVTEN_CONNECTDONEEN |
2437                         DWC3_DEVTEN_USBRSTEN |
2438                         DWC3_DEVTEN_DISCONNEVTEN);
2439         dwc3_writel(dwc->regs, DWC3_DEVTEN, reg);
2440
2441         /* Enable USB2 LPM and automatic phy suspend only on recent versions */
2442         if (dwc->revision >= DWC3_REVISION_194A) {
2443                 reg = dwc3_readl(dwc->regs, DWC3_DCFG);
2444                 reg |= DWC3_DCFG_LPM_CAP;
2445                 dwc3_writel(dwc->regs, DWC3_DCFG, reg);
2446
2447                 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
2448                 reg &= ~(DWC3_DCTL_HIRD_THRES_MASK | DWC3_DCTL_L1_HIBER_EN);
2449
2450                 /* TODO: This should be configurable */
2451                 reg |= DWC3_DCTL_HIRD_THRES(28);
2452
2453                 dwc3_writel(dwc->regs, DWC3_DCTL, reg);
2454
2455                 dwc3_gadget_usb2_phy_suspend(dwc, false);
2456                 dwc3_gadget_usb3_phy_suspend(dwc, false);
2457         }
2458
2459         ret = device_register(&dwc->gadget.dev);
2460         if (ret) {
2461                 dev_err(dwc->dev, "failed to register gadget device\n");
2462                 put_device(&dwc->gadget.dev);
2463                 goto err6;
2464         }
2465
2466         ret = usb_add_gadget_udc(dwc->dev, &dwc->gadget);
2467         if (ret) {
2468                 dev_err(dwc->dev, "failed to register udc\n");
2469                 goto err7;
2470         }
2471
2472         return 0;
2473
2474 err7:
2475         device_unregister(&dwc->gadget.dev);
2476
2477 err6:
2478         dwc3_writel(dwc->regs, DWC3_DEVTEN, 0x00);
2479         free_irq(irq, dwc);
2480
2481 err5:
2482         dwc3_gadget_free_endpoints(dwc);
2483
2484 err4:
2485         dma_free_coherent(dwc->dev, DWC3_EP0_BOUNCE_SIZE,
2486                         dwc->ep0_bounce, dwc->ep0_bounce_addr);
2487
2488 err3:
2489         kfree(dwc->setup_buf);
2490
2491 err2:
2492         dma_free_coherent(dwc->dev, sizeof(*dwc->ep0_trb),
2493                         dwc->ep0_trb, dwc->ep0_trb_addr);
2494
2495 err1:
2496         dma_free_coherent(dwc->dev, sizeof(*dwc->ctrl_req),
2497                         dwc->ctrl_req, dwc->ctrl_req_addr);
2498
2499 err0:
2500         return ret;
2501 }
2502
2503 void dwc3_gadget_exit(struct dwc3 *dwc)
2504 {
2505         int                     irq;
2506
2507         usb_del_gadget_udc(&dwc->gadget);
2508         irq = platform_get_irq(to_platform_device(dwc->dev), 0);
2509
2510         dwc3_writel(dwc->regs, DWC3_DEVTEN, 0x00);
2511         free_irq(irq, dwc);
2512
2513         dwc3_gadget_free_endpoints(dwc);
2514
2515         dma_free_coherent(dwc->dev, DWC3_EP0_BOUNCE_SIZE,
2516                         dwc->ep0_bounce, dwc->ep0_bounce_addr);
2517
2518         kfree(dwc->setup_buf);
2519
2520         dma_free_coherent(dwc->dev, sizeof(*dwc->ep0_trb),
2521                         dwc->ep0_trb, dwc->ep0_trb_addr);
2522
2523         dma_free_coherent(dwc->dev, sizeof(*dwc->ctrl_req),
2524                         dwc->ctrl_req, dwc->ctrl_req_addr);
2525
2526         device_unregister(&dwc->gadget.dev);
2527 }