drivers/usb/dwc3/gadget.c

   1 /**
   2  * gadget.c - DesignWare USB3 DRD Controller Gadget Framework Link
   3  *
   4  * Copyright (C) 2010-2011 Texas Instruments Incorporated - http://www.ti.com
   5  *
   6  * Authors: Felipe Balbi <balbi@ti.com>,
   7  *          Sebastian Andrzej Siewior <bigeasy@linutronix.de>
   8  *
   9  * Redistribution and use in source and binary forms, with or without
  10  * modification, are permitted provided that the following conditions
  11  * are met:
  12  * 1. Redistributions of source code must retain the above copyright
  13  *    notice, this list of conditions, and the following disclaimer,
  14  *    without modification.
  15  * 2. Redistributions in binary form must reproduce the above copyright
  16  *    notice, this list of conditions and the following disclaimer in the
  17  *    documentation and/or other materials provided with the distribution.
  18  * 3. The names of the above-listed copyright holders may not be used
  19  *    to endorse or promote products derived from this software without
  20  *    specific prior written permission.
  21  *
  22  * ALTERNATIVELY, this software may be distributed under the terms of the
  23  * GNU General Public License ("GPL") version 2, as published by the Free
  24  * Software Foundation.
  25  *
  26  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
  27  * IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
  28  * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  29  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
  30  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
  31  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
  32  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
  33  * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
  34  * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
  35  * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
  36  * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  37  */
  38
  39 #include <linux/kernel.h>
  40 #include <linux/delay.h>
  41 #include <linux/slab.h>
  42 #include <linux/spinlock.h>
  43 #include <linux/platform_device.h>
  44 #include <linux/pm_runtime.h>
  45 #include <linux/interrupt.h>
  46 #include <linux/io.h>
  47 #include <linux/list.h>
  48 #include <linux/dma-mapping.h>
  49
  50 #include <linux/usb/ch9.h>
  51 #include <linux/usb/gadget.h>
  52
  53 #include "core.h"
  54 #include "gadget.h"
  55 #include "io.h"
  56
  57 #define DMA_ADDR_INVALID        (~(dma_addr_t)0)
  58
  59 void dwc3_map_buffer_to_dma(struct dwc3_request *req)
  60 {
  61         struct dwc3                     *dwc = req->dep->dwc;
  62
  63         if (req->request.length == 0) {
  64                 /* req->request.dma = dwc->setup_buf_addr; */
  65                 return;
  66         }
  67
  68         if (req->request.dma == DMA_ADDR_INVALID) {
  69                 req->request.dma = dma_map_single(dwc->dev, req->request.buf,
  70                                 req->request.length, req->direction
  71                                 ? DMA_TO_DEVICE : DMA_FROM_DEVICE);
  72                 req->mapped = true;
  73         }
  74 }
  75
  76 void dwc3_unmap_buffer_from_dma(struct dwc3_request *req)
  77 {
  78         struct dwc3                     *dwc = req->dep->dwc;
  79
  80         if (req->request.length == 0) {
  81                 req->request.dma = DMA_ADDR_INVALID;
  82                 return;
  83         }
  84
  85         if (req->mapped) {
  86                 dma_unmap_single(dwc->dev, req->request.dma,
  87                                 req->request.length, req->direction
  88                                 ? DMA_TO_DEVICE : DMA_FROM_DEVICE);
  89                 req->mapped = 0;
  90                 req->request.dma = DMA_ADDR_INVALID;
  91         }
  92 }
  93
  94 void dwc3_gadget_giveback(struct dwc3_ep *dep, struct dwc3_request *req,
  95                 int status)
  96 {
  97         struct dwc3                     *dwc = dep->dwc;
  98
  99         if (req->queued) {
 100                 dep->busy_slot++;
 101                 /*
 102                  * Skip LINK TRB. We can't use req->trb and check for
 103                  * DWC3_TRBCTL_LINK_TRB because it points the TRB we just
 104                  * completed (not the LINK TRB).
 105                  */
 106                 if (((dep->busy_slot & DWC3_TRB_MASK) == DWC3_TRB_NUM - 1) &&
 107                                 usb_endpoint_xfer_isoc(dep->desc))
 108                         dep->busy_slot++;
 109         }
 110         list_del(&req->list);
 111
 112         if (req->request.status == -EINPROGRESS)
 113                 req->request.status = status;
 114
 115         dwc3_unmap_buffer_from_dma(req);
 116
 117         dev_dbg(dwc->dev, "request %p from %s completed %d/%d ===> %d\n",
 118                         req, dep->name, req->request.actual,
 119                         req->request.length, status);
 120
 121         spin_unlock(&dwc->lock);
 122         req->request.complete(&req->dep->endpoint, &req->request);
 123         spin_lock(&dwc->lock);
 124 }
 125
 126 static const char *dwc3_gadget_ep_cmd_string(u8 cmd)
 127 {
 128         switch (cmd) {
 129         case DWC3_DEPCMD_DEPSTARTCFG:
 130                 return "Start New Configuration";
 131         case DWC3_DEPCMD_ENDTRANSFER:
 132                 return "End Transfer";
 133         case DWC3_DEPCMD_UPDATETRANSFER:
 134                 return "Update Transfer";
 135         case DWC3_DEPCMD_STARTTRANSFER:
 136                 return "Start Transfer";
 137         case DWC3_DEPCMD_CLEARSTALL:
 138                 return "Clear Stall";
 139         case DWC3_DEPCMD_SETSTALL:
 140                 return "Set Stall";
 141         case DWC3_DEPCMD_GETSEQNUMBER:
 142                 return "Get Data Sequence Number";
 143         case DWC3_DEPCMD_SETTRANSFRESOURCE:
 144                 return "Set Endpoint Transfer Resource";
 145         case DWC3_DEPCMD_SETEPCONFIG:
 146                 return "Set Endpoint Configuration";
 147         default:
 148                 return "UNKNOWN command";
 149         }
 150 }
 151
 152 int dwc3_send_gadget_ep_cmd(struct dwc3 *dwc, unsigned ep,
 153                 unsigned cmd, struct dwc3_gadget_ep_cmd_params *params)
 154 {
 155         struct dwc3_ep          *dep = dwc->eps[ep];
 156         u32                     timeout = 500;
 157         u32                     reg;
 158
 159         dev_vdbg(dwc->dev, "%s: cmd '%s' params %08x %08x %08x\n",
 160                         dep->name,
 161                         dwc3_gadget_ep_cmd_string(cmd), params->param0.raw,
 162                         params->param1.raw, params->param2.raw);
 163
 164         dwc3_writel(dwc->regs, DWC3_DEPCMDPAR0(ep), params->param0.raw);
 165         dwc3_writel(dwc->regs, DWC3_DEPCMDPAR1(ep), params->param1.raw);
 166         dwc3_writel(dwc->regs, DWC3_DEPCMDPAR2(ep), params->param2.raw);
 167
 168         dwc3_writel(dwc->regs, DWC3_DEPCMD(ep), cmd | DWC3_DEPCMD_CMDACT);
 169         do {
 170                 reg = dwc3_readl(dwc->regs, DWC3_DEPCMD(ep));
 171                 if (!(reg & DWC3_DEPCMD_CMDACT)) {
 172                         dev_vdbg(dwc->dev, "Command Complete --> %d\n",
 173                                         DWC3_DEPCMD_STATUS(reg));
 174                         return 0;
 175                 }
 176
 177                 /*
 178                  * We can't sleep here, because it is also called from
 179                  * interrupt context.
 180                  */
 181                 timeout--;
 182                 if (!timeout)
 183                         return -ETIMEDOUT;
 184
 185                 udelay(1);
 186         } while (1);
 187 }
 188
 189 static dma_addr_t dwc3_trb_dma_offset(struct dwc3_ep *dep,
 190                 struct dwc3_trb_hw *trb)
 191 {
 192         u32             offset = (char *) trb - (char *) dep->trb_pool;
 193
 194         return dep->trb_pool_dma + offset;
 195 }
 196
 197 static int dwc3_alloc_trb_pool(struct dwc3_ep *dep)
 198 {
 199         struct dwc3             *dwc = dep->dwc;
 200
 201         if (dep->trb_pool)
 202                 return 0;
 203
 204         if (dep->number == 0 || dep->number == 1)
 205                 return 0;
 206
 207         dep->trb_pool = dma_alloc_coherent(dwc->dev,
 208                         sizeof(struct dwc3_trb) * DWC3_TRB_NUM,
 209                         &dep->trb_pool_dma, GFP_KERNEL);
 210         if (!dep->trb_pool) {
 211                 dev_err(dep->dwc->dev, "failed to allocate trb pool for %s\n",
 212                                 dep->name);
 213                 return -ENOMEM;
 214         }
 215
 216         return 0;
 217 }
 218
 219 static void dwc3_free_trb_pool(struct dwc3_ep *dep)
 220 {
 221         struct dwc3             *dwc = dep->dwc;
 222
 223         dma_free_coherent(dwc->dev, sizeof(struct dwc3_trb) * DWC3_TRB_NUM,
 224                         dep->trb_pool, dep->trb_pool_dma);
 225
 226         dep->trb_pool = NULL;
 227         dep->trb_pool_dma = 0;
 228 }
 229
 230 static int dwc3_gadget_start_config(struct dwc3 *dwc, struct dwc3_ep *dep)
 231 {
 232         struct dwc3_gadget_ep_cmd_params params;
 233         u32                     cmd;
 234
 235         memset(&params, 0x00, sizeof(params));
 236
 237         if (dep->number != 1) {
 238                 cmd = DWC3_DEPCMD_DEPSTARTCFG;
 239                 /* XferRscIdx == 0 for ep0 and 2 for the remaining */
 240                 if (dep->number > 1) {
 241                         if (dwc->start_config_issued)
 242                                 return 0;
 243                         dwc->start_config_issued = true;
 244                         cmd |= DWC3_DEPCMD_PARAM(2);
 245                 }
 246
 247                 return dwc3_send_gadget_ep_cmd(dwc, 0, cmd, &params);
 248         }
 249
 250         return 0;
 251 }
 252
 253 static int dwc3_gadget_set_ep_config(struct dwc3 *dwc, struct dwc3_ep *dep,
 254                 const struct usb_endpoint_descriptor *desc)
 255 {
 256         struct dwc3_gadget_ep_cmd_params params;
 257
 258         memset(&params, 0x00, sizeof(params));
 259
 260         params.param0.depcfg.ep_type = usb_endpoint_type(desc);
 261         params.param0.depcfg.max_packet_size = usb_endpoint_maxp(desc);
 262
 263         params.param1.depcfg.xfer_complete_enable = true;
 264         params.param1.depcfg.xfer_not_ready_enable = true;
 265
 266         if (usb_endpoint_xfer_isoc(desc))
 267                 params.param1.depcfg.xfer_in_progress_enable = true;
 268
 269         /*
 270          * We are doing 1:1 mapping for endpoints, meaning
 271          * Physical Endpoints 2 maps to Logical Endpoint 2 and
 272          * so on. We consider the direction bit as part of the physical
 273          * endpoint number. So USB endpoint 0x81 is 0x03.
 274          */
 275         params.param1.depcfg.ep_number = dep->number;
 276
 277         /*
 278          * We must use the lower 16 TX FIFOs even though
 279          * HW might have more
 280          */
 281         if (dep->direction)
 282                 params.param0.depcfg.fifo_number = dep->number >> 1;
 283
 284         if (desc->bInterval) {
 285                 params.param1.depcfg.binterval_m1 = desc->bInterval - 1;
 286                 dep->interval = 1 << (desc->bInterval - 1);
 287         }
 288
 289         return dwc3_send_gadget_ep_cmd(dwc, dep->number,
 290                         DWC3_DEPCMD_SETEPCONFIG, &params);
 291 }
 292
 293 static int dwc3_gadget_set_xfer_resource(struct dwc3 *dwc, struct dwc3_ep *dep)
 294 {
 295         struct dwc3_gadget_ep_cmd_params params;
 296
 297         memset(&params, 0x00, sizeof(params));
 298
 299         params.param0.depxfercfg.number_xfer_resources = 1;
 300
 301         return dwc3_send_gadget_ep_cmd(dwc, dep->number,
 302                         DWC3_DEPCMD_SETTRANSFRESOURCE, &params);
 303 }
 304
 305 /**
 306  * __dwc3_gadget_ep_enable - Initializes a HW endpoint
 307  * @dep: endpoint to be initialized
 308  * @desc: USB Endpoint Descriptor
 309  *
 310  * Caller should take care of locking
 311  */
 312 static int __dwc3_gadget_ep_enable(struct dwc3_ep *dep,
 313                 const struct usb_endpoint_descriptor *desc)
 314 {
 315         struct dwc3             *dwc = dep->dwc;
 316         u32                     reg;
 317         int                     ret = -ENOMEM;
 318
 319         if (!(dep->flags & DWC3_EP_ENABLED)) {
 320                 ret = dwc3_gadget_start_config(dwc, dep);
 321                 if (ret)
 322                         return ret;
 323         }
 324
 325         ret = dwc3_gadget_set_ep_config(dwc, dep, desc);
 326         if (ret)
 327                 return ret;
 328
 329         if (!(dep->flags & DWC3_EP_ENABLED)) {
 330                 struct dwc3_trb_hw      *trb_st_hw;
 331                 struct dwc3_trb_hw      *trb_link_hw;
 332                 struct dwc3_trb         trb_link;
 333
 334                 ret = dwc3_gadget_set_xfer_resource(dwc, dep);
 335                 if (ret)
 336                         return ret;
 337
 338                 dep->desc = desc;
 339                 dep->type = usb_endpoint_type(desc);
 340                 dep->flags |= DWC3_EP_ENABLED;
 341
 342                 reg = dwc3_readl(dwc->regs, DWC3_DALEPENA);
 343                 reg |= DWC3_DALEPENA_EP(dep->number);
 344                 dwc3_writel(dwc->regs, DWC3_DALEPENA, reg);
 345
 346                 if (!usb_endpoint_xfer_isoc(desc))
 347                         return 0;
 348
 349                 memset(&trb_link, 0, sizeof(trb_link));
 350
 351                 /* Link TRB for ISOC. The HWO but is never reset */
 352                 trb_st_hw = &dep->trb_pool[0];
 353
 354                 trb_link.bplh = dwc3_trb_dma_offset(dep, trb_st_hw);
 355                 trb_link.trbctl = DWC3_TRBCTL_LINK_TRB;
 356                 trb_link.hwo = true;
 357
 358                 trb_link_hw = &dep->trb_pool[DWC3_TRB_NUM - 1];
 359                 dwc3_trb_to_hw(&trb_link, trb_link_hw);
 360         }
 361
 362         return 0;
 363 }
 364
 365 static void dwc3_stop_active_transfer(struct dwc3 *dwc, u32 epnum);
 366 static void dwc3_remove_requests(struct dwc3 *dwc, struct dwc3_ep *dep)
 367 {
 368         struct dwc3_request             *req;
 369
 370         if (!list_empty(&dep->req_queued))
 371                 dwc3_stop_active_transfer(dwc, dep->number);
 372
 373         while (!list_empty(&dep->request_list)) {
 374                 req = next_request(&dep->request_list);
 375
 376                 dwc3_gadget_giveback(dep, req, -ESHUTDOWN);
 377         }
 378 }
 379
 380 /**
 381  * __dwc3_gadget_ep_disable - Disables a HW endpoint
 382  * @dep: the endpoint to disable
 383  *
 384  * This function also removes requests which are currently processed ny the
 385  * hardware and those which are not yet scheduled.
 386  * Caller should take care of locking.
 387  */
 388 static int __dwc3_gadget_ep_disable(struct dwc3_ep *dep)
 389 {
 390         struct dwc3             *dwc = dep->dwc;
 391         u32                     reg;
 392
 393         dep->flags &= ~DWC3_EP_ENABLED;
 394         dwc3_remove_requests(dwc, dep);
 395
 396         reg = dwc3_readl(dwc->regs, DWC3_DALEPENA);
 397         reg &= ~DWC3_DALEPENA_EP(dep->number);
 398         dwc3_writel(dwc->regs, DWC3_DALEPENA, reg);
 399
 400         dep->desc = NULL;
 401         dep->type = 0;
 402
 403         return 0;
 404 }
 405
 406 /* -------------------------------------------------------------------------- */
 407
 408 static int dwc3_gadget_ep0_enable(struct usb_ep *ep,
 409                 const struct usb_endpoint_descriptor *desc)
 410 {
 411         return -EINVAL;
 412 }
 413
 414 static int dwc3_gadget_ep0_disable(struct usb_ep *ep)
 415 {
 416         return -EINVAL;
 417 }
 418
 419 /* -------------------------------------------------------------------------- */
 420
 421 static int dwc3_gadget_ep_enable(struct usb_ep *ep,
 422                 const struct usb_endpoint_descriptor *desc)
 423 {
 424         struct dwc3_ep                  *dep;
 425         struct dwc3                     *dwc;
 426         unsigned long                   flags;
 427         int                             ret;
 428
 429         if (!ep || !desc || desc->bDescriptorType != USB_DT_ENDPOINT) {
 430                 pr_debug("dwc3: invalid parameters\n");
 431                 return -EINVAL;
 432         }
 433
 434         if (!desc->wMaxPacketSize) {
 435                 pr_debug("dwc3: missing wMaxPacketSize\n");
 436                 return -EINVAL;
 437         }
 438
 439         dep = to_dwc3_ep(ep);
 440         dwc = dep->dwc;
 441
 442         switch (usb_endpoint_type(desc)) {
 443         case USB_ENDPOINT_XFER_CONTROL:
 444                 strncat(dep->name, "-control", sizeof(dep->name));
 445                 break;
 446         case USB_ENDPOINT_XFER_ISOC:
 447                 strncat(dep->name, "-isoc", sizeof(dep->name));
 448                 break;
 449         case USB_ENDPOINT_XFER_BULK:
 450                 strncat(dep->name, "-bulk", sizeof(dep->name));
 451                 break;
 452         case USB_ENDPOINT_XFER_INT:
 453                 strncat(dep->name, "-int", sizeof(dep->name));
 454                 break;
 455         default:
 456                 dev_err(dwc->dev, "invalid endpoint transfer type\n");
 457         }
 458
 459         if (dep->flags & DWC3_EP_ENABLED) {
 460                 dev_WARN_ONCE(dwc->dev, true, "%s is already enabled\n",
 461                                 dep->name);
 462                 return 0;
 463         }
 464
 465         dev_vdbg(dwc->dev, "Enabling %s\n", dep->name);
 466
 467         spin_lock_irqsave(&dwc->lock, flags);
 468         ret = __dwc3_gadget_ep_enable(dep, desc);
 469         spin_unlock_irqrestore(&dwc->lock, flags);
 470
 471         return ret;
 472 }
 473
 474 static int dwc3_gadget_ep_disable(struct usb_ep *ep)
 475 {
 476         struct dwc3_ep                  *dep;
 477         struct dwc3                     *dwc;
 478         unsigned long                   flags;
 479         int                             ret;
 480
 481         if (!ep) {
 482                 pr_debug("dwc3: invalid parameters\n");
 483                 return -EINVAL;
 484         }
 485
 486         dep = to_dwc3_ep(ep);
 487         dwc = dep->dwc;
 488
 489         if (!(dep->flags & DWC3_EP_ENABLED)) {
 490                 dev_WARN_ONCE(dwc->dev, true, "%s is already disabled\n",
 491                                 dep->name);
 492                 return 0;
 493         }
 494
 495         snprintf(dep->name, sizeof(dep->name), "ep%d%s",
 496                         dep->number >> 1,
 497                         (dep->number & 1) ? "in" : "out");
 498
 499         spin_lock_irqsave(&dwc->lock, flags);
 500         ret = __dwc3_gadget_ep_disable(dep);
 501         spin_unlock_irqrestore(&dwc->lock, flags);
 502
 503         return ret;
 504 }
 505
 506 static struct usb_request *dwc3_gadget_ep_alloc_request(struct usb_ep *ep,
 507         gfp_t gfp_flags)
 508 {
 509         struct dwc3_request             *req;
 510         struct dwc3_ep                  *dep = to_dwc3_ep(ep);
 511         struct dwc3                     *dwc = dep->dwc;
 512
 513         req = kzalloc(sizeof(*req), gfp_flags);
 514         if (!req) {
 515                 dev_err(dwc->dev, "not enough memory\n");
 516                 return NULL;
 517         }
 518
 519         req->epnum      = dep->number;
 520         req->dep        = dep;
 521         req->request.dma = DMA_ADDR_INVALID;
 522
 523         return &req->request;
 524 }
 525
 526 static void dwc3_gadget_ep_free_request(struct usb_ep *ep,
 527                 struct usb_request *request)
 528 {
 529         struct dwc3_request             *req = to_dwc3_request(request);
 530
 531         kfree(req);
 532 }
 533
 534 /*
 535  * dwc3_prepare_trbs - setup TRBs from requests
 536  * @dep: endpoint for which requests are being prepared
 537  * @starting: true if the endpoint is idle and no requests are queued.
 538  *
 539  * The functions goes through the requests list and setups TRBs for the
 540  * transfers. The functions returns once there are not more TRBs available or
 541  * it run out of requests.
 542  */
 543 static struct dwc3_request *dwc3_prepare_trbs(struct dwc3_ep *dep,
 544                 bool starting)
 545 {
 546         struct dwc3_request     *req, *n, *ret = NULL;
 547         struct dwc3_trb_hw      *trb_hw;
 548         struct dwc3_trb         trb;
 549         u32                     trbs_left;
 550
 551         BUILD_BUG_ON_NOT_POWER_OF_2(DWC3_TRB_NUM);
 552
 553         /* the first request must not be queued */
 554         trbs_left = (dep->busy_slot - dep->free_slot) & DWC3_TRB_MASK;
 555         /*
 556          * if busy & slot are equal than it is either full or empty. If we are
 557          * starting to proceed requests then we are empty. Otherwise we ar
 558          * full and don't do anything
 559          */
 560         if (!trbs_left) {
 561                 if (!starting)
 562                         return NULL;
 563                 trbs_left = DWC3_TRB_NUM;
 564                 /*
 565                  * In case we start from scratch, we queue the ISOC requests
 566                  * starting from slot 1. This is done because we use ring
 567                  * buffer and have no LST bit to stop us. Instead, we place
 568                  * IOC bit TRB_NUM/4. We try to avoid to having an interrupt
 569                  * after the first request so we start at slot 1 and have
 570                  * 7 requests proceed before we hit the first IOC.
 571                  * Other transfer types don't use the ring buffer and are
 572                  * processed from the first TRB until the last one. Since we
 573                  * don't wrap around we have to start at the beginning.
 574                  */
 575                 if (usb_endpoint_xfer_isoc(dep->desc)) {
 576                         dep->busy_slot = 1;
 577                         dep->free_slot = 1;
 578                 } else {
 579                         dep->busy_slot = 0;
 580                         dep->free_slot = 0;
 581                 }
 582         }
 583
 584         /* The last TRB is a link TRB, not used for xfer */
 585         if ((trbs_left <= 1) && usb_endpoint_xfer_isoc(dep->desc))
 586                 return NULL;
 587
 588         list_for_each_entry_safe(req, n, &dep->request_list, list) {
 589                 unsigned int last_one = 0;
 590                 unsigned int cur_slot;
 591
 592                 trb_hw = &dep->trb_pool[dep->free_slot & DWC3_TRB_MASK];
 593                 cur_slot = dep->free_slot;
 594                 dep->free_slot++;
 595
 596                 /* Skip the LINK-TRB on ISOC */
 597                 if (((cur_slot & DWC3_TRB_MASK) == DWC3_TRB_NUM - 1) &&
 598                                 usb_endpoint_xfer_isoc(dep->desc))
 599                         continue;
 600
 601                 dwc3_gadget_move_request_queued(req);
 602                 memset(&trb, 0, sizeof(trb));
 603                 trbs_left--;
 604
 605                 /* Is our TRB pool empty? */
 606                 if (!trbs_left)
 607                         last_one = 1;
 608                 /* Is this the last request? */
 609                 if (list_empty(&dep->request_list))
 610                         last_one = 1;
 611
 612                 /*
 613                  * FIXME we shouldn't need to set LST bit always but we are
 614                  * facing some weird problem with the Hardware where it doesn't
 615                  * complete even though it has been previously started.
 616                  *
 617                  * While we're debugging the problem, as a workaround to
 618                  * multiple TRBs handling, use only one TRB at a time.
 619                  */
 620                 last_one = 1;
 621
 622                 req->trb = trb_hw;
 623                 if (!ret)
 624                         ret = req;
 625
 626                 trb.bplh = req->request.dma;
 627
 628                 if (usb_endpoint_xfer_isoc(dep->desc)) {
 629                         trb.isp_imi = true;
 630                         trb.csp = true;
 631                 } else {
 632                         trb.lst = last_one;
 633                 }
 634
 635                 switch (usb_endpoint_type(dep->desc)) {
 636                 case USB_ENDPOINT_XFER_CONTROL:
 637                         trb.trbctl = DWC3_TRBCTL_CONTROL_SETUP;
 638                         break;
 639
 640                 case USB_ENDPOINT_XFER_ISOC:
 641                         trb.trbctl = DWC3_TRBCTL_ISOCHRONOUS_FIRST;
 642
 643                         /* IOC every DWC3_TRB_NUM / 4 so we can refill */
 644                         if (!(cur_slot % (DWC3_TRB_NUM / 4)))
 645                                 trb.ioc = last_one;
 646                         break;
 647
 648                 case USB_ENDPOINT_XFER_BULK:
 649                 case USB_ENDPOINT_XFER_INT:
 650                         trb.trbctl = DWC3_TRBCTL_NORMAL;
 651                         break;
 652                 default:
 653                         /*
 654                          * This is only possible with faulty memory because we
 655                          * checked it already :)
 656                          */
 657                         BUG();
 658                 }
 659
 660                 trb.length      = req->request.length;
 661                 trb.hwo = true;
 662
 663                 dwc3_trb_to_hw(&trb, trb_hw);
 664                 req->trb_dma = dwc3_trb_dma_offset(dep, trb_hw);
 665
 666                 if (last_one)
 667                         break;
 668         }
 669
 670         return ret;
 671 }
 672
 673 static int __dwc3_gadget_kick_transfer(struct dwc3_ep *dep, u16 cmd_param,
 674                 int start_new)
 675 {
 676         struct dwc3_gadget_ep_cmd_params params;
 677         struct dwc3_request             *req;
 678         struct dwc3                     *dwc = dep->dwc;
 679         int                             ret;
 680         u32                             cmd;
 681
 682         if (start_new && (dep->flags & DWC3_EP_BUSY)) {
 683                 dev_vdbg(dwc->dev, "%s: endpoint busy\n", dep->name);
 684                 return -EBUSY;
 685         }
 686         dep->flags &= ~DWC3_EP_PENDING_REQUEST;
 687
 688         /*
 689          * If we are getting here after a short-out-packet we don't enqueue any
 690          * new requests as we try to set the IOC bit only on the last request.
 691          */
 692         if (start_new) {
 693                 if (list_empty(&dep->req_queued))
 694                         dwc3_prepare_trbs(dep, start_new);
 695
 696                 /* req points to the first request which will be sent */
 697                 req = next_request(&dep->req_queued);
 698         } else {
 699                 /*
 700                  * req points to the first request where HWO changed
 701                  * from 0 to 1
 702                  */
 703                 req = dwc3_prepare_trbs(dep, start_new);
 704         }
 705         if (!req) {
 706                 dep->flags |= DWC3_EP_PENDING_REQUEST;
 707                 return 0;
 708         }
 709
 710         memset(&params, 0, sizeof(params));
 711         params.param0.depstrtxfer.transfer_desc_addr_high =
 712                 upper_32_bits(req->trb_dma);
 713         params.param1.depstrtxfer.transfer_desc_addr_low =
 714                 lower_32_bits(req->trb_dma);
 715
 716         if (start_new)
 717                 cmd = DWC3_DEPCMD_STARTTRANSFER;
 718         else
 719                 cmd = DWC3_DEPCMD_UPDATETRANSFER;
 720
 721         cmd |= DWC3_DEPCMD_PARAM(cmd_param);
 722         ret = dwc3_send_gadget_ep_cmd(dwc, dep->number, cmd, &params);
 723         if (ret < 0) {
 724                 dev_dbg(dwc->dev, "failed to send STARTTRANSFER command\n");
 725
 726                 /*
 727                  * FIXME we need to iterate over the list of requests
 728                  * here and stop, unmap, free and del each of the linked
 729                  * requests instead of we do now.
 730                  */
 731                 dwc3_unmap_buffer_from_dma(req);
 732                 list_del(&req->list);
 733                 return ret;
 734         }
 735
 736         dep->flags |= DWC3_EP_BUSY;
 737         dep->res_trans_idx = dwc3_gadget_ep_get_transfer_index(dwc,
 738                         dep->number);
 739         if (!dep->res_trans_idx)
 740                 printk_once(KERN_ERR "%s() res_trans_idx is invalid\n", __func__);
 741         return 0;
 742 }
 743
 744 static int __dwc3_gadget_ep_queue(struct dwc3_ep *dep, struct dwc3_request *req)
 745 {
 746         req->request.actual     = 0;
 747         req->request.status     = -EINPROGRESS;
 748         req->direction          = dep->direction;
 749         req->epnum              = dep->number;
 750
 751         /*
 752          * We only add to our list of requests now and
 753          * start consuming the list once we get XferNotReady
 754          * IRQ.
 755          *
 756          * That way, we avoid doing anything that we don't need
 757          * to do now and defer it until the point we receive a
 758          * particular token from the Host side.
 759          *
 760          * This will also avoid Host cancelling URBs due to too
 761          * many NACKs.
 762          */
 763         dwc3_map_buffer_to_dma(req);
 764         list_add_tail(&req->list, &dep->request_list);
 765
 766         /*
 767          * There is one special case: XferNotReady with
 768          * empty list of requests. We need to kick the
 769          * transfer here in that situation, otherwise
 770          * we will be NAKing forever.
 771          *
 772          * If we get XferNotReady before gadget driver
 773          * has a chance to queue a request, we will ACK
 774          * the IRQ but won't be able to receive the data
 775          * until the next request is queued. The following
 776          * code is handling exactly that.
 777          */
 778         if (dep->flags & DWC3_EP_PENDING_REQUEST) {
 779                 int ret;
 780                 int start_trans;
 781
 782                 start_trans = 1;
 783                 if (usb_endpoint_xfer_isoc(dep->endpoint.desc) &&
 784                                 dep->flags & DWC3_EP_BUSY)
 785                         start_trans = 0;
 786
 787                 ret =  __dwc3_gadget_kick_transfer(dep, 0, start_trans);
 788                 if (ret && ret != -EBUSY) {
 789                         struct dwc3     *dwc = dep->dwc;
 790
 791                         dev_dbg(dwc->dev, "%s: failed to kick transfers\n",
 792                                         dep->name);
 793                 }
 794         };
 795
 796         return 0;
 797 }
 798
 799 static int dwc3_gadget_ep_queue(struct usb_ep *ep, struct usb_request *request,
 800         gfp_t gfp_flags)
 801 {
 802         struct dwc3_request             *req = to_dwc3_request(request);
 803         struct dwc3_ep                  *dep = to_dwc3_ep(ep);
 804         struct dwc3                     *dwc = dep->dwc;
 805
 806         unsigned long                   flags;
 807
 808         int                             ret;
 809
 810         if (!dep->desc) {
 811                 dev_dbg(dwc->dev, "trying to queue request %p to disabled %s\n",
 812                                 request, ep->name);
 813                 return -ESHUTDOWN;
 814         }
 815
 816         dev_vdbg(dwc->dev, "queing request %p to %s length %d\n",
 817                         request, ep->name, request->length);
 818
 819         spin_lock_irqsave(&dwc->lock, flags);
 820         ret = __dwc3_gadget_ep_queue(dep, req);
 821         spin_unlock_irqrestore(&dwc->lock, flags);
 822
 823         return ret;
 824 }
 825
 826 static int dwc3_gadget_ep_dequeue(struct usb_ep *ep,
 827                 struct usb_request *request)
 828 {
 829         struct dwc3_request             *req = to_dwc3_request(request);
 830         struct dwc3_request             *r = NULL;
 831
 832         struct dwc3_ep                  *dep = to_dwc3_ep(ep);
 833         struct dwc3                     *dwc = dep->dwc;
 834
 835         unsigned long                   flags;
 836         int                             ret = 0;
 837
 838         spin_lock_irqsave(&dwc->lock, flags);
 839
 840         list_for_each_entry(r, &dep->request_list, list) {
 841                 if (r == req)
 842                         break;
 843         }
 844
 845         if (r != req) {
 846                 list_for_each_entry(r, &dep->req_queued, list) {
 847                         if (r == req)
 848                                 break;
 849                 }
 850                 if (r == req) {
 851                         /* wait until it is processed */
 852                         dwc3_stop_active_transfer(dwc, dep->number);
 853                         goto out0;
 854                 }
 855                 dev_err(dwc->dev, "request %p was not queued to %s\n",
 856                                 request, ep->name);
 857                 ret = -EINVAL;
 858                 goto out0;
 859         }
 860
 861         /* giveback the request */
 862         dwc3_gadget_giveback(dep, req, -ECONNRESET);
 863
 864 out0:
 865         spin_unlock_irqrestore(&dwc->lock, flags);
 866
 867         return ret;
 868 }
 869
 870 int __dwc3_gadget_ep_set_halt(struct dwc3_ep *dep, int value)
 871 {
 872         struct dwc3_gadget_ep_cmd_params        params;
 873         struct dwc3                             *dwc = dep->dwc;
 874         int                                     ret;
 875
 876         memset(&params, 0x00, sizeof(params));
 877
 878         if (value) {
 879                 if (dep->number == 0 || dep->number == 1) {
 880                         /*
 881                          * Whenever EP0 is stalled, we will restart
 882                          * the state machine, thus moving back to
 883                          * Setup Phase
 884                          */
 885                         dwc->ep0state = EP0_SETUP_PHASE;
 886                 }
 887
 888                 ret = dwc3_send_gadget_ep_cmd(dwc, dep->number,
 889                         DWC3_DEPCMD_SETSTALL, &params);
 890                 if (ret)
 891                         dev_err(dwc->dev, "failed to %s STALL on %s\n",
 892                                         value ? "set" : "clear",
 893                                         dep->name);
 894                 else
 895                         dep->flags |= DWC3_EP_STALL;
 896         } else {
 897                 if (dep->flags & DWC3_EP_WEDGE)
 898                         return 0;
 899
 900                 ret = dwc3_send_gadget_ep_cmd(dwc, dep->number,
 901                         DWC3_DEPCMD_CLEARSTALL, &params);
 902                 if (ret)
 903                         dev_err(dwc->dev, "failed to %s STALL on %s\n",
 904                                         value ? "set" : "clear",
 905                                         dep->name);
 906                 else
 907                         dep->flags &= ~DWC3_EP_STALL;
 908         }
 909
 910         return ret;
 911 }
 912
 913 static int dwc3_gadget_ep_set_halt(struct usb_ep *ep, int value)
 914 {
 915         struct dwc3_ep                  *dep = to_dwc3_ep(ep);
 916         struct dwc3                     *dwc = dep->dwc;
 917
 918         unsigned long                   flags;
 919
 920         int                             ret;
 921
 922         spin_lock_irqsave(&dwc->lock, flags);
 923
 924         if (usb_endpoint_xfer_isoc(dep->desc)) {
 925                 dev_err(dwc->dev, "%s is of Isochronous type\n", dep->name);
 926                 ret = -EINVAL;
 927                 goto out;
 928         }
 929
 930         ret = __dwc3_gadget_ep_set_halt(dep, value);
 931 out:
 932         spin_unlock_irqrestore(&dwc->lock, flags);
 933
 934         return ret;
 935 }
 936
 937 static int dwc3_gadget_ep_set_wedge(struct usb_ep *ep)
 938 {
 939         struct dwc3_ep                  *dep = to_dwc3_ep(ep);
 940
 941         dep->flags |= DWC3_EP_WEDGE;
 942
 943         return dwc3_gadget_ep_set_halt(ep, 1);
 944 }
 945
 946 /* -------------------------------------------------------------------------- */
 947
 948 static struct usb_endpoint_descriptor dwc3_gadget_ep0_desc = {
 949         .bLength        = USB_DT_ENDPOINT_SIZE,
 950         .bDescriptorType = USB_DT_ENDPOINT,
 951         .bmAttributes   = USB_ENDPOINT_XFER_CONTROL,
 952 };
 953
 954 static const struct usb_ep_ops dwc3_gadget_ep0_ops = {
 955         .enable         = dwc3_gadget_ep0_enable,
 956         .disable        = dwc3_gadget_ep0_disable,
 957         .alloc_request  = dwc3_gadget_ep_alloc_request,
 958         .free_request   = dwc3_gadget_ep_free_request,
 959         .queue          = dwc3_gadget_ep0_queue,
 960         .dequeue        = dwc3_gadget_ep_dequeue,
 961         .set_halt       = dwc3_gadget_ep_set_halt,
 962         .set_wedge      = dwc3_gadget_ep_set_wedge,
 963 };
 964
 965 static const struct usb_ep_ops dwc3_gadget_ep_ops = {
 966         .enable         = dwc3_gadget_ep_enable,
 967         .disable        = dwc3_gadget_ep_disable,
 968         .alloc_request  = dwc3_gadget_ep_alloc_request,
 969         .free_request   = dwc3_gadget_ep_free_request,
 970         .queue          = dwc3_gadget_ep_queue,
 971         .dequeue        = dwc3_gadget_ep_dequeue,
 972         .set_halt       = dwc3_gadget_ep_set_halt,
 973         .set_wedge      = dwc3_gadget_ep_set_wedge,
 974 };
 975
 976 /* -------------------------------------------------------------------------- */
 977
 978 static int dwc3_gadget_get_frame(struct usb_gadget *g)
 979 {
 980         struct dwc3             *dwc = gadget_to_dwc(g);
 981         u32                     reg;
 982
 983         reg = dwc3_readl(dwc->regs, DWC3_DSTS);
 984         return DWC3_DSTS_SOFFN(reg);
 985 }
 986
 987 static int dwc3_gadget_wakeup(struct usb_gadget *g)
 988 {
 989         struct dwc3             *dwc = gadget_to_dwc(g);
 990
 991         unsigned long           timeout;
 992         unsigned long           flags;
 993
 994         u32                     reg;
 995
 996         int                     ret = 0;
 997
 998         u8                      link_state;
 999         u8                      speed;
1000
1001         spin_lock_irqsave(&dwc->lock, flags);
1002
1003         /*
1004          * According to the Databook Remote wakeup request should
1005          * be issued only when the device is in early suspend state.
1006          *
1007          * We can check that via USB Link State bits in DSTS register.
1008          */
1009         reg = dwc3_readl(dwc->regs, DWC3_DSTS);
1010
1011         speed = reg & DWC3_DSTS_CONNECTSPD;
1012         if (speed == DWC3_DSTS_SUPERSPEED) {
1013                 dev_dbg(dwc->dev, "no wakeup on SuperSpeed\n");
1014                 ret = -EINVAL;
1015                 goto out;
1016         }
1017
1018         link_state = DWC3_DSTS_USBLNKST(reg);
1019
1020         switch (link_state) {
1021         case DWC3_LINK_STATE_RX_DET:    /* in HS, means Early Suspend */
1022         case DWC3_LINK_STATE_U3:        /* in HS, means SUSPEND */
1023                 break;
1024         default:
1025                 dev_dbg(dwc->dev, "can't wakeup from link state %d\n",
1026                                 link_state);
1027                 ret = -EINVAL;
1028                 goto out;
1029         }
1030
1031         reg = dwc3_readl(dwc->regs, DWC3_DCTL);
1032
1033         /*
1034          * Switch link state to Recovery. In HS/FS/LS this means
1035          * RemoteWakeup Request
1036          */
1037         reg |= DWC3_DCTL_ULSTCHNG_RECOVERY;
1038         dwc3_writel(dwc->regs, DWC3_DCTL, reg);
1039
1040         /* wait for at least 2000us */
1041         usleep_range(2000, 2500);
1042
1043         /* write zeroes to Link Change Request */
1044         reg &= ~DWC3_DCTL_ULSTCHNGREQ_MASK;
1045         dwc3_writel(dwc->regs, DWC3_DCTL, reg);
1046
1047         /* pool until Link State change to ON */
1048         timeout = jiffies + msecs_to_jiffies(100);
1049
1050         while (!(time_after(jiffies, timeout))) {
1051                 reg = dwc3_readl(dwc->regs, DWC3_DSTS);
1052
1053                 /* in HS, means ON */
1054                 if (DWC3_DSTS_USBLNKST(reg) == DWC3_LINK_STATE_U0)
1055                         break;
1056         }
1057
1058         if (DWC3_DSTS_USBLNKST(reg) != DWC3_LINK_STATE_U0) {
1059                 dev_err(dwc->dev, "failed to send remote wakeup\n");
1060                 ret = -EINVAL;
1061         }
1062
1063 out:
1064         spin_unlock_irqrestore(&dwc->lock, flags);
1065
1066         return ret;
1067 }
1068
1069 static int dwc3_gadget_set_selfpowered(struct usb_gadget *g,
1070                 int is_selfpowered)
1071 {
1072         struct dwc3             *dwc = gadget_to_dwc(g);
1073
1074         dwc->is_selfpowered = !!is_selfpowered;
1075
1076         return 0;
1077 }
1078
1079 static void dwc3_gadget_run_stop(struct dwc3 *dwc, int is_on)
1080 {
1081         u32                     reg;
1082         u32                     timeout = 500;
1083
1084         reg = dwc3_readl(dwc->regs, DWC3_DCTL);
1085         if (is_on)
1086                 reg |= DWC3_DCTL_RUN_STOP;
1087         else
1088                 reg &= ~DWC3_DCTL_RUN_STOP;
1089
1090         dwc3_writel(dwc->regs, DWC3_DCTL, reg);
1091
1092         do {
1093                 reg = dwc3_readl(dwc->regs, DWC3_DSTS);
1094                 if (is_on) {
1095                         if (!(reg & DWC3_DSTS_DEVCTRLHLT))
1096                                 break;
1097                 } else {
1098                         if (reg & DWC3_DSTS_DEVCTRLHLT)
1099                                 break;
1100                 }
1101                 timeout--;
1102                 if (!timeout)
1103                         break;
1104                 udelay(1);
1105         } while (1);
1106
1107         dev_vdbg(dwc->dev, "gadget %s data soft-%s\n",
1108                         dwc->gadget_driver
1109                         ? dwc->gadget_driver->function : "no-function",
1110                         is_on ? "connect" : "disconnect");
1111 }
1112
1113 static int dwc3_gadget_pullup(struct usb_gadget *g, int is_on)
1114 {
1115         struct dwc3             *dwc = gadget_to_dwc(g);
1116         unsigned long           flags;
1117
1118         is_on = !!is_on;
1119
1120         spin_lock_irqsave(&dwc->lock, flags);
1121         dwc3_gadget_run_stop(dwc, is_on);
1122         spin_unlock_irqrestore(&dwc->lock, flags);
1123
1124         return 0;
1125 }
1126
1127 static int dwc3_gadget_start(struct usb_gadget *g,
1128                 struct usb_gadget_driver *driver)
1129 {
1130         struct dwc3             *dwc = gadget_to_dwc(g);
1131         struct dwc3_ep          *dep;
1132         unsigned long           flags;
1133         int                     ret = 0;
1134         u32                     reg;
1135
1136         spin_lock_irqsave(&dwc->lock, flags);
1137
1138         if (dwc->gadget_driver) {
1139                 dev_err(dwc->dev, "%s is already bound to %s\n",
1140                                 dwc->gadget.name,
1141                                 dwc->gadget_driver->driver.name);
1142                 ret = -EBUSY;
1143                 goto err0;
1144         }
1145
1146         dwc->gadget_driver      = driver;
1147         dwc->gadget.dev.driver  = &driver->driver;
1148
1149         reg = dwc3_readl(dwc->regs, DWC3_GCTL);
1150
1151         reg &= ~DWC3_GCTL_SCALEDOWN(3);
1152         reg &= ~DWC3_GCTL_PRTCAPDIR(DWC3_GCTL_PRTCAP_OTG);
1153         reg &= ~DWC3_GCTL_DISSCRAMBLE;
1154         reg |= DWC3_GCTL_PRTCAPDIR(DWC3_GCTL_PRTCAP_DEVICE);
1155
1156         /*
1157          * WORKAROUND: DWC3 revisions <1.90a have a bug
1158          * when The device fails to connect at SuperSpeed
1159          * and falls back to high-speed mode which causes
1160          * the device to enter in a Connect/Disconnect loop
1161          */
1162         if (dwc->revision < DWC3_REVISION_190A)
1163                 reg |= DWC3_GCTL_U2RSTECN;
1164
1165         dwc3_writel(dwc->regs, DWC3_GCTL, reg);
1166
1167         reg = dwc3_readl(dwc->regs, DWC3_DCFG);
1168         reg &= ~(DWC3_DCFG_SPEED_MASK);
1169         reg |= DWC3_DCFG_SUPERSPEED;
1170         dwc3_writel(dwc->regs, DWC3_DCFG, reg);
1171
1172         dwc->start_config_issued = false;
1173
1174         /* Start with SuperSpeed Default */
1175         dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(512);
1176
1177         dep = dwc->eps[0];
1178         ret = __dwc3_gadget_ep_enable(dep, &dwc3_gadget_ep0_desc);
1179         if (ret) {
1180                 dev_err(dwc->dev, "failed to enable %s\n", dep->name);
1181                 goto err0;
1182         }
1183
1184         dep = dwc->eps[1];
1185         ret = __dwc3_gadget_ep_enable(dep, &dwc3_gadget_ep0_desc);
1186         if (ret) {
1187                 dev_err(dwc->dev, "failed to enable %s\n", dep->name);
1188                 goto err1;
1189         }
1190
1191         /* begin to receive SETUP packets */
1192         dwc->ep0state = EP0_SETUP_PHASE;
1193         dwc3_ep0_out_start(dwc);
1194
1195         spin_unlock_irqrestore(&dwc->lock, flags);
1196
1197         return 0;
1198
1199 err1:
1200         __dwc3_gadget_ep_disable(dwc->eps[0]);
1201
1202 err0:
1203         spin_unlock_irqrestore(&dwc->lock, flags);
1204
1205         return ret;
1206 }
1207
1208 static int dwc3_gadget_stop(struct usb_gadget *g,
1209                 struct usb_gadget_driver *driver)
1210 {
1211         struct dwc3             *dwc = gadget_to_dwc(g);
1212         unsigned long           flags;
1213
1214         spin_lock_irqsave(&dwc->lock, flags);
1215
1216         __dwc3_gadget_ep_disable(dwc->eps[0]);
1217         __dwc3_gadget_ep_disable(dwc->eps[1]);
1218
1219         dwc->gadget_driver      = NULL;
1220         dwc->gadget.dev.driver  = NULL;
1221
1222         spin_unlock_irqrestore(&dwc->lock, flags);
1223
1224         return 0;
1225 }
1226 static const struct usb_gadget_ops dwc3_gadget_ops = {
1227         .get_frame              = dwc3_gadget_get_frame,
1228         .wakeup                 = dwc3_gadget_wakeup,
1229         .set_selfpowered        = dwc3_gadget_set_selfpowered,
1230         .pullup                 = dwc3_gadget_pullup,
1231         .udc_start              = dwc3_gadget_start,
1232         .udc_stop               = dwc3_gadget_stop,
1233 };
1234
1235 /* -------------------------------------------------------------------------- */
1236
1237 static int __devinit dwc3_gadget_init_endpoints(struct dwc3 *dwc)
1238 {
1239         struct dwc3_ep                  *dep;
1240         u8                              epnum;
1241
1242         INIT_LIST_HEAD(&dwc->gadget.ep_list);
1243
1244         for (epnum = 0; epnum < DWC3_ENDPOINTS_NUM; epnum++) {
1245                 dep = kzalloc(sizeof(*dep), GFP_KERNEL);
1246                 if (!dep) {
1247                         dev_err(dwc->dev, "can't allocate endpoint %d\n",
1248                                         epnum);
1249                         return -ENOMEM;
1250                 }
1251
1252                 dep->dwc = dwc;
1253                 dep->number = epnum;
1254                 dwc->eps[epnum] = dep;
1255
1256                 snprintf(dep->name, sizeof(dep->name), "ep%d%s", epnum >> 1,
1257                                 (epnum & 1) ? "in" : "out");
1258                 dep->endpoint.name = dep->name;
1259                 dep->direction = (epnum & 1);
1260
1261                 if (epnum == 0 || epnum == 1) {
1262                         dep->endpoint.maxpacket = 512;
1263                         dep->endpoint.ops = &dwc3_gadget_ep0_ops;
1264                         if (!epnum)
1265                                 dwc->gadget.ep0 = &dep->endpoint;
1266                 } else {
1267                         int             ret;
1268
1269                         dep->endpoint.maxpacket = 1024;
1270                         dep->endpoint.ops = &dwc3_gadget_ep_ops;
1271                         list_add_tail(&dep->endpoint.ep_list,
1272                                         &dwc->gadget.ep_list);
1273
1274                         ret = dwc3_alloc_trb_pool(dep);
1275                         if (ret) {
1276                                 dev_err(dwc->dev, "%s: failed to allocate TRB pool\n", dep->name);
1277                                 return ret;
1278                         }
1279                 }
1280                 INIT_LIST_HEAD(&dep->request_list);
1281                 INIT_LIST_HEAD(&dep->req_queued);
1282         }
1283
1284         return 0;
1285 }
1286
1287 static void dwc3_gadget_free_endpoints(struct dwc3 *dwc)
1288 {
1289         struct dwc3_ep                  *dep;
1290         u8                              epnum;
1291
1292         for (epnum = 0; epnum < DWC3_ENDPOINTS_NUM; epnum++) {
1293                 dep = dwc->eps[epnum];
1294                 dwc3_free_trb_pool(dep);
1295
1296                 if (epnum != 0 && epnum != 1)
1297                         list_del(&dep->endpoint.ep_list);
1298
1299                 kfree(dep);
1300         }
1301 }
1302
1303 static void dwc3_gadget_release(struct device *dev)
1304 {
1305         dev_dbg(dev, "%s\n", __func__);
1306 }
1307
1308 /* -------------------------------------------------------------------------- */
1309 static int dwc3_cleanup_done_reqs(struct dwc3 *dwc, struct dwc3_ep *dep,
1310                 const struct dwc3_event_depevt *event, int status)
1311 {
1312         struct dwc3_request     *req;
1313         struct dwc3_trb         trb;
1314         unsigned int            count;
1315         unsigned int            s_pkt = 0;
1316
1317         do {
1318                 req = next_request(&dep->req_queued);
1319                 if (!req)
1320                         break;
1321
1322                 dwc3_trb_to_nat(req->trb, &trb);
1323
1324                 if (trb.hwo && status != -ESHUTDOWN)
1325                         /*
1326                          * We continue despite the error. There is not much we
1327                          * can do. If we don't clean in up we loop for ever. If
1328                          * we skip the TRB than it gets overwritten reused after
1329                          * a while since we use them in a ring buffer. a BUG()
1330                          * would help. Lets hope that if this occures, someone
1331                          * fixes the root cause instead of looking away :)
1332                          */
1333                         dev_err(dwc->dev, "%s's TRB (%p) still owned by HW\n",
1334                                         dep->name, req->trb);
1335                 count = trb.length;
1336
1337                 if (dep->direction) {
1338                         if (count) {
1339                                 dev_err(dwc->dev, "incomplete IN transfer %s\n",
1340                                                 dep->name);
1341                                 status = -ECONNRESET;
1342                         }
1343                 } else {
1344                         if (count && (event->status & DEPEVT_STATUS_SHORT))
1345                                 s_pkt = 1;
1346                 }
1347
1348                 /*
1349                  * We assume here we will always receive the entire data block
1350                  * which we should receive. Meaning, if we program RX to
1351                  * receive 4K but we receive only 2K, we assume that's all we
1352                  * should receive and we simply bounce the request back to the
1353                  * gadget driver for further processing.
1354                  */
1355                 req->request.actual += req->request.length - count;
1356                 dwc3_gadget_giveback(dep, req, status);
1357                 if (s_pkt)
1358                         break;
1359                 if ((event->status & DEPEVT_STATUS_LST) && trb.lst)
1360                         break;
1361                 if ((event->status & DEPEVT_STATUS_IOC) && trb.ioc)
1362                         break;
1363         } while (1);
1364
1365         if ((event->status & DEPEVT_STATUS_IOC) && trb.ioc)
1366                 return 0;
1367         return 1;
1368 }
1369
1370 static void dwc3_endpoint_transfer_complete(struct dwc3 *dwc,
1371                 struct dwc3_ep *dep, const struct dwc3_event_depevt *event,
1372                 int start_new)
1373 {
1374         unsigned                status = 0;
1375         int                     clean_busy;
1376
1377         if (event->status & DEPEVT_STATUS_BUSERR)
1378                 status = -ECONNRESET;
1379
1380         clean_busy =  dwc3_cleanup_done_reqs(dwc, dep, event, status);
1381         if (clean_busy) {
1382                 dep->flags &= ~DWC3_EP_BUSY;
1383                 dep->res_trans_idx = 0;
1384         }
1385 }
1386
1387 static void dwc3_gadget_start_isoc(struct dwc3 *dwc,
1388                 struct dwc3_ep *dep, const struct dwc3_event_depevt *event)
1389 {
1390         u32 uf;
1391
1392         if (list_empty(&dep->request_list)) {
1393                 dev_vdbg(dwc->dev, "ISOC ep %s run out for requests.\n",
1394                         dep->name);
1395                 return;
1396         }
1397
1398         if (event->parameters) {
1399                 u32 mask;
1400
1401                 mask = ~(dep->interval - 1);
1402                 uf = event->parameters & mask;
1403                 /* 4 micro frames in the future */
1404                 uf += dep->interval * 4;
1405         } else {
1406                 uf = 0;
1407         }
1408
1409         __dwc3_gadget_kick_transfer(dep, uf, 1);
1410 }
1411
1412 static void dwc3_process_ep_cmd_complete(struct dwc3_ep *dep,
1413                 const struct dwc3_event_depevt *event)
1414 {
1415         struct dwc3 *dwc = dep->dwc;
1416         struct dwc3_event_depevt mod_ev = *event;
1417
1418         /*
1419          * We were asked to remove one requests. It is possible that this
1420          * request and a few other were started together and have the same
1421          * transfer index. Since we stopped the complete endpoint we don't
1422          * know how many requests were already completed (and not yet)
1423          * reported and how could be done (later). We purge them all until
1424          * the end of the list.
1425          */
1426         mod_ev.status = DEPEVT_STATUS_LST;
1427         dwc3_cleanup_done_reqs(dwc, dep, &mod_ev, -ESHUTDOWN);
1428         dep->flags &= ~DWC3_EP_BUSY;
1429         /* pending requets are ignored and are queued on XferNotReady */
1430 }
1431
1432 static void dwc3_ep_cmd_compl(struct dwc3_ep *dep,
1433                 const struct dwc3_event_depevt *event)
1434 {
1435         u32 param = event->parameters;
1436         u32 cmd_type = (param >> 8) & ((1 << 5) - 1);
1437
1438         switch (cmd_type) {
1439         case DWC3_DEPCMD_ENDTRANSFER:
1440                 dwc3_process_ep_cmd_complete(dep, event);
1441                 break;
1442         case DWC3_DEPCMD_STARTTRANSFER:
1443                 dep->res_trans_idx = param & 0x7f;
1444                 break;
1445         default:
1446                 printk(KERN_ERR "%s() unknown /unexpected type: %d\n",
1447                                 __func__, cmd_type);
1448                 break;
1449         };
1450 }
1451
1452 static void dwc3_endpoint_interrupt(struct dwc3 *dwc,
1453                 const struct dwc3_event_depevt *event)
1454 {
1455         struct dwc3_ep          *dep;
1456         u8                      epnum = event->endpoint_number;
1457
1458         dep = dwc->eps[epnum];
1459
1460         dev_vdbg(dwc->dev, "%s: %s\n", dep->name,
1461                         dwc3_ep_event_string(event->endpoint_event));
1462
1463         if (epnum == 0 || epnum == 1) {
1464                 dwc3_ep0_interrupt(dwc, event);
1465                 return;
1466         }
1467
1468         switch (event->endpoint_event) {
1469         case DWC3_DEPEVT_XFERCOMPLETE:
1470                 if (usb_endpoint_xfer_isoc(dep->desc)) {
1471                         dev_dbg(dwc->dev, "%s is an Isochronous endpoint\n",
1472                                         dep->name);
1473                         return;
1474                 }
1475
1476                 dwc3_endpoint_transfer_complete(dwc, dep, event, 1);
1477                 break;
1478         case DWC3_DEPEVT_XFERINPROGRESS:
1479                 if (!usb_endpoint_xfer_isoc(dep->desc)) {
1480                         dev_dbg(dwc->dev, "%s is not an Isochronous endpoint\n",
1481                                         dep->name);
1482                         return;
1483                 }
1484
1485                 dwc3_endpoint_transfer_complete(dwc, dep, event, 0);
1486                 break;
1487         case DWC3_DEPEVT_XFERNOTREADY:
1488                 if (usb_endpoint_xfer_isoc(dep->desc)) {
1489                         dwc3_gadget_start_isoc(dwc, dep, event);
1490                 } else {
1491                         int ret;
1492
1493                         dev_vdbg(dwc->dev, "%s: reason %s\n",
1494                                         dep->name, event->status
1495                                         ? "Transfer Active"
1496                                         : "Transfer Not Active");
1497
1498                         ret = __dwc3_gadget_kick_transfer(dep, 0, 1);
1499                         if (!ret || ret == -EBUSY)
1500                                 return;
1501
1502                         dev_dbg(dwc->dev, "%s: failed to kick transfers\n",
1503                                         dep->name);
1504                 }
1505
1506                 break;
1507         case DWC3_DEPEVT_RXTXFIFOEVT:
1508                 dev_dbg(dwc->dev, "%s FIFO Overrun\n", dep->name);
1509                 break;
1510         case DWC3_DEPEVT_STREAMEVT:
1511                 dev_dbg(dwc->dev, "%s Stream Event\n", dep->name);
1512                 break;
1513         case DWC3_DEPEVT_EPCMDCMPLT:
1514                 dwc3_ep_cmd_compl(dep, event);
1515                 break;
1516         }
1517 }
1518
1519 static void dwc3_disconnect_gadget(struct dwc3 *dwc)
1520 {
1521         if (dwc->gadget_driver && dwc->gadget_driver->disconnect) {
1522                 spin_unlock(&dwc->lock);
1523                 dwc->gadget_driver->disconnect(&dwc->gadget);
1524                 spin_lock(&dwc->lock);
1525         }
1526 }
1527
1528 static void dwc3_stop_active_transfer(struct dwc3 *dwc, u32 epnum)
1529 {
1530         struct dwc3_ep *dep;
1531         struct dwc3_gadget_ep_cmd_params params;
1532         u32 cmd;
1533         int ret;
1534
1535         dep = dwc->eps[epnum];
1536
1537         WARN_ON(!dep->res_trans_idx);
1538         if (dep->res_trans_idx) {
1539                 cmd = DWC3_DEPCMD_ENDTRANSFER;
1540                 cmd |= DWC3_DEPCMD_HIPRI_FORCERM | DWC3_DEPCMD_CMDIOC;
1541                 cmd |= DWC3_DEPCMD_PARAM(dep->res_trans_idx);
1542                 memset(&params, 0, sizeof(params));
1543                 ret = dwc3_send_gadget_ep_cmd(dwc, dep->number, cmd, &params);
1544                 WARN_ON_ONCE(ret);
1545                 dep->res_trans_idx = 0;
1546         }
1547 }
1548
1549 static void dwc3_stop_active_transfers(struct dwc3 *dwc)
1550 {
1551         u32 epnum;
1552
1553         for (epnum = 2; epnum < DWC3_ENDPOINTS_NUM; epnum++) {
1554                 struct dwc3_ep *dep;
1555
1556                 dep = dwc->eps[epnum];
1557                 if (!(dep->flags & DWC3_EP_ENABLED))
1558                         continue;
1559
1560                 dwc3_remove_requests(dwc, dep);
1561         }
1562 }
1563
1564 static void dwc3_clear_stall_all_ep(struct dwc3 *dwc)
1565 {
1566         u32 epnum;
1567
1568         for (epnum = 1; epnum < DWC3_ENDPOINTS_NUM; epnum++) {
1569                 struct dwc3_ep *dep;
1570                 struct dwc3_gadget_ep_cmd_params params;
1571                 int ret;
1572
1573                 dep = dwc->eps[epnum];
1574
1575                 if (!(dep->flags & DWC3_EP_STALL))
1576                         continue;
1577
1578                 dep->flags &= ~DWC3_EP_STALL;
1579
1580                 memset(&params, 0, sizeof(params));
1581                 ret = dwc3_send_gadget_ep_cmd(dwc, dep->number,
1582                                 DWC3_DEPCMD_CLEARSTALL, &params);
1583                 WARN_ON_ONCE(ret);
1584         }
1585 }
1586
1587 static void dwc3_gadget_disconnect_interrupt(struct dwc3 *dwc)
1588 {
1589         dev_vdbg(dwc->dev, "%s\n", __func__);
1590 #if 0
1591         XXX
1592         U1/U2 is powersave optimization. Skip it for now. Anyway we need to
1593         enable it before we can disable it.
1594
1595         reg = dwc3_readl(dwc->regs, DWC3_DCTL);
1596         reg &= ~DWC3_DCTL_INITU1ENA;
1597         dwc3_writel(dwc->regs, DWC3_DCTL, reg);
1598
1599         reg &= ~DWC3_DCTL_INITU2ENA;
1600         dwc3_writel(dwc->regs, DWC3_DCTL, reg);
1601 #endif
1602
1603         dwc3_stop_active_transfers(dwc);
1604         dwc3_disconnect_gadget(dwc);
1605         dwc->start_config_issued = false;
1606
1607         dwc->gadget.speed = USB_SPEED_UNKNOWN;
1608 }
1609
1610 static void dwc3_gadget_usb3_phy_power(struct dwc3 *dwc, int on)
1611 {
1612         u32                     reg;
1613
1614         reg = dwc3_readl(dwc->regs, DWC3_GUSB3PIPECTL(0));
1615
1616         if (on)
1617                 reg &= ~DWC3_GUSB3PIPECTL_SUSPHY;
1618         else
1619                 reg |= DWC3_GUSB3PIPECTL_SUSPHY;
1620
1621         dwc3_writel(dwc->regs, DWC3_GUSB3PIPECTL(0), reg);
1622 }
1623
1624 static void dwc3_gadget_usb2_phy_power(struct dwc3 *dwc, int on)
1625 {
1626         u32                     reg;
1627
1628         reg = dwc3_readl(dwc->regs, DWC3_GUSB2PHYCFG(0));
1629
1630         if (on)
1631                 reg &= ~DWC3_GUSB2PHYCFG_SUSPHY;
1632         else
1633                 reg |= DWC3_GUSB2PHYCFG_SUSPHY;
1634
1635         dwc3_writel(dwc->regs, DWC3_GUSB2PHYCFG(0), reg);
1636 }
1637
1638 static void dwc3_gadget_reset_interrupt(struct dwc3 *dwc)
1639 {
1640         u32                     reg;
1641
1642         dev_vdbg(dwc->dev, "%s\n", __func__);
1643
1644         /* Enable PHYs */
1645         dwc3_gadget_usb2_phy_power(dwc, true);
1646         dwc3_gadget_usb3_phy_power(dwc, true);
1647
1648         if (dwc->gadget.speed != USB_SPEED_UNKNOWN)
1649                 dwc3_disconnect_gadget(dwc);
1650
1651         reg = dwc3_readl(dwc->regs, DWC3_DCTL);
1652         reg &= ~DWC3_DCTL_TSTCTRL_MASK;
1653         dwc3_writel(dwc->regs, DWC3_DCTL, reg);
1654
1655         dwc3_stop_active_transfers(dwc);
1656         dwc3_clear_stall_all_ep(dwc);
1657         dwc->start_config_issued = false;
1658
1659         /* Reset device address to zero */
1660         reg = dwc3_readl(dwc->regs, DWC3_DCFG);
1661         reg &= ~(DWC3_DCFG_DEVADDR_MASK);
1662         dwc3_writel(dwc->regs, DWC3_DCFG, reg);
1663 }
1664
1665 static void dwc3_update_ram_clk_sel(struct dwc3 *dwc, u32 speed)
1666 {
1667         u32 reg;
1668         u32 usb30_clock = DWC3_GCTL_CLK_BUS;
1669
1670         /*
1671          * We change the clock only at SS but I dunno why I would want to do
1672          * this. Maybe it becomes part of the power saving plan.
1673          */
1674
1675         if (speed != DWC3_DSTS_SUPERSPEED)
1676                 return;
1677
1678         /*
1679          * RAMClkSel is reset to 0 after USB reset, so it must be reprogrammed
1680          * each time on Connect Done.
1681          */
1682         if (!usb30_clock)
1683                 return;
1684
1685         reg = dwc3_readl(dwc->regs, DWC3_GCTL);
1686         reg |= DWC3_GCTL_RAMCLKSEL(usb30_clock);
1687         dwc3_writel(dwc->regs, DWC3_GCTL, reg);
1688 }
1689
1690 static void dwc3_gadget_disable_phy(struct dwc3 *dwc, u8 speed)
1691 {
1692         switch (speed) {
1693         case USB_SPEED_SUPER:
1694                 dwc3_gadget_usb2_phy_power(dwc, false);
1695                 break;
1696         case USB_SPEED_HIGH:
1697         case USB_SPEED_FULL:
1698         case USB_SPEED_LOW:
1699                 dwc3_gadget_usb3_phy_power(dwc, false);
1700                 break;
1701         }
1702 }
1703
1704 static void dwc3_gadget_conndone_interrupt(struct dwc3 *dwc)
1705 {
1706         struct dwc3_gadget_ep_cmd_params params;
1707         struct dwc3_ep          *dep;
1708         int                     ret;
1709         u32                     reg;
1710         u8                      speed;
1711
1712         dev_vdbg(dwc->dev, "%s\n", __func__);
1713
1714         memset(&params, 0x00, sizeof(params));
1715
1716         reg = dwc3_readl(dwc->regs, DWC3_DSTS);
1717         speed = reg & DWC3_DSTS_CONNECTSPD;
1718         dwc->speed = speed;
1719
1720         dwc3_update_ram_clk_sel(dwc, speed);
1721
1722         switch (speed) {
1723         case DWC3_DCFG_SUPERSPEED:
1724                 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(512);
1725                 dwc->gadget.ep0->maxpacket = 512;
1726                 dwc->gadget.speed = USB_SPEED_SUPER;
1727                 break;
1728         case DWC3_DCFG_HIGHSPEED:
1729                 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(64);
1730                 dwc->gadget.ep0->maxpacket = 64;
1731                 dwc->gadget.speed = USB_SPEED_HIGH;
1732                 break;
1733         case DWC3_DCFG_FULLSPEED2:
1734         case DWC3_DCFG_FULLSPEED1:
1735                 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(64);
1736                 dwc->gadget.ep0->maxpacket = 64;
1737                 dwc->gadget.speed = USB_SPEED_FULL;
1738                 break;
1739         case DWC3_DCFG_LOWSPEED:
1740                 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(8);
1741                 dwc->gadget.ep0->maxpacket = 8;
1742                 dwc->gadget.speed = USB_SPEED_LOW;
1743                 break;
1744         }
1745
1746         /* Disable unneded PHY */
1747         dwc3_gadget_disable_phy(dwc, dwc->gadget.speed);
1748
1749         dep = dwc->eps[0];
1750         ret = __dwc3_gadget_ep_enable(dep, &dwc3_gadget_ep0_desc);
1751         if (ret) {
1752                 dev_err(dwc->dev, "failed to enable %s\n", dep->name);
1753                 return;
1754         }
1755
1756         dep = dwc->eps[1];
1757         ret = __dwc3_gadget_ep_enable(dep, &dwc3_gadget_ep0_desc);
1758         if (ret) {
1759                 dev_err(dwc->dev, "failed to enable %s\n", dep->name);
1760                 return;
1761         }
1762
1763         /*
1764          * Configure PHY via GUSB3PIPECTLn if required.
1765          *
1766          * Update GTXFIFOSIZn
1767          *
1768          * In both cases reset values should be sufficient.
1769          */
1770 }
1771
1772 static void dwc3_gadget_wakeup_interrupt(struct dwc3 *dwc)
1773 {
1774         dev_vdbg(dwc->dev, "%s\n", __func__);
1775
1776         /*
1777          * TODO take core out of low power mode when that's
1778          * implemented.
1779          */
1780
1781         dwc->gadget_driver->resume(&dwc->gadget);
1782 }
1783
1784 static void dwc3_gadget_linksts_change_interrupt(struct dwc3 *dwc,
1785                 unsigned int evtinfo)
1786 {
1787         /*  The fith bit says SuperSpeed yes or no. */
1788         dwc->link_state = evtinfo & DWC3_LINK_STATE_MASK;
1789
1790         dev_vdbg(dwc->dev, "%s link %d\n", __func__, dwc->link_state);
1791 }
1792
1793 static void dwc3_gadget_interrupt(struct dwc3 *dwc,
1794                 const struct dwc3_event_devt *event)
1795 {
1796         switch (event->type) {
1797         case DWC3_DEVICE_EVENT_DISCONNECT:
1798                 dwc3_gadget_disconnect_interrupt(dwc);
1799                 break;
1800         case DWC3_DEVICE_EVENT_RESET:
1801                 dwc3_gadget_reset_interrupt(dwc);
1802                 break;
1803         case DWC3_DEVICE_EVENT_CONNECT_DONE:
1804                 dwc3_gadget_conndone_interrupt(dwc);
1805                 break;
1806         case DWC3_DEVICE_EVENT_WAKEUP:
1807                 dwc3_gadget_wakeup_interrupt(dwc);
1808                 break;
1809         case DWC3_DEVICE_EVENT_LINK_STATUS_CHANGE:
1810                 dwc3_gadget_linksts_change_interrupt(dwc, event->event_info);
1811                 break;
1812         case DWC3_DEVICE_EVENT_EOPF:
1813                 dev_vdbg(dwc->dev, "End of Periodic Frame\n");
1814                 break;
1815         case DWC3_DEVICE_EVENT_SOF:
1816                 dev_vdbg(dwc->dev, "Start of Periodic Frame\n");
1817                 break;
1818         case DWC3_DEVICE_EVENT_ERRATIC_ERROR:
1819                 dev_vdbg(dwc->dev, "Erratic Error\n");
1820                 break;
1821         case DWC3_DEVICE_EVENT_CMD_CMPL:
1822                 dev_vdbg(dwc->dev, "Command Complete\n");
1823                 break;
1824         case DWC3_DEVICE_EVENT_OVERFLOW:
1825                 dev_vdbg(dwc->dev, "Overflow\n");
1826                 break;
1827         default:
1828                 dev_dbg(dwc->dev, "UNKNOWN IRQ %d\n", event->type);
1829         }
1830 }
1831
1832 static void dwc3_process_event_entry(struct dwc3 *dwc,
1833                 const union dwc3_event *event)
1834 {
1835         /* Endpoint IRQ, handle it and return early */
1836         if (event->type.is_devspec == 0) {
1837                 /* depevt */
1838                 return dwc3_endpoint_interrupt(dwc, &event->depevt);
1839         }
1840
1841         switch (event->type.type) {
1842         case DWC3_EVENT_TYPE_DEV:
1843                 dwc3_gadget_interrupt(dwc, &event->devt);
1844                 break;
1845         /* REVISIT what to do with Carkit and I2C events ? */
1846         default:
1847                 dev_err(dwc->dev, "UNKNOWN IRQ type %d\n", event->raw);
1848         }
1849 }
1850
1851 static irqreturn_t dwc3_process_event_buf(struct dwc3 *dwc, u32 buf)
1852 {
1853         struct dwc3_event_buffer *evt;
1854         int left;
1855         u32 count;
1856
1857         count = dwc3_readl(dwc->regs, DWC3_GEVNTCOUNT(buf));
1858         count &= DWC3_GEVNTCOUNT_MASK;
1859         if (!count)
1860                 return IRQ_NONE;
1861
1862         evt = dwc->ev_buffs[buf];
1863         left = count;
1864
1865         while (left > 0) {
1866                 union dwc3_event event;
1867
1868                 memcpy(&event.raw, (evt->buf + evt->lpos), sizeof(event.raw));
1869                 dwc3_process_event_entry(dwc, &event);
1870                 /*
1871                  * XXX we wrap around correctly to the next entry as almost all
1872                  * entries are 4 bytes in size. There is one entry which has 12
1873                  * bytes which is a regular entry followed by 8 bytes data. ATM
1874                  * I don't know how things are organized if were get next to the
1875                  * a boundary so I worry about that once we try to handle that.
1876                  */
1877                 evt->lpos = (evt->lpos + 4) % DWC3_EVENT_BUFFERS_SIZE;
1878                 left -= 4;
1879
1880                 dwc3_writel(dwc->regs, DWC3_GEVNTCOUNT(buf), 4);
1881         }
1882
1883         return IRQ_HANDLED;
1884 }
1885
1886 static irqreturn_t dwc3_interrupt(int irq, void *_dwc)
1887 {
1888         struct dwc3                     *dwc = _dwc;
1889         int                             i;
1890         irqreturn_t                     ret = IRQ_NONE;
1891
1892         spin_lock(&dwc->lock);
1893
1894         for (i = 0; i < DWC3_EVENT_BUFFERS_NUM; i++) {
1895                 irqreturn_t status;
1896
1897                 status = dwc3_process_event_buf(dwc, i);
1898                 if (status == IRQ_HANDLED)
1899                         ret = status;
1900         }
1901
1902         spin_unlock(&dwc->lock);
1903
1904         return ret;
1905 }
1906
1907 /**
1908  * dwc3_gadget_init - Initializes gadget related registers
1909  * @dwc: Pointer to out controller context structure
1910  *
1911  * Returns 0 on success otherwise negative errno.
1912  */
1913 int __devinit dwc3_gadget_init(struct dwc3 *dwc)
1914 {
1915         u32                                     reg;
1916         int                                     ret;
1917         int                                     irq;
1918
1919         dwc->ctrl_req = dma_alloc_coherent(dwc->dev, sizeof(*dwc->ctrl_req),
1920                         &dwc->ctrl_req_addr, GFP_KERNEL);
1921         if (!dwc->ctrl_req) {
1922                 dev_err(dwc->dev, "failed to allocate ctrl request\n");
1923                 ret = -ENOMEM;
1924                 goto err0;
1925         }
1926
1927         dwc->ep0_trb = dma_alloc_coherent(dwc->dev, sizeof(*dwc->ep0_trb),
1928                         &dwc->ep0_trb_addr, GFP_KERNEL);
1929         if (!dwc->ep0_trb) {
1930                 dev_err(dwc->dev, "failed to allocate ep0 trb\n");
1931                 ret = -ENOMEM;
1932                 goto err1;
1933         }
1934
1935         dwc->setup_buf = dma_alloc_coherent(dwc->dev,
1936                         sizeof(*dwc->setup_buf) * 2,
1937                         &dwc->setup_buf_addr, GFP_KERNEL);
1938         if (!dwc->setup_buf) {
1939                 dev_err(dwc->dev, "failed to allocate setup buffer\n");
1940                 ret = -ENOMEM;
1941                 goto err2;
1942         }
1943
1944         dwc->ep0_bounce = dma_alloc_coherent(dwc->dev,
1945                         512, &dwc->ep0_bounce_addr, GFP_KERNEL);
1946         if (!dwc->ep0_bounce) {
1947                 dev_err(dwc->dev, "failed to allocate ep0 bounce buffer\n");
1948                 ret = -ENOMEM;
1949                 goto err3;
1950         }
1951
1952         dev_set_name(&dwc->gadget.dev, "gadget");
1953
1954         dwc->gadget.ops                 = &dwc3_gadget_ops;
1955         dwc->gadget.is_dualspeed        = true;
1956         dwc->gadget.speed               = USB_SPEED_UNKNOWN;
1957         dwc->gadget.dev.parent          = dwc->dev;
1958
1959         dma_set_coherent_mask(&dwc->gadget.dev, dwc->dev->coherent_dma_mask);
1960
1961         dwc->gadget.dev.dma_parms       = dwc->dev->dma_parms;
1962         dwc->gadget.dev.dma_mask        = dwc->dev->dma_mask;
1963         dwc->gadget.dev.release         = dwc3_gadget_release;
1964         dwc->gadget.name                = "dwc3-gadget";
1965
1966         /*
1967          * REVISIT: Here we should clear all pending IRQs to be
1968          * sure we're starting from a well known location.
1969          */
1970
1971         ret = dwc3_gadget_init_endpoints(dwc);
1972         if (ret)
1973                 goto err4;
1974
1975         irq = platform_get_irq(to_platform_device(dwc->dev), 0);
1976
1977         ret = request_irq(irq, dwc3_interrupt, IRQF_SHARED,
1978                         "dwc3", dwc);
1979         if (ret) {
1980                 dev_err(dwc->dev, "failed to request irq #%d --> %d\n",
1981                                 irq, ret);
1982                 goto err5;
1983         }
1984
1985         /* Enable all but Start and End of Frame IRQs */
1986         reg = (DWC3_DEVTEN_VNDRDEVTSTRCVEDEN |
1987                         DWC3_DEVTEN_EVNTOVERFLOWEN |
1988                         DWC3_DEVTEN_CMDCMPLTEN |
1989                         DWC3_DEVTEN_ERRTICERREN |
1990                         DWC3_DEVTEN_WKUPEVTEN |
1991                         DWC3_DEVTEN_ULSTCNGEN |
1992                         DWC3_DEVTEN_CONNECTDONEEN |
1993                         DWC3_DEVTEN_USBRSTEN |
1994                         DWC3_DEVTEN_DISCONNEVTEN);
1995         dwc3_writel(dwc->regs, DWC3_DEVTEN, reg);
1996
1997         ret = device_register(&dwc->gadget.dev);
1998         if (ret) {
1999                 dev_err(dwc->dev, "failed to register gadget device\n");
2000                 put_device(&dwc->gadget.dev);
2001                 goto err6;
2002         }
2003
2004         ret = usb_add_gadget_udc(dwc->dev, &dwc->gadget);
2005         if (ret) {
2006                 dev_err(dwc->dev, "failed to register udc\n");
2007                 goto err7;
2008         }
2009
2010         return 0;
2011
2012 err7:
2013         device_unregister(&dwc->gadget.dev);
2014
2015 err6:
2016         dwc3_writel(dwc->regs, DWC3_DEVTEN, 0x00);
2017         free_irq(irq, dwc);
2018
2019 err5:
2020         dwc3_gadget_free_endpoints(dwc);
2021
2022 err4:
2023         dma_free_coherent(dwc->dev, 512, dwc->ep0_bounce,
2024                         dwc->ep0_bounce_addr);
2025
2026 err3:
2027         dma_free_coherent(dwc->dev, sizeof(*dwc->setup_buf) * 2,
2028                         dwc->setup_buf, dwc->setup_buf_addr);
2029
2030 err2:
2031         dma_free_coherent(dwc->dev, sizeof(*dwc->ep0_trb),
2032                         dwc->ep0_trb, dwc->ep0_trb_addr);
2033
2034 err1:
2035         dma_free_coherent(dwc->dev, sizeof(*dwc->ctrl_req),
2036                         dwc->ctrl_req, dwc->ctrl_req_addr);
2037
2038 err0:
2039         return ret;
2040 }
2041
2042 void dwc3_gadget_exit(struct dwc3 *dwc)
2043 {
2044         int                     irq;
2045         int                     i;
2046
2047         usb_del_gadget_udc(&dwc->gadget);
2048         irq = platform_get_irq(to_platform_device(dwc->dev), 0);
2049
2050         dwc3_writel(dwc->regs, DWC3_DEVTEN, 0x00);
2051         free_irq(irq, dwc);
2052
2053         for (i = 0; i < ARRAY_SIZE(dwc->eps); i++)
2054                 __dwc3_gadget_ep_disable(dwc->eps[i]);
2055
2056         dwc3_gadget_free_endpoints(dwc);
2057
2058         dma_free_coherent(dwc->dev, 512, dwc->ep0_bounce,
2059                         dwc->ep0_bounce_addr);
2060
2061         dma_free_coherent(dwc->dev, sizeof(*dwc->setup_buf) * 2,
2062                         dwc->setup_buf, dwc->setup_buf_addr);
2063
2064         dma_free_coherent(dwc->dev, sizeof(*dwc->ep0_trb),
2065                         dwc->ep0_trb, dwc->ep0_trb_addr);
2066
2067         dma_free_coherent(dwc->dev, sizeof(*dwc->ctrl_req),
2068                         dwc->ctrl_req, dwc->ctrl_req_addr);
2069
2070         device_unregister(&dwc->gadget.dev);
2071 }