drivers/usb/gadget/udc/net2280.c

   1 /*
   2  * Driver for the PLX NET2280 USB device controller.
   3  * Specs and errata are available from <http://www.plxtech.com>.
   4  *
   5  * PLX Technology Inc. (formerly NetChip Technology) supported the
   6  * development of this driver.
   7  *
   8  *
   9  * CODE STATUS HIGHLIGHTS
  10  *
  11  * This driver should work well with most "gadget" drivers, including
  12  * the Mass Storage, Serial, and Ethernet/RNDIS gadget drivers
  13  * as well as Gadget Zero and Gadgetfs.
  14  *
  15  * DMA is enabled by default.
  16  *
  17  * MSI is enabled by default.  The legacy IRQ is used if MSI couldn't
  18  * be enabled.
  19  *
  20  * Note that almost all the errata workarounds here are only needed for
  21  * rev1 chips.  Rev1a silicon (0110) fixes almost all of them.
  22  */
  23
  24 /*
  25  * Copyright (C) 2003 David Brownell
  26  * Copyright (C) 2003-2005 PLX Technology, Inc.
  27  * Copyright (C) 2014 Ricardo Ribalda - Qtechnology/AS
  28  *
  29  * Modified Seth Levy 2005 PLX Technology, Inc. to provide compatibility
  30  *      with 2282 chip
  31  *
  32  * Modified Ricardo Ribalda Qtechnology AS  to provide compatibility
  33  *      with usb 338x chip. Based on PLX driver
  34  *
  35  * This program is free software; you can redistribute it and/or modify
  36  * it under the terms of the GNU General Public License as published by
  37  * the Free Software Foundation; either version 2 of the License, or
  38  * (at your option) any later version.
  39  */
  40
  41 #include <linux/module.h>
  42 #include <linux/pci.h>
  43 #include <linux/dma-mapping.h>
  44 #include <linux/kernel.h>
  45 #include <linux/delay.h>
  46 #include <linux/ioport.h>
  47 #include <linux/slab.h>
  48 #include <linux/errno.h>
  49 #include <linux/init.h>
  50 #include <linux/timer.h>
  51 #include <linux/list.h>
  52 #include <linux/interrupt.h>
  53 #include <linux/moduleparam.h>
  54 #include <linux/device.h>
  55 #include <linux/usb/ch9.h>
  56 #include <linux/usb/gadget.h>
  57 #include <linux/prefetch.h>
  58 #include <linux/io.h>
  59
  60 #include <asm/byteorder.h>
  61 #include <asm/irq.h>
  62 #include <asm/unaligned.h>
  63
  64 #define DRIVER_DESC             "PLX NET228x/USB338x USB Peripheral Controller"
  65 #define DRIVER_VERSION          "2005 Sept 27/v3.0"
  66
  67 #define EP_DONTUSE              13      /* nonzero */
  68
  69 #define USE_RDK_LEDS            /* GPIO pins control three LEDs */
  70
  71
  72 static const char driver_name[] = "net2280";
  73 static const char driver_desc[] = DRIVER_DESC;
  74
  75 static const u32 ep_bit[9] = { 0, 17, 2, 19, 4, 1, 18, 3, 20 };
  76 static const char ep0name[] = "ep0";
  77
  78 #define EP_INFO(_name, _caps) \
  79         { \
  80                 .name = _name, \
  81                 .caps = _caps, \
  82         }
  83
  84 static const struct {
  85         const char *name;
  86         const struct usb_ep_caps caps;
  87 } ep_info_dft[] = { /* Default endpoint configuration */
  88         EP_INFO(ep0name,
  89                 USB_EP_CAPS(USB_EP_CAPS_TYPE_CONTROL, USB_EP_CAPS_DIR_ALL)),
  90         EP_INFO("ep-a",
  91                 USB_EP_CAPS(USB_EP_CAPS_TYPE_ALL, USB_EP_CAPS_DIR_ALL)),
  92         EP_INFO("ep-b",
  93                 USB_EP_CAPS(USB_EP_CAPS_TYPE_ALL, USB_EP_CAPS_DIR_ALL)),
  94         EP_INFO("ep-c",
  95                 USB_EP_CAPS(USB_EP_CAPS_TYPE_ALL, USB_EP_CAPS_DIR_ALL)),
  96         EP_INFO("ep-d",
  97                 USB_EP_CAPS(USB_EP_CAPS_TYPE_ALL, USB_EP_CAPS_DIR_ALL)),
  98         EP_INFO("ep-e",
  99                 USB_EP_CAPS(USB_EP_CAPS_TYPE_ALL, USB_EP_CAPS_DIR_ALL)),
 100         EP_INFO("ep-f",
 101                 USB_EP_CAPS(USB_EP_CAPS_TYPE_ALL, USB_EP_CAPS_DIR_ALL)),
 102         EP_INFO("ep-g",
 103                 USB_EP_CAPS(USB_EP_CAPS_TYPE_ALL, USB_EP_CAPS_DIR_ALL)),
 104         EP_INFO("ep-h",
 105                 USB_EP_CAPS(USB_EP_CAPS_TYPE_ALL, USB_EP_CAPS_DIR_ALL)),
 106 }, ep_info_adv[] = { /* Endpoints for usb3380 advance mode */
 107         EP_INFO(ep0name,
 108                 USB_EP_CAPS(USB_EP_CAPS_TYPE_CONTROL, USB_EP_CAPS_DIR_ALL)),
 109         EP_INFO("ep1in",
 110                 USB_EP_CAPS(USB_EP_CAPS_TYPE_ALL, USB_EP_CAPS_DIR_IN)),
 111         EP_INFO("ep2out",
 112                 USB_EP_CAPS(USB_EP_CAPS_TYPE_ALL, USB_EP_CAPS_DIR_OUT)),
 113         EP_INFO("ep3in",
 114                 USB_EP_CAPS(USB_EP_CAPS_TYPE_ALL, USB_EP_CAPS_DIR_IN)),
 115         EP_INFO("ep4out",
 116                 USB_EP_CAPS(USB_EP_CAPS_TYPE_ALL, USB_EP_CAPS_DIR_OUT)),
 117         EP_INFO("ep1out",
 118                 USB_EP_CAPS(USB_EP_CAPS_TYPE_ALL, USB_EP_CAPS_DIR_OUT)),
 119         EP_INFO("ep2in",
 120                 USB_EP_CAPS(USB_EP_CAPS_TYPE_ALL, USB_EP_CAPS_DIR_IN)),
 121         EP_INFO("ep3out",
 122                 USB_EP_CAPS(USB_EP_CAPS_TYPE_ALL, USB_EP_CAPS_DIR_OUT)),
 123         EP_INFO("ep4in",
 124                 USB_EP_CAPS(USB_EP_CAPS_TYPE_ALL, USB_EP_CAPS_DIR_IN)),
 125 };
 126
 127 #undef EP_INFO
 128
 129 /* mode 0 == ep-{a,b,c,d} 1K fifo each
 130  * mode 1 == ep-{a,b} 2K fifo each, ep-{c,d} unavailable
 131  * mode 2 == ep-a 2K fifo, ep-{b,c} 1K each, ep-d unavailable
 132  */
 133 static ushort fifo_mode;
 134
 135 /* "modprobe net2280 fifo_mode=1" etc */
 136 module_param(fifo_mode, ushort, 0644);
 137
 138 /* enable_suspend -- When enabled, the driver will respond to
 139  * USB suspend requests by powering down the NET2280.  Otherwise,
 140  * USB suspend requests will be ignored.  This is acceptable for
 141  * self-powered devices
 142  */
 143 static bool enable_suspend;
 144
 145 /* "modprobe net2280 enable_suspend=1" etc */
 146 module_param(enable_suspend, bool, 0444);
 147
 148 #define DIR_STRING(bAddress) (((bAddress) & USB_DIR_IN) ? "in" : "out")
 149
 150 static char *type_string(u8 bmAttributes)
 151 {
 152         switch ((bmAttributes) & USB_ENDPOINT_XFERTYPE_MASK) {
 153         case USB_ENDPOINT_XFER_BULK:    return "bulk";
 154         case USB_ENDPOINT_XFER_ISOC:    return "iso";
 155         case USB_ENDPOINT_XFER_INT:     return "intr";
 156         }
 157         return "control";
 158 }
 159
 160 #include "net2280.h"
 161
 162 #define valid_bit       cpu_to_le32(BIT(VALID_BIT))
 163 #define dma_done_ie     cpu_to_le32(BIT(DMA_DONE_INTERRUPT_ENABLE))
 164
 165 static void ep_clear_seqnum(struct net2280_ep *ep);
 166 static void stop_activity(struct net2280 *dev,
 167                                         struct usb_gadget_driver *driver);
 168 static void ep0_start(struct net2280 *dev);
 169
 170 /*-------------------------------------------------------------------------*/
 171 static inline void enable_pciirqenb(struct net2280_ep *ep)
 172 {
 173         u32 tmp = readl(&ep->dev->regs->pciirqenb0);
 174
 175         if (ep->dev->quirks & PLX_LEGACY)
 176                 tmp |= BIT(ep->num);
 177         else
 178                 tmp |= BIT(ep_bit[ep->num]);
 179         writel(tmp, &ep->dev->regs->pciirqenb0);
 180
 181         return;
 182 }
 183
 184 static int
 185 net2280_enable(struct usb_ep *_ep, const struct usb_endpoint_descriptor *desc)
 186 {
 187         struct net2280          *dev;
 188         struct net2280_ep       *ep;
 189         u32                     max;
 190         u32 tmp = 0;
 191         u32 type;
 192         unsigned long           flags;
 193         static const u32 ep_key[9] = { 1, 0, 1, 0, 1, 1, 0, 1, 0 };
 194         int ret = 0;
 195
 196         ep = container_of(_ep, struct net2280_ep, ep);
 197         if (!_ep || !desc || ep->desc || _ep->name == ep0name ||
 198                         desc->bDescriptorType != USB_DT_ENDPOINT) {
 199                 pr_err("%s: failed at line=%d\n", __func__, __LINE__);
 200                 return -EINVAL;
 201         }
 202         dev = ep->dev;
 203         if (!dev->driver || dev->gadget.speed == USB_SPEED_UNKNOWN) {
 204                 ret = -ESHUTDOWN;
 205                 goto print_err;
 206         }
 207
 208         /* erratum 0119 workaround ties up an endpoint number */
 209         if ((desc->bEndpointAddress & 0x0f) == EP_DONTUSE) {
 210                 ret = -EDOM;
 211                 goto print_err;
 212         }
 213
 214         if (dev->quirks & PLX_SUPERSPEED) {
 215                 if ((desc->bEndpointAddress & 0x0f) >= 0x0c) {
 216                         ret = -EDOM;
 217                         goto print_err;
 218                 }
 219                 ep->is_in = !!usb_endpoint_dir_in(desc);
 220                 if (dev->enhanced_mode && ep->is_in && ep_key[ep->num]) {
 221                         ret = -EINVAL;
 222                         goto print_err;
 223                 }
 224         }
 225
 226         /* sanity check ep-e/ep-f since their fifos are small */
 227         max = usb_endpoint_maxp(desc) & 0x1fff;
 228         if (ep->num > 4 && max > 64 && (dev->quirks & PLX_LEGACY)) {
 229                 ret = -ERANGE;
 230                 goto print_err;
 231         }
 232
 233         spin_lock_irqsave(&dev->lock, flags);
 234         _ep->maxpacket = max & 0x7ff;
 235         ep->desc = desc;
 236
 237         /* ep_reset() has already been called */
 238         ep->stopped = 0;
 239         ep->wedged = 0;
 240         ep->out_overflow = 0;
 241
 242         /* set speed-dependent max packet; may kick in high bandwidth */
 243         set_max_speed(ep, max);
 244
 245         /* set type, direction, address; reset fifo counters */
 246         writel(BIT(FIFO_FLUSH), &ep->regs->ep_stat);
 247
 248         if ((dev->quirks & PLX_SUPERSPEED) && dev->enhanced_mode) {
 249                 tmp = readl(&ep->cfg->ep_cfg);
 250                 /* If USB ep number doesn't match hardware ep number */
 251                 if ((tmp & 0xf) != usb_endpoint_num(desc)) {
 252                         ret = -EINVAL;
 253                         spin_unlock_irqrestore(&dev->lock, flags);
 254                         goto print_err;
 255                 }
 256                 if (ep->is_in)
 257                         tmp &= ~USB3380_EP_CFG_MASK_IN;
 258                 else
 259                         tmp &= ~USB3380_EP_CFG_MASK_OUT;
 260         }
 261         type = (desc->bmAttributes & USB_ENDPOINT_XFERTYPE_MASK);
 262         if (type == USB_ENDPOINT_XFER_INT) {
 263                 /* erratum 0105 workaround prevents hs NYET */
 264                 if (dev->chiprev == 0100 &&
 265                                 dev->gadget.speed == USB_SPEED_HIGH &&
 266                                 !(desc->bEndpointAddress & USB_DIR_IN))
 267                         writel(BIT(CLEAR_NAK_OUT_PACKETS_MODE),
 268                                 &ep->regs->ep_rsp);
 269         } else if (type == USB_ENDPOINT_XFER_BULK) {
 270                 /* catch some particularly blatant driver bugs */
 271                 if ((dev->gadget.speed == USB_SPEED_SUPER && max != 1024) ||
 272                     (dev->gadget.speed == USB_SPEED_HIGH && max != 512) ||
 273                     (dev->gadget.speed == USB_SPEED_FULL && max > 64)) {
 274                         spin_unlock_irqrestore(&dev->lock, flags);
 275                         ret = -ERANGE;
 276                         goto print_err;
 277                 }
 278         }
 279         ep->is_iso = (type == USB_ENDPOINT_XFER_ISOC);
 280         /* Enable this endpoint */
 281         if (dev->quirks & PLX_LEGACY) {
 282                 tmp |= type << ENDPOINT_TYPE;
 283                 tmp |= desc->bEndpointAddress;
 284                 /* default full fifo lines */
 285                 tmp |= (4 << ENDPOINT_BYTE_COUNT);
 286                 tmp |= BIT(ENDPOINT_ENABLE);
 287                 ep->is_in = (tmp & USB_DIR_IN) != 0;
 288         } else {
 289                 /* In Legacy mode, only OUT endpoints are used */
 290                 if (dev->enhanced_mode && ep->is_in) {
 291                         tmp |= type << IN_ENDPOINT_TYPE;
 292                         tmp |= BIT(IN_ENDPOINT_ENABLE);
 293                 } else {
 294                         tmp |= type << OUT_ENDPOINT_TYPE;
 295                         tmp |= BIT(OUT_ENDPOINT_ENABLE);
 296                         tmp |= (ep->is_in << ENDPOINT_DIRECTION);
 297                 }
 298
 299                 tmp |= (4 << ENDPOINT_BYTE_COUNT);
 300                 if (!dev->enhanced_mode)
 301                         tmp |= usb_endpoint_num(desc);
 302                 tmp |= (ep->ep.maxburst << MAX_BURST_SIZE);
 303         }
 304
 305         /* Make sure all the registers are written before ep_rsp*/
 306         wmb();
 307
 308         /* for OUT transfers, block the rx fifo until a read is posted */
 309         if (!ep->is_in)
 310                 writel(BIT(SET_NAK_OUT_PACKETS), &ep->regs->ep_rsp);
 311         else if (!(dev->quirks & PLX_2280)) {
 312                 /* Added for 2282, Don't use nak packets on an in endpoint,
 313                  * this was ignored on 2280
 314                  */
 315                 writel(BIT(CLEAR_NAK_OUT_PACKETS) |
 316                         BIT(CLEAR_NAK_OUT_PACKETS_MODE), &ep->regs->ep_rsp);
 317         }
 318
 319         if (dev->quirks & PLX_SUPERSPEED)
 320                 ep_clear_seqnum(ep);
 321         writel(tmp, &ep->cfg->ep_cfg);
 322
 323         /* enable irqs */
 324         if (!ep->dma) {                         /* pio, per-packet */
 325                 enable_pciirqenb(ep);
 326
 327                 tmp = BIT(DATA_PACKET_RECEIVED_INTERRUPT_ENABLE) |
 328                         BIT(DATA_PACKET_TRANSMITTED_INTERRUPT_ENABLE);
 329                 if (dev->quirks & PLX_2280)
 330                         tmp |= readl(&ep->regs->ep_irqenb);
 331                 writel(tmp, &ep->regs->ep_irqenb);
 332         } else {                                /* dma, per-request */
 333                 tmp = BIT((8 + ep->num));       /* completion */
 334                 tmp |= readl(&dev->regs->pciirqenb1);
 335                 writel(tmp, &dev->regs->pciirqenb1);
 336
 337                 /* for short OUT transfers, dma completions can't
 338                  * advance the queue; do it pio-style, by hand.
 339                  * NOTE erratum 0112 workaround #2
 340                  */
 341                 if ((desc->bEndpointAddress & USB_DIR_IN) == 0) {
 342                         tmp = BIT(SHORT_PACKET_TRANSFERRED_INTERRUPT_ENABLE);
 343                         writel(tmp, &ep->regs->ep_irqenb);
 344
 345                         enable_pciirqenb(ep);
 346                 }
 347         }
 348
 349         tmp = desc->bEndpointAddress;
 350         ep_dbg(dev, "enabled %s (ep%d%s-%s) %s max %04x\n",
 351                 _ep->name, tmp & 0x0f, DIR_STRING(tmp),
 352                 type_string(desc->bmAttributes),
 353                 ep->dma ? "dma" : "pio", max);
 354
 355         /* pci writes may still be posted */
 356         spin_unlock_irqrestore(&dev->lock, flags);
 357         return ret;
 358
 359 print_err:
 360         dev_err(&ep->dev->pdev->dev, "%s: error=%d\n", __func__, ret);
 361         return ret;
 362 }
 363
 364 static int handshake(u32 __iomem *ptr, u32 mask, u32 done, int usec)
 365 {
 366         u32     result;
 367
 368         do {
 369                 result = readl(ptr);
 370                 if (result == ~(u32)0)          /* "device unplugged" */
 371                         return -ENODEV;
 372                 result &= mask;
 373                 if (result == done)
 374                         return 0;
 375                 udelay(1);
 376                 usec--;
 377         } while (usec > 0);
 378         return -ETIMEDOUT;
 379 }
 380
 381 static const struct usb_ep_ops net2280_ep_ops;
 382
 383 static void ep_reset_228x(struct net2280_regs __iomem *regs,
 384                           struct net2280_ep *ep)
 385 {
 386         u32             tmp;
 387
 388         ep->desc = NULL;
 389         INIT_LIST_HEAD(&ep->queue);
 390
 391         usb_ep_set_maxpacket_limit(&ep->ep, ~0);
 392         ep->ep.ops = &net2280_ep_ops;
 393
 394         /* disable the dma, irqs, endpoint... */
 395         if (ep->dma) {
 396                 writel(0, &ep->dma->dmactl);
 397                 writel(BIT(DMA_SCATTER_GATHER_DONE_INTERRUPT) |
 398                         BIT(DMA_TRANSACTION_DONE_INTERRUPT) |
 399                         BIT(DMA_ABORT),
 400                         &ep->dma->dmastat);
 401
 402                 tmp = readl(&regs->pciirqenb0);
 403                 tmp &= ~BIT(ep->num);
 404                 writel(tmp, &regs->pciirqenb0);
 405         } else {
 406                 tmp = readl(&regs->pciirqenb1);
 407                 tmp &= ~BIT((8 + ep->num));     /* completion */
 408                 writel(tmp, &regs->pciirqenb1);
 409         }
 410         writel(0, &ep->regs->ep_irqenb);
 411
 412         /* init to our chosen defaults, notably so that we NAK OUT
 413          * packets until the driver queues a read (+note erratum 0112)
 414          */
 415         if (!ep->is_in || (ep->dev->quirks & PLX_2280)) {
 416                 tmp = BIT(SET_NAK_OUT_PACKETS_MODE) |
 417                 BIT(SET_NAK_OUT_PACKETS) |
 418                 BIT(CLEAR_EP_HIDE_STATUS_PHASE) |
 419                 BIT(CLEAR_INTERRUPT_MODE);
 420         } else {
 421                 /* added for 2282 */
 422                 tmp = BIT(CLEAR_NAK_OUT_PACKETS_MODE) |
 423                 BIT(CLEAR_NAK_OUT_PACKETS) |
 424                 BIT(CLEAR_EP_HIDE_STATUS_PHASE) |
 425                 BIT(CLEAR_INTERRUPT_MODE);
 426         }
 427
 428         if (ep->num != 0) {
 429                 tmp |= BIT(CLEAR_ENDPOINT_TOGGLE) |
 430                         BIT(CLEAR_ENDPOINT_HALT);
 431         }
 432         writel(tmp, &ep->regs->ep_rsp);
 433
 434         /* scrub most status bits, and flush any fifo state */
 435         if (ep->dev->quirks & PLX_2280)
 436                 tmp = BIT(FIFO_OVERFLOW) |
 437                         BIT(FIFO_UNDERFLOW);
 438         else
 439                 tmp = 0;
 440
 441         writel(tmp | BIT(TIMEOUT) |
 442                 BIT(USB_STALL_SENT) |
 443                 BIT(USB_IN_NAK_SENT) |
 444                 BIT(USB_IN_ACK_RCVD) |
 445                 BIT(USB_OUT_PING_NAK_SENT) |
 446                 BIT(USB_OUT_ACK_SENT) |
 447                 BIT(FIFO_FLUSH) |
 448                 BIT(SHORT_PACKET_OUT_DONE_INTERRUPT) |
 449                 BIT(SHORT_PACKET_TRANSFERRED_INTERRUPT) |
 450                 BIT(DATA_PACKET_RECEIVED_INTERRUPT) |
 451                 BIT(DATA_PACKET_TRANSMITTED_INTERRUPT) |
 452                 BIT(DATA_OUT_PING_TOKEN_INTERRUPT) |
 453                 BIT(DATA_IN_TOKEN_INTERRUPT),
 454                 &ep->regs->ep_stat);
 455
 456         /* fifo size is handled separately */
 457 }
 458
 459 static void ep_reset_338x(struct net2280_regs __iomem *regs,
 460                                         struct net2280_ep *ep)
 461 {
 462         u32 tmp, dmastat;
 463
 464         ep->desc = NULL;
 465         INIT_LIST_HEAD(&ep->queue);
 466
 467         usb_ep_set_maxpacket_limit(&ep->ep, ~0);
 468         ep->ep.ops = &net2280_ep_ops;
 469
 470         /* disable the dma, irqs, endpoint... */
 471         if (ep->dma) {
 472                 writel(0, &ep->dma->dmactl);
 473                 writel(BIT(DMA_ABORT_DONE_INTERRUPT) |
 474                        BIT(DMA_PAUSE_DONE_INTERRUPT) |
 475                        BIT(DMA_SCATTER_GATHER_DONE_INTERRUPT) |
 476                        BIT(DMA_TRANSACTION_DONE_INTERRUPT),
 477                        /* | BIT(DMA_ABORT), */
 478                        &ep->dma->dmastat);
 479
 480                 dmastat = readl(&ep->dma->dmastat);
 481                 if (dmastat == 0x5002) {
 482                         ep_warn(ep->dev, "The dmastat return = %x!!\n",
 483                                dmastat);
 484                         writel(0x5a, &ep->dma->dmastat);
 485                 }
 486
 487                 tmp = readl(&regs->pciirqenb0);
 488                 tmp &= ~BIT(ep_bit[ep->num]);
 489                 writel(tmp, &regs->pciirqenb0);
 490         } else {
 491                 if (ep->num < 5) {
 492                         tmp = readl(&regs->pciirqenb1);
 493                         tmp &= ~BIT((8 + ep->num));     /* completion */
 494                         writel(tmp, &regs->pciirqenb1);
 495                 }
 496         }
 497         writel(0, &ep->regs->ep_irqenb);
 498
 499         writel(BIT(SHORT_PACKET_OUT_DONE_INTERRUPT) |
 500                BIT(SHORT_PACKET_TRANSFERRED_INTERRUPT) |
 501                BIT(FIFO_OVERFLOW) |
 502                BIT(DATA_PACKET_RECEIVED_INTERRUPT) |
 503                BIT(DATA_PACKET_TRANSMITTED_INTERRUPT) |
 504                BIT(DATA_OUT_PING_TOKEN_INTERRUPT) |
 505                BIT(DATA_IN_TOKEN_INTERRUPT), &ep->regs->ep_stat);
 506
 507         tmp = readl(&ep->cfg->ep_cfg);
 508         if (ep->is_in)
 509                 tmp &= ~USB3380_EP_CFG_MASK_IN;
 510         else
 511                 tmp &= ~USB3380_EP_CFG_MASK_OUT;
 512         writel(tmp, &ep->cfg->ep_cfg);
 513 }
 514
 515 static void nuke(struct net2280_ep *);
 516
 517 static int net2280_disable(struct usb_ep *_ep)
 518 {
 519         struct net2280_ep       *ep;
 520         unsigned long           flags;
 521
 522         ep = container_of(_ep, struct net2280_ep, ep);
 523         if (!_ep || !ep->desc || _ep->name == ep0name) {
 524                 pr_err("%s: Invalid ep=%p or ep->desc\n", __func__, _ep);
 525                 return -EINVAL;
 526         }
 527         spin_lock_irqsave(&ep->dev->lock, flags);
 528         nuke(ep);
 529
 530         if (ep->dev->quirks & PLX_SUPERSPEED)
 531                 ep_reset_338x(ep->dev->regs, ep);
 532         else
 533                 ep_reset_228x(ep->dev->regs, ep);
 534
 535         ep_vdbg(ep->dev, "disabled %s %s\n",
 536                         ep->dma ? "dma" : "pio", _ep->name);
 537
 538         /* synch memory views with the device */
 539         (void)readl(&ep->cfg->ep_cfg);
 540
 541         if (!ep->dma && ep->num >= 1 && ep->num <= 4)
 542                 ep->dma = &ep->dev->dma[ep->num - 1];
 543
 544         spin_unlock_irqrestore(&ep->dev->lock, flags);
 545         return 0;
 546 }
 547
 548 /*-------------------------------------------------------------------------*/
 549
 550 static struct usb_request
 551 *net2280_alloc_request(struct usb_ep *_ep, gfp_t gfp_flags)
 552 {
 553         struct net2280_ep       *ep;
 554         struct net2280_request  *req;
 555
 556         if (!_ep) {
 557                 pr_err("%s: Invalid ep\n", __func__);
 558                 return NULL;
 559         }
 560         ep = container_of(_ep, struct net2280_ep, ep);
 561
 562         req = kzalloc(sizeof(*req), gfp_flags);
 563         if (!req)
 564                 return NULL;
 565
 566         INIT_LIST_HEAD(&req->queue);
 567
 568         /* this dma descriptor may be swapped with the previous dummy */
 569         if (ep->dma) {
 570                 struct net2280_dma      *td;
 571
 572                 td = pci_pool_alloc(ep->dev->requests, gfp_flags,
 573                                 &req->td_dma);
 574                 if (!td) {
 575                         kfree(req);
 576                         return NULL;
 577                 }
 578                 td->dmacount = 0;       /* not VALID */
 579                 td->dmadesc = td->dmaaddr;
 580                 req->td = td;
 581         }
 582         return &req->req;
 583 }
 584
 585 static void net2280_free_request(struct usb_ep *_ep, struct usb_request *_req)
 586 {
 587         struct net2280_ep       *ep;
 588         struct net2280_request  *req;
 589
 590         ep = container_of(_ep, struct net2280_ep, ep);
 591         if (!_ep || !_req) {
 592                 dev_err(&ep->dev->pdev->dev, "%s: Inavlid ep=%p or req=%p\n",
 593                                                         __func__, _ep, _req);
 594                 return;
 595         }
 596
 597         req = container_of(_req, struct net2280_request, req);
 598         WARN_ON(!list_empty(&req->queue));
 599         if (req->td)
 600                 pci_pool_free(ep->dev->requests, req->td, req->td_dma);
 601         kfree(req);
 602 }
 603
 604 /*-------------------------------------------------------------------------*/
 605
 606 /* load a packet into the fifo we use for usb IN transfers.
 607  * works for all endpoints.
 608  *
 609  * NOTE: pio with ep-a..ep-d could stuff multiple packets into the fifo
 610  * at a time, but this code is simpler because it knows it only writes
 611  * one packet.  ep-a..ep-d should use dma instead.
 612  */
 613 static void write_fifo(struct net2280_ep *ep, struct usb_request *req)
 614 {
 615         struct net2280_ep_regs  __iomem *regs = ep->regs;
 616         u8                      *buf;
 617         u32                     tmp;
 618         unsigned                count, total;
 619
 620         /* INVARIANT:  fifo is currently empty. (testable) */
 621
 622         if (req) {
 623                 buf = req->buf + req->actual;
 624                 prefetch(buf);
 625                 total = req->length - req->actual;
 626         } else {
 627                 total = 0;
 628                 buf = NULL;
 629         }
 630
 631         /* write just one packet at a time */
 632         count = ep->ep.maxpacket;
 633         if (count > total)      /* min() cannot be used on a bitfield */
 634                 count = total;
 635
 636         ep_vdbg(ep->dev, "write %s fifo (IN) %d bytes%s req %p\n",
 637                         ep->ep.name, count,
 638                         (count != ep->ep.maxpacket) ? " (short)" : "",
 639                         req);
 640         while (count >= 4) {
 641                 /* NOTE be careful if you try to align these. fifo lines
 642                  * should normally be full (4 bytes) and successive partial
 643                  * lines are ok only in certain cases.
 644                  */
 645                 tmp = get_unaligned((u32 *)buf);
 646                 cpu_to_le32s(&tmp);
 647                 writel(tmp, &regs->ep_data);
 648                 buf += 4;
 649                 count -= 4;
 650         }
 651
 652         /* last fifo entry is "short" unless we wrote a full packet.
 653          * also explicitly validate last word in (periodic) transfers
 654          * when maxpacket is not a multiple of 4 bytes.
 655          */
 656         if (count || total < ep->ep.maxpacket) {
 657                 tmp = count ? get_unaligned((u32 *)buf) : count;
 658                 cpu_to_le32s(&tmp);
 659                 set_fifo_bytecount(ep, count & 0x03);
 660                 writel(tmp, &regs->ep_data);
 661         }
 662
 663         /* pci writes may still be posted */
 664 }
 665
 666 /* work around erratum 0106: PCI and USB race over the OUT fifo.
 667  * caller guarantees chiprev 0100, out endpoint is NAKing, and
 668  * there's no real data in the fifo.
 669  *
 670  * NOTE:  also used in cases where that erratum doesn't apply:
 671  * where the host wrote "too much" data to us.
 672  */
 673 static void out_flush(struct net2280_ep *ep)
 674 {
 675         u32     __iomem *statp;
 676         u32     tmp;
 677
 678         statp = &ep->regs->ep_stat;
 679
 680         tmp = readl(statp);
 681         if (tmp & BIT(NAK_OUT_PACKETS)) {
 682                 ep_dbg(ep->dev, "%s %s %08x !NAK\n",
 683                         ep->ep.name, __func__, tmp);
 684                 writel(BIT(SET_NAK_OUT_PACKETS), &ep->regs->ep_rsp);
 685         }
 686
 687         writel(BIT(DATA_OUT_PING_TOKEN_INTERRUPT) |
 688                 BIT(DATA_PACKET_RECEIVED_INTERRUPT),
 689                 statp);
 690         writel(BIT(FIFO_FLUSH), statp);
 691         /* Make sure that stap is written */
 692         mb();
 693         tmp = readl(statp);
 694         if (tmp & BIT(DATA_OUT_PING_TOKEN_INTERRUPT) &&
 695                         /* high speed did bulk NYET; fifo isn't filling */
 696                         ep->dev->gadget.speed == USB_SPEED_FULL) {
 697                 unsigned        usec;
 698
 699                 usec = 50;              /* 64 byte bulk/interrupt */
 700                 handshake(statp, BIT(USB_OUT_PING_NAK_SENT),
 701                                 BIT(USB_OUT_PING_NAK_SENT), usec);
 702                 /* NAK done; now CLEAR_NAK_OUT_PACKETS is safe */
 703         }
 704 }
 705
 706 /* unload packet(s) from the fifo we use for usb OUT transfers.
 707  * returns true iff the request completed, because of short packet
 708  * or the request buffer having filled with full packets.
 709  *
 710  * for ep-a..ep-d this will read multiple packets out when they
 711  * have been accepted.
 712  */
 713 static int read_fifo(struct net2280_ep *ep, struct net2280_request *req)
 714 {
 715         struct net2280_ep_regs  __iomem *regs = ep->regs;
 716         u8                      *buf = req->req.buf + req->req.actual;
 717         unsigned                count, tmp, is_short;
 718         unsigned                cleanup = 0, prevent = 0;
 719
 720         /* erratum 0106 ... packets coming in during fifo reads might
 721          * be incompletely rejected.  not all cases have workarounds.
 722          */
 723         if (ep->dev->chiprev == 0x0100 &&
 724                         ep->dev->gadget.speed == USB_SPEED_FULL) {
 725                 udelay(1);
 726                 tmp = readl(&ep->regs->ep_stat);
 727                 if ((tmp & BIT(NAK_OUT_PACKETS)))
 728                         cleanup = 1;
 729                 else if ((tmp & BIT(FIFO_FULL))) {
 730                         start_out_naking(ep);
 731                         prevent = 1;
 732                 }
 733                 /* else: hope we don't see the problem */
 734         }
 735
 736         /* never overflow the rx buffer. the fifo reads packets until
 737          * it sees a short one; we might not be ready for them all.
 738          */
 739         prefetchw(buf);
 740         count = readl(&regs->ep_avail);
 741         if (unlikely(count == 0)) {
 742                 udelay(1);
 743                 tmp = readl(&ep->regs->ep_stat);
 744                 count = readl(&regs->ep_avail);
 745                 /* handled that data already? */
 746                 if (count == 0 && (tmp & BIT(NAK_OUT_PACKETS)) == 0)
 747                         return 0;
 748         }
 749
 750         tmp = req->req.length - req->req.actual;
 751         if (count > tmp) {
 752                 /* as with DMA, data overflow gets flushed */
 753                 if ((tmp % ep->ep.maxpacket) != 0) {
 754                         ep_err(ep->dev,
 755                                 "%s out fifo %d bytes, expected %d\n",
 756                                 ep->ep.name, count, tmp);
 757                         req->req.status = -EOVERFLOW;
 758                         cleanup = 1;
 759                         /* NAK_OUT_PACKETS will be set, so flushing is safe;
 760                          * the next read will start with the next packet
 761                          */
 762                 } /* else it's a ZLP, no worries */
 763                 count = tmp;
 764         }
 765         req->req.actual += count;
 766
 767         is_short = (count == 0) || ((count % ep->ep.maxpacket) != 0);
 768
 769         ep_vdbg(ep->dev, "read %s fifo (OUT) %d bytes%s%s%s req %p %d/%d\n",
 770                         ep->ep.name, count, is_short ? " (short)" : "",
 771                         cleanup ? " flush" : "", prevent ? " nak" : "",
 772                         req, req->req.actual, req->req.length);
 773
 774         while (count >= 4) {
 775                 tmp = readl(&regs->ep_data);
 776                 cpu_to_le32s(&tmp);
 777                 put_unaligned(tmp, (u32 *)buf);
 778                 buf += 4;
 779                 count -= 4;
 780         }
 781         if (count) {
 782                 tmp = readl(&regs->ep_data);
 783                 /* LE conversion is implicit here: */
 784                 do {
 785                         *buf++ = (u8) tmp;
 786                         tmp >>= 8;
 787                 } while (--count);
 788         }
 789         if (cleanup)
 790                 out_flush(ep);
 791         if (prevent) {
 792                 writel(BIT(CLEAR_NAK_OUT_PACKETS), &ep->regs->ep_rsp);
 793                 (void) readl(&ep->regs->ep_rsp);
 794         }
 795
 796         return is_short || ((req->req.actual == req->req.length) &&
 797                         !req->req.zero);
 798 }
 799
 800 /* fill out dma descriptor to match a given request */
 801 static void fill_dma_desc(struct net2280_ep *ep,
 802                                         struct net2280_request *req, int valid)
 803 {
 804         struct net2280_dma      *td = req->td;
 805         u32                     dmacount = req->req.length;
 806
 807         /* don't let DMA continue after a short OUT packet,
 808          * so overruns can't affect the next transfer.
 809          * in case of overruns on max-size packets, we can't
 810          * stop the fifo from filling but we can flush it.
 811          */
 812         if (ep->is_in)
 813                 dmacount |= BIT(DMA_DIRECTION);
 814         if ((!ep->is_in && (dmacount % ep->ep.maxpacket) != 0) ||
 815                                         !(ep->dev->quirks & PLX_2280))
 816                 dmacount |= BIT(END_OF_CHAIN);
 817
 818         req->valid = valid;
 819         if (valid)
 820                 dmacount |= BIT(VALID_BIT);
 821         dmacount |= BIT(DMA_DONE_INTERRUPT_ENABLE);
 822
 823         /* td->dmadesc = previously set by caller */
 824         td->dmaaddr = cpu_to_le32 (req->req.dma);
 825
 826         /* 2280 may be polling VALID_BIT through ep->dma->dmadesc */
 827         wmb();
 828         td->dmacount = cpu_to_le32(dmacount);
 829 }
 830
 831 static const u32 dmactl_default =
 832                 BIT(DMA_SCATTER_GATHER_DONE_INTERRUPT) |
 833                 BIT(DMA_CLEAR_COUNT_ENABLE) |
 834                 /* erratum 0116 workaround part 1 (use POLLING) */
 835                 (POLL_100_USEC << DESCRIPTOR_POLLING_RATE) |
 836                 BIT(DMA_VALID_BIT_POLLING_ENABLE) |
 837                 BIT(DMA_VALID_BIT_ENABLE) |
 838                 BIT(DMA_SCATTER_GATHER_ENABLE) |
 839                 /* erratum 0116 workaround part 2 (no AUTOSTART) */
 840                 BIT(DMA_ENABLE);
 841
 842 static inline void spin_stop_dma(struct net2280_dma_regs __iomem *dma)
 843 {
 844         handshake(&dma->dmactl, BIT(DMA_ENABLE), 0, 50);
 845 }
 846
 847 static inline void stop_dma(struct net2280_dma_regs __iomem *dma)
 848 {
 849         writel(readl(&dma->dmactl) & ~BIT(DMA_ENABLE), &dma->dmactl);
 850         spin_stop_dma(dma);
 851 }
 852
 853 static void start_queue(struct net2280_ep *ep, u32 dmactl, u32 td_dma)
 854 {
 855         struct net2280_dma_regs __iomem *dma = ep->dma;
 856         unsigned int tmp = BIT(VALID_BIT) | (ep->is_in << DMA_DIRECTION);
 857
 858         if (!(ep->dev->quirks & PLX_2280))
 859                 tmp |= BIT(END_OF_CHAIN);
 860
 861         writel(tmp, &dma->dmacount);
 862         writel(readl(&dma->dmastat), &dma->dmastat);
 863
 864         writel(td_dma, &dma->dmadesc);
 865         if (ep->dev->quirks & PLX_SUPERSPEED)
 866                 dmactl |= BIT(DMA_REQUEST_OUTSTANDING);
 867         writel(dmactl, &dma->dmactl);
 868
 869         /* erratum 0116 workaround part 3:  pci arbiter away from net2280 */
 870         (void) readl(&ep->dev->pci->pcimstctl);
 871
 872         writel(BIT(DMA_START), &dma->dmastat);
 873
 874         if (!ep->is_in)
 875                 stop_out_naking(ep);
 876 }
 877
 878 static void start_dma(struct net2280_ep *ep, struct net2280_request *req)
 879 {
 880         u32                     tmp;
 881         struct net2280_dma_regs __iomem *dma = ep->dma;
 882
 883         /* FIXME can't use DMA for ZLPs */
 884
 885         /* on this path we "know" there's no dma active (yet) */
 886         WARN_ON(readl(&dma->dmactl) & BIT(DMA_ENABLE));
 887         writel(0, &ep->dma->dmactl);
 888
 889         /* previous OUT packet might have been short */
 890         if (!ep->is_in && (readl(&ep->regs->ep_stat) &
 891                                 BIT(NAK_OUT_PACKETS))) {
 892                 writel(BIT(SHORT_PACKET_TRANSFERRED_INTERRUPT),
 893                         &ep->regs->ep_stat);
 894
 895                 tmp = readl(&ep->regs->ep_avail);
 896                 if (tmp) {
 897                         writel(readl(&dma->dmastat), &dma->dmastat);
 898
 899                         /* transfer all/some fifo data */
 900                         writel(req->req.dma, &dma->dmaaddr);
 901                         tmp = min(tmp, req->req.length);
 902
 903                         /* dma irq, faking scatterlist status */
 904                         req->td->dmacount = cpu_to_le32(req->req.length - tmp);
 905                         writel(BIT(DMA_DONE_INTERRUPT_ENABLE) | tmp,
 906                                         &dma->dmacount);
 907                         req->td->dmadesc = 0;
 908                         req->valid = 1;
 909
 910                         writel(BIT(DMA_ENABLE), &dma->dmactl);
 911                         writel(BIT(DMA_START), &dma->dmastat);
 912                         return;
 913                 }
 914         }
 915
 916         tmp = dmactl_default;
 917
 918         /* force packet boundaries between dma requests, but prevent the
 919          * controller from automagically writing a last "short" packet
 920          * (zero length) unless the driver explicitly said to do that.
 921          */
 922         if (ep->is_in) {
 923                 if (likely((req->req.length % ep->ep.maxpacket) ||
 924                                                         req->req.zero)){
 925                         tmp |= BIT(DMA_FIFO_VALIDATE);
 926                         ep->in_fifo_validate = 1;
 927                 } else
 928                         ep->in_fifo_validate = 0;
 929         }
 930
 931         /* init req->td, pointing to the current dummy */
 932         req->td->dmadesc = cpu_to_le32 (ep->td_dma);
 933         fill_dma_desc(ep, req, 1);
 934
 935         req->td->dmacount |= cpu_to_le32(BIT(END_OF_CHAIN));
 936
 937         start_queue(ep, tmp, req->td_dma);
 938 }
 939
 940 static inline void
 941 queue_dma(struct net2280_ep *ep, struct net2280_request *req, int valid)
 942 {
 943         struct net2280_dma      *end;
 944         dma_addr_t              tmp;
 945
 946         /* swap new dummy for old, link; fill and maybe activate */
 947         end = ep->dummy;
 948         ep->dummy = req->td;
 949         req->td = end;
 950
 951         tmp = ep->td_dma;
 952         ep->td_dma = req->td_dma;
 953         req->td_dma = tmp;
 954
 955         end->dmadesc = cpu_to_le32 (ep->td_dma);
 956
 957         fill_dma_desc(ep, req, valid);
 958 }
 959
 960 static void
 961 done(struct net2280_ep *ep, struct net2280_request *req, int status)
 962 {
 963         struct net2280          *dev;
 964         unsigned                stopped = ep->stopped;
 965
 966         list_del_init(&req->queue);
 967
 968         if (req->req.status == -EINPROGRESS)
 969                 req->req.status = status;
 970         else
 971                 status = req->req.status;
 972
 973         dev = ep->dev;
 974         if (ep->dma)
 975                 usb_gadget_unmap_request(&dev->gadget, &req->req, ep->is_in);
 976
 977         if (status && status != -ESHUTDOWN)
 978                 ep_vdbg(dev, "complete %s req %p stat %d len %u/%u\n",
 979                         ep->ep.name, &req->req, status,
 980                         req->req.actual, req->req.length);
 981
 982         /* don't modify queue heads during completion callback */
 983         ep->stopped = 1;
 984         spin_unlock(&dev->lock);
 985         usb_gadget_giveback_request(&ep->ep, &req->req);
 986         spin_lock(&dev->lock);
 987         ep->stopped = stopped;
 988 }
 989
 990 /*-------------------------------------------------------------------------*/
 991
 992 static int
 993 net2280_queue(struct usb_ep *_ep, struct usb_request *_req, gfp_t gfp_flags)
 994 {
 995         struct net2280_request  *req;
 996         struct net2280_ep       *ep;
 997         struct net2280          *dev;
 998         unsigned long           flags;
 999         int ret = 0;
1000
1001         /* we always require a cpu-view buffer, so that we can
1002          * always use pio (as fallback or whatever).
1003          */
1004         ep = container_of(_ep, struct net2280_ep, ep);
1005         if (!_ep || (!ep->desc && ep->num != 0)) {
1006                 pr_err("%s: Invalid ep=%p or ep->desc\n", __func__, _ep);
1007                 return -EINVAL;
1008         }
1009         req = container_of(_req, struct net2280_request, req);
1010         if (!_req || !_req->complete || !_req->buf ||
1011                                 !list_empty(&req->queue)) {
1012                 ret = -EINVAL;
1013                 goto print_err;
1014         }
1015         if (_req->length > (~0 & DMA_BYTE_COUNT_MASK)) {
1016                 ret = -EDOM;
1017                 goto print_err;
1018         }
1019         dev = ep->dev;
1020         if (!dev->driver || dev->gadget.speed == USB_SPEED_UNKNOWN) {
1021                 ret = -ESHUTDOWN;
1022                 goto print_err;
1023         }
1024
1025         /* FIXME implement PIO fallback for ZLPs with DMA */
1026         if (ep->dma && _req->length == 0) {
1027                 ret = -EOPNOTSUPP;
1028                 goto print_err;
1029         }
1030
1031         /* set up dma mapping in case the caller didn't */
1032         if (ep->dma) {
1033                 ret = usb_gadget_map_request(&dev->gadget, _req,
1034                                 ep->is_in);
1035                 if (ret)
1036                         goto print_err;
1037         }
1038
1039         ep_vdbg(dev, "%s queue req %p, len %d buf %p\n",
1040                         _ep->name, _req, _req->length, _req->buf);
1041
1042         spin_lock_irqsave(&dev->lock, flags);
1043
1044         _req->status = -EINPROGRESS;
1045         _req->actual = 0;
1046
1047         /* kickstart this i/o queue? */
1048         if  (list_empty(&ep->queue) && !ep->stopped &&
1049                 !((dev->quirks & PLX_SUPERSPEED) && ep->dma &&
1050                   (readl(&ep->regs->ep_rsp) & BIT(CLEAR_ENDPOINT_HALT)))) {
1051
1052                 /* use DMA if the endpoint supports it, else pio */
1053                 if (ep->dma)
1054                         start_dma(ep, req);
1055                 else {
1056                         /* maybe there's no control data, just status ack */
1057                         if (ep->num == 0 && _req->length == 0) {
1058                                 allow_status(ep);
1059                                 done(ep, req, 0);
1060                                 ep_vdbg(dev, "%s status ack\n", ep->ep.name);
1061                                 goto done;
1062                         }
1063
1064                         /* PIO ... stuff the fifo, or unblock it.  */
1065                         if (ep->is_in)
1066                                 write_fifo(ep, _req);
1067                         else if (list_empty(&ep->queue)) {
1068                                 u32     s;
1069
1070                                 /* OUT FIFO might have packet(s) buffered */
1071                                 s = readl(&ep->regs->ep_stat);
1072                                 if ((s & BIT(FIFO_EMPTY)) == 0) {
1073                                         /* note:  _req->short_not_ok is
1074                                          * ignored here since PIO _always_
1075                                          * stops queue advance here, and
1076                                          * _req->status doesn't change for
1077                                          * short reads (only _req->actual)
1078                                          */
1079                                         if (read_fifo(ep, req) &&
1080                                                         ep->num == 0) {
1081                                                 done(ep, req, 0);
1082                                                 allow_status(ep);
1083                                                 /* don't queue it */
1084                                                 req = NULL;
1085                                         } else if (read_fifo(ep, req) &&
1086                                                         ep->num != 0) {
1087                                                 done(ep, req, 0);
1088                                                 req = NULL;
1089                                         } else
1090                                                 s = readl(&ep->regs->ep_stat);
1091                                 }
1092
1093                                 /* don't NAK, let the fifo fill */
1094                                 if (req && (s & BIT(NAK_OUT_PACKETS)))
1095                                         writel(BIT(CLEAR_NAK_OUT_PACKETS),
1096                                                         &ep->regs->ep_rsp);
1097                         }
1098                 }
1099
1100         } else if (ep->dma) {
1101                 int     valid = 1;
1102
1103                 if (ep->is_in) {
1104                         int     expect;
1105
1106                         /* preventing magic zlps is per-engine state, not
1107                          * per-transfer; irq logic must recover hiccups.
1108                          */
1109                         expect = likely(req->req.zero ||
1110                                 (req->req.length % ep->ep.maxpacket));
1111                         if (expect != ep->in_fifo_validate)
1112                                 valid = 0;
1113                 }
1114                 queue_dma(ep, req, valid);
1115
1116         } /* else the irq handler advances the queue. */
1117
1118         ep->responded = 1;
1119         if (req)
1120                 list_add_tail(&req->queue, &ep->queue);
1121 done:
1122         spin_unlock_irqrestore(&dev->lock, flags);
1123
1124         /* pci writes may still be posted */
1125         return ret;
1126
1127 print_err:
1128         dev_err(&ep->dev->pdev->dev, "%s: error=%d\n", __func__, ret);
1129         return ret;
1130 }
1131
1132 static inline void
1133 dma_done(struct net2280_ep *ep, struct net2280_request *req, u32 dmacount,
1134                 int status)
1135 {
1136         req->req.actual = req->req.length - (DMA_BYTE_COUNT_MASK & dmacount);
1137         done(ep, req, status);
1138 }
1139
1140 static void scan_dma_completions(struct net2280_ep *ep)
1141 {
1142         /* only look at descriptors that were "naturally" retired,
1143          * so fifo and list head state won't matter
1144          */
1145         while (!list_empty(&ep->queue)) {
1146                 struct net2280_request  *req;
1147                 u32                     tmp;
1148
1149                 req = list_entry(ep->queue.next,
1150                                 struct net2280_request, queue);
1151                 if (!req->valid)
1152                         break;
1153                 rmb();
1154                 tmp = le32_to_cpup(&req->td->dmacount);
1155                 if ((tmp & BIT(VALID_BIT)) != 0)
1156                         break;
1157
1158                 /* SHORT_PACKET_TRANSFERRED_INTERRUPT handles "usb-short"
1159                  * cases where DMA must be aborted; this code handles
1160                  * all non-abort DMA completions.
1161                  */
1162                 if (unlikely(req->td->dmadesc == 0)) {
1163                         /* paranoia */
1164                         tmp = readl(&ep->dma->dmacount);
1165                         if (tmp & DMA_BYTE_COUNT_MASK)
1166                                 break;
1167                         /* single transfer mode */
1168                         dma_done(ep, req, tmp, 0);
1169                         break;
1170                 } else if (!ep->is_in &&
1171                            (req->req.length % ep->ep.maxpacket) &&
1172                            !(ep->dev->quirks & PLX_SUPERSPEED)) {
1173
1174                         tmp = readl(&ep->regs->ep_stat);
1175                         /* AVOID TROUBLE HERE by not issuing short reads from
1176                          * your gadget driver.  That helps avoids errata 0121,
1177                          * 0122, and 0124; not all cases trigger the warning.
1178                          */
1179                         if ((tmp & BIT(NAK_OUT_PACKETS)) == 0) {
1180                                 ep_warn(ep->dev, "%s lost packet sync!\n",
1181                                                 ep->ep.name);
1182                                 req->req.status = -EOVERFLOW;
1183                         } else {
1184                                 tmp = readl(&ep->regs->ep_avail);
1185                                 if (tmp) {
1186                                         /* fifo gets flushed later */
1187                                         ep->out_overflow = 1;
1188                                         ep_dbg(ep->dev,
1189                                                 "%s dma, discard %d len %d\n",
1190                                                 ep->ep.name, tmp,
1191                                                 req->req.length);
1192                                         req->req.status = -EOVERFLOW;
1193                                 }
1194                         }
1195                 }
1196                 dma_done(ep, req, tmp, 0);
1197         }
1198 }
1199
1200 static void restart_dma(struct net2280_ep *ep)
1201 {
1202         struct net2280_request  *req;
1203
1204         if (ep->stopped)
1205                 return;
1206         req = list_entry(ep->queue.next, struct net2280_request, queue);
1207
1208         start_dma(ep, req);
1209 }
1210
1211 static void abort_dma(struct net2280_ep *ep)
1212 {
1213         /* abort the current transfer */
1214         if (likely(!list_empty(&ep->queue))) {
1215                 /* FIXME work around errata 0121, 0122, 0124 */
1216                 writel(BIT(DMA_ABORT), &ep->dma->dmastat);
1217                 spin_stop_dma(ep->dma);
1218         } else
1219                 stop_dma(ep->dma);
1220         scan_dma_completions(ep);
1221 }
1222
1223 /* dequeue ALL requests */
1224 static void nuke(struct net2280_ep *ep)
1225 {
1226         struct net2280_request  *req;
1227
1228         /* called with spinlock held */
1229         ep->stopped = 1;
1230         if (ep->dma)
1231                 abort_dma(ep);
1232         while (!list_empty(&ep->queue)) {
1233                 req = list_entry(ep->queue.next,
1234                                 struct net2280_request,
1235                                 queue);
1236                 done(ep, req, -ESHUTDOWN);
1237         }
1238 }
1239
1240 /* dequeue JUST ONE request */
1241 static int net2280_dequeue(struct usb_ep *_ep, struct usb_request *_req)
1242 {
1243         struct net2280_ep       *ep;
1244         struct net2280_request  *req;
1245         unsigned long           flags;
1246         u32                     dmactl;
1247         int                     stopped;
1248
1249         ep = container_of(_ep, struct net2280_ep, ep);
1250         if (!_ep || (!ep->desc && ep->num != 0) || !_req) {
1251                 pr_err("%s: Invalid ep=%p or ep->desc or req=%p\n",
1252                                                 __func__, _ep, _req);
1253                 return -EINVAL;
1254         }
1255
1256         spin_lock_irqsave(&ep->dev->lock, flags);
1257         stopped = ep->stopped;
1258
1259         /* quiesce dma while we patch the queue */
1260         dmactl = 0;
1261         ep->stopped = 1;
1262         if (ep->dma) {
1263                 dmactl = readl(&ep->dma->dmactl);
1264                 /* WARNING erratum 0127 may kick in ... */
1265                 stop_dma(ep->dma);
1266                 scan_dma_completions(ep);
1267         }
1268
1269         /* make sure it's still queued on this endpoint */
1270         list_for_each_entry(req, &ep->queue, queue) {
1271                 if (&req->req == _req)
1272                         break;
1273         }
1274         if (&req->req != _req) {
1275                 spin_unlock_irqrestore(&ep->dev->lock, flags);
1276                 dev_err(&ep->dev->pdev->dev, "%s: Request mismatch\n",
1277                                                                 __func__);
1278                 return -EINVAL;
1279         }
1280
1281         /* queue head may be partially complete. */
1282         if (ep->queue.next == &req->queue) {
1283                 if (ep->dma) {
1284                         ep_dbg(ep->dev, "unlink (%s) dma\n", _ep->name);
1285                         _req->status = -ECONNRESET;
1286                         abort_dma(ep);
1287                         if (likely(ep->queue.next == &req->queue)) {
1288                                 /* NOTE: misreports single-transfer mode*/
1289                                 req->td->dmacount = 0;  /* invalidate */
1290                                 dma_done(ep, req,
1291                                         readl(&ep->dma->dmacount),
1292                                         -ECONNRESET);
1293                         }
1294                 } else {
1295                         ep_dbg(ep->dev, "unlink (%s) pio\n", _ep->name);
1296                         done(ep, req, -ECONNRESET);
1297                 }
1298                 req = NULL;
1299         }
1300
1301         if (req)
1302                 done(ep, req, -ECONNRESET);
1303         ep->stopped = stopped;
1304
1305         if (ep->dma) {
1306                 /* turn off dma on inactive queues */
1307                 if (list_empty(&ep->queue))
1308                         stop_dma(ep->dma);
1309                 else if (!ep->stopped) {
1310                         /* resume current request, or start new one */
1311                         if (req)
1312                                 writel(dmactl, &ep->dma->dmactl);
1313                         else
1314                                 start_dma(ep, list_entry(ep->queue.next,
1315                                         struct net2280_request, queue));
1316                 }
1317         }
1318
1319         spin_unlock_irqrestore(&ep->dev->lock, flags);
1320         return 0;
1321 }
1322
1323 /*-------------------------------------------------------------------------*/
1324
1325 static int net2280_fifo_status(struct usb_ep *_ep);
1326
1327 static int
1328 net2280_set_halt_and_wedge(struct usb_ep *_ep, int value, int wedged)
1329 {
1330         struct net2280_ep       *ep;
1331         unsigned long           flags;
1332         int                     retval = 0;
1333
1334         ep = container_of(_ep, struct net2280_ep, ep);
1335         if (!_ep || (!ep->desc && ep->num != 0)) {
1336                 pr_err("%s: Invalid ep=%p or ep->desc\n", __func__, _ep);
1337                 return -EINVAL;
1338         }
1339         if (!ep->dev->driver || ep->dev->gadget.speed == USB_SPEED_UNKNOWN) {
1340                 retval = -ESHUTDOWN;
1341                 goto print_err;
1342         }
1343         if (ep->desc /* not ep0 */ && (ep->desc->bmAttributes & 0x03)
1344                                                 == USB_ENDPOINT_XFER_ISOC) {
1345                 retval = -EINVAL;
1346                 goto print_err;
1347         }
1348
1349         spin_lock_irqsave(&ep->dev->lock, flags);
1350         if (!list_empty(&ep->queue)) {
1351                 retval = -EAGAIN;
1352                 goto print_unlock;
1353         } else if (ep->is_in && value && net2280_fifo_status(_ep) != 0) {
1354                 retval = -EAGAIN;
1355                 goto print_unlock;
1356         } else {
1357                 ep_vdbg(ep->dev, "%s %s %s\n", _ep->name,
1358                                 value ? "set" : "clear",
1359                                 wedged ? "wedge" : "halt");
1360                 /* set/clear, then synch memory views with the device */
1361                 if (value) {
1362                         if (ep->num == 0)
1363                                 ep->dev->protocol_stall = 1;
1364                         else
1365                                 set_halt(ep);
1366                         if (wedged)
1367                                 ep->wedged = 1;
1368                 } else {
1369                         clear_halt(ep);
1370                         if (ep->dev->quirks & PLX_SUPERSPEED &&
1371                                 !list_empty(&ep->queue) && ep->td_dma)
1372                                         restart_dma(ep);
1373                         ep->wedged = 0;
1374                 }
1375                 (void) readl(&ep->regs->ep_rsp);
1376         }
1377         spin_unlock_irqrestore(&ep->dev->lock, flags);
1378
1379         return retval;
1380
1381 print_unlock:
1382         spin_unlock_irqrestore(&ep->dev->lock, flags);
1383 print_err:
1384         dev_err(&ep->dev->pdev->dev, "%s: error=%d\n", __func__, retval);
1385         return retval;
1386 }
1387
1388 static int net2280_set_halt(struct usb_ep *_ep, int value)
1389 {
1390         return net2280_set_halt_and_wedge(_ep, value, 0);
1391 }
1392
1393 static int net2280_set_wedge(struct usb_ep *_ep)
1394 {
1395         if (!_ep || _ep->name == ep0name) {
1396                 pr_err("%s: Invalid ep=%p or ep0\n", __func__, _ep);
1397                 return -EINVAL;
1398         }
1399         return net2280_set_halt_and_wedge(_ep, 1, 1);
1400 }
1401
1402 static int net2280_fifo_status(struct usb_ep *_ep)
1403 {
1404         struct net2280_ep       *ep;
1405         u32                     avail;
1406
1407         ep = container_of(_ep, struct net2280_ep, ep);
1408         if (!_ep || (!ep->desc && ep->num != 0)) {
1409                 pr_err("%s: Invalid ep=%p or ep->desc\n", __func__, _ep);
1410                 return -ENODEV;
1411         }
1412         if (!ep->dev->driver || ep->dev->gadget.speed == USB_SPEED_UNKNOWN) {
1413                 dev_err(&ep->dev->pdev->dev,
1414                         "%s: Invalid driver=%p or speed=%d\n",
1415                         __func__, ep->dev->driver, ep->dev->gadget.speed);
1416                 return -ESHUTDOWN;
1417         }
1418
1419         avail = readl(&ep->regs->ep_avail) & (BIT(12) - 1);
1420         if (avail > ep->fifo_size) {
1421                 dev_err(&ep->dev->pdev->dev, "%s: Fifo overflow\n", __func__);
1422                 return -EOVERFLOW;
1423         }
1424         if (ep->is_in)
1425                 avail = ep->fifo_size - avail;
1426         return avail;
1427 }
1428
1429 static void net2280_fifo_flush(struct usb_ep *_ep)
1430 {
1431         struct net2280_ep       *ep;
1432
1433         ep = container_of(_ep, struct net2280_ep, ep);
1434         if (!_ep || (!ep->desc && ep->num != 0)) {
1435                 pr_err("%s: Invalid ep=%p or ep->desc\n", __func__, _ep);
1436                 return;
1437         }
1438         if (!ep->dev->driver || ep->dev->gadget.speed == USB_SPEED_UNKNOWN) {
1439                 dev_err(&ep->dev->pdev->dev,
1440                         "%s: Invalid driver=%p or speed=%d\n",
1441                         __func__, ep->dev->driver, ep->dev->gadget.speed);
1442                 return;
1443         }
1444
1445         writel(BIT(FIFO_FLUSH), &ep->regs->ep_stat);
1446         (void) readl(&ep->regs->ep_rsp);
1447 }
1448
1449 static const struct usb_ep_ops net2280_ep_ops = {
1450         .enable         = net2280_enable,
1451         .disable        = net2280_disable,
1452
1453         .alloc_request  = net2280_alloc_request,
1454         .free_request   = net2280_free_request,
1455
1456         .queue          = net2280_queue,
1457         .dequeue        = net2280_dequeue,
1458
1459         .set_halt       = net2280_set_halt,
1460         .set_wedge      = net2280_set_wedge,
1461         .fifo_status    = net2280_fifo_status,
1462         .fifo_flush     = net2280_fifo_flush,
1463 };
1464
1465 /*-------------------------------------------------------------------------*/
1466
1467 static int net2280_get_frame(struct usb_gadget *_gadget)
1468 {
1469         struct net2280          *dev;
1470         unsigned long           flags;
1471         u16                     retval;
1472
1473         if (!_gadget)
1474                 return -ENODEV;
1475         dev = container_of(_gadget, struct net2280, gadget);
1476         spin_lock_irqsave(&dev->lock, flags);
1477         retval = get_idx_reg(dev->regs, REG_FRAME) & 0x03ff;
1478         spin_unlock_irqrestore(&dev->lock, flags);
1479         return retval;
1480 }
1481
1482 static int net2280_wakeup(struct usb_gadget *_gadget)
1483 {
1484         struct net2280          *dev;
1485         u32                     tmp;
1486         unsigned long           flags;
1487
1488         if (!_gadget)
1489                 return 0;
1490         dev = container_of(_gadget, struct net2280, gadget);
1491
1492         spin_lock_irqsave(&dev->lock, flags);
1493         tmp = readl(&dev->usb->usbctl);
1494         if (tmp & BIT(DEVICE_REMOTE_WAKEUP_ENABLE))
1495                 writel(BIT(GENERATE_RESUME), &dev->usb->usbstat);
1496         spin_unlock_irqrestore(&dev->lock, flags);
1497
1498         /* pci writes may still be posted */
1499         return 0;
1500 }
1501
1502 static int net2280_set_selfpowered(struct usb_gadget *_gadget, int value)
1503 {
1504         struct net2280          *dev;
1505         u32                     tmp;
1506         unsigned long           flags;
1507
1508         if (!_gadget)
1509                 return 0;
1510         dev = container_of(_gadget, struct net2280, gadget);
1511
1512         spin_lock_irqsave(&dev->lock, flags);
1513         tmp = readl(&dev->usb->usbctl);
1514         if (value) {
1515                 tmp |= BIT(SELF_POWERED_STATUS);
1516                 _gadget->is_selfpowered = 1;
1517         } else {
1518                 tmp &= ~BIT(SELF_POWERED_STATUS);
1519                 _gadget->is_selfpowered = 0;
1520         }
1521         writel(tmp, &dev->usb->usbctl);
1522         spin_unlock_irqrestore(&dev->lock, flags);
1523
1524         return 0;
1525 }
1526
1527 static int net2280_pullup(struct usb_gadget *_gadget, int is_on)
1528 {
1529         struct net2280  *dev;
1530         u32             tmp;
1531         unsigned long   flags;
1532
1533         if (!_gadget)
1534                 return -ENODEV;
1535         dev = container_of(_gadget, struct net2280, gadget);
1536
1537         spin_lock_irqsave(&dev->lock, flags);
1538         tmp = readl(&dev->usb->usbctl);
1539         dev->softconnect = (is_on != 0);
1540         if (is_on) {
1541                 ep0_start(dev);
1542                 writel(tmp | BIT(USB_DETECT_ENABLE), &dev->usb->usbctl);
1543         } else {
1544                 writel(tmp & ~BIT(USB_DETECT_ENABLE), &dev->usb->usbctl);
1545                 stop_activity(dev, dev->driver);
1546         }
1547
1548         spin_unlock_irqrestore(&dev->lock, flags);
1549
1550         return 0;
1551 }
1552
1553 static struct usb_ep *net2280_match_ep(struct usb_gadget *_gadget,
1554                 struct usb_endpoint_descriptor *desc,
1555                 struct usb_ss_ep_comp_descriptor *ep_comp)
1556 {
1557         char name[8];
1558         struct usb_ep *ep;
1559
1560         if (usb_endpoint_type(desc) == USB_ENDPOINT_XFER_INT) {
1561                 /* ep-e, ep-f are PIO with only 64 byte fifos */
1562                 ep = gadget_find_ep_by_name(_gadget, "ep-e");
1563                 if (ep && usb_gadget_ep_match_desc(_gadget, ep, desc, ep_comp))
1564                         return ep;
1565                 ep = gadget_find_ep_by_name(_gadget, "ep-f");
1566                 if (ep && usb_gadget_ep_match_desc(_gadget, ep, desc, ep_comp))
1567                         return ep;
1568         }
1569
1570         /* USB3380: use same address for usb and hardware endpoints */
1571         snprintf(name, sizeof(name), "ep%d%s", usb_endpoint_num(desc),
1572                         usb_endpoint_dir_in(desc) ? "in" : "out");
1573         ep = gadget_find_ep_by_name(_gadget, name);
1574         if (ep && usb_gadget_ep_match_desc(_gadget, ep, desc, ep_comp))
1575                 return ep;
1576
1577         return NULL;
1578 }
1579
1580 static int net2280_start(struct usb_gadget *_gadget,
1581                 struct usb_gadget_driver *driver);
1582 static int net2280_stop(struct usb_gadget *_gadget);
1583
1584 static const struct usb_gadget_ops net2280_ops = {
1585         .get_frame      = net2280_get_frame,
1586         .wakeup         = net2280_wakeup,
1587         .set_selfpowered = net2280_set_selfpowered,
1588         .pullup         = net2280_pullup,
1589         .udc_start      = net2280_start,
1590         .udc_stop       = net2280_stop,
1591         .match_ep       = net2280_match_ep,
1592 };
1593
1594 /*-------------------------------------------------------------------------*/
1595
1596 #ifdef  CONFIG_USB_GADGET_DEBUG_FILES
1597
1598 /* FIXME move these into procfs, and use seq_file.
1599  * Sysfs _still_ doesn't behave for arbitrarily sized files,
1600  * and also doesn't help products using this with 2.4 kernels.
1601  */
1602
1603 /* "function" sysfs attribute */
1604 static ssize_t function_show(struct device *_dev, struct device_attribute *attr,
1605                              char *buf)
1606 {
1607         struct net2280  *dev = dev_get_drvdata(_dev);
1608
1609         if (!dev->driver || !dev->driver->function ||
1610                         strlen(dev->driver->function) > PAGE_SIZE)
1611                 return 0;
1612         return scnprintf(buf, PAGE_SIZE, "%s\n", dev->driver->function);
1613 }
1614 static DEVICE_ATTR_RO(function);
1615
1616 static ssize_t registers_show(struct device *_dev,
1617                               struct device_attribute *attr, char *buf)
1618 {
1619         struct net2280          *dev;
1620         char                    *next;
1621         unsigned                size, t;
1622         unsigned long           flags;
1623         int                     i;
1624         u32                     t1, t2;
1625         const char              *s;
1626
1627         dev = dev_get_drvdata(_dev);
1628         next = buf;
1629         size = PAGE_SIZE;
1630         spin_lock_irqsave(&dev->lock, flags);
1631
1632         if (dev->driver)
1633                 s = dev->driver->driver.name;
1634         else
1635                 s = "(none)";
1636
1637         /* Main Control Registers */
1638         t = scnprintf(next, size, "%s version " DRIVER_VERSION
1639                         ", chiprev %04x\n\n"
1640                         "devinit %03x fifoctl %08x gadget '%s'\n"
1641                         "pci irqenb0 %02x irqenb1 %08x "
1642                         "irqstat0 %04x irqstat1 %08x\n",
1643                         driver_name, dev->chiprev,
1644                         readl(&dev->regs->devinit),
1645                         readl(&dev->regs->fifoctl),
1646                         s,
1647                         readl(&dev->regs->pciirqenb0),
1648                         readl(&dev->regs->pciirqenb1),
1649                         readl(&dev->regs->irqstat0),
1650                         readl(&dev->regs->irqstat1));
1651         size -= t;
1652         next += t;
1653
1654         /* USB Control Registers */
1655         t1 = readl(&dev->usb->usbctl);
1656         t2 = readl(&dev->usb->usbstat);
1657         if (t1 & BIT(VBUS_PIN)) {
1658                 if (t2 & BIT(HIGH_SPEED))
1659                         s = "high speed";
1660                 else if (dev->gadget.speed == USB_SPEED_UNKNOWN)
1661                         s = "powered";
1662                 else
1663                         s = "full speed";
1664                 /* full speed bit (6) not working?? */
1665         } else
1666                         s = "not attached";
1667         t = scnprintf(next, size,
1668                         "stdrsp %08x usbctl %08x usbstat %08x "
1669                                 "addr 0x%02x (%s)\n",
1670                         readl(&dev->usb->stdrsp), t1, t2,
1671                         readl(&dev->usb->ouraddr), s);
1672         size -= t;
1673         next += t;
1674
1675         /* PCI Master Control Registers */
1676
1677         /* DMA Control Registers */
1678
1679         /* Configurable EP Control Registers */
1680         for (i = 0; i < dev->n_ep; i++) {
1681                 struct net2280_ep       *ep;
1682
1683                 ep = &dev->ep[i];
1684                 if (i && !ep->desc)
1685                         continue;
1686
1687                 t1 = readl(&ep->cfg->ep_cfg);
1688                 t2 = readl(&ep->regs->ep_rsp) & 0xff;
1689                 t = scnprintf(next, size,
1690                                 "\n%s\tcfg %05x rsp (%02x) %s%s%s%s%s%s%s%s"
1691                                         "irqenb %02x\n",
1692                                 ep->ep.name, t1, t2,
1693                                 (t2 & BIT(CLEAR_NAK_OUT_PACKETS))
1694                                         ? "NAK " : "",
1695                                 (t2 & BIT(CLEAR_EP_HIDE_STATUS_PHASE))
1696                                         ? "hide " : "",
1697                                 (t2 & BIT(CLEAR_EP_FORCE_CRC_ERROR))
1698                                         ? "CRC " : "",
1699                                 (t2 & BIT(CLEAR_INTERRUPT_MODE))
1700                                         ? "interrupt " : "",
1701                                 (t2 & BIT(CLEAR_CONTROL_STATUS_PHASE_HANDSHAKE))
1702                                         ? "status " : "",
1703                                 (t2 & BIT(CLEAR_NAK_OUT_PACKETS_MODE))
1704                                         ? "NAKmode " : "",
1705                                 (t2 & BIT(CLEAR_ENDPOINT_TOGGLE))
1706                                         ? "DATA1 " : "DATA0 ",
1707                                 (t2 & BIT(CLEAR_ENDPOINT_HALT))
1708                                         ? "HALT " : "",
1709                                 readl(&ep->regs->ep_irqenb));
1710                 size -= t;
1711                 next += t;
1712
1713                 t = scnprintf(next, size,
1714                                 "\tstat %08x avail %04x "
1715                                 "(ep%d%s-%s)%s\n",
1716                                 readl(&ep->regs->ep_stat),
1717                                 readl(&ep->regs->ep_avail),
1718                                 t1 & 0x0f, DIR_STRING(t1),
1719                                 type_string(t1 >> 8),
1720                                 ep->stopped ? "*" : "");
1721                 size -= t;
1722                 next += t;
1723
1724                 if (!ep->dma)
1725                         continue;
1726
1727                 t = scnprintf(next, size,
1728                                 "  dma\tctl %08x stat %08x count %08x\n"
1729                                 "\taddr %08x desc %08x\n",
1730                                 readl(&ep->dma->dmactl),
1731                                 readl(&ep->dma->dmastat),
1732                                 readl(&ep->dma->dmacount),
1733                                 readl(&ep->dma->dmaaddr),
1734                                 readl(&ep->dma->dmadesc));
1735                 size -= t;
1736                 next += t;
1737
1738         }
1739
1740         /* Indexed Registers (none yet) */
1741
1742         /* Statistics */
1743         t = scnprintf(next, size, "\nirqs:  ");
1744         size -= t;
1745         next += t;
1746         for (i = 0; i < dev->n_ep; i++) {
1747                 struct net2280_ep       *ep;
1748
1749                 ep = &dev->ep[i];
1750                 if (i && !ep->irqs)
1751                         continue;
1752                 t = scnprintf(next, size, " %s/%lu", ep->ep.name, ep->irqs);
1753                 size -= t;
1754                 next += t;
1755
1756         }
1757         t = scnprintf(next, size, "\n");
1758         size -= t;
1759         next += t;
1760
1761         spin_unlock_irqrestore(&dev->lock, flags);
1762
1763         return PAGE_SIZE - size;
1764 }
1765 static DEVICE_ATTR_RO(registers);
1766
1767 static ssize_t queues_show(struct device *_dev, struct device_attribute *attr,
1768                            char *buf)
1769 {
1770         struct net2280          *dev;
1771         char                    *next;
1772         unsigned                size;
1773         unsigned long           flags;
1774         int                     i;
1775
1776         dev = dev_get_drvdata(_dev);
1777         next = buf;
1778         size = PAGE_SIZE;
1779         spin_lock_irqsave(&dev->lock, flags);
1780
1781         for (i = 0; i < dev->n_ep; i++) {
1782                 struct net2280_ep               *ep = &dev->ep[i];
1783                 struct net2280_request          *req;
1784                 int                             t;
1785
1786                 if (i != 0) {
1787                         const struct usb_endpoint_descriptor    *d;
1788
1789                         d = ep->desc;
1790                         if (!d)
1791                                 continue;
1792                         t = d->bEndpointAddress;
1793                         t = scnprintf(next, size,
1794                                 "\n%s (ep%d%s-%s) max %04x %s fifo %d\n",
1795                                 ep->ep.name, t & USB_ENDPOINT_NUMBER_MASK,
1796                                 (t & USB_DIR_IN) ? "in" : "out",
1797                                 type_string(d->bmAttributes),
1798                                 usb_endpoint_maxp(d) & 0x1fff,
1799                                 ep->dma ? "dma" : "pio", ep->fifo_size
1800                                 );
1801                 } else /* ep0 should only have one transfer queued */
1802                         t = scnprintf(next, size, "ep0 max 64 pio %s\n",
1803                                         ep->is_in ? "in" : "out");
1804                 if (t <= 0 || t > size)
1805                         goto done;
1806                 size -= t;
1807                 next += t;
1808
1809                 if (list_empty(&ep->queue)) {
1810                         t = scnprintf(next, size, "\t(nothing queued)\n");
1811                         if (t <= 0 || t > size)
1812                                 goto done;
1813                         size -= t;
1814                         next += t;
1815                         continue;
1816                 }
1817                 list_for_each_entry(req, &ep->queue, queue) {
1818                         if (ep->dma && req->td_dma == readl(&ep->dma->dmadesc))
1819                                 t = scnprintf(next, size,
1820                                         "\treq %p len %d/%d "
1821                                         "buf %p (dmacount %08x)\n",
1822                                         &req->req, req->req.actual,
1823                                         req->req.length, req->req.buf,
1824                                         readl(&ep->dma->dmacount));
1825                         else
1826                                 t = scnprintf(next, size,
1827                                         "\treq %p len %d/%d buf %p\n",
1828                                         &req->req, req->req.actual,
1829                                         req->req.length, req->req.buf);
1830                         if (t <= 0 || t > size)
1831                                 goto done;
1832                         size -= t;
1833                         next += t;
1834
1835                         if (ep->dma) {
1836                                 struct net2280_dma      *td;
1837
1838                                 td = req->td;
1839                                 t = scnprintf(next, size, "\t    td %08x "
1840                                         " count %08x buf %08x desc %08x\n",
1841                                         (u32) req->td_dma,
1842                                         le32_to_cpu(td->dmacount),
1843                                         le32_to_cpu(td->dmaaddr),
1844                                         le32_to_cpu(td->dmadesc));
1845                                 if (t <= 0 || t > size)
1846                                         goto done;
1847                                 size -= t;
1848                                 next += t;
1849                         }
1850                 }
1851         }
1852
1853 done:
1854         spin_unlock_irqrestore(&dev->lock, flags);
1855         return PAGE_SIZE - size;
1856 }
1857 static DEVICE_ATTR_RO(queues);
1858
1859
1860 #else
1861
1862 #define device_create_file(a, b)        (0)
1863 #define device_remove_file(a, b)        do { } while (0)
1864
1865 #endif
1866
1867 /*-------------------------------------------------------------------------*/
1868
1869 /* another driver-specific mode might be a request type doing dma
1870  * to/from another device fifo instead of to/from memory.
1871  */
1872
1873 static void set_fifo_mode(struct net2280 *dev, int mode)
1874 {
1875         /* keeping high bits preserves BAR2 */
1876         writel((0xffff << PCI_BASE2_RANGE) | mode, &dev->regs->fifoctl);
1877
1878         /* always ep-{a,b,e,f} ... maybe not ep-c or ep-d */
1879         INIT_LIST_HEAD(&dev->gadget.ep_list);
1880         list_add_tail(&dev->ep[1].ep.ep_list, &dev->gadget.ep_list);
1881         list_add_tail(&dev->ep[2].ep.ep_list, &dev->gadget.ep_list);
1882         switch (mode) {
1883         case 0:
1884                 list_add_tail(&dev->ep[3].ep.ep_list, &dev->gadget.ep_list);
1885                 list_add_tail(&dev->ep[4].ep.ep_list, &dev->gadget.ep_list);
1886                 dev->ep[1].fifo_size = dev->ep[2].fifo_size = 1024;
1887                 break;
1888         case 1:
1889                 dev->ep[1].fifo_size = dev->ep[2].fifo_size = 2048;
1890                 break;
1891         case 2:
1892                 list_add_tail(&dev->ep[3].ep.ep_list, &dev->gadget.ep_list);
1893                 dev->ep[1].fifo_size = 2048;
1894                 dev->ep[2].fifo_size = 1024;
1895                 break;
1896         }
1897         /* fifo sizes for ep0, ep-c, ep-d, ep-e, and ep-f never change */
1898         list_add_tail(&dev->ep[5].ep.ep_list, &dev->gadget.ep_list);
1899         list_add_tail(&dev->ep[6].ep.ep_list, &dev->gadget.ep_list);
1900 }
1901
1902 static void defect7374_disable_data_eps(struct net2280 *dev)
1903 {
1904         /*
1905          * For Defect 7374, disable data EPs (and more):
1906          *  - This phase undoes the earlier phase of the Defect 7374 workaround,
1907          *    returing ep regs back to normal.
1908          */
1909         struct net2280_ep *ep;
1910         int i;
1911         unsigned char ep_sel;
1912         u32 tmp_reg;
1913
1914         for (i = 1; i < 5; i++) {
1915                 ep = &dev->ep[i];
1916                 writel(i, &ep->cfg->ep_cfg);
1917         }
1918
1919         /* CSROUT, CSRIN, PCIOUT, PCIIN, STATIN, RCIN */
1920         for (i = 0; i < 6; i++)
1921                 writel(0, &dev->dep[i].dep_cfg);
1922
1923         for (ep_sel = 0; ep_sel <= 21; ep_sel++) {
1924                 /* Select an endpoint for subsequent operations: */
1925                 tmp_reg = readl(&dev->plregs->pl_ep_ctrl);
1926                 writel(((tmp_reg & ~0x1f) | ep_sel), &dev->plregs->pl_ep_ctrl);
1927
1928                 if (ep_sel < 2 || (ep_sel > 9 && ep_sel < 14) ||
1929                                         ep_sel == 18 || ep_sel == 20)
1930                         continue;
1931
1932                 /* Change settings on some selected endpoints */
1933                 tmp_reg = readl(&dev->plregs->pl_ep_cfg_4);
1934                 tmp_reg &= ~BIT(NON_CTRL_IN_TOLERATE_BAD_DIR);
1935                 writel(tmp_reg, &dev->plregs->pl_ep_cfg_4);
1936                 tmp_reg = readl(&dev->plregs->pl_ep_ctrl);
1937                 tmp_reg |= BIT(EP_INITIALIZED);
1938                 writel(tmp_reg, &dev->plregs->pl_ep_ctrl);
1939         }
1940 }
1941
1942 static void defect7374_enable_data_eps_zero(struct net2280 *dev)
1943 {
1944         u32 tmp = 0, tmp_reg;
1945         u32 scratch;
1946         int i;
1947         unsigned char ep_sel;
1948
1949         scratch = get_idx_reg(dev->regs, SCRATCH);
1950
1951         WARN_ON((scratch & (0xf << DEFECT7374_FSM_FIELD))
1952                 == DEFECT7374_FSM_SS_CONTROL_READ);
1953
1954         scratch &= ~(0xf << DEFECT7374_FSM_FIELD);
1955
1956         ep_warn(dev, "Operate Defect 7374 workaround soft this time");
1957         ep_warn(dev, "It will operate on cold-reboot and SS connect");
1958
1959         /*GPEPs:*/
1960         tmp = ((0 << ENDPOINT_NUMBER) | BIT(ENDPOINT_DIRECTION) |
1961                         (2 << OUT_ENDPOINT_TYPE) | (2 << IN_ENDPOINT_TYPE) |
1962                         ((dev->enhanced_mode) ?
1963                          BIT(OUT_ENDPOINT_ENABLE) | BIT(IN_ENDPOINT_ENABLE) :
1964                          BIT(ENDPOINT_ENABLE)));
1965
1966         for (i = 1; i < 5; i++)
1967                 writel(tmp, &dev->ep[i].cfg->ep_cfg);
1968
1969         /* CSRIN, PCIIN, STATIN, RCIN*/
1970         tmp = ((0 << ENDPOINT_NUMBER) | BIT(ENDPOINT_ENABLE));
1971         writel(tmp, &dev->dep[1].dep_cfg);
1972         writel(tmp, &dev->dep[3].dep_cfg);
1973         writel(tmp, &dev->dep[4].dep_cfg);
1974         writel(tmp, &dev->dep[5].dep_cfg);
1975
1976         /*Implemented for development and debug.
1977          * Can be refined/tuned later.*/
1978         for (ep_sel = 0; ep_sel <= 21; ep_sel++) {
1979                 /* Select an endpoint for subsequent operations: */
1980                 tmp_reg = readl(&dev->plregs->pl_ep_ctrl);
1981                 writel(((tmp_reg & ~0x1f) | ep_sel),
1982                                 &dev->plregs->pl_ep_ctrl);
1983
1984                 if (ep_sel == 1) {
1985                         tmp =
1986                                 (readl(&dev->plregs->pl_ep_ctrl) |
1987                                  BIT(CLEAR_ACK_ERROR_CODE) | 0);
1988                         writel(tmp, &dev->plregs->pl_ep_ctrl);
1989                         continue;
1990                 }
1991
1992                 if (ep_sel == 0 || (ep_sel > 9 && ep_sel < 14) ||
1993                                 ep_sel == 18  || ep_sel == 20)
1994                         continue;
1995
1996                 tmp = (readl(&dev->plregs->pl_ep_cfg_4) |
1997                                 BIT(NON_CTRL_IN_TOLERATE_BAD_DIR) | 0);
1998                 writel(tmp, &dev->plregs->pl_ep_cfg_4);
1999
2000                 tmp = readl(&dev->plregs->pl_ep_ctrl) &
2001                         ~BIT(EP_INITIALIZED);
2002                 writel(tmp, &dev->plregs->pl_ep_ctrl);
2003
2004         }
2005
2006         /* Set FSM to focus on the first Control Read:
2007          * - Tip: Connection speed is known upon the first
2008          * setup request.*/
2009         scratch |= DEFECT7374_FSM_WAITING_FOR_CONTROL_READ;
2010         set_idx_reg(dev->regs, SCRATCH, scratch);
2011
2012 }
2013
2014 /* keeping it simple:
2015  * - one bus driver, initted first;
2016  * - one function driver, initted second
2017  *
2018  * most of the work to support multiple net2280 controllers would
2019  * be to associate this gadget driver (yes?) with all of them, or
2020  * perhaps to bind specific drivers to specific devices.
2021  */
2022
2023 static void usb_reset_228x(struct net2280 *dev)
2024 {
2025         u32     tmp;
2026
2027         dev->gadget.speed = USB_SPEED_UNKNOWN;
2028         (void) readl(&dev->usb->usbctl);
2029
2030         net2280_led_init(dev);
2031
2032         /* disable automatic responses, and irqs */
2033         writel(0, &dev->usb->stdrsp);
2034         writel(0, &dev->regs->pciirqenb0);
2035         writel(0, &dev->regs->pciirqenb1);
2036
2037         /* clear old dma and irq state */
2038         for (tmp = 0; tmp < 4; tmp++) {
2039                 struct net2280_ep       *ep = &dev->ep[tmp + 1];
2040                 if (ep->dma)
2041                         abort_dma(ep);
2042         }
2043
2044         writel(~0, &dev->regs->irqstat0),
2045         writel(~(u32)BIT(SUSPEND_REQUEST_INTERRUPT), &dev->regs->irqstat1),
2046
2047         /* reset, and enable pci */
2048         tmp = readl(&dev->regs->devinit) |
2049                 BIT(PCI_ENABLE) |
2050                 BIT(FIFO_SOFT_RESET) |
2051                 BIT(USB_SOFT_RESET) |
2052                 BIT(M8051_RESET);
2053         writel(tmp, &dev->regs->devinit);
2054
2055         /* standard fifo and endpoint allocations */
2056         set_fifo_mode(dev, (fifo_mode <= 2) ? fifo_mode : 0);
2057 }
2058
2059 static void usb_reset_338x(struct net2280 *dev)
2060 {
2061         u32 tmp;
2062
2063         dev->gadget.speed = USB_SPEED_UNKNOWN;
2064         (void)readl(&dev->usb->usbctl);
2065
2066         net2280_led_init(dev);
2067
2068         if (dev->bug7734_patched) {
2069                 /* disable automatic responses, and irqs */
2070                 writel(0, &dev->usb->stdrsp);
2071                 writel(0, &dev->regs->pciirqenb0);
2072                 writel(0, &dev->regs->pciirqenb1);
2073         }
2074
2075         /* clear old dma and irq state */
2076         for (tmp = 0; tmp < 4; tmp++) {
2077                 struct net2280_ep *ep = &dev->ep[tmp + 1];
2078                 struct net2280_dma_regs __iomem *dma;
2079
2080                 if (ep->dma) {
2081                         abort_dma(ep);
2082                 } else {
2083                         dma = &dev->dma[tmp];
2084                         writel(BIT(DMA_ABORT), &dma->dmastat);
2085                         writel(0, &dma->dmactl);
2086                 }
2087         }
2088
2089         writel(~0, &dev->regs->irqstat0), writel(~0, &dev->regs->irqstat1);
2090
2091         if (dev->bug7734_patched) {
2092                 /* reset, and enable pci */
2093                 tmp = readl(&dev->regs->devinit) |
2094                     BIT(PCI_ENABLE) |
2095                     BIT(FIFO_SOFT_RESET) |
2096                     BIT(USB_SOFT_RESET) |
2097                     BIT(M8051_RESET);
2098
2099                 writel(tmp, &dev->regs->devinit);
2100         }
2101
2102         /* always ep-{1,2,3,4} ... maybe not ep-3 or ep-4 */
2103         INIT_LIST_HEAD(&dev->gadget.ep_list);
2104
2105         for (tmp = 1; tmp < dev->n_ep; tmp++)
2106                 list_add_tail(&dev->ep[tmp].ep.ep_list, &dev->gadget.ep_list);
2107
2108 }
2109
2110 static void usb_reset(struct net2280 *dev)
2111 {
2112         if (dev->quirks & PLX_LEGACY)
2113                 return usb_reset_228x(dev);
2114         return usb_reset_338x(dev);
2115 }
2116
2117 static void usb_reinit_228x(struct net2280 *dev)
2118 {
2119         u32     tmp;
2120
2121         /* basic endpoint init */
2122         for (tmp = 0; tmp < 7; tmp++) {
2123                 struct net2280_ep       *ep = &dev->ep[tmp];
2124
2125                 ep->ep.name = ep_info_dft[tmp].name;
2126                 ep->ep.caps = ep_info_dft[tmp].caps;
2127                 ep->dev = dev;
2128                 ep->num = tmp;
2129
2130                 if (tmp > 0 && tmp <= 4) {
2131                         ep->fifo_size = 1024;
2132                         ep->dma = &dev->dma[tmp - 1];
2133                 } else
2134                         ep->fifo_size = 64;
2135                 ep->regs = &dev->epregs[tmp];
2136                 ep->cfg = &dev->epregs[tmp];
2137                 ep_reset_228x(dev->regs, ep);
2138         }
2139         usb_ep_set_maxpacket_limit(&dev->ep[0].ep, 64);
2140         usb_ep_set_maxpacket_limit(&dev->ep[5].ep, 64);
2141         usb_ep_set_maxpacket_limit(&dev->ep[6].ep, 64);
2142
2143         dev->gadget.ep0 = &dev->ep[0].ep;
2144         dev->ep[0].stopped = 0;
2145         INIT_LIST_HEAD(&dev->gadget.ep0->ep_list);
2146
2147         /* we want to prevent lowlevel/insecure access from the USB host,
2148          * but erratum 0119 means this enable bit is ignored
2149          */
2150         for (tmp = 0; tmp < 5; tmp++)
2151                 writel(EP_DONTUSE, &dev->dep[tmp].dep_cfg);
2152 }
2153
2154 static void usb_reinit_338x(struct net2280 *dev)
2155 {
2156         int i;
2157         u32 tmp, val;
2158         static const u32 ne[9] = { 0, 1, 2, 3, 4, 1, 2, 3, 4 };
2159         static const u32 ep_reg_addr[9] = { 0x00, 0xC0, 0x00, 0xC0, 0x00,
2160                                                 0x00, 0xC0, 0x00, 0xC0 };
2161
2162         /* basic endpoint init */
2163         for (i = 0; i < dev->n_ep; i++) {
2164                 struct net2280_ep *ep = &dev->ep[i];
2165
2166                 ep->ep.name = dev->enhanced_mode ? ep_info_adv[i].name :
2167                                                    ep_info_dft[i].name;
2168                 ep->ep.caps = dev->enhanced_mode ? ep_info_adv[i].caps :
2169                                                    ep_info_dft[i].caps;
2170                 ep->dev = dev;
2171                 ep->num = i;
2172
2173                 if (i > 0 && i <= 4)
2174                         ep->dma = &dev->dma[i - 1];
2175
2176                 if (dev->enhanced_mode) {
2177                         ep->cfg = &dev->epregs[ne[i]];
2178                         /*
2179                          * Set USB endpoint number, hardware allows same number
2180                          * in both directions.
2181                          */
2182                          if (i > 0 && i < 5)
2183                                 writel(ne[i], &ep->cfg->ep_cfg);
2184                         ep->regs = (struct net2280_ep_regs __iomem *)
2185                                 (((void __iomem *)&dev->epregs[ne[i]]) +
2186                                 ep_reg_addr[i]);
2187                 } else {
2188                         ep->cfg = &dev->epregs[i];
2189                         ep->regs = &dev->epregs[i];
2190                 }
2191
2192                 ep->fifo_size = (i != 0) ? 2048 : 512;
2193
2194                 ep_reset_338x(dev->regs, ep);
2195         }
2196         usb_ep_set_maxpacket_limit(&dev->ep[0].ep, 512);
2197
2198         dev->gadget.ep0 = &dev->ep[0].ep;
2199         dev->ep[0].stopped = 0;
2200
2201         /* Link layer set up */
2202         if (dev->bug7734_patched) {
2203                 tmp = readl(&dev->usb_ext->usbctl2) &
2204                     ~(BIT(U1_ENABLE) | BIT(U2_ENABLE) | BIT(LTM_ENABLE));
2205                 writel(tmp, &dev->usb_ext->usbctl2);
2206         }
2207
2208         /* Hardware Defect and Workaround */
2209         val = readl(&dev->ll_lfps_regs->ll_lfps_5);
2210         val &= ~(0xf << TIMER_LFPS_6US);
2211         val |= 0x5 << TIMER_LFPS_6US;
2212         writel(val, &dev->ll_lfps_regs->ll_lfps_5);
2213
2214         val = readl(&dev->ll_lfps_regs->ll_lfps_6);
2215         val &= ~(0xffff << TIMER_LFPS_80US);
2216         val |= 0x0100 << TIMER_LFPS_80US;
2217         writel(val, &dev->ll_lfps_regs->ll_lfps_6);
2218
2219         /*
2220          * AA_AB Errata. Issue 4. Workaround for SuperSpeed USB
2221          * Hot Reset Exit Handshake may Fail in Specific Case using
2222          * Default Register Settings. Workaround for Enumeration test.
2223          */
2224         val = readl(&dev->ll_tsn_regs->ll_tsn_counters_2);
2225         val &= ~(0x1f << HOT_TX_NORESET_TS2);
2226         val |= 0x10 << HOT_TX_NORESET_TS2;
2227         writel(val, &dev->ll_tsn_regs->ll_tsn_counters_2);
2228
2229         val = readl(&dev->ll_tsn_regs->ll_tsn_counters_3);
2230         val &= ~(0x1f << HOT_RX_RESET_TS2);
2231         val |= 0x3 << HOT_RX_RESET_TS2;
2232         writel(val, &dev->ll_tsn_regs->ll_tsn_counters_3);
2233
2234         /*
2235          * Set Recovery Idle to Recover bit:
2236          * - On SS connections, setting Recovery Idle to Recover Fmw improves
2237          *   link robustness with various hosts and hubs.
2238          * - It is safe to set for all connection speeds; all chip revisions.
2239          * - R-M-W to leave other bits undisturbed.
2240          * - Reference PLX TT-7372
2241         */
2242         val = readl(&dev->ll_chicken_reg->ll_tsn_chicken_bit);
2243         val |= BIT(RECOVERY_IDLE_TO_RECOVER_FMW);
2244         writel(val, &dev->ll_chicken_reg->ll_tsn_chicken_bit);
2245
2246         INIT_LIST_HEAD(&dev->gadget.ep0->ep_list);
2247
2248         /* disable dedicated endpoints */
2249         writel(0x0D, &dev->dep[0].dep_cfg);
2250         writel(0x0D, &dev->dep[1].dep_cfg);
2251         writel(0x0E, &dev->dep[2].dep_cfg);
2252         writel(0x0E, &dev->dep[3].dep_cfg);
2253         writel(0x0F, &dev->dep[4].dep_cfg);
2254         writel(0x0C, &dev->dep[5].dep_cfg);
2255 }
2256
2257 static void usb_reinit(struct net2280 *dev)
2258 {
2259         if (dev->quirks & PLX_LEGACY)
2260                 return usb_reinit_228x(dev);
2261         return usb_reinit_338x(dev);
2262 }
2263
2264 static void ep0_start_228x(struct net2280 *dev)
2265 {
2266         writel(BIT(CLEAR_EP_HIDE_STATUS_PHASE) |
2267                 BIT(CLEAR_NAK_OUT_PACKETS) |
2268                 BIT(CLEAR_CONTROL_STATUS_PHASE_HANDSHAKE),
2269                 &dev->epregs[0].ep_rsp);
2270
2271         /*
2272          * hardware optionally handles a bunch of standard requests
2273          * that the API hides from drivers anyway.  have it do so.
2274          * endpoint status/features are handled in software, to
2275          * help pass tests for some dubious behavior.
2276          */
2277         writel(BIT(SET_TEST_MODE) |
2278                 BIT(SET_ADDRESS) |
2279                 BIT(DEVICE_SET_CLEAR_DEVICE_REMOTE_WAKEUP) |
2280                 BIT(GET_DEVICE_STATUS) |
2281                 BIT(GET_INTERFACE_STATUS),
2282                 &dev->usb->stdrsp);
2283         writel(BIT(USB_ROOT_PORT_WAKEUP_ENABLE) |
2284                 BIT(SELF_POWERED_USB_DEVICE) |
2285                 BIT(REMOTE_WAKEUP_SUPPORT) |
2286                 (dev->softconnect << USB_DETECT_ENABLE) |
2287                 BIT(SELF_POWERED_STATUS),
2288                 &dev->usb->usbctl);
2289
2290         /* enable irqs so we can see ep0 and general operation  */
2291         writel(BIT(SETUP_PACKET_INTERRUPT_ENABLE) |
2292                 BIT(ENDPOINT_0_INTERRUPT_ENABLE),
2293                 &dev->regs->pciirqenb0);
2294         writel(BIT(PCI_INTERRUPT_ENABLE) |
2295                 BIT(PCI_MASTER_ABORT_RECEIVED_INTERRUPT_ENABLE) |
2296                 BIT(PCI_TARGET_ABORT_RECEIVED_INTERRUPT_ENABLE) |
2297                 BIT(PCI_RETRY_ABORT_INTERRUPT_ENABLE) |
2298                 BIT(VBUS_INTERRUPT_ENABLE) |
2299                 BIT(ROOT_PORT_RESET_INTERRUPT_ENABLE) |
2300                 BIT(SUSPEND_REQUEST_CHANGE_INTERRUPT_ENABLE),
2301                 &dev->regs->pciirqenb1);
2302
2303         /* don't leave any writes posted */
2304         (void) readl(&dev->usb->usbctl);
2305 }
2306
2307 static void ep0_start_338x(struct net2280 *dev)
2308 {
2309
2310         if (dev->bug7734_patched)
2311                 writel(BIT(CLEAR_NAK_OUT_PACKETS_MODE) |
2312                        BIT(SET_EP_HIDE_STATUS_PHASE),
2313                        &dev->epregs[0].ep_rsp);
2314
2315         /*
2316          * hardware optionally handles a bunch of standard requests
2317          * that the API hides from drivers anyway.  have it do so.
2318          * endpoint status/features are handled in software, to
2319          * help pass tests for some dubious behavior.
2320          */
2321         writel(BIT(SET_ISOCHRONOUS_DELAY) |
2322                BIT(SET_SEL) |
2323                BIT(SET_TEST_MODE) |
2324                BIT(SET_ADDRESS) |
2325                BIT(GET_INTERFACE_STATUS) |
2326                BIT(GET_DEVICE_STATUS),
2327                 &dev->usb->stdrsp);
2328         dev->wakeup_enable = 1;
2329         writel(BIT(USB_ROOT_PORT_WAKEUP_ENABLE) |
2330                (dev->softconnect << USB_DETECT_ENABLE) |
2331                BIT(DEVICE_REMOTE_WAKEUP_ENABLE),
2332                &dev->usb->usbctl);
2333
2334         /* enable irqs so we can see ep0 and general operation  */
2335         writel(BIT(SETUP_PACKET_INTERRUPT_ENABLE) |
2336                BIT(ENDPOINT_0_INTERRUPT_ENABLE),
2337                &dev->regs->pciirqenb0);
2338         writel(BIT(PCI_INTERRUPT_ENABLE) |
2339                BIT(ROOT_PORT_RESET_INTERRUPT_ENABLE) |
2340                BIT(SUSPEND_REQUEST_CHANGE_INTERRUPT_ENABLE) |
2341                BIT(VBUS_INTERRUPT_ENABLE),
2342                &dev->regs->pciirqenb1);
2343
2344         /* don't leave any writes posted */
2345         (void)readl(&dev->usb->usbctl);
2346 }
2347
2348 static void ep0_start(struct net2280 *dev)
2349 {
2350         if (dev->quirks & PLX_LEGACY)
2351                 return ep0_start_228x(dev);
2352         return ep0_start_338x(dev);
2353 }
2354
2355 /* when a driver is successfully registered, it will receive
2356  * control requests including set_configuration(), which enables
2357  * non-control requests.  then usb traffic follows until a
2358  * disconnect is reported.  then a host may connect again, or
2359  * the driver might get unbound.
2360  */
2361 static int net2280_start(struct usb_gadget *_gadget,
2362                 struct usb_gadget_driver *driver)
2363 {
2364         struct net2280          *dev;
2365         int                     retval;
2366         unsigned                i;
2367
2368         /* insist on high speed support from the driver, since
2369          * (dev->usb->xcvrdiag & FORCE_FULL_SPEED_MODE)
2370          * "must not be used in normal operation"
2371          */
2372         if (!driver || driver->max_speed < USB_SPEED_HIGH ||
2373                         !driver->setup)
2374                 return -EINVAL;
2375
2376         dev = container_of(_gadget, struct net2280, gadget);
2377
2378         for (i = 0; i < dev->n_ep; i++)
2379                 dev->ep[i].irqs = 0;
2380
2381         /* hook up the driver ... */
2382         driver->driver.bus = NULL;
2383         dev->driver = driver;
2384
2385         retval = device_create_file(&dev->pdev->dev, &dev_attr_function);
2386         if (retval)
2387                 goto err_unbind;
2388         retval = device_create_file(&dev->pdev->dev, &dev_attr_queues);
2389         if (retval)
2390                 goto err_func;
2391
2392         /* enable host detection and ep0; and we're ready
2393          * for set_configuration as well as eventual disconnect.
2394          */
2395         net2280_led_active(dev, 1);
2396
2397         if ((dev->quirks & PLX_SUPERSPEED) && !dev->bug7734_patched)
2398                 defect7374_enable_data_eps_zero(dev);
2399
2400         ep0_start(dev);
2401
2402         /* pci writes may still be posted */
2403         return 0;
2404
2405 err_func:
2406         device_remove_file(&dev->pdev->dev, &dev_attr_function);
2407 err_unbind:
2408         dev->driver = NULL;
2409         return retval;
2410 }
2411
2412 static void stop_activity(struct net2280 *dev, struct usb_gadget_driver *driver)
2413 {
2414         int                     i;
2415
2416         /* don't disconnect if it's not connected */
2417         if (dev->gadget.speed == USB_SPEED_UNKNOWN)
2418                 driver = NULL;
2419
2420         /* stop hardware; prevent new request submissions;
2421          * and kill any outstanding requests.
2422          */
2423         usb_reset(dev);
2424         for (i = 0; i < dev->n_ep; i++)
2425                 nuke(&dev->ep[i]);
2426
2427         /* report disconnect; the driver is already quiesced */
2428         if (driver) {
2429                 spin_unlock(&dev->lock);
2430                 driver->disconnect(&dev->gadget);
2431                 spin_lock(&dev->lock);
2432         }
2433
2434         usb_reinit(dev);
2435 }
2436
2437 static int net2280_stop(struct usb_gadget *_gadget)
2438 {
2439         struct net2280  *dev;
2440         unsigned long   flags;
2441
2442         dev = container_of(_gadget, struct net2280, gadget);
2443
2444         spin_lock_irqsave(&dev->lock, flags);
2445         stop_activity(dev, NULL);
2446         spin_unlock_irqrestore(&dev->lock, flags);
2447
2448         net2280_led_active(dev, 0);
2449
2450         device_remove_file(&dev->pdev->dev, &dev_attr_function);
2451         device_remove_file(&dev->pdev->dev, &dev_attr_queues);
2452
2453         dev->driver = NULL;
2454
2455         return 0;
2456 }
2457
2458 /*-------------------------------------------------------------------------*/
2459
2460 /* handle ep0, ep-e, ep-f with 64 byte packets: packet per irq.
2461  * also works for dma-capable endpoints, in pio mode or just
2462  * to manually advance the queue after short OUT transfers.
2463  */
2464 static void handle_ep_small(struct net2280_ep *ep)
2465 {
2466         struct net2280_request  *req;
2467         u32                     t;
2468         /* 0 error, 1 mid-data, 2 done */
2469         int                     mode = 1;
2470
2471         if (!list_empty(&ep->queue))
2472                 req = list_entry(ep->queue.next,
2473                         struct net2280_request, queue);
2474         else
2475                 req = NULL;
2476
2477         /* ack all, and handle what we care about */
2478         t = readl(&ep->regs->ep_stat);
2479         ep->irqs++;
2480
2481         ep_vdbg(ep->dev, "%s ack ep_stat %08x, req %p\n",
2482                         ep->ep.name, t, req ? &req->req : NULL);
2483
2484         if (!ep->is_in || (ep->dev->quirks & PLX_2280))
2485                 writel(t & ~BIT(NAK_OUT_PACKETS), &ep->regs->ep_stat);
2486         else
2487                 /* Added for 2282 */
2488                 writel(t, &ep->regs->ep_stat);
2489
2490         /* for ep0, monitor token irqs to catch data stage length errors
2491          * and to synchronize on status.
2492          *
2493          * also, to defer reporting of protocol stalls ... here's where
2494          * data or status first appears, handling stalls here should never
2495          * cause trouble on the host side..
2496          *
2497          * control requests could be slightly faster without token synch for
2498          * status, but status can jam up that way.
2499          */
2500         if (unlikely(ep->num == 0)) {
2501                 if (ep->is_in) {
2502                         /* status; stop NAKing */
2503                         if (t & BIT(DATA_OUT_PING_TOKEN_INTERRUPT)) {
2504                                 if (ep->dev->protocol_stall) {
2505                                         ep->stopped = 1;
2506                                         set_halt(ep);
2507                                 }
2508                                 if (!req)
2509                                         allow_status(ep);
2510                                 mode = 2;
2511                         /* reply to extra IN data tokens with a zlp */
2512                         } else if (t & BIT(DATA_IN_TOKEN_INTERRUPT)) {
2513                                 if (ep->dev->protocol_stall) {
2514                                         ep->stopped = 1;
2515                                         set_halt(ep);
2516                                         mode = 2;
2517                                 } else if (ep->responded &&
2518                                                 !req && !ep->stopped)
2519                                         write_fifo(ep, NULL);
2520                         }
2521                 } else {
2522                         /* status; stop NAKing */
2523                         if (t & BIT(DATA_IN_TOKEN_INTERRUPT)) {
2524                                 if (ep->dev->protocol_stall) {
2525                                         ep->stopped = 1;
2526                                         set_halt(ep);
2527                                 }
2528                                 mode = 2;
2529                         /* an extra OUT token is an error */
2530                         } else if (((t & BIT(DATA_OUT_PING_TOKEN_INTERRUPT)) &&
2531                                         req &&
2532                                         req->req.actual == req->req.length) ||
2533                                         (ep->responded && !req)) {
2534                                 ep->dev->protocol_stall = 1;
2535                                 set_halt(ep);
2536                                 ep->stopped = 1;
2537                                 if (req)
2538                                         done(ep, req, -EOVERFLOW);
2539                                 req = NULL;
2540                         }
2541                 }
2542         }
2543
2544         if (unlikely(!req))
2545                 return;
2546
2547         /* manual DMA queue advance after short OUT */
2548         if (likely(ep->dma)) {
2549                 if (t & BIT(SHORT_PACKET_TRANSFERRED_INTERRUPT)) {
2550                         u32     count;
2551                         int     stopped = ep->stopped;
2552
2553                         /* TRANSFERRED works around OUT_DONE erratum 0112.
2554                          * we expect (N <= maxpacket) bytes; host wrote M.
2555                          * iff (M < N) we won't ever see a DMA interrupt.
2556                          */
2557                         ep->stopped = 1;
2558                         for (count = 0; ; t = readl(&ep->regs->ep_stat)) {
2559
2560                                 /* any preceding dma transfers must finish.
2561                                  * dma handles (M >= N), may empty the queue
2562                                  */
2563                                 scan_dma_completions(ep);
2564                                 if (unlikely(list_empty(&ep->queue) ||
2565                                                 ep->out_overflow)) {
2566                                         req = NULL;
2567                                         break;
2568                                 }
2569                                 req = list_entry(ep->queue.next,
2570                                         struct net2280_request, queue);
2571
2572                                 /* here either (M < N), a "real" short rx;
2573                                  * or (M == N) and the queue didn't empty
2574                                  */
2575                                 if (likely(t & BIT(FIFO_EMPTY))) {
2576                                         count = readl(&ep->dma->dmacount);
2577                                         count &= DMA_BYTE_COUNT_MASK;
2578                                         if (readl(&ep->dma->dmadesc)
2579                                                         != req->td_dma)
2580                                                 req = NULL;
2581                                         break;
2582                                 }
2583                                 udelay(1);
2584                         }
2585
2586                         /* stop DMA, leave ep NAKing */
2587                         writel(BIT(DMA_ABORT), &ep->dma->dmastat);
2588                         spin_stop_dma(ep->dma);
2589
2590                         if (likely(req)) {
2591                                 req->td->dmacount = 0;
2592                                 t = readl(&ep->regs->ep_avail);
2593                                 dma_done(ep, req, count,
2594                                         (ep->out_overflow || t)
2595                                                 ? -EOVERFLOW : 0);
2596                         }
2597
2598                         /* also flush to prevent erratum 0106 trouble */
2599                         if (unlikely(ep->out_overflow ||
2600                                         (ep->dev->chiprev == 0x0100 &&
2601                                         ep->dev->gadget.speed
2602                                         == USB_SPEED_FULL))) {
2603                                 out_flush(ep);
2604                                 ep->out_overflow = 0;
2605                         }
2606
2607                         /* (re)start dma if needed, stop NAKing */
2608                         ep->stopped = stopped;
2609                         if (!list_empty(&ep->queue))
2610                                 restart_dma(ep);
2611                 } else
2612                         ep_dbg(ep->dev, "%s dma ep_stat %08x ??\n",
2613                                         ep->ep.name, t);
2614                 return;
2615
2616         /* data packet(s) received (in the fifo, OUT) */
2617         } else if (t & BIT(DATA_PACKET_RECEIVED_INTERRUPT)) {
2618                 if (read_fifo(ep, req) && ep->num != 0)
2619                         mode = 2;
2620
2621         /* data packet(s) transmitted (IN) */
2622         } else if (t & BIT(DATA_PACKET_TRANSMITTED_INTERRUPT)) {
2623                 unsigned        len;
2624
2625                 len = req->req.length - req->req.actual;
2626                 if (len > ep->ep.maxpacket)
2627                         len = ep->ep.maxpacket;
2628                 req->req.actual += len;
2629
2630                 /* if we wrote it all, we're usually done */
2631                 /* send zlps until the status stage */
2632                 if ((req->req.actual == req->req.length) &&
2633                         (!req->req.zero || len != ep->ep.maxpacket) && ep->num)
2634                                 mode = 2;
2635
2636         /* there was nothing to do ...  */
2637         } else if (mode == 1)
2638                 return;
2639
2640         /* done */
2641         if (mode == 2) {
2642                 /* stream endpoints often resubmit/unlink in completion */
2643                 done(ep, req, 0);
2644
2645                 /* maybe advance queue to next request */
2646                 if (ep->num == 0) {
2647                         /* NOTE:  net2280 could let gadget driver start the
2648                          * status stage later. since not all controllers let
2649                          * them control that, the api doesn't (yet) allow it.
2650                          */
2651                         if (!ep->stopped)
2652                                 allow_status(ep);
2653                         req = NULL;
2654                 } else {
2655                         if (!list_empty(&ep->queue) && !ep->stopped)
2656                                 req = list_entry(ep->queue.next,
2657                                         struct net2280_request, queue);
2658                         else
2659                                 req = NULL;
2660                         if (req && !ep->is_in)
2661                                 stop_out_naking(ep);
2662                 }
2663         }
2664
2665         /* is there a buffer for the next packet?
2666          * for best streaming performance, make sure there is one.
2667          */
2668         if (req && !ep->stopped) {
2669
2670                 /* load IN fifo with next packet (may be zlp) */
2671                 if (t & BIT(DATA_PACKET_TRANSMITTED_INTERRUPT))
2672                         write_fifo(ep, &req->req);
2673         }
2674 }
2675
2676 static struct net2280_ep *get_ep_by_addr(struct net2280 *dev, u16 wIndex)
2677 {
2678         struct net2280_ep       *ep;
2679
2680         if ((wIndex & USB_ENDPOINT_NUMBER_MASK) == 0)
2681                 return &dev->ep[0];
2682         list_for_each_entry(ep, &dev->gadget.ep_list, ep.ep_list) {
2683                 u8      bEndpointAddress;
2684
2685                 if (!ep->desc)
2686                         continue;
2687                 bEndpointAddress = ep->desc->bEndpointAddress;
2688                 if ((wIndex ^ bEndpointAddress) & USB_DIR_IN)
2689                         continue;
2690                 if ((wIndex & 0x0f) == (bEndpointAddress & 0x0f))
2691                         return ep;
2692         }
2693         return NULL;
2694 }
2695
2696 static void defect7374_workaround(struct net2280 *dev, struct usb_ctrlrequest r)
2697 {
2698         u32 scratch, fsmvalue;
2699         u32 ack_wait_timeout, state;
2700
2701         /* Workaround for Defect 7374 (U1/U2 erroneously rejected): */
2702         scratch = get_idx_reg(dev->regs, SCRATCH);
2703         fsmvalue = scratch & (0xf << DEFECT7374_FSM_FIELD);
2704         scratch &= ~(0xf << DEFECT7374_FSM_FIELD);
2705
2706         if (!((fsmvalue == DEFECT7374_FSM_WAITING_FOR_CONTROL_READ) &&
2707                                 (r.bRequestType & USB_DIR_IN)))
2708                 return;
2709
2710         /* This is the first Control Read for this connection: */
2711         if (!(readl(&dev->usb->usbstat) & BIT(SUPER_SPEED_MODE))) {
2712                 /*
2713                  * Connection is NOT SS:
2714                  * - Connection must be FS or HS.
2715                  * - This FSM state should allow workaround software to
2716                  * run after the next USB connection.
2717                  */
2718                 scratch |= DEFECT7374_FSM_NON_SS_CONTROL_READ;
2719                 dev->bug7734_patched = 1;
2720                 goto restore_data_eps;
2721         }
2722
2723         /* Connection is SS: */
2724         for (ack_wait_timeout = 0;
2725                         ack_wait_timeout < DEFECT_7374_NUMBEROF_MAX_WAIT_LOOPS;
2726                         ack_wait_timeout++) {
2727
2728                 state = readl(&dev->plregs->pl_ep_status_1)
2729                         & (0xff << STATE);
2730                 if ((state >= (ACK_GOOD_NORMAL << STATE)) &&
2731                         (state <= (ACK_GOOD_MORE_ACKS_TO_COME << STATE))) {
2732                         scratch |= DEFECT7374_FSM_SS_CONTROL_READ;
2733                         dev->bug7734_patched = 1;
2734                         break;
2735                 }
2736
2737                 /*
2738                  * We have not yet received host's Data Phase ACK
2739                  * - Wait and try again.
2740                  */
2741                 udelay(DEFECT_7374_PROCESSOR_WAIT_TIME);
2742
2743                 continue;
2744         }
2745
2746
2747         if (ack_wait_timeout >= DEFECT_7374_NUMBEROF_MAX_WAIT_LOOPS) {
2748                 ep_err(dev, "FAIL: Defect 7374 workaround waited but failed "
2749                 "to detect SS host's data phase ACK.");
2750                 ep_err(dev, "PL_EP_STATUS_1(23:16):.Expected from 0x11 to 0x16"
2751                 "got 0x%2.2x.\n", state >> STATE);
2752         } else {
2753                 ep_warn(dev, "INFO: Defect 7374 workaround waited about\n"
2754                 "%duSec for Control Read Data Phase ACK\n",
2755                         DEFECT_7374_PROCESSOR_WAIT_TIME * ack_wait_timeout);
2756         }
2757
2758 restore_data_eps:
2759         /*
2760          * Restore data EPs to their pre-workaround settings (disabled,
2761          * initialized, and other details).
2762          */
2763         defect7374_disable_data_eps(dev);
2764
2765         set_idx_reg(dev->regs, SCRATCH, scratch);
2766
2767         return;
2768 }
2769
2770 static void ep_clear_seqnum(struct net2280_ep *ep)
2771 {
2772         struct net2280 *dev = ep->dev;
2773         u32 val;
2774         static const u32 ep_pl[9] = { 0, 3, 4, 7, 8, 2, 5, 6, 9 };
2775
2776         val = readl(&dev->plregs->pl_ep_ctrl) & ~0x1f;
2777         val |= ep_pl[ep->num];
2778         writel(val, &dev->plregs->pl_ep_ctrl);
2779         val |= BIT(SEQUENCE_NUMBER_RESET);
2780         writel(val, &dev->plregs->pl_ep_ctrl);
2781
2782         return;
2783 }
2784
2785 static void handle_stat0_irqs_superspeed(struct net2280 *dev,
2786                 struct net2280_ep *ep, struct usb_ctrlrequest r)
2787 {
2788         int tmp = 0;
2789
2790 #define w_value         le16_to_cpu(r.wValue)
2791 #define w_index         le16_to_cpu(r.wIndex)
2792 #define w_length        le16_to_cpu(r.wLength)
2793
2794         switch (r.bRequest) {
2795                 struct net2280_ep *e;
2796                 u16 status;
2797
2798         case USB_REQ_SET_CONFIGURATION:
2799                 dev->addressed_state = !w_value;
2800                 goto usb3_delegate;
2801
2802         case USB_REQ_GET_STATUS:
2803                 switch (r.bRequestType) {
2804                 case (USB_DIR_IN | USB_TYPE_STANDARD | USB_RECIP_DEVICE):
2805                         status = dev->wakeup_enable ? 0x02 : 0x00;
2806                         if (dev->gadget.is_selfpowered)
2807                                 status |= BIT(0);
2808                         status |= (dev->u1_enable << 2 | dev->u2_enable << 3 |
2809                                                         dev->ltm_enable << 4);
2810                         writel(0, &dev->epregs[0].ep_irqenb);
2811                         set_fifo_bytecount(ep, sizeof(status));
2812                         writel((__force u32) status, &dev->epregs[0].ep_data);
2813                         allow_status_338x(ep);
2814                         break;
2815
2816                 case (USB_DIR_IN | USB_TYPE_STANDARD | USB_RECIP_ENDPOINT):
2817                         e = get_ep_by_addr(dev, w_index);
2818                         if (!e)
2819                                 goto do_stall3;
2820                         status = readl(&e->regs->ep_rsp) &
2821                                                 BIT(CLEAR_ENDPOINT_HALT);
2822                         writel(0, &dev->epregs[0].ep_irqenb);
2823                         set_fifo_bytecount(ep, sizeof(status));
2824                         writel((__force u32) status, &dev->epregs[0].ep_data);
2825                         allow_status_338x(ep);
2826                         break;
2827
2828                 default:
2829                         goto usb3_delegate;
2830                 }
2831                 break;
2832
2833         case USB_REQ_CLEAR_FEATURE:
2834                 switch (r.bRequestType) {
2835                 case (USB_DIR_OUT | USB_TYPE_STANDARD | USB_RECIP_DEVICE):
2836                         if (!dev->addressed_state) {
2837                                 switch (w_value) {
2838                                 case USB_DEVICE_U1_ENABLE:
2839                                         dev->u1_enable = 0;
2840                                         writel(readl(&dev->usb_ext->usbctl2) &
2841                                                 ~BIT(U1_ENABLE),
2842                                                 &dev->usb_ext->usbctl2);
2843                                         allow_status_338x(ep);
2844                                         goto next_endpoints3;
2845
2846                                 case USB_DEVICE_U2_ENABLE:
2847                                         dev->u2_enable = 0;
2848                                         writel(readl(&dev->usb_ext->usbctl2) &
2849                                                 ~BIT(U2_ENABLE),
2850                                                 &dev->usb_ext->usbctl2);
2851                                         allow_status_338x(ep);
2852                                         goto next_endpoints3;
2853
2854                                 case USB_DEVICE_LTM_ENABLE:
2855                                         dev->ltm_enable = 0;
2856                                         writel(readl(&dev->usb_ext->usbctl2) &
2857                                                 ~BIT(LTM_ENABLE),
2858                                                 &dev->usb_ext->usbctl2);
2859                                         allow_status_338x(ep);
2860                                         goto next_endpoints3;
2861
2862                                 default:
2863                                         break;
2864                                 }
2865                         }
2866                         if (w_value == USB_DEVICE_REMOTE_WAKEUP) {
2867                                 dev->wakeup_enable = 0;
2868                                 writel(readl(&dev->usb->usbctl) &
2869                                         ~BIT(DEVICE_REMOTE_WAKEUP_ENABLE),
2870                                         &dev->usb->usbctl);
2871                                 allow_status_338x(ep);
2872                                 break;
2873                         }
2874                         goto usb3_delegate;
2875
2876                 case (USB_DIR_OUT | USB_TYPE_STANDARD | USB_RECIP_ENDPOINT):
2877                         e = get_ep_by_addr(dev, w_index);
2878                         if (!e)
2879                                 goto do_stall3;
2880                         if (w_value != USB_ENDPOINT_HALT)
2881                                 goto do_stall3;
2882                         ep_vdbg(dev, "%s clear halt\n", e->ep.name);
2883                         /*
2884                          * Workaround for SS SeqNum not cleared via
2885                          * Endpoint Halt (Clear) bit. select endpoint
2886                          */
2887                         ep_clear_seqnum(e);
2888                         clear_halt(e);
2889                         if (!list_empty(&e->queue) && e->td_dma)
2890                                 restart_dma(e);
2891                         allow_status(ep);
2892                         ep->stopped = 1;
2893                         break;
2894
2895                 default:
2896                         goto usb3_delegate;
2897                 }
2898                 break;
2899         case USB_REQ_SET_FEATURE:
2900                 switch (r.bRequestType) {
2901                 case (USB_DIR_OUT | USB_TYPE_STANDARD | USB_RECIP_DEVICE):
2902                         if (!dev->addressed_state) {
2903                                 switch (w_value) {
2904                                 case USB_DEVICE_U1_ENABLE:
2905                                         dev->u1_enable = 1;
2906                                         writel(readl(&dev->usb_ext->usbctl2) |
2907                                                 BIT(U1_ENABLE),
2908                                                 &dev->usb_ext->usbctl2);
2909                                         allow_status_338x(ep);
2910                                         goto next_endpoints3;
2911
2912                                 case USB_DEVICE_U2_ENABLE:
2913                                         dev->u2_enable = 1;
2914                                         writel(readl(&dev->usb_ext->usbctl2) |
2915                                                 BIT(U2_ENABLE),
2916                                                 &dev->usb_ext->usbctl2);
2917                                         allow_status_338x(ep);
2918                                         goto next_endpoints3;
2919
2920                                 case USB_DEVICE_LTM_ENABLE:
2921                                         dev->ltm_enable = 1;
2922                                         writel(readl(&dev->usb_ext->usbctl2) |
2923                                                 BIT(LTM_ENABLE),
2924                                                 &dev->usb_ext->usbctl2);
2925                                         allow_status_338x(ep);
2926                                         goto next_endpoints3;
2927                                 default:
2928                                         break;
2929                                 }
2930                         }
2931
2932                         if (w_value == USB_DEVICE_REMOTE_WAKEUP) {
2933                                 dev->wakeup_enable = 1;
2934                                 writel(readl(&dev->usb->usbctl) |
2935                                         BIT(DEVICE_REMOTE_WAKEUP_ENABLE),
2936                                         &dev->usb->usbctl);
2937                                 allow_status_338x(ep);
2938                                 break;
2939                         }
2940                         goto usb3_delegate;
2941
2942                 case (USB_DIR_OUT | USB_TYPE_STANDARD | USB_RECIP_ENDPOINT):
2943                         e = get_ep_by_addr(dev, w_index);
2944                         if (!e || (w_value != USB_ENDPOINT_HALT))
2945                                 goto do_stall3;
2946                         ep->stopped = 1;
2947                         if (ep->num == 0)
2948                                 ep->dev->protocol_stall = 1;
2949                         else {
2950                                 if (ep->dma)
2951                                         abort_dma(ep);
2952                                 set_halt(ep);
2953                         }
2954                         allow_status_338x(ep);
2955                         break;
2956
2957                 default:
2958                         goto usb3_delegate;
2959                 }
2960
2961                 break;
2962         default:
2963
2964 usb3_delegate:
2965                 ep_vdbg(dev, "setup %02x.%02x v%04x i%04x l%04x ep_cfg %08x\n",
2966                                 r.bRequestType, r.bRequest,
2967                                 w_value, w_index, w_length,
2968                                 readl(&ep->cfg->ep_cfg));
2969
2970                 ep->responded = 0;
2971                 spin_unlock(&dev->lock);
2972                 tmp = dev->driver->setup(&dev->gadget, &r);
2973                 spin_lock(&dev->lock);
2974         }
2975 do_stall3:
2976         if (tmp < 0) {
2977                 ep_vdbg(dev, "req %02x.%02x protocol STALL; stat %d\n",
2978                                 r.bRequestType, r.bRequest, tmp);
2979                 dev->protocol_stall = 1;
2980                 /* TD 9.9 Halt Endpoint test. TD 9.22 Set feature test */
2981                 set_halt(ep);
2982         }
2983
2984 next_endpoints3:
2985
2986 #undef  w_value
2987 #undef  w_index
2988 #undef  w_length
2989
2990         return;
2991 }
2992
2993 static void usb338x_handle_ep_intr(struct net2280 *dev, u32 stat0)
2994 {
2995         u32 index;
2996         u32 bit;
2997
2998         for (index = 0; index < ARRAY_SIZE(ep_bit); index++) {
2999                 bit = BIT(ep_bit[index]);
3000
3001                 if (!stat0)
3002                         break;
3003
3004                 if (!(stat0 & bit))
3005                         continue;
3006
3007                 stat0 &= ~bit;
3008
3009                 handle_ep_small(&dev->ep[index]);
3010         }
3011 }
3012
3013 static void handle_stat0_irqs(struct net2280 *dev, u32 stat)
3014 {
3015         struct net2280_ep       *ep;
3016         u32                     num, scratch;
3017
3018         /* most of these don't need individual acks */
3019         stat &= ~BIT(INTA_ASSERTED);
3020         if (!stat)
3021                 return;
3022         /* ep_dbg(dev, "irqstat0 %04x\n", stat); */
3023
3024         /* starting a control request? */
3025         if (unlikely(stat & BIT(SETUP_PACKET_INTERRUPT))) {
3026                 union {
3027                         u32                     raw[2];
3028                         struct usb_ctrlrequest  r;
3029                 } u;
3030                 int                             tmp;
3031                 struct net2280_request          *req;
3032
3033                 if (dev->gadget.speed == USB_SPEED_UNKNOWN) {
3034                         u32 val = readl(&dev->usb->usbstat);
3035                         if (val & BIT(SUPER_SPEED)) {
3036                                 dev->gadget.speed = USB_SPEED_SUPER;
3037                                 usb_ep_set_maxpacket_limit(&dev->ep[0].ep,
3038                                                 EP0_SS_MAX_PACKET_SIZE);
3039                         } else if (val & BIT(HIGH_SPEED)) {
3040                                 dev->gadget.speed = USB_SPEED_HIGH;
3041                                 usb_ep_set_maxpacket_limit(&dev->ep[0].ep,
3042                                                 EP0_HS_MAX_PACKET_SIZE);
3043                         } else {
3044                                 dev->gadget.speed = USB_SPEED_FULL;
3045                                 usb_ep_set_maxpacket_limit(&dev->ep[0].ep,
3046                                                 EP0_HS_MAX_PACKET_SIZE);
3047                         }
3048                         net2280_led_speed(dev, dev->gadget.speed);
3049                         ep_dbg(dev, "%s\n",
3050                                         usb_speed_string(dev->gadget.speed));
3051                 }
3052
3053                 ep = &dev->ep[0];
3054                 ep->irqs++;
3055
3056                 /* make sure any leftover request state is cleared */
3057                 stat &= ~BIT(ENDPOINT_0_INTERRUPT);
3058                 while (!list_empty(&ep->queue)) {
3059                         req = list_entry(ep->queue.next,
3060                                         struct net2280_request, queue);
3061                         done(ep, req, (req->req.actual == req->req.length)
3062                                                 ? 0 : -EPROTO);
3063                 }
3064                 ep->stopped = 0;
3065                 dev->protocol_stall = 0;
3066                 if (!(dev->quirks & PLX_SUPERSPEED)) {
3067                         if (ep->dev->quirks & PLX_2280)
3068                                 tmp = BIT(FIFO_OVERFLOW) |
3069                                     BIT(FIFO_UNDERFLOW);
3070                         else
3071                                 tmp = 0;
3072
3073                         writel(tmp | BIT(TIMEOUT) |
3074                                    BIT(USB_STALL_SENT) |
3075                                    BIT(USB_IN_NAK_SENT) |
3076                                    BIT(USB_IN_ACK_RCVD) |
3077                                    BIT(USB_OUT_PING_NAK_SENT) |
3078                                    BIT(USB_OUT_ACK_SENT) |
3079                                    BIT(SHORT_PACKET_OUT_DONE_INTERRUPT) |
3080                                    BIT(SHORT_PACKET_TRANSFERRED_INTERRUPT) |
3081                                    BIT(DATA_PACKET_RECEIVED_INTERRUPT) |
3082                                    BIT(DATA_PACKET_TRANSMITTED_INTERRUPT) |
3083                                    BIT(DATA_OUT_PING_TOKEN_INTERRUPT) |
3084                                    BIT(DATA_IN_TOKEN_INTERRUPT),
3085                                    &ep->regs->ep_stat);
3086                 }
3087                 u.raw[0] = readl(&dev->usb->setup0123);
3088                 u.raw[1] = readl(&dev->usb->setup4567);
3089
3090                 cpu_to_le32s(&u.raw[0]);
3091                 cpu_to_le32s(&u.raw[1]);
3092
3093                 if ((dev->quirks & PLX_SUPERSPEED) && !dev->bug7734_patched)
3094                         defect7374_workaround(dev, u.r);
3095
3096                 tmp = 0;
3097
3098 #define w_value         le16_to_cpu(u.r.wValue)
3099 #define w_index         le16_to_cpu(u.r.wIndex)
3100 #define w_length        le16_to_cpu(u.r.wLength)
3101
3102                 /* ack the irq */
3103                 writel(BIT(SETUP_PACKET_INTERRUPT), &dev->regs->irqstat0);
3104                 stat ^= BIT(SETUP_PACKET_INTERRUPT);
3105
3106                 /* watch control traffic at the token level, and force
3107                  * synchronization before letting the status stage happen.
3108                  * FIXME ignore tokens we'll NAK, until driver responds.
3109                  * that'll mean a lot less irqs for some drivers.
3110                  */
3111                 ep->is_in = (u.r.bRequestType & USB_DIR_IN) != 0;
3112                 if (ep->is_in) {
3113                         scratch = BIT(DATA_PACKET_TRANSMITTED_INTERRUPT) |
3114                                 BIT(DATA_OUT_PING_TOKEN_INTERRUPT) |
3115                                 BIT(DATA_IN_TOKEN_INTERRUPT);
3116                         stop_out_naking(ep);
3117                 } else
3118                         scratch = BIT(DATA_PACKET_RECEIVED_INTERRUPT) |
3119                                 BIT(DATA_OUT_PING_TOKEN_INTERRUPT) |
3120                                 BIT(DATA_IN_TOKEN_INTERRUPT);
3121                 writel(scratch, &dev->epregs[0].ep_irqenb);
3122
3123                 /* we made the hardware handle most lowlevel requests;
3124                  * everything else goes uplevel to the gadget code.
3125                  */
3126                 ep->responded = 1;
3127
3128                 if (dev->gadget.speed == USB_SPEED_SUPER) {
3129                         handle_stat0_irqs_superspeed(dev, ep, u.r);
3130                         goto next_endpoints;
3131                 }
3132
3133                 switch (u.r.bRequest) {
3134                 case USB_REQ_GET_STATUS: {
3135                         struct net2280_ep       *e;
3136                         __le32                  status;
3137
3138                         /* hw handles device and interface status */
3139                         if (u.r.bRequestType != (USB_DIR_IN|USB_RECIP_ENDPOINT))
3140                                 goto delegate;
3141                         e = get_ep_by_addr(dev, w_index);
3142                         if (!e || w_length > 2)
3143                                 goto do_stall;
3144
3145                         if (readl(&e->regs->ep_rsp) & BIT(SET_ENDPOINT_HALT))
3146                                 status = cpu_to_le32(1);
3147                         else
3148                                 status = cpu_to_le32(0);
3149
3150                         /* don't bother with a request object! */
3151                         writel(0, &dev->epregs[0].ep_irqenb);
3152                         set_fifo_bytecount(ep, w_length);
3153                         writel((__force u32)status, &dev->epregs[0].ep_data);
3154                         allow_status(ep);
3155                         ep_vdbg(dev, "%s stat %02x\n", ep->ep.name, status);
3156                         goto next_endpoints;
3157                         }
3158                         break;
3159                 case USB_REQ_CLEAR_FEATURE: {
3160                         struct net2280_ep       *e;
3161
3162                         /* hw handles device features */
3163                         if (u.r.bRequestType != USB_RECIP_ENDPOINT)
3164                                 goto delegate;
3165                         if (w_value != USB_ENDPOINT_HALT || w_length != 0)
3166                                 goto do_stall;
3167                         e = get_ep_by_addr(dev, w_index);
3168                         if (!e)
3169                                 goto do_stall;
3170                         if (e->wedged) {
3171                                 ep_vdbg(dev, "%s wedged, halt not cleared\n",
3172                                                 ep->ep.name);
3173                         } else {
3174                                 ep_vdbg(dev, "%s clear halt\n", e->ep.name);
3175                                 clear_halt(e);
3176                                 if ((ep->dev->quirks & PLX_SUPERSPEED) &&
3177                                         !list_empty(&e->queue) && e->td_dma)
3178                                                 restart_dma(e);
3179                         }
3180                         allow_status(ep);
3181                         goto next_endpoints;
3182                         }
3183                         break;
3184                 case USB_REQ_SET_FEATURE: {
3185                         struct net2280_ep       *e;
3186
3187                         /* hw handles device features */
3188                         if (u.r.bRequestType != USB_RECIP_ENDPOINT)
3189                                 goto delegate;
3190                         if (w_value != USB_ENDPOINT_HALT || w_length != 0)
3191                                 goto do_stall;
3192                         e = get_ep_by_addr(dev, w_index);
3193                         if (!e)
3194                                 goto do_stall;
3195                         if (e->ep.name == ep0name)
3196                                 goto do_stall;
3197                         set_halt(e);
3198                         if ((dev->quirks & PLX_SUPERSPEED) && e->dma)
3199                                 abort_dma(e);
3200                         allow_status(ep);
3201                         ep_vdbg(dev, "%s set halt\n", ep->ep.name);
3202                         goto next_endpoints;
3203                         }
3204                         break;
3205                 default:
3206 delegate:
3207                         ep_vdbg(dev, "setup %02x.%02x v%04x i%04x l%04x "
3208                                 "ep_cfg %08x\n",
3209                                 u.r.bRequestType, u.r.bRequest,
3210                                 w_value, w_index, w_length,
3211                                 readl(&ep->cfg->ep_cfg));
3212                         ep->responded = 0;
3213                         spin_unlock(&dev->lock);
3214                         tmp = dev->driver->setup(&dev->gadget, &u.r);
3215                         spin_lock(&dev->lock);
3216                 }
3217
3218                 /* stall ep0 on error */
3219                 if (tmp < 0) {
3220 do_stall:
3221                         ep_vdbg(dev, "req %02x.%02x protocol STALL; stat %d\n",
3222                                         u.r.bRequestType, u.r.bRequest, tmp);
3223                         dev->protocol_stall = 1;
3224                 }
3225
3226                 /* some in/out token irq should follow; maybe stall then.
3227                  * driver must queue a request (even zlp) or halt ep0
3228                  * before the host times out.
3229                  */
3230         }
3231
3232 #undef  w_value
3233 #undef  w_index
3234 #undef  w_length
3235
3236 next_endpoints:
3237         if ((dev->quirks & PLX_SUPERSPEED) && dev->enhanced_mode) {
3238                 u32 mask = (BIT(ENDPOINT_0_INTERRUPT) |
3239                         USB3380_IRQSTAT0_EP_INTR_MASK_IN |
3240                         USB3380_IRQSTAT0_EP_INTR_MASK_OUT);
3241
3242                 if (stat & mask) {
3243                         usb338x_handle_ep_intr(dev, stat & mask);
3244                         stat &= ~mask;
3245                 }
3246         } else {
3247                 /* endpoint data irq ? */
3248                 scratch = stat & 0x7f;
3249                 stat &= ~0x7f;
3250                 for (num = 0; scratch; num++) {
3251                         u32             t;
3252
3253                         /* do this endpoint's FIFO and queue need tending? */
3254                         t = BIT(num);
3255                         if ((scratch & t) == 0)
3256                                 continue;
3257                         scratch ^= t;
3258
3259                         ep = &dev->ep[num];
3260                         handle_ep_small(ep);
3261                 }
3262         }
3263
3264         if (stat)
3265                 ep_dbg(dev, "unhandled irqstat0 %08x\n", stat);
3266 }
3267
3268 #define DMA_INTERRUPTS (BIT(DMA_D_INTERRUPT) | \
3269                 BIT(DMA_C_INTERRUPT) | \
3270                 BIT(DMA_B_INTERRUPT) | \
3271                 BIT(DMA_A_INTERRUPT))
3272 #define PCI_ERROR_INTERRUPTS ( \
3273                 BIT(PCI_MASTER_ABORT_RECEIVED_INTERRUPT) | \
3274                 BIT(PCI_TARGET_ABORT_RECEIVED_INTERRUPT) | \
3275                 BIT(PCI_RETRY_ABORT_INTERRUPT))
3276
3277 static void handle_stat1_irqs(struct net2280 *dev, u32 stat)
3278 __releases(dev->lock)
3279 __acquires(dev->lock)
3280 {
3281         struct net2280_ep       *ep;
3282         u32                     tmp, num, mask, scratch;
3283
3284         /* after disconnect there's nothing else to do! */
3285         tmp = BIT(VBUS_INTERRUPT) | BIT(ROOT_PORT_RESET_INTERRUPT);
3286         mask = BIT(SUPER_SPEED) | BIT(HIGH_SPEED) | BIT(FULL_SPEED);
3287
3288         /* VBUS disconnect is indicated by VBUS_PIN and VBUS_INTERRUPT set.
3289          * Root Port Reset is indicated by ROOT_PORT_RESET_INTERRUPT set and
3290          * both HIGH_SPEED and FULL_SPEED clear (as ROOT_PORT_RESET_INTERRUPT
3291          * only indicates a change in the reset state).
3292          */
3293         if (stat & tmp) {
3294                 bool    reset = false;
3295                 bool    disconnect = false;
3296
3297                 /*
3298                  * Ignore disconnects and resets if the speed hasn't been set.
3299                  * VBUS can bounce and there's always an initial reset.
3300                  */
3301                 writel(tmp, &dev->regs->irqstat1);
3302                 if (dev->gadget.speed != USB_SPEED_UNKNOWN) {
3303                         if ((stat & BIT(VBUS_INTERRUPT)) &&
3304                                         (readl(&dev->usb->usbctl) &
3305                                                 BIT(VBUS_PIN)) == 0) {
3306                                 disconnect = true;
3307                                 ep_dbg(dev, "disconnect %s\n",
3308                                                 dev->driver->driver.name);
3309                         } else if ((stat & BIT(ROOT_PORT_RESET_INTERRUPT)) &&
3310                                         (readl(&dev->usb->usbstat) & mask)
3311                                                 == 0) {
3312                                 reset = true;
3313                                 ep_dbg(dev, "reset %s\n",
3314                                                 dev->driver->driver.name);
3315                         }
3316
3317                         if (disconnect || reset) {
3318                                 stop_activity(dev, dev->driver);
3319                                 ep0_start(dev);
3320                                 spin_unlock(&dev->lock);
3321                                 if (reset)
3322                                         usb_gadget_udc_reset
3323                                                 (&dev->gadget, dev->driver);
3324                                 else
3325                                         (dev->driver->disconnect)
3326                                                 (&dev->gadget);
3327                                 spin_lock(&dev->lock);
3328                                 return;
3329                         }
3330                 }
3331                 stat &= ~tmp;
3332
3333                 /* vBUS can bounce ... one of many reasons to ignore the
3334                  * notion of hotplug events on bus connect/disconnect!
3335                  */
3336                 if (!stat)
3337                         return;
3338         }
3339
3340         /* NOTE: chip stays in PCI D0 state for now, but it could
3341          * enter D1 to save more power
3342          */
3343         tmp = BIT(SUSPEND_REQUEST_CHANGE_INTERRUPT);
3344         if (stat & tmp) {
3345                 writel(tmp, &dev->regs->irqstat1);
3346                 if (stat & BIT(SUSPEND_REQUEST_INTERRUPT)) {
3347                         if (dev->driver->suspend)
3348                                 dev->driver->suspend(&dev->gadget);
3349                         if (!enable_suspend)
3350                                 stat &= ~BIT(SUSPEND_REQUEST_INTERRUPT);
3351                 } else {
3352                         if (dev->driver->resume)
3353                                 dev->driver->resume(&dev->gadget);
3354                         /* at high speed, note erratum 0133 */
3355                 }
3356                 stat &= ~tmp;
3357         }
3358
3359         /* clear any other status/irqs */
3360         if (stat)
3361                 writel(stat, &dev->regs->irqstat1);
3362
3363         /* some status we can just ignore */
3364         if (dev->quirks & PLX_2280)
3365                 stat &= ~(BIT(CONTROL_STATUS_INTERRUPT) |
3366                           BIT(SUSPEND_REQUEST_INTERRUPT) |
3367                           BIT(RESUME_INTERRUPT) |
3368                           BIT(SOF_INTERRUPT));
3369         else
3370                 stat &= ~(BIT(CONTROL_STATUS_INTERRUPT) |
3371                           BIT(RESUME_INTERRUPT) |
3372                           BIT(SOF_DOWN_INTERRUPT) |
3373                           BIT(SOF_INTERRUPT));
3374
3375         if (!stat)
3376                 return;
3377         /* ep_dbg(dev, "irqstat1 %08x\n", stat);*/
3378
3379         /* DMA status, for ep-{a,b,c,d} */
3380         scratch = stat & DMA_INTERRUPTS;
3381         stat &= ~DMA_INTERRUPTS;
3382         scratch >>= 9;
3383         for (num = 0; scratch; num++) {
3384                 struct net2280_dma_regs __iomem *dma;
3385
3386                 tmp = BIT(num);
3387                 if ((tmp & scratch) == 0)
3388                         continue;
3389                 scratch ^= tmp;
3390
3391                 ep = &dev->ep[num + 1];
3392                 dma = ep->dma;
3393
3394                 if (!dma)
3395                         continue;
3396
3397                 /* clear ep's dma status */
3398                 tmp = readl(&dma->dmastat);
3399                 writel(tmp, &dma->dmastat);
3400
3401                 /* dma sync*/
3402                 if (dev->quirks & PLX_SUPERSPEED) {
3403                         u32 r_dmacount = readl(&dma->dmacount);
3404                         if (!ep->is_in &&  (r_dmacount & 0x00FFFFFF) &&
3405                             (tmp & BIT(DMA_TRANSACTION_DONE_INTERRUPT)))
3406                                 continue;
3407                 }
3408
3409                 if (!(tmp & BIT(DMA_TRANSACTION_DONE_INTERRUPT))) {
3410                         ep_dbg(ep->dev, "%s no xact done? %08x\n",
3411                                 ep->ep.name, tmp);
3412                         continue;
3413                 }
3414                 stop_dma(ep->dma);
3415
3416                 /* OUT transfers terminate when the data from the
3417                  * host is in our memory.  Process whatever's done.
3418                  * On this path, we know transfer's last packet wasn't
3419                  * less than req->length. NAK_OUT_PACKETS may be set,
3420                  * or the FIFO may already be holding new packets.
3421                  *
3422                  * IN transfers can linger in the FIFO for a very
3423                  * long time ... we ignore that for now, accounting
3424                  * precisely (like PIO does) needs per-packet irqs
3425                  */
3426                 scan_dma_completions(ep);
3427
3428                 /* disable dma on inactive queues; else maybe restart */
3429                 if (!list_empty(&ep->queue)) {
3430                         tmp = readl(&dma->dmactl);
3431                         restart_dma(ep);
3432                 }
3433                 ep->irqs++;
3434         }
3435
3436         /* NOTE:  there are other PCI errors we might usefully notice.
3437          * if they appear very often, here's where to try recovering.
3438          */
3439         if (stat & PCI_ERROR_INTERRUPTS) {
3440                 ep_err(dev, "pci dma error; stat %08x\n", stat);
3441                 stat &= ~PCI_ERROR_INTERRUPTS;
3442                 /* these are fatal errors, but "maybe" they won't
3443                  * happen again ...
3444                  */
3445                 stop_activity(dev, dev->driver);
3446                 ep0_start(dev);
3447                 stat = 0;
3448         }
3449
3450         if (stat)
3451                 ep_dbg(dev, "unhandled irqstat1 %08x\n", stat);
3452 }
3453
3454 static irqreturn_t net2280_irq(int irq, void *_dev)
3455 {
3456         struct net2280          *dev = _dev;
3457
3458         /* shared interrupt, not ours */
3459         if ((dev->quirks & PLX_LEGACY) &&
3460                 (!(readl(&dev->regs->irqstat0) & BIT(INTA_ASSERTED))))
3461                 return IRQ_NONE;
3462
3463         spin_lock(&dev->lock);
3464
3465         /* handle disconnect, dma, and more */
3466         handle_stat1_irqs(dev, readl(&dev->regs->irqstat1));
3467
3468         /* control requests and PIO */
3469         handle_stat0_irqs(dev, readl(&dev->regs->irqstat0));
3470
3471         if (dev->quirks & PLX_SUPERSPEED) {
3472                 /* re-enable interrupt to trigger any possible new interrupt */
3473                 u32 pciirqenb1 = readl(&dev->regs->pciirqenb1);
3474                 writel(pciirqenb1 & 0x7FFFFFFF, &dev->regs->pciirqenb1);
3475                 writel(pciirqenb1, &dev->regs->pciirqenb1);
3476         }
3477
3478         spin_unlock(&dev->lock);
3479
3480         return IRQ_HANDLED;
3481 }
3482
3483 /*-------------------------------------------------------------------------*/
3484
3485 static void gadget_release(struct device *_dev)
3486 {
3487         struct net2280  *dev = dev_get_drvdata(_dev);
3488
3489         kfree(dev);
3490 }
3491
3492 /* tear down the binding between this driver and the pci device */
3493
3494 static void net2280_remove(struct pci_dev *pdev)
3495 {
3496         struct net2280          *dev = pci_get_drvdata(pdev);
3497
3498         usb_del_gadget_udc(&dev->gadget);
3499
3500         BUG_ON(dev->driver);
3501
3502         /* then clean up the resources we allocated during probe() */
3503         net2280_led_shutdown(dev);
3504         if (dev->requests) {
3505                 int             i;
3506                 for (i = 1; i < 5; i++) {
3507                         if (!dev->ep[i].dummy)
3508                                 continue;
3509                         pci_pool_free(dev->requests, dev->ep[i].dummy,
3510                                         dev->ep[i].td_dma);
3511                 }
3512                 pci_pool_destroy(dev->requests);
3513         }
3514         if (dev->got_irq)
3515                 free_irq(pdev->irq, dev);
3516         if (dev->quirks & PLX_SUPERSPEED)
3517                 pci_disable_msi(pdev);
3518         if (dev->regs)
3519                 iounmap(dev->regs);
3520         if (dev->region)
3521                 release_mem_region(pci_resource_start(pdev, 0),
3522                                 pci_resource_len(pdev, 0));
3523         if (dev->enabled)
3524                 pci_disable_device(pdev);
3525         device_remove_file(&pdev->dev, &dev_attr_registers);
3526
3527         ep_info(dev, "unbind\n");
3528 }
3529
3530 /* wrap this driver around the specified device, but
3531  * don't respond over USB until a gadget driver binds to us.
3532  */
3533
3534 static int net2280_probe(struct pci_dev *pdev, const struct pci_device_id *id)
3535 {
3536         struct net2280          *dev;
3537         unsigned long           resource, len;
3538         void                    __iomem *base = NULL;
3539         int                     retval, i;
3540
3541         /* alloc, and start init */
3542         dev = kzalloc(sizeof(*dev), GFP_KERNEL);
3543         if (dev == NULL) {
3544                 retval = -ENOMEM;
3545                 goto done;
3546         }
3547
3548         pci_set_drvdata(pdev, dev);
3549         spin_lock_init(&dev->lock);
3550         dev->quirks = id->driver_data;
3551         dev->pdev = pdev;
3552         dev->gadget.ops = &net2280_ops;
3553         dev->gadget.max_speed = (dev->quirks & PLX_SUPERSPEED) ?
3554                                 USB_SPEED_SUPER : USB_SPEED_HIGH;
3555
3556         /* the "gadget" abstracts/virtualizes the controller */
3557         dev->gadget.name = driver_name;
3558
3559         /* now all the pci goodies ... */
3560         if (pci_enable_device(pdev) < 0) {
3561                 retval = -ENODEV;
3562                 goto done;
3563         }
3564         dev->enabled = 1;
3565
3566         /* BAR 0 holds all the registers
3567          * BAR 1 is 8051 memory; unused here (note erratum 0103)
3568          * BAR 2 is fifo memory; unused here
3569          */
3570         resource = pci_resource_start(pdev, 0);
3571         len = pci_resource_len(pdev, 0);
3572         if (!request_mem_region(resource, len, driver_name)) {
3573                 ep_dbg(dev, "controller already in use\n");
3574                 retval = -EBUSY;
3575                 goto done;
3576         }
3577         dev->region = 1;
3578
3579         /* FIXME provide firmware download interface to put
3580          * 8051 code into the chip, e.g. to turn on PCI PM.
3581          */
3582
3583         base = ioremap_nocache(resource, len);
3584         if (base == NULL) {
3585                 ep_dbg(dev, "can't map memory\n");
3586                 retval = -EFAULT;
3587                 goto done;
3588         }
3589         dev->regs = (struct net2280_regs __iomem *) base;
3590         dev->usb = (struct net2280_usb_regs __iomem *) (base + 0x0080);
3591         dev->pci = (struct net2280_pci_regs __iomem *) (base + 0x0100);
3592         dev->dma = (struct net2280_dma_regs __iomem *) (base + 0x0180);
3593         dev->dep = (struct net2280_dep_regs __iomem *) (base + 0x0200);
3594         dev->epregs = (struct net2280_ep_regs __iomem *) (base + 0x0300);
3595
3596         if (dev->quirks & PLX_SUPERSPEED) {
3597                 u32 fsmvalue;
3598                 u32 usbstat;
3599                 dev->usb_ext = (struct usb338x_usb_ext_regs __iomem *)
3600                                                         (base + 0x00b4);
3601                 dev->llregs = (struct usb338x_ll_regs __iomem *)
3602                                                         (base + 0x0700);
3603                 dev->ll_lfps_regs = (struct usb338x_ll_lfps_regs __iomem *)
3604                                                         (base + 0x0748);
3605                 dev->ll_tsn_regs = (struct usb338x_ll_tsn_regs __iomem *)
3606                                                         (base + 0x077c);
3607                 dev->ll_chicken_reg = (struct usb338x_ll_chi_regs __iomem *)
3608                                                         (base + 0x079c);
3609                 dev->plregs = (struct usb338x_pl_regs __iomem *)
3610                                                         (base + 0x0800);
3611                 usbstat = readl(&dev->usb->usbstat);
3612                 dev->enhanced_mode = !!(usbstat & BIT(11));
3613                 dev->n_ep = (dev->enhanced_mode) ? 9 : 5;
3614                 /* put into initial config, link up all endpoints */
3615                 fsmvalue = get_idx_reg(dev->regs, SCRATCH) &
3616                                         (0xf << DEFECT7374_FSM_FIELD);
3617                 /* See if firmware needs to set up for workaround: */
3618                 if (fsmvalue == DEFECT7374_FSM_SS_CONTROL_READ) {
3619                         dev->bug7734_patched = 1;
3620                         writel(0, &dev->usb->usbctl);
3621                 } else
3622                         dev->bug7734_patched = 0;
3623         } else {
3624                 dev->enhanced_mode = 0;
3625                 dev->n_ep = 7;
3626                 /* put into initial config, link up all endpoints */
3627                 writel(0, &dev->usb->usbctl);
3628         }
3629
3630         usb_reset(dev);
3631         usb_reinit(dev);
3632
3633         /* irq setup after old hardware is cleaned up */
3634         if (!pdev->irq) {
3635                 ep_err(dev, "No IRQ.  Check PCI setup!\n");
3636                 retval = -ENODEV;
3637                 goto done;
3638         }
3639
3640         if (dev->quirks & PLX_SUPERSPEED)
3641                 if (pci_enable_msi(pdev))
3642                         ep_err(dev, "Failed to enable MSI mode\n");
3643
3644         if (request_irq(pdev->irq, net2280_irq, IRQF_SHARED,
3645                                                         driver_name, dev)) {
3646                 ep_err(dev, "request interrupt %d failed\n", pdev->irq);
3647                 retval = -EBUSY;
3648                 goto done;
3649         }
3650         dev->got_irq = 1;
3651
3652         /* DMA setup */
3653         /* NOTE:  we know only the 32 LSBs of dma addresses may be nonzero */
3654         dev->requests = pci_pool_create("requests", pdev,
3655                 sizeof(struct net2280_dma),
3656                 0 /* no alignment requirements */,
3657                 0 /* or page-crossing issues */);
3658         if (!dev->requests) {
3659                 ep_dbg(dev, "can't get request pool\n");
3660                 retval = -ENOMEM;
3661                 goto done;
3662         }
3663         for (i = 1; i < 5; i++) {
3664                 struct net2280_dma      *td;
3665
3666                 td = pci_pool_alloc(dev->requests, GFP_KERNEL,
3667                                 &dev->ep[i].td_dma);
3668                 if (!td) {
3669                         ep_dbg(dev, "can't get dummy %d\n", i);
3670                         retval = -ENOMEM;
3671                         goto done;
3672                 }
3673                 td->dmacount = 0;       /* not VALID */
3674                 td->dmadesc = td->dmaaddr;
3675                 dev->ep[i].dummy = td;
3676         }
3677
3678         /* enable lower-overhead pci memory bursts during DMA */
3679         if (dev->quirks & PLX_LEGACY)
3680                 writel(BIT(DMA_MEMORY_WRITE_AND_INVALIDATE_ENABLE) |
3681                         /*
3682                          * 256 write retries may not be enough...
3683                            BIT(PCI_RETRY_ABORT_ENABLE) |
3684                         */
3685                         BIT(DMA_READ_MULTIPLE_ENABLE) |
3686                         BIT(DMA_READ_LINE_ENABLE),
3687                         &dev->pci->pcimstctl);
3688         /* erratum 0115 shouldn't appear: Linux inits PCI_LATENCY_TIMER */
3689         pci_set_master(pdev);
3690         pci_try_set_mwi(pdev);
3691
3692         /* ... also flushes any posted pci writes */
3693         dev->chiprev = get_idx_reg(dev->regs, REG_CHIPREV) & 0xffff;
3694
3695         /* done */
3696         ep_info(dev, "%s\n", driver_desc);
3697         ep_info(dev, "irq %d, pci mem %p, chip rev %04x\n",
3698                         pdev->irq, base, dev->chiprev);
3699         ep_info(dev, "version: " DRIVER_VERSION "; %s\n",
3700                 dev->enhanced_mode ? "enhanced mode" : "legacy mode");
3701         retval = device_create_file(&pdev->dev, &dev_attr_registers);
3702         if (retval)
3703                 goto done;
3704
3705         retval = usb_add_gadget_udc_release(&pdev->dev, &dev->gadget,
3706                         gadget_release);
3707         if (retval)
3708                 goto done;
3709         return 0;
3710
3711 done:
3712         if (dev)
3713                 net2280_remove(pdev);
3714         return retval;
3715 }
3716
3717 /* make sure the board is quiescent; otherwise it will continue
3718  * generating IRQs across the upcoming reboot.
3719  */
3720
3721 static void net2280_shutdown(struct pci_dev *pdev)
3722 {
3723         struct net2280          *dev = pci_get_drvdata(pdev);
3724
3725         /* disable IRQs */
3726         writel(0, &dev->regs->pciirqenb0);
3727         writel(0, &dev->regs->pciirqenb1);
3728
3729         /* disable the pullup so the host will think we're gone */
3730         writel(0, &dev->usb->usbctl);
3731
3732 }
3733
3734
3735 /*-------------------------------------------------------------------------*/
3736
3737 static const struct pci_device_id pci_ids[] = { {
3738         .class =        ((PCI_CLASS_SERIAL_USB << 8) | 0xfe),
3739         .class_mask =   ~0,
3740         .vendor =       PCI_VENDOR_ID_PLX_LEGACY,
3741         .device =       0x2280,
3742         .subvendor =    PCI_ANY_ID,
3743         .subdevice =    PCI_ANY_ID,
3744         .driver_data =  PLX_LEGACY | PLX_2280,
3745         }, {
3746         .class =        ((PCI_CLASS_SERIAL_USB << 8) | 0xfe),
3747         .class_mask =   ~0,
3748         .vendor =       PCI_VENDOR_ID_PLX_LEGACY,
3749         .device =       0x2282,
3750         .subvendor =    PCI_ANY_ID,
3751         .subdevice =    PCI_ANY_ID,
3752         .driver_data =  PLX_LEGACY,
3753         },
3754         {
3755         .class =        ((PCI_CLASS_SERIAL_USB << 8) | 0xfe),
3756         .class_mask =   ~0,
3757         .vendor =       PCI_VENDOR_ID_PLX,
3758         .device =       0x3380,
3759         .subvendor =    PCI_ANY_ID,
3760         .subdevice =    PCI_ANY_ID,
3761         .driver_data =  PLX_SUPERSPEED,
3762          },
3763         {
3764         .class =        ((PCI_CLASS_SERIAL_USB << 8) | 0xfe),
3765         .class_mask =   ~0,
3766         .vendor =       PCI_VENDOR_ID_PLX,
3767         .device =       0x3382,
3768         .subvendor =    PCI_ANY_ID,
3769         .subdevice =    PCI_ANY_ID,
3770         .driver_data =  PLX_SUPERSPEED,
3771          },
3772 { /* end: all zeroes */ }
3773 };
3774 MODULE_DEVICE_TABLE(pci, pci_ids);
3775
3776 /* pci driver glue; this is a "new style" PCI driver module */
3777 static struct pci_driver net2280_pci_driver = {
3778         .name =         (char *) driver_name,
3779         .id_table =     pci_ids,
3780
3781         .probe =        net2280_probe,
3782         .remove =       net2280_remove,
3783         .shutdown =     net2280_shutdown,
3784
3785         /* FIXME add power management support */
3786 };
3787
3788 module_pci_driver(net2280_pci_driver);
3789
3790 MODULE_DESCRIPTION(DRIVER_DESC);
3791 MODULE_AUTHOR("David Brownell");
3792 MODULE_LICENSE("GPL");