drivers/staging/dwc2/hcd_intr.c

   1 /*
   2  * hcd_intr.c - DesignWare HS OTG Controller host-mode interrupt handling
   3  *
   4  * Copyright (C) 2004-2013 Synopsys, Inc.
   5  *
   6  * Redistribution and use in source and binary forms, with or without
   7  * modification, are permitted provided that the following conditions
   8  * are met:
   9  * 1. Redistributions of source code must retain the above copyright
  10  *    notice, this list of conditions, and the following disclaimer,
  11  *    without modification.
  12  * 2. Redistributions in binary form must reproduce the above copyright
  13  *    notice, this list of conditions and the following disclaimer in the
  14  *    documentation and/or other materials provided with the distribution.
  15  * 3. The names of the above-listed copyright holders may not be used
  16  *    to endorse or promote products derived from this software without
  17  *    specific prior written permission.
  18  *
  19  * ALTERNATIVELY, this software may be distributed under the terms of the
  20  * GNU General Public License ("GPL") as published by the Free Software
  21  * Foundation; either version 2 of the License, or (at your option) any
  22  * later version.
  23  *
  24  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
  25  * IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
  26  * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  27  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
  28  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
  29  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
  30  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
  31  * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
  32  * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
  33  * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
  34  * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  35  */
  36
  37 /*
  38  * This file contains the interrupt handlers for Host mode
  39  */
  40 #include <linux/kernel.h>
  41 #include <linux/module.h>
  42 #include <linux/spinlock.h>
  43 #include <linux/interrupt.h>
  44 #include <linux/dma-mapping.h>
  45 #include <linux/io.h>
  46 #include <linux/slab.h>
  47 #include <linux/usb.h>
  48
  49 #include <linux/usb/hcd.h>
  50 #include <linux/usb/ch11.h>
  51
  52 #include "core.h"
  53 #include "hcd.h"
  54
  55 /* This function is for debug only */
  56 static void dwc2_track_missed_sofs(struct dwc2_hsotg *hsotg)
  57 {
  58 #ifdef CONFIG_USB_DWC2_TRACK_MISSED_SOFS
  59 #warning Compiling code to track missed SOFs
  60
  61         u16 curr_frame_number = hsotg->frame_number;
  62
  63         if (hsotg->frame_num_idx < FRAME_NUM_ARRAY_SIZE) {
  64                 if (((hsotg->last_frame_num + 1) & HFNUM_MAX_FRNUM) !=
  65                     curr_frame_number) {
  66                         hsotg->frame_num_array[hsotg->frame_num_idx] =
  67                                         curr_frame_number;
  68                         hsotg->last_frame_num_array[hsotg->frame_num_idx] =
  69                                         hsotg->last_frame_num;
  70                         hsotg->frame_num_idx++;
  71                 }
  72         } else if (!hsotg->dumped_frame_num_array) {
  73                 int i;
  74
  75                 dev_info(hsotg->dev, "Frame     Last Frame\n");
  76                 dev_info(hsotg->dev, "-----     ----------\n");
  77                 for (i = 0; i < FRAME_NUM_ARRAY_SIZE; i++) {
  78                         dev_info(hsotg->dev, "0x%04x    0x%04x\n",
  79                                  hsotg->frame_num_array[i],
  80                                  hsotg->last_frame_num_array[i]);
  81                 }
  82                 hsotg->dumped_frame_num_array = 1;
  83         }
  84         hsotg->last_frame_num = curr_frame_number;
  85 #endif
  86 }
  87
  88 static void dwc2_hc_handle_tt_clear(struct dwc2_hsotg *hsotg,
  89                                     struct dwc2_host_chan *chan,
  90                                     struct dwc2_qtd *qtd)
  91 {
  92         struct urb *usb_urb;
  93
  94         if (!chan->qh || !qtd->urb)
  95                 return;
  96
  97         usb_urb = qtd->urb->priv;
  98         if (!usb_urb || !usb_urb->dev)
  99                 return;
 100
 101         if (chan->qh->dev_speed != USB_SPEED_HIGH &&
 102             qtd->urb->status != -EPIPE && qtd->urb->status != -EREMOTEIO) {
 103                 chan->qh->tt_buffer_dirty = 1;
 104                 if (usb_hub_clear_tt_buffer(usb_urb))
 105                         /* Clear failed; let's hope things work anyway */
 106                         chan->qh->tt_buffer_dirty = 0;
 107         }
 108 }
 109
 110 /*
 111  * Handles the start-of-frame interrupt in host mode. Non-periodic
 112  * transactions may be queued to the DWC_otg controller for the current
 113  * (micro)frame. Periodic transactions may be queued to the controller
 114  * for the next (micro)frame.
 115  */
 116 static void dwc2_sof_intr(struct dwc2_hsotg *hsotg)
 117 {
 118         struct list_head *qh_entry;
 119         struct dwc2_qh *qh;
 120         u32 hfnum;
 121         enum dwc2_transaction_type tr_type;
 122
 123 #ifdef DEBUG_SOF
 124         dev_vdbg(hsotg->dev, "--Start of Frame Interrupt--\n");
 125 #endif
 126
 127         hfnum = readl(hsotg->regs + HFNUM);
 128         hsotg->frame_number = hfnum >> HFNUM_FRNUM_SHIFT &
 129                             HFNUM_FRNUM_MASK >> HFNUM_FRNUM_SHIFT;
 130
 131         dwc2_track_missed_sofs(hsotg);
 132
 133         /* Determine whether any periodic QHs should be executed */
 134         qh_entry = hsotg->periodic_sched_inactive.next;
 135         while (qh_entry != &hsotg->periodic_sched_inactive) {
 136                 qh = list_entry(qh_entry, struct dwc2_qh, qh_list_entry);
 137                 qh_entry = qh_entry->next;
 138                 if (dwc2_frame_num_le(qh->sched_frame, hsotg->frame_number))
 139                         /*
 140                          * Move QH to the ready list to be executed next
 141                          * (micro)frame
 142                          */
 143                         list_move(&qh->qh_list_entry,
 144                                   &hsotg->periodic_sched_ready);
 145         }
 146         tr_type = dwc2_hcd_select_transactions(hsotg);
 147         if (tr_type != DWC2_TRANSACTION_NONE)
 148                 dwc2_hcd_queue_transactions(hsotg, tr_type);
 149
 150         /* Clear interrupt */
 151         writel(GINTSTS_SOF, hsotg->regs + GINTSTS);
 152 }
 153
 154 /*
 155  * Handles the Rx FIFO Level Interrupt, which indicates that there is
 156  * at least one packet in the Rx FIFO. The packets are moved from the FIFO to
 157  * memory if the DWC_otg controller is operating in Slave mode.
 158  */
 159 static void dwc2_rx_fifo_level_intr(struct dwc2_hsotg *hsotg)
 160 {
 161         u32 grxsts, chnum, bcnt, dpid, pktsts;
 162         struct dwc2_host_chan *chan;
 163
 164         dev_vdbg(hsotg->dev, "--RxFIFO Level Interrupt--\n");
 165
 166         grxsts = readl(hsotg->regs + GRXSTSP);
 167         chnum = grxsts >> GRXSTS_HCHNUM_SHIFT &
 168                 GRXSTS_HCHNUM_MASK >> GRXSTS_HCHNUM_SHIFT;
 169         chan = hsotg->hc_ptr_array[chnum];
 170         if (!chan) {
 171                 dev_err(hsotg->dev, "Unable to get corresponding channel\n");
 172                 return;
 173         }
 174
 175         bcnt = grxsts >> GRXSTS_BYTECNT_SHIFT &
 176                GRXSTS_BYTECNT_MASK >> GRXSTS_BYTECNT_SHIFT;
 177         dpid = grxsts >> GRXSTS_DPID_SHIFT &
 178                GRXSTS_DPID_MASK >> GRXSTS_DPID_SHIFT;
 179         pktsts = grxsts & GRXSTS_PKTSTS_MASK;
 180
 181         /* Packet Status */
 182         dev_vdbg(hsotg->dev, "    Ch num = %d\n", chnum);
 183         dev_vdbg(hsotg->dev, "    Count = %d\n", bcnt);
 184         dev_vdbg(hsotg->dev, "    DPID = %d, chan.dpid = %d\n", dpid,
 185                  chan->data_pid_start);
 186         dev_vdbg(hsotg->dev, "    PStatus = %d\n",
 187                  pktsts >> GRXSTS_PKTSTS_SHIFT &
 188                  GRXSTS_PKTSTS_MASK >> GRXSTS_PKTSTS_SHIFT);
 189
 190         switch (pktsts) {
 191         case GRXSTS_PKTSTS_HCHIN:
 192                 /* Read the data into the host buffer */
 193                 if (bcnt > 0) {
 194                         dwc2_read_packet(hsotg, chan->xfer_buf, bcnt);
 195
 196                         /* Update the HC fields for the next packet received */
 197                         chan->xfer_count += bcnt;
 198                         chan->xfer_buf += bcnt;
 199                 }
 200                 break;
 201         case GRXSTS_PKTSTS_HCHIN_XFER_COMP:
 202         case GRXSTS_PKTSTS_DATATOGGLEERR:
 203         case GRXSTS_PKTSTS_HCHHALTED:
 204                 /* Handled in interrupt, just ignore data */
 205                 break;
 206         default:
 207                 dev_err(hsotg->dev,
 208                         "RxFIFO Level Interrupt: Unknown status %d\n", pktsts);
 209                 break;
 210         }
 211 }
 212
 213 /*
 214  * This interrupt occurs when the non-periodic Tx FIFO is half-empty. More
 215  * data packets may be written to the FIFO for OUT transfers. More requests
 216  * may be written to the non-periodic request queue for IN transfers. This
 217  * interrupt is enabled only in Slave mode.
 218  */
 219 static void dwc2_np_tx_fifo_empty_intr(struct dwc2_hsotg *hsotg)
 220 {
 221         dev_vdbg(hsotg->dev, "--Non-Periodic TxFIFO Empty Interrupt--\n");
 222         dwc2_hcd_queue_transactions(hsotg, DWC2_TRANSACTION_NON_PERIODIC);
 223 }
 224
 225 /*
 226  * This interrupt occurs when the periodic Tx FIFO is half-empty. More data
 227  * packets may be written to the FIFO for OUT transfers. More requests may be
 228  * written to the periodic request queue for IN transfers. This interrupt is
 229  * enabled only in Slave mode.
 230  */
 231 static void dwc2_perio_tx_fifo_empty_intr(struct dwc2_hsotg *hsotg)
 232 {
 233         dev_vdbg(hsotg->dev, "--Periodic TxFIFO Empty Interrupt--\n");
 234         dwc2_hcd_queue_transactions(hsotg, DWC2_TRANSACTION_PERIODIC);
 235 }
 236
 237 static void dwc2_hprt0_enable(struct dwc2_hsotg *hsotg, u32 hprt0,
 238                               u32 *hprt0_modify)
 239 {
 240         struct dwc2_core_params *params = hsotg->core_params;
 241         int do_reset = 0;
 242         u32 usbcfg;
 243         u32 prtspd;
 244         u32 hcfg;
 245         u32 hfir;
 246
 247         dev_vdbg(hsotg->dev, "%s(%p)\n", __func__, hsotg);
 248
 249         /* Every time when port enables calculate HFIR.FrInterval */
 250         hfir = readl(hsotg->regs + HFIR);
 251         hfir &= ~HFIR_FRINT_MASK;
 252         hfir |= dwc2_calc_frame_interval(hsotg) << HFIR_FRINT_SHIFT &
 253                 HFIR_FRINT_MASK;
 254         writel(hfir, hsotg->regs + HFIR);
 255
 256         /* Check if we need to adjust the PHY clock speed for low power */
 257         if (!params->host_support_fs_ls_low_power) {
 258                 /* Port has been enabled, set the reset change flag */
 259                 hsotg->flags.b.port_reset_change = 1;
 260                 return;
 261         }
 262
 263         usbcfg = readl(hsotg->regs + GUSBCFG);
 264         prtspd = hprt0 & HPRT0_SPD_MASK;
 265
 266         if (prtspd == HPRT0_SPD_LOW_SPEED || prtspd == HPRT0_SPD_FULL_SPEED) {
 267                 /* Low power */
 268                 if (!(usbcfg & GUSBCFG_PHY_LP_CLK_SEL)) {
 269                         /* Set PHY low power clock select for FS/LS devices */
 270                         usbcfg |= GUSBCFG_PHY_LP_CLK_SEL;
 271                         writel(usbcfg, hsotg->regs + GUSBCFG);
 272                         do_reset = 1;
 273                 }
 274
 275                 hcfg = readl(hsotg->regs + HCFG);
 276
 277                 if (prtspd == HPRT0_SPD_LOW_SPEED &&
 278                     params->host_ls_low_power_phy_clk ==
 279                     DWC2_HOST_LS_LOW_POWER_PHY_CLK_PARAM_6MHZ) {
 280                         /* 6 MHZ */
 281                         dev_vdbg(hsotg->dev,
 282                                  "FS_PHY programming HCFG to 6 MHz\n");
 283                         if ((hcfg & HCFG_FSLSPCLKSEL_MASK) !=
 284                             HCFG_FSLSPCLKSEL_6_MHZ) {
 285                                 hcfg &= ~HCFG_FSLSPCLKSEL_MASK;
 286                                 hcfg |= HCFG_FSLSPCLKSEL_6_MHZ;
 287                                 writel(hcfg, hsotg->regs + HCFG);
 288                                 do_reset = 1;
 289                         }
 290                 } else {
 291                         /* 48 MHZ */
 292                         dev_vdbg(hsotg->dev,
 293                                  "FS_PHY programming HCFG to 48 MHz\n");
 294                         if ((hcfg & HCFG_FSLSPCLKSEL_MASK) !=
 295                             HCFG_FSLSPCLKSEL_48_MHZ) {
 296                                 hcfg &= ~HCFG_FSLSPCLKSEL_MASK;
 297                                 hcfg |= HCFG_FSLSPCLKSEL_48_MHZ;
 298                                 writel(hcfg, hsotg->regs + HCFG);
 299                                 do_reset = 1;
 300                         }
 301                 }
 302         } else {
 303                 /* Not low power */
 304                 if (usbcfg & GUSBCFG_PHY_LP_CLK_SEL) {
 305                         usbcfg &= ~GUSBCFG_PHY_LP_CLK_SEL;
 306                         writel(usbcfg, hsotg->regs + GUSBCFG);
 307                         do_reset = 1;
 308                 }
 309         }
 310
 311         if (do_reset) {
 312                 *hprt0_modify |= HPRT0_RST;
 313                 queue_delayed_work(hsotg->wq_otg, &hsotg->reset_work,
 314                                    msecs_to_jiffies(60));
 315         } else {
 316                 /* Port has been enabled, set the reset change flag */
 317                 hsotg->flags.b.port_reset_change = 1;
 318         }
 319 }
 320
 321 /*
 322  * There are multiple conditions that can cause a port interrupt. This function
 323  * determines which interrupt conditions have occurred and handles them
 324  * appropriately.
 325  */
 326 static void dwc2_port_intr(struct dwc2_hsotg *hsotg)
 327 {
 328         u32 hprt0;
 329         u32 hprt0_modify;
 330
 331         dev_vdbg(hsotg->dev, "--Port Interrupt--\n");
 332
 333         hprt0 = readl(hsotg->regs + HPRT0);
 334         hprt0_modify = hprt0;
 335
 336         /*
 337          * Clear appropriate bits in HPRT0 to clear the interrupt bit in
 338          * GINTSTS
 339          */
 340         hprt0_modify &= ~(HPRT0_ENA | HPRT0_CONNDET | HPRT0_ENACHG |
 341                           HPRT0_OVRCURRCHG);
 342
 343         /*
 344          * Port Connect Detected
 345          * Set flag and clear if detected
 346          */
 347         if (hprt0 & HPRT0_CONNDET) {
 348                 dev_vdbg(hsotg->dev,
 349                          "--Port Interrupt HPRT0=0x%08x Port Connect Detected--\n",
 350                          hprt0);
 351                 hsotg->flags.b.port_connect_status_change = 1;
 352                 hsotg->flags.b.port_connect_status = 1;
 353                 hprt0_modify |= HPRT0_CONNDET;
 354
 355                 /*
 356                  * The Hub driver asserts a reset when it sees port connect
 357                  * status change flag
 358                  */
 359         }
 360
 361         /*
 362          * Port Enable Changed
 363          * Clear if detected - Set internal flag if disabled
 364          */
 365         if (hprt0 & HPRT0_ENACHG) {
 366                 dev_vdbg(hsotg->dev,
 367                          "  --Port Interrupt HPRT0=0x%08x Port Enable Changed (now %d)--\n",
 368                          hprt0, !!(hprt0 & HPRT0_ENA));
 369                 hprt0_modify |= HPRT0_ENACHG;
 370                 if (hprt0 & HPRT0_ENA)
 371                         dwc2_hprt0_enable(hsotg, hprt0, &hprt0_modify);
 372                 else
 373                         hsotg->flags.b.port_enable_change = 1;
 374         }
 375
 376         /* Overcurrent Change Interrupt */
 377         if (hprt0 & HPRT0_OVRCURRCHG) {
 378                 dev_vdbg(hsotg->dev,
 379                          "  --Port Interrupt HPRT0=0x%08x Port Overcurrent Changed--\n",
 380                          hprt0);
 381                 hsotg->flags.b.port_over_current_change = 1;
 382                 hprt0_modify |= HPRT0_OVRCURRCHG;
 383         }
 384
 385         /* Clear Port Interrupts */
 386         writel(hprt0_modify, hsotg->regs + HPRT0);
 387 }
 388
 389 /*
 390  * Gets the actual length of a transfer after the transfer halts. halt_status
 391  * holds the reason for the halt.
 392  *
 393  * For IN transfers where halt_status is DWC2_HC_XFER_COMPLETE, *short_read
 394  * is set to 1 upon return if less than the requested number of bytes were
 395  * transferred. short_read may also be NULL on entry, in which case it remains
 396  * unchanged.
 397  */
 398 static u32 dwc2_get_actual_xfer_length(struct dwc2_hsotg *hsotg,
 399                                        struct dwc2_host_chan *chan, int chnum,
 400                                        struct dwc2_qtd *qtd,
 401                                        enum dwc2_halt_status halt_status,
 402                                        int *short_read)
 403 {
 404         u32 hctsiz, count, length;
 405
 406         hctsiz = readl(hsotg->regs + HCTSIZ(chnum));
 407
 408         if (halt_status == DWC2_HC_XFER_COMPLETE) {
 409                 if (chan->ep_is_in) {
 410                         count = hctsiz >> TSIZ_XFERSIZE_SHIFT &
 411                                 TSIZ_XFERSIZE_MASK >> TSIZ_XFERSIZE_SHIFT;
 412                         length = chan->xfer_len - count;
 413                         if (short_read != NULL)
 414                                 *short_read = (count != 0);
 415                 } else if (chan->qh->do_split) {
 416                         length = qtd->ssplit_out_xfer_count;
 417                 } else {
 418                         length = chan->xfer_len;
 419                 }
 420         } else {
 421                 /*
 422                  * Must use the hctsiz.pktcnt field to determine how much data
 423                  * has been transferred. This field reflects the number of
 424                  * packets that have been transferred via the USB. This is
 425                  * always an integral number of packets if the transfer was
 426                  * halted before its normal completion. (Can't use the
 427                  * hctsiz.xfersize field because that reflects the number of
 428                  * bytes transferred via the AHB, not the USB).
 429                  */
 430                 count = hctsiz >> TSIZ_PKTCNT_SHIFT &
 431                         TSIZ_PKTCNT_MASK >> TSIZ_PKTCNT_SHIFT;
 432                 length = (chan->start_pkt_count - count) * chan->max_packet;
 433         }
 434
 435         return length;
 436 }
 437
 438 /**
 439  * dwc2_update_urb_state() - Updates the state of the URB after a Transfer
 440  * Complete interrupt on the host channel. Updates the actual_length field
 441  * of the URB based on the number of bytes transferred via the host channel.
 442  * Sets the URB status if the data transfer is finished.
 443  *
 444  * Return: 1 if the data transfer specified by the URB is completely finished,
 445  * 0 otherwise
 446  */
 447 static int dwc2_update_urb_state(struct dwc2_hsotg *hsotg,
 448                                  struct dwc2_host_chan *chan, int chnum,
 449                                  struct dwc2_hcd_urb *urb,
 450                                  struct dwc2_qtd *qtd)
 451 {
 452         u32 hctsiz;
 453         int xfer_done = 0;
 454         int short_read = 0;
 455         int xfer_length = dwc2_get_actual_xfer_length(hsotg, chan, chnum, qtd,
 456                                                       DWC2_HC_XFER_COMPLETE,
 457                                                       &short_read);
 458
 459         if (urb->actual_length + xfer_length > urb->length) {
 460                 dev_warn(hsotg->dev, "%s(): trimming xfer length\n", __func__);
 461                 xfer_length = urb->length - urb->actual_length;
 462         }
 463
 464         /* Non DWORD-aligned buffer case handling */
 465         if (chan->align_buf && xfer_length && chan->ep_is_in) {
 466                 dev_dbg(hsotg->dev, "%s(): non-aligned buffer\n", __func__);
 467                 dma_sync_single_for_cpu(hsotg->dev, urb->dma, urb->length,
 468                                         DMA_FROM_DEVICE);
 469                 memcpy(urb->buf + urb->actual_length, chan->qh->dw_align_buf,
 470                        xfer_length);
 471                 dma_sync_single_for_device(hsotg->dev, urb->dma, urb->length,
 472                                            DMA_FROM_DEVICE);
 473         }
 474
 475         dev_vdbg(hsotg->dev, "urb->actual_length=%d xfer_length=%d\n",
 476                  urb->actual_length, xfer_length);
 477         urb->actual_length += xfer_length;
 478
 479         if (xfer_length && chan->ep_type == USB_ENDPOINT_XFER_BULK &&
 480             (urb->flags & URB_SEND_ZERO_PACKET) &&
 481             urb->actual_length >= urb->length &&
 482             !(urb->length % chan->max_packet)) {
 483                 xfer_done = 0;
 484         } else if (short_read || urb->actual_length >= urb->length) {
 485                 xfer_done = 1;
 486                 urb->status = 0;
 487         }
 488
 489         hctsiz = readl(hsotg->regs + HCTSIZ(chnum));
 490         dev_vdbg(hsotg->dev, "DWC_otg: %s: %s, channel %d\n",
 491                  __func__, (chan->ep_is_in ? "IN" : "OUT"), chnum);
 492         dev_vdbg(hsotg->dev, "  chan->xfer_len %d\n", chan->xfer_len);
 493         dev_vdbg(hsotg->dev, "  hctsiz.xfersize %d\n",
 494                  hctsiz >> TSIZ_XFERSIZE_SHIFT &
 495                  TSIZ_XFERSIZE_MASK >> TSIZ_XFERSIZE_SHIFT);
 496         dev_vdbg(hsotg->dev, "  urb->transfer_buffer_length %d\n", urb->length);
 497         dev_vdbg(hsotg->dev, "  urb->actual_length %d\n", urb->actual_length);
 498         dev_vdbg(hsotg->dev, "  short_read %d, xfer_done %d\n", short_read,
 499                  xfer_done);
 500
 501         return xfer_done;
 502 }
 503
 504 /*
 505  * Save the starting data toggle for the next transfer. The data toggle is
 506  * saved in the QH for non-control transfers and it's saved in the QTD for
 507  * control transfers.
 508  */
 509 void dwc2_hcd_save_data_toggle(struct dwc2_hsotg *hsotg,
 510                                struct dwc2_host_chan *chan, int chnum,
 511                                struct dwc2_qtd *qtd)
 512 {
 513         u32 hctsiz = readl(hsotg->regs + HCTSIZ(chnum));
 514         u32 pid = hctsiz & TSIZ_SC_MC_PID_MASK;
 515
 516         if (chan->ep_type != USB_ENDPOINT_XFER_CONTROL) {
 517                 if (pid == TSIZ_SC_MC_PID_DATA0)
 518                         chan->qh->data_toggle = DWC2_HC_PID_DATA0;
 519                 else
 520                         chan->qh->data_toggle = DWC2_HC_PID_DATA1;
 521         } else {
 522                 if (pid == TSIZ_SC_MC_PID_DATA0)
 523                         qtd->data_toggle = DWC2_HC_PID_DATA0;
 524                 else
 525                         qtd->data_toggle = DWC2_HC_PID_DATA1;
 526         }
 527 }
 528
 529 /**
 530  * dwc2_update_isoc_urb_state() - Updates the state of an Isochronous URB when
 531  * the transfer is stopped for any reason. The fields of the current entry in
 532  * the frame descriptor array are set based on the transfer state and the input
 533  * halt_status. Completes the Isochronous URB if all the URB frames have been
 534  * completed.
 535  *
 536  * Return: DWC2_HC_XFER_COMPLETE if there are more frames remaining to be
 537  * transferred in the URB. Otherwise return DWC2_HC_XFER_URB_COMPLETE.
 538  */
 539 static enum dwc2_halt_status dwc2_update_isoc_urb_state(
 540                 struct dwc2_hsotg *hsotg, struct dwc2_host_chan *chan,
 541                 int chnum, struct dwc2_qtd *qtd,
 542                 enum dwc2_halt_status halt_status)
 543 {
 544         struct dwc2_hcd_iso_packet_desc *frame_desc;
 545         struct dwc2_hcd_urb *urb = qtd->urb;
 546
 547         if (!urb)
 548                 return DWC2_HC_XFER_NO_HALT_STATUS;
 549
 550         frame_desc = &urb->iso_descs[qtd->isoc_frame_index];
 551
 552         switch (halt_status) {
 553         case DWC2_HC_XFER_COMPLETE:
 554                 frame_desc->status = 0;
 555                 frame_desc->actual_length = dwc2_get_actual_xfer_length(hsotg,
 556                                         chan, chnum, qtd, halt_status, NULL);
 557
 558                 /* Non DWORD-aligned buffer case handling */
 559                 if (chan->align_buf && frame_desc->actual_length &&
 560                     chan->ep_is_in) {
 561                         dev_dbg(hsotg->dev, "%s(): non-aligned buffer\n",
 562                                 __func__);
 563                         dma_sync_single_for_cpu(hsotg->dev, urb->dma,
 564                                                 urb->length, DMA_FROM_DEVICE);
 565                         memcpy(urb->buf + frame_desc->offset +
 566                                qtd->isoc_split_offset, chan->qh->dw_align_buf,
 567                                frame_desc->actual_length);
 568                         dma_sync_single_for_device(hsotg->dev, urb->dma,
 569                                                    urb->length,
 570                                                    DMA_FROM_DEVICE);
 571                 }
 572                 break;
 573         case DWC2_HC_XFER_FRAME_OVERRUN:
 574                 urb->error_count++;
 575                 if (chan->ep_is_in)
 576                         frame_desc->status = -ENOSR;
 577                 else
 578                         frame_desc->status = -ECOMM;
 579                 frame_desc->actual_length = 0;
 580                 break;
 581         case DWC2_HC_XFER_BABBLE_ERR:
 582                 urb->error_count++;
 583                 frame_desc->status = -EOVERFLOW;
 584                 /* Don't need to update actual_length in this case */
 585                 break;
 586         case DWC2_HC_XFER_XACT_ERR:
 587                 urb->error_count++;
 588                 frame_desc->status = -EPROTO;
 589                 frame_desc->actual_length = dwc2_get_actual_xfer_length(hsotg,
 590                                         chan, chnum, qtd, halt_status, NULL);
 591
 592                 /* Non DWORD-aligned buffer case handling */
 593                 if (chan->align_buf && frame_desc->actual_length &&
 594                     chan->ep_is_in) {
 595                         dev_dbg(hsotg->dev, "%s(): non-aligned buffer\n",
 596                                 __func__);
 597                         dma_sync_single_for_cpu(hsotg->dev, urb->dma,
 598                                                 urb->length, DMA_FROM_DEVICE);
 599                         memcpy(urb->buf + frame_desc->offset +
 600                                qtd->isoc_split_offset, chan->qh->dw_align_buf,
 601                                frame_desc->actual_length);
 602                         dma_sync_single_for_device(hsotg->dev, urb->dma,
 603                                                    urb->length,
 604                                                    DMA_FROM_DEVICE);
 605                 }
 606
 607                 /* Skip whole frame */
 608                 if (chan->qh->do_split &&
 609                     chan->ep_type == USB_ENDPOINT_XFER_ISOC && chan->ep_is_in &&
 610                     hsotg->core_params->dma_enable > 0) {
 611                         qtd->complete_split = 0;
 612                         qtd->isoc_split_offset = 0;
 613                 }
 614
 615                 break;
 616         default:
 617                 dev_err(hsotg->dev, "Unhandled halt_status (%d)\n",
 618                         halt_status);
 619                 break;
 620         }
 621
 622         if (++qtd->isoc_frame_index == urb->packet_count) {
 623                 /*
 624                  * urb->status is not used for isoc transfers. The individual
 625                  * frame_desc statuses are used instead.
 626                  */
 627                 dwc2_host_complete(hsotg, urb->priv, urb, 0);
 628                 halt_status = DWC2_HC_XFER_URB_COMPLETE;
 629         } else {
 630                 halt_status = DWC2_HC_XFER_COMPLETE;
 631         }
 632
 633         return halt_status;
 634 }
 635
 636 /*
 637  * Frees the first QTD in the QH's list if free_qtd is 1. For non-periodic
 638  * QHs, removes the QH from the active non-periodic schedule. If any QTDs are
 639  * still linked to the QH, the QH is added to the end of the inactive
 640  * non-periodic schedule. For periodic QHs, removes the QH from the periodic
 641  * schedule if no more QTDs are linked to the QH.
 642  */
 643 static void dwc2_deactivate_qh(struct dwc2_hsotg *hsotg, struct dwc2_qh *qh,
 644                                int free_qtd)
 645 {
 646         int continue_split = 0;
 647         struct dwc2_qtd *qtd;
 648
 649         dev_vdbg(hsotg->dev, "  %s(%p,%p,%d)\n", __func__, hsotg, qh, free_qtd);
 650
 651         if (list_empty(&qh->qtd_list)) {
 652                 dev_dbg(hsotg->dev, "## QTD list empty ##\n");
 653                 goto no_qtd;
 654         }
 655
 656         qtd = list_first_entry(&qh->qtd_list, struct dwc2_qtd, qtd_list_entry);
 657
 658         if (qtd->complete_split)
 659                 continue_split = 1;
 660         else if (qtd->isoc_split_pos == DWC2_HCSPLT_XACTPOS_MID ||
 661                  qtd->isoc_split_pos == DWC2_HCSPLT_XACTPOS_END)
 662                 continue_split = 1;
 663
 664         if (free_qtd) {
 665                 dwc2_hcd_qtd_unlink_and_free(hsotg, qtd, qh);
 666                 continue_split = 0;
 667         }
 668
 669 no_qtd:
 670         if (qh->channel)
 671                 qh->channel->align_buf = 0;
 672         qh->channel = NULL;
 673         dwc2_hcd_qh_deactivate(hsotg, qh, continue_split);
 674 }
 675
 676 /**
 677  * dwc2_release_channel() - Releases a host channel for use by other transfers
 678  *
 679  * @hsotg:       The HCD state structure
 680  * @chan:        The host channel to release
 681  * @qtd:         The QTD associated with the host channel. This QTD may be
 682  *               freed if the transfer is complete or an error has occurred.
 683  * @halt_status: Reason the channel is being released. This status
 684  *               determines the actions taken by this function.
 685  *
 686  * Also attempts to select and queue more transactions since at least one host
 687  * channel is available.
 688  */
 689 static void dwc2_release_channel(struct dwc2_hsotg *hsotg,
 690                                  struct dwc2_host_chan *chan,
 691                                  struct dwc2_qtd *qtd,
 692                                  enum dwc2_halt_status halt_status)
 693 {
 694         enum dwc2_transaction_type tr_type;
 695         u32 haintmsk;
 696         int free_qtd = 0;
 697
 698         dev_vdbg(hsotg->dev, "  %s: channel %d, halt_status %d\n",
 699                  __func__, chan->hc_num, halt_status);
 700
 701         switch (halt_status) {
 702         case DWC2_HC_XFER_URB_COMPLETE:
 703                 free_qtd = 1;
 704                 break;
 705         case DWC2_HC_XFER_AHB_ERR:
 706         case DWC2_HC_XFER_STALL:
 707         case DWC2_HC_XFER_BABBLE_ERR:
 708                 free_qtd = 1;
 709                 break;
 710         case DWC2_HC_XFER_XACT_ERR:
 711                 if (qtd->error_count >= 3) {
 712                         dev_vdbg(hsotg->dev,
 713                                  "  Complete URB with transaction error\n");
 714                         free_qtd = 1;
 715                         if (qtd->urb) {
 716                                 qtd->urb->status = -EPROTO;
 717                                 dwc2_host_complete(hsotg, qtd->urb->priv,
 718                                                    qtd->urb, -EPROTO);
 719                         }
 720                 }
 721                 break;
 722         case DWC2_HC_XFER_URB_DEQUEUE:
 723                 /*
 724                  * The QTD has already been removed and the QH has been
 725                  * deactivated. Don't want to do anything except release the
 726                  * host channel and try to queue more transfers.
 727                  */
 728                 goto cleanup;
 729         case DWC2_HC_XFER_PERIODIC_INCOMPLETE:
 730                 dev_vdbg(hsotg->dev, "  Complete URB with I/O error\n");
 731                 free_qtd = 1;
 732                 if (qtd->urb) {
 733                         qtd->urb->status = -EIO;
 734                         dwc2_host_complete(hsotg, qtd->urb->priv, qtd->urb,
 735                                            -EIO);
 736                 }
 737                 break;
 738         case DWC2_HC_XFER_NO_HALT_STATUS:
 739         default:
 740                 break;
 741         }
 742
 743         dwc2_deactivate_qh(hsotg, chan->qh, free_qtd);
 744
 745 cleanup:
 746         /*
 747          * Release the host channel for use by other transfers. The cleanup
 748          * function clears the channel interrupt enables and conditions, so
 749          * there's no need to clear the Channel Halted interrupt separately.
 750          */
 751         if (!list_empty(&chan->hc_list_entry))
 752                 list_del(&chan->hc_list_entry);
 753         dwc2_hc_cleanup(hsotg, chan);
 754         list_add_tail(&chan->hc_list_entry, &hsotg->free_hc_list);
 755
 756         switch (chan->ep_type) {
 757         case USB_ENDPOINT_XFER_CONTROL:
 758         case USB_ENDPOINT_XFER_BULK:
 759                 hsotg->non_periodic_channels--;
 760                 break;
 761         default:
 762                 /*
 763                  * Don't release reservations for periodic channels here.
 764                  * That's done when a periodic transfer is descheduled (i.e.
 765                  * when the QH is removed from the periodic schedule).
 766                  */
 767                 break;
 768         }
 769
 770         haintmsk = readl(hsotg->regs + HAINTMSK);
 771         haintmsk &= ~(1 << chan->hc_num);
 772         writel(haintmsk, hsotg->regs + HAINTMSK);
 773
 774         /* Try to queue more transfers now that there's a free channel */
 775         tr_type = dwc2_hcd_select_transactions(hsotg);
 776         if (tr_type != DWC2_TRANSACTION_NONE)
 777                 dwc2_hcd_queue_transactions(hsotg, tr_type);
 778 }
 779
 780 /*
 781  * Halts a host channel. If the channel cannot be halted immediately because
 782  * the request queue is full, this function ensures that the FIFO empty
 783  * interrupt for the appropriate queue is enabled so that the halt request can
 784  * be queued when there is space in the request queue.
 785  *
 786  * This function may also be called in DMA mode. In that case, the channel is
 787  * simply released since the core always halts the channel automatically in
 788  * DMA mode.
 789  */
 790 static void dwc2_halt_channel(struct dwc2_hsotg *hsotg,
 791                               struct dwc2_host_chan *chan, struct dwc2_qtd *qtd,
 792                               enum dwc2_halt_status halt_status)
 793 {
 794         dev_vdbg(hsotg->dev, "%s()\n", __func__);
 795
 796         if (hsotg->core_params->dma_enable > 0) {
 797                 dev_vdbg(hsotg->dev, "DMA enabled\n");
 798                 dwc2_release_channel(hsotg, chan, qtd, halt_status);
 799                 return;
 800         }
 801
 802         /* Slave mode processing */
 803         dwc2_hc_halt(hsotg, chan, halt_status);
 804
 805         if (chan->halt_on_queue) {
 806                 u32 gintmsk;
 807
 808                 dev_vdbg(hsotg->dev, "Halt on queue\n");
 809                 if (chan->ep_type == USB_ENDPOINT_XFER_CONTROL ||
 810                     chan->ep_type == USB_ENDPOINT_XFER_BULK) {
 811                         dev_vdbg(hsotg->dev, "control/bulk\n");
 812                         /*
 813                          * Make sure the Non-periodic Tx FIFO empty interrupt
 814                          * is enabled so that the non-periodic schedule will
 815                          * be processed
 816                          */
 817                         gintmsk = readl(hsotg->regs + GINTMSK);
 818                         gintmsk |= GINTSTS_NPTXFEMP;
 819                         writel(gintmsk, hsotg->regs + GINTMSK);
 820                 } else {
 821                         dev_vdbg(hsotg->dev, "isoc/intr\n");
 822                         /*
 823                          * Move the QH from the periodic queued schedule to
 824                          * the periodic assigned schedule. This allows the
 825                          * halt to be queued when the periodic schedule is
 826                          * processed.
 827                          */
 828                         list_move(&chan->qh->qh_list_entry,
 829                                   &hsotg->periodic_sched_assigned);
 830
 831                         /*
 832                          * Make sure the Periodic Tx FIFO Empty interrupt is
 833                          * enabled so that the periodic schedule will be
 834                          * processed
 835                          */
 836                         gintmsk = readl(hsotg->regs + GINTMSK);
 837                         gintmsk |= GINTSTS_PTXFEMP;
 838                         writel(gintmsk, hsotg->regs + GINTMSK);
 839                 }
 840         }
 841 }
 842
 843 /*
 844  * Performs common cleanup for non-periodic transfers after a Transfer
 845  * Complete interrupt. This function should be called after any endpoint type
 846  * specific handling is finished to release the host channel.
 847  */
 848 static void dwc2_complete_non_periodic_xfer(struct dwc2_hsotg *hsotg,
 849                                             struct dwc2_host_chan *chan,
 850                                             int chnum, struct dwc2_qtd *qtd,
 851                                             enum dwc2_halt_status halt_status)
 852 {
 853         dev_vdbg(hsotg->dev, "%s()\n", __func__);
 854
 855         qtd->error_count = 0;
 856
 857         if (chan->hcint & HCINTMSK_NYET) {
 858                 /*
 859                  * Got a NYET on the last transaction of the transfer. This
 860                  * means that the endpoint should be in the PING state at the
 861                  * beginning of the next transfer.
 862                  */
 863                 dev_vdbg(hsotg->dev, "got NYET\n");
 864                 chan->qh->ping_state = 1;
 865         }
 866
 867         /*
 868          * Always halt and release the host channel to make it available for
 869          * more transfers. There may still be more phases for a control
 870          * transfer or more data packets for a bulk transfer at this point,
 871          * but the host channel is still halted. A channel will be reassigned
 872          * to the transfer when the non-periodic schedule is processed after
 873          * the channel is released. This allows transactions to be queued
 874          * properly via dwc2_hcd_queue_transactions, which also enables the
 875          * Tx FIFO Empty interrupt if necessary.
 876          */
 877         if (chan->ep_is_in) {
 878                 /*
 879                  * IN transfers in Slave mode require an explicit disable to
 880                  * halt the channel. (In DMA mode, this call simply releases
 881                  * the channel.)
 882                  */
 883                 dwc2_halt_channel(hsotg, chan, qtd, halt_status);
 884         } else {
 885                 /*
 886                  * The channel is automatically disabled by the core for OUT
 887                  * transfers in Slave mode
 888                  */
 889                 dwc2_release_channel(hsotg, chan, qtd, halt_status);
 890         }
 891 }
 892
 893 /*
 894  * Performs common cleanup for periodic transfers after a Transfer Complete
 895  * interrupt. This function should be called after any endpoint type specific
 896  * handling is finished to release the host channel.
 897  */
 898 static void dwc2_complete_periodic_xfer(struct dwc2_hsotg *hsotg,
 899                                         struct dwc2_host_chan *chan, int chnum,
 900                                         struct dwc2_qtd *qtd,
 901                                         enum dwc2_halt_status halt_status)
 902 {
 903         u32 hctsiz = readl(hsotg->regs + HCTSIZ(chnum));
 904
 905         qtd->error_count = 0;
 906
 907         if (!chan->ep_is_in || (hctsiz & TSIZ_PKTCNT_MASK) == 0)
 908                 /* Core halts channel in these cases */
 909                 dwc2_release_channel(hsotg, chan, qtd, halt_status);
 910         else
 911                 /* Flush any outstanding requests from the Tx queue */
 912                 dwc2_halt_channel(hsotg, chan, qtd, halt_status);
 913 }
 914
 915 static int dwc2_xfercomp_isoc_split_in(struct dwc2_hsotg *hsotg,
 916                                        struct dwc2_host_chan *chan, int chnum,
 917                                        struct dwc2_qtd *qtd)
 918 {
 919         struct dwc2_hcd_iso_packet_desc *frame_desc;
 920         u32 len;
 921
 922         if (!qtd->urb)
 923                 return 0;
 924
 925         frame_desc = &qtd->urb->iso_descs[qtd->isoc_frame_index];
 926         len = dwc2_get_actual_xfer_length(hsotg, chan, chnum, qtd,
 927                                           DWC2_HC_XFER_COMPLETE, NULL);
 928         if (!len) {
 929                 qtd->complete_split = 0;
 930                 qtd->isoc_split_offset = 0;
 931                 return 0;
 932         }
 933
 934         frame_desc->actual_length += len;
 935
 936         if (chan->align_buf && len) {
 937                 dev_dbg(hsotg->dev, "%s(): non-aligned buffer\n", __func__);
 938                 dma_sync_single_for_cpu(hsotg->dev, qtd->urb->dma,
 939                                         qtd->urb->length, DMA_FROM_DEVICE);
 940                 memcpy(qtd->urb->buf + frame_desc->offset +
 941                        qtd->isoc_split_offset, chan->qh->dw_align_buf, len);
 942                 dma_sync_single_for_device(hsotg->dev, qtd->urb->dma,
 943                                            qtd->urb->length, DMA_FROM_DEVICE);
 944         }
 945
 946         qtd->isoc_split_offset += len;
 947
 948         if (frame_desc->actual_length >= frame_desc->length) {
 949                 frame_desc->status = 0;
 950                 qtd->isoc_frame_index++;
 951                 qtd->complete_split = 0;
 952                 qtd->isoc_split_offset = 0;
 953         }
 954
 955         if (qtd->isoc_frame_index == qtd->urb->packet_count) {
 956                 dwc2_host_complete(hsotg, qtd->urb->priv, qtd->urb, 0);
 957                 dwc2_release_channel(hsotg, chan, qtd,
 958                                      DWC2_HC_XFER_URB_COMPLETE);
 959         } else {
 960                 dwc2_release_channel(hsotg, chan, qtd,
 961                                      DWC2_HC_XFER_NO_HALT_STATUS);
 962         }
 963
 964         return 1;       /* Indicates that channel released */
 965 }
 966
 967 /*
 968  * Handles a host channel Transfer Complete interrupt. This handler may be
 969  * called in either DMA mode or Slave mode.
 970  */
 971 static void dwc2_hc_xfercomp_intr(struct dwc2_hsotg *hsotg,
 972                                   struct dwc2_host_chan *chan, int chnum,
 973                                   struct dwc2_qtd *qtd)
 974 {
 975         struct dwc2_hcd_urb *urb = qtd->urb;
 976         int pipe_type = dwc2_hcd_get_pipe_type(&urb->pipe_info);
 977         enum dwc2_halt_status halt_status = DWC2_HC_XFER_COMPLETE;
 978         int urb_xfer_done;
 979
 980         dev_vdbg(hsotg->dev,
 981                  "--Host Channel %d Interrupt: Transfer Complete--\n", chnum);
 982
 983         if (hsotg->core_params->dma_desc_enable > 0) {
 984                 dwc2_hcd_complete_xfer_ddma(hsotg, chan, chnum, halt_status);
 985                 if (pipe_type == USB_ENDPOINT_XFER_ISOC)
 986                         /* Do not disable the interrupt, just clear it */
 987                         return;
 988                 goto handle_xfercomp_done;
 989         }
 990
 991         /* Handle xfer complete on CSPLIT */
 992         if (chan->qh->do_split) {
 993                 if (chan->ep_type == USB_ENDPOINT_XFER_ISOC && chan->ep_is_in &&
 994                     hsotg->core_params->dma_enable > 0) {
 995                         if (qtd->complete_split &&
 996                             dwc2_xfercomp_isoc_split_in(hsotg, chan, chnum,
 997                                                         qtd))
 998                                 goto handle_xfercomp_done;
 999                 } else {
1000                         qtd->complete_split = 0;
1001                 }
1002         }
1003
1004         if (!urb)
1005                 goto handle_xfercomp_done;
1006
1007         /* Update the QTD and URB states */
1008         switch (pipe_type) {
1009         case USB_ENDPOINT_XFER_CONTROL:
1010                 switch (qtd->control_phase) {
1011                 case DWC2_CONTROL_SETUP:
1012                         if (urb->length > 0)
1013                                 qtd->control_phase = DWC2_CONTROL_DATA;
1014                         else
1015                                 qtd->control_phase = DWC2_CONTROL_STATUS;
1016                         dev_vdbg(hsotg->dev,
1017                                  "  Control setup transaction done\n");
1018                         halt_status = DWC2_HC_XFER_COMPLETE;
1019                         break;
1020                 case DWC2_CONTROL_DATA:
1021                         urb_xfer_done = dwc2_update_urb_state(hsotg, chan,
1022                                                               chnum, urb, qtd);
1023                         if (urb_xfer_done) {
1024                                 qtd->control_phase = DWC2_CONTROL_STATUS;
1025                                 dev_vdbg(hsotg->dev,
1026                                          "  Control data transfer done\n");
1027                         } else {
1028                                 dwc2_hcd_save_data_toggle(hsotg, chan, chnum,
1029                                                           qtd);
1030                         }
1031                         halt_status = DWC2_HC_XFER_COMPLETE;
1032                         break;
1033                 case DWC2_CONTROL_STATUS:
1034                         dev_vdbg(hsotg->dev, "  Control transfer complete\n");
1035                         if (urb->status == -EINPROGRESS)
1036                                 urb->status = 0;
1037                         dwc2_host_complete(hsotg, urb->priv, urb, urb->status);
1038                         halt_status = DWC2_HC_XFER_URB_COMPLETE;
1039                         break;
1040                 }
1041
1042                 dwc2_complete_non_periodic_xfer(hsotg, chan, chnum, qtd,
1043                                                 halt_status);
1044                 break;
1045         case USB_ENDPOINT_XFER_BULK:
1046                 dev_vdbg(hsotg->dev, "  Bulk transfer complete\n");
1047                 urb_xfer_done = dwc2_update_urb_state(hsotg, chan, chnum, urb,
1048                                                       qtd);
1049                 if (urb_xfer_done) {
1050                         dwc2_host_complete(hsotg, urb->priv, urb, urb->status);
1051                         halt_status = DWC2_HC_XFER_URB_COMPLETE;
1052                 } else {
1053                         halt_status = DWC2_HC_XFER_COMPLETE;
1054                 }
1055
1056                 dwc2_hcd_save_data_toggle(hsotg, chan, chnum, qtd);
1057                 dwc2_complete_non_periodic_xfer(hsotg, chan, chnum, qtd,
1058                                                 halt_status);
1059                 break;
1060         case USB_ENDPOINT_XFER_INT:
1061                 dev_vdbg(hsotg->dev, "  Interrupt transfer complete\n");
1062                 urb_xfer_done = dwc2_update_urb_state(hsotg, chan, chnum, urb,
1063                                                       qtd);
1064
1065                 /*
1066                  * Interrupt URB is done on the first transfer complete
1067                  * interrupt
1068                  */
1069                 if (urb_xfer_done) {
1070                                 dwc2_host_complete(hsotg, urb->priv, urb,
1071                                                    urb->status);
1072                                 halt_status = DWC2_HC_XFER_URB_COMPLETE;
1073                 } else {
1074                                 halt_status = DWC2_HC_XFER_COMPLETE;
1075                 }
1076
1077                 dwc2_hcd_save_data_toggle(hsotg, chan, chnum, qtd);
1078                 dwc2_complete_periodic_xfer(hsotg, chan, chnum, qtd,
1079                                             halt_status);
1080                 break;
1081         case USB_ENDPOINT_XFER_ISOC:
1082                 dev_vdbg(hsotg->dev, "  Isochronous transfer complete\n");
1083                 if (qtd->isoc_split_pos == DWC2_HCSPLT_XACTPOS_ALL)
1084                         halt_status = dwc2_update_isoc_urb_state(hsotg, chan,
1085                                         chnum, qtd, DWC2_HC_XFER_COMPLETE);
1086                 dwc2_complete_periodic_xfer(hsotg, chan, chnum, qtd,
1087                                             halt_status);
1088                 break;
1089         }
1090
1091 handle_xfercomp_done:
1092         disable_hc_int(hsotg, chnum, HCINTMSK_XFERCOMPL);
1093 }
1094
1095 /*
1096  * Handles a host channel STALL interrupt. This handler may be called in
1097  * either DMA mode or Slave mode.
1098  */
1099 static void dwc2_hc_stall_intr(struct dwc2_hsotg *hsotg,
1100                                struct dwc2_host_chan *chan, int chnum,
1101                                struct dwc2_qtd *qtd)
1102 {
1103         struct dwc2_hcd_urb *urb = qtd->urb;
1104         int pipe_type = dwc2_hcd_get_pipe_type(&urb->pipe_info);
1105
1106         dev_dbg(hsotg->dev, "--Host Channel %d Interrupt: STALL Received--\n",
1107                 chnum);
1108
1109         if (hsotg->core_params->dma_desc_enable > 0) {
1110                 dwc2_hcd_complete_xfer_ddma(hsotg, chan, chnum,
1111                                             DWC2_HC_XFER_STALL);
1112                 goto handle_stall_done;
1113         }
1114
1115         if (!urb)
1116                 goto handle_stall_halt;
1117
1118         if (pipe_type == USB_ENDPOINT_XFER_CONTROL)
1119                 dwc2_host_complete(hsotg, urb->priv, urb, -EPIPE);
1120
1121         if (pipe_type == USB_ENDPOINT_XFER_BULK ||
1122             pipe_type == USB_ENDPOINT_XFER_INT) {
1123                 dwc2_host_complete(hsotg, urb->priv, urb, -EPIPE);
1124                 /*
1125                  * USB protocol requires resetting the data toggle for bulk
1126                  * and interrupt endpoints when a CLEAR_FEATURE(ENDPOINT_HALT)
1127                  * setup command is issued to the endpoint. Anticipate the
1128                  * CLEAR_FEATURE command since a STALL has occurred and reset
1129                  * the data toggle now.
1130                  */
1131                 chan->qh->data_toggle = 0;
1132         }
1133
1134 handle_stall_halt:
1135         dwc2_halt_channel(hsotg, chan, qtd, DWC2_HC_XFER_STALL);
1136
1137 handle_stall_done:
1138         disable_hc_int(hsotg, chnum, HCINTMSK_STALL);
1139 }
1140
1141 /*
1142  * Updates the state of the URB when a transfer has been stopped due to an
1143  * abnormal condition before the transfer completes. Modifies the
1144  * actual_length field of the URB to reflect the number of bytes that have
1145  * actually been transferred via the host channel.
1146  */
1147 static void dwc2_update_urb_state_abn(struct dwc2_hsotg *hsotg,
1148                                       struct dwc2_host_chan *chan, int chnum,
1149                                       struct dwc2_hcd_urb *urb,
1150                                       struct dwc2_qtd *qtd,
1151                                       enum dwc2_halt_status halt_status)
1152 {
1153         u32 xfer_length = dwc2_get_actual_xfer_length(hsotg, chan, chnum,
1154                                                       qtd, halt_status, NULL);
1155         u32 hctsiz;
1156
1157         if (urb->actual_length + xfer_length > urb->length) {
1158                 dev_warn(hsotg->dev, "%s(): trimming xfer length\n", __func__);
1159                 xfer_length = urb->length - urb->actual_length;
1160         }
1161
1162         /* Non DWORD-aligned buffer case handling */
1163         if (chan->align_buf && xfer_length && chan->ep_is_in) {
1164                 dev_dbg(hsotg->dev, "%s(): non-aligned buffer\n", __func__);
1165                 dma_sync_single_for_cpu(hsotg->dev, urb->dma, urb->length,
1166                                         DMA_FROM_DEVICE);
1167                 memcpy(urb->buf + urb->actual_length, chan->qh->dw_align_buf,
1168                        xfer_length);
1169                 dma_sync_single_for_device(hsotg->dev, urb->dma, urb->length,
1170                                            DMA_FROM_DEVICE);
1171         }
1172
1173         urb->actual_length += xfer_length;
1174
1175         hctsiz = readl(hsotg->regs + HCTSIZ(chnum));
1176         dev_vdbg(hsotg->dev, "DWC_otg: %s: %s, channel %d\n",
1177                  __func__, (chan->ep_is_in ? "IN" : "OUT"), chnum);
1178         dev_vdbg(hsotg->dev, "  chan->start_pkt_count %d\n",
1179                  chan->start_pkt_count);
1180         dev_vdbg(hsotg->dev, "  hctsiz.pktcnt %d\n",
1181                  hctsiz >> TSIZ_PKTCNT_SHIFT &
1182                  TSIZ_PKTCNT_MASK >> TSIZ_PKTCNT_SHIFT);
1183         dev_vdbg(hsotg->dev, "  chan->max_packet %d\n", chan->max_packet);
1184         dev_vdbg(hsotg->dev, "  bytes_transferred %d\n",
1185                  xfer_length);
1186         dev_vdbg(hsotg->dev, "  urb->actual_length %d\n",
1187                  urb->actual_length);
1188         dev_vdbg(hsotg->dev, "  urb->transfer_buffer_length %d\n",
1189                  urb->length);
1190 }
1191
1192 /*
1193  * Handles a host channel NAK interrupt. This handler may be called in either
1194  * DMA mode or Slave mode.
1195  */
1196 static void dwc2_hc_nak_intr(struct dwc2_hsotg *hsotg,
1197                              struct dwc2_host_chan *chan, int chnum,
1198                              struct dwc2_qtd *qtd)
1199 {
1200         dev_vdbg(hsotg->dev, "--Host Channel %d Interrupt: NAK Received--\n",
1201                  chnum);
1202
1203         /*
1204          * Handle NAK for IN/OUT SSPLIT/CSPLIT transfers, bulk, control, and
1205          * interrupt. Re-start the SSPLIT transfer.
1206          */
1207         if (chan->do_split) {
1208                 if (chan->complete_split)
1209                         qtd->error_count = 0;
1210                 qtd->complete_split = 0;
1211                 dwc2_halt_channel(hsotg, chan, qtd, DWC2_HC_XFER_NAK);
1212                 goto handle_nak_done;
1213         }
1214
1215         switch (dwc2_hcd_get_pipe_type(&qtd->urb->pipe_info)) {
1216         case USB_ENDPOINT_XFER_CONTROL:
1217         case USB_ENDPOINT_XFER_BULK:
1218                 if (hsotg->core_params->dma_enable > 0 && chan->ep_is_in) {
1219                         /*
1220                          * NAK interrupts are enabled on bulk/control IN
1221                          * transfers in DMA mode for the sole purpose of
1222                          * resetting the error count after a transaction error
1223                          * occurs. The core will continue transferring data.
1224                          */
1225                         qtd->error_count = 0;
1226                         break;
1227                 }
1228
1229                 /*
1230                  * NAK interrupts normally occur during OUT transfers in DMA
1231                  * or Slave mode. For IN transfers, more requests will be
1232                  * queued as request queue space is available.
1233                  */
1234                 qtd->error_count = 0;
1235
1236                 if (!chan->qh->ping_state) {
1237                         dwc2_update_urb_state_abn(hsotg, chan, chnum, qtd->urb,
1238                                                   qtd, DWC2_HC_XFER_NAK);
1239                         dwc2_hcd_save_data_toggle(hsotg, chan, chnum, qtd);
1240
1241                         if (chan->speed == USB_SPEED_HIGH)
1242                                 chan->qh->ping_state = 1;
1243                 }
1244
1245                 /*
1246                  * Halt the channel so the transfer can be re-started from
1247                  * the appropriate point or the PING protocol will
1248                  * start/continue
1249                  */
1250                 dwc2_halt_channel(hsotg, chan, qtd, DWC2_HC_XFER_NAK);
1251                 break;
1252         case USB_ENDPOINT_XFER_INT:
1253                 qtd->error_count = 0;
1254                 dwc2_halt_channel(hsotg, chan, qtd, DWC2_HC_XFER_NAK);
1255                 break;
1256         case USB_ENDPOINT_XFER_ISOC:
1257                 /* Should never get called for isochronous transfers */
1258                 dev_err(hsotg->dev, "NACK interrupt for ISOC transfer\n");
1259                 break;
1260         }
1261
1262 handle_nak_done:
1263         disable_hc_int(hsotg, chnum, HCINTMSK_NAK);
1264 }
1265
1266 /*
1267  * Handles a host channel ACK interrupt. This interrupt is enabled when
1268  * performing the PING protocol in Slave mode, when errors occur during
1269  * either Slave mode or DMA mode, and during Start Split transactions.
1270  */
1271 static void dwc2_hc_ack_intr(struct dwc2_hsotg *hsotg,
1272                              struct dwc2_host_chan *chan, int chnum,
1273                              struct dwc2_qtd *qtd)
1274 {
1275         struct dwc2_hcd_iso_packet_desc *frame_desc;
1276
1277         dev_vdbg(hsotg->dev, "--Host Channel %d Interrupt: ACK Received--\n",
1278                  chnum);
1279
1280         if (chan->do_split) {
1281                 /* Handle ACK on SSPLIT. ACK should not occur in CSPLIT. */
1282                 if (!chan->ep_is_in &&
1283                     chan->data_pid_start != DWC2_HC_PID_SETUP)
1284                         qtd->ssplit_out_xfer_count = chan->xfer_len;
1285
1286                 if (chan->ep_type != USB_ENDPOINT_XFER_ISOC || chan->ep_is_in) {
1287                         qtd->complete_split = 1;
1288                         dwc2_halt_channel(hsotg, chan, qtd, DWC2_HC_XFER_ACK);
1289                 } else {
1290                         /* ISOC OUT */
1291                         switch (chan->xact_pos) {
1292                         case DWC2_HCSPLT_XACTPOS_ALL:
1293                                 break;
1294                         case DWC2_HCSPLT_XACTPOS_END:
1295                                 qtd->isoc_split_pos = DWC2_HCSPLT_XACTPOS_ALL;
1296                                 qtd->isoc_split_offset = 0;
1297                                 break;
1298                         case DWC2_HCSPLT_XACTPOS_BEGIN:
1299                         case DWC2_HCSPLT_XACTPOS_MID:
1300                                 /*
1301                                  * For BEGIN or MID, calculate the length for
1302                                  * the next microframe to determine the correct
1303                                  * SSPLIT token, either MID or END
1304                                  */
1305                                 frame_desc = &qtd->urb->iso_descs[
1306                                                 qtd->isoc_frame_index];
1307                                 qtd->isoc_split_offset += 188;
1308
1309                                 if (frame_desc->length - qtd->isoc_split_offset
1310                                                         <= 188)
1311                                         qtd->isoc_split_pos =
1312                                                         DWC2_HCSPLT_XACTPOS_END;
1313                                 else
1314                                         qtd->isoc_split_pos =
1315                                                         DWC2_HCSPLT_XACTPOS_MID;
1316                                 break;
1317                         }
1318                 }
1319         } else {
1320                 qtd->error_count = 0;
1321
1322                 if (chan->qh->ping_state) {
1323                         chan->qh->ping_state = 0;
1324                         /*
1325                          * Halt the channel so the transfer can be re-started
1326                          * from the appropriate point. This only happens in
1327                          * Slave mode. In DMA mode, the ping_state is cleared
1328                          * when the transfer is started because the core
1329                          * automatically executes the PING, then the transfer.
1330                          */
1331                         dwc2_halt_channel(hsotg, chan, qtd, DWC2_HC_XFER_ACK);
1332                 }
1333         }
1334
1335         /*
1336          * If the ACK occurred when _not_ in the PING state, let the channel
1337          * continue transferring data after clearing the error count
1338          */
1339         disable_hc_int(hsotg, chnum, HCINTMSK_ACK);
1340 }
1341
1342 /*
1343  * Handles a host channel NYET interrupt. This interrupt should only occur on
1344  * Bulk and Control OUT endpoints and for complete split transactions. If a
1345  * NYET occurs at the same time as a Transfer Complete interrupt, it is
1346  * handled in the xfercomp interrupt handler, not here. This handler may be
1347  * called in either DMA mode or Slave mode.
1348  */
1349 static void dwc2_hc_nyet_intr(struct dwc2_hsotg *hsotg,
1350                               struct dwc2_host_chan *chan, int chnum,
1351                               struct dwc2_qtd *qtd)
1352 {
1353         dev_vdbg(hsotg->dev, "--Host Channel %d Interrupt: NYET Received--\n",
1354                  chnum);
1355
1356         /*
1357          * NYET on CSPLIT
1358          * re-do the CSPLIT immediately on non-periodic
1359          */
1360         if (chan->do_split && chan->complete_split) {
1361                 if (chan->ep_is_in && chan->ep_type == USB_ENDPOINT_XFER_ISOC &&
1362                     hsotg->core_params->dma_enable > 0) {
1363                         qtd->complete_split = 0;
1364                         qtd->isoc_split_offset = 0;
1365                         if (++qtd->isoc_frame_index == qtd->urb->packet_count) {
1366                                 if (qtd->urb)
1367                                         dwc2_host_complete(hsotg,
1368                                                            qtd->urb->priv,
1369                                                            qtd->urb, 0);
1370                                 dwc2_release_channel(hsotg, chan, qtd,
1371                                                 DWC2_HC_XFER_URB_COMPLETE);
1372                         } else {
1373                                 dwc2_release_channel(hsotg, chan, qtd,
1374                                                 DWC2_HC_XFER_NO_HALT_STATUS);
1375                         }
1376                         goto handle_nyet_done;
1377                 }
1378
1379                 if (chan->ep_type == USB_ENDPOINT_XFER_INT ||
1380                     chan->ep_type == USB_ENDPOINT_XFER_ISOC) {
1381                         int frnum = dwc2_hcd_get_frame_number(hsotg);
1382
1383                         if (dwc2_full_frame_num(frnum) !=
1384                             dwc2_full_frame_num(chan->qh->sched_frame)) {
1385                                 /*
1386                                  * No longer in the same full speed frame.
1387                                  * Treat this as a transaction error.
1388                                  */
1389 #if 0
1390                                 /*
1391                                  * Todo: Fix system performance so this can
1392                                  * be treated as an error. Right now complete
1393                                  * splits cannot be scheduled precisely enough
1394                                  * due to other system activity, so this error
1395                                  * occurs regularly in Slave mode.
1396                                  */
1397                                 qtd->error_count++;
1398 #endif
1399                                 qtd->complete_split = 0;
1400                                 dwc2_halt_channel(hsotg, chan, qtd,
1401                                                   DWC2_HC_XFER_XACT_ERR);
1402                                 /* Todo: add support for isoc release */
1403                                 goto handle_nyet_done;
1404                         }
1405                 }
1406
1407                 dwc2_halt_channel(hsotg, chan, qtd, DWC2_HC_XFER_NYET);
1408                 goto handle_nyet_done;
1409         }
1410
1411         chan->qh->ping_state = 1;
1412         qtd->error_count = 0;
1413
1414         dwc2_update_urb_state_abn(hsotg, chan, chnum, qtd->urb, qtd,
1415                                   DWC2_HC_XFER_NYET);
1416         dwc2_hcd_save_data_toggle(hsotg, chan, chnum, qtd);
1417
1418         /*
1419          * Halt the channel and re-start the transfer so the PING protocol
1420          * will start
1421          */
1422         dwc2_halt_channel(hsotg, chan, qtd, DWC2_HC_XFER_NYET);
1423
1424 handle_nyet_done:
1425         disable_hc_int(hsotg, chnum, HCINTMSK_NYET);
1426 }
1427
1428 /*
1429  * Handles a host channel babble interrupt. This handler may be called in
1430  * either DMA mode or Slave mode.
1431  */
1432 static void dwc2_hc_babble_intr(struct dwc2_hsotg *hsotg,
1433                                 struct dwc2_host_chan *chan, int chnum,
1434                                 struct dwc2_qtd *qtd)
1435 {
1436         dev_dbg(hsotg->dev, "--Host Channel %d Interrupt: Babble Error--\n",
1437                 chnum);
1438
1439         if (hsotg->core_params->dma_desc_enable > 0) {
1440                 dwc2_hcd_complete_xfer_ddma(hsotg, chan, chnum,
1441                                             DWC2_HC_XFER_BABBLE_ERR);
1442                 goto handle_babble_done;
1443         }
1444
1445         if (chan->ep_type != USB_ENDPOINT_XFER_ISOC) {
1446                 if (qtd->urb)
1447                         dwc2_host_complete(hsotg, qtd->urb->priv, qtd->urb,
1448                                            -EOVERFLOW);
1449                 dwc2_halt_channel(hsotg, chan, qtd, DWC2_HC_XFER_BABBLE_ERR);
1450         } else {
1451                 enum dwc2_halt_status halt_status;
1452
1453                 halt_status = dwc2_update_isoc_urb_state(hsotg, chan, chnum,
1454                                                 qtd, DWC2_HC_XFER_BABBLE_ERR);
1455                 dwc2_halt_channel(hsotg, chan, qtd, halt_status);
1456         }
1457
1458 handle_babble_done:
1459         dwc2_hc_handle_tt_clear(hsotg, chan, qtd);
1460         disable_hc_int(hsotg, chnum, HCINTMSK_BBLERR);
1461 }
1462
1463 /*
1464  * Handles a host channel AHB error interrupt. This handler is only called in
1465  * DMA mode.
1466  */
1467 static void dwc2_hc_ahberr_intr(struct dwc2_hsotg *hsotg,
1468                                 struct dwc2_host_chan *chan, int chnum,
1469                                 struct dwc2_qtd *qtd)
1470 {
1471         struct dwc2_hcd_urb *urb = qtd->urb;
1472         char *pipetype, *speed;
1473         u32 hcchar;
1474         u32 hcsplt;
1475         u32 hctsiz;
1476         u32 hc_dma;
1477
1478         dev_dbg(hsotg->dev, "--Host Channel %d Interrupt: AHB Error--\n",
1479                 chnum);
1480
1481         if (!urb)
1482                 goto handle_ahberr_halt;
1483
1484         hcchar = readl(hsotg->regs + HCCHAR(chnum));
1485         hcsplt = readl(hsotg->regs + HCSPLT(chnum));
1486         hctsiz = readl(hsotg->regs + HCTSIZ(chnum));
1487         hc_dma = readl(hsotg->regs + HCDMA(chnum));
1488
1489         dev_err(hsotg->dev, "AHB ERROR, Channel %d\n", chnum);
1490         dev_err(hsotg->dev, "  hcchar 0x%08x, hcsplt 0x%08x\n", hcchar, hcsplt);
1491         dev_err(hsotg->dev, "  hctsiz 0x%08x, hc_dma 0x%08x\n", hctsiz, hc_dma);
1492         dev_err(hsotg->dev, "  Device address: %d\n",
1493                 dwc2_hcd_get_dev_addr(&urb->pipe_info));
1494         dev_err(hsotg->dev, "  Endpoint: %d, %s\n",
1495                 dwc2_hcd_get_ep_num(&urb->pipe_info),
1496                 dwc2_hcd_is_pipe_in(&urb->pipe_info) ? "IN" : "OUT");
1497
1498         switch (dwc2_hcd_get_pipe_type(&urb->pipe_info)) {
1499         case USB_ENDPOINT_XFER_CONTROL:
1500                 pipetype = "CONTROL";
1501                 break;
1502         case USB_ENDPOINT_XFER_BULK:
1503                 pipetype = "BULK";
1504                 break;
1505         case USB_ENDPOINT_XFER_INT:
1506                 pipetype = "INTERRUPT";
1507                 break;
1508         case USB_ENDPOINT_XFER_ISOC:
1509                 pipetype = "ISOCHRONOUS";
1510                 break;
1511         default:
1512                 pipetype = "UNKNOWN";
1513                 break;
1514         }
1515
1516         dev_err(hsotg->dev, "  Endpoint type: %s\n", pipetype);
1517
1518         switch (chan->speed) {
1519         case USB_SPEED_HIGH:
1520                 speed = "HIGH";
1521                 break;
1522         case USB_SPEED_FULL:
1523                 speed = "FULL";
1524                 break;
1525         case USB_SPEED_LOW:
1526                 speed = "LOW";
1527                 break;
1528         default:
1529                 speed = "UNKNOWN";
1530                 break;
1531         }
1532
1533         dev_err(hsotg->dev, "  Speed: %s\n", speed);
1534
1535         dev_err(hsotg->dev, "  Max packet size: %d\n",
1536                 dwc2_hcd_get_mps(&urb->pipe_info));
1537         dev_err(hsotg->dev, "  Data buffer length: %d\n", urb->length);
1538         dev_err(hsotg->dev, "  Transfer buffer: %p, Transfer DMA: %p\n",
1539                 urb->buf, (void *)urb->dma);
1540         dev_err(hsotg->dev, "  Setup buffer: %p, Setup DMA: %p\n",
1541                 urb->setup_packet, (void *)urb->setup_dma);
1542         dev_err(hsotg->dev, "  Interval: %d\n", urb->interval);
1543
1544         /* Core halts the channel for Descriptor DMA mode */
1545         if (hsotg->core_params->dma_desc_enable > 0) {
1546                 dwc2_hcd_complete_xfer_ddma(hsotg, chan, chnum,
1547                                             DWC2_HC_XFER_AHB_ERR);
1548                 goto handle_ahberr_done;
1549         }
1550
1551         dwc2_host_complete(hsotg, urb->priv, urb, -EIO);
1552
1553 handle_ahberr_halt:
1554         /*
1555          * Force a channel halt. Don't call dwc2_halt_channel because that won't
1556          * write to the HCCHARn register in DMA mode to force the halt.
1557          */
1558         dwc2_hc_halt(hsotg, chan, DWC2_HC_XFER_AHB_ERR);
1559
1560 handle_ahberr_done:
1561         dwc2_hc_handle_tt_clear(hsotg, chan, qtd);
1562         disable_hc_int(hsotg, chnum, HCINTMSK_AHBERR);
1563 }
1564
1565 /*
1566  * Handles a host channel transaction error interrupt. This handler may be
1567  * called in either DMA mode or Slave mode.
1568  */
1569 static void dwc2_hc_xacterr_intr(struct dwc2_hsotg *hsotg,
1570                                  struct dwc2_host_chan *chan, int chnum,
1571                                  struct dwc2_qtd *qtd)
1572 {
1573         dev_dbg(hsotg->dev,
1574                 "--Host Channel %d Interrupt: Transaction Error--\n", chnum);
1575
1576         if (hsotg->core_params->dma_desc_enable > 0) {
1577                 dwc2_hcd_complete_xfer_ddma(hsotg, chan, chnum,
1578                                             DWC2_HC_XFER_XACT_ERR);
1579                 goto handle_xacterr_done;
1580         }
1581
1582         switch (dwc2_hcd_get_pipe_type(&qtd->urb->pipe_info)) {
1583         case USB_ENDPOINT_XFER_CONTROL:
1584         case USB_ENDPOINT_XFER_BULK:
1585                 qtd->error_count++;
1586                 if (!chan->qh->ping_state) {
1587
1588                         dwc2_update_urb_state_abn(hsotg, chan, chnum, qtd->urb,
1589                                                   qtd, DWC2_HC_XFER_XACT_ERR);
1590                         dwc2_hcd_save_data_toggle(hsotg, chan, chnum, qtd);
1591                         if (!chan->ep_is_in && chan->speed == USB_SPEED_HIGH)
1592                                 chan->qh->ping_state = 1;
1593                 }
1594
1595                 /*
1596                  * Halt the channel so the transfer can be re-started from
1597                  * the appropriate point or the PING protocol will start
1598                  */
1599                 dwc2_halt_channel(hsotg, chan, qtd, DWC2_HC_XFER_XACT_ERR);
1600                 break;
1601         case USB_ENDPOINT_XFER_INT:
1602                 qtd->error_count++;
1603                 if (chan->do_split && chan->complete_split)
1604                         qtd->complete_split = 0;
1605                 dwc2_halt_channel(hsotg, chan, qtd, DWC2_HC_XFER_XACT_ERR);
1606                 break;
1607         case USB_ENDPOINT_XFER_ISOC:
1608                 {
1609                         enum dwc2_halt_status halt_status;
1610
1611                         halt_status = dwc2_update_isoc_urb_state(hsotg, chan,
1612                                         chnum, qtd, DWC2_HC_XFER_XACT_ERR);
1613                         dwc2_halt_channel(hsotg, chan, qtd, halt_status);
1614                 }
1615                 break;
1616         }
1617
1618 handle_xacterr_done:
1619         dwc2_hc_handle_tt_clear(hsotg, chan, qtd);
1620         disable_hc_int(hsotg, chnum, HCINTMSK_XACTERR);
1621 }
1622
1623 /*
1624  * Handles a host channel frame overrun interrupt. This handler may be called
1625  * in either DMA mode or Slave mode.
1626  */
1627 static void dwc2_hc_frmovrun_intr(struct dwc2_hsotg *hsotg,
1628                                   struct dwc2_host_chan *chan, int chnum,
1629                                   struct dwc2_qtd *qtd)
1630 {
1631         enum dwc2_halt_status halt_status;
1632
1633         dev_dbg(hsotg->dev, "--Host Channel %d Interrupt: Frame Overrun--\n",
1634                 chnum);
1635
1636         switch (dwc2_hcd_get_pipe_type(&qtd->urb->pipe_info)) {
1637         case USB_ENDPOINT_XFER_CONTROL:
1638         case USB_ENDPOINT_XFER_BULK:
1639                 break;
1640         case USB_ENDPOINT_XFER_INT:
1641                 dwc2_halt_channel(hsotg, chan, qtd, DWC2_HC_XFER_FRAME_OVERRUN);
1642                 break;
1643         case USB_ENDPOINT_XFER_ISOC:
1644                 halt_status = dwc2_update_isoc_urb_state(hsotg, chan, chnum,
1645                                         qtd, DWC2_HC_XFER_FRAME_OVERRUN);
1646                 dwc2_halt_channel(hsotg, chan, qtd, halt_status);
1647                 break;
1648         }
1649
1650         dwc2_hc_handle_tt_clear(hsotg, chan, qtd);
1651         disable_hc_int(hsotg, chnum, HCINTMSK_FRMOVRUN);
1652 }
1653
1654 /*
1655  * Handles a host channel data toggle error interrupt. This handler may be
1656  * called in either DMA mode or Slave mode.
1657  */
1658 static void dwc2_hc_datatglerr_intr(struct dwc2_hsotg *hsotg,
1659                                     struct dwc2_host_chan *chan, int chnum,
1660                                     struct dwc2_qtd *qtd)
1661 {
1662         dev_dbg(hsotg->dev,
1663                 "--Host Channel %d Interrupt: Data Toggle Error--\n", chnum);
1664
1665         if (chan->ep_is_in)
1666                 qtd->error_count = 0;
1667         else
1668                 dev_err(hsotg->dev,
1669                         "Data Toggle Error on OUT transfer, channel %d\n",
1670                         chnum);
1671
1672         dwc2_hc_handle_tt_clear(hsotg, chan, qtd);
1673         disable_hc_int(hsotg, chnum, HCINTMSK_DATATGLERR);
1674 }
1675
1676 /*
1677  * For debug only. It checks that a valid halt status is set and that
1678  * HCCHARn.chdis is clear. If there's a problem, corrective action is
1679  * taken and a warning is issued.
1680  *
1681  * Return: true if halt status is ok, false otherwise
1682  */
1683 static bool dwc2_halt_status_ok(struct dwc2_hsotg *hsotg,
1684                                 struct dwc2_host_chan *chan, int chnum,
1685                                 struct dwc2_qtd *qtd)
1686 {
1687 #ifdef DEBUG
1688         u32 hcchar;
1689         u32 hctsiz;
1690         u32 hcintmsk;
1691         u32 hcsplt;
1692
1693         if (chan->halt_status == DWC2_HC_XFER_NO_HALT_STATUS) {
1694                 /*
1695                  * This code is here only as a check. This condition should
1696                  * never happen. Ignore the halt if it does occur.
1697                  */
1698                 hcchar = readl(hsotg->regs + HCCHAR(chnum));
1699                 hctsiz = readl(hsotg->regs + HCTSIZ(chnum));
1700                 hcintmsk = readl(hsotg->regs + HCINTMSK(chnum));
1701                 hcsplt = readl(hsotg->regs + HCSPLT(chnum));
1702                 dev_dbg(hsotg->dev,
1703                         "%s: chan->halt_status DWC2_HC_XFER_NO_HALT_STATUS,\n",
1704                          __func__);
1705                 dev_dbg(hsotg->dev,
1706                         "channel %d, hcchar 0x%08x, hctsiz 0x%08x,\n",
1707                         chnum, hcchar, hctsiz);
1708                 dev_dbg(hsotg->dev,
1709                         "hcint 0x%08x, hcintmsk 0x%08x, hcsplt 0x%08x,\n",
1710                         chan->hcint, hcintmsk, hcsplt);
1711                 dev_dbg(hsotg->dev, "qtd->complete_split %d\n",
1712                         qtd->complete_split);
1713                 dev_warn(hsotg->dev,
1714                          "%s: no halt status, channel %d, ignoring interrupt\n",
1715                          __func__, chnum);
1716                 return false;
1717         }
1718
1719         /*
1720          * This code is here only as a check. hcchar.chdis should never be set
1721          * when the halt interrupt occurs. Halt the channel again if it does
1722          * occur.
1723          */
1724         hcchar = readl(hsotg->regs + HCCHAR(chnum));
1725         if (hcchar & HCCHAR_CHDIS) {
1726                 dev_warn(hsotg->dev,
1727                          "%s: hcchar.chdis set unexpectedly, hcchar 0x%08x, trying to halt again\n",
1728                          __func__, hcchar);
1729                 chan->halt_pending = 0;
1730                 dwc2_halt_channel(hsotg, chan, qtd, chan->halt_status);
1731                 return false;
1732         }
1733 #endif
1734
1735         return true;
1736 }
1737
1738 /*
1739  * Handles a host Channel Halted interrupt in DMA mode. This handler
1740  * determines the reason the channel halted and proceeds accordingly.
1741  */
1742 static void dwc2_hc_chhltd_intr_dma(struct dwc2_hsotg *hsotg,
1743                                     struct dwc2_host_chan *chan, int chnum,
1744                                     struct dwc2_qtd *qtd)
1745 {
1746         u32 hcintmsk;
1747         int out_nak_enh = 0;
1748
1749         dev_vdbg(hsotg->dev,
1750                  "--Host Channel %d Interrupt: DMA Channel Halted--\n", chnum);
1751
1752         /*
1753          * For core with OUT NAK enhancement, the flow for high-speed
1754          * CONTROL/BULK OUT is handled a little differently
1755          */
1756         if (hsotg->snpsid >= DWC2_CORE_REV_2_71a) {
1757                 if (chan->speed == USB_SPEED_HIGH && !chan->ep_is_in &&
1758                     (chan->ep_type == USB_ENDPOINT_XFER_CONTROL ||
1759                      chan->ep_type == USB_ENDPOINT_XFER_BULK)) {
1760                         out_nak_enh = 1;
1761                 }
1762         }
1763
1764         if (chan->halt_status == DWC2_HC_XFER_URB_DEQUEUE ||
1765             (chan->halt_status == DWC2_HC_XFER_AHB_ERR &&
1766              hsotg->core_params->dma_desc_enable <= 0)) {
1767                 if (hsotg->core_params->dma_desc_enable > 0)
1768                         dwc2_hcd_complete_xfer_ddma(hsotg, chan, chnum,
1769                                                     chan->halt_status);
1770                 else
1771                         /*
1772                          * Just release the channel. A dequeue can happen on a
1773                          * transfer timeout. In the case of an AHB Error, the
1774                          * channel was forced to halt because there's no way to
1775                          * gracefully recover.
1776                          */
1777                         dwc2_release_channel(hsotg, chan, qtd,
1778                                              chan->halt_status);
1779                 return;
1780         }
1781
1782         hcintmsk = readl(hsotg->regs + HCINTMSK(chnum));
1783
1784         if (chan->hcint & HCINTMSK_XFERCOMPL) {
1785                 /*
1786                  * Todo: This is here because of a possible hardware bug. Spec
1787                  * says that on SPLIT-ISOC OUT transfers in DMA mode that a HALT
1788                  * interrupt w/ACK bit set should occur, but I only see the
1789                  * XFERCOMP bit, even with it masked out. This is a workaround
1790                  * for that behavior. Should fix this when hardware is fixed.
1791                  */
1792                 if (chan->ep_type == USB_ENDPOINT_XFER_ISOC && !chan->ep_is_in)
1793                         dwc2_hc_ack_intr(hsotg, chan, chnum, qtd);
1794                 dwc2_hc_xfercomp_intr(hsotg, chan, chnum, qtd);
1795         } else if (chan->hcint & HCINTMSK_STALL) {
1796                 dwc2_hc_stall_intr(hsotg, chan, chnum, qtd);
1797         } else if ((chan->hcint & HCINTMSK_XACTERR) &&
1798                    hsotg->core_params->dma_desc_enable <= 0) {
1799                 if (out_nak_enh) {
1800                         if (chan->hcint &
1801                             (HCINTMSK_NYET | HCINTMSK_NAK | HCINTMSK_ACK)) {
1802                                 dev_vdbg(hsotg->dev,
1803                                          "XactErr with NYET/NAK/ACK\n");
1804                                 qtd->error_count = 0;
1805                         } else {
1806                                 dev_vdbg(hsotg->dev,
1807                                          "XactErr without NYET/NAK/ACK\n");
1808                         }
1809                 }
1810
1811                 /*
1812                  * Must handle xacterr before nak or ack. Could get a xacterr
1813                  * at the same time as either of these on a BULK/CONTROL OUT
1814                  * that started with a PING. The xacterr takes precedence.
1815                  */
1816                 dwc2_hc_xacterr_intr(hsotg, chan, chnum, qtd);
1817         } else if ((chan->hcint & HCINTMSK_XCS_XACT) &&
1818                    hsotg->core_params->dma_desc_enable > 0) {
1819                 dwc2_hc_xacterr_intr(hsotg, chan, chnum, qtd);
1820         } else if ((chan->hcint & HCINTMSK_AHBERR) &&
1821                    hsotg->core_params->dma_desc_enable > 0) {
1822                 dwc2_hc_ahberr_intr(hsotg, chan, chnum, qtd);
1823         } else if (chan->hcint & HCINTMSK_BBLERR) {
1824                 dwc2_hc_babble_intr(hsotg, chan, chnum, qtd);
1825         } else if (chan->hcint & HCINTMSK_FRMOVRUN) {
1826                 dwc2_hc_frmovrun_intr(hsotg, chan, chnum, qtd);
1827         } else if (!out_nak_enh) {
1828                 if (chan->hcint & HCINTMSK_NYET) {
1829                         /*
1830                          * Must handle nyet before nak or ack. Could get a nyet
1831                          * at the same time as either of those on a BULK/CONTROL
1832                          * OUT that started with a PING. The nyet takes
1833                          * precedence.
1834                          */
1835                         dwc2_hc_nyet_intr(hsotg, chan, chnum, qtd);
1836                 } else if ((chan->hcint & HCINTMSK_NAK) &&
1837                            !(hcintmsk & HCINTMSK_NAK)) {
1838                         /*
1839                          * If nak is not masked, it's because a non-split IN
1840                          * transfer is in an error state. In that case, the nak
1841                          * is handled by the nak interrupt handler, not here.
1842                          * Handle nak here for BULK/CONTROL OUT transfers, which
1843                          * halt on a NAK to allow rewinding the buffer pointer.
1844                          */
1845                         dwc2_hc_nak_intr(hsotg, chan, chnum, qtd);
1846                 } else if ((chan->hcint & HCINTMSK_ACK) &&
1847                            !(hcintmsk & HCINTMSK_ACK)) {
1848                         /*
1849                          * If ack is not masked, it's because a non-split IN
1850                          * transfer is in an error state. In that case, the ack
1851                          * is handled by the ack interrupt handler, not here.
1852                          * Handle ack here for split transfers. Start splits
1853                          * halt on ACK.
1854                          */
1855                         dwc2_hc_ack_intr(hsotg, chan, chnum, qtd);
1856                 } else {
1857                         if (chan->ep_type == USB_ENDPOINT_XFER_INT ||
1858                             chan->ep_type == USB_ENDPOINT_XFER_ISOC) {
1859                                 /*
1860                                  * A periodic transfer halted with no other
1861                                  * channel interrupts set. Assume it was halted
1862                                  * by the core because it could not be completed
1863                                  * in its scheduled (micro)frame.
1864                                  */
1865                                 dev_dbg(hsotg->dev,
1866                                         "%s: Halt channel %d (assume incomplete periodic transfer)\n",
1867                                         __func__, chnum);
1868                                 dwc2_halt_channel(hsotg, chan, qtd,
1869                                         DWC2_HC_XFER_PERIODIC_INCOMPLETE);
1870                         } else {
1871                                 dev_err(hsotg->dev,
1872                                         "%s: Channel %d - ChHltd set, but reason is unknown\n",
1873                                         __func__, chnum);
1874                                 dev_err(hsotg->dev,
1875                                         "hcint 0x%08x, intsts 0x%08x\n",
1876                                         chan->hcint,
1877                                         readl(hsotg->regs + GINTSTS));
1878                         }
1879                 }
1880         } else {
1881                 dev_info(hsotg->dev,
1882                          "NYET/NAK/ACK/other in non-error case, 0x%08x\n",
1883                          chan->hcint);
1884         }
1885 }
1886
1887 /*
1888  * Handles a host channel Channel Halted interrupt
1889  *
1890  * In slave mode, this handler is called only when the driver specifically
1891  * requests a halt. This occurs during handling other host channel interrupts
1892  * (e.g. nak, xacterr, stall, nyet, etc.).
1893  *
1894  * In DMA mode, this is the interrupt that occurs when the core has finished
1895  * processing a transfer on a channel. Other host channel interrupts (except
1896  * ahberr) are disabled in DMA mode.
1897  */
1898 static void dwc2_hc_chhltd_intr(struct dwc2_hsotg *hsotg,
1899                                 struct dwc2_host_chan *chan, int chnum,
1900                                 struct dwc2_qtd *qtd)
1901 {
1902         dev_vdbg(hsotg->dev, "--Host Channel %d Interrupt: Channel Halted--\n",
1903                  chnum);
1904
1905         if (hsotg->core_params->dma_enable > 0) {
1906                 dwc2_hc_chhltd_intr_dma(hsotg, chan, chnum, qtd);
1907         } else {
1908                 if (!dwc2_halt_status_ok(hsotg, chan, chnum, qtd))
1909                         return;
1910                 dwc2_release_channel(hsotg, chan, qtd, chan->halt_status);
1911         }
1912 }
1913
1914 /* Handles interrupt for a specific Host Channel */
1915 static void dwc2_hc_n_intr(struct dwc2_hsotg *hsotg, int chnum)
1916 {
1917         struct dwc2_qtd *qtd;
1918         struct dwc2_host_chan *chan;
1919         u32 hcint, hcintmsk;
1920
1921         dev_vdbg(hsotg->dev, "--Host Channel Interrupt--, Channel %d\n", chnum);
1922
1923         hcint = readl(hsotg->regs + HCINT(chnum));
1924         hcintmsk = readl(hsotg->regs + HCINTMSK(chnum));
1925         dev_vdbg(hsotg->dev,
1926                  "  hcint 0x%08x, hcintmsk 0x%08x, hcint&hcintmsk 0x%08x\n",
1927                  hcint, hcintmsk, hcint & hcintmsk);
1928
1929         chan = hsotg->hc_ptr_array[chnum];
1930         if (!chan) {
1931                 dev_err(hsotg->dev, "## hc_ptr_array for channel is NULL ##\n");
1932                 writel(hcint, hsotg->regs + HCINT(chnum));
1933                 return;
1934         }
1935
1936         writel(hcint, hsotg->regs + HCINT(chnum));
1937         chan->hcint = hcint;
1938         hcint &= hcintmsk;
1939
1940         if (list_empty(&chan->qh->qtd_list)) {
1941                 dev_dbg(hsotg->dev, "## no QTD queued for channel %d ##\n",
1942                         chnum);
1943                 dev_dbg(hsotg->dev,
1944                         "  hcint 0x%08x, hcintmsk 0x%08x, hcint&hcintmsk 0x%08x\n",
1945                         chan->hcint, hcintmsk, hcint);
1946                 chan->halt_status = DWC2_HC_XFER_NO_HALT_STATUS;
1947                 disable_hc_int(hsotg, chnum, HCINTMSK_CHHLTD);
1948                 chan->hcint = 0;
1949                 return;
1950         }
1951
1952         qtd = list_first_entry(&chan->qh->qtd_list, struct dwc2_qtd,
1953                                qtd_list_entry);
1954
1955         if (hsotg->core_params->dma_enable <= 0) {
1956                 if ((hcint & HCINTMSK_CHHLTD) && hcint != HCINTMSK_CHHLTD)
1957                         hcint &= ~HCINTMSK_CHHLTD;
1958         }
1959
1960         if (hcint & HCINTMSK_XFERCOMPL) {
1961                 dwc2_hc_xfercomp_intr(hsotg, chan, chnum, qtd);
1962                 /*
1963                  * If NYET occurred at same time as Xfer Complete, the NYET is
1964                  * handled by the Xfer Complete interrupt handler. Don't want
1965                  * to call the NYET interrupt handler in this case.
1966                  */
1967                 hcint &= ~HCINTMSK_NYET;
1968         }
1969         if (hcint & HCINTMSK_CHHLTD)
1970                 dwc2_hc_chhltd_intr(hsotg, chan, chnum, qtd);
1971         if (hcint & HCINTMSK_AHBERR)
1972                 dwc2_hc_ahberr_intr(hsotg, chan, chnum, qtd);
1973         if (hcint & HCINTMSK_STALL)
1974                 dwc2_hc_stall_intr(hsotg, chan, chnum, qtd);
1975         if (hcint & HCINTMSK_NAK)
1976                 dwc2_hc_nak_intr(hsotg, chan, chnum, qtd);
1977         if (hcint & HCINTMSK_ACK)
1978                 dwc2_hc_ack_intr(hsotg, chan, chnum, qtd);
1979         if (hcint & HCINTMSK_NYET)
1980                 dwc2_hc_nyet_intr(hsotg, chan, chnum, qtd);
1981         if (hcint & HCINTMSK_XACTERR)
1982                 dwc2_hc_xacterr_intr(hsotg, chan, chnum, qtd);
1983         if (hcint & HCINTMSK_BBLERR)
1984                 dwc2_hc_babble_intr(hsotg, chan, chnum, qtd);
1985         if (hcint & HCINTMSK_FRMOVRUN)
1986                 dwc2_hc_frmovrun_intr(hsotg, chan, chnum, qtd);
1987         if (hcint & HCINTMSK_DATATGLERR)
1988                 dwc2_hc_datatglerr_intr(hsotg, chan, chnum, qtd);
1989
1990         chan->hcint = 0;
1991 }
1992
1993 /*
1994  * This interrupt indicates that one or more host channels has a pending
1995  * interrupt. There are multiple conditions that can cause each host channel
1996  * interrupt. This function determines which conditions have occurred for each
1997  * host channel interrupt and handles them appropriately.
1998  */
1999 static void dwc2_hc_intr(struct dwc2_hsotg *hsotg)
2000 {
2001         u32 haint;
2002         int i;
2003
2004         dev_vdbg(hsotg->dev, "%s()\n", __func__);
2005
2006         haint = readl(hsotg->regs + HAINT);
2007         dev_vdbg(hsotg->dev, "HAINT=%08x\n", haint);
2008
2009         for (i = 0; i < hsotg->core_params->host_channels; i++) {
2010                 if (haint & (1 << i))
2011                         dwc2_hc_n_intr(hsotg, i);
2012         }
2013 }
2014
2015 /* This function handles interrupts for the HCD */
2016 int dwc2_hcd_intr(struct dwc2_hsotg *hsotg)
2017 {
2018         u32 gintsts;
2019         int retval = 0;
2020
2021         if (dwc2_check_core_status(hsotg) < 0) {
2022                 dev_warn(hsotg->dev, "Controller is disconnected");
2023                 return 0;
2024         }
2025
2026         spin_lock(&hsotg->lock);
2027
2028         /* Check if HOST Mode */
2029         if (dwc2_is_host_mode(hsotg)) {
2030                 gintsts = dwc2_read_core_intr(hsotg);
2031                 if (!gintsts) {
2032                         spin_unlock(&hsotg->lock);
2033                         return 0;
2034                 }
2035
2036                 retval = 1;
2037
2038 #ifndef DEBUG_SOF
2039                 /* Don't print debug message in the interrupt handler on SOF */
2040                 if (gintsts != GINTSTS_SOF)
2041 #endif
2042                         dev_vdbg(hsotg->dev,
2043                                  "DWC OTG HCD Interrupt Detected gintsts&gintmsk=0x%08x\n",
2044                                  gintsts);
2045
2046                 if (gintsts & GINTSTS_SOF)
2047                         dwc2_sof_intr(hsotg);
2048                 if (gintsts & GINTSTS_RXFLVL)
2049                         dwc2_rx_fifo_level_intr(hsotg);
2050                 if (gintsts & GINTSTS_NPTXFEMP)
2051                         dwc2_np_tx_fifo_empty_intr(hsotg);
2052                 if (gintsts & GINTSTS_I2CINT)
2053                         /* Todo: Implement i2cintr handler */
2054                         writel(GINTSTS_I2CINT, hsotg->regs + GINTSTS);
2055                 if (gintsts & GINTSTS_PRTINT)
2056                         dwc2_port_intr(hsotg);
2057                 if (gintsts & GINTSTS_HCHINT)
2058                         dwc2_hc_intr(hsotg);
2059                 if (gintsts & GINTSTS_PTXFEMP)
2060                         dwc2_perio_tx_fifo_empty_intr(hsotg);
2061
2062 #ifndef DEBUG_SOF
2063                 if (gintsts != GINTSTS_SOF) {
2064 #endif
2065                         dev_vdbg(hsotg->dev,
2066                                  "DWC OTG HCD Finished Servicing Interrupts\n");
2067                         dev_vdbg(hsotg->dev,
2068                                  "DWC OTG HCD gintsts=0x%08x gintmsk=0x%08x\n",
2069                                  readl(hsotg->regs + GINTSTS),
2070                                  readl(hsotg->regs + GINTMSK));
2071 #ifndef DEBUG_SOF
2072                 }
2073 #endif
2074         }
2075
2076         spin_unlock(&hsotg->lock);
2077
2078         return retval;
2079 }