drivers/net/ethernet/intel/i40e/i40e_txrx.c

   1 /*******************************************************************************
   2  *
   3  * Intel Ethernet Controller XL710 Family Linux Driver
   4  * Copyright(c) 2013 - 2014 Intel Corporation.
   5  *
   6  * This program is free software; you can redistribute it and/or modify it
   7  * under the terms and conditions of the GNU General Public License,
   8  * version 2, as published by the Free Software Foundation.
   9  *
  10  * This program is distributed in the hope it will be useful, but WITHOUT
  11  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  12  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  13  * more details.
  14  *
  15  * You should have received a copy of the GNU General Public License along
  16  * with this program.  If not, see <http://www.gnu.org/licenses/>.
  17  *
  18  * The full GNU General Public License is included in this distribution in
  19  * the file called "COPYING".
  20  *
  21  * Contact Information:
  22  * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
  23  * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
  24  *
  25  ******************************************************************************/
  26
  27 #include <linux/prefetch.h>
  28 #include "i40e.h"
  29 #include "i40e_prototype.h"
  30
  31 static inline __le64 build_ctob(u32 td_cmd, u32 td_offset, unsigned int size,
  32                                 u32 td_tag)
  33 {
  34         return cpu_to_le64(I40E_TX_DESC_DTYPE_DATA |
  35                            ((u64)td_cmd  << I40E_TXD_QW1_CMD_SHIFT) |
  36                            ((u64)td_offset << I40E_TXD_QW1_OFFSET_SHIFT) |
  37                            ((u64)size  << I40E_TXD_QW1_TX_BUF_SZ_SHIFT) |
  38                            ((u64)td_tag  << I40E_TXD_QW1_L2TAG1_SHIFT));
  39 }
  40
  41 #define I40E_TXD_CMD (I40E_TX_DESC_CMD_EOP | I40E_TX_DESC_CMD_RS)
  42 /**
  43  * i40e_program_fdir_filter - Program a Flow Director filter
  44  * @fdir_data: Packet data that will be filter parameters
  45  * @raw_packet: the pre-allocated packet buffer for FDir
  46  * @pf: The pf pointer
  47  * @add: True for add/update, False for remove
  48  **/
  49 int i40e_program_fdir_filter(struct i40e_fdir_filter *fdir_data, u8 *raw_packet,
  50                              struct i40e_pf *pf, bool add)
  51 {
  52         struct i40e_filter_program_desc *fdir_desc;
  53         struct i40e_tx_buffer *tx_buf;
  54         struct i40e_tx_desc *tx_desc;
  55         struct i40e_ring *tx_ring;
  56         unsigned int fpt, dcc;
  57         struct i40e_vsi *vsi;
  58         struct device *dev;
  59         dma_addr_t dma;
  60         u32 td_cmd = 0;
  61         u16 i;
  62
  63         /* find existing FDIR VSI */
  64         vsi = NULL;
  65         for (i = 0; i < pf->hw.func_caps.num_vsis; i++)
  66                 if (pf->vsi[i] && pf->vsi[i]->type == I40E_VSI_FDIR)
  67                         vsi = pf->vsi[i];
  68         if (!vsi)
  69                 return -ENOENT;
  70
  71         tx_ring = vsi->tx_rings[0];
  72         dev = tx_ring->dev;
  73
  74         dma = dma_map_single(dev, raw_packet,
  75                              I40E_FDIR_MAX_RAW_PACKET_SIZE, DMA_TO_DEVICE);
  76         if (dma_mapping_error(dev, dma))
  77                 goto dma_fail;
  78
  79         /* grab the next descriptor */
  80         i = tx_ring->next_to_use;
  81         fdir_desc = I40E_TX_FDIRDESC(tx_ring, i);
  82
  83         tx_ring->next_to_use = (i + 1 < tx_ring->count) ? i + 1 : 0;
  84
  85         fpt = (fdir_data->q_index << I40E_TXD_FLTR_QW0_QINDEX_SHIFT) &
  86               I40E_TXD_FLTR_QW0_QINDEX_MASK;
  87
  88         fpt |= (fdir_data->flex_off << I40E_TXD_FLTR_QW0_FLEXOFF_SHIFT) &
  89                I40E_TXD_FLTR_QW0_FLEXOFF_MASK;
  90
  91         fpt |= (fdir_data->pctype << I40E_TXD_FLTR_QW0_PCTYPE_SHIFT) &
  92                I40E_TXD_FLTR_QW0_PCTYPE_MASK;
  93
  94         /* Use LAN VSI Id if not programmed by user */
  95         if (fdir_data->dest_vsi == 0)
  96                 fpt |= (pf->vsi[pf->lan_vsi]->id) <<
  97                        I40E_TXD_FLTR_QW0_DEST_VSI_SHIFT;
  98         else
  99                 fpt |= ((u32)fdir_data->dest_vsi <<
 100                         I40E_TXD_FLTR_QW0_DEST_VSI_SHIFT) &
 101                        I40E_TXD_FLTR_QW0_DEST_VSI_MASK;
 102
 103         fdir_desc->qindex_flex_ptype_vsi = cpu_to_le32(fpt);
 104
 105         dcc = I40E_TX_DESC_DTYPE_FILTER_PROG;
 106
 107         if (add)
 108                 dcc |= I40E_FILTER_PROGRAM_DESC_PCMD_ADD_UPDATE <<
 109                        I40E_TXD_FLTR_QW1_PCMD_SHIFT;
 110         else
 111                 dcc |= I40E_FILTER_PROGRAM_DESC_PCMD_REMOVE <<
 112                        I40E_TXD_FLTR_QW1_PCMD_SHIFT;
 113
 114         dcc |= (fdir_data->dest_ctl << I40E_TXD_FLTR_QW1_DEST_SHIFT) &
 115                I40E_TXD_FLTR_QW1_DEST_MASK;
 116
 117         dcc |= (fdir_data->fd_status << I40E_TXD_FLTR_QW1_FD_STATUS_SHIFT) &
 118                I40E_TXD_FLTR_QW1_FD_STATUS_MASK;
 119
 120         if (fdir_data->cnt_index != 0) {
 121                 dcc |= I40E_TXD_FLTR_QW1_CNT_ENA_MASK;
 122                 dcc |= ((u32)fdir_data->cnt_index <<
 123                         I40E_TXD_FLTR_QW1_CNTINDEX_SHIFT) &
 124                        I40E_TXD_FLTR_QW1_CNTINDEX_MASK;
 125         }
 126
 127         fdir_desc->dtype_cmd_cntindex = cpu_to_le32(dcc);
 128         fdir_desc->fd_id = cpu_to_le32(fdir_data->fd_id);
 129
 130         /* Now program a dummy descriptor */
 131         i = tx_ring->next_to_use;
 132         tx_desc = I40E_TX_DESC(tx_ring, i);
 133         tx_buf = &tx_ring->tx_bi[i];
 134
 135         tx_ring->next_to_use = (i + 1 < tx_ring->count) ? i + 1 : 0;
 136
 137         /* record length, and DMA address */
 138         dma_unmap_len_set(tx_buf, len, I40E_FDIR_MAX_RAW_PACKET_SIZE);
 139         dma_unmap_addr_set(tx_buf, dma, dma);
 140
 141         tx_desc->buffer_addr = cpu_to_le64(dma);
 142         td_cmd = I40E_TXD_CMD | I40E_TX_DESC_CMD_DUMMY;
 143
 144         tx_desc->cmd_type_offset_bsz =
 145                 build_ctob(td_cmd, 0, I40E_FDIR_MAX_RAW_PACKET_SIZE, 0);
 146
 147         /* set the timestamp */
 148         tx_buf->time_stamp = jiffies;
 149
 150         /* Force memory writes to complete before letting h/w
 151          * know there are new descriptors to fetch.  (Only
 152          * applicable for weak-ordered memory model archs,
 153          * such as IA-64).
 154          */
 155         wmb();
 156
 157         /* Mark the data descriptor to be watched */
 158         tx_buf->next_to_watch = tx_desc;
 159
 160         writel(tx_ring->next_to_use, tx_ring->tail);
 161         return 0;
 162
 163 dma_fail:
 164         return -1;
 165 }
 166
 167 #define IP_HEADER_OFFSET 14
 168 #define I40E_UDPIP_DUMMY_PACKET_LEN 42
 169 /**
 170  * i40e_add_del_fdir_udpv4 - Add/Remove UDPv4 filters
 171  * @vsi: pointer to the targeted VSI
 172  * @fd_data: the flow director data required for the FDir descriptor
 173  * @raw_packet: the pre-allocated packet buffer for FDir
 174  * @add: true adds a filter, false removes it
 175  *
 176  * Returns 0 if the filters were successfully added or removed
 177  **/
 178 static int i40e_add_del_fdir_udpv4(struct i40e_vsi *vsi,
 179                                    struct i40e_fdir_filter *fd_data,
 180                                    u8 *raw_packet, bool add)
 181 {
 182         struct i40e_pf *pf = vsi->back;
 183         struct udphdr *udp;
 184         struct iphdr *ip;
 185         bool err = false;
 186         int ret;
 187         static char packet[] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x08, 0,
 188                 0x45, 0, 0, 0x1c, 0, 0, 0x40, 0, 0x40, 0x11, 0, 0, 0, 0, 0, 0,
 189                 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
 190
 191         memcpy(raw_packet, packet, I40E_UDPIP_DUMMY_PACKET_LEN);
 192
 193         ip = (struct iphdr *)(raw_packet + IP_HEADER_OFFSET);
 194         udp = (struct udphdr *)(raw_packet + IP_HEADER_OFFSET
 195               + sizeof(struct iphdr));
 196
 197         ip->daddr = fd_data->dst_ip[0];
 198         udp->dest = fd_data->dst_port;
 199         ip->saddr = fd_data->src_ip[0];
 200         udp->source = fd_data->src_port;
 201
 202         fd_data->pctype = I40E_FILTER_PCTYPE_NONF_IPV4_UDP;
 203         ret = i40e_program_fdir_filter(fd_data, raw_packet, pf, add);
 204         if (ret) {
 205                 dev_info(&pf->pdev->dev,
 206                          "Filter command send failed for PCTYPE %d (ret = %d)\n",
 207                          fd_data->pctype, ret);
 208                 err = true;
 209         } else {
 210                 dev_info(&pf->pdev->dev,
 211                          "Filter OK for PCTYPE %d (ret = %d)\n",
 212                          fd_data->pctype, ret);
 213         }
 214
 215         return err ? -EOPNOTSUPP : 0;
 216 }
 217
 218 #define I40E_TCPIP_DUMMY_PACKET_LEN 54
 219 /**
 220  * i40e_add_del_fdir_tcpv4 - Add/Remove TCPv4 filters
 221  * @vsi: pointer to the targeted VSI
 222  * @fd_data: the flow director data required for the FDir descriptor
 223  * @raw_packet: the pre-allocated packet buffer for FDir
 224  * @add: true adds a filter, false removes it
 225  *
 226  * Returns 0 if the filters were successfully added or removed
 227  **/
 228 static int i40e_add_del_fdir_tcpv4(struct i40e_vsi *vsi,
 229                                    struct i40e_fdir_filter *fd_data,
 230                                    u8 *raw_packet, bool add)
 231 {
 232         struct i40e_pf *pf = vsi->back;
 233         struct tcphdr *tcp;
 234         struct iphdr *ip;
 235         bool err = false;
 236         int ret;
 237         /* Dummy packet */
 238         static char packet[] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x08, 0,
 239                 0x45, 0, 0, 0x28, 0, 0, 0x40, 0, 0x40, 0x6, 0, 0, 0, 0, 0, 0,
 240                 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x80, 0x11,
 241                 0x0, 0x72, 0, 0, 0, 0};
 242
 243         memcpy(raw_packet, packet, I40E_TCPIP_DUMMY_PACKET_LEN);
 244
 245         ip = (struct iphdr *)(raw_packet + IP_HEADER_OFFSET);
 246         tcp = (struct tcphdr *)(raw_packet + IP_HEADER_OFFSET
 247               + sizeof(struct iphdr));
 248
 249         ip->daddr = fd_data->dst_ip[0];
 250         tcp->dest = fd_data->dst_port;
 251         ip->saddr = fd_data->src_ip[0];
 252         tcp->source = fd_data->src_port;
 253
 254         if (add) {
 255                 if (pf->flags & I40E_FLAG_FD_ATR_ENABLED) {
 256                         dev_info(&pf->pdev->dev, "Forcing ATR off, sideband rules for TCP/IPv4 flow being applied\n");
 257                         pf->flags &= ~I40E_FLAG_FD_ATR_ENABLED;
 258                 }
 259         }
 260
 261         fd_data->pctype = I40E_FILTER_PCTYPE_NONF_IPV4_TCP;
 262         ret = i40e_program_fdir_filter(fd_data, raw_packet, pf, add);
 263
 264         if (ret) {
 265                 dev_info(&pf->pdev->dev,
 266                          "Filter command send failed for PCTYPE %d (ret = %d)\n",
 267                          fd_data->pctype, ret);
 268                 err = true;
 269         } else {
 270                 dev_info(&pf->pdev->dev, "Filter OK for PCTYPE %d (ret = %d)\n",
 271                          fd_data->pctype, ret);
 272         }
 273
 274         fd_data->pctype = I40E_FILTER_PCTYPE_NONF_IPV4_TCP;
 275
 276         ret = i40e_program_fdir_filter(fd_data, raw_packet, pf, add);
 277         if (ret) {
 278                 dev_info(&pf->pdev->dev,
 279                          "Filter command send failed for PCTYPE %d (ret = %d)\n",
 280                          fd_data->pctype, ret);
 281                 err = true;
 282         } else {
 283                 dev_info(&pf->pdev->dev, "Filter OK for PCTYPE %d (ret = %d)\n",
 284                           fd_data->pctype, ret);
 285         }
 286
 287         return err ? -EOPNOTSUPP : 0;
 288 }
 289
 290 /**
 291  * i40e_add_del_fdir_sctpv4 - Add/Remove SCTPv4 Flow Director filters for
 292  * a specific flow spec
 293  * @vsi: pointer to the targeted VSI
 294  * @fd_data: the flow director data required for the FDir descriptor
 295  * @raw_packet: the pre-allocated packet buffer for FDir
 296  * @add: true adds a filter, false removes it
 297  *
 298  * Always returns -EOPNOTSUPP
 299  **/
 300 static int i40e_add_del_fdir_sctpv4(struct i40e_vsi *vsi,
 301                                     struct i40e_fdir_filter *fd_data,
 302                                     u8 *raw_packet, bool add)
 303 {
 304         return -EOPNOTSUPP;
 305 }
 306
 307 #define I40E_IP_DUMMY_PACKET_LEN 34
 308 /**
 309  * i40e_add_del_fdir_ipv4 - Add/Remove IPv4 Flow Director filters for
 310  * a specific flow spec
 311  * @vsi: pointer to the targeted VSI
 312  * @fd_data: the flow director data required for the FDir descriptor
 313  * @raw_packet: the pre-allocated packet buffer for FDir
 314  * @add: true adds a filter, false removes it
 315  *
 316  * Returns 0 if the filters were successfully added or removed
 317  **/
 318 static int i40e_add_del_fdir_ipv4(struct i40e_vsi *vsi,
 319                                   struct i40e_fdir_filter *fd_data,
 320                                   u8 *raw_packet, bool add)
 321 {
 322         struct i40e_pf *pf = vsi->back;
 323         struct iphdr *ip;
 324         bool err = false;
 325         int ret;
 326         int i;
 327         static char packet[] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x08, 0,
 328                 0x45, 0, 0, 0x14, 0, 0, 0x40, 0, 0x40, 0x10, 0, 0, 0, 0, 0, 0,
 329                 0, 0, 0, 0};
 330
 331         memcpy(raw_packet, packet, I40E_IP_DUMMY_PACKET_LEN);
 332         ip = (struct iphdr *)(raw_packet + IP_HEADER_OFFSET);
 333
 334         ip->saddr = fd_data->src_ip[0];
 335         ip->daddr = fd_data->dst_ip[0];
 336         ip->protocol = 0;
 337
 338         for (i = I40E_FILTER_PCTYPE_NONF_IPV4_OTHER;
 339              i <= I40E_FILTER_PCTYPE_FRAG_IPV4; i++) {
 340                 fd_data->pctype = i;
 341                 ret = i40e_program_fdir_filter(fd_data, raw_packet, pf, add);
 342
 343                 if (ret) {
 344                         dev_info(&pf->pdev->dev,
 345                                  "Filter command send failed for PCTYPE %d (ret = %d)\n",
 346                                  fd_data->pctype, ret);
 347                         err = true;
 348                 } else {
 349                         dev_info(&pf->pdev->dev,
 350                                  "Filter OK for PCTYPE %d (ret = %d)\n",
 351                                  fd_data->pctype, ret);
 352                 }
 353         }
 354
 355         return err ? -EOPNOTSUPP : 0;
 356 }
 357
 358 /**
 359  * i40e_add_del_fdir - Build raw packets to add/del fdir filter
 360  * @vsi: pointer to the targeted VSI
 361  * @cmd: command to get or set RX flow classification rules
 362  * @add: true adds a filter, false removes it
 363  *
 364  **/
 365 int i40e_add_del_fdir(struct i40e_vsi *vsi,
 366                       struct i40e_fdir_filter *input, bool add)
 367 {
 368         struct i40e_pf *pf = vsi->back;
 369         u8 *raw_packet;
 370         int ret;
 371
 372         /* Populate the Flow Director that we have at the moment
 373          * and allocate the raw packet buffer for the calling functions
 374          */
 375         raw_packet = kzalloc(I40E_FDIR_MAX_RAW_PACKET_SIZE, GFP_KERNEL);
 376         if (!raw_packet)
 377                 return -ENOMEM;
 378
 379         switch (input->flow_type & ~FLOW_EXT) {
 380         case TCP_V4_FLOW:
 381                 ret = i40e_add_del_fdir_tcpv4(vsi, input, raw_packet,
 382                                               add);
 383                 break;
 384         case UDP_V4_FLOW:
 385                 ret = i40e_add_del_fdir_udpv4(vsi, input, raw_packet,
 386                                               add);
 387                 break;
 388         case SCTP_V4_FLOW:
 389                 ret = i40e_add_del_fdir_sctpv4(vsi, input, raw_packet,
 390                                                add);
 391                 break;
 392         case IPV4_FLOW:
 393                 ret = i40e_add_del_fdir_ipv4(vsi, input, raw_packet,
 394                                              add);
 395                 break;
 396         case IP_USER_FLOW:
 397                 switch (input->ip4_proto) {
 398                 case IPPROTO_TCP:
 399                         ret = i40e_add_del_fdir_tcpv4(vsi, input,
 400                                                       raw_packet, add);
 401                         break;
 402                 case IPPROTO_UDP:
 403                         ret = i40e_add_del_fdir_udpv4(vsi, input,
 404                                                       raw_packet, add);
 405                         break;
 406                 case IPPROTO_SCTP:
 407                         ret = i40e_add_del_fdir_sctpv4(vsi, input,
 408                                                        raw_packet, add);
 409                         break;
 410                 default:
 411                         ret = i40e_add_del_fdir_ipv4(vsi, input,
 412                                                      raw_packet, add);
 413                         break;
 414                 }
 415                 break;
 416         default:
 417                 dev_info(&pf->pdev->dev, "Could not specify spec type %d\n",
 418                          input->flow_type);
 419                 ret = -EINVAL;
 420         }
 421
 422         kfree(raw_packet);
 423         return ret;
 424 }
 425
 426 /**
 427  * i40e_fd_handle_status - check the Programming Status for FD
 428  * @rx_ring: the Rx ring for this descriptor
 429  * @rx_desc: the Rx descriptor for programming Status, not a packet descriptor.
 430  * @prog_id: the id originally used for programming
 431  *
 432  * This is used to verify if the FD programming or invalidation
 433  * requested by SW to the HW is successful or not and take actions accordingly.
 434  **/
 435 static void i40e_fd_handle_status(struct i40e_ring *rx_ring,
 436                                   union i40e_rx_desc *rx_desc, u8 prog_id)
 437 {
 438         struct i40e_pf *pf = rx_ring->vsi->back;
 439         struct pci_dev *pdev = pf->pdev;
 440         u32 fcnt_prog, fcnt_avail;
 441         u32 error;
 442         u64 qw;
 443
 444         qw = le64_to_cpu(rx_desc->wb.qword1.status_error_len);
 445         error = (qw & I40E_RX_PROG_STATUS_DESC_QW1_ERROR_MASK) >>
 446                 I40E_RX_PROG_STATUS_DESC_QW1_ERROR_SHIFT;
 447
 448         if (error == (0x1 << I40E_RX_PROG_STATUS_DESC_FD_TBL_FULL_SHIFT)) {
 449                 dev_warn(&pdev->dev, "ntuple filter loc = %d, could not be added\n",
 450                          rx_desc->wb.qword0.hi_dword.fd_id);
 451
 452                 /* filter programming failed most likely due to table full */
 453                 fcnt_prog = i40e_get_current_fd_count(pf);
 454                 fcnt_avail = pf->hw.fdir_shared_filter_count +
 455                                                        pf->fdir_pf_filter_count;
 456
 457                 /* If ATR is running fcnt_prog can quickly change,
 458                  * if we are very close to full, it makes sense to disable
 459                  * FD ATR/SB and then re-enable it when there is room.
 460                  */
 461                 if (fcnt_prog >= (fcnt_avail - I40E_FDIR_BUFFER_FULL_MARGIN)) {
 462                         /* Turn off ATR first */
 463                         if (pf->flags | I40E_FLAG_FD_ATR_ENABLED) {
 464                                 pf->flags &= ~I40E_FLAG_FD_ATR_ENABLED;
 465                                 dev_warn(&pdev->dev, "FD filter space full, ATR for further flows will be turned off\n");
 466                                 pf->auto_disable_flags |=
 467                                                        I40E_FLAG_FD_ATR_ENABLED;
 468                                 pf->flags |= I40E_FLAG_FDIR_REQUIRES_REINIT;
 469                         } else if (pf->flags | I40E_FLAG_FD_SB_ENABLED) {
 470                                 pf->flags &= ~I40E_FLAG_FD_SB_ENABLED;
 471                                 dev_warn(&pdev->dev, "FD filter space full, new ntuple rules will not be added\n");
 472                                 pf->auto_disable_flags |=
 473                                                         I40E_FLAG_FD_SB_ENABLED;
 474                                 pf->flags |= I40E_FLAG_FDIR_REQUIRES_REINIT;
 475                         }
 476                 } else {
 477                         dev_info(&pdev->dev, "FD filter programming error\n");
 478                 }
 479         } else if (error ==
 480                           (0x1 << I40E_RX_PROG_STATUS_DESC_NO_FD_ENTRY_SHIFT)) {
 481                 if (I40E_DEBUG_FD & pf->hw.debug_mask)
 482                         dev_info(&pdev->dev, "ntuple filter loc = %d, could not be removed\n",
 483                                  rx_desc->wb.qword0.hi_dword.fd_id);
 484         }
 485 }
 486
 487 /**
 488  * i40e_unmap_and_free_tx_resource - Release a Tx buffer
 489  * @ring:      the ring that owns the buffer
 490  * @tx_buffer: the buffer to free
 491  **/
 492 static void i40e_unmap_and_free_tx_resource(struct i40e_ring *ring,
 493                                             struct i40e_tx_buffer *tx_buffer)
 494 {
 495         if (tx_buffer->skb) {
 496                 dev_kfree_skb_any(tx_buffer->skb);
 497                 if (dma_unmap_len(tx_buffer, len))
 498                         dma_unmap_single(ring->dev,
 499                                          dma_unmap_addr(tx_buffer, dma),
 500                                          dma_unmap_len(tx_buffer, len),
 501                                          DMA_TO_DEVICE);
 502         } else if (dma_unmap_len(tx_buffer, len)) {
 503                 dma_unmap_page(ring->dev,
 504                                dma_unmap_addr(tx_buffer, dma),
 505                                dma_unmap_len(tx_buffer, len),
 506                                DMA_TO_DEVICE);
 507         }
 508         tx_buffer->next_to_watch = NULL;
 509         tx_buffer->skb = NULL;
 510         dma_unmap_len_set(tx_buffer, len, 0);
 511         /* tx_buffer must be completely set up in the transmit path */
 512 }
 513
 514 /**
 515  * i40e_clean_tx_ring - Free any empty Tx buffers
 516  * @tx_ring: ring to be cleaned
 517  **/
 518 void i40e_clean_tx_ring(struct i40e_ring *tx_ring)
 519 {
 520         unsigned long bi_size;
 521         u16 i;
 522
 523         /* ring already cleared, nothing to do */
 524         if (!tx_ring->tx_bi)
 525                 return;
 526
 527         /* Free all the Tx ring sk_buffs */
 528         for (i = 0; i < tx_ring->count; i++)
 529                 i40e_unmap_and_free_tx_resource(tx_ring, &tx_ring->tx_bi[i]);
 530
 531         bi_size = sizeof(struct i40e_tx_buffer) * tx_ring->count;
 532         memset(tx_ring->tx_bi, 0, bi_size);
 533
 534         /* Zero out the descriptor ring */
 535         memset(tx_ring->desc, 0, tx_ring->size);
 536
 537         tx_ring->next_to_use = 0;
 538         tx_ring->next_to_clean = 0;
 539
 540         if (!tx_ring->netdev)
 541                 return;
 542
 543         /* cleanup Tx queue statistics */
 544         netdev_tx_reset_queue(netdev_get_tx_queue(tx_ring->netdev,
 545                                                   tx_ring->queue_index));
 546 }
 547
 548 /**
 549  * i40e_free_tx_resources - Free Tx resources per queue
 550  * @tx_ring: Tx descriptor ring for a specific queue
 551  *
 552  * Free all transmit software resources
 553  **/
 554 void i40e_free_tx_resources(struct i40e_ring *tx_ring)
 555 {
 556         i40e_clean_tx_ring(tx_ring);
 557         kfree(tx_ring->tx_bi);
 558         tx_ring->tx_bi = NULL;
 559
 560         if (tx_ring->desc) {
 561                 dma_free_coherent(tx_ring->dev, tx_ring->size,
 562                                   tx_ring->desc, tx_ring->dma);
 563                 tx_ring->desc = NULL;
 564         }
 565 }
 566
 567 /**
 568  * i40e_get_tx_pending - how many tx descriptors not processed
 569  * @tx_ring: the ring of descriptors
 570  *
 571  * Since there is no access to the ring head register
 572  * in XL710, we need to use our local copies
 573  **/
 574 static u32 i40e_get_tx_pending(struct i40e_ring *ring)
 575 {
 576         u32 ntu = ((ring->next_to_clean <= ring->next_to_use)
 577                         ? ring->next_to_use
 578                         : ring->next_to_use + ring->count);
 579         return ntu - ring->next_to_clean;
 580 }
 581
 582 /**
 583  * i40e_check_tx_hang - Is there a hang in the Tx queue
 584  * @tx_ring: the ring of descriptors
 585  **/
 586 static bool i40e_check_tx_hang(struct i40e_ring *tx_ring)
 587 {
 588         u32 tx_pending = i40e_get_tx_pending(tx_ring);
 589         bool ret = false;
 590
 591         clear_check_for_tx_hang(tx_ring);
 592
 593         /* Check for a hung queue, but be thorough. This verifies
 594          * that a transmit has been completed since the previous
 595          * check AND there is at least one packet pending. The
 596          * ARMED bit is set to indicate a potential hang. The
 597          * bit is cleared if a pause frame is received to remove
 598          * false hang detection due to PFC or 802.3x frames. By
 599          * requiring this to fail twice we avoid races with
 600          * PFC clearing the ARMED bit and conditions where we
 601          * run the check_tx_hang logic with a transmit completion
 602          * pending but without time to complete it yet.
 603          */
 604         if ((tx_ring->tx_stats.tx_done_old == tx_ring->stats.packets) &&
 605             tx_pending) {
 606                 /* make sure it is true for two checks in a row */
 607                 ret = test_and_set_bit(__I40E_HANG_CHECK_ARMED,
 608                                        &tx_ring->state);
 609         } else {
 610                 /* update completed stats and disarm the hang check */
 611                 tx_ring->tx_stats.tx_done_old = tx_ring->stats.packets;
 612                 clear_bit(__I40E_HANG_CHECK_ARMED, &tx_ring->state);
 613         }
 614
 615         return ret;
 616 }
 617
 618 /**
 619  * i40e_get_head - Retrieve head from head writeback
 620  * @tx_ring:  tx ring to fetch head of
 621  *
 622  * Returns value of Tx ring head based on value stored
 623  * in head write-back location
 624  **/
 625 static inline u32 i40e_get_head(struct i40e_ring *tx_ring)
 626 {
 627         void *head = (struct i40e_tx_desc *)tx_ring->desc + tx_ring->count;
 628
 629         return le32_to_cpu(*(volatile __le32 *)head);
 630 }
 631
 632 /**
 633  * i40e_clean_tx_irq - Reclaim resources after transmit completes
 634  * @tx_ring:  tx ring to clean
 635  * @budget:   how many cleans we're allowed
 636  *
 637  * Returns true if there's any budget left (e.g. the clean is finished)
 638  **/
 639 static bool i40e_clean_tx_irq(struct i40e_ring *tx_ring, int budget)
 640 {
 641         u16 i = tx_ring->next_to_clean;
 642         struct i40e_tx_buffer *tx_buf;
 643         struct i40e_tx_desc *tx_head;
 644         struct i40e_tx_desc *tx_desc;
 645         unsigned int total_packets = 0;
 646         unsigned int total_bytes = 0;
 647
 648         tx_buf = &tx_ring->tx_bi[i];
 649         tx_desc = I40E_TX_DESC(tx_ring, i);
 650         i -= tx_ring->count;
 651
 652         tx_head = I40E_TX_DESC(tx_ring, i40e_get_head(tx_ring));
 653
 654         do {
 655                 struct i40e_tx_desc *eop_desc = tx_buf->next_to_watch;
 656
 657                 /* if next_to_watch is not set then there is no work pending */
 658                 if (!eop_desc)
 659                         break;
 660
 661                 /* prevent any other reads prior to eop_desc */
 662                 read_barrier_depends();
 663
 664                 /* we have caught up to head, no work left to do */
 665                 if (tx_head == tx_desc)
 666                         break;
 667
 668                 /* clear next_to_watch to prevent false hangs */
 669                 tx_buf->next_to_watch = NULL;
 670
 671                 /* update the statistics for this packet */
 672                 total_bytes += tx_buf->bytecount;
 673                 total_packets += tx_buf->gso_segs;
 674
 675                 /* free the skb */
 676                 dev_kfree_skb_any(tx_buf->skb);
 677
 678                 /* unmap skb header data */
 679                 dma_unmap_single(tx_ring->dev,
 680                                  dma_unmap_addr(tx_buf, dma),
 681                                  dma_unmap_len(tx_buf, len),
 682                                  DMA_TO_DEVICE);
 683
 684                 /* clear tx_buffer data */
 685                 tx_buf->skb = NULL;
 686                 dma_unmap_len_set(tx_buf, len, 0);
 687
 688                 /* unmap remaining buffers */
 689                 while (tx_desc != eop_desc) {
 690
 691                         tx_buf++;
 692                         tx_desc++;
 693                         i++;
 694                         if (unlikely(!i)) {
 695                                 i -= tx_ring->count;
 696                                 tx_buf = tx_ring->tx_bi;
 697                                 tx_desc = I40E_TX_DESC(tx_ring, 0);
 698                         }
 699
 700                         /* unmap any remaining paged data */
 701                         if (dma_unmap_len(tx_buf, len)) {
 702                                 dma_unmap_page(tx_ring->dev,
 703                                                dma_unmap_addr(tx_buf, dma),
 704                                                dma_unmap_len(tx_buf, len),
 705                                                DMA_TO_DEVICE);
 706                                 dma_unmap_len_set(tx_buf, len, 0);
 707                         }
 708                 }
 709
 710                 /* move us one more past the eop_desc for start of next pkt */
 711                 tx_buf++;
 712                 tx_desc++;
 713                 i++;
 714                 if (unlikely(!i)) {
 715                         i -= tx_ring->count;
 716                         tx_buf = tx_ring->tx_bi;
 717                         tx_desc = I40E_TX_DESC(tx_ring, 0);
 718                 }
 719
 720                 /* update budget accounting */
 721                 budget--;
 722         } while (likely(budget));
 723
 724         i += tx_ring->count;
 725         tx_ring->next_to_clean = i;
 726         u64_stats_update_begin(&tx_ring->syncp);
 727         tx_ring->stats.bytes += total_bytes;
 728         tx_ring->stats.packets += total_packets;
 729         u64_stats_update_end(&tx_ring->syncp);
 730         tx_ring->q_vector->tx.total_bytes += total_bytes;
 731         tx_ring->q_vector->tx.total_packets += total_packets;
 732
 733         if (check_for_tx_hang(tx_ring) && i40e_check_tx_hang(tx_ring)) {
 734                 /* schedule immediate reset if we believe we hung */
 735                 dev_info(tx_ring->dev, "Detected Tx Unit Hang\n"
 736                          "  VSI                  <%d>\n"
 737                          "  Tx Queue             <%d>\n"
 738                          "  next_to_use          <%x>\n"
 739                          "  next_to_clean        <%x>\n",
 740                          tx_ring->vsi->seid,
 741                          tx_ring->queue_index,
 742                          tx_ring->next_to_use, i);
 743                 dev_info(tx_ring->dev, "tx_bi[next_to_clean]\n"
 744                          "  time_stamp           <%lx>\n"
 745                          "  jiffies              <%lx>\n",
 746                          tx_ring->tx_bi[i].time_stamp, jiffies);
 747
 748                 netif_stop_subqueue(tx_ring->netdev, tx_ring->queue_index);
 749
 750                 dev_info(tx_ring->dev,
 751                          "tx hang detected on queue %d, resetting adapter\n",
 752                          tx_ring->queue_index);
 753
 754                 tx_ring->netdev->netdev_ops->ndo_tx_timeout(tx_ring->netdev);
 755
 756                 /* the adapter is about to reset, no point in enabling stuff */
 757                 return true;
 758         }
 759
 760         netdev_tx_completed_queue(netdev_get_tx_queue(tx_ring->netdev,
 761                                                       tx_ring->queue_index),
 762                                   total_packets, total_bytes);
 763
 764 #define TX_WAKE_THRESHOLD (DESC_NEEDED * 2)
 765         if (unlikely(total_packets && netif_carrier_ok(tx_ring->netdev) &&
 766                      (I40E_DESC_UNUSED(tx_ring) >= TX_WAKE_THRESHOLD))) {
 767                 /* Make sure that anybody stopping the queue after this
 768                  * sees the new next_to_clean.
 769                  */
 770                 smp_mb();
 771                 if (__netif_subqueue_stopped(tx_ring->netdev,
 772                                              tx_ring->queue_index) &&
 773                    !test_bit(__I40E_DOWN, &tx_ring->vsi->state)) {
 774                         netif_wake_subqueue(tx_ring->netdev,
 775                                             tx_ring->queue_index);
 776                         ++tx_ring->tx_stats.restart_queue;
 777                 }
 778         }
 779
 780         return budget > 0;
 781 }
 782
 783 /**
 784  * i40e_set_new_dynamic_itr - Find new ITR level
 785  * @rc: structure containing ring performance data
 786  *
 787  * Stores a new ITR value based on packets and byte counts during
 788  * the last interrupt.  The advantage of per interrupt computation
 789  * is faster updates and more accurate ITR for the current traffic
 790  * pattern.  Constants in this function were computed based on
 791  * theoretical maximum wire speed and thresholds were set based on
 792  * testing data as well as attempting to minimize response time
 793  * while increasing bulk throughput.
 794  **/
 795 static void i40e_set_new_dynamic_itr(struct i40e_ring_container *rc)
 796 {
 797         enum i40e_latency_range new_latency_range = rc->latency_range;
 798         u32 new_itr = rc->itr;
 799         int bytes_per_int;
 800
 801         if (rc->total_packets == 0 || !rc->itr)
 802                 return;
 803
 804         /* simple throttlerate management
 805          *   0-10MB/s   lowest (100000 ints/s)
 806          *  10-20MB/s   low    (20000 ints/s)
 807          *  20-1249MB/s bulk   (8000 ints/s)
 808          */
 809         bytes_per_int = rc->total_bytes / rc->itr;
 810         switch (rc->itr) {
 811         case I40E_LOWEST_LATENCY:
 812                 if (bytes_per_int > 10)
 813                         new_latency_range = I40E_LOW_LATENCY;
 814                 break;
 815         case I40E_LOW_LATENCY:
 816                 if (bytes_per_int > 20)
 817                         new_latency_range = I40E_BULK_LATENCY;
 818                 else if (bytes_per_int <= 10)
 819                         new_latency_range = I40E_LOWEST_LATENCY;
 820                 break;
 821         case I40E_BULK_LATENCY:
 822                 if (bytes_per_int <= 20)
 823                         rc->latency_range = I40E_LOW_LATENCY;
 824                 break;
 825         }
 826
 827         switch (new_latency_range) {
 828         case I40E_LOWEST_LATENCY:
 829                 new_itr = I40E_ITR_100K;
 830                 break;
 831         case I40E_LOW_LATENCY:
 832                 new_itr = I40E_ITR_20K;
 833                 break;
 834         case I40E_BULK_LATENCY:
 835                 new_itr = I40E_ITR_8K;
 836                 break;
 837         default:
 838                 break;
 839         }
 840
 841         if (new_itr != rc->itr) {
 842                 /* do an exponential smoothing */
 843                 new_itr = (10 * new_itr * rc->itr) /
 844                           ((9 * new_itr) + rc->itr);
 845                 rc->itr = new_itr & I40E_MAX_ITR;
 846         }
 847
 848         rc->total_bytes = 0;
 849         rc->total_packets = 0;
 850 }
 851
 852 /**
 853  * i40e_update_dynamic_itr - Adjust ITR based on bytes per int
 854  * @q_vector: the vector to adjust
 855  **/
 856 static void i40e_update_dynamic_itr(struct i40e_q_vector *q_vector)
 857 {
 858         u16 vector = q_vector->vsi->base_vector + q_vector->v_idx;
 859         struct i40e_hw *hw = &q_vector->vsi->back->hw;
 860         u32 reg_addr;
 861         u16 old_itr;
 862
 863         reg_addr = I40E_PFINT_ITRN(I40E_RX_ITR, vector - 1);
 864         old_itr = q_vector->rx.itr;
 865         i40e_set_new_dynamic_itr(&q_vector->rx);
 866         if (old_itr != q_vector->rx.itr)
 867                 wr32(hw, reg_addr, q_vector->rx.itr);
 868
 869         reg_addr = I40E_PFINT_ITRN(I40E_TX_ITR, vector - 1);
 870         old_itr = q_vector->tx.itr;
 871         i40e_set_new_dynamic_itr(&q_vector->tx);
 872         if (old_itr != q_vector->tx.itr)
 873                 wr32(hw, reg_addr, q_vector->tx.itr);
 874 }
 875
 876 /**
 877  * i40e_clean_programming_status - clean the programming status descriptor
 878  * @rx_ring: the rx ring that has this descriptor
 879  * @rx_desc: the rx descriptor written back by HW
 880  *
 881  * Flow director should handle FD_FILTER_STATUS to check its filter programming
 882  * status being successful or not and take actions accordingly. FCoE should
 883  * handle its context/filter programming/invalidation status and take actions.
 884  *
 885  **/
 886 static void i40e_clean_programming_status(struct i40e_ring *rx_ring,
 887                                           union i40e_rx_desc *rx_desc)
 888 {
 889         u64 qw;
 890         u8 id;
 891
 892         qw = le64_to_cpu(rx_desc->wb.qword1.status_error_len);
 893         id = (qw & I40E_RX_PROG_STATUS_DESC_QW1_PROGID_MASK) >>
 894                   I40E_RX_PROG_STATUS_DESC_QW1_PROGID_SHIFT;
 895
 896         if (id == I40E_RX_PROG_STATUS_DESC_FD_FILTER_STATUS)
 897                 i40e_fd_handle_status(rx_ring, rx_desc, id);
 898 }
 899
 900 /**
 901  * i40e_setup_tx_descriptors - Allocate the Tx descriptors
 902  * @tx_ring: the tx ring to set up
 903  *
 904  * Return 0 on success, negative on error
 905  **/
 906 int i40e_setup_tx_descriptors(struct i40e_ring *tx_ring)
 907 {
 908         struct device *dev = tx_ring->dev;
 909         int bi_size;
 910
 911         if (!dev)
 912                 return -ENOMEM;
 913
 914         bi_size = sizeof(struct i40e_tx_buffer) * tx_ring->count;
 915         tx_ring->tx_bi = kzalloc(bi_size, GFP_KERNEL);
 916         if (!tx_ring->tx_bi)
 917                 goto err;
 918
 919         /* round up to nearest 4K */
 920         tx_ring->size = tx_ring->count * sizeof(struct i40e_tx_desc);
 921         /* add u32 for head writeback, align after this takes care of
 922          * guaranteeing this is at least one cache line in size
 923          */
 924         tx_ring->size += sizeof(u32);
 925         tx_ring->size = ALIGN(tx_ring->size, 4096);
 926         tx_ring->desc = dma_alloc_coherent(dev, tx_ring->size,
 927                                            &tx_ring->dma, GFP_KERNEL);
 928         if (!tx_ring->desc) {
 929                 dev_info(dev, "Unable to allocate memory for the Tx descriptor ring, size=%d\n",
 930                          tx_ring->size);
 931                 goto err;
 932         }
 933
 934         tx_ring->next_to_use = 0;
 935         tx_ring->next_to_clean = 0;
 936         return 0;
 937
 938 err:
 939         kfree(tx_ring->tx_bi);
 940         tx_ring->tx_bi = NULL;
 941         return -ENOMEM;
 942 }
 943
 944 /**
 945  * i40e_clean_rx_ring - Free Rx buffers
 946  * @rx_ring: ring to be cleaned
 947  **/
 948 void i40e_clean_rx_ring(struct i40e_ring *rx_ring)
 949 {
 950         struct device *dev = rx_ring->dev;
 951         struct i40e_rx_buffer *rx_bi;
 952         unsigned long bi_size;
 953         u16 i;
 954
 955         /* ring already cleared, nothing to do */
 956         if (!rx_ring->rx_bi)
 957                 return;
 958
 959         /* Free all the Rx ring sk_buffs */
 960         for (i = 0; i < rx_ring->count; i++) {
 961                 rx_bi = &rx_ring->rx_bi[i];
 962                 if (rx_bi->dma) {
 963                         dma_unmap_single(dev,
 964                                          rx_bi->dma,
 965                                          rx_ring->rx_buf_len,
 966                                          DMA_FROM_DEVICE);
 967                         rx_bi->dma = 0;
 968                 }
 969                 if (rx_bi->skb) {
 970                         dev_kfree_skb(rx_bi->skb);
 971                         rx_bi->skb = NULL;
 972                 }
 973                 if (rx_bi->page) {
 974                         if (rx_bi->page_dma) {
 975                                 dma_unmap_page(dev,
 976                                                rx_bi->page_dma,
 977                                                PAGE_SIZE / 2,
 978                                                DMA_FROM_DEVICE);
 979                                 rx_bi->page_dma = 0;
 980                         }
 981                         __free_page(rx_bi->page);
 982                         rx_bi->page = NULL;
 983                         rx_bi->page_offset = 0;
 984                 }
 985         }
 986
 987         bi_size = sizeof(struct i40e_rx_buffer) * rx_ring->count;
 988         memset(rx_ring->rx_bi, 0, bi_size);
 989
 990         /* Zero out the descriptor ring */
 991         memset(rx_ring->desc, 0, rx_ring->size);
 992
 993         rx_ring->next_to_clean = 0;
 994         rx_ring->next_to_use = 0;
 995 }
 996
 997 /**
 998  * i40e_free_rx_resources - Free Rx resources
 999  * @rx_ring: ring to clean the resources from
1000  *
1001  * Free all receive software resources
1002  **/
1003 void i40e_free_rx_resources(struct i40e_ring *rx_ring)
1004 {
1005         i40e_clean_rx_ring(rx_ring);
1006         kfree(rx_ring->rx_bi);
1007         rx_ring->rx_bi = NULL;
1008
1009         if (rx_ring->desc) {
1010                 dma_free_coherent(rx_ring->dev, rx_ring->size,
1011                                   rx_ring->desc, rx_ring->dma);
1012                 rx_ring->desc = NULL;
1013         }
1014 }
1015
1016 /**
1017  * i40e_setup_rx_descriptors - Allocate Rx descriptors
1018  * @rx_ring: Rx descriptor ring (for a specific queue) to setup
1019  *
1020  * Returns 0 on success, negative on failure
1021  **/
1022 int i40e_setup_rx_descriptors(struct i40e_ring *rx_ring)
1023 {
1024         struct device *dev = rx_ring->dev;
1025         int bi_size;
1026
1027         bi_size = sizeof(struct i40e_rx_buffer) * rx_ring->count;
1028         rx_ring->rx_bi = kzalloc(bi_size, GFP_KERNEL);
1029         if (!rx_ring->rx_bi)
1030                 goto err;
1031
1032         /* Round up to nearest 4K */
1033         rx_ring->size = ring_is_16byte_desc_enabled(rx_ring)
1034                 ? rx_ring->count * sizeof(union i40e_16byte_rx_desc)
1035                 : rx_ring->count * sizeof(union i40e_32byte_rx_desc);
1036         rx_ring->size = ALIGN(rx_ring->size, 4096);
1037         rx_ring->desc = dma_alloc_coherent(dev, rx_ring->size,
1038                                            &rx_ring->dma, GFP_KERNEL);
1039
1040         if (!rx_ring->desc) {
1041                 dev_info(dev, "Unable to allocate memory for the Rx descriptor ring, size=%d\n",
1042                          rx_ring->size);
1043                 goto err;
1044         }
1045
1046         rx_ring->next_to_clean = 0;
1047         rx_ring->next_to_use = 0;
1048
1049         return 0;
1050 err:
1051         kfree(rx_ring->rx_bi);
1052         rx_ring->rx_bi = NULL;
1053         return -ENOMEM;
1054 }
1055
1056 /**
1057  * i40e_release_rx_desc - Store the new tail and head values
1058  * @rx_ring: ring to bump
1059  * @val: new head index
1060  **/
1061 static inline void i40e_release_rx_desc(struct i40e_ring *rx_ring, u32 val)
1062 {
1063         rx_ring->next_to_use = val;
1064         /* Force memory writes to complete before letting h/w
1065          * know there are new descriptors to fetch.  (Only
1066          * applicable for weak-ordered memory model archs,
1067          * such as IA-64).
1068          */
1069         wmb();
1070         writel(val, rx_ring->tail);
1071 }
1072
1073 /**
1074  * i40e_alloc_rx_buffers - Replace used receive buffers; packet split
1075  * @rx_ring: ring to place buffers on
1076  * @cleaned_count: number of buffers to replace
1077  **/
1078 void i40e_alloc_rx_buffers(struct i40e_ring *rx_ring, u16 cleaned_count)
1079 {
1080         u16 i = rx_ring->next_to_use;
1081         union i40e_rx_desc *rx_desc;
1082         struct i40e_rx_buffer *bi;
1083         struct sk_buff *skb;
1084
1085         /* do nothing if no valid netdev defined */
1086         if (!rx_ring->netdev || !cleaned_count)
1087                 return;
1088
1089         while (cleaned_count--) {
1090                 rx_desc = I40E_RX_DESC(rx_ring, i);
1091                 bi = &rx_ring->rx_bi[i];
1092                 skb = bi->skb;
1093
1094                 if (!skb) {
1095                         skb = netdev_alloc_skb_ip_align(rx_ring->netdev,
1096                                                         rx_ring->rx_buf_len);
1097                         if (!skb) {
1098                                 rx_ring->rx_stats.alloc_buff_failed++;
1099                                 goto no_buffers;
1100                         }
1101                         /* initialize queue mapping */
1102                         skb_record_rx_queue(skb, rx_ring->queue_index);
1103                         bi->skb = skb;
1104                 }
1105
1106                 if (!bi->dma) {
1107                         bi->dma = dma_map_single(rx_ring->dev,
1108                                                  skb->data,
1109                                                  rx_ring->rx_buf_len,
1110                                                  DMA_FROM_DEVICE);
1111                         if (dma_mapping_error(rx_ring->dev, bi->dma)) {
1112                                 rx_ring->rx_stats.alloc_buff_failed++;
1113                                 bi->dma = 0;
1114                                 goto no_buffers;
1115                         }
1116                 }
1117
1118                 if (ring_is_ps_enabled(rx_ring)) {
1119                         if (!bi->page) {
1120                                 bi->page = alloc_page(GFP_ATOMIC);
1121                                 if (!bi->page) {
1122                                         rx_ring->rx_stats.alloc_page_failed++;
1123                                         goto no_buffers;
1124                                 }
1125                         }
1126
1127                         if (!bi->page_dma) {
1128                                 /* use a half page if we're re-using */
1129                                 bi->page_offset ^= PAGE_SIZE / 2;
1130                                 bi->page_dma = dma_map_page(rx_ring->dev,
1131                                                             bi->page,
1132                                                             bi->page_offset,
1133                                                             PAGE_SIZE / 2,
1134                                                             DMA_FROM_DEVICE);
1135                                 if (dma_mapping_error(rx_ring->dev,
1136                                                       bi->page_dma)) {
1137                                         rx_ring->rx_stats.alloc_page_failed++;
1138                                         bi->page_dma = 0;
1139                                         goto no_buffers;
1140                                 }
1141                         }
1142
1143                         /* Refresh the desc even if buffer_addrs didn't change
1144                          * because each write-back erases this info.
1145                          */
1146                         rx_desc->read.pkt_addr = cpu_to_le64(bi->page_dma);
1147                         rx_desc->read.hdr_addr = cpu_to_le64(bi->dma);
1148                 } else {
1149                         rx_desc->read.pkt_addr = cpu_to_le64(bi->dma);
1150                         rx_desc->read.hdr_addr = 0;
1151                 }
1152                 i++;
1153                 if (i == rx_ring->count)
1154                         i = 0;
1155         }
1156
1157 no_buffers:
1158         if (rx_ring->next_to_use != i)
1159                 i40e_release_rx_desc(rx_ring, i);
1160 }
1161
1162 /**
1163  * i40e_receive_skb - Send a completed packet up the stack
1164  * @rx_ring:  rx ring in play
1165  * @skb: packet to send up
1166  * @vlan_tag: vlan tag for packet
1167  **/
1168 static void i40e_receive_skb(struct i40e_ring *rx_ring,
1169                              struct sk_buff *skb, u16 vlan_tag)
1170 {
1171         struct i40e_q_vector *q_vector = rx_ring->q_vector;
1172         struct i40e_vsi *vsi = rx_ring->vsi;
1173         u64 flags = vsi->back->flags;
1174
1175         if (vlan_tag & VLAN_VID_MASK)
1176                 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vlan_tag);
1177
1178         if (flags & I40E_FLAG_IN_NETPOLL)
1179                 netif_rx(skb);
1180         else
1181                 napi_gro_receive(&q_vector->napi, skb);
1182 }
1183
1184 /**
1185  * i40e_rx_checksum - Indicate in skb if hw indicated a good cksum
1186  * @vsi: the VSI we care about
1187  * @skb: skb currently being received and modified
1188  * @rx_status: status value of last descriptor in packet
1189  * @rx_error: error value of last descriptor in packet
1190  * @rx_ptype: ptype value of last descriptor in packet
1191  **/
1192 static inline void i40e_rx_checksum(struct i40e_vsi *vsi,
1193                                     struct sk_buff *skb,
1194                                     u32 rx_status,
1195                                     u32 rx_error,
1196                                     u16 rx_ptype)
1197 {
1198         bool ipv4_tunnel, ipv6_tunnel;
1199         __wsum rx_udp_csum;
1200         __sum16 csum;
1201         struct iphdr *iph;
1202
1203         ipv4_tunnel = (rx_ptype > I40E_RX_PTYPE_GRENAT4_MAC_PAY3) &&
1204                       (rx_ptype < I40E_RX_PTYPE_GRENAT4_MACVLAN_IPV6_ICMP_PAY4);
1205         ipv6_tunnel = (rx_ptype > I40E_RX_PTYPE_GRENAT6_MAC_PAY3) &&
1206                       (rx_ptype < I40E_RX_PTYPE_GRENAT6_MACVLAN_IPV6_ICMP_PAY4);
1207
1208         skb->encapsulation = ipv4_tunnel || ipv6_tunnel;
1209         skb->ip_summed = CHECKSUM_NONE;
1210
1211         /* Rx csum enabled and ip headers found? */
1212         if (!(vsi->netdev->features & NETIF_F_RXCSUM &&
1213               rx_status & (1 << I40E_RX_DESC_STATUS_L3L4P_SHIFT)))
1214                 return;
1215
1216         /* likely incorrect csum if alternate IP extension headers found */
1217         if (rx_status & (1 << I40E_RX_DESC_STATUS_IPV6EXADD_SHIFT))
1218                 return;
1219
1220         /* IP or L4 or outmost IP checksum error */
1221         if (rx_error & ((1 << I40E_RX_DESC_ERROR_IPE_SHIFT) |
1222                         (1 << I40E_RX_DESC_ERROR_L4E_SHIFT) |
1223                         (1 << I40E_RX_DESC_ERROR_EIPE_SHIFT))) {
1224                 vsi->back->hw_csum_rx_error++;
1225                 return;
1226         }
1227
1228         if (ipv4_tunnel &&
1229             !(rx_status & (1 << I40E_RX_DESC_STATUS_UDP_0_SHIFT))) {
1230                 /* If VXLAN traffic has an outer UDPv4 checksum we need to check
1231                  * it in the driver, hardware does not do it for us.
1232                  * Since L3L4P bit was set we assume a valid IHL value (>=5)
1233                  * so the total length of IPv4 header is IHL*4 bytes
1234                  */
1235                 skb->transport_header = skb->mac_header +
1236                                         sizeof(struct ethhdr) +
1237                                         (ip_hdr(skb)->ihl * 4);
1238
1239                 /* Add 4 bytes for VLAN tagged packets */
1240                 skb->transport_header += (skb->protocol == htons(ETH_P_8021Q) ||
1241                                           skb->protocol == htons(ETH_P_8021AD))
1242                                           ? VLAN_HLEN : 0;
1243
1244                 rx_udp_csum = udp_csum(skb);
1245                 iph = ip_hdr(skb);
1246                 csum = csum_tcpudp_magic(
1247                                 iph->saddr, iph->daddr,
1248                                 (skb->len - skb_transport_offset(skb)),
1249                                 IPPROTO_UDP, rx_udp_csum);
1250
1251                 if (udp_hdr(skb)->check != csum) {
1252                         vsi->back->hw_csum_rx_error++;
1253                         return;
1254                 }
1255         }
1256
1257         skb->ip_summed = CHECKSUM_UNNECESSARY;
1258 }
1259
1260 /**
1261  * i40e_rx_hash - returns the hash value from the Rx descriptor
1262  * @ring: descriptor ring
1263  * @rx_desc: specific descriptor
1264  **/
1265 static inline u32 i40e_rx_hash(struct i40e_ring *ring,
1266                                union i40e_rx_desc *rx_desc)
1267 {
1268         const __le64 rss_mask =
1269                 cpu_to_le64((u64)I40E_RX_DESC_FLTSTAT_RSS_HASH <<
1270                             I40E_RX_DESC_STATUS_FLTSTAT_SHIFT);
1271
1272         if ((ring->netdev->features & NETIF_F_RXHASH) &&
1273             (rx_desc->wb.qword1.status_error_len & rss_mask) == rss_mask)
1274                 return le32_to_cpu(rx_desc->wb.qword0.hi_dword.rss);
1275         else
1276                 return 0;
1277 }
1278
1279 /**
1280  * i40e_ptype_to_hash - get a hash type
1281  * @ptype: the ptype value from the descriptor
1282  *
1283  * Returns a hash type to be used by skb_set_hash
1284  **/
1285 static inline enum pkt_hash_types i40e_ptype_to_hash(u8 ptype)
1286 {
1287         struct i40e_rx_ptype_decoded decoded = decode_rx_desc_ptype(ptype);
1288
1289         if (!decoded.known)
1290                 return PKT_HASH_TYPE_NONE;
1291
1292         if (decoded.outer_ip == I40E_RX_PTYPE_OUTER_IP &&
1293             decoded.payload_layer == I40E_RX_PTYPE_PAYLOAD_LAYER_PAY4)
1294                 return PKT_HASH_TYPE_L4;
1295         else if (decoded.outer_ip == I40E_RX_PTYPE_OUTER_IP &&
1296                  decoded.payload_layer == I40E_RX_PTYPE_PAYLOAD_LAYER_PAY3)
1297                 return PKT_HASH_TYPE_L3;
1298         else
1299                 return PKT_HASH_TYPE_L2;
1300 }
1301
1302 /**
1303  * i40e_clean_rx_irq - Reclaim resources after receive completes
1304  * @rx_ring:  rx ring to clean
1305  * @budget:   how many cleans we're allowed
1306  *
1307  * Returns true if there's any budget left (e.g. the clean is finished)
1308  **/
1309 static int i40e_clean_rx_irq(struct i40e_ring *rx_ring, int budget)
1310 {
1311         unsigned int total_rx_bytes = 0, total_rx_packets = 0;
1312         u16 rx_packet_len, rx_header_len, rx_sph, rx_hbo;
1313         u16 cleaned_count = I40E_DESC_UNUSED(rx_ring);
1314         const int current_node = numa_node_id();
1315         struct i40e_vsi *vsi = rx_ring->vsi;
1316         u16 i = rx_ring->next_to_clean;
1317         union i40e_rx_desc *rx_desc;
1318         u32 rx_error, rx_status;
1319         u8 rx_ptype;
1320         u64 qword;
1321
1322         if (budget <= 0)
1323                 return 0;
1324
1325         rx_desc = I40E_RX_DESC(rx_ring, i);
1326         qword = le64_to_cpu(rx_desc->wb.qword1.status_error_len);
1327         rx_status = (qword & I40E_RXD_QW1_STATUS_MASK) >>
1328                     I40E_RXD_QW1_STATUS_SHIFT;
1329
1330         while (rx_status & (1 << I40E_RX_DESC_STATUS_DD_SHIFT)) {
1331                 union i40e_rx_desc *next_rxd;
1332                 struct i40e_rx_buffer *rx_bi;
1333                 struct sk_buff *skb;
1334                 u16 vlan_tag;
1335                 if (i40e_rx_is_programming_status(qword)) {
1336                         i40e_clean_programming_status(rx_ring, rx_desc);
1337                         I40E_RX_NEXT_DESC_PREFETCH(rx_ring, i, next_rxd);
1338                         goto next_desc;
1339                 }
1340                 rx_bi = &rx_ring->rx_bi[i];
1341                 skb = rx_bi->skb;
1342                 prefetch(skb->data);
1343
1344                 rx_packet_len = (qword & I40E_RXD_QW1_LENGTH_PBUF_MASK) >>
1345                                 I40E_RXD_QW1_LENGTH_PBUF_SHIFT;
1346                 rx_header_len = (qword & I40E_RXD_QW1_LENGTH_HBUF_MASK) >>
1347                                 I40E_RXD_QW1_LENGTH_HBUF_SHIFT;
1348                 rx_sph = (qword & I40E_RXD_QW1_LENGTH_SPH_MASK) >>
1349                          I40E_RXD_QW1_LENGTH_SPH_SHIFT;
1350
1351                 rx_error = (qword & I40E_RXD_QW1_ERROR_MASK) >>
1352                            I40E_RXD_QW1_ERROR_SHIFT;
1353                 rx_hbo = rx_error & (1 << I40E_RX_DESC_ERROR_HBO_SHIFT);
1354                 rx_error &= ~(1 << I40E_RX_DESC_ERROR_HBO_SHIFT);
1355
1356                 rx_ptype = (qword & I40E_RXD_QW1_PTYPE_MASK) >>
1357                            I40E_RXD_QW1_PTYPE_SHIFT;
1358                 rx_bi->skb = NULL;
1359
1360                 /* This memory barrier is needed to keep us from reading
1361                  * any other fields out of the rx_desc until we know the
1362                  * STATUS_DD bit is set
1363                  */
1364                 rmb();
1365
1366                 /* Get the header and possibly the whole packet
1367                  * If this is an skb from previous receive dma will be 0
1368                  */
1369                 if (rx_bi->dma) {
1370                         u16 len;
1371
1372                         if (rx_hbo)
1373                                 len = I40E_RX_HDR_SIZE;
1374                         else if (rx_sph)
1375                                 len = rx_header_len;
1376                         else if (rx_packet_len)
1377                                 len = rx_packet_len;   /* 1buf/no split found */
1378                         else
1379                                 len = rx_header_len;   /* split always mode */
1380
1381                         skb_put(skb, len);
1382                         dma_unmap_single(rx_ring->dev,
1383                                          rx_bi->dma,
1384                                          rx_ring->rx_buf_len,
1385                                          DMA_FROM_DEVICE);
1386                         rx_bi->dma = 0;
1387                 }
1388
1389                 /* Get the rest of the data if this was a header split */
1390                 if (ring_is_ps_enabled(rx_ring) && rx_packet_len) {
1391
1392                         skb_fill_page_desc(skb, skb_shinfo(skb)->nr_frags,
1393                                            rx_bi->page,
1394                                            rx_bi->page_offset,
1395                                            rx_packet_len);
1396
1397                         skb->len += rx_packet_len;
1398                         skb->data_len += rx_packet_len;
1399                         skb->truesize += rx_packet_len;
1400
1401                         if ((page_count(rx_bi->page) == 1) &&
1402                             (page_to_nid(rx_bi->page) == current_node))
1403                                 get_page(rx_bi->page);
1404                         else
1405                                 rx_bi->page = NULL;
1406
1407                         dma_unmap_page(rx_ring->dev,
1408                                        rx_bi->page_dma,
1409                                        PAGE_SIZE / 2,
1410                                        DMA_FROM_DEVICE);
1411                         rx_bi->page_dma = 0;
1412                 }
1413                 I40E_RX_NEXT_DESC_PREFETCH(rx_ring, i, next_rxd);
1414
1415                 if (unlikely(
1416                     !(rx_status & (1 << I40E_RX_DESC_STATUS_EOF_SHIFT)))) {
1417                         struct i40e_rx_buffer *next_buffer;
1418
1419                         next_buffer = &rx_ring->rx_bi[i];
1420
1421                         if (ring_is_ps_enabled(rx_ring)) {
1422                                 rx_bi->skb = next_buffer->skb;
1423                                 rx_bi->dma = next_buffer->dma;
1424                                 next_buffer->skb = skb;
1425                                 next_buffer->dma = 0;
1426                         }
1427                         rx_ring->rx_stats.non_eop_descs++;
1428                         goto next_desc;
1429                 }
1430
1431                 /* ERR_MASK will only have valid bits if EOP set */
1432                 if (unlikely(rx_error & (1 << I40E_RX_DESC_ERROR_RXE_SHIFT))) {
1433                         dev_kfree_skb_any(skb);
1434                         goto next_desc;
1435                 }
1436
1437                 skb_set_hash(skb, i40e_rx_hash(rx_ring, rx_desc),
1438                              i40e_ptype_to_hash(rx_ptype));
1439                 if (unlikely(rx_status & I40E_RXD_QW1_STATUS_TSYNVALID_MASK)) {
1440                         i40e_ptp_rx_hwtstamp(vsi->back, skb, (rx_status &
1441                                            I40E_RXD_QW1_STATUS_TSYNINDX_MASK) >>
1442                                            I40E_RXD_QW1_STATUS_TSYNINDX_SHIFT);
1443                         rx_ring->last_rx_timestamp = jiffies;
1444                 }
1445
1446                 /* probably a little skewed due to removing CRC */
1447                 total_rx_bytes += skb->len;
1448                 total_rx_packets++;
1449
1450                 skb->protocol = eth_type_trans(skb, rx_ring->netdev);
1451
1452                 i40e_rx_checksum(vsi, skb, rx_status, rx_error, rx_ptype);
1453
1454                 vlan_tag = rx_status & (1 << I40E_RX_DESC_STATUS_L2TAG1P_SHIFT)
1455                          ? le16_to_cpu(rx_desc->wb.qword0.lo_dword.l2tag1)
1456                          : 0;
1457                 i40e_receive_skb(rx_ring, skb, vlan_tag);
1458
1459                 rx_ring->netdev->last_rx = jiffies;
1460                 budget--;
1461 next_desc:
1462                 rx_desc->wb.qword1.status_error_len = 0;
1463                 if (!budget)
1464                         break;
1465
1466                 cleaned_count++;
1467                 /* return some buffers to hardware, one at a time is too slow */
1468                 if (cleaned_count >= I40E_RX_BUFFER_WRITE) {
1469                         i40e_alloc_rx_buffers(rx_ring, cleaned_count);
1470                         cleaned_count = 0;
1471                 }
1472
1473                 /* use prefetched values */
1474                 rx_desc = next_rxd;
1475                 qword = le64_to_cpu(rx_desc->wb.qword1.status_error_len);
1476                 rx_status = (qword & I40E_RXD_QW1_STATUS_MASK) >>
1477                             I40E_RXD_QW1_STATUS_SHIFT;
1478         }
1479
1480         rx_ring->next_to_clean = i;
1481         u64_stats_update_begin(&rx_ring->syncp);
1482         rx_ring->stats.packets += total_rx_packets;
1483         rx_ring->stats.bytes += total_rx_bytes;
1484         u64_stats_update_end(&rx_ring->syncp);
1485         rx_ring->q_vector->rx.total_packets += total_rx_packets;
1486         rx_ring->q_vector->rx.total_bytes += total_rx_bytes;
1487
1488         if (cleaned_count)
1489                 i40e_alloc_rx_buffers(rx_ring, cleaned_count);
1490
1491         return budget > 0;
1492 }
1493
1494 /**
1495  * i40e_napi_poll - NAPI polling Rx/Tx cleanup routine
1496  * @napi: napi struct with our devices info in it
1497  * @budget: amount of work driver is allowed to do this pass, in packets
1498  *
1499  * This function will clean all queues associated with a q_vector.
1500  *
1501  * Returns the amount of work done
1502  **/
1503 int i40e_napi_poll(struct napi_struct *napi, int budget)
1504 {
1505         struct i40e_q_vector *q_vector =
1506                                container_of(napi, struct i40e_q_vector, napi);
1507         struct i40e_vsi *vsi = q_vector->vsi;
1508         struct i40e_ring *ring;
1509         bool clean_complete = true;
1510         int budget_per_ring;
1511
1512         if (test_bit(__I40E_DOWN, &vsi->state)) {
1513                 napi_complete(napi);
1514                 return 0;
1515         }
1516
1517         /* Since the actual Tx work is minimal, we can give the Tx a larger
1518          * budget and be more aggressive about cleaning up the Tx descriptors.
1519          */
1520         i40e_for_each_ring(ring, q_vector->tx)
1521                 clean_complete &= i40e_clean_tx_irq(ring, vsi->work_limit);
1522
1523         /* We attempt to distribute budget to each Rx queue fairly, but don't
1524          * allow the budget to go below 1 because that would exit polling early.
1525          */
1526         budget_per_ring = max(budget/q_vector->num_ringpairs, 1);
1527
1528         i40e_for_each_ring(ring, q_vector->rx)
1529                 clean_complete &= i40e_clean_rx_irq(ring, budget_per_ring);
1530
1531         /* If work not completed, return budget and polling will return */
1532         if (!clean_complete)
1533                 return budget;
1534
1535         /* Work is done so exit the polling mode and re-enable the interrupt */
1536         napi_complete(napi);
1537         if (ITR_IS_DYNAMIC(vsi->rx_itr_setting) ||
1538             ITR_IS_DYNAMIC(vsi->tx_itr_setting))
1539                 i40e_update_dynamic_itr(q_vector);
1540
1541         if (!test_bit(__I40E_DOWN, &vsi->state)) {
1542                 if (vsi->back->flags & I40E_FLAG_MSIX_ENABLED) {
1543                         i40e_irq_dynamic_enable(vsi,
1544                                         q_vector->v_idx + vsi->base_vector);
1545                 } else {
1546                         struct i40e_hw *hw = &vsi->back->hw;
1547                         /* We re-enable the queue 0 cause, but
1548                          * don't worry about dynamic_enable
1549                          * because we left it on for the other
1550                          * possible interrupts during napi
1551                          */
1552                         u32 qval = rd32(hw, I40E_QINT_RQCTL(0));
1553                         qval |= I40E_QINT_RQCTL_CAUSE_ENA_MASK;
1554                         wr32(hw, I40E_QINT_RQCTL(0), qval);
1555
1556                         qval = rd32(hw, I40E_QINT_TQCTL(0));
1557                         qval |= I40E_QINT_TQCTL_CAUSE_ENA_MASK;
1558                         wr32(hw, I40E_QINT_TQCTL(0), qval);
1559
1560                         i40e_irq_dynamic_enable_icr0(vsi->back);
1561                 }
1562         }
1563
1564         return 0;
1565 }
1566
1567 /**
1568  * i40e_atr - Add a Flow Director ATR filter
1569  * @tx_ring:  ring to add programming descriptor to
1570  * @skb:      send buffer
1571  * @flags:    send flags
1572  * @protocol: wire protocol
1573  **/
1574 static void i40e_atr(struct i40e_ring *tx_ring, struct sk_buff *skb,
1575                      u32 flags, __be16 protocol)
1576 {
1577         struct i40e_filter_program_desc *fdir_desc;
1578         struct i40e_pf *pf = tx_ring->vsi->back;
1579         union {
1580                 unsigned char *network;
1581                 struct iphdr *ipv4;
1582                 struct ipv6hdr *ipv6;
1583         } hdr;
1584         struct tcphdr *th;
1585         unsigned int hlen;
1586         u32 flex_ptype, dtype_cmd;
1587         u16 i;
1588
1589         /* make sure ATR is enabled */
1590         if (!(pf->flags & I40E_FLAG_FD_ATR_ENABLED))
1591                 return;
1592
1593         /* if sampling is disabled do nothing */
1594         if (!tx_ring->atr_sample_rate)
1595                 return;
1596
1597         /* snag network header to get L4 type and address */
1598         hdr.network = skb_network_header(skb);
1599
1600         /* Currently only IPv4/IPv6 with TCP is supported */
1601         if (protocol == htons(ETH_P_IP)) {
1602                 if (hdr.ipv4->protocol != IPPROTO_TCP)
1603                         return;
1604
1605                 /* access ihl as a u8 to avoid unaligned access on ia64 */
1606                 hlen = (hdr.network[0] & 0x0F) << 2;
1607         } else if (protocol == htons(ETH_P_IPV6)) {
1608                 if (hdr.ipv6->nexthdr != IPPROTO_TCP)
1609                         return;
1610
1611                 hlen = sizeof(struct ipv6hdr);
1612         } else {
1613                 return;
1614         }
1615
1616         th = (struct tcphdr *)(hdr.network + hlen);
1617
1618         /* Due to lack of space, no more new filters can be programmed */
1619         if (th->syn && (pf->auto_disable_flags & I40E_FLAG_FD_ATR_ENABLED))
1620                 return;
1621
1622         tx_ring->atr_count++;
1623
1624         /* sample on all syn/fin/rst packets or once every atr sample rate */
1625         if (!th->fin &&
1626             !th->syn &&
1627             !th->rst &&
1628             (tx_ring->atr_count < tx_ring->atr_sample_rate))
1629                 return;
1630
1631         tx_ring->atr_count = 0;
1632
1633         /* grab the next descriptor */
1634         i = tx_ring->next_to_use;
1635         fdir_desc = I40E_TX_FDIRDESC(tx_ring, i);
1636
1637         i++;
1638         tx_ring->next_to_use = (i < tx_ring->count) ? i : 0;
1639
1640         flex_ptype = (tx_ring->queue_index << I40E_TXD_FLTR_QW0_QINDEX_SHIFT) &
1641                       I40E_TXD_FLTR_QW0_QINDEX_MASK;
1642         flex_ptype |= (protocol == htons(ETH_P_IP)) ?
1643                       (I40E_FILTER_PCTYPE_NONF_IPV4_TCP <<
1644                        I40E_TXD_FLTR_QW0_PCTYPE_SHIFT) :
1645                       (I40E_FILTER_PCTYPE_NONF_IPV6_TCP <<
1646                        I40E_TXD_FLTR_QW0_PCTYPE_SHIFT);
1647
1648         flex_ptype |= tx_ring->vsi->id << I40E_TXD_FLTR_QW0_DEST_VSI_SHIFT;
1649
1650         dtype_cmd = I40E_TX_DESC_DTYPE_FILTER_PROG;
1651
1652         dtype_cmd |= (th->fin || th->rst) ?
1653                      (I40E_FILTER_PROGRAM_DESC_PCMD_REMOVE <<
1654                       I40E_TXD_FLTR_QW1_PCMD_SHIFT) :
1655                      (I40E_FILTER_PROGRAM_DESC_PCMD_ADD_UPDATE <<
1656                       I40E_TXD_FLTR_QW1_PCMD_SHIFT);
1657
1658         dtype_cmd |= I40E_FILTER_PROGRAM_DESC_DEST_DIRECT_PACKET_QINDEX <<
1659                      I40E_TXD_FLTR_QW1_DEST_SHIFT;
1660
1661         dtype_cmd |= I40E_FILTER_PROGRAM_DESC_FD_STATUS_FD_ID <<
1662                      I40E_TXD_FLTR_QW1_FD_STATUS_SHIFT;
1663
1664         fdir_desc->qindex_flex_ptype_vsi = cpu_to_le32(flex_ptype);
1665         fdir_desc->dtype_cmd_cntindex = cpu_to_le32(dtype_cmd);
1666 }
1667
1668 /**
1669  * i40e_tx_prepare_vlan_flags - prepare generic TX VLAN tagging flags for HW
1670  * @skb:     send buffer
1671  * @tx_ring: ring to send buffer on
1672  * @flags:   the tx flags to be set
1673  *
1674  * Checks the skb and set up correspondingly several generic transmit flags
1675  * related to VLAN tagging for the HW, such as VLAN, DCB, etc.
1676  *
1677  * Returns error code indicate the frame should be dropped upon error and the
1678  * otherwise  returns 0 to indicate the flags has been set properly.
1679  **/
1680 static int i40e_tx_prepare_vlan_flags(struct sk_buff *skb,
1681                                       struct i40e_ring *tx_ring,
1682                                       u32 *flags)
1683 {
1684         __be16 protocol = skb->protocol;
1685         u32  tx_flags = 0;
1686
1687         /* if we have a HW VLAN tag being added, default to the HW one */
1688         if (vlan_tx_tag_present(skb)) {
1689                 tx_flags |= vlan_tx_tag_get(skb) << I40E_TX_FLAGS_VLAN_SHIFT;
1690                 tx_flags |= I40E_TX_FLAGS_HW_VLAN;
1691         /* else if it is a SW VLAN, check the next protocol and store the tag */
1692         } else if (protocol == htons(ETH_P_8021Q)) {
1693                 struct vlan_hdr *vhdr, _vhdr;
1694                 vhdr = skb_header_pointer(skb, ETH_HLEN, sizeof(_vhdr), &_vhdr);
1695                 if (!vhdr)
1696                         return -EINVAL;
1697
1698                 protocol = vhdr->h_vlan_encapsulated_proto;
1699                 tx_flags |= ntohs(vhdr->h_vlan_TCI) << I40E_TX_FLAGS_VLAN_SHIFT;
1700                 tx_flags |= I40E_TX_FLAGS_SW_VLAN;
1701         }
1702
1703         /* Insert 802.1p priority into VLAN header */
1704         if ((tx_ring->vsi->back->flags & I40E_FLAG_DCB_ENABLED) &&
1705             ((tx_flags & (I40E_TX_FLAGS_HW_VLAN | I40E_TX_FLAGS_SW_VLAN)) ||
1706              (skb->priority != TC_PRIO_CONTROL))) {
1707                 tx_flags &= ~I40E_TX_FLAGS_VLAN_PRIO_MASK;
1708                 tx_flags |= (skb->priority & 0x7) <<
1709                                 I40E_TX_FLAGS_VLAN_PRIO_SHIFT;
1710                 if (tx_flags & I40E_TX_FLAGS_SW_VLAN) {
1711                         struct vlan_ethhdr *vhdr;
1712                         int rc;
1713
1714                         rc = skb_cow_head(skb, 0);
1715                         if (rc < 0)
1716                                 return rc;
1717                         vhdr = (struct vlan_ethhdr *)skb->data;
1718                         vhdr->h_vlan_TCI = htons(tx_flags >>
1719                                                  I40E_TX_FLAGS_VLAN_SHIFT);
1720                 } else {
1721                         tx_flags |= I40E_TX_FLAGS_HW_VLAN;
1722                 }
1723         }
1724         *flags = tx_flags;
1725         return 0;
1726 }
1727
1728 /**
1729  * i40e_tso - set up the tso context descriptor
1730  * @tx_ring:  ptr to the ring to send
1731  * @skb:      ptr to the skb we're sending
1732  * @tx_flags: the collected send information
1733  * @protocol: the send protocol
1734  * @hdr_len:  ptr to the size of the packet header
1735  * @cd_tunneling: ptr to context descriptor bits
1736  *
1737  * Returns 0 if no TSO can happen, 1 if tso is going, or error
1738  **/
1739 static int i40e_tso(struct i40e_ring *tx_ring, struct sk_buff *skb,
1740                     u32 tx_flags, __be16 protocol, u8 *hdr_len,
1741                     u64 *cd_type_cmd_tso_mss, u32 *cd_tunneling)
1742 {
1743         u32 cd_cmd, cd_tso_len, cd_mss;
1744         struct ipv6hdr *ipv6h;
1745         struct tcphdr *tcph;
1746         struct iphdr *iph;
1747         u32 l4len;
1748         int err;
1749
1750         if (!skb_is_gso(skb))
1751                 return 0;
1752
1753         err = skb_cow_head(skb, 0);
1754         if (err < 0)
1755                 return err;
1756
1757         if (protocol == htons(ETH_P_IP)) {
1758                 iph = skb->encapsulation ? inner_ip_hdr(skb) : ip_hdr(skb);
1759                 tcph = skb->encapsulation ? inner_tcp_hdr(skb) : tcp_hdr(skb);
1760                 iph->tot_len = 0;
1761                 iph->check = 0;
1762                 tcph->check = ~csum_tcpudp_magic(iph->saddr, iph->daddr,
1763                                                  0, IPPROTO_TCP, 0);
1764         } else if (skb_is_gso_v6(skb)) {
1765
1766                 ipv6h = skb->encapsulation ? inner_ipv6_hdr(skb)
1767                                            : ipv6_hdr(skb);
1768                 tcph = skb->encapsulation ? inner_tcp_hdr(skb) : tcp_hdr(skb);
1769                 ipv6h->payload_len = 0;
1770                 tcph->check = ~csum_ipv6_magic(&ipv6h->saddr, &ipv6h->daddr,
1771                                                0, IPPROTO_TCP, 0);
1772         }
1773
1774         l4len = skb->encapsulation ? inner_tcp_hdrlen(skb) : tcp_hdrlen(skb);
1775         *hdr_len = (skb->encapsulation
1776                     ? (skb_inner_transport_header(skb) - skb->data)
1777                     : skb_transport_offset(skb)) + l4len;
1778
1779         /* find the field values */
1780         cd_cmd = I40E_TX_CTX_DESC_TSO;
1781         cd_tso_len = skb->len - *hdr_len;
1782         cd_mss = skb_shinfo(skb)->gso_size;
1783         *cd_type_cmd_tso_mss |= ((u64)cd_cmd << I40E_TXD_CTX_QW1_CMD_SHIFT) |
1784                                 ((u64)cd_tso_len <<
1785                                  I40E_TXD_CTX_QW1_TSO_LEN_SHIFT) |
1786                                 ((u64)cd_mss << I40E_TXD_CTX_QW1_MSS_SHIFT);
1787         return 1;
1788 }
1789
1790 /**
1791  * i40e_tsyn - set up the tsyn context descriptor
1792  * @tx_ring:  ptr to the ring to send
1793  * @skb:      ptr to the skb we're sending
1794  * @tx_flags: the collected send information
1795  *
1796  * Returns 0 if no Tx timestamp can happen and 1 if the timestamp will happen
1797  **/
1798 static int i40e_tsyn(struct i40e_ring *tx_ring, struct sk_buff *skb,
1799                      u32 tx_flags, u64 *cd_type_cmd_tso_mss)
1800 {
1801         struct i40e_pf *pf;
1802
1803         if (likely(!(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)))
1804                 return 0;
1805
1806         /* Tx timestamps cannot be sampled when doing TSO */
1807         if (tx_flags & I40E_TX_FLAGS_TSO)
1808                 return 0;
1809
1810         /* only timestamp the outbound packet if the user has requested it and
1811          * we are not already transmitting a packet to be timestamped
1812          */
1813         pf = i40e_netdev_to_pf(tx_ring->netdev);
1814         if (pf->ptp_tx && !pf->ptp_tx_skb) {
1815                 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
1816                 pf->ptp_tx_skb = skb_get(skb);
1817         } else {
1818                 return 0;
1819         }
1820
1821         *cd_type_cmd_tso_mss |= (u64)I40E_TX_CTX_DESC_TSYN <<
1822                                 I40E_TXD_CTX_QW1_CMD_SHIFT;
1823
1824         return 1;
1825 }
1826
1827 /**
1828  * i40e_tx_enable_csum - Enable Tx checksum offloads
1829  * @skb: send buffer
1830  * @tx_flags: Tx flags currently set
1831  * @td_cmd: Tx descriptor command bits to set
1832  * @td_offset: Tx descriptor header offsets to set
1833  * @cd_tunneling: ptr to context desc bits
1834  **/
1835 static void i40e_tx_enable_csum(struct sk_buff *skb, u32 tx_flags,
1836                                 u32 *td_cmd, u32 *td_offset,
1837                                 struct i40e_ring *tx_ring,
1838                                 u32 *cd_tunneling)
1839 {
1840         struct ipv6hdr *this_ipv6_hdr;
1841         unsigned int this_tcp_hdrlen;
1842         struct iphdr *this_ip_hdr;
1843         u32 network_hdr_len;
1844         u8 l4_hdr = 0;
1845
1846         if (skb->encapsulation) {
1847                 network_hdr_len = skb_inner_network_header_len(skb);
1848                 this_ip_hdr = inner_ip_hdr(skb);
1849                 this_ipv6_hdr = inner_ipv6_hdr(skb);
1850                 this_tcp_hdrlen = inner_tcp_hdrlen(skb);
1851
1852                 if (tx_flags & I40E_TX_FLAGS_IPV4) {
1853
1854                         if (tx_flags & I40E_TX_FLAGS_TSO) {
1855                                 *cd_tunneling |= I40E_TX_CTX_EXT_IP_IPV4;
1856                                 ip_hdr(skb)->check = 0;
1857                         } else {
1858                                 *cd_tunneling |=
1859                                          I40E_TX_CTX_EXT_IP_IPV4_NO_CSUM;
1860                         }
1861                 } else if (tx_flags & I40E_TX_FLAGS_IPV6) {
1862                         if (tx_flags & I40E_TX_FLAGS_TSO) {
1863                                 *cd_tunneling |= I40E_TX_CTX_EXT_IP_IPV6;
1864                                 ip_hdr(skb)->check = 0;
1865                         } else {
1866                                 *cd_tunneling |=
1867                                          I40E_TX_CTX_EXT_IP_IPV4_NO_CSUM;
1868                         }
1869                 }
1870
1871                 /* Now set the ctx descriptor fields */
1872                 *cd_tunneling |= (skb_network_header_len(skb) >> 2) <<
1873                                         I40E_TXD_CTX_QW0_EXT_IPLEN_SHIFT |
1874                                    I40E_TXD_CTX_UDP_TUNNELING            |
1875                                    ((skb_inner_network_offset(skb) -
1876                                         skb_transport_offset(skb)) >> 1) <<
1877                                    I40E_TXD_CTX_QW0_NATLEN_SHIFT;
1878
1879         } else {
1880                 network_hdr_len = skb_network_header_len(skb);
1881                 this_ip_hdr = ip_hdr(skb);
1882                 this_ipv6_hdr = ipv6_hdr(skb);
1883                 this_tcp_hdrlen = tcp_hdrlen(skb);
1884         }
1885
1886         /* Enable IP checksum offloads */
1887         if (tx_flags & I40E_TX_FLAGS_IPV4) {
1888                 l4_hdr = this_ip_hdr->protocol;
1889                 /* the stack computes the IP header already, the only time we
1890                  * need the hardware to recompute it is in the case of TSO.
1891                  */
1892                 if (tx_flags & I40E_TX_FLAGS_TSO) {
1893                         *td_cmd |= I40E_TX_DESC_CMD_IIPT_IPV4_CSUM;
1894                         this_ip_hdr->check = 0;
1895                 } else {
1896                         *td_cmd |= I40E_TX_DESC_CMD_IIPT_IPV4;
1897                 }
1898                 /* Now set the td_offset for IP header length */
1899                 *td_offset = (network_hdr_len >> 2) <<
1900                               I40E_TX_DESC_LENGTH_IPLEN_SHIFT;
1901         } else if (tx_flags & I40E_TX_FLAGS_IPV6) {
1902                 l4_hdr = this_ipv6_hdr->nexthdr;
1903                 *td_cmd |= I40E_TX_DESC_CMD_IIPT_IPV6;
1904                 /* Now set the td_offset for IP header length */
1905                 *td_offset = (network_hdr_len >> 2) <<
1906                               I40E_TX_DESC_LENGTH_IPLEN_SHIFT;
1907         }
1908         /* words in MACLEN + dwords in IPLEN + dwords in L4Len */
1909         *td_offset |= (skb_network_offset(skb) >> 1) <<
1910                        I40E_TX_DESC_LENGTH_MACLEN_SHIFT;
1911
1912         /* Enable L4 checksum offloads */
1913         switch (l4_hdr) {
1914         case IPPROTO_TCP:
1915                 /* enable checksum offloads */
1916                 *td_cmd |= I40E_TX_DESC_CMD_L4T_EOFT_TCP;
1917                 *td_offset |= (this_tcp_hdrlen >> 2) <<
1918                                I40E_TX_DESC_LENGTH_L4_FC_LEN_SHIFT;
1919                 break;
1920         case IPPROTO_SCTP:
1921                 /* enable SCTP checksum offload */
1922                 *td_cmd |= I40E_TX_DESC_CMD_L4T_EOFT_SCTP;
1923                 *td_offset |= (sizeof(struct sctphdr) >> 2) <<
1924                                I40E_TX_DESC_LENGTH_L4_FC_LEN_SHIFT;
1925                 break;
1926         case IPPROTO_UDP:
1927                 /* enable UDP checksum offload */
1928                 *td_cmd |= I40E_TX_DESC_CMD_L4T_EOFT_UDP;
1929                 *td_offset |= (sizeof(struct udphdr) >> 2) <<
1930                                I40E_TX_DESC_LENGTH_L4_FC_LEN_SHIFT;
1931                 break;
1932         default:
1933                 break;
1934         }
1935 }
1936
1937 /**
1938  * i40e_create_tx_ctx Build the Tx context descriptor
1939  * @tx_ring:  ring to create the descriptor on
1940  * @cd_type_cmd_tso_mss: Quad Word 1
1941  * @cd_tunneling: Quad Word 0 - bits 0-31
1942  * @cd_l2tag2: Quad Word 0 - bits 32-63
1943  **/
1944 static void i40e_create_tx_ctx(struct i40e_ring *tx_ring,
1945                                const u64 cd_type_cmd_tso_mss,
1946                                const u32 cd_tunneling, const u32 cd_l2tag2)
1947 {
1948         struct i40e_tx_context_desc *context_desc;
1949         int i = tx_ring->next_to_use;
1950
1951         if ((cd_type_cmd_tso_mss == I40E_TX_DESC_DTYPE_CONTEXT) &&
1952             !cd_tunneling && !cd_l2tag2)
1953                 return;
1954
1955         /* grab the next descriptor */
1956         context_desc = I40E_TX_CTXTDESC(tx_ring, i);
1957
1958         i++;
1959         tx_ring->next_to_use = (i < tx_ring->count) ? i : 0;
1960
1961         /* cpu_to_le32 and assign to struct fields */
1962         context_desc->tunneling_params = cpu_to_le32(cd_tunneling);
1963         context_desc->l2tag2 = cpu_to_le16(cd_l2tag2);
1964         context_desc->type_cmd_tso_mss = cpu_to_le64(cd_type_cmd_tso_mss);
1965 }
1966
1967 /**
1968  * i40e_tx_map - Build the Tx descriptor
1969  * @tx_ring:  ring to send buffer on
1970  * @skb:      send buffer
1971  * @first:    first buffer info buffer to use
1972  * @tx_flags: collected send information
1973  * @hdr_len:  size of the packet header
1974  * @td_cmd:   the command field in the descriptor
1975  * @td_offset: offset for checksum or crc
1976  **/
1977 static void i40e_tx_map(struct i40e_ring *tx_ring, struct sk_buff *skb,
1978                         struct i40e_tx_buffer *first, u32 tx_flags,
1979                         const u8 hdr_len, u32 td_cmd, u32 td_offset)
1980 {
1981         unsigned int data_len = skb->data_len;
1982         unsigned int size = skb_headlen(skb);
1983         struct skb_frag_struct *frag;
1984         struct i40e_tx_buffer *tx_bi;
1985         struct i40e_tx_desc *tx_desc;
1986         u16 i = tx_ring->next_to_use;
1987         u32 td_tag = 0;
1988         dma_addr_t dma;
1989         u16 gso_segs;
1990
1991         if (tx_flags & I40E_TX_FLAGS_HW_VLAN) {
1992                 td_cmd |= I40E_TX_DESC_CMD_IL2TAG1;
1993                 td_tag = (tx_flags & I40E_TX_FLAGS_VLAN_MASK) >>
1994                          I40E_TX_FLAGS_VLAN_SHIFT;
1995         }
1996
1997         if (tx_flags & (I40E_TX_FLAGS_TSO | I40E_TX_FLAGS_FSO))
1998                 gso_segs = skb_shinfo(skb)->gso_segs;
1999         else
2000                 gso_segs = 1;
2001
2002         /* multiply data chunks by size of headers */
2003         first->bytecount = skb->len - hdr_len + (gso_segs * hdr_len);
2004         first->gso_segs = gso_segs;
2005         first->skb = skb;
2006         first->tx_flags = tx_flags;
2007
2008         dma = dma_map_single(tx_ring->dev, skb->data, size, DMA_TO_DEVICE);
2009
2010         tx_desc = I40E_TX_DESC(tx_ring, i);
2011         tx_bi = first;
2012
2013         for (frag = &skb_shinfo(skb)->frags[0];; frag++) {
2014                 if (dma_mapping_error(tx_ring->dev, dma))
2015                         goto dma_error;
2016
2017                 /* record length, and DMA address */
2018                 dma_unmap_len_set(tx_bi, len, size);
2019                 dma_unmap_addr_set(tx_bi, dma, dma);
2020
2021                 tx_desc->buffer_addr = cpu_to_le64(dma);
2022
2023                 while (unlikely(size > I40E_MAX_DATA_PER_TXD)) {
2024                         tx_desc->cmd_type_offset_bsz =
2025                                 build_ctob(td_cmd, td_offset,
2026                                            I40E_MAX_DATA_PER_TXD, td_tag);
2027
2028                         tx_desc++;
2029                         i++;
2030                         if (i == tx_ring->count) {
2031                                 tx_desc = I40E_TX_DESC(tx_ring, 0);
2032                                 i = 0;
2033                         }
2034
2035                         dma += I40E_MAX_DATA_PER_TXD;
2036                         size -= I40E_MAX_DATA_PER_TXD;
2037
2038                         tx_desc->buffer_addr = cpu_to_le64(dma);
2039                 }
2040
2041                 if (likely(!data_len))
2042                         break;
2043
2044                 tx_desc->cmd_type_offset_bsz = build_ctob(td_cmd, td_offset,
2045                                                           size, td_tag);
2046
2047                 tx_desc++;
2048                 i++;
2049                 if (i == tx_ring->count) {
2050                         tx_desc = I40E_TX_DESC(tx_ring, 0);
2051                         i = 0;
2052                 }
2053
2054                 size = skb_frag_size(frag);
2055                 data_len -= size;
2056
2057                 dma = skb_frag_dma_map(tx_ring->dev, frag, 0, size,
2058                                        DMA_TO_DEVICE);
2059
2060                 tx_bi = &tx_ring->tx_bi[i];
2061         }
2062
2063         /* Place RS bit on last descriptor of any packet that spans across the
2064          * 4th descriptor (WB_STRIDE aka 0x3) in a 64B cacheline.
2065          */
2066 #define WB_STRIDE 0x3
2067         if (((i & WB_STRIDE) != WB_STRIDE) &&
2068             (first <= &tx_ring->tx_bi[i]) &&
2069             (first >= &tx_ring->tx_bi[i & ~WB_STRIDE])) {
2070                 tx_desc->cmd_type_offset_bsz =
2071                         build_ctob(td_cmd, td_offset, size, td_tag) |
2072                         cpu_to_le64((u64)I40E_TX_DESC_CMD_EOP <<
2073                                          I40E_TXD_QW1_CMD_SHIFT);
2074         } else {
2075                 tx_desc->cmd_type_offset_bsz =
2076                         build_ctob(td_cmd, td_offset, size, td_tag) |
2077                         cpu_to_le64((u64)I40E_TXD_CMD <<
2078                                          I40E_TXD_QW1_CMD_SHIFT);
2079         }
2080
2081         netdev_tx_sent_queue(netdev_get_tx_queue(tx_ring->netdev,
2082                                                  tx_ring->queue_index),
2083                              first->bytecount);
2084
2085         /* set the timestamp */
2086         first->time_stamp = jiffies;
2087
2088         /* Force memory writes to complete before letting h/w
2089          * know there are new descriptors to fetch.  (Only
2090          * applicable for weak-ordered memory model archs,
2091          * such as IA-64).
2092          */
2093         wmb();
2094
2095         /* set next_to_watch value indicating a packet is present */
2096         first->next_to_watch = tx_desc;
2097
2098         i++;
2099         if (i == tx_ring->count)
2100                 i = 0;
2101
2102         tx_ring->next_to_use = i;
2103
2104         /* notify HW of packet */
2105         writel(i, tx_ring->tail);
2106
2107         return;
2108
2109 dma_error:
2110         dev_info(tx_ring->dev, "TX DMA map failed\n");
2111
2112         /* clear dma mappings for failed tx_bi map */
2113         for (;;) {
2114                 tx_bi = &tx_ring->tx_bi[i];
2115                 i40e_unmap_and_free_tx_resource(tx_ring, tx_bi);
2116                 if (tx_bi == first)
2117                         break;
2118                 if (i == 0)
2119                         i = tx_ring->count;
2120                 i--;
2121         }
2122
2123         tx_ring->next_to_use = i;
2124 }
2125
2126 /**
2127  * __i40e_maybe_stop_tx - 2nd level check for tx stop conditions
2128  * @tx_ring: the ring to be checked
2129  * @size:    the size buffer we want to assure is available
2130  *
2131  * Returns -EBUSY if a stop is needed, else 0
2132  **/
2133 static inline int __i40e_maybe_stop_tx(struct i40e_ring *tx_ring, int size)
2134 {
2135         netif_stop_subqueue(tx_ring->netdev, tx_ring->queue_index);
2136         /* Memory barrier before checking head and tail */
2137         smp_mb();
2138
2139         /* Check again in a case another CPU has just made room available. */
2140         if (likely(I40E_DESC_UNUSED(tx_ring) < size))
2141                 return -EBUSY;
2142
2143         /* A reprieve! - use start_queue because it doesn't call schedule */
2144         netif_start_subqueue(tx_ring->netdev, tx_ring->queue_index);
2145         ++tx_ring->tx_stats.restart_queue;
2146         return 0;
2147 }
2148
2149 /**
2150  * i40e_maybe_stop_tx - 1st level check for tx stop conditions
2151  * @tx_ring: the ring to be checked
2152  * @size:    the size buffer we want to assure is available
2153  *
2154  * Returns 0 if stop is not needed
2155  **/
2156 static int i40e_maybe_stop_tx(struct i40e_ring *tx_ring, int size)
2157 {
2158         if (likely(I40E_DESC_UNUSED(tx_ring) >= size))
2159                 return 0;
2160         return __i40e_maybe_stop_tx(tx_ring, size);
2161 }
2162
2163 /**
2164  * i40e_xmit_descriptor_count - calculate number of tx descriptors needed
2165  * @skb:     send buffer
2166  * @tx_ring: ring to send buffer on
2167  *
2168  * Returns number of data descriptors needed for this skb. Returns 0 to indicate
2169  * there is not enough descriptors available in this ring since we need at least
2170  * one descriptor.
2171  **/
2172 static int i40e_xmit_descriptor_count(struct sk_buff *skb,
2173                                       struct i40e_ring *tx_ring)
2174 {
2175 #if PAGE_SIZE > I40E_MAX_DATA_PER_TXD
2176         unsigned int f;
2177 #endif
2178         int count = 0;
2179
2180         /* need: 1 descriptor per page * PAGE_SIZE/I40E_MAX_DATA_PER_TXD,
2181          *       + 1 desc for skb_head_len/I40E_MAX_DATA_PER_TXD,
2182          *       + 4 desc gap to avoid the cache line where head is,
2183          *       + 1 desc for context descriptor,
2184          * otherwise try next time
2185          */
2186 #if PAGE_SIZE > I40E_MAX_DATA_PER_TXD
2187         for (f = 0; f < skb_shinfo(skb)->nr_frags; f++)
2188                 count += TXD_USE_COUNT(skb_shinfo(skb)->frags[f].size);
2189 #else
2190         count += skb_shinfo(skb)->nr_frags;
2191 #endif
2192         count += TXD_USE_COUNT(skb_headlen(skb));
2193         if (i40e_maybe_stop_tx(tx_ring, count + 4 + 1)) {
2194                 tx_ring->tx_stats.tx_busy++;
2195                 return 0;
2196         }
2197         return count;
2198 }
2199
2200 /**
2201  * i40e_xmit_frame_ring - Sends buffer on Tx ring
2202  * @skb:     send buffer
2203  * @tx_ring: ring to send buffer on
2204  *
2205  * Returns NETDEV_TX_OK if sent, else an error code
2206  **/
2207 static netdev_tx_t i40e_xmit_frame_ring(struct sk_buff *skb,
2208                                         struct i40e_ring *tx_ring)
2209 {
2210         u64 cd_type_cmd_tso_mss = I40E_TX_DESC_DTYPE_CONTEXT;
2211         u32 cd_tunneling = 0, cd_l2tag2 = 0;
2212         struct i40e_tx_buffer *first;
2213         u32 td_offset = 0;
2214         u32 tx_flags = 0;
2215         __be16 protocol;
2216         u32 td_cmd = 0;
2217         u8 hdr_len = 0;
2218         int tsyn;
2219         int tso;
2220         if (0 == i40e_xmit_descriptor_count(skb, tx_ring))
2221                 return NETDEV_TX_BUSY;
2222
2223         /* prepare the xmit flags */
2224         if (i40e_tx_prepare_vlan_flags(skb, tx_ring, &tx_flags))
2225                 goto out_drop;
2226
2227         /* obtain protocol of skb */
2228         protocol = skb->protocol;
2229
2230         /* record the location of the first descriptor for this packet */
2231         first = &tx_ring->tx_bi[tx_ring->next_to_use];
2232
2233         /* setup IPv4/IPv6 offloads */
2234         if (protocol == htons(ETH_P_IP))
2235                 tx_flags |= I40E_TX_FLAGS_IPV4;
2236         else if (protocol == htons(ETH_P_IPV6))
2237                 tx_flags |= I40E_TX_FLAGS_IPV6;
2238
2239         tso = i40e_tso(tx_ring, skb, tx_flags, protocol, &hdr_len,
2240                        &cd_type_cmd_tso_mss, &cd_tunneling);
2241
2242         if (tso < 0)
2243                 goto out_drop;
2244         else if (tso)
2245                 tx_flags |= I40E_TX_FLAGS_TSO;
2246
2247         skb_tx_timestamp(skb);
2248
2249         tsyn = i40e_tsyn(tx_ring, skb, tx_flags, &cd_type_cmd_tso_mss);
2250
2251         if (tsyn)
2252                 tx_flags |= I40E_TX_FLAGS_TSYN;
2253
2254         /* always enable CRC insertion offload */
2255         td_cmd |= I40E_TX_DESC_CMD_ICRC;
2256
2257         /* Always offload the checksum, since it's in the data descriptor */
2258         if (skb->ip_summed == CHECKSUM_PARTIAL) {
2259                 tx_flags |= I40E_TX_FLAGS_CSUM;
2260
2261                 i40e_tx_enable_csum(skb, tx_flags, &td_cmd, &td_offset,
2262                                     tx_ring, &cd_tunneling);
2263         }
2264
2265         i40e_create_tx_ctx(tx_ring, cd_type_cmd_tso_mss,
2266                            cd_tunneling, cd_l2tag2);
2267
2268         /* Add Flow Director ATR if it's enabled.
2269          *
2270          * NOTE: this must always be directly before the data descriptor.
2271          */
2272         i40e_atr(tx_ring, skb, tx_flags, protocol);
2273
2274         i40e_tx_map(tx_ring, skb, first, tx_flags, hdr_len,
2275                     td_cmd, td_offset);
2276
2277         i40e_maybe_stop_tx(tx_ring, DESC_NEEDED);
2278
2279         return NETDEV_TX_OK;
2280
2281 out_drop:
2282         dev_kfree_skb_any(skb);
2283         return NETDEV_TX_OK;
2284 }
2285
2286 /**
2287  * i40e_lan_xmit_frame - Selects the correct VSI and Tx queue to send buffer
2288  * @skb:    send buffer
2289  * @netdev: network interface device structure
2290  *
2291  * Returns NETDEV_TX_OK if sent, else an error code
2292  **/
2293 netdev_tx_t i40e_lan_xmit_frame(struct sk_buff *skb, struct net_device *netdev)
2294 {
2295         struct i40e_netdev_priv *np = netdev_priv(netdev);
2296         struct i40e_vsi *vsi = np->vsi;
2297         struct i40e_ring *tx_ring = vsi->tx_rings[skb->queue_mapping];
2298
2299         /* hardware can't handle really short frames, hardware padding works
2300          * beyond this point
2301          */
2302         if (unlikely(skb->len < I40E_MIN_TX_LEN)) {
2303                 if (skb_pad(skb, I40E_MIN_TX_LEN - skb->len))
2304                         return NETDEV_TX_OK;
2305                 skb->len = I40E_MIN_TX_LEN;
2306                 skb_set_tail_pointer(skb, I40E_MIN_TX_LEN);
2307         }
2308
2309         return i40e_xmit_frame_ring(skb, tx_ring);
2310 }