drivers/net/ethernet/broadcom/bnx2x/bnx2x_cmn.c

   1 /* bnx2x_cmn.c: Broadcom Everest network driver.
   2  *
   3  * Copyright (c) 2007-2012 Broadcom Corporation
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of the GNU General Public License as published by
   7  * the Free Software Foundation.
   8  *
   9  * Maintained by: Eilon Greenstein <eilong@broadcom.com>
  10  * Written by: Eliezer Tamir
  11  * Based on code from Michael Chan's bnx2 driver
  12  * UDP CSUM errata workaround by Arik Gendelman
  13  * Slowpath and fastpath rework by Vladislav Zolotarov
  14  * Statistics and Link management by Yitchak Gertner
  15  *
  16  */
  17
  18 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  19
  20 #include <linux/etherdevice.h>
  21 #include <linux/if_vlan.h>
  22 #include <linux/interrupt.h>
  23 #include <linux/ip.h>
  24 #include <net/ipv6.h>
  25 #include <net/ip6_checksum.h>
  26 #include <linux/prefetch.h>
  27 #include "bnx2x_cmn.h"
  28 #include "bnx2x_init.h"
  29 #include "bnx2x_sp.h"
  30
  31
  32
  33 /**
  34  * bnx2x_move_fp - move content of the fastpath structure.
  35  *
  36  * @bp:         driver handle
  37  * @from:       source FP index
  38  * @to:         destination FP index
  39  *
  40  * Makes sure the contents of the bp->fp[to].napi is kept
  41  * intact. This is done by first copying the napi struct from
  42  * the target to the source, and then mem copying the entire
  43  * source onto the target. Update txdata pointers and related
  44  * content.
  45  */
  46 static inline void bnx2x_move_fp(struct bnx2x *bp, int from, int to)
  47 {
  48         struct bnx2x_fastpath *from_fp = &bp->fp[from];
  49         struct bnx2x_fastpath *to_fp = &bp->fp[to];
  50         struct bnx2x_sp_objs *from_sp_objs = &bp->sp_objs[from];
  51         struct bnx2x_sp_objs *to_sp_objs = &bp->sp_objs[to];
  52         struct bnx2x_fp_stats *from_fp_stats = &bp->fp_stats[from];
  53         struct bnx2x_fp_stats *to_fp_stats = &bp->fp_stats[to];
  54         int old_max_eth_txqs, new_max_eth_txqs;
  55         int old_txdata_index = 0, new_txdata_index = 0;
  56
  57         /* Copy the NAPI object as it has been already initialized */
  58         from_fp->napi = to_fp->napi;
  59
  60         /* Move bnx2x_fastpath contents */
  61         memcpy(to_fp, from_fp, sizeof(*to_fp));
  62         to_fp->index = to;
  63
  64         /* move sp_objs contents as well, as their indices match fp ones */
  65         memcpy(to_sp_objs, from_sp_objs, sizeof(*to_sp_objs));
  66
  67         /* move fp_stats contents as well, as their indices match fp ones */
  68         memcpy(to_fp_stats, from_fp_stats, sizeof(*to_fp_stats));
  69
  70         /* Update txdata pointers in fp and move txdata content accordingly:
  71          * Each fp consumes 'max_cos' txdata structures, so the index should be
  72          * decremented by max_cos x delta.
  73          */
  74
  75         old_max_eth_txqs = BNX2X_NUM_ETH_QUEUES(bp) * (bp)->max_cos;
  76         new_max_eth_txqs = (BNX2X_NUM_ETH_QUEUES(bp) - from + to) *
  77                                 (bp)->max_cos;
  78         if (from == FCOE_IDX(bp)) {
  79                 old_txdata_index = old_max_eth_txqs + FCOE_TXQ_IDX_OFFSET;
  80                 new_txdata_index = new_max_eth_txqs + FCOE_TXQ_IDX_OFFSET;
  81         }
  82
  83         memcpy(&bp->bnx2x_txq[old_txdata_index],
  84                &bp->bnx2x_txq[new_txdata_index],
  85                sizeof(struct bnx2x_fp_txdata));
  86         to_fp->txdata_ptr[0] = &bp->bnx2x_txq[new_txdata_index];
  87 }
  88
  89 int load_count[2][3] = { {0} }; /* per-path: 0-common, 1-port0, 2-port1 */
  90
  91 /* free skb in the packet ring at pos idx
  92  * return idx of last bd freed
  93  */
  94 static u16 bnx2x_free_tx_pkt(struct bnx2x *bp, struct bnx2x_fp_txdata *txdata,
  95                              u16 idx, unsigned int *pkts_compl,
  96                              unsigned int *bytes_compl)
  97 {
  98         struct sw_tx_bd *tx_buf = &txdata->tx_buf_ring[idx];
  99         struct eth_tx_start_bd *tx_start_bd;
 100         struct eth_tx_bd *tx_data_bd;
 101         struct sk_buff *skb = tx_buf->skb;
 102         u16 bd_idx = TX_BD(tx_buf->first_bd), new_cons;
 103         int nbd;
 104
 105         /* prefetch skb end pointer to speedup dev_kfree_skb() */
 106         prefetch(&skb->end);
 107
 108         DP(NETIF_MSG_TX_DONE, "fp[%d]: pkt_idx %d  buff @(%p)->skb %p\n",
 109            txdata->txq_index, idx, tx_buf, skb);
 110
 111         /* unmap first bd */
 112         tx_start_bd = &txdata->tx_desc_ring[bd_idx].start_bd;
 113         dma_unmap_single(&bp->pdev->dev, BD_UNMAP_ADDR(tx_start_bd),
 114                          BD_UNMAP_LEN(tx_start_bd), DMA_TO_DEVICE);
 115
 116
 117         nbd = le16_to_cpu(tx_start_bd->nbd) - 1;
 118 #ifdef BNX2X_STOP_ON_ERROR
 119         if ((nbd - 1) > (MAX_SKB_FRAGS + 2)) {
 120                 BNX2X_ERR("BAD nbd!\n");
 121                 bnx2x_panic();
 122         }
 123 #endif
 124         new_cons = nbd + tx_buf->first_bd;
 125
 126         /* Get the next bd */
 127         bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
 128
 129         /* Skip a parse bd... */
 130         --nbd;
 131         bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
 132
 133         /* ...and the TSO split header bd since they have no mapping */
 134         if (tx_buf->flags & BNX2X_TSO_SPLIT_BD) {
 135                 --nbd;
 136                 bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
 137         }
 138
 139         /* now free frags */
 140         while (nbd > 0) {
 141
 142                 tx_data_bd = &txdata->tx_desc_ring[bd_idx].reg_bd;
 143                 dma_unmap_page(&bp->pdev->dev, BD_UNMAP_ADDR(tx_data_bd),
 144                                BD_UNMAP_LEN(tx_data_bd), DMA_TO_DEVICE);
 145                 if (--nbd)
 146                         bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
 147         }
 148
 149         /* release skb */
 150         WARN_ON(!skb);
 151         if (likely(skb)) {
 152                 (*pkts_compl)++;
 153                 (*bytes_compl) += skb->len;
 154         }
 155
 156         dev_kfree_skb_any(skb);
 157         tx_buf->first_bd = 0;
 158         tx_buf->skb = NULL;
 159
 160         return new_cons;
 161 }
 162
 163 int bnx2x_tx_int(struct bnx2x *bp, struct bnx2x_fp_txdata *txdata)
 164 {
 165         struct netdev_queue *txq;
 166         u16 hw_cons, sw_cons, bd_cons = txdata->tx_bd_cons;
 167         unsigned int pkts_compl = 0, bytes_compl = 0;
 168
 169 #ifdef BNX2X_STOP_ON_ERROR
 170         if (unlikely(bp->panic))
 171                 return -1;
 172 #endif
 173
 174         txq = netdev_get_tx_queue(bp->dev, txdata->txq_index);
 175         hw_cons = le16_to_cpu(*txdata->tx_cons_sb);
 176         sw_cons = txdata->tx_pkt_cons;
 177
 178         while (sw_cons != hw_cons) {
 179                 u16 pkt_cons;
 180
 181                 pkt_cons = TX_BD(sw_cons);
 182
 183                 DP(NETIF_MSG_TX_DONE,
 184                    "queue[%d]: hw_cons %u  sw_cons %u  pkt_cons %u\n",
 185                    txdata->txq_index, hw_cons, sw_cons, pkt_cons);
 186
 187                 bd_cons = bnx2x_free_tx_pkt(bp, txdata, pkt_cons,
 188                     &pkts_compl, &bytes_compl);
 189
 190                 sw_cons++;
 191         }
 192
 193         netdev_tx_completed_queue(txq, pkts_compl, bytes_compl);
 194
 195         txdata->tx_pkt_cons = sw_cons;
 196         txdata->tx_bd_cons = bd_cons;
 197
 198         /* Need to make the tx_bd_cons update visible to start_xmit()
 199          * before checking for netif_tx_queue_stopped().  Without the
 200          * memory barrier, there is a small possibility that
 201          * start_xmit() will miss it and cause the queue to be stopped
 202          * forever.
 203          * On the other hand we need an rmb() here to ensure the proper
 204          * ordering of bit testing in the following
 205          * netif_tx_queue_stopped(txq) call.
 206          */
 207         smp_mb();
 208
 209         if (unlikely(netif_tx_queue_stopped(txq))) {
 210                 /* Taking tx_lock() is needed to prevent reenabling the queue
 211                  * while it's empty. This could have happen if rx_action() gets
 212                  * suspended in bnx2x_tx_int() after the condition before
 213                  * netif_tx_wake_queue(), while tx_action (bnx2x_start_xmit()):
 214                  *
 215                  * stops the queue->sees fresh tx_bd_cons->releases the queue->
 216                  * sends some packets consuming the whole queue again->
 217                  * stops the queue
 218                  */
 219
 220                 __netif_tx_lock(txq, smp_processor_id());
 221
 222                 if ((netif_tx_queue_stopped(txq)) &&
 223                     (bp->state == BNX2X_STATE_OPEN) &&
 224                     (bnx2x_tx_avail(bp, txdata) >= MAX_DESC_PER_TX_PKT))
 225                         netif_tx_wake_queue(txq);
 226
 227                 __netif_tx_unlock(txq);
 228         }
 229         return 0;
 230 }
 231
 232 static inline void bnx2x_update_last_max_sge(struct bnx2x_fastpath *fp,
 233                                              u16 idx)
 234 {
 235         u16 last_max = fp->last_max_sge;
 236
 237         if (SUB_S16(idx, last_max) > 0)
 238                 fp->last_max_sge = idx;
 239 }
 240
 241 static inline void bnx2x_update_sge_prod(struct bnx2x_fastpath *fp,
 242                                          u16 sge_len,
 243                                          struct eth_end_agg_rx_cqe *cqe)
 244 {
 245         struct bnx2x *bp = fp->bp;
 246         u16 last_max, last_elem, first_elem;
 247         u16 delta = 0;
 248         u16 i;
 249
 250         if (!sge_len)
 251                 return;
 252
 253         /* First mark all used pages */
 254         for (i = 0; i < sge_len; i++)
 255                 BIT_VEC64_CLEAR_BIT(fp->sge_mask,
 256                         RX_SGE(le16_to_cpu(cqe->sgl_or_raw_data.sgl[i])));
 257
 258         DP(NETIF_MSG_RX_STATUS, "fp_cqe->sgl[%d] = %d\n",
 259            sge_len - 1, le16_to_cpu(cqe->sgl_or_raw_data.sgl[sge_len - 1]));
 260
 261         /* Here we assume that the last SGE index is the biggest */
 262         prefetch((void *)(fp->sge_mask));
 263         bnx2x_update_last_max_sge(fp,
 264                 le16_to_cpu(cqe->sgl_or_raw_data.sgl[sge_len - 1]));
 265
 266         last_max = RX_SGE(fp->last_max_sge);
 267         last_elem = last_max >> BIT_VEC64_ELEM_SHIFT;
 268         first_elem = RX_SGE(fp->rx_sge_prod) >> BIT_VEC64_ELEM_SHIFT;
 269
 270         /* If ring is not full */
 271         if (last_elem + 1 != first_elem)
 272                 last_elem++;
 273
 274         /* Now update the prod */
 275         for (i = first_elem; i != last_elem; i = NEXT_SGE_MASK_ELEM(i)) {
 276                 if (likely(fp->sge_mask[i]))
 277                         break;
 278
 279                 fp->sge_mask[i] = BIT_VEC64_ELEM_ONE_MASK;
 280                 delta += BIT_VEC64_ELEM_SZ;
 281         }
 282
 283         if (delta > 0) {
 284                 fp->rx_sge_prod += delta;
 285                 /* clear page-end entries */
 286                 bnx2x_clear_sge_mask_next_elems(fp);
 287         }
 288
 289         DP(NETIF_MSG_RX_STATUS,
 290            "fp->last_max_sge = %d  fp->rx_sge_prod = %d\n",
 291            fp->last_max_sge, fp->rx_sge_prod);
 292 }
 293
 294 /* Set Toeplitz hash value in the skb using the value from the
 295  * CQE (calculated by HW).
 296  */
 297 static u32 bnx2x_get_rxhash(const struct bnx2x *bp,
 298                             const struct eth_fast_path_rx_cqe *cqe,
 299                             bool *l4_rxhash)
 300 {
 301         /* Set Toeplitz hash from CQE */
 302         if ((bp->dev->features & NETIF_F_RXHASH) &&
 303             (cqe->status_flags & ETH_FAST_PATH_RX_CQE_RSS_HASH_FLG)) {
 304                 enum eth_rss_hash_type htype;
 305
 306                 htype = cqe->status_flags & ETH_FAST_PATH_RX_CQE_RSS_HASH_TYPE;
 307                 *l4_rxhash = (htype == TCP_IPV4_HASH_TYPE) ||
 308                              (htype == TCP_IPV6_HASH_TYPE);
 309                 return le32_to_cpu(cqe->rss_hash_result);
 310         }
 311         *l4_rxhash = false;
 312         return 0;
 313 }
 314
 315 static void bnx2x_tpa_start(struct bnx2x_fastpath *fp, u16 queue,
 316                             u16 cons, u16 prod,
 317                             struct eth_fast_path_rx_cqe *cqe)
 318 {
 319         struct bnx2x *bp = fp->bp;
 320         struct sw_rx_bd *cons_rx_buf = &fp->rx_buf_ring[cons];
 321         struct sw_rx_bd *prod_rx_buf = &fp->rx_buf_ring[prod];
 322         struct eth_rx_bd *prod_bd = &fp->rx_desc_ring[prod];
 323         dma_addr_t mapping;
 324         struct bnx2x_agg_info *tpa_info = &fp->tpa_info[queue];
 325         struct sw_rx_bd *first_buf = &tpa_info->first_buf;
 326
 327         /* print error if current state != stop */
 328         if (tpa_info->tpa_state != BNX2X_TPA_STOP)
 329                 BNX2X_ERR("start of bin not in stop [%d]\n", queue);
 330
 331         /* Try to map an empty data buffer from the aggregation info  */
 332         mapping = dma_map_single(&bp->pdev->dev,
 333                                  first_buf->data + NET_SKB_PAD,
 334                                  fp->rx_buf_size, DMA_FROM_DEVICE);
 335         /*
 336          *  ...if it fails - move the skb from the consumer to the producer
 337          *  and set the current aggregation state as ERROR to drop it
 338          *  when TPA_STOP arrives.
 339          */
 340
 341         if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) {
 342                 /* Move the BD from the consumer to the producer */
 343                 bnx2x_reuse_rx_data(fp, cons, prod);
 344                 tpa_info->tpa_state = BNX2X_TPA_ERROR;
 345                 return;
 346         }
 347
 348         /* move empty data from pool to prod */
 349         prod_rx_buf->data = first_buf->data;
 350         dma_unmap_addr_set(prod_rx_buf, mapping, mapping);
 351         /* point prod_bd to new data */
 352         prod_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
 353         prod_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
 354
 355         /* move partial skb from cons to pool (don't unmap yet) */
 356         *first_buf = *cons_rx_buf;
 357
 358         /* mark bin state as START */
 359         tpa_info->parsing_flags =
 360                 le16_to_cpu(cqe->pars_flags.flags);
 361         tpa_info->vlan_tag = le16_to_cpu(cqe->vlan_tag);
 362         tpa_info->tpa_state = BNX2X_TPA_START;
 363         tpa_info->len_on_bd = le16_to_cpu(cqe->len_on_bd);
 364         tpa_info->placement_offset = cqe->placement_offset;
 365         tpa_info->rxhash = bnx2x_get_rxhash(bp, cqe, &tpa_info->l4_rxhash);
 366         if (fp->mode == TPA_MODE_GRO) {
 367                 u16 gro_size = le16_to_cpu(cqe->pkt_len_or_gro_seg_len);
 368                 tpa_info->full_page =
 369                         SGE_PAGE_SIZE * PAGES_PER_SGE / gro_size * gro_size;
 370                 tpa_info->gro_size = gro_size;
 371         }
 372
 373 #ifdef BNX2X_STOP_ON_ERROR
 374         fp->tpa_queue_used |= (1 << queue);
 375 #ifdef _ASM_GENERIC_INT_L64_H
 376         DP(NETIF_MSG_RX_STATUS, "fp->tpa_queue_used = 0x%lx\n",
 377 #else
 378         DP(NETIF_MSG_RX_STATUS, "fp->tpa_queue_used = 0x%llx\n",
 379 #endif
 380            fp->tpa_queue_used);
 381 #endif
 382 }
 383
 384 /* Timestamp option length allowed for TPA aggregation:
 385  *
 386  *              nop nop kind length echo val
 387  */
 388 #define TPA_TSTAMP_OPT_LEN      12
 389 /**
 390  * bnx2x_set_lro_mss - calculate the approximate value of the MSS
 391  *
 392  * @bp:                 driver handle
 393  * @parsing_flags:      parsing flags from the START CQE
 394  * @len_on_bd:          total length of the first packet for the
 395  *                      aggregation.
 396  *
 397  * Approximate value of the MSS for this aggregation calculated using
 398  * the first packet of it.
 399  */
 400 static u16 bnx2x_set_lro_mss(struct bnx2x *bp, u16 parsing_flags,
 401                              u16 len_on_bd)
 402 {
 403         /*
 404          * TPA arrgregation won't have either IP options or TCP options
 405          * other than timestamp or IPv6 extension headers.
 406          */
 407         u16 hdrs_len = ETH_HLEN + sizeof(struct tcphdr);
 408
 409         if (GET_FLAG(parsing_flags, PARSING_FLAGS_OVER_ETHERNET_PROTOCOL) ==
 410             PRS_FLAG_OVERETH_IPV6)
 411                 hdrs_len += sizeof(struct ipv6hdr);
 412         else /* IPv4 */
 413                 hdrs_len += sizeof(struct iphdr);
 414
 415
 416         /* Check if there was a TCP timestamp, if there is it's will
 417          * always be 12 bytes length: nop nop kind length echo val.
 418          *
 419          * Otherwise FW would close the aggregation.
 420          */
 421         if (parsing_flags & PARSING_FLAGS_TIME_STAMP_EXIST_FLAG)
 422                 hdrs_len += TPA_TSTAMP_OPT_LEN;
 423
 424         return len_on_bd - hdrs_len;
 425 }
 426
 427 static int bnx2x_alloc_rx_sge(struct bnx2x *bp,
 428                               struct bnx2x_fastpath *fp, u16 index)
 429 {
 430         struct page *page = alloc_pages(GFP_ATOMIC, PAGES_PER_SGE_SHIFT);
 431         struct sw_rx_page *sw_buf = &fp->rx_page_ring[index];
 432         struct eth_rx_sge *sge = &fp->rx_sge_ring[index];
 433         dma_addr_t mapping;
 434
 435         if (unlikely(page == NULL)) {
 436                 BNX2X_ERR("Can't alloc sge\n");
 437                 return -ENOMEM;
 438         }
 439
 440         mapping = dma_map_page(&bp->pdev->dev, page, 0,
 441                                SGE_PAGE_SIZE*PAGES_PER_SGE, DMA_FROM_DEVICE);
 442         if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) {
 443                 __free_pages(page, PAGES_PER_SGE_SHIFT);
 444                 BNX2X_ERR("Can't map sge\n");
 445                 return -ENOMEM;
 446         }
 447
 448         sw_buf->page = page;
 449         dma_unmap_addr_set(sw_buf, mapping, mapping);
 450
 451         sge->addr_hi = cpu_to_le32(U64_HI(mapping));
 452         sge->addr_lo = cpu_to_le32(U64_LO(mapping));
 453
 454         return 0;
 455 }
 456
 457 static int bnx2x_fill_frag_skb(struct bnx2x *bp, struct bnx2x_fastpath *fp,
 458                                struct bnx2x_agg_info *tpa_info,
 459                                u16 pages,
 460                                struct sk_buff *skb,
 461                                struct eth_end_agg_rx_cqe *cqe,
 462                                u16 cqe_idx)
 463 {
 464         struct sw_rx_page *rx_pg, old_rx_pg;
 465         u32 i, frag_len, frag_size;
 466         int err, j, frag_id = 0;
 467         u16 len_on_bd = tpa_info->len_on_bd;
 468         u16 full_page = 0, gro_size = 0;
 469
 470         frag_size = le16_to_cpu(cqe->pkt_len) - len_on_bd;
 471
 472         if (fp->mode == TPA_MODE_GRO) {
 473                 gro_size = tpa_info->gro_size;
 474                 full_page = tpa_info->full_page;
 475         }
 476
 477         /* This is needed in order to enable forwarding support */
 478         if (frag_size) {
 479                 skb_shinfo(skb)->gso_size = bnx2x_set_lro_mss(bp,
 480                                         tpa_info->parsing_flags, len_on_bd);
 481
 482                 /* set for GRO */
 483                 if (fp->mode == TPA_MODE_GRO)
 484                         skb_shinfo(skb)->gso_type =
 485                             (GET_FLAG(tpa_info->parsing_flags,
 486                                       PARSING_FLAGS_OVER_ETHERNET_PROTOCOL) ==
 487                                                 PRS_FLAG_OVERETH_IPV6) ?
 488                                 SKB_GSO_TCPV6 : SKB_GSO_TCPV4;
 489         }
 490
 491
 492 #ifdef BNX2X_STOP_ON_ERROR
 493         if (pages > min_t(u32, 8, MAX_SKB_FRAGS)*SGE_PAGE_SIZE*PAGES_PER_SGE) {
 494                 BNX2X_ERR("SGL length is too long: %d. CQE index is %d\n",
 495                           pages, cqe_idx);
 496                 BNX2X_ERR("cqe->pkt_len = %d\n", cqe->pkt_len);
 497                 bnx2x_panic();
 498                 return -EINVAL;
 499         }
 500 #endif
 501
 502         /* Run through the SGL and compose the fragmented skb */
 503         for (i = 0, j = 0; i < pages; i += PAGES_PER_SGE, j++) {
 504                 u16 sge_idx = RX_SGE(le16_to_cpu(cqe->sgl_or_raw_data.sgl[j]));
 505
 506                 /* FW gives the indices of the SGE as if the ring is an array
 507                    (meaning that "next" element will consume 2 indices) */
 508                 if (fp->mode == TPA_MODE_GRO)
 509                         frag_len = min_t(u32, frag_size, (u32)full_page);
 510                 else /* LRO */
 511                         frag_len = min_t(u32, frag_size,
 512                                          (u32)(SGE_PAGE_SIZE * PAGES_PER_SGE));
 513
 514                 rx_pg = &fp->rx_page_ring[sge_idx];
 515                 old_rx_pg = *rx_pg;
 516
 517                 /* If we fail to allocate a substitute page, we simply stop
 518                    where we are and drop the whole packet */
 519                 err = bnx2x_alloc_rx_sge(bp, fp, sge_idx);
 520                 if (unlikely(err)) {
 521                         bnx2x_fp_qstats(bp, fp)->rx_skb_alloc_failed++;
 522                         return err;
 523                 }
 524
 525                 /* Unmap the page as we r going to pass it to the stack */
 526                 dma_unmap_page(&bp->pdev->dev,
 527                                dma_unmap_addr(&old_rx_pg, mapping),
 528                                SGE_PAGE_SIZE*PAGES_PER_SGE, DMA_FROM_DEVICE);
 529                 /* Add one frag and update the appropriate fields in the skb */
 530                 if (fp->mode == TPA_MODE_LRO)
 531                         skb_fill_page_desc(skb, j, old_rx_pg.page, 0, frag_len);
 532                 else { /* GRO */
 533                         int rem;
 534                         int offset = 0;
 535                         for (rem = frag_len; rem > 0; rem -= gro_size) {
 536                                 int len = rem > gro_size ? gro_size : rem;
 537                                 skb_fill_page_desc(skb, frag_id++,
 538                                                    old_rx_pg.page, offset, len);
 539                                 if (offset)
 540                                         get_page(old_rx_pg.page);
 541                                 offset += len;
 542                         }
 543                 }
 544
 545                 skb->data_len += frag_len;
 546                 skb->truesize += SGE_PAGE_SIZE * PAGES_PER_SGE;
 547                 skb->len += frag_len;
 548
 549                 frag_size -= frag_len;
 550         }
 551
 552         return 0;
 553 }
 554
 555 static void bnx2x_frag_free(const struct bnx2x_fastpath *fp, void *data)
 556 {
 557         if (fp->rx_frag_size)
 558                 put_page(virt_to_head_page(data));
 559         else
 560                 kfree(data);
 561 }
 562
 563 static void *bnx2x_frag_alloc(const struct bnx2x_fastpath *fp)
 564 {
 565         if (fp->rx_frag_size)
 566                 return netdev_alloc_frag(fp->rx_frag_size);
 567
 568         return kmalloc(fp->rx_buf_size + NET_SKB_PAD, GFP_ATOMIC);
 569 }
 570
 571
 572 static void bnx2x_tpa_stop(struct bnx2x *bp, struct bnx2x_fastpath *fp,
 573                            struct bnx2x_agg_info *tpa_info,
 574                            u16 pages,
 575                            struct eth_end_agg_rx_cqe *cqe,
 576                            u16 cqe_idx)
 577 {
 578         struct sw_rx_bd *rx_buf = &tpa_info->first_buf;
 579         u8 pad = tpa_info->placement_offset;
 580         u16 len = tpa_info->len_on_bd;
 581         struct sk_buff *skb = NULL;
 582         u8 *new_data, *data = rx_buf->data;
 583         u8 old_tpa_state = tpa_info->tpa_state;
 584
 585         tpa_info->tpa_state = BNX2X_TPA_STOP;
 586
 587         /* If we there was an error during the handling of the TPA_START -
 588          * drop this aggregation.
 589          */
 590         if (old_tpa_state == BNX2X_TPA_ERROR)
 591                 goto drop;
 592
 593         /* Try to allocate the new data */
 594         new_data = bnx2x_frag_alloc(fp);
 595         /* Unmap skb in the pool anyway, as we are going to change
 596            pool entry status to BNX2X_TPA_STOP even if new skb allocation
 597            fails. */
 598         dma_unmap_single(&bp->pdev->dev, dma_unmap_addr(rx_buf, mapping),
 599                          fp->rx_buf_size, DMA_FROM_DEVICE);
 600         if (likely(new_data))
 601                 skb = build_skb(data, fp->rx_frag_size);
 602
 603         if (likely(skb)) {
 604 #ifdef BNX2X_STOP_ON_ERROR
 605                 if (pad + len > fp->rx_buf_size) {
 606                         BNX2X_ERR("skb_put is about to fail...  pad %d  len %d  rx_buf_size %d\n",
 607                                   pad, len, fp->rx_buf_size);
 608                         bnx2x_panic();
 609                         return;
 610                 }
 611 #endif
 612
 613                 skb_reserve(skb, pad + NET_SKB_PAD);
 614                 skb_put(skb, len);
 615                 skb->rxhash = tpa_info->rxhash;
 616                 skb->l4_rxhash = tpa_info->l4_rxhash;
 617
 618                 skb->protocol = eth_type_trans(skb, bp->dev);
 619                 skb->ip_summed = CHECKSUM_UNNECESSARY;
 620
 621                 if (!bnx2x_fill_frag_skb(bp, fp, tpa_info, pages,
 622                                          skb, cqe, cqe_idx)) {
 623                         if (tpa_info->parsing_flags & PARSING_FLAGS_VLAN)
 624                                 __vlan_hwaccel_put_tag(skb, tpa_info->vlan_tag);
 625                         napi_gro_receive(&fp->napi, skb);
 626                 } else {
 627                         DP(NETIF_MSG_RX_STATUS,
 628                            "Failed to allocate new pages - dropping packet!\n");
 629                         dev_kfree_skb_any(skb);
 630                 }
 631
 632
 633                 /* put new data in bin */
 634                 rx_buf->data = new_data;
 635
 636                 return;
 637         }
 638         bnx2x_frag_free(fp, new_data);
 639 drop:
 640         /* drop the packet and keep the buffer in the bin */
 641         DP(NETIF_MSG_RX_STATUS,
 642            "Failed to allocate or map a new skb - dropping packet!\n");
 643         bnx2x_fp_stats(bp, fp)->eth_q_stats.rx_skb_alloc_failed++;
 644 }
 645
 646 static int bnx2x_alloc_rx_data(struct bnx2x *bp,
 647                                struct bnx2x_fastpath *fp, u16 index)
 648 {
 649         u8 *data;
 650         struct sw_rx_bd *rx_buf = &fp->rx_buf_ring[index];
 651         struct eth_rx_bd *rx_bd = &fp->rx_desc_ring[index];
 652         dma_addr_t mapping;
 653
 654         data = bnx2x_frag_alloc(fp);
 655         if (unlikely(data == NULL))
 656                 return -ENOMEM;
 657
 658         mapping = dma_map_single(&bp->pdev->dev, data + NET_SKB_PAD,
 659                                  fp->rx_buf_size,
 660                                  DMA_FROM_DEVICE);
 661         if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) {
 662                 bnx2x_frag_free(fp, data);
 663                 BNX2X_ERR("Can't map rx data\n");
 664                 return -ENOMEM;
 665         }
 666
 667         rx_buf->data = data;
 668         dma_unmap_addr_set(rx_buf, mapping, mapping);
 669
 670         rx_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
 671         rx_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
 672
 673         return 0;
 674 }
 675
 676 static
 677 void bnx2x_csum_validate(struct sk_buff *skb, union eth_rx_cqe *cqe,
 678                                  struct bnx2x_fastpath *fp,
 679                                  struct bnx2x_eth_q_stats *qstats)
 680 {
 681         /* Do nothing if no L4 csum validation was done.
 682          * We do not check whether IP csum was validated. For IPv4 we assume
 683          * that if the card got as far as validating the L4 csum, it also
 684          * validated the IP csum. IPv6 has no IP csum.
 685          */
 686         if (cqe->fast_path_cqe.status_flags &
 687             ETH_FAST_PATH_RX_CQE_L4_XSUM_NO_VALIDATION_FLG)
 688                 return;
 689
 690         /* If L4 validation was done, check if an error was found. */
 691
 692         if (cqe->fast_path_cqe.type_error_flags &
 693             (ETH_FAST_PATH_RX_CQE_IP_BAD_XSUM_FLG |
 694              ETH_FAST_PATH_RX_CQE_L4_BAD_XSUM_FLG))
 695                 qstats->hw_csum_err++;
 696         else
 697                 skb->ip_summed = CHECKSUM_UNNECESSARY;
 698 }
 699
 700 int bnx2x_rx_int(struct bnx2x_fastpath *fp, int budget)
 701 {
 702         struct bnx2x *bp = fp->bp;
 703         u16 bd_cons, bd_prod, bd_prod_fw, comp_ring_cons;
 704         u16 hw_comp_cons, sw_comp_cons, sw_comp_prod;
 705         int rx_pkt = 0;
 706
 707 #ifdef BNX2X_STOP_ON_ERROR
 708         if (unlikely(bp->panic))
 709                 return 0;
 710 #endif
 711
 712         /* CQ "next element" is of the size of the regular element,
 713            that's why it's ok here */
 714         hw_comp_cons = le16_to_cpu(*fp->rx_cons_sb);
 715         if ((hw_comp_cons & MAX_RCQ_DESC_CNT) == MAX_RCQ_DESC_CNT)
 716                 hw_comp_cons++;
 717
 718         bd_cons = fp->rx_bd_cons;
 719         bd_prod = fp->rx_bd_prod;
 720         bd_prod_fw = bd_prod;
 721         sw_comp_cons = fp->rx_comp_cons;
 722         sw_comp_prod = fp->rx_comp_prod;
 723
 724         /* Memory barrier necessary as speculative reads of the rx
 725          * buffer can be ahead of the index in the status block
 726          */
 727         rmb();
 728
 729         DP(NETIF_MSG_RX_STATUS,
 730            "queue[%d]:  hw_comp_cons %u  sw_comp_cons %u\n",
 731            fp->index, hw_comp_cons, sw_comp_cons);
 732
 733         while (sw_comp_cons != hw_comp_cons) {
 734                 struct sw_rx_bd *rx_buf = NULL;
 735                 struct sk_buff *skb;
 736                 union eth_rx_cqe *cqe;
 737                 struct eth_fast_path_rx_cqe *cqe_fp;
 738                 u8 cqe_fp_flags;
 739                 enum eth_rx_cqe_type cqe_fp_type;
 740                 u16 len, pad, queue;
 741                 u8 *data;
 742                 bool l4_rxhash;
 743
 744 #ifdef BNX2X_STOP_ON_ERROR
 745                 if (unlikely(bp->panic))
 746                         return 0;
 747 #endif
 748
 749                 comp_ring_cons = RCQ_BD(sw_comp_cons);
 750                 bd_prod = RX_BD(bd_prod);
 751                 bd_cons = RX_BD(bd_cons);
 752
 753                 cqe = &fp->rx_comp_ring[comp_ring_cons];
 754                 cqe_fp = &cqe->fast_path_cqe;
 755                 cqe_fp_flags = cqe_fp->type_error_flags;
 756                 cqe_fp_type = cqe_fp_flags & ETH_FAST_PATH_RX_CQE_TYPE;
 757
 758                 DP(NETIF_MSG_RX_STATUS,
 759                    "CQE type %x  err %x  status %x  queue %x  vlan %x  len %u\n",
 760                    CQE_TYPE(cqe_fp_flags),
 761                    cqe_fp_flags, cqe_fp->status_flags,
 762                    le32_to_cpu(cqe_fp->rss_hash_result),
 763                    le16_to_cpu(cqe_fp->vlan_tag),
 764                    le16_to_cpu(cqe_fp->pkt_len_or_gro_seg_len));
 765
 766                 /* is this a slowpath msg? */
 767                 if (unlikely(CQE_TYPE_SLOW(cqe_fp_type))) {
 768                         bnx2x_sp_event(fp, cqe);
 769                         goto next_cqe;
 770                 }
 771
 772                 rx_buf = &fp->rx_buf_ring[bd_cons];
 773                 data = rx_buf->data;
 774
 775                 if (!CQE_TYPE_FAST(cqe_fp_type)) {
 776                         struct bnx2x_agg_info *tpa_info;
 777                         u16 frag_size, pages;
 778 #ifdef BNX2X_STOP_ON_ERROR
 779                         /* sanity check */
 780                         if (fp->disable_tpa &&
 781                             (CQE_TYPE_START(cqe_fp_type) ||
 782                              CQE_TYPE_STOP(cqe_fp_type)))
 783                                 BNX2X_ERR("START/STOP packet while disable_tpa type %x\n",
 784                                           CQE_TYPE(cqe_fp_type));
 785 #endif
 786
 787                         if (CQE_TYPE_START(cqe_fp_type)) {
 788                                 u16 queue = cqe_fp->queue_index;
 789                                 DP(NETIF_MSG_RX_STATUS,
 790                                    "calling tpa_start on queue %d\n",
 791                                    queue);
 792
 793                                 bnx2x_tpa_start(fp, queue,
 794                                                 bd_cons, bd_prod,
 795                                                 cqe_fp);
 796
 797                                 goto next_rx;
 798
 799                         }
 800                         queue = cqe->end_agg_cqe.queue_index;
 801                         tpa_info = &fp->tpa_info[queue];
 802                         DP(NETIF_MSG_RX_STATUS,
 803                            "calling tpa_stop on queue %d\n",
 804                            queue);
 805
 806                         frag_size = le16_to_cpu(cqe->end_agg_cqe.pkt_len) -
 807                                     tpa_info->len_on_bd;
 808
 809                         if (fp->mode == TPA_MODE_GRO)
 810                                 pages = (frag_size + tpa_info->full_page - 1) /
 811                                          tpa_info->full_page;
 812                         else
 813                                 pages = SGE_PAGE_ALIGN(frag_size) >>
 814                                         SGE_PAGE_SHIFT;
 815
 816                         bnx2x_tpa_stop(bp, fp, tpa_info, pages,
 817                                        &cqe->end_agg_cqe, comp_ring_cons);
 818 #ifdef BNX2X_STOP_ON_ERROR
 819                         if (bp->panic)
 820                                 return 0;
 821 #endif
 822
 823                         bnx2x_update_sge_prod(fp, pages, &cqe->end_agg_cqe);
 824                         goto next_cqe;
 825                 }
 826                 /* non TPA */
 827                 len = le16_to_cpu(cqe_fp->pkt_len_or_gro_seg_len);
 828                 pad = cqe_fp->placement_offset;
 829                 dma_sync_single_for_cpu(&bp->pdev->dev,
 830                                         dma_unmap_addr(rx_buf, mapping),
 831                                         pad + RX_COPY_THRESH,
 832                                         DMA_FROM_DEVICE);
 833                 pad += NET_SKB_PAD;
 834                 prefetch(data + pad); /* speedup eth_type_trans() */
 835                 /* is this an error packet? */
 836                 if (unlikely(cqe_fp_flags & ETH_RX_ERROR_FALGS)) {
 837                         DP(NETIF_MSG_RX_ERR | NETIF_MSG_RX_STATUS,
 838                            "ERROR  flags %x  rx packet %u\n",
 839                            cqe_fp_flags, sw_comp_cons);
 840                         bnx2x_fp_qstats(bp, fp)->rx_err_discard_pkt++;
 841                         goto reuse_rx;
 842                 }
 843
 844                 /* Since we don't have a jumbo ring
 845                  * copy small packets if mtu > 1500
 846                  */
 847                 if ((bp->dev->mtu > ETH_MAX_PACKET_SIZE) &&
 848                     (len <= RX_COPY_THRESH)) {
 849                         skb = netdev_alloc_skb_ip_align(bp->dev, len);
 850                         if (skb == NULL) {
 851                                 DP(NETIF_MSG_RX_ERR | NETIF_MSG_RX_STATUS,
 852                                    "ERROR  packet dropped because of alloc failure\n");
 853                                 bnx2x_fp_qstats(bp, fp)->rx_skb_alloc_failed++;
 854                                 goto reuse_rx;
 855                         }
 856                         memcpy(skb->data, data + pad, len);
 857                         bnx2x_reuse_rx_data(fp, bd_cons, bd_prod);
 858                 } else {
 859                         if (likely(bnx2x_alloc_rx_data(bp, fp, bd_prod) == 0)) {
 860                                 dma_unmap_single(&bp->pdev->dev,
 861                                                  dma_unmap_addr(rx_buf, mapping),
 862                                                  fp->rx_buf_size,
 863                                                  DMA_FROM_DEVICE);
 864                                 skb = build_skb(data, fp->rx_frag_size);
 865                                 if (unlikely(!skb)) {
 866                                         bnx2x_frag_free(fp, data);
 867                                         bnx2x_fp_qstats(bp, fp)->
 868                                                         rx_skb_alloc_failed++;
 869                                         goto next_rx;
 870                                 }
 871                                 skb_reserve(skb, pad);
 872                         } else {
 873                                 DP(NETIF_MSG_RX_ERR | NETIF_MSG_RX_STATUS,
 874                                    "ERROR  packet dropped because of alloc failure\n");
 875                                 bnx2x_fp_qstats(bp, fp)->rx_skb_alloc_failed++;
 876 reuse_rx:
 877                                 bnx2x_reuse_rx_data(fp, bd_cons, bd_prod);
 878                                 goto next_rx;
 879                         }
 880                 }
 881
 882                 skb_put(skb, len);
 883                 skb->protocol = eth_type_trans(skb, bp->dev);
 884
 885                 /* Set Toeplitz hash for a none-LRO skb */
 886                 skb->rxhash = bnx2x_get_rxhash(bp, cqe_fp, &l4_rxhash);
 887                 skb->l4_rxhash = l4_rxhash;
 888
 889                 skb_checksum_none_assert(skb);
 890
 891                 if (bp->dev->features & NETIF_F_RXCSUM)
 892                         bnx2x_csum_validate(skb, cqe, fp,
 893                                             bnx2x_fp_qstats(bp, fp));
 894
 895                 skb_record_rx_queue(skb, fp->rx_queue);
 896
 897                 if (le16_to_cpu(cqe_fp->pars_flags.flags) &
 898                     PARSING_FLAGS_VLAN)
 899                         __vlan_hwaccel_put_tag(skb,
 900                                                le16_to_cpu(cqe_fp->vlan_tag));
 901                 napi_gro_receive(&fp->napi, skb);
 902
 903
 904 next_rx:
 905                 rx_buf->data = NULL;
 906
 907                 bd_cons = NEXT_RX_IDX(bd_cons);
 908                 bd_prod = NEXT_RX_IDX(bd_prod);
 909                 bd_prod_fw = NEXT_RX_IDX(bd_prod_fw);
 910                 rx_pkt++;
 911 next_cqe:
 912                 sw_comp_prod = NEXT_RCQ_IDX(sw_comp_prod);
 913                 sw_comp_cons = NEXT_RCQ_IDX(sw_comp_cons);
 914
 915                 if (rx_pkt == budget)
 916                         break;
 917         } /* while */
 918
 919         fp->rx_bd_cons = bd_cons;
 920         fp->rx_bd_prod = bd_prod_fw;
 921         fp->rx_comp_cons = sw_comp_cons;
 922         fp->rx_comp_prod = sw_comp_prod;
 923
 924         /* Update producers */
 925         bnx2x_update_rx_prod(bp, fp, bd_prod_fw, sw_comp_prod,
 926                              fp->rx_sge_prod);
 927
 928         fp->rx_pkt += rx_pkt;
 929         fp->rx_calls++;
 930
 931         return rx_pkt;
 932 }
 933
 934 static irqreturn_t bnx2x_msix_fp_int(int irq, void *fp_cookie)
 935 {
 936         struct bnx2x_fastpath *fp = fp_cookie;
 937         struct bnx2x *bp = fp->bp;
 938         u8 cos;
 939
 940         DP(NETIF_MSG_INTR,
 941            "got an MSI-X interrupt on IDX:SB [fp %d fw_sd %d igusb %d]\n",
 942            fp->index, fp->fw_sb_id, fp->igu_sb_id);
 943         bnx2x_ack_sb(bp, fp->igu_sb_id, USTORM_ID, 0, IGU_INT_DISABLE, 0);
 944
 945 #ifdef BNX2X_STOP_ON_ERROR
 946         if (unlikely(bp->panic))
 947                 return IRQ_HANDLED;
 948 #endif
 949
 950         /* Handle Rx and Tx according to MSI-X vector */
 951         prefetch(fp->rx_cons_sb);
 952
 953         for_each_cos_in_tx_queue(fp, cos)
 954                 prefetch(fp->txdata_ptr[cos]->tx_cons_sb);
 955
 956         prefetch(&fp->sb_running_index[SM_RX_ID]);
 957         napi_schedule(&bnx2x_fp(bp, fp->index, napi));
 958
 959         return IRQ_HANDLED;
 960 }
 961
 962 /* HW Lock for shared dual port PHYs */
 963 void bnx2x_acquire_phy_lock(struct bnx2x *bp)
 964 {
 965         mutex_lock(&bp->port.phy_mutex);
 966
 967         bnx2x_acquire_hw_lock(bp, HW_LOCK_RESOURCE_MDIO);
 968 }
 969
 970 void bnx2x_release_phy_lock(struct bnx2x *bp)
 971 {
 972         bnx2x_release_hw_lock(bp, HW_LOCK_RESOURCE_MDIO);
 973
 974         mutex_unlock(&bp->port.phy_mutex);
 975 }
 976
 977 /* calculates MF speed according to current linespeed and MF configuration */
 978 u16 bnx2x_get_mf_speed(struct bnx2x *bp)
 979 {
 980         u16 line_speed = bp->link_vars.line_speed;
 981         if (IS_MF(bp)) {
 982                 u16 maxCfg = bnx2x_extract_max_cfg(bp,
 983                                                    bp->mf_config[BP_VN(bp)]);
 984
 985                 /* Calculate the current MAX line speed limit for the MF
 986                  * devices
 987                  */
 988                 if (IS_MF_SI(bp))
 989                         line_speed = (line_speed * maxCfg) / 100;
 990                 else { /* SD mode */
 991                         u16 vn_max_rate = maxCfg * 100;
 992
 993                         if (vn_max_rate < line_speed)
 994                                 line_speed = vn_max_rate;
 995                 }
 996         }
 997
 998         return line_speed;
 999 }
1000
1001 /**
1002  * bnx2x_fill_report_data - fill link report data to report
1003  *
1004  * @bp:         driver handle
1005  * @data:       link state to update
1006  *
1007  * It uses a none-atomic bit operations because is called under the mutex.
1008  */
1009 static void bnx2x_fill_report_data(struct bnx2x *bp,
1010                                    struct bnx2x_link_report_data *data)
1011 {
1012         u16 line_speed = bnx2x_get_mf_speed(bp);
1013
1014         memset(data, 0, sizeof(*data));
1015
1016         /* Fill the report data: efective line speed */
1017         data->line_speed = line_speed;
1018
1019         /* Link is down */
1020         if (!bp->link_vars.link_up || (bp->flags & MF_FUNC_DIS))
1021                 __set_bit(BNX2X_LINK_REPORT_LINK_DOWN,
1022                           &data->link_report_flags);
1023
1024         /* Full DUPLEX */
1025         if (bp->link_vars.duplex == DUPLEX_FULL)
1026                 __set_bit(BNX2X_LINK_REPORT_FD, &data->link_report_flags);
1027
1028         /* Rx Flow Control is ON */
1029         if (bp->link_vars.flow_ctrl & BNX2X_FLOW_CTRL_RX)
1030                 __set_bit(BNX2X_LINK_REPORT_RX_FC_ON, &data->link_report_flags);
1031
1032         /* Tx Flow Control is ON */
1033         if (bp->link_vars.flow_ctrl & BNX2X_FLOW_CTRL_TX)
1034                 __set_bit(BNX2X_LINK_REPORT_TX_FC_ON, &data->link_report_flags);
1035 }
1036
1037 /**
1038  * bnx2x_link_report - report link status to OS.
1039  *
1040  * @bp:         driver handle
1041  *
1042  * Calls the __bnx2x_link_report() under the same locking scheme
1043  * as a link/PHY state managing code to ensure a consistent link
1044  * reporting.
1045  */
1046
1047 void bnx2x_link_report(struct bnx2x *bp)
1048 {
1049         bnx2x_acquire_phy_lock(bp);
1050         __bnx2x_link_report(bp);
1051         bnx2x_release_phy_lock(bp);
1052 }
1053
1054 /**
1055  * __bnx2x_link_report - report link status to OS.
1056  *
1057  * @bp:         driver handle
1058  *
1059  * None atomic inmlementation.
1060  * Should be called under the phy_lock.
1061  */
1062 void __bnx2x_link_report(struct bnx2x *bp)
1063 {
1064         struct bnx2x_link_report_data cur_data;
1065
1066         /* reread mf_cfg */
1067         if (!CHIP_IS_E1(bp))
1068                 bnx2x_read_mf_cfg(bp);
1069
1070         /* Read the current link report info */
1071         bnx2x_fill_report_data(bp, &cur_data);
1072
1073         /* Don't report link down or exactly the same link status twice */
1074         if (!memcmp(&cur_data, &bp->last_reported_link, sizeof(cur_data)) ||
1075             (test_bit(BNX2X_LINK_REPORT_LINK_DOWN,
1076                       &bp->last_reported_link.link_report_flags) &&
1077              test_bit(BNX2X_LINK_REPORT_LINK_DOWN,
1078                       &cur_data.link_report_flags)))
1079                 return;
1080
1081         bp->link_cnt++;
1082
1083         /* We are going to report a new link parameters now -
1084          * remember the current data for the next time.
1085          */
1086         memcpy(&bp->last_reported_link, &cur_data, sizeof(cur_data));
1087
1088         if (test_bit(BNX2X_LINK_REPORT_LINK_DOWN,
1089                      &cur_data.link_report_flags)) {
1090                 netif_carrier_off(bp->dev);
1091                 netdev_err(bp->dev, "NIC Link is Down\n");
1092                 return;
1093         } else {
1094                 const char *duplex;
1095                 const char *flow;
1096
1097                 netif_carrier_on(bp->dev);
1098
1099                 if (test_and_clear_bit(BNX2X_LINK_REPORT_FD,
1100                                        &cur_data.link_report_flags))
1101                         duplex = "full";
1102                 else
1103                         duplex = "half";
1104
1105                 /* Handle the FC at the end so that only these flags would be
1106                  * possibly set. This way we may easily check if there is no FC
1107                  * enabled.
1108                  */
1109                 if (cur_data.link_report_flags) {
1110                         if (test_bit(BNX2X_LINK_REPORT_RX_FC_ON,
1111                                      &cur_data.link_report_flags)) {
1112                                 if (test_bit(BNX2X_LINK_REPORT_TX_FC_ON,
1113                                      &cur_data.link_report_flags))
1114                                         flow = "ON - receive & transmit";
1115                                 else
1116                                         flow = "ON - receive";
1117                         } else {
1118                                 flow = "ON - transmit";
1119                         }
1120                 } else {
1121                         flow = "none";
1122                 }
1123                 netdev_info(bp->dev, "NIC Link is Up, %d Mbps %s duplex, Flow control: %s\n",
1124                             cur_data.line_speed, duplex, flow);
1125         }
1126 }
1127
1128 static void bnx2x_set_next_page_sgl(struct bnx2x_fastpath *fp)
1129 {
1130         int i;
1131
1132         for (i = 1; i <= NUM_RX_SGE_PAGES; i++) {
1133                 struct eth_rx_sge *sge;
1134
1135                 sge = &fp->rx_sge_ring[RX_SGE_CNT * i - 2];
1136                 sge->addr_hi =
1137                         cpu_to_le32(U64_HI(fp->rx_sge_mapping +
1138                         BCM_PAGE_SIZE*(i % NUM_RX_SGE_PAGES)));
1139
1140                 sge->addr_lo =
1141                         cpu_to_le32(U64_LO(fp->rx_sge_mapping +
1142                         BCM_PAGE_SIZE*(i % NUM_RX_SGE_PAGES)));
1143         }
1144 }
1145
1146 static void bnx2x_free_tpa_pool(struct bnx2x *bp,
1147                                 struct bnx2x_fastpath *fp, int last)
1148 {
1149         int i;
1150
1151         for (i = 0; i < last; i++) {
1152                 struct bnx2x_agg_info *tpa_info = &fp->tpa_info[i];
1153                 struct sw_rx_bd *first_buf = &tpa_info->first_buf;
1154                 u8 *data = first_buf->data;
1155
1156                 if (data == NULL) {
1157                         DP(NETIF_MSG_IFDOWN, "tpa bin %d empty on free\n", i);
1158                         continue;
1159                 }
1160                 if (tpa_info->tpa_state == BNX2X_TPA_START)
1161                         dma_unmap_single(&bp->pdev->dev,
1162                                          dma_unmap_addr(first_buf, mapping),
1163                                          fp->rx_buf_size, DMA_FROM_DEVICE);
1164                 bnx2x_frag_free(fp, data);
1165                 first_buf->data = NULL;
1166         }
1167 }
1168
1169 void bnx2x_init_rx_rings_cnic(struct bnx2x *bp)
1170 {
1171         int j;
1172
1173         for_each_rx_queue_cnic(bp, j) {
1174                 struct bnx2x_fastpath *fp = &bp->fp[j];
1175
1176                 fp->rx_bd_cons = 0;
1177
1178                 /* Activate BD ring */
1179                 /* Warning!
1180                  * this will generate an interrupt (to the TSTORM)
1181                  * must only be done after chip is initialized
1182                  */
1183                 bnx2x_update_rx_prod(bp, fp, fp->rx_bd_prod, fp->rx_comp_prod,
1184                                      fp->rx_sge_prod);
1185         }
1186 }
1187
1188 void bnx2x_init_rx_rings(struct bnx2x *bp)
1189 {
1190         int func = BP_FUNC(bp);
1191         u16 ring_prod;
1192         int i, j;
1193
1194         /* Allocate TPA resources */
1195         for_each_eth_queue(bp, j) {
1196                 struct bnx2x_fastpath *fp = &bp->fp[j];
1197
1198                 DP(NETIF_MSG_IFUP,
1199                    "mtu %d  rx_buf_size %d\n", bp->dev->mtu, fp->rx_buf_size);
1200
1201                 if (!fp->disable_tpa) {
1202                         /* Fill the per-aggregtion pool */
1203                         for (i = 0; i < MAX_AGG_QS(bp); i++) {
1204                                 struct bnx2x_agg_info *tpa_info =
1205                                         &fp->tpa_info[i];
1206                                 struct sw_rx_bd *first_buf =
1207                                         &tpa_info->first_buf;
1208
1209                                 first_buf->data = bnx2x_frag_alloc(fp);
1210                                 if (!first_buf->data) {
1211                                         BNX2X_ERR("Failed to allocate TPA skb pool for queue[%d] - disabling TPA on this queue!\n",
1212                                                   j);
1213                                         bnx2x_free_tpa_pool(bp, fp, i);
1214                                         fp->disable_tpa = 1;
1215                                         break;
1216                                 }
1217                                 dma_unmap_addr_set(first_buf, mapping, 0);
1218                                 tpa_info->tpa_state = BNX2X_TPA_STOP;
1219                         }
1220
1221                         /* "next page" elements initialization */
1222                         bnx2x_set_next_page_sgl(fp);
1223
1224                         /* set SGEs bit mask */
1225                         bnx2x_init_sge_ring_bit_mask(fp);
1226
1227                         /* Allocate SGEs and initialize the ring elements */
1228                         for (i = 0, ring_prod = 0;
1229                              i < MAX_RX_SGE_CNT*NUM_RX_SGE_PAGES; i++) {
1230
1231                                 if (bnx2x_alloc_rx_sge(bp, fp, ring_prod) < 0) {
1232                                         BNX2X_ERR("was only able to allocate %d rx sges\n",
1233                                                   i);
1234                                         BNX2X_ERR("disabling TPA for queue[%d]\n",
1235                                                   j);
1236                                         /* Cleanup already allocated elements */
1237                                         bnx2x_free_rx_sge_range(bp, fp,
1238                                                                 ring_prod);
1239                                         bnx2x_free_tpa_pool(bp, fp,
1240                                                             MAX_AGG_QS(bp));
1241                                         fp->disable_tpa = 1;
1242                                         ring_prod = 0;
1243                                         break;
1244                                 }
1245                                 ring_prod = NEXT_SGE_IDX(ring_prod);
1246                         }
1247
1248                         fp->rx_sge_prod = ring_prod;
1249                 }
1250         }
1251
1252         for_each_eth_queue(bp, j) {
1253                 struct bnx2x_fastpath *fp = &bp->fp[j];
1254
1255                 fp->rx_bd_cons = 0;
1256
1257                 /* Activate BD ring */
1258                 /* Warning!
1259                  * this will generate an interrupt (to the TSTORM)
1260                  * must only be done after chip is initialized
1261                  */
1262                 bnx2x_update_rx_prod(bp, fp, fp->rx_bd_prod, fp->rx_comp_prod,
1263                                      fp->rx_sge_prod);
1264
1265                 if (j != 0)
1266                         continue;
1267
1268                 if (CHIP_IS_E1(bp)) {
1269                         REG_WR(bp, BAR_USTRORM_INTMEM +
1270                                USTORM_MEM_WORKAROUND_ADDRESS_OFFSET(func),
1271                                U64_LO(fp->rx_comp_mapping));
1272                         REG_WR(bp, BAR_USTRORM_INTMEM +
1273                                USTORM_MEM_WORKAROUND_ADDRESS_OFFSET(func) + 4,
1274                                U64_HI(fp->rx_comp_mapping));
1275                 }
1276         }
1277 }
1278
1279 static void bnx2x_free_tx_skbs_queue(struct bnx2x_fastpath *fp)
1280 {
1281         u8 cos;
1282         struct bnx2x *bp = fp->bp;
1283
1284         for_each_cos_in_tx_queue(fp, cos) {
1285                 struct bnx2x_fp_txdata *txdata = fp->txdata_ptr[cos];
1286                 unsigned pkts_compl = 0, bytes_compl = 0;
1287
1288                 u16 sw_prod = txdata->tx_pkt_prod;
1289                 u16 sw_cons = txdata->tx_pkt_cons;
1290
1291                 while (sw_cons != sw_prod) {
1292                         bnx2x_free_tx_pkt(bp, txdata, TX_BD(sw_cons),
1293                                           &pkts_compl, &bytes_compl);
1294                         sw_cons++;
1295                 }
1296
1297                 netdev_tx_reset_queue(
1298                         netdev_get_tx_queue(bp->dev,
1299                                             txdata->txq_index));
1300         }
1301 }
1302
1303 static void bnx2x_free_tx_skbs_cnic(struct bnx2x *bp)
1304 {
1305         int i;
1306
1307         for_each_tx_queue_cnic(bp, i) {
1308                 bnx2x_free_tx_skbs_queue(&bp->fp[i]);
1309         }
1310 }
1311
1312 static void bnx2x_free_tx_skbs(struct bnx2x *bp)
1313 {
1314         int i;
1315
1316         for_each_eth_queue(bp, i) {
1317                 bnx2x_free_tx_skbs_queue(&bp->fp[i]);
1318         }
1319 }
1320
1321 static void bnx2x_free_rx_bds(struct bnx2x_fastpath *fp)
1322 {
1323         struct bnx2x *bp = fp->bp;
1324         int i;
1325
1326         /* ring wasn't allocated */
1327         if (fp->rx_buf_ring == NULL)
1328                 return;
1329
1330         for (i = 0; i < NUM_RX_BD; i++) {
1331                 struct sw_rx_bd *rx_buf = &fp->rx_buf_ring[i];
1332                 u8 *data = rx_buf->data;
1333
1334                 if (data == NULL)
1335                         continue;
1336                 dma_unmap_single(&bp->pdev->dev,
1337                                  dma_unmap_addr(rx_buf, mapping),
1338                                  fp->rx_buf_size, DMA_FROM_DEVICE);
1339
1340                 rx_buf->data = NULL;
1341                 bnx2x_frag_free(fp, data);
1342         }
1343 }
1344
1345 static void bnx2x_free_rx_skbs_cnic(struct bnx2x *bp)
1346 {
1347         int j;
1348
1349         for_each_rx_queue_cnic(bp, j) {
1350                 bnx2x_free_rx_bds(&bp->fp[j]);
1351         }
1352 }
1353
1354 static void bnx2x_free_rx_skbs(struct bnx2x *bp)
1355 {
1356         int j;
1357
1358         for_each_eth_queue(bp, j) {
1359                 struct bnx2x_fastpath *fp = &bp->fp[j];
1360
1361                 bnx2x_free_rx_bds(fp);
1362
1363                 if (!fp->disable_tpa)
1364                         bnx2x_free_tpa_pool(bp, fp, MAX_AGG_QS(bp));
1365         }
1366 }
1367
1368 void bnx2x_free_skbs_cnic(struct bnx2x *bp)
1369 {
1370         bnx2x_free_tx_skbs_cnic(bp);
1371         bnx2x_free_rx_skbs_cnic(bp);
1372 }
1373
1374 void bnx2x_free_skbs(struct bnx2x *bp)
1375 {
1376         bnx2x_free_tx_skbs(bp);
1377         bnx2x_free_rx_skbs(bp);
1378 }
1379
1380 void bnx2x_update_max_mf_config(struct bnx2x *bp, u32 value)
1381 {
1382         /* load old values */
1383         u32 mf_cfg = bp->mf_config[BP_VN(bp)];
1384
1385         if (value != bnx2x_extract_max_cfg(bp, mf_cfg)) {
1386                 /* leave all but MAX value */
1387                 mf_cfg &= ~FUNC_MF_CFG_MAX_BW_MASK;
1388
1389                 /* set new MAX value */
1390                 mf_cfg |= (value << FUNC_MF_CFG_MAX_BW_SHIFT)
1391                                 & FUNC_MF_CFG_MAX_BW_MASK;
1392
1393                 bnx2x_fw_command(bp, DRV_MSG_CODE_SET_MF_BW, mf_cfg);
1394         }
1395 }
1396
1397 /**
1398  * bnx2x_free_msix_irqs - free previously requested MSI-X IRQ vectors
1399  *
1400  * @bp:         driver handle
1401  * @nvecs:      number of vectors to be released
1402  */
1403 static void bnx2x_free_msix_irqs(struct bnx2x *bp, int nvecs)
1404 {
1405         int i, offset = 0;
1406
1407         if (nvecs == offset)
1408                 return;
1409         free_irq(bp->msix_table[offset].vector, bp->dev);
1410         DP(NETIF_MSG_IFDOWN, "released sp irq (%d)\n",
1411            bp->msix_table[offset].vector);
1412         offset++;
1413
1414         if (CNIC_SUPPORT(bp)) {
1415                 if (nvecs == offset)
1416                         return;
1417                 offset++;
1418         }
1419
1420         for_each_eth_queue(bp, i) {
1421                 if (nvecs == offset)
1422                         return;
1423                 DP(NETIF_MSG_IFDOWN, "about to release fp #%d->%d irq\n",
1424                    i, bp->msix_table[offset].vector);
1425
1426                 free_irq(bp->msix_table[offset++].vector, &bp->fp[i]);
1427         }
1428 }
1429
1430 void bnx2x_free_irq(struct bnx2x *bp)
1431 {
1432         if (bp->flags & USING_MSIX_FLAG &&
1433             !(bp->flags & USING_SINGLE_MSIX_FLAG))
1434                 bnx2x_free_msix_irqs(bp, BNX2X_NUM_ETH_QUEUES(bp) +
1435                                      CNIC_SUPPORT(bp) + 1);
1436         else
1437                 free_irq(bp->dev->irq, bp->dev);
1438 }
1439
1440 int bnx2x_enable_msix(struct bnx2x *bp)
1441 {
1442         int msix_vec = 0, i, rc, req_cnt;
1443
1444         bp->msix_table[msix_vec].entry = msix_vec;
1445         BNX2X_DEV_INFO("msix_table[0].entry = %d (slowpath)\n",
1446            bp->msix_table[0].entry);
1447         msix_vec++;
1448
1449         /* Cnic requires an msix vector for itself */
1450         if (CNIC_SUPPORT(bp)) {
1451                 bp->msix_table[msix_vec].entry = msix_vec;
1452                 BNX2X_DEV_INFO("msix_table[%d].entry = %d (CNIC)\n",
1453                                msix_vec, bp->msix_table[msix_vec].entry);
1454                 msix_vec++;
1455         }
1456
1457         /* We need separate vectors for ETH queues only (not FCoE) */
1458         for_each_eth_queue(bp, i) {
1459                 bp->msix_table[msix_vec].entry = msix_vec;
1460                 BNX2X_DEV_INFO("msix_table[%d].entry = %d (fastpath #%u)\n",
1461                                msix_vec, msix_vec, i);
1462                 msix_vec++;
1463         }
1464
1465         req_cnt = BNX2X_NUM_ETH_QUEUES(bp) + CNIC_SUPPORT(bp) + 1;
1466
1467         rc = pci_enable_msix(bp->pdev, &bp->msix_table[0], req_cnt);
1468
1469         /*
1470          * reconfigure number of tx/rx queues according to available
1471          * MSI-X vectors
1472          */
1473         if (rc >= BNX2X_MIN_MSIX_VEC_CNT(bp)) {
1474                 /* how less vectors we will have? */
1475                 int diff = req_cnt - rc;
1476
1477                 BNX2X_DEV_INFO("Trying to use less MSI-X vectors: %d\n", rc);
1478
1479                 rc = pci_enable_msix(bp->pdev, &bp->msix_table[0], rc);
1480
1481                 if (rc) {
1482                         BNX2X_DEV_INFO("MSI-X is not attainable rc %d\n", rc);
1483                         goto no_msix;
1484                 }
1485                 /*
1486                  * decrease number of queues by number of unallocated entries
1487                  */
1488                 bp->num_ethernet_queues -= diff;
1489                 bp->num_queues = bp->num_ethernet_queues + bp->num_cnic_queues;
1490
1491                 BNX2X_DEV_INFO("New queue configuration set: %d\n",
1492                                bp->num_queues);
1493         } else if (rc > 0) {
1494                 /* Get by with single vector */
1495                 rc = pci_enable_msix(bp->pdev, &bp->msix_table[0], 1);
1496                 if (rc) {
1497                         BNX2X_DEV_INFO("Single MSI-X is not attainable rc %d\n",
1498                                        rc);
1499                         goto no_msix;
1500                 }
1501
1502                 BNX2X_DEV_INFO("Using single MSI-X vector\n");
1503                 bp->flags |= USING_SINGLE_MSIX_FLAG;
1504
1505                 BNX2X_DEV_INFO("set number of queues to 1\n");
1506                 bp->num_ethernet_queues = 1;
1507                 bp->num_queues = bp->num_ethernet_queues + bp->num_cnic_queues;
1508         } else if (rc < 0) {
1509                 BNX2X_DEV_INFO("MSI-X is not attainable  rc %d\n", rc);
1510                 goto no_msix;
1511         }
1512
1513         bp->flags |= USING_MSIX_FLAG;
1514
1515         return 0;
1516
1517 no_msix:
1518         /* fall to INTx if not enough memory */
1519         if (rc == -ENOMEM)
1520                 bp->flags |= DISABLE_MSI_FLAG;
1521
1522         return rc;
1523 }
1524
1525 static int bnx2x_req_msix_irqs(struct bnx2x *bp)
1526 {
1527         int i, rc, offset = 0;
1528
1529         rc = request_irq(bp->msix_table[offset++].vector,
1530                          bnx2x_msix_sp_int, 0,
1531                          bp->dev->name, bp->dev);
1532         if (rc) {
1533                 BNX2X_ERR("request sp irq failed\n");
1534                 return -EBUSY;
1535         }
1536
1537         if (CNIC_SUPPORT(bp))
1538                 offset++;
1539
1540         for_each_eth_queue(bp, i) {
1541                 struct bnx2x_fastpath *fp = &bp->fp[i];
1542                 snprintf(fp->name, sizeof(fp->name), "%s-fp-%d",
1543                          bp->dev->name, i);
1544
1545                 rc = request_irq(bp->msix_table[offset].vector,
1546                                  bnx2x_msix_fp_int, 0, fp->name, fp);
1547                 if (rc) {
1548                         BNX2X_ERR("request fp #%d irq (%d) failed  rc %d\n", i,
1549                               bp->msix_table[offset].vector, rc);
1550                         bnx2x_free_msix_irqs(bp, offset);
1551                         return -EBUSY;
1552                 }
1553
1554                 offset++;
1555         }
1556
1557         i = BNX2X_NUM_ETH_QUEUES(bp);
1558         offset = 1 + CNIC_SUPPORT(bp);
1559         netdev_info(bp->dev, "using MSI-X  IRQs: sp %d  fp[%d] %d ... fp[%d] %d\n",
1560                bp->msix_table[0].vector,
1561                0, bp->msix_table[offset].vector,
1562                i - 1, bp->msix_table[offset + i - 1].vector);
1563
1564         return 0;
1565 }
1566
1567 int bnx2x_enable_msi(struct bnx2x *bp)
1568 {
1569         int rc;
1570
1571         rc = pci_enable_msi(bp->pdev);
1572         if (rc) {
1573                 BNX2X_DEV_INFO("MSI is not attainable\n");
1574                 return -1;
1575         }
1576         bp->flags |= USING_MSI_FLAG;
1577
1578         return 0;
1579 }
1580
1581 static int bnx2x_req_irq(struct bnx2x *bp)
1582 {
1583         unsigned long flags;
1584         unsigned int irq;
1585
1586         if (bp->flags & (USING_MSI_FLAG | USING_MSIX_FLAG))
1587                 flags = 0;
1588         else
1589                 flags = IRQF_SHARED;
1590
1591         if (bp->flags & USING_MSIX_FLAG)
1592                 irq = bp->msix_table[0].vector;
1593         else
1594                 irq = bp->pdev->irq;
1595
1596         return request_irq(irq, bnx2x_interrupt, flags, bp->dev->name, bp->dev);
1597 }
1598
1599 static int bnx2x_setup_irqs(struct bnx2x *bp)
1600 {
1601         int rc = 0;
1602         if (bp->flags & USING_MSIX_FLAG &&
1603             !(bp->flags & USING_SINGLE_MSIX_FLAG)) {
1604                 rc = bnx2x_req_msix_irqs(bp);
1605                 if (rc)
1606                         return rc;
1607         } else {
1608                 bnx2x_ack_int(bp);
1609                 rc = bnx2x_req_irq(bp);
1610                 if (rc) {
1611                         BNX2X_ERR("IRQ request failed  rc %d, aborting\n", rc);
1612                         return rc;
1613                 }
1614                 if (bp->flags & USING_MSI_FLAG) {
1615                         bp->dev->irq = bp->pdev->irq;
1616                         netdev_info(bp->dev, "using MSI IRQ %d\n",
1617                                     bp->dev->irq);
1618                 }
1619                 if (bp->flags & USING_MSIX_FLAG) {
1620                         bp->dev->irq = bp->msix_table[0].vector;
1621                         netdev_info(bp->dev, "using MSIX IRQ %d\n",
1622                                     bp->dev->irq);
1623                 }
1624         }
1625
1626         return 0;
1627 }
1628
1629 static void bnx2x_napi_enable_cnic(struct bnx2x *bp)
1630 {
1631         int i;
1632
1633         for_each_rx_queue_cnic(bp, i)
1634                 napi_enable(&bnx2x_fp(bp, i, napi));
1635 }
1636
1637 static void bnx2x_napi_enable(struct bnx2x *bp)
1638 {
1639         int i;
1640
1641         for_each_eth_queue(bp, i)
1642                 napi_enable(&bnx2x_fp(bp, i, napi));
1643 }
1644
1645 static void bnx2x_napi_disable_cnic(struct bnx2x *bp)
1646 {
1647         int i;
1648
1649         for_each_rx_queue_cnic(bp, i)
1650                 napi_disable(&bnx2x_fp(bp, i, napi));
1651 }
1652
1653 static void bnx2x_napi_disable(struct bnx2x *bp)
1654 {
1655         int i;
1656
1657         for_each_eth_queue(bp, i)
1658                 napi_disable(&bnx2x_fp(bp, i, napi));
1659 }
1660
1661 void bnx2x_netif_start(struct bnx2x *bp)
1662 {
1663         if (netif_running(bp->dev)) {
1664                 bnx2x_napi_enable(bp);
1665                 if (CNIC_LOADED(bp))
1666                         bnx2x_napi_enable_cnic(bp);
1667                 bnx2x_int_enable(bp);
1668                 if (bp->state == BNX2X_STATE_OPEN)
1669                         netif_tx_wake_all_queues(bp->dev);
1670         }
1671 }
1672
1673 void bnx2x_netif_stop(struct bnx2x *bp, int disable_hw)
1674 {
1675         bnx2x_int_disable_sync(bp, disable_hw);
1676         bnx2x_napi_disable(bp);
1677         if (CNIC_LOADED(bp))
1678                 bnx2x_napi_disable_cnic(bp);
1679 }
1680
1681 u16 bnx2x_select_queue(struct net_device *dev, struct sk_buff *skb)
1682 {
1683         struct bnx2x *bp = netdev_priv(dev);
1684
1685         if (CNIC_LOADED(bp) && !NO_FCOE(bp)) {
1686                 struct ethhdr *hdr = (struct ethhdr *)skb->data;
1687                 u16 ether_type = ntohs(hdr->h_proto);
1688
1689                 /* Skip VLAN tag if present */
1690                 if (ether_type == ETH_P_8021Q) {
1691                         struct vlan_ethhdr *vhdr =
1692                                 (struct vlan_ethhdr *)skb->data;
1693
1694                         ether_type = ntohs(vhdr->h_vlan_encapsulated_proto);
1695                 }
1696
1697                 /* If ethertype is FCoE or FIP - use FCoE ring */
1698                 if ((ether_type == ETH_P_FCOE) || (ether_type == ETH_P_FIP))
1699                         return bnx2x_fcoe_tx(bp, txq_index);
1700         }
1701
1702         /* select a non-FCoE queue */
1703         return __skb_tx_hash(dev, skb, BNX2X_NUM_ETH_QUEUES(bp));
1704 }
1705
1706
1707 void bnx2x_set_num_queues(struct bnx2x *bp)
1708 {
1709         /* RSS queues */
1710         bp->num_ethernet_queues = bnx2x_calc_num_queues(bp);
1711
1712         /* override in STORAGE SD modes */
1713         if (IS_MF_STORAGE_SD(bp) || IS_MF_FCOE_AFEX(bp))
1714                 bp->num_ethernet_queues = 1;
1715
1716         /* Add special queues */
1717         bp->num_cnic_queues = CNIC_SUPPORT(bp); /* For FCOE */
1718         bp->num_queues = bp->num_ethernet_queues + bp->num_cnic_queues;
1719
1720         BNX2X_DEV_INFO("set number of queues to %d\n", bp->num_queues);
1721 }
1722
1723 /**
1724  * bnx2x_set_real_num_queues - configure netdev->real_num_[tx,rx]_queues
1725  *
1726  * @bp:         Driver handle
1727  *
1728  * We currently support for at most 16 Tx queues for each CoS thus we will
1729  * allocate a multiple of 16 for ETH L2 rings according to the value of the
1730  * bp->max_cos.
1731  *
1732  * If there is an FCoE L2 queue the appropriate Tx queue will have the next
1733  * index after all ETH L2 indices.
1734  *
1735  * If the actual number of Tx queues (for each CoS) is less than 16 then there
1736  * will be the holes at the end of each group of 16 ETh L2 indices (0..15,
1737  * 16..31,...) with indicies that are not coupled with any real Tx queue.
1738  *
1739  * The proper configuration of skb->queue_mapping is handled by
1740  * bnx2x_select_queue() and __skb_tx_hash().
1741  *
1742  * bnx2x_setup_tc() takes care of the proper TC mappings so that __skb_tx_hash()
1743  * will return a proper Tx index if TC is enabled (netdev->num_tc > 0).
1744  */
1745 static int bnx2x_set_real_num_queues(struct bnx2x *bp, int include_cnic)
1746 {
1747         int rc, tx, rx;
1748
1749         tx = BNX2X_NUM_ETH_QUEUES(bp) * bp->max_cos;
1750         rx = BNX2X_NUM_ETH_QUEUES(bp);
1751
1752 /* account for fcoe queue */
1753         if (include_cnic && !NO_FCOE(bp)) {
1754                 rx++;
1755                 tx++;
1756         }
1757
1758         rc = netif_set_real_num_tx_queues(bp->dev, tx);
1759         if (rc) {
1760                 BNX2X_ERR("Failed to set real number of Tx queues: %d\n", rc);
1761                 return rc;
1762         }
1763         rc = netif_set_real_num_rx_queues(bp->dev, rx);
1764         if (rc) {
1765                 BNX2X_ERR("Failed to set real number of Rx queues: %d\n", rc);
1766                 return rc;
1767         }
1768
1769         DP(NETIF_MSG_IFUP, "Setting real num queues to (tx, rx) (%d, %d)\n",
1770                           tx, rx);
1771
1772         return rc;
1773 }
1774
1775 static void bnx2x_set_rx_buf_size(struct bnx2x *bp)
1776 {
1777         int i;
1778
1779         for_each_queue(bp, i) {
1780                 struct bnx2x_fastpath *fp = &bp->fp[i];
1781                 u32 mtu;
1782
1783                 /* Always use a mini-jumbo MTU for the FCoE L2 ring */
1784                 if (IS_FCOE_IDX(i))
1785                         /*
1786                          * Although there are no IP frames expected to arrive to
1787                          * this ring we still want to add an
1788                          * IP_HEADER_ALIGNMENT_PADDING to prevent a buffer
1789                          * overrun attack.
1790                          */
1791                         mtu = BNX2X_FCOE_MINI_JUMBO_MTU;
1792                 else
1793                         mtu = bp->dev->mtu;
1794                 fp->rx_buf_size = BNX2X_FW_RX_ALIGN_START +
1795                                   IP_HEADER_ALIGNMENT_PADDING +
1796                                   ETH_OVREHEAD +
1797                                   mtu +
1798                                   BNX2X_FW_RX_ALIGN_END;
1799                 /* Note : rx_buf_size doesnt take into account NET_SKB_PAD */
1800                 if (fp->rx_buf_size + NET_SKB_PAD <= PAGE_SIZE)
1801                         fp->rx_frag_size = fp->rx_buf_size + NET_SKB_PAD;
1802                 else
1803                         fp->rx_frag_size = 0;
1804         }
1805 }
1806
1807 static int bnx2x_init_rss_pf(struct bnx2x *bp)
1808 {
1809         int i;
1810         u8 num_eth_queues = BNX2X_NUM_ETH_QUEUES(bp);
1811
1812         /* Prepare the initial contents fo the indirection table if RSS is
1813          * enabled
1814          */
1815         for (i = 0; i < sizeof(bp->rss_conf_obj.ind_table); i++)
1816                 bp->rss_conf_obj.ind_table[i] =
1817                         bp->fp->cl_id +
1818                         ethtool_rxfh_indir_default(i, num_eth_queues);
1819
1820         /*
1821          * For 57710 and 57711 SEARCHER configuration (rss_keys) is
1822          * per-port, so if explicit configuration is needed , do it only
1823          * for a PMF.
1824          *
1825          * For 57712 and newer on the other hand it's a per-function
1826          * configuration.
1827          */
1828         return bnx2x_config_rss_eth(bp, bp->port.pmf || !CHIP_IS_E1x(bp));
1829 }
1830
1831 int bnx2x_config_rss_pf(struct bnx2x *bp, struct bnx2x_rss_config_obj *rss_obj,
1832                         bool config_hash)
1833 {
1834         struct bnx2x_config_rss_params params = {NULL};
1835
1836         /* Although RSS is meaningless when there is a single HW queue we
1837          * still need it enabled in order to have HW Rx hash generated.
1838          *
1839          * if (!is_eth_multi(bp))
1840          *      bp->multi_mode = ETH_RSS_MODE_DISABLED;
1841          */
1842
1843         params.rss_obj = rss_obj;
1844
1845         __set_bit(RAMROD_COMP_WAIT, &params.ramrod_flags);
1846
1847         __set_bit(BNX2X_RSS_MODE_REGULAR, &params.rss_flags);
1848
1849         /* RSS configuration */
1850         __set_bit(BNX2X_RSS_IPV4, &params.rss_flags);
1851         __set_bit(BNX2X_RSS_IPV4_TCP, &params.rss_flags);
1852         __set_bit(BNX2X_RSS_IPV6, &params.rss_flags);
1853         __set_bit(BNX2X_RSS_IPV6_TCP, &params.rss_flags);
1854         if (rss_obj->udp_rss_v4)
1855                 __set_bit(BNX2X_RSS_IPV4_UDP, &params.rss_flags);
1856         if (rss_obj->udp_rss_v6)
1857                 __set_bit(BNX2X_RSS_IPV6_UDP, &params.rss_flags);
1858
1859         /* Hash bits */
1860         params.rss_result_mask = MULTI_MASK;
1861
1862         memcpy(params.ind_table, rss_obj->ind_table, sizeof(params.ind_table));
1863
1864         if (config_hash) {
1865                 /* RSS keys */
1866                 prandom_bytes(params.rss_key, sizeof(params.rss_key));
1867                 __set_bit(BNX2X_RSS_SET_SRCH, &params.rss_flags);
1868         }
1869
1870         return bnx2x_config_rss(bp, &params);
1871 }
1872
1873 static int bnx2x_init_hw(struct bnx2x *bp, u32 load_code)
1874 {
1875         struct bnx2x_func_state_params func_params = {NULL};
1876
1877         /* Prepare parameters for function state transitions */
1878         __set_bit(RAMROD_COMP_WAIT, &func_params.ramrod_flags);
1879
1880         func_params.f_obj = &bp->func_obj;
1881         func_params.cmd = BNX2X_F_CMD_HW_INIT;
1882
1883         func_params.params.hw_init.load_phase = load_code;
1884
1885         return bnx2x_func_state_change(bp, &func_params);
1886 }
1887
1888 /*
1889  * Cleans the object that have internal lists without sending
1890  * ramrods. Should be run when interrutps are disabled.
1891  */
1892 static void bnx2x_squeeze_objects(struct bnx2x *bp)
1893 {
1894         int rc;
1895         unsigned long ramrod_flags = 0, vlan_mac_flags = 0;
1896         struct bnx2x_mcast_ramrod_params rparam = {NULL};
1897         struct bnx2x_vlan_mac_obj *mac_obj = &bp->sp_objs->mac_obj;
1898
1899         /***************** Cleanup MACs' object first *************************/
1900
1901         /* Wait for completion of requested */
1902         __set_bit(RAMROD_COMP_WAIT, &ramrod_flags);
1903         /* Perform a dry cleanup */
1904         __set_bit(RAMROD_DRV_CLR_ONLY, &ramrod_flags);
1905
1906         /* Clean ETH primary MAC */
1907         __set_bit(BNX2X_ETH_MAC, &vlan_mac_flags);
1908         rc = mac_obj->delete_all(bp, &bp->sp_objs->mac_obj, &vlan_mac_flags,
1909                                  &ramrod_flags);
1910         if (rc != 0)
1911                 BNX2X_ERR("Failed to clean ETH MACs: %d\n", rc);
1912
1913         /* Cleanup UC list */
1914         vlan_mac_flags = 0;
1915         __set_bit(BNX2X_UC_LIST_MAC, &vlan_mac_flags);
1916         rc = mac_obj->delete_all(bp, mac_obj, &vlan_mac_flags,
1917                                  &ramrod_flags);
1918         if (rc != 0)
1919                 BNX2X_ERR("Failed to clean UC list MACs: %d\n", rc);
1920
1921         /***************** Now clean mcast object *****************************/
1922         rparam.mcast_obj = &bp->mcast_obj;
1923         __set_bit(RAMROD_DRV_CLR_ONLY, &rparam.ramrod_flags);
1924
1925         /* Add a DEL command... */
1926         rc = bnx2x_config_mcast(bp, &rparam, BNX2X_MCAST_CMD_DEL);
1927         if (rc < 0)
1928                 BNX2X_ERR("Failed to add a new DEL command to a multi-cast object: %d\n",
1929                           rc);
1930
1931         /* ...and wait until all pending commands are cleared */
1932         rc = bnx2x_config_mcast(bp, &rparam, BNX2X_MCAST_CMD_CONT);
1933         while (rc != 0) {
1934                 if (rc < 0) {
1935                         BNX2X_ERR("Failed to clean multi-cast object: %d\n",
1936                                   rc);
1937                         return;
1938                 }
1939
1940                 rc = bnx2x_config_mcast(bp, &rparam, BNX2X_MCAST_CMD_CONT);
1941         }
1942 }
1943
1944 #ifndef BNX2X_STOP_ON_ERROR
1945 #define LOAD_ERROR_EXIT(bp, label) \
1946         do { \
1947                 (bp)->state = BNX2X_STATE_ERROR; \
1948                 goto label; \
1949         } while (0)
1950
1951 #define LOAD_ERROR_EXIT_CNIC(bp, label) \
1952         do { \
1953                 bp->cnic_loaded = false; \
1954                 goto label; \
1955         } while (0)
1956 #else /*BNX2X_STOP_ON_ERROR*/
1957 #define LOAD_ERROR_EXIT(bp, label) \
1958         do { \
1959                 (bp)->state = BNX2X_STATE_ERROR; \
1960                 (bp)->panic = 1; \
1961                 return -EBUSY; \
1962         } while (0)
1963 #define LOAD_ERROR_EXIT_CNIC(bp, label) \
1964         do { \
1965                 bp->cnic_loaded = false; \
1966                 (bp)->panic = 1; \
1967                 return -EBUSY; \
1968         } while (0)
1969 #endif /*BNX2X_STOP_ON_ERROR*/
1970
1971 bool bnx2x_test_firmware_version(struct bnx2x *bp, bool is_err)
1972 {
1973         /* build FW version dword */
1974         u32 my_fw = (BCM_5710_FW_MAJOR_VERSION) +
1975                     (BCM_5710_FW_MINOR_VERSION << 8) +
1976                     (BCM_5710_FW_REVISION_VERSION << 16) +
1977                     (BCM_5710_FW_ENGINEERING_VERSION << 24);
1978
1979         /* read loaded FW from chip */
1980         u32 loaded_fw = REG_RD(bp, XSEM_REG_PRAM);
1981
1982         DP(NETIF_MSG_IFUP, "loaded fw %x, my fw %x\n", loaded_fw, my_fw);
1983
1984         if (loaded_fw != my_fw) {
1985                 if (is_err)
1986                         BNX2X_ERR("bnx2x with FW %x was already loaded, which mismatches my %x FW. aborting\n",
1987                                   loaded_fw, my_fw);
1988                 return false;
1989         }
1990
1991         return true;
1992 }
1993
1994 /**
1995  * bnx2x_bz_fp - zero content of the fastpath structure.
1996  *
1997  * @bp:         driver handle
1998  * @index:      fastpath index to be zeroed
1999  *
2000  * Makes sure the contents of the bp->fp[index].napi is kept
2001  * intact.
2002  */
2003 static void bnx2x_bz_fp(struct bnx2x *bp, int index)
2004 {
2005         struct bnx2x_fastpath *fp = &bp->fp[index];
2006         struct bnx2x_fp_stats *fp_stats = &bp->fp_stats[index];
2007
2008         int cos;
2009         struct napi_struct orig_napi = fp->napi;
2010         struct bnx2x_agg_info *orig_tpa_info = fp->tpa_info;
2011         /* bzero bnx2x_fastpath contents */
2012         if (bp->stats_init) {
2013                 memset(fp->tpa_info, 0, sizeof(*fp->tpa_info));
2014                 memset(fp, 0, sizeof(*fp));
2015         } else {
2016                 /* Keep Queue statistics */
2017                 struct bnx2x_eth_q_stats *tmp_eth_q_stats;
2018                 struct bnx2x_eth_q_stats_old *tmp_eth_q_stats_old;
2019
2020                 tmp_eth_q_stats = kzalloc(sizeof(struct bnx2x_eth_q_stats),
2021                                           GFP_KERNEL);
2022                 if (tmp_eth_q_stats)
2023                         memcpy(tmp_eth_q_stats, &fp_stats->eth_q_stats,
2024                                sizeof(struct bnx2x_eth_q_stats));
2025
2026                 tmp_eth_q_stats_old =
2027                         kzalloc(sizeof(struct bnx2x_eth_q_stats_old),
2028                                 GFP_KERNEL);
2029                 if (tmp_eth_q_stats_old)
2030                         memcpy(tmp_eth_q_stats_old, &fp_stats->eth_q_stats_old,
2031                                sizeof(struct bnx2x_eth_q_stats_old));
2032
2033                 memset(fp->tpa_info, 0, sizeof(*fp->tpa_info));
2034                 memset(fp, 0, sizeof(*fp));
2035
2036                 if (tmp_eth_q_stats) {
2037                         memcpy(&fp_stats->eth_q_stats, tmp_eth_q_stats,
2038                                sizeof(struct bnx2x_eth_q_stats));
2039                         kfree(tmp_eth_q_stats);
2040                 }
2041
2042                 if (tmp_eth_q_stats_old) {
2043                         memcpy(&fp_stats->eth_q_stats_old, tmp_eth_q_stats_old,
2044                                sizeof(struct bnx2x_eth_q_stats_old));
2045                         kfree(tmp_eth_q_stats_old);
2046                 }
2047
2048         }
2049
2050         /* Restore the NAPI object as it has been already initialized */
2051         fp->napi = orig_napi;
2052         fp->tpa_info = orig_tpa_info;
2053         fp->bp = bp;
2054         fp->index = index;
2055         if (IS_ETH_FP(fp))
2056                 fp->max_cos = bp->max_cos;
2057         else
2058                 /* Special queues support only one CoS */
2059                 fp->max_cos = 1;
2060
2061         /* Init txdata pointers */
2062         if (IS_FCOE_FP(fp))
2063                 fp->txdata_ptr[0] = &bp->bnx2x_txq[FCOE_TXQ_IDX(bp)];
2064         if (IS_ETH_FP(fp))
2065                 for_each_cos_in_tx_queue(fp, cos)
2066                         fp->txdata_ptr[cos] = &bp->bnx2x_txq[cos *
2067                                 BNX2X_NUM_ETH_QUEUES(bp) + index];
2068
2069         /*
2070          * set the tpa flag for each queue. The tpa flag determines the queue
2071          * minimal size so it must be set prior to queue memory allocation
2072          */
2073         fp->disable_tpa = !(bp->flags & TPA_ENABLE_FLAG ||
2074                                   (bp->flags & GRO_ENABLE_FLAG &&
2075                                    bnx2x_mtu_allows_gro(bp->dev->mtu)));
2076         if (bp->flags & TPA_ENABLE_FLAG)
2077                 fp->mode = TPA_MODE_LRO;
2078         else if (bp->flags & GRO_ENABLE_FLAG)
2079                 fp->mode = TPA_MODE_GRO;
2080
2081         /* We don't want TPA on an FCoE L2 ring */
2082         if (IS_FCOE_FP(fp))
2083                 fp->disable_tpa = 1;
2084 }
2085
2086 int bnx2x_load_cnic(struct bnx2x *bp)
2087 {
2088         int i, rc, port = BP_PORT(bp);
2089
2090         DP(NETIF_MSG_IFUP, "Starting CNIC-related load\n");
2091
2092         mutex_init(&bp->cnic_mutex);
2093
2094         rc = bnx2x_alloc_mem_cnic(bp);
2095         if (rc) {
2096                 BNX2X_ERR("Unable to allocate bp memory for cnic\n");
2097                 LOAD_ERROR_EXIT_CNIC(bp, load_error_cnic0);
2098         }
2099
2100         rc = bnx2x_alloc_fp_mem_cnic(bp);
2101         if (rc) {
2102                 BNX2X_ERR("Unable to allocate memory for cnic fps\n");
2103                 LOAD_ERROR_EXIT_CNIC(bp, load_error_cnic0);
2104         }
2105
2106         /* Update the number of queues with the cnic queues */
2107         rc = bnx2x_set_real_num_queues(bp, 1);
2108         if (rc) {
2109                 BNX2X_ERR("Unable to set real_num_queues including cnic\n");
2110                 LOAD_ERROR_EXIT_CNIC(bp, load_error_cnic0);
2111         }
2112
2113         /* Add all CNIC NAPI objects */
2114         bnx2x_add_all_napi_cnic(bp);
2115         DP(NETIF_MSG_IFUP, "cnic napi added\n");
2116         bnx2x_napi_enable_cnic(bp);
2117
2118         rc = bnx2x_init_hw_func_cnic(bp);
2119         if (rc)
2120                 LOAD_ERROR_EXIT_CNIC(bp, load_error_cnic1);
2121
2122         bnx2x_nic_init_cnic(bp);
2123
2124         /* Enable Timer scan */
2125         REG_WR(bp, TM_REG_EN_LINEAR0_TIMER + port*4, 1);
2126
2127         for_each_cnic_queue(bp, i) {
2128                 rc = bnx2x_setup_queue(bp, &bp->fp[i], 0);
2129                 if (rc) {
2130                         BNX2X_ERR("Queue setup failed\n");
2131                         LOAD_ERROR_EXIT(bp, load_error_cnic2);
2132                 }
2133         }
2134
2135         /* Initialize Rx filter. */
2136         netif_addr_lock_bh(bp->dev);
2137         bnx2x_set_rx_mode(bp->dev);
2138         netif_addr_unlock_bh(bp->dev);
2139
2140         /* re-read iscsi info */
2141         bnx2x_get_iscsi_info(bp);
2142         bnx2x_setup_cnic_irq_info(bp);
2143         bnx2x_setup_cnic_info(bp);
2144         bp->cnic_loaded = true;
2145         if (bp->state == BNX2X_STATE_OPEN)
2146                 bnx2x_cnic_notify(bp, CNIC_CTL_START_CMD);
2147
2148
2149         DP(NETIF_MSG_IFUP, "Ending successfully CNIC-related load\n");
2150
2151         return 0;
2152
2153 #ifndef BNX2X_STOP_ON_ERROR
2154 load_error_cnic2:
2155         /* Disable Timer scan */
2156         REG_WR(bp, TM_REG_EN_LINEAR0_TIMER + port*4, 0);
2157
2158 load_error_cnic1:
2159         bnx2x_napi_disable_cnic(bp);
2160         /* Update the number of queues without the cnic queues */
2161         rc = bnx2x_set_real_num_queues(bp, 0);
2162         if (rc)
2163                 BNX2X_ERR("Unable to set real_num_queues not including cnic\n");
2164 load_error_cnic0:
2165         BNX2X_ERR("CNIC-related load failed\n");
2166         bnx2x_free_fp_mem_cnic(bp);
2167         bnx2x_free_mem_cnic(bp);
2168         return rc;
2169 #endif /* ! BNX2X_STOP_ON_ERROR */
2170 }
2171
2172
2173 /* must be called with rtnl_lock */
2174 int bnx2x_nic_load(struct bnx2x *bp, int load_mode)
2175 {
2176         int port = BP_PORT(bp);
2177         u32 load_code;
2178         int i, rc;
2179
2180         DP(NETIF_MSG_IFUP, "Starting NIC load\n");
2181         DP(NETIF_MSG_IFUP,
2182            "CNIC is %s\n", CNIC_ENABLED(bp) ? "enabled" : "disabled");
2183
2184 #ifdef BNX2X_STOP_ON_ERROR
2185         if (unlikely(bp->panic)) {
2186                 BNX2X_ERR("Can't load NIC when there is panic\n");
2187                 return -EPERM;
2188         }
2189 #endif
2190
2191         bp->state = BNX2X_STATE_OPENING_WAIT4_LOAD;
2192
2193         /* Set the initial link reported state to link down */
2194         bnx2x_acquire_phy_lock(bp);
2195         memset(&bp->last_reported_link, 0, sizeof(bp->last_reported_link));
2196         __set_bit(BNX2X_LINK_REPORT_LINK_DOWN,
2197                 &bp->last_reported_link.link_report_flags);
2198         bnx2x_release_phy_lock(bp);
2199
2200         /* must be called before memory allocation and HW init */
2201         bnx2x_ilt_set_info(bp);
2202
2203         /*
2204          * Zero fastpath structures preserving invariants like napi, which are
2205          * allocated only once, fp index, max_cos, bp pointer.
2206          * Also set fp->disable_tpa and txdata_ptr.
2207          */
2208         DP(NETIF_MSG_IFUP, "num queues: %d", bp->num_queues);
2209         for_each_queue(bp, i)
2210                 bnx2x_bz_fp(bp, i);
2211         memset(bp->bnx2x_txq, 0, (BNX2X_MAX_RSS_COUNT(bp) * BNX2X_MULTI_TX_COS +
2212                                   bp->num_cnic_queues) *
2213                                   sizeof(struct bnx2x_fp_txdata));
2214
2215         bp->fcoe_init = false;
2216
2217         /* Set the receive queues buffer size */
2218         bnx2x_set_rx_buf_size(bp);
2219
2220         if (bnx2x_alloc_mem(bp))
2221                 return -ENOMEM;
2222
2223         /* As long as bnx2x_alloc_mem() may possibly update
2224          * bp->num_queues, bnx2x_set_real_num_queues() should always
2225          * come after it. At this stage cnic queues are not counted.
2226          */
2227         rc = bnx2x_set_real_num_queues(bp, 0);
2228         if (rc) {
2229                 BNX2X_ERR("Unable to set real_num_queues\n");
2230                 LOAD_ERROR_EXIT(bp, load_error0);
2231         }
2232
2233         /* configure multi cos mappings in kernel.
2234          * this configuration may be overriden by a multi class queue discipline
2235          * or by a dcbx negotiation result.
2236          */
2237         bnx2x_setup_tc(bp->dev, bp->max_cos);
2238
2239         /* Add all NAPI objects */
2240         bnx2x_add_all_napi(bp);
2241         DP(NETIF_MSG_IFUP, "napi added\n");
2242         bnx2x_napi_enable(bp);
2243
2244         /* set pf load just before approaching the MCP */
2245         bnx2x_set_pf_load(bp);
2246
2247         /* Send LOAD_REQUEST command to MCP
2248          * Returns the type of LOAD command:
2249          * if it is the first port to be initialized
2250          * common blocks should be initialized, otherwise - not
2251          */
2252         if (!BP_NOMCP(bp)) {
2253                 /* init fw_seq */
2254                 bp->fw_seq =
2255                         (SHMEM_RD(bp, func_mb[BP_FW_MB_IDX(bp)].drv_mb_header) &
2256                          DRV_MSG_SEQ_NUMBER_MASK);
2257                 BNX2X_DEV_INFO("fw_seq 0x%08x\n", bp->fw_seq);
2258
2259                 /* Get current FW pulse sequence */
2260                 bp->fw_drv_pulse_wr_seq =
2261                         (SHMEM_RD(bp, func_mb[BP_FW_MB_IDX(bp)].drv_pulse_mb) &
2262                          DRV_PULSE_SEQ_MASK);
2263                 BNX2X_DEV_INFO("drv_pulse 0x%x\n", bp->fw_drv_pulse_wr_seq);
2264
2265                 load_code = bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_REQ,
2266                                              DRV_MSG_CODE_LOAD_REQ_WITH_LFA);
2267                 if (!load_code) {
2268                         BNX2X_ERR("MCP response failure, aborting\n");
2269                         rc = -EBUSY;
2270                         LOAD_ERROR_EXIT(bp, load_error1);
2271                 }
2272                 if (load_code == FW_MSG_CODE_DRV_LOAD_REFUSED) {
2273                         BNX2X_ERR("Driver load refused\n");
2274                         rc = -EBUSY; /* other port in diagnostic mode */
2275                         LOAD_ERROR_EXIT(bp, load_error1);
2276                 }
2277                 if (load_code != FW_MSG_CODE_DRV_LOAD_COMMON_CHIP &&
2278                     load_code != FW_MSG_CODE_DRV_LOAD_COMMON) {
2279                         /* abort nic load if version mismatch */
2280                         if (!bnx2x_test_firmware_version(bp, true)) {
2281                                 rc = -EBUSY;
2282                                 LOAD_ERROR_EXIT(bp, load_error2);
2283                         }
2284                 }
2285
2286         } else {
2287                 int path = BP_PATH(bp);
2288
2289                 DP(NETIF_MSG_IFUP, "NO MCP - load counts[%d]      %d, %d, %d\n",
2290                    path, load_count[path][0], load_count[path][1],
2291                    load_count[path][2]);
2292                 load_count[path][0]++;
2293                 load_count[path][1 + port]++;
2294                 DP(NETIF_MSG_IFUP, "NO MCP - new load counts[%d]  %d, %d, %d\n",
2295                    path, load_count[path][0], load_count[path][1],
2296                    load_count[path][2]);
2297                 if (load_count[path][0] == 1)
2298                         load_code = FW_MSG_CODE_DRV_LOAD_COMMON;
2299                 else if (load_count[path][1 + port] == 1)
2300                         load_code = FW_MSG_CODE_DRV_LOAD_PORT;
2301                 else
2302                         load_code = FW_MSG_CODE_DRV_LOAD_FUNCTION;
2303         }
2304
2305         if ((load_code == FW_MSG_CODE_DRV_LOAD_COMMON) ||
2306             (load_code == FW_MSG_CODE_DRV_LOAD_COMMON_CHIP) ||
2307             (load_code == FW_MSG_CODE_DRV_LOAD_PORT)) {
2308                 bp->port.pmf = 1;
2309                 /*
2310                  * We need the barrier to ensure the ordering between the
2311                  * writing to bp->port.pmf here and reading it from the
2312                  * bnx2x_periodic_task().
2313                  */
2314                 smp_mb();
2315         } else
2316                 bp->port.pmf = 0;
2317
2318         DP(NETIF_MSG_IFUP, "pmf %d\n", bp->port.pmf);
2319
2320         /* Init Function state controlling object */
2321         bnx2x__init_func_obj(bp);
2322
2323         /* Initialize HW */
2324         rc = bnx2x_init_hw(bp, load_code);
2325         if (rc) {
2326                 BNX2X_ERR("HW init failed, aborting\n");
2327                 bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
2328                 LOAD_ERROR_EXIT(bp, load_error2);
2329         }
2330
2331         /* Connect to IRQs */
2332         rc = bnx2x_setup_irqs(bp);
2333         if (rc) {
2334                 BNX2X_ERR("IRQs setup failed\n");
2335                 bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
2336                 LOAD_ERROR_EXIT(bp, load_error2);
2337         }
2338
2339         /* Setup NIC internals and enable interrupts */
2340         bnx2x_nic_init(bp, load_code);
2341
2342         /* Init per-function objects */
2343         bnx2x_init_bp_objs(bp);
2344
2345         if (((load_code == FW_MSG_CODE_DRV_LOAD_COMMON) ||
2346             (load_code == FW_MSG_CODE_DRV_LOAD_COMMON_CHIP)) &&
2347             (bp->common.shmem2_base)) {
2348                 if (SHMEM2_HAS(bp, dcc_support))
2349                         SHMEM2_WR(bp, dcc_support,
2350                                   (SHMEM_DCC_SUPPORT_DISABLE_ENABLE_PF_TLV |
2351                                    SHMEM_DCC_SUPPORT_BANDWIDTH_ALLOCATION_TLV));
2352                 if (SHMEM2_HAS(bp, afex_driver_support))
2353                         SHMEM2_WR(bp, afex_driver_support,
2354                                   SHMEM_AFEX_SUPPORTED_VERSION_ONE);
2355         }
2356
2357         /* Set AFEX default VLAN tag to an invalid value */
2358         bp->afex_def_vlan_tag = -1;
2359
2360         bp->state = BNX2X_STATE_OPENING_WAIT4_PORT;
2361         rc = bnx2x_func_start(bp);
2362         if (rc) {
2363                 BNX2X_ERR("Function start failed!\n");
2364                 bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
2365                 LOAD_ERROR_EXIT(bp, load_error3);
2366         }
2367
2368         /* Send LOAD_DONE command to MCP */
2369         if (!BP_NOMCP(bp)) {
2370                 load_code = bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
2371                 if (!load_code) {
2372                         BNX2X_ERR("MCP response failure, aborting\n");
2373                         rc = -EBUSY;
2374                         LOAD_ERROR_EXIT(bp, load_error3);
2375                 }
2376         }
2377
2378         rc = bnx2x_setup_leading(bp);
2379         if (rc) {
2380                 BNX2X_ERR("Setup leading failed!\n");
2381                 LOAD_ERROR_EXIT(bp, load_error3);
2382         }
2383
2384         for_each_nondefault_eth_queue(bp, i) {
2385                 rc = bnx2x_setup_queue(bp, &bp->fp[i], 0);
2386                 if (rc) {
2387                         BNX2X_ERR("Queue setup failed\n");
2388                         LOAD_ERROR_EXIT(bp, load_error3);
2389                 }
2390         }
2391
2392         rc = bnx2x_init_rss_pf(bp);
2393         if (rc) {
2394                 BNX2X_ERR("PF RSS init failed\n");
2395                 LOAD_ERROR_EXIT(bp, load_error3);
2396         }
2397
2398         /* Now when Clients are configured we are ready to work */
2399         bp->state = BNX2X_STATE_OPEN;
2400
2401         /* Configure a ucast MAC */
2402         rc = bnx2x_set_eth_mac(bp, true);
2403         if (rc) {
2404                 BNX2X_ERR("Setting Ethernet MAC failed\n");
2405                 LOAD_ERROR_EXIT(bp, load_error3);
2406         }
2407
2408         if (bp->pending_max) {
2409                 bnx2x_update_max_mf_config(bp, bp->pending_max);
2410                 bp->pending_max = 0;
2411         }
2412
2413         if (bp->port.pmf)
2414                 bnx2x_initial_phy_init(bp, load_mode);
2415         bp->link_params.feature_config_flags &= ~FEATURE_CONFIG_BOOT_FROM_SAN;
2416
2417         /* Start fast path */
2418
2419         /* Initialize Rx filter. */
2420         netif_addr_lock_bh(bp->dev);
2421         bnx2x_set_rx_mode(bp->dev);
2422         netif_addr_unlock_bh(bp->dev);
2423
2424         /* Start the Tx */
2425         switch (load_mode) {
2426         case LOAD_NORMAL:
2427                 /* Tx queue should be only reenabled */
2428                 netif_tx_wake_all_queues(bp->dev);
2429                 break;
2430
2431         case LOAD_OPEN:
2432                 netif_tx_start_all_queues(bp->dev);
2433                 smp_mb__after_clear_bit();
2434                 break;
2435
2436         case LOAD_DIAG:
2437         case LOAD_LOOPBACK_EXT:
2438                 bp->state = BNX2X_STATE_DIAG;
2439                 break;
2440
2441         default:
2442                 break;
2443         }
2444
2445         if (bp->port.pmf)
2446                 bnx2x_update_drv_flags(bp, 1 << DRV_FLAGS_PORT_MASK, 0);
2447         else
2448                 bnx2x__link_status_update(bp);
2449
2450         /* start the timer */
2451         mod_timer(&bp->timer, jiffies + bp->current_interval);
2452
2453         if (CNIC_ENABLED(bp))
2454                 bnx2x_load_cnic(bp);
2455
2456         /* mark driver is loaded in shmem2 */
2457         if (SHMEM2_HAS(bp, drv_capabilities_flag)) {
2458                 u32 val;
2459                 val = SHMEM2_RD(bp, drv_capabilities_flag[BP_FW_MB_IDX(bp)]);
2460                 SHMEM2_WR(bp, drv_capabilities_flag[BP_FW_MB_IDX(bp)],
2461                           val | DRV_FLAGS_CAPABILITIES_LOADED_SUPPORTED |
2462                           DRV_FLAGS_CAPABILITIES_LOADED_L2);
2463         }
2464
2465         /* Wait for all pending SP commands to complete */
2466         if (!bnx2x_wait_sp_comp(bp, ~0x0UL)) {
2467                 BNX2X_ERR("Timeout waiting for SP elements to complete\n");
2468                 bnx2x_nic_unload(bp, UNLOAD_CLOSE, false);
2469                 return -EBUSY;
2470         }
2471
2472         /* If PMF - send ADMIN DCBX msg to MFW to initiate DCBX FSM */
2473         if (bp->port.pmf && (bp->state != BNX2X_STATE_DIAG))
2474                 bnx2x_dcbx_init(bp, false);
2475
2476         DP(NETIF_MSG_IFUP, "Ending successfully NIC load\n");
2477
2478         return 0;
2479
2480 #ifndef BNX2X_STOP_ON_ERROR
2481 load_error3:
2482         bnx2x_int_disable_sync(bp, 1);
2483
2484         /* Clean queueable objects */
2485         bnx2x_squeeze_objects(bp);
2486
2487         /* Free SKBs, SGEs, TPA pool and driver internals */
2488         bnx2x_free_skbs(bp);
2489         for_each_rx_queue(bp, i)
2490                 bnx2x_free_rx_sge_range(bp, bp->fp + i, NUM_RX_SGE);
2491
2492         /* Release IRQs */
2493         bnx2x_free_irq(bp);
2494 load_error2:
2495         if (!BP_NOMCP(bp)) {
2496                 bnx2x_fw_command(bp, DRV_MSG_CODE_UNLOAD_REQ_WOL_MCP, 0);
2497                 bnx2x_fw_command(bp, DRV_MSG_CODE_UNLOAD_DONE, 0);
2498         }
2499
2500         bp->port.pmf = 0;
2501 load_error1:
2502         bnx2x_napi_disable(bp);
2503         /* clear pf_load status, as it was already set */
2504         bnx2x_clear_pf_load(bp);
2505 load_error0:
2506         bnx2x_free_mem(bp);
2507
2508         return rc;
2509 #endif /* ! BNX2X_STOP_ON_ERROR */
2510 }
2511
2512 /* must be called with rtnl_lock */
2513 int bnx2x_nic_unload(struct bnx2x *bp, int unload_mode, bool keep_link)
2514 {
2515         int i;
2516         bool global = false;
2517
2518         DP(NETIF_MSG_IFUP, "Starting NIC unload\n");
2519
2520         /* mark driver is unloaded in shmem2 */
2521         if (SHMEM2_HAS(bp, drv_capabilities_flag)) {
2522                 u32 val;
2523                 val = SHMEM2_RD(bp, drv_capabilities_flag[BP_FW_MB_IDX(bp)]);
2524                 SHMEM2_WR(bp, drv_capabilities_flag[BP_FW_MB_IDX(bp)],
2525                           val & ~DRV_FLAGS_CAPABILITIES_LOADED_L2);
2526         }
2527
2528         if ((bp->state == BNX2X_STATE_CLOSED) ||
2529             (bp->state == BNX2X_STATE_ERROR)) {
2530                 /* We can get here if the driver has been unloaded
2531                  * during parity error recovery and is either waiting for a
2532                  * leader to complete or for other functions to unload and
2533                  * then ifdown has been issued. In this case we want to
2534                  * unload and let other functions to complete a recovery
2535                  * process.
2536                  */
2537                 bp->recovery_state = BNX2X_RECOVERY_DONE;
2538                 bp->is_leader = 0;
2539                 bnx2x_release_leader_lock(bp);
2540                 smp_mb();
2541
2542                 DP(NETIF_MSG_IFDOWN, "Releasing a leadership...\n");
2543                 BNX2X_ERR("Can't unload in closed or error state\n");
2544                 return -EINVAL;
2545         }
2546
2547         /*
2548          * It's important to set the bp->state to the value different from
2549          * BNX2X_STATE_OPEN and only then stop the Tx. Otherwise bnx2x_tx_int()
2550          * may restart the Tx from the NAPI context (see bnx2x_tx_int()).
2551          */
2552         bp->state = BNX2X_STATE_CLOSING_WAIT4_HALT;
2553         smp_mb();
2554
2555         if (CNIC_LOADED(bp))
2556                 bnx2x_cnic_notify(bp, CNIC_CTL_STOP_CMD);
2557
2558         /* Stop Tx */
2559         bnx2x_tx_disable(bp);
2560         netdev_reset_tc(bp->dev);
2561
2562         bp->rx_mode = BNX2X_RX_MODE_NONE;
2563
2564         del_timer_sync(&bp->timer);
2565
2566         /* Set ALWAYS_ALIVE bit in shmem */
2567         bp->fw_drv_pulse_wr_seq |= DRV_PULSE_ALWAYS_ALIVE;
2568
2569         bnx2x_drv_pulse(bp);
2570
2571         bnx2x_stats_handle(bp, STATS_EVENT_STOP);
2572         bnx2x_save_statistics(bp);
2573
2574         /* Cleanup the chip if needed */
2575         if (unload_mode != UNLOAD_RECOVERY)
2576                 bnx2x_chip_cleanup(bp, unload_mode, keep_link);
2577         else {
2578                 /* Send the UNLOAD_REQUEST to the MCP */
2579                 bnx2x_send_unload_req(bp, unload_mode);
2580
2581                 /*
2582                  * Prevent transactions to host from the functions on the
2583                  * engine that doesn't reset global blocks in case of global
2584                  * attention once gloabl blocks are reset and gates are opened
2585                  * (the engine which leader will perform the recovery
2586                  * last).
2587                  */
2588                 if (!CHIP_IS_E1x(bp))
2589                         bnx2x_pf_disable(bp);
2590
2591                 /* Disable HW interrupts, NAPI */
2592                 bnx2x_netif_stop(bp, 1);
2593                 /* Delete all NAPI objects */
2594                 bnx2x_del_all_napi(bp);
2595                 if (CNIC_LOADED(bp))
2596                         bnx2x_del_all_napi_cnic(bp);
2597                 /* Release IRQs */
2598                 bnx2x_free_irq(bp);
2599
2600                 /* Report UNLOAD_DONE to MCP */
2601                 bnx2x_send_unload_done(bp, false);
2602         }
2603
2604         /*
2605          * At this stage no more interrupts will arrive so we may safly clean
2606          * the queueable objects here in case they failed to get cleaned so far.
2607          */
2608         bnx2x_squeeze_objects(bp);
2609
2610         /* There should be no more pending SP commands at this stage */
2611         bp->sp_state = 0;
2612
2613         bp->port.pmf = 0;
2614
2615         /* Free SKBs, SGEs, TPA pool and driver internals */
2616         bnx2x_free_skbs(bp);
2617         if (CNIC_LOADED(bp))
2618                 bnx2x_free_skbs_cnic(bp);
2619         for_each_rx_queue(bp, i)
2620                 bnx2x_free_rx_sge_range(bp, bp->fp + i, NUM_RX_SGE);
2621
2622         if (CNIC_LOADED(bp)) {
2623                 bnx2x_free_fp_mem_cnic(bp);
2624                 bnx2x_free_mem_cnic(bp);
2625         }
2626         bnx2x_free_mem(bp);
2627
2628         bp->state = BNX2X_STATE_CLOSED;
2629         bp->cnic_loaded = false;
2630
2631         /* Check if there are pending parity attentions. If there are - set
2632          * RECOVERY_IN_PROGRESS.
2633          */
2634         if (bnx2x_chk_parity_attn(bp, &global, false)) {
2635                 bnx2x_set_reset_in_progress(bp);
2636
2637                 /* Set RESET_IS_GLOBAL if needed */
2638                 if (global)
2639                         bnx2x_set_reset_global(bp);
2640         }
2641
2642
2643         /* The last driver must disable a "close the gate" if there is no
2644          * parity attention or "process kill" pending.
2645          */
2646         if (!bnx2x_clear_pf_load(bp) && bnx2x_reset_is_done(bp, BP_PATH(bp)))
2647                 bnx2x_disable_close_the_gate(bp);
2648
2649         DP(NETIF_MSG_IFUP, "Ending NIC unload\n");
2650
2651         return 0;
2652 }
2653
2654 int bnx2x_set_power_state(struct bnx2x *bp, pci_power_t state)
2655 {
2656         u16 pmcsr;
2657
2658         /* If there is no power capability, silently succeed */
2659         if (!bp->pm_cap) {
2660                 BNX2X_DEV_INFO("No power capability. Breaking.\n");
2661                 return 0;
2662         }
2663
2664         pci_read_config_word(bp->pdev, bp->pm_cap + PCI_PM_CTRL, &pmcsr);
2665
2666         switch (state) {
2667         case PCI_D0:
2668                 pci_write_config_word(bp->pdev, bp->pm_cap + PCI_PM_CTRL,
2669                                       ((pmcsr & ~PCI_PM_CTRL_STATE_MASK) |
2670                                        PCI_PM_CTRL_PME_STATUS));
2671
2672                 if (pmcsr & PCI_PM_CTRL_STATE_MASK)
2673                         /* delay required during transition out of D3hot */
2674                         msleep(20);
2675                 break;
2676
2677         case PCI_D3hot:
2678                 /* If there are other clients above don't
2679                    shut down the power */
2680                 if (atomic_read(&bp->pdev->enable_cnt) != 1)
2681                         return 0;
2682                 /* Don't shut down the power for emulation and FPGA */
2683                 if (CHIP_REV_IS_SLOW(bp))
2684                         return 0;
2685
2686                 pmcsr &= ~PCI_PM_CTRL_STATE_MASK;
2687                 pmcsr |= 3;
2688
2689                 if (bp->wol)
2690                         pmcsr |= PCI_PM_CTRL_PME_ENABLE;
2691
2692                 pci_write_config_word(bp->pdev, bp->pm_cap + PCI_PM_CTRL,
2693                                       pmcsr);
2694
2695                 /* No more memory access after this point until
2696                 * device is brought back to D0.
2697                 */
2698                 break;
2699
2700         default:
2701                 dev_err(&bp->pdev->dev, "Can't support state = %d\n", state);
2702                 return -EINVAL;
2703         }
2704         return 0;
2705 }
2706
2707 /*
2708  * net_device service functions
2709  */
2710 int bnx2x_poll(struct napi_struct *napi, int budget)
2711 {
2712         int work_done = 0;
2713         u8 cos;
2714         struct bnx2x_fastpath *fp = container_of(napi, struct bnx2x_fastpath,
2715                                                  napi);
2716         struct bnx2x *bp = fp->bp;
2717
2718         while (1) {
2719 #ifdef BNX2X_STOP_ON_ERROR
2720                 if (unlikely(bp->panic)) {
2721                         napi_complete(napi);
2722                         return 0;
2723                 }
2724 #endif
2725
2726                 for_each_cos_in_tx_queue(fp, cos)
2727                         if (bnx2x_tx_queue_has_work(fp->txdata_ptr[cos]))
2728                                 bnx2x_tx_int(bp, fp->txdata_ptr[cos]);
2729
2730
2731                 if (bnx2x_has_rx_work(fp)) {
2732                         work_done += bnx2x_rx_int(fp, budget - work_done);
2733
2734                         /* must not complete if we consumed full budget */
2735                         if (work_done >= budget)
2736                                 break;
2737                 }
2738
2739                 /* Fall out from the NAPI loop if needed */
2740                 if (!(bnx2x_has_rx_work(fp) || bnx2x_has_tx_work(fp))) {
2741
2742                         /* No need to update SB for FCoE L2 ring as long as
2743                          * it's connected to the default SB and the SB
2744                          * has been updated when NAPI was scheduled.
2745                          */
2746                         if (IS_FCOE_FP(fp)) {
2747                                 napi_complete(napi);
2748                                 break;
2749                         }
2750                         bnx2x_update_fpsb_idx(fp);
2751                         /* bnx2x_has_rx_work() reads the status block,
2752                          * thus we need to ensure that status block indices
2753                          * have been actually read (bnx2x_update_fpsb_idx)
2754                          * prior to this check (bnx2x_has_rx_work) so that
2755                          * we won't write the "newer" value of the status block
2756                          * to IGU (if there was a DMA right after
2757                          * bnx2x_has_rx_work and if there is no rmb, the memory
2758                          * reading (bnx2x_update_fpsb_idx) may be postponed
2759                          * to right before bnx2x_ack_sb). In this case there
2760                          * will never be another interrupt until there is
2761                          * another update of the status block, while there
2762                          * is still unhandled work.
2763                          */
2764                         rmb();
2765
2766                         if (!(bnx2x_has_rx_work(fp) || bnx2x_has_tx_work(fp))) {
2767                                 napi_complete(napi);
2768                                 /* Re-enable interrupts */
2769                                 DP(NETIF_MSG_RX_STATUS,
2770                                    "Update index to %d\n", fp->fp_hc_idx);
2771                                 bnx2x_ack_sb(bp, fp->igu_sb_id, USTORM_ID,
2772                                              le16_to_cpu(fp->fp_hc_idx),
2773                                              IGU_INT_ENABLE, 1);
2774                                 break;
2775                         }
2776                 }
2777         }
2778
2779         return work_done;
2780 }
2781
2782 /* we split the first BD into headers and data BDs
2783  * to ease the pain of our fellow microcode engineers
2784  * we use one mapping for both BDs
2785  */
2786 static noinline u16 bnx2x_tx_split(struct bnx2x *bp,
2787                                    struct bnx2x_fp_txdata *txdata,
2788                                    struct sw_tx_bd *tx_buf,
2789                                    struct eth_tx_start_bd **tx_bd, u16 hlen,
2790                                    u16 bd_prod, int nbd)
2791 {
2792         struct eth_tx_start_bd *h_tx_bd = *tx_bd;
2793         struct eth_tx_bd *d_tx_bd;
2794         dma_addr_t mapping;
2795         int old_len = le16_to_cpu(h_tx_bd->nbytes);
2796
2797         /* first fix first BD */
2798         h_tx_bd->nbd = cpu_to_le16(nbd);
2799         h_tx_bd->nbytes = cpu_to_le16(hlen);
2800
2801         DP(NETIF_MSG_TX_QUEUED, "TSO split header size is %d (%x:%x) nbd %d\n",
2802            h_tx_bd->nbytes, h_tx_bd->addr_hi, h_tx_bd->addr_lo, h_tx_bd->nbd);
2803
2804         /* now get a new data BD
2805          * (after the pbd) and fill it */
2806         bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
2807         d_tx_bd = &txdata->tx_desc_ring[bd_prod].reg_bd;
2808
2809         mapping = HILO_U64(le32_to_cpu(h_tx_bd->addr_hi),
2810                            le32_to_cpu(h_tx_bd->addr_lo)) + hlen;
2811
2812         d_tx_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
2813         d_tx_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
2814         d_tx_bd->nbytes = cpu_to_le16(old_len - hlen);
2815
2816         /* this marks the BD as one that has no individual mapping */
2817         tx_buf->flags |= BNX2X_TSO_SPLIT_BD;
2818
2819         DP(NETIF_MSG_TX_QUEUED,
2820            "TSO split data size is %d (%x:%x)\n",
2821            d_tx_bd->nbytes, d_tx_bd->addr_hi, d_tx_bd->addr_lo);
2822
2823         /* update tx_bd */
2824         *tx_bd = (struct eth_tx_start_bd *)d_tx_bd;
2825
2826         return bd_prod;
2827 }
2828
2829 static inline u16 bnx2x_csum_fix(unsigned char *t_header, u16 csum, s8 fix)
2830 {
2831         if (fix > 0)
2832                 csum = (u16) ~csum_fold(csum_sub(csum,
2833                                 csum_partial(t_header - fix, fix, 0)));
2834
2835         else if (fix < 0)
2836                 csum = (u16) ~csum_fold(csum_add(csum,
2837                                 csum_partial(t_header, -fix, 0)));
2838
2839         return swab16(csum);
2840 }
2841
2842 static inline u32 bnx2x_xmit_type(struct bnx2x *bp, struct sk_buff *skb)
2843 {
2844         u32 rc;
2845
2846         if (skb->ip_summed != CHECKSUM_PARTIAL)
2847                 rc = XMIT_PLAIN;
2848
2849         else {
2850                 if (vlan_get_protocol(skb) == htons(ETH_P_IPV6)) {
2851                         rc = XMIT_CSUM_V6;
2852                         if (ipv6_hdr(skb)->nexthdr == IPPROTO_TCP)
2853                                 rc |= XMIT_CSUM_TCP;
2854
2855                 } else {
2856                         rc = XMIT_CSUM_V4;
2857                         if (ip_hdr(skb)->protocol == IPPROTO_TCP)
2858                                 rc |= XMIT_CSUM_TCP;
2859                 }
2860         }
2861
2862         if (skb_is_gso_v6(skb))
2863                 rc |= XMIT_GSO_V6 | XMIT_CSUM_TCP | XMIT_CSUM_V6;
2864         else if (skb_is_gso(skb))
2865                 rc |= XMIT_GSO_V4 | XMIT_CSUM_V4 | XMIT_CSUM_TCP;
2866
2867         return rc;
2868 }
2869
2870 #if (MAX_SKB_FRAGS >= MAX_FETCH_BD - 3)
2871 /* check if packet requires linearization (packet is too fragmented)
2872    no need to check fragmentation if page size > 8K (there will be no
2873    violation to FW restrictions) */
2874 static int bnx2x_pkt_req_lin(struct bnx2x *bp, struct sk_buff *skb,
2875                              u32 xmit_type)
2876 {
2877         int to_copy = 0;
2878         int hlen = 0;
2879         int first_bd_sz = 0;
2880
2881         /* 3 = 1 (for linear data BD) + 2 (for PBD and last BD) */
2882         if (skb_shinfo(skb)->nr_frags >= (MAX_FETCH_BD - 3)) {
2883
2884                 if (xmit_type & XMIT_GSO) {
2885                         unsigned short lso_mss = skb_shinfo(skb)->gso_size;
2886                         /* Check if LSO packet needs to be copied:
2887                            3 = 1 (for headers BD) + 2 (for PBD and last BD) */
2888                         int wnd_size = MAX_FETCH_BD - 3;
2889                         /* Number of windows to check */
2890                         int num_wnds = skb_shinfo(skb)->nr_frags - wnd_size;
2891                         int wnd_idx = 0;
2892                         int frag_idx = 0;
2893                         u32 wnd_sum = 0;
2894
2895                         /* Headers length */
2896                         hlen = (int)(skb_transport_header(skb) - skb->data) +
2897                                 tcp_hdrlen(skb);
2898
2899                         /* Amount of data (w/o headers) on linear part of SKB*/
2900                         first_bd_sz = skb_headlen(skb) - hlen;
2901
2902                         wnd_sum  = first_bd_sz;
2903
2904                         /* Calculate the first sum - it's special */
2905                         for (frag_idx = 0; frag_idx < wnd_size - 1; frag_idx++)
2906                                 wnd_sum +=
2907                                         skb_frag_size(&skb_shinfo(skb)->frags[frag_idx]);
2908
2909                         /* If there was data on linear skb data - check it */
2910                         if (first_bd_sz > 0) {
2911                                 if (unlikely(wnd_sum < lso_mss)) {
2912                                         to_copy = 1;
2913                                         goto exit_lbl;
2914                                 }
2915
2916                                 wnd_sum -= first_bd_sz;
2917                         }
2918
2919                         /* Others are easier: run through the frag list and
2920                            check all windows */
2921                         for (wnd_idx = 0; wnd_idx <= num_wnds; wnd_idx++) {
2922                                 wnd_sum +=
2923                           skb_frag_size(&skb_shinfo(skb)->frags[wnd_idx + wnd_size - 1]);
2924
2925                                 if (unlikely(wnd_sum < lso_mss)) {
2926                                         to_copy = 1;
2927                                         break;
2928                                 }
2929                                 wnd_sum -=
2930                                         skb_frag_size(&skb_shinfo(skb)->frags[wnd_idx]);
2931                         }
2932                 } else {
2933                         /* in non-LSO too fragmented packet should always
2934                            be linearized */
2935                         to_copy = 1;
2936                 }
2937         }
2938
2939 exit_lbl:
2940         if (unlikely(to_copy))
2941                 DP(NETIF_MSG_TX_QUEUED,
2942                    "Linearization IS REQUIRED for %s packet. num_frags %d  hlen %d  first_bd_sz %d\n",
2943                    (xmit_type & XMIT_GSO) ? "LSO" : "non-LSO",
2944                    skb_shinfo(skb)->nr_frags, hlen, first_bd_sz);
2945
2946         return to_copy;
2947 }
2948 #endif
2949
2950 static inline void bnx2x_set_pbd_gso_e2(struct sk_buff *skb, u32 *parsing_data,
2951                                         u32 xmit_type)
2952 {
2953         *parsing_data |= (skb_shinfo(skb)->gso_size <<
2954                               ETH_TX_PARSE_BD_E2_LSO_MSS_SHIFT) &
2955                               ETH_TX_PARSE_BD_E2_LSO_MSS;
2956         if ((xmit_type & XMIT_GSO_V6) &&
2957             (ipv6_hdr(skb)->nexthdr == NEXTHDR_IPV6))
2958                 *parsing_data |= ETH_TX_PARSE_BD_E2_IPV6_WITH_EXT_HDR;
2959 }
2960
2961 /**
2962  * bnx2x_set_pbd_gso - update PBD in GSO case.
2963  *
2964  * @skb:        packet skb
2965  * @pbd:        parse BD
2966  * @xmit_type:  xmit flags
2967  */
2968 static inline void bnx2x_set_pbd_gso(struct sk_buff *skb,
2969                                      struct eth_tx_parse_bd_e1x *pbd,
2970                                      u32 xmit_type)
2971 {
2972         pbd->lso_mss = cpu_to_le16(skb_shinfo(skb)->gso_size);
2973         pbd->tcp_send_seq = swab32(tcp_hdr(skb)->seq);
2974         pbd->tcp_flags = pbd_tcp_flags(skb);
2975
2976         if (xmit_type & XMIT_GSO_V4) {
2977                 pbd->ip_id = swab16(ip_hdr(skb)->id);
2978                 pbd->tcp_pseudo_csum =
2979                         swab16(~csum_tcpudp_magic(ip_hdr(skb)->saddr,
2980                                                   ip_hdr(skb)->daddr,
2981                                                   0, IPPROTO_TCP, 0));
2982
2983         } else
2984                 pbd->tcp_pseudo_csum =
2985                         swab16(~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
2986                                                 &ipv6_hdr(skb)->daddr,
2987                                                 0, IPPROTO_TCP, 0));
2988
2989         pbd->global_data |= ETH_TX_PARSE_BD_E1X_PSEUDO_CS_WITHOUT_LEN;
2990 }
2991
2992 /**
2993  * bnx2x_set_pbd_csum_e2 - update PBD with checksum and return header length
2994  *
2995  * @bp:                 driver handle
2996  * @skb:                packet skb
2997  * @parsing_data:       data to be updated
2998  * @xmit_type:          xmit flags
2999  *
3000  * 57712 related
3001  */
3002 static inline  u8 bnx2x_set_pbd_csum_e2(struct bnx2x *bp, struct sk_buff *skb,
3003         u32 *parsing_data, u32 xmit_type)
3004 {
3005         *parsing_data |=
3006                         ((((u8 *)skb_transport_header(skb) - skb->data) >> 1) <<
3007                         ETH_TX_PARSE_BD_E2_TCP_HDR_START_OFFSET_W_SHIFT) &
3008                         ETH_TX_PARSE_BD_E2_TCP_HDR_START_OFFSET_W;
3009
3010         if (xmit_type & XMIT_CSUM_TCP) {
3011                 *parsing_data |= ((tcp_hdrlen(skb) / 4) <<
3012                         ETH_TX_PARSE_BD_E2_TCP_HDR_LENGTH_DW_SHIFT) &
3013                         ETH_TX_PARSE_BD_E2_TCP_HDR_LENGTH_DW;
3014
3015                 return skb_transport_header(skb) + tcp_hdrlen(skb) - skb->data;
3016         } else
3017                 /* We support checksum offload for TCP and UDP only.
3018                  * No need to pass the UDP header length - it's a constant.
3019                  */
3020                 return skb_transport_header(skb) +
3021                                 sizeof(struct udphdr) - skb->data;
3022 }
3023
3024 static inline void bnx2x_set_sbd_csum(struct bnx2x *bp, struct sk_buff *skb,
3025         struct eth_tx_start_bd *tx_start_bd, u32 xmit_type)
3026 {
3027         tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_L4_CSUM;
3028
3029         if (xmit_type & XMIT_CSUM_V4)
3030                 tx_start_bd->bd_flags.as_bitfield |=
3031                                         ETH_TX_BD_FLAGS_IP_CSUM;
3032         else
3033                 tx_start_bd->bd_flags.as_bitfield |=
3034                                         ETH_TX_BD_FLAGS_IPV6;
3035
3036         if (!(xmit_type & XMIT_CSUM_TCP))
3037                 tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_IS_UDP;
3038 }
3039
3040 /**
3041  * bnx2x_set_pbd_csum - update PBD with checksum and return header length
3042  *
3043  * @bp:         driver handle
3044  * @skb:        packet skb
3045  * @pbd:        parse BD to be updated
3046  * @xmit_type:  xmit flags
3047  */
3048 static inline u8 bnx2x_set_pbd_csum(struct bnx2x *bp, struct sk_buff *skb,
3049         struct eth_tx_parse_bd_e1x *pbd,
3050         u32 xmit_type)
3051 {
3052         u8 hlen = (skb_network_header(skb) - skb->data) >> 1;
3053
3054         /* for now NS flag is not used in Linux */
3055         pbd->global_data =
3056                 (hlen | ((skb->protocol == cpu_to_be16(ETH_P_8021Q)) <<
3057                          ETH_TX_PARSE_BD_E1X_LLC_SNAP_EN_SHIFT));
3058
3059         pbd->ip_hlen_w = (skb_transport_header(skb) -
3060                         skb_network_header(skb)) >> 1;
3061
3062         hlen += pbd->ip_hlen_w;
3063
3064         /* We support checksum offload for TCP and UDP only */
3065         if (xmit_type & XMIT_CSUM_TCP)
3066                 hlen += tcp_hdrlen(skb) / 2;
3067         else
3068                 hlen += sizeof(struct udphdr) / 2;
3069
3070         pbd->total_hlen_w = cpu_to_le16(hlen);
3071         hlen = hlen*2;
3072
3073         if (xmit_type & XMIT_CSUM_TCP) {
3074                 pbd->tcp_pseudo_csum = swab16(tcp_hdr(skb)->check);
3075
3076         } else {
3077                 s8 fix = SKB_CS_OFF(skb); /* signed! */
3078
3079                 DP(NETIF_MSG_TX_QUEUED,
3080                    "hlen %d  fix %d  csum before fix %x\n",
3081                    le16_to_cpu(pbd->total_hlen_w), fix, SKB_CS(skb));
3082
3083                 /* HW bug: fixup the CSUM */
3084                 pbd->tcp_pseudo_csum =
3085                         bnx2x_csum_fix(skb_transport_header(skb),
3086                                        SKB_CS(skb), fix);
3087
3088                 DP(NETIF_MSG_TX_QUEUED, "csum after fix %x\n",
3089                    pbd->tcp_pseudo_csum);
3090         }
3091
3092         return hlen;
3093 }
3094
3095 /* called with netif_tx_lock
3096  * bnx2x_tx_int() runs without netif_tx_lock unless it needs to call
3097  * netif_wake_queue()
3098  */
3099 netdev_tx_t bnx2x_start_xmit(struct sk_buff *skb, struct net_device *dev)
3100 {
3101         struct bnx2x *bp = netdev_priv(dev);
3102
3103         struct netdev_queue *txq;
3104         struct bnx2x_fp_txdata *txdata;
3105         struct sw_tx_bd *tx_buf;
3106         struct eth_tx_start_bd *tx_start_bd, *first_bd;
3107         struct eth_tx_bd *tx_data_bd, *total_pkt_bd = NULL;
3108         struct eth_tx_parse_bd_e1x *pbd_e1x = NULL;
3109         struct eth_tx_parse_bd_e2 *pbd_e2 = NULL;
3110         u32 pbd_e2_parsing_data = 0;
3111         u16 pkt_prod, bd_prod;
3112         int nbd, txq_index;
3113         dma_addr_t mapping;
3114         u32 xmit_type = bnx2x_xmit_type(bp, skb);
3115         int i;
3116         u8 hlen = 0;
3117         __le16 pkt_size = 0;
3118         struct ethhdr *eth;
3119         u8 mac_type = UNICAST_ADDRESS;
3120
3121 #ifdef BNX2X_STOP_ON_ERROR
3122         if (unlikely(bp->panic))
3123                 return NETDEV_TX_BUSY;
3124 #endif
3125
3126         txq_index = skb_get_queue_mapping(skb);
3127         txq = netdev_get_tx_queue(dev, txq_index);
3128
3129         BUG_ON(txq_index >= MAX_ETH_TXQ_IDX(bp) + (CNIC_LOADED(bp) ? 1 : 0));
3130
3131         txdata = &bp->bnx2x_txq[txq_index];
3132
3133         /* enable this debug print to view the transmission queue being used
3134         DP(NETIF_MSG_TX_QUEUED, "indices: txq %d, fp %d, txdata %d\n",
3135            txq_index, fp_index, txdata_index); */
3136
3137         /* enable this debug print to view the tranmission details
3138         DP(NETIF_MSG_TX_QUEUED,
3139            "transmitting packet cid %d fp index %d txdata_index %d tx_data ptr %p fp pointer %p\n",
3140            txdata->cid, fp_index, txdata_index, txdata, fp); */
3141
3142         if (unlikely(bnx2x_tx_avail(bp, txdata) <
3143                         skb_shinfo(skb)->nr_frags +
3144                         BDS_PER_TX_PKT +
3145                         NEXT_CNT_PER_TX_PKT(MAX_BDS_PER_TX_PKT))) {
3146                 /* Handle special storage cases separately */
3147                 if (txdata->tx_ring_size == 0) {
3148                         struct bnx2x_eth_q_stats *q_stats =
3149                                 bnx2x_fp_qstats(bp, txdata->parent_fp);
3150                         q_stats->driver_filtered_tx_pkt++;
3151                         dev_kfree_skb(skb);
3152                         return NETDEV_TX_OK;
3153                 }
3154                         bnx2x_fp_qstats(bp, txdata->parent_fp)->driver_xoff++;
3155                         netif_tx_stop_queue(txq);
3156                 BNX2X_ERR("BUG! Tx ring full when queue awake!\n");
3157
3158                 return NETDEV_TX_BUSY;
3159         }
3160
3161         DP(NETIF_MSG_TX_QUEUED,
3162            "queue[%d]: SKB: summed %x  protocol %x protocol(%x,%x) gso type %x  xmit_type %x\n",
3163            txq_index, skb->ip_summed, skb->protocol, ipv6_hdr(skb)->nexthdr,
3164            ip_hdr(skb)->protocol, skb_shinfo(skb)->gso_type, xmit_type);
3165
3166         eth = (struct ethhdr *)skb->data;
3167
3168         /* set flag according to packet type (UNICAST_ADDRESS is default)*/
3169         if (unlikely(is_multicast_ether_addr(eth->h_dest))) {
3170                 if (is_broadcast_ether_addr(eth->h_dest))
3171                         mac_type = BROADCAST_ADDRESS;
3172                 else
3173                         mac_type = MULTICAST_ADDRESS;
3174         }
3175
3176 #if (MAX_SKB_FRAGS >= MAX_FETCH_BD - 3)
3177         /* First, check if we need to linearize the skb (due to FW
3178            restrictions). No need to check fragmentation if page size > 8K
3179            (there will be no violation to FW restrictions) */
3180         if (bnx2x_pkt_req_lin(bp, skb, xmit_type)) {
3181                 /* Statistics of linearization */
3182                 bp->lin_cnt++;
3183                 if (skb_linearize(skb) != 0) {
3184                         DP(NETIF_MSG_TX_QUEUED,
3185                            "SKB linearization failed - silently dropping this SKB\n");
3186                         dev_kfree_skb_any(skb);
3187                         return NETDEV_TX_OK;
3188                 }
3189         }
3190 #endif
3191         /* Map skb linear data for DMA */
3192         mapping = dma_map_single(&bp->pdev->dev, skb->data,
3193                                  skb_headlen(skb), DMA_TO_DEVICE);
3194         if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) {
3195                 DP(NETIF_MSG_TX_QUEUED,
3196                    "SKB mapping failed - silently dropping this SKB\n");
3197                 dev_kfree_skb_any(skb);
3198                 return NETDEV_TX_OK;
3199         }
3200         /*
3201         Please read carefully. First we use one BD which we mark as start,
3202         then we have a parsing info BD (used for TSO or xsum),
3203         and only then we have the rest of the TSO BDs.
3204         (don't forget to mark the last one as last,
3205         and to unmap only AFTER you write to the BD ...)
3206         And above all, all pdb sizes are in words - NOT DWORDS!
3207         */
3208
3209         /* get current pkt produced now - advance it just before sending packet
3210          * since mapping of pages may fail and cause packet to be dropped
3211          */
3212         pkt_prod = txdata->tx_pkt_prod;
3213         bd_prod = TX_BD(txdata->tx_bd_prod);
3214
3215         /* get a tx_buf and first BD
3216          * tx_start_bd may be changed during SPLIT,
3217          * but first_bd will always stay first
3218          */
3219         tx_buf = &txdata->tx_buf_ring[TX_BD(pkt_prod)];
3220         tx_start_bd = &txdata->tx_desc_ring[bd_prod].start_bd;
3221         first_bd = tx_start_bd;
3222
3223         tx_start_bd->bd_flags.as_bitfield = ETH_TX_BD_FLAGS_START_BD;
3224         SET_FLAG(tx_start_bd->general_data,
3225                  ETH_TX_START_BD_PARSE_NBDS,
3226                  0);
3227
3228         /* header nbd */
3229         SET_FLAG(tx_start_bd->general_data, ETH_TX_START_BD_HDR_NBDS, 1);
3230
3231         /* remember the first BD of the packet */
3232         tx_buf->first_bd = txdata->tx_bd_prod;
3233         tx_buf->skb = skb;
3234         tx_buf->flags = 0;
3235
3236         DP(NETIF_MSG_TX_QUEUED,
3237            "sending pkt %u @%p  next_idx %u  bd %u @%p\n",
3238            pkt_prod, tx_buf, txdata->tx_pkt_prod, bd_prod, tx_start_bd);
3239
3240         if (vlan_tx_tag_present(skb)) {
3241                 tx_start_bd->vlan_or_ethertype =
3242                     cpu_to_le16(vlan_tx_tag_get(skb));
3243                 tx_start_bd->bd_flags.as_bitfield |=
3244                     (X_ETH_OUTBAND_VLAN << ETH_TX_BD_FLAGS_VLAN_MODE_SHIFT);
3245         } else
3246                 tx_start_bd->vlan_or_ethertype = cpu_to_le16(pkt_prod);
3247
3248         /* turn on parsing and get a BD */
3249         bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
3250
3251         if (xmit_type & XMIT_CSUM)
3252                 bnx2x_set_sbd_csum(bp, skb, tx_start_bd, xmit_type);
3253
3254         if (!CHIP_IS_E1x(bp)) {
3255                 pbd_e2 = &txdata->tx_desc_ring[bd_prod].parse_bd_e2;
3256                 memset(pbd_e2, 0, sizeof(struct eth_tx_parse_bd_e2));
3257                 /* Set PBD in checksum offload case */
3258                 if (xmit_type & XMIT_CSUM)
3259                         hlen = bnx2x_set_pbd_csum_e2(bp, skb,
3260                                                      &pbd_e2_parsing_data,
3261                                                      xmit_type);
3262                 if (IS_MF_SI(bp)) {
3263                         /*
3264                          * fill in the MAC addresses in the PBD - for local
3265                          * switching
3266                          */
3267                         bnx2x_set_fw_mac_addr(&pbd_e2->src_mac_addr_hi,
3268                                               &pbd_e2->src_mac_addr_mid,
3269                                               &pbd_e2->src_mac_addr_lo,
3270                                               eth->h_source);
3271                         bnx2x_set_fw_mac_addr(&pbd_e2->dst_mac_addr_hi,
3272                                               &pbd_e2->dst_mac_addr_mid,
3273                                               &pbd_e2->dst_mac_addr_lo,
3274                                               eth->h_dest);
3275                 }
3276
3277                 SET_FLAG(pbd_e2_parsing_data,
3278                          ETH_TX_PARSE_BD_E2_ETH_ADDR_TYPE, mac_type);
3279         } else {
3280                 u16 global_data = 0;
3281                 pbd_e1x = &txdata->tx_desc_ring[bd_prod].parse_bd_e1x;
3282                 memset(pbd_e1x, 0, sizeof(struct eth_tx_parse_bd_e1x));
3283                 /* Set PBD in checksum offload case */
3284                 if (xmit_type & XMIT_CSUM)
3285                         hlen = bnx2x_set_pbd_csum(bp, skb, pbd_e1x, xmit_type);
3286
3287                 SET_FLAG(global_data,
3288                          ETH_TX_PARSE_BD_E1X_ETH_ADDR_TYPE, mac_type);
3289                 pbd_e1x->global_data |= cpu_to_le16(global_data);
3290         }
3291
3292         /* Setup the data pointer of the first BD of the packet */
3293         tx_start_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
3294         tx_start_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
3295         nbd = 2; /* start_bd + pbd + frags (updated when pages are mapped) */
3296         tx_start_bd->nbytes = cpu_to_le16(skb_headlen(skb));
3297         pkt_size = tx_start_bd->nbytes;
3298
3299         DP(NETIF_MSG_TX_QUEUED,
3300            "first bd @%p  addr (%x:%x)  nbd %d  nbytes %d  flags %x  vlan %x\n",
3301            tx_start_bd, tx_start_bd->addr_hi, tx_start_bd->addr_lo,
3302            le16_to_cpu(tx_start_bd->nbd), le16_to_cpu(tx_start_bd->nbytes),
3303            tx_start_bd->bd_flags.as_bitfield,
3304            le16_to_cpu(tx_start_bd->vlan_or_ethertype));
3305
3306         if (xmit_type & XMIT_GSO) {
3307
3308                 DP(NETIF_MSG_TX_QUEUED,
3309                    "TSO packet len %d  hlen %d  total len %d  tso size %d\n",
3310                    skb->len, hlen, skb_headlen(skb),
3311                    skb_shinfo(skb)->gso_size);
3312
3313                 tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_SW_LSO;
3314
3315                 if (unlikely(skb_headlen(skb) > hlen))
3316                         bd_prod = bnx2x_tx_split(bp, txdata, tx_buf,
3317                                                  &tx_start_bd, hlen,
3318                                                  bd_prod, ++nbd);
3319                 if (!CHIP_IS_E1x(bp))
3320                         bnx2x_set_pbd_gso_e2(skb, &pbd_e2_parsing_data,
3321                                              xmit_type);
3322                 else
3323                         bnx2x_set_pbd_gso(skb, pbd_e1x, xmit_type);
3324         }
3325
3326         /* Set the PBD's parsing_data field if not zero
3327          * (for the chips newer than 57711).
3328          */
3329         if (pbd_e2_parsing_data)
3330                 pbd_e2->parsing_data = cpu_to_le32(pbd_e2_parsing_data);
3331
3332         tx_data_bd = (struct eth_tx_bd *)tx_start_bd;
3333
3334         /* Handle fragmented skb */
3335         for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
3336                 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
3337
3338                 mapping = skb_frag_dma_map(&bp->pdev->dev, frag, 0,
3339                                            skb_frag_size(frag), DMA_TO_DEVICE);
3340                 if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) {
3341                         unsigned int pkts_compl = 0, bytes_compl = 0;
3342
3343                         DP(NETIF_MSG_TX_QUEUED,
3344                            "Unable to map page - dropping packet...\n");
3345
3346                         /* we need unmap all buffers already mapped
3347                          * for this SKB;
3348                          * first_bd->nbd need to be properly updated
3349                          * before call to bnx2x_free_tx_pkt
3350                          */
3351                         first_bd->nbd = cpu_to_le16(nbd);
3352                         bnx2x_free_tx_pkt(bp, txdata,
3353                                           TX_BD(txdata->tx_pkt_prod),
3354                                           &pkts_compl, &bytes_compl);
3355                         return NETDEV_TX_OK;
3356                 }
3357
3358                 bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
3359                 tx_data_bd = &txdata->tx_desc_ring[bd_prod].reg_bd;
3360                 if (total_pkt_bd == NULL)
3361                         total_pkt_bd = &txdata->tx_desc_ring[bd_prod].reg_bd;
3362
3363                 tx_data_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
3364                 tx_data_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
3365                 tx_data_bd->nbytes = cpu_to_le16(skb_frag_size(frag));
3366                 le16_add_cpu(&pkt_size, skb_frag_size(frag));
3367                 nbd++;
3368
3369                 DP(NETIF_MSG_TX_QUEUED,
3370                    "frag %d  bd @%p  addr (%x:%x)  nbytes %d\n",
3371                    i, tx_data_bd, tx_data_bd->addr_hi, tx_data_bd->addr_lo,
3372                    le16_to_cpu(tx_data_bd->nbytes));
3373         }
3374
3375         DP(NETIF_MSG_TX_QUEUED, "last bd @%p\n", tx_data_bd);
3376
3377         /* update with actual num BDs */
3378         first_bd->nbd = cpu_to_le16(nbd);
3379
3380         bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
3381
3382         /* now send a tx doorbell, counting the next BD
3383          * if the packet contains or ends with it
3384          */
3385         if (TX_BD_POFF(bd_prod) < nbd)
3386                 nbd++;
3387
3388         /* total_pkt_bytes should be set on the first data BD if
3389          * it's not an LSO packet and there is more than one
3390          * data BD. In this case pkt_size is limited by an MTU value.
3391          * However we prefer to set it for an LSO packet (while we don't
3392          * have to) in order to save some CPU cycles in a none-LSO
3393          * case, when we much more care about them.
3394          */
3395         if (total_pkt_bd != NULL)
3396                 total_pkt_bd->total_pkt_bytes = pkt_size;
3397
3398         if (pbd_e1x)
3399                 DP(NETIF_MSG_TX_QUEUED,
3400                    "PBD (E1X) @%p  ip_data %x  ip_hlen %u  ip_id %u  lso_mss %u  tcp_flags %x  xsum %x  seq %u  hlen %u\n",
3401                    pbd_e1x, pbd_e1x->global_data, pbd_e1x->ip_hlen_w,
3402                    pbd_e1x->ip_id, pbd_e1x->lso_mss, pbd_e1x->tcp_flags,
3403                    pbd_e1x->tcp_pseudo_csum, pbd_e1x->tcp_send_seq,
3404                     le16_to_cpu(pbd_e1x->total_hlen_w));
3405         if (pbd_e2)
3406                 DP(NETIF_MSG_TX_QUEUED,
3407                    "PBD (E2) @%p  dst %x %x %x src %x %x %x parsing_data %x\n",
3408                    pbd_e2, pbd_e2->dst_mac_addr_hi, pbd_e2->dst_mac_addr_mid,
3409                    pbd_e2->dst_mac_addr_lo, pbd_e2->src_mac_addr_hi,
3410                    pbd_e2->src_mac_addr_mid, pbd_e2->src_mac_addr_lo,
3411                    pbd_e2->parsing_data);
3412         DP(NETIF_MSG_TX_QUEUED, "doorbell: nbd %d  bd %u\n", nbd, bd_prod);
3413
3414         netdev_tx_sent_queue(txq, skb->len);
3415
3416         skb_tx_timestamp(skb);
3417
3418         txdata->tx_pkt_prod++;
3419         /*
3420          * Make sure that the BD data is updated before updating the producer
3421          * since FW might read the BD right after the producer is updated.
3422          * This is only applicable for weak-ordered memory model archs such
3423          * as IA-64. The following barrier is also mandatory since FW will
3424          * assumes packets must have BDs.
3425          */
3426         wmb();
3427
3428         txdata->tx_db.data.prod += nbd;
3429         barrier();
3430
3431         DOORBELL(bp, txdata->cid, txdata->tx_db.raw);
3432
3433         mmiowb();
3434
3435         txdata->tx_bd_prod += nbd;
3436
3437         if (unlikely(bnx2x_tx_avail(bp, txdata) < MAX_DESC_PER_TX_PKT)) {
3438                 netif_tx_stop_queue(txq);
3439
3440                 /* paired memory barrier is in bnx2x_tx_int(), we have to keep
3441                  * ordering of set_bit() in netif_tx_stop_queue() and read of
3442                  * fp->bd_tx_cons */
3443                 smp_mb();
3444
3445                 bnx2x_fp_qstats(bp, txdata->parent_fp)->driver_xoff++;
3446                 if (bnx2x_tx_avail(bp, txdata) >= MAX_DESC_PER_TX_PKT)
3447                         netif_tx_wake_queue(txq);
3448         }
3449         txdata->tx_pkt++;
3450
3451         return NETDEV_TX_OK;
3452 }
3453
3454 /**
3455  * bnx2x_setup_tc - routine to configure net_device for multi tc
3456  *
3457  * @netdev: net device to configure
3458  * @tc: number of traffic classes to enable
3459  *
3460  * callback connected to the ndo_setup_tc function pointer
3461  */
3462 int bnx2x_setup_tc(struct net_device *dev, u8 num_tc)
3463 {
3464         int cos, prio, count, offset;
3465         struct bnx2x *bp = netdev_priv(dev);
3466
3467         /* setup tc must be called under rtnl lock */
3468         ASSERT_RTNL();
3469
3470         /* no traffic classes requested. aborting */
3471         if (!num_tc) {
3472                 netdev_reset_tc(dev);
3473                 return 0;
3474         }
3475
3476         /* requested to support too many traffic classes */
3477         if (num_tc > bp->max_cos) {
3478                 BNX2X_ERR("support for too many traffic classes requested: %d. max supported is %d\n",
3479                           num_tc, bp->max_cos);
3480                 return -EINVAL;
3481         }
3482
3483         /* declare amount of supported traffic classes */
3484         if (netdev_set_num_tc(dev, num_tc)) {
3485                 BNX2X_ERR("failed to declare %d traffic classes\n", num_tc);
3486                 return -EINVAL;
3487         }
3488
3489         /* configure priority to traffic class mapping */
3490         for (prio = 0; prio < BNX2X_MAX_PRIORITY; prio++) {
3491                 netdev_set_prio_tc_map(dev, prio, bp->prio_to_cos[prio]);
3492                 DP(BNX2X_MSG_SP | NETIF_MSG_IFUP,
3493                    "mapping priority %d to tc %d\n",
3494                    prio, bp->prio_to_cos[prio]);
3495         }
3496
3497
3498         /* Use this configuration to diffrentiate tc0 from other COSes
3499            This can be used for ets or pfc, and save the effort of setting
3500            up a multio class queue disc or negotiating DCBX with a switch
3501         netdev_set_prio_tc_map(dev, 0, 0);
3502         DP(BNX2X_MSG_SP, "mapping priority %d to tc %d\n", 0, 0);
3503         for (prio = 1; prio < 16; prio++) {
3504                 netdev_set_prio_tc_map(dev, prio, 1);
3505                 DP(BNX2X_MSG_SP, "mapping priority %d to tc %d\n", prio, 1);
3506         } */
3507
3508         /* configure traffic class to transmission queue mapping */
3509         for (cos = 0; cos < bp->max_cos; cos++) {
3510                 count = BNX2X_NUM_ETH_QUEUES(bp);
3511                 offset = cos * BNX2X_NUM_NON_CNIC_QUEUES(bp);
3512                 netdev_set_tc_queue(dev, cos, count, offset);
3513                 DP(BNX2X_MSG_SP | NETIF_MSG_IFUP,
3514                    "mapping tc %d to offset %d count %d\n",
3515                    cos, offset, count);
3516         }
3517
3518         return 0;
3519 }
3520
3521 /* called with rtnl_lock */
3522 int bnx2x_change_mac_addr(struct net_device *dev, void *p)
3523 {
3524         struct sockaddr *addr = p;
3525         struct bnx2x *bp = netdev_priv(dev);
3526         int rc = 0;
3527
3528         if (!bnx2x_is_valid_ether_addr(bp, addr->sa_data)) {
3529                 BNX2X_ERR("Requested MAC address is not valid\n");
3530                 return -EINVAL;
3531         }
3532
3533         if ((IS_MF_STORAGE_SD(bp) || IS_MF_FCOE_AFEX(bp)) &&
3534             !is_zero_ether_addr(addr->sa_data)) {
3535                 BNX2X_ERR("Can't configure non-zero address on iSCSI or FCoE functions in MF-SD mode\n");
3536                 return -EINVAL;
3537         }
3538
3539         if (netif_running(dev))  {
3540                 rc = bnx2x_set_eth_mac(bp, false);
3541                 if (rc)
3542                         return rc;
3543         }
3544
3545         dev->addr_assign_type &= ~NET_ADDR_RANDOM;
3546         memcpy(dev->dev_addr, addr->sa_data, dev->addr_len);
3547
3548         if (netif_running(dev))
3549                 rc = bnx2x_set_eth_mac(bp, true);
3550
3551         return rc;
3552 }
3553
3554 static void bnx2x_free_fp_mem_at(struct bnx2x *bp, int fp_index)
3555 {
3556         union host_hc_status_block *sb = &bnx2x_fp(bp, fp_index, status_blk);
3557         struct bnx2x_fastpath *fp = &bp->fp[fp_index];
3558         u8 cos;
3559
3560         /* Common */
3561
3562         if (IS_FCOE_IDX(fp_index)) {
3563                 memset(sb, 0, sizeof(union host_hc_status_block));
3564                 fp->status_blk_mapping = 0;
3565         } else {
3566                 /* status blocks */
3567                 if (!CHIP_IS_E1x(bp))
3568                         BNX2X_PCI_FREE(sb->e2_sb,
3569                                        bnx2x_fp(bp, fp_index,
3570                                                 status_blk_mapping),
3571                                        sizeof(struct host_hc_status_block_e2));
3572                 else
3573                         BNX2X_PCI_FREE(sb->e1x_sb,
3574                                        bnx2x_fp(bp, fp_index,
3575                                                 status_blk_mapping),
3576                                        sizeof(struct host_hc_status_block_e1x));
3577         }
3578
3579         /* Rx */
3580         if (!skip_rx_queue(bp, fp_index)) {
3581                 bnx2x_free_rx_bds(fp);
3582
3583                 /* fastpath rx rings: rx_buf rx_desc rx_comp */
3584                 BNX2X_FREE(bnx2x_fp(bp, fp_index, rx_buf_ring));
3585                 BNX2X_PCI_FREE(bnx2x_fp(bp, fp_index, rx_desc_ring),
3586                                bnx2x_fp(bp, fp_index, rx_desc_mapping),
3587                                sizeof(struct eth_rx_bd) * NUM_RX_BD);
3588
3589                 BNX2X_PCI_FREE(bnx2x_fp(bp, fp_index, rx_comp_ring),
3590                                bnx2x_fp(bp, fp_index, rx_comp_mapping),
3591                                sizeof(struct eth_fast_path_rx_cqe) *
3592                                NUM_RCQ_BD);
3593
3594                 /* SGE ring */
3595                 BNX2X_FREE(bnx2x_fp(bp, fp_index, rx_page_ring));
3596                 BNX2X_PCI_FREE(bnx2x_fp(bp, fp_index, rx_sge_ring),
3597                                bnx2x_fp(bp, fp_index, rx_sge_mapping),
3598                                BCM_PAGE_SIZE * NUM_RX_SGE_PAGES);
3599         }
3600
3601         /* Tx */
3602         if (!skip_tx_queue(bp, fp_index)) {
3603                 /* fastpath tx rings: tx_buf tx_desc */
3604                 for_each_cos_in_tx_queue(fp, cos) {
3605                         struct bnx2x_fp_txdata *txdata = fp->txdata_ptr[cos];
3606
3607                         DP(NETIF_MSG_IFDOWN,
3608                            "freeing tx memory of fp %d cos %d cid %d\n",
3609                            fp_index, cos, txdata->cid);
3610
3611                         BNX2X_FREE(txdata->tx_buf_ring);
3612                         BNX2X_PCI_FREE(txdata->tx_desc_ring,
3613                                 txdata->tx_desc_mapping,
3614                                 sizeof(union eth_tx_bd_types) * NUM_TX_BD);
3615                 }
3616         }
3617         /* end of fastpath */
3618 }
3619
3620 void bnx2x_free_fp_mem_cnic(struct bnx2x *bp)
3621 {
3622         int i;
3623         for_each_cnic_queue(bp, i)
3624                 bnx2x_free_fp_mem_at(bp, i);
3625 }
3626
3627 void bnx2x_free_fp_mem(struct bnx2x *bp)
3628 {
3629         int i;
3630         for_each_eth_queue(bp, i)
3631                 bnx2x_free_fp_mem_at(bp, i);
3632 }
3633
3634 static void set_sb_shortcuts(struct bnx2x *bp, int index)
3635 {
3636         union host_hc_status_block status_blk = bnx2x_fp(bp, index, status_blk);
3637         if (!CHIP_IS_E1x(bp)) {
3638                 bnx2x_fp(bp, index, sb_index_values) =
3639                         (__le16 *)status_blk.e2_sb->sb.index_values;
3640                 bnx2x_fp(bp, index, sb_running_index) =
3641                         (__le16 *)status_blk.e2_sb->sb.running_index;
3642         } else {
3643                 bnx2x_fp(bp, index, sb_index_values) =
3644                         (__le16 *)status_blk.e1x_sb->sb.index_values;
3645                 bnx2x_fp(bp, index, sb_running_index) =
3646                         (__le16 *)status_blk.e1x_sb->sb.running_index;
3647         }
3648 }
3649
3650 /* Returns the number of actually allocated BDs */
3651 static int bnx2x_alloc_rx_bds(struct bnx2x_fastpath *fp,
3652                               int rx_ring_size)
3653 {
3654         struct bnx2x *bp = fp->bp;
3655         u16 ring_prod, cqe_ring_prod;
3656         int i, failure_cnt = 0;
3657
3658         fp->rx_comp_cons = 0;
3659         cqe_ring_prod = ring_prod = 0;
3660
3661         /* This routine is called only during fo init so
3662          * fp->eth_q_stats.rx_skb_alloc_failed = 0
3663          */
3664         for (i = 0; i < rx_ring_size; i++) {
3665                 if (bnx2x_alloc_rx_data(bp, fp, ring_prod) < 0) {
3666                         failure_cnt++;
3667                         continue;
3668                 }
3669                 ring_prod = NEXT_RX_IDX(ring_prod);
3670                 cqe_ring_prod = NEXT_RCQ_IDX(cqe_ring_prod);
3671                 WARN_ON(ring_prod <= (i - failure_cnt));
3672         }
3673
3674         if (failure_cnt)
3675                 BNX2X_ERR("was only able to allocate %d rx skbs on queue[%d]\n",
3676                           i - failure_cnt, fp->index);
3677
3678         fp->rx_bd_prod = ring_prod;
3679         /* Limit the CQE producer by the CQE ring size */
3680         fp->rx_comp_prod = min_t(u16, NUM_RCQ_RINGS*RCQ_DESC_CNT,
3681                                cqe_ring_prod);
3682         fp->rx_pkt = fp->rx_calls = 0;
3683
3684         bnx2x_fp_stats(bp, fp)->eth_q_stats.rx_skb_alloc_failed += failure_cnt;
3685
3686         return i - failure_cnt;
3687 }
3688
3689 static void bnx2x_set_next_page_rx_cq(struct bnx2x_fastpath *fp)
3690 {
3691         int i;
3692
3693         for (i = 1; i <= NUM_RCQ_RINGS; i++) {
3694                 struct eth_rx_cqe_next_page *nextpg;
3695
3696                 nextpg = (struct eth_rx_cqe_next_page *)
3697                         &fp->rx_comp_ring[RCQ_DESC_CNT * i - 1];
3698                 nextpg->addr_hi =
3699                         cpu_to_le32(U64_HI(fp->rx_comp_mapping +
3700                                    BCM_PAGE_SIZE*(i % NUM_RCQ_RINGS)));
3701                 nextpg->addr_lo =
3702                         cpu_to_le32(U64_LO(fp->rx_comp_mapping +
3703                                    BCM_PAGE_SIZE*(i % NUM_RCQ_RINGS)));
3704         }
3705 }
3706
3707 static int bnx2x_alloc_fp_mem_at(struct bnx2x *bp, int index)
3708 {
3709         union host_hc_status_block *sb;
3710         struct bnx2x_fastpath *fp = &bp->fp[index];
3711         int ring_size = 0;
3712         u8 cos;
3713         int rx_ring_size = 0;
3714
3715         if (!bp->rx_ring_size &&
3716             (IS_MF_STORAGE_SD(bp) || IS_MF_FCOE_AFEX(bp))) {
3717                 rx_ring_size = MIN_RX_SIZE_NONTPA;
3718                 bp->rx_ring_size = rx_ring_size;
3719         } else if (!bp->rx_ring_size) {
3720                 rx_ring_size = MAX_RX_AVAIL/BNX2X_NUM_RX_QUEUES(bp);
3721
3722                 if (CHIP_IS_E3(bp)) {
3723                         u32 cfg = SHMEM_RD(bp,
3724                                            dev_info.port_hw_config[BP_PORT(bp)].
3725                                            default_cfg);
3726
3727                         /* Decrease ring size for 1G functions */
3728                         if ((cfg & PORT_HW_CFG_NET_SERDES_IF_MASK) ==
3729                             PORT_HW_CFG_NET_SERDES_IF_SGMII)
3730                                 rx_ring_size /= 10;
3731                 }
3732
3733                 /* allocate at least number of buffers required by FW */
3734                 rx_ring_size = max_t(int, bp->disable_tpa ? MIN_RX_SIZE_NONTPA :
3735                                      MIN_RX_SIZE_TPA, rx_ring_size);
3736
3737                 bp->rx_ring_size = rx_ring_size;
3738         } else /* if rx_ring_size specified - use it */
3739                 rx_ring_size = bp->rx_ring_size;
3740
3741         /* Common */
3742         sb = &bnx2x_fp(bp, index, status_blk);
3743
3744         if (!IS_FCOE_IDX(index)) {
3745                 /* status blocks */
3746                 if (!CHIP_IS_E1x(bp))
3747                         BNX2X_PCI_ALLOC(sb->e2_sb,
3748                                 &bnx2x_fp(bp, index, status_blk_mapping),
3749                                 sizeof(struct host_hc_status_block_e2));
3750                 else
3751                         BNX2X_PCI_ALLOC(sb->e1x_sb,
3752                                 &bnx2x_fp(bp, index, status_blk_mapping),
3753                             sizeof(struct host_hc_status_block_e1x));
3754         }
3755
3756         /* FCoE Queue uses Default SB and doesn't ACK the SB, thus no need to
3757          * set shortcuts for it.
3758          */
3759         if (!IS_FCOE_IDX(index))
3760                 set_sb_shortcuts(bp, index);
3761
3762         /* Tx */
3763         if (!skip_tx_queue(bp, index)) {
3764                 /* fastpath tx rings: tx_buf tx_desc */
3765                 for_each_cos_in_tx_queue(fp, cos) {
3766                         struct bnx2x_fp_txdata *txdata = fp->txdata_ptr[cos];
3767
3768                         DP(NETIF_MSG_IFUP,
3769                            "allocating tx memory of fp %d cos %d\n",
3770                            index, cos);
3771
3772                         BNX2X_ALLOC(txdata->tx_buf_ring,
3773                                 sizeof(struct sw_tx_bd) * NUM_TX_BD);
3774                         BNX2X_PCI_ALLOC(txdata->tx_desc_ring,
3775                                 &txdata->tx_desc_mapping,
3776                                 sizeof(union eth_tx_bd_types) * NUM_TX_BD);
3777                 }
3778         }
3779
3780         /* Rx */
3781         if (!skip_rx_queue(bp, index)) {
3782                 /* fastpath rx rings: rx_buf rx_desc rx_comp */
3783                 BNX2X_ALLOC(bnx2x_fp(bp, index, rx_buf_ring),
3784                                 sizeof(struct sw_rx_bd) * NUM_RX_BD);
3785                 BNX2X_PCI_ALLOC(bnx2x_fp(bp, index, rx_desc_ring),
3786                                 &bnx2x_fp(bp, index, rx_desc_mapping),
3787                                 sizeof(struct eth_rx_bd) * NUM_RX_BD);
3788
3789                 BNX2X_PCI_ALLOC(bnx2x_fp(bp, index, rx_comp_ring),
3790                                 &bnx2x_fp(bp, index, rx_comp_mapping),
3791                                 sizeof(struct eth_fast_path_rx_cqe) *
3792                                 NUM_RCQ_BD);
3793
3794                 /* SGE ring */
3795                 BNX2X_ALLOC(bnx2x_fp(bp, index, rx_page_ring),
3796                                 sizeof(struct sw_rx_page) * NUM_RX_SGE);
3797                 BNX2X_PCI_ALLOC(bnx2x_fp(bp, index, rx_sge_ring),
3798                                 &bnx2x_fp(bp, index, rx_sge_mapping),
3799                                 BCM_PAGE_SIZE * NUM_RX_SGE_PAGES);
3800                 /* RX BD ring */
3801                 bnx2x_set_next_page_rx_bd(fp);
3802
3803                 /* CQ ring */
3804                 bnx2x_set_next_page_rx_cq(fp);
3805
3806                 /* BDs */
3807                 ring_size = bnx2x_alloc_rx_bds(fp, rx_ring_size);
3808                 if (ring_size < rx_ring_size)
3809                         goto alloc_mem_err;
3810         }
3811
3812         return 0;
3813
3814 /* handles low memory cases */
3815 alloc_mem_err:
3816         BNX2X_ERR("Unable to allocate full memory for queue %d (size %d)\n",
3817                                                 index, ring_size);
3818         /* FW will drop all packets if queue is not big enough,
3819          * In these cases we disable the queue
3820          * Min size is different for OOO, TPA and non-TPA queues
3821          */
3822         if (ring_size < (fp->disable_tpa ?
3823                                 MIN_RX_SIZE_NONTPA : MIN_RX_SIZE_TPA)) {
3824                         /* release memory allocated for this queue */
3825                         bnx2x_free_fp_mem_at(bp, index);
3826                         return -ENOMEM;
3827         }
3828         return 0;
3829 }
3830
3831 int bnx2x_alloc_fp_mem_cnic(struct bnx2x *bp)
3832 {
3833         if (!NO_FCOE(bp))
3834                 /* FCoE */
3835                 if (bnx2x_alloc_fp_mem_at(bp, FCOE_IDX(bp)))
3836                         /* we will fail load process instead of mark
3837                          * NO_FCOE_FLAG
3838                          */
3839                         return -ENOMEM;
3840
3841         return 0;
3842 }
3843
3844 int bnx2x_alloc_fp_mem(struct bnx2x *bp)
3845 {
3846         int i;
3847
3848         /* 1. Allocate FP for leading - fatal if error
3849          * 2. Allocate RSS - fix number of queues if error
3850          */
3851
3852         /* leading */
3853         if (bnx2x_alloc_fp_mem_at(bp, 0))
3854                 return -ENOMEM;
3855
3856         /* RSS */
3857         for_each_nondefault_eth_queue(bp, i)
3858                 if (bnx2x_alloc_fp_mem_at(bp, i))
3859                         break;
3860
3861         /* handle memory failures */
3862         if (i != BNX2X_NUM_ETH_QUEUES(bp)) {
3863                 int delta = BNX2X_NUM_ETH_QUEUES(bp) - i;
3864
3865                 WARN_ON(delta < 0);
3866                 if (CNIC_SUPPORT(bp))
3867                         /* move non eth FPs next to last eth FP
3868                          * must be done in that order
3869                          * FCOE_IDX < FWD_IDX < OOO_IDX
3870                          */
3871
3872                         /* move FCoE fp even NO_FCOE_FLAG is on */
3873                         bnx2x_move_fp(bp, FCOE_IDX(bp), FCOE_IDX(bp) - delta);
3874                 bp->num_ethernet_queues -= delta;
3875                 bp->num_queues = bp->num_ethernet_queues +
3876                                  bp->num_cnic_queues;
3877                 BNX2X_ERR("Adjusted num of queues from %d to %d\n",
3878                           bp->num_queues + delta, bp->num_queues);
3879         }
3880
3881         return 0;
3882 }
3883
3884 void bnx2x_free_mem_bp(struct bnx2x *bp)
3885 {
3886         kfree(bp->fp->tpa_info);
3887         kfree(bp->fp);
3888         kfree(bp->sp_objs);
3889         kfree(bp->fp_stats);
3890         kfree(bp->bnx2x_txq);
3891         kfree(bp->msix_table);
3892         kfree(bp->ilt);
3893 }
3894
3895 int bnx2x_alloc_mem_bp(struct bnx2x *bp)
3896 {
3897         struct bnx2x_fastpath *fp;
3898         struct msix_entry *tbl;
3899         struct bnx2x_ilt *ilt;
3900         int msix_table_size = 0;
3901         int fp_array_size, txq_array_size;
3902         int i;
3903
3904         /*
3905          * The biggest MSI-X table we might need is as a maximum number of fast
3906          * path IGU SBs plus default SB (for PF).
3907          */
3908         msix_table_size = bp->igu_sb_cnt + 1;
3909
3910         /* fp array: RSS plus CNIC related L2 queues */
3911         fp_array_size = BNX2X_MAX_RSS_COUNT(bp) + CNIC_SUPPORT(bp);
3912         BNX2X_DEV_INFO("fp_array_size %d", fp_array_size);
3913
3914         fp = kcalloc(fp_array_size, sizeof(*fp), GFP_KERNEL);
3915         if (!fp)
3916                 goto alloc_err;
3917         for (i = 0; i < fp_array_size; i++) {
3918                 fp[i].tpa_info =
3919                         kcalloc(ETH_MAX_AGGREGATION_QUEUES_E1H_E2,
3920                                 sizeof(struct bnx2x_agg_info), GFP_KERNEL);
3921                 if (!(fp[i].tpa_info))
3922                         goto alloc_err;
3923         }
3924
3925         bp->fp = fp;
3926
3927         /* allocate sp objs */
3928         bp->sp_objs = kcalloc(fp_array_size, sizeof(struct bnx2x_sp_objs),
3929                               GFP_KERNEL);
3930         if (!bp->sp_objs)
3931                 goto alloc_err;
3932
3933         /* allocate fp_stats */
3934         bp->fp_stats = kcalloc(fp_array_size, sizeof(struct bnx2x_fp_stats),
3935                                GFP_KERNEL);
3936         if (!bp->fp_stats)
3937                 goto alloc_err;
3938
3939         /* Allocate memory for the transmission queues array */
3940         txq_array_size =
3941                 BNX2X_MAX_RSS_COUNT(bp) * BNX2X_MULTI_TX_COS + CNIC_SUPPORT(bp);
3942         BNX2X_DEV_INFO("txq_array_size %d", txq_array_size);
3943
3944         bp->bnx2x_txq = kcalloc(txq_array_size, sizeof(struct bnx2x_fp_txdata),
3945                                 GFP_KERNEL);
3946         if (!bp->bnx2x_txq)
3947                 goto alloc_err;
3948
3949         /* msix table */
3950         tbl = kcalloc(msix_table_size, sizeof(*tbl), GFP_KERNEL);
3951         if (!tbl)
3952                 goto alloc_err;
3953         bp->msix_table = tbl;
3954
3955         /* ilt */
3956         ilt = kzalloc(sizeof(*ilt), GFP_KERNEL);
3957         if (!ilt)
3958                 goto alloc_err;
3959         bp->ilt = ilt;
3960
3961         return 0;
3962 alloc_err:
3963         bnx2x_free_mem_bp(bp);
3964         return -ENOMEM;
3965
3966 }
3967
3968 int bnx2x_reload_if_running(struct net_device *dev)
3969 {
3970         struct bnx2x *bp = netdev_priv(dev);
3971
3972         if (unlikely(!netif_running(dev)))
3973                 return 0;
3974
3975         bnx2x_nic_unload(bp, UNLOAD_NORMAL, true);
3976         return bnx2x_nic_load(bp, LOAD_NORMAL);
3977 }
3978
3979 int bnx2x_get_cur_phy_idx(struct bnx2x *bp)
3980 {
3981         u32 sel_phy_idx = 0;
3982         if (bp->link_params.num_phys <= 1)
3983                 return INT_PHY;
3984
3985         if (bp->link_vars.link_up) {
3986                 sel_phy_idx = EXT_PHY1;
3987                 /* In case link is SERDES, check if the EXT_PHY2 is the one */
3988                 if ((bp->link_vars.link_status & LINK_STATUS_SERDES_LINK) &&
3989                     (bp->link_params.phy[EXT_PHY2].supported & SUPPORTED_FIBRE))
3990                         sel_phy_idx = EXT_PHY2;
3991         } else {
3992
3993                 switch (bnx2x_phy_selection(&bp->link_params)) {
3994                 case PORT_HW_CFG_PHY_SELECTION_HARDWARE_DEFAULT:
3995                 case PORT_HW_CFG_PHY_SELECTION_FIRST_PHY:
3996                 case PORT_HW_CFG_PHY_SELECTION_FIRST_PHY_PRIORITY:
3997                        sel_phy_idx = EXT_PHY1;
3998                        break;
3999                 case PORT_HW_CFG_PHY_SELECTION_SECOND_PHY:
4000                 case PORT_HW_CFG_PHY_SELECTION_SECOND_PHY_PRIORITY:
4001                        sel_phy_idx = EXT_PHY2;
4002                        break;
4003                 }
4004         }
4005
4006         return sel_phy_idx;
4007
4008 }
4009 int bnx2x_get_link_cfg_idx(struct bnx2x *bp)
4010 {
4011         u32 sel_phy_idx = bnx2x_get_cur_phy_idx(bp);
4012         /*
4013          * The selected actived PHY is always after swapping (in case PHY
4014          * swapping is enabled). So when swapping is enabled, we need to reverse
4015          * the configuration
4016          */
4017
4018         if (bp->link_params.multi_phy_config &
4019             PORT_HW_CFG_PHY_SWAPPED_ENABLED) {
4020                 if (sel_phy_idx == EXT_PHY1)
4021                         sel_phy_idx = EXT_PHY2;
4022                 else if (sel_phy_idx == EXT_PHY2)
4023                         sel_phy_idx = EXT_PHY1;
4024         }
4025         return LINK_CONFIG_IDX(sel_phy_idx);
4026 }
4027
4028 #ifdef NETDEV_FCOE_WWNN
4029 int bnx2x_fcoe_get_wwn(struct net_device *dev, u64 *wwn, int type)
4030 {
4031         struct bnx2x *bp = netdev_priv(dev);
4032         struct cnic_eth_dev *cp = &bp->cnic_eth_dev;
4033
4034         switch (type) {
4035         case NETDEV_FCOE_WWNN:
4036                 *wwn = HILO_U64(cp->fcoe_wwn_node_name_hi,
4037                                 cp->fcoe_wwn_node_name_lo);
4038                 break;
4039         case NETDEV_FCOE_WWPN:
4040                 *wwn = HILO_U64(cp->fcoe_wwn_port_name_hi,
4041                                 cp->fcoe_wwn_port_name_lo);
4042                 break;
4043         default:
4044                 BNX2X_ERR("Wrong WWN type requested - %d\n", type);
4045                 return -EINVAL;
4046         }
4047
4048         return 0;
4049 }
4050 #endif
4051
4052 /* called with rtnl_lock */
4053 int bnx2x_change_mtu(struct net_device *dev, int new_mtu)
4054 {
4055         struct bnx2x *bp = netdev_priv(dev);
4056
4057         if (bp->recovery_state != BNX2X_RECOVERY_DONE) {
4058                 BNX2X_ERR("Can't perform change MTU during parity recovery\n");
4059                 return -EAGAIN;
4060         }
4061
4062         if ((new_mtu > ETH_MAX_JUMBO_PACKET_SIZE) ||
4063             ((new_mtu + ETH_HLEN) < ETH_MIN_PACKET_SIZE)) {
4064                 BNX2X_ERR("Can't support requested MTU size\n");
4065                 return -EINVAL;
4066         }
4067
4068         /* This does not race with packet allocation
4069          * because the actual alloc size is
4070          * only updated as part of load
4071          */
4072         dev->mtu = new_mtu;
4073
4074         return bnx2x_reload_if_running(dev);
4075 }
4076
4077 netdev_features_t bnx2x_fix_features(struct net_device *dev,
4078                                      netdev_features_t features)
4079 {
4080         struct bnx2x *bp = netdev_priv(dev);
4081
4082         /* TPA requires Rx CSUM offloading */
4083         if (!(features & NETIF_F_RXCSUM) || bp->disable_tpa) {
4084                 features &= ~NETIF_F_LRO;
4085                 features &= ~NETIF_F_GRO;
4086         }
4087
4088         return features;
4089 }
4090
4091 int bnx2x_set_features(struct net_device *dev, netdev_features_t features)
4092 {
4093         struct bnx2x *bp = netdev_priv(dev);
4094         u32 flags = bp->flags;
4095         bool bnx2x_reload = false;
4096
4097         if (features & NETIF_F_LRO)
4098                 flags |= TPA_ENABLE_FLAG;
4099         else
4100                 flags &= ~TPA_ENABLE_FLAG;
4101
4102         if (features & NETIF_F_GRO)
4103                 flags |= GRO_ENABLE_FLAG;
4104         else
4105                 flags &= ~GRO_ENABLE_FLAG;
4106
4107         if (features & NETIF_F_LOOPBACK) {
4108                 if (bp->link_params.loopback_mode != LOOPBACK_BMAC) {
4109                         bp->link_params.loopback_mode = LOOPBACK_BMAC;
4110                         bnx2x_reload = true;
4111                 }
4112         } else {
4113                 if (bp->link_params.loopback_mode != LOOPBACK_NONE) {
4114                         bp->link_params.loopback_mode = LOOPBACK_NONE;
4115                         bnx2x_reload = true;
4116                 }
4117         }
4118
4119         if (flags ^ bp->flags) {
4120                 bp->flags = flags;
4121                 bnx2x_reload = true;
4122         }
4123
4124         if (bnx2x_reload) {
4125                 if (bp->recovery_state == BNX2X_RECOVERY_DONE)
4126                         return bnx2x_reload_if_running(dev);
4127                 /* else: bnx2x_nic_load() will be called at end of recovery */
4128         }
4129
4130         return 0;
4131 }
4132
4133 void bnx2x_tx_timeout(struct net_device *dev)
4134 {
4135         struct bnx2x *bp = netdev_priv(dev);
4136
4137 #ifdef BNX2X_STOP_ON_ERROR
4138         if (!bp->panic)
4139                 bnx2x_panic();
4140 #endif
4141
4142         smp_mb__before_clear_bit();
4143         set_bit(BNX2X_SP_RTNL_TX_TIMEOUT, &bp->sp_rtnl_state);
4144         smp_mb__after_clear_bit();
4145
4146         /* This allows the netif to be shutdown gracefully before resetting */
4147         schedule_delayed_work(&bp->sp_rtnl_task, 0);
4148 }
4149
4150 int bnx2x_suspend(struct pci_dev *pdev, pm_message_t state)
4151 {
4152         struct net_device *dev = pci_get_drvdata(pdev);
4153         struct bnx2x *bp;
4154
4155         if (!dev) {
4156                 dev_err(&pdev->dev, "BAD net device from bnx2x_init_one\n");
4157                 return -ENODEV;
4158         }
4159         bp = netdev_priv(dev);
4160
4161         rtnl_lock();
4162
4163         pci_save_state(pdev);
4164
4165         if (!netif_running(dev)) {
4166                 rtnl_unlock();
4167                 return 0;
4168         }
4169
4170         netif_device_detach(dev);
4171
4172         bnx2x_nic_unload(bp, UNLOAD_CLOSE, false);
4173
4174         bnx2x_set_power_state(bp, pci_choose_state(pdev, state));
4175
4176         rtnl_unlock();
4177
4178         return 0;
4179 }
4180
4181 int bnx2x_resume(struct pci_dev *pdev)
4182 {
4183         struct net_device *dev = pci_get_drvdata(pdev);
4184         struct bnx2x *bp;
4185         int rc;
4186
4187         if (!dev) {
4188                 dev_err(&pdev->dev, "BAD net device from bnx2x_init_one\n");
4189                 return -ENODEV;
4190         }
4191         bp = netdev_priv(dev);
4192
4193         if (bp->recovery_state != BNX2X_RECOVERY_DONE) {
4194                 BNX2X_ERR("Handling parity error recovery. Try again later\n");
4195                 return -EAGAIN;
4196         }
4197
4198         rtnl_lock();
4199
4200         pci_restore_state(pdev);
4201
4202         if (!netif_running(dev)) {
4203                 rtnl_unlock();
4204                 return 0;
4205         }
4206
4207         bnx2x_set_power_state(bp, PCI_D0);
4208         netif_device_attach(dev);
4209
4210         rc = bnx2x_nic_load(bp, LOAD_OPEN);
4211
4212         rtnl_unlock();
4213
4214         return rc;
4215 }
4216
4217
4218 void bnx2x_set_ctx_validation(struct bnx2x *bp, struct eth_context *cxt,
4219                               u32 cid)
4220 {
4221         /* ustorm cxt validation */
4222         cxt->ustorm_ag_context.cdu_usage =
4223                 CDU_RSRVD_VALUE_TYPE_A(HW_CID(bp, cid),
4224                         CDU_REGION_NUMBER_UCM_AG, ETH_CONNECTION_TYPE);
4225         /* xcontext validation */
4226         cxt->xstorm_ag_context.cdu_reserved =
4227                 CDU_RSRVD_VALUE_TYPE_A(HW_CID(bp, cid),
4228                         CDU_REGION_NUMBER_XCM_AG, ETH_CONNECTION_TYPE);
4229 }
4230
4231 static void storm_memset_hc_timeout(struct bnx2x *bp, u8 port,
4232                                     u8 fw_sb_id, u8 sb_index,
4233                                     u8 ticks)
4234 {
4235
4236         u32 addr = BAR_CSTRORM_INTMEM +
4237                    CSTORM_STATUS_BLOCK_DATA_TIMEOUT_OFFSET(fw_sb_id, sb_index);
4238         REG_WR8(bp, addr, ticks);
4239         DP(NETIF_MSG_IFUP,
4240            "port %x fw_sb_id %d sb_index %d ticks %d\n",
4241            port, fw_sb_id, sb_index, ticks);
4242 }
4243
4244 static void storm_memset_hc_disable(struct bnx2x *bp, u8 port,
4245                                     u16 fw_sb_id, u8 sb_index,
4246                                     u8 disable)
4247 {
4248         u32 enable_flag = disable ? 0 : (1 << HC_INDEX_DATA_HC_ENABLED_SHIFT);
4249         u32 addr = BAR_CSTRORM_INTMEM +
4250                    CSTORM_STATUS_BLOCK_DATA_FLAGS_OFFSET(fw_sb_id, sb_index);
4251         u16 flags = REG_RD16(bp, addr);
4252         /* clear and set */
4253         flags &= ~HC_INDEX_DATA_HC_ENABLED;
4254         flags |= enable_flag;
4255         REG_WR16(bp, addr, flags);
4256         DP(NETIF_MSG_IFUP,
4257            "port %x fw_sb_id %d sb_index %d disable %d\n",
4258            port, fw_sb_id, sb_index, disable);
4259 }
4260
4261 void bnx2x_update_coalesce_sb_index(struct bnx2x *bp, u8 fw_sb_id,
4262                                     u8 sb_index, u8 disable, u16 usec)
4263 {
4264         int port = BP_PORT(bp);
4265         u8 ticks = usec / BNX2X_BTR;
4266
4267         storm_memset_hc_timeout(bp, port, fw_sb_id, sb_index, ticks);
4268
4269         disable = disable ? 1 : (usec ? 0 : 1);
4270         storm_memset_hc_disable(bp, port, fw_sb_id, sb_index, disable);
4271 }