drivers/net/ethernet/broadcom/bnx2x/bnx2x_cmn.c

   1 /* bnx2x_cmn.c: Broadcom Everest network driver.
   2  *
   3  * Copyright (c) 2007-2012 Broadcom Corporation
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of the GNU General Public License as published by
   7  * the Free Software Foundation.
   8  *
   9  * Maintained by: Eilon Greenstein <eilong@broadcom.com>
  10  * Written by: Eliezer Tamir
  11  * Based on code from Michael Chan's bnx2 driver
  12  * UDP CSUM errata workaround by Arik Gendelman
  13  * Slowpath and fastpath rework by Vladislav Zolotarov
  14  * Statistics and Link management by Yitchak Gertner
  15  *
  16  */
  17
  18 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  19
  20 #include <linux/etherdevice.h>
  21 #include <linux/if_vlan.h>
  22 #include <linux/interrupt.h>
  23 #include <linux/ip.h>
  24 #include <net/ipv6.h>
  25 #include <net/ip6_checksum.h>
  26 #include <linux/prefetch.h>
  27 #include "bnx2x_cmn.h"
  28 #include "bnx2x_init.h"
  29 #include "bnx2x_sp.h"
  30
  31
  32
  33 /**
  34  * bnx2x_move_fp - move content of the fastpath structure.
  35  *
  36  * @bp:         driver handle
  37  * @from:       source FP index
  38  * @to:         destination FP index
  39  *
  40  * Makes sure the contents of the bp->fp[to].napi is kept
  41  * intact. This is done by first copying the napi struct from
  42  * the target to the source, and then mem copying the entire
  43  * source onto the target. Update txdata pointers and related
  44  * content.
  45  */
  46 static inline void bnx2x_move_fp(struct bnx2x *bp, int from, int to)
  47 {
  48         struct bnx2x_fastpath *from_fp = &bp->fp[from];
  49         struct bnx2x_fastpath *to_fp = &bp->fp[to];
  50         struct bnx2x_sp_objs *from_sp_objs = &bp->sp_objs[from];
  51         struct bnx2x_sp_objs *to_sp_objs = &bp->sp_objs[to];
  52         struct bnx2x_fp_stats *from_fp_stats = &bp->fp_stats[from];
  53         struct bnx2x_fp_stats *to_fp_stats = &bp->fp_stats[to];
  54         int old_max_eth_txqs, new_max_eth_txqs;
  55         int old_txdata_index = 0, new_txdata_index = 0;
  56
  57         /* Copy the NAPI object as it has been already initialized */
  58         from_fp->napi = to_fp->napi;
  59
  60         /* Move bnx2x_fastpath contents */
  61         memcpy(to_fp, from_fp, sizeof(*to_fp));
  62         to_fp->index = to;
  63
  64         /* move sp_objs contents as well, as their indices match fp ones */
  65         memcpy(to_sp_objs, from_sp_objs, sizeof(*to_sp_objs));
  66
  67         /* move fp_stats contents as well, as their indices match fp ones */
  68         memcpy(to_fp_stats, from_fp_stats, sizeof(*to_fp_stats));
  69
  70         /* Update txdata pointers in fp and move txdata content accordingly:
  71          * Each fp consumes 'max_cos' txdata structures, so the index should be
  72          * decremented by max_cos x delta.
  73          */
  74
  75         old_max_eth_txqs = BNX2X_NUM_ETH_QUEUES(bp) * (bp)->max_cos;
  76         new_max_eth_txqs = (BNX2X_NUM_ETH_QUEUES(bp) - from + to) *
  77                                 (bp)->max_cos;
  78         if (from == FCOE_IDX(bp)) {
  79                 old_txdata_index = old_max_eth_txqs + FCOE_TXQ_IDX_OFFSET;
  80                 new_txdata_index = new_max_eth_txqs + FCOE_TXQ_IDX_OFFSET;
  81         }
  82
  83         memcpy(&bp->bnx2x_txq[old_txdata_index],
  84                &bp->bnx2x_txq[new_txdata_index],
  85                sizeof(struct bnx2x_fp_txdata));
  86         to_fp->txdata_ptr[0] = &bp->bnx2x_txq[new_txdata_index];
  87 }
  88
  89 int load_count[2][3] = { {0} }; /* per-path: 0-common, 1-port0, 2-port1 */
  90
  91 /* free skb in the packet ring at pos idx
  92  * return idx of last bd freed
  93  */
  94 static u16 bnx2x_free_tx_pkt(struct bnx2x *bp, struct bnx2x_fp_txdata *txdata,
  95                              u16 idx, unsigned int *pkts_compl,
  96                              unsigned int *bytes_compl)
  97 {
  98         struct sw_tx_bd *tx_buf = &txdata->tx_buf_ring[idx];
  99         struct eth_tx_start_bd *tx_start_bd;
 100         struct eth_tx_bd *tx_data_bd;
 101         struct sk_buff *skb = tx_buf->skb;
 102         u16 bd_idx = TX_BD(tx_buf->first_bd), new_cons;
 103         int nbd;
 104
 105         /* prefetch skb end pointer to speedup dev_kfree_skb() */
 106         prefetch(&skb->end);
 107
 108         DP(NETIF_MSG_TX_DONE, "fp[%d]: pkt_idx %d  buff @(%p)->skb %p\n",
 109            txdata->txq_index, idx, tx_buf, skb);
 110
 111         /* unmap first bd */
 112         tx_start_bd = &txdata->tx_desc_ring[bd_idx].start_bd;
 113         dma_unmap_single(&bp->pdev->dev, BD_UNMAP_ADDR(tx_start_bd),
 114                          BD_UNMAP_LEN(tx_start_bd), DMA_TO_DEVICE);
 115
 116
 117         nbd = le16_to_cpu(tx_start_bd->nbd) - 1;
 118 #ifdef BNX2X_STOP_ON_ERROR
 119         if ((nbd - 1) > (MAX_SKB_FRAGS + 2)) {
 120                 BNX2X_ERR("BAD nbd!\n");
 121                 bnx2x_panic();
 122         }
 123 #endif
 124         new_cons = nbd + tx_buf->first_bd;
 125
 126         /* Get the next bd */
 127         bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
 128
 129         /* Skip a parse bd... */
 130         --nbd;
 131         bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
 132
 133         /* ...and the TSO split header bd since they have no mapping */
 134         if (tx_buf->flags & BNX2X_TSO_SPLIT_BD) {
 135                 --nbd;
 136                 bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
 137         }
 138
 139         /* now free frags */
 140         while (nbd > 0) {
 141
 142                 tx_data_bd = &txdata->tx_desc_ring[bd_idx].reg_bd;
 143                 dma_unmap_page(&bp->pdev->dev, BD_UNMAP_ADDR(tx_data_bd),
 144                                BD_UNMAP_LEN(tx_data_bd), DMA_TO_DEVICE);
 145                 if (--nbd)
 146                         bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
 147         }
 148
 149         /* release skb */
 150         WARN_ON(!skb);
 151         if (likely(skb)) {
 152                 (*pkts_compl)++;
 153                 (*bytes_compl) += skb->len;
 154         }
 155
 156         dev_kfree_skb_any(skb);
 157         tx_buf->first_bd = 0;
 158         tx_buf->skb = NULL;
 159
 160         return new_cons;
 161 }
 162
 163 int bnx2x_tx_int(struct bnx2x *bp, struct bnx2x_fp_txdata *txdata)
 164 {
 165         struct netdev_queue *txq;
 166         u16 hw_cons, sw_cons, bd_cons = txdata->tx_bd_cons;
 167         unsigned int pkts_compl = 0, bytes_compl = 0;
 168
 169 #ifdef BNX2X_STOP_ON_ERROR
 170         if (unlikely(bp->panic))
 171                 return -1;
 172 #endif
 173
 174         txq = netdev_get_tx_queue(bp->dev, txdata->txq_index);
 175         hw_cons = le16_to_cpu(*txdata->tx_cons_sb);
 176         sw_cons = txdata->tx_pkt_cons;
 177
 178         while (sw_cons != hw_cons) {
 179                 u16 pkt_cons;
 180
 181                 pkt_cons = TX_BD(sw_cons);
 182
 183                 DP(NETIF_MSG_TX_DONE,
 184                    "queue[%d]: hw_cons %u  sw_cons %u  pkt_cons %u\n",
 185                    txdata->txq_index, hw_cons, sw_cons, pkt_cons);
 186
 187                 bd_cons = bnx2x_free_tx_pkt(bp, txdata, pkt_cons,
 188                     &pkts_compl, &bytes_compl);
 189
 190                 sw_cons++;
 191         }
 192
 193         netdev_tx_completed_queue(txq, pkts_compl, bytes_compl);
 194
 195         txdata->tx_pkt_cons = sw_cons;
 196         txdata->tx_bd_cons = bd_cons;
 197
 198         /* Need to make the tx_bd_cons update visible to start_xmit()
 199          * before checking for netif_tx_queue_stopped().  Without the
 200          * memory barrier, there is a small possibility that
 201          * start_xmit() will miss it and cause the queue to be stopped
 202          * forever.
 203          * On the other hand we need an rmb() here to ensure the proper
 204          * ordering of bit testing in the following
 205          * netif_tx_queue_stopped(txq) call.
 206          */
 207         smp_mb();
 208
 209         if (unlikely(netif_tx_queue_stopped(txq))) {
 210                 /* Taking tx_lock() is needed to prevent reenabling the queue
 211                  * while it's empty. This could have happen if rx_action() gets
 212                  * suspended in bnx2x_tx_int() after the condition before
 213                  * netif_tx_wake_queue(), while tx_action (bnx2x_start_xmit()):
 214                  *
 215                  * stops the queue->sees fresh tx_bd_cons->releases the queue->
 216                  * sends some packets consuming the whole queue again->
 217                  * stops the queue
 218                  */
 219
 220                 __netif_tx_lock(txq, smp_processor_id());
 221
 222                 if ((netif_tx_queue_stopped(txq)) &&
 223                     (bp->state == BNX2X_STATE_OPEN) &&
 224                     (bnx2x_tx_avail(bp, txdata) >= MAX_DESC_PER_TX_PKT))
 225                         netif_tx_wake_queue(txq);
 226
 227                 __netif_tx_unlock(txq);
 228         }
 229         return 0;
 230 }
 231
 232 static inline void bnx2x_update_last_max_sge(struct bnx2x_fastpath *fp,
 233                                              u16 idx)
 234 {
 235         u16 last_max = fp->last_max_sge;
 236
 237         if (SUB_S16(idx, last_max) > 0)
 238                 fp->last_max_sge = idx;
 239 }
 240
 241 static inline void bnx2x_update_sge_prod(struct bnx2x_fastpath *fp,
 242                                          u16 sge_len,
 243                                          struct eth_end_agg_rx_cqe *cqe)
 244 {
 245         struct bnx2x *bp = fp->bp;
 246         u16 last_max, last_elem, first_elem;
 247         u16 delta = 0;
 248         u16 i;
 249
 250         if (!sge_len)
 251                 return;
 252
 253         /* First mark all used pages */
 254         for (i = 0; i < sge_len; i++)
 255                 BIT_VEC64_CLEAR_BIT(fp->sge_mask,
 256                         RX_SGE(le16_to_cpu(cqe->sgl_or_raw_data.sgl[i])));
 257
 258         DP(NETIF_MSG_RX_STATUS, "fp_cqe->sgl[%d] = %d\n",
 259            sge_len - 1, le16_to_cpu(cqe->sgl_or_raw_data.sgl[sge_len - 1]));
 260
 261         /* Here we assume that the last SGE index is the biggest */
 262         prefetch((void *)(fp->sge_mask));
 263         bnx2x_update_last_max_sge(fp,
 264                 le16_to_cpu(cqe->sgl_or_raw_data.sgl[sge_len - 1]));
 265
 266         last_max = RX_SGE(fp->last_max_sge);
 267         last_elem = last_max >> BIT_VEC64_ELEM_SHIFT;
 268         first_elem = RX_SGE(fp->rx_sge_prod) >> BIT_VEC64_ELEM_SHIFT;
 269
 270         /* If ring is not full */
 271         if (last_elem + 1 != first_elem)
 272                 last_elem++;
 273
 274         /* Now update the prod */
 275         for (i = first_elem; i != last_elem; i = NEXT_SGE_MASK_ELEM(i)) {
 276                 if (likely(fp->sge_mask[i]))
 277                         break;
 278
 279                 fp->sge_mask[i] = BIT_VEC64_ELEM_ONE_MASK;
 280                 delta += BIT_VEC64_ELEM_SZ;
 281         }
 282
 283         if (delta > 0) {
 284                 fp->rx_sge_prod += delta;
 285                 /* clear page-end entries */
 286                 bnx2x_clear_sge_mask_next_elems(fp);
 287         }
 288
 289         DP(NETIF_MSG_RX_STATUS,
 290            "fp->last_max_sge = %d  fp->rx_sge_prod = %d\n",
 291            fp->last_max_sge, fp->rx_sge_prod);
 292 }
 293
 294 /* Set Toeplitz hash value in the skb using the value from the
 295  * CQE (calculated by HW).
 296  */
 297 static u32 bnx2x_get_rxhash(const struct bnx2x *bp,
 298                             const struct eth_fast_path_rx_cqe *cqe,
 299                             bool *l4_rxhash)
 300 {
 301         /* Set Toeplitz hash from CQE */
 302         if ((bp->dev->features & NETIF_F_RXHASH) &&
 303             (cqe->status_flags & ETH_FAST_PATH_RX_CQE_RSS_HASH_FLG)) {
 304                 enum eth_rss_hash_type htype;
 305
 306                 htype = cqe->status_flags & ETH_FAST_PATH_RX_CQE_RSS_HASH_TYPE;
 307                 *l4_rxhash = (htype == TCP_IPV4_HASH_TYPE) ||
 308                              (htype == TCP_IPV6_HASH_TYPE);
 309                 return le32_to_cpu(cqe->rss_hash_result);
 310         }
 311         *l4_rxhash = false;
 312         return 0;
 313 }
 314
 315 static void bnx2x_tpa_start(struct bnx2x_fastpath *fp, u16 queue,
 316                             u16 cons, u16 prod,
 317                             struct eth_fast_path_rx_cqe *cqe)
 318 {
 319         struct bnx2x *bp = fp->bp;
 320         struct sw_rx_bd *cons_rx_buf = &fp->rx_buf_ring[cons];
 321         struct sw_rx_bd *prod_rx_buf = &fp->rx_buf_ring[prod];
 322         struct eth_rx_bd *prod_bd = &fp->rx_desc_ring[prod];
 323         dma_addr_t mapping;
 324         struct bnx2x_agg_info *tpa_info = &fp->tpa_info[queue];
 325         struct sw_rx_bd *first_buf = &tpa_info->first_buf;
 326
 327         /* print error if current state != stop */
 328         if (tpa_info->tpa_state != BNX2X_TPA_STOP)
 329                 BNX2X_ERR("start of bin not in stop [%d]\n", queue);
 330
 331         /* Try to map an empty data buffer from the aggregation info  */
 332         mapping = dma_map_single(&bp->pdev->dev,
 333                                  first_buf->data + NET_SKB_PAD,
 334                                  fp->rx_buf_size, DMA_FROM_DEVICE);
 335         /*
 336          *  ...if it fails - move the skb from the consumer to the producer
 337          *  and set the current aggregation state as ERROR to drop it
 338          *  when TPA_STOP arrives.
 339          */
 340
 341         if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) {
 342                 /* Move the BD from the consumer to the producer */
 343                 bnx2x_reuse_rx_data(fp, cons, prod);
 344                 tpa_info->tpa_state = BNX2X_TPA_ERROR;
 345                 return;
 346         }
 347
 348         /* move empty data from pool to prod */
 349         prod_rx_buf->data = first_buf->data;
 350         dma_unmap_addr_set(prod_rx_buf, mapping, mapping);
 351         /* point prod_bd to new data */
 352         prod_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
 353         prod_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
 354
 355         /* move partial skb from cons to pool (don't unmap yet) */
 356         *first_buf = *cons_rx_buf;
 357
 358         /* mark bin state as START */
 359         tpa_info->parsing_flags =
 360                 le16_to_cpu(cqe->pars_flags.flags);
 361         tpa_info->vlan_tag = le16_to_cpu(cqe->vlan_tag);
 362         tpa_info->tpa_state = BNX2X_TPA_START;
 363         tpa_info->len_on_bd = le16_to_cpu(cqe->len_on_bd);
 364         tpa_info->placement_offset = cqe->placement_offset;
 365         tpa_info->rxhash = bnx2x_get_rxhash(bp, cqe, &tpa_info->l4_rxhash);
 366         if (fp->mode == TPA_MODE_GRO) {
 367                 u16 gro_size = le16_to_cpu(cqe->pkt_len_or_gro_seg_len);
 368                 tpa_info->full_page =
 369                         SGE_PAGE_SIZE * PAGES_PER_SGE / gro_size * gro_size;
 370                 tpa_info->gro_size = gro_size;
 371         }
 372
 373 #ifdef BNX2X_STOP_ON_ERROR
 374         fp->tpa_queue_used |= (1 << queue);
 375 #ifdef _ASM_GENERIC_INT_L64_H
 376         DP(NETIF_MSG_RX_STATUS, "fp->tpa_queue_used = 0x%lx\n",
 377 #else
 378         DP(NETIF_MSG_RX_STATUS, "fp->tpa_queue_used = 0x%llx\n",
 379 #endif
 380            fp->tpa_queue_used);
 381 #endif
 382 }
 383
 384 /* Timestamp option length allowed for TPA aggregation:
 385  *
 386  *              nop nop kind length echo val
 387  */
 388 #define TPA_TSTAMP_OPT_LEN      12
 389 /**
 390  * bnx2x_set_lro_mss - calculate the approximate value of the MSS
 391  *
 392  * @bp:                 driver handle
 393  * @parsing_flags:      parsing flags from the START CQE
 394  * @len_on_bd:          total length of the first packet for the
 395  *                      aggregation.
 396  *
 397  * Approximate value of the MSS for this aggregation calculated using
 398  * the first packet of it.
 399  */
 400 static u16 bnx2x_set_lro_mss(struct bnx2x *bp, u16 parsing_flags,
 401                              u16 len_on_bd)
 402 {
 403         /*
 404          * TPA arrgregation won't have either IP options or TCP options
 405          * other than timestamp or IPv6 extension headers.
 406          */
 407         u16 hdrs_len = ETH_HLEN + sizeof(struct tcphdr);
 408
 409         if (GET_FLAG(parsing_flags, PARSING_FLAGS_OVER_ETHERNET_PROTOCOL) ==
 410             PRS_FLAG_OVERETH_IPV6)
 411                 hdrs_len += sizeof(struct ipv6hdr);
 412         else /* IPv4 */
 413                 hdrs_len += sizeof(struct iphdr);
 414
 415
 416         /* Check if there was a TCP timestamp, if there is it's will
 417          * always be 12 bytes length: nop nop kind length echo val.
 418          *
 419          * Otherwise FW would close the aggregation.
 420          */
 421         if (parsing_flags & PARSING_FLAGS_TIME_STAMP_EXIST_FLAG)
 422                 hdrs_len += TPA_TSTAMP_OPT_LEN;
 423
 424         return len_on_bd - hdrs_len;
 425 }
 426
 427 static int bnx2x_alloc_rx_sge(struct bnx2x *bp,
 428                               struct bnx2x_fastpath *fp, u16 index)
 429 {
 430         struct page *page = alloc_pages(GFP_ATOMIC, PAGES_PER_SGE_SHIFT);
 431         struct sw_rx_page *sw_buf = &fp->rx_page_ring[index];
 432         struct eth_rx_sge *sge = &fp->rx_sge_ring[index];
 433         dma_addr_t mapping;
 434
 435         if (unlikely(page == NULL)) {
 436                 BNX2X_ERR("Can't alloc sge\n");
 437                 return -ENOMEM;
 438         }
 439
 440         mapping = dma_map_page(&bp->pdev->dev, page, 0,
 441                                SGE_PAGE_SIZE*PAGES_PER_SGE, DMA_FROM_DEVICE);
 442         if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) {
 443                 __free_pages(page, PAGES_PER_SGE_SHIFT);
 444                 BNX2X_ERR("Can't map sge\n");
 445                 return -ENOMEM;
 446         }
 447
 448         sw_buf->page = page;
 449         dma_unmap_addr_set(sw_buf, mapping, mapping);
 450
 451         sge->addr_hi = cpu_to_le32(U64_HI(mapping));
 452         sge->addr_lo = cpu_to_le32(U64_LO(mapping));
 453
 454         return 0;
 455 }
 456
 457 static int bnx2x_fill_frag_skb(struct bnx2x *bp, struct bnx2x_fastpath *fp,
 458                                struct bnx2x_agg_info *tpa_info,
 459                                u16 pages,
 460                                struct sk_buff *skb,
 461                                struct eth_end_agg_rx_cqe *cqe,
 462                                u16 cqe_idx)
 463 {
 464         struct sw_rx_page *rx_pg, old_rx_pg;
 465         u32 i, frag_len, frag_size;
 466         int err, j, frag_id = 0;
 467         u16 len_on_bd = tpa_info->len_on_bd;
 468         u16 full_page = 0, gro_size = 0;
 469
 470         frag_size = le16_to_cpu(cqe->pkt_len) - len_on_bd;
 471
 472         if (fp->mode == TPA_MODE_GRO) {
 473                 gro_size = tpa_info->gro_size;
 474                 full_page = tpa_info->full_page;
 475         }
 476
 477         /* This is needed in order to enable forwarding support */
 478         if (frag_size) {
 479                 skb_shinfo(skb)->gso_size = bnx2x_set_lro_mss(bp,
 480                                         tpa_info->parsing_flags, len_on_bd);
 481
 482                 /* set for GRO */
 483                 if (fp->mode == TPA_MODE_GRO)
 484                         skb_shinfo(skb)->gso_type =
 485                             (GET_FLAG(tpa_info->parsing_flags,
 486                                       PARSING_FLAGS_OVER_ETHERNET_PROTOCOL) ==
 487                                                 PRS_FLAG_OVERETH_IPV6) ?
 488                                 SKB_GSO_TCPV6 : SKB_GSO_TCPV4;
 489         }
 490
 491
 492 #ifdef BNX2X_STOP_ON_ERROR
 493         if (pages > min_t(u32, 8, MAX_SKB_FRAGS)*SGE_PAGE_SIZE*PAGES_PER_SGE) {
 494                 BNX2X_ERR("SGL length is too long: %d. CQE index is %d\n",
 495                           pages, cqe_idx);
 496                 BNX2X_ERR("cqe->pkt_len = %d\n", cqe->pkt_len);
 497                 bnx2x_panic();
 498                 return -EINVAL;
 499         }
 500 #endif
 501
 502         /* Run through the SGL and compose the fragmented skb */
 503         for (i = 0, j = 0; i < pages; i += PAGES_PER_SGE, j++) {
 504                 u16 sge_idx = RX_SGE(le16_to_cpu(cqe->sgl_or_raw_data.sgl[j]));
 505
 506                 /* FW gives the indices of the SGE as if the ring is an array
 507                    (meaning that "next" element will consume 2 indices) */
 508                 if (fp->mode == TPA_MODE_GRO)
 509                         frag_len = min_t(u32, frag_size, (u32)full_page);
 510                 else /* LRO */
 511                         frag_len = min_t(u32, frag_size,
 512                                          (u32)(SGE_PAGE_SIZE * PAGES_PER_SGE));
 513
 514                 rx_pg = &fp->rx_page_ring[sge_idx];
 515                 old_rx_pg = *rx_pg;
 516
 517                 /* If we fail to allocate a substitute page, we simply stop
 518                    where we are and drop the whole packet */
 519                 err = bnx2x_alloc_rx_sge(bp, fp, sge_idx);
 520                 if (unlikely(err)) {
 521                         bnx2x_fp_qstats(bp, fp)->rx_skb_alloc_failed++;
 522                         return err;
 523                 }
 524
 525                 /* Unmap the page as we r going to pass it to the stack */
 526                 dma_unmap_page(&bp->pdev->dev,
 527                                dma_unmap_addr(&old_rx_pg, mapping),
 528                                SGE_PAGE_SIZE*PAGES_PER_SGE, DMA_FROM_DEVICE);
 529                 /* Add one frag and update the appropriate fields in the skb */
 530                 if (fp->mode == TPA_MODE_LRO)
 531                         skb_fill_page_desc(skb, j, old_rx_pg.page, 0, frag_len);
 532                 else { /* GRO */
 533                         int rem;
 534                         int offset = 0;
 535                         for (rem = frag_len; rem > 0; rem -= gro_size) {
 536                                 int len = rem > gro_size ? gro_size : rem;
 537                                 skb_fill_page_desc(skb, frag_id++,
 538                                                    old_rx_pg.page, offset, len);
 539                                 if (offset)
 540                                         get_page(old_rx_pg.page);
 541                                 offset += len;
 542                         }
 543                 }
 544
 545                 skb->data_len += frag_len;
 546                 skb->truesize += SGE_PAGE_SIZE * PAGES_PER_SGE;
 547                 skb->len += frag_len;
 548
 549                 frag_size -= frag_len;
 550         }
 551
 552         return 0;
 553 }
 554
 555 static void bnx2x_tpa_stop(struct bnx2x *bp, struct bnx2x_fastpath *fp,
 556                            struct bnx2x_agg_info *tpa_info,
 557                            u16 pages,
 558                            struct eth_end_agg_rx_cqe *cqe,
 559                            u16 cqe_idx)
 560 {
 561         struct sw_rx_bd *rx_buf = &tpa_info->first_buf;
 562         u8 pad = tpa_info->placement_offset;
 563         u16 len = tpa_info->len_on_bd;
 564         struct sk_buff *skb = NULL;
 565         u8 *new_data, *data = rx_buf->data;
 566         u8 old_tpa_state = tpa_info->tpa_state;
 567
 568         tpa_info->tpa_state = BNX2X_TPA_STOP;
 569
 570         /* If we there was an error during the handling of the TPA_START -
 571          * drop this aggregation.
 572          */
 573         if (old_tpa_state == BNX2X_TPA_ERROR)
 574                 goto drop;
 575
 576         /* Try to allocate the new data */
 577         new_data = kmalloc(fp->rx_buf_size + NET_SKB_PAD, GFP_ATOMIC);
 578
 579         /* Unmap skb in the pool anyway, as we are going to change
 580            pool entry status to BNX2X_TPA_STOP even if new skb allocation
 581            fails. */
 582         dma_unmap_single(&bp->pdev->dev, dma_unmap_addr(rx_buf, mapping),
 583                          fp->rx_buf_size, DMA_FROM_DEVICE);
 584         if (likely(new_data))
 585                 skb = build_skb(data, 0);
 586
 587         if (likely(skb)) {
 588 #ifdef BNX2X_STOP_ON_ERROR
 589                 if (pad + len > fp->rx_buf_size) {
 590                         BNX2X_ERR("skb_put is about to fail...  pad %d  len %d  rx_buf_size %d\n",
 591                                   pad, len, fp->rx_buf_size);
 592                         bnx2x_panic();
 593                         return;
 594                 }
 595 #endif
 596
 597                 skb_reserve(skb, pad + NET_SKB_PAD);
 598                 skb_put(skb, len);
 599                 skb->rxhash = tpa_info->rxhash;
 600                 skb->l4_rxhash = tpa_info->l4_rxhash;
 601
 602                 skb->protocol = eth_type_trans(skb, bp->dev);
 603                 skb->ip_summed = CHECKSUM_UNNECESSARY;
 604
 605                 if (!bnx2x_fill_frag_skb(bp, fp, tpa_info, pages,
 606                                          skb, cqe, cqe_idx)) {
 607                         if (tpa_info->parsing_flags & PARSING_FLAGS_VLAN)
 608                                 __vlan_hwaccel_put_tag(skb, tpa_info->vlan_tag);
 609                         napi_gro_receive(&fp->napi, skb);
 610                 } else {
 611                         DP(NETIF_MSG_RX_STATUS,
 612                            "Failed to allocate new pages - dropping packet!\n");
 613                         dev_kfree_skb_any(skb);
 614                 }
 615
 616
 617                 /* put new data in bin */
 618                 rx_buf->data = new_data;
 619
 620                 return;
 621         }
 622         kfree(new_data);
 623 drop:
 624         /* drop the packet and keep the buffer in the bin */
 625         DP(NETIF_MSG_RX_STATUS,
 626            "Failed to allocate or map a new skb - dropping packet!\n");
 627         bnx2x_fp_stats(bp, fp)->eth_q_stats.rx_skb_alloc_failed++;
 628 }
 629
 630 static int bnx2x_alloc_rx_data(struct bnx2x *bp,
 631                                struct bnx2x_fastpath *fp, u16 index)
 632 {
 633         u8 *data;
 634         struct sw_rx_bd *rx_buf = &fp->rx_buf_ring[index];
 635         struct eth_rx_bd *rx_bd = &fp->rx_desc_ring[index];
 636         dma_addr_t mapping;
 637
 638         data = kmalloc(fp->rx_buf_size + NET_SKB_PAD, GFP_ATOMIC);
 639         if (unlikely(data == NULL))
 640                 return -ENOMEM;
 641
 642         mapping = dma_map_single(&bp->pdev->dev, data + NET_SKB_PAD,
 643                                  fp->rx_buf_size,
 644                                  DMA_FROM_DEVICE);
 645         if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) {
 646                 kfree(data);
 647                 BNX2X_ERR("Can't map rx data\n");
 648                 return -ENOMEM;
 649         }
 650
 651         rx_buf->data = data;
 652         dma_unmap_addr_set(rx_buf, mapping, mapping);
 653
 654         rx_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
 655         rx_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
 656
 657         return 0;
 658 }
 659
 660 static
 661 void bnx2x_csum_validate(struct sk_buff *skb, union eth_rx_cqe *cqe,
 662                                  struct bnx2x_fastpath *fp,
 663                                  struct bnx2x_eth_q_stats *qstats)
 664 {
 665         /* Do nothing if no IP/L4 csum validation was done */
 666
 667         if (cqe->fast_path_cqe.status_flags &
 668             (ETH_FAST_PATH_RX_CQE_IP_XSUM_NO_VALIDATION_FLG |
 669              ETH_FAST_PATH_RX_CQE_L4_XSUM_NO_VALIDATION_FLG))
 670                 return;
 671
 672         /* If both IP/L4 validation were done, check if an error was found. */
 673
 674         if (cqe->fast_path_cqe.type_error_flags &
 675             (ETH_FAST_PATH_RX_CQE_IP_BAD_XSUM_FLG |
 676              ETH_FAST_PATH_RX_CQE_L4_BAD_XSUM_FLG))
 677                 qstats->hw_csum_err++;
 678         else
 679                 skb->ip_summed = CHECKSUM_UNNECESSARY;
 680 }
 681
 682 int bnx2x_rx_int(struct bnx2x_fastpath *fp, int budget)
 683 {
 684         struct bnx2x *bp = fp->bp;
 685         u16 bd_cons, bd_prod, bd_prod_fw, comp_ring_cons;
 686         u16 hw_comp_cons, sw_comp_cons, sw_comp_prod;
 687         int rx_pkt = 0;
 688
 689 #ifdef BNX2X_STOP_ON_ERROR
 690         if (unlikely(bp->panic))
 691                 return 0;
 692 #endif
 693
 694         /* CQ "next element" is of the size of the regular element,
 695            that's why it's ok here */
 696         hw_comp_cons = le16_to_cpu(*fp->rx_cons_sb);
 697         if ((hw_comp_cons & MAX_RCQ_DESC_CNT) == MAX_RCQ_DESC_CNT)
 698                 hw_comp_cons++;
 699
 700         bd_cons = fp->rx_bd_cons;
 701         bd_prod = fp->rx_bd_prod;
 702         bd_prod_fw = bd_prod;
 703         sw_comp_cons = fp->rx_comp_cons;
 704         sw_comp_prod = fp->rx_comp_prod;
 705
 706         /* Memory barrier necessary as speculative reads of the rx
 707          * buffer can be ahead of the index in the status block
 708          */
 709         rmb();
 710
 711         DP(NETIF_MSG_RX_STATUS,
 712            "queue[%d]:  hw_comp_cons %u  sw_comp_cons %u\n",
 713            fp->index, hw_comp_cons, sw_comp_cons);
 714
 715         while (sw_comp_cons != hw_comp_cons) {
 716                 struct sw_rx_bd *rx_buf = NULL;
 717                 struct sk_buff *skb;
 718                 union eth_rx_cqe *cqe;
 719                 struct eth_fast_path_rx_cqe *cqe_fp;
 720                 u8 cqe_fp_flags;
 721                 enum eth_rx_cqe_type cqe_fp_type;
 722                 u16 len, pad, queue;
 723                 u8 *data;
 724                 bool l4_rxhash;
 725
 726 #ifdef BNX2X_STOP_ON_ERROR
 727                 if (unlikely(bp->panic))
 728                         return 0;
 729 #endif
 730
 731                 comp_ring_cons = RCQ_BD(sw_comp_cons);
 732                 bd_prod = RX_BD(bd_prod);
 733                 bd_cons = RX_BD(bd_cons);
 734
 735                 cqe = &fp->rx_comp_ring[comp_ring_cons];
 736                 cqe_fp = &cqe->fast_path_cqe;
 737                 cqe_fp_flags = cqe_fp->type_error_flags;
 738                 cqe_fp_type = cqe_fp_flags & ETH_FAST_PATH_RX_CQE_TYPE;
 739
 740                 DP(NETIF_MSG_RX_STATUS,
 741                    "CQE type %x  err %x  status %x  queue %x  vlan %x  len %u\n",
 742                    CQE_TYPE(cqe_fp_flags),
 743                    cqe_fp_flags, cqe_fp->status_flags,
 744                    le32_to_cpu(cqe_fp->rss_hash_result),
 745                    le16_to_cpu(cqe_fp->vlan_tag),
 746                    le16_to_cpu(cqe_fp->pkt_len_or_gro_seg_len));
 747
 748                 /* is this a slowpath msg? */
 749                 if (unlikely(CQE_TYPE_SLOW(cqe_fp_type))) {
 750                         bnx2x_sp_event(fp, cqe);
 751                         goto next_cqe;
 752                 }
 753
 754                 rx_buf = &fp->rx_buf_ring[bd_cons];
 755                 data = rx_buf->data;
 756
 757                 if (!CQE_TYPE_FAST(cqe_fp_type)) {
 758                         struct bnx2x_agg_info *tpa_info;
 759                         u16 frag_size, pages;
 760 #ifdef BNX2X_STOP_ON_ERROR
 761                         /* sanity check */
 762                         if (fp->disable_tpa &&
 763                             (CQE_TYPE_START(cqe_fp_type) ||
 764                              CQE_TYPE_STOP(cqe_fp_type)))
 765                                 BNX2X_ERR("START/STOP packet while disable_tpa type %x\n",
 766                                           CQE_TYPE(cqe_fp_type));
 767 #endif
 768
 769                         if (CQE_TYPE_START(cqe_fp_type)) {
 770                                 u16 queue = cqe_fp->queue_index;
 771                                 DP(NETIF_MSG_RX_STATUS,
 772                                    "calling tpa_start on queue %d\n",
 773                                    queue);
 774
 775                                 bnx2x_tpa_start(fp, queue,
 776                                                 bd_cons, bd_prod,
 777                                                 cqe_fp);
 778
 779                                 goto next_rx;
 780
 781                         }
 782                         queue = cqe->end_agg_cqe.queue_index;
 783                         tpa_info = &fp->tpa_info[queue];
 784                         DP(NETIF_MSG_RX_STATUS,
 785                            "calling tpa_stop on queue %d\n",
 786                            queue);
 787
 788                         frag_size = le16_to_cpu(cqe->end_agg_cqe.pkt_len) -
 789                                     tpa_info->len_on_bd;
 790
 791                         if (fp->mode == TPA_MODE_GRO)
 792                                 pages = (frag_size + tpa_info->full_page - 1) /
 793                                          tpa_info->full_page;
 794                         else
 795                                 pages = SGE_PAGE_ALIGN(frag_size) >>
 796                                         SGE_PAGE_SHIFT;
 797
 798                         bnx2x_tpa_stop(bp, fp, tpa_info, pages,
 799                                        &cqe->end_agg_cqe, comp_ring_cons);
 800 #ifdef BNX2X_STOP_ON_ERROR
 801                         if (bp->panic)
 802                                 return 0;
 803 #endif
 804
 805                         bnx2x_update_sge_prod(fp, pages, &cqe->end_agg_cqe);
 806                         goto next_cqe;
 807                 }
 808                 /* non TPA */
 809                 len = le16_to_cpu(cqe_fp->pkt_len_or_gro_seg_len);
 810                 pad = cqe_fp->placement_offset;
 811                 dma_sync_single_for_cpu(&bp->pdev->dev,
 812                                         dma_unmap_addr(rx_buf, mapping),
 813                                         pad + RX_COPY_THRESH,
 814                                         DMA_FROM_DEVICE);
 815                 pad += NET_SKB_PAD;
 816                 prefetch(data + pad); /* speedup eth_type_trans() */
 817                 /* is this an error packet? */
 818                 if (unlikely(cqe_fp_flags & ETH_RX_ERROR_FALGS)) {
 819                         DP(NETIF_MSG_RX_ERR | NETIF_MSG_RX_STATUS,
 820                            "ERROR  flags %x  rx packet %u\n",
 821                            cqe_fp_flags, sw_comp_cons);
 822                         bnx2x_fp_qstats(bp, fp)->rx_err_discard_pkt++;
 823                         goto reuse_rx;
 824                 }
 825
 826                 /* Since we don't have a jumbo ring
 827                  * copy small packets if mtu > 1500
 828                  */
 829                 if ((bp->dev->mtu > ETH_MAX_PACKET_SIZE) &&
 830                     (len <= RX_COPY_THRESH)) {
 831                         skb = netdev_alloc_skb_ip_align(bp->dev, len);
 832                         if (skb == NULL) {
 833                                 DP(NETIF_MSG_RX_ERR | NETIF_MSG_RX_STATUS,
 834                                    "ERROR  packet dropped because of alloc failure\n");
 835                                 bnx2x_fp_qstats(bp, fp)->rx_skb_alloc_failed++;
 836                                 goto reuse_rx;
 837                         }
 838                         memcpy(skb->data, data + pad, len);
 839                         bnx2x_reuse_rx_data(fp, bd_cons, bd_prod);
 840                 } else {
 841                         if (likely(bnx2x_alloc_rx_data(bp, fp, bd_prod) == 0)) {
 842                                 dma_unmap_single(&bp->pdev->dev,
 843                                                  dma_unmap_addr(rx_buf, mapping),
 844                                                  fp->rx_buf_size,
 845                                                  DMA_FROM_DEVICE);
 846                                 skb = build_skb(data, 0);
 847                                 if (unlikely(!skb)) {
 848                                         kfree(data);
 849                                         bnx2x_fp_qstats(bp, fp)->
 850                                                         rx_skb_alloc_failed++;
 851                                         goto next_rx;
 852                                 }
 853                                 skb_reserve(skb, pad);
 854                         } else {
 855                                 DP(NETIF_MSG_RX_ERR | NETIF_MSG_RX_STATUS,
 856                                    "ERROR  packet dropped because of alloc failure\n");
 857                                 bnx2x_fp_qstats(bp, fp)->rx_skb_alloc_failed++;
 858 reuse_rx:
 859                                 bnx2x_reuse_rx_data(fp, bd_cons, bd_prod);
 860                                 goto next_rx;
 861                         }
 862                 }
 863
 864                 skb_put(skb, len);
 865                 skb->protocol = eth_type_trans(skb, bp->dev);
 866
 867                 /* Set Toeplitz hash for a none-LRO skb */
 868                 skb->rxhash = bnx2x_get_rxhash(bp, cqe_fp, &l4_rxhash);
 869                 skb->l4_rxhash = l4_rxhash;
 870
 871                 skb_checksum_none_assert(skb);
 872
 873                 if (bp->dev->features & NETIF_F_RXCSUM)
 874                         bnx2x_csum_validate(skb, cqe, fp,
 875                                             bnx2x_fp_qstats(bp, fp));
 876
 877                 skb_record_rx_queue(skb, fp->rx_queue);
 878
 879                 if (le16_to_cpu(cqe_fp->pars_flags.flags) &
 880                     PARSING_FLAGS_VLAN)
 881                         __vlan_hwaccel_put_tag(skb,
 882                                                le16_to_cpu(cqe_fp->vlan_tag));
 883                 napi_gro_receive(&fp->napi, skb);
 884
 885
 886 next_rx:
 887                 rx_buf->data = NULL;
 888
 889                 bd_cons = NEXT_RX_IDX(bd_cons);
 890                 bd_prod = NEXT_RX_IDX(bd_prod);
 891                 bd_prod_fw = NEXT_RX_IDX(bd_prod_fw);
 892                 rx_pkt++;
 893 next_cqe:
 894                 sw_comp_prod = NEXT_RCQ_IDX(sw_comp_prod);
 895                 sw_comp_cons = NEXT_RCQ_IDX(sw_comp_cons);
 896
 897                 if (rx_pkt == budget)
 898                         break;
 899         } /* while */
 900
 901         fp->rx_bd_cons = bd_cons;
 902         fp->rx_bd_prod = bd_prod_fw;
 903         fp->rx_comp_cons = sw_comp_cons;
 904         fp->rx_comp_prod = sw_comp_prod;
 905
 906         /* Update producers */
 907         bnx2x_update_rx_prod(bp, fp, bd_prod_fw, sw_comp_prod,
 908                              fp->rx_sge_prod);
 909
 910         fp->rx_pkt += rx_pkt;
 911         fp->rx_calls++;
 912
 913         return rx_pkt;
 914 }
 915
 916 static irqreturn_t bnx2x_msix_fp_int(int irq, void *fp_cookie)
 917 {
 918         struct bnx2x_fastpath *fp = fp_cookie;
 919         struct bnx2x *bp = fp->bp;
 920         u8 cos;
 921
 922         DP(NETIF_MSG_INTR,
 923            "got an MSI-X interrupt on IDX:SB [fp %d fw_sd %d igusb %d]\n",
 924            fp->index, fp->fw_sb_id, fp->igu_sb_id);
 925         bnx2x_ack_sb(bp, fp->igu_sb_id, USTORM_ID, 0, IGU_INT_DISABLE, 0);
 926
 927 #ifdef BNX2X_STOP_ON_ERROR
 928         if (unlikely(bp->panic))
 929                 return IRQ_HANDLED;
 930 #endif
 931
 932         /* Handle Rx and Tx according to MSI-X vector */
 933         prefetch(fp->rx_cons_sb);
 934
 935         for_each_cos_in_tx_queue(fp, cos)
 936                 prefetch(fp->txdata_ptr[cos]->tx_cons_sb);
 937
 938         prefetch(&fp->sb_running_index[SM_RX_ID]);
 939         napi_schedule(&bnx2x_fp(bp, fp->index, napi));
 940
 941         return IRQ_HANDLED;
 942 }
 943
 944 /* HW Lock for shared dual port PHYs */
 945 void bnx2x_acquire_phy_lock(struct bnx2x *bp)
 946 {
 947         mutex_lock(&bp->port.phy_mutex);
 948
 949         if (bp->port.need_hw_lock)
 950                 bnx2x_acquire_hw_lock(bp, HW_LOCK_RESOURCE_MDIO);
 951 }
 952
 953 void bnx2x_release_phy_lock(struct bnx2x *bp)
 954 {
 955         if (bp->port.need_hw_lock)
 956                 bnx2x_release_hw_lock(bp, HW_LOCK_RESOURCE_MDIO);
 957
 958         mutex_unlock(&bp->port.phy_mutex);
 959 }
 960
 961 /* calculates MF speed according to current linespeed and MF configuration */
 962 u16 bnx2x_get_mf_speed(struct bnx2x *bp)
 963 {
 964         u16 line_speed = bp->link_vars.line_speed;
 965         if (IS_MF(bp)) {
 966                 u16 maxCfg = bnx2x_extract_max_cfg(bp,
 967                                                    bp->mf_config[BP_VN(bp)]);
 968
 969                 /* Calculate the current MAX line speed limit for the MF
 970                  * devices
 971                  */
 972                 if (IS_MF_SI(bp))
 973                         line_speed = (line_speed * maxCfg) / 100;
 974                 else { /* SD mode */
 975                         u16 vn_max_rate = maxCfg * 100;
 976
 977                         if (vn_max_rate < line_speed)
 978                                 line_speed = vn_max_rate;
 979                 }
 980         }
 981
 982         return line_speed;
 983 }
 984
 985 /**
 986  * bnx2x_fill_report_data - fill link report data to report
 987  *
 988  * @bp:         driver handle
 989  * @data:       link state to update
 990  *
 991  * It uses a none-atomic bit operations because is called under the mutex.
 992  */
 993 static void bnx2x_fill_report_data(struct bnx2x *bp,
 994                                    struct bnx2x_link_report_data *data)
 995 {
 996         u16 line_speed = bnx2x_get_mf_speed(bp);
 997
 998         memset(data, 0, sizeof(*data));
 999
1000         /* Fill the report data: efective line speed */
1001         data->line_speed = line_speed;
1002
1003         /* Link is down */
1004         if (!bp->link_vars.link_up || (bp->flags & MF_FUNC_DIS))
1005                 __set_bit(BNX2X_LINK_REPORT_LINK_DOWN,
1006                           &data->link_report_flags);
1007
1008         /* Full DUPLEX */
1009         if (bp->link_vars.duplex == DUPLEX_FULL)
1010                 __set_bit(BNX2X_LINK_REPORT_FD, &data->link_report_flags);
1011
1012         /* Rx Flow Control is ON */
1013         if (bp->link_vars.flow_ctrl & BNX2X_FLOW_CTRL_RX)
1014                 __set_bit(BNX2X_LINK_REPORT_RX_FC_ON, &data->link_report_flags);
1015
1016         /* Tx Flow Control is ON */
1017         if (bp->link_vars.flow_ctrl & BNX2X_FLOW_CTRL_TX)
1018                 __set_bit(BNX2X_LINK_REPORT_TX_FC_ON, &data->link_report_flags);
1019 }
1020
1021 /**
1022  * bnx2x_link_report - report link status to OS.
1023  *
1024  * @bp:         driver handle
1025  *
1026  * Calls the __bnx2x_link_report() under the same locking scheme
1027  * as a link/PHY state managing code to ensure a consistent link
1028  * reporting.
1029  */
1030
1031 void bnx2x_link_report(struct bnx2x *bp)
1032 {
1033         bnx2x_acquire_phy_lock(bp);
1034         __bnx2x_link_report(bp);
1035         bnx2x_release_phy_lock(bp);
1036 }
1037
1038 /**
1039  * __bnx2x_link_report - report link status to OS.
1040  *
1041  * @bp:         driver handle
1042  *
1043  * None atomic inmlementation.
1044  * Should be called under the phy_lock.
1045  */
1046 void __bnx2x_link_report(struct bnx2x *bp)
1047 {
1048         struct bnx2x_link_report_data cur_data;
1049
1050         /* reread mf_cfg */
1051         if (!CHIP_IS_E1(bp))
1052                 bnx2x_read_mf_cfg(bp);
1053
1054         /* Read the current link report info */
1055         bnx2x_fill_report_data(bp, &cur_data);
1056
1057         /* Don't report link down or exactly the same link status twice */
1058         if (!memcmp(&cur_data, &bp->last_reported_link, sizeof(cur_data)) ||
1059             (test_bit(BNX2X_LINK_REPORT_LINK_DOWN,
1060                       &bp->last_reported_link.link_report_flags) &&
1061              test_bit(BNX2X_LINK_REPORT_LINK_DOWN,
1062                       &cur_data.link_report_flags)))
1063                 return;
1064
1065         bp->link_cnt++;
1066
1067         /* We are going to report a new link parameters now -
1068          * remember the current data for the next time.
1069          */
1070         memcpy(&bp->last_reported_link, &cur_data, sizeof(cur_data));
1071
1072         if (test_bit(BNX2X_LINK_REPORT_LINK_DOWN,
1073                      &cur_data.link_report_flags)) {
1074                 netif_carrier_off(bp->dev);
1075                 netdev_err(bp->dev, "NIC Link is Down\n");
1076                 return;
1077         } else {
1078                 const char *duplex;
1079                 const char *flow;
1080
1081                 netif_carrier_on(bp->dev);
1082
1083                 if (test_and_clear_bit(BNX2X_LINK_REPORT_FD,
1084                                        &cur_data.link_report_flags))
1085                         duplex = "full";
1086                 else
1087                         duplex = "half";
1088
1089                 /* Handle the FC at the end so that only these flags would be
1090                  * possibly set. This way we may easily check if there is no FC
1091                  * enabled.
1092                  */
1093                 if (cur_data.link_report_flags) {
1094                         if (test_bit(BNX2X_LINK_REPORT_RX_FC_ON,
1095                                      &cur_data.link_report_flags)) {
1096                                 if (test_bit(BNX2X_LINK_REPORT_TX_FC_ON,
1097                                      &cur_data.link_report_flags))
1098                                         flow = "ON - receive & transmit";
1099                                 else
1100                                         flow = "ON - receive";
1101                         } else {
1102                                 flow = "ON - transmit";
1103                         }
1104                 } else {
1105                         flow = "none";
1106                 }
1107                 netdev_info(bp->dev, "NIC Link is Up, %d Mbps %s duplex, Flow control: %s\n",
1108                             cur_data.line_speed, duplex, flow);
1109         }
1110 }
1111
1112 static void bnx2x_set_next_page_sgl(struct bnx2x_fastpath *fp)
1113 {
1114         int i;
1115
1116         for (i = 1; i <= NUM_RX_SGE_PAGES; i++) {
1117                 struct eth_rx_sge *sge;
1118
1119                 sge = &fp->rx_sge_ring[RX_SGE_CNT * i - 2];
1120                 sge->addr_hi =
1121                         cpu_to_le32(U64_HI(fp->rx_sge_mapping +
1122                         BCM_PAGE_SIZE*(i % NUM_RX_SGE_PAGES)));
1123
1124                 sge->addr_lo =
1125                         cpu_to_le32(U64_LO(fp->rx_sge_mapping +
1126                         BCM_PAGE_SIZE*(i % NUM_RX_SGE_PAGES)));
1127         }
1128 }
1129
1130 static void bnx2x_free_tpa_pool(struct bnx2x *bp,
1131                                 struct bnx2x_fastpath *fp, int last)
1132 {
1133         int i;
1134
1135         for (i = 0; i < last; i++) {
1136                 struct bnx2x_agg_info *tpa_info = &fp->tpa_info[i];
1137                 struct sw_rx_bd *first_buf = &tpa_info->first_buf;
1138                 u8 *data = first_buf->data;
1139
1140                 if (data == NULL) {
1141                         DP(NETIF_MSG_IFDOWN, "tpa bin %d empty on free\n", i);
1142                         continue;
1143                 }
1144                 if (tpa_info->tpa_state == BNX2X_TPA_START)
1145                         dma_unmap_single(&bp->pdev->dev,
1146                                          dma_unmap_addr(first_buf, mapping),
1147                                          fp->rx_buf_size, DMA_FROM_DEVICE);
1148                 kfree(data);
1149                 first_buf->data = NULL;
1150         }
1151 }
1152
1153 void bnx2x_init_rx_rings(struct bnx2x *bp)
1154 {
1155         int func = BP_FUNC(bp);
1156         u16 ring_prod;
1157         int i, j;
1158
1159         /* Allocate TPA resources */
1160         for_each_rx_queue(bp, j) {
1161                 struct bnx2x_fastpath *fp = &bp->fp[j];
1162
1163                 DP(NETIF_MSG_IFUP,
1164                    "mtu %d  rx_buf_size %d\n", bp->dev->mtu, fp->rx_buf_size);
1165
1166                 if (!fp->disable_tpa) {
1167                         /* Fill the per-aggregtion pool */
1168                         for (i = 0; i < MAX_AGG_QS(bp); i++) {
1169                                 struct bnx2x_agg_info *tpa_info =
1170                                         &fp->tpa_info[i];
1171                                 struct sw_rx_bd *first_buf =
1172                                         &tpa_info->first_buf;
1173
1174                                 first_buf->data = kmalloc(fp->rx_buf_size + NET_SKB_PAD,
1175                                                           GFP_ATOMIC);
1176                                 if (!first_buf->data) {
1177                                         BNX2X_ERR("Failed to allocate TPA skb pool for queue[%d] - disabling TPA on this queue!\n",
1178                                                   j);
1179                                         bnx2x_free_tpa_pool(bp, fp, i);
1180                                         fp->disable_tpa = 1;
1181                                         break;
1182                                 }
1183                                 dma_unmap_addr_set(first_buf, mapping, 0);
1184                                 tpa_info->tpa_state = BNX2X_TPA_STOP;
1185                         }
1186
1187                         /* "next page" elements initialization */
1188                         bnx2x_set_next_page_sgl(fp);
1189
1190                         /* set SGEs bit mask */
1191                         bnx2x_init_sge_ring_bit_mask(fp);
1192
1193                         /* Allocate SGEs and initialize the ring elements */
1194                         for (i = 0, ring_prod = 0;
1195                              i < MAX_RX_SGE_CNT*NUM_RX_SGE_PAGES; i++) {
1196
1197                                 if (bnx2x_alloc_rx_sge(bp, fp, ring_prod) < 0) {
1198                                         BNX2X_ERR("was only able to allocate %d rx sges\n",
1199                                                   i);
1200                                         BNX2X_ERR("disabling TPA for queue[%d]\n",
1201                                                   j);
1202                                         /* Cleanup already allocated elements */
1203                                         bnx2x_free_rx_sge_range(bp, fp,
1204                                                                 ring_prod);
1205                                         bnx2x_free_tpa_pool(bp, fp,
1206                                                             MAX_AGG_QS(bp));
1207                                         fp->disable_tpa = 1;
1208                                         ring_prod = 0;
1209                                         break;
1210                                 }
1211                                 ring_prod = NEXT_SGE_IDX(ring_prod);
1212                         }
1213
1214                         fp->rx_sge_prod = ring_prod;
1215                 }
1216         }
1217
1218         for_each_rx_queue(bp, j) {
1219                 struct bnx2x_fastpath *fp = &bp->fp[j];
1220
1221                 fp->rx_bd_cons = 0;
1222
1223                 /* Activate BD ring */
1224                 /* Warning!
1225                  * this will generate an interrupt (to the TSTORM)
1226                  * must only be done after chip is initialized
1227                  */
1228                 bnx2x_update_rx_prod(bp, fp, fp->rx_bd_prod, fp->rx_comp_prod,
1229                                      fp->rx_sge_prod);
1230
1231                 if (j != 0)
1232                         continue;
1233
1234                 if (CHIP_IS_E1(bp)) {
1235                         REG_WR(bp, BAR_USTRORM_INTMEM +
1236                                USTORM_MEM_WORKAROUND_ADDRESS_OFFSET(func),
1237                                U64_LO(fp->rx_comp_mapping));
1238                         REG_WR(bp, BAR_USTRORM_INTMEM +
1239                                USTORM_MEM_WORKAROUND_ADDRESS_OFFSET(func) + 4,
1240                                U64_HI(fp->rx_comp_mapping));
1241                 }
1242         }
1243 }
1244
1245 static void bnx2x_free_tx_skbs(struct bnx2x *bp)
1246 {
1247         int i;
1248         u8 cos;
1249
1250         for_each_tx_queue(bp, i) {
1251                 struct bnx2x_fastpath *fp = &bp->fp[i];
1252                 for_each_cos_in_tx_queue(fp, cos) {
1253                         struct bnx2x_fp_txdata *txdata = fp->txdata_ptr[cos];
1254                         unsigned pkts_compl = 0, bytes_compl = 0;
1255
1256                         u16 sw_prod = txdata->tx_pkt_prod;
1257                         u16 sw_cons = txdata->tx_pkt_cons;
1258
1259                         while (sw_cons != sw_prod) {
1260                                 bnx2x_free_tx_pkt(bp, txdata, TX_BD(sw_cons),
1261                                     &pkts_compl, &bytes_compl);
1262                                 sw_cons++;
1263                         }
1264                         netdev_tx_reset_queue(
1265                                 netdev_get_tx_queue(bp->dev,
1266                                                     txdata->txq_index));
1267                 }
1268         }
1269 }
1270
1271 static void bnx2x_free_rx_bds(struct bnx2x_fastpath *fp)
1272 {
1273         struct bnx2x *bp = fp->bp;
1274         int i;
1275
1276         /* ring wasn't allocated */
1277         if (fp->rx_buf_ring == NULL)
1278                 return;
1279
1280         for (i = 0; i < NUM_RX_BD; i++) {
1281                 struct sw_rx_bd *rx_buf = &fp->rx_buf_ring[i];
1282                 u8 *data = rx_buf->data;
1283
1284                 if (data == NULL)
1285                         continue;
1286                 dma_unmap_single(&bp->pdev->dev,
1287                                  dma_unmap_addr(rx_buf, mapping),
1288                                  fp->rx_buf_size, DMA_FROM_DEVICE);
1289
1290                 rx_buf->data = NULL;
1291                 kfree(data);
1292         }
1293 }
1294
1295 static void bnx2x_free_rx_skbs(struct bnx2x *bp)
1296 {
1297         int j;
1298
1299         for_each_rx_queue(bp, j) {
1300                 struct bnx2x_fastpath *fp = &bp->fp[j];
1301
1302                 bnx2x_free_rx_bds(fp);
1303
1304                 if (!fp->disable_tpa)
1305                         bnx2x_free_tpa_pool(bp, fp, MAX_AGG_QS(bp));
1306         }
1307 }
1308
1309 void bnx2x_free_skbs(struct bnx2x *bp)
1310 {
1311         bnx2x_free_tx_skbs(bp);
1312         bnx2x_free_rx_skbs(bp);
1313 }
1314
1315 void bnx2x_update_max_mf_config(struct bnx2x *bp, u32 value)
1316 {
1317         /* load old values */
1318         u32 mf_cfg = bp->mf_config[BP_VN(bp)];
1319
1320         if (value != bnx2x_extract_max_cfg(bp, mf_cfg)) {
1321                 /* leave all but MAX value */
1322                 mf_cfg &= ~FUNC_MF_CFG_MAX_BW_MASK;
1323
1324                 /* set new MAX value */
1325                 mf_cfg |= (value << FUNC_MF_CFG_MAX_BW_SHIFT)
1326                                 & FUNC_MF_CFG_MAX_BW_MASK;
1327
1328                 bnx2x_fw_command(bp, DRV_MSG_CODE_SET_MF_BW, mf_cfg);
1329         }
1330 }
1331
1332 /**
1333  * bnx2x_free_msix_irqs - free previously requested MSI-X IRQ vectors
1334  *
1335  * @bp:         driver handle
1336  * @nvecs:      number of vectors to be released
1337  */
1338 static void bnx2x_free_msix_irqs(struct bnx2x *bp, int nvecs)
1339 {
1340         int i, offset = 0;
1341
1342         if (nvecs == offset)
1343                 return;
1344         free_irq(bp->msix_table[offset].vector, bp->dev);
1345         DP(NETIF_MSG_IFDOWN, "released sp irq (%d)\n",
1346            bp->msix_table[offset].vector);
1347         offset++;
1348 #ifdef BCM_CNIC
1349         if (nvecs == offset)
1350                 return;
1351         offset++;
1352 #endif
1353
1354         for_each_eth_queue(bp, i) {
1355                 if (nvecs == offset)
1356                         return;
1357                 DP(NETIF_MSG_IFDOWN, "about to release fp #%d->%d irq\n",
1358                    i, bp->msix_table[offset].vector);
1359
1360                 free_irq(bp->msix_table[offset++].vector, &bp->fp[i]);
1361         }
1362 }
1363
1364 void bnx2x_free_irq(struct bnx2x *bp)
1365 {
1366         if (bp->flags & USING_MSIX_FLAG &&
1367             !(bp->flags & USING_SINGLE_MSIX_FLAG))
1368                 bnx2x_free_msix_irqs(bp, BNX2X_NUM_ETH_QUEUES(bp) +
1369                                      CNIC_PRESENT + 1);
1370         else
1371                 free_irq(bp->dev->irq, bp->dev);
1372 }
1373
1374 int bnx2x_enable_msix(struct bnx2x *bp)
1375 {
1376         int msix_vec = 0, i, rc, req_cnt;
1377
1378         bp->msix_table[msix_vec].entry = msix_vec;
1379         BNX2X_DEV_INFO("msix_table[0].entry = %d (slowpath)\n",
1380            bp->msix_table[0].entry);
1381         msix_vec++;
1382
1383 #ifdef BCM_CNIC
1384         bp->msix_table[msix_vec].entry = msix_vec;
1385         BNX2X_DEV_INFO("msix_table[%d].entry = %d (CNIC)\n",
1386            bp->msix_table[msix_vec].entry, bp->msix_table[msix_vec].entry);
1387         msix_vec++;
1388 #endif
1389         /* We need separate vectors for ETH queues only (not FCoE) */
1390         for_each_eth_queue(bp, i) {
1391                 bp->msix_table[msix_vec].entry = msix_vec;
1392                 BNX2X_DEV_INFO("msix_table[%d].entry = %d (fastpath #%u)\n",
1393                                msix_vec, msix_vec, i);
1394                 msix_vec++;
1395         }
1396
1397         req_cnt = BNX2X_NUM_ETH_QUEUES(bp) + CNIC_PRESENT + 1;
1398
1399         rc = pci_enable_msix(bp->pdev, &bp->msix_table[0], req_cnt);
1400
1401         /*
1402          * reconfigure number of tx/rx queues according to available
1403          * MSI-X vectors
1404          */
1405         if (rc >= BNX2X_MIN_MSIX_VEC_CNT) {
1406                 /* how less vectors we will have? */
1407                 int diff = req_cnt - rc;
1408
1409                 BNX2X_DEV_INFO("Trying to use less MSI-X vectors: %d\n", rc);
1410
1411                 rc = pci_enable_msix(bp->pdev, &bp->msix_table[0], rc);
1412
1413                 if (rc) {
1414                         BNX2X_DEV_INFO("MSI-X is not attainable rc %d\n", rc);
1415                         goto no_msix;
1416                 }
1417                 /*
1418                  * decrease number of queues by number of unallocated entries
1419                  */
1420                 bp->num_queues -= diff;
1421
1422                 BNX2X_DEV_INFO("New queue configuration set: %d\n",
1423                                bp->num_queues);
1424         } else if (rc > 0) {
1425                 /* Get by with single vector */
1426                 rc = pci_enable_msix(bp->pdev, &bp->msix_table[0], 1);
1427                 if (rc) {
1428                         BNX2X_DEV_INFO("Single MSI-X is not attainable rc %d\n",
1429                                        rc);
1430                         goto no_msix;
1431                 }
1432
1433                 BNX2X_DEV_INFO("Using single MSI-X vector\n");
1434                 bp->flags |= USING_SINGLE_MSIX_FLAG;
1435
1436         } else if (rc < 0) {
1437                 BNX2X_DEV_INFO("MSI-X is not attainable  rc %d\n", rc);
1438                 goto no_msix;
1439         }
1440
1441         bp->flags |= USING_MSIX_FLAG;
1442
1443         return 0;
1444
1445 no_msix:
1446         /* fall to INTx if not enough memory */
1447         if (rc == -ENOMEM)
1448                 bp->flags |= DISABLE_MSI_FLAG;
1449
1450         return rc;
1451 }
1452
1453 static int bnx2x_req_msix_irqs(struct bnx2x *bp)
1454 {
1455         int i, rc, offset = 0;
1456
1457         rc = request_irq(bp->msix_table[offset++].vector,
1458                          bnx2x_msix_sp_int, 0,
1459                          bp->dev->name, bp->dev);
1460         if (rc) {
1461                 BNX2X_ERR("request sp irq failed\n");
1462                 return -EBUSY;
1463         }
1464
1465 #ifdef BCM_CNIC
1466         offset++;
1467 #endif
1468         for_each_eth_queue(bp, i) {
1469                 struct bnx2x_fastpath *fp = &bp->fp[i];
1470                 snprintf(fp->name, sizeof(fp->name), "%s-fp-%d",
1471                          bp->dev->name, i);
1472
1473                 rc = request_irq(bp->msix_table[offset].vector,
1474                                  bnx2x_msix_fp_int, 0, fp->name, fp);
1475                 if (rc) {
1476                         BNX2X_ERR("request fp #%d irq (%d) failed  rc %d\n", i,
1477                               bp->msix_table[offset].vector, rc);
1478                         bnx2x_free_msix_irqs(bp, offset);
1479                         return -EBUSY;
1480                 }
1481
1482                 offset++;
1483         }
1484
1485         i = BNX2X_NUM_ETH_QUEUES(bp);
1486         offset = 1 + CNIC_PRESENT;
1487         netdev_info(bp->dev, "using MSI-X  IRQs: sp %d  fp[%d] %d ... fp[%d] %d\n",
1488                bp->msix_table[0].vector,
1489                0, bp->msix_table[offset].vector,
1490                i - 1, bp->msix_table[offset + i - 1].vector);
1491
1492         return 0;
1493 }
1494
1495 int bnx2x_enable_msi(struct bnx2x *bp)
1496 {
1497         int rc;
1498
1499         rc = pci_enable_msi(bp->pdev);
1500         if (rc) {
1501                 BNX2X_DEV_INFO("MSI is not attainable\n");
1502                 return -1;
1503         }
1504         bp->flags |= USING_MSI_FLAG;
1505
1506         return 0;
1507 }
1508
1509 static int bnx2x_req_irq(struct bnx2x *bp)
1510 {
1511         unsigned long flags;
1512         unsigned int irq;
1513
1514         if (bp->flags & (USING_MSI_FLAG | USING_MSIX_FLAG))
1515                 flags = 0;
1516         else
1517                 flags = IRQF_SHARED;
1518
1519         if (bp->flags & USING_MSIX_FLAG)
1520                 irq = bp->msix_table[0].vector;
1521         else
1522                 irq = bp->pdev->irq;
1523
1524         return request_irq(irq, bnx2x_interrupt, flags, bp->dev->name, bp->dev);
1525 }
1526
1527 static int bnx2x_setup_irqs(struct bnx2x *bp)
1528 {
1529         int rc = 0;
1530         if (bp->flags & USING_MSIX_FLAG &&
1531             !(bp->flags & USING_SINGLE_MSIX_FLAG)) {
1532                 rc = bnx2x_req_msix_irqs(bp);
1533                 if (rc)
1534                         return rc;
1535         } else {
1536                 bnx2x_ack_int(bp);
1537                 rc = bnx2x_req_irq(bp);
1538                 if (rc) {
1539                         BNX2X_ERR("IRQ request failed  rc %d, aborting\n", rc);
1540                         return rc;
1541                 }
1542                 if (bp->flags & USING_MSI_FLAG) {
1543                         bp->dev->irq = bp->pdev->irq;
1544                         netdev_info(bp->dev, "using MSI IRQ %d\n",
1545                                     bp->dev->irq);
1546                 }
1547                 if (bp->flags & USING_MSIX_FLAG) {
1548                         bp->dev->irq = bp->msix_table[0].vector;
1549                         netdev_info(bp->dev, "using MSIX IRQ %d\n",
1550                                     bp->dev->irq);
1551                 }
1552         }
1553
1554         return 0;
1555 }
1556
1557 static void bnx2x_napi_enable(struct bnx2x *bp)
1558 {
1559         int i;
1560
1561         for_each_rx_queue(bp, i)
1562                 napi_enable(&bnx2x_fp(bp, i, napi));
1563 }
1564
1565 static void bnx2x_napi_disable(struct bnx2x *bp)
1566 {
1567         int i;
1568
1569         for_each_rx_queue(bp, i)
1570                 napi_disable(&bnx2x_fp(bp, i, napi));
1571 }
1572
1573 void bnx2x_netif_start(struct bnx2x *bp)
1574 {
1575         if (netif_running(bp->dev)) {
1576                 bnx2x_napi_enable(bp);
1577                 bnx2x_int_enable(bp);
1578                 if (bp->state == BNX2X_STATE_OPEN)
1579                         netif_tx_wake_all_queues(bp->dev);
1580         }
1581 }
1582
1583 void bnx2x_netif_stop(struct bnx2x *bp, int disable_hw)
1584 {
1585         bnx2x_int_disable_sync(bp, disable_hw);
1586         bnx2x_napi_disable(bp);
1587 }
1588
1589 u16 bnx2x_select_queue(struct net_device *dev, struct sk_buff *skb)
1590 {
1591         struct bnx2x *bp = netdev_priv(dev);
1592
1593 #ifdef BCM_CNIC
1594         if (!NO_FCOE(bp)) {
1595                 struct ethhdr *hdr = (struct ethhdr *)skb->data;
1596                 u16 ether_type = ntohs(hdr->h_proto);
1597
1598                 /* Skip VLAN tag if present */
1599                 if (ether_type == ETH_P_8021Q) {
1600                         struct vlan_ethhdr *vhdr =
1601                                 (struct vlan_ethhdr *)skb->data;
1602
1603                         ether_type = ntohs(vhdr->h_vlan_encapsulated_proto);
1604                 }
1605
1606                 /* If ethertype is FCoE or FIP - use FCoE ring */
1607                 if ((ether_type == ETH_P_FCOE) || (ether_type == ETH_P_FIP))
1608                         return bnx2x_fcoe_tx(bp, txq_index);
1609         }
1610 #endif
1611         /* select a non-FCoE queue */
1612         return __skb_tx_hash(dev, skb, BNX2X_NUM_ETH_QUEUES(bp));
1613 }
1614
1615
1616 void bnx2x_set_num_queues(struct bnx2x *bp)
1617 {
1618         /* RSS queues */
1619         bp->num_queues = bnx2x_calc_num_queues(bp);
1620
1621 #ifdef BCM_CNIC
1622         /* override in STORAGE SD modes */
1623         if (IS_MF_STORAGE_SD(bp) || IS_MF_FCOE_AFEX(bp))
1624                 bp->num_queues = 1;
1625 #endif
1626         /* Add special queues */
1627         bp->num_queues += NON_ETH_CONTEXT_USE;
1628
1629         BNX2X_DEV_INFO("set number of queues to %d\n", bp->num_queues);
1630 }
1631
1632 /**
1633  * bnx2x_set_real_num_queues - configure netdev->real_num_[tx,rx]_queues
1634  *
1635  * @bp:         Driver handle
1636  *
1637  * We currently support for at most 16 Tx queues for each CoS thus we will
1638  * allocate a multiple of 16 for ETH L2 rings according to the value of the
1639  * bp->max_cos.
1640  *
1641  * If there is an FCoE L2 queue the appropriate Tx queue will have the next
1642  * index after all ETH L2 indices.
1643  *
1644  * If the actual number of Tx queues (for each CoS) is less than 16 then there
1645  * will be the holes at the end of each group of 16 ETh L2 indices (0..15,
1646  * 16..31,...) with indicies that are not coupled with any real Tx queue.
1647  *
1648  * The proper configuration of skb->queue_mapping is handled by
1649  * bnx2x_select_queue() and __skb_tx_hash().
1650  *
1651  * bnx2x_setup_tc() takes care of the proper TC mappings so that __skb_tx_hash()
1652  * will return a proper Tx index if TC is enabled (netdev->num_tc > 0).
1653  */
1654 static int bnx2x_set_real_num_queues(struct bnx2x *bp)
1655 {
1656         int rc, tx, rx;
1657
1658         tx = BNX2X_NUM_ETH_QUEUES(bp) * bp->max_cos;
1659         rx = BNX2X_NUM_QUEUES(bp) - NON_ETH_CONTEXT_USE;
1660
1661 /* account for fcoe queue */
1662 #ifdef BCM_CNIC
1663         if (!NO_FCOE(bp)) {
1664                 rx += FCOE_PRESENT;
1665                 tx += FCOE_PRESENT;
1666         }
1667 #endif
1668
1669         rc = netif_set_real_num_tx_queues(bp->dev, tx);
1670         if (rc) {
1671                 BNX2X_ERR("Failed to set real number of Tx queues: %d\n", rc);
1672                 return rc;
1673         }
1674         rc = netif_set_real_num_rx_queues(bp->dev, rx);
1675         if (rc) {
1676                 BNX2X_ERR("Failed to set real number of Rx queues: %d\n", rc);
1677                 return rc;
1678         }
1679
1680         DP(NETIF_MSG_IFUP, "Setting real num queues to (tx, rx) (%d, %d)\n",
1681                           tx, rx);
1682
1683         return rc;
1684 }
1685
1686 static void bnx2x_set_rx_buf_size(struct bnx2x *bp)
1687 {
1688         int i;
1689
1690         for_each_queue(bp, i) {
1691                 struct bnx2x_fastpath *fp = &bp->fp[i];
1692                 u32 mtu;
1693
1694                 /* Always use a mini-jumbo MTU for the FCoE L2 ring */
1695                 if (IS_FCOE_IDX(i))
1696                         /*
1697                          * Although there are no IP frames expected to arrive to
1698                          * this ring we still want to add an
1699                          * IP_HEADER_ALIGNMENT_PADDING to prevent a buffer
1700                          * overrun attack.
1701                          */
1702                         mtu = BNX2X_FCOE_MINI_JUMBO_MTU;
1703                 else
1704                         mtu = bp->dev->mtu;
1705                 fp->rx_buf_size = BNX2X_FW_RX_ALIGN_START +
1706                                   IP_HEADER_ALIGNMENT_PADDING +
1707                                   ETH_OVREHEAD +
1708                                   mtu +
1709                                   BNX2X_FW_RX_ALIGN_END;
1710                 /* Note : rx_buf_size doesnt take into account NET_SKB_PAD */
1711         }
1712 }
1713
1714 static int bnx2x_init_rss_pf(struct bnx2x *bp)
1715 {
1716         int i;
1717         u8 num_eth_queues = BNX2X_NUM_ETH_QUEUES(bp);
1718
1719         /* Prepare the initial contents fo the indirection table if RSS is
1720          * enabled
1721          */
1722         for (i = 0; i < sizeof(bp->rss_conf_obj.ind_table); i++)
1723                 bp->rss_conf_obj.ind_table[i] =
1724                         bp->fp->cl_id +
1725                         ethtool_rxfh_indir_default(i, num_eth_queues);
1726
1727         /*
1728          * For 57710 and 57711 SEARCHER configuration (rss_keys) is
1729          * per-port, so if explicit configuration is needed , do it only
1730          * for a PMF.
1731          *
1732          * For 57712 and newer on the other hand it's a per-function
1733          * configuration.
1734          */
1735         return bnx2x_config_rss_eth(bp, bp->port.pmf || !CHIP_IS_E1x(bp));
1736 }
1737
1738 int bnx2x_config_rss_pf(struct bnx2x *bp, struct bnx2x_rss_config_obj *rss_obj,
1739                         bool config_hash)
1740 {
1741         struct bnx2x_config_rss_params params = {NULL};
1742         int i;
1743
1744         /* Although RSS is meaningless when there is a single HW queue we
1745          * still need it enabled in order to have HW Rx hash generated.
1746          *
1747          * if (!is_eth_multi(bp))
1748          *      bp->multi_mode = ETH_RSS_MODE_DISABLED;
1749          */
1750
1751         params.rss_obj = rss_obj;
1752
1753         __set_bit(RAMROD_COMP_WAIT, &params.ramrod_flags);
1754
1755         __set_bit(BNX2X_RSS_MODE_REGULAR, &params.rss_flags);
1756
1757         /* RSS configuration */
1758         __set_bit(BNX2X_RSS_IPV4, &params.rss_flags);
1759         __set_bit(BNX2X_RSS_IPV4_TCP, &params.rss_flags);
1760         __set_bit(BNX2X_RSS_IPV6, &params.rss_flags);
1761         __set_bit(BNX2X_RSS_IPV6_TCP, &params.rss_flags);
1762         if (rss_obj->udp_rss_v4)
1763                 __set_bit(BNX2X_RSS_IPV4_UDP, &params.rss_flags);
1764         if (rss_obj->udp_rss_v6)
1765                 __set_bit(BNX2X_RSS_IPV6_UDP, &params.rss_flags);
1766
1767         /* Hash bits */
1768         params.rss_result_mask = MULTI_MASK;
1769
1770         memcpy(params.ind_table, rss_obj->ind_table, sizeof(params.ind_table));
1771
1772         if (config_hash) {
1773                 /* RSS keys */
1774                 for (i = 0; i < sizeof(params.rss_key) / 4; i++)
1775                         params.rss_key[i] = random32();
1776
1777                 __set_bit(BNX2X_RSS_SET_SRCH, &params.rss_flags);
1778         }
1779
1780         return bnx2x_config_rss(bp, &params);
1781 }
1782
1783 static int bnx2x_init_hw(struct bnx2x *bp, u32 load_code)
1784 {
1785         struct bnx2x_func_state_params func_params = {NULL};
1786
1787         /* Prepare parameters for function state transitions */
1788         __set_bit(RAMROD_COMP_WAIT, &func_params.ramrod_flags);
1789
1790         func_params.f_obj = &bp->func_obj;
1791         func_params.cmd = BNX2X_F_CMD_HW_INIT;
1792
1793         func_params.params.hw_init.load_phase = load_code;
1794
1795         return bnx2x_func_state_change(bp, &func_params);
1796 }
1797
1798 /*
1799  * Cleans the object that have internal lists without sending
1800  * ramrods. Should be run when interrutps are disabled.
1801  */
1802 static void bnx2x_squeeze_objects(struct bnx2x *bp)
1803 {
1804         int rc;
1805         unsigned long ramrod_flags = 0, vlan_mac_flags = 0;
1806         struct bnx2x_mcast_ramrod_params rparam = {NULL};
1807         struct bnx2x_vlan_mac_obj *mac_obj = &bp->sp_objs->mac_obj;
1808
1809         /***************** Cleanup MACs' object first *************************/
1810
1811         /* Wait for completion of requested */
1812         __set_bit(RAMROD_COMP_WAIT, &ramrod_flags);
1813         /* Perform a dry cleanup */
1814         __set_bit(RAMROD_DRV_CLR_ONLY, &ramrod_flags);
1815
1816         /* Clean ETH primary MAC */
1817         __set_bit(BNX2X_ETH_MAC, &vlan_mac_flags);
1818         rc = mac_obj->delete_all(bp, &bp->sp_objs->mac_obj, &vlan_mac_flags,
1819                                  &ramrod_flags);
1820         if (rc != 0)
1821                 BNX2X_ERR("Failed to clean ETH MACs: %d\n", rc);
1822
1823         /* Cleanup UC list */
1824         vlan_mac_flags = 0;
1825         __set_bit(BNX2X_UC_LIST_MAC, &vlan_mac_flags);
1826         rc = mac_obj->delete_all(bp, mac_obj, &vlan_mac_flags,
1827                                  &ramrod_flags);
1828         if (rc != 0)
1829                 BNX2X_ERR("Failed to clean UC list MACs: %d\n", rc);
1830
1831         /***************** Now clean mcast object *****************************/
1832         rparam.mcast_obj = &bp->mcast_obj;
1833         __set_bit(RAMROD_DRV_CLR_ONLY, &rparam.ramrod_flags);
1834
1835         /* Add a DEL command... */
1836         rc = bnx2x_config_mcast(bp, &rparam, BNX2X_MCAST_CMD_DEL);
1837         if (rc < 0)
1838                 BNX2X_ERR("Failed to add a new DEL command to a multi-cast object: %d\n",
1839                           rc);
1840
1841         /* ...and wait until all pending commands are cleared */
1842         rc = bnx2x_config_mcast(bp, &rparam, BNX2X_MCAST_CMD_CONT);
1843         while (rc != 0) {
1844                 if (rc < 0) {
1845                         BNX2X_ERR("Failed to clean multi-cast object: %d\n",
1846                                   rc);
1847                         return;
1848                 }
1849
1850                 rc = bnx2x_config_mcast(bp, &rparam, BNX2X_MCAST_CMD_CONT);
1851         }
1852 }
1853
1854 #ifndef BNX2X_STOP_ON_ERROR
1855 #define LOAD_ERROR_EXIT(bp, label) \
1856         do { \
1857                 (bp)->state = BNX2X_STATE_ERROR; \
1858                 goto label; \
1859         } while (0)
1860 #else
1861 #define LOAD_ERROR_EXIT(bp, label) \
1862         do { \
1863                 (bp)->state = BNX2X_STATE_ERROR; \
1864                 (bp)->panic = 1; \
1865                 return -EBUSY; \
1866         } while (0)
1867 #endif
1868
1869 bool bnx2x_test_firmware_version(struct bnx2x *bp, bool is_err)
1870 {
1871         /* build FW version dword */
1872         u32 my_fw = (BCM_5710_FW_MAJOR_VERSION) +
1873                     (BCM_5710_FW_MINOR_VERSION << 8) +
1874                     (BCM_5710_FW_REVISION_VERSION << 16) +
1875                     (BCM_5710_FW_ENGINEERING_VERSION << 24);
1876
1877         /* read loaded FW from chip */
1878         u32 loaded_fw = REG_RD(bp, XSEM_REG_PRAM);
1879
1880         DP(NETIF_MSG_IFUP, "loaded fw %x, my fw %x\n", loaded_fw, my_fw);
1881
1882         if (loaded_fw != my_fw) {
1883                 if (is_err)
1884                         BNX2X_ERR("bnx2x with FW %x was already loaded, which mismatches my %x FW. aborting\n",
1885                                   loaded_fw, my_fw);
1886                 return false;
1887         }
1888
1889         return true;
1890 }
1891
1892 /**
1893  * bnx2x_bz_fp - zero content of the fastpath structure.
1894  *
1895  * @bp:         driver handle
1896  * @index:      fastpath index to be zeroed
1897  *
1898  * Makes sure the contents of the bp->fp[index].napi is kept
1899  * intact.
1900  */
1901 static void bnx2x_bz_fp(struct bnx2x *bp, int index)
1902 {
1903         struct bnx2x_fastpath *fp = &bp->fp[index];
1904         struct bnx2x_fp_stats *fp_stats = &bp->fp_stats[index];
1905
1906         int cos;
1907         struct napi_struct orig_napi = fp->napi;
1908         struct bnx2x_agg_info *orig_tpa_info = fp->tpa_info;
1909         /* bzero bnx2x_fastpath contents */
1910         if (bp->stats_init) {
1911                 memset(fp->tpa_info, 0, sizeof(*fp->tpa_info));
1912                 memset(fp, 0, sizeof(*fp));
1913         } else {
1914                 /* Keep Queue statistics */
1915                 struct bnx2x_eth_q_stats *tmp_eth_q_stats;
1916                 struct bnx2x_eth_q_stats_old *tmp_eth_q_stats_old;
1917
1918                 tmp_eth_q_stats = kzalloc(sizeof(struct bnx2x_eth_q_stats),
1919                                           GFP_KERNEL);
1920                 if (tmp_eth_q_stats)
1921                         memcpy(tmp_eth_q_stats, &fp_stats->eth_q_stats,
1922                                sizeof(struct bnx2x_eth_q_stats));
1923
1924                 tmp_eth_q_stats_old =
1925                         kzalloc(sizeof(struct bnx2x_eth_q_stats_old),
1926                                 GFP_KERNEL);
1927                 if (tmp_eth_q_stats_old)
1928                         memcpy(tmp_eth_q_stats_old, &fp_stats->eth_q_stats_old,
1929                                sizeof(struct bnx2x_eth_q_stats_old));
1930
1931                 memset(fp->tpa_info, 0, sizeof(*fp->tpa_info));
1932                 memset(fp, 0, sizeof(*fp));
1933
1934                 if (tmp_eth_q_stats) {
1935                         memcpy(&fp_stats->eth_q_stats, tmp_eth_q_stats,
1936                                sizeof(struct bnx2x_eth_q_stats));
1937                         kfree(tmp_eth_q_stats);
1938                 }
1939
1940                 if (tmp_eth_q_stats_old) {
1941                         memcpy(&fp_stats->eth_q_stats_old, tmp_eth_q_stats_old,
1942                                sizeof(struct bnx2x_eth_q_stats_old));
1943                         kfree(tmp_eth_q_stats_old);
1944                 }
1945
1946         }
1947
1948         /* Restore the NAPI object as it has been already initialized */
1949         fp->napi = orig_napi;
1950         fp->tpa_info = orig_tpa_info;
1951         fp->bp = bp;
1952         fp->index = index;
1953         if (IS_ETH_FP(fp))
1954                 fp->max_cos = bp->max_cos;
1955         else
1956                 /* Special queues support only one CoS */
1957                 fp->max_cos = 1;
1958
1959         /* Init txdata pointers */
1960 #ifdef BCM_CNIC
1961         if (IS_FCOE_FP(fp))
1962                 fp->txdata_ptr[0] = &bp->bnx2x_txq[FCOE_TXQ_IDX(bp)];
1963 #endif
1964         if (IS_ETH_FP(fp))
1965                 for_each_cos_in_tx_queue(fp, cos)
1966                         fp->txdata_ptr[cos] = &bp->bnx2x_txq[cos *
1967                                 BNX2X_NUM_ETH_QUEUES(bp) + index];
1968
1969         /*
1970          * set the tpa flag for each queue. The tpa flag determines the queue
1971          * minimal size so it must be set prior to queue memory allocation
1972          */
1973         fp->disable_tpa = !(bp->flags & TPA_ENABLE_FLAG ||
1974                                   (bp->flags & GRO_ENABLE_FLAG &&
1975                                    bnx2x_mtu_allows_gro(bp->dev->mtu)));
1976         if (bp->flags & TPA_ENABLE_FLAG)
1977                 fp->mode = TPA_MODE_LRO;
1978         else if (bp->flags & GRO_ENABLE_FLAG)
1979                 fp->mode = TPA_MODE_GRO;
1980
1981 #ifdef BCM_CNIC
1982         /* We don't want TPA on an FCoE L2 ring */
1983         if (IS_FCOE_FP(fp))
1984                 fp->disable_tpa = 1;
1985 #endif
1986 }
1987
1988
1989 /* must be called with rtnl_lock */
1990 int bnx2x_nic_load(struct bnx2x *bp, int load_mode)
1991 {
1992         int port = BP_PORT(bp);
1993         u32 load_code;
1994         int i, rc;
1995
1996 #ifdef BNX2X_STOP_ON_ERROR
1997         if (unlikely(bp->panic)) {
1998                 BNX2X_ERR("Can't load NIC when there is panic\n");
1999                 return -EPERM;
2000         }
2001 #endif
2002
2003         bp->state = BNX2X_STATE_OPENING_WAIT4_LOAD;
2004
2005         /* Set the initial link reported state to link down */
2006         bnx2x_acquire_phy_lock(bp);
2007         memset(&bp->last_reported_link, 0, sizeof(bp->last_reported_link));
2008         __set_bit(BNX2X_LINK_REPORT_LINK_DOWN,
2009                 &bp->last_reported_link.link_report_flags);
2010         bnx2x_release_phy_lock(bp);
2011
2012         /* must be called before memory allocation and HW init */
2013         bnx2x_ilt_set_info(bp);
2014
2015         /*
2016          * Zero fastpath structures preserving invariants like napi, which are
2017          * allocated only once, fp index, max_cos, bp pointer.
2018          * Also set fp->disable_tpa and txdata_ptr.
2019          */
2020         DP(NETIF_MSG_IFUP, "num queues: %d", bp->num_queues);
2021         for_each_queue(bp, i)
2022                 bnx2x_bz_fp(bp, i);
2023         memset(bp->bnx2x_txq, 0, bp->bnx2x_txq_size *
2024                sizeof(struct bnx2x_fp_txdata));
2025
2026
2027         /* Set the receive queues buffer size */
2028         bnx2x_set_rx_buf_size(bp);
2029
2030         if (bnx2x_alloc_mem(bp))
2031                 return -ENOMEM;
2032
2033         /* As long as bnx2x_alloc_mem() may possibly update
2034          * bp->num_queues, bnx2x_set_real_num_queues() should always
2035          * come after it.
2036          */
2037         rc = bnx2x_set_real_num_queues(bp);
2038         if (rc) {
2039                 BNX2X_ERR("Unable to set real_num_queues\n");
2040                 LOAD_ERROR_EXIT(bp, load_error0);
2041         }
2042
2043         /* configure multi cos mappings in kernel.
2044          * this configuration may be overriden by a multi class queue discipline
2045          * or by a dcbx negotiation result.
2046          */
2047         bnx2x_setup_tc(bp->dev, bp->max_cos);
2048
2049         bnx2x_napi_enable(bp);
2050
2051         /* set pf load just before approaching the MCP */
2052         bnx2x_set_pf_load(bp);
2053
2054         /* Send LOAD_REQUEST command to MCP
2055          * Returns the type of LOAD command:
2056          * if it is the first port to be initialized
2057          * common blocks should be initialized, otherwise - not
2058          */
2059         if (!BP_NOMCP(bp)) {
2060                 /* init fw_seq */
2061                 bp->fw_seq =
2062                         (SHMEM_RD(bp, func_mb[BP_FW_MB_IDX(bp)].drv_mb_header) &
2063                          DRV_MSG_SEQ_NUMBER_MASK);
2064                 BNX2X_DEV_INFO("fw_seq 0x%08x\n", bp->fw_seq);
2065
2066                 /* Get current FW pulse sequence */
2067                 bp->fw_drv_pulse_wr_seq =
2068                         (SHMEM_RD(bp, func_mb[BP_FW_MB_IDX(bp)].drv_pulse_mb) &
2069                          DRV_PULSE_SEQ_MASK);
2070                 BNX2X_DEV_INFO("drv_pulse 0x%x\n", bp->fw_drv_pulse_wr_seq);
2071
2072                 load_code = bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_REQ, 0);
2073                 if (!load_code) {
2074                         BNX2X_ERR("MCP response failure, aborting\n");
2075                         rc = -EBUSY;
2076                         LOAD_ERROR_EXIT(bp, load_error1);
2077                 }
2078                 if (load_code == FW_MSG_CODE_DRV_LOAD_REFUSED) {
2079                         BNX2X_ERR("Driver load refused\n");
2080                         rc = -EBUSY; /* other port in diagnostic mode */
2081                         LOAD_ERROR_EXIT(bp, load_error1);
2082                 }
2083                 if (load_code != FW_MSG_CODE_DRV_LOAD_COMMON_CHIP &&
2084                     load_code != FW_MSG_CODE_DRV_LOAD_COMMON) {
2085                         /* abort nic load if version mismatch */
2086                         if (!bnx2x_test_firmware_version(bp, true)) {
2087                                 rc = -EBUSY;
2088                                 LOAD_ERROR_EXIT(bp, load_error2);
2089                         }
2090                 }
2091
2092         } else {
2093                 int path = BP_PATH(bp);
2094
2095                 DP(NETIF_MSG_IFUP, "NO MCP - load counts[%d]      %d, %d, %d\n",
2096                    path, load_count[path][0], load_count[path][1],
2097                    load_count[path][2]);
2098                 load_count[path][0]++;
2099                 load_count[path][1 + port]++;
2100                 DP(NETIF_MSG_IFUP, "NO MCP - new load counts[%d]  %d, %d, %d\n",
2101                    path, load_count[path][0], load_count[path][1],
2102                    load_count[path][2]);
2103                 if (load_count[path][0] == 1)
2104                         load_code = FW_MSG_CODE_DRV_LOAD_COMMON;
2105                 else if (load_count[path][1 + port] == 1)
2106                         load_code = FW_MSG_CODE_DRV_LOAD_PORT;
2107                 else
2108                         load_code = FW_MSG_CODE_DRV_LOAD_FUNCTION;
2109         }
2110
2111         if ((load_code == FW_MSG_CODE_DRV_LOAD_COMMON) ||
2112             (load_code == FW_MSG_CODE_DRV_LOAD_COMMON_CHIP) ||
2113             (load_code == FW_MSG_CODE_DRV_LOAD_PORT)) {
2114                 bp->port.pmf = 1;
2115                 /*
2116                  * We need the barrier to ensure the ordering between the
2117                  * writing to bp->port.pmf here and reading it from the
2118                  * bnx2x_periodic_task().
2119                  */
2120                 smp_mb();
2121         } else
2122                 bp->port.pmf = 0;
2123
2124         DP(NETIF_MSG_IFUP, "pmf %d\n", bp->port.pmf);
2125
2126         /* Init Function state controlling object */
2127         bnx2x__init_func_obj(bp);
2128
2129         /* Initialize HW */
2130         rc = bnx2x_init_hw(bp, load_code);
2131         if (rc) {
2132                 BNX2X_ERR("HW init failed, aborting\n");
2133                 bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
2134                 LOAD_ERROR_EXIT(bp, load_error2);
2135         }
2136
2137         /* Connect to IRQs */
2138         rc = bnx2x_setup_irqs(bp);
2139         if (rc) {
2140                 BNX2X_ERR("IRQs setup failed\n");
2141                 bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
2142                 LOAD_ERROR_EXIT(bp, load_error2);
2143         }
2144
2145         /* Setup NIC internals and enable interrupts */
2146         bnx2x_nic_init(bp, load_code);
2147
2148         /* Init per-function objects */
2149         bnx2x_init_bp_objs(bp);
2150
2151         if (((load_code == FW_MSG_CODE_DRV_LOAD_COMMON) ||
2152             (load_code == FW_MSG_CODE_DRV_LOAD_COMMON_CHIP)) &&
2153             (bp->common.shmem2_base)) {
2154                 if (SHMEM2_HAS(bp, dcc_support))
2155                         SHMEM2_WR(bp, dcc_support,
2156                                   (SHMEM_DCC_SUPPORT_DISABLE_ENABLE_PF_TLV |
2157                                    SHMEM_DCC_SUPPORT_BANDWIDTH_ALLOCATION_TLV));
2158                 if (SHMEM2_HAS(bp, afex_driver_support))
2159                         SHMEM2_WR(bp, afex_driver_support,
2160                                   SHMEM_AFEX_SUPPORTED_VERSION_ONE);
2161         }
2162
2163         /* Set AFEX default VLAN tag to an invalid value */
2164         bp->afex_def_vlan_tag = -1;
2165
2166         bp->state = BNX2X_STATE_OPENING_WAIT4_PORT;
2167         rc = bnx2x_func_start(bp);
2168         if (rc) {
2169                 BNX2X_ERR("Function start failed!\n");
2170                 bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
2171                 LOAD_ERROR_EXIT(bp, load_error3);
2172         }
2173
2174         /* Send LOAD_DONE command to MCP */
2175         if (!BP_NOMCP(bp)) {
2176                 load_code = bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
2177                 if (!load_code) {
2178                         BNX2X_ERR("MCP response failure, aborting\n");
2179                         rc = -EBUSY;
2180                         LOAD_ERROR_EXIT(bp, load_error3);
2181                 }
2182         }
2183
2184         rc = bnx2x_setup_leading(bp);
2185         if (rc) {
2186                 BNX2X_ERR("Setup leading failed!\n");
2187                 LOAD_ERROR_EXIT(bp, load_error3);
2188         }
2189
2190 #ifdef BCM_CNIC
2191         /* Enable Timer scan */
2192         REG_WR(bp, TM_REG_EN_LINEAR0_TIMER + port*4, 1);
2193 #endif
2194
2195         for_each_nondefault_queue(bp, i) {
2196                 rc = bnx2x_setup_queue(bp, &bp->fp[i], 0);
2197                 if (rc) {
2198                         BNX2X_ERR("Queue setup failed\n");
2199                         LOAD_ERROR_EXIT(bp, load_error4);
2200                 }
2201         }
2202
2203         rc = bnx2x_init_rss_pf(bp);
2204         if (rc) {
2205                 BNX2X_ERR("PF RSS init failed\n");
2206                 LOAD_ERROR_EXIT(bp, load_error4);
2207         }
2208
2209         /* Now when Clients are configured we are ready to work */
2210         bp->state = BNX2X_STATE_OPEN;
2211
2212         /* Configure a ucast MAC */
2213         rc = bnx2x_set_eth_mac(bp, true);
2214         if (rc) {
2215                 BNX2X_ERR("Setting Ethernet MAC failed\n");
2216                 LOAD_ERROR_EXIT(bp, load_error4);
2217         }
2218
2219         if (bp->pending_max) {
2220                 bnx2x_update_max_mf_config(bp, bp->pending_max);
2221                 bp->pending_max = 0;
2222         }
2223
2224         if (bp->port.pmf)
2225                 bnx2x_initial_phy_init(bp, load_mode);
2226
2227         /* Start fast path */
2228
2229         /* Initialize Rx filter. */
2230         netif_addr_lock_bh(bp->dev);
2231         bnx2x_set_rx_mode(bp->dev);
2232         netif_addr_unlock_bh(bp->dev);
2233
2234         /* Start the Tx */
2235         switch (load_mode) {
2236         case LOAD_NORMAL:
2237                 /* Tx queue should be only reenabled */
2238                 netif_tx_wake_all_queues(bp->dev);
2239                 break;
2240
2241         case LOAD_OPEN:
2242                 netif_tx_start_all_queues(bp->dev);
2243                 smp_mb__after_clear_bit();
2244                 break;
2245
2246         case LOAD_DIAG:
2247         case LOAD_LOOPBACK_EXT:
2248                 bp->state = BNX2X_STATE_DIAG;
2249                 break;
2250
2251         default:
2252                 break;
2253         }
2254
2255         if (bp->port.pmf)
2256                 bnx2x_update_drv_flags(bp, 1 << DRV_FLAGS_DCB_CONFIGURED, 0);
2257         else
2258                 bnx2x__link_status_update(bp);
2259
2260         /* start the timer */
2261         mod_timer(&bp->timer, jiffies + bp->current_interval);
2262
2263 #ifdef BCM_CNIC
2264         /* re-read iscsi info */
2265         bnx2x_get_iscsi_info(bp);
2266         bnx2x_setup_cnic_irq_info(bp);
2267         bnx2x_setup_cnic_info(bp);
2268         if (bp->state == BNX2X_STATE_OPEN)
2269                 bnx2x_cnic_notify(bp, CNIC_CTL_START_CMD);
2270 #endif
2271
2272         /* mark driver is loaded in shmem2 */
2273         if (SHMEM2_HAS(bp, drv_capabilities_flag)) {
2274                 u32 val;
2275                 val = SHMEM2_RD(bp, drv_capabilities_flag[BP_FW_MB_IDX(bp)]);
2276                 SHMEM2_WR(bp, drv_capabilities_flag[BP_FW_MB_IDX(bp)],
2277                           val | DRV_FLAGS_CAPABILITIES_LOADED_SUPPORTED |
2278                           DRV_FLAGS_CAPABILITIES_LOADED_L2);
2279         }
2280
2281         /* Wait for all pending SP commands to complete */
2282         if (!bnx2x_wait_sp_comp(bp, ~0x0UL)) {
2283                 BNX2X_ERR("Timeout waiting for SP elements to complete\n");
2284                 bnx2x_nic_unload(bp, UNLOAD_CLOSE);
2285                 return -EBUSY;
2286         }
2287
2288         /* If PMF - send ADMIN DCBX msg to MFW to initiate DCBX FSM */
2289         if (bp->port.pmf && (bp->state != BNX2X_STATE_DIAG))
2290                 bnx2x_dcbx_init(bp, false);
2291
2292         return 0;
2293
2294 #ifndef BNX2X_STOP_ON_ERROR
2295 load_error4:
2296 #ifdef BCM_CNIC
2297         /* Disable Timer scan */
2298         REG_WR(bp, TM_REG_EN_LINEAR0_TIMER + port*4, 0);
2299 #endif
2300 load_error3:
2301         bnx2x_int_disable_sync(bp, 1);
2302
2303         /* Clean queueable objects */
2304         bnx2x_squeeze_objects(bp);
2305
2306         /* Free SKBs, SGEs, TPA pool and driver internals */
2307         bnx2x_free_skbs(bp);
2308         for_each_rx_queue(bp, i)
2309                 bnx2x_free_rx_sge_range(bp, bp->fp + i, NUM_RX_SGE);
2310
2311         /* Release IRQs */
2312         bnx2x_free_irq(bp);
2313 load_error2:
2314         if (!BP_NOMCP(bp)) {
2315                 bnx2x_fw_command(bp, DRV_MSG_CODE_UNLOAD_REQ_WOL_MCP, 0);
2316                 bnx2x_fw_command(bp, DRV_MSG_CODE_UNLOAD_DONE, 0);
2317         }
2318
2319         bp->port.pmf = 0;
2320 load_error1:
2321         bnx2x_napi_disable(bp);
2322         /* clear pf_load status, as it was already set */
2323         bnx2x_clear_pf_load(bp);
2324 load_error0:
2325         bnx2x_free_mem(bp);
2326
2327         return rc;
2328 #endif /* ! BNX2X_STOP_ON_ERROR */
2329 }
2330
2331 /* must be called with rtnl_lock */
2332 int bnx2x_nic_unload(struct bnx2x *bp, int unload_mode)
2333 {
2334         int i;
2335         bool global = false;
2336
2337         /* mark driver is unloaded in shmem2 */
2338         if (SHMEM2_HAS(bp, drv_capabilities_flag)) {
2339                 u32 val;
2340                 val = SHMEM2_RD(bp, drv_capabilities_flag[BP_FW_MB_IDX(bp)]);
2341                 SHMEM2_WR(bp, drv_capabilities_flag[BP_FW_MB_IDX(bp)],
2342                           val & ~DRV_FLAGS_CAPABILITIES_LOADED_L2);
2343         }
2344
2345         if ((bp->state == BNX2X_STATE_CLOSED) ||
2346             (bp->state == BNX2X_STATE_ERROR)) {
2347                 /* We can get here if the driver has been unloaded
2348                  * during parity error recovery and is either waiting for a
2349                  * leader to complete or for other functions to unload and
2350                  * then ifdown has been issued. In this case we want to
2351                  * unload and let other functions to complete a recovery
2352                  * process.
2353                  */
2354                 bp->recovery_state = BNX2X_RECOVERY_DONE;
2355                 bp->is_leader = 0;
2356                 bnx2x_release_leader_lock(bp);
2357                 smp_mb();
2358
2359                 DP(NETIF_MSG_IFDOWN, "Releasing a leadership...\n");
2360                 BNX2X_ERR("Can't unload in closed or error state\n");
2361                 return -EINVAL;
2362         }
2363
2364         /*
2365          * It's important to set the bp->state to the value different from
2366          * BNX2X_STATE_OPEN and only then stop the Tx. Otherwise bnx2x_tx_int()
2367          * may restart the Tx from the NAPI context (see bnx2x_tx_int()).
2368          */
2369         bp->state = BNX2X_STATE_CLOSING_WAIT4_HALT;
2370         smp_mb();
2371
2372         /* Stop Tx */
2373         bnx2x_tx_disable(bp);
2374         netdev_reset_tc(bp->dev);
2375
2376 #ifdef BCM_CNIC
2377         bnx2x_cnic_notify(bp, CNIC_CTL_STOP_CMD);
2378 #endif
2379
2380         bp->rx_mode = BNX2X_RX_MODE_NONE;
2381
2382         del_timer_sync(&bp->timer);
2383
2384         /* Set ALWAYS_ALIVE bit in shmem */
2385         bp->fw_drv_pulse_wr_seq |= DRV_PULSE_ALWAYS_ALIVE;
2386
2387         bnx2x_drv_pulse(bp);
2388
2389         bnx2x_stats_handle(bp, STATS_EVENT_STOP);
2390         bnx2x_save_statistics(bp);
2391
2392         /* Cleanup the chip if needed */
2393         if (unload_mode != UNLOAD_RECOVERY)
2394                 bnx2x_chip_cleanup(bp, unload_mode);
2395         else {
2396                 /* Send the UNLOAD_REQUEST to the MCP */
2397                 bnx2x_send_unload_req(bp, unload_mode);
2398
2399                 /*
2400                  * Prevent transactions to host from the functions on the
2401                  * engine that doesn't reset global blocks in case of global
2402                  * attention once gloabl blocks are reset and gates are opened
2403                  * (the engine which leader will perform the recovery
2404                  * last).
2405                  */
2406                 if (!CHIP_IS_E1x(bp))
2407                         bnx2x_pf_disable(bp);
2408
2409                 /* Disable HW interrupts, NAPI */
2410                 bnx2x_netif_stop(bp, 1);
2411
2412                 /* Release IRQs */
2413                 bnx2x_free_irq(bp);
2414
2415                 /* Report UNLOAD_DONE to MCP */
2416                 bnx2x_send_unload_done(bp);
2417         }
2418
2419         /*
2420          * At this stage no more interrupts will arrive so we may safly clean
2421          * the queueable objects here in case they failed to get cleaned so far.
2422          */
2423         bnx2x_squeeze_objects(bp);
2424
2425         /* There should be no more pending SP commands at this stage */
2426         bp->sp_state = 0;
2427
2428         bp->port.pmf = 0;
2429
2430         /* Free SKBs, SGEs, TPA pool and driver internals */
2431         bnx2x_free_skbs(bp);
2432         for_each_rx_queue(bp, i)
2433                 bnx2x_free_rx_sge_range(bp, bp->fp + i, NUM_RX_SGE);
2434
2435         bnx2x_free_mem(bp);
2436
2437         bp->state = BNX2X_STATE_CLOSED;
2438
2439         /* Check if there are pending parity attentions. If there are - set
2440          * RECOVERY_IN_PROGRESS.
2441          */
2442         if (bnx2x_chk_parity_attn(bp, &global, false)) {
2443                 bnx2x_set_reset_in_progress(bp);
2444
2445                 /* Set RESET_IS_GLOBAL if needed */
2446                 if (global)
2447                         bnx2x_set_reset_global(bp);
2448         }
2449
2450
2451         /* The last driver must disable a "close the gate" if there is no
2452          * parity attention or "process kill" pending.
2453          */
2454         if (!bnx2x_clear_pf_load(bp) && bnx2x_reset_is_done(bp, BP_PATH(bp)))
2455                 bnx2x_disable_close_the_gate(bp);
2456
2457         return 0;
2458 }
2459
2460 int bnx2x_set_power_state(struct bnx2x *bp, pci_power_t state)
2461 {
2462         u16 pmcsr;
2463
2464         /* If there is no power capability, silently succeed */
2465         if (!bp->pm_cap) {
2466                 BNX2X_DEV_INFO("No power capability. Breaking.\n");
2467                 return 0;
2468         }
2469
2470         pci_read_config_word(bp->pdev, bp->pm_cap + PCI_PM_CTRL, &pmcsr);
2471
2472         switch (state) {
2473         case PCI_D0:
2474                 pci_write_config_word(bp->pdev, bp->pm_cap + PCI_PM_CTRL,
2475                                       ((pmcsr & ~PCI_PM_CTRL_STATE_MASK) |
2476                                        PCI_PM_CTRL_PME_STATUS));
2477
2478                 if (pmcsr & PCI_PM_CTRL_STATE_MASK)
2479                         /* delay required during transition out of D3hot */
2480                         msleep(20);
2481                 break;
2482
2483         case PCI_D3hot:
2484                 /* If there are other clients above don't
2485                    shut down the power */
2486                 if (atomic_read(&bp->pdev->enable_cnt) != 1)
2487                         return 0;
2488                 /* Don't shut down the power for emulation and FPGA */
2489                 if (CHIP_REV_IS_SLOW(bp))
2490                         return 0;
2491
2492                 pmcsr &= ~PCI_PM_CTRL_STATE_MASK;
2493                 pmcsr |= 3;
2494
2495                 if (bp->wol)
2496                         pmcsr |= PCI_PM_CTRL_PME_ENABLE;
2497
2498                 pci_write_config_word(bp->pdev, bp->pm_cap + PCI_PM_CTRL,
2499                                       pmcsr);
2500
2501                 /* No more memory access after this point until
2502                 * device is brought back to D0.
2503                 */
2504                 break;
2505
2506         default:
2507                 dev_err(&bp->pdev->dev, "Can't support state = %d\n", state);
2508                 return -EINVAL;
2509         }
2510         return 0;
2511 }
2512
2513 /*
2514  * net_device service functions
2515  */
2516 int bnx2x_poll(struct napi_struct *napi, int budget)
2517 {
2518         int work_done = 0;
2519         u8 cos;
2520         struct bnx2x_fastpath *fp = container_of(napi, struct bnx2x_fastpath,
2521                                                  napi);
2522         struct bnx2x *bp = fp->bp;
2523
2524         while (1) {
2525 #ifdef BNX2X_STOP_ON_ERROR
2526                 if (unlikely(bp->panic)) {
2527                         napi_complete(napi);
2528                         return 0;
2529                 }
2530 #endif
2531
2532                 for_each_cos_in_tx_queue(fp, cos)
2533                         if (bnx2x_tx_queue_has_work(fp->txdata_ptr[cos]))
2534                                 bnx2x_tx_int(bp, fp->txdata_ptr[cos]);
2535
2536
2537                 if (bnx2x_has_rx_work(fp)) {
2538                         work_done += bnx2x_rx_int(fp, budget - work_done);
2539
2540                         /* must not complete if we consumed full budget */
2541                         if (work_done >= budget)
2542                                 break;
2543                 }
2544
2545                 /* Fall out from the NAPI loop if needed */
2546                 if (!(bnx2x_has_rx_work(fp) || bnx2x_has_tx_work(fp))) {
2547 #ifdef BCM_CNIC
2548                         /* No need to update SB for FCoE L2 ring as long as
2549                          * it's connected to the default SB and the SB
2550                          * has been updated when NAPI was scheduled.
2551                          */
2552                         if (IS_FCOE_FP(fp)) {
2553                                 napi_complete(napi);
2554                                 break;
2555                         }
2556 #endif
2557
2558                         bnx2x_update_fpsb_idx(fp);
2559                         /* bnx2x_has_rx_work() reads the status block,
2560                          * thus we need to ensure that status block indices
2561                          * have been actually read (bnx2x_update_fpsb_idx)
2562                          * prior to this check (bnx2x_has_rx_work) so that
2563                          * we won't write the "newer" value of the status block
2564                          * to IGU (if there was a DMA right after
2565                          * bnx2x_has_rx_work and if there is no rmb, the memory
2566                          * reading (bnx2x_update_fpsb_idx) may be postponed
2567                          * to right before bnx2x_ack_sb). In this case there
2568                          * will never be another interrupt until there is
2569                          * another update of the status block, while there
2570                          * is still unhandled work.
2571                          */
2572                         rmb();
2573
2574                         if (!(bnx2x_has_rx_work(fp) || bnx2x_has_tx_work(fp))) {
2575                                 napi_complete(napi);
2576                                 /* Re-enable interrupts */
2577                                 DP(NETIF_MSG_RX_STATUS,
2578                                    "Update index to %d\n", fp->fp_hc_idx);
2579                                 bnx2x_ack_sb(bp, fp->igu_sb_id, USTORM_ID,
2580                                              le16_to_cpu(fp->fp_hc_idx),
2581                                              IGU_INT_ENABLE, 1);
2582                                 break;
2583                         }
2584                 }
2585         }
2586
2587         return work_done;
2588 }
2589
2590 /* we split the first BD into headers and data BDs
2591  * to ease the pain of our fellow microcode engineers
2592  * we use one mapping for both BDs
2593  */
2594 static noinline u16 bnx2x_tx_split(struct bnx2x *bp,
2595                                    struct bnx2x_fp_txdata *txdata,
2596                                    struct sw_tx_bd *tx_buf,
2597                                    struct eth_tx_start_bd **tx_bd, u16 hlen,
2598                                    u16 bd_prod, int nbd)
2599 {
2600         struct eth_tx_start_bd *h_tx_bd = *tx_bd;
2601         struct eth_tx_bd *d_tx_bd;
2602         dma_addr_t mapping;
2603         int old_len = le16_to_cpu(h_tx_bd->nbytes);
2604
2605         /* first fix first BD */
2606         h_tx_bd->nbd = cpu_to_le16(nbd);
2607         h_tx_bd->nbytes = cpu_to_le16(hlen);
2608
2609         DP(NETIF_MSG_TX_QUEUED, "TSO split header size is %d (%x:%x) nbd %d\n",
2610            h_tx_bd->nbytes, h_tx_bd->addr_hi, h_tx_bd->addr_lo, h_tx_bd->nbd);
2611
2612         /* now get a new data BD
2613          * (after the pbd) and fill it */
2614         bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
2615         d_tx_bd = &txdata->tx_desc_ring[bd_prod].reg_bd;
2616
2617         mapping = HILO_U64(le32_to_cpu(h_tx_bd->addr_hi),
2618                            le32_to_cpu(h_tx_bd->addr_lo)) + hlen;
2619
2620         d_tx_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
2621         d_tx_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
2622         d_tx_bd->nbytes = cpu_to_le16(old_len - hlen);
2623
2624         /* this marks the BD as one that has no individual mapping */
2625         tx_buf->flags |= BNX2X_TSO_SPLIT_BD;
2626
2627         DP(NETIF_MSG_TX_QUEUED,
2628            "TSO split data size is %d (%x:%x)\n",
2629            d_tx_bd->nbytes, d_tx_bd->addr_hi, d_tx_bd->addr_lo);
2630
2631         /* update tx_bd */
2632         *tx_bd = (struct eth_tx_start_bd *)d_tx_bd;
2633
2634         return bd_prod;
2635 }
2636
2637 static inline u16 bnx2x_csum_fix(unsigned char *t_header, u16 csum, s8 fix)
2638 {
2639         if (fix > 0)
2640                 csum = (u16) ~csum_fold(csum_sub(csum,
2641                                 csum_partial(t_header - fix, fix, 0)));
2642
2643         else if (fix < 0)
2644                 csum = (u16) ~csum_fold(csum_add(csum,
2645                                 csum_partial(t_header, -fix, 0)));
2646
2647         return swab16(csum);
2648 }
2649
2650 static inline u32 bnx2x_xmit_type(struct bnx2x *bp, struct sk_buff *skb)
2651 {
2652         u32 rc;
2653
2654         if (skb->ip_summed != CHECKSUM_PARTIAL)
2655                 rc = XMIT_PLAIN;
2656
2657         else {
2658                 if (vlan_get_protocol(skb) == htons(ETH_P_IPV6)) {
2659                         rc = XMIT_CSUM_V6;
2660                         if (ipv6_hdr(skb)->nexthdr == IPPROTO_TCP)
2661                                 rc |= XMIT_CSUM_TCP;
2662
2663                 } else {
2664                         rc = XMIT_CSUM_V4;
2665                         if (ip_hdr(skb)->protocol == IPPROTO_TCP)
2666                                 rc |= XMIT_CSUM_TCP;
2667                 }
2668         }
2669
2670         if (skb_is_gso_v6(skb))
2671                 rc |= XMIT_GSO_V6 | XMIT_CSUM_TCP | XMIT_CSUM_V6;
2672         else if (skb_is_gso(skb))
2673                 rc |= XMIT_GSO_V4 | XMIT_CSUM_V4 | XMIT_CSUM_TCP;
2674
2675         return rc;
2676 }
2677
2678 #if (MAX_SKB_FRAGS >= MAX_FETCH_BD - 3)
2679 /* check if packet requires linearization (packet is too fragmented)
2680    no need to check fragmentation if page size > 8K (there will be no
2681    violation to FW restrictions) */
2682 static int bnx2x_pkt_req_lin(struct bnx2x *bp, struct sk_buff *skb,
2683                              u32 xmit_type)
2684 {
2685         int to_copy = 0;
2686         int hlen = 0;
2687         int first_bd_sz = 0;
2688
2689         /* 3 = 1 (for linear data BD) + 2 (for PBD and last BD) */
2690         if (skb_shinfo(skb)->nr_frags >= (MAX_FETCH_BD - 3)) {
2691
2692                 if (xmit_type & XMIT_GSO) {
2693                         unsigned short lso_mss = skb_shinfo(skb)->gso_size;
2694                         /* Check if LSO packet needs to be copied:
2695                            3 = 1 (for headers BD) + 2 (for PBD and last BD) */
2696                         int wnd_size = MAX_FETCH_BD - 3;
2697                         /* Number of windows to check */
2698                         int num_wnds = skb_shinfo(skb)->nr_frags - wnd_size;
2699                         int wnd_idx = 0;
2700                         int frag_idx = 0;
2701                         u32 wnd_sum = 0;
2702
2703                         /* Headers length */
2704                         hlen = (int)(skb_transport_header(skb) - skb->data) +
2705                                 tcp_hdrlen(skb);
2706
2707                         /* Amount of data (w/o headers) on linear part of SKB*/
2708                         first_bd_sz = skb_headlen(skb) - hlen;
2709
2710                         wnd_sum  = first_bd_sz;
2711
2712                         /* Calculate the first sum - it's special */
2713                         for (frag_idx = 0; frag_idx < wnd_size - 1; frag_idx++)
2714                                 wnd_sum +=
2715                                         skb_frag_size(&skb_shinfo(skb)->frags[frag_idx]);
2716
2717                         /* If there was data on linear skb data - check it */
2718                         if (first_bd_sz > 0) {
2719                                 if (unlikely(wnd_sum < lso_mss)) {
2720                                         to_copy = 1;
2721                                         goto exit_lbl;
2722                                 }
2723
2724                                 wnd_sum -= first_bd_sz;
2725                         }
2726
2727                         /* Others are easier: run through the frag list and
2728                            check all windows */
2729                         for (wnd_idx = 0; wnd_idx <= num_wnds; wnd_idx++) {
2730                                 wnd_sum +=
2731                           skb_frag_size(&skb_shinfo(skb)->frags[wnd_idx + wnd_size - 1]);
2732
2733                                 if (unlikely(wnd_sum < lso_mss)) {
2734                                         to_copy = 1;
2735                                         break;
2736                                 }
2737                                 wnd_sum -=
2738                                         skb_frag_size(&skb_shinfo(skb)->frags[wnd_idx]);
2739                         }
2740                 } else {
2741                         /* in non-LSO too fragmented packet should always
2742                            be linearized */
2743                         to_copy = 1;
2744                 }
2745         }
2746
2747 exit_lbl:
2748         if (unlikely(to_copy))
2749                 DP(NETIF_MSG_TX_QUEUED,
2750                    "Linearization IS REQUIRED for %s packet. num_frags %d  hlen %d  first_bd_sz %d\n",
2751                    (xmit_type & XMIT_GSO) ? "LSO" : "non-LSO",
2752                    skb_shinfo(skb)->nr_frags, hlen, first_bd_sz);
2753
2754         return to_copy;
2755 }
2756 #endif
2757
2758 static inline void bnx2x_set_pbd_gso_e2(struct sk_buff *skb, u32 *parsing_data,
2759                                         u32 xmit_type)
2760 {
2761         *parsing_data |= (skb_shinfo(skb)->gso_size <<
2762                               ETH_TX_PARSE_BD_E2_LSO_MSS_SHIFT) &
2763                               ETH_TX_PARSE_BD_E2_LSO_MSS;
2764         if ((xmit_type & XMIT_GSO_V6) &&
2765             (ipv6_hdr(skb)->nexthdr == NEXTHDR_IPV6))
2766                 *parsing_data |= ETH_TX_PARSE_BD_E2_IPV6_WITH_EXT_HDR;
2767 }
2768
2769 /**
2770  * bnx2x_set_pbd_gso - update PBD in GSO case.
2771  *
2772  * @skb:        packet skb
2773  * @pbd:        parse BD
2774  * @xmit_type:  xmit flags
2775  */
2776 static inline void bnx2x_set_pbd_gso(struct sk_buff *skb,
2777                                      struct eth_tx_parse_bd_e1x *pbd,
2778                                      u32 xmit_type)
2779 {
2780         pbd->lso_mss = cpu_to_le16(skb_shinfo(skb)->gso_size);
2781         pbd->tcp_send_seq = swab32(tcp_hdr(skb)->seq);
2782         pbd->tcp_flags = pbd_tcp_flags(skb);
2783
2784         if (xmit_type & XMIT_GSO_V4) {
2785                 pbd->ip_id = swab16(ip_hdr(skb)->id);
2786                 pbd->tcp_pseudo_csum =
2787                         swab16(~csum_tcpudp_magic(ip_hdr(skb)->saddr,
2788                                                   ip_hdr(skb)->daddr,
2789                                                   0, IPPROTO_TCP, 0));
2790
2791         } else
2792                 pbd->tcp_pseudo_csum =
2793                         swab16(~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
2794                                                 &ipv6_hdr(skb)->daddr,
2795                                                 0, IPPROTO_TCP, 0));
2796
2797         pbd->global_data |= ETH_TX_PARSE_BD_E1X_PSEUDO_CS_WITHOUT_LEN;
2798 }
2799
2800 /**
2801  * bnx2x_set_pbd_csum_e2 - update PBD with checksum and return header length
2802  *
2803  * @bp:                 driver handle
2804  * @skb:                packet skb
2805  * @parsing_data:       data to be updated
2806  * @xmit_type:          xmit flags
2807  *
2808  * 57712 related
2809  */
2810 static inline  u8 bnx2x_set_pbd_csum_e2(struct bnx2x *bp, struct sk_buff *skb,
2811         u32 *parsing_data, u32 xmit_type)
2812 {
2813         *parsing_data |=
2814                         ((((u8 *)skb_transport_header(skb) - skb->data) >> 1) <<
2815                         ETH_TX_PARSE_BD_E2_TCP_HDR_START_OFFSET_W_SHIFT) &
2816                         ETH_TX_PARSE_BD_E2_TCP_HDR_START_OFFSET_W;
2817
2818         if (xmit_type & XMIT_CSUM_TCP) {
2819                 *parsing_data |= ((tcp_hdrlen(skb) / 4) <<
2820                         ETH_TX_PARSE_BD_E2_TCP_HDR_LENGTH_DW_SHIFT) &
2821                         ETH_TX_PARSE_BD_E2_TCP_HDR_LENGTH_DW;
2822
2823                 return skb_transport_header(skb) + tcp_hdrlen(skb) - skb->data;
2824         } else
2825                 /* We support checksum offload for TCP and UDP only.
2826                  * No need to pass the UDP header length - it's a constant.
2827                  */
2828                 return skb_transport_header(skb) +
2829                                 sizeof(struct udphdr) - skb->data;
2830 }
2831
2832 static inline void bnx2x_set_sbd_csum(struct bnx2x *bp, struct sk_buff *skb,
2833         struct eth_tx_start_bd *tx_start_bd, u32 xmit_type)
2834 {
2835         tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_L4_CSUM;
2836
2837         if (xmit_type & XMIT_CSUM_V4)
2838                 tx_start_bd->bd_flags.as_bitfield |=
2839                                         ETH_TX_BD_FLAGS_IP_CSUM;
2840         else
2841                 tx_start_bd->bd_flags.as_bitfield |=
2842                                         ETH_TX_BD_FLAGS_IPV6;
2843
2844         if (!(xmit_type & XMIT_CSUM_TCP))
2845                 tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_IS_UDP;
2846 }
2847
2848 /**
2849  * bnx2x_set_pbd_csum - update PBD with checksum and return header length
2850  *
2851  * @bp:         driver handle
2852  * @skb:        packet skb
2853  * @pbd:        parse BD to be updated
2854  * @xmit_type:  xmit flags
2855  */
2856 static inline u8 bnx2x_set_pbd_csum(struct bnx2x *bp, struct sk_buff *skb,
2857         struct eth_tx_parse_bd_e1x *pbd,
2858         u32 xmit_type)
2859 {
2860         u8 hlen = (skb_network_header(skb) - skb->data) >> 1;
2861
2862         /* for now NS flag is not used in Linux */
2863         pbd->global_data =
2864                 (hlen | ((skb->protocol == cpu_to_be16(ETH_P_8021Q)) <<
2865                          ETH_TX_PARSE_BD_E1X_LLC_SNAP_EN_SHIFT));
2866
2867         pbd->ip_hlen_w = (skb_transport_header(skb) -
2868                         skb_network_header(skb)) >> 1;
2869
2870         hlen += pbd->ip_hlen_w;
2871
2872         /* We support checksum offload for TCP and UDP only */
2873         if (xmit_type & XMIT_CSUM_TCP)
2874                 hlen += tcp_hdrlen(skb) / 2;
2875         else
2876                 hlen += sizeof(struct udphdr) / 2;
2877
2878         pbd->total_hlen_w = cpu_to_le16(hlen);
2879         hlen = hlen*2;
2880
2881         if (xmit_type & XMIT_CSUM_TCP) {
2882                 pbd->tcp_pseudo_csum = swab16(tcp_hdr(skb)->check);
2883
2884         } else {
2885                 s8 fix = SKB_CS_OFF(skb); /* signed! */
2886
2887                 DP(NETIF_MSG_TX_QUEUED,
2888                    "hlen %d  fix %d  csum before fix %x\n",
2889                    le16_to_cpu(pbd->total_hlen_w), fix, SKB_CS(skb));
2890
2891                 /* HW bug: fixup the CSUM */
2892                 pbd->tcp_pseudo_csum =
2893                         bnx2x_csum_fix(skb_transport_header(skb),
2894                                        SKB_CS(skb), fix);
2895
2896                 DP(NETIF_MSG_TX_QUEUED, "csum after fix %x\n",
2897                    pbd->tcp_pseudo_csum);
2898         }
2899
2900         return hlen;
2901 }
2902
2903 /* called with netif_tx_lock
2904  * bnx2x_tx_int() runs without netif_tx_lock unless it needs to call
2905  * netif_wake_queue()
2906  */
2907 netdev_tx_t bnx2x_start_xmit(struct sk_buff *skb, struct net_device *dev)
2908 {
2909         struct bnx2x *bp = netdev_priv(dev);
2910
2911         struct netdev_queue *txq;
2912         struct bnx2x_fp_txdata *txdata;
2913         struct sw_tx_bd *tx_buf;
2914         struct eth_tx_start_bd *tx_start_bd, *first_bd;
2915         struct eth_tx_bd *tx_data_bd, *total_pkt_bd = NULL;
2916         struct eth_tx_parse_bd_e1x *pbd_e1x = NULL;
2917         struct eth_tx_parse_bd_e2 *pbd_e2 = NULL;
2918         u32 pbd_e2_parsing_data = 0;
2919         u16 pkt_prod, bd_prod;
2920         int nbd, txq_index;
2921         dma_addr_t mapping;
2922         u32 xmit_type = bnx2x_xmit_type(bp, skb);
2923         int i;
2924         u8 hlen = 0;
2925         __le16 pkt_size = 0;
2926         struct ethhdr *eth;
2927         u8 mac_type = UNICAST_ADDRESS;
2928
2929 #ifdef BNX2X_STOP_ON_ERROR
2930         if (unlikely(bp->panic))
2931                 return NETDEV_TX_BUSY;
2932 #endif
2933
2934         txq_index = skb_get_queue_mapping(skb);
2935         txq = netdev_get_tx_queue(dev, txq_index);
2936
2937         BUG_ON(txq_index >= MAX_ETH_TXQ_IDX(bp) + FCOE_PRESENT);
2938
2939         txdata = &bp->bnx2x_txq[txq_index];
2940
2941         /* enable this debug print to view the transmission queue being used
2942         DP(NETIF_MSG_TX_QUEUED, "indices: txq %d, fp %d, txdata %d\n",
2943            txq_index, fp_index, txdata_index); */
2944
2945         /* enable this debug print to view the tranmission details
2946         DP(NETIF_MSG_TX_QUEUED,
2947            "transmitting packet cid %d fp index %d txdata_index %d tx_data ptr %p fp pointer %p\n",
2948            txdata->cid, fp_index, txdata_index, txdata, fp); */
2949
2950         if (unlikely(bnx2x_tx_avail(bp, txdata) <
2951                         skb_shinfo(skb)->nr_frags +
2952                         BDS_PER_TX_PKT +
2953                         NEXT_CNT_PER_TX_PKT(MAX_BDS_PER_TX_PKT))) {
2954                 bnx2x_fp_qstats(bp, txdata->parent_fp)->driver_xoff++;
2955                 netif_tx_stop_queue(txq);
2956                 BNX2X_ERR("BUG! Tx ring full when queue awake!\n");
2957                 return NETDEV_TX_BUSY;
2958         }
2959
2960         DP(NETIF_MSG_TX_QUEUED,
2961            "queue[%d]: SKB: summed %x  protocol %x protocol(%x,%x) gso type %x  xmit_type %x\n",
2962            txq_index, skb->ip_summed, skb->protocol, ipv6_hdr(skb)->nexthdr,
2963            ip_hdr(skb)->protocol, skb_shinfo(skb)->gso_type, xmit_type);
2964
2965         eth = (struct ethhdr *)skb->data;
2966
2967         /* set flag according to packet type (UNICAST_ADDRESS is default)*/
2968         if (unlikely(is_multicast_ether_addr(eth->h_dest))) {
2969                 if (is_broadcast_ether_addr(eth->h_dest))
2970                         mac_type = BROADCAST_ADDRESS;
2971                 else
2972                         mac_type = MULTICAST_ADDRESS;
2973         }
2974
2975 #if (MAX_SKB_FRAGS >= MAX_FETCH_BD - 3)
2976         /* First, check if we need to linearize the skb (due to FW
2977            restrictions). No need to check fragmentation if page size > 8K
2978            (there will be no violation to FW restrictions) */
2979         if (bnx2x_pkt_req_lin(bp, skb, xmit_type)) {
2980                 /* Statistics of linearization */
2981                 bp->lin_cnt++;
2982                 if (skb_linearize(skb) != 0) {
2983                         DP(NETIF_MSG_TX_QUEUED,
2984                            "SKB linearization failed - silently dropping this SKB\n");
2985                         dev_kfree_skb_any(skb);
2986                         return NETDEV_TX_OK;
2987                 }
2988         }
2989 #endif
2990         /* Map skb linear data for DMA */
2991         mapping = dma_map_single(&bp->pdev->dev, skb->data,
2992                                  skb_headlen(skb), DMA_TO_DEVICE);
2993         if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) {
2994                 DP(NETIF_MSG_TX_QUEUED,
2995                    "SKB mapping failed - silently dropping this SKB\n");
2996                 dev_kfree_skb_any(skb);
2997                 return NETDEV_TX_OK;
2998         }
2999         /*
3000         Please read carefully. First we use one BD which we mark as start,
3001         then we have a parsing info BD (used for TSO or xsum),
3002         and only then we have the rest of the TSO BDs.
3003         (don't forget to mark the last one as last,
3004         and to unmap only AFTER you write to the BD ...)
3005         And above all, all pdb sizes are in words - NOT DWORDS!
3006         */
3007
3008         /* get current pkt produced now - advance it just before sending packet
3009          * since mapping of pages may fail and cause packet to be dropped
3010          */
3011         pkt_prod = txdata->tx_pkt_prod;
3012         bd_prod = TX_BD(txdata->tx_bd_prod);
3013
3014         /* get a tx_buf and first BD
3015          * tx_start_bd may be changed during SPLIT,
3016          * but first_bd will always stay first
3017          */
3018         tx_buf = &txdata->tx_buf_ring[TX_BD(pkt_prod)];
3019         tx_start_bd = &txdata->tx_desc_ring[bd_prod].start_bd;
3020         first_bd = tx_start_bd;
3021
3022         tx_start_bd->bd_flags.as_bitfield = ETH_TX_BD_FLAGS_START_BD;
3023         SET_FLAG(tx_start_bd->general_data, ETH_TX_START_BD_ETH_ADDR_TYPE,
3024                  mac_type);
3025
3026         /* header nbd */
3027         SET_FLAG(tx_start_bd->general_data, ETH_TX_START_BD_HDR_NBDS, 1);
3028
3029         /* remember the first BD of the packet */
3030         tx_buf->first_bd = txdata->tx_bd_prod;
3031         tx_buf->skb = skb;
3032         tx_buf->flags = 0;
3033
3034         DP(NETIF_MSG_TX_QUEUED,
3035            "sending pkt %u @%p  next_idx %u  bd %u @%p\n",
3036            pkt_prod, tx_buf, txdata->tx_pkt_prod, bd_prod, tx_start_bd);
3037
3038         if (vlan_tx_tag_present(skb)) {
3039                 tx_start_bd->vlan_or_ethertype =
3040                     cpu_to_le16(vlan_tx_tag_get(skb));
3041                 tx_start_bd->bd_flags.as_bitfield |=
3042                     (X_ETH_OUTBAND_VLAN << ETH_TX_BD_FLAGS_VLAN_MODE_SHIFT);
3043         } else
3044                 tx_start_bd->vlan_or_ethertype = cpu_to_le16(pkt_prod);
3045
3046         /* turn on parsing and get a BD */
3047         bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
3048
3049         if (xmit_type & XMIT_CSUM)
3050                 bnx2x_set_sbd_csum(bp, skb, tx_start_bd, xmit_type);
3051
3052         if (!CHIP_IS_E1x(bp)) {
3053                 pbd_e2 = &txdata->tx_desc_ring[bd_prod].parse_bd_e2;
3054                 memset(pbd_e2, 0, sizeof(struct eth_tx_parse_bd_e2));
3055                 /* Set PBD in checksum offload case */
3056                 if (xmit_type & XMIT_CSUM)
3057                         hlen = bnx2x_set_pbd_csum_e2(bp, skb,
3058                                                      &pbd_e2_parsing_data,
3059                                                      xmit_type);
3060                 if (IS_MF_SI(bp)) {
3061                         /*
3062                          * fill in the MAC addresses in the PBD - for local
3063                          * switching
3064                          */
3065                         bnx2x_set_fw_mac_addr(&pbd_e2->src_mac_addr_hi,
3066                                               &pbd_e2->src_mac_addr_mid,
3067                                               &pbd_e2->src_mac_addr_lo,
3068                                               eth->h_source);
3069                         bnx2x_set_fw_mac_addr(&pbd_e2->dst_mac_addr_hi,
3070                                               &pbd_e2->dst_mac_addr_mid,
3071                                               &pbd_e2->dst_mac_addr_lo,
3072                                               eth->h_dest);
3073                 }
3074         } else {
3075                 pbd_e1x = &txdata->tx_desc_ring[bd_prod].parse_bd_e1x;
3076                 memset(pbd_e1x, 0, sizeof(struct eth_tx_parse_bd_e1x));
3077                 /* Set PBD in checksum offload case */
3078                 if (xmit_type & XMIT_CSUM)
3079                         hlen = bnx2x_set_pbd_csum(bp, skb, pbd_e1x, xmit_type);
3080
3081         }
3082
3083         /* Setup the data pointer of the first BD of the packet */
3084         tx_start_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
3085         tx_start_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
3086         nbd = 2; /* start_bd + pbd + frags (updated when pages are mapped) */
3087         tx_start_bd->nbytes = cpu_to_le16(skb_headlen(skb));
3088         pkt_size = tx_start_bd->nbytes;
3089
3090         DP(NETIF_MSG_TX_QUEUED,
3091            "first bd @%p  addr (%x:%x)  nbd %d  nbytes %d  flags %x  vlan %x\n",
3092            tx_start_bd, tx_start_bd->addr_hi, tx_start_bd->addr_lo,
3093            le16_to_cpu(tx_start_bd->nbd), le16_to_cpu(tx_start_bd->nbytes),
3094            tx_start_bd->bd_flags.as_bitfield,
3095            le16_to_cpu(tx_start_bd->vlan_or_ethertype));
3096
3097         if (xmit_type & XMIT_GSO) {
3098
3099                 DP(NETIF_MSG_TX_QUEUED,
3100                    "TSO packet len %d  hlen %d  total len %d  tso size %d\n",
3101                    skb->len, hlen, skb_headlen(skb),
3102                    skb_shinfo(skb)->gso_size);
3103
3104                 tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_SW_LSO;
3105
3106                 if (unlikely(skb_headlen(skb) > hlen))
3107                         bd_prod = bnx2x_tx_split(bp, txdata, tx_buf,
3108                                                  &tx_start_bd, hlen,
3109                                                  bd_prod, ++nbd);
3110                 if (!CHIP_IS_E1x(bp))
3111                         bnx2x_set_pbd_gso_e2(skb, &pbd_e2_parsing_data,
3112                                              xmit_type);
3113                 else
3114                         bnx2x_set_pbd_gso(skb, pbd_e1x, xmit_type);
3115         }
3116
3117         /* Set the PBD's parsing_data field if not zero
3118          * (for the chips newer than 57711).
3119          */
3120         if (pbd_e2_parsing_data)
3121                 pbd_e2->parsing_data = cpu_to_le32(pbd_e2_parsing_data);
3122
3123         tx_data_bd = (struct eth_tx_bd *)tx_start_bd;
3124
3125         /* Handle fragmented skb */
3126         for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
3127                 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
3128
3129                 mapping = skb_frag_dma_map(&bp->pdev->dev, frag, 0,
3130                                            skb_frag_size(frag), DMA_TO_DEVICE);
3131                 if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) {
3132                         unsigned int pkts_compl = 0, bytes_compl = 0;
3133
3134                         DP(NETIF_MSG_TX_QUEUED,
3135                            "Unable to map page - dropping packet...\n");
3136
3137                         /* we need unmap all buffers already mapped
3138                          * for this SKB;
3139                          * first_bd->nbd need to be properly updated
3140                          * before call to bnx2x_free_tx_pkt
3141                          */
3142                         first_bd->nbd = cpu_to_le16(nbd);
3143                         bnx2x_free_tx_pkt(bp, txdata,
3144                                           TX_BD(txdata->tx_pkt_prod),
3145                                           &pkts_compl, &bytes_compl);
3146                         return NETDEV_TX_OK;
3147                 }
3148
3149                 bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
3150                 tx_data_bd = &txdata->tx_desc_ring[bd_prod].reg_bd;
3151                 if (total_pkt_bd == NULL)
3152                         total_pkt_bd = &txdata->tx_desc_ring[bd_prod].reg_bd;
3153
3154                 tx_data_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
3155                 tx_data_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
3156                 tx_data_bd->nbytes = cpu_to_le16(skb_frag_size(frag));
3157                 le16_add_cpu(&pkt_size, skb_frag_size(frag));
3158                 nbd++;
3159
3160                 DP(NETIF_MSG_TX_QUEUED,
3161                    "frag %d  bd @%p  addr (%x:%x)  nbytes %d\n",
3162                    i, tx_data_bd, tx_data_bd->addr_hi, tx_data_bd->addr_lo,
3163                    le16_to_cpu(tx_data_bd->nbytes));
3164         }
3165
3166         DP(NETIF_MSG_TX_QUEUED, "last bd @%p\n", tx_data_bd);
3167
3168         /* update with actual num BDs */
3169         first_bd->nbd = cpu_to_le16(nbd);
3170
3171         bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
3172
3173         /* now send a tx doorbell, counting the next BD
3174          * if the packet contains or ends with it
3175          */
3176         if (TX_BD_POFF(bd_prod) < nbd)
3177                 nbd++;
3178
3179         /* total_pkt_bytes should be set on the first data BD if
3180          * it's not an LSO packet and there is more than one
3181          * data BD. In this case pkt_size is limited by an MTU value.
3182          * However we prefer to set it for an LSO packet (while we don't
3183          * have to) in order to save some CPU cycles in a none-LSO
3184          * case, when we much more care about them.
3185          */
3186         if (total_pkt_bd != NULL)
3187                 total_pkt_bd->total_pkt_bytes = pkt_size;
3188
3189         if (pbd_e1x)
3190                 DP(NETIF_MSG_TX_QUEUED,
3191                    "PBD (E1X) @%p  ip_data %x  ip_hlen %u  ip_id %u  lso_mss %u  tcp_flags %x  xsum %x  seq %u  hlen %u\n",
3192                    pbd_e1x, pbd_e1x->global_data, pbd_e1x->ip_hlen_w,
3193                    pbd_e1x->ip_id, pbd_e1x->lso_mss, pbd_e1x->tcp_flags,
3194                    pbd_e1x->tcp_pseudo_csum, pbd_e1x->tcp_send_seq,
3195                     le16_to_cpu(pbd_e1x->total_hlen_w));
3196         if (pbd_e2)
3197                 DP(NETIF_MSG_TX_QUEUED,
3198                    "PBD (E2) @%p  dst %x %x %x src %x %x %x parsing_data %x\n",
3199                    pbd_e2, pbd_e2->dst_mac_addr_hi, pbd_e2->dst_mac_addr_mid,
3200                    pbd_e2->dst_mac_addr_lo, pbd_e2->src_mac_addr_hi,
3201                    pbd_e2->src_mac_addr_mid, pbd_e2->src_mac_addr_lo,
3202                    pbd_e2->parsing_data);
3203         DP(NETIF_MSG_TX_QUEUED, "doorbell: nbd %d  bd %u\n", nbd, bd_prod);
3204
3205         netdev_tx_sent_queue(txq, skb->len);
3206
3207         skb_tx_timestamp(skb);
3208
3209         txdata->tx_pkt_prod++;
3210         /*
3211          * Make sure that the BD data is updated before updating the producer
3212          * since FW might read the BD right after the producer is updated.
3213          * This is only applicable for weak-ordered memory model archs such
3214          * as IA-64. The following barrier is also mandatory since FW will
3215          * assumes packets must have BDs.
3216          */
3217         wmb();
3218
3219         txdata->tx_db.data.prod += nbd;
3220         barrier();
3221
3222         DOORBELL(bp, txdata->cid, txdata->tx_db.raw);
3223
3224         mmiowb();
3225
3226         txdata->tx_bd_prod += nbd;
3227
3228         if (unlikely(bnx2x_tx_avail(bp, txdata) < MAX_DESC_PER_TX_PKT)) {
3229                 netif_tx_stop_queue(txq);
3230
3231                 /* paired memory barrier is in bnx2x_tx_int(), we have to keep
3232                  * ordering of set_bit() in netif_tx_stop_queue() and read of
3233                  * fp->bd_tx_cons */
3234                 smp_mb();
3235
3236                 bnx2x_fp_qstats(bp, txdata->parent_fp)->driver_xoff++;
3237                 if (bnx2x_tx_avail(bp, txdata) >= MAX_DESC_PER_TX_PKT)
3238                         netif_tx_wake_queue(txq);
3239         }
3240         txdata->tx_pkt++;
3241
3242         return NETDEV_TX_OK;
3243 }
3244
3245 /**
3246  * bnx2x_setup_tc - routine to configure net_device for multi tc
3247  *
3248  * @netdev: net device to configure
3249  * @tc: number of traffic classes to enable
3250  *
3251  * callback connected to the ndo_setup_tc function pointer
3252  */
3253 int bnx2x_setup_tc(struct net_device *dev, u8 num_tc)
3254 {
3255         int cos, prio, count, offset;
3256         struct bnx2x *bp = netdev_priv(dev);
3257
3258         /* setup tc must be called under rtnl lock */
3259         ASSERT_RTNL();
3260
3261         /* no traffic classes requested. aborting */
3262         if (!num_tc) {
3263                 netdev_reset_tc(dev);
3264                 return 0;
3265         }
3266
3267         /* requested to support too many traffic classes */
3268         if (num_tc > bp->max_cos) {
3269                 BNX2X_ERR("support for too many traffic classes requested: %d. max supported is %d\n",
3270                           num_tc, bp->max_cos);
3271                 return -EINVAL;
3272         }
3273
3274         /* declare amount of supported traffic classes */
3275         if (netdev_set_num_tc(dev, num_tc)) {
3276                 BNX2X_ERR("failed to declare %d traffic classes\n", num_tc);
3277                 return -EINVAL;
3278         }
3279
3280         /* configure priority to traffic class mapping */
3281         for (prio = 0; prio < BNX2X_MAX_PRIORITY; prio++) {
3282                 netdev_set_prio_tc_map(dev, prio, bp->prio_to_cos[prio]);
3283                 DP(BNX2X_MSG_SP | NETIF_MSG_IFUP,
3284                    "mapping priority %d to tc %d\n",
3285                    prio, bp->prio_to_cos[prio]);
3286         }
3287
3288
3289         /* Use this configuration to diffrentiate tc0 from other COSes
3290            This can be used for ets or pfc, and save the effort of setting
3291            up a multio class queue disc or negotiating DCBX with a switch
3292         netdev_set_prio_tc_map(dev, 0, 0);
3293         DP(BNX2X_MSG_SP, "mapping priority %d to tc %d\n", 0, 0);
3294         for (prio = 1; prio < 16; prio++) {
3295                 netdev_set_prio_tc_map(dev, prio, 1);
3296                 DP(BNX2X_MSG_SP, "mapping priority %d to tc %d\n", prio, 1);
3297         } */
3298
3299         /* configure traffic class to transmission queue mapping */
3300         for (cos = 0; cos < bp->max_cos; cos++) {
3301                 count = BNX2X_NUM_ETH_QUEUES(bp);
3302                 offset = cos * BNX2X_NUM_NON_CNIC_QUEUES(bp);
3303                 netdev_set_tc_queue(dev, cos, count, offset);
3304                 DP(BNX2X_MSG_SP | NETIF_MSG_IFUP,
3305                    "mapping tc %d to offset %d count %d\n",
3306                    cos, offset, count);
3307         }
3308
3309         return 0;
3310 }
3311
3312 /* called with rtnl_lock */
3313 int bnx2x_change_mac_addr(struct net_device *dev, void *p)
3314 {
3315         struct sockaddr *addr = p;
3316         struct bnx2x *bp = netdev_priv(dev);
3317         int rc = 0;
3318
3319         if (!bnx2x_is_valid_ether_addr(bp, addr->sa_data)) {
3320                 BNX2X_ERR("Requested MAC address is not valid\n");
3321                 return -EINVAL;
3322         }
3323
3324 #ifdef BCM_CNIC
3325         if ((IS_MF_STORAGE_SD(bp) || IS_MF_FCOE_AFEX(bp)) &&
3326             !is_zero_ether_addr(addr->sa_data)) {
3327                 BNX2X_ERR("Can't configure non-zero address on iSCSI or FCoE functions in MF-SD mode\n");
3328                 return -EINVAL;
3329         }
3330 #endif
3331
3332         if (netif_running(dev))  {
3333                 rc = bnx2x_set_eth_mac(bp, false);
3334                 if (rc)
3335                         return rc;
3336         }
3337
3338         dev->addr_assign_type &= ~NET_ADDR_RANDOM;
3339         memcpy(dev->dev_addr, addr->sa_data, dev->addr_len);
3340
3341         if (netif_running(dev))
3342                 rc = bnx2x_set_eth_mac(bp, true);
3343
3344         return rc;
3345 }
3346
3347 static void bnx2x_free_fp_mem_at(struct bnx2x *bp, int fp_index)
3348 {
3349         union host_hc_status_block *sb = &bnx2x_fp(bp, fp_index, status_blk);
3350         struct bnx2x_fastpath *fp = &bp->fp[fp_index];
3351         u8 cos;
3352
3353         /* Common */
3354 #ifdef BCM_CNIC
3355         if (IS_FCOE_IDX(fp_index)) {
3356                 memset(sb, 0, sizeof(union host_hc_status_block));
3357                 fp->status_blk_mapping = 0;
3358
3359         } else {
3360 #endif
3361                 /* status blocks */
3362                 if (!CHIP_IS_E1x(bp))
3363                         BNX2X_PCI_FREE(sb->e2_sb,
3364                                        bnx2x_fp(bp, fp_index,
3365                                                 status_blk_mapping),
3366                                        sizeof(struct host_hc_status_block_e2));
3367                 else
3368                         BNX2X_PCI_FREE(sb->e1x_sb,
3369                                        bnx2x_fp(bp, fp_index,
3370                                                 status_blk_mapping),
3371                                        sizeof(struct host_hc_status_block_e1x));
3372 #ifdef BCM_CNIC
3373         }
3374 #endif
3375         /* Rx */
3376         if (!skip_rx_queue(bp, fp_index)) {
3377                 bnx2x_free_rx_bds(fp);
3378
3379                 /* fastpath rx rings: rx_buf rx_desc rx_comp */
3380                 BNX2X_FREE(bnx2x_fp(bp, fp_index, rx_buf_ring));
3381                 BNX2X_PCI_FREE(bnx2x_fp(bp, fp_index, rx_desc_ring),
3382                                bnx2x_fp(bp, fp_index, rx_desc_mapping),
3383                                sizeof(struct eth_rx_bd) * NUM_RX_BD);
3384
3385                 BNX2X_PCI_FREE(bnx2x_fp(bp, fp_index, rx_comp_ring),
3386                                bnx2x_fp(bp, fp_index, rx_comp_mapping),
3387                                sizeof(struct eth_fast_path_rx_cqe) *
3388                                NUM_RCQ_BD);
3389
3390                 /* SGE ring */
3391                 BNX2X_FREE(bnx2x_fp(bp, fp_index, rx_page_ring));
3392                 BNX2X_PCI_FREE(bnx2x_fp(bp, fp_index, rx_sge_ring),
3393                                bnx2x_fp(bp, fp_index, rx_sge_mapping),
3394                                BCM_PAGE_SIZE * NUM_RX_SGE_PAGES);
3395         }
3396
3397         /* Tx */
3398         if (!skip_tx_queue(bp, fp_index)) {
3399                 /* fastpath tx rings: tx_buf tx_desc */
3400                 for_each_cos_in_tx_queue(fp, cos) {
3401                         struct bnx2x_fp_txdata *txdata = fp->txdata_ptr[cos];
3402
3403                         DP(NETIF_MSG_IFDOWN,
3404                            "freeing tx memory of fp %d cos %d cid %d\n",
3405                            fp_index, cos, txdata->cid);
3406
3407                         BNX2X_FREE(txdata->tx_buf_ring);
3408                         BNX2X_PCI_FREE(txdata->tx_desc_ring,
3409                                 txdata->tx_desc_mapping,
3410                                 sizeof(union eth_tx_bd_types) * NUM_TX_BD);
3411                 }
3412         }
3413         /* end of fastpath */
3414 }
3415
3416 void bnx2x_free_fp_mem(struct bnx2x *bp)
3417 {
3418         int i;
3419         for_each_queue(bp, i)
3420                 bnx2x_free_fp_mem_at(bp, i);
3421 }
3422
3423 static void set_sb_shortcuts(struct bnx2x *bp, int index)
3424 {
3425         union host_hc_status_block status_blk = bnx2x_fp(bp, index, status_blk);
3426         if (!CHIP_IS_E1x(bp)) {
3427                 bnx2x_fp(bp, index, sb_index_values) =
3428                         (__le16 *)status_blk.e2_sb->sb.index_values;
3429                 bnx2x_fp(bp, index, sb_running_index) =
3430                         (__le16 *)status_blk.e2_sb->sb.running_index;
3431         } else {
3432                 bnx2x_fp(bp, index, sb_index_values) =
3433                         (__le16 *)status_blk.e1x_sb->sb.index_values;
3434                 bnx2x_fp(bp, index, sb_running_index) =
3435                         (__le16 *)status_blk.e1x_sb->sb.running_index;
3436         }
3437 }
3438
3439 /* Returns the number of actually allocated BDs */
3440 static int bnx2x_alloc_rx_bds(struct bnx2x_fastpath *fp,
3441                               int rx_ring_size)
3442 {
3443         struct bnx2x *bp = fp->bp;
3444         u16 ring_prod, cqe_ring_prod;
3445         int i, failure_cnt = 0;
3446
3447         fp->rx_comp_cons = 0;
3448         cqe_ring_prod = ring_prod = 0;
3449
3450         /* This routine is called only during fo init so
3451          * fp->eth_q_stats.rx_skb_alloc_failed = 0
3452          */
3453         for (i = 0; i < rx_ring_size; i++) {
3454                 if (bnx2x_alloc_rx_data(bp, fp, ring_prod) < 0) {
3455                         failure_cnt++;
3456                         continue;
3457                 }
3458                 ring_prod = NEXT_RX_IDX(ring_prod);
3459                 cqe_ring_prod = NEXT_RCQ_IDX(cqe_ring_prod);
3460                 WARN_ON(ring_prod <= (i - failure_cnt));
3461         }
3462
3463         if (failure_cnt)
3464                 BNX2X_ERR("was only able to allocate %d rx skbs on queue[%d]\n",
3465                           i - failure_cnt, fp->index);
3466
3467         fp->rx_bd_prod = ring_prod;
3468         /* Limit the CQE producer by the CQE ring size */
3469         fp->rx_comp_prod = min_t(u16, NUM_RCQ_RINGS*RCQ_DESC_CNT,
3470                                cqe_ring_prod);
3471         fp->rx_pkt = fp->rx_calls = 0;
3472
3473         bnx2x_fp_stats(bp, fp)->eth_q_stats.rx_skb_alloc_failed += failure_cnt;
3474
3475         return i - failure_cnt;
3476 }
3477
3478 static void bnx2x_set_next_page_rx_cq(struct bnx2x_fastpath *fp)
3479 {
3480         int i;
3481
3482         for (i = 1; i <= NUM_RCQ_RINGS; i++) {
3483                 struct eth_rx_cqe_next_page *nextpg;
3484
3485                 nextpg = (struct eth_rx_cqe_next_page *)
3486                         &fp->rx_comp_ring[RCQ_DESC_CNT * i - 1];
3487                 nextpg->addr_hi =
3488                         cpu_to_le32(U64_HI(fp->rx_comp_mapping +
3489                                    BCM_PAGE_SIZE*(i % NUM_RCQ_RINGS)));
3490                 nextpg->addr_lo =
3491                         cpu_to_le32(U64_LO(fp->rx_comp_mapping +
3492                                    BCM_PAGE_SIZE*(i % NUM_RCQ_RINGS)));
3493         }
3494 }
3495
3496 static int bnx2x_alloc_fp_mem_at(struct bnx2x *bp, int index)
3497 {
3498         union host_hc_status_block *sb;
3499         struct bnx2x_fastpath *fp = &bp->fp[index];
3500         int ring_size = 0;
3501         u8 cos;
3502         int rx_ring_size = 0;
3503
3504 #ifdef BCM_CNIC
3505         if (!bp->rx_ring_size &&
3506             (IS_MF_STORAGE_SD(bp) || IS_MF_FCOE_AFEX(bp))) {
3507                 rx_ring_size = MIN_RX_SIZE_NONTPA;
3508                 bp->rx_ring_size = rx_ring_size;
3509         } else
3510 #endif
3511         if (!bp->rx_ring_size) {
3512                 u32 cfg = SHMEM_RD(bp,
3513                              dev_info.port_hw_config[BP_PORT(bp)].default_cfg);
3514
3515                 rx_ring_size = MAX_RX_AVAIL/BNX2X_NUM_RX_QUEUES(bp);
3516
3517                 /* Dercease ring size for 1G functions */
3518                 if ((cfg & PORT_HW_CFG_NET_SERDES_IF_MASK) ==
3519                     PORT_HW_CFG_NET_SERDES_IF_SGMII)
3520                         rx_ring_size /= 10;
3521
3522                 /* allocate at least number of buffers required by FW */
3523                 rx_ring_size = max_t(int, bp->disable_tpa ? MIN_RX_SIZE_NONTPA :
3524                                      MIN_RX_SIZE_TPA, rx_ring_size);
3525
3526                 bp->rx_ring_size = rx_ring_size;
3527         } else /* if rx_ring_size specified - use it */
3528                 rx_ring_size = bp->rx_ring_size;
3529
3530         /* Common */
3531         sb = &bnx2x_fp(bp, index, status_blk);
3532 #ifdef BCM_CNIC
3533         if (!IS_FCOE_IDX(index)) {
3534 #endif
3535                 /* status blocks */
3536                 if (!CHIP_IS_E1x(bp))
3537                         BNX2X_PCI_ALLOC(sb->e2_sb,
3538                                 &bnx2x_fp(bp, index, status_blk_mapping),
3539                                 sizeof(struct host_hc_status_block_e2));
3540                 else
3541                         BNX2X_PCI_ALLOC(sb->e1x_sb,
3542                                 &bnx2x_fp(bp, index, status_blk_mapping),
3543                             sizeof(struct host_hc_status_block_e1x));
3544 #ifdef BCM_CNIC
3545         }
3546 #endif
3547
3548         /* FCoE Queue uses Default SB and doesn't ACK the SB, thus no need to
3549          * set shortcuts for it.
3550          */
3551         if (!IS_FCOE_IDX(index))
3552                 set_sb_shortcuts(bp, index);
3553
3554         /* Tx */
3555         if (!skip_tx_queue(bp, index)) {
3556                 /* fastpath tx rings: tx_buf tx_desc */
3557                 for_each_cos_in_tx_queue(fp, cos) {
3558                         struct bnx2x_fp_txdata *txdata = fp->txdata_ptr[cos];
3559
3560                         DP(NETIF_MSG_IFUP,
3561                            "allocating tx memory of fp %d cos %d\n",
3562                            index, cos);
3563
3564                         BNX2X_ALLOC(txdata->tx_buf_ring,
3565                                 sizeof(struct sw_tx_bd) * NUM_TX_BD);
3566                         BNX2X_PCI_ALLOC(txdata->tx_desc_ring,
3567                                 &txdata->tx_desc_mapping,
3568                                 sizeof(union eth_tx_bd_types) * NUM_TX_BD);
3569                 }
3570         }
3571
3572         /* Rx */
3573         if (!skip_rx_queue(bp, index)) {
3574                 /* fastpath rx rings: rx_buf rx_desc rx_comp */
3575                 BNX2X_ALLOC(bnx2x_fp(bp, index, rx_buf_ring),
3576                                 sizeof(struct sw_rx_bd) * NUM_RX_BD);
3577                 BNX2X_PCI_ALLOC(bnx2x_fp(bp, index, rx_desc_ring),
3578                                 &bnx2x_fp(bp, index, rx_desc_mapping),
3579                                 sizeof(struct eth_rx_bd) * NUM_RX_BD);
3580
3581                 BNX2X_PCI_ALLOC(bnx2x_fp(bp, index, rx_comp_ring),
3582                                 &bnx2x_fp(bp, index, rx_comp_mapping),
3583                                 sizeof(struct eth_fast_path_rx_cqe) *
3584                                 NUM_RCQ_BD);
3585
3586                 /* SGE ring */
3587                 BNX2X_ALLOC(bnx2x_fp(bp, index, rx_page_ring),
3588                                 sizeof(struct sw_rx_page) * NUM_RX_SGE);
3589                 BNX2X_PCI_ALLOC(bnx2x_fp(bp, index, rx_sge_ring),
3590                                 &bnx2x_fp(bp, index, rx_sge_mapping),
3591                                 BCM_PAGE_SIZE * NUM_RX_SGE_PAGES);
3592                 /* RX BD ring */
3593                 bnx2x_set_next_page_rx_bd(fp);
3594
3595                 /* CQ ring */
3596                 bnx2x_set_next_page_rx_cq(fp);
3597
3598                 /* BDs */
3599                 ring_size = bnx2x_alloc_rx_bds(fp, rx_ring_size);
3600                 if (ring_size < rx_ring_size)
3601                         goto alloc_mem_err;
3602         }
3603
3604         return 0;
3605
3606 /* handles low memory cases */
3607 alloc_mem_err:
3608         BNX2X_ERR("Unable to allocate full memory for queue %d (size %d)\n",
3609                                                 index, ring_size);
3610         /* FW will drop all packets if queue is not big enough,
3611          * In these cases we disable the queue
3612          * Min size is different for OOO, TPA and non-TPA queues
3613          */
3614         if (ring_size < (fp->disable_tpa ?
3615                                 MIN_RX_SIZE_NONTPA : MIN_RX_SIZE_TPA)) {
3616                         /* release memory allocated for this queue */
3617                         bnx2x_free_fp_mem_at(bp, index);
3618                         return -ENOMEM;
3619         }
3620         return 0;
3621 }
3622
3623 int bnx2x_alloc_fp_mem(struct bnx2x *bp)
3624 {
3625         int i;
3626
3627         /**
3628          * 1. Allocate FP for leading - fatal if error
3629          * 2. {CNIC} Allocate FCoE FP - fatal if error
3630          * 3. {CNIC} Allocate OOO + FWD - disable OOO if error
3631          * 4. Allocate RSS - fix number of queues if error
3632          */
3633
3634         /* leading */
3635         if (bnx2x_alloc_fp_mem_at(bp, 0))
3636                 return -ENOMEM;
3637
3638 #ifdef BCM_CNIC
3639         if (!NO_FCOE(bp))
3640                 /* FCoE */
3641                 if (bnx2x_alloc_fp_mem_at(bp, FCOE_IDX(bp)))
3642                         /* we will fail load process instead of mark
3643                          * NO_FCOE_FLAG
3644                          */
3645                         return -ENOMEM;
3646 #endif
3647
3648         /* RSS */
3649         for_each_nondefault_eth_queue(bp, i)
3650                 if (bnx2x_alloc_fp_mem_at(bp, i))
3651                         break;
3652
3653         /* handle memory failures */
3654         if (i != BNX2X_NUM_ETH_QUEUES(bp)) {
3655                 int delta = BNX2X_NUM_ETH_QUEUES(bp) - i;
3656
3657                 WARN_ON(delta < 0);
3658 #ifdef BCM_CNIC
3659                 /**
3660                  * move non eth FPs next to last eth FP
3661                  * must be done in that order
3662                  * FCOE_IDX < FWD_IDX < OOO_IDX
3663                  */
3664
3665                 /* move FCoE fp even NO_FCOE_FLAG is on */
3666                 bnx2x_move_fp(bp, FCOE_IDX(bp), FCOE_IDX(bp) - delta);
3667 #endif
3668                 bp->num_queues -= delta;
3669                 BNX2X_ERR("Adjusted num of queues from %d to %d\n",
3670                           bp->num_queues + delta, bp->num_queues);
3671         }
3672
3673         return 0;
3674 }
3675
3676 void bnx2x_free_mem_bp(struct bnx2x *bp)
3677 {
3678         kfree(bp->fp->tpa_info);
3679         kfree(bp->fp);
3680         kfree(bp->sp_objs);
3681         kfree(bp->fp_stats);
3682         kfree(bp->bnx2x_txq);
3683         kfree(bp->msix_table);
3684         kfree(bp->ilt);
3685 }
3686
3687 int __devinit bnx2x_alloc_mem_bp(struct bnx2x *bp)
3688 {
3689         struct bnx2x_fastpath *fp;
3690         struct msix_entry *tbl;
3691         struct bnx2x_ilt *ilt;
3692         int msix_table_size = 0;
3693         int fp_array_size;
3694         int i;
3695
3696         /*
3697          * The biggest MSI-X table we might need is as a maximum number of fast
3698          * path IGU SBs plus default SB (for PF).
3699          */
3700         msix_table_size = bp->igu_sb_cnt + 1;
3701
3702         /* fp array: RSS plus CNIC related L2 queues */
3703         fp_array_size = BNX2X_MAX_RSS_COUNT(bp) + NON_ETH_CONTEXT_USE;
3704         BNX2X_DEV_INFO("fp_array_size %d", fp_array_size);
3705
3706         fp = kcalloc(fp_array_size, sizeof(*fp), GFP_KERNEL);
3707         if (!fp)
3708                 goto alloc_err;
3709         for (i = 0; i < fp_array_size; i++) {
3710                 fp[i].tpa_info =
3711                         kcalloc(ETH_MAX_AGGREGATION_QUEUES_E1H_E2,
3712                                 sizeof(struct bnx2x_agg_info), GFP_KERNEL);
3713                 if (!(fp[i].tpa_info))
3714                         goto alloc_err;
3715         }
3716
3717         bp->fp = fp;
3718
3719         /* allocate sp objs */
3720         bp->sp_objs = kcalloc(fp_array_size, sizeof(struct bnx2x_sp_objs),
3721                               GFP_KERNEL);
3722         if (!bp->sp_objs)
3723                 goto alloc_err;
3724
3725         /* allocate fp_stats */
3726         bp->fp_stats = kcalloc(fp_array_size, sizeof(struct bnx2x_fp_stats),
3727                                GFP_KERNEL);
3728         if (!bp->fp_stats)
3729                 goto alloc_err;
3730
3731         /* Allocate memory for the transmission queues array */
3732         bp->bnx2x_txq_size = BNX2X_MAX_RSS_COUNT(bp) * BNX2X_MULTI_TX_COS;
3733 #ifdef BCM_CNIC
3734         bp->bnx2x_txq_size++;
3735 #endif
3736         bp->bnx2x_txq = kcalloc(bp->bnx2x_txq_size,
3737                                 sizeof(struct bnx2x_fp_txdata), GFP_KERNEL);
3738         if (!bp->bnx2x_txq)
3739                 goto alloc_err;
3740
3741         /* msix table */
3742         tbl = kcalloc(msix_table_size, sizeof(*tbl), GFP_KERNEL);
3743         if (!tbl)
3744                 goto alloc_err;
3745         bp->msix_table = tbl;
3746
3747         /* ilt */
3748         ilt = kzalloc(sizeof(*ilt), GFP_KERNEL);
3749         if (!ilt)
3750                 goto alloc_err;
3751         bp->ilt = ilt;
3752
3753         return 0;
3754 alloc_err:
3755         bnx2x_free_mem_bp(bp);
3756         return -ENOMEM;
3757
3758 }
3759
3760 int bnx2x_reload_if_running(struct net_device *dev)
3761 {
3762         struct bnx2x *bp = netdev_priv(dev);
3763
3764         if (unlikely(!netif_running(dev)))
3765                 return 0;
3766
3767         bnx2x_nic_unload(bp, UNLOAD_NORMAL);
3768         return bnx2x_nic_load(bp, LOAD_NORMAL);
3769 }
3770
3771 int bnx2x_get_cur_phy_idx(struct bnx2x *bp)
3772 {
3773         u32 sel_phy_idx = 0;
3774         if (bp->link_params.num_phys <= 1)
3775                 return INT_PHY;
3776
3777         if (bp->link_vars.link_up) {
3778                 sel_phy_idx = EXT_PHY1;
3779                 /* In case link is SERDES, check if the EXT_PHY2 is the one */
3780                 if ((bp->link_vars.link_status & LINK_STATUS_SERDES_LINK) &&
3781                     (bp->link_params.phy[EXT_PHY2].supported & SUPPORTED_FIBRE))
3782                         sel_phy_idx = EXT_PHY2;
3783         } else {
3784
3785                 switch (bnx2x_phy_selection(&bp->link_params)) {
3786                 case PORT_HW_CFG_PHY_SELECTION_HARDWARE_DEFAULT:
3787                 case PORT_HW_CFG_PHY_SELECTION_FIRST_PHY:
3788                 case PORT_HW_CFG_PHY_SELECTION_FIRST_PHY_PRIORITY:
3789                        sel_phy_idx = EXT_PHY1;
3790                        break;
3791                 case PORT_HW_CFG_PHY_SELECTION_SECOND_PHY:
3792                 case PORT_HW_CFG_PHY_SELECTION_SECOND_PHY_PRIORITY:
3793                        sel_phy_idx = EXT_PHY2;
3794                        break;
3795                 }
3796         }
3797
3798         return sel_phy_idx;
3799
3800 }
3801 int bnx2x_get_link_cfg_idx(struct bnx2x *bp)
3802 {
3803         u32 sel_phy_idx = bnx2x_get_cur_phy_idx(bp);
3804         /*
3805          * The selected actived PHY is always after swapping (in case PHY
3806          * swapping is enabled). So when swapping is enabled, we need to reverse
3807          * the configuration
3808          */
3809
3810         if (bp->link_params.multi_phy_config &
3811             PORT_HW_CFG_PHY_SWAPPED_ENABLED) {
3812                 if (sel_phy_idx == EXT_PHY1)
3813                         sel_phy_idx = EXT_PHY2;
3814                 else if (sel_phy_idx == EXT_PHY2)
3815                         sel_phy_idx = EXT_PHY1;
3816         }
3817         return LINK_CONFIG_IDX(sel_phy_idx);
3818 }
3819
3820 #if defined(NETDEV_FCOE_WWNN) && defined(BCM_CNIC)
3821 int bnx2x_fcoe_get_wwn(struct net_device *dev, u64 *wwn, int type)
3822 {
3823         struct bnx2x *bp = netdev_priv(dev);
3824         struct cnic_eth_dev *cp = &bp->cnic_eth_dev;
3825
3826         switch (type) {
3827         case NETDEV_FCOE_WWNN:
3828                 *wwn = HILO_U64(cp->fcoe_wwn_node_name_hi,
3829                                 cp->fcoe_wwn_node_name_lo);
3830                 break;
3831         case NETDEV_FCOE_WWPN:
3832                 *wwn = HILO_U64(cp->fcoe_wwn_port_name_hi,
3833                                 cp->fcoe_wwn_port_name_lo);
3834                 break;
3835         default:
3836                 BNX2X_ERR("Wrong WWN type requested - %d\n", type);
3837                 return -EINVAL;
3838         }
3839
3840         return 0;
3841 }
3842 #endif
3843
3844 /* called with rtnl_lock */
3845 int bnx2x_change_mtu(struct net_device *dev, int new_mtu)
3846 {
3847         struct bnx2x *bp = netdev_priv(dev);
3848
3849         if (bp->recovery_state != BNX2X_RECOVERY_DONE) {
3850                 BNX2X_ERR("Can't perform change MTU during parity recovery\n");
3851                 return -EAGAIN;
3852         }
3853
3854         if ((new_mtu > ETH_MAX_JUMBO_PACKET_SIZE) ||
3855             ((new_mtu + ETH_HLEN) < ETH_MIN_PACKET_SIZE)) {
3856                 BNX2X_ERR("Can't support requested MTU size\n");
3857                 return -EINVAL;
3858         }
3859
3860         /* This does not race with packet allocation
3861          * because the actual alloc size is
3862          * only updated as part of load
3863          */
3864         dev->mtu = new_mtu;
3865
3866         return bnx2x_reload_if_running(dev);
3867 }
3868
3869 netdev_features_t bnx2x_fix_features(struct net_device *dev,
3870                                      netdev_features_t features)
3871 {
3872         struct bnx2x *bp = netdev_priv(dev);
3873
3874         /* TPA requires Rx CSUM offloading */
3875         if (!(features & NETIF_F_RXCSUM) || bp->disable_tpa) {
3876                 features &= ~NETIF_F_LRO;
3877                 features &= ~NETIF_F_GRO;
3878         }
3879
3880         return features;
3881 }
3882
3883 int bnx2x_set_features(struct net_device *dev, netdev_features_t features)
3884 {
3885         struct bnx2x *bp = netdev_priv(dev);
3886         u32 flags = bp->flags;
3887         bool bnx2x_reload = false;
3888
3889         if (features & NETIF_F_LRO)
3890                 flags |= TPA_ENABLE_FLAG;
3891         else
3892                 flags &= ~TPA_ENABLE_FLAG;
3893
3894         if (features & NETIF_F_GRO)
3895                 flags |= GRO_ENABLE_FLAG;
3896         else
3897                 flags &= ~GRO_ENABLE_FLAG;
3898
3899         if (features & NETIF_F_LOOPBACK) {
3900                 if (bp->link_params.loopback_mode != LOOPBACK_BMAC) {
3901                         bp->link_params.loopback_mode = LOOPBACK_BMAC;
3902                         bnx2x_reload = true;
3903                 }
3904         } else {
3905                 if (bp->link_params.loopback_mode != LOOPBACK_NONE) {
3906                         bp->link_params.loopback_mode = LOOPBACK_NONE;
3907                         bnx2x_reload = true;
3908                 }
3909         }
3910
3911         if (flags ^ bp->flags) {
3912                 bp->flags = flags;
3913                 bnx2x_reload = true;
3914         }
3915
3916         if (bnx2x_reload) {
3917                 if (bp->recovery_state == BNX2X_RECOVERY_DONE)
3918                         return bnx2x_reload_if_running(dev);
3919                 /* else: bnx2x_nic_load() will be called at end of recovery */
3920         }
3921
3922         return 0;
3923 }
3924
3925 void bnx2x_tx_timeout(struct net_device *dev)
3926 {
3927         struct bnx2x *bp = netdev_priv(dev);
3928
3929 #ifdef BNX2X_STOP_ON_ERROR
3930         if (!bp->panic)
3931                 bnx2x_panic();
3932 #endif
3933
3934         smp_mb__before_clear_bit();
3935         set_bit(BNX2X_SP_RTNL_TX_TIMEOUT, &bp->sp_rtnl_state);
3936         smp_mb__after_clear_bit();
3937
3938         /* This allows the netif to be shutdown gracefully before resetting */
3939         schedule_delayed_work(&bp->sp_rtnl_task, 0);
3940 }
3941
3942 int bnx2x_suspend(struct pci_dev *pdev, pm_message_t state)
3943 {
3944         struct net_device *dev = pci_get_drvdata(pdev);
3945         struct bnx2x *bp;
3946
3947         if (!dev) {
3948                 dev_err(&pdev->dev, "BAD net device from bnx2x_init_one\n");
3949                 return -ENODEV;
3950         }
3951         bp = netdev_priv(dev);
3952
3953         rtnl_lock();
3954
3955         pci_save_state(pdev);
3956
3957         if (!netif_running(dev)) {
3958                 rtnl_unlock();
3959                 return 0;
3960         }
3961
3962         netif_device_detach(dev);
3963
3964         bnx2x_nic_unload(bp, UNLOAD_CLOSE);
3965
3966         bnx2x_set_power_state(bp, pci_choose_state(pdev, state));
3967
3968         rtnl_unlock();
3969
3970         return 0;
3971 }
3972
3973 int bnx2x_resume(struct pci_dev *pdev)
3974 {
3975         struct net_device *dev = pci_get_drvdata(pdev);
3976         struct bnx2x *bp;
3977         int rc;
3978
3979         if (!dev) {
3980                 dev_err(&pdev->dev, "BAD net device from bnx2x_init_one\n");
3981                 return -ENODEV;
3982         }
3983         bp = netdev_priv(dev);
3984
3985         if (bp->recovery_state != BNX2X_RECOVERY_DONE) {
3986                 BNX2X_ERR("Handling parity error recovery. Try again later\n");
3987                 return -EAGAIN;
3988         }
3989
3990         rtnl_lock();
3991
3992         pci_restore_state(pdev);
3993
3994         if (!netif_running(dev)) {
3995                 rtnl_unlock();
3996                 return 0;
3997         }
3998
3999         bnx2x_set_power_state(bp, PCI_D0);
4000         netif_device_attach(dev);
4001
4002         rc = bnx2x_nic_load(bp, LOAD_OPEN);
4003
4004         rtnl_unlock();
4005
4006         return rc;
4007 }
4008
4009
4010 void bnx2x_set_ctx_validation(struct bnx2x *bp, struct eth_context *cxt,
4011                               u32 cid)
4012 {
4013         /* ustorm cxt validation */
4014         cxt->ustorm_ag_context.cdu_usage =
4015                 CDU_RSRVD_VALUE_TYPE_A(HW_CID(bp, cid),
4016                         CDU_REGION_NUMBER_UCM_AG, ETH_CONNECTION_TYPE);
4017         /* xcontext validation */
4018         cxt->xstorm_ag_context.cdu_reserved =
4019                 CDU_RSRVD_VALUE_TYPE_A(HW_CID(bp, cid),
4020                         CDU_REGION_NUMBER_XCM_AG, ETH_CONNECTION_TYPE);
4021 }
4022
4023 static void storm_memset_hc_timeout(struct bnx2x *bp, u8 port,
4024                                     u8 fw_sb_id, u8 sb_index,
4025                                     u8 ticks)
4026 {
4027
4028         u32 addr = BAR_CSTRORM_INTMEM +
4029                    CSTORM_STATUS_BLOCK_DATA_TIMEOUT_OFFSET(fw_sb_id, sb_index);
4030         REG_WR8(bp, addr, ticks);
4031         DP(NETIF_MSG_IFUP,
4032            "port %x fw_sb_id %d sb_index %d ticks %d\n",
4033            port, fw_sb_id, sb_index, ticks);
4034 }
4035
4036 static void storm_memset_hc_disable(struct bnx2x *bp, u8 port,
4037                                     u16 fw_sb_id, u8 sb_index,
4038                                     u8 disable)
4039 {
4040         u32 enable_flag = disable ? 0 : (1 << HC_INDEX_DATA_HC_ENABLED_SHIFT);
4041         u32 addr = BAR_CSTRORM_INTMEM +
4042                    CSTORM_STATUS_BLOCK_DATA_FLAGS_OFFSET(fw_sb_id, sb_index);
4043         u16 flags = REG_RD16(bp, addr);
4044         /* clear and set */
4045         flags &= ~HC_INDEX_DATA_HC_ENABLED;
4046         flags |= enable_flag;
4047         REG_WR16(bp, addr, flags);
4048         DP(NETIF_MSG_IFUP,
4049            "port %x fw_sb_id %d sb_index %d disable %d\n",
4050            port, fw_sb_id, sb_index, disable);
4051 }
4052
4053 void bnx2x_update_coalesce_sb_index(struct bnx2x *bp, u8 fw_sb_id,
4054                                     u8 sb_index, u8 disable, u16 usec)
4055 {
4056         int port = BP_PORT(bp);
4057         u8 ticks = usec / BNX2X_BTR;
4058
4059         storm_memset_hc_timeout(bp, port, fw_sb_id, sb_index, ticks);
4060
4061         disable = disable ? 1 : (usec ? 0 : 1);
4062         storm_memset_hc_disable(bp, port, fw_sb_id, sb_index, disable);
4063 }