drivers/net/ethernet/emulex/benet/be_main.c

   1 /*
   2  * Copyright (C) 2005 - 2014 Emulex
   3  * All rights reserved.
   4  *
   5  * This program is free software; you can redistribute it and/or
   6  * modify it under the terms of the GNU General Public License version 2
   7  * as published by the Free Software Foundation.  The full GNU General
   8  * Public License is included in this distribution in the file called COPYING.
   9  *
  10  * Contact Information:
  11  * linux-drivers@emulex.com
  12  *
  13  * Emulex
  14  * 3333 Susan Street
  15  * Costa Mesa, CA 92626
  16  */
  17
  18 #include <linux/prefetch.h>
  19 #include <linux/module.h>
  20 #include "be.h"
  21 #include "be_cmds.h"
  22 #include <asm/div64.h>
  23 #include <linux/aer.h>
  24 #include <linux/if_bridge.h>
  25 #include <net/busy_poll.h>
  26 #include <net/vxlan.h>
  27
  28 MODULE_VERSION(DRV_VER);
  29 MODULE_DEVICE_TABLE(pci, be_dev_ids);
  30 MODULE_DESCRIPTION(DRV_DESC " " DRV_VER);
  31 MODULE_AUTHOR("Emulex Corporation");
  32 MODULE_LICENSE("GPL");
  33
  34 static unsigned int num_vfs;
  35 module_param(num_vfs, uint, S_IRUGO);
  36 MODULE_PARM_DESC(num_vfs, "Number of PCI VFs to initialize");
  37
  38 static ushort rx_frag_size = 2048;
  39 module_param(rx_frag_size, ushort, S_IRUGO);
  40 MODULE_PARM_DESC(rx_frag_size, "Size of a fragment that holds rcvd data.");
  41
  42 static const struct pci_device_id be_dev_ids[] = {
  43         { PCI_DEVICE(BE_VENDOR_ID, BE_DEVICE_ID1) },
  44         { PCI_DEVICE(BE_VENDOR_ID, BE_DEVICE_ID2) },
  45         { PCI_DEVICE(BE_VENDOR_ID, OC_DEVICE_ID1) },
  46         { PCI_DEVICE(BE_VENDOR_ID, OC_DEVICE_ID2) },
  47         { PCI_DEVICE(EMULEX_VENDOR_ID, OC_DEVICE_ID3)},
  48         { PCI_DEVICE(EMULEX_VENDOR_ID, OC_DEVICE_ID4)},
  49         { PCI_DEVICE(EMULEX_VENDOR_ID, OC_DEVICE_ID5)},
  50         { PCI_DEVICE(EMULEX_VENDOR_ID, OC_DEVICE_ID6)},
  51         { 0 }
  52 };
  53 MODULE_DEVICE_TABLE(pci, be_dev_ids);
  54 /* UE Status Low CSR */
  55 static const char * const ue_status_low_desc[] = {
  56         "CEV",
  57         "CTX",
  58         "DBUF",
  59         "ERX",
  60         "Host",
  61         "MPU",
  62         "NDMA",
  63         "PTC ",
  64         "RDMA ",
  65         "RXF ",
  66         "RXIPS ",
  67         "RXULP0 ",
  68         "RXULP1 ",
  69         "RXULP2 ",
  70         "TIM ",
  71         "TPOST ",
  72         "TPRE ",
  73         "TXIPS ",
  74         "TXULP0 ",
  75         "TXULP1 ",
  76         "UC ",
  77         "WDMA ",
  78         "TXULP2 ",
  79         "HOST1 ",
  80         "P0_OB_LINK ",
  81         "P1_OB_LINK ",
  82         "HOST_GPIO ",
  83         "MBOX ",
  84         "ERX2 ",
  85         "SPARE ",
  86         "JTAG ",
  87         "MPU_INTPEND "
  88 };
  89 /* UE Status High CSR */
  90 static const char * const ue_status_hi_desc[] = {
  91         "LPCMEMHOST",
  92         "MGMT_MAC",
  93         "PCS0ONLINE",
  94         "MPU_IRAM",
  95         "PCS1ONLINE",
  96         "PCTL0",
  97         "PCTL1",
  98         "PMEM",
  99         "RR",
 100         "TXPB",
 101         "RXPP",
 102         "XAUI",
 103         "TXP",
 104         "ARM",
 105         "IPC",
 106         "HOST2",
 107         "HOST3",
 108         "HOST4",
 109         "HOST5",
 110         "HOST6",
 111         "HOST7",
 112         "ECRC",
 113         "Poison TLP",
 114         "NETC",
 115         "PERIPH",
 116         "LLTXULP",
 117         "D2P",
 118         "RCON",
 119         "LDMA",
 120         "LLTXP",
 121         "LLTXPB",
 122         "Unknown"
 123 };
 124
 125
 126 static void be_queue_free(struct be_adapter *adapter, struct be_queue_info *q)
 127 {
 128         struct be_dma_mem *mem = &q->dma_mem;
 129         if (mem->va) {
 130                 dma_free_coherent(&adapter->pdev->dev, mem->size, mem->va,
 131                                   mem->dma);
 132                 mem->va = NULL;
 133         }
 134 }
 135
 136 static int be_queue_alloc(struct be_adapter *adapter, struct be_queue_info *q,
 137                           u16 len, u16 entry_size)
 138 {
 139         struct be_dma_mem *mem = &q->dma_mem;
 140
 141         memset(q, 0, sizeof(*q));
 142         q->len = len;
 143         q->entry_size = entry_size;
 144         mem->size = len * entry_size;
 145         mem->va = dma_zalloc_coherent(&adapter->pdev->dev, mem->size, &mem->dma,
 146                                       GFP_KERNEL);
 147         if (!mem->va)
 148                 return -ENOMEM;
 149         return 0;
 150 }
 151
 152 static void be_reg_intr_set(struct be_adapter *adapter, bool enable)
 153 {
 154         u32 reg, enabled;
 155
 156         pci_read_config_dword(adapter->pdev, PCICFG_MEMBAR_CTRL_INT_CTRL_OFFSET,
 157                               &reg);
 158         enabled = reg & MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK;
 159
 160         if (!enabled && enable)
 161                 reg |= MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK;
 162         else if (enabled && !enable)
 163                 reg &= ~MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK;
 164         else
 165                 return;
 166
 167         pci_write_config_dword(adapter->pdev,
 168                                PCICFG_MEMBAR_CTRL_INT_CTRL_OFFSET, reg);
 169 }
 170
 171 static void be_intr_set(struct be_adapter *adapter, bool enable)
 172 {
 173         int status = 0;
 174
 175         /* On lancer interrupts can't be controlled via this register */
 176         if (lancer_chip(adapter))
 177                 return;
 178
 179         if (adapter->eeh_error)
 180                 return;
 181
 182         status = be_cmd_intr_set(adapter, enable);
 183         if (status)
 184                 be_reg_intr_set(adapter, enable);
 185 }
 186
 187 static void be_rxq_notify(struct be_adapter *adapter, u16 qid, u16 posted)
 188 {
 189         u32 val = 0;
 190         val |= qid & DB_RQ_RING_ID_MASK;
 191         val |= posted << DB_RQ_NUM_POSTED_SHIFT;
 192
 193         wmb();
 194         iowrite32(val, adapter->db + DB_RQ_OFFSET);
 195 }
 196
 197 static void be_txq_notify(struct be_adapter *adapter, struct be_tx_obj *txo,
 198                           u16 posted)
 199 {
 200         u32 val = 0;
 201         val |= txo->q.id & DB_TXULP_RING_ID_MASK;
 202         val |= (posted & DB_TXULP_NUM_POSTED_MASK) << DB_TXULP_NUM_POSTED_SHIFT;
 203
 204         wmb();
 205         iowrite32(val, adapter->db + txo->db_offset);
 206 }
 207
 208 static void be_eq_notify(struct be_adapter *adapter, u16 qid,
 209                          bool arm, bool clear_int, u16 num_popped)
 210 {
 211         u32 val = 0;
 212         val |= qid & DB_EQ_RING_ID_MASK;
 213         val |= ((qid & DB_EQ_RING_ID_EXT_MASK) << DB_EQ_RING_ID_EXT_MASK_SHIFT);
 214
 215         if (adapter->eeh_error)
 216                 return;
 217
 218         if (arm)
 219                 val |= 1 << DB_EQ_REARM_SHIFT;
 220         if (clear_int)
 221                 val |= 1 << DB_EQ_CLR_SHIFT;
 222         val |= 1 << DB_EQ_EVNT_SHIFT;
 223         val |= num_popped << DB_EQ_NUM_POPPED_SHIFT;
 224         iowrite32(val, adapter->db + DB_EQ_OFFSET);
 225 }
 226
 227 void be_cq_notify(struct be_adapter *adapter, u16 qid, bool arm, u16 num_popped)
 228 {
 229         u32 val = 0;
 230         val |= qid & DB_CQ_RING_ID_MASK;
 231         val |= ((qid & DB_CQ_RING_ID_EXT_MASK) <<
 232                         DB_CQ_RING_ID_EXT_MASK_SHIFT);
 233
 234         if (adapter->eeh_error)
 235                 return;
 236
 237         if (arm)
 238                 val |= 1 << DB_CQ_REARM_SHIFT;
 239         val |= num_popped << DB_CQ_NUM_POPPED_SHIFT;
 240         iowrite32(val, adapter->db + DB_CQ_OFFSET);
 241 }
 242
 243 static int be_mac_addr_set(struct net_device *netdev, void *p)
 244 {
 245         struct be_adapter *adapter = netdev_priv(netdev);
 246         struct device *dev = &adapter->pdev->dev;
 247         struct sockaddr *addr = p;
 248         int status;
 249         u8 mac[ETH_ALEN];
 250         u32 old_pmac_id = adapter->pmac_id[0], curr_pmac_id = 0;
 251
 252         if (!is_valid_ether_addr(addr->sa_data))
 253                 return -EADDRNOTAVAIL;
 254
 255         /* Proceed further only if, User provided MAC is different
 256          * from active MAC
 257          */
 258         if (ether_addr_equal(addr->sa_data, netdev->dev_addr))
 259                 return 0;
 260
 261         /* The PMAC_ADD cmd may fail if the VF doesn't have FILTMGMT
 262          * privilege or if PF did not provision the new MAC address.
 263          * On BE3, this cmd will always fail if the VF doesn't have the
 264          * FILTMGMT privilege. This failure is OK, only if the PF programmed
 265          * the MAC for the VF.
 266          */
 267         status = be_cmd_pmac_add(adapter, (u8 *)addr->sa_data,
 268                                  adapter->if_handle, &adapter->pmac_id[0], 0);
 269         if (!status) {
 270                 curr_pmac_id = adapter->pmac_id[0];
 271
 272                 /* Delete the old programmed MAC. This call may fail if the
 273                  * old MAC was already deleted by the PF driver.
 274                  */
 275                 if (adapter->pmac_id[0] != old_pmac_id)
 276                         be_cmd_pmac_del(adapter, adapter->if_handle,
 277                                         old_pmac_id, 0);
 278         }
 279
 280         /* Decide if the new MAC is successfully activated only after
 281          * querying the FW
 282          */
 283         status = be_cmd_get_active_mac(adapter, curr_pmac_id, mac,
 284                                        adapter->if_handle, true, 0);
 285         if (status)
 286                 goto err;
 287
 288         /* The MAC change did not happen, either due to lack of privilege
 289          * or PF didn't pre-provision.
 290          */
 291         if (!ether_addr_equal(addr->sa_data, mac)) {
 292                 status = -EPERM;
 293                 goto err;
 294         }
 295
 296         memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len);
 297         dev_info(dev, "MAC address changed to %pM\n", mac);
 298         return 0;
 299 err:
 300         dev_warn(dev, "MAC address change to %pM failed\n", addr->sa_data);
 301         return status;
 302 }
 303
 304 /* BE2 supports only v0 cmd */
 305 static void *hw_stats_from_cmd(struct be_adapter *adapter)
 306 {
 307         if (BE2_chip(adapter)) {
 308                 struct be_cmd_resp_get_stats_v0 *cmd = adapter->stats_cmd.va;
 309
 310                 return &cmd->hw_stats;
 311         } else if (BE3_chip(adapter)) {
 312                 struct be_cmd_resp_get_stats_v1 *cmd = adapter->stats_cmd.va;
 313
 314                 return &cmd->hw_stats;
 315         } else {
 316                 struct be_cmd_resp_get_stats_v2 *cmd = adapter->stats_cmd.va;
 317
 318                 return &cmd->hw_stats;
 319         }
 320 }
 321
 322 /* BE2 supports only v0 cmd */
 323 static void *be_erx_stats_from_cmd(struct be_adapter *adapter)
 324 {
 325         if (BE2_chip(adapter)) {
 326                 struct be_hw_stats_v0 *hw_stats = hw_stats_from_cmd(adapter);
 327
 328                 return &hw_stats->erx;
 329         } else if (BE3_chip(adapter)) {
 330                 struct be_hw_stats_v1 *hw_stats = hw_stats_from_cmd(adapter);
 331
 332                 return &hw_stats->erx;
 333         } else {
 334                 struct be_hw_stats_v2 *hw_stats = hw_stats_from_cmd(adapter);
 335
 336                 return &hw_stats->erx;
 337         }
 338 }
 339
 340 static void populate_be_v0_stats(struct be_adapter *adapter)
 341 {
 342         struct be_hw_stats_v0 *hw_stats = hw_stats_from_cmd(adapter);
 343         struct be_pmem_stats *pmem_sts = &hw_stats->pmem;
 344         struct be_rxf_stats_v0 *rxf_stats = &hw_stats->rxf;
 345         struct be_port_rxf_stats_v0 *port_stats =
 346                                         &rxf_stats->port[adapter->port_num];
 347         struct be_drv_stats *drvs = &adapter->drv_stats;
 348
 349         be_dws_le_to_cpu(hw_stats, sizeof(*hw_stats));
 350         drvs->rx_pause_frames = port_stats->rx_pause_frames;
 351         drvs->rx_crc_errors = port_stats->rx_crc_errors;
 352         drvs->rx_control_frames = port_stats->rx_control_frames;
 353         drvs->rx_in_range_errors = port_stats->rx_in_range_errors;
 354         drvs->rx_frame_too_long = port_stats->rx_frame_too_long;
 355         drvs->rx_dropped_runt = port_stats->rx_dropped_runt;
 356         drvs->rx_ip_checksum_errs = port_stats->rx_ip_checksum_errs;
 357         drvs->rx_tcp_checksum_errs = port_stats->rx_tcp_checksum_errs;
 358         drvs->rx_udp_checksum_errs = port_stats->rx_udp_checksum_errs;
 359         drvs->rxpp_fifo_overflow_drop = port_stats->rx_fifo_overflow;
 360         drvs->rx_dropped_tcp_length = port_stats->rx_dropped_tcp_length;
 361         drvs->rx_dropped_too_small = port_stats->rx_dropped_too_small;
 362         drvs->rx_dropped_too_short = port_stats->rx_dropped_too_short;
 363         drvs->rx_out_range_errors = port_stats->rx_out_range_errors;
 364         drvs->rx_input_fifo_overflow_drop = port_stats->rx_input_fifo_overflow;
 365         drvs->rx_dropped_header_too_small =
 366                 port_stats->rx_dropped_header_too_small;
 367         drvs->rx_address_filtered =
 368                                         port_stats->rx_address_filtered +
 369                                         port_stats->rx_vlan_filtered;
 370         drvs->rx_alignment_symbol_errors =
 371                 port_stats->rx_alignment_symbol_errors;
 372
 373         drvs->tx_pauseframes = port_stats->tx_pauseframes;
 374         drvs->tx_controlframes = port_stats->tx_controlframes;
 375
 376         if (adapter->port_num)
 377                 drvs->jabber_events = rxf_stats->port1_jabber_events;
 378         else
 379                 drvs->jabber_events = rxf_stats->port0_jabber_events;
 380         drvs->rx_drops_no_pbuf = rxf_stats->rx_drops_no_pbuf;
 381         drvs->rx_drops_no_erx_descr = rxf_stats->rx_drops_no_erx_descr;
 382         drvs->forwarded_packets = rxf_stats->forwarded_packets;
 383         drvs->rx_drops_mtu = rxf_stats->rx_drops_mtu;
 384         drvs->rx_drops_no_tpre_descr = rxf_stats->rx_drops_no_tpre_descr;
 385         drvs->rx_drops_too_many_frags = rxf_stats->rx_drops_too_many_frags;
 386         adapter->drv_stats.eth_red_drops = pmem_sts->eth_red_drops;
 387 }
 388
 389 static void populate_be_v1_stats(struct be_adapter *adapter)
 390 {
 391         struct be_hw_stats_v1 *hw_stats = hw_stats_from_cmd(adapter);
 392         struct be_pmem_stats *pmem_sts = &hw_stats->pmem;
 393         struct be_rxf_stats_v1 *rxf_stats = &hw_stats->rxf;
 394         struct be_port_rxf_stats_v1 *port_stats =
 395                                         &rxf_stats->port[adapter->port_num];
 396         struct be_drv_stats *drvs = &adapter->drv_stats;
 397
 398         be_dws_le_to_cpu(hw_stats, sizeof(*hw_stats));
 399         drvs->pmem_fifo_overflow_drop = port_stats->pmem_fifo_overflow_drop;
 400         drvs->rx_priority_pause_frames = port_stats->rx_priority_pause_frames;
 401         drvs->rx_pause_frames = port_stats->rx_pause_frames;
 402         drvs->rx_crc_errors = port_stats->rx_crc_errors;
 403         drvs->rx_control_frames = port_stats->rx_control_frames;
 404         drvs->rx_in_range_errors = port_stats->rx_in_range_errors;
 405         drvs->rx_frame_too_long = port_stats->rx_frame_too_long;
 406         drvs->rx_dropped_runt = port_stats->rx_dropped_runt;
 407         drvs->rx_ip_checksum_errs = port_stats->rx_ip_checksum_errs;
 408         drvs->rx_tcp_checksum_errs = port_stats->rx_tcp_checksum_errs;
 409         drvs->rx_udp_checksum_errs = port_stats->rx_udp_checksum_errs;
 410         drvs->rx_dropped_tcp_length = port_stats->rx_dropped_tcp_length;
 411         drvs->rx_dropped_too_small = port_stats->rx_dropped_too_small;
 412         drvs->rx_dropped_too_short = port_stats->rx_dropped_too_short;
 413         drvs->rx_out_range_errors = port_stats->rx_out_range_errors;
 414         drvs->rx_dropped_header_too_small =
 415                 port_stats->rx_dropped_header_too_small;
 416         drvs->rx_input_fifo_overflow_drop =
 417                 port_stats->rx_input_fifo_overflow_drop;
 418         drvs->rx_address_filtered = port_stats->rx_address_filtered;
 419         drvs->rx_alignment_symbol_errors =
 420                 port_stats->rx_alignment_symbol_errors;
 421         drvs->rxpp_fifo_overflow_drop = port_stats->rxpp_fifo_overflow_drop;
 422         drvs->tx_pauseframes = port_stats->tx_pauseframes;
 423         drvs->tx_controlframes = port_stats->tx_controlframes;
 424         drvs->tx_priority_pauseframes = port_stats->tx_priority_pauseframes;
 425         drvs->jabber_events = port_stats->jabber_events;
 426         drvs->rx_drops_no_pbuf = rxf_stats->rx_drops_no_pbuf;
 427         drvs->rx_drops_no_erx_descr = rxf_stats->rx_drops_no_erx_descr;
 428         drvs->forwarded_packets = rxf_stats->forwarded_packets;
 429         drvs->rx_drops_mtu = rxf_stats->rx_drops_mtu;
 430         drvs->rx_drops_no_tpre_descr = rxf_stats->rx_drops_no_tpre_descr;
 431         drvs->rx_drops_too_many_frags = rxf_stats->rx_drops_too_many_frags;
 432         adapter->drv_stats.eth_red_drops = pmem_sts->eth_red_drops;
 433 }
 434
 435 static void populate_be_v2_stats(struct be_adapter *adapter)
 436 {
 437         struct be_hw_stats_v2 *hw_stats = hw_stats_from_cmd(adapter);
 438         struct be_pmem_stats *pmem_sts = &hw_stats->pmem;
 439         struct be_rxf_stats_v2 *rxf_stats = &hw_stats->rxf;
 440         struct be_port_rxf_stats_v2 *port_stats =
 441                                         &rxf_stats->port[adapter->port_num];
 442         struct be_drv_stats *drvs = &adapter->drv_stats;
 443
 444         be_dws_le_to_cpu(hw_stats, sizeof(*hw_stats));
 445         drvs->pmem_fifo_overflow_drop = port_stats->pmem_fifo_overflow_drop;
 446         drvs->rx_priority_pause_frames = port_stats->rx_priority_pause_frames;
 447         drvs->rx_pause_frames = port_stats->rx_pause_frames;
 448         drvs->rx_crc_errors = port_stats->rx_crc_errors;
 449         drvs->rx_control_frames = port_stats->rx_control_frames;
 450         drvs->rx_in_range_errors = port_stats->rx_in_range_errors;
 451         drvs->rx_frame_too_long = port_stats->rx_frame_too_long;
 452         drvs->rx_dropped_runt = port_stats->rx_dropped_runt;
 453         drvs->rx_ip_checksum_errs = port_stats->rx_ip_checksum_errs;
 454         drvs->rx_tcp_checksum_errs = port_stats->rx_tcp_checksum_errs;
 455         drvs->rx_udp_checksum_errs = port_stats->rx_udp_checksum_errs;
 456         drvs->rx_dropped_tcp_length = port_stats->rx_dropped_tcp_length;
 457         drvs->rx_dropped_too_small = port_stats->rx_dropped_too_small;
 458         drvs->rx_dropped_too_short = port_stats->rx_dropped_too_short;
 459         drvs->rx_out_range_errors = port_stats->rx_out_range_errors;
 460         drvs->rx_dropped_header_too_small =
 461                 port_stats->rx_dropped_header_too_small;
 462         drvs->rx_input_fifo_overflow_drop =
 463                 port_stats->rx_input_fifo_overflow_drop;
 464         drvs->rx_address_filtered = port_stats->rx_address_filtered;
 465         drvs->rx_alignment_symbol_errors =
 466                 port_stats->rx_alignment_symbol_errors;
 467         drvs->rxpp_fifo_overflow_drop = port_stats->rxpp_fifo_overflow_drop;
 468         drvs->tx_pauseframes = port_stats->tx_pauseframes;
 469         drvs->tx_controlframes = port_stats->tx_controlframes;
 470         drvs->tx_priority_pauseframes = port_stats->tx_priority_pauseframes;
 471         drvs->jabber_events = port_stats->jabber_events;
 472         drvs->rx_drops_no_pbuf = rxf_stats->rx_drops_no_pbuf;
 473         drvs->rx_drops_no_erx_descr = rxf_stats->rx_drops_no_erx_descr;
 474         drvs->forwarded_packets = rxf_stats->forwarded_packets;
 475         drvs->rx_drops_mtu = rxf_stats->rx_drops_mtu;
 476         drvs->rx_drops_no_tpre_descr = rxf_stats->rx_drops_no_tpre_descr;
 477         drvs->rx_drops_too_many_frags = rxf_stats->rx_drops_too_many_frags;
 478         adapter->drv_stats.eth_red_drops = pmem_sts->eth_red_drops;
 479         if (be_roce_supported(adapter)) {
 480                 drvs->rx_roce_bytes_lsd = port_stats->roce_bytes_received_lsd;
 481                 drvs->rx_roce_bytes_msd = port_stats->roce_bytes_received_msd;
 482                 drvs->rx_roce_frames = port_stats->roce_frames_received;
 483                 drvs->roce_drops_crc = port_stats->roce_drops_crc;
 484                 drvs->roce_drops_payload_len =
 485                         port_stats->roce_drops_payload_len;
 486         }
 487 }
 488
 489 static void populate_lancer_stats(struct be_adapter *adapter)
 490 {
 491
 492         struct be_drv_stats *drvs = &adapter->drv_stats;
 493         struct lancer_pport_stats *pport_stats = pport_stats_from_cmd(adapter);
 494
 495         be_dws_le_to_cpu(pport_stats, sizeof(*pport_stats));
 496         drvs->rx_pause_frames = pport_stats->rx_pause_frames_lo;
 497         drvs->rx_crc_errors = pport_stats->rx_crc_errors_lo;
 498         drvs->rx_control_frames = pport_stats->rx_control_frames_lo;
 499         drvs->rx_in_range_errors = pport_stats->rx_in_range_errors;
 500         drvs->rx_frame_too_long = pport_stats->rx_frames_too_long_lo;
 501         drvs->rx_dropped_runt = pport_stats->rx_dropped_runt;
 502         drvs->rx_ip_checksum_errs = pport_stats->rx_ip_checksum_errors;
 503         drvs->rx_tcp_checksum_errs = pport_stats->rx_tcp_checksum_errors;
 504         drvs->rx_udp_checksum_errs = pport_stats->rx_udp_checksum_errors;
 505         drvs->rx_dropped_tcp_length =
 506                                 pport_stats->rx_dropped_invalid_tcp_length;
 507         drvs->rx_dropped_too_small = pport_stats->rx_dropped_too_small;
 508         drvs->rx_dropped_too_short = pport_stats->rx_dropped_too_short;
 509         drvs->rx_out_range_errors = pport_stats->rx_out_of_range_errors;
 510         drvs->rx_dropped_header_too_small =
 511                                 pport_stats->rx_dropped_header_too_small;
 512         drvs->rx_input_fifo_overflow_drop = pport_stats->rx_fifo_overflow;
 513         drvs->rx_address_filtered =
 514                                         pport_stats->rx_address_filtered +
 515                                         pport_stats->rx_vlan_filtered;
 516         drvs->rx_alignment_symbol_errors = pport_stats->rx_symbol_errors_lo;
 517         drvs->rxpp_fifo_overflow_drop = pport_stats->rx_fifo_overflow;
 518         drvs->tx_pauseframes = pport_stats->tx_pause_frames_lo;
 519         drvs->tx_controlframes = pport_stats->tx_control_frames_lo;
 520         drvs->jabber_events = pport_stats->rx_jabbers;
 521         drvs->forwarded_packets = pport_stats->num_forwards_lo;
 522         drvs->rx_drops_mtu = pport_stats->rx_drops_mtu_lo;
 523         drvs->rx_drops_too_many_frags =
 524                                 pport_stats->rx_drops_too_many_frags_lo;
 525 }
 526
 527 static void accumulate_16bit_val(u32 *acc, u16 val)
 528 {
 529 #define lo(x)                   (x & 0xFFFF)
 530 #define hi(x)                   (x & 0xFFFF0000)
 531         bool wrapped = val < lo(*acc);
 532         u32 newacc = hi(*acc) + val;
 533
 534         if (wrapped)
 535                 newacc += 65536;
 536         ACCESS_ONCE(*acc) = newacc;
 537 }
 538
 539 static void populate_erx_stats(struct be_adapter *adapter,
 540                                struct be_rx_obj *rxo, u32 erx_stat)
 541 {
 542         if (!BEx_chip(adapter))
 543                 rx_stats(rxo)->rx_drops_no_frags = erx_stat;
 544         else
 545                 /* below erx HW counter can actually wrap around after
 546                  * 65535. Driver accumulates a 32-bit value
 547                  */
 548                 accumulate_16bit_val(&rx_stats(rxo)->rx_drops_no_frags,
 549                                      (u16)erx_stat);
 550 }
 551
 552 void be_parse_stats(struct be_adapter *adapter)
 553 {
 554         struct be_erx_stats_v2 *erx = be_erx_stats_from_cmd(adapter);
 555         struct be_rx_obj *rxo;
 556         int i;
 557         u32 erx_stat;
 558
 559         if (lancer_chip(adapter)) {
 560                 populate_lancer_stats(adapter);
 561         } else {
 562                 if (BE2_chip(adapter))
 563                         populate_be_v0_stats(adapter);
 564                 else if (BE3_chip(adapter))
 565                         /* for BE3 */
 566                         populate_be_v1_stats(adapter);
 567                 else
 568                         populate_be_v2_stats(adapter);
 569
 570                 /* erx_v2 is longer than v0, v1. use v2 for v0, v1 access */
 571                 for_all_rx_queues(adapter, rxo, i) {
 572                         erx_stat = erx->rx_drops_no_fragments[rxo->q.id];
 573                         populate_erx_stats(adapter, rxo, erx_stat);
 574                 }
 575         }
 576 }
 577
 578 static struct rtnl_link_stats64 *be_get_stats64(struct net_device *netdev,
 579                                                 struct rtnl_link_stats64 *stats)
 580 {
 581         struct be_adapter *adapter = netdev_priv(netdev);
 582         struct be_drv_stats *drvs = &adapter->drv_stats;
 583         struct be_rx_obj *rxo;
 584         struct be_tx_obj *txo;
 585         u64 pkts, bytes;
 586         unsigned int start;
 587         int i;
 588
 589         for_all_rx_queues(adapter, rxo, i) {
 590                 const struct be_rx_stats *rx_stats = rx_stats(rxo);
 591                 do {
 592                         start = u64_stats_fetch_begin_irq(&rx_stats->sync);
 593                         pkts = rx_stats(rxo)->rx_pkts;
 594                         bytes = rx_stats(rxo)->rx_bytes;
 595                 } while (u64_stats_fetch_retry_irq(&rx_stats->sync, start));
 596                 stats->rx_packets += pkts;
 597                 stats->rx_bytes += bytes;
 598                 stats->multicast += rx_stats(rxo)->rx_mcast_pkts;
 599                 stats->rx_dropped += rx_stats(rxo)->rx_drops_no_skbs +
 600                                         rx_stats(rxo)->rx_drops_no_frags;
 601         }
 602
 603         for_all_tx_queues(adapter, txo, i) {
 604                 const struct be_tx_stats *tx_stats = tx_stats(txo);
 605                 do {
 606                         start = u64_stats_fetch_begin_irq(&tx_stats->sync);
 607                         pkts = tx_stats(txo)->tx_pkts;
 608                         bytes = tx_stats(txo)->tx_bytes;
 609                 } while (u64_stats_fetch_retry_irq(&tx_stats->sync, start));
 610                 stats->tx_packets += pkts;
 611                 stats->tx_bytes += bytes;
 612         }
 613
 614         /* bad pkts received */
 615         stats->rx_errors = drvs->rx_crc_errors +
 616                 drvs->rx_alignment_symbol_errors +
 617                 drvs->rx_in_range_errors +
 618                 drvs->rx_out_range_errors +
 619                 drvs->rx_frame_too_long +
 620                 drvs->rx_dropped_too_small +
 621                 drvs->rx_dropped_too_short +
 622                 drvs->rx_dropped_header_too_small +
 623                 drvs->rx_dropped_tcp_length +
 624                 drvs->rx_dropped_runt;
 625
 626         /* detailed rx errors */
 627         stats->rx_length_errors = drvs->rx_in_range_errors +
 628                 drvs->rx_out_range_errors +
 629                 drvs->rx_frame_too_long;
 630
 631         stats->rx_crc_errors = drvs->rx_crc_errors;
 632
 633         /* frame alignment errors */
 634         stats->rx_frame_errors = drvs->rx_alignment_symbol_errors;
 635
 636         /* receiver fifo overrun */
 637         /* drops_no_pbuf is no per i/f, it's per BE card */
 638         stats->rx_fifo_errors = drvs->rxpp_fifo_overflow_drop +
 639                                 drvs->rx_input_fifo_overflow_drop +
 640                                 drvs->rx_drops_no_pbuf;
 641         return stats;
 642 }
 643
 644 void be_link_status_update(struct be_adapter *adapter, u8 link_status)
 645 {
 646         struct net_device *netdev = adapter->netdev;
 647
 648         if (!(adapter->flags & BE_FLAGS_LINK_STATUS_INIT)) {
 649                 netif_carrier_off(netdev);
 650                 adapter->flags |= BE_FLAGS_LINK_STATUS_INIT;
 651         }
 652
 653         if (link_status)
 654                 netif_carrier_on(netdev);
 655         else
 656                 netif_carrier_off(netdev);
 657 }
 658
 659 static void be_tx_stats_update(struct be_tx_obj *txo,
 660                                u32 wrb_cnt, u32 copied, u32 gso_segs,
 661                                bool stopped)
 662 {
 663         struct be_tx_stats *stats = tx_stats(txo);
 664
 665         u64_stats_update_begin(&stats->sync);
 666         stats->tx_reqs++;
 667         stats->tx_wrbs += wrb_cnt;
 668         stats->tx_bytes += copied;
 669         stats->tx_pkts += (gso_segs ? gso_segs : 1);
 670         if (stopped)
 671                 stats->tx_stops++;
 672         u64_stats_update_end(&stats->sync);
 673 }
 674
 675 /* Determine number of WRB entries needed to xmit data in an skb */
 676 static u32 wrb_cnt_for_skb(struct be_adapter *adapter, struct sk_buff *skb,
 677                            bool *dummy)
 678 {
 679         int cnt = (skb->len > skb->data_len);
 680
 681         cnt += skb_shinfo(skb)->nr_frags;
 682
 683         /* to account for hdr wrb */
 684         cnt++;
 685         if (lancer_chip(adapter) || !(cnt & 1)) {
 686                 *dummy = false;
 687         } else {
 688                 /* add a dummy to make it an even num */
 689                 cnt++;
 690                 *dummy = true;
 691         }
 692         BUG_ON(cnt > BE_MAX_TX_FRAG_COUNT);
 693         return cnt;
 694 }
 695
 696 static inline void wrb_fill(struct be_eth_wrb *wrb, u64 addr, int len)
 697 {
 698         wrb->frag_pa_hi = upper_32_bits(addr);
 699         wrb->frag_pa_lo = addr & 0xFFFFFFFF;
 700         wrb->frag_len = len & ETH_WRB_FRAG_LEN_MASK;
 701         wrb->rsvd0 = 0;
 702 }
 703
 704 static inline u16 be_get_tx_vlan_tag(struct be_adapter *adapter,
 705                                      struct sk_buff *skb)
 706 {
 707         u8 vlan_prio;
 708         u16 vlan_tag;
 709
 710         vlan_tag = vlan_tx_tag_get(skb);
 711         vlan_prio = (vlan_tag & VLAN_PRIO_MASK) >> VLAN_PRIO_SHIFT;
 712         /* If vlan priority provided by OS is NOT in available bmap */
 713         if (!(adapter->vlan_prio_bmap & (1 << vlan_prio)))
 714                 vlan_tag = (vlan_tag & ~VLAN_PRIO_MASK) |
 715                                 adapter->recommended_prio;
 716
 717         return vlan_tag;
 718 }
 719
 720 /* Used only for IP tunnel packets */
 721 static u16 skb_inner_ip_proto(struct sk_buff *skb)
 722 {
 723         return (inner_ip_hdr(skb)->version == 4) ?
 724                 inner_ip_hdr(skb)->protocol : inner_ipv6_hdr(skb)->nexthdr;
 725 }
 726
 727 static u16 skb_ip_proto(struct sk_buff *skb)
 728 {
 729         return (ip_hdr(skb)->version == 4) ?
 730                 ip_hdr(skb)->protocol : ipv6_hdr(skb)->nexthdr;
 731 }
 732
 733 static void wrb_fill_hdr(struct be_adapter *adapter, struct be_eth_hdr_wrb *hdr,
 734                          struct sk_buff *skb, u32 wrb_cnt, u32 len,
 735                          bool skip_hw_vlan)
 736 {
 737         u16 vlan_tag, proto;
 738
 739         memset(hdr, 0, sizeof(*hdr));
 740
 741         SET_TX_WRB_HDR_BITS(crc, hdr, 1);
 742
 743         if (skb_is_gso(skb)) {
 744                 SET_TX_WRB_HDR_BITS(lso, hdr, 1);
 745                 SET_TX_WRB_HDR_BITS(lso_mss, hdr, skb_shinfo(skb)->gso_size);
 746                 if (skb_is_gso_v6(skb) && !lancer_chip(adapter))
 747                         SET_TX_WRB_HDR_BITS(lso6, hdr, 1);
 748         } else if (skb->ip_summed == CHECKSUM_PARTIAL) {
 749                 if (skb->encapsulation) {
 750                         SET_TX_WRB_HDR_BITS(ipcs, hdr, 1);
 751                         proto = skb_inner_ip_proto(skb);
 752                 } else {
 753                         proto = skb_ip_proto(skb);
 754                 }
 755                 if (proto == IPPROTO_TCP)
 756                         SET_TX_WRB_HDR_BITS(tcpcs, hdr, 1);
 757                 else if (proto == IPPROTO_UDP)
 758                         SET_TX_WRB_HDR_BITS(udpcs, hdr, 1);
 759         }
 760
 761         if (vlan_tx_tag_present(skb)) {
 762                 SET_TX_WRB_HDR_BITS(vlan, hdr, 1);
 763                 vlan_tag = be_get_tx_vlan_tag(adapter, skb);
 764                 SET_TX_WRB_HDR_BITS(vlan_tag, hdr, vlan_tag);
 765         }
 766
 767         /* To skip HW VLAN tagging: evt = 1, compl = 0 */
 768         SET_TX_WRB_HDR_BITS(complete, hdr, !skip_hw_vlan);
 769         SET_TX_WRB_HDR_BITS(event, hdr, 1);
 770         SET_TX_WRB_HDR_BITS(num_wrb, hdr, wrb_cnt);
 771         SET_TX_WRB_HDR_BITS(len, hdr, len);
 772 }
 773
 774 static void unmap_tx_frag(struct device *dev, struct be_eth_wrb *wrb,
 775                           bool unmap_single)
 776 {
 777         dma_addr_t dma;
 778
 779         be_dws_le_to_cpu(wrb, sizeof(*wrb));
 780
 781         dma = (u64)wrb->frag_pa_hi << 32 | (u64)wrb->frag_pa_lo;
 782         if (wrb->frag_len) {
 783                 if (unmap_single)
 784                         dma_unmap_single(dev, dma, wrb->frag_len,
 785                                          DMA_TO_DEVICE);
 786                 else
 787                         dma_unmap_page(dev, dma, wrb->frag_len, DMA_TO_DEVICE);
 788         }
 789 }
 790
 791 static int make_tx_wrbs(struct be_adapter *adapter, struct be_queue_info *txq,
 792                         struct sk_buff *skb, u32 wrb_cnt, bool dummy_wrb,
 793                         bool skip_hw_vlan)
 794 {
 795         dma_addr_t busaddr;
 796         int i, copied = 0;
 797         struct device *dev = &adapter->pdev->dev;
 798         struct sk_buff *first_skb = skb;
 799         struct be_eth_wrb *wrb;
 800         struct be_eth_hdr_wrb *hdr;
 801         bool map_single = false;
 802         u16 map_head;
 803
 804         hdr = queue_head_node(txq);
 805         queue_head_inc(txq);
 806         map_head = txq->head;
 807
 808         if (skb->len > skb->data_len) {
 809                 int len = skb_headlen(skb);
 810                 busaddr = dma_map_single(dev, skb->data, len, DMA_TO_DEVICE);
 811                 if (dma_mapping_error(dev, busaddr))
 812                         goto dma_err;
 813                 map_single = true;
 814                 wrb = queue_head_node(txq);
 815                 wrb_fill(wrb, busaddr, len);
 816                 be_dws_cpu_to_le(wrb, sizeof(*wrb));
 817                 queue_head_inc(txq);
 818                 copied += len;
 819         }
 820
 821         for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
 822                 const struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[i];
 823                 busaddr = skb_frag_dma_map(dev, frag, 0,
 824                                            skb_frag_size(frag), DMA_TO_DEVICE);
 825                 if (dma_mapping_error(dev, busaddr))
 826                         goto dma_err;
 827                 wrb = queue_head_node(txq);
 828                 wrb_fill(wrb, busaddr, skb_frag_size(frag));
 829                 be_dws_cpu_to_le(wrb, sizeof(*wrb));
 830                 queue_head_inc(txq);
 831                 copied += skb_frag_size(frag);
 832         }
 833
 834         if (dummy_wrb) {
 835                 wrb = queue_head_node(txq);
 836                 wrb_fill(wrb, 0, 0);
 837                 be_dws_cpu_to_le(wrb, sizeof(*wrb));
 838                 queue_head_inc(txq);
 839         }
 840
 841         wrb_fill_hdr(adapter, hdr, first_skb, wrb_cnt, copied, skip_hw_vlan);
 842         be_dws_cpu_to_le(hdr, sizeof(*hdr));
 843
 844         return copied;
 845 dma_err:
 846         txq->head = map_head;
 847         while (copied) {
 848                 wrb = queue_head_node(txq);
 849                 unmap_tx_frag(dev, wrb, map_single);
 850                 map_single = false;
 851                 copied -= wrb->frag_len;
 852                 adapter->drv_stats.dma_map_errors++;
 853                 queue_head_inc(txq);
 854         }
 855         return 0;
 856 }
 857
 858 static struct sk_buff *be_insert_vlan_in_pkt(struct be_adapter *adapter,
 859                                              struct sk_buff *skb,
 860                                              bool *skip_hw_vlan)
 861 {
 862         u16 vlan_tag = 0;
 863
 864         skb = skb_share_check(skb, GFP_ATOMIC);
 865         if (unlikely(!skb))
 866                 return skb;
 867
 868         if (vlan_tx_tag_present(skb))
 869                 vlan_tag = be_get_tx_vlan_tag(adapter, skb);
 870
 871         if (qnq_async_evt_rcvd(adapter) && adapter->pvid) {
 872                 if (!vlan_tag)
 873                         vlan_tag = adapter->pvid;
 874                 /* f/w workaround to set skip_hw_vlan = 1, informs the F/W to
 875                  * skip VLAN insertion
 876                  */
 877                 if (skip_hw_vlan)
 878                         *skip_hw_vlan = true;
 879         }
 880
 881         if (vlan_tag) {
 882                 skb = __vlan_put_tag(skb, htons(ETH_P_8021Q), vlan_tag);
 883                 if (unlikely(!skb))
 884                         return skb;
 885                 skb->vlan_tci = 0;
 886         }
 887
 888         /* Insert the outer VLAN, if any */
 889         if (adapter->qnq_vid) {
 890                 vlan_tag = adapter->qnq_vid;
 891                 skb = __vlan_put_tag(skb, htons(ETH_P_8021Q), vlan_tag);
 892                 if (unlikely(!skb))
 893                         return skb;
 894                 if (skip_hw_vlan)
 895                         *skip_hw_vlan = true;
 896         }
 897
 898         return skb;
 899 }
 900
 901 static bool be_ipv6_exthdr_check(struct sk_buff *skb)
 902 {
 903         struct ethhdr *eh = (struct ethhdr *)skb->data;
 904         u16 offset = ETH_HLEN;
 905
 906         if (eh->h_proto == htons(ETH_P_IPV6)) {
 907                 struct ipv6hdr *ip6h = (struct ipv6hdr *)(skb->data + offset);
 908
 909                 offset += sizeof(struct ipv6hdr);
 910                 if (ip6h->nexthdr != NEXTHDR_TCP &&
 911                     ip6h->nexthdr != NEXTHDR_UDP) {
 912                         struct ipv6_opt_hdr *ehdr =
 913                                 (struct ipv6_opt_hdr *) (skb->data + offset);
 914
 915                         /* offending pkt: 2nd byte following IPv6 hdr is 0xff */
 916                         if (ehdr->hdrlen == 0xff)
 917                                 return true;
 918                 }
 919         }
 920         return false;
 921 }
 922
 923 static int be_vlan_tag_tx_chk(struct be_adapter *adapter, struct sk_buff *skb)
 924 {
 925         return vlan_tx_tag_present(skb) || adapter->pvid || adapter->qnq_vid;
 926 }
 927
 928 static int be_ipv6_tx_stall_chk(struct be_adapter *adapter, struct sk_buff *skb)
 929 {
 930         return BE3_chip(adapter) && be_ipv6_exthdr_check(skb);
 931 }
 932
 933 static struct sk_buff *be_lancer_xmit_workarounds(struct be_adapter *adapter,
 934                                                   struct sk_buff *skb,
 935                                                   bool *skip_hw_vlan)
 936 {
 937         struct vlan_ethhdr *veh = (struct vlan_ethhdr *)skb->data;
 938         unsigned int eth_hdr_len;
 939         struct iphdr *ip;
 940
 941         /* For padded packets, BE HW modifies tot_len field in IP header
 942          * incorrecly when VLAN tag is inserted by HW.
 943          * For padded packets, Lancer computes incorrect checksum.
 944          */
 945         eth_hdr_len = ntohs(skb->protocol) == ETH_P_8021Q ?
 946                                                 VLAN_ETH_HLEN : ETH_HLEN;
 947         if (skb->len <= 60 &&
 948             (lancer_chip(adapter) || vlan_tx_tag_present(skb)) &&
 949             is_ipv4_pkt(skb)) {
 950                 ip = (struct iphdr *)ip_hdr(skb);
 951                 pskb_trim(skb, eth_hdr_len + ntohs(ip->tot_len));
 952         }
 953
 954         /* If vlan tag is already inlined in the packet, skip HW VLAN
 955          * tagging in pvid-tagging mode
 956          */
 957         if (be_pvid_tagging_enabled(adapter) &&
 958             veh->h_vlan_proto == htons(ETH_P_8021Q))
 959                 *skip_hw_vlan = true;
 960
 961         /* HW has a bug wherein it will calculate CSUM for VLAN
 962          * pkts even though it is disabled.
 963          * Manually insert VLAN in pkt.
 964          */
 965         if (skb->ip_summed != CHECKSUM_PARTIAL &&
 966             vlan_tx_tag_present(skb)) {
 967                 skb = be_insert_vlan_in_pkt(adapter, skb, skip_hw_vlan);
 968                 if (unlikely(!skb))
 969                         goto err;
 970         }
 971
 972         /* HW may lockup when VLAN HW tagging is requested on
 973          * certain ipv6 packets. Drop such pkts if the HW workaround to
 974          * skip HW tagging is not enabled by FW.
 975          */
 976         if (unlikely(be_ipv6_tx_stall_chk(adapter, skb) &&
 977             (adapter->pvid || adapter->qnq_vid) &&
 978             !qnq_async_evt_rcvd(adapter)))
 979                 goto tx_drop;
 980
 981         /* Manual VLAN tag insertion to prevent:
 982          * ASIC lockup when the ASIC inserts VLAN tag into
 983          * certain ipv6 packets. Insert VLAN tags in driver,
 984          * and set event, completion, vlan bits accordingly
 985          * in the Tx WRB.
 986          */
 987         if (be_ipv6_tx_stall_chk(adapter, skb) &&
 988             be_vlan_tag_tx_chk(adapter, skb)) {
 989                 skb = be_insert_vlan_in_pkt(adapter, skb, skip_hw_vlan);
 990                 if (unlikely(!skb))
 991                         goto err;
 992         }
 993
 994         return skb;
 995 tx_drop:
 996         dev_kfree_skb_any(skb);
 997 err:
 998         return NULL;
 999 }
1000
1001 static struct sk_buff *be_xmit_workarounds(struct be_adapter *adapter,
1002                                            struct sk_buff *skb,
1003                                            bool *skip_hw_vlan)
1004 {
1005         /* Lancer, SH-R ASICs have a bug wherein Packets that are 32 bytes or
1006          * less may cause a transmit stall on that port. So the work-around is
1007          * to pad short packets (<= 32 bytes) to a 36-byte length.
1008          */
1009         if (unlikely(!BEx_chip(adapter) && skb->len <= 32)) {
1010                 if (skb_padto(skb, 36))
1011                         return NULL;
1012                 skb->len = 36;
1013         }
1014
1015         if (BEx_chip(adapter) || lancer_chip(adapter)) {
1016                 skb = be_lancer_xmit_workarounds(adapter, skb, skip_hw_vlan);
1017                 if (!skb)
1018                         return NULL;
1019         }
1020
1021         return skb;
1022 }
1023
1024 static netdev_tx_t be_xmit(struct sk_buff *skb, struct net_device *netdev)
1025 {
1026         struct be_adapter *adapter = netdev_priv(netdev);
1027         struct be_tx_obj *txo = &adapter->tx_obj[skb_get_queue_mapping(skb)];
1028         struct be_queue_info *txq = &txo->q;
1029         bool dummy_wrb, stopped = false;
1030         u32 wrb_cnt = 0, copied = 0;
1031         bool skip_hw_vlan = false;
1032         u32 start = txq->head;
1033
1034         skb = be_xmit_workarounds(adapter, skb, &skip_hw_vlan);
1035         if (!skb) {
1036                 tx_stats(txo)->tx_drv_drops++;
1037                 return NETDEV_TX_OK;
1038         }
1039
1040         wrb_cnt = wrb_cnt_for_skb(adapter, skb, &dummy_wrb);
1041
1042         copied = make_tx_wrbs(adapter, txq, skb, wrb_cnt, dummy_wrb,
1043                               skip_hw_vlan);
1044         if (copied) {
1045                 int gso_segs = skb_shinfo(skb)->gso_segs;
1046
1047                 /* record the sent skb in the sent_skb table */
1048                 BUG_ON(txo->sent_skb_list[start]);
1049                 txo->sent_skb_list[start] = skb;
1050
1051                 /* Ensure txq has space for the next skb; Else stop the queue
1052                  * *BEFORE* ringing the tx doorbell, so that we serialze the
1053                  * tx compls of the current transmit which'll wake up the queue
1054                  */
1055                 atomic_add(wrb_cnt, &txq->used);
1056                 if ((BE_MAX_TX_FRAG_COUNT + atomic_read(&txq->used)) >=
1057                                                                 txq->len) {
1058                         netif_stop_subqueue(netdev, skb_get_queue_mapping(skb));
1059                         stopped = true;
1060                 }
1061
1062                 be_txq_notify(adapter, txo, wrb_cnt);
1063
1064                 be_tx_stats_update(txo, wrb_cnt, copied, gso_segs, stopped);
1065         } else {
1066                 txq->head = start;
1067                 tx_stats(txo)->tx_drv_drops++;
1068                 dev_kfree_skb_any(skb);
1069         }
1070         return NETDEV_TX_OK;
1071 }
1072
1073 static int be_change_mtu(struct net_device *netdev, int new_mtu)
1074 {
1075         struct be_adapter *adapter = netdev_priv(netdev);
1076         if (new_mtu < BE_MIN_MTU ||
1077             new_mtu > (BE_MAX_JUMBO_FRAME_SIZE - (ETH_HLEN + ETH_FCS_LEN))) {
1078                 dev_info(&adapter->pdev->dev,
1079                          "MTU must be between %d and %d bytes\n",
1080                          BE_MIN_MTU,
1081                          (BE_MAX_JUMBO_FRAME_SIZE - (ETH_HLEN + ETH_FCS_LEN)));
1082                 return -EINVAL;
1083         }
1084         dev_info(&adapter->pdev->dev, "MTU changed from %d to %d bytes\n",
1085                  netdev->mtu, new_mtu);
1086         netdev->mtu = new_mtu;
1087         return 0;
1088 }
1089
1090 /*
1091  * A max of 64 (BE_NUM_VLANS_SUPPORTED) vlans can be configured in BE.
1092  * If the user configures more, place BE in vlan promiscuous mode.
1093  */
1094 static int be_vid_config(struct be_adapter *adapter)
1095 {
1096         u16 vids[BE_NUM_VLANS_SUPPORTED];
1097         u16 num = 0, i = 0;
1098         int status = 0;
1099
1100         /* No need to further configure vids if in promiscuous mode */
1101         if (adapter->promiscuous)
1102                 return 0;
1103
1104         if (adapter->vlans_added > be_max_vlans(adapter))
1105                 goto set_vlan_promisc;
1106
1107         /* Construct VLAN Table to give to HW */
1108         for_each_set_bit(i, adapter->vids, VLAN_N_VID)
1109                 vids[num++] = cpu_to_le16(i);
1110
1111         status = be_cmd_vlan_config(adapter, adapter->if_handle, vids, num);
1112         if (status) {
1113                 /* Set to VLAN promisc mode as setting VLAN filter failed */
1114                 if (addl_status(status) ==
1115                                 MCC_ADDL_STATUS_INSUFFICIENT_RESOURCES)
1116                         goto set_vlan_promisc;
1117                 dev_err(&adapter->pdev->dev,
1118                         "Setting HW VLAN filtering failed.\n");
1119         } else {
1120                 if (adapter->flags & BE_FLAGS_VLAN_PROMISC) {
1121                         /* hw VLAN filtering re-enabled. */
1122                         status = be_cmd_rx_filter(adapter,
1123                                                   BE_FLAGS_VLAN_PROMISC, OFF);
1124                         if (!status) {
1125                                 dev_info(&adapter->pdev->dev,
1126                                          "Disabling VLAN Promiscuous mode.\n");
1127                                 adapter->flags &= ~BE_FLAGS_VLAN_PROMISC;
1128                         }
1129                 }
1130         }
1131
1132         return status;
1133
1134 set_vlan_promisc:
1135         if (adapter->flags & BE_FLAGS_VLAN_PROMISC)
1136                 return 0;
1137
1138         status = be_cmd_rx_filter(adapter, BE_FLAGS_VLAN_PROMISC, ON);
1139         if (!status) {
1140                 dev_info(&adapter->pdev->dev, "Enable VLAN Promiscuous mode\n");
1141                 adapter->flags |= BE_FLAGS_VLAN_PROMISC;
1142         } else
1143                 dev_err(&adapter->pdev->dev,
1144                         "Failed to enable VLAN Promiscuous mode.\n");
1145         return status;
1146 }
1147
1148 static int be_vlan_add_vid(struct net_device *netdev, __be16 proto, u16 vid)
1149 {
1150         struct be_adapter *adapter = netdev_priv(netdev);
1151         int status = 0;
1152
1153         /* Packets with VID 0 are always received by Lancer by default */
1154         if (lancer_chip(adapter) && vid == 0)
1155                 return status;
1156
1157         if (test_bit(vid, adapter->vids))
1158                 return status;
1159
1160         set_bit(vid, adapter->vids);
1161         adapter->vlans_added++;
1162
1163         status = be_vid_config(adapter);
1164         if (status) {
1165                 adapter->vlans_added--;
1166                 clear_bit(vid, adapter->vids);
1167         }
1168
1169         return status;
1170 }
1171
1172 static int be_vlan_rem_vid(struct net_device *netdev, __be16 proto, u16 vid)
1173 {
1174         struct be_adapter *adapter = netdev_priv(netdev);
1175
1176         /* Packets with VID 0 are always received by Lancer by default */
1177         if (lancer_chip(adapter) && vid == 0)
1178                 return 0;
1179
1180         clear_bit(vid, adapter->vids);
1181         adapter->vlans_added--;
1182
1183         return be_vid_config(adapter);
1184 }
1185
1186 static void be_clear_promisc(struct be_adapter *adapter)
1187 {
1188         adapter->promiscuous = false;
1189         adapter->flags &= ~(BE_FLAGS_VLAN_PROMISC | BE_FLAGS_MCAST_PROMISC);
1190
1191         be_cmd_rx_filter(adapter, IFF_PROMISC, OFF);
1192 }
1193
1194 static void be_set_rx_mode(struct net_device *netdev)
1195 {
1196         struct be_adapter *adapter = netdev_priv(netdev);
1197         int status;
1198
1199         if (netdev->flags & IFF_PROMISC) {
1200                 be_cmd_rx_filter(adapter, IFF_PROMISC, ON);
1201                 adapter->promiscuous = true;
1202                 goto done;
1203         }
1204
1205         /* BE was previously in promiscuous mode; disable it */
1206         if (adapter->promiscuous) {
1207                 be_clear_promisc(adapter);
1208                 if (adapter->vlans_added)
1209                         be_vid_config(adapter);
1210         }
1211
1212         /* Enable multicast promisc if num configured exceeds what we support */
1213         if (netdev->flags & IFF_ALLMULTI ||
1214             netdev_mc_count(netdev) > be_max_mc(adapter))
1215                 goto set_mcast_promisc;
1216
1217         if (netdev_uc_count(netdev) != adapter->uc_macs) {
1218                 struct netdev_hw_addr *ha;
1219                 int i = 1; /* First slot is claimed by the Primary MAC */
1220
1221                 for (; adapter->uc_macs > 0; adapter->uc_macs--, i++) {
1222                         be_cmd_pmac_del(adapter, adapter->if_handle,
1223                                         adapter->pmac_id[i], 0);
1224                 }
1225
1226                 if (netdev_uc_count(netdev) > be_max_uc(adapter)) {
1227                         be_cmd_rx_filter(adapter, IFF_PROMISC, ON);
1228                         adapter->promiscuous = true;
1229                         goto done;
1230                 }
1231
1232                 netdev_for_each_uc_addr(ha, adapter->netdev) {
1233                         adapter->uc_macs++; /* First slot is for Primary MAC */
1234                         be_cmd_pmac_add(adapter, (u8 *)ha->addr,
1235                                         adapter->if_handle,
1236                                         &adapter->pmac_id[adapter->uc_macs], 0);
1237                 }
1238         }
1239
1240         status = be_cmd_rx_filter(adapter, IFF_MULTICAST, ON);
1241         if (!status) {
1242                 if (adapter->flags & BE_FLAGS_MCAST_PROMISC)
1243                         adapter->flags &= ~BE_FLAGS_MCAST_PROMISC;
1244                 goto done;
1245         }
1246
1247 set_mcast_promisc:
1248         if (adapter->flags & BE_FLAGS_MCAST_PROMISC)
1249                 return;
1250
1251         /* Set to MCAST promisc mode if setting MULTICAST address fails
1252          * or if num configured exceeds what we support
1253          */
1254         status = be_cmd_rx_filter(adapter, IFF_ALLMULTI, ON);
1255         if (!status)
1256                 adapter->flags |= BE_FLAGS_MCAST_PROMISC;
1257 done:
1258         return;
1259 }
1260
1261 static int be_set_vf_mac(struct net_device *netdev, int vf, u8 *mac)
1262 {
1263         struct be_adapter *adapter = netdev_priv(netdev);
1264         struct be_vf_cfg *vf_cfg = &adapter->vf_cfg[vf];
1265         int status;
1266
1267         if (!sriov_enabled(adapter))
1268                 return -EPERM;
1269
1270         if (!is_valid_ether_addr(mac) || vf >= adapter->num_vfs)
1271                 return -EINVAL;
1272
1273         /* Proceed further only if user provided MAC is different
1274          * from active MAC
1275          */
1276         if (ether_addr_equal(mac, vf_cfg->mac_addr))
1277                 return 0;
1278
1279         if (BEx_chip(adapter)) {
1280                 be_cmd_pmac_del(adapter, vf_cfg->if_handle, vf_cfg->pmac_id,
1281                                 vf + 1);
1282
1283                 status = be_cmd_pmac_add(adapter, mac, vf_cfg->if_handle,
1284                                          &vf_cfg->pmac_id, vf + 1);
1285         } else {
1286                 status = be_cmd_set_mac(adapter, mac, vf_cfg->if_handle,
1287                                         vf + 1);
1288         }
1289
1290         if (status) {
1291                 dev_err(&adapter->pdev->dev, "MAC %pM set on VF %d Failed: %#x",
1292                         mac, vf, status);
1293                 return be_cmd_status(status);
1294         }
1295
1296         ether_addr_copy(vf_cfg->mac_addr, mac);
1297
1298         return 0;
1299 }
1300
1301 static int be_get_vf_config(struct net_device *netdev, int vf,
1302                             struct ifla_vf_info *vi)
1303 {
1304         struct be_adapter *adapter = netdev_priv(netdev);
1305         struct be_vf_cfg *vf_cfg = &adapter->vf_cfg[vf];
1306
1307         if (!sriov_enabled(adapter))
1308                 return -EPERM;
1309
1310         if (vf >= adapter->num_vfs)
1311                 return -EINVAL;
1312
1313         vi->vf = vf;
1314         vi->max_tx_rate = vf_cfg->tx_rate;
1315         vi->min_tx_rate = 0;
1316         vi->vlan = vf_cfg->vlan_tag & VLAN_VID_MASK;
1317         vi->qos = vf_cfg->vlan_tag >> VLAN_PRIO_SHIFT;
1318         memcpy(&vi->mac, vf_cfg->mac_addr, ETH_ALEN);
1319         vi->linkstate = adapter->vf_cfg[vf].plink_tracking;
1320
1321         return 0;
1322 }
1323
1324 static int be_set_vf_vlan(struct net_device *netdev, int vf, u16 vlan, u8 qos)
1325 {
1326         struct be_adapter *adapter = netdev_priv(netdev);
1327         struct be_vf_cfg *vf_cfg = &adapter->vf_cfg[vf];
1328         int status = 0;
1329
1330         if (!sriov_enabled(adapter))
1331                 return -EPERM;
1332
1333         if (vf >= adapter->num_vfs || vlan > 4095 || qos > 7)
1334                 return -EINVAL;
1335
1336         if (vlan || qos) {
1337                 vlan |= qos << VLAN_PRIO_SHIFT;
1338                 if (vf_cfg->vlan_tag != vlan)
1339                         status = be_cmd_set_hsw_config(adapter, vlan, vf + 1,
1340                                                        vf_cfg->if_handle, 0);
1341         } else {
1342                 /* Reset Transparent Vlan Tagging. */
1343                 status = be_cmd_set_hsw_config(adapter, BE_RESET_VLAN_TAG_ID,
1344                                                vf + 1, vf_cfg->if_handle, 0);
1345         }
1346
1347         if (status) {
1348                 dev_err(&adapter->pdev->dev,
1349                         "VLAN %d config on VF %d failed : %#x\n", vlan,
1350                         vf, status);
1351                 return be_cmd_status(status);
1352         }
1353
1354         vf_cfg->vlan_tag = vlan;
1355
1356         return 0;
1357 }
1358
1359 static int be_set_vf_tx_rate(struct net_device *netdev, int vf,
1360                              int min_tx_rate, int max_tx_rate)
1361 {
1362         struct be_adapter *adapter = netdev_priv(netdev);
1363         struct device *dev = &adapter->pdev->dev;
1364         int percent_rate, status = 0;
1365         u16 link_speed = 0;
1366         u8 link_status;
1367
1368         if (!sriov_enabled(adapter))
1369                 return -EPERM;
1370
1371         if (vf >= adapter->num_vfs)
1372                 return -EINVAL;
1373
1374         if (min_tx_rate)
1375                 return -EINVAL;
1376
1377         if (!max_tx_rate)
1378                 goto config_qos;
1379
1380         status = be_cmd_link_status_query(adapter, &link_speed,
1381                                           &link_status, 0);
1382         if (status)
1383                 goto err;
1384
1385         if (!link_status) {
1386                 dev_err(dev, "TX-rate setting not allowed when link is down\n");
1387                 status = -ENETDOWN;
1388                 goto err;
1389         }
1390
1391         if (max_tx_rate < 100 || max_tx_rate > link_speed) {
1392                 dev_err(dev, "TX-rate must be between 100 and %d Mbps\n",
1393                         link_speed);
1394                 status = -EINVAL;
1395                 goto err;
1396         }
1397
1398         /* On Skyhawk the QOS setting must be done only as a % value */
1399         percent_rate = link_speed / 100;
1400         if (skyhawk_chip(adapter) && (max_tx_rate % percent_rate)) {
1401                 dev_err(dev, "TX-rate must be a multiple of %d Mbps\n",
1402                         percent_rate);
1403                 status = -EINVAL;
1404                 goto err;
1405         }
1406
1407 config_qos:
1408         status = be_cmd_config_qos(adapter, max_tx_rate, link_speed, vf + 1);
1409         if (status)
1410                 goto err;
1411
1412         adapter->vf_cfg[vf].tx_rate = max_tx_rate;
1413         return 0;
1414
1415 err:
1416         dev_err(dev, "TX-rate setting of %dMbps on VF%d failed\n",
1417                 max_tx_rate, vf);
1418         return be_cmd_status(status);
1419 }
1420 static int be_set_vf_link_state(struct net_device *netdev, int vf,
1421                                 int link_state)
1422 {
1423         struct be_adapter *adapter = netdev_priv(netdev);
1424         int status;
1425
1426         if (!sriov_enabled(adapter))
1427                 return -EPERM;
1428
1429         if (vf >= adapter->num_vfs)
1430                 return -EINVAL;
1431
1432         status = be_cmd_set_logical_link_config(adapter, link_state, vf+1);
1433         if (status) {
1434                 dev_err(&adapter->pdev->dev,
1435                         "Link state change on VF %d failed: %#x\n", vf, status);
1436                 return be_cmd_status(status);
1437         }
1438
1439         adapter->vf_cfg[vf].plink_tracking = link_state;
1440
1441         return 0;
1442 }
1443
1444 static void be_aic_update(struct be_aic_obj *aic, u64 rx_pkts, u64 tx_pkts,
1445                           ulong now)
1446 {
1447         aic->rx_pkts_prev = rx_pkts;
1448         aic->tx_reqs_prev = tx_pkts;
1449         aic->jiffies = now;
1450 }
1451
1452 static void be_eqd_update(struct be_adapter *adapter)
1453 {
1454         struct be_set_eqd set_eqd[MAX_EVT_QS];
1455         int eqd, i, num = 0, start;
1456         struct be_aic_obj *aic;
1457         struct be_eq_obj *eqo;
1458         struct be_rx_obj *rxo;
1459         struct be_tx_obj *txo;
1460         u64 rx_pkts, tx_pkts;
1461         ulong now;
1462         u32 pps, delta;
1463
1464         for_all_evt_queues(adapter, eqo, i) {
1465                 aic = &adapter->aic_obj[eqo->idx];
1466                 if (!aic->enable) {
1467                         if (aic->jiffies)
1468                                 aic->jiffies = 0;
1469                         eqd = aic->et_eqd;
1470                         goto modify_eqd;
1471                 }
1472
1473                 rxo = &adapter->rx_obj[eqo->idx];
1474                 do {
1475                         start = u64_stats_fetch_begin_irq(&rxo->stats.sync);
1476                         rx_pkts = rxo->stats.rx_pkts;
1477                 } while (u64_stats_fetch_retry_irq(&rxo->stats.sync, start));
1478
1479                 txo = &adapter->tx_obj[eqo->idx];
1480                 do {
1481                         start = u64_stats_fetch_begin_irq(&txo->stats.sync);
1482                         tx_pkts = txo->stats.tx_reqs;
1483                 } while (u64_stats_fetch_retry_irq(&txo->stats.sync, start));
1484
1485
1486                 /* Skip, if wrapped around or first calculation */
1487                 now = jiffies;
1488                 if (!aic->jiffies || time_before(now, aic->jiffies) ||
1489                     rx_pkts < aic->rx_pkts_prev ||
1490                     tx_pkts < aic->tx_reqs_prev) {
1491                         be_aic_update(aic, rx_pkts, tx_pkts, now);
1492                         continue;
1493                 }
1494
1495                 delta = jiffies_to_msecs(now - aic->jiffies);
1496                 pps = (((u32)(rx_pkts - aic->rx_pkts_prev) * 1000) / delta) +
1497                         (((u32)(tx_pkts - aic->tx_reqs_prev) * 1000) / delta);
1498                 eqd = (pps / 15000) << 2;
1499
1500                 if (eqd < 8)
1501                         eqd = 0;
1502                 eqd = min_t(u32, eqd, aic->max_eqd);
1503                 eqd = max_t(u32, eqd, aic->min_eqd);
1504
1505                 be_aic_update(aic, rx_pkts, tx_pkts, now);
1506 modify_eqd:
1507                 if (eqd != aic->prev_eqd) {
1508                         set_eqd[num].delay_multiplier = (eqd * 65)/100;
1509                         set_eqd[num].eq_id = eqo->q.id;
1510                         aic->prev_eqd = eqd;
1511                         num++;
1512                 }
1513         }
1514
1515         if (num)
1516                 be_cmd_modify_eqd(adapter, set_eqd, num);
1517 }
1518
1519 static void be_rx_stats_update(struct be_rx_obj *rxo,
1520                                struct be_rx_compl_info *rxcp)
1521 {
1522         struct be_rx_stats *stats = rx_stats(rxo);
1523
1524         u64_stats_update_begin(&stats->sync);
1525         stats->rx_compl++;
1526         stats->rx_bytes += rxcp->pkt_size;
1527         stats->rx_pkts++;
1528         if (rxcp->pkt_type == BE_MULTICAST_PACKET)
1529                 stats->rx_mcast_pkts++;
1530         if (rxcp->err)
1531                 stats->rx_compl_err++;
1532         u64_stats_update_end(&stats->sync);
1533 }
1534
1535 static inline bool csum_passed(struct be_rx_compl_info *rxcp)
1536 {
1537         /* L4 checksum is not reliable for non TCP/UDP packets.
1538          * Also ignore ipcksm for ipv6 pkts
1539          */
1540         return (rxcp->tcpf || rxcp->udpf) && rxcp->l4_csum &&
1541                 (rxcp->ip_csum || rxcp->ipv6) && !rxcp->err;
1542 }
1543
1544 static struct be_rx_page_info *get_rx_page_info(struct be_rx_obj *rxo)
1545 {
1546         struct be_adapter *adapter = rxo->adapter;
1547         struct be_rx_page_info *rx_page_info;
1548         struct be_queue_info *rxq = &rxo->q;
1549         u16 frag_idx = rxq->tail;
1550
1551         rx_page_info = &rxo->page_info_tbl[frag_idx];
1552         BUG_ON(!rx_page_info->page);
1553
1554         if (rx_page_info->last_frag) {
1555                 dma_unmap_page(&adapter->pdev->dev,
1556                                dma_unmap_addr(rx_page_info, bus),
1557                                adapter->big_page_size, DMA_FROM_DEVICE);
1558                 rx_page_info->last_frag = false;
1559         } else {
1560                 dma_sync_single_for_cpu(&adapter->pdev->dev,
1561                                         dma_unmap_addr(rx_page_info, bus),
1562                                         rx_frag_size, DMA_FROM_DEVICE);
1563         }
1564
1565         queue_tail_inc(rxq);
1566         atomic_dec(&rxq->used);
1567         return rx_page_info;
1568 }
1569
1570 /* Throwaway the data in the Rx completion */
1571 static void be_rx_compl_discard(struct be_rx_obj *rxo,
1572                                 struct be_rx_compl_info *rxcp)
1573 {
1574         struct be_rx_page_info *page_info;
1575         u16 i, num_rcvd = rxcp->num_rcvd;
1576
1577         for (i = 0; i < num_rcvd; i++) {
1578                 page_info = get_rx_page_info(rxo);
1579                 put_page(page_info->page);
1580                 memset(page_info, 0, sizeof(*page_info));
1581         }
1582 }
1583
1584 /*
1585  * skb_fill_rx_data forms a complete skb for an ether frame
1586  * indicated by rxcp.
1587  */
1588 static void skb_fill_rx_data(struct be_rx_obj *rxo, struct sk_buff *skb,
1589                              struct be_rx_compl_info *rxcp)
1590 {
1591         struct be_rx_page_info *page_info;
1592         u16 i, j;
1593         u16 hdr_len, curr_frag_len, remaining;
1594         u8 *start;
1595
1596         page_info = get_rx_page_info(rxo);
1597         start = page_address(page_info->page) + page_info->page_offset;
1598         prefetch(start);
1599
1600         /* Copy data in the first descriptor of this completion */
1601         curr_frag_len = min(rxcp->pkt_size, rx_frag_size);
1602
1603         skb->len = curr_frag_len;
1604         if (curr_frag_len <= BE_HDR_LEN) { /* tiny packet */
1605                 memcpy(skb->data, start, curr_frag_len);
1606                 /* Complete packet has now been moved to data */
1607                 put_page(page_info->page);
1608                 skb->data_len = 0;
1609                 skb->tail += curr_frag_len;
1610         } else {
1611                 hdr_len = ETH_HLEN;
1612                 memcpy(skb->data, start, hdr_len);
1613                 skb_shinfo(skb)->nr_frags = 1;
1614                 skb_frag_set_page(skb, 0, page_info->page);
1615                 skb_shinfo(skb)->frags[0].page_offset =
1616                                         page_info->page_offset + hdr_len;
1617                 skb_frag_size_set(&skb_shinfo(skb)->frags[0],
1618                                   curr_frag_len - hdr_len);
1619                 skb->data_len = curr_frag_len - hdr_len;
1620                 skb->truesize += rx_frag_size;
1621                 skb->tail += hdr_len;
1622         }
1623         page_info->page = NULL;
1624
1625         if (rxcp->pkt_size <= rx_frag_size) {
1626                 BUG_ON(rxcp->num_rcvd != 1);
1627                 return;
1628         }
1629
1630         /* More frags present for this completion */
1631         remaining = rxcp->pkt_size - curr_frag_len;
1632         for (i = 1, j = 0; i < rxcp->num_rcvd; i++) {
1633                 page_info = get_rx_page_info(rxo);
1634                 curr_frag_len = min(remaining, rx_frag_size);
1635
1636                 /* Coalesce all frags from the same physical page in one slot */
1637                 if (page_info->page_offset == 0) {
1638                         /* Fresh page */
1639                         j++;
1640                         skb_frag_set_page(skb, j, page_info->page);
1641                         skb_shinfo(skb)->frags[j].page_offset =
1642                                                         page_info->page_offset;
1643                         skb_frag_size_set(&skb_shinfo(skb)->frags[j], 0);
1644                         skb_shinfo(skb)->nr_frags++;
1645                 } else {
1646                         put_page(page_info->page);
1647                 }
1648
1649                 skb_frag_size_add(&skb_shinfo(skb)->frags[j], curr_frag_len);
1650                 skb->len += curr_frag_len;
1651                 skb->data_len += curr_frag_len;
1652                 skb->truesize += rx_frag_size;
1653                 remaining -= curr_frag_len;
1654                 page_info->page = NULL;
1655         }
1656         BUG_ON(j > MAX_SKB_FRAGS);
1657 }
1658
1659 /* Process the RX completion indicated by rxcp when GRO is disabled */
1660 static void be_rx_compl_process(struct be_rx_obj *rxo, struct napi_struct *napi,
1661                                 struct be_rx_compl_info *rxcp)
1662 {
1663         struct be_adapter *adapter = rxo->adapter;
1664         struct net_device *netdev = adapter->netdev;
1665         struct sk_buff *skb;
1666
1667         skb = netdev_alloc_skb_ip_align(netdev, BE_RX_SKB_ALLOC_SIZE);
1668         if (unlikely(!skb)) {
1669                 rx_stats(rxo)->rx_drops_no_skbs++;
1670                 be_rx_compl_discard(rxo, rxcp);
1671                 return;
1672         }
1673
1674         skb_fill_rx_data(rxo, skb, rxcp);
1675
1676         if (likely((netdev->features & NETIF_F_RXCSUM) && csum_passed(rxcp)))
1677                 skb->ip_summed = CHECKSUM_UNNECESSARY;
1678         else
1679                 skb_checksum_none_assert(skb);
1680
1681         skb->protocol = eth_type_trans(skb, netdev);
1682         skb_record_rx_queue(skb, rxo - &adapter->rx_obj[0]);
1683         if (netdev->features & NETIF_F_RXHASH)
1684                 skb_set_hash(skb, rxcp->rss_hash, PKT_HASH_TYPE_L3);
1685
1686         skb->csum_level = rxcp->tunneled;
1687         skb_mark_napi_id(skb, napi);
1688
1689         if (rxcp->vlanf)
1690                 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), rxcp->vlan_tag);
1691
1692         netif_receive_skb(skb);
1693 }
1694
1695 /* Process the RX completion indicated by rxcp when GRO is enabled */
1696 static void be_rx_compl_process_gro(struct be_rx_obj *rxo,
1697                                     struct napi_struct *napi,
1698                                     struct be_rx_compl_info *rxcp)
1699 {
1700         struct be_adapter *adapter = rxo->adapter;
1701         struct be_rx_page_info *page_info;
1702         struct sk_buff *skb = NULL;
1703         u16 remaining, curr_frag_len;
1704         u16 i, j;
1705
1706         skb = napi_get_frags(napi);
1707         if (!skb) {
1708                 be_rx_compl_discard(rxo, rxcp);
1709                 return;
1710         }
1711
1712         remaining = rxcp->pkt_size;
1713         for (i = 0, j = -1; i < rxcp->num_rcvd; i++) {
1714                 page_info = get_rx_page_info(rxo);
1715
1716                 curr_frag_len = min(remaining, rx_frag_size);
1717
1718                 /* Coalesce all frags from the same physical page in one slot */
1719                 if (i == 0 || page_info->page_offset == 0) {
1720                         /* First frag or Fresh page */
1721                         j++;
1722                         skb_frag_set_page(skb, j, page_info->page);
1723                         skb_shinfo(skb)->frags[j].page_offset =
1724                                                         page_info->page_offset;
1725                         skb_frag_size_set(&skb_shinfo(skb)->frags[j], 0);
1726                 } else {
1727                         put_page(page_info->page);
1728                 }
1729                 skb_frag_size_add(&skb_shinfo(skb)->frags[j], curr_frag_len);
1730                 skb->truesize += rx_frag_size;
1731                 remaining -= curr_frag_len;
1732                 memset(page_info, 0, sizeof(*page_info));
1733         }
1734         BUG_ON(j > MAX_SKB_FRAGS);
1735
1736         skb_shinfo(skb)->nr_frags = j + 1;
1737         skb->len = rxcp->pkt_size;
1738         skb->data_len = rxcp->pkt_size;
1739         skb->ip_summed = CHECKSUM_UNNECESSARY;
1740         skb_record_rx_queue(skb, rxo - &adapter->rx_obj[0]);
1741         if (adapter->netdev->features & NETIF_F_RXHASH)
1742                 skb_set_hash(skb, rxcp->rss_hash, PKT_HASH_TYPE_L3);
1743
1744         skb->csum_level = rxcp->tunneled;
1745         skb_mark_napi_id(skb, napi);
1746
1747         if (rxcp->vlanf)
1748                 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), rxcp->vlan_tag);
1749
1750         napi_gro_frags(napi);
1751 }
1752
1753 static void be_parse_rx_compl_v1(struct be_eth_rx_compl *compl,
1754                                  struct be_rx_compl_info *rxcp)
1755 {
1756         rxcp->pkt_size = GET_RX_COMPL_V1_BITS(pktsize, compl);
1757         rxcp->vlanf = GET_RX_COMPL_V1_BITS(vtp, compl);
1758         rxcp->err = GET_RX_COMPL_V1_BITS(err, compl);
1759         rxcp->tcpf = GET_RX_COMPL_V1_BITS(tcpf, compl);
1760         rxcp->udpf = GET_RX_COMPL_V1_BITS(udpf, compl);
1761         rxcp->ip_csum = GET_RX_COMPL_V1_BITS(ipcksm, compl);
1762         rxcp->l4_csum = GET_RX_COMPL_V1_BITS(l4_cksm, compl);
1763         rxcp->ipv6 = GET_RX_COMPL_V1_BITS(ip_version, compl);
1764         rxcp->num_rcvd = GET_RX_COMPL_V1_BITS(numfrags, compl);
1765         rxcp->pkt_type = GET_RX_COMPL_V1_BITS(cast_enc, compl);
1766         rxcp->rss_hash = GET_RX_COMPL_V1_BITS(rsshash, compl);
1767         if (rxcp->vlanf) {
1768                 rxcp->qnq = GET_RX_COMPL_V1_BITS(qnq, compl);
1769                 rxcp->vlan_tag = GET_RX_COMPL_V1_BITS(vlan_tag, compl);
1770         }
1771         rxcp->port = GET_RX_COMPL_V1_BITS(port, compl);
1772         rxcp->tunneled =
1773                 GET_RX_COMPL_V1_BITS(tunneled, compl);
1774 }
1775
1776 static void be_parse_rx_compl_v0(struct be_eth_rx_compl *compl,
1777                                  struct be_rx_compl_info *rxcp)
1778 {
1779         rxcp->pkt_size = GET_RX_COMPL_V0_BITS(pktsize, compl);
1780         rxcp->vlanf = GET_RX_COMPL_V0_BITS(vtp, compl);
1781         rxcp->err = GET_RX_COMPL_V0_BITS(err, compl);
1782         rxcp->tcpf = GET_RX_COMPL_V0_BITS(tcpf, compl);
1783         rxcp->udpf = GET_RX_COMPL_V0_BITS(udpf, compl);
1784         rxcp->ip_csum = GET_RX_COMPL_V0_BITS(ipcksm, compl);
1785         rxcp->l4_csum = GET_RX_COMPL_V0_BITS(l4_cksm, compl);
1786         rxcp->ipv6 = GET_RX_COMPL_V0_BITS(ip_version, compl);
1787         rxcp->num_rcvd = GET_RX_COMPL_V0_BITS(numfrags, compl);
1788         rxcp->pkt_type = GET_RX_COMPL_V0_BITS(cast_enc, compl);
1789         rxcp->rss_hash = GET_RX_COMPL_V0_BITS(rsshash, compl);
1790         if (rxcp->vlanf) {
1791                 rxcp->qnq = GET_RX_COMPL_V0_BITS(qnq, compl);
1792                 rxcp->vlan_tag = GET_RX_COMPL_V0_BITS(vlan_tag, compl);
1793         }
1794         rxcp->port = GET_RX_COMPL_V0_BITS(port, compl);
1795         rxcp->ip_frag = GET_RX_COMPL_V0_BITS(ip_frag, compl);
1796 }
1797
1798 static struct be_rx_compl_info *be_rx_compl_get(struct be_rx_obj *rxo)
1799 {
1800         struct be_eth_rx_compl *compl = queue_tail_node(&rxo->cq);
1801         struct be_rx_compl_info *rxcp = &rxo->rxcp;
1802         struct be_adapter *adapter = rxo->adapter;
1803
1804         /* For checking the valid bit it is Ok to use either definition as the
1805          * valid bit is at the same position in both v0 and v1 Rx compl */
1806         if (compl->dw[offsetof(struct amap_eth_rx_compl_v1, valid) / 32] == 0)
1807                 return NULL;
1808
1809         rmb();
1810         be_dws_le_to_cpu(compl, sizeof(*compl));
1811
1812         if (adapter->be3_native)
1813                 be_parse_rx_compl_v1(compl, rxcp);
1814         else
1815                 be_parse_rx_compl_v0(compl, rxcp);
1816
1817         if (rxcp->ip_frag)
1818                 rxcp->l4_csum = 0;
1819
1820         if (rxcp->vlanf) {
1821                 /* In QNQ modes, if qnq bit is not set, then the packet was
1822                  * tagged only with the transparent outer vlan-tag and must
1823                  * not be treated as a vlan packet by host
1824                  */
1825                 if (be_is_qnq_mode(adapter) && !rxcp->qnq)
1826                         rxcp->vlanf = 0;
1827
1828                 if (!lancer_chip(adapter))
1829                         rxcp->vlan_tag = swab16(rxcp->vlan_tag);
1830
1831                 if (adapter->pvid == (rxcp->vlan_tag & VLAN_VID_MASK) &&
1832                     !test_bit(rxcp->vlan_tag, adapter->vids))
1833                         rxcp->vlanf = 0;
1834         }
1835
1836         /* As the compl has been parsed, reset it; we wont touch it again */
1837         compl->dw[offsetof(struct amap_eth_rx_compl_v1, valid) / 32] = 0;
1838
1839         queue_tail_inc(&rxo->cq);
1840         return rxcp;
1841 }
1842
1843 static inline struct page *be_alloc_pages(u32 size, gfp_t gfp)
1844 {
1845         u32 order = get_order(size);
1846
1847         if (order > 0)
1848                 gfp |= __GFP_COMP;
1849         return  alloc_pages(gfp, order);
1850 }
1851
1852 /*
1853  * Allocate a page, split it to fragments of size rx_frag_size and post as
1854  * receive buffers to BE
1855  */
1856 static void be_post_rx_frags(struct be_rx_obj *rxo, gfp_t gfp)
1857 {
1858         struct be_adapter *adapter = rxo->adapter;
1859         struct be_rx_page_info *page_info = NULL, *prev_page_info = NULL;
1860         struct be_queue_info *rxq = &rxo->q;
1861         struct page *pagep = NULL;
1862         struct device *dev = &adapter->pdev->dev;
1863         struct be_eth_rx_d *rxd;
1864         u64 page_dmaaddr = 0, frag_dmaaddr;
1865         u32 posted, page_offset = 0;
1866
1867         page_info = &rxo->page_info_tbl[rxq->head];
1868         for (posted = 0; posted < MAX_RX_POST && !page_info->page; posted++) {
1869                 if (!pagep) {
1870                         pagep = be_alloc_pages(adapter->big_page_size, gfp);
1871                         if (unlikely(!pagep)) {
1872                                 rx_stats(rxo)->rx_post_fail++;
1873                                 break;
1874                         }
1875                         page_dmaaddr = dma_map_page(dev, pagep, 0,
1876                                                     adapter->big_page_size,
1877                                                     DMA_FROM_DEVICE);
1878                         if (dma_mapping_error(dev, page_dmaaddr)) {
1879                                 put_page(pagep);
1880                                 pagep = NULL;
1881                                 adapter->drv_stats.dma_map_errors++;
1882                                 break;
1883                         }
1884                         page_offset = 0;
1885                 } else {
1886                         get_page(pagep);
1887                         page_offset += rx_frag_size;
1888                 }
1889                 page_info->page_offset = page_offset;
1890                 page_info->page = pagep;
1891
1892                 rxd = queue_head_node(rxq);
1893                 frag_dmaaddr = page_dmaaddr + page_info->page_offset;
1894                 rxd->fragpa_lo = cpu_to_le32(frag_dmaaddr & 0xFFFFFFFF);
1895                 rxd->fragpa_hi = cpu_to_le32(upper_32_bits(frag_dmaaddr));
1896
1897                 /* Any space left in the current big page for another frag? */
1898                 if ((page_offset + rx_frag_size + rx_frag_size) >
1899                                         adapter->big_page_size) {
1900                         pagep = NULL;
1901                         page_info->last_frag = true;
1902                         dma_unmap_addr_set(page_info, bus, page_dmaaddr);
1903                 } else {
1904                         dma_unmap_addr_set(page_info, bus, frag_dmaaddr);
1905                 }
1906
1907                 prev_page_info = page_info;
1908                 queue_head_inc(rxq);
1909                 page_info = &rxo->page_info_tbl[rxq->head];
1910         }
1911
1912         /* Mark the last frag of a page when we break out of the above loop
1913          * with no more slots available in the RXQ
1914          */
1915         if (pagep) {
1916                 prev_page_info->last_frag = true;
1917                 dma_unmap_addr_set(prev_page_info, bus, page_dmaaddr);
1918         }
1919
1920         if (posted) {
1921                 atomic_add(posted, &rxq->used);
1922                 if (rxo->rx_post_starved)
1923                         rxo->rx_post_starved = false;
1924                 be_rxq_notify(adapter, rxq->id, posted);
1925         } else if (atomic_read(&rxq->used) == 0) {
1926                 /* Let be_worker replenish when memory is available */
1927                 rxo->rx_post_starved = true;
1928         }
1929 }
1930
1931 static struct be_eth_tx_compl *be_tx_compl_get(struct be_queue_info *tx_cq)
1932 {
1933         struct be_eth_tx_compl *txcp = queue_tail_node(tx_cq);
1934
1935         if (txcp->dw[offsetof(struct amap_eth_tx_compl, valid) / 32] == 0)
1936                 return NULL;
1937
1938         rmb();
1939         be_dws_le_to_cpu(txcp, sizeof(*txcp));
1940
1941         txcp->dw[offsetof(struct amap_eth_tx_compl, valid) / 32] = 0;
1942
1943         queue_tail_inc(tx_cq);
1944         return txcp;
1945 }
1946
1947 static u16 be_tx_compl_process(struct be_adapter *adapter,
1948                                struct be_tx_obj *txo, u16 last_index)
1949 {
1950         struct be_queue_info *txq = &txo->q;
1951         struct be_eth_wrb *wrb;
1952         struct sk_buff **sent_skbs = txo->sent_skb_list;
1953         struct sk_buff *sent_skb;
1954         u16 cur_index, num_wrbs = 1; /* account for hdr wrb */
1955         bool unmap_skb_hdr = true;
1956
1957         sent_skb = sent_skbs[txq->tail];
1958         BUG_ON(!sent_skb);
1959         sent_skbs[txq->tail] = NULL;
1960
1961         /* skip header wrb */
1962         queue_tail_inc(txq);
1963
1964         do {
1965                 cur_index = txq->tail;
1966                 wrb = queue_tail_node(txq);
1967                 unmap_tx_frag(&adapter->pdev->dev, wrb,
1968                               (unmap_skb_hdr && skb_headlen(sent_skb)));
1969                 unmap_skb_hdr = false;
1970
1971                 num_wrbs++;
1972                 queue_tail_inc(txq);
1973         } while (cur_index != last_index);
1974
1975         dev_consume_skb_any(sent_skb);
1976         return num_wrbs;
1977 }
1978
1979 /* Return the number of events in the event queue */
1980 static inline int events_get(struct be_eq_obj *eqo)
1981 {
1982         struct be_eq_entry *eqe;
1983         int num = 0;
1984
1985         do {
1986                 eqe = queue_tail_node(&eqo->q);
1987                 if (eqe->evt == 0)
1988                         break;
1989
1990                 rmb();
1991                 eqe->evt = 0;
1992                 num++;
1993                 queue_tail_inc(&eqo->q);
1994         } while (true);
1995
1996         return num;
1997 }
1998
1999 /* Leaves the EQ is disarmed state */
2000 static void be_eq_clean(struct be_eq_obj *eqo)
2001 {
2002         int num = events_get(eqo);
2003
2004         be_eq_notify(eqo->adapter, eqo->q.id, false, true, num);
2005 }
2006
2007 static void be_rx_cq_clean(struct be_rx_obj *rxo)
2008 {
2009         struct be_rx_page_info *page_info;
2010         struct be_queue_info *rxq = &rxo->q;
2011         struct be_queue_info *rx_cq = &rxo->cq;
2012         struct be_rx_compl_info *rxcp;
2013         struct be_adapter *adapter = rxo->adapter;
2014         int flush_wait = 0;
2015
2016         /* Consume pending rx completions.
2017          * Wait for the flush completion (identified by zero num_rcvd)
2018          * to arrive. Notify CQ even when there are no more CQ entries
2019          * for HW to flush partially coalesced CQ entries.
2020          * In Lancer, there is no need to wait for flush compl.
2021          */
2022         for (;;) {
2023                 rxcp = be_rx_compl_get(rxo);
2024                 if (!rxcp) {
2025                         if (lancer_chip(adapter))
2026                                 break;
2027
2028                         if (flush_wait++ > 10 || be_hw_error(adapter)) {
2029                                 dev_warn(&adapter->pdev->dev,
2030                                          "did not receive flush compl\n");
2031                                 break;
2032                         }
2033                         be_cq_notify(adapter, rx_cq->id, true, 0);
2034                         mdelay(1);
2035                 } else {
2036                         be_rx_compl_discard(rxo, rxcp);
2037                         be_cq_notify(adapter, rx_cq->id, false, 1);
2038                         if (rxcp->num_rcvd == 0)
2039                                 break;
2040                 }
2041         }
2042
2043         /* After cleanup, leave the CQ in unarmed state */
2044         be_cq_notify(adapter, rx_cq->id, false, 0);
2045
2046         /* Then free posted rx buffers that were not used */
2047         while (atomic_read(&rxq->used) > 0) {
2048                 page_info = get_rx_page_info(rxo);
2049                 put_page(page_info->page);
2050                 memset(page_info, 0, sizeof(*page_info));
2051         }
2052         BUG_ON(atomic_read(&rxq->used));
2053         rxq->tail = rxq->head = 0;
2054 }
2055
2056 static void be_tx_compl_clean(struct be_adapter *adapter)
2057 {
2058         struct be_tx_obj *txo;
2059         struct be_queue_info *txq;
2060         struct be_eth_tx_compl *txcp;
2061         u16 end_idx, cmpl = 0, timeo = 0, num_wrbs = 0;
2062         struct sk_buff *sent_skb;
2063         bool dummy_wrb;
2064         int i, pending_txqs;
2065
2066         /* Stop polling for compls when HW has been silent for 10ms */
2067         do {
2068                 pending_txqs = adapter->num_tx_qs;
2069
2070                 for_all_tx_queues(adapter, txo, i) {
2071                         cmpl = 0;
2072                         num_wrbs = 0;
2073                         txq = &txo->q;
2074                         while ((txcp = be_tx_compl_get(&txo->cq))) {
2075                                 end_idx = GET_TX_COMPL_BITS(wrb_index, txcp);
2076                                 num_wrbs += be_tx_compl_process(adapter, txo,
2077                                                                 end_idx);
2078                                 cmpl++;
2079                         }
2080                         if (cmpl) {
2081                                 be_cq_notify(adapter, txo->cq.id, false, cmpl);
2082                                 atomic_sub(num_wrbs, &txq->used);
2083                                 timeo = 0;
2084                         }
2085                         if (atomic_read(&txq->used) == 0)
2086                                 pending_txqs--;
2087                 }
2088
2089                 if (pending_txqs == 0 || ++timeo > 10 || be_hw_error(adapter))
2090                         break;
2091
2092                 mdelay(1);
2093         } while (true);
2094
2095         for_all_tx_queues(adapter, txo, i) {
2096                 txq = &txo->q;
2097                 if (atomic_read(&txq->used))
2098                         dev_err(&adapter->pdev->dev, "%d pending tx-compls\n",
2099                                 atomic_read(&txq->used));
2100
2101                 /* free posted tx for which compls will never arrive */
2102                 while (atomic_read(&txq->used)) {
2103                         sent_skb = txo->sent_skb_list[txq->tail];
2104                         end_idx = txq->tail;
2105                         num_wrbs = wrb_cnt_for_skb(adapter, sent_skb,
2106                                                    &dummy_wrb);
2107                         index_adv(&end_idx, num_wrbs - 1, txq->len);
2108                         num_wrbs = be_tx_compl_process(adapter, txo, end_idx);
2109                         atomic_sub(num_wrbs, &txq->used);
2110                 }
2111         }
2112 }
2113
2114 static void be_evt_queues_destroy(struct be_adapter *adapter)
2115 {
2116         struct be_eq_obj *eqo;
2117         int i;
2118
2119         for_all_evt_queues(adapter, eqo, i) {
2120                 if (eqo->q.created) {
2121                         be_eq_clean(eqo);
2122                         be_cmd_q_destroy(adapter, &eqo->q, QTYPE_EQ);
2123                         napi_hash_del(&eqo->napi);
2124                         netif_napi_del(&eqo->napi);
2125                 }
2126                 be_queue_free(adapter, &eqo->q);
2127         }
2128 }
2129
2130 static int be_evt_queues_create(struct be_adapter *adapter)
2131 {
2132         struct be_queue_info *eq;
2133         struct be_eq_obj *eqo;
2134         struct be_aic_obj *aic;
2135         int i, rc;
2136
2137         adapter->num_evt_qs = min_t(u16, num_irqs(adapter),
2138                                     adapter->cfg_num_qs);
2139
2140         for_all_evt_queues(adapter, eqo, i) {
2141                 netif_napi_add(adapter->netdev, &eqo->napi, be_poll,
2142                                BE_NAPI_WEIGHT);
2143                 napi_hash_add(&eqo->napi);
2144                 aic = &adapter->aic_obj[i];
2145                 eqo->adapter = adapter;
2146                 eqo->tx_budget = BE_TX_BUDGET;
2147                 eqo->idx = i;
2148                 aic->max_eqd = BE_MAX_EQD;
2149                 aic->enable = true;
2150
2151                 eq = &eqo->q;
2152                 rc = be_queue_alloc(adapter, eq, EVNT_Q_LEN,
2153                                     sizeof(struct be_eq_entry));
2154                 if (rc)
2155                         return rc;
2156
2157                 rc = be_cmd_eq_create(adapter, eqo);
2158                 if (rc)
2159                         return rc;
2160         }
2161         return 0;
2162 }
2163
2164 static void be_mcc_queues_destroy(struct be_adapter *adapter)
2165 {
2166         struct be_queue_info *q;
2167
2168         q = &adapter->mcc_obj.q;
2169         if (q->created)
2170                 be_cmd_q_destroy(adapter, q, QTYPE_MCCQ);
2171         be_queue_free(adapter, q);
2172
2173         q = &adapter->mcc_obj.cq;
2174         if (q->created)
2175                 be_cmd_q_destroy(adapter, q, QTYPE_CQ);
2176         be_queue_free(adapter, q);
2177 }
2178
2179 /* Must be called only after TX qs are created as MCC shares TX EQ */
2180 static int be_mcc_queues_create(struct be_adapter *adapter)
2181 {
2182         struct be_queue_info *q, *cq;
2183
2184         cq = &adapter->mcc_obj.cq;
2185         if (be_queue_alloc(adapter, cq, MCC_CQ_LEN,
2186                            sizeof(struct be_mcc_compl)))
2187                 goto err;
2188
2189         /* Use the default EQ for MCC completions */
2190         if (be_cmd_cq_create(adapter, cq, &mcc_eqo(adapter)->q, true, 0))
2191                 goto mcc_cq_free;
2192
2193         q = &adapter->mcc_obj.q;
2194         if (be_queue_alloc(adapter, q, MCC_Q_LEN, sizeof(struct be_mcc_wrb)))
2195                 goto mcc_cq_destroy;
2196
2197         if (be_cmd_mccq_create(adapter, q, cq))
2198                 goto mcc_q_free;
2199
2200         return 0;
2201
2202 mcc_q_free:
2203         be_queue_free(adapter, q);
2204 mcc_cq_destroy:
2205         be_cmd_q_destroy(adapter, cq, QTYPE_CQ);
2206 mcc_cq_free:
2207         be_queue_free(adapter, cq);
2208 err:
2209         return -1;
2210 }
2211
2212 static void be_tx_queues_destroy(struct be_adapter *adapter)
2213 {
2214         struct be_queue_info *q;
2215         struct be_tx_obj *txo;
2216         u8 i;
2217
2218         for_all_tx_queues(adapter, txo, i) {
2219                 q = &txo->q;
2220                 if (q->created)
2221                         be_cmd_q_destroy(adapter, q, QTYPE_TXQ);
2222                 be_queue_free(adapter, q);
2223
2224                 q = &txo->cq;
2225                 if (q->created)
2226                         be_cmd_q_destroy(adapter, q, QTYPE_CQ);
2227                 be_queue_free(adapter, q);
2228         }
2229 }
2230
2231 static int be_tx_qs_create(struct be_adapter *adapter)
2232 {
2233         struct be_queue_info *cq, *eq;
2234         struct be_tx_obj *txo;
2235         int status, i;
2236
2237         adapter->num_tx_qs = min(adapter->num_evt_qs, be_max_txqs(adapter));
2238
2239         for_all_tx_queues(adapter, txo, i) {
2240                 cq = &txo->cq;
2241                 status = be_queue_alloc(adapter, cq, TX_CQ_LEN,
2242                                         sizeof(struct be_eth_tx_compl));
2243                 if (status)
2244                         return status;
2245
2246                 u64_stats_init(&txo->stats.sync);
2247                 u64_stats_init(&txo->stats.sync_compl);
2248
2249                 /* If num_evt_qs is less than num_tx_qs, then more than
2250                  * one txq share an eq
2251                  */
2252                 eq = &adapter->eq_obj[i % adapter->num_evt_qs].q;
2253                 status = be_cmd_cq_create(adapter, cq, eq, false, 3);
2254                 if (status)
2255                         return status;
2256
2257                 status = be_queue_alloc(adapter, &txo->q, TX_Q_LEN,
2258                                         sizeof(struct be_eth_wrb));
2259                 if (status)
2260                         return status;
2261
2262                 status = be_cmd_txq_create(adapter, txo);
2263                 if (status)
2264                         return status;
2265         }
2266
2267         dev_info(&adapter->pdev->dev, "created %d TX queue(s)\n",
2268                  adapter->num_tx_qs);
2269         return 0;
2270 }
2271
2272 static void be_rx_cqs_destroy(struct be_adapter *adapter)
2273 {
2274         struct be_queue_info *q;
2275         struct be_rx_obj *rxo;
2276         int i;
2277
2278         for_all_rx_queues(adapter, rxo, i) {
2279                 q = &rxo->cq;
2280                 if (q->created)
2281                         be_cmd_q_destroy(adapter, q, QTYPE_CQ);
2282                 be_queue_free(adapter, q);
2283         }
2284 }
2285
2286 static int be_rx_cqs_create(struct be_adapter *adapter)
2287 {
2288         struct be_queue_info *eq, *cq;
2289         struct be_rx_obj *rxo;
2290         int rc, i;
2291
2292         /* We can create as many RSS rings as there are EQs. */
2293         adapter->num_rx_qs = adapter->num_evt_qs;
2294
2295         /* We'll use RSS only if atleast 2 RSS rings are supported.
2296          * When RSS is used, we'll need a default RXQ for non-IP traffic.
2297          */
2298         if (adapter->num_rx_qs > 1)
2299                 adapter->num_rx_qs++;
2300
2301         adapter->big_page_size = (1 << get_order(rx_frag_size)) * PAGE_SIZE;
2302         for_all_rx_queues(adapter, rxo, i) {
2303                 rxo->adapter = adapter;
2304                 cq = &rxo->cq;
2305                 rc = be_queue_alloc(adapter, cq, RX_CQ_LEN,
2306                                     sizeof(struct be_eth_rx_compl));
2307                 if (rc)
2308                         return rc;
2309
2310                 u64_stats_init(&rxo->stats.sync);
2311                 eq = &adapter->eq_obj[i % adapter->num_evt_qs].q;
2312                 rc = be_cmd_cq_create(adapter, cq, eq, false, 3);
2313                 if (rc)
2314                         return rc;
2315         }
2316
2317         dev_info(&adapter->pdev->dev,
2318                  "created %d RSS queue(s) and 1 default RX queue\n",
2319                  adapter->num_rx_qs - 1);
2320         return 0;
2321 }
2322
2323 static irqreturn_t be_intx(int irq, void *dev)
2324 {
2325         struct be_eq_obj *eqo = dev;
2326         struct be_adapter *adapter = eqo->adapter;
2327         int num_evts = 0;
2328
2329         /* IRQ is not expected when NAPI is scheduled as the EQ
2330          * will not be armed.
2331          * But, this can happen on Lancer INTx where it takes
2332          * a while to de-assert INTx or in BE2 where occasionaly
2333          * an interrupt may be raised even when EQ is unarmed.
2334          * If NAPI is already scheduled, then counting & notifying
2335          * events will orphan them.
2336          */
2337         if (napi_schedule_prep(&eqo->napi)) {
2338                 num_evts = events_get(eqo);
2339                 __napi_schedule(&eqo->napi);
2340                 if (num_evts)
2341                         eqo->spurious_intr = 0;
2342         }
2343         be_eq_notify(adapter, eqo->q.id, false, true, num_evts);
2344
2345         /* Return IRQ_HANDLED only for the the first spurious intr
2346          * after a valid intr to stop the kernel from branding
2347          * this irq as a bad one!
2348          */
2349         if (num_evts || eqo->spurious_intr++ == 0)
2350                 return IRQ_HANDLED;
2351         else
2352                 return IRQ_NONE;
2353 }
2354
2355 static irqreturn_t be_msix(int irq, void *dev)
2356 {
2357         struct be_eq_obj *eqo = dev;
2358
2359         be_eq_notify(eqo->adapter, eqo->q.id, false, true, 0);
2360         napi_schedule(&eqo->napi);
2361         return IRQ_HANDLED;
2362 }
2363
2364 static inline bool do_gro(struct be_rx_compl_info *rxcp)
2365 {
2366         return (rxcp->tcpf && !rxcp->err && rxcp->l4_csum) ? true : false;
2367 }
2368
2369 static int be_process_rx(struct be_rx_obj *rxo, struct napi_struct *napi,
2370                          int budget, int polling)
2371 {
2372         struct be_adapter *adapter = rxo->adapter;
2373         struct be_queue_info *rx_cq = &rxo->cq;
2374         struct be_rx_compl_info *rxcp;
2375         u32 work_done;
2376
2377         for (work_done = 0; work_done < budget; work_done++) {
2378                 rxcp = be_rx_compl_get(rxo);
2379                 if (!rxcp)
2380                         break;
2381
2382                 /* Is it a flush compl that has no data */
2383                 if (unlikely(rxcp->num_rcvd == 0))
2384                         goto loop_continue;
2385
2386                 /* Discard compl with partial DMA Lancer B0 */
2387                 if (unlikely(!rxcp->pkt_size)) {
2388                         be_rx_compl_discard(rxo, rxcp);
2389                         goto loop_continue;
2390                 }
2391
2392                 /* On BE drop pkts that arrive due to imperfect filtering in
2393                  * promiscuous mode on some skews
2394                  */
2395                 if (unlikely(rxcp->port != adapter->port_num &&
2396                              !lancer_chip(adapter))) {
2397                         be_rx_compl_discard(rxo, rxcp);
2398                         goto loop_continue;
2399                 }
2400
2401                 /* Don't do gro when we're busy_polling */
2402                 if (do_gro(rxcp) && polling != BUSY_POLLING)
2403                         be_rx_compl_process_gro(rxo, napi, rxcp);
2404                 else
2405                         be_rx_compl_process(rxo, napi, rxcp);
2406
2407 loop_continue:
2408                 be_rx_stats_update(rxo, rxcp);
2409         }
2410
2411         if (work_done) {
2412                 be_cq_notify(adapter, rx_cq->id, true, work_done);
2413
2414                 /* When an rx-obj gets into post_starved state, just
2415                  * let be_worker do the posting.
2416                  */
2417                 if (atomic_read(&rxo->q.used) < RX_FRAGS_REFILL_WM &&
2418                     !rxo->rx_post_starved)
2419                         be_post_rx_frags(rxo, GFP_ATOMIC);
2420         }
2421
2422         return work_done;
2423 }
2424
2425 static inline void be_update_tx_err(struct be_tx_obj *txo, u32 status)
2426 {
2427         switch (status) {
2428         case BE_TX_COMP_HDR_PARSE_ERR:
2429                 tx_stats(txo)->tx_hdr_parse_err++;
2430                 break;
2431         case BE_TX_COMP_NDMA_ERR:
2432                 tx_stats(txo)->tx_dma_err++;
2433                 break;
2434         case BE_TX_COMP_ACL_ERR:
2435                 tx_stats(txo)->tx_spoof_check_err++;
2436                 break;
2437         }
2438 }
2439
2440 static inline void lancer_update_tx_err(struct be_tx_obj *txo, u32 status)
2441 {
2442         switch (status) {
2443         case LANCER_TX_COMP_LSO_ERR:
2444                 tx_stats(txo)->tx_tso_err++;
2445                 break;
2446         case LANCER_TX_COMP_HSW_DROP_MAC_ERR:
2447         case LANCER_TX_COMP_HSW_DROP_VLAN_ERR:
2448                 tx_stats(txo)->tx_spoof_check_err++;
2449                 break;
2450         case LANCER_TX_COMP_QINQ_ERR:
2451                 tx_stats(txo)->tx_qinq_err++;
2452                 break;
2453         case LANCER_TX_COMP_PARITY_ERR:
2454                 tx_stats(txo)->tx_internal_parity_err++;
2455                 break;
2456         case LANCER_TX_COMP_DMA_ERR:
2457                 tx_stats(txo)->tx_dma_err++;
2458                 break;
2459         }
2460 }
2461
2462 static bool be_process_tx(struct be_adapter *adapter, struct be_tx_obj *txo,
2463                           int budget, int idx)
2464 {
2465         struct be_eth_tx_compl *txcp;
2466         int num_wrbs = 0, work_done;
2467         u32 compl_status;
2468
2469         for (work_done = 0; work_done < budget; work_done++) {
2470                 txcp = be_tx_compl_get(&txo->cq);
2471                 if (!txcp)
2472                         break;
2473                 num_wrbs += be_tx_compl_process(adapter, txo,
2474                                                 GET_TX_COMPL_BITS(wrb_index,
2475                                                                   txcp));
2476                 compl_status = GET_TX_COMPL_BITS(status, txcp);
2477                 if (compl_status) {
2478                         if (lancer_chip(adapter))
2479                                 lancer_update_tx_err(txo, compl_status);
2480                         else
2481                                 be_update_tx_err(txo, compl_status);
2482                 }
2483         }
2484
2485         if (work_done) {
2486                 be_cq_notify(adapter, txo->cq.id, true, work_done);
2487                 atomic_sub(num_wrbs, &txo->q.used);
2488
2489                 /* As Tx wrbs have been freed up, wake up netdev queue
2490                  * if it was stopped due to lack of tx wrbs.  */
2491                 if (__netif_subqueue_stopped(adapter->netdev, idx) &&
2492                     atomic_read(&txo->q.used) < txo->q.len / 2) {
2493                         netif_wake_subqueue(adapter->netdev, idx);
2494                 }
2495
2496                 u64_stats_update_begin(&tx_stats(txo)->sync_compl);
2497                 tx_stats(txo)->tx_compl += work_done;
2498                 u64_stats_update_end(&tx_stats(txo)->sync_compl);
2499         }
2500         return (work_done < budget); /* Done */
2501 }
2502
2503 int be_poll(struct napi_struct *napi, int budget)
2504 {
2505         struct be_eq_obj *eqo = container_of(napi, struct be_eq_obj, napi);
2506         struct be_adapter *adapter = eqo->adapter;
2507         int max_work = 0, work, i, num_evts;
2508         struct be_rx_obj *rxo;
2509         bool tx_done;
2510
2511         num_evts = events_get(eqo);
2512
2513         /* Process all TXQs serviced by this EQ */
2514         for (i = eqo->idx; i < adapter->num_tx_qs; i += adapter->num_evt_qs) {
2515                 tx_done = be_process_tx(adapter, &adapter->tx_obj[i],
2516                                         eqo->tx_budget, i);
2517                 if (!tx_done)
2518                         max_work = budget;
2519         }
2520
2521         if (be_lock_napi(eqo)) {
2522                 /* This loop will iterate twice for EQ0 in which
2523                  * completions of the last RXQ (default one) are also processed
2524                  * For other EQs the loop iterates only once
2525                  */
2526                 for_all_rx_queues_on_eq(adapter, eqo, rxo, i) {
2527                         work = be_process_rx(rxo, napi, budget, NAPI_POLLING);
2528                         max_work = max(work, max_work);
2529                 }
2530                 be_unlock_napi(eqo);
2531         } else {
2532                 max_work = budget;
2533         }
2534
2535         if (is_mcc_eqo(eqo))
2536                 be_process_mcc(adapter);
2537
2538         if (max_work < budget) {
2539                 napi_complete(napi);
2540                 be_eq_notify(adapter, eqo->q.id, true, false, num_evts);
2541         } else {
2542                 /* As we'll continue in polling mode, count and clear events */
2543                 be_eq_notify(adapter, eqo->q.id, false, false, num_evts);
2544         }
2545         return max_work;
2546 }
2547
2548 #ifdef CONFIG_NET_RX_BUSY_POLL
2549 static int be_busy_poll(struct napi_struct *napi)
2550 {
2551         struct be_eq_obj *eqo = container_of(napi, struct be_eq_obj, napi);
2552         struct be_adapter *adapter = eqo->adapter;
2553         struct be_rx_obj *rxo;
2554         int i, work = 0;
2555
2556         if (!be_lock_busy_poll(eqo))
2557                 return LL_FLUSH_BUSY;
2558
2559         for_all_rx_queues_on_eq(adapter, eqo, rxo, i) {
2560                 work = be_process_rx(rxo, napi, 4, BUSY_POLLING);
2561                 if (work)
2562                         break;
2563         }
2564
2565         be_unlock_busy_poll(eqo);
2566         return work;
2567 }
2568 #endif
2569
2570 void be_detect_error(struct be_adapter *adapter)
2571 {
2572         u32 ue_lo = 0, ue_hi = 0, ue_lo_mask = 0, ue_hi_mask = 0;
2573         u32 sliport_status = 0, sliport_err1 = 0, sliport_err2 = 0;
2574         u32 i;
2575         bool error_detected = false;
2576         struct device *dev = &adapter->pdev->dev;
2577         struct net_device *netdev = adapter->netdev;
2578
2579         if (be_hw_error(adapter))
2580                 return;
2581
2582         if (lancer_chip(adapter)) {
2583                 sliport_status = ioread32(adapter->db + SLIPORT_STATUS_OFFSET);
2584                 if (sliport_status & SLIPORT_STATUS_ERR_MASK) {
2585                         sliport_err1 = ioread32(adapter->db +
2586                                                 SLIPORT_ERROR1_OFFSET);
2587                         sliport_err2 = ioread32(adapter->db +
2588                                                 SLIPORT_ERROR2_OFFSET);
2589                         adapter->hw_error = true;
2590                         /* Do not log error messages if its a FW reset */
2591                         if (sliport_err1 == SLIPORT_ERROR_FW_RESET1 &&
2592                             sliport_err2 == SLIPORT_ERROR_FW_RESET2) {
2593                                 dev_info(dev, "Firmware update in progress\n");
2594                         } else {
2595                                 error_detected = true;
2596                                 dev_err(dev, "Error detected in the card\n");
2597                                 dev_err(dev, "ERR: sliport status 0x%x\n",
2598                                         sliport_status);
2599                                 dev_err(dev, "ERR: sliport error1 0x%x\n",
2600                                         sliport_err1);
2601                                 dev_err(dev, "ERR: sliport error2 0x%x\n",
2602                                         sliport_err2);
2603                         }
2604                 }
2605         } else {
2606                 pci_read_config_dword(adapter->pdev,
2607                                       PCICFG_UE_STATUS_LOW, &ue_lo);
2608                 pci_read_config_dword(adapter->pdev,
2609                                       PCICFG_UE_STATUS_HIGH, &ue_hi);
2610                 pci_read_config_dword(adapter->pdev,
2611                                       PCICFG_UE_STATUS_LOW_MASK, &ue_lo_mask);
2612                 pci_read_config_dword(adapter->pdev,
2613                                       PCICFG_UE_STATUS_HI_MASK, &ue_hi_mask);
2614
2615                 ue_lo = (ue_lo & ~ue_lo_mask);
2616                 ue_hi = (ue_hi & ~ue_hi_mask);
2617
2618                 /* On certain platforms BE hardware can indicate spurious UEs.
2619                  * Allow HW to stop working completely in case of a real UE.
2620                  * Hence not setting the hw_error for UE detection.
2621                  */
2622
2623                 if (ue_lo || ue_hi) {
2624                         error_detected = true;
2625                         dev_err(dev,
2626                                 "Unrecoverable Error detected in the adapter");
2627                         dev_err(dev, "Please reboot server to recover");
2628                         if (skyhawk_chip(adapter))
2629                                 adapter->hw_error = true;
2630                         for (i = 0; ue_lo; ue_lo >>= 1, i++) {
2631                                 if (ue_lo & 1)
2632                                         dev_err(dev, "UE: %s bit set\n",
2633                                                 ue_status_low_desc[i]);
2634                         }
2635                         for (i = 0; ue_hi; ue_hi >>= 1, i++) {
2636                                 if (ue_hi & 1)
2637                                         dev_err(dev, "UE: %s bit set\n",
2638                                                 ue_status_hi_desc[i]);
2639                         }
2640                 }
2641         }
2642         if (error_detected)
2643                 netif_carrier_off(netdev);
2644 }
2645
2646 static void be_msix_disable(struct be_adapter *adapter)
2647 {
2648         if (msix_enabled(adapter)) {
2649                 pci_disable_msix(adapter->pdev);
2650                 adapter->num_msix_vec = 0;
2651                 adapter->num_msix_roce_vec = 0;
2652         }
2653 }
2654
2655 static int be_msix_enable(struct be_adapter *adapter)
2656 {
2657         int i, num_vec;
2658         struct device *dev = &adapter->pdev->dev;
2659
2660         /* If RoCE is supported, program the max number of NIC vectors that
2661          * may be configured via set-channels, along with vectors needed for
2662          * RoCe. Else, just program the number we'll use initially.
2663          */
2664         if (be_roce_supported(adapter))
2665                 num_vec = min_t(int, 2 * be_max_eqs(adapter),
2666                                 2 * num_online_cpus());
2667         else
2668                 num_vec = adapter->cfg_num_qs;
2669
2670         for (i = 0; i < num_vec; i++)
2671                 adapter->msix_entries[i].entry = i;
2672
2673         num_vec = pci_enable_msix_range(adapter->pdev, adapter->msix_entries,
2674                                         MIN_MSIX_VECTORS, num_vec);
2675         if (num_vec < 0)
2676                 goto fail;
2677
2678         if (be_roce_supported(adapter) && num_vec > MIN_MSIX_VECTORS) {
2679                 adapter->num_msix_roce_vec = num_vec / 2;
2680                 dev_info(dev, "enabled %d MSI-x vector(s) for RoCE\n",
2681                          adapter->num_msix_roce_vec);
2682         }
2683
2684         adapter->num_msix_vec = num_vec - adapter->num_msix_roce_vec;
2685
2686         dev_info(dev, "enabled %d MSI-x vector(s) for NIC\n",
2687                  adapter->num_msix_vec);
2688         return 0;
2689
2690 fail:
2691         dev_warn(dev, "MSIx enable failed\n");
2692
2693         /* INTx is not supported in VFs, so fail probe if enable_msix fails */
2694         if (!be_physfn(adapter))
2695                 return num_vec;
2696         return 0;
2697 }
2698
2699 static inline int be_msix_vec_get(struct be_adapter *adapter,
2700                                   struct be_eq_obj *eqo)
2701 {
2702         return adapter->msix_entries[eqo->msix_idx].vector;
2703 }
2704
2705 static int be_msix_register(struct be_adapter *adapter)
2706 {
2707         struct net_device *netdev = adapter->netdev;
2708         struct be_eq_obj *eqo;
2709         int status, i, vec;
2710
2711         for_all_evt_queues(adapter, eqo, i) {
2712                 sprintf(eqo->desc, "%s-q%d", netdev->name, i);
2713                 vec = be_msix_vec_get(adapter, eqo);
2714                 status = request_irq(vec, be_msix, 0, eqo->desc, eqo);
2715                 if (status)
2716                         goto err_msix;
2717         }
2718
2719         return 0;
2720 err_msix:
2721         for (i--, eqo = &adapter->eq_obj[i]; i >= 0; i--, eqo--)
2722                 free_irq(be_msix_vec_get(adapter, eqo), eqo);
2723         dev_warn(&adapter->pdev->dev, "MSIX Request IRQ failed - err %d\n",
2724                  status);
2725         be_msix_disable(adapter);
2726         return status;
2727 }
2728
2729 static int be_irq_register(struct be_adapter *adapter)
2730 {
2731         struct net_device *netdev = adapter->netdev;
2732         int status;
2733
2734         if (msix_enabled(adapter)) {
2735                 status = be_msix_register(adapter);
2736                 if (status == 0)
2737                         goto done;
2738                 /* INTx is not supported for VF */
2739                 if (!be_physfn(adapter))
2740                         return status;
2741         }
2742
2743         /* INTx: only the first EQ is used */
2744         netdev->irq = adapter->pdev->irq;
2745         status = request_irq(netdev->irq, be_intx, IRQF_SHARED, netdev->name,
2746                              &adapter->eq_obj[0]);
2747         if (status) {
2748                 dev_err(&adapter->pdev->dev,
2749                         "INTx request IRQ failed - err %d\n", status);
2750                 return status;
2751         }
2752 done:
2753         adapter->isr_registered = true;
2754         return 0;
2755 }
2756
2757 static void be_irq_unregister(struct be_adapter *adapter)
2758 {
2759         struct net_device *netdev = adapter->netdev;
2760         struct be_eq_obj *eqo;
2761         int i;
2762
2763         if (!adapter->isr_registered)
2764                 return;
2765
2766         /* INTx */
2767         if (!msix_enabled(adapter)) {
2768                 free_irq(netdev->irq, &adapter->eq_obj[0]);
2769                 goto done;
2770         }
2771
2772         /* MSIx */
2773         for_all_evt_queues(adapter, eqo, i)
2774                 free_irq(be_msix_vec_get(adapter, eqo), eqo);
2775
2776 done:
2777         adapter->isr_registered = false;
2778 }
2779
2780 static void be_rx_qs_destroy(struct be_adapter *adapter)
2781 {
2782         struct be_queue_info *q;
2783         struct be_rx_obj *rxo;
2784         int i;
2785
2786         for_all_rx_queues(adapter, rxo, i) {
2787                 q = &rxo->q;
2788                 if (q->created) {
2789                         be_cmd_rxq_destroy(adapter, q);
2790                         be_rx_cq_clean(rxo);
2791                 }
2792                 be_queue_free(adapter, q);
2793         }
2794 }
2795
2796 static int be_close(struct net_device *netdev)
2797 {
2798         struct be_adapter *adapter = netdev_priv(netdev);
2799         struct be_eq_obj *eqo;
2800         int i;
2801
2802         /* This protection is needed as be_close() may be called even when the
2803          * adapter is in cleared state (after eeh perm failure)
2804          */
2805         if (!(adapter->flags & BE_FLAGS_SETUP_DONE))
2806                 return 0;
2807
2808         be_roce_dev_close(adapter);
2809
2810         if (adapter->flags & BE_FLAGS_NAPI_ENABLED) {
2811                 for_all_evt_queues(adapter, eqo, i) {
2812                         napi_disable(&eqo->napi);
2813                         be_disable_busy_poll(eqo);
2814                 }
2815                 adapter->flags &= ~BE_FLAGS_NAPI_ENABLED;
2816         }
2817
2818         be_async_mcc_disable(adapter);
2819
2820         /* Wait for all pending tx completions to arrive so that
2821          * all tx skbs are freed.
2822          */
2823         netif_tx_disable(netdev);
2824         be_tx_compl_clean(adapter);
2825
2826         be_rx_qs_destroy(adapter);
2827
2828         for (i = 1; i < (adapter->uc_macs + 1); i++)
2829                 be_cmd_pmac_del(adapter, adapter->if_handle,
2830                                 adapter->pmac_id[i], 0);
2831         adapter->uc_macs = 0;
2832
2833         for_all_evt_queues(adapter, eqo, i) {
2834                 if (msix_enabled(adapter))
2835                         synchronize_irq(be_msix_vec_get(adapter, eqo));
2836                 else
2837                         synchronize_irq(netdev->irq);
2838                 be_eq_clean(eqo);
2839         }
2840
2841         be_irq_unregister(adapter);
2842
2843         return 0;
2844 }
2845
2846 static int be_rx_qs_create(struct be_adapter *adapter)
2847 {
2848         struct be_rx_obj *rxo;
2849         int rc, i, j;
2850         u8 rss_hkey[RSS_HASH_KEY_LEN];
2851         struct rss_info *rss = &adapter->rss_info;
2852
2853         for_all_rx_queues(adapter, rxo, i) {
2854                 rc = be_queue_alloc(adapter, &rxo->q, RX_Q_LEN,
2855                                     sizeof(struct be_eth_rx_d));
2856                 if (rc)
2857                         return rc;
2858         }
2859
2860         /* The FW would like the default RXQ to be created first */
2861         rxo = default_rxo(adapter);
2862         rc = be_cmd_rxq_create(adapter, &rxo->q, rxo->cq.id, rx_frag_size,
2863                                adapter->if_handle, false, &rxo->rss_id);
2864         if (rc)
2865                 return rc;
2866
2867         for_all_rss_queues(adapter, rxo, i) {
2868                 rc = be_cmd_rxq_create(adapter, &rxo->q, rxo->cq.id,
2869                                        rx_frag_size, adapter->if_handle,
2870                                        true, &rxo->rss_id);
2871                 if (rc)
2872                         return rc;
2873         }
2874
2875         if (be_multi_rxq(adapter)) {
2876                 for (j = 0; j < RSS_INDIR_TABLE_LEN;
2877                         j += adapter->num_rx_qs - 1) {
2878                         for_all_rss_queues(adapter, rxo, i) {
2879                                 if ((j + i) >= RSS_INDIR_TABLE_LEN)
2880                                         break;
2881                                 rss->rsstable[j + i] = rxo->rss_id;
2882                                 rss->rss_queue[j + i] = i;
2883                         }
2884                 }
2885                 rss->rss_flags = RSS_ENABLE_TCP_IPV4 | RSS_ENABLE_IPV4 |
2886                         RSS_ENABLE_TCP_IPV6 | RSS_ENABLE_IPV6;
2887
2888                 if (!BEx_chip(adapter))
2889                         rss->rss_flags |= RSS_ENABLE_UDP_IPV4 |
2890                                 RSS_ENABLE_UDP_IPV6;
2891         } else {
2892                 /* Disable RSS, if only default RX Q is created */
2893                 rss->rss_flags = RSS_ENABLE_NONE;
2894         }
2895
2896         get_random_bytes(rss_hkey, RSS_HASH_KEY_LEN);
2897         rc = be_cmd_rss_config(adapter, rss->rsstable, rss->rss_flags,
2898                                128, rss_hkey);
2899         if (rc) {
2900                 rss->rss_flags = RSS_ENABLE_NONE;
2901                 return rc;
2902         }
2903
2904         memcpy(rss->rss_hkey, rss_hkey, RSS_HASH_KEY_LEN);
2905
2906         /* First time posting */
2907         for_all_rx_queues(adapter, rxo, i)
2908                 be_post_rx_frags(rxo, GFP_KERNEL);
2909         return 0;
2910 }
2911
2912 static int be_open(struct net_device *netdev)
2913 {
2914         struct be_adapter *adapter = netdev_priv(netdev);
2915         struct be_eq_obj *eqo;
2916         struct be_rx_obj *rxo;
2917         struct be_tx_obj *txo;
2918         u8 link_status;
2919         int status, i;
2920
2921         status = be_rx_qs_create(adapter);
2922         if (status)
2923                 goto err;
2924
2925         status = be_irq_register(adapter);
2926         if (status)
2927                 goto err;
2928
2929         for_all_rx_queues(adapter, rxo, i)
2930                 be_cq_notify(adapter, rxo->cq.id, true, 0);
2931
2932         for_all_tx_queues(adapter, txo, i)
2933                 be_cq_notify(adapter, txo->cq.id, true, 0);
2934
2935         be_async_mcc_enable(adapter);
2936
2937         for_all_evt_queues(adapter, eqo, i) {
2938                 napi_enable(&eqo->napi);
2939                 be_enable_busy_poll(eqo);
2940                 be_eq_notify(adapter, eqo->q.id, true, true, 0);
2941         }
2942         adapter->flags |= BE_FLAGS_NAPI_ENABLED;
2943
2944         status = be_cmd_link_status_query(adapter, NULL, &link_status, 0);
2945         if (!status)
2946                 be_link_status_update(adapter, link_status);
2947
2948         netif_tx_start_all_queues(netdev);
2949         be_roce_dev_open(adapter);
2950
2951 #ifdef CONFIG_BE2NET_VXLAN
2952         if (skyhawk_chip(adapter))
2953                 vxlan_get_rx_port(netdev);
2954 #endif
2955
2956         return 0;
2957 err:
2958         be_close(adapter->netdev);
2959         return -EIO;
2960 }
2961
2962 static int be_setup_wol(struct be_adapter *adapter, bool enable)
2963 {
2964         struct be_dma_mem cmd;
2965         int status = 0;
2966         u8 mac[ETH_ALEN];
2967
2968         memset(mac, 0, ETH_ALEN);
2969
2970         cmd.size = sizeof(struct be_cmd_req_acpi_wol_magic_config);
2971         cmd.va = dma_zalloc_coherent(&adapter->pdev->dev, cmd.size, &cmd.dma,
2972                                      GFP_KERNEL);
2973         if (!cmd.va)
2974                 return -ENOMEM;
2975
2976         if (enable) {
2977                 status = pci_write_config_dword(adapter->pdev,
2978                                                 PCICFG_PM_CONTROL_OFFSET,
2979                                                 PCICFG_PM_CONTROL_MASK);
2980                 if (status) {
2981                         dev_err(&adapter->pdev->dev,
2982                                 "Could not enable Wake-on-lan\n");
2983                         dma_free_coherent(&adapter->pdev->dev, cmd.size, cmd.va,
2984                                           cmd.dma);
2985                         return status;
2986                 }
2987                 status = be_cmd_enable_magic_wol(adapter,
2988                                                  adapter->netdev->dev_addr,
2989                                                  &cmd);
2990                 pci_enable_wake(adapter->pdev, PCI_D3hot, 1);
2991                 pci_enable_wake(adapter->pdev, PCI_D3cold, 1);
2992         } else {
2993                 status = be_cmd_enable_magic_wol(adapter, mac, &cmd);
2994                 pci_enable_wake(adapter->pdev, PCI_D3hot, 0);
2995                 pci_enable_wake(adapter->pdev, PCI_D3cold, 0);
2996         }
2997
2998         dma_free_coherent(&adapter->pdev->dev, cmd.size, cmd.va, cmd.dma);
2999         return status;
3000 }
3001
3002 /*
3003  * Generate a seed MAC address from the PF MAC Address using jhash.
3004  * MAC Address for VFs are assigned incrementally starting from the seed.
3005  * These addresses are programmed in the ASIC by the PF and the VF driver
3006  * queries for the MAC address during its probe.
3007  */
3008 static int be_vf_eth_addr_config(struct be_adapter *adapter)
3009 {
3010         u32 vf;
3011         int status = 0;
3012         u8 mac[ETH_ALEN];
3013         struct be_vf_cfg *vf_cfg;
3014
3015         be_vf_eth_addr_generate(adapter, mac);
3016
3017         for_all_vfs(adapter, vf_cfg, vf) {
3018                 if (BEx_chip(adapter))
3019                         status = be_cmd_pmac_add(adapter, mac,
3020                                                  vf_cfg->if_handle,
3021                                                  &vf_cfg->pmac_id, vf + 1);
3022                 else
3023                         status = be_cmd_set_mac(adapter, mac, vf_cfg->if_handle,
3024                                                 vf + 1);
3025
3026                 if (status)
3027                         dev_err(&adapter->pdev->dev,
3028                                 "Mac address assignment failed for VF %d\n",
3029                                 vf);
3030                 else
3031                         memcpy(vf_cfg->mac_addr, mac, ETH_ALEN);
3032
3033                 mac[5] += 1;
3034         }
3035         return status;
3036 }
3037
3038 static int be_vfs_mac_query(struct be_adapter *adapter)
3039 {
3040         int status, vf;
3041         u8 mac[ETH_ALEN];
3042         struct be_vf_cfg *vf_cfg;
3043
3044         for_all_vfs(adapter, vf_cfg, vf) {
3045                 status = be_cmd_get_active_mac(adapter, vf_cfg->pmac_id,
3046                                                mac, vf_cfg->if_handle,
3047                                                false, vf+1);
3048                 if (status)
3049                         return status;
3050                 memcpy(vf_cfg->mac_addr, mac, ETH_ALEN);
3051         }
3052         return 0;
3053 }
3054
3055 static void be_vf_clear(struct be_adapter *adapter)
3056 {
3057         struct be_vf_cfg *vf_cfg;
3058         u32 vf;
3059
3060         if (pci_vfs_assigned(adapter->pdev)) {
3061                 dev_warn(&adapter->pdev->dev,
3062                          "VFs are assigned to VMs: not disabling VFs\n");
3063                 goto done;
3064         }
3065
3066         pci_disable_sriov(adapter->pdev);
3067
3068         for_all_vfs(adapter, vf_cfg, vf) {
3069                 if (BEx_chip(adapter))
3070                         be_cmd_pmac_del(adapter, vf_cfg->if_handle,
3071                                         vf_cfg->pmac_id, vf + 1);
3072                 else
3073                         be_cmd_set_mac(adapter, NULL, vf_cfg->if_handle,
3074                                        vf + 1);
3075
3076                 be_cmd_if_destroy(adapter, vf_cfg->if_handle, vf + 1);
3077         }
3078 done:
3079         kfree(adapter->vf_cfg);
3080         adapter->num_vfs = 0;
3081         adapter->flags &= ~BE_FLAGS_SRIOV_ENABLED;
3082 }
3083
3084 static void be_clear_queues(struct be_adapter *adapter)
3085 {
3086         be_mcc_queues_destroy(adapter);
3087         be_rx_cqs_destroy(adapter);
3088         be_tx_queues_destroy(adapter);
3089         be_evt_queues_destroy(adapter);
3090 }
3091
3092 static void be_cancel_worker(struct be_adapter *adapter)
3093 {
3094         if (adapter->flags & BE_FLAGS_WORKER_SCHEDULED) {
3095                 cancel_delayed_work_sync(&adapter->work);
3096                 adapter->flags &= ~BE_FLAGS_WORKER_SCHEDULED;
3097         }
3098 }
3099
3100 static void be_mac_clear(struct be_adapter *adapter)
3101 {
3102         int i;
3103
3104         if (adapter->pmac_id) {
3105                 for (i = 0; i < (adapter->uc_macs + 1); i++)
3106                         be_cmd_pmac_del(adapter, adapter->if_handle,
3107                                         adapter->pmac_id[i], 0);
3108                 adapter->uc_macs = 0;
3109
3110                 kfree(adapter->pmac_id);
3111                 adapter->pmac_id = NULL;
3112         }
3113 }
3114
3115 #ifdef CONFIG_BE2NET_VXLAN
3116 static void be_disable_vxlan_offloads(struct be_adapter *adapter)
3117 {
3118         if (adapter->flags & BE_FLAGS_VXLAN_OFFLOADS)
3119                 be_cmd_manage_iface(adapter, adapter->if_handle,
3120                                     OP_CONVERT_TUNNEL_TO_NORMAL);
3121
3122         if (adapter->vxlan_port)
3123                 be_cmd_set_vxlan_port(adapter, 0);
3124
3125         adapter->flags &= ~BE_FLAGS_VXLAN_OFFLOADS;
3126         adapter->vxlan_port = 0;
3127 }
3128 #endif
3129
3130 static int be_clear(struct be_adapter *adapter)
3131 {
3132         be_cancel_worker(adapter);
3133
3134         if (sriov_enabled(adapter))
3135                 be_vf_clear(adapter);
3136
3137         /* Re-configure FW to distribute resources evenly across max-supported
3138          * number of VFs, only when VFs are not already enabled.
3139          */
3140         if (be_physfn(adapter) && !pci_vfs_assigned(adapter->pdev))
3141                 be_cmd_set_sriov_config(adapter, adapter->pool_res,
3142                                         pci_sriov_get_totalvfs(adapter->pdev));
3143
3144 #ifdef CONFIG_BE2NET_VXLAN
3145         be_disable_vxlan_offloads(adapter);
3146 #endif
3147         /* delete the primary mac along with the uc-mac list */
3148         be_mac_clear(adapter);
3149
3150         be_cmd_if_destroy(adapter, adapter->if_handle,  0);
3151
3152         be_clear_queues(adapter);
3153
3154         be_msix_disable(adapter);
3155         adapter->flags &= ~BE_FLAGS_SETUP_DONE;
3156         return 0;
3157 }
3158
3159 static int be_vfs_if_create(struct be_adapter *adapter)
3160 {
3161         struct be_resources res = {0};
3162         struct be_vf_cfg *vf_cfg;
3163         u32 cap_flags, en_flags, vf;
3164         int status = 0;
3165
3166         cap_flags = BE_IF_FLAGS_UNTAGGED | BE_IF_FLAGS_BROADCAST |
3167                     BE_IF_FLAGS_MULTICAST;
3168
3169         for_all_vfs(adapter, vf_cfg, vf) {
3170                 if (!BE3_chip(adapter)) {
3171                         status = be_cmd_get_profile_config(adapter, &res,
3172                                                            vf + 1);
3173                         if (!status)
3174                                 cap_flags = res.if_cap_flags;
3175                 }
3176
3177                 /* If a FW profile exists, then cap_flags are updated */
3178                 en_flags = cap_flags & (BE_IF_FLAGS_UNTAGGED |
3179                                         BE_IF_FLAGS_BROADCAST |
3180                                         BE_IF_FLAGS_MULTICAST);
3181                 status =
3182                     be_cmd_if_create(adapter, cap_flags, en_flags,
3183                                      &vf_cfg->if_handle, vf + 1);
3184                 if (status)
3185                         goto err;
3186         }
3187 err:
3188         return status;
3189 }
3190
3191 static int be_vf_setup_init(struct be_adapter *adapter)
3192 {
3193         struct be_vf_cfg *vf_cfg;
3194         int vf;
3195
3196         adapter->vf_cfg = kcalloc(adapter->num_vfs, sizeof(*vf_cfg),
3197                                   GFP_KERNEL);
3198         if (!adapter->vf_cfg)
3199                 return -ENOMEM;
3200
3201         for_all_vfs(adapter, vf_cfg, vf) {
3202                 vf_cfg->if_handle = -1;
3203                 vf_cfg->pmac_id = -1;
3204         }
3205         return 0;
3206 }
3207
3208 static int be_vf_setup(struct be_adapter *adapter)
3209 {
3210         struct device *dev = &adapter->pdev->dev;
3211         struct be_vf_cfg *vf_cfg;
3212         int status, old_vfs, vf;
3213         u32 privileges;
3214
3215         old_vfs = pci_num_vf(adapter->pdev);
3216
3217         status = be_vf_setup_init(adapter);
3218         if (status)
3219                 goto err;
3220
3221         if (old_vfs) {
3222                 for_all_vfs(adapter, vf_cfg, vf) {
3223                         status = be_cmd_get_if_id(adapter, vf_cfg, vf);
3224                         if (status)
3225                                 goto err;
3226                 }
3227
3228                 status = be_vfs_mac_query(adapter);
3229                 if (status)
3230                         goto err;
3231         } else {
3232                 status = be_vfs_if_create(adapter);
3233                 if (status)
3234                         goto err;
3235
3236                 status = be_vf_eth_addr_config(adapter);
3237                 if (status)
3238                         goto err;
3239         }
3240
3241         for_all_vfs(adapter, vf_cfg, vf) {
3242                 /* Allow VFs to programs MAC/VLAN filters */
3243                 status = be_cmd_get_fn_privileges(adapter, &privileges, vf + 1);
3244                 if (!status && !(privileges & BE_PRIV_FILTMGMT)) {
3245                         status = be_cmd_set_fn_privileges(adapter,
3246                                                           privileges |
3247                                                           BE_PRIV_FILTMGMT,
3248                                                           vf + 1);
3249                         if (!status)
3250                                 dev_info(dev, "VF%d has FILTMGMT privilege\n",
3251                                          vf);
3252                 }
3253
3254                 /* Allow full available bandwidth */
3255                 if (!old_vfs)
3256                         be_cmd_config_qos(adapter, 0, 0, vf + 1);
3257
3258                 if (!old_vfs) {
3259                         be_cmd_enable_vf(adapter, vf + 1);
3260                         be_cmd_set_logical_link_config(adapter,
3261                                                        IFLA_VF_LINK_STATE_AUTO,
3262                                                        vf+1);
3263                 }
3264         }
3265
3266         if (!old_vfs) {
3267                 status = pci_enable_sriov(adapter->pdev, adapter->num_vfs);
3268                 if (status) {
3269                         dev_err(dev, "SRIOV enable failed\n");
3270                         adapter->num_vfs = 0;
3271                         goto err;
3272                 }
3273         }
3274
3275         adapter->flags |= BE_FLAGS_SRIOV_ENABLED;
3276         return 0;
3277 err:
3278         dev_err(dev, "VF setup failed\n");
3279         be_vf_clear(adapter);
3280         return status;
3281 }
3282
3283 /* Converting function_mode bits on BE3 to SH mc_type enums */
3284
3285 static u8 be_convert_mc_type(u32 function_mode)
3286 {
3287         if (function_mode & VNIC_MODE && function_mode & QNQ_MODE)
3288                 return vNIC1;
3289         else if (function_mode & QNQ_MODE)
3290                 return FLEX10;
3291         else if (function_mode & VNIC_MODE)
3292                 return vNIC2;
3293         else if (function_mode & UMC_ENABLED)
3294                 return UMC;
3295         else
3296                 return MC_NONE;
3297 }
3298
3299 /* On BE2/BE3 FW does not suggest the supported limits */
3300 static void BEx_get_resources(struct be_adapter *adapter,
3301                               struct be_resources *res)
3302 {
3303         bool use_sriov = adapter->num_vfs ? 1 : 0;
3304
3305         if (be_physfn(adapter))
3306                 res->max_uc_mac = BE_UC_PMAC_COUNT;
3307         else
3308                 res->max_uc_mac = BE_VF_UC_PMAC_COUNT;
3309
3310         adapter->mc_type = be_convert_mc_type(adapter->function_mode);
3311
3312         if (be_is_mc(adapter)) {
3313                 /* Assuming that there are 4 channels per port,
3314                  * when multi-channel is enabled
3315                  */
3316                 if (be_is_qnq_mode(adapter))
3317                         res->max_vlans = BE_NUM_VLANS_SUPPORTED/8;
3318                 else
3319                         /* In a non-qnq multichannel mode, the pvid
3320                          * takes up one vlan entry
3321                          */
3322                         res->max_vlans = (BE_NUM_VLANS_SUPPORTED / 4) - 1;
3323         } else {
3324                 res->max_vlans = BE_NUM_VLANS_SUPPORTED;
3325         }
3326
3327         res->max_mcast_mac = BE_MAX_MC;
3328
3329         /* 1) For BE3 1Gb ports, FW does not support multiple TXQs
3330          * 2) Create multiple TX rings on a BE3-R multi-channel interface
3331          *    *only* if it is RSS-capable.
3332          */
3333         if (BE2_chip(adapter) || use_sriov ||  (adapter->port_num > 1) ||
3334             !be_physfn(adapter) || (be_is_mc(adapter) &&
3335             !(adapter->function_caps & BE_FUNCTION_CAPS_RSS)))
3336                 res->max_tx_qs = 1;
3337         else
3338                 res->max_tx_qs = BE3_MAX_TX_QS;
3339
3340         if ((adapter->function_caps & BE_FUNCTION_CAPS_RSS) &&
3341             !use_sriov && be_physfn(adapter))
3342                 res->max_rss_qs = (adapter->be3_native) ?
3343                                            BE3_MAX_RSS_QS : BE2_MAX_RSS_QS;
3344         res->max_rx_qs = res->max_rss_qs + 1;
3345
3346         if (be_physfn(adapter))
3347                 res->max_evt_qs = (be_max_vfs(adapter) > 0) ?
3348                                         BE3_SRIOV_MAX_EVT_QS : BE3_MAX_EVT_QS;
3349         else
3350                 res->max_evt_qs = 1;
3351
3352         res->if_cap_flags = BE_IF_CAP_FLAGS_WANT;
3353         if (!(adapter->function_caps & BE_FUNCTION_CAPS_RSS))
3354                 res->if_cap_flags &= ~BE_IF_FLAGS_RSS;
3355 }
3356
3357 static void be_setup_init(struct be_adapter *adapter)
3358 {
3359         adapter->vlan_prio_bmap = 0xff;
3360         adapter->phy.link_speed = -1;
3361         adapter->if_handle = -1;
3362         adapter->be3_native = false;
3363         adapter->promiscuous = false;
3364         if (be_physfn(adapter))
3365                 adapter->cmd_privileges = MAX_PRIVILEGES;
3366         else
3367                 adapter->cmd_privileges = MIN_PRIVILEGES;
3368 }
3369
3370 static int be_get_sriov_config(struct be_adapter *adapter)
3371 {
3372         struct device *dev = &adapter->pdev->dev;
3373         struct be_resources res = {0};
3374         int max_vfs, old_vfs;
3375
3376         /* Some old versions of BE3 FW don't report max_vfs value */
3377         be_cmd_get_profile_config(adapter, &res, 0);
3378
3379         if (BE3_chip(adapter) && !res.max_vfs) {
3380                 max_vfs = pci_sriov_get_totalvfs(adapter->pdev);
3381                 res.max_vfs = max_vfs > 0 ? min(MAX_VFS, max_vfs) : 0;
3382         }
3383
3384         adapter->pool_res = res;
3385
3386         if (!be_max_vfs(adapter)) {
3387                 if (num_vfs)
3388                         dev_warn(dev, "device doesn't support SRIOV\n");
3389                 adapter->num_vfs = 0;
3390                 return 0;
3391         }
3392
3393         pci_sriov_set_totalvfs(adapter->pdev, be_max_vfs(adapter));
3394
3395         /* validate num_vfs module param */
3396         old_vfs = pci_num_vf(adapter->pdev);
3397         if (old_vfs) {
3398                 dev_info(dev, "%d VFs are already enabled\n", old_vfs);
3399                 if (old_vfs != num_vfs)
3400                         dev_warn(dev, "Ignoring num_vfs=%d setting\n", num_vfs);
3401                 adapter->num_vfs = old_vfs;
3402         } else {
3403                 if (num_vfs > be_max_vfs(adapter)) {
3404                         dev_info(dev, "Resources unavailable to init %d VFs\n",
3405                                  num_vfs);
3406                         dev_info(dev, "Limiting to %d VFs\n",
3407                                  be_max_vfs(adapter));
3408                 }
3409                 adapter->num_vfs = min_t(u16, num_vfs, be_max_vfs(adapter));
3410         }
3411
3412         return 0;
3413 }
3414
3415 static int be_get_resources(struct be_adapter *adapter)
3416 {
3417         struct device *dev = &adapter->pdev->dev;
3418         struct be_resources res = {0};
3419         int status;
3420
3421         if (BEx_chip(adapter)) {
3422                 BEx_get_resources(adapter, &res);
3423                 adapter->res = res;
3424         }
3425
3426         /* For Lancer, SH etc read per-function resource limits from FW.
3427          * GET_FUNC_CONFIG returns per function guaranteed limits.
3428          * GET_PROFILE_CONFIG returns PCI-E related limits PF-pool limits
3429          */
3430         if (!BEx_chip(adapter)) {
3431                 status = be_cmd_get_func_config(adapter, &res);
3432                 if (status)
3433                         return status;
3434
3435                 /* If RoCE may be enabled stash away half the EQs for RoCE */
3436                 if (be_roce_supported(adapter))
3437                         res.max_evt_qs /= 2;
3438                 adapter->res = res;
3439         }
3440
3441         dev_info(dev, "Max: txqs %d, rxqs %d, rss %d, eqs %d, vfs %d\n",
3442                  be_max_txqs(adapter), be_max_rxqs(adapter),
3443                  be_max_rss(adapter), be_max_eqs(adapter),
3444                  be_max_vfs(adapter));
3445         dev_info(dev, "Max: uc-macs %d, mc-macs %d, vlans %d\n",
3446                  be_max_uc(adapter), be_max_mc(adapter),
3447                  be_max_vlans(adapter));
3448
3449         return 0;
3450 }
3451
3452 static void be_sriov_config(struct be_adapter *adapter)
3453 {
3454         struct device *dev = &adapter->pdev->dev;
3455         int status;
3456
3457         status = be_get_sriov_config(adapter);
3458         if (status) {
3459                 dev_err(dev, "Failed to query SR-IOV configuration\n");
3460                 dev_err(dev, "SR-IOV cannot be enabled\n");
3461                 return;
3462         }
3463
3464         /* When the HW is in SRIOV capable configuration, the PF-pool
3465          * resources are equally distributed across the max-number of
3466          * VFs. The user may request only a subset of the max-vfs to be
3467          * enabled. Based on num_vfs, redistribute the resources across
3468          * num_vfs so that each VF will have access to more number of
3469          * resources. This facility is not available in BE3 FW.
3470          * Also, this is done by FW in Lancer chip.
3471          */
3472         if (be_max_vfs(adapter) && !pci_num_vf(adapter->pdev)) {
3473                 status = be_cmd_set_sriov_config(adapter,
3474                                                  adapter->pool_res,
3475                                                  adapter->num_vfs);
3476                 if (status)
3477                         dev_err(dev, "Failed to optimize SR-IOV resources\n");
3478         }
3479 }
3480
3481 static int be_get_config(struct be_adapter *adapter)
3482 {
3483         u16 profile_id;
3484         int status;
3485
3486         status = be_cmd_query_fw_cfg(adapter);
3487         if (status)
3488                 return status;
3489
3490          if (be_physfn(adapter)) {
3491                 status = be_cmd_get_active_profile(adapter, &profile_id);
3492                 if (!status)
3493                         dev_info(&adapter->pdev->dev,
3494                                  "Using profile 0x%x\n", profile_id);
3495         }
3496
3497         if (!BE2_chip(adapter) && be_physfn(adapter))
3498                 be_sriov_config(adapter);
3499
3500         status = be_get_resources(adapter);
3501         if (status)
3502                 return status;
3503
3504         adapter->pmac_id = kcalloc(be_max_uc(adapter),
3505                                    sizeof(*adapter->pmac_id), GFP_KERNEL);
3506         if (!adapter->pmac_id)
3507                 return -ENOMEM;
3508
3509         /* Sanitize cfg_num_qs based on HW and platform limits */
3510         adapter->cfg_num_qs = min(adapter->cfg_num_qs, be_max_qs(adapter));
3511
3512         return 0;
3513 }
3514
3515 static int be_mac_setup(struct be_adapter *adapter)
3516 {
3517         u8 mac[ETH_ALEN];
3518         int status;
3519
3520         if (is_zero_ether_addr(adapter->netdev->dev_addr)) {
3521                 status = be_cmd_get_perm_mac(adapter, mac);
3522                 if (status)
3523                         return status;
3524
3525                 memcpy(adapter->netdev->dev_addr, mac, ETH_ALEN);
3526                 memcpy(adapter->netdev->perm_addr, mac, ETH_ALEN);
3527         } else {
3528                 /* Maybe the HW was reset; dev_addr must be re-programmed */
3529                 memcpy(mac, adapter->netdev->dev_addr, ETH_ALEN);
3530         }
3531
3532         /* For BE3-R VFs, the PF programs the initial MAC address */
3533         if (!(BEx_chip(adapter) && be_virtfn(adapter)))
3534                 be_cmd_pmac_add(adapter, mac, adapter->if_handle,
3535                                 &adapter->pmac_id[0], 0);
3536         return 0;
3537 }
3538
3539 static void be_schedule_worker(struct be_adapter *adapter)
3540 {
3541         schedule_delayed_work(&adapter->work, msecs_to_jiffies(1000));
3542         adapter->flags |= BE_FLAGS_WORKER_SCHEDULED;
3543 }
3544
3545 static int be_setup_queues(struct be_adapter *adapter)
3546 {
3547         struct net_device *netdev = adapter->netdev;
3548         int status;
3549
3550         status = be_evt_queues_create(adapter);
3551         if (status)
3552                 goto err;
3553
3554         status = be_tx_qs_create(adapter);
3555         if (status)
3556                 goto err;
3557
3558         status = be_rx_cqs_create(adapter);
3559         if (status)
3560                 goto err;
3561
3562         status = be_mcc_queues_create(adapter);
3563         if (status)
3564                 goto err;
3565
3566         status = netif_set_real_num_rx_queues(netdev, adapter->num_rx_qs);
3567         if (status)
3568                 goto err;
3569
3570         status = netif_set_real_num_tx_queues(netdev, adapter->num_tx_qs);
3571         if (status)
3572                 goto err;
3573
3574         return 0;
3575 err:
3576         dev_err(&adapter->pdev->dev, "queue_setup failed\n");
3577         return status;
3578 }
3579
3580 int be_update_queues(struct be_adapter *adapter)
3581 {
3582         struct net_device *netdev = adapter->netdev;
3583         int status;
3584
3585         if (netif_running(netdev))
3586                 be_close(netdev);
3587
3588         be_cancel_worker(adapter);
3589
3590         /* If any vectors have been shared with RoCE we cannot re-program
3591          * the MSIx table.
3592          */
3593         if (!adapter->num_msix_roce_vec)
3594                 be_msix_disable(adapter);
3595
3596         be_clear_queues(adapter);
3597
3598         if (!msix_enabled(adapter)) {
3599                 status = be_msix_enable(adapter);
3600                 if (status)
3601                         return status;
3602         }
3603
3604         status = be_setup_queues(adapter);
3605         if (status)
3606                 return status;
3607
3608         be_schedule_worker(adapter);
3609
3610         if (netif_running(netdev))
3611                 status = be_open(netdev);
3612
3613         return status;
3614 }
3615
3616 static int be_setup(struct be_adapter *adapter)
3617 {
3618         struct device *dev = &adapter->pdev->dev;
3619         u32 tx_fc, rx_fc, en_flags;
3620         int status;
3621
3622         be_setup_init(adapter);
3623
3624         if (!lancer_chip(adapter))
3625                 be_cmd_req_native_mode(adapter);
3626
3627         status = be_get_config(adapter);
3628         if (status)
3629                 goto err;
3630
3631         status = be_msix_enable(adapter);
3632         if (status)
3633                 goto err;
3634
3635         en_flags = BE_IF_FLAGS_UNTAGGED | BE_IF_FLAGS_BROADCAST |
3636                    BE_IF_FLAGS_MULTICAST | BE_IF_FLAGS_PASS_L3L4_ERRORS;
3637         if (adapter->function_caps & BE_FUNCTION_CAPS_RSS)
3638                 en_flags |= BE_IF_FLAGS_RSS;
3639         en_flags = en_flags & be_if_cap_flags(adapter);
3640         status = be_cmd_if_create(adapter, be_if_cap_flags(adapter), en_flags,
3641                                   &adapter->if_handle, 0);
3642         if (status)
3643                 goto err;
3644
3645         /* Updating real_num_tx/rx_queues() requires rtnl_lock() */
3646         rtnl_lock();
3647         status = be_setup_queues(adapter);
3648         rtnl_unlock();
3649         if (status)
3650                 goto err;
3651
3652         be_cmd_get_fn_privileges(adapter, &adapter->cmd_privileges, 0);
3653
3654         status = be_mac_setup(adapter);
3655         if (status)
3656                 goto err;
3657
3658         be_cmd_get_fw_ver(adapter);
3659         dev_info(dev, "FW version is %s\n", adapter->fw_ver);
3660
3661         if (BE2_chip(adapter) && fw_major_num(adapter->fw_ver) < 4) {
3662                 dev_err(dev, "Firmware on card is old(%s), IRQs may not work.",
3663                         adapter->fw_ver);
3664                 dev_err(dev, "Please upgrade firmware to version >= 4.0\n");
3665         }
3666
3667         if (adapter->vlans_added)
3668                 be_vid_config(adapter);
3669
3670         be_set_rx_mode(adapter->netdev);
3671
3672         be_cmd_get_acpi_wol_cap(adapter);
3673
3674         be_cmd_get_flow_control(adapter, &tx_fc, &rx_fc);
3675
3676         if (rx_fc != adapter->rx_fc || tx_fc != adapter->tx_fc)
3677                 be_cmd_set_flow_control(adapter, adapter->tx_fc,
3678                                         adapter->rx_fc);
3679
3680         if (be_physfn(adapter))
3681                 be_cmd_set_logical_link_config(adapter,
3682                                                IFLA_VF_LINK_STATE_AUTO, 0);
3683
3684         if (adapter->num_vfs)
3685                 be_vf_setup(adapter);
3686
3687         status = be_cmd_get_phy_info(adapter);
3688         if (!status && be_pause_supported(adapter))
3689                 adapter->phy.fc_autoneg = 1;
3690
3691         be_schedule_worker(adapter);
3692         adapter->flags |= BE_FLAGS_SETUP_DONE;
3693         return 0;
3694 err:
3695         be_clear(adapter);
3696         return status;
3697 }
3698
3699 #ifdef CONFIG_NET_POLL_CONTROLLER
3700 static void be_netpoll(struct net_device *netdev)
3701 {
3702         struct be_adapter *adapter = netdev_priv(netdev);
3703         struct be_eq_obj *eqo;
3704         int i;
3705
3706         for_all_evt_queues(adapter, eqo, i) {
3707                 be_eq_notify(eqo->adapter, eqo->q.id, false, true, 0);
3708                 napi_schedule(&eqo->napi);
3709         }
3710
3711         return;
3712 }
3713 #endif
3714
3715 static char flash_cookie[2][16] = {"*** SE FLAS", "H DIRECTORY *** "};
3716
3717 static bool phy_flashing_required(struct be_adapter *adapter)
3718 {
3719         return (adapter->phy.phy_type == TN_8022 &&
3720                 adapter->phy.interface_type == PHY_TYPE_BASET_10GB);
3721 }
3722
3723 static bool is_comp_in_ufi(struct be_adapter *adapter,
3724                            struct flash_section_info *fsec, int type)
3725 {
3726         int i = 0, img_type = 0;
3727         struct flash_section_info_g2 *fsec_g2 = NULL;
3728
3729         if (BE2_chip(adapter))
3730                 fsec_g2 = (struct flash_section_info_g2 *)fsec;
3731
3732         for (i = 0; i < MAX_FLASH_COMP; i++) {
3733                 if (fsec_g2)
3734                         img_type = le32_to_cpu(fsec_g2->fsec_entry[i].type);
3735                 else
3736                         img_type = le32_to_cpu(fsec->fsec_entry[i].type);
3737
3738                 if (img_type == type)
3739                         return true;
3740         }
3741         return false;
3742
3743 }
3744
3745 static struct flash_section_info *get_fsec_info(struct be_adapter *adapter,
3746                                                 int header_size,
3747                                                 const struct firmware *fw)
3748 {
3749         struct flash_section_info *fsec = NULL;
3750         const u8 *p = fw->data;
3751
3752         p += header_size;
3753         while (p < (fw->data + fw->size)) {
3754                 fsec = (struct flash_section_info *)p;
3755                 if (!memcmp(flash_cookie, fsec->cookie, sizeof(flash_cookie)))
3756                         return fsec;
3757                 p += 32;
3758         }
3759         return NULL;
3760 }
3761
3762 static int be_check_flash_crc(struct be_adapter *adapter, const u8 *p,
3763                               u32 img_offset, u32 img_size, int hdr_size,
3764                               u16 img_optype, bool *crc_match)
3765 {
3766         u32 crc_offset;
3767         int status;
3768         u8 crc[4];
3769
3770         status = be_cmd_get_flash_crc(adapter, crc, img_optype, img_size - 4);
3771         if (status)
3772                 return status;
3773
3774         crc_offset = hdr_size + img_offset + img_size - 4;
3775
3776         /* Skip flashing, if crc of flashed region matches */
3777         if (!memcmp(crc, p + crc_offset, 4))
3778                 *crc_match = true;
3779         else
3780                 *crc_match = false;
3781
3782         return status;
3783 }
3784
3785 static int be_flash(struct be_adapter *adapter, const u8 *img,
3786                     struct be_dma_mem *flash_cmd, int optype, int img_size)
3787 {
3788         struct be_cmd_write_flashrom *req = flash_cmd->va;
3789         u32 total_bytes, flash_op, num_bytes;
3790         int status;
3791
3792         total_bytes = img_size;
3793         while (total_bytes) {
3794                 num_bytes = min_t(u32, 32*1024, total_bytes);
3795
3796                 total_bytes -= num_bytes;
3797
3798                 if (!total_bytes) {
3799                         if (optype == OPTYPE_PHY_FW)
3800                                 flash_op = FLASHROM_OPER_PHY_FLASH;
3801                         else
3802                                 flash_op = FLASHROM_OPER_FLASH;
3803                 } else {
3804                         if (optype == OPTYPE_PHY_FW)
3805                                 flash_op = FLASHROM_OPER_PHY_SAVE;
3806                         else
3807                                 flash_op = FLASHROM_OPER_SAVE;
3808                 }
3809
3810                 memcpy(req->data_buf, img, num_bytes);
3811                 img += num_bytes;
3812                 status = be_cmd_write_flashrom(adapter, flash_cmd, optype,
3813                                                flash_op, num_bytes);
3814                 if (base_status(status) == MCC_STATUS_ILLEGAL_REQUEST &&
3815                     optype == OPTYPE_PHY_FW)
3816                         break;
3817                 else if (status)
3818                         return status;
3819         }
3820         return 0;
3821 }
3822
3823 /* For BE2, BE3 and BE3-R */
3824 static int be_flash_BEx(struct be_adapter *adapter,
3825                         const struct firmware *fw,
3826                         struct be_dma_mem *flash_cmd, int num_of_images)
3827 {
3828         int img_hdrs_size = (num_of_images * sizeof(struct image_hdr));
3829         struct device *dev = &adapter->pdev->dev;
3830         struct flash_section_info *fsec = NULL;
3831         int status, i, filehdr_size, num_comp;
3832         const struct flash_comp *pflashcomp;
3833         bool crc_match;
3834         const u8 *p;
3835
3836         struct flash_comp gen3_flash_types[] = {
3837                 { FLASH_iSCSI_PRIMARY_IMAGE_START_g3, OPTYPE_ISCSI_ACTIVE,
3838                         FLASH_IMAGE_MAX_SIZE_g3, IMAGE_FIRMWARE_iSCSI},
3839                 { FLASH_REDBOOT_START_g3, OPTYPE_REDBOOT,
3840                         FLASH_REDBOOT_IMAGE_MAX_SIZE_g3, IMAGE_BOOT_CODE},
3841                 { FLASH_iSCSI_BIOS_START_g3, OPTYPE_BIOS,
3842                         FLASH_BIOS_IMAGE_MAX_SIZE_g3, IMAGE_OPTION_ROM_ISCSI},
3843                 { FLASH_PXE_BIOS_START_g3, OPTYPE_PXE_BIOS,
3844                         FLASH_BIOS_IMAGE_MAX_SIZE_g3, IMAGE_OPTION_ROM_PXE},
3845                 { FLASH_FCoE_BIOS_START_g3, OPTYPE_FCOE_BIOS,
3846                         FLASH_BIOS_IMAGE_MAX_SIZE_g3, IMAGE_OPTION_ROM_FCoE},
3847                 { FLASH_iSCSI_BACKUP_IMAGE_START_g3, OPTYPE_ISCSI_BACKUP,
3848                         FLASH_IMAGE_MAX_SIZE_g3, IMAGE_FIRMWARE_BACKUP_iSCSI},
3849                 { FLASH_FCoE_PRIMARY_IMAGE_START_g3, OPTYPE_FCOE_FW_ACTIVE,
3850                         FLASH_IMAGE_MAX_SIZE_g3, IMAGE_FIRMWARE_FCoE},
3851                 { FLASH_FCoE_BACKUP_IMAGE_START_g3, OPTYPE_FCOE_FW_BACKUP,
3852                         FLASH_IMAGE_MAX_SIZE_g3, IMAGE_FIRMWARE_BACKUP_FCoE},
3853                 { FLASH_NCSI_START_g3, OPTYPE_NCSI_FW,
3854                         FLASH_NCSI_IMAGE_MAX_SIZE_g3, IMAGE_NCSI},
3855                 { FLASH_PHY_FW_START_g3, OPTYPE_PHY_FW,
3856                         FLASH_PHY_FW_IMAGE_MAX_SIZE_g3, IMAGE_FIRMWARE_PHY}
3857         };
3858
3859         struct flash_comp gen2_flash_types[] = {
3860                 { FLASH_iSCSI_PRIMARY_IMAGE_START_g2, OPTYPE_ISCSI_ACTIVE,
3861                         FLASH_IMAGE_MAX_SIZE_g2, IMAGE_FIRMWARE_iSCSI},
3862                 { FLASH_REDBOOT_START_g2, OPTYPE_REDBOOT,
3863                         FLASH_REDBOOT_IMAGE_MAX_SIZE_g2, IMAGE_BOOT_CODE},
3864                 { FLASH_iSCSI_BIOS_START_g2, OPTYPE_BIOS,
3865                         FLASH_BIOS_IMAGE_MAX_SIZE_g2, IMAGE_OPTION_ROM_ISCSI},
3866                 { FLASH_PXE_BIOS_START_g2, OPTYPE_PXE_BIOS,
3867                         FLASH_BIOS_IMAGE_MAX_SIZE_g2, IMAGE_OPTION_ROM_PXE},
3868                 { FLASH_FCoE_BIOS_START_g2, OPTYPE_FCOE_BIOS,
3869                         FLASH_BIOS_IMAGE_MAX_SIZE_g2, IMAGE_OPTION_ROM_FCoE},
3870                 { FLASH_iSCSI_BACKUP_IMAGE_START_g2, OPTYPE_ISCSI_BACKUP,
3871                         FLASH_IMAGE_MAX_SIZE_g2, IMAGE_FIRMWARE_BACKUP_iSCSI},
3872                 { FLASH_FCoE_PRIMARY_IMAGE_START_g2, OPTYPE_FCOE_FW_ACTIVE,
3873                         FLASH_IMAGE_MAX_SIZE_g2, IMAGE_FIRMWARE_FCoE},
3874                 { FLASH_FCoE_BACKUP_IMAGE_START_g2, OPTYPE_FCOE_FW_BACKUP,
3875                          FLASH_IMAGE_MAX_SIZE_g2, IMAGE_FIRMWARE_BACKUP_FCoE}
3876         };
3877
3878         if (BE3_chip(adapter)) {
3879                 pflashcomp = gen3_flash_types;
3880                 filehdr_size = sizeof(struct flash_file_hdr_g3);
3881                 num_comp = ARRAY_SIZE(gen3_flash_types);
3882         } else {
3883                 pflashcomp = gen2_flash_types;
3884                 filehdr_size = sizeof(struct flash_file_hdr_g2);
3885                 num_comp = ARRAY_SIZE(gen2_flash_types);
3886         }
3887
3888         /* Get flash section info*/
3889         fsec = get_fsec_info(adapter, filehdr_size + img_hdrs_size, fw);
3890         if (!fsec) {
3891                 dev_err(dev, "Invalid Cookie. FW image may be corrupted\n");
3892                 return -1;
3893         }
3894         for (i = 0; i < num_comp; i++) {
3895                 if (!is_comp_in_ufi(adapter, fsec, pflashcomp[i].img_type))
3896                         continue;
3897
3898                 if ((pflashcomp[i].optype == OPTYPE_NCSI_FW) &&
3899                     memcmp(adapter->fw_ver, "3.102.148.0", 11) < 0)
3900                         continue;
3901
3902                 if (pflashcomp[i].optype == OPTYPE_PHY_FW  &&
3903                     !phy_flashing_required(adapter))
3904                                 continue;
3905
3906                 if (pflashcomp[i].optype == OPTYPE_REDBOOT) {
3907                         status = be_check_flash_crc(adapter, fw->data,
3908                                                     pflashcomp[i].offset,
3909                                                     pflashcomp[i].size,
3910                                                     filehdr_size +
3911                                                     img_hdrs_size,
3912                                                     OPTYPE_REDBOOT, &crc_match);
3913                         if (status) {
3914                                 dev_err(dev,
3915                                         "Could not get CRC for 0x%x region\n",
3916                                         pflashcomp[i].optype);
3917                                 continue;
3918                         }
3919
3920                         if (crc_match)
3921                                 continue;
3922                 }
3923
3924                 p = fw->data + filehdr_size + pflashcomp[i].offset +
3925                         img_hdrs_size;
3926                 if (p + pflashcomp[i].size > fw->data + fw->size)
3927                         return -1;
3928
3929                 status = be_flash(adapter, p, flash_cmd, pflashcomp[i].optype,
3930                                   pflashcomp[i].size);
3931                 if (status) {
3932                         dev_err(dev, "Flashing section type 0x%x failed\n",
3933                                 pflashcomp[i].img_type);
3934                         return status;
3935                 }
3936         }
3937         return 0;
3938 }
3939
3940 static u16 be_get_img_optype(struct flash_section_entry fsec_entry)
3941 {
3942         u32 img_type = le32_to_cpu(fsec_entry.type);
3943         u16 img_optype = le16_to_cpu(fsec_entry.optype);
3944
3945         if (img_optype != 0xFFFF)
3946                 return img_optype;
3947
3948         switch (img_type) {
3949         case IMAGE_FIRMWARE_iSCSI:
3950                 img_optype = OPTYPE_ISCSI_ACTIVE;
3951                 break;
3952         case IMAGE_BOOT_CODE:
3953                 img_optype = OPTYPE_REDBOOT;
3954                 break;
3955         case IMAGE_OPTION_ROM_ISCSI:
3956                 img_optype = OPTYPE_BIOS;
3957                 break;
3958         case IMAGE_OPTION_ROM_PXE:
3959                 img_optype = OPTYPE_PXE_BIOS;
3960                 break;
3961         case IMAGE_OPTION_ROM_FCoE:
3962                 img_optype = OPTYPE_FCOE_BIOS;
3963                 break;
3964         case IMAGE_FIRMWARE_BACKUP_iSCSI:
3965                 img_optype = OPTYPE_ISCSI_BACKUP;
3966                 break;
3967         case IMAGE_NCSI:
3968                 img_optype = OPTYPE_NCSI_FW;
3969                 break;
3970         case IMAGE_FLASHISM_JUMPVECTOR:
3971                 img_optype = OPTYPE_FLASHISM_JUMPVECTOR;
3972                 break;
3973         case IMAGE_FIRMWARE_PHY:
3974                 img_optype = OPTYPE_SH_PHY_FW;
3975                 break;
3976         case IMAGE_REDBOOT_DIR:
3977                 img_optype = OPTYPE_REDBOOT_DIR;
3978                 break;
3979         case IMAGE_REDBOOT_CONFIG:
3980                 img_optype = OPTYPE_REDBOOT_CONFIG;
3981                 break;
3982         case IMAGE_UFI_DIR:
3983                 img_optype = OPTYPE_UFI_DIR;
3984                 break;
3985         default:
3986                 break;
3987         }
3988
3989         return img_optype;
3990 }
3991
3992 static int be_flash_skyhawk(struct be_adapter *adapter,
3993                             const struct firmware *fw,
3994                             struct be_dma_mem *flash_cmd, int num_of_images)
3995 {
3996         int img_hdrs_size = num_of_images * sizeof(struct image_hdr);
3997         struct device *dev = &adapter->pdev->dev;
3998         struct flash_section_info *fsec = NULL;
3999         u32 img_offset, img_size, img_type;
4000         int status, i, filehdr_size;
4001         bool crc_match, old_fw_img;
4002         u16 img_optype;
4003         const u8 *p;
4004
4005         filehdr_size = sizeof(struct flash_file_hdr_g3);
4006         fsec = get_fsec_info(adapter, filehdr_size + img_hdrs_size, fw);
4007         if (!fsec) {
4008                 dev_err(dev, "Invalid Cookie. FW image may be corrupted\n");
4009                 return -EINVAL;
4010         }
4011
4012         for (i = 0; i < le32_to_cpu(fsec->fsec_hdr.num_images); i++) {
4013                 img_offset = le32_to_cpu(fsec->fsec_entry[i].offset);
4014                 img_size   = le32_to_cpu(fsec->fsec_entry[i].pad_size);
4015                 img_type   = le32_to_cpu(fsec->fsec_entry[i].type);
4016                 img_optype = be_get_img_optype(fsec->fsec_entry[i]);
4017                 old_fw_img = fsec->fsec_entry[i].optype == 0xFFFF;
4018
4019                 if (img_optype == 0xFFFF)
4020                         continue;
4021                 /* Don't bother verifying CRC if an old FW image is being
4022                  * flashed
4023                  */
4024                 if (old_fw_img)
4025                         goto flash;
4026
4027                 status = be_check_flash_crc(adapter, fw->data, img_offset,
4028                                             img_size, filehdr_size +
4029                                             img_hdrs_size, img_optype,
4030                                             &crc_match);
4031                 /* The current FW image on the card does not recognize the new
4032                  * FLASH op_type. The FW download is partially complete.
4033                  * Reboot the server now to enable FW image to recognize the
4034                  * new FLASH op_type. To complete the remaining process,
4035                  * download the same FW again after the reboot.
4036                  */
4037                 if (base_status(status) == MCC_STATUS_ILLEGAL_REQUEST ||
4038                     base_status(status) == MCC_STATUS_ILLEGAL_FIELD) {
4039                         dev_err(dev, "Flash incomplete. Reset the server\n");
4040                         dev_err(dev, "Download FW image again after reset\n");
4041                         return -EAGAIN;
4042                 } else if (status) {
4043                         dev_err(dev, "Could not get CRC for 0x%x region\n",
4044                                 img_optype);
4045                         return -EFAULT;
4046                 }
4047
4048                 if (crc_match)
4049                         continue;
4050
4051 flash:
4052                 p = fw->data + filehdr_size + img_offset + img_hdrs_size;
4053                 if (p + img_size > fw->data + fw->size)
4054                         return -1;
4055
4056                 status = be_flash(adapter, p, flash_cmd, img_optype, img_size);
4057                 /* For old FW images ignore ILLEGAL_FIELD error or errors on
4058                  * UFI_DIR region
4059                  */
4060                 if (old_fw_img &&
4061                     (base_status(status) == MCC_STATUS_ILLEGAL_FIELD ||
4062                      (img_optype == OPTYPE_UFI_DIR &&
4063                       base_status(status) == MCC_STATUS_FAILED))) {
4064                         continue;
4065                 } else if (status) {
4066                         dev_err(dev, "Flashing section type 0x%x failed\n",
4067                                 img_type);
4068                         return -EFAULT;
4069                 }
4070         }
4071         return 0;
4072 }
4073
4074 static int lancer_fw_download(struct be_adapter *adapter,
4075                               const struct firmware *fw)
4076 {
4077 #define LANCER_FW_DOWNLOAD_CHUNK      (32 * 1024)
4078 #define LANCER_FW_DOWNLOAD_LOCATION   "/prg"
4079         struct device *dev = &adapter->pdev->dev;
4080         struct be_dma_mem flash_cmd;
4081         const u8 *data_ptr = NULL;
4082         u8 *dest_image_ptr = NULL;
4083         size_t image_size = 0;
4084         u32 chunk_size = 0;
4085         u32 data_written = 0;
4086         u32 offset = 0;
4087         int status = 0;
4088         u8 add_status = 0;
4089         u8 change_status;
4090
4091         if (!IS_ALIGNED(fw->size, sizeof(u32))) {
4092                 dev_err(dev, "FW image size should be multiple of 4\n");
4093                 return -EINVAL;
4094         }
4095
4096         flash_cmd.size = sizeof(struct lancer_cmd_req_write_object)
4097                                 + LANCER_FW_DOWNLOAD_CHUNK;
4098         flash_cmd.va = dma_alloc_coherent(dev, flash_cmd.size,
4099                                           &flash_cmd.dma, GFP_KERNEL);
4100         if (!flash_cmd.va)
4101                 return -ENOMEM;
4102
4103         dest_image_ptr = flash_cmd.va +
4104                                 sizeof(struct lancer_cmd_req_write_object);
4105         image_size = fw->size;
4106         data_ptr = fw->data;
4107
4108         while (image_size) {
4109                 chunk_size = min_t(u32, image_size, LANCER_FW_DOWNLOAD_CHUNK);
4110
4111                 /* Copy the image chunk content. */
4112                 memcpy(dest_image_ptr, data_ptr, chunk_size);
4113
4114                 status = lancer_cmd_write_object(adapter, &flash_cmd,
4115                                                  chunk_size, offset,
4116                                                  LANCER_FW_DOWNLOAD_LOCATION,
4117                                                  &data_written, &change_status,
4118                                                  &add_status);
4119                 if (status)
4120                         break;
4121
4122                 offset += data_written;
4123                 data_ptr += data_written;
4124                 image_size -= data_written;
4125         }
4126
4127         if (!status) {
4128                 /* Commit the FW written */
4129                 status = lancer_cmd_write_object(adapter, &flash_cmd,
4130                                                  0, offset,
4131                                                  LANCER_FW_DOWNLOAD_LOCATION,
4132                                                  &data_written, &change_status,
4133                                                  &add_status);
4134         }
4135
4136         dma_free_coherent(dev, flash_cmd.size, flash_cmd.va, flash_cmd.dma);
4137         if (status) {
4138                 dev_err(dev, "Firmware load error\n");
4139                 return be_cmd_status(status);
4140         }
4141
4142         dev_info(dev, "Firmware flashed successfully\n");
4143
4144         if (change_status == LANCER_FW_RESET_NEEDED) {
4145                 dev_info(dev, "Resetting adapter to activate new FW\n");
4146                 status = lancer_physdev_ctrl(adapter,
4147                                              PHYSDEV_CONTROL_FW_RESET_MASK);
4148                 if (status) {
4149                         dev_err(dev, "Adapter busy, could not reset FW\n");
4150                         dev_err(dev, "Reboot server to activate new FW\n");
4151                 }
4152         } else if (change_status != LANCER_NO_RESET_NEEDED) {
4153                 dev_info(dev, "Reboot server to activate new FW\n");
4154         }
4155
4156         return 0;
4157 }
4158
4159 #define UFI_TYPE2               2
4160 #define UFI_TYPE3               3
4161 #define UFI_TYPE3R              10
4162 #define UFI_TYPE4               4
4163 static int be_get_ufi_type(struct be_adapter *adapter,
4164                            struct flash_file_hdr_g3 *fhdr)
4165 {
4166         if (!fhdr)
4167                 goto be_get_ufi_exit;
4168
4169         if (skyhawk_chip(adapter) && fhdr->build[0] == '4')
4170                 return UFI_TYPE4;
4171         else if (BE3_chip(adapter) && fhdr->build[0] == '3') {
4172                 if (fhdr->asic_type_rev == 0x10)
4173                         return UFI_TYPE3R;
4174                 else
4175                         return UFI_TYPE3;
4176         } else if (BE2_chip(adapter) && fhdr->build[0] == '2')
4177                 return UFI_TYPE2;
4178
4179 be_get_ufi_exit:
4180         dev_err(&adapter->pdev->dev,
4181                 "UFI and Interface are not compatible for flashing\n");
4182         return -1;
4183 }
4184
4185 static int be_fw_download(struct be_adapter *adapter, const struct firmware* fw)
4186 {
4187         struct flash_file_hdr_g3 *fhdr3;
4188         struct image_hdr *img_hdr_ptr = NULL;
4189         struct be_dma_mem flash_cmd;
4190         const u8 *p;
4191         int status = 0, i = 0, num_imgs = 0, ufi_type = 0;
4192
4193         flash_cmd.size = sizeof(struct be_cmd_write_flashrom);
4194         flash_cmd.va = dma_alloc_coherent(&adapter->pdev->dev, flash_cmd.size,
4195                                           &flash_cmd.dma, GFP_KERNEL);
4196         if (!flash_cmd.va) {
4197                 status = -ENOMEM;
4198                 goto be_fw_exit;
4199         }
4200
4201         p = fw->data;
4202         fhdr3 = (struct flash_file_hdr_g3 *)p;
4203
4204         ufi_type = be_get_ufi_type(adapter, fhdr3);
4205
4206         num_imgs = le32_to_cpu(fhdr3->num_imgs);
4207         for (i = 0; i < num_imgs; i++) {
4208                 img_hdr_ptr = (struct image_hdr *)(fw->data +
4209                                 (sizeof(struct flash_file_hdr_g3) +
4210                                  i * sizeof(struct image_hdr)));
4211                 if (le32_to_cpu(img_hdr_ptr->imageid) == 1) {
4212                         switch (ufi_type) {
4213                         case UFI_TYPE4:
4214                                 status = be_flash_skyhawk(adapter, fw,
4215                                                           &flash_cmd, num_imgs);
4216                                 break;
4217                         case UFI_TYPE3R:
4218                                 status = be_flash_BEx(adapter, fw, &flash_cmd,
4219                                                       num_imgs);
4220                                 break;
4221                         case UFI_TYPE3:
4222                                 /* Do not flash this ufi on BE3-R cards */
4223                                 if (adapter->asic_rev < 0x10)
4224                                         status = be_flash_BEx(adapter, fw,
4225                                                               &flash_cmd,
4226                                                               num_imgs);
4227                                 else {
4228                                         status = -EINVAL;
4229                                         dev_err(&adapter->pdev->dev,
4230                                                 "Can't load BE3 UFI on BE3R\n");
4231                                 }
4232                         }
4233                 }
4234         }
4235
4236         if (ufi_type == UFI_TYPE2)
4237                 status = be_flash_BEx(adapter, fw, &flash_cmd, 0);
4238         else if (ufi_type == -1)
4239                 status = -EINVAL;
4240
4241         dma_free_coherent(&adapter->pdev->dev, flash_cmd.size, flash_cmd.va,
4242                           flash_cmd.dma);
4243         if (status) {
4244                 dev_err(&adapter->pdev->dev, "Firmware load error\n");
4245                 goto be_fw_exit;
4246         }
4247
4248         dev_info(&adapter->pdev->dev, "Firmware flashed successfully\n");
4249
4250 be_fw_exit:
4251         return status;
4252 }
4253
4254 int be_load_fw(struct be_adapter *adapter, u8 *fw_file)
4255 {
4256         const struct firmware *fw;
4257         int status;
4258
4259         if (!netif_running(adapter->netdev)) {
4260                 dev_err(&adapter->pdev->dev,
4261                         "Firmware load not allowed (interface is down)\n");
4262                 return -ENETDOWN;
4263         }
4264
4265         status = request_firmware(&fw, fw_file, &adapter->pdev->dev);
4266         if (status)
4267                 goto fw_exit;
4268
4269         dev_info(&adapter->pdev->dev, "Flashing firmware file %s\n", fw_file);
4270
4271         if (lancer_chip(adapter))
4272                 status = lancer_fw_download(adapter, fw);
4273         else
4274                 status = be_fw_download(adapter, fw);
4275
4276         if (!status)
4277                 be_cmd_get_fw_ver(adapter);
4278
4279 fw_exit:
4280         release_firmware(fw);
4281         return status;
4282 }
4283
4284 static int be_ndo_bridge_setlink(struct net_device *dev, struct nlmsghdr *nlh)
4285 {
4286         struct be_adapter *adapter = netdev_priv(dev);
4287         struct nlattr *attr, *br_spec;
4288         int rem;
4289         int status = 0;
4290         u16 mode = 0;
4291
4292         if (!sriov_enabled(adapter))
4293                 return -EOPNOTSUPP;
4294
4295         br_spec = nlmsg_find_attr(nlh, sizeof(struct ifinfomsg), IFLA_AF_SPEC);
4296
4297         nla_for_each_nested(attr, br_spec, rem) {
4298                 if (nla_type(attr) != IFLA_BRIDGE_MODE)
4299                         continue;
4300
4301                 mode = nla_get_u16(attr);
4302                 if (mode != BRIDGE_MODE_VEPA && mode != BRIDGE_MODE_VEB)
4303                         return -EINVAL;
4304
4305                 status = be_cmd_set_hsw_config(adapter, 0, 0,
4306                                                adapter->if_handle,
4307                                                mode == BRIDGE_MODE_VEPA ?
4308                                                PORT_FWD_TYPE_VEPA :
4309                                                PORT_FWD_TYPE_VEB);
4310                 if (status)
4311                         goto err;
4312
4313                 dev_info(&adapter->pdev->dev, "enabled switch mode: %s\n",
4314                          mode == BRIDGE_MODE_VEPA ? "VEPA" : "VEB");
4315
4316                 return status;
4317         }
4318 err:
4319         dev_err(&adapter->pdev->dev, "Failed to set switch mode %s\n",
4320                 mode == BRIDGE_MODE_VEPA ? "VEPA" : "VEB");
4321
4322         return status;
4323 }
4324
4325 static int be_ndo_bridge_getlink(struct sk_buff *skb, u32 pid, u32 seq,
4326                                  struct net_device *dev, u32 filter_mask)
4327 {
4328         struct be_adapter *adapter = netdev_priv(dev);
4329         int status = 0;
4330         u8 hsw_mode;
4331
4332         if (!sriov_enabled(adapter))
4333                 return 0;
4334
4335         /* BE and Lancer chips support VEB mode only */
4336         if (BEx_chip(adapter) || lancer_chip(adapter)) {
4337                 hsw_mode = PORT_FWD_TYPE_VEB;
4338         } else {
4339                 status = be_cmd_get_hsw_config(adapter, NULL, 0,
4340                                                adapter->if_handle, &hsw_mode);
4341                 if (status)
4342                         return 0;
4343         }
4344
4345         return ndo_dflt_bridge_getlink(skb, pid, seq, dev,
4346                                        hsw_mode == PORT_FWD_TYPE_VEPA ?
4347                                        BRIDGE_MODE_VEPA : BRIDGE_MODE_VEB);
4348 }
4349
4350 #ifdef CONFIG_BE2NET_VXLAN
4351 static void be_add_vxlan_port(struct net_device *netdev, sa_family_t sa_family,
4352                               __be16 port)
4353 {
4354         struct be_adapter *adapter = netdev_priv(netdev);
4355         struct device *dev = &adapter->pdev->dev;
4356         int status;
4357
4358         if (lancer_chip(adapter) || BEx_chip(adapter))
4359                 return;
4360
4361         if (adapter->flags & BE_FLAGS_VXLAN_OFFLOADS) {
4362                 dev_warn(dev, "Cannot add UDP port %d for VxLAN offloads\n",
4363                          be16_to_cpu(port));
4364                 dev_info(dev,
4365                          "Only one UDP port supported for VxLAN offloads\n");
4366                 return;
4367         }
4368
4369         status = be_cmd_manage_iface(adapter, adapter->if_handle,
4370                                      OP_CONVERT_NORMAL_TO_TUNNEL);
4371         if (status) {
4372                 dev_warn(dev, "Failed to convert normal interface to tunnel\n");
4373                 goto err;
4374         }
4375
4376         status = be_cmd_set_vxlan_port(adapter, port);
4377         if (status) {
4378                 dev_warn(dev, "Failed to add VxLAN port\n");
4379                 goto err;
4380         }
4381         adapter->flags |= BE_FLAGS_VXLAN_OFFLOADS;
4382         adapter->vxlan_port = port;
4383
4384         dev_info(dev, "Enabled VxLAN offloads for UDP port %d\n",
4385                  be16_to_cpu(port));
4386         return;
4387 err:
4388         be_disable_vxlan_offloads(adapter);
4389         return;
4390 }
4391
4392 static void be_del_vxlan_port(struct net_device *netdev, sa_family_t sa_family,
4393                               __be16 port)
4394 {
4395         struct be_adapter *adapter = netdev_priv(netdev);
4396
4397         if (lancer_chip(adapter) || BEx_chip(adapter))
4398                 return;
4399
4400         if (adapter->vxlan_port != port)
4401                 return;
4402
4403         be_disable_vxlan_offloads(adapter);
4404
4405         dev_info(&adapter->pdev->dev,
4406                  "Disabled VxLAN offloads for UDP port %d\n",
4407                  be16_to_cpu(port));
4408 }
4409 #endif
4410
4411 static const struct net_device_ops be_netdev_ops = {
4412         .ndo_open               = be_open,
4413         .ndo_stop               = be_close,
4414         .ndo_start_xmit         = be_xmit,
4415         .ndo_set_rx_mode        = be_set_rx_mode,
4416         .ndo_set_mac_address    = be_mac_addr_set,
4417         .ndo_change_mtu         = be_change_mtu,
4418         .ndo_get_stats64        = be_get_stats64,
4419         .ndo_validate_addr      = eth_validate_addr,
4420         .ndo_vlan_rx_add_vid    = be_vlan_add_vid,
4421         .ndo_vlan_rx_kill_vid   = be_vlan_rem_vid,
4422         .ndo_set_vf_mac         = be_set_vf_mac,
4423         .ndo_set_vf_vlan        = be_set_vf_vlan,
4424         .ndo_set_vf_rate        = be_set_vf_tx_rate,
4425         .ndo_get_vf_config      = be_get_vf_config,
4426         .ndo_set_vf_link_state  = be_set_vf_link_state,
4427 #ifdef CONFIG_NET_POLL_CONTROLLER
4428         .ndo_poll_controller    = be_netpoll,
4429 #endif
4430         .ndo_bridge_setlink     = be_ndo_bridge_setlink,
4431         .ndo_bridge_getlink     = be_ndo_bridge_getlink,
4432 #ifdef CONFIG_NET_RX_BUSY_POLL
4433         .ndo_busy_poll          = be_busy_poll,
4434 #endif
4435 #ifdef CONFIG_BE2NET_VXLAN
4436         .ndo_add_vxlan_port     = be_add_vxlan_port,
4437         .ndo_del_vxlan_port     = be_del_vxlan_port,
4438 #endif
4439 };
4440
4441 static void be_netdev_init(struct net_device *netdev)
4442 {
4443         struct be_adapter *adapter = netdev_priv(netdev);
4444
4445         if (skyhawk_chip(adapter)) {
4446                 netdev->hw_enc_features |= NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |
4447                                            NETIF_F_TSO | NETIF_F_TSO6 |
4448                                            NETIF_F_GSO_UDP_TUNNEL;
4449                 netdev->hw_features |= NETIF_F_GSO_UDP_TUNNEL;
4450         }
4451         netdev->hw_features |= NETIF_F_SG | NETIF_F_TSO | NETIF_F_TSO6 |
4452                 NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | NETIF_F_RXCSUM |
4453                 NETIF_F_HW_VLAN_CTAG_TX;
4454         if (be_multi_rxq(adapter))
4455                 netdev->hw_features |= NETIF_F_RXHASH;
4456
4457         netdev->features |= netdev->hw_features |
4458                 NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_CTAG_FILTER;
4459
4460         netdev->vlan_features |= NETIF_F_SG | NETIF_F_TSO | NETIF_F_TSO6 |
4461                 NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM;
4462
4463         netdev->priv_flags |= IFF_UNICAST_FLT;
4464
4465         netdev->flags |= IFF_MULTICAST;
4466
4467         netif_set_gso_max_size(netdev, 65535 - ETH_HLEN);
4468
4469         netdev->netdev_ops = &be_netdev_ops;
4470
4471         netdev->ethtool_ops = &be_ethtool_ops;
4472 }
4473
4474 static void be_unmap_pci_bars(struct be_adapter *adapter)
4475 {
4476         if (adapter->csr)
4477                 pci_iounmap(adapter->pdev, adapter->csr);
4478         if (adapter->db)
4479                 pci_iounmap(adapter->pdev, adapter->db);
4480 }
4481
4482 static int db_bar(struct be_adapter *adapter)
4483 {
4484         if (lancer_chip(adapter) || !be_physfn(adapter))
4485                 return 0;
4486         else
4487                 return 4;
4488 }
4489
4490 static int be_roce_map_pci_bars(struct be_adapter *adapter)
4491 {
4492         if (skyhawk_chip(adapter)) {
4493                 adapter->roce_db.size = 4096;
4494                 adapter->roce_db.io_addr = pci_resource_start(adapter->pdev,
4495                                                               db_bar(adapter));
4496                 adapter->roce_db.total_size = pci_resource_len(adapter->pdev,
4497                                                                db_bar(adapter));
4498         }
4499         return 0;
4500 }
4501
4502 static int be_map_pci_bars(struct be_adapter *adapter)
4503 {
4504         u8 __iomem *addr;
4505
4506         if (BEx_chip(adapter) && be_physfn(adapter)) {
4507                 adapter->csr = pci_iomap(adapter->pdev, 2, 0);
4508                 if (!adapter->csr)
4509                         return -ENOMEM;
4510         }
4511
4512         addr = pci_iomap(adapter->pdev, db_bar(adapter), 0);
4513         if (!addr)
4514                 goto pci_map_err;
4515         adapter->db = addr;
4516
4517         be_roce_map_pci_bars(adapter);
4518         return 0;
4519
4520 pci_map_err:
4521         dev_err(&adapter->pdev->dev, "Error in mapping PCI BARs\n");
4522         be_unmap_pci_bars(adapter);
4523         return -ENOMEM;
4524 }
4525
4526 static void be_ctrl_cleanup(struct be_adapter *adapter)
4527 {
4528         struct be_dma_mem *mem = &adapter->mbox_mem_alloced;
4529
4530         be_unmap_pci_bars(adapter);
4531
4532         if (mem->va)
4533                 dma_free_coherent(&adapter->pdev->dev, mem->size, mem->va,
4534                                   mem->dma);
4535
4536         mem = &adapter->rx_filter;
4537         if (mem->va)
4538                 dma_free_coherent(&adapter->pdev->dev, mem->size, mem->va,
4539                                   mem->dma);
4540 }
4541
4542 static int be_ctrl_init(struct be_adapter *adapter)
4543 {
4544         struct be_dma_mem *mbox_mem_alloc = &adapter->mbox_mem_alloced;
4545         struct be_dma_mem *mbox_mem_align = &adapter->mbox_mem;
4546         struct be_dma_mem *rx_filter = &adapter->rx_filter;
4547         u32 sli_intf;
4548         int status;
4549
4550         pci_read_config_dword(adapter->pdev, SLI_INTF_REG_OFFSET, &sli_intf);
4551         adapter->sli_family = (sli_intf & SLI_INTF_FAMILY_MASK) >>
4552                                  SLI_INTF_FAMILY_SHIFT;
4553         adapter->virtfn = (sli_intf & SLI_INTF_FT_MASK) ? 1 : 0;
4554
4555         status = be_map_pci_bars(adapter);
4556         if (status)
4557                 goto done;
4558
4559         mbox_mem_alloc->size = sizeof(struct be_mcc_mailbox) + 16;
4560         mbox_mem_alloc->va = dma_alloc_coherent(&adapter->pdev->dev,
4561                                                 mbox_mem_alloc->size,
4562                                                 &mbox_mem_alloc->dma,
4563                                                 GFP_KERNEL);
4564         if (!mbox_mem_alloc->va) {
4565                 status = -ENOMEM;
4566                 goto unmap_pci_bars;
4567         }
4568         mbox_mem_align->size = sizeof(struct be_mcc_mailbox);
4569         mbox_mem_align->va = PTR_ALIGN(mbox_mem_alloc->va, 16);
4570         mbox_mem_align->dma = PTR_ALIGN(mbox_mem_alloc->dma, 16);
4571         memset(mbox_mem_align->va, 0, sizeof(struct be_mcc_mailbox));
4572
4573         rx_filter->size = sizeof(struct be_cmd_req_rx_filter);
4574         rx_filter->va = dma_zalloc_coherent(&adapter->pdev->dev,
4575                                             rx_filter->size, &rx_filter->dma,
4576                                             GFP_KERNEL);
4577         if (!rx_filter->va) {
4578                 status = -ENOMEM;
4579                 goto free_mbox;
4580         }
4581
4582         mutex_init(&adapter->mbox_lock);
4583         spin_lock_init(&adapter->mcc_lock);
4584         spin_lock_init(&adapter->mcc_cq_lock);
4585
4586         init_completion(&adapter->et_cmd_compl);
4587         pci_save_state(adapter->pdev);
4588         return 0;
4589
4590 free_mbox:
4591         dma_free_coherent(&adapter->pdev->dev, mbox_mem_alloc->size,
4592                           mbox_mem_alloc->va, mbox_mem_alloc->dma);
4593
4594 unmap_pci_bars:
4595         be_unmap_pci_bars(adapter);
4596
4597 done:
4598         return status;
4599 }
4600
4601 static void be_stats_cleanup(struct be_adapter *adapter)
4602 {
4603         struct be_dma_mem *cmd = &adapter->stats_cmd;
4604
4605         if (cmd->va)
4606                 dma_free_coherent(&adapter->pdev->dev, cmd->size,
4607                                   cmd->va, cmd->dma);
4608 }
4609
4610 static int be_stats_init(struct be_adapter *adapter)
4611 {
4612         struct be_dma_mem *cmd = &adapter->stats_cmd;
4613
4614         if (lancer_chip(adapter))
4615                 cmd->size = sizeof(struct lancer_cmd_req_pport_stats);
4616         else if (BE2_chip(adapter))
4617                 cmd->size = sizeof(struct be_cmd_req_get_stats_v0);
4618         else if (BE3_chip(adapter))
4619                 cmd->size = sizeof(struct be_cmd_req_get_stats_v1);
4620         else
4621                 /* ALL non-BE ASICs */
4622                 cmd->size = sizeof(struct be_cmd_req_get_stats_v2);
4623
4624         cmd->va = dma_zalloc_coherent(&adapter->pdev->dev, cmd->size, &cmd->dma,
4625                                       GFP_KERNEL);
4626         if (!cmd->va)
4627                 return -ENOMEM;
4628         return 0;
4629 }
4630
4631 static void be_remove(struct pci_dev *pdev)
4632 {
4633         struct be_adapter *adapter = pci_get_drvdata(pdev);
4634
4635         if (!adapter)
4636                 return;
4637
4638         be_roce_dev_remove(adapter);
4639         be_intr_set(adapter, false);
4640
4641         cancel_delayed_work_sync(&adapter->func_recovery_work);
4642
4643         unregister_netdev(adapter->netdev);
4644
4645         be_clear(adapter);
4646
4647         /* tell fw we're done with firing cmds */
4648         be_cmd_fw_clean(adapter);
4649
4650         be_stats_cleanup(adapter);
4651
4652         be_ctrl_cleanup(adapter);
4653
4654         pci_disable_pcie_error_reporting(pdev);
4655
4656         pci_release_regions(pdev);
4657         pci_disable_device(pdev);
4658
4659         free_netdev(adapter->netdev);
4660 }
4661
4662 static int be_get_initial_config(struct be_adapter *adapter)
4663 {
4664         int status, level;
4665
4666         status = be_cmd_get_cntl_attributes(adapter);
4667         if (status)
4668                 return status;
4669
4670         /* Must be a power of 2 or else MODULO will BUG_ON */
4671         adapter->be_get_temp_freq = 64;
4672
4673         if (BEx_chip(adapter)) {
4674                 level = be_cmd_get_fw_log_level(adapter);
4675                 adapter->msg_enable =
4676                         level <= FW_LOG_LEVEL_DEFAULT ? NETIF_MSG_HW : 0;
4677         }
4678
4679         adapter->cfg_num_qs = netif_get_num_default_rss_queues();
4680         return 0;
4681 }
4682
4683 static int lancer_recover_func(struct be_adapter *adapter)
4684 {
4685         struct device *dev = &adapter->pdev->dev;
4686         int status;
4687
4688         status = lancer_test_and_set_rdy_state(adapter);
4689         if (status)
4690                 goto err;
4691
4692         if (netif_running(adapter->netdev))
4693                 be_close(adapter->netdev);
4694
4695         be_clear(adapter);
4696
4697         be_clear_all_error(adapter);
4698
4699         status = be_setup(adapter);
4700         if (status)
4701                 goto err;
4702
4703         if (netif_running(adapter->netdev)) {
4704                 status = be_open(adapter->netdev);
4705                 if (status)
4706                         goto err;
4707         }
4708
4709         dev_err(dev, "Adapter recovery successful\n");
4710         return 0;
4711 err:
4712         if (status == -EAGAIN)
4713                 dev_err(dev, "Waiting for resource provisioning\n");
4714         else
4715                 dev_err(dev, "Adapter recovery failed\n");
4716
4717         return status;
4718 }
4719
4720 static void be_func_recovery_task(struct work_struct *work)
4721 {
4722         struct be_adapter *adapter =
4723                 container_of(work, struct be_adapter,  func_recovery_work.work);
4724         int status = 0;
4725
4726         be_detect_error(adapter);
4727
4728         if (adapter->hw_error && lancer_chip(adapter)) {
4729
4730                 rtnl_lock();
4731                 netif_device_detach(adapter->netdev);
4732                 rtnl_unlock();
4733
4734                 status = lancer_recover_func(adapter);
4735                 if (!status)
4736                         netif_device_attach(adapter->netdev);
4737         }
4738
4739         /* In Lancer, for all errors other than provisioning error (-EAGAIN),
4740          * no need to attempt further recovery.
4741          */
4742         if (!status || status == -EAGAIN)
4743                 schedule_delayed_work(&adapter->func_recovery_work,
4744                                       msecs_to_jiffies(1000));
4745 }
4746
4747 static void be_worker(struct work_struct *work)
4748 {
4749         struct be_adapter *adapter =
4750                 container_of(work, struct be_adapter, work.work);
4751         struct be_rx_obj *rxo;
4752         int i;
4753
4754         /* when interrupts are not yet enabled, just reap any pending
4755         * mcc completions */
4756         if (!netif_running(adapter->netdev)) {
4757                 local_bh_disable();
4758                 be_process_mcc(adapter);
4759                 local_bh_enable();
4760                 goto reschedule;
4761         }
4762
4763         if (!adapter->stats_cmd_sent) {
4764                 if (lancer_chip(adapter))
4765                         lancer_cmd_get_pport_stats(adapter,
4766                                                 &adapter->stats_cmd);
4767                 else
4768                         be_cmd_get_stats(adapter, &adapter->stats_cmd);
4769         }
4770
4771         if (be_physfn(adapter) &&
4772             MODULO(adapter->work_counter, adapter->be_get_temp_freq) == 0)
4773                 be_cmd_get_die_temperature(adapter);
4774
4775         for_all_rx_queues(adapter, rxo, i) {
4776                 /* Replenish RX-queues starved due to memory
4777                  * allocation failures.
4778                  */
4779                 if (rxo->rx_post_starved)
4780                         be_post_rx_frags(rxo, GFP_KERNEL);
4781         }
4782
4783         be_eqd_update(adapter);
4784
4785 reschedule:
4786         adapter->work_counter++;
4787         schedule_delayed_work(&adapter->work, msecs_to_jiffies(1000));
4788 }
4789
4790 /* If any VFs are already enabled don't FLR the PF */
4791 static bool be_reset_required(struct be_adapter *adapter)
4792 {
4793         return pci_num_vf(adapter->pdev) ? false : true;
4794 }
4795
4796 static char *mc_name(struct be_adapter *adapter)
4797 {
4798         char *str = ""; /* default */
4799
4800         switch (adapter->mc_type) {
4801         case UMC:
4802                 str = "UMC";
4803                 break;
4804         case FLEX10:
4805                 str = "FLEX10";
4806                 break;
4807         case vNIC1:
4808                 str = "vNIC-1";
4809                 break;
4810         case nPAR:
4811                 str = "nPAR";
4812                 break;
4813         case UFP:
4814                 str = "UFP";
4815                 break;
4816         case vNIC2:
4817                 str = "vNIC-2";
4818                 break;
4819         default:
4820                 str = "";
4821         }
4822
4823         return str;
4824 }
4825
4826 static inline char *func_name(struct be_adapter *adapter)
4827 {
4828         return be_physfn(adapter) ? "PF" : "VF";
4829 }
4830
4831 static int be_probe(struct pci_dev *pdev, const struct pci_device_id *pdev_id)
4832 {
4833         int status = 0;
4834         struct be_adapter *adapter;
4835         struct net_device *netdev;
4836         char port_name;
4837
4838         dev_info(&pdev->dev, "%s version is %s\n", DRV_NAME, DRV_VER);
4839
4840         status = pci_enable_device(pdev);
4841         if (status)
4842                 goto do_none;
4843
4844         status = pci_request_regions(pdev, DRV_NAME);
4845         if (status)
4846                 goto disable_dev;
4847         pci_set_master(pdev);
4848
4849         netdev = alloc_etherdev_mqs(sizeof(*adapter), MAX_TX_QS, MAX_RX_QS);
4850         if (!netdev) {
4851                 status = -ENOMEM;
4852                 goto rel_reg;
4853         }
4854         adapter = netdev_priv(netdev);
4855         adapter->pdev = pdev;
4856         pci_set_drvdata(pdev, adapter);
4857         adapter->netdev = netdev;
4858         SET_NETDEV_DEV(netdev, &pdev->dev);
4859
4860         status = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
4861         if (!status) {
4862                 netdev->features |= NETIF_F_HIGHDMA;
4863         } else {
4864                 status = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
4865                 if (status) {
4866                         dev_err(&pdev->dev, "Could not set PCI DMA Mask\n");
4867                         goto free_netdev;
4868                 }
4869         }
4870
4871         if (be_physfn(adapter)) {
4872                 status = pci_enable_pcie_error_reporting(pdev);
4873                 if (!status)
4874                         dev_info(&pdev->dev, "PCIe error reporting enabled\n");
4875         }
4876
4877         status = be_ctrl_init(adapter);
4878         if (status)
4879                 goto free_netdev;
4880
4881         /* sync up with fw's ready state */
4882         if (be_physfn(adapter)) {
4883                 status = be_fw_wait_ready(adapter);
4884                 if (status)
4885                         goto ctrl_clean;
4886         }
4887
4888         if (be_reset_required(adapter)) {
4889                 status = be_cmd_reset_function(adapter);
4890                 if (status)
4891                         goto ctrl_clean;
4892
4893                 /* Wait for interrupts to quiesce after an FLR */
4894                 msleep(100);
4895         }
4896
4897         /* Allow interrupts for other ULPs running on NIC function */
4898         be_intr_set(adapter, true);
4899
4900         /* tell fw we're ready to fire cmds */
4901         status = be_cmd_fw_init(adapter);
4902         if (status)
4903                 goto ctrl_clean;
4904
4905         status = be_stats_init(adapter);
4906         if (status)
4907                 goto ctrl_clean;
4908
4909         status = be_get_initial_config(adapter);
4910         if (status)
4911                 goto stats_clean;
4912
4913         INIT_DELAYED_WORK(&adapter->work, be_worker);
4914         INIT_DELAYED_WORK(&adapter->func_recovery_work, be_func_recovery_task);
4915         adapter->rx_fc = adapter->tx_fc = true;
4916
4917         status = be_setup(adapter);
4918         if (status)
4919                 goto stats_clean;
4920
4921         be_netdev_init(netdev);
4922         status = register_netdev(netdev);
4923         if (status != 0)
4924                 goto unsetup;
4925
4926         be_roce_dev_add(adapter);
4927
4928         schedule_delayed_work(&adapter->func_recovery_work,
4929                               msecs_to_jiffies(1000));
4930
4931         be_cmd_query_port_name(adapter, &port_name);
4932
4933         dev_info(&pdev->dev, "%s: %s %s port %c\n", nic_name(pdev),
4934                  func_name(adapter), mc_name(adapter), port_name);
4935
4936         return 0;
4937
4938 unsetup:
4939         be_clear(adapter);
4940 stats_clean:
4941         be_stats_cleanup(adapter);
4942 ctrl_clean:
4943         be_ctrl_cleanup(adapter);
4944 free_netdev:
4945         free_netdev(netdev);
4946 rel_reg:
4947         pci_release_regions(pdev);
4948 disable_dev:
4949         pci_disable_device(pdev);
4950 do_none:
4951         dev_err(&pdev->dev, "%s initialization failed\n", nic_name(pdev));
4952         return status;
4953 }
4954
4955 static int be_suspend(struct pci_dev *pdev, pm_message_t state)
4956 {
4957         struct be_adapter *adapter = pci_get_drvdata(pdev);
4958         struct net_device *netdev =  adapter->netdev;
4959
4960         if (adapter->wol_en)
4961                 be_setup_wol(adapter, true);
4962
4963         be_intr_set(adapter, false);
4964         cancel_delayed_work_sync(&adapter->func_recovery_work);
4965
4966         netif_device_detach(netdev);
4967         if (netif_running(netdev)) {
4968                 rtnl_lock();
4969                 be_close(netdev);
4970                 rtnl_unlock();
4971         }
4972         be_clear(adapter);
4973
4974         pci_save_state(pdev);
4975         pci_disable_device(pdev);
4976         pci_set_power_state(pdev, pci_choose_state(pdev, state));
4977         return 0;
4978 }
4979
4980 static int be_resume(struct pci_dev *pdev)
4981 {
4982         int status = 0;
4983         struct be_adapter *adapter = pci_get_drvdata(pdev);
4984         struct net_device *netdev =  adapter->netdev;
4985
4986         netif_device_detach(netdev);
4987
4988         status = pci_enable_device(pdev);
4989         if (status)
4990                 return status;
4991
4992         pci_set_power_state(pdev, PCI_D0);
4993         pci_restore_state(pdev);
4994
4995         status = be_fw_wait_ready(adapter);
4996         if (status)
4997                 return status;
4998
4999         be_intr_set(adapter, true);
5000         /* tell fw we're ready to fire cmds */
5001         status = be_cmd_fw_init(adapter);
5002         if (status)
5003                 return status;
5004
5005         be_setup(adapter);
5006         if (netif_running(netdev)) {
5007                 rtnl_lock();
5008                 be_open(netdev);
5009                 rtnl_unlock();
5010         }
5011
5012         schedule_delayed_work(&adapter->func_recovery_work,
5013                               msecs_to_jiffies(1000));
5014         netif_device_attach(netdev);
5015
5016         if (adapter->wol_en)
5017                 be_setup_wol(adapter, false);
5018
5019         return 0;
5020 }
5021
5022 /*
5023  * An FLR will stop BE from DMAing any data.
5024  */
5025 static void be_shutdown(struct pci_dev *pdev)
5026 {
5027         struct be_adapter *adapter = pci_get_drvdata(pdev);
5028
5029         if (!adapter)
5030                 return;
5031
5032         be_roce_dev_shutdown(adapter);
5033         cancel_delayed_work_sync(&adapter->work);
5034         cancel_delayed_work_sync(&adapter->func_recovery_work);
5035
5036         netif_device_detach(adapter->netdev);
5037
5038         be_cmd_reset_function(adapter);
5039
5040         pci_disable_device(pdev);
5041 }
5042
5043 static pci_ers_result_t be_eeh_err_detected(struct pci_dev *pdev,
5044                                             pci_channel_state_t state)
5045 {
5046         struct be_adapter *adapter = pci_get_drvdata(pdev);
5047         struct net_device *netdev =  adapter->netdev;
5048
5049         dev_err(&adapter->pdev->dev, "EEH error detected\n");
5050
5051         if (!adapter->eeh_error) {
5052                 adapter->eeh_error = true;
5053
5054                 cancel_delayed_work_sync(&adapter->func_recovery_work);
5055
5056                 rtnl_lock();
5057                 netif_device_detach(netdev);
5058                 if (netif_running(netdev))
5059                         be_close(netdev);
5060                 rtnl_unlock();
5061
5062                 be_clear(adapter);
5063         }
5064
5065         if (state == pci_channel_io_perm_failure)
5066                 return PCI_ERS_RESULT_DISCONNECT;
5067
5068         pci_disable_device(pdev);
5069
5070         /* The error could cause the FW to trigger a flash debug dump.
5071          * Resetting the card while flash dump is in progress
5072          * can cause it not to recover; wait for it to finish.
5073          * Wait only for first function as it is needed only once per
5074          * adapter.
5075          */
5076         if (pdev->devfn == 0)
5077                 ssleep(30);
5078
5079         return PCI_ERS_RESULT_NEED_RESET;
5080 }
5081
5082 static pci_ers_result_t be_eeh_reset(struct pci_dev *pdev)
5083 {
5084         struct be_adapter *adapter = pci_get_drvdata(pdev);
5085         int status;
5086
5087         dev_info(&adapter->pdev->dev, "EEH reset\n");
5088
5089         status = pci_enable_device(pdev);
5090         if (status)
5091                 return PCI_ERS_RESULT_DISCONNECT;
5092
5093         pci_set_master(pdev);
5094         pci_set_power_state(pdev, PCI_D0);
5095         pci_restore_state(pdev);
5096
5097         /* Check if card is ok and fw is ready */
5098         dev_info(&adapter->pdev->dev,
5099                  "Waiting for FW to be ready after EEH reset\n");
5100         status = be_fw_wait_ready(adapter);
5101         if (status)
5102                 return PCI_ERS_RESULT_DISCONNECT;
5103
5104         pci_cleanup_aer_uncorrect_error_status(pdev);
5105         be_clear_all_error(adapter);
5106         return PCI_ERS_RESULT_RECOVERED;
5107 }
5108
5109 static void be_eeh_resume(struct pci_dev *pdev)
5110 {
5111         int status = 0;
5112         struct be_adapter *adapter = pci_get_drvdata(pdev);
5113         struct net_device *netdev =  adapter->netdev;
5114
5115         dev_info(&adapter->pdev->dev, "EEH resume\n");
5116
5117         pci_save_state(pdev);
5118
5119         status = be_cmd_reset_function(adapter);
5120         if (status)
5121                 goto err;
5122
5123         /* On some BE3 FW versions, after a HW reset,
5124          * interrupts will remain disabled for each function.
5125          * So, explicitly enable interrupts
5126          */
5127         be_intr_set(adapter, true);
5128
5129         /* tell fw we're ready to fire cmds */
5130         status = be_cmd_fw_init(adapter);
5131         if (status)
5132                 goto err;
5133
5134         status = be_setup(adapter);
5135         if (status)
5136                 goto err;
5137
5138         if (netif_running(netdev)) {
5139                 status = be_open(netdev);
5140                 if (status)
5141                         goto err;
5142         }
5143
5144         schedule_delayed_work(&adapter->func_recovery_work,
5145                               msecs_to_jiffies(1000));
5146         netif_device_attach(netdev);
5147         return;
5148 err:
5149         dev_err(&adapter->pdev->dev, "EEH resume failed\n");
5150 }
5151
5152 static const struct pci_error_handlers be_eeh_handlers = {
5153         .error_detected = be_eeh_err_detected,
5154         .slot_reset = be_eeh_reset,
5155         .resume = be_eeh_resume,
5156 };
5157
5158 static struct pci_driver be_driver = {
5159         .name = DRV_NAME,
5160         .id_table = be_dev_ids,
5161         .probe = be_probe,
5162         .remove = be_remove,
5163         .suspend = be_suspend,
5164         .resume = be_resume,
5165         .shutdown = be_shutdown,
5166         .err_handler = &be_eeh_handlers
5167 };
5168
5169 static int __init be_init_module(void)
5170 {
5171         if (rx_frag_size != 8192 && rx_frag_size != 4096 &&
5172             rx_frag_size != 2048) {
5173                 printk(KERN_WARNING DRV_NAME
5174                         " : Module param rx_frag_size must be 2048/4096/8192."
5175                         " Using 2048\n");
5176                 rx_frag_size = 2048;
5177         }
5178
5179         return pci_register_driver(&be_driver);
5180 }
5181 module_init(be_init_module);
5182
5183 static void __exit be_exit_module(void)
5184 {
5185         pci_unregister_driver(&be_driver);
5186 }
5187 module_exit(be_exit_module);