drivers/net/ethernet/emulex/benet/be_main.c

   1 /*
   2  * Copyright (C) 2005 - 2014 Emulex
   3  * All rights reserved.
   4  *
   5  * This program is free software; you can redistribute it and/or
   6  * modify it under the terms of the GNU General Public License version 2
   7  * as published by the Free Software Foundation.  The full GNU General
   8  * Public License is included in this distribution in the file called COPYING.
   9  *
  10  * Contact Information:
  11  * linux-drivers@emulex.com
  12  *
  13  * Emulex
  14  * 3333 Susan Street
  15  * Costa Mesa, CA 92626
  16  */
  17
  18 #include <linux/prefetch.h>
  19 #include <linux/module.h>
  20 #include "be.h"
  21 #include "be_cmds.h"
  22 #include <asm/div64.h>
  23 #include <linux/aer.h>
  24 #include <linux/if_bridge.h>
  25 #include <net/busy_poll.h>
  26 #include <net/vxlan.h>
  27
  28 MODULE_VERSION(DRV_VER);
  29 MODULE_DEVICE_TABLE(pci, be_dev_ids);
  30 MODULE_DESCRIPTION(DRV_DESC " " DRV_VER);
  31 MODULE_AUTHOR("Emulex Corporation");
  32 MODULE_LICENSE("GPL");
  33
  34 static unsigned int num_vfs;
  35 module_param(num_vfs, uint, S_IRUGO);
  36 MODULE_PARM_DESC(num_vfs, "Number of PCI VFs to initialize");
  37
  38 static ushort rx_frag_size = 2048;
  39 module_param(rx_frag_size, ushort, S_IRUGO);
  40 MODULE_PARM_DESC(rx_frag_size, "Size of a fragment that holds rcvd data.");
  41
  42 static const struct pci_device_id be_dev_ids[] = {
  43         { PCI_DEVICE(BE_VENDOR_ID, BE_DEVICE_ID1) },
  44         { PCI_DEVICE(BE_VENDOR_ID, BE_DEVICE_ID2) },
  45         { PCI_DEVICE(BE_VENDOR_ID, OC_DEVICE_ID1) },
  46         { PCI_DEVICE(BE_VENDOR_ID, OC_DEVICE_ID2) },
  47         { PCI_DEVICE(EMULEX_VENDOR_ID, OC_DEVICE_ID3)},
  48         { PCI_DEVICE(EMULEX_VENDOR_ID, OC_DEVICE_ID4)},
  49         { PCI_DEVICE(EMULEX_VENDOR_ID, OC_DEVICE_ID5)},
  50         { PCI_DEVICE(EMULEX_VENDOR_ID, OC_DEVICE_ID6)},
  51         { 0 }
  52 };
  53 MODULE_DEVICE_TABLE(pci, be_dev_ids);
  54 /* UE Status Low CSR */
  55 static const char * const ue_status_low_desc[] = {
  56         "CEV",
  57         "CTX",
  58         "DBUF",
  59         "ERX",
  60         "Host",
  61         "MPU",
  62         "NDMA",
  63         "PTC ",
  64         "RDMA ",
  65         "RXF ",
  66         "RXIPS ",
  67         "RXULP0 ",
  68         "RXULP1 ",
  69         "RXULP2 ",
  70         "TIM ",
  71         "TPOST ",
  72         "TPRE ",
  73         "TXIPS ",
  74         "TXULP0 ",
  75         "TXULP1 ",
  76         "UC ",
  77         "WDMA ",
  78         "TXULP2 ",
  79         "HOST1 ",
  80         "P0_OB_LINK ",
  81         "P1_OB_LINK ",
  82         "HOST_GPIO ",
  83         "MBOX ",
  84         "ERX2 ",
  85         "SPARE ",
  86         "JTAG ",
  87         "MPU_INTPEND "
  88 };
  89
  90 /* UE Status High CSR */
  91 static const char * const ue_status_hi_desc[] = {
  92         "LPCMEMHOST",
  93         "MGMT_MAC",
  94         "PCS0ONLINE",
  95         "MPU_IRAM",
  96         "PCS1ONLINE",
  97         "PCTL0",
  98         "PCTL1",
  99         "PMEM",
 100         "RR",
 101         "TXPB",
 102         "RXPP",
 103         "XAUI",
 104         "TXP",
 105         "ARM",
 106         "IPC",
 107         "HOST2",
 108         "HOST3",
 109         "HOST4",
 110         "HOST5",
 111         "HOST6",
 112         "HOST7",
 113         "ECRC",
 114         "Poison TLP",
 115         "NETC",
 116         "PERIPH",
 117         "LLTXULP",
 118         "D2P",
 119         "RCON",
 120         "LDMA",
 121         "LLTXP",
 122         "LLTXPB",
 123         "Unknown"
 124 };
 125
 126 static void be_queue_free(struct be_adapter *adapter, struct be_queue_info *q)
 127 {
 128         struct be_dma_mem *mem = &q->dma_mem;
 129
 130         if (mem->va) {
 131                 dma_free_coherent(&adapter->pdev->dev, mem->size, mem->va,
 132                                   mem->dma);
 133                 mem->va = NULL;
 134         }
 135 }
 136
 137 static int be_queue_alloc(struct be_adapter *adapter, struct be_queue_info *q,
 138                           u16 len, u16 entry_size)
 139 {
 140         struct be_dma_mem *mem = &q->dma_mem;
 141
 142         memset(q, 0, sizeof(*q));
 143         q->len = len;
 144         q->entry_size = entry_size;
 145         mem->size = len * entry_size;
 146         mem->va = dma_zalloc_coherent(&adapter->pdev->dev, mem->size, &mem->dma,
 147                                       GFP_KERNEL);
 148         if (!mem->va)
 149                 return -ENOMEM;
 150         return 0;
 151 }
 152
 153 static void be_reg_intr_set(struct be_adapter *adapter, bool enable)
 154 {
 155         u32 reg, enabled;
 156
 157         pci_read_config_dword(adapter->pdev, PCICFG_MEMBAR_CTRL_INT_CTRL_OFFSET,
 158                               &reg);
 159         enabled = reg & MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK;
 160
 161         if (!enabled && enable)
 162                 reg |= MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK;
 163         else if (enabled && !enable)
 164                 reg &= ~MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK;
 165         else
 166                 return;
 167
 168         pci_write_config_dword(adapter->pdev,
 169                                PCICFG_MEMBAR_CTRL_INT_CTRL_OFFSET, reg);
 170 }
 171
 172 static void be_intr_set(struct be_adapter *adapter, bool enable)
 173 {
 174         int status = 0;
 175
 176         /* On lancer interrupts can't be controlled via this register */
 177         if (lancer_chip(adapter))
 178                 return;
 179
 180         if (adapter->eeh_error)
 181                 return;
 182
 183         status = be_cmd_intr_set(adapter, enable);
 184         if (status)
 185                 be_reg_intr_set(adapter, enable);
 186 }
 187
 188 static void be_rxq_notify(struct be_adapter *adapter, u16 qid, u16 posted)
 189 {
 190         u32 val = 0;
 191
 192         val |= qid & DB_RQ_RING_ID_MASK;
 193         val |= posted << DB_RQ_NUM_POSTED_SHIFT;
 194
 195         wmb();
 196         iowrite32(val, adapter->db + DB_RQ_OFFSET);
 197 }
 198
 199 static void be_txq_notify(struct be_adapter *adapter, struct be_tx_obj *txo,
 200                           u16 posted)
 201 {
 202         u32 val = 0;
 203
 204         val |= txo->q.id & DB_TXULP_RING_ID_MASK;
 205         val |= (posted & DB_TXULP_NUM_POSTED_MASK) << DB_TXULP_NUM_POSTED_SHIFT;
 206
 207         wmb();
 208         iowrite32(val, adapter->db + txo->db_offset);
 209 }
 210
 211 static void be_eq_notify(struct be_adapter *adapter, u16 qid,
 212                          bool arm, bool clear_int, u16 num_popped)
 213 {
 214         u32 val = 0;
 215
 216         val |= qid & DB_EQ_RING_ID_MASK;
 217         val |= ((qid & DB_EQ_RING_ID_EXT_MASK) << DB_EQ_RING_ID_EXT_MASK_SHIFT);
 218
 219         if (adapter->eeh_error)
 220                 return;
 221
 222         if (arm)
 223                 val |= 1 << DB_EQ_REARM_SHIFT;
 224         if (clear_int)
 225                 val |= 1 << DB_EQ_CLR_SHIFT;
 226         val |= 1 << DB_EQ_EVNT_SHIFT;
 227         val |= num_popped << DB_EQ_NUM_POPPED_SHIFT;
 228         iowrite32(val, adapter->db + DB_EQ_OFFSET);
 229 }
 230
 231 void be_cq_notify(struct be_adapter *adapter, u16 qid, bool arm, u16 num_popped)
 232 {
 233         u32 val = 0;
 234
 235         val |= qid & DB_CQ_RING_ID_MASK;
 236         val |= ((qid & DB_CQ_RING_ID_EXT_MASK) <<
 237                         DB_CQ_RING_ID_EXT_MASK_SHIFT);
 238
 239         if (adapter->eeh_error)
 240                 return;
 241
 242         if (arm)
 243                 val |= 1 << DB_CQ_REARM_SHIFT;
 244         val |= num_popped << DB_CQ_NUM_POPPED_SHIFT;
 245         iowrite32(val, adapter->db + DB_CQ_OFFSET);
 246 }
 247
 248 static int be_mac_addr_set(struct net_device *netdev, void *p)
 249 {
 250         struct be_adapter *adapter = netdev_priv(netdev);
 251         struct device *dev = &adapter->pdev->dev;
 252         struct sockaddr *addr = p;
 253         int status;
 254         u8 mac[ETH_ALEN];
 255         u32 old_pmac_id = adapter->pmac_id[0], curr_pmac_id = 0;
 256
 257         if (!is_valid_ether_addr(addr->sa_data))
 258                 return -EADDRNOTAVAIL;
 259
 260         /* Proceed further only if, User provided MAC is different
 261          * from active MAC
 262          */
 263         if (ether_addr_equal(addr->sa_data, netdev->dev_addr))
 264                 return 0;
 265
 266         /* The PMAC_ADD cmd may fail if the VF doesn't have FILTMGMT
 267          * privilege or if PF did not provision the new MAC address.
 268          * On BE3, this cmd will always fail if the VF doesn't have the
 269          * FILTMGMT privilege. This failure is OK, only if the PF programmed
 270          * the MAC for the VF.
 271          */
 272         status = be_cmd_pmac_add(adapter, (u8 *)addr->sa_data,
 273                                  adapter->if_handle, &adapter->pmac_id[0], 0);
 274         if (!status) {
 275                 curr_pmac_id = adapter->pmac_id[0];
 276
 277                 /* Delete the old programmed MAC. This call may fail if the
 278                  * old MAC was already deleted by the PF driver.
 279                  */
 280                 if (adapter->pmac_id[0] != old_pmac_id)
 281                         be_cmd_pmac_del(adapter, adapter->if_handle,
 282                                         old_pmac_id, 0);
 283         }
 284
 285         /* Decide if the new MAC is successfully activated only after
 286          * querying the FW
 287          */
 288         status = be_cmd_get_active_mac(adapter, curr_pmac_id, mac,
 289                                        adapter->if_handle, true, 0);
 290         if (status)
 291                 goto err;
 292
 293         /* The MAC change did not happen, either due to lack of privilege
 294          * or PF didn't pre-provision.
 295          */
 296         if (!ether_addr_equal(addr->sa_data, mac)) {
 297                 status = -EPERM;
 298                 goto err;
 299         }
 300
 301         memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len);
 302         dev_info(dev, "MAC address changed to %pM\n", mac);
 303         return 0;
 304 err:
 305         dev_warn(dev, "MAC address change to %pM failed\n", addr->sa_data);
 306         return status;
 307 }
 308
 309 /* BE2 supports only v0 cmd */
 310 static void *hw_stats_from_cmd(struct be_adapter *adapter)
 311 {
 312         if (BE2_chip(adapter)) {
 313                 struct be_cmd_resp_get_stats_v0 *cmd = adapter->stats_cmd.va;
 314
 315                 return &cmd->hw_stats;
 316         } else if (BE3_chip(adapter)) {
 317                 struct be_cmd_resp_get_stats_v1 *cmd = adapter->stats_cmd.va;
 318
 319                 return &cmd->hw_stats;
 320         } else {
 321                 struct be_cmd_resp_get_stats_v2 *cmd = adapter->stats_cmd.va;
 322
 323                 return &cmd->hw_stats;
 324         }
 325 }
 326
 327 /* BE2 supports only v0 cmd */
 328 static void *be_erx_stats_from_cmd(struct be_adapter *adapter)
 329 {
 330         if (BE2_chip(adapter)) {
 331                 struct be_hw_stats_v0 *hw_stats = hw_stats_from_cmd(adapter);
 332
 333                 return &hw_stats->erx;
 334         } else if (BE3_chip(adapter)) {
 335                 struct be_hw_stats_v1 *hw_stats = hw_stats_from_cmd(adapter);
 336
 337                 return &hw_stats->erx;
 338         } else {
 339                 struct be_hw_stats_v2 *hw_stats = hw_stats_from_cmd(adapter);
 340
 341                 return &hw_stats->erx;
 342         }
 343 }
 344
 345 static void populate_be_v0_stats(struct be_adapter *adapter)
 346 {
 347         struct be_hw_stats_v0 *hw_stats = hw_stats_from_cmd(adapter);
 348         struct be_pmem_stats *pmem_sts = &hw_stats->pmem;
 349         struct be_rxf_stats_v0 *rxf_stats = &hw_stats->rxf;
 350         struct be_port_rxf_stats_v0 *port_stats =
 351                                         &rxf_stats->port[adapter->port_num];
 352         struct be_drv_stats *drvs = &adapter->drv_stats;
 353
 354         be_dws_le_to_cpu(hw_stats, sizeof(*hw_stats));
 355         drvs->rx_pause_frames = port_stats->rx_pause_frames;
 356         drvs->rx_crc_errors = port_stats->rx_crc_errors;
 357         drvs->rx_control_frames = port_stats->rx_control_frames;
 358         drvs->rx_in_range_errors = port_stats->rx_in_range_errors;
 359         drvs->rx_frame_too_long = port_stats->rx_frame_too_long;
 360         drvs->rx_dropped_runt = port_stats->rx_dropped_runt;
 361         drvs->rx_ip_checksum_errs = port_stats->rx_ip_checksum_errs;
 362         drvs->rx_tcp_checksum_errs = port_stats->rx_tcp_checksum_errs;
 363         drvs->rx_udp_checksum_errs = port_stats->rx_udp_checksum_errs;
 364         drvs->rxpp_fifo_overflow_drop = port_stats->rx_fifo_overflow;
 365         drvs->rx_dropped_tcp_length = port_stats->rx_dropped_tcp_length;
 366         drvs->rx_dropped_too_small = port_stats->rx_dropped_too_small;
 367         drvs->rx_dropped_too_short = port_stats->rx_dropped_too_short;
 368         drvs->rx_out_range_errors = port_stats->rx_out_range_errors;
 369         drvs->rx_input_fifo_overflow_drop = port_stats->rx_input_fifo_overflow;
 370         drvs->rx_dropped_header_too_small =
 371                 port_stats->rx_dropped_header_too_small;
 372         drvs->rx_address_filtered =
 373                                         port_stats->rx_address_filtered +
 374                                         port_stats->rx_vlan_filtered;
 375         drvs->rx_alignment_symbol_errors =
 376                 port_stats->rx_alignment_symbol_errors;
 377
 378         drvs->tx_pauseframes = port_stats->tx_pauseframes;
 379         drvs->tx_controlframes = port_stats->tx_controlframes;
 380
 381         if (adapter->port_num)
 382                 drvs->jabber_events = rxf_stats->port1_jabber_events;
 383         else
 384                 drvs->jabber_events = rxf_stats->port0_jabber_events;
 385         drvs->rx_drops_no_pbuf = rxf_stats->rx_drops_no_pbuf;
 386         drvs->rx_drops_no_erx_descr = rxf_stats->rx_drops_no_erx_descr;
 387         drvs->forwarded_packets = rxf_stats->forwarded_packets;
 388         drvs->rx_drops_mtu = rxf_stats->rx_drops_mtu;
 389         drvs->rx_drops_no_tpre_descr = rxf_stats->rx_drops_no_tpre_descr;
 390         drvs->rx_drops_too_many_frags = rxf_stats->rx_drops_too_many_frags;
 391         adapter->drv_stats.eth_red_drops = pmem_sts->eth_red_drops;
 392 }
 393
 394 static void populate_be_v1_stats(struct be_adapter *adapter)
 395 {
 396         struct be_hw_stats_v1 *hw_stats = hw_stats_from_cmd(adapter);
 397         struct be_pmem_stats *pmem_sts = &hw_stats->pmem;
 398         struct be_rxf_stats_v1 *rxf_stats = &hw_stats->rxf;
 399         struct be_port_rxf_stats_v1 *port_stats =
 400                                         &rxf_stats->port[adapter->port_num];
 401         struct be_drv_stats *drvs = &adapter->drv_stats;
 402
 403         be_dws_le_to_cpu(hw_stats, sizeof(*hw_stats));
 404         drvs->pmem_fifo_overflow_drop = port_stats->pmem_fifo_overflow_drop;
 405         drvs->rx_priority_pause_frames = port_stats->rx_priority_pause_frames;
 406         drvs->rx_pause_frames = port_stats->rx_pause_frames;
 407         drvs->rx_crc_errors = port_stats->rx_crc_errors;
 408         drvs->rx_control_frames = port_stats->rx_control_frames;
 409         drvs->rx_in_range_errors = port_stats->rx_in_range_errors;
 410         drvs->rx_frame_too_long = port_stats->rx_frame_too_long;
 411         drvs->rx_dropped_runt = port_stats->rx_dropped_runt;
 412         drvs->rx_ip_checksum_errs = port_stats->rx_ip_checksum_errs;
 413         drvs->rx_tcp_checksum_errs = port_stats->rx_tcp_checksum_errs;
 414         drvs->rx_udp_checksum_errs = port_stats->rx_udp_checksum_errs;
 415         drvs->rx_dropped_tcp_length = port_stats->rx_dropped_tcp_length;
 416         drvs->rx_dropped_too_small = port_stats->rx_dropped_too_small;
 417         drvs->rx_dropped_too_short = port_stats->rx_dropped_too_short;
 418         drvs->rx_out_range_errors = port_stats->rx_out_range_errors;
 419         drvs->rx_dropped_header_too_small =
 420                 port_stats->rx_dropped_header_too_small;
 421         drvs->rx_input_fifo_overflow_drop =
 422                 port_stats->rx_input_fifo_overflow_drop;
 423         drvs->rx_address_filtered = port_stats->rx_address_filtered;
 424         drvs->rx_alignment_symbol_errors =
 425                 port_stats->rx_alignment_symbol_errors;
 426         drvs->rxpp_fifo_overflow_drop = port_stats->rxpp_fifo_overflow_drop;
 427         drvs->tx_pauseframes = port_stats->tx_pauseframes;
 428         drvs->tx_controlframes = port_stats->tx_controlframes;
 429         drvs->tx_priority_pauseframes = port_stats->tx_priority_pauseframes;
 430         drvs->jabber_events = port_stats->jabber_events;
 431         drvs->rx_drops_no_pbuf = rxf_stats->rx_drops_no_pbuf;
 432         drvs->rx_drops_no_erx_descr = rxf_stats->rx_drops_no_erx_descr;
 433         drvs->forwarded_packets = rxf_stats->forwarded_packets;
 434         drvs->rx_drops_mtu = rxf_stats->rx_drops_mtu;
 435         drvs->rx_drops_no_tpre_descr = rxf_stats->rx_drops_no_tpre_descr;
 436         drvs->rx_drops_too_many_frags = rxf_stats->rx_drops_too_many_frags;
 437         adapter->drv_stats.eth_red_drops = pmem_sts->eth_red_drops;
 438 }
 439
 440 static void populate_be_v2_stats(struct be_adapter *adapter)
 441 {
 442         struct be_hw_stats_v2 *hw_stats = hw_stats_from_cmd(adapter);
 443         struct be_pmem_stats *pmem_sts = &hw_stats->pmem;
 444         struct be_rxf_stats_v2 *rxf_stats = &hw_stats->rxf;
 445         struct be_port_rxf_stats_v2 *port_stats =
 446                                         &rxf_stats->port[adapter->port_num];
 447         struct be_drv_stats *drvs = &adapter->drv_stats;
 448
 449         be_dws_le_to_cpu(hw_stats, sizeof(*hw_stats));
 450         drvs->pmem_fifo_overflow_drop = port_stats->pmem_fifo_overflow_drop;
 451         drvs->rx_priority_pause_frames = port_stats->rx_priority_pause_frames;
 452         drvs->rx_pause_frames = port_stats->rx_pause_frames;
 453         drvs->rx_crc_errors = port_stats->rx_crc_errors;
 454         drvs->rx_control_frames = port_stats->rx_control_frames;
 455         drvs->rx_in_range_errors = port_stats->rx_in_range_errors;
 456         drvs->rx_frame_too_long = port_stats->rx_frame_too_long;
 457         drvs->rx_dropped_runt = port_stats->rx_dropped_runt;
 458         drvs->rx_ip_checksum_errs = port_stats->rx_ip_checksum_errs;
 459         drvs->rx_tcp_checksum_errs = port_stats->rx_tcp_checksum_errs;
 460         drvs->rx_udp_checksum_errs = port_stats->rx_udp_checksum_errs;
 461         drvs->rx_dropped_tcp_length = port_stats->rx_dropped_tcp_length;
 462         drvs->rx_dropped_too_small = port_stats->rx_dropped_too_small;
 463         drvs->rx_dropped_too_short = port_stats->rx_dropped_too_short;
 464         drvs->rx_out_range_errors = port_stats->rx_out_range_errors;
 465         drvs->rx_dropped_header_too_small =
 466                 port_stats->rx_dropped_header_too_small;
 467         drvs->rx_input_fifo_overflow_drop =
 468                 port_stats->rx_input_fifo_overflow_drop;
 469         drvs->rx_address_filtered = port_stats->rx_address_filtered;
 470         drvs->rx_alignment_symbol_errors =
 471                 port_stats->rx_alignment_symbol_errors;
 472         drvs->rxpp_fifo_overflow_drop = port_stats->rxpp_fifo_overflow_drop;
 473         drvs->tx_pauseframes = port_stats->tx_pauseframes;
 474         drvs->tx_controlframes = port_stats->tx_controlframes;
 475         drvs->tx_priority_pauseframes = port_stats->tx_priority_pauseframes;
 476         drvs->jabber_events = port_stats->jabber_events;
 477         drvs->rx_drops_no_pbuf = rxf_stats->rx_drops_no_pbuf;
 478         drvs->rx_drops_no_erx_descr = rxf_stats->rx_drops_no_erx_descr;
 479         drvs->forwarded_packets = rxf_stats->forwarded_packets;
 480         drvs->rx_drops_mtu = rxf_stats->rx_drops_mtu;
 481         drvs->rx_drops_no_tpre_descr = rxf_stats->rx_drops_no_tpre_descr;
 482         drvs->rx_drops_too_many_frags = rxf_stats->rx_drops_too_many_frags;
 483         adapter->drv_stats.eth_red_drops = pmem_sts->eth_red_drops;
 484         if (be_roce_supported(adapter)) {
 485                 drvs->rx_roce_bytes_lsd = port_stats->roce_bytes_received_lsd;
 486                 drvs->rx_roce_bytes_msd = port_stats->roce_bytes_received_msd;
 487                 drvs->rx_roce_frames = port_stats->roce_frames_received;
 488                 drvs->roce_drops_crc = port_stats->roce_drops_crc;
 489                 drvs->roce_drops_payload_len =
 490                         port_stats->roce_drops_payload_len;
 491         }
 492 }
 493
 494 static void populate_lancer_stats(struct be_adapter *adapter)
 495 {
 496         struct be_drv_stats *drvs = &adapter->drv_stats;
 497         struct lancer_pport_stats *pport_stats = pport_stats_from_cmd(adapter);
 498
 499         be_dws_le_to_cpu(pport_stats, sizeof(*pport_stats));
 500         drvs->rx_pause_frames = pport_stats->rx_pause_frames_lo;
 501         drvs->rx_crc_errors = pport_stats->rx_crc_errors_lo;
 502         drvs->rx_control_frames = pport_stats->rx_control_frames_lo;
 503         drvs->rx_in_range_errors = pport_stats->rx_in_range_errors;
 504         drvs->rx_frame_too_long = pport_stats->rx_frames_too_long_lo;
 505         drvs->rx_dropped_runt = pport_stats->rx_dropped_runt;
 506         drvs->rx_ip_checksum_errs = pport_stats->rx_ip_checksum_errors;
 507         drvs->rx_tcp_checksum_errs = pport_stats->rx_tcp_checksum_errors;
 508         drvs->rx_udp_checksum_errs = pport_stats->rx_udp_checksum_errors;
 509         drvs->rx_dropped_tcp_length =
 510                                 pport_stats->rx_dropped_invalid_tcp_length;
 511         drvs->rx_dropped_too_small = pport_stats->rx_dropped_too_small;
 512         drvs->rx_dropped_too_short = pport_stats->rx_dropped_too_short;
 513         drvs->rx_out_range_errors = pport_stats->rx_out_of_range_errors;
 514         drvs->rx_dropped_header_too_small =
 515                                 pport_stats->rx_dropped_header_too_small;
 516         drvs->rx_input_fifo_overflow_drop = pport_stats->rx_fifo_overflow;
 517         drvs->rx_address_filtered =
 518                                         pport_stats->rx_address_filtered +
 519                                         pport_stats->rx_vlan_filtered;
 520         drvs->rx_alignment_symbol_errors = pport_stats->rx_symbol_errors_lo;
 521         drvs->rxpp_fifo_overflow_drop = pport_stats->rx_fifo_overflow;
 522         drvs->tx_pauseframes = pport_stats->tx_pause_frames_lo;
 523         drvs->tx_controlframes = pport_stats->tx_control_frames_lo;
 524         drvs->jabber_events = pport_stats->rx_jabbers;
 525         drvs->forwarded_packets = pport_stats->num_forwards_lo;
 526         drvs->rx_drops_mtu = pport_stats->rx_drops_mtu_lo;
 527         drvs->rx_drops_too_many_frags =
 528                                 pport_stats->rx_drops_too_many_frags_lo;
 529 }
 530
 531 static void accumulate_16bit_val(u32 *acc, u16 val)
 532 {
 533 #define lo(x)                   (x & 0xFFFF)
 534 #define hi(x)                   (x & 0xFFFF0000)
 535         bool wrapped = val < lo(*acc);
 536         u32 newacc = hi(*acc) + val;
 537
 538         if (wrapped)
 539                 newacc += 65536;
 540         ACCESS_ONCE(*acc) = newacc;
 541 }
 542
 543 static void populate_erx_stats(struct be_adapter *adapter,
 544                                struct be_rx_obj *rxo, u32 erx_stat)
 545 {
 546         if (!BEx_chip(adapter))
 547                 rx_stats(rxo)->rx_drops_no_frags = erx_stat;
 548         else
 549                 /* below erx HW counter can actually wrap around after
 550                  * 65535. Driver accumulates a 32-bit value
 551                  */
 552                 accumulate_16bit_val(&rx_stats(rxo)->rx_drops_no_frags,
 553                                      (u16)erx_stat);
 554 }
 555
 556 void be_parse_stats(struct be_adapter *adapter)
 557 {
 558         struct be_erx_stats_v2 *erx = be_erx_stats_from_cmd(adapter);
 559         struct be_rx_obj *rxo;
 560         int i;
 561         u32 erx_stat;
 562
 563         if (lancer_chip(adapter)) {
 564                 populate_lancer_stats(adapter);
 565         } else {
 566                 if (BE2_chip(adapter))
 567                         populate_be_v0_stats(adapter);
 568                 else if (BE3_chip(adapter))
 569                         /* for BE3 */
 570                         populate_be_v1_stats(adapter);
 571                 else
 572                         populate_be_v2_stats(adapter);
 573
 574                 /* erx_v2 is longer than v0, v1. use v2 for v0, v1 access */
 575                 for_all_rx_queues(adapter, rxo, i) {
 576                         erx_stat = erx->rx_drops_no_fragments[rxo->q.id];
 577                         populate_erx_stats(adapter, rxo, erx_stat);
 578                 }
 579         }
 580 }
 581
 582 static struct rtnl_link_stats64 *be_get_stats64(struct net_device *netdev,
 583                                                 struct rtnl_link_stats64 *stats)
 584 {
 585         struct be_adapter *adapter = netdev_priv(netdev);
 586         struct be_drv_stats *drvs = &adapter->drv_stats;
 587         struct be_rx_obj *rxo;
 588         struct be_tx_obj *txo;
 589         u64 pkts, bytes;
 590         unsigned int start;
 591         int i;
 592
 593         for_all_rx_queues(adapter, rxo, i) {
 594                 const struct be_rx_stats *rx_stats = rx_stats(rxo);
 595
 596                 do {
 597                         start = u64_stats_fetch_begin_irq(&rx_stats->sync);
 598                         pkts = rx_stats(rxo)->rx_pkts;
 599                         bytes = rx_stats(rxo)->rx_bytes;
 600                 } while (u64_stats_fetch_retry_irq(&rx_stats->sync, start));
 601                 stats->rx_packets += pkts;
 602                 stats->rx_bytes += bytes;
 603                 stats->multicast += rx_stats(rxo)->rx_mcast_pkts;
 604                 stats->rx_dropped += rx_stats(rxo)->rx_drops_no_skbs +
 605                                         rx_stats(rxo)->rx_drops_no_frags;
 606         }
 607
 608         for_all_tx_queues(adapter, txo, i) {
 609                 const struct be_tx_stats *tx_stats = tx_stats(txo);
 610
 611                 do {
 612                         start = u64_stats_fetch_begin_irq(&tx_stats->sync);
 613                         pkts = tx_stats(txo)->tx_pkts;
 614                         bytes = tx_stats(txo)->tx_bytes;
 615                 } while (u64_stats_fetch_retry_irq(&tx_stats->sync, start));
 616                 stats->tx_packets += pkts;
 617                 stats->tx_bytes += bytes;
 618         }
 619
 620         /* bad pkts received */
 621         stats->rx_errors = drvs->rx_crc_errors +
 622                 drvs->rx_alignment_symbol_errors +
 623                 drvs->rx_in_range_errors +
 624                 drvs->rx_out_range_errors +
 625                 drvs->rx_frame_too_long +
 626                 drvs->rx_dropped_too_small +
 627                 drvs->rx_dropped_too_short +
 628                 drvs->rx_dropped_header_too_small +
 629                 drvs->rx_dropped_tcp_length +
 630                 drvs->rx_dropped_runt;
 631
 632         /* detailed rx errors */
 633         stats->rx_length_errors = drvs->rx_in_range_errors +
 634                 drvs->rx_out_range_errors +
 635                 drvs->rx_frame_too_long;
 636
 637         stats->rx_crc_errors = drvs->rx_crc_errors;
 638
 639         /* frame alignment errors */
 640         stats->rx_frame_errors = drvs->rx_alignment_symbol_errors;
 641
 642         /* receiver fifo overrun */
 643         /* drops_no_pbuf is no per i/f, it's per BE card */
 644         stats->rx_fifo_errors = drvs->rxpp_fifo_overflow_drop +
 645                                 drvs->rx_input_fifo_overflow_drop +
 646                                 drvs->rx_drops_no_pbuf;
 647         return stats;
 648 }
 649
 650 void be_link_status_update(struct be_adapter *adapter, u8 link_status)
 651 {
 652         struct net_device *netdev = adapter->netdev;
 653
 654         if (!(adapter->flags & BE_FLAGS_LINK_STATUS_INIT)) {
 655                 netif_carrier_off(netdev);
 656                 adapter->flags |= BE_FLAGS_LINK_STATUS_INIT;
 657         }
 658
 659         if (link_status)
 660                 netif_carrier_on(netdev);
 661         else
 662                 netif_carrier_off(netdev);
 663 }
 664
 665 static void be_tx_stats_update(struct be_tx_obj *txo, struct sk_buff *skb)
 666 {
 667         struct be_tx_stats *stats = tx_stats(txo);
 668
 669         u64_stats_update_begin(&stats->sync);
 670         stats->tx_reqs++;
 671         stats->tx_bytes += skb->len;
 672         stats->tx_pkts += (skb_shinfo(skb)->gso_segs ? : 1);
 673         u64_stats_update_end(&stats->sync);
 674 }
 675
 676 /* Returns number of WRBs needed for the skb */
 677 static u32 skb_wrb_cnt(struct sk_buff *skb)
 678 {
 679         /* +1 for the header wrb */
 680         return 1 + (skb_headlen(skb) ? 1 : 0) + skb_shinfo(skb)->nr_frags;
 681 }
 682
 683 static inline void wrb_fill(struct be_eth_wrb *wrb, u64 addr, int len)
 684 {
 685         wrb->frag_pa_hi = upper_32_bits(addr);
 686         wrb->frag_pa_lo = addr & 0xFFFFFFFF;
 687         wrb->frag_len = len & ETH_WRB_FRAG_LEN_MASK;
 688         wrb->rsvd0 = 0;
 689 }
 690
 691 static inline u16 be_get_tx_vlan_tag(struct be_adapter *adapter,
 692                                      struct sk_buff *skb)
 693 {
 694         u8 vlan_prio;
 695         u16 vlan_tag;
 696
 697         vlan_tag = skb_vlan_tag_get(skb);
 698         vlan_prio = (vlan_tag & VLAN_PRIO_MASK) >> VLAN_PRIO_SHIFT;
 699         /* If vlan priority provided by OS is NOT in available bmap */
 700         if (!(adapter->vlan_prio_bmap & (1 << vlan_prio)))
 701                 vlan_tag = (vlan_tag & ~VLAN_PRIO_MASK) |
 702                                 adapter->recommended_prio;
 703
 704         return vlan_tag;
 705 }
 706
 707 /* Used only for IP tunnel packets */
 708 static u16 skb_inner_ip_proto(struct sk_buff *skb)
 709 {
 710         return (inner_ip_hdr(skb)->version == 4) ?
 711                 inner_ip_hdr(skb)->protocol : inner_ipv6_hdr(skb)->nexthdr;
 712 }
 713
 714 static u16 skb_ip_proto(struct sk_buff *skb)
 715 {
 716         return (ip_hdr(skb)->version == 4) ?
 717                 ip_hdr(skb)->protocol : ipv6_hdr(skb)->nexthdr;
 718 }
 719
 720 static void wrb_fill_hdr(struct be_adapter *adapter, struct be_eth_hdr_wrb *hdr,
 721                          struct sk_buff *skb, u32 wrb_cnt, u32 len,
 722                          bool skip_hw_vlan)
 723 {
 724         u16 vlan_tag, proto;
 725
 726         memset(hdr, 0, sizeof(*hdr));
 727
 728         SET_TX_WRB_HDR_BITS(crc, hdr, 1);
 729
 730         if (skb_is_gso(skb)) {
 731                 SET_TX_WRB_HDR_BITS(lso, hdr, 1);
 732                 SET_TX_WRB_HDR_BITS(lso_mss, hdr, skb_shinfo(skb)->gso_size);
 733                 if (skb_is_gso_v6(skb) && !lancer_chip(adapter))
 734                         SET_TX_WRB_HDR_BITS(lso6, hdr, 1);
 735         } else if (skb->ip_summed == CHECKSUM_PARTIAL) {
 736                 if (skb->encapsulation) {
 737                         SET_TX_WRB_HDR_BITS(ipcs, hdr, 1);
 738                         proto = skb_inner_ip_proto(skb);
 739                 } else {
 740                         proto = skb_ip_proto(skb);
 741                 }
 742                 if (proto == IPPROTO_TCP)
 743                         SET_TX_WRB_HDR_BITS(tcpcs, hdr, 1);
 744                 else if (proto == IPPROTO_UDP)
 745                         SET_TX_WRB_HDR_BITS(udpcs, hdr, 1);
 746         }
 747
 748         if (skb_vlan_tag_present(skb)) {
 749                 SET_TX_WRB_HDR_BITS(vlan, hdr, 1);
 750                 vlan_tag = be_get_tx_vlan_tag(adapter, skb);
 751                 SET_TX_WRB_HDR_BITS(vlan_tag, hdr, vlan_tag);
 752         }
 753
 754         SET_TX_WRB_HDR_BITS(num_wrb, hdr, wrb_cnt);
 755         SET_TX_WRB_HDR_BITS(len, hdr, len);
 756
 757         /* Hack to skip HW VLAN tagging needs evt = 1, compl = 0
 758          * When this hack is not needed, the evt bit is set while ringing DB
 759          */
 760         if (skip_hw_vlan)
 761                 SET_TX_WRB_HDR_BITS(event, hdr, 1);
 762 }
 763
 764 static void unmap_tx_frag(struct device *dev, struct be_eth_wrb *wrb,
 765                           bool unmap_single)
 766 {
 767         dma_addr_t dma;
 768
 769         be_dws_le_to_cpu(wrb, sizeof(*wrb));
 770
 771         dma = (u64)wrb->frag_pa_hi << 32 | (u64)wrb->frag_pa_lo;
 772         if (wrb->frag_len) {
 773                 if (unmap_single)
 774                         dma_unmap_single(dev, dma, wrb->frag_len,
 775                                          DMA_TO_DEVICE);
 776                 else
 777                         dma_unmap_page(dev, dma, wrb->frag_len, DMA_TO_DEVICE);
 778         }
 779 }
 780
 781 /* Returns the number of WRBs used up by the skb */
 782 static u32 be_xmit_enqueue(struct be_adapter *adapter, struct be_tx_obj *txo,
 783                            struct sk_buff *skb, bool skip_hw_vlan)
 784 {
 785         u32 i, copied = 0, wrb_cnt = skb_wrb_cnt(skb);
 786         struct device *dev = &adapter->pdev->dev;
 787         struct be_queue_info *txq = &txo->q;
 788         struct be_eth_hdr_wrb *hdr;
 789         bool map_single = false;
 790         struct be_eth_wrb *wrb;
 791         dma_addr_t busaddr;
 792         u16 head = txq->head;
 793
 794         hdr = queue_head_node(txq);
 795         wrb_fill_hdr(adapter, hdr, skb, wrb_cnt, skb->len, skip_hw_vlan);
 796         be_dws_cpu_to_le(hdr, sizeof(*hdr));
 797
 798         queue_head_inc(txq);
 799
 800         if (skb->len > skb->data_len) {
 801                 int len = skb_headlen(skb);
 802
 803                 busaddr = dma_map_single(dev, skb->data, len, DMA_TO_DEVICE);
 804                 if (dma_mapping_error(dev, busaddr))
 805                         goto dma_err;
 806                 map_single = true;
 807                 wrb = queue_head_node(txq);
 808                 wrb_fill(wrb, busaddr, len);
 809                 be_dws_cpu_to_le(wrb, sizeof(*wrb));
 810                 queue_head_inc(txq);
 811                 copied += len;
 812         }
 813
 814         for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
 815                 const struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[i];
 816
 817                 busaddr = skb_frag_dma_map(dev, frag, 0,
 818                                            skb_frag_size(frag), DMA_TO_DEVICE);
 819                 if (dma_mapping_error(dev, busaddr))
 820                         goto dma_err;
 821                 wrb = queue_head_node(txq);
 822                 wrb_fill(wrb, busaddr, skb_frag_size(frag));
 823                 be_dws_cpu_to_le(wrb, sizeof(*wrb));
 824                 queue_head_inc(txq);
 825                 copied += skb_frag_size(frag);
 826         }
 827
 828         BUG_ON(txo->sent_skb_list[head]);
 829         txo->sent_skb_list[head] = skb;
 830         txo->last_req_hdr = head;
 831         atomic_add(wrb_cnt, &txq->used);
 832         txo->last_req_wrb_cnt = wrb_cnt;
 833         txo->pend_wrb_cnt += wrb_cnt;
 834
 835         be_tx_stats_update(txo, skb);
 836         return wrb_cnt;
 837
 838 dma_err:
 839         /* Bring the queue back to the state it was in before this
 840          * routine was invoked.
 841          */
 842         txq->head = head;
 843         /* skip the first wrb (hdr); it's not mapped */
 844         queue_head_inc(txq);
 845         while (copied) {
 846                 wrb = queue_head_node(txq);
 847                 unmap_tx_frag(dev, wrb, map_single);
 848                 map_single = false;
 849                 copied -= wrb->frag_len;
 850                 adapter->drv_stats.dma_map_errors++;
 851                 queue_head_inc(txq);
 852         }
 853         txq->head = head;
 854         return 0;
 855 }
 856
 857 static struct sk_buff *be_insert_vlan_in_pkt(struct be_adapter *adapter,
 858                                              struct sk_buff *skb,
 859                                              bool *skip_hw_vlan)
 860 {
 861         u16 vlan_tag = 0;
 862
 863         skb = skb_share_check(skb, GFP_ATOMIC);
 864         if (unlikely(!skb))
 865                 return skb;
 866
 867         if (skb_vlan_tag_present(skb))
 868                 vlan_tag = be_get_tx_vlan_tag(adapter, skb);
 869
 870         if (qnq_async_evt_rcvd(adapter) && adapter->pvid) {
 871                 if (!vlan_tag)
 872                         vlan_tag = adapter->pvid;
 873                 /* f/w workaround to set skip_hw_vlan = 1, informs the F/W to
 874                  * skip VLAN insertion
 875                  */
 876                 if (skip_hw_vlan)
 877                         *skip_hw_vlan = true;
 878         }
 879
 880         if (vlan_tag) {
 881                 skb = vlan_insert_tag_set_proto(skb, htons(ETH_P_8021Q),
 882                                                 vlan_tag);
 883                 if (unlikely(!skb))
 884                         return skb;
 885                 skb->vlan_tci = 0;
 886         }
 887
 888         /* Insert the outer VLAN, if any */
 889         if (adapter->qnq_vid) {
 890                 vlan_tag = adapter->qnq_vid;
 891                 skb = vlan_insert_tag_set_proto(skb, htons(ETH_P_8021Q),
 892                                                 vlan_tag);
 893                 if (unlikely(!skb))
 894                         return skb;
 895                 if (skip_hw_vlan)
 896                         *skip_hw_vlan = true;
 897         }
 898
 899         return skb;
 900 }
 901
 902 static bool be_ipv6_exthdr_check(struct sk_buff *skb)
 903 {
 904         struct ethhdr *eh = (struct ethhdr *)skb->data;
 905         u16 offset = ETH_HLEN;
 906
 907         if (eh->h_proto == htons(ETH_P_IPV6)) {
 908                 struct ipv6hdr *ip6h = (struct ipv6hdr *)(skb->data + offset);
 909
 910                 offset += sizeof(struct ipv6hdr);
 911                 if (ip6h->nexthdr != NEXTHDR_TCP &&
 912                     ip6h->nexthdr != NEXTHDR_UDP) {
 913                         struct ipv6_opt_hdr *ehdr =
 914                                 (struct ipv6_opt_hdr *)(skb->data + offset);
 915
 916                         /* offending pkt: 2nd byte following IPv6 hdr is 0xff */
 917                         if (ehdr->hdrlen == 0xff)
 918                                 return true;
 919                 }
 920         }
 921         return false;
 922 }
 923
 924 static int be_vlan_tag_tx_chk(struct be_adapter *adapter, struct sk_buff *skb)
 925 {
 926         return skb_vlan_tag_present(skb) || adapter->pvid || adapter->qnq_vid;
 927 }
 928
 929 static int be_ipv6_tx_stall_chk(struct be_adapter *adapter, struct sk_buff *skb)
 930 {
 931         return BE3_chip(adapter) && be_ipv6_exthdr_check(skb);
 932 }
 933
 934 static struct sk_buff *be_lancer_xmit_workarounds(struct be_adapter *adapter,
 935                                                   struct sk_buff *skb,
 936                                                   bool *skip_hw_vlan)
 937 {
 938         struct vlan_ethhdr *veh = (struct vlan_ethhdr *)skb->data;
 939         unsigned int eth_hdr_len;
 940         struct iphdr *ip;
 941
 942         /* For padded packets, BE HW modifies tot_len field in IP header
 943          * incorrecly when VLAN tag is inserted by HW.
 944          * For padded packets, Lancer computes incorrect checksum.
 945          */
 946         eth_hdr_len = ntohs(skb->protocol) == ETH_P_8021Q ?
 947                                                 VLAN_ETH_HLEN : ETH_HLEN;
 948         if (skb->len <= 60 &&
 949             (lancer_chip(adapter) || skb_vlan_tag_present(skb)) &&
 950             is_ipv4_pkt(skb)) {
 951                 ip = (struct iphdr *)ip_hdr(skb);
 952                 pskb_trim(skb, eth_hdr_len + ntohs(ip->tot_len));
 953         }
 954
 955         /* If vlan tag is already inlined in the packet, skip HW VLAN
 956          * tagging in pvid-tagging mode
 957          */
 958         if (be_pvid_tagging_enabled(adapter) &&
 959             veh->h_vlan_proto == htons(ETH_P_8021Q))
 960                 *skip_hw_vlan = true;
 961
 962         /* HW has a bug wherein it will calculate CSUM for VLAN
 963          * pkts even though it is disabled.
 964          * Manually insert VLAN in pkt.
 965          */
 966         if (skb->ip_summed != CHECKSUM_PARTIAL &&
 967             skb_vlan_tag_present(skb)) {
 968                 skb = be_insert_vlan_in_pkt(adapter, skb, skip_hw_vlan);
 969                 if (unlikely(!skb))
 970                         goto err;
 971         }
 972
 973         /* HW may lockup when VLAN HW tagging is requested on
 974          * certain ipv6 packets. Drop such pkts if the HW workaround to
 975          * skip HW tagging is not enabled by FW.
 976          */
 977         if (unlikely(be_ipv6_tx_stall_chk(adapter, skb) &&
 978                      (adapter->pvid || adapter->qnq_vid) &&
 979                      !qnq_async_evt_rcvd(adapter)))
 980                 goto tx_drop;
 981
 982         /* Manual VLAN tag insertion to prevent:
 983          * ASIC lockup when the ASIC inserts VLAN tag into
 984          * certain ipv6 packets. Insert VLAN tags in driver,
 985          * and set event, completion, vlan bits accordingly
 986          * in the Tx WRB.
 987          */
 988         if (be_ipv6_tx_stall_chk(adapter, skb) &&
 989             be_vlan_tag_tx_chk(adapter, skb)) {
 990                 skb = be_insert_vlan_in_pkt(adapter, skb, skip_hw_vlan);
 991                 if (unlikely(!skb))
 992                         goto err;
 993         }
 994
 995         return skb;
 996 tx_drop:
 997         dev_kfree_skb_any(skb);
 998 err:
 999         return NULL;
1000 }
1001
1002 static struct sk_buff *be_xmit_workarounds(struct be_adapter *adapter,
1003                                            struct sk_buff *skb,
1004                                            bool *skip_hw_vlan)
1005 {
1006         /* Lancer, SH-R ASICs have a bug wherein Packets that are 32 bytes or
1007          * less may cause a transmit stall on that port. So the work-around is
1008          * to pad short packets (<= 32 bytes) to a 36-byte length.
1009          */
1010         if (unlikely(!BEx_chip(adapter) && skb->len <= 32)) {
1011                 if (skb_put_padto(skb, 36))
1012                         return NULL;
1013         }
1014
1015         if (BEx_chip(adapter) || lancer_chip(adapter)) {
1016                 skb = be_lancer_xmit_workarounds(adapter, skb, skip_hw_vlan);
1017                 if (!skb)
1018                         return NULL;
1019         }
1020
1021         return skb;
1022 }
1023
1024 static void be_xmit_flush(struct be_adapter *adapter, struct be_tx_obj *txo)
1025 {
1026         struct be_queue_info *txq = &txo->q;
1027         struct be_eth_hdr_wrb *hdr = queue_index_node(txq, txo->last_req_hdr);
1028
1029         /* Mark the last request eventable if it hasn't been marked already */
1030         if (!(hdr->dw[2] & cpu_to_le32(TX_HDR_WRB_EVT)))
1031                 hdr->dw[2] |= cpu_to_le32(TX_HDR_WRB_EVT | TX_HDR_WRB_COMPL);
1032
1033         /* compose a dummy wrb if there are odd set of wrbs to notify */
1034         if (!lancer_chip(adapter) && (txo->pend_wrb_cnt & 1)) {
1035                 wrb_fill(queue_head_node(txq), 0, 0);
1036                 queue_head_inc(txq);
1037                 atomic_inc(&txq->used);
1038                 txo->pend_wrb_cnt++;
1039                 hdr->dw[2] &= ~cpu_to_le32(TX_HDR_WRB_NUM_MASK <<
1040                                            TX_HDR_WRB_NUM_SHIFT);
1041                 hdr->dw[2] |= cpu_to_le32((txo->last_req_wrb_cnt + 1) <<
1042                                           TX_HDR_WRB_NUM_SHIFT);
1043         }
1044         be_txq_notify(adapter, txo, txo->pend_wrb_cnt);
1045         txo->pend_wrb_cnt = 0;
1046 }
1047
1048 static netdev_tx_t be_xmit(struct sk_buff *skb, struct net_device *netdev)
1049 {
1050         bool skip_hw_vlan = false, flush = !skb->xmit_more;
1051         struct be_adapter *adapter = netdev_priv(netdev);
1052         u16 q_idx = skb_get_queue_mapping(skb);
1053         struct be_tx_obj *txo = &adapter->tx_obj[q_idx];
1054         struct be_queue_info *txq = &txo->q;
1055         u16 wrb_cnt;
1056
1057         skb = be_xmit_workarounds(adapter, skb, &skip_hw_vlan);
1058         if (unlikely(!skb))
1059                 goto drop;
1060
1061         wrb_cnt = be_xmit_enqueue(adapter, txo, skb, skip_hw_vlan);
1062         if (unlikely(!wrb_cnt)) {
1063                 dev_kfree_skb_any(skb);
1064                 goto drop;
1065         }
1066
1067         if ((atomic_read(&txq->used) + BE_MAX_TX_FRAG_COUNT) >= txq->len) {
1068                 netif_stop_subqueue(netdev, q_idx);
1069                 tx_stats(txo)->tx_stops++;
1070         }
1071
1072         if (flush || __netif_subqueue_stopped(netdev, q_idx))
1073                 be_xmit_flush(adapter, txo);
1074
1075         return NETDEV_TX_OK;
1076 drop:
1077         tx_stats(txo)->tx_drv_drops++;
1078         /* Flush the already enqueued tx requests */
1079         if (flush && txo->pend_wrb_cnt)
1080                 be_xmit_flush(adapter, txo);
1081
1082         return NETDEV_TX_OK;
1083 }
1084
1085 static int be_change_mtu(struct net_device *netdev, int new_mtu)
1086 {
1087         struct be_adapter *adapter = netdev_priv(netdev);
1088         struct device *dev = &adapter->pdev->dev;
1089
1090         if (new_mtu < BE_MIN_MTU || new_mtu > BE_MAX_MTU) {
1091                 dev_info(dev, "MTU must be between %d and %d bytes\n",
1092                          BE_MIN_MTU, BE_MAX_MTU);
1093                 return -EINVAL;
1094         }
1095
1096         dev_info(dev, "MTU changed from %d to %d bytes\n",
1097                  netdev->mtu, new_mtu);
1098         netdev->mtu = new_mtu;
1099         return 0;
1100 }
1101
1102 /*
1103  * A max of 64 (BE_NUM_VLANS_SUPPORTED) vlans can be configured in BE.
1104  * If the user configures more, place BE in vlan promiscuous mode.
1105  */
1106 static int be_vid_config(struct be_adapter *adapter)
1107 {
1108         struct device *dev = &adapter->pdev->dev;
1109         u16 vids[BE_NUM_VLANS_SUPPORTED];
1110         u16 num = 0, i = 0;
1111         int status = 0;
1112
1113         /* No need to further configure vids if in promiscuous mode */
1114         if (adapter->promiscuous)
1115                 return 0;
1116
1117         if (adapter->vlans_added > be_max_vlans(adapter))
1118                 goto set_vlan_promisc;
1119
1120         /* Construct VLAN Table to give to HW */
1121         for_each_set_bit(i, adapter->vids, VLAN_N_VID)
1122                 vids[num++] = cpu_to_le16(i);
1123
1124         status = be_cmd_vlan_config(adapter, adapter->if_handle, vids, num);
1125         if (status) {
1126                 /* Set to VLAN promisc mode as setting VLAN filter failed */
1127                 if (addl_status(status) ==
1128                                 MCC_ADDL_STATUS_INSUFFICIENT_RESOURCES)
1129                         goto set_vlan_promisc;
1130                 dev_err(dev, "Setting HW VLAN filtering failed\n");
1131         } else {
1132                 if (adapter->flags & BE_FLAGS_VLAN_PROMISC) {
1133                         /* hw VLAN filtering re-enabled. */
1134                         status = be_cmd_rx_filter(adapter,
1135                                                   BE_FLAGS_VLAN_PROMISC, OFF);
1136                         if (!status) {
1137                                 dev_info(dev,
1138                                          "Disabling VLAN Promiscuous mode\n");
1139                                 adapter->flags &= ~BE_FLAGS_VLAN_PROMISC;
1140                         }
1141                 }
1142         }
1143
1144         return status;
1145
1146 set_vlan_promisc:
1147         if (adapter->flags & BE_FLAGS_VLAN_PROMISC)
1148                 return 0;
1149
1150         status = be_cmd_rx_filter(adapter, BE_FLAGS_VLAN_PROMISC, ON);
1151         if (!status) {
1152                 dev_info(dev, "Enable VLAN Promiscuous mode\n");
1153                 adapter->flags |= BE_FLAGS_VLAN_PROMISC;
1154         } else
1155                 dev_err(dev, "Failed to enable VLAN Promiscuous mode\n");
1156         return status;
1157 }
1158
1159 static int be_vlan_add_vid(struct net_device *netdev, __be16 proto, u16 vid)
1160 {
1161         struct be_adapter *adapter = netdev_priv(netdev);
1162         int status = 0;
1163
1164         /* Packets with VID 0 are always received by Lancer by default */
1165         if (lancer_chip(adapter) && vid == 0)
1166                 return status;
1167
1168         if (test_bit(vid, adapter->vids))
1169                 return status;
1170
1171         set_bit(vid, adapter->vids);
1172         adapter->vlans_added++;
1173
1174         status = be_vid_config(adapter);
1175         if (status) {
1176                 adapter->vlans_added--;
1177                 clear_bit(vid, adapter->vids);
1178         }
1179
1180         return status;
1181 }
1182
1183 static int be_vlan_rem_vid(struct net_device *netdev, __be16 proto, u16 vid)
1184 {
1185         struct be_adapter *adapter = netdev_priv(netdev);
1186
1187         /* Packets with VID 0 are always received by Lancer by default */
1188         if (lancer_chip(adapter) && vid == 0)
1189                 return 0;
1190
1191         clear_bit(vid, adapter->vids);
1192         adapter->vlans_added--;
1193
1194         return be_vid_config(adapter);
1195 }
1196
1197 static void be_clear_promisc(struct be_adapter *adapter)
1198 {
1199         adapter->promiscuous = false;
1200         adapter->flags &= ~(BE_FLAGS_VLAN_PROMISC | BE_FLAGS_MCAST_PROMISC);
1201
1202         be_cmd_rx_filter(adapter, IFF_PROMISC, OFF);
1203 }
1204
1205 static void be_set_rx_mode(struct net_device *netdev)
1206 {
1207         struct be_adapter *adapter = netdev_priv(netdev);
1208         int status;
1209
1210         if (netdev->flags & IFF_PROMISC) {
1211                 be_cmd_rx_filter(adapter, IFF_PROMISC, ON);
1212                 adapter->promiscuous = true;
1213                 goto done;
1214         }
1215
1216         /* BE was previously in promiscuous mode; disable it */
1217         if (adapter->promiscuous) {
1218                 be_clear_promisc(adapter);
1219                 if (adapter->vlans_added)
1220                         be_vid_config(adapter);
1221         }
1222
1223         /* Enable multicast promisc if num configured exceeds what we support */
1224         if (netdev->flags & IFF_ALLMULTI ||
1225             netdev_mc_count(netdev) > be_max_mc(adapter))
1226                 goto set_mcast_promisc;
1227
1228         if (netdev_uc_count(netdev) != adapter->uc_macs) {
1229                 struct netdev_hw_addr *ha;
1230                 int i = 1; /* First slot is claimed by the Primary MAC */
1231
1232                 for (; adapter->uc_macs > 0; adapter->uc_macs--, i++) {
1233                         be_cmd_pmac_del(adapter, adapter->if_handle,
1234                                         adapter->pmac_id[i], 0);
1235                 }
1236
1237                 if (netdev_uc_count(netdev) > be_max_uc(adapter)) {
1238                         be_cmd_rx_filter(adapter, IFF_PROMISC, ON);
1239                         adapter->promiscuous = true;
1240                         goto done;
1241                 }
1242
1243                 netdev_for_each_uc_addr(ha, adapter->netdev) {
1244                         adapter->uc_macs++; /* First slot is for Primary MAC */
1245                         be_cmd_pmac_add(adapter, (u8 *)ha->addr,
1246                                         adapter->if_handle,
1247                                         &adapter->pmac_id[adapter->uc_macs], 0);
1248                 }
1249         }
1250
1251         status = be_cmd_rx_filter(adapter, IFF_MULTICAST, ON);
1252         if (!status) {
1253                 if (adapter->flags & BE_FLAGS_MCAST_PROMISC)
1254                         adapter->flags &= ~BE_FLAGS_MCAST_PROMISC;
1255                 goto done;
1256         }
1257
1258 set_mcast_promisc:
1259         if (adapter->flags & BE_FLAGS_MCAST_PROMISC)
1260                 return;
1261
1262         /* Set to MCAST promisc mode if setting MULTICAST address fails
1263          * or if num configured exceeds what we support
1264          */
1265         status = be_cmd_rx_filter(adapter, IFF_ALLMULTI, ON);
1266         if (!status)
1267                 adapter->flags |= BE_FLAGS_MCAST_PROMISC;
1268 done:
1269         return;
1270 }
1271
1272 static int be_set_vf_mac(struct net_device *netdev, int vf, u8 *mac)
1273 {
1274         struct be_adapter *adapter = netdev_priv(netdev);
1275         struct be_vf_cfg *vf_cfg = &adapter->vf_cfg[vf];
1276         int status;
1277
1278         if (!sriov_enabled(adapter))
1279                 return -EPERM;
1280
1281         if (!is_valid_ether_addr(mac) || vf >= adapter->num_vfs)
1282                 return -EINVAL;
1283
1284         /* Proceed further only if user provided MAC is different
1285          * from active MAC
1286          */
1287         if (ether_addr_equal(mac, vf_cfg->mac_addr))
1288                 return 0;
1289
1290         if (BEx_chip(adapter)) {
1291                 be_cmd_pmac_del(adapter, vf_cfg->if_handle, vf_cfg->pmac_id,
1292                                 vf + 1);
1293
1294                 status = be_cmd_pmac_add(adapter, mac, vf_cfg->if_handle,
1295                                          &vf_cfg->pmac_id, vf + 1);
1296         } else {
1297                 status = be_cmd_set_mac(adapter, mac, vf_cfg->if_handle,
1298                                         vf + 1);
1299         }
1300
1301         if (status) {
1302                 dev_err(&adapter->pdev->dev, "MAC %pM set on VF %d Failed: %#x",
1303                         mac, vf, status);
1304                 return be_cmd_status(status);
1305         }
1306
1307         ether_addr_copy(vf_cfg->mac_addr, mac);
1308
1309         return 0;
1310 }
1311
1312 static int be_get_vf_config(struct net_device *netdev, int vf,
1313                             struct ifla_vf_info *vi)
1314 {
1315         struct be_adapter *adapter = netdev_priv(netdev);
1316         struct be_vf_cfg *vf_cfg = &adapter->vf_cfg[vf];
1317
1318         if (!sriov_enabled(adapter))
1319                 return -EPERM;
1320
1321         if (vf >= adapter->num_vfs)
1322                 return -EINVAL;
1323
1324         vi->vf = vf;
1325         vi->max_tx_rate = vf_cfg->tx_rate;
1326         vi->min_tx_rate = 0;
1327         vi->vlan = vf_cfg->vlan_tag & VLAN_VID_MASK;
1328         vi->qos = vf_cfg->vlan_tag >> VLAN_PRIO_SHIFT;
1329         memcpy(&vi->mac, vf_cfg->mac_addr, ETH_ALEN);
1330         vi->linkstate = adapter->vf_cfg[vf].plink_tracking;
1331
1332         return 0;
1333 }
1334
1335 static int be_set_vf_vlan(struct net_device *netdev, int vf, u16 vlan, u8 qos)
1336 {
1337         struct be_adapter *adapter = netdev_priv(netdev);
1338         struct be_vf_cfg *vf_cfg = &adapter->vf_cfg[vf];
1339         int status = 0;
1340
1341         if (!sriov_enabled(adapter))
1342                 return -EPERM;
1343
1344         if (vf >= adapter->num_vfs || vlan > 4095 || qos > 7)
1345                 return -EINVAL;
1346
1347         if (vlan || qos) {
1348                 vlan |= qos << VLAN_PRIO_SHIFT;
1349                 if (vf_cfg->vlan_tag != vlan)
1350                         status = be_cmd_set_hsw_config(adapter, vlan, vf + 1,
1351                                                        vf_cfg->if_handle, 0);
1352         } else {
1353                 /* Reset Transparent Vlan Tagging. */
1354                 status = be_cmd_set_hsw_config(adapter, BE_RESET_VLAN_TAG_ID,
1355                                                vf + 1, vf_cfg->if_handle, 0);
1356         }
1357
1358         if (status) {
1359                 dev_err(&adapter->pdev->dev,
1360                         "VLAN %d config on VF %d failed : %#x\n", vlan,
1361                         vf, status);
1362                 return be_cmd_status(status);
1363         }
1364
1365         vf_cfg->vlan_tag = vlan;
1366
1367         return 0;
1368 }
1369
1370 static int be_set_vf_tx_rate(struct net_device *netdev, int vf,
1371                              int min_tx_rate, int max_tx_rate)
1372 {
1373         struct be_adapter *adapter = netdev_priv(netdev);
1374         struct device *dev = &adapter->pdev->dev;
1375         int percent_rate, status = 0;
1376         u16 link_speed = 0;
1377         u8 link_status;
1378
1379         if (!sriov_enabled(adapter))
1380                 return -EPERM;
1381
1382         if (vf >= adapter->num_vfs)
1383                 return -EINVAL;
1384
1385         if (min_tx_rate)
1386                 return -EINVAL;
1387
1388         if (!max_tx_rate)
1389                 goto config_qos;
1390
1391         status = be_cmd_link_status_query(adapter, &link_speed,
1392                                           &link_status, 0);
1393         if (status)
1394                 goto err;
1395
1396         if (!link_status) {
1397                 dev_err(dev, "TX-rate setting not allowed when link is down\n");
1398                 status = -ENETDOWN;
1399                 goto err;
1400         }
1401
1402         if (max_tx_rate < 100 || max_tx_rate > link_speed) {
1403                 dev_err(dev, "TX-rate must be between 100 and %d Mbps\n",
1404                         link_speed);
1405                 status = -EINVAL;
1406                 goto err;
1407         }
1408
1409         /* On Skyhawk the QOS setting must be done only as a % value */
1410         percent_rate = link_speed / 100;
1411         if (skyhawk_chip(adapter) && (max_tx_rate % percent_rate)) {
1412                 dev_err(dev, "TX-rate must be a multiple of %d Mbps\n",
1413                         percent_rate);
1414                 status = -EINVAL;
1415                 goto err;
1416         }
1417
1418 config_qos:
1419         status = be_cmd_config_qos(adapter, max_tx_rate, link_speed, vf + 1);
1420         if (status)
1421                 goto err;
1422
1423         adapter->vf_cfg[vf].tx_rate = max_tx_rate;
1424         return 0;
1425
1426 err:
1427         dev_err(dev, "TX-rate setting of %dMbps on VF%d failed\n",
1428                 max_tx_rate, vf);
1429         return be_cmd_status(status);
1430 }
1431
1432 static int be_set_vf_link_state(struct net_device *netdev, int vf,
1433                                 int link_state)
1434 {
1435         struct be_adapter *adapter = netdev_priv(netdev);
1436         int status;
1437
1438         if (!sriov_enabled(adapter))
1439                 return -EPERM;
1440
1441         if (vf >= adapter->num_vfs)
1442                 return -EINVAL;
1443
1444         status = be_cmd_set_logical_link_config(adapter, link_state, vf+1);
1445         if (status) {
1446                 dev_err(&adapter->pdev->dev,
1447                         "Link state change on VF %d failed: %#x\n", vf, status);
1448                 return be_cmd_status(status);
1449         }
1450
1451         adapter->vf_cfg[vf].plink_tracking = link_state;
1452
1453         return 0;
1454 }
1455
1456 static void be_aic_update(struct be_aic_obj *aic, u64 rx_pkts, u64 tx_pkts,
1457                           ulong now)
1458 {
1459         aic->rx_pkts_prev = rx_pkts;
1460         aic->tx_reqs_prev = tx_pkts;
1461         aic->jiffies = now;
1462 }
1463
1464 static void be_eqd_update(struct be_adapter *adapter)
1465 {
1466         struct be_set_eqd set_eqd[MAX_EVT_QS];
1467         int eqd, i, num = 0, start;
1468         struct be_aic_obj *aic;
1469         struct be_eq_obj *eqo;
1470         struct be_rx_obj *rxo;
1471         struct be_tx_obj *txo;
1472         u64 rx_pkts, tx_pkts;
1473         ulong now;
1474         u32 pps, delta;
1475
1476         for_all_evt_queues(adapter, eqo, i) {
1477                 aic = &adapter->aic_obj[eqo->idx];
1478                 if (!aic->enable) {
1479                         if (aic->jiffies)
1480                                 aic->jiffies = 0;
1481                         eqd = aic->et_eqd;
1482                         goto modify_eqd;
1483                 }
1484
1485                 rxo = &adapter->rx_obj[eqo->idx];
1486                 do {
1487                         start = u64_stats_fetch_begin_irq(&rxo->stats.sync);
1488                         rx_pkts = rxo->stats.rx_pkts;
1489                 } while (u64_stats_fetch_retry_irq(&rxo->stats.sync, start));
1490
1491                 txo = &adapter->tx_obj[eqo->idx];
1492                 do {
1493                         start = u64_stats_fetch_begin_irq(&txo->stats.sync);
1494                         tx_pkts = txo->stats.tx_reqs;
1495                 } while (u64_stats_fetch_retry_irq(&txo->stats.sync, start));
1496
1497                 /* Skip, if wrapped around or first calculation */
1498                 now = jiffies;
1499                 if (!aic->jiffies || time_before(now, aic->jiffies) ||
1500                     rx_pkts < aic->rx_pkts_prev ||
1501                     tx_pkts < aic->tx_reqs_prev) {
1502                         be_aic_update(aic, rx_pkts, tx_pkts, now);
1503                         continue;
1504                 }
1505
1506                 delta = jiffies_to_msecs(now - aic->jiffies);
1507                 pps = (((u32)(rx_pkts - aic->rx_pkts_prev) * 1000) / delta) +
1508                         (((u32)(tx_pkts - aic->tx_reqs_prev) * 1000) / delta);
1509                 eqd = (pps / 15000) << 2;
1510
1511                 if (eqd < 8)
1512                         eqd = 0;
1513                 eqd = min_t(u32, eqd, aic->max_eqd);
1514                 eqd = max_t(u32, eqd, aic->min_eqd);
1515
1516                 be_aic_update(aic, rx_pkts, tx_pkts, now);
1517 modify_eqd:
1518                 if (eqd != aic->prev_eqd) {
1519                         set_eqd[num].delay_multiplier = (eqd * 65)/100;
1520                         set_eqd[num].eq_id = eqo->q.id;
1521                         aic->prev_eqd = eqd;
1522                         num++;
1523                 }
1524         }
1525
1526         if (num)
1527                 be_cmd_modify_eqd(adapter, set_eqd, num);
1528 }
1529
1530 static void be_rx_stats_update(struct be_rx_obj *rxo,
1531                                struct be_rx_compl_info *rxcp)
1532 {
1533         struct be_rx_stats *stats = rx_stats(rxo);
1534
1535         u64_stats_update_begin(&stats->sync);
1536         stats->rx_compl++;
1537         stats->rx_bytes += rxcp->pkt_size;
1538         stats->rx_pkts++;
1539         if (rxcp->pkt_type == BE_MULTICAST_PACKET)
1540                 stats->rx_mcast_pkts++;
1541         if (rxcp->err)
1542                 stats->rx_compl_err++;
1543         u64_stats_update_end(&stats->sync);
1544 }
1545
1546 static inline bool csum_passed(struct be_rx_compl_info *rxcp)
1547 {
1548         /* L4 checksum is not reliable for non TCP/UDP packets.
1549          * Also ignore ipcksm for ipv6 pkts
1550          */
1551         return (rxcp->tcpf || rxcp->udpf) && rxcp->l4_csum &&
1552                 (rxcp->ip_csum || rxcp->ipv6) && !rxcp->err;
1553 }
1554
1555 static struct be_rx_page_info *get_rx_page_info(struct be_rx_obj *rxo)
1556 {
1557         struct be_adapter *adapter = rxo->adapter;
1558         struct be_rx_page_info *rx_page_info;
1559         struct be_queue_info *rxq = &rxo->q;
1560         u16 frag_idx = rxq->tail;
1561
1562         rx_page_info = &rxo->page_info_tbl[frag_idx];
1563         BUG_ON(!rx_page_info->page);
1564
1565         if (rx_page_info->last_frag) {
1566                 dma_unmap_page(&adapter->pdev->dev,
1567                                dma_unmap_addr(rx_page_info, bus),
1568                                adapter->big_page_size, DMA_FROM_DEVICE);
1569                 rx_page_info->last_frag = false;
1570         } else {
1571                 dma_sync_single_for_cpu(&adapter->pdev->dev,
1572                                         dma_unmap_addr(rx_page_info, bus),
1573                                         rx_frag_size, DMA_FROM_DEVICE);
1574         }
1575
1576         queue_tail_inc(rxq);
1577         atomic_dec(&rxq->used);
1578         return rx_page_info;
1579 }
1580
1581 /* Throwaway the data in the Rx completion */
1582 static void be_rx_compl_discard(struct be_rx_obj *rxo,
1583                                 struct be_rx_compl_info *rxcp)
1584 {
1585         struct be_rx_page_info *page_info;
1586         u16 i, num_rcvd = rxcp->num_rcvd;
1587
1588         for (i = 0; i < num_rcvd; i++) {
1589                 page_info = get_rx_page_info(rxo);
1590                 put_page(page_info->page);
1591                 memset(page_info, 0, sizeof(*page_info));
1592         }
1593 }
1594
1595 /*
1596  * skb_fill_rx_data forms a complete skb for an ether frame
1597  * indicated by rxcp.
1598  */
1599 static void skb_fill_rx_data(struct be_rx_obj *rxo, struct sk_buff *skb,
1600                              struct be_rx_compl_info *rxcp)
1601 {
1602         struct be_rx_page_info *page_info;
1603         u16 i, j;
1604         u16 hdr_len, curr_frag_len, remaining;
1605         u8 *start;
1606
1607         page_info = get_rx_page_info(rxo);
1608         start = page_address(page_info->page) + page_info->page_offset;
1609         prefetch(start);
1610
1611         /* Copy data in the first descriptor of this completion */
1612         curr_frag_len = min(rxcp->pkt_size, rx_frag_size);
1613
1614         skb->len = curr_frag_len;
1615         if (curr_frag_len <= BE_HDR_LEN) { /* tiny packet */
1616                 memcpy(skb->data, start, curr_frag_len);
1617                 /* Complete packet has now been moved to data */
1618                 put_page(page_info->page);
1619                 skb->data_len = 0;
1620                 skb->tail += curr_frag_len;
1621         } else {
1622                 hdr_len = ETH_HLEN;
1623                 memcpy(skb->data, start, hdr_len);
1624                 skb_shinfo(skb)->nr_frags = 1;
1625                 skb_frag_set_page(skb, 0, page_info->page);
1626                 skb_shinfo(skb)->frags[0].page_offset =
1627                                         page_info->page_offset + hdr_len;
1628                 skb_frag_size_set(&skb_shinfo(skb)->frags[0],
1629                                   curr_frag_len - hdr_len);
1630                 skb->data_len = curr_frag_len - hdr_len;
1631                 skb->truesize += rx_frag_size;
1632                 skb->tail += hdr_len;
1633         }
1634         page_info->page = NULL;
1635
1636         if (rxcp->pkt_size <= rx_frag_size) {
1637                 BUG_ON(rxcp->num_rcvd != 1);
1638                 return;
1639         }
1640
1641         /* More frags present for this completion */
1642         remaining = rxcp->pkt_size - curr_frag_len;
1643         for (i = 1, j = 0; i < rxcp->num_rcvd; i++) {
1644                 page_info = get_rx_page_info(rxo);
1645                 curr_frag_len = min(remaining, rx_frag_size);
1646
1647                 /* Coalesce all frags from the same physical page in one slot */
1648                 if (page_info->page_offset == 0) {
1649                         /* Fresh page */
1650                         j++;
1651                         skb_frag_set_page(skb, j, page_info->page);
1652                         skb_shinfo(skb)->frags[j].page_offset =
1653                                                         page_info->page_offset;
1654                         skb_frag_size_set(&skb_shinfo(skb)->frags[j], 0);
1655                         skb_shinfo(skb)->nr_frags++;
1656                 } else {
1657                         put_page(page_info->page);
1658                 }
1659
1660                 skb_frag_size_add(&skb_shinfo(skb)->frags[j], curr_frag_len);
1661                 skb->len += curr_frag_len;
1662                 skb->data_len += curr_frag_len;
1663                 skb->truesize += rx_frag_size;
1664                 remaining -= curr_frag_len;
1665                 page_info->page = NULL;
1666         }
1667         BUG_ON(j > MAX_SKB_FRAGS);
1668 }
1669
1670 /* Process the RX completion indicated by rxcp when GRO is disabled */
1671 static void be_rx_compl_process(struct be_rx_obj *rxo, struct napi_struct *napi,
1672                                 struct be_rx_compl_info *rxcp)
1673 {
1674         struct be_adapter *adapter = rxo->adapter;
1675         struct net_device *netdev = adapter->netdev;
1676         struct sk_buff *skb;
1677
1678         skb = netdev_alloc_skb_ip_align(netdev, BE_RX_SKB_ALLOC_SIZE);
1679         if (unlikely(!skb)) {
1680                 rx_stats(rxo)->rx_drops_no_skbs++;
1681                 be_rx_compl_discard(rxo, rxcp);
1682                 return;
1683         }
1684
1685         skb_fill_rx_data(rxo, skb, rxcp);
1686
1687         if (likely((netdev->features & NETIF_F_RXCSUM) && csum_passed(rxcp)))
1688                 skb->ip_summed = CHECKSUM_UNNECESSARY;
1689         else
1690                 skb_checksum_none_assert(skb);
1691
1692         skb->protocol = eth_type_trans(skb, netdev);
1693         skb_record_rx_queue(skb, rxo - &adapter->rx_obj[0]);
1694         if (netdev->features & NETIF_F_RXHASH)
1695                 skb_set_hash(skb, rxcp->rss_hash, PKT_HASH_TYPE_L3);
1696
1697         skb->csum_level = rxcp->tunneled;
1698         skb_mark_napi_id(skb, napi);
1699
1700         if (rxcp->vlanf)
1701                 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), rxcp->vlan_tag);
1702
1703         netif_receive_skb(skb);
1704 }
1705
1706 /* Process the RX completion indicated by rxcp when GRO is enabled */
1707 static void be_rx_compl_process_gro(struct be_rx_obj *rxo,
1708                                     struct napi_struct *napi,
1709                                     struct be_rx_compl_info *rxcp)
1710 {
1711         struct be_adapter *adapter = rxo->adapter;
1712         struct be_rx_page_info *page_info;
1713         struct sk_buff *skb = NULL;
1714         u16 remaining, curr_frag_len;
1715         u16 i, j;
1716
1717         skb = napi_get_frags(napi);
1718         if (!skb) {
1719                 be_rx_compl_discard(rxo, rxcp);
1720                 return;
1721         }
1722
1723         remaining = rxcp->pkt_size;
1724         for (i = 0, j = -1; i < rxcp->num_rcvd; i++) {
1725                 page_info = get_rx_page_info(rxo);
1726
1727                 curr_frag_len = min(remaining, rx_frag_size);
1728
1729                 /* Coalesce all frags from the same physical page in one slot */
1730                 if (i == 0 || page_info->page_offset == 0) {
1731                         /* First frag or Fresh page */
1732                         j++;
1733                         skb_frag_set_page(skb, j, page_info->page);
1734                         skb_shinfo(skb)->frags[j].page_offset =
1735                                                         page_info->page_offset;
1736                         skb_frag_size_set(&skb_shinfo(skb)->frags[j], 0);
1737                 } else {
1738                         put_page(page_info->page);
1739                 }
1740                 skb_frag_size_add(&skb_shinfo(skb)->frags[j], curr_frag_len);
1741                 skb->truesize += rx_frag_size;
1742                 remaining -= curr_frag_len;
1743                 memset(page_info, 0, sizeof(*page_info));
1744         }
1745         BUG_ON(j > MAX_SKB_FRAGS);
1746
1747         skb_shinfo(skb)->nr_frags = j + 1;
1748         skb->len = rxcp->pkt_size;
1749         skb->data_len = rxcp->pkt_size;
1750         skb->ip_summed = CHECKSUM_UNNECESSARY;
1751         skb_record_rx_queue(skb, rxo - &adapter->rx_obj[0]);
1752         if (adapter->netdev->features & NETIF_F_RXHASH)
1753                 skb_set_hash(skb, rxcp->rss_hash, PKT_HASH_TYPE_L3);
1754
1755         skb->csum_level = rxcp->tunneled;
1756         skb_mark_napi_id(skb, napi);
1757
1758         if (rxcp->vlanf)
1759                 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), rxcp->vlan_tag);
1760
1761         napi_gro_frags(napi);
1762 }
1763
1764 static void be_parse_rx_compl_v1(struct be_eth_rx_compl *compl,
1765                                  struct be_rx_compl_info *rxcp)
1766 {
1767         rxcp->pkt_size = GET_RX_COMPL_V1_BITS(pktsize, compl);
1768         rxcp->vlanf = GET_RX_COMPL_V1_BITS(vtp, compl);
1769         rxcp->err = GET_RX_COMPL_V1_BITS(err, compl);
1770         rxcp->tcpf = GET_RX_COMPL_V1_BITS(tcpf, compl);
1771         rxcp->udpf = GET_RX_COMPL_V1_BITS(udpf, compl);
1772         rxcp->ip_csum = GET_RX_COMPL_V1_BITS(ipcksm, compl);
1773         rxcp->l4_csum = GET_RX_COMPL_V1_BITS(l4_cksm, compl);
1774         rxcp->ipv6 = GET_RX_COMPL_V1_BITS(ip_version, compl);
1775         rxcp->num_rcvd = GET_RX_COMPL_V1_BITS(numfrags, compl);
1776         rxcp->pkt_type = GET_RX_COMPL_V1_BITS(cast_enc, compl);
1777         rxcp->rss_hash = GET_RX_COMPL_V1_BITS(rsshash, compl);
1778         if (rxcp->vlanf) {
1779                 rxcp->qnq = GET_RX_COMPL_V1_BITS(qnq, compl);
1780                 rxcp->vlan_tag = GET_RX_COMPL_V1_BITS(vlan_tag, compl);
1781         }
1782         rxcp->port = GET_RX_COMPL_V1_BITS(port, compl);
1783         rxcp->tunneled =
1784                 GET_RX_COMPL_V1_BITS(tunneled, compl);
1785 }
1786
1787 static void be_parse_rx_compl_v0(struct be_eth_rx_compl *compl,
1788                                  struct be_rx_compl_info *rxcp)
1789 {
1790         rxcp->pkt_size = GET_RX_COMPL_V0_BITS(pktsize, compl);
1791         rxcp->vlanf = GET_RX_COMPL_V0_BITS(vtp, compl);
1792         rxcp->err = GET_RX_COMPL_V0_BITS(err, compl);
1793         rxcp->tcpf = GET_RX_COMPL_V0_BITS(tcpf, compl);
1794         rxcp->udpf = GET_RX_COMPL_V0_BITS(udpf, compl);
1795         rxcp->ip_csum = GET_RX_COMPL_V0_BITS(ipcksm, compl);
1796         rxcp->l4_csum = GET_RX_COMPL_V0_BITS(l4_cksm, compl);
1797         rxcp->ipv6 = GET_RX_COMPL_V0_BITS(ip_version, compl);
1798         rxcp->num_rcvd = GET_RX_COMPL_V0_BITS(numfrags, compl);
1799         rxcp->pkt_type = GET_RX_COMPL_V0_BITS(cast_enc, compl);
1800         rxcp->rss_hash = GET_RX_COMPL_V0_BITS(rsshash, compl);
1801         if (rxcp->vlanf) {
1802                 rxcp->qnq = GET_RX_COMPL_V0_BITS(qnq, compl);
1803                 rxcp->vlan_tag = GET_RX_COMPL_V0_BITS(vlan_tag, compl);
1804         }
1805         rxcp->port = GET_RX_COMPL_V0_BITS(port, compl);
1806         rxcp->ip_frag = GET_RX_COMPL_V0_BITS(ip_frag, compl);
1807 }
1808
1809 static struct be_rx_compl_info *be_rx_compl_get(struct be_rx_obj *rxo)
1810 {
1811         struct be_eth_rx_compl *compl = queue_tail_node(&rxo->cq);
1812         struct be_rx_compl_info *rxcp = &rxo->rxcp;
1813         struct be_adapter *adapter = rxo->adapter;
1814
1815         /* For checking the valid bit it is Ok to use either definition as the
1816          * valid bit is at the same position in both v0 and v1 Rx compl */
1817         if (compl->dw[offsetof(struct amap_eth_rx_compl_v1, valid) / 32] == 0)
1818                 return NULL;
1819
1820         rmb();
1821         be_dws_le_to_cpu(compl, sizeof(*compl));
1822
1823         if (adapter->be3_native)
1824                 be_parse_rx_compl_v1(compl, rxcp);
1825         else
1826                 be_parse_rx_compl_v0(compl, rxcp);
1827
1828         if (rxcp->ip_frag)
1829                 rxcp->l4_csum = 0;
1830
1831         if (rxcp->vlanf) {
1832                 /* In QNQ modes, if qnq bit is not set, then the packet was
1833                  * tagged only with the transparent outer vlan-tag and must
1834                  * not be treated as a vlan packet by host
1835                  */
1836                 if (be_is_qnq_mode(adapter) && !rxcp->qnq)
1837                         rxcp->vlanf = 0;
1838
1839                 if (!lancer_chip(adapter))
1840                         rxcp->vlan_tag = swab16(rxcp->vlan_tag);
1841
1842                 if (adapter->pvid == (rxcp->vlan_tag & VLAN_VID_MASK) &&
1843                     !test_bit(rxcp->vlan_tag, adapter->vids))
1844                         rxcp->vlanf = 0;
1845         }
1846
1847         /* As the compl has been parsed, reset it; we wont touch it again */
1848         compl->dw[offsetof(struct amap_eth_rx_compl_v1, valid) / 32] = 0;
1849
1850         queue_tail_inc(&rxo->cq);
1851         return rxcp;
1852 }
1853
1854 static inline struct page *be_alloc_pages(u32 size, gfp_t gfp)
1855 {
1856         u32 order = get_order(size);
1857
1858         if (order > 0)
1859                 gfp |= __GFP_COMP;
1860         return  alloc_pages(gfp, order);
1861 }
1862
1863 /*
1864  * Allocate a page, split it to fragments of size rx_frag_size and post as
1865  * receive buffers to BE
1866  */
1867 static void be_post_rx_frags(struct be_rx_obj *rxo, gfp_t gfp, u32 frags_needed)
1868 {
1869         struct be_adapter *adapter = rxo->adapter;
1870         struct be_rx_page_info *page_info = NULL, *prev_page_info = NULL;
1871         struct be_queue_info *rxq = &rxo->q;
1872         struct page *pagep = NULL;
1873         struct device *dev = &adapter->pdev->dev;
1874         struct be_eth_rx_d *rxd;
1875         u64 page_dmaaddr = 0, frag_dmaaddr;
1876         u32 posted, page_offset = 0, notify = 0;
1877
1878         page_info = &rxo->page_info_tbl[rxq->head];
1879         for (posted = 0; posted < frags_needed && !page_info->page; posted++) {
1880                 if (!pagep) {
1881                         pagep = be_alloc_pages(adapter->big_page_size, gfp);
1882                         if (unlikely(!pagep)) {
1883                                 rx_stats(rxo)->rx_post_fail++;
1884                                 break;
1885                         }
1886                         page_dmaaddr = dma_map_page(dev, pagep, 0,
1887                                                     adapter->big_page_size,
1888                                                     DMA_FROM_DEVICE);
1889                         if (dma_mapping_error(dev, page_dmaaddr)) {
1890                                 put_page(pagep);
1891                                 pagep = NULL;
1892                                 adapter->drv_stats.dma_map_errors++;
1893                                 break;
1894                         }
1895                         page_offset = 0;
1896                 } else {
1897                         get_page(pagep);
1898                         page_offset += rx_frag_size;
1899                 }
1900                 page_info->page_offset = page_offset;
1901                 page_info->page = pagep;
1902
1903                 rxd = queue_head_node(rxq);
1904                 frag_dmaaddr = page_dmaaddr + page_info->page_offset;
1905                 rxd->fragpa_lo = cpu_to_le32(frag_dmaaddr & 0xFFFFFFFF);
1906                 rxd->fragpa_hi = cpu_to_le32(upper_32_bits(frag_dmaaddr));
1907
1908                 /* Any space left in the current big page for another frag? */
1909                 if ((page_offset + rx_frag_size + rx_frag_size) >
1910                                         adapter->big_page_size) {
1911                         pagep = NULL;
1912                         page_info->last_frag = true;
1913                         dma_unmap_addr_set(page_info, bus, page_dmaaddr);
1914                 } else {
1915                         dma_unmap_addr_set(page_info, bus, frag_dmaaddr);
1916                 }
1917
1918                 prev_page_info = page_info;
1919                 queue_head_inc(rxq);
1920                 page_info = &rxo->page_info_tbl[rxq->head];
1921         }
1922
1923         /* Mark the last frag of a page when we break out of the above loop
1924          * with no more slots available in the RXQ
1925          */
1926         if (pagep) {
1927                 prev_page_info->last_frag = true;
1928                 dma_unmap_addr_set(prev_page_info, bus, page_dmaaddr);
1929         }
1930
1931         if (posted) {
1932                 atomic_add(posted, &rxq->used);
1933                 if (rxo->rx_post_starved)
1934                         rxo->rx_post_starved = false;
1935                 do {
1936                         notify = min(256u, posted);
1937                         be_rxq_notify(adapter, rxq->id, notify);
1938                         posted -= notify;
1939                 } while (posted);
1940         } else if (atomic_read(&rxq->used) == 0) {
1941                 /* Let be_worker replenish when memory is available */
1942                 rxo->rx_post_starved = true;
1943         }
1944 }
1945
1946 static struct be_eth_tx_compl *be_tx_compl_get(struct be_queue_info *tx_cq)
1947 {
1948         struct be_eth_tx_compl *txcp = queue_tail_node(tx_cq);
1949
1950         if (txcp->dw[offsetof(struct amap_eth_tx_compl, valid) / 32] == 0)
1951                 return NULL;
1952
1953         rmb();
1954         be_dws_le_to_cpu(txcp, sizeof(*txcp));
1955
1956         txcp->dw[offsetof(struct amap_eth_tx_compl, valid) / 32] = 0;
1957
1958         queue_tail_inc(tx_cq);
1959         return txcp;
1960 }
1961
1962 static u16 be_tx_compl_process(struct be_adapter *adapter,
1963                                struct be_tx_obj *txo, u16 last_index)
1964 {
1965         struct sk_buff **sent_skbs = txo->sent_skb_list;
1966         struct be_queue_info *txq = &txo->q;
1967         u16 frag_index, num_wrbs = 0;
1968         struct sk_buff *skb = NULL;
1969         bool unmap_skb_hdr = false;
1970         struct be_eth_wrb *wrb;
1971
1972         do {
1973                 if (sent_skbs[txq->tail]) {
1974                         /* Free skb from prev req */
1975                         if (skb)
1976                                 dev_consume_skb_any(skb);
1977                         skb = sent_skbs[txq->tail];
1978                         sent_skbs[txq->tail] = NULL;
1979                         queue_tail_inc(txq);  /* skip hdr wrb */
1980                         num_wrbs++;
1981                         unmap_skb_hdr = true;
1982                 }
1983                 wrb = queue_tail_node(txq);
1984                 frag_index = txq->tail;
1985                 unmap_tx_frag(&adapter->pdev->dev, wrb,
1986                               (unmap_skb_hdr && skb_headlen(skb)));
1987                 unmap_skb_hdr = false;
1988                 queue_tail_inc(txq);
1989                 num_wrbs++;
1990         } while (frag_index != last_index);
1991         dev_consume_skb_any(skb);
1992
1993         return num_wrbs;
1994 }
1995
1996 /* Return the number of events in the event queue */
1997 static inline int events_get(struct be_eq_obj *eqo)
1998 {
1999         struct be_eq_entry *eqe;
2000         int num = 0;
2001
2002         do {
2003                 eqe = queue_tail_node(&eqo->q);
2004                 if (eqe->evt == 0)
2005                         break;
2006
2007                 rmb();
2008                 eqe->evt = 0;
2009                 num++;
2010                 queue_tail_inc(&eqo->q);
2011         } while (true);
2012
2013         return num;
2014 }
2015
2016 /* Leaves the EQ is disarmed state */
2017 static void be_eq_clean(struct be_eq_obj *eqo)
2018 {
2019         int num = events_get(eqo);
2020
2021         be_eq_notify(eqo->adapter, eqo->q.id, false, true, num);
2022 }
2023
2024 static void be_rx_cq_clean(struct be_rx_obj *rxo)
2025 {
2026         struct be_rx_page_info *page_info;
2027         struct be_queue_info *rxq = &rxo->q;
2028         struct be_queue_info *rx_cq = &rxo->cq;
2029         struct be_rx_compl_info *rxcp;
2030         struct be_adapter *adapter = rxo->adapter;
2031         int flush_wait = 0;
2032
2033         /* Consume pending rx completions.
2034          * Wait for the flush completion (identified by zero num_rcvd)
2035          * to arrive. Notify CQ even when there are no more CQ entries
2036          * for HW to flush partially coalesced CQ entries.
2037          * In Lancer, there is no need to wait for flush compl.
2038          */
2039         for (;;) {
2040                 rxcp = be_rx_compl_get(rxo);
2041                 if (!rxcp) {
2042                         if (lancer_chip(adapter))
2043                                 break;
2044
2045                         if (flush_wait++ > 10 || be_hw_error(adapter)) {
2046                                 dev_warn(&adapter->pdev->dev,
2047                                          "did not receive flush compl\n");
2048                                 break;
2049                         }
2050                         be_cq_notify(adapter, rx_cq->id, true, 0);
2051                         mdelay(1);
2052                 } else {
2053                         be_rx_compl_discard(rxo, rxcp);
2054                         be_cq_notify(adapter, rx_cq->id, false, 1);
2055                         if (rxcp->num_rcvd == 0)
2056                                 break;
2057                 }
2058         }
2059
2060         /* After cleanup, leave the CQ in unarmed state */
2061         be_cq_notify(adapter, rx_cq->id, false, 0);
2062
2063         /* Then free posted rx buffers that were not used */
2064         while (atomic_read(&rxq->used) > 0) {
2065                 page_info = get_rx_page_info(rxo);
2066                 put_page(page_info->page);
2067                 memset(page_info, 0, sizeof(*page_info));
2068         }
2069         BUG_ON(atomic_read(&rxq->used));
2070         rxq->tail = 0;
2071         rxq->head = 0;
2072 }
2073
2074 static void be_tx_compl_clean(struct be_adapter *adapter)
2075 {
2076         u16 end_idx, notified_idx, cmpl = 0, timeo = 0, num_wrbs = 0;
2077         struct device *dev = &adapter->pdev->dev;
2078         struct be_tx_obj *txo;
2079         struct be_queue_info *txq;
2080         struct be_eth_tx_compl *txcp;
2081         int i, pending_txqs;
2082
2083         /* Stop polling for compls when HW has been silent for 10ms */
2084         do {
2085                 pending_txqs = adapter->num_tx_qs;
2086
2087                 for_all_tx_queues(adapter, txo, i) {
2088                         cmpl = 0;
2089                         num_wrbs = 0;
2090                         txq = &txo->q;
2091                         while ((txcp = be_tx_compl_get(&txo->cq))) {
2092                                 end_idx = GET_TX_COMPL_BITS(wrb_index, txcp);
2093                                 num_wrbs += be_tx_compl_process(adapter, txo,
2094                                                                 end_idx);
2095                                 cmpl++;
2096                         }
2097                         if (cmpl) {
2098                                 be_cq_notify(adapter, txo->cq.id, false, cmpl);
2099                                 atomic_sub(num_wrbs, &txq->used);
2100                                 timeo = 0;
2101                         }
2102                         if (atomic_read(&txq->used) == txo->pend_wrb_cnt)
2103                                 pending_txqs--;
2104                 }
2105
2106                 if (pending_txqs == 0 || ++timeo > 10 || be_hw_error(adapter))
2107                         break;
2108
2109                 mdelay(1);
2110         } while (true);
2111
2112         /* Free enqueued TX that was never notified to HW */
2113         for_all_tx_queues(adapter, txo, i) {
2114                 txq = &txo->q;
2115
2116                 if (atomic_read(&txq->used)) {
2117                         dev_info(dev, "txq%d: cleaning %d pending tx-wrbs\n",
2118                                  i, atomic_read(&txq->used));
2119                         notified_idx = txq->tail;
2120                         end_idx = txq->tail;
2121                         index_adv(&end_idx, atomic_read(&txq->used) - 1,
2122                                   txq->len);
2123                         /* Use the tx-compl process logic to handle requests
2124                          * that were not sent to the HW.
2125                          */
2126                         num_wrbs = be_tx_compl_process(adapter, txo, end_idx);
2127                         atomic_sub(num_wrbs, &txq->used);
2128                         BUG_ON(atomic_read(&txq->used));
2129                         txo->pend_wrb_cnt = 0;
2130                         /* Since hw was never notified of these requests,
2131                          * reset TXQ indices
2132                          */
2133                         txq->head = notified_idx;
2134                         txq->tail = notified_idx;
2135                 }
2136         }
2137 }
2138
2139 static void be_evt_queues_destroy(struct be_adapter *adapter)
2140 {
2141         struct be_eq_obj *eqo;
2142         int i;
2143
2144         for_all_evt_queues(adapter, eqo, i) {
2145                 if (eqo->q.created) {
2146                         be_eq_clean(eqo);
2147                         be_cmd_q_destroy(adapter, &eqo->q, QTYPE_EQ);
2148                         napi_hash_del(&eqo->napi);
2149                         netif_napi_del(&eqo->napi);
2150                 }
2151                 be_queue_free(adapter, &eqo->q);
2152         }
2153 }
2154
2155 static int be_evt_queues_create(struct be_adapter *adapter)
2156 {
2157         struct be_queue_info *eq;
2158         struct be_eq_obj *eqo;
2159         struct be_aic_obj *aic;
2160         int i, rc;
2161
2162         adapter->num_evt_qs = min_t(u16, num_irqs(adapter),
2163                                     adapter->cfg_num_qs);
2164
2165         for_all_evt_queues(adapter, eqo, i) {
2166                 netif_napi_add(adapter->netdev, &eqo->napi, be_poll,
2167                                BE_NAPI_WEIGHT);
2168                 napi_hash_add(&eqo->napi);
2169                 aic = &adapter->aic_obj[i];
2170                 eqo->adapter = adapter;
2171                 eqo->idx = i;
2172                 aic->max_eqd = BE_MAX_EQD;
2173                 aic->enable = true;
2174
2175                 eq = &eqo->q;
2176                 rc = be_queue_alloc(adapter, eq, EVNT_Q_LEN,
2177                                     sizeof(struct be_eq_entry));
2178                 if (rc)
2179                         return rc;
2180
2181                 rc = be_cmd_eq_create(adapter, eqo);
2182                 if (rc)
2183                         return rc;
2184         }
2185         return 0;
2186 }
2187
2188 static void be_mcc_queues_destroy(struct be_adapter *adapter)
2189 {
2190         struct be_queue_info *q;
2191
2192         q = &adapter->mcc_obj.q;
2193         if (q->created)
2194                 be_cmd_q_destroy(adapter, q, QTYPE_MCCQ);
2195         be_queue_free(adapter, q);
2196
2197         q = &adapter->mcc_obj.cq;
2198         if (q->created)
2199                 be_cmd_q_destroy(adapter, q, QTYPE_CQ);
2200         be_queue_free(adapter, q);
2201 }
2202
2203 /* Must be called only after TX qs are created as MCC shares TX EQ */
2204 static int be_mcc_queues_create(struct be_adapter *adapter)
2205 {
2206         struct be_queue_info *q, *cq;
2207
2208         cq = &adapter->mcc_obj.cq;
2209         if (be_queue_alloc(adapter, cq, MCC_CQ_LEN,
2210                            sizeof(struct be_mcc_compl)))
2211                 goto err;
2212
2213         /* Use the default EQ for MCC completions */
2214         if (be_cmd_cq_create(adapter, cq, &mcc_eqo(adapter)->q, true, 0))
2215                 goto mcc_cq_free;
2216
2217         q = &adapter->mcc_obj.q;
2218         if (be_queue_alloc(adapter, q, MCC_Q_LEN, sizeof(struct be_mcc_wrb)))
2219                 goto mcc_cq_destroy;
2220
2221         if (be_cmd_mccq_create(adapter, q, cq))
2222                 goto mcc_q_free;
2223
2224         return 0;
2225
2226 mcc_q_free:
2227         be_queue_free(adapter, q);
2228 mcc_cq_destroy:
2229         be_cmd_q_destroy(adapter, cq, QTYPE_CQ);
2230 mcc_cq_free:
2231         be_queue_free(adapter, cq);
2232 err:
2233         return -1;
2234 }
2235
2236 static void be_tx_queues_destroy(struct be_adapter *adapter)
2237 {
2238         struct be_queue_info *q;
2239         struct be_tx_obj *txo;
2240         u8 i;
2241
2242         for_all_tx_queues(adapter, txo, i) {
2243                 q = &txo->q;
2244                 if (q->created)
2245                         be_cmd_q_destroy(adapter, q, QTYPE_TXQ);
2246                 be_queue_free(adapter, q);
2247
2248                 q = &txo->cq;
2249                 if (q->created)
2250                         be_cmd_q_destroy(adapter, q, QTYPE_CQ);
2251                 be_queue_free(adapter, q);
2252         }
2253 }
2254
2255 static int be_tx_qs_create(struct be_adapter *adapter)
2256 {
2257         struct be_queue_info *cq, *eq;
2258         struct be_tx_obj *txo;
2259         int status, i;
2260
2261         adapter->num_tx_qs = min(adapter->num_evt_qs, be_max_txqs(adapter));
2262
2263         for_all_tx_queues(adapter, txo, i) {
2264                 cq = &txo->cq;
2265                 status = be_queue_alloc(adapter, cq, TX_CQ_LEN,
2266                                         sizeof(struct be_eth_tx_compl));
2267                 if (status)
2268                         return status;
2269
2270                 u64_stats_init(&txo->stats.sync);
2271                 u64_stats_init(&txo->stats.sync_compl);
2272
2273                 /* If num_evt_qs is less than num_tx_qs, then more than
2274                  * one txq share an eq
2275                  */
2276                 eq = &adapter->eq_obj[i % adapter->num_evt_qs].q;
2277                 status = be_cmd_cq_create(adapter, cq, eq, false, 3);
2278                 if (status)
2279                         return status;
2280
2281                 status = be_queue_alloc(adapter, &txo->q, TX_Q_LEN,
2282                                         sizeof(struct be_eth_wrb));
2283                 if (status)
2284                         return status;
2285
2286                 status = be_cmd_txq_create(adapter, txo);
2287                 if (status)
2288                         return status;
2289         }
2290
2291         dev_info(&adapter->pdev->dev, "created %d TX queue(s)\n",
2292                  adapter->num_tx_qs);
2293         return 0;
2294 }
2295
2296 static void be_rx_cqs_destroy(struct be_adapter *adapter)
2297 {
2298         struct be_queue_info *q;
2299         struct be_rx_obj *rxo;
2300         int i;
2301
2302         for_all_rx_queues(adapter, rxo, i) {
2303                 q = &rxo->cq;
2304                 if (q->created)
2305                         be_cmd_q_destroy(adapter, q, QTYPE_CQ);
2306                 be_queue_free(adapter, q);
2307         }
2308 }
2309
2310 static int be_rx_cqs_create(struct be_adapter *adapter)
2311 {
2312         struct be_queue_info *eq, *cq;
2313         struct be_rx_obj *rxo;
2314         int rc, i;
2315
2316         /* We can create as many RSS rings as there are EQs. */
2317         adapter->num_rx_qs = adapter->num_evt_qs;
2318
2319         /* We'll use RSS only if atleast 2 RSS rings are supported.
2320          * When RSS is used, we'll need a default RXQ for non-IP traffic.
2321          */
2322         if (adapter->num_rx_qs > 1)
2323                 adapter->num_rx_qs++;
2324
2325         adapter->big_page_size = (1 << get_order(rx_frag_size)) * PAGE_SIZE;
2326         for_all_rx_queues(adapter, rxo, i) {
2327                 rxo->adapter = adapter;
2328                 cq = &rxo->cq;
2329                 rc = be_queue_alloc(adapter, cq, RX_CQ_LEN,
2330                                     sizeof(struct be_eth_rx_compl));
2331                 if (rc)
2332                         return rc;
2333
2334                 u64_stats_init(&rxo->stats.sync);
2335                 eq = &adapter->eq_obj[i % adapter->num_evt_qs].q;
2336                 rc = be_cmd_cq_create(adapter, cq, eq, false, 3);
2337                 if (rc)
2338                         return rc;
2339         }
2340
2341         dev_info(&adapter->pdev->dev,
2342                  "created %d RSS queue(s) and 1 default RX queue\n",
2343                  adapter->num_rx_qs - 1);
2344         return 0;
2345 }
2346
2347 static irqreturn_t be_intx(int irq, void *dev)
2348 {
2349         struct be_eq_obj *eqo = dev;
2350         struct be_adapter *adapter = eqo->adapter;
2351         int num_evts = 0;
2352
2353         /* IRQ is not expected when NAPI is scheduled as the EQ
2354          * will not be armed.
2355          * But, this can happen on Lancer INTx where it takes
2356          * a while to de-assert INTx or in BE2 where occasionaly
2357          * an interrupt may be raised even when EQ is unarmed.
2358          * If NAPI is already scheduled, then counting & notifying
2359          * events will orphan them.
2360          */
2361         if (napi_schedule_prep(&eqo->napi)) {
2362                 num_evts = events_get(eqo);
2363                 __napi_schedule(&eqo->napi);
2364                 if (num_evts)
2365                         eqo->spurious_intr = 0;
2366         }
2367         be_eq_notify(adapter, eqo->q.id, false, true, num_evts);
2368
2369         /* Return IRQ_HANDLED only for the the first spurious intr
2370          * after a valid intr to stop the kernel from branding
2371          * this irq as a bad one!
2372          */
2373         if (num_evts || eqo->spurious_intr++ == 0)
2374                 return IRQ_HANDLED;
2375         else
2376                 return IRQ_NONE;
2377 }
2378
2379 static irqreturn_t be_msix(int irq, void *dev)
2380 {
2381         struct be_eq_obj *eqo = dev;
2382
2383         be_eq_notify(eqo->adapter, eqo->q.id, false, true, 0);
2384         napi_schedule(&eqo->napi);
2385         return IRQ_HANDLED;
2386 }
2387
2388 static inline bool do_gro(struct be_rx_compl_info *rxcp)
2389 {
2390         return (rxcp->tcpf && !rxcp->err && rxcp->l4_csum) ? true : false;
2391 }
2392
2393 static int be_process_rx(struct be_rx_obj *rxo, struct napi_struct *napi,
2394                          int budget, int polling)
2395 {
2396         struct be_adapter *adapter = rxo->adapter;
2397         struct be_queue_info *rx_cq = &rxo->cq;
2398         struct be_rx_compl_info *rxcp;
2399         u32 work_done;
2400         u32 frags_consumed = 0;
2401
2402         for (work_done = 0; work_done < budget; work_done++) {
2403                 rxcp = be_rx_compl_get(rxo);
2404                 if (!rxcp)
2405                         break;
2406
2407                 /* Is it a flush compl that has no data */
2408                 if (unlikely(rxcp->num_rcvd == 0))
2409                         goto loop_continue;
2410
2411                 /* Discard compl with partial DMA Lancer B0 */
2412                 if (unlikely(!rxcp->pkt_size)) {
2413                         be_rx_compl_discard(rxo, rxcp);
2414                         goto loop_continue;
2415                 }
2416
2417                 /* On BE drop pkts that arrive due to imperfect filtering in
2418                  * promiscuous mode on some skews
2419                  */
2420                 if (unlikely(rxcp->port != adapter->port_num &&
2421                              !lancer_chip(adapter))) {
2422                         be_rx_compl_discard(rxo, rxcp);
2423                         goto loop_continue;
2424                 }
2425
2426                 /* Don't do gro when we're busy_polling */
2427                 if (do_gro(rxcp) && polling != BUSY_POLLING)
2428                         be_rx_compl_process_gro(rxo, napi, rxcp);
2429                 else
2430                         be_rx_compl_process(rxo, napi, rxcp);
2431
2432 loop_continue:
2433                 frags_consumed += rxcp->num_rcvd;
2434                 be_rx_stats_update(rxo, rxcp);
2435         }
2436
2437         if (work_done) {
2438                 be_cq_notify(adapter, rx_cq->id, true, work_done);
2439
2440                 /* When an rx-obj gets into post_starved state, just
2441                  * let be_worker do the posting.
2442                  */
2443                 if (atomic_read(&rxo->q.used) < RX_FRAGS_REFILL_WM &&
2444                     !rxo->rx_post_starved)
2445                         be_post_rx_frags(rxo, GFP_ATOMIC,
2446                                          max_t(u32, MAX_RX_POST,
2447                                                frags_consumed));
2448         }
2449
2450         return work_done;
2451 }
2452
2453 static inline void be_update_tx_err(struct be_tx_obj *txo, u32 status)
2454 {
2455         switch (status) {
2456         case BE_TX_COMP_HDR_PARSE_ERR:
2457                 tx_stats(txo)->tx_hdr_parse_err++;
2458                 break;
2459         case BE_TX_COMP_NDMA_ERR:
2460                 tx_stats(txo)->tx_dma_err++;
2461                 break;
2462         case BE_TX_COMP_ACL_ERR:
2463                 tx_stats(txo)->tx_spoof_check_err++;
2464                 break;
2465         }
2466 }
2467
2468 static inline void lancer_update_tx_err(struct be_tx_obj *txo, u32 status)
2469 {
2470         switch (status) {
2471         case LANCER_TX_COMP_LSO_ERR:
2472                 tx_stats(txo)->tx_tso_err++;
2473                 break;
2474         case LANCER_TX_COMP_HSW_DROP_MAC_ERR:
2475         case LANCER_TX_COMP_HSW_DROP_VLAN_ERR:
2476                 tx_stats(txo)->tx_spoof_check_err++;
2477                 break;
2478         case LANCER_TX_COMP_QINQ_ERR:
2479                 tx_stats(txo)->tx_qinq_err++;
2480                 break;
2481         case LANCER_TX_COMP_PARITY_ERR:
2482                 tx_stats(txo)->tx_internal_parity_err++;
2483                 break;
2484         case LANCER_TX_COMP_DMA_ERR:
2485                 tx_stats(txo)->tx_dma_err++;
2486                 break;
2487         }
2488 }
2489
2490 static void be_process_tx(struct be_adapter *adapter, struct be_tx_obj *txo,
2491                           int idx)
2492 {
2493         struct be_eth_tx_compl *txcp;
2494         int num_wrbs = 0, work_done = 0;
2495         u32 compl_status;
2496         u16 last_idx;
2497
2498         while ((txcp = be_tx_compl_get(&txo->cq))) {
2499                 last_idx = GET_TX_COMPL_BITS(wrb_index, txcp);
2500                 num_wrbs += be_tx_compl_process(adapter, txo, last_idx);
2501                 work_done++;
2502
2503                 compl_status = GET_TX_COMPL_BITS(status, txcp);
2504                 if (compl_status) {
2505                         if (lancer_chip(adapter))
2506                                 lancer_update_tx_err(txo, compl_status);
2507                         else
2508                                 be_update_tx_err(txo, compl_status);
2509                 }
2510         }
2511
2512         if (work_done) {
2513                 be_cq_notify(adapter, txo->cq.id, true, work_done);
2514                 atomic_sub(num_wrbs, &txo->q.used);
2515
2516                 /* As Tx wrbs have been freed up, wake up netdev queue
2517                  * if it was stopped due to lack of tx wrbs.  */
2518                 if (__netif_subqueue_stopped(adapter->netdev, idx) &&
2519                     atomic_read(&txo->q.used) < txo->q.len / 2) {
2520                         netif_wake_subqueue(adapter->netdev, idx);
2521                 }
2522
2523                 u64_stats_update_begin(&tx_stats(txo)->sync_compl);
2524                 tx_stats(txo)->tx_compl += work_done;
2525                 u64_stats_update_end(&tx_stats(txo)->sync_compl);
2526         }
2527 }
2528
2529 int be_poll(struct napi_struct *napi, int budget)
2530 {
2531         struct be_eq_obj *eqo = container_of(napi, struct be_eq_obj, napi);
2532         struct be_adapter *adapter = eqo->adapter;
2533         int max_work = 0, work, i, num_evts;
2534         struct be_rx_obj *rxo;
2535         struct be_tx_obj *txo;
2536
2537         num_evts = events_get(eqo);
2538
2539         for_all_tx_queues_on_eq(adapter, eqo, txo, i)
2540                 be_process_tx(adapter, txo, i);
2541
2542         if (be_lock_napi(eqo)) {
2543                 /* This loop will iterate twice for EQ0 in which
2544                  * completions of the last RXQ (default one) are also processed
2545                  * For other EQs the loop iterates only once
2546                  */
2547                 for_all_rx_queues_on_eq(adapter, eqo, rxo, i) {
2548                         work = be_process_rx(rxo, napi, budget, NAPI_POLLING);
2549                         max_work = max(work, max_work);
2550                 }
2551                 be_unlock_napi(eqo);
2552         } else {
2553                 max_work = budget;
2554         }
2555
2556         if (is_mcc_eqo(eqo))
2557                 be_process_mcc(adapter);
2558
2559         if (max_work < budget) {
2560                 napi_complete(napi);
2561                 be_eq_notify(adapter, eqo->q.id, true, false, num_evts);
2562         } else {
2563                 /* As we'll continue in polling mode, count and clear events */
2564                 be_eq_notify(adapter, eqo->q.id, false, false, num_evts);
2565         }
2566         return max_work;
2567 }
2568
2569 #ifdef CONFIG_NET_RX_BUSY_POLL
2570 static int be_busy_poll(struct napi_struct *napi)
2571 {
2572         struct be_eq_obj *eqo = container_of(napi, struct be_eq_obj, napi);
2573         struct be_adapter *adapter = eqo->adapter;
2574         struct be_rx_obj *rxo;
2575         int i, work = 0;
2576
2577         if (!be_lock_busy_poll(eqo))
2578                 return LL_FLUSH_BUSY;
2579
2580         for_all_rx_queues_on_eq(adapter, eqo, rxo, i) {
2581                 work = be_process_rx(rxo, napi, 4, BUSY_POLLING);
2582                 if (work)
2583                         break;
2584         }
2585
2586         be_unlock_busy_poll(eqo);
2587         return work;
2588 }
2589 #endif
2590
2591 void be_detect_error(struct be_adapter *adapter)
2592 {
2593         u32 ue_lo = 0, ue_hi = 0, ue_lo_mask = 0, ue_hi_mask = 0;
2594         u32 sliport_status = 0, sliport_err1 = 0, sliport_err2 = 0;
2595         u32 i;
2596         bool error_detected = false;
2597         struct device *dev = &adapter->pdev->dev;
2598         struct net_device *netdev = adapter->netdev;
2599
2600         if (be_hw_error(adapter))
2601                 return;
2602
2603         if (lancer_chip(adapter)) {
2604                 sliport_status = ioread32(adapter->db + SLIPORT_STATUS_OFFSET);
2605                 if (sliport_status & SLIPORT_STATUS_ERR_MASK) {
2606                         sliport_err1 = ioread32(adapter->db +
2607                                                 SLIPORT_ERROR1_OFFSET);
2608                         sliport_err2 = ioread32(adapter->db +
2609                                                 SLIPORT_ERROR2_OFFSET);
2610                         adapter->hw_error = true;
2611                         /* Do not log error messages if its a FW reset */
2612                         if (sliport_err1 == SLIPORT_ERROR_FW_RESET1 &&
2613                             sliport_err2 == SLIPORT_ERROR_FW_RESET2) {
2614                                 dev_info(dev, "Firmware update in progress\n");
2615                         } else {
2616                                 error_detected = true;
2617                                 dev_err(dev, "Error detected in the card\n");
2618                                 dev_err(dev, "ERR: sliport status 0x%x\n",
2619                                         sliport_status);
2620                                 dev_err(dev, "ERR: sliport error1 0x%x\n",
2621                                         sliport_err1);
2622                                 dev_err(dev, "ERR: sliport error2 0x%x\n",
2623                                         sliport_err2);
2624                         }
2625                 }
2626         } else {
2627                 pci_read_config_dword(adapter->pdev,
2628                                       PCICFG_UE_STATUS_LOW, &ue_lo);
2629                 pci_read_config_dword(adapter->pdev,
2630                                       PCICFG_UE_STATUS_HIGH, &ue_hi);
2631                 pci_read_config_dword(adapter->pdev,
2632                                       PCICFG_UE_STATUS_LOW_MASK, &ue_lo_mask);
2633                 pci_read_config_dword(adapter->pdev,
2634                                       PCICFG_UE_STATUS_HI_MASK, &ue_hi_mask);
2635
2636                 ue_lo = (ue_lo & ~ue_lo_mask);
2637                 ue_hi = (ue_hi & ~ue_hi_mask);
2638
2639                 /* On certain platforms BE hardware can indicate spurious UEs.
2640                  * Allow HW to stop working completely in case of a real UE.
2641                  * Hence not setting the hw_error for UE detection.
2642                  */
2643
2644                 if (ue_lo || ue_hi) {
2645                         error_detected = true;
2646                         dev_err(dev,
2647                                 "Unrecoverable Error detected in the adapter");
2648                         dev_err(dev, "Please reboot server to recover");
2649                         if (skyhawk_chip(adapter))
2650                                 adapter->hw_error = true;
2651                         for (i = 0; ue_lo; ue_lo >>= 1, i++) {
2652                                 if (ue_lo & 1)
2653                                         dev_err(dev, "UE: %s bit set\n",
2654                                                 ue_status_low_desc[i]);
2655                         }
2656                         for (i = 0; ue_hi; ue_hi >>= 1, i++) {
2657                                 if (ue_hi & 1)
2658                                         dev_err(dev, "UE: %s bit set\n",
2659                                                 ue_status_hi_desc[i]);
2660                         }
2661                 }
2662         }
2663         if (error_detected)
2664                 netif_carrier_off(netdev);
2665 }
2666
2667 static void be_msix_disable(struct be_adapter *adapter)
2668 {
2669         if (msix_enabled(adapter)) {
2670                 pci_disable_msix(adapter->pdev);
2671                 adapter->num_msix_vec = 0;
2672                 adapter->num_msix_roce_vec = 0;
2673         }
2674 }
2675
2676 static int be_msix_enable(struct be_adapter *adapter)
2677 {
2678         int i, num_vec;
2679         struct device *dev = &adapter->pdev->dev;
2680
2681         /* If RoCE is supported, program the max number of NIC vectors that
2682          * may be configured via set-channels, along with vectors needed for
2683          * RoCe. Else, just program the number we'll use initially.
2684          */
2685         if (be_roce_supported(adapter))
2686                 num_vec = min_t(int, 2 * be_max_eqs(adapter),
2687                                 2 * num_online_cpus());
2688         else
2689                 num_vec = adapter->cfg_num_qs;
2690
2691         for (i = 0; i < num_vec; i++)
2692                 adapter->msix_entries[i].entry = i;
2693
2694         num_vec = pci_enable_msix_range(adapter->pdev, adapter->msix_entries,
2695                                         MIN_MSIX_VECTORS, num_vec);
2696         if (num_vec < 0)
2697                 goto fail;
2698
2699         if (be_roce_supported(adapter) && num_vec > MIN_MSIX_VECTORS) {
2700                 adapter->num_msix_roce_vec = num_vec / 2;
2701                 dev_info(dev, "enabled %d MSI-x vector(s) for RoCE\n",
2702                          adapter->num_msix_roce_vec);
2703         }
2704
2705         adapter->num_msix_vec = num_vec - adapter->num_msix_roce_vec;
2706
2707         dev_info(dev, "enabled %d MSI-x vector(s) for NIC\n",
2708                  adapter->num_msix_vec);
2709         return 0;
2710
2711 fail:
2712         dev_warn(dev, "MSIx enable failed\n");
2713
2714         /* INTx is not supported in VFs, so fail probe if enable_msix fails */
2715         if (!be_physfn(adapter))
2716                 return num_vec;
2717         return 0;
2718 }
2719
2720 static inline int be_msix_vec_get(struct be_adapter *adapter,
2721                                   struct be_eq_obj *eqo)
2722 {
2723         return adapter->msix_entries[eqo->msix_idx].vector;
2724 }
2725
2726 static int be_msix_register(struct be_adapter *adapter)
2727 {
2728         struct net_device *netdev = adapter->netdev;
2729         struct be_eq_obj *eqo;
2730         int status, i, vec;
2731
2732         for_all_evt_queues(adapter, eqo, i) {
2733                 sprintf(eqo->desc, "%s-q%d", netdev->name, i);
2734                 vec = be_msix_vec_get(adapter, eqo);
2735                 status = request_irq(vec, be_msix, 0, eqo->desc, eqo);
2736                 if (status)
2737                         goto err_msix;
2738         }
2739
2740         return 0;
2741 err_msix:
2742         for (i--, eqo = &adapter->eq_obj[i]; i >= 0; i--, eqo--)
2743                 free_irq(be_msix_vec_get(adapter, eqo), eqo);
2744         dev_warn(&adapter->pdev->dev, "MSIX Request IRQ failed - err %d\n",
2745                  status);
2746         be_msix_disable(adapter);
2747         return status;
2748 }
2749
2750 static int be_irq_register(struct be_adapter *adapter)
2751 {
2752         struct net_device *netdev = adapter->netdev;
2753         int status;
2754
2755         if (msix_enabled(adapter)) {
2756                 status = be_msix_register(adapter);
2757                 if (status == 0)
2758                         goto done;
2759                 /* INTx is not supported for VF */
2760                 if (!be_physfn(adapter))
2761                         return status;
2762         }
2763
2764         /* INTx: only the first EQ is used */
2765         netdev->irq = adapter->pdev->irq;
2766         status = request_irq(netdev->irq, be_intx, IRQF_SHARED, netdev->name,
2767                              &adapter->eq_obj[0]);
2768         if (status) {
2769                 dev_err(&adapter->pdev->dev,
2770                         "INTx request IRQ failed - err %d\n", status);
2771                 return status;
2772         }
2773 done:
2774         adapter->isr_registered = true;
2775         return 0;
2776 }
2777
2778 static void be_irq_unregister(struct be_adapter *adapter)
2779 {
2780         struct net_device *netdev = adapter->netdev;
2781         struct be_eq_obj *eqo;
2782         int i;
2783
2784         if (!adapter->isr_registered)
2785                 return;
2786
2787         /* INTx */
2788         if (!msix_enabled(adapter)) {
2789                 free_irq(netdev->irq, &adapter->eq_obj[0]);
2790                 goto done;
2791         }
2792
2793         /* MSIx */
2794         for_all_evt_queues(adapter, eqo, i)
2795                 free_irq(be_msix_vec_get(adapter, eqo), eqo);
2796
2797 done:
2798         adapter->isr_registered = false;
2799 }
2800
2801 static void be_rx_qs_destroy(struct be_adapter *adapter)
2802 {
2803         struct be_queue_info *q;
2804         struct be_rx_obj *rxo;
2805         int i;
2806
2807         for_all_rx_queues(adapter, rxo, i) {
2808                 q = &rxo->q;
2809                 if (q->created) {
2810                         be_cmd_rxq_destroy(adapter, q);
2811                         be_rx_cq_clean(rxo);
2812                 }
2813                 be_queue_free(adapter, q);
2814         }
2815 }
2816
2817 static int be_close(struct net_device *netdev)
2818 {
2819         struct be_adapter *adapter = netdev_priv(netdev);
2820         struct be_eq_obj *eqo;
2821         int i;
2822
2823         /* This protection is needed as be_close() may be called even when the
2824          * adapter is in cleared state (after eeh perm failure)
2825          */
2826         if (!(adapter->flags & BE_FLAGS_SETUP_DONE))
2827                 return 0;
2828
2829         be_roce_dev_close(adapter);
2830
2831         if (adapter->flags & BE_FLAGS_NAPI_ENABLED) {
2832                 for_all_evt_queues(adapter, eqo, i) {
2833                         napi_disable(&eqo->napi);
2834                         be_disable_busy_poll(eqo);
2835                 }
2836                 adapter->flags &= ~BE_FLAGS_NAPI_ENABLED;
2837         }
2838
2839         be_async_mcc_disable(adapter);
2840
2841         /* Wait for all pending tx completions to arrive so that
2842          * all tx skbs are freed.
2843          */
2844         netif_tx_disable(netdev);
2845         be_tx_compl_clean(adapter);
2846
2847         be_rx_qs_destroy(adapter);
2848
2849         for (i = 1; i < (adapter->uc_macs + 1); i++)
2850                 be_cmd_pmac_del(adapter, adapter->if_handle,
2851                                 adapter->pmac_id[i], 0);
2852         adapter->uc_macs = 0;
2853
2854         for_all_evt_queues(adapter, eqo, i) {
2855                 if (msix_enabled(adapter))
2856                         synchronize_irq(be_msix_vec_get(adapter, eqo));
2857                 else
2858                         synchronize_irq(netdev->irq);
2859                 be_eq_clean(eqo);
2860         }
2861
2862         be_irq_unregister(adapter);
2863
2864         return 0;
2865 }
2866
2867 static int be_rx_qs_create(struct be_adapter *adapter)
2868 {
2869         struct rss_info *rss = &adapter->rss_info;
2870         u8 rss_key[RSS_HASH_KEY_LEN];
2871         struct be_rx_obj *rxo;
2872         int rc, i, j;
2873
2874         for_all_rx_queues(adapter, rxo, i) {
2875                 rc = be_queue_alloc(adapter, &rxo->q, RX_Q_LEN,
2876                                     sizeof(struct be_eth_rx_d));
2877                 if (rc)
2878                         return rc;
2879         }
2880
2881         /* The FW would like the default RXQ to be created first */
2882         rxo = default_rxo(adapter);
2883         rc = be_cmd_rxq_create(adapter, &rxo->q, rxo->cq.id, rx_frag_size,
2884                                adapter->if_handle, false, &rxo->rss_id);
2885         if (rc)
2886                 return rc;
2887
2888         for_all_rss_queues(adapter, rxo, i) {
2889                 rc = be_cmd_rxq_create(adapter, &rxo->q, rxo->cq.id,
2890                                        rx_frag_size, adapter->if_handle,
2891                                        true, &rxo->rss_id);
2892                 if (rc)
2893                         return rc;
2894         }
2895
2896         if (be_multi_rxq(adapter)) {
2897                 for (j = 0; j < RSS_INDIR_TABLE_LEN;
2898                         j += adapter->num_rx_qs - 1) {
2899                         for_all_rss_queues(adapter, rxo, i) {
2900                                 if ((j + i) >= RSS_INDIR_TABLE_LEN)
2901                                         break;
2902                                 rss->rsstable[j + i] = rxo->rss_id;
2903                                 rss->rss_queue[j + i] = i;
2904                         }
2905                 }
2906                 rss->rss_flags = RSS_ENABLE_TCP_IPV4 | RSS_ENABLE_IPV4 |
2907                         RSS_ENABLE_TCP_IPV6 | RSS_ENABLE_IPV6;
2908
2909                 if (!BEx_chip(adapter))
2910                         rss->rss_flags |= RSS_ENABLE_UDP_IPV4 |
2911                                 RSS_ENABLE_UDP_IPV6;
2912         } else {
2913                 /* Disable RSS, if only default RX Q is created */
2914                 rss->rss_flags = RSS_ENABLE_NONE;
2915         }
2916
2917         netdev_rss_key_fill(rss_key, RSS_HASH_KEY_LEN);
2918         rc = be_cmd_rss_config(adapter, rss->rsstable, rss->rss_flags,
2919                                128, rss_key);
2920         if (rc) {
2921                 rss->rss_flags = RSS_ENABLE_NONE;
2922                 return rc;
2923         }
2924
2925         memcpy(rss->rss_hkey, rss_key, RSS_HASH_KEY_LEN);
2926
2927         /* First time posting */
2928         for_all_rx_queues(adapter, rxo, i)
2929                 be_post_rx_frags(rxo, GFP_KERNEL, MAX_RX_POST);
2930         return 0;
2931 }
2932
2933 static int be_open(struct net_device *netdev)
2934 {
2935         struct be_adapter *adapter = netdev_priv(netdev);
2936         struct be_eq_obj *eqo;
2937         struct be_rx_obj *rxo;
2938         struct be_tx_obj *txo;
2939         u8 link_status;
2940         int status, i;
2941
2942         status = be_rx_qs_create(adapter);
2943         if (status)
2944                 goto err;
2945
2946         status = be_irq_register(adapter);
2947         if (status)
2948                 goto err;
2949
2950         for_all_rx_queues(adapter, rxo, i)
2951                 be_cq_notify(adapter, rxo->cq.id, true, 0);
2952
2953         for_all_tx_queues(adapter, txo, i)
2954                 be_cq_notify(adapter, txo->cq.id, true, 0);
2955
2956         be_async_mcc_enable(adapter);
2957
2958         for_all_evt_queues(adapter, eqo, i) {
2959                 napi_enable(&eqo->napi);
2960                 be_enable_busy_poll(eqo);
2961                 be_eq_notify(adapter, eqo->q.id, true, true, 0);
2962         }
2963         adapter->flags |= BE_FLAGS_NAPI_ENABLED;
2964
2965         status = be_cmd_link_status_query(adapter, NULL, &link_status, 0);
2966         if (!status)
2967                 be_link_status_update(adapter, link_status);
2968
2969         netif_tx_start_all_queues(netdev);
2970         be_roce_dev_open(adapter);
2971
2972 #ifdef CONFIG_BE2NET_VXLAN
2973         if (skyhawk_chip(adapter))
2974                 vxlan_get_rx_port(netdev);
2975 #endif
2976
2977         return 0;
2978 err:
2979         be_close(adapter->netdev);
2980         return -EIO;
2981 }
2982
2983 static int be_setup_wol(struct be_adapter *adapter, bool enable)
2984 {
2985         struct be_dma_mem cmd;
2986         int status = 0;
2987         u8 mac[ETH_ALEN];
2988
2989         memset(mac, 0, ETH_ALEN);
2990
2991         cmd.size = sizeof(struct be_cmd_req_acpi_wol_magic_config);
2992         cmd.va = dma_zalloc_coherent(&adapter->pdev->dev, cmd.size, &cmd.dma,
2993                                      GFP_KERNEL);
2994         if (!cmd.va)
2995                 return -ENOMEM;
2996
2997         if (enable) {
2998                 status = pci_write_config_dword(adapter->pdev,
2999                                                 PCICFG_PM_CONTROL_OFFSET,
3000                                                 PCICFG_PM_CONTROL_MASK);
3001                 if (status) {
3002                         dev_err(&adapter->pdev->dev,
3003                                 "Could not enable Wake-on-lan\n");
3004                         dma_free_coherent(&adapter->pdev->dev, cmd.size, cmd.va,
3005                                           cmd.dma);
3006                         return status;
3007                 }
3008                 status = be_cmd_enable_magic_wol(adapter,
3009                                                  adapter->netdev->dev_addr,
3010                                                  &cmd);
3011                 pci_enable_wake(adapter->pdev, PCI_D3hot, 1);
3012                 pci_enable_wake(adapter->pdev, PCI_D3cold, 1);
3013         } else {
3014                 status = be_cmd_enable_magic_wol(adapter, mac, &cmd);
3015                 pci_enable_wake(adapter->pdev, PCI_D3hot, 0);
3016                 pci_enable_wake(adapter->pdev, PCI_D3cold, 0);
3017         }
3018
3019         dma_free_coherent(&adapter->pdev->dev, cmd.size, cmd.va, cmd.dma);
3020         return status;
3021 }
3022
3023 /*
3024  * Generate a seed MAC address from the PF MAC Address using jhash.
3025  * MAC Address for VFs are assigned incrementally starting from the seed.
3026  * These addresses are programmed in the ASIC by the PF and the VF driver
3027  * queries for the MAC address during its probe.
3028  */
3029 static int be_vf_eth_addr_config(struct be_adapter *adapter)
3030 {
3031         u32 vf;
3032         int status = 0;
3033         u8 mac[ETH_ALEN];
3034         struct be_vf_cfg *vf_cfg;
3035
3036         be_vf_eth_addr_generate(adapter, mac);
3037
3038         for_all_vfs(adapter, vf_cfg, vf) {
3039                 if (BEx_chip(adapter))
3040                         status = be_cmd_pmac_add(adapter, mac,
3041                                                  vf_cfg->if_handle,
3042                                                  &vf_cfg->pmac_id, vf + 1);
3043                 else
3044                         status = be_cmd_set_mac(adapter, mac, vf_cfg->if_handle,
3045                                                 vf + 1);
3046
3047                 if (status)
3048                         dev_err(&adapter->pdev->dev,
3049                                 "Mac address assignment failed for VF %d\n",
3050                                 vf);
3051                 else
3052                         memcpy(vf_cfg->mac_addr, mac, ETH_ALEN);
3053
3054                 mac[5] += 1;
3055         }
3056         return status;
3057 }
3058
3059 static int be_vfs_mac_query(struct be_adapter *adapter)
3060 {
3061         int status, vf;
3062         u8 mac[ETH_ALEN];
3063         struct be_vf_cfg *vf_cfg;
3064
3065         for_all_vfs(adapter, vf_cfg, vf) {
3066                 status = be_cmd_get_active_mac(adapter, vf_cfg->pmac_id,
3067                                                mac, vf_cfg->if_handle,
3068                                                false, vf+1);
3069                 if (status)
3070                         return status;
3071                 memcpy(vf_cfg->mac_addr, mac, ETH_ALEN);
3072         }
3073         return 0;
3074 }
3075
3076 static void be_vf_clear(struct be_adapter *adapter)
3077 {
3078         struct be_vf_cfg *vf_cfg;
3079         u32 vf;
3080
3081         if (pci_vfs_assigned(adapter->pdev)) {
3082                 dev_warn(&adapter->pdev->dev,
3083                          "VFs are assigned to VMs: not disabling VFs\n");
3084                 goto done;
3085         }
3086
3087         pci_disable_sriov(adapter->pdev);
3088
3089         for_all_vfs(adapter, vf_cfg, vf) {
3090                 if (BEx_chip(adapter))
3091                         be_cmd_pmac_del(adapter, vf_cfg->if_handle,
3092                                         vf_cfg->pmac_id, vf + 1);
3093                 else
3094                         be_cmd_set_mac(adapter, NULL, vf_cfg->if_handle,
3095                                        vf + 1);
3096
3097                 be_cmd_if_destroy(adapter, vf_cfg->if_handle, vf + 1);
3098         }
3099 done:
3100         kfree(adapter->vf_cfg);
3101         adapter->num_vfs = 0;
3102         adapter->flags &= ~BE_FLAGS_SRIOV_ENABLED;
3103 }
3104
3105 static void be_clear_queues(struct be_adapter *adapter)
3106 {
3107         be_mcc_queues_destroy(adapter);
3108         be_rx_cqs_destroy(adapter);
3109         be_tx_queues_destroy(adapter);
3110         be_evt_queues_destroy(adapter);
3111 }
3112
3113 static void be_cancel_worker(struct be_adapter *adapter)
3114 {
3115         if (adapter->flags & BE_FLAGS_WORKER_SCHEDULED) {
3116                 cancel_delayed_work_sync(&adapter->work);
3117                 adapter->flags &= ~BE_FLAGS_WORKER_SCHEDULED;
3118         }
3119 }
3120
3121 static void be_mac_clear(struct be_adapter *adapter)
3122 {
3123         int i;
3124
3125         if (adapter->pmac_id) {
3126                 for (i = 0; i < (adapter->uc_macs + 1); i++)
3127                         be_cmd_pmac_del(adapter, adapter->if_handle,
3128                                         adapter->pmac_id[i], 0);
3129                 adapter->uc_macs = 0;
3130
3131                 kfree(adapter->pmac_id);
3132                 adapter->pmac_id = NULL;
3133         }
3134 }
3135
3136 #ifdef CONFIG_BE2NET_VXLAN
3137 static void be_disable_vxlan_offloads(struct be_adapter *adapter)
3138 {
3139         struct net_device *netdev = adapter->netdev;
3140
3141         if (adapter->flags & BE_FLAGS_VXLAN_OFFLOADS)
3142                 be_cmd_manage_iface(adapter, adapter->if_handle,
3143                                     OP_CONVERT_TUNNEL_TO_NORMAL);
3144
3145         if (adapter->vxlan_port)
3146                 be_cmd_set_vxlan_port(adapter, 0);
3147
3148         adapter->flags &= ~BE_FLAGS_VXLAN_OFFLOADS;
3149         adapter->vxlan_port = 0;
3150
3151         netdev->hw_enc_features = 0;
3152         netdev->hw_features &= ~(NETIF_F_GSO_UDP_TUNNEL);
3153         netdev->features &= ~(NETIF_F_GSO_UDP_TUNNEL);
3154 }
3155 #endif
3156
3157 static int be_clear(struct be_adapter *adapter)
3158 {
3159         be_cancel_worker(adapter);
3160
3161         if (sriov_enabled(adapter))
3162                 be_vf_clear(adapter);
3163
3164         /* Re-configure FW to distribute resources evenly across max-supported
3165          * number of VFs, only when VFs are not already enabled.
3166          */
3167         if (be_physfn(adapter) && !pci_vfs_assigned(adapter->pdev))
3168                 be_cmd_set_sriov_config(adapter, adapter->pool_res,
3169                                         pci_sriov_get_totalvfs(adapter->pdev));
3170
3171 #ifdef CONFIG_BE2NET_VXLAN
3172         be_disable_vxlan_offloads(adapter);
3173 #endif
3174         /* delete the primary mac along with the uc-mac list */
3175         be_mac_clear(adapter);
3176
3177         be_cmd_if_destroy(adapter, adapter->if_handle,  0);
3178
3179         be_clear_queues(adapter);
3180
3181         be_msix_disable(adapter);
3182         adapter->flags &= ~BE_FLAGS_SETUP_DONE;
3183         return 0;
3184 }
3185
3186 static int be_if_create(struct be_adapter *adapter, u32 *if_handle,
3187                         u32 cap_flags, u32 vf)
3188 {
3189         u32 en_flags;
3190         int status;
3191
3192         en_flags = BE_IF_FLAGS_UNTAGGED | BE_IF_FLAGS_BROADCAST |
3193                    BE_IF_FLAGS_MULTICAST | BE_IF_FLAGS_PASS_L3L4_ERRORS |
3194                    BE_IF_FLAGS_RSS;
3195
3196         en_flags &= cap_flags;
3197
3198         status = be_cmd_if_create(adapter, cap_flags, en_flags,
3199                                   if_handle, vf);
3200
3201         return status;
3202 }
3203
3204 static int be_vfs_if_create(struct be_adapter *adapter)
3205 {
3206         struct be_resources res = {0};
3207         struct be_vf_cfg *vf_cfg;
3208         u32 cap_flags, vf;
3209         int status;
3210
3211         /* If a FW profile exists, then cap_flags are updated */
3212         cap_flags = BE_IF_FLAGS_UNTAGGED | BE_IF_FLAGS_BROADCAST |
3213                     BE_IF_FLAGS_MULTICAST;
3214
3215         for_all_vfs(adapter, vf_cfg, vf) {
3216                 if (!BE3_chip(adapter)) {
3217                         status = be_cmd_get_profile_config(adapter, &res,
3218                                                            vf + 1);
3219                         if (!status)
3220                                 cap_flags = res.if_cap_flags;
3221                 }
3222
3223                 status = be_if_create(adapter, &vf_cfg->if_handle,
3224                                       cap_flags, vf + 1);
3225                 if (status)
3226                         return status;
3227         }
3228
3229         return 0;
3230 }
3231
3232 static int be_vf_setup_init(struct be_adapter *adapter)
3233 {
3234         struct be_vf_cfg *vf_cfg;
3235         int vf;
3236
3237         adapter->vf_cfg = kcalloc(adapter->num_vfs, sizeof(*vf_cfg),
3238                                   GFP_KERNEL);
3239         if (!adapter->vf_cfg)
3240                 return -ENOMEM;
3241
3242         for_all_vfs(adapter, vf_cfg, vf) {
3243                 vf_cfg->if_handle = -1;
3244                 vf_cfg->pmac_id = -1;
3245         }
3246         return 0;
3247 }
3248
3249 static int be_vf_setup(struct be_adapter *adapter)
3250 {
3251         struct device *dev = &adapter->pdev->dev;
3252         struct be_vf_cfg *vf_cfg;
3253         int status, old_vfs, vf;
3254         u32 privileges;
3255
3256         old_vfs = pci_num_vf(adapter->pdev);
3257
3258         status = be_vf_setup_init(adapter);
3259         if (status)
3260                 goto err;
3261
3262         if (old_vfs) {
3263                 for_all_vfs(adapter, vf_cfg, vf) {
3264                         status = be_cmd_get_if_id(adapter, vf_cfg, vf);
3265                         if (status)
3266                                 goto err;
3267                 }
3268
3269                 status = be_vfs_mac_query(adapter);
3270                 if (status)
3271                         goto err;
3272         } else {
3273                 status = be_vfs_if_create(adapter);
3274                 if (status)
3275                         goto err;
3276
3277                 status = be_vf_eth_addr_config(adapter);
3278                 if (status)
3279                         goto err;
3280         }
3281
3282         for_all_vfs(adapter, vf_cfg, vf) {
3283                 /* Allow VFs to programs MAC/VLAN filters */
3284                 status = be_cmd_get_fn_privileges(adapter, &privileges, vf + 1);
3285                 if (!status && !(privileges & BE_PRIV_FILTMGMT)) {
3286                         status = be_cmd_set_fn_privileges(adapter,
3287                                                           privileges |
3288                                                           BE_PRIV_FILTMGMT,
3289                                                           vf + 1);
3290                         if (!status)
3291                                 dev_info(dev, "VF%d has FILTMGMT privilege\n",
3292                                          vf);
3293                 }
3294
3295                 /* Allow full available bandwidth */
3296                 if (!old_vfs)
3297                         be_cmd_config_qos(adapter, 0, 0, vf + 1);
3298
3299                 if (!old_vfs) {
3300                         be_cmd_enable_vf(adapter, vf + 1);
3301                         be_cmd_set_logical_link_config(adapter,
3302                                                        IFLA_VF_LINK_STATE_AUTO,
3303                                                        vf+1);
3304                 }
3305         }
3306
3307         if (!old_vfs) {
3308                 status = pci_enable_sriov(adapter->pdev, adapter->num_vfs);
3309                 if (status) {
3310                         dev_err(dev, "SRIOV enable failed\n");
3311                         adapter->num_vfs = 0;
3312                         goto err;
3313                 }
3314         }
3315
3316         adapter->flags |= BE_FLAGS_SRIOV_ENABLED;
3317         return 0;
3318 err:
3319         dev_err(dev, "VF setup failed\n");
3320         be_vf_clear(adapter);
3321         return status;
3322 }
3323
3324 /* Converting function_mode bits on BE3 to SH mc_type enums */
3325
3326 static u8 be_convert_mc_type(u32 function_mode)
3327 {
3328         if (function_mode & VNIC_MODE && function_mode & QNQ_MODE)
3329                 return vNIC1;
3330         else if (function_mode & QNQ_MODE)
3331                 return FLEX10;
3332         else if (function_mode & VNIC_MODE)
3333                 return vNIC2;
3334         else if (function_mode & UMC_ENABLED)
3335                 return UMC;
3336         else
3337                 return MC_NONE;
3338 }
3339
3340 /* On BE2/BE3 FW does not suggest the supported limits */
3341 static void BEx_get_resources(struct be_adapter *adapter,
3342                               struct be_resources *res)
3343 {
3344         bool use_sriov = adapter->num_vfs ? 1 : 0;
3345
3346         if (be_physfn(adapter))
3347                 res->max_uc_mac = BE_UC_PMAC_COUNT;
3348         else
3349                 res->max_uc_mac = BE_VF_UC_PMAC_COUNT;
3350
3351         adapter->mc_type = be_convert_mc_type(adapter->function_mode);
3352
3353         if (be_is_mc(adapter)) {
3354                 /* Assuming that there are 4 channels per port,
3355                  * when multi-channel is enabled
3356                  */
3357                 if (be_is_qnq_mode(adapter))
3358                         res->max_vlans = BE_NUM_VLANS_SUPPORTED/8;
3359                 else
3360                         /* In a non-qnq multichannel mode, the pvid
3361                          * takes up one vlan entry
3362                          */
3363                         res->max_vlans = (BE_NUM_VLANS_SUPPORTED / 4) - 1;
3364         } else {
3365                 res->max_vlans = BE_NUM_VLANS_SUPPORTED;
3366         }
3367
3368         res->max_mcast_mac = BE_MAX_MC;
3369
3370         /* 1) For BE3 1Gb ports, FW does not support multiple TXQs
3371          * 2) Create multiple TX rings on a BE3-R multi-channel interface
3372          *    *only* if it is RSS-capable.
3373          */
3374         if (BE2_chip(adapter) || use_sriov ||  (adapter->port_num > 1) ||
3375             !be_physfn(adapter) || (be_is_mc(adapter) &&
3376             !(adapter->function_caps & BE_FUNCTION_CAPS_RSS))) {
3377                 res->max_tx_qs = 1;
3378         } else if (adapter->function_caps & BE_FUNCTION_CAPS_SUPER_NIC) {
3379                 struct be_resources super_nic_res = {0};
3380
3381                 /* On a SuperNIC profile, the driver needs to use the
3382                  * GET_PROFILE_CONFIG cmd to query the per-function TXQ limits
3383                  */
3384                 be_cmd_get_profile_config(adapter, &super_nic_res, 0);
3385                 /* Some old versions of BE3 FW don't report max_tx_qs value */
3386                 res->max_tx_qs = super_nic_res.max_tx_qs ? : BE3_MAX_TX_QS;
3387         } else {
3388                 res->max_tx_qs = BE3_MAX_TX_QS;
3389         }
3390
3391         if ((adapter->function_caps & BE_FUNCTION_CAPS_RSS) &&
3392             !use_sriov && be_physfn(adapter))
3393                 res->max_rss_qs = (adapter->be3_native) ?
3394                                            BE3_MAX_RSS_QS : BE2_MAX_RSS_QS;
3395         res->max_rx_qs = res->max_rss_qs + 1;
3396
3397         if (be_physfn(adapter))
3398                 res->max_evt_qs = (be_max_vfs(adapter) > 0) ?
3399                                         BE3_SRIOV_MAX_EVT_QS : BE3_MAX_EVT_QS;
3400         else
3401                 res->max_evt_qs = 1;
3402
3403         res->if_cap_flags = BE_IF_CAP_FLAGS_WANT;
3404         if (!(adapter->function_caps & BE_FUNCTION_CAPS_RSS))
3405                 res->if_cap_flags &= ~BE_IF_FLAGS_RSS;
3406 }
3407
3408 static void be_setup_init(struct be_adapter *adapter)
3409 {
3410         adapter->vlan_prio_bmap = 0xff;
3411         adapter->phy.link_speed = -1;
3412         adapter->if_handle = -1;
3413         adapter->be3_native = false;
3414         adapter->promiscuous = false;
3415         if (be_physfn(adapter))
3416                 adapter->cmd_privileges = MAX_PRIVILEGES;
3417         else
3418                 adapter->cmd_privileges = MIN_PRIVILEGES;
3419 }
3420
3421 static int be_get_sriov_config(struct be_adapter *adapter)
3422 {
3423         struct device *dev = &adapter->pdev->dev;
3424         struct be_resources res = {0};
3425         int max_vfs, old_vfs;
3426
3427         /* Some old versions of BE3 FW don't report max_vfs value */
3428         be_cmd_get_profile_config(adapter, &res, 0);
3429
3430         if (BE3_chip(adapter) && !res.max_vfs) {
3431                 max_vfs = pci_sriov_get_totalvfs(adapter->pdev);
3432                 res.max_vfs = max_vfs > 0 ? min(MAX_VFS, max_vfs) : 0;
3433         }
3434
3435         adapter->pool_res = res;
3436
3437         if (!be_max_vfs(adapter)) {
3438                 if (num_vfs)
3439                         dev_warn(dev, "SRIOV is disabled. Ignoring num_vfs\n");
3440                 adapter->num_vfs = 0;
3441                 return 0;
3442         }
3443
3444         pci_sriov_set_totalvfs(adapter->pdev, be_max_vfs(adapter));
3445
3446         /* validate num_vfs module param */
3447         old_vfs = pci_num_vf(adapter->pdev);
3448         if (old_vfs) {
3449                 dev_info(dev, "%d VFs are already enabled\n", old_vfs);
3450                 if (old_vfs != num_vfs)
3451                         dev_warn(dev, "Ignoring num_vfs=%d setting\n", num_vfs);
3452                 adapter->num_vfs = old_vfs;
3453         } else {
3454                 if (num_vfs > be_max_vfs(adapter)) {
3455                         dev_info(dev, "Resources unavailable to init %d VFs\n",
3456                                  num_vfs);
3457                         dev_info(dev, "Limiting to %d VFs\n",
3458                                  be_max_vfs(adapter));
3459                 }
3460                 adapter->num_vfs = min_t(u16, num_vfs, be_max_vfs(adapter));
3461         }
3462
3463         return 0;
3464 }
3465
3466 static int be_get_resources(struct be_adapter *adapter)
3467 {
3468         struct device *dev = &adapter->pdev->dev;
3469         struct be_resources res = {0};
3470         int status;
3471
3472         if (BEx_chip(adapter)) {
3473                 BEx_get_resources(adapter, &res);
3474                 adapter->res = res;
3475         }
3476
3477         /* For Lancer, SH etc read per-function resource limits from FW.
3478          * GET_FUNC_CONFIG returns per function guaranteed limits.
3479          * GET_PROFILE_CONFIG returns PCI-E related limits PF-pool limits
3480          */
3481         if (!BEx_chip(adapter)) {
3482                 status = be_cmd_get_func_config(adapter, &res);
3483                 if (status)
3484                         return status;
3485
3486                 /* If RoCE may be enabled stash away half the EQs for RoCE */
3487                 if (be_roce_supported(adapter))
3488                         res.max_evt_qs /= 2;
3489                 adapter->res = res;
3490         }
3491
3492         dev_info(dev, "Max: txqs %d, rxqs %d, rss %d, eqs %d, vfs %d\n",
3493                  be_max_txqs(adapter), be_max_rxqs(adapter),
3494                  be_max_rss(adapter), be_max_eqs(adapter),
3495                  be_max_vfs(adapter));
3496         dev_info(dev, "Max: uc-macs %d, mc-macs %d, vlans %d\n",
3497                  be_max_uc(adapter), be_max_mc(adapter),
3498                  be_max_vlans(adapter));
3499
3500         return 0;
3501 }
3502
3503 static void be_sriov_config(struct be_adapter *adapter)
3504 {
3505         struct device *dev = &adapter->pdev->dev;
3506         int status;
3507
3508         status = be_get_sriov_config(adapter);
3509         if (status) {
3510                 dev_err(dev, "Failed to query SR-IOV configuration\n");
3511                 dev_err(dev, "SR-IOV cannot be enabled\n");
3512                 return;
3513         }
3514
3515         /* When the HW is in SRIOV capable configuration, the PF-pool
3516          * resources are equally distributed across the max-number of
3517          * VFs. The user may request only a subset of the max-vfs to be
3518          * enabled. Based on num_vfs, redistribute the resources across
3519          * num_vfs so that each VF will have access to more number of
3520          * resources. This facility is not available in BE3 FW.
3521          * Also, this is done by FW in Lancer chip.
3522          */
3523         if (be_max_vfs(adapter) && !pci_num_vf(adapter->pdev)) {
3524                 status = be_cmd_set_sriov_config(adapter,
3525                                                  adapter->pool_res,
3526                                                  adapter->num_vfs);
3527                 if (status)
3528                         dev_err(dev, "Failed to optimize SR-IOV resources\n");
3529         }
3530 }
3531
3532 static int be_get_config(struct be_adapter *adapter)
3533 {
3534         u16 profile_id;
3535         int status;
3536
3537         status = be_cmd_query_fw_cfg(adapter);
3538         if (status)
3539                 return status;
3540
3541          if (be_physfn(adapter)) {
3542                 status = be_cmd_get_active_profile(adapter, &profile_id);
3543                 if (!status)
3544                         dev_info(&adapter->pdev->dev,
3545                                  "Using profile 0x%x\n", profile_id);
3546         }
3547
3548         if (!BE2_chip(adapter) && be_physfn(adapter))
3549                 be_sriov_config(adapter);
3550
3551         status = be_get_resources(adapter);
3552         if (status)
3553                 return status;
3554
3555         adapter->pmac_id = kcalloc(be_max_uc(adapter),
3556                                    sizeof(*adapter->pmac_id), GFP_KERNEL);
3557         if (!adapter->pmac_id)
3558                 return -ENOMEM;
3559
3560         /* Sanitize cfg_num_qs based on HW and platform limits */
3561         adapter->cfg_num_qs = min(adapter->cfg_num_qs, be_max_qs(adapter));
3562
3563         return 0;
3564 }
3565
3566 static int be_mac_setup(struct be_adapter *adapter)
3567 {
3568         u8 mac[ETH_ALEN];
3569         int status;
3570
3571         if (is_zero_ether_addr(adapter->netdev->dev_addr)) {
3572                 status = be_cmd_get_perm_mac(adapter, mac);
3573                 if (status)
3574                         return status;
3575
3576                 memcpy(adapter->netdev->dev_addr, mac, ETH_ALEN);
3577                 memcpy(adapter->netdev->perm_addr, mac, ETH_ALEN);
3578         } else {
3579                 /* Maybe the HW was reset; dev_addr must be re-programmed */
3580                 memcpy(mac, adapter->netdev->dev_addr, ETH_ALEN);
3581         }
3582
3583         /* For BE3-R VFs, the PF programs the initial MAC address */
3584         if (!(BEx_chip(adapter) && be_virtfn(adapter)))
3585                 be_cmd_pmac_add(adapter, mac, adapter->if_handle,
3586                                 &adapter->pmac_id[0], 0);
3587         return 0;
3588 }
3589
3590 static void be_schedule_worker(struct be_adapter *adapter)
3591 {
3592         schedule_delayed_work(&adapter->work, msecs_to_jiffies(1000));
3593         adapter->flags |= BE_FLAGS_WORKER_SCHEDULED;
3594 }
3595
3596 static int be_setup_queues(struct be_adapter *adapter)
3597 {
3598         struct net_device *netdev = adapter->netdev;
3599         int status;
3600
3601         status = be_evt_queues_create(adapter);
3602         if (status)
3603                 goto err;
3604
3605         status = be_tx_qs_create(adapter);
3606         if (status)
3607                 goto err;
3608
3609         status = be_rx_cqs_create(adapter);
3610         if (status)
3611                 goto err;
3612
3613         status = be_mcc_queues_create(adapter);
3614         if (status)
3615                 goto err;
3616
3617         status = netif_set_real_num_rx_queues(netdev, adapter->num_rx_qs);
3618         if (status)
3619                 goto err;
3620
3621         status = netif_set_real_num_tx_queues(netdev, adapter->num_tx_qs);
3622         if (status)
3623                 goto err;
3624
3625         return 0;
3626 err:
3627         dev_err(&adapter->pdev->dev, "queue_setup failed\n");
3628         return status;
3629 }
3630
3631 int be_update_queues(struct be_adapter *adapter)
3632 {
3633         struct net_device *netdev = adapter->netdev;
3634         int status;
3635
3636         if (netif_running(netdev))
3637                 be_close(netdev);
3638
3639         be_cancel_worker(adapter);
3640
3641         /* If any vectors have been shared with RoCE we cannot re-program
3642          * the MSIx table.
3643          */
3644         if (!adapter->num_msix_roce_vec)
3645                 be_msix_disable(adapter);
3646
3647         be_clear_queues(adapter);
3648
3649         if (!msix_enabled(adapter)) {
3650                 status = be_msix_enable(adapter);
3651                 if (status)
3652                         return status;
3653         }
3654
3655         status = be_setup_queues(adapter);
3656         if (status)
3657                 return status;
3658
3659         be_schedule_worker(adapter);
3660
3661         if (netif_running(netdev))
3662                 status = be_open(netdev);
3663
3664         return status;
3665 }
3666
3667 static int be_setup(struct be_adapter *adapter)
3668 {
3669         struct device *dev = &adapter->pdev->dev;
3670         int status;
3671
3672         be_setup_init(adapter);
3673
3674         if (!lancer_chip(adapter))
3675                 be_cmd_req_native_mode(adapter);
3676
3677         status = be_get_config(adapter);
3678         if (status)
3679                 goto err;
3680
3681         status = be_msix_enable(adapter);
3682         if (status)
3683                 goto err;
3684
3685         status = be_if_create(adapter, &adapter->if_handle,
3686                               be_if_cap_flags(adapter), 0);
3687         if (status)
3688                 goto err;
3689
3690         /* Updating real_num_tx/rx_queues() requires rtnl_lock() */
3691         rtnl_lock();
3692         status = be_setup_queues(adapter);
3693         rtnl_unlock();
3694         if (status)
3695                 goto err;
3696
3697         be_cmd_get_fn_privileges(adapter, &adapter->cmd_privileges, 0);
3698
3699         status = be_mac_setup(adapter);
3700         if (status)
3701                 goto err;
3702
3703         be_cmd_get_fw_ver(adapter);
3704         dev_info(dev, "FW version is %s\n", adapter->fw_ver);
3705
3706         if (BE2_chip(adapter) && fw_major_num(adapter->fw_ver) < 4) {
3707                 dev_err(dev, "Firmware on card is old(%s), IRQs may not work",
3708                         adapter->fw_ver);
3709                 dev_err(dev, "Please upgrade firmware to version >= 4.0\n");
3710         }
3711
3712         if (adapter->vlans_added)
3713                 be_vid_config(adapter);
3714
3715         be_set_rx_mode(adapter->netdev);
3716
3717         be_cmd_get_acpi_wol_cap(adapter);
3718
3719         status = be_cmd_set_flow_control(adapter, adapter->tx_fc,
3720                                          adapter->rx_fc);
3721         if (status)
3722                 be_cmd_get_flow_control(adapter, &adapter->tx_fc,
3723                                         &adapter->rx_fc);
3724
3725         dev_info(&adapter->pdev->dev, "HW Flow control - TX:%d RX:%d\n",
3726                  adapter->tx_fc, adapter->rx_fc);
3727
3728         if (be_physfn(adapter))
3729                 be_cmd_set_logical_link_config(adapter,
3730                                                IFLA_VF_LINK_STATE_AUTO, 0);
3731
3732         if (adapter->num_vfs)
3733                 be_vf_setup(adapter);
3734
3735         status = be_cmd_get_phy_info(adapter);
3736         if (!status && be_pause_supported(adapter))
3737                 adapter->phy.fc_autoneg = 1;
3738
3739         be_schedule_worker(adapter);
3740         adapter->flags |= BE_FLAGS_SETUP_DONE;
3741         return 0;
3742 err:
3743         be_clear(adapter);
3744         return status;
3745 }
3746
3747 #ifdef CONFIG_NET_POLL_CONTROLLER
3748 static void be_netpoll(struct net_device *netdev)
3749 {
3750         struct be_adapter *adapter = netdev_priv(netdev);
3751         struct be_eq_obj *eqo;
3752         int i;
3753
3754         for_all_evt_queues(adapter, eqo, i) {
3755                 be_eq_notify(eqo->adapter, eqo->q.id, false, true, 0);
3756                 napi_schedule(&eqo->napi);
3757         }
3758 }
3759 #endif
3760
3761 static char flash_cookie[2][16] = {"*** SE FLAS", "H DIRECTORY *** "};
3762
3763 static bool phy_flashing_required(struct be_adapter *adapter)
3764 {
3765         return (adapter->phy.phy_type == PHY_TYPE_TN_8022 &&
3766                 adapter->phy.interface_type == PHY_TYPE_BASET_10GB);
3767 }
3768
3769 static bool is_comp_in_ufi(struct be_adapter *adapter,
3770                            struct flash_section_info *fsec, int type)
3771 {
3772         int i = 0, img_type = 0;
3773         struct flash_section_info_g2 *fsec_g2 = NULL;
3774
3775         if (BE2_chip(adapter))
3776                 fsec_g2 = (struct flash_section_info_g2 *)fsec;
3777
3778         for (i = 0; i < MAX_FLASH_COMP; i++) {
3779                 if (fsec_g2)
3780                         img_type = le32_to_cpu(fsec_g2->fsec_entry[i].type);
3781                 else
3782                         img_type = le32_to_cpu(fsec->fsec_entry[i].type);
3783
3784                 if (img_type == type)
3785                         return true;
3786         }
3787         return false;
3788
3789 }
3790
3791 static struct flash_section_info *get_fsec_info(struct be_adapter *adapter,
3792                                                 int header_size,
3793                                                 const struct firmware *fw)
3794 {
3795         struct flash_section_info *fsec = NULL;
3796         const u8 *p = fw->data;
3797
3798         p += header_size;
3799         while (p < (fw->data + fw->size)) {
3800                 fsec = (struct flash_section_info *)p;
3801                 if (!memcmp(flash_cookie, fsec->cookie, sizeof(flash_cookie)))
3802                         return fsec;
3803                 p += 32;
3804         }
3805         return NULL;
3806 }
3807
3808 static int be_check_flash_crc(struct be_adapter *adapter, const u8 *p,
3809                               u32 img_offset, u32 img_size, int hdr_size,
3810                               u16 img_optype, bool *crc_match)
3811 {
3812         u32 crc_offset;
3813         int status;
3814         u8 crc[4];
3815
3816         status = be_cmd_get_flash_crc(adapter, crc, img_optype, img_size - 4);
3817         if (status)
3818                 return status;
3819
3820         crc_offset = hdr_size + img_offset + img_size - 4;
3821
3822         /* Skip flashing, if crc of flashed region matches */
3823         if (!memcmp(crc, p + crc_offset, 4))
3824                 *crc_match = true;
3825         else
3826                 *crc_match = false;
3827
3828         return status;
3829 }
3830
3831 static int be_flash(struct be_adapter *adapter, const u8 *img,
3832                     struct be_dma_mem *flash_cmd, int optype, int img_size)
3833 {
3834         struct be_cmd_write_flashrom *req = flash_cmd->va;
3835         u32 total_bytes, flash_op, num_bytes;
3836         int status;
3837
3838         total_bytes = img_size;
3839         while (total_bytes) {
3840                 num_bytes = min_t(u32, 32*1024, total_bytes);
3841
3842                 total_bytes -= num_bytes;
3843
3844                 if (!total_bytes) {
3845                         if (optype == OPTYPE_PHY_FW)
3846                                 flash_op = FLASHROM_OPER_PHY_FLASH;
3847                         else
3848                                 flash_op = FLASHROM_OPER_FLASH;
3849                 } else {
3850                         if (optype == OPTYPE_PHY_FW)
3851                                 flash_op = FLASHROM_OPER_PHY_SAVE;
3852                         else
3853                                 flash_op = FLASHROM_OPER_SAVE;
3854                 }
3855
3856                 memcpy(req->data_buf, img, num_bytes);
3857                 img += num_bytes;
3858                 status = be_cmd_write_flashrom(adapter, flash_cmd, optype,
3859                                                flash_op, num_bytes);
3860                 if (base_status(status) == MCC_STATUS_ILLEGAL_REQUEST &&
3861                     optype == OPTYPE_PHY_FW)
3862                         break;
3863                 else if (status)
3864                         return status;
3865         }
3866         return 0;
3867 }
3868
3869 /* For BE2, BE3 and BE3-R */
3870 static int be_flash_BEx(struct be_adapter *adapter,
3871                         const struct firmware *fw,
3872                         struct be_dma_mem *flash_cmd, int num_of_images)
3873 {
3874         int img_hdrs_size = (num_of_images * sizeof(struct image_hdr));
3875         struct device *dev = &adapter->pdev->dev;
3876         struct flash_section_info *fsec = NULL;
3877         int status, i, filehdr_size, num_comp;
3878         const struct flash_comp *pflashcomp;
3879         bool crc_match;
3880         const u8 *p;
3881
3882         struct flash_comp gen3_flash_types[] = {
3883                 { FLASH_iSCSI_PRIMARY_IMAGE_START_g3, OPTYPE_ISCSI_ACTIVE,
3884                         FLASH_IMAGE_MAX_SIZE_g3, IMAGE_FIRMWARE_iSCSI},
3885                 { FLASH_REDBOOT_START_g3, OPTYPE_REDBOOT,
3886                         FLASH_REDBOOT_IMAGE_MAX_SIZE_g3, IMAGE_BOOT_CODE},
3887                 { FLASH_iSCSI_BIOS_START_g3, OPTYPE_BIOS,
3888                         FLASH_BIOS_IMAGE_MAX_SIZE_g3, IMAGE_OPTION_ROM_ISCSI},
3889                 { FLASH_PXE_BIOS_START_g3, OPTYPE_PXE_BIOS,
3890                         FLASH_BIOS_IMAGE_MAX_SIZE_g3, IMAGE_OPTION_ROM_PXE},
3891                 { FLASH_FCoE_BIOS_START_g3, OPTYPE_FCOE_BIOS,
3892                         FLASH_BIOS_IMAGE_MAX_SIZE_g3, IMAGE_OPTION_ROM_FCoE},
3893                 { FLASH_iSCSI_BACKUP_IMAGE_START_g3, OPTYPE_ISCSI_BACKUP,
3894                         FLASH_IMAGE_MAX_SIZE_g3, IMAGE_FIRMWARE_BACKUP_iSCSI},
3895                 { FLASH_FCoE_PRIMARY_IMAGE_START_g3, OPTYPE_FCOE_FW_ACTIVE,
3896                         FLASH_IMAGE_MAX_SIZE_g3, IMAGE_FIRMWARE_FCoE},
3897                 { FLASH_FCoE_BACKUP_IMAGE_START_g3, OPTYPE_FCOE_FW_BACKUP,
3898                         FLASH_IMAGE_MAX_SIZE_g3, IMAGE_FIRMWARE_BACKUP_FCoE},
3899                 { FLASH_NCSI_START_g3, OPTYPE_NCSI_FW,
3900                         FLASH_NCSI_IMAGE_MAX_SIZE_g3, IMAGE_NCSI},
3901                 { FLASH_PHY_FW_START_g3, OPTYPE_PHY_FW,
3902                         FLASH_PHY_FW_IMAGE_MAX_SIZE_g3, IMAGE_FIRMWARE_PHY}
3903         };
3904
3905         struct flash_comp gen2_flash_types[] = {
3906                 { FLASH_iSCSI_PRIMARY_IMAGE_START_g2, OPTYPE_ISCSI_ACTIVE,
3907                         FLASH_IMAGE_MAX_SIZE_g2, IMAGE_FIRMWARE_iSCSI},
3908                 { FLASH_REDBOOT_START_g2, OPTYPE_REDBOOT,
3909                         FLASH_REDBOOT_IMAGE_MAX_SIZE_g2, IMAGE_BOOT_CODE},
3910                 { FLASH_iSCSI_BIOS_START_g2, OPTYPE_BIOS,
3911                         FLASH_BIOS_IMAGE_MAX_SIZE_g2, IMAGE_OPTION_ROM_ISCSI},
3912                 { FLASH_PXE_BIOS_START_g2, OPTYPE_PXE_BIOS,
3913                         FLASH_BIOS_IMAGE_MAX_SIZE_g2, IMAGE_OPTION_ROM_PXE},
3914                 { FLASH_FCoE_BIOS_START_g2, OPTYPE_FCOE_BIOS,
3915                         FLASH_BIOS_IMAGE_MAX_SIZE_g2, IMAGE_OPTION_ROM_FCoE},
3916                 { FLASH_iSCSI_BACKUP_IMAGE_START_g2, OPTYPE_ISCSI_BACKUP,
3917                         FLASH_IMAGE_MAX_SIZE_g2, IMAGE_FIRMWARE_BACKUP_iSCSI},
3918                 { FLASH_FCoE_PRIMARY_IMAGE_START_g2, OPTYPE_FCOE_FW_ACTIVE,
3919                         FLASH_IMAGE_MAX_SIZE_g2, IMAGE_FIRMWARE_FCoE},
3920                 { FLASH_FCoE_BACKUP_IMAGE_START_g2, OPTYPE_FCOE_FW_BACKUP,
3921                          FLASH_IMAGE_MAX_SIZE_g2, IMAGE_FIRMWARE_BACKUP_FCoE}
3922         };
3923
3924         if (BE3_chip(adapter)) {
3925                 pflashcomp = gen3_flash_types;
3926                 filehdr_size = sizeof(struct flash_file_hdr_g3);
3927                 num_comp = ARRAY_SIZE(gen3_flash_types);
3928         } else {
3929                 pflashcomp = gen2_flash_types;
3930                 filehdr_size = sizeof(struct flash_file_hdr_g2);
3931                 num_comp = ARRAY_SIZE(gen2_flash_types);
3932         }
3933
3934         /* Get flash section info*/
3935         fsec = get_fsec_info(adapter, filehdr_size + img_hdrs_size, fw);
3936         if (!fsec) {
3937                 dev_err(dev, "Invalid Cookie. FW image may be corrupted\n");
3938                 return -1;
3939         }
3940         for (i = 0; i < num_comp; i++) {
3941                 if (!is_comp_in_ufi(adapter, fsec, pflashcomp[i].img_type))
3942                         continue;
3943
3944                 if ((pflashcomp[i].optype == OPTYPE_NCSI_FW) &&
3945                     memcmp(adapter->fw_ver, "3.102.148.0", 11) < 0)
3946                         continue;
3947
3948                 if (pflashcomp[i].optype == OPTYPE_PHY_FW  &&
3949                     !phy_flashing_required(adapter))
3950                                 continue;
3951
3952                 if (pflashcomp[i].optype == OPTYPE_REDBOOT) {
3953                         status = be_check_flash_crc(adapter, fw->data,
3954                                                     pflashcomp[i].offset,
3955                                                     pflashcomp[i].size,
3956                                                     filehdr_size +
3957                                                     img_hdrs_size,
3958                                                     OPTYPE_REDBOOT, &crc_match);
3959                         if (status) {
3960                                 dev_err(dev,
3961                                         "Could not get CRC for 0x%x region\n",
3962                                         pflashcomp[i].optype);
3963                                 continue;
3964                         }
3965
3966                         if (crc_match)
3967                                 continue;
3968                 }
3969
3970                 p = fw->data + filehdr_size + pflashcomp[i].offset +
3971                         img_hdrs_size;
3972                 if (p + pflashcomp[i].size > fw->data + fw->size)
3973                         return -1;
3974
3975                 status = be_flash(adapter, p, flash_cmd, pflashcomp[i].optype,
3976                                   pflashcomp[i].size);
3977                 if (status) {
3978                         dev_err(dev, "Flashing section type 0x%x failed\n",
3979                                 pflashcomp[i].img_type);
3980                         return status;
3981                 }
3982         }
3983         return 0;
3984 }
3985
3986 static u16 be_get_img_optype(struct flash_section_entry fsec_entry)
3987 {
3988         u32 img_type = le32_to_cpu(fsec_entry.type);
3989         u16 img_optype = le16_to_cpu(fsec_entry.optype);
3990
3991         if (img_optype != 0xFFFF)
3992                 return img_optype;
3993
3994         switch (img_type) {
3995         case IMAGE_FIRMWARE_iSCSI:
3996                 img_optype = OPTYPE_ISCSI_ACTIVE;
3997                 break;
3998         case IMAGE_BOOT_CODE:
3999                 img_optype = OPTYPE_REDBOOT;
4000                 break;
4001         case IMAGE_OPTION_ROM_ISCSI:
4002                 img_optype = OPTYPE_BIOS;
4003                 break;
4004         case IMAGE_OPTION_ROM_PXE:
4005                 img_optype = OPTYPE_PXE_BIOS;
4006                 break;
4007         case IMAGE_OPTION_ROM_FCoE:
4008                 img_optype = OPTYPE_FCOE_BIOS;
4009                 break;
4010         case IMAGE_FIRMWARE_BACKUP_iSCSI:
4011                 img_optype = OPTYPE_ISCSI_BACKUP;
4012                 break;
4013         case IMAGE_NCSI:
4014                 img_optype = OPTYPE_NCSI_FW;
4015                 break;
4016         case IMAGE_FLASHISM_JUMPVECTOR:
4017                 img_optype = OPTYPE_FLASHISM_JUMPVECTOR;
4018                 break;
4019         case IMAGE_FIRMWARE_PHY:
4020                 img_optype = OPTYPE_SH_PHY_FW;
4021                 break;
4022         case IMAGE_REDBOOT_DIR:
4023                 img_optype = OPTYPE_REDBOOT_DIR;
4024                 break;
4025         case IMAGE_REDBOOT_CONFIG:
4026                 img_optype = OPTYPE_REDBOOT_CONFIG;
4027                 break;
4028         case IMAGE_UFI_DIR:
4029                 img_optype = OPTYPE_UFI_DIR;
4030                 break;
4031         default:
4032                 break;
4033         }
4034
4035         return img_optype;
4036 }
4037
4038 static int be_flash_skyhawk(struct be_adapter *adapter,
4039                             const struct firmware *fw,
4040                             struct be_dma_mem *flash_cmd, int num_of_images)
4041 {
4042         int img_hdrs_size = num_of_images * sizeof(struct image_hdr);
4043         struct device *dev = &adapter->pdev->dev;
4044         struct flash_section_info *fsec = NULL;
4045         u32 img_offset, img_size, img_type;
4046         int status, i, filehdr_size;
4047         bool crc_match, old_fw_img;
4048         u16 img_optype;
4049         const u8 *p;
4050
4051         filehdr_size = sizeof(struct flash_file_hdr_g3);
4052         fsec = get_fsec_info(adapter, filehdr_size + img_hdrs_size, fw);
4053         if (!fsec) {
4054                 dev_err(dev, "Invalid Cookie. FW image may be corrupted\n");
4055                 return -EINVAL;
4056         }
4057
4058         for (i = 0; i < le32_to_cpu(fsec->fsec_hdr.num_images); i++) {
4059                 img_offset = le32_to_cpu(fsec->fsec_entry[i].offset);
4060                 img_size   = le32_to_cpu(fsec->fsec_entry[i].pad_size);
4061                 img_type   = le32_to_cpu(fsec->fsec_entry[i].type);
4062                 img_optype = be_get_img_optype(fsec->fsec_entry[i]);
4063                 old_fw_img = fsec->fsec_entry[i].optype == 0xFFFF;
4064
4065                 if (img_optype == 0xFFFF)
4066                         continue;
4067                 /* Don't bother verifying CRC if an old FW image is being
4068                  * flashed
4069                  */
4070                 if (old_fw_img)
4071                         goto flash;
4072
4073                 status = be_check_flash_crc(adapter, fw->data, img_offset,
4074                                             img_size, filehdr_size +
4075                                             img_hdrs_size, img_optype,
4076                                             &crc_match);
4077                 /* The current FW image on the card does not recognize the new
4078                  * FLASH op_type. The FW download is partially complete.
4079                  * Reboot the server now to enable FW image to recognize the
4080                  * new FLASH op_type. To complete the remaining process,
4081                  * download the same FW again after the reboot.
4082                  */
4083                 if (base_status(status) == MCC_STATUS_ILLEGAL_REQUEST ||
4084                     base_status(status) == MCC_STATUS_ILLEGAL_FIELD) {
4085                         dev_err(dev, "Flash incomplete. Reset the server\n");
4086                         dev_err(dev, "Download FW image again after reset\n");
4087                         return -EAGAIN;
4088                 } else if (status) {
4089                         dev_err(dev, "Could not get CRC for 0x%x region\n",
4090                                 img_optype);
4091                         return -EFAULT;
4092                 }
4093
4094                 if (crc_match)
4095                         continue;
4096
4097 flash:
4098                 p = fw->data + filehdr_size + img_offset + img_hdrs_size;
4099                 if (p + img_size > fw->data + fw->size)
4100                         return -1;
4101
4102                 status = be_flash(adapter, p, flash_cmd, img_optype, img_size);
4103                 /* For old FW images ignore ILLEGAL_FIELD error or errors on
4104                  * UFI_DIR region
4105                  */
4106                 if (old_fw_img &&
4107                     (base_status(status) == MCC_STATUS_ILLEGAL_FIELD ||
4108                      (img_optype == OPTYPE_UFI_DIR &&
4109                       base_status(status) == MCC_STATUS_FAILED))) {
4110                         continue;
4111                 } else if (status) {
4112                         dev_err(dev, "Flashing section type 0x%x failed\n",
4113                                 img_type);
4114                         return -EFAULT;
4115                 }
4116         }
4117         return 0;
4118 }
4119
4120 static int lancer_fw_download(struct be_adapter *adapter,
4121                               const struct firmware *fw)
4122 {
4123 #define LANCER_FW_DOWNLOAD_CHUNK      (32 * 1024)
4124 #define LANCER_FW_DOWNLOAD_LOCATION   "/prg"
4125         struct device *dev = &adapter->pdev->dev;
4126         struct be_dma_mem flash_cmd;
4127         const u8 *data_ptr = NULL;
4128         u8 *dest_image_ptr = NULL;
4129         size_t image_size = 0;
4130         u32 chunk_size = 0;
4131         u32 data_written = 0;
4132         u32 offset = 0;
4133         int status = 0;
4134         u8 add_status = 0;
4135         u8 change_status;
4136
4137         if (!IS_ALIGNED(fw->size, sizeof(u32))) {
4138                 dev_err(dev, "FW image size should be multiple of 4\n");
4139                 return -EINVAL;
4140         }
4141
4142         flash_cmd.size = sizeof(struct lancer_cmd_req_write_object)
4143                                 + LANCER_FW_DOWNLOAD_CHUNK;
4144         flash_cmd.va = dma_alloc_coherent(dev, flash_cmd.size,
4145                                           &flash_cmd.dma, GFP_KERNEL);
4146         if (!flash_cmd.va)
4147                 return -ENOMEM;
4148
4149         dest_image_ptr = flash_cmd.va +
4150                                 sizeof(struct lancer_cmd_req_write_object);
4151         image_size = fw->size;
4152         data_ptr = fw->data;
4153
4154         while (image_size) {
4155                 chunk_size = min_t(u32, image_size, LANCER_FW_DOWNLOAD_CHUNK);
4156
4157                 /* Copy the image chunk content. */
4158                 memcpy(dest_image_ptr, data_ptr, chunk_size);
4159
4160                 status = lancer_cmd_write_object(adapter, &flash_cmd,
4161                                                  chunk_size, offset,
4162                                                  LANCER_FW_DOWNLOAD_LOCATION,
4163                                                  &data_written, &change_status,
4164                                                  &add_status);
4165                 if (status)
4166                         break;
4167
4168                 offset += data_written;
4169                 data_ptr += data_written;
4170                 image_size -= data_written;
4171         }
4172
4173         if (!status) {
4174                 /* Commit the FW written */
4175                 status = lancer_cmd_write_object(adapter, &flash_cmd,
4176                                                  0, offset,
4177                                                  LANCER_FW_DOWNLOAD_LOCATION,
4178                                                  &data_written, &change_status,
4179                                                  &add_status);
4180         }
4181
4182         dma_free_coherent(dev, flash_cmd.size, flash_cmd.va, flash_cmd.dma);
4183         if (status) {
4184                 dev_err(dev, "Firmware load error\n");
4185                 return be_cmd_status(status);
4186         }
4187
4188         dev_info(dev, "Firmware flashed successfully\n");
4189
4190         if (change_status == LANCER_FW_RESET_NEEDED) {
4191                 dev_info(dev, "Resetting adapter to activate new FW\n");
4192                 status = lancer_physdev_ctrl(adapter,
4193                                              PHYSDEV_CONTROL_FW_RESET_MASK);
4194                 if (status) {
4195                         dev_err(dev, "Adapter busy, could not reset FW\n");
4196                         dev_err(dev, "Reboot server to activate new FW\n");
4197                 }
4198         } else if (change_status != LANCER_NO_RESET_NEEDED) {
4199                 dev_info(dev, "Reboot server to activate new FW\n");
4200         }
4201
4202         return 0;
4203 }
4204
4205 #define UFI_TYPE2               2
4206 #define UFI_TYPE3               3
4207 #define UFI_TYPE3R              10
4208 #define UFI_TYPE4               4
4209 static int be_get_ufi_type(struct be_adapter *adapter,
4210                            struct flash_file_hdr_g3 *fhdr)
4211 {
4212         if (!fhdr)
4213                 goto be_get_ufi_exit;
4214
4215         if (skyhawk_chip(adapter) && fhdr->build[0] == '4')
4216                 return UFI_TYPE4;
4217         else if (BE3_chip(adapter) && fhdr->build[0] == '3') {
4218                 if (fhdr->asic_type_rev == 0x10)
4219                         return UFI_TYPE3R;
4220                 else
4221                         return UFI_TYPE3;
4222         } else if (BE2_chip(adapter) && fhdr->build[0] == '2')
4223                 return UFI_TYPE2;
4224
4225 be_get_ufi_exit:
4226         dev_err(&adapter->pdev->dev,
4227                 "UFI and Interface are not compatible for flashing\n");
4228         return -1;
4229 }
4230
4231 static int be_fw_download(struct be_adapter *adapter, const struct firmware* fw)
4232 {
4233         struct flash_file_hdr_g3 *fhdr3;
4234         struct image_hdr *img_hdr_ptr = NULL;
4235         struct be_dma_mem flash_cmd;
4236         const u8 *p;
4237         int status = 0, i = 0, num_imgs = 0, ufi_type = 0;
4238
4239         flash_cmd.size = sizeof(struct be_cmd_write_flashrom);
4240         flash_cmd.va = dma_alloc_coherent(&adapter->pdev->dev, flash_cmd.size,
4241                                           &flash_cmd.dma, GFP_KERNEL);
4242         if (!flash_cmd.va) {
4243                 status = -ENOMEM;
4244                 goto be_fw_exit;
4245         }
4246
4247         p = fw->data;
4248         fhdr3 = (struct flash_file_hdr_g3 *)p;
4249
4250         ufi_type = be_get_ufi_type(adapter, fhdr3);
4251
4252         num_imgs = le32_to_cpu(fhdr3->num_imgs);
4253         for (i = 0; i < num_imgs; i++) {
4254                 img_hdr_ptr = (struct image_hdr *)(fw->data +
4255                                 (sizeof(struct flash_file_hdr_g3) +
4256                                  i * sizeof(struct image_hdr)));
4257                 if (le32_to_cpu(img_hdr_ptr->imageid) == 1) {
4258                         switch (ufi_type) {
4259                         case UFI_TYPE4:
4260                                 status = be_flash_skyhawk(adapter, fw,
4261                                                           &flash_cmd, num_imgs);
4262                                 break;
4263                         case UFI_TYPE3R:
4264                                 status = be_flash_BEx(adapter, fw, &flash_cmd,
4265                                                       num_imgs);
4266                                 break;
4267                         case UFI_TYPE3:
4268                                 /* Do not flash this ufi on BE3-R cards */
4269                                 if (adapter->asic_rev < 0x10)
4270                                         status = be_flash_BEx(adapter, fw,
4271                                                               &flash_cmd,
4272                                                               num_imgs);
4273                                 else {
4274                                         status = -EINVAL;
4275                                         dev_err(&adapter->pdev->dev,
4276                                                 "Can't load BE3 UFI on BE3R\n");
4277                                 }
4278                         }
4279                 }
4280         }
4281
4282         if (ufi_type == UFI_TYPE2)
4283                 status = be_flash_BEx(adapter, fw, &flash_cmd, 0);
4284         else if (ufi_type == -1)
4285                 status = -EINVAL;
4286
4287         dma_free_coherent(&adapter->pdev->dev, flash_cmd.size, flash_cmd.va,
4288                           flash_cmd.dma);
4289         if (status) {
4290                 dev_err(&adapter->pdev->dev, "Firmware load error\n");
4291                 goto be_fw_exit;
4292         }
4293
4294         dev_info(&adapter->pdev->dev, "Firmware flashed successfully\n");
4295
4296 be_fw_exit:
4297         return status;
4298 }
4299
4300 int be_load_fw(struct be_adapter *adapter, u8 *fw_file)
4301 {
4302         const struct firmware *fw;
4303         int status;
4304
4305         if (!netif_running(adapter->netdev)) {
4306                 dev_err(&adapter->pdev->dev,
4307                         "Firmware load not allowed (interface is down)\n");
4308                 return -ENETDOWN;
4309         }
4310
4311         status = request_firmware(&fw, fw_file, &adapter->pdev->dev);
4312         if (status)
4313                 goto fw_exit;
4314
4315         dev_info(&adapter->pdev->dev, "Flashing firmware file %s\n", fw_file);
4316
4317         if (lancer_chip(adapter))
4318                 status = lancer_fw_download(adapter, fw);
4319         else
4320                 status = be_fw_download(adapter, fw);
4321
4322         if (!status)
4323                 be_cmd_get_fw_ver(adapter);
4324
4325 fw_exit:
4326         release_firmware(fw);
4327         return status;
4328 }
4329
4330 static int be_ndo_bridge_setlink(struct net_device *dev, struct nlmsghdr *nlh,
4331                                  u16 flags)
4332 {
4333         struct be_adapter *adapter = netdev_priv(dev);
4334         struct nlattr *attr, *br_spec;
4335         int rem;
4336         int status = 0;
4337         u16 mode = 0;
4338
4339         if (!sriov_enabled(adapter))
4340                 return -EOPNOTSUPP;
4341
4342         br_spec = nlmsg_find_attr(nlh, sizeof(struct ifinfomsg), IFLA_AF_SPEC);
4343         if (!br_spec)
4344                 return -EINVAL;
4345
4346         nla_for_each_nested(attr, br_spec, rem) {
4347                 if (nla_type(attr) != IFLA_BRIDGE_MODE)
4348                         continue;
4349
4350                 if (nla_len(attr) < sizeof(mode))
4351                         return -EINVAL;
4352
4353                 mode = nla_get_u16(attr);
4354                 if (mode != BRIDGE_MODE_VEPA && mode != BRIDGE_MODE_VEB)
4355                         return -EINVAL;
4356
4357                 status = be_cmd_set_hsw_config(adapter, 0, 0,
4358                                                adapter->if_handle,
4359                                                mode == BRIDGE_MODE_VEPA ?
4360                                                PORT_FWD_TYPE_VEPA :
4361                                                PORT_FWD_TYPE_VEB);
4362                 if (status)
4363                         goto err;
4364
4365                 dev_info(&adapter->pdev->dev, "enabled switch mode: %s\n",
4366                          mode == BRIDGE_MODE_VEPA ? "VEPA" : "VEB");
4367
4368                 return status;
4369         }
4370 err:
4371         dev_err(&adapter->pdev->dev, "Failed to set switch mode %s\n",
4372                 mode == BRIDGE_MODE_VEPA ? "VEPA" : "VEB");
4373
4374         return status;
4375 }
4376
4377 static int be_ndo_bridge_getlink(struct sk_buff *skb, u32 pid, u32 seq,
4378                                  struct net_device *dev, u32 filter_mask)
4379 {
4380         struct be_adapter *adapter = netdev_priv(dev);
4381         int status = 0;
4382         u8 hsw_mode;
4383
4384         if (!sriov_enabled(adapter))
4385                 return 0;
4386
4387         /* BE and Lancer chips support VEB mode only */
4388         if (BEx_chip(adapter) || lancer_chip(adapter)) {
4389                 hsw_mode = PORT_FWD_TYPE_VEB;
4390         } else {
4391                 status = be_cmd_get_hsw_config(adapter, NULL, 0,
4392                                                adapter->if_handle, &hsw_mode);
4393                 if (status)
4394                         return 0;
4395         }
4396
4397         return ndo_dflt_bridge_getlink(skb, pid, seq, dev,
4398                                        hsw_mode == PORT_FWD_TYPE_VEPA ?
4399                                        BRIDGE_MODE_VEPA : BRIDGE_MODE_VEB,
4400                                        0, 0);
4401 }
4402
4403 #ifdef CONFIG_BE2NET_VXLAN
4404 /* VxLAN offload Notes:
4405  *
4406  * The stack defines tunnel offload flags (hw_enc_features) for IP and doesn't
4407  * distinguish various types of transports (VxLAN, GRE, NVGRE ..). So, offload
4408  * is expected to work across all types of IP tunnels once exported. Skyhawk
4409  * supports offloads for either VxLAN or NVGRE, exclusively. So we export VxLAN
4410  * offloads in hw_enc_features only when a VxLAN port is added. If other (non
4411  * VxLAN) tunnels are configured while VxLAN offloads are enabled, offloads for
4412  * those other tunnels are unexported on the fly through ndo_features_check().
4413  *
4414  * Skyhawk supports VxLAN offloads only for one UDP dport. So, if the stack
4415  * adds more than one port, disable offloads and don't re-enable them again
4416  * until after all the tunnels are removed.
4417  */
4418 static void be_add_vxlan_port(struct net_device *netdev, sa_family_t sa_family,
4419                               __be16 port)
4420 {
4421         struct be_adapter *adapter = netdev_priv(netdev);
4422         struct device *dev = &adapter->pdev->dev;
4423         int status;
4424
4425         if (lancer_chip(adapter) || BEx_chip(adapter))
4426                 return;
4427
4428         if (adapter->flags & BE_FLAGS_VXLAN_OFFLOADS) {
4429                 dev_info(dev,
4430                          "Only one UDP port supported for VxLAN offloads\n");
4431                 dev_info(dev, "Disabling VxLAN offloads\n");
4432                 adapter->vxlan_port_count++;
4433                 goto err;
4434         }
4435
4436         if (adapter->vxlan_port_count++ >= 1)
4437                 return;
4438
4439         status = be_cmd_manage_iface(adapter, adapter->if_handle,
4440                                      OP_CONVERT_NORMAL_TO_TUNNEL);
4441         if (status) {
4442                 dev_warn(dev, "Failed to convert normal interface to tunnel\n");
4443                 goto err;
4444         }
4445
4446         status = be_cmd_set_vxlan_port(adapter, port);
4447         if (status) {
4448                 dev_warn(dev, "Failed to add VxLAN port\n");
4449                 goto err;
4450         }
4451         adapter->flags |= BE_FLAGS_VXLAN_OFFLOADS;
4452         adapter->vxlan_port = port;
4453
4454         netdev->hw_enc_features |= NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |
4455                                    NETIF_F_TSO | NETIF_F_TSO6 |
4456                                    NETIF_F_GSO_UDP_TUNNEL;
4457         netdev->hw_features |= NETIF_F_GSO_UDP_TUNNEL;
4458         netdev->features |= NETIF_F_GSO_UDP_TUNNEL;
4459
4460         dev_info(dev, "Enabled VxLAN offloads for UDP port %d\n",
4461                  be16_to_cpu(port));
4462         return;
4463 err:
4464         be_disable_vxlan_offloads(adapter);
4465 }
4466
4467 static void be_del_vxlan_port(struct net_device *netdev, sa_family_t sa_family,
4468                               __be16 port)
4469 {
4470         struct be_adapter *adapter = netdev_priv(netdev);
4471
4472         if (lancer_chip(adapter) || BEx_chip(adapter))
4473                 return;
4474
4475         if (adapter->vxlan_port != port)
4476                 goto done;
4477
4478         be_disable_vxlan_offloads(adapter);
4479
4480         dev_info(&adapter->pdev->dev,
4481                  "Disabled VxLAN offloads for UDP port %d\n",
4482                  be16_to_cpu(port));
4483 done:
4484         adapter->vxlan_port_count--;
4485 }
4486
4487 static netdev_features_t be_features_check(struct sk_buff *skb,
4488                                            struct net_device *dev,
4489                                            netdev_features_t features)
4490 {
4491         struct be_adapter *adapter = netdev_priv(dev);
4492         u8 l4_hdr = 0;
4493
4494         /* The code below restricts offload features for some tunneled packets.
4495          * Offload features for normal (non tunnel) packets are unchanged.
4496          */
4497         if (!skb->encapsulation ||
4498             !(adapter->flags & BE_FLAGS_VXLAN_OFFLOADS))
4499                 return features;
4500
4501         /* It's an encapsulated packet and VxLAN offloads are enabled. We
4502          * should disable tunnel offload features if it's not a VxLAN packet,
4503          * as tunnel offloads have been enabled only for VxLAN. This is done to
4504          * allow other tunneled traffic like GRE work fine while VxLAN
4505          * offloads are configured in Skyhawk-R.
4506          */
4507         switch (vlan_get_protocol(skb)) {
4508         case htons(ETH_P_IP):
4509                 l4_hdr = ip_hdr(skb)->protocol;
4510                 break;
4511         case htons(ETH_P_IPV6):
4512                 l4_hdr = ipv6_hdr(skb)->nexthdr;
4513                 break;
4514         default:
4515                 return features;
4516         }
4517
4518         if (l4_hdr != IPPROTO_UDP ||
4519             skb->inner_protocol_type != ENCAP_TYPE_ETHER ||
4520             skb->inner_protocol != htons(ETH_P_TEB) ||
4521             skb_inner_mac_header(skb) - skb_transport_header(skb) !=
4522             sizeof(struct udphdr) + sizeof(struct vxlanhdr))
4523                 return features & ~(NETIF_F_ALL_CSUM | NETIF_F_GSO_MASK);
4524
4525         return features;
4526 }
4527 #endif
4528
4529 static const struct net_device_ops be_netdev_ops = {
4530         .ndo_open               = be_open,
4531         .ndo_stop               = be_close,
4532         .ndo_start_xmit         = be_xmit,
4533         .ndo_set_rx_mode        = be_set_rx_mode,
4534         .ndo_set_mac_address    = be_mac_addr_set,
4535         .ndo_change_mtu         = be_change_mtu,
4536         .ndo_get_stats64        = be_get_stats64,
4537         .ndo_validate_addr      = eth_validate_addr,
4538         .ndo_vlan_rx_add_vid    = be_vlan_add_vid,
4539         .ndo_vlan_rx_kill_vid   = be_vlan_rem_vid,
4540         .ndo_set_vf_mac         = be_set_vf_mac,
4541         .ndo_set_vf_vlan        = be_set_vf_vlan,
4542         .ndo_set_vf_rate        = be_set_vf_tx_rate,
4543         .ndo_get_vf_config      = be_get_vf_config,
4544         .ndo_set_vf_link_state  = be_set_vf_link_state,
4545 #ifdef CONFIG_NET_POLL_CONTROLLER
4546         .ndo_poll_controller    = be_netpoll,
4547 #endif
4548         .ndo_bridge_setlink     = be_ndo_bridge_setlink,
4549         .ndo_bridge_getlink     = be_ndo_bridge_getlink,
4550 #ifdef CONFIG_NET_RX_BUSY_POLL
4551         .ndo_busy_poll          = be_busy_poll,
4552 #endif
4553 #ifdef CONFIG_BE2NET_VXLAN
4554         .ndo_add_vxlan_port     = be_add_vxlan_port,
4555         .ndo_del_vxlan_port     = be_del_vxlan_port,
4556         .ndo_features_check     = be_features_check,
4557 #endif
4558 };
4559
4560 static void be_netdev_init(struct net_device *netdev)
4561 {
4562         struct be_adapter *adapter = netdev_priv(netdev);
4563
4564         netdev->hw_features |= NETIF_F_SG | NETIF_F_TSO | NETIF_F_TSO6 |
4565                 NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | NETIF_F_RXCSUM |
4566                 NETIF_F_HW_VLAN_CTAG_TX;
4567         if (be_multi_rxq(adapter))
4568                 netdev->hw_features |= NETIF_F_RXHASH;
4569
4570         netdev->features |= netdev->hw_features |
4571                 NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_CTAG_FILTER;
4572
4573         netdev->vlan_features |= NETIF_F_SG | NETIF_F_TSO | NETIF_F_TSO6 |
4574                 NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM;
4575
4576         netdev->priv_flags |= IFF_UNICAST_FLT;
4577
4578         netdev->flags |= IFF_MULTICAST;
4579
4580         netif_set_gso_max_size(netdev, 65535 - ETH_HLEN);
4581
4582         netdev->netdev_ops = &be_netdev_ops;
4583
4584         netdev->ethtool_ops = &be_ethtool_ops;
4585 }
4586
4587 static void be_unmap_pci_bars(struct be_adapter *adapter)
4588 {
4589         if (adapter->csr)
4590                 pci_iounmap(adapter->pdev, adapter->csr);
4591         if (adapter->db)
4592                 pci_iounmap(adapter->pdev, adapter->db);
4593 }
4594
4595 static int db_bar(struct be_adapter *adapter)
4596 {
4597         if (lancer_chip(adapter) || !be_physfn(adapter))
4598                 return 0;
4599         else
4600                 return 4;
4601 }
4602
4603 static int be_roce_map_pci_bars(struct be_adapter *adapter)
4604 {
4605         if (skyhawk_chip(adapter)) {
4606                 adapter->roce_db.size = 4096;
4607                 adapter->roce_db.io_addr = pci_resource_start(adapter->pdev,
4608                                                               db_bar(adapter));
4609                 adapter->roce_db.total_size = pci_resource_len(adapter->pdev,
4610                                                                db_bar(adapter));
4611         }
4612         return 0;
4613 }
4614
4615 static int be_map_pci_bars(struct be_adapter *adapter)
4616 {
4617         u8 __iomem *addr;
4618
4619         if (BEx_chip(adapter) && be_physfn(adapter)) {
4620                 adapter->csr = pci_iomap(adapter->pdev, 2, 0);
4621                 if (!adapter->csr)
4622                         return -ENOMEM;
4623         }
4624
4625         addr = pci_iomap(adapter->pdev, db_bar(adapter), 0);
4626         if (!addr)
4627                 goto pci_map_err;
4628         adapter->db = addr;
4629
4630         be_roce_map_pci_bars(adapter);
4631         return 0;
4632
4633 pci_map_err:
4634         dev_err(&adapter->pdev->dev, "Error in mapping PCI BARs\n");
4635         be_unmap_pci_bars(adapter);
4636         return -ENOMEM;
4637 }
4638
4639 static void be_ctrl_cleanup(struct be_adapter *adapter)
4640 {
4641         struct be_dma_mem *mem = &adapter->mbox_mem_alloced;
4642
4643         be_unmap_pci_bars(adapter);
4644
4645         if (mem->va)
4646                 dma_free_coherent(&adapter->pdev->dev, mem->size, mem->va,
4647                                   mem->dma);
4648
4649         mem = &adapter->rx_filter;
4650         if (mem->va)
4651                 dma_free_coherent(&adapter->pdev->dev, mem->size, mem->va,
4652                                   mem->dma);
4653 }
4654
4655 static int be_ctrl_init(struct be_adapter *adapter)
4656 {
4657         struct be_dma_mem *mbox_mem_alloc = &adapter->mbox_mem_alloced;
4658         struct be_dma_mem *mbox_mem_align = &adapter->mbox_mem;
4659         struct be_dma_mem *rx_filter = &adapter->rx_filter;
4660         u32 sli_intf;
4661         int status;
4662
4663         pci_read_config_dword(adapter->pdev, SLI_INTF_REG_OFFSET, &sli_intf);
4664         adapter->sli_family = (sli_intf & SLI_INTF_FAMILY_MASK) >>
4665                                  SLI_INTF_FAMILY_SHIFT;
4666         adapter->virtfn = (sli_intf & SLI_INTF_FT_MASK) ? 1 : 0;
4667
4668         status = be_map_pci_bars(adapter);
4669         if (status)
4670                 goto done;
4671
4672         mbox_mem_alloc->size = sizeof(struct be_mcc_mailbox) + 16;
4673         mbox_mem_alloc->va = dma_alloc_coherent(&adapter->pdev->dev,
4674                                                 mbox_mem_alloc->size,
4675                                                 &mbox_mem_alloc->dma,
4676                                                 GFP_KERNEL);
4677         if (!mbox_mem_alloc->va) {
4678                 status = -ENOMEM;
4679                 goto unmap_pci_bars;
4680         }
4681         mbox_mem_align->size = sizeof(struct be_mcc_mailbox);
4682         mbox_mem_align->va = PTR_ALIGN(mbox_mem_alloc->va, 16);
4683         mbox_mem_align->dma = PTR_ALIGN(mbox_mem_alloc->dma, 16);
4684         memset(mbox_mem_align->va, 0, sizeof(struct be_mcc_mailbox));
4685
4686         rx_filter->size = sizeof(struct be_cmd_req_rx_filter);
4687         rx_filter->va = dma_zalloc_coherent(&adapter->pdev->dev,
4688                                             rx_filter->size, &rx_filter->dma,
4689                                             GFP_KERNEL);
4690         if (!rx_filter->va) {
4691                 status = -ENOMEM;
4692                 goto free_mbox;
4693         }
4694
4695         mutex_init(&adapter->mbox_lock);
4696         spin_lock_init(&adapter->mcc_lock);
4697         spin_lock_init(&adapter->mcc_cq_lock);
4698
4699         init_completion(&adapter->et_cmd_compl);
4700         pci_save_state(adapter->pdev);
4701         return 0;
4702
4703 free_mbox:
4704         dma_free_coherent(&adapter->pdev->dev, mbox_mem_alloc->size,
4705                           mbox_mem_alloc->va, mbox_mem_alloc->dma);
4706
4707 unmap_pci_bars:
4708         be_unmap_pci_bars(adapter);
4709
4710 done:
4711         return status;
4712 }
4713
4714 static void be_stats_cleanup(struct be_adapter *adapter)
4715 {
4716         struct be_dma_mem *cmd = &adapter->stats_cmd;
4717
4718         if (cmd->va)
4719                 dma_free_coherent(&adapter->pdev->dev, cmd->size,
4720                                   cmd->va, cmd->dma);
4721 }
4722
4723 static int be_stats_init(struct be_adapter *adapter)
4724 {
4725         struct be_dma_mem *cmd = &adapter->stats_cmd;
4726
4727         if (lancer_chip(adapter))
4728                 cmd->size = sizeof(struct lancer_cmd_req_pport_stats);
4729         else if (BE2_chip(adapter))
4730                 cmd->size = sizeof(struct be_cmd_req_get_stats_v0);
4731         else if (BE3_chip(adapter))
4732                 cmd->size = sizeof(struct be_cmd_req_get_stats_v1);
4733         else
4734                 /* ALL non-BE ASICs */
4735                 cmd->size = sizeof(struct be_cmd_req_get_stats_v2);
4736
4737         cmd->va = dma_zalloc_coherent(&adapter->pdev->dev, cmd->size, &cmd->dma,
4738                                       GFP_KERNEL);
4739         if (!cmd->va)
4740                 return -ENOMEM;
4741         return 0;
4742 }
4743
4744 static void be_remove(struct pci_dev *pdev)
4745 {
4746         struct be_adapter *adapter = pci_get_drvdata(pdev);
4747
4748         if (!adapter)
4749                 return;
4750
4751         be_roce_dev_remove(adapter);
4752         be_intr_set(adapter, false);
4753
4754         cancel_delayed_work_sync(&adapter->func_recovery_work);
4755
4756         unregister_netdev(adapter->netdev);
4757
4758         be_clear(adapter);
4759
4760         /* tell fw we're done with firing cmds */
4761         be_cmd_fw_clean(adapter);
4762
4763         be_stats_cleanup(adapter);
4764
4765         be_ctrl_cleanup(adapter);
4766
4767         pci_disable_pcie_error_reporting(pdev);
4768
4769         pci_release_regions(pdev);
4770         pci_disable_device(pdev);
4771
4772         free_netdev(adapter->netdev);
4773 }
4774
4775 static int be_get_initial_config(struct be_adapter *adapter)
4776 {
4777         int status, level;
4778
4779         status = be_cmd_get_cntl_attributes(adapter);
4780         if (status)
4781                 return status;
4782
4783         /* Must be a power of 2 or else MODULO will BUG_ON */
4784         adapter->be_get_temp_freq = 64;
4785
4786         if (BEx_chip(adapter)) {
4787                 level = be_cmd_get_fw_log_level(adapter);
4788                 adapter->msg_enable =
4789                         level <= FW_LOG_LEVEL_DEFAULT ? NETIF_MSG_HW : 0;
4790         }
4791
4792         adapter->cfg_num_qs = netif_get_num_default_rss_queues();
4793         return 0;
4794 }
4795
4796 static int lancer_recover_func(struct be_adapter *adapter)
4797 {
4798         struct device *dev = &adapter->pdev->dev;
4799         int status;
4800
4801         status = lancer_test_and_set_rdy_state(adapter);
4802         if (status)
4803                 goto err;
4804
4805         if (netif_running(adapter->netdev))
4806                 be_close(adapter->netdev);
4807
4808         be_clear(adapter);
4809
4810         be_clear_all_error(adapter);
4811
4812         status = be_setup(adapter);
4813         if (status)
4814                 goto err;
4815
4816         if (netif_running(adapter->netdev)) {
4817                 status = be_open(adapter->netdev);
4818                 if (status)
4819                         goto err;
4820         }
4821
4822         dev_err(dev, "Adapter recovery successful\n");
4823         return 0;
4824 err:
4825         if (status == -EAGAIN)
4826                 dev_err(dev, "Waiting for resource provisioning\n");
4827         else
4828                 dev_err(dev, "Adapter recovery failed\n");
4829
4830         return status;
4831 }
4832
4833 static void be_func_recovery_task(struct work_struct *work)
4834 {
4835         struct be_adapter *adapter =
4836                 container_of(work, struct be_adapter,  func_recovery_work.work);
4837         int status = 0;
4838
4839         be_detect_error(adapter);
4840
4841         if (adapter->hw_error && lancer_chip(adapter)) {
4842                 rtnl_lock();
4843                 netif_device_detach(adapter->netdev);
4844                 rtnl_unlock();
4845
4846                 status = lancer_recover_func(adapter);
4847                 if (!status)
4848                         netif_device_attach(adapter->netdev);
4849         }
4850
4851         /* In Lancer, for all errors other than provisioning error (-EAGAIN),
4852          * no need to attempt further recovery.
4853          */
4854         if (!status || status == -EAGAIN)
4855                 schedule_delayed_work(&adapter->func_recovery_work,
4856                                       msecs_to_jiffies(1000));
4857 }
4858
4859 static void be_worker(struct work_struct *work)
4860 {
4861         struct be_adapter *adapter =
4862                 container_of(work, struct be_adapter, work.work);
4863         struct be_rx_obj *rxo;
4864         int i;
4865
4866         /* when interrupts are not yet enabled, just reap any pending
4867         * mcc completions */
4868         if (!netif_running(adapter->netdev)) {
4869                 local_bh_disable();
4870                 be_process_mcc(adapter);
4871                 local_bh_enable();
4872                 goto reschedule;
4873         }
4874
4875         if (!adapter->stats_cmd_sent) {
4876                 if (lancer_chip(adapter))
4877                         lancer_cmd_get_pport_stats(adapter,
4878                                                    &adapter->stats_cmd);
4879                 else
4880                         be_cmd_get_stats(adapter, &adapter->stats_cmd);
4881         }
4882
4883         if (be_physfn(adapter) &&
4884             MODULO(adapter->work_counter, adapter->be_get_temp_freq) == 0)
4885                 be_cmd_get_die_temperature(adapter);
4886
4887         for_all_rx_queues(adapter, rxo, i) {
4888                 /* Replenish RX-queues starved due to memory
4889                  * allocation failures.
4890                  */
4891                 if (rxo->rx_post_starved)
4892                         be_post_rx_frags(rxo, GFP_KERNEL, MAX_RX_POST);
4893         }
4894
4895         be_eqd_update(adapter);
4896
4897 reschedule:
4898         adapter->work_counter++;
4899         schedule_delayed_work(&adapter->work, msecs_to_jiffies(1000));
4900 }
4901
4902 /* If any VFs are already enabled don't FLR the PF */
4903 static bool be_reset_required(struct be_adapter *adapter)
4904 {
4905         return pci_num_vf(adapter->pdev) ? false : true;
4906 }
4907
4908 static char *mc_name(struct be_adapter *adapter)
4909 {
4910         char *str = ""; /* default */
4911
4912         switch (adapter->mc_type) {
4913         case UMC:
4914                 str = "UMC";
4915                 break;
4916         case FLEX10:
4917                 str = "FLEX10";
4918                 break;
4919         case vNIC1:
4920                 str = "vNIC-1";
4921                 break;
4922         case nPAR:
4923                 str = "nPAR";
4924                 break;
4925         case UFP:
4926                 str = "UFP";
4927                 break;
4928         case vNIC2:
4929                 str = "vNIC-2";
4930                 break;
4931         default:
4932                 str = "";
4933         }
4934
4935         return str;
4936 }
4937
4938 static inline char *func_name(struct be_adapter *adapter)
4939 {
4940         return be_physfn(adapter) ? "PF" : "VF";
4941 }
4942
4943 static int be_probe(struct pci_dev *pdev, const struct pci_device_id *pdev_id)
4944 {
4945         int status = 0;
4946         struct be_adapter *adapter;
4947         struct net_device *netdev;
4948         char port_name;
4949
4950         dev_info(&pdev->dev, "%s version is %s\n", DRV_NAME, DRV_VER);
4951
4952         status = pci_enable_device(pdev);
4953         if (status)
4954                 goto do_none;
4955
4956         status = pci_request_regions(pdev, DRV_NAME);
4957         if (status)
4958                 goto disable_dev;
4959         pci_set_master(pdev);
4960
4961         netdev = alloc_etherdev_mqs(sizeof(*adapter), MAX_TX_QS, MAX_RX_QS);
4962         if (!netdev) {
4963                 status = -ENOMEM;
4964                 goto rel_reg;
4965         }
4966         adapter = netdev_priv(netdev);
4967         adapter->pdev = pdev;
4968         pci_set_drvdata(pdev, adapter);
4969         adapter->netdev = netdev;
4970         SET_NETDEV_DEV(netdev, &pdev->dev);
4971
4972         status = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
4973         if (!status) {
4974                 netdev->features |= NETIF_F_HIGHDMA;
4975         } else {
4976                 status = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
4977                 if (status) {
4978                         dev_err(&pdev->dev, "Could not set PCI DMA Mask\n");
4979                         goto free_netdev;
4980                 }
4981         }
4982
4983         status = pci_enable_pcie_error_reporting(pdev);
4984         if (!status)
4985                 dev_info(&pdev->dev, "PCIe error reporting enabled\n");
4986
4987         status = be_ctrl_init(adapter);
4988         if (status)
4989                 goto free_netdev;
4990
4991         /* sync up with fw's ready state */
4992         if (be_physfn(adapter)) {
4993                 status = be_fw_wait_ready(adapter);
4994                 if (status)
4995                         goto ctrl_clean;
4996         }
4997
4998         if (be_reset_required(adapter)) {
4999                 status = be_cmd_reset_function(adapter);
5000                 if (status)
5001                         goto ctrl_clean;
5002
5003                 /* Wait for interrupts to quiesce after an FLR */
5004                 msleep(100);
5005         }
5006
5007         /* Allow interrupts for other ULPs running on NIC function */
5008         be_intr_set(adapter, true);
5009
5010         /* tell fw we're ready to fire cmds */
5011         status = be_cmd_fw_init(adapter);
5012         if (status)
5013                 goto ctrl_clean;
5014
5015         status = be_stats_init(adapter);
5016         if (status)
5017                 goto ctrl_clean;
5018
5019         status = be_get_initial_config(adapter);
5020         if (status)
5021                 goto stats_clean;
5022
5023         INIT_DELAYED_WORK(&adapter->work, be_worker);
5024         INIT_DELAYED_WORK(&adapter->func_recovery_work, be_func_recovery_task);
5025         adapter->rx_fc = true;
5026         adapter->tx_fc = true;
5027
5028         status = be_setup(adapter);
5029         if (status)
5030                 goto stats_clean;
5031
5032         be_netdev_init(netdev);
5033         status = register_netdev(netdev);
5034         if (status != 0)
5035                 goto unsetup;
5036
5037         be_roce_dev_add(adapter);
5038
5039         schedule_delayed_work(&adapter->func_recovery_work,
5040                               msecs_to_jiffies(1000));
5041
5042         be_cmd_query_port_name(adapter, &port_name);
5043
5044         dev_info(&pdev->dev, "%s: %s %s port %c\n", nic_name(pdev),
5045                  func_name(adapter), mc_name(adapter), port_name);
5046
5047         return 0;
5048
5049 unsetup:
5050         be_clear(adapter);
5051 stats_clean:
5052         be_stats_cleanup(adapter);
5053 ctrl_clean:
5054         be_ctrl_cleanup(adapter);
5055 free_netdev:
5056         free_netdev(netdev);
5057 rel_reg:
5058         pci_release_regions(pdev);
5059 disable_dev:
5060         pci_disable_device(pdev);
5061 do_none:
5062         dev_err(&pdev->dev, "%s initialization failed\n", nic_name(pdev));
5063         return status;
5064 }
5065
5066 static int be_suspend(struct pci_dev *pdev, pm_message_t state)
5067 {
5068         struct be_adapter *adapter = pci_get_drvdata(pdev);
5069         struct net_device *netdev =  adapter->netdev;
5070
5071         if (adapter->wol_en)
5072                 be_setup_wol(adapter, true);
5073
5074         be_intr_set(adapter, false);
5075         cancel_delayed_work_sync(&adapter->func_recovery_work);
5076
5077         netif_device_detach(netdev);
5078         if (netif_running(netdev)) {
5079                 rtnl_lock();
5080                 be_close(netdev);
5081                 rtnl_unlock();
5082         }
5083         be_clear(adapter);
5084
5085         pci_save_state(pdev);
5086         pci_disable_device(pdev);
5087         pci_set_power_state(pdev, pci_choose_state(pdev, state));
5088         return 0;
5089 }
5090
5091 static int be_resume(struct pci_dev *pdev)
5092 {
5093         int status = 0;
5094         struct be_adapter *adapter = pci_get_drvdata(pdev);
5095         struct net_device *netdev =  adapter->netdev;
5096
5097         netif_device_detach(netdev);
5098
5099         status = pci_enable_device(pdev);
5100         if (status)
5101                 return status;
5102
5103         pci_set_power_state(pdev, PCI_D0);
5104         pci_restore_state(pdev);
5105
5106         status = be_fw_wait_ready(adapter);
5107         if (status)
5108                 return status;
5109
5110         status = be_cmd_reset_function(adapter);
5111         if (status)
5112                 return status;
5113
5114         be_intr_set(adapter, true);
5115         /* tell fw we're ready to fire cmds */
5116         status = be_cmd_fw_init(adapter);
5117         if (status)
5118                 return status;
5119
5120         be_setup(adapter);
5121         if (netif_running(netdev)) {
5122                 rtnl_lock();
5123                 be_open(netdev);
5124                 rtnl_unlock();
5125         }
5126
5127         schedule_delayed_work(&adapter->func_recovery_work,
5128                               msecs_to_jiffies(1000));
5129         netif_device_attach(netdev);
5130
5131         if (adapter->wol_en)
5132                 be_setup_wol(adapter, false);
5133
5134         return 0;
5135 }
5136
5137 /*
5138  * An FLR will stop BE from DMAing any data.
5139  */
5140 static void be_shutdown(struct pci_dev *pdev)
5141 {
5142         struct be_adapter *adapter = pci_get_drvdata(pdev);
5143
5144         if (!adapter)
5145                 return;
5146
5147         be_roce_dev_shutdown(adapter);
5148         cancel_delayed_work_sync(&adapter->work);
5149         cancel_delayed_work_sync(&adapter->func_recovery_work);
5150
5151         netif_device_detach(adapter->netdev);
5152
5153         be_cmd_reset_function(adapter);
5154
5155         pci_disable_device(pdev);
5156 }
5157
5158 static pci_ers_result_t be_eeh_err_detected(struct pci_dev *pdev,
5159                                             pci_channel_state_t state)
5160 {
5161         struct be_adapter *adapter = pci_get_drvdata(pdev);
5162         struct net_device *netdev =  adapter->netdev;
5163
5164         dev_err(&adapter->pdev->dev, "EEH error detected\n");
5165
5166         if (!adapter->eeh_error) {
5167                 adapter->eeh_error = true;
5168
5169                 cancel_delayed_work_sync(&adapter->func_recovery_work);
5170
5171                 rtnl_lock();
5172                 netif_device_detach(netdev);
5173                 if (netif_running(netdev))
5174                         be_close(netdev);
5175                 rtnl_unlock();
5176
5177                 be_clear(adapter);
5178         }
5179
5180         if (state == pci_channel_io_perm_failure)
5181                 return PCI_ERS_RESULT_DISCONNECT;
5182
5183         pci_disable_device(pdev);
5184
5185         /* The error could cause the FW to trigger a flash debug dump.
5186          * Resetting the card while flash dump is in progress
5187          * can cause it not to recover; wait for it to finish.
5188          * Wait only for first function as it is needed only once per
5189          * adapter.
5190          */
5191         if (pdev->devfn == 0)
5192                 ssleep(30);
5193
5194         return PCI_ERS_RESULT_NEED_RESET;
5195 }
5196
5197 static pci_ers_result_t be_eeh_reset(struct pci_dev *pdev)
5198 {
5199         struct be_adapter *adapter = pci_get_drvdata(pdev);
5200         int status;
5201
5202         dev_info(&adapter->pdev->dev, "EEH reset\n");
5203
5204         status = pci_enable_device(pdev);
5205         if (status)
5206                 return PCI_ERS_RESULT_DISCONNECT;
5207
5208         pci_set_master(pdev);
5209         pci_set_power_state(pdev, PCI_D0);
5210         pci_restore_state(pdev);
5211
5212         /* Check if card is ok and fw is ready */
5213         dev_info(&adapter->pdev->dev,
5214                  "Waiting for FW to be ready after EEH reset\n");
5215         status = be_fw_wait_ready(adapter);
5216         if (status)
5217                 return PCI_ERS_RESULT_DISCONNECT;
5218
5219         pci_cleanup_aer_uncorrect_error_status(pdev);
5220         be_clear_all_error(adapter);
5221         return PCI_ERS_RESULT_RECOVERED;
5222 }
5223
5224 static void be_eeh_resume(struct pci_dev *pdev)
5225 {
5226         int status = 0;
5227         struct be_adapter *adapter = pci_get_drvdata(pdev);
5228         struct net_device *netdev =  adapter->netdev;
5229
5230         dev_info(&adapter->pdev->dev, "EEH resume\n");
5231
5232         pci_save_state(pdev);
5233
5234         status = be_cmd_reset_function(adapter);
5235         if (status)
5236                 goto err;
5237
5238         /* On some BE3 FW versions, after a HW reset,
5239          * interrupts will remain disabled for each function.
5240          * So, explicitly enable interrupts
5241          */
5242         be_intr_set(adapter, true);
5243
5244         /* tell fw we're ready to fire cmds */
5245         status = be_cmd_fw_init(adapter);
5246         if (status)
5247                 goto err;
5248
5249         status = be_setup(adapter);
5250         if (status)
5251                 goto err;
5252
5253         if (netif_running(netdev)) {
5254                 status = be_open(netdev);
5255                 if (status)
5256                         goto err;
5257         }
5258
5259         schedule_delayed_work(&adapter->func_recovery_work,
5260                               msecs_to_jiffies(1000));
5261         netif_device_attach(netdev);
5262         return;
5263 err:
5264         dev_err(&adapter->pdev->dev, "EEH resume failed\n");
5265 }
5266
5267 static const struct pci_error_handlers be_eeh_handlers = {
5268         .error_detected = be_eeh_err_detected,
5269         .slot_reset = be_eeh_reset,
5270         .resume = be_eeh_resume,
5271 };
5272
5273 static struct pci_driver be_driver = {
5274         .name = DRV_NAME,
5275         .id_table = be_dev_ids,
5276         .probe = be_probe,
5277         .remove = be_remove,
5278         .suspend = be_suspend,
5279         .resume = be_resume,
5280         .shutdown = be_shutdown,
5281         .err_handler = &be_eeh_handlers
5282 };
5283
5284 static int __init be_init_module(void)
5285 {
5286         if (rx_frag_size != 8192 && rx_frag_size != 4096 &&
5287             rx_frag_size != 2048) {
5288                 printk(KERN_WARNING DRV_NAME
5289                         " : Module param rx_frag_size must be 2048/4096/8192."
5290                         " Using 2048\n");
5291                 rx_frag_size = 2048;
5292         }
5293
5294         return pci_register_driver(&be_driver);
5295 }
5296 module_init(be_init_module);
5297
5298 static void __exit be_exit_module(void)
5299 {
5300         pci_unregister_driver(&be_driver);
5301 }
5302 module_exit(be_exit_module);