drivers/net/ethernet/chelsio/cxgb3/cxgb3_main.c

   1 /*
   2  * Copyright (c) 2003-2008 Chelsio, Inc. All rights reserved.
   3  *
   4  * This software is available to you under a choice of one of two
   5  * licenses.  You may choose to be licensed under the terms of the GNU
   6  * General Public License (GPL) Version 2, available from the file
   7  * COPYING in the main directory of this source tree, or the
   8  * OpenIB.org BSD license below:
   9  *
  10  *     Redistribution and use in source and binary forms, with or
  11  *     without modification, are permitted provided that the following
  12  *     conditions are met:
  13  *
  14  *      - Redistributions of source code must retain the above
  15  *        copyright notice, this list of conditions and the following
  16  *        disclaimer.
  17  *
  18  *      - Redistributions in binary form must reproduce the above
  19  *        copyright notice, this list of conditions and the following
  20  *        disclaimer in the documentation and/or other materials
  21  *        provided with the distribution.
  22  *
  23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  24  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  25  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  26  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  27  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  28  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  29  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  30  * SOFTWARE.
  31  */
  32
  33 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  34
  35 #include <linux/module.h>
  36 #include <linux/moduleparam.h>
  37 #include <linux/init.h>
  38 #include <linux/pci.h>
  39 #include <linux/dma-mapping.h>
  40 #include <linux/netdevice.h>
  41 #include <linux/etherdevice.h>
  42 #include <linux/if_vlan.h>
  43 #include <linux/mdio.h>
  44 #include <linux/sockios.h>
  45 #include <linux/workqueue.h>
  46 #include <linux/proc_fs.h>
  47 #include <linux/rtnetlink.h>
  48 #include <linux/firmware.h>
  49 #include <linux/log2.h>
  50 #include <linux/stringify.h>
  51 #include <linux/sched.h>
  52 #include <linux/slab.h>
  53 #include <asm/uaccess.h>
  54
  55 #include "common.h"
  56 #include "cxgb3_ioctl.h"
  57 #include "regs.h"
  58 #include "cxgb3_offload.h"
  59 #include "version.h"
  60
  61 #include "cxgb3_ctl_defs.h"
  62 #include "t3_cpl.h"
  63 #include "firmware_exports.h"
  64
  65 enum {
  66         MAX_TXQ_ENTRIES = 16384,
  67         MAX_CTRL_TXQ_ENTRIES = 1024,
  68         MAX_RSPQ_ENTRIES = 16384,
  69         MAX_RX_BUFFERS = 16384,
  70         MAX_RX_JUMBO_BUFFERS = 16384,
  71         MIN_TXQ_ENTRIES = 4,
  72         MIN_CTRL_TXQ_ENTRIES = 4,
  73         MIN_RSPQ_ENTRIES = 32,
  74         MIN_FL_ENTRIES = 32
  75 };
  76
  77 #define PORT_MASK ((1 << MAX_NPORTS) - 1)
  78
  79 #define DFLT_MSG_ENABLE (NETIF_MSG_DRV | NETIF_MSG_PROBE | NETIF_MSG_LINK | \
  80                          NETIF_MSG_TIMER | NETIF_MSG_IFDOWN | NETIF_MSG_IFUP |\
  81                          NETIF_MSG_RX_ERR | NETIF_MSG_TX_ERR)
  82
  83 #define EEPROM_MAGIC 0x38E2F10C
  84
  85 #define CH_DEVICE(devid, idx) \
  86         { PCI_VENDOR_ID_CHELSIO, devid, PCI_ANY_ID, PCI_ANY_ID, 0, 0, idx }
  87
  88 static DEFINE_PCI_DEVICE_TABLE(cxgb3_pci_tbl) = {
  89         CH_DEVICE(0x20, 0),     /* PE9000 */
  90         CH_DEVICE(0x21, 1),     /* T302E */
  91         CH_DEVICE(0x22, 2),     /* T310E */
  92         CH_DEVICE(0x23, 3),     /* T320X */
  93         CH_DEVICE(0x24, 1),     /* T302X */
  94         CH_DEVICE(0x25, 3),     /* T320E */
  95         CH_DEVICE(0x26, 2),     /* T310X */
  96         CH_DEVICE(0x30, 2),     /* T3B10 */
  97         CH_DEVICE(0x31, 3),     /* T3B20 */
  98         CH_DEVICE(0x32, 1),     /* T3B02 */
  99         CH_DEVICE(0x35, 6),     /* T3C20-derived T3C10 */
 100         CH_DEVICE(0x36, 3),     /* S320E-CR */
 101         CH_DEVICE(0x37, 7),     /* N320E-G2 */
 102         {0,}
 103 };
 104
 105 MODULE_DESCRIPTION(DRV_DESC);
 106 MODULE_AUTHOR("Chelsio Communications");
 107 MODULE_LICENSE("Dual BSD/GPL");
 108 MODULE_VERSION(DRV_VERSION);
 109 MODULE_DEVICE_TABLE(pci, cxgb3_pci_tbl);
 110
 111 static int dflt_msg_enable = DFLT_MSG_ENABLE;
 112
 113 module_param(dflt_msg_enable, int, 0644);
 114 MODULE_PARM_DESC(dflt_msg_enable, "Chelsio T3 default message enable bitmap");
 115
 116 /*
 117  * The driver uses the best interrupt scheme available on a platform in the
 118  * order MSI-X, MSI, legacy pin interrupts.  This parameter determines which
 119  * of these schemes the driver may consider as follows:
 120  *
 121  * msi = 2: choose from among all three options
 122  * msi = 1: only consider MSI and pin interrupts
 123  * msi = 0: force pin interrupts
 124  */
 125 static int msi = 2;
 126
 127 module_param(msi, int, 0644);
 128 MODULE_PARM_DESC(msi, "whether to use MSI or MSI-X");
 129
 130 /*
 131  * The driver enables offload as a default.
 132  * To disable it, use ofld_disable = 1.
 133  */
 134
 135 static int ofld_disable = 0;
 136
 137 module_param(ofld_disable, int, 0644);
 138 MODULE_PARM_DESC(ofld_disable, "whether to enable offload at init time or not");
 139
 140 /*
 141  * We have work elements that we need to cancel when an interface is taken
 142  * down.  Normally the work elements would be executed by keventd but that
 143  * can deadlock because of linkwatch.  If our close method takes the rtnl
 144  * lock and linkwatch is ahead of our work elements in keventd, linkwatch
 145  * will block keventd as it needs the rtnl lock, and we'll deadlock waiting
 146  * for our work to complete.  Get our own work queue to solve this.
 147  */
 148 struct workqueue_struct *cxgb3_wq;
 149
 150 /**
 151  *      link_report - show link status and link speed/duplex
 152  *      @p: the port whose settings are to be reported
 153  *
 154  *      Shows the link status, speed, and duplex of a port.
 155  */
 156 static void link_report(struct net_device *dev)
 157 {
 158         if (!netif_carrier_ok(dev))
 159                 netdev_info(dev, "link down\n");
 160         else {
 161                 const char *s = "10Mbps";
 162                 const struct port_info *p = netdev_priv(dev);
 163
 164                 switch (p->link_config.speed) {
 165                 case SPEED_10000:
 166                         s = "10Gbps";
 167                         break;
 168                 case SPEED_1000:
 169                         s = "1000Mbps";
 170                         break;
 171                 case SPEED_100:
 172                         s = "100Mbps";
 173                         break;
 174                 }
 175
 176                 netdev_info(dev, "link up, %s, %s-duplex\n",
 177                             s, p->link_config.duplex == DUPLEX_FULL
 178                             ? "full" : "half");
 179         }
 180 }
 181
 182 static void enable_tx_fifo_drain(struct adapter *adapter,
 183                                  struct port_info *pi)
 184 {
 185         t3_set_reg_field(adapter, A_XGM_TXFIFO_CFG + pi->mac.offset, 0,
 186                          F_ENDROPPKT);
 187         t3_write_reg(adapter, A_XGM_RX_CTRL + pi->mac.offset, 0);
 188         t3_write_reg(adapter, A_XGM_TX_CTRL + pi->mac.offset, F_TXEN);
 189         t3_write_reg(adapter, A_XGM_RX_CTRL + pi->mac.offset, F_RXEN);
 190 }
 191
 192 static void disable_tx_fifo_drain(struct adapter *adapter,
 193                                   struct port_info *pi)
 194 {
 195         t3_set_reg_field(adapter, A_XGM_TXFIFO_CFG + pi->mac.offset,
 196                          F_ENDROPPKT, 0);
 197 }
 198
 199 void t3_os_link_fault(struct adapter *adap, int port_id, int state)
 200 {
 201         struct net_device *dev = adap->port[port_id];
 202         struct port_info *pi = netdev_priv(dev);
 203
 204         if (state == netif_carrier_ok(dev))
 205                 return;
 206
 207         if (state) {
 208                 struct cmac *mac = &pi->mac;
 209
 210                 netif_carrier_on(dev);
 211
 212                 disable_tx_fifo_drain(adap, pi);
 213
 214                 /* Clear local faults */
 215                 t3_xgm_intr_disable(adap, pi->port_id);
 216                 t3_read_reg(adap, A_XGM_INT_STATUS +
 217                                     pi->mac.offset);
 218                 t3_write_reg(adap,
 219                              A_XGM_INT_CAUSE + pi->mac.offset,
 220                              F_XGM_INT);
 221
 222                 t3_set_reg_field(adap,
 223                                  A_XGM_INT_ENABLE +
 224                                  pi->mac.offset,
 225                                  F_XGM_INT, F_XGM_INT);
 226                 t3_xgm_intr_enable(adap, pi->port_id);
 227
 228                 t3_mac_enable(mac, MAC_DIRECTION_TX);
 229         } else {
 230                 netif_carrier_off(dev);
 231
 232                 /* Flush TX FIFO */
 233                 enable_tx_fifo_drain(adap, pi);
 234         }
 235         link_report(dev);
 236 }
 237
 238 /**
 239  *      t3_os_link_changed - handle link status changes
 240  *      @adapter: the adapter associated with the link change
 241  *      @port_id: the port index whose limk status has changed
 242  *      @link_stat: the new status of the link
 243  *      @speed: the new speed setting
 244  *      @duplex: the new duplex setting
 245  *      @pause: the new flow-control setting
 246  *
 247  *      This is the OS-dependent handler for link status changes.  The OS
 248  *      neutral handler takes care of most of the processing for these events,
 249  *      then calls this handler for any OS-specific processing.
 250  */
 251 void t3_os_link_changed(struct adapter *adapter, int port_id, int link_stat,
 252                         int speed, int duplex, int pause)
 253 {
 254         struct net_device *dev = adapter->port[port_id];
 255         struct port_info *pi = netdev_priv(dev);
 256         struct cmac *mac = &pi->mac;
 257
 258         /* Skip changes from disabled ports. */
 259         if (!netif_running(dev))
 260                 return;
 261
 262         if (link_stat != netif_carrier_ok(dev)) {
 263                 if (link_stat) {
 264                         disable_tx_fifo_drain(adapter, pi);
 265
 266                         t3_mac_enable(mac, MAC_DIRECTION_RX);
 267
 268                         /* Clear local faults */
 269                         t3_xgm_intr_disable(adapter, pi->port_id);
 270                         t3_read_reg(adapter, A_XGM_INT_STATUS +
 271                                     pi->mac.offset);
 272                         t3_write_reg(adapter,
 273                                      A_XGM_INT_CAUSE + pi->mac.offset,
 274                                      F_XGM_INT);
 275
 276                         t3_set_reg_field(adapter,
 277                                          A_XGM_INT_ENABLE + pi->mac.offset,
 278                                          F_XGM_INT, F_XGM_INT);
 279                         t3_xgm_intr_enable(adapter, pi->port_id);
 280
 281                         netif_carrier_on(dev);
 282                 } else {
 283                         netif_carrier_off(dev);
 284
 285                         t3_xgm_intr_disable(adapter, pi->port_id);
 286                         t3_read_reg(adapter, A_XGM_INT_STATUS + pi->mac.offset);
 287                         t3_set_reg_field(adapter,
 288                                          A_XGM_INT_ENABLE + pi->mac.offset,
 289                                          F_XGM_INT, 0);
 290
 291                         if (is_10G(adapter))
 292                                 pi->phy.ops->power_down(&pi->phy, 1);
 293
 294                         t3_read_reg(adapter, A_XGM_INT_STATUS + pi->mac.offset);
 295                         t3_mac_disable(mac, MAC_DIRECTION_RX);
 296                         t3_link_start(&pi->phy, mac, &pi->link_config);
 297
 298                         /* Flush TX FIFO */
 299                         enable_tx_fifo_drain(adapter, pi);
 300                 }
 301
 302                 link_report(dev);
 303         }
 304 }
 305
 306 /**
 307  *      t3_os_phymod_changed - handle PHY module changes
 308  *      @phy: the PHY reporting the module change
 309  *      @mod_type: new module type
 310  *
 311  *      This is the OS-dependent handler for PHY module changes.  It is
 312  *      invoked when a PHY module is removed or inserted for any OS-specific
 313  *      processing.
 314  */
 315 void t3_os_phymod_changed(struct adapter *adap, int port_id)
 316 {
 317         static const char *mod_str[] = {
 318                 NULL, "SR", "LR", "LRM", "TWINAX", "TWINAX", "unknown"
 319         };
 320
 321         const struct net_device *dev = adap->port[port_id];
 322         const struct port_info *pi = netdev_priv(dev);
 323
 324         if (pi->phy.modtype == phy_modtype_none)
 325                 netdev_info(dev, "PHY module unplugged\n");
 326         else
 327                 netdev_info(dev, "%s PHY module inserted\n",
 328                             mod_str[pi->phy.modtype]);
 329 }
 330
 331 static void cxgb_set_rxmode(struct net_device *dev)
 332 {
 333         struct port_info *pi = netdev_priv(dev);
 334
 335         t3_mac_set_rx_mode(&pi->mac, dev);
 336 }
 337
 338 /**
 339  *      link_start - enable a port
 340  *      @dev: the device to enable
 341  *
 342  *      Performs the MAC and PHY actions needed to enable a port.
 343  */
 344 static void link_start(struct net_device *dev)
 345 {
 346         struct port_info *pi = netdev_priv(dev);
 347         struct cmac *mac = &pi->mac;
 348
 349         t3_mac_reset(mac);
 350         t3_mac_set_num_ucast(mac, MAX_MAC_IDX);
 351         t3_mac_set_mtu(mac, dev->mtu);
 352         t3_mac_set_address(mac, LAN_MAC_IDX, dev->dev_addr);
 353         t3_mac_set_address(mac, SAN_MAC_IDX, pi->iscsic.mac_addr);
 354         t3_mac_set_rx_mode(mac, dev);
 355         t3_link_start(&pi->phy, mac, &pi->link_config);
 356         t3_mac_enable(mac, MAC_DIRECTION_RX | MAC_DIRECTION_TX);
 357 }
 358
 359 static inline void cxgb_disable_msi(struct adapter *adapter)
 360 {
 361         if (adapter->flags & USING_MSIX) {
 362                 pci_disable_msix(adapter->pdev);
 363                 adapter->flags &= ~USING_MSIX;
 364         } else if (adapter->flags & USING_MSI) {
 365                 pci_disable_msi(adapter->pdev);
 366                 adapter->flags &= ~USING_MSI;
 367         }
 368 }
 369
 370 /*
 371  * Interrupt handler for asynchronous events used with MSI-X.
 372  */
 373 static irqreturn_t t3_async_intr_handler(int irq, void *cookie)
 374 {
 375         t3_slow_intr_handler(cookie);
 376         return IRQ_HANDLED;
 377 }
 378
 379 /*
 380  * Name the MSI-X interrupts.
 381  */
 382 static void name_msix_vecs(struct adapter *adap)
 383 {
 384         int i, j, msi_idx = 1, n = sizeof(adap->msix_info[0].desc) - 1;
 385
 386         snprintf(adap->msix_info[0].desc, n, "%s", adap->name);
 387         adap->msix_info[0].desc[n] = 0;
 388
 389         for_each_port(adap, j) {
 390                 struct net_device *d = adap->port[j];
 391                 const struct port_info *pi = netdev_priv(d);
 392
 393                 for (i = 0; i < pi->nqsets; i++, msi_idx++) {
 394                         snprintf(adap->msix_info[msi_idx].desc, n,
 395                                  "%s-%d", d->name, pi->first_qset + i);
 396                         adap->msix_info[msi_idx].desc[n] = 0;
 397                 }
 398         }
 399 }
 400
 401 static int request_msix_data_irqs(struct adapter *adap)
 402 {
 403         int i, j, err, qidx = 0;
 404
 405         for_each_port(adap, i) {
 406                 int nqsets = adap2pinfo(adap, i)->nqsets;
 407
 408                 for (j = 0; j < nqsets; ++j) {
 409                         err = request_irq(adap->msix_info[qidx + 1].vec,
 410                                           t3_intr_handler(adap,
 411                                                           adap->sge.qs[qidx].
 412                                                           rspq.polling), 0,
 413                                           adap->msix_info[qidx + 1].desc,
 414                                           &adap->sge.qs[qidx]);
 415                         if (err) {
 416                                 while (--qidx >= 0)
 417                                         free_irq(adap->msix_info[qidx + 1].vec,
 418                                                  &adap->sge.qs[qidx]);
 419                                 return err;
 420                         }
 421                         qidx++;
 422                 }
 423         }
 424         return 0;
 425 }
 426
 427 static void free_irq_resources(struct adapter *adapter)
 428 {
 429         if (adapter->flags & USING_MSIX) {
 430                 int i, n = 0;
 431
 432                 free_irq(adapter->msix_info[0].vec, adapter);
 433                 for_each_port(adapter, i)
 434                         n += adap2pinfo(adapter, i)->nqsets;
 435
 436                 for (i = 0; i < n; ++i)
 437                         free_irq(adapter->msix_info[i + 1].vec,
 438                                  &adapter->sge.qs[i]);
 439         } else
 440                 free_irq(adapter->pdev->irq, adapter);
 441 }
 442
 443 static int await_mgmt_replies(struct adapter *adap, unsigned long init_cnt,
 444                               unsigned long n)
 445 {
 446         int attempts = 10;
 447
 448         while (adap->sge.qs[0].rspq.offload_pkts < init_cnt + n) {
 449                 if (!--attempts)
 450                         return -ETIMEDOUT;
 451                 msleep(10);
 452         }
 453         return 0;
 454 }
 455
 456 static int init_tp_parity(struct adapter *adap)
 457 {
 458         int i;
 459         struct sk_buff *skb;
 460         struct cpl_set_tcb_field *greq;
 461         unsigned long cnt = adap->sge.qs[0].rspq.offload_pkts;
 462
 463         t3_tp_set_offload_mode(adap, 1);
 464
 465         for (i = 0; i < 16; i++) {
 466                 struct cpl_smt_write_req *req;
 467
 468                 skb = alloc_skb(sizeof(*req), GFP_KERNEL);
 469                 if (!skb)
 470                         skb = adap->nofail_skb;
 471                 if (!skb)
 472                         goto alloc_skb_fail;
 473
 474                 req = (struct cpl_smt_write_req *)__skb_put(skb, sizeof(*req));
 475                 memset(req, 0, sizeof(*req));
 476                 req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
 477                 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SMT_WRITE_REQ, i));
 478                 req->mtu_idx = NMTUS - 1;
 479                 req->iff = i;
 480                 t3_mgmt_tx(adap, skb);
 481                 if (skb == adap->nofail_skb) {
 482                         await_mgmt_replies(adap, cnt, i + 1);
 483                         adap->nofail_skb = alloc_skb(sizeof(*greq), GFP_KERNEL);
 484                         if (!adap->nofail_skb)
 485                                 goto alloc_skb_fail;
 486                 }
 487         }
 488
 489         for (i = 0; i < 2048; i++) {
 490                 struct cpl_l2t_write_req *req;
 491
 492                 skb = alloc_skb(sizeof(*req), GFP_KERNEL);
 493                 if (!skb)
 494                         skb = adap->nofail_skb;
 495                 if (!skb)
 496                         goto alloc_skb_fail;
 497
 498                 req = (struct cpl_l2t_write_req *)__skb_put(skb, sizeof(*req));
 499                 memset(req, 0, sizeof(*req));
 500                 req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
 501                 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_L2T_WRITE_REQ, i));
 502                 req->params = htonl(V_L2T_W_IDX(i));
 503                 t3_mgmt_tx(adap, skb);
 504                 if (skb == adap->nofail_skb) {
 505                         await_mgmt_replies(adap, cnt, 16 + i + 1);
 506                         adap->nofail_skb = alloc_skb(sizeof(*greq), GFP_KERNEL);
 507                         if (!adap->nofail_skb)
 508                                 goto alloc_skb_fail;
 509                 }
 510         }
 511
 512         for (i = 0; i < 2048; i++) {
 513                 struct cpl_rte_write_req *req;
 514
 515                 skb = alloc_skb(sizeof(*req), GFP_KERNEL);
 516                 if (!skb)
 517                         skb = adap->nofail_skb;
 518                 if (!skb)
 519                         goto alloc_skb_fail;
 520
 521                 req = (struct cpl_rte_write_req *)__skb_put(skb, sizeof(*req));
 522                 memset(req, 0, sizeof(*req));
 523                 req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
 524                 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_RTE_WRITE_REQ, i));
 525                 req->l2t_idx = htonl(V_L2T_W_IDX(i));
 526                 t3_mgmt_tx(adap, skb);
 527                 if (skb == adap->nofail_skb) {
 528                         await_mgmt_replies(adap, cnt, 16 + 2048 + i + 1);
 529                         adap->nofail_skb = alloc_skb(sizeof(*greq), GFP_KERNEL);
 530                         if (!adap->nofail_skb)
 531                                 goto alloc_skb_fail;
 532                 }
 533         }
 534
 535         skb = alloc_skb(sizeof(*greq), GFP_KERNEL);
 536         if (!skb)
 537                 skb = adap->nofail_skb;
 538         if (!skb)
 539                 goto alloc_skb_fail;
 540
 541         greq = (struct cpl_set_tcb_field *)__skb_put(skb, sizeof(*greq));
 542         memset(greq, 0, sizeof(*greq));
 543         greq->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
 544         OPCODE_TID(greq) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD, 0));
 545         greq->mask = cpu_to_be64(1);
 546         t3_mgmt_tx(adap, skb);
 547
 548         i = await_mgmt_replies(adap, cnt, 16 + 2048 + 2048 + 1);
 549         if (skb == adap->nofail_skb) {
 550                 i = await_mgmt_replies(adap, cnt, 16 + 2048 + 2048 + 1);
 551                 adap->nofail_skb = alloc_skb(sizeof(*greq), GFP_KERNEL);
 552         }
 553
 554         t3_tp_set_offload_mode(adap, 0);
 555         return i;
 556
 557 alloc_skb_fail:
 558         t3_tp_set_offload_mode(adap, 0);
 559         return -ENOMEM;
 560 }
 561
 562 /**
 563  *      setup_rss - configure RSS
 564  *      @adap: the adapter
 565  *
 566  *      Sets up RSS to distribute packets to multiple receive queues.  We
 567  *      configure the RSS CPU lookup table to distribute to the number of HW
 568  *      receive queues, and the response queue lookup table to narrow that
 569  *      down to the response queues actually configured for each port.
 570  *      We always configure the RSS mapping for two ports since the mapping
 571  *      table has plenty of entries.
 572  */
 573 static void setup_rss(struct adapter *adap)
 574 {
 575         int i;
 576         unsigned int nq0 = adap2pinfo(adap, 0)->nqsets;
 577         unsigned int nq1 = adap->port[1] ? adap2pinfo(adap, 1)->nqsets : 1;
 578         u8 cpus[SGE_QSETS + 1];
 579         u16 rspq_map[RSS_TABLE_SIZE];
 580
 581         for (i = 0; i < SGE_QSETS; ++i)
 582                 cpus[i] = i;
 583         cpus[SGE_QSETS] = 0xff; /* terminator */
 584
 585         for (i = 0; i < RSS_TABLE_SIZE / 2; ++i) {
 586                 rspq_map[i] = i % nq0;
 587                 rspq_map[i + RSS_TABLE_SIZE / 2] = (i % nq1) + nq0;
 588         }
 589
 590         t3_config_rss(adap, F_RQFEEDBACKENABLE | F_TNLLKPEN | F_TNLMAPEN |
 591                       F_TNLPRTEN | F_TNL2TUPEN | F_TNL4TUPEN |
 592                       V_RRCPLCPUSIZE(6) | F_HASHTOEPLITZ, cpus, rspq_map);
 593 }
 594
 595 static void ring_dbs(struct adapter *adap)
 596 {
 597         int i, j;
 598
 599         for (i = 0; i < SGE_QSETS; i++) {
 600                 struct sge_qset *qs = &adap->sge.qs[i];
 601
 602                 if (qs->adap)
 603                         for (j = 0; j < SGE_TXQ_PER_SET; j++)
 604                                 t3_write_reg(adap, A_SG_KDOORBELL, F_SELEGRCNTX | V_EGRCNTX(qs->txq[j].cntxt_id));
 605         }
 606 }
 607
 608 static void init_napi(struct adapter *adap)
 609 {
 610         int i;
 611
 612         for (i = 0; i < SGE_QSETS; i++) {
 613                 struct sge_qset *qs = &adap->sge.qs[i];
 614
 615                 if (qs->adap)
 616                         netif_napi_add(qs->netdev, &qs->napi, qs->napi.poll,
 617                                        64);
 618         }
 619
 620         /*
 621          * netif_napi_add() can be called only once per napi_struct because it
 622          * adds each new napi_struct to a list.  Be careful not to call it a
 623          * second time, e.g., during EEH recovery, by making a note of it.
 624          */
 625         adap->flags |= NAPI_INIT;
 626 }
 627
 628 /*
 629  * Wait until all NAPI handlers are descheduled.  This includes the handlers of
 630  * both netdevices representing interfaces and the dummy ones for the extra
 631  * queues.
 632  */
 633 static void quiesce_rx(struct adapter *adap)
 634 {
 635         int i;
 636
 637         for (i = 0; i < SGE_QSETS; i++)
 638                 if (adap->sge.qs[i].adap)
 639                         napi_disable(&adap->sge.qs[i].napi);
 640 }
 641
 642 static void enable_all_napi(struct adapter *adap)
 643 {
 644         int i;
 645         for (i = 0; i < SGE_QSETS; i++)
 646                 if (adap->sge.qs[i].adap)
 647                         napi_enable(&adap->sge.qs[i].napi);
 648 }
 649
 650 /**
 651  *      setup_sge_qsets - configure SGE Tx/Rx/response queues
 652  *      @adap: the adapter
 653  *
 654  *      Determines how many sets of SGE queues to use and initializes them.
 655  *      We support multiple queue sets per port if we have MSI-X, otherwise
 656  *      just one queue set per port.
 657  */
 658 static int setup_sge_qsets(struct adapter *adap)
 659 {
 660         int i, j, err, irq_idx = 0, qset_idx = 0;
 661         unsigned int ntxq = SGE_TXQ_PER_SET;
 662
 663         if (adap->params.rev > 0 && !(adap->flags & USING_MSI))
 664                 irq_idx = -1;
 665
 666         for_each_port(adap, i) {
 667                 struct net_device *dev = adap->port[i];
 668                 struct port_info *pi = netdev_priv(dev);
 669
 670                 pi->qs = &adap->sge.qs[pi->first_qset];
 671                 for (j = 0; j < pi->nqsets; ++j, ++qset_idx) {
 672                         err = t3_sge_alloc_qset(adap, qset_idx, 1,
 673                                 (adap->flags & USING_MSIX) ? qset_idx + 1 :
 674                                                              irq_idx,
 675                                 &adap->params.sge.qset[qset_idx], ntxq, dev,
 676                                 netdev_get_tx_queue(dev, j));
 677                         if (err) {
 678                                 t3_free_sge_resources(adap);
 679                                 return err;
 680                         }
 681                 }
 682         }
 683
 684         return 0;
 685 }
 686
 687 static ssize_t attr_show(struct device *d, char *buf,
 688                          ssize_t(*format) (struct net_device *, char *))
 689 {
 690         ssize_t len;
 691
 692         /* Synchronize with ioctls that may shut down the device */
 693         rtnl_lock();
 694         len = (*format) (to_net_dev(d), buf);
 695         rtnl_unlock();
 696         return len;
 697 }
 698
 699 static ssize_t attr_store(struct device *d,
 700                           const char *buf, size_t len,
 701                           ssize_t(*set) (struct net_device *, unsigned int),
 702                           unsigned int min_val, unsigned int max_val)
 703 {
 704         char *endp;
 705         ssize_t ret;
 706         unsigned int val;
 707
 708         if (!capable(CAP_NET_ADMIN))
 709                 return -EPERM;
 710
 711         val = simple_strtoul(buf, &endp, 0);
 712         if (endp == buf || val < min_val || val > max_val)
 713                 return -EINVAL;
 714
 715         rtnl_lock();
 716         ret = (*set) (to_net_dev(d), val);
 717         if (!ret)
 718                 ret = len;
 719         rtnl_unlock();
 720         return ret;
 721 }
 722
 723 #define CXGB3_SHOW(name, val_expr) \
 724 static ssize_t format_##name(struct net_device *dev, char *buf) \
 725 { \
 726         struct port_info *pi = netdev_priv(dev); \
 727         struct adapter *adap = pi->adapter; \
 728         return sprintf(buf, "%u\n", val_expr); \
 729 } \
 730 static ssize_t show_##name(struct device *d, struct device_attribute *attr, \
 731                            char *buf) \
 732 { \
 733         return attr_show(d, buf, format_##name); \
 734 }
 735
 736 static ssize_t set_nfilters(struct net_device *dev, unsigned int val)
 737 {
 738         struct port_info *pi = netdev_priv(dev);
 739         struct adapter *adap = pi->adapter;
 740         int min_tids = is_offload(adap) ? MC5_MIN_TIDS : 0;
 741
 742         if (adap->flags & FULL_INIT_DONE)
 743                 return -EBUSY;
 744         if (val && adap->params.rev == 0)
 745                 return -EINVAL;
 746         if (val > t3_mc5_size(&adap->mc5) - adap->params.mc5.nservers -
 747             min_tids)
 748                 return -EINVAL;
 749         adap->params.mc5.nfilters = val;
 750         return 0;
 751 }
 752
 753 static ssize_t store_nfilters(struct device *d, struct device_attribute *attr,
 754                               const char *buf, size_t len)
 755 {
 756         return attr_store(d, buf, len, set_nfilters, 0, ~0);
 757 }
 758
 759 static ssize_t set_nservers(struct net_device *dev, unsigned int val)
 760 {
 761         struct port_info *pi = netdev_priv(dev);
 762         struct adapter *adap = pi->adapter;
 763
 764         if (adap->flags & FULL_INIT_DONE)
 765                 return -EBUSY;
 766         if (val > t3_mc5_size(&adap->mc5) - adap->params.mc5.nfilters -
 767             MC5_MIN_TIDS)
 768                 return -EINVAL;
 769         adap->params.mc5.nservers = val;
 770         return 0;
 771 }
 772
 773 static ssize_t store_nservers(struct device *d, struct device_attribute *attr,
 774                               const char *buf, size_t len)
 775 {
 776         return attr_store(d, buf, len, set_nservers, 0, ~0);
 777 }
 778
 779 #define CXGB3_ATTR_R(name, val_expr) \
 780 CXGB3_SHOW(name, val_expr) \
 781 static DEVICE_ATTR(name, S_IRUGO, show_##name, NULL)
 782
 783 #define CXGB3_ATTR_RW(name, val_expr, store_method) \
 784 CXGB3_SHOW(name, val_expr) \
 785 static DEVICE_ATTR(name, S_IRUGO | S_IWUSR, show_##name, store_method)
 786
 787 CXGB3_ATTR_R(cam_size, t3_mc5_size(&adap->mc5));
 788 CXGB3_ATTR_RW(nfilters, adap->params.mc5.nfilters, store_nfilters);
 789 CXGB3_ATTR_RW(nservers, adap->params.mc5.nservers, store_nservers);
 790
 791 static struct attribute *cxgb3_attrs[] = {
 792         &dev_attr_cam_size.attr,
 793         &dev_attr_nfilters.attr,
 794         &dev_attr_nservers.attr,
 795         NULL
 796 };
 797
 798 static struct attribute_group cxgb3_attr_group = {.attrs = cxgb3_attrs };
 799
 800 static ssize_t tm_attr_show(struct device *d,
 801                             char *buf, int sched)
 802 {
 803         struct port_info *pi = netdev_priv(to_net_dev(d));
 804         struct adapter *adap = pi->adapter;
 805         unsigned int v, addr, bpt, cpt;
 806         ssize_t len;
 807
 808         addr = A_TP_TX_MOD_Q1_Q0_RATE_LIMIT - sched / 2;
 809         rtnl_lock();
 810         t3_write_reg(adap, A_TP_TM_PIO_ADDR, addr);
 811         v = t3_read_reg(adap, A_TP_TM_PIO_DATA);
 812         if (sched & 1)
 813                 v >>= 16;
 814         bpt = (v >> 8) & 0xff;
 815         cpt = v & 0xff;
 816         if (!cpt)
 817                 len = sprintf(buf, "disabled\n");
 818         else {
 819                 v = (adap->params.vpd.cclk * 1000) / cpt;
 820                 len = sprintf(buf, "%u Kbps\n", (v * bpt) / 125);
 821         }
 822         rtnl_unlock();
 823         return len;
 824 }
 825
 826 static ssize_t tm_attr_store(struct device *d,
 827                              const char *buf, size_t len, int sched)
 828 {
 829         struct port_info *pi = netdev_priv(to_net_dev(d));
 830         struct adapter *adap = pi->adapter;
 831         unsigned int val;
 832         char *endp;
 833         ssize_t ret;
 834
 835         if (!capable(CAP_NET_ADMIN))
 836                 return -EPERM;
 837
 838         val = simple_strtoul(buf, &endp, 0);
 839         if (endp == buf || val > 10000000)
 840                 return -EINVAL;
 841
 842         rtnl_lock();
 843         ret = t3_config_sched(adap, val, sched);
 844         if (!ret)
 845                 ret = len;
 846         rtnl_unlock();
 847         return ret;
 848 }
 849
 850 #define TM_ATTR(name, sched) \
 851 static ssize_t show_##name(struct device *d, struct device_attribute *attr, \
 852                            char *buf) \
 853 { \
 854         return tm_attr_show(d, buf, sched); \
 855 } \
 856 static ssize_t store_##name(struct device *d, struct device_attribute *attr, \
 857                             const char *buf, size_t len) \
 858 { \
 859         return tm_attr_store(d, buf, len, sched); \
 860 } \
 861 static DEVICE_ATTR(name, S_IRUGO | S_IWUSR, show_##name, store_##name)
 862
 863 TM_ATTR(sched0, 0);
 864 TM_ATTR(sched1, 1);
 865 TM_ATTR(sched2, 2);
 866 TM_ATTR(sched3, 3);
 867 TM_ATTR(sched4, 4);
 868 TM_ATTR(sched5, 5);
 869 TM_ATTR(sched6, 6);
 870 TM_ATTR(sched7, 7);
 871
 872 static struct attribute *offload_attrs[] = {
 873         &dev_attr_sched0.attr,
 874         &dev_attr_sched1.attr,
 875         &dev_attr_sched2.attr,
 876         &dev_attr_sched3.attr,
 877         &dev_attr_sched4.attr,
 878         &dev_attr_sched5.attr,
 879         &dev_attr_sched6.attr,
 880         &dev_attr_sched7.attr,
 881         NULL
 882 };
 883
 884 static struct attribute_group offload_attr_group = {.attrs = offload_attrs };
 885
 886 /*
 887  * Sends an sk_buff to an offload queue driver
 888  * after dealing with any active network taps.
 889  */
 890 static inline int offload_tx(struct t3cdev *tdev, struct sk_buff *skb)
 891 {
 892         int ret;
 893
 894         local_bh_disable();
 895         ret = t3_offload_tx(tdev, skb);
 896         local_bh_enable();
 897         return ret;
 898 }
 899
 900 static int write_smt_entry(struct adapter *adapter, int idx)
 901 {
 902         struct cpl_smt_write_req *req;
 903         struct port_info *pi = netdev_priv(adapter->port[idx]);
 904         struct sk_buff *skb = alloc_skb(sizeof(*req), GFP_KERNEL);
 905
 906         if (!skb)
 907                 return -ENOMEM;
 908
 909         req = (struct cpl_smt_write_req *)__skb_put(skb, sizeof(*req));
 910         req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
 911         OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SMT_WRITE_REQ, idx));
 912         req->mtu_idx = NMTUS - 1;       /* should be 0 but there's a T3 bug */
 913         req->iff = idx;
 914         memcpy(req->src_mac0, adapter->port[idx]->dev_addr, ETH_ALEN);
 915         memcpy(req->src_mac1, pi->iscsic.mac_addr, ETH_ALEN);
 916         skb->priority = 1;
 917         offload_tx(&adapter->tdev, skb);
 918         return 0;
 919 }
 920
 921 static int init_smt(struct adapter *adapter)
 922 {
 923         int i;
 924
 925         for_each_port(adapter, i)
 926             write_smt_entry(adapter, i);
 927         return 0;
 928 }
 929
 930 static void init_port_mtus(struct adapter *adapter)
 931 {
 932         unsigned int mtus = adapter->port[0]->mtu;
 933
 934         if (adapter->port[1])
 935                 mtus |= adapter->port[1]->mtu << 16;
 936         t3_write_reg(adapter, A_TP_MTU_PORT_TABLE, mtus);
 937 }
 938
 939 static int send_pktsched_cmd(struct adapter *adap, int sched, int qidx, int lo,
 940                               int hi, int port)
 941 {
 942         struct sk_buff *skb;
 943         struct mngt_pktsched_wr *req;
 944         int ret;
 945
 946         skb = alloc_skb(sizeof(*req), GFP_KERNEL);
 947         if (!skb)
 948                 skb = adap->nofail_skb;
 949         if (!skb)
 950                 return -ENOMEM;
 951
 952         req = (struct mngt_pktsched_wr *)skb_put(skb, sizeof(*req));
 953         req->wr_hi = htonl(V_WR_OP(FW_WROPCODE_MNGT));
 954         req->mngt_opcode = FW_MNGTOPCODE_PKTSCHED_SET;
 955         req->sched = sched;
 956         req->idx = qidx;
 957         req->min = lo;
 958         req->max = hi;
 959         req->binding = port;
 960         ret = t3_mgmt_tx(adap, skb);
 961         if (skb == adap->nofail_skb) {
 962                 adap->nofail_skb = alloc_skb(sizeof(struct cpl_set_tcb_field),
 963                                              GFP_KERNEL);
 964                 if (!adap->nofail_skb)
 965                         ret = -ENOMEM;
 966         }
 967
 968         return ret;
 969 }
 970
 971 static int bind_qsets(struct adapter *adap)
 972 {
 973         int i, j, err = 0;
 974
 975         for_each_port(adap, i) {
 976                 const struct port_info *pi = adap2pinfo(adap, i);
 977
 978                 for (j = 0; j < pi->nqsets; ++j) {
 979                         int ret = send_pktsched_cmd(adap, 1,
 980                                                     pi->first_qset + j, -1,
 981                                                     -1, i);
 982                         if (ret)
 983                                 err = ret;
 984                 }
 985         }
 986
 987         return err;
 988 }
 989
 990 #define FW_VERSION __stringify(FW_VERSION_MAJOR) "."                    \
 991         __stringify(FW_VERSION_MINOR) "." __stringify(FW_VERSION_MICRO)
 992 #define FW_FNAME "cxgb3/t3fw-" FW_VERSION ".bin"
 993 #define TPSRAM_VERSION __stringify(TP_VERSION_MAJOR) "."                \
 994         __stringify(TP_VERSION_MINOR) "." __stringify(TP_VERSION_MICRO)
 995 #define TPSRAM_NAME "cxgb3/t3%c_psram-" TPSRAM_VERSION ".bin"
 996 #define AEL2005_OPT_EDC_NAME "cxgb3/ael2005_opt_edc.bin"
 997 #define AEL2005_TWX_EDC_NAME "cxgb3/ael2005_twx_edc.bin"
 998 #define AEL2020_TWX_EDC_NAME "cxgb3/ael2020_twx_edc.bin"
 999 MODULE_FIRMWARE(FW_FNAME);
1000 MODULE_FIRMWARE("cxgb3/t3b_psram-" TPSRAM_VERSION ".bin");
1001 MODULE_FIRMWARE("cxgb3/t3c_psram-" TPSRAM_VERSION ".bin");
1002 MODULE_FIRMWARE(AEL2005_OPT_EDC_NAME);
1003 MODULE_FIRMWARE(AEL2005_TWX_EDC_NAME);
1004 MODULE_FIRMWARE(AEL2020_TWX_EDC_NAME);
1005
1006 static inline const char *get_edc_fw_name(int edc_idx)
1007 {
1008         const char *fw_name = NULL;
1009
1010         switch (edc_idx) {
1011         case EDC_OPT_AEL2005:
1012                 fw_name = AEL2005_OPT_EDC_NAME;
1013                 break;
1014         case EDC_TWX_AEL2005:
1015                 fw_name = AEL2005_TWX_EDC_NAME;
1016                 break;
1017         case EDC_TWX_AEL2020:
1018                 fw_name = AEL2020_TWX_EDC_NAME;
1019                 break;
1020         }
1021         return fw_name;
1022 }
1023
1024 int t3_get_edc_fw(struct cphy *phy, int edc_idx, int size)
1025 {
1026         struct adapter *adapter = phy->adapter;
1027         const struct firmware *fw;
1028         char buf[64];
1029         u32 csum;
1030         const __be32 *p;
1031         u16 *cache = phy->phy_cache;
1032         int i, ret;
1033
1034         snprintf(buf, sizeof(buf), get_edc_fw_name(edc_idx));
1035
1036         ret = request_firmware(&fw, buf, &adapter->pdev->dev);
1037         if (ret < 0) {
1038                 dev_err(&adapter->pdev->dev,
1039                         "could not upgrade firmware: unable to load %s\n",
1040                         buf);
1041                 return ret;
1042         }
1043
1044         /* check size, take checksum in account */
1045         if (fw->size > size + 4) {
1046                 CH_ERR(adapter, "firmware image too large %u, expected %d\n",
1047                        (unsigned int)fw->size, size + 4);
1048                 ret = -EINVAL;
1049         }
1050
1051         /* compute checksum */
1052         p = (const __be32 *)fw->data;
1053         for (csum = 0, i = 0; i < fw->size / sizeof(csum); i++)
1054                 csum += ntohl(p[i]);
1055
1056         if (csum != 0xffffffff) {
1057                 CH_ERR(adapter, "corrupted firmware image, checksum %u\n",
1058                        csum);
1059                 ret = -EINVAL;
1060         }
1061
1062         for (i = 0; i < size / 4 ; i++) {
1063                 *cache++ = (be32_to_cpu(p[i]) & 0xffff0000) >> 16;
1064                 *cache++ = be32_to_cpu(p[i]) & 0xffff;
1065         }
1066
1067         release_firmware(fw);
1068
1069         return ret;
1070 }
1071
1072 static int upgrade_fw(struct adapter *adap)
1073 {
1074         int ret;
1075         const struct firmware *fw;
1076         struct device *dev = &adap->pdev->dev;
1077
1078         ret = request_firmware(&fw, FW_FNAME, dev);
1079         if (ret < 0) {
1080                 dev_err(dev, "could not upgrade firmware: unable to load %s\n",
1081                         FW_FNAME);
1082                 return ret;
1083         }
1084         ret = t3_load_fw(adap, fw->data, fw->size);
1085         release_firmware(fw);
1086
1087         if (ret == 0)
1088                 dev_info(dev, "successful upgrade to firmware %d.%d.%d\n",
1089                          FW_VERSION_MAJOR, FW_VERSION_MINOR, FW_VERSION_MICRO);
1090         else
1091                 dev_err(dev, "failed to upgrade to firmware %d.%d.%d\n",
1092                         FW_VERSION_MAJOR, FW_VERSION_MINOR, FW_VERSION_MICRO);
1093
1094         return ret;
1095 }
1096
1097 static inline char t3rev2char(struct adapter *adapter)
1098 {
1099         char rev = 0;
1100
1101         switch(adapter->params.rev) {
1102         case T3_REV_B:
1103         case T3_REV_B2:
1104                 rev = 'b';
1105                 break;
1106         case T3_REV_C:
1107                 rev = 'c';
1108                 break;
1109         }
1110         return rev;
1111 }
1112
1113 static int update_tpsram(struct adapter *adap)
1114 {
1115         const struct firmware *tpsram;
1116         char buf[64];
1117         struct device *dev = &adap->pdev->dev;
1118         int ret;
1119         char rev;
1120
1121         rev = t3rev2char(adap);
1122         if (!rev)
1123                 return 0;
1124
1125         snprintf(buf, sizeof(buf), TPSRAM_NAME, rev);
1126
1127         ret = request_firmware(&tpsram, buf, dev);
1128         if (ret < 0) {
1129                 dev_err(dev, "could not load TP SRAM: unable to load %s\n",
1130                         buf);
1131                 return ret;
1132         }
1133
1134         ret = t3_check_tpsram(adap, tpsram->data, tpsram->size);
1135         if (ret)
1136                 goto release_tpsram;
1137
1138         ret = t3_set_proto_sram(adap, tpsram->data);
1139         if (ret == 0)
1140                 dev_info(dev,
1141                          "successful update of protocol engine "
1142                          "to %d.%d.%d\n",
1143                          TP_VERSION_MAJOR, TP_VERSION_MINOR, TP_VERSION_MICRO);
1144         else
1145                 dev_err(dev, "failed to update of protocol engine %d.%d.%d\n",
1146                         TP_VERSION_MAJOR, TP_VERSION_MINOR, TP_VERSION_MICRO);
1147         if (ret)
1148                 dev_err(dev, "loading protocol SRAM failed\n");
1149
1150 release_tpsram:
1151         release_firmware(tpsram);
1152
1153         return ret;
1154 }
1155
1156 /**
1157  * t3_synchronize_rx - wait for current Rx processing on a port to complete
1158  * @adap: the adapter
1159  * @p: the port
1160  *
1161  * Ensures that current Rx processing on any of the queues associated with
1162  * the given port completes before returning.  We do this by acquiring and
1163  * releasing the locks of the response queues associated with the port.
1164  */
1165 static void t3_synchronize_rx(struct adapter *adap, const struct port_info *p)
1166 {
1167         int i;
1168
1169         for (i = p->first_qset; i < p->first_qset + p->nqsets; i++) {
1170                 struct sge_rspq *q = &adap->sge.qs[i].rspq;
1171
1172                 spin_lock_irq(&q->lock);
1173                 spin_unlock_irq(&q->lock);
1174         }
1175 }
1176
1177 static void cxgb_vlan_mode(struct net_device *dev, netdev_features_t features)
1178 {
1179         struct port_info *pi = netdev_priv(dev);
1180         struct adapter *adapter = pi->adapter;
1181
1182         if (adapter->params.rev > 0) {
1183                 t3_set_vlan_accel(adapter, 1 << pi->port_id,
1184                                   features & NETIF_F_HW_VLAN_RX);
1185         } else {
1186                 /* single control for all ports */
1187                 unsigned int i, have_vlans = features & NETIF_F_HW_VLAN_RX;
1188
1189                 for_each_port(adapter, i)
1190                         have_vlans |=
1191                                 adapter->port[i]->features & NETIF_F_HW_VLAN_RX;
1192
1193                 t3_set_vlan_accel(adapter, 1, have_vlans);
1194         }
1195         t3_synchronize_rx(adapter, pi);
1196 }
1197
1198 /**
1199  *      cxgb_up - enable the adapter
1200  *      @adapter: adapter being enabled
1201  *
1202  *      Called when the first port is enabled, this function performs the
1203  *      actions necessary to make an adapter operational, such as completing
1204  *      the initialization of HW modules, and enabling interrupts.
1205  *
1206  *      Must be called with the rtnl lock held.
1207  */
1208 static int cxgb_up(struct adapter *adap)
1209 {
1210         int i, err;
1211
1212         if (!(adap->flags & FULL_INIT_DONE)) {
1213                 err = t3_check_fw_version(adap);
1214                 if (err == -EINVAL) {
1215                         err = upgrade_fw(adap);
1216                         CH_WARN(adap, "FW upgrade to %d.%d.%d %s\n",
1217                                 FW_VERSION_MAJOR, FW_VERSION_MINOR,
1218                                 FW_VERSION_MICRO, err ? "failed" : "succeeded");
1219                 }
1220
1221                 err = t3_check_tpsram_version(adap);
1222                 if (err == -EINVAL) {
1223                         err = update_tpsram(adap);
1224                         CH_WARN(adap, "TP upgrade to %d.%d.%d %s\n",
1225                                 TP_VERSION_MAJOR, TP_VERSION_MINOR,
1226                                 TP_VERSION_MICRO, err ? "failed" : "succeeded");
1227                 }
1228
1229                 /*
1230                  * Clear interrupts now to catch errors if t3_init_hw fails.
1231                  * We clear them again later as initialization may trigger
1232                  * conditions that can interrupt.
1233                  */
1234                 t3_intr_clear(adap);
1235
1236                 err = t3_init_hw(adap, 0);
1237                 if (err)
1238                         goto out;
1239
1240                 t3_set_reg_field(adap, A_TP_PARA_REG5, 0, F_RXDDPOFFINIT);
1241                 t3_write_reg(adap, A_ULPRX_TDDP_PSZ, V_HPZ0(PAGE_SHIFT - 12));
1242
1243                 err = setup_sge_qsets(adap);
1244                 if (err)
1245                         goto out;
1246
1247                 for_each_port(adap, i)
1248                         cxgb_vlan_mode(adap->port[i], adap->port[i]->features);
1249
1250                 setup_rss(adap);
1251                 if (!(adap->flags & NAPI_INIT))
1252                         init_napi(adap);
1253
1254                 t3_start_sge_timers(adap);
1255                 adap->flags |= FULL_INIT_DONE;
1256         }
1257
1258         t3_intr_clear(adap);
1259
1260         if (adap->flags & USING_MSIX) {
1261                 name_msix_vecs(adap);
1262                 err = request_irq(adap->msix_info[0].vec,
1263                                   t3_async_intr_handler, 0,
1264                                   adap->msix_info[0].desc, adap);
1265                 if (err)
1266                         goto irq_err;
1267
1268                 err = request_msix_data_irqs(adap);
1269                 if (err) {
1270                         free_irq(adap->msix_info[0].vec, adap);
1271                         goto irq_err;
1272                 }
1273         } else if ((err = request_irq(adap->pdev->irq,
1274                                       t3_intr_handler(adap,
1275                                                       adap->sge.qs[0].rspq.
1276                                                       polling),
1277                                       (adap->flags & USING_MSI) ?
1278                                        0 : IRQF_SHARED,
1279                                       adap->name, adap)))
1280                 goto irq_err;
1281
1282         enable_all_napi(adap);
1283         t3_sge_start(adap);
1284         t3_intr_enable(adap);
1285
1286         if (adap->params.rev >= T3_REV_C && !(adap->flags & TP_PARITY_INIT) &&
1287             is_offload(adap) && init_tp_parity(adap) == 0)
1288                 adap->flags |= TP_PARITY_INIT;
1289
1290         if (adap->flags & TP_PARITY_INIT) {
1291                 t3_write_reg(adap, A_TP_INT_CAUSE,
1292                              F_CMCACHEPERR | F_ARPLUTPERR);
1293                 t3_write_reg(adap, A_TP_INT_ENABLE, 0x7fbfffff);
1294         }
1295
1296         if (!(adap->flags & QUEUES_BOUND)) {
1297                 int ret = bind_qsets(adap);
1298
1299                 if (ret < 0) {
1300                         CH_ERR(adap, "failed to bind qsets, err %d\n", ret);
1301                         t3_intr_disable(adap);
1302                         free_irq_resources(adap);
1303                         err = ret;
1304                         goto out;
1305                 }
1306                 adap->flags |= QUEUES_BOUND;
1307         }
1308
1309 out:
1310         return err;
1311 irq_err:
1312         CH_ERR(adap, "request_irq failed, err %d\n", err);
1313         goto out;
1314 }
1315
1316 /*
1317  * Release resources when all the ports and offloading have been stopped.
1318  */
1319 static void cxgb_down(struct adapter *adapter, int on_wq)
1320 {
1321         t3_sge_stop(adapter);
1322         spin_lock_irq(&adapter->work_lock);     /* sync with PHY intr task */
1323         t3_intr_disable(adapter);
1324         spin_unlock_irq(&adapter->work_lock);
1325
1326         free_irq_resources(adapter);
1327         quiesce_rx(adapter);
1328         t3_sge_stop(adapter);
1329         if (!on_wq)
1330                 flush_workqueue(cxgb3_wq);/* wait for external IRQ handler */
1331 }
1332
1333 static void schedule_chk_task(struct adapter *adap)
1334 {
1335         unsigned int timeo;
1336
1337         timeo = adap->params.linkpoll_period ?
1338             (HZ * adap->params.linkpoll_period) / 10 :
1339             adap->params.stats_update_period * HZ;
1340         if (timeo)
1341                 queue_delayed_work(cxgb3_wq, &adap->adap_check_task, timeo);
1342 }
1343
1344 static int offload_open(struct net_device *dev)
1345 {
1346         struct port_info *pi = netdev_priv(dev);
1347         struct adapter *adapter = pi->adapter;
1348         struct t3cdev *tdev = dev2t3cdev(dev);
1349         int adap_up = adapter->open_device_map & PORT_MASK;
1350         int err;
1351
1352         if (test_and_set_bit(OFFLOAD_DEVMAP_BIT, &adapter->open_device_map))
1353                 return 0;
1354
1355         if (!adap_up && (err = cxgb_up(adapter)) < 0)
1356                 goto out;
1357
1358         t3_tp_set_offload_mode(adapter, 1);
1359         tdev->lldev = adapter->port[0];
1360         err = cxgb3_offload_activate(adapter);
1361         if (err)
1362                 goto out;
1363
1364         init_port_mtus(adapter);
1365         t3_load_mtus(adapter, adapter->params.mtus, adapter->params.a_wnd,
1366                      adapter->params.b_wnd,
1367                      adapter->params.rev == 0 ?
1368                      adapter->port[0]->mtu : 0xffff);
1369         init_smt(adapter);
1370
1371         if (sysfs_create_group(&tdev->lldev->dev.kobj, &offload_attr_group))
1372                 dev_dbg(&dev->dev, "cannot create sysfs group\n");
1373
1374         /* Call back all registered clients */
1375         cxgb3_add_clients(tdev);
1376
1377 out:
1378         /* restore them in case the offload module has changed them */
1379         if (err) {
1380                 t3_tp_set_offload_mode(adapter, 0);
1381                 clear_bit(OFFLOAD_DEVMAP_BIT, &adapter->open_device_map);
1382                 cxgb3_set_dummy_ops(tdev);
1383         }
1384         return err;
1385 }
1386
1387 static int offload_close(struct t3cdev *tdev)
1388 {
1389         struct adapter *adapter = tdev2adap(tdev);
1390         struct t3c_data *td = T3C_DATA(tdev);
1391
1392         if (!test_bit(OFFLOAD_DEVMAP_BIT, &adapter->open_device_map))
1393                 return 0;
1394
1395         /* Call back all registered clients */
1396         cxgb3_remove_clients(tdev);
1397
1398         sysfs_remove_group(&tdev->lldev->dev.kobj, &offload_attr_group);
1399
1400         /* Flush work scheduled while releasing TIDs */
1401         flush_work(&td->tid_release_task);
1402
1403         tdev->lldev = NULL;
1404         cxgb3_set_dummy_ops(tdev);
1405         t3_tp_set_offload_mode(adapter, 0);
1406         clear_bit(OFFLOAD_DEVMAP_BIT, &adapter->open_device_map);
1407
1408         if (!adapter->open_device_map)
1409                 cxgb_down(adapter, 0);
1410
1411         cxgb3_offload_deactivate(adapter);
1412         return 0;
1413 }
1414
1415 static int cxgb_open(struct net_device *dev)
1416 {
1417         struct port_info *pi = netdev_priv(dev);
1418         struct adapter *adapter = pi->adapter;
1419         int other_ports = adapter->open_device_map & PORT_MASK;
1420         int err;
1421
1422         if (!adapter->open_device_map && (err = cxgb_up(adapter)) < 0)
1423                 return err;
1424
1425         set_bit(pi->port_id, &adapter->open_device_map);
1426         if (is_offload(adapter) && !ofld_disable) {
1427                 err = offload_open(dev);
1428                 if (err)
1429                         pr_warn("Could not initialize offload capabilities\n");
1430         }
1431
1432         netif_set_real_num_tx_queues(dev, pi->nqsets);
1433         err = netif_set_real_num_rx_queues(dev, pi->nqsets);
1434         if (err)
1435                 return err;
1436         link_start(dev);
1437         t3_port_intr_enable(adapter, pi->port_id);
1438         netif_tx_start_all_queues(dev);
1439         if (!other_ports)
1440                 schedule_chk_task(adapter);
1441
1442         cxgb3_event_notify(&adapter->tdev, OFFLOAD_PORT_UP, pi->port_id);
1443         return 0;
1444 }
1445
1446 static int __cxgb_close(struct net_device *dev, int on_wq)
1447 {
1448         struct port_info *pi = netdev_priv(dev);
1449         struct adapter *adapter = pi->adapter;
1450
1451
1452         if (!adapter->open_device_map)
1453                 return 0;
1454
1455         /* Stop link fault interrupts */
1456         t3_xgm_intr_disable(adapter, pi->port_id);
1457         t3_read_reg(adapter, A_XGM_INT_STATUS + pi->mac.offset);
1458
1459         t3_port_intr_disable(adapter, pi->port_id);
1460         netif_tx_stop_all_queues(dev);
1461         pi->phy.ops->power_down(&pi->phy, 1);
1462         netif_carrier_off(dev);
1463         t3_mac_disable(&pi->mac, MAC_DIRECTION_TX | MAC_DIRECTION_RX);
1464
1465         spin_lock_irq(&adapter->work_lock);     /* sync with update task */
1466         clear_bit(pi->port_id, &adapter->open_device_map);
1467         spin_unlock_irq(&adapter->work_lock);
1468
1469         if (!(adapter->open_device_map & PORT_MASK))
1470                 cancel_delayed_work_sync(&adapter->adap_check_task);
1471
1472         if (!adapter->open_device_map)
1473                 cxgb_down(adapter, on_wq);
1474
1475         cxgb3_event_notify(&adapter->tdev, OFFLOAD_PORT_DOWN, pi->port_id);
1476         return 0;
1477 }
1478
1479 static int cxgb_close(struct net_device *dev)
1480 {
1481         return __cxgb_close(dev, 0);
1482 }
1483
1484 static struct net_device_stats *cxgb_get_stats(struct net_device *dev)
1485 {
1486         struct port_info *pi = netdev_priv(dev);
1487         struct adapter *adapter = pi->adapter;
1488         struct net_device_stats *ns = &pi->netstats;
1489         const struct mac_stats *pstats;
1490
1491         spin_lock(&adapter->stats_lock);
1492         pstats = t3_mac_update_stats(&pi->mac);
1493         spin_unlock(&adapter->stats_lock);
1494
1495         ns->tx_bytes = pstats->tx_octets;
1496         ns->tx_packets = pstats->tx_frames;
1497         ns->rx_bytes = pstats->rx_octets;
1498         ns->rx_packets = pstats->rx_frames;
1499         ns->multicast = pstats->rx_mcast_frames;
1500
1501         ns->tx_errors = pstats->tx_underrun;
1502         ns->rx_errors = pstats->rx_symbol_errs + pstats->rx_fcs_errs +
1503             pstats->rx_too_long + pstats->rx_jabber + pstats->rx_short +
1504             pstats->rx_fifo_ovfl;
1505
1506         /* detailed rx_errors */
1507         ns->rx_length_errors = pstats->rx_jabber + pstats->rx_too_long;
1508         ns->rx_over_errors = 0;
1509         ns->rx_crc_errors = pstats->rx_fcs_errs;
1510         ns->rx_frame_errors = pstats->rx_symbol_errs;
1511         ns->rx_fifo_errors = pstats->rx_fifo_ovfl;
1512         ns->rx_missed_errors = pstats->rx_cong_drops;
1513
1514         /* detailed tx_errors */
1515         ns->tx_aborted_errors = 0;
1516         ns->tx_carrier_errors = 0;
1517         ns->tx_fifo_errors = pstats->tx_underrun;
1518         ns->tx_heartbeat_errors = 0;
1519         ns->tx_window_errors = 0;
1520         return ns;
1521 }
1522
1523 static u32 get_msglevel(struct net_device *dev)
1524 {
1525         struct port_info *pi = netdev_priv(dev);
1526         struct adapter *adapter = pi->adapter;
1527
1528         return adapter->msg_enable;
1529 }
1530
1531 static void set_msglevel(struct net_device *dev, u32 val)
1532 {
1533         struct port_info *pi = netdev_priv(dev);
1534         struct adapter *adapter = pi->adapter;
1535
1536         adapter->msg_enable = val;
1537 }
1538
1539 static char stats_strings[][ETH_GSTRING_LEN] = {
1540         "TxOctetsOK         ",
1541         "TxFramesOK         ",
1542         "TxMulticastFramesOK",
1543         "TxBroadcastFramesOK",
1544         "TxPauseFrames      ",
1545         "TxUnderrun         ",
1546         "TxExtUnderrun      ",
1547
1548         "TxFrames64         ",
1549         "TxFrames65To127    ",
1550         "TxFrames128To255   ",
1551         "TxFrames256To511   ",
1552         "TxFrames512To1023  ",
1553         "TxFrames1024To1518 ",
1554         "TxFrames1519ToMax  ",
1555
1556         "RxOctetsOK         ",
1557         "RxFramesOK         ",
1558         "RxMulticastFramesOK",
1559         "RxBroadcastFramesOK",
1560         "RxPauseFrames      ",
1561         "RxFCSErrors        ",
1562         "RxSymbolErrors     ",
1563         "RxShortErrors      ",
1564         "RxJabberErrors     ",
1565         "RxLengthErrors     ",
1566         "RxFIFOoverflow     ",
1567
1568         "RxFrames64         ",
1569         "RxFrames65To127    ",
1570         "RxFrames128To255   ",
1571         "RxFrames256To511   ",
1572         "RxFrames512To1023  ",
1573         "RxFrames1024To1518 ",
1574         "RxFrames1519ToMax  ",
1575
1576         "PhyFIFOErrors      ",
1577         "TSO                ",
1578         "VLANextractions    ",
1579         "VLANinsertions     ",
1580         "TxCsumOffload      ",
1581         "RxCsumGood         ",
1582         "LroAggregated      ",
1583         "LroFlushed         ",
1584         "LroNoDesc          ",
1585         "RxDrops            ",
1586
1587         "CheckTXEnToggled   ",
1588         "CheckResets        ",
1589
1590         "LinkFaults         ",
1591 };
1592
1593 static int get_sset_count(struct net_device *dev, int sset)
1594 {
1595         switch (sset) {
1596         case ETH_SS_STATS:
1597                 return ARRAY_SIZE(stats_strings);
1598         default:
1599                 return -EOPNOTSUPP;
1600         }
1601 }
1602
1603 #define T3_REGMAP_SIZE (3 * 1024)
1604
1605 static int get_regs_len(struct net_device *dev)
1606 {
1607         return T3_REGMAP_SIZE;
1608 }
1609
1610 static int get_eeprom_len(struct net_device *dev)
1611 {
1612         return EEPROMSIZE;
1613 }
1614
1615 static void get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info)
1616 {
1617         struct port_info *pi = netdev_priv(dev);
1618         struct adapter *adapter = pi->adapter;
1619         u32 fw_vers = 0;
1620         u32 tp_vers = 0;
1621
1622         spin_lock(&adapter->stats_lock);
1623         t3_get_fw_version(adapter, &fw_vers);
1624         t3_get_tp_version(adapter, &tp_vers);
1625         spin_unlock(&adapter->stats_lock);
1626
1627         strlcpy(info->driver, DRV_NAME, sizeof(info->driver));
1628         strlcpy(info->version, DRV_VERSION, sizeof(info->version));
1629         strlcpy(info->bus_info, pci_name(adapter->pdev),
1630                 sizeof(info->bus_info));
1631         if (fw_vers)
1632                 snprintf(info->fw_version, sizeof(info->fw_version),
1633                          "%s %u.%u.%u TP %u.%u.%u",
1634                          G_FW_VERSION_TYPE(fw_vers) ? "T" : "N",
1635                          G_FW_VERSION_MAJOR(fw_vers),
1636                          G_FW_VERSION_MINOR(fw_vers),
1637                          G_FW_VERSION_MICRO(fw_vers),
1638                          G_TP_VERSION_MAJOR(tp_vers),
1639                          G_TP_VERSION_MINOR(tp_vers),
1640                          G_TP_VERSION_MICRO(tp_vers));
1641 }
1642
1643 static void get_strings(struct net_device *dev, u32 stringset, u8 * data)
1644 {
1645         if (stringset == ETH_SS_STATS)
1646                 memcpy(data, stats_strings, sizeof(stats_strings));
1647 }
1648
1649 static unsigned long collect_sge_port_stats(struct adapter *adapter,
1650                                             struct port_info *p, int idx)
1651 {
1652         int i;
1653         unsigned long tot = 0;
1654
1655         for (i = p->first_qset; i < p->first_qset + p->nqsets; ++i)
1656                 tot += adapter->sge.qs[i].port_stats[idx];
1657         return tot;
1658 }
1659
1660 static void get_stats(struct net_device *dev, struct ethtool_stats *stats,
1661                       u64 *data)
1662 {
1663         struct port_info *pi = netdev_priv(dev);
1664         struct adapter *adapter = pi->adapter;
1665         const struct mac_stats *s;
1666
1667         spin_lock(&adapter->stats_lock);
1668         s = t3_mac_update_stats(&pi->mac);
1669         spin_unlock(&adapter->stats_lock);
1670
1671         *data++ = s->tx_octets;
1672         *data++ = s->tx_frames;
1673         *data++ = s->tx_mcast_frames;
1674         *data++ = s->tx_bcast_frames;
1675         *data++ = s->tx_pause;
1676         *data++ = s->tx_underrun;
1677         *data++ = s->tx_fifo_urun;
1678
1679         *data++ = s->tx_frames_64;
1680         *data++ = s->tx_frames_65_127;
1681         *data++ = s->tx_frames_128_255;
1682         *data++ = s->tx_frames_256_511;
1683         *data++ = s->tx_frames_512_1023;
1684         *data++ = s->tx_frames_1024_1518;
1685         *data++ = s->tx_frames_1519_max;
1686
1687         *data++ = s->rx_octets;
1688         *data++ = s->rx_frames;
1689         *data++ = s->rx_mcast_frames;
1690         *data++ = s->rx_bcast_frames;
1691         *data++ = s->rx_pause;
1692         *data++ = s->rx_fcs_errs;
1693         *data++ = s->rx_symbol_errs;
1694         *data++ = s->rx_short;
1695         *data++ = s->rx_jabber;
1696         *data++ = s->rx_too_long;
1697         *data++ = s->rx_fifo_ovfl;
1698
1699         *data++ = s->rx_frames_64;
1700         *data++ = s->rx_frames_65_127;
1701         *data++ = s->rx_frames_128_255;
1702         *data++ = s->rx_frames_256_511;
1703         *data++ = s->rx_frames_512_1023;
1704         *data++ = s->rx_frames_1024_1518;
1705         *data++ = s->rx_frames_1519_max;
1706
1707         *data++ = pi->phy.fifo_errors;
1708
1709         *data++ = collect_sge_port_stats(adapter, pi, SGE_PSTAT_TSO);
1710         *data++ = collect_sge_port_stats(adapter, pi, SGE_PSTAT_VLANEX);
1711         *data++ = collect_sge_port_stats(adapter, pi, SGE_PSTAT_VLANINS);
1712         *data++ = collect_sge_port_stats(adapter, pi, SGE_PSTAT_TX_CSUM);
1713         *data++ = collect_sge_port_stats(adapter, pi, SGE_PSTAT_RX_CSUM_GOOD);
1714         *data++ = 0;
1715         *data++ = 0;
1716         *data++ = 0;
1717         *data++ = s->rx_cong_drops;
1718
1719         *data++ = s->num_toggled;
1720         *data++ = s->num_resets;
1721
1722         *data++ = s->link_faults;
1723 }
1724
1725 static inline void reg_block_dump(struct adapter *ap, void *buf,
1726                                   unsigned int start, unsigned int end)
1727 {
1728         u32 *p = buf + start;
1729
1730         for (; start <= end; start += sizeof(u32))
1731                 *p++ = t3_read_reg(ap, start);
1732 }
1733
1734 static void get_regs(struct net_device *dev, struct ethtool_regs *regs,
1735                      void *buf)
1736 {
1737         struct port_info *pi = netdev_priv(dev);
1738         struct adapter *ap = pi->adapter;
1739
1740         /*
1741          * Version scheme:
1742          * bits 0..9: chip version
1743          * bits 10..15: chip revision
1744          * bit 31: set for PCIe cards
1745          */
1746         regs->version = 3 | (ap->params.rev << 10) | (is_pcie(ap) << 31);
1747
1748         /*
1749          * We skip the MAC statistics registers because they are clear-on-read.
1750          * Also reading multi-register stats would need to synchronize with the
1751          * periodic mac stats accumulation.  Hard to justify the complexity.
1752          */
1753         memset(buf, 0, T3_REGMAP_SIZE);
1754         reg_block_dump(ap, buf, 0, A_SG_RSPQ_CREDIT_RETURN);
1755         reg_block_dump(ap, buf, A_SG_HI_DRB_HI_THRSH, A_ULPRX_PBL_ULIMIT);
1756         reg_block_dump(ap, buf, A_ULPTX_CONFIG, A_MPS_INT_CAUSE);
1757         reg_block_dump(ap, buf, A_CPL_SWITCH_CNTRL, A_CPL_MAP_TBL_DATA);
1758         reg_block_dump(ap, buf, A_SMB_GLOBAL_TIME_CFG, A_XGM_SERDES_STAT3);
1759         reg_block_dump(ap, buf, A_XGM_SERDES_STATUS0,
1760                        XGM_REG(A_XGM_SERDES_STAT3, 1));
1761         reg_block_dump(ap, buf, XGM_REG(A_XGM_SERDES_STATUS0, 1),
1762                        XGM_REG(A_XGM_RX_SPI4_SOP_EOP_CNT, 1));
1763 }
1764
1765 static int restart_autoneg(struct net_device *dev)
1766 {
1767         struct port_info *p = netdev_priv(dev);
1768
1769         if (!netif_running(dev))
1770                 return -EAGAIN;
1771         if (p->link_config.autoneg != AUTONEG_ENABLE)
1772                 return -EINVAL;
1773         p->phy.ops->autoneg_restart(&p->phy);
1774         return 0;
1775 }
1776
1777 static int set_phys_id(struct net_device *dev,
1778                        enum ethtool_phys_id_state state)
1779 {
1780         struct port_info *pi = netdev_priv(dev);
1781         struct adapter *adapter = pi->adapter;
1782
1783         switch (state) {
1784         case ETHTOOL_ID_ACTIVE:
1785                 return 1;       /* cycle on/off once per second */
1786
1787         case ETHTOOL_ID_OFF:
1788                 t3_set_reg_field(adapter, A_T3DBG_GPIO_EN, F_GPIO0_OUT_VAL, 0);
1789                 break;
1790
1791         case ETHTOOL_ID_ON:
1792         case ETHTOOL_ID_INACTIVE:
1793                 t3_set_reg_field(adapter, A_T3DBG_GPIO_EN, F_GPIO0_OUT_VAL,
1794                          F_GPIO0_OUT_VAL);
1795         }
1796
1797         return 0;
1798 }
1799
1800 static int get_settings(struct net_device *dev, struct ethtool_cmd *cmd)
1801 {
1802         struct port_info *p = netdev_priv(dev);
1803
1804         cmd->supported = p->link_config.supported;
1805         cmd->advertising = p->link_config.advertising;
1806
1807         if (netif_carrier_ok(dev)) {
1808                 ethtool_cmd_speed_set(cmd, p->link_config.speed);
1809                 cmd->duplex = p->link_config.duplex;
1810         } else {
1811                 ethtool_cmd_speed_set(cmd, -1);
1812                 cmd->duplex = -1;
1813         }
1814
1815         cmd->port = (cmd->supported & SUPPORTED_TP) ? PORT_TP : PORT_FIBRE;
1816         cmd->phy_address = p->phy.mdio.prtad;
1817         cmd->transceiver = XCVR_EXTERNAL;
1818         cmd->autoneg = p->link_config.autoneg;
1819         cmd->maxtxpkt = 0;
1820         cmd->maxrxpkt = 0;
1821         return 0;
1822 }
1823
1824 static int speed_duplex_to_caps(int speed, int duplex)
1825 {
1826         int cap = 0;
1827
1828         switch (speed) {
1829         case SPEED_10:
1830                 if (duplex == DUPLEX_FULL)
1831                         cap = SUPPORTED_10baseT_Full;
1832                 else
1833                         cap = SUPPORTED_10baseT_Half;
1834                 break;
1835         case SPEED_100:
1836                 if (duplex == DUPLEX_FULL)
1837                         cap = SUPPORTED_100baseT_Full;
1838                 else
1839                         cap = SUPPORTED_100baseT_Half;
1840                 break;
1841         case SPEED_1000:
1842                 if (duplex == DUPLEX_FULL)
1843                         cap = SUPPORTED_1000baseT_Full;
1844                 else
1845                         cap = SUPPORTED_1000baseT_Half;
1846                 break;
1847         case SPEED_10000:
1848                 if (duplex == DUPLEX_FULL)
1849                         cap = SUPPORTED_10000baseT_Full;
1850         }
1851         return cap;
1852 }
1853
1854 #define ADVERTISED_MASK (ADVERTISED_10baseT_Half | ADVERTISED_10baseT_Full | \
1855                       ADVERTISED_100baseT_Half | ADVERTISED_100baseT_Full | \
1856                       ADVERTISED_1000baseT_Half | ADVERTISED_1000baseT_Full | \
1857                       ADVERTISED_10000baseT_Full)
1858
1859 static int set_settings(struct net_device *dev, struct ethtool_cmd *cmd)
1860 {
1861         struct port_info *p = netdev_priv(dev);
1862         struct link_config *lc = &p->link_config;
1863
1864         if (!(lc->supported & SUPPORTED_Autoneg)) {
1865                 /*
1866                  * PHY offers a single speed/duplex.  See if that's what's
1867                  * being requested.
1868                  */
1869                 if (cmd->autoneg == AUTONEG_DISABLE) {
1870                         u32 speed = ethtool_cmd_speed(cmd);
1871                         int cap = speed_duplex_to_caps(speed, cmd->duplex);
1872                         if (lc->supported & cap)
1873                                 return 0;
1874                 }
1875                 return -EINVAL;
1876         }
1877
1878         if (cmd->autoneg == AUTONEG_DISABLE) {
1879                 u32 speed = ethtool_cmd_speed(cmd);
1880                 int cap = speed_duplex_to_caps(speed, cmd->duplex);
1881
1882                 if (!(lc->supported & cap) || (speed == SPEED_1000))
1883                         return -EINVAL;
1884                 lc->requested_speed = speed;
1885                 lc->requested_duplex = cmd->duplex;
1886                 lc->advertising = 0;
1887         } else {
1888                 cmd->advertising &= ADVERTISED_MASK;
1889                 cmd->advertising &= lc->supported;
1890                 if (!cmd->advertising)
1891                         return -EINVAL;
1892                 lc->requested_speed = SPEED_INVALID;
1893                 lc->requested_duplex = DUPLEX_INVALID;
1894                 lc->advertising = cmd->advertising | ADVERTISED_Autoneg;
1895         }
1896         lc->autoneg = cmd->autoneg;
1897         if (netif_running(dev))
1898                 t3_link_start(&p->phy, &p->mac, lc);
1899         return 0;
1900 }
1901
1902 static void get_pauseparam(struct net_device *dev,
1903                            struct ethtool_pauseparam *epause)
1904 {
1905         struct port_info *p = netdev_priv(dev);
1906
1907         epause->autoneg = (p->link_config.requested_fc & PAUSE_AUTONEG) != 0;
1908         epause->rx_pause = (p->link_config.fc & PAUSE_RX) != 0;
1909         epause->tx_pause = (p->link_config.fc & PAUSE_TX) != 0;
1910 }
1911
1912 static int set_pauseparam(struct net_device *dev,
1913                           struct ethtool_pauseparam *epause)
1914 {
1915         struct port_info *p = netdev_priv(dev);
1916         struct link_config *lc = &p->link_config;
1917
1918         if (epause->autoneg == AUTONEG_DISABLE)
1919                 lc->requested_fc = 0;
1920         else if (lc->supported & SUPPORTED_Autoneg)
1921                 lc->requested_fc = PAUSE_AUTONEG;
1922         else
1923                 return -EINVAL;
1924
1925         if (epause->rx_pause)
1926                 lc->requested_fc |= PAUSE_RX;
1927         if (epause->tx_pause)
1928                 lc->requested_fc |= PAUSE_TX;
1929         if (lc->autoneg == AUTONEG_ENABLE) {
1930                 if (netif_running(dev))
1931                         t3_link_start(&p->phy, &p->mac, lc);
1932         } else {
1933                 lc->fc = lc->requested_fc & (PAUSE_RX | PAUSE_TX);
1934                 if (netif_running(dev))
1935                         t3_mac_set_speed_duplex_fc(&p->mac, -1, -1, lc->fc);
1936         }
1937         return 0;
1938 }
1939
1940 static void get_sge_param(struct net_device *dev, struct ethtool_ringparam *e)
1941 {
1942         struct port_info *pi = netdev_priv(dev);
1943         struct adapter *adapter = pi->adapter;
1944         const struct qset_params *q = &adapter->params.sge.qset[pi->first_qset];
1945
1946         e->rx_max_pending = MAX_RX_BUFFERS;
1947         e->rx_jumbo_max_pending = MAX_RX_JUMBO_BUFFERS;
1948         e->tx_max_pending = MAX_TXQ_ENTRIES;
1949
1950         e->rx_pending = q->fl_size;
1951         e->rx_mini_pending = q->rspq_size;
1952         e->rx_jumbo_pending = q->jumbo_size;
1953         e->tx_pending = q->txq_size[0];
1954 }
1955
1956 static int set_sge_param(struct net_device *dev, struct ethtool_ringparam *e)
1957 {
1958         struct port_info *pi = netdev_priv(dev);
1959         struct adapter *adapter = pi->adapter;
1960         struct qset_params *q;
1961         int i;
1962
1963         if (e->rx_pending > MAX_RX_BUFFERS ||
1964             e->rx_jumbo_pending > MAX_RX_JUMBO_BUFFERS ||
1965             e->tx_pending > MAX_TXQ_ENTRIES ||
1966             e->rx_mini_pending > MAX_RSPQ_ENTRIES ||
1967             e->rx_mini_pending < MIN_RSPQ_ENTRIES ||
1968             e->rx_pending < MIN_FL_ENTRIES ||
1969             e->rx_jumbo_pending < MIN_FL_ENTRIES ||
1970             e->tx_pending < adapter->params.nports * MIN_TXQ_ENTRIES)
1971                 return -EINVAL;
1972
1973         if (adapter->flags & FULL_INIT_DONE)
1974                 return -EBUSY;
1975
1976         q = &adapter->params.sge.qset[pi->first_qset];
1977         for (i = 0; i < pi->nqsets; ++i, ++q) {
1978                 q->rspq_size = e->rx_mini_pending;
1979                 q->fl_size = e->rx_pending;
1980                 q->jumbo_size = e->rx_jumbo_pending;
1981                 q->txq_size[0] = e->tx_pending;
1982                 q->txq_size[1] = e->tx_pending;
1983                 q->txq_size[2] = e->tx_pending;
1984         }
1985         return 0;
1986 }
1987
1988 static int set_coalesce(struct net_device *dev, struct ethtool_coalesce *c)
1989 {
1990         struct port_info *pi = netdev_priv(dev);
1991         struct adapter *adapter = pi->adapter;
1992         struct qset_params *qsp;
1993         struct sge_qset *qs;
1994         int i;
1995
1996         if (c->rx_coalesce_usecs * 10 > M_NEWTIMER)
1997                 return -EINVAL;
1998
1999         for (i = 0; i < pi->nqsets; i++) {
2000                 qsp = &adapter->params.sge.qset[i];
2001                 qs = &adapter->sge.qs[i];
2002                 qsp->coalesce_usecs = c->rx_coalesce_usecs;
2003                 t3_update_qset_coalesce(qs, qsp);
2004         }
2005
2006         return 0;
2007 }
2008
2009 static int get_coalesce(struct net_device *dev, struct ethtool_coalesce *c)
2010 {
2011         struct port_info *pi = netdev_priv(dev);
2012         struct adapter *adapter = pi->adapter;
2013         struct qset_params *q = adapter->params.sge.qset;
2014
2015         c->rx_coalesce_usecs = q->coalesce_usecs;
2016         return 0;
2017 }
2018
2019 static int get_eeprom(struct net_device *dev, struct ethtool_eeprom *e,
2020                       u8 * data)
2021 {
2022         struct port_info *pi = netdev_priv(dev);
2023         struct adapter *adapter = pi->adapter;
2024         int i, err = 0;
2025
2026         u8 *buf = kmalloc(EEPROMSIZE, GFP_KERNEL);
2027         if (!buf)
2028                 return -ENOMEM;
2029
2030         e->magic = EEPROM_MAGIC;
2031         for (i = e->offset & ~3; !err && i < e->offset + e->len; i += 4)
2032                 err = t3_seeprom_read(adapter, i, (__le32 *) & buf[i]);
2033
2034         if (!err)
2035                 memcpy(data, buf + e->offset, e->len);
2036         kfree(buf);
2037         return err;
2038 }
2039
2040 static int set_eeprom(struct net_device *dev, struct ethtool_eeprom *eeprom,
2041                       u8 * data)
2042 {
2043         struct port_info *pi = netdev_priv(dev);
2044         struct adapter *adapter = pi->adapter;
2045         u32 aligned_offset, aligned_len;
2046         __le32 *p;
2047         u8 *buf;
2048         int err;
2049
2050         if (eeprom->magic != EEPROM_MAGIC)
2051                 return -EINVAL;
2052
2053         aligned_offset = eeprom->offset & ~3;
2054         aligned_len = (eeprom->len + (eeprom->offset & 3) + 3) & ~3;
2055
2056         if (aligned_offset != eeprom->offset || aligned_len != eeprom->len) {
2057                 buf = kmalloc(aligned_len, GFP_KERNEL);
2058                 if (!buf)
2059                         return -ENOMEM;
2060                 err = t3_seeprom_read(adapter, aligned_offset, (__le32 *) buf);
2061                 if (!err && aligned_len > 4)
2062                         err = t3_seeprom_read(adapter,
2063                                               aligned_offset + aligned_len - 4,
2064                                               (__le32 *) & buf[aligned_len - 4]);
2065                 if (err)
2066                         goto out;
2067                 memcpy(buf + (eeprom->offset & 3), data, eeprom->len);
2068         } else
2069                 buf = data;
2070
2071         err = t3_seeprom_wp(adapter, 0);
2072         if (err)
2073                 goto out;
2074
2075         for (p = (__le32 *) buf; !err && aligned_len; aligned_len -= 4, p++) {
2076                 err = t3_seeprom_write(adapter, aligned_offset, *p);
2077                 aligned_offset += 4;
2078         }
2079
2080         if (!err)
2081                 err = t3_seeprom_wp(adapter, 1);
2082 out:
2083         if (buf != data)
2084                 kfree(buf);
2085         return err;
2086 }
2087
2088 static void get_wol(struct net_device *dev, struct ethtool_wolinfo *wol)
2089 {
2090         wol->supported = 0;
2091         wol->wolopts = 0;
2092         memset(&wol->sopass, 0, sizeof(wol->sopass));
2093 }
2094
2095 static const struct ethtool_ops cxgb_ethtool_ops = {
2096         .get_settings = get_settings,
2097         .set_settings = set_settings,
2098         .get_drvinfo = get_drvinfo,
2099         .get_msglevel = get_msglevel,
2100         .set_msglevel = set_msglevel,
2101         .get_ringparam = get_sge_param,
2102         .set_ringparam = set_sge_param,
2103         .get_coalesce = get_coalesce,
2104         .set_coalesce = set_coalesce,
2105         .get_eeprom_len = get_eeprom_len,
2106         .get_eeprom = get_eeprom,
2107         .set_eeprom = set_eeprom,
2108         .get_pauseparam = get_pauseparam,
2109         .set_pauseparam = set_pauseparam,
2110         .get_link = ethtool_op_get_link,
2111         .get_strings = get_strings,
2112         .set_phys_id = set_phys_id,
2113         .nway_reset = restart_autoneg,
2114         .get_sset_count = get_sset_count,
2115         .get_ethtool_stats = get_stats,
2116         .get_regs_len = get_regs_len,
2117         .get_regs = get_regs,
2118         .get_wol = get_wol,
2119 };
2120
2121 static int in_range(int val, int lo, int hi)
2122 {
2123         return val < 0 || (val <= hi && val >= lo);
2124 }
2125
2126 static int cxgb_extension_ioctl(struct net_device *dev, void __user *useraddr)
2127 {
2128         struct port_info *pi = netdev_priv(dev);
2129         struct adapter *adapter = pi->adapter;
2130         u32 cmd;
2131         int ret;
2132
2133         if (copy_from_user(&cmd, useraddr, sizeof(cmd)))
2134                 return -EFAULT;
2135
2136         switch (cmd) {
2137         case CHELSIO_SET_QSET_PARAMS:{
2138                 int i;
2139                 struct qset_params *q;
2140                 struct ch_qset_params t;
2141                 int q1 = pi->first_qset;
2142                 int nqsets = pi->nqsets;
2143
2144                 if (!capable(CAP_NET_ADMIN))
2145                         return -EPERM;
2146                 if (copy_from_user(&t, useraddr, sizeof(t)))
2147                         return -EFAULT;
2148                 if (t.qset_idx >= SGE_QSETS)
2149                         return -EINVAL;
2150                 if (!in_range(t.intr_lat, 0, M_NEWTIMER) ||
2151                     !in_range(t.cong_thres, 0, 255) ||
2152                     !in_range(t.txq_size[0], MIN_TXQ_ENTRIES,
2153                               MAX_TXQ_ENTRIES) ||
2154                     !in_range(t.txq_size[1], MIN_TXQ_ENTRIES,
2155                               MAX_TXQ_ENTRIES) ||
2156                     !in_range(t.txq_size[2], MIN_CTRL_TXQ_ENTRIES,
2157                               MAX_CTRL_TXQ_ENTRIES) ||
2158                     !in_range(t.fl_size[0], MIN_FL_ENTRIES,
2159                               MAX_RX_BUFFERS) ||
2160                     !in_range(t.fl_size[1], MIN_FL_ENTRIES,
2161                               MAX_RX_JUMBO_BUFFERS) ||
2162                     !in_range(t.rspq_size, MIN_RSPQ_ENTRIES,
2163                               MAX_RSPQ_ENTRIES))
2164                         return -EINVAL;
2165
2166                 if ((adapter->flags & FULL_INIT_DONE) &&
2167                         (t.rspq_size >= 0 || t.fl_size[0] >= 0 ||
2168                         t.fl_size[1] >= 0 || t.txq_size[0] >= 0 ||
2169                         t.txq_size[1] >= 0 || t.txq_size[2] >= 0 ||
2170                         t.polling >= 0 || t.cong_thres >= 0))
2171                         return -EBUSY;
2172
2173                 /* Allow setting of any available qset when offload enabled */
2174                 if (test_bit(OFFLOAD_DEVMAP_BIT, &adapter->open_device_map)) {
2175                         q1 = 0;
2176                         for_each_port(adapter, i) {
2177                                 pi = adap2pinfo(adapter, i);
2178                                 nqsets += pi->first_qset + pi->nqsets;
2179                         }
2180                 }
2181
2182                 if (t.qset_idx < q1)
2183                         return -EINVAL;
2184                 if (t.qset_idx > q1 + nqsets - 1)
2185                         return -EINVAL;
2186
2187                 q = &adapter->params.sge.qset[t.qset_idx];
2188
2189                 if (t.rspq_size >= 0)
2190                         q->rspq_size = t.rspq_size;
2191                 if (t.fl_size[0] >= 0)
2192                         q->fl_size = t.fl_size[0];
2193                 if (t.fl_size[1] >= 0)
2194                         q->jumbo_size = t.fl_size[1];
2195                 if (t.txq_size[0] >= 0)
2196                         q->txq_size[0] = t.txq_size[0];
2197                 if (t.txq_size[1] >= 0)
2198                         q->txq_size[1] = t.txq_size[1];
2199                 if (t.txq_size[2] >= 0)
2200                         q->txq_size[2] = t.txq_size[2];
2201                 if (t.cong_thres >= 0)
2202                         q->cong_thres = t.cong_thres;
2203                 if (t.intr_lat >= 0) {
2204                         struct sge_qset *qs =
2205                                 &adapter->sge.qs[t.qset_idx];
2206
2207                         q->coalesce_usecs = t.intr_lat;
2208                         t3_update_qset_coalesce(qs, q);
2209                 }
2210                 if (t.polling >= 0) {
2211                         if (adapter->flags & USING_MSIX)
2212                                 q->polling = t.polling;
2213                         else {
2214                                 /* No polling with INTx for T3A */
2215                                 if (adapter->params.rev == 0 &&
2216                                         !(adapter->flags & USING_MSI))
2217                                         t.polling = 0;
2218
2219                                 for (i = 0; i < SGE_QSETS; i++) {
2220                                         q = &adapter->params.sge.
2221                                                 qset[i];
2222                                         q->polling = t.polling;
2223                                 }
2224                         }
2225                 }
2226
2227                 if (t.lro >= 0) {
2228                         if (t.lro)
2229                                 dev->wanted_features |= NETIF_F_GRO;
2230                         else
2231                                 dev->wanted_features &= ~NETIF_F_GRO;
2232                         netdev_update_features(dev);
2233                 }
2234
2235                 break;
2236         }
2237         case CHELSIO_GET_QSET_PARAMS:{
2238                 struct qset_params *q;
2239                 struct ch_qset_params t;
2240                 int q1 = pi->first_qset;
2241                 int nqsets = pi->nqsets;
2242                 int i;
2243
2244                 if (copy_from_user(&t, useraddr, sizeof(t)))
2245                         return -EFAULT;
2246
2247                 /* Display qsets for all ports when offload enabled */
2248                 if (test_bit(OFFLOAD_DEVMAP_BIT, &adapter->open_device_map)) {
2249                         q1 = 0;
2250                         for_each_port(adapter, i) {
2251                                 pi = adap2pinfo(adapter, i);
2252                                 nqsets = pi->first_qset + pi->nqsets;
2253                         }
2254                 }
2255
2256                 if (t.qset_idx >= nqsets)
2257                         return -EINVAL;
2258
2259                 q = &adapter->params.sge.qset[q1 + t.qset_idx];
2260                 t.rspq_size = q->rspq_size;
2261                 t.txq_size[0] = q->txq_size[0];
2262                 t.txq_size[1] = q->txq_size[1];
2263                 t.txq_size[2] = q->txq_size[2];
2264                 t.fl_size[0] = q->fl_size;
2265                 t.fl_size[1] = q->jumbo_size;
2266                 t.polling = q->polling;
2267                 t.lro = !!(dev->features & NETIF_F_GRO);
2268                 t.intr_lat = q->coalesce_usecs;
2269                 t.cong_thres = q->cong_thres;
2270                 t.qnum = q1;
2271
2272                 if (adapter->flags & USING_MSIX)
2273                         t.vector = adapter->msix_info[q1 + t.qset_idx + 1].vec;
2274                 else
2275                         t.vector = adapter->pdev->irq;
2276
2277                 if (copy_to_user(useraddr, &t, sizeof(t)))
2278                         return -EFAULT;
2279                 break;
2280         }
2281         case CHELSIO_SET_QSET_NUM:{
2282                 struct ch_reg edata;
2283                 unsigned int i, first_qset = 0, other_qsets = 0;
2284
2285                 if (!capable(CAP_NET_ADMIN))
2286                         return -EPERM;
2287                 if (adapter->flags & FULL_INIT_DONE)
2288                         return -EBUSY;
2289                 if (copy_from_user(&edata, useraddr, sizeof(edata)))
2290                         return -EFAULT;
2291                 if (edata.val < 1 ||
2292                         (edata.val > 1 && !(adapter->flags & USING_MSIX)))
2293                         return -EINVAL;
2294
2295                 for_each_port(adapter, i)
2296                         if (adapter->port[i] && adapter->port[i] != dev)
2297                                 other_qsets += adap2pinfo(adapter, i)->nqsets;
2298
2299                 if (edata.val + other_qsets > SGE_QSETS)
2300                         return -EINVAL;
2301
2302                 pi->nqsets = edata.val;
2303
2304                 for_each_port(adapter, i)
2305                         if (adapter->port[i]) {
2306                                 pi = adap2pinfo(adapter, i);
2307                                 pi->first_qset = first_qset;
2308                                 first_qset += pi->nqsets;
2309                         }
2310                 break;
2311         }
2312         case CHELSIO_GET_QSET_NUM:{
2313                 struct ch_reg edata;
2314
2315                 memset(&edata, 0, sizeof(struct ch_reg));
2316
2317                 edata.cmd = CHELSIO_GET_QSET_NUM;
2318                 edata.val = pi->nqsets;
2319                 if (copy_to_user(useraddr, &edata, sizeof(edata)))
2320                         return -EFAULT;
2321                 break;
2322         }
2323         case CHELSIO_LOAD_FW:{
2324                 u8 *fw_data;
2325                 struct ch_mem_range t;
2326
2327                 if (!capable(CAP_SYS_RAWIO))
2328                         return -EPERM;
2329                 if (copy_from_user(&t, useraddr, sizeof(t)))
2330                         return -EFAULT;
2331                 /* Check t.len sanity ? */
2332                 fw_data = memdup_user(useraddr + sizeof(t), t.len);
2333                 if (IS_ERR(fw_data))
2334                         return PTR_ERR(fw_data);
2335
2336                 ret = t3_load_fw(adapter, fw_data, t.len);
2337                 kfree(fw_data);
2338                 if (ret)
2339                         return ret;
2340                 break;
2341         }
2342         case CHELSIO_SETMTUTAB:{
2343                 struct ch_mtus m;
2344                 int i;
2345
2346                 if (!is_offload(adapter))
2347                         return -EOPNOTSUPP;
2348                 if (!capable(CAP_NET_ADMIN))
2349                         return -EPERM;
2350                 if (offload_running(adapter))
2351                         return -EBUSY;
2352                 if (copy_from_user(&m, useraddr, sizeof(m)))
2353                         return -EFAULT;
2354                 if (m.nmtus != NMTUS)
2355                         return -EINVAL;
2356                 if (m.mtus[0] < 81)     /* accommodate SACK */
2357                         return -EINVAL;
2358
2359                 /* MTUs must be in ascending order */
2360                 for (i = 1; i < NMTUS; ++i)
2361                         if (m.mtus[i] < m.mtus[i - 1])
2362                                 return -EINVAL;
2363
2364                 memcpy(adapter->params.mtus, m.mtus,
2365                         sizeof(adapter->params.mtus));
2366                 break;
2367         }
2368         case CHELSIO_GET_PM:{
2369                 struct tp_params *p = &adapter->params.tp;
2370                 struct ch_pm m = {.cmd = CHELSIO_GET_PM };
2371
2372                 if (!is_offload(adapter))
2373                         return -EOPNOTSUPP;
2374                 m.tx_pg_sz = p->tx_pg_size;
2375                 m.tx_num_pg = p->tx_num_pgs;
2376                 m.rx_pg_sz = p->rx_pg_size;
2377                 m.rx_num_pg = p->rx_num_pgs;
2378                 m.pm_total = p->pmtx_size + p->chan_rx_size * p->nchan;
2379                 if (copy_to_user(useraddr, &m, sizeof(m)))
2380                         return -EFAULT;
2381                 break;
2382         }
2383         case CHELSIO_SET_PM:{
2384                 struct ch_pm m;
2385                 struct tp_params *p = &adapter->params.tp;
2386
2387                 if (!is_offload(adapter))
2388                         return -EOPNOTSUPP;
2389                 if (!capable(CAP_NET_ADMIN))
2390                         return -EPERM;
2391                 if (adapter->flags & FULL_INIT_DONE)
2392                         return -EBUSY;
2393                 if (copy_from_user(&m, useraddr, sizeof(m)))
2394                         return -EFAULT;
2395                 if (!is_power_of_2(m.rx_pg_sz) ||
2396                         !is_power_of_2(m.tx_pg_sz))
2397                         return -EINVAL; /* not power of 2 */
2398                 if (!(m.rx_pg_sz & 0x14000))
2399                         return -EINVAL; /* not 16KB or 64KB */
2400                 if (!(m.tx_pg_sz & 0x1554000))
2401                         return -EINVAL;
2402                 if (m.tx_num_pg == -1)
2403                         m.tx_num_pg = p->tx_num_pgs;
2404                 if (m.rx_num_pg == -1)
2405                         m.rx_num_pg = p->rx_num_pgs;
2406                 if (m.tx_num_pg % 24 || m.rx_num_pg % 24)
2407                         return -EINVAL;
2408                 if (m.rx_num_pg * m.rx_pg_sz > p->chan_rx_size ||
2409                         m.tx_num_pg * m.tx_pg_sz > p->chan_tx_size)
2410                         return -EINVAL;
2411                 p->rx_pg_size = m.rx_pg_sz;
2412                 p->tx_pg_size = m.tx_pg_sz;
2413                 p->rx_num_pgs = m.rx_num_pg;
2414                 p->tx_num_pgs = m.tx_num_pg;
2415                 break;
2416         }
2417         case CHELSIO_GET_MEM:{
2418                 struct ch_mem_range t;
2419                 struct mc7 *mem;
2420                 u64 buf[32];
2421
2422                 if (!is_offload(adapter))
2423                         return -EOPNOTSUPP;
2424                 if (!(adapter->flags & FULL_INIT_DONE))
2425                         return -EIO;    /* need the memory controllers */
2426                 if (copy_from_user(&t, useraddr, sizeof(t)))
2427                         return -EFAULT;
2428                 if ((t.addr & 7) || (t.len & 7))
2429                         return -EINVAL;
2430                 if (t.mem_id == MEM_CM)
2431                         mem = &adapter->cm;
2432                 else if (t.mem_id == MEM_PMRX)
2433                         mem = &adapter->pmrx;
2434                 else if (t.mem_id == MEM_PMTX)
2435                         mem = &adapter->pmtx;
2436                 else
2437                         return -EINVAL;
2438
2439                 /*
2440                  * Version scheme:
2441                  * bits 0..9: chip version
2442                  * bits 10..15: chip revision
2443                  */
2444                 t.version = 3 | (adapter->params.rev << 10);
2445                 if (copy_to_user(useraddr, &t, sizeof(t)))
2446                         return -EFAULT;
2447
2448                 /*
2449                  * Read 256 bytes at a time as len can be large and we don't
2450                  * want to use huge intermediate buffers.
2451                  */
2452                 useraddr += sizeof(t);  /* advance to start of buffer */
2453                 while (t.len) {
2454                         unsigned int chunk =
2455                                 min_t(unsigned int, t.len, sizeof(buf));
2456
2457                         ret =
2458                                 t3_mc7_bd_read(mem, t.addr / 8, chunk / 8,
2459                                                 buf);
2460                         if (ret)
2461                                 return ret;
2462                         if (copy_to_user(useraddr, buf, chunk))
2463                                 return -EFAULT;
2464                         useraddr += chunk;
2465                         t.addr += chunk;
2466                         t.len -= chunk;
2467                 }
2468                 break;
2469         }
2470         case CHELSIO_SET_TRACE_FILTER:{
2471                 struct ch_trace t;
2472                 const struct trace_params *tp;
2473
2474                 if (!capable(CAP_NET_ADMIN))
2475                         return -EPERM;
2476                 if (!offload_running(adapter))
2477                         return -EAGAIN;
2478                 if (copy_from_user(&t, useraddr, sizeof(t)))
2479                         return -EFAULT;
2480
2481                 tp = (const struct trace_params *)&t.sip;
2482                 if (t.config_tx)
2483                         t3_config_trace_filter(adapter, tp, 0,
2484                                                 t.invert_match,
2485                                                 t.trace_tx);
2486                 if (t.config_rx)
2487                         t3_config_trace_filter(adapter, tp, 1,
2488                                                 t.invert_match,
2489                                                 t.trace_rx);
2490                 break;
2491         }
2492         default:
2493                 return -EOPNOTSUPP;
2494         }
2495         return 0;
2496 }
2497
2498 static int cxgb_ioctl(struct net_device *dev, struct ifreq *req, int cmd)
2499 {
2500         struct mii_ioctl_data *data = if_mii(req);
2501         struct port_info *pi = netdev_priv(dev);
2502         struct adapter *adapter = pi->adapter;
2503
2504         switch (cmd) {
2505         case SIOCGMIIREG:
2506         case SIOCSMIIREG:
2507                 /* Convert phy_id from older PRTAD/DEVAD format */
2508                 if (is_10G(adapter) &&
2509                     !mdio_phy_id_is_c45(data->phy_id) &&
2510                     (data->phy_id & 0x1f00) &&
2511                     !(data->phy_id & 0xe0e0))
2512                         data->phy_id = mdio_phy_id_c45(data->phy_id >> 8,
2513                                                        data->phy_id & 0x1f);
2514                 /* FALLTHRU */
2515         case SIOCGMIIPHY:
2516                 return mdio_mii_ioctl(&pi->phy.mdio, data, cmd);
2517         case SIOCCHIOCTL:
2518                 return cxgb_extension_ioctl(dev, req->ifr_data);
2519         default:
2520                 return -EOPNOTSUPP;
2521         }
2522 }
2523
2524 static int cxgb_change_mtu(struct net_device *dev, int new_mtu)
2525 {
2526         struct port_info *pi = netdev_priv(dev);
2527         struct adapter *adapter = pi->adapter;
2528         int ret;
2529
2530         if (new_mtu < 81)       /* accommodate SACK */
2531                 return -EINVAL;
2532         if ((ret = t3_mac_set_mtu(&pi->mac, new_mtu)))
2533                 return ret;
2534         dev->mtu = new_mtu;
2535         init_port_mtus(adapter);
2536         if (adapter->params.rev == 0 && offload_running(adapter))
2537                 t3_load_mtus(adapter, adapter->params.mtus,
2538                              adapter->params.a_wnd, adapter->params.b_wnd,
2539                              adapter->port[0]->mtu);
2540         return 0;
2541 }
2542
2543 static int cxgb_set_mac_addr(struct net_device *dev, void *p)
2544 {
2545         struct port_info *pi = netdev_priv(dev);
2546         struct adapter *adapter = pi->adapter;
2547         struct sockaddr *addr = p;
2548
2549         if (!is_valid_ether_addr(addr->sa_data))
2550                 return -EADDRNOTAVAIL;
2551
2552         memcpy(dev->dev_addr, addr->sa_data, dev->addr_len);
2553         t3_mac_set_address(&pi->mac, LAN_MAC_IDX, dev->dev_addr);
2554         if (offload_running(adapter))
2555                 write_smt_entry(adapter, pi->port_id);
2556         return 0;
2557 }
2558
2559 static netdev_features_t cxgb_fix_features(struct net_device *dev,
2560         netdev_features_t features)
2561 {
2562         /*
2563          * Since there is no support for separate rx/tx vlan accel
2564          * enable/disable make sure tx flag is always in same state as rx.
2565          */
2566         if (features & NETIF_F_HW_VLAN_RX)
2567                 features |= NETIF_F_HW_VLAN_TX;
2568         else
2569                 features &= ~NETIF_F_HW_VLAN_TX;
2570
2571         return features;
2572 }
2573
2574 static int cxgb_set_features(struct net_device *dev, netdev_features_t features)
2575 {
2576         netdev_features_t changed = dev->features ^ features;
2577
2578         if (changed & NETIF_F_HW_VLAN_RX)
2579                 cxgb_vlan_mode(dev, features);
2580
2581         return 0;
2582 }
2583
2584 #ifdef CONFIG_NET_POLL_CONTROLLER
2585 static void cxgb_netpoll(struct net_device *dev)
2586 {
2587         struct port_info *pi = netdev_priv(dev);
2588         struct adapter *adapter = pi->adapter;
2589         int qidx;
2590
2591         for (qidx = pi->first_qset; qidx < pi->first_qset + pi->nqsets; qidx++) {
2592                 struct sge_qset *qs = &adapter->sge.qs[qidx];
2593                 void *source;
2594
2595                 if (adapter->flags & USING_MSIX)
2596                         source = qs;
2597                 else
2598                         source = adapter;
2599
2600                 t3_intr_handler(adapter, qs->rspq.polling) (0, source);
2601         }
2602 }
2603 #endif
2604
2605 /*
2606  * Periodic accumulation of MAC statistics.
2607  */
2608 static void mac_stats_update(struct adapter *adapter)
2609 {
2610         int i;
2611
2612         for_each_port(adapter, i) {
2613                 struct net_device *dev = adapter->port[i];
2614                 struct port_info *p = netdev_priv(dev);
2615
2616                 if (netif_running(dev)) {
2617                         spin_lock(&adapter->stats_lock);
2618                         t3_mac_update_stats(&p->mac);
2619                         spin_unlock(&adapter->stats_lock);
2620                 }
2621         }
2622 }
2623
2624 static void check_link_status(struct adapter *adapter)
2625 {
2626         int i;
2627
2628         for_each_port(adapter, i) {
2629                 struct net_device *dev = adapter->port[i];
2630                 struct port_info *p = netdev_priv(dev);
2631                 int link_fault;
2632
2633                 spin_lock_irq(&adapter->work_lock);
2634                 link_fault = p->link_fault;
2635                 spin_unlock_irq(&adapter->work_lock);
2636
2637                 if (link_fault) {
2638                         t3_link_fault(adapter, i);
2639                         continue;
2640                 }
2641
2642                 if (!(p->phy.caps & SUPPORTED_IRQ) && netif_running(dev)) {
2643                         t3_xgm_intr_disable(adapter, i);
2644                         t3_read_reg(adapter, A_XGM_INT_STATUS + p->mac.offset);
2645
2646                         t3_link_changed(adapter, i);
2647                         t3_xgm_intr_enable(adapter, i);
2648                 }
2649         }
2650 }
2651
2652 static void check_t3b2_mac(struct adapter *adapter)
2653 {
2654         int i;
2655
2656         if (!rtnl_trylock())    /* synchronize with ifdown */
2657                 return;
2658
2659         for_each_port(adapter, i) {
2660                 struct net_device *dev = adapter->port[i];
2661                 struct port_info *p = netdev_priv(dev);
2662                 int status;
2663
2664                 if (!netif_running(dev))
2665                         continue;
2666
2667                 status = 0;
2668                 if (netif_running(dev) && netif_carrier_ok(dev))
2669                         status = t3b2_mac_watchdog_task(&p->mac);
2670                 if (status == 1)
2671                         p->mac.stats.num_toggled++;
2672                 else if (status == 2) {
2673                         struct cmac *mac = &p->mac;
2674
2675                         t3_mac_set_mtu(mac, dev->mtu);
2676                         t3_mac_set_address(mac, LAN_MAC_IDX, dev->dev_addr);
2677                         cxgb_set_rxmode(dev);
2678                         t3_link_start(&p->phy, mac, &p->link_config);
2679                         t3_mac_enable(mac, MAC_DIRECTION_RX | MAC_DIRECTION_TX);
2680                         t3_port_intr_enable(adapter, p->port_id);
2681                         p->mac.stats.num_resets++;
2682                 }
2683         }
2684         rtnl_unlock();
2685 }
2686
2687
2688 static void t3_adap_check_task(struct work_struct *work)
2689 {
2690         struct adapter *adapter = container_of(work, struct adapter,
2691                                                adap_check_task.work);
2692         const struct adapter_params *p = &adapter->params;
2693         int port;
2694         unsigned int v, status, reset;
2695
2696         adapter->check_task_cnt++;
2697
2698         check_link_status(adapter);
2699
2700         /* Accumulate MAC stats if needed */
2701         if (!p->linkpoll_period ||
2702             (adapter->check_task_cnt * p->linkpoll_period) / 10 >=
2703             p->stats_update_period) {
2704                 mac_stats_update(adapter);
2705                 adapter->check_task_cnt = 0;
2706         }
2707
2708         if (p->rev == T3_REV_B2)
2709                 check_t3b2_mac(adapter);
2710
2711         /*
2712          * Scan the XGMAC's to check for various conditions which we want to
2713          * monitor in a periodic polling manner rather than via an interrupt
2714          * condition.  This is used for conditions which would otherwise flood
2715          * the system with interrupts and we only really need to know that the
2716          * conditions are "happening" ...  For each condition we count the
2717          * detection of the condition and reset it for the next polling loop.
2718          */
2719         for_each_port(adapter, port) {
2720                 struct cmac *mac =  &adap2pinfo(adapter, port)->mac;
2721                 u32 cause;
2722
2723                 cause = t3_read_reg(adapter, A_XGM_INT_CAUSE + mac->offset);
2724                 reset = 0;
2725                 if (cause & F_RXFIFO_OVERFLOW) {
2726                         mac->stats.rx_fifo_ovfl++;
2727                         reset |= F_RXFIFO_OVERFLOW;
2728                 }
2729
2730                 t3_write_reg(adapter, A_XGM_INT_CAUSE + mac->offset, reset);
2731         }
2732
2733         /*
2734          * We do the same as above for FL_EMPTY interrupts.
2735          */
2736         status = t3_read_reg(adapter, A_SG_INT_CAUSE);
2737         reset = 0;
2738
2739         if (status & F_FLEMPTY) {
2740                 struct sge_qset *qs = &adapter->sge.qs[0];
2741                 int i = 0;
2742
2743                 reset |= F_FLEMPTY;
2744
2745                 v = (t3_read_reg(adapter, A_SG_RSPQ_FL_STATUS) >> S_FL0EMPTY) &
2746                     0xffff;
2747
2748                 while (v) {
2749                         qs->fl[i].empty += (v & 1);
2750                         if (i)
2751                                 qs++;
2752                         i ^= 1;
2753                         v >>= 1;
2754                 }
2755         }
2756
2757         t3_write_reg(adapter, A_SG_INT_CAUSE, reset);
2758
2759         /* Schedule the next check update if any port is active. */
2760         spin_lock_irq(&adapter->work_lock);
2761         if (adapter->open_device_map & PORT_MASK)
2762                 schedule_chk_task(adapter);
2763         spin_unlock_irq(&adapter->work_lock);
2764 }
2765
2766 static void db_full_task(struct work_struct *work)
2767 {
2768         struct adapter *adapter = container_of(work, struct adapter,
2769                                                db_full_task);
2770
2771         cxgb3_event_notify(&adapter->tdev, OFFLOAD_DB_FULL, 0);
2772 }
2773
2774 static void db_empty_task(struct work_struct *work)
2775 {
2776         struct adapter *adapter = container_of(work, struct adapter,
2777                                                db_empty_task);
2778
2779         cxgb3_event_notify(&adapter->tdev, OFFLOAD_DB_EMPTY, 0);
2780 }
2781
2782 static void db_drop_task(struct work_struct *work)
2783 {
2784         struct adapter *adapter = container_of(work, struct adapter,
2785                                                db_drop_task);
2786         unsigned long delay = 1000;
2787         unsigned short r;
2788
2789         cxgb3_event_notify(&adapter->tdev, OFFLOAD_DB_DROP, 0);
2790
2791         /*
2792          * Sleep a while before ringing the driver qset dbs.
2793          * The delay is between 1000-2023 usecs.
2794          */
2795         get_random_bytes(&r, 2);
2796         delay += r & 1023;
2797         set_current_state(TASK_UNINTERRUPTIBLE);
2798         schedule_timeout(usecs_to_jiffies(delay));
2799         ring_dbs(adapter);
2800 }
2801
2802 /*
2803  * Processes external (PHY) interrupts in process context.
2804  */
2805 static void ext_intr_task(struct work_struct *work)
2806 {
2807         struct adapter *adapter = container_of(work, struct adapter,
2808                                                ext_intr_handler_task);
2809         int i;
2810
2811         /* Disable link fault interrupts */
2812         for_each_port(adapter, i) {
2813                 struct net_device *dev = adapter->port[i];
2814                 struct port_info *p = netdev_priv(dev);
2815
2816                 t3_xgm_intr_disable(adapter, i);
2817                 t3_read_reg(adapter, A_XGM_INT_STATUS + p->mac.offset);
2818         }
2819
2820         /* Re-enable link fault interrupts */
2821         t3_phy_intr_handler(adapter);
2822
2823         for_each_port(adapter, i)
2824                 t3_xgm_intr_enable(adapter, i);
2825
2826         /* Now reenable external interrupts */
2827         spin_lock_irq(&adapter->work_lock);
2828         if (adapter->slow_intr_mask) {
2829                 adapter->slow_intr_mask |= F_T3DBG;
2830                 t3_write_reg(adapter, A_PL_INT_CAUSE0, F_T3DBG);
2831                 t3_write_reg(adapter, A_PL_INT_ENABLE0,
2832                              adapter->slow_intr_mask);
2833         }
2834         spin_unlock_irq(&adapter->work_lock);
2835 }
2836
2837 /*
2838  * Interrupt-context handler for external (PHY) interrupts.
2839  */
2840 void t3_os_ext_intr_handler(struct adapter *adapter)
2841 {
2842         /*
2843          * Schedule a task to handle external interrupts as they may be slow
2844          * and we use a mutex to protect MDIO registers.  We disable PHY
2845          * interrupts in the meantime and let the task reenable them when
2846          * it's done.
2847          */
2848         spin_lock(&adapter->work_lock);
2849         if (adapter->slow_intr_mask) {
2850                 adapter->slow_intr_mask &= ~F_T3DBG;
2851                 t3_write_reg(adapter, A_PL_INT_ENABLE0,
2852                              adapter->slow_intr_mask);
2853                 queue_work(cxgb3_wq, &adapter->ext_intr_handler_task);
2854         }
2855         spin_unlock(&adapter->work_lock);
2856 }
2857
2858 void t3_os_link_fault_handler(struct adapter *adapter, int port_id)
2859 {
2860         struct net_device *netdev = adapter->port[port_id];
2861         struct port_info *pi = netdev_priv(netdev);
2862
2863         spin_lock(&adapter->work_lock);
2864         pi->link_fault = 1;
2865         spin_unlock(&adapter->work_lock);
2866 }
2867
2868 static int t3_adapter_error(struct adapter *adapter, int reset, int on_wq)
2869 {
2870         int i, ret = 0;
2871
2872         if (is_offload(adapter) &&
2873             test_bit(OFFLOAD_DEVMAP_BIT, &adapter->open_device_map)) {
2874                 cxgb3_event_notify(&adapter->tdev, OFFLOAD_STATUS_DOWN, 0);
2875                 offload_close(&adapter->tdev);
2876         }
2877
2878         /* Stop all ports */
2879         for_each_port(adapter, i) {
2880                 struct net_device *netdev = adapter->port[i];
2881
2882                 if (netif_running(netdev))
2883                         __cxgb_close(netdev, on_wq);
2884         }
2885
2886         /* Stop SGE timers */
2887         t3_stop_sge_timers(adapter);
2888
2889         adapter->flags &= ~FULL_INIT_DONE;
2890
2891         if (reset)
2892                 ret = t3_reset_adapter(adapter);
2893
2894         pci_disable_device(adapter->pdev);
2895
2896         return ret;
2897 }
2898
2899 static int t3_reenable_adapter(struct adapter *adapter)
2900 {
2901         if (pci_enable_device(adapter->pdev)) {
2902                 dev_err(&adapter->pdev->dev,
2903                         "Cannot re-enable PCI device after reset.\n");
2904                 goto err;
2905         }
2906         pci_set_master(adapter->pdev);
2907         pci_restore_state(adapter->pdev);
2908         pci_save_state(adapter->pdev);
2909
2910         /* Free sge resources */
2911         t3_free_sge_resources(adapter);
2912
2913         if (t3_replay_prep_adapter(adapter))
2914                 goto err;
2915
2916         return 0;
2917 err:
2918         return -1;
2919 }
2920
2921 static void t3_resume_ports(struct adapter *adapter)
2922 {
2923         int i;
2924
2925         /* Restart the ports */
2926         for_each_port(adapter, i) {
2927                 struct net_device *netdev = adapter->port[i];
2928
2929                 if (netif_running(netdev)) {
2930                         if (cxgb_open(netdev)) {
2931                                 dev_err(&adapter->pdev->dev,
2932                                         "can't bring device back up"
2933                                         " after reset\n");
2934                                 continue;
2935                         }
2936                 }
2937         }
2938
2939         if (is_offload(adapter) && !ofld_disable)
2940                 cxgb3_event_notify(&adapter->tdev, OFFLOAD_STATUS_UP, 0);
2941 }
2942
2943 /*
2944  * processes a fatal error.
2945  * Bring the ports down, reset the chip, bring the ports back up.
2946  */
2947 static void fatal_error_task(struct work_struct *work)
2948 {
2949         struct adapter *adapter = container_of(work, struct adapter,
2950                                                fatal_error_handler_task);
2951         int err = 0;
2952
2953         rtnl_lock();
2954         err = t3_adapter_error(adapter, 1, 1);
2955         if (!err)
2956                 err = t3_reenable_adapter(adapter);
2957         if (!err)
2958                 t3_resume_ports(adapter);
2959
2960         CH_ALERT(adapter, "adapter reset %s\n", err ? "failed" : "succeeded");
2961         rtnl_unlock();
2962 }
2963
2964 void t3_fatal_err(struct adapter *adapter)
2965 {
2966         unsigned int fw_status[4];
2967
2968         if (adapter->flags & FULL_INIT_DONE) {
2969                 t3_sge_stop(adapter);
2970                 t3_write_reg(adapter, A_XGM_TX_CTRL, 0);
2971                 t3_write_reg(adapter, A_XGM_RX_CTRL, 0);
2972                 t3_write_reg(adapter, XGM_REG(A_XGM_TX_CTRL, 1), 0);
2973                 t3_write_reg(adapter, XGM_REG(A_XGM_RX_CTRL, 1), 0);
2974
2975                 spin_lock(&adapter->work_lock);
2976                 t3_intr_disable(adapter);
2977                 queue_work(cxgb3_wq, &adapter->fatal_error_handler_task);
2978                 spin_unlock(&adapter->work_lock);
2979         }
2980         CH_ALERT(adapter, "encountered fatal error, operation suspended\n");
2981         if (!t3_cim_ctl_blk_read(adapter, 0xa0, 4, fw_status))
2982                 CH_ALERT(adapter, "FW status: 0x%x, 0x%x, 0x%x, 0x%x\n",
2983                          fw_status[0], fw_status[1],
2984                          fw_status[2], fw_status[3]);
2985 }
2986
2987 /**
2988  * t3_io_error_detected - called when PCI error is detected
2989  * @pdev: Pointer to PCI device
2990  * @state: The current pci connection state
2991  *
2992  * This function is called after a PCI bus error affecting
2993  * this device has been detected.
2994  */
2995 static pci_ers_result_t t3_io_error_detected(struct pci_dev *pdev,
2996                                              pci_channel_state_t state)
2997 {
2998         struct adapter *adapter = pci_get_drvdata(pdev);
2999
3000         if (state == pci_channel_io_perm_failure)
3001                 return PCI_ERS_RESULT_DISCONNECT;
3002
3003         t3_adapter_error(adapter, 0, 0);
3004
3005         /* Request a slot reset. */
3006         return PCI_ERS_RESULT_NEED_RESET;
3007 }
3008
3009 /**
3010  * t3_io_slot_reset - called after the pci bus has been reset.
3011  * @pdev: Pointer to PCI device
3012  *
3013  * Restart the card from scratch, as if from a cold-boot.
3014  */
3015 static pci_ers_result_t t3_io_slot_reset(struct pci_dev *pdev)
3016 {
3017         struct adapter *adapter = pci_get_drvdata(pdev);
3018
3019         if (!t3_reenable_adapter(adapter))
3020                 return PCI_ERS_RESULT_RECOVERED;
3021
3022         return PCI_ERS_RESULT_DISCONNECT;
3023 }
3024
3025 /**
3026  * t3_io_resume - called when traffic can start flowing again.
3027  * @pdev: Pointer to PCI device
3028  *
3029  * This callback is called when the error recovery driver tells us that
3030  * its OK to resume normal operation.
3031  */
3032 static void t3_io_resume(struct pci_dev *pdev)
3033 {
3034         struct adapter *adapter = pci_get_drvdata(pdev);
3035
3036         CH_ALERT(adapter, "adapter recovering, PEX ERR 0x%x\n",
3037                  t3_read_reg(adapter, A_PCIE_PEX_ERR));
3038
3039         t3_resume_ports(adapter);
3040 }
3041
3042 static const struct pci_error_handlers t3_err_handler = {
3043         .error_detected = t3_io_error_detected,
3044         .slot_reset = t3_io_slot_reset,
3045         .resume = t3_io_resume,
3046 };
3047
3048 /*
3049  * Set the number of qsets based on the number of CPUs and the number of ports,
3050  * not to exceed the number of available qsets, assuming there are enough qsets
3051  * per port in HW.
3052  */
3053 static void set_nqsets(struct adapter *adap)
3054 {
3055         int i, j = 0;
3056         int num_cpus = netif_get_num_default_rss_queues();
3057         int hwports = adap->params.nports;
3058         int nqsets = adap->msix_nvectors - 1;
3059
3060         if (adap->params.rev > 0 && adap->flags & USING_MSIX) {
3061                 if (hwports == 2 &&
3062                     (hwports * nqsets > SGE_QSETS ||
3063                      num_cpus >= nqsets / hwports))
3064                         nqsets /= hwports;
3065                 if (nqsets > num_cpus)
3066                         nqsets = num_cpus;
3067                 if (nqsets < 1 || hwports == 4)
3068                         nqsets = 1;
3069         } else
3070                 nqsets = 1;
3071
3072         for_each_port(adap, i) {
3073                 struct port_info *pi = adap2pinfo(adap, i);
3074
3075                 pi->first_qset = j;
3076                 pi->nqsets = nqsets;
3077                 j = pi->first_qset + nqsets;
3078
3079                 dev_info(&adap->pdev->dev,
3080                          "Port %d using %d queue sets.\n", i, nqsets);
3081         }
3082 }
3083
3084 static int cxgb_enable_msix(struct adapter *adap)
3085 {
3086         struct msix_entry entries[SGE_QSETS + 1];
3087         int vectors;
3088         int i, err;
3089
3090         vectors = ARRAY_SIZE(entries);
3091         for (i = 0; i < vectors; ++i)
3092                 entries[i].entry = i;
3093
3094         while ((err = pci_enable_msix(adap->pdev, entries, vectors)) > 0)
3095                 vectors = err;
3096
3097         if (err < 0)
3098                 pci_disable_msix(adap->pdev);
3099
3100         if (!err && vectors < (adap->params.nports + 1)) {
3101                 pci_disable_msix(adap->pdev);
3102                 err = -1;
3103         }
3104
3105         if (!err) {
3106                 for (i = 0; i < vectors; ++i)
3107                         adap->msix_info[i].vec = entries[i].vector;
3108                 adap->msix_nvectors = vectors;
3109         }
3110
3111         return err;
3112 }
3113
3114 static void print_port_info(struct adapter *adap, const struct adapter_info *ai)
3115 {
3116         static const char *pci_variant[] = {
3117                 "PCI", "PCI-X", "PCI-X ECC", "PCI-X 266", "PCI Express"
3118         };
3119
3120         int i;
3121         char buf[80];
3122
3123         if (is_pcie(adap))
3124                 snprintf(buf, sizeof(buf), "%s x%d",
3125                          pci_variant[adap->params.pci.variant],
3126                          adap->params.pci.width);
3127         else
3128                 snprintf(buf, sizeof(buf), "%s %dMHz/%d-bit",
3129                          pci_variant[adap->params.pci.variant],
3130                          adap->params.pci.speed, adap->params.pci.width);
3131
3132         for_each_port(adap, i) {
3133                 struct net_device *dev = adap->port[i];
3134                 const struct port_info *pi = netdev_priv(dev);
3135
3136                 if (!test_bit(i, &adap->registered_device_map))
3137                         continue;
3138                 netdev_info(dev, "%s %s %sNIC (rev %d) %s%s\n",
3139                             ai->desc, pi->phy.desc,
3140                             is_offload(adap) ? "R" : "", adap->params.rev, buf,
3141                             (adap->flags & USING_MSIX) ? " MSI-X" :
3142                             (adap->flags & USING_MSI) ? " MSI" : "");
3143                 if (adap->name == dev->name && adap->params.vpd.mclk)
3144                         pr_info("%s: %uMB CM, %uMB PMTX, %uMB PMRX, S/N: %s\n",
3145                                adap->name, t3_mc7_size(&adap->cm) >> 20,
3146                                t3_mc7_size(&adap->pmtx) >> 20,
3147                                t3_mc7_size(&adap->pmrx) >> 20,
3148                                adap->params.vpd.sn);
3149         }
3150 }
3151
3152 static const struct net_device_ops cxgb_netdev_ops = {
3153         .ndo_open               = cxgb_open,
3154         .ndo_stop               = cxgb_close,
3155         .ndo_start_xmit         = t3_eth_xmit,
3156         .ndo_get_stats          = cxgb_get_stats,
3157         .ndo_validate_addr      = eth_validate_addr,
3158         .ndo_set_rx_mode        = cxgb_set_rxmode,
3159         .ndo_do_ioctl           = cxgb_ioctl,
3160         .ndo_change_mtu         = cxgb_change_mtu,
3161         .ndo_set_mac_address    = cxgb_set_mac_addr,
3162         .ndo_fix_features       = cxgb_fix_features,
3163         .ndo_set_features       = cxgb_set_features,
3164 #ifdef CONFIG_NET_POLL_CONTROLLER
3165         .ndo_poll_controller    = cxgb_netpoll,
3166 #endif
3167 };
3168
3169 static void cxgb3_init_iscsi_mac(struct net_device *dev)
3170 {
3171         struct port_info *pi = netdev_priv(dev);
3172
3173         memcpy(pi->iscsic.mac_addr, dev->dev_addr, ETH_ALEN);
3174         pi->iscsic.mac_addr[3] |= 0x80;
3175 }
3176
3177 #define TSO_FLAGS (NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_TSO_ECN)
3178 #define VLAN_FEAT (NETIF_F_SG | NETIF_F_IP_CSUM | TSO_FLAGS | \
3179                         NETIF_F_IPV6_CSUM | NETIF_F_HIGHDMA)
3180 static int init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
3181 {
3182         int i, err, pci_using_dac = 0;
3183         resource_size_t mmio_start, mmio_len;
3184         const struct adapter_info *ai;
3185         struct adapter *adapter = NULL;
3186         struct port_info *pi;
3187
3188         pr_info_once("%s - version %s\n", DRV_DESC, DRV_VERSION);
3189
3190         if (!cxgb3_wq) {
3191                 cxgb3_wq = create_singlethread_workqueue(DRV_NAME);
3192                 if (!cxgb3_wq) {
3193                         pr_err("cannot initialize work queue\n");
3194                         return -ENOMEM;
3195                 }
3196         }
3197
3198         err = pci_enable_device(pdev);
3199         if (err) {
3200                 dev_err(&pdev->dev, "cannot enable PCI device\n");
3201                 goto out;
3202         }
3203
3204         err = pci_request_regions(pdev, DRV_NAME);
3205         if (err) {
3206                 /* Just info, some other driver may have claimed the device. */
3207                 dev_info(&pdev->dev, "cannot obtain PCI resources\n");
3208                 goto out_disable_device;
3209         }
3210
3211         if (!pci_set_dma_mask(pdev, DMA_BIT_MASK(64))) {
3212                 pci_using_dac = 1;
3213                 err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
3214                 if (err) {
3215                         dev_err(&pdev->dev, "unable to obtain 64-bit DMA for "
3216                                "coherent allocations\n");
3217                         goto out_release_regions;
3218                 }
3219         } else if ((err = pci_set_dma_mask(pdev, DMA_BIT_MASK(32))) != 0) {
3220                 dev_err(&pdev->dev, "no usable DMA configuration\n");
3221                 goto out_release_regions;
3222         }
3223
3224         pci_set_master(pdev);
3225         pci_save_state(pdev);
3226
3227         mmio_start = pci_resource_start(pdev, 0);
3228         mmio_len = pci_resource_len(pdev, 0);
3229         ai = t3_get_adapter_info(ent->driver_data);
3230
3231         adapter = kzalloc(sizeof(*adapter), GFP_KERNEL);
3232         if (!adapter) {
3233                 err = -ENOMEM;
3234                 goto out_release_regions;
3235         }
3236
3237         adapter->nofail_skb =
3238                 alloc_skb(sizeof(struct cpl_set_tcb_field), GFP_KERNEL);
3239         if (!adapter->nofail_skb) {
3240                 dev_err(&pdev->dev, "cannot allocate nofail buffer\n");
3241                 err = -ENOMEM;
3242                 goto out_free_adapter;
3243         }
3244
3245         adapter->regs = ioremap_nocache(mmio_start, mmio_len);
3246         if (!adapter->regs) {
3247                 dev_err(&pdev->dev, "cannot map device registers\n");
3248                 err = -ENOMEM;
3249                 goto out_free_adapter;
3250         }
3251
3252         adapter->pdev = pdev;
3253         adapter->name = pci_name(pdev);
3254         adapter->msg_enable = dflt_msg_enable;
3255         adapter->mmio_len = mmio_len;
3256
3257         mutex_init(&adapter->mdio_lock);
3258         spin_lock_init(&adapter->work_lock);
3259         spin_lock_init(&adapter->stats_lock);
3260
3261         INIT_LIST_HEAD(&adapter->adapter_list);
3262         INIT_WORK(&adapter->ext_intr_handler_task, ext_intr_task);
3263         INIT_WORK(&adapter->fatal_error_handler_task, fatal_error_task);
3264
3265         INIT_WORK(&adapter->db_full_task, db_full_task);
3266         INIT_WORK(&adapter->db_empty_task, db_empty_task);
3267         INIT_WORK(&adapter->db_drop_task, db_drop_task);
3268
3269         INIT_DELAYED_WORK(&adapter->adap_check_task, t3_adap_check_task);
3270
3271         for (i = 0; i < ai->nports0 + ai->nports1; ++i) {
3272                 struct net_device *netdev;
3273
3274                 netdev = alloc_etherdev_mq(sizeof(struct port_info), SGE_QSETS);
3275                 if (!netdev) {
3276                         err = -ENOMEM;
3277                         goto out_free_dev;
3278                 }
3279
3280                 SET_NETDEV_DEV(netdev, &pdev->dev);
3281
3282                 adapter->port[i] = netdev;
3283                 pi = netdev_priv(netdev);
3284                 pi->adapter = adapter;
3285                 pi->port_id = i;
3286                 netif_carrier_off(netdev);
3287                 netdev->irq = pdev->irq;
3288                 netdev->mem_start = mmio_start;
3289                 netdev->mem_end = mmio_start + mmio_len - 1;
3290                 netdev->hw_features = NETIF_F_SG | NETIF_F_IP_CSUM |
3291                         NETIF_F_TSO | NETIF_F_RXCSUM | NETIF_F_HW_VLAN_RX;
3292                 netdev->features |= netdev->hw_features | NETIF_F_HW_VLAN_TX;
3293                 netdev->vlan_features |= netdev->features & VLAN_FEAT;
3294                 if (pci_using_dac)
3295                         netdev->features |= NETIF_F_HIGHDMA;
3296
3297                 netdev->netdev_ops = &cxgb_netdev_ops;
3298                 SET_ETHTOOL_OPS(netdev, &cxgb_ethtool_ops);
3299         }
3300
3301         pci_set_drvdata(pdev, adapter);
3302         if (t3_prep_adapter(adapter, ai, 1) < 0) {
3303                 err = -ENODEV;
3304                 goto out_free_dev;
3305         }
3306
3307         /*
3308          * The card is now ready to go.  If any errors occur during device
3309          * registration we do not fail the whole card but rather proceed only
3310          * with the ports we manage to register successfully.  However we must
3311          * register at least one net device.
3312          */
3313         for_each_port(adapter, i) {
3314                 err = register_netdev(adapter->port[i]);
3315                 if (err)
3316                         dev_warn(&pdev->dev,
3317                                  "cannot register net device %s, skipping\n",
3318                                  adapter->port[i]->name);
3319                 else {
3320                         /*
3321                          * Change the name we use for messages to the name of
3322                          * the first successfully registered interface.
3323                          */
3324                         if (!adapter->registered_device_map)
3325                                 adapter->name = adapter->port[i]->name;
3326
3327                         __set_bit(i, &adapter->registered_device_map);
3328                 }
3329         }
3330         if (!adapter->registered_device_map) {
3331                 dev_err(&pdev->dev, "could not register any net devices\n");
3332                 goto out_free_dev;
3333         }
3334
3335         for_each_port(adapter, i)
3336                 cxgb3_init_iscsi_mac(adapter->port[i]);
3337
3338         /* Driver's ready. Reflect it on LEDs */
3339         t3_led_ready(adapter);
3340
3341         if (is_offload(adapter)) {
3342                 __set_bit(OFFLOAD_DEVMAP_BIT, &adapter->registered_device_map);
3343                 cxgb3_adapter_ofld(adapter);
3344         }
3345
3346         /* See what interrupts we'll be using */
3347         if (msi > 1 && cxgb_enable_msix(adapter) == 0)
3348                 adapter->flags |= USING_MSIX;
3349         else if (msi > 0 && pci_enable_msi(pdev) == 0)
3350                 adapter->flags |= USING_MSI;
3351
3352         set_nqsets(adapter);
3353
3354         err = sysfs_create_group(&adapter->port[0]->dev.kobj,
3355                                  &cxgb3_attr_group);
3356
3357         print_port_info(adapter, ai);
3358         return 0;
3359
3360 out_free_dev:
3361         iounmap(adapter->regs);
3362         for (i = ai->nports0 + ai->nports1 - 1; i >= 0; --i)
3363                 if (adapter->port[i])
3364                         free_netdev(adapter->port[i]);
3365
3366 out_free_adapter:
3367         kfree(adapter);
3368
3369 out_release_regions:
3370         pci_release_regions(pdev);
3371 out_disable_device:
3372         pci_disable_device(pdev);
3373         pci_set_drvdata(pdev, NULL);
3374 out:
3375         return err;
3376 }
3377
3378 static void remove_one(struct pci_dev *pdev)
3379 {
3380         struct adapter *adapter = pci_get_drvdata(pdev);
3381
3382         if (adapter) {
3383                 int i;
3384
3385                 t3_sge_stop(adapter);
3386                 sysfs_remove_group(&adapter->port[0]->dev.kobj,
3387                                    &cxgb3_attr_group);
3388
3389                 if (is_offload(adapter)) {
3390                         cxgb3_adapter_unofld(adapter);
3391                         if (test_bit(OFFLOAD_DEVMAP_BIT,
3392                                      &adapter->open_device_map))
3393                                 offload_close(&adapter->tdev);
3394                 }
3395
3396                 for_each_port(adapter, i)
3397                     if (test_bit(i, &adapter->registered_device_map))
3398                         unregister_netdev(adapter->port[i]);
3399
3400                 t3_stop_sge_timers(adapter);
3401                 t3_free_sge_resources(adapter);
3402                 cxgb_disable_msi(adapter);
3403
3404                 for_each_port(adapter, i)
3405                         if (adapter->port[i])
3406                                 free_netdev(adapter->port[i]);
3407
3408                 iounmap(adapter->regs);
3409                 if (adapter->nofail_skb)
3410                         kfree_skb(adapter->nofail_skb);
3411                 kfree(adapter);
3412                 pci_release_regions(pdev);
3413                 pci_disable_device(pdev);
3414                 pci_set_drvdata(pdev, NULL);
3415         }
3416 }
3417
3418 static struct pci_driver driver = {
3419         .name = DRV_NAME,
3420         .id_table = cxgb3_pci_tbl,
3421         .probe = init_one,
3422         .remove = remove_one,
3423         .err_handler = &t3_err_handler,
3424 };
3425
3426 static int __init cxgb3_init_module(void)
3427 {
3428         int ret;
3429
3430         cxgb3_offload_init();
3431
3432         ret = pci_register_driver(&driver);
3433         return ret;
3434 }
3435
3436 static void __exit cxgb3_cleanup_module(void)
3437 {
3438         pci_unregister_driver(&driver);
3439         if (cxgb3_wq)
3440                 destroy_workqueue(cxgb3_wq);
3441 }
3442
3443 module_init(cxgb3_init_module);
3444 module_exit(cxgb3_cleanup_module);