drivers/scsi/be2iscsi/be_main.c

   1 /**
   2  * Copyright (C) 2005 - 2011 Emulex
   3  * All rights reserved.
   4  *
   5  * This program is free software; you can redistribute it and/or
   6  * modify it under the terms of the GNU General Public License version 2
   7  * as published by the Free Software Foundation.  The full GNU General
   8  * Public License is included in this distribution in the file called COPYING.
   9  *
  10  * Written by: Jayamohan Kallickal (jayamohan.kallickal@emulex.com)
  11  *
  12  * Contact Information:
  13  * linux-drivers@emulex.com
  14  *
  15  * Emulex
  16  * 3333 Susan Street
  17  * Costa Mesa, CA 92626
  18  */
  19
  20 #include <linux/reboot.h>
  21 #include <linux/delay.h>
  22 #include <linux/slab.h>
  23 #include <linux/interrupt.h>
  24 #include <linux/blkdev.h>
  25 #include <linux/pci.h>
  26 #include <linux/string.h>
  27 #include <linux/kernel.h>
  28 #include <linux/semaphore.h>
  29 #include <linux/iscsi_boot_sysfs.h>
  30 #include <linux/module.h>
  31 #include <linux/bsg-lib.h>
  32
  33 #include <scsi/libiscsi.h>
  34 #include <scsi/scsi_bsg_iscsi.h>
  35 #include <scsi/scsi_netlink.h>
  36 #include <scsi/scsi_transport_iscsi.h>
  37 #include <scsi/scsi_transport.h>
  38 #include <scsi/scsi_cmnd.h>
  39 #include <scsi/scsi_device.h>
  40 #include <scsi/scsi_host.h>
  41 #include <scsi/scsi.h>
  42 #include "be_main.h"
  43 #include "be_iscsi.h"
  44 #include "be_mgmt.h"
  45 #include "be_cmds.h"
  46
  47 static unsigned int be_iopoll_budget = 10;
  48 static unsigned int be_max_phys_size = 64;
  49 static unsigned int enable_msix = 1;
  50
  51 MODULE_DEVICE_TABLE(pci, beiscsi_pci_id_table);
  52 MODULE_DESCRIPTION(DRV_DESC " " BUILD_STR);
  53 MODULE_VERSION(BUILD_STR);
  54 MODULE_AUTHOR("Emulex Corporation");
  55 MODULE_LICENSE("GPL");
  56 module_param(be_iopoll_budget, int, 0);
  57 module_param(enable_msix, int, 0);
  58 module_param(be_max_phys_size, uint, S_IRUGO);
  59 MODULE_PARM_DESC(be_max_phys_size,
  60                 "Maximum Size (In Kilobytes) of physically contiguous "
  61                 "memory that can be allocated. Range is 16 - 128");
  62
  63 #define beiscsi_disp_param(_name)\
  64 ssize_t \
  65 beiscsi_##_name##_disp(struct device *dev,\
  66                         struct device_attribute *attrib, char *buf)     \
  67 {       \
  68         struct Scsi_Host *shost = class_to_shost(dev);\
  69         struct beiscsi_hba *phba = iscsi_host_priv(shost); \
  70         uint32_t param_val = 0; \
  71         param_val = phba->attr_##_name;\
  72         return snprintf(buf, PAGE_SIZE, "%d\n",\
  73                         phba->attr_##_name);\
  74 }
  75
  76 #define beiscsi_change_param(_name, _minval, _maxval, _defaval)\
  77 int \
  78 beiscsi_##_name##_change(struct beiscsi_hba *phba, uint32_t val)\
  79 {\
  80         if (val >= _minval && val <= _maxval) {\
  81                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,\
  82                             "BA_%d : beiscsi_"#_name" updated "\
  83                             "from 0x%x ==> 0x%x\n",\
  84                             phba->attr_##_name, val); \
  85                 phba->attr_##_name = val;\
  86                 return 0;\
  87         } \
  88         beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT, \
  89                     "BA_%d beiscsi_"#_name" attribute "\
  90                     "cannot be updated to 0x%x, "\
  91                     "range allowed is ["#_minval" - "#_maxval"]\n", val);\
  92                 return -EINVAL;\
  93 }
  94
  95 #define beiscsi_store_param(_name)  \
  96 ssize_t \
  97 beiscsi_##_name##_store(struct device *dev,\
  98                          struct device_attribute *attr, const char *buf,\
  99                          size_t count) \
 100 { \
 101         struct Scsi_Host  *shost = class_to_shost(dev);\
 102         struct beiscsi_hba *phba = iscsi_host_priv(shost);\
 103         uint32_t param_val = 0;\
 104         if (!isdigit(buf[0]))\
 105                 return -EINVAL;\
 106         if (sscanf(buf, "%i", &param_val) != 1)\
 107                 return -EINVAL;\
 108         if (beiscsi_##_name##_change(phba, param_val) == 0) \
 109                 return strlen(buf);\
 110         else \
 111                 return -EINVAL;\
 112 }
 113
 114 #define beiscsi_init_param(_name, _minval, _maxval, _defval) \
 115 int \
 116 beiscsi_##_name##_init(struct beiscsi_hba *phba, uint32_t val) \
 117 { \
 118         if (val >= _minval && val <= _maxval) {\
 119                 phba->attr_##_name = val;\
 120                 return 0;\
 121         } \
 122         beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,\
 123                     "BA_%d beiscsi_"#_name" attribute " \
 124                     "cannot be updated to 0x%x, "\
 125                     "range allowed is ["#_minval" - "#_maxval"]\n", val);\
 126         phba->attr_##_name = _defval;\
 127         return -EINVAL;\
 128 }
 129
 130 #define BEISCSI_RW_ATTR(_name, _minval, _maxval, _defval, _descp) \
 131 static uint beiscsi_##_name = _defval;\
 132 module_param(beiscsi_##_name, uint, S_IRUGO);\
 133 MODULE_PARM_DESC(beiscsi_##_name, _descp);\
 134 beiscsi_disp_param(_name)\
 135 beiscsi_change_param(_name, _minval, _maxval, _defval)\
 136 beiscsi_store_param(_name)\
 137 beiscsi_init_param(_name, _minval, _maxval, _defval)\
 138 DEVICE_ATTR(beiscsi_##_name, S_IRUGO | S_IWUSR,\
 139               beiscsi_##_name##_disp, beiscsi_##_name##_store)
 140
 141 /*
 142  * When new log level added update the
 143  * the MAX allowed value for log_enable
 144  */
 145 BEISCSI_RW_ATTR(log_enable, 0x00,
 146                 0xFF, 0x00, "Enable logging Bit Mask\n"
 147                 "\t\t\t\tInitialization Events  : 0x01\n"
 148                 "\t\t\t\tMailbox Events         : 0x02\n"
 149                 "\t\t\t\tMiscellaneous Events   : 0x04\n"
 150                 "\t\t\t\tError Handling         : 0x08\n"
 151                 "\t\t\t\tIO Path Events         : 0x10\n"
 152                 "\t\t\t\tConfiguration Path     : 0x20\n");
 153
 154 DEVICE_ATTR(beiscsi_drvr_ver, S_IRUGO, beiscsi_drvr_ver_disp, NULL);
 155 struct device_attribute *beiscsi_attrs[] = {
 156         &dev_attr_beiscsi_log_enable,
 157         &dev_attr_beiscsi_drvr_ver,
 158         NULL,
 159 };
 160
 161 static char const *cqe_desc[] = {
 162         "RESERVED_DESC",
 163         "SOL_CMD_COMPLETE",
 164         "SOL_CMD_KILLED_DATA_DIGEST_ERR",
 165         "CXN_KILLED_PDU_SIZE_EXCEEDS_DSL",
 166         "CXN_KILLED_BURST_LEN_MISMATCH",
 167         "CXN_KILLED_AHS_RCVD",
 168         "CXN_KILLED_HDR_DIGEST_ERR",
 169         "CXN_KILLED_UNKNOWN_HDR",
 170         "CXN_KILLED_STALE_ITT_TTT_RCVD",
 171         "CXN_KILLED_INVALID_ITT_TTT_RCVD",
 172         "CXN_KILLED_RST_RCVD",
 173         "CXN_KILLED_TIMED_OUT",
 174         "CXN_KILLED_RST_SENT",
 175         "CXN_KILLED_FIN_RCVD",
 176         "CXN_KILLED_BAD_UNSOL_PDU_RCVD",
 177         "CXN_KILLED_BAD_WRB_INDEX_ERROR",
 178         "CXN_KILLED_OVER_RUN_RESIDUAL",
 179         "CXN_KILLED_UNDER_RUN_RESIDUAL",
 180         "CMD_KILLED_INVALID_STATSN_RCVD",
 181         "CMD_KILLED_INVALID_R2T_RCVD",
 182         "CMD_CXN_KILLED_LUN_INVALID",
 183         "CMD_CXN_KILLED_ICD_INVALID",
 184         "CMD_CXN_KILLED_ITT_INVALID",
 185         "CMD_CXN_KILLED_SEQ_OUTOFORDER",
 186         "CMD_CXN_KILLED_INVALID_DATASN_RCVD",
 187         "CXN_INVALIDATE_NOTIFY",
 188         "CXN_INVALIDATE_INDEX_NOTIFY",
 189         "CMD_INVALIDATED_NOTIFY",
 190         "UNSOL_HDR_NOTIFY",
 191         "UNSOL_DATA_NOTIFY",
 192         "UNSOL_DATA_DIGEST_ERROR_NOTIFY",
 193         "DRIVERMSG_NOTIFY",
 194         "CXN_KILLED_CMND_DATA_NOT_ON_SAME_CONN",
 195         "SOL_CMD_KILLED_DIF_ERR",
 196         "CXN_KILLED_SYN_RCVD",
 197         "CXN_KILLED_IMM_DATA_RCVD"
 198 };
 199
 200 static int beiscsi_slave_configure(struct scsi_device *sdev)
 201 {
 202         blk_queue_max_segment_size(sdev->request_queue, 65536);
 203         return 0;
 204 }
 205
 206 static int beiscsi_eh_abort(struct scsi_cmnd *sc)
 207 {
 208         struct iscsi_cls_session *cls_session;
 209         struct iscsi_task *aborted_task = (struct iscsi_task *)sc->SCp.ptr;
 210         struct beiscsi_io_task *aborted_io_task;
 211         struct iscsi_conn *conn;
 212         struct beiscsi_conn *beiscsi_conn;
 213         struct beiscsi_hba *phba;
 214         struct iscsi_session *session;
 215         struct invalidate_command_table *inv_tbl;
 216         struct be_dma_mem nonemb_cmd;
 217         unsigned int cid, tag, num_invalidate;
 218
 219         cls_session = starget_to_session(scsi_target(sc->device));
 220         session = cls_session->dd_data;
 221
 222         spin_lock_bh(&session->lock);
 223         if (!aborted_task || !aborted_task->sc) {
 224                 /* we raced */
 225                 spin_unlock_bh(&session->lock);
 226                 return SUCCESS;
 227         }
 228
 229         aborted_io_task = aborted_task->dd_data;
 230         if (!aborted_io_task->scsi_cmnd) {
 231                 /* raced or invalid command */
 232                 spin_unlock_bh(&session->lock);
 233                 return SUCCESS;
 234         }
 235         spin_unlock_bh(&session->lock);
 236         conn = aborted_task->conn;
 237         beiscsi_conn = conn->dd_data;
 238         phba = beiscsi_conn->phba;
 239
 240         /* invalidate iocb */
 241         cid = beiscsi_conn->beiscsi_conn_cid;
 242         inv_tbl = phba->inv_tbl;
 243         memset(inv_tbl, 0x0, sizeof(*inv_tbl));
 244         inv_tbl->cid = cid;
 245         inv_tbl->icd = aborted_io_task->psgl_handle->sgl_index;
 246         num_invalidate = 1;
 247         nonemb_cmd.va = pci_alloc_consistent(phba->ctrl.pdev,
 248                                 sizeof(struct invalidate_commands_params_in),
 249                                 &nonemb_cmd.dma);
 250         if (nonemb_cmd.va == NULL) {
 251                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_EH,
 252                             "BM_%d : Failed to allocate memory for"
 253                             "mgmt_invalidate_icds\n");
 254                 return FAILED;
 255         }
 256         nonemb_cmd.size = sizeof(struct invalidate_commands_params_in);
 257
 258         tag = mgmt_invalidate_icds(phba, inv_tbl, num_invalidate,
 259                                    cid, &nonemb_cmd);
 260         if (!tag) {
 261                 beiscsi_log(phba, KERN_WARNING, BEISCSI_LOG_EH,
 262                             "BM_%d : mgmt_invalidate_icds could not be"
 263                             "submitted\n");
 264                 pci_free_consistent(phba->ctrl.pdev, nonemb_cmd.size,
 265                                     nonemb_cmd.va, nonemb_cmd.dma);
 266
 267                 return FAILED;
 268         } else {
 269                 wait_event_interruptible(phba->ctrl.mcc_wait[tag],
 270                                          phba->ctrl.mcc_numtag[tag]);
 271                 free_mcc_tag(&phba->ctrl, tag);
 272         }
 273         pci_free_consistent(phba->ctrl.pdev, nonemb_cmd.size,
 274                             nonemb_cmd.va, nonemb_cmd.dma);
 275         return iscsi_eh_abort(sc);
 276 }
 277
 278 static int beiscsi_eh_device_reset(struct scsi_cmnd *sc)
 279 {
 280         struct iscsi_task *abrt_task;
 281         struct beiscsi_io_task *abrt_io_task;
 282         struct iscsi_conn *conn;
 283         struct beiscsi_conn *beiscsi_conn;
 284         struct beiscsi_hba *phba;
 285         struct iscsi_session *session;
 286         struct iscsi_cls_session *cls_session;
 287         struct invalidate_command_table *inv_tbl;
 288         struct be_dma_mem nonemb_cmd;
 289         unsigned int cid, tag, i, num_invalidate;
 290
 291         /* invalidate iocbs */
 292         cls_session = starget_to_session(scsi_target(sc->device));
 293         session = cls_session->dd_data;
 294         spin_lock_bh(&session->lock);
 295         if (!session->leadconn || session->state != ISCSI_STATE_LOGGED_IN) {
 296                 spin_unlock_bh(&session->lock);
 297                 return FAILED;
 298         }
 299         conn = session->leadconn;
 300         beiscsi_conn = conn->dd_data;
 301         phba = beiscsi_conn->phba;
 302         cid = beiscsi_conn->beiscsi_conn_cid;
 303         inv_tbl = phba->inv_tbl;
 304         memset(inv_tbl, 0x0, sizeof(*inv_tbl) * BE2_CMDS_PER_CXN);
 305         num_invalidate = 0;
 306         for (i = 0; i < conn->session->cmds_max; i++) {
 307                 abrt_task = conn->session->cmds[i];
 308                 abrt_io_task = abrt_task->dd_data;
 309                 if (!abrt_task->sc || abrt_task->state == ISCSI_TASK_FREE)
 310                         continue;
 311
 312                 if (abrt_task->sc->device->lun != abrt_task->sc->device->lun)
 313                         continue;
 314
 315                 inv_tbl->cid = cid;
 316                 inv_tbl->icd = abrt_io_task->psgl_handle->sgl_index;
 317                 num_invalidate++;
 318                 inv_tbl++;
 319         }
 320         spin_unlock_bh(&session->lock);
 321         inv_tbl = phba->inv_tbl;
 322
 323         nonemb_cmd.va = pci_alloc_consistent(phba->ctrl.pdev,
 324                                 sizeof(struct invalidate_commands_params_in),
 325                                 &nonemb_cmd.dma);
 326         if (nonemb_cmd.va == NULL) {
 327                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_EH,
 328                             "BM_%d : Failed to allocate memory for"
 329                             "mgmt_invalidate_icds\n");
 330                 return FAILED;
 331         }
 332         nonemb_cmd.size = sizeof(struct invalidate_commands_params_in);
 333         memset(nonemb_cmd.va, 0, nonemb_cmd.size);
 334         tag = mgmt_invalidate_icds(phba, inv_tbl, num_invalidate,
 335                                    cid, &nonemb_cmd);
 336         if (!tag) {
 337                 beiscsi_log(phba, KERN_WARNING, BEISCSI_LOG_EH,
 338                             "BM_%d : mgmt_invalidate_icds could not be"
 339                             " submitted\n");
 340                 pci_free_consistent(phba->ctrl.pdev, nonemb_cmd.size,
 341                                     nonemb_cmd.va, nonemb_cmd.dma);
 342                 return FAILED;
 343         } else {
 344                 wait_event_interruptible(phba->ctrl.mcc_wait[tag],
 345                                          phba->ctrl.mcc_numtag[tag]);
 346                 free_mcc_tag(&phba->ctrl, tag);
 347         }
 348         pci_free_consistent(phba->ctrl.pdev, nonemb_cmd.size,
 349                             nonemb_cmd.va, nonemb_cmd.dma);
 350         return iscsi_eh_device_reset(sc);
 351 }
 352
 353 static ssize_t beiscsi_show_boot_tgt_info(void *data, int type, char *buf)
 354 {
 355         struct beiscsi_hba *phba = data;
 356         struct mgmt_session_info *boot_sess = &phba->boot_sess;
 357         struct mgmt_conn_info *boot_conn = &boot_sess->conn_list[0];
 358         char *str = buf;
 359         int rc;
 360
 361         switch (type) {
 362         case ISCSI_BOOT_TGT_NAME:
 363                 rc = sprintf(buf, "%.*s\n",
 364                             (int)strlen(boot_sess->target_name),
 365                             (char *)&boot_sess->target_name);
 366                 break;
 367         case ISCSI_BOOT_TGT_IP_ADDR:
 368                 if (boot_conn->dest_ipaddr.ip_type == 0x1)
 369                         rc = sprintf(buf, "%pI4\n",
 370                                 (char *)&boot_conn->dest_ipaddr.addr);
 371                 else
 372                         rc = sprintf(str, "%pI6\n",
 373                                 (char *)&boot_conn->dest_ipaddr.addr);
 374                 break;
 375         case ISCSI_BOOT_TGT_PORT:
 376                 rc = sprintf(str, "%d\n", boot_conn->dest_port);
 377                 break;
 378
 379         case ISCSI_BOOT_TGT_CHAP_NAME:
 380                 rc = sprintf(str,  "%.*s\n",
 381                              boot_conn->negotiated_login_options.auth_data.chap.
 382                              target_chap_name_length,
 383                              (char *)&boot_conn->negotiated_login_options.
 384                              auth_data.chap.target_chap_name);
 385                 break;
 386         case ISCSI_BOOT_TGT_CHAP_SECRET:
 387                 rc = sprintf(str,  "%.*s\n",
 388                              boot_conn->negotiated_login_options.auth_data.chap.
 389                              target_secret_length,
 390                              (char *)&boot_conn->negotiated_login_options.
 391                              auth_data.chap.target_secret);
 392                 break;
 393         case ISCSI_BOOT_TGT_REV_CHAP_NAME:
 394                 rc = sprintf(str,  "%.*s\n",
 395                              boot_conn->negotiated_login_options.auth_data.chap.
 396                              intr_chap_name_length,
 397                              (char *)&boot_conn->negotiated_login_options.
 398                              auth_data.chap.intr_chap_name);
 399                 break;
 400         case ISCSI_BOOT_TGT_REV_CHAP_SECRET:
 401                 rc = sprintf(str,  "%.*s\n",
 402                              boot_conn->negotiated_login_options.auth_data.chap.
 403                              intr_secret_length,
 404                              (char *)&boot_conn->negotiated_login_options.
 405                              auth_data.chap.intr_secret);
 406                 break;
 407         case ISCSI_BOOT_TGT_FLAGS:
 408                 rc = sprintf(str, "2\n");
 409                 break;
 410         case ISCSI_BOOT_TGT_NIC_ASSOC:
 411                 rc = sprintf(str, "0\n");
 412                 break;
 413         default:
 414                 rc = -ENOSYS;
 415                 break;
 416         }
 417         return rc;
 418 }
 419
 420 static ssize_t beiscsi_show_boot_ini_info(void *data, int type, char *buf)
 421 {
 422         struct beiscsi_hba *phba = data;
 423         char *str = buf;
 424         int rc;
 425
 426         switch (type) {
 427         case ISCSI_BOOT_INI_INITIATOR_NAME:
 428                 rc = sprintf(str, "%s\n", phba->boot_sess.initiator_iscsiname);
 429                 break;
 430         default:
 431                 rc = -ENOSYS;
 432                 break;
 433         }
 434         return rc;
 435 }
 436
 437 static ssize_t beiscsi_show_boot_eth_info(void *data, int type, char *buf)
 438 {
 439         struct beiscsi_hba *phba = data;
 440         char *str = buf;
 441         int rc;
 442
 443         switch (type) {
 444         case ISCSI_BOOT_ETH_FLAGS:
 445                 rc = sprintf(str, "2\n");
 446                 break;
 447         case ISCSI_BOOT_ETH_INDEX:
 448                 rc = sprintf(str, "0\n");
 449                 break;
 450         case ISCSI_BOOT_ETH_MAC:
 451                 rc  = beiscsi_get_macaddr(str, phba);
 452                 break;
 453         default:
 454                 rc = -ENOSYS;
 455                 break;
 456         }
 457         return rc;
 458 }
 459
 460
 461 static umode_t beiscsi_tgt_get_attr_visibility(void *data, int type)
 462 {
 463         umode_t rc;
 464
 465         switch (type) {
 466         case ISCSI_BOOT_TGT_NAME:
 467         case ISCSI_BOOT_TGT_IP_ADDR:
 468         case ISCSI_BOOT_TGT_PORT:
 469         case ISCSI_BOOT_TGT_CHAP_NAME:
 470         case ISCSI_BOOT_TGT_CHAP_SECRET:
 471         case ISCSI_BOOT_TGT_REV_CHAP_NAME:
 472         case ISCSI_BOOT_TGT_REV_CHAP_SECRET:
 473         case ISCSI_BOOT_TGT_NIC_ASSOC:
 474         case ISCSI_BOOT_TGT_FLAGS:
 475                 rc = S_IRUGO;
 476                 break;
 477         default:
 478                 rc = 0;
 479                 break;
 480         }
 481         return rc;
 482 }
 483
 484 static umode_t beiscsi_ini_get_attr_visibility(void *data, int type)
 485 {
 486         umode_t rc;
 487
 488         switch (type) {
 489         case ISCSI_BOOT_INI_INITIATOR_NAME:
 490                 rc = S_IRUGO;
 491                 break;
 492         default:
 493                 rc = 0;
 494                 break;
 495         }
 496         return rc;
 497 }
 498
 499
 500 static umode_t beiscsi_eth_get_attr_visibility(void *data, int type)
 501 {
 502         umode_t rc;
 503
 504         switch (type) {
 505         case ISCSI_BOOT_ETH_FLAGS:
 506         case ISCSI_BOOT_ETH_MAC:
 507         case ISCSI_BOOT_ETH_INDEX:
 508                 rc = S_IRUGO;
 509                 break;
 510         default:
 511                 rc = 0;
 512                 break;
 513         }
 514         return rc;
 515 }
 516
 517 /*------------------- PCI Driver operations and data ----------------- */
 518 static DEFINE_PCI_DEVICE_TABLE(beiscsi_pci_id_table) = {
 519         { PCI_DEVICE(BE_VENDOR_ID, BE_DEVICE_ID1) },
 520         { PCI_DEVICE(BE_VENDOR_ID, BE_DEVICE_ID2) },
 521         { PCI_DEVICE(BE_VENDOR_ID, OC_DEVICE_ID1) },
 522         { PCI_DEVICE(BE_VENDOR_ID, OC_DEVICE_ID2) },
 523         { PCI_DEVICE(BE_VENDOR_ID, OC_DEVICE_ID3) },
 524         { PCI_DEVICE(ELX_VENDOR_ID, OC_SKH_ID1) },
 525         { 0 }
 526 };
 527 MODULE_DEVICE_TABLE(pci, beiscsi_pci_id_table);
 528
 529
 530 static struct scsi_host_template beiscsi_sht = {
 531         .module = THIS_MODULE,
 532         .name = "Emulex 10Gbe open-iscsi Initiator Driver",
 533         .proc_name = DRV_NAME,
 534         .queuecommand = iscsi_queuecommand,
 535         .change_queue_depth = iscsi_change_queue_depth,
 536         .slave_configure = beiscsi_slave_configure,
 537         .target_alloc = iscsi_target_alloc,
 538         .eh_abort_handler = beiscsi_eh_abort,
 539         .eh_device_reset_handler = beiscsi_eh_device_reset,
 540         .eh_target_reset_handler = iscsi_eh_session_reset,
 541         .shost_attrs = beiscsi_attrs,
 542         .sg_tablesize = BEISCSI_SGLIST_ELEMENTS,
 543         .can_queue = BE2_IO_DEPTH,
 544         .this_id = -1,
 545         .max_sectors = BEISCSI_MAX_SECTORS,
 546         .cmd_per_lun = BEISCSI_CMD_PER_LUN,
 547         .use_clustering = ENABLE_CLUSTERING,
 548         .vendor_id = SCSI_NL_VID_TYPE_PCI | BE_VENDOR_ID,
 549
 550 };
 551
 552 static struct scsi_transport_template *beiscsi_scsi_transport;
 553
 554 static struct beiscsi_hba *beiscsi_hba_alloc(struct pci_dev *pcidev)
 555 {
 556         struct beiscsi_hba *phba;
 557         struct Scsi_Host *shost;
 558
 559         shost = iscsi_host_alloc(&beiscsi_sht, sizeof(*phba), 0);
 560         if (!shost) {
 561                 dev_err(&pcidev->dev,
 562                         "beiscsi_hba_alloc - iscsi_host_alloc failed\n");
 563                 return NULL;
 564         }
 565         shost->dma_boundary = pcidev->dma_mask;
 566         shost->max_id = BE2_MAX_SESSIONS;
 567         shost->max_channel = 0;
 568         shost->max_cmd_len = BEISCSI_MAX_CMD_LEN;
 569         shost->max_lun = BEISCSI_NUM_MAX_LUN;
 570         shost->transportt = beiscsi_scsi_transport;
 571         phba = iscsi_host_priv(shost);
 572         memset(phba, 0, sizeof(*phba));
 573         phba->shost = shost;
 574         phba->pcidev = pci_dev_get(pcidev);
 575         pci_set_drvdata(pcidev, phba);
 576         phba->interface_handle = 0xFFFFFFFF;
 577
 578         if (iscsi_host_add(shost, &phba->pcidev->dev))
 579                 goto free_devices;
 580
 581         return phba;
 582
 583 free_devices:
 584         pci_dev_put(phba->pcidev);
 585         iscsi_host_free(phba->shost);
 586         return NULL;
 587 }
 588
 589 static void beiscsi_unmap_pci_function(struct beiscsi_hba *phba)
 590 {
 591         if (phba->csr_va) {
 592                 iounmap(phba->csr_va);
 593                 phba->csr_va = NULL;
 594         }
 595         if (phba->db_va) {
 596                 iounmap(phba->db_va);
 597                 phba->db_va = NULL;
 598         }
 599         if (phba->pci_va) {
 600                 iounmap(phba->pci_va);
 601                 phba->pci_va = NULL;
 602         }
 603 }
 604
 605 static int beiscsi_map_pci_bars(struct beiscsi_hba *phba,
 606                                 struct pci_dev *pcidev)
 607 {
 608         u8 __iomem *addr;
 609         int pcicfg_reg;
 610
 611         addr = ioremap_nocache(pci_resource_start(pcidev, 2),
 612                                pci_resource_len(pcidev, 2));
 613         if (addr == NULL)
 614                 return -ENOMEM;
 615         phba->ctrl.csr = addr;
 616         phba->csr_va = addr;
 617         phba->csr_pa.u.a64.address = pci_resource_start(pcidev, 2);
 618
 619         addr = ioremap_nocache(pci_resource_start(pcidev, 4), 128 * 1024);
 620         if (addr == NULL)
 621                 goto pci_map_err;
 622         phba->ctrl.db = addr;
 623         phba->db_va = addr;
 624         phba->db_pa.u.a64.address =  pci_resource_start(pcidev, 4);
 625
 626         if (phba->generation == BE_GEN2)
 627                 pcicfg_reg = 1;
 628         else
 629                 pcicfg_reg = 0;
 630
 631         addr = ioremap_nocache(pci_resource_start(pcidev, pcicfg_reg),
 632                                pci_resource_len(pcidev, pcicfg_reg));
 633
 634         if (addr == NULL)
 635                 goto pci_map_err;
 636         phba->ctrl.pcicfg = addr;
 637         phba->pci_va = addr;
 638         phba->pci_pa.u.a64.address = pci_resource_start(pcidev, pcicfg_reg);
 639         return 0;
 640
 641 pci_map_err:
 642         beiscsi_unmap_pci_function(phba);
 643         return -ENOMEM;
 644 }
 645
 646 static int beiscsi_enable_pci(struct pci_dev *pcidev)
 647 {
 648         int ret;
 649
 650         ret = pci_enable_device(pcidev);
 651         if (ret) {
 652                 dev_err(&pcidev->dev,
 653                         "beiscsi_enable_pci - enable device failed\n");
 654                 return ret;
 655         }
 656
 657         pci_set_master(pcidev);
 658         if (pci_set_consistent_dma_mask(pcidev, DMA_BIT_MASK(64))) {
 659                 ret = pci_set_consistent_dma_mask(pcidev, DMA_BIT_MASK(32));
 660                 if (ret) {
 661                         dev_err(&pcidev->dev, "Could not set PCI DMA Mask\n");
 662                         pci_disable_device(pcidev);
 663                         return ret;
 664                 }
 665         }
 666         return 0;
 667 }
 668
 669 static int be_ctrl_init(struct beiscsi_hba *phba, struct pci_dev *pdev)
 670 {
 671         struct be_ctrl_info *ctrl = &phba->ctrl;
 672         struct be_dma_mem *mbox_mem_alloc = &ctrl->mbox_mem_alloced;
 673         struct be_dma_mem *mbox_mem_align = &ctrl->mbox_mem;
 674         int status = 0;
 675
 676         ctrl->pdev = pdev;
 677         status = beiscsi_map_pci_bars(phba, pdev);
 678         if (status)
 679                 return status;
 680         mbox_mem_alloc->size = sizeof(struct be_mcc_mailbox) + 16;
 681         mbox_mem_alloc->va = pci_alloc_consistent(pdev,
 682                                                   mbox_mem_alloc->size,
 683                                                   &mbox_mem_alloc->dma);
 684         if (!mbox_mem_alloc->va) {
 685                 beiscsi_unmap_pci_function(phba);
 686                 return -ENOMEM;
 687         }
 688
 689         mbox_mem_align->size = sizeof(struct be_mcc_mailbox);
 690         mbox_mem_align->va = PTR_ALIGN(mbox_mem_alloc->va, 16);
 691         mbox_mem_align->dma = PTR_ALIGN(mbox_mem_alloc->dma, 16);
 692         memset(mbox_mem_align->va, 0, sizeof(struct be_mcc_mailbox));
 693         spin_lock_init(&ctrl->mbox_lock);
 694         spin_lock_init(&phba->ctrl.mcc_lock);
 695         spin_lock_init(&phba->ctrl.mcc_cq_lock);
 696
 697         return status;
 698 }
 699
 700 static void beiscsi_get_params(struct beiscsi_hba *phba)
 701 {
 702         phba->params.ios_per_ctrl = (phba->fw_config.iscsi_icd_count
 703                                     - (phba->fw_config.iscsi_cid_count
 704                                     + BE2_TMFS
 705                                     + BE2_NOPOUT_REQ));
 706         phba->params.cxns_per_ctrl = phba->fw_config.iscsi_cid_count;
 707         phba->params.asyncpdus_per_ctrl = phba->fw_config.iscsi_cid_count * 2;
 708         phba->params.icds_per_ctrl = phba->fw_config.iscsi_icd_count;
 709         phba->params.num_sge_per_io = BE2_SGE;
 710         phba->params.defpdu_hdr_sz = BE2_DEFPDU_HDR_SZ;
 711         phba->params.defpdu_data_sz = BE2_DEFPDU_DATA_SZ;
 712         phba->params.eq_timer = 64;
 713         phba->params.num_eq_entries =
 714             (((BE2_CMDS_PER_CXN * 2 + phba->fw_config.iscsi_cid_count * 2
 715                                     + BE2_TMFS) / 512) + 1) * 512;
 716         phba->params.num_eq_entries = (phba->params.num_eq_entries < 1024)
 717                                 ? 1024 : phba->params.num_eq_entries;
 718         beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
 719                     "BM_%d : phba->params.num_eq_entries=%d\n",
 720                     phba->params.num_eq_entries);
 721         phba->params.num_cq_entries =
 722             (((BE2_CMDS_PER_CXN * 2 +  phba->fw_config.iscsi_cid_count * 2
 723                                     + BE2_TMFS) / 512) + 1) * 512;
 724         phba->params.wrbs_per_cxn = 256;
 725 }
 726
 727 static void hwi_ring_eq_db(struct beiscsi_hba *phba,
 728                            unsigned int id, unsigned int clr_interrupt,
 729                            unsigned int num_processed,
 730                            unsigned char rearm, unsigned char event)
 731 {
 732         u32 val = 0;
 733         val |= id & DB_EQ_RING_ID_MASK;
 734         if (rearm)
 735                 val |= 1 << DB_EQ_REARM_SHIFT;
 736         if (clr_interrupt)
 737                 val |= 1 << DB_EQ_CLR_SHIFT;
 738         if (event)
 739                 val |= 1 << DB_EQ_EVNT_SHIFT;
 740         val |= num_processed << DB_EQ_NUM_POPPED_SHIFT;
 741         iowrite32(val, phba->db_va + DB_EQ_OFFSET);
 742 }
 743
 744 /**
 745  * be_isr_mcc - The isr routine of the driver.
 746  * @irq: Not used
 747  * @dev_id: Pointer to host adapter structure
 748  */
 749 static irqreturn_t be_isr_mcc(int irq, void *dev_id)
 750 {
 751         struct beiscsi_hba *phba;
 752         struct be_eq_entry *eqe = NULL;
 753         struct be_queue_info *eq;
 754         struct be_queue_info *mcc;
 755         unsigned int num_eq_processed;
 756         struct be_eq_obj *pbe_eq;
 757         unsigned long flags;
 758
 759         pbe_eq = dev_id;
 760         eq = &pbe_eq->q;
 761         phba =  pbe_eq->phba;
 762         mcc = &phba->ctrl.mcc_obj.cq;
 763         eqe = queue_tail_node(eq);
 764
 765         num_eq_processed = 0;
 766
 767         while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32]
 768                                 & EQE_VALID_MASK) {
 769                 if (((eqe->dw[offsetof(struct amap_eq_entry,
 770                      resource_id) / 32] &
 771                      EQE_RESID_MASK) >> 16) == mcc->id) {
 772                         spin_lock_irqsave(&phba->isr_lock, flags);
 773                         pbe_eq->todo_mcc_cq = true;
 774                         spin_unlock_irqrestore(&phba->isr_lock, flags);
 775                 }
 776                 AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0);
 777                 queue_tail_inc(eq);
 778                 eqe = queue_tail_node(eq);
 779                 num_eq_processed++;
 780         }
 781         if (pbe_eq->todo_mcc_cq)
 782                 queue_work(phba->wq, &pbe_eq->work_cqs);
 783         if (num_eq_processed)
 784                 hwi_ring_eq_db(phba, eq->id, 1, num_eq_processed, 1, 1);
 785
 786         return IRQ_HANDLED;
 787 }
 788
 789 /**
 790  * be_isr_msix - The isr routine of the driver.
 791  * @irq: Not used
 792  * @dev_id: Pointer to host adapter structure
 793  */
 794 static irqreturn_t be_isr_msix(int irq, void *dev_id)
 795 {
 796         struct beiscsi_hba *phba;
 797         struct be_eq_entry *eqe = NULL;
 798         struct be_queue_info *eq;
 799         struct be_queue_info *cq;
 800         unsigned int num_eq_processed;
 801         struct be_eq_obj *pbe_eq;
 802         unsigned long flags;
 803
 804         pbe_eq = dev_id;
 805         eq = &pbe_eq->q;
 806         cq = pbe_eq->cq;
 807         eqe = queue_tail_node(eq);
 808
 809         phba = pbe_eq->phba;
 810         num_eq_processed = 0;
 811         if (blk_iopoll_enabled) {
 812                 while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32]
 813                                         & EQE_VALID_MASK) {
 814                         if (!blk_iopoll_sched_prep(&pbe_eq->iopoll))
 815                                 blk_iopoll_sched(&pbe_eq->iopoll);
 816
 817                         AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0);
 818                         queue_tail_inc(eq);
 819                         eqe = queue_tail_node(eq);
 820                         num_eq_processed++;
 821                 }
 822         } else {
 823                 while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32]
 824                                                 & EQE_VALID_MASK) {
 825                         spin_lock_irqsave(&phba->isr_lock, flags);
 826                         pbe_eq->todo_cq = true;
 827                         spin_unlock_irqrestore(&phba->isr_lock, flags);
 828                         AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0);
 829                         queue_tail_inc(eq);
 830                         eqe = queue_tail_node(eq);
 831                         num_eq_processed++;
 832                 }
 833
 834                 if (pbe_eq->todo_cq)
 835                         queue_work(phba->wq, &pbe_eq->work_cqs);
 836         }
 837
 838         if (num_eq_processed)
 839                 hwi_ring_eq_db(phba, eq->id, 1, num_eq_processed, 0, 1);
 840
 841         return IRQ_HANDLED;
 842 }
 843
 844 /**
 845  * be_isr - The isr routine of the driver.
 846  * @irq: Not used
 847  * @dev_id: Pointer to host adapter structure
 848  */
 849 static irqreturn_t be_isr(int irq, void *dev_id)
 850 {
 851         struct beiscsi_hba *phba;
 852         struct hwi_controller *phwi_ctrlr;
 853         struct hwi_context_memory *phwi_context;
 854         struct be_eq_entry *eqe = NULL;
 855         struct be_queue_info *eq;
 856         struct be_queue_info *cq;
 857         struct be_queue_info *mcc;
 858         unsigned long flags, index;
 859         unsigned int num_mcceq_processed, num_ioeq_processed;
 860         struct be_ctrl_info *ctrl;
 861         struct be_eq_obj *pbe_eq;
 862         int isr;
 863
 864         phba = dev_id;
 865         ctrl = &phba->ctrl;
 866         isr = ioread32(ctrl->csr + CEV_ISR0_OFFSET +
 867                        (PCI_FUNC(ctrl->pdev->devfn) * CEV_ISR_SIZE));
 868         if (!isr)
 869                 return IRQ_NONE;
 870
 871         phwi_ctrlr = phba->phwi_ctrlr;
 872         phwi_context = phwi_ctrlr->phwi_ctxt;
 873         pbe_eq = &phwi_context->be_eq[0];
 874
 875         eq = &phwi_context->be_eq[0].q;
 876         mcc = &phba->ctrl.mcc_obj.cq;
 877         index = 0;
 878         eqe = queue_tail_node(eq);
 879
 880         num_ioeq_processed = 0;
 881         num_mcceq_processed = 0;
 882         if (blk_iopoll_enabled) {
 883                 while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32]
 884                                         & EQE_VALID_MASK) {
 885                         if (((eqe->dw[offsetof(struct amap_eq_entry,
 886                              resource_id) / 32] &
 887                              EQE_RESID_MASK) >> 16) == mcc->id) {
 888                                 spin_lock_irqsave(&phba->isr_lock, flags);
 889                                 pbe_eq->todo_mcc_cq = true;
 890                                 spin_unlock_irqrestore(&phba->isr_lock, flags);
 891                                 num_mcceq_processed++;
 892                         } else {
 893                                 if (!blk_iopoll_sched_prep(&pbe_eq->iopoll))
 894                                         blk_iopoll_sched(&pbe_eq->iopoll);
 895                                 num_ioeq_processed++;
 896                         }
 897                         AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0);
 898                         queue_tail_inc(eq);
 899                         eqe = queue_tail_node(eq);
 900                 }
 901                 if (num_ioeq_processed || num_mcceq_processed) {
 902                         if (pbe_eq->todo_mcc_cq)
 903                                 queue_work(phba->wq, &pbe_eq->work_cqs);
 904
 905                         if ((num_mcceq_processed) && (!num_ioeq_processed))
 906                                 hwi_ring_eq_db(phba, eq->id, 0,
 907                                               (num_ioeq_processed +
 908                                                num_mcceq_processed) , 1, 1);
 909                         else
 910                                 hwi_ring_eq_db(phba, eq->id, 0,
 911                                                (num_ioeq_processed +
 912                                                 num_mcceq_processed), 0, 1);
 913
 914                         return IRQ_HANDLED;
 915                 } else
 916                         return IRQ_NONE;
 917         } else {
 918                 cq = &phwi_context->be_cq[0];
 919                 while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32]
 920                                                 & EQE_VALID_MASK) {
 921
 922                         if (((eqe->dw[offsetof(struct amap_eq_entry,
 923                              resource_id) / 32] &
 924                              EQE_RESID_MASK) >> 16) != cq->id) {
 925                                 spin_lock_irqsave(&phba->isr_lock, flags);
 926                                 pbe_eq->todo_mcc_cq = true;
 927                                 spin_unlock_irqrestore(&phba->isr_lock, flags);
 928                         } else {
 929                                 spin_lock_irqsave(&phba->isr_lock, flags);
 930                                 pbe_eq->todo_cq = true;
 931                                 spin_unlock_irqrestore(&phba->isr_lock, flags);
 932                         }
 933                         AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0);
 934                         queue_tail_inc(eq);
 935                         eqe = queue_tail_node(eq);
 936                         num_ioeq_processed++;
 937                 }
 938                 if (pbe_eq->todo_cq || pbe_eq->todo_mcc_cq)
 939                         queue_work(phba->wq, &pbe_eq->work_cqs);
 940
 941                 if (num_ioeq_processed) {
 942                         hwi_ring_eq_db(phba, eq->id, 0,
 943                                        num_ioeq_processed, 1, 1);
 944                         return IRQ_HANDLED;
 945                 } else
 946                         return IRQ_NONE;
 947         }
 948 }
 949
 950 static int beiscsi_init_irqs(struct beiscsi_hba *phba)
 951 {
 952         struct pci_dev *pcidev = phba->pcidev;
 953         struct hwi_controller *phwi_ctrlr;
 954         struct hwi_context_memory *phwi_context;
 955         int ret, msix_vec, i, j;
 956
 957         phwi_ctrlr = phba->phwi_ctrlr;
 958         phwi_context = phwi_ctrlr->phwi_ctxt;
 959
 960         if (phba->msix_enabled) {
 961                 for (i = 0; i < phba->num_cpus; i++) {
 962                         phba->msi_name[i] = kzalloc(BEISCSI_MSI_NAME,
 963                                                     GFP_KERNEL);
 964                         if (!phba->msi_name[i]) {
 965                                 ret = -ENOMEM;
 966                                 goto free_msix_irqs;
 967                         }
 968
 969                         sprintf(phba->msi_name[i], "beiscsi_%02x_%02x",
 970                                 phba->shost->host_no, i);
 971                         msix_vec = phba->msix_entries[i].vector;
 972                         ret = request_irq(msix_vec, be_isr_msix, 0,
 973                                           phba->msi_name[i],
 974                                           &phwi_context->be_eq[i]);
 975                         if (ret) {
 976                                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
 977                                             "BM_%d : beiscsi_init_irqs-Failed to"
 978                                             "register msix for i = %d\n",
 979                                             i);
 980                                 kfree(phba->msi_name[i]);
 981                                 goto free_msix_irqs;
 982                         }
 983                 }
 984                 phba->msi_name[i] = kzalloc(BEISCSI_MSI_NAME, GFP_KERNEL);
 985                 if (!phba->msi_name[i]) {
 986                         ret = -ENOMEM;
 987                         goto free_msix_irqs;
 988                 }
 989                 sprintf(phba->msi_name[i], "beiscsi_mcc_%02x",
 990                         phba->shost->host_no);
 991                 msix_vec = phba->msix_entries[i].vector;
 992                 ret = request_irq(msix_vec, be_isr_mcc, 0, phba->msi_name[i],
 993                                   &phwi_context->be_eq[i]);
 994                 if (ret) {
 995                         beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT ,
 996                                     "BM_%d : beiscsi_init_irqs-"
 997                                     "Failed to register beiscsi_msix_mcc\n");
 998                         kfree(phba->msi_name[i]);
 999                         goto free_msix_irqs;
1000                 }
1001
1002         } else {
1003                 ret = request_irq(pcidev->irq, be_isr, IRQF_SHARED,
1004                                   "beiscsi", phba);
1005                 if (ret) {
1006                         beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
1007                                     "BM_%d : beiscsi_init_irqs-"
1008                                     "Failed to register irq\\n");
1009                         return ret;
1010                 }
1011         }
1012         return 0;
1013 free_msix_irqs:
1014         for (j = i - 1; j >= 0; j--) {
1015                 kfree(phba->msi_name[j]);
1016                 msix_vec = phba->msix_entries[j].vector;
1017                 free_irq(msix_vec, &phwi_context->be_eq[j]);
1018         }
1019         return ret;
1020 }
1021
1022 static void hwi_ring_cq_db(struct beiscsi_hba *phba,
1023                            unsigned int id, unsigned int num_processed,
1024                            unsigned char rearm, unsigned char event)
1025 {
1026         u32 val = 0;
1027         val |= id & DB_CQ_RING_ID_MASK;
1028         if (rearm)
1029                 val |= 1 << DB_CQ_REARM_SHIFT;
1030         val |= num_processed << DB_CQ_NUM_POPPED_SHIFT;
1031         iowrite32(val, phba->db_va + DB_CQ_OFFSET);
1032 }
1033
1034 static unsigned int
1035 beiscsi_process_async_pdu(struct beiscsi_conn *beiscsi_conn,
1036                           struct beiscsi_hba *phba,
1037                           unsigned short cid,
1038                           struct pdu_base *ppdu,
1039                           unsigned long pdu_len,
1040                           void *pbuffer, unsigned long buf_len)
1041 {
1042         struct iscsi_conn *conn = beiscsi_conn->conn;
1043         struct iscsi_session *session = conn->session;
1044         struct iscsi_task *task;
1045         struct beiscsi_io_task *io_task;
1046         struct iscsi_hdr *login_hdr;
1047
1048         switch (ppdu->dw[offsetof(struct amap_pdu_base, opcode) / 32] &
1049                                                 PDUBASE_OPCODE_MASK) {
1050         case ISCSI_OP_NOOP_IN:
1051                 pbuffer = NULL;
1052                 buf_len = 0;
1053                 break;
1054         case ISCSI_OP_ASYNC_EVENT:
1055                 break;
1056         case ISCSI_OP_REJECT:
1057                 WARN_ON(!pbuffer);
1058                 WARN_ON(!(buf_len == 48));
1059                 beiscsi_log(phba, KERN_ERR,
1060                             BEISCSI_LOG_CONFIG | BEISCSI_LOG_IO,
1061                             "BM_%d : In ISCSI_OP_REJECT\n");
1062                 break;
1063         case ISCSI_OP_LOGIN_RSP:
1064         case ISCSI_OP_TEXT_RSP:
1065                 task = conn->login_task;
1066                 io_task = task->dd_data;
1067                 login_hdr = (struct iscsi_hdr *)ppdu;
1068                 login_hdr->itt = io_task->libiscsi_itt;
1069                 break;
1070         default:
1071                 beiscsi_log(phba, KERN_WARNING,
1072                             BEISCSI_LOG_IO | BEISCSI_LOG_CONFIG,
1073                             "BM_%d : Unrecognized opcode 0x%x in async msg\n",
1074                             (ppdu->
1075                              dw[offsetof(struct amap_pdu_base, opcode) / 32]
1076                              & PDUBASE_OPCODE_MASK));
1077                 return 1;
1078         }
1079
1080         spin_lock_bh(&session->lock);
1081         __iscsi_complete_pdu(conn, (struct iscsi_hdr *)ppdu, pbuffer, buf_len);
1082         spin_unlock_bh(&session->lock);
1083         return 0;
1084 }
1085
1086 static struct sgl_handle *alloc_io_sgl_handle(struct beiscsi_hba *phba)
1087 {
1088         struct sgl_handle *psgl_handle;
1089
1090         if (phba->io_sgl_hndl_avbl) {
1091                 beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_IO,
1092                             "BM_%d : In alloc_io_sgl_handle,"
1093                             " io_sgl_alloc_index=%d\n",
1094                             phba->io_sgl_alloc_index);
1095
1096                 psgl_handle = phba->io_sgl_hndl_base[phba->
1097                                                 io_sgl_alloc_index];
1098                 phba->io_sgl_hndl_base[phba->io_sgl_alloc_index] = NULL;
1099                 phba->io_sgl_hndl_avbl--;
1100                 if (phba->io_sgl_alloc_index == (phba->params.
1101                                                  ios_per_ctrl - 1))
1102                         phba->io_sgl_alloc_index = 0;
1103                 else
1104                         phba->io_sgl_alloc_index++;
1105         } else
1106                 psgl_handle = NULL;
1107         return psgl_handle;
1108 }
1109
1110 static void
1111 free_io_sgl_handle(struct beiscsi_hba *phba, struct sgl_handle *psgl_handle)
1112 {
1113         beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_IO,
1114                     "BM_%d : In free_,io_sgl_free_index=%d\n",
1115                     phba->io_sgl_free_index);
1116
1117         if (phba->io_sgl_hndl_base[phba->io_sgl_free_index]) {
1118                 /*
1119                  * this can happen if clean_task is called on a task that
1120                  * failed in xmit_task or alloc_pdu.
1121                  */
1122                  beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_IO,
1123                              "BM_%d : Double Free in IO SGL io_sgl_free_index=%d,"
1124                              "value there=%p\n", phba->io_sgl_free_index,
1125                              phba->io_sgl_hndl_base
1126                              [phba->io_sgl_free_index]);
1127                 return;
1128         }
1129         phba->io_sgl_hndl_base[phba->io_sgl_free_index] = psgl_handle;
1130         phba->io_sgl_hndl_avbl++;
1131         if (phba->io_sgl_free_index == (phba->params.ios_per_ctrl - 1))
1132                 phba->io_sgl_free_index = 0;
1133         else
1134                 phba->io_sgl_free_index++;
1135 }
1136
1137 /**
1138  * alloc_wrb_handle - To allocate a wrb handle
1139  * @phba: The hba pointer
1140  * @cid: The cid to use for allocation
1141  *
1142  * This happens under session_lock until submission to chip
1143  */
1144 struct wrb_handle *alloc_wrb_handle(struct beiscsi_hba *phba, unsigned int cid)
1145 {
1146         struct hwi_wrb_context *pwrb_context;
1147         struct hwi_controller *phwi_ctrlr;
1148         struct wrb_handle *pwrb_handle, *pwrb_handle_tmp;
1149
1150         phwi_ctrlr = phba->phwi_ctrlr;
1151         pwrb_context = &phwi_ctrlr->wrb_context[cid];
1152         if (pwrb_context->wrb_handles_available >= 2) {
1153                 pwrb_handle = pwrb_context->pwrb_handle_base[
1154                                             pwrb_context->alloc_index];
1155                 pwrb_context->wrb_handles_available--;
1156                 if (pwrb_context->alloc_index ==
1157                                                 (phba->params.wrbs_per_cxn - 1))
1158                         pwrb_context->alloc_index = 0;
1159                 else
1160                         pwrb_context->alloc_index++;
1161                 pwrb_handle_tmp = pwrb_context->pwrb_handle_base[
1162                                                 pwrb_context->alloc_index];
1163                 pwrb_handle->nxt_wrb_index = pwrb_handle_tmp->wrb_index;
1164         } else
1165                 pwrb_handle = NULL;
1166         return pwrb_handle;
1167 }
1168
1169 /**
1170  * free_wrb_handle - To free the wrb handle back to pool
1171  * @phba: The hba pointer
1172  * @pwrb_context: The context to free from
1173  * @pwrb_handle: The wrb_handle to free
1174  *
1175  * This happens under session_lock until submission to chip
1176  */
1177 static void
1178 free_wrb_handle(struct beiscsi_hba *phba, struct hwi_wrb_context *pwrb_context,
1179                 struct wrb_handle *pwrb_handle)
1180 {
1181         pwrb_context->pwrb_handle_base[pwrb_context->free_index] = pwrb_handle;
1182         pwrb_context->wrb_handles_available++;
1183         if (pwrb_context->free_index == (phba->params.wrbs_per_cxn - 1))
1184                 pwrb_context->free_index = 0;
1185         else
1186                 pwrb_context->free_index++;
1187
1188         beiscsi_log(phba, KERN_INFO,
1189                     BEISCSI_LOG_IO | BEISCSI_LOG_CONFIG,
1190                     "BM_%d : FREE WRB: pwrb_handle=%p free_index=0x%x"
1191                     "wrb_handles_available=%d\n",
1192                     pwrb_handle, pwrb_context->free_index,
1193                     pwrb_context->wrb_handles_available);
1194 }
1195
1196 static struct sgl_handle *alloc_mgmt_sgl_handle(struct beiscsi_hba *phba)
1197 {
1198         struct sgl_handle *psgl_handle;
1199
1200         if (phba->eh_sgl_hndl_avbl) {
1201                 psgl_handle = phba->eh_sgl_hndl_base[phba->eh_sgl_alloc_index];
1202                 phba->eh_sgl_hndl_base[phba->eh_sgl_alloc_index] = NULL;
1203                 beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_CONFIG,
1204                             "BM_%d : mgmt_sgl_alloc_index=%d=0x%x\n",
1205                             phba->eh_sgl_alloc_index,
1206                             phba->eh_sgl_alloc_index);
1207
1208                 phba->eh_sgl_hndl_avbl--;
1209                 if (phba->eh_sgl_alloc_index ==
1210                     (phba->params.icds_per_ctrl - phba->params.ios_per_ctrl -
1211                      1))
1212                         phba->eh_sgl_alloc_index = 0;
1213                 else
1214                         phba->eh_sgl_alloc_index++;
1215         } else
1216                 psgl_handle = NULL;
1217         return psgl_handle;
1218 }
1219
1220 void
1221 free_mgmt_sgl_handle(struct beiscsi_hba *phba, struct sgl_handle *psgl_handle)
1222 {
1223
1224         beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_CONFIG,
1225                     "BM_%d : In  free_mgmt_sgl_handle,"
1226                     "eh_sgl_free_index=%d\n",
1227                     phba->eh_sgl_free_index);
1228
1229         if (phba->eh_sgl_hndl_base[phba->eh_sgl_free_index]) {
1230                 /*
1231                  * this can happen if clean_task is called on a task that
1232                  * failed in xmit_task or alloc_pdu.
1233                  */
1234                 beiscsi_log(phba, KERN_WARNING, BEISCSI_LOG_CONFIG,
1235                             "BM_%d : Double Free in eh SGL ,"
1236                             "eh_sgl_free_index=%d\n",
1237                             phba->eh_sgl_free_index);
1238                 return;
1239         }
1240         phba->eh_sgl_hndl_base[phba->eh_sgl_free_index] = psgl_handle;
1241         phba->eh_sgl_hndl_avbl++;
1242         if (phba->eh_sgl_free_index ==
1243             (phba->params.icds_per_ctrl - phba->params.ios_per_ctrl - 1))
1244                 phba->eh_sgl_free_index = 0;
1245         else
1246                 phba->eh_sgl_free_index++;
1247 }
1248
1249 static void
1250 be_complete_io(struct beiscsi_conn *beiscsi_conn,
1251                struct iscsi_task *task, struct sol_cqe *psol)
1252 {
1253         struct beiscsi_io_task *io_task = task->dd_data;
1254         struct be_status_bhs *sts_bhs =
1255                                 (struct be_status_bhs *)io_task->cmd_bhs;
1256         struct iscsi_conn *conn = beiscsi_conn->conn;
1257         unsigned char *sense;
1258         u32 resid = 0, exp_cmdsn, max_cmdsn;
1259         u8 rsp, status, flags;
1260
1261         exp_cmdsn = (psol->
1262                         dw[offsetof(struct amap_sol_cqe, i_exp_cmd_sn) / 32]
1263                         & SOL_EXP_CMD_SN_MASK);
1264         max_cmdsn = ((psol->
1265                         dw[offsetof(struct amap_sol_cqe, i_exp_cmd_sn) / 32]
1266                         & SOL_EXP_CMD_SN_MASK) +
1267                         ((psol->dw[offsetof(struct amap_sol_cqe, i_cmd_wnd)
1268                                 / 32] & SOL_CMD_WND_MASK) >> 24) - 1);
1269         rsp = ((psol->dw[offsetof(struct amap_sol_cqe, i_resp) / 32]
1270                                                 & SOL_RESP_MASK) >> 16);
1271         status = ((psol->dw[offsetof(struct amap_sol_cqe, i_sts) / 32]
1272                                                 & SOL_STS_MASK) >> 8);
1273         flags = ((psol->dw[offsetof(struct amap_sol_cqe, i_flags) / 32]
1274                                         & SOL_FLAGS_MASK) >> 24) | 0x80;
1275         if (!task->sc) {
1276                 if (io_task->scsi_cmnd)
1277                         scsi_dma_unmap(io_task->scsi_cmnd);
1278
1279                 return;
1280         }
1281         task->sc->result = (DID_OK << 16) | status;
1282         if (rsp != ISCSI_STATUS_CMD_COMPLETED) {
1283                 task->sc->result = DID_ERROR << 16;
1284                 goto unmap;
1285         }
1286
1287         /* bidi not initially supported */
1288         if (flags & (ISCSI_FLAG_CMD_UNDERFLOW | ISCSI_FLAG_CMD_OVERFLOW)) {
1289                 resid = (psol->dw[offsetof(struct amap_sol_cqe, i_res_cnt) /
1290                                 32] & SOL_RES_CNT_MASK);
1291
1292                 if (!status && (flags & ISCSI_FLAG_CMD_OVERFLOW))
1293                         task->sc->result = DID_ERROR << 16;
1294
1295                 if (flags & ISCSI_FLAG_CMD_UNDERFLOW) {
1296                         scsi_set_resid(task->sc, resid);
1297                         if (!status && (scsi_bufflen(task->sc) - resid <
1298                             task->sc->underflow))
1299                                 task->sc->result = DID_ERROR << 16;
1300                 }
1301         }
1302
1303         if (status == SAM_STAT_CHECK_CONDITION) {
1304                 u16 sense_len;
1305                 unsigned short *slen = (unsigned short *)sts_bhs->sense_info;
1306
1307                 sense = sts_bhs->sense_info + sizeof(unsigned short);
1308                 sense_len = be16_to_cpu(*slen);
1309                 memcpy(task->sc->sense_buffer, sense,
1310                        min_t(u16, sense_len, SCSI_SENSE_BUFFERSIZE));
1311         }
1312
1313         if (io_task->cmd_bhs->iscsi_hdr.flags & ISCSI_FLAG_CMD_READ) {
1314                 if (psol->dw[offsetof(struct amap_sol_cqe, i_res_cnt) / 32]
1315                                                         & SOL_RES_CNT_MASK)
1316                          conn->rxdata_octets += (psol->
1317                              dw[offsetof(struct amap_sol_cqe, i_res_cnt) / 32]
1318                              & SOL_RES_CNT_MASK);
1319         }
1320 unmap:
1321         scsi_dma_unmap(io_task->scsi_cmnd);
1322         iscsi_complete_scsi_task(task, exp_cmdsn, max_cmdsn);
1323 }
1324
1325 static void
1326 be_complete_logout(struct beiscsi_conn *beiscsi_conn,
1327                    struct iscsi_task *task, struct sol_cqe *psol)
1328 {
1329         struct iscsi_logout_rsp *hdr;
1330         struct beiscsi_io_task *io_task = task->dd_data;
1331         struct iscsi_conn *conn = beiscsi_conn->conn;
1332
1333         hdr = (struct iscsi_logout_rsp *)task->hdr;
1334         hdr->opcode = ISCSI_OP_LOGOUT_RSP;
1335         hdr->t2wait = 5;
1336         hdr->t2retain = 0;
1337         hdr->flags = ((psol->dw[offsetof(struct amap_sol_cqe, i_flags) / 32]
1338                                         & SOL_FLAGS_MASK) >> 24) | 0x80;
1339         hdr->response = (psol->dw[offsetof(struct amap_sol_cqe, i_resp) /
1340                                         32] & SOL_RESP_MASK);
1341         hdr->exp_cmdsn = cpu_to_be32(psol->
1342                         dw[offsetof(struct amap_sol_cqe, i_exp_cmd_sn) / 32]
1343                                         & SOL_EXP_CMD_SN_MASK);
1344         hdr->max_cmdsn = be32_to_cpu((psol->
1345                          dw[offsetof(struct amap_sol_cqe, i_exp_cmd_sn) / 32]
1346                                         & SOL_EXP_CMD_SN_MASK) +
1347                         ((psol->dw[offsetof(struct amap_sol_cqe, i_cmd_wnd)
1348                                         / 32] & SOL_CMD_WND_MASK) >> 24) - 1);
1349         hdr->dlength[0] = 0;
1350         hdr->dlength[1] = 0;
1351         hdr->dlength[2] = 0;
1352         hdr->hlength = 0;
1353         hdr->itt = io_task->libiscsi_itt;
1354         __iscsi_complete_pdu(conn, (struct iscsi_hdr *)hdr, NULL, 0);
1355 }
1356
1357 static void
1358 be_complete_tmf(struct beiscsi_conn *beiscsi_conn,
1359                 struct iscsi_task *task, struct sol_cqe *psol)
1360 {
1361         struct iscsi_tm_rsp *hdr;
1362         struct iscsi_conn *conn = beiscsi_conn->conn;
1363         struct beiscsi_io_task *io_task = task->dd_data;
1364
1365         hdr = (struct iscsi_tm_rsp *)task->hdr;
1366         hdr->opcode = ISCSI_OP_SCSI_TMFUNC_RSP;
1367         hdr->flags = ((psol->dw[offsetof(struct amap_sol_cqe, i_flags) / 32]
1368                                         & SOL_FLAGS_MASK) >> 24) | 0x80;
1369         hdr->response = (psol->dw[offsetof(struct amap_sol_cqe, i_resp) /
1370                                         32] & SOL_RESP_MASK);
1371         hdr->exp_cmdsn = cpu_to_be32(psol->dw[offsetof(struct amap_sol_cqe,
1372                                     i_exp_cmd_sn) / 32] & SOL_EXP_CMD_SN_MASK);
1373         hdr->max_cmdsn = be32_to_cpu((psol->dw[offsetof(struct amap_sol_cqe,
1374                         i_exp_cmd_sn) / 32] & SOL_EXP_CMD_SN_MASK) +
1375                         ((psol->dw[offsetof(struct amap_sol_cqe, i_cmd_wnd)
1376                         / 32] & SOL_CMD_WND_MASK) >> 24) - 1);
1377         hdr->itt = io_task->libiscsi_itt;
1378         __iscsi_complete_pdu(conn, (struct iscsi_hdr *)hdr, NULL, 0);
1379 }
1380
1381 static void
1382 hwi_complete_drvr_msgs(struct beiscsi_conn *beiscsi_conn,
1383                        struct beiscsi_hba *phba, struct sol_cqe *psol)
1384 {
1385         struct hwi_wrb_context *pwrb_context;
1386         struct wrb_handle *pwrb_handle = NULL;
1387         struct hwi_controller *phwi_ctrlr;
1388         struct iscsi_task *task;
1389         struct beiscsi_io_task *io_task;
1390         struct iscsi_conn *conn = beiscsi_conn->conn;
1391         struct iscsi_session *session = conn->session;
1392
1393         phwi_ctrlr = phba->phwi_ctrlr;
1394         pwrb_context = &phwi_ctrlr->wrb_context[((psol->
1395                                 dw[offsetof(struct amap_sol_cqe, cid) / 32] &
1396                                 SOL_CID_MASK) >> 6) -
1397                                 phba->fw_config.iscsi_cid_start];
1398         pwrb_handle = pwrb_context->pwrb_handle_basestd[((psol->
1399                                 dw[offsetof(struct amap_sol_cqe, wrb_index) /
1400                                 32] & SOL_WRB_INDEX_MASK) >> 16)];
1401         task = pwrb_handle->pio_handle;
1402
1403         io_task = task->dd_data;
1404         spin_lock_bh(&phba->mgmt_sgl_lock);
1405         free_mgmt_sgl_handle(phba, io_task->psgl_handle);
1406         spin_unlock_bh(&phba->mgmt_sgl_lock);
1407         spin_lock_bh(&session->lock);
1408         free_wrb_handle(phba, pwrb_context, pwrb_handle);
1409         spin_unlock_bh(&session->lock);
1410 }
1411
1412 static void
1413 be_complete_nopin_resp(struct beiscsi_conn *beiscsi_conn,
1414                        struct iscsi_task *task, struct sol_cqe *psol)
1415 {
1416         struct iscsi_nopin *hdr;
1417         struct iscsi_conn *conn = beiscsi_conn->conn;
1418         struct beiscsi_io_task *io_task = task->dd_data;
1419
1420         hdr = (struct iscsi_nopin *)task->hdr;
1421         hdr->flags = ((psol->dw[offsetof(struct amap_sol_cqe, i_flags) / 32]
1422                         & SOL_FLAGS_MASK) >> 24) | 0x80;
1423         hdr->exp_cmdsn = cpu_to_be32(psol->dw[offsetof(struct amap_sol_cqe,
1424                                      i_exp_cmd_sn) / 32] & SOL_EXP_CMD_SN_MASK);
1425         hdr->max_cmdsn = be32_to_cpu((psol->dw[offsetof(struct amap_sol_cqe,
1426                         i_exp_cmd_sn) / 32] & SOL_EXP_CMD_SN_MASK) +
1427                         ((psol->dw[offsetof(struct amap_sol_cqe, i_cmd_wnd)
1428                         / 32] & SOL_CMD_WND_MASK) >> 24) - 1);
1429         hdr->opcode = ISCSI_OP_NOOP_IN;
1430         hdr->itt = io_task->libiscsi_itt;
1431         __iscsi_complete_pdu(conn, (struct iscsi_hdr *)hdr, NULL, 0);
1432 }
1433
1434 static void hwi_complete_cmd(struct beiscsi_conn *beiscsi_conn,
1435                              struct beiscsi_hba *phba, struct sol_cqe *psol)
1436 {
1437         struct hwi_wrb_context *pwrb_context;
1438         struct wrb_handle *pwrb_handle;
1439         struct iscsi_wrb *pwrb = NULL;
1440         struct hwi_controller *phwi_ctrlr;
1441         struct iscsi_task *task;
1442         unsigned int type;
1443         struct iscsi_conn *conn = beiscsi_conn->conn;
1444         struct iscsi_session *session = conn->session;
1445
1446         phwi_ctrlr = phba->phwi_ctrlr;
1447         pwrb_context = &phwi_ctrlr->wrb_context[((psol->dw[offsetof
1448                                 (struct amap_sol_cqe, cid) / 32]
1449                                 & SOL_CID_MASK) >> 6) -
1450                                 phba->fw_config.iscsi_cid_start];
1451         pwrb_handle = pwrb_context->pwrb_handle_basestd[((psol->
1452                                 dw[offsetof(struct amap_sol_cqe, wrb_index) /
1453                                 32] & SOL_WRB_INDEX_MASK) >> 16)];
1454         task = pwrb_handle->pio_handle;
1455         pwrb = pwrb_handle->pwrb;
1456         type = (pwrb->dw[offsetof(struct amap_iscsi_wrb, type) / 32] &
1457                                  WRB_TYPE_MASK) >> 28;
1458
1459         spin_lock_bh(&session->lock);
1460         switch (type) {
1461         case HWH_TYPE_IO:
1462         case HWH_TYPE_IO_RD:
1463                 if ((task->hdr->opcode & ISCSI_OPCODE_MASK) ==
1464                      ISCSI_OP_NOOP_OUT)
1465                         be_complete_nopin_resp(beiscsi_conn, task, psol);
1466                 else
1467                         be_complete_io(beiscsi_conn, task, psol);
1468                 break;
1469
1470         case HWH_TYPE_LOGOUT:
1471                 if ((task->hdr->opcode & ISCSI_OPCODE_MASK) == ISCSI_OP_LOGOUT)
1472                         be_complete_logout(beiscsi_conn, task, psol);
1473                 else
1474                         be_complete_tmf(beiscsi_conn, task, psol);
1475
1476                 break;
1477
1478         case HWH_TYPE_LOGIN:
1479                 beiscsi_log(phba, KERN_ERR,
1480                             BEISCSI_LOG_CONFIG | BEISCSI_LOG_IO,
1481                             "BM_%d :\t\t No HWH_TYPE_LOGIN Expected in"
1482                             " hwi_complete_cmd- Solicited path\n");
1483                 break;
1484
1485         case HWH_TYPE_NOP:
1486                 be_complete_nopin_resp(beiscsi_conn, task, psol);
1487                 break;
1488
1489         default:
1490                 beiscsi_log(phba, KERN_WARNING,
1491                             BEISCSI_LOG_CONFIG | BEISCSI_LOG_IO,
1492                             "BM_%d : In hwi_complete_cmd, unknown type = %d"
1493                             "wrb_index 0x%x CID 0x%x\n", type,
1494                             ((psol->dw[offsetof(struct amap_iscsi_wrb,
1495                             type) / 32] & SOL_WRB_INDEX_MASK) >> 16),
1496                             ((psol->dw[offsetof(struct amap_sol_cqe,
1497                             cid) / 32] & SOL_CID_MASK) >> 6));
1498                 break;
1499         }
1500
1501         spin_unlock_bh(&session->lock);
1502 }
1503
1504 static struct list_head *hwi_get_async_busy_list(struct hwi_async_pdu_context
1505                                           *pasync_ctx, unsigned int is_header,
1506                                           unsigned int host_write_ptr)
1507 {
1508         if (is_header)
1509                 return &pasync_ctx->async_entry[host_write_ptr].
1510                     header_busy_list;
1511         else
1512                 return &pasync_ctx->async_entry[host_write_ptr].data_busy_list;
1513 }
1514
1515 static struct async_pdu_handle *
1516 hwi_get_async_handle(struct beiscsi_hba *phba,
1517                      struct beiscsi_conn *beiscsi_conn,
1518                      struct hwi_async_pdu_context *pasync_ctx,
1519                      struct i_t_dpdu_cqe *pdpdu_cqe, unsigned int *pcq_index)
1520 {
1521         struct be_bus_address phys_addr;
1522         struct list_head *pbusy_list;
1523         struct async_pdu_handle *pasync_handle = NULL;
1524         unsigned char is_header = 0;
1525
1526         phys_addr.u.a32.address_lo =
1527             pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe, db_addr_lo) / 32] -
1528             ((pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe, dpl) / 32]
1529                                                 & PDUCQE_DPL_MASK) >> 16);
1530         phys_addr.u.a32.address_hi =
1531             pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe, db_addr_hi) / 32];
1532
1533         phys_addr.u.a64.address =
1534                         *((unsigned long long *)(&phys_addr.u.a64.address));
1535
1536         switch (pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe, code) / 32]
1537                         & PDUCQE_CODE_MASK) {
1538         case UNSOL_HDR_NOTIFY:
1539                 is_header = 1;
1540
1541                 pbusy_list = hwi_get_async_busy_list(pasync_ctx, 1,
1542                         (pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe,
1543                         index) / 32] & PDUCQE_INDEX_MASK));
1544                 break;
1545         case UNSOL_DATA_NOTIFY:
1546                 pbusy_list = hwi_get_async_busy_list(pasync_ctx, 0, (pdpdu_cqe->
1547                                         dw[offsetof(struct amap_i_t_dpdu_cqe,
1548                                         index) / 32] & PDUCQE_INDEX_MASK));
1549                 break;
1550         default:
1551                 pbusy_list = NULL;
1552                 beiscsi_log(phba, KERN_WARNING,
1553                             BEISCSI_LOG_IO | BEISCSI_LOG_CONFIG,
1554                             "BM_%d : Unexpected code=%d\n",
1555                             pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe,
1556                             code) / 32] & PDUCQE_CODE_MASK);
1557                 return NULL;
1558         }
1559
1560         WARN_ON(list_empty(pbusy_list));
1561         list_for_each_entry(pasync_handle, pbusy_list, link) {
1562                 if (pasync_handle->pa.u.a64.address == phys_addr.u.a64.address)
1563                         break;
1564         }
1565
1566         WARN_ON(!pasync_handle);
1567
1568         pasync_handle->cri = (unsigned short)beiscsi_conn->beiscsi_conn_cid -
1569                                              phba->fw_config.iscsi_cid_start;
1570         pasync_handle->is_header = is_header;
1571         pasync_handle->buffer_len = ((pdpdu_cqe->
1572                         dw[offsetof(struct amap_i_t_dpdu_cqe, dpl) / 32]
1573                         & PDUCQE_DPL_MASK) >> 16);
1574
1575         *pcq_index = (pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe,
1576                         index) / 32] & PDUCQE_INDEX_MASK);
1577         return pasync_handle;
1578 }
1579
1580 static unsigned int
1581 hwi_update_async_writables(struct beiscsi_hba *phba,
1582                             struct hwi_async_pdu_context *pasync_ctx,
1583                             unsigned int is_header, unsigned int cq_index)
1584 {
1585         struct list_head *pbusy_list;
1586         struct async_pdu_handle *pasync_handle;
1587         unsigned int num_entries, writables = 0;
1588         unsigned int *pep_read_ptr, *pwritables;
1589
1590         num_entries = pasync_ctx->num_entries;
1591         if (is_header) {
1592                 pep_read_ptr = &pasync_ctx->async_header.ep_read_ptr;
1593                 pwritables = &pasync_ctx->async_header.writables;
1594         } else {
1595                 pep_read_ptr = &pasync_ctx->async_data.ep_read_ptr;
1596                 pwritables = &pasync_ctx->async_data.writables;
1597         }
1598
1599         while ((*pep_read_ptr) != cq_index) {
1600                 (*pep_read_ptr)++;
1601                 *pep_read_ptr = (*pep_read_ptr) % num_entries;
1602
1603                 pbusy_list = hwi_get_async_busy_list(pasync_ctx, is_header,
1604                                                      *pep_read_ptr);
1605                 if (writables == 0)
1606                         WARN_ON(list_empty(pbusy_list));
1607
1608                 if (!list_empty(pbusy_list)) {
1609                         pasync_handle = list_entry(pbusy_list->next,
1610                                                    struct async_pdu_handle,
1611                                                    link);
1612                         WARN_ON(!pasync_handle);
1613                         pasync_handle->consumed = 1;
1614                 }
1615
1616                 writables++;
1617         }
1618
1619         if (!writables) {
1620                 beiscsi_log(phba, KERN_ERR,
1621                             BEISCSI_LOG_CONFIG | BEISCSI_LOG_IO,
1622                             "BM_%d : Duplicate notification received - index 0x%x!!\n",
1623                             cq_index);
1624                 WARN_ON(1);
1625         }
1626
1627         *pwritables = *pwritables + writables;
1628         return 0;
1629 }
1630
1631 static void hwi_free_async_msg(struct beiscsi_hba *phba,
1632                                        unsigned int cri)
1633 {
1634         struct hwi_controller *phwi_ctrlr;
1635         struct hwi_async_pdu_context *pasync_ctx;
1636         struct async_pdu_handle *pasync_handle, *tmp_handle;
1637         struct list_head *plist;
1638
1639         phwi_ctrlr = phba->phwi_ctrlr;
1640         pasync_ctx = HWI_GET_ASYNC_PDU_CTX(phwi_ctrlr);
1641
1642         plist  = &pasync_ctx->async_entry[cri].wait_queue.list;
1643
1644         list_for_each_entry_safe(pasync_handle, tmp_handle, plist, link) {
1645                 list_del(&pasync_handle->link);
1646
1647                 if (pasync_handle->is_header) {
1648                         list_add_tail(&pasync_handle->link,
1649                                       &pasync_ctx->async_header.free_list);
1650                         pasync_ctx->async_header.free_entries++;
1651                 } else {
1652                         list_add_tail(&pasync_handle->link,
1653                                       &pasync_ctx->async_data.free_list);
1654                         pasync_ctx->async_data.free_entries++;
1655                 }
1656         }
1657
1658         INIT_LIST_HEAD(&pasync_ctx->async_entry[cri].wait_queue.list);
1659         pasync_ctx->async_entry[cri].wait_queue.hdr_received = 0;
1660         pasync_ctx->async_entry[cri].wait_queue.bytes_received = 0;
1661 }
1662
1663 static struct phys_addr *
1664 hwi_get_ring_address(struct hwi_async_pdu_context *pasync_ctx,
1665                      unsigned int is_header, unsigned int host_write_ptr)
1666 {
1667         struct phys_addr *pasync_sge = NULL;
1668
1669         if (is_header)
1670                 pasync_sge = pasync_ctx->async_header.ring_base;
1671         else
1672                 pasync_sge = pasync_ctx->async_data.ring_base;
1673
1674         return pasync_sge + host_write_ptr;
1675 }
1676
1677 static void hwi_post_async_buffers(struct beiscsi_hba *phba,
1678                                    unsigned int is_header)
1679 {
1680         struct hwi_controller *phwi_ctrlr;
1681         struct hwi_async_pdu_context *pasync_ctx;
1682         struct async_pdu_handle *pasync_handle;
1683         struct list_head *pfree_link, *pbusy_list;
1684         struct phys_addr *pasync_sge;
1685         unsigned int ring_id, num_entries;
1686         unsigned int host_write_num;
1687         unsigned int writables;
1688         unsigned int i = 0;
1689         u32 doorbell = 0;
1690
1691         phwi_ctrlr = phba->phwi_ctrlr;
1692         pasync_ctx = HWI_GET_ASYNC_PDU_CTX(phwi_ctrlr);
1693         num_entries = pasync_ctx->num_entries;
1694
1695         if (is_header) {
1696                 writables = min(pasync_ctx->async_header.writables,
1697                                 pasync_ctx->async_header.free_entries);
1698                 pfree_link = pasync_ctx->async_header.free_list.next;
1699                 host_write_num = pasync_ctx->async_header.host_write_ptr;
1700                 ring_id = phwi_ctrlr->default_pdu_hdr.id;
1701         } else {
1702                 writables = min(pasync_ctx->async_data.writables,
1703                                 pasync_ctx->async_data.free_entries);
1704                 pfree_link = pasync_ctx->async_data.free_list.next;
1705                 host_write_num = pasync_ctx->async_data.host_write_ptr;
1706                 ring_id = phwi_ctrlr->default_pdu_data.id;
1707         }
1708
1709         writables = (writables / 8) * 8;
1710         if (writables) {
1711                 for (i = 0; i < writables; i++) {
1712                         pbusy_list =
1713                             hwi_get_async_busy_list(pasync_ctx, is_header,
1714                                                     host_write_num);
1715                         pasync_handle =
1716                             list_entry(pfree_link, struct async_pdu_handle,
1717                                                                 link);
1718                         WARN_ON(!pasync_handle);
1719                         pasync_handle->consumed = 0;
1720
1721                         pfree_link = pfree_link->next;
1722
1723                         pasync_sge = hwi_get_ring_address(pasync_ctx,
1724                                                 is_header, host_write_num);
1725
1726                         pasync_sge->hi = pasync_handle->pa.u.a32.address_lo;
1727                         pasync_sge->lo = pasync_handle->pa.u.a32.address_hi;
1728
1729                         list_move(&pasync_handle->link, pbusy_list);
1730
1731                         host_write_num++;
1732                         host_write_num = host_write_num % num_entries;
1733                 }
1734
1735                 if (is_header) {
1736                         pasync_ctx->async_header.host_write_ptr =
1737                                                         host_write_num;
1738                         pasync_ctx->async_header.free_entries -= writables;
1739                         pasync_ctx->async_header.writables -= writables;
1740                         pasync_ctx->async_header.busy_entries += writables;
1741                 } else {
1742                         pasync_ctx->async_data.host_write_ptr = host_write_num;
1743                         pasync_ctx->async_data.free_entries -= writables;
1744                         pasync_ctx->async_data.writables -= writables;
1745                         pasync_ctx->async_data.busy_entries += writables;
1746                 }
1747
1748                 doorbell |= ring_id & DB_DEF_PDU_RING_ID_MASK;
1749                 doorbell |= 1 << DB_DEF_PDU_REARM_SHIFT;
1750                 doorbell |= 0 << DB_DEF_PDU_EVENT_SHIFT;
1751                 doorbell |= (writables & DB_DEF_PDU_CQPROC_MASK)
1752                                         << DB_DEF_PDU_CQPROC_SHIFT;
1753
1754                 iowrite32(doorbell, phba->db_va + DB_RXULP0_OFFSET);
1755         }
1756 }
1757
1758 static void hwi_flush_default_pdu_buffer(struct beiscsi_hba *phba,
1759                                          struct beiscsi_conn *beiscsi_conn,
1760                                          struct i_t_dpdu_cqe *pdpdu_cqe)
1761 {
1762         struct hwi_controller *phwi_ctrlr;
1763         struct hwi_async_pdu_context *pasync_ctx;
1764         struct async_pdu_handle *pasync_handle = NULL;
1765         unsigned int cq_index = -1;
1766
1767         phwi_ctrlr = phba->phwi_ctrlr;
1768         pasync_ctx = HWI_GET_ASYNC_PDU_CTX(phwi_ctrlr);
1769
1770         pasync_handle = hwi_get_async_handle(phba, beiscsi_conn, pasync_ctx,
1771                                              pdpdu_cqe, &cq_index);
1772         BUG_ON(pasync_handle->is_header != 0);
1773         if (pasync_handle->consumed == 0)
1774                 hwi_update_async_writables(phba, pasync_ctx,
1775                                            pasync_handle->is_header, cq_index);
1776
1777         hwi_free_async_msg(phba, pasync_handle->cri);
1778         hwi_post_async_buffers(phba, pasync_handle->is_header);
1779 }
1780
1781 static unsigned int
1782 hwi_fwd_async_msg(struct beiscsi_conn *beiscsi_conn,
1783                   struct beiscsi_hba *phba,
1784                   struct hwi_async_pdu_context *pasync_ctx, unsigned short cri)
1785 {
1786         struct list_head *plist;
1787         struct async_pdu_handle *pasync_handle;
1788         void *phdr = NULL;
1789         unsigned int hdr_len = 0, buf_len = 0;
1790         unsigned int status, index = 0, offset = 0;
1791         void *pfirst_buffer = NULL;
1792         unsigned int num_buf = 0;
1793
1794         plist = &pasync_ctx->async_entry[cri].wait_queue.list;
1795
1796         list_for_each_entry(pasync_handle, plist, link) {
1797                 if (index == 0) {
1798                         phdr = pasync_handle->pbuffer;
1799                         hdr_len = pasync_handle->buffer_len;
1800                 } else {
1801                         buf_len = pasync_handle->buffer_len;
1802                         if (!num_buf) {
1803                                 pfirst_buffer = pasync_handle->pbuffer;
1804                                 num_buf++;
1805                         }
1806                         memcpy(pfirst_buffer + offset,
1807                                pasync_handle->pbuffer, buf_len);
1808                         offset += buf_len;
1809                 }
1810                 index++;
1811         }
1812
1813         status = beiscsi_process_async_pdu(beiscsi_conn, phba,
1814                                            (beiscsi_conn->beiscsi_conn_cid -
1815                                             phba->fw_config.iscsi_cid_start),
1816                                             phdr, hdr_len, pfirst_buffer,
1817                                             offset);
1818
1819         hwi_free_async_msg(phba, cri);
1820         return 0;
1821 }
1822
1823 static unsigned int
1824 hwi_gather_async_pdu(struct beiscsi_conn *beiscsi_conn,
1825                      struct beiscsi_hba *phba,
1826                      struct async_pdu_handle *pasync_handle)
1827 {
1828         struct hwi_async_pdu_context *pasync_ctx;
1829         struct hwi_controller *phwi_ctrlr;
1830         unsigned int bytes_needed = 0, status = 0;
1831         unsigned short cri = pasync_handle->cri;
1832         struct pdu_base *ppdu;
1833
1834         phwi_ctrlr = phba->phwi_ctrlr;
1835         pasync_ctx = HWI_GET_ASYNC_PDU_CTX(phwi_ctrlr);
1836
1837         list_del(&pasync_handle->link);
1838         if (pasync_handle->is_header) {
1839                 pasync_ctx->async_header.busy_entries--;
1840                 if (pasync_ctx->async_entry[cri].wait_queue.hdr_received) {
1841                         hwi_free_async_msg(phba, cri);
1842                         BUG();
1843                 }
1844
1845                 pasync_ctx->async_entry[cri].wait_queue.bytes_received = 0;
1846                 pasync_ctx->async_entry[cri].wait_queue.hdr_received = 1;
1847                 pasync_ctx->async_entry[cri].wait_queue.hdr_len =
1848                                 (unsigned short)pasync_handle->buffer_len;
1849                 list_add_tail(&pasync_handle->link,
1850                               &pasync_ctx->async_entry[cri].wait_queue.list);
1851
1852                 ppdu = pasync_handle->pbuffer;
1853                 bytes_needed = ((((ppdu->dw[offsetof(struct amap_pdu_base,
1854                         data_len_hi) / 32] & PDUBASE_DATALENHI_MASK) << 8) &
1855                         0xFFFF0000) | ((be16_to_cpu((ppdu->
1856                         dw[offsetof(struct amap_pdu_base, data_len_lo) / 32]
1857                         & PDUBASE_DATALENLO_MASK) >> 16)) & 0x0000FFFF));
1858
1859                 if (status == 0) {
1860                         pasync_ctx->async_entry[cri].wait_queue.bytes_needed =
1861                             bytes_needed;
1862
1863                         if (bytes_needed == 0)
1864                                 status = hwi_fwd_async_msg(beiscsi_conn, phba,
1865                                                            pasync_ctx, cri);
1866                 }
1867         } else {
1868                 pasync_ctx->async_data.busy_entries--;
1869                 if (pasync_ctx->async_entry[cri].wait_queue.hdr_received) {
1870                         list_add_tail(&pasync_handle->link,
1871                                       &pasync_ctx->async_entry[cri].wait_queue.
1872                                       list);
1873                         pasync_ctx->async_entry[cri].wait_queue.
1874                                 bytes_received +=
1875                                 (unsigned short)pasync_handle->buffer_len;
1876
1877                         if (pasync_ctx->async_entry[cri].wait_queue.
1878                             bytes_received >=
1879                             pasync_ctx->async_entry[cri].wait_queue.
1880                             bytes_needed)
1881                                 status = hwi_fwd_async_msg(beiscsi_conn, phba,
1882                                                            pasync_ctx, cri);
1883                 }
1884         }
1885         return status;
1886 }
1887
1888 static void hwi_process_default_pdu_ring(struct beiscsi_conn *beiscsi_conn,
1889                                          struct beiscsi_hba *phba,
1890                                          struct i_t_dpdu_cqe *pdpdu_cqe)
1891 {
1892         struct hwi_controller *phwi_ctrlr;
1893         struct hwi_async_pdu_context *pasync_ctx;
1894         struct async_pdu_handle *pasync_handle = NULL;
1895         unsigned int cq_index = -1;
1896
1897         phwi_ctrlr = phba->phwi_ctrlr;
1898         pasync_ctx = HWI_GET_ASYNC_PDU_CTX(phwi_ctrlr);
1899         pasync_handle = hwi_get_async_handle(phba, beiscsi_conn, pasync_ctx,
1900                                              pdpdu_cqe, &cq_index);
1901
1902         if (pasync_handle->consumed == 0)
1903                 hwi_update_async_writables(phba, pasync_ctx,
1904                                            pasync_handle->is_header, cq_index);
1905
1906         hwi_gather_async_pdu(beiscsi_conn, phba, pasync_handle);
1907         hwi_post_async_buffers(phba, pasync_handle->is_header);
1908 }
1909
1910 static void  beiscsi_process_mcc_isr(struct beiscsi_hba *phba)
1911 {
1912         struct be_queue_info *mcc_cq;
1913         struct  be_mcc_compl *mcc_compl;
1914         unsigned int num_processed = 0;
1915
1916         mcc_cq = &phba->ctrl.mcc_obj.cq;
1917         mcc_compl = queue_tail_node(mcc_cq);
1918         mcc_compl->flags = le32_to_cpu(mcc_compl->flags);
1919         while (mcc_compl->flags & CQE_FLAGS_VALID_MASK) {
1920
1921                 if (num_processed >= 32) {
1922                         hwi_ring_cq_db(phba, mcc_cq->id,
1923                                         num_processed, 0, 0);
1924                         num_processed = 0;
1925                 }
1926                 if (mcc_compl->flags & CQE_FLAGS_ASYNC_MASK) {
1927                         /* Interpret flags as an async trailer */
1928                         if (is_link_state_evt(mcc_compl->flags))
1929                                 /* Interpret compl as a async link evt */
1930                                 beiscsi_async_link_state_process(phba,
1931                                 (struct be_async_event_link_state *) mcc_compl);
1932                         else
1933                                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_MBOX,
1934                                             "BM_%d :  Unsupported Async Event, flags"
1935                                             " = 0x%08x\n",
1936                                             mcc_compl->flags);
1937                 } else if (mcc_compl->flags & CQE_FLAGS_COMPLETED_MASK) {
1938                         be_mcc_compl_process_isr(&phba->ctrl, mcc_compl);
1939                         atomic_dec(&phba->ctrl.mcc_obj.q.used);
1940                 }
1941
1942                 mcc_compl->flags = 0;
1943                 queue_tail_inc(mcc_cq);
1944                 mcc_compl = queue_tail_node(mcc_cq);
1945                 mcc_compl->flags = le32_to_cpu(mcc_compl->flags);
1946                 num_processed++;
1947         }
1948
1949         if (num_processed > 0)
1950                 hwi_ring_cq_db(phba, mcc_cq->id, num_processed, 1, 0);
1951
1952 }
1953
1954 /**
1955  * beiscsi_process_cq()- Process the Completion Queue
1956  * @pbe_eq: Event Q on which the Completion has come
1957  *
1958  * return
1959  *     Number of Completion Entries processed.
1960  **/
1961 static unsigned int beiscsi_process_cq(struct be_eq_obj *pbe_eq)
1962 {
1963         struct be_queue_info *cq;
1964         struct sol_cqe *sol;
1965         struct dmsg_cqe *dmsg;
1966         unsigned int num_processed = 0;
1967         unsigned int tot_nump = 0;
1968         unsigned short code = 0, cid = 0;
1969         struct beiscsi_conn *beiscsi_conn;
1970         struct beiscsi_endpoint *beiscsi_ep;
1971         struct iscsi_endpoint *ep;
1972         struct beiscsi_hba *phba;
1973
1974         cq = pbe_eq->cq;
1975         sol = queue_tail_node(cq);
1976         phba = pbe_eq->phba;
1977
1978         while (sol->dw[offsetof(struct amap_sol_cqe, valid) / 32] &
1979                CQE_VALID_MASK) {
1980                 be_dws_le_to_cpu(sol, sizeof(struct sol_cqe));
1981
1982                 cid = ((sol->dw[offsetof(struct amap_sol_cqe, cid)/32] &
1983                       CQE_CID_MASK) >> 6);
1984                 code = (sol->dw[offsetof(struct amap_sol_cqe, code)/32] &
1985                        CQE_CODE_MASK);
1986                 ep = phba->ep_array[cid - phba->fw_config.iscsi_cid_start];
1987
1988                 beiscsi_ep = ep->dd_data;
1989                 beiscsi_conn = beiscsi_ep->conn;
1990
1991                 if (num_processed >= 32) {
1992                         hwi_ring_cq_db(phba, cq->id,
1993                                         num_processed, 0, 0);
1994                         tot_nump += num_processed;
1995                         num_processed = 0;
1996                 }
1997
1998                 switch (code) {
1999                 case SOL_CMD_COMPLETE:
2000                         hwi_complete_cmd(beiscsi_conn, phba, sol);
2001                         break;
2002                 case DRIVERMSG_NOTIFY:
2003                         beiscsi_log(phba, KERN_INFO,
2004                                     BEISCSI_LOG_IO | BEISCSI_LOG_CONFIG,
2005                                     "BM_%d : Received %s[%d] on CID : %d\n",
2006                                     cqe_desc[code], code, cid);
2007
2008                         dmsg = (struct dmsg_cqe *)sol;
2009                         hwi_complete_drvr_msgs(beiscsi_conn, phba, sol);
2010                         break;
2011                 case UNSOL_HDR_NOTIFY:
2012                         beiscsi_log(phba, KERN_INFO,
2013                                     BEISCSI_LOG_IO | BEISCSI_LOG_CONFIG,
2014                                     "BM_%d : Received %s[%d] on CID : %d\n",
2015                                     cqe_desc[code], code, cid);
2016
2017                         hwi_process_default_pdu_ring(beiscsi_conn, phba,
2018                                              (struct i_t_dpdu_cqe *)sol);
2019                         break;
2020                 case UNSOL_DATA_NOTIFY:
2021                         beiscsi_log(phba, KERN_INFO,
2022                                     BEISCSI_LOG_CONFIG | BEISCSI_LOG_IO,
2023                                     "BM_%d : Received %s[%d] on CID : %d\n",
2024                                     cqe_desc[code], code, cid);
2025
2026                         hwi_process_default_pdu_ring(beiscsi_conn, phba,
2027                                              (struct i_t_dpdu_cqe *)sol);
2028                         break;
2029                 case CXN_INVALIDATE_INDEX_NOTIFY:
2030                 case CMD_INVALIDATED_NOTIFY:
2031                 case CXN_INVALIDATE_NOTIFY:
2032                         beiscsi_log(phba, KERN_ERR,
2033                                     BEISCSI_LOG_IO | BEISCSI_LOG_CONFIG,
2034                                     "BM_%d : Ignoring %s[%d] on CID : %d\n",
2035                                     cqe_desc[code], code, cid);
2036                         break;
2037                 case SOL_CMD_KILLED_DATA_DIGEST_ERR:
2038                 case CMD_KILLED_INVALID_STATSN_RCVD:
2039                 case CMD_KILLED_INVALID_R2T_RCVD:
2040                 case CMD_CXN_KILLED_LUN_INVALID:
2041                 case CMD_CXN_KILLED_ICD_INVALID:
2042                 case CMD_CXN_KILLED_ITT_INVALID:
2043                 case CMD_CXN_KILLED_SEQ_OUTOFORDER:
2044                 case CMD_CXN_KILLED_INVALID_DATASN_RCVD:
2045                         beiscsi_log(phba, KERN_ERR,
2046                                     BEISCSI_LOG_CONFIG | BEISCSI_LOG_IO,
2047                                     "BM_%d : Cmd Notification %s[%d] on CID : %d\n",
2048                                     cqe_desc[code], code,  cid);
2049                         break;
2050                 case UNSOL_DATA_DIGEST_ERROR_NOTIFY:
2051                         beiscsi_log(phba, KERN_ERR,
2052                                     BEISCSI_LOG_IO | BEISCSI_LOG_CONFIG,
2053                                     "BM_%d :  Dropping %s[%d] on DPDU ring on CID : %d\n",
2054                                     cqe_desc[code], code, cid);
2055                         hwi_flush_default_pdu_buffer(phba, beiscsi_conn,
2056                                              (struct i_t_dpdu_cqe *) sol);
2057                         break;
2058                 case CXN_KILLED_PDU_SIZE_EXCEEDS_DSL:
2059                 case CXN_KILLED_BURST_LEN_MISMATCH:
2060                 case CXN_KILLED_AHS_RCVD:
2061                 case CXN_KILLED_HDR_DIGEST_ERR:
2062                 case CXN_KILLED_UNKNOWN_HDR:
2063                 case CXN_KILLED_STALE_ITT_TTT_RCVD:
2064                 case CXN_KILLED_INVALID_ITT_TTT_RCVD:
2065                 case CXN_KILLED_TIMED_OUT:
2066                 case CXN_KILLED_FIN_RCVD:
2067                 case CXN_KILLED_RST_SENT:
2068                 case CXN_KILLED_RST_RCVD:
2069                 case CXN_KILLED_BAD_UNSOL_PDU_RCVD:
2070                 case CXN_KILLED_BAD_WRB_INDEX_ERROR:
2071                 case CXN_KILLED_OVER_RUN_RESIDUAL:
2072                 case CXN_KILLED_UNDER_RUN_RESIDUAL:
2073                 case CXN_KILLED_CMND_DATA_NOT_ON_SAME_CONN:
2074                         beiscsi_log(phba, KERN_ERR,
2075                                     BEISCSI_LOG_IO | BEISCSI_LOG_CONFIG,
2076                                     "BM_%d : Event %s[%d] received on CID : %d\n",
2077                                     cqe_desc[code], code, cid);
2078                         if (beiscsi_conn)
2079                                 iscsi_conn_failure(beiscsi_conn->conn,
2080                                                    ISCSI_ERR_CONN_FAILED);
2081                         break;
2082                 default:
2083                         beiscsi_log(phba, KERN_ERR,
2084                                     BEISCSI_LOG_IO | BEISCSI_LOG_CONFIG,
2085                                     "BM_%d : Invalid CQE Event Received Code : %d"
2086                                     "CID 0x%x...\n",
2087                                     code, cid);
2088                         break;
2089                 }
2090
2091                 AMAP_SET_BITS(struct amap_sol_cqe, valid, sol, 0);
2092                 queue_tail_inc(cq);
2093                 sol = queue_tail_node(cq);
2094                 num_processed++;
2095         }
2096
2097         if (num_processed > 0) {
2098                 tot_nump += num_processed;
2099                 hwi_ring_cq_db(phba, cq->id, num_processed, 1, 0);
2100         }
2101         return tot_nump;
2102 }
2103
2104 void beiscsi_process_all_cqs(struct work_struct *work)
2105 {
2106         unsigned long flags;
2107         struct hwi_controller *phwi_ctrlr;
2108         struct hwi_context_memory *phwi_context;
2109         struct beiscsi_hba *phba;
2110         struct be_eq_obj *pbe_eq =
2111             container_of(work, struct be_eq_obj, work_cqs);
2112
2113         phba = pbe_eq->phba;
2114         phwi_ctrlr = phba->phwi_ctrlr;
2115         phwi_context = phwi_ctrlr->phwi_ctxt;
2116
2117         if (pbe_eq->todo_mcc_cq) {
2118                 spin_lock_irqsave(&phba->isr_lock, flags);
2119                 pbe_eq->todo_mcc_cq = false;
2120                 spin_unlock_irqrestore(&phba->isr_lock, flags);
2121                 beiscsi_process_mcc_isr(phba);
2122         }
2123
2124         if (pbe_eq->todo_cq) {
2125                 spin_lock_irqsave(&phba->isr_lock, flags);
2126                 pbe_eq->todo_cq = false;
2127                 spin_unlock_irqrestore(&phba->isr_lock, flags);
2128                 beiscsi_process_cq(pbe_eq);
2129         }
2130
2131         /* rearm EQ for further interrupts */
2132         hwi_ring_eq_db(phba, pbe_eq->q.id, 0, 0, 1, 1);
2133 }
2134
2135 static int be_iopoll(struct blk_iopoll *iop, int budget)
2136 {
2137         static unsigned int ret;
2138         struct beiscsi_hba *phba;
2139         struct be_eq_obj *pbe_eq;
2140
2141         pbe_eq = container_of(iop, struct be_eq_obj, iopoll);
2142         ret = beiscsi_process_cq(pbe_eq);
2143         if (ret < budget) {
2144                 phba = pbe_eq->phba;
2145                 blk_iopoll_complete(iop);
2146                 beiscsi_log(phba, KERN_INFO,
2147                             BEISCSI_LOG_CONFIG | BEISCSI_LOG_IO,
2148                             "BM_%d : rearm pbe_eq->q.id =%d\n",
2149                             pbe_eq->q.id);
2150                 hwi_ring_eq_db(phba, pbe_eq->q.id, 0, 0, 1, 1);
2151         }
2152         return ret;
2153 }
2154
2155 static void
2156 hwi_write_sgl(struct iscsi_wrb *pwrb, struct scatterlist *sg,
2157               unsigned int num_sg, struct beiscsi_io_task *io_task)
2158 {
2159         struct iscsi_sge *psgl;
2160         unsigned int sg_len, index;
2161         unsigned int sge_len = 0;
2162         unsigned long long addr;
2163         struct scatterlist *l_sg;
2164         unsigned int offset;
2165
2166         AMAP_SET_BITS(struct amap_iscsi_wrb, iscsi_bhs_addr_lo, pwrb,
2167                                       io_task->bhs_pa.u.a32.address_lo);
2168         AMAP_SET_BITS(struct amap_iscsi_wrb, iscsi_bhs_addr_hi, pwrb,
2169                                       io_task->bhs_pa.u.a32.address_hi);
2170
2171         l_sg = sg;
2172         for (index = 0; (index < num_sg) && (index < 2); index++,
2173                                                          sg = sg_next(sg)) {
2174                 if (index == 0) {
2175                         sg_len = sg_dma_len(sg);
2176                         addr = (u64) sg_dma_address(sg);
2177                         AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_addr_lo, pwrb,
2178                                                 ((u32)(addr & 0xFFFFFFFF)));
2179                         AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_addr_hi, pwrb,
2180                                                         ((u32)(addr >> 32)));
2181                         AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_len, pwrb,
2182                                                         sg_len);
2183                         sge_len = sg_len;
2184                 } else {
2185                         AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_r2t_offset,
2186                                                         pwrb, sge_len);
2187                         sg_len = sg_dma_len(sg);
2188                         addr = (u64) sg_dma_address(sg);
2189                         AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_addr_lo, pwrb,
2190                                                 ((u32)(addr & 0xFFFFFFFF)));
2191                         AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_addr_hi, pwrb,
2192                                                         ((u32)(addr >> 32)));
2193                         AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_len, pwrb,
2194                                                         sg_len);
2195                 }
2196         }
2197         psgl = (struct iscsi_sge *)io_task->psgl_handle->pfrag;
2198         memset(psgl, 0, sizeof(*psgl) * BE2_SGE);
2199
2200         AMAP_SET_BITS(struct amap_iscsi_sge, len, psgl, io_task->bhs_len - 2);
2201
2202         AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, psgl,
2203                         io_task->bhs_pa.u.a32.address_hi);
2204         AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, psgl,
2205                         io_task->bhs_pa.u.a32.address_lo);
2206
2207         if (num_sg == 1) {
2208                 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_last, pwrb,
2209                                                                 1);
2210                 AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_last, pwrb,
2211                                                                 0);
2212         } else if (num_sg == 2) {
2213                 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_last, pwrb,
2214                                                                 0);
2215                 AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_last, pwrb,
2216                                                                 1);
2217         } else {
2218                 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_last, pwrb,
2219                                                                 0);
2220                 AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_last, pwrb,
2221                                                                 0);
2222         }
2223         sg = l_sg;
2224         psgl++;
2225         psgl++;
2226         offset = 0;
2227         for (index = 0; index < num_sg; index++, sg = sg_next(sg), psgl++) {
2228                 sg_len = sg_dma_len(sg);
2229                 addr = (u64) sg_dma_address(sg);
2230                 AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, psgl,
2231                                                 (addr & 0xFFFFFFFF));
2232                 AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, psgl,
2233                                                 (addr >> 32));
2234                 AMAP_SET_BITS(struct amap_iscsi_sge, len, psgl, sg_len);
2235                 AMAP_SET_BITS(struct amap_iscsi_sge, sge_offset, psgl, offset);
2236                 AMAP_SET_BITS(struct amap_iscsi_sge, last_sge, psgl, 0);
2237                 offset += sg_len;
2238         }
2239         psgl--;
2240         AMAP_SET_BITS(struct amap_iscsi_sge, last_sge, psgl, 1);
2241 }
2242
2243 /**
2244  * hwi_write_buffer()- Populate the WRB with task info
2245  * @pwrb: ptr to the WRB entry
2246  * @task: iscsi task which is to be executed
2247  **/
2248 static void hwi_write_buffer(struct iscsi_wrb *pwrb, struct iscsi_task *task)
2249 {
2250         struct iscsi_sge *psgl;
2251         struct beiscsi_io_task *io_task = task->dd_data;
2252         struct beiscsi_conn *beiscsi_conn = io_task->conn;
2253         struct beiscsi_hba *phba = beiscsi_conn->phba;
2254
2255         io_task->bhs_len = sizeof(struct be_nonio_bhs) - 2;
2256         AMAP_SET_BITS(struct amap_iscsi_wrb, iscsi_bhs_addr_lo, pwrb,
2257                                 io_task->bhs_pa.u.a32.address_lo);
2258         AMAP_SET_BITS(struct amap_iscsi_wrb, iscsi_bhs_addr_hi, pwrb,
2259                                 io_task->bhs_pa.u.a32.address_hi);
2260
2261         if (task->data) {
2262                 if (task->data_count) {
2263                         AMAP_SET_BITS(struct amap_iscsi_wrb, dsp, pwrb, 1);
2264                         io_task->mtask_addr = pci_map_single(phba->pcidev,
2265                                                              task->data,
2266                                                              task->data_count,
2267                                                              PCI_DMA_TODEVICE);
2268
2269                         io_task->mtask_data_count = task->data_count;
2270                 } else {
2271                         AMAP_SET_BITS(struct amap_iscsi_wrb, dsp, pwrb, 0);
2272                         io_task->mtask_addr = 0;
2273                 }
2274                 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_addr_lo, pwrb,
2275                               lower_32_bits(io_task->mtask_addr));
2276                 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_addr_hi, pwrb,
2277                               upper_32_bits(io_task->mtask_addr));
2278                 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_len, pwrb,
2279                                                 task->data_count);
2280
2281                 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_last, pwrb, 1);
2282         } else {
2283                 AMAP_SET_BITS(struct amap_iscsi_wrb, dsp, pwrb, 0);
2284                 io_task->mtask_addr = 0;
2285         }
2286
2287         psgl = (struct iscsi_sge *)io_task->psgl_handle->pfrag;
2288
2289         AMAP_SET_BITS(struct amap_iscsi_sge, len, psgl, io_task->bhs_len);
2290
2291         AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, psgl,
2292                       io_task->bhs_pa.u.a32.address_hi);
2293         AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, psgl,
2294                       io_task->bhs_pa.u.a32.address_lo);
2295         if (task->data) {
2296                 psgl++;
2297                 AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, psgl, 0);
2298                 AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, psgl, 0);
2299                 AMAP_SET_BITS(struct amap_iscsi_sge, len, psgl, 0);
2300                 AMAP_SET_BITS(struct amap_iscsi_sge, sge_offset, psgl, 0);
2301                 AMAP_SET_BITS(struct amap_iscsi_sge, rsvd0, psgl, 0);
2302                 AMAP_SET_BITS(struct amap_iscsi_sge, last_sge, psgl, 0);
2303
2304                 psgl++;
2305                 if (task->data) {
2306                         AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, psgl,
2307                                       lower_32_bits(io_task->mtask_addr));
2308                         AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, psgl,
2309                                       upper_32_bits(io_task->mtask_addr));
2310                 }
2311                 AMAP_SET_BITS(struct amap_iscsi_sge, len, psgl, 0x106);
2312         }
2313         AMAP_SET_BITS(struct amap_iscsi_sge, last_sge, psgl, 1);
2314 }
2315
2316 static void beiscsi_find_mem_req(struct beiscsi_hba *phba)
2317 {
2318         unsigned int num_cq_pages, num_async_pdu_buf_pages;
2319         unsigned int num_async_pdu_data_pages, wrb_sz_per_cxn;
2320         unsigned int num_async_pdu_buf_sgl_pages, num_async_pdu_data_sgl_pages;
2321
2322         num_cq_pages = PAGES_REQUIRED(phba->params.num_cq_entries * \
2323                                       sizeof(struct sol_cqe));
2324         num_async_pdu_buf_pages =
2325                         PAGES_REQUIRED(phba->params.asyncpdus_per_ctrl * \
2326                                        phba->params.defpdu_hdr_sz);
2327         num_async_pdu_buf_sgl_pages =
2328                         PAGES_REQUIRED(phba->params.asyncpdus_per_ctrl * \
2329                                        sizeof(struct phys_addr));
2330         num_async_pdu_data_pages =
2331                         PAGES_REQUIRED(phba->params.asyncpdus_per_ctrl * \
2332                                        phba->params.defpdu_data_sz);
2333         num_async_pdu_data_sgl_pages =
2334                         PAGES_REQUIRED(phba->params.asyncpdus_per_ctrl * \
2335                                        sizeof(struct phys_addr));
2336
2337         phba->params.hwi_ws_sz = sizeof(struct hwi_controller);
2338
2339         phba->mem_req[ISCSI_MEM_GLOBAL_HEADER] = 2 *
2340                                                  BE_ISCSI_PDU_HEADER_SIZE;
2341         phba->mem_req[HWI_MEM_ADDN_CONTEXT] =
2342                                             sizeof(struct hwi_context_memory);
2343
2344
2345         phba->mem_req[HWI_MEM_WRB] = sizeof(struct iscsi_wrb)
2346             * (phba->params.wrbs_per_cxn)
2347             * phba->params.cxns_per_ctrl;
2348         wrb_sz_per_cxn =  sizeof(struct wrb_handle) *
2349                                  (phba->params.wrbs_per_cxn);
2350         phba->mem_req[HWI_MEM_WRBH] = roundup_pow_of_two((wrb_sz_per_cxn) *
2351                                 phba->params.cxns_per_ctrl);
2352
2353         phba->mem_req[HWI_MEM_SGLH] = sizeof(struct sgl_handle) *
2354                 phba->params.icds_per_ctrl;
2355         phba->mem_req[HWI_MEM_SGE] = sizeof(struct iscsi_sge) *
2356                 phba->params.num_sge_per_io * phba->params.icds_per_ctrl;
2357
2358         phba->mem_req[HWI_MEM_ASYNC_HEADER_BUF] =
2359                 num_async_pdu_buf_pages * PAGE_SIZE;
2360         phba->mem_req[HWI_MEM_ASYNC_DATA_BUF] =
2361                 num_async_pdu_data_pages * PAGE_SIZE;
2362         phba->mem_req[HWI_MEM_ASYNC_HEADER_RING] =
2363                 num_async_pdu_buf_sgl_pages * PAGE_SIZE;
2364         phba->mem_req[HWI_MEM_ASYNC_DATA_RING] =
2365                 num_async_pdu_data_sgl_pages * PAGE_SIZE;
2366         phba->mem_req[HWI_MEM_ASYNC_HEADER_HANDLE] =
2367                 phba->params.asyncpdus_per_ctrl *
2368                 sizeof(struct async_pdu_handle);
2369         phba->mem_req[HWI_MEM_ASYNC_DATA_HANDLE] =
2370                 phba->params.asyncpdus_per_ctrl *
2371                 sizeof(struct async_pdu_handle);
2372         phba->mem_req[HWI_MEM_ASYNC_PDU_CONTEXT] =
2373                 sizeof(struct hwi_async_pdu_context) +
2374                 (phba->params.cxns_per_ctrl * sizeof(struct hwi_async_entry));
2375 }
2376
2377 static int beiscsi_alloc_mem(struct beiscsi_hba *phba)
2378 {
2379         struct be_mem_descriptor *mem_descr;
2380         dma_addr_t bus_add;
2381         struct mem_array *mem_arr, *mem_arr_orig;
2382         unsigned int i, j, alloc_size, curr_alloc_size;
2383
2384         phba->phwi_ctrlr = kzalloc(phba->params.hwi_ws_sz, GFP_KERNEL);
2385         if (!phba->phwi_ctrlr)
2386                 return -ENOMEM;
2387
2388         phba->init_mem = kcalloc(SE_MEM_MAX, sizeof(*mem_descr),
2389                                  GFP_KERNEL);
2390         if (!phba->init_mem) {
2391                 kfree(phba->phwi_ctrlr);
2392                 return -ENOMEM;
2393         }
2394
2395         mem_arr_orig = kmalloc(sizeof(*mem_arr_orig) * BEISCSI_MAX_FRAGS_INIT,
2396                                GFP_KERNEL);
2397         if (!mem_arr_orig) {
2398                 kfree(phba->init_mem);
2399                 kfree(phba->phwi_ctrlr);
2400                 return -ENOMEM;
2401         }
2402
2403         mem_descr = phba->init_mem;
2404         for (i = 0; i < SE_MEM_MAX; i++) {
2405                 j = 0;
2406                 mem_arr = mem_arr_orig;
2407                 alloc_size = phba->mem_req[i];
2408                 memset(mem_arr, 0, sizeof(struct mem_array) *
2409                        BEISCSI_MAX_FRAGS_INIT);
2410                 curr_alloc_size = min(be_max_phys_size * 1024, alloc_size);
2411                 do {
2412                         mem_arr->virtual_address = pci_alloc_consistent(
2413                                                         phba->pcidev,
2414                                                         curr_alloc_size,
2415                                                         &bus_add);
2416                         if (!mem_arr->virtual_address) {
2417                                 if (curr_alloc_size <= BE_MIN_MEM_SIZE)
2418                                         goto free_mem;
2419                                 if (curr_alloc_size -
2420                                         rounddown_pow_of_two(curr_alloc_size))
2421                                         curr_alloc_size = rounddown_pow_of_two
2422                                                              (curr_alloc_size);
2423                                 else
2424                                         curr_alloc_size = curr_alloc_size / 2;
2425                         } else {
2426                                 mem_arr->bus_address.u.
2427                                     a64.address = (__u64) bus_add;
2428                                 mem_arr->size = curr_alloc_size;
2429                                 alloc_size -= curr_alloc_size;
2430                                 curr_alloc_size = min(be_max_phys_size *
2431                                                       1024, alloc_size);
2432                                 j++;
2433                                 mem_arr++;
2434                         }
2435                 } while (alloc_size);
2436                 mem_descr->num_elements = j;
2437                 mem_descr->size_in_bytes = phba->mem_req[i];
2438                 mem_descr->mem_array = kmalloc(sizeof(*mem_arr) * j,
2439                                                GFP_KERNEL);
2440                 if (!mem_descr->mem_array)
2441                         goto free_mem;
2442
2443                 memcpy(mem_descr->mem_array, mem_arr_orig,
2444                        sizeof(struct mem_array) * j);
2445                 mem_descr++;
2446         }
2447         kfree(mem_arr_orig);
2448         return 0;
2449 free_mem:
2450         mem_descr->num_elements = j;
2451         while ((i) || (j)) {
2452                 for (j = mem_descr->num_elements; j > 0; j--) {
2453                         pci_free_consistent(phba->pcidev,
2454                                             mem_descr->mem_array[j - 1].size,
2455                                             mem_descr->mem_array[j - 1].
2456                                             virtual_address,
2457                                             (unsigned long)mem_descr->
2458                                             mem_array[j - 1].
2459                                             bus_address.u.a64.address);
2460                 }
2461                 if (i) {
2462                         i--;
2463                         kfree(mem_descr->mem_array);
2464                         mem_descr--;
2465                 }
2466         }
2467         kfree(mem_arr_orig);
2468         kfree(phba->init_mem);
2469         kfree(phba->phwi_ctrlr);
2470         return -ENOMEM;
2471 }
2472
2473 static int beiscsi_get_memory(struct beiscsi_hba *phba)
2474 {
2475         beiscsi_find_mem_req(phba);
2476         return beiscsi_alloc_mem(phba);
2477 }
2478
2479 static void iscsi_init_global_templates(struct beiscsi_hba *phba)
2480 {
2481         struct pdu_data_out *pdata_out;
2482         struct pdu_nop_out *pnop_out;
2483         struct be_mem_descriptor *mem_descr;
2484
2485         mem_descr = phba->init_mem;
2486         mem_descr += ISCSI_MEM_GLOBAL_HEADER;
2487         pdata_out =
2488             (struct pdu_data_out *)mem_descr->mem_array[0].virtual_address;
2489         memset(pdata_out, 0, BE_ISCSI_PDU_HEADER_SIZE);
2490
2491         AMAP_SET_BITS(struct amap_pdu_data_out, opcode, pdata_out,
2492                       IIOC_SCSI_DATA);
2493
2494         pnop_out =
2495             (struct pdu_nop_out *)((unsigned char *)mem_descr->mem_array[0].
2496                                    virtual_address + BE_ISCSI_PDU_HEADER_SIZE);
2497
2498         memset(pnop_out, 0, BE_ISCSI_PDU_HEADER_SIZE);
2499         AMAP_SET_BITS(struct amap_pdu_nop_out, ttt, pnop_out, 0xFFFFFFFF);
2500         AMAP_SET_BITS(struct amap_pdu_nop_out, f_bit, pnop_out, 1);
2501         AMAP_SET_BITS(struct amap_pdu_nop_out, i_bit, pnop_out, 0);
2502 }
2503
2504 static int beiscsi_init_wrb_handle(struct beiscsi_hba *phba)
2505 {
2506         struct be_mem_descriptor *mem_descr_wrbh, *mem_descr_wrb;
2507         struct wrb_handle *pwrb_handle = NULL;
2508         struct hwi_controller *phwi_ctrlr;
2509         struct hwi_wrb_context *pwrb_context;
2510         struct iscsi_wrb *pwrb = NULL;
2511         unsigned int num_cxn_wrbh = 0;
2512         unsigned int num_cxn_wrb = 0, j, idx = 0, index;
2513
2514         mem_descr_wrbh = phba->init_mem;
2515         mem_descr_wrbh += HWI_MEM_WRBH;
2516
2517         mem_descr_wrb = phba->init_mem;
2518         mem_descr_wrb += HWI_MEM_WRB;
2519         phwi_ctrlr = phba->phwi_ctrlr;
2520
2521         for (index = 0; index < phba->params.cxns_per_ctrl * 2; index += 2) {
2522                 pwrb_context = &phwi_ctrlr->wrb_context[index];
2523                 pwrb_context->pwrb_handle_base =
2524                                 kzalloc(sizeof(struct wrb_handle *) *
2525                                         phba->params.wrbs_per_cxn, GFP_KERNEL);
2526                 if (!pwrb_context->pwrb_handle_base) {
2527                         beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
2528                                     "BM_%d : Mem Alloc Failed. Failing to load\n");
2529                         goto init_wrb_hndl_failed;
2530                 }
2531                 pwrb_context->pwrb_handle_basestd =
2532                                 kzalloc(sizeof(struct wrb_handle *) *
2533                                         phba->params.wrbs_per_cxn, GFP_KERNEL);
2534                 if (!pwrb_context->pwrb_handle_basestd) {
2535                         beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
2536                                     "BM_%d : Mem Alloc Failed. Failing to load\n");
2537                         goto init_wrb_hndl_failed;
2538                 }
2539                 if (!num_cxn_wrbh) {
2540                         pwrb_handle =
2541                                 mem_descr_wrbh->mem_array[idx].virtual_address;
2542                         num_cxn_wrbh = ((mem_descr_wrbh->mem_array[idx].size) /
2543                                         ((sizeof(struct wrb_handle)) *
2544                                          phba->params.wrbs_per_cxn));
2545                         idx++;
2546                 }
2547                 pwrb_context->alloc_index = 0;
2548                 pwrb_context->wrb_handles_available = 0;
2549                 pwrb_context->free_index = 0;
2550
2551                 if (num_cxn_wrbh) {
2552                         for (j = 0; j < phba->params.wrbs_per_cxn; j++) {
2553                                 pwrb_context->pwrb_handle_base[j] = pwrb_handle;
2554                                 pwrb_context->pwrb_handle_basestd[j] =
2555                                                                 pwrb_handle;
2556                                 pwrb_context->wrb_handles_available++;
2557                                 pwrb_handle->wrb_index = j;
2558                                 pwrb_handle++;
2559                         }
2560                         num_cxn_wrbh--;
2561                 }
2562         }
2563         idx = 0;
2564         for (index = 0; index < phba->params.cxns_per_ctrl * 2; index += 2) {
2565                 pwrb_context = &phwi_ctrlr->wrb_context[index];
2566                 if (!num_cxn_wrb) {
2567                         pwrb = mem_descr_wrb->mem_array[idx].virtual_address;
2568                         num_cxn_wrb = (mem_descr_wrb->mem_array[idx].size) /
2569                                 ((sizeof(struct iscsi_wrb) *
2570                                   phba->params.wrbs_per_cxn));
2571                         idx++;
2572                 }
2573
2574                 if (num_cxn_wrb) {
2575                         for (j = 0; j < phba->params.wrbs_per_cxn; j++) {
2576                                 pwrb_handle = pwrb_context->pwrb_handle_base[j];
2577                                 pwrb_handle->pwrb = pwrb;
2578                                 pwrb++;
2579                         }
2580                         num_cxn_wrb--;
2581                 }
2582         }
2583         return 0;
2584 init_wrb_hndl_failed:
2585         for (j = index; j > 0; j--) {
2586                 pwrb_context = &phwi_ctrlr->wrb_context[j];
2587                 kfree(pwrb_context->pwrb_handle_base);
2588                 kfree(pwrb_context->pwrb_handle_basestd);
2589         }
2590         return -ENOMEM;
2591 }
2592
2593 static void hwi_init_async_pdu_ctx(struct beiscsi_hba *phba)
2594 {
2595         struct hwi_controller *phwi_ctrlr;
2596         struct hba_parameters *p = &phba->params;
2597         struct hwi_async_pdu_context *pasync_ctx;
2598         struct async_pdu_handle *pasync_header_h, *pasync_data_h;
2599         unsigned int index, idx, num_per_mem, num_async_data;
2600         struct be_mem_descriptor *mem_descr;
2601
2602         mem_descr = (struct be_mem_descriptor *)phba->init_mem;
2603         mem_descr += HWI_MEM_ASYNC_PDU_CONTEXT;
2604
2605         phwi_ctrlr = phba->phwi_ctrlr;
2606         phwi_ctrlr->phwi_ctxt->pasync_ctx = (struct hwi_async_pdu_context *)
2607                                 mem_descr->mem_array[0].virtual_address;
2608         pasync_ctx = phwi_ctrlr->phwi_ctxt->pasync_ctx;
2609         memset(pasync_ctx, 0, sizeof(*pasync_ctx));
2610
2611         pasync_ctx->num_entries = p->asyncpdus_per_ctrl;
2612         pasync_ctx->buffer_size = p->defpdu_hdr_sz;
2613
2614         mem_descr = (struct be_mem_descriptor *)phba->init_mem;
2615         mem_descr += HWI_MEM_ASYNC_HEADER_BUF;
2616         if (mem_descr->mem_array[0].virtual_address) {
2617                 beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
2618                             "BM_%d : hwi_init_async_pdu_ctx"
2619                             " HWI_MEM_ASYNC_HEADER_BUF va=%p\n",
2620                             mem_descr->mem_array[0].virtual_address);
2621         } else
2622                 beiscsi_log(phba, KERN_WARNING, BEISCSI_LOG_INIT,
2623                             "BM_%d : No Virtual address\n");
2624
2625         pasync_ctx->async_header.va_base =
2626                         mem_descr->mem_array[0].virtual_address;
2627
2628         pasync_ctx->async_header.pa_base.u.a64.address =
2629                         mem_descr->mem_array[0].bus_address.u.a64.address;
2630
2631         mem_descr = (struct be_mem_descriptor *)phba->init_mem;
2632         mem_descr += HWI_MEM_ASYNC_HEADER_RING;
2633         if (mem_descr->mem_array[0].virtual_address) {
2634                 beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
2635                             "BM_%d : hwi_init_async_pdu_ctx"
2636                             " HWI_MEM_ASYNC_HEADER_RING va=%p\n",
2637                             mem_descr->mem_array[0].virtual_address);
2638         } else
2639                 beiscsi_log(phba, KERN_WARNING, BEISCSI_LOG_INIT,
2640                             "BM_%d : No Virtual address\n");
2641
2642         pasync_ctx->async_header.ring_base =
2643                         mem_descr->mem_array[0].virtual_address;
2644
2645         mem_descr = (struct be_mem_descriptor *)phba->init_mem;
2646         mem_descr += HWI_MEM_ASYNC_HEADER_HANDLE;
2647         if (mem_descr->mem_array[0].virtual_address) {
2648                 beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
2649                             "BM_%d : hwi_init_async_pdu_ctx"
2650                             " HWI_MEM_ASYNC_HEADER_HANDLE va=%p\n",
2651                             mem_descr->mem_array[0].virtual_address);
2652         } else
2653                 beiscsi_log(phba, KERN_WARNING, BEISCSI_LOG_INIT,
2654                             "BM_%d : No Virtual address\n");
2655
2656         pasync_ctx->async_header.handle_base =
2657                         mem_descr->mem_array[0].virtual_address;
2658         pasync_ctx->async_header.writables = 0;
2659         INIT_LIST_HEAD(&pasync_ctx->async_header.free_list);
2660
2661
2662         mem_descr = (struct be_mem_descriptor *)phba->init_mem;
2663         mem_descr += HWI_MEM_ASYNC_DATA_RING;
2664         if (mem_descr->mem_array[0].virtual_address) {
2665                 beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
2666                             "BM_%d : hwi_init_async_pdu_ctx"
2667                             " HWI_MEM_ASYNC_DATA_RING va=%p\n",
2668                             mem_descr->mem_array[0].virtual_address);
2669         } else
2670                 beiscsi_log(phba, KERN_WARNING, BEISCSI_LOG_INIT,
2671                             "BM_%d : No Virtual address\n");
2672
2673         pasync_ctx->async_data.ring_base =
2674                         mem_descr->mem_array[0].virtual_address;
2675
2676         mem_descr = (struct be_mem_descriptor *)phba->init_mem;
2677         mem_descr += HWI_MEM_ASYNC_DATA_HANDLE;
2678         if (!mem_descr->mem_array[0].virtual_address)
2679                 beiscsi_log(phba, KERN_WARNING, BEISCSI_LOG_INIT,
2680                             "BM_%d : No Virtual address\n");
2681
2682         pasync_ctx->async_data.handle_base =
2683                         mem_descr->mem_array[0].virtual_address;
2684         pasync_ctx->async_data.writables = 0;
2685         INIT_LIST_HEAD(&pasync_ctx->async_data.free_list);
2686
2687         pasync_header_h =
2688                 (struct async_pdu_handle *)pasync_ctx->async_header.handle_base;
2689         pasync_data_h =
2690                 (struct async_pdu_handle *)pasync_ctx->async_data.handle_base;
2691
2692         mem_descr = (struct be_mem_descriptor *)phba->init_mem;
2693         mem_descr += HWI_MEM_ASYNC_DATA_BUF;
2694         if (mem_descr->mem_array[0].virtual_address) {
2695                 beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
2696                             "BM_%d : hwi_init_async_pdu_ctx"
2697                             " HWI_MEM_ASYNC_DATA_BUF va=%p\n",
2698                             mem_descr->mem_array[0].virtual_address);
2699         } else
2700                 beiscsi_log(phba, KERN_WARNING, BEISCSI_LOG_INIT,
2701                             "BM_%d : No Virtual address\n");
2702
2703         idx = 0;
2704         pasync_ctx->async_data.va_base =
2705                         mem_descr->mem_array[idx].virtual_address;
2706         pasync_ctx->async_data.pa_base.u.a64.address =
2707                         mem_descr->mem_array[idx].bus_address.u.a64.address;
2708
2709         num_async_data = ((mem_descr->mem_array[idx].size) /
2710                                 phba->params.defpdu_data_sz);
2711         num_per_mem = 0;
2712
2713         for (index = 0; index < p->asyncpdus_per_ctrl; index++) {
2714                 pasync_header_h->cri = -1;
2715                 pasync_header_h->index = (char)index;
2716                 INIT_LIST_HEAD(&pasync_header_h->link);
2717                 pasync_header_h->pbuffer =
2718                         (void *)((unsigned long)
2719                         (pasync_ctx->async_header.va_base) +
2720                         (p->defpdu_hdr_sz * index));
2721
2722                 pasync_header_h->pa.u.a64.address =
2723                         pasync_ctx->async_header.pa_base.u.a64.address +
2724                         (p->defpdu_hdr_sz * index);
2725
2726                 list_add_tail(&pasync_header_h->link,
2727                                 &pasync_ctx->async_header.free_list);
2728                 pasync_header_h++;
2729                 pasync_ctx->async_header.free_entries++;
2730                 pasync_ctx->async_header.writables++;
2731
2732                 INIT_LIST_HEAD(&pasync_ctx->async_entry[index].wait_queue.list);
2733                 INIT_LIST_HEAD(&pasync_ctx->async_entry[index].
2734                                header_busy_list);
2735                 pasync_data_h->cri = -1;
2736                 pasync_data_h->index = (char)index;
2737                 INIT_LIST_HEAD(&pasync_data_h->link);
2738
2739                 if (!num_async_data) {
2740                         num_per_mem = 0;
2741                         idx++;
2742                         pasync_ctx->async_data.va_base =
2743                                 mem_descr->mem_array[idx].virtual_address;
2744                         pasync_ctx->async_data.pa_base.u.a64.address =
2745                                 mem_descr->mem_array[idx].
2746                                 bus_address.u.a64.address;
2747
2748                         num_async_data = ((mem_descr->mem_array[idx].size) /
2749                                         phba->params.defpdu_data_sz);
2750                 }
2751                 pasync_data_h->pbuffer =
2752                         (void *)((unsigned long)
2753                         (pasync_ctx->async_data.va_base) +
2754                         (p->defpdu_data_sz * num_per_mem));
2755
2756                 pasync_data_h->pa.u.a64.address =
2757                     pasync_ctx->async_data.pa_base.u.a64.address +
2758                     (p->defpdu_data_sz * num_per_mem);
2759                 num_per_mem++;
2760                 num_async_data--;
2761
2762                 list_add_tail(&pasync_data_h->link,
2763                               &pasync_ctx->async_data.free_list);
2764                 pasync_data_h++;
2765                 pasync_ctx->async_data.free_entries++;
2766                 pasync_ctx->async_data.writables++;
2767
2768                 INIT_LIST_HEAD(&pasync_ctx->async_entry[index].data_busy_list);
2769         }
2770
2771         pasync_ctx->async_header.host_write_ptr = 0;
2772         pasync_ctx->async_header.ep_read_ptr = -1;
2773         pasync_ctx->async_data.host_write_ptr = 0;
2774         pasync_ctx->async_data.ep_read_ptr = -1;
2775 }
2776
2777 static int
2778 be_sgl_create_contiguous(void *virtual_address,
2779                          u64 physical_address, u32 length,
2780                          struct be_dma_mem *sgl)
2781 {
2782         WARN_ON(!virtual_address);
2783         WARN_ON(!physical_address);
2784         WARN_ON(!length > 0);
2785         WARN_ON(!sgl);
2786
2787         sgl->va = virtual_address;
2788         sgl->dma = (unsigned long)physical_address;
2789         sgl->size = length;
2790
2791         return 0;
2792 }
2793
2794 static void be_sgl_destroy_contiguous(struct be_dma_mem *sgl)
2795 {
2796         memset(sgl, 0, sizeof(*sgl));
2797 }
2798
2799 static void
2800 hwi_build_be_sgl_arr(struct beiscsi_hba *phba,
2801                      struct mem_array *pmem, struct be_dma_mem *sgl)
2802 {
2803         if (sgl->va)
2804                 be_sgl_destroy_contiguous(sgl);
2805
2806         be_sgl_create_contiguous(pmem->virtual_address,
2807                                  pmem->bus_address.u.a64.address,
2808                                  pmem->size, sgl);
2809 }
2810
2811 static void
2812 hwi_build_be_sgl_by_offset(struct beiscsi_hba *phba,
2813                            struct mem_array *pmem, struct be_dma_mem *sgl)
2814 {
2815         if (sgl->va)
2816                 be_sgl_destroy_contiguous(sgl);
2817
2818         be_sgl_create_contiguous((unsigned char *)pmem->virtual_address,
2819                                  pmem->bus_address.u.a64.address,
2820                                  pmem->size, sgl);
2821 }
2822
2823 static int be_fill_queue(struct be_queue_info *q,
2824                 u16 len, u16 entry_size, void *vaddress)
2825 {
2826         struct be_dma_mem *mem = &q->dma_mem;
2827
2828         memset(q, 0, sizeof(*q));
2829         q->len = len;
2830         q->entry_size = entry_size;
2831         mem->size = len * entry_size;
2832         mem->va = vaddress;
2833         if (!mem->va)
2834                 return -ENOMEM;
2835         memset(mem->va, 0, mem->size);
2836         return 0;
2837 }
2838
2839 static int beiscsi_create_eqs(struct beiscsi_hba *phba,
2840                              struct hwi_context_memory *phwi_context)
2841 {
2842         unsigned int i, num_eq_pages;
2843         int ret = 0, eq_for_mcc;
2844         struct be_queue_info *eq;
2845         struct be_dma_mem *mem;
2846         void *eq_vaddress;
2847         dma_addr_t paddr;
2848
2849         num_eq_pages = PAGES_REQUIRED(phba->params.num_eq_entries * \
2850                                       sizeof(struct be_eq_entry));
2851
2852         if (phba->msix_enabled)
2853                 eq_for_mcc = 1;
2854         else
2855                 eq_for_mcc = 0;
2856         for (i = 0; i < (phba->num_cpus + eq_for_mcc); i++) {
2857                 eq = &phwi_context->be_eq[i].q;
2858                 mem = &eq->dma_mem;
2859                 phwi_context->be_eq[i].phba = phba;
2860                 eq_vaddress = pci_alloc_consistent(phba->pcidev,
2861                                                      num_eq_pages * PAGE_SIZE,
2862                                                      &paddr);
2863                 if (!eq_vaddress)
2864                         goto create_eq_error;
2865
2866                 mem->va = eq_vaddress;
2867                 ret = be_fill_queue(eq, phba->params.num_eq_entries,
2868                                     sizeof(struct be_eq_entry), eq_vaddress);
2869                 if (ret) {
2870                         beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
2871                                     "BM_%d : be_fill_queue Failed for EQ\n");
2872                         goto create_eq_error;
2873                 }
2874
2875                 mem->dma = paddr;
2876                 ret = beiscsi_cmd_eq_create(&phba->ctrl, eq,
2877                                             phwi_context->cur_eqd);
2878                 if (ret) {
2879                         beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
2880                                     "BM_%d : beiscsi_cmd_eq_create"
2881                                     "Failed for EQ\n");
2882                         goto create_eq_error;
2883                 }
2884
2885                 beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
2886                             "BM_%d : eqid = %d\n",
2887                             phwi_context->be_eq[i].q.id);
2888         }
2889         return 0;
2890 create_eq_error:
2891         for (i = 0; i < (phba->num_cpus + eq_for_mcc); i++) {
2892                 eq = &phwi_context->be_eq[i].q;
2893                 mem = &eq->dma_mem;
2894                 if (mem->va)
2895                         pci_free_consistent(phba->pcidev, num_eq_pages
2896                                             * PAGE_SIZE,
2897                                             mem->va, mem->dma);
2898         }
2899         return ret;
2900 }
2901
2902 static int beiscsi_create_cqs(struct beiscsi_hba *phba,
2903                              struct hwi_context_memory *phwi_context)
2904 {
2905         unsigned int i, num_cq_pages;
2906         int ret = 0;
2907         struct be_queue_info *cq, *eq;
2908         struct be_dma_mem *mem;
2909         struct be_eq_obj *pbe_eq;
2910         void *cq_vaddress;
2911         dma_addr_t paddr;
2912
2913         num_cq_pages = PAGES_REQUIRED(phba->params.num_cq_entries * \
2914                                       sizeof(struct sol_cqe));
2915
2916         for (i = 0; i < phba->num_cpus; i++) {
2917                 cq = &phwi_context->be_cq[i];
2918                 eq = &phwi_context->be_eq[i].q;
2919                 pbe_eq = &phwi_context->be_eq[i];
2920                 pbe_eq->cq = cq;
2921                 pbe_eq->phba = phba;
2922                 mem = &cq->dma_mem;
2923                 cq_vaddress = pci_alloc_consistent(phba->pcidev,
2924                                                      num_cq_pages * PAGE_SIZE,
2925                                                      &paddr);
2926                 if (!cq_vaddress)
2927                         goto create_cq_error;
2928                 ret = be_fill_queue(cq, phba->params.num_cq_entries,
2929                                     sizeof(struct sol_cqe), cq_vaddress);
2930                 if (ret) {
2931                         beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
2932                                     "BM_%d : be_fill_queue Failed "
2933                                     "for ISCSI CQ\n");
2934                         goto create_cq_error;
2935                 }
2936
2937                 mem->dma = paddr;
2938                 ret = beiscsi_cmd_cq_create(&phba->ctrl, cq, eq, false,
2939                                             false, 0);
2940                 if (ret) {
2941                         beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
2942                                     "BM_%d : beiscsi_cmd_eq_create"
2943                                     "Failed for ISCSI CQ\n");
2944                         goto create_cq_error;
2945                 }
2946                 beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
2947                             "BM_%d : iscsi cq_id is %d for eq_id %d\n"
2948                             "iSCSI CQ CREATED\n", cq->id, eq->id);
2949         }
2950         return 0;
2951
2952 create_cq_error:
2953         for (i = 0; i < phba->num_cpus; i++) {
2954                 cq = &phwi_context->be_cq[i];
2955                 mem = &cq->dma_mem;
2956                 if (mem->va)
2957                         pci_free_consistent(phba->pcidev, num_cq_pages
2958                                             * PAGE_SIZE,
2959                                             mem->va, mem->dma);
2960         }
2961         return ret;
2962
2963 }
2964
2965 static int
2966 beiscsi_create_def_hdr(struct beiscsi_hba *phba,
2967                        struct hwi_context_memory *phwi_context,
2968                        struct hwi_controller *phwi_ctrlr,
2969                        unsigned int def_pdu_ring_sz)
2970 {
2971         unsigned int idx;
2972         int ret;
2973         struct be_queue_info *dq, *cq;
2974         struct be_dma_mem *mem;
2975         struct be_mem_descriptor *mem_descr;
2976         void *dq_vaddress;
2977
2978         idx = 0;
2979         dq = &phwi_context->be_def_hdrq;
2980         cq = &phwi_context->be_cq[0];
2981         mem = &dq->dma_mem;
2982         mem_descr = phba->init_mem;
2983         mem_descr += HWI_MEM_ASYNC_HEADER_RING;
2984         dq_vaddress = mem_descr->mem_array[idx].virtual_address;
2985         ret = be_fill_queue(dq, mem_descr->mem_array[0].size /
2986                             sizeof(struct phys_addr),
2987                             sizeof(struct phys_addr), dq_vaddress);
2988         if (ret) {
2989                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
2990                             "BM_%d : be_fill_queue Failed for DEF PDU HDR\n");
2991                 return ret;
2992         }
2993         mem->dma = (unsigned long)mem_descr->mem_array[idx].
2994                                   bus_address.u.a64.address;
2995         ret = be_cmd_create_default_pdu_queue(&phba->ctrl, cq, dq,
2996                                               def_pdu_ring_sz,
2997                                               phba->params.defpdu_hdr_sz);
2998         if (ret) {
2999                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3000                             "BM_%d : be_cmd_create_default_pdu_queue Failed DEFHDR\n");
3001                 return ret;
3002         }
3003         phwi_ctrlr->default_pdu_hdr.id = phwi_context->be_def_hdrq.id;
3004         beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
3005                     "BM_%d : iscsi def pdu id is %d\n",
3006                     phwi_context->be_def_hdrq.id);
3007
3008         hwi_post_async_buffers(phba, 1);
3009         return 0;
3010 }
3011
3012 static int
3013 beiscsi_create_def_data(struct beiscsi_hba *phba,
3014                         struct hwi_context_memory *phwi_context,
3015                         struct hwi_controller *phwi_ctrlr,
3016                         unsigned int def_pdu_ring_sz)
3017 {
3018         unsigned int idx;
3019         int ret;
3020         struct be_queue_info *dataq, *cq;
3021         struct be_dma_mem *mem;
3022         struct be_mem_descriptor *mem_descr;
3023         void *dq_vaddress;
3024
3025         idx = 0;
3026         dataq = &phwi_context->be_def_dataq;
3027         cq = &phwi_context->be_cq[0];
3028         mem = &dataq->dma_mem;
3029         mem_descr = phba->init_mem;
3030         mem_descr += HWI_MEM_ASYNC_DATA_RING;
3031         dq_vaddress = mem_descr->mem_array[idx].virtual_address;
3032         ret = be_fill_queue(dataq, mem_descr->mem_array[0].size /
3033                             sizeof(struct phys_addr),
3034                             sizeof(struct phys_addr), dq_vaddress);
3035         if (ret) {
3036                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3037                             "BM_%d : be_fill_queue Failed for DEF PDU DATA\n");
3038                 return ret;
3039         }
3040         mem->dma = (unsigned long)mem_descr->mem_array[idx].
3041                                   bus_address.u.a64.address;
3042         ret = be_cmd_create_default_pdu_queue(&phba->ctrl, cq, dataq,
3043                                               def_pdu_ring_sz,
3044                                               phba->params.defpdu_data_sz);
3045         if (ret) {
3046                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3047                             "BM_%d be_cmd_create_default_pdu_queue"
3048                             " Failed for DEF PDU DATA\n");
3049                 return ret;
3050         }
3051         phwi_ctrlr->default_pdu_data.id = phwi_context->be_def_dataq.id;
3052         beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
3053                     "BM_%d : iscsi def data id is %d\n",
3054                     phwi_context->be_def_dataq.id);
3055
3056         hwi_post_async_buffers(phba, 0);
3057         beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
3058                     "BM_%d : DEFAULT PDU DATA RING CREATED\n");
3059
3060         return 0;
3061 }
3062
3063 static int
3064 beiscsi_post_pages(struct beiscsi_hba *phba)
3065 {
3066         struct be_mem_descriptor *mem_descr;
3067         struct mem_array *pm_arr;
3068         unsigned int page_offset, i;
3069         struct be_dma_mem sgl;
3070         int status;
3071
3072         mem_descr = phba->init_mem;
3073         mem_descr += HWI_MEM_SGE;
3074         pm_arr = mem_descr->mem_array;
3075
3076         page_offset = (sizeof(struct iscsi_sge) * phba->params.num_sge_per_io *
3077                         phba->fw_config.iscsi_icd_start) / PAGE_SIZE;
3078         for (i = 0; i < mem_descr->num_elements; i++) {
3079                 hwi_build_be_sgl_arr(phba, pm_arr, &sgl);
3080                 status = be_cmd_iscsi_post_sgl_pages(&phba->ctrl, &sgl,
3081                                                 page_offset,
3082                                                 (pm_arr->size / PAGE_SIZE));
3083                 page_offset += pm_arr->size / PAGE_SIZE;
3084                 if (status != 0) {
3085                         beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3086                                     "BM_%d : post sgl failed.\n");
3087                         return status;
3088                 }
3089                 pm_arr++;
3090         }
3091         beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
3092                     "BM_%d : POSTED PAGES\n");
3093         return 0;
3094 }
3095
3096 static void be_queue_free(struct beiscsi_hba *phba, struct be_queue_info *q)
3097 {
3098         struct be_dma_mem *mem = &q->dma_mem;
3099         if (mem->va) {
3100                 pci_free_consistent(phba->pcidev, mem->size,
3101                         mem->va, mem->dma);
3102                 mem->va = NULL;
3103         }
3104 }
3105
3106 static int be_queue_alloc(struct beiscsi_hba *phba, struct be_queue_info *q,
3107                 u16 len, u16 entry_size)
3108 {
3109         struct be_dma_mem *mem = &q->dma_mem;
3110
3111         memset(q, 0, sizeof(*q));
3112         q->len = len;
3113         q->entry_size = entry_size;
3114         mem->size = len * entry_size;
3115         mem->va = pci_alloc_consistent(phba->pcidev, mem->size, &mem->dma);
3116         if (!mem->va)
3117                 return -ENOMEM;
3118         memset(mem->va, 0, mem->size);
3119         return 0;
3120 }
3121
3122 static int
3123 beiscsi_create_wrb_rings(struct beiscsi_hba *phba,
3124                          struct hwi_context_memory *phwi_context,
3125                          struct hwi_controller *phwi_ctrlr)
3126 {
3127         unsigned int wrb_mem_index, offset, size, num_wrb_rings;
3128         u64 pa_addr_lo;
3129         unsigned int idx, num, i;
3130         struct mem_array *pwrb_arr;
3131         void *wrb_vaddr;
3132         struct be_dma_mem sgl;
3133         struct be_mem_descriptor *mem_descr;
3134         int status;
3135
3136         idx = 0;
3137         mem_descr = phba->init_mem;
3138         mem_descr += HWI_MEM_WRB;
3139         pwrb_arr = kmalloc(sizeof(*pwrb_arr) * phba->params.cxns_per_ctrl,
3140                            GFP_KERNEL);
3141         if (!pwrb_arr) {
3142                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3143                             "BM_%d : Memory alloc failed in create wrb ring.\n");
3144                 return -ENOMEM;
3145         }
3146         wrb_vaddr = mem_descr->mem_array[idx].virtual_address;
3147         pa_addr_lo = mem_descr->mem_array[idx].bus_address.u.a64.address;
3148         num_wrb_rings = mem_descr->mem_array[idx].size /
3149                 (phba->params.wrbs_per_cxn * sizeof(struct iscsi_wrb));
3150
3151         for (num = 0; num < phba->params.cxns_per_ctrl; num++) {
3152                 if (num_wrb_rings) {
3153                         pwrb_arr[num].virtual_address = wrb_vaddr;
3154                         pwrb_arr[num].bus_address.u.a64.address = pa_addr_lo;
3155                         pwrb_arr[num].size = phba->params.wrbs_per_cxn *
3156                                             sizeof(struct iscsi_wrb);
3157                         wrb_vaddr += pwrb_arr[num].size;
3158                         pa_addr_lo += pwrb_arr[num].size;
3159                         num_wrb_rings--;
3160                 } else {
3161                         idx++;
3162                         wrb_vaddr = mem_descr->mem_array[idx].virtual_address;
3163                         pa_addr_lo = mem_descr->mem_array[idx].\
3164                                         bus_address.u.a64.address;
3165                         num_wrb_rings = mem_descr->mem_array[idx].size /
3166                                         (phba->params.wrbs_per_cxn *
3167                                         sizeof(struct iscsi_wrb));
3168                         pwrb_arr[num].virtual_address = wrb_vaddr;
3169                         pwrb_arr[num].bus_address.u.a64.address\
3170                                                 = pa_addr_lo;
3171                         pwrb_arr[num].size = phba->params.wrbs_per_cxn *
3172                                                  sizeof(struct iscsi_wrb);
3173                         wrb_vaddr += pwrb_arr[num].size;
3174                         pa_addr_lo   += pwrb_arr[num].size;
3175                         num_wrb_rings--;
3176                 }
3177         }
3178         for (i = 0; i < phba->params.cxns_per_ctrl; i++) {
3179                 wrb_mem_index = 0;
3180                 offset = 0;
3181                 size = 0;
3182
3183                 hwi_build_be_sgl_by_offset(phba, &pwrb_arr[i], &sgl);
3184                 status = be_cmd_wrbq_create(&phba->ctrl, &sgl,
3185                                             &phwi_context->be_wrbq[i]);
3186                 if (status != 0) {
3187                         beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3188                                     "BM_%d : wrbq create failed.");
3189                         kfree(pwrb_arr);
3190                         return status;
3191                 }
3192                 phwi_ctrlr->wrb_context[i * 2].cid = phwi_context->be_wrbq[i].
3193                                                                    id;
3194         }
3195         kfree(pwrb_arr);
3196         return 0;
3197 }
3198
3199 static void free_wrb_handles(struct beiscsi_hba *phba)
3200 {
3201         unsigned int index;
3202         struct hwi_controller *phwi_ctrlr;
3203         struct hwi_wrb_context *pwrb_context;
3204
3205         phwi_ctrlr = phba->phwi_ctrlr;
3206         for (index = 0; index < phba->params.cxns_per_ctrl * 2; index += 2) {
3207                 pwrb_context = &phwi_ctrlr->wrb_context[index];
3208                 kfree(pwrb_context->pwrb_handle_base);
3209                 kfree(pwrb_context->pwrb_handle_basestd);
3210         }
3211 }
3212
3213 static void be_mcc_queues_destroy(struct beiscsi_hba *phba)
3214 {
3215         struct be_queue_info *q;
3216         struct be_ctrl_info *ctrl = &phba->ctrl;
3217
3218         q = &phba->ctrl.mcc_obj.q;
3219         if (q->created)
3220                 beiscsi_cmd_q_destroy(ctrl, q, QTYPE_MCCQ);
3221         be_queue_free(phba, q);
3222
3223         q = &phba->ctrl.mcc_obj.cq;
3224         if (q->created)
3225                 beiscsi_cmd_q_destroy(ctrl, q, QTYPE_CQ);
3226         be_queue_free(phba, q);
3227 }
3228
3229 static void hwi_cleanup(struct beiscsi_hba *phba)
3230 {
3231         struct be_queue_info *q;
3232         struct be_ctrl_info *ctrl = &phba->ctrl;
3233         struct hwi_controller *phwi_ctrlr;
3234         struct hwi_context_memory *phwi_context;
3235         int i, eq_num;
3236
3237         phwi_ctrlr = phba->phwi_ctrlr;
3238         phwi_context = phwi_ctrlr->phwi_ctxt;
3239         for (i = 0; i < phba->params.cxns_per_ctrl; i++) {
3240                 q = &phwi_context->be_wrbq[i];
3241                 if (q->created)
3242                         beiscsi_cmd_q_destroy(ctrl, q, QTYPE_WRBQ);
3243         }
3244         free_wrb_handles(phba);
3245
3246         q = &phwi_context->be_def_hdrq;
3247         if (q->created)
3248                 beiscsi_cmd_q_destroy(ctrl, q, QTYPE_DPDUQ);
3249
3250         q = &phwi_context->be_def_dataq;
3251         if (q->created)
3252                 beiscsi_cmd_q_destroy(ctrl, q, QTYPE_DPDUQ);
3253
3254         beiscsi_cmd_q_destroy(ctrl, NULL, QTYPE_SGL);
3255
3256         for (i = 0; i < (phba->num_cpus); i++) {
3257                 q = &phwi_context->be_cq[i];
3258                 if (q->created)
3259                         beiscsi_cmd_q_destroy(ctrl, q, QTYPE_CQ);
3260         }
3261         if (phba->msix_enabled)
3262                 eq_num = 1;
3263         else
3264                 eq_num = 0;
3265         for (i = 0; i < (phba->num_cpus + eq_num); i++) {
3266                 q = &phwi_context->be_eq[i].q;
3267                 if (q->created)
3268                         beiscsi_cmd_q_destroy(ctrl, q, QTYPE_EQ);
3269         }
3270         be_mcc_queues_destroy(phba);
3271 }
3272
3273 static int be_mcc_queues_create(struct beiscsi_hba *phba,
3274                                 struct hwi_context_memory *phwi_context)
3275 {
3276         struct be_queue_info *q, *cq;
3277         struct be_ctrl_info *ctrl = &phba->ctrl;
3278
3279         /* Alloc MCC compl queue */
3280         cq = &phba->ctrl.mcc_obj.cq;
3281         if (be_queue_alloc(phba, cq, MCC_CQ_LEN,
3282                         sizeof(struct be_mcc_compl)))
3283                 goto err;
3284         /* Ask BE to create MCC compl queue; */
3285         if (phba->msix_enabled) {
3286                 if (beiscsi_cmd_cq_create(ctrl, cq, &phwi_context->be_eq
3287                                          [phba->num_cpus].q, false, true, 0))
3288                 goto mcc_cq_free;
3289         } else {
3290                 if (beiscsi_cmd_cq_create(ctrl, cq, &phwi_context->be_eq[0].q,
3291                                           false, true, 0))
3292                 goto mcc_cq_free;
3293         }
3294
3295         /* Alloc MCC queue */
3296         q = &phba->ctrl.mcc_obj.q;
3297         if (be_queue_alloc(phba, q, MCC_Q_LEN, sizeof(struct be_mcc_wrb)))
3298                 goto mcc_cq_destroy;
3299
3300         /* Ask BE to create MCC queue */
3301         if (beiscsi_cmd_mccq_create(phba, q, cq))
3302                 goto mcc_q_free;
3303
3304         return 0;
3305
3306 mcc_q_free:
3307         be_queue_free(phba, q);
3308 mcc_cq_destroy:
3309         beiscsi_cmd_q_destroy(ctrl, cq, QTYPE_CQ);
3310 mcc_cq_free:
3311         be_queue_free(phba, cq);
3312 err:
3313         return -ENOMEM;
3314 }
3315
3316 /**
3317  * find_num_cpus()- Get the CPU online count
3318  * @phba: ptr to priv structure
3319  *
3320  * CPU count is used for creating EQ.
3321  **/
3322 static void find_num_cpus(struct beiscsi_hba *phba)
3323 {
3324         int  num_cpus = 0;
3325
3326         num_cpus = num_online_cpus();
3327
3328         switch (phba->generation) {
3329         case BE_GEN2:
3330         case BE_GEN3:
3331                 phba->num_cpus = (num_cpus > BEISCSI_MAX_NUM_CPUS) ?
3332                                   BEISCSI_MAX_NUM_CPUS : num_cpus;
3333                 break;
3334         case BE_GEN4:
3335                 phba->num_cpus = (num_cpus > OC_SKH_MAX_NUM_CPUS) ?
3336                                   OC_SKH_MAX_NUM_CPUS : num_cpus;
3337                 break;
3338         default:
3339                 phba->num_cpus = 1;
3340         }
3341 }
3342
3343 static int hwi_init_port(struct beiscsi_hba *phba)
3344 {
3345         struct hwi_controller *phwi_ctrlr;
3346         struct hwi_context_memory *phwi_context;
3347         unsigned int def_pdu_ring_sz;
3348         struct be_ctrl_info *ctrl = &phba->ctrl;
3349         int status;
3350
3351         def_pdu_ring_sz =
3352                 phba->params.asyncpdus_per_ctrl * sizeof(struct phys_addr);
3353         phwi_ctrlr = phba->phwi_ctrlr;
3354         phwi_context = phwi_ctrlr->phwi_ctxt;
3355         phwi_context->max_eqd = 0;
3356         phwi_context->min_eqd = 0;
3357         phwi_context->cur_eqd = 64;
3358         be_cmd_fw_initialize(&phba->ctrl);
3359
3360         status = beiscsi_create_eqs(phba, phwi_context);
3361         if (status != 0) {
3362                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3363                             "BM_%d : EQ not created\n");
3364                 goto error;
3365         }
3366
3367         status = be_mcc_queues_create(phba, phwi_context);
3368         if (status != 0)
3369                 goto error;
3370
3371         status = mgmt_check_supported_fw(ctrl, phba);
3372         if (status != 0) {
3373                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3374                             "BM_%d : Unsupported fw version\n");
3375                 goto error;
3376         }
3377
3378         status = beiscsi_create_cqs(phba, phwi_context);
3379         if (status != 0) {
3380                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3381                             "BM_%d : CQ not created\n");
3382                 goto error;
3383         }
3384
3385         status = beiscsi_create_def_hdr(phba, phwi_context, phwi_ctrlr,
3386                                         def_pdu_ring_sz);
3387         if (status != 0) {
3388                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3389                             "BM_%d : Default Header not created\n");
3390                 goto error;
3391         }
3392
3393         status = beiscsi_create_def_data(phba, phwi_context,
3394                                          phwi_ctrlr, def_pdu_ring_sz);
3395         if (status != 0) {
3396                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3397                             "BM_%d : Default Data not created\n");
3398                 goto error;
3399         }
3400
3401         status = beiscsi_post_pages(phba);
3402         if (status != 0) {
3403                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3404                             "BM_%d : Post SGL Pages Failed\n");
3405                 goto error;
3406         }
3407
3408         status = beiscsi_create_wrb_rings(phba, phwi_context, phwi_ctrlr);
3409         if (status != 0) {
3410                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3411                             "BM_%d : WRB Rings not created\n");
3412                 goto error;
3413         }
3414
3415         beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
3416                     "BM_%d : hwi_init_port success\n");
3417         return 0;
3418
3419 error:
3420         beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3421                     "BM_%d : hwi_init_port failed");
3422         hwi_cleanup(phba);
3423         return status;
3424 }
3425
3426 static int hwi_init_controller(struct beiscsi_hba *phba)
3427 {
3428         struct hwi_controller *phwi_ctrlr;
3429
3430         phwi_ctrlr = phba->phwi_ctrlr;
3431         if (1 == phba->init_mem[HWI_MEM_ADDN_CONTEXT].num_elements) {
3432                 phwi_ctrlr->phwi_ctxt = (struct hwi_context_memory *)phba->
3433                     init_mem[HWI_MEM_ADDN_CONTEXT].mem_array[0].virtual_address;
3434                 beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
3435                             "BM_%d :  phwi_ctrlr->phwi_ctxt=%p\n",
3436                             phwi_ctrlr->phwi_ctxt);
3437         } else {
3438                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3439                             "BM_%d : HWI_MEM_ADDN_CONTEXT is more "
3440                             "than one element.Failing to load\n");
3441                 return -ENOMEM;
3442         }
3443
3444         iscsi_init_global_templates(phba);
3445         if (beiscsi_init_wrb_handle(phba))
3446                 return -ENOMEM;
3447
3448         hwi_init_async_pdu_ctx(phba);
3449         if (hwi_init_port(phba) != 0) {
3450                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3451                             "BM_%d : hwi_init_controller failed\n");
3452
3453                 return -ENOMEM;
3454         }
3455         return 0;
3456 }
3457
3458 static void beiscsi_free_mem(struct beiscsi_hba *phba)
3459 {
3460         struct be_mem_descriptor *mem_descr;
3461         int i, j;
3462
3463         mem_descr = phba->init_mem;
3464         i = 0;
3465         j = 0;
3466         for (i = 0; i < SE_MEM_MAX; i++) {
3467                 for (j = mem_descr->num_elements; j > 0; j--) {
3468                         pci_free_consistent(phba->pcidev,
3469                           mem_descr->mem_array[j - 1].size,
3470                           mem_descr->mem_array[j - 1].virtual_address,
3471                           (unsigned long)mem_descr->mem_array[j - 1].
3472                           bus_address.u.a64.address);
3473                 }
3474                 kfree(mem_descr->mem_array);
3475                 mem_descr++;
3476         }
3477         kfree(phba->init_mem);
3478         kfree(phba->phwi_ctrlr);
3479 }
3480
3481 static int beiscsi_init_controller(struct beiscsi_hba *phba)
3482 {
3483         int ret = -ENOMEM;
3484
3485         ret = beiscsi_get_memory(phba);
3486         if (ret < 0) {
3487                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3488                             "BM_%d : beiscsi_dev_probe -"
3489                             "Failed in beiscsi_alloc_memory\n");
3490                 return ret;
3491         }
3492
3493         ret = hwi_init_controller(phba);
3494         if (ret)
3495                 goto free_init;
3496         beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
3497                     "BM_%d : Return success from beiscsi_init_controller");
3498
3499         return 0;
3500
3501 free_init:
3502         beiscsi_free_mem(phba);
3503         return ret;
3504 }
3505
3506 static int beiscsi_init_sgl_handle(struct beiscsi_hba *phba)
3507 {
3508         struct be_mem_descriptor *mem_descr_sglh, *mem_descr_sg;
3509         struct sgl_handle *psgl_handle;
3510         struct iscsi_sge *pfrag;
3511         unsigned int arr_index, i, idx;
3512
3513         phba->io_sgl_hndl_avbl = 0;
3514         phba->eh_sgl_hndl_avbl = 0;
3515
3516         mem_descr_sglh = phba->init_mem;
3517         mem_descr_sglh += HWI_MEM_SGLH;
3518         if (1 == mem_descr_sglh->num_elements) {
3519                 phba->io_sgl_hndl_base = kzalloc(sizeof(struct sgl_handle *) *
3520                                                  phba->params.ios_per_ctrl,
3521                                                  GFP_KERNEL);
3522                 if (!phba->io_sgl_hndl_base) {
3523                         beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3524                                     "BM_%d : Mem Alloc Failed. Failing to load\n");
3525                         return -ENOMEM;
3526                 }
3527                 phba->eh_sgl_hndl_base = kzalloc(sizeof(struct sgl_handle *) *
3528                                                  (phba->params.icds_per_ctrl -
3529                                                  phba->params.ios_per_ctrl),
3530                                                  GFP_KERNEL);
3531                 if (!phba->eh_sgl_hndl_base) {
3532                         kfree(phba->io_sgl_hndl_base);
3533                         beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3534                                     "BM_%d : Mem Alloc Failed. Failing to load\n");
3535                         return -ENOMEM;
3536                 }
3537         } else {
3538                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3539                             "BM_%d : HWI_MEM_SGLH is more than one element."
3540                             "Failing to load\n");
3541                 return -ENOMEM;
3542         }
3543
3544         arr_index = 0;
3545         idx = 0;
3546         while (idx < mem_descr_sglh->num_elements) {
3547                 psgl_handle = mem_descr_sglh->mem_array[idx].virtual_address;
3548
3549                 for (i = 0; i < (mem_descr_sglh->mem_array[idx].size /
3550                       sizeof(struct sgl_handle)); i++) {
3551                         if (arr_index < phba->params.ios_per_ctrl) {
3552                                 phba->io_sgl_hndl_base[arr_index] = psgl_handle;
3553                                 phba->io_sgl_hndl_avbl++;
3554                                 arr_index++;
3555                         } else {
3556                                 phba->eh_sgl_hndl_base[arr_index -
3557                                         phba->params.ios_per_ctrl] =
3558                                                                 psgl_handle;
3559                                 arr_index++;
3560                                 phba->eh_sgl_hndl_avbl++;
3561                         }
3562                         psgl_handle++;
3563                 }
3564                 idx++;
3565         }
3566         beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
3567                     "BM_%d : phba->io_sgl_hndl_avbl=%d"
3568                     "phba->eh_sgl_hndl_avbl=%d\n",
3569                     phba->io_sgl_hndl_avbl,
3570                     phba->eh_sgl_hndl_avbl);
3571
3572         mem_descr_sg = phba->init_mem;
3573         mem_descr_sg += HWI_MEM_SGE;
3574         beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
3575                     "\n BM_%d : mem_descr_sg->num_elements=%d\n",
3576                     mem_descr_sg->num_elements);
3577
3578         arr_index = 0;
3579         idx = 0;
3580         while (idx < mem_descr_sg->num_elements) {
3581                 pfrag = mem_descr_sg->mem_array[idx].virtual_address;
3582
3583                 for (i = 0;
3584                      i < (mem_descr_sg->mem_array[idx].size) /
3585                      (sizeof(struct iscsi_sge) * phba->params.num_sge_per_io);
3586                      i++) {
3587                         if (arr_index < phba->params.ios_per_ctrl)
3588                                 psgl_handle = phba->io_sgl_hndl_base[arr_index];
3589                         else
3590                                 psgl_handle = phba->eh_sgl_hndl_base[arr_index -
3591                                                 phba->params.ios_per_ctrl];
3592                         psgl_handle->pfrag = pfrag;
3593                         AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, pfrag, 0);
3594                         AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, pfrag, 0);
3595                         pfrag += phba->params.num_sge_per_io;
3596                         psgl_handle->sgl_index =
3597                                 phba->fw_config.iscsi_icd_start + arr_index++;
3598                 }
3599                 idx++;
3600         }
3601         phba->io_sgl_free_index = 0;
3602         phba->io_sgl_alloc_index = 0;
3603         phba->eh_sgl_free_index = 0;
3604         phba->eh_sgl_alloc_index = 0;
3605         return 0;
3606 }
3607
3608 static int hba_setup_cid_tbls(struct beiscsi_hba *phba)
3609 {
3610         int i, new_cid;
3611
3612         phba->cid_array = kzalloc(sizeof(void *) * phba->params.cxns_per_ctrl,
3613                                   GFP_KERNEL);
3614         if (!phba->cid_array) {
3615                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3616                             "BM_%d : Failed to allocate memory in "
3617                             "hba_setup_cid_tbls\n");
3618                 return -ENOMEM;
3619         }
3620         phba->ep_array = kzalloc(sizeof(struct iscsi_endpoint *) *
3621                                  phba->params.cxns_per_ctrl * 2, GFP_KERNEL);
3622         if (!phba->ep_array) {
3623                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3624                             "BM_%d : Failed to allocate memory in "
3625                             "hba_setup_cid_tbls\n");
3626                 kfree(phba->cid_array);
3627                 return -ENOMEM;
3628         }
3629         new_cid = phba->fw_config.iscsi_cid_start;
3630         for (i = 0; i < phba->params.cxns_per_ctrl; i++) {
3631                 phba->cid_array[i] = new_cid;
3632                 new_cid += 2;
3633         }
3634         phba->avlbl_cids = phba->params.cxns_per_ctrl;
3635         return 0;
3636 }
3637
3638 static void hwi_enable_intr(struct beiscsi_hba *phba)
3639 {
3640         struct be_ctrl_info *ctrl = &phba->ctrl;
3641         struct hwi_controller *phwi_ctrlr;
3642         struct hwi_context_memory *phwi_context;
3643         struct be_queue_info *eq;
3644         u8 __iomem *addr;
3645         u32 reg, i;
3646         u32 enabled;
3647
3648         phwi_ctrlr = phba->phwi_ctrlr;
3649         phwi_context = phwi_ctrlr->phwi_ctxt;
3650
3651         addr = (u8 __iomem *) ((u8 __iomem *) ctrl->pcicfg +
3652                         PCICFG_MEMBAR_CTRL_INT_CTRL_OFFSET);
3653         reg = ioread32(addr);
3654
3655         enabled = reg & MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK;
3656         if (!enabled) {
3657                 reg |= MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK;
3658                 beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
3659                             "BM_%d : reg =x%08x addr=%p\n", reg, addr);
3660                 iowrite32(reg, addr);
3661         }
3662
3663         if (!phba->msix_enabled) {
3664                 eq = &phwi_context->be_eq[0].q;
3665                 beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
3666                             "BM_%d : eq->id=%d\n", eq->id);
3667
3668                 hwi_ring_eq_db(phba, eq->id, 0, 0, 1, 1);
3669         } else {
3670                 for (i = 0; i <= phba->num_cpus; i++) {
3671                         eq = &phwi_context->be_eq[i].q;
3672                         beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
3673                                     "BM_%d : eq->id=%d\n", eq->id);
3674                         hwi_ring_eq_db(phba, eq->id, 0, 0, 1, 1);
3675                 }
3676         }
3677 }
3678
3679 static void hwi_disable_intr(struct beiscsi_hba *phba)
3680 {
3681         struct be_ctrl_info *ctrl = &phba->ctrl;
3682
3683         u8 __iomem *addr = ctrl->pcicfg + PCICFG_MEMBAR_CTRL_INT_CTRL_OFFSET;
3684         u32 reg = ioread32(addr);
3685
3686         u32 enabled = reg & MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK;
3687         if (enabled) {
3688                 reg &= ~MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK;
3689                 iowrite32(reg, addr);
3690         } else
3691                 beiscsi_log(phba, KERN_WARNING, BEISCSI_LOG_INIT,
3692                             "BM_%d : In hwi_disable_intr, Already Disabled\n");
3693 }
3694
3695 /**
3696  * beiscsi_get_boot_info()- Get the boot session info
3697  * @phba: The device priv structure instance
3698  *
3699  * Get the boot target info and store in driver priv structure
3700  *
3701  * return values
3702  *      Success: 0
3703  *      Failure: Non-Zero Value
3704  **/
3705 static int beiscsi_get_boot_info(struct beiscsi_hba *phba)
3706 {
3707         struct be_cmd_get_session_resp *session_resp;
3708         struct be_mcc_wrb *wrb;
3709         struct be_dma_mem nonemb_cmd;
3710         unsigned int tag, wrb_num;
3711         unsigned short status, extd_status;
3712         unsigned int s_handle;
3713         struct be_queue_info *mccq = &phba->ctrl.mcc_obj.q;
3714         int ret = -ENOMEM;
3715
3716         /* Get the session handle of the boot target */
3717         ret = be_mgmt_get_boot_shandle(phba, &s_handle);
3718         if (ret) {
3719                 beiscsi_log(phba, KERN_ERR,
3720                             BEISCSI_LOG_INIT | BEISCSI_LOG_CONFIG,
3721                             "BM_%d : No boot session\n");
3722                 return ret;
3723         }
3724         nonemb_cmd.va = pci_alloc_consistent(phba->ctrl.pdev,
3725                                 sizeof(*session_resp),
3726                                 &nonemb_cmd.dma);
3727         if (nonemb_cmd.va == NULL) {
3728                 beiscsi_log(phba, KERN_ERR,
3729                             BEISCSI_LOG_INIT | BEISCSI_LOG_CONFIG,
3730                             "BM_%d : Failed to allocate memory for"
3731                             "beiscsi_get_session_info\n");
3732
3733                 return -ENOMEM;
3734         }
3735
3736         memset(nonemb_cmd.va, 0, sizeof(*session_resp));
3737         tag = mgmt_get_session_info(phba, s_handle,
3738                                     &nonemb_cmd);
3739         if (!tag) {
3740                 beiscsi_log(phba, KERN_ERR,
3741                             BEISCSI_LOG_INIT | BEISCSI_LOG_CONFIG,
3742                             "BM_%d : beiscsi_get_session_info"
3743                             " Failed\n");
3744
3745                 goto boot_freemem;
3746         } else
3747                 wait_event_interruptible(phba->ctrl.mcc_wait[tag],
3748                                          phba->ctrl.mcc_numtag[tag]);
3749
3750         wrb_num = (phba->ctrl.mcc_numtag[tag] & 0x00FF0000) >> 16;
3751         extd_status = (phba->ctrl.mcc_numtag[tag] & 0x0000FF00) >> 8;
3752         status = phba->ctrl.mcc_numtag[tag] & 0x000000FF;
3753         if (status || extd_status) {
3754                 beiscsi_log(phba, KERN_ERR,
3755                             BEISCSI_LOG_INIT | BEISCSI_LOG_CONFIG,
3756                             "BM_%d : beiscsi_get_session_info Failed"
3757                             " status = %d extd_status = %d\n",
3758                             status, extd_status);
3759
3760                 free_mcc_tag(&phba->ctrl, tag);
3761                 goto boot_freemem;
3762         }
3763         wrb = queue_get_wrb(mccq, wrb_num);
3764         free_mcc_tag(&phba->ctrl, tag);
3765         session_resp = nonemb_cmd.va ;
3766
3767         memcpy(&phba->boot_sess, &session_resp->session_info,
3768                sizeof(struct mgmt_session_info));
3769         ret = 0;
3770
3771 boot_freemem:
3772         pci_free_consistent(phba->ctrl.pdev, nonemb_cmd.size,
3773                     nonemb_cmd.va, nonemb_cmd.dma);
3774         return ret;
3775 }
3776
3777 static void beiscsi_boot_release(void *data)
3778 {
3779         struct beiscsi_hba *phba = data;
3780
3781         scsi_host_put(phba->shost);
3782 }
3783
3784 static int beiscsi_setup_boot_info(struct beiscsi_hba *phba)
3785 {
3786         struct iscsi_boot_kobj *boot_kobj;
3787
3788         /* get boot info using mgmt cmd */
3789         if (beiscsi_get_boot_info(phba))
3790                 /* Try to see if we can carry on without this */
3791                 return 0;
3792
3793         phba->boot_kset = iscsi_boot_create_host_kset(phba->shost->host_no);
3794         if (!phba->boot_kset)
3795                 return -ENOMEM;
3796
3797         /* get a ref because the show function will ref the phba */
3798         if (!scsi_host_get(phba->shost))
3799                 goto free_kset;
3800         boot_kobj = iscsi_boot_create_target(phba->boot_kset, 0, phba,
3801                                              beiscsi_show_boot_tgt_info,
3802                                              beiscsi_tgt_get_attr_visibility,
3803                                              beiscsi_boot_release);
3804         if (!boot_kobj)
3805                 goto put_shost;
3806
3807         if (!scsi_host_get(phba->shost))
3808                 goto free_kset;
3809         boot_kobj = iscsi_boot_create_initiator(phba->boot_kset, 0, phba,
3810                                                 beiscsi_show_boot_ini_info,
3811                                                 beiscsi_ini_get_attr_visibility,
3812                                                 beiscsi_boot_release);
3813         if (!boot_kobj)
3814                 goto put_shost;
3815
3816         if (!scsi_host_get(phba->shost))
3817                 goto free_kset;
3818         boot_kobj = iscsi_boot_create_ethernet(phba->boot_kset, 0, phba,
3819                                                beiscsi_show_boot_eth_info,
3820                                                beiscsi_eth_get_attr_visibility,
3821                                                beiscsi_boot_release);
3822         if (!boot_kobj)
3823                 goto put_shost;
3824         return 0;
3825
3826 put_shost:
3827         scsi_host_put(phba->shost);
3828 free_kset:
3829         iscsi_boot_destroy_kset(phba->boot_kset);
3830         return -ENOMEM;
3831 }
3832
3833 static int beiscsi_init_port(struct beiscsi_hba *phba)
3834 {
3835         int ret;
3836
3837         ret = beiscsi_init_controller(phba);
3838         if (ret < 0) {
3839                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3840                             "BM_%d : beiscsi_dev_probe - Failed in"
3841                             "beiscsi_init_controller\n");
3842                 return ret;
3843         }
3844         ret = beiscsi_init_sgl_handle(phba);
3845         if (ret < 0) {
3846                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3847                             "BM_%d : beiscsi_dev_probe - Failed in"
3848                             "beiscsi_init_sgl_handle\n");
3849                 goto do_cleanup_ctrlr;
3850         }
3851
3852         if (hba_setup_cid_tbls(phba)) {
3853                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3854                             "BM_%d : Failed in hba_setup_cid_tbls\n");
3855                 kfree(phba->io_sgl_hndl_base);
3856                 kfree(phba->eh_sgl_hndl_base);
3857                 goto do_cleanup_ctrlr;
3858         }
3859
3860         return ret;
3861
3862 do_cleanup_ctrlr:
3863         hwi_cleanup(phba);
3864         return ret;
3865 }
3866
3867 static void hwi_purge_eq(struct beiscsi_hba *phba)
3868 {
3869         struct hwi_controller *phwi_ctrlr;
3870         struct hwi_context_memory *phwi_context;
3871         struct be_queue_info *eq;
3872         struct be_eq_entry *eqe = NULL;
3873         int i, eq_msix;
3874         unsigned int num_processed;
3875
3876         phwi_ctrlr = phba->phwi_ctrlr;
3877         phwi_context = phwi_ctrlr->phwi_ctxt;
3878         if (phba->msix_enabled)
3879                 eq_msix = 1;
3880         else
3881                 eq_msix = 0;
3882
3883         for (i = 0; i < (phba->num_cpus + eq_msix); i++) {
3884                 eq = &phwi_context->be_eq[i].q;
3885                 eqe = queue_tail_node(eq);
3886                 num_processed = 0;
3887                 while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32]
3888                                         & EQE_VALID_MASK) {
3889                         AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0);
3890                         queue_tail_inc(eq);
3891                         eqe = queue_tail_node(eq);
3892                         num_processed++;
3893                 }
3894
3895                 if (num_processed)
3896                         hwi_ring_eq_db(phba, eq->id, 1, num_processed, 1, 1);
3897         }
3898 }
3899
3900 static void beiscsi_clean_port(struct beiscsi_hba *phba)
3901 {
3902         int mgmt_status;
3903
3904         mgmt_status = mgmt_epfw_cleanup(phba, CMD_CONNECTION_CHUTE_0);
3905         if (mgmt_status)
3906                 beiscsi_log(phba, KERN_WARNING, BEISCSI_LOG_INIT,
3907                             "BM_%d : mgmt_epfw_cleanup FAILED\n");
3908
3909         hwi_purge_eq(phba);
3910         hwi_cleanup(phba);
3911         kfree(phba->io_sgl_hndl_base);
3912         kfree(phba->eh_sgl_hndl_base);
3913         kfree(phba->cid_array);
3914         kfree(phba->ep_array);
3915 }
3916
3917 /**
3918  * beiscsi_cleanup_task()- Free driver resources of the task
3919  * @task: ptr to the iscsi task
3920  *
3921  **/
3922 static void beiscsi_cleanup_task(struct iscsi_task *task)
3923 {
3924         struct beiscsi_io_task *io_task = task->dd_data;
3925         struct iscsi_conn *conn = task->conn;
3926         struct beiscsi_conn *beiscsi_conn = conn->dd_data;
3927         struct beiscsi_hba *phba = beiscsi_conn->phba;
3928         struct beiscsi_session *beiscsi_sess = beiscsi_conn->beiscsi_sess;
3929         struct hwi_wrb_context *pwrb_context;
3930         struct hwi_controller *phwi_ctrlr;
3931
3932         phwi_ctrlr = phba->phwi_ctrlr;
3933         pwrb_context = &phwi_ctrlr->wrb_context[beiscsi_conn->beiscsi_conn_cid
3934                         - phba->fw_config.iscsi_cid_start];
3935
3936         if (io_task->cmd_bhs) {
3937                 pci_pool_free(beiscsi_sess->bhs_pool, io_task->cmd_bhs,
3938                               io_task->bhs_pa.u.a64.address);
3939                 io_task->cmd_bhs = NULL;
3940         }
3941
3942         if (task->sc) {
3943                 if (io_task->pwrb_handle) {
3944                         free_wrb_handle(phba, pwrb_context,
3945                                         io_task->pwrb_handle);
3946                         io_task->pwrb_handle = NULL;
3947                 }
3948
3949                 if (io_task->psgl_handle) {
3950                         spin_lock(&phba->io_sgl_lock);
3951                         free_io_sgl_handle(phba, io_task->psgl_handle);
3952                         spin_unlock(&phba->io_sgl_lock);
3953                         io_task->psgl_handle = NULL;
3954                 }
3955         } else {
3956                 if (!beiscsi_conn->login_in_progress) {
3957                         if (io_task->pwrb_handle) {
3958                                 free_wrb_handle(phba, pwrb_context,
3959                                                 io_task->pwrb_handle);
3960                                 io_task->pwrb_handle = NULL;
3961                         }
3962                         if (io_task->psgl_handle) {
3963                                 spin_lock(&phba->mgmt_sgl_lock);
3964                                 free_mgmt_sgl_handle(phba,
3965                                                      io_task->psgl_handle);
3966                                 spin_unlock(&phba->mgmt_sgl_lock);
3967                                 io_task->psgl_handle = NULL;
3968                         }
3969                         if (io_task->mtask_addr) {
3970                                 pci_unmap_single(phba->pcidev,
3971                                                  io_task->mtask_addr,
3972                                                  io_task->mtask_data_count,
3973                                                  PCI_DMA_TODEVICE);
3974                                 io_task->mtask_addr = 0;
3975                         }
3976                 }
3977         }
3978 }
3979
3980 void
3981 beiscsi_offload_connection(struct beiscsi_conn *beiscsi_conn,
3982                            struct beiscsi_offload_params *params)
3983 {
3984         struct wrb_handle *pwrb_handle;
3985         struct iscsi_target_context_update_wrb *pwrb = NULL;
3986         struct be_mem_descriptor *mem_descr;
3987         struct beiscsi_hba *phba = beiscsi_conn->phba;
3988         struct iscsi_task *task = beiscsi_conn->task;
3989         struct iscsi_session *session = task->conn->session;
3990         u32 doorbell = 0;
3991
3992         /*
3993          * We can always use 0 here because it is reserved by libiscsi for
3994          * login/startup related tasks.
3995          */
3996         beiscsi_conn->login_in_progress = 0;
3997         spin_lock_bh(&session->lock);
3998         beiscsi_cleanup_task(task);
3999         spin_unlock_bh(&session->lock);
4000
4001         pwrb_handle = alloc_wrb_handle(phba, (beiscsi_conn->beiscsi_conn_cid -
4002                                        phba->fw_config.iscsi_cid_start));
4003         pwrb = (struct iscsi_target_context_update_wrb *)pwrb_handle->pwrb;
4004         memset(pwrb, 0, sizeof(*pwrb));
4005         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb,
4006                       max_burst_length, pwrb, params->dw[offsetof
4007                       (struct amap_beiscsi_offload_params,
4008                       max_burst_length) / 32]);
4009         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb,
4010                       max_send_data_segment_length, pwrb,
4011                       params->dw[offsetof(struct amap_beiscsi_offload_params,
4012                       max_send_data_segment_length) / 32]);
4013         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb,
4014                       first_burst_length,
4015                       pwrb,
4016                       params->dw[offsetof(struct amap_beiscsi_offload_params,
4017                       first_burst_length) / 32]);
4018
4019         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, erl, pwrb,
4020                       (params->dw[offsetof(struct amap_beiscsi_offload_params,
4021                       erl) / 32] & OFFLD_PARAMS_ERL));
4022         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, dde, pwrb,
4023                       (params->dw[offsetof(struct amap_beiscsi_offload_params,
4024                       dde) / 32] & OFFLD_PARAMS_DDE) >> 2);
4025         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, hde, pwrb,
4026                       (params->dw[offsetof(struct amap_beiscsi_offload_params,
4027                       hde) / 32] & OFFLD_PARAMS_HDE) >> 3);
4028         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, ir2t, pwrb,
4029                       (params->dw[offsetof(struct amap_beiscsi_offload_params,
4030                       ir2t) / 32] & OFFLD_PARAMS_IR2T) >> 4);
4031         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, imd, pwrb,
4032                       (params->dw[offsetof(struct amap_beiscsi_offload_params,
4033                        imd) / 32] & OFFLD_PARAMS_IMD) >> 5);
4034         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, stat_sn,
4035                       pwrb,
4036                       (params->dw[offsetof(struct amap_beiscsi_offload_params,
4037                       exp_statsn) / 32] + 1));
4038         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, type, pwrb,
4039                       0x7);
4040         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, wrb_idx,
4041                       pwrb, pwrb_handle->wrb_index);
4042         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, ptr2nextwrb,
4043                       pwrb, pwrb_handle->nxt_wrb_index);
4044         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb,
4045                         session_state, pwrb, 0);
4046         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, compltonack,
4047                       pwrb, 1);
4048         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, notpredblq,
4049                       pwrb, 0);
4050         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb, mode, pwrb,
4051                       0);
4052
4053         mem_descr = phba->init_mem;
4054         mem_descr += ISCSI_MEM_GLOBAL_HEADER;
4055
4056         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb,
4057                         pad_buffer_addr_hi, pwrb,
4058                       mem_descr->mem_array[0].bus_address.u.a32.address_hi);
4059         AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb,
4060                         pad_buffer_addr_lo, pwrb,
4061                       mem_descr->mem_array[0].bus_address.u.a32.address_lo);
4062
4063         be_dws_le_to_cpu(pwrb, sizeof(struct iscsi_target_context_update_wrb));
4064
4065         doorbell |= beiscsi_conn->beiscsi_conn_cid & DB_WRB_POST_CID_MASK;
4066         doorbell |= (pwrb_handle->wrb_index & DB_DEF_PDU_WRB_INDEX_MASK)
4067                              << DB_DEF_PDU_WRB_INDEX_SHIFT;
4068         doorbell |= 1 << DB_DEF_PDU_NUM_POSTED_SHIFT;
4069
4070         iowrite32(doorbell, phba->db_va + DB_TXULP0_OFFSET);
4071 }
4072
4073 static void beiscsi_parse_pdu(struct iscsi_conn *conn, itt_t itt,
4074                               int *index, int *age)
4075 {
4076         *index = (int)itt;
4077         if (age)
4078                 *age = conn->session->age;
4079 }
4080
4081 /**
4082  * beiscsi_alloc_pdu - allocates pdu and related resources
4083  * @task: libiscsi task
4084  * @opcode: opcode of pdu for task
4085  *
4086  * This is called with the session lock held. It will allocate
4087  * the wrb and sgl if needed for the command. And it will prep
4088  * the pdu's itt. beiscsi_parse_pdu will later translate
4089  * the pdu itt to the libiscsi task itt.
4090  */
4091 static int beiscsi_alloc_pdu(struct iscsi_task *task, uint8_t opcode)
4092 {
4093         struct beiscsi_io_task *io_task = task->dd_data;
4094         struct iscsi_conn *conn = task->conn;
4095         struct beiscsi_conn *beiscsi_conn = conn->dd_data;
4096         struct beiscsi_hba *phba = beiscsi_conn->phba;
4097         struct hwi_wrb_context *pwrb_context;
4098         struct hwi_controller *phwi_ctrlr;
4099         itt_t itt;
4100         struct beiscsi_session *beiscsi_sess = beiscsi_conn->beiscsi_sess;
4101         dma_addr_t paddr;
4102
4103         io_task->cmd_bhs = pci_pool_alloc(beiscsi_sess->bhs_pool,
4104                                           GFP_ATOMIC, &paddr);
4105         if (!io_task->cmd_bhs)
4106                 return -ENOMEM;
4107         io_task->bhs_pa.u.a64.address = paddr;
4108         io_task->libiscsi_itt = (itt_t)task->itt;
4109         io_task->conn = beiscsi_conn;
4110
4111         task->hdr = (struct iscsi_hdr *)&io_task->cmd_bhs->iscsi_hdr;
4112         task->hdr_max = sizeof(struct be_cmd_bhs);
4113         io_task->psgl_handle = NULL;
4114         io_task->pwrb_handle = NULL;
4115
4116         if (task->sc) {
4117                 spin_lock(&phba->io_sgl_lock);
4118                 io_task->psgl_handle = alloc_io_sgl_handle(phba);
4119                 spin_unlock(&phba->io_sgl_lock);
4120                 if (!io_task->psgl_handle) {
4121                         beiscsi_log(phba, KERN_ERR,
4122                                     BEISCSI_LOG_IO | BEISCSI_LOG_CONFIG,
4123                                     "BM_%d : Alloc of IO_SGL_ICD Failed"
4124                                     "for the CID : %d\n",
4125                                     beiscsi_conn->beiscsi_conn_cid);
4126                         goto free_hndls;
4127                 }
4128                 io_task->pwrb_handle = alloc_wrb_handle(phba,
4129                                         beiscsi_conn->beiscsi_conn_cid -
4130                                         phba->fw_config.iscsi_cid_start);
4131                 if (!io_task->pwrb_handle) {
4132                         beiscsi_log(phba, KERN_ERR,
4133                                     BEISCSI_LOG_IO | BEISCSI_LOG_CONFIG,
4134                                     "BM_%d : Alloc of WRB_HANDLE Failed"
4135                                     "for the CID : %d\n",
4136                                     beiscsi_conn->beiscsi_conn_cid);
4137                         goto free_io_hndls;
4138                 }
4139         } else {
4140                 io_task->scsi_cmnd = NULL;
4141                 if ((opcode & ISCSI_OPCODE_MASK) == ISCSI_OP_LOGIN) {
4142                         if (!beiscsi_conn->login_in_progress) {
4143                                 spin_lock(&phba->mgmt_sgl_lock);
4144                                 io_task->psgl_handle = (struct sgl_handle *)
4145                                                 alloc_mgmt_sgl_handle(phba);
4146                                 spin_unlock(&phba->mgmt_sgl_lock);
4147                                 if (!io_task->psgl_handle) {
4148                                         beiscsi_log(phba, KERN_ERR,
4149                                                     BEISCSI_LOG_IO |
4150                                                     BEISCSI_LOG_CONFIG,
4151                                                     "BM_%d : Alloc of MGMT_SGL_ICD Failed"
4152                                                     "for the CID : %d\n",
4153                                                     beiscsi_conn->
4154                                                     beiscsi_conn_cid);
4155                                         goto free_hndls;
4156                                 }
4157
4158                                 beiscsi_conn->login_in_progress = 1;
4159                                 beiscsi_conn->plogin_sgl_handle =
4160                                                         io_task->psgl_handle;
4161                                 io_task->pwrb_handle =
4162                                         alloc_wrb_handle(phba,
4163                                         beiscsi_conn->beiscsi_conn_cid -
4164                                         phba->fw_config.iscsi_cid_start);
4165                                 if (!io_task->pwrb_handle) {
4166                                         beiscsi_log(phba, KERN_ERR,
4167                                                     BEISCSI_LOG_IO |
4168                                                     BEISCSI_LOG_CONFIG,
4169                                                     "BM_%d : Alloc of WRB_HANDLE Failed"
4170                                                     "for the CID : %d\n",
4171                                                     beiscsi_conn->
4172                                                     beiscsi_conn_cid);
4173                                         goto free_mgmt_hndls;
4174                                 }
4175                                 beiscsi_conn->plogin_wrb_handle =
4176                                                         io_task->pwrb_handle;
4177
4178                         } else {
4179                                 io_task->psgl_handle =
4180                                                 beiscsi_conn->plogin_sgl_handle;
4181                                 io_task->pwrb_handle =
4182                                                 beiscsi_conn->plogin_wrb_handle;
4183                         }
4184                         beiscsi_conn->task = task;
4185                 } else {
4186                         spin_lock(&phba->mgmt_sgl_lock);
4187                         io_task->psgl_handle = alloc_mgmt_sgl_handle(phba);
4188                         spin_unlock(&phba->mgmt_sgl_lock);
4189                         if (!io_task->psgl_handle) {
4190                                 beiscsi_log(phba, KERN_ERR,
4191                                             BEISCSI_LOG_IO |
4192                                             BEISCSI_LOG_CONFIG,
4193                                             "BM_%d : Alloc of MGMT_SGL_ICD Failed"
4194                                             "for the CID : %d\n",
4195                                             beiscsi_conn->
4196                                             beiscsi_conn_cid);
4197                                 goto free_hndls;
4198                         }
4199                         io_task->pwrb_handle =
4200                                         alloc_wrb_handle(phba,
4201                                         beiscsi_conn->beiscsi_conn_cid -
4202                                         phba->fw_config.iscsi_cid_start);
4203                         if (!io_task->pwrb_handle) {
4204                                 beiscsi_log(phba, KERN_ERR,
4205                                             BEISCSI_LOG_IO | BEISCSI_LOG_CONFIG,
4206                                             "BM_%d : Alloc of WRB_HANDLE Failed"
4207                                             "for the CID : %d\n",
4208                                             beiscsi_conn->beiscsi_conn_cid);
4209                                 goto free_mgmt_hndls;
4210                         }
4211
4212                 }
4213         }
4214         itt = (itt_t) cpu_to_be32(((unsigned int)io_task->pwrb_handle->
4215                                  wrb_index << 16) | (unsigned int)
4216                                 (io_task->psgl_handle->sgl_index));
4217         io_task->pwrb_handle->pio_handle = task;
4218
4219         io_task->cmd_bhs->iscsi_hdr.itt = itt;
4220         return 0;
4221
4222 free_io_hndls:
4223         spin_lock(&phba->io_sgl_lock);
4224         free_io_sgl_handle(phba, io_task->psgl_handle);
4225         spin_unlock(&phba->io_sgl_lock);
4226         goto free_hndls;
4227 free_mgmt_hndls:
4228         spin_lock(&phba->mgmt_sgl_lock);
4229         free_mgmt_sgl_handle(phba, io_task->psgl_handle);
4230         spin_unlock(&phba->mgmt_sgl_lock);
4231 free_hndls:
4232         phwi_ctrlr = phba->phwi_ctrlr;
4233         pwrb_context = &phwi_ctrlr->wrb_context[
4234                         beiscsi_conn->beiscsi_conn_cid -
4235                         phba->fw_config.iscsi_cid_start];
4236         if (io_task->pwrb_handle)
4237                 free_wrb_handle(phba, pwrb_context, io_task->pwrb_handle);
4238         io_task->pwrb_handle = NULL;
4239         pci_pool_free(beiscsi_sess->bhs_pool, io_task->cmd_bhs,
4240                       io_task->bhs_pa.u.a64.address);
4241         io_task->cmd_bhs = NULL;
4242         return -ENOMEM;
4243 }
4244
4245 static int beiscsi_iotask(struct iscsi_task *task, struct scatterlist *sg,
4246                           unsigned int num_sg, unsigned int xferlen,
4247                           unsigned int writedir)
4248 {
4249
4250         struct beiscsi_io_task *io_task = task->dd_data;
4251         struct iscsi_conn *conn = task->conn;
4252         struct beiscsi_conn *beiscsi_conn = conn->dd_data;
4253         struct beiscsi_hba *phba = beiscsi_conn->phba;
4254         struct iscsi_wrb *pwrb = NULL;
4255         unsigned int doorbell = 0;
4256
4257         pwrb = io_task->pwrb_handle->pwrb;
4258         io_task->cmd_bhs->iscsi_hdr.exp_statsn = 0;
4259         io_task->bhs_len = sizeof(struct be_cmd_bhs);
4260
4261         if (writedir) {
4262                 AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb,
4263                               INI_WR_CMD);
4264                 AMAP_SET_BITS(struct amap_iscsi_wrb, dsp, pwrb, 1);
4265         } else {
4266                 AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb,
4267                               INI_RD_CMD);
4268                 AMAP_SET_BITS(struct amap_iscsi_wrb, dsp, pwrb, 0);
4269         }
4270
4271         AMAP_SET_BITS(struct amap_iscsi_wrb, lun, pwrb,
4272                       cpu_to_be16(*(unsigned short *)
4273                                   &io_task->cmd_bhs->iscsi_hdr.lun));
4274         AMAP_SET_BITS(struct amap_iscsi_wrb, r2t_exp_dtl, pwrb, xferlen);
4275         AMAP_SET_BITS(struct amap_iscsi_wrb, wrb_idx, pwrb,
4276                       io_task->pwrb_handle->wrb_index);
4277         AMAP_SET_BITS(struct amap_iscsi_wrb, cmdsn_itt, pwrb,
4278                       be32_to_cpu(task->cmdsn));
4279         AMAP_SET_BITS(struct amap_iscsi_wrb, sgl_icd_idx, pwrb,
4280                       io_task->psgl_handle->sgl_index);
4281
4282         hwi_write_sgl(pwrb, sg, num_sg, io_task);
4283
4284         AMAP_SET_BITS(struct amap_iscsi_wrb, ptr2nextwrb, pwrb,
4285                       io_task->pwrb_handle->nxt_wrb_index);
4286         be_dws_le_to_cpu(pwrb, sizeof(struct iscsi_wrb));
4287
4288         doorbell |= beiscsi_conn->beiscsi_conn_cid & DB_WRB_POST_CID_MASK;
4289         doorbell |= (io_task->pwrb_handle->wrb_index &
4290                      DB_DEF_PDU_WRB_INDEX_MASK) << DB_DEF_PDU_WRB_INDEX_SHIFT;
4291         doorbell |= 1 << DB_DEF_PDU_NUM_POSTED_SHIFT;
4292
4293         iowrite32(doorbell, phba->db_va + DB_TXULP0_OFFSET);
4294         return 0;
4295 }
4296
4297 static int beiscsi_mtask(struct iscsi_task *task)
4298 {
4299         struct beiscsi_io_task *io_task = task->dd_data;
4300         struct iscsi_conn *conn = task->conn;
4301         struct beiscsi_conn *beiscsi_conn = conn->dd_data;
4302         struct beiscsi_hba *phba = beiscsi_conn->phba;
4303         struct iscsi_wrb *pwrb = NULL;
4304         unsigned int doorbell = 0;
4305         unsigned int cid;
4306
4307         cid = beiscsi_conn->beiscsi_conn_cid;
4308         pwrb = io_task->pwrb_handle->pwrb;
4309         memset(pwrb, 0, sizeof(*pwrb));
4310         AMAP_SET_BITS(struct amap_iscsi_wrb, cmdsn_itt, pwrb,
4311                       be32_to_cpu(task->cmdsn));
4312         AMAP_SET_BITS(struct amap_iscsi_wrb, wrb_idx, pwrb,
4313                       io_task->pwrb_handle->wrb_index);
4314         AMAP_SET_BITS(struct amap_iscsi_wrb, sgl_icd_idx, pwrb,
4315                       io_task->psgl_handle->sgl_index);
4316
4317         switch (task->hdr->opcode & ISCSI_OPCODE_MASK) {
4318         case ISCSI_OP_LOGIN:
4319                 AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb,
4320                               TGT_DM_CMD);
4321                 AMAP_SET_BITS(struct amap_iscsi_wrb, dmsg, pwrb, 0);
4322                 AMAP_SET_BITS(struct amap_iscsi_wrb, cmdsn_itt, pwrb, 1);
4323                 hwi_write_buffer(pwrb, task);
4324                 break;
4325         case ISCSI_OP_NOOP_OUT:
4326                 if (task->hdr->ttt != ISCSI_RESERVED_TAG) {
4327                         AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb,
4328                                       TGT_DM_CMD);
4329                         AMAP_SET_BITS(struct amap_iscsi_wrb, cmdsn_itt,
4330                                       pwrb, 0);
4331                         AMAP_SET_BITS(struct amap_iscsi_wrb, dmsg, pwrb, 1);
4332                 } else {
4333                         AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb,
4334                                       INI_RD_CMD);
4335                         AMAP_SET_BITS(struct amap_iscsi_wrb, dmsg, pwrb, 0);
4336                 }
4337                 hwi_write_buffer(pwrb, task);
4338                 break;
4339         case ISCSI_OP_TEXT:
4340                 AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb,
4341                               TGT_DM_CMD);
4342                 AMAP_SET_BITS(struct amap_iscsi_wrb, dmsg, pwrb, 0);
4343                 hwi_write_buffer(pwrb, task);
4344                 break;
4345         case ISCSI_OP_SCSI_TMFUNC:
4346                 AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb,
4347                               INI_TMF_CMD);
4348                 AMAP_SET_BITS(struct amap_iscsi_wrb, dmsg, pwrb, 0);
4349                 hwi_write_buffer(pwrb, task);
4350                 break;
4351         case ISCSI_OP_LOGOUT:
4352                 AMAP_SET_BITS(struct amap_iscsi_wrb, dmsg, pwrb, 0);
4353                 AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb,
4354                               HWH_TYPE_LOGOUT);
4355                 hwi_write_buffer(pwrb, task);
4356                 break;
4357
4358         default:
4359                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_CONFIG,
4360                             "BM_%d : opcode =%d Not supported\n",
4361                             task->hdr->opcode & ISCSI_OPCODE_MASK);
4362
4363                 return -EINVAL;
4364         }
4365
4366         AMAP_SET_BITS(struct amap_iscsi_wrb, r2t_exp_dtl, pwrb,
4367                       task->data_count);
4368         AMAP_SET_BITS(struct amap_iscsi_wrb, ptr2nextwrb, pwrb,
4369                       io_task->pwrb_handle->nxt_wrb_index);
4370         be_dws_le_to_cpu(pwrb, sizeof(struct iscsi_wrb));
4371
4372         doorbell |= cid & DB_WRB_POST_CID_MASK;
4373         doorbell |= (io_task->pwrb_handle->wrb_index &
4374                      DB_DEF_PDU_WRB_INDEX_MASK) << DB_DEF_PDU_WRB_INDEX_SHIFT;
4375         doorbell |= 1 << DB_DEF_PDU_NUM_POSTED_SHIFT;
4376         iowrite32(doorbell, phba->db_va + DB_TXULP0_OFFSET);
4377         return 0;
4378 }
4379
4380 static int beiscsi_task_xmit(struct iscsi_task *task)
4381 {
4382         struct beiscsi_io_task *io_task = task->dd_data;
4383         struct scsi_cmnd *sc = task->sc;
4384         struct scatterlist *sg;
4385         int num_sg;
4386         unsigned int  writedir = 0, xferlen = 0;
4387
4388         if (!sc)
4389                 return beiscsi_mtask(task);
4390
4391         io_task->scsi_cmnd = sc;
4392         num_sg = scsi_dma_map(sc);
4393         if (num_sg < 0) {
4394                 struct iscsi_conn *conn = task->conn;
4395                 struct beiscsi_hba *phba = NULL;
4396
4397                 phba = ((struct beiscsi_conn *)conn->dd_data)->phba;
4398                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_IO,
4399                             "BM_%d : scsi_dma_map Failed\n");
4400
4401                 return num_sg;
4402         }
4403         xferlen = scsi_bufflen(sc);
4404         sg = scsi_sglist(sc);
4405         if (sc->sc_data_direction == DMA_TO_DEVICE)
4406                 writedir = 1;
4407          else
4408                 writedir = 0;
4409
4410         return beiscsi_iotask(task, sg, num_sg, xferlen, writedir);
4411 }
4412
4413 /**
4414  * beiscsi_bsg_request - handle bsg request from ISCSI transport
4415  * @job: job to handle
4416  */
4417 static int beiscsi_bsg_request(struct bsg_job *job)
4418 {
4419         struct Scsi_Host *shost;
4420         struct beiscsi_hba *phba;
4421         struct iscsi_bsg_request *bsg_req = job->request;
4422         int rc = -EINVAL;
4423         unsigned int tag;
4424         struct be_dma_mem nonemb_cmd;
4425         struct be_cmd_resp_hdr *resp;
4426         struct iscsi_bsg_reply *bsg_reply = job->reply;
4427         unsigned short status, extd_status;
4428
4429         shost = iscsi_job_to_shost(job);
4430         phba = iscsi_host_priv(shost);
4431
4432         switch (bsg_req->msgcode) {
4433         case ISCSI_BSG_HST_VENDOR:
4434                 nonemb_cmd.va = pci_alloc_consistent(phba->ctrl.pdev,
4435                                         job->request_payload.payload_len,
4436                                         &nonemb_cmd.dma);
4437                 if (nonemb_cmd.va == NULL) {
4438                         beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_CONFIG,
4439                                     "BM_%d : Failed to allocate memory for "
4440                                     "beiscsi_bsg_request\n");
4441                         return -ENOMEM;
4442                 }
4443                 tag = mgmt_vendor_specific_fw_cmd(&phba->ctrl, phba, job,
4444                                                   &nonemb_cmd);
4445                 if (!tag) {
4446                         beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_CONFIG,
4447                                     "BM_%d : MBX Tag Allocation Failed\n");
4448
4449                         pci_free_consistent(phba->ctrl.pdev, nonemb_cmd.size,
4450                                             nonemb_cmd.va, nonemb_cmd.dma);
4451                         return -EAGAIN;
4452                 } else
4453                         wait_event_interruptible(phba->ctrl.mcc_wait[tag],
4454                                                  phba->ctrl.mcc_numtag[tag]);
4455                 extd_status = (phba->ctrl.mcc_numtag[tag] & 0x0000FF00) >> 8;
4456                 status = phba->ctrl.mcc_numtag[tag] & 0x000000FF;
4457                 free_mcc_tag(&phba->ctrl, tag);
4458                 resp = (struct be_cmd_resp_hdr *)nonemb_cmd.va;
4459                 sg_copy_from_buffer(job->reply_payload.sg_list,
4460                                     job->reply_payload.sg_cnt,
4461                                     nonemb_cmd.va, (resp->response_length
4462                                     + sizeof(*resp)));
4463                 bsg_reply->reply_payload_rcv_len = resp->response_length;
4464                 bsg_reply->result = status;
4465                 bsg_job_done(job, bsg_reply->result,
4466                              bsg_reply->reply_payload_rcv_len);
4467                 pci_free_consistent(phba->ctrl.pdev, nonemb_cmd.size,
4468                                     nonemb_cmd.va, nonemb_cmd.dma);
4469                 if (status || extd_status) {
4470                         beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_CONFIG,
4471                                     "BM_%d : MBX Cmd Failed"
4472                                     " status = %d extd_status = %d\n",
4473                                     status, extd_status);
4474
4475                         return -EIO;
4476                 } else {
4477                         rc = 0;
4478                 }
4479                 break;
4480
4481         default:
4482                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_CONFIG,
4483                                 "BM_%d : Unsupported bsg command: 0x%x\n",
4484                                 bsg_req->msgcode);
4485                 break;
4486         }
4487
4488         return rc;
4489 }
4490
4491 void beiscsi_hba_attrs_init(struct beiscsi_hba *phba)
4492 {
4493         /* Set the logging parameter */
4494         beiscsi_log_enable_init(phba, beiscsi_log_enable);
4495 }
4496
4497 /*
4498  * beiscsi_quiesce()- Cleanup Driver resources
4499  * @phba: Instance Priv structure
4500  *
4501  * Free the OS and HW resources held by the driver
4502  **/
4503 static void beiscsi_quiesce(struct beiscsi_hba *phba)
4504 {
4505         struct hwi_controller *phwi_ctrlr;
4506         struct hwi_context_memory *phwi_context;
4507         struct be_eq_obj *pbe_eq;
4508         unsigned int i, msix_vec;
4509
4510         phwi_ctrlr = phba->phwi_ctrlr;
4511         phwi_context = phwi_ctrlr->phwi_ctxt;
4512         hwi_disable_intr(phba);
4513         if (phba->msix_enabled) {
4514                 for (i = 0; i <= phba->num_cpus; i++) {
4515                         msix_vec = phba->msix_entries[i].vector;
4516                         free_irq(msix_vec, &phwi_context->be_eq[i]);
4517                         kfree(phba->msi_name[i]);
4518                 }
4519         } else
4520                 if (phba->pcidev->irq)
4521                         free_irq(phba->pcidev->irq, phba);
4522         pci_disable_msix(phba->pcidev);
4523         destroy_workqueue(phba->wq);
4524         if (blk_iopoll_enabled)
4525                 for (i = 0; i < phba->num_cpus; i++) {
4526                         pbe_eq = &phwi_context->be_eq[i];
4527                         blk_iopoll_disable(&pbe_eq->iopoll);
4528                 }
4529
4530         beiscsi_clean_port(phba);
4531         beiscsi_free_mem(phba);
4532
4533         beiscsi_unmap_pci_function(phba);
4534         pci_free_consistent(phba->pcidev,
4535                             phba->ctrl.mbox_mem_alloced.size,
4536                             phba->ctrl.mbox_mem_alloced.va,
4537                             phba->ctrl.mbox_mem_alloced.dma);
4538 }
4539
4540 static void beiscsi_remove(struct pci_dev *pcidev)
4541 {
4542
4543         struct beiscsi_hba *phba = NULL;
4544
4545         phba = pci_get_drvdata(pcidev);
4546         if (!phba) {
4547                 dev_err(&pcidev->dev, "beiscsi_remove called with no phba\n");
4548                 return;
4549         }
4550
4551         beiscsi_destroy_def_ifaces(phba);
4552         beiscsi_quiesce(phba);
4553         iscsi_boot_destroy_kset(phba->boot_kset);
4554         iscsi_host_remove(phba->shost);
4555         pci_dev_put(phba->pcidev);
4556         iscsi_host_free(phba->shost);
4557         pci_disable_device(pcidev);
4558 }
4559
4560 static void beiscsi_shutdown(struct pci_dev *pcidev)
4561 {
4562
4563         struct beiscsi_hba *phba = NULL;
4564
4565         phba = (struct beiscsi_hba *)pci_get_drvdata(pcidev);
4566         if (!phba) {
4567                 dev_err(&pcidev->dev, "beiscsi_shutdown called with no phba\n");
4568                 return;
4569         }
4570
4571         beiscsi_quiesce(phba);
4572         pci_disable_device(pcidev);
4573 }
4574
4575 static void beiscsi_msix_enable(struct beiscsi_hba *phba)
4576 {
4577         int i, status;
4578
4579         for (i = 0; i <= phba->num_cpus; i++)
4580                 phba->msix_entries[i].entry = i;
4581
4582         status = pci_enable_msix(phba->pcidev, phba->msix_entries,
4583                                  (phba->num_cpus + 1));
4584         if (!status)
4585                 phba->msix_enabled = true;
4586
4587         return;
4588 }
4589
4590 static int __devinit beiscsi_dev_probe(struct pci_dev *pcidev,
4591                                 const struct pci_device_id *id)
4592 {
4593         struct beiscsi_hba *phba = NULL;
4594         struct hwi_controller *phwi_ctrlr;
4595         struct hwi_context_memory *phwi_context;
4596         struct be_eq_obj *pbe_eq;
4597         int ret, i;
4598
4599         ret = beiscsi_enable_pci(pcidev);
4600         if (ret < 0) {
4601                 dev_err(&pcidev->dev,
4602                         "beiscsi_dev_probe - Failed to enable pci device\n");
4603                 return ret;
4604         }
4605
4606         phba = beiscsi_hba_alloc(pcidev);
4607         if (!phba) {
4608                 dev_err(&pcidev->dev,
4609                         "beiscsi_dev_probe - Failed in beiscsi_hba_alloc\n");
4610                 goto disable_pci;
4611         }
4612
4613         /* Initialize Driver configuration Paramters */
4614         beiscsi_hba_attrs_init(phba);
4615
4616         switch (pcidev->device) {
4617         case BE_DEVICE_ID1:
4618         case OC_DEVICE_ID1:
4619         case OC_DEVICE_ID2:
4620                 phba->generation = BE_GEN2;
4621                 break;
4622         case BE_DEVICE_ID2:
4623         case OC_DEVICE_ID3:
4624                 phba->generation = BE_GEN3;
4625                 break;
4626         case OC_SKH_ID1:
4627                 phba->generation = BE_GEN4;
4628         default:
4629                 phba->generation = 0;
4630         }
4631
4632         if (enable_msix)
4633                 find_num_cpus(phba);
4634         else
4635                 phba->num_cpus = 1;
4636
4637         beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
4638                     "BM_%d : num_cpus = %d\n",
4639                     phba->num_cpus);
4640
4641         if (enable_msix) {
4642                 beiscsi_msix_enable(phba);
4643                 if (!phba->msix_enabled)
4644                         phba->num_cpus = 1;
4645         }
4646         ret = be_ctrl_init(phba, pcidev);
4647         if (ret) {
4648                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
4649                             "BM_%d : beiscsi_dev_probe-"
4650                             "Failed in be_ctrl_init\n");
4651                 goto hba_free;
4652         }
4653
4654         ret = beiscsi_cmd_reset_function(phba);
4655         if (ret) {
4656                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
4657                             "BM_%d : Reset Failed. Aborting Crashdump\n");
4658                 goto hba_free;
4659         }
4660         ret = be_chk_reset_complete(phba);
4661         if (ret) {
4662                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
4663                             "BM_%d : Failed to get out of reset."
4664                             "Aborting Crashdump\n");
4665                 goto hba_free;
4666         }
4667
4668         spin_lock_init(&phba->io_sgl_lock);
4669         spin_lock_init(&phba->mgmt_sgl_lock);
4670         spin_lock_init(&phba->isr_lock);
4671         ret = mgmt_get_fw_config(&phba->ctrl, phba);
4672         if (ret != 0) {
4673                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
4674                             "BM_%d : Error getting fw config\n");
4675                 goto free_port;
4676         }
4677         phba->shost->max_id = phba->fw_config.iscsi_cid_count;
4678         beiscsi_get_params(phba);
4679         phba->shost->can_queue = phba->params.ios_per_ctrl;
4680         ret = beiscsi_init_port(phba);
4681         if (ret < 0) {
4682                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
4683                             "BM_%d : beiscsi_dev_probe-"
4684                             "Failed in beiscsi_init_port\n");
4685                 goto free_port;
4686         }
4687
4688         for (i = 0; i < MAX_MCC_CMD ; i++) {
4689                 init_waitqueue_head(&phba->ctrl.mcc_wait[i + 1]);
4690                 phba->ctrl.mcc_tag[i] = i + 1;
4691                 phba->ctrl.mcc_numtag[i + 1] = 0;
4692                 phba->ctrl.mcc_tag_available++;
4693         }
4694
4695         phba->ctrl.mcc_alloc_index = phba->ctrl.mcc_free_index = 0;
4696
4697         snprintf(phba->wq_name, sizeof(phba->wq_name), "beiscsi_%02x_wq",
4698                  phba->shost->host_no);
4699         phba->wq = alloc_workqueue(phba->wq_name, WQ_MEM_RECLAIM, 1);
4700         if (!phba->wq) {
4701                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
4702                             "BM_%d : beiscsi_dev_probe-"
4703                             "Failed to allocate work queue\n");
4704                 goto free_twq;
4705         }
4706
4707
4708         phwi_ctrlr = phba->phwi_ctrlr;
4709         phwi_context = phwi_ctrlr->phwi_ctxt;
4710
4711         if (blk_iopoll_enabled) {
4712                 for (i = 0; i < phba->num_cpus; i++) {
4713                         pbe_eq = &phwi_context->be_eq[i];
4714                         blk_iopoll_init(&pbe_eq->iopoll, be_iopoll_budget,
4715                                         be_iopoll);
4716                         blk_iopoll_enable(&pbe_eq->iopoll);
4717                 }
4718
4719                 i = (phba->msix_enabled) ? i : 0;
4720                 /* Work item for MCC handling */
4721                 pbe_eq = &phwi_context->be_eq[i];
4722                 INIT_WORK(&pbe_eq->work_cqs, beiscsi_process_all_cqs);
4723         } else {
4724                 if (phba->msix_enabled) {
4725                         for (i = 0; i <= phba->num_cpus; i++) {
4726                                 pbe_eq = &phwi_context->be_eq[i];
4727                                 INIT_WORK(&pbe_eq->work_cqs,
4728                                           beiscsi_process_all_cqs);
4729                         }
4730                 } else {
4731                                 pbe_eq = &phwi_context->be_eq[0];
4732                                 INIT_WORK(&pbe_eq->work_cqs,
4733                                           beiscsi_process_all_cqs);
4734                         }
4735         }
4736
4737         ret = beiscsi_init_irqs(phba);
4738         if (ret < 0) {
4739                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
4740                             "BM_%d : beiscsi_dev_probe-"
4741                             "Failed to beiscsi_init_irqs\n");
4742                 goto free_blkenbld;
4743         }
4744         hwi_enable_intr(phba);
4745
4746         if (beiscsi_setup_boot_info(phba))
4747                 /*
4748                  * log error but continue, because we may not be using
4749                  * iscsi boot.
4750                  */
4751                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
4752                             "BM_%d : Could not set up "
4753                             "iSCSI boot info.\n");
4754
4755         beiscsi_create_def_ifaces(phba);
4756         beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
4757                     "\n\n\n BM_%d : SUCCESS - DRIVER LOADED\n\n\n");
4758         return 0;
4759
4760 free_blkenbld:
4761         destroy_workqueue(phba->wq);
4762         if (blk_iopoll_enabled)
4763                 for (i = 0; i < phba->num_cpus; i++) {
4764                         pbe_eq = &phwi_context->be_eq[i];
4765                         blk_iopoll_disable(&pbe_eq->iopoll);
4766                 }
4767 free_twq:
4768         beiscsi_clean_port(phba);
4769         beiscsi_free_mem(phba);
4770 free_port:
4771         pci_free_consistent(phba->pcidev,
4772                             phba->ctrl.mbox_mem_alloced.size,
4773                             phba->ctrl.mbox_mem_alloced.va,
4774                            phba->ctrl.mbox_mem_alloced.dma);
4775         beiscsi_unmap_pci_function(phba);
4776 hba_free:
4777         if (phba->msix_enabled)
4778                 pci_disable_msix(phba->pcidev);
4779         iscsi_host_remove(phba->shost);
4780         pci_dev_put(phba->pcidev);
4781         iscsi_host_free(phba->shost);
4782 disable_pci:
4783         pci_disable_device(pcidev);
4784         return ret;
4785 }
4786
4787 struct iscsi_transport beiscsi_iscsi_transport = {
4788         .owner = THIS_MODULE,
4789         .name = DRV_NAME,
4790         .caps = CAP_RECOVERY_L0 | CAP_HDRDGST | CAP_TEXT_NEGO |
4791                 CAP_MULTI_R2T | CAP_DATADGST | CAP_DATA_PATH_OFFLOAD,
4792         .create_session = beiscsi_session_create,
4793         .destroy_session = beiscsi_session_destroy,
4794         .create_conn = beiscsi_conn_create,
4795         .bind_conn = beiscsi_conn_bind,
4796         .destroy_conn = iscsi_conn_teardown,
4797         .attr_is_visible = be2iscsi_attr_is_visible,
4798         .set_iface_param = be2iscsi_iface_set_param,
4799         .get_iface_param = be2iscsi_iface_get_param,
4800         .set_param = beiscsi_set_param,
4801         .get_conn_param = iscsi_conn_get_param,
4802         .get_session_param = iscsi_session_get_param,
4803         .get_host_param = beiscsi_get_host_param,
4804         .start_conn = beiscsi_conn_start,
4805         .stop_conn = iscsi_conn_stop,
4806         .send_pdu = iscsi_conn_send_pdu,
4807         .xmit_task = beiscsi_task_xmit,
4808         .cleanup_task = beiscsi_cleanup_task,
4809         .alloc_pdu = beiscsi_alloc_pdu,
4810         .parse_pdu_itt = beiscsi_parse_pdu,
4811         .get_stats = beiscsi_conn_get_stats,
4812         .get_ep_param = beiscsi_ep_get_param,
4813         .ep_connect = beiscsi_ep_connect,
4814         .ep_poll = beiscsi_ep_poll,
4815         .ep_disconnect = beiscsi_ep_disconnect,
4816         .session_recovery_timedout = iscsi_session_recovery_timedout,
4817         .bsg_request = beiscsi_bsg_request,
4818 };
4819
4820 static struct pci_driver beiscsi_pci_driver = {
4821         .name = DRV_NAME,
4822         .probe = beiscsi_dev_probe,
4823         .remove = beiscsi_remove,
4824         .shutdown = beiscsi_shutdown,
4825         .id_table = beiscsi_pci_id_table
4826 };
4827
4828
4829 static int __init beiscsi_module_init(void)
4830 {
4831         int ret;
4832
4833         beiscsi_scsi_transport =
4834                         iscsi_register_transport(&beiscsi_iscsi_transport);
4835         if (!beiscsi_scsi_transport) {
4836                 printk(KERN_ERR
4837                        "beiscsi_module_init - Unable to  register beiscsi transport.\n");
4838                 return -ENOMEM;
4839         }
4840         printk(KERN_INFO "In beiscsi_module_init, tt=%p\n",
4841                &beiscsi_iscsi_transport);
4842
4843         ret = pci_register_driver(&beiscsi_pci_driver);
4844         if (ret) {
4845                 printk(KERN_ERR
4846                        "beiscsi_module_init - Unable to  register beiscsi pci driver.\n");
4847                 goto unregister_iscsi_transport;
4848         }
4849         return 0;
4850
4851 unregister_iscsi_transport:
4852         iscsi_unregister_transport(&beiscsi_iscsi_transport);
4853         return ret;
4854 }
4855
4856 static void __exit beiscsi_module_exit(void)
4857 {
4858         pci_unregister_driver(&beiscsi_pci_driver);
4859         iscsi_unregister_transport(&beiscsi_iscsi_transport);
4860 }
4861
4862 module_init(beiscsi_module_init);
4863 module_exit(beiscsi_module_exit);