drivers/scsi/be2iscsi/be_main.c

   1 /**
   2  * Copyright (C) 2005 - 2011 Emulex
   3  * All rights reserved.
   4  *
   5  * This program is free software; you can redistribute it and/or
   6  * modify it under the terms of the GNU General Public License version 2
   7  * as published by the Free Software Foundation.  The full GNU General
   8  * Public License is included in this distribution in the file called COPYING.
   9  *
  10  * Written by: Jayamohan Kallickal (jayamohan.kallickal@emulex.com)
  11  *
  12  * Contact Information:
  13  * linux-drivers@emulex.com
  14  *
  15  * Emulex
  16  * 3333 Susan Street
  17  * Costa Mesa, CA 92626
  18  */
  19
  20 #include <linux/reboot.h>
  21 #include <linux/delay.h>
  22 #include <linux/slab.h>
  23 #include <linux/interrupt.h>
  24 #include <linux/blkdev.h>
  25 #include <linux/pci.h>
  26 #include <linux/string.h>
  27 #include <linux/kernel.h>
  28 #include <linux/semaphore.h>
  29 #include <linux/iscsi_boot_sysfs.h>
  30 #include <linux/module.h>
  31 #include <linux/bsg-lib.h>
  32
  33 #include <scsi/libiscsi.h>
  34 #include <scsi/scsi_bsg_iscsi.h>
  35 #include <scsi/scsi_netlink.h>
  36 #include <scsi/scsi_transport_iscsi.h>
  37 #include <scsi/scsi_transport.h>
  38 #include <scsi/scsi_cmnd.h>
  39 #include <scsi/scsi_device.h>
  40 #include <scsi/scsi_host.h>
  41 #include <scsi/scsi.h>
  42 #include "be_main.h"
  43 #include "be_iscsi.h"
  44 #include "be_mgmt.h"
  45 #include "be_cmds.h"
  46
  47 static unsigned int be_iopoll_budget = 10;
  48 static unsigned int be_max_phys_size = 64;
  49 static unsigned int enable_msix = 1;
  50
  51 MODULE_DEVICE_TABLE(pci, beiscsi_pci_id_table);
  52 MODULE_DESCRIPTION(DRV_DESC " " BUILD_STR);
  53 MODULE_VERSION(BUILD_STR);
  54 MODULE_AUTHOR("Emulex Corporation");
  55 MODULE_LICENSE("GPL");
  56 module_param(be_iopoll_budget, int, 0);
  57 module_param(enable_msix, int, 0);
  58 module_param(be_max_phys_size, uint, S_IRUGO);
  59 MODULE_PARM_DESC(be_max_phys_size,
  60                 "Maximum Size (In Kilobytes) of physically contiguous "
  61                 "memory that can be allocated. Range is 16 - 128");
  62
  63 #define beiscsi_disp_param(_name)\
  64 ssize_t \
  65 beiscsi_##_name##_disp(struct device *dev,\
  66                         struct device_attribute *attrib, char *buf)     \
  67 {       \
  68         struct Scsi_Host *shost = class_to_shost(dev);\
  69         struct beiscsi_hba *phba = iscsi_host_priv(shost); \
  70         uint32_t param_val = 0; \
  71         param_val = phba->attr_##_name;\
  72         return snprintf(buf, PAGE_SIZE, "%d\n",\
  73                         phba->attr_##_name);\
  74 }
  75
  76 #define beiscsi_change_param(_name, _minval, _maxval, _defaval)\
  77 int \
  78 beiscsi_##_name##_change(struct beiscsi_hba *phba, uint32_t val)\
  79 {\
  80         if (val >= _minval && val <= _maxval) {\
  81                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,\
  82                             "BA_%d : beiscsi_"#_name" updated "\
  83                             "from 0x%x ==> 0x%x\n",\
  84                             phba->attr_##_name, val); \
  85                 phba->attr_##_name = val;\
  86                 return 0;\
  87         } \
  88         beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT, \
  89                     "BA_%d beiscsi_"#_name" attribute "\
  90                     "cannot be updated to 0x%x, "\
  91                     "range allowed is ["#_minval" - "#_maxval"]\n", val);\
  92                 return -EINVAL;\
  93 }
  94
  95 #define beiscsi_store_param(_name)  \
  96 ssize_t \
  97 beiscsi_##_name##_store(struct device *dev,\
  98                          struct device_attribute *attr, const char *buf,\
  99                          size_t count) \
 100 { \
 101         struct Scsi_Host  *shost = class_to_shost(dev);\
 102         struct beiscsi_hba *phba = iscsi_host_priv(shost);\
 103         uint32_t param_val = 0;\
 104         if (!isdigit(buf[0]))\
 105                 return -EINVAL;\
 106         if (sscanf(buf, "%i", &param_val) != 1)\
 107                 return -EINVAL;\
 108         if (beiscsi_##_name##_change(phba, param_val) == 0) \
 109                 return strlen(buf);\
 110         else \
 111                 return -EINVAL;\
 112 }
 113
 114 #define beiscsi_init_param(_name, _minval, _maxval, _defval) \
 115 int \
 116 beiscsi_##_name##_init(struct beiscsi_hba *phba, uint32_t val) \
 117 { \
 118         if (val >= _minval && val <= _maxval) {\
 119                 phba->attr_##_name = val;\
 120                 return 0;\
 121         } \
 122         beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,\
 123                     "BA_%d beiscsi_"#_name" attribute " \
 124                     "cannot be updated to 0x%x, "\
 125                     "range allowed is ["#_minval" - "#_maxval"]\n", val);\
 126         phba->attr_##_name = _defval;\
 127         return -EINVAL;\
 128 }
 129
 130 #define BEISCSI_RW_ATTR(_name, _minval, _maxval, _defval, _descp) \
 131 static uint beiscsi_##_name = _defval;\
 132 module_param(beiscsi_##_name, uint, S_IRUGO);\
 133 MODULE_PARM_DESC(beiscsi_##_name, _descp);\
 134 beiscsi_disp_param(_name)\
 135 beiscsi_change_param(_name, _minval, _maxval, _defval)\
 136 beiscsi_store_param(_name)\
 137 beiscsi_init_param(_name, _minval, _maxval, _defval)\
 138 DEVICE_ATTR(beiscsi_##_name, S_IRUGO | S_IWUSR,\
 139               beiscsi_##_name##_disp, beiscsi_##_name##_store)
 140
 141 /*
 142  * When new log level added update the
 143  * the MAX allowed value for log_enable
 144  */
 145 BEISCSI_RW_ATTR(log_enable, 0x00,
 146                 0xFF, 0x00, "Enable logging Bit Mask\n"
 147                 "\t\t\t\tInitialization Events  : 0x01\n"
 148                 "\t\t\t\tMailbox Events         : 0x02\n"
 149                 "\t\t\t\tMiscellaneous Events   : 0x04\n"
 150                 "\t\t\t\tError Handling         : 0x08\n"
 151                 "\t\t\t\tIO Path Events         : 0x10\n"
 152                 "\t\t\t\tConfiguration Path     : 0x20\n");
 153
 154 DEVICE_ATTR(beiscsi_drvr_ver, S_IRUGO, beiscsi_drvr_ver_disp, NULL);
 155 DEVICE_ATTR(beiscsi_adapter_family, S_IRUGO, beiscsi_adap_family_disp, NULL);
 156 struct device_attribute *beiscsi_attrs[] = {
 157         &dev_attr_beiscsi_log_enable,
 158         &dev_attr_beiscsi_drvr_ver,
 159         &dev_attr_beiscsi_adapter_family,
 160         NULL,
 161 };
 162
 163 static char const *cqe_desc[] = {
 164         "RESERVED_DESC",
 165         "SOL_CMD_COMPLETE",
 166         "SOL_CMD_KILLED_DATA_DIGEST_ERR",
 167         "CXN_KILLED_PDU_SIZE_EXCEEDS_DSL",
 168         "CXN_KILLED_BURST_LEN_MISMATCH",
 169         "CXN_KILLED_AHS_RCVD",
 170         "CXN_KILLED_HDR_DIGEST_ERR",
 171         "CXN_KILLED_UNKNOWN_HDR",
 172         "CXN_KILLED_STALE_ITT_TTT_RCVD",
 173         "CXN_KILLED_INVALID_ITT_TTT_RCVD",
 174         "CXN_KILLED_RST_RCVD",
 175         "CXN_KILLED_TIMED_OUT",
 176         "CXN_KILLED_RST_SENT",
 177         "CXN_KILLED_FIN_RCVD",
 178         "CXN_KILLED_BAD_UNSOL_PDU_RCVD",
 179         "CXN_KILLED_BAD_WRB_INDEX_ERROR",
 180         "CXN_KILLED_OVER_RUN_RESIDUAL",
 181         "CXN_KILLED_UNDER_RUN_RESIDUAL",
 182         "CMD_KILLED_INVALID_STATSN_RCVD",
 183         "CMD_KILLED_INVALID_R2T_RCVD",
 184         "CMD_CXN_KILLED_LUN_INVALID",
 185         "CMD_CXN_KILLED_ICD_INVALID",
 186         "CMD_CXN_KILLED_ITT_INVALID",
 187         "CMD_CXN_KILLED_SEQ_OUTOFORDER",
 188         "CMD_CXN_KILLED_INVALID_DATASN_RCVD",
 189         "CXN_INVALIDATE_NOTIFY",
 190         "CXN_INVALIDATE_INDEX_NOTIFY",
 191         "CMD_INVALIDATED_NOTIFY",
 192         "UNSOL_HDR_NOTIFY",
 193         "UNSOL_DATA_NOTIFY",
 194         "UNSOL_DATA_DIGEST_ERROR_NOTIFY",
 195         "DRIVERMSG_NOTIFY",
 196         "CXN_KILLED_CMND_DATA_NOT_ON_SAME_CONN",
 197         "SOL_CMD_KILLED_DIF_ERR",
 198         "CXN_KILLED_SYN_RCVD",
 199         "CXN_KILLED_IMM_DATA_RCVD"
 200 };
 201
 202 static int beiscsi_slave_configure(struct scsi_device *sdev)
 203 {
 204         blk_queue_max_segment_size(sdev->request_queue, 65536);
 205         return 0;
 206 }
 207
 208 static int beiscsi_eh_abort(struct scsi_cmnd *sc)
 209 {
 210         struct iscsi_cls_session *cls_session;
 211         struct iscsi_task *aborted_task = (struct iscsi_task *)sc->SCp.ptr;
 212         struct beiscsi_io_task *aborted_io_task;
 213         struct iscsi_conn *conn;
 214         struct beiscsi_conn *beiscsi_conn;
 215         struct beiscsi_hba *phba;
 216         struct iscsi_session *session;
 217         struct invalidate_command_table *inv_tbl;
 218         struct be_dma_mem nonemb_cmd;
 219         unsigned int cid, tag, num_invalidate;
 220
 221         cls_session = starget_to_session(scsi_target(sc->device));
 222         session = cls_session->dd_data;
 223
 224         spin_lock_bh(&session->lock);
 225         if (!aborted_task || !aborted_task->sc) {
 226                 /* we raced */
 227                 spin_unlock_bh(&session->lock);
 228                 return SUCCESS;
 229         }
 230
 231         aborted_io_task = aborted_task->dd_data;
 232         if (!aborted_io_task->scsi_cmnd) {
 233                 /* raced or invalid command */
 234                 spin_unlock_bh(&session->lock);
 235                 return SUCCESS;
 236         }
 237         spin_unlock_bh(&session->lock);
 238         conn = aborted_task->conn;
 239         beiscsi_conn = conn->dd_data;
 240         phba = beiscsi_conn->phba;
 241
 242         /* invalidate iocb */
 243         cid = beiscsi_conn->beiscsi_conn_cid;
 244         inv_tbl = phba->inv_tbl;
 245         memset(inv_tbl, 0x0, sizeof(*inv_tbl));
 246         inv_tbl->cid = cid;
 247         inv_tbl->icd = aborted_io_task->psgl_handle->sgl_index;
 248         num_invalidate = 1;
 249         nonemb_cmd.va = pci_alloc_consistent(phba->ctrl.pdev,
 250                                 sizeof(struct invalidate_commands_params_in),
 251                                 &nonemb_cmd.dma);
 252         if (nonemb_cmd.va == NULL) {
 253                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_EH,
 254                             "BM_%d : Failed to allocate memory for"
 255                             "mgmt_invalidate_icds\n");
 256                 return FAILED;
 257         }
 258         nonemb_cmd.size = sizeof(struct invalidate_commands_params_in);
 259
 260         tag = mgmt_invalidate_icds(phba, inv_tbl, num_invalidate,
 261                                    cid, &nonemb_cmd);
 262         if (!tag) {
 263                 beiscsi_log(phba, KERN_WARNING, BEISCSI_LOG_EH,
 264                             "BM_%d : mgmt_invalidate_icds could not be"
 265                             "submitted\n");
 266                 pci_free_consistent(phba->ctrl.pdev, nonemb_cmd.size,
 267                                     nonemb_cmd.va, nonemb_cmd.dma);
 268
 269                 return FAILED;
 270         } else {
 271                 wait_event_interruptible(phba->ctrl.mcc_wait[tag],
 272                                          phba->ctrl.mcc_numtag[tag]);
 273                 free_mcc_tag(&phba->ctrl, tag);
 274         }
 275         pci_free_consistent(phba->ctrl.pdev, nonemb_cmd.size,
 276                             nonemb_cmd.va, nonemb_cmd.dma);
 277         return iscsi_eh_abort(sc);
 278 }
 279
 280 static int beiscsi_eh_device_reset(struct scsi_cmnd *sc)
 281 {
 282         struct iscsi_task *abrt_task;
 283         struct beiscsi_io_task *abrt_io_task;
 284         struct iscsi_conn *conn;
 285         struct beiscsi_conn *beiscsi_conn;
 286         struct beiscsi_hba *phba;
 287         struct iscsi_session *session;
 288         struct iscsi_cls_session *cls_session;
 289         struct invalidate_command_table *inv_tbl;
 290         struct be_dma_mem nonemb_cmd;
 291         unsigned int cid, tag, i, num_invalidate;
 292
 293         /* invalidate iocbs */
 294         cls_session = starget_to_session(scsi_target(sc->device));
 295         session = cls_session->dd_data;
 296         spin_lock_bh(&session->lock);
 297         if (!session->leadconn || session->state != ISCSI_STATE_LOGGED_IN) {
 298                 spin_unlock_bh(&session->lock);
 299                 return FAILED;
 300         }
 301         conn = session->leadconn;
 302         beiscsi_conn = conn->dd_data;
 303         phba = beiscsi_conn->phba;
 304         cid = beiscsi_conn->beiscsi_conn_cid;
 305         inv_tbl = phba->inv_tbl;
 306         memset(inv_tbl, 0x0, sizeof(*inv_tbl) * BE2_CMDS_PER_CXN);
 307         num_invalidate = 0;
 308         for (i = 0; i < conn->session->cmds_max; i++) {
 309                 abrt_task = conn->session->cmds[i];
 310                 abrt_io_task = abrt_task->dd_data;
 311                 if (!abrt_task->sc || abrt_task->state == ISCSI_TASK_FREE)
 312                         continue;
 313
 314                 if (abrt_task->sc->device->lun != abrt_task->sc->device->lun)
 315                         continue;
 316
 317                 inv_tbl->cid = cid;
 318                 inv_tbl->icd = abrt_io_task->psgl_handle->sgl_index;
 319                 num_invalidate++;
 320                 inv_tbl++;
 321         }
 322         spin_unlock_bh(&session->lock);
 323         inv_tbl = phba->inv_tbl;
 324
 325         nonemb_cmd.va = pci_alloc_consistent(phba->ctrl.pdev,
 326                                 sizeof(struct invalidate_commands_params_in),
 327                                 &nonemb_cmd.dma);
 328         if (nonemb_cmd.va == NULL) {
 329                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_EH,
 330                             "BM_%d : Failed to allocate memory for"
 331                             "mgmt_invalidate_icds\n");
 332                 return FAILED;
 333         }
 334         nonemb_cmd.size = sizeof(struct invalidate_commands_params_in);
 335         memset(nonemb_cmd.va, 0, nonemb_cmd.size);
 336         tag = mgmt_invalidate_icds(phba, inv_tbl, num_invalidate,
 337                                    cid, &nonemb_cmd);
 338         if (!tag) {
 339                 beiscsi_log(phba, KERN_WARNING, BEISCSI_LOG_EH,
 340                             "BM_%d : mgmt_invalidate_icds could not be"
 341                             " submitted\n");
 342                 pci_free_consistent(phba->ctrl.pdev, nonemb_cmd.size,
 343                                     nonemb_cmd.va, nonemb_cmd.dma);
 344                 return FAILED;
 345         } else {
 346                 wait_event_interruptible(phba->ctrl.mcc_wait[tag],
 347                                          phba->ctrl.mcc_numtag[tag]);
 348                 free_mcc_tag(&phba->ctrl, tag);
 349         }
 350         pci_free_consistent(phba->ctrl.pdev, nonemb_cmd.size,
 351                             nonemb_cmd.va, nonemb_cmd.dma);
 352         return iscsi_eh_device_reset(sc);
 353 }
 354
 355 static ssize_t beiscsi_show_boot_tgt_info(void *data, int type, char *buf)
 356 {
 357         struct beiscsi_hba *phba = data;
 358         struct mgmt_session_info *boot_sess = &phba->boot_sess;
 359         struct mgmt_conn_info *boot_conn = &boot_sess->conn_list[0];
 360         char *str = buf;
 361         int rc;
 362
 363         switch (type) {
 364         case ISCSI_BOOT_TGT_NAME:
 365                 rc = sprintf(buf, "%.*s\n",
 366                             (int)strlen(boot_sess->target_name),
 367                             (char *)&boot_sess->target_name);
 368                 break;
 369         case ISCSI_BOOT_TGT_IP_ADDR:
 370                 if (boot_conn->dest_ipaddr.ip_type == 0x1)
 371                         rc = sprintf(buf, "%pI4\n",
 372                                 (char *)&boot_conn->dest_ipaddr.addr);
 373                 else
 374                         rc = sprintf(str, "%pI6\n",
 375                                 (char *)&boot_conn->dest_ipaddr.addr);
 376                 break;
 377         case ISCSI_BOOT_TGT_PORT:
 378                 rc = sprintf(str, "%d\n", boot_conn->dest_port);
 379                 break;
 380
 381         case ISCSI_BOOT_TGT_CHAP_NAME:
 382                 rc = sprintf(str,  "%.*s\n",
 383                              boot_conn->negotiated_login_options.auth_data.chap.
 384                              target_chap_name_length,
 385                              (char *)&boot_conn->negotiated_login_options.
 386                              auth_data.chap.target_chap_name);
 387                 break;
 388         case ISCSI_BOOT_TGT_CHAP_SECRET:
 389                 rc = sprintf(str,  "%.*s\n",
 390                              boot_conn->negotiated_login_options.auth_data.chap.
 391                              target_secret_length,
 392                              (char *)&boot_conn->negotiated_login_options.
 393                              auth_data.chap.target_secret);
 394                 break;
 395         case ISCSI_BOOT_TGT_REV_CHAP_NAME:
 396                 rc = sprintf(str,  "%.*s\n",
 397                              boot_conn->negotiated_login_options.auth_data.chap.
 398                              intr_chap_name_length,
 399                              (char *)&boot_conn->negotiated_login_options.
 400                              auth_data.chap.intr_chap_name);
 401                 break;
 402         case ISCSI_BOOT_TGT_REV_CHAP_SECRET:
 403                 rc = sprintf(str,  "%.*s\n",
 404                              boot_conn->negotiated_login_options.auth_data.chap.
 405                              intr_secret_length,
 406                              (char *)&boot_conn->negotiated_login_options.
 407                              auth_data.chap.intr_secret);
 408                 break;
 409         case ISCSI_BOOT_TGT_FLAGS:
 410                 rc = sprintf(str, "2\n");
 411                 break;
 412         case ISCSI_BOOT_TGT_NIC_ASSOC:
 413                 rc = sprintf(str, "0\n");
 414                 break;
 415         default:
 416                 rc = -ENOSYS;
 417                 break;
 418         }
 419         return rc;
 420 }
 421
 422 static ssize_t beiscsi_show_boot_ini_info(void *data, int type, char *buf)
 423 {
 424         struct beiscsi_hba *phba = data;
 425         char *str = buf;
 426         int rc;
 427
 428         switch (type) {
 429         case ISCSI_BOOT_INI_INITIATOR_NAME:
 430                 rc = sprintf(str, "%s\n", phba->boot_sess.initiator_iscsiname);
 431                 break;
 432         default:
 433                 rc = -ENOSYS;
 434                 break;
 435         }
 436         return rc;
 437 }
 438
 439 static ssize_t beiscsi_show_boot_eth_info(void *data, int type, char *buf)
 440 {
 441         struct beiscsi_hba *phba = data;
 442         char *str = buf;
 443         int rc;
 444
 445         switch (type) {
 446         case ISCSI_BOOT_ETH_FLAGS:
 447                 rc = sprintf(str, "2\n");
 448                 break;
 449         case ISCSI_BOOT_ETH_INDEX:
 450                 rc = sprintf(str, "0\n");
 451                 break;
 452         case ISCSI_BOOT_ETH_MAC:
 453                 rc  = beiscsi_get_macaddr(str, phba);
 454                 break;
 455         default:
 456                 rc = -ENOSYS;
 457                 break;
 458         }
 459         return rc;
 460 }
 461
 462
 463 static umode_t beiscsi_tgt_get_attr_visibility(void *data, int type)
 464 {
 465         umode_t rc;
 466
 467         switch (type) {
 468         case ISCSI_BOOT_TGT_NAME:
 469         case ISCSI_BOOT_TGT_IP_ADDR:
 470         case ISCSI_BOOT_TGT_PORT:
 471         case ISCSI_BOOT_TGT_CHAP_NAME:
 472         case ISCSI_BOOT_TGT_CHAP_SECRET:
 473         case ISCSI_BOOT_TGT_REV_CHAP_NAME:
 474         case ISCSI_BOOT_TGT_REV_CHAP_SECRET:
 475         case ISCSI_BOOT_TGT_NIC_ASSOC:
 476         case ISCSI_BOOT_TGT_FLAGS:
 477                 rc = S_IRUGO;
 478                 break;
 479         default:
 480                 rc = 0;
 481                 break;
 482         }
 483         return rc;
 484 }
 485
 486 static umode_t beiscsi_ini_get_attr_visibility(void *data, int type)
 487 {
 488         umode_t rc;
 489
 490         switch (type) {
 491         case ISCSI_BOOT_INI_INITIATOR_NAME:
 492                 rc = S_IRUGO;
 493                 break;
 494         default:
 495                 rc = 0;
 496                 break;
 497         }
 498         return rc;
 499 }
 500
 501
 502 static umode_t beiscsi_eth_get_attr_visibility(void *data, int type)
 503 {
 504         umode_t rc;
 505
 506         switch (type) {
 507         case ISCSI_BOOT_ETH_FLAGS:
 508         case ISCSI_BOOT_ETH_MAC:
 509         case ISCSI_BOOT_ETH_INDEX:
 510                 rc = S_IRUGO;
 511                 break;
 512         default:
 513                 rc = 0;
 514                 break;
 515         }
 516         return rc;
 517 }
 518
 519 /*------------------- PCI Driver operations and data ----------------- */
 520 static DEFINE_PCI_DEVICE_TABLE(beiscsi_pci_id_table) = {
 521         { PCI_DEVICE(BE_VENDOR_ID, BE_DEVICE_ID1) },
 522         { PCI_DEVICE(BE_VENDOR_ID, BE_DEVICE_ID2) },
 523         { PCI_DEVICE(BE_VENDOR_ID, OC_DEVICE_ID1) },
 524         { PCI_DEVICE(BE_VENDOR_ID, OC_DEVICE_ID2) },
 525         { PCI_DEVICE(BE_VENDOR_ID, OC_DEVICE_ID3) },
 526         { PCI_DEVICE(ELX_VENDOR_ID, OC_SKH_ID1) },
 527         { 0 }
 528 };
 529 MODULE_DEVICE_TABLE(pci, beiscsi_pci_id_table);
 530
 531
 532 static struct scsi_host_template beiscsi_sht = {
 533         .module = THIS_MODULE,
 534         .name = "Emulex 10Gbe open-iscsi Initiator Driver",
 535         .proc_name = DRV_NAME,
 536         .queuecommand = iscsi_queuecommand,
 537         .change_queue_depth = iscsi_change_queue_depth,
 538         .slave_configure = beiscsi_slave_configure,
 539         .target_alloc = iscsi_target_alloc,
 540         .eh_abort_handler = beiscsi_eh_abort,
 541         .eh_device_reset_handler = beiscsi_eh_device_reset,
 542         .eh_target_reset_handler = iscsi_eh_session_reset,
 543         .shost_attrs = beiscsi_attrs,
 544         .sg_tablesize = BEISCSI_SGLIST_ELEMENTS,
 545         .can_queue = BE2_IO_DEPTH,
 546         .this_id = -1,
 547         .max_sectors = BEISCSI_MAX_SECTORS,
 548         .cmd_per_lun = BEISCSI_CMD_PER_LUN,
 549         .use_clustering = ENABLE_CLUSTERING,
 550         .vendor_id = SCSI_NL_VID_TYPE_PCI | BE_VENDOR_ID,
 551
 552 };
 553
 554 static struct scsi_transport_template *beiscsi_scsi_transport;
 555
 556 static struct beiscsi_hba *beiscsi_hba_alloc(struct pci_dev *pcidev)
 557 {
 558         struct beiscsi_hba *phba;
 559         struct Scsi_Host *shost;
 560
 561         shost = iscsi_host_alloc(&beiscsi_sht, sizeof(*phba), 0);
 562         if (!shost) {
 563                 dev_err(&pcidev->dev,
 564                         "beiscsi_hba_alloc - iscsi_host_alloc failed\n");
 565                 return NULL;
 566         }
 567         shost->dma_boundary = pcidev->dma_mask;
 568         shost->max_id = BE2_MAX_SESSIONS;
 569         shost->max_channel = 0;
 570         shost->max_cmd_len = BEISCSI_MAX_CMD_LEN;
 571         shost->max_lun = BEISCSI_NUM_MAX_LUN;
 572         shost->transportt = beiscsi_scsi_transport;
 573         phba = iscsi_host_priv(shost);
 574         memset(phba, 0, sizeof(*phba));
 575         phba->shost = shost;
 576         phba->pcidev = pci_dev_get(pcidev);
 577         pci_set_drvdata(pcidev, phba);
 578         phba->interface_handle = 0xFFFFFFFF;
 579
 580         if (iscsi_host_add(shost, &phba->pcidev->dev))
 581                 goto free_devices;
 582
 583         return phba;
 584
 585 free_devices:
 586         pci_dev_put(phba->pcidev);
 587         iscsi_host_free(phba->shost);
 588         return NULL;
 589 }
 590
 591 static void beiscsi_unmap_pci_function(struct beiscsi_hba *phba)
 592 {
 593         if (phba->csr_va) {
 594                 iounmap(phba->csr_va);
 595                 phba->csr_va = NULL;
 596         }
 597         if (phba->db_va) {
 598                 iounmap(phba->db_va);
 599                 phba->db_va = NULL;
 600         }
 601         if (phba->pci_va) {
 602                 iounmap(phba->pci_va);
 603                 phba->pci_va = NULL;
 604         }
 605 }
 606
 607 static int beiscsi_map_pci_bars(struct beiscsi_hba *phba,
 608                                 struct pci_dev *pcidev)
 609 {
 610         u8 __iomem *addr;
 611         int pcicfg_reg;
 612
 613         addr = ioremap_nocache(pci_resource_start(pcidev, 2),
 614                                pci_resource_len(pcidev, 2));
 615         if (addr == NULL)
 616                 return -ENOMEM;
 617         phba->ctrl.csr = addr;
 618         phba->csr_va = addr;
 619         phba->csr_pa.u.a64.address = pci_resource_start(pcidev, 2);
 620
 621         addr = ioremap_nocache(pci_resource_start(pcidev, 4), 128 * 1024);
 622         if (addr == NULL)
 623                 goto pci_map_err;
 624         phba->ctrl.db = addr;
 625         phba->db_va = addr;
 626         phba->db_pa.u.a64.address =  pci_resource_start(pcidev, 4);
 627
 628         if (phba->generation == BE_GEN2)
 629                 pcicfg_reg = 1;
 630         else
 631                 pcicfg_reg = 0;
 632
 633         addr = ioremap_nocache(pci_resource_start(pcidev, pcicfg_reg),
 634                                pci_resource_len(pcidev, pcicfg_reg));
 635
 636         if (addr == NULL)
 637                 goto pci_map_err;
 638         phba->ctrl.pcicfg = addr;
 639         phba->pci_va = addr;
 640         phba->pci_pa.u.a64.address = pci_resource_start(pcidev, pcicfg_reg);
 641         return 0;
 642
 643 pci_map_err:
 644         beiscsi_unmap_pci_function(phba);
 645         return -ENOMEM;
 646 }
 647
 648 static int beiscsi_enable_pci(struct pci_dev *pcidev)
 649 {
 650         int ret;
 651
 652         ret = pci_enable_device(pcidev);
 653         if (ret) {
 654                 dev_err(&pcidev->dev,
 655                         "beiscsi_enable_pci - enable device failed\n");
 656                 return ret;
 657         }
 658
 659         pci_set_master(pcidev);
 660         if (pci_set_consistent_dma_mask(pcidev, DMA_BIT_MASK(64))) {
 661                 ret = pci_set_consistent_dma_mask(pcidev, DMA_BIT_MASK(32));
 662                 if (ret) {
 663                         dev_err(&pcidev->dev, "Could not set PCI DMA Mask\n");
 664                         pci_disable_device(pcidev);
 665                         return ret;
 666                 }
 667         }
 668         return 0;
 669 }
 670
 671 static int be_ctrl_init(struct beiscsi_hba *phba, struct pci_dev *pdev)
 672 {
 673         struct be_ctrl_info *ctrl = &phba->ctrl;
 674         struct be_dma_mem *mbox_mem_alloc = &ctrl->mbox_mem_alloced;
 675         struct be_dma_mem *mbox_mem_align = &ctrl->mbox_mem;
 676         int status = 0;
 677
 678         ctrl->pdev = pdev;
 679         status = beiscsi_map_pci_bars(phba, pdev);
 680         if (status)
 681                 return status;
 682         mbox_mem_alloc->size = sizeof(struct be_mcc_mailbox) + 16;
 683         mbox_mem_alloc->va = pci_alloc_consistent(pdev,
 684                                                   mbox_mem_alloc->size,
 685                                                   &mbox_mem_alloc->dma);
 686         if (!mbox_mem_alloc->va) {
 687                 beiscsi_unmap_pci_function(phba);
 688                 return -ENOMEM;
 689         }
 690
 691         mbox_mem_align->size = sizeof(struct be_mcc_mailbox);
 692         mbox_mem_align->va = PTR_ALIGN(mbox_mem_alloc->va, 16);
 693         mbox_mem_align->dma = PTR_ALIGN(mbox_mem_alloc->dma, 16);
 694         memset(mbox_mem_align->va, 0, sizeof(struct be_mcc_mailbox));
 695         spin_lock_init(&ctrl->mbox_lock);
 696         spin_lock_init(&phba->ctrl.mcc_lock);
 697         spin_lock_init(&phba->ctrl.mcc_cq_lock);
 698
 699         return status;
 700 }
 701
 702 static void beiscsi_get_params(struct beiscsi_hba *phba)
 703 {
 704         phba->params.ios_per_ctrl = (phba->fw_config.iscsi_icd_count
 705                                     - (phba->fw_config.iscsi_cid_count
 706                                     + BE2_TMFS
 707                                     + BE2_NOPOUT_REQ));
 708         phba->params.cxns_per_ctrl = phba->fw_config.iscsi_cid_count;
 709         phba->params.asyncpdus_per_ctrl = phba->fw_config.iscsi_cid_count * 2;
 710         phba->params.icds_per_ctrl = phba->fw_config.iscsi_icd_count;
 711         phba->params.num_sge_per_io = BE2_SGE;
 712         phba->params.defpdu_hdr_sz = BE2_DEFPDU_HDR_SZ;
 713         phba->params.defpdu_data_sz = BE2_DEFPDU_DATA_SZ;
 714         phba->params.eq_timer = 64;
 715         phba->params.num_eq_entries =
 716             (((BE2_CMDS_PER_CXN * 2 + phba->fw_config.iscsi_cid_count * 2
 717                                     + BE2_TMFS) / 512) + 1) * 512;
 718         phba->params.num_eq_entries = (phba->params.num_eq_entries < 1024)
 719                                 ? 1024 : phba->params.num_eq_entries;
 720         beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
 721                     "BM_%d : phba->params.num_eq_entries=%d\n",
 722                     phba->params.num_eq_entries);
 723         phba->params.num_cq_entries =
 724             (((BE2_CMDS_PER_CXN * 2 +  phba->fw_config.iscsi_cid_count * 2
 725                                     + BE2_TMFS) / 512) + 1) * 512;
 726         phba->params.wrbs_per_cxn = 256;
 727 }
 728
 729 static void hwi_ring_eq_db(struct beiscsi_hba *phba,
 730                            unsigned int id, unsigned int clr_interrupt,
 731                            unsigned int num_processed,
 732                            unsigned char rearm, unsigned char event)
 733 {
 734         u32 val = 0;
 735         val |= id & DB_EQ_RING_ID_MASK;
 736         if (rearm)
 737                 val |= 1 << DB_EQ_REARM_SHIFT;
 738         if (clr_interrupt)
 739                 val |= 1 << DB_EQ_CLR_SHIFT;
 740         if (event)
 741                 val |= 1 << DB_EQ_EVNT_SHIFT;
 742         val |= num_processed << DB_EQ_NUM_POPPED_SHIFT;
 743         iowrite32(val, phba->db_va + DB_EQ_OFFSET);
 744 }
 745
 746 /**
 747  * be_isr_mcc - The isr routine of the driver.
 748  * @irq: Not used
 749  * @dev_id: Pointer to host adapter structure
 750  */
 751 static irqreturn_t be_isr_mcc(int irq, void *dev_id)
 752 {
 753         struct beiscsi_hba *phba;
 754         struct be_eq_entry *eqe = NULL;
 755         struct be_queue_info *eq;
 756         struct be_queue_info *mcc;
 757         unsigned int num_eq_processed;
 758         struct be_eq_obj *pbe_eq;
 759         unsigned long flags;
 760
 761         pbe_eq = dev_id;
 762         eq = &pbe_eq->q;
 763         phba =  pbe_eq->phba;
 764         mcc = &phba->ctrl.mcc_obj.cq;
 765         eqe = queue_tail_node(eq);
 766
 767         num_eq_processed = 0;
 768
 769         while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32]
 770                                 & EQE_VALID_MASK) {
 771                 if (((eqe->dw[offsetof(struct amap_eq_entry,
 772                      resource_id) / 32] &
 773                      EQE_RESID_MASK) >> 16) == mcc->id) {
 774                         spin_lock_irqsave(&phba->isr_lock, flags);
 775                         pbe_eq->todo_mcc_cq = true;
 776                         spin_unlock_irqrestore(&phba->isr_lock, flags);
 777                 }
 778                 AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0);
 779                 queue_tail_inc(eq);
 780                 eqe = queue_tail_node(eq);
 781                 num_eq_processed++;
 782         }
 783         if (pbe_eq->todo_mcc_cq)
 784                 queue_work(phba->wq, &pbe_eq->work_cqs);
 785         if (num_eq_processed)
 786                 hwi_ring_eq_db(phba, eq->id, 1, num_eq_processed, 1, 1);
 787
 788         return IRQ_HANDLED;
 789 }
 790
 791 /**
 792  * be_isr_msix - The isr routine of the driver.
 793  * @irq: Not used
 794  * @dev_id: Pointer to host adapter structure
 795  */
 796 static irqreturn_t be_isr_msix(int irq, void *dev_id)
 797 {
 798         struct beiscsi_hba *phba;
 799         struct be_eq_entry *eqe = NULL;
 800         struct be_queue_info *eq;
 801         struct be_queue_info *cq;
 802         unsigned int num_eq_processed;
 803         struct be_eq_obj *pbe_eq;
 804         unsigned long flags;
 805
 806         pbe_eq = dev_id;
 807         eq = &pbe_eq->q;
 808         cq = pbe_eq->cq;
 809         eqe = queue_tail_node(eq);
 810
 811         phba = pbe_eq->phba;
 812         num_eq_processed = 0;
 813         if (blk_iopoll_enabled) {
 814                 while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32]
 815                                         & EQE_VALID_MASK) {
 816                         if (!blk_iopoll_sched_prep(&pbe_eq->iopoll))
 817                                 blk_iopoll_sched(&pbe_eq->iopoll);
 818
 819                         AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0);
 820                         queue_tail_inc(eq);
 821                         eqe = queue_tail_node(eq);
 822                         num_eq_processed++;
 823                 }
 824         } else {
 825                 while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32]
 826                                                 & EQE_VALID_MASK) {
 827                         spin_lock_irqsave(&phba->isr_lock, flags);
 828                         pbe_eq->todo_cq = true;
 829                         spin_unlock_irqrestore(&phba->isr_lock, flags);
 830                         AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0);
 831                         queue_tail_inc(eq);
 832                         eqe = queue_tail_node(eq);
 833                         num_eq_processed++;
 834                 }
 835
 836                 if (pbe_eq->todo_cq)
 837                         queue_work(phba->wq, &pbe_eq->work_cqs);
 838         }
 839
 840         if (num_eq_processed)
 841                 hwi_ring_eq_db(phba, eq->id, 1, num_eq_processed, 0, 1);
 842
 843         return IRQ_HANDLED;
 844 }
 845
 846 /**
 847  * be_isr - The isr routine of the driver.
 848  * @irq: Not used
 849  * @dev_id: Pointer to host adapter structure
 850  */
 851 static irqreturn_t be_isr(int irq, void *dev_id)
 852 {
 853         struct beiscsi_hba *phba;
 854         struct hwi_controller *phwi_ctrlr;
 855         struct hwi_context_memory *phwi_context;
 856         struct be_eq_entry *eqe = NULL;
 857         struct be_queue_info *eq;
 858         struct be_queue_info *cq;
 859         struct be_queue_info *mcc;
 860         unsigned long flags, index;
 861         unsigned int num_mcceq_processed, num_ioeq_processed;
 862         struct be_ctrl_info *ctrl;
 863         struct be_eq_obj *pbe_eq;
 864         int isr;
 865
 866         phba = dev_id;
 867         ctrl = &phba->ctrl;
 868         isr = ioread32(ctrl->csr + CEV_ISR0_OFFSET +
 869                        (PCI_FUNC(ctrl->pdev->devfn) * CEV_ISR_SIZE));
 870         if (!isr)
 871                 return IRQ_NONE;
 872
 873         phwi_ctrlr = phba->phwi_ctrlr;
 874         phwi_context = phwi_ctrlr->phwi_ctxt;
 875         pbe_eq = &phwi_context->be_eq[0];
 876
 877         eq = &phwi_context->be_eq[0].q;
 878         mcc = &phba->ctrl.mcc_obj.cq;
 879         index = 0;
 880         eqe = queue_tail_node(eq);
 881
 882         num_ioeq_processed = 0;
 883         num_mcceq_processed = 0;
 884         if (blk_iopoll_enabled) {
 885                 while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32]
 886                                         & EQE_VALID_MASK) {
 887                         if (((eqe->dw[offsetof(struct amap_eq_entry,
 888                              resource_id) / 32] &
 889                              EQE_RESID_MASK) >> 16) == mcc->id) {
 890                                 spin_lock_irqsave(&phba->isr_lock, flags);
 891                                 pbe_eq->todo_mcc_cq = true;
 892                                 spin_unlock_irqrestore(&phba->isr_lock, flags);
 893                                 num_mcceq_processed++;
 894                         } else {
 895                                 if (!blk_iopoll_sched_prep(&pbe_eq->iopoll))
 896                                         blk_iopoll_sched(&pbe_eq->iopoll);
 897                                 num_ioeq_processed++;
 898                         }
 899                         AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0);
 900                         queue_tail_inc(eq);
 901                         eqe = queue_tail_node(eq);
 902                 }
 903                 if (num_ioeq_processed || num_mcceq_processed) {
 904                         if (pbe_eq->todo_mcc_cq)
 905                                 queue_work(phba->wq, &pbe_eq->work_cqs);
 906
 907                         if ((num_mcceq_processed) && (!num_ioeq_processed))
 908                                 hwi_ring_eq_db(phba, eq->id, 0,
 909                                               (num_ioeq_processed +
 910                                                num_mcceq_processed) , 1, 1);
 911                         else
 912                                 hwi_ring_eq_db(phba, eq->id, 0,
 913                                                (num_ioeq_processed +
 914                                                 num_mcceq_processed), 0, 1);
 915
 916                         return IRQ_HANDLED;
 917                 } else
 918                         return IRQ_NONE;
 919         } else {
 920                 cq = &phwi_context->be_cq[0];
 921                 while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32]
 922                                                 & EQE_VALID_MASK) {
 923
 924                         if (((eqe->dw[offsetof(struct amap_eq_entry,
 925                              resource_id) / 32] &
 926                              EQE_RESID_MASK) >> 16) != cq->id) {
 927                                 spin_lock_irqsave(&phba->isr_lock, flags);
 928                                 pbe_eq->todo_mcc_cq = true;
 929                                 spin_unlock_irqrestore(&phba->isr_lock, flags);
 930                         } else {
 931                                 spin_lock_irqsave(&phba->isr_lock, flags);
 932                                 pbe_eq->todo_cq = true;
 933                                 spin_unlock_irqrestore(&phba->isr_lock, flags);
 934                         }
 935                         AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0);
 936                         queue_tail_inc(eq);
 937                         eqe = queue_tail_node(eq);
 938                         num_ioeq_processed++;
 939                 }
 940                 if (pbe_eq->todo_cq || pbe_eq->todo_mcc_cq)
 941                         queue_work(phba->wq, &pbe_eq->work_cqs);
 942
 943                 if (num_ioeq_processed) {
 944                         hwi_ring_eq_db(phba, eq->id, 0,
 945                                        num_ioeq_processed, 1, 1);
 946                         return IRQ_HANDLED;
 947                 } else
 948                         return IRQ_NONE;
 949         }
 950 }
 951
 952 static int beiscsi_init_irqs(struct beiscsi_hba *phba)
 953 {
 954         struct pci_dev *pcidev = phba->pcidev;
 955         struct hwi_controller *phwi_ctrlr;
 956         struct hwi_context_memory *phwi_context;
 957         int ret, msix_vec, i, j;
 958
 959         phwi_ctrlr = phba->phwi_ctrlr;
 960         phwi_context = phwi_ctrlr->phwi_ctxt;
 961
 962         if (phba->msix_enabled) {
 963                 for (i = 0; i < phba->num_cpus; i++) {
 964                         phba->msi_name[i] = kzalloc(BEISCSI_MSI_NAME,
 965                                                     GFP_KERNEL);
 966                         if (!phba->msi_name[i]) {
 967                                 ret = -ENOMEM;
 968                                 goto free_msix_irqs;
 969                         }
 970
 971                         sprintf(phba->msi_name[i], "beiscsi_%02x_%02x",
 972                                 phba->shost->host_no, i);
 973                         msix_vec = phba->msix_entries[i].vector;
 974                         ret = request_irq(msix_vec, be_isr_msix, 0,
 975                                           phba->msi_name[i],
 976                                           &phwi_context->be_eq[i]);
 977                         if (ret) {
 978                                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
 979                                             "BM_%d : beiscsi_init_irqs-Failed to"
 980                                             "register msix for i = %d\n",
 981                                             i);
 982                                 kfree(phba->msi_name[i]);
 983                                 goto free_msix_irqs;
 984                         }
 985                 }
 986                 phba->msi_name[i] = kzalloc(BEISCSI_MSI_NAME, GFP_KERNEL);
 987                 if (!phba->msi_name[i]) {
 988                         ret = -ENOMEM;
 989                         goto free_msix_irqs;
 990                 }
 991                 sprintf(phba->msi_name[i], "beiscsi_mcc_%02x",
 992                         phba->shost->host_no);
 993                 msix_vec = phba->msix_entries[i].vector;
 994                 ret = request_irq(msix_vec, be_isr_mcc, 0, phba->msi_name[i],
 995                                   &phwi_context->be_eq[i]);
 996                 if (ret) {
 997                         beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT ,
 998                                     "BM_%d : beiscsi_init_irqs-"
 999                                     "Failed to register beiscsi_msix_mcc\n");
1000                         kfree(phba->msi_name[i]);
1001                         goto free_msix_irqs;
1002                 }
1003
1004         } else {
1005                 ret = request_irq(pcidev->irq, be_isr, IRQF_SHARED,
1006                                   "beiscsi", phba);
1007                 if (ret) {
1008                         beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
1009                                     "BM_%d : beiscsi_init_irqs-"
1010                                     "Failed to register irq\\n");
1011                         return ret;
1012                 }
1013         }
1014         return 0;
1015 free_msix_irqs:
1016         for (j = i - 1; j >= 0; j--) {
1017                 kfree(phba->msi_name[j]);
1018                 msix_vec = phba->msix_entries[j].vector;
1019                 free_irq(msix_vec, &phwi_context->be_eq[j]);
1020         }
1021         return ret;
1022 }
1023
1024 static void hwi_ring_cq_db(struct beiscsi_hba *phba,
1025                            unsigned int id, unsigned int num_processed,
1026                            unsigned char rearm, unsigned char event)
1027 {
1028         u32 val = 0;
1029         val |= id & DB_CQ_RING_ID_MASK;
1030         if (rearm)
1031                 val |= 1 << DB_CQ_REARM_SHIFT;
1032         val |= num_processed << DB_CQ_NUM_POPPED_SHIFT;
1033         iowrite32(val, phba->db_va + DB_CQ_OFFSET);
1034 }
1035
1036 static unsigned int
1037 beiscsi_process_async_pdu(struct beiscsi_conn *beiscsi_conn,
1038                           struct beiscsi_hba *phba,
1039                           unsigned short cid,
1040                           struct pdu_base *ppdu,
1041                           unsigned long pdu_len,
1042                           void *pbuffer, unsigned long buf_len)
1043 {
1044         struct iscsi_conn *conn = beiscsi_conn->conn;
1045         struct iscsi_session *session = conn->session;
1046         struct iscsi_task *task;
1047         struct beiscsi_io_task *io_task;
1048         struct iscsi_hdr *login_hdr;
1049
1050         switch (ppdu->dw[offsetof(struct amap_pdu_base, opcode) / 32] &
1051                                                 PDUBASE_OPCODE_MASK) {
1052         case ISCSI_OP_NOOP_IN:
1053                 pbuffer = NULL;
1054                 buf_len = 0;
1055                 break;
1056         case ISCSI_OP_ASYNC_EVENT:
1057                 break;
1058         case ISCSI_OP_REJECT:
1059                 WARN_ON(!pbuffer);
1060                 WARN_ON(!(buf_len == 48));
1061                 beiscsi_log(phba, KERN_ERR,
1062                             BEISCSI_LOG_CONFIG | BEISCSI_LOG_IO,
1063                             "BM_%d : In ISCSI_OP_REJECT\n");
1064                 break;
1065         case ISCSI_OP_LOGIN_RSP:
1066         case ISCSI_OP_TEXT_RSP:
1067                 task = conn->login_task;
1068                 io_task = task->dd_data;
1069                 login_hdr = (struct iscsi_hdr *)ppdu;
1070                 login_hdr->itt = io_task->libiscsi_itt;
1071                 break;
1072         default:
1073                 beiscsi_log(phba, KERN_WARNING,
1074                             BEISCSI_LOG_IO | BEISCSI_LOG_CONFIG,
1075                             "BM_%d : Unrecognized opcode 0x%x in async msg\n",
1076                             (ppdu->
1077                              dw[offsetof(struct amap_pdu_base, opcode) / 32]
1078                              & PDUBASE_OPCODE_MASK));
1079                 return 1;
1080         }
1081
1082         spin_lock_bh(&session->lock);
1083         __iscsi_complete_pdu(conn, (struct iscsi_hdr *)ppdu, pbuffer, buf_len);
1084         spin_unlock_bh(&session->lock);
1085         return 0;
1086 }
1087
1088 static struct sgl_handle *alloc_io_sgl_handle(struct beiscsi_hba *phba)
1089 {
1090         struct sgl_handle *psgl_handle;
1091
1092         if (phba->io_sgl_hndl_avbl) {
1093                 beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_IO,
1094                             "BM_%d : In alloc_io_sgl_handle,"
1095                             " io_sgl_alloc_index=%d\n",
1096                             phba->io_sgl_alloc_index);
1097
1098                 psgl_handle = phba->io_sgl_hndl_base[phba->
1099                                                 io_sgl_alloc_index];
1100                 phba->io_sgl_hndl_base[phba->io_sgl_alloc_index] = NULL;
1101                 phba->io_sgl_hndl_avbl--;
1102                 if (phba->io_sgl_alloc_index == (phba->params.
1103                                                  ios_per_ctrl - 1))
1104                         phba->io_sgl_alloc_index = 0;
1105                 else
1106                         phba->io_sgl_alloc_index++;
1107         } else
1108                 psgl_handle = NULL;
1109         return psgl_handle;
1110 }
1111
1112 static void
1113 free_io_sgl_handle(struct beiscsi_hba *phba, struct sgl_handle *psgl_handle)
1114 {
1115         beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_IO,
1116                     "BM_%d : In free_,io_sgl_free_index=%d\n",
1117                     phba->io_sgl_free_index);
1118
1119         if (phba->io_sgl_hndl_base[phba->io_sgl_free_index]) {
1120                 /*
1121                  * this can happen if clean_task is called on a task that
1122                  * failed in xmit_task or alloc_pdu.
1123                  */
1124                  beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_IO,
1125                              "BM_%d : Double Free in IO SGL io_sgl_free_index=%d,"
1126                              "value there=%p\n", phba->io_sgl_free_index,
1127                              phba->io_sgl_hndl_base
1128                              [phba->io_sgl_free_index]);
1129                 return;
1130         }
1131         phba->io_sgl_hndl_base[phba->io_sgl_free_index] = psgl_handle;
1132         phba->io_sgl_hndl_avbl++;
1133         if (phba->io_sgl_free_index == (phba->params.ios_per_ctrl - 1))
1134                 phba->io_sgl_free_index = 0;
1135         else
1136                 phba->io_sgl_free_index++;
1137 }
1138
1139 /**
1140  * alloc_wrb_handle - To allocate a wrb handle
1141  * @phba: The hba pointer
1142  * @cid: The cid to use for allocation
1143  *
1144  * This happens under session_lock until submission to chip
1145  */
1146 struct wrb_handle *alloc_wrb_handle(struct beiscsi_hba *phba, unsigned int cid)
1147 {
1148         struct hwi_wrb_context *pwrb_context;
1149         struct hwi_controller *phwi_ctrlr;
1150         struct wrb_handle *pwrb_handle, *pwrb_handle_tmp;
1151
1152         phwi_ctrlr = phba->phwi_ctrlr;
1153         pwrb_context = &phwi_ctrlr->wrb_context[cid];
1154         if (pwrb_context->wrb_handles_available >= 2) {
1155                 pwrb_handle = pwrb_context->pwrb_handle_base[
1156                                             pwrb_context->alloc_index];
1157                 pwrb_context->wrb_handles_available--;
1158                 if (pwrb_context->alloc_index ==
1159                                                 (phba->params.wrbs_per_cxn - 1))
1160                         pwrb_context->alloc_index = 0;
1161                 else
1162                         pwrb_context->alloc_index++;
1163                 pwrb_handle_tmp = pwrb_context->pwrb_handle_base[
1164                                                 pwrb_context->alloc_index];
1165                 pwrb_handle->nxt_wrb_index = pwrb_handle_tmp->wrb_index;
1166         } else
1167                 pwrb_handle = NULL;
1168         return pwrb_handle;
1169 }
1170
1171 /**
1172  * free_wrb_handle - To free the wrb handle back to pool
1173  * @phba: The hba pointer
1174  * @pwrb_context: The context to free from
1175  * @pwrb_handle: The wrb_handle to free
1176  *
1177  * This happens under session_lock until submission to chip
1178  */
1179 static void
1180 free_wrb_handle(struct beiscsi_hba *phba, struct hwi_wrb_context *pwrb_context,
1181                 struct wrb_handle *pwrb_handle)
1182 {
1183         pwrb_context->pwrb_handle_base[pwrb_context->free_index] = pwrb_handle;
1184         pwrb_context->wrb_handles_available++;
1185         if (pwrb_context->free_index == (phba->params.wrbs_per_cxn - 1))
1186                 pwrb_context->free_index = 0;
1187         else
1188                 pwrb_context->free_index++;
1189
1190         beiscsi_log(phba, KERN_INFO,
1191                     BEISCSI_LOG_IO | BEISCSI_LOG_CONFIG,
1192                     "BM_%d : FREE WRB: pwrb_handle=%p free_index=0x%x"
1193                     "wrb_handles_available=%d\n",
1194                     pwrb_handle, pwrb_context->free_index,
1195                     pwrb_context->wrb_handles_available);
1196 }
1197
1198 static struct sgl_handle *alloc_mgmt_sgl_handle(struct beiscsi_hba *phba)
1199 {
1200         struct sgl_handle *psgl_handle;
1201
1202         if (phba->eh_sgl_hndl_avbl) {
1203                 psgl_handle = phba->eh_sgl_hndl_base[phba->eh_sgl_alloc_index];
1204                 phba->eh_sgl_hndl_base[phba->eh_sgl_alloc_index] = NULL;
1205                 beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_CONFIG,
1206                             "BM_%d : mgmt_sgl_alloc_index=%d=0x%x\n",
1207                             phba->eh_sgl_alloc_index,
1208                             phba->eh_sgl_alloc_index);
1209
1210                 phba->eh_sgl_hndl_avbl--;
1211                 if (phba->eh_sgl_alloc_index ==
1212                     (phba->params.icds_per_ctrl - phba->params.ios_per_ctrl -
1213                      1))
1214                         phba->eh_sgl_alloc_index = 0;
1215                 else
1216                         phba->eh_sgl_alloc_index++;
1217         } else
1218                 psgl_handle = NULL;
1219         return psgl_handle;
1220 }
1221
1222 void
1223 free_mgmt_sgl_handle(struct beiscsi_hba *phba, struct sgl_handle *psgl_handle)
1224 {
1225
1226         beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_CONFIG,
1227                     "BM_%d : In  free_mgmt_sgl_handle,"
1228                     "eh_sgl_free_index=%d\n",
1229                     phba->eh_sgl_free_index);
1230
1231         if (phba->eh_sgl_hndl_base[phba->eh_sgl_free_index]) {
1232                 /*
1233                  * this can happen if clean_task is called on a task that
1234                  * failed in xmit_task or alloc_pdu.
1235                  */
1236                 beiscsi_log(phba, KERN_WARNING, BEISCSI_LOG_CONFIG,
1237                             "BM_%d : Double Free in eh SGL ,"
1238                             "eh_sgl_free_index=%d\n",
1239                             phba->eh_sgl_free_index);
1240                 return;
1241         }
1242         phba->eh_sgl_hndl_base[phba->eh_sgl_free_index] = psgl_handle;
1243         phba->eh_sgl_hndl_avbl++;
1244         if (phba->eh_sgl_free_index ==
1245             (phba->params.icds_per_ctrl - phba->params.ios_per_ctrl - 1))
1246                 phba->eh_sgl_free_index = 0;
1247         else
1248                 phba->eh_sgl_free_index++;
1249 }
1250
1251 static void
1252 be_complete_io(struct beiscsi_conn *beiscsi_conn,
1253                 struct iscsi_task *task,
1254                 struct common_sol_cqe *csol_cqe)
1255 {
1256         struct beiscsi_io_task *io_task = task->dd_data;
1257         struct be_status_bhs *sts_bhs =
1258                                 (struct be_status_bhs *)io_task->cmd_bhs;
1259         struct iscsi_conn *conn = beiscsi_conn->conn;
1260         unsigned char *sense;
1261         u32 resid = 0, exp_cmdsn, max_cmdsn;
1262         u8 rsp, status, flags;
1263
1264         exp_cmdsn = csol_cqe->exp_cmdsn;
1265         max_cmdsn = (csol_cqe->exp_cmdsn +
1266                      csol_cqe->cmd_wnd - 1);
1267         rsp = csol_cqe->i_resp;
1268         status = csol_cqe->i_sts;
1269         flags = csol_cqe->i_flags;
1270         resid = csol_cqe->res_cnt;
1271
1272         if (!task->sc) {
1273                 if (io_task->scsi_cmnd)
1274                         scsi_dma_unmap(io_task->scsi_cmnd);
1275
1276                 return;
1277         }
1278         task->sc->result = (DID_OK << 16) | status;
1279         if (rsp != ISCSI_STATUS_CMD_COMPLETED) {
1280                 task->sc->result = DID_ERROR << 16;
1281                 goto unmap;
1282         }
1283
1284         /* bidi not initially supported */
1285         if (flags & (ISCSI_FLAG_CMD_UNDERFLOW | ISCSI_FLAG_CMD_OVERFLOW)) {
1286                 if (!status && (flags & ISCSI_FLAG_CMD_OVERFLOW))
1287                         task->sc->result = DID_ERROR << 16;
1288
1289                 if (flags & ISCSI_FLAG_CMD_UNDERFLOW) {
1290                         scsi_set_resid(task->sc, resid);
1291                         if (!status && (scsi_bufflen(task->sc) - resid <
1292                             task->sc->underflow))
1293                                 task->sc->result = DID_ERROR << 16;
1294                 }
1295         }
1296
1297         if (status == SAM_STAT_CHECK_CONDITION) {
1298                 u16 sense_len;
1299                 unsigned short *slen = (unsigned short *)sts_bhs->sense_info;
1300
1301                 sense = sts_bhs->sense_info + sizeof(unsigned short);
1302                 sense_len = be16_to_cpu(*slen);
1303                 memcpy(task->sc->sense_buffer, sense,
1304                        min_t(u16, sense_len, SCSI_SENSE_BUFFERSIZE));
1305         }
1306
1307         if (io_task->cmd_bhs->iscsi_hdr.flags & ISCSI_FLAG_CMD_READ)
1308                 conn->rxdata_octets += resid;
1309 unmap:
1310         scsi_dma_unmap(io_task->scsi_cmnd);
1311         iscsi_complete_scsi_task(task, exp_cmdsn, max_cmdsn);
1312 }
1313
1314 static void
1315 be_complete_logout(struct beiscsi_conn *beiscsi_conn,
1316                     struct iscsi_task *task,
1317                     struct common_sol_cqe *csol_cqe)
1318 {
1319         struct iscsi_logout_rsp *hdr;
1320         struct beiscsi_io_task *io_task = task->dd_data;
1321         struct iscsi_conn *conn = beiscsi_conn->conn;
1322
1323         hdr = (struct iscsi_logout_rsp *)task->hdr;
1324         hdr->opcode = ISCSI_OP_LOGOUT_RSP;
1325         hdr->t2wait = 5;
1326         hdr->t2retain = 0;
1327         hdr->flags = csol_cqe->i_flags;
1328         hdr->response = csol_cqe->i_resp;
1329         hdr->exp_cmdsn = csol_cqe->exp_cmdsn;
1330         hdr->max_cmdsn = (csol_cqe->exp_cmdsn + csol_cqe->cmd_wnd - 1);
1331
1332         hdr->dlength[0] = 0;
1333         hdr->dlength[1] = 0;
1334         hdr->dlength[2] = 0;
1335         hdr->hlength = 0;
1336         hdr->itt = io_task->libiscsi_itt;
1337         __iscsi_complete_pdu(conn, (struct iscsi_hdr *)hdr, NULL, 0);
1338 }
1339
1340 static void
1341 be_complete_tmf(struct beiscsi_conn *beiscsi_conn,
1342                  struct iscsi_task *task,
1343                  struct common_sol_cqe *csol_cqe)
1344 {
1345         struct iscsi_tm_rsp *hdr;
1346         struct iscsi_conn *conn = beiscsi_conn->conn;
1347         struct beiscsi_io_task *io_task = task->dd_data;
1348
1349         hdr = (struct iscsi_tm_rsp *)task->hdr;
1350         hdr->opcode = ISCSI_OP_SCSI_TMFUNC_RSP;
1351         hdr->flags = csol_cqe->i_flags;
1352         hdr->response = csol_cqe->i_resp;
1353         hdr->exp_cmdsn = csol_cqe->exp_cmdsn;
1354         hdr->max_cmdsn = (csol_cqe->exp_cmdsn +
1355                           csol_cqe->cmd_wnd - 1);
1356
1357         hdr->itt = io_task->libiscsi_itt;
1358         __iscsi_complete_pdu(conn, (struct iscsi_hdr *)hdr, NULL, 0);
1359 }
1360
1361 static void
1362 hwi_complete_drvr_msgs(struct beiscsi_conn *beiscsi_conn,
1363                        struct beiscsi_hba *phba, struct sol_cqe *psol)
1364 {
1365         struct hwi_wrb_context *pwrb_context;
1366         struct wrb_handle *pwrb_handle = NULL;
1367         struct hwi_controller *phwi_ctrlr;
1368         struct iscsi_task *task;
1369         struct beiscsi_io_task *io_task;
1370         struct iscsi_conn *conn = beiscsi_conn->conn;
1371         struct iscsi_session *session = conn->session;
1372         uint16_t wrb_index, cid;
1373
1374         phwi_ctrlr = phba->phwi_ctrlr;
1375         if (chip_skh_r(phba->pcidev)) {
1376                 wrb_index = AMAP_GET_BITS(struct amap_it_dmsg_cqe_v2,
1377                                           wrb_idx, psol);
1378                 cid = AMAP_GET_BITS(struct amap_it_dmsg_cqe_v2,
1379                                     cid, psol);
1380         } else {
1381                 wrb_index = AMAP_GET_BITS(struct amap_it_dmsg_cqe,
1382                                           wrb_idx, psol);
1383                 cid = AMAP_GET_BITS(struct amap_it_dmsg_cqe,
1384                                     cid, psol);
1385         }
1386
1387         pwrb_context = &phwi_ctrlr->wrb_context[
1388                         cid - phba->fw_config.iscsi_cid_start];
1389         pwrb_handle = pwrb_context->pwrb_handle_basestd[wrb_index];
1390         task = pwrb_handle->pio_handle;
1391
1392         io_task = task->dd_data;
1393         spin_lock_bh(&phba->mgmt_sgl_lock);
1394         free_mgmt_sgl_handle(phba, io_task->psgl_handle);
1395         spin_unlock_bh(&phba->mgmt_sgl_lock);
1396         spin_lock_bh(&session->lock);
1397         free_wrb_handle(phba, pwrb_context, pwrb_handle);
1398         spin_unlock_bh(&session->lock);
1399 }
1400
1401 static void
1402 be_complete_nopin_resp(struct beiscsi_conn *beiscsi_conn,
1403                         struct iscsi_task *task,
1404                         struct common_sol_cqe *csol_cqe)
1405 {
1406         struct iscsi_nopin *hdr;
1407         struct iscsi_conn *conn = beiscsi_conn->conn;
1408         struct beiscsi_io_task *io_task = task->dd_data;
1409
1410         hdr = (struct iscsi_nopin *)task->hdr;
1411         hdr->flags = csol_cqe->i_flags;
1412         hdr->exp_cmdsn = cpu_to_be32(csol_cqe->exp_cmdsn);
1413         hdr->max_cmdsn = be32_to_cpu(hdr->exp_cmdsn +
1414                          csol_cqe->cmd_wnd - 1);
1415
1416         hdr->opcode = ISCSI_OP_NOOP_IN;
1417         hdr->itt = io_task->libiscsi_itt;
1418         __iscsi_complete_pdu(conn, (struct iscsi_hdr *)hdr, NULL, 0);
1419 }
1420
1421 static void adapter_get_sol_cqe(struct beiscsi_hba *phba,
1422                 struct sol_cqe *psol,
1423                 struct common_sol_cqe *csol_cqe)
1424 {
1425         if (chip_skh_r(phba->pcidev)) {
1426                 csol_cqe->exp_cmdsn = AMAP_GET_BITS(struct amap_sol_cqe_v2,
1427                                                     i_exp_cmd_sn, psol);
1428                 csol_cqe->res_cnt = AMAP_GET_BITS(struct amap_sol_cqe_v2,
1429                                                   i_res_cnt, psol);
1430                 csol_cqe->wrb_index = AMAP_GET_BITS(struct amap_sol_cqe_v2,
1431                                                     wrb_index, psol);
1432                 csol_cqe->cid = AMAP_GET_BITS(struct amap_sol_cqe_v2,
1433                                               cid, psol);
1434                 csol_cqe->hw_sts = AMAP_GET_BITS(struct amap_sol_cqe_v2,
1435                                                  hw_sts, psol);
1436                 csol_cqe->cmd_wnd = AMAP_GET_BITS(struct amap_sol_cqe,
1437                                                   i_cmd_wnd, psol);
1438                 if (AMAP_GET_BITS(struct amap_sol_cqe_v2,
1439                                   cmd_cmpl, psol))
1440                         csol_cqe->i_sts = AMAP_GET_BITS(struct amap_sol_cqe_v2,
1441                                                         i_sts, psol);
1442                 else
1443                         csol_cqe->i_resp = AMAP_GET_BITS(struct amap_sol_cqe_v2,
1444                                                          i_sts, psol);
1445                 if (AMAP_GET_BITS(struct amap_sol_cqe_v2,
1446                                   u, psol))
1447                         csol_cqe->i_flags = ISCSI_FLAG_CMD_UNDERFLOW;
1448
1449                 if (AMAP_GET_BITS(struct amap_sol_cqe_v2,
1450                                   o, psol))
1451                         csol_cqe->i_flags |= ISCSI_FLAG_CMD_OVERFLOW;
1452         } else {
1453                 csol_cqe->exp_cmdsn = AMAP_GET_BITS(struct amap_sol_cqe,
1454                                                     i_exp_cmd_sn, psol);
1455                 csol_cqe->res_cnt = AMAP_GET_BITS(struct amap_sol_cqe,
1456                                                   i_res_cnt, psol);
1457                 csol_cqe->cmd_wnd = AMAP_GET_BITS(struct amap_sol_cqe,
1458                                                   i_cmd_wnd, psol);
1459                 csol_cqe->wrb_index = AMAP_GET_BITS(struct amap_sol_cqe,
1460                                                     wrb_index, psol);
1461                 csol_cqe->cid = AMAP_GET_BITS(struct amap_sol_cqe,
1462                                               cid, psol);
1463                 csol_cqe->hw_sts = AMAP_GET_BITS(struct amap_sol_cqe,
1464                                                  hw_sts, psol);
1465                 csol_cqe->i_resp = AMAP_GET_BITS(struct amap_sol_cqe,
1466                                                  i_resp, psol);
1467                 csol_cqe->i_sts = AMAP_GET_BITS(struct amap_sol_cqe,
1468                                                 i_sts, psol);
1469                 csol_cqe->i_flags = AMAP_GET_BITS(struct amap_sol_cqe,
1470                                                   i_flags, psol);
1471         }
1472 }
1473
1474
1475 static void hwi_complete_cmd(struct beiscsi_conn *beiscsi_conn,
1476                              struct beiscsi_hba *phba, struct sol_cqe *psol)
1477 {
1478         struct hwi_wrb_context *pwrb_context;
1479         struct wrb_handle *pwrb_handle;
1480         struct iscsi_wrb *pwrb = NULL;
1481         struct hwi_controller *phwi_ctrlr;
1482         struct iscsi_task *task;
1483         unsigned int type;
1484         struct iscsi_conn *conn = beiscsi_conn->conn;
1485         struct iscsi_session *session = conn->session;
1486         struct common_sol_cqe csol_cqe = {0};
1487
1488         phwi_ctrlr = phba->phwi_ctrlr;
1489
1490         /* Copy the elements to a common structure */
1491         adapter_get_sol_cqe(phba, psol, &csol_cqe);
1492
1493         pwrb_context = &phwi_ctrlr->wrb_context[
1494                         csol_cqe.cid - phba->fw_config.iscsi_cid_start];
1495
1496         pwrb_handle = pwrb_context->pwrb_handle_basestd[
1497                       csol_cqe.wrb_index];
1498
1499         task = pwrb_handle->pio_handle;
1500         pwrb = pwrb_handle->pwrb;
1501         type = ((struct beiscsi_io_task *)task->dd_data)->wrb_type;
1502
1503         spin_lock_bh(&session->lock);
1504         switch (type) {
1505         case HWH_TYPE_IO:
1506         case HWH_TYPE_IO_RD:
1507                 if ((task->hdr->opcode & ISCSI_OPCODE_MASK) ==
1508                      ISCSI_OP_NOOP_OUT)
1509                         be_complete_nopin_resp(beiscsi_conn, task, &csol_cqe);
1510                 else
1511                         be_complete_io(beiscsi_conn, task, &csol_cqe);
1512                 break;
1513
1514         case HWH_TYPE_LOGOUT:
1515                 if ((task->hdr->opcode & ISCSI_OPCODE_MASK) == ISCSI_OP_LOGOUT)
1516                         be_complete_logout(beiscsi_conn, task, &csol_cqe);
1517                 else
1518                         be_complete_tmf(beiscsi_conn, task, &csol_cqe);
1519                 break;
1520
1521         case HWH_TYPE_LOGIN:
1522                 beiscsi_log(phba, KERN_ERR,
1523                             BEISCSI_LOG_CONFIG | BEISCSI_LOG_IO,
1524                             "BM_%d :\t\t No HWH_TYPE_LOGIN Expected in"
1525                             " hwi_complete_cmd- Solicited path\n");
1526                 break;
1527
1528         case HWH_TYPE_NOP:
1529                 be_complete_nopin_resp(beiscsi_conn, task, &csol_cqe);
1530                 break;
1531
1532         default:
1533                 beiscsi_log(phba, KERN_WARNING,
1534                             BEISCSI_LOG_CONFIG | BEISCSI_LOG_IO,
1535                             "BM_%d : In hwi_complete_cmd, unknown type = %d"
1536                             "wrb_index 0x%x CID 0x%x\n", type,
1537                             csol_cqe.wrb_index,
1538                             csol_cqe.cid);
1539                 break;
1540         }
1541
1542         spin_unlock_bh(&session->lock);
1543 }
1544
1545 static struct list_head *hwi_get_async_busy_list(struct hwi_async_pdu_context
1546                                           *pasync_ctx, unsigned int is_header,
1547                                           unsigned int host_write_ptr)
1548 {
1549         if (is_header)
1550                 return &pasync_ctx->async_entry[host_write_ptr].
1551                     header_busy_list;
1552         else
1553                 return &pasync_ctx->async_entry[host_write_ptr].data_busy_list;
1554 }
1555
1556 static struct async_pdu_handle *
1557 hwi_get_async_handle(struct beiscsi_hba *phba,
1558                      struct beiscsi_conn *beiscsi_conn,
1559                      struct hwi_async_pdu_context *pasync_ctx,
1560                      struct i_t_dpdu_cqe *pdpdu_cqe, unsigned int *pcq_index)
1561 {
1562         struct be_bus_address phys_addr;
1563         struct list_head *pbusy_list;
1564         struct async_pdu_handle *pasync_handle = NULL;
1565         unsigned char is_header = 0;
1566         unsigned int index, dpl;
1567
1568         if (chip_skh_r(phba->pcidev)) {
1569                 dpl = AMAP_GET_BITS(struct amap_i_t_dpdu_cqe_v2,
1570                                     dpl, pdpdu_cqe);
1571                 index = AMAP_GET_BITS(struct amap_i_t_dpdu_cqe_v2,
1572                                       index, pdpdu_cqe);
1573         } else {
1574                 dpl = AMAP_GET_BITS(struct amap_i_t_dpdu_cqe,
1575                                     dpl, pdpdu_cqe);
1576                 index = AMAP_GET_BITS(struct amap_i_t_dpdu_cqe,
1577                                       index, pdpdu_cqe);
1578         }
1579
1580         phys_addr.u.a32.address_lo =
1581                 (pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe,
1582                                         db_addr_lo) / 32] - dpl);
1583         phys_addr.u.a32.address_hi =
1584                 pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe,
1585                                        db_addr_hi) / 32];
1586
1587         phys_addr.u.a64.address =
1588                         *((unsigned long long *)(&phys_addr.u.a64.address));
1589
1590         switch (pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe, code) / 32]
1591                         & PDUCQE_CODE_MASK) {
1592         case UNSOL_HDR_NOTIFY:
1593                 is_header = 1;
1594
1595                  pbusy_list = hwi_get_async_busy_list(pasync_ctx,
1596                                                       is_header, index);
1597                 break;
1598         case UNSOL_DATA_NOTIFY:
1599                  pbusy_list = hwi_get_async_busy_list(pasync_ctx,
1600                                                       is_header, index);
1601                 break;
1602         default:
1603                 pbusy_list = NULL;
1604                 beiscsi_log(phba, KERN_WARNING,
1605                             BEISCSI_LOG_IO | BEISCSI_LOG_CONFIG,
1606                             "BM_%d : Unexpected code=%d\n",
1607                             pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe,
1608                             code) / 32] & PDUCQE_CODE_MASK);
1609                 return NULL;
1610         }
1611
1612         WARN_ON(list_empty(pbusy_list));
1613         list_for_each_entry(pasync_handle, pbusy_list, link) {
1614                 if (pasync_handle->pa.u.a64.address == phys_addr.u.a64.address)
1615                         break;
1616         }
1617
1618         WARN_ON(!pasync_handle);
1619
1620         pasync_handle->cri = (unsigned short)beiscsi_conn->beiscsi_conn_cid -
1621                                              phba->fw_config.iscsi_cid_start;
1622         pasync_handle->is_header = is_header;
1623         pasync_handle->buffer_len = dpl;
1624         *pcq_index = index;
1625
1626         return pasync_handle;
1627 }
1628
1629 static unsigned int
1630 hwi_update_async_writables(struct beiscsi_hba *phba,
1631                             struct hwi_async_pdu_context *pasync_ctx,
1632                             unsigned int is_header, unsigned int cq_index)
1633 {
1634         struct list_head *pbusy_list;
1635         struct async_pdu_handle *pasync_handle;
1636         unsigned int num_entries, writables = 0;
1637         unsigned int *pep_read_ptr, *pwritables;
1638
1639         num_entries = pasync_ctx->num_entries;
1640         if (is_header) {
1641                 pep_read_ptr = &pasync_ctx->async_header.ep_read_ptr;
1642                 pwritables = &pasync_ctx->async_header.writables;
1643         } else {
1644                 pep_read_ptr = &pasync_ctx->async_data.ep_read_ptr;
1645                 pwritables = &pasync_ctx->async_data.writables;
1646         }
1647
1648         while ((*pep_read_ptr) != cq_index) {
1649                 (*pep_read_ptr)++;
1650                 *pep_read_ptr = (*pep_read_ptr) % num_entries;
1651
1652                 pbusy_list = hwi_get_async_busy_list(pasync_ctx, is_header,
1653                                                      *pep_read_ptr);
1654                 if (writables == 0)
1655                         WARN_ON(list_empty(pbusy_list));
1656
1657                 if (!list_empty(pbusy_list)) {
1658                         pasync_handle = list_entry(pbusy_list->next,
1659                                                    struct async_pdu_handle,
1660                                                    link);
1661                         WARN_ON(!pasync_handle);
1662                         pasync_handle->consumed = 1;
1663                 }
1664
1665                 writables++;
1666         }
1667
1668         if (!writables) {
1669                 beiscsi_log(phba, KERN_ERR,
1670                             BEISCSI_LOG_CONFIG | BEISCSI_LOG_IO,
1671                             "BM_%d : Duplicate notification received - index 0x%x!!\n",
1672                             cq_index);
1673                 WARN_ON(1);
1674         }
1675
1676         *pwritables = *pwritables + writables;
1677         return 0;
1678 }
1679
1680 static void hwi_free_async_msg(struct beiscsi_hba *phba,
1681                                        unsigned int cri)
1682 {
1683         struct hwi_controller *phwi_ctrlr;
1684         struct hwi_async_pdu_context *pasync_ctx;
1685         struct async_pdu_handle *pasync_handle, *tmp_handle;
1686         struct list_head *plist;
1687
1688         phwi_ctrlr = phba->phwi_ctrlr;
1689         pasync_ctx = HWI_GET_ASYNC_PDU_CTX(phwi_ctrlr);
1690
1691         plist  = &pasync_ctx->async_entry[cri].wait_queue.list;
1692
1693         list_for_each_entry_safe(pasync_handle, tmp_handle, plist, link) {
1694                 list_del(&pasync_handle->link);
1695
1696                 if (pasync_handle->is_header) {
1697                         list_add_tail(&pasync_handle->link,
1698                                       &pasync_ctx->async_header.free_list);
1699                         pasync_ctx->async_header.free_entries++;
1700                 } else {
1701                         list_add_tail(&pasync_handle->link,
1702                                       &pasync_ctx->async_data.free_list);
1703                         pasync_ctx->async_data.free_entries++;
1704                 }
1705         }
1706
1707         INIT_LIST_HEAD(&pasync_ctx->async_entry[cri].wait_queue.list);
1708         pasync_ctx->async_entry[cri].wait_queue.hdr_received = 0;
1709         pasync_ctx->async_entry[cri].wait_queue.bytes_received = 0;
1710 }
1711
1712 static struct phys_addr *
1713 hwi_get_ring_address(struct hwi_async_pdu_context *pasync_ctx,
1714                      unsigned int is_header, unsigned int host_write_ptr)
1715 {
1716         struct phys_addr *pasync_sge = NULL;
1717
1718         if (is_header)
1719                 pasync_sge = pasync_ctx->async_header.ring_base;
1720         else
1721                 pasync_sge = pasync_ctx->async_data.ring_base;
1722
1723         return pasync_sge + host_write_ptr;
1724 }
1725
1726 static void hwi_post_async_buffers(struct beiscsi_hba *phba,
1727                                    unsigned int is_header)
1728 {
1729         struct hwi_controller *phwi_ctrlr;
1730         struct hwi_async_pdu_context *pasync_ctx;
1731         struct async_pdu_handle *pasync_handle;
1732         struct list_head *pfree_link, *pbusy_list;
1733         struct phys_addr *pasync_sge;
1734         unsigned int ring_id, num_entries;
1735         unsigned int host_write_num;
1736         unsigned int writables;
1737         unsigned int i = 0;
1738         u32 doorbell = 0;
1739
1740         phwi_ctrlr = phba->phwi_ctrlr;
1741         pasync_ctx = HWI_GET_ASYNC_PDU_CTX(phwi_ctrlr);
1742         num_entries = pasync_ctx->num_entries;
1743
1744         if (is_header) {
1745                 writables = min(pasync_ctx->async_header.writables,
1746                                 pasync_ctx->async_header.free_entries);
1747                 pfree_link = pasync_ctx->async_header.free_list.next;
1748                 host_write_num = pasync_ctx->async_header.host_write_ptr;
1749                 ring_id = phwi_ctrlr->default_pdu_hdr.id;
1750         } else {
1751                 writables = min(pasync_ctx->async_data.writables,
1752                                 pasync_ctx->async_data.free_entries);
1753                 pfree_link = pasync_ctx->async_data.free_list.next;
1754                 host_write_num = pasync_ctx->async_data.host_write_ptr;
1755                 ring_id = phwi_ctrlr->default_pdu_data.id;
1756         }
1757
1758         writables = (writables / 8) * 8;
1759         if (writables) {
1760                 for (i = 0; i < writables; i++) {
1761                         pbusy_list =
1762                             hwi_get_async_busy_list(pasync_ctx, is_header,
1763                                                     host_write_num);
1764                         pasync_handle =
1765                             list_entry(pfree_link, struct async_pdu_handle,
1766                                                                 link);
1767                         WARN_ON(!pasync_handle);
1768                         pasync_handle->consumed = 0;
1769
1770                         pfree_link = pfree_link->next;
1771
1772                         pasync_sge = hwi_get_ring_address(pasync_ctx,
1773                                                 is_header, host_write_num);
1774
1775                         pasync_sge->hi = pasync_handle->pa.u.a32.address_lo;
1776                         pasync_sge->lo = pasync_handle->pa.u.a32.address_hi;
1777
1778                         list_move(&pasync_handle->link, pbusy_list);
1779
1780                         host_write_num++;
1781                         host_write_num = host_write_num % num_entries;
1782                 }
1783
1784                 if (is_header) {
1785                         pasync_ctx->async_header.host_write_ptr =
1786                                                         host_write_num;
1787                         pasync_ctx->async_header.free_entries -= writables;
1788                         pasync_ctx->async_header.writables -= writables;
1789                         pasync_ctx->async_header.busy_entries += writables;
1790                 } else {
1791                         pasync_ctx->async_data.host_write_ptr = host_write_num;
1792                         pasync_ctx->async_data.free_entries -= writables;
1793                         pasync_ctx->async_data.writables -= writables;
1794                         pasync_ctx->async_data.busy_entries += writables;
1795                 }
1796
1797                 doorbell |= ring_id & DB_DEF_PDU_RING_ID_MASK;
1798                 doorbell |= 1 << DB_DEF_PDU_REARM_SHIFT;
1799                 doorbell |= 0 << DB_DEF_PDU_EVENT_SHIFT;
1800                 doorbell |= (writables & DB_DEF_PDU_CQPROC_MASK)
1801                                         << DB_DEF_PDU_CQPROC_SHIFT;
1802
1803                 iowrite32(doorbell, phba->db_va + DB_RXULP0_OFFSET);
1804         }
1805 }
1806
1807 static void hwi_flush_default_pdu_buffer(struct beiscsi_hba *phba,
1808                                          struct beiscsi_conn *beiscsi_conn,
1809                                          struct i_t_dpdu_cqe *pdpdu_cqe)
1810 {
1811         struct hwi_controller *phwi_ctrlr;
1812         struct hwi_async_pdu_context *pasync_ctx;
1813         struct async_pdu_handle *pasync_handle = NULL;
1814         unsigned int cq_index = -1;
1815
1816         phwi_ctrlr = phba->phwi_ctrlr;
1817         pasync_ctx = HWI_GET_ASYNC_PDU_CTX(phwi_ctrlr);
1818
1819         pasync_handle = hwi_get_async_handle(phba, beiscsi_conn, pasync_ctx,
1820                                              pdpdu_cqe, &cq_index);
1821         BUG_ON(pasync_handle->is_header != 0);
1822         if (pasync_handle->consumed == 0)
1823                 hwi_update_async_writables(phba, pasync_ctx,
1824                                            pasync_handle->is_header, cq_index);
1825
1826         hwi_free_async_msg(phba, pasync_handle->cri);
1827         hwi_post_async_buffers(phba, pasync_handle->is_header);
1828 }
1829
1830 static unsigned int
1831 hwi_fwd_async_msg(struct beiscsi_conn *beiscsi_conn,
1832                   struct beiscsi_hba *phba,
1833                   struct hwi_async_pdu_context *pasync_ctx, unsigned short cri)
1834 {
1835         struct list_head *plist;
1836         struct async_pdu_handle *pasync_handle;
1837         void *phdr = NULL;
1838         unsigned int hdr_len = 0, buf_len = 0;
1839         unsigned int status, index = 0, offset = 0;
1840         void *pfirst_buffer = NULL;
1841         unsigned int num_buf = 0;
1842
1843         plist = &pasync_ctx->async_entry[cri].wait_queue.list;
1844
1845         list_for_each_entry(pasync_handle, plist, link) {
1846                 if (index == 0) {
1847                         phdr = pasync_handle->pbuffer;
1848                         hdr_len = pasync_handle->buffer_len;
1849                 } else {
1850                         buf_len = pasync_handle->buffer_len;
1851                         if (!num_buf) {
1852                                 pfirst_buffer = pasync_handle->pbuffer;
1853                                 num_buf++;
1854                         }
1855                         memcpy(pfirst_buffer + offset,
1856                                pasync_handle->pbuffer, buf_len);
1857                         offset += buf_len;
1858                 }
1859                 index++;
1860         }
1861
1862         status = beiscsi_process_async_pdu(beiscsi_conn, phba,
1863                                            (beiscsi_conn->beiscsi_conn_cid -
1864                                             phba->fw_config.iscsi_cid_start),
1865                                             phdr, hdr_len, pfirst_buffer,
1866                                             offset);
1867
1868         hwi_free_async_msg(phba, cri);
1869         return 0;
1870 }
1871
1872 static unsigned int
1873 hwi_gather_async_pdu(struct beiscsi_conn *beiscsi_conn,
1874                      struct beiscsi_hba *phba,
1875                      struct async_pdu_handle *pasync_handle)
1876 {
1877         struct hwi_async_pdu_context *pasync_ctx;
1878         struct hwi_controller *phwi_ctrlr;
1879         unsigned int bytes_needed = 0, status = 0;
1880         unsigned short cri = pasync_handle->cri;
1881         struct pdu_base *ppdu;
1882
1883         phwi_ctrlr = phba->phwi_ctrlr;
1884         pasync_ctx = HWI_GET_ASYNC_PDU_CTX(phwi_ctrlr);
1885
1886         list_del(&pasync_handle->link);
1887         if (pasync_handle->is_header) {
1888                 pasync_ctx->async_header.busy_entries--;
1889                 if (pasync_ctx->async_entry[cri].wait_queue.hdr_received) {
1890                         hwi_free_async_msg(phba, cri);
1891                         BUG();
1892                 }
1893
1894                 pasync_ctx->async_entry[cri].wait_queue.bytes_received = 0;
1895                 pasync_ctx->async_entry[cri].wait_queue.hdr_received = 1;
1896                 pasync_ctx->async_entry[cri].wait_queue.hdr_len =
1897                                 (unsigned short)pasync_handle->buffer_len;
1898                 list_add_tail(&pasync_handle->link,
1899                               &pasync_ctx->async_entry[cri].wait_queue.list);
1900
1901                 ppdu = pasync_handle->pbuffer;
1902                 bytes_needed = ((((ppdu->dw[offsetof(struct amap_pdu_base,
1903                         data_len_hi) / 32] & PDUBASE_DATALENHI_MASK) << 8) &
1904                         0xFFFF0000) | ((be16_to_cpu((ppdu->
1905                         dw[offsetof(struct amap_pdu_base, data_len_lo) / 32]
1906                         & PDUBASE_DATALENLO_MASK) >> 16)) & 0x0000FFFF));
1907
1908                 if (status == 0) {
1909                         pasync_ctx->async_entry[cri].wait_queue.bytes_needed =
1910                             bytes_needed;
1911
1912                         if (bytes_needed == 0)
1913                                 status = hwi_fwd_async_msg(beiscsi_conn, phba,
1914                                                            pasync_ctx, cri);
1915                 }
1916         } else {
1917                 pasync_ctx->async_data.busy_entries--;
1918                 if (pasync_ctx->async_entry[cri].wait_queue.hdr_received) {
1919                         list_add_tail(&pasync_handle->link,
1920                                       &pasync_ctx->async_entry[cri].wait_queue.
1921                                       list);
1922                         pasync_ctx->async_entry[cri].wait_queue.
1923                                 bytes_received +=
1924                                 (unsigned short)pasync_handle->buffer_len;
1925
1926                         if (pasync_ctx->async_entry[cri].wait_queue.
1927                             bytes_received >=
1928                             pasync_ctx->async_entry[cri].wait_queue.
1929                             bytes_needed)
1930                                 status = hwi_fwd_async_msg(beiscsi_conn, phba,
1931                                                            pasync_ctx, cri);
1932                 }
1933         }
1934         return status;
1935 }
1936
1937 static void hwi_process_default_pdu_ring(struct beiscsi_conn *beiscsi_conn,
1938                                          struct beiscsi_hba *phba,
1939                                          struct i_t_dpdu_cqe *pdpdu_cqe)
1940 {
1941         struct hwi_controller *phwi_ctrlr;
1942         struct hwi_async_pdu_context *pasync_ctx;
1943         struct async_pdu_handle *pasync_handle = NULL;
1944         unsigned int cq_index = -1;
1945
1946         phwi_ctrlr = phba->phwi_ctrlr;
1947         pasync_ctx = HWI_GET_ASYNC_PDU_CTX(phwi_ctrlr);
1948         pasync_handle = hwi_get_async_handle(phba, beiscsi_conn, pasync_ctx,
1949                                              pdpdu_cqe, &cq_index);
1950
1951         if (pasync_handle->consumed == 0)
1952                 hwi_update_async_writables(phba, pasync_ctx,
1953                                            pasync_handle->is_header, cq_index);
1954
1955         hwi_gather_async_pdu(beiscsi_conn, phba, pasync_handle);
1956         hwi_post_async_buffers(phba, pasync_handle->is_header);
1957 }
1958
1959 static void  beiscsi_process_mcc_isr(struct beiscsi_hba *phba)
1960 {
1961         struct be_queue_info *mcc_cq;
1962         struct  be_mcc_compl *mcc_compl;
1963         unsigned int num_processed = 0;
1964
1965         mcc_cq = &phba->ctrl.mcc_obj.cq;
1966         mcc_compl = queue_tail_node(mcc_cq);
1967         mcc_compl->flags = le32_to_cpu(mcc_compl->flags);
1968         while (mcc_compl->flags & CQE_FLAGS_VALID_MASK) {
1969
1970                 if (num_processed >= 32) {
1971                         hwi_ring_cq_db(phba, mcc_cq->id,
1972                                         num_processed, 0, 0);
1973                         num_processed = 0;
1974                 }
1975                 if (mcc_compl->flags & CQE_FLAGS_ASYNC_MASK) {
1976                         /* Interpret flags as an async trailer */
1977                         if (is_link_state_evt(mcc_compl->flags))
1978                                 /* Interpret compl as a async link evt */
1979                                 beiscsi_async_link_state_process(phba,
1980                                 (struct be_async_event_link_state *) mcc_compl);
1981                         else
1982                                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_MBOX,
1983                                             "BM_%d :  Unsupported Async Event, flags"
1984                                             " = 0x%08x\n",
1985                                             mcc_compl->flags);
1986                 } else if (mcc_compl->flags & CQE_FLAGS_COMPLETED_MASK) {
1987                         be_mcc_compl_process_isr(&phba->ctrl, mcc_compl);
1988                         atomic_dec(&phba->ctrl.mcc_obj.q.used);
1989                 }
1990
1991                 mcc_compl->flags = 0;
1992                 queue_tail_inc(mcc_cq);
1993                 mcc_compl = queue_tail_node(mcc_cq);
1994                 mcc_compl->flags = le32_to_cpu(mcc_compl->flags);
1995                 num_processed++;
1996         }
1997
1998         if (num_processed > 0)
1999                 hwi_ring_cq_db(phba, mcc_cq->id, num_processed, 1, 0);
2000
2001 }
2002
2003 /**
2004  * beiscsi_process_cq()- Process the Completion Queue
2005  * @pbe_eq: Event Q on which the Completion has come
2006  *
2007  * return
2008  *     Number of Completion Entries processed.
2009  **/
2010 static unsigned int beiscsi_process_cq(struct be_eq_obj *pbe_eq)
2011 {
2012         struct be_queue_info *cq;
2013         struct sol_cqe *sol;
2014         struct dmsg_cqe *dmsg;
2015         unsigned int num_processed = 0;
2016         unsigned int tot_nump = 0;
2017         unsigned short code = 0, cid = 0;
2018         struct beiscsi_conn *beiscsi_conn;
2019         struct beiscsi_endpoint *beiscsi_ep;
2020         struct iscsi_endpoint *ep;
2021         struct beiscsi_hba *phba;
2022
2023         cq = pbe_eq->cq;
2024         sol = queue_tail_node(cq);
2025         phba = pbe_eq->phba;
2026
2027         while (sol->dw[offsetof(struct amap_sol_cqe, valid) / 32] &
2028                CQE_VALID_MASK) {
2029                 be_dws_le_to_cpu(sol, sizeof(struct sol_cqe));
2030
2031                  code = (sol->dw[offsetof(struct amap_sol_cqe, code) /
2032                          32] & CQE_CODE_MASK);
2033
2034                  /* Get the CID */
2035                 if (chip_skh_r(phba->pcidev)) {
2036                         if ((code == DRIVERMSG_NOTIFY) ||
2037                             (code == UNSOL_HDR_NOTIFY) ||
2038                             (code == UNSOL_DATA_NOTIFY))
2039                                 cid = AMAP_GET_BITS(
2040                                                     struct amap_i_t_dpdu_cqe_v2,
2041                                                     cid, sol);
2042                          else
2043                                  cid = AMAP_GET_BITS(struct amap_sol_cqe_v2,
2044                                                      cid, sol);
2045                    } else
2046                          cid = AMAP_GET_BITS(struct amap_sol_cqe, cid, sol);
2047
2048                 ep = phba->ep_array[cid - phba->fw_config.iscsi_cid_start];
2049                 beiscsi_ep = ep->dd_data;
2050                 beiscsi_conn = beiscsi_ep->conn;
2051
2052                 if (num_processed >= 32) {
2053                         hwi_ring_cq_db(phba, cq->id,
2054                                         num_processed, 0, 0);
2055                         tot_nump += num_processed;
2056                         num_processed = 0;
2057                 }
2058
2059                 switch (code) {
2060                 case SOL_CMD_COMPLETE:
2061                         hwi_complete_cmd(beiscsi_conn, phba, sol);
2062                         break;
2063                 case DRIVERMSG_NOTIFY:
2064                         beiscsi_log(phba, KERN_INFO,
2065                                     BEISCSI_LOG_IO | BEISCSI_LOG_CONFIG,
2066                                     "BM_%d : Received %s[%d] on CID : %d\n",
2067                                     cqe_desc[code], code, cid);
2068
2069                         dmsg = (struct dmsg_cqe *)sol;
2070                         hwi_complete_drvr_msgs(beiscsi_conn, phba, sol);
2071                         break;
2072                 case UNSOL_HDR_NOTIFY:
2073                         beiscsi_log(phba, KERN_INFO,
2074                                     BEISCSI_LOG_IO | BEISCSI_LOG_CONFIG,
2075                                     "BM_%d : Received %s[%d] on CID : %d\n",
2076                                     cqe_desc[code], code, cid);
2077
2078                         hwi_process_default_pdu_ring(beiscsi_conn, phba,
2079                                              (struct i_t_dpdu_cqe *)sol);
2080                         break;
2081                 case UNSOL_DATA_NOTIFY:
2082                         beiscsi_log(phba, KERN_INFO,
2083                                     BEISCSI_LOG_CONFIG | BEISCSI_LOG_IO,
2084                                     "BM_%d : Received %s[%d] on CID : %d\n",
2085                                     cqe_desc[code], code, cid);
2086
2087                         hwi_process_default_pdu_ring(beiscsi_conn, phba,
2088                                              (struct i_t_dpdu_cqe *)sol);
2089                         break;
2090                 case CXN_INVALIDATE_INDEX_NOTIFY:
2091                 case CMD_INVALIDATED_NOTIFY:
2092                 case CXN_INVALIDATE_NOTIFY:
2093                         beiscsi_log(phba, KERN_ERR,
2094                                     BEISCSI_LOG_IO | BEISCSI_LOG_CONFIG,
2095                                     "BM_%d : Ignoring %s[%d] on CID : %d\n",
2096                                     cqe_desc[code], code, cid);
2097                         break;
2098                 case SOL_CMD_KILLED_DATA_DIGEST_ERR:
2099                 case CMD_KILLED_INVALID_STATSN_RCVD:
2100                 case CMD_KILLED_INVALID_R2T_RCVD:
2101                 case CMD_CXN_KILLED_LUN_INVALID:
2102                 case CMD_CXN_KILLED_ICD_INVALID:
2103                 case CMD_CXN_KILLED_ITT_INVALID:
2104                 case CMD_CXN_KILLED_SEQ_OUTOFORDER:
2105                 case CMD_CXN_KILLED_INVALID_DATASN_RCVD:
2106                         beiscsi_log(phba, KERN_ERR,
2107                                     BEISCSI_LOG_CONFIG | BEISCSI_LOG_IO,
2108                                     "BM_%d : Cmd Notification %s[%d] on CID : %d\n",
2109                                     cqe_desc[code], code,  cid);
2110                         break;
2111                 case UNSOL_DATA_DIGEST_ERROR_NOTIFY:
2112                         beiscsi_log(phba, KERN_ERR,
2113                                     BEISCSI_LOG_IO | BEISCSI_LOG_CONFIG,
2114                                     "BM_%d :  Dropping %s[%d] on DPDU ring on CID : %d\n",
2115                                     cqe_desc[code], code, cid);
2116                         hwi_flush_default_pdu_buffer(phba, beiscsi_conn,
2117                                              (struct i_t_dpdu_cqe *) sol);
2118                         break;
2119                 case CXN_KILLED_PDU_SIZE_EXCEEDS_DSL:
2120                 case CXN_KILLED_BURST_LEN_MISMATCH:
2121                 case CXN_KILLED_AHS_RCVD:
2122                 case CXN_KILLED_HDR_DIGEST_ERR:
2123                 case CXN_KILLED_UNKNOWN_HDR:
2124                 case CXN_KILLED_STALE_ITT_TTT_RCVD:
2125                 case CXN_KILLED_INVALID_ITT_TTT_RCVD:
2126                 case CXN_KILLED_TIMED_OUT:
2127                 case CXN_KILLED_FIN_RCVD:
2128                 case CXN_KILLED_RST_SENT:
2129                 case CXN_KILLED_RST_RCVD:
2130                 case CXN_KILLED_BAD_UNSOL_PDU_RCVD:
2131                 case CXN_KILLED_BAD_WRB_INDEX_ERROR:
2132                 case CXN_KILLED_OVER_RUN_RESIDUAL:
2133                 case CXN_KILLED_UNDER_RUN_RESIDUAL:
2134                 case CXN_KILLED_CMND_DATA_NOT_ON_SAME_CONN:
2135                         beiscsi_log(phba, KERN_ERR,
2136                                     BEISCSI_LOG_IO | BEISCSI_LOG_CONFIG,
2137                                     "BM_%d : Event %s[%d] received on CID : %d\n",
2138                                     cqe_desc[code], code, cid);
2139                         if (beiscsi_conn)
2140                                 iscsi_conn_failure(beiscsi_conn->conn,
2141                                                    ISCSI_ERR_CONN_FAILED);
2142                         break;
2143                 default:
2144                         beiscsi_log(phba, KERN_ERR,
2145                                     BEISCSI_LOG_IO | BEISCSI_LOG_CONFIG,
2146                                     "BM_%d : Invalid CQE Event Received Code : %d"
2147                                     "CID 0x%x...\n",
2148                                     code, cid);
2149                         break;
2150                 }
2151
2152                 AMAP_SET_BITS(struct amap_sol_cqe, valid, sol, 0);
2153                 queue_tail_inc(cq);
2154                 sol = queue_tail_node(cq);
2155                 num_processed++;
2156         }
2157
2158         if (num_processed > 0) {
2159                 tot_nump += num_processed;
2160                 hwi_ring_cq_db(phba, cq->id, num_processed, 1, 0);
2161         }
2162         return tot_nump;
2163 }
2164
2165 void beiscsi_process_all_cqs(struct work_struct *work)
2166 {
2167         unsigned long flags;
2168         struct hwi_controller *phwi_ctrlr;
2169         struct hwi_context_memory *phwi_context;
2170         struct beiscsi_hba *phba;
2171         struct be_eq_obj *pbe_eq =
2172             container_of(work, struct be_eq_obj, work_cqs);
2173
2174         phba = pbe_eq->phba;
2175         phwi_ctrlr = phba->phwi_ctrlr;
2176         phwi_context = phwi_ctrlr->phwi_ctxt;
2177
2178         if (pbe_eq->todo_mcc_cq) {
2179                 spin_lock_irqsave(&phba->isr_lock, flags);
2180                 pbe_eq->todo_mcc_cq = false;
2181                 spin_unlock_irqrestore(&phba->isr_lock, flags);
2182                 beiscsi_process_mcc_isr(phba);
2183         }
2184
2185         if (pbe_eq->todo_cq) {
2186                 spin_lock_irqsave(&phba->isr_lock, flags);
2187                 pbe_eq->todo_cq = false;
2188                 spin_unlock_irqrestore(&phba->isr_lock, flags);
2189                 beiscsi_process_cq(pbe_eq);
2190         }
2191
2192         /* rearm EQ for further interrupts */
2193         hwi_ring_eq_db(phba, pbe_eq->q.id, 0, 0, 1, 1);
2194 }
2195
2196 static int be_iopoll(struct blk_iopoll *iop, int budget)
2197 {
2198         static unsigned int ret;
2199         struct beiscsi_hba *phba;
2200         struct be_eq_obj *pbe_eq;
2201
2202         pbe_eq = container_of(iop, struct be_eq_obj, iopoll);
2203         ret = beiscsi_process_cq(pbe_eq);
2204         if (ret < budget) {
2205                 phba = pbe_eq->phba;
2206                 blk_iopoll_complete(iop);
2207                 beiscsi_log(phba, KERN_INFO,
2208                             BEISCSI_LOG_CONFIG | BEISCSI_LOG_IO,
2209                             "BM_%d : rearm pbe_eq->q.id =%d\n",
2210                             pbe_eq->q.id);
2211                 hwi_ring_eq_db(phba, pbe_eq->q.id, 0, 0, 1, 1);
2212         }
2213         return ret;
2214 }
2215
2216 static void
2217 hwi_write_sgl_v2(struct iscsi_wrb *pwrb, struct scatterlist *sg,
2218                   unsigned int num_sg, struct beiscsi_io_task *io_task)
2219 {
2220         struct iscsi_sge *psgl;
2221         unsigned int sg_len, index;
2222         unsigned int sge_len = 0;
2223         unsigned long long addr;
2224         struct scatterlist *l_sg;
2225         unsigned int offset;
2226
2227         AMAP_SET_BITS(struct amap_iscsi_wrb_v2, iscsi_bhs_addr_lo, pwrb,
2228                       io_task->bhs_pa.u.a32.address_lo);
2229         AMAP_SET_BITS(struct amap_iscsi_wrb_v2, iscsi_bhs_addr_hi, pwrb,
2230                       io_task->bhs_pa.u.a32.address_hi);
2231
2232         l_sg = sg;
2233         for (index = 0; (index < num_sg) && (index < 2); index++,
2234                         sg = sg_next(sg)) {
2235                 if (index == 0) {
2236                         sg_len = sg_dma_len(sg);
2237                         addr = (u64) sg_dma_address(sg);
2238                         AMAP_SET_BITS(struct amap_iscsi_wrb_v2,
2239                                       sge0_addr_lo, pwrb,
2240                                       lower_32_bits(addr));
2241                         AMAP_SET_BITS(struct amap_iscsi_wrb_v2,
2242                                       sge0_addr_hi, pwrb,
2243                                       upper_32_bits(addr));
2244                         AMAP_SET_BITS(struct amap_iscsi_wrb_v2,
2245                                       sge0_len, pwrb,
2246                                       sg_len);
2247                         sge_len = sg_len;
2248                 } else {
2249                         AMAP_SET_BITS(struct amap_iscsi_wrb_v2, sge1_r2t_offset,
2250                                       pwrb, sge_len);
2251                         sg_len = sg_dma_len(sg);
2252                         addr = (u64) sg_dma_address(sg);
2253                         AMAP_SET_BITS(struct amap_iscsi_wrb_v2,
2254                                       sge1_addr_lo, pwrb,
2255                                       lower_32_bits(addr));
2256                         AMAP_SET_BITS(struct amap_iscsi_wrb_v2,
2257                                       sge1_addr_hi, pwrb,
2258                                       upper_32_bits(addr));
2259                         AMAP_SET_BITS(struct amap_iscsi_wrb_v2,
2260                                       sge1_len, pwrb,
2261                                       sg_len);
2262                 }
2263         }
2264         psgl = (struct iscsi_sge *)io_task->psgl_handle->pfrag;
2265         memset(psgl, 0, sizeof(*psgl) * BE2_SGE);
2266
2267         AMAP_SET_BITS(struct amap_iscsi_sge, len, psgl, io_task->bhs_len - 2);
2268
2269         AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, psgl,
2270                       io_task->bhs_pa.u.a32.address_hi);
2271         AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, psgl,
2272                       io_task->bhs_pa.u.a32.address_lo);
2273
2274         if (num_sg == 1) {
2275                 AMAP_SET_BITS(struct amap_iscsi_wrb_v2, sge0_last, pwrb,
2276                               1);
2277                 AMAP_SET_BITS(struct amap_iscsi_wrb_v2, sge1_last, pwrb,
2278                               0);
2279         } else if (num_sg == 2) {
2280                 AMAP_SET_BITS(struct amap_iscsi_wrb_v2, sge0_last, pwrb,
2281                               0);
2282                 AMAP_SET_BITS(struct amap_iscsi_wrb_v2, sge1_last, pwrb,
2283                               1);
2284         } else {
2285                 AMAP_SET_BITS(struct amap_iscsi_wrb_v2, sge0_last, pwrb,
2286                               0);
2287                 AMAP_SET_BITS(struct amap_iscsi_wrb_v2, sge1_last, pwrb,
2288                               0);
2289         }
2290
2291         sg = l_sg;
2292         psgl++;
2293         psgl++;
2294         offset = 0;
2295         for (index = 0; index < num_sg; index++, sg = sg_next(sg), psgl++) {
2296                 sg_len = sg_dma_len(sg);
2297                 addr = (u64) sg_dma_address(sg);
2298                 AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, psgl,
2299                               lower_32_bits(addr));
2300                 AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, psgl,
2301                               upper_32_bits(addr));
2302                 AMAP_SET_BITS(struct amap_iscsi_sge, len, psgl, sg_len);
2303                 AMAP_SET_BITS(struct amap_iscsi_sge, sge_offset, psgl, offset);
2304                 AMAP_SET_BITS(struct amap_iscsi_sge, last_sge, psgl, 0);
2305                 offset += sg_len;
2306         }
2307         psgl--;
2308         AMAP_SET_BITS(struct amap_iscsi_sge, last_sge, psgl, 1);
2309 }
2310
2311 static void
2312 hwi_write_sgl(struct iscsi_wrb *pwrb, struct scatterlist *sg,
2313               unsigned int num_sg, struct beiscsi_io_task *io_task)
2314 {
2315         struct iscsi_sge *psgl;
2316         unsigned int sg_len, index;
2317         unsigned int sge_len = 0;
2318         unsigned long long addr;
2319         struct scatterlist *l_sg;
2320         unsigned int offset;
2321
2322         AMAP_SET_BITS(struct amap_iscsi_wrb, iscsi_bhs_addr_lo, pwrb,
2323                                       io_task->bhs_pa.u.a32.address_lo);
2324         AMAP_SET_BITS(struct amap_iscsi_wrb, iscsi_bhs_addr_hi, pwrb,
2325                                       io_task->bhs_pa.u.a32.address_hi);
2326
2327         l_sg = sg;
2328         for (index = 0; (index < num_sg) && (index < 2); index++,
2329                                                          sg = sg_next(sg)) {
2330                 if (index == 0) {
2331                         sg_len = sg_dma_len(sg);
2332                         addr = (u64) sg_dma_address(sg);
2333                         AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_addr_lo, pwrb,
2334                                                 ((u32)(addr & 0xFFFFFFFF)));
2335                         AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_addr_hi, pwrb,
2336                                                         ((u32)(addr >> 32)));
2337                         AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_len, pwrb,
2338                                                         sg_len);
2339                         sge_len = sg_len;
2340                 } else {
2341                         AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_r2t_offset,
2342                                                         pwrb, sge_len);
2343                         sg_len = sg_dma_len(sg);
2344                         addr = (u64) sg_dma_address(sg);
2345                         AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_addr_lo, pwrb,
2346                                                 ((u32)(addr & 0xFFFFFFFF)));
2347                         AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_addr_hi, pwrb,
2348                                                         ((u32)(addr >> 32)));
2349                         AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_len, pwrb,
2350                                                         sg_len);
2351                 }
2352         }
2353         psgl = (struct iscsi_sge *)io_task->psgl_handle->pfrag;
2354         memset(psgl, 0, sizeof(*psgl) * BE2_SGE);
2355
2356         AMAP_SET_BITS(struct amap_iscsi_sge, len, psgl, io_task->bhs_len - 2);
2357
2358         AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, psgl,
2359                         io_task->bhs_pa.u.a32.address_hi);
2360         AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, psgl,
2361                         io_task->bhs_pa.u.a32.address_lo);
2362
2363         if (num_sg == 1) {
2364                 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_last, pwrb,
2365                                                                 1);
2366                 AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_last, pwrb,
2367                                                                 0);
2368         } else if (num_sg == 2) {
2369                 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_last, pwrb,
2370                                                                 0);
2371                 AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_last, pwrb,
2372                                                                 1);
2373         } else {
2374                 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_last, pwrb,
2375                                                                 0);
2376                 AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_last, pwrb,
2377                                                                 0);
2378         }
2379         sg = l_sg;
2380         psgl++;
2381         psgl++;
2382         offset = 0;
2383         for (index = 0; index < num_sg; index++, sg = sg_next(sg), psgl++) {
2384                 sg_len = sg_dma_len(sg);
2385                 addr = (u64) sg_dma_address(sg);
2386                 AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, psgl,
2387                                                 (addr & 0xFFFFFFFF));
2388                 AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, psgl,
2389                                                 (addr >> 32));
2390                 AMAP_SET_BITS(struct amap_iscsi_sge, len, psgl, sg_len);
2391                 AMAP_SET_BITS(struct amap_iscsi_sge, sge_offset, psgl, offset);
2392                 AMAP_SET_BITS(struct amap_iscsi_sge, last_sge, psgl, 0);
2393                 offset += sg_len;
2394         }
2395         psgl--;
2396         AMAP_SET_BITS(struct amap_iscsi_sge, last_sge, psgl, 1);
2397 }
2398
2399 /**
2400  * hwi_write_buffer()- Populate the WRB with task info
2401  * @pwrb: ptr to the WRB entry
2402  * @task: iscsi task which is to be executed
2403  **/
2404 static void hwi_write_buffer(struct iscsi_wrb *pwrb, struct iscsi_task *task)
2405 {
2406         struct iscsi_sge *psgl;
2407         struct beiscsi_io_task *io_task = task->dd_data;
2408         struct beiscsi_conn *beiscsi_conn = io_task->conn;
2409         struct beiscsi_hba *phba = beiscsi_conn->phba;
2410         uint8_t dsp_value = 0;
2411
2412         io_task->bhs_len = sizeof(struct be_nonio_bhs) - 2;
2413         AMAP_SET_BITS(struct amap_iscsi_wrb, iscsi_bhs_addr_lo, pwrb,
2414                                 io_task->bhs_pa.u.a32.address_lo);
2415         AMAP_SET_BITS(struct amap_iscsi_wrb, iscsi_bhs_addr_hi, pwrb,
2416                                 io_task->bhs_pa.u.a32.address_hi);
2417
2418         if (task->data) {
2419
2420                 /* Check for the data_count */
2421                 dsp_value = (task->data_count) ? 1 : 0;
2422
2423                 if (chip_skh_r(phba->pcidev))
2424                         AMAP_SET_BITS(struct amap_iscsi_wrb_v2, dsp,
2425                                       pwrb, dsp_value);
2426                 else
2427                         AMAP_SET_BITS(struct amap_iscsi_wrb, dsp,
2428                                       pwrb, dsp_value);
2429
2430                 /* Map addr only if there is data_count */
2431                 if (dsp_value) {
2432                         io_task->mtask_addr = pci_map_single(phba->pcidev,
2433                                                              task->data,
2434                                                              task->data_count,
2435                                                              PCI_DMA_TODEVICE);
2436                         io_task->mtask_data_count = task->data_count;
2437                 } else
2438                         io_task->mtask_addr = 0;
2439
2440                 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_addr_lo, pwrb,
2441                               lower_32_bits(io_task->mtask_addr));
2442                 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_addr_hi, pwrb,
2443                               upper_32_bits(io_task->mtask_addr));
2444                 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_len, pwrb,
2445                                                 task->data_count);
2446
2447                 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_last, pwrb, 1);
2448         } else {
2449                 AMAP_SET_BITS(struct amap_iscsi_wrb, dsp, pwrb, 0);
2450                 io_task->mtask_addr = 0;
2451         }
2452
2453         psgl = (struct iscsi_sge *)io_task->psgl_handle->pfrag;
2454
2455         AMAP_SET_BITS(struct amap_iscsi_sge, len, psgl, io_task->bhs_len);
2456
2457         AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, psgl,
2458                       io_task->bhs_pa.u.a32.address_hi);
2459         AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, psgl,
2460                       io_task->bhs_pa.u.a32.address_lo);
2461         if (task->data) {
2462                 psgl++;
2463                 AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, psgl, 0);
2464                 AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, psgl, 0);
2465                 AMAP_SET_BITS(struct amap_iscsi_sge, len, psgl, 0);
2466                 AMAP_SET_BITS(struct amap_iscsi_sge, sge_offset, psgl, 0);
2467                 AMAP_SET_BITS(struct amap_iscsi_sge, rsvd0, psgl, 0);
2468                 AMAP_SET_BITS(struct amap_iscsi_sge, last_sge, psgl, 0);
2469
2470                 psgl++;
2471                 if (task->data) {
2472                         AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, psgl,
2473                                       lower_32_bits(io_task->mtask_addr));
2474                         AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, psgl,
2475                                       upper_32_bits(io_task->mtask_addr));
2476                 }
2477                 AMAP_SET_BITS(struct amap_iscsi_sge, len, psgl, 0x106);
2478         }
2479         AMAP_SET_BITS(struct amap_iscsi_sge, last_sge, psgl, 1);
2480 }
2481
2482 static void beiscsi_find_mem_req(struct beiscsi_hba *phba)
2483 {
2484         unsigned int num_cq_pages, num_async_pdu_buf_pages;
2485         unsigned int num_async_pdu_data_pages, wrb_sz_per_cxn;
2486         unsigned int num_async_pdu_buf_sgl_pages, num_async_pdu_data_sgl_pages;
2487
2488         num_cq_pages = PAGES_REQUIRED(phba->params.num_cq_entries * \
2489                                       sizeof(struct sol_cqe));
2490         num_async_pdu_buf_pages =
2491                         PAGES_REQUIRED(phba->params.asyncpdus_per_ctrl * \
2492                                        phba->params.defpdu_hdr_sz);
2493         num_async_pdu_buf_sgl_pages =
2494                         PAGES_REQUIRED(phba->params.asyncpdus_per_ctrl * \
2495                                        sizeof(struct phys_addr));
2496         num_async_pdu_data_pages =
2497                         PAGES_REQUIRED(phba->params.asyncpdus_per_ctrl * \
2498                                        phba->params.defpdu_data_sz);
2499         num_async_pdu_data_sgl_pages =
2500                         PAGES_REQUIRED(phba->params.asyncpdus_per_ctrl * \
2501                                        sizeof(struct phys_addr));
2502
2503         phba->params.hwi_ws_sz = sizeof(struct hwi_controller);
2504
2505         phba->mem_req[ISCSI_MEM_GLOBAL_HEADER] = 2 *
2506                                                  BE_ISCSI_PDU_HEADER_SIZE;
2507         phba->mem_req[HWI_MEM_ADDN_CONTEXT] =
2508                                             sizeof(struct hwi_context_memory);
2509
2510
2511         phba->mem_req[HWI_MEM_WRB] = sizeof(struct iscsi_wrb)
2512             * (phba->params.wrbs_per_cxn)
2513             * phba->params.cxns_per_ctrl;
2514         wrb_sz_per_cxn =  sizeof(struct wrb_handle) *
2515                                  (phba->params.wrbs_per_cxn);
2516         phba->mem_req[HWI_MEM_WRBH] = roundup_pow_of_two((wrb_sz_per_cxn) *
2517                                 phba->params.cxns_per_ctrl);
2518
2519         phba->mem_req[HWI_MEM_SGLH] = sizeof(struct sgl_handle) *
2520                 phba->params.icds_per_ctrl;
2521         phba->mem_req[HWI_MEM_SGE] = sizeof(struct iscsi_sge) *
2522                 phba->params.num_sge_per_io * phba->params.icds_per_ctrl;
2523
2524         phba->mem_req[HWI_MEM_ASYNC_HEADER_BUF] =
2525                 num_async_pdu_buf_pages * PAGE_SIZE;
2526         phba->mem_req[HWI_MEM_ASYNC_DATA_BUF] =
2527                 num_async_pdu_data_pages * PAGE_SIZE;
2528         phba->mem_req[HWI_MEM_ASYNC_HEADER_RING] =
2529                 num_async_pdu_buf_sgl_pages * PAGE_SIZE;
2530         phba->mem_req[HWI_MEM_ASYNC_DATA_RING] =
2531                 num_async_pdu_data_sgl_pages * PAGE_SIZE;
2532         phba->mem_req[HWI_MEM_ASYNC_HEADER_HANDLE] =
2533                 phba->params.asyncpdus_per_ctrl *
2534                 sizeof(struct async_pdu_handle);
2535         phba->mem_req[HWI_MEM_ASYNC_DATA_HANDLE] =
2536                 phba->params.asyncpdus_per_ctrl *
2537                 sizeof(struct async_pdu_handle);
2538         phba->mem_req[HWI_MEM_ASYNC_PDU_CONTEXT] =
2539                 sizeof(struct hwi_async_pdu_context) +
2540                 (phba->params.cxns_per_ctrl * sizeof(struct hwi_async_entry));
2541 }
2542
2543 static int beiscsi_alloc_mem(struct beiscsi_hba *phba)
2544 {
2545         struct be_mem_descriptor *mem_descr;
2546         dma_addr_t bus_add;
2547         struct mem_array *mem_arr, *mem_arr_orig;
2548         unsigned int i, j, alloc_size, curr_alloc_size;
2549
2550         phba->phwi_ctrlr = kzalloc(phba->params.hwi_ws_sz, GFP_KERNEL);
2551         if (!phba->phwi_ctrlr)
2552                 return -ENOMEM;
2553
2554         phba->init_mem = kcalloc(SE_MEM_MAX, sizeof(*mem_descr),
2555                                  GFP_KERNEL);
2556         if (!phba->init_mem) {
2557                 kfree(phba->phwi_ctrlr);
2558                 return -ENOMEM;
2559         }
2560
2561         mem_arr_orig = kmalloc(sizeof(*mem_arr_orig) * BEISCSI_MAX_FRAGS_INIT,
2562                                GFP_KERNEL);
2563         if (!mem_arr_orig) {
2564                 kfree(phba->init_mem);
2565                 kfree(phba->phwi_ctrlr);
2566                 return -ENOMEM;
2567         }
2568
2569         mem_descr = phba->init_mem;
2570         for (i = 0; i < SE_MEM_MAX; i++) {
2571                 j = 0;
2572                 mem_arr = mem_arr_orig;
2573                 alloc_size = phba->mem_req[i];
2574                 memset(mem_arr, 0, sizeof(struct mem_array) *
2575                        BEISCSI_MAX_FRAGS_INIT);
2576                 curr_alloc_size = min(be_max_phys_size * 1024, alloc_size);
2577                 do {
2578                         mem_arr->virtual_address = pci_alloc_consistent(
2579                                                         phba->pcidev,
2580                                                         curr_alloc_size,
2581                                                         &bus_add);
2582                         if (!mem_arr->virtual_address) {
2583                                 if (curr_alloc_size <= BE_MIN_MEM_SIZE)
2584                                         goto free_mem;
2585                                 if (curr_alloc_size -
2586                                         rounddown_pow_of_two(curr_alloc_size))
2587                                         curr_alloc_size = rounddown_pow_of_two
2588                                                              (curr_alloc_size);
2589                                 else
2590                                         curr_alloc_size = curr_alloc_size / 2;
2591                         } else {
2592                                 mem_arr->bus_address.u.
2593                                     a64.address = (__u64) bus_add;
2594                                 mem_arr->size = curr_alloc_size;
2595                                 alloc_size -= curr_alloc_size;
2596                                 curr_alloc_size = min(be_max_phys_size *
2597                                                       1024, alloc_size);
2598                                 j++;
2599                                 mem_arr++;
2600                         }
2601                 } while (alloc_size);
2602                 mem_descr->num_elements = j;
2603                 mem_descr->size_in_bytes = phba->mem_req[i];
2604                 mem_descr->mem_array = kmalloc(sizeof(*mem_arr) * j,
2605                                                GFP_KERNEL);
2606                 if (!mem_descr->mem_array)
2607                         goto free_mem;
2608
2609                 memcpy(mem_descr->mem_array, mem_arr_orig,
2610                        sizeof(struct mem_array) * j);
2611                 mem_descr++;
2612         }
2613         kfree(mem_arr_orig);
2614         return 0;
2615 free_mem:
2616         mem_descr->num_elements = j;
2617         while ((i) || (j)) {
2618                 for (j = mem_descr->num_elements; j > 0; j--) {
2619                         pci_free_consistent(phba->pcidev,
2620                                             mem_descr->mem_array[j - 1].size,
2621                                             mem_descr->mem_array[j - 1].
2622                                             virtual_address,
2623                                             (unsigned long)mem_descr->
2624                                             mem_array[j - 1].
2625                                             bus_address.u.a64.address);
2626                 }
2627                 if (i) {
2628                         i--;
2629                         kfree(mem_descr->mem_array);
2630                         mem_descr--;
2631                 }
2632         }
2633         kfree(mem_arr_orig);
2634         kfree(phba->init_mem);
2635         kfree(phba->phwi_ctrlr);
2636         return -ENOMEM;
2637 }
2638
2639 static int beiscsi_get_memory(struct beiscsi_hba *phba)
2640 {
2641         beiscsi_find_mem_req(phba);
2642         return beiscsi_alloc_mem(phba);
2643 }
2644
2645 static void iscsi_init_global_templates(struct beiscsi_hba *phba)
2646 {
2647         struct pdu_data_out *pdata_out;
2648         struct pdu_nop_out *pnop_out;
2649         struct be_mem_descriptor *mem_descr;
2650
2651         mem_descr = phba->init_mem;
2652         mem_descr += ISCSI_MEM_GLOBAL_HEADER;
2653         pdata_out =
2654             (struct pdu_data_out *)mem_descr->mem_array[0].virtual_address;
2655         memset(pdata_out, 0, BE_ISCSI_PDU_HEADER_SIZE);
2656
2657         AMAP_SET_BITS(struct amap_pdu_data_out, opcode, pdata_out,
2658                       IIOC_SCSI_DATA);
2659
2660         pnop_out =
2661             (struct pdu_nop_out *)((unsigned char *)mem_descr->mem_array[0].
2662                                    virtual_address + BE_ISCSI_PDU_HEADER_SIZE);
2663
2664         memset(pnop_out, 0, BE_ISCSI_PDU_HEADER_SIZE);
2665         AMAP_SET_BITS(struct amap_pdu_nop_out, ttt, pnop_out, 0xFFFFFFFF);
2666         AMAP_SET_BITS(struct amap_pdu_nop_out, f_bit, pnop_out, 1);
2667         AMAP_SET_BITS(struct amap_pdu_nop_out, i_bit, pnop_out, 0);
2668 }
2669
2670 static int beiscsi_init_wrb_handle(struct beiscsi_hba *phba)
2671 {
2672         struct be_mem_descriptor *mem_descr_wrbh, *mem_descr_wrb;
2673         struct wrb_handle *pwrb_handle = NULL;
2674         struct hwi_controller *phwi_ctrlr;
2675         struct hwi_wrb_context *pwrb_context;
2676         struct iscsi_wrb *pwrb = NULL;
2677         unsigned int num_cxn_wrbh = 0;
2678         unsigned int num_cxn_wrb = 0, j, idx = 0, index;
2679
2680         mem_descr_wrbh = phba->init_mem;
2681         mem_descr_wrbh += HWI_MEM_WRBH;
2682
2683         mem_descr_wrb = phba->init_mem;
2684         mem_descr_wrb += HWI_MEM_WRB;
2685         phwi_ctrlr = phba->phwi_ctrlr;
2686
2687         for (index = 0; index < phba->params.cxns_per_ctrl * 2; index += 2) {
2688                 pwrb_context = &phwi_ctrlr->wrb_context[index];
2689                 pwrb_context->pwrb_handle_base =
2690                                 kzalloc(sizeof(struct wrb_handle *) *
2691                                         phba->params.wrbs_per_cxn, GFP_KERNEL);
2692                 if (!pwrb_context->pwrb_handle_base) {
2693                         beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
2694                                     "BM_%d : Mem Alloc Failed. Failing to load\n");
2695                         goto init_wrb_hndl_failed;
2696                 }
2697                 pwrb_context->pwrb_handle_basestd =
2698                                 kzalloc(sizeof(struct wrb_handle *) *
2699                                         phba->params.wrbs_per_cxn, GFP_KERNEL);
2700                 if (!pwrb_context->pwrb_handle_basestd) {
2701                         beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
2702                                     "BM_%d : Mem Alloc Failed. Failing to load\n");
2703                         goto init_wrb_hndl_failed;
2704                 }
2705                 if (!num_cxn_wrbh) {
2706                         pwrb_handle =
2707                                 mem_descr_wrbh->mem_array[idx].virtual_address;
2708                         num_cxn_wrbh = ((mem_descr_wrbh->mem_array[idx].size) /
2709                                         ((sizeof(struct wrb_handle)) *
2710                                          phba->params.wrbs_per_cxn));
2711                         idx++;
2712                 }
2713                 pwrb_context->alloc_index = 0;
2714                 pwrb_context->wrb_handles_available = 0;
2715                 pwrb_context->free_index = 0;
2716
2717                 if (num_cxn_wrbh) {
2718                         for (j = 0; j < phba->params.wrbs_per_cxn; j++) {
2719                                 pwrb_context->pwrb_handle_base[j] = pwrb_handle;
2720                                 pwrb_context->pwrb_handle_basestd[j] =
2721                                                                 pwrb_handle;
2722                                 pwrb_context->wrb_handles_available++;
2723                                 pwrb_handle->wrb_index = j;
2724                                 pwrb_handle++;
2725                         }
2726                         num_cxn_wrbh--;
2727                 }
2728         }
2729         idx = 0;
2730         for (index = 0; index < phba->params.cxns_per_ctrl * 2; index += 2) {
2731                 pwrb_context = &phwi_ctrlr->wrb_context[index];
2732                 if (!num_cxn_wrb) {
2733                         pwrb = mem_descr_wrb->mem_array[idx].virtual_address;
2734                         num_cxn_wrb = (mem_descr_wrb->mem_array[idx].size) /
2735                                 ((sizeof(struct iscsi_wrb) *
2736                                   phba->params.wrbs_per_cxn));
2737                         idx++;
2738                 }
2739
2740                 if (num_cxn_wrb) {
2741                         for (j = 0; j < phba->params.wrbs_per_cxn; j++) {
2742                                 pwrb_handle = pwrb_context->pwrb_handle_base[j];
2743                                 pwrb_handle->pwrb = pwrb;
2744                                 pwrb++;
2745                         }
2746                         num_cxn_wrb--;
2747                 }
2748         }
2749         return 0;
2750 init_wrb_hndl_failed:
2751         for (j = index; j > 0; j--) {
2752                 pwrb_context = &phwi_ctrlr->wrb_context[j];
2753                 kfree(pwrb_context->pwrb_handle_base);
2754                 kfree(pwrb_context->pwrb_handle_basestd);
2755         }
2756         return -ENOMEM;
2757 }
2758
2759 static void hwi_init_async_pdu_ctx(struct beiscsi_hba *phba)
2760 {
2761         struct hwi_controller *phwi_ctrlr;
2762         struct hba_parameters *p = &phba->params;
2763         struct hwi_async_pdu_context *pasync_ctx;
2764         struct async_pdu_handle *pasync_header_h, *pasync_data_h;
2765         unsigned int index, idx, num_per_mem, num_async_data;
2766         struct be_mem_descriptor *mem_descr;
2767
2768         mem_descr = (struct be_mem_descriptor *)phba->init_mem;
2769         mem_descr += HWI_MEM_ASYNC_PDU_CONTEXT;
2770
2771         phwi_ctrlr = phba->phwi_ctrlr;
2772         phwi_ctrlr->phwi_ctxt->pasync_ctx = (struct hwi_async_pdu_context *)
2773                                 mem_descr->mem_array[0].virtual_address;
2774         pasync_ctx = phwi_ctrlr->phwi_ctxt->pasync_ctx;
2775         memset(pasync_ctx, 0, sizeof(*pasync_ctx));
2776
2777         pasync_ctx->num_entries = p->asyncpdus_per_ctrl;
2778         pasync_ctx->buffer_size = p->defpdu_hdr_sz;
2779
2780         mem_descr = (struct be_mem_descriptor *)phba->init_mem;
2781         mem_descr += HWI_MEM_ASYNC_HEADER_BUF;
2782         if (mem_descr->mem_array[0].virtual_address) {
2783                 beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
2784                             "BM_%d : hwi_init_async_pdu_ctx"
2785                             " HWI_MEM_ASYNC_HEADER_BUF va=%p\n",
2786                             mem_descr->mem_array[0].virtual_address);
2787         } else
2788                 beiscsi_log(phba, KERN_WARNING, BEISCSI_LOG_INIT,
2789                             "BM_%d : No Virtual address\n");
2790
2791         pasync_ctx->async_header.va_base =
2792                         mem_descr->mem_array[0].virtual_address;
2793
2794         pasync_ctx->async_header.pa_base.u.a64.address =
2795                         mem_descr->mem_array[0].bus_address.u.a64.address;
2796
2797         mem_descr = (struct be_mem_descriptor *)phba->init_mem;
2798         mem_descr += HWI_MEM_ASYNC_HEADER_RING;
2799         if (mem_descr->mem_array[0].virtual_address) {
2800                 beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
2801                             "BM_%d : hwi_init_async_pdu_ctx"
2802                             " HWI_MEM_ASYNC_HEADER_RING va=%p\n",
2803                             mem_descr->mem_array[0].virtual_address);
2804         } else
2805                 beiscsi_log(phba, KERN_WARNING, BEISCSI_LOG_INIT,
2806                             "BM_%d : No Virtual address\n");
2807
2808         pasync_ctx->async_header.ring_base =
2809                         mem_descr->mem_array[0].virtual_address;
2810
2811         mem_descr = (struct be_mem_descriptor *)phba->init_mem;
2812         mem_descr += HWI_MEM_ASYNC_HEADER_HANDLE;
2813         if (mem_descr->mem_array[0].virtual_address) {
2814                 beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
2815                             "BM_%d : hwi_init_async_pdu_ctx"
2816                             " HWI_MEM_ASYNC_HEADER_HANDLE va=%p\n",
2817                             mem_descr->mem_array[0].virtual_address);
2818         } else
2819                 beiscsi_log(phba, KERN_WARNING, BEISCSI_LOG_INIT,
2820                             "BM_%d : No Virtual address\n");
2821
2822         pasync_ctx->async_header.handle_base =
2823                         mem_descr->mem_array[0].virtual_address;
2824         pasync_ctx->async_header.writables = 0;
2825         INIT_LIST_HEAD(&pasync_ctx->async_header.free_list);
2826
2827
2828         mem_descr = (struct be_mem_descriptor *)phba->init_mem;
2829         mem_descr += HWI_MEM_ASYNC_DATA_RING;
2830         if (mem_descr->mem_array[0].virtual_address) {
2831                 beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
2832                             "BM_%d : hwi_init_async_pdu_ctx"
2833                             " HWI_MEM_ASYNC_DATA_RING va=%p\n",
2834                             mem_descr->mem_array[0].virtual_address);
2835         } else
2836                 beiscsi_log(phba, KERN_WARNING, BEISCSI_LOG_INIT,
2837                             "BM_%d : No Virtual address\n");
2838
2839         pasync_ctx->async_data.ring_base =
2840                         mem_descr->mem_array[0].virtual_address;
2841
2842         mem_descr = (struct be_mem_descriptor *)phba->init_mem;
2843         mem_descr += HWI_MEM_ASYNC_DATA_HANDLE;
2844         if (!mem_descr->mem_array[0].virtual_address)
2845                 beiscsi_log(phba, KERN_WARNING, BEISCSI_LOG_INIT,
2846                             "BM_%d : No Virtual address\n");
2847
2848         pasync_ctx->async_data.handle_base =
2849                         mem_descr->mem_array[0].virtual_address;
2850         pasync_ctx->async_data.writables = 0;
2851         INIT_LIST_HEAD(&pasync_ctx->async_data.free_list);
2852
2853         pasync_header_h =
2854                 (struct async_pdu_handle *)pasync_ctx->async_header.handle_base;
2855         pasync_data_h =
2856                 (struct async_pdu_handle *)pasync_ctx->async_data.handle_base;
2857
2858         mem_descr = (struct be_mem_descriptor *)phba->init_mem;
2859         mem_descr += HWI_MEM_ASYNC_DATA_BUF;
2860         if (mem_descr->mem_array[0].virtual_address) {
2861                 beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
2862                             "BM_%d : hwi_init_async_pdu_ctx"
2863                             " HWI_MEM_ASYNC_DATA_BUF va=%p\n",
2864                             mem_descr->mem_array[0].virtual_address);
2865         } else
2866                 beiscsi_log(phba, KERN_WARNING, BEISCSI_LOG_INIT,
2867                             "BM_%d : No Virtual address\n");
2868
2869         idx = 0;
2870         pasync_ctx->async_data.va_base =
2871                         mem_descr->mem_array[idx].virtual_address;
2872         pasync_ctx->async_data.pa_base.u.a64.address =
2873                         mem_descr->mem_array[idx].bus_address.u.a64.address;
2874
2875         num_async_data = ((mem_descr->mem_array[idx].size) /
2876                                 phba->params.defpdu_data_sz);
2877         num_per_mem = 0;
2878
2879         for (index = 0; index < p->asyncpdus_per_ctrl; index++) {
2880                 pasync_header_h->cri = -1;
2881                 pasync_header_h->index = (char)index;
2882                 INIT_LIST_HEAD(&pasync_header_h->link);
2883                 pasync_header_h->pbuffer =
2884                         (void *)((unsigned long)
2885                         (pasync_ctx->async_header.va_base) +
2886                         (p->defpdu_hdr_sz * index));
2887
2888                 pasync_header_h->pa.u.a64.address =
2889                         pasync_ctx->async_header.pa_base.u.a64.address +
2890                         (p->defpdu_hdr_sz * index);
2891
2892                 list_add_tail(&pasync_header_h->link,
2893                                 &pasync_ctx->async_header.free_list);
2894                 pasync_header_h++;
2895                 pasync_ctx->async_header.free_entries++;
2896                 pasync_ctx->async_header.writables++;
2897
2898                 INIT_LIST_HEAD(&pasync_ctx->async_entry[index].wait_queue.list);
2899                 INIT_LIST_HEAD(&pasync_ctx->async_entry[index].
2900                                header_busy_list);
2901                 pasync_data_h->cri = -1;
2902                 pasync_data_h->index = (char)index;
2903                 INIT_LIST_HEAD(&pasync_data_h->link);
2904
2905                 if (!num_async_data) {
2906                         num_per_mem = 0;
2907                         idx++;
2908                         pasync_ctx->async_data.va_base =
2909                                 mem_descr->mem_array[idx].virtual_address;
2910                         pasync_ctx->async_data.pa_base.u.a64.address =
2911                                 mem_descr->mem_array[idx].
2912                                 bus_address.u.a64.address;
2913
2914                         num_async_data = ((mem_descr->mem_array[idx].size) /
2915                                         phba->params.defpdu_data_sz);
2916                 }
2917                 pasync_data_h->pbuffer =
2918                         (void *)((unsigned long)
2919                         (pasync_ctx->async_data.va_base) +
2920                         (p->defpdu_data_sz * num_per_mem));
2921
2922                 pasync_data_h->pa.u.a64.address =
2923                     pasync_ctx->async_data.pa_base.u.a64.address +
2924                     (p->defpdu_data_sz * num_per_mem);
2925                 num_per_mem++;
2926                 num_async_data--;
2927
2928                 list_add_tail(&pasync_data_h->link,
2929                               &pasync_ctx->async_data.free_list);
2930                 pasync_data_h++;
2931                 pasync_ctx->async_data.free_entries++;
2932                 pasync_ctx->async_data.writables++;
2933
2934                 INIT_LIST_HEAD(&pasync_ctx->async_entry[index].data_busy_list);
2935         }
2936
2937         pasync_ctx->async_header.host_write_ptr = 0;
2938         pasync_ctx->async_header.ep_read_ptr = -1;
2939         pasync_ctx->async_data.host_write_ptr = 0;
2940         pasync_ctx->async_data.ep_read_ptr = -1;
2941 }
2942
2943 static int
2944 be_sgl_create_contiguous(void *virtual_address,
2945                          u64 physical_address, u32 length,
2946                          struct be_dma_mem *sgl)
2947 {
2948         WARN_ON(!virtual_address);
2949         WARN_ON(!physical_address);
2950         WARN_ON(!length > 0);
2951         WARN_ON(!sgl);
2952
2953         sgl->va = virtual_address;
2954         sgl->dma = (unsigned long)physical_address;
2955         sgl->size = length;
2956
2957         return 0;
2958 }
2959
2960 static void be_sgl_destroy_contiguous(struct be_dma_mem *sgl)
2961 {
2962         memset(sgl, 0, sizeof(*sgl));
2963 }
2964
2965 static void
2966 hwi_build_be_sgl_arr(struct beiscsi_hba *phba,
2967                      struct mem_array *pmem, struct be_dma_mem *sgl)
2968 {
2969         if (sgl->va)
2970                 be_sgl_destroy_contiguous(sgl);
2971
2972         be_sgl_create_contiguous(pmem->virtual_address,
2973                                  pmem->bus_address.u.a64.address,
2974                                  pmem->size, sgl);
2975 }
2976
2977 static void
2978 hwi_build_be_sgl_by_offset(struct beiscsi_hba *phba,
2979                            struct mem_array *pmem, struct be_dma_mem *sgl)
2980 {
2981         if (sgl->va)
2982                 be_sgl_destroy_contiguous(sgl);
2983
2984         be_sgl_create_contiguous((unsigned char *)pmem->virtual_address,
2985                                  pmem->bus_address.u.a64.address,
2986                                  pmem->size, sgl);
2987 }
2988
2989 static int be_fill_queue(struct be_queue_info *q,
2990                 u16 len, u16 entry_size, void *vaddress)
2991 {
2992         struct be_dma_mem *mem = &q->dma_mem;
2993
2994         memset(q, 0, sizeof(*q));
2995         q->len = len;
2996         q->entry_size = entry_size;
2997         mem->size = len * entry_size;
2998         mem->va = vaddress;
2999         if (!mem->va)
3000                 return -ENOMEM;
3001         memset(mem->va, 0, mem->size);
3002         return 0;
3003 }
3004
3005 static int beiscsi_create_eqs(struct beiscsi_hba *phba,
3006                              struct hwi_context_memory *phwi_context)
3007 {
3008         unsigned int i, num_eq_pages;
3009         int ret = 0, eq_for_mcc;
3010         struct be_queue_info *eq;
3011         struct be_dma_mem *mem;
3012         void *eq_vaddress;
3013         dma_addr_t paddr;
3014
3015         num_eq_pages = PAGES_REQUIRED(phba->params.num_eq_entries * \
3016                                       sizeof(struct be_eq_entry));
3017
3018         if (phba->msix_enabled)
3019                 eq_for_mcc = 1;
3020         else
3021                 eq_for_mcc = 0;
3022         for (i = 0; i < (phba->num_cpus + eq_for_mcc); i++) {
3023                 eq = &phwi_context->be_eq[i].q;
3024                 mem = &eq->dma_mem;
3025                 phwi_context->be_eq[i].phba = phba;
3026                 eq_vaddress = pci_alloc_consistent(phba->pcidev,
3027                                                      num_eq_pages * PAGE_SIZE,
3028                                                      &paddr);
3029                 if (!eq_vaddress)
3030                         goto create_eq_error;
3031
3032                 mem->va = eq_vaddress;
3033                 ret = be_fill_queue(eq, phba->params.num_eq_entries,
3034                                     sizeof(struct be_eq_entry), eq_vaddress);
3035                 if (ret) {
3036                         beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3037                                     "BM_%d : be_fill_queue Failed for EQ\n");
3038                         goto create_eq_error;
3039                 }
3040
3041                 mem->dma = paddr;
3042                 ret = beiscsi_cmd_eq_create(&phba->ctrl, eq,
3043                                             phwi_context->cur_eqd);
3044                 if (ret) {
3045                         beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3046                                     "BM_%d : beiscsi_cmd_eq_create"
3047                                     "Failed for EQ\n");
3048                         goto create_eq_error;
3049                 }
3050
3051                 beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
3052                             "BM_%d : eqid = %d\n",
3053                             phwi_context->be_eq[i].q.id);
3054         }
3055         return 0;
3056 create_eq_error:
3057         for (i = 0; i < (phba->num_cpus + eq_for_mcc); i++) {
3058                 eq = &phwi_context->be_eq[i].q;
3059                 mem = &eq->dma_mem;
3060                 if (mem->va)
3061                         pci_free_consistent(phba->pcidev, num_eq_pages
3062                                             * PAGE_SIZE,
3063                                             mem->va, mem->dma);
3064         }
3065         return ret;
3066 }
3067
3068 static int beiscsi_create_cqs(struct beiscsi_hba *phba,
3069                              struct hwi_context_memory *phwi_context)
3070 {
3071         unsigned int i, num_cq_pages;
3072         int ret = 0;
3073         struct be_queue_info *cq, *eq;
3074         struct be_dma_mem *mem;
3075         struct be_eq_obj *pbe_eq;
3076         void *cq_vaddress;
3077         dma_addr_t paddr;
3078
3079         num_cq_pages = PAGES_REQUIRED(phba->params.num_cq_entries * \
3080                                       sizeof(struct sol_cqe));
3081
3082         for (i = 0; i < phba->num_cpus; i++) {
3083                 cq = &phwi_context->be_cq[i];
3084                 eq = &phwi_context->be_eq[i].q;
3085                 pbe_eq = &phwi_context->be_eq[i];
3086                 pbe_eq->cq = cq;
3087                 pbe_eq->phba = phba;
3088                 mem = &cq->dma_mem;
3089                 cq_vaddress = pci_alloc_consistent(phba->pcidev,
3090                                                      num_cq_pages * PAGE_SIZE,
3091                                                      &paddr);
3092                 if (!cq_vaddress)
3093                         goto create_cq_error;
3094                 ret = be_fill_queue(cq, phba->params.num_cq_entries,
3095                                     sizeof(struct sol_cqe), cq_vaddress);
3096                 if (ret) {
3097                         beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3098                                     "BM_%d : be_fill_queue Failed "
3099                                     "for ISCSI CQ\n");
3100                         goto create_cq_error;
3101                 }
3102
3103                 mem->dma = paddr;
3104                 ret = beiscsi_cmd_cq_create(&phba->ctrl, cq, eq, false,
3105                                             false, 0);
3106                 if (ret) {
3107                         beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3108                                     "BM_%d : beiscsi_cmd_eq_create"
3109                                     "Failed for ISCSI CQ\n");
3110                         goto create_cq_error;
3111                 }
3112                 beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
3113                             "BM_%d : iscsi cq_id is %d for eq_id %d\n"
3114                             "iSCSI CQ CREATED\n", cq->id, eq->id);
3115         }
3116         return 0;
3117
3118 create_cq_error:
3119         for (i = 0; i < phba->num_cpus; i++) {
3120                 cq = &phwi_context->be_cq[i];
3121                 mem = &cq->dma_mem;
3122                 if (mem->va)
3123                         pci_free_consistent(phba->pcidev, num_cq_pages
3124                                             * PAGE_SIZE,
3125                                             mem->va, mem->dma);
3126         }
3127         return ret;
3128
3129 }
3130
3131 static int
3132 beiscsi_create_def_hdr(struct beiscsi_hba *phba,
3133                        struct hwi_context_memory *phwi_context,
3134                        struct hwi_controller *phwi_ctrlr,
3135                        unsigned int def_pdu_ring_sz)
3136 {
3137         unsigned int idx;
3138         int ret;
3139         struct be_queue_info *dq, *cq;
3140         struct be_dma_mem *mem;
3141         struct be_mem_descriptor *mem_descr;
3142         void *dq_vaddress;
3143
3144         idx = 0;
3145         dq = &phwi_context->be_def_hdrq;
3146         cq = &phwi_context->be_cq[0];
3147         mem = &dq->dma_mem;
3148         mem_descr = phba->init_mem;
3149         mem_descr += HWI_MEM_ASYNC_HEADER_RING;
3150         dq_vaddress = mem_descr->mem_array[idx].virtual_address;
3151         ret = be_fill_queue(dq, mem_descr->mem_array[0].size /
3152                             sizeof(struct phys_addr),
3153                             sizeof(struct phys_addr), dq_vaddress);
3154         if (ret) {
3155                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3156                             "BM_%d : be_fill_queue Failed for DEF PDU HDR\n");
3157                 return ret;
3158         }
3159         mem->dma = (unsigned long)mem_descr->mem_array[idx].
3160                                   bus_address.u.a64.address;
3161         ret = be_cmd_create_default_pdu_queue(&phba->ctrl, cq, dq,
3162                                               def_pdu_ring_sz,
3163                                               phba->params.defpdu_hdr_sz);
3164         if (ret) {
3165                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3166                             "BM_%d : be_cmd_create_default_pdu_queue Failed DEFHDR\n");
3167                 return ret;
3168         }
3169         phwi_ctrlr->default_pdu_hdr.id = phwi_context->be_def_hdrq.id;
3170         beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
3171                     "BM_%d : iscsi def pdu id is %d\n",
3172                     phwi_context->be_def_hdrq.id);
3173
3174         hwi_post_async_buffers(phba, 1);
3175         return 0;
3176 }
3177
3178 static int
3179 beiscsi_create_def_data(struct beiscsi_hba *phba,
3180                         struct hwi_context_memory *phwi_context,
3181                         struct hwi_controller *phwi_ctrlr,
3182                         unsigned int def_pdu_ring_sz)
3183 {
3184         unsigned int idx;
3185         int ret;
3186         struct be_queue_info *dataq, *cq;
3187         struct be_dma_mem *mem;
3188         struct be_mem_descriptor *mem_descr;
3189         void *dq_vaddress;
3190
3191         idx = 0;
3192         dataq = &phwi_context->be_def_dataq;
3193         cq = &phwi_context->be_cq[0];
3194         mem = &dataq->dma_mem;
3195         mem_descr = phba->init_mem;
3196         mem_descr += HWI_MEM_ASYNC_DATA_RING;
3197         dq_vaddress = mem_descr->mem_array[idx].virtual_address;
3198         ret = be_fill_queue(dataq, mem_descr->mem_array[0].size /
3199                             sizeof(struct phys_addr),
3200                             sizeof(struct phys_addr), dq_vaddress);
3201         if (ret) {
3202                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3203                             "BM_%d : be_fill_queue Failed for DEF PDU DATA\n");
3204                 return ret;
3205         }
3206         mem->dma = (unsigned long)mem_descr->mem_array[idx].
3207                                   bus_address.u.a64.address;
3208         ret = be_cmd_create_default_pdu_queue(&phba->ctrl, cq, dataq,
3209                                               def_pdu_ring_sz,
3210                                               phba->params.defpdu_data_sz);
3211         if (ret) {
3212                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3213                             "BM_%d be_cmd_create_default_pdu_queue"
3214                             " Failed for DEF PDU DATA\n");
3215                 return ret;
3216         }
3217         phwi_ctrlr->default_pdu_data.id = phwi_context->be_def_dataq.id;
3218         beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
3219                     "BM_%d : iscsi def data id is %d\n",
3220                     phwi_context->be_def_dataq.id);
3221
3222         hwi_post_async_buffers(phba, 0);
3223         beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
3224                     "BM_%d : DEFAULT PDU DATA RING CREATED\n");
3225
3226         return 0;
3227 }
3228
3229 static int
3230 beiscsi_post_pages(struct beiscsi_hba *phba)
3231 {
3232         struct be_mem_descriptor *mem_descr;
3233         struct mem_array *pm_arr;
3234         unsigned int page_offset, i;
3235         struct be_dma_mem sgl;
3236         int status;
3237
3238         mem_descr = phba->init_mem;
3239         mem_descr += HWI_MEM_SGE;
3240         pm_arr = mem_descr->mem_array;
3241
3242         page_offset = (sizeof(struct iscsi_sge) * phba->params.num_sge_per_io *
3243                         phba->fw_config.iscsi_icd_start) / PAGE_SIZE;
3244         for (i = 0; i < mem_descr->num_elements; i++) {
3245                 hwi_build_be_sgl_arr(phba, pm_arr, &sgl);
3246                 status = be_cmd_iscsi_post_sgl_pages(&phba->ctrl, &sgl,
3247                                                 page_offset,
3248                                                 (pm_arr->size / PAGE_SIZE));
3249                 page_offset += pm_arr->size / PAGE_SIZE;
3250                 if (status != 0) {
3251                         beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3252                                     "BM_%d : post sgl failed.\n");
3253                         return status;
3254                 }
3255                 pm_arr++;
3256         }
3257         beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
3258                     "BM_%d : POSTED PAGES\n");
3259         return 0;
3260 }
3261
3262 static void be_queue_free(struct beiscsi_hba *phba, struct be_queue_info *q)
3263 {
3264         struct be_dma_mem *mem = &q->dma_mem;
3265         if (mem->va) {
3266                 pci_free_consistent(phba->pcidev, mem->size,
3267                         mem->va, mem->dma);
3268                 mem->va = NULL;
3269         }
3270 }
3271
3272 static int be_queue_alloc(struct beiscsi_hba *phba, struct be_queue_info *q,
3273                 u16 len, u16 entry_size)
3274 {
3275         struct be_dma_mem *mem = &q->dma_mem;
3276
3277         memset(q, 0, sizeof(*q));
3278         q->len = len;
3279         q->entry_size = entry_size;
3280         mem->size = len * entry_size;
3281         mem->va = pci_alloc_consistent(phba->pcidev, mem->size, &mem->dma);
3282         if (!mem->va)
3283                 return -ENOMEM;
3284         memset(mem->va, 0, mem->size);
3285         return 0;
3286 }
3287
3288 static int
3289 beiscsi_create_wrb_rings(struct beiscsi_hba *phba,
3290                          struct hwi_context_memory *phwi_context,
3291                          struct hwi_controller *phwi_ctrlr)
3292 {
3293         unsigned int wrb_mem_index, offset, size, num_wrb_rings;
3294         u64 pa_addr_lo;
3295         unsigned int idx, num, i;
3296         struct mem_array *pwrb_arr;
3297         void *wrb_vaddr;
3298         struct be_dma_mem sgl;
3299         struct be_mem_descriptor *mem_descr;
3300         int status;
3301
3302         idx = 0;
3303         mem_descr = phba->init_mem;
3304         mem_descr += HWI_MEM_WRB;
3305         pwrb_arr = kmalloc(sizeof(*pwrb_arr) * phba->params.cxns_per_ctrl,
3306                            GFP_KERNEL);
3307         if (!pwrb_arr) {
3308                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3309                             "BM_%d : Memory alloc failed in create wrb ring.\n");
3310                 return -ENOMEM;
3311         }
3312         wrb_vaddr = mem_descr->mem_array[idx].virtual_address;
3313         pa_addr_lo = mem_descr->mem_array[idx].bus_address.u.a64.address;
3314         num_wrb_rings = mem_descr->mem_array[idx].size /
3315                 (phba->params.wrbs_per_cxn * sizeof(struct iscsi_wrb));
3316
3317         for (num = 0; num < phba->params.cxns_per_ctrl; num++) {
3318                 if (num_wrb_rings) {
3319                         pwrb_arr[num].virtual_address = wrb_vaddr;
3320                         pwrb_arr[num].bus_address.u.a64.address = pa_addr_lo;
3321                         pwrb_arr[num].size = phba->params.wrbs_per_cxn *
3322                                             sizeof(struct iscsi_wrb);
3323                         wrb_vaddr += pwrb_arr[num].size;
3324                         pa_addr_lo += pwrb_arr[num].size;
3325                         num_wrb_rings--;
3326                 } else {
3327                         idx++;
3328                         wrb_vaddr = mem_descr->mem_array[idx].virtual_address;
3329                         pa_addr_lo = mem_descr->mem_array[idx].\
3330                                         bus_address.u.a64.address;
3331                         num_wrb_rings = mem_descr->mem_array[idx].size /
3332                                         (phba->params.wrbs_per_cxn *
3333                                         sizeof(struct iscsi_wrb));
3334                         pwrb_arr[num].virtual_address = wrb_vaddr;
3335                         pwrb_arr[num].bus_address.u.a64.address\
3336                                                 = pa_addr_lo;
3337                         pwrb_arr[num].size = phba->params.wrbs_per_cxn *
3338                                                  sizeof(struct iscsi_wrb);
3339                         wrb_vaddr += pwrb_arr[num].size;
3340                         pa_addr_lo   += pwrb_arr[num].size;
3341                         num_wrb_rings--;
3342                 }
3343         }
3344         for (i = 0; i < phba->params.cxns_per_ctrl; i++) {
3345                 wrb_mem_index = 0;
3346                 offset = 0;
3347                 size = 0;
3348
3349                 hwi_build_be_sgl_by_offset(phba, &pwrb_arr[i], &sgl);
3350                 status = be_cmd_wrbq_create(&phba->ctrl, &sgl,
3351                                             &phwi_context->be_wrbq[i]);
3352                 if (status != 0) {
3353                         beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3354                                     "BM_%d : wrbq create failed.");
3355                         kfree(pwrb_arr);
3356                         return status;
3357                 }
3358                 phwi_ctrlr->wrb_context[i * 2].cid = phwi_context->be_wrbq[i].
3359                                                                    id;
3360         }
3361         kfree(pwrb_arr);
3362         return 0;
3363 }
3364
3365 static void free_wrb_handles(struct beiscsi_hba *phba)
3366 {
3367         unsigned int index;
3368         struct hwi_controller *phwi_ctrlr;
3369         struct hwi_wrb_context *pwrb_context;
3370
3371         phwi_ctrlr = phba->phwi_ctrlr;
3372         for (index = 0; index < phba->params.cxns_per_ctrl * 2; index += 2) {
3373                 pwrb_context = &phwi_ctrlr->wrb_context[index];
3374                 kfree(pwrb_context->pwrb_handle_base);
3375                 kfree(pwrb_context->pwrb_handle_basestd);
3376         }
3377 }
3378
3379 static void be_mcc_queues_destroy(struct beiscsi_hba *phba)
3380 {
3381         struct be_queue_info *q;
3382         struct be_ctrl_info *ctrl = &phba->ctrl;
3383
3384         q = &phba->ctrl.mcc_obj.q;
3385         if (q->created)
3386                 beiscsi_cmd_q_destroy(ctrl, q, QTYPE_MCCQ);
3387         be_queue_free(phba, q);
3388
3389         q = &phba->ctrl.mcc_obj.cq;
3390         if (q->created)
3391                 beiscsi_cmd_q_destroy(ctrl, q, QTYPE_CQ);
3392         be_queue_free(phba, q);
3393 }
3394
3395 static void hwi_cleanup(struct beiscsi_hba *phba)
3396 {
3397         struct be_queue_info *q;
3398         struct be_ctrl_info *ctrl = &phba->ctrl;
3399         struct hwi_controller *phwi_ctrlr;
3400         struct hwi_context_memory *phwi_context;
3401         int i, eq_num;
3402
3403         phwi_ctrlr = phba->phwi_ctrlr;
3404         phwi_context = phwi_ctrlr->phwi_ctxt;
3405         for (i = 0; i < phba->params.cxns_per_ctrl; i++) {
3406                 q = &phwi_context->be_wrbq[i];
3407                 if (q->created)
3408                         beiscsi_cmd_q_destroy(ctrl, q, QTYPE_WRBQ);
3409         }
3410         free_wrb_handles(phba);
3411
3412         q = &phwi_context->be_def_hdrq;
3413         if (q->created)
3414                 beiscsi_cmd_q_destroy(ctrl, q, QTYPE_DPDUQ);
3415
3416         q = &phwi_context->be_def_dataq;
3417         if (q->created)
3418                 beiscsi_cmd_q_destroy(ctrl, q, QTYPE_DPDUQ);
3419
3420         beiscsi_cmd_q_destroy(ctrl, NULL, QTYPE_SGL);
3421
3422         for (i = 0; i < (phba->num_cpus); i++) {
3423                 q = &phwi_context->be_cq[i];
3424                 if (q->created)
3425                         beiscsi_cmd_q_destroy(ctrl, q, QTYPE_CQ);
3426         }
3427         if (phba->msix_enabled)
3428                 eq_num = 1;
3429         else
3430                 eq_num = 0;
3431         for (i = 0; i < (phba->num_cpus + eq_num); i++) {
3432                 q = &phwi_context->be_eq[i].q;
3433                 if (q->created)
3434                         beiscsi_cmd_q_destroy(ctrl, q, QTYPE_EQ);
3435         }
3436         be_mcc_queues_destroy(phba);
3437 }
3438
3439 static int be_mcc_queues_create(struct beiscsi_hba *phba,
3440                                 struct hwi_context_memory *phwi_context)
3441 {
3442         struct be_queue_info *q, *cq;
3443         struct be_ctrl_info *ctrl = &phba->ctrl;
3444
3445         /* Alloc MCC compl queue */
3446         cq = &phba->ctrl.mcc_obj.cq;
3447         if (be_queue_alloc(phba, cq, MCC_CQ_LEN,
3448                         sizeof(struct be_mcc_compl)))
3449                 goto err;
3450         /* Ask BE to create MCC compl queue; */
3451         if (phba->msix_enabled) {
3452                 if (beiscsi_cmd_cq_create(ctrl, cq, &phwi_context->be_eq
3453                                          [phba->num_cpus].q, false, true, 0))
3454                 goto mcc_cq_free;
3455         } else {
3456                 if (beiscsi_cmd_cq_create(ctrl, cq, &phwi_context->be_eq[0].q,
3457                                           false, true, 0))
3458                 goto mcc_cq_free;
3459         }
3460
3461         /* Alloc MCC queue */
3462         q = &phba->ctrl.mcc_obj.q;
3463         if (be_queue_alloc(phba, q, MCC_Q_LEN, sizeof(struct be_mcc_wrb)))
3464                 goto mcc_cq_destroy;
3465
3466         /* Ask BE to create MCC queue */
3467         if (beiscsi_cmd_mccq_create(phba, q, cq))
3468                 goto mcc_q_free;
3469
3470         return 0;
3471
3472 mcc_q_free:
3473         be_queue_free(phba, q);
3474 mcc_cq_destroy:
3475         beiscsi_cmd_q_destroy(ctrl, cq, QTYPE_CQ);
3476 mcc_cq_free:
3477         be_queue_free(phba, cq);
3478 err:
3479         return -ENOMEM;
3480 }
3481
3482 /**
3483  * find_num_cpus()- Get the CPU online count
3484  * @phba: ptr to priv structure
3485  *
3486  * CPU count is used for creating EQ.
3487  **/
3488 static void find_num_cpus(struct beiscsi_hba *phba)
3489 {
3490         int  num_cpus = 0;
3491
3492         num_cpus = num_online_cpus();
3493
3494         switch (phba->generation) {
3495         case BE_GEN2:
3496         case BE_GEN3:
3497                 phba->num_cpus = (num_cpus > BEISCSI_MAX_NUM_CPUS) ?
3498                                   BEISCSI_MAX_NUM_CPUS : num_cpus;
3499                 break;
3500         case BE_GEN4:
3501                 phba->num_cpus = (num_cpus > OC_SKH_MAX_NUM_CPUS) ?
3502                                   OC_SKH_MAX_NUM_CPUS : num_cpus;
3503                 break;
3504         default:
3505                 phba->num_cpus = 1;
3506         }
3507 }
3508
3509 static int hwi_init_port(struct beiscsi_hba *phba)
3510 {
3511         struct hwi_controller *phwi_ctrlr;
3512         struct hwi_context_memory *phwi_context;
3513         unsigned int def_pdu_ring_sz;
3514         struct be_ctrl_info *ctrl = &phba->ctrl;
3515         int status;
3516
3517         def_pdu_ring_sz =
3518                 phba->params.asyncpdus_per_ctrl * sizeof(struct phys_addr);
3519         phwi_ctrlr = phba->phwi_ctrlr;
3520         phwi_context = phwi_ctrlr->phwi_ctxt;
3521         phwi_context->max_eqd = 0;
3522         phwi_context->min_eqd = 0;
3523         phwi_context->cur_eqd = 64;
3524         be_cmd_fw_initialize(&phba->ctrl);
3525
3526         status = beiscsi_create_eqs(phba, phwi_context);
3527         if (status != 0) {
3528                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3529                             "BM_%d : EQ not created\n");
3530                 goto error;
3531         }
3532
3533         status = be_mcc_queues_create(phba, phwi_context);
3534         if (status != 0)
3535                 goto error;
3536
3537         status = mgmt_check_supported_fw(ctrl, phba);
3538         if (status != 0) {
3539                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3540                             "BM_%d : Unsupported fw version\n");
3541                 goto error;
3542         }
3543
3544         status = beiscsi_create_cqs(phba, phwi_context);
3545         if (status != 0) {
3546                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3547                             "BM_%d : CQ not created\n");
3548                 goto error;
3549         }
3550
3551         status = beiscsi_create_def_hdr(phba, phwi_context, phwi_ctrlr,
3552                                         def_pdu_ring_sz);
3553         if (status != 0) {
3554                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3555                             "BM_%d : Default Header not created\n");
3556                 goto error;
3557         }
3558
3559         status = beiscsi_create_def_data(phba, phwi_context,
3560                                          phwi_ctrlr, def_pdu_ring_sz);
3561         if (status != 0) {
3562                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3563                             "BM_%d : Default Data not created\n");
3564                 goto error;
3565         }
3566
3567         status = beiscsi_post_pages(phba);
3568         if (status != 0) {
3569                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3570                             "BM_%d : Post SGL Pages Failed\n");
3571                 goto error;
3572         }
3573
3574         status = beiscsi_create_wrb_rings(phba, phwi_context, phwi_ctrlr);
3575         if (status != 0) {
3576                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3577                             "BM_%d : WRB Rings not created\n");
3578                 goto error;
3579         }
3580
3581         beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
3582                     "BM_%d : hwi_init_port success\n");
3583         return 0;
3584
3585 error:
3586         beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3587                     "BM_%d : hwi_init_port failed");
3588         hwi_cleanup(phba);
3589         return status;
3590 }
3591
3592 static int hwi_init_controller(struct beiscsi_hba *phba)
3593 {
3594         struct hwi_controller *phwi_ctrlr;
3595
3596         phwi_ctrlr = phba->phwi_ctrlr;
3597         if (1 == phba->init_mem[HWI_MEM_ADDN_CONTEXT].num_elements) {
3598                 phwi_ctrlr->phwi_ctxt = (struct hwi_context_memory *)phba->
3599                     init_mem[HWI_MEM_ADDN_CONTEXT].mem_array[0].virtual_address;
3600                 beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
3601                             "BM_%d :  phwi_ctrlr->phwi_ctxt=%p\n",
3602                             phwi_ctrlr->phwi_ctxt);
3603         } else {
3604                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3605                             "BM_%d : HWI_MEM_ADDN_CONTEXT is more "
3606                             "than one element.Failing to load\n");
3607                 return -ENOMEM;
3608         }
3609
3610         iscsi_init_global_templates(phba);
3611         if (beiscsi_init_wrb_handle(phba))
3612                 return -ENOMEM;
3613
3614         hwi_init_async_pdu_ctx(phba);
3615         if (hwi_init_port(phba) != 0) {
3616                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3617                             "BM_%d : hwi_init_controller failed\n");
3618
3619                 return -ENOMEM;
3620         }
3621         return 0;
3622 }
3623
3624 static void beiscsi_free_mem(struct beiscsi_hba *phba)
3625 {
3626         struct be_mem_descriptor *mem_descr;
3627         int i, j;
3628
3629         mem_descr = phba->init_mem;
3630         i = 0;
3631         j = 0;
3632         for (i = 0; i < SE_MEM_MAX; i++) {
3633                 for (j = mem_descr->num_elements; j > 0; j--) {
3634                         pci_free_consistent(phba->pcidev,
3635                           mem_descr->mem_array[j - 1].size,
3636                           mem_descr->mem_array[j - 1].virtual_address,
3637                           (unsigned long)mem_descr->mem_array[j - 1].
3638                           bus_address.u.a64.address);
3639                 }
3640                 kfree(mem_descr->mem_array);
3641                 mem_descr++;
3642         }
3643         kfree(phba->init_mem);
3644         kfree(phba->phwi_ctrlr);
3645 }
3646
3647 static int beiscsi_init_controller(struct beiscsi_hba *phba)
3648 {
3649         int ret = -ENOMEM;
3650
3651         ret = beiscsi_get_memory(phba);
3652         if (ret < 0) {
3653                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3654                             "BM_%d : beiscsi_dev_probe -"
3655                             "Failed in beiscsi_alloc_memory\n");
3656                 return ret;
3657         }
3658
3659         ret = hwi_init_controller(phba);
3660         if (ret)
3661                 goto free_init;
3662         beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
3663                     "BM_%d : Return success from beiscsi_init_controller");
3664
3665         return 0;
3666
3667 free_init:
3668         beiscsi_free_mem(phba);
3669         return ret;
3670 }
3671
3672 static int beiscsi_init_sgl_handle(struct beiscsi_hba *phba)
3673 {
3674         struct be_mem_descriptor *mem_descr_sglh, *mem_descr_sg;
3675         struct sgl_handle *psgl_handle;
3676         struct iscsi_sge *pfrag;
3677         unsigned int arr_index, i, idx;
3678
3679         phba->io_sgl_hndl_avbl = 0;
3680         phba->eh_sgl_hndl_avbl = 0;
3681
3682         mem_descr_sglh = phba->init_mem;
3683         mem_descr_sglh += HWI_MEM_SGLH;
3684         if (1 == mem_descr_sglh->num_elements) {
3685                 phba->io_sgl_hndl_base = kzalloc(sizeof(struct sgl_handle *) *
3686                                                  phba->params.ios_per_ctrl,
3687                                                  GFP_KERNEL);
3688                 if (!phba->io_sgl_hndl_base) {
3689                         beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3690                                     "BM_%d : Mem Alloc Failed. Failing to load\n");
3691                         return -ENOMEM;
3692                 }
3693                 phba->eh_sgl_hndl_base = kzalloc(sizeof(struct sgl_handle *) *
3694                                                  (phba->params.icds_per_ctrl -
3695                                                  phba->params.ios_per_ctrl),
3696                                                  GFP_KERNEL);
3697                 if (!phba->eh_sgl_hndl_base) {
3698                         kfree(phba->io_sgl_hndl_base);
3699                         beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3700                                     "BM_%d : Mem Alloc Failed. Failing to load\n");
3701                         return -ENOMEM;
3702                 }
3703         } else {
3704                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3705                             "BM_%d : HWI_MEM_SGLH is more than one element."
3706                             "Failing to load\n");
3707                 return -ENOMEM;
3708         }
3709
3710         arr_index = 0;
3711         idx = 0;
3712         while (idx < mem_descr_sglh->num_elements) {
3713                 psgl_handle = mem_descr_sglh->mem_array[idx].virtual_address;
3714
3715                 for (i = 0; i < (mem_descr_sglh->mem_array[idx].size /
3716                       sizeof(struct sgl_handle)); i++) {
3717                         if (arr_index < phba->params.ios_per_ctrl) {
3718                                 phba->io_sgl_hndl_base[arr_index] = psgl_handle;
3719                                 phba->io_sgl_hndl_avbl++;
3720                                 arr_index++;
3721                         } else {
3722                                 phba->eh_sgl_hndl_base[arr_index -
3723                                         phba->params.ios_per_ctrl] =
3724                                                                 psgl_handle;
3725                                 arr_index++;
3726                                 phba->eh_sgl_hndl_avbl++;
3727                         }
3728                         psgl_handle++;
3729                 }
3730                 idx++;
3731         }
3732         beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
3733                     "BM_%d : phba->io_sgl_hndl_avbl=%d"
3734                     "phba->eh_sgl_hndl_avbl=%d\n",
3735                     phba->io_sgl_hndl_avbl,
3736                     phba->eh_sgl_hndl_avbl);
3737
3738         mem_descr_sg = phba->init_mem;
3739         mem_descr_sg += HWI_MEM_SGE;
3740         beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
3741                     "\n BM_%d : mem_descr_sg->num_elements=%d\n",
3742                     mem_descr_sg->num_elements);
3743
3744         arr_index = 0;
3745         idx = 0;
3746         while (idx < mem_descr_sg->num_elements) {
3747                 pfrag = mem_descr_sg->mem_array[idx].virtual_address;
3748
3749                 for (i = 0;
3750                      i < (mem_descr_sg->mem_array[idx].size) /
3751                      (sizeof(struct iscsi_sge) * phba->params.num_sge_per_io);
3752                      i++) {
3753                         if (arr_index < phba->params.ios_per_ctrl)
3754                                 psgl_handle = phba->io_sgl_hndl_base[arr_index];
3755                         else
3756                                 psgl_handle = phba->eh_sgl_hndl_base[arr_index -
3757                                                 phba->params.ios_per_ctrl];
3758                         psgl_handle->pfrag = pfrag;
3759                         AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, pfrag, 0);
3760                         AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, pfrag, 0);
3761                         pfrag += phba->params.num_sge_per_io;
3762                         psgl_handle->sgl_index =
3763                                 phba->fw_config.iscsi_icd_start + arr_index++;
3764                 }
3765                 idx++;
3766         }
3767         phba->io_sgl_free_index = 0;
3768         phba->io_sgl_alloc_index = 0;
3769         phba->eh_sgl_free_index = 0;
3770         phba->eh_sgl_alloc_index = 0;
3771         return 0;
3772 }
3773
3774 static int hba_setup_cid_tbls(struct beiscsi_hba *phba)
3775 {
3776         int i, new_cid;
3777
3778         phba->cid_array = kzalloc(sizeof(void *) * phba->params.cxns_per_ctrl,
3779                                   GFP_KERNEL);
3780         if (!phba->cid_array) {
3781                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3782                             "BM_%d : Failed to allocate memory in "
3783                             "hba_setup_cid_tbls\n");
3784                 return -ENOMEM;
3785         }
3786         phba->ep_array = kzalloc(sizeof(struct iscsi_endpoint *) *
3787                                  phba->params.cxns_per_ctrl * 2, GFP_KERNEL);
3788         if (!phba->ep_array) {
3789                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3790                             "BM_%d : Failed to allocate memory in "
3791                             "hba_setup_cid_tbls\n");
3792                 kfree(phba->cid_array);
3793                 return -ENOMEM;
3794         }
3795         new_cid = phba->fw_config.iscsi_cid_start;
3796         for (i = 0; i < phba->params.cxns_per_ctrl; i++) {
3797                 phba->cid_array[i] = new_cid;
3798                 new_cid += 2;
3799         }
3800         phba->avlbl_cids = phba->params.cxns_per_ctrl;
3801         return 0;
3802 }
3803
3804 static void hwi_enable_intr(struct beiscsi_hba *phba)
3805 {
3806         struct be_ctrl_info *ctrl = &phba->ctrl;
3807         struct hwi_controller *phwi_ctrlr;
3808         struct hwi_context_memory *phwi_context;
3809         struct be_queue_info *eq;
3810         u8 __iomem *addr;
3811         u32 reg, i;
3812         u32 enabled;
3813
3814         phwi_ctrlr = phba->phwi_ctrlr;
3815         phwi_context = phwi_ctrlr->phwi_ctxt;
3816
3817         addr = (u8 __iomem *) ((u8 __iomem *) ctrl->pcicfg +
3818                         PCICFG_MEMBAR_CTRL_INT_CTRL_OFFSET);
3819         reg = ioread32(addr);
3820
3821         enabled = reg & MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK;
3822         if (!enabled) {
3823                 reg |= MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK;
3824                 beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
3825                             "BM_%d : reg =x%08x addr=%p\n", reg, addr);
3826                 iowrite32(reg, addr);
3827         }
3828
3829         if (!phba->msix_enabled) {
3830                 eq = &phwi_context->be_eq[0].q;
3831                 beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
3832                             "BM_%d : eq->id=%d\n", eq->id);
3833
3834                 hwi_ring_eq_db(phba, eq->id, 0, 0, 1, 1);
3835         } else {
3836                 for (i = 0; i <= phba->num_cpus; i++) {
3837                         eq = &phwi_context->be_eq[i].q;
3838                         beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
3839                                     "BM_%d : eq->id=%d\n", eq->id);
3840                         hwi_ring_eq_db(phba, eq->id, 0, 0, 1, 1);
3841                 }
3842         }
3843 }
3844
3845 static void hwi_disable_intr(struct beiscsi_hba *phba)
3846 {
3847         struct be_ctrl_info *ctrl = &phba->ctrl;
3848
3849         u8 __iomem *addr = ctrl->pcicfg + PCICFG_MEMBAR_CTRL_INT_CTRL_OFFSET;
3850         u32 reg = ioread32(addr);
3851
3852         u32 enabled = reg & MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK;
3853         if (enabled) {
3854                 reg &= ~MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK;
3855                 iowrite32(reg, addr);
3856         } else
3857                 beiscsi_log(phba, KERN_WARNING, BEISCSI_LOG_INIT,
3858                             "BM_%d : In hwi_disable_intr, Already Disabled\n");
3859 }
3860
3861 /**
3862  * beiscsi_get_boot_info()- Get the boot session info
3863  * @phba: The device priv structure instance
3864  *
3865  * Get the boot target info and store in driver priv structure
3866  *
3867  * return values
3868  *      Success: 0
3869  *      Failure: Non-Zero Value
3870  **/
3871 static int beiscsi_get_boot_info(struct beiscsi_hba *phba)
3872 {
3873         struct be_cmd_get_session_resp *session_resp;
3874         struct be_mcc_wrb *wrb;
3875         struct be_dma_mem nonemb_cmd;
3876         unsigned int tag, wrb_num;
3877         unsigned short status, extd_status;
3878         unsigned int s_handle;
3879         struct be_queue_info *mccq = &phba->ctrl.mcc_obj.q;
3880         int ret = -ENOMEM;
3881
3882         /* Get the session handle of the boot target */
3883         ret = be_mgmt_get_boot_shandle(phba, &s_handle);
3884         if (ret) {
3885                 beiscsi_log(phba, KERN_ERR,
3886                             BEISCSI_LOG_INIT | BEISCSI_LOG_CONFIG,
3887                             "BM_%d : No boot session\n");
3888                 return ret;
3889         }
3890         nonemb_cmd.va = pci_alloc_consistent(phba->ctrl.pdev,
3891                                 sizeof(*session_resp),
3892                                 &nonemb_cmd.dma);
3893         if (nonemb_cmd.va == NULL) {
3894                 beiscsi_log(phba, KERN_ERR,
3895                             BEISCSI_LOG_INIT | BEISCSI_LOG_CONFIG,
3896                             "BM_%d : Failed to allocate memory for"
3897                             "beiscsi_get_session_info\n");
3898
3899                 return -ENOMEM;
3900         }
3901
3902         memset(nonemb_cmd.va, 0, sizeof(*session_resp));
3903         tag = mgmt_get_session_info(phba, s_handle,
3904                                     &nonemb_cmd);
3905         if (!tag) {
3906                 beiscsi_log(phba, KERN_ERR,
3907                             BEISCSI_LOG_INIT | BEISCSI_LOG_CONFIG,
3908                             "BM_%d : beiscsi_get_session_info"
3909                             " Failed\n");
3910
3911                 goto boot_freemem;
3912         } else
3913                 wait_event_interruptible(phba->ctrl.mcc_wait[tag],
3914                                          phba->ctrl.mcc_numtag[tag]);
3915
3916         wrb_num = (phba->ctrl.mcc_numtag[tag] & 0x00FF0000) >> 16;
3917         extd_status = (phba->ctrl.mcc_numtag[tag] & 0x0000FF00) >> 8;
3918         status = phba->ctrl.mcc_numtag[tag] & 0x000000FF;
3919         if (status || extd_status) {
3920                 beiscsi_log(phba, KERN_ERR,
3921                             BEISCSI_LOG_INIT | BEISCSI_LOG_CONFIG,
3922                             "BM_%d : beiscsi_get_session_info Failed"
3923                             " status = %d extd_status = %d\n",
3924                             status, extd_status);
3925
3926                 free_mcc_tag(&phba->ctrl, tag);
3927                 goto boot_freemem;
3928         }
3929         wrb = queue_get_wrb(mccq, wrb_num);
3930         free_mcc_tag(&phba->ctrl, tag);
3931         session_resp = nonemb_cmd.va ;
3932
3933         memcpy(&phba->boot_sess, &session_resp->session_info,
3934                sizeof(struct mgmt_session_info));
3935         ret = 0;
3936
3937 boot_freemem:
3938         pci_free_consistent(phba->ctrl.pdev, nonemb_cmd.size,
3939                     nonemb_cmd.va, nonemb_cmd.dma);
3940         return ret;
3941 }
3942
3943 static void beiscsi_boot_release(void *data)
3944 {
3945         struct beiscsi_hba *phba = data;
3946
3947         scsi_host_put(phba->shost);
3948 }
3949
3950 static int beiscsi_setup_boot_info(struct beiscsi_hba *phba)
3951 {
3952         struct iscsi_boot_kobj *boot_kobj;
3953
3954         /* get boot info using mgmt cmd */
3955         if (beiscsi_get_boot_info(phba))
3956                 /* Try to see if we can carry on without this */
3957                 return 0;
3958
3959         phba->boot_kset = iscsi_boot_create_host_kset(phba->shost->host_no);
3960         if (!phba->boot_kset)
3961                 return -ENOMEM;
3962
3963         /* get a ref because the show function will ref the phba */
3964         if (!scsi_host_get(phba->shost))
3965                 goto free_kset;
3966         boot_kobj = iscsi_boot_create_target(phba->boot_kset, 0, phba,
3967                                              beiscsi_show_boot_tgt_info,
3968                                              beiscsi_tgt_get_attr_visibility,
3969                                              beiscsi_boot_release);
3970         if (!boot_kobj)
3971                 goto put_shost;
3972
3973         if (!scsi_host_get(phba->shost))
3974                 goto free_kset;
3975         boot_kobj = iscsi_boot_create_initiator(phba->boot_kset, 0, phba,
3976                                                 beiscsi_show_boot_ini_info,
3977                                                 beiscsi_ini_get_attr_visibility,
3978                                                 beiscsi_boot_release);
3979         if (!boot_kobj)
3980                 goto put_shost;
3981
3982         if (!scsi_host_get(phba->shost))
3983                 goto free_kset;
3984         boot_kobj = iscsi_boot_create_ethernet(phba->boot_kset, 0, phba,
3985                                                beiscsi_show_boot_eth_info,
3986                                                beiscsi_eth_get_attr_visibility,
3987                                                beiscsi_boot_release);
3988         if (!boot_kobj)
3989                 goto put_shost;
3990         return 0;
3991
3992 put_shost:
3993         scsi_host_put(phba->shost);
3994 free_kset:
3995         iscsi_boot_destroy_kset(phba->boot_kset);
3996         return -ENOMEM;
3997 }
3998
3999 static int beiscsi_init_port(struct beiscsi_hba *phba)
4000 {
4001         int ret;
4002
4003         ret = beiscsi_init_controller(phba);
4004         if (ret < 0) {
4005                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
4006                             "BM_%d : beiscsi_dev_probe - Failed in"
4007                             "beiscsi_init_controller\n");
4008                 return ret;
4009         }
4010         ret = beiscsi_init_sgl_handle(phba);
4011         if (ret < 0) {
4012                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
4013                             "BM_%d : beiscsi_dev_probe - Failed in"
4014                             "beiscsi_init_sgl_handle\n");
4015                 goto do_cleanup_ctrlr;
4016         }
4017
4018         if (hba_setup_cid_tbls(phba)) {
4019                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
4020                             "BM_%d : Failed in hba_setup_cid_tbls\n");
4021                 kfree(phba->io_sgl_hndl_base);
4022                 kfree(phba->eh_sgl_hndl_base);
4023                 goto do_cleanup_ctrlr;
4024         }
4025
4026         return ret;
4027
4028 do_cleanup_ctrlr:
4029         hwi_cleanup(phba);
4030         return ret;
4031 }
4032
4033 static void hwi_purge_eq(struct beiscsi_hba *phba)
4034 {
4035         struct hwi_controller *phwi_ctrlr;
4036         struct hwi_context_memory *phwi_context;
4037         struct be_queue_info *eq;
4038         struct be_eq_entry *eqe = NULL;
4039         int i, eq_msix;
4040         unsigned int num_processed;
4041
4042         phwi_ctrlr = phba->phwi_ctrlr;
4043         phwi_context = phwi_ctrlr->phwi_ctxt;
4044         if (phba->msix_enabled)
4045                 eq_msix = 1;
4046         else
4047                 eq_msix = 0;
4048
4049         for (i = 0; i < (phba->num_cpus + eq_msix); i++) {
4050                 eq = &phwi_context->be_eq[i].q;
4051                 eqe = queue_tail_node(eq);
4052                 num_processed = 0;
4053                 while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32]
4054                                         & EQE_VALID_MASK) {
4055                         AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0);
4056                         queue_tail_inc(eq);
4057                         eqe = queue_tail_node(eq);
4058                         num_processed++;
4059                 }
4060
4061                 if (num_processed)
4062                         hwi_ring_eq_db(phba, eq->id, 1, num_processed, 1, 1);
4063         }
4064 }
4065
4066 static void beiscsi_clean_port(struct beiscsi_hba *phba)
4067 {
4068         int mgmt_status;
4069
4070         mgmt_status = mgmt_epfw_cleanup(phba, CMD_CONNECTION_CHUTE_0);
4071         if (mgmt_status)
4072                 beiscsi_log(phba, KERN_WARNING, BEISCSI_LOG_INIT,
4073                             "BM_%d : mgmt_epfw_cleanup FAILED\n");
4074
4075         hwi_purge_eq(phba);
4076         hwi_cleanup(phba);
4077         kfree(phba->io_sgl_hndl_base);
4078         kfree(phba->eh_sgl_hndl_base);
4079         kfree(phba->cid_array);
4080         kfree(phba->ep_array);
4081 }
4082
4083 /**
4084  * beiscsi_cleanup_task()- Free driver resources of the task
4085  * @task: ptr to the iscsi task
4086  *
4087  **/
4088 static void beiscsi_cleanup_task(struct iscsi_task *task)
4089 {
4090         struct beiscsi_io_task *io_task = task->dd_data;
4091         struct iscsi_conn *conn = task->conn;
4092         struct beiscsi_conn *beiscsi_conn = conn->dd_data;
4093         struct beiscsi_hba *phba = beiscsi_conn->phba;
4094         struct beiscsi_session *beiscsi_sess = beiscsi_conn->beiscsi_sess;
4095         struct hwi_wrb_context *pwrb_context;
4096         struct hwi_controller *phwi_ctrlr;
4097
4098         phwi_ctrlr = phba->phwi_ctrlr;
4099         pwrb_context = &phwi_ctrlr->wrb_context[beiscsi_conn->beiscsi_conn_cid
4100                         - phba->fw_config.iscsi_cid_start];
4101
4102         if (io_task->cmd_bhs) {
4103                 pci_pool_free(beiscsi_sess->bhs_pool, io_task->cmd_bhs,
4104                               io_task->bhs_pa.u.a64.address);
4105                 io_task->cmd_bhs = NULL;
4106         }
4107
4108         if (task->sc) {
4109                 if (io_task->pwrb_handle) {
4110                         free_wrb_handle(phba, pwrb_context,
4111                                         io_task->pwrb_handle);
4112                         io_task->pwrb_handle = NULL;
4113                 }
4114
4115                 if (io_task->psgl_handle) {
4116                         spin_lock(&phba->io_sgl_lock);
4117                         free_io_sgl_handle(phba, io_task->psgl_handle);
4118                         spin_unlock(&phba->io_sgl_lock);
4119                         io_task->psgl_handle = NULL;
4120                 }
4121         } else {
4122                 if (!beiscsi_conn->login_in_progress) {
4123                         if (io_task->pwrb_handle) {
4124                                 free_wrb_handle(phba, pwrb_context,
4125                                                 io_task->pwrb_handle);
4126                                 io_task->pwrb_handle = NULL;
4127                         }
4128                         if (io_task->psgl_handle) {
4129                                 spin_lock(&phba->mgmt_sgl_lock);
4130                                 free_mgmt_sgl_handle(phba,
4131                                                      io_task->psgl_handle);
4132                                 spin_unlock(&phba->mgmt_sgl_lock);
4133                                 io_task->psgl_handle = NULL;
4134                         }
4135                         if (io_task->mtask_addr) {
4136                                 pci_unmap_single(phba->pcidev,
4137                                                  io_task->mtask_addr,
4138                                                  io_task->mtask_data_count,
4139                                                  PCI_DMA_TODEVICE);
4140                                 io_task->mtask_addr = 0;
4141                         }
4142                 }
4143         }
4144 }
4145
4146 void
4147 beiscsi_offload_connection(struct beiscsi_conn *beiscsi_conn,
4148                            struct beiscsi_offload_params *params)
4149 {
4150         struct wrb_handle *pwrb_handle;
4151         struct beiscsi_hba *phba = beiscsi_conn->phba;
4152         struct iscsi_task *task = beiscsi_conn->task;
4153         struct iscsi_session *session = task->conn->session;
4154         u32 doorbell = 0;
4155
4156         /*
4157          * We can always use 0 here because it is reserved by libiscsi for
4158          * login/startup related tasks.
4159          */
4160         beiscsi_conn->login_in_progress = 0;
4161         spin_lock_bh(&session->lock);
4162         beiscsi_cleanup_task(task);
4163         spin_unlock_bh(&session->lock);
4164
4165         pwrb_handle = alloc_wrb_handle(phba, (beiscsi_conn->beiscsi_conn_cid -
4166                                        phba->fw_config.iscsi_cid_start));
4167
4168         /* Check for the adapter family */
4169         if (chip_skh_r(phba->pcidev))
4170                 beiscsi_offload_cxn_v2(params, pwrb_handle);
4171         else
4172                 beiscsi_offload_cxn_v0(params, pwrb_handle,
4173                                        phba->init_mem);
4174
4175         be_dws_le_to_cpu(pwrb_handle->pwrb,
4176                          sizeof(struct iscsi_target_context_update_wrb));
4177
4178         doorbell |= beiscsi_conn->beiscsi_conn_cid & DB_WRB_POST_CID_MASK;
4179         doorbell |= (pwrb_handle->wrb_index & DB_DEF_PDU_WRB_INDEX_MASK)
4180                              << DB_DEF_PDU_WRB_INDEX_SHIFT;
4181         doorbell |= 1 << DB_DEF_PDU_NUM_POSTED_SHIFT;
4182
4183         iowrite32(doorbell, phba->db_va + DB_TXULP0_OFFSET);
4184 }
4185
4186 static void beiscsi_parse_pdu(struct iscsi_conn *conn, itt_t itt,
4187                               int *index, int *age)
4188 {
4189         *index = (int)itt;
4190         if (age)
4191                 *age = conn->session->age;
4192 }
4193
4194 /**
4195  * beiscsi_alloc_pdu - allocates pdu and related resources
4196  * @task: libiscsi task
4197  * @opcode: opcode of pdu for task
4198  *
4199  * This is called with the session lock held. It will allocate
4200  * the wrb and sgl if needed for the command. And it will prep
4201  * the pdu's itt. beiscsi_parse_pdu will later translate
4202  * the pdu itt to the libiscsi task itt.
4203  */
4204 static int beiscsi_alloc_pdu(struct iscsi_task *task, uint8_t opcode)
4205 {
4206         struct beiscsi_io_task *io_task = task->dd_data;
4207         struct iscsi_conn *conn = task->conn;
4208         struct beiscsi_conn *beiscsi_conn = conn->dd_data;
4209         struct beiscsi_hba *phba = beiscsi_conn->phba;
4210         struct hwi_wrb_context *pwrb_context;
4211         struct hwi_controller *phwi_ctrlr;
4212         itt_t itt;
4213         struct beiscsi_session *beiscsi_sess = beiscsi_conn->beiscsi_sess;
4214         dma_addr_t paddr;
4215
4216         io_task->cmd_bhs = pci_pool_alloc(beiscsi_sess->bhs_pool,
4217                                           GFP_ATOMIC, &paddr);
4218         if (!io_task->cmd_bhs)
4219                 return -ENOMEM;
4220         io_task->bhs_pa.u.a64.address = paddr;
4221         io_task->libiscsi_itt = (itt_t)task->itt;
4222         io_task->conn = beiscsi_conn;
4223
4224         task->hdr = (struct iscsi_hdr *)&io_task->cmd_bhs->iscsi_hdr;
4225         task->hdr_max = sizeof(struct be_cmd_bhs);
4226         io_task->psgl_handle = NULL;
4227         io_task->pwrb_handle = NULL;
4228
4229         if (task->sc) {
4230                 spin_lock(&phba->io_sgl_lock);
4231                 io_task->psgl_handle = alloc_io_sgl_handle(phba);
4232                 spin_unlock(&phba->io_sgl_lock);
4233                 if (!io_task->psgl_handle) {
4234                         beiscsi_log(phba, KERN_ERR,
4235                                     BEISCSI_LOG_IO | BEISCSI_LOG_CONFIG,
4236                                     "BM_%d : Alloc of IO_SGL_ICD Failed"
4237                                     "for the CID : %d\n",
4238                                     beiscsi_conn->beiscsi_conn_cid);
4239                         goto free_hndls;
4240                 }
4241                 io_task->pwrb_handle = alloc_wrb_handle(phba,
4242                                         beiscsi_conn->beiscsi_conn_cid -
4243                                         phba->fw_config.iscsi_cid_start);
4244                 if (!io_task->pwrb_handle) {
4245                         beiscsi_log(phba, KERN_ERR,
4246                                     BEISCSI_LOG_IO | BEISCSI_LOG_CONFIG,
4247                                     "BM_%d : Alloc of WRB_HANDLE Failed"
4248                                     "for the CID : %d\n",
4249                                     beiscsi_conn->beiscsi_conn_cid);
4250                         goto free_io_hndls;
4251                 }
4252         } else {
4253                 io_task->scsi_cmnd = NULL;
4254                 if ((opcode & ISCSI_OPCODE_MASK) == ISCSI_OP_LOGIN) {
4255                         if (!beiscsi_conn->login_in_progress) {
4256                                 spin_lock(&phba->mgmt_sgl_lock);
4257                                 io_task->psgl_handle = (struct sgl_handle *)
4258                                                 alloc_mgmt_sgl_handle(phba);
4259                                 spin_unlock(&phba->mgmt_sgl_lock);
4260                                 if (!io_task->psgl_handle) {
4261                                         beiscsi_log(phba, KERN_ERR,
4262                                                     BEISCSI_LOG_IO |
4263                                                     BEISCSI_LOG_CONFIG,
4264                                                     "BM_%d : Alloc of MGMT_SGL_ICD Failed"
4265                                                     "for the CID : %d\n",
4266                                                     beiscsi_conn->
4267                                                     beiscsi_conn_cid);
4268                                         goto free_hndls;
4269                                 }
4270
4271                                 beiscsi_conn->login_in_progress = 1;
4272                                 beiscsi_conn->plogin_sgl_handle =
4273                                                         io_task->psgl_handle;
4274                                 io_task->pwrb_handle =
4275                                         alloc_wrb_handle(phba,
4276                                         beiscsi_conn->beiscsi_conn_cid -
4277                                         phba->fw_config.iscsi_cid_start);
4278                                 if (!io_task->pwrb_handle) {
4279                                         beiscsi_log(phba, KERN_ERR,
4280                                                     BEISCSI_LOG_IO |
4281                                                     BEISCSI_LOG_CONFIG,
4282                                                     "BM_%d : Alloc of WRB_HANDLE Failed"
4283                                                     "for the CID : %d\n",
4284                                                     beiscsi_conn->
4285                                                     beiscsi_conn_cid);
4286                                         goto free_mgmt_hndls;
4287                                 }
4288                                 beiscsi_conn->plogin_wrb_handle =
4289                                                         io_task->pwrb_handle;
4290
4291                         } else {
4292                                 io_task->psgl_handle =
4293                                                 beiscsi_conn->plogin_sgl_handle;
4294                                 io_task->pwrb_handle =
4295                                                 beiscsi_conn->plogin_wrb_handle;
4296                         }
4297                         beiscsi_conn->task = task;
4298                 } else {
4299                         spin_lock(&phba->mgmt_sgl_lock);
4300                         io_task->psgl_handle = alloc_mgmt_sgl_handle(phba);
4301                         spin_unlock(&phba->mgmt_sgl_lock);
4302                         if (!io_task->psgl_handle) {
4303                                 beiscsi_log(phba, KERN_ERR,
4304                                             BEISCSI_LOG_IO |
4305                                             BEISCSI_LOG_CONFIG,
4306                                             "BM_%d : Alloc of MGMT_SGL_ICD Failed"
4307                                             "for the CID : %d\n",
4308                                             beiscsi_conn->
4309                                             beiscsi_conn_cid);
4310                                 goto free_hndls;
4311                         }
4312                         io_task->pwrb_handle =
4313                                         alloc_wrb_handle(phba,
4314                                         beiscsi_conn->beiscsi_conn_cid -
4315                                         phba->fw_config.iscsi_cid_start);
4316                         if (!io_task->pwrb_handle) {
4317                                 beiscsi_log(phba, KERN_ERR,
4318                                             BEISCSI_LOG_IO | BEISCSI_LOG_CONFIG,
4319                                             "BM_%d : Alloc of WRB_HANDLE Failed"
4320                                             "for the CID : %d\n",
4321                                             beiscsi_conn->beiscsi_conn_cid);
4322                                 goto free_mgmt_hndls;
4323                         }
4324
4325                 }
4326         }
4327         itt = (itt_t) cpu_to_be32(((unsigned int)io_task->pwrb_handle->
4328                                  wrb_index << 16) | (unsigned int)
4329                                 (io_task->psgl_handle->sgl_index));
4330         io_task->pwrb_handle->pio_handle = task;
4331
4332         io_task->cmd_bhs->iscsi_hdr.itt = itt;
4333         return 0;
4334
4335 free_io_hndls:
4336         spin_lock(&phba->io_sgl_lock);
4337         free_io_sgl_handle(phba, io_task->psgl_handle);
4338         spin_unlock(&phba->io_sgl_lock);
4339         goto free_hndls;
4340 free_mgmt_hndls:
4341         spin_lock(&phba->mgmt_sgl_lock);
4342         free_mgmt_sgl_handle(phba, io_task->psgl_handle);
4343         spin_unlock(&phba->mgmt_sgl_lock);
4344 free_hndls:
4345         phwi_ctrlr = phba->phwi_ctrlr;
4346         pwrb_context = &phwi_ctrlr->wrb_context[
4347                         beiscsi_conn->beiscsi_conn_cid -
4348                         phba->fw_config.iscsi_cid_start];
4349         if (io_task->pwrb_handle)
4350                 free_wrb_handle(phba, pwrb_context, io_task->pwrb_handle);
4351         io_task->pwrb_handle = NULL;
4352         pci_pool_free(beiscsi_sess->bhs_pool, io_task->cmd_bhs,
4353                       io_task->bhs_pa.u.a64.address);
4354         io_task->cmd_bhs = NULL;
4355         return -ENOMEM;
4356 }
4357 int beiscsi_iotask_v2(struct iscsi_task *task, struct scatterlist *sg,
4358                        unsigned int num_sg, unsigned int xferlen,
4359                        unsigned int writedir)
4360 {
4361
4362         struct beiscsi_io_task *io_task = task->dd_data;
4363         struct iscsi_conn *conn = task->conn;
4364         struct beiscsi_conn *beiscsi_conn = conn->dd_data;
4365         struct beiscsi_hba *phba = beiscsi_conn->phba;
4366         struct iscsi_wrb *pwrb = NULL;
4367         unsigned int doorbell = 0;
4368
4369         pwrb = io_task->pwrb_handle->pwrb;
4370         memset(pwrb, 0, sizeof(*pwrb));
4371
4372         io_task->cmd_bhs->iscsi_hdr.exp_statsn = 0;
4373         io_task->bhs_len = sizeof(struct be_cmd_bhs);
4374
4375         if (writedir) {
4376                 AMAP_SET_BITS(struct amap_iscsi_wrb_v2, type, pwrb,
4377                               INI_WR_CMD);
4378                 AMAP_SET_BITS(struct amap_iscsi_wrb_v2, dsp, pwrb, 1);
4379         } else {
4380                 AMAP_SET_BITS(struct amap_iscsi_wrb_v2, type, pwrb,
4381                               INI_RD_CMD);
4382                 AMAP_SET_BITS(struct amap_iscsi_wrb_v2, dsp, pwrb, 0);
4383         }
4384
4385         io_task->wrb_type = AMAP_GET_BITS(struct amap_iscsi_wrb_v2,
4386                                           type, pwrb);
4387
4388         AMAP_SET_BITS(struct amap_iscsi_wrb_v2, lun, pwrb,
4389                       cpu_to_be16(*(unsigned short *)
4390                       &io_task->cmd_bhs->iscsi_hdr.lun));
4391         AMAP_SET_BITS(struct amap_iscsi_wrb_v2, r2t_exp_dtl, pwrb, xferlen);
4392         AMAP_SET_BITS(struct amap_iscsi_wrb_v2, wrb_idx, pwrb,
4393                       io_task->pwrb_handle->wrb_index);
4394         AMAP_SET_BITS(struct amap_iscsi_wrb_v2, cmdsn_itt, pwrb,
4395                       be32_to_cpu(task->cmdsn));
4396         AMAP_SET_BITS(struct amap_iscsi_wrb_v2, sgl_idx, pwrb,
4397                       io_task->psgl_handle->sgl_index);
4398
4399         hwi_write_sgl_v2(pwrb, sg, num_sg, io_task);
4400         AMAP_SET_BITS(struct amap_iscsi_wrb_v2, ptr2nextwrb, pwrb,
4401                       io_task->pwrb_handle->nxt_wrb_index);
4402
4403         be_dws_le_to_cpu(pwrb, sizeof(struct iscsi_wrb));
4404
4405         doorbell |= beiscsi_conn->beiscsi_conn_cid & DB_WRB_POST_CID_MASK;
4406         doorbell |= (io_task->pwrb_handle->wrb_index &
4407                      DB_DEF_PDU_WRB_INDEX_MASK) <<
4408                      DB_DEF_PDU_WRB_INDEX_SHIFT;
4409         doorbell |= 1 << DB_DEF_PDU_NUM_POSTED_SHIFT;
4410         iowrite32(doorbell, phba->db_va + DB_TXULP0_OFFSET);
4411         return 0;
4412 }
4413
4414 static int beiscsi_iotask(struct iscsi_task *task, struct scatterlist *sg,
4415                           unsigned int num_sg, unsigned int xferlen,
4416                           unsigned int writedir)
4417 {
4418
4419         struct beiscsi_io_task *io_task = task->dd_data;
4420         struct iscsi_conn *conn = task->conn;
4421         struct beiscsi_conn *beiscsi_conn = conn->dd_data;
4422         struct beiscsi_hba *phba = beiscsi_conn->phba;
4423         struct iscsi_wrb *pwrb = NULL;
4424         unsigned int doorbell = 0;
4425
4426         pwrb = io_task->pwrb_handle->pwrb;
4427         io_task->cmd_bhs->iscsi_hdr.exp_statsn = 0;
4428         io_task->bhs_len = sizeof(struct be_cmd_bhs);
4429
4430         if (writedir) {
4431                 AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb,
4432                               INI_WR_CMD);
4433                 AMAP_SET_BITS(struct amap_iscsi_wrb, dsp, pwrb, 1);
4434         } else {
4435                 AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb,
4436                               INI_RD_CMD);
4437                 AMAP_SET_BITS(struct amap_iscsi_wrb, dsp, pwrb, 0);
4438         }
4439
4440         io_task->wrb_type = AMAP_GET_BITS(struct amap_iscsi_wrb,
4441                                           type, pwrb);
4442
4443         AMAP_SET_BITS(struct amap_iscsi_wrb, lun, pwrb,
4444                       cpu_to_be16(*(unsigned short *)
4445                                   &io_task->cmd_bhs->iscsi_hdr.lun));
4446         AMAP_SET_BITS(struct amap_iscsi_wrb, r2t_exp_dtl, pwrb, xferlen);
4447         AMAP_SET_BITS(struct amap_iscsi_wrb, wrb_idx, pwrb,
4448                       io_task->pwrb_handle->wrb_index);
4449         AMAP_SET_BITS(struct amap_iscsi_wrb, cmdsn_itt, pwrb,
4450                       be32_to_cpu(task->cmdsn));
4451         AMAP_SET_BITS(struct amap_iscsi_wrb, sgl_icd_idx, pwrb,
4452                       io_task->psgl_handle->sgl_index);
4453
4454         hwi_write_sgl(pwrb, sg, num_sg, io_task);
4455
4456         AMAP_SET_BITS(struct amap_iscsi_wrb, ptr2nextwrb, pwrb,
4457                       io_task->pwrb_handle->nxt_wrb_index);
4458         be_dws_le_to_cpu(pwrb, sizeof(struct iscsi_wrb));
4459
4460         doorbell |= beiscsi_conn->beiscsi_conn_cid & DB_WRB_POST_CID_MASK;
4461         doorbell |= (io_task->pwrb_handle->wrb_index &
4462                      DB_DEF_PDU_WRB_INDEX_MASK) << DB_DEF_PDU_WRB_INDEX_SHIFT;
4463         doorbell |= 1 << DB_DEF_PDU_NUM_POSTED_SHIFT;
4464
4465         iowrite32(doorbell, phba->db_va + DB_TXULP0_OFFSET);
4466         return 0;
4467 }
4468
4469 static int beiscsi_mtask(struct iscsi_task *task)
4470 {
4471         struct beiscsi_io_task *io_task = task->dd_data;
4472         struct iscsi_conn *conn = task->conn;
4473         struct beiscsi_conn *beiscsi_conn = conn->dd_data;
4474         struct beiscsi_hba *phba = beiscsi_conn->phba;
4475         struct iscsi_wrb *pwrb = NULL;
4476         unsigned int doorbell = 0;
4477         unsigned int cid;
4478         unsigned int pwrb_typeoffset = 0;
4479
4480         cid = beiscsi_conn->beiscsi_conn_cid;
4481         pwrb = io_task->pwrb_handle->pwrb;
4482         memset(pwrb, 0, sizeof(*pwrb));
4483
4484         if (chip_skh_r(phba->pcidev)) {
4485                 AMAP_SET_BITS(struct amap_iscsi_wrb_v2, cmdsn_itt, pwrb,
4486                               be32_to_cpu(task->cmdsn));
4487                 AMAP_SET_BITS(struct amap_iscsi_wrb_v2, wrb_idx, pwrb,
4488                               io_task->pwrb_handle->wrb_index);
4489                 AMAP_SET_BITS(struct amap_iscsi_wrb_v2, sgl_idx, pwrb,
4490                               io_task->psgl_handle->sgl_index);
4491                 AMAP_SET_BITS(struct amap_iscsi_wrb_v2, r2t_exp_dtl, pwrb,
4492                               task->data_count);
4493                 AMAP_SET_BITS(struct amap_iscsi_wrb_v2, ptr2nextwrb, pwrb,
4494                               io_task->pwrb_handle->nxt_wrb_index);
4495                 pwrb_typeoffset = SKH_WRB_TYPE_OFFSET;
4496         } else {
4497                 AMAP_SET_BITS(struct amap_iscsi_wrb, cmdsn_itt, pwrb,
4498                               be32_to_cpu(task->cmdsn));
4499                 AMAP_SET_BITS(struct amap_iscsi_wrb, wrb_idx, pwrb,
4500                               io_task->pwrb_handle->wrb_index);
4501                 AMAP_SET_BITS(struct amap_iscsi_wrb, sgl_icd_idx, pwrb,
4502                               io_task->psgl_handle->sgl_index);
4503                 AMAP_SET_BITS(struct amap_iscsi_wrb, r2t_exp_dtl, pwrb,
4504                               task->data_count);
4505                 AMAP_SET_BITS(struct amap_iscsi_wrb, ptr2nextwrb, pwrb,
4506                               io_task->pwrb_handle->nxt_wrb_index);
4507                 pwrb_typeoffset = BE_WRB_TYPE_OFFSET;
4508         }
4509
4510
4511         switch (task->hdr->opcode & ISCSI_OPCODE_MASK) {
4512         case ISCSI_OP_LOGIN:
4513                 AMAP_SET_BITS(struct amap_iscsi_wrb, cmdsn_itt, pwrb, 1);
4514                 ADAPTER_SET_WRB_TYPE(pwrb, TGT_DM_CMD, pwrb_typeoffset);
4515                 hwi_write_buffer(pwrb, task);
4516                 break;
4517         case ISCSI_OP_NOOP_OUT:
4518                 if (task->hdr->ttt != ISCSI_RESERVED_TAG) {
4519                         ADAPTER_SET_WRB_TYPE(pwrb, TGT_DM_CMD, pwrb_typeoffset);
4520                         if (chip_skh_r(phba->pcidev))
4521                                 AMAP_SET_BITS(struct amap_iscsi_wrb_v2,
4522                                               dmsg, pwrb, 1);
4523                         else
4524                                 AMAP_SET_BITS(struct amap_iscsi_wrb,
4525                                               dmsg, pwrb, 1);
4526                 } else {
4527                         ADAPTER_SET_WRB_TYPE(pwrb, INI_RD_CMD, pwrb_typeoffset);
4528                         if (chip_skh_r(phba->pcidev))
4529                                 AMAP_SET_BITS(struct amap_iscsi_wrb_v2,
4530                                               dmsg, pwrb, 0);
4531                         else
4532                                 AMAP_SET_BITS(struct amap_iscsi_wrb,
4533                                               dmsg, pwrb, 0);
4534                 }
4535                 hwi_write_buffer(pwrb, task);
4536                 break;
4537         case ISCSI_OP_TEXT:
4538                 ADAPTER_SET_WRB_TYPE(pwrb, TGT_DM_CMD, pwrb_typeoffset);
4539                 hwi_write_buffer(pwrb, task);
4540                 break;
4541         case ISCSI_OP_SCSI_TMFUNC:
4542                 ADAPTER_SET_WRB_TYPE(pwrb, INI_TMF_CMD, pwrb_typeoffset);
4543                 hwi_write_buffer(pwrb, task);
4544                 break;
4545         case ISCSI_OP_LOGOUT:
4546                 ADAPTER_SET_WRB_TYPE(pwrb, HWH_TYPE_LOGOUT, pwrb_typeoffset);
4547                 hwi_write_buffer(pwrb, task);
4548                 break;
4549
4550         default:
4551                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_CONFIG,
4552                             "BM_%d : opcode =%d Not supported\n",
4553                             task->hdr->opcode & ISCSI_OPCODE_MASK);
4554
4555                 return -EINVAL;
4556         }
4557
4558         /* Set the task type */
4559         io_task->wrb_type = (chip_skh_r(phba->pcidev)) ?
4560                 AMAP_GET_BITS(struct amap_iscsi_wrb_v2, type, pwrb) :
4561                 AMAP_GET_BITS(struct amap_iscsi_wrb, type, pwrb);
4562
4563         doorbell |= cid & DB_WRB_POST_CID_MASK;
4564         doorbell |= (io_task->pwrb_handle->wrb_index &
4565                      DB_DEF_PDU_WRB_INDEX_MASK) << DB_DEF_PDU_WRB_INDEX_SHIFT;
4566         doorbell |= 1 << DB_DEF_PDU_NUM_POSTED_SHIFT;
4567         iowrite32(doorbell, phba->db_va + DB_TXULP0_OFFSET);
4568         return 0;
4569 }
4570
4571 static int beiscsi_task_xmit(struct iscsi_task *task)
4572 {
4573         struct beiscsi_io_task *io_task = task->dd_data;
4574         struct scsi_cmnd *sc = task->sc;
4575         struct beiscsi_hba *phba = NULL;
4576         struct scatterlist *sg;
4577         int num_sg;
4578         unsigned int  writedir = 0, xferlen = 0;
4579
4580         phba = ((struct beiscsi_conn *)task->conn->dd_data)->phba;
4581
4582         if (!sc)
4583                 return beiscsi_mtask(task);
4584
4585         io_task->scsi_cmnd = sc;
4586         num_sg = scsi_dma_map(sc);
4587         if (num_sg < 0) {
4588                 struct iscsi_conn *conn = task->conn;
4589                 struct beiscsi_hba *phba = NULL;
4590
4591                 phba = ((struct beiscsi_conn *)conn->dd_data)->phba;
4592                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_IO,
4593                             "BM_%d : scsi_dma_map Failed\n");
4594
4595                 return num_sg;
4596         }
4597         xferlen = scsi_bufflen(sc);
4598         sg = scsi_sglist(sc);
4599         if (sc->sc_data_direction == DMA_TO_DEVICE)
4600                 writedir = 1;
4601          else
4602                 writedir = 0;
4603
4604          return phba->iotask_fn(task, sg, num_sg, xferlen, writedir);
4605 }
4606
4607 /**
4608  * beiscsi_bsg_request - handle bsg request from ISCSI transport
4609  * @job: job to handle
4610  */
4611 static int beiscsi_bsg_request(struct bsg_job *job)
4612 {
4613         struct Scsi_Host *shost;
4614         struct beiscsi_hba *phba;
4615         struct iscsi_bsg_request *bsg_req = job->request;
4616         int rc = -EINVAL;
4617         unsigned int tag;
4618         struct be_dma_mem nonemb_cmd;
4619         struct be_cmd_resp_hdr *resp;
4620         struct iscsi_bsg_reply *bsg_reply = job->reply;
4621         unsigned short status, extd_status;
4622
4623         shost = iscsi_job_to_shost(job);
4624         phba = iscsi_host_priv(shost);
4625
4626         switch (bsg_req->msgcode) {
4627         case ISCSI_BSG_HST_VENDOR:
4628                 nonemb_cmd.va = pci_alloc_consistent(phba->ctrl.pdev,
4629                                         job->request_payload.payload_len,
4630                                         &nonemb_cmd.dma);
4631                 if (nonemb_cmd.va == NULL) {
4632                         beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_CONFIG,
4633                                     "BM_%d : Failed to allocate memory for "
4634                                     "beiscsi_bsg_request\n");
4635                         return -ENOMEM;
4636                 }
4637                 tag = mgmt_vendor_specific_fw_cmd(&phba->ctrl, phba, job,
4638                                                   &nonemb_cmd);
4639                 if (!tag) {
4640                         beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_CONFIG,
4641                                     "BM_%d : MBX Tag Allocation Failed\n");
4642
4643                         pci_free_consistent(phba->ctrl.pdev, nonemb_cmd.size,
4644                                             nonemb_cmd.va, nonemb_cmd.dma);
4645                         return -EAGAIN;
4646                 } else
4647                         wait_event_interruptible(phba->ctrl.mcc_wait[tag],
4648                                                  phba->ctrl.mcc_numtag[tag]);
4649                 extd_status = (phba->ctrl.mcc_numtag[tag] & 0x0000FF00) >> 8;
4650                 status = phba->ctrl.mcc_numtag[tag] & 0x000000FF;
4651                 free_mcc_tag(&phba->ctrl, tag);
4652                 resp = (struct be_cmd_resp_hdr *)nonemb_cmd.va;
4653                 sg_copy_from_buffer(job->reply_payload.sg_list,
4654                                     job->reply_payload.sg_cnt,
4655                                     nonemb_cmd.va, (resp->response_length
4656                                     + sizeof(*resp)));
4657                 bsg_reply->reply_payload_rcv_len = resp->response_length;
4658                 bsg_reply->result = status;
4659                 bsg_job_done(job, bsg_reply->result,
4660                              bsg_reply->reply_payload_rcv_len);
4661                 pci_free_consistent(phba->ctrl.pdev, nonemb_cmd.size,
4662                                     nonemb_cmd.va, nonemb_cmd.dma);
4663                 if (status || extd_status) {
4664                         beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_CONFIG,
4665                                     "BM_%d : MBX Cmd Failed"
4666                                     " status = %d extd_status = %d\n",
4667                                     status, extd_status);
4668
4669                         return -EIO;
4670                 } else {
4671                         rc = 0;
4672                 }
4673                 break;
4674
4675         default:
4676                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_CONFIG,
4677                                 "BM_%d : Unsupported bsg command: 0x%x\n",
4678                                 bsg_req->msgcode);
4679                 break;
4680         }
4681
4682         return rc;
4683 }
4684
4685 void beiscsi_hba_attrs_init(struct beiscsi_hba *phba)
4686 {
4687         /* Set the logging parameter */
4688         beiscsi_log_enable_init(phba, beiscsi_log_enable);
4689 }
4690
4691 /*
4692  * beiscsi_quiesce()- Cleanup Driver resources
4693  * @phba: Instance Priv structure
4694  *
4695  * Free the OS and HW resources held by the driver
4696  **/
4697 static void beiscsi_quiesce(struct beiscsi_hba *phba)
4698 {
4699         struct hwi_controller *phwi_ctrlr;
4700         struct hwi_context_memory *phwi_context;
4701         struct be_eq_obj *pbe_eq;
4702         unsigned int i, msix_vec;
4703
4704         phwi_ctrlr = phba->phwi_ctrlr;
4705         phwi_context = phwi_ctrlr->phwi_ctxt;
4706         hwi_disable_intr(phba);
4707         if (phba->msix_enabled) {
4708                 for (i = 0; i <= phba->num_cpus; i++) {
4709                         msix_vec = phba->msix_entries[i].vector;
4710                         free_irq(msix_vec, &phwi_context->be_eq[i]);
4711                         kfree(phba->msi_name[i]);
4712                 }
4713         } else
4714                 if (phba->pcidev->irq)
4715                         free_irq(phba->pcidev->irq, phba);
4716         pci_disable_msix(phba->pcidev);
4717         destroy_workqueue(phba->wq);
4718         if (blk_iopoll_enabled)
4719                 for (i = 0; i < phba->num_cpus; i++) {
4720                         pbe_eq = &phwi_context->be_eq[i];
4721                         blk_iopoll_disable(&pbe_eq->iopoll);
4722                 }
4723
4724         beiscsi_clean_port(phba);
4725         beiscsi_free_mem(phba);
4726
4727         beiscsi_unmap_pci_function(phba);
4728         pci_free_consistent(phba->pcidev,
4729                             phba->ctrl.mbox_mem_alloced.size,
4730                             phba->ctrl.mbox_mem_alloced.va,
4731                             phba->ctrl.mbox_mem_alloced.dma);
4732 }
4733
4734 static void beiscsi_remove(struct pci_dev *pcidev)
4735 {
4736
4737         struct beiscsi_hba *phba = NULL;
4738
4739         phba = pci_get_drvdata(pcidev);
4740         if (!phba) {
4741                 dev_err(&pcidev->dev, "beiscsi_remove called with no phba\n");
4742                 return;
4743         }
4744
4745         beiscsi_destroy_def_ifaces(phba);
4746         beiscsi_quiesce(phba);
4747         iscsi_boot_destroy_kset(phba->boot_kset);
4748         iscsi_host_remove(phba->shost);
4749         pci_dev_put(phba->pcidev);
4750         iscsi_host_free(phba->shost);
4751         pci_disable_device(pcidev);
4752 }
4753
4754 static void beiscsi_shutdown(struct pci_dev *pcidev)
4755 {
4756
4757         struct beiscsi_hba *phba = NULL;
4758
4759         phba = (struct beiscsi_hba *)pci_get_drvdata(pcidev);
4760         if (!phba) {
4761                 dev_err(&pcidev->dev, "beiscsi_shutdown called with no phba\n");
4762                 return;
4763         }
4764
4765         beiscsi_quiesce(phba);
4766         pci_disable_device(pcidev);
4767 }
4768
4769 static void beiscsi_msix_enable(struct beiscsi_hba *phba)
4770 {
4771         int i, status;
4772
4773         for (i = 0; i <= phba->num_cpus; i++)
4774                 phba->msix_entries[i].entry = i;
4775
4776         status = pci_enable_msix(phba->pcidev, phba->msix_entries,
4777                                  (phba->num_cpus + 1));
4778         if (!status)
4779                 phba->msix_enabled = true;
4780
4781         return;
4782 }
4783
4784 static int __devinit beiscsi_dev_probe(struct pci_dev *pcidev,
4785                                 const struct pci_device_id *id)
4786 {
4787         struct beiscsi_hba *phba = NULL;
4788         struct hwi_controller *phwi_ctrlr;
4789         struct hwi_context_memory *phwi_context;
4790         struct be_eq_obj *pbe_eq;
4791         int ret, i;
4792
4793         ret = beiscsi_enable_pci(pcidev);
4794         if (ret < 0) {
4795                 dev_err(&pcidev->dev,
4796                         "beiscsi_dev_probe - Failed to enable pci device\n");
4797                 return ret;
4798         }
4799
4800         phba = beiscsi_hba_alloc(pcidev);
4801         if (!phba) {
4802                 dev_err(&pcidev->dev,
4803                         "beiscsi_dev_probe - Failed in beiscsi_hba_alloc\n");
4804                 goto disable_pci;
4805         }
4806
4807         /* Initialize Driver configuration Paramters */
4808         beiscsi_hba_attrs_init(phba);
4809
4810         switch (pcidev->device) {
4811         case BE_DEVICE_ID1:
4812         case OC_DEVICE_ID1:
4813         case OC_DEVICE_ID2:
4814                 phba->generation = BE_GEN2;
4815                 phba->iotask_fn = beiscsi_iotask;
4816                 break;
4817         case BE_DEVICE_ID2:
4818         case OC_DEVICE_ID3:
4819                 phba->generation = BE_GEN3;
4820                 phba->iotask_fn = beiscsi_iotask;
4821                 break;
4822         case OC_SKH_ID1:
4823                 phba->generation = BE_GEN4;
4824                 phba->iotask_fn = beiscsi_iotask_v2;
4825         default:
4826                 phba->generation = 0;
4827         }
4828
4829         if (enable_msix)
4830                 find_num_cpus(phba);
4831         else
4832                 phba->num_cpus = 1;
4833
4834         beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
4835                     "BM_%d : num_cpus = %d\n",
4836                     phba->num_cpus);
4837
4838         if (enable_msix) {
4839                 beiscsi_msix_enable(phba);
4840                 if (!phba->msix_enabled)
4841                         phba->num_cpus = 1;
4842         }
4843         ret = be_ctrl_init(phba, pcidev);
4844         if (ret) {
4845                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
4846                             "BM_%d : beiscsi_dev_probe-"
4847                             "Failed in be_ctrl_init\n");
4848                 goto hba_free;
4849         }
4850
4851         ret = beiscsi_cmd_reset_function(phba);
4852         if (ret) {
4853                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
4854                             "BM_%d : Reset Failed. Aborting Crashdump\n");
4855                 goto hba_free;
4856         }
4857         ret = be_chk_reset_complete(phba);
4858         if (ret) {
4859                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
4860                             "BM_%d : Failed to get out of reset."
4861                             "Aborting Crashdump\n");
4862                 goto hba_free;
4863         }
4864
4865         spin_lock_init(&phba->io_sgl_lock);
4866         spin_lock_init(&phba->mgmt_sgl_lock);
4867         spin_lock_init(&phba->isr_lock);
4868         ret = mgmt_get_fw_config(&phba->ctrl, phba);
4869         if (ret != 0) {
4870                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
4871                             "BM_%d : Error getting fw config\n");
4872                 goto free_port;
4873         }
4874         phba->shost->max_id = phba->fw_config.iscsi_cid_count;
4875         beiscsi_get_params(phba);
4876         phba->shost->can_queue = phba->params.ios_per_ctrl;
4877         ret = beiscsi_init_port(phba);
4878         if (ret < 0) {
4879                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
4880                             "BM_%d : beiscsi_dev_probe-"
4881                             "Failed in beiscsi_init_port\n");
4882                 goto free_port;
4883         }
4884
4885         for (i = 0; i < MAX_MCC_CMD ; i++) {
4886                 init_waitqueue_head(&phba->ctrl.mcc_wait[i + 1]);
4887                 phba->ctrl.mcc_tag[i] = i + 1;
4888                 phba->ctrl.mcc_numtag[i + 1] = 0;
4889                 phba->ctrl.mcc_tag_available++;
4890         }
4891
4892         phba->ctrl.mcc_alloc_index = phba->ctrl.mcc_free_index = 0;
4893
4894         snprintf(phba->wq_name, sizeof(phba->wq_name), "beiscsi_%02x_wq",
4895                  phba->shost->host_no);
4896         phba->wq = alloc_workqueue(phba->wq_name, WQ_MEM_RECLAIM, 1);
4897         if (!phba->wq) {
4898                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
4899                             "BM_%d : beiscsi_dev_probe-"
4900                             "Failed to allocate work queue\n");
4901                 goto free_twq;
4902         }
4903
4904
4905         phwi_ctrlr = phba->phwi_ctrlr;
4906         phwi_context = phwi_ctrlr->phwi_ctxt;
4907
4908         if (blk_iopoll_enabled) {
4909                 for (i = 0; i < phba->num_cpus; i++) {
4910                         pbe_eq = &phwi_context->be_eq[i];
4911                         blk_iopoll_init(&pbe_eq->iopoll, be_iopoll_budget,
4912                                         be_iopoll);
4913                         blk_iopoll_enable(&pbe_eq->iopoll);
4914                 }
4915
4916                 i = (phba->msix_enabled) ? i : 0;
4917                 /* Work item for MCC handling */
4918                 pbe_eq = &phwi_context->be_eq[i];
4919                 INIT_WORK(&pbe_eq->work_cqs, beiscsi_process_all_cqs);
4920         } else {
4921                 if (phba->msix_enabled) {
4922                         for (i = 0; i <= phba->num_cpus; i++) {
4923                                 pbe_eq = &phwi_context->be_eq[i];
4924                                 INIT_WORK(&pbe_eq->work_cqs,
4925                                           beiscsi_process_all_cqs);
4926                         }
4927                 } else {
4928                                 pbe_eq = &phwi_context->be_eq[0];
4929                                 INIT_WORK(&pbe_eq->work_cqs,
4930                                           beiscsi_process_all_cqs);
4931                         }
4932         }
4933
4934         ret = beiscsi_init_irqs(phba);
4935         if (ret < 0) {
4936                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
4937                             "BM_%d : beiscsi_dev_probe-"
4938                             "Failed to beiscsi_init_irqs\n");
4939                 goto free_blkenbld;
4940         }
4941         hwi_enable_intr(phba);
4942
4943         if (beiscsi_setup_boot_info(phba))
4944                 /*
4945                  * log error but continue, because we may not be using
4946                  * iscsi boot.
4947                  */
4948                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
4949                             "BM_%d : Could not set up "
4950                             "iSCSI boot info.\n");
4951
4952         beiscsi_create_def_ifaces(phba);
4953         beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
4954                     "\n\n\n BM_%d : SUCCESS - DRIVER LOADED\n\n\n");
4955         return 0;
4956
4957 free_blkenbld:
4958         destroy_workqueue(phba->wq);
4959         if (blk_iopoll_enabled)
4960                 for (i = 0; i < phba->num_cpus; i++) {
4961                         pbe_eq = &phwi_context->be_eq[i];
4962                         blk_iopoll_disable(&pbe_eq->iopoll);
4963                 }
4964 free_twq:
4965         beiscsi_clean_port(phba);
4966         beiscsi_free_mem(phba);
4967 free_port:
4968         pci_free_consistent(phba->pcidev,
4969                             phba->ctrl.mbox_mem_alloced.size,
4970                             phba->ctrl.mbox_mem_alloced.va,
4971                            phba->ctrl.mbox_mem_alloced.dma);
4972         beiscsi_unmap_pci_function(phba);
4973 hba_free:
4974         if (phba->msix_enabled)
4975                 pci_disable_msix(phba->pcidev);
4976         iscsi_host_remove(phba->shost);
4977         pci_dev_put(phba->pcidev);
4978         iscsi_host_free(phba->shost);
4979 disable_pci:
4980         pci_disable_device(pcidev);
4981         return ret;
4982 }
4983
4984 struct iscsi_transport beiscsi_iscsi_transport = {
4985         .owner = THIS_MODULE,
4986         .name = DRV_NAME,
4987         .caps = CAP_RECOVERY_L0 | CAP_HDRDGST | CAP_TEXT_NEGO |
4988                 CAP_MULTI_R2T | CAP_DATADGST | CAP_DATA_PATH_OFFLOAD,
4989         .create_session = beiscsi_session_create,
4990         .destroy_session = beiscsi_session_destroy,
4991         .create_conn = beiscsi_conn_create,
4992         .bind_conn = beiscsi_conn_bind,
4993         .destroy_conn = iscsi_conn_teardown,
4994         .attr_is_visible = be2iscsi_attr_is_visible,
4995         .set_iface_param = be2iscsi_iface_set_param,
4996         .get_iface_param = be2iscsi_iface_get_param,
4997         .set_param = beiscsi_set_param,
4998         .get_conn_param = iscsi_conn_get_param,
4999         .get_session_param = iscsi_session_get_param,
5000         .get_host_param = beiscsi_get_host_param,
5001         .start_conn = beiscsi_conn_start,
5002         .stop_conn = iscsi_conn_stop,
5003         .send_pdu = iscsi_conn_send_pdu,
5004         .xmit_task = beiscsi_task_xmit,
5005         .cleanup_task = beiscsi_cleanup_task,
5006         .alloc_pdu = beiscsi_alloc_pdu,
5007         .parse_pdu_itt = beiscsi_parse_pdu,
5008         .get_stats = beiscsi_conn_get_stats,
5009         .get_ep_param = beiscsi_ep_get_param,
5010         .ep_connect = beiscsi_ep_connect,
5011         .ep_poll = beiscsi_ep_poll,
5012         .ep_disconnect = beiscsi_ep_disconnect,
5013         .session_recovery_timedout = iscsi_session_recovery_timedout,
5014         .bsg_request = beiscsi_bsg_request,
5015 };
5016
5017 static struct pci_driver beiscsi_pci_driver = {
5018         .name = DRV_NAME,
5019         .probe = beiscsi_dev_probe,
5020         .remove = beiscsi_remove,
5021         .shutdown = beiscsi_shutdown,
5022         .id_table = beiscsi_pci_id_table
5023 };
5024
5025
5026 static int __init beiscsi_module_init(void)
5027 {
5028         int ret;
5029
5030         beiscsi_scsi_transport =
5031                         iscsi_register_transport(&beiscsi_iscsi_transport);
5032         if (!beiscsi_scsi_transport) {
5033                 printk(KERN_ERR
5034                        "beiscsi_module_init - Unable to  register beiscsi transport.\n");
5035                 return -ENOMEM;
5036         }
5037         printk(KERN_INFO "In beiscsi_module_init, tt=%p\n",
5038                &beiscsi_iscsi_transport);
5039
5040         ret = pci_register_driver(&beiscsi_pci_driver);
5041         if (ret) {
5042                 printk(KERN_ERR
5043                        "beiscsi_module_init - Unable to  register beiscsi pci driver.\n");
5044                 goto unregister_iscsi_transport;
5045         }
5046         return 0;
5047
5048 unregister_iscsi_transport:
5049         iscsi_unregister_transport(&beiscsi_iscsi_transport);
5050         return ret;
5051 }
5052
5053 static void __exit beiscsi_module_exit(void)
5054 {
5055         pci_unregister_driver(&beiscsi_pci_driver);
5056         iscsi_unregister_transport(&beiscsi_iscsi_transport);
5057 }
5058
5059 module_init(beiscsi_module_init);
5060 module_exit(beiscsi_module_exit);