drivers/infiniband/ulp/iser/iser_memory.c

   1 /*
   2  * Copyright (c) 2004, 2005, 2006 Voltaire, Inc. All rights reserved.
   3  * Copyright (c) 2013-2014 Mellanox Technologies. All rights reserved.
   4  *
   5  * This software is available to you under a choice of one of two
   6  * licenses.  You may choose to be licensed under the terms of the GNU
   7  * General Public License (GPL) Version 2, available from the file
   8  * COPYING in the main directory of this source tree, or the
   9  * OpenIB.org BSD license below:
  10  *
  11  *     Redistribution and use in source and binary forms, with or
  12  *     without modification, are permitted provided that the following
  13  *     conditions are met:
  14  *
  15  *      - Redistributions of source code must retain the above
  16  *        copyright notice, this list of conditions and the following
  17  *        disclaimer.
  18  *
  19  *      - Redistributions in binary form must reproduce the above
  20  *        copyright notice, this list of conditions and the following
  21  *        disclaimer in the documentation and/or other materials
  22  *        provided with the distribution.
  23  *
  24  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  25  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  26  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  27  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  28  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  29  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  30  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  31  * SOFTWARE.
  32  */
  33 #include <linux/module.h>
  34 #include <linux/kernel.h>
  35 #include <linux/slab.h>
  36 #include <linux/mm.h>
  37 #include <linux/highmem.h>
  38 #include <linux/scatterlist.h>
  39
  40 #include "iscsi_iser.h"
  41
  42 #define ISER_KMALLOC_THRESHOLD 0x20000 /* 128K - kmalloc limit */
  43
  44 /**
  45  * iser_start_rdma_unaligned_sg
  46  */
  47 static int iser_start_rdma_unaligned_sg(struct iscsi_iser_task *iser_task,
  48                                         struct iser_data_buf *data,
  49                                         struct iser_data_buf *data_copy,
  50                                         enum iser_data_dir cmd_dir)
  51 {
  52         struct ib_device *dev = iser_task->iser_conn->ib_conn.device->ib_device;
  53         struct scatterlist *sgl = (struct scatterlist *)data->buf;
  54         struct scatterlist *sg;
  55         char *mem = NULL;
  56         unsigned long  cmd_data_len = 0;
  57         int dma_nents, i;
  58
  59         for_each_sg(sgl, sg, data->size, i)
  60                 cmd_data_len += ib_sg_dma_len(dev, sg);
  61
  62         if (cmd_data_len > ISER_KMALLOC_THRESHOLD)
  63                 mem = (void *)__get_free_pages(GFP_ATOMIC,
  64                       ilog2(roundup_pow_of_two(cmd_data_len)) - PAGE_SHIFT);
  65         else
  66                 mem = kmalloc(cmd_data_len, GFP_ATOMIC);
  67
  68         if (mem == NULL) {
  69                 iser_err("Failed to allocate mem size %d %d for copying sglist\n",
  70                          data->size, (int)cmd_data_len);
  71                 return -ENOMEM;
  72         }
  73
  74         if (cmd_dir == ISER_DIR_OUT) {
  75                 /* copy the unaligned sg the buffer which is used for RDMA */
  76                 char *p, *from;
  77
  78                 sgl = (struct scatterlist *)data->buf;
  79                 p = mem;
  80                 for_each_sg(sgl, sg, data->size, i) {
  81                         from = kmap_atomic(sg_page(sg));
  82                         memcpy(p,
  83                                from + sg->offset,
  84                                sg->length);
  85                         kunmap_atomic(from);
  86                         p += sg->length;
  87                 }
  88         }
  89
  90         sg_init_one(&data_copy->sg_single, mem, cmd_data_len);
  91         data_copy->buf = &data_copy->sg_single;
  92         data_copy->size = 1;
  93         data_copy->copy_buf = mem;
  94
  95         dma_nents = ib_dma_map_sg(dev, &data_copy->sg_single, 1,
  96                                   (cmd_dir == ISER_DIR_OUT) ?
  97                                   DMA_TO_DEVICE : DMA_FROM_DEVICE);
  98         BUG_ON(dma_nents == 0);
  99
 100         data_copy->dma_nents = dma_nents;
 101         data_copy->data_len = cmd_data_len;
 102
 103         return 0;
 104 }
 105
 106 /**
 107  * iser_finalize_rdma_unaligned_sg
 108  */
 109
 110 void iser_finalize_rdma_unaligned_sg(struct iscsi_iser_task *iser_task,
 111                                      struct iser_data_buf *data,
 112                                      struct iser_data_buf *data_copy,
 113                                      enum iser_data_dir cmd_dir)
 114 {
 115         struct ib_device *dev;
 116         unsigned long  cmd_data_len;
 117
 118         dev = iser_task->iser_conn->ib_conn.device->ib_device;
 119
 120         ib_dma_unmap_sg(dev, &data_copy->sg_single, 1,
 121                         (cmd_dir == ISER_DIR_OUT) ?
 122                         DMA_TO_DEVICE : DMA_FROM_DEVICE);
 123
 124         if (cmd_dir == ISER_DIR_IN) {
 125                 char *mem;
 126                 struct scatterlist *sgl, *sg;
 127                 unsigned char *p, *to;
 128                 unsigned int sg_size;
 129                 int i;
 130
 131                 /* copy back read RDMA to unaligned sg */
 132                 mem = data_copy->copy_buf;
 133
 134                 sgl = (struct scatterlist *)data->buf;
 135                 sg_size = data->size;
 136
 137                 p = mem;
 138                 for_each_sg(sgl, sg, sg_size, i) {
 139                         to = kmap_atomic(sg_page(sg));
 140                         memcpy(to + sg->offset,
 141                                p,
 142                                sg->length);
 143                         kunmap_atomic(to);
 144                         p += sg->length;
 145                 }
 146         }
 147
 148         cmd_data_len = data->data_len;
 149
 150         if (cmd_data_len > ISER_KMALLOC_THRESHOLD)
 151                 free_pages((unsigned long)data_copy->copy_buf,
 152                            ilog2(roundup_pow_of_two(cmd_data_len)) - PAGE_SHIFT);
 153         else
 154                 kfree(data_copy->copy_buf);
 155
 156         data_copy->copy_buf = NULL;
 157 }
 158
 159 #define IS_4K_ALIGNED(addr)     ((((unsigned long)addr) & ~MASK_4K) == 0)
 160
 161 /**
 162  * iser_sg_to_page_vec - Translates scatterlist entries to physical addresses
 163  * and returns the length of resulting physical address array (may be less than
 164  * the original due to possible compaction).
 165  *
 166  * we build a "page vec" under the assumption that the SG meets the RDMA
 167  * alignment requirements. Other then the first and last SG elements, all
 168  * the "internal" elements can be compacted into a list whose elements are
 169  * dma addresses of physical pages. The code supports also the weird case
 170  * where --few fragments of the same page-- are present in the SG as
 171  * consecutive elements. Also, it handles one entry SG.
 172  */
 173
 174 static int iser_sg_to_page_vec(struct iser_data_buf *data,
 175                                struct ib_device *ibdev, u64 *pages,
 176                                int *offset, int *data_size)
 177 {
 178         struct scatterlist *sg, *sgl = (struct scatterlist *)data->buf;
 179         u64 start_addr, end_addr, page, chunk_start = 0;
 180         unsigned long total_sz = 0;
 181         unsigned int dma_len;
 182         int i, new_chunk, cur_page, last_ent = data->dma_nents - 1;
 183
 184         /* compute the offset of first element */
 185         *offset = (u64) sgl[0].offset & ~MASK_4K;
 186
 187         new_chunk = 1;
 188         cur_page  = 0;
 189         for_each_sg(sgl, sg, data->dma_nents, i) {
 190                 start_addr = ib_sg_dma_address(ibdev, sg);
 191                 if (new_chunk)
 192                         chunk_start = start_addr;
 193                 dma_len = ib_sg_dma_len(ibdev, sg);
 194                 end_addr = start_addr + dma_len;
 195                 total_sz += dma_len;
 196
 197                 /* collect page fragments until aligned or end of SG list */
 198                 if (!IS_4K_ALIGNED(end_addr) && i < last_ent) {
 199                         new_chunk = 0;
 200                         continue;
 201                 }
 202                 new_chunk = 1;
 203
 204                 /* address of the first page in the contiguous chunk;
 205                    masking relevant for the very first SG entry,
 206                    which might be unaligned */
 207                 page = chunk_start & MASK_4K;
 208                 do {
 209                         pages[cur_page++] = page;
 210                         page += SIZE_4K;
 211                 } while (page < end_addr);
 212         }
 213
 214         *data_size = total_sz;
 215         iser_dbg("page_vec->data_size:%d cur_page %d\n",
 216                  *data_size, cur_page);
 217         return cur_page;
 218 }
 219
 220
 221 /**
 222  * iser_data_buf_aligned_len - Tries to determine the maximal correctly aligned
 223  * for RDMA sub-list of a scatter-gather list of memory buffers, and  returns
 224  * the number of entries which are aligned correctly. Supports the case where
 225  * consecutive SG elements are actually fragments of the same physcial page.
 226  */
 227 static int iser_data_buf_aligned_len(struct iser_data_buf *data,
 228                                       struct ib_device *ibdev)
 229 {
 230         struct scatterlist *sgl, *sg, *next_sg = NULL;
 231         u64 start_addr, end_addr;
 232         int i, ret_len, start_check = 0;
 233
 234         if (data->dma_nents == 1)
 235                 return 1;
 236
 237         sgl = (struct scatterlist *)data->buf;
 238         start_addr  = ib_sg_dma_address(ibdev, sgl);
 239
 240         for_each_sg(sgl, sg, data->dma_nents, i) {
 241                 if (start_check && !IS_4K_ALIGNED(start_addr))
 242                         break;
 243
 244                 next_sg = sg_next(sg);
 245                 if (!next_sg)
 246                         break;
 247
 248                 end_addr    = start_addr + ib_sg_dma_len(ibdev, sg);
 249                 start_addr  = ib_sg_dma_address(ibdev, next_sg);
 250
 251                 if (end_addr == start_addr) {
 252                         start_check = 0;
 253                         continue;
 254                 } else
 255                         start_check = 1;
 256
 257                 if (!IS_4K_ALIGNED(end_addr))
 258                         break;
 259         }
 260         ret_len = (next_sg) ? i : i+1;
 261         iser_dbg("Found %d aligned entries out of %d in sg:0x%p\n",
 262                  ret_len, data->dma_nents, data);
 263         return ret_len;
 264 }
 265
 266 static void iser_data_buf_dump(struct iser_data_buf *data,
 267                                struct ib_device *ibdev)
 268 {
 269         struct scatterlist *sgl = (struct scatterlist *)data->buf;
 270         struct scatterlist *sg;
 271         int i;
 272
 273         for_each_sg(sgl, sg, data->dma_nents, i)
 274                 iser_dbg("sg[%d] dma_addr:0x%lX page:0x%p "
 275                          "off:0x%x sz:0x%x dma_len:0x%x\n",
 276                          i, (unsigned long)ib_sg_dma_address(ibdev, sg),
 277                          sg_page(sg), sg->offset,
 278                          sg->length, ib_sg_dma_len(ibdev, sg));
 279 }
 280
 281 static void iser_dump_page_vec(struct iser_page_vec *page_vec)
 282 {
 283         int i;
 284
 285         iser_err("page vec length %d data size %d\n",
 286                  page_vec->length, page_vec->data_size);
 287         for (i = 0; i < page_vec->length; i++)
 288                 iser_err("%d %lx\n",i,(unsigned long)page_vec->pages[i]);
 289 }
 290
 291 static void iser_page_vec_build(struct iser_data_buf *data,
 292                                 struct iser_page_vec *page_vec,
 293                                 struct ib_device *ibdev)
 294 {
 295         int page_vec_len = 0;
 296
 297         page_vec->length = 0;
 298         page_vec->offset = 0;
 299
 300         iser_dbg("Translating sg sz: %d\n", data->dma_nents);
 301         page_vec_len = iser_sg_to_page_vec(data, ibdev, page_vec->pages,
 302                                            &page_vec->offset,
 303                                            &page_vec->data_size);
 304         iser_dbg("sg len %d page_vec_len %d\n", data->dma_nents, page_vec_len);
 305
 306         page_vec->length = page_vec_len;
 307
 308         if (page_vec_len * SIZE_4K < page_vec->data_size) {
 309                 iser_err("page_vec too short to hold this SG\n");
 310                 iser_data_buf_dump(data, ibdev);
 311                 iser_dump_page_vec(page_vec);
 312                 BUG();
 313         }
 314 }
 315
 316 int iser_dma_map_task_data(struct iscsi_iser_task *iser_task,
 317                             struct iser_data_buf *data,
 318                             enum iser_data_dir iser_dir,
 319                             enum dma_data_direction dma_dir)
 320 {
 321         struct ib_device *dev;
 322
 323         iser_task->dir[iser_dir] = 1;
 324         dev = iser_task->iser_conn->ib_conn.device->ib_device;
 325
 326         data->dma_nents = ib_dma_map_sg(dev, data->buf, data->size, dma_dir);
 327         if (data->dma_nents == 0) {
 328                 iser_err("dma_map_sg failed!!!\n");
 329                 return -EINVAL;
 330         }
 331         return 0;
 332 }
 333
 334 void iser_dma_unmap_task_data(struct iscsi_iser_task *iser_task,
 335                               struct iser_data_buf *data)
 336 {
 337         struct ib_device *dev;
 338
 339         dev = iser_task->iser_conn->ib_conn.device->ib_device;
 340         ib_dma_unmap_sg(dev, data->buf, data->size, DMA_FROM_DEVICE);
 341 }
 342
 343 static int fall_to_bounce_buf(struct iscsi_iser_task *iser_task,
 344                               struct ib_device *ibdev,
 345                               struct iser_data_buf *mem,
 346                               struct iser_data_buf *mem_copy,
 347                               enum iser_data_dir cmd_dir,
 348                               int aligned_len)
 349 {
 350         struct iscsi_conn    *iscsi_conn = iser_task->iser_conn->iscsi_conn;
 351
 352         iscsi_conn->fmr_unalign_cnt++;
 353         iser_warn("rdma alignment violation (%d/%d aligned) or FMR not supported\n",
 354                   aligned_len, mem->size);
 355
 356         if (iser_debug_level > 0)
 357                 iser_data_buf_dump(mem, ibdev);
 358
 359         /* unmap the command data before accessing it */
 360         iser_dma_unmap_task_data(iser_task, mem);
 361
 362         /* allocate copy buf, if we are writing, copy the */
 363         /* unaligned scatterlist, dma map the copy        */
 364         if (iser_start_rdma_unaligned_sg(iser_task, mem, mem_copy, cmd_dir) != 0)
 365                 return -ENOMEM;
 366
 367         return 0;
 368 }
 369
 370 /**
 371  * iser_reg_rdma_mem_fmr - Registers memory intended for RDMA,
 372  * using FMR (if possible) obtaining rkey and va
 373  *
 374  * returns 0 on success, errno code on failure
 375  */
 376 int iser_reg_rdma_mem_fmr(struct iscsi_iser_task *iser_task,
 377                           enum iser_data_dir cmd_dir)
 378 {
 379         struct ib_conn *ib_conn = &iser_task->iser_conn->ib_conn;
 380         struct iser_device   *device = ib_conn->device;
 381         struct ib_device     *ibdev = device->ib_device;
 382         struct iser_data_buf *mem = &iser_task->data[cmd_dir];
 383         struct iser_regd_buf *regd_buf;
 384         int aligned_len;
 385         int err;
 386         int i;
 387         struct scatterlist *sg;
 388
 389         regd_buf = &iser_task->rdma_regd[cmd_dir];
 390
 391         aligned_len = iser_data_buf_aligned_len(mem, ibdev);
 392         if (aligned_len != mem->dma_nents) {
 393                 err = fall_to_bounce_buf(iser_task, ibdev, mem,
 394                                          &iser_task->data_copy[cmd_dir],
 395                                          cmd_dir, aligned_len);
 396                 if (err) {
 397                         iser_err("failed to allocate bounce buffer\n");
 398                         return err;
 399                 }
 400                 mem = &iser_task->data_copy[cmd_dir];
 401         }
 402
 403         /* if there a single dma entry, FMR is not needed */
 404         if (mem->dma_nents == 1) {
 405                 sg = (struct scatterlist *)mem->buf;
 406
 407                 regd_buf->reg.lkey = device->mr->lkey;
 408                 regd_buf->reg.rkey = device->mr->rkey;
 409                 regd_buf->reg.len  = ib_sg_dma_len(ibdev, &sg[0]);
 410                 regd_buf->reg.va   = ib_sg_dma_address(ibdev, &sg[0]);
 411
 412                 iser_dbg("PHYSICAL Mem.register: lkey: 0x%08X rkey: 0x%08X  "
 413                          "va: 0x%08lX sz: %ld]\n",
 414                          (unsigned int)regd_buf->reg.lkey,
 415                          (unsigned int)regd_buf->reg.rkey,
 416                          (unsigned long)regd_buf->reg.va,
 417                          (unsigned long)regd_buf->reg.len);
 418         } else { /* use FMR for multiple dma entries */
 419                 iser_page_vec_build(mem, ib_conn->fmr.page_vec, ibdev);
 420                 err = iser_reg_page_vec(ib_conn, ib_conn->fmr.page_vec,
 421                                         &regd_buf->reg);
 422                 if (err && err != -EAGAIN) {
 423                         iser_data_buf_dump(mem, ibdev);
 424                         iser_err("mem->dma_nents = %d (dlength = 0x%x)\n",
 425                                  mem->dma_nents,
 426                                  ntoh24(iser_task->desc.iscsi_header.dlength));
 427                         iser_err("page_vec: data_size = 0x%x, length = %d, offset = 0x%x\n",
 428                                  ib_conn->fmr.page_vec->data_size,
 429                                  ib_conn->fmr.page_vec->length,
 430                                  ib_conn->fmr.page_vec->offset);
 431                         for (i = 0; i < ib_conn->fmr.page_vec->length; i++)
 432                                 iser_err("page_vec[%d] = 0x%llx\n", i,
 433                                          (unsigned long long)ib_conn->fmr.page_vec->pages[i]);
 434                 }
 435                 if (err)
 436                         return err;
 437         }
 438         return 0;
 439 }
 440
 441 static void
 442 iser_set_dif_domain(struct scsi_cmnd *sc, struct ib_sig_attrs *sig_attrs,
 443                     struct ib_sig_domain *domain)
 444 {
 445         domain->sig_type = IB_SIG_TYPE_T10_DIF;
 446         domain->sig.dif.pi_interval = scsi_prot_interval(sc);
 447         domain->sig.dif.ref_tag = scsi_prot_ref_tag(sc);
 448         /*
 449          * At the moment we hard code those, but in the future
 450          * we will take them from sc.
 451          */
 452         domain->sig.dif.apptag_check_mask = 0xffff;
 453         domain->sig.dif.app_escape = true;
 454         domain->sig.dif.ref_escape = true;
 455         if (sc->prot_flags & SCSI_PROT_REF_INCREMENT)
 456                 domain->sig.dif.ref_remap = true;
 457 };
 458
 459 static int
 460 iser_set_sig_attrs(struct scsi_cmnd *sc, struct ib_sig_attrs *sig_attrs)
 461 {
 462         switch (scsi_get_prot_op(sc)) {
 463         case SCSI_PROT_WRITE_INSERT:
 464         case SCSI_PROT_READ_STRIP:
 465                 sig_attrs->mem.sig_type = IB_SIG_TYPE_NONE;
 466                 iser_set_dif_domain(sc, sig_attrs, &sig_attrs->wire);
 467                 sig_attrs->wire.sig.dif.bg_type = IB_T10DIF_CRC;
 468                 break;
 469         case SCSI_PROT_READ_INSERT:
 470         case SCSI_PROT_WRITE_STRIP:
 471                 sig_attrs->wire.sig_type = IB_SIG_TYPE_NONE;
 472                 iser_set_dif_domain(sc, sig_attrs, &sig_attrs->mem);
 473                 sig_attrs->mem.sig.dif.bg_type = sc->prot_flags & SCSI_PROT_IP_CHECKSUM ?
 474                                                 IB_T10DIF_CSUM : IB_T10DIF_CRC;
 475                 break;
 476         case SCSI_PROT_READ_PASS:
 477         case SCSI_PROT_WRITE_PASS:
 478                 iser_set_dif_domain(sc, sig_attrs, &sig_attrs->wire);
 479                 sig_attrs->wire.sig.dif.bg_type = IB_T10DIF_CRC;
 480                 iser_set_dif_domain(sc, sig_attrs, &sig_attrs->mem);
 481                 sig_attrs->mem.sig.dif.bg_type = sc->prot_flags & SCSI_PROT_IP_CHECKSUM ?
 482                                                 IB_T10DIF_CSUM : IB_T10DIF_CRC;
 483                 break;
 484         default:
 485                 iser_err("Unsupported PI operation %d\n",
 486                          scsi_get_prot_op(sc));
 487                 return -EINVAL;
 488         }
 489
 490         return 0;
 491 }
 492
 493 static inline void
 494 iser_set_prot_checks(struct scsi_cmnd *sc, u8 *mask)
 495 {
 496         *mask = 0;
 497         if (sc->prot_flags & SCSI_PROT_REF_CHECK)
 498                 *mask |= ISER_CHECK_REFTAG;
 499         if (sc->prot_flags & SCSI_PROT_GUARD_CHECK)
 500                 *mask |= ISER_CHECK_GUARD;
 501 }
 502
 503 static void
 504 iser_inv_rkey(struct ib_send_wr *inv_wr, struct ib_mr *mr)
 505 {
 506         u32 rkey;
 507
 508         memset(inv_wr, 0, sizeof(*inv_wr));
 509         inv_wr->opcode = IB_WR_LOCAL_INV;
 510         inv_wr->wr_id = ISER_FASTREG_LI_WRID;
 511         inv_wr->ex.invalidate_rkey = mr->rkey;
 512
 513         rkey = ib_inc_rkey(mr->rkey);
 514         ib_update_fast_reg_key(mr, rkey);
 515 }
 516
 517 static int
 518 iser_reg_sig_mr(struct iscsi_iser_task *iser_task,
 519                 struct fast_reg_descriptor *desc, struct ib_sge *data_sge,
 520                 struct ib_sge *prot_sge, struct ib_sge *sig_sge)
 521 {
 522         struct ib_conn *ib_conn = &iser_task->iser_conn->ib_conn;
 523         struct iser_pi_context *pi_ctx = desc->pi_ctx;
 524         struct ib_send_wr sig_wr, inv_wr;
 525         struct ib_send_wr *bad_wr, *wr = NULL;
 526         struct ib_sig_attrs sig_attrs;
 527         int ret;
 528
 529         memset(&sig_attrs, 0, sizeof(sig_attrs));
 530         ret = iser_set_sig_attrs(iser_task->sc, &sig_attrs);
 531         if (ret)
 532                 goto err;
 533
 534         iser_set_prot_checks(iser_task->sc, &sig_attrs.check_mask);
 535
 536         if (!(desc->reg_indicators & ISER_SIG_KEY_VALID)) {
 537                 iser_inv_rkey(&inv_wr, pi_ctx->sig_mr);
 538                 wr = &inv_wr;
 539         }
 540
 541         memset(&sig_wr, 0, sizeof(sig_wr));
 542         sig_wr.opcode = IB_WR_REG_SIG_MR;
 543         sig_wr.wr_id = ISER_FASTREG_LI_WRID;
 544         sig_wr.sg_list = data_sge;
 545         sig_wr.num_sge = 1;
 546         sig_wr.wr.sig_handover.sig_attrs = &sig_attrs;
 547         sig_wr.wr.sig_handover.sig_mr = pi_ctx->sig_mr;
 548         if (scsi_prot_sg_count(iser_task->sc))
 549                 sig_wr.wr.sig_handover.prot = prot_sge;
 550         sig_wr.wr.sig_handover.access_flags = IB_ACCESS_LOCAL_WRITE |
 551                                               IB_ACCESS_REMOTE_READ |
 552                                               IB_ACCESS_REMOTE_WRITE;
 553
 554         if (!wr)
 555                 wr = &sig_wr;
 556         else
 557                 wr->next = &sig_wr;
 558
 559         ret = ib_post_send(ib_conn->qp, wr, &bad_wr);
 560         if (ret) {
 561                 iser_err("reg_sig_mr failed, ret:%d\n", ret);
 562                 goto err;
 563         }
 564         desc->reg_indicators &= ~ISER_SIG_KEY_VALID;
 565
 566         sig_sge->lkey = pi_ctx->sig_mr->lkey;
 567         sig_sge->addr = 0;
 568         sig_sge->length = scsi_transfer_length(iser_task->sc);
 569
 570         iser_dbg("sig_sge: addr: 0x%llx  length: %u lkey: 0x%x\n",
 571                  sig_sge->addr, sig_sge->length,
 572                  sig_sge->lkey);
 573 err:
 574         return ret;
 575 }
 576
 577 static int iser_fast_reg_mr(struct iscsi_iser_task *iser_task,
 578                             struct iser_regd_buf *regd_buf,
 579                             struct iser_data_buf *mem,
 580                             enum iser_reg_indicator ind,
 581                             struct ib_sge *sge)
 582 {
 583         struct fast_reg_descriptor *desc = regd_buf->reg.mem_h;
 584         struct ib_conn *ib_conn = &iser_task->iser_conn->ib_conn;
 585         struct iser_device *device = ib_conn->device;
 586         struct ib_device *ibdev = device->ib_device;
 587         struct ib_mr *mr;
 588         struct ib_fast_reg_page_list *frpl;
 589         struct ib_send_wr fastreg_wr, inv_wr;
 590         struct ib_send_wr *bad_wr, *wr = NULL;
 591         int ret, offset, size, plen;
 592
 593         /* if there a single dma entry, dma mr suffices */
 594         if (mem->dma_nents == 1) {
 595                 struct scatterlist *sg = (struct scatterlist *)mem->buf;
 596
 597                 sge->lkey = device->mr->lkey;
 598                 sge->addr   = ib_sg_dma_address(ibdev, &sg[0]);
 599                 sge->length  = ib_sg_dma_len(ibdev, &sg[0]);
 600
 601                 iser_dbg("Single DMA entry: lkey=0x%x, addr=0x%llx, length=0x%x\n",
 602                          sge->lkey, sge->addr, sge->length);
 603                 return 0;
 604         }
 605
 606         if (ind == ISER_DATA_KEY_VALID) {
 607                 mr = desc->data_mr;
 608                 frpl = desc->data_frpl;
 609         } else {
 610                 mr = desc->pi_ctx->prot_mr;
 611                 frpl = desc->pi_ctx->prot_frpl;
 612         }
 613
 614         plen = iser_sg_to_page_vec(mem, device->ib_device, frpl->page_list,
 615                                    &offset, &size);
 616         if (plen * SIZE_4K < size) {
 617                 iser_err("fast reg page_list too short to hold this SG\n");
 618                 return -EINVAL;
 619         }
 620
 621         if (!(desc->reg_indicators & ind)) {
 622                 iser_inv_rkey(&inv_wr, mr);
 623                 wr = &inv_wr;
 624         }
 625
 626         /* Prepare FASTREG WR */
 627         memset(&fastreg_wr, 0, sizeof(fastreg_wr));
 628         fastreg_wr.wr_id = ISER_FASTREG_LI_WRID;
 629         fastreg_wr.opcode = IB_WR_FAST_REG_MR;
 630         fastreg_wr.wr.fast_reg.iova_start = frpl->page_list[0] + offset;
 631         fastreg_wr.wr.fast_reg.page_list = frpl;
 632         fastreg_wr.wr.fast_reg.page_list_len = plen;
 633         fastreg_wr.wr.fast_reg.page_shift = SHIFT_4K;
 634         fastreg_wr.wr.fast_reg.length = size;
 635         fastreg_wr.wr.fast_reg.rkey = mr->rkey;
 636         fastreg_wr.wr.fast_reg.access_flags = (IB_ACCESS_LOCAL_WRITE  |
 637                                                IB_ACCESS_REMOTE_WRITE |
 638                                                IB_ACCESS_REMOTE_READ);
 639
 640         if (!wr)
 641                 wr = &fastreg_wr;
 642         else
 643                 wr->next = &fastreg_wr;
 644
 645         ret = ib_post_send(ib_conn->qp, wr, &bad_wr);
 646         if (ret) {
 647                 iser_err("fast registration failed, ret:%d\n", ret);
 648                 return ret;
 649         }
 650         desc->reg_indicators &= ~ind;
 651
 652         sge->lkey = mr->lkey;
 653         sge->addr = frpl->page_list[0] + offset;
 654         sge->length = size;
 655
 656         return ret;
 657 }
 658
 659 /**
 660  * iser_reg_rdma_mem_fastreg - Registers memory intended for RDMA,
 661  * using Fast Registration WR (if possible) obtaining rkey and va
 662  *
 663  * returns 0 on success, errno code on failure
 664  */
 665 int iser_reg_rdma_mem_fastreg(struct iscsi_iser_task *iser_task,
 666                               enum iser_data_dir cmd_dir)
 667 {
 668         struct ib_conn *ib_conn = &iser_task->iser_conn->ib_conn;
 669         struct iser_device *device = ib_conn->device;
 670         struct ib_device *ibdev = device->ib_device;
 671         struct iser_data_buf *mem = &iser_task->data[cmd_dir];
 672         struct iser_regd_buf *regd_buf = &iser_task->rdma_regd[cmd_dir];
 673         struct fast_reg_descriptor *desc = NULL;
 674         struct ib_sge data_sge;
 675         int err, aligned_len;
 676         unsigned long flags;
 677
 678         aligned_len = iser_data_buf_aligned_len(mem, ibdev);
 679         if (aligned_len != mem->dma_nents) {
 680                 err = fall_to_bounce_buf(iser_task, ibdev, mem,
 681                                          &iser_task->data_copy[cmd_dir],
 682                                          cmd_dir, aligned_len);
 683                 if (err) {
 684                         iser_err("failed to allocate bounce buffer\n");
 685                         return err;
 686                 }
 687                 mem = &iser_task->data_copy[cmd_dir];
 688         }
 689
 690         if (mem->dma_nents != 1 ||
 691             scsi_get_prot_op(iser_task->sc) != SCSI_PROT_NORMAL) {
 692                 spin_lock_irqsave(&ib_conn->lock, flags);
 693                 desc = list_first_entry(&ib_conn->fastreg.pool,
 694                                         struct fast_reg_descriptor, list);
 695                 list_del(&desc->list);
 696                 spin_unlock_irqrestore(&ib_conn->lock, flags);
 697                 regd_buf->reg.mem_h = desc;
 698         }
 699
 700         err = iser_fast_reg_mr(iser_task, regd_buf, mem,
 701                                ISER_DATA_KEY_VALID, &data_sge);
 702         if (err)
 703                 goto err_reg;
 704
 705         if (scsi_get_prot_op(iser_task->sc) != SCSI_PROT_NORMAL) {
 706                 struct ib_sge prot_sge, sig_sge;
 707
 708                 memset(&prot_sge, 0, sizeof(prot_sge));
 709                 if (scsi_prot_sg_count(iser_task->sc)) {
 710                         mem = &iser_task->prot[cmd_dir];
 711                         aligned_len = iser_data_buf_aligned_len(mem, ibdev);
 712                         if (aligned_len != mem->dma_nents) {
 713                                 err = fall_to_bounce_buf(iser_task, ibdev, mem,
 714                                                          &iser_task->prot_copy[cmd_dir],
 715                                                          cmd_dir, aligned_len);
 716                                 if (err) {
 717                                         iser_err("failed to allocate bounce buffer\n");
 718                                         return err;
 719                                 }
 720                                 mem = &iser_task->prot_copy[cmd_dir];
 721                         }
 722
 723                         err = iser_fast_reg_mr(iser_task, regd_buf, mem,
 724                                                ISER_PROT_KEY_VALID, &prot_sge);
 725                         if (err)
 726                                 goto err_reg;
 727                 }
 728
 729                 err = iser_reg_sig_mr(iser_task, desc, &data_sge,
 730                                       &prot_sge, &sig_sge);
 731                 if (err) {
 732                         iser_err("Failed to register signature mr\n");
 733                         return err;
 734                 }
 735                 desc->reg_indicators |= ISER_FASTREG_PROTECTED;
 736
 737                 regd_buf->reg.lkey = sig_sge.lkey;
 738                 regd_buf->reg.rkey = desc->pi_ctx->sig_mr->rkey;
 739                 regd_buf->reg.va = sig_sge.addr;
 740                 regd_buf->reg.len = sig_sge.length;
 741         } else {
 742                 if (desc)
 743                         regd_buf->reg.rkey = desc->data_mr->rkey;
 744                 else
 745                         regd_buf->reg.rkey = device->mr->rkey;
 746
 747                 regd_buf->reg.lkey = data_sge.lkey;
 748                 regd_buf->reg.va = data_sge.addr;
 749                 regd_buf->reg.len = data_sge.length;
 750         }
 751
 752         return 0;
 753 err_reg:
 754         if (desc) {
 755                 spin_lock_irqsave(&ib_conn->lock, flags);
 756                 list_add_tail(&desc->list, &ib_conn->fastreg.pool);
 757                 spin_unlock_irqrestore(&ib_conn->lock, flags);
 758         }
 759
 760         return err;
 761 }