fs/nfs/pagelist.c

   1 /*
   2  * linux/fs/nfs/pagelist.c
   3  *
   4  * A set of helper functions for managing NFS read and write requests.
   5  * The main purpose of these routines is to provide support for the
   6  * coalescing of several requests into a single RPC call.
   7  *
   8  * Copyright 2000, 2001 (c) Trond Myklebust <trond.myklebust@fys.uio.no>
   9  *
  10  */
  11
  12 #include <linux/slab.h>
  13 #include <linux/file.h>
  14 #include <linux/sched.h>
  15 #include <linux/sunrpc/clnt.h>
  16 #include <linux/nfs.h>
  17 #include <linux/nfs3.h>
  18 #include <linux/nfs4.h>
  19 #include <linux/nfs_page.h>
  20 #include <linux/nfs_fs.h>
  21 #include <linux/nfs_mount.h>
  22 #include <linux/export.h>
  23
  24 #include "internal.h"
  25 #include "pnfs.h"
  26
  27 #define NFSDBG_FACILITY         NFSDBG_PAGECACHE
  28
  29 static struct kmem_cache *nfs_page_cachep;
  30 static const struct rpc_call_ops nfs_pgio_common_ops;
  31
  32 static bool nfs_pgarray_set(struct nfs_page_array *p, unsigned int pagecount)
  33 {
  34         p->npages = pagecount;
  35         if (pagecount <= ARRAY_SIZE(p->page_array))
  36                 p->pagevec = p->page_array;
  37         else {
  38                 p->pagevec = kcalloc(pagecount, sizeof(struct page *), GFP_KERNEL);
  39                 if (!p->pagevec)
  40                         p->npages = 0;
  41         }
  42         return p->pagevec != NULL;
  43 }
  44
  45 void nfs_pgheader_init(struct nfs_pageio_descriptor *desc,
  46                        struct nfs_pgio_header *hdr,
  47                        void (*release)(struct nfs_pgio_header *hdr))
  48 {
  49         struct nfs_pgio_mirror *mirror = &desc->pg_mirrors[desc->pg_mirror_idx];
  50
  51
  52         hdr->req = nfs_list_entry(mirror->pg_list.next);
  53         hdr->inode = desc->pg_inode;
  54         hdr->cred = hdr->req->wb_context->cred;
  55         hdr->io_start = req_offset(hdr->req);
  56         hdr->good_bytes = mirror->pg_count;
  57         hdr->dreq = desc->pg_dreq;
  58         hdr->layout_private = desc->pg_layout_private;
  59         hdr->release = release;
  60         hdr->completion_ops = desc->pg_completion_ops;
  61         if (hdr->completion_ops->init_hdr)
  62                 hdr->completion_ops->init_hdr(hdr);
  63
  64         hdr->pgio_mirror_idx = desc->pg_mirror_idx;
  65 }
  66 EXPORT_SYMBOL_GPL(nfs_pgheader_init);
  67
  68 void nfs_set_pgio_error(struct nfs_pgio_header *hdr, int error, loff_t pos)
  69 {
  70         spin_lock(&hdr->lock);
  71         if (pos < hdr->io_start + hdr->good_bytes) {
  72                 set_bit(NFS_IOHDR_ERROR, &hdr->flags);
  73                 clear_bit(NFS_IOHDR_EOF, &hdr->flags);
  74                 hdr->good_bytes = pos - hdr->io_start;
  75                 hdr->error = error;
  76         }
  77         spin_unlock(&hdr->lock);
  78 }
  79
  80 static inline struct nfs_page *
  81 nfs_page_alloc(void)
  82 {
  83         struct nfs_page *p = kmem_cache_zalloc(nfs_page_cachep, GFP_NOIO);
  84         if (p)
  85                 INIT_LIST_HEAD(&p->wb_list);
  86         return p;
  87 }
  88
  89 static inline void
  90 nfs_page_free(struct nfs_page *p)
  91 {
  92         kmem_cache_free(nfs_page_cachep, p);
  93 }
  94
  95 static void
  96 nfs_iocounter_inc(struct nfs_io_counter *c)
  97 {
  98         atomic_inc(&c->io_count);
  99 }
 100
 101 static void
 102 nfs_iocounter_dec(struct nfs_io_counter *c)
 103 {
 104         if (atomic_dec_and_test(&c->io_count)) {
 105                 clear_bit(NFS_IO_INPROGRESS, &c->flags);
 106                 smp_mb__after_atomic();
 107                 wake_up_bit(&c->flags, NFS_IO_INPROGRESS);
 108         }
 109 }
 110
 111 static int
 112 __nfs_iocounter_wait(struct nfs_io_counter *c)
 113 {
 114         wait_queue_head_t *wq = bit_waitqueue(&c->flags, NFS_IO_INPROGRESS);
 115         DEFINE_WAIT_BIT(q, &c->flags, NFS_IO_INPROGRESS);
 116         int ret = 0;
 117
 118         do {
 119                 prepare_to_wait(wq, &q.wait, TASK_KILLABLE);
 120                 set_bit(NFS_IO_INPROGRESS, &c->flags);
 121                 if (atomic_read(&c->io_count) == 0)
 122                         break;
 123                 ret = nfs_wait_bit_killable(&q.key);
 124         } while (atomic_read(&c->io_count) != 0 && !ret);
 125         finish_wait(wq, &q.wait);
 126         return ret;
 127 }
 128
 129 /**
 130  * nfs_iocounter_wait - wait for i/o to complete
 131  * @c: nfs_io_counter to use
 132  *
 133  * returns -ERESTARTSYS if interrupted by a fatal signal.
 134  * Otherwise returns 0 once the io_count hits 0.
 135  */
 136 int
 137 nfs_iocounter_wait(struct nfs_io_counter *c)
 138 {
 139         if (atomic_read(&c->io_count) == 0)
 140                 return 0;
 141         return __nfs_iocounter_wait(c);
 142 }
 143
 144 /*
 145  * nfs_page_group_lock - lock the head of the page group
 146  * @req - request in group that is to be locked
 147  * @nonblock - if true don't block waiting for lock
 148  *
 149  * this lock must be held if modifying the page group list
 150  *
 151  * return 0 on success, < 0 on error: -EDELAY if nonblocking or the
 152  * result from wait_on_bit_lock
 153  *
 154  * NOTE: calling with nonblock=false should always have set the
 155  *       lock bit (see fs/buffer.c and other uses of wait_on_bit_lock
 156  *       with TASK_UNINTERRUPTIBLE), so there is no need to check the result.
 157  */
 158 int
 159 nfs_page_group_lock(struct nfs_page *req, bool nonblock)
 160 {
 161         struct nfs_page *head = req->wb_head;
 162
 163         WARN_ON_ONCE(head != head->wb_head);
 164
 165         if (!test_and_set_bit(PG_HEADLOCK, &head->wb_flags))
 166                 return 0;
 167
 168         if (!nonblock)
 169                 return wait_on_bit_lock(&head->wb_flags, PG_HEADLOCK,
 170                                 TASK_UNINTERRUPTIBLE);
 171
 172         return -EAGAIN;
 173 }
 174
 175 /*
 176  * nfs_page_group_lock_wait - wait for the lock to clear, but don't grab it
 177  * @req - a request in the group
 178  *
 179  * This is a blocking call to wait for the group lock to be cleared.
 180  */
 181 void
 182 nfs_page_group_lock_wait(struct nfs_page *req)
 183 {
 184         struct nfs_page *head = req->wb_head;
 185
 186         WARN_ON_ONCE(head != head->wb_head);
 187
 188         wait_on_bit(&head->wb_flags, PG_HEADLOCK,
 189                 TASK_UNINTERRUPTIBLE);
 190 }
 191
 192 /*
 193  * nfs_page_group_unlock - unlock the head of the page group
 194  * @req - request in group that is to be unlocked
 195  */
 196 void
 197 nfs_page_group_unlock(struct nfs_page *req)
 198 {
 199         struct nfs_page *head = req->wb_head;
 200
 201         WARN_ON_ONCE(head != head->wb_head);
 202
 203         smp_mb__before_atomic();
 204         clear_bit(PG_HEADLOCK, &head->wb_flags);
 205         smp_mb__after_atomic();
 206         wake_up_bit(&head->wb_flags, PG_HEADLOCK);
 207 }
 208
 209 /*
 210  * nfs_page_group_sync_on_bit_locked
 211  *
 212  * must be called with page group lock held
 213  */
 214 static bool
 215 nfs_page_group_sync_on_bit_locked(struct nfs_page *req, unsigned int bit)
 216 {
 217         struct nfs_page *head = req->wb_head;
 218         struct nfs_page *tmp;
 219
 220         WARN_ON_ONCE(!test_bit(PG_HEADLOCK, &head->wb_flags));
 221         WARN_ON_ONCE(test_and_set_bit(bit, &req->wb_flags));
 222
 223         tmp = req->wb_this_page;
 224         while (tmp != req) {
 225                 if (!test_bit(bit, &tmp->wb_flags))
 226                         return false;
 227                 tmp = tmp->wb_this_page;
 228         }
 229
 230         /* true! reset all bits */
 231         tmp = req;
 232         do {
 233                 clear_bit(bit, &tmp->wb_flags);
 234                 tmp = tmp->wb_this_page;
 235         } while (tmp != req);
 236
 237         return true;
 238 }
 239
 240 /*
 241  * nfs_page_group_sync_on_bit - set bit on current request, but only
 242  *   return true if the bit is set for all requests in page group
 243  * @req - request in page group
 244  * @bit - PG_* bit that is used to sync page group
 245  */
 246 bool nfs_page_group_sync_on_bit(struct nfs_page *req, unsigned int bit)
 247 {
 248         bool ret;
 249
 250         nfs_page_group_lock(req, false);
 251         ret = nfs_page_group_sync_on_bit_locked(req, bit);
 252         nfs_page_group_unlock(req);
 253
 254         return ret;
 255 }
 256
 257 /*
 258  * nfs_page_group_init - Initialize the page group linkage for @req
 259  * @req - a new nfs request
 260  * @prev - the previous request in page group, or NULL if @req is the first
 261  *         or only request in the group (the head).
 262  */
 263 static inline void
 264 nfs_page_group_init(struct nfs_page *req, struct nfs_page *prev)
 265 {
 266         struct inode *inode;
 267         WARN_ON_ONCE(prev == req);
 268
 269         if (!prev) {
 270                 /* a head request */
 271                 req->wb_head = req;
 272                 req->wb_this_page = req;
 273         } else {
 274                 /* a subrequest */
 275                 WARN_ON_ONCE(prev->wb_this_page != prev->wb_head);
 276                 WARN_ON_ONCE(!test_bit(PG_HEADLOCK, &prev->wb_head->wb_flags));
 277                 req->wb_head = prev->wb_head;
 278                 req->wb_this_page = prev->wb_this_page;
 279                 prev->wb_this_page = req;
 280
 281                 /* All subrequests take a ref on the head request until
 282                  * nfs_page_group_destroy is called */
 283                 kref_get(&req->wb_head->wb_kref);
 284
 285                 /* grab extra ref and bump the request count if head request
 286                  * has extra ref from the write/commit path to handle handoff
 287                  * between write and commit lists. */
 288                 if (test_bit(PG_INODE_REF, &prev->wb_head->wb_flags)) {
 289                         inode = page_file_mapping(req->wb_page)->host;
 290                         set_bit(PG_INODE_REF, &req->wb_flags);
 291                         kref_get(&req->wb_kref);
 292                         spin_lock(&inode->i_lock);
 293                         NFS_I(inode)->nrequests++;
 294                         spin_unlock(&inode->i_lock);
 295                 }
 296         }
 297 }
 298
 299 /*
 300  * nfs_page_group_destroy - sync the destruction of page groups
 301  * @req - request that no longer needs the page group
 302  *
 303  * releases the page group reference from each member once all
 304  * members have called this function.
 305  */
 306 static void
 307 nfs_page_group_destroy(struct kref *kref)
 308 {
 309         struct nfs_page *req = container_of(kref, struct nfs_page, wb_kref);
 310         struct nfs_page *tmp, *next;
 311
 312         /* subrequests must release the ref on the head request */
 313         if (req->wb_head != req)
 314                 nfs_release_request(req->wb_head);
 315
 316         if (!nfs_page_group_sync_on_bit(req, PG_TEARDOWN))
 317                 return;
 318
 319         tmp = req;
 320         do {
 321                 next = tmp->wb_this_page;
 322                 /* unlink and free */
 323                 tmp->wb_this_page = tmp;
 324                 tmp->wb_head = tmp;
 325                 nfs_free_request(tmp);
 326                 tmp = next;
 327         } while (tmp != req);
 328 }
 329
 330 /**
 331  * nfs_create_request - Create an NFS read/write request.
 332  * @ctx: open context to use
 333  * @page: page to write
 334  * @last: last nfs request created for this page group or NULL if head
 335  * @offset: starting offset within the page for the write
 336  * @count: number of bytes to read/write
 337  *
 338  * The page must be locked by the caller. This makes sure we never
 339  * create two different requests for the same page.
 340  * User should ensure it is safe to sleep in this function.
 341  */
 342 struct nfs_page *
 343 nfs_create_request(struct nfs_open_context *ctx, struct page *page,
 344                    struct nfs_page *last, unsigned int offset,
 345                    unsigned int count)
 346 {
 347         struct nfs_page         *req;
 348         struct nfs_lock_context *l_ctx;
 349
 350         if (test_bit(NFS_CONTEXT_BAD, &ctx->flags))
 351                 return ERR_PTR(-EBADF);
 352         /* try to allocate the request struct */
 353         req = nfs_page_alloc();
 354         if (req == NULL)
 355                 return ERR_PTR(-ENOMEM);
 356
 357         /* get lock context early so we can deal with alloc failures */
 358         l_ctx = nfs_get_lock_context(ctx);
 359         if (IS_ERR(l_ctx)) {
 360                 nfs_page_free(req);
 361                 return ERR_CAST(l_ctx);
 362         }
 363         req->wb_lock_context = l_ctx;
 364         nfs_iocounter_inc(&l_ctx->io_count);
 365
 366         /* Initialize the request struct. Initially, we assume a
 367          * long write-back delay. This will be adjusted in
 368          * update_nfs_request below if the region is not locked. */
 369         req->wb_page    = page;
 370         req->wb_index   = page_file_index(page);
 371         page_cache_get(page);
 372         req->wb_offset  = offset;
 373         req->wb_pgbase  = offset;
 374         req->wb_bytes   = count;
 375         req->wb_context = get_nfs_open_context(ctx);
 376         kref_init(&req->wb_kref);
 377         nfs_page_group_init(req, last);
 378         return req;
 379 }
 380
 381 /**
 382  * nfs_unlock_request - Unlock request and wake up sleepers.
 383  * @req:
 384  */
 385 void nfs_unlock_request(struct nfs_page *req)
 386 {
 387         if (!NFS_WBACK_BUSY(req)) {
 388                 printk(KERN_ERR "NFS: Invalid unlock attempted\n");
 389                 BUG();
 390         }
 391         smp_mb__before_atomic();
 392         clear_bit(PG_BUSY, &req->wb_flags);
 393         smp_mb__after_atomic();
 394         wake_up_bit(&req->wb_flags, PG_BUSY);
 395 }
 396
 397 /**
 398  * nfs_unlock_and_release_request - Unlock request and release the nfs_page
 399  * @req:
 400  */
 401 void nfs_unlock_and_release_request(struct nfs_page *req)
 402 {
 403         nfs_unlock_request(req);
 404         nfs_release_request(req);
 405 }
 406
 407 /*
 408  * nfs_clear_request - Free up all resources allocated to the request
 409  * @req:
 410  *
 411  * Release page and open context resources associated with a read/write
 412  * request after it has completed.
 413  */
 414 static void nfs_clear_request(struct nfs_page *req)
 415 {
 416         struct page *page = req->wb_page;
 417         struct nfs_open_context *ctx = req->wb_context;
 418         struct nfs_lock_context *l_ctx = req->wb_lock_context;
 419
 420         if (page != NULL) {
 421                 page_cache_release(page);
 422                 req->wb_page = NULL;
 423         }
 424         if (l_ctx != NULL) {
 425                 nfs_iocounter_dec(&l_ctx->io_count);
 426                 nfs_put_lock_context(l_ctx);
 427                 req->wb_lock_context = NULL;
 428         }
 429         if (ctx != NULL) {
 430                 put_nfs_open_context(ctx);
 431                 req->wb_context = NULL;
 432         }
 433 }
 434
 435 /**
 436  * nfs_release_request - Release the count on an NFS read/write request
 437  * @req: request to release
 438  *
 439  * Note: Should never be called with the spinlock held!
 440  */
 441 void nfs_free_request(struct nfs_page *req)
 442 {
 443         WARN_ON_ONCE(req->wb_this_page != req);
 444
 445         /* extra debug: make sure no sync bits are still set */
 446         WARN_ON_ONCE(test_bit(PG_TEARDOWN, &req->wb_flags));
 447         WARN_ON_ONCE(test_bit(PG_UNLOCKPAGE, &req->wb_flags));
 448         WARN_ON_ONCE(test_bit(PG_UPTODATE, &req->wb_flags));
 449         WARN_ON_ONCE(test_bit(PG_WB_END, &req->wb_flags));
 450         WARN_ON_ONCE(test_bit(PG_REMOVE, &req->wb_flags));
 451
 452         /* Release struct file and open context */
 453         nfs_clear_request(req);
 454         nfs_page_free(req);
 455 }
 456
 457 void nfs_release_request(struct nfs_page *req)
 458 {
 459         kref_put(&req->wb_kref, nfs_page_group_destroy);
 460 }
 461
 462 /**
 463  * nfs_wait_on_request - Wait for a request to complete.
 464  * @req: request to wait upon.
 465  *
 466  * Interruptible by fatal signals only.
 467  * The user is responsible for holding a count on the request.
 468  */
 469 int
 470 nfs_wait_on_request(struct nfs_page *req)
 471 {
 472         return wait_on_bit_io(&req->wb_flags, PG_BUSY,
 473                               TASK_UNINTERRUPTIBLE);
 474 }
 475
 476 /*
 477  * nfs_generic_pg_test - determine if requests can be coalesced
 478  * @desc: pointer to descriptor
 479  * @prev: previous request in desc, or NULL
 480  * @req: this request
 481  *
 482  * Returns zero if @req can be coalesced into @desc, otherwise it returns
 483  * the size of the request.
 484  */
 485 size_t nfs_generic_pg_test(struct nfs_pageio_descriptor *desc,
 486                            struct nfs_page *prev, struct nfs_page *req)
 487 {
 488         struct nfs_pgio_mirror *mirror = &desc->pg_mirrors[desc->pg_mirror_idx];
 489
 490
 491         if (mirror->pg_count > mirror->pg_bsize) {
 492                 /* should never happen */
 493                 WARN_ON_ONCE(1);
 494                 return 0;
 495         }
 496
 497         /*
 498          * Limit the request size so that we can still allocate a page array
 499          * for it without upsetting the slab allocator.
 500          */
 501         if (((mirror->pg_count + req->wb_bytes) >> PAGE_SHIFT) *
 502                         sizeof(struct page) > PAGE_SIZE)
 503                 return 0;
 504
 505         return min(mirror->pg_bsize - mirror->pg_count, (size_t)req->wb_bytes);
 506 }
 507 EXPORT_SYMBOL_GPL(nfs_generic_pg_test);
 508
 509 struct nfs_pgio_header *nfs_pgio_header_alloc(const struct nfs_rw_ops *ops)
 510 {
 511         struct nfs_pgio_header *hdr = ops->rw_alloc_header();
 512
 513         if (hdr) {
 514                 INIT_LIST_HEAD(&hdr->pages);
 515                 spin_lock_init(&hdr->lock);
 516                 hdr->rw_ops = ops;
 517         }
 518         return hdr;
 519 }
 520 EXPORT_SYMBOL_GPL(nfs_pgio_header_alloc);
 521
 522 /*
 523  * nfs_pgio_header_free - Free a read or write header
 524  * @hdr: The header to free
 525  */
 526 void nfs_pgio_header_free(struct nfs_pgio_header *hdr)
 527 {
 528         hdr->rw_ops->rw_free_header(hdr);
 529 }
 530 EXPORT_SYMBOL_GPL(nfs_pgio_header_free);
 531
 532 /**
 533  * nfs_pgio_data_destroy - make @hdr suitable for reuse
 534  *
 535  * Frees memory and releases refs from nfs_generic_pgio, so that it may
 536  * be called again.
 537  *
 538  * @hdr: A header that has had nfs_generic_pgio called
 539  */
 540 void nfs_pgio_data_destroy(struct nfs_pgio_header *hdr)
 541 {
 542         if (hdr->args.context)
 543                 put_nfs_open_context(hdr->args.context);
 544         if (hdr->page_array.pagevec != hdr->page_array.page_array)
 545                 kfree(hdr->page_array.pagevec);
 546 }
 547 EXPORT_SYMBOL_GPL(nfs_pgio_data_destroy);
 548
 549 /**
 550  * nfs_pgio_rpcsetup - Set up arguments for a pageio call
 551  * @hdr: The pageio hdr
 552  * @count: Number of bytes to read
 553  * @offset: Initial offset
 554  * @how: How to commit data (writes only)
 555  * @cinfo: Commit information for the call (writes only)
 556  */
 557 static void nfs_pgio_rpcsetup(struct nfs_pgio_header *hdr,
 558                               unsigned int count, unsigned int offset,
 559                               int how, struct nfs_commit_info *cinfo)
 560 {
 561         struct nfs_page *req = hdr->req;
 562
 563         /* Set up the RPC argument and reply structs
 564          * NB: take care not to mess about with hdr->commit et al. */
 565
 566         hdr->args.fh     = NFS_FH(hdr->inode);
 567         hdr->args.offset = req_offset(req) + offset;
 568         /* pnfs_set_layoutcommit needs this */
 569         hdr->mds_offset = hdr->args.offset;
 570         hdr->args.pgbase = req->wb_pgbase + offset;
 571         hdr->args.pages  = hdr->page_array.pagevec;
 572         hdr->args.count  = count;
 573         hdr->args.context = get_nfs_open_context(req->wb_context);
 574         hdr->args.lock_context = req->wb_lock_context;
 575         hdr->args.stable  = NFS_UNSTABLE;
 576         switch (how & (FLUSH_STABLE | FLUSH_COND_STABLE)) {
 577         case 0:
 578                 break;
 579         case FLUSH_COND_STABLE:
 580                 if (nfs_reqs_to_commit(cinfo))
 581                         break;
 582         default:
 583                 hdr->args.stable = NFS_FILE_SYNC;
 584         }
 585
 586         hdr->res.fattr   = &hdr->fattr;
 587         hdr->res.count   = count;
 588         hdr->res.eof     = 0;
 589         hdr->res.verf    = &hdr->verf;
 590         nfs_fattr_init(&hdr->fattr);
 591 }
 592
 593 /**
 594  * nfs_pgio_prepare - Prepare pageio hdr to go over the wire
 595  * @task: The current task
 596  * @calldata: pageio header to prepare
 597  */
 598 static void nfs_pgio_prepare(struct rpc_task *task, void *calldata)
 599 {
 600         struct nfs_pgio_header *hdr = calldata;
 601         int err;
 602         err = NFS_PROTO(hdr->inode)->pgio_rpc_prepare(task, hdr);
 603         if (err)
 604                 rpc_exit(task, err);
 605 }
 606
 607 int nfs_initiate_pgio(struct rpc_clnt *clnt, struct nfs_pgio_header *hdr,
 608                       struct rpc_cred *cred, const struct nfs_rpc_ops *rpc_ops,
 609                       const struct rpc_call_ops *call_ops, int how, int flags)
 610 {
 611         struct rpc_task *task;
 612         struct rpc_message msg = {
 613                 .rpc_argp = &hdr->args,
 614                 .rpc_resp = &hdr->res,
 615                 .rpc_cred = cred,
 616         };
 617         struct rpc_task_setup task_setup_data = {
 618                 .rpc_client = clnt,
 619                 .task = &hdr->task,
 620                 .rpc_message = &msg,
 621                 .callback_ops = call_ops,
 622                 .callback_data = hdr,
 623                 .workqueue = nfsiod_workqueue,
 624                 .flags = RPC_TASK_ASYNC | flags,
 625         };
 626         int ret = 0;
 627
 628         hdr->rw_ops->rw_initiate(hdr, &msg, rpc_ops, &task_setup_data, how);
 629
 630         dprintk("NFS: %5u initiated pgio call "
 631                 "(req %s/%llu, %u bytes @ offset %llu)\n",
 632                 hdr->task.tk_pid,
 633                 hdr->inode->i_sb->s_id,
 634                 (unsigned long long)NFS_FILEID(hdr->inode),
 635                 hdr->args.count,
 636                 (unsigned long long)hdr->args.offset);
 637
 638         task = rpc_run_task(&task_setup_data);
 639         if (IS_ERR(task)) {
 640                 ret = PTR_ERR(task);
 641                 goto out;
 642         }
 643         if (how & FLUSH_SYNC) {
 644                 ret = rpc_wait_for_completion_task(task);
 645                 if (ret == 0)
 646                         ret = task->tk_status;
 647         }
 648         rpc_put_task(task);
 649 out:
 650         return ret;
 651 }
 652 EXPORT_SYMBOL_GPL(nfs_initiate_pgio);
 653
 654 /**
 655  * nfs_pgio_error - Clean up from a pageio error
 656  * @desc: IO descriptor
 657  * @hdr: pageio header
 658  */
 659 static int nfs_pgio_error(struct nfs_pageio_descriptor *desc,
 660                           struct nfs_pgio_header *hdr)
 661 {
 662         struct nfs_pgio_mirror *mirror;
 663         u32 midx;
 664
 665         set_bit(NFS_IOHDR_REDO, &hdr->flags);
 666         nfs_pgio_data_destroy(hdr);
 667         hdr->completion_ops->completion(hdr);
 668         /* TODO: Make sure it's right to clean up all mirrors here
 669          *       and not just hdr->pgio_mirror_idx */
 670         for (midx = 0; midx < desc->pg_mirror_count; midx++) {
 671                 mirror = &desc->pg_mirrors[midx];
 672                 desc->pg_completion_ops->error_cleanup(&mirror->pg_list);
 673         }
 674         return -ENOMEM;
 675 }
 676
 677 /**
 678  * nfs_pgio_release - Release pageio data
 679  * @calldata: The pageio header to release
 680  */
 681 static void nfs_pgio_release(void *calldata)
 682 {
 683         struct nfs_pgio_header *hdr = calldata;
 684         if (hdr->rw_ops->rw_release)
 685                 hdr->rw_ops->rw_release(hdr);
 686         nfs_pgio_data_destroy(hdr);
 687         hdr->completion_ops->completion(hdr);
 688 }
 689
 690 static void nfs_pageio_mirror_init(struct nfs_pgio_mirror *mirror,
 691                                    unsigned int bsize)
 692 {
 693         INIT_LIST_HEAD(&mirror->pg_list);
 694         mirror->pg_bytes_written = 0;
 695         mirror->pg_count = 0;
 696         mirror->pg_bsize = bsize;
 697         mirror->pg_base = 0;
 698         mirror->pg_recoalesce = 0;
 699 }
 700
 701 /**
 702  * nfs_pageio_init - initialise a page io descriptor
 703  * @desc: pointer to descriptor
 704  * @inode: pointer to inode
 705  * @doio: pointer to io function
 706  * @bsize: io block size
 707  * @io_flags: extra parameters for the io function
 708  */
 709 void nfs_pageio_init(struct nfs_pageio_descriptor *desc,
 710                      struct inode *inode,
 711                      const struct nfs_pageio_ops *pg_ops,
 712                      const struct nfs_pgio_completion_ops *compl_ops,
 713                      const struct nfs_rw_ops *rw_ops,
 714                      size_t bsize,
 715                      int io_flags)
 716 {
 717         struct nfs_pgio_mirror *new;
 718         int i;
 719
 720         desc->pg_moreio = 0;
 721         desc->pg_inode = inode;
 722         desc->pg_ops = pg_ops;
 723         desc->pg_completion_ops = compl_ops;
 724         desc->pg_rw_ops = rw_ops;
 725         desc->pg_ioflags = io_flags;
 726         desc->pg_error = 0;
 727         desc->pg_lseg = NULL;
 728         desc->pg_dreq = NULL;
 729         desc->pg_layout_private = NULL;
 730         desc->pg_bsize = bsize;
 731
 732         desc->pg_mirror_count = 1;
 733         desc->pg_mirror_idx = 0;
 734
 735         if (pg_ops->pg_get_mirror_count) {
 736                 /* until we have a request, we don't have an lseg and no
 737                  * idea how many mirrors there will be */
 738                 new = kcalloc(NFS_PAGEIO_DESCRIPTOR_MIRROR_MAX,
 739                               sizeof(struct nfs_pgio_mirror), GFP_KERNEL);
 740                 desc->pg_mirrors_dynamic = new;
 741                 desc->pg_mirrors = new;
 742
 743                 for (i = 0; i < NFS_PAGEIO_DESCRIPTOR_MIRROR_MAX; i++)
 744                         nfs_pageio_mirror_init(&desc->pg_mirrors[i], bsize);
 745         } else {
 746                 desc->pg_mirrors_dynamic = NULL;
 747                 desc->pg_mirrors = desc->pg_mirrors_static;
 748                 nfs_pageio_mirror_init(&desc->pg_mirrors[0], bsize);
 749         }
 750 }
 751 EXPORT_SYMBOL_GPL(nfs_pageio_init);
 752
 753 /**
 754  * nfs_pgio_result - Basic pageio error handling
 755  * @task: The task that ran
 756  * @calldata: Pageio header to check
 757  */
 758 static void nfs_pgio_result(struct rpc_task *task, void *calldata)
 759 {
 760         struct nfs_pgio_header *hdr = calldata;
 761         struct inode *inode = hdr->inode;
 762
 763         dprintk("NFS: %s: %5u, (status %d)\n", __func__,
 764                 task->tk_pid, task->tk_status);
 765
 766         if (hdr->rw_ops->rw_done(task, hdr, inode) != 0)
 767                 return;
 768         if (task->tk_status < 0)
 769                 nfs_set_pgio_error(hdr, task->tk_status, hdr->args.offset);
 770         else
 771                 hdr->rw_ops->rw_result(task, hdr);
 772 }
 773
 774 /*
 775  * Create an RPC task for the given read or write request and kick it.
 776  * The page must have been locked by the caller.
 777  *
 778  * It may happen that the page we're passed is not marked dirty.
 779  * This is the case if nfs_updatepage detects a conflicting request
 780  * that has been written but not committed.
 781  */
 782 int nfs_generic_pgio(struct nfs_pageio_descriptor *desc,
 783                      struct nfs_pgio_header *hdr)
 784 {
 785         struct nfs_pgio_mirror *mirror = &desc->pg_mirrors[desc->pg_mirror_idx];
 786
 787         struct nfs_page         *req;
 788         struct page             **pages,
 789                                 *last_page;
 790         struct list_head *head = &mirror->pg_list;
 791         struct nfs_commit_info cinfo;
 792         unsigned int pagecount, pageused;
 793
 794         pagecount = nfs_page_array_len(mirror->pg_base, mirror->pg_count);
 795         if (!nfs_pgarray_set(&hdr->page_array, pagecount))
 796                 return nfs_pgio_error(desc, hdr);
 797
 798         nfs_init_cinfo(&cinfo, desc->pg_inode, desc->pg_dreq);
 799         pages = hdr->page_array.pagevec;
 800         last_page = NULL;
 801         pageused = 0;
 802         while (!list_empty(head)) {
 803                 req = nfs_list_entry(head->next);
 804                 nfs_list_remove_request(req);
 805                 nfs_list_add_request(req, &hdr->pages);
 806
 807                 if (!last_page || last_page != req->wb_page) {
 808                         pageused++;
 809                         if (pageused > pagecount)
 810                                 break;
 811                         *pages++ = last_page = req->wb_page;
 812                 }
 813         }
 814         if (WARN_ON_ONCE(pageused != pagecount))
 815                 return nfs_pgio_error(desc, hdr);
 816
 817         if ((desc->pg_ioflags & FLUSH_COND_STABLE) &&
 818             (desc->pg_moreio || nfs_reqs_to_commit(&cinfo)))
 819                 desc->pg_ioflags &= ~FLUSH_COND_STABLE;
 820
 821         /* Set up the argument struct */
 822         nfs_pgio_rpcsetup(hdr, mirror->pg_count, 0, desc->pg_ioflags, &cinfo);
 823         desc->pg_rpc_callops = &nfs_pgio_common_ops;
 824         return 0;
 825 }
 826 EXPORT_SYMBOL_GPL(nfs_generic_pgio);
 827
 828 static int nfs_generic_pg_pgios(struct nfs_pageio_descriptor *desc)
 829 {
 830         struct nfs_pgio_mirror *mirror;
 831         struct nfs_pgio_header *hdr;
 832         int ret;
 833
 834         mirror = &desc->pg_mirrors[desc->pg_mirror_idx];
 835
 836         hdr = nfs_pgio_header_alloc(desc->pg_rw_ops);
 837         if (!hdr) {
 838                 /* TODO: make sure this is right with mirroring - or
 839                  *       should it back out all mirrors? */
 840                 desc->pg_completion_ops->error_cleanup(&mirror->pg_list);
 841                 return -ENOMEM;
 842         }
 843         nfs_pgheader_init(desc, hdr, nfs_pgio_header_free);
 844         ret = nfs_generic_pgio(desc, hdr);
 845         if (ret == 0)
 846                 ret = nfs_initiate_pgio(NFS_CLIENT(hdr->inode),
 847                                         hdr,
 848                                         hdr->cred,
 849                                         NFS_PROTO(hdr->inode),
 850                                         desc->pg_rpc_callops,
 851                                         desc->pg_ioflags, 0);
 852         return ret;
 853 }
 854
 855 /*
 856  * nfs_pageio_setup_mirroring - determine if mirroring is to be used
 857  *                              by calling the pg_get_mirror_count op
 858  */
 859 static int nfs_pageio_setup_mirroring(struct nfs_pageio_descriptor *pgio,
 860                                        struct nfs_page *req)
 861 {
 862         int mirror_count = 1;
 863
 864         if (!pgio->pg_ops->pg_get_mirror_count)
 865                 return 0;
 866
 867         mirror_count = pgio->pg_ops->pg_get_mirror_count(pgio, req);
 868
 869         if (!mirror_count || mirror_count > NFS_PAGEIO_DESCRIPTOR_MIRROR_MAX)
 870                 return -EINVAL;
 871
 872         if (WARN_ON_ONCE(!pgio->pg_mirrors_dynamic))
 873                 return -EINVAL;
 874
 875         pgio->pg_mirror_count = mirror_count;
 876
 877         return 0;
 878 }
 879
 880 /*
 881  * nfs_pageio_stop_mirroring - stop using mirroring (set mirror count to 1)
 882  */
 883 void nfs_pageio_stop_mirroring(struct nfs_pageio_descriptor *pgio)
 884 {
 885         pgio->pg_mirror_count = 1;
 886         pgio->pg_mirror_idx = 0;
 887 }
 888
 889 static void nfs_pageio_cleanup_mirroring(struct nfs_pageio_descriptor *pgio)
 890 {
 891         pgio->pg_mirror_count = 1;
 892         pgio->pg_mirror_idx = 0;
 893         pgio->pg_mirrors = pgio->pg_mirrors_static;
 894         kfree(pgio->pg_mirrors_dynamic);
 895         pgio->pg_mirrors_dynamic = NULL;
 896 }
 897
 898 static bool nfs_match_open_context(const struct nfs_open_context *ctx1,
 899                 const struct nfs_open_context *ctx2)
 900 {
 901         return ctx1->cred == ctx2->cred && ctx1->state == ctx2->state;
 902 }
 903
 904 static bool nfs_match_lock_context(const struct nfs_lock_context *l1,
 905                 const struct nfs_lock_context *l2)
 906 {
 907         return l1->lockowner.l_owner == l2->lockowner.l_owner
 908                 && l1->lockowner.l_pid == l2->lockowner.l_pid;
 909 }
 910
 911 /**
 912  * nfs_can_coalesce_requests - test two requests for compatibility
 913  * @prev: pointer to nfs_page
 914  * @req: pointer to nfs_page
 915  *
 916  * The nfs_page structures 'prev' and 'req' are compared to ensure that the
 917  * page data area they describe is contiguous, and that their RPC
 918  * credentials, NFSv4 open state, and lockowners are the same.
 919  *
 920  * Return 'true' if this is the case, else return 'false'.
 921  */
 922 static bool nfs_can_coalesce_requests(struct nfs_page *prev,
 923                                       struct nfs_page *req,
 924                                       struct nfs_pageio_descriptor *pgio)
 925 {
 926         size_t size;
 927
 928         if (prev) {
 929                 if (!nfs_match_open_context(req->wb_context, prev->wb_context))
 930                         return false;
 931                 if (req->wb_context->dentry->d_inode->i_flock != NULL &&
 932                     !nfs_match_lock_context(req->wb_lock_context,
 933                                             prev->wb_lock_context))
 934                         return false;
 935                 if (req_offset(req) != req_offset(prev) + prev->wb_bytes)
 936                         return false;
 937                 if (req->wb_page == prev->wb_page) {
 938                         if (req->wb_pgbase != prev->wb_pgbase + prev->wb_bytes)
 939                                 return false;
 940                 } else {
 941                         if (req->wb_pgbase != 0 ||
 942                             prev->wb_pgbase + prev->wb_bytes != PAGE_CACHE_SIZE)
 943                                 return false;
 944                 }
 945         }
 946         size = pgio->pg_ops->pg_test(pgio, prev, req);
 947         WARN_ON_ONCE(size > req->wb_bytes);
 948         if (size && size < req->wb_bytes)
 949                 req->wb_bytes = size;
 950         return size > 0;
 951 }
 952
 953 /**
 954  * nfs_pageio_do_add_request - Attempt to coalesce a request into a page list.
 955  * @desc: destination io descriptor
 956  * @req: request
 957  *
 958  * Returns true if the request 'req' was successfully coalesced into the
 959  * existing list of pages 'desc'.
 960  */
 961 static int nfs_pageio_do_add_request(struct nfs_pageio_descriptor *desc,
 962                                      struct nfs_page *req)
 963 {
 964         struct nfs_pgio_mirror *mirror = &desc->pg_mirrors[desc->pg_mirror_idx];
 965
 966         struct nfs_page *prev = NULL;
 967
 968         if (mirror->pg_count != 0) {
 969                 prev = nfs_list_entry(mirror->pg_list.prev);
 970         } else {
 971                 if (desc->pg_ops->pg_init)
 972                         desc->pg_ops->pg_init(desc, req);
 973                 mirror->pg_base = req->wb_pgbase;
 974         }
 975         if (!nfs_can_coalesce_requests(prev, req, desc))
 976                 return 0;
 977         nfs_list_remove_request(req);
 978         nfs_list_add_request(req, &mirror->pg_list);
 979         mirror->pg_count += req->wb_bytes;
 980         return 1;
 981 }
 982
 983 /*
 984  * Helper for nfs_pageio_add_request and nfs_pageio_complete
 985  */
 986 static void nfs_pageio_doio(struct nfs_pageio_descriptor *desc)
 987 {
 988         struct nfs_pgio_mirror *mirror = &desc->pg_mirrors[desc->pg_mirror_idx];
 989
 990
 991         if (!list_empty(&mirror->pg_list)) {
 992                 int error = desc->pg_ops->pg_doio(desc);
 993                 if (error < 0)
 994                         desc->pg_error = error;
 995                 else
 996                         mirror->pg_bytes_written += mirror->pg_count;
 997         }
 998         if (list_empty(&mirror->pg_list)) {
 999                 mirror->pg_count = 0;
1000                 mirror->pg_base = 0;
1001         }
1002 }
1003
1004 /**
1005  * nfs_pageio_add_request - Attempt to coalesce a request into a page list.
1006  * @desc: destination io descriptor
1007  * @req: request
1008  *
1009  * This may split a request into subrequests which are all part of the
1010  * same page group.
1011  *
1012  * Returns true if the request 'req' was successfully coalesced into the
1013  * existing list of pages 'desc'.
1014  */
1015 static int __nfs_pageio_add_request(struct nfs_pageio_descriptor *desc,
1016                            struct nfs_page *req)
1017 {
1018         struct nfs_pgio_mirror *mirror = &desc->pg_mirrors[desc->pg_mirror_idx];
1019
1020         struct nfs_page *subreq;
1021         unsigned int bytes_left = 0;
1022         unsigned int offset, pgbase;
1023
1024         nfs_page_group_lock(req, false);
1025
1026         subreq = req;
1027         bytes_left = subreq->wb_bytes;
1028         offset = subreq->wb_offset;
1029         pgbase = subreq->wb_pgbase;
1030
1031         do {
1032                 if (!nfs_pageio_do_add_request(desc, subreq)) {
1033                         /* make sure pg_test call(s) did nothing */
1034                         WARN_ON_ONCE(subreq->wb_bytes != bytes_left);
1035                         WARN_ON_ONCE(subreq->wb_offset != offset);
1036                         WARN_ON_ONCE(subreq->wb_pgbase != pgbase);
1037
1038                         nfs_page_group_unlock(req);
1039                         desc->pg_moreio = 1;
1040                         nfs_pageio_doio(desc);
1041                         if (desc->pg_error < 0)
1042                                 return 0;
1043                         if (mirror->pg_recoalesce)
1044                                 return 0;
1045                         /* retry add_request for this subreq */
1046                         nfs_page_group_lock(req, false);
1047                         continue;
1048                 }
1049
1050                 /* check for buggy pg_test call(s) */
1051                 WARN_ON_ONCE(subreq->wb_bytes + subreq->wb_pgbase > PAGE_SIZE);
1052                 WARN_ON_ONCE(subreq->wb_bytes > bytes_left);
1053                 WARN_ON_ONCE(subreq->wb_bytes == 0);
1054
1055                 bytes_left -= subreq->wb_bytes;
1056                 offset += subreq->wb_bytes;
1057                 pgbase += subreq->wb_bytes;
1058
1059                 if (bytes_left) {
1060                         subreq = nfs_create_request(req->wb_context,
1061                                         req->wb_page,
1062                                         subreq, pgbase, bytes_left);
1063                         if (IS_ERR(subreq))
1064                                 goto err_ptr;
1065                         nfs_lock_request(subreq);
1066                         subreq->wb_offset  = offset;
1067                         subreq->wb_index = req->wb_index;
1068                 }
1069         } while (bytes_left > 0);
1070
1071         nfs_page_group_unlock(req);
1072         return 1;
1073 err_ptr:
1074         desc->pg_error = PTR_ERR(subreq);
1075         nfs_page_group_unlock(req);
1076         return 0;
1077 }
1078
1079 static int nfs_do_recoalesce(struct nfs_pageio_descriptor *desc)
1080 {
1081         struct nfs_pgio_mirror *mirror = &desc->pg_mirrors[desc->pg_mirror_idx];
1082         LIST_HEAD(head);
1083
1084         do {
1085                 list_splice_init(&mirror->pg_list, &head);
1086                 mirror->pg_bytes_written -= mirror->pg_count;
1087                 mirror->pg_count = 0;
1088                 mirror->pg_base = 0;
1089                 mirror->pg_recoalesce = 0;
1090
1091                 desc->pg_moreio = 0;
1092
1093                 while (!list_empty(&head)) {
1094                         struct nfs_page *req;
1095
1096                         req = list_first_entry(&head, struct nfs_page, wb_list);
1097                         nfs_list_remove_request(req);
1098                         if (__nfs_pageio_add_request(desc, req))
1099                                 continue;
1100                         if (desc->pg_error < 0)
1101                                 return 0;
1102                         break;
1103                 }
1104         } while (mirror->pg_recoalesce);
1105         return 1;
1106 }
1107
1108 static int nfs_pageio_add_request_mirror(struct nfs_pageio_descriptor *desc,
1109                 struct nfs_page *req)
1110 {
1111         int ret;
1112
1113         do {
1114                 ret = __nfs_pageio_add_request(desc, req);
1115                 if (ret)
1116                         break;
1117                 if (desc->pg_error < 0)
1118                         break;
1119                 ret = nfs_do_recoalesce(desc);
1120         } while (ret);
1121
1122         return ret;
1123 }
1124
1125 int nfs_pageio_add_request(struct nfs_pageio_descriptor *desc,
1126                            struct nfs_page *req)
1127 {
1128         u32 midx;
1129         unsigned int pgbase, offset, bytes;
1130         struct nfs_page *dupreq, *lastreq;
1131
1132         pgbase = req->wb_pgbase;
1133         offset = req->wb_offset;
1134         bytes = req->wb_bytes;
1135
1136         nfs_pageio_setup_mirroring(desc, req);
1137
1138         for (midx = 0; midx < desc->pg_mirror_count; midx++) {
1139                 if (midx) {
1140                         nfs_page_group_lock(req, false);
1141
1142                         /* find the last request */
1143                         for (lastreq = req->wb_head;
1144                              lastreq->wb_this_page != req->wb_head;
1145                              lastreq = lastreq->wb_this_page)
1146                                 ;
1147
1148                         dupreq = nfs_create_request(req->wb_context,
1149                                         req->wb_page, lastreq, pgbase, bytes);
1150
1151                         if (IS_ERR(dupreq)) {
1152                                 nfs_page_group_unlock(req);
1153                                 return 0;
1154                         }
1155
1156                         nfs_lock_request(dupreq);
1157                         nfs_page_group_unlock(req);
1158                         dupreq->wb_offset = offset;
1159                         dupreq->wb_index = req->wb_index;
1160                 } else
1161                         dupreq = req;
1162
1163                 if (nfs_pgio_has_mirroring(desc))
1164                         desc->pg_mirror_idx = midx;
1165                 if (!nfs_pageio_add_request_mirror(desc, dupreq))
1166                         return 0;
1167         }
1168
1169         return 1;
1170 }
1171
1172 /*
1173  * nfs_pageio_complete_mirror - Complete I/O on the current mirror of an
1174  *                              nfs_pageio_descriptor
1175  * @desc: pointer to io descriptor
1176  */
1177 static void nfs_pageio_complete_mirror(struct nfs_pageio_descriptor *desc,
1178                                        u32 mirror_idx)
1179 {
1180         struct nfs_pgio_mirror *mirror = &desc->pg_mirrors[mirror_idx];
1181         u32 restore_idx = desc->pg_mirror_idx;
1182
1183         if (nfs_pgio_has_mirroring(desc))
1184                 desc->pg_mirror_idx = mirror_idx;
1185         for (;;) {
1186                 nfs_pageio_doio(desc);
1187                 if (!mirror->pg_recoalesce)
1188                         break;
1189                 if (!nfs_do_recoalesce(desc))
1190                         break;
1191         }
1192         desc->pg_mirror_idx = restore_idx;
1193 }
1194
1195 /*
1196  * nfs_pageio_resend - Transfer requests to new descriptor and resend
1197  * @hdr - the pgio header to move request from
1198  * @desc - the pageio descriptor to add requests to
1199  *
1200  * Try to move each request (nfs_page) from @hdr to @desc then attempt
1201  * to send them.
1202  *
1203  * Returns 0 on success and < 0 on error.
1204  */
1205 int nfs_pageio_resend(struct nfs_pageio_descriptor *desc,
1206                       struct nfs_pgio_header *hdr)
1207 {
1208         LIST_HEAD(failed);
1209
1210         desc->pg_dreq = hdr->dreq;
1211         while (!list_empty(&hdr->pages)) {
1212                 struct nfs_page *req = nfs_list_entry(hdr->pages.next);
1213
1214                 nfs_list_remove_request(req);
1215                 if (!nfs_pageio_add_request(desc, req))
1216                         nfs_list_add_request(req, &failed);
1217         }
1218         nfs_pageio_complete(desc);
1219         if (!list_empty(&failed)) {
1220                 list_move(&failed, &hdr->pages);
1221                 return -EIO;
1222         }
1223         return 0;
1224 }
1225 EXPORT_SYMBOL_GPL(nfs_pageio_resend);
1226
1227 /**
1228  * nfs_pageio_complete - Complete I/O then cleanup an nfs_pageio_descriptor
1229  * @desc: pointer to io descriptor
1230  */
1231 void nfs_pageio_complete(struct nfs_pageio_descriptor *desc)
1232 {
1233         u32 midx;
1234
1235         for (midx = 0; midx < desc->pg_mirror_count; midx++)
1236                 nfs_pageio_complete_mirror(desc, midx);
1237
1238         if (desc->pg_ops->pg_cleanup)
1239                 desc->pg_ops->pg_cleanup(desc);
1240         nfs_pageio_cleanup_mirroring(desc);
1241 }
1242
1243 /**
1244  * nfs_pageio_cond_complete - Conditional I/O completion
1245  * @desc: pointer to io descriptor
1246  * @index: page index
1247  *
1248  * It is important to ensure that processes don't try to take locks
1249  * on non-contiguous ranges of pages as that might deadlock. This
1250  * function should be called before attempting to wait on a locked
1251  * nfs_page. It will complete the I/O if the page index 'index'
1252  * is not contiguous with the existing list of pages in 'desc'.
1253  */
1254 void nfs_pageio_cond_complete(struct nfs_pageio_descriptor *desc, pgoff_t index)
1255 {
1256         struct nfs_pgio_mirror *mirror;
1257         struct nfs_page *prev;
1258         u32 midx;
1259
1260         for (midx = 0; midx < desc->pg_mirror_count; midx++) {
1261                 mirror = &desc->pg_mirrors[midx];
1262                 if (!list_empty(&mirror->pg_list)) {
1263                         prev = nfs_list_entry(mirror->pg_list.prev);
1264                         if (index != prev->wb_index + 1)
1265                                 nfs_pageio_complete_mirror(desc, midx);
1266                 }
1267         }
1268 }
1269
1270 int __init nfs_init_nfspagecache(void)
1271 {
1272         nfs_page_cachep = kmem_cache_create("nfs_page",
1273                                             sizeof(struct nfs_page),
1274                                             0, SLAB_HWCACHE_ALIGN,
1275                                             NULL);
1276         if (nfs_page_cachep == NULL)
1277                 return -ENOMEM;
1278
1279         return 0;
1280 }
1281
1282 void nfs_destroy_nfspagecache(void)
1283 {
1284         kmem_cache_destroy(nfs_page_cachep);
1285 }
1286
1287 static const struct rpc_call_ops nfs_pgio_common_ops = {
1288         .rpc_call_prepare = nfs_pgio_prepare,
1289         .rpc_call_done = nfs_pgio_result,
1290         .rpc_release = nfs_pgio_release,
1291 };
1292
1293 const struct nfs_pageio_ops nfs_pgio_rw_ops = {
1294         .pg_test = nfs_generic_pg_test,
1295         .pg_doio = nfs_generic_pg_pgios,
1296 };