fs/xfs/linux-2.6/xfs_buf.c

   1 /*
   2  * Copyright (c) 2000-2006 Silicon Graphics, Inc.
   3  * All Rights Reserved.
   4  *
   5  * This program is free software; you can redistribute it and/or
   6  * modify it under the terms of the GNU General Public License as
   7  * published by the Free Software Foundation.
   8  *
   9  * This program is distributed in the hope that it would be useful,
  10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12  * GNU General Public License for more details.
  13  *
  14  * You should have received a copy of the GNU General Public License
  15  * along with this program; if not, write the Free Software Foundation,
  16  * Inc.,  51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
  17  */
  18 #include <linux/stddef.h>
  19 #include <linux/errno.h>
  20 #include <linux/slab.h>
  21 #include <linux/pagemap.h>
  22 #include <linux/init.h>
  23 #include <linux/vmalloc.h>
  24 #include <linux/bio.h>
  25 #include <linux/sysctl.h>
  26 #include <linux/proc_fs.h>
  27 #include <linux/workqueue.h>
  28 #include <linux/percpu.h>
  29 #include <linux/blkdev.h>
  30 #include <linux/hash.h>
  31 #include <linux/kthread.h>
  32 #include <linux/migrate.h>
  33 #include "xfs_linux.h"
  34
  35 STATIC kmem_zone_t *xfs_buf_zone;
  36 STATIC kmem_shaker_t xfs_buf_shake;
  37 STATIC int xfsbufd(void *);
  38 STATIC int xfsbufd_wakeup(int, gfp_t);
  39 STATIC void xfs_buf_delwri_queue(xfs_buf_t *, int);
  40
  41 STATIC struct workqueue_struct *xfslogd_workqueue;
  42 struct workqueue_struct *xfsdatad_workqueue;
  43
  44 #ifdef XFS_BUF_TRACE
  45 void
  46 xfs_buf_trace(
  47         xfs_buf_t       *bp,
  48         char            *id,
  49         void            *data,
  50         void            *ra)
  51 {
  52         ktrace_enter(xfs_buf_trace_buf,
  53                 bp, id,
  54                 (void *)(unsigned long)bp->b_flags,
  55                 (void *)(unsigned long)bp->b_hold.counter,
  56                 (void *)(unsigned long)bp->b_sema.count.counter,
  57                 (void *)current,
  58                 data, ra,
  59                 (void *)(unsigned long)((bp->b_file_offset>>32) & 0xffffffff),
  60                 (void *)(unsigned long)(bp->b_file_offset & 0xffffffff),
  61                 (void *)(unsigned long)bp->b_buffer_length,
  62                 NULL, NULL, NULL, NULL, NULL);
  63 }
  64 ktrace_t *xfs_buf_trace_buf;
  65 #define XFS_BUF_TRACE_SIZE      4096
  66 #define XB_TRACE(bp, id, data)  \
  67         xfs_buf_trace(bp, id, (void *)data, (void *)__builtin_return_address(0))
  68 #else
  69 #define XB_TRACE(bp, id, data)  do { } while (0)
  70 #endif
  71
  72 #ifdef XFS_BUF_LOCK_TRACKING
  73 # define XB_SET_OWNER(bp)       ((bp)->b_last_holder = current->pid)
  74 # define XB_CLEAR_OWNER(bp)     ((bp)->b_last_holder = -1)
  75 # define XB_GET_OWNER(bp)       ((bp)->b_last_holder)
  76 #else
  77 # define XB_SET_OWNER(bp)       do { } while (0)
  78 # define XB_CLEAR_OWNER(bp)     do { } while (0)
  79 # define XB_GET_OWNER(bp)       do { } while (0)
  80 #endif
  81
  82 #define xb_to_gfp(flags) \
  83         ((((flags) & XBF_READ_AHEAD) ? __GFP_NORETRY : \
  84           ((flags) & XBF_DONT_BLOCK) ? GFP_NOFS : GFP_KERNEL) | __GFP_NOWARN)
  85
  86 #define xb_to_km(flags) \
  87          (((flags) & XBF_DONT_BLOCK) ? KM_NOFS : KM_SLEEP)
  88
  89 #define xfs_buf_allocate(flags) \
  90         kmem_zone_alloc(xfs_buf_zone, xb_to_km(flags))
  91 #define xfs_buf_deallocate(bp) \
  92         kmem_zone_free(xfs_buf_zone, (bp));
  93
  94 /*
  95  *      Page Region interfaces.
  96  *
  97  *      For pages in filesystems where the blocksize is smaller than the
  98  *      pagesize, we use the page->private field (long) to hold a bitmap
  99  *      of uptodate regions within the page.
 100  *
 101  *      Each such region is "bytes per page / bits per long" bytes long.
 102  *
 103  *      NBPPR == number-of-bytes-per-page-region
 104  *      BTOPR == bytes-to-page-region (rounded up)
 105  *      BTOPRT == bytes-to-page-region-truncated (rounded down)
 106  */
 107 #if (BITS_PER_LONG == 32)
 108 #define PRSHIFT         (PAGE_CACHE_SHIFT - 5)  /* (32 == 1<<5) */
 109 #elif (BITS_PER_LONG == 64)
 110 #define PRSHIFT         (PAGE_CACHE_SHIFT - 6)  /* (64 == 1<<6) */
 111 #else
 112 #error BITS_PER_LONG must be 32 or 64
 113 #endif
 114 #define NBPPR           (PAGE_CACHE_SIZE/BITS_PER_LONG)
 115 #define BTOPR(b)        (((unsigned int)(b) + (NBPPR - 1)) >> PRSHIFT)
 116 #define BTOPRT(b)       (((unsigned int)(b) >> PRSHIFT))
 117
 118 STATIC unsigned long
 119 page_region_mask(
 120         size_t          offset,
 121         size_t          length)
 122 {
 123         unsigned long   mask;
 124         int             first, final;
 125
 126         first = BTOPR(offset);
 127         final = BTOPRT(offset + length - 1);
 128         first = min(first, final);
 129
 130         mask = ~0UL;
 131         mask <<= BITS_PER_LONG - (final - first);
 132         mask >>= BITS_PER_LONG - (final);
 133
 134         ASSERT(offset + length <= PAGE_CACHE_SIZE);
 135         ASSERT((final - first) < BITS_PER_LONG && (final - first) >= 0);
 136
 137         return mask;
 138 }
 139
 140 STATIC inline void
 141 set_page_region(
 142         struct page     *page,
 143         size_t          offset,
 144         size_t          length)
 145 {
 146         set_page_private(page,
 147                 page_private(page) | page_region_mask(offset, length));
 148         if (page_private(page) == ~0UL)
 149                 SetPageUptodate(page);
 150 }
 151
 152 STATIC inline int
 153 test_page_region(
 154         struct page     *page,
 155         size_t          offset,
 156         size_t          length)
 157 {
 158         unsigned long   mask = page_region_mask(offset, length);
 159
 160         return (mask && (page_private(page) & mask) == mask);
 161 }
 162
 163 /*
 164  *      Mapping of multi-page buffers into contiguous virtual space
 165  */
 166
 167 typedef struct a_list {
 168         void            *vm_addr;
 169         struct a_list   *next;
 170 } a_list_t;
 171
 172 STATIC a_list_t         *as_free_head;
 173 STATIC int              as_list_len;
 174 STATIC DEFINE_SPINLOCK(as_lock);
 175
 176 /*
 177  *      Try to batch vunmaps because they are costly.
 178  */
 179 STATIC void
 180 free_address(
 181         void            *addr)
 182 {
 183         a_list_t        *aentry;
 184
 185         aentry = kmalloc(sizeof(a_list_t), GFP_NOWAIT);
 186         if (likely(aentry)) {
 187                 spin_lock(&as_lock);
 188                 aentry->next = as_free_head;
 189                 aentry->vm_addr = addr;
 190                 as_free_head = aentry;
 191                 as_list_len++;
 192                 spin_unlock(&as_lock);
 193         } else {
 194                 vunmap(addr);
 195         }
 196 }
 197
 198 STATIC void
 199 purge_addresses(void)
 200 {
 201         a_list_t        *aentry, *old;
 202
 203         if (as_free_head == NULL)
 204                 return;
 205
 206         spin_lock(&as_lock);
 207         aentry = as_free_head;
 208         as_free_head = NULL;
 209         as_list_len = 0;
 210         spin_unlock(&as_lock);
 211
 212         while ((old = aentry) != NULL) {
 213                 vunmap(aentry->vm_addr);
 214                 aentry = aentry->next;
 215                 kfree(old);
 216         }
 217 }
 218
 219 /*
 220  *      Internal xfs_buf_t object manipulation
 221  */
 222
 223 STATIC void
 224 _xfs_buf_initialize(
 225         xfs_buf_t               *bp,
 226         xfs_buftarg_t           *target,
 227         xfs_off_t               range_base,
 228         size_t                  range_length,
 229         xfs_buf_flags_t         flags)
 230 {
 231         /*
 232          * We don't want certain flags to appear in b_flags.
 233          */
 234         flags &= ~(XBF_LOCK|XBF_MAPPED|XBF_DONT_BLOCK|XBF_READ_AHEAD);
 235
 236         memset(bp, 0, sizeof(xfs_buf_t));
 237         atomic_set(&bp->b_hold, 1);
 238         init_MUTEX_LOCKED(&bp->b_iodonesema);
 239         INIT_LIST_HEAD(&bp->b_list);
 240         INIT_LIST_HEAD(&bp->b_hash_list);
 241         init_MUTEX_LOCKED(&bp->b_sema); /* held, no waiters */
 242         XB_SET_OWNER(bp);
 243         bp->b_target = target;
 244         bp->b_file_offset = range_base;
 245         /*
 246          * Set buffer_length and count_desired to the same value initially.
 247          * I/O routines should use count_desired, which will be the same in
 248          * most cases but may be reset (e.g. XFS recovery).
 249          */
 250         bp->b_buffer_length = bp->b_count_desired = range_length;
 251         bp->b_flags = flags;
 252         bp->b_bn = XFS_BUF_DADDR_NULL;
 253         atomic_set(&bp->b_pin_count, 0);
 254         init_waitqueue_head(&bp->b_waiters);
 255
 256         XFS_STATS_INC(xb_create);
 257         XB_TRACE(bp, "initialize", target);
 258 }
 259
 260 /*
 261  *      Allocate a page array capable of holding a specified number
 262  *      of pages, and point the page buf at it.
 263  */
 264 STATIC int
 265 _xfs_buf_get_pages(
 266         xfs_buf_t               *bp,
 267         int                     page_count,
 268         xfs_buf_flags_t         flags)
 269 {
 270         /* Make sure that we have a page list */
 271         if (bp->b_pages == NULL) {
 272                 bp->b_offset = xfs_buf_poff(bp->b_file_offset);
 273                 bp->b_page_count = page_count;
 274                 if (page_count <= XB_PAGES) {
 275                         bp->b_pages = bp->b_page_array;
 276                 } else {
 277                         bp->b_pages = kmem_alloc(sizeof(struct page *) *
 278                                         page_count, xb_to_km(flags));
 279                         if (bp->b_pages == NULL)
 280                                 return -ENOMEM;
 281                 }
 282                 memset(bp->b_pages, 0, sizeof(struct page *) * page_count);
 283         }
 284         return 0;
 285 }
 286
 287 /*
 288  *      Frees b_pages if it was allocated.
 289  */
 290 STATIC void
 291 _xfs_buf_free_pages(
 292         xfs_buf_t       *bp)
 293 {
 294         if (bp->b_pages != bp->b_page_array) {
 295                 kmem_free(bp->b_pages,
 296                           bp->b_page_count * sizeof(struct page *));
 297         }
 298 }
 299
 300 /*
 301  *      Releases the specified buffer.
 302  *
 303  *      The modification state of any associated pages is left unchanged.
 304  *      The buffer most not be on any hash - use xfs_buf_rele instead for
 305  *      hashed and refcounted buffers
 306  */
 307 void
 308 xfs_buf_free(
 309         xfs_buf_t               *bp)
 310 {
 311         XB_TRACE(bp, "free", 0);
 312
 313         ASSERT(list_empty(&bp->b_hash_list));
 314
 315         if (bp->b_flags & _XBF_PAGE_CACHE) {
 316                 uint            i;
 317
 318                 if ((bp->b_flags & XBF_MAPPED) && (bp->b_page_count > 1))
 319                         free_address(bp->b_addr - bp->b_offset);
 320
 321                 for (i = 0; i < bp->b_page_count; i++) {
 322                         struct page     *page = bp->b_pages[i];
 323
 324                         ASSERT(!PagePrivate(page));
 325                         page_cache_release(page);
 326                 }
 327                 _xfs_buf_free_pages(bp);
 328         } else if (bp->b_flags & _XBF_KMEM_ALLOC) {
 329                  /*
 330                   * XXX(hch): bp->b_count_desired might be incorrect (see
 331                   * xfs_buf_associate_memory for details), but fortunately
 332                   * the Linux version of kmem_free ignores the len argument..
 333                   */
 334                 kmem_free(bp->b_addr, bp->b_count_desired);
 335                 _xfs_buf_free_pages(bp);
 336         }
 337
 338         xfs_buf_deallocate(bp);
 339 }
 340
 341 /*
 342  *      Finds all pages for buffer in question and builds it's page list.
 343  */
 344 STATIC int
 345 _xfs_buf_lookup_pages(
 346         xfs_buf_t               *bp,
 347         uint                    flags)
 348 {
 349         struct address_space    *mapping = bp->b_target->bt_mapping;
 350         size_t                  blocksize = bp->b_target->bt_bsize;
 351         size_t                  size = bp->b_count_desired;
 352         size_t                  nbytes, offset;
 353         gfp_t                   gfp_mask = xb_to_gfp(flags);
 354         unsigned short          page_count, i;
 355         pgoff_t                 first;
 356         xfs_off_t               end;
 357         int                     error;
 358
 359         end = bp->b_file_offset + bp->b_buffer_length;
 360         page_count = xfs_buf_btoc(end) - xfs_buf_btoct(bp->b_file_offset);
 361
 362         error = _xfs_buf_get_pages(bp, page_count, flags);
 363         if (unlikely(error))
 364                 return error;
 365         bp->b_flags |= _XBF_PAGE_CACHE;
 366
 367         offset = bp->b_offset;
 368         first = bp->b_file_offset >> PAGE_CACHE_SHIFT;
 369
 370         for (i = 0; i < bp->b_page_count; i++) {
 371                 struct page     *page;
 372                 uint            retries = 0;
 373
 374               retry:
 375                 page = find_or_create_page(mapping, first + i, gfp_mask);
 376                 if (unlikely(page == NULL)) {
 377                         if (flags & XBF_READ_AHEAD) {
 378                                 bp->b_page_count = i;
 379                                 for (i = 0; i < bp->b_page_count; i++)
 380                                         unlock_page(bp->b_pages[i]);
 381                                 return -ENOMEM;
 382                         }
 383
 384                         /*
 385                          * This could deadlock.
 386                          *
 387                          * But until all the XFS lowlevel code is revamped to
 388                          * handle buffer allocation failures we can't do much.
 389                          */
 390                         if (!(++retries % 100))
 391                                 printk(KERN_ERR
 392                                         "XFS: possible memory allocation "
 393                                         "deadlock in %s (mode:0x%x)\n",
 394                                         __FUNCTION__, gfp_mask);
 395
 396                         XFS_STATS_INC(xb_page_retries);
 397                         xfsbufd_wakeup(0, gfp_mask);
 398                         blk_congestion_wait(WRITE, HZ/50);
 399                         goto retry;
 400                 }
 401
 402                 XFS_STATS_INC(xb_page_found);
 403
 404                 nbytes = min_t(size_t, size, PAGE_CACHE_SIZE - offset);
 405                 size -= nbytes;
 406
 407                 ASSERT(!PagePrivate(page));
 408                 if (!PageUptodate(page)) {
 409                         page_count--;
 410                         if (blocksize >= PAGE_CACHE_SIZE) {
 411                                 if (flags & XBF_READ)
 412                                         bp->b_locked = 1;
 413                         } else if (!PagePrivate(page)) {
 414                                 if (test_page_region(page, offset, nbytes))
 415                                         page_count++;
 416                         }
 417                 }
 418
 419                 bp->b_pages[i] = page;
 420                 offset = 0;
 421         }
 422
 423         if (!bp->b_locked) {
 424                 for (i = 0; i < bp->b_page_count; i++)
 425                         unlock_page(bp->b_pages[i]);
 426         }
 427
 428         if (page_count == bp->b_page_count)
 429                 bp->b_flags |= XBF_DONE;
 430
 431         XB_TRACE(bp, "lookup_pages", (long)page_count);
 432         return error;
 433 }
 434
 435 /*
 436  *      Map buffer into kernel address-space if nessecary.
 437  */
 438 STATIC int
 439 _xfs_buf_map_pages(
 440         xfs_buf_t               *bp,
 441         uint                    flags)
 442 {
 443         /* A single page buffer is always mappable */
 444         if (bp->b_page_count == 1) {
 445                 bp->b_addr = page_address(bp->b_pages[0]) + bp->b_offset;
 446                 bp->b_flags |= XBF_MAPPED;
 447         } else if (flags & XBF_MAPPED) {
 448                 if (as_list_len > 64)
 449                         purge_addresses();
 450                 bp->b_addr = vmap(bp->b_pages, bp->b_page_count,
 451                                         VM_MAP, PAGE_KERNEL);
 452                 if (unlikely(bp->b_addr == NULL))
 453                         return -ENOMEM;
 454                 bp->b_addr += bp->b_offset;
 455                 bp->b_flags |= XBF_MAPPED;
 456         }
 457
 458         return 0;
 459 }
 460
 461 /*
 462  *      Finding and Reading Buffers
 463  */
 464
 465 /*
 466  *      Look up, and creates if absent, a lockable buffer for
 467  *      a given range of an inode.  The buffer is returned
 468  *      locked.  If other overlapping buffers exist, they are
 469  *      released before the new buffer is created and locked,
 470  *      which may imply that this call will block until those buffers
 471  *      are unlocked.  No I/O is implied by this call.
 472  */
 473 xfs_buf_t *
 474 _xfs_buf_find(
 475         xfs_buftarg_t           *btp,   /* block device target          */
 476         xfs_off_t               ioff,   /* starting offset of range     */
 477         size_t                  isize,  /* length of range              */
 478         xfs_buf_flags_t         flags,
 479         xfs_buf_t               *new_bp)
 480 {
 481         xfs_off_t               range_base;
 482         size_t                  range_length;
 483         xfs_bufhash_t           *hash;
 484         xfs_buf_t               *bp, *n;
 485
 486         range_base = (ioff << BBSHIFT);
 487         range_length = (isize << BBSHIFT);
 488
 489         /* Check for IOs smaller than the sector size / not sector aligned */
 490         ASSERT(!(range_length < (1 << btp->bt_sshift)));
 491         ASSERT(!(range_base & (xfs_off_t)btp->bt_smask));
 492
 493         hash = &btp->bt_hash[hash_long((unsigned long)ioff, btp->bt_hashshift)];
 494
 495         spin_lock(&hash->bh_lock);
 496
 497         list_for_each_entry_safe(bp, n, &hash->bh_list, b_hash_list) {
 498                 ASSERT(btp == bp->b_target);
 499                 if (bp->b_file_offset == range_base &&
 500                     bp->b_buffer_length == range_length) {
 501                         /*
 502                          * If we look at something, bring it to the
 503                          * front of the list for next time.
 504                          */
 505                         atomic_inc(&bp->b_hold);
 506                         list_move(&bp->b_hash_list, &hash->bh_list);
 507                         goto found;
 508                 }
 509         }
 510
 511         /* No match found */
 512         if (new_bp) {
 513                 _xfs_buf_initialize(new_bp, btp, range_base,
 514                                 range_length, flags);
 515                 new_bp->b_hash = hash;
 516                 list_add(&new_bp->b_hash_list, &hash->bh_list);
 517         } else {
 518                 XFS_STATS_INC(xb_miss_locked);
 519         }
 520
 521         spin_unlock(&hash->bh_lock);
 522         return new_bp;
 523
 524 found:
 525         spin_unlock(&hash->bh_lock);
 526
 527         /* Attempt to get the semaphore without sleeping,
 528          * if this does not work then we need to drop the
 529          * spinlock and do a hard attempt on the semaphore.
 530          */
 531         if (down_trylock(&bp->b_sema)) {
 532                 if (!(flags & XBF_TRYLOCK)) {
 533                         /* wait for buffer ownership */
 534                         XB_TRACE(bp, "get_lock", 0);
 535                         xfs_buf_lock(bp);
 536                         XFS_STATS_INC(xb_get_locked_waited);
 537                 } else {
 538                         /* We asked for a trylock and failed, no need
 539                          * to look at file offset and length here, we
 540                          * know that this buffer at least overlaps our
 541                          * buffer and is locked, therefore our buffer
 542                          * either does not exist, or is this buffer.
 543                          */
 544                         xfs_buf_rele(bp);
 545                         XFS_STATS_INC(xb_busy_locked);
 546                         return NULL;
 547                 }
 548         } else {
 549                 /* trylock worked */
 550                 XB_SET_OWNER(bp);
 551         }
 552
 553         if (bp->b_flags & XBF_STALE) {
 554                 ASSERT((bp->b_flags & _XBF_DELWRI_Q) == 0);
 555                 bp->b_flags &= XBF_MAPPED;
 556         }
 557         XB_TRACE(bp, "got_lock", 0);
 558         XFS_STATS_INC(xb_get_locked);
 559         return bp;
 560 }
 561
 562 /*
 563  *      Assembles a buffer covering the specified range.
 564  *      Storage in memory for all portions of the buffer will be allocated,
 565  *      although backing storage may not be.
 566  */
 567 xfs_buf_t *
 568 xfs_buf_get_flags(
 569         xfs_buftarg_t           *target,/* target for buffer            */
 570         xfs_off_t               ioff,   /* starting offset of range     */
 571         size_t                  isize,  /* length of range              */
 572         xfs_buf_flags_t         flags)
 573 {
 574         xfs_buf_t               *bp, *new_bp;
 575         int                     error = 0, i;
 576
 577         new_bp = xfs_buf_allocate(flags);
 578         if (unlikely(!new_bp))
 579                 return NULL;
 580
 581         bp = _xfs_buf_find(target, ioff, isize, flags, new_bp);
 582         if (bp == new_bp) {
 583                 error = _xfs_buf_lookup_pages(bp, flags);
 584                 if (error)
 585                         goto no_buffer;
 586         } else {
 587                 xfs_buf_deallocate(new_bp);
 588                 if (unlikely(bp == NULL))
 589                         return NULL;
 590         }
 591
 592         for (i = 0; i < bp->b_page_count; i++)
 593                 mark_page_accessed(bp->b_pages[i]);
 594
 595         if (!(bp->b_flags & XBF_MAPPED)) {
 596                 error = _xfs_buf_map_pages(bp, flags);
 597                 if (unlikely(error)) {
 598                         printk(KERN_WARNING "%s: failed to map pages\n",
 599                                         __FUNCTION__);
 600                         goto no_buffer;
 601                 }
 602         }
 603
 604         XFS_STATS_INC(xb_get);
 605
 606         /*
 607          * Always fill in the block number now, the mapped cases can do
 608          * their own overlay of this later.
 609          */
 610         bp->b_bn = ioff;
 611         bp->b_count_desired = bp->b_buffer_length;
 612
 613         XB_TRACE(bp, "get", (unsigned long)flags);
 614         return bp;
 615
 616  no_buffer:
 617         if (flags & (XBF_LOCK | XBF_TRYLOCK))
 618                 xfs_buf_unlock(bp);
 619         xfs_buf_rele(bp);
 620         return NULL;
 621 }
 622
 623 xfs_buf_t *
 624 xfs_buf_read_flags(
 625         xfs_buftarg_t           *target,
 626         xfs_off_t               ioff,
 627         size_t                  isize,
 628         xfs_buf_flags_t         flags)
 629 {
 630         xfs_buf_t               *bp;
 631
 632         flags |= XBF_READ;
 633
 634         bp = xfs_buf_get_flags(target, ioff, isize, flags);
 635         if (bp) {
 636                 if (!XFS_BUF_ISDONE(bp)) {
 637                         XB_TRACE(bp, "read", (unsigned long)flags);
 638                         XFS_STATS_INC(xb_get_read);
 639                         xfs_buf_iostart(bp, flags);
 640                 } else if (flags & XBF_ASYNC) {
 641                         XB_TRACE(bp, "read_async", (unsigned long)flags);
 642                         /*
 643                          * Read ahead call which is already satisfied,
 644                          * drop the buffer
 645                          */
 646                         goto no_buffer;
 647                 } else {
 648                         XB_TRACE(bp, "read_done", (unsigned long)flags);
 649                         /* We do not want read in the flags */
 650                         bp->b_flags &= ~XBF_READ;
 651                 }
 652         }
 653
 654         return bp;
 655
 656  no_buffer:
 657         if (flags & (XBF_LOCK | XBF_TRYLOCK))
 658                 xfs_buf_unlock(bp);
 659         xfs_buf_rele(bp);
 660         return NULL;
 661 }
 662
 663 /*
 664  *      If we are not low on memory then do the readahead in a deadlock
 665  *      safe manner.
 666  */
 667 void
 668 xfs_buf_readahead(
 669         xfs_buftarg_t           *target,
 670         xfs_off_t               ioff,
 671         size_t                  isize,
 672         xfs_buf_flags_t         flags)
 673 {
 674         struct backing_dev_info *bdi;
 675
 676         bdi = target->bt_mapping->backing_dev_info;
 677         if (bdi_read_congested(bdi))
 678                 return;
 679
 680         flags |= (XBF_TRYLOCK|XBF_ASYNC|XBF_READ_AHEAD);
 681         xfs_buf_read_flags(target, ioff, isize, flags);
 682 }
 683
 684 xfs_buf_t *
 685 xfs_buf_get_empty(
 686         size_t                  len,
 687         xfs_buftarg_t           *target)
 688 {
 689         xfs_buf_t               *bp;
 690
 691         bp = xfs_buf_allocate(0);
 692         if (bp)
 693                 _xfs_buf_initialize(bp, target, 0, len, 0);
 694         return bp;
 695 }
 696
 697 static inline struct page *
 698 mem_to_page(
 699         void                    *addr)
 700 {
 701         if (((unsigned long)addr < VMALLOC_START) ||
 702             ((unsigned long)addr >= VMALLOC_END)) {
 703                 return virt_to_page(addr);
 704         } else {
 705                 return vmalloc_to_page(addr);
 706         }
 707 }
 708
 709 int
 710 xfs_buf_associate_memory(
 711         xfs_buf_t               *bp,
 712         void                    *mem,
 713         size_t                  len)
 714 {
 715         int                     rval;
 716         int                     i = 0;
 717         size_t                  ptr;
 718         size_t                  end, end_cur;
 719         off_t                   offset;
 720         int                     page_count;
 721
 722         page_count = PAGE_CACHE_ALIGN(len) >> PAGE_CACHE_SHIFT;
 723         offset = (off_t) mem - ((off_t)mem & PAGE_CACHE_MASK);
 724         if (offset && (len > PAGE_CACHE_SIZE))
 725                 page_count++;
 726
 727         /* Free any previous set of page pointers */
 728         if (bp->b_pages)
 729                 _xfs_buf_free_pages(bp);
 730
 731         bp->b_pages = NULL;
 732         bp->b_addr = mem;
 733
 734         rval = _xfs_buf_get_pages(bp, page_count, 0);
 735         if (rval)
 736                 return rval;
 737
 738         bp->b_offset = offset;
 739         ptr = (size_t) mem & PAGE_CACHE_MASK;
 740         end = PAGE_CACHE_ALIGN((size_t) mem + len);
 741         end_cur = end;
 742         /* set up first page */
 743         bp->b_pages[0] = mem_to_page(mem);
 744
 745         ptr += PAGE_CACHE_SIZE;
 746         bp->b_page_count = ++i;
 747         while (ptr < end) {
 748                 bp->b_pages[i] = mem_to_page((void *)ptr);
 749                 bp->b_page_count = ++i;
 750                 ptr += PAGE_CACHE_SIZE;
 751         }
 752         bp->b_locked = 0;
 753
 754         bp->b_count_desired = bp->b_buffer_length = len;
 755         bp->b_flags |= XBF_MAPPED;
 756
 757         return 0;
 758 }
 759
 760 xfs_buf_t *
 761 xfs_buf_get_noaddr(
 762         size_t                  len,
 763         xfs_buftarg_t           *target)
 764 {
 765         size_t                  malloc_len = len;
 766         xfs_buf_t               *bp;
 767         void                    *data;
 768         int                     error;
 769
 770         bp = xfs_buf_allocate(0);
 771         if (unlikely(bp == NULL))
 772                 goto fail;
 773         _xfs_buf_initialize(bp, target, 0, len, 0);
 774
 775  try_again:
 776         data = kmem_alloc(malloc_len, KM_SLEEP | KM_MAYFAIL | KM_LARGE);
 777         if (unlikely(data == NULL))
 778                 goto fail_free_buf;
 779
 780         /* check whether alignment matches.. */
 781         if ((__psunsigned_t)data !=
 782             ((__psunsigned_t)data & ~target->bt_smask)) {
 783                 /* .. else double the size and try again */
 784                 kmem_free(data, malloc_len);
 785                 malloc_len <<= 1;
 786                 goto try_again;
 787         }
 788
 789         error = xfs_buf_associate_memory(bp, data, len);
 790         if (error)
 791                 goto fail_free_mem;
 792         bp->b_flags |= _XBF_KMEM_ALLOC;
 793
 794         xfs_buf_unlock(bp);
 795
 796         XB_TRACE(bp, "no_daddr", data);
 797         return bp;
 798  fail_free_mem:
 799         kmem_free(data, malloc_len);
 800  fail_free_buf:
 801         xfs_buf_free(bp);
 802  fail:
 803         return NULL;
 804 }
 805
 806 /*
 807  *      Increment reference count on buffer, to hold the buffer concurrently
 808  *      with another thread which may release (free) the buffer asynchronously.
 809  *      Must hold the buffer already to call this function.
 810  */
 811 void
 812 xfs_buf_hold(
 813         xfs_buf_t               *bp)
 814 {
 815         atomic_inc(&bp->b_hold);
 816         XB_TRACE(bp, "hold", 0);
 817 }
 818
 819 /*
 820  *      Releases a hold on the specified buffer.  If the
 821  *      the hold count is 1, calls xfs_buf_free.
 822  */
 823 void
 824 xfs_buf_rele(
 825         xfs_buf_t               *bp)
 826 {
 827         xfs_bufhash_t           *hash = bp->b_hash;
 828
 829         XB_TRACE(bp, "rele", bp->b_relse);
 830
 831         if (unlikely(!hash)) {
 832                 ASSERT(!bp->b_relse);
 833                 if (atomic_dec_and_test(&bp->b_hold))
 834                         xfs_buf_free(bp);
 835                 return;
 836         }
 837
 838         if (atomic_dec_and_lock(&bp->b_hold, &hash->bh_lock)) {
 839                 if (bp->b_relse) {
 840                         atomic_inc(&bp->b_hold);
 841                         spin_unlock(&hash->bh_lock);
 842                         (*(bp->b_relse)) (bp);
 843                 } else if (bp->b_flags & XBF_FS_MANAGED) {
 844                         spin_unlock(&hash->bh_lock);
 845                 } else {
 846                         ASSERT(!(bp->b_flags & (XBF_DELWRI|_XBF_DELWRI_Q)));
 847                         list_del_init(&bp->b_hash_list);
 848                         spin_unlock(&hash->bh_lock);
 849                         xfs_buf_free(bp);
 850                 }
 851         } else {
 852                 /*
 853                  * Catch reference count leaks
 854                  */
 855                 ASSERT(atomic_read(&bp->b_hold) >= 0);
 856         }
 857 }
 858
 859
 860 /*
 861  *      Mutual exclusion on buffers.  Locking model:
 862  *
 863  *      Buffers associated with inodes for which buffer locking
 864  *      is not enabled are not protected by semaphores, and are
 865  *      assumed to be exclusively owned by the caller.  There is a
 866  *      spinlock in the buffer, used by the caller when concurrent
 867  *      access is possible.
 868  */
 869
 870 /*
 871  *      Locks a buffer object, if it is not already locked.
 872  *      Note that this in no way locks the underlying pages, so it is only
 873  *      useful for synchronizing concurrent use of buffer objects, not for
 874  *      synchronizing independent access to the underlying pages.
 875  */
 876 int
 877 xfs_buf_cond_lock(
 878         xfs_buf_t               *bp)
 879 {
 880         int                     locked;
 881
 882         locked = down_trylock(&bp->b_sema) == 0;
 883         if (locked) {
 884                 XB_SET_OWNER(bp);
 885         }
 886         XB_TRACE(bp, "cond_lock", (long)locked);
 887         return locked ? 0 : -EBUSY;
 888 }
 889
 890 #if defined(DEBUG) || defined(XFS_BLI_TRACE)
 891 int
 892 xfs_buf_lock_value(
 893         xfs_buf_t               *bp)
 894 {
 895         return atomic_read(&bp->b_sema.count);
 896 }
 897 #endif
 898
 899 /*
 900  *      Locks a buffer object.
 901  *      Note that this in no way locks the underlying pages, so it is only
 902  *      useful for synchronizing concurrent use of buffer objects, not for
 903  *      synchronizing independent access to the underlying pages.
 904  */
 905 void
 906 xfs_buf_lock(
 907         xfs_buf_t               *bp)
 908 {
 909         XB_TRACE(bp, "lock", 0);
 910         if (atomic_read(&bp->b_io_remaining))
 911                 blk_run_address_space(bp->b_target->bt_mapping);
 912         down(&bp->b_sema);
 913         XB_SET_OWNER(bp);
 914         XB_TRACE(bp, "locked", 0);
 915 }
 916
 917 /*
 918  *      Releases the lock on the buffer object.
 919  *      If the buffer is marked delwri but is not queued, do so before we
 920  *      unlock the buffer as we need to set flags correctly.  We also need to
 921  *      take a reference for the delwri queue because the unlocker is going to
 922  *      drop their's and they don't know we just queued it.
 923  */
 924 void
 925 xfs_buf_unlock(
 926         xfs_buf_t               *bp)
 927 {
 928         if ((bp->b_flags & (XBF_DELWRI|_XBF_DELWRI_Q)) == XBF_DELWRI) {
 929                 atomic_inc(&bp->b_hold);
 930                 bp->b_flags |= XBF_ASYNC;
 931                 xfs_buf_delwri_queue(bp, 0);
 932         }
 933
 934         XB_CLEAR_OWNER(bp);
 935         up(&bp->b_sema);
 936         XB_TRACE(bp, "unlock", 0);
 937 }
 938
 939
 940 /*
 941  *      Pinning Buffer Storage in Memory
 942  *      Ensure that no attempt to force a buffer to disk will succeed.
 943  */
 944 void
 945 xfs_buf_pin(
 946         xfs_buf_t               *bp)
 947 {
 948         atomic_inc(&bp->b_pin_count);
 949         XB_TRACE(bp, "pin", (long)bp->b_pin_count.counter);
 950 }
 951
 952 void
 953 xfs_buf_unpin(
 954         xfs_buf_t               *bp)
 955 {
 956         if (atomic_dec_and_test(&bp->b_pin_count))
 957                 wake_up_all(&bp->b_waiters);
 958         XB_TRACE(bp, "unpin", (long)bp->b_pin_count.counter);
 959 }
 960
 961 int
 962 xfs_buf_ispin(
 963         xfs_buf_t               *bp)
 964 {
 965         return atomic_read(&bp->b_pin_count);
 966 }
 967
 968 STATIC void
 969 xfs_buf_wait_unpin(
 970         xfs_buf_t               *bp)
 971 {
 972         DECLARE_WAITQUEUE       (wait, current);
 973
 974         if (atomic_read(&bp->b_pin_count) == 0)
 975                 return;
 976
 977         add_wait_queue(&bp->b_waiters, &wait);
 978         for (;;) {
 979                 set_current_state(TASK_UNINTERRUPTIBLE);
 980                 if (atomic_read(&bp->b_pin_count) == 0)
 981                         break;
 982                 if (atomic_read(&bp->b_io_remaining))
 983                         blk_run_address_space(bp->b_target->bt_mapping);
 984                 schedule();
 985         }
 986         remove_wait_queue(&bp->b_waiters, &wait);
 987         set_current_state(TASK_RUNNING);
 988 }
 989
 990 /*
 991  *      Buffer Utility Routines
 992  */
 993
 994 STATIC void
 995 xfs_buf_iodone_work(
 996         void                    *v)
 997 {
 998         xfs_buf_t               *bp = (xfs_buf_t *)v;
 999
1000         if (bp->b_iodone)
1001                 (*(bp->b_iodone))(bp);
1002         else if (bp->b_flags & XBF_ASYNC)
1003                 xfs_buf_relse(bp);
1004 }
1005
1006 void
1007 xfs_buf_ioend(
1008         xfs_buf_t               *bp,
1009         int                     schedule)
1010 {
1011         bp->b_flags &= ~(XBF_READ | XBF_WRITE);
1012         if (bp->b_error == 0)
1013                 bp->b_flags |= XBF_DONE;
1014
1015         XB_TRACE(bp, "iodone", bp->b_iodone);
1016
1017         if ((bp->b_iodone) || (bp->b_flags & XBF_ASYNC)) {
1018                 if (schedule) {
1019                         INIT_WORK(&bp->b_iodone_work, xfs_buf_iodone_work, bp);
1020                         queue_work(xfslogd_workqueue, &bp->b_iodone_work);
1021                 } else {
1022                         xfs_buf_iodone_work(bp);
1023                 }
1024         } else {
1025                 up(&bp->b_iodonesema);
1026         }
1027 }
1028
1029 void
1030 xfs_buf_ioerror(
1031         xfs_buf_t               *bp,
1032         int                     error)
1033 {
1034         ASSERT(error >= 0 && error <= 0xffff);
1035         bp->b_error = (unsigned short)error;
1036         XB_TRACE(bp, "ioerror", (unsigned long)error);
1037 }
1038
1039 /*
1040  *      Initiate I/O on a buffer, based on the flags supplied.
1041  *      The b_iodone routine in the buffer supplied will only be called
1042  *      when all of the subsidiary I/O requests, if any, have been completed.
1043  */
1044 int
1045 xfs_buf_iostart(
1046         xfs_buf_t               *bp,
1047         xfs_buf_flags_t         flags)
1048 {
1049         int                     status = 0;
1050
1051         XB_TRACE(bp, "iostart", (unsigned long)flags);
1052
1053         if (flags & XBF_DELWRI) {
1054                 bp->b_flags &= ~(XBF_READ | XBF_WRITE | XBF_ASYNC);
1055                 bp->b_flags |= flags & (XBF_DELWRI | XBF_ASYNC);
1056                 xfs_buf_delwri_queue(bp, 1);
1057                 return status;
1058         }
1059
1060         bp->b_flags &= ~(XBF_READ | XBF_WRITE | XBF_ASYNC | XBF_DELWRI | \
1061                         XBF_READ_AHEAD | _XBF_RUN_QUEUES);
1062         bp->b_flags |= flags & (XBF_READ | XBF_WRITE | XBF_ASYNC | \
1063                         XBF_READ_AHEAD | _XBF_RUN_QUEUES);
1064
1065         BUG_ON(bp->b_bn == XFS_BUF_DADDR_NULL);
1066
1067         /* For writes allow an alternate strategy routine to precede
1068          * the actual I/O request (which may not be issued at all in
1069          * a shutdown situation, for example).
1070          */
1071         status = (flags & XBF_WRITE) ?
1072                 xfs_buf_iostrategy(bp) : xfs_buf_iorequest(bp);
1073
1074         /* Wait for I/O if we are not an async request.
1075          * Note: async I/O request completion will release the buffer,
1076          * and that can already be done by this point.  So using the
1077          * buffer pointer from here on, after async I/O, is invalid.
1078          */
1079         if (!status && !(flags & XBF_ASYNC))
1080                 status = xfs_buf_iowait(bp);
1081
1082         return status;
1083 }
1084
1085 STATIC __inline__ int
1086 _xfs_buf_iolocked(
1087         xfs_buf_t               *bp)
1088 {
1089         ASSERT(bp->b_flags & (XBF_READ | XBF_WRITE));
1090         if (bp->b_flags & XBF_READ)
1091                 return bp->b_locked;
1092         return 0;
1093 }
1094
1095 STATIC __inline__ void
1096 _xfs_buf_ioend(
1097         xfs_buf_t               *bp,
1098         int                     schedule)
1099 {
1100         if (atomic_dec_and_test(&bp->b_io_remaining) == 1) {
1101                 bp->b_locked = 0;
1102                 xfs_buf_ioend(bp, schedule);
1103         }
1104 }
1105
1106 STATIC int
1107 xfs_buf_bio_end_io(
1108         struct bio              *bio,
1109         unsigned int            bytes_done,
1110         int                     error)
1111 {
1112         xfs_buf_t               *bp = (xfs_buf_t *)bio->bi_private;
1113         unsigned int            blocksize = bp->b_target->bt_bsize;
1114         struct bio_vec          *bvec = bio->bi_io_vec + bio->bi_vcnt - 1;
1115
1116         if (bio->bi_size)
1117                 return 1;
1118
1119         if (!test_bit(BIO_UPTODATE, &bio->bi_flags))
1120                 bp->b_error = EIO;
1121
1122         do {
1123                 struct page     *page = bvec->bv_page;
1124
1125                 ASSERT(!PagePrivate(page));
1126                 if (unlikely(bp->b_error)) {
1127                         if (bp->b_flags & XBF_READ)
1128                                 ClearPageUptodate(page);
1129                 } else if (blocksize >= PAGE_CACHE_SIZE) {
1130                         SetPageUptodate(page);
1131                 } else if (!PagePrivate(page) &&
1132                                 (bp->b_flags & _XBF_PAGE_CACHE)) {
1133                         set_page_region(page, bvec->bv_offset, bvec->bv_len);
1134                 }
1135
1136                 if (--bvec >= bio->bi_io_vec)
1137                         prefetchw(&bvec->bv_page->flags);
1138
1139                 if (_xfs_buf_iolocked(bp)) {
1140                         unlock_page(page);
1141                 }
1142         } while (bvec >= bio->bi_io_vec);
1143
1144         _xfs_buf_ioend(bp, 1);
1145         bio_put(bio);
1146         return 0;
1147 }
1148
1149 STATIC void
1150 _xfs_buf_ioapply(
1151         xfs_buf_t               *bp)
1152 {
1153         int                     i, rw, map_i, total_nr_pages, nr_pages;
1154         struct bio              *bio;
1155         int                     offset = bp->b_offset;
1156         int                     size = bp->b_count_desired;
1157         sector_t                sector = bp->b_bn;
1158         unsigned int            blocksize = bp->b_target->bt_bsize;
1159         int                     locking = _xfs_buf_iolocked(bp);
1160
1161         total_nr_pages = bp->b_page_count;
1162         map_i = 0;
1163
1164         if (bp->b_flags & XBF_ORDERED) {
1165                 ASSERT(!(bp->b_flags & XBF_READ));
1166                 rw = WRITE_BARRIER;
1167         } else if (bp->b_flags & _XBF_RUN_QUEUES) {
1168                 ASSERT(!(bp->b_flags & XBF_READ_AHEAD));
1169                 bp->b_flags &= ~_XBF_RUN_QUEUES;
1170                 rw = (bp->b_flags & XBF_WRITE) ? WRITE_SYNC : READ_SYNC;
1171         } else {
1172                 rw = (bp->b_flags & XBF_WRITE) ? WRITE :
1173                      (bp->b_flags & XBF_READ_AHEAD) ? READA : READ;
1174         }
1175
1176         /* Special code path for reading a sub page size buffer in --
1177          * we populate up the whole page, and hence the other metadata
1178          * in the same page.  This optimization is only valid when the
1179          * filesystem block size is not smaller than the page size.
1180          */
1181         if ((bp->b_buffer_length < PAGE_CACHE_SIZE) &&
1182             (bp->b_flags & XBF_READ) && locking &&
1183             (blocksize >= PAGE_CACHE_SIZE)) {
1184                 bio = bio_alloc(GFP_NOIO, 1);
1185
1186                 bio->bi_bdev = bp->b_target->bt_bdev;
1187                 bio->bi_sector = sector - (offset >> BBSHIFT);
1188                 bio->bi_end_io = xfs_buf_bio_end_io;
1189                 bio->bi_private = bp;
1190
1191                 bio_add_page(bio, bp->b_pages[0], PAGE_CACHE_SIZE, 0);
1192                 size = 0;
1193
1194                 atomic_inc(&bp->b_io_remaining);
1195
1196                 goto submit_io;
1197         }
1198
1199         /* Lock down the pages which we need to for the request */
1200         if (locking && (bp->b_flags & XBF_WRITE) && (bp->b_locked == 0)) {
1201                 for (i = 0; size; i++) {
1202                         int             nbytes = PAGE_CACHE_SIZE - offset;
1203                         struct page     *page = bp->b_pages[i];
1204
1205                         if (nbytes > size)
1206                                 nbytes = size;
1207
1208                         lock_page(page);
1209
1210                         size -= nbytes;
1211                         offset = 0;
1212                 }
1213                 offset = bp->b_offset;
1214                 size = bp->b_count_desired;
1215         }
1216
1217 next_chunk:
1218         atomic_inc(&bp->b_io_remaining);
1219         nr_pages = BIO_MAX_SECTORS >> (PAGE_SHIFT - BBSHIFT);
1220         if (nr_pages > total_nr_pages)
1221                 nr_pages = total_nr_pages;
1222
1223         bio = bio_alloc(GFP_NOIO, nr_pages);
1224         bio->bi_bdev = bp->b_target->bt_bdev;
1225         bio->bi_sector = sector;
1226         bio->bi_end_io = xfs_buf_bio_end_io;
1227         bio->bi_private = bp;
1228
1229         for (; size && nr_pages; nr_pages--, map_i++) {
1230                 int     rbytes, nbytes = PAGE_CACHE_SIZE - offset;
1231
1232                 if (nbytes > size)
1233                         nbytes = size;
1234
1235                 rbytes = bio_add_page(bio, bp->b_pages[map_i], nbytes, offset);
1236                 if (rbytes < nbytes)
1237                         break;
1238
1239                 offset = 0;
1240                 sector += nbytes >> BBSHIFT;
1241                 size -= nbytes;
1242                 total_nr_pages--;
1243         }
1244
1245 submit_io:
1246         if (likely(bio->bi_size)) {
1247                 submit_bio(rw, bio);
1248                 if (size)
1249                         goto next_chunk;
1250         } else {
1251                 bio_put(bio);
1252                 xfs_buf_ioerror(bp, EIO);
1253         }
1254 }
1255
1256 int
1257 xfs_buf_iorequest(
1258         xfs_buf_t               *bp)
1259 {
1260         XB_TRACE(bp, "iorequest", 0);
1261
1262         if (bp->b_flags & XBF_DELWRI) {
1263                 xfs_buf_delwri_queue(bp, 1);
1264                 return 0;
1265         }
1266
1267         if (bp->b_flags & XBF_WRITE) {
1268                 xfs_buf_wait_unpin(bp);
1269         }
1270
1271         xfs_buf_hold(bp);
1272
1273         /* Set the count to 1 initially, this will stop an I/O
1274          * completion callout which happens before we have started
1275          * all the I/O from calling xfs_buf_ioend too early.
1276          */
1277         atomic_set(&bp->b_io_remaining, 1);
1278         _xfs_buf_ioapply(bp);
1279         _xfs_buf_ioend(bp, 0);
1280
1281         xfs_buf_rele(bp);
1282         return 0;
1283 }
1284
1285 /*
1286  *      Waits for I/O to complete on the buffer supplied.
1287  *      It returns immediately if no I/O is pending.
1288  *      It returns the I/O error code, if any, or 0 if there was no error.
1289  */
1290 int
1291 xfs_buf_iowait(
1292         xfs_buf_t               *bp)
1293 {
1294         XB_TRACE(bp, "iowait", 0);
1295         if (atomic_read(&bp->b_io_remaining))
1296                 blk_run_address_space(bp->b_target->bt_mapping);
1297         down(&bp->b_iodonesema);
1298         XB_TRACE(bp, "iowaited", (long)bp->b_error);
1299         return bp->b_error;
1300 }
1301
1302 xfs_caddr_t
1303 xfs_buf_offset(
1304         xfs_buf_t               *bp,
1305         size_t                  offset)
1306 {
1307         struct page             *page;
1308
1309         if (bp->b_flags & XBF_MAPPED)
1310                 return XFS_BUF_PTR(bp) + offset;
1311
1312         offset += bp->b_offset;
1313         page = bp->b_pages[offset >> PAGE_CACHE_SHIFT];
1314         return (xfs_caddr_t)page_address(page) + (offset & (PAGE_CACHE_SIZE-1));
1315 }
1316
1317 /*
1318  *      Move data into or out of a buffer.
1319  */
1320 void
1321 xfs_buf_iomove(
1322         xfs_buf_t               *bp,    /* buffer to process            */
1323         size_t                  boff,   /* starting buffer offset       */
1324         size_t                  bsize,  /* length to copy               */
1325         caddr_t                 data,   /* data address                 */
1326         xfs_buf_rw_t            mode)   /* read/write/zero flag         */
1327 {
1328         size_t                  bend, cpoff, csize;
1329         struct page             *page;
1330
1331         bend = boff + bsize;
1332         while (boff < bend) {
1333                 page = bp->b_pages[xfs_buf_btoct(boff + bp->b_offset)];
1334                 cpoff = xfs_buf_poff(boff + bp->b_offset);
1335                 csize = min_t(size_t,
1336                               PAGE_CACHE_SIZE-cpoff, bp->b_count_desired-boff);
1337
1338                 ASSERT(((csize + cpoff) <= PAGE_CACHE_SIZE));
1339
1340                 switch (mode) {
1341                 case XBRW_ZERO:
1342                         memset(page_address(page) + cpoff, 0, csize);
1343                         break;
1344                 case XBRW_READ:
1345                         memcpy(data, page_address(page) + cpoff, csize);
1346                         break;
1347                 case XBRW_WRITE:
1348                         memcpy(page_address(page) + cpoff, data, csize);
1349                 }
1350
1351                 boff += csize;
1352                 data += csize;
1353         }
1354 }
1355
1356 /*
1357  *      Handling of buffer targets (buftargs).
1358  */
1359
1360 /*
1361  *      Wait for any bufs with callbacks that have been submitted but
1362  *      have not yet returned... walk the hash list for the target.
1363  */
1364 void
1365 xfs_wait_buftarg(
1366         xfs_buftarg_t   *btp)
1367 {
1368         xfs_buf_t       *bp, *n;
1369         xfs_bufhash_t   *hash;
1370         uint            i;
1371
1372         for (i = 0; i < (1 << btp->bt_hashshift); i++) {
1373                 hash = &btp->bt_hash[i];
1374 again:
1375                 spin_lock(&hash->bh_lock);
1376                 list_for_each_entry_safe(bp, n, &hash->bh_list, b_hash_list) {
1377                         ASSERT(btp == bp->b_target);
1378                         if (!(bp->b_flags & XBF_FS_MANAGED)) {
1379                                 spin_unlock(&hash->bh_lock);
1380                                 /*
1381                                  * Catch superblock reference count leaks
1382                                  * immediately
1383                                  */
1384                                 BUG_ON(bp->b_bn == 0);
1385                                 delay(100);
1386                                 goto again;
1387                         }
1388                 }
1389                 spin_unlock(&hash->bh_lock);
1390         }
1391 }
1392
1393 /*
1394  *      Allocate buffer hash table for a given target.
1395  *      For devices containing metadata (i.e. not the log/realtime devices)
1396  *      we need to allocate a much larger hash table.
1397  */
1398 STATIC void
1399 xfs_alloc_bufhash(
1400         xfs_buftarg_t           *btp,
1401         int                     external)
1402 {
1403         unsigned int            i;
1404
1405         btp->bt_hashshift = external ? 3 : 8;   /* 8 or 256 buckets */
1406         btp->bt_hashmask = (1 << btp->bt_hashshift) - 1;
1407         btp->bt_hash = kmem_zalloc((1 << btp->bt_hashshift) *
1408                                         sizeof(xfs_bufhash_t), KM_SLEEP);
1409         for (i = 0; i < (1 << btp->bt_hashshift); i++) {
1410                 spin_lock_init(&btp->bt_hash[i].bh_lock);
1411                 INIT_LIST_HEAD(&btp->bt_hash[i].bh_list);
1412         }
1413 }
1414
1415 STATIC void
1416 xfs_free_bufhash(
1417         xfs_buftarg_t           *btp)
1418 {
1419         kmem_free(btp->bt_hash, (1<<btp->bt_hashshift) * sizeof(xfs_bufhash_t));
1420         btp->bt_hash = NULL;
1421 }
1422
1423 /*
1424  *      buftarg list for delwrite queue processing
1425  */
1426 STATIC LIST_HEAD(xfs_buftarg_list);
1427 STATIC DEFINE_SPINLOCK(xfs_buftarg_lock);
1428
1429 STATIC void
1430 xfs_register_buftarg(
1431         xfs_buftarg_t           *btp)
1432 {
1433         spin_lock(&xfs_buftarg_lock);
1434         list_add(&btp->bt_list, &xfs_buftarg_list);
1435         spin_unlock(&xfs_buftarg_lock);
1436 }
1437
1438 STATIC void
1439 xfs_unregister_buftarg(
1440         xfs_buftarg_t           *btp)
1441 {
1442         spin_lock(&xfs_buftarg_lock);
1443         list_del(&btp->bt_list);
1444         spin_unlock(&xfs_buftarg_lock);
1445 }
1446
1447 void
1448 xfs_free_buftarg(
1449         xfs_buftarg_t           *btp,
1450         int                     external)
1451 {
1452         xfs_flush_buftarg(btp, 1);
1453         if (external)
1454                 xfs_blkdev_put(btp->bt_bdev);
1455         xfs_free_bufhash(btp);
1456         iput(btp->bt_mapping->host);
1457
1458         /* Unregister the buftarg first so that we don't get a
1459          * wakeup finding a non-existent task
1460          */
1461         xfs_unregister_buftarg(btp);
1462         kthread_stop(btp->bt_task);
1463
1464         kmem_free(btp, sizeof(*btp));
1465 }
1466
1467 STATIC int
1468 xfs_setsize_buftarg_flags(
1469         xfs_buftarg_t           *btp,
1470         unsigned int            blocksize,
1471         unsigned int            sectorsize,
1472         int                     verbose)
1473 {
1474         btp->bt_bsize = blocksize;
1475         btp->bt_sshift = ffs(sectorsize) - 1;
1476         btp->bt_smask = sectorsize - 1;
1477
1478         if (set_blocksize(btp->bt_bdev, sectorsize)) {
1479                 printk(KERN_WARNING
1480                         "XFS: Cannot set_blocksize to %u on device %s\n",
1481                         sectorsize, XFS_BUFTARG_NAME(btp));
1482                 return EINVAL;
1483         }
1484
1485         if (verbose &&
1486             (PAGE_CACHE_SIZE / BITS_PER_LONG) > sectorsize) {
1487                 printk(KERN_WARNING
1488                         "XFS: %u byte sectors in use on device %s.  "
1489                         "This is suboptimal; %u or greater is ideal.\n",
1490                         sectorsize, XFS_BUFTARG_NAME(btp),
1491                         (unsigned int)PAGE_CACHE_SIZE / BITS_PER_LONG);
1492         }
1493
1494         return 0;
1495 }
1496
1497 /*
1498  *      When allocating the initial buffer target we have not yet
1499  *      read in the superblock, so don't know what sized sectors
1500  *      are being used is at this early stage.  Play safe.
1501  */
1502 STATIC int
1503 xfs_setsize_buftarg_early(
1504         xfs_buftarg_t           *btp,
1505         struct block_device     *bdev)
1506 {
1507         return xfs_setsize_buftarg_flags(btp,
1508                         PAGE_CACHE_SIZE, bdev_hardsect_size(bdev), 0);
1509 }
1510
1511 int
1512 xfs_setsize_buftarg(
1513         xfs_buftarg_t           *btp,
1514         unsigned int            blocksize,
1515         unsigned int            sectorsize)
1516 {
1517         return xfs_setsize_buftarg_flags(btp, blocksize, sectorsize, 1);
1518 }
1519
1520 STATIC int
1521 xfs_mapping_buftarg(
1522         xfs_buftarg_t           *btp,
1523         struct block_device     *bdev)
1524 {
1525         struct backing_dev_info *bdi;
1526         struct inode            *inode;
1527         struct address_space    *mapping;
1528         static const struct address_space_operations mapping_aops = {
1529                 .sync_page = block_sync_page,
1530                 .migratepage = fail_migrate_page,
1531         };
1532
1533         inode = new_inode(bdev->bd_inode->i_sb);
1534         if (!inode) {
1535                 printk(KERN_WARNING
1536                         "XFS: Cannot allocate mapping inode for device %s\n",
1537                         XFS_BUFTARG_NAME(btp));
1538                 return ENOMEM;
1539         }
1540         inode->i_mode = S_IFBLK;
1541         inode->i_bdev = bdev;
1542         inode->i_rdev = bdev->bd_dev;
1543         bdi = blk_get_backing_dev_info(bdev);
1544         if (!bdi)
1545                 bdi = &default_backing_dev_info;
1546         mapping = &inode->i_data;
1547         mapping->a_ops = &mapping_aops;
1548         mapping->backing_dev_info = bdi;
1549         mapping_set_gfp_mask(mapping, GFP_NOFS);
1550         btp->bt_mapping = mapping;
1551         return 0;
1552 }
1553
1554 STATIC int
1555 xfs_alloc_delwrite_queue(
1556         xfs_buftarg_t           *btp)
1557 {
1558         int     error = 0;
1559
1560         INIT_LIST_HEAD(&btp->bt_list);
1561         INIT_LIST_HEAD(&btp->bt_delwrite_queue);
1562         spinlock_init(&btp->bt_delwrite_lock, "delwri_lock");
1563         btp->bt_flags = 0;
1564         btp->bt_task = kthread_run(xfsbufd, btp, "xfsbufd");
1565         if (IS_ERR(btp->bt_task)) {
1566                 error = PTR_ERR(btp->bt_task);
1567                 goto out_error;
1568         }
1569         xfs_register_buftarg(btp);
1570 out_error:
1571         return error;
1572 }
1573
1574 xfs_buftarg_t *
1575 xfs_alloc_buftarg(
1576         struct block_device     *bdev,
1577         int                     external)
1578 {
1579         xfs_buftarg_t           *btp;
1580
1581         btp = kmem_zalloc(sizeof(*btp), KM_SLEEP);
1582
1583         btp->bt_dev =  bdev->bd_dev;
1584         btp->bt_bdev = bdev;
1585         if (xfs_setsize_buftarg_early(btp, bdev))
1586                 goto error;
1587         if (xfs_mapping_buftarg(btp, bdev))
1588                 goto error;
1589         if (xfs_alloc_delwrite_queue(btp))
1590                 goto error;
1591         xfs_alloc_bufhash(btp, external);
1592         return btp;
1593
1594 error:
1595         kmem_free(btp, sizeof(*btp));
1596         return NULL;
1597 }
1598
1599
1600 /*
1601  *      Delayed write buffer handling
1602  */
1603 STATIC void
1604 xfs_buf_delwri_queue(
1605         xfs_buf_t               *bp,
1606         int                     unlock)
1607 {
1608         struct list_head        *dwq = &bp->b_target->bt_delwrite_queue;
1609         spinlock_t              *dwlk = &bp->b_target->bt_delwrite_lock;
1610
1611         XB_TRACE(bp, "delwri_q", (long)unlock);
1612         ASSERT((bp->b_flags&(XBF_DELWRI|XBF_ASYNC)) == (XBF_DELWRI|XBF_ASYNC));
1613
1614         spin_lock(dwlk);
1615         /* If already in the queue, dequeue and place at tail */
1616         if (!list_empty(&bp->b_list)) {
1617                 ASSERT(bp->b_flags & _XBF_DELWRI_Q);
1618                 if (unlock)
1619                         atomic_dec(&bp->b_hold);
1620                 list_del(&bp->b_list);
1621         }
1622
1623         bp->b_flags |= _XBF_DELWRI_Q;
1624         list_add_tail(&bp->b_list, dwq);
1625         bp->b_queuetime = jiffies;
1626         spin_unlock(dwlk);
1627
1628         if (unlock)
1629                 xfs_buf_unlock(bp);
1630 }
1631
1632 void
1633 xfs_buf_delwri_dequeue(
1634         xfs_buf_t               *bp)
1635 {
1636         spinlock_t              *dwlk = &bp->b_target->bt_delwrite_lock;
1637         int                     dequeued = 0;
1638
1639         spin_lock(dwlk);
1640         if ((bp->b_flags & XBF_DELWRI) && !list_empty(&bp->b_list)) {
1641                 ASSERT(bp->b_flags & _XBF_DELWRI_Q);
1642                 list_del_init(&bp->b_list);
1643                 dequeued = 1;
1644         }
1645         bp->b_flags &= ~(XBF_DELWRI|_XBF_DELWRI_Q);
1646         spin_unlock(dwlk);
1647
1648         if (dequeued)
1649                 xfs_buf_rele(bp);
1650
1651         XB_TRACE(bp, "delwri_dq", (long)dequeued);
1652 }
1653
1654 STATIC void
1655 xfs_buf_runall_queues(
1656         struct workqueue_struct *queue)
1657 {
1658         flush_workqueue(queue);
1659 }
1660
1661 STATIC int
1662 xfsbufd_wakeup(
1663         int                     priority,
1664         gfp_t                   mask)
1665 {
1666         xfs_buftarg_t           *btp;
1667
1668         spin_lock(&xfs_buftarg_lock);
1669         list_for_each_entry(btp, &xfs_buftarg_list, bt_list) {
1670                 if (test_bit(XBT_FORCE_SLEEP, &btp->bt_flags))
1671                         continue;
1672                 set_bit(XBT_FORCE_FLUSH, &btp->bt_flags);
1673                 wake_up_process(btp->bt_task);
1674         }
1675         spin_unlock(&xfs_buftarg_lock);
1676         return 0;
1677 }
1678
1679 STATIC int
1680 xfsbufd(
1681         void                    *data)
1682 {
1683         struct list_head        tmp;
1684         unsigned long           age;
1685         xfs_buftarg_t           *target = (xfs_buftarg_t *)data;
1686         xfs_buf_t               *bp, *n;
1687         struct list_head        *dwq = &target->bt_delwrite_queue;
1688         spinlock_t              *dwlk = &target->bt_delwrite_lock;
1689         int                     count;
1690
1691         current->flags |= PF_MEMALLOC;
1692
1693         INIT_LIST_HEAD(&tmp);
1694         do {
1695                 if (unlikely(freezing(current))) {
1696                         set_bit(XBT_FORCE_SLEEP, &target->bt_flags);
1697                         refrigerator();
1698                 } else {
1699                         clear_bit(XBT_FORCE_SLEEP, &target->bt_flags);
1700                 }
1701
1702                 schedule_timeout_interruptible(
1703                         xfs_buf_timer_centisecs * msecs_to_jiffies(10));
1704
1705                 count = 0;
1706                 age = xfs_buf_age_centisecs * msecs_to_jiffies(10);
1707                 spin_lock(dwlk);
1708                 list_for_each_entry_safe(bp, n, dwq, b_list) {
1709                         XB_TRACE(bp, "walkq1", (long)xfs_buf_ispin(bp));
1710                         ASSERT(bp->b_flags & XBF_DELWRI);
1711
1712                         if (!xfs_buf_ispin(bp) && !xfs_buf_cond_lock(bp)) {
1713                                 if (!test_bit(XBT_FORCE_FLUSH,
1714                                                 &target->bt_flags) &&
1715                                     time_before(jiffies,
1716                                                 bp->b_queuetime + age)) {
1717                                         xfs_buf_unlock(bp);
1718                                         break;
1719                                 }
1720
1721                                 bp->b_flags &= ~(XBF_DELWRI|_XBF_DELWRI_Q|
1722                                                  _XBF_RUN_QUEUES);
1723                                 bp->b_flags |= XBF_WRITE;
1724                                 list_move_tail(&bp->b_list, &tmp);
1725                                 count++;
1726                         }
1727                 }
1728                 spin_unlock(dwlk);
1729
1730                 while (!list_empty(&tmp)) {
1731                         bp = list_entry(tmp.next, xfs_buf_t, b_list);
1732                         ASSERT(target == bp->b_target);
1733
1734                         list_del_init(&bp->b_list);
1735                         xfs_buf_iostrategy(bp);
1736                 }
1737
1738                 if (as_list_len > 0)
1739                         purge_addresses();
1740                 if (count)
1741                         blk_run_address_space(target->bt_mapping);
1742
1743                 clear_bit(XBT_FORCE_FLUSH, &target->bt_flags);
1744         } while (!kthread_should_stop());
1745
1746         return 0;
1747 }
1748
1749 /*
1750  *      Go through all incore buffers, and release buffers if they belong to
1751  *      the given device. This is used in filesystem error handling to
1752  *      preserve the consistency of its metadata.
1753  */
1754 int
1755 xfs_flush_buftarg(
1756         xfs_buftarg_t           *target,
1757         int                     wait)
1758 {
1759         struct list_head        tmp;
1760         xfs_buf_t               *bp, *n;
1761         int                     pincount = 0;
1762         struct list_head        *dwq = &target->bt_delwrite_queue;
1763         spinlock_t              *dwlk = &target->bt_delwrite_lock;
1764
1765         xfs_buf_runall_queues(xfsdatad_workqueue);
1766         xfs_buf_runall_queues(xfslogd_workqueue);
1767
1768         INIT_LIST_HEAD(&tmp);
1769         spin_lock(dwlk);
1770         list_for_each_entry_safe(bp, n, dwq, b_list) {
1771                 ASSERT(bp->b_target == target);
1772                 ASSERT(bp->b_flags & (XBF_DELWRI | _XBF_DELWRI_Q));
1773                 XB_TRACE(bp, "walkq2", (long)xfs_buf_ispin(bp));
1774                 if (xfs_buf_ispin(bp)) {
1775                         pincount++;
1776                         continue;
1777                 }
1778
1779                 list_move_tail(&bp->b_list, &tmp);
1780         }
1781         spin_unlock(dwlk);
1782
1783         /*
1784          * Dropped the delayed write list lock, now walk the temporary list
1785          */
1786         list_for_each_entry_safe(bp, n, &tmp, b_list) {
1787                 xfs_buf_lock(bp);
1788                 bp->b_flags &= ~(XBF_DELWRI|_XBF_DELWRI_Q|_XBF_RUN_QUEUES);
1789                 bp->b_flags |= XBF_WRITE;
1790                 if (wait)
1791                         bp->b_flags &= ~XBF_ASYNC;
1792                 else
1793                         list_del_init(&bp->b_list);
1794
1795                 xfs_buf_iostrategy(bp);
1796         }
1797
1798         if (wait)
1799                 blk_run_address_space(target->bt_mapping);
1800
1801         /*
1802          * Remaining list items must be flushed before returning
1803          */
1804         while (!list_empty(&tmp)) {
1805                 bp = list_entry(tmp.next, xfs_buf_t, b_list);
1806
1807                 list_del_init(&bp->b_list);
1808                 xfs_iowait(bp);
1809                 xfs_buf_relse(bp);
1810         }
1811
1812         return pincount;
1813 }
1814
1815 int __init
1816 xfs_buf_init(void)
1817 {
1818 #ifdef XFS_BUF_TRACE
1819         xfs_buf_trace_buf = ktrace_alloc(XFS_BUF_TRACE_SIZE, KM_SLEEP);
1820 #endif
1821
1822         xfs_buf_zone = kmem_zone_init_flags(sizeof(xfs_buf_t), "xfs_buf",
1823                                                 KM_ZONE_HWALIGN, NULL);
1824         if (!xfs_buf_zone)
1825                 goto out_free_trace_buf;
1826
1827         xfslogd_workqueue = create_workqueue("xfslogd");
1828         if (!xfslogd_workqueue)
1829                 goto out_free_buf_zone;
1830
1831         xfsdatad_workqueue = create_workqueue("xfsdatad");
1832         if (!xfsdatad_workqueue)
1833                 goto out_destroy_xfslogd_workqueue;
1834
1835         xfs_buf_shake = kmem_shake_register(xfsbufd_wakeup);
1836         if (!xfs_buf_shake)
1837                 goto out_destroy_xfsdatad_workqueue;
1838
1839         return 0;
1840
1841  out_destroy_xfsdatad_workqueue:
1842         destroy_workqueue(xfsdatad_workqueue);
1843  out_destroy_xfslogd_workqueue:
1844         destroy_workqueue(xfslogd_workqueue);
1845  out_free_buf_zone:
1846         kmem_zone_destroy(xfs_buf_zone);
1847  out_free_trace_buf:
1848 #ifdef XFS_BUF_TRACE
1849         ktrace_free(xfs_buf_trace_buf);
1850 #endif
1851         return -ENOMEM;
1852 }
1853
1854 void
1855 xfs_buf_terminate(void)
1856 {
1857         kmem_shake_deregister(xfs_buf_shake);
1858         destroy_workqueue(xfsdatad_workqueue);
1859         destroy_workqueue(xfslogd_workqueue);
1860         kmem_zone_destroy(xfs_buf_zone);
1861 #ifdef XFS_BUF_TRACE
1862         ktrace_free(xfs_buf_trace_buf);
1863 #endif
1864 }