drivers/md/bitmap.c

   1 /*
   2  * bitmap.c two-level bitmap (C) Peter T. Breuer (ptb@ot.uc3m.es) 2003
   3  *
   4  * bitmap_create  - sets up the bitmap structure
   5  * bitmap_destroy - destroys the bitmap structure
   6  *
   7  * additions, Copyright (C) 2003-2004, Paul Clements, SteelEye Technology, Inc.:
   8  * - added disk storage for bitmap
   9  * - changes to allow various bitmap chunk sizes
  10  */
  11
  12 /*
  13  * Still to do:
  14  *
  15  * flush after percent set rather than just time based. (maybe both).
  16  */
  17
  18 #include <linux/blkdev.h>
  19 #include <linux/module.h>
  20 #include <linux/errno.h>
  21 #include <linux/slab.h>
  22 #include <linux/init.h>
  23 #include <linux/timer.h>
  24 #include <linux/sched.h>
  25 #include <linux/list.h>
  26 #include <linux/file.h>
  27 #include <linux/mount.h>
  28 #include <linux/buffer_head.h>
  29 #include <linux/seq_file.h>
  30 #include "md.h"
  31 #include "bitmap.h"
  32
  33 static inline char *bmname(struct bitmap *bitmap)
  34 {
  35         return bitmap->mddev ? mdname(bitmap->mddev) : "mdX";
  36 }
  37
  38 /*
  39  * check a page and, if necessary, allocate it (or hijack it if the alloc fails)
  40  *
  41  * 1) check to see if this page is allocated, if it's not then try to alloc
  42  * 2) if the alloc fails, set the page's hijacked flag so we'll use the
  43  *    page pointer directly as a counter
  44  *
  45  * if we find our page, we increment the page's refcount so that it stays
  46  * allocated while we're using it
  47  */
  48 static int bitmap_checkpage(struct bitmap_counts *bitmap,
  49                             unsigned long page, int create)
  50 __releases(bitmap->lock)
  51 __acquires(bitmap->lock)
  52 {
  53         unsigned char *mappage;
  54
  55         if (page >= bitmap->pages) {
  56                 /* This can happen if bitmap_start_sync goes beyond
  57                  * End-of-device while looking for a whole page.
  58                  * It is harmless.
  59                  */
  60                 return -EINVAL;
  61         }
  62
  63         if (bitmap->bp[page].hijacked) /* it's hijacked, don't try to alloc */
  64                 return 0;
  65
  66         if (bitmap->bp[page].map) /* page is already allocated, just return */
  67                 return 0;
  68
  69         if (!create)
  70                 return -ENOENT;
  71
  72         /* this page has not been allocated yet */
  73
  74         spin_unlock_irq(&bitmap->lock);
  75         /* It is possible that this is being called inside a
  76          * prepare_to_wait/finish_wait loop from raid5c:make_request().
  77          * In general it is not permitted to sleep in that context as it
  78          * can cause the loop to spin freely.
  79          * That doesn't apply here as we can only reach this point
  80          * once with any loop.
  81          * When this function completes, either bp[page].map or
  82          * bp[page].hijacked.  In either case, this function will
  83          * abort before getting to this point again.  So there is
  84          * no risk of a free-spin, and so it is safe to assert
  85          * that sleeping here is allowed.
  86          */
  87         sched_annotate_sleep();
  88         mappage = kzalloc(PAGE_SIZE, GFP_NOIO);
  89         spin_lock_irq(&bitmap->lock);
  90
  91         if (mappage == NULL) {
  92                 pr_debug("md/bitmap: map page allocation failed, hijacking\n");
  93                 /* failed - set the hijacked flag so that we can use the
  94                  * pointer as a counter */
  95                 if (!bitmap->bp[page].map)
  96                         bitmap->bp[page].hijacked = 1;
  97         } else if (bitmap->bp[page].map ||
  98                    bitmap->bp[page].hijacked) {
  99                 /* somebody beat us to getting the page */
 100                 kfree(mappage);
 101                 return 0;
 102         } else {
 103
 104                 /* no page was in place and we have one, so install it */
 105
 106                 bitmap->bp[page].map = mappage;
 107                 bitmap->missing_pages--;
 108         }
 109         return 0;
 110 }
 111
 112 /* if page is completely empty, put it back on the free list, or dealloc it */
 113 /* if page was hijacked, unmark the flag so it might get alloced next time */
 114 /* Note: lock should be held when calling this */
 115 static void bitmap_checkfree(struct bitmap_counts *bitmap, unsigned long page)
 116 {
 117         char *ptr;
 118
 119         if (bitmap->bp[page].count) /* page is still busy */
 120                 return;
 121
 122         /* page is no longer in use, it can be released */
 123
 124         if (bitmap->bp[page].hijacked) { /* page was hijacked, undo this now */
 125                 bitmap->bp[page].hijacked = 0;
 126                 bitmap->bp[page].map = NULL;
 127         } else {
 128                 /* normal case, free the page */
 129                 ptr = bitmap->bp[page].map;
 130                 bitmap->bp[page].map = NULL;
 131                 bitmap->missing_pages++;
 132                 kfree(ptr);
 133         }
 134 }
 135
 136 /*
 137  * bitmap file handling - read and write the bitmap file and its superblock
 138  */
 139
 140 /*
 141  * basic page I/O operations
 142  */
 143
 144 /* IO operations when bitmap is stored near all superblocks */
 145 static int read_sb_page(struct mddev *mddev, loff_t offset,
 146                         struct page *page,
 147                         unsigned long index, int size)
 148 {
 149         /* choose a good rdev and read the page from there */
 150
 151         struct md_rdev *rdev;
 152         sector_t target;
 153
 154         rdev_for_each(rdev, mddev) {
 155                 if (! test_bit(In_sync, &rdev->flags)
 156                     || test_bit(Faulty, &rdev->flags))
 157                         continue;
 158
 159                 target = offset + index * (PAGE_SIZE/512);
 160
 161                 if (sync_page_io(rdev, target,
 162                                  roundup(size, bdev_logical_block_size(rdev->bdev)),
 163                                  page, READ, true)) {
 164                         page->index = index;
 165                         return 0;
 166                 }
 167         }
 168         return -EIO;
 169 }
 170
 171 static struct md_rdev *next_active_rdev(struct md_rdev *rdev, struct mddev *mddev)
 172 {
 173         /* Iterate the disks of an mddev, using rcu to protect access to the
 174          * linked list, and raising the refcount of devices we return to ensure
 175          * they don't disappear while in use.
 176          * As devices are only added or removed when raid_disk is < 0 and
 177          * nr_pending is 0 and In_sync is clear, the entries we return will
 178          * still be in the same position on the list when we re-enter
 179          * list_for_each_entry_continue_rcu.
 180          *
 181          * Note that if entered with 'rdev == NULL' to start at the
 182          * beginning, we temporarily assign 'rdev' to an address which
 183          * isn't really an rdev, but which can be used by
 184          * list_for_each_entry_continue_rcu() to find the first entry.
 185          */
 186         rcu_read_lock();
 187         if (rdev == NULL)
 188                 /* start at the beginning */
 189                 rdev = list_entry(&mddev->disks, struct md_rdev, same_set);
 190         else {
 191                 /* release the previous rdev and start from there. */
 192                 rdev_dec_pending(rdev, mddev);
 193         }
 194         list_for_each_entry_continue_rcu(rdev, &mddev->disks, same_set) {
 195                 if (rdev->raid_disk >= 0 &&
 196                     !test_bit(Faulty, &rdev->flags)) {
 197                         /* this is a usable devices */
 198                         atomic_inc(&rdev->nr_pending);
 199                         rcu_read_unlock();
 200                         return rdev;
 201                 }
 202         }
 203         rcu_read_unlock();
 204         return NULL;
 205 }
 206
 207 static int write_sb_page(struct bitmap *bitmap, struct page *page, int wait)
 208 {
 209         struct md_rdev *rdev = NULL;
 210         struct block_device *bdev;
 211         struct mddev *mddev = bitmap->mddev;
 212         struct bitmap_storage *store = &bitmap->storage;
 213         int node_offset = 0;
 214
 215         if (mddev_is_clustered(bitmap->mddev))
 216                 node_offset = bitmap->cluster_slot * store->file_pages;
 217
 218         while ((rdev = next_active_rdev(rdev, mddev)) != NULL) {
 219                 int size = PAGE_SIZE;
 220                 loff_t offset = mddev->bitmap_info.offset;
 221
 222                 bdev = (rdev->meta_bdev) ? rdev->meta_bdev : rdev->bdev;
 223
 224                 if (page->index == store->file_pages-1) {
 225                         int last_page_size = store->bytes & (PAGE_SIZE-1);
 226                         if (last_page_size == 0)
 227                                 last_page_size = PAGE_SIZE;
 228                         size = roundup(last_page_size,
 229                                        bdev_logical_block_size(bdev));
 230                 }
 231                 /* Just make sure we aren't corrupting data or
 232                  * metadata
 233                  */
 234                 if (mddev->external) {
 235                         /* Bitmap could be anywhere. */
 236                         if (rdev->sb_start + offset + (page->index
 237                                                        * (PAGE_SIZE/512))
 238                             > rdev->data_offset
 239                             &&
 240                             rdev->sb_start + offset
 241                             < (rdev->data_offset + mddev->dev_sectors
 242                              + (PAGE_SIZE/512)))
 243                                 goto bad_alignment;
 244                 } else if (offset < 0) {
 245                         /* DATA  BITMAP METADATA  */
 246                         if (offset
 247                             + (long)(page->index * (PAGE_SIZE/512))
 248                             + size/512 > 0)
 249                                 /* bitmap runs in to metadata */
 250                                 goto bad_alignment;
 251                         if (rdev->data_offset + mddev->dev_sectors
 252                             > rdev->sb_start + offset)
 253                                 /* data runs in to bitmap */
 254                                 goto bad_alignment;
 255                 } else if (rdev->sb_start < rdev->data_offset) {
 256                         /* METADATA BITMAP DATA */
 257                         if (rdev->sb_start
 258                             + offset
 259                             + page->index*(PAGE_SIZE/512) + size/512
 260                             > rdev->data_offset)
 261                                 /* bitmap runs in to data */
 262                                 goto bad_alignment;
 263                 } else {
 264                         /* DATA METADATA BITMAP - no problems */
 265                 }
 266                 md_super_write(mddev, rdev,
 267                                rdev->sb_start + offset
 268                                + page->index * (PAGE_SIZE/512),
 269                                size,
 270                                page);
 271         }
 272
 273         if (wait)
 274                 md_super_wait(mddev);
 275         return 0;
 276
 277  bad_alignment:
 278         return -EINVAL;
 279 }
 280
 281 static void bitmap_file_kick(struct bitmap *bitmap);
 282 /*
 283  * write out a page to a file
 284  */
 285 static void write_page(struct bitmap *bitmap, struct page *page, int wait)
 286 {
 287         struct buffer_head *bh;
 288
 289         if (bitmap->storage.file == NULL) {
 290                 switch (write_sb_page(bitmap, page, wait)) {
 291                 case -EINVAL:
 292                         set_bit(BITMAP_WRITE_ERROR, &bitmap->flags);
 293                 }
 294         } else {
 295
 296                 bh = page_buffers(page);
 297
 298                 while (bh && bh->b_blocknr) {
 299                         atomic_inc(&bitmap->pending_writes);
 300                         set_buffer_locked(bh);
 301                         set_buffer_mapped(bh);
 302                         submit_bh(WRITE | REQ_SYNC, bh);
 303                         bh = bh->b_this_page;
 304                 }
 305
 306                 if (wait)
 307                         wait_event(bitmap->write_wait,
 308                                    atomic_read(&bitmap->pending_writes)==0);
 309         }
 310         if (test_bit(BITMAP_WRITE_ERROR, &bitmap->flags))
 311                 bitmap_file_kick(bitmap);
 312 }
 313
 314 static void end_bitmap_write(struct buffer_head *bh, int uptodate)
 315 {
 316         struct bitmap *bitmap = bh->b_private;
 317
 318         if (!uptodate)
 319                 set_bit(BITMAP_WRITE_ERROR, &bitmap->flags);
 320         if (atomic_dec_and_test(&bitmap->pending_writes))
 321                 wake_up(&bitmap->write_wait);
 322 }
 323
 324 /* copied from buffer.c */
 325 static void
 326 __clear_page_buffers(struct page *page)
 327 {
 328         ClearPagePrivate(page);
 329         set_page_private(page, 0);
 330         page_cache_release(page);
 331 }
 332 static void free_buffers(struct page *page)
 333 {
 334         struct buffer_head *bh;
 335
 336         if (!PagePrivate(page))
 337                 return;
 338
 339         bh = page_buffers(page);
 340         while (bh) {
 341                 struct buffer_head *next = bh->b_this_page;
 342                 free_buffer_head(bh);
 343                 bh = next;
 344         }
 345         __clear_page_buffers(page);
 346         put_page(page);
 347 }
 348
 349 /* read a page from a file.
 350  * We both read the page, and attach buffers to the page to record the
 351  * address of each block (using bmap).  These addresses will be used
 352  * to write the block later, completely bypassing the filesystem.
 353  * This usage is similar to how swap files are handled, and allows us
 354  * to write to a file with no concerns of memory allocation failing.
 355  */
 356 static int read_page(struct file *file, unsigned long index,
 357                      struct bitmap *bitmap,
 358                      unsigned long count,
 359                      struct page *page)
 360 {
 361         int ret = 0;
 362         struct inode *inode = file_inode(file);
 363         struct buffer_head *bh;
 364         sector_t block;
 365
 366         pr_debug("read bitmap file (%dB @ %llu)\n", (int)PAGE_SIZE,
 367                  (unsigned long long)index << PAGE_SHIFT);
 368
 369         bh = alloc_page_buffers(page, 1<<inode->i_blkbits, 0);
 370         if (!bh) {
 371                 ret = -ENOMEM;
 372                 goto out;
 373         }
 374         attach_page_buffers(page, bh);
 375         block = index << (PAGE_SHIFT - inode->i_blkbits);
 376         while (bh) {
 377                 if (count == 0)
 378                         bh->b_blocknr = 0;
 379                 else {
 380                         bh->b_blocknr = bmap(inode, block);
 381                         if (bh->b_blocknr == 0) {
 382                                 /* Cannot use this file! */
 383                                 ret = -EINVAL;
 384                                 goto out;
 385                         }
 386                         bh->b_bdev = inode->i_sb->s_bdev;
 387                         if (count < (1<<inode->i_blkbits))
 388                                 count = 0;
 389                         else
 390                                 count -= (1<<inode->i_blkbits);
 391
 392                         bh->b_end_io = end_bitmap_write;
 393                         bh->b_private = bitmap;
 394                         atomic_inc(&bitmap->pending_writes);
 395                         set_buffer_locked(bh);
 396                         set_buffer_mapped(bh);
 397                         submit_bh(READ, bh);
 398                 }
 399                 block++;
 400                 bh = bh->b_this_page;
 401         }
 402         page->index = index;
 403
 404         wait_event(bitmap->write_wait,
 405                    atomic_read(&bitmap->pending_writes)==0);
 406         if (test_bit(BITMAP_WRITE_ERROR, &bitmap->flags))
 407                 ret = -EIO;
 408 out:
 409         if (ret)
 410                 printk(KERN_ALERT "md: bitmap read error: (%dB @ %llu): %d\n",
 411                         (int)PAGE_SIZE,
 412                         (unsigned long long)index << PAGE_SHIFT,
 413                         ret);
 414         return ret;
 415 }
 416
 417 /*
 418  * bitmap file superblock operations
 419  */
 420
 421 /* update the event counter and sync the superblock to disk */
 422 void bitmap_update_sb(struct bitmap *bitmap)
 423 {
 424         bitmap_super_t *sb;
 425
 426         if (!bitmap || !bitmap->mddev) /* no bitmap for this array */
 427                 return;
 428         if (bitmap->mddev->bitmap_info.external)
 429                 return;
 430         if (!bitmap->storage.sb_page) /* no superblock */
 431                 return;
 432         sb = kmap_atomic(bitmap->storage.sb_page);
 433         sb->events = cpu_to_le64(bitmap->mddev->events);
 434         if (bitmap->mddev->events < bitmap->events_cleared)
 435                 /* rocking back to read-only */
 436                 bitmap->events_cleared = bitmap->mddev->events;
 437         sb->events_cleared = cpu_to_le64(bitmap->events_cleared);
 438         sb->state = cpu_to_le32(bitmap->flags);
 439         /* Just in case these have been changed via sysfs: */
 440         sb->daemon_sleep = cpu_to_le32(bitmap->mddev->bitmap_info.daemon_sleep/HZ);
 441         sb->write_behind = cpu_to_le32(bitmap->mddev->bitmap_info.max_write_behind);
 442         /* This might have been changed by a reshape */
 443         sb->sync_size = cpu_to_le64(bitmap->mddev->resync_max_sectors);
 444         sb->chunksize = cpu_to_le32(bitmap->mddev->bitmap_info.chunksize);
 445         sb->nodes = cpu_to_le32(bitmap->mddev->bitmap_info.nodes);
 446         sb->sectors_reserved = cpu_to_le32(bitmap->mddev->
 447                                            bitmap_info.space);
 448         kunmap_atomic(sb);
 449         write_page(bitmap, bitmap->storage.sb_page, 1);
 450 }
 451
 452 /* print out the bitmap file superblock */
 453 void bitmap_print_sb(struct bitmap *bitmap)
 454 {
 455         bitmap_super_t *sb;
 456
 457         if (!bitmap || !bitmap->storage.sb_page)
 458                 return;
 459         sb = kmap_atomic(bitmap->storage.sb_page);
 460         printk(KERN_DEBUG "%s: bitmap file superblock:\n", bmname(bitmap));
 461         printk(KERN_DEBUG "         magic: %08x\n", le32_to_cpu(sb->magic));
 462         printk(KERN_DEBUG "       version: %d\n", le32_to_cpu(sb->version));
 463         printk(KERN_DEBUG "          uuid: %08x.%08x.%08x.%08x\n",
 464                                         *(__u32 *)(sb->uuid+0),
 465                                         *(__u32 *)(sb->uuid+4),
 466                                         *(__u32 *)(sb->uuid+8),
 467                                         *(__u32 *)(sb->uuid+12));
 468         printk(KERN_DEBUG "        events: %llu\n",
 469                         (unsigned long long) le64_to_cpu(sb->events));
 470         printk(KERN_DEBUG "events cleared: %llu\n",
 471                         (unsigned long long) le64_to_cpu(sb->events_cleared));
 472         printk(KERN_DEBUG "         state: %08x\n", le32_to_cpu(sb->state));
 473         printk(KERN_DEBUG "     chunksize: %d B\n", le32_to_cpu(sb->chunksize));
 474         printk(KERN_DEBUG "  daemon sleep: %ds\n", le32_to_cpu(sb->daemon_sleep));
 475         printk(KERN_DEBUG "     sync size: %llu KB\n",
 476                         (unsigned long long)le64_to_cpu(sb->sync_size)/2);
 477         printk(KERN_DEBUG "max write behind: %d\n", le32_to_cpu(sb->write_behind));
 478         kunmap_atomic(sb);
 479 }
 480
 481 /*
 482  * bitmap_new_disk_sb
 483  * @bitmap
 484  *
 485  * This function is somewhat the reverse of bitmap_read_sb.  bitmap_read_sb
 486  * reads and verifies the on-disk bitmap superblock and populates bitmap_info.
 487  * This function verifies 'bitmap_info' and populates the on-disk bitmap
 488  * structure, which is to be written to disk.
 489  *
 490  * Returns: 0 on success, -Exxx on error
 491  */
 492 static int bitmap_new_disk_sb(struct bitmap *bitmap)
 493 {
 494         bitmap_super_t *sb;
 495         unsigned long chunksize, daemon_sleep, write_behind;
 496
 497         bitmap->storage.sb_page = alloc_page(GFP_KERNEL);
 498         if (bitmap->storage.sb_page == NULL)
 499                 return -ENOMEM;
 500         bitmap->storage.sb_page->index = 0;
 501
 502         sb = kmap_atomic(bitmap->storage.sb_page);
 503
 504         sb->magic = cpu_to_le32(BITMAP_MAGIC);
 505         sb->version = cpu_to_le32(BITMAP_MAJOR_HI);
 506
 507         chunksize = bitmap->mddev->bitmap_info.chunksize;
 508         BUG_ON(!chunksize);
 509         if (!is_power_of_2(chunksize)) {
 510                 kunmap_atomic(sb);
 511                 printk(KERN_ERR "bitmap chunksize not a power of 2\n");
 512                 return -EINVAL;
 513         }
 514         sb->chunksize = cpu_to_le32(chunksize);
 515
 516         daemon_sleep = bitmap->mddev->bitmap_info.daemon_sleep;
 517         if (!daemon_sleep ||
 518             (daemon_sleep < 1) || (daemon_sleep > MAX_SCHEDULE_TIMEOUT)) {
 519                 printk(KERN_INFO "Choosing daemon_sleep default (5 sec)\n");
 520                 daemon_sleep = 5 * HZ;
 521         }
 522         sb->daemon_sleep = cpu_to_le32(daemon_sleep);
 523         bitmap->mddev->bitmap_info.daemon_sleep = daemon_sleep;
 524
 525         /*
 526          * FIXME: write_behind for RAID1.  If not specified, what
 527          * is a good choice?  We choose COUNTER_MAX / 2 arbitrarily.
 528          */
 529         write_behind = bitmap->mddev->bitmap_info.max_write_behind;
 530         if (write_behind > COUNTER_MAX)
 531                 write_behind = COUNTER_MAX / 2;
 532         sb->write_behind = cpu_to_le32(write_behind);
 533         bitmap->mddev->bitmap_info.max_write_behind = write_behind;
 534
 535         /* keep the array size field of the bitmap superblock up to date */
 536         sb->sync_size = cpu_to_le64(bitmap->mddev->resync_max_sectors);
 537
 538         memcpy(sb->uuid, bitmap->mddev->uuid, 16);
 539
 540         set_bit(BITMAP_STALE, &bitmap->flags);
 541         sb->state = cpu_to_le32(bitmap->flags);
 542         bitmap->events_cleared = bitmap->mddev->events;
 543         sb->events_cleared = cpu_to_le64(bitmap->mddev->events);
 544
 545         kunmap_atomic(sb);
 546
 547         return 0;
 548 }
 549
 550 /* read the superblock from the bitmap file and initialize some bitmap fields */
 551 static int bitmap_read_sb(struct bitmap *bitmap)
 552 {
 553         char *reason = NULL;
 554         bitmap_super_t *sb;
 555         unsigned long chunksize, daemon_sleep, write_behind;
 556         unsigned long long events;
 557         int nodes = 0;
 558         unsigned long sectors_reserved = 0;
 559         int err = -EINVAL;
 560         struct page *sb_page;
 561
 562         if (!bitmap->storage.file && !bitmap->mddev->bitmap_info.offset) {
 563                 chunksize = 128 * 1024 * 1024;
 564                 daemon_sleep = 5 * HZ;
 565                 write_behind = 0;
 566                 set_bit(BITMAP_STALE, &bitmap->flags);
 567                 err = 0;
 568                 goto out_no_sb;
 569         }
 570         /* page 0 is the superblock, read it... */
 571         sb_page = alloc_page(GFP_KERNEL);
 572         if (!sb_page)
 573                 return -ENOMEM;
 574         bitmap->storage.sb_page = sb_page;
 575
 576 re_read:
 577         /* If cluster_slot is set, the cluster is setup */
 578         if (bitmap->cluster_slot >= 0) {
 579                 sector_t bm_blocks = bitmap->mddev->resync_max_sectors;
 580
 581                 sector_div(bm_blocks,
 582                            bitmap->mddev->bitmap_info.chunksize >> 9);
 583                 /* bits to bytes */
 584                 bm_blocks = ((bm_blocks+7) >> 3) + sizeof(bitmap_super_t);
 585                 /* to 4k blocks */
 586                 bm_blocks = DIV_ROUND_UP_SECTOR_T(bm_blocks, 4096);
 587                 bitmap->mddev->bitmap_info.offset += bitmap->cluster_slot * (bm_blocks << 3);
 588                 pr_info("%s:%d bm slot: %d offset: %llu\n", __func__, __LINE__,
 589                         bitmap->cluster_slot, (unsigned long long)bitmap->mddev->bitmap_info.offset);
 590         }
 591
 592         if (bitmap->storage.file) {
 593                 loff_t isize = i_size_read(bitmap->storage.file->f_mapping->host);
 594                 int bytes = isize > PAGE_SIZE ? PAGE_SIZE : isize;
 595
 596                 err = read_page(bitmap->storage.file, 0,
 597                                 bitmap, bytes, sb_page);
 598         } else {
 599                 err = read_sb_page(bitmap->mddev,
 600                                    bitmap->mddev->bitmap_info.offset,
 601                                    sb_page,
 602                                    0, sizeof(bitmap_super_t));
 603         }
 604         if (err)
 605                 return err;
 606
 607         err = -EINVAL;
 608         sb = kmap_atomic(sb_page);
 609
 610         chunksize = le32_to_cpu(sb->chunksize);
 611         daemon_sleep = le32_to_cpu(sb->daemon_sleep) * HZ;
 612         write_behind = le32_to_cpu(sb->write_behind);
 613         sectors_reserved = le32_to_cpu(sb->sectors_reserved);
 614         nodes = le32_to_cpu(sb->nodes);
 615         strlcpy(bitmap->mddev->bitmap_info.cluster_name, sb->cluster_name, 64);
 616
 617         /* verify that the bitmap-specific fields are valid */
 618         if (sb->magic != cpu_to_le32(BITMAP_MAGIC))
 619                 reason = "bad magic";
 620         else if (le32_to_cpu(sb->version) < BITMAP_MAJOR_LO ||
 621                  le32_to_cpu(sb->version) > BITMAP_MAJOR_HI)
 622                 reason = "unrecognized superblock version";
 623         else if (chunksize < 512)
 624                 reason = "bitmap chunksize too small";
 625         else if (!is_power_of_2(chunksize))
 626                 reason = "bitmap chunksize not a power of 2";
 627         else if (daemon_sleep < 1 || daemon_sleep > MAX_SCHEDULE_TIMEOUT)
 628                 reason = "daemon sleep period out of range";
 629         else if (write_behind > COUNTER_MAX)
 630                 reason = "write-behind limit out of range (0 - 16383)";
 631         if (reason) {
 632                 printk(KERN_INFO "%s: invalid bitmap file superblock: %s\n",
 633                         bmname(bitmap), reason);
 634                 goto out;
 635         }
 636
 637         /* keep the array size field of the bitmap superblock up to date */
 638         sb->sync_size = cpu_to_le64(bitmap->mddev->resync_max_sectors);
 639
 640         if (bitmap->mddev->persistent) {
 641                 /*
 642                  * We have a persistent array superblock, so compare the
 643                  * bitmap's UUID and event counter to the mddev's
 644                  */
 645                 if (memcmp(sb->uuid, bitmap->mddev->uuid, 16)) {
 646                         printk(KERN_INFO
 647                                "%s: bitmap superblock UUID mismatch\n",
 648                                bmname(bitmap));
 649                         goto out;
 650                 }
 651                 events = le64_to_cpu(sb->events);
 652                 if (!nodes && (events < bitmap->mddev->events)) {
 653                         printk(KERN_INFO
 654                                "%s: bitmap file is out of date (%llu < %llu) "
 655                                "-- forcing full recovery\n",
 656                                bmname(bitmap), events,
 657                                (unsigned long long) bitmap->mddev->events);
 658                         set_bit(BITMAP_STALE, &bitmap->flags);
 659                 }
 660         }
 661
 662         /* assign fields using values from superblock */
 663         bitmap->flags |= le32_to_cpu(sb->state);
 664         if (le32_to_cpu(sb->version) == BITMAP_MAJOR_HOSTENDIAN)
 665                 set_bit(BITMAP_HOSTENDIAN, &bitmap->flags);
 666         bitmap->events_cleared = le64_to_cpu(sb->events_cleared);
 667         strlcpy(bitmap->mddev->bitmap_info.cluster_name, sb->cluster_name, 64);
 668         err = 0;
 669
 670 out:
 671         kunmap_atomic(sb);
 672         /* Assiging chunksize is required for "re_read" */
 673         bitmap->mddev->bitmap_info.chunksize = chunksize;
 674         if (nodes && (bitmap->cluster_slot < 0)) {
 675                 err = md_setup_cluster(bitmap->mddev, nodes);
 676                 if (err) {
 677                         pr_err("%s: Could not setup cluster service (%d)\n",
 678                                         bmname(bitmap), err);
 679                         goto out_no_sb;
 680                 }
 681                 bitmap->cluster_slot = md_cluster_ops->slot_number(bitmap->mddev);
 682                 goto re_read;
 683         }
 684
 685
 686 out_no_sb:
 687         if (test_bit(BITMAP_STALE, &bitmap->flags))
 688                 bitmap->events_cleared = bitmap->mddev->events;
 689         bitmap->mddev->bitmap_info.chunksize = chunksize;
 690         bitmap->mddev->bitmap_info.daemon_sleep = daemon_sleep;
 691         bitmap->mddev->bitmap_info.max_write_behind = write_behind;
 692         bitmap->mddev->bitmap_info.nodes = nodes;
 693         if (bitmap->mddev->bitmap_info.space == 0 ||
 694             bitmap->mddev->bitmap_info.space > sectors_reserved)
 695                 bitmap->mddev->bitmap_info.space = sectors_reserved;
 696         if (err) {
 697                 bitmap_print_sb(bitmap);
 698                 if (bitmap->cluster_slot < 0)
 699                         md_cluster_stop(bitmap->mddev);
 700         }
 701         return err;
 702 }
 703
 704 /*
 705  * general bitmap file operations
 706  */
 707
 708 /*
 709  * on-disk bitmap:
 710  *
 711  * Use one bit per "chunk" (block set). We do the disk I/O on the bitmap
 712  * file a page at a time. There's a superblock at the start of the file.
 713  */
 714 /* calculate the index of the page that contains this bit */
 715 static inline unsigned long file_page_index(struct bitmap_storage *store,
 716                                             unsigned long chunk)
 717 {
 718         if (store->sb_page)
 719                 chunk += sizeof(bitmap_super_t) << 3;
 720         return chunk >> PAGE_BIT_SHIFT;
 721 }
 722
 723 /* calculate the (bit) offset of this bit within a page */
 724 static inline unsigned long file_page_offset(struct bitmap_storage *store,
 725                                              unsigned long chunk)
 726 {
 727         if (store->sb_page)
 728                 chunk += sizeof(bitmap_super_t) << 3;
 729         return chunk & (PAGE_BITS - 1);
 730 }
 731
 732 /*
 733  * return a pointer to the page in the filemap that contains the given bit
 734  *
 735  */
 736 static inline struct page *filemap_get_page(struct bitmap_storage *store,
 737                                             unsigned long chunk)
 738 {
 739         if (file_page_index(store, chunk) >= store->file_pages)
 740                 return NULL;
 741         return store->filemap[file_page_index(store, chunk)];
 742 }
 743
 744 static int bitmap_storage_alloc(struct bitmap_storage *store,
 745                                 unsigned long chunks, int with_super,
 746                                 int slot_number)
 747 {
 748         int pnum, offset = 0;
 749         unsigned long num_pages;
 750         unsigned long bytes;
 751
 752         bytes = DIV_ROUND_UP(chunks, 8);
 753         if (with_super)
 754                 bytes += sizeof(bitmap_super_t);
 755
 756         num_pages = DIV_ROUND_UP(bytes, PAGE_SIZE);
 757         offset = slot_number * (num_pages - 1);
 758
 759         store->filemap = kmalloc(sizeof(struct page *)
 760                                  * num_pages, GFP_KERNEL);
 761         if (!store->filemap)
 762                 return -ENOMEM;
 763
 764         if (with_super && !store->sb_page) {
 765                 store->sb_page = alloc_page(GFP_KERNEL|__GFP_ZERO);
 766                 if (store->sb_page == NULL)
 767                         return -ENOMEM;
 768         }
 769
 770         pnum = 0;
 771         if (store->sb_page) {
 772                 store->filemap[0] = store->sb_page;
 773                 pnum = 1;
 774                 store->sb_page->index = offset;
 775         }
 776
 777         for ( ; pnum < num_pages; pnum++) {
 778                 store->filemap[pnum] = alloc_page(GFP_KERNEL|__GFP_ZERO);
 779                 if (!store->filemap[pnum]) {
 780                         store->file_pages = pnum;
 781                         return -ENOMEM;
 782                 }
 783                 store->filemap[pnum]->index = pnum + offset;
 784         }
 785         store->file_pages = pnum;
 786
 787         /* We need 4 bits per page, rounded up to a multiple
 788          * of sizeof(unsigned long) */
 789         store->filemap_attr = kzalloc(
 790                 roundup(DIV_ROUND_UP(num_pages*4, 8), sizeof(unsigned long)),
 791                 GFP_KERNEL);
 792         if (!store->filemap_attr)
 793                 return -ENOMEM;
 794
 795         store->bytes = bytes;
 796
 797         return 0;
 798 }
 799
 800 static void bitmap_file_unmap(struct bitmap_storage *store)
 801 {
 802         struct page **map, *sb_page;
 803         int pages;
 804         struct file *file;
 805
 806         file = store->file;
 807         map = store->filemap;
 808         pages = store->file_pages;
 809         sb_page = store->sb_page;
 810
 811         while (pages--)
 812                 if (map[pages] != sb_page) /* 0 is sb_page, release it below */
 813                         free_buffers(map[pages]);
 814         kfree(map);
 815         kfree(store->filemap_attr);
 816
 817         if (sb_page)
 818                 free_buffers(sb_page);
 819
 820         if (file) {
 821                 struct inode *inode = file_inode(file);
 822                 invalidate_mapping_pages(inode->i_mapping, 0, -1);
 823                 fput(file);
 824         }
 825 }
 826
 827 /*
 828  * bitmap_file_kick - if an error occurs while manipulating the bitmap file
 829  * then it is no longer reliable, so we stop using it and we mark the file
 830  * as failed in the superblock
 831  */
 832 static void bitmap_file_kick(struct bitmap *bitmap)
 833 {
 834         char *path, *ptr = NULL;
 835
 836         if (!test_and_set_bit(BITMAP_STALE, &bitmap->flags)) {
 837                 bitmap_update_sb(bitmap);
 838
 839                 if (bitmap->storage.file) {
 840                         path = kmalloc(PAGE_SIZE, GFP_KERNEL);
 841                         if (path)
 842                                 ptr = d_path(&bitmap->storage.file->f_path,
 843                                              path, PAGE_SIZE);
 844
 845                         printk(KERN_ALERT
 846                               "%s: kicking failed bitmap file %s from array!\n",
 847                               bmname(bitmap), IS_ERR(ptr) ? "" : ptr);
 848
 849                         kfree(path);
 850                 } else
 851                         printk(KERN_ALERT
 852                                "%s: disabling internal bitmap due to errors\n",
 853                                bmname(bitmap));
 854         }
 855 }
 856
 857 enum bitmap_page_attr {
 858         BITMAP_PAGE_DIRTY = 0,     /* there are set bits that need to be synced */
 859         BITMAP_PAGE_PENDING = 1,   /* there are bits that are being cleaned.
 860                                     * i.e. counter is 1 or 2. */
 861         BITMAP_PAGE_NEEDWRITE = 2, /* there are cleared bits that need to be synced */
 862 };
 863
 864 static inline void set_page_attr(struct bitmap *bitmap, int pnum,
 865                                  enum bitmap_page_attr attr)
 866 {
 867         set_bit((pnum<<2) + attr, bitmap->storage.filemap_attr);
 868 }
 869
 870 static inline void clear_page_attr(struct bitmap *bitmap, int pnum,
 871                                    enum bitmap_page_attr attr)
 872 {
 873         clear_bit((pnum<<2) + attr, bitmap->storage.filemap_attr);
 874 }
 875
 876 static inline int test_page_attr(struct bitmap *bitmap, int pnum,
 877                                  enum bitmap_page_attr attr)
 878 {
 879         return test_bit((pnum<<2) + attr, bitmap->storage.filemap_attr);
 880 }
 881
 882 static inline int test_and_clear_page_attr(struct bitmap *bitmap, int pnum,
 883                                            enum bitmap_page_attr attr)
 884 {
 885         return test_and_clear_bit((pnum<<2) + attr,
 886                                   bitmap->storage.filemap_attr);
 887 }
 888 /*
 889  * bitmap_file_set_bit -- called before performing a write to the md device
 890  * to set (and eventually sync) a particular bit in the bitmap file
 891  *
 892  * we set the bit immediately, then we record the page number so that
 893  * when an unplug occurs, we can flush the dirty pages out to disk
 894  */
 895 static void bitmap_file_set_bit(struct bitmap *bitmap, sector_t block)
 896 {
 897         unsigned long bit;
 898         struct page *page;
 899         void *kaddr;
 900         unsigned long chunk = block >> bitmap->counts.chunkshift;
 901
 902         page = filemap_get_page(&bitmap->storage, chunk);
 903         if (!page)
 904                 return;
 905         bit = file_page_offset(&bitmap->storage, chunk);
 906
 907         /* set the bit */
 908         kaddr = kmap_atomic(page);
 909         if (test_bit(BITMAP_HOSTENDIAN, &bitmap->flags))
 910                 set_bit(bit, kaddr);
 911         else
 912                 set_bit_le(bit, kaddr);
 913         kunmap_atomic(kaddr);
 914         pr_debug("set file bit %lu page %lu\n", bit, page->index);
 915         /* record page number so it gets flushed to disk when unplug occurs */
 916         set_page_attr(bitmap, page->index, BITMAP_PAGE_DIRTY);
 917 }
 918
 919 static void bitmap_file_clear_bit(struct bitmap *bitmap, sector_t block)
 920 {
 921         unsigned long bit;
 922         struct page *page;
 923         void *paddr;
 924         unsigned long chunk = block >> bitmap->counts.chunkshift;
 925
 926         page = filemap_get_page(&bitmap->storage, chunk);
 927         if (!page)
 928                 return;
 929         bit = file_page_offset(&bitmap->storage, chunk);
 930         paddr = kmap_atomic(page);
 931         if (test_bit(BITMAP_HOSTENDIAN, &bitmap->flags))
 932                 clear_bit(bit, paddr);
 933         else
 934                 clear_bit_le(bit, paddr);
 935         kunmap_atomic(paddr);
 936         if (!test_page_attr(bitmap, page->index, BITMAP_PAGE_NEEDWRITE)) {
 937                 set_page_attr(bitmap, page->index, BITMAP_PAGE_PENDING);
 938                 bitmap->allclean = 0;
 939         }
 940 }
 941
 942 static int bitmap_file_test_bit(struct bitmap *bitmap, sector_t block)
 943 {
 944         unsigned long bit;
 945         struct page *page;
 946         void *paddr;
 947         unsigned long chunk = block >> bitmap->counts.chunkshift;
 948         int set = 0;
 949
 950         page = filemap_get_page(&bitmap->storage, chunk);
 951         if (!page)
 952                 return -EINVAL;
 953         bit = file_page_offset(&bitmap->storage, chunk);
 954         paddr = kmap_atomic(page);
 955         if (test_bit(BITMAP_HOSTENDIAN, &bitmap->flags))
 956                 set = test_bit(bit, paddr);
 957         else
 958                 set = test_bit_le(bit, paddr);
 959         kunmap_atomic(paddr);
 960         return set;
 961 }
 962
 963
 964 /* this gets called when the md device is ready to unplug its underlying
 965  * (slave) device queues -- before we let any writes go down, we need to
 966  * sync the dirty pages of the bitmap file to disk */
 967 void bitmap_unplug(struct bitmap *bitmap)
 968 {
 969         unsigned long i;
 970         int dirty, need_write;
 971
 972         if (!bitmap || !bitmap->storage.filemap ||
 973             test_bit(BITMAP_STALE, &bitmap->flags))
 974                 return;
 975
 976         /* look at each page to see if there are any set bits that need to be
 977          * flushed out to disk */
 978         for (i = 0; i < bitmap->storage.file_pages; i++) {
 979                 if (!bitmap->storage.filemap)
 980                         return;
 981                 dirty = test_and_clear_page_attr(bitmap, i, BITMAP_PAGE_DIRTY);
 982                 need_write = test_and_clear_page_attr(bitmap, i,
 983                                                       BITMAP_PAGE_NEEDWRITE);
 984                 if (dirty || need_write) {
 985                         clear_page_attr(bitmap, i, BITMAP_PAGE_PENDING);
 986                         write_page(bitmap, bitmap->storage.filemap[i], 0);
 987                 }
 988         }
 989         if (bitmap->storage.file)
 990                 wait_event(bitmap->write_wait,
 991                            atomic_read(&bitmap->pending_writes)==0);
 992         else
 993                 md_super_wait(bitmap->mddev);
 994
 995         if (test_bit(BITMAP_WRITE_ERROR, &bitmap->flags))
 996                 bitmap_file_kick(bitmap);
 997 }
 998 EXPORT_SYMBOL(bitmap_unplug);
 999
1000 static void bitmap_set_memory_bits(struct bitmap *bitmap, sector_t offset, int needed);
1001 /* * bitmap_init_from_disk -- called at bitmap_create time to initialize
1002  * the in-memory bitmap from the on-disk bitmap -- also, sets up the
1003  * memory mapping of the bitmap file
1004  * Special cases:
1005  *   if there's no bitmap file, or if the bitmap file had been
1006  *   previously kicked from the array, we mark all the bits as
1007  *   1's in order to cause a full resync.
1008  *
1009  * We ignore all bits for sectors that end earlier than 'start'.
1010  * This is used when reading an out-of-date bitmap...
1011  */
1012 static int bitmap_init_from_disk(struct bitmap *bitmap, sector_t start)
1013 {
1014         unsigned long i, chunks, index, oldindex, bit, node_offset = 0;
1015         struct page *page = NULL;
1016         unsigned long bit_cnt = 0;
1017         struct file *file;
1018         unsigned long offset;
1019         int outofdate;
1020         int ret = -ENOSPC;
1021         void *paddr;
1022         struct bitmap_storage *store = &bitmap->storage;
1023
1024         chunks = bitmap->counts.chunks;
1025         file = store->file;
1026
1027         if (!file && !bitmap->mddev->bitmap_info.offset) {
1028                 /* No permanent bitmap - fill with '1s'. */
1029                 store->filemap = NULL;
1030                 store->file_pages = 0;
1031                 for (i = 0; i < chunks ; i++) {
1032                         /* if the disk bit is set, set the memory bit */
1033                         int needed = ((sector_t)(i+1) << (bitmap->counts.chunkshift)
1034                                       >= start);
1035                         bitmap_set_memory_bits(bitmap,
1036                                                (sector_t)i << bitmap->counts.chunkshift,
1037                                                needed);
1038                 }
1039                 return 0;
1040         }
1041
1042         outofdate = test_bit(BITMAP_STALE, &bitmap->flags);
1043         if (outofdate)
1044                 printk(KERN_INFO "%s: bitmap file is out of date, doing full "
1045                         "recovery\n", bmname(bitmap));
1046
1047         if (file && i_size_read(file->f_mapping->host) < store->bytes) {
1048                 printk(KERN_INFO "%s: bitmap file too short %lu < %lu\n",
1049                        bmname(bitmap),
1050                        (unsigned long) i_size_read(file->f_mapping->host),
1051                        store->bytes);
1052                 goto err;
1053         }
1054
1055         oldindex = ~0L;
1056         offset = 0;
1057         if (!bitmap->mddev->bitmap_info.external)
1058                 offset = sizeof(bitmap_super_t);
1059
1060         if (mddev_is_clustered(bitmap->mddev))
1061                 node_offset = bitmap->cluster_slot * (DIV_ROUND_UP(store->bytes, PAGE_SIZE));
1062
1063         for (i = 0; i < chunks; i++) {
1064                 int b;
1065                 index = file_page_index(&bitmap->storage, i);
1066                 bit = file_page_offset(&bitmap->storage, i);
1067                 if (index != oldindex) { /* this is a new page, read it in */
1068                         int count;
1069                         /* unmap the old page, we're done with it */
1070                         if (index == store->file_pages-1)
1071                                 count = store->bytes - index * PAGE_SIZE;
1072                         else
1073                                 count = PAGE_SIZE;
1074                         page = store->filemap[index];
1075                         if (file)
1076                                 ret = read_page(file, index, bitmap,
1077                                                 count, page);
1078                         else
1079                                 ret = read_sb_page(
1080                                         bitmap->mddev,
1081                                         bitmap->mddev->bitmap_info.offset,
1082                                         page,
1083                                         index + node_offset, count);
1084
1085                         if (ret)
1086                                 goto err;
1087
1088                         oldindex = index;
1089
1090                         if (outofdate) {
1091                                 /*
1092                                  * if bitmap is out of date, dirty the
1093                                  * whole page and write it out
1094                                  */
1095                                 paddr = kmap_atomic(page);
1096                                 memset(paddr + offset, 0xff,
1097                                        PAGE_SIZE - offset);
1098                                 kunmap_atomic(paddr);
1099                                 write_page(bitmap, page, 1);
1100
1101                                 ret = -EIO;
1102                                 if (test_bit(BITMAP_WRITE_ERROR,
1103                                              &bitmap->flags))
1104                                         goto err;
1105                         }
1106                 }
1107                 paddr = kmap_atomic(page);
1108                 if (test_bit(BITMAP_HOSTENDIAN, &bitmap->flags))
1109                         b = test_bit(bit, paddr);
1110                 else
1111                         b = test_bit_le(bit, paddr);
1112                 kunmap_atomic(paddr);
1113                 if (b) {
1114                         /* if the disk bit is set, set the memory bit */
1115                         int needed = ((sector_t)(i+1) << bitmap->counts.chunkshift
1116                                       >= start);
1117                         bitmap_set_memory_bits(bitmap,
1118                                                (sector_t)i << bitmap->counts.chunkshift,
1119                                                needed);
1120                         bit_cnt++;
1121                 }
1122                 offset = 0;
1123         }
1124
1125         printk(KERN_INFO "%s: bitmap initialized from disk: "
1126                "read %lu pages, set %lu of %lu bits\n",
1127                bmname(bitmap), store->file_pages,
1128                bit_cnt, chunks);
1129
1130         return 0;
1131
1132  err:
1133         printk(KERN_INFO "%s: bitmap initialisation failed: %d\n",
1134                bmname(bitmap), ret);
1135         return ret;
1136 }
1137
1138 void bitmap_write_all(struct bitmap *bitmap)
1139 {
1140         /* We don't actually write all bitmap blocks here,
1141          * just flag them as needing to be written
1142          */
1143         int i;
1144
1145         if (!bitmap || !bitmap->storage.filemap)
1146                 return;
1147         if (bitmap->storage.file)
1148                 /* Only one copy, so nothing needed */
1149                 return;
1150
1151         for (i = 0; i < bitmap->storage.file_pages; i++)
1152                 set_page_attr(bitmap, i,
1153                               BITMAP_PAGE_NEEDWRITE);
1154         bitmap->allclean = 0;
1155 }
1156
1157 static void bitmap_count_page(struct bitmap_counts *bitmap,
1158                               sector_t offset, int inc)
1159 {
1160         sector_t chunk = offset >> bitmap->chunkshift;
1161         unsigned long page = chunk >> PAGE_COUNTER_SHIFT;
1162         bitmap->bp[page].count += inc;
1163         bitmap_checkfree(bitmap, page);
1164 }
1165
1166 static void bitmap_set_pending(struct bitmap_counts *bitmap, sector_t offset)
1167 {
1168         sector_t chunk = offset >> bitmap->chunkshift;
1169         unsigned long page = chunk >> PAGE_COUNTER_SHIFT;
1170         struct bitmap_page *bp = &bitmap->bp[page];
1171
1172         if (!bp->pending)
1173                 bp->pending = 1;
1174 }
1175
1176 static bitmap_counter_t *bitmap_get_counter(struct bitmap_counts *bitmap,
1177                                             sector_t offset, sector_t *blocks,
1178                                             int create);
1179
1180 /*
1181  * bitmap daemon -- periodically wakes up to clean bits and flush pages
1182  *                      out to disk
1183  */
1184
1185 void bitmap_daemon_work(struct mddev *mddev)
1186 {
1187         struct bitmap *bitmap;
1188         unsigned long j;
1189         unsigned long nextpage;
1190         sector_t blocks;
1191         struct bitmap_counts *counts;
1192
1193         /* Use a mutex to guard daemon_work against
1194          * bitmap_destroy.
1195          */
1196         mutex_lock(&mddev->bitmap_info.mutex);
1197         bitmap = mddev->bitmap;
1198         if (bitmap == NULL) {
1199                 mutex_unlock(&mddev->bitmap_info.mutex);
1200                 return;
1201         }
1202         if (time_before(jiffies, bitmap->daemon_lastrun
1203                         + mddev->bitmap_info.daemon_sleep))
1204                 goto done;
1205
1206         bitmap->daemon_lastrun = jiffies;
1207         if (bitmap->allclean) {
1208                 mddev->thread->timeout = MAX_SCHEDULE_TIMEOUT;
1209                 goto done;
1210         }
1211         bitmap->allclean = 1;
1212
1213         /* Any file-page which is PENDING now needs to be written.
1214          * So set NEEDWRITE now, then after we make any last-minute changes
1215          * we will write it.
1216          */
1217         for (j = 0; j < bitmap->storage.file_pages; j++)
1218                 if (test_and_clear_page_attr(bitmap, j,
1219                                              BITMAP_PAGE_PENDING))
1220                         set_page_attr(bitmap, j,
1221                                       BITMAP_PAGE_NEEDWRITE);
1222
1223         if (bitmap->need_sync &&
1224             mddev->bitmap_info.external == 0) {
1225                 /* Arrange for superblock update as well as
1226                  * other changes */
1227                 bitmap_super_t *sb;
1228                 bitmap->need_sync = 0;
1229                 if (bitmap->storage.filemap) {
1230                         sb = kmap_atomic(bitmap->storage.sb_page);
1231                         sb->events_cleared =
1232                                 cpu_to_le64(bitmap->events_cleared);
1233                         kunmap_atomic(sb);
1234                         set_page_attr(bitmap, 0,
1235                                       BITMAP_PAGE_NEEDWRITE);
1236                 }
1237         }
1238         /* Now look at the bitmap counters and if any are '2' or '1',
1239          * decrement and handle accordingly.
1240          */
1241         counts = &bitmap->counts;
1242         spin_lock_irq(&counts->lock);
1243         nextpage = 0;
1244         for (j = 0; j < counts->chunks; j++) {
1245                 bitmap_counter_t *bmc;
1246                 sector_t  block = (sector_t)j << counts->chunkshift;
1247
1248                 if (j == nextpage) {
1249                         nextpage += PAGE_COUNTER_RATIO;
1250                         if (!counts->bp[j >> PAGE_COUNTER_SHIFT].pending) {
1251                                 j |= PAGE_COUNTER_MASK;
1252                                 continue;
1253                         }
1254                         counts->bp[j >> PAGE_COUNTER_SHIFT].pending = 0;
1255                 }
1256                 bmc = bitmap_get_counter(counts,
1257                                          block,
1258                                          &blocks, 0);
1259
1260                 if (!bmc) {
1261                         j |= PAGE_COUNTER_MASK;
1262                         continue;
1263                 }
1264                 if (*bmc == 1 && !bitmap->need_sync) {
1265                         /* We can clear the bit */
1266                         *bmc = 0;
1267                         bitmap_count_page(counts, block, -1);
1268                         bitmap_file_clear_bit(bitmap, block);
1269                 } else if (*bmc && *bmc <= 2) {
1270                         *bmc = 1;
1271                         bitmap_set_pending(counts, block);
1272                         bitmap->allclean = 0;
1273                 }
1274         }
1275         spin_unlock_irq(&counts->lock);
1276
1277         /* Now start writeout on any page in NEEDWRITE that isn't DIRTY.
1278          * DIRTY pages need to be written by bitmap_unplug so it can wait
1279          * for them.
1280          * If we find any DIRTY page we stop there and let bitmap_unplug
1281          * handle all the rest.  This is important in the case where
1282          * the first blocking holds the superblock and it has been updated.
1283          * We mustn't write any other blocks before the superblock.
1284          */
1285         for (j = 0;
1286              j < bitmap->storage.file_pages
1287                      && !test_bit(BITMAP_STALE, &bitmap->flags);
1288              j++) {
1289                 if (test_page_attr(bitmap, j,
1290                                    BITMAP_PAGE_DIRTY))
1291                         /* bitmap_unplug will handle the rest */
1292                         break;
1293                 if (test_and_clear_page_attr(bitmap, j,
1294                                              BITMAP_PAGE_NEEDWRITE)) {
1295                         write_page(bitmap, bitmap->storage.filemap[j], 0);
1296                 }
1297         }
1298
1299  done:
1300         if (bitmap->allclean == 0)
1301                 mddev->thread->timeout =
1302                         mddev->bitmap_info.daemon_sleep;
1303         mutex_unlock(&mddev->bitmap_info.mutex);
1304 }
1305
1306 static bitmap_counter_t *bitmap_get_counter(struct bitmap_counts *bitmap,
1307                                             sector_t offset, sector_t *blocks,
1308                                             int create)
1309 __releases(bitmap->lock)
1310 __acquires(bitmap->lock)
1311 {
1312         /* If 'create', we might release the lock and reclaim it.
1313          * The lock must have been taken with interrupts enabled.
1314          * If !create, we don't release the lock.
1315          */
1316         sector_t chunk = offset >> bitmap->chunkshift;
1317         unsigned long page = chunk >> PAGE_COUNTER_SHIFT;
1318         unsigned long pageoff = (chunk & PAGE_COUNTER_MASK) << COUNTER_BYTE_SHIFT;
1319         sector_t csize;
1320         int err;
1321
1322         err = bitmap_checkpage(bitmap, page, create);
1323
1324         if (bitmap->bp[page].hijacked ||
1325             bitmap->bp[page].map == NULL)
1326                 csize = ((sector_t)1) << (bitmap->chunkshift +
1327                                           PAGE_COUNTER_SHIFT - 1);
1328         else
1329                 csize = ((sector_t)1) << bitmap->chunkshift;
1330         *blocks = csize - (offset & (csize - 1));
1331
1332         if (err < 0)
1333                 return NULL;
1334
1335         /* now locked ... */
1336
1337         if (bitmap->bp[page].hijacked) { /* hijacked pointer */
1338                 /* should we use the first or second counter field
1339                  * of the hijacked pointer? */
1340                 int hi = (pageoff > PAGE_COUNTER_MASK);
1341                 return  &((bitmap_counter_t *)
1342                           &bitmap->bp[page].map)[hi];
1343         } else /* page is allocated */
1344                 return (bitmap_counter_t *)
1345                         &(bitmap->bp[page].map[pageoff]);
1346 }
1347
1348 int bitmap_startwrite(struct bitmap *bitmap, sector_t offset, unsigned long sectors, int behind)
1349 {
1350         if (!bitmap)
1351                 return 0;
1352
1353         if (behind) {
1354                 int bw;
1355                 atomic_inc(&bitmap->behind_writes);
1356                 bw = atomic_read(&bitmap->behind_writes);
1357                 if (bw > bitmap->behind_writes_used)
1358                         bitmap->behind_writes_used = bw;
1359
1360                 pr_debug("inc write-behind count %d/%lu\n",
1361                          bw, bitmap->mddev->bitmap_info.max_write_behind);
1362         }
1363
1364         while (sectors) {
1365                 sector_t blocks;
1366                 bitmap_counter_t *bmc;
1367
1368                 spin_lock_irq(&bitmap->counts.lock);
1369                 bmc = bitmap_get_counter(&bitmap->counts, offset, &blocks, 1);
1370                 if (!bmc) {
1371                         spin_unlock_irq(&bitmap->counts.lock);
1372                         return 0;
1373                 }
1374
1375                 if (unlikely(COUNTER(*bmc) == COUNTER_MAX)) {
1376                         DEFINE_WAIT(__wait);
1377                         /* note that it is safe to do the prepare_to_wait
1378                          * after the test as long as we do it before dropping
1379                          * the spinlock.
1380                          */
1381                         prepare_to_wait(&bitmap->overflow_wait, &__wait,
1382                                         TASK_UNINTERRUPTIBLE);
1383                         spin_unlock_irq(&bitmap->counts.lock);
1384                         schedule();
1385                         finish_wait(&bitmap->overflow_wait, &__wait);
1386                         continue;
1387                 }
1388
1389                 switch (*bmc) {
1390                 case 0:
1391                         bitmap_file_set_bit(bitmap, offset);
1392                         bitmap_count_page(&bitmap->counts, offset, 1);
1393                         /* fall through */
1394                 case 1:
1395                         *bmc = 2;
1396                 }
1397
1398                 (*bmc)++;
1399
1400                 spin_unlock_irq(&bitmap->counts.lock);
1401
1402                 offset += blocks;
1403                 if (sectors > blocks)
1404                         sectors -= blocks;
1405                 else
1406                         sectors = 0;
1407         }
1408         return 0;
1409 }
1410 EXPORT_SYMBOL(bitmap_startwrite);
1411
1412 void bitmap_endwrite(struct bitmap *bitmap, sector_t offset, unsigned long sectors,
1413                      int success, int behind)
1414 {
1415         if (!bitmap)
1416                 return;
1417         if (behind) {
1418                 if (atomic_dec_and_test(&bitmap->behind_writes))
1419                         wake_up(&bitmap->behind_wait);
1420                 pr_debug("dec write-behind count %d/%lu\n",
1421                          atomic_read(&bitmap->behind_writes),
1422                          bitmap->mddev->bitmap_info.max_write_behind);
1423         }
1424
1425         while (sectors) {
1426                 sector_t blocks;
1427                 unsigned long flags;
1428                 bitmap_counter_t *bmc;
1429
1430                 spin_lock_irqsave(&bitmap->counts.lock, flags);
1431                 bmc = bitmap_get_counter(&bitmap->counts, offset, &blocks, 0);
1432                 if (!bmc) {
1433                         spin_unlock_irqrestore(&bitmap->counts.lock, flags);
1434                         return;
1435                 }
1436
1437                 if (success && !bitmap->mddev->degraded &&
1438                     bitmap->events_cleared < bitmap->mddev->events) {
1439                         bitmap->events_cleared = bitmap->mddev->events;
1440                         bitmap->need_sync = 1;
1441                         sysfs_notify_dirent_safe(bitmap->sysfs_can_clear);
1442                 }
1443
1444                 if (!success && !NEEDED(*bmc))
1445                         *bmc |= NEEDED_MASK;
1446
1447                 if (COUNTER(*bmc) == COUNTER_MAX)
1448                         wake_up(&bitmap->overflow_wait);
1449
1450                 (*bmc)--;
1451                 if (*bmc <= 2) {
1452                         bitmap_set_pending(&bitmap->counts, offset);
1453                         bitmap->allclean = 0;
1454                 }
1455                 spin_unlock_irqrestore(&bitmap->counts.lock, flags);
1456                 offset += blocks;
1457                 if (sectors > blocks)
1458                         sectors -= blocks;
1459                 else
1460                         sectors = 0;
1461         }
1462 }
1463 EXPORT_SYMBOL(bitmap_endwrite);
1464
1465 static int __bitmap_start_sync(struct bitmap *bitmap, sector_t offset, sector_t *blocks,
1466                                int degraded)
1467 {
1468         bitmap_counter_t *bmc;
1469         int rv;
1470         if (bitmap == NULL) {/* FIXME or bitmap set as 'failed' */
1471                 *blocks = 1024;
1472                 return 1; /* always resync if no bitmap */
1473         }
1474         spin_lock_irq(&bitmap->counts.lock);
1475         bmc = bitmap_get_counter(&bitmap->counts, offset, blocks, 0);
1476         rv = 0;
1477         if (bmc) {
1478                 /* locked */
1479                 if (RESYNC(*bmc))
1480                         rv = 1;
1481                 else if (NEEDED(*bmc)) {
1482                         rv = 1;
1483                         if (!degraded) { /* don't set/clear bits if degraded */
1484                                 *bmc |= RESYNC_MASK;
1485                                 *bmc &= ~NEEDED_MASK;
1486                         }
1487                 }
1488         }
1489         spin_unlock_irq(&bitmap->counts.lock);
1490         return rv;
1491 }
1492
1493 int bitmap_start_sync(struct bitmap *bitmap, sector_t offset, sector_t *blocks,
1494                       int degraded)
1495 {
1496         /* bitmap_start_sync must always report on multiples of whole
1497          * pages, otherwise resync (which is very PAGE_SIZE based) will
1498          * get confused.
1499          * So call __bitmap_start_sync repeatedly (if needed) until
1500          * At least PAGE_SIZE>>9 blocks are covered.
1501          * Return the 'or' of the result.
1502          */
1503         int rv = 0;
1504         sector_t blocks1;
1505
1506         *blocks = 0;
1507         while (*blocks < (PAGE_SIZE>>9)) {
1508                 rv |= __bitmap_start_sync(bitmap, offset,
1509                                           &blocks1, degraded);
1510                 offset += blocks1;
1511                 *blocks += blocks1;
1512         }
1513         return rv;
1514 }
1515 EXPORT_SYMBOL(bitmap_start_sync);
1516
1517 void bitmap_end_sync(struct bitmap *bitmap, sector_t offset, sector_t *blocks, int aborted)
1518 {
1519         bitmap_counter_t *bmc;
1520         unsigned long flags;
1521
1522         if (bitmap == NULL) {
1523                 *blocks = 1024;
1524                 return;
1525         }
1526         spin_lock_irqsave(&bitmap->counts.lock, flags);
1527         bmc = bitmap_get_counter(&bitmap->counts, offset, blocks, 0);
1528         if (bmc == NULL)
1529                 goto unlock;
1530         /* locked */
1531         if (RESYNC(*bmc)) {
1532                 *bmc &= ~RESYNC_MASK;
1533
1534                 if (!NEEDED(*bmc) && aborted)
1535                         *bmc |= NEEDED_MASK;
1536                 else {
1537                         if (*bmc <= 2) {
1538                                 bitmap_set_pending(&bitmap->counts, offset);
1539                                 bitmap->allclean = 0;
1540                         }
1541                 }
1542         }
1543  unlock:
1544         spin_unlock_irqrestore(&bitmap->counts.lock, flags);
1545 }
1546 EXPORT_SYMBOL(bitmap_end_sync);
1547
1548 void bitmap_close_sync(struct bitmap *bitmap)
1549 {
1550         /* Sync has finished, and any bitmap chunks that weren't synced
1551          * properly have been aborted.  It remains to us to clear the
1552          * RESYNC bit wherever it is still on
1553          */
1554         sector_t sector = 0;
1555         sector_t blocks;
1556         if (!bitmap)
1557                 return;
1558         while (sector < bitmap->mddev->resync_max_sectors) {
1559                 bitmap_end_sync(bitmap, sector, &blocks, 0);
1560                 sector += blocks;
1561         }
1562 }
1563 EXPORT_SYMBOL(bitmap_close_sync);
1564
1565 void bitmap_cond_end_sync(struct bitmap *bitmap, sector_t sector)
1566 {
1567         sector_t s = 0;
1568         sector_t blocks;
1569
1570         if (!bitmap)
1571                 return;
1572         if (sector == 0) {
1573                 bitmap->last_end_sync = jiffies;
1574                 return;
1575         }
1576         if (time_before(jiffies, (bitmap->last_end_sync
1577                                   + bitmap->mddev->bitmap_info.daemon_sleep)))
1578                 return;
1579         wait_event(bitmap->mddev->recovery_wait,
1580                    atomic_read(&bitmap->mddev->recovery_active) == 0);
1581
1582         bitmap->mddev->curr_resync_completed = sector;
1583         set_bit(MD_CHANGE_CLEAN, &bitmap->mddev->flags);
1584         sector &= ~((1ULL << bitmap->counts.chunkshift) - 1);
1585         s = 0;
1586         while (s < sector && s < bitmap->mddev->resync_max_sectors) {
1587                 bitmap_end_sync(bitmap, s, &blocks, 0);
1588                 s += blocks;
1589         }
1590         bitmap->last_end_sync = jiffies;
1591         sysfs_notify(&bitmap->mddev->kobj, NULL, "sync_completed");
1592 }
1593 EXPORT_SYMBOL(bitmap_cond_end_sync);
1594
1595 static void bitmap_set_memory_bits(struct bitmap *bitmap, sector_t offset, int needed)
1596 {
1597         /* For each chunk covered by any of these sectors, set the
1598          * counter to 2 and possibly set resync_needed.  They should all
1599          * be 0 at this point
1600          */
1601
1602         sector_t secs;
1603         bitmap_counter_t *bmc;
1604         spin_lock_irq(&bitmap->counts.lock);
1605         bmc = bitmap_get_counter(&bitmap->counts, offset, &secs, 1);
1606         if (!bmc) {
1607                 spin_unlock_irq(&bitmap->counts.lock);
1608                 return;
1609         }
1610         if (!*bmc) {
1611                 *bmc = 2;
1612                 bitmap_count_page(&bitmap->counts, offset, 1);
1613                 bitmap_set_pending(&bitmap->counts, offset);
1614                 bitmap->allclean = 0;
1615         }
1616         if (needed)
1617                 *bmc |= NEEDED_MASK;
1618         spin_unlock_irq(&bitmap->counts.lock);
1619 }
1620
1621 /* dirty the memory and file bits for bitmap chunks "s" to "e" */
1622 void bitmap_dirty_bits(struct bitmap *bitmap, unsigned long s, unsigned long e)
1623 {
1624         unsigned long chunk;
1625
1626         for (chunk = s; chunk <= e; chunk++) {
1627                 sector_t sec = (sector_t)chunk << bitmap->counts.chunkshift;
1628                 bitmap_set_memory_bits(bitmap, sec, 1);
1629                 bitmap_file_set_bit(bitmap, sec);
1630                 if (sec < bitmap->mddev->recovery_cp)
1631                         /* We are asserting that the array is dirty,
1632                          * so move the recovery_cp address back so
1633                          * that it is obvious that it is dirty
1634                          */
1635                         bitmap->mddev->recovery_cp = sec;
1636         }
1637 }
1638
1639 /*
1640  * flush out any pending updates
1641  */
1642 void bitmap_flush(struct mddev *mddev)
1643 {
1644         struct bitmap *bitmap = mddev->bitmap;
1645         long sleep;
1646
1647         if (!bitmap) /* there was no bitmap */
1648                 return;
1649
1650         /* run the daemon_work three time to ensure everything is flushed
1651          * that can be
1652          */
1653         sleep = mddev->bitmap_info.daemon_sleep * 2;
1654         bitmap->daemon_lastrun -= sleep;
1655         bitmap_daemon_work(mddev);
1656         bitmap->daemon_lastrun -= sleep;
1657         bitmap_daemon_work(mddev);
1658         bitmap->daemon_lastrun -= sleep;
1659         bitmap_daemon_work(mddev);
1660         bitmap_update_sb(bitmap);
1661 }
1662
1663 /*
1664  * free memory that was allocated
1665  */
1666 static void bitmap_free(struct bitmap *bitmap)
1667 {
1668         unsigned long k, pages;
1669         struct bitmap_page *bp;
1670
1671         if (!bitmap) /* there was no bitmap */
1672                 return;
1673
1674         if (mddev_is_clustered(bitmap->mddev) && bitmap->mddev->cluster_info &&
1675                 bitmap->cluster_slot == md_cluster_ops->slot_number(bitmap->mddev))
1676                 md_cluster_stop(bitmap->mddev);
1677
1678         /* Shouldn't be needed - but just in case.... */
1679         wait_event(bitmap->write_wait,
1680                    atomic_read(&bitmap->pending_writes) == 0);
1681
1682         /* release the bitmap file  */
1683         bitmap_file_unmap(&bitmap->storage);
1684
1685         bp = bitmap->counts.bp;
1686         pages = bitmap->counts.pages;
1687
1688         /* free all allocated memory */
1689
1690         if (bp) /* deallocate the page memory */
1691                 for (k = 0; k < pages; k++)
1692                         if (bp[k].map && !bp[k].hijacked)
1693                                 kfree(bp[k].map);
1694         kfree(bp);
1695         kfree(bitmap);
1696 }
1697
1698 void bitmap_destroy(struct mddev *mddev)
1699 {
1700         struct bitmap *bitmap = mddev->bitmap;
1701
1702         if (!bitmap) /* there was no bitmap */
1703                 return;
1704
1705         mutex_lock(&mddev->bitmap_info.mutex);
1706         spin_lock(&mddev->lock);
1707         mddev->bitmap = NULL; /* disconnect from the md device */
1708         spin_unlock(&mddev->lock);
1709         mutex_unlock(&mddev->bitmap_info.mutex);
1710         if (mddev->thread)
1711                 mddev->thread->timeout = MAX_SCHEDULE_TIMEOUT;
1712
1713         if (bitmap->sysfs_can_clear)
1714                 sysfs_put(bitmap->sysfs_can_clear);
1715
1716         bitmap_free(bitmap);
1717 }
1718
1719 /*
1720  * initialize the bitmap structure
1721  * if this returns an error, bitmap_destroy must be called to do clean up
1722  */
1723 struct bitmap *bitmap_create(struct mddev *mddev, int slot)
1724 {
1725         struct bitmap *bitmap;
1726         sector_t blocks = mddev->resync_max_sectors;
1727         struct file *file = mddev->bitmap_info.file;
1728         int err;
1729         struct kernfs_node *bm = NULL;
1730
1731         BUILD_BUG_ON(sizeof(bitmap_super_t) != 256);
1732
1733         BUG_ON(file && mddev->bitmap_info.offset);
1734
1735         bitmap = kzalloc(sizeof(*bitmap), GFP_KERNEL);
1736         if (!bitmap)
1737                 return ERR_PTR(-ENOMEM);
1738
1739         spin_lock_init(&bitmap->counts.lock);
1740         atomic_set(&bitmap->pending_writes, 0);
1741         init_waitqueue_head(&bitmap->write_wait);
1742         init_waitqueue_head(&bitmap->overflow_wait);
1743         init_waitqueue_head(&bitmap->behind_wait);
1744
1745         bitmap->mddev = mddev;
1746         bitmap->cluster_slot = slot;
1747
1748         if (mddev->kobj.sd)
1749                 bm = sysfs_get_dirent(mddev->kobj.sd, "bitmap");
1750         if (bm) {
1751                 bitmap->sysfs_can_clear = sysfs_get_dirent(bm, "can_clear");
1752                 sysfs_put(bm);
1753         } else
1754                 bitmap->sysfs_can_clear = NULL;
1755
1756         bitmap->storage.file = file;
1757         if (file) {
1758                 get_file(file);
1759                 /* As future accesses to this file will use bmap,
1760                  * and bypass the page cache, we must sync the file
1761                  * first.
1762                  */
1763                 vfs_fsync(file, 1);
1764         }
1765         /* read superblock from bitmap file (this sets mddev->bitmap_info.chunksize) */
1766         if (!mddev->bitmap_info.external) {
1767                 /*
1768                  * If 'MD_ARRAY_FIRST_USE' is set, then device-mapper is
1769                  * instructing us to create a new on-disk bitmap instance.
1770                  */
1771                 if (test_and_clear_bit(MD_ARRAY_FIRST_USE, &mddev->flags))
1772                         err = bitmap_new_disk_sb(bitmap);
1773                 else
1774                         err = bitmap_read_sb(bitmap);
1775         } else {
1776                 err = 0;
1777                 if (mddev->bitmap_info.chunksize == 0 ||
1778                     mddev->bitmap_info.daemon_sleep == 0)
1779                         /* chunksize and time_base need to be
1780                          * set first. */
1781                         err = -EINVAL;
1782         }
1783         if (err)
1784                 goto error;
1785
1786         bitmap->daemon_lastrun = jiffies;
1787         err = bitmap_resize(bitmap, blocks, mddev->bitmap_info.chunksize, 1);
1788         if (err)
1789                 goto error;
1790
1791         printk(KERN_INFO "created bitmap (%lu pages) for device %s\n",
1792                bitmap->counts.pages, bmname(bitmap));
1793
1794         err = test_bit(BITMAP_WRITE_ERROR, &bitmap->flags) ? -EIO : 0;
1795         if (err)
1796                 goto error;
1797
1798         return bitmap;
1799  error:
1800         bitmap_free(bitmap);
1801         return ERR_PTR(err);
1802 }
1803
1804 int bitmap_load(struct mddev *mddev)
1805 {
1806         int err = 0;
1807         sector_t start = 0;
1808         sector_t sector = 0;
1809         struct bitmap *bitmap = mddev->bitmap;
1810
1811         if (!bitmap)
1812                 goto out;
1813
1814         /* Clear out old bitmap info first:  Either there is none, or we
1815          * are resuming after someone else has possibly changed things,
1816          * so we should forget old cached info.
1817          * All chunks should be clean, but some might need_sync.
1818          */
1819         while (sector < mddev->resync_max_sectors) {
1820                 sector_t blocks;
1821                 bitmap_start_sync(bitmap, sector, &blocks, 0);
1822                 sector += blocks;
1823         }
1824         bitmap_close_sync(bitmap);
1825
1826         if (mddev->degraded == 0
1827             || bitmap->events_cleared == mddev->events)
1828                 /* no need to keep dirty bits to optimise a
1829                  * re-add of a missing device */
1830                 start = mddev->recovery_cp;
1831
1832         mutex_lock(&mddev->bitmap_info.mutex);
1833         err = bitmap_init_from_disk(bitmap, start);
1834         mutex_unlock(&mddev->bitmap_info.mutex);
1835
1836         if (err)
1837                 goto out;
1838         clear_bit(BITMAP_STALE, &bitmap->flags);
1839
1840         /* Kick recovery in case any bits were set */
1841         set_bit(MD_RECOVERY_NEEDED, &bitmap->mddev->recovery);
1842
1843         mddev->thread->timeout = mddev->bitmap_info.daemon_sleep;
1844         md_wakeup_thread(mddev->thread);
1845
1846         bitmap_update_sb(bitmap);
1847
1848         if (test_bit(BITMAP_WRITE_ERROR, &bitmap->flags))
1849                 err = -EIO;
1850 out:
1851         return err;
1852 }
1853 EXPORT_SYMBOL_GPL(bitmap_load);
1854
1855 /* Loads the bitmap associated with slot and copies the resync information
1856  * to our bitmap
1857  */
1858 int bitmap_copy_from_slot(struct mddev *mddev, int slot,
1859                 sector_t *low, sector_t *high, bool clear_bits)
1860 {
1861         int rv = 0, i, j;
1862         sector_t block, lo = 0, hi = 0;
1863         struct bitmap_counts *counts;
1864         struct bitmap *bitmap = bitmap_create(mddev, slot);
1865
1866         if (IS_ERR(bitmap))
1867                 return PTR_ERR(bitmap);
1868
1869         rv = bitmap_read_sb(bitmap);
1870         if (rv)
1871                 goto err;
1872
1873         rv = bitmap_init_from_disk(bitmap, 0);
1874         if (rv)
1875                 goto err;
1876
1877         counts = &bitmap->counts;
1878         for (j = 0; j < counts->chunks; j++) {
1879                 block = (sector_t)j << counts->chunkshift;
1880                 if (bitmap_file_test_bit(bitmap, block)) {
1881                         if (!lo)
1882                                 lo = block;
1883                         hi = block;
1884                         bitmap_file_clear_bit(bitmap, block);
1885                         bitmap_set_memory_bits(mddev->bitmap, block, 1);
1886                         bitmap_file_set_bit(mddev->bitmap, block);
1887                 }
1888         }
1889
1890         if (clear_bits) {
1891                 bitmap_update_sb(bitmap);
1892                 /* Setting this for the ev_page should be enough.
1893                  * And we do not require both write_all and PAGE_DIRT either
1894                  */
1895                 for (i = 0; i < bitmap->storage.file_pages; i++)
1896                         set_page_attr(bitmap, i, BITMAP_PAGE_DIRTY);
1897                 bitmap_write_all(bitmap);
1898                 bitmap_unplug(bitmap);
1899         }
1900         *low = lo;
1901         *high = hi;
1902 err:
1903         bitmap_free(bitmap);
1904         return rv;
1905 }
1906 EXPORT_SYMBOL_GPL(bitmap_copy_from_slot);
1907
1908
1909 void bitmap_status(struct seq_file *seq, struct bitmap *bitmap)
1910 {
1911         unsigned long chunk_kb;
1912         struct bitmap_counts *counts;
1913
1914         if (!bitmap)
1915                 return;
1916
1917         counts = &bitmap->counts;
1918
1919         chunk_kb = bitmap->mddev->bitmap_info.chunksize >> 10;
1920         seq_printf(seq, "bitmap: %lu/%lu pages [%luKB], "
1921                    "%lu%s chunk",
1922                    counts->pages - counts->missing_pages,
1923                    counts->pages,
1924                    (counts->pages - counts->missing_pages)
1925                    << (PAGE_SHIFT - 10),
1926                    chunk_kb ? chunk_kb : bitmap->mddev->bitmap_info.chunksize,
1927                    chunk_kb ? "KB" : "B");
1928         if (bitmap->storage.file) {
1929                 seq_printf(seq, ", file: ");
1930                 seq_path(seq, &bitmap->storage.file->f_path, " \t\n");
1931         }
1932
1933         seq_printf(seq, "\n");
1934 }
1935
1936 int bitmap_resize(struct bitmap *bitmap, sector_t blocks,
1937                   int chunksize, int init)
1938 {
1939         /* If chunk_size is 0, choose an appropriate chunk size.
1940          * Then possibly allocate new storage space.
1941          * Then quiesce, copy bits, replace bitmap, and re-start
1942          *
1943          * This function is called both to set up the initial bitmap
1944          * and to resize the bitmap while the array is active.
1945          * If this happens as a result of the array being resized,
1946          * chunksize will be zero, and we need to choose a suitable
1947          * chunksize, otherwise we use what we are given.
1948          */
1949         struct bitmap_storage store;
1950         struct bitmap_counts old_counts;
1951         unsigned long chunks;
1952         sector_t block;
1953         sector_t old_blocks, new_blocks;
1954         int chunkshift;
1955         int ret = 0;
1956         long pages;
1957         struct bitmap_page *new_bp;
1958
1959         if (chunksize == 0) {
1960                 /* If there is enough space, leave the chunk size unchanged,
1961                  * else increase by factor of two until there is enough space.
1962                  */
1963                 long bytes;
1964                 long space = bitmap->mddev->bitmap_info.space;
1965
1966                 if (space == 0) {
1967                         /* We don't know how much space there is, so limit
1968                          * to current size - in sectors.
1969                          */
1970                         bytes = DIV_ROUND_UP(bitmap->counts.chunks, 8);
1971                         if (!bitmap->mddev->bitmap_info.external)
1972                                 bytes += sizeof(bitmap_super_t);
1973                         space = DIV_ROUND_UP(bytes, 512);
1974                         bitmap->mddev->bitmap_info.space = space;
1975                 }
1976                 chunkshift = bitmap->counts.chunkshift;
1977                 chunkshift--;
1978                 do {
1979                         /* 'chunkshift' is shift from block size to chunk size */
1980                         chunkshift++;
1981                         chunks = DIV_ROUND_UP_SECTOR_T(blocks, 1 << chunkshift);
1982                         bytes = DIV_ROUND_UP(chunks, 8);
1983                         if (!bitmap->mddev->bitmap_info.external)
1984                                 bytes += sizeof(bitmap_super_t);
1985                 } while (bytes > (space << 9));
1986         } else
1987                 chunkshift = ffz(~chunksize) - BITMAP_BLOCK_SHIFT;
1988
1989         chunks = DIV_ROUND_UP_SECTOR_T(blocks, 1 << chunkshift);
1990         memset(&store, 0, sizeof(store));
1991         if (bitmap->mddev->bitmap_info.offset || bitmap->mddev->bitmap_info.file)
1992                 ret = bitmap_storage_alloc(&store, chunks,
1993                                            !bitmap->mddev->bitmap_info.external,
1994                                            bitmap->cluster_slot);
1995         if (ret)
1996                 goto err;
1997
1998         pages = DIV_ROUND_UP(chunks, PAGE_COUNTER_RATIO);
1999
2000         new_bp = kzalloc(pages * sizeof(*new_bp), GFP_KERNEL);
2001         ret = -ENOMEM;
2002         if (!new_bp) {
2003                 bitmap_file_unmap(&store);
2004                 goto err;
2005         }
2006
2007         if (!init)
2008                 bitmap->mddev->pers->quiesce(bitmap->mddev, 1);
2009
2010         store.file = bitmap->storage.file;
2011         bitmap->storage.file = NULL;
2012
2013         if (store.sb_page && bitmap->storage.sb_page)
2014                 memcpy(page_address(store.sb_page),
2015                        page_address(bitmap->storage.sb_page),
2016                        sizeof(bitmap_super_t));
2017         bitmap_file_unmap(&bitmap->storage);
2018         bitmap->storage = store;
2019
2020         old_counts = bitmap->counts;
2021         bitmap->counts.bp = new_bp;
2022         bitmap->counts.pages = pages;
2023         bitmap->counts.missing_pages = pages;
2024         bitmap->counts.chunkshift = chunkshift;
2025         bitmap->counts.chunks = chunks;
2026         bitmap->mddev->bitmap_info.chunksize = 1 << (chunkshift +
2027                                                      BITMAP_BLOCK_SHIFT);
2028
2029         blocks = min(old_counts.chunks << old_counts.chunkshift,
2030                      chunks << chunkshift);
2031
2032         spin_lock_irq(&bitmap->counts.lock);
2033         for (block = 0; block < blocks; ) {
2034                 bitmap_counter_t *bmc_old, *bmc_new;
2035                 int set;
2036
2037                 bmc_old = bitmap_get_counter(&old_counts, block,
2038                                              &old_blocks, 0);
2039                 set = bmc_old && NEEDED(*bmc_old);
2040
2041                 if (set) {
2042                         bmc_new = bitmap_get_counter(&bitmap->counts, block,
2043                                                      &new_blocks, 1);
2044                         if (*bmc_new == 0) {
2045                                 /* need to set on-disk bits too. */
2046                                 sector_t end = block + new_blocks;
2047                                 sector_t start = block >> chunkshift;
2048                                 start <<= chunkshift;
2049                                 while (start < end) {
2050                                         bitmap_file_set_bit(bitmap, block);
2051                                         start += 1 << chunkshift;
2052                                 }
2053                                 *bmc_new = 2;
2054                                 bitmap_count_page(&bitmap->counts,
2055                                                   block, 1);
2056                                 bitmap_set_pending(&bitmap->counts,
2057                                                    block);
2058                         }
2059                         *bmc_new |= NEEDED_MASK;
2060                         if (new_blocks < old_blocks)
2061                                 old_blocks = new_blocks;
2062                 }
2063                 block += old_blocks;
2064         }
2065
2066         if (!init) {
2067                 int i;
2068                 while (block < (chunks << chunkshift)) {
2069                         bitmap_counter_t *bmc;
2070                         bmc = bitmap_get_counter(&bitmap->counts, block,
2071                                                  &new_blocks, 1);
2072                         if (bmc) {
2073                                 /* new space.  It needs to be resynced, so
2074                                  * we set NEEDED_MASK.
2075                                  */
2076                                 if (*bmc == 0) {
2077                                         *bmc = NEEDED_MASK | 2;
2078                                         bitmap_count_page(&bitmap->counts,
2079                                                           block, 1);
2080                                         bitmap_set_pending(&bitmap->counts,
2081                                                            block);
2082                                 }
2083                         }
2084                         block += new_blocks;
2085                 }
2086                 for (i = 0; i < bitmap->storage.file_pages; i++)
2087                         set_page_attr(bitmap, i, BITMAP_PAGE_DIRTY);
2088         }
2089         spin_unlock_irq(&bitmap->counts.lock);
2090
2091         if (!init) {
2092                 bitmap_unplug(bitmap);
2093                 bitmap->mddev->pers->quiesce(bitmap->mddev, 0);
2094         }
2095         ret = 0;
2096 err:
2097         return ret;
2098 }
2099 EXPORT_SYMBOL_GPL(bitmap_resize);
2100
2101 static ssize_t
2102 location_show(struct mddev *mddev, char *page)
2103 {
2104         ssize_t len;
2105         if (mddev->bitmap_info.file)
2106                 len = sprintf(page, "file");
2107         else if (mddev->bitmap_info.offset)
2108                 len = sprintf(page, "%+lld", (long long)mddev->bitmap_info.offset);
2109         else
2110                 len = sprintf(page, "none");
2111         len += sprintf(page+len, "\n");
2112         return len;
2113 }
2114
2115 static ssize_t
2116 location_store(struct mddev *mddev, const char *buf, size_t len)
2117 {
2118
2119         if (mddev->pers) {
2120                 if (!mddev->pers->quiesce)
2121                         return -EBUSY;
2122                 if (mddev->recovery || mddev->sync_thread)
2123                         return -EBUSY;
2124         }
2125
2126         if (mddev->bitmap || mddev->bitmap_info.file ||
2127             mddev->bitmap_info.offset) {
2128                 /* bitmap already configured.  Only option is to clear it */
2129                 if (strncmp(buf, "none", 4) != 0)
2130                         return -EBUSY;
2131                 if (mddev->pers) {
2132                         mddev->pers->quiesce(mddev, 1);
2133                         bitmap_destroy(mddev);
2134                         mddev->pers->quiesce(mddev, 0);
2135                 }
2136                 mddev->bitmap_info.offset = 0;
2137                 if (mddev->bitmap_info.file) {
2138                         struct file *f = mddev->bitmap_info.file;
2139                         mddev->bitmap_info.file = NULL;
2140                         fput(f);
2141                 }
2142         } else {
2143                 /* No bitmap, OK to set a location */
2144                 long long offset;
2145                 if (strncmp(buf, "none", 4) == 0)
2146                         /* nothing to be done */;
2147                 else if (strncmp(buf, "file:", 5) == 0) {
2148                         /* Not supported yet */
2149                         return -EINVAL;
2150                 } else {
2151                         int rv;
2152                         if (buf[0] == '+')
2153                                 rv = kstrtoll(buf+1, 10, &offset);
2154                         else
2155                                 rv = kstrtoll(buf, 10, &offset);
2156                         if (rv)
2157                                 return rv;
2158                         if (offset == 0)
2159                                 return -EINVAL;
2160                         if (mddev->bitmap_info.external == 0 &&
2161                             mddev->major_version == 0 &&
2162                             offset != mddev->bitmap_info.default_offset)
2163                                 return -EINVAL;
2164                         mddev->bitmap_info.offset = offset;
2165                         if (mddev->pers) {
2166                                 struct bitmap *bitmap;
2167                                 mddev->pers->quiesce(mddev, 1);
2168                                 bitmap = bitmap_create(mddev, -1);
2169                                 if (IS_ERR(bitmap))
2170                                         rv = PTR_ERR(bitmap);
2171                                 else {
2172                                         mddev->bitmap = bitmap;
2173                                         rv = bitmap_load(mddev);
2174                                         if (rv) {
2175                                                 bitmap_destroy(mddev);
2176                                                 mddev->bitmap_info.offset = 0;
2177                                         }
2178                                 }
2179                                 mddev->pers->quiesce(mddev, 0);
2180                                 if (rv)
2181                                         return rv;
2182                         }
2183                 }
2184         }
2185         if (!mddev->external) {
2186                 /* Ensure new bitmap info is stored in
2187                  * metadata promptly.
2188                  */
2189                 set_bit(MD_CHANGE_DEVS, &mddev->flags);
2190                 md_wakeup_thread(mddev->thread);
2191         }
2192         return len;
2193 }
2194
2195 static struct md_sysfs_entry bitmap_location =
2196 __ATTR(location, S_IRUGO|S_IWUSR, location_show, location_store);
2197
2198 /* 'bitmap/space' is the space available at 'location' for the
2199  * bitmap.  This allows the kernel to know when it is safe to
2200  * resize the bitmap to match a resized array.
2201  */
2202 static ssize_t
2203 space_show(struct mddev *mddev, char *page)
2204 {
2205         return sprintf(page, "%lu\n", mddev->bitmap_info.space);
2206 }
2207
2208 static ssize_t
2209 space_store(struct mddev *mddev, const char *buf, size_t len)
2210 {
2211         unsigned long sectors;
2212         int rv;
2213
2214         rv = kstrtoul(buf, 10, &sectors);
2215         if (rv)
2216                 return rv;
2217
2218         if (sectors == 0)
2219                 return -EINVAL;
2220
2221         if (mddev->bitmap &&
2222             sectors < (mddev->bitmap->storage.bytes + 511) >> 9)
2223                 return -EFBIG; /* Bitmap is too big for this small space */
2224
2225         /* could make sure it isn't too big, but that isn't really
2226          * needed - user-space should be careful.
2227          */
2228         mddev->bitmap_info.space = sectors;
2229         return len;
2230 }
2231
2232 static struct md_sysfs_entry bitmap_space =
2233 __ATTR(space, S_IRUGO|S_IWUSR, space_show, space_store);
2234
2235 static ssize_t
2236 timeout_show(struct mddev *mddev, char *page)
2237 {
2238         ssize_t len;
2239         unsigned long secs = mddev->bitmap_info.daemon_sleep / HZ;
2240         unsigned long jifs = mddev->bitmap_info.daemon_sleep % HZ;
2241
2242         len = sprintf(page, "%lu", secs);
2243         if (jifs)
2244                 len += sprintf(page+len, ".%03u", jiffies_to_msecs(jifs));
2245         len += sprintf(page+len, "\n");
2246         return len;
2247 }
2248
2249 static ssize_t
2250 timeout_store(struct mddev *mddev, const char *buf, size_t len)
2251 {
2252         /* timeout can be set at any time */
2253         unsigned long timeout;
2254         int rv = strict_strtoul_scaled(buf, &timeout, 4);
2255         if (rv)
2256                 return rv;
2257
2258         /* just to make sure we don't overflow... */
2259         if (timeout >= LONG_MAX / HZ)
2260                 return -EINVAL;
2261
2262         timeout = timeout * HZ / 10000;
2263
2264         if (timeout >= MAX_SCHEDULE_TIMEOUT)
2265                 timeout = MAX_SCHEDULE_TIMEOUT-1;
2266         if (timeout < 1)
2267                 timeout = 1;
2268         mddev->bitmap_info.daemon_sleep = timeout;
2269         if (mddev->thread) {
2270                 /* if thread->timeout is MAX_SCHEDULE_TIMEOUT, then
2271                  * the bitmap is all clean and we don't need to
2272                  * adjust the timeout right now
2273                  */
2274                 if (mddev->thread->timeout < MAX_SCHEDULE_TIMEOUT) {
2275                         mddev->thread->timeout = timeout;
2276                         md_wakeup_thread(mddev->thread);
2277                 }
2278         }
2279         return len;
2280 }
2281
2282 static struct md_sysfs_entry bitmap_timeout =
2283 __ATTR(time_base, S_IRUGO|S_IWUSR, timeout_show, timeout_store);
2284
2285 static ssize_t
2286 backlog_show(struct mddev *mddev, char *page)
2287 {
2288         return sprintf(page, "%lu\n", mddev->bitmap_info.max_write_behind);
2289 }
2290
2291 static ssize_t
2292 backlog_store(struct mddev *mddev, const char *buf, size_t len)
2293 {
2294         unsigned long backlog;
2295         int rv = kstrtoul(buf, 10, &backlog);
2296         if (rv)
2297                 return rv;
2298         if (backlog > COUNTER_MAX)
2299                 return -EINVAL;
2300         mddev->bitmap_info.max_write_behind = backlog;
2301         return len;
2302 }
2303
2304 static struct md_sysfs_entry bitmap_backlog =
2305 __ATTR(backlog, S_IRUGO|S_IWUSR, backlog_show, backlog_store);
2306
2307 static ssize_t
2308 chunksize_show(struct mddev *mddev, char *page)
2309 {
2310         return sprintf(page, "%lu\n", mddev->bitmap_info.chunksize);
2311 }
2312
2313 static ssize_t
2314 chunksize_store(struct mddev *mddev, const char *buf, size_t len)
2315 {
2316         /* Can only be changed when no bitmap is active */
2317         int rv;
2318         unsigned long csize;
2319         if (mddev->bitmap)
2320                 return -EBUSY;
2321         rv = kstrtoul(buf, 10, &csize);
2322         if (rv)
2323                 return rv;
2324         if (csize < 512 ||
2325             !is_power_of_2(csize))
2326                 return -EINVAL;
2327         mddev->bitmap_info.chunksize = csize;
2328         return len;
2329 }
2330
2331 static struct md_sysfs_entry bitmap_chunksize =
2332 __ATTR(chunksize, S_IRUGO|S_IWUSR, chunksize_show, chunksize_store);
2333
2334 static ssize_t metadata_show(struct mddev *mddev, char *page)
2335 {
2336         if (mddev_is_clustered(mddev))
2337                 return sprintf(page, "clustered\n");
2338         return sprintf(page, "%s\n", (mddev->bitmap_info.external
2339                                       ? "external" : "internal"));
2340 }
2341
2342 static ssize_t metadata_store(struct mddev *mddev, const char *buf, size_t len)
2343 {
2344         if (mddev->bitmap ||
2345             mddev->bitmap_info.file ||
2346             mddev->bitmap_info.offset)
2347                 return -EBUSY;
2348         if (strncmp(buf, "external", 8) == 0)
2349                 mddev->bitmap_info.external = 1;
2350         else if ((strncmp(buf, "internal", 8) == 0) ||
2351                         (strncmp(buf, "clustered", 9) == 0))
2352                 mddev->bitmap_info.external = 0;
2353         else
2354                 return -EINVAL;
2355         return len;
2356 }
2357
2358 static struct md_sysfs_entry bitmap_metadata =
2359 __ATTR(metadata, S_IRUGO|S_IWUSR, metadata_show, metadata_store);
2360
2361 static ssize_t can_clear_show(struct mddev *mddev, char *page)
2362 {
2363         int len;
2364         spin_lock(&mddev->lock);
2365         if (mddev->bitmap)
2366                 len = sprintf(page, "%s\n", (mddev->bitmap->need_sync ?
2367                                              "false" : "true"));
2368         else
2369                 len = sprintf(page, "\n");
2370         spin_unlock(&mddev->lock);
2371         return len;
2372 }
2373
2374 static ssize_t can_clear_store(struct mddev *mddev, const char *buf, size_t len)
2375 {
2376         if (mddev->bitmap == NULL)
2377                 return -ENOENT;
2378         if (strncmp(buf, "false", 5) == 0)
2379                 mddev->bitmap->need_sync = 1;
2380         else if (strncmp(buf, "true", 4) == 0) {
2381                 if (mddev->degraded)
2382                         return -EBUSY;
2383                 mddev->bitmap->need_sync = 0;
2384         } else
2385                 return -EINVAL;
2386         return len;
2387 }
2388
2389 static struct md_sysfs_entry bitmap_can_clear =
2390 __ATTR(can_clear, S_IRUGO|S_IWUSR, can_clear_show, can_clear_store);
2391
2392 static ssize_t
2393 behind_writes_used_show(struct mddev *mddev, char *page)
2394 {
2395         ssize_t ret;
2396         spin_lock(&mddev->lock);
2397         if (mddev->bitmap == NULL)
2398                 ret = sprintf(page, "0\n");
2399         else
2400                 ret = sprintf(page, "%lu\n",
2401                               mddev->bitmap->behind_writes_used);
2402         spin_unlock(&mddev->lock);
2403         return ret;
2404 }
2405
2406 static ssize_t
2407 behind_writes_used_reset(struct mddev *mddev, const char *buf, size_t len)
2408 {
2409         if (mddev->bitmap)
2410                 mddev->bitmap->behind_writes_used = 0;
2411         return len;
2412 }
2413
2414 static struct md_sysfs_entry max_backlog_used =
2415 __ATTR(max_backlog_used, S_IRUGO | S_IWUSR,
2416        behind_writes_used_show, behind_writes_used_reset);
2417
2418 static struct attribute *md_bitmap_attrs[] = {
2419         &bitmap_location.attr,
2420         &bitmap_space.attr,
2421         &bitmap_timeout.attr,
2422         &bitmap_backlog.attr,
2423         &bitmap_chunksize.attr,
2424         &bitmap_metadata.attr,
2425         &bitmap_can_clear.attr,
2426         &max_backlog_used.attr,
2427         NULL
2428 };
2429 struct attribute_group md_bitmap_group = {
2430         .name = "bitmap",
2431         .attrs = md_bitmap_attrs,
2432 };
2433