drivers/md/dm-thin.c

   1 /*
   2  * Copyright (C) 2011 Red Hat UK.
   3  *
   4  * This file is released under the GPL.
   5  */
   6
   7 #include "dm-thin-metadata.h"
   8
   9 #include <linux/device-mapper.h>
  10 #include <linux/dm-io.h>
  11 #include <linux/dm-kcopyd.h>
  12 #include <linux/list.h>
  13 #include <linux/init.h>
  14 #include <linux/module.h>
  15 #include <linux/slab.h>
  16
  17 #define DM_MSG_PREFIX   "thin"
  18
  19 /*
  20  * Tunable constants
  21  */
  22 #define ENDIO_HOOK_POOL_SIZE 10240
  23 #define DEFERRED_SET_SIZE 64
  24 #define MAPPING_POOL_SIZE 1024
  25 #define PRISON_CELLS 1024
  26 #define COMMIT_PERIOD HZ
  27
  28 /*
  29  * The block size of the device holding pool data must be
  30  * between 64KB and 1GB.
  31  */
  32 #define DATA_DEV_BLOCK_SIZE_MIN_SECTORS (64 * 1024 >> SECTOR_SHIFT)
  33 #define DATA_DEV_BLOCK_SIZE_MAX_SECTORS (1024 * 1024 * 1024 >> SECTOR_SHIFT)
  34
  35 /*
  36  * Device id is restricted to 24 bits.
  37  */
  38 #define MAX_DEV_ID ((1 << 24) - 1)
  39
  40 /*
  41  * How do we handle breaking sharing of data blocks?
  42  * =================================================
  43  *
  44  * We use a standard copy-on-write btree to store the mappings for the
  45  * devices (note I'm talking about copy-on-write of the metadata here, not
  46  * the data).  When you take an internal snapshot you clone the root node
  47  * of the origin btree.  After this there is no concept of an origin or a
  48  * snapshot.  They are just two device trees that happen to point to the
  49  * same data blocks.
  50  *
  51  * When we get a write in we decide if it's to a shared data block using
  52  * some timestamp magic.  If it is, we have to break sharing.
  53  *
  54  * Let's say we write to a shared block in what was the origin.  The
  55  * steps are:
  56  *
  57  * i) plug io further to this physical block. (see bio_prison code).
  58  *
  59  * ii) quiesce any read io to that shared data block.  Obviously
  60  * including all devices that share this block.  (see deferred_set code)
  61  *
  62  * iii) copy the data block to a newly allocate block.  This step can be
  63  * missed out if the io covers the block. (schedule_copy).
  64  *
  65  * iv) insert the new mapping into the origin's btree
  66  * (process_prepared_mapping).  This act of inserting breaks some
  67  * sharing of btree nodes between the two devices.  Breaking sharing only
  68  * effects the btree of that specific device.  Btrees for the other
  69  * devices that share the block never change.  The btree for the origin
  70  * device as it was after the last commit is untouched, ie. we're using
  71  * persistent data structures in the functional programming sense.
  72  *
  73  * v) unplug io to this physical block, including the io that triggered
  74  * the breaking of sharing.
  75  *
  76  * Steps (ii) and (iii) occur in parallel.
  77  *
  78  * The metadata _doesn't_ need to be committed before the io continues.  We
  79  * get away with this because the io is always written to a _new_ block.
  80  * If there's a crash, then:
  81  *
  82  * - The origin mapping will point to the old origin block (the shared
  83  * one).  This will contain the data as it was before the io that triggered
  84  * the breaking of sharing came in.
  85  *
  86  * - The snap mapping still points to the old block.  As it would after
  87  * the commit.
  88  *
  89  * The downside of this scheme is the timestamp magic isn't perfect, and
  90  * will continue to think that data block in the snapshot device is shared
  91  * even after the write to the origin has broken sharing.  I suspect data
  92  * blocks will typically be shared by many different devices, so we're
  93  * breaking sharing n + 1 times, rather than n, where n is the number of
  94  * devices that reference this data block.  At the moment I think the
  95  * benefits far, far outweigh the disadvantages.
  96  */
  97
  98 /*----------------------------------------------------------------*/
  99
 100 /*
 101  * Sometimes we can't deal with a bio straight away.  We put them in prison
 102  * where they can't cause any mischief.  Bios are put in a cell identified
 103  * by a key, multiple bios can be in the same cell.  When the cell is
 104  * subsequently unlocked the bios become available.
 105  */
 106 struct bio_prison;
 107
 108 struct cell_key {
 109         int virtual;
 110         dm_thin_id dev;
 111         dm_block_t block;
 112 };
 113
 114 struct cell {
 115         struct hlist_node list;
 116         struct bio_prison *prison;
 117         struct cell_key key;
 118         struct bio *holder;
 119         struct bio_list bios;
 120 };
 121
 122 struct bio_prison {
 123         spinlock_t lock;
 124         mempool_t *cell_pool;
 125
 126         unsigned nr_buckets;
 127         unsigned hash_mask;
 128         struct hlist_head *cells;
 129 };
 130
 131 static uint32_t calc_nr_buckets(unsigned nr_cells)
 132 {
 133         uint32_t n = 128;
 134
 135         nr_cells /= 4;
 136         nr_cells = min(nr_cells, 8192u);
 137
 138         while (n < nr_cells)
 139                 n <<= 1;
 140
 141         return n;
 142 }
 143
 144 /*
 145  * @nr_cells should be the number of cells you want in use _concurrently_.
 146  * Don't confuse it with the number of distinct keys.
 147  */
 148 static struct bio_prison *prison_create(unsigned nr_cells)
 149 {
 150         unsigned i;
 151         uint32_t nr_buckets = calc_nr_buckets(nr_cells);
 152         size_t len = sizeof(struct bio_prison) +
 153                 (sizeof(struct hlist_head) * nr_buckets);
 154         struct bio_prison *prison = kmalloc(len, GFP_KERNEL);
 155
 156         if (!prison)
 157                 return NULL;
 158
 159         spin_lock_init(&prison->lock);
 160         prison->cell_pool = mempool_create_kmalloc_pool(nr_cells,
 161                                                         sizeof(struct cell));
 162         if (!prison->cell_pool) {
 163                 kfree(prison);
 164                 return NULL;
 165         }
 166
 167         prison->nr_buckets = nr_buckets;
 168         prison->hash_mask = nr_buckets - 1;
 169         prison->cells = (struct hlist_head *) (prison + 1);
 170         for (i = 0; i < nr_buckets; i++)
 171                 INIT_HLIST_HEAD(prison->cells + i);
 172
 173         return prison;
 174 }
 175
 176 static void prison_destroy(struct bio_prison *prison)
 177 {
 178         mempool_destroy(prison->cell_pool);
 179         kfree(prison);
 180 }
 181
 182 static uint32_t hash_key(struct bio_prison *prison, struct cell_key *key)
 183 {
 184         const unsigned long BIG_PRIME = 4294967291UL;
 185         uint64_t hash = key->block * BIG_PRIME;
 186
 187         return (uint32_t) (hash & prison->hash_mask);
 188 }
 189
 190 static int keys_equal(struct cell_key *lhs, struct cell_key *rhs)
 191 {
 192                return (lhs->virtual == rhs->virtual) &&
 193                        (lhs->dev == rhs->dev) &&
 194                        (lhs->block == rhs->block);
 195 }
 196
 197 static struct cell *__search_bucket(struct hlist_head *bucket,
 198                                     struct cell_key *key)
 199 {
 200         struct cell *cell;
 201         struct hlist_node *tmp;
 202
 203         hlist_for_each_entry(cell, tmp, bucket, list)
 204                 if (keys_equal(&cell->key, key))
 205                         return cell;
 206
 207         return NULL;
 208 }
 209
 210 /*
 211  * This may block if a new cell needs allocating.  You must ensure that
 212  * cells will be unlocked even if the calling thread is blocked.
 213  *
 214  * Returns 1 if the cell was already held, 0 if @inmate is the new holder.
 215  */
 216 static int bio_detain(struct bio_prison *prison, struct cell_key *key,
 217                       struct bio *inmate, struct cell **ref)
 218 {
 219         int r = 1;
 220         unsigned long flags;
 221         uint32_t hash = hash_key(prison, key);
 222         struct cell *cell, *cell2;
 223
 224         BUG_ON(hash > prison->nr_buckets);
 225
 226         spin_lock_irqsave(&prison->lock, flags);
 227
 228         cell = __search_bucket(prison->cells + hash, key);
 229         if (cell) {
 230                 bio_list_add(&cell->bios, inmate);
 231                 goto out;
 232         }
 233
 234         /*
 235          * Allocate a new cell
 236          */
 237         spin_unlock_irqrestore(&prison->lock, flags);
 238         cell2 = mempool_alloc(prison->cell_pool, GFP_NOIO);
 239         spin_lock_irqsave(&prison->lock, flags);
 240
 241         /*
 242          * We've been unlocked, so we have to double check that
 243          * nobody else has inserted this cell in the meantime.
 244          */
 245         cell = __search_bucket(prison->cells + hash, key);
 246         if (cell) {
 247                 mempool_free(cell2, prison->cell_pool);
 248                 bio_list_add(&cell->bios, inmate);
 249                 goto out;
 250         }
 251
 252         /*
 253          * Use new cell.
 254          */
 255         cell = cell2;
 256
 257         cell->prison = prison;
 258         memcpy(&cell->key, key, sizeof(cell->key));
 259         cell->holder = inmate;
 260         bio_list_init(&cell->bios);
 261         hlist_add_head(&cell->list, prison->cells + hash);
 262
 263         r = 0;
 264
 265 out:
 266         spin_unlock_irqrestore(&prison->lock, flags);
 267
 268         *ref = cell;
 269
 270         return r;
 271 }
 272
 273 /*
 274  * @inmates must have been initialised prior to this call
 275  */
 276 static void __cell_release(struct cell *cell, struct bio_list *inmates)
 277 {
 278         struct bio_prison *prison = cell->prison;
 279
 280         hlist_del(&cell->list);
 281
 282         bio_list_add(inmates, cell->holder);
 283         bio_list_merge(inmates, &cell->bios);
 284
 285         mempool_free(cell, prison->cell_pool);
 286 }
 287
 288 static void cell_release(struct cell *cell, struct bio_list *bios)
 289 {
 290         unsigned long flags;
 291         struct bio_prison *prison = cell->prison;
 292
 293         spin_lock_irqsave(&prison->lock, flags);
 294         __cell_release(cell, bios);
 295         spin_unlock_irqrestore(&prison->lock, flags);
 296 }
 297
 298 /*
 299  * There are a couple of places where we put a bio into a cell briefly
 300  * before taking it out again.  In these situations we know that no other
 301  * bio may be in the cell.  This function releases the cell, and also does
 302  * a sanity check.
 303  */
 304 static void __cell_release_singleton(struct cell *cell, struct bio *bio)
 305 {
 306         hlist_del(&cell->list);
 307         BUG_ON(cell->holder != bio);
 308         BUG_ON(!bio_list_empty(&cell->bios));
 309 }
 310
 311 static void cell_release_singleton(struct cell *cell, struct bio *bio)
 312 {
 313         unsigned long flags;
 314         struct bio_prison *prison = cell->prison;
 315
 316         spin_lock_irqsave(&prison->lock, flags);
 317         __cell_release_singleton(cell, bio);
 318         spin_unlock_irqrestore(&prison->lock, flags);
 319 }
 320
 321 /*
 322  * Sometimes we don't want the holder, just the additional bios.
 323  */
 324 static void __cell_release_no_holder(struct cell *cell, struct bio_list *inmates)
 325 {
 326         struct bio_prison *prison = cell->prison;
 327
 328         hlist_del(&cell->list);
 329         bio_list_merge(inmates, &cell->bios);
 330
 331         mempool_free(cell, prison->cell_pool);
 332 }
 333
 334 static void cell_release_no_holder(struct cell *cell, struct bio_list *inmates)
 335 {
 336         unsigned long flags;
 337         struct bio_prison *prison = cell->prison;
 338
 339         spin_lock_irqsave(&prison->lock, flags);
 340         __cell_release_no_holder(cell, inmates);
 341         spin_unlock_irqrestore(&prison->lock, flags);
 342 }
 343
 344 static void cell_error(struct cell *cell)
 345 {
 346         struct bio_prison *prison = cell->prison;
 347         struct bio_list bios;
 348         struct bio *bio;
 349         unsigned long flags;
 350
 351         bio_list_init(&bios);
 352
 353         spin_lock_irqsave(&prison->lock, flags);
 354         __cell_release(cell, &bios);
 355         spin_unlock_irqrestore(&prison->lock, flags);
 356
 357         while ((bio = bio_list_pop(&bios)))
 358                 bio_io_error(bio);
 359 }
 360
 361 /*----------------------------------------------------------------*/
 362
 363 /*
 364  * We use the deferred set to keep track of pending reads to shared blocks.
 365  * We do this to ensure the new mapping caused by a write isn't performed
 366  * until these prior reads have completed.  Otherwise the insertion of the
 367  * new mapping could free the old block that the read bios are mapped to.
 368  */
 369
 370 struct deferred_set;
 371 struct deferred_entry {
 372         struct deferred_set *ds;
 373         unsigned count;
 374         struct list_head work_items;
 375 };
 376
 377 struct deferred_set {
 378         spinlock_t lock;
 379         unsigned current_entry;
 380         unsigned sweeper;
 381         struct deferred_entry entries[DEFERRED_SET_SIZE];
 382 };
 383
 384 static void ds_init(struct deferred_set *ds)
 385 {
 386         int i;
 387
 388         spin_lock_init(&ds->lock);
 389         ds->current_entry = 0;
 390         ds->sweeper = 0;
 391         for (i = 0; i < DEFERRED_SET_SIZE; i++) {
 392                 ds->entries[i].ds = ds;
 393                 ds->entries[i].count = 0;
 394                 INIT_LIST_HEAD(&ds->entries[i].work_items);
 395         }
 396 }
 397
 398 static struct deferred_entry *ds_inc(struct deferred_set *ds)
 399 {
 400         unsigned long flags;
 401         struct deferred_entry *entry;
 402
 403         spin_lock_irqsave(&ds->lock, flags);
 404         entry = ds->entries + ds->current_entry;
 405         entry->count++;
 406         spin_unlock_irqrestore(&ds->lock, flags);
 407
 408         return entry;
 409 }
 410
 411 static unsigned ds_next(unsigned index)
 412 {
 413         return (index + 1) % DEFERRED_SET_SIZE;
 414 }
 415
 416 static void __sweep(struct deferred_set *ds, struct list_head *head)
 417 {
 418         while ((ds->sweeper != ds->current_entry) &&
 419                !ds->entries[ds->sweeper].count) {
 420                 list_splice_init(&ds->entries[ds->sweeper].work_items, head);
 421                 ds->sweeper = ds_next(ds->sweeper);
 422         }
 423
 424         if ((ds->sweeper == ds->current_entry) && !ds->entries[ds->sweeper].count)
 425                 list_splice_init(&ds->entries[ds->sweeper].work_items, head);
 426 }
 427
 428 static void ds_dec(struct deferred_entry *entry, struct list_head *head)
 429 {
 430         unsigned long flags;
 431
 432         spin_lock_irqsave(&entry->ds->lock, flags);
 433         BUG_ON(!entry->count);
 434         --entry->count;
 435         __sweep(entry->ds, head);
 436         spin_unlock_irqrestore(&entry->ds->lock, flags);
 437 }
 438
 439 /*
 440  * Returns 1 if deferred or 0 if no pending items to delay job.
 441  */
 442 static int ds_add_work(struct deferred_set *ds, struct list_head *work)
 443 {
 444         int r = 1;
 445         unsigned long flags;
 446         unsigned next_entry;
 447
 448         spin_lock_irqsave(&ds->lock, flags);
 449         if ((ds->sweeper == ds->current_entry) &&
 450             !ds->entries[ds->current_entry].count)
 451                 r = 0;
 452         else {
 453                 list_add(work, &ds->entries[ds->current_entry].work_items);
 454                 next_entry = ds_next(ds->current_entry);
 455                 if (!ds->entries[next_entry].count)
 456                         ds->current_entry = next_entry;
 457         }
 458         spin_unlock_irqrestore(&ds->lock, flags);
 459
 460         return r;
 461 }
 462
 463 /*----------------------------------------------------------------*/
 464
 465 /*
 466  * Key building.
 467  */
 468 static void build_data_key(struct dm_thin_device *td,
 469                            dm_block_t b, struct cell_key *key)
 470 {
 471         key->virtual = 0;
 472         key->dev = dm_thin_dev_id(td);
 473         key->block = b;
 474 }
 475
 476 static void build_virtual_key(struct dm_thin_device *td, dm_block_t b,
 477                               struct cell_key *key)
 478 {
 479         key->virtual = 1;
 480         key->dev = dm_thin_dev_id(td);
 481         key->block = b;
 482 }
 483
 484 /*----------------------------------------------------------------*/
 485
 486 /*
 487  * A pool device ties together a metadata device and a data device.  It
 488  * also provides the interface for creating and destroying internal
 489  * devices.
 490  */
 491 struct new_mapping;
 492 struct pool {
 493         struct list_head list;
 494         struct dm_target *ti;   /* Only set if a pool target is bound */
 495
 496         struct mapped_device *pool_md;
 497         struct block_device *md_dev;
 498         struct dm_pool_metadata *pmd;
 499
 500         uint32_t sectors_per_block;
 501         unsigned block_shift;
 502         dm_block_t offset_mask;
 503         dm_block_t low_water_blocks;
 504
 505         unsigned zero_new_blocks:1;
 506         unsigned low_water_triggered:1; /* A dm event has been sent */
 507         unsigned no_free_space:1;       /* A -ENOSPC warning has been issued */
 508
 509         struct bio_prison *prison;
 510         struct dm_kcopyd_client *copier;
 511
 512         struct workqueue_struct *wq;
 513         struct work_struct worker;
 514         struct delayed_work waker;
 515
 516         unsigned ref_count;
 517         unsigned long last_commit_jiffies;
 518
 519         spinlock_t lock;
 520         struct bio_list deferred_bios;
 521         struct bio_list deferred_flush_bios;
 522         struct list_head prepared_mappings;
 523
 524         struct bio_list retry_on_resume_list;
 525
 526         struct deferred_set ds; /* FIXME: move to thin_c */
 527
 528         struct new_mapping *next_mapping;
 529         mempool_t *mapping_pool;
 530         mempool_t *endio_hook_pool;
 531 };
 532
 533 /*
 534  * Target context for a pool.
 535  */
 536 struct pool_c {
 537         struct dm_target *ti;
 538         struct pool *pool;
 539         struct dm_dev *data_dev;
 540         struct dm_dev *metadata_dev;
 541         struct dm_target_callbacks callbacks;
 542
 543         dm_block_t low_water_blocks;
 544         unsigned zero_new_blocks:1;
 545 };
 546
 547 /*
 548  * Target context for a thin.
 549  */
 550 struct thin_c {
 551         struct dm_dev *pool_dev;
 552         dm_thin_id dev_id;
 553
 554         struct pool *pool;
 555         struct dm_thin_device *td;
 556 };
 557
 558 /*----------------------------------------------------------------*/
 559
 560 /*
 561  * A global list of pools that uses a struct mapped_device as a key.
 562  */
 563 static struct dm_thin_pool_table {
 564         struct mutex mutex;
 565         struct list_head pools;
 566 } dm_thin_pool_table;
 567
 568 static void pool_table_init(void)
 569 {
 570         mutex_init(&dm_thin_pool_table.mutex);
 571         INIT_LIST_HEAD(&dm_thin_pool_table.pools);
 572 }
 573
 574 static void __pool_table_insert(struct pool *pool)
 575 {
 576         BUG_ON(!mutex_is_locked(&dm_thin_pool_table.mutex));
 577         list_add(&pool->list, &dm_thin_pool_table.pools);
 578 }
 579
 580 static void __pool_table_remove(struct pool *pool)
 581 {
 582         BUG_ON(!mutex_is_locked(&dm_thin_pool_table.mutex));
 583         list_del(&pool->list);
 584 }
 585
 586 static struct pool *__pool_table_lookup(struct mapped_device *md)
 587 {
 588         struct pool *pool = NULL, *tmp;
 589
 590         BUG_ON(!mutex_is_locked(&dm_thin_pool_table.mutex));
 591
 592         list_for_each_entry(tmp, &dm_thin_pool_table.pools, list) {
 593                 if (tmp->pool_md == md) {
 594                         pool = tmp;
 595                         break;
 596                 }
 597         }
 598
 599         return pool;
 600 }
 601
 602 static struct pool *__pool_table_lookup_metadata_dev(struct block_device *md_dev)
 603 {
 604         struct pool *pool = NULL, *tmp;
 605
 606         BUG_ON(!mutex_is_locked(&dm_thin_pool_table.mutex));
 607
 608         list_for_each_entry(tmp, &dm_thin_pool_table.pools, list) {
 609                 if (tmp->md_dev == md_dev) {
 610                         pool = tmp;
 611                         break;
 612                 }
 613         }
 614
 615         return pool;
 616 }
 617
 618 /*----------------------------------------------------------------*/
 619
 620 static void __requeue_bio_list(struct thin_c *tc, struct bio_list *master)
 621 {
 622         struct bio *bio;
 623         struct bio_list bios;
 624
 625         bio_list_init(&bios);
 626         bio_list_merge(&bios, master);
 627         bio_list_init(master);
 628
 629         while ((bio = bio_list_pop(&bios))) {
 630                 if (dm_get_mapinfo(bio)->ptr == tc)
 631                         bio_endio(bio, DM_ENDIO_REQUEUE);
 632                 else
 633                         bio_list_add(master, bio);
 634         }
 635 }
 636
 637 static void requeue_io(struct thin_c *tc)
 638 {
 639         struct pool *pool = tc->pool;
 640         unsigned long flags;
 641
 642         spin_lock_irqsave(&pool->lock, flags);
 643         __requeue_bio_list(tc, &pool->deferred_bios);
 644         __requeue_bio_list(tc, &pool->retry_on_resume_list);
 645         spin_unlock_irqrestore(&pool->lock, flags);
 646 }
 647
 648 /*
 649  * This section of code contains the logic for processing a thin device's IO.
 650  * Much of the code depends on pool object resources (lists, workqueues, etc)
 651  * but most is exclusively called from the thin target rather than the thin-pool
 652  * target.
 653  */
 654
 655 static dm_block_t get_bio_block(struct thin_c *tc, struct bio *bio)
 656 {
 657         return bio->bi_sector >> tc->pool->block_shift;
 658 }
 659
 660 static void remap(struct thin_c *tc, struct bio *bio, dm_block_t block)
 661 {
 662         struct pool *pool = tc->pool;
 663
 664         bio->bi_bdev = tc->pool_dev->bdev;
 665         bio->bi_sector = (block << pool->block_shift) +
 666                 (bio->bi_sector & pool->offset_mask);
 667 }
 668
 669 static void remap_and_issue(struct thin_c *tc, struct bio *bio,
 670                             dm_block_t block)
 671 {
 672         struct pool *pool = tc->pool;
 673         unsigned long flags;
 674
 675         remap(tc, bio, block);
 676
 677         /*
 678          * Batch together any FUA/FLUSH bios we find and then issue
 679          * a single commit for them in process_deferred_bios().
 680          */
 681         if (bio->bi_rw & (REQ_FLUSH | REQ_FUA)) {
 682                 spin_lock_irqsave(&pool->lock, flags);
 683                 bio_list_add(&pool->deferred_flush_bios, bio);
 684                 spin_unlock_irqrestore(&pool->lock, flags);
 685         } else
 686                 generic_make_request(bio);
 687 }
 688
 689 /*
 690  * wake_worker() is used when new work is queued and when pool_resume is
 691  * ready to continue deferred IO processing.
 692  */
 693 static void wake_worker(struct pool *pool)
 694 {
 695         queue_work(pool->wq, &pool->worker);
 696 }
 697
 698 /*----------------------------------------------------------------*/
 699
 700 /*
 701  * Bio endio functions.
 702  */
 703 struct endio_hook {
 704         struct thin_c *tc;
 705         bio_end_io_t *saved_bi_end_io;
 706         struct deferred_entry *entry;
 707 };
 708
 709 struct new_mapping {
 710         struct list_head list;
 711
 712         int prepared;
 713
 714         struct thin_c *tc;
 715         dm_block_t virt_block;
 716         dm_block_t data_block;
 717         struct cell *cell;
 718         int err;
 719
 720         /*
 721          * If the bio covers the whole area of a block then we can avoid
 722          * zeroing or copying.  Instead this bio is hooked.  The bio will
 723          * still be in the cell, so care has to be taken to avoid issuing
 724          * the bio twice.
 725          */
 726         struct bio *bio;
 727         bio_end_io_t *saved_bi_end_io;
 728 };
 729
 730 static void __maybe_add_mapping(struct new_mapping *m)
 731 {
 732         struct pool *pool = m->tc->pool;
 733
 734         if (list_empty(&m->list) && m->prepared) {
 735                 list_add(&m->list, &pool->prepared_mappings);
 736                 wake_worker(pool);
 737         }
 738 }
 739
 740 static void copy_complete(int read_err, unsigned long write_err, void *context)
 741 {
 742         unsigned long flags;
 743         struct new_mapping *m = context;
 744         struct pool *pool = m->tc->pool;
 745
 746         m->err = read_err || write_err ? -EIO : 0;
 747
 748         spin_lock_irqsave(&pool->lock, flags);
 749         m->prepared = 1;
 750         __maybe_add_mapping(m);
 751         spin_unlock_irqrestore(&pool->lock, flags);
 752 }
 753
 754 static void overwrite_endio(struct bio *bio, int err)
 755 {
 756         unsigned long flags;
 757         struct new_mapping *m = dm_get_mapinfo(bio)->ptr;
 758         struct pool *pool = m->tc->pool;
 759
 760         m->err = err;
 761
 762         spin_lock_irqsave(&pool->lock, flags);
 763         m->prepared = 1;
 764         __maybe_add_mapping(m);
 765         spin_unlock_irqrestore(&pool->lock, flags);
 766 }
 767
 768 static void shared_read_endio(struct bio *bio, int err)
 769 {
 770         struct list_head mappings;
 771         struct new_mapping *m, *tmp;
 772         struct endio_hook *h = dm_get_mapinfo(bio)->ptr;
 773         unsigned long flags;
 774         struct pool *pool = h->tc->pool;
 775
 776         bio->bi_end_io = h->saved_bi_end_io;
 777         bio_endio(bio, err);
 778
 779         INIT_LIST_HEAD(&mappings);
 780         ds_dec(h->entry, &mappings);
 781
 782         spin_lock_irqsave(&pool->lock, flags);
 783         list_for_each_entry_safe(m, tmp, &mappings, list) {
 784                 list_del(&m->list);
 785                 INIT_LIST_HEAD(&m->list);
 786                 __maybe_add_mapping(m);
 787         }
 788         spin_unlock_irqrestore(&pool->lock, flags);
 789
 790         mempool_free(h, pool->endio_hook_pool);
 791 }
 792
 793 /*----------------------------------------------------------------*/
 794
 795 /*
 796  * Workqueue.
 797  */
 798
 799 /*
 800  * Prepared mapping jobs.
 801  */
 802
 803 /*
 804  * This sends the bios in the cell back to the deferred_bios list.
 805  */
 806 static void cell_defer(struct thin_c *tc, struct cell *cell,
 807                        dm_block_t data_block)
 808 {
 809         struct pool *pool = tc->pool;
 810         unsigned long flags;
 811
 812         spin_lock_irqsave(&pool->lock, flags);
 813         cell_release(cell, &pool->deferred_bios);
 814         spin_unlock_irqrestore(&tc->pool->lock, flags);
 815
 816         wake_worker(pool);
 817 }
 818
 819 /*
 820  * Same as cell_defer above, except it omits one particular detainee,
 821  * a write bio that covers the block and has already been processed.
 822  */
 823 static void cell_defer_except(struct thin_c *tc, struct cell *cell)
 824 {
 825         struct bio_list bios;
 826         struct pool *pool = tc->pool;
 827         unsigned long flags;
 828
 829         bio_list_init(&bios);
 830
 831         spin_lock_irqsave(&pool->lock, flags);
 832         cell_release_no_holder(cell, &pool->deferred_bios);
 833         spin_unlock_irqrestore(&pool->lock, flags);
 834
 835         wake_worker(pool);
 836 }
 837
 838 static void process_prepared_mapping(struct new_mapping *m)
 839 {
 840         struct thin_c *tc = m->tc;
 841         struct bio *bio;
 842         int r;
 843
 844         bio = m->bio;
 845         if (bio)
 846                 bio->bi_end_io = m->saved_bi_end_io;
 847
 848         if (m->err) {
 849                 cell_error(m->cell);
 850                 return;
 851         }
 852
 853         /*
 854          * Commit the prepared block into the mapping btree.
 855          * Any I/O for this block arriving after this point will get
 856          * remapped to it directly.
 857          */
 858         r = dm_thin_insert_block(tc->td, m->virt_block, m->data_block);
 859         if (r) {
 860                 DMERR("dm_thin_insert_block() failed");
 861                 cell_error(m->cell);
 862                 return;
 863         }
 864
 865         /*
 866          * Release any bios held while the block was being provisioned.
 867          * If we are processing a write bio that completely covers the block,
 868          * we already processed it so can ignore it now when processing
 869          * the bios in the cell.
 870          */
 871         if (bio) {
 872                 cell_defer_except(tc, m->cell);
 873                 bio_endio(bio, 0);
 874         } else
 875                 cell_defer(tc, m->cell, m->data_block);
 876
 877         list_del(&m->list);
 878         mempool_free(m, tc->pool->mapping_pool);
 879 }
 880
 881 static void process_prepared_mappings(struct pool *pool)
 882 {
 883         unsigned long flags;
 884         struct list_head maps;
 885         struct new_mapping *m, *tmp;
 886
 887         INIT_LIST_HEAD(&maps);
 888         spin_lock_irqsave(&pool->lock, flags);
 889         list_splice_init(&pool->prepared_mappings, &maps);
 890         spin_unlock_irqrestore(&pool->lock, flags);
 891
 892         list_for_each_entry_safe(m, tmp, &maps, list)
 893                 process_prepared_mapping(m);
 894 }
 895
 896 /*
 897  * Deferred bio jobs.
 898  */
 899 static int io_overwrites_block(struct pool *pool, struct bio *bio)
 900 {
 901         return ((bio_data_dir(bio) == WRITE) &&
 902                 !(bio->bi_sector & pool->offset_mask)) &&
 903                 (bio->bi_size == (pool->sectors_per_block << SECTOR_SHIFT));
 904 }
 905
 906 static void save_and_set_endio(struct bio *bio, bio_end_io_t **save,
 907                                bio_end_io_t *fn)
 908 {
 909         *save = bio->bi_end_io;
 910         bio->bi_end_io = fn;
 911 }
 912
 913 static int ensure_next_mapping(struct pool *pool)
 914 {
 915         if (pool->next_mapping)
 916                 return 0;
 917
 918         pool->next_mapping = mempool_alloc(pool->mapping_pool, GFP_ATOMIC);
 919
 920         return pool->next_mapping ? 0 : -ENOMEM;
 921 }
 922
 923 static struct new_mapping *get_next_mapping(struct pool *pool)
 924 {
 925         struct new_mapping *r = pool->next_mapping;
 926
 927         BUG_ON(!pool->next_mapping);
 928
 929         pool->next_mapping = NULL;
 930
 931         return r;
 932 }
 933
 934 static void schedule_copy(struct thin_c *tc, dm_block_t virt_block,
 935                           dm_block_t data_origin, dm_block_t data_dest,
 936                           struct cell *cell, struct bio *bio)
 937 {
 938         int r;
 939         struct pool *pool = tc->pool;
 940         struct new_mapping *m = get_next_mapping(pool);
 941
 942         INIT_LIST_HEAD(&m->list);
 943         m->prepared = 0;
 944         m->tc = tc;
 945         m->virt_block = virt_block;
 946         m->data_block = data_dest;
 947         m->cell = cell;
 948         m->err = 0;
 949         m->bio = NULL;
 950
 951         ds_add_work(&pool->ds, &m->list);
 952
 953         /*
 954          * IO to pool_dev remaps to the pool target's data_dev.
 955          *
 956          * If the whole block of data is being overwritten, we can issue the
 957          * bio immediately. Otherwise we use kcopyd to clone the data first.
 958          */
 959         if (io_overwrites_block(pool, bio)) {
 960                 m->bio = bio;
 961                 save_and_set_endio(bio, &m->saved_bi_end_io, overwrite_endio);
 962                 dm_get_mapinfo(bio)->ptr = m;
 963                 remap_and_issue(tc, bio, data_dest);
 964         } else {
 965                 struct dm_io_region from, to;
 966
 967                 from.bdev = tc->pool_dev->bdev;
 968                 from.sector = data_origin * pool->sectors_per_block;
 969                 from.count = pool->sectors_per_block;
 970
 971                 to.bdev = tc->pool_dev->bdev;
 972                 to.sector = data_dest * pool->sectors_per_block;
 973                 to.count = pool->sectors_per_block;
 974
 975                 r = dm_kcopyd_copy(pool->copier, &from, 1, &to,
 976                                    0, copy_complete, m);
 977                 if (r < 0) {
 978                         mempool_free(m, pool->mapping_pool);
 979                         DMERR("dm_kcopyd_copy() failed");
 980                         cell_error(cell);
 981                 }
 982         }
 983 }
 984
 985 static void schedule_zero(struct thin_c *tc, dm_block_t virt_block,
 986                           dm_block_t data_block, struct cell *cell,
 987                           struct bio *bio)
 988 {
 989         struct pool *pool = tc->pool;
 990         struct new_mapping *m = get_next_mapping(pool);
 991
 992         INIT_LIST_HEAD(&m->list);
 993         m->prepared = 0;
 994         m->tc = tc;
 995         m->virt_block = virt_block;
 996         m->data_block = data_block;
 997         m->cell = cell;
 998         m->err = 0;
 999         m->bio = NULL;
1000
1001         /*
1002          * If the whole block of data is being overwritten or we are not
1003          * zeroing pre-existing data, we can issue the bio immediately.
1004          * Otherwise we use kcopyd to zero the data first.
1005          */
1006         if (!pool->zero_new_blocks)
1007                 process_prepared_mapping(m);
1008
1009         else if (io_overwrites_block(pool, bio)) {
1010                 m->bio = bio;
1011                 save_and_set_endio(bio, &m->saved_bi_end_io, overwrite_endio);
1012                 dm_get_mapinfo(bio)->ptr = m;
1013                 remap_and_issue(tc, bio, data_block);
1014
1015         } else {
1016                 int r;
1017                 struct dm_io_region to;
1018
1019                 to.bdev = tc->pool_dev->bdev;
1020                 to.sector = data_block * pool->sectors_per_block;
1021                 to.count = pool->sectors_per_block;
1022
1023                 r = dm_kcopyd_zero(pool->copier, 1, &to, 0, copy_complete, m);
1024                 if (r < 0) {
1025                         mempool_free(m, pool->mapping_pool);
1026                         DMERR("dm_kcopyd_zero() failed");
1027                         cell_error(cell);
1028                 }
1029         }
1030 }
1031
1032 static int alloc_data_block(struct thin_c *tc, dm_block_t *result)
1033 {
1034         int r;
1035         dm_block_t free_blocks;
1036         unsigned long flags;
1037         struct pool *pool = tc->pool;
1038
1039         r = dm_pool_get_free_block_count(pool->pmd, &free_blocks);
1040         if (r)
1041                 return r;
1042
1043         if (free_blocks <= pool->low_water_blocks && !pool->low_water_triggered) {
1044                 DMWARN("%s: reached low water mark, sending event.",
1045                        dm_device_name(pool->pool_md));
1046                 spin_lock_irqsave(&pool->lock, flags);
1047                 pool->low_water_triggered = 1;
1048                 spin_unlock_irqrestore(&pool->lock, flags);
1049                 dm_table_event(pool->ti->table);
1050         }
1051
1052         if (!free_blocks) {
1053                 if (pool->no_free_space)
1054                         return -ENOSPC;
1055                 else {
1056                         /*
1057                          * Try to commit to see if that will free up some
1058                          * more space.
1059                          */
1060                         r = dm_pool_commit_metadata(pool->pmd);
1061                         if (r) {
1062                                 DMERR("%s: dm_pool_commit_metadata() failed, error = %d",
1063                                       __func__, r);
1064                                 return r;
1065                         }
1066
1067                         r = dm_pool_get_free_block_count(pool->pmd, &free_blocks);
1068                         if (r)
1069                                 return r;
1070
1071                         /*
1072                          * If we still have no space we set a flag to avoid
1073                          * doing all this checking and return -ENOSPC.
1074                          */
1075                         if (!free_blocks) {
1076                                 DMWARN("%s: no free space available.",
1077                                        dm_device_name(pool->pool_md));
1078                                 spin_lock_irqsave(&pool->lock, flags);
1079                                 pool->no_free_space = 1;
1080                                 spin_unlock_irqrestore(&pool->lock, flags);
1081                                 return -ENOSPC;
1082                         }
1083                 }
1084         }
1085
1086         r = dm_pool_alloc_data_block(pool->pmd, result);
1087         if (r)
1088                 return r;
1089
1090         return 0;
1091 }
1092
1093 /*
1094  * If we have run out of space, queue bios until the device is
1095  * resumed, presumably after having been reloaded with more space.
1096  */
1097 static void retry_on_resume(struct bio *bio)
1098 {
1099         struct thin_c *tc = dm_get_mapinfo(bio)->ptr;
1100         struct pool *pool = tc->pool;
1101         unsigned long flags;
1102
1103         spin_lock_irqsave(&pool->lock, flags);
1104         bio_list_add(&pool->retry_on_resume_list, bio);
1105         spin_unlock_irqrestore(&pool->lock, flags);
1106 }
1107
1108 static void no_space(struct cell *cell)
1109 {
1110         struct bio *bio;
1111         struct bio_list bios;
1112
1113         bio_list_init(&bios);
1114         cell_release(cell, &bios);
1115
1116         while ((bio = bio_list_pop(&bios)))
1117                 retry_on_resume(bio);
1118 }
1119
1120 static void break_sharing(struct thin_c *tc, struct bio *bio, dm_block_t block,
1121                           struct cell_key *key,
1122                           struct dm_thin_lookup_result *lookup_result,
1123                           struct cell *cell)
1124 {
1125         int r;
1126         dm_block_t data_block;
1127
1128         r = alloc_data_block(tc, &data_block);
1129         switch (r) {
1130         case 0:
1131                 schedule_copy(tc, block, lookup_result->block,
1132                               data_block, cell, bio);
1133                 break;
1134
1135         case -ENOSPC:
1136                 no_space(cell);
1137                 break;
1138
1139         default:
1140                 DMERR("%s: alloc_data_block() failed, error = %d", __func__, r);
1141                 cell_error(cell);
1142                 break;
1143         }
1144 }
1145
1146 static void process_shared_bio(struct thin_c *tc, struct bio *bio,
1147                                dm_block_t block,
1148                                struct dm_thin_lookup_result *lookup_result)
1149 {
1150         struct cell *cell;
1151         struct pool *pool = tc->pool;
1152         struct cell_key key;
1153
1154         /*
1155          * If cell is already occupied, then sharing is already in the process
1156          * of being broken so we have nothing further to do here.
1157          */
1158         build_data_key(tc->td, lookup_result->block, &key);
1159         if (bio_detain(pool->prison, &key, bio, &cell))
1160                 return;
1161
1162         if (bio_data_dir(bio) == WRITE)
1163                 break_sharing(tc, bio, block, &key, lookup_result, cell);
1164         else {
1165                 struct endio_hook *h;
1166                 h = mempool_alloc(pool->endio_hook_pool, GFP_NOIO);
1167
1168                 h->tc = tc;
1169                 h->entry = ds_inc(&pool->ds);
1170                 save_and_set_endio(bio, &h->saved_bi_end_io, shared_read_endio);
1171                 dm_get_mapinfo(bio)->ptr = h;
1172
1173                 cell_release_singleton(cell, bio);
1174                 remap_and_issue(tc, bio, lookup_result->block);
1175         }
1176 }
1177
1178 static void provision_block(struct thin_c *tc, struct bio *bio, dm_block_t block,
1179                             struct cell *cell)
1180 {
1181         int r;
1182         dm_block_t data_block;
1183
1184         /*
1185          * Remap empty bios (flushes) immediately, without provisioning.
1186          */
1187         if (!bio->bi_size) {
1188                 cell_release_singleton(cell, bio);
1189                 remap_and_issue(tc, bio, 0);
1190                 return;
1191         }
1192
1193         /*
1194          * Fill read bios with zeroes and complete them immediately.
1195          */
1196         if (bio_data_dir(bio) == READ) {
1197                 zero_fill_bio(bio);
1198                 cell_release_singleton(cell, bio);
1199                 bio_endio(bio, 0);
1200                 return;
1201         }
1202
1203         r = alloc_data_block(tc, &data_block);
1204         switch (r) {
1205         case 0:
1206                 schedule_zero(tc, block, data_block, cell, bio);
1207                 break;
1208
1209         case -ENOSPC:
1210                 no_space(cell);
1211                 break;
1212
1213         default:
1214                 DMERR("%s: alloc_data_block() failed, error = %d", __func__, r);
1215                 cell_error(cell);
1216                 break;
1217         }
1218 }
1219
1220 static void process_bio(struct thin_c *tc, struct bio *bio)
1221 {
1222         int r;
1223         dm_block_t block = get_bio_block(tc, bio);
1224         struct cell *cell;
1225         struct cell_key key;
1226         struct dm_thin_lookup_result lookup_result;
1227
1228         /*
1229          * If cell is already occupied, then the block is already
1230          * being provisioned so we have nothing further to do here.
1231          */
1232         build_virtual_key(tc->td, block, &key);
1233         if (bio_detain(tc->pool->prison, &key, bio, &cell))
1234                 return;
1235
1236         r = dm_thin_find_block(tc->td, block, 1, &lookup_result);
1237         switch (r) {
1238         case 0:
1239                 /*
1240                  * We can release this cell now.  This thread is the only
1241                  * one that puts bios into a cell, and we know there were
1242                  * no preceding bios.
1243                  */
1244                 /*
1245                  * TODO: this will probably have to change when discard goes
1246                  * back in.
1247                  */
1248                 cell_release_singleton(cell, bio);
1249
1250                 if (lookup_result.shared)
1251                         process_shared_bio(tc, bio, block, &lookup_result);
1252                 else
1253                         remap_and_issue(tc, bio, lookup_result.block);
1254                 break;
1255
1256         case -ENODATA:
1257                 provision_block(tc, bio, block, cell);
1258                 break;
1259
1260         default:
1261                 DMERR("dm_thin_find_block() failed, error = %d", r);
1262                 bio_io_error(bio);
1263                 break;
1264         }
1265 }
1266
1267 static int need_commit_due_to_time(struct pool *pool)
1268 {
1269         return jiffies < pool->last_commit_jiffies ||
1270                jiffies > pool->last_commit_jiffies + COMMIT_PERIOD;
1271 }
1272
1273 static void process_deferred_bios(struct pool *pool)
1274 {
1275         unsigned long flags;
1276         struct bio *bio;
1277         struct bio_list bios;
1278         int r;
1279
1280         bio_list_init(&bios);
1281
1282         spin_lock_irqsave(&pool->lock, flags);
1283         bio_list_merge(&bios, &pool->deferred_bios);
1284         bio_list_init(&pool->deferred_bios);
1285         spin_unlock_irqrestore(&pool->lock, flags);
1286
1287         while ((bio = bio_list_pop(&bios))) {
1288                 struct thin_c *tc = dm_get_mapinfo(bio)->ptr;
1289                 /*
1290                  * If we've got no free new_mapping structs, and processing
1291                  * this bio might require one, we pause until there are some
1292                  * prepared mappings to process.
1293                  */
1294                 if (ensure_next_mapping(pool)) {
1295                         spin_lock_irqsave(&pool->lock, flags);
1296                         bio_list_merge(&pool->deferred_bios, &bios);
1297                         spin_unlock_irqrestore(&pool->lock, flags);
1298
1299                         break;
1300                 }
1301                 process_bio(tc, bio);
1302         }
1303
1304         /*
1305          * If there are any deferred flush bios, we must commit
1306          * the metadata before issuing them.
1307          */
1308         bio_list_init(&bios);
1309         spin_lock_irqsave(&pool->lock, flags);
1310         bio_list_merge(&bios, &pool->deferred_flush_bios);
1311         bio_list_init(&pool->deferred_flush_bios);
1312         spin_unlock_irqrestore(&pool->lock, flags);
1313
1314         if (bio_list_empty(&bios) && !need_commit_due_to_time(pool))
1315                 return;
1316
1317         r = dm_pool_commit_metadata(pool->pmd);
1318         if (r) {
1319                 DMERR("%s: dm_pool_commit_metadata() failed, error = %d",
1320                       __func__, r);
1321                 while ((bio = bio_list_pop(&bios)))
1322                         bio_io_error(bio);
1323                 return;
1324         }
1325         pool->last_commit_jiffies = jiffies;
1326
1327         while ((bio = bio_list_pop(&bios)))
1328                 generic_make_request(bio);
1329 }
1330
1331 static void do_worker(struct work_struct *ws)
1332 {
1333         struct pool *pool = container_of(ws, struct pool, worker);
1334
1335         process_prepared_mappings(pool);
1336         process_deferred_bios(pool);
1337 }
1338
1339 /*
1340  * We want to commit periodically so that not too much
1341  * unwritten data builds up.
1342  */
1343 static void do_waker(struct work_struct *ws)
1344 {
1345         struct pool *pool = container_of(to_delayed_work(ws), struct pool, waker);
1346         wake_worker(pool);
1347         queue_delayed_work(pool->wq, &pool->waker, COMMIT_PERIOD);
1348 }
1349
1350 /*----------------------------------------------------------------*/
1351
1352 /*
1353  * Mapping functions.
1354  */
1355
1356 /*
1357  * Called only while mapping a thin bio to hand it over to the workqueue.
1358  */
1359 static void thin_defer_bio(struct thin_c *tc, struct bio *bio)
1360 {
1361         unsigned long flags;
1362         struct pool *pool = tc->pool;
1363
1364         spin_lock_irqsave(&pool->lock, flags);
1365         bio_list_add(&pool->deferred_bios, bio);
1366         spin_unlock_irqrestore(&pool->lock, flags);
1367
1368         wake_worker(pool);
1369 }
1370
1371 /*
1372  * Non-blocking function called from the thin target's map function.
1373  */
1374 static int thin_bio_map(struct dm_target *ti, struct bio *bio,
1375                         union map_info *map_context)
1376 {
1377         int r;
1378         struct thin_c *tc = ti->private;
1379         dm_block_t block = get_bio_block(tc, bio);
1380         struct dm_thin_device *td = tc->td;
1381         struct dm_thin_lookup_result result;
1382
1383         /*
1384          * Save the thin context for easy access from the deferred bio later.
1385          */
1386         map_context->ptr = tc;
1387
1388         if (bio->bi_rw & (REQ_FLUSH | REQ_FUA)) {
1389                 thin_defer_bio(tc, bio);
1390                 return DM_MAPIO_SUBMITTED;
1391         }
1392
1393         r = dm_thin_find_block(td, block, 0, &result);
1394
1395         /*
1396          * Note that we defer readahead too.
1397          */
1398         switch (r) {
1399         case 0:
1400                 if (unlikely(result.shared)) {
1401                         /*
1402                          * We have a race condition here between the
1403                          * result.shared value returned by the lookup and
1404                          * snapshot creation, which may cause new
1405                          * sharing.
1406                          *
1407                          * To avoid this always quiesce the origin before
1408                          * taking the snap.  You want to do this anyway to
1409                          * ensure a consistent application view
1410                          * (i.e. lockfs).
1411                          *
1412                          * More distant ancestors are irrelevant. The
1413                          * shared flag will be set in their case.
1414                          */
1415                         thin_defer_bio(tc, bio);
1416                         r = DM_MAPIO_SUBMITTED;
1417                 } else {
1418                         remap(tc, bio, result.block);
1419                         r = DM_MAPIO_REMAPPED;
1420                 }
1421                 break;
1422
1423         case -ENODATA:
1424                 /*
1425                  * In future, the failed dm_thin_find_block above could
1426                  * provide the hint to load the metadata into cache.
1427                  */
1428         case -EWOULDBLOCK:
1429                 thin_defer_bio(tc, bio);
1430                 r = DM_MAPIO_SUBMITTED;
1431                 break;
1432         }
1433
1434         return r;
1435 }
1436
1437 static int pool_is_congested(struct dm_target_callbacks *cb, int bdi_bits)
1438 {
1439         int r;
1440         unsigned long flags;
1441         struct pool_c *pt = container_of(cb, struct pool_c, callbacks);
1442
1443         spin_lock_irqsave(&pt->pool->lock, flags);
1444         r = !bio_list_empty(&pt->pool->retry_on_resume_list);
1445         spin_unlock_irqrestore(&pt->pool->lock, flags);
1446
1447         if (!r) {
1448                 struct request_queue *q = bdev_get_queue(pt->data_dev->bdev);
1449                 r = bdi_congested(&q->backing_dev_info, bdi_bits);
1450         }
1451
1452         return r;
1453 }
1454
1455 static void __requeue_bios(struct pool *pool)
1456 {
1457         bio_list_merge(&pool->deferred_bios, &pool->retry_on_resume_list);
1458         bio_list_init(&pool->retry_on_resume_list);
1459 }
1460
1461 /*----------------------------------------------------------------
1462  * Binding of control targets to a pool object
1463  *--------------------------------------------------------------*/
1464 static int bind_control_target(struct pool *pool, struct dm_target *ti)
1465 {
1466         struct pool_c *pt = ti->private;
1467
1468         pool->ti = ti;
1469         pool->low_water_blocks = pt->low_water_blocks;
1470         pool->zero_new_blocks = pt->zero_new_blocks;
1471
1472         return 0;
1473 }
1474
1475 static void unbind_control_target(struct pool *pool, struct dm_target *ti)
1476 {
1477         if (pool->ti == ti)
1478                 pool->ti = NULL;
1479 }
1480
1481 /*----------------------------------------------------------------
1482  * Pool creation
1483  *--------------------------------------------------------------*/
1484 static void __pool_destroy(struct pool *pool)
1485 {
1486         __pool_table_remove(pool);
1487
1488         if (dm_pool_metadata_close(pool->pmd) < 0)
1489                 DMWARN("%s: dm_pool_metadata_close() failed.", __func__);
1490
1491         prison_destroy(pool->prison);
1492         dm_kcopyd_client_destroy(pool->copier);
1493
1494         if (pool->wq)
1495                 destroy_workqueue(pool->wq);
1496
1497         if (pool->next_mapping)
1498                 mempool_free(pool->next_mapping, pool->mapping_pool);
1499         mempool_destroy(pool->mapping_pool);
1500         mempool_destroy(pool->endio_hook_pool);
1501         kfree(pool);
1502 }
1503
1504 static struct pool *pool_create(struct mapped_device *pool_md,
1505                                 struct block_device *metadata_dev,
1506                                 unsigned long block_size, char **error)
1507 {
1508         int r;
1509         void *err_p;
1510         struct pool *pool;
1511         struct dm_pool_metadata *pmd;
1512
1513         pmd = dm_pool_metadata_open(metadata_dev, block_size);
1514         if (IS_ERR(pmd)) {
1515                 *error = "Error creating metadata object";
1516                 return (struct pool *)pmd;
1517         }
1518
1519         pool = kmalloc(sizeof(*pool), GFP_KERNEL);
1520         if (!pool) {
1521                 *error = "Error allocating memory for pool";
1522                 err_p = ERR_PTR(-ENOMEM);
1523                 goto bad_pool;
1524         }
1525
1526         pool->pmd = pmd;
1527         pool->sectors_per_block = block_size;
1528         pool->block_shift = ffs(block_size) - 1;
1529         pool->offset_mask = block_size - 1;
1530         pool->low_water_blocks = 0;
1531         pool->zero_new_blocks = 1;
1532         pool->prison = prison_create(PRISON_CELLS);
1533         if (!pool->prison) {
1534                 *error = "Error creating pool's bio prison";
1535                 err_p = ERR_PTR(-ENOMEM);
1536                 goto bad_prison;
1537         }
1538
1539         pool->copier = dm_kcopyd_client_create();
1540         if (IS_ERR(pool->copier)) {
1541                 r = PTR_ERR(pool->copier);
1542                 *error = "Error creating pool's kcopyd client";
1543                 err_p = ERR_PTR(r);
1544                 goto bad_kcopyd_client;
1545         }
1546
1547         /*
1548          * Create singlethreaded workqueue that will service all devices
1549          * that use this metadata.
1550          */
1551         pool->wq = alloc_ordered_workqueue("dm-" DM_MSG_PREFIX, WQ_MEM_RECLAIM);
1552         if (!pool->wq) {
1553                 *error = "Error creating pool's workqueue";
1554                 err_p = ERR_PTR(-ENOMEM);
1555                 goto bad_wq;
1556         }
1557
1558         INIT_WORK(&pool->worker, do_worker);
1559         INIT_DELAYED_WORK(&pool->waker, do_waker);
1560         spin_lock_init(&pool->lock);
1561         bio_list_init(&pool->deferred_bios);
1562         bio_list_init(&pool->deferred_flush_bios);
1563         INIT_LIST_HEAD(&pool->prepared_mappings);
1564         pool->low_water_triggered = 0;
1565         pool->no_free_space = 0;
1566         bio_list_init(&pool->retry_on_resume_list);
1567         ds_init(&pool->ds);
1568
1569         pool->next_mapping = NULL;
1570         pool->mapping_pool =
1571                 mempool_create_kmalloc_pool(MAPPING_POOL_SIZE, sizeof(struct new_mapping));
1572         if (!pool->mapping_pool) {
1573                 *error = "Error creating pool's mapping mempool";
1574                 err_p = ERR_PTR(-ENOMEM);
1575                 goto bad_mapping_pool;
1576         }
1577
1578         pool->endio_hook_pool =
1579                 mempool_create_kmalloc_pool(ENDIO_HOOK_POOL_SIZE, sizeof(struct endio_hook));
1580         if (!pool->endio_hook_pool) {
1581                 *error = "Error creating pool's endio_hook mempool";
1582                 err_p = ERR_PTR(-ENOMEM);
1583                 goto bad_endio_hook_pool;
1584         }
1585         pool->ref_count = 1;
1586         pool->last_commit_jiffies = jiffies;
1587         pool->pool_md = pool_md;
1588         pool->md_dev = metadata_dev;
1589         __pool_table_insert(pool);
1590
1591         return pool;
1592
1593 bad_endio_hook_pool:
1594         mempool_destroy(pool->mapping_pool);
1595 bad_mapping_pool:
1596         destroy_workqueue(pool->wq);
1597 bad_wq:
1598         dm_kcopyd_client_destroy(pool->copier);
1599 bad_kcopyd_client:
1600         prison_destroy(pool->prison);
1601 bad_prison:
1602         kfree(pool);
1603 bad_pool:
1604         if (dm_pool_metadata_close(pmd))
1605                 DMWARN("%s: dm_pool_metadata_close() failed.", __func__);
1606
1607         return err_p;
1608 }
1609
1610 static void __pool_inc(struct pool *pool)
1611 {
1612         BUG_ON(!mutex_is_locked(&dm_thin_pool_table.mutex));
1613         pool->ref_count++;
1614 }
1615
1616 static void __pool_dec(struct pool *pool)
1617 {
1618         BUG_ON(!mutex_is_locked(&dm_thin_pool_table.mutex));
1619         BUG_ON(!pool->ref_count);
1620         if (!--pool->ref_count)
1621                 __pool_destroy(pool);
1622 }
1623
1624 static struct pool *__pool_find(struct mapped_device *pool_md,
1625                                 struct block_device *metadata_dev,
1626                                 unsigned long block_size, char **error)
1627 {
1628         struct pool *pool = __pool_table_lookup_metadata_dev(metadata_dev);
1629
1630         if (pool) {
1631                 if (pool->pool_md != pool_md)
1632                         return ERR_PTR(-EBUSY);
1633                 __pool_inc(pool);
1634
1635         } else {
1636                 pool = __pool_table_lookup(pool_md);
1637                 if (pool) {
1638                         if (pool->md_dev != metadata_dev)
1639                                 return ERR_PTR(-EINVAL);
1640                         __pool_inc(pool);
1641
1642                 } else
1643                         pool = pool_create(pool_md, metadata_dev, block_size, error);
1644         }
1645
1646         return pool;
1647 }
1648
1649 /*----------------------------------------------------------------
1650  * Pool target methods
1651  *--------------------------------------------------------------*/
1652 static void pool_dtr(struct dm_target *ti)
1653 {
1654         struct pool_c *pt = ti->private;
1655
1656         mutex_lock(&dm_thin_pool_table.mutex);
1657
1658         unbind_control_target(pt->pool, ti);
1659         __pool_dec(pt->pool);
1660         dm_put_device(ti, pt->metadata_dev);
1661         dm_put_device(ti, pt->data_dev);
1662         kfree(pt);
1663
1664         mutex_unlock(&dm_thin_pool_table.mutex);
1665 }
1666
1667 struct pool_features {
1668         unsigned zero_new_blocks:1;
1669 };
1670
1671 static int parse_pool_features(struct dm_arg_set *as, struct pool_features *pf,
1672                                struct dm_target *ti)
1673 {
1674         int r;
1675         unsigned argc;
1676         const char *arg_name;
1677
1678         static struct dm_arg _args[] = {
1679                 {0, 1, "Invalid number of pool feature arguments"},
1680         };
1681
1682         /*
1683          * No feature arguments supplied.
1684          */
1685         if (!as->argc)
1686                 return 0;
1687
1688         r = dm_read_arg_group(_args, as, &argc, &ti->error);
1689         if (r)
1690                 return -EINVAL;
1691
1692         while (argc && !r) {
1693                 arg_name = dm_shift_arg(as);
1694                 argc--;
1695
1696                 if (!strcasecmp(arg_name, "skip_block_zeroing")) {
1697                         pf->zero_new_blocks = 0;
1698                         continue;
1699                 }
1700
1701                 ti->error = "Unrecognised pool feature requested";
1702                 r = -EINVAL;
1703         }
1704
1705         return r;
1706 }
1707
1708 /*
1709  * thin-pool <metadata dev> <data dev>
1710  *           <data block size (sectors)>
1711  *           <low water mark (blocks)>
1712  *           [<#feature args> [<arg>]*]
1713  *
1714  * Optional feature arguments are:
1715  *           skip_block_zeroing: skips the zeroing of newly-provisioned blocks.
1716  */
1717 static int pool_ctr(struct dm_target *ti, unsigned argc, char **argv)
1718 {
1719         int r;
1720         struct pool_c *pt;
1721         struct pool *pool;
1722         struct pool_features pf;
1723         struct dm_arg_set as;
1724         struct dm_dev *data_dev;
1725         unsigned long block_size;
1726         dm_block_t low_water_blocks;
1727         struct dm_dev *metadata_dev;
1728         sector_t metadata_dev_size;
1729         char b[BDEVNAME_SIZE];
1730
1731         /*
1732          * FIXME Remove validation from scope of lock.
1733          */
1734         mutex_lock(&dm_thin_pool_table.mutex);
1735
1736         if (argc < 4) {
1737                 ti->error = "Invalid argument count";
1738                 r = -EINVAL;
1739                 goto out_unlock;
1740         }
1741         as.argc = argc;
1742         as.argv = argv;
1743
1744         r = dm_get_device(ti, argv[0], FMODE_READ | FMODE_WRITE, &metadata_dev);
1745         if (r) {
1746                 ti->error = "Error opening metadata block device";
1747                 goto out_unlock;
1748         }
1749
1750         metadata_dev_size = i_size_read(metadata_dev->bdev->bd_inode) >> SECTOR_SHIFT;
1751         if (metadata_dev_size > THIN_METADATA_MAX_SECTORS_WARNING)
1752                 DMWARN("Metadata device %s is larger than %u sectors: excess space will not be used.",
1753                        bdevname(metadata_dev->bdev, b), THIN_METADATA_MAX_SECTORS);
1754
1755         r = dm_get_device(ti, argv[1], FMODE_READ | FMODE_WRITE, &data_dev);
1756         if (r) {
1757                 ti->error = "Error getting data device";
1758                 goto out_metadata;
1759         }
1760
1761         if (kstrtoul(argv[2], 10, &block_size) || !block_size ||
1762             block_size < DATA_DEV_BLOCK_SIZE_MIN_SECTORS ||
1763             block_size > DATA_DEV_BLOCK_SIZE_MAX_SECTORS ||
1764             !is_power_of_2(block_size)) {
1765                 ti->error = "Invalid block size";
1766                 r = -EINVAL;
1767                 goto out;
1768         }
1769
1770         if (kstrtoull(argv[3], 10, (unsigned long long *)&low_water_blocks)) {
1771                 ti->error = "Invalid low water mark";
1772                 r = -EINVAL;
1773                 goto out;
1774         }
1775
1776         /*
1777          * Set default pool features.
1778          */
1779         memset(&pf, 0, sizeof(pf));
1780         pf.zero_new_blocks = 1;
1781
1782         dm_consume_args(&as, 4);
1783         r = parse_pool_features(&as, &pf, ti);
1784         if (r)
1785                 goto out;
1786
1787         pt = kzalloc(sizeof(*pt), GFP_KERNEL);
1788         if (!pt) {
1789                 r = -ENOMEM;
1790                 goto out;
1791         }
1792
1793         pool = __pool_find(dm_table_get_md(ti->table), metadata_dev->bdev,
1794                            block_size, &ti->error);
1795         if (IS_ERR(pool)) {
1796                 r = PTR_ERR(pool);
1797                 goto out_free_pt;
1798         }
1799
1800         pt->pool = pool;
1801         pt->ti = ti;
1802         pt->metadata_dev = metadata_dev;
1803         pt->data_dev = data_dev;
1804         pt->low_water_blocks = low_water_blocks;
1805         pt->zero_new_blocks = pf.zero_new_blocks;
1806         ti->num_flush_requests = 1;
1807         ti->num_discard_requests = 0;
1808         ti->private = pt;
1809
1810         pt->callbacks.congested_fn = pool_is_congested;
1811         dm_table_add_target_callbacks(ti->table, &pt->callbacks);
1812
1813         mutex_unlock(&dm_thin_pool_table.mutex);
1814
1815         return 0;
1816
1817 out_free_pt:
1818         kfree(pt);
1819 out:
1820         dm_put_device(ti, data_dev);
1821 out_metadata:
1822         dm_put_device(ti, metadata_dev);
1823 out_unlock:
1824         mutex_unlock(&dm_thin_pool_table.mutex);
1825
1826         return r;
1827 }
1828
1829 static int pool_map(struct dm_target *ti, struct bio *bio,
1830                     union map_info *map_context)
1831 {
1832         int r;
1833         struct pool_c *pt = ti->private;
1834         struct pool *pool = pt->pool;
1835         unsigned long flags;
1836
1837         /*
1838          * As this is a singleton target, ti->begin is always zero.
1839          */
1840         spin_lock_irqsave(&pool->lock, flags);
1841         bio->bi_bdev = pt->data_dev->bdev;
1842         r = DM_MAPIO_REMAPPED;
1843         spin_unlock_irqrestore(&pool->lock, flags);
1844
1845         return r;
1846 }
1847
1848 /*
1849  * Retrieves the number of blocks of the data device from
1850  * the superblock and compares it to the actual device size,
1851  * thus resizing the data device in case it has grown.
1852  *
1853  * This both copes with opening preallocated data devices in the ctr
1854  * being followed by a resume
1855  * -and-
1856  * calling the resume method individually after userspace has
1857  * grown the data device in reaction to a table event.
1858  */
1859 static int pool_preresume(struct dm_target *ti)
1860 {
1861         int r;
1862         struct pool_c *pt = ti->private;
1863         struct pool *pool = pt->pool;
1864         dm_block_t data_size, sb_data_size;
1865
1866         /*
1867          * Take control of the pool object.
1868          */
1869         r = bind_control_target(pool, ti);
1870         if (r)
1871                 return r;
1872
1873         data_size = ti->len >> pool->block_shift;
1874         r = dm_pool_get_data_dev_size(pool->pmd, &sb_data_size);
1875         if (r) {
1876                 DMERR("failed to retrieve data device size");
1877                 return r;
1878         }
1879
1880         if (data_size < sb_data_size) {
1881                 DMERR("pool target too small, is %llu blocks (expected %llu)",
1882                       data_size, sb_data_size);
1883                 return -EINVAL;
1884
1885         } else if (data_size > sb_data_size) {
1886                 r = dm_pool_resize_data_dev(pool->pmd, data_size);
1887                 if (r) {
1888                         DMERR("failed to resize data device");
1889                         return r;
1890                 }
1891
1892                 r = dm_pool_commit_metadata(pool->pmd);
1893                 if (r) {
1894                         DMERR("%s: dm_pool_commit_metadata() failed, error = %d",
1895                               __func__, r);
1896                         return r;
1897                 }
1898         }
1899
1900         return 0;
1901 }
1902
1903 static void pool_resume(struct dm_target *ti)
1904 {
1905         struct pool_c *pt = ti->private;
1906         struct pool *pool = pt->pool;
1907         unsigned long flags;
1908
1909         spin_lock_irqsave(&pool->lock, flags);
1910         pool->low_water_triggered = 0;
1911         pool->no_free_space = 0;
1912         __requeue_bios(pool);
1913         spin_unlock_irqrestore(&pool->lock, flags);
1914
1915         do_waker(&pool->waker.work);
1916 }
1917
1918 static void pool_postsuspend(struct dm_target *ti)
1919 {
1920         int r;
1921         struct pool_c *pt = ti->private;
1922         struct pool *pool = pt->pool;
1923
1924         cancel_delayed_work(&pool->waker);
1925         flush_workqueue(pool->wq);
1926
1927         r = dm_pool_commit_metadata(pool->pmd);
1928         if (r < 0) {
1929                 DMERR("%s: dm_pool_commit_metadata() failed, error = %d",
1930                       __func__, r);
1931                 /* FIXME: invalidate device? error the next FUA or FLUSH bio ?*/
1932         }
1933 }
1934
1935 static int check_arg_count(unsigned argc, unsigned args_required)
1936 {
1937         if (argc != args_required) {
1938                 DMWARN("Message received with %u arguments instead of %u.",
1939                        argc, args_required);
1940                 return -EINVAL;
1941         }
1942
1943         return 0;
1944 }
1945
1946 static int read_dev_id(char *arg, dm_thin_id *dev_id, int warning)
1947 {
1948         if (!kstrtoull(arg, 10, (unsigned long long *)dev_id) &&
1949             *dev_id <= MAX_DEV_ID)
1950                 return 0;
1951
1952         if (warning)
1953                 DMWARN("Message received with invalid device id: %s", arg);
1954
1955         return -EINVAL;
1956 }
1957
1958 static int process_create_thin_mesg(unsigned argc, char **argv, struct pool *pool)
1959 {
1960         dm_thin_id dev_id;
1961         int r;
1962
1963         r = check_arg_count(argc, 2);
1964         if (r)
1965                 return r;
1966
1967         r = read_dev_id(argv[1], &dev_id, 1);
1968         if (r)
1969                 return r;
1970
1971         r = dm_pool_create_thin(pool->pmd, dev_id);
1972         if (r) {
1973                 DMWARN("Creation of new thinly-provisioned device with id %s failed.",
1974                        argv[1]);
1975                 return r;
1976         }
1977
1978         return 0;
1979 }
1980
1981 static int process_create_snap_mesg(unsigned argc, char **argv, struct pool *pool)
1982 {
1983         dm_thin_id dev_id;
1984         dm_thin_id origin_dev_id;
1985         int r;
1986
1987         r = check_arg_count(argc, 3);
1988         if (r)
1989                 return r;
1990
1991         r = read_dev_id(argv[1], &dev_id, 1);
1992         if (r)
1993                 return r;
1994
1995         r = read_dev_id(argv[2], &origin_dev_id, 1);
1996         if (r)
1997                 return r;
1998
1999         r = dm_pool_create_snap(pool->pmd, dev_id, origin_dev_id);
2000         if (r) {
2001                 DMWARN("Creation of new snapshot %s of device %s failed.",
2002                        argv[1], argv[2]);
2003                 return r;
2004         }
2005
2006         return 0;
2007 }
2008
2009 static int process_delete_mesg(unsigned argc, char **argv, struct pool *pool)
2010 {
2011         dm_thin_id dev_id;
2012         int r;
2013
2014         r = check_arg_count(argc, 2);
2015         if (r)
2016                 return r;
2017
2018         r = read_dev_id(argv[1], &dev_id, 1);
2019         if (r)
2020                 return r;
2021
2022         r = dm_pool_delete_thin_device(pool->pmd, dev_id);
2023         if (r)
2024                 DMWARN("Deletion of thin device %s failed.", argv[1]);
2025
2026         return r;
2027 }
2028
2029 static int process_set_transaction_id_mesg(unsigned argc, char **argv, struct pool *pool)
2030 {
2031         dm_thin_id old_id, new_id;
2032         int r;
2033
2034         r = check_arg_count(argc, 3);
2035         if (r)
2036                 return r;
2037
2038         if (kstrtoull(argv[1], 10, (unsigned long long *)&old_id)) {
2039                 DMWARN("set_transaction_id message: Unrecognised id %s.", argv[1]);
2040                 return -EINVAL;
2041         }
2042
2043         if (kstrtoull(argv[2], 10, (unsigned long long *)&new_id)) {
2044                 DMWARN("set_transaction_id message: Unrecognised new id %s.", argv[2]);
2045                 return -EINVAL;
2046         }
2047
2048         r = dm_pool_set_metadata_transaction_id(pool->pmd, old_id, new_id);
2049         if (r) {
2050                 DMWARN("Failed to change transaction id from %s to %s.",
2051                        argv[1], argv[2]);
2052                 return r;
2053         }
2054
2055         return 0;
2056 }
2057
2058 /*
2059  * Messages supported:
2060  *   create_thin        <dev_id>
2061  *   create_snap        <dev_id> <origin_id>
2062  *   delete             <dev_id>
2063  *   trim               <dev_id> <new_size_in_sectors>
2064  *   set_transaction_id <current_trans_id> <new_trans_id>
2065  */
2066 static int pool_message(struct dm_target *ti, unsigned argc, char **argv)
2067 {
2068         int r = -EINVAL;
2069         struct pool_c *pt = ti->private;
2070         struct pool *pool = pt->pool;
2071
2072         if (!strcasecmp(argv[0], "create_thin"))
2073                 r = process_create_thin_mesg(argc, argv, pool);
2074
2075         else if (!strcasecmp(argv[0], "create_snap"))
2076                 r = process_create_snap_mesg(argc, argv, pool);
2077
2078         else if (!strcasecmp(argv[0], "delete"))
2079                 r = process_delete_mesg(argc, argv, pool);
2080
2081         else if (!strcasecmp(argv[0], "set_transaction_id"))
2082                 r = process_set_transaction_id_mesg(argc, argv, pool);
2083
2084         else
2085                 DMWARN("Unrecognised thin pool target message received: %s", argv[0]);
2086
2087         if (!r) {
2088                 r = dm_pool_commit_metadata(pool->pmd);
2089                 if (r)
2090                         DMERR("%s message: dm_pool_commit_metadata() failed, error = %d",
2091                               argv[0], r);
2092         }
2093
2094         return r;
2095 }
2096
2097 /*
2098  * Status line is:
2099  *    <transaction id> <used metadata sectors>/<total metadata sectors>
2100  *    <used data sectors>/<total data sectors> <held metadata root>
2101  */
2102 static int pool_status(struct dm_target *ti, status_type_t type,
2103                        char *result, unsigned maxlen)
2104 {
2105         int r;
2106         unsigned sz = 0;
2107         uint64_t transaction_id;
2108         dm_block_t nr_free_blocks_data;
2109         dm_block_t nr_free_blocks_metadata;
2110         dm_block_t nr_blocks_data;
2111         dm_block_t nr_blocks_metadata;
2112         dm_block_t held_root;
2113         char buf[BDEVNAME_SIZE];
2114         char buf2[BDEVNAME_SIZE];
2115         struct pool_c *pt = ti->private;
2116         struct pool *pool = pt->pool;
2117
2118         switch (type) {
2119         case STATUSTYPE_INFO:
2120                 r = dm_pool_get_metadata_transaction_id(pool->pmd,
2121                                                         &transaction_id);
2122                 if (r)
2123                         return r;
2124
2125                 r = dm_pool_get_free_metadata_block_count(pool->pmd,
2126                                                           &nr_free_blocks_metadata);
2127                 if (r)
2128                         return r;
2129
2130                 r = dm_pool_get_metadata_dev_size(pool->pmd, &nr_blocks_metadata);
2131                 if (r)
2132                         return r;
2133
2134                 r = dm_pool_get_free_block_count(pool->pmd,
2135                                                  &nr_free_blocks_data);
2136                 if (r)
2137                         return r;
2138
2139                 r = dm_pool_get_data_dev_size(pool->pmd, &nr_blocks_data);
2140                 if (r)
2141                         return r;
2142
2143                 r = dm_pool_get_held_metadata_root(pool->pmd, &held_root);
2144                 if (r)
2145                         return r;
2146
2147                 DMEMIT("%llu %llu/%llu %llu/%llu ",
2148                        (unsigned long long)transaction_id,
2149                        (unsigned long long)(nr_blocks_metadata - nr_free_blocks_metadata),
2150                        (unsigned long long)nr_blocks_metadata,
2151                        (unsigned long long)(nr_blocks_data - nr_free_blocks_data),
2152                        (unsigned long long)nr_blocks_data);
2153
2154                 if (held_root)
2155                         DMEMIT("%llu", held_root);
2156                 else
2157                         DMEMIT("-");
2158
2159                 break;
2160
2161         case STATUSTYPE_TABLE:
2162                 DMEMIT("%s %s %lu %llu ",
2163                        format_dev_t(buf, pt->metadata_dev->bdev->bd_dev),
2164                        format_dev_t(buf2, pt->data_dev->bdev->bd_dev),
2165                        (unsigned long)pool->sectors_per_block,
2166                        (unsigned long long)pt->low_water_blocks);
2167
2168                 DMEMIT("%u ", !pool->zero_new_blocks);
2169
2170                 if (!pool->zero_new_blocks)
2171                         DMEMIT("skip_block_zeroing ");
2172                 break;
2173         }
2174
2175         return 0;
2176 }
2177
2178 static int pool_iterate_devices(struct dm_target *ti,
2179                                 iterate_devices_callout_fn fn, void *data)
2180 {
2181         struct pool_c *pt = ti->private;
2182
2183         return fn(ti, pt->data_dev, 0, ti->len, data);
2184 }
2185
2186 static int pool_merge(struct dm_target *ti, struct bvec_merge_data *bvm,
2187                       struct bio_vec *biovec, int max_size)
2188 {
2189         struct pool_c *pt = ti->private;
2190         struct request_queue *q = bdev_get_queue(pt->data_dev->bdev);
2191
2192         if (!q->merge_bvec_fn)
2193                 return max_size;
2194
2195         bvm->bi_bdev = pt->data_dev->bdev;
2196
2197         return min(max_size, q->merge_bvec_fn(q, bvm, biovec));
2198 }
2199
2200 static void pool_io_hints(struct dm_target *ti, struct queue_limits *limits)
2201 {
2202         struct pool_c *pt = ti->private;
2203         struct pool *pool = pt->pool;
2204
2205         blk_limits_io_min(limits, 0);
2206         blk_limits_io_opt(limits, pool->sectors_per_block << SECTOR_SHIFT);
2207 }
2208
2209 static struct target_type pool_target = {
2210         .name = "thin-pool",
2211         .features = DM_TARGET_SINGLETON | DM_TARGET_ALWAYS_WRITEABLE |
2212                     DM_TARGET_IMMUTABLE,
2213         .version = {1, 0, 0},
2214         .module = THIS_MODULE,
2215         .ctr = pool_ctr,
2216         .dtr = pool_dtr,
2217         .map = pool_map,
2218         .postsuspend = pool_postsuspend,
2219         .preresume = pool_preresume,
2220         .resume = pool_resume,
2221         .message = pool_message,
2222         .status = pool_status,
2223         .merge = pool_merge,
2224         .iterate_devices = pool_iterate_devices,
2225         .io_hints = pool_io_hints,
2226 };
2227
2228 /*----------------------------------------------------------------
2229  * Thin target methods
2230  *--------------------------------------------------------------*/
2231 static void thin_dtr(struct dm_target *ti)
2232 {
2233         struct thin_c *tc = ti->private;
2234
2235         mutex_lock(&dm_thin_pool_table.mutex);
2236
2237         __pool_dec(tc->pool);
2238         dm_pool_close_thin_device(tc->td);
2239         dm_put_device(ti, tc->pool_dev);
2240         kfree(tc);
2241
2242         mutex_unlock(&dm_thin_pool_table.mutex);
2243 }
2244
2245 /*
2246  * Thin target parameters:
2247  *
2248  * <pool_dev> <dev_id>
2249  *
2250  * pool_dev: the path to the pool (eg, /dev/mapper/my_pool)
2251  * dev_id: the internal device identifier
2252  */
2253 static int thin_ctr(struct dm_target *ti, unsigned argc, char **argv)
2254 {
2255         int r;
2256         struct thin_c *tc;
2257         struct dm_dev *pool_dev;
2258         struct mapped_device *pool_md;
2259
2260         mutex_lock(&dm_thin_pool_table.mutex);
2261
2262         if (argc != 2) {
2263                 ti->error = "Invalid argument count";
2264                 r = -EINVAL;
2265                 goto out_unlock;
2266         }
2267
2268         tc = ti->private = kzalloc(sizeof(*tc), GFP_KERNEL);
2269         if (!tc) {
2270                 ti->error = "Out of memory";
2271                 r = -ENOMEM;
2272                 goto out_unlock;
2273         }
2274
2275         r = dm_get_device(ti, argv[0], dm_table_get_mode(ti->table), &pool_dev);
2276         if (r) {
2277                 ti->error = "Error opening pool device";
2278                 goto bad_pool_dev;
2279         }
2280         tc->pool_dev = pool_dev;
2281
2282         if (read_dev_id(argv[1], (unsigned long long *)&tc->dev_id, 0)) {
2283                 ti->error = "Invalid device id";
2284                 r = -EINVAL;
2285                 goto bad_common;
2286         }
2287
2288         pool_md = dm_get_md(tc->pool_dev->bdev->bd_dev);
2289         if (!pool_md) {
2290                 ti->error = "Couldn't get pool mapped device";
2291                 r = -EINVAL;
2292                 goto bad_common;
2293         }
2294
2295         tc->pool = __pool_table_lookup(pool_md);
2296         if (!tc->pool) {
2297                 ti->error = "Couldn't find pool object";
2298                 r = -EINVAL;
2299                 goto bad_pool_lookup;
2300         }
2301         __pool_inc(tc->pool);
2302
2303         r = dm_pool_open_thin_device(tc->pool->pmd, tc->dev_id, &tc->td);
2304         if (r) {
2305                 ti->error = "Couldn't open thin internal device";
2306                 goto bad_thin_open;
2307         }
2308
2309         ti->split_io = tc->pool->sectors_per_block;
2310         ti->num_flush_requests = 1;
2311         ti->num_discard_requests = 0;
2312         ti->discards_supported = 0;
2313
2314         dm_put(pool_md);
2315
2316         mutex_unlock(&dm_thin_pool_table.mutex);
2317
2318         return 0;
2319
2320 bad_thin_open:
2321         __pool_dec(tc->pool);
2322 bad_pool_lookup:
2323         dm_put(pool_md);
2324 bad_common:
2325         dm_put_device(ti, tc->pool_dev);
2326 bad_pool_dev:
2327         kfree(tc);
2328 out_unlock:
2329         mutex_unlock(&dm_thin_pool_table.mutex);
2330
2331         return r;
2332 }
2333
2334 static int thin_map(struct dm_target *ti, struct bio *bio,
2335                     union map_info *map_context)
2336 {
2337         bio->bi_sector -= ti->begin;
2338
2339         return thin_bio_map(ti, bio, map_context);
2340 }
2341
2342 static void thin_postsuspend(struct dm_target *ti)
2343 {
2344         if (dm_noflush_suspending(ti))
2345                 requeue_io((struct thin_c *)ti->private);
2346 }
2347
2348 /*
2349  * <nr mapped sectors> <highest mapped sector>
2350  */
2351 static int thin_status(struct dm_target *ti, status_type_t type,
2352                        char *result, unsigned maxlen)
2353 {
2354         int r;
2355         ssize_t sz = 0;
2356         dm_block_t mapped, highest;
2357         char buf[BDEVNAME_SIZE];
2358         struct thin_c *tc = ti->private;
2359
2360         if (!tc->td)
2361                 DMEMIT("-");
2362         else {
2363                 switch (type) {
2364                 case STATUSTYPE_INFO:
2365                         r = dm_thin_get_mapped_count(tc->td, &mapped);
2366                         if (r)
2367                                 return r;
2368
2369                         r = dm_thin_get_highest_mapped_block(tc->td, &highest);
2370                         if (r < 0)
2371                                 return r;
2372
2373                         DMEMIT("%llu ", mapped * tc->pool->sectors_per_block);
2374                         if (r)
2375                                 DMEMIT("%llu", ((highest + 1) *
2376                                                 tc->pool->sectors_per_block) - 1);
2377                         else
2378                                 DMEMIT("-");
2379                         break;
2380
2381                 case STATUSTYPE_TABLE:
2382                         DMEMIT("%s %lu",
2383                                format_dev_t(buf, tc->pool_dev->bdev->bd_dev),
2384                                (unsigned long) tc->dev_id);
2385                         break;
2386                 }
2387         }
2388
2389         return 0;
2390 }
2391
2392 static int thin_iterate_devices(struct dm_target *ti,
2393                                 iterate_devices_callout_fn fn, void *data)
2394 {
2395         dm_block_t blocks;
2396         struct thin_c *tc = ti->private;
2397
2398         /*
2399          * We can't call dm_pool_get_data_dev_size() since that blocks.  So
2400          * we follow a more convoluted path through to the pool's target.
2401          */
2402         if (!tc->pool->ti)
2403                 return 0;       /* nothing is bound */
2404
2405         blocks = tc->pool->ti->len >> tc->pool->block_shift;
2406         if (blocks)
2407                 return fn(ti, tc->pool_dev, 0, tc->pool->sectors_per_block * blocks, data);
2408
2409         return 0;
2410 }
2411
2412 static void thin_io_hints(struct dm_target *ti, struct queue_limits *limits)
2413 {
2414         struct thin_c *tc = ti->private;
2415
2416         blk_limits_io_min(limits, 0);
2417         blk_limits_io_opt(limits, tc->pool->sectors_per_block << SECTOR_SHIFT);
2418 }
2419
2420 static struct target_type thin_target = {
2421         .name = "thin",
2422         .version = {1, 0, 0},
2423         .module = THIS_MODULE,
2424         .ctr = thin_ctr,
2425         .dtr = thin_dtr,
2426         .map = thin_map,
2427         .postsuspend = thin_postsuspend,
2428         .status = thin_status,
2429         .iterate_devices = thin_iterate_devices,
2430         .io_hints = thin_io_hints,
2431 };
2432
2433 /*----------------------------------------------------------------*/
2434
2435 static int __init dm_thin_init(void)
2436 {
2437         int r;
2438
2439         pool_table_init();
2440
2441         r = dm_register_target(&thin_target);
2442         if (r)
2443                 return r;
2444
2445         r = dm_register_target(&pool_target);
2446         if (r)
2447                 dm_unregister_target(&thin_target);
2448
2449         return r;
2450 }
2451
2452 static void dm_thin_exit(void)
2453 {
2454         dm_unregister_target(&thin_target);
2455         dm_unregister_target(&pool_target);
2456 }
2457
2458 module_init(dm_thin_init);
2459 module_exit(dm_thin_exit);
2460
2461 MODULE_DESCRIPTION(DM_NAME "device-mapper thin provisioning target");
2462 MODULE_AUTHOR("Joe Thornber <dm-devel@redhat.com>");
2463 MODULE_LICENSE("GPL");