drivers/md/dm-verity-target.c

   1 /*
   2  * Copyright (C) 2012 Red Hat, Inc.
   3  *
   4  * Author: Mikulas Patocka <mpatocka@redhat.com>
   5  *
   6  * Based on Chromium dm-verity driver (C) 2011 The Chromium OS Authors
   7  *
   8  * This file is released under the GPLv2.
   9  *
  10  * In the file "/sys/module/dm_verity/parameters/prefetch_cluster" you can set
  11  * default prefetch value. Data are read in "prefetch_cluster" chunks from the
  12  * hash device. Setting this greatly improves performance when data and hash
  13  * are on the same disk on different partitions on devices with poor random
  14  * access behavior.
  15  */
  16
  17 #include "dm-bufio.h"
  18
  19 #include <linux/module.h>
  20 #include <linux/device-mapper.h>
  21 #include <linux/reboot.h>
  22 #include <crypto/hash.h>
  23
  24 #define DM_MSG_PREFIX                   "verity"
  25
  26 #define DM_VERITY_ENV_LENGTH            42
  27 #define DM_VERITY_ENV_VAR_NAME          "DM_VERITY_ERR_BLOCK_NR"
  28
  29 #define DM_VERITY_DEFAULT_PREFETCH_SIZE 262144
  30
  31 #define DM_VERITY_MAX_LEVELS            63
  32 #define DM_VERITY_MAX_CORRUPTED_ERRS    100
  33
  34 #define DM_VERITY_OPT_LOGGING           "ignore_corruption"
  35 #define DM_VERITY_OPT_RESTART           "restart_on_corruption"
  36
  37 #define DM_VERITY_OPTS_MAX              1
  38
  39 static unsigned dm_verity_prefetch_cluster = DM_VERITY_DEFAULT_PREFETCH_SIZE;
  40
  41 module_param_named(prefetch_cluster, dm_verity_prefetch_cluster, uint, S_IRUGO | S_IWUSR);
  42
  43 enum verity_mode {
  44         DM_VERITY_MODE_EIO,
  45         DM_VERITY_MODE_LOGGING,
  46         DM_VERITY_MODE_RESTART
  47 };
  48
  49 enum verity_block_type {
  50         DM_VERITY_BLOCK_TYPE_DATA,
  51         DM_VERITY_BLOCK_TYPE_METADATA
  52 };
  53
  54 struct dm_verity {
  55         struct dm_dev *data_dev;
  56         struct dm_dev *hash_dev;
  57         struct dm_target *ti;
  58         struct dm_bufio_client *bufio;
  59         char *alg_name;
  60         struct crypto_shash *tfm;
  61         u8 *root_digest;        /* digest of the root block */
  62         u8 *salt;               /* salt: its size is salt_size */
  63         unsigned salt_size;
  64         sector_t data_start;    /* data offset in 512-byte sectors */
  65         sector_t hash_start;    /* hash start in blocks */
  66         sector_t data_blocks;   /* the number of data blocks */
  67         sector_t hash_blocks;   /* the number of hash blocks */
  68         unsigned char data_dev_block_bits;      /* log2(data blocksize) */
  69         unsigned char hash_dev_block_bits;      /* log2(hash blocksize) */
  70         unsigned char hash_per_block_bits;      /* log2(hashes in hash block) */
  71         unsigned char levels;   /* the number of tree levels */
  72         unsigned char version;
  73         unsigned digest_size;   /* digest size for the current hash algorithm */
  74         unsigned shash_descsize;/* the size of temporary space for crypto */
  75         int hash_failed;        /* set to 1 if hash of any block failed */
  76         enum verity_mode mode;  /* mode for handling verification errors */
  77         unsigned corrupted_errs;/* Number of errors for corrupted blocks */
  78
  79         struct workqueue_struct *verify_wq;
  80
  81         /* starting blocks for each tree level. 0 is the lowest level. */
  82         sector_t hash_level_block[DM_VERITY_MAX_LEVELS];
  83 };
  84
  85 struct dm_verity_io {
  86         struct dm_verity *v;
  87
  88         /* original value of bio->bi_end_io */
  89         bio_end_io_t *orig_bi_end_io;
  90
  91         sector_t block;
  92         unsigned n_blocks;
  93
  94         struct bvec_iter iter;
  95
  96         struct work_struct work;
  97
  98         /*
  99          * Three variably-size fields follow this struct:
 100          *
 101          * u8 hash_desc[v->shash_descsize];
 102          * u8 real_digest[v->digest_size];
 103          * u8 want_digest[v->digest_size];
 104          *
 105          * To access them use: io_hash_desc(), io_real_digest() and io_want_digest().
 106          */
 107 };
 108
 109 struct dm_verity_prefetch_work {
 110         struct work_struct work;
 111         struct dm_verity *v;
 112         sector_t block;
 113         unsigned n_blocks;
 114 };
 115
 116 static struct shash_desc *io_hash_desc(struct dm_verity *v, struct dm_verity_io *io)
 117 {
 118         return (struct shash_desc *)(io + 1);
 119 }
 120
 121 static u8 *io_real_digest(struct dm_verity *v, struct dm_verity_io *io)
 122 {
 123         return (u8 *)(io + 1) + v->shash_descsize;
 124 }
 125
 126 static u8 *io_want_digest(struct dm_verity *v, struct dm_verity_io *io)
 127 {
 128         return (u8 *)(io + 1) + v->shash_descsize + v->digest_size;
 129 }
 130
 131 /*
 132  * Auxiliary structure appended to each dm-bufio buffer. If the value
 133  * hash_verified is nonzero, hash of the block has been verified.
 134  *
 135  * The variable hash_verified is set to 0 when allocating the buffer, then
 136  * it can be changed to 1 and it is never reset to 0 again.
 137  *
 138  * There is no lock around this value, a race condition can at worst cause
 139  * that multiple processes verify the hash of the same buffer simultaneously
 140  * and write 1 to hash_verified simultaneously.
 141  * This condition is harmless, so we don't need locking.
 142  */
 143 struct buffer_aux {
 144         int hash_verified;
 145 };
 146
 147 /*
 148  * Initialize struct buffer_aux for a freshly created buffer.
 149  */
 150 static void dm_bufio_alloc_callback(struct dm_buffer *buf)
 151 {
 152         struct buffer_aux *aux = dm_bufio_get_aux_data(buf);
 153
 154         aux->hash_verified = 0;
 155 }
 156
 157 /*
 158  * Translate input sector number to the sector number on the target device.
 159  */
 160 static sector_t verity_map_sector(struct dm_verity *v, sector_t bi_sector)
 161 {
 162         return v->data_start + dm_target_offset(v->ti, bi_sector);
 163 }
 164
 165 /*
 166  * Return hash position of a specified block at a specified tree level
 167  * (0 is the lowest level).
 168  * The lowest "hash_per_block_bits"-bits of the result denote hash position
 169  * inside a hash block. The remaining bits denote location of the hash block.
 170  */
 171 static sector_t verity_position_at_level(struct dm_verity *v, sector_t block,
 172                                          int level)
 173 {
 174         return block >> (level * v->hash_per_block_bits);
 175 }
 176
 177 /*
 178  * Wrapper for crypto_shash_init, which handles verity salting.
 179  */
 180 static int verity_hash_init(struct dm_verity *v, struct shash_desc *desc)
 181 {
 182         int r;
 183
 184         desc->tfm = v->tfm;
 185         desc->flags = CRYPTO_TFM_REQ_MAY_SLEEP;
 186
 187         r = crypto_shash_init(desc);
 188
 189         if (unlikely(r < 0)) {
 190                 DMERR("crypto_shash_init failed: %d", r);
 191                 return r;
 192         }
 193
 194         if (likely(v->version >= 1)) {
 195                 r = crypto_shash_update(desc, v->salt, v->salt_size);
 196
 197                 if (unlikely(r < 0)) {
 198                         DMERR("crypto_shash_update failed: %d", r);
 199                         return r;
 200                 }
 201         }
 202
 203         return 0;
 204 }
 205
 206 static int verity_hash_update(struct dm_verity *v, struct shash_desc *desc,
 207                               const u8 *data, size_t len)
 208 {
 209         int r = crypto_shash_update(desc, data, len);
 210
 211         if (unlikely(r < 0))
 212                 DMERR("crypto_shash_update failed: %d", r);
 213
 214         return r;
 215 }
 216
 217 static int verity_hash_final(struct dm_verity *v, struct shash_desc *desc,
 218                              u8 *digest)
 219 {
 220         int r;
 221
 222         if (unlikely(!v->version)) {
 223                 r = crypto_shash_update(desc, v->salt, v->salt_size);
 224
 225                 if (r < 0) {
 226                         DMERR("crypto_shash_update failed: %d", r);
 227                         return r;
 228                 }
 229         }
 230
 231         r = crypto_shash_final(desc, digest);
 232
 233         if (unlikely(r < 0))
 234                 DMERR("crypto_shash_final failed: %d", r);
 235
 236         return r;
 237 }
 238
 239 static int verity_hash(struct dm_verity *v, struct shash_desc *desc,
 240                        const u8 *data, size_t len, u8 *digest)
 241 {
 242         int r;
 243
 244         r = verity_hash_init(v, desc);
 245         if (unlikely(r < 0))
 246                 return r;
 247
 248         r = verity_hash_update(v, desc, data, len);
 249         if (unlikely(r < 0))
 250                 return r;
 251
 252         return verity_hash_final(v, desc, digest);
 253 }
 254
 255 static void verity_hash_at_level(struct dm_verity *v, sector_t block, int level,
 256                                  sector_t *hash_block, unsigned *offset)
 257 {
 258         sector_t position = verity_position_at_level(v, block, level);
 259         unsigned idx;
 260
 261         *hash_block = v->hash_level_block[level] + (position >> v->hash_per_block_bits);
 262
 263         if (!offset)
 264                 return;
 265
 266         idx = position & ((1 << v->hash_per_block_bits) - 1);
 267         if (!v->version)
 268                 *offset = idx * v->digest_size;
 269         else
 270                 *offset = idx << (v->hash_dev_block_bits - v->hash_per_block_bits);
 271 }
 272
 273 /*
 274  * Handle verification errors.
 275  */
 276 static int verity_handle_err(struct dm_verity *v, enum verity_block_type type,
 277                              unsigned long long block)
 278 {
 279         char verity_env[DM_VERITY_ENV_LENGTH];
 280         char *envp[] = { verity_env, NULL };
 281         const char *type_str = "";
 282         struct mapped_device *md = dm_table_get_md(v->ti->table);
 283
 284         /* Corruption should be visible in device status in all modes */
 285         v->hash_failed = 1;
 286
 287         if (v->corrupted_errs >= DM_VERITY_MAX_CORRUPTED_ERRS)
 288                 goto out;
 289
 290         v->corrupted_errs++;
 291
 292         switch (type) {
 293         case DM_VERITY_BLOCK_TYPE_DATA:
 294                 type_str = "data";
 295                 break;
 296         case DM_VERITY_BLOCK_TYPE_METADATA:
 297                 type_str = "metadata";
 298                 break;
 299         default:
 300                 BUG();
 301         }
 302
 303         DMERR("%s: %s block %llu is corrupted", v->data_dev->name, type_str,
 304                 block);
 305
 306         if (v->corrupted_errs == DM_VERITY_MAX_CORRUPTED_ERRS)
 307                 DMERR("%s: reached maximum errors", v->data_dev->name);
 308
 309         snprintf(verity_env, DM_VERITY_ENV_LENGTH, "%s=%d,%llu",
 310                 DM_VERITY_ENV_VAR_NAME, type, block);
 311
 312         kobject_uevent_env(&disk_to_dev(dm_disk(md))->kobj, KOBJ_CHANGE, envp);
 313
 314 out:
 315         if (v->mode == DM_VERITY_MODE_LOGGING)
 316                 return 0;
 317
 318         if (v->mode == DM_VERITY_MODE_RESTART)
 319                 kernel_restart("dm-verity device corrupted");
 320
 321         return 1;
 322 }
 323
 324 /*
 325  * Verify hash of a metadata block pertaining to the specified data block
 326  * ("block" argument) at a specified level ("level" argument).
 327  *
 328  * On successful return, io_want_digest(v, io) contains the hash value for
 329  * a lower tree level or for the data block (if we're at the lowest leve).
 330  *
 331  * If "skip_unverified" is true, unverified buffer is skipped and 1 is returned.
 332  * If "skip_unverified" is false, unverified buffer is hashed and verified
 333  * against current value of io_want_digest(v, io).
 334  */
 335 static int verity_verify_level(struct dm_verity *v, struct dm_verity_io *io,
 336                                sector_t block, int level, bool skip_unverified,
 337                                u8 *want_digest)
 338 {
 339         struct dm_buffer *buf;
 340         struct buffer_aux *aux;
 341         u8 *data;
 342         int r;
 343         sector_t hash_block;
 344         unsigned offset;
 345
 346         verity_hash_at_level(v, block, level, &hash_block, &offset);
 347
 348         data = dm_bufio_read(v->bufio, hash_block, &buf);
 349         if (IS_ERR(data))
 350                 return PTR_ERR(data);
 351
 352         aux = dm_bufio_get_aux_data(buf);
 353
 354         if (!aux->hash_verified) {
 355                 if (skip_unverified) {
 356                         r = 1;
 357                         goto release_ret_r;
 358                 }
 359
 360                 r = verity_hash(v, io_hash_desc(v, io),
 361                                 data, 1 << v->hash_dev_block_bits,
 362                                 io_real_digest(v, io));
 363                 if (unlikely(r < 0))
 364                         goto release_ret_r;
 365
 366                 if (likely(memcmp(io_real_digest(v, io), want_digest,
 367                                   v->digest_size) == 0))
 368                         aux->hash_verified = 1;
 369                 else if (verity_handle_err(v,
 370                                            DM_VERITY_BLOCK_TYPE_METADATA,
 371                                            hash_block)) {
 372                         r = -EIO;
 373                         goto release_ret_r;
 374                 }
 375         }
 376
 377         data += offset;
 378         memcpy(want_digest, data, v->digest_size);
 379         r = 0;
 380
 381 release_ret_r:
 382         dm_bufio_release(buf);
 383         return r;
 384 }
 385
 386 /*
 387  * Find a hash for a given block, write it to digest and verify the integrity
 388  * of the hash tree if necessary.
 389  */
 390 static int verity_hash_for_block(struct dm_verity *v, struct dm_verity_io *io,
 391                                  sector_t block, u8 *digest)
 392 {
 393         int i;
 394         int r;
 395
 396         if (likely(v->levels)) {
 397                 /*
 398                  * First, we try to get the requested hash for
 399                  * the current block. If the hash block itself is
 400                  * verified, zero is returned. If it isn't, this
 401                  * function returns 1 and we fall back to whole
 402                  * chain verification.
 403                  */
 404                 r = verity_verify_level(v, io, block, 0, true, digest);
 405                 if (likely(r <= 0))
 406                         return r;
 407         }
 408
 409         memcpy(digest, v->root_digest, v->digest_size);
 410
 411         for (i = v->levels - 1; i >= 0; i--) {
 412                 r = verity_verify_level(v, io, block, i, false, digest);
 413                 if (unlikely(r))
 414                         return r;
 415         }
 416
 417         return 0;
 418 }
 419
 420 /*
 421  * Verify one "dm_verity_io" structure.
 422  */
 423 static int verity_verify_io(struct dm_verity_io *io)
 424 {
 425         struct dm_verity *v = io->v;
 426         struct bio *bio = dm_bio_from_per_bio_data(io,
 427                                                    v->ti->per_bio_data_size);
 428         unsigned b;
 429
 430         for (b = 0; b < io->n_blocks; b++) {
 431                 int r;
 432                 unsigned todo;
 433                 struct shash_desc *desc = io_hash_desc(v, io);
 434
 435                 r = verity_hash_for_block(v, io, io->block + b,
 436                                           io_want_digest(v, io));
 437                 if (unlikely(r < 0))
 438                         return r;
 439
 440                 r = verity_hash_init(v, desc);
 441                 if (unlikely(r < 0))
 442                         return r;
 443
 444                 todo = 1 << v->data_dev_block_bits;
 445                 do {
 446                         u8 *page;
 447                         unsigned len;
 448                         struct bio_vec bv = bio_iter_iovec(bio, io->iter);
 449
 450                         page = kmap_atomic(bv.bv_page);
 451                         len = bv.bv_len;
 452                         if (likely(len >= todo))
 453                                 len = todo;
 454                         r = verity_hash_update(v, desc,  page + bv.bv_offset,
 455                                                len);
 456                         kunmap_atomic(page);
 457
 458                         if (unlikely(r < 0))
 459                                 return r;
 460
 461                         bio_advance_iter(bio, &io->iter, len);
 462                         todo -= len;
 463                 } while (todo);
 464
 465                 r = verity_hash_final(v, desc, io_real_digest(v, io));
 466                 if (unlikely(r < 0))
 467                         return r;
 468
 469                 if (likely(memcmp(io_real_digest(v, io),
 470                                 io_want_digest(v, io), v->digest_size) == 0))
 471                         continue;
 472                 else if (verity_handle_err(v, DM_VERITY_BLOCK_TYPE_DATA,
 473                                 io->block + b))
 474                         return -EIO;
 475         }
 476
 477         return 0;
 478 }
 479
 480 /*
 481  * End one "io" structure with a given error.
 482  */
 483 static void verity_finish_io(struct dm_verity_io *io, int error)
 484 {
 485         struct dm_verity *v = io->v;
 486         struct bio *bio = dm_bio_from_per_bio_data(io, v->ti->per_bio_data_size);
 487
 488         bio->bi_end_io = io->orig_bi_end_io;
 489         bio->bi_error = error;
 490
 491         bio_endio(bio);
 492 }
 493
 494 static void verity_work(struct work_struct *w)
 495 {
 496         struct dm_verity_io *io = container_of(w, struct dm_verity_io, work);
 497
 498         verity_finish_io(io, verity_verify_io(io));
 499 }
 500
 501 static void verity_end_io(struct bio *bio)
 502 {
 503         struct dm_verity_io *io = bio->bi_private;
 504
 505         if (bio->bi_error) {
 506                 verity_finish_io(io, bio->bi_error);
 507                 return;
 508         }
 509
 510         INIT_WORK(&io->work, verity_work);
 511         queue_work(io->v->verify_wq, &io->work);
 512 }
 513
 514 /*
 515  * Prefetch buffers for the specified io.
 516  * The root buffer is not prefetched, it is assumed that it will be cached
 517  * all the time.
 518  */
 519 static void verity_prefetch_io(struct work_struct *work)
 520 {
 521         struct dm_verity_prefetch_work *pw =
 522                 container_of(work, struct dm_verity_prefetch_work, work);
 523         struct dm_verity *v = pw->v;
 524         int i;
 525
 526         for (i = v->levels - 2; i >= 0; i--) {
 527                 sector_t hash_block_start;
 528                 sector_t hash_block_end;
 529                 verity_hash_at_level(v, pw->block, i, &hash_block_start, NULL);
 530                 verity_hash_at_level(v, pw->block + pw->n_blocks - 1, i, &hash_block_end, NULL);
 531                 if (!i) {
 532                         unsigned cluster = ACCESS_ONCE(dm_verity_prefetch_cluster);
 533
 534                         cluster >>= v->data_dev_block_bits;
 535                         if (unlikely(!cluster))
 536                                 goto no_prefetch_cluster;
 537
 538                         if (unlikely(cluster & (cluster - 1)))
 539                                 cluster = 1 << __fls(cluster);
 540
 541                         hash_block_start &= ~(sector_t)(cluster - 1);
 542                         hash_block_end |= cluster - 1;
 543                         if (unlikely(hash_block_end >= v->hash_blocks))
 544                                 hash_block_end = v->hash_blocks - 1;
 545                 }
 546 no_prefetch_cluster:
 547                 dm_bufio_prefetch(v->bufio, hash_block_start,
 548                                   hash_block_end - hash_block_start + 1);
 549         }
 550
 551         kfree(pw);
 552 }
 553
 554 static void verity_submit_prefetch(struct dm_verity *v, struct dm_verity_io *io)
 555 {
 556         struct dm_verity_prefetch_work *pw;
 557
 558         pw = kmalloc(sizeof(struct dm_verity_prefetch_work),
 559                 GFP_NOIO | __GFP_NORETRY | __GFP_NOMEMALLOC | __GFP_NOWARN);
 560
 561         if (!pw)
 562                 return;
 563
 564         INIT_WORK(&pw->work, verity_prefetch_io);
 565         pw->v = v;
 566         pw->block = io->block;
 567         pw->n_blocks = io->n_blocks;
 568         queue_work(v->verify_wq, &pw->work);
 569 }
 570
 571 /*
 572  * Bio map function. It allocates dm_verity_io structure and bio vector and
 573  * fills them. Then it issues prefetches and the I/O.
 574  */
 575 static int verity_map(struct dm_target *ti, struct bio *bio)
 576 {
 577         struct dm_verity *v = ti->private;
 578         struct dm_verity_io *io;
 579
 580         bio->bi_bdev = v->data_dev->bdev;
 581         bio->bi_iter.bi_sector = verity_map_sector(v, bio->bi_iter.bi_sector);
 582
 583         if (((unsigned)bio->bi_iter.bi_sector | bio_sectors(bio)) &
 584             ((1 << (v->data_dev_block_bits - SECTOR_SHIFT)) - 1)) {
 585                 DMERR_LIMIT("unaligned io");
 586                 return -EIO;
 587         }
 588
 589         if (bio_end_sector(bio) >>
 590             (v->data_dev_block_bits - SECTOR_SHIFT) > v->data_blocks) {
 591                 DMERR_LIMIT("io out of range");
 592                 return -EIO;
 593         }
 594
 595         if (bio_data_dir(bio) == WRITE)
 596                 return -EIO;
 597
 598         io = dm_per_bio_data(bio, ti->per_bio_data_size);
 599         io->v = v;
 600         io->orig_bi_end_io = bio->bi_end_io;
 601         io->block = bio->bi_iter.bi_sector >> (v->data_dev_block_bits - SECTOR_SHIFT);
 602         io->n_blocks = bio->bi_iter.bi_size >> v->data_dev_block_bits;
 603
 604         bio->bi_end_io = verity_end_io;
 605         bio->bi_private = io;
 606         io->iter = bio->bi_iter;
 607
 608         verity_submit_prefetch(v, io);
 609
 610         generic_make_request(bio);
 611
 612         return DM_MAPIO_SUBMITTED;
 613 }
 614
 615 /*
 616  * Status: V (valid) or C (corruption found)
 617  */
 618 static void verity_status(struct dm_target *ti, status_type_t type,
 619                           unsigned status_flags, char *result, unsigned maxlen)
 620 {
 621         struct dm_verity *v = ti->private;
 622         unsigned sz = 0;
 623         unsigned x;
 624
 625         switch (type) {
 626         case STATUSTYPE_INFO:
 627                 DMEMIT("%c", v->hash_failed ? 'C' : 'V');
 628                 break;
 629         case STATUSTYPE_TABLE:
 630                 DMEMIT("%u %s %s %u %u %llu %llu %s ",
 631                         v->version,
 632                         v->data_dev->name,
 633                         v->hash_dev->name,
 634                         1 << v->data_dev_block_bits,
 635                         1 << v->hash_dev_block_bits,
 636                         (unsigned long long)v->data_blocks,
 637                         (unsigned long long)v->hash_start,
 638                         v->alg_name
 639                         );
 640                 for (x = 0; x < v->digest_size; x++)
 641                         DMEMIT("%02x", v->root_digest[x]);
 642                 DMEMIT(" ");
 643                 if (!v->salt_size)
 644                         DMEMIT("-");
 645                 else
 646                         for (x = 0; x < v->salt_size; x++)
 647                                 DMEMIT("%02x", v->salt[x]);
 648                 if (v->mode != DM_VERITY_MODE_EIO) {
 649                         DMEMIT(" 1 ");
 650                         switch (v->mode) {
 651                         case DM_VERITY_MODE_LOGGING:
 652                                 DMEMIT(DM_VERITY_OPT_LOGGING);
 653                                 break;
 654                         case DM_VERITY_MODE_RESTART:
 655                                 DMEMIT(DM_VERITY_OPT_RESTART);
 656                                 break;
 657                         default:
 658                                 BUG();
 659                         }
 660                 }
 661                 break;
 662         }
 663 }
 664
 665 static int verity_prepare_ioctl(struct dm_target *ti,
 666                 struct block_device **bdev, fmode_t *mode)
 667 {
 668         struct dm_verity *v = ti->private;
 669
 670         *bdev = v->data_dev->bdev;
 671
 672         if (v->data_start ||
 673             ti->len != i_size_read(v->data_dev->bdev->bd_inode) >> SECTOR_SHIFT)
 674                 return 1;
 675         return 0;
 676 }
 677
 678 static int verity_iterate_devices(struct dm_target *ti,
 679                                   iterate_devices_callout_fn fn, void *data)
 680 {
 681         struct dm_verity *v = ti->private;
 682
 683         return fn(ti, v->data_dev, v->data_start, ti->len, data);
 684 }
 685
 686 static void verity_io_hints(struct dm_target *ti, struct queue_limits *limits)
 687 {
 688         struct dm_verity *v = ti->private;
 689
 690         if (limits->logical_block_size < 1 << v->data_dev_block_bits)
 691                 limits->logical_block_size = 1 << v->data_dev_block_bits;
 692
 693         if (limits->physical_block_size < 1 << v->data_dev_block_bits)
 694                 limits->physical_block_size = 1 << v->data_dev_block_bits;
 695
 696         blk_limits_io_min(limits, limits->logical_block_size);
 697 }
 698
 699 static void verity_dtr(struct dm_target *ti)
 700 {
 701         struct dm_verity *v = ti->private;
 702
 703         if (v->verify_wq)
 704                 destroy_workqueue(v->verify_wq);
 705
 706         if (v->bufio)
 707                 dm_bufio_client_destroy(v->bufio);
 708
 709         kfree(v->salt);
 710         kfree(v->root_digest);
 711
 712         if (v->tfm)
 713                 crypto_free_shash(v->tfm);
 714
 715         kfree(v->alg_name);
 716
 717         if (v->hash_dev)
 718                 dm_put_device(ti, v->hash_dev);
 719
 720         if (v->data_dev)
 721                 dm_put_device(ti, v->data_dev);
 722
 723         kfree(v);
 724 }
 725
 726 static int verity_parse_opt_args(struct dm_arg_set *as, struct dm_verity *v)
 727 {
 728         int r;
 729         unsigned argc;
 730         struct dm_target *ti = v->ti;
 731         const char *arg_name;
 732
 733         static struct dm_arg _args[] = {
 734                 {0, DM_VERITY_OPTS_MAX, "Invalid number of feature args"},
 735         };
 736
 737         r = dm_read_arg_group(_args, as, &argc, &ti->error);
 738         if (r)
 739                 return -EINVAL;
 740
 741         if (!argc)
 742                 return 0;
 743
 744         do {
 745                 arg_name = dm_shift_arg(as);
 746                 argc--;
 747
 748                 if (!strcasecmp(arg_name, DM_VERITY_OPT_LOGGING)) {
 749                         v->mode = DM_VERITY_MODE_LOGGING;
 750                         continue;
 751
 752                 } else if (!strcasecmp(arg_name, DM_VERITY_OPT_RESTART)) {
 753                         v->mode = DM_VERITY_MODE_RESTART;
 754                         continue;
 755                 }
 756
 757                 ti->error = "Unrecognized verity feature request";
 758                 return -EINVAL;
 759         } while (argc && !r);
 760
 761         return r;
 762 }
 763
 764 /*
 765  * Target parameters:
 766  *      <version>       The current format is version 1.
 767  *                      Vsn 0 is compatible with original Chromium OS releases.
 768  *      <data device>
 769  *      <hash device>
 770  *      <data block size>
 771  *      <hash block size>
 772  *      <the number of data blocks>
 773  *      <hash start block>
 774  *      <algorithm>
 775  *      <digest>
 776  *      <salt>          Hex string or "-" if no salt.
 777  */
 778 static int verity_ctr(struct dm_target *ti, unsigned argc, char **argv)
 779 {
 780         struct dm_verity *v;
 781         struct dm_arg_set as;
 782         unsigned int num;
 783         unsigned long long num_ll;
 784         int r;
 785         int i;
 786         sector_t hash_position;
 787         char dummy;
 788
 789         v = kzalloc(sizeof(struct dm_verity), GFP_KERNEL);
 790         if (!v) {
 791                 ti->error = "Cannot allocate verity structure";
 792                 return -ENOMEM;
 793         }
 794         ti->private = v;
 795         v->ti = ti;
 796
 797         if ((dm_table_get_mode(ti->table) & ~FMODE_READ)) {
 798                 ti->error = "Device must be readonly";
 799                 r = -EINVAL;
 800                 goto bad;
 801         }
 802
 803         if (argc < 10) {
 804                 ti->error = "Not enough arguments";
 805                 r = -EINVAL;
 806                 goto bad;
 807         }
 808
 809         if (sscanf(argv[0], "%u%c", &num, &dummy) != 1 ||
 810             num > 1) {
 811                 ti->error = "Invalid version";
 812                 r = -EINVAL;
 813                 goto bad;
 814         }
 815         v->version = num;
 816
 817         r = dm_get_device(ti, argv[1], FMODE_READ, &v->data_dev);
 818         if (r) {
 819                 ti->error = "Data device lookup failed";
 820                 goto bad;
 821         }
 822
 823         r = dm_get_device(ti, argv[2], FMODE_READ, &v->hash_dev);
 824         if (r) {
 825                 ti->error = "Data device lookup failed";
 826                 goto bad;
 827         }
 828
 829         if (sscanf(argv[3], "%u%c", &num, &dummy) != 1 ||
 830             !num || (num & (num - 1)) ||
 831             num < bdev_logical_block_size(v->data_dev->bdev) ||
 832             num > PAGE_SIZE) {
 833                 ti->error = "Invalid data device block size";
 834                 r = -EINVAL;
 835                 goto bad;
 836         }
 837         v->data_dev_block_bits = __ffs(num);
 838
 839         if (sscanf(argv[4], "%u%c", &num, &dummy) != 1 ||
 840             !num || (num & (num - 1)) ||
 841             num < bdev_logical_block_size(v->hash_dev->bdev) ||
 842             num > INT_MAX) {
 843                 ti->error = "Invalid hash device block size";
 844                 r = -EINVAL;
 845                 goto bad;
 846         }
 847         v->hash_dev_block_bits = __ffs(num);
 848
 849         if (sscanf(argv[5], "%llu%c", &num_ll, &dummy) != 1 ||
 850             (sector_t)(num_ll << (v->data_dev_block_bits - SECTOR_SHIFT))
 851             >> (v->data_dev_block_bits - SECTOR_SHIFT) != num_ll) {
 852                 ti->error = "Invalid data blocks";
 853                 r = -EINVAL;
 854                 goto bad;
 855         }
 856         v->data_blocks = num_ll;
 857
 858         if (ti->len > (v->data_blocks << (v->data_dev_block_bits - SECTOR_SHIFT))) {
 859                 ti->error = "Data device is too small";
 860                 r = -EINVAL;
 861                 goto bad;
 862         }
 863
 864         if (sscanf(argv[6], "%llu%c", &num_ll, &dummy) != 1 ||
 865             (sector_t)(num_ll << (v->hash_dev_block_bits - SECTOR_SHIFT))
 866             >> (v->hash_dev_block_bits - SECTOR_SHIFT) != num_ll) {
 867                 ti->error = "Invalid hash start";
 868                 r = -EINVAL;
 869                 goto bad;
 870         }
 871         v->hash_start = num_ll;
 872
 873         v->alg_name = kstrdup(argv[7], GFP_KERNEL);
 874         if (!v->alg_name) {
 875                 ti->error = "Cannot allocate algorithm name";
 876                 r = -ENOMEM;
 877                 goto bad;
 878         }
 879
 880         v->tfm = crypto_alloc_shash(v->alg_name, 0, 0);
 881         if (IS_ERR(v->tfm)) {
 882                 ti->error = "Cannot initialize hash function";
 883                 r = PTR_ERR(v->tfm);
 884                 v->tfm = NULL;
 885                 goto bad;
 886         }
 887         v->digest_size = crypto_shash_digestsize(v->tfm);
 888         if ((1 << v->hash_dev_block_bits) < v->digest_size * 2) {
 889                 ti->error = "Digest size too big";
 890                 r = -EINVAL;
 891                 goto bad;
 892         }
 893         v->shash_descsize =
 894                 sizeof(struct shash_desc) + crypto_shash_descsize(v->tfm);
 895
 896         v->root_digest = kmalloc(v->digest_size, GFP_KERNEL);
 897         if (!v->root_digest) {
 898                 ti->error = "Cannot allocate root digest";
 899                 r = -ENOMEM;
 900                 goto bad;
 901         }
 902         if (strlen(argv[8]) != v->digest_size * 2 ||
 903             hex2bin(v->root_digest, argv[8], v->digest_size)) {
 904                 ti->error = "Invalid root digest";
 905                 r = -EINVAL;
 906                 goto bad;
 907         }
 908
 909         if (strcmp(argv[9], "-")) {
 910                 v->salt_size = strlen(argv[9]) / 2;
 911                 v->salt = kmalloc(v->salt_size, GFP_KERNEL);
 912                 if (!v->salt) {
 913                         ti->error = "Cannot allocate salt";
 914                         r = -ENOMEM;
 915                         goto bad;
 916                 }
 917                 if (strlen(argv[9]) != v->salt_size * 2 ||
 918                     hex2bin(v->salt, argv[9], v->salt_size)) {
 919                         ti->error = "Invalid salt";
 920                         r = -EINVAL;
 921                         goto bad;
 922                 }
 923         }
 924
 925         argv += 10;
 926         argc -= 10;
 927
 928         /* Optional parameters */
 929         if (argc) {
 930                 as.argc = argc;
 931                 as.argv = argv;
 932
 933                 r = verity_parse_opt_args(&as, v);
 934                 if (r < 0)
 935                         goto bad;
 936         }
 937
 938         v->hash_per_block_bits =
 939                 __fls((1 << v->hash_dev_block_bits) / v->digest_size);
 940
 941         v->levels = 0;
 942         if (v->data_blocks)
 943                 while (v->hash_per_block_bits * v->levels < 64 &&
 944                        (unsigned long long)(v->data_blocks - 1) >>
 945                        (v->hash_per_block_bits * v->levels))
 946                         v->levels++;
 947
 948         if (v->levels > DM_VERITY_MAX_LEVELS) {
 949                 ti->error = "Too many tree levels";
 950                 r = -E2BIG;
 951                 goto bad;
 952         }
 953
 954         hash_position = v->hash_start;
 955         for (i = v->levels - 1; i >= 0; i--) {
 956                 sector_t s;
 957                 v->hash_level_block[i] = hash_position;
 958                 s = (v->data_blocks + ((sector_t)1 << ((i + 1) * v->hash_per_block_bits)) - 1)
 959                                         >> ((i + 1) * v->hash_per_block_bits);
 960                 if (hash_position + s < hash_position) {
 961                         ti->error = "Hash device offset overflow";
 962                         r = -E2BIG;
 963                         goto bad;
 964                 }
 965                 hash_position += s;
 966         }
 967         v->hash_blocks = hash_position;
 968
 969         v->bufio = dm_bufio_client_create(v->hash_dev->bdev,
 970                 1 << v->hash_dev_block_bits, 1, sizeof(struct buffer_aux),
 971                 dm_bufio_alloc_callback, NULL);
 972         if (IS_ERR(v->bufio)) {
 973                 ti->error = "Cannot initialize dm-bufio";
 974                 r = PTR_ERR(v->bufio);
 975                 v->bufio = NULL;
 976                 goto bad;
 977         }
 978
 979         if (dm_bufio_get_device_size(v->bufio) < v->hash_blocks) {
 980                 ti->error = "Hash device is too small";
 981                 r = -E2BIG;
 982                 goto bad;
 983         }
 984
 985         ti->per_bio_data_size = roundup(sizeof(struct dm_verity_io) + v->shash_descsize + v->digest_size * 2, __alignof__(struct dm_verity_io));
 986
 987         /* WQ_UNBOUND greatly improves performance when running on ramdisk */
 988         v->verify_wq = alloc_workqueue("kverityd", WQ_CPU_INTENSIVE | WQ_MEM_RECLAIM | WQ_UNBOUND, num_online_cpus());
 989         if (!v->verify_wq) {
 990                 ti->error = "Cannot allocate workqueue";
 991                 r = -ENOMEM;
 992                 goto bad;
 993         }
 994
 995         return 0;
 996
 997 bad:
 998         verity_dtr(ti);
 999
1000         return r;
1001 }
1002
1003 static struct target_type verity_target = {
1004         .name           = "verity",
1005         .version        = {1, 2, 0},
1006         .module         = THIS_MODULE,
1007         .ctr            = verity_ctr,
1008         .dtr            = verity_dtr,
1009         .map            = verity_map,
1010         .status         = verity_status,
1011         .prepare_ioctl  = verity_prepare_ioctl,
1012         .iterate_devices = verity_iterate_devices,
1013         .io_hints       = verity_io_hints,
1014 };
1015
1016 static int __init dm_verity_init(void)
1017 {
1018         int r;
1019
1020         r = dm_register_target(&verity_target);
1021         if (r < 0)
1022                 DMERR("register failed %d", r);
1023
1024         return r;
1025 }
1026
1027 static void __exit dm_verity_exit(void)
1028 {
1029         dm_unregister_target(&verity_target);
1030 }
1031
1032 module_init(dm_verity_init);
1033 module_exit(dm_verity_exit);
1034
1035 MODULE_AUTHOR("Mikulas Patocka <mpatocka@redhat.com>");
1036 MODULE_AUTHOR("Mandeep Baines <msb@chromium.org>");
1037 MODULE_AUTHOR("Will Drewry <wad@chromium.org>");
1038 MODULE_DESCRIPTION(DM_NAME " target for transparent disk integrity checking");
1039 MODULE_LICENSE("GPL");