block/blk-mq.c

   1 /*
   2  * Block multiqueue core code
   3  *
   4  * Copyright (C) 2013-2014 Jens Axboe
   5  * Copyright (C) 2013-2014 Christoph Hellwig
   6  */
   7 #include <linux/kernel.h>
   8 #include <linux/module.h>
   9 #include <linux/backing-dev.h>
  10 #include <linux/bio.h>
  11 #include <linux/blkdev.h>
  12 #include <linux/mm.h>
  13 #include <linux/init.h>
  14 #include <linux/slab.h>
  15 #include <linux/workqueue.h>
  16 #include <linux/smp.h>
  17 #include <linux/llist.h>
  18 #include <linux/list_sort.h>
  19 #include <linux/cpu.h>
  20 #include <linux/cache.h>
  21 #include <linux/sched/sysctl.h>
  22 #include <linux/delay.h>
  23
  24 #include <trace/events/block.h>
  25
  26 #include <linux/blk-mq.h>
  27 #include "blk.h"
  28 #include "blk-mq.h"
  29 #include "blk-mq-tag.h"
  30
  31 static DEFINE_MUTEX(all_q_mutex);
  32 static LIST_HEAD(all_q_list);
  33
  34 static void __blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx);
  35
  36 /*
  37  * Check if any of the ctx's have pending work in this hardware queue
  38  */
  39 static bool blk_mq_hctx_has_pending(struct blk_mq_hw_ctx *hctx)
  40 {
  41         unsigned int i;
  42
  43         for (i = 0; i < hctx->ctx_map.map_size; i++)
  44                 if (hctx->ctx_map.map[i].word)
  45                         return true;
  46
  47         return false;
  48 }
  49
  50 static inline struct blk_align_bitmap *get_bm(struct blk_mq_hw_ctx *hctx,
  51                                               struct blk_mq_ctx *ctx)
  52 {
  53         return &hctx->ctx_map.map[ctx->index_hw / hctx->ctx_map.bits_per_word];
  54 }
  55
  56 #define CTX_TO_BIT(hctx, ctx)   \
  57         ((ctx)->index_hw & ((hctx)->ctx_map.bits_per_word - 1))
  58
  59 /*
  60  * Mark this ctx as having pending work in this hardware queue
  61  */
  62 static void blk_mq_hctx_mark_pending(struct blk_mq_hw_ctx *hctx,
  63                                      struct blk_mq_ctx *ctx)
  64 {
  65         struct blk_align_bitmap *bm = get_bm(hctx, ctx);
  66
  67         if (!test_bit(CTX_TO_BIT(hctx, ctx), &bm->word))
  68                 set_bit(CTX_TO_BIT(hctx, ctx), &bm->word);
  69 }
  70
  71 static void blk_mq_hctx_clear_pending(struct blk_mq_hw_ctx *hctx,
  72                                       struct blk_mq_ctx *ctx)
  73 {
  74         struct blk_align_bitmap *bm = get_bm(hctx, ctx);
  75
  76         clear_bit(CTX_TO_BIT(hctx, ctx), &bm->word);
  77 }
  78
  79 static int blk_mq_queue_enter(struct request_queue *q)
  80 {
  81         int ret;
  82
  83         __percpu_counter_add(&q->mq_usage_counter, 1, 1000000);
  84         smp_wmb();
  85         /* we have problems to freeze the queue if it's initializing */
  86         if (!blk_queue_bypass(q) || !blk_queue_init_done(q))
  87                 return 0;
  88
  89         __percpu_counter_add(&q->mq_usage_counter, -1, 1000000);
  90
  91         spin_lock_irq(q->queue_lock);
  92         ret = wait_event_interruptible_lock_irq(q->mq_freeze_wq,
  93                 !blk_queue_bypass(q) || blk_queue_dying(q),
  94                 *q->queue_lock);
  95         /* inc usage with lock hold to avoid freeze_queue runs here */
  96         if (!ret && !blk_queue_dying(q))
  97                 __percpu_counter_add(&q->mq_usage_counter, 1, 1000000);
  98         else if (blk_queue_dying(q))
  99                 ret = -ENODEV;
 100         spin_unlock_irq(q->queue_lock);
 101
 102         return ret;
 103 }
 104
 105 static void blk_mq_queue_exit(struct request_queue *q)
 106 {
 107         __percpu_counter_add(&q->mq_usage_counter, -1, 1000000);
 108 }
 109
 110 static void __blk_mq_drain_queue(struct request_queue *q)
 111 {
 112         while (true) {
 113                 s64 count;
 114
 115                 spin_lock_irq(q->queue_lock);
 116                 count = percpu_counter_sum(&q->mq_usage_counter);
 117                 spin_unlock_irq(q->queue_lock);
 118
 119                 if (count == 0)
 120                         break;
 121                 blk_mq_run_queues(q, false);
 122                 msleep(10);
 123         }
 124 }
 125
 126 /*
 127  * Guarantee no request is in use, so we can change any data structure of
 128  * the queue afterward.
 129  */
 130 static void blk_mq_freeze_queue(struct request_queue *q)
 131 {
 132         bool drain;
 133
 134         spin_lock_irq(q->queue_lock);
 135         drain = !q->bypass_depth++;
 136         queue_flag_set(QUEUE_FLAG_BYPASS, q);
 137         spin_unlock_irq(q->queue_lock);
 138
 139         if (drain)
 140                 __blk_mq_drain_queue(q);
 141 }
 142
 143 void blk_mq_drain_queue(struct request_queue *q)
 144 {
 145         __blk_mq_drain_queue(q);
 146 }
 147
 148 static void blk_mq_unfreeze_queue(struct request_queue *q)
 149 {
 150         bool wake = false;
 151
 152         spin_lock_irq(q->queue_lock);
 153         if (!--q->bypass_depth) {
 154                 queue_flag_clear(QUEUE_FLAG_BYPASS, q);
 155                 wake = true;
 156         }
 157         WARN_ON_ONCE(q->bypass_depth < 0);
 158         spin_unlock_irq(q->queue_lock);
 159         if (wake)
 160                 wake_up_all(&q->mq_freeze_wq);
 161 }
 162
 163 bool blk_mq_can_queue(struct blk_mq_hw_ctx *hctx)
 164 {
 165         return blk_mq_has_free_tags(hctx->tags);
 166 }
 167 EXPORT_SYMBOL(blk_mq_can_queue);
 168
 169 static void blk_mq_rq_ctx_init(struct request_queue *q, struct blk_mq_ctx *ctx,
 170                                struct request *rq, unsigned int rw_flags)
 171 {
 172         if (blk_queue_io_stat(q))
 173                 rw_flags |= REQ_IO_STAT;
 174
 175         INIT_LIST_HEAD(&rq->queuelist);
 176         /* csd/requeue_work/fifo_time is initialized before use */
 177         rq->q = q;
 178         rq->mq_ctx = ctx;
 179         rq->cmd_flags |= rw_flags;
 180         /* do not touch atomic flags, it needs atomic ops against the timer */
 181         rq->cpu = -1;
 182         INIT_HLIST_NODE(&rq->hash);
 183         RB_CLEAR_NODE(&rq->rb_node);
 184         rq->rq_disk = NULL;
 185         rq->part = NULL;
 186 #ifdef CONFIG_BLK_CGROUP
 187         rq->rl = NULL;
 188         set_start_time_ns(rq);
 189         rq->io_start_time_ns = 0;
 190 #endif
 191         rq->nr_phys_segments = 0;
 192 #if defined(CONFIG_BLK_DEV_INTEGRITY)
 193         rq->nr_integrity_segments = 0;
 194 #endif
 195         rq->special = NULL;
 196         /* tag was already set */
 197         rq->errors = 0;
 198
 199         rq->extra_len = 0;
 200         rq->sense_len = 0;
 201         rq->resid_len = 0;
 202         rq->sense = NULL;
 203
 204         INIT_LIST_HEAD(&rq->timeout_list);
 205         rq->end_io = NULL;
 206         rq->end_io_data = NULL;
 207         rq->next_rq = NULL;
 208
 209         ctx->rq_dispatched[rw_is_sync(rw_flags)]++;
 210 }
 211
 212 static struct request *
 213 __blk_mq_alloc_request(struct request_queue *q, struct blk_mq_hw_ctx *hctx,
 214                 struct blk_mq_ctx *ctx, int rw, gfp_t gfp, bool reserved)
 215 {
 216         struct request *rq;
 217         unsigned int tag;
 218
 219         tag = blk_mq_get_tag(hctx, &ctx->last_tag, gfp, reserved);
 220         if (tag != BLK_MQ_TAG_FAIL) {
 221                 rq = hctx->tags->rqs[tag];
 222
 223                 rq->cmd_flags = 0;
 224                 if (blk_mq_tag_busy(hctx)) {
 225                         rq->cmd_flags = REQ_MQ_INFLIGHT;
 226                         atomic_inc(&hctx->nr_active);
 227                 }
 228
 229                 rq->tag = tag;
 230                 blk_mq_rq_ctx_init(q, ctx, rq, rw);
 231                 return rq;
 232         }
 233
 234         return NULL;
 235 }
 236
 237 struct request *blk_mq_alloc_request(struct request_queue *q, int rw, gfp_t gfp,
 238                 bool reserved)
 239 {
 240         struct blk_mq_ctx *ctx;
 241         struct blk_mq_hw_ctx *hctx;
 242         struct request *rq;
 243
 244         if (blk_mq_queue_enter(q))
 245                 return NULL;
 246
 247         ctx = blk_mq_get_ctx(q);
 248         hctx = q->mq_ops->map_queue(q, ctx->cpu);
 249
 250         rq = __blk_mq_alloc_request(q, hctx, ctx, rw, gfp & ~__GFP_WAIT,
 251                                     reserved);
 252         if (!rq && (gfp & __GFP_WAIT)) {
 253                 __blk_mq_run_hw_queue(hctx);
 254                 blk_mq_put_ctx(ctx);
 255
 256                 ctx = blk_mq_get_ctx(q);
 257                 hctx = q->mq_ops->map_queue(q, ctx->cpu);
 258                 rq =  __blk_mq_alloc_request(q, hctx, ctx, rw, gfp, reserved);
 259         }
 260         blk_mq_put_ctx(ctx);
 261         return rq;
 262 }
 263 EXPORT_SYMBOL(blk_mq_alloc_request);
 264
 265 static void __blk_mq_free_request(struct blk_mq_hw_ctx *hctx,
 266                                   struct blk_mq_ctx *ctx, struct request *rq)
 267 {
 268         const int tag = rq->tag;
 269         struct request_queue *q = rq->q;
 270
 271         if (rq->cmd_flags & REQ_MQ_INFLIGHT)
 272                 atomic_dec(&hctx->nr_active);
 273
 274         clear_bit(REQ_ATOM_STARTED, &rq->atomic_flags);
 275         blk_mq_put_tag(hctx, tag, &ctx->last_tag);
 276         blk_mq_queue_exit(q);
 277 }
 278
 279 void blk_mq_free_request(struct request *rq)
 280 {
 281         struct blk_mq_ctx *ctx = rq->mq_ctx;
 282         struct blk_mq_hw_ctx *hctx;
 283         struct request_queue *q = rq->q;
 284
 285         ctx->rq_completed[rq_is_sync(rq)]++;
 286
 287         hctx = q->mq_ops->map_queue(q, ctx->cpu);
 288         __blk_mq_free_request(hctx, ctx, rq);
 289 }
 290
 291 /*
 292  * Clone all relevant state from a request that has been put on hold in
 293  * the flush state machine into the preallocated flush request that hangs
 294  * off the request queue.
 295  *
 296  * For a driver the flush request should be invisible, that's why we are
 297  * impersonating the original request here.
 298  */
 299 void blk_mq_clone_flush_request(struct request *flush_rq,
 300                 struct request *orig_rq)
 301 {
 302         struct blk_mq_hw_ctx *hctx =
 303                 orig_rq->q->mq_ops->map_queue(orig_rq->q, orig_rq->mq_ctx->cpu);
 304
 305         flush_rq->mq_ctx = orig_rq->mq_ctx;
 306         flush_rq->tag = orig_rq->tag;
 307         memcpy(blk_mq_rq_to_pdu(flush_rq), blk_mq_rq_to_pdu(orig_rq),
 308                 hctx->cmd_size);
 309 }
 310
 311 inline void __blk_mq_end_io(struct request *rq, int error)
 312 {
 313         blk_account_io_done(rq);
 314
 315         if (rq->end_io) {
 316                 rq->end_io(rq, error);
 317         } else {
 318                 if (unlikely(blk_bidi_rq(rq)))
 319                         blk_mq_free_request(rq->next_rq);
 320                 blk_mq_free_request(rq);
 321         }
 322 }
 323 EXPORT_SYMBOL(__blk_mq_end_io);
 324
 325 void blk_mq_end_io(struct request *rq, int error)
 326 {
 327         if (blk_update_request(rq, error, blk_rq_bytes(rq)))
 328                 BUG();
 329         __blk_mq_end_io(rq, error);
 330 }
 331 EXPORT_SYMBOL(blk_mq_end_io);
 332
 333 static void __blk_mq_complete_request_remote(void *data)
 334 {
 335         struct request *rq = data;
 336
 337         rq->q->softirq_done_fn(rq);
 338 }
 339
 340 static void blk_mq_ipi_complete_request(struct request *rq)
 341 {
 342         struct blk_mq_ctx *ctx = rq->mq_ctx;
 343         bool shared = false;
 344         int cpu;
 345
 346         if (!test_bit(QUEUE_FLAG_SAME_COMP, &rq->q->queue_flags)) {
 347                 rq->q->softirq_done_fn(rq);
 348                 return;
 349         }
 350
 351         cpu = get_cpu();
 352         if (!test_bit(QUEUE_FLAG_SAME_FORCE, &rq->q->queue_flags))
 353                 shared = cpus_share_cache(cpu, ctx->cpu);
 354
 355         if (cpu != ctx->cpu && !shared && cpu_online(ctx->cpu)) {
 356                 rq->csd.func = __blk_mq_complete_request_remote;
 357                 rq->csd.info = rq;
 358                 rq->csd.flags = 0;
 359                 smp_call_function_single_async(ctx->cpu, &rq->csd);
 360         } else {
 361                 rq->q->softirq_done_fn(rq);
 362         }
 363         put_cpu();
 364 }
 365
 366 void __blk_mq_complete_request(struct request *rq)
 367 {
 368         struct request_queue *q = rq->q;
 369
 370         if (!q->softirq_done_fn)
 371                 blk_mq_end_io(rq, rq->errors);
 372         else
 373                 blk_mq_ipi_complete_request(rq);
 374 }
 375
 376 /**
 377  * blk_mq_complete_request - end I/O on a request
 378  * @rq:         the request being processed
 379  *
 380  * Description:
 381  *      Ends all I/O on a request. It does not handle partial completions.
 382  *      The actual completion happens out-of-order, through a IPI handler.
 383  **/
 384 void blk_mq_complete_request(struct request *rq)
 385 {
 386         struct request_queue *q = rq->q;
 387
 388         if (unlikely(blk_should_fake_timeout(q)))
 389                 return;
 390         if (!blk_mark_rq_complete(rq))
 391                 __blk_mq_complete_request(rq);
 392 }
 393 EXPORT_SYMBOL(blk_mq_complete_request);
 394
 395 static void blk_mq_start_request(struct request *rq, bool last)
 396 {
 397         struct request_queue *q = rq->q;
 398
 399         trace_block_rq_issue(q, rq);
 400
 401         rq->resid_len = blk_rq_bytes(rq);
 402         if (unlikely(blk_bidi_rq(rq)))
 403                 rq->next_rq->resid_len = blk_rq_bytes(rq->next_rq);
 404
 405         /*
 406          * Just mark start time and set the started bit. Due to memory
 407          * ordering, we know we'll see the correct deadline as long as
 408          * REQ_ATOMIC_STARTED is seen. Use the default queue timeout,
 409          * unless one has been set in the request.
 410          */
 411         if (!rq->timeout)
 412                 rq->deadline = jiffies + q->rq_timeout;
 413         else
 414                 rq->deadline = jiffies + rq->timeout;
 415
 416         /*
 417          * Mark us as started and clear complete. Complete might have been
 418          * set if requeue raced with timeout, which then marked it as
 419          * complete. So be sure to clear complete again when we start
 420          * the request, otherwise we'll ignore the completion event.
 421          */
 422         if (!test_bit(REQ_ATOM_STARTED, &rq->atomic_flags))
 423                 set_bit(REQ_ATOM_STARTED, &rq->atomic_flags);
 424         if (test_bit(REQ_ATOM_COMPLETE, &rq->atomic_flags))
 425                 clear_bit(REQ_ATOM_COMPLETE, &rq->atomic_flags);
 426
 427         if (q->dma_drain_size && blk_rq_bytes(rq)) {
 428                 /*
 429                  * Make sure space for the drain appears.  We know we can do
 430                  * this because max_hw_segments has been adjusted to be one
 431                  * fewer than the device can handle.
 432                  */
 433                 rq->nr_phys_segments++;
 434         }
 435
 436         /*
 437          * Flag the last request in the series so that drivers know when IO
 438          * should be kicked off, if they don't do it on a per-request basis.
 439          *
 440          * Note: the flag isn't the only condition drivers should do kick off.
 441          * If drive is busy, the last request might not have the bit set.
 442          */
 443         if (last)
 444                 rq->cmd_flags |= REQ_END;
 445 }
 446
 447 static void __blk_mq_requeue_request(struct request *rq)
 448 {
 449         struct request_queue *q = rq->q;
 450
 451         trace_block_rq_requeue(q, rq);
 452         clear_bit(REQ_ATOM_STARTED, &rq->atomic_flags);
 453
 454         rq->cmd_flags &= ~REQ_END;
 455
 456         if (q->dma_drain_size && blk_rq_bytes(rq))
 457                 rq->nr_phys_segments--;
 458 }
 459
 460 void blk_mq_requeue_request(struct request *rq)
 461 {
 462         __blk_mq_requeue_request(rq);
 463         blk_clear_rq_complete(rq);
 464
 465         BUG_ON(blk_queued_rq(rq));
 466         blk_mq_add_to_requeue_list(rq, true);
 467 }
 468 EXPORT_SYMBOL(blk_mq_requeue_request);
 469
 470 static void blk_mq_requeue_work(struct work_struct *work)
 471 {
 472         struct request_queue *q =
 473                 container_of(work, struct request_queue, requeue_work);
 474         LIST_HEAD(rq_list);
 475         struct request *rq, *next;
 476         unsigned long flags;
 477
 478         spin_lock_irqsave(&q->requeue_lock, flags);
 479         list_splice_init(&q->requeue_list, &rq_list);
 480         spin_unlock_irqrestore(&q->requeue_lock, flags);
 481
 482         list_for_each_entry_safe(rq, next, &rq_list, queuelist) {
 483                 if (!(rq->cmd_flags & REQ_SOFTBARRIER))
 484                         continue;
 485
 486                 rq->cmd_flags &= ~REQ_SOFTBARRIER;
 487                 list_del_init(&rq->queuelist);
 488                 blk_mq_insert_request(rq, true, false, false);
 489         }
 490
 491         while (!list_empty(&rq_list)) {
 492                 rq = list_entry(rq_list.next, struct request, queuelist);
 493                 list_del_init(&rq->queuelist);
 494                 blk_mq_insert_request(rq, false, false, false);
 495         }
 496
 497         blk_mq_run_queues(q, false);
 498 }
 499
 500 void blk_mq_add_to_requeue_list(struct request *rq, bool at_head)
 501 {
 502         struct request_queue *q = rq->q;
 503         unsigned long flags;
 504
 505         /*
 506          * We abuse this flag that is otherwise used by the I/O scheduler to
 507          * request head insertation from the workqueue.
 508          */
 509         BUG_ON(rq->cmd_flags & REQ_SOFTBARRIER);
 510
 511         spin_lock_irqsave(&q->requeue_lock, flags);
 512         if (at_head) {
 513                 rq->cmd_flags |= REQ_SOFTBARRIER;
 514                 list_add(&rq->queuelist, &q->requeue_list);
 515         } else {
 516                 list_add_tail(&rq->queuelist, &q->requeue_list);
 517         }
 518         spin_unlock_irqrestore(&q->requeue_lock, flags);
 519 }
 520 EXPORT_SYMBOL(blk_mq_add_to_requeue_list);
 521
 522 void blk_mq_kick_requeue_list(struct request_queue *q)
 523 {
 524         kblockd_schedule_work(&q->requeue_work);
 525 }
 526 EXPORT_SYMBOL(blk_mq_kick_requeue_list);
 527
 528 struct request *blk_mq_tag_to_rq(struct blk_mq_hw_ctx *hctx, unsigned int tag)
 529 {
 530         struct request_queue *q = hctx->queue;
 531
 532         if ((q->flush_rq->cmd_flags & REQ_FLUSH_SEQ) &&
 533             q->flush_rq->tag == tag)
 534                 return q->flush_rq;
 535
 536         return hctx->tags->rqs[tag];
 537 }
 538 EXPORT_SYMBOL(blk_mq_tag_to_rq);
 539
 540 struct blk_mq_timeout_data {
 541         struct blk_mq_hw_ctx *hctx;
 542         unsigned long *next;
 543         unsigned int *next_set;
 544 };
 545
 546 static void blk_mq_timeout_check(void *__data, unsigned long *free_tags)
 547 {
 548         struct blk_mq_timeout_data *data = __data;
 549         struct blk_mq_hw_ctx *hctx = data->hctx;
 550         unsigned int tag;
 551
 552          /* It may not be in flight yet (this is where
 553          * the REQ_ATOMIC_STARTED flag comes in). The requests are
 554          * statically allocated, so we know it's always safe to access the
 555          * memory associated with a bit offset into ->rqs[].
 556          */
 557         tag = 0;
 558         do {
 559                 struct request *rq;
 560
 561                 tag = find_next_zero_bit(free_tags, hctx->tags->nr_tags, tag);
 562                 if (tag >= hctx->tags->nr_tags)
 563                         break;
 564
 565                 rq = blk_mq_tag_to_rq(hctx, tag++);
 566                 if (rq->q != hctx->queue)
 567                         continue;
 568                 if (!test_bit(REQ_ATOM_STARTED, &rq->atomic_flags))
 569                         continue;
 570
 571                 blk_rq_check_expired(rq, data->next, data->next_set);
 572         } while (1);
 573 }
 574
 575 static void blk_mq_hw_ctx_check_timeout(struct blk_mq_hw_ctx *hctx,
 576                                         unsigned long *next,
 577                                         unsigned int *next_set)
 578 {
 579         struct blk_mq_timeout_data data = {
 580                 .hctx           = hctx,
 581                 .next           = next,
 582                 .next_set       = next_set,
 583         };
 584
 585         /*
 586          * Ask the tagging code to iterate busy requests, so we can
 587          * check them for timeout.
 588          */
 589         blk_mq_tag_busy_iter(hctx->tags, blk_mq_timeout_check, &data);
 590 }
 591
 592 static enum blk_eh_timer_return blk_mq_rq_timed_out(struct request *rq)
 593 {
 594         struct request_queue *q = rq->q;
 595
 596         /*
 597          * We know that complete is set at this point. If STARTED isn't set
 598          * anymore, then the request isn't active and the "timeout" should
 599          * just be ignored. This can happen due to the bitflag ordering.
 600          * Timeout first checks if STARTED is set, and if it is, assumes
 601          * the request is active. But if we race with completion, then
 602          * we both flags will get cleared. So check here again, and ignore
 603          * a timeout event with a request that isn't active.
 604          */
 605         if (!test_bit(REQ_ATOM_STARTED, &rq->atomic_flags))
 606                 return BLK_EH_NOT_HANDLED;
 607
 608         if (!q->mq_ops->timeout)
 609                 return BLK_EH_RESET_TIMER;
 610
 611         return q->mq_ops->timeout(rq);
 612 }
 613
 614 static void blk_mq_rq_timer(unsigned long data)
 615 {
 616         struct request_queue *q = (struct request_queue *) data;
 617         struct blk_mq_hw_ctx *hctx;
 618         unsigned long next = 0;
 619         int i, next_set = 0;
 620
 621         queue_for_each_hw_ctx(q, hctx, i) {
 622                 /*
 623                  * If not software queues are currently mapped to this
 624                  * hardware queue, there's nothing to check
 625                  */
 626                 if (!hctx->nr_ctx || !hctx->tags)
 627                         continue;
 628
 629                 blk_mq_hw_ctx_check_timeout(hctx, &next, &next_set);
 630         }
 631
 632         if (next_set) {
 633                 next = blk_rq_timeout(round_jiffies_up(next));
 634                 mod_timer(&q->timeout, next);
 635         } else {
 636                 queue_for_each_hw_ctx(q, hctx, i)
 637                         blk_mq_tag_idle(hctx);
 638         }
 639 }
 640
 641 /*
 642  * Reverse check our software queue for entries that we could potentially
 643  * merge with. Currently includes a hand-wavy stop count of 8, to not spend
 644  * too much time checking for merges.
 645  */
 646 static bool blk_mq_attempt_merge(struct request_queue *q,
 647                                  struct blk_mq_ctx *ctx, struct bio *bio)
 648 {
 649         struct request *rq;
 650         int checked = 8;
 651
 652         list_for_each_entry_reverse(rq, &ctx->rq_list, queuelist) {
 653                 int el_ret;
 654
 655                 if (!checked--)
 656                         break;
 657
 658                 if (!blk_rq_merge_ok(rq, bio))
 659                         continue;
 660
 661                 el_ret = blk_try_merge(rq, bio);
 662                 if (el_ret == ELEVATOR_BACK_MERGE) {
 663                         if (bio_attempt_back_merge(q, rq, bio)) {
 664                                 ctx->rq_merged++;
 665                                 return true;
 666                         }
 667                         break;
 668                 } else if (el_ret == ELEVATOR_FRONT_MERGE) {
 669                         if (bio_attempt_front_merge(q, rq, bio)) {
 670                                 ctx->rq_merged++;
 671                                 return true;
 672                         }
 673                         break;
 674                 }
 675         }
 676
 677         return false;
 678 }
 679
 680 /*
 681  * Process software queues that have been marked busy, splicing them
 682  * to the for-dispatch
 683  */
 684 static void flush_busy_ctxs(struct blk_mq_hw_ctx *hctx, struct list_head *list)
 685 {
 686         struct blk_mq_ctx *ctx;
 687         int i;
 688
 689         for (i = 0; i < hctx->ctx_map.map_size; i++) {
 690                 struct blk_align_bitmap *bm = &hctx->ctx_map.map[i];
 691                 unsigned int off, bit;
 692
 693                 if (!bm->word)
 694                         continue;
 695
 696                 bit = 0;
 697                 off = i * hctx->ctx_map.bits_per_word;
 698                 do {
 699                         bit = find_next_bit(&bm->word, bm->depth, bit);
 700                         if (bit >= bm->depth)
 701                                 break;
 702
 703                         ctx = hctx->ctxs[bit + off];
 704                         clear_bit(bit, &bm->word);
 705                         spin_lock(&ctx->lock);
 706                         list_splice_tail_init(&ctx->rq_list, list);
 707                         spin_unlock(&ctx->lock);
 708
 709                         bit++;
 710                 } while (1);
 711         }
 712 }
 713
 714 /*
 715  * Run this hardware queue, pulling any software queues mapped to it in.
 716  * Note that this function currently has various problems around ordering
 717  * of IO. In particular, we'd like FIFO behaviour on handling existing
 718  * items on the hctx->dispatch list. Ignore that for now.
 719  */
 720 static void __blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx)
 721 {
 722         struct request_queue *q = hctx->queue;
 723         struct request *rq;
 724         LIST_HEAD(rq_list);
 725         int queued;
 726
 727         WARN_ON(!cpumask_test_cpu(raw_smp_processor_id(), hctx->cpumask));
 728
 729         if (unlikely(test_bit(BLK_MQ_S_STOPPED, &hctx->state)))
 730                 return;
 731
 732         hctx->run++;
 733
 734         /*
 735          * Touch any software queue that has pending entries.
 736          */
 737         flush_busy_ctxs(hctx, &rq_list);
 738
 739         /*
 740          * If we have previous entries on our dispatch list, grab them
 741          * and stuff them at the front for more fair dispatch.
 742          */
 743         if (!list_empty_careful(&hctx->dispatch)) {
 744                 spin_lock(&hctx->lock);
 745                 if (!list_empty(&hctx->dispatch))
 746                         list_splice_init(&hctx->dispatch, &rq_list);
 747                 spin_unlock(&hctx->lock);
 748         }
 749
 750         /*
 751          * Now process all the entries, sending them to the driver.
 752          */
 753         queued = 0;
 754         while (!list_empty(&rq_list)) {
 755                 int ret;
 756
 757                 rq = list_first_entry(&rq_list, struct request, queuelist);
 758                 list_del_init(&rq->queuelist);
 759
 760                 blk_mq_start_request(rq, list_empty(&rq_list));
 761
 762                 ret = q->mq_ops->queue_rq(hctx, rq);
 763                 switch (ret) {
 764                 case BLK_MQ_RQ_QUEUE_OK:
 765                         queued++;
 766                         continue;
 767                 case BLK_MQ_RQ_QUEUE_BUSY:
 768                         list_add(&rq->queuelist, &rq_list);
 769                         __blk_mq_requeue_request(rq);
 770                         break;
 771                 default:
 772                         pr_err("blk-mq: bad return on queue: %d\n", ret);
 773                 case BLK_MQ_RQ_QUEUE_ERROR:
 774                         rq->errors = -EIO;
 775                         blk_mq_end_io(rq, rq->errors);
 776                         break;
 777                 }
 778
 779                 if (ret == BLK_MQ_RQ_QUEUE_BUSY)
 780                         break;
 781         }
 782
 783         if (!queued)
 784                 hctx->dispatched[0]++;
 785         else if (queued < (1 << (BLK_MQ_MAX_DISPATCH_ORDER - 1)))
 786                 hctx->dispatched[ilog2(queued) + 1]++;
 787
 788         /*
 789          * Any items that need requeuing? Stuff them into hctx->dispatch,
 790          * that is where we will continue on next queue run.
 791          */
 792         if (!list_empty(&rq_list)) {
 793                 spin_lock(&hctx->lock);
 794                 list_splice(&rq_list, &hctx->dispatch);
 795                 spin_unlock(&hctx->lock);
 796         }
 797 }
 798
 799 /*
 800  * It'd be great if the workqueue API had a way to pass
 801  * in a mask and had some smarts for more clever placement.
 802  * For now we just round-robin here, switching for every
 803  * BLK_MQ_CPU_WORK_BATCH queued items.
 804  */
 805 static int blk_mq_hctx_next_cpu(struct blk_mq_hw_ctx *hctx)
 806 {
 807         int cpu = hctx->next_cpu;
 808
 809         if (--hctx->next_cpu_batch <= 0) {
 810                 int next_cpu;
 811
 812                 next_cpu = cpumask_next(hctx->next_cpu, hctx->cpumask);
 813                 if (next_cpu >= nr_cpu_ids)
 814                         next_cpu = cpumask_first(hctx->cpumask);
 815
 816                 hctx->next_cpu = next_cpu;
 817                 hctx->next_cpu_batch = BLK_MQ_CPU_WORK_BATCH;
 818         }
 819
 820         return cpu;
 821 }
 822
 823 void blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx, bool async)
 824 {
 825         if (unlikely(test_bit(BLK_MQ_S_STOPPED, &hctx->state)))
 826                 return;
 827
 828         if (!async && cpumask_test_cpu(smp_processor_id(), hctx->cpumask))
 829                 __blk_mq_run_hw_queue(hctx);
 830         else if (hctx->queue->nr_hw_queues == 1)
 831                 kblockd_schedule_delayed_work(&hctx->run_work, 0);
 832         else {
 833                 unsigned int cpu;
 834
 835                 cpu = blk_mq_hctx_next_cpu(hctx);
 836                 kblockd_schedule_delayed_work_on(cpu, &hctx->run_work, 0);
 837         }
 838 }
 839
 840 void blk_mq_run_queues(struct request_queue *q, bool async)
 841 {
 842         struct blk_mq_hw_ctx *hctx;
 843         int i;
 844
 845         queue_for_each_hw_ctx(q, hctx, i) {
 846                 if ((!blk_mq_hctx_has_pending(hctx) &&
 847                     list_empty_careful(&hctx->dispatch)) ||
 848                     test_bit(BLK_MQ_S_STOPPED, &hctx->state))
 849                         continue;
 850
 851                 preempt_disable();
 852                 blk_mq_run_hw_queue(hctx, async);
 853                 preempt_enable();
 854         }
 855 }
 856 EXPORT_SYMBOL(blk_mq_run_queues);
 857
 858 void blk_mq_stop_hw_queue(struct blk_mq_hw_ctx *hctx)
 859 {
 860         cancel_delayed_work(&hctx->run_work);
 861         cancel_delayed_work(&hctx->delay_work);
 862         set_bit(BLK_MQ_S_STOPPED, &hctx->state);
 863 }
 864 EXPORT_SYMBOL(blk_mq_stop_hw_queue);
 865
 866 void blk_mq_stop_hw_queues(struct request_queue *q)
 867 {
 868         struct blk_mq_hw_ctx *hctx;
 869         int i;
 870
 871         queue_for_each_hw_ctx(q, hctx, i)
 872                 blk_mq_stop_hw_queue(hctx);
 873 }
 874 EXPORT_SYMBOL(blk_mq_stop_hw_queues);
 875
 876 void blk_mq_start_hw_queue(struct blk_mq_hw_ctx *hctx)
 877 {
 878         clear_bit(BLK_MQ_S_STOPPED, &hctx->state);
 879
 880         preempt_disable();
 881         __blk_mq_run_hw_queue(hctx);
 882         preempt_enable();
 883 }
 884 EXPORT_SYMBOL(blk_mq_start_hw_queue);
 885
 886 void blk_mq_start_hw_queues(struct request_queue *q)
 887 {
 888         struct blk_mq_hw_ctx *hctx;
 889         int i;
 890
 891         queue_for_each_hw_ctx(q, hctx, i)
 892                 blk_mq_start_hw_queue(hctx);
 893 }
 894 EXPORT_SYMBOL(blk_mq_start_hw_queues);
 895
 896
 897 void blk_mq_start_stopped_hw_queues(struct request_queue *q, bool async)
 898 {
 899         struct blk_mq_hw_ctx *hctx;
 900         int i;
 901
 902         queue_for_each_hw_ctx(q, hctx, i) {
 903                 if (!test_bit(BLK_MQ_S_STOPPED, &hctx->state))
 904                         continue;
 905
 906                 clear_bit(BLK_MQ_S_STOPPED, &hctx->state);
 907                 preempt_disable();
 908                 blk_mq_run_hw_queue(hctx, async);
 909                 preempt_enable();
 910         }
 911 }
 912 EXPORT_SYMBOL(blk_mq_start_stopped_hw_queues);
 913
 914 static void blk_mq_run_work_fn(struct work_struct *work)
 915 {
 916         struct blk_mq_hw_ctx *hctx;
 917
 918         hctx = container_of(work, struct blk_mq_hw_ctx, run_work.work);
 919
 920         __blk_mq_run_hw_queue(hctx);
 921 }
 922
 923 static void blk_mq_delay_work_fn(struct work_struct *work)
 924 {
 925         struct blk_mq_hw_ctx *hctx;
 926
 927         hctx = container_of(work, struct blk_mq_hw_ctx, delay_work.work);
 928
 929         if (test_and_clear_bit(BLK_MQ_S_STOPPED, &hctx->state))
 930                 __blk_mq_run_hw_queue(hctx);
 931 }
 932
 933 void blk_mq_delay_queue(struct blk_mq_hw_ctx *hctx, unsigned long msecs)
 934 {
 935         unsigned long tmo = msecs_to_jiffies(msecs);
 936
 937         if (hctx->queue->nr_hw_queues == 1)
 938                 kblockd_schedule_delayed_work(&hctx->delay_work, tmo);
 939         else {
 940                 unsigned int cpu;
 941
 942                 cpu = blk_mq_hctx_next_cpu(hctx);
 943                 kblockd_schedule_delayed_work_on(cpu, &hctx->delay_work, tmo);
 944         }
 945 }
 946 EXPORT_SYMBOL(blk_mq_delay_queue);
 947
 948 static void __blk_mq_insert_request(struct blk_mq_hw_ctx *hctx,
 949                                     struct request *rq, bool at_head)
 950 {
 951         struct blk_mq_ctx *ctx = rq->mq_ctx;
 952
 953         trace_block_rq_insert(hctx->queue, rq);
 954
 955         if (at_head)
 956                 list_add(&rq->queuelist, &ctx->rq_list);
 957         else
 958                 list_add_tail(&rq->queuelist, &ctx->rq_list);
 959
 960         blk_mq_hctx_mark_pending(hctx, ctx);
 961
 962         /*
 963          * We do this early, to ensure we are on the right CPU.
 964          */
 965         blk_add_timer(rq);
 966 }
 967
 968 void blk_mq_insert_request(struct request *rq, bool at_head, bool run_queue,
 969                 bool async)
 970 {
 971         struct request_queue *q = rq->q;
 972         struct blk_mq_hw_ctx *hctx;
 973         struct blk_mq_ctx *ctx = rq->mq_ctx, *current_ctx;
 974
 975         current_ctx = blk_mq_get_ctx(q);
 976         if (!cpu_online(ctx->cpu))
 977                 rq->mq_ctx = ctx = current_ctx;
 978
 979         hctx = q->mq_ops->map_queue(q, ctx->cpu);
 980
 981         if (rq->cmd_flags & (REQ_FLUSH | REQ_FUA) &&
 982             !(rq->cmd_flags & (REQ_FLUSH_SEQ))) {
 983                 blk_insert_flush(rq);
 984         } else {
 985                 spin_lock(&ctx->lock);
 986                 __blk_mq_insert_request(hctx, rq, at_head);
 987                 spin_unlock(&ctx->lock);
 988         }
 989
 990         if (run_queue)
 991                 blk_mq_run_hw_queue(hctx, async);
 992
 993         blk_mq_put_ctx(current_ctx);
 994 }
 995
 996 static void blk_mq_insert_requests(struct request_queue *q,
 997                                      struct blk_mq_ctx *ctx,
 998                                      struct list_head *list,
 999                                      int depth,
1000                                      bool from_schedule)
1001
1002 {
1003         struct blk_mq_hw_ctx *hctx;
1004         struct blk_mq_ctx *current_ctx;
1005
1006         trace_block_unplug(q, depth, !from_schedule);
1007
1008         current_ctx = blk_mq_get_ctx(q);
1009
1010         if (!cpu_online(ctx->cpu))
1011                 ctx = current_ctx;
1012         hctx = q->mq_ops->map_queue(q, ctx->cpu);
1013
1014         /*
1015          * preemption doesn't flush plug list, so it's possible ctx->cpu is
1016          * offline now
1017          */
1018         spin_lock(&ctx->lock);
1019         while (!list_empty(list)) {
1020                 struct request *rq;
1021
1022                 rq = list_first_entry(list, struct request, queuelist);
1023                 list_del_init(&rq->queuelist);
1024                 rq->mq_ctx = ctx;
1025                 __blk_mq_insert_request(hctx, rq, false);
1026         }
1027         spin_unlock(&ctx->lock);
1028
1029         blk_mq_run_hw_queue(hctx, from_schedule);
1030         blk_mq_put_ctx(current_ctx);
1031 }
1032
1033 static int plug_ctx_cmp(void *priv, struct list_head *a, struct list_head *b)
1034 {
1035         struct request *rqa = container_of(a, struct request, queuelist);
1036         struct request *rqb = container_of(b, struct request, queuelist);
1037
1038         return !(rqa->mq_ctx < rqb->mq_ctx ||
1039                  (rqa->mq_ctx == rqb->mq_ctx &&
1040                   blk_rq_pos(rqa) < blk_rq_pos(rqb)));
1041 }
1042
1043 void blk_mq_flush_plug_list(struct blk_plug *plug, bool from_schedule)
1044 {
1045         struct blk_mq_ctx *this_ctx;
1046         struct request_queue *this_q;
1047         struct request *rq;
1048         LIST_HEAD(list);
1049         LIST_HEAD(ctx_list);
1050         unsigned int depth;
1051
1052         list_splice_init(&plug->mq_list, &list);
1053
1054         list_sort(NULL, &list, plug_ctx_cmp);
1055
1056         this_q = NULL;
1057         this_ctx = NULL;
1058         depth = 0;
1059
1060         while (!list_empty(&list)) {
1061                 rq = list_entry_rq(list.next);
1062                 list_del_init(&rq->queuelist);
1063                 BUG_ON(!rq->q);
1064                 if (rq->mq_ctx != this_ctx) {
1065                         if (this_ctx) {
1066                                 blk_mq_insert_requests(this_q, this_ctx,
1067                                                         &ctx_list, depth,
1068                                                         from_schedule);
1069                         }
1070
1071                         this_ctx = rq->mq_ctx;
1072                         this_q = rq->q;
1073                         depth = 0;
1074                 }
1075
1076                 depth++;
1077                 list_add_tail(&rq->queuelist, &ctx_list);
1078         }
1079
1080         /*
1081          * If 'this_ctx' is set, we know we have entries to complete
1082          * on 'ctx_list'. Do those.
1083          */
1084         if (this_ctx) {
1085                 blk_mq_insert_requests(this_q, this_ctx, &ctx_list, depth,
1086                                        from_schedule);
1087         }
1088 }
1089
1090 static void blk_mq_bio_to_request(struct request *rq, struct bio *bio)
1091 {
1092         init_request_from_bio(rq, bio);
1093
1094         if (blk_do_io_stat(rq)) {
1095                 rq->start_time = jiffies;
1096                 blk_account_io_start(rq, 1);
1097         }
1098 }
1099
1100 static inline bool blk_mq_merge_queue_io(struct blk_mq_hw_ctx *hctx,
1101                                          struct blk_mq_ctx *ctx,
1102                                          struct request *rq, struct bio *bio)
1103 {
1104         struct request_queue *q = hctx->queue;
1105
1106         if (!(hctx->flags & BLK_MQ_F_SHOULD_MERGE)) {
1107                 blk_mq_bio_to_request(rq, bio);
1108                 spin_lock(&ctx->lock);
1109 insert_rq:
1110                 __blk_mq_insert_request(hctx, rq, false);
1111                 spin_unlock(&ctx->lock);
1112                 return false;
1113         } else {
1114                 spin_lock(&ctx->lock);
1115                 if (!blk_mq_attempt_merge(q, ctx, bio)) {
1116                         blk_mq_bio_to_request(rq, bio);
1117                         goto insert_rq;
1118                 }
1119
1120                 spin_unlock(&ctx->lock);
1121                 __blk_mq_free_request(hctx, ctx, rq);
1122                 return true;
1123         }
1124 }
1125
1126 struct blk_map_ctx {
1127         struct blk_mq_hw_ctx *hctx;
1128         struct blk_mq_ctx *ctx;
1129 };
1130
1131 static struct request *blk_mq_map_request(struct request_queue *q,
1132                                           struct bio *bio,
1133                                           struct blk_map_ctx *data)
1134 {
1135         struct blk_mq_hw_ctx *hctx;
1136         struct blk_mq_ctx *ctx;
1137         struct request *rq;
1138         int rw = bio_data_dir(bio);
1139
1140         if (unlikely(blk_mq_queue_enter(q))) {
1141                 bio_endio(bio, -EIO);
1142                 return NULL;
1143         }
1144
1145         ctx = blk_mq_get_ctx(q);
1146         hctx = q->mq_ops->map_queue(q, ctx->cpu);
1147
1148         if (rw_is_sync(bio->bi_rw))
1149                 rw |= REQ_SYNC;
1150
1151         trace_block_getrq(q, bio, rw);
1152         rq = __blk_mq_alloc_request(q, hctx, ctx, rw, GFP_ATOMIC, false);
1153         if (unlikely(!rq)) {
1154                 __blk_mq_run_hw_queue(hctx);
1155                 blk_mq_put_ctx(ctx);
1156                 trace_block_sleeprq(q, bio, rw);
1157
1158                 ctx = blk_mq_get_ctx(q);
1159                 hctx = q->mq_ops->map_queue(q, ctx->cpu);
1160                 rq = __blk_mq_alloc_request(q, hctx, ctx, rw,
1161                                             __GFP_WAIT|GFP_ATOMIC, false);
1162         }
1163
1164         hctx->queued++;
1165         data->hctx = hctx;
1166         data->ctx = ctx;
1167         return rq;
1168 }
1169
1170 /*
1171  * Multiple hardware queue variant. This will not use per-process plugs,
1172  * but will attempt to bypass the hctx queueing if we can go straight to
1173  * hardware for SYNC IO.
1174  */
1175 static void blk_mq_make_request(struct request_queue *q, struct bio *bio)
1176 {
1177         const int is_sync = rw_is_sync(bio->bi_rw);
1178         const int is_flush_fua = bio->bi_rw & (REQ_FLUSH | REQ_FUA);
1179         struct blk_map_ctx data;
1180         struct request *rq;
1181
1182         blk_queue_bounce(q, &bio);
1183
1184         if (bio_integrity_enabled(bio) && bio_integrity_prep(bio)) {
1185                 bio_endio(bio, -EIO);
1186                 return;
1187         }
1188
1189         rq = blk_mq_map_request(q, bio, &data);
1190         if (unlikely(!rq))
1191                 return;
1192
1193         if (unlikely(is_flush_fua)) {
1194                 blk_mq_bio_to_request(rq, bio);
1195                 blk_insert_flush(rq);
1196                 goto run_queue;
1197         }
1198
1199         if (is_sync) {
1200                 int ret;
1201
1202                 blk_mq_bio_to_request(rq, bio);
1203                 blk_mq_start_request(rq, true);
1204                 blk_add_timer(rq);
1205
1206                 /*
1207                  * For OK queue, we are done. For error, kill it. Any other
1208                  * error (busy), just add it to our list as we previously
1209                  * would have done
1210                  */
1211                 ret = q->mq_ops->queue_rq(data.hctx, rq);
1212                 if (ret == BLK_MQ_RQ_QUEUE_OK)
1213                         goto done;
1214                 else {
1215                         __blk_mq_requeue_request(rq);
1216
1217                         if (ret == BLK_MQ_RQ_QUEUE_ERROR) {
1218                                 rq->errors = -EIO;
1219                                 blk_mq_end_io(rq, rq->errors);
1220                                 goto done;
1221                         }
1222                 }
1223         }
1224
1225         if (!blk_mq_merge_queue_io(data.hctx, data.ctx, rq, bio)) {
1226                 /*
1227                  * For a SYNC request, send it to the hardware immediately. For
1228                  * an ASYNC request, just ensure that we run it later on. The
1229                  * latter allows for merging opportunities and more efficient
1230                  * dispatching.
1231                  */
1232 run_queue:
1233                 blk_mq_run_hw_queue(data.hctx, !is_sync || is_flush_fua);
1234         }
1235 done:
1236         blk_mq_put_ctx(data.ctx);
1237 }
1238
1239 /*
1240  * Single hardware queue variant. This will attempt to use any per-process
1241  * plug for merging and IO deferral.
1242  */
1243 static void blk_sq_make_request(struct request_queue *q, struct bio *bio)
1244 {
1245         const int is_sync = rw_is_sync(bio->bi_rw);
1246         const int is_flush_fua = bio->bi_rw & (REQ_FLUSH | REQ_FUA);
1247         unsigned int use_plug, request_count = 0;
1248         struct blk_map_ctx data;
1249         struct request *rq;
1250
1251         /*
1252          * If we have multiple hardware queues, just go directly to
1253          * one of those for sync IO.
1254          */
1255         use_plug = !is_flush_fua && !is_sync;
1256
1257         blk_queue_bounce(q, &bio);
1258
1259         if (bio_integrity_enabled(bio) && bio_integrity_prep(bio)) {
1260                 bio_endio(bio, -EIO);
1261                 return;
1262         }
1263
1264         if (use_plug && !blk_queue_nomerges(q) &&
1265             blk_attempt_plug_merge(q, bio, &request_count))
1266                 return;
1267
1268         rq = blk_mq_map_request(q, bio, &data);
1269
1270         if (unlikely(is_flush_fua)) {
1271                 blk_mq_bio_to_request(rq, bio);
1272                 blk_insert_flush(rq);
1273                 goto run_queue;
1274         }
1275
1276         /*
1277          * A task plug currently exists. Since this is completely lockless,
1278          * utilize that to temporarily store requests until the task is
1279          * either done or scheduled away.
1280          */
1281         if (use_plug) {
1282                 struct blk_plug *plug = current->plug;
1283
1284                 if (plug) {
1285                         blk_mq_bio_to_request(rq, bio);
1286                         if (list_empty(&plug->mq_list))
1287                                 trace_block_plug(q);
1288                         else if (request_count >= BLK_MAX_REQUEST_COUNT) {
1289                                 blk_flush_plug_list(plug, false);
1290                                 trace_block_plug(q);
1291                         }
1292                         list_add_tail(&rq->queuelist, &plug->mq_list);
1293                         blk_mq_put_ctx(data.ctx);
1294                         return;
1295                 }
1296         }
1297
1298         if (!blk_mq_merge_queue_io(data.hctx, data.ctx, rq, bio)) {
1299                 /*
1300                  * For a SYNC request, send it to the hardware immediately. For
1301                  * an ASYNC request, just ensure that we run it later on. The
1302                  * latter allows for merging opportunities and more efficient
1303                  * dispatching.
1304                  */
1305 run_queue:
1306                 blk_mq_run_hw_queue(data.hctx, !is_sync || is_flush_fua);
1307         }
1308
1309         blk_mq_put_ctx(data.ctx);
1310 }
1311
1312 /*
1313  * Default mapping to a software queue, since we use one per CPU.
1314  */
1315 struct blk_mq_hw_ctx *blk_mq_map_queue(struct request_queue *q, const int cpu)
1316 {
1317         return q->queue_hw_ctx[q->mq_map[cpu]];
1318 }
1319 EXPORT_SYMBOL(blk_mq_map_queue);
1320
1321 static void blk_mq_free_rq_map(struct blk_mq_tag_set *set,
1322                 struct blk_mq_tags *tags, unsigned int hctx_idx)
1323 {
1324         struct page *page;
1325
1326         if (tags->rqs && set->ops->exit_request) {
1327                 int i;
1328
1329                 for (i = 0; i < tags->nr_tags; i++) {
1330                         if (!tags->rqs[i])
1331                                 continue;
1332                         set->ops->exit_request(set->driver_data, tags->rqs[i],
1333                                                 hctx_idx, i);
1334                 }
1335         }
1336
1337         while (!list_empty(&tags->page_list)) {
1338                 page = list_first_entry(&tags->page_list, struct page, lru);
1339                 list_del_init(&page->lru);
1340                 __free_pages(page, page->private);
1341         }
1342
1343         kfree(tags->rqs);
1344
1345         blk_mq_free_tags(tags);
1346 }
1347
1348 static size_t order_to_size(unsigned int order)
1349 {
1350         return (size_t)PAGE_SIZE << order;
1351 }
1352
1353 static struct blk_mq_tags *blk_mq_init_rq_map(struct blk_mq_tag_set *set,
1354                 unsigned int hctx_idx)
1355 {
1356         struct blk_mq_tags *tags;
1357         unsigned int i, j, entries_per_page, max_order = 4;
1358         size_t rq_size, left;
1359
1360         tags = blk_mq_init_tags(set->queue_depth, set->reserved_tags,
1361                                 set->numa_node);
1362         if (!tags)
1363                 return NULL;
1364
1365         INIT_LIST_HEAD(&tags->page_list);
1366
1367         tags->rqs = kmalloc_node(set->queue_depth * sizeof(struct request *),
1368                                         GFP_KERNEL, set->numa_node);
1369         if (!tags->rqs) {
1370                 blk_mq_free_tags(tags);
1371                 return NULL;
1372         }
1373
1374         /*
1375          * rq_size is the size of the request plus driver payload, rounded
1376          * to the cacheline size
1377          */
1378         rq_size = round_up(sizeof(struct request) + set->cmd_size,
1379                                 cache_line_size());
1380         left = rq_size * set->queue_depth;
1381
1382         for (i = 0; i < set->queue_depth; ) {
1383                 int this_order = max_order;
1384                 struct page *page;
1385                 int to_do;
1386                 void *p;
1387
1388                 while (left < order_to_size(this_order - 1) && this_order)
1389                         this_order--;
1390
1391                 do {
1392                         page = alloc_pages_node(set->numa_node, GFP_KERNEL,
1393                                                 this_order);
1394                         if (page)
1395                                 break;
1396                         if (!this_order--)
1397                                 break;
1398                         if (order_to_size(this_order) < rq_size)
1399                                 break;
1400                 } while (1);
1401
1402                 if (!page)
1403                         goto fail;
1404
1405                 page->private = this_order;
1406                 list_add_tail(&page->lru, &tags->page_list);
1407
1408                 p = page_address(page);
1409                 entries_per_page = order_to_size(this_order) / rq_size;
1410                 to_do = min(entries_per_page, set->queue_depth - i);
1411                 left -= to_do * rq_size;
1412                 for (j = 0; j < to_do; j++) {
1413                         tags->rqs[i] = p;
1414                         if (set->ops->init_request) {
1415                                 if (set->ops->init_request(set->driver_data,
1416                                                 tags->rqs[i], hctx_idx, i,
1417                                                 set->numa_node))
1418                                         goto fail;
1419                         }
1420
1421                         p += rq_size;
1422                         i++;
1423                 }
1424         }
1425
1426         return tags;
1427
1428 fail:
1429         pr_warn("%s: failed to allocate requests\n", __func__);
1430         blk_mq_free_rq_map(set, tags, hctx_idx);
1431         return NULL;
1432 }
1433
1434 static void blk_mq_free_bitmap(struct blk_mq_ctxmap *bitmap)
1435 {
1436         kfree(bitmap->map);
1437 }
1438
1439 static int blk_mq_alloc_bitmap(struct blk_mq_ctxmap *bitmap, int node)
1440 {
1441         unsigned int bpw = 8, total, num_maps, i;
1442
1443         bitmap->bits_per_word = bpw;
1444
1445         num_maps = ALIGN(nr_cpu_ids, bpw) / bpw;
1446         bitmap->map = kzalloc_node(num_maps * sizeof(struct blk_align_bitmap),
1447                                         GFP_KERNEL, node);
1448         if (!bitmap->map)
1449                 return -ENOMEM;
1450
1451         bitmap->map_size = num_maps;
1452
1453         total = nr_cpu_ids;
1454         for (i = 0; i < num_maps; i++) {
1455                 bitmap->map[i].depth = min(total, bitmap->bits_per_word);
1456                 total -= bitmap->map[i].depth;
1457         }
1458
1459         return 0;
1460 }
1461
1462 static int blk_mq_hctx_cpu_offline(struct blk_mq_hw_ctx *hctx, int cpu)
1463 {
1464         struct request_queue *q = hctx->queue;
1465         struct blk_mq_ctx *ctx;
1466         LIST_HEAD(tmp);
1467
1468         /*
1469          * Move ctx entries to new CPU, if this one is going away.
1470          */
1471         ctx = __blk_mq_get_ctx(q, cpu);
1472
1473         spin_lock(&ctx->lock);
1474         if (!list_empty(&ctx->rq_list)) {
1475                 list_splice_init(&ctx->rq_list, &tmp);
1476                 blk_mq_hctx_clear_pending(hctx, ctx);
1477         }
1478         spin_unlock(&ctx->lock);
1479
1480         if (list_empty(&tmp))
1481                 return NOTIFY_OK;
1482
1483         ctx = blk_mq_get_ctx(q);
1484         spin_lock(&ctx->lock);
1485
1486         while (!list_empty(&tmp)) {
1487                 struct request *rq;
1488
1489                 rq = list_first_entry(&tmp, struct request, queuelist);
1490                 rq->mq_ctx = ctx;
1491                 list_move_tail(&rq->queuelist, &ctx->rq_list);
1492         }
1493
1494         hctx = q->mq_ops->map_queue(q, ctx->cpu);
1495         blk_mq_hctx_mark_pending(hctx, ctx);
1496
1497         spin_unlock(&ctx->lock);
1498
1499         blk_mq_run_hw_queue(hctx, true);
1500         blk_mq_put_ctx(ctx);
1501         return NOTIFY_OK;
1502 }
1503
1504 static int blk_mq_hctx_cpu_online(struct blk_mq_hw_ctx *hctx, int cpu)
1505 {
1506         struct request_queue *q = hctx->queue;
1507         struct blk_mq_tag_set *set = q->tag_set;
1508
1509         if (set->tags[hctx->queue_num])
1510                 return NOTIFY_OK;
1511
1512         set->tags[hctx->queue_num] = blk_mq_init_rq_map(set, hctx->queue_num);
1513         if (!set->tags[hctx->queue_num])
1514                 return NOTIFY_STOP;
1515
1516         hctx->tags = set->tags[hctx->queue_num];
1517         return NOTIFY_OK;
1518 }
1519
1520 static int blk_mq_hctx_notify(void *data, unsigned long action,
1521                               unsigned int cpu)
1522 {
1523         struct blk_mq_hw_ctx *hctx = data;
1524
1525         if (action == CPU_DEAD || action == CPU_DEAD_FROZEN)
1526                 return blk_mq_hctx_cpu_offline(hctx, cpu);
1527         else if (action == CPU_ONLINE || action == CPU_ONLINE_FROZEN)
1528                 return blk_mq_hctx_cpu_online(hctx, cpu);
1529
1530         return NOTIFY_OK;
1531 }
1532
1533 static void blk_mq_exit_hw_queues(struct request_queue *q,
1534                 struct blk_mq_tag_set *set, int nr_queue)
1535 {
1536         struct blk_mq_hw_ctx *hctx;
1537         unsigned int i;
1538
1539         queue_for_each_hw_ctx(q, hctx, i) {
1540                 if (i == nr_queue)
1541                         break;
1542
1543                 if (set->ops->exit_hctx)
1544                         set->ops->exit_hctx(hctx, i);
1545
1546                 blk_mq_unregister_cpu_notifier(&hctx->cpu_notifier);
1547                 kfree(hctx->ctxs);
1548                 blk_mq_free_bitmap(&hctx->ctx_map);
1549         }
1550
1551 }
1552
1553 static void blk_mq_free_hw_queues(struct request_queue *q,
1554                 struct blk_mq_tag_set *set)
1555 {
1556         struct blk_mq_hw_ctx *hctx;
1557         unsigned int i;
1558
1559         queue_for_each_hw_ctx(q, hctx, i) {
1560                 free_cpumask_var(hctx->cpumask);
1561                 kfree(hctx);
1562         }
1563 }
1564
1565 static int blk_mq_init_hw_queues(struct request_queue *q,
1566                 struct blk_mq_tag_set *set)
1567 {
1568         struct blk_mq_hw_ctx *hctx;
1569         unsigned int i;
1570
1571         /*
1572          * Initialize hardware queues
1573          */
1574         queue_for_each_hw_ctx(q, hctx, i) {
1575                 int node;
1576
1577                 node = hctx->numa_node;
1578                 if (node == NUMA_NO_NODE)
1579                         node = hctx->numa_node = set->numa_node;
1580
1581                 INIT_DELAYED_WORK(&hctx->run_work, blk_mq_run_work_fn);
1582                 INIT_DELAYED_WORK(&hctx->delay_work, blk_mq_delay_work_fn);
1583                 spin_lock_init(&hctx->lock);
1584                 INIT_LIST_HEAD(&hctx->dispatch);
1585                 hctx->queue = q;
1586                 hctx->queue_num = i;
1587                 hctx->flags = set->flags;
1588                 hctx->cmd_size = set->cmd_size;
1589
1590                 blk_mq_init_cpu_notifier(&hctx->cpu_notifier,
1591                                                 blk_mq_hctx_notify, hctx);
1592                 blk_mq_register_cpu_notifier(&hctx->cpu_notifier);
1593
1594                 hctx->tags = set->tags[i];
1595
1596                 /*
1597                  * Allocate space for all possible cpus to avoid allocation in
1598                  * runtime
1599                  */
1600                 hctx->ctxs = kmalloc_node(nr_cpu_ids * sizeof(void *),
1601                                                 GFP_KERNEL, node);
1602                 if (!hctx->ctxs)
1603                         break;
1604
1605                 if (blk_mq_alloc_bitmap(&hctx->ctx_map, node))
1606                         break;
1607
1608                 hctx->nr_ctx = 0;
1609
1610                 if (set->ops->init_hctx &&
1611                     set->ops->init_hctx(hctx, set->driver_data, i))
1612                         break;
1613         }
1614
1615         if (i == q->nr_hw_queues)
1616                 return 0;
1617
1618         /*
1619          * Init failed
1620          */
1621         blk_mq_exit_hw_queues(q, set, i);
1622
1623         return 1;
1624 }
1625
1626 static void blk_mq_init_cpu_queues(struct request_queue *q,
1627                                    unsigned int nr_hw_queues)
1628 {
1629         unsigned int i;
1630
1631         for_each_possible_cpu(i) {
1632                 struct blk_mq_ctx *__ctx = per_cpu_ptr(q->queue_ctx, i);
1633                 struct blk_mq_hw_ctx *hctx;
1634
1635                 memset(__ctx, 0, sizeof(*__ctx));
1636                 __ctx->cpu = i;
1637                 spin_lock_init(&__ctx->lock);
1638                 INIT_LIST_HEAD(&__ctx->rq_list);
1639                 __ctx->queue = q;
1640
1641                 /* If the cpu isn't online, the cpu is mapped to first hctx */
1642                 if (!cpu_online(i))
1643                         continue;
1644
1645                 hctx = q->mq_ops->map_queue(q, i);
1646                 cpumask_set_cpu(i, hctx->cpumask);
1647                 hctx->nr_ctx++;
1648
1649                 /*
1650                  * Set local node, IFF we have more than one hw queue. If
1651                  * not, we remain on the home node of the device
1652                  */
1653                 if (nr_hw_queues > 1 && hctx->numa_node == NUMA_NO_NODE)
1654                         hctx->numa_node = cpu_to_node(i);
1655         }
1656 }
1657
1658 static void blk_mq_map_swqueue(struct request_queue *q)
1659 {
1660         unsigned int i;
1661         struct blk_mq_hw_ctx *hctx;
1662         struct blk_mq_ctx *ctx;
1663
1664         queue_for_each_hw_ctx(q, hctx, i) {
1665                 cpumask_clear(hctx->cpumask);
1666                 hctx->nr_ctx = 0;
1667         }
1668
1669         /*
1670          * Map software to hardware queues
1671          */
1672         queue_for_each_ctx(q, ctx, i) {
1673                 /* If the cpu isn't online, the cpu is mapped to first hctx */
1674                 if (!cpu_online(i))
1675                         continue;
1676
1677                 hctx = q->mq_ops->map_queue(q, i);
1678                 cpumask_set_cpu(i, hctx->cpumask);
1679                 ctx->index_hw = hctx->nr_ctx;
1680                 hctx->ctxs[hctx->nr_ctx++] = ctx;
1681         }
1682
1683         queue_for_each_hw_ctx(q, hctx, i) {
1684                 /*
1685                  * If not software queues are mapped to this hardware queue,
1686                  * disable it and free the request entries
1687                  */
1688                 if (!hctx->nr_ctx) {
1689                         struct blk_mq_tag_set *set = q->tag_set;
1690
1691                         if (set->tags[i]) {
1692                                 blk_mq_free_rq_map(set, set->tags[i], i);
1693                                 set->tags[i] = NULL;
1694                                 hctx->tags = NULL;
1695                         }
1696                         continue;
1697                 }
1698
1699                 /*
1700                  * Initialize batch roundrobin counts
1701                  */
1702                 hctx->next_cpu = cpumask_first(hctx->cpumask);
1703                 hctx->next_cpu_batch = BLK_MQ_CPU_WORK_BATCH;
1704         }
1705 }
1706
1707 static void blk_mq_update_tag_set_depth(struct blk_mq_tag_set *set)
1708 {
1709         struct blk_mq_hw_ctx *hctx;
1710         struct request_queue *q;
1711         bool shared;
1712         int i;
1713
1714         if (set->tag_list.next == set->tag_list.prev)
1715                 shared = false;
1716         else
1717                 shared = true;
1718
1719         list_for_each_entry(q, &set->tag_list, tag_set_list) {
1720                 blk_mq_freeze_queue(q);
1721
1722                 queue_for_each_hw_ctx(q, hctx, i) {
1723                         if (shared)
1724                                 hctx->flags |= BLK_MQ_F_TAG_SHARED;
1725                         else
1726                                 hctx->flags &= ~BLK_MQ_F_TAG_SHARED;
1727                 }
1728                 blk_mq_unfreeze_queue(q);
1729         }
1730 }
1731
1732 static void blk_mq_del_queue_tag_set(struct request_queue *q)
1733 {
1734         struct blk_mq_tag_set *set = q->tag_set;
1735
1736         blk_mq_freeze_queue(q);
1737
1738         mutex_lock(&set->tag_list_lock);
1739         list_del_init(&q->tag_set_list);
1740         blk_mq_update_tag_set_depth(set);
1741         mutex_unlock(&set->tag_list_lock);
1742
1743         blk_mq_unfreeze_queue(q);
1744 }
1745
1746 static void blk_mq_add_queue_tag_set(struct blk_mq_tag_set *set,
1747                                      struct request_queue *q)
1748 {
1749         q->tag_set = set;
1750
1751         mutex_lock(&set->tag_list_lock);
1752         list_add_tail(&q->tag_set_list, &set->tag_list);
1753         blk_mq_update_tag_set_depth(set);
1754         mutex_unlock(&set->tag_list_lock);
1755 }
1756
1757 struct request_queue *blk_mq_init_queue(struct blk_mq_tag_set *set)
1758 {
1759         struct blk_mq_hw_ctx **hctxs;
1760         struct blk_mq_ctx *ctx;
1761         struct request_queue *q;
1762         unsigned int *map;
1763         int i;
1764
1765         ctx = alloc_percpu(struct blk_mq_ctx);
1766         if (!ctx)
1767                 return ERR_PTR(-ENOMEM);
1768
1769         hctxs = kmalloc_node(set->nr_hw_queues * sizeof(*hctxs), GFP_KERNEL,
1770                         set->numa_node);
1771
1772         if (!hctxs)
1773                 goto err_percpu;
1774
1775         map = blk_mq_make_queue_map(set);
1776         if (!map)
1777                 goto err_map;
1778
1779         for (i = 0; i < set->nr_hw_queues; i++) {
1780                 int node = blk_mq_hw_queue_to_node(map, i);
1781
1782                 hctxs[i] = kzalloc_node(sizeof(struct blk_mq_hw_ctx),
1783                                         GFP_KERNEL, node);
1784                 if (!hctxs[i])
1785                         goto err_hctxs;
1786
1787                 if (!zalloc_cpumask_var(&hctxs[i]->cpumask, GFP_KERNEL))
1788                         goto err_hctxs;
1789
1790                 atomic_set(&hctxs[i]->nr_active, 0);
1791                 hctxs[i]->numa_node = node;
1792                 hctxs[i]->queue_num = i;
1793         }
1794
1795         q = blk_alloc_queue_node(GFP_KERNEL, set->numa_node);
1796         if (!q)
1797                 goto err_hctxs;
1798
1799         if (percpu_counter_init(&q->mq_usage_counter, 0))
1800                 goto err_map;
1801
1802         setup_timer(&q->timeout, blk_mq_rq_timer, (unsigned long) q);
1803         blk_queue_rq_timeout(q, 30000);
1804
1805         q->nr_queues = nr_cpu_ids;
1806         q->nr_hw_queues = set->nr_hw_queues;
1807         q->mq_map = map;
1808
1809         q->queue_ctx = ctx;
1810         q->queue_hw_ctx = hctxs;
1811
1812         q->mq_ops = set->ops;
1813         q->queue_flags |= QUEUE_FLAG_MQ_DEFAULT;
1814
1815         if (!(set->flags & BLK_MQ_F_SG_MERGE))
1816                 q->queue_flags |= 1 << QUEUE_FLAG_NO_SG_MERGE;
1817
1818         q->sg_reserved_size = INT_MAX;
1819
1820         INIT_WORK(&q->requeue_work, blk_mq_requeue_work);
1821         INIT_LIST_HEAD(&q->requeue_list);
1822         spin_lock_init(&q->requeue_lock);
1823
1824         if (q->nr_hw_queues > 1)
1825                 blk_queue_make_request(q, blk_mq_make_request);
1826         else
1827                 blk_queue_make_request(q, blk_sq_make_request);
1828
1829         blk_queue_rq_timed_out(q, blk_mq_rq_timed_out);
1830         if (set->timeout)
1831                 blk_queue_rq_timeout(q, set->timeout);
1832
1833         /*
1834          * Do this after blk_queue_make_request() overrides it...
1835          */
1836         q->nr_requests = set->queue_depth;
1837
1838         if (set->ops->complete)
1839                 blk_queue_softirq_done(q, set->ops->complete);
1840
1841         blk_mq_init_flush(q);
1842         blk_mq_init_cpu_queues(q, set->nr_hw_queues);
1843
1844         q->flush_rq = kzalloc(round_up(sizeof(struct request) +
1845                                 set->cmd_size, cache_line_size()),
1846                                 GFP_KERNEL);
1847         if (!q->flush_rq)
1848                 goto err_hw;
1849
1850         if (blk_mq_init_hw_queues(q, set))
1851                 goto err_flush_rq;
1852
1853         mutex_lock(&all_q_mutex);
1854         list_add_tail(&q->all_q_node, &all_q_list);
1855         mutex_unlock(&all_q_mutex);
1856
1857         blk_mq_add_queue_tag_set(set, q);
1858
1859         blk_mq_map_swqueue(q);
1860
1861         return q;
1862
1863 err_flush_rq:
1864         kfree(q->flush_rq);
1865 err_hw:
1866         blk_cleanup_queue(q);
1867 err_hctxs:
1868         kfree(map);
1869         for (i = 0; i < set->nr_hw_queues; i++) {
1870                 if (!hctxs[i])
1871                         break;
1872                 free_cpumask_var(hctxs[i]->cpumask);
1873                 kfree(hctxs[i]);
1874         }
1875 err_map:
1876         kfree(hctxs);
1877 err_percpu:
1878         free_percpu(ctx);
1879         return ERR_PTR(-ENOMEM);
1880 }
1881 EXPORT_SYMBOL(blk_mq_init_queue);
1882
1883 void blk_mq_free_queue(struct request_queue *q)
1884 {
1885         struct blk_mq_tag_set   *set = q->tag_set;
1886
1887         blk_mq_del_queue_tag_set(q);
1888
1889         blk_mq_exit_hw_queues(q, set, set->nr_hw_queues);
1890         blk_mq_free_hw_queues(q, set);
1891
1892         percpu_counter_destroy(&q->mq_usage_counter);
1893
1894         free_percpu(q->queue_ctx);
1895         kfree(q->queue_hw_ctx);
1896         kfree(q->mq_map);
1897
1898         q->queue_ctx = NULL;
1899         q->queue_hw_ctx = NULL;
1900         q->mq_map = NULL;
1901
1902         mutex_lock(&all_q_mutex);
1903         list_del_init(&q->all_q_node);
1904         mutex_unlock(&all_q_mutex);
1905 }
1906
1907 /* Basically redo blk_mq_init_queue with queue frozen */
1908 static void blk_mq_queue_reinit(struct request_queue *q)
1909 {
1910         blk_mq_freeze_queue(q);
1911
1912         blk_mq_sysfs_unregister(q);
1913
1914         blk_mq_update_queue_map(q->mq_map, q->nr_hw_queues);
1915
1916         /*
1917          * redo blk_mq_init_cpu_queues and blk_mq_init_hw_queues. FIXME: maybe
1918          * we should change hctx numa_node according to new topology (this
1919          * involves free and re-allocate memory, worthy doing?)
1920          */
1921
1922         blk_mq_map_swqueue(q);
1923
1924         blk_mq_sysfs_register(q);
1925
1926         blk_mq_unfreeze_queue(q);
1927 }
1928
1929 static int blk_mq_queue_reinit_notify(struct notifier_block *nb,
1930                                       unsigned long action, void *hcpu)
1931 {
1932         struct request_queue *q;
1933
1934         /*
1935          * Before new mappings are established, hotadded cpu might already
1936          * start handling requests. This doesn't break anything as we map
1937          * offline CPUs to first hardware queue. We will re-init the queue
1938          * below to get optimal settings.
1939          */
1940         if (action != CPU_DEAD && action != CPU_DEAD_FROZEN &&
1941             action != CPU_ONLINE && action != CPU_ONLINE_FROZEN)
1942                 return NOTIFY_OK;
1943
1944         mutex_lock(&all_q_mutex);
1945         list_for_each_entry(q, &all_q_list, all_q_node)
1946                 blk_mq_queue_reinit(q);
1947         mutex_unlock(&all_q_mutex);
1948         return NOTIFY_OK;
1949 }
1950
1951 int blk_mq_alloc_tag_set(struct blk_mq_tag_set *set)
1952 {
1953         int i;
1954
1955         if (!set->nr_hw_queues)
1956                 return -EINVAL;
1957         if (!set->queue_depth || set->queue_depth > BLK_MQ_MAX_DEPTH)
1958                 return -EINVAL;
1959         if (set->queue_depth < set->reserved_tags + BLK_MQ_TAG_MIN)
1960                 return -EINVAL;
1961
1962         if (!set->nr_hw_queues || !set->ops->queue_rq || !set->ops->map_queue)
1963                 return -EINVAL;
1964
1965
1966         set->tags = kmalloc_node(set->nr_hw_queues *
1967                                  sizeof(struct blk_mq_tags *),
1968                                  GFP_KERNEL, set->numa_node);
1969         if (!set->tags)
1970                 goto out;
1971
1972         for (i = 0; i < set->nr_hw_queues; i++) {
1973                 set->tags[i] = blk_mq_init_rq_map(set, i);
1974                 if (!set->tags[i])
1975                         goto out_unwind;
1976         }
1977
1978         mutex_init(&set->tag_list_lock);
1979         INIT_LIST_HEAD(&set->tag_list);
1980
1981         return 0;
1982
1983 out_unwind:
1984         while (--i >= 0)
1985                 blk_mq_free_rq_map(set, set->tags[i], i);
1986 out:
1987         return -ENOMEM;
1988 }
1989 EXPORT_SYMBOL(blk_mq_alloc_tag_set);
1990
1991 void blk_mq_free_tag_set(struct blk_mq_tag_set *set)
1992 {
1993         int i;
1994
1995         for (i = 0; i < set->nr_hw_queues; i++) {
1996                 if (set->tags[i])
1997                         blk_mq_free_rq_map(set, set->tags[i], i);
1998         }
1999
2000         kfree(set->tags);
2001 }
2002 EXPORT_SYMBOL(blk_mq_free_tag_set);
2003
2004 int blk_mq_update_nr_requests(struct request_queue *q, unsigned int nr)
2005 {
2006         struct blk_mq_tag_set *set = q->tag_set;
2007         struct blk_mq_hw_ctx *hctx;
2008         int i, ret;
2009
2010         if (!set || nr > set->queue_depth)
2011                 return -EINVAL;
2012
2013         ret = 0;
2014         queue_for_each_hw_ctx(q, hctx, i) {
2015                 ret = blk_mq_tag_update_depth(hctx->tags, nr);
2016                 if (ret)
2017                         break;
2018         }
2019
2020         if (!ret)
2021                 q->nr_requests = nr;
2022
2023         return ret;
2024 }
2025
2026 void blk_mq_disable_hotplug(void)
2027 {
2028         mutex_lock(&all_q_mutex);
2029 }
2030
2031 void blk_mq_enable_hotplug(void)
2032 {
2033         mutex_unlock(&all_q_mutex);
2034 }
2035
2036 static int __init blk_mq_init(void)
2037 {
2038         blk_mq_cpu_init();
2039
2040         /* Must be called after percpu_counter_hotcpu_callback() */
2041         hotcpu_notifier(blk_mq_queue_reinit_notify, -10);
2042
2043         return 0;
2044 }
2045 subsys_initcall(blk_mq_init);