block/blk-mq-cpumap.c

   1 #include <linux/kernel.h>
   2 #include <linux/threads.h>
   3 #include <linux/module.h>
   4 #include <linux/mm.h>
   5 #include <linux/smp.h>
   6 #include <linux/cpu.h>
   7
   8 #include <linux/blk-mq.h>
   9 #include "blk.h"
  10 #include "blk-mq.h"
  11
  12 static void show_map(unsigned int *map, unsigned int nr)
  13 {
  14         int i;
  15
  16         pr_info("blk-mq: CPU -> queue map\n");
  17         for_each_online_cpu(i)
  18                 pr_info("  CPU%2u -> Queue %u\n", i, map[i]);
  19 }
  20
  21 static int cpu_to_queue_index(unsigned int nr_cpus, unsigned int nr_queues,
  22                               const int cpu)
  23 {
  24         return cpu / ((nr_cpus + nr_queues - 1) / nr_queues);
  25 }
  26
  27 static int get_first_sibling(unsigned int cpu)
  28 {
  29         unsigned int ret;
  30
  31         ret = cpumask_first(topology_thread_cpumask(cpu));
  32         if (ret < nr_cpu_ids)
  33                 return ret;
  34
  35         return cpu;
  36 }
  37
  38 int blk_mq_update_queue_map(unsigned int *map, unsigned int nr_queues)
  39 {
  40         unsigned int i, nr_cpus, nr_uniq_cpus, queue, first_sibling;
  41         cpumask_var_t cpus;
  42
  43         if (!alloc_cpumask_var(&cpus, GFP_ATOMIC))
  44                 return 1;
  45
  46         cpumask_clear(cpus);
  47         nr_cpus = nr_uniq_cpus = 0;
  48         for_each_online_cpu(i) {
  49                 nr_cpus++;
  50                 first_sibling = get_first_sibling(i);
  51                 if (!cpumask_test_cpu(first_sibling, cpus))
  52                         nr_uniq_cpus++;
  53                 cpumask_set_cpu(i, cpus);
  54         }
  55
  56         queue = 0;
  57         for_each_possible_cpu(i) {
  58                 if (!cpu_online(i)) {
  59                         map[i] = 0;
  60                         continue;
  61                 }
  62
  63                 /*
  64                  * Easy case - we have equal or more hardware queues. Or
  65                  * there are no thread siblings to take into account. Do
  66                  * 1:1 if enough, or sequential mapping if less.
  67                  */
  68                 if (nr_queues >= nr_cpus || nr_cpus == nr_uniq_cpus) {
  69                         map[i] = cpu_to_queue_index(nr_cpus, nr_queues, queue);
  70                         queue++;
  71                         continue;
  72                 }
  73
  74                 /*
  75                  * Less then nr_cpus queues, and we have some number of
  76                  * threads per cores. Map sibling threads to the same
  77                  * queue.
  78                  */
  79                 first_sibling = get_first_sibling(i);
  80                 if (first_sibling == i) {
  81                         map[i] = cpu_to_queue_index(nr_uniq_cpus, nr_queues,
  82                                                         queue);
  83                         queue++;
  84                 } else
  85                         map[i] = map[first_sibling];
  86         }
  87
  88         show_map(map, nr_cpus);
  89         free_cpumask_var(cpus);
  90         return 0;
  91 }
  92
  93 unsigned int *blk_mq_make_queue_map(struct blk_mq_reg *reg)
  94 {
  95         unsigned int *map;
  96
  97         /* If cpus are offline, map them to first hctx */
  98         map = kzalloc_node(sizeof(*map) * num_possible_cpus(), GFP_KERNEL,
  99                                 reg->numa_node);
 100         if (!map)
 101                 return NULL;
 102
 103         if (!blk_mq_update_queue_map(map, reg->nr_hw_queues))
 104                 return map;
 105
 106         kfree(map);
 107         return NULL;
 108 }