Merge git://git.kvack.org/~bcrl/aio-next
[firefly-linux-kernel-4.4.55.git] / block / blk-mq-cpumap.c
1 #include <linux/kernel.h>
2 #include <linux/threads.h>
3 #include <linux/module.h>
4 #include <linux/mm.h>
5 #include <linux/smp.h>
6 #include <linux/cpu.h>
7
8 #include <linux/blk-mq.h>
9 #include "blk.h"
10 #include "blk-mq.h"
11
12 static void show_map(unsigned int *map, unsigned int nr)
13 {
14         int i;
15
16         pr_info("blk-mq: CPU -> queue map\n");
17         for_each_online_cpu(i)
18                 pr_info("  CPU%2u -> Queue %u\n", i, map[i]);
19 }
20
21 static int cpu_to_queue_index(unsigned int nr_cpus, unsigned int nr_queues,
22                               const int cpu)
23 {
24         return cpu / ((nr_cpus + nr_queues - 1) / nr_queues);
25 }
26
27 static int get_first_sibling(unsigned int cpu)
28 {
29         unsigned int ret;
30
31         ret = cpumask_first(topology_thread_cpumask(cpu));
32         if (ret < nr_cpu_ids)
33                 return ret;
34
35         return cpu;
36 }
37
38 int blk_mq_update_queue_map(unsigned int *map, unsigned int nr_queues)
39 {
40         unsigned int i, nr_cpus, nr_uniq_cpus, queue, first_sibling;
41         cpumask_var_t cpus;
42
43         if (!alloc_cpumask_var(&cpus, GFP_ATOMIC))
44                 return 1;
45
46         cpumask_clear(cpus);
47         nr_cpus = nr_uniq_cpus = 0;
48         for_each_online_cpu(i) {
49                 nr_cpus++;
50                 first_sibling = get_first_sibling(i);
51                 if (!cpumask_test_cpu(first_sibling, cpus))
52                         nr_uniq_cpus++;
53                 cpumask_set_cpu(i, cpus);
54         }
55
56         queue = 0;
57         for_each_possible_cpu(i) {
58                 if (!cpu_online(i)) {
59                         map[i] = 0;
60                         continue;
61                 }
62
63                 /*
64                  * Easy case - we have equal or more hardware queues. Or
65                  * there are no thread siblings to take into account. Do
66                  * 1:1 if enough, or sequential mapping if less.
67                  */
68                 if (nr_queues >= nr_cpus || nr_cpus == nr_uniq_cpus) {
69                         map[i] = cpu_to_queue_index(nr_cpus, nr_queues, queue);
70                         queue++;
71                         continue;
72                 }
73
74                 /*
75                  * Less then nr_cpus queues, and we have some number of
76                  * threads per cores. Map sibling threads to the same
77                  * queue.
78                  */
79                 first_sibling = get_first_sibling(i);
80                 if (first_sibling == i) {
81                         map[i] = cpu_to_queue_index(nr_uniq_cpus, nr_queues,
82                                                         queue);
83                         queue++;
84                 } else
85                         map[i] = map[first_sibling];
86         }
87
88         show_map(map, nr_cpus);
89         free_cpumask_var(cpus);
90         return 0;
91 }
92
93 unsigned int *blk_mq_make_queue_map(struct blk_mq_reg *reg)
94 {
95         unsigned int *map;
96
97         /* If cpus are offline, map them to first hctx */
98         map = kzalloc_node(sizeof(*map) * num_possible_cpus(), GFP_KERNEL,
99                                 reg->numa_node);
100         if (!map)
101                 return NULL;
102
103         if (!blk_mq_update_queue_map(map, reg->nr_hw_queues))
104                 return map;
105
106         kfree(map);
107         return NULL;
108 }