Merge branch 'x86-uaccess-for-linus' of git://git.kernel.org/pub/scm/linux/kernel...

[firefly-linux-kernel-4.4.55.git] / kernel / stop_machine.c

diff --git a/kernel/stop_machine.c b/kernel/stop_machine.c
index 32a6c44d8f7837f2f4b5d0fe86ce873cf767dc85..84571e09c9079e8887f73a0f24beefe58e00ff77 100644 (file)
--- a/kernel/stop_machine.c
+++ b/kernel/stop_machine.c
@@ -20,6 +20,7 @@
 #include <linux/kallsyms.h>
 #include <linux/smpboot.h>
 #include <linux/atomic.h>
+#include <linux/lglock.h>
 
 /*
  * Structure to determine completion condition and record errors.  May
@@ -43,6 +44,14 @@ static DEFINE_PER_CPU(struct cpu_stopper, cpu_stopper);
 static DEFINE_PER_CPU(struct task_struct *, cpu_stopper_task);
 static bool stop_machine_initialized = false;
 
+/*
+ * Avoids a race between stop_two_cpus and global stop_cpus, where
+ * the stoppers could get queued up in reverse order, leading to
+ * system deadlock. Using an lglock means stop_two_cpus remains
+ * relatively cheap.
+ */
+DEFINE_STATIC_LGLOCK(stop_cpus_lock);
+
 static void cpu_stop_init_done(struct cpu_stop_done *done, unsigned int nr_todo)
 {
        memset(done, 0, sizeof(*done));
@@ -234,11 +243,13 @@ static void irq_cpu_stop_queue_work(void *arg)
  */
 int stop_two_cpus(unsigned int cpu1, unsigned int cpu2, cpu_stop_fn_t fn, void *arg)
 {
-       int call_cpu;
        struct cpu_stop_done done;
        struct cpu_stop_work work1, work2;
        struct irq_cpu_stop_queue_work_info call_args;
-       struct multi_stop_data msdata = {
+       struct multi_stop_data msdata;
+
+       preempt_disable();
+       msdata = (struct multi_stop_data){
                .fn = fn,
                .data = arg,
                .num_threads = 2,
@@ -261,17 +272,33 @@ int stop_two_cpus(unsigned int cpu1, unsigned int cpu2, cpu_stop_fn_t fn, void *
        cpu_stop_init_done(&done, 2);
        set_state(&msdata, MULTI_STOP_PREPARE);
 
+       /*
+        * If we observe both CPUs active we know _cpu_down() cannot yet have
+        * queued its stop_machine works and therefore ours will get executed
+        * first. Or its not either one of our CPUs that's getting unplugged,
+        * in which case we don't care.
+        *
+        * This relies on the stopper workqueues to be FIFO.
+        */
+       if (!cpu_active(cpu1) || !cpu_active(cpu2)) {
+               preempt_enable();
+               return -ENOENT;
+       }
+
+       lg_local_lock(&stop_cpus_lock);
        /*
         * Queuing needs to be done by the lowest numbered CPU, to ensure
         * that works are always queued in the same order on every CPU.
         * This prevents deadlocks.
         */
-       call_cpu = min(cpu1, cpu2);
-
-       smp_call_function_single(call_cpu, &irq_cpu_stop_queue_work,
+       smp_call_function_single(min(cpu1, cpu2),
+                                &irq_cpu_stop_queue_work,
                                 &call_args, 0);
+       lg_local_unlock(&stop_cpus_lock);
+       preempt_enable();
 
        wait_for_completion(&done.completion);
+
        return done.executed ? done.ret : -ENOENT;
 }
 
@@ -319,10 +346,10 @@ static void queue_stop_cpus_work(const struct cpumask *cpumask,
         * preempted by a stopper which might wait for other stoppers
         * to enter @fn which can lead to deadlock.
         */
-       preempt_disable();
+       lg_global_lock(&stop_cpus_lock);
        for_each_cpu(cpu, cpumask)
                cpu_stop_queue_work(cpu, &per_cpu(stop_cpus_work, cpu));
-       preempt_enable();
+       lg_global_unlock(&stop_cpus_lock);
 }
 
 static int __stop_cpus(const struct cpumask *cpumask,