mm/migrate: correct failure handling if !hugepage_migration_support()

[firefly-linux-kernel-4.4.55.git] / kernel / sched / clock.c

diff --git a/kernel/sched/clock.c b/kernel/sched/clock.c
index c3ae1446461c0ffc22f41cc03a7504a84576b48b..6bd6a6731b21220cae9c9691583c59d9c992ff6f 100644 (file)
--- a/kernel/sched/clock.c
+++ b/kernel/sched/clock.c
@@ -26,9 +26,10 @@
  * at 0 on boot (but people really shouldn't rely on that).
  *
  * cpu_clock(i)       -- can be used from any context, including NMI.
- * sched_clock_cpu(i) -- must be used with local IRQs disabled (implied by NMI)
  * local_clock()      -- is cpu_clock() on the current cpu.
  *
+ * sched_clock_cpu(i)
+ *
  * How:
  *
  * The implementation either uses sched_clock() when
@@ -50,15 +51,6 @@
  * Furthermore, explicit sleep and wakeup hooks allow us to account for time
  * that is otherwise invisible (TSC gets stopped).
  *
- *
- * Notes:
- *
- * The !IRQ-safetly of sched_clock() and sched_clock_cpu() comes from things
- * like cpufreq interrupts that can change the base clock (TSC) multiplier
- * and cause funny jumps in time -- although the filtering provided by
- * sched_clock_cpu() should mitigate serious artifacts we cannot rely on it
- * in general since for !CONFIG_HAVE_UNSTABLE_SCHED_CLOCK we fully rely on
- * sched_clock().
  */
 #include <linux/spinlock.h>
 #include <linux/hardirq.h>
@@ -66,6 +58,8 @@
 #include <linux/percpu.h>
 #include <linux/ktime.h>
 #include <linux/sched.h>
+#include <linux/static_key.h>
+#include <linux/workqueue.h>
 
 /*
  * Scheduler clock - returns current time in nanosec units.
@@ -82,7 +76,37 @@ EXPORT_SYMBOL_GPL(sched_clock);
 __read_mostly int sched_clock_running;
 
 #ifdef CONFIG_HAVE_UNSTABLE_SCHED_CLOCK
-__read_mostly int sched_clock_stable;
+static struct static_key __sched_clock_stable = STATIC_KEY_INIT;
+
+int sched_clock_stable(void)
+{
+       if (static_key_false(&__sched_clock_stable))
+               return false;
+       return true;
+}
+
+void set_sched_clock_stable(void)
+{
+       if (!sched_clock_stable())
+               static_key_slow_dec(&__sched_clock_stable);
+}
+
+static void __clear_sched_clock_stable(struct work_struct *work)
+{
+       /* XXX worry about clock continuity */
+       if (sched_clock_stable())
+               static_key_slow_inc(&__sched_clock_stable);
+}
+
+static DECLARE_WORK(sched_clock_work, __clear_sched_clock_stable);
+
+void clear_sched_clock_stable(void)
+{
+       if (keventd_up())
+               schedule_work(&sched_clock_work);
+       else
+               __clear_sched_clock_stable(&sched_clock_work);
+}
 
 struct sched_clock_data {
        u64                     tick_raw;
@@ -242,20 +266,20 @@ u64 sched_clock_cpu(int cpu)
        struct sched_clock_data *scd;
        u64 clock;
 
-       WARN_ON_ONCE(!irqs_disabled());
-
-       if (sched_clock_stable)
+       if (sched_clock_stable())
                return sched_clock();
 
        if (unlikely(!sched_clock_running))
                return 0ull;
 
+       preempt_disable();
        scd = cpu_sdc(cpu);
 
        if (cpu != smp_processor_id())
                clock = sched_clock_remote(scd);
        else
                clock = sched_clock_local(scd);
+       preempt_enable();
 
        return clock;
 }
@@ -265,7 +289,7 @@ void sched_clock_tick(void)
        struct sched_clock_data *scd;
        u64 now, now_gtod;
 
-       if (sched_clock_stable)
+       if (sched_clock_stable())
                return;
 
        if (unlikely(!sched_clock_running))
@@ -316,14 +340,10 @@ EXPORT_SYMBOL_GPL(sched_clock_idle_wakeup_event);
  */
 u64 cpu_clock(int cpu)
 {
-       u64 clock;
-       unsigned long flags;
-
-       local_irq_save(flags);
-       clock = sched_clock_cpu(cpu);
-       local_irq_restore(flags);
+       if (static_key_false(&__sched_clock_stable))
+               return sched_clock_cpu(cpu);
 
-       return clock;
+       return sched_clock();
 }
 
 /*
@@ -335,14 +355,10 @@ u64 cpu_clock(int cpu)
  */
 u64 local_clock(void)
 {
-       u64 clock;
-       unsigned long flags;
+       if (static_key_false(&__sched_clock_stable))
+               return sched_clock_cpu(raw_smp_processor_id());
 
-       local_irq_save(flags);
-       clock = sched_clock_cpu(smp_processor_id());
-       local_irq_restore(flags);
-
-       return clock;
+       return sched_clock();
 }
 
 #else /* CONFIG_HAVE_UNSTABLE_SCHED_CLOCK */
@@ -362,12 +378,12 @@ u64 sched_clock_cpu(int cpu)
 
 u64 cpu_clock(int cpu)
 {
-       return sched_clock_cpu(cpu);
+       return sched_clock();
 }
 
 u64 local_clock(void)
 {
-       return sched_clock_cpu(0);
+       return sched_clock();
 }
 
 #endif /* CONFIG_HAVE_UNSTABLE_SCHED_CLOCK */