Merge branches 'pm-cpu', 'pm-cpuidle' and 'pm-domains'

[firefly-linux-kernel-4.4.55.git] / arch / x86 / lib / delay.c

diff --git a/arch/x86/lib/delay.c b/arch/x86/lib/delay.c
index f24bc59ab0a0ac8261769ac49d6875025d1db480..e912b2f6d36e5392b33bf6749f19a53cff6d7a06 100644 (file)
--- a/arch/x86/lib/delay.c
+++ b/arch/x86/lib/delay.c
@@ -20,6 +20,7 @@
 #include <asm/processor.h>
 #include <asm/delay.h>
 #include <asm/timer.h>
+#include <asm/mwait.h>
 
 #ifdef CONFIG_SMP
 # include <asm/smp.h>
@@ -54,11 +55,9 @@ static void delay_tsc(unsigned long __loops)
 
        preempt_disable();
        cpu = smp_processor_id();
-       rdtsc_barrier();
-       bclock = rdtsc();
+       bclock = rdtsc_ordered();
        for (;;) {
-               rdtsc_barrier();
-               now = rdtsc();
+               now = rdtsc_ordered();
                if ((now - bclock) >= loops)
                        break;
 
@@ -79,13 +78,50 @@ static void delay_tsc(unsigned long __loops)
                if (unlikely(cpu != smp_processor_id())) {
                        loops -= (now - bclock);
                        cpu = smp_processor_id();
-                       rdtsc_barrier();
-                       bclock = rdtsc();
+                       bclock = rdtsc_ordered();
                }
        }
        preempt_enable();
 }
 
+/*
+ * On some AMD platforms, MWAITX has a configurable 32-bit timer, that
+ * counts with TSC frequency. The input value is the loop of the
+ * counter, it will exit when the timer expires.
+ */
+static void delay_mwaitx(unsigned long __loops)
+{
+       u64 start, end, delay, loops = __loops;
+
+       start = rdtsc_ordered();
+
+       for (;;) {
+               delay = min_t(u64, MWAITX_MAX_LOOPS, loops);
+
+               /*
+                * Use cpu_tss as a cacheline-aligned, seldomly
+                * accessed per-cpu variable as the monitor target.
+                */
+               __monitorx(this_cpu_ptr(&cpu_tss), 0, 0);
+
+               /*
+                * AMD, like Intel, supports the EAX hint and EAX=0xf
+                * means, do not enter any deep C-state and we use it
+                * here in delay() to minimize wakeup latency.
+                */
+               __mwaitx(MWAITX_DISABLE_CSTATES, delay, MWAITX_ECX_TIMER_ENABLE);
+
+               end = rdtsc_ordered();
+
+               if (loops <= end - start)
+                       break;
+
+               loops -= end - start;
+
+               start = end;
+       }
+}
+
 /*
  * Since we calibrate only once at boot, this
  * function should be set once at boot and not changed
@@ -94,7 +130,13 @@ static void (*delay_fn)(unsigned long) = delay_loop;
 
 void use_tsc_delay(void)
 {
-       delay_fn = delay_tsc;
+       if (delay_fn == delay_loop)
+               delay_fn = delay_tsc;
+}
+
+void use_mwaitx_delay(void)
+{
+       delay_fn = delay_mwaitx;
 }
 
 int read_current_timer(unsigned long *timer_val)