Merge remote-tracking branch 'lts/linux-4.4.y' into linux-linaro-lsk-v4.4

[firefly-linux-kernel-4.4.55.git] / arch / arm64 / kernel / entry.S

diff --git a/arch/arm64/kernel/entry.S b/arch/arm64/kernel/entry.S
index 5a3753d09e20d607ab9419cec2d27e3aef50b8b0..588c8e1778d44a6cde02ef8ebb4b6db33652b809 100644 (file)
--- a/arch/arm64/kernel/entry.S
+++ b/arch/arm64/kernel/entry.S
@@ -27,6 +27,7 @@
 #include <asm/cpufeature.h>
 #include <asm/errno.h>
 #include <asm/esr.h>
+#include <asm/irq.h>
 #include <asm/memory.h>
 #include <asm/thread_info.h>
 #include <asm/unistd.h>
@@ -89,9 +90,12 @@
 
        .if     \el == 0
        mrs     x21, sp_el0
-       get_thread_info tsk                     // Ensure MDSCR_EL1.SS is clear,
+       mov     tsk, sp
+       and     tsk, tsk, #~(THREAD_SIZE - 1)   // Ensure MDSCR_EL1.SS is clear,
        ldr     x19, [tsk, #TI_FLAGS]           // since we can unmask debug
        disable_step_tsk x19, x20               // exceptions when scheduling.
+
+       mov     x29, xzr                        // fp pointed to user-space
        .else
        add     x21, sp, #S_FRAME_SIZE
        get_thread_info tsk
@@ -114,6 +118,13 @@
        str     x21, [sp, #S_SYSCALLNO]
        .endif
 
+       /*
+        * Set sp_el0 to current thread_info.
+        */
+       .if     \el == 0
+       msr     sp_el0, tsk
+       .endif
+
        /*
         * Registers that may be useful after this macro is invoked:
         *
@@ -177,8 +188,44 @@ alternative_endif
        .endm
 
        .macro  get_thread_info, rd
-       mov     \rd, sp
-       and     \rd, \rd, #~(THREAD_SIZE - 1)   // top of stack
+       mrs     \rd, sp_el0
+       .endm
+
+       .macro  irq_stack_entry
+       mov     x19, sp                 // preserve the original sp
+
+       /*
+        * Compare sp with the current thread_info, if the top
+        * ~(THREAD_SIZE - 1) bits match, we are on a task stack, and
+        * should switch to the irq stack.
+        */
+       and     x25, x19, #~(THREAD_SIZE - 1)
+       cmp     x25, tsk
+       b.ne    9998f
+
+       this_cpu_ptr irq_stack, x25, x26
+       mov     x26, #IRQ_STACK_START_SP
+       add     x26, x25, x26
+
+       /* switch to the irq stack */
+       mov     sp, x26
+
+       /*
+        * Add a dummy stack frame, this non-standard format is fixed up
+        * by unwind_frame()
+        */
+       stp     x29, x19, [sp, #-16]!
+       mov     x29, sp
+
+9998:
+       .endm
+
+       /*
+        * x19 should be preserved between irq_stack_entry and
+        * irq_stack_exit.
+        */
+       .macro  irq_stack_exit
+       mov     sp, x19
        .endm
 
 /*
@@ -196,10 +243,11 @@ tsk       .req    x28             // current thread_info
  * Interrupt handling.
  */
        .macro  irq_handler
-       adrp    x1, handle_arch_irq
-       ldr     x1, [x1, #:lo12:handle_arch_irq]
+       ldr_l   x1, handle_arch_irq
        mov     x0, sp
+       irq_stack_entry
        blr     x1
+       irq_stack_exit
        .endm
 
        .text
@@ -371,10 +419,10 @@ el1_irq:
        bl      trace_hardirqs_off
 #endif
 
+       get_thread_info tsk
        irq_handler
 
 #ifdef CONFIG_PREEMPT
-       get_thread_info tsk
        ldr     w24, [tsk, #TI_PREEMPT]         // get preempt count
        cbnz    w24, 1f                         // preempt count != 0
        ldr     x0, [tsk, #TI_FLAGS]            // get flags
@@ -612,6 +660,8 @@ ENTRY(cpu_switch_to)
        ldp     x29, x9, [x8], #16
        ldr     lr, [x8]
        mov     sp, x9
+       and     x9, x9, #~(THREAD_SIZE - 1)
+       msr     sp_el0, x9
        ret
 ENDPROC(cpu_switch_to)
 
@@ -639,14 +689,14 @@ ret_fast_syscall_trace:
 work_pending:
        tbnz    x1, #TIF_NEED_RESCHED, work_resched
        /* TIF_SIGPENDING, TIF_NOTIFY_RESUME or TIF_FOREIGN_FPSTATE case */
-       ldr     x2, [sp, #S_PSTATE]
        mov     x0, sp                          // 'regs'
-       tst     x2, #PSR_MODE_MASK              // user mode regs?
-       b.ne    no_work_pending                 // returning to kernel
        enable_irq                              // enable interrupts for do_notify_resume()
        bl      do_notify_resume
        b       ret_to_user
 work_resched:
+#ifdef CONFIG_TRACE_IRQFLAGS
+       bl      trace_hardirqs_off              // the IRQs are off here, inform the tracing code
+#endif
        bl      schedule
 
 /*
@@ -658,7 +708,6 @@ ret_to_user:
        and     x2, x1, #_TIF_WORK_MASK
        cbnz    x2, work_pending
        enable_step_tsk x1, x2
-no_work_pending:
        kernel_exit 0
 ENDPROC(ret_to_user)