arm64: add abstractions for FPSIMD state manipulation

There are two tacit assumptions in the FPSIMD handling code that will no longer
hold after the next patch that optimizes away some FPSIMD state restores:
. the FPSIMD registers of this CPU contain the userland FPSIMD state of
  task 'current';
. when switching to a task, its FPSIMD state will always be restored from
  memory.

This patch adds the following functions to abstract away from straight FPSIMD
register file saves and restores:
- fpsimd_preserve_current_state -> ensure current's FPSIMD state is saved
- fpsimd_update_current_state -> replace current's FPSIMD state

Where necessary, the signal handling and fork code are updated to use the above
wrappers instead of poking into the FPSIMD registers directly.

Signed-off-by: Ard Biesheuvel <ard.biesheuvel@linaro.org>
Conflicts:
	arch/arm64/kernel/fpsimd.c

Change-Id: I53ae7082427cb1c5cc32e1f2ddbd4218115601ba

diff --git a/arch/arm64/include/asm/fpsimd.h b/arch/arm64/include/asm/fpsimd.h
index c43b4ac13008ffec8f0b39101e5cfd77e43fd3b5..f4e524b67e912296ef7c539e535987b1f3405322 100644 (file)
--- a/arch/arm64/include/asm/fpsimd.h
+++ b/arch/arm64/include/asm/fpsimd.h
@@ -58,6 +58,9 @@ extern void fpsimd_load_state(struct fpsimd_state *state);
 extern void fpsimd_thread_switch(struct task_struct *next);
 extern void fpsimd_flush_thread(void);
 
+extern void fpsimd_preserve_current_state(void);
+extern void fpsimd_update_current_state(struct fpsimd_state *state);
+
 #endif
 
 #endif

diff --git a/arch/arm64/kernel/fpsimd.c b/arch/arm64/kernel/fpsimd.c
index e8b8357aedb42e1fb23d6c736a7fcd2ea5003429..135cfb88d2aa922e92cb1ad6a52322c6fca3e60f 100644 (file)
--- a/arch/arm64/kernel/fpsimd.c
+++ b/arch/arm64/kernel/fpsimd.c
@@ -83,6 +83,86 @@ void fpsimd_flush_thread(void)
        fpsimd_load_state(&current->thread.fpsimd_state);
 }
 
+/*
+ * Save the userland FPSIMD state of 'current' to memory
+ */
+void fpsimd_preserve_current_state(void)
+{
+       preempt_disable();
+       fpsimd_save_state(&current->thread.fpsimd_state);
+       preempt_enable();
+}
+
+/*
+ * Load an updated userland FPSIMD state for 'current' from memory
+ */
+void fpsimd_update_current_state(struct fpsimd_state *state)
+{
+       preempt_disable();
+       fpsimd_load_state(state);
+       preempt_enable();
+}
+
+#ifdef CONFIG_KERNEL_MODE_NEON
+
+/*
+ * Kernel-side NEON support functions
+ */
+void kernel_neon_begin(void)
+{
+       /* Avoid using the NEON in interrupt context */
+       BUG_ON(in_interrupt());
+       preempt_disable();
+
+       if (current->mm)
+               fpsimd_save_state(&current->thread.fpsimd_state);
+}
+EXPORT_SYMBOL(kernel_neon_begin);
+
+void kernel_neon_end(void)
+{
+       if (current->mm)
+               fpsimd_load_state(&current->thread.fpsimd_state);
+
+       preempt_enable();
+}
+EXPORT_SYMBOL(kernel_neon_end);
+
+#endif /* CONFIG_KERNEL_MODE_NEON */
+
+#ifdef CONFIG_CPU_PM
+static int fpsimd_cpu_pm_notifier(struct notifier_block *self,
+                                 unsigned long cmd, void *v)
+{
+       switch (cmd) {
+       case CPU_PM_ENTER:
+               if (current->mm)
+                       fpsimd_save_state(&current->thread.fpsimd_state);
+               break;
+       case CPU_PM_EXIT:
+               if (current->mm)
+                       fpsimd_load_state(&current->thread.fpsimd_state);
+               break;
+       case CPU_PM_ENTER_FAILED:
+       default:
+               return NOTIFY_DONE;
+       }
+       return NOTIFY_OK;
+}
+
+static struct notifier_block fpsimd_cpu_pm_notifier_block = {
+       .notifier_call = fpsimd_cpu_pm_notifier,
+};
+
+static void fpsimd_pm_init(void)
+{
+       cpu_pm_register_notifier(&fpsimd_cpu_pm_notifier_block);
+}
+
+#else
+static inline void fpsimd_pm_init(void) { }
+#endif /* CONFIG_CPU_PM */
+
 /*
  * FP/SIMD support code initialisation.
  */

diff --git a/arch/arm64/kernel/process.c b/arch/arm64/kernel/process.c
index 46f02c3b5015ece9b1ca0ab4d57d9540a3288e67..e2eb9453d3a1caaa8ea4f294ff826409bde57357 100644 (file)
--- a/arch/arm64/kernel/process.c
+++ b/arch/arm64/kernel/process.c
@@ -184,7 +184,7 @@ void release_thread(struct task_struct *dead_task)
 
 int arch_dup_task_struct(struct task_struct *dst, struct task_struct *src)
 {
-       fpsimd_save_state(&current->thread.fpsimd_state);
+       fpsimd_preserve_current_state();
        *dst = *src;
        return 0;
 }

diff --git a/arch/arm64/kernel/signal.c b/arch/arm64/kernel/signal.c
index 890a591f75dd1848e28e1ea62413775be5394c6d..06448a77ff5351a18ad98409311675144453462a 100644 (file)
--- a/arch/arm64/kernel/signal.c
+++ b/arch/arm64/kernel/signal.c
@@ -51,7 +51,7 @@ static int preserve_fpsimd_context(struct fpsimd_context __user *ctx)
        int err;
 
        /* dump the hardware registers to the fpsimd_state structure */
-       fpsimd_save_state(fpsimd);
+       fpsimd_preserve_current_state();
 
        /* copy the FP and status/control registers */
        err = __copy_to_user(ctx->vregs, fpsimd->vregs, sizeof(fpsimd->vregs));
@@ -86,11 +86,8 @@ static int restore_fpsimd_context(struct fpsimd_context __user *ctx)
        __get_user_error(fpsimd.fpcr, &ctx->fpcr, err);
 
        /* load the hardware registers from the fpsimd_state structure */
-       if (!err) {
-               preempt_disable();
-               fpsimd_load_state(&fpsimd);
-               preempt_enable();
-       }
+       if (!err)
+               fpsimd_update_current_state(&fpsimd);
 
        return err ? -EFAULT : 0;
 }

diff --git a/arch/arm64/kernel/signal32.c b/arch/arm64/kernel/signal32.c
index e393174fe859721e7f6538bc377577cb69bb68aa..31fbeaf4dd62aa0d08730a243248e7c2b1213a15 100644 (file)
--- a/arch/arm64/kernel/signal32.c
+++ b/arch/arm64/kernel/signal32.c
@@ -247,7 +247,7 @@ static int compat_preserve_vfp_context(struct compat_vfp_sigframe __user *frame)
         * Note that this also saves V16-31, which aren't visible
         * in AArch32.
         */
-       fpsimd_save_state(fpsimd);
+       fpsimd_preserve_current_state();
 
        /* Place structure header on the stack */
        __put_user_error(magic, &frame->magic, err);
@@ -310,11 +310,8 @@ static int compat_restore_vfp_context(struct compat_vfp_sigframe __user *frame)
         * We don't need to touch the exception register, so
         * reload the hardware state.
         */
-       if (!err) {
-               preempt_disable();
-               fpsimd_load_state(&fpsimd);
-               preempt_enable();
-       }
+       if (!err)
+               fpsimd_update_current_state(&fpsimd);
 
        return err ? -EFAULT : 0;
 }