arm64, jump label: optimize jump label implementation

Optimize jump label implementation for ARM64 by dynamically patching
kernel text.

Reviewed-by: Will Deacon <will.deacon@arm.com>
Signed-off-by: Jiang Liu <liuj97@gmail.com>
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
(cherry picked from commit 9732cafd9dc0206479be919baf0067239f0a63ca)
Signed-off-by: Mark Brown <broonie@linaro.org>
Conflicts:
	arch/arm64/kernel/Makefile

diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig
index 56b3f6d447ae10b8d53f9cfb767826fe90a97c11..8a3aa8f7729fa941fb2a316704a01e303ecc26c0 100644 (file)
--- a/arch/arm64/Kconfig
+++ b/arch/arm64/Kconfig
@@ -16,6 +16,7 @@ config ARM64
        select GENERIC_SMP_IDLE_THREAD
        select GENERIC_TIME_VSYSCALL
        select HARDIRQS_SW_RESEND
+       select HAVE_ARCH_JUMP_LABEL
        select HAVE_ARCH_TRACEHOOK
        select HAVE_DEBUG_BUGVERBOSE
        select HAVE_DEBUG_KMEMLEAK

diff --git a/arch/arm64/include/asm/jump_label.h b/arch/arm64/include/asm/jump_label.h
new file mode 100644 (file)
index 0000000..076a1c7
--- /dev/null
+++ b/arch/arm64/include/asm/jump_label.h
@@ -0,0 +1,52 @@
+/*
+ * Copyright (C) 2013 Huawei Ltd.
+ * Author: Jiang Liu <liuj97@gmail.com>
+ *
+ * Based on arch/arm/include/asm/jump_label.h
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+#ifndef __ASM_JUMP_LABEL_H
+#define __ASM_JUMP_LABEL_H
+#include <linux/types.h>
+#include <asm/insn.h>
+
+#ifdef __KERNEL__
+
+#define JUMP_LABEL_NOP_SIZE            AARCH64_INSN_SIZE
+
+static __always_inline bool arch_static_branch(struct static_key *key)
+{
+       asm goto("1: nop\n\t"
+                ".pushsection __jump_table,  \"aw\"\n\t"
+                ".align 3\n\t"
+                ".quad 1b, %l[l_yes], %c0\n\t"
+                ".popsection\n\t"
+                :  :  "i"(key) :  : l_yes);
+
+       return false;
+l_yes:
+       return true;
+}
+
+#endif /* __KERNEL__ */
+
+typedef u64 jump_label_t;
+
+struct jump_entry {
+       jump_label_t code;
+       jump_label_t target;
+       jump_label_t key;
+};
+
+#endif /* __ASM_JUMP_LABEL_H */

diff --git a/arch/arm64/kernel/Makefile b/arch/arm64/kernel/Makefile
index 9af6cb322d7466091713b240c82d28bd77f2e6b7..b7db65ea2971e400f123f1f91bf028aa88d41c58 100644 (file)
--- a/arch/arm64/kernel/Makefile
+++ b/arch/arm64/kernel/Makefile
@@ -18,6 +18,7 @@ arm64-obj-$(CONFIG_SMP)                       += smp.o smp_spin_table.o smp_psci.o
 arm64-obj-$(CONFIG_HW_PERF_EVENTS)     += perf_event.o
 arm64-obj-$(CONFIG_HAVE_HW_BREAKPOINT)+= hw_breakpoint.o
 arm64-obj-$(CONFIG_EARLY_PRINTK)       += early_printk.o
+arm64-obj-$(CONFIG_JUMP_LABEL)         += jump_label.o
 
 obj-y                                  += $(arm64-obj-y) vdso/
 obj-m                                  += $(arm64-obj-m)

diff --git a/arch/arm64/kernel/jump_label.c b/arch/arm64/kernel/jump_label.c
new file mode 100644 (file)
index 0000000..263a166
--- /dev/null
+++ b/arch/arm64/kernel/jump_label.c
@@ -0,0 +1,58 @@
+/*
+ * Copyright (C) 2013 Huawei Ltd.
+ * Author: Jiang Liu <liuj97@gmail.com>
+ *
+ * Based on arch/arm/kernel/jump_label.c
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+#include <linux/kernel.h>
+#include <linux/jump_label.h>
+#include <asm/insn.h>
+
+#ifdef HAVE_JUMP_LABEL
+
+static void __arch_jump_label_transform(struct jump_entry *entry,
+                                       enum jump_label_type type,
+                                       bool is_static)
+{
+       void *addr = (void *)entry->code;
+       u32 insn;
+
+       if (type == JUMP_LABEL_ENABLE) {
+               insn = aarch64_insn_gen_branch_imm(entry->code,
+                                                  entry->target,
+                                                  AARCH64_INSN_BRANCH_NOLINK);
+       } else {
+               insn = aarch64_insn_gen_nop();
+       }
+
+       if (is_static)
+               aarch64_insn_patch_text_nosync(addr, insn);
+       else
+               aarch64_insn_patch_text(&addr, &insn, 1);
+}
+
+void arch_jump_label_transform(struct jump_entry *entry,
+                              enum jump_label_type type)
+{
+       __arch_jump_label_transform(entry, type, false);
+}
+
+void arch_jump_label_transform_static(struct jump_entry *entry,
+                                     enum jump_label_type type)
+{
+       __arch_jump_label_transform(entry, type, true);
+}
+
+#endif /* HAVE_JUMP_LABEL */