BACKPORT: arm64: vmlinux.ld: Add mmuoff data sections and move mmuoff text into idmap

Resume from hibernate needs to clean any text executed by the kernel with
the MMU off to the PoC. Collect these functions together into the
.idmap.text section as all this code is tightly coupled and also needs
the same cleaning after resume.

Data is more complicated, secondary_holding_pen_release is written with
the MMU on, clean and invalidated, then read with the MMU off. In contrast
__boot_cpu_mode is written with the MMU off, the corresponding cache line
is invalidated, so when we read it with the MMU on we don't get stale data.
These cache maintenance operations conflict with each other if the values
are within a Cache Writeback Granule (CWG) of each other.
Collect the data into two sections .mmuoff.data.read and .mmuoff.data.write,
the linker script ensures mmuoff.data.write section is aligned to the
architectural maximum CWG of 2KB.

Signed-off-by: James Morse <james.morse@arm.com>
Cc: Ard Biesheuvel <ard.biesheuvel@linaro.org>
Cc: Mark Rutland <mark.rutland@arm.com>
Reviewed-by: Catalin Marinas <catalin.marinas@arm.com>
Signed-off-by: Will Deacon <will.deacon@arm.com>
(cherry picked from commit b61130381120398876b86282082ad9f24976dfcf)
Signed-off-by: Amit Pundir <amit.pundir@linaro.org>

diff --git a/arch/arm64/include/asm/sections.h b/arch/arm64/include/asm/sections.h
index 237fcdd134456ab2674c27c75448028e6c2e4e66..4e7e7067afdb048d464d9287f42f16ebc26aeb7b 100644 (file)
--- a/arch/arm64/include/asm/sections.h
+++ b/arch/arm64/include/asm/sections.h
@@ -25,5 +25,6 @@ extern char __hyp_idmap_text_start[], __hyp_idmap_text_end[];
 extern char __hyp_text_start[], __hyp_text_end[];
 extern char __idmap_text_start[], __idmap_text_end[];
 extern char __irqentry_text_start[], __irqentry_text_end[];
+extern char __mmuoff_data_start[], __mmuoff_data_end[];
 
 #endif /* __ASM_SECTIONS_H */

diff --git a/arch/arm64/kernel/head.S b/arch/arm64/kernel/head.S
index 8cfd5ab377434b2ccafd5b8d7c4701386faf8a95..6d9d36a42967435f32b47fccd63a37fc4ac3a75b 100644 (file)
--- a/arch/arm64/kernel/head.S
+++ b/arch/arm64/kernel/head.S
@@ -463,7 +463,7 @@ ENDPROC(__primary_switched)
  * end early head section, begin head code that is also used for
  * hotplug and needs to have the same protections as the text region
  */
-       .section ".text","ax"
+       .section ".idmap.text","ax"
 
 ENTRY(kimage_vaddr)
        .quad           _text - TEXT_OFFSET
@@ -584,6 +584,13 @@ set_cpu_boot_mode_flag:
        ret
 ENDPROC(set_cpu_boot_mode_flag)
 
+/*
+ * These values are written with the MMU off, but read with the MMU on.
+ * Writers will invalidate the corresponding address, discarding up to a
+ * 'Cache Writeback Granule' (CWG) worth of data. The linker script ensures
+ * sufficient alignment that the CWG doesn't overlap another section.
+ */
+       .pushsection ".mmuoff.data.write", "aw"
 /*
  * We need to find out the CPU boot mode long after boot, so we need to
  * store it in a writable variable.
@@ -591,11 +598,16 @@ ENDPROC(set_cpu_boot_mode_flag)
  * This is not in .bss, because we set it sufficiently early that the boot-time
  * zeroing of .bss would clobber it.
  */
-       .pushsection    .data..cacheline_aligned
-       .align  L1_CACHE_SHIFT
 ENTRY(__boot_cpu_mode)
        .long   BOOT_CPU_MODE_EL2
        .long   BOOT_CPU_MODE_EL1
+/*
+ * The booting CPU updates the failed status @__early_cpu_boot_status,
+ * with MMU turned off.
+ */
+ENTRY(__early_cpu_boot_status)
+       .long   0
+
        .popsection
 
        /*
@@ -662,7 +674,6 @@ ENDPROC(__secondary_switched)
  * Checks if the selected granule size is supported by the CPU.
  * If it isn't, park the CPU
  */
-       .section        ".idmap.text", "ax"
 ENTRY(__enable_mmu)
        mrs     x18, sctlr_el1                  // preserve old SCTLR_EL1 value
        mrs     x1, ID_AA64MMFR0_EL1

diff --git a/arch/arm64/kernel/sleep.S b/arch/arm64/kernel/sleep.S
index c2bf5a58039f1b7f749912bc18f58bf5007e80fb..6f9ba222955c5c14c911a6d9c36ce84dad99ce20 100644 (file)
--- a/arch/arm64/kernel/sleep.S
+++ b/arch/arm64/kernel/sleep.S
@@ -97,6 +97,7 @@ ENTRY(__cpu_suspend_enter)
 ENDPROC(__cpu_suspend_enter)
        .ltorg
 
+       .pushsection ".idmap.text", "ax"
 ENTRY(cpu_resume)
        bl      el2_setup               // if in EL2 drop to EL1 cleanly
        /* enable the MMU early - so we can access sleep_save_stash by va */

diff --git a/arch/arm64/kernel/smp_spin_table.c b/arch/arm64/kernel/smp_spin_table.c
index aef3605a8c47845ce2ce2d32f75e0fd59a8e4405..4b4b0ad6ebde615154d1c221a84234dcbdfd070b 100644 (file)
--- a/arch/arm64/kernel/smp_spin_table.c
+++ b/arch/arm64/kernel/smp_spin_table.c
@@ -29,7 +29,8 @@
 #include <asm/smp_plat.h>
 
 extern void secondary_holding_pen(void);
-volatile unsigned long secondary_holding_pen_release = INVALID_HWID;
+volatile unsigned long __section(".mmuoff.data.read")
+secondary_holding_pen_release = INVALID_HWID;
 
 static phys_addr_t cpu_release_addr[NR_CPUS];
 

diff --git a/arch/arm64/kernel/vmlinux.lds.S b/arch/arm64/kernel/vmlinux.lds.S
index 7a5228c7abdd7cc9c8226df887a2ec9e7b4d053d..f2d34d4c0a6b9dcaf7573ed7d31af6536ca25c4f 100644 (file)
--- a/arch/arm64/kernel/vmlinux.lds.S
+++ b/arch/arm64/kernel/vmlinux.lds.S
@@ -183,6 +183,25 @@ SECTIONS
        _data = .;
        _sdata = .;
        RW_DATA_SECTION(L1_CACHE_BYTES, PAGE_SIZE, THREAD_SIZE)
+
+       /*
+        * Data written with the MMU off but read with the MMU on requires
+        * cache lines to be invalidated, discarding up to a Cache Writeback
+        * Granule (CWG) of data from the cache. Keep the section that
+        * requires this type of maintenance to be in its own Cache Writeback
+        * Granule (CWG) area so the cache maintenance operations don't
+        * interfere with adjacent data.
+        */
+       .mmuoff.data.write : ALIGN(SZ_2K) {
+               __mmuoff_data_start = .;
+               *(.mmuoff.data.write)
+       }
+       . = ALIGN(SZ_2K);
+       .mmuoff.data.read : {
+               *(.mmuoff.data.read)
+               __mmuoff_data_end = .;
+       }
+
        PECOFF_EDATA_PADDING
        _edata = .;
 

diff --git a/arch/arm64/mm/proc.S b/arch/arm64/mm/proc.S
index d88a2a80ada88c081915b6d40269c6e3a0914b45..c2adb0cb952a75ad9ef03db67acde19318b5ba1b 100644 (file)
--- a/arch/arm64/mm/proc.S
+++ b/arch/arm64/mm/proc.S
@@ -83,6 +83,7 @@ ENDPROC(cpu_do_suspend)
  *
  * x0: Address of context pointer
  */
+       .pushsection ".idmap.text", "ax"
 ENTRY(cpu_do_resume)
        ldp     x2, x3, [x0]
        ldp     x4, x5, [x0, #16]
@@ -120,6 +121,7 @@ ENTRY(cpu_do_resume)
        isb
        ret
 ENDPROC(cpu_do_resume)
+       .popsection
 #endif
 
 /*
@@ -172,6 +174,7 @@ ENDPROC(idmap_cpu_replace_ttbr1)
  *     Initialise the processor for turning the MMU on.  Return in x0 the
  *     value of the SCTLR_EL1 register.
  */
+       .pushsection ".idmap.text", "ax"
 ENTRY(__cpu_setup)
        tlbi    vmalle1                         // Invalidate local TLB
        dsb     nsh
@@ -257,3 +260,4 @@ ENDPROC(__cpu_setup)
 crval:
        .word   0xfcffffff                      // clear
        .word   0x34d5d91d                      // set
+       .popsection