From 3c1a3bcea945f9d59ab1fe3d319c67c0ff56100f Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Fri, 13 Mar 2015 13:13:36 +0100 Subject: [PATCH] s390/maccess: improve s390_kernel_write() Use the sturg instruction instead of the stura instruction. This allows to modify up to eight bytes in a row instead of only four. For function tracer enabling and disabling this reduces the time needed to modify the text sections by 50%, since for each mcount call site six bytes need to be changed. Also remove the EXTABLE entries, since calls to this function are not supposed to fail. Signed-off-by: Heiko Carstens Signed-off-by: Martin Schwidefsky --- arch/s390/mm/maccess.c | 45 +++++++++++++++++++----------------------- 1 file changed, 20 insertions(+), 25 deletions(-) diff --git a/arch/s390/mm/maccess.c b/arch/s390/mm/maccess.c index fb737e9e0683..8a993a53fcd6 100644 --- a/arch/s390/mm/maccess.c +++ b/arch/s390/mm/maccess.c @@ -1,7 +1,7 @@ /* * Access kernel memory without faulting -- s390 specific implementation. * - * Copyright IBM Corp. 2009 + * Copyright IBM Corp. 2009, 2015 * * Author(s): Heiko Carstens , * @@ -18,28 +18,25 @@ static notrace long s390_kernel_write_odd(void *dst, const void *src, size_t size) { - unsigned long count, aligned; - int offset, mask; - int rc = -EFAULT; + unsigned long aligned, offset, count; + char tmp[8]; - aligned = (unsigned long) dst & ~3UL; - offset = (unsigned long) dst & 3; - count = min_t(unsigned long, 4 - offset, size); - mask = (0xf << (4 - count)) & 0xf; - mask >>= offset; + aligned = (unsigned long) dst & ~7UL; + offset = (unsigned long) dst & 7UL; + size = min(8UL - offset, size); + count = size - 1; asm volatile( " bras 1,0f\n" - " icm 0,0,0(%3)\n" - "0: l 0,0(%1)\n" - " lra %1,0(%1)\n" - "1: ex %2,0(1)\n" - "2: stura 0,%1\n" - " la %0,0\n" - "3:\n" - EX_TABLE(0b,3b) EX_TABLE(1b,3b) EX_TABLE(2b,3b) - : "+d" (rc), "+a" (aligned) - : "a" (mask), "a" (src) : "cc", "memory", "0", "1"); - return rc ? rc : count; + " mvc 0(1,%4),0(%5)\n" + "0: mvc 0(8,%3),0(%0)\n" + " ex %1,0(1)\n" + " lg %1,0(%3)\n" + " lra %0,0(%0)\n" + " sturg %1,%0\n" + : "+&a" (aligned), "+&a" (count), "=m" (tmp) + : "a" (&tmp), "a" (&tmp[offset]), "a" (src) + : "cc", "memory", "1"); + return size; } /* @@ -50,8 +47,8 @@ static notrace long s390_kernel_write_odd(void *dst, const void *src, size_t siz * * This function writes to kernel memory bypassing DAT and possible page table * write protection. It writes to the destination using the sturg instruction. - * Therefore we have a read-modify-write sequence: the function reads four - * bytes from destination at a four byte boundary, modifies the bytes + * Therefore we have a read-modify-write sequence: the function reads eight + * bytes from destination at an eight byte boundary, modifies the bytes * requested and writes the result back in a loop. * * Note: this means that this function may not be called concurrently on @@ -60,12 +57,10 @@ static notrace long s390_kernel_write_odd(void *dst, const void *src, size_t siz */ void notrace s390_kernel_write(void *dst, const void *src, size_t size) { - long copied = 0; + long copied; while (size) { copied = s390_kernel_write_odd(dst, src, size); - if (copied < 0) - break; dst += copied; src += copied; size -= copied; -- 2.34.1