arm64/crypto: AES using ARMv8 Crypto Extensions
authorArd Biesheuvel <ard.biesheuvel@linaro.org>
Wed, 5 Feb 2014 17:13:38 +0000 (18:13 +0100)
committerArd Biesheuvel <ard.biesheuvel@linaro.org>
Wed, 14 May 2014 17:04:11 +0000 (10:04 -0700)
This patch adds support for the AES symmetric encryption algorithm for CPUs
that have support for the AES part of the ARM v8 Crypto Extensions.

Signed-off-by: Ard Biesheuvel <ard.biesheuvel@linaro.org>
Acked-by: Herbert Xu <herbert@gondor.apana.org.au>
arch/arm64/crypto/Kconfig
arch/arm64/crypto/Makefile
arch/arm64/crypto/aes-ce-cipher.c [new file with mode: 0644]

index 0c50859ee7b9b059e17cd18381cc29079dfecbaa..9ba32c0da871ee765be66cbb6d7e6fa21b0d2d21 100644 (file)
@@ -18,10 +18,15 @@ config CRYPTO_SHA2_ARM64_CE
        depends on ARM64 && KERNEL_MODE_NEON
        select CRYPTO_HASH
 
-
 config CRYPTO_GHASH_ARM64_CE
        tristate "GHASH (for GCM chaining mode) using ARMv8 Crypto Extensions"
        depends on ARM64 && KERNEL_MODE_NEON
        select CRYPTO_HASH
 
+config CRYPTO_AES_ARM64_CE
+       tristate "AES core cipher using ARMv8 Crypto Extensions"
+       depends on ARM64 && KERNEL_MODE_NEON
+       select CRYPTO_ALGAPI
+       select CRYPTO_AES
+
 endif
index e8c81a068868ea4448272f1900c418702ecf7a62..908abd9242b17890829c06ed73800d2101509677 100644 (file)
@@ -16,3 +16,6 @@ sha2-ce-y := sha2-ce-glue.o sha2-ce-core.o
 
 obj-$(CONFIG_CRYPTO_GHASH_ARM64_CE) += ghash-ce.o
 ghash-ce-y := ghash-ce-glue.o ghash-ce-core.o
+
+obj-$(CONFIG_CRYPTO_AES_ARM64_CE) += aes-ce-cipher.o
+CFLAGS_aes-ce-cipher.o += -march=armv8-a+crypto
diff --git a/arch/arm64/crypto/aes-ce-cipher.c b/arch/arm64/crypto/aes-ce-cipher.c
new file mode 100644 (file)
index 0000000..2075e1a
--- /dev/null
@@ -0,0 +1,155 @@
+/*
+ * aes-ce-cipher.c - core AES cipher using ARMv8 Crypto Extensions
+ *
+ * Copyright (C) 2013 - 2014 Linaro Ltd <ard.biesheuvel@linaro.org>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#include <asm/neon.h>
+#include <crypto/aes.h>
+#include <linux/cpufeature.h>
+#include <linux/crypto.h>
+#include <linux/module.h>
+
+MODULE_DESCRIPTION("Synchronous AES cipher using ARMv8 Crypto Extensions");
+MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
+MODULE_LICENSE("GPL v2");
+
+struct aes_block {
+       u8 b[AES_BLOCK_SIZE];
+};
+
+static int num_rounds(struct crypto_aes_ctx *ctx)
+{
+       /*
+        * # of rounds specified by AES:
+        * 128 bit key          10 rounds
+        * 192 bit key          12 rounds
+        * 256 bit key          14 rounds
+        * => n byte key        => 6 + (n/4) rounds
+        */
+       return 6 + ctx->key_length / 4;
+}
+
+static void aes_cipher_encrypt(struct crypto_tfm *tfm, u8 dst[], u8 const src[])
+{
+       struct crypto_aes_ctx *ctx = crypto_tfm_ctx(tfm);
+       struct aes_block *out = (struct aes_block *)dst;
+       struct aes_block const *in = (struct aes_block *)src;
+       void *dummy0;
+       int dummy1;
+
+       kernel_neon_begin_partial(4);
+
+       __asm__("       ld1     {v0.16b}, %[in]                 ;"
+               "       ld1     {v1.2d}, [%[key]], #16          ;"
+               "       cmp     %w[rounds], #10                 ;"
+               "       bmi     0f                              ;"
+               "       bne     3f                              ;"
+               "       mov     v3.16b, v1.16b                  ;"
+               "       b       2f                              ;"
+               "0:     mov     v2.16b, v1.16b                  ;"
+               "       ld1     {v3.2d}, [%[key]], #16          ;"
+               "1:     aese    v0.16b, v2.16b                  ;"
+               "       aesmc   v0.16b, v0.16b                  ;"
+               "2:     ld1     {v1.2d}, [%[key]], #16          ;"
+               "       aese    v0.16b, v3.16b                  ;"
+               "       aesmc   v0.16b, v0.16b                  ;"
+               "3:     ld1     {v2.2d}, [%[key]], #16          ;"
+               "       subs    %w[rounds], %w[rounds], #3      ;"
+               "       aese    v0.16b, v1.16b                  ;"
+               "       aesmc   v0.16b, v0.16b                  ;"
+               "       ld1     {v3.2d}, [%[key]], #16          ;"
+               "       bpl     1b                              ;"
+               "       aese    v0.16b, v2.16b                  ;"
+               "       eor     v0.16b, v0.16b, v3.16b          ;"
+               "       st1     {v0.16b}, %[out]                ;"
+
+       :       [out]           "=Q"(*out),
+               [key]           "=r"(dummy0),
+               [rounds]        "=r"(dummy1)
+       :       [in]            "Q"(*in),
+                               "1"(ctx->key_enc),
+                               "2"(num_rounds(ctx) - 2)
+       :       "cc");
+
+       kernel_neon_end();
+}
+
+static void aes_cipher_decrypt(struct crypto_tfm *tfm, u8 dst[], u8 const src[])
+{
+       struct crypto_aes_ctx *ctx = crypto_tfm_ctx(tfm);
+       struct aes_block *out = (struct aes_block *)dst;
+       struct aes_block const *in = (struct aes_block *)src;
+       void *dummy0;
+       int dummy1;
+
+       kernel_neon_begin_partial(4);
+
+       __asm__("       ld1     {v0.16b}, %[in]                 ;"
+               "       ld1     {v1.2d}, [%[key]], #16          ;"
+               "       cmp     %w[rounds], #10                 ;"
+               "       bmi     0f                              ;"
+               "       bne     3f                              ;"
+               "       mov     v3.16b, v1.16b                  ;"
+               "       b       2f                              ;"
+               "0:     mov     v2.16b, v1.16b                  ;"
+               "       ld1     {v3.2d}, [%[key]], #16          ;"
+               "1:     aesd    v0.16b, v2.16b                  ;"
+               "       aesimc  v0.16b, v0.16b                  ;"
+               "2:     ld1     {v1.2d}, [%[key]], #16          ;"
+               "       aesd    v0.16b, v3.16b                  ;"
+               "       aesimc  v0.16b, v0.16b                  ;"
+               "3:     ld1     {v2.2d}, [%[key]], #16          ;"
+               "       subs    %w[rounds], %w[rounds], #3      ;"
+               "       aesd    v0.16b, v1.16b                  ;"
+               "       aesimc  v0.16b, v0.16b                  ;"
+               "       ld1     {v3.2d}, [%[key]], #16          ;"
+               "       bpl     1b                              ;"
+               "       aesd    v0.16b, v2.16b                  ;"
+               "       eor     v0.16b, v0.16b, v3.16b          ;"
+               "       st1     {v0.16b}, %[out]                ;"
+
+       :       [out]           "=Q"(*out),
+               [key]           "=r"(dummy0),
+               [rounds]        "=r"(dummy1)
+       :       [in]            "Q"(*in),
+                               "1"(ctx->key_dec),
+                               "2"(num_rounds(ctx) - 2)
+       :       "cc");
+
+       kernel_neon_end();
+}
+
+static struct crypto_alg aes_alg = {
+       .cra_name               = "aes",
+       .cra_driver_name        = "aes-ce",
+       .cra_priority           = 300,
+       .cra_flags              = CRYPTO_ALG_TYPE_CIPHER,
+       .cra_blocksize          = AES_BLOCK_SIZE,
+       .cra_ctxsize            = sizeof(struct crypto_aes_ctx),
+       .cra_module             = THIS_MODULE,
+       .cra_cipher = {
+               .cia_min_keysize        = AES_MIN_KEY_SIZE,
+               .cia_max_keysize        = AES_MAX_KEY_SIZE,
+               .cia_setkey             = crypto_aes_set_key,
+               .cia_encrypt            = aes_cipher_encrypt,
+               .cia_decrypt            = aes_cipher_decrypt
+       }
+};
+
+static int __init aes_mod_init(void)
+{
+       return crypto_register_alg(&aes_alg);
+}
+
+static void __exit aes_mod_exit(void)
+{
+       crypto_unregister_alg(&aes_alg);
+}
+
+module_cpu_feature_match(AES, aes_mod_init);
+module_exit(aes_mod_exit);