arm64/crypto: AES in CCM mode using ARMv8 Crypto Extensions
authorArd Biesheuvel <ard.biesheuvel@linaro.org>
Mon, 10 Feb 2014 10:26:29 +0000 (11:26 +0100)
committerArd Biesheuvel <ard.biesheuvel@linaro.org>
Wed, 14 May 2014 17:04:15 +0000 (10:04 -0700)
This patch adds support for the AES-CCM encryption algorithm for CPUs that
have support for the AES part of the ARM v8 Crypto Extensions.

Signed-off-by: Ard Biesheuvel <ard.biesheuvel@linaro.org>
Acked-by: Herbert Xu <herbert@gondor.apana.org.au>
arch/arm64/crypto/Kconfig
arch/arm64/crypto/Makefile
arch/arm64/crypto/aes-ce-ccm-core.S [new file with mode: 0644]
arch/arm64/crypto/aes-ce-ccm-glue.c [new file with mode: 0644]

index 9ba32c0da871ee765be66cbb6d7e6fa21b0d2d21..8fffd5af65efac93c71790446e807fb80ed479c0 100644 (file)
@@ -29,4 +29,11 @@ config CRYPTO_AES_ARM64_CE
        select CRYPTO_ALGAPI
        select CRYPTO_AES
 
+config CRYPTO_AES_ARM64_CE_CCM
+       tristate "AES in CCM mode using ARMv8 Crypto Extensions"
+       depends on ARM64 && KERNEL_MODE_NEON
+       select CRYPTO_ALGAPI
+       select CRYPTO_AES
+       select CRYPTO_AEAD
+
 endif
index 908abd9242b17890829c06ed73800d2101509677..311287d680781b570978e06441dd8239703ce0dc 100644 (file)
@@ -19,3 +19,6 @@ ghash-ce-y := ghash-ce-glue.o ghash-ce-core.o
 
 obj-$(CONFIG_CRYPTO_AES_ARM64_CE) += aes-ce-cipher.o
 CFLAGS_aes-ce-cipher.o += -march=armv8-a+crypto
+
+obj-$(CONFIG_CRYPTO_AES_ARM64_CE_CCM) += aes-ce-ccm.o
+aes-ce-ccm-y := aes-ce-ccm-glue.o aes-ce-ccm-core.o
diff --git a/arch/arm64/crypto/aes-ce-ccm-core.S b/arch/arm64/crypto/aes-ce-ccm-core.S
new file mode 100644 (file)
index 0000000..432e484
--- /dev/null
@@ -0,0 +1,222 @@
+/*
+ * aesce-ccm-core.S - AES-CCM transform for ARMv8 with Crypto Extensions
+ *
+ * Copyright (C) 2013 - 2014 Linaro Ltd <ard.biesheuvel@linaro.org>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#include <linux/linkage.h>
+
+       .text
+       .arch   armv8-a+crypto
+
+       /*
+        * void ce_aes_ccm_auth_data(u8 mac[], u8 const in[], u32 abytes,
+        *                           u32 *macp, u8 const rk[], u32 rounds);
+        */
+ENTRY(ce_aes_ccm_auth_data)
+       ldr     w8, [x3]                        /* leftover from prev round? */
+       ld1     {v0.2d}, [x0]                   /* load mac */
+       cbz     w8, 1f
+       sub     w8, w8, #16
+       eor     v1.16b, v1.16b, v1.16b
+0:     ldrb    w7, [x1], #1                    /* get 1 byte of input */
+       subs    w2, w2, #1
+       add     w8, w8, #1
+       ins     v1.b[0], w7
+       ext     v1.16b, v1.16b, v1.16b, #1      /* rotate in the input bytes */
+       beq     8f                              /* out of input? */
+       cbnz    w8, 0b
+       eor     v0.16b, v0.16b, v1.16b
+1:     ld1     {v3.2d}, [x4]                   /* load first round key */
+       prfm    pldl1strm, [x1]
+       cmp     w5, #12                         /* which key size? */
+       add     x6, x4, #16
+       sub     w7, w5, #2                      /* modified # of rounds */
+       bmi     2f
+       bne     5f
+       mov     v5.16b, v3.16b
+       b       4f
+2:     mov     v4.16b, v3.16b
+       ld1     {v5.2d}, [x6], #16              /* load 2nd round key */
+3:     aese    v0.16b, v4.16b
+       aesmc   v0.16b, v0.16b
+4:     ld1     {v3.2d}, [x6], #16              /* load next round key */
+       aese    v0.16b, v5.16b
+       aesmc   v0.16b, v0.16b
+5:     ld1     {v4.2d}, [x6], #16              /* load next round key */
+       subs    w7, w7, #3
+       aese    v0.16b, v3.16b
+       aesmc   v0.16b, v0.16b
+       ld1     {v5.2d}, [x6], #16              /* load next round key */
+       bpl     3b
+       aese    v0.16b, v4.16b
+       subs    w2, w2, #16                     /* last data? */
+       eor     v0.16b, v0.16b, v5.16b          /* final round */
+       bmi     6f
+       ld1     {v1.16b}, [x1], #16             /* load next input block */
+       eor     v0.16b, v0.16b, v1.16b          /* xor with mac */
+       bne     1b
+6:     st1     {v0.2d}, [x0]                   /* store mac */
+       beq     10f
+       adds    w2, w2, #16
+       beq     10f
+       mov     w8, w2
+7:     ldrb    w7, [x1], #1
+       umov    w6, v0.b[0]
+       eor     w6, w6, w7
+       strb    w6, [x0], #1
+       subs    w2, w2, #1
+       beq     10f
+       ext     v0.16b, v0.16b, v0.16b, #1      /* rotate out the mac bytes */
+       b       7b
+8:     mov     w7, w8
+       add     w8, w8, #16
+9:     ext     v1.16b, v1.16b, v1.16b, #1
+       adds    w7, w7, #1
+       bne     9b
+       eor     v0.16b, v0.16b, v1.16b
+       st1     {v0.2d}, [x0]
+10:    str     w8, [x3]
+       ret
+ENDPROC(ce_aes_ccm_auth_data)
+
+       /*
+        * void ce_aes_ccm_final(u8 mac[], u8 const ctr[], u8 const rk[],
+        *                       u32 rounds);
+        */
+ENTRY(ce_aes_ccm_final)
+       ld1     {v3.2d}, [x2], #16              /* load first round key */
+       ld1     {v0.2d}, [x0]                   /* load mac */
+       cmp     w3, #12                         /* which key size? */
+       sub     w3, w3, #2                      /* modified # of rounds */
+       ld1     {v1.2d}, [x1]                   /* load 1st ctriv */
+       bmi     0f
+       bne     3f
+       mov     v5.16b, v3.16b
+       b       2f
+0:     mov     v4.16b, v3.16b
+1:     ld1     {v5.2d}, [x2], #16              /* load next round key */
+       aese    v0.16b, v4.16b
+       aese    v1.16b, v4.16b
+       aesmc   v0.16b, v0.16b
+       aesmc   v1.16b, v1.16b
+2:     ld1     {v3.2d}, [x2], #16              /* load next round key */
+       aese    v0.16b, v5.16b
+       aese    v1.16b, v5.16b
+       aesmc   v0.16b, v0.16b
+       aesmc   v1.16b, v1.16b
+3:     ld1     {v4.2d}, [x2], #16              /* load next round key */
+       subs    w3, w3, #3
+       aese    v0.16b, v3.16b
+       aese    v1.16b, v3.16b
+       aesmc   v0.16b, v0.16b
+       aesmc   v1.16b, v1.16b
+       bpl     1b
+       aese    v0.16b, v4.16b
+       aese    v1.16b, v4.16b
+       /* final round key cancels out */
+       eor     v0.16b, v0.16b, v1.16b          /* en-/decrypt the mac */
+       st1     {v0.2d}, [x0]                   /* store result */
+       ret
+ENDPROC(ce_aes_ccm_final)
+
+       .macro  aes_ccm_do_crypt,enc
+       ldr     x8, [x6, #8]                    /* load lower ctr */
+       ld1     {v0.2d}, [x5]                   /* load mac */
+       rev     x8, x8                          /* keep swabbed ctr in reg */
+0:     /* outer loop */
+       ld1     {v1.1d}, [x6]                   /* load upper ctr */
+       prfm    pldl1strm, [x1]
+       add     x8, x8, #1
+       rev     x9, x8
+       cmp     w4, #12                         /* which key size? */
+       sub     w7, w4, #2                      /* get modified # of rounds */
+       ins     v1.d[1], x9                     /* no carry in lower ctr */
+       ld1     {v3.2d}, [x3]                   /* load first round key */
+       add     x10, x3, #16
+       bmi     1f
+       bne     4f
+       mov     v5.16b, v3.16b
+       b       3f
+1:     mov     v4.16b, v3.16b
+       ld1     {v5.2d}, [x10], #16             /* load 2nd round key */
+2:     /* inner loop: 3 rounds, 2x interleaved */
+       aese    v0.16b, v4.16b
+       aese    v1.16b, v4.16b
+       aesmc   v0.16b, v0.16b
+       aesmc   v1.16b, v1.16b
+3:     ld1     {v3.2d}, [x10], #16             /* load next round key */
+       aese    v0.16b, v5.16b
+       aese    v1.16b, v5.16b
+       aesmc   v0.16b, v0.16b
+       aesmc   v1.16b, v1.16b
+4:     ld1     {v4.2d}, [x10], #16             /* load next round key */
+       subs    w7, w7, #3
+       aese    v0.16b, v3.16b
+       aese    v1.16b, v3.16b
+       aesmc   v0.16b, v0.16b
+       aesmc   v1.16b, v1.16b
+       ld1     {v5.2d}, [x10], #16             /* load next round key */
+       bpl     2b
+       aese    v0.16b, v4.16b
+       aese    v1.16b, v4.16b
+       subs    w2, w2, #16
+       bmi     6f                              /* partial block? */
+       ld1     {v2.16b}, [x1], #16             /* load next input block */
+       .if     \enc == 1
+       eor     v2.16b, v2.16b, v5.16b          /* final round enc+mac */
+       eor     v1.16b, v1.16b, v2.16b          /* xor with crypted ctr */
+       .else
+       eor     v2.16b, v2.16b, v1.16b          /* xor with crypted ctr */
+       eor     v1.16b, v2.16b, v5.16b          /* final round enc */
+       .endif
+       eor     v0.16b, v0.16b, v2.16b          /* xor mac with pt ^ rk[last] */
+       st1     {v1.16b}, [x0], #16             /* write output block */
+       bne     0b
+       rev     x8, x8
+       st1     {v0.2d}, [x5]                   /* store mac */
+       str     x8, [x6, #8]                    /* store lsb end of ctr (BE) */
+5:     ret
+
+6:     eor     v0.16b, v0.16b, v5.16b          /* final round mac */
+       eor     v1.16b, v1.16b, v5.16b          /* final round enc */
+       st1     {v0.2d}, [x5]                   /* store mac */
+       add     w2, w2, #16                     /* process partial tail block */
+7:     ldrb    w9, [x1], #1                    /* get 1 byte of input */
+       umov    w6, v1.b[0]                     /* get top crypted ctr byte */
+       umov    w7, v0.b[0]                     /* get top mac byte */
+       .if     \enc == 1
+       eor     w7, w7, w9
+       eor     w9, w9, w6
+       .else
+       eor     w9, w9, w6
+       eor     w7, w7, w9
+       .endif
+       strb    w9, [x0], #1                    /* store out byte */
+       strb    w7, [x5], #1                    /* store mac byte */
+       subs    w2, w2, #1
+       beq     5b
+       ext     v0.16b, v0.16b, v0.16b, #1      /* shift out mac byte */
+       ext     v1.16b, v1.16b, v1.16b, #1      /* shift out ctr byte */
+       b       7b
+       .endm
+
+       /*
+        * void ce_aes_ccm_encrypt(u8 out[], u8 const in[], u32 cbytes,
+        *                         u8 const rk[], u32 rounds, u8 mac[],
+        *                         u8 ctr[]);
+        * void ce_aes_ccm_decrypt(u8 out[], u8 const in[], u32 cbytes,
+        *                         u8 const rk[], u32 rounds, u8 mac[],
+        *                         u8 ctr[]);
+        */
+ENTRY(ce_aes_ccm_encrypt)
+       aes_ccm_do_crypt        1
+ENDPROC(ce_aes_ccm_encrypt)
+
+ENTRY(ce_aes_ccm_decrypt)
+       aes_ccm_do_crypt        0
+ENDPROC(ce_aes_ccm_decrypt)
diff --git a/arch/arm64/crypto/aes-ce-ccm-glue.c b/arch/arm64/crypto/aes-ce-ccm-glue.c
new file mode 100644 (file)
index 0000000..9e6cdde
--- /dev/null
@@ -0,0 +1,297 @@
+/*
+ * aes-ccm-glue.c - AES-CCM transform for ARMv8 with Crypto Extensions
+ *
+ * Copyright (C) 2013 - 2014 Linaro Ltd <ard.biesheuvel@linaro.org>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#include <asm/neon.h>
+#include <asm/unaligned.h>
+#include <crypto/aes.h>
+#include <crypto/algapi.h>
+#include <crypto/scatterwalk.h>
+#include <linux/crypto.h>
+#include <linux/module.h>
+
+static int num_rounds(struct crypto_aes_ctx *ctx)
+{
+       /*
+        * # of rounds specified by AES:
+        * 128 bit key          10 rounds
+        * 192 bit key          12 rounds
+        * 256 bit key          14 rounds
+        * => n byte key        => 6 + (n/4) rounds
+        */
+       return 6 + ctx->key_length / 4;
+}
+
+asmlinkage void ce_aes_ccm_auth_data(u8 mac[], u8 const in[], u32 abytes,
+                                    u32 *macp, u32 const rk[], u32 rounds);
+
+asmlinkage void ce_aes_ccm_encrypt(u8 out[], u8 const in[], u32 cbytes,
+                                  u32 const rk[], u32 rounds, u8 mac[],
+                                  u8 ctr[]);
+
+asmlinkage void ce_aes_ccm_decrypt(u8 out[], u8 const in[], u32 cbytes,
+                                  u32 const rk[], u32 rounds, u8 mac[],
+                                  u8 ctr[]);
+
+asmlinkage void ce_aes_ccm_final(u8 mac[], u8 const ctr[], u32 const rk[],
+                                u32 rounds);
+
+static int ccm_setkey(struct crypto_aead *tfm, const u8 *in_key,
+                     unsigned int key_len)
+{
+       struct crypto_aes_ctx *ctx = crypto_aead_ctx(tfm);
+       int ret;
+
+       ret = crypto_aes_expand_key(ctx, in_key, key_len);
+       if (!ret)
+               return 0;
+
+       tfm->base.crt_flags |= CRYPTO_TFM_RES_BAD_KEY_LEN;
+       return -EINVAL;
+}
+
+static int ccm_setauthsize(struct crypto_aead *tfm, unsigned int authsize)
+{
+       if ((authsize & 1) || authsize < 4)
+               return -EINVAL;
+       return 0;
+}
+
+static int ccm_init_mac(struct aead_request *req, u8 maciv[], u32 msglen)
+{
+       struct crypto_aead *aead = crypto_aead_reqtfm(req);
+       __be32 *n = (__be32 *)&maciv[AES_BLOCK_SIZE - 8];
+       u32 l = req->iv[0] + 1;
+
+       /* verify that CCM dimension 'L' is set correctly in the IV */
+       if (l < 2 || l > 8)
+               return -EINVAL;
+
+       /* verify that msglen can in fact be represented in L bytes */
+       if (l < 4 && msglen >> (8 * l))
+               return -EOVERFLOW;
+
+       /*
+        * Even if the CCM spec allows L values of up to 8, the Linux cryptoapi
+        * uses a u32 type to represent msglen so the top 4 bytes are always 0.
+        */
+       n[0] = 0;
+       n[1] = cpu_to_be32(msglen);
+
+       memcpy(maciv, req->iv, AES_BLOCK_SIZE - l);
+
+       /*
+        * Meaning of byte 0 according to CCM spec (RFC 3610/NIST 800-38C)
+        * - bits 0..2  : max # of bytes required to represent msglen, minus 1
+        *                (already set by caller)
+        * - bits 3..5  : size of auth tag (1 => 4 bytes, 2 => 6 bytes, etc)
+        * - bit 6      : indicates presence of authenticate-only data
+        */
+       maciv[0] |= (crypto_aead_authsize(aead) - 2) << 2;
+       if (req->assoclen)
+               maciv[0] |= 0x40;
+
+       memset(&req->iv[AES_BLOCK_SIZE - l], 0, l);
+       return 0;
+}
+
+static void ccm_calculate_auth_mac(struct aead_request *req, u8 mac[])
+{
+       struct crypto_aead *aead = crypto_aead_reqtfm(req);
+       struct crypto_aes_ctx *ctx = crypto_aead_ctx(aead);
+       struct __packed { __be16 l; __be32 h; u16 len; } ltag;
+       struct scatter_walk walk;
+       u32 len = req->assoclen;
+       u32 macp = 0;
+
+       /* prepend the AAD with a length tag */
+       if (len < 0xff00) {
+               ltag.l = cpu_to_be16(len);
+               ltag.len = 2;
+       } else  {
+               ltag.l = cpu_to_be16(0xfffe);
+               put_unaligned_be32(len, &ltag.h);
+               ltag.len = 6;
+       }
+
+       ce_aes_ccm_auth_data(mac, (u8 *)&ltag, ltag.len, &macp, ctx->key_enc,
+                            num_rounds(ctx));
+       scatterwalk_start(&walk, req->assoc);
+
+       do {
+               u32 n = scatterwalk_clamp(&walk, len);
+               u8 *p;
+
+               if (!n) {
+                       scatterwalk_start(&walk, sg_next(walk.sg));
+                       n = scatterwalk_clamp(&walk, len);
+               }
+               p = scatterwalk_map(&walk);
+               ce_aes_ccm_auth_data(mac, p, n, &macp, ctx->key_enc,
+                                    num_rounds(ctx));
+               len -= n;
+
+               scatterwalk_unmap(p);
+               scatterwalk_advance(&walk, n);
+               scatterwalk_done(&walk, 0, len);
+       } while (len);
+}
+
+static int ccm_encrypt(struct aead_request *req)
+{
+       struct crypto_aead *aead = crypto_aead_reqtfm(req);
+       struct crypto_aes_ctx *ctx = crypto_aead_ctx(aead);
+       struct blkcipher_desc desc = { .info = req->iv };
+       struct blkcipher_walk walk;
+       u8 __aligned(8) mac[AES_BLOCK_SIZE];
+       u8 buf[AES_BLOCK_SIZE];
+       u32 len = req->cryptlen;
+       int err;
+
+       err = ccm_init_mac(req, mac, len);
+       if (err)
+               return err;
+
+       kernel_neon_begin_partial(6);
+
+       if (req->assoclen)
+               ccm_calculate_auth_mac(req, mac);
+
+       /* preserve the original iv for the final round */
+       memcpy(buf, req->iv, AES_BLOCK_SIZE);
+
+       blkcipher_walk_init(&walk, req->dst, req->src, len);
+       err = blkcipher_aead_walk_virt_block(&desc, &walk, aead,
+                                            AES_BLOCK_SIZE);
+
+       while (walk.nbytes) {
+               u32 tail = walk.nbytes % AES_BLOCK_SIZE;
+
+               if (walk.nbytes == len)
+                       tail = 0;
+
+               ce_aes_ccm_encrypt(walk.dst.virt.addr, walk.src.virt.addr,
+                                  walk.nbytes - tail, ctx->key_enc,
+                                  num_rounds(ctx), mac, walk.iv);
+
+               len -= walk.nbytes - tail;
+               err = blkcipher_walk_done(&desc, &walk, tail);
+       }
+       if (!err)
+               ce_aes_ccm_final(mac, buf, ctx->key_enc, num_rounds(ctx));
+
+       kernel_neon_end();
+
+       if (err)
+               return err;
+
+       /* copy authtag to end of dst */
+       scatterwalk_map_and_copy(mac, req->dst, req->cryptlen,
+                                crypto_aead_authsize(aead), 1);
+
+       return 0;
+}
+
+static int ccm_decrypt(struct aead_request *req)
+{
+       struct crypto_aead *aead = crypto_aead_reqtfm(req);
+       struct crypto_aes_ctx *ctx = crypto_aead_ctx(aead);
+       unsigned int authsize = crypto_aead_authsize(aead);
+       struct blkcipher_desc desc = { .info = req->iv };
+       struct blkcipher_walk walk;
+       u8 __aligned(8) mac[AES_BLOCK_SIZE];
+       u8 buf[AES_BLOCK_SIZE];
+       u32 len = req->cryptlen - authsize;
+       int err;
+
+       err = ccm_init_mac(req, mac, len);
+       if (err)
+               return err;
+
+       kernel_neon_begin_partial(6);
+
+       if (req->assoclen)
+               ccm_calculate_auth_mac(req, mac);
+
+       /* preserve the original iv for the final round */
+       memcpy(buf, req->iv, AES_BLOCK_SIZE);
+
+       blkcipher_walk_init(&walk, req->dst, req->src, len);
+       err = blkcipher_aead_walk_virt_block(&desc, &walk, aead,
+                                            AES_BLOCK_SIZE);
+
+       while (walk.nbytes) {
+               u32 tail = walk.nbytes % AES_BLOCK_SIZE;
+
+               if (walk.nbytes == len)
+                       tail = 0;
+
+               ce_aes_ccm_decrypt(walk.dst.virt.addr, walk.src.virt.addr,
+                                  walk.nbytes - tail, ctx->key_enc,
+                                  num_rounds(ctx), mac, walk.iv);
+
+               len -= walk.nbytes - tail;
+               err = blkcipher_walk_done(&desc, &walk, tail);
+       }
+       if (!err)
+               ce_aes_ccm_final(mac, buf, ctx->key_enc, num_rounds(ctx));
+
+       kernel_neon_end();
+
+       if (err)
+               return err;
+
+       /* compare calculated auth tag with the stored one */
+       scatterwalk_map_and_copy(buf, req->src, req->cryptlen - authsize,
+                                authsize, 0);
+
+       if (memcmp(mac, buf, authsize))
+               return -EBADMSG;
+       return 0;
+}
+
+static struct crypto_alg ccm_aes_alg = {
+       .cra_name               = "ccm(aes)",
+       .cra_driver_name        = "ccm-aes-ce",
+       .cra_priority           = 300,
+       .cra_flags              = CRYPTO_ALG_TYPE_AEAD,
+       .cra_blocksize          = 1,
+       .cra_ctxsize            = sizeof(struct crypto_aes_ctx),
+       .cra_alignmask          = 7,
+       .cra_type               = &crypto_aead_type,
+       .cra_module             = THIS_MODULE,
+       .cra_aead = {
+               .ivsize         = AES_BLOCK_SIZE,
+               .maxauthsize    = AES_BLOCK_SIZE,
+               .setkey         = ccm_setkey,
+               .setauthsize    = ccm_setauthsize,
+               .encrypt        = ccm_encrypt,
+               .decrypt        = ccm_decrypt,
+       }
+};
+
+static int __init aes_mod_init(void)
+{
+       if (!(elf_hwcap & HWCAP_AES))
+               return -ENODEV;
+       return crypto_register_alg(&ccm_aes_alg);
+}
+
+static void __exit aes_mod_exit(void)
+{
+       crypto_unregister_alg(&ccm_aes_alg);
+}
+
+module_init(aes_mod_init);
+module_exit(aes_mod_exit);
+
+MODULE_DESCRIPTION("Synchronous AES in CCM mode using ARMv8 Crypto Extensions");
+MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
+MODULE_LICENSE("GPL v2");
+MODULE_ALIAS("ccm(aes)");