arch/x86/crypto/poly1305_glue.c

   1 /*
   2  * Poly1305 authenticator algorithm, RFC7539, SIMD glue code
   3  *
   4  * Copyright (C) 2015 Martin Willi
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License as published by
   8  * the Free Software Foundation; either version 2 of the License, or
   9  * (at your option) any later version.
  10  */
  11
  12 #include <crypto/algapi.h>
  13 #include <crypto/internal/hash.h>
  14 #include <crypto/poly1305.h>
  15 #include <linux/crypto.h>
  16 #include <linux/kernel.h>
  17 #include <linux/module.h>
  18 #include <asm/fpu/api.h>
  19 #include <asm/simd.h>
  20
  21 asmlinkage void poly1305_block_sse2(u32 *h, const u8 *src,
  22                                     const u32 *r, unsigned int blocks);
  23
  24 static unsigned int poly1305_simd_blocks(struct poly1305_desc_ctx *dctx,
  25                                          const u8 *src, unsigned int srclen)
  26 {
  27         unsigned int blocks, datalen;
  28
  29         if (unlikely(!dctx->sset)) {
  30                 datalen = crypto_poly1305_setdesckey(dctx, src, srclen);
  31                 src += srclen - datalen;
  32                 srclen = datalen;
  33         }
  34
  35         if (srclen >= POLY1305_BLOCK_SIZE) {
  36                 blocks = srclen / POLY1305_BLOCK_SIZE;
  37                 poly1305_block_sse2(dctx->h, src, dctx->r, blocks);
  38                 srclen -= POLY1305_BLOCK_SIZE * blocks;
  39         }
  40         return srclen;
  41 }
  42
  43 static int poly1305_simd_update(struct shash_desc *desc,
  44                                 const u8 *src, unsigned int srclen)
  45 {
  46         struct poly1305_desc_ctx *dctx = shash_desc_ctx(desc);
  47         unsigned int bytes;
  48
  49         /* kernel_fpu_begin/end is costly, use fallback for small updates */
  50         if (srclen <= 288 || !may_use_simd())
  51                 return crypto_poly1305_update(desc, src, srclen);
  52
  53         kernel_fpu_begin();
  54
  55         if (unlikely(dctx->buflen)) {
  56                 bytes = min(srclen, POLY1305_BLOCK_SIZE - dctx->buflen);
  57                 memcpy(dctx->buf + dctx->buflen, src, bytes);
  58                 src += bytes;
  59                 srclen -= bytes;
  60                 dctx->buflen += bytes;
  61
  62                 if (dctx->buflen == POLY1305_BLOCK_SIZE) {
  63                         poly1305_simd_blocks(dctx, dctx->buf,
  64                                              POLY1305_BLOCK_SIZE);
  65                         dctx->buflen = 0;
  66                 }
  67         }
  68
  69         if (likely(srclen >= POLY1305_BLOCK_SIZE)) {
  70                 bytes = poly1305_simd_blocks(dctx, src, srclen);
  71                 src += srclen - bytes;
  72                 srclen = bytes;
  73         }
  74
  75         kernel_fpu_end();
  76
  77         if (unlikely(srclen)) {
  78                 dctx->buflen = srclen;
  79                 memcpy(dctx->buf, src, srclen);
  80         }
  81
  82         return 0;
  83 }
  84
  85 static struct shash_alg alg = {
  86         .digestsize     = POLY1305_DIGEST_SIZE,
  87         .init           = crypto_poly1305_init,
  88         .update         = poly1305_simd_update,
  89         .final          = crypto_poly1305_final,
  90         .setkey         = crypto_poly1305_setkey,
  91         .descsize       = sizeof(struct poly1305_desc_ctx),
  92         .base           = {
  93                 .cra_name               = "poly1305",
  94                 .cra_driver_name        = "poly1305-simd",
  95                 .cra_priority           = 300,
  96                 .cra_flags              = CRYPTO_ALG_TYPE_SHASH,
  97                 .cra_alignmask          = sizeof(u32) - 1,
  98                 .cra_blocksize          = POLY1305_BLOCK_SIZE,
  99                 .cra_module             = THIS_MODULE,
 100         },
 101 };
 102
 103 static int __init poly1305_simd_mod_init(void)
 104 {
 105         if (!cpu_has_xmm2)
 106                 return -ENODEV;
 107
 108         return crypto_register_shash(&alg);
 109 }
 110
 111 static void __exit poly1305_simd_mod_exit(void)
 112 {
 113         crypto_unregister_shash(&alg);
 114 }
 115
 116 module_init(poly1305_simd_mod_init);
 117 module_exit(poly1305_simd_mod_exit);
 118
 119 MODULE_LICENSE("GPL");
 120 MODULE_AUTHOR("Martin Willi <martin@strongswan.org>");
 121 MODULE_DESCRIPTION("Poly1305 authenticator");
 122 MODULE_ALIAS_CRYPTO("poly1305");
 123 MODULE_ALIAS_CRYPTO("poly1305-simd");