2 * Glue Code for the AVX assembler implemention of the Cast6 Cipher
4 * Copyright (C) 2012 Johannes Goetzfried
5 * <Johannes.Goetzfried@informatik.stud.uni-erlangen.de>
7 * Copyright © 2013 Jussi Kivilinna <jussi.kivilinna@iki.fi>
9 * This program is free software; you can redistribute it and/or modify
10 * it under the terms of the GNU General Public License as published by
11 * the Free Software Foundation; either version 2 of the License, or
12 * (at your option) any later version.
14 * This program is distributed in the hope that it will be useful,
15 * but WITHOUT ANY WARRANTY; without even the implied warranty of
16 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 * GNU General Public License for more details.
19 * You should have received a copy of the GNU General Public License
20 * along with this program; if not, write to the Free Software
21 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
26 #include <linux/module.h>
27 #include <linux/hardirq.h>
28 #include <linux/types.h>
29 #include <linux/crypto.h>
30 #include <linux/err.h>
31 #include <crypto/ablk_helper.h>
32 #include <crypto/algapi.h>
33 #include <crypto/cast6.h>
34 #include <crypto/cryptd.h>
35 #include <crypto/b128ops.h>
36 #include <crypto/ctr.h>
37 #include <crypto/lrw.h>
38 #include <crypto/xts.h>
39 #include <asm/fpu/api.h>
40 #include <asm/crypto/glue_helper.h>
42 #define CAST6_PARALLEL_BLOCKS 8
44 asmlinkage void cast6_ecb_enc_8way(struct cast6_ctx *ctx, u8 *dst,
46 asmlinkage void cast6_ecb_dec_8way(struct cast6_ctx *ctx, u8 *dst,
49 asmlinkage void cast6_cbc_dec_8way(struct cast6_ctx *ctx, u8 *dst,
51 asmlinkage void cast6_ctr_8way(struct cast6_ctx *ctx, u8 *dst, const u8 *src,
54 asmlinkage void cast6_xts_enc_8way(struct cast6_ctx *ctx, u8 *dst,
55 const u8 *src, le128 *iv);
56 asmlinkage void cast6_xts_dec_8way(struct cast6_ctx *ctx, u8 *dst,
57 const u8 *src, le128 *iv);
59 static void cast6_xts_enc(void *ctx, u128 *dst, const u128 *src, le128 *iv)
61 glue_xts_crypt_128bit_one(ctx, dst, src, iv,
62 GLUE_FUNC_CAST(__cast6_encrypt));
65 static void cast6_xts_dec(void *ctx, u128 *dst, const u128 *src, le128 *iv)
67 glue_xts_crypt_128bit_one(ctx, dst, src, iv,
68 GLUE_FUNC_CAST(__cast6_decrypt));
71 static void cast6_crypt_ctr(void *ctx, u128 *dst, const u128 *src, le128 *iv)
75 le128_to_be128(&ctrblk, iv);
78 __cast6_encrypt(ctx, (u8 *)&ctrblk, (u8 *)&ctrblk);
79 u128_xor(dst, src, (u128 *)&ctrblk);
82 static const struct common_glue_ctx cast6_enc = {
84 .fpu_blocks_limit = CAST6_PARALLEL_BLOCKS,
87 .num_blocks = CAST6_PARALLEL_BLOCKS,
88 .fn_u = { .ecb = GLUE_FUNC_CAST(cast6_ecb_enc_8way) }
91 .fn_u = { .ecb = GLUE_FUNC_CAST(__cast6_encrypt) }
95 static const struct common_glue_ctx cast6_ctr = {
97 .fpu_blocks_limit = CAST6_PARALLEL_BLOCKS,
100 .num_blocks = CAST6_PARALLEL_BLOCKS,
101 .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(cast6_ctr_8way) }
104 .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(cast6_crypt_ctr) }
108 static const struct common_glue_ctx cast6_enc_xts = {
110 .fpu_blocks_limit = CAST6_PARALLEL_BLOCKS,
113 .num_blocks = CAST6_PARALLEL_BLOCKS,
114 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(cast6_xts_enc_8way) }
117 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(cast6_xts_enc) }
121 static const struct common_glue_ctx cast6_dec = {
123 .fpu_blocks_limit = CAST6_PARALLEL_BLOCKS,
126 .num_blocks = CAST6_PARALLEL_BLOCKS,
127 .fn_u = { .ecb = GLUE_FUNC_CAST(cast6_ecb_dec_8way) }
130 .fn_u = { .ecb = GLUE_FUNC_CAST(__cast6_decrypt) }
134 static const struct common_glue_ctx cast6_dec_cbc = {
136 .fpu_blocks_limit = CAST6_PARALLEL_BLOCKS,
139 .num_blocks = CAST6_PARALLEL_BLOCKS,
140 .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(cast6_cbc_dec_8way) }
143 .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(__cast6_decrypt) }
147 static const struct common_glue_ctx cast6_dec_xts = {
149 .fpu_blocks_limit = CAST6_PARALLEL_BLOCKS,
152 .num_blocks = CAST6_PARALLEL_BLOCKS,
153 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(cast6_xts_dec_8way) }
156 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(cast6_xts_dec) }
160 static int ecb_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
161 struct scatterlist *src, unsigned int nbytes)
163 return glue_ecb_crypt_128bit(&cast6_enc, desc, dst, src, nbytes);
166 static int ecb_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
167 struct scatterlist *src, unsigned int nbytes)
169 return glue_ecb_crypt_128bit(&cast6_dec, desc, dst, src, nbytes);
172 static int cbc_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
173 struct scatterlist *src, unsigned int nbytes)
175 return glue_cbc_encrypt_128bit(GLUE_FUNC_CAST(__cast6_encrypt), desc,
179 static int cbc_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
180 struct scatterlist *src, unsigned int nbytes)
182 return glue_cbc_decrypt_128bit(&cast6_dec_cbc, desc, dst, src,
186 static int ctr_crypt(struct blkcipher_desc *desc, struct scatterlist *dst,
187 struct scatterlist *src, unsigned int nbytes)
189 return glue_ctr_crypt_128bit(&cast6_ctr, desc, dst, src, nbytes);
192 static inline bool cast6_fpu_begin(bool fpu_enabled, unsigned int nbytes)
194 return glue_fpu_begin(CAST6_BLOCK_SIZE, CAST6_PARALLEL_BLOCKS,
195 NULL, fpu_enabled, nbytes);
198 static inline void cast6_fpu_end(bool fpu_enabled)
200 glue_fpu_end(fpu_enabled);
204 struct cast6_ctx *ctx;
208 static void encrypt_callback(void *priv, u8 *srcdst, unsigned int nbytes)
210 const unsigned int bsize = CAST6_BLOCK_SIZE;
211 struct crypt_priv *ctx = priv;
214 ctx->fpu_enabled = cast6_fpu_begin(ctx->fpu_enabled, nbytes);
216 if (nbytes == bsize * CAST6_PARALLEL_BLOCKS) {
217 cast6_ecb_enc_8way(ctx->ctx, srcdst, srcdst);
221 for (i = 0; i < nbytes / bsize; i++, srcdst += bsize)
222 __cast6_encrypt(ctx->ctx, srcdst, srcdst);
225 static void decrypt_callback(void *priv, u8 *srcdst, unsigned int nbytes)
227 const unsigned int bsize = CAST6_BLOCK_SIZE;
228 struct crypt_priv *ctx = priv;
231 ctx->fpu_enabled = cast6_fpu_begin(ctx->fpu_enabled, nbytes);
233 if (nbytes == bsize * CAST6_PARALLEL_BLOCKS) {
234 cast6_ecb_dec_8way(ctx->ctx, srcdst, srcdst);
238 for (i = 0; i < nbytes / bsize; i++, srcdst += bsize)
239 __cast6_decrypt(ctx->ctx, srcdst, srcdst);
242 struct cast6_lrw_ctx {
243 struct lrw_table_ctx lrw_table;
244 struct cast6_ctx cast6_ctx;
247 static int lrw_cast6_setkey(struct crypto_tfm *tfm, const u8 *key,
250 struct cast6_lrw_ctx *ctx = crypto_tfm_ctx(tfm);
253 err = __cast6_setkey(&ctx->cast6_ctx, key, keylen - CAST6_BLOCK_SIZE,
258 return lrw_init_table(&ctx->lrw_table, key + keylen - CAST6_BLOCK_SIZE);
261 static int lrw_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
262 struct scatterlist *src, unsigned int nbytes)
264 struct cast6_lrw_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
265 be128 buf[CAST6_PARALLEL_BLOCKS];
266 struct crypt_priv crypt_ctx = {
267 .ctx = &ctx->cast6_ctx,
268 .fpu_enabled = false,
270 struct lrw_crypt_req req = {
272 .tbuflen = sizeof(buf),
274 .table_ctx = &ctx->lrw_table,
275 .crypt_ctx = &crypt_ctx,
276 .crypt_fn = encrypt_callback,
280 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
281 ret = lrw_crypt(desc, dst, src, nbytes, &req);
282 cast6_fpu_end(crypt_ctx.fpu_enabled);
287 static int lrw_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
288 struct scatterlist *src, unsigned int nbytes)
290 struct cast6_lrw_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
291 be128 buf[CAST6_PARALLEL_BLOCKS];
292 struct crypt_priv crypt_ctx = {
293 .ctx = &ctx->cast6_ctx,
294 .fpu_enabled = false,
296 struct lrw_crypt_req req = {
298 .tbuflen = sizeof(buf),
300 .table_ctx = &ctx->lrw_table,
301 .crypt_ctx = &crypt_ctx,
302 .crypt_fn = decrypt_callback,
306 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
307 ret = lrw_crypt(desc, dst, src, nbytes, &req);
308 cast6_fpu_end(crypt_ctx.fpu_enabled);
313 static void lrw_exit_tfm(struct crypto_tfm *tfm)
315 struct cast6_lrw_ctx *ctx = crypto_tfm_ctx(tfm);
317 lrw_free_table(&ctx->lrw_table);
320 struct cast6_xts_ctx {
321 struct cast6_ctx tweak_ctx;
322 struct cast6_ctx crypt_ctx;
325 static int xts_cast6_setkey(struct crypto_tfm *tfm, const u8 *key,
328 struct cast6_xts_ctx *ctx = crypto_tfm_ctx(tfm);
329 u32 *flags = &tfm->crt_flags;
332 /* key consists of keys of equal size concatenated, therefore
333 * the length must be even
336 *flags |= CRYPTO_TFM_RES_BAD_KEY_LEN;
340 /* first half of xts-key is for crypt */
341 err = __cast6_setkey(&ctx->crypt_ctx, key, keylen / 2, flags);
345 /* second half of xts-key is for tweak */
346 return __cast6_setkey(&ctx->tweak_ctx, key + keylen / 2, keylen / 2,
350 static int xts_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
351 struct scatterlist *src, unsigned int nbytes)
353 struct cast6_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
355 return glue_xts_crypt_128bit(&cast6_enc_xts, desc, dst, src, nbytes,
356 XTS_TWEAK_CAST(__cast6_encrypt),
357 &ctx->tweak_ctx, &ctx->crypt_ctx);
360 static int xts_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
361 struct scatterlist *src, unsigned int nbytes)
363 struct cast6_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
365 return glue_xts_crypt_128bit(&cast6_dec_xts, desc, dst, src, nbytes,
366 XTS_TWEAK_CAST(__cast6_encrypt),
367 &ctx->tweak_ctx, &ctx->crypt_ctx);
370 static struct crypto_alg cast6_algs[10] = { {
371 .cra_name = "__ecb-cast6-avx",
372 .cra_driver_name = "__driver-ecb-cast6-avx",
374 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER |
376 .cra_blocksize = CAST6_BLOCK_SIZE,
377 .cra_ctxsize = sizeof(struct cast6_ctx),
379 .cra_type = &crypto_blkcipher_type,
380 .cra_module = THIS_MODULE,
383 .min_keysize = CAST6_MIN_KEY_SIZE,
384 .max_keysize = CAST6_MAX_KEY_SIZE,
385 .setkey = cast6_setkey,
386 .encrypt = ecb_encrypt,
387 .decrypt = ecb_decrypt,
391 .cra_name = "__cbc-cast6-avx",
392 .cra_driver_name = "__driver-cbc-cast6-avx",
394 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER |
396 .cra_blocksize = CAST6_BLOCK_SIZE,
397 .cra_ctxsize = sizeof(struct cast6_ctx),
399 .cra_type = &crypto_blkcipher_type,
400 .cra_module = THIS_MODULE,
403 .min_keysize = CAST6_MIN_KEY_SIZE,
404 .max_keysize = CAST6_MAX_KEY_SIZE,
405 .setkey = cast6_setkey,
406 .encrypt = cbc_encrypt,
407 .decrypt = cbc_decrypt,
411 .cra_name = "__ctr-cast6-avx",
412 .cra_driver_name = "__driver-ctr-cast6-avx",
414 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER |
417 .cra_ctxsize = sizeof(struct cast6_ctx),
419 .cra_type = &crypto_blkcipher_type,
420 .cra_module = THIS_MODULE,
423 .min_keysize = CAST6_MIN_KEY_SIZE,
424 .max_keysize = CAST6_MAX_KEY_SIZE,
425 .ivsize = CAST6_BLOCK_SIZE,
426 .setkey = cast6_setkey,
427 .encrypt = ctr_crypt,
428 .decrypt = ctr_crypt,
432 .cra_name = "__lrw-cast6-avx",
433 .cra_driver_name = "__driver-lrw-cast6-avx",
435 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER |
437 .cra_blocksize = CAST6_BLOCK_SIZE,
438 .cra_ctxsize = sizeof(struct cast6_lrw_ctx),
440 .cra_type = &crypto_blkcipher_type,
441 .cra_module = THIS_MODULE,
442 .cra_exit = lrw_exit_tfm,
445 .min_keysize = CAST6_MIN_KEY_SIZE +
447 .max_keysize = CAST6_MAX_KEY_SIZE +
449 .ivsize = CAST6_BLOCK_SIZE,
450 .setkey = lrw_cast6_setkey,
451 .encrypt = lrw_encrypt,
452 .decrypt = lrw_decrypt,
456 .cra_name = "__xts-cast6-avx",
457 .cra_driver_name = "__driver-xts-cast6-avx",
459 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER |
461 .cra_blocksize = CAST6_BLOCK_SIZE,
462 .cra_ctxsize = sizeof(struct cast6_xts_ctx),
464 .cra_type = &crypto_blkcipher_type,
465 .cra_module = THIS_MODULE,
468 .min_keysize = CAST6_MIN_KEY_SIZE * 2,
469 .max_keysize = CAST6_MAX_KEY_SIZE * 2,
470 .ivsize = CAST6_BLOCK_SIZE,
471 .setkey = xts_cast6_setkey,
472 .encrypt = xts_encrypt,
473 .decrypt = xts_decrypt,
477 .cra_name = "ecb(cast6)",
478 .cra_driver_name = "ecb-cast6-avx",
480 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
481 .cra_blocksize = CAST6_BLOCK_SIZE,
482 .cra_ctxsize = sizeof(struct async_helper_ctx),
484 .cra_type = &crypto_ablkcipher_type,
485 .cra_module = THIS_MODULE,
486 .cra_init = ablk_init,
487 .cra_exit = ablk_exit,
490 .min_keysize = CAST6_MIN_KEY_SIZE,
491 .max_keysize = CAST6_MAX_KEY_SIZE,
492 .setkey = ablk_set_key,
493 .encrypt = ablk_encrypt,
494 .decrypt = ablk_decrypt,
498 .cra_name = "cbc(cast6)",
499 .cra_driver_name = "cbc-cast6-avx",
501 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
502 .cra_blocksize = CAST6_BLOCK_SIZE,
503 .cra_ctxsize = sizeof(struct async_helper_ctx),
505 .cra_type = &crypto_ablkcipher_type,
506 .cra_module = THIS_MODULE,
507 .cra_init = ablk_init,
508 .cra_exit = ablk_exit,
511 .min_keysize = CAST6_MIN_KEY_SIZE,
512 .max_keysize = CAST6_MAX_KEY_SIZE,
513 .ivsize = CAST6_BLOCK_SIZE,
514 .setkey = ablk_set_key,
515 .encrypt = __ablk_encrypt,
516 .decrypt = ablk_decrypt,
520 .cra_name = "ctr(cast6)",
521 .cra_driver_name = "ctr-cast6-avx",
523 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
525 .cra_ctxsize = sizeof(struct async_helper_ctx),
527 .cra_type = &crypto_ablkcipher_type,
528 .cra_module = THIS_MODULE,
529 .cra_init = ablk_init,
530 .cra_exit = ablk_exit,
533 .min_keysize = CAST6_MIN_KEY_SIZE,
534 .max_keysize = CAST6_MAX_KEY_SIZE,
535 .ivsize = CAST6_BLOCK_SIZE,
536 .setkey = ablk_set_key,
537 .encrypt = ablk_encrypt,
538 .decrypt = ablk_encrypt,
543 .cra_name = "lrw(cast6)",
544 .cra_driver_name = "lrw-cast6-avx",
546 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
547 .cra_blocksize = CAST6_BLOCK_SIZE,
548 .cra_ctxsize = sizeof(struct async_helper_ctx),
550 .cra_type = &crypto_ablkcipher_type,
551 .cra_module = THIS_MODULE,
552 .cra_init = ablk_init,
553 .cra_exit = ablk_exit,
556 .min_keysize = CAST6_MIN_KEY_SIZE +
558 .max_keysize = CAST6_MAX_KEY_SIZE +
560 .ivsize = CAST6_BLOCK_SIZE,
561 .setkey = ablk_set_key,
562 .encrypt = ablk_encrypt,
563 .decrypt = ablk_decrypt,
567 .cra_name = "xts(cast6)",
568 .cra_driver_name = "xts-cast6-avx",
570 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
571 .cra_blocksize = CAST6_BLOCK_SIZE,
572 .cra_ctxsize = sizeof(struct async_helper_ctx),
574 .cra_type = &crypto_ablkcipher_type,
575 .cra_module = THIS_MODULE,
576 .cra_init = ablk_init,
577 .cra_exit = ablk_exit,
580 .min_keysize = CAST6_MIN_KEY_SIZE * 2,
581 .max_keysize = CAST6_MAX_KEY_SIZE * 2,
582 .ivsize = CAST6_BLOCK_SIZE,
583 .setkey = ablk_set_key,
584 .encrypt = ablk_encrypt,
585 .decrypt = ablk_decrypt,
590 static int __init cast6_init(void)
592 const char *feature_name;
594 if (!cpu_has_xfeatures(XSTATE_SSE | XSTATE_YMM, &feature_name)) {
595 pr_info("CPU feature '%s' is not supported.\n", feature_name);
599 return crypto_register_algs(cast6_algs, ARRAY_SIZE(cast6_algs));
602 static void __exit cast6_exit(void)
604 crypto_unregister_algs(cast6_algs, ARRAY_SIZE(cast6_algs));
607 module_init(cast6_init);
608 module_exit(cast6_exit);
610 MODULE_DESCRIPTION("Cast6 Cipher Algorithm, AVX optimized");
611 MODULE_LICENSE("GPL");
612 MODULE_ALIAS_CRYPTO("cast6");