drivers/crypto/caam/caamalg.c

   1 /*
   2  * caam - Freescale FSL CAAM support for crypto API
   3  *
   4  * Copyright 2008-2011 Freescale Semiconductor, Inc.
   5  *
   6  * Based on talitos crypto API driver.
   7  *
   8  * relationship of job descriptors to shared descriptors (SteveC Dec 10 2008):
   9  *
  10  * ---------------                     ---------------
  11  * | JobDesc #1  |-------------------->|  ShareDesc  |
  12  * | *(packet 1) |                     |   (PDB)     |
  13  * ---------------      |------------->|  (hashKey)  |
  14  *       .              |              | (cipherKey) |
  15  *       .              |    |-------->| (operation) |
  16  * ---------------      |    |         ---------------
  17  * | JobDesc #2  |------|    |
  18  * | *(packet 2) |           |
  19  * ---------------           |
  20  *       .                   |
  21  *       .                   |
  22  * ---------------           |
  23  * | JobDesc #3  |------------
  24  * | *(packet 3) |
  25  * ---------------
  26  *
  27  * The SharedDesc never changes for a connection unless rekeyed, but
  28  * each packet will likely be in a different place. So all we need
  29  * to know to process the packet is where the input is, where the
  30  * output goes, and what context we want to process with. Context is
  31  * in the SharedDesc, packet references in the JobDesc.
  32  *
  33  * So, a job desc looks like:
  34  *
  35  * ---------------------
  36  * | Header            |
  37  * | ShareDesc Pointer |
  38  * | SEQ_OUT_PTR       |
  39  * | (output buffer)   |
  40  * | (output length)   |
  41  * | SEQ_IN_PTR        |
  42  * | (input buffer)    |
  43  * | (input length)    |
  44  * ---------------------
  45  */
  46
  47 #include "compat.h"
  48
  49 #include "regs.h"
  50 #include "intern.h"
  51 #include "desc_constr.h"
  52 #include "jr.h"
  53 #include "error.h"
  54 #include "sg_sw_sec4.h"
  55 #include "key_gen.h"
  56
  57 /*
  58  * crypto alg
  59  */
  60 #define CAAM_CRA_PRIORITY               3000
  61 /* max key is sum of AES_MAX_KEY_SIZE, max split key size */
  62 #define CAAM_MAX_KEY_SIZE               (AES_MAX_KEY_SIZE + \
  63                                          SHA512_DIGEST_SIZE * 2)
  64 /* max IV is max of AES_BLOCK_SIZE, DES3_EDE_BLOCK_SIZE */
  65 #define CAAM_MAX_IV_LENGTH              16
  66
  67 /* length of descriptors text */
  68 #define DESC_AEAD_BASE                  (4 * CAAM_CMD_SZ)
  69 #define DESC_AEAD_ENC_LEN               (DESC_AEAD_BASE + 16 * CAAM_CMD_SZ)
  70 #define DESC_AEAD_DEC_LEN               (DESC_AEAD_BASE + 21 * CAAM_CMD_SZ)
  71 #define DESC_AEAD_GIVENC_LEN            (DESC_AEAD_ENC_LEN + 7 * CAAM_CMD_SZ)
  72
  73 #define DESC_ABLKCIPHER_BASE            (3 * CAAM_CMD_SZ)
  74 #define DESC_ABLKCIPHER_ENC_LEN         (DESC_ABLKCIPHER_BASE + \
  75                                          20 * CAAM_CMD_SZ)
  76 #define DESC_ABLKCIPHER_DEC_LEN         (DESC_ABLKCIPHER_BASE + \
  77                                          15 * CAAM_CMD_SZ)
  78
  79 #define DESC_MAX_USED_BYTES             (DESC_AEAD_GIVENC_LEN + \
  80                                          CAAM_MAX_KEY_SIZE)
  81 #define DESC_MAX_USED_LEN               (DESC_MAX_USED_BYTES / CAAM_CMD_SZ)
  82
  83 #ifdef DEBUG
  84 /* for print_hex_dumps with line references */
  85 #define debug(format, arg...) printk(format, arg)
  86 #else
  87 #define debug(format, arg...)
  88 #endif
  89 static struct list_head alg_list;
  90
  91 /* Set DK bit in class 1 operation if shared */
  92 static inline void append_dec_op1(u32 *desc, u32 type)
  93 {
  94         u32 *jump_cmd, *uncond_jump_cmd;
  95
  96         jump_cmd = append_jump(desc, JUMP_TEST_ALL | JUMP_COND_SHRD);
  97         append_operation(desc, type | OP_ALG_AS_INITFINAL |
  98                          OP_ALG_DECRYPT);
  99         uncond_jump_cmd = append_jump(desc, JUMP_TEST_ALL);
 100         set_jump_tgt_here(desc, jump_cmd);
 101         append_operation(desc, type | OP_ALG_AS_INITFINAL |
 102                          OP_ALG_DECRYPT | OP_ALG_AAI_DK);
 103         set_jump_tgt_here(desc, uncond_jump_cmd);
 104 }
 105
 106 /*
 107  * Wait for completion of class 1 key loading before allowing
 108  * error propagation
 109  */
 110 static inline void append_dec_shr_done(u32 *desc)
 111 {
 112         u32 *jump_cmd;
 113
 114         jump_cmd = append_jump(desc, JUMP_CLASS_CLASS1 | JUMP_TEST_ALL);
 115         set_jump_tgt_here(desc, jump_cmd);
 116         append_cmd(desc, SET_OK_NO_PROP_ERRORS | CMD_LOAD);
 117 }
 118
 119 /*
 120  * For aead functions, read payload and write payload,
 121  * both of which are specified in req->src and req->dst
 122  */
 123 static inline void aead_append_src_dst(u32 *desc, u32 msg_type)
 124 {
 125         append_seq_fifo_load(desc, 0, FIFOLD_CLASS_BOTH |
 126                              KEY_VLF | msg_type | FIFOLD_TYPE_LASTBOTH);
 127         append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | KEY_VLF);
 128 }
 129
 130 /*
 131  * For aead encrypt and decrypt, read iv for both classes
 132  */
 133 static inline void aead_append_ld_iv(u32 *desc, int ivsize)
 134 {
 135         append_cmd(desc, CMD_SEQ_LOAD | LDST_SRCDST_BYTE_CONTEXT |
 136                    LDST_CLASS_1_CCB | ivsize);
 137         append_move(desc, MOVE_SRC_CLASS1CTX | MOVE_DEST_CLASS2INFIFO | ivsize);
 138 }
 139
 140 /*
 141  * For ablkcipher encrypt and decrypt, read from req->src and
 142  * write to req->dst
 143  */
 144 static inline void ablkcipher_append_src_dst(u32 *desc)
 145 {
 146         append_math_add(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
 147         append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
 148         append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 |
 149                              KEY_VLF | FIFOLD_TYPE_MSG | FIFOLD_TYPE_LAST1);
 150         append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | KEY_VLF);
 151 }
 152
 153 /*
 154  * If all data, including src (with assoc and iv) or dst (with iv only) are
 155  * contiguous
 156  */
 157 #define GIV_SRC_CONTIG          1
 158 #define GIV_DST_CONTIG          (1 << 1)
 159
 160 /*
 161  * per-session context
 162  */
 163 struct caam_ctx {
 164         struct device *jrdev;
 165         u32 sh_desc_enc[DESC_MAX_USED_LEN];
 166         u32 sh_desc_dec[DESC_MAX_USED_LEN];
 167         u32 sh_desc_givenc[DESC_MAX_USED_LEN];
 168         dma_addr_t sh_desc_enc_dma;
 169         dma_addr_t sh_desc_dec_dma;
 170         dma_addr_t sh_desc_givenc_dma;
 171         u32 class1_alg_type;
 172         u32 class2_alg_type;
 173         u32 alg_op;
 174         u8 key[CAAM_MAX_KEY_SIZE];
 175         dma_addr_t key_dma;
 176         unsigned int enckeylen;
 177         unsigned int split_key_len;
 178         unsigned int split_key_pad_len;
 179         unsigned int authsize;
 180 };
 181
 182 static void append_key_aead(u32 *desc, struct caam_ctx *ctx,
 183                             int keys_fit_inline)
 184 {
 185         if (keys_fit_inline) {
 186                 append_key_as_imm(desc, ctx->key, ctx->split_key_pad_len,
 187                                   ctx->split_key_len, CLASS_2 |
 188                                   KEY_DEST_MDHA_SPLIT | KEY_ENC);
 189                 append_key_as_imm(desc, (void *)ctx->key +
 190                                   ctx->split_key_pad_len, ctx->enckeylen,
 191                                   ctx->enckeylen, CLASS_1 | KEY_DEST_CLASS_REG);
 192         } else {
 193                 append_key(desc, ctx->key_dma, ctx->split_key_len, CLASS_2 |
 194                            KEY_DEST_MDHA_SPLIT | KEY_ENC);
 195                 append_key(desc, ctx->key_dma + ctx->split_key_pad_len,
 196                            ctx->enckeylen, CLASS_1 | KEY_DEST_CLASS_REG);
 197         }
 198 }
 199
 200 static void init_sh_desc_key_aead(u32 *desc, struct caam_ctx *ctx,
 201                                   int keys_fit_inline)
 202 {
 203         u32 *key_jump_cmd;
 204
 205         init_sh_desc(desc, HDR_SHARE_SERIAL);
 206
 207         /* Skip if already shared */
 208         key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
 209                                    JUMP_COND_SHRD);
 210
 211         append_key_aead(desc, ctx, keys_fit_inline);
 212
 213         set_jump_tgt_here(desc, key_jump_cmd);
 214
 215         /* Propagate errors from shared to job descriptor */
 216         append_cmd(desc, SET_OK_NO_PROP_ERRORS | CMD_LOAD);
 217 }
 218
 219 static int aead_set_sh_desc(struct crypto_aead *aead)
 220 {
 221         struct aead_tfm *tfm = &aead->base.crt_aead;
 222         struct caam_ctx *ctx = crypto_aead_ctx(aead);
 223         struct device *jrdev = ctx->jrdev;
 224         bool keys_fit_inline = false;
 225         u32 *key_jump_cmd, *jump_cmd;
 226         u32 geniv, moveiv;
 227         u32 *desc;
 228
 229         if (!ctx->enckeylen || !ctx->authsize)
 230                 return 0;
 231
 232         /*
 233          * Job Descriptor and Shared Descriptors
 234          * must all fit into the 64-word Descriptor h/w Buffer
 235          */
 236         if (DESC_AEAD_ENC_LEN + DESC_JOB_IO_LEN +
 237             ctx->split_key_pad_len + ctx->enckeylen <=
 238             CAAM_DESC_BYTES_MAX)
 239                 keys_fit_inline = true;
 240
 241         /* aead_encrypt shared descriptor */
 242         desc = ctx->sh_desc_enc;
 243
 244         init_sh_desc_key_aead(desc, ctx, keys_fit_inline);
 245
 246         /* Class 2 operation */
 247         append_operation(desc, ctx->class2_alg_type |
 248                          OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT);
 249
 250         /* cryptlen = seqoutlen - authsize */
 251         append_math_sub_imm_u32(desc, REG3, SEQOUTLEN, IMM, ctx->authsize);
 252
 253         /* assoclen + cryptlen = seqinlen - ivsize */
 254         append_math_sub_imm_u32(desc, REG2, SEQINLEN, IMM, tfm->ivsize);
 255
 256         /* assoclen + cryptlen = (assoclen + cryptlen) - cryptlen */
 257         append_math_sub(desc, VARSEQINLEN, REG2, REG3, CAAM_CMD_SZ);
 258
 259         /* read assoc before reading payload */
 260         append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG |
 261                              KEY_VLF);
 262         aead_append_ld_iv(desc, tfm->ivsize);
 263
 264         /* Class 1 operation */
 265         append_operation(desc, ctx->class1_alg_type |
 266                          OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT);
 267
 268         /* Read and write cryptlen bytes */
 269         append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ);
 270         append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
 271         aead_append_src_dst(desc, FIFOLD_TYPE_MSG1OUT2);
 272
 273         /* Write ICV */
 274         append_seq_store(desc, ctx->authsize, LDST_CLASS_2_CCB |
 275                          LDST_SRCDST_BYTE_CONTEXT);
 276
 277         ctx->sh_desc_enc_dma = dma_map_single(jrdev, desc,
 278                                               desc_bytes(desc),
 279                                               DMA_TO_DEVICE);
 280         if (dma_mapping_error(jrdev, ctx->sh_desc_enc_dma)) {
 281                 dev_err(jrdev, "unable to map shared descriptor\n");
 282                 return -ENOMEM;
 283         }
 284 #ifdef DEBUG
 285         print_hex_dump(KERN_ERR, "aead enc shdesc@"__stringify(__LINE__)": ",
 286                        DUMP_PREFIX_ADDRESS, 16, 4, desc,
 287                        desc_bytes(desc), 1);
 288 #endif
 289
 290         /*
 291          * Job Descriptor and Shared Descriptors
 292          * must all fit into the 64-word Descriptor h/w Buffer
 293          */
 294         if (DESC_AEAD_DEC_LEN + DESC_JOB_IO_LEN +
 295             ctx->split_key_pad_len + ctx->enckeylen <=
 296             CAAM_DESC_BYTES_MAX)
 297                 keys_fit_inline = true;
 298
 299         desc = ctx->sh_desc_dec;
 300
 301         /* aead_decrypt shared descriptor */
 302         init_sh_desc(desc, HDR_SHARE_SERIAL);
 303
 304         /* Skip if already shared */
 305         key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
 306                                    JUMP_COND_SHRD);
 307
 308         append_key_aead(desc, ctx, keys_fit_inline);
 309
 310         /* Only propagate error immediately if shared */
 311         jump_cmd = append_jump(desc, JUMP_TEST_ALL);
 312         set_jump_tgt_here(desc, key_jump_cmd);
 313         append_cmd(desc, SET_OK_NO_PROP_ERRORS | CMD_LOAD);
 314         set_jump_tgt_here(desc, jump_cmd);
 315
 316         /* Class 2 operation */
 317         append_operation(desc, ctx->class2_alg_type |
 318                          OP_ALG_AS_INITFINAL | OP_ALG_DECRYPT | OP_ALG_ICV_ON);
 319
 320         /* assoclen + cryptlen = seqinlen - ivsize */
 321         append_math_sub_imm_u32(desc, REG3, SEQINLEN, IMM,
 322                                 ctx->authsize + tfm->ivsize)
 323         /* assoclen = (assoclen + cryptlen) - cryptlen */
 324         append_math_sub(desc, REG2, SEQOUTLEN, REG0, CAAM_CMD_SZ);
 325         append_math_sub(desc, VARSEQINLEN, REG3, REG2, CAAM_CMD_SZ);
 326
 327         /* read assoc before reading payload */
 328         append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG |
 329                              KEY_VLF);
 330
 331         aead_append_ld_iv(desc, tfm->ivsize);
 332
 333         append_dec_op1(desc, ctx->class1_alg_type);
 334
 335         /* Read and write cryptlen bytes */
 336         append_math_add(desc, VARSEQINLEN, ZERO, REG2, CAAM_CMD_SZ);
 337         append_math_add(desc, VARSEQOUTLEN, ZERO, REG2, CAAM_CMD_SZ);
 338         aead_append_src_dst(desc, FIFOLD_TYPE_MSG);
 339
 340         /* Load ICV */
 341         append_seq_fifo_load(desc, ctx->authsize, FIFOLD_CLASS_CLASS2 |
 342                              FIFOLD_TYPE_LAST2 | FIFOLD_TYPE_ICV);
 343         append_dec_shr_done(desc);
 344
 345         ctx->sh_desc_dec_dma = dma_map_single(jrdev, desc,
 346                                               desc_bytes(desc),
 347                                               DMA_TO_DEVICE);
 348         if (dma_mapping_error(jrdev, ctx->sh_desc_dec_dma)) {
 349                 dev_err(jrdev, "unable to map shared descriptor\n");
 350                 return -ENOMEM;
 351         }
 352 #ifdef DEBUG
 353         print_hex_dump(KERN_ERR, "aead dec shdesc@"__stringify(__LINE__)": ",
 354                        DUMP_PREFIX_ADDRESS, 16, 4, desc,
 355                        desc_bytes(desc), 1);
 356 #endif
 357
 358         /*
 359          * Job Descriptor and Shared Descriptors
 360          * must all fit into the 64-word Descriptor h/w Buffer
 361          */
 362         if (DESC_AEAD_GIVENC_LEN + DESC_JOB_IO_LEN +
 363             ctx->split_key_pad_len + ctx->enckeylen <=
 364             CAAM_DESC_BYTES_MAX)
 365                 keys_fit_inline = true;
 366
 367         /* aead_givencrypt shared descriptor */
 368         desc = ctx->sh_desc_givenc;
 369
 370         init_sh_desc_key_aead(desc, ctx, keys_fit_inline);
 371
 372         /* Generate IV */
 373         geniv = NFIFOENTRY_STYPE_PAD | NFIFOENTRY_DEST_DECO |
 374                 NFIFOENTRY_DTYPE_MSG | NFIFOENTRY_LC1 |
 375                 NFIFOENTRY_PTYPE_RND | (tfm->ivsize << NFIFOENTRY_DLEN_SHIFT);
 376         append_load_imm_u32(desc, geniv, LDST_CLASS_IND_CCB |
 377                             LDST_SRCDST_WORD_INFO_FIFO | LDST_IMM);
 378         append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO);
 379         append_move(desc, MOVE_SRC_INFIFO |
 380                     MOVE_DEST_CLASS1CTX | (tfm->ivsize << MOVE_LEN_SHIFT));
 381         append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO);
 382
 383         /* Copy IV to class 1 context */
 384         append_move(desc, MOVE_SRC_CLASS1CTX |
 385                     MOVE_DEST_OUTFIFO | (tfm->ivsize << MOVE_LEN_SHIFT));
 386
 387         /* Return to encryption */
 388         append_operation(desc, ctx->class2_alg_type |
 389                          OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT);
 390
 391         /* ivsize + cryptlen = seqoutlen - authsize */
 392         append_math_sub_imm_u32(desc, REG3, SEQOUTLEN, IMM, ctx->authsize);
 393
 394         /* assoclen = seqinlen - (ivsize + cryptlen) */
 395         append_math_sub(desc, VARSEQINLEN, SEQINLEN, REG3, CAAM_CMD_SZ);
 396
 397         /* read assoc before reading payload */
 398         append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG |
 399                              KEY_VLF);
 400
 401         /* Copy iv from class 1 ctx to class 2 fifo*/
 402         moveiv = NFIFOENTRY_STYPE_OFIFO | NFIFOENTRY_DEST_CLASS2 |
 403                  NFIFOENTRY_DTYPE_MSG | (tfm->ivsize << NFIFOENTRY_DLEN_SHIFT);
 404         append_load_imm_u32(desc, moveiv, LDST_CLASS_IND_CCB |
 405                             LDST_SRCDST_WORD_INFO_FIFO | LDST_IMM);
 406         append_load_imm_u32(desc, tfm->ivsize, LDST_CLASS_2_CCB |
 407                             LDST_SRCDST_WORD_DATASZ_REG | LDST_IMM);
 408
 409         /* Class 1 operation */
 410         append_operation(desc, ctx->class1_alg_type |
 411                          OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT);
 412
 413         /* Will write ivsize + cryptlen */
 414         append_math_add(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
 415
 416         /* Not need to reload iv */
 417         append_seq_fifo_load(desc, tfm->ivsize,
 418                              FIFOLD_CLASS_SKIP);
 419
 420         /* Will read cryptlen */
 421         append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
 422         aead_append_src_dst(desc, FIFOLD_TYPE_MSG1OUT2);
 423
 424         /* Write ICV */
 425         append_seq_store(desc, ctx->authsize, LDST_CLASS_2_CCB |
 426                          LDST_SRCDST_BYTE_CONTEXT);
 427
 428         ctx->sh_desc_givenc_dma = dma_map_single(jrdev, desc,
 429                                                  desc_bytes(desc),
 430                                                  DMA_TO_DEVICE);
 431         if (dma_mapping_error(jrdev, ctx->sh_desc_givenc_dma)) {
 432                 dev_err(jrdev, "unable to map shared descriptor\n");
 433                 return -ENOMEM;
 434         }
 435 #ifdef DEBUG
 436         print_hex_dump(KERN_ERR, "aead givenc shdesc@"__stringify(__LINE__)": ",
 437                        DUMP_PREFIX_ADDRESS, 16, 4, desc,
 438                        desc_bytes(desc), 1);
 439 #endif
 440
 441         return 0;
 442 }
 443
 444 static int aead_setauthsize(struct crypto_aead *authenc,
 445                                     unsigned int authsize)
 446 {
 447         struct caam_ctx *ctx = crypto_aead_ctx(authenc);
 448
 449         ctx->authsize = authsize;
 450         aead_set_sh_desc(authenc);
 451
 452         return 0;
 453 }
 454
 455 static u32 gen_split_aead_key(struct caam_ctx *ctx, const u8 *key_in,
 456                               u32 authkeylen)
 457 {
 458         return gen_split_key(ctx->jrdev, ctx->key, ctx->split_key_len,
 459                                ctx->split_key_pad_len, key_in, authkeylen,
 460                                ctx->alg_op);
 461 }
 462
 463 static int aead_setkey(struct crypto_aead *aead,
 464                                const u8 *key, unsigned int keylen)
 465 {
 466         /* Sizes for MDHA pads (*not* keys): MD5, SHA1, 224, 256, 384, 512 */
 467         static const u8 mdpadlen[] = { 16, 20, 32, 32, 64, 64 };
 468         struct caam_ctx *ctx = crypto_aead_ctx(aead);
 469         struct device *jrdev = ctx->jrdev;
 470         struct crypto_authenc_keys keys;
 471         int ret = 0;
 472
 473         if (crypto_authenc_extractkeys(&keys, key, keylen) != 0)
 474                 goto badkey;
 475
 476         /* Pick class 2 key length from algorithm submask */
 477         ctx->split_key_len = mdpadlen[(ctx->alg_op & OP_ALG_ALGSEL_SUBMASK) >>
 478                                       OP_ALG_ALGSEL_SHIFT] * 2;
 479         ctx->split_key_pad_len = ALIGN(ctx->split_key_len, 16);
 480
 481         if (ctx->split_key_pad_len + keys.enckeylen > CAAM_MAX_KEY_SIZE)
 482                 goto badkey;
 483
 484 #ifdef DEBUG
 485         printk(KERN_ERR "keylen %d enckeylen %d authkeylen %d\n",
 486                keys.authkeylen + keys.enckeylen, keys.enckeylen,
 487                keys.authkeylen);
 488         printk(KERN_ERR "split_key_len %d split_key_pad_len %d\n",
 489                ctx->split_key_len, ctx->split_key_pad_len);
 490         print_hex_dump(KERN_ERR, "key in @"__stringify(__LINE__)": ",
 491                        DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
 492 #endif
 493
 494         ret = gen_split_aead_key(ctx, keys.authkey, keys.authkeylen);
 495         if (ret) {
 496                 goto badkey;
 497         }
 498
 499         /* postpend encryption key to auth split key */
 500         memcpy(ctx->key + ctx->split_key_pad_len, keys.enckey, keys.enckeylen);
 501
 502         ctx->key_dma = dma_map_single(jrdev, ctx->key, ctx->split_key_pad_len +
 503                                       keys.enckeylen, DMA_TO_DEVICE);
 504         if (dma_mapping_error(jrdev, ctx->key_dma)) {
 505                 dev_err(jrdev, "unable to map key i/o memory\n");
 506                 return -ENOMEM;
 507         }
 508 #ifdef DEBUG
 509         print_hex_dump(KERN_ERR, "ctx.key@"__stringify(__LINE__)": ",
 510                        DUMP_PREFIX_ADDRESS, 16, 4, ctx->key,
 511                        ctx->split_key_pad_len + keys.enckeylen, 1);
 512 #endif
 513
 514         ctx->enckeylen = keys.enckeylen;
 515
 516         ret = aead_set_sh_desc(aead);
 517         if (ret) {
 518                 dma_unmap_single(jrdev, ctx->key_dma, ctx->split_key_pad_len +
 519                                  keys.enckeylen, DMA_TO_DEVICE);
 520         }
 521
 522         return ret;
 523 badkey:
 524         crypto_aead_set_flags(aead, CRYPTO_TFM_RES_BAD_KEY_LEN);
 525         return -EINVAL;
 526 }
 527
 528 static int ablkcipher_setkey(struct crypto_ablkcipher *ablkcipher,
 529                              const u8 *key, unsigned int keylen)
 530 {
 531         struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher);
 532         struct ablkcipher_tfm *tfm = &ablkcipher->base.crt_ablkcipher;
 533         struct device *jrdev = ctx->jrdev;
 534         int ret = 0;
 535         u32 *key_jump_cmd, *jump_cmd;
 536         u32 *desc;
 537
 538 #ifdef DEBUG
 539         print_hex_dump(KERN_ERR, "key in @"__stringify(__LINE__)": ",
 540                        DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
 541 #endif
 542
 543         memcpy(ctx->key, key, keylen);
 544         ctx->key_dma = dma_map_single(jrdev, ctx->key, keylen,
 545                                       DMA_TO_DEVICE);
 546         if (dma_mapping_error(jrdev, ctx->key_dma)) {
 547                 dev_err(jrdev, "unable to map key i/o memory\n");
 548                 return -ENOMEM;
 549         }
 550         ctx->enckeylen = keylen;
 551
 552         /* ablkcipher_encrypt shared descriptor */
 553         desc = ctx->sh_desc_enc;
 554         init_sh_desc(desc, HDR_SHARE_SERIAL);
 555         /* Skip if already shared */
 556         key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
 557                                    JUMP_COND_SHRD);
 558
 559         /* Load class1 key only */
 560         append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen,
 561                           ctx->enckeylen, CLASS_1 |
 562                           KEY_DEST_CLASS_REG);
 563
 564         set_jump_tgt_here(desc, key_jump_cmd);
 565
 566         /* Propagate errors from shared to job descriptor */
 567         append_cmd(desc, SET_OK_NO_PROP_ERRORS | CMD_LOAD);
 568
 569         /* Load iv */
 570         append_cmd(desc, CMD_SEQ_LOAD | LDST_SRCDST_BYTE_CONTEXT |
 571                    LDST_CLASS_1_CCB | tfm->ivsize);
 572
 573         /* Load operation */
 574         append_operation(desc, ctx->class1_alg_type |
 575                          OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT);
 576
 577         /* Perform operation */
 578         ablkcipher_append_src_dst(desc);
 579
 580         ctx->sh_desc_enc_dma = dma_map_single(jrdev, desc,
 581                                               desc_bytes(desc),
 582                                               DMA_TO_DEVICE);
 583         if (dma_mapping_error(jrdev, ctx->sh_desc_enc_dma)) {
 584                 dev_err(jrdev, "unable to map shared descriptor\n");
 585                 return -ENOMEM;
 586         }
 587 #ifdef DEBUG
 588         print_hex_dump(KERN_ERR,
 589                        "ablkcipher enc shdesc@"__stringify(__LINE__)": ",
 590                        DUMP_PREFIX_ADDRESS, 16, 4, desc,
 591                        desc_bytes(desc), 1);
 592 #endif
 593         /* ablkcipher_decrypt shared descriptor */
 594         desc = ctx->sh_desc_dec;
 595
 596         init_sh_desc(desc, HDR_SHARE_SERIAL);
 597         /* Skip if already shared */
 598         key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
 599                                    JUMP_COND_SHRD);
 600
 601         /* Load class1 key only */
 602         append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen,
 603                           ctx->enckeylen, CLASS_1 |
 604                           KEY_DEST_CLASS_REG);
 605
 606         /* For aead, only propagate error immediately if shared */
 607         jump_cmd = append_jump(desc, JUMP_TEST_ALL);
 608         set_jump_tgt_here(desc, key_jump_cmd);
 609         append_cmd(desc, SET_OK_NO_PROP_ERRORS | CMD_LOAD);
 610         set_jump_tgt_here(desc, jump_cmd);
 611
 612         /* load IV */
 613         append_cmd(desc, CMD_SEQ_LOAD | LDST_SRCDST_BYTE_CONTEXT |
 614                    LDST_CLASS_1_CCB | tfm->ivsize);
 615
 616         /* Choose operation */
 617         append_dec_op1(desc, ctx->class1_alg_type);
 618
 619         /* Perform operation */
 620         ablkcipher_append_src_dst(desc);
 621
 622         /* Wait for key to load before allowing propagating error */
 623         append_dec_shr_done(desc);
 624
 625         ctx->sh_desc_dec_dma = dma_map_single(jrdev, desc,
 626                                               desc_bytes(desc),
 627                                               DMA_TO_DEVICE);
 628         if (dma_mapping_error(jrdev, ctx->sh_desc_enc_dma)) {
 629                 dev_err(jrdev, "unable to map shared descriptor\n");
 630                 return -ENOMEM;
 631         }
 632
 633 #ifdef DEBUG
 634         print_hex_dump(KERN_ERR,
 635                        "ablkcipher dec shdesc@"__stringify(__LINE__)": ",
 636                        DUMP_PREFIX_ADDRESS, 16, 4, desc,
 637                        desc_bytes(desc), 1);
 638 #endif
 639
 640         return ret;
 641 }
 642
 643 /*
 644  * aead_edesc - s/w-extended aead descriptor
 645  * @assoc_nents: number of segments in associated data (SPI+Seq) scatterlist
 646  * @assoc_chained: if source is chained
 647  * @src_nents: number of segments in input scatterlist
 648  * @src_chained: if source is chained
 649  * @dst_nents: number of segments in output scatterlist
 650  * @dst_chained: if destination is chained
 651  * @iv_dma: dma address of iv for checking continuity and link table
 652  * @desc: h/w descriptor (variable length; must not exceed MAX_CAAM_DESCSIZE)
 653  * @sec4_sg_bytes: length of dma mapped sec4_sg space
 654  * @sec4_sg_dma: bus physical mapped address of h/w link table
 655  * @hw_desc: the h/w job descriptor followed by any referenced link tables
 656  */
 657 struct aead_edesc {
 658         int assoc_nents;
 659         bool assoc_chained;
 660         int src_nents;
 661         bool src_chained;
 662         int dst_nents;
 663         bool dst_chained;
 664         dma_addr_t iv_dma;
 665         int sec4_sg_bytes;
 666         dma_addr_t sec4_sg_dma;
 667         struct sec4_sg_entry *sec4_sg;
 668         u32 hw_desc[0];
 669 };
 670
 671 /*
 672  * ablkcipher_edesc - s/w-extended ablkcipher descriptor
 673  * @src_nents: number of segments in input scatterlist
 674  * @src_chained: if source is chained
 675  * @dst_nents: number of segments in output scatterlist
 676  * @dst_chained: if destination is chained
 677  * @iv_dma: dma address of iv for checking continuity and link table
 678  * @desc: h/w descriptor (variable length; must not exceed MAX_CAAM_DESCSIZE)
 679  * @sec4_sg_bytes: length of dma mapped sec4_sg space
 680  * @sec4_sg_dma: bus physical mapped address of h/w link table
 681  * @hw_desc: the h/w job descriptor followed by any referenced link tables
 682  */
 683 struct ablkcipher_edesc {
 684         int src_nents;
 685         bool src_chained;
 686         int dst_nents;
 687         bool dst_chained;
 688         dma_addr_t iv_dma;
 689         int sec4_sg_bytes;
 690         dma_addr_t sec4_sg_dma;
 691         struct sec4_sg_entry *sec4_sg;
 692         u32 hw_desc[0];
 693 };
 694
 695 static void caam_unmap(struct device *dev, struct scatterlist *src,
 696                        struct scatterlist *dst, int src_nents,
 697                        bool src_chained, int dst_nents, bool dst_chained,
 698                        dma_addr_t iv_dma, int ivsize, dma_addr_t sec4_sg_dma,
 699                        int sec4_sg_bytes)
 700 {
 701         if (dst != src) {
 702                 dma_unmap_sg_chained(dev, src, src_nents ? : 1, DMA_TO_DEVICE,
 703                                      src_chained);
 704                 dma_unmap_sg_chained(dev, dst, dst_nents ? : 1, DMA_FROM_DEVICE,
 705                                      dst_chained);
 706         } else {
 707                 dma_unmap_sg_chained(dev, src, src_nents ? : 1,
 708                                      DMA_BIDIRECTIONAL, src_chained);
 709         }
 710
 711         if (iv_dma)
 712                 dma_unmap_single(dev, iv_dma, ivsize, DMA_TO_DEVICE);
 713         if (sec4_sg_bytes)
 714                 dma_unmap_single(dev, sec4_sg_dma, sec4_sg_bytes,
 715                                  DMA_TO_DEVICE);
 716 }
 717
 718 static void aead_unmap(struct device *dev,
 719                        struct aead_edesc *edesc,
 720                        struct aead_request *req)
 721 {
 722         struct crypto_aead *aead = crypto_aead_reqtfm(req);
 723         int ivsize = crypto_aead_ivsize(aead);
 724
 725         dma_unmap_sg_chained(dev, req->assoc, edesc->assoc_nents,
 726                              DMA_TO_DEVICE, edesc->assoc_chained);
 727
 728         caam_unmap(dev, req->src, req->dst,
 729                    edesc->src_nents, edesc->src_chained, edesc->dst_nents,
 730                    edesc->dst_chained, edesc->iv_dma, ivsize,
 731                    edesc->sec4_sg_dma, edesc->sec4_sg_bytes);
 732 }
 733
 734 static void ablkcipher_unmap(struct device *dev,
 735                              struct ablkcipher_edesc *edesc,
 736                              struct ablkcipher_request *req)
 737 {
 738         struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req);
 739         int ivsize = crypto_ablkcipher_ivsize(ablkcipher);
 740
 741         caam_unmap(dev, req->src, req->dst,
 742                    edesc->src_nents, edesc->src_chained, edesc->dst_nents,
 743                    edesc->dst_chained, edesc->iv_dma, ivsize,
 744                    edesc->sec4_sg_dma, edesc->sec4_sg_bytes);
 745 }
 746
 747 static void aead_encrypt_done(struct device *jrdev, u32 *desc, u32 err,
 748                                    void *context)
 749 {
 750         struct aead_request *req = context;
 751         struct aead_edesc *edesc;
 752 #ifdef DEBUG
 753         struct crypto_aead *aead = crypto_aead_reqtfm(req);
 754         struct caam_ctx *ctx = crypto_aead_ctx(aead);
 755         int ivsize = crypto_aead_ivsize(aead);
 756
 757         dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
 758 #endif
 759
 760         edesc = (struct aead_edesc *)((char *)desc -
 761                  offsetof(struct aead_edesc, hw_desc));
 762
 763         if (err) {
 764                 char tmp[CAAM_ERROR_STR_MAX];
 765
 766                 dev_err(jrdev, "%08x: %s\n", err, caam_jr_strstatus(tmp, err));
 767         }
 768
 769         aead_unmap(jrdev, edesc, req);
 770
 771 #ifdef DEBUG
 772         print_hex_dump(KERN_ERR, "assoc  @"__stringify(__LINE__)": ",
 773                        DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->assoc),
 774                        req->assoclen , 1);
 775         print_hex_dump(KERN_ERR, "dstiv  @"__stringify(__LINE__)": ",
 776                        DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src) - ivsize,
 777                        edesc->src_nents ? 100 : ivsize, 1);
 778         print_hex_dump(KERN_ERR, "dst    @"__stringify(__LINE__)": ",
 779                        DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src),
 780                        edesc->src_nents ? 100 : req->cryptlen +
 781                        ctx->authsize + 4, 1);
 782 #endif
 783
 784         kfree(edesc);
 785
 786         aead_request_complete(req, err);
 787 }
 788
 789 static void aead_decrypt_done(struct device *jrdev, u32 *desc, u32 err,
 790                                    void *context)
 791 {
 792         struct aead_request *req = context;
 793         struct aead_edesc *edesc;
 794 #ifdef DEBUG
 795         struct crypto_aead *aead = crypto_aead_reqtfm(req);
 796         struct caam_ctx *ctx = crypto_aead_ctx(aead);
 797         int ivsize = crypto_aead_ivsize(aead);
 798
 799         dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
 800 #endif
 801
 802         edesc = (struct aead_edesc *)((char *)desc -
 803                  offsetof(struct aead_edesc, hw_desc));
 804
 805 #ifdef DEBUG
 806         print_hex_dump(KERN_ERR, "dstiv  @"__stringify(__LINE__)": ",
 807                        DUMP_PREFIX_ADDRESS, 16, 4, req->iv,
 808                        ivsize, 1);
 809         print_hex_dump(KERN_ERR, "dst    @"__stringify(__LINE__)": ",
 810                        DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->dst),
 811                        req->cryptlen - ctx->authsize, 1);
 812 #endif
 813
 814         if (err) {
 815                 char tmp[CAAM_ERROR_STR_MAX];
 816
 817                 dev_err(jrdev, "%08x: %s\n", err, caam_jr_strstatus(tmp, err));
 818         }
 819
 820         aead_unmap(jrdev, edesc, req);
 821
 822         /*
 823          * verify hw auth check passed else return -EBADMSG
 824          */
 825         if ((err & JRSTA_CCBERR_ERRID_MASK) == JRSTA_CCBERR_ERRID_ICVCHK)
 826                 err = -EBADMSG;
 827
 828 #ifdef DEBUG
 829         print_hex_dump(KERN_ERR, "iphdrout@"__stringify(__LINE__)": ",
 830                        DUMP_PREFIX_ADDRESS, 16, 4,
 831                        ((char *)sg_virt(req->assoc) - sizeof(struct iphdr)),
 832                        sizeof(struct iphdr) + req->assoclen +
 833                        ((req->cryptlen > 1500) ? 1500 : req->cryptlen) +
 834                        ctx->authsize + 36, 1);
 835         if (!err && edesc->sec4_sg_bytes) {
 836                 struct scatterlist *sg = sg_last(req->src, edesc->src_nents);
 837                 print_hex_dump(KERN_ERR, "sglastout@"__stringify(__LINE__)": ",
 838                                DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(sg),
 839                         sg->length + ctx->authsize + 16, 1);
 840         }
 841 #endif
 842
 843         kfree(edesc);
 844
 845         aead_request_complete(req, err);
 846 }
 847
 848 static void ablkcipher_encrypt_done(struct device *jrdev, u32 *desc, u32 err,
 849                                    void *context)
 850 {
 851         struct ablkcipher_request *req = context;
 852         struct ablkcipher_edesc *edesc;
 853 #ifdef DEBUG
 854         struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req);
 855         int ivsize = crypto_ablkcipher_ivsize(ablkcipher);
 856
 857         dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
 858 #endif
 859
 860         edesc = (struct ablkcipher_edesc *)((char *)desc -
 861                  offsetof(struct ablkcipher_edesc, hw_desc));
 862
 863         if (err) {
 864                 char tmp[CAAM_ERROR_STR_MAX];
 865
 866                 dev_err(jrdev, "%08x: %s\n", err, caam_jr_strstatus(tmp, err));
 867         }
 868
 869 #ifdef DEBUG
 870         print_hex_dump(KERN_ERR, "dstiv  @"__stringify(__LINE__)": ",
 871                        DUMP_PREFIX_ADDRESS, 16, 4, req->info,
 872                        edesc->src_nents > 1 ? 100 : ivsize, 1);
 873         print_hex_dump(KERN_ERR, "dst    @"__stringify(__LINE__)": ",
 874                        DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src),
 875                        edesc->dst_nents > 1 ? 100 : req->nbytes, 1);
 876 #endif
 877
 878         ablkcipher_unmap(jrdev, edesc, req);
 879         kfree(edesc);
 880
 881         ablkcipher_request_complete(req, err);
 882 }
 883
 884 static void ablkcipher_decrypt_done(struct device *jrdev, u32 *desc, u32 err,
 885                                     void *context)
 886 {
 887         struct ablkcipher_request *req = context;
 888         struct ablkcipher_edesc *edesc;
 889 #ifdef DEBUG
 890         struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req);
 891         int ivsize = crypto_ablkcipher_ivsize(ablkcipher);
 892
 893         dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
 894 #endif
 895
 896         edesc = (struct ablkcipher_edesc *)((char *)desc -
 897                  offsetof(struct ablkcipher_edesc, hw_desc));
 898         if (err) {
 899                 char tmp[CAAM_ERROR_STR_MAX];
 900
 901                 dev_err(jrdev, "%08x: %s\n", err, caam_jr_strstatus(tmp, err));
 902         }
 903
 904 #ifdef DEBUG
 905         print_hex_dump(KERN_ERR, "dstiv  @"__stringify(__LINE__)": ",
 906                        DUMP_PREFIX_ADDRESS, 16, 4, req->info,
 907                        ivsize, 1);
 908         print_hex_dump(KERN_ERR, "dst    @"__stringify(__LINE__)": ",
 909                        DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src),
 910                        edesc->dst_nents > 1 ? 100 : req->nbytes, 1);
 911 #endif
 912
 913         ablkcipher_unmap(jrdev, edesc, req);
 914         kfree(edesc);
 915
 916         ablkcipher_request_complete(req, err);
 917 }
 918
 919 /*
 920  * Fill in aead job descriptor
 921  */
 922 static void init_aead_job(u32 *sh_desc, dma_addr_t ptr,
 923                           struct aead_edesc *edesc,
 924                           struct aead_request *req,
 925                           bool all_contig, bool encrypt)
 926 {
 927         struct crypto_aead *aead = crypto_aead_reqtfm(req);
 928         struct caam_ctx *ctx = crypto_aead_ctx(aead);
 929         int ivsize = crypto_aead_ivsize(aead);
 930         int authsize = ctx->authsize;
 931         u32 *desc = edesc->hw_desc;
 932         u32 out_options = 0, in_options;
 933         dma_addr_t dst_dma, src_dma;
 934         int len, sec4_sg_index = 0;
 935
 936 #ifdef DEBUG
 937         debug("assoclen %d cryptlen %d authsize %d\n",
 938               req->assoclen, req->cryptlen, authsize);
 939         print_hex_dump(KERN_ERR, "assoc  @"__stringify(__LINE__)": ",
 940                        DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->assoc),
 941                        req->assoclen , 1);
 942         print_hex_dump(KERN_ERR, "presciv@"__stringify(__LINE__)": ",
 943                        DUMP_PREFIX_ADDRESS, 16, 4, req->iv,
 944                        edesc->src_nents ? 100 : ivsize, 1);
 945         print_hex_dump(KERN_ERR, "src    @"__stringify(__LINE__)": ",
 946                        DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src),
 947                         edesc->src_nents ? 100 : req->cryptlen, 1);
 948         print_hex_dump(KERN_ERR, "shrdesc@"__stringify(__LINE__)": ",
 949                        DUMP_PREFIX_ADDRESS, 16, 4, sh_desc,
 950                        desc_bytes(sh_desc), 1);
 951 #endif
 952
 953         len = desc_len(sh_desc);
 954         init_job_desc_shared(desc, ptr, len, HDR_SHARE_DEFER | HDR_REVERSE);
 955
 956         if (all_contig) {
 957                 src_dma = sg_dma_address(req->assoc);
 958                 in_options = 0;
 959         } else {
 960                 src_dma = edesc->sec4_sg_dma;
 961                 sec4_sg_index += (edesc->assoc_nents ? : 1) + 1 +
 962                                  (edesc->src_nents ? : 1);
 963                 in_options = LDST_SGF;
 964         }
 965
 966         append_seq_in_ptr(desc, src_dma, req->assoclen + ivsize + req->cryptlen,
 967                           in_options);
 968
 969         if (likely(req->src == req->dst)) {
 970                 if (all_contig) {
 971                         dst_dma = sg_dma_address(req->src);
 972                 } else {
 973                         dst_dma = src_dma + sizeof(struct sec4_sg_entry) *
 974                                   ((edesc->assoc_nents ? : 1) + 1);
 975                         out_options = LDST_SGF;
 976                 }
 977         } else {
 978                 if (!edesc->dst_nents) {
 979                         dst_dma = sg_dma_address(req->dst);
 980                 } else {
 981                         dst_dma = edesc->sec4_sg_dma +
 982                                   sec4_sg_index *
 983                                   sizeof(struct sec4_sg_entry);
 984                         out_options = LDST_SGF;
 985                 }
 986         }
 987         if (encrypt)
 988                 append_seq_out_ptr(desc, dst_dma, req->cryptlen + authsize,
 989                                    out_options);
 990         else
 991                 append_seq_out_ptr(desc, dst_dma, req->cryptlen - authsize,
 992                                    out_options);
 993 }
 994
 995 /*
 996  * Fill in aead givencrypt job descriptor
 997  */
 998 static void init_aead_giv_job(u32 *sh_desc, dma_addr_t ptr,
 999                               struct aead_edesc *edesc,
1000                               struct aead_request *req,
1001                               int contig)
1002 {
1003         struct crypto_aead *aead = crypto_aead_reqtfm(req);
1004         struct caam_ctx *ctx = crypto_aead_ctx(aead);
1005         int ivsize = crypto_aead_ivsize(aead);
1006         int authsize = ctx->authsize;
1007         u32 *desc = edesc->hw_desc;
1008         u32 out_options = 0, in_options;
1009         dma_addr_t dst_dma, src_dma;
1010         int len, sec4_sg_index = 0;
1011
1012 #ifdef DEBUG
1013         debug("assoclen %d cryptlen %d authsize %d\n",
1014               req->assoclen, req->cryptlen, authsize);
1015         print_hex_dump(KERN_ERR, "assoc  @"__stringify(__LINE__)": ",
1016                        DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->assoc),
1017                        req->assoclen , 1);
1018         print_hex_dump(KERN_ERR, "presciv@"__stringify(__LINE__)": ",
1019                        DUMP_PREFIX_ADDRESS, 16, 4, req->iv, ivsize, 1);
1020         print_hex_dump(KERN_ERR, "src    @"__stringify(__LINE__)": ",
1021                        DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src),
1022                         edesc->src_nents > 1 ? 100 : req->cryptlen, 1);
1023         print_hex_dump(KERN_ERR, "shrdesc@"__stringify(__LINE__)": ",
1024                        DUMP_PREFIX_ADDRESS, 16, 4, sh_desc,
1025                        desc_bytes(sh_desc), 1);
1026 #endif
1027
1028         len = desc_len(sh_desc);
1029         init_job_desc_shared(desc, ptr, len, HDR_SHARE_DEFER | HDR_REVERSE);
1030
1031         if (contig & GIV_SRC_CONTIG) {
1032                 src_dma = sg_dma_address(req->assoc);
1033                 in_options = 0;
1034         } else {
1035                 src_dma = edesc->sec4_sg_dma;
1036                 sec4_sg_index += edesc->assoc_nents + 1 + edesc->src_nents;
1037                 in_options = LDST_SGF;
1038         }
1039         append_seq_in_ptr(desc, src_dma, req->assoclen + ivsize + req->cryptlen,
1040                           in_options);
1041
1042         if (contig & GIV_DST_CONTIG) {
1043                 dst_dma = edesc->iv_dma;
1044         } else {
1045                 if (likely(req->src == req->dst)) {
1046                         dst_dma = src_dma + sizeof(struct sec4_sg_entry) *
1047                                   edesc->assoc_nents;
1048                         out_options = LDST_SGF;
1049                 } else {
1050                         dst_dma = edesc->sec4_sg_dma +
1051                                   sec4_sg_index *
1052                                   sizeof(struct sec4_sg_entry);
1053                         out_options = LDST_SGF;
1054                 }
1055         }
1056
1057         append_seq_out_ptr(desc, dst_dma, ivsize + req->cryptlen + authsize,
1058                            out_options);
1059 }
1060
1061 /*
1062  * Fill in ablkcipher job descriptor
1063  */
1064 static void init_ablkcipher_job(u32 *sh_desc, dma_addr_t ptr,
1065                                 struct ablkcipher_edesc *edesc,
1066                                 struct ablkcipher_request *req,
1067                                 bool iv_contig)
1068 {
1069         struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req);
1070         int ivsize = crypto_ablkcipher_ivsize(ablkcipher);
1071         u32 *desc = edesc->hw_desc;
1072         u32 out_options = 0, in_options;
1073         dma_addr_t dst_dma, src_dma;
1074         int len, sec4_sg_index = 0;
1075
1076 #ifdef DEBUG
1077         print_hex_dump(KERN_ERR, "presciv@"__stringify(__LINE__)": ",
1078                        DUMP_PREFIX_ADDRESS, 16, 4, req->info,
1079                        ivsize, 1);
1080         print_hex_dump(KERN_ERR, "src    @"__stringify(__LINE__)": ",
1081                        DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src),
1082                        edesc->src_nents ? 100 : req->nbytes, 1);
1083 #endif
1084
1085         len = desc_len(sh_desc);
1086         init_job_desc_shared(desc, ptr, len, HDR_SHARE_DEFER | HDR_REVERSE);
1087
1088         if (iv_contig) {
1089                 src_dma = edesc->iv_dma;
1090                 in_options = 0;
1091         } else {
1092                 src_dma = edesc->sec4_sg_dma;
1093                 sec4_sg_index += (iv_contig ? 0 : 1) + edesc->src_nents;
1094                 in_options = LDST_SGF;
1095         }
1096         append_seq_in_ptr(desc, src_dma, req->nbytes + ivsize, in_options);
1097
1098         if (likely(req->src == req->dst)) {
1099                 if (!edesc->src_nents && iv_contig) {
1100                         dst_dma = sg_dma_address(req->src);
1101                 } else {
1102                         dst_dma = edesc->sec4_sg_dma +
1103                                 sizeof(struct sec4_sg_entry);
1104                         out_options = LDST_SGF;
1105                 }
1106         } else {
1107                 if (!edesc->dst_nents) {
1108                         dst_dma = sg_dma_address(req->dst);
1109                 } else {
1110                         dst_dma = edesc->sec4_sg_dma +
1111                                 sec4_sg_index * sizeof(struct sec4_sg_entry);
1112                         out_options = LDST_SGF;
1113                 }
1114         }
1115         append_seq_out_ptr(desc, dst_dma, req->nbytes, out_options);
1116 }
1117
1118 /*
1119  * allocate and map the aead extended descriptor
1120  */
1121 static struct aead_edesc *aead_edesc_alloc(struct aead_request *req,
1122                                            int desc_bytes, bool *all_contig_ptr,
1123                                            bool encrypt)
1124 {
1125         struct crypto_aead *aead = crypto_aead_reqtfm(req);
1126         struct caam_ctx *ctx = crypto_aead_ctx(aead);
1127         struct device *jrdev = ctx->jrdev;
1128         gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG |
1129                        CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC;
1130         int assoc_nents, src_nents, dst_nents = 0;
1131         struct aead_edesc *edesc;
1132         dma_addr_t iv_dma = 0;
1133         int sgc;
1134         bool all_contig = true;
1135         bool assoc_chained = false, src_chained = false, dst_chained = false;
1136         int ivsize = crypto_aead_ivsize(aead);
1137         int sec4_sg_index, sec4_sg_len = 0, sec4_sg_bytes;
1138         unsigned int authsize = ctx->authsize;
1139
1140         assoc_nents = sg_count(req->assoc, req->assoclen, &assoc_chained);
1141
1142         if (unlikely(req->dst != req->src)) {
1143                 src_nents = sg_count(req->src, req->cryptlen, &src_chained);
1144                 dst_nents = sg_count(req->dst,
1145                                      req->cryptlen +
1146                                         (encrypt ? authsize : (-authsize)),
1147                                      &dst_chained);
1148         } else {
1149                 src_nents = sg_count(req->src,
1150                                      req->cryptlen +
1151                                         (encrypt ? authsize : 0),
1152                                      &src_chained);
1153         }
1154
1155         sgc = dma_map_sg_chained(jrdev, req->assoc, assoc_nents ? : 1,
1156                                  DMA_TO_DEVICE, assoc_chained);
1157         if (likely(req->src == req->dst)) {
1158                 sgc = dma_map_sg_chained(jrdev, req->src, src_nents ? : 1,
1159                                          DMA_BIDIRECTIONAL, src_chained);
1160         } else {
1161                 sgc = dma_map_sg_chained(jrdev, req->src, src_nents ? : 1,
1162                                          DMA_TO_DEVICE, src_chained);
1163                 sgc = dma_map_sg_chained(jrdev, req->dst, dst_nents ? : 1,
1164                                          DMA_FROM_DEVICE, dst_chained);
1165         }
1166
1167         /* Check if data are contiguous */
1168         iv_dma = dma_map_single(jrdev, req->iv, ivsize, DMA_TO_DEVICE);
1169         if (assoc_nents || sg_dma_address(req->assoc) + req->assoclen !=
1170             iv_dma || src_nents || iv_dma + ivsize !=
1171             sg_dma_address(req->src)) {
1172                 all_contig = false;
1173                 assoc_nents = assoc_nents ? : 1;
1174                 src_nents = src_nents ? : 1;
1175                 sec4_sg_len = assoc_nents + 1 + src_nents;
1176         }
1177         sec4_sg_len += dst_nents;
1178
1179         sec4_sg_bytes = sec4_sg_len * sizeof(struct sec4_sg_entry);
1180
1181         /* allocate space for base edesc and hw desc commands, link tables */
1182         edesc = kmalloc(sizeof(struct aead_edesc) + desc_bytes +
1183                         sec4_sg_bytes, GFP_DMA | flags);
1184         if (!edesc) {
1185                 dev_err(jrdev, "could not allocate extended descriptor\n");
1186                 return ERR_PTR(-ENOMEM);
1187         }
1188
1189         edesc->assoc_nents = assoc_nents;
1190         edesc->assoc_chained = assoc_chained;
1191         edesc->src_nents = src_nents;
1192         edesc->src_chained = src_chained;
1193         edesc->dst_nents = dst_nents;
1194         edesc->dst_chained = dst_chained;
1195         edesc->iv_dma = iv_dma;
1196         edesc->sec4_sg_bytes = sec4_sg_bytes;
1197         edesc->sec4_sg = (void *)edesc + sizeof(struct aead_edesc) +
1198                          desc_bytes;
1199         edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg,
1200                                             sec4_sg_bytes, DMA_TO_DEVICE);
1201         *all_contig_ptr = all_contig;
1202
1203         sec4_sg_index = 0;
1204         if (!all_contig) {
1205                 sg_to_sec4_sg(req->assoc,
1206                               (assoc_nents ? : 1),
1207                               edesc->sec4_sg +
1208                               sec4_sg_index, 0);
1209                 sec4_sg_index += assoc_nents ? : 1;
1210                 dma_to_sec4_sg_one(edesc->sec4_sg + sec4_sg_index,
1211                                    iv_dma, ivsize, 0);
1212                 sec4_sg_index += 1;
1213                 sg_to_sec4_sg_last(req->src,
1214                                    (src_nents ? : 1),
1215                                    edesc->sec4_sg +
1216                                    sec4_sg_index, 0);
1217                 sec4_sg_index += src_nents ? : 1;
1218         }
1219         if (dst_nents) {
1220                 sg_to_sec4_sg_last(req->dst, dst_nents,
1221                                    edesc->sec4_sg + sec4_sg_index, 0);
1222         }
1223
1224         return edesc;
1225 }
1226
1227 static int aead_encrypt(struct aead_request *req)
1228 {
1229         struct aead_edesc *edesc;
1230         struct crypto_aead *aead = crypto_aead_reqtfm(req);
1231         struct caam_ctx *ctx = crypto_aead_ctx(aead);
1232         struct device *jrdev = ctx->jrdev;
1233         bool all_contig;
1234         u32 *desc;
1235         int ret = 0;
1236
1237         /* allocate extended descriptor */
1238         edesc = aead_edesc_alloc(req, DESC_JOB_IO_LEN *
1239                                  CAAM_CMD_SZ, &all_contig, true);
1240         if (IS_ERR(edesc))
1241                 return PTR_ERR(edesc);
1242
1243         /* Create and submit job descriptor */
1244         init_aead_job(ctx->sh_desc_enc, ctx->sh_desc_enc_dma, edesc, req,
1245                       all_contig, true);
1246 #ifdef DEBUG
1247         print_hex_dump(KERN_ERR, "aead jobdesc@"__stringify(__LINE__)": ",
1248                        DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc,
1249                        desc_bytes(edesc->hw_desc), 1);
1250 #endif
1251
1252         desc = edesc->hw_desc;
1253         ret = caam_jr_enqueue(jrdev, desc, aead_encrypt_done, req);
1254         if (!ret) {
1255                 ret = -EINPROGRESS;
1256         } else {
1257                 aead_unmap(jrdev, edesc, req);
1258                 kfree(edesc);
1259         }
1260
1261         return ret;
1262 }
1263
1264 static int aead_decrypt(struct aead_request *req)
1265 {
1266         struct aead_edesc *edesc;
1267         struct crypto_aead *aead = crypto_aead_reqtfm(req);
1268         struct caam_ctx *ctx = crypto_aead_ctx(aead);
1269         struct device *jrdev = ctx->jrdev;
1270         bool all_contig;
1271         u32 *desc;
1272         int ret = 0;
1273
1274         /* allocate extended descriptor */
1275         edesc = aead_edesc_alloc(req, DESC_JOB_IO_LEN *
1276                                  CAAM_CMD_SZ, &all_contig, false);
1277         if (IS_ERR(edesc))
1278                 return PTR_ERR(edesc);
1279
1280 #ifdef DEBUG
1281         print_hex_dump(KERN_ERR, "dec src@"__stringify(__LINE__)": ",
1282                        DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src),
1283                        req->cryptlen, 1);
1284 #endif
1285
1286         /* Create and submit job descriptor*/
1287         init_aead_job(ctx->sh_desc_dec,
1288                       ctx->sh_desc_dec_dma, edesc, req, all_contig, false);
1289 #ifdef DEBUG
1290         print_hex_dump(KERN_ERR, "aead jobdesc@"__stringify(__LINE__)": ",
1291                        DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc,
1292                        desc_bytes(edesc->hw_desc), 1);
1293 #endif
1294
1295         desc = edesc->hw_desc;
1296         ret = caam_jr_enqueue(jrdev, desc, aead_decrypt_done, req);
1297         if (!ret) {
1298                 ret = -EINPROGRESS;
1299         } else {
1300                 aead_unmap(jrdev, edesc, req);
1301                 kfree(edesc);
1302         }
1303
1304         return ret;
1305 }
1306
1307 /*
1308  * allocate and map the aead extended descriptor for aead givencrypt
1309  */
1310 static struct aead_edesc *aead_giv_edesc_alloc(struct aead_givcrypt_request
1311                                                *greq, int desc_bytes,
1312                                                u32 *contig_ptr)
1313 {
1314         struct aead_request *req = &greq->areq;
1315         struct crypto_aead *aead = crypto_aead_reqtfm(req);
1316         struct caam_ctx *ctx = crypto_aead_ctx(aead);
1317         struct device *jrdev = ctx->jrdev;
1318         gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG |
1319                        CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC;
1320         int assoc_nents, src_nents, dst_nents = 0;
1321         struct aead_edesc *edesc;
1322         dma_addr_t iv_dma = 0;
1323         int sgc;
1324         u32 contig = GIV_SRC_CONTIG | GIV_DST_CONTIG;
1325         int ivsize = crypto_aead_ivsize(aead);
1326         bool assoc_chained = false, src_chained = false, dst_chained = false;
1327         int sec4_sg_index, sec4_sg_len = 0, sec4_sg_bytes;
1328
1329         assoc_nents = sg_count(req->assoc, req->assoclen, &assoc_chained);
1330         src_nents = sg_count(req->src, req->cryptlen, &src_chained);
1331
1332         if (unlikely(req->dst != req->src))
1333                 dst_nents = sg_count(req->dst, req->cryptlen + ctx->authsize,
1334                                      &dst_chained);
1335
1336         sgc = dma_map_sg_chained(jrdev, req->assoc, assoc_nents ? : 1,
1337                                  DMA_TO_DEVICE, assoc_chained);
1338         if (likely(req->src == req->dst)) {
1339                 sgc = dma_map_sg_chained(jrdev, req->src, src_nents ? : 1,
1340                                          DMA_BIDIRECTIONAL, src_chained);
1341         } else {
1342                 sgc = dma_map_sg_chained(jrdev, req->src, src_nents ? : 1,
1343                                          DMA_TO_DEVICE, src_chained);
1344                 sgc = dma_map_sg_chained(jrdev, req->dst, dst_nents ? : 1,
1345                                          DMA_FROM_DEVICE, dst_chained);
1346         }
1347
1348         /* Check if data are contiguous */
1349         iv_dma = dma_map_single(jrdev, greq->giv, ivsize, DMA_TO_DEVICE);
1350         if (assoc_nents || sg_dma_address(req->assoc) + req->assoclen !=
1351             iv_dma || src_nents || iv_dma + ivsize != sg_dma_address(req->src))
1352                 contig &= ~GIV_SRC_CONTIG;
1353         if (dst_nents || iv_dma + ivsize != sg_dma_address(req->dst))
1354                 contig &= ~GIV_DST_CONTIG;
1355         if (unlikely(req->src != req->dst)) {
1356                 dst_nents = dst_nents ? : 1;
1357                 sec4_sg_len += 1;
1358         }
1359         if (!(contig & GIV_SRC_CONTIG)) {
1360                 assoc_nents = assoc_nents ? : 1;
1361                 src_nents = src_nents ? : 1;
1362                 sec4_sg_len += assoc_nents + 1 + src_nents;
1363                 if (likely(req->src == req->dst))
1364                         contig &= ~GIV_DST_CONTIG;
1365         }
1366         sec4_sg_len += dst_nents;
1367
1368         sec4_sg_bytes = sec4_sg_len * sizeof(struct sec4_sg_entry);
1369
1370         /* allocate space for base edesc and hw desc commands, link tables */
1371         edesc = kmalloc(sizeof(struct aead_edesc) + desc_bytes +
1372                         sec4_sg_bytes, GFP_DMA | flags);
1373         if (!edesc) {
1374                 dev_err(jrdev, "could not allocate extended descriptor\n");
1375                 return ERR_PTR(-ENOMEM);
1376         }
1377
1378         edesc->assoc_nents = assoc_nents;
1379         edesc->assoc_chained = assoc_chained;
1380         edesc->src_nents = src_nents;
1381         edesc->src_chained = src_chained;
1382         edesc->dst_nents = dst_nents;
1383         edesc->dst_chained = dst_chained;
1384         edesc->iv_dma = iv_dma;
1385         edesc->sec4_sg_bytes = sec4_sg_bytes;
1386         edesc->sec4_sg = (void *)edesc + sizeof(struct aead_edesc) +
1387                          desc_bytes;
1388         edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg,
1389                                             sec4_sg_bytes, DMA_TO_DEVICE);
1390         *contig_ptr = contig;
1391
1392         sec4_sg_index = 0;
1393         if (!(contig & GIV_SRC_CONTIG)) {
1394                 sg_to_sec4_sg(req->assoc, assoc_nents,
1395                               edesc->sec4_sg +
1396                               sec4_sg_index, 0);
1397                 sec4_sg_index += assoc_nents;
1398                 dma_to_sec4_sg_one(edesc->sec4_sg + sec4_sg_index,
1399                                    iv_dma, ivsize, 0);
1400                 sec4_sg_index += 1;
1401                 sg_to_sec4_sg_last(req->src, src_nents,
1402                                    edesc->sec4_sg +
1403                                    sec4_sg_index, 0);
1404                 sec4_sg_index += src_nents;
1405         }
1406         if (unlikely(req->src != req->dst && !(contig & GIV_DST_CONTIG))) {
1407                 dma_to_sec4_sg_one(edesc->sec4_sg + sec4_sg_index,
1408                                    iv_dma, ivsize, 0);
1409                 sec4_sg_index += 1;
1410                 sg_to_sec4_sg_last(req->dst, dst_nents,
1411                                    edesc->sec4_sg + sec4_sg_index, 0);
1412         }
1413
1414         return edesc;
1415 }
1416
1417 static int aead_givencrypt(struct aead_givcrypt_request *areq)
1418 {
1419         struct aead_request *req = &areq->areq;
1420         struct aead_edesc *edesc;
1421         struct crypto_aead *aead = crypto_aead_reqtfm(req);
1422         struct caam_ctx *ctx = crypto_aead_ctx(aead);
1423         struct device *jrdev = ctx->jrdev;
1424         u32 contig;
1425         u32 *desc;
1426         int ret = 0;
1427
1428         /* allocate extended descriptor */
1429         edesc = aead_giv_edesc_alloc(areq, DESC_JOB_IO_LEN *
1430                                      CAAM_CMD_SZ, &contig);
1431
1432         if (IS_ERR(edesc))
1433                 return PTR_ERR(edesc);
1434
1435 #ifdef DEBUG
1436         print_hex_dump(KERN_ERR, "giv src@"__stringify(__LINE__)": ",
1437                        DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src),
1438                        req->cryptlen, 1);
1439 #endif
1440
1441         /* Create and submit job descriptor*/
1442         init_aead_giv_job(ctx->sh_desc_givenc,
1443                           ctx->sh_desc_givenc_dma, edesc, req, contig);
1444 #ifdef DEBUG
1445         print_hex_dump(KERN_ERR, "aead jobdesc@"__stringify(__LINE__)": ",
1446                        DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc,
1447                        desc_bytes(edesc->hw_desc), 1);
1448 #endif
1449
1450         desc = edesc->hw_desc;
1451         ret = caam_jr_enqueue(jrdev, desc, aead_encrypt_done, req);
1452         if (!ret) {
1453                 ret = -EINPROGRESS;
1454         } else {
1455                 aead_unmap(jrdev, edesc, req);
1456                 kfree(edesc);
1457         }
1458
1459         return ret;
1460 }
1461
1462 /*
1463  * allocate and map the ablkcipher extended descriptor for ablkcipher
1464  */
1465 static struct ablkcipher_edesc *ablkcipher_edesc_alloc(struct ablkcipher_request
1466                                                        *req, int desc_bytes,
1467                                                        bool *iv_contig_out)
1468 {
1469         struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req);
1470         struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher);
1471         struct device *jrdev = ctx->jrdev;
1472         gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG |
1473                                           CRYPTO_TFM_REQ_MAY_SLEEP)) ?
1474                        GFP_KERNEL : GFP_ATOMIC;
1475         int src_nents, dst_nents = 0, sec4_sg_bytes;
1476         struct ablkcipher_edesc *edesc;
1477         dma_addr_t iv_dma = 0;
1478         bool iv_contig = false;
1479         int sgc;
1480         int ivsize = crypto_ablkcipher_ivsize(ablkcipher);
1481         bool src_chained = false, dst_chained = false;
1482         int sec4_sg_index;
1483
1484         src_nents = sg_count(req->src, req->nbytes, &src_chained);
1485
1486         if (req->dst != req->src)
1487                 dst_nents = sg_count(req->dst, req->nbytes, &dst_chained);
1488
1489         if (likely(req->src == req->dst)) {
1490                 sgc = dma_map_sg_chained(jrdev, req->src, src_nents ? : 1,
1491                                          DMA_BIDIRECTIONAL, src_chained);
1492         } else {
1493                 sgc = dma_map_sg_chained(jrdev, req->src, src_nents ? : 1,
1494                                          DMA_TO_DEVICE, src_chained);
1495                 sgc = dma_map_sg_chained(jrdev, req->dst, dst_nents ? : 1,
1496                                          DMA_FROM_DEVICE, dst_chained);
1497         }
1498
1499         /*
1500          * Check if iv can be contiguous with source and destination.
1501          * If so, include it. If not, create scatterlist.
1502          */
1503         iv_dma = dma_map_single(jrdev, req->info, ivsize, DMA_TO_DEVICE);
1504         if (!src_nents && iv_dma + ivsize == sg_dma_address(req->src))
1505                 iv_contig = true;
1506         else
1507                 src_nents = src_nents ? : 1;
1508         sec4_sg_bytes = ((iv_contig ? 0 : 1) + src_nents + dst_nents) *
1509                         sizeof(struct sec4_sg_entry);
1510
1511         /* allocate space for base edesc and hw desc commands, link tables */
1512         edesc = kmalloc(sizeof(struct ablkcipher_edesc) + desc_bytes +
1513                         sec4_sg_bytes, GFP_DMA | flags);
1514         if (!edesc) {
1515                 dev_err(jrdev, "could not allocate extended descriptor\n");
1516                 return ERR_PTR(-ENOMEM);
1517         }
1518
1519         edesc->src_nents = src_nents;
1520         edesc->src_chained = src_chained;
1521         edesc->dst_nents = dst_nents;
1522         edesc->dst_chained = dst_chained;
1523         edesc->sec4_sg_bytes = sec4_sg_bytes;
1524         edesc->sec4_sg = (void *)edesc + sizeof(struct ablkcipher_edesc) +
1525                          desc_bytes;
1526
1527         sec4_sg_index = 0;
1528         if (!iv_contig) {
1529                 dma_to_sec4_sg_one(edesc->sec4_sg, iv_dma, ivsize, 0);
1530                 sg_to_sec4_sg_last(req->src, src_nents,
1531                                    edesc->sec4_sg + 1, 0);
1532                 sec4_sg_index += 1 + src_nents;
1533         }
1534
1535         if (dst_nents) {
1536                 sg_to_sec4_sg_last(req->dst, dst_nents,
1537                         edesc->sec4_sg + sec4_sg_index, 0);
1538         }
1539
1540         edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg,
1541                                             sec4_sg_bytes, DMA_TO_DEVICE);
1542         edesc->iv_dma = iv_dma;
1543
1544 #ifdef DEBUG
1545         print_hex_dump(KERN_ERR, "ablkcipher sec4_sg@"__stringify(__LINE__)": ",
1546                        DUMP_PREFIX_ADDRESS, 16, 4, edesc->sec4_sg,
1547                        sec4_sg_bytes, 1);
1548 #endif
1549
1550         *iv_contig_out = iv_contig;
1551         return edesc;
1552 }
1553
1554 static int ablkcipher_encrypt(struct ablkcipher_request *req)
1555 {
1556         struct ablkcipher_edesc *edesc;
1557         struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req);
1558         struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher);
1559         struct device *jrdev = ctx->jrdev;
1560         bool iv_contig;
1561         u32 *desc;
1562         int ret = 0;
1563
1564         /* allocate extended descriptor */
1565         edesc = ablkcipher_edesc_alloc(req, DESC_JOB_IO_LEN *
1566                                        CAAM_CMD_SZ, &iv_contig);
1567         if (IS_ERR(edesc))
1568                 return PTR_ERR(edesc);
1569
1570         /* Create and submit job descriptor*/
1571         init_ablkcipher_job(ctx->sh_desc_enc,
1572                 ctx->sh_desc_enc_dma, edesc, req, iv_contig);
1573 #ifdef DEBUG
1574         print_hex_dump(KERN_ERR, "ablkcipher jobdesc@"__stringify(__LINE__)": ",
1575                        DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc,
1576                        desc_bytes(edesc->hw_desc), 1);
1577 #endif
1578         desc = edesc->hw_desc;
1579         ret = caam_jr_enqueue(jrdev, desc, ablkcipher_encrypt_done, req);
1580
1581         if (!ret) {
1582                 ret = -EINPROGRESS;
1583         } else {
1584                 ablkcipher_unmap(jrdev, edesc, req);
1585                 kfree(edesc);
1586         }
1587
1588         return ret;
1589 }
1590
1591 static int ablkcipher_decrypt(struct ablkcipher_request *req)
1592 {
1593         struct ablkcipher_edesc *edesc;
1594         struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req);
1595         struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher);
1596         struct device *jrdev = ctx->jrdev;
1597         bool iv_contig;
1598         u32 *desc;
1599         int ret = 0;
1600
1601         /* allocate extended descriptor */
1602         edesc = ablkcipher_edesc_alloc(req, DESC_JOB_IO_LEN *
1603                                        CAAM_CMD_SZ, &iv_contig);
1604         if (IS_ERR(edesc))
1605                 return PTR_ERR(edesc);
1606
1607         /* Create and submit job descriptor*/
1608         init_ablkcipher_job(ctx->sh_desc_dec,
1609                 ctx->sh_desc_dec_dma, edesc, req, iv_contig);
1610         desc = edesc->hw_desc;
1611 #ifdef DEBUG
1612         print_hex_dump(KERN_ERR, "ablkcipher jobdesc@"__stringify(__LINE__)": ",
1613                        DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc,
1614                        desc_bytes(edesc->hw_desc), 1);
1615 #endif
1616
1617         ret = caam_jr_enqueue(jrdev, desc, ablkcipher_decrypt_done, req);
1618         if (!ret) {
1619                 ret = -EINPROGRESS;
1620         } else {
1621                 ablkcipher_unmap(jrdev, edesc, req);
1622                 kfree(edesc);
1623         }
1624
1625         return ret;
1626 }
1627
1628 #define template_aead           template_u.aead
1629 #define template_ablkcipher     template_u.ablkcipher
1630 struct caam_alg_template {
1631         char name[CRYPTO_MAX_ALG_NAME];
1632         char driver_name[CRYPTO_MAX_ALG_NAME];
1633         unsigned int blocksize;
1634         u32 type;
1635         union {
1636                 struct ablkcipher_alg ablkcipher;
1637                 struct aead_alg aead;
1638                 struct blkcipher_alg blkcipher;
1639                 struct cipher_alg cipher;
1640                 struct compress_alg compress;
1641                 struct rng_alg rng;
1642         } template_u;
1643         u32 class1_alg_type;
1644         u32 class2_alg_type;
1645         u32 alg_op;
1646 };
1647
1648 static struct caam_alg_template driver_algs[] = {
1649         /* single-pass ipsec_esp descriptor */
1650         {
1651                 .name = "authenc(hmac(md5),cbc(aes))",
1652                 .driver_name = "authenc-hmac-md5-cbc-aes-caam",
1653                 .blocksize = AES_BLOCK_SIZE,
1654                 .type = CRYPTO_ALG_TYPE_AEAD,
1655                 .template_aead = {
1656                         .setkey = aead_setkey,
1657                         .setauthsize = aead_setauthsize,
1658                         .encrypt = aead_encrypt,
1659                         .decrypt = aead_decrypt,
1660                         .givencrypt = aead_givencrypt,
1661                         .geniv = "<built-in>",
1662                         .ivsize = AES_BLOCK_SIZE,
1663                         .maxauthsize = MD5_DIGEST_SIZE,
1664                         },
1665                 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
1666                 .class2_alg_type = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC_PRECOMP,
1667                 .alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC,
1668         },
1669         {
1670                 .name = "authenc(hmac(sha1),cbc(aes))",
1671                 .driver_name = "authenc-hmac-sha1-cbc-aes-caam",
1672                 .blocksize = AES_BLOCK_SIZE,
1673                 .type = CRYPTO_ALG_TYPE_AEAD,
1674                 .template_aead = {
1675                         .setkey = aead_setkey,
1676                         .setauthsize = aead_setauthsize,
1677                         .encrypt = aead_encrypt,
1678                         .decrypt = aead_decrypt,
1679                         .givencrypt = aead_givencrypt,
1680                         .geniv = "<built-in>",
1681                         .ivsize = AES_BLOCK_SIZE,
1682                         .maxauthsize = SHA1_DIGEST_SIZE,
1683                         },
1684                 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
1685                 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC_PRECOMP,
1686                 .alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC,
1687         },
1688         {
1689                 .name = "authenc(hmac(sha224),cbc(aes))",
1690                 .driver_name = "authenc-hmac-sha224-cbc-aes-caam",
1691                 .blocksize = AES_BLOCK_SIZE,
1692                 .type = CRYPTO_ALG_TYPE_AEAD,
1693                 .template_aead = {
1694                         .setkey = aead_setkey,
1695                         .setauthsize = aead_setauthsize,
1696                         .encrypt = aead_encrypt,
1697                         .decrypt = aead_decrypt,
1698                         .givencrypt = aead_givencrypt,
1699                         .geniv = "<built-in>",
1700                         .ivsize = AES_BLOCK_SIZE,
1701                         .maxauthsize = SHA224_DIGEST_SIZE,
1702                         },
1703                 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
1704                 .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
1705                                    OP_ALG_AAI_HMAC_PRECOMP,
1706                 .alg_op = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC,
1707         },
1708         {
1709                 .name = "authenc(hmac(sha256),cbc(aes))",
1710                 .driver_name = "authenc-hmac-sha256-cbc-aes-caam",
1711                 .blocksize = AES_BLOCK_SIZE,
1712                 .type = CRYPTO_ALG_TYPE_AEAD,
1713                 .template_aead = {
1714                         .setkey = aead_setkey,
1715                         .setauthsize = aead_setauthsize,
1716                         .encrypt = aead_encrypt,
1717                         .decrypt = aead_decrypt,
1718                         .givencrypt = aead_givencrypt,
1719                         .geniv = "<built-in>",
1720                         .ivsize = AES_BLOCK_SIZE,
1721                         .maxauthsize = SHA256_DIGEST_SIZE,
1722                         },
1723                 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
1724                 .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
1725                                    OP_ALG_AAI_HMAC_PRECOMP,
1726                 .alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC,
1727         },
1728         {
1729                 .name = "authenc(hmac(sha384),cbc(aes))",
1730                 .driver_name = "authenc-hmac-sha384-cbc-aes-caam",
1731                 .blocksize = AES_BLOCK_SIZE,
1732                 .type = CRYPTO_ALG_TYPE_AEAD,
1733                 .template_aead = {
1734                         .setkey = aead_setkey,
1735                         .setauthsize = aead_setauthsize,
1736                         .encrypt = aead_encrypt,
1737                         .decrypt = aead_decrypt,
1738                         .givencrypt = aead_givencrypt,
1739                         .geniv = "<built-in>",
1740                         .ivsize = AES_BLOCK_SIZE,
1741                         .maxauthsize = SHA384_DIGEST_SIZE,
1742                         },
1743                 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
1744                 .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
1745                                    OP_ALG_AAI_HMAC_PRECOMP,
1746                 .alg_op = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC,
1747         },
1748
1749         {
1750                 .name = "authenc(hmac(sha512),cbc(aes))",
1751                 .driver_name = "authenc-hmac-sha512-cbc-aes-caam",
1752                 .blocksize = AES_BLOCK_SIZE,
1753                 .type = CRYPTO_ALG_TYPE_AEAD,
1754                 .template_aead = {
1755                         .setkey = aead_setkey,
1756                         .setauthsize = aead_setauthsize,
1757                         .encrypt = aead_encrypt,
1758                         .decrypt = aead_decrypt,
1759                         .givencrypt = aead_givencrypt,
1760                         .geniv = "<built-in>",
1761                         .ivsize = AES_BLOCK_SIZE,
1762                         .maxauthsize = SHA512_DIGEST_SIZE,
1763                         },
1764                 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
1765                 .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
1766                                    OP_ALG_AAI_HMAC_PRECOMP,
1767                 .alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC,
1768         },
1769         {
1770                 .name = "authenc(hmac(md5),cbc(des3_ede))",
1771                 .driver_name = "authenc-hmac-md5-cbc-des3_ede-caam",
1772                 .blocksize = DES3_EDE_BLOCK_SIZE,
1773                 .type = CRYPTO_ALG_TYPE_AEAD,
1774                 .template_aead = {
1775                         .setkey = aead_setkey,
1776                         .setauthsize = aead_setauthsize,
1777                         .encrypt = aead_encrypt,
1778                         .decrypt = aead_decrypt,
1779                         .givencrypt = aead_givencrypt,
1780                         .geniv = "<built-in>",
1781                         .ivsize = DES3_EDE_BLOCK_SIZE,
1782                         .maxauthsize = MD5_DIGEST_SIZE,
1783                         },
1784                 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
1785                 .class2_alg_type = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC_PRECOMP,
1786                 .alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC,
1787         },
1788         {
1789                 .name = "authenc(hmac(sha1),cbc(des3_ede))",
1790                 .driver_name = "authenc-hmac-sha1-cbc-des3_ede-caam",
1791                 .blocksize = DES3_EDE_BLOCK_SIZE,
1792                 .type = CRYPTO_ALG_TYPE_AEAD,
1793                 .template_aead = {
1794                         .setkey = aead_setkey,
1795                         .setauthsize = aead_setauthsize,
1796                         .encrypt = aead_encrypt,
1797                         .decrypt = aead_decrypt,
1798                         .givencrypt = aead_givencrypt,
1799                         .geniv = "<built-in>",
1800                         .ivsize = DES3_EDE_BLOCK_SIZE,
1801                         .maxauthsize = SHA1_DIGEST_SIZE,
1802                         },
1803                 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
1804                 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC_PRECOMP,
1805                 .alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC,
1806         },
1807         {
1808                 .name = "authenc(hmac(sha224),cbc(des3_ede))",
1809                 .driver_name = "authenc-hmac-sha224-cbc-des3_ede-caam",
1810                 .blocksize = DES3_EDE_BLOCK_SIZE,
1811                 .type = CRYPTO_ALG_TYPE_AEAD,
1812                 .template_aead = {
1813                         .setkey = aead_setkey,
1814                         .setauthsize = aead_setauthsize,
1815                         .encrypt = aead_encrypt,
1816                         .decrypt = aead_decrypt,
1817                         .givencrypt = aead_givencrypt,
1818                         .geniv = "<built-in>",
1819                         .ivsize = DES3_EDE_BLOCK_SIZE,
1820                         .maxauthsize = SHA224_DIGEST_SIZE,
1821                         },
1822                 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
1823                 .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
1824                                    OP_ALG_AAI_HMAC_PRECOMP,
1825                 .alg_op = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC,
1826         },
1827         {
1828                 .name = "authenc(hmac(sha256),cbc(des3_ede))",
1829                 .driver_name = "authenc-hmac-sha256-cbc-des3_ede-caam",
1830                 .blocksize = DES3_EDE_BLOCK_SIZE,
1831                 .type = CRYPTO_ALG_TYPE_AEAD,
1832                 .template_aead = {
1833                         .setkey = aead_setkey,
1834                         .setauthsize = aead_setauthsize,
1835                         .encrypt = aead_encrypt,
1836                         .decrypt = aead_decrypt,
1837                         .givencrypt = aead_givencrypt,
1838                         .geniv = "<built-in>",
1839                         .ivsize = DES3_EDE_BLOCK_SIZE,
1840                         .maxauthsize = SHA256_DIGEST_SIZE,
1841                         },
1842                 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
1843                 .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
1844                                    OP_ALG_AAI_HMAC_PRECOMP,
1845                 .alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC,
1846         },
1847         {
1848                 .name = "authenc(hmac(sha384),cbc(des3_ede))",
1849                 .driver_name = "authenc-hmac-sha384-cbc-des3_ede-caam",
1850                 .blocksize = DES3_EDE_BLOCK_SIZE,
1851                 .type = CRYPTO_ALG_TYPE_AEAD,
1852                 .template_aead = {
1853                         .setkey = aead_setkey,
1854                         .setauthsize = aead_setauthsize,
1855                         .encrypt = aead_encrypt,
1856                         .decrypt = aead_decrypt,
1857                         .givencrypt = aead_givencrypt,
1858                         .geniv = "<built-in>",
1859                         .ivsize = DES3_EDE_BLOCK_SIZE,
1860                         .maxauthsize = SHA384_DIGEST_SIZE,
1861                         },
1862                 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
1863                 .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
1864                                    OP_ALG_AAI_HMAC_PRECOMP,
1865                 .alg_op = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC,
1866         },
1867         {
1868                 .name = "authenc(hmac(sha512),cbc(des3_ede))",
1869                 .driver_name = "authenc-hmac-sha512-cbc-des3_ede-caam",
1870                 .blocksize = DES3_EDE_BLOCK_SIZE,
1871                 .type = CRYPTO_ALG_TYPE_AEAD,
1872                 .template_aead = {
1873                         .setkey = aead_setkey,
1874                         .setauthsize = aead_setauthsize,
1875                         .encrypt = aead_encrypt,
1876                         .decrypt = aead_decrypt,
1877                         .givencrypt = aead_givencrypt,
1878                         .geniv = "<built-in>",
1879                         .ivsize = DES3_EDE_BLOCK_SIZE,
1880                         .maxauthsize = SHA512_DIGEST_SIZE,
1881                         },
1882                 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
1883                 .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
1884                                    OP_ALG_AAI_HMAC_PRECOMP,
1885                 .alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC,
1886         },
1887         {
1888                 .name = "authenc(hmac(md5),cbc(des))",
1889                 .driver_name = "authenc-hmac-md5-cbc-des-caam",
1890                 .blocksize = DES_BLOCK_SIZE,
1891                 .type = CRYPTO_ALG_TYPE_AEAD,
1892                 .template_aead = {
1893                         .setkey = aead_setkey,
1894                         .setauthsize = aead_setauthsize,
1895                         .encrypt = aead_encrypt,
1896                         .decrypt = aead_decrypt,
1897                         .givencrypt = aead_givencrypt,
1898                         .geniv = "<built-in>",
1899                         .ivsize = DES_BLOCK_SIZE,
1900                         .maxauthsize = MD5_DIGEST_SIZE,
1901                         },
1902                 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
1903                 .class2_alg_type = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC_PRECOMP,
1904                 .alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC,
1905         },
1906         {
1907                 .name = "authenc(hmac(sha1),cbc(des))",
1908                 .driver_name = "authenc-hmac-sha1-cbc-des-caam",
1909                 .blocksize = DES_BLOCK_SIZE,
1910                 .type = CRYPTO_ALG_TYPE_AEAD,
1911                 .template_aead = {
1912                         .setkey = aead_setkey,
1913                         .setauthsize = aead_setauthsize,
1914                         .encrypt = aead_encrypt,
1915                         .decrypt = aead_decrypt,
1916                         .givencrypt = aead_givencrypt,
1917                         .geniv = "<built-in>",
1918                         .ivsize = DES_BLOCK_SIZE,
1919                         .maxauthsize = SHA1_DIGEST_SIZE,
1920                         },
1921                 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
1922                 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC_PRECOMP,
1923                 .alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC,
1924         },
1925         {
1926                 .name = "authenc(hmac(sha224),cbc(des))",
1927                 .driver_name = "authenc-hmac-sha224-cbc-des-caam",
1928                 .blocksize = DES_BLOCK_SIZE,
1929                 .type = CRYPTO_ALG_TYPE_AEAD,
1930                 .template_aead = {
1931                         .setkey = aead_setkey,
1932                         .setauthsize = aead_setauthsize,
1933                         .encrypt = aead_encrypt,
1934                         .decrypt = aead_decrypt,
1935                         .givencrypt = aead_givencrypt,
1936                         .geniv = "<built-in>",
1937                         .ivsize = DES_BLOCK_SIZE,
1938                         .maxauthsize = SHA224_DIGEST_SIZE,
1939                         },
1940                 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
1941                 .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
1942                                    OP_ALG_AAI_HMAC_PRECOMP,
1943                 .alg_op = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC,
1944         },
1945         {
1946                 .name = "authenc(hmac(sha256),cbc(des))",
1947                 .driver_name = "authenc-hmac-sha256-cbc-des-caam",
1948                 .blocksize = DES_BLOCK_SIZE,
1949                 .type = CRYPTO_ALG_TYPE_AEAD,
1950                 .template_aead = {
1951                         .setkey = aead_setkey,
1952                         .setauthsize = aead_setauthsize,
1953                         .encrypt = aead_encrypt,
1954                         .decrypt = aead_decrypt,
1955                         .givencrypt = aead_givencrypt,
1956                         .geniv = "<built-in>",
1957                         .ivsize = DES_BLOCK_SIZE,
1958                         .maxauthsize = SHA256_DIGEST_SIZE,
1959                         },
1960                 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
1961                 .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
1962                                    OP_ALG_AAI_HMAC_PRECOMP,
1963                 .alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC,
1964         },
1965         {
1966                 .name = "authenc(hmac(sha384),cbc(des))",
1967                 .driver_name = "authenc-hmac-sha384-cbc-des-caam",
1968                 .blocksize = DES_BLOCK_SIZE,
1969                 .type = CRYPTO_ALG_TYPE_AEAD,
1970                 .template_aead = {
1971                         .setkey = aead_setkey,
1972                         .setauthsize = aead_setauthsize,
1973                         .encrypt = aead_encrypt,
1974                         .decrypt = aead_decrypt,
1975                         .givencrypt = aead_givencrypt,
1976                         .geniv = "<built-in>",
1977                         .ivsize = DES_BLOCK_SIZE,
1978                         .maxauthsize = SHA384_DIGEST_SIZE,
1979                         },
1980                 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
1981                 .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
1982                                    OP_ALG_AAI_HMAC_PRECOMP,
1983                 .alg_op = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC,
1984         },
1985         {
1986                 .name = "authenc(hmac(sha512),cbc(des))",
1987                 .driver_name = "authenc-hmac-sha512-cbc-des-caam",
1988                 .blocksize = DES_BLOCK_SIZE,
1989                 .type = CRYPTO_ALG_TYPE_AEAD,
1990                 .template_aead = {
1991                         .setkey = aead_setkey,
1992                         .setauthsize = aead_setauthsize,
1993                         .encrypt = aead_encrypt,
1994                         .decrypt = aead_decrypt,
1995                         .givencrypt = aead_givencrypt,
1996                         .geniv = "<built-in>",
1997                         .ivsize = DES_BLOCK_SIZE,
1998                         .maxauthsize = SHA512_DIGEST_SIZE,
1999                         },
2000                 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2001                 .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
2002                                    OP_ALG_AAI_HMAC_PRECOMP,
2003                 .alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC,
2004         },
2005         /* ablkcipher descriptor */
2006         {
2007                 .name = "cbc(aes)",
2008                 .driver_name = "cbc-aes-caam",
2009                 .blocksize = AES_BLOCK_SIZE,
2010                 .type = CRYPTO_ALG_TYPE_ABLKCIPHER,
2011                 .template_ablkcipher = {
2012                         .setkey = ablkcipher_setkey,
2013                         .encrypt = ablkcipher_encrypt,
2014                         .decrypt = ablkcipher_decrypt,
2015                         .geniv = "eseqiv",
2016                         .min_keysize = AES_MIN_KEY_SIZE,
2017                         .max_keysize = AES_MAX_KEY_SIZE,
2018                         .ivsize = AES_BLOCK_SIZE,
2019                         },
2020                 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
2021         },
2022         {
2023                 .name = "cbc(des3_ede)",
2024                 .driver_name = "cbc-3des-caam",
2025                 .blocksize = DES3_EDE_BLOCK_SIZE,
2026                 .type = CRYPTO_ALG_TYPE_ABLKCIPHER,
2027                 .template_ablkcipher = {
2028                         .setkey = ablkcipher_setkey,
2029                         .encrypt = ablkcipher_encrypt,
2030                         .decrypt = ablkcipher_decrypt,
2031                         .geniv = "eseqiv",
2032                         .min_keysize = DES3_EDE_KEY_SIZE,
2033                         .max_keysize = DES3_EDE_KEY_SIZE,
2034                         .ivsize = DES3_EDE_BLOCK_SIZE,
2035                         },
2036                 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2037         },
2038         {
2039                 .name = "cbc(des)",
2040                 .driver_name = "cbc-des-caam",
2041                 .blocksize = DES_BLOCK_SIZE,
2042                 .type = CRYPTO_ALG_TYPE_ABLKCIPHER,
2043                 .template_ablkcipher = {
2044                         .setkey = ablkcipher_setkey,
2045                         .encrypt = ablkcipher_encrypt,
2046                         .decrypt = ablkcipher_decrypt,
2047                         .geniv = "eseqiv",
2048                         .min_keysize = DES_KEY_SIZE,
2049                         .max_keysize = DES_KEY_SIZE,
2050                         .ivsize = DES_BLOCK_SIZE,
2051                         },
2052                 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2053         }
2054 };
2055
2056 struct caam_crypto_alg {
2057         struct list_head entry;
2058         int class1_alg_type;
2059         int class2_alg_type;
2060         int alg_op;
2061         struct crypto_alg crypto_alg;
2062 };
2063
2064 static int caam_cra_init(struct crypto_tfm *tfm)
2065 {
2066         struct crypto_alg *alg = tfm->__crt_alg;
2067         struct caam_crypto_alg *caam_alg =
2068                  container_of(alg, struct caam_crypto_alg, crypto_alg);
2069         struct caam_ctx *ctx = crypto_tfm_ctx(tfm);
2070
2071         ctx->jrdev = caam_jr_alloc();
2072         if (IS_ERR(ctx->jrdev)) {
2073                 pr_err("Job Ring Device allocation for transform failed\n");
2074                 return PTR_ERR(ctx->jrdev);
2075         }
2076
2077         /* copy descriptor header template value */
2078         ctx->class1_alg_type = OP_TYPE_CLASS1_ALG | caam_alg->class1_alg_type;
2079         ctx->class2_alg_type = OP_TYPE_CLASS2_ALG | caam_alg->class2_alg_type;
2080         ctx->alg_op = OP_TYPE_CLASS2_ALG | caam_alg->alg_op;
2081
2082         return 0;
2083 }
2084
2085 static void caam_cra_exit(struct crypto_tfm *tfm)
2086 {
2087         struct caam_ctx *ctx = crypto_tfm_ctx(tfm);
2088
2089         if (ctx->sh_desc_enc_dma &&
2090             !dma_mapping_error(ctx->jrdev, ctx->sh_desc_enc_dma))
2091                 dma_unmap_single(ctx->jrdev, ctx->sh_desc_enc_dma,
2092                                  desc_bytes(ctx->sh_desc_enc), DMA_TO_DEVICE);
2093         if (ctx->sh_desc_dec_dma &&
2094             !dma_mapping_error(ctx->jrdev, ctx->sh_desc_dec_dma))
2095                 dma_unmap_single(ctx->jrdev, ctx->sh_desc_dec_dma,
2096                                  desc_bytes(ctx->sh_desc_dec), DMA_TO_DEVICE);
2097         if (ctx->sh_desc_givenc_dma &&
2098             !dma_mapping_error(ctx->jrdev, ctx->sh_desc_givenc_dma))
2099                 dma_unmap_single(ctx->jrdev, ctx->sh_desc_givenc_dma,
2100                                  desc_bytes(ctx->sh_desc_givenc),
2101                                  DMA_TO_DEVICE);
2102
2103         caam_jr_free(ctx->jrdev);
2104 }
2105
2106 static void __exit caam_algapi_exit(void)
2107 {
2108
2109         struct caam_crypto_alg *t_alg, *n;
2110
2111         if (!alg_list.next)
2112                 return;
2113
2114         list_for_each_entry_safe(t_alg, n, &alg_list, entry) {
2115                 crypto_unregister_alg(&t_alg->crypto_alg);
2116                 list_del(&t_alg->entry);
2117                 kfree(t_alg);
2118         }
2119 }
2120
2121 static struct caam_crypto_alg *caam_alg_alloc(struct caam_alg_template
2122                                               *template)
2123 {
2124         struct caam_crypto_alg *t_alg;
2125         struct crypto_alg *alg;
2126
2127         t_alg = kzalloc(sizeof(struct caam_crypto_alg), GFP_KERNEL);
2128         if (!t_alg) {
2129                 pr_err("failed to allocate t_alg\n");
2130                 return ERR_PTR(-ENOMEM);
2131         }
2132
2133         alg = &t_alg->crypto_alg;
2134
2135         snprintf(alg->cra_name, CRYPTO_MAX_ALG_NAME, "%s", template->name);
2136         snprintf(alg->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s",
2137                  template->driver_name);
2138         alg->cra_module = THIS_MODULE;
2139         alg->cra_init = caam_cra_init;
2140         alg->cra_exit = caam_cra_exit;
2141         alg->cra_priority = CAAM_CRA_PRIORITY;
2142         alg->cra_blocksize = template->blocksize;
2143         alg->cra_alignmask = 0;
2144         alg->cra_ctxsize = sizeof(struct caam_ctx);
2145         alg->cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY |
2146                          template->type;
2147         switch (template->type) {
2148         case CRYPTO_ALG_TYPE_ABLKCIPHER:
2149                 alg->cra_type = &crypto_ablkcipher_type;
2150                 alg->cra_ablkcipher = template->template_ablkcipher;
2151                 break;
2152         case CRYPTO_ALG_TYPE_AEAD:
2153                 alg->cra_type = &crypto_aead_type;
2154                 alg->cra_aead = template->template_aead;
2155                 break;
2156         }
2157
2158         t_alg->class1_alg_type = template->class1_alg_type;
2159         t_alg->class2_alg_type = template->class2_alg_type;
2160         t_alg->alg_op = template->alg_op;
2161
2162         return t_alg;
2163 }
2164
2165 static int __init caam_algapi_init(void)
2166 {
2167         int i = 0, err = 0;
2168
2169         INIT_LIST_HEAD(&alg_list);
2170
2171         /* register crypto algorithms the device supports */
2172         for (i = 0; i < ARRAY_SIZE(driver_algs); i++) {
2173                 /* TODO: check if h/w supports alg */
2174                 struct caam_crypto_alg *t_alg;
2175
2176                 t_alg = caam_alg_alloc(&driver_algs[i]);
2177                 if (IS_ERR(t_alg)) {
2178                         err = PTR_ERR(t_alg);
2179                         pr_warn("%s alg allocation failed\n",
2180                                 driver_algs[i].driver_name);
2181                         continue;
2182                 }
2183
2184                 err = crypto_register_alg(&t_alg->crypto_alg);
2185                 if (err) {
2186                         pr_warn("%s alg registration failed\n",
2187                                 t_alg->crypto_alg.cra_driver_name);
2188                         kfree(t_alg);
2189                 } else
2190                         list_add_tail(&t_alg->entry, &alg_list);
2191         }
2192         if (!list_empty(&alg_list))
2193                 pr_info("caam algorithms registered in /proc/crypto\n");
2194
2195         return err;
2196 }
2197
2198 module_init(caam_algapi_init);
2199 module_exit(caam_algapi_exit);
2200
2201 MODULE_LICENSE("GPL");
2202 MODULE_DESCRIPTION("FSL CAAM support for crypto API");
2203 MODULE_AUTHOR("Freescale Semiconductor - NMG/STC");