drivers/crypto/caam/caamalg.c

   1 /*
   2  * caam - Freescale FSL CAAM support for crypto API
   3  *
   4  * Copyright 2008-2011 Freescale Semiconductor, Inc.
   5  *
   6  * Based on talitos crypto API driver.
   7  *
   8  * relationship of job descriptors to shared descriptors (SteveC Dec 10 2008):
   9  *
  10  * ---------------                     ---------------
  11  * | JobDesc #1  |-------------------->|  ShareDesc  |
  12  * | *(packet 1) |                     |   (PDB)     |
  13  * ---------------      |------------->|  (hashKey)  |
  14  *       .              |              | (cipherKey) |
  15  *       .              |    |-------->| (operation) |
  16  * ---------------      |    |         ---------------
  17  * | JobDesc #2  |------|    |
  18  * | *(packet 2) |           |
  19  * ---------------           |
  20  *       .                   |
  21  *       .                   |
  22  * ---------------           |
  23  * | JobDesc #3  |------------
  24  * | *(packet 3) |
  25  * ---------------
  26  *
  27  * The SharedDesc never changes for a connection unless rekeyed, but
  28  * each packet will likely be in a different place. So all we need
  29  * to know to process the packet is where the input is, where the
  30  * output goes, and what context we want to process with. Context is
  31  * in the SharedDesc, packet references in the JobDesc.
  32  *
  33  * So, a job desc looks like:
  34  *
  35  * ---------------------
  36  * | Header            |
  37  * | ShareDesc Pointer |
  38  * | SEQ_OUT_PTR       |
  39  * | (output buffer)   |
  40  * | (output length)   |
  41  * | SEQ_IN_PTR        |
  42  * | (input buffer)    |
  43  * | (input length)    |
  44  * ---------------------
  45  */
  46
  47 #include "compat.h"
  48
  49 #include "regs.h"
  50 #include "intern.h"
  51 #include "desc_constr.h"
  52 #include "jr.h"
  53 #include "error.h"
  54 #include "sg_sw_sec4.h"
  55 #include "key_gen.h"
  56
  57 /*
  58  * crypto alg
  59  */
  60 #define CAAM_CRA_PRIORITY               3000
  61 /* max key is sum of AES_MAX_KEY_SIZE, max split key size */
  62 #define CAAM_MAX_KEY_SIZE               (AES_MAX_KEY_SIZE + \
  63                                          SHA512_DIGEST_SIZE * 2)
  64 /* max IV is max of AES_BLOCK_SIZE, DES3_EDE_BLOCK_SIZE */
  65 #define CAAM_MAX_IV_LENGTH              16
  66
  67 /* length of descriptors text */
  68 #define DESC_AEAD_BASE                  (4 * CAAM_CMD_SZ)
  69 #define DESC_AEAD_ENC_LEN               (DESC_AEAD_BASE + 15 * CAAM_CMD_SZ)
  70 #define DESC_AEAD_DEC_LEN               (DESC_AEAD_BASE + 18 * CAAM_CMD_SZ)
  71 #define DESC_AEAD_GIVENC_LEN            (DESC_AEAD_ENC_LEN + 7 * CAAM_CMD_SZ)
  72
  73 #define DESC_AEAD_NULL_BASE             (3 * CAAM_CMD_SZ)
  74 #define DESC_AEAD_NULL_ENC_LEN          (DESC_AEAD_NULL_BASE + 14 * CAAM_CMD_SZ)
  75 #define DESC_AEAD_NULL_DEC_LEN          (DESC_AEAD_NULL_BASE + 17 * CAAM_CMD_SZ)
  76
  77 #define DESC_ABLKCIPHER_BASE            (3 * CAAM_CMD_SZ)
  78 #define DESC_ABLKCIPHER_ENC_LEN         (DESC_ABLKCIPHER_BASE + \
  79                                          20 * CAAM_CMD_SZ)
  80 #define DESC_ABLKCIPHER_DEC_LEN         (DESC_ABLKCIPHER_BASE + \
  81                                          15 * CAAM_CMD_SZ)
  82
  83 #define DESC_MAX_USED_BYTES             (DESC_AEAD_GIVENC_LEN + \
  84                                          CAAM_MAX_KEY_SIZE)
  85 #define DESC_MAX_USED_LEN               (DESC_MAX_USED_BYTES / CAAM_CMD_SZ)
  86
  87 #ifdef DEBUG
  88 /* for print_hex_dumps with line references */
  89 #define debug(format, arg...) printk(format, arg)
  90 #else
  91 #define debug(format, arg...)
  92 #endif
  93 static struct list_head alg_list;
  94
  95 /* Set DK bit in class 1 operation if shared */
  96 static inline void append_dec_op1(u32 *desc, u32 type)
  97 {
  98         u32 *jump_cmd, *uncond_jump_cmd;
  99
 100         jump_cmd = append_jump(desc, JUMP_TEST_ALL | JUMP_COND_SHRD);
 101         append_operation(desc, type | OP_ALG_AS_INITFINAL |
 102                          OP_ALG_DECRYPT);
 103         uncond_jump_cmd = append_jump(desc, JUMP_TEST_ALL);
 104         set_jump_tgt_here(desc, jump_cmd);
 105         append_operation(desc, type | OP_ALG_AS_INITFINAL |
 106                          OP_ALG_DECRYPT | OP_ALG_AAI_DK);
 107         set_jump_tgt_here(desc, uncond_jump_cmd);
 108 }
 109
 110 /*
 111  * For aead functions, read payload and write payload,
 112  * both of which are specified in req->src and req->dst
 113  */
 114 static inline void aead_append_src_dst(u32 *desc, u32 msg_type)
 115 {
 116         append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | KEY_VLF);
 117         append_seq_fifo_load(desc, 0, FIFOLD_CLASS_BOTH |
 118                              KEY_VLF | msg_type | FIFOLD_TYPE_LASTBOTH);
 119 }
 120
 121 /*
 122  * For aead encrypt and decrypt, read iv for both classes
 123  */
 124 static inline void aead_append_ld_iv(u32 *desc, int ivsize)
 125 {
 126         append_cmd(desc, CMD_SEQ_LOAD | LDST_SRCDST_BYTE_CONTEXT |
 127                    LDST_CLASS_1_CCB | ivsize);
 128         append_move(desc, MOVE_SRC_CLASS1CTX | MOVE_DEST_CLASS2INFIFO | ivsize);
 129 }
 130
 131 /*
 132  * For ablkcipher encrypt and decrypt, read from req->src and
 133  * write to req->dst
 134  */
 135 static inline void ablkcipher_append_src_dst(u32 *desc)
 136 {
 137         append_math_add(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
 138         append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
 139         append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 |
 140                              KEY_VLF | FIFOLD_TYPE_MSG | FIFOLD_TYPE_LAST1);
 141         append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | KEY_VLF);
 142 }
 143
 144 /*
 145  * If all data, including src (with assoc and iv) or dst (with iv only) are
 146  * contiguous
 147  */
 148 #define GIV_SRC_CONTIG          1
 149 #define GIV_DST_CONTIG          (1 << 1)
 150
 151 /*
 152  * per-session context
 153  */
 154 struct caam_ctx {
 155         struct device *jrdev;
 156         u32 sh_desc_enc[DESC_MAX_USED_LEN];
 157         u32 sh_desc_dec[DESC_MAX_USED_LEN];
 158         u32 sh_desc_givenc[DESC_MAX_USED_LEN];
 159         dma_addr_t sh_desc_enc_dma;
 160         dma_addr_t sh_desc_dec_dma;
 161         dma_addr_t sh_desc_givenc_dma;
 162         u32 class1_alg_type;
 163         u32 class2_alg_type;
 164         u32 alg_op;
 165         u8 key[CAAM_MAX_KEY_SIZE];
 166         dma_addr_t key_dma;
 167         unsigned int enckeylen;
 168         unsigned int split_key_len;
 169         unsigned int split_key_pad_len;
 170         unsigned int authsize;
 171 };
 172
 173 static void append_key_aead(u32 *desc, struct caam_ctx *ctx,
 174                             int keys_fit_inline)
 175 {
 176         if (keys_fit_inline) {
 177                 append_key_as_imm(desc, ctx->key, ctx->split_key_pad_len,
 178                                   ctx->split_key_len, CLASS_2 |
 179                                   KEY_DEST_MDHA_SPLIT | KEY_ENC);
 180                 append_key_as_imm(desc, (void *)ctx->key +
 181                                   ctx->split_key_pad_len, ctx->enckeylen,
 182                                   ctx->enckeylen, CLASS_1 | KEY_DEST_CLASS_REG);
 183         } else {
 184                 append_key(desc, ctx->key_dma, ctx->split_key_len, CLASS_2 |
 185                            KEY_DEST_MDHA_SPLIT | KEY_ENC);
 186                 append_key(desc, ctx->key_dma + ctx->split_key_pad_len,
 187                            ctx->enckeylen, CLASS_1 | KEY_DEST_CLASS_REG);
 188         }
 189 }
 190
 191 static void init_sh_desc_key_aead(u32 *desc, struct caam_ctx *ctx,
 192                                   int keys_fit_inline)
 193 {
 194         u32 *key_jump_cmd;
 195
 196         init_sh_desc(desc, HDR_SHARE_SERIAL);
 197
 198         /* Skip if already shared */
 199         key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
 200                                    JUMP_COND_SHRD);
 201
 202         append_key_aead(desc, ctx, keys_fit_inline);
 203
 204         set_jump_tgt_here(desc, key_jump_cmd);
 205 }
 206
 207 static int aead_null_set_sh_desc(struct crypto_aead *aead)
 208 {
 209         struct aead_tfm *tfm = &aead->base.crt_aead;
 210         struct caam_ctx *ctx = crypto_aead_ctx(aead);
 211         struct device *jrdev = ctx->jrdev;
 212         bool keys_fit_inline = false;
 213         u32 *key_jump_cmd, *jump_cmd, *read_move_cmd, *write_move_cmd;
 214         u32 *desc;
 215
 216         /*
 217          * Job Descriptor and Shared Descriptors
 218          * must all fit into the 64-word Descriptor h/w Buffer
 219          */
 220         if (DESC_AEAD_NULL_ENC_LEN + DESC_JOB_IO_LEN +
 221             ctx->split_key_pad_len <= CAAM_DESC_BYTES_MAX)
 222                 keys_fit_inline = true;
 223
 224         /* aead_encrypt shared descriptor */
 225         desc = ctx->sh_desc_enc;
 226
 227         init_sh_desc(desc, HDR_SHARE_SERIAL);
 228
 229         /* Skip if already shared */
 230         key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
 231                                    JUMP_COND_SHRD);
 232         if (keys_fit_inline)
 233                 append_key_as_imm(desc, ctx->key, ctx->split_key_pad_len,
 234                                   ctx->split_key_len, CLASS_2 |
 235                                   KEY_DEST_MDHA_SPLIT | KEY_ENC);
 236         else
 237                 append_key(desc, ctx->key_dma, ctx->split_key_len, CLASS_2 |
 238                            KEY_DEST_MDHA_SPLIT | KEY_ENC);
 239         set_jump_tgt_here(desc, key_jump_cmd);
 240
 241         /* cryptlen = seqoutlen - authsize */
 242         append_math_sub_imm_u32(desc, REG3, SEQOUTLEN, IMM, ctx->authsize);
 243
 244         /*
 245          * NULL encryption; IV is zero
 246          * assoclen = (assoclen + cryptlen) - cryptlen
 247          */
 248         append_math_sub(desc, VARSEQINLEN, SEQINLEN, REG3, CAAM_CMD_SZ);
 249
 250         /* read assoc before reading payload */
 251         append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG |
 252                              KEY_VLF);
 253
 254         /* Prepare to read and write cryptlen bytes */
 255         append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ);
 256         append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
 257
 258         /*
 259          * MOVE_LEN opcode is not available in all SEC HW revisions,
 260          * thus need to do some magic, i.e. self-patch the descriptor
 261          * buffer.
 262          */
 263         read_move_cmd = append_move(desc, MOVE_SRC_DESCBUF |
 264                                     MOVE_DEST_MATH3 |
 265                                     (0x6 << MOVE_LEN_SHIFT));
 266         write_move_cmd = append_move(desc, MOVE_SRC_MATH3 |
 267                                      MOVE_DEST_DESCBUF |
 268                                      MOVE_WAITCOMP |
 269                                      (0x8 << MOVE_LEN_SHIFT));
 270
 271         /* Class 2 operation */
 272         append_operation(desc, ctx->class2_alg_type |
 273                          OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT);
 274
 275         /* Read and write cryptlen bytes */
 276         aead_append_src_dst(desc, FIFOLD_TYPE_MSG | FIFOLD_TYPE_FLUSH1);
 277
 278         set_move_tgt_here(desc, read_move_cmd);
 279         set_move_tgt_here(desc, write_move_cmd);
 280         append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO);
 281         append_move(desc, MOVE_SRC_INFIFO_CL | MOVE_DEST_OUTFIFO |
 282                     MOVE_AUX_LS);
 283
 284         /* Write ICV */
 285         append_seq_store(desc, ctx->authsize, LDST_CLASS_2_CCB |
 286                          LDST_SRCDST_BYTE_CONTEXT);
 287
 288         ctx->sh_desc_enc_dma = dma_map_single(jrdev, desc,
 289                                               desc_bytes(desc),
 290                                               DMA_TO_DEVICE);
 291         if (dma_mapping_error(jrdev, ctx->sh_desc_enc_dma)) {
 292                 dev_err(jrdev, "unable to map shared descriptor\n");
 293                 return -ENOMEM;
 294         }
 295 #ifdef DEBUG
 296         print_hex_dump(KERN_ERR,
 297                        "aead null enc shdesc@"__stringify(__LINE__)": ",
 298                        DUMP_PREFIX_ADDRESS, 16, 4, desc,
 299                        desc_bytes(desc), 1);
 300 #endif
 301
 302         /*
 303          * Job Descriptor and Shared Descriptors
 304          * must all fit into the 64-word Descriptor h/w Buffer
 305          */
 306         keys_fit_inline = false;
 307         if (DESC_AEAD_NULL_DEC_LEN + DESC_JOB_IO_LEN +
 308             ctx->split_key_pad_len <= CAAM_DESC_BYTES_MAX)
 309                 keys_fit_inline = true;
 310
 311         desc = ctx->sh_desc_dec;
 312
 313         /* aead_decrypt shared descriptor */
 314         init_sh_desc(desc, HDR_SHARE_SERIAL);
 315
 316         /* Skip if already shared */
 317         key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
 318                                    JUMP_COND_SHRD);
 319         if (keys_fit_inline)
 320                 append_key_as_imm(desc, ctx->key, ctx->split_key_pad_len,
 321                                   ctx->split_key_len, CLASS_2 |
 322                                   KEY_DEST_MDHA_SPLIT | KEY_ENC);
 323         else
 324                 append_key(desc, ctx->key_dma, ctx->split_key_len, CLASS_2 |
 325                            KEY_DEST_MDHA_SPLIT | KEY_ENC);
 326         set_jump_tgt_here(desc, key_jump_cmd);
 327
 328         /* Class 2 operation */
 329         append_operation(desc, ctx->class2_alg_type |
 330                          OP_ALG_AS_INITFINAL | OP_ALG_DECRYPT | OP_ALG_ICV_ON);
 331
 332         /* assoclen + cryptlen = seqinlen - ivsize - authsize */
 333         append_math_sub_imm_u32(desc, REG3, SEQINLEN, IMM,
 334                                 ctx->authsize + tfm->ivsize);
 335         /* assoclen = (assoclen + cryptlen) - cryptlen */
 336         append_math_sub(desc, REG2, SEQOUTLEN, REG0, CAAM_CMD_SZ);
 337         append_math_sub(desc, VARSEQINLEN, REG3, REG2, CAAM_CMD_SZ);
 338
 339         /* read assoc before reading payload */
 340         append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG |
 341                              KEY_VLF);
 342
 343         /* Prepare to read and write cryptlen bytes */
 344         append_math_add(desc, VARSEQINLEN, ZERO, REG2, CAAM_CMD_SZ);
 345         append_math_add(desc, VARSEQOUTLEN, ZERO, REG2, CAAM_CMD_SZ);
 346
 347         /*
 348          * MOVE_LEN opcode is not available in all SEC HW revisions,
 349          * thus need to do some magic, i.e. self-patch the descriptor
 350          * buffer.
 351          */
 352         read_move_cmd = append_move(desc, MOVE_SRC_DESCBUF |
 353                                     MOVE_DEST_MATH2 |
 354                                     (0x6 << MOVE_LEN_SHIFT));
 355         write_move_cmd = append_move(desc, MOVE_SRC_MATH2 |
 356                                      MOVE_DEST_DESCBUF |
 357                                      MOVE_WAITCOMP |
 358                                      (0x8 << MOVE_LEN_SHIFT));
 359
 360         /* Read and write cryptlen bytes */
 361         aead_append_src_dst(desc, FIFOLD_TYPE_MSG | FIFOLD_TYPE_FLUSH1);
 362
 363         /*
 364          * Insert a NOP here, since we need at least 4 instructions between
 365          * code patching the descriptor buffer and the location being patched.
 366          */
 367         jump_cmd = append_jump(desc, JUMP_TEST_ALL);
 368         set_jump_tgt_here(desc, jump_cmd);
 369
 370         set_move_tgt_here(desc, read_move_cmd);
 371         set_move_tgt_here(desc, write_move_cmd);
 372         append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO);
 373         append_move(desc, MOVE_SRC_INFIFO_CL | MOVE_DEST_OUTFIFO |
 374                     MOVE_AUX_LS);
 375         append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO);
 376
 377         /* Load ICV */
 378         append_seq_fifo_load(desc, ctx->authsize, FIFOLD_CLASS_CLASS2 |
 379                              FIFOLD_TYPE_LAST2 | FIFOLD_TYPE_ICV);
 380
 381         ctx->sh_desc_dec_dma = dma_map_single(jrdev, desc,
 382                                               desc_bytes(desc),
 383                                               DMA_TO_DEVICE);
 384         if (dma_mapping_error(jrdev, ctx->sh_desc_dec_dma)) {
 385                 dev_err(jrdev, "unable to map shared descriptor\n");
 386                 return -ENOMEM;
 387         }
 388 #ifdef DEBUG
 389         print_hex_dump(KERN_ERR,
 390                        "aead null dec shdesc@"__stringify(__LINE__)": ",
 391                        DUMP_PREFIX_ADDRESS, 16, 4, desc,
 392                        desc_bytes(desc), 1);
 393 #endif
 394
 395         return 0;
 396 }
 397
 398 static int aead_set_sh_desc(struct crypto_aead *aead)
 399 {
 400         struct aead_tfm *tfm = &aead->base.crt_aead;
 401         struct caam_ctx *ctx = crypto_aead_ctx(aead);
 402         struct device *jrdev = ctx->jrdev;
 403         bool keys_fit_inline = false;
 404         u32 geniv, moveiv;
 405         u32 *desc;
 406
 407         if (!ctx->authsize)
 408                 return 0;
 409
 410         /* NULL encryption / decryption */
 411         if (!ctx->enckeylen)
 412                 return aead_null_set_sh_desc(aead);
 413
 414         /*
 415          * Job Descriptor and Shared Descriptors
 416          * must all fit into the 64-word Descriptor h/w Buffer
 417          */
 418         if (DESC_AEAD_ENC_LEN + DESC_JOB_IO_LEN +
 419             ctx->split_key_pad_len + ctx->enckeylen <=
 420             CAAM_DESC_BYTES_MAX)
 421                 keys_fit_inline = true;
 422
 423         /* aead_encrypt shared descriptor */
 424         desc = ctx->sh_desc_enc;
 425
 426         init_sh_desc_key_aead(desc, ctx, keys_fit_inline);
 427
 428         /* Class 2 operation */
 429         append_operation(desc, ctx->class2_alg_type |
 430                          OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT);
 431
 432         /* cryptlen = seqoutlen - authsize */
 433         append_math_sub_imm_u32(desc, REG3, SEQOUTLEN, IMM, ctx->authsize);
 434
 435         /* assoclen + cryptlen = seqinlen - ivsize */
 436         append_math_sub_imm_u32(desc, REG2, SEQINLEN, IMM, tfm->ivsize);
 437
 438         /* assoclen = (assoclen + cryptlen) - cryptlen */
 439         append_math_sub(desc, VARSEQINLEN, REG2, REG3, CAAM_CMD_SZ);
 440
 441         /* read assoc before reading payload */
 442         append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG |
 443                              KEY_VLF);
 444         aead_append_ld_iv(desc, tfm->ivsize);
 445
 446         /* Class 1 operation */
 447         append_operation(desc, ctx->class1_alg_type |
 448                          OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT);
 449
 450         /* Read and write cryptlen bytes */
 451         append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ);
 452         append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
 453         aead_append_src_dst(desc, FIFOLD_TYPE_MSG1OUT2);
 454
 455         /* Write ICV */
 456         append_seq_store(desc, ctx->authsize, LDST_CLASS_2_CCB |
 457                          LDST_SRCDST_BYTE_CONTEXT);
 458
 459         ctx->sh_desc_enc_dma = dma_map_single(jrdev, desc,
 460                                               desc_bytes(desc),
 461                                               DMA_TO_DEVICE);
 462         if (dma_mapping_error(jrdev, ctx->sh_desc_enc_dma)) {
 463                 dev_err(jrdev, "unable to map shared descriptor\n");
 464                 return -ENOMEM;
 465         }
 466 #ifdef DEBUG
 467         print_hex_dump(KERN_ERR, "aead enc shdesc@"__stringify(__LINE__)": ",
 468                        DUMP_PREFIX_ADDRESS, 16, 4, desc,
 469                        desc_bytes(desc), 1);
 470 #endif
 471
 472         /*
 473          * Job Descriptor and Shared Descriptors
 474          * must all fit into the 64-word Descriptor h/w Buffer
 475          */
 476         keys_fit_inline = false;
 477         if (DESC_AEAD_DEC_LEN + DESC_JOB_IO_LEN +
 478             ctx->split_key_pad_len + ctx->enckeylen <=
 479             CAAM_DESC_BYTES_MAX)
 480                 keys_fit_inline = true;
 481
 482         /* aead_decrypt shared descriptor */
 483         desc = ctx->sh_desc_dec;
 484
 485         init_sh_desc_key_aead(desc, ctx, keys_fit_inline);
 486
 487         /* Class 2 operation */
 488         append_operation(desc, ctx->class2_alg_type |
 489                          OP_ALG_AS_INITFINAL | OP_ALG_DECRYPT | OP_ALG_ICV_ON);
 490
 491         /* assoclen + cryptlen = seqinlen - ivsize - authsize */
 492         append_math_sub_imm_u32(desc, REG3, SEQINLEN, IMM,
 493                                 ctx->authsize + tfm->ivsize);
 494         /* assoclen = (assoclen + cryptlen) - cryptlen */
 495         append_math_sub(desc, REG2, SEQOUTLEN, REG0, CAAM_CMD_SZ);
 496         append_math_sub(desc, VARSEQINLEN, REG3, REG2, CAAM_CMD_SZ);
 497
 498         /* read assoc before reading payload */
 499         append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG |
 500                              KEY_VLF);
 501
 502         aead_append_ld_iv(desc, tfm->ivsize);
 503
 504         append_dec_op1(desc, ctx->class1_alg_type);
 505
 506         /* Read and write cryptlen bytes */
 507         append_math_add(desc, VARSEQINLEN, ZERO, REG2, CAAM_CMD_SZ);
 508         append_math_add(desc, VARSEQOUTLEN, ZERO, REG2, CAAM_CMD_SZ);
 509         aead_append_src_dst(desc, FIFOLD_TYPE_MSG);
 510
 511         /* Load ICV */
 512         append_seq_fifo_load(desc, ctx->authsize, FIFOLD_CLASS_CLASS2 |
 513                              FIFOLD_TYPE_LAST2 | FIFOLD_TYPE_ICV);
 514
 515         ctx->sh_desc_dec_dma = dma_map_single(jrdev, desc,
 516                                               desc_bytes(desc),
 517                                               DMA_TO_DEVICE);
 518         if (dma_mapping_error(jrdev, ctx->sh_desc_dec_dma)) {
 519                 dev_err(jrdev, "unable to map shared descriptor\n");
 520                 return -ENOMEM;
 521         }
 522 #ifdef DEBUG
 523         print_hex_dump(KERN_ERR, "aead dec shdesc@"__stringify(__LINE__)": ",
 524                        DUMP_PREFIX_ADDRESS, 16, 4, desc,
 525                        desc_bytes(desc), 1);
 526 #endif
 527
 528         /*
 529          * Job Descriptor and Shared Descriptors
 530          * must all fit into the 64-word Descriptor h/w Buffer
 531          */
 532         keys_fit_inline = false;
 533         if (DESC_AEAD_GIVENC_LEN + DESC_JOB_IO_LEN +
 534             ctx->split_key_pad_len + ctx->enckeylen <=
 535             CAAM_DESC_BYTES_MAX)
 536                 keys_fit_inline = true;
 537
 538         /* aead_givencrypt shared descriptor */
 539         desc = ctx->sh_desc_givenc;
 540
 541         init_sh_desc_key_aead(desc, ctx, keys_fit_inline);
 542
 543         /* Generate IV */
 544         geniv = NFIFOENTRY_STYPE_PAD | NFIFOENTRY_DEST_DECO |
 545                 NFIFOENTRY_DTYPE_MSG | NFIFOENTRY_LC1 |
 546                 NFIFOENTRY_PTYPE_RND | (tfm->ivsize << NFIFOENTRY_DLEN_SHIFT);
 547         append_load_imm_u32(desc, geniv, LDST_CLASS_IND_CCB |
 548                             LDST_SRCDST_WORD_INFO_FIFO | LDST_IMM);
 549         append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO);
 550         append_move(desc, MOVE_SRC_INFIFO |
 551                     MOVE_DEST_CLASS1CTX | (tfm->ivsize << MOVE_LEN_SHIFT));
 552         append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO);
 553
 554         /* Copy IV to class 1 context */
 555         append_move(desc, MOVE_SRC_CLASS1CTX |
 556                     MOVE_DEST_OUTFIFO | (tfm->ivsize << MOVE_LEN_SHIFT));
 557
 558         /* Return to encryption */
 559         append_operation(desc, ctx->class2_alg_type |
 560                          OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT);
 561
 562         /* ivsize + cryptlen = seqoutlen - authsize */
 563         append_math_sub_imm_u32(desc, REG3, SEQOUTLEN, IMM, ctx->authsize);
 564
 565         /* assoclen = seqinlen - (ivsize + cryptlen) */
 566         append_math_sub(desc, VARSEQINLEN, SEQINLEN, REG3, CAAM_CMD_SZ);
 567
 568         /* read assoc before reading payload */
 569         append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG |
 570                              KEY_VLF);
 571
 572         /* Copy iv from class 1 ctx to class 2 fifo*/
 573         moveiv = NFIFOENTRY_STYPE_OFIFO | NFIFOENTRY_DEST_CLASS2 |
 574                  NFIFOENTRY_DTYPE_MSG | (tfm->ivsize << NFIFOENTRY_DLEN_SHIFT);
 575         append_load_imm_u32(desc, moveiv, LDST_CLASS_IND_CCB |
 576                             LDST_SRCDST_WORD_INFO_FIFO | LDST_IMM);
 577         append_load_imm_u32(desc, tfm->ivsize, LDST_CLASS_2_CCB |
 578                             LDST_SRCDST_WORD_DATASZ_REG | LDST_IMM);
 579
 580         /* Class 1 operation */
 581         append_operation(desc, ctx->class1_alg_type |
 582                          OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT);
 583
 584         /* Will write ivsize + cryptlen */
 585         append_math_add(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
 586
 587         /* Not need to reload iv */
 588         append_seq_fifo_load(desc, tfm->ivsize,
 589                              FIFOLD_CLASS_SKIP);
 590
 591         /* Will read cryptlen */
 592         append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
 593         aead_append_src_dst(desc, FIFOLD_TYPE_MSG1OUT2);
 594
 595         /* Write ICV */
 596         append_seq_store(desc, ctx->authsize, LDST_CLASS_2_CCB |
 597                          LDST_SRCDST_BYTE_CONTEXT);
 598
 599         ctx->sh_desc_givenc_dma = dma_map_single(jrdev, desc,
 600                                                  desc_bytes(desc),
 601                                                  DMA_TO_DEVICE);
 602         if (dma_mapping_error(jrdev, ctx->sh_desc_givenc_dma)) {
 603                 dev_err(jrdev, "unable to map shared descriptor\n");
 604                 return -ENOMEM;
 605         }
 606 #ifdef DEBUG
 607         print_hex_dump(KERN_ERR, "aead givenc shdesc@"__stringify(__LINE__)": ",
 608                        DUMP_PREFIX_ADDRESS, 16, 4, desc,
 609                        desc_bytes(desc), 1);
 610 #endif
 611
 612         return 0;
 613 }
 614
 615 static int aead_setauthsize(struct crypto_aead *authenc,
 616                                     unsigned int authsize)
 617 {
 618         struct caam_ctx *ctx = crypto_aead_ctx(authenc);
 619
 620         ctx->authsize = authsize;
 621         aead_set_sh_desc(authenc);
 622
 623         return 0;
 624 }
 625
 626 static u32 gen_split_aead_key(struct caam_ctx *ctx, const u8 *key_in,
 627                               u32 authkeylen)
 628 {
 629         return gen_split_key(ctx->jrdev, ctx->key, ctx->split_key_len,
 630                                ctx->split_key_pad_len, key_in, authkeylen,
 631                                ctx->alg_op);
 632 }
 633
 634 static int aead_setkey(struct crypto_aead *aead,
 635                                const u8 *key, unsigned int keylen)
 636 {
 637         /* Sizes for MDHA pads (*not* keys): MD5, SHA1, 224, 256, 384, 512 */
 638         static const u8 mdpadlen[] = { 16, 20, 32, 32, 64, 64 };
 639         struct caam_ctx *ctx = crypto_aead_ctx(aead);
 640         struct device *jrdev = ctx->jrdev;
 641         struct crypto_authenc_keys keys;
 642         int ret = 0;
 643
 644         if (crypto_authenc_extractkeys(&keys, key, keylen) != 0)
 645                 goto badkey;
 646
 647         /* Pick class 2 key length from algorithm submask */
 648         ctx->split_key_len = mdpadlen[(ctx->alg_op & OP_ALG_ALGSEL_SUBMASK) >>
 649                                       OP_ALG_ALGSEL_SHIFT] * 2;
 650         ctx->split_key_pad_len = ALIGN(ctx->split_key_len, 16);
 651
 652         if (ctx->split_key_pad_len + keys.enckeylen > CAAM_MAX_KEY_SIZE)
 653                 goto badkey;
 654
 655 #ifdef DEBUG
 656         printk(KERN_ERR "keylen %d enckeylen %d authkeylen %d\n",
 657                keys.authkeylen + keys.enckeylen, keys.enckeylen,
 658                keys.authkeylen);
 659         printk(KERN_ERR "split_key_len %d split_key_pad_len %d\n",
 660                ctx->split_key_len, ctx->split_key_pad_len);
 661         print_hex_dump(KERN_ERR, "key in @"__stringify(__LINE__)": ",
 662                        DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
 663 #endif
 664
 665         ret = gen_split_aead_key(ctx, keys.authkey, keys.authkeylen);
 666         if (ret) {
 667                 goto badkey;
 668         }
 669
 670         /* postpend encryption key to auth split key */
 671         memcpy(ctx->key + ctx->split_key_pad_len, keys.enckey, keys.enckeylen);
 672
 673         ctx->key_dma = dma_map_single(jrdev, ctx->key, ctx->split_key_pad_len +
 674                                       keys.enckeylen, DMA_TO_DEVICE);
 675         if (dma_mapping_error(jrdev, ctx->key_dma)) {
 676                 dev_err(jrdev, "unable to map key i/o memory\n");
 677                 return -ENOMEM;
 678         }
 679 #ifdef DEBUG
 680         print_hex_dump(KERN_ERR, "ctx.key@"__stringify(__LINE__)": ",
 681                        DUMP_PREFIX_ADDRESS, 16, 4, ctx->key,
 682                        ctx->split_key_pad_len + keys.enckeylen, 1);
 683 #endif
 684
 685         ctx->enckeylen = keys.enckeylen;
 686
 687         ret = aead_set_sh_desc(aead);
 688         if (ret) {
 689                 dma_unmap_single(jrdev, ctx->key_dma, ctx->split_key_pad_len +
 690                                  keys.enckeylen, DMA_TO_DEVICE);
 691         }
 692
 693         return ret;
 694 badkey:
 695         crypto_aead_set_flags(aead, CRYPTO_TFM_RES_BAD_KEY_LEN);
 696         return -EINVAL;
 697 }
 698
 699 static int ablkcipher_setkey(struct crypto_ablkcipher *ablkcipher,
 700                              const u8 *key, unsigned int keylen)
 701 {
 702         struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher);
 703         struct ablkcipher_tfm *tfm = &ablkcipher->base.crt_ablkcipher;
 704         struct device *jrdev = ctx->jrdev;
 705         int ret = 0;
 706         u32 *key_jump_cmd;
 707         u32 *desc;
 708
 709 #ifdef DEBUG
 710         print_hex_dump(KERN_ERR, "key in @"__stringify(__LINE__)": ",
 711                        DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
 712 #endif
 713
 714         memcpy(ctx->key, key, keylen);
 715         ctx->key_dma = dma_map_single(jrdev, ctx->key, keylen,
 716                                       DMA_TO_DEVICE);
 717         if (dma_mapping_error(jrdev, ctx->key_dma)) {
 718                 dev_err(jrdev, "unable to map key i/o memory\n");
 719                 return -ENOMEM;
 720         }
 721         ctx->enckeylen = keylen;
 722
 723         /* ablkcipher_encrypt shared descriptor */
 724         desc = ctx->sh_desc_enc;
 725         init_sh_desc(desc, HDR_SHARE_SERIAL);
 726         /* Skip if already shared */
 727         key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
 728                                    JUMP_COND_SHRD);
 729
 730         /* Load class1 key only */
 731         append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen,
 732                           ctx->enckeylen, CLASS_1 |
 733                           KEY_DEST_CLASS_REG);
 734
 735         set_jump_tgt_here(desc, key_jump_cmd);
 736
 737         /* Load iv */
 738         append_cmd(desc, CMD_SEQ_LOAD | LDST_SRCDST_BYTE_CONTEXT |
 739                    LDST_CLASS_1_CCB | tfm->ivsize);
 740
 741         /* Load operation */
 742         append_operation(desc, ctx->class1_alg_type |
 743                          OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT);
 744
 745         /* Perform operation */
 746         ablkcipher_append_src_dst(desc);
 747
 748         ctx->sh_desc_enc_dma = dma_map_single(jrdev, desc,
 749                                               desc_bytes(desc),
 750                                               DMA_TO_DEVICE);
 751         if (dma_mapping_error(jrdev, ctx->sh_desc_enc_dma)) {
 752                 dev_err(jrdev, "unable to map shared descriptor\n");
 753                 return -ENOMEM;
 754         }
 755 #ifdef DEBUG
 756         print_hex_dump(KERN_ERR,
 757                        "ablkcipher enc shdesc@"__stringify(__LINE__)": ",
 758                        DUMP_PREFIX_ADDRESS, 16, 4, desc,
 759                        desc_bytes(desc), 1);
 760 #endif
 761         /* ablkcipher_decrypt shared descriptor */
 762         desc = ctx->sh_desc_dec;
 763
 764         init_sh_desc(desc, HDR_SHARE_SERIAL);
 765         /* Skip if already shared */
 766         key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
 767                                    JUMP_COND_SHRD);
 768
 769         /* Load class1 key only */
 770         append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen,
 771                           ctx->enckeylen, CLASS_1 |
 772                           KEY_DEST_CLASS_REG);
 773
 774         set_jump_tgt_here(desc, key_jump_cmd);
 775
 776         /* load IV */
 777         append_cmd(desc, CMD_SEQ_LOAD | LDST_SRCDST_BYTE_CONTEXT |
 778                    LDST_CLASS_1_CCB | tfm->ivsize);
 779
 780         /* Choose operation */
 781         append_dec_op1(desc, ctx->class1_alg_type);
 782
 783         /* Perform operation */
 784         ablkcipher_append_src_dst(desc);
 785
 786         ctx->sh_desc_dec_dma = dma_map_single(jrdev, desc,
 787                                               desc_bytes(desc),
 788                                               DMA_TO_DEVICE);
 789         if (dma_mapping_error(jrdev, ctx->sh_desc_enc_dma)) {
 790                 dev_err(jrdev, "unable to map shared descriptor\n");
 791                 return -ENOMEM;
 792         }
 793
 794 #ifdef DEBUG
 795         print_hex_dump(KERN_ERR,
 796                        "ablkcipher dec shdesc@"__stringify(__LINE__)": ",
 797                        DUMP_PREFIX_ADDRESS, 16, 4, desc,
 798                        desc_bytes(desc), 1);
 799 #endif
 800
 801         return ret;
 802 }
 803
 804 /*
 805  * aead_edesc - s/w-extended aead descriptor
 806  * @assoc_nents: number of segments in associated data (SPI+Seq) scatterlist
 807  * @assoc_chained: if source is chained
 808  * @src_nents: number of segments in input scatterlist
 809  * @src_chained: if source is chained
 810  * @dst_nents: number of segments in output scatterlist
 811  * @dst_chained: if destination is chained
 812  * @iv_dma: dma address of iv for checking continuity and link table
 813  * @desc: h/w descriptor (variable length; must not exceed MAX_CAAM_DESCSIZE)
 814  * @sec4_sg_bytes: length of dma mapped sec4_sg space
 815  * @sec4_sg_dma: bus physical mapped address of h/w link table
 816  * @hw_desc: the h/w job descriptor followed by any referenced link tables
 817  */
 818 struct aead_edesc {
 819         int assoc_nents;
 820         bool assoc_chained;
 821         int src_nents;
 822         bool src_chained;
 823         int dst_nents;
 824         bool dst_chained;
 825         dma_addr_t iv_dma;
 826         int sec4_sg_bytes;
 827         dma_addr_t sec4_sg_dma;
 828         struct sec4_sg_entry *sec4_sg;
 829         u32 hw_desc[0];
 830 };
 831
 832 /*
 833  * ablkcipher_edesc - s/w-extended ablkcipher descriptor
 834  * @src_nents: number of segments in input scatterlist
 835  * @src_chained: if source is chained
 836  * @dst_nents: number of segments in output scatterlist
 837  * @dst_chained: if destination is chained
 838  * @iv_dma: dma address of iv for checking continuity and link table
 839  * @desc: h/w descriptor (variable length; must not exceed MAX_CAAM_DESCSIZE)
 840  * @sec4_sg_bytes: length of dma mapped sec4_sg space
 841  * @sec4_sg_dma: bus physical mapped address of h/w link table
 842  * @hw_desc: the h/w job descriptor followed by any referenced link tables
 843  */
 844 struct ablkcipher_edesc {
 845         int src_nents;
 846         bool src_chained;
 847         int dst_nents;
 848         bool dst_chained;
 849         dma_addr_t iv_dma;
 850         int sec4_sg_bytes;
 851         dma_addr_t sec4_sg_dma;
 852         struct sec4_sg_entry *sec4_sg;
 853         u32 hw_desc[0];
 854 };
 855
 856 static void caam_unmap(struct device *dev, struct scatterlist *src,
 857                        struct scatterlist *dst, int src_nents,
 858                        bool src_chained, int dst_nents, bool dst_chained,
 859                        dma_addr_t iv_dma, int ivsize, dma_addr_t sec4_sg_dma,
 860                        int sec4_sg_bytes)
 861 {
 862         if (dst != src) {
 863                 dma_unmap_sg_chained(dev, src, src_nents ? : 1, DMA_TO_DEVICE,
 864                                      src_chained);
 865                 dma_unmap_sg_chained(dev, dst, dst_nents ? : 1, DMA_FROM_DEVICE,
 866                                      dst_chained);
 867         } else {
 868                 dma_unmap_sg_chained(dev, src, src_nents ? : 1,
 869                                      DMA_BIDIRECTIONAL, src_chained);
 870         }
 871
 872         if (iv_dma)
 873                 dma_unmap_single(dev, iv_dma, ivsize, DMA_TO_DEVICE);
 874         if (sec4_sg_bytes)
 875                 dma_unmap_single(dev, sec4_sg_dma, sec4_sg_bytes,
 876                                  DMA_TO_DEVICE);
 877 }
 878
 879 static void aead_unmap(struct device *dev,
 880                        struct aead_edesc *edesc,
 881                        struct aead_request *req)
 882 {
 883         struct crypto_aead *aead = crypto_aead_reqtfm(req);
 884         int ivsize = crypto_aead_ivsize(aead);
 885
 886         dma_unmap_sg_chained(dev, req->assoc, edesc->assoc_nents,
 887                              DMA_TO_DEVICE, edesc->assoc_chained);
 888
 889         caam_unmap(dev, req->src, req->dst,
 890                    edesc->src_nents, edesc->src_chained, edesc->dst_nents,
 891                    edesc->dst_chained, edesc->iv_dma, ivsize,
 892                    edesc->sec4_sg_dma, edesc->sec4_sg_bytes);
 893 }
 894
 895 static void ablkcipher_unmap(struct device *dev,
 896                              struct ablkcipher_edesc *edesc,
 897                              struct ablkcipher_request *req)
 898 {
 899         struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req);
 900         int ivsize = crypto_ablkcipher_ivsize(ablkcipher);
 901
 902         caam_unmap(dev, req->src, req->dst,
 903                    edesc->src_nents, edesc->src_chained, edesc->dst_nents,
 904                    edesc->dst_chained, edesc->iv_dma, ivsize,
 905                    edesc->sec4_sg_dma, edesc->sec4_sg_bytes);
 906 }
 907
 908 static void aead_encrypt_done(struct device *jrdev, u32 *desc, u32 err,
 909                                    void *context)
 910 {
 911         struct aead_request *req = context;
 912         struct aead_edesc *edesc;
 913 #ifdef DEBUG
 914         struct crypto_aead *aead = crypto_aead_reqtfm(req);
 915         struct caam_ctx *ctx = crypto_aead_ctx(aead);
 916         int ivsize = crypto_aead_ivsize(aead);
 917
 918         dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
 919 #endif
 920
 921         edesc = (struct aead_edesc *)((char *)desc -
 922                  offsetof(struct aead_edesc, hw_desc));
 923
 924         if (err)
 925                 caam_jr_strstatus(jrdev, err);
 926
 927         aead_unmap(jrdev, edesc, req);
 928
 929 #ifdef DEBUG
 930         print_hex_dump(KERN_ERR, "assoc  @"__stringify(__LINE__)": ",
 931                        DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->assoc),
 932                        req->assoclen , 1);
 933         print_hex_dump(KERN_ERR, "dstiv  @"__stringify(__LINE__)": ",
 934                        DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src) - ivsize,
 935                        edesc->src_nents ? 100 : ivsize, 1);
 936         print_hex_dump(KERN_ERR, "dst    @"__stringify(__LINE__)": ",
 937                        DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src),
 938                        edesc->src_nents ? 100 : req->cryptlen +
 939                        ctx->authsize + 4, 1);
 940 #endif
 941
 942         kfree(edesc);
 943
 944         aead_request_complete(req, err);
 945 }
 946
 947 static void aead_decrypt_done(struct device *jrdev, u32 *desc, u32 err,
 948                                    void *context)
 949 {
 950         struct aead_request *req = context;
 951         struct aead_edesc *edesc;
 952 #ifdef DEBUG
 953         struct crypto_aead *aead = crypto_aead_reqtfm(req);
 954         struct caam_ctx *ctx = crypto_aead_ctx(aead);
 955         int ivsize = crypto_aead_ivsize(aead);
 956
 957         dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
 958 #endif
 959
 960         edesc = (struct aead_edesc *)((char *)desc -
 961                  offsetof(struct aead_edesc, hw_desc));
 962
 963 #ifdef DEBUG
 964         print_hex_dump(KERN_ERR, "dstiv  @"__stringify(__LINE__)": ",
 965                        DUMP_PREFIX_ADDRESS, 16, 4, req->iv,
 966                        ivsize, 1);
 967         print_hex_dump(KERN_ERR, "dst    @"__stringify(__LINE__)": ",
 968                        DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->dst),
 969                        req->cryptlen - ctx->authsize, 1);
 970 #endif
 971
 972         if (err)
 973                 caam_jr_strstatus(jrdev, err);
 974
 975         aead_unmap(jrdev, edesc, req);
 976
 977         /*
 978          * verify hw auth check passed else return -EBADMSG
 979          */
 980         if ((err & JRSTA_CCBERR_ERRID_MASK) == JRSTA_CCBERR_ERRID_ICVCHK)
 981                 err = -EBADMSG;
 982
 983 #ifdef DEBUG
 984         print_hex_dump(KERN_ERR, "iphdrout@"__stringify(__LINE__)": ",
 985                        DUMP_PREFIX_ADDRESS, 16, 4,
 986                        ((char *)sg_virt(req->assoc) - sizeof(struct iphdr)),
 987                        sizeof(struct iphdr) + req->assoclen +
 988                        ((req->cryptlen > 1500) ? 1500 : req->cryptlen) +
 989                        ctx->authsize + 36, 1);
 990         if (!err && edesc->sec4_sg_bytes) {
 991                 struct scatterlist *sg = sg_last(req->src, edesc->src_nents);
 992                 print_hex_dump(KERN_ERR, "sglastout@"__stringify(__LINE__)": ",
 993                                DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(sg),
 994                         sg->length + ctx->authsize + 16, 1);
 995         }
 996 #endif
 997
 998         kfree(edesc);
 999
1000         aead_request_complete(req, err);
1001 }
1002
1003 static void ablkcipher_encrypt_done(struct device *jrdev, u32 *desc, u32 err,
1004                                    void *context)
1005 {
1006         struct ablkcipher_request *req = context;
1007         struct ablkcipher_edesc *edesc;
1008 #ifdef DEBUG
1009         struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req);
1010         int ivsize = crypto_ablkcipher_ivsize(ablkcipher);
1011
1012         dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
1013 #endif
1014
1015         edesc = (struct ablkcipher_edesc *)((char *)desc -
1016                  offsetof(struct ablkcipher_edesc, hw_desc));
1017
1018         if (err)
1019                 caam_jr_strstatus(jrdev, err);
1020
1021 #ifdef DEBUG
1022         print_hex_dump(KERN_ERR, "dstiv  @"__stringify(__LINE__)": ",
1023                        DUMP_PREFIX_ADDRESS, 16, 4, req->info,
1024                        edesc->src_nents > 1 ? 100 : ivsize, 1);
1025         print_hex_dump(KERN_ERR, "dst    @"__stringify(__LINE__)": ",
1026                        DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src),
1027                        edesc->dst_nents > 1 ? 100 : req->nbytes, 1);
1028 #endif
1029
1030         ablkcipher_unmap(jrdev, edesc, req);
1031         kfree(edesc);
1032
1033         ablkcipher_request_complete(req, err);
1034 }
1035
1036 static void ablkcipher_decrypt_done(struct device *jrdev, u32 *desc, u32 err,
1037                                     void *context)
1038 {
1039         struct ablkcipher_request *req = context;
1040         struct ablkcipher_edesc *edesc;
1041 #ifdef DEBUG
1042         struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req);
1043         int ivsize = crypto_ablkcipher_ivsize(ablkcipher);
1044
1045         dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
1046 #endif
1047
1048         edesc = (struct ablkcipher_edesc *)((char *)desc -
1049                  offsetof(struct ablkcipher_edesc, hw_desc));
1050         if (err)
1051                 caam_jr_strstatus(jrdev, err);
1052
1053 #ifdef DEBUG
1054         print_hex_dump(KERN_ERR, "dstiv  @"__stringify(__LINE__)": ",
1055                        DUMP_PREFIX_ADDRESS, 16, 4, req->info,
1056                        ivsize, 1);
1057         print_hex_dump(KERN_ERR, "dst    @"__stringify(__LINE__)": ",
1058                        DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src),
1059                        edesc->dst_nents > 1 ? 100 : req->nbytes, 1);
1060 #endif
1061
1062         ablkcipher_unmap(jrdev, edesc, req);
1063         kfree(edesc);
1064
1065         ablkcipher_request_complete(req, err);
1066 }
1067
1068 /*
1069  * Fill in aead job descriptor
1070  */
1071 static void init_aead_job(u32 *sh_desc, dma_addr_t ptr,
1072                           struct aead_edesc *edesc,
1073                           struct aead_request *req,
1074                           bool all_contig, bool encrypt)
1075 {
1076         struct crypto_aead *aead = crypto_aead_reqtfm(req);
1077         struct caam_ctx *ctx = crypto_aead_ctx(aead);
1078         int ivsize = crypto_aead_ivsize(aead);
1079         int authsize = ctx->authsize;
1080         u32 *desc = edesc->hw_desc;
1081         u32 out_options = 0, in_options;
1082         dma_addr_t dst_dma, src_dma;
1083         int len, sec4_sg_index = 0;
1084
1085 #ifdef DEBUG
1086         debug("assoclen %d cryptlen %d authsize %d\n",
1087               req->assoclen, req->cryptlen, authsize);
1088         print_hex_dump(KERN_ERR, "assoc  @"__stringify(__LINE__)": ",
1089                        DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->assoc),
1090                        req->assoclen , 1);
1091         print_hex_dump(KERN_ERR, "presciv@"__stringify(__LINE__)": ",
1092                        DUMP_PREFIX_ADDRESS, 16, 4, req->iv,
1093                        edesc->src_nents ? 100 : ivsize, 1);
1094         print_hex_dump(KERN_ERR, "src    @"__stringify(__LINE__)": ",
1095                        DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src),
1096                         edesc->src_nents ? 100 : req->cryptlen, 1);
1097         print_hex_dump(KERN_ERR, "shrdesc@"__stringify(__LINE__)": ",
1098                        DUMP_PREFIX_ADDRESS, 16, 4, sh_desc,
1099                        desc_bytes(sh_desc), 1);
1100 #endif
1101
1102         len = desc_len(sh_desc);
1103         init_job_desc_shared(desc, ptr, len, HDR_SHARE_DEFER | HDR_REVERSE);
1104
1105         if (all_contig) {
1106                 src_dma = sg_dma_address(req->assoc);
1107                 in_options = 0;
1108         } else {
1109                 src_dma = edesc->sec4_sg_dma;
1110                 sec4_sg_index += (edesc->assoc_nents ? : 1) + 1 +
1111                                  (edesc->src_nents ? : 1);
1112                 in_options = LDST_SGF;
1113         }
1114
1115         append_seq_in_ptr(desc, src_dma, req->assoclen + ivsize + req->cryptlen,
1116                           in_options);
1117
1118         if (likely(req->src == req->dst)) {
1119                 if (all_contig) {
1120                         dst_dma = sg_dma_address(req->src);
1121                 } else {
1122                         dst_dma = src_dma + sizeof(struct sec4_sg_entry) *
1123                                   ((edesc->assoc_nents ? : 1) + 1);
1124                         out_options = LDST_SGF;
1125                 }
1126         } else {
1127                 if (!edesc->dst_nents) {
1128                         dst_dma = sg_dma_address(req->dst);
1129                 } else {
1130                         dst_dma = edesc->sec4_sg_dma +
1131                                   sec4_sg_index *
1132                                   sizeof(struct sec4_sg_entry);
1133                         out_options = LDST_SGF;
1134                 }
1135         }
1136         if (encrypt)
1137                 append_seq_out_ptr(desc, dst_dma, req->cryptlen + authsize,
1138                                    out_options);
1139         else
1140                 append_seq_out_ptr(desc, dst_dma, req->cryptlen - authsize,
1141                                    out_options);
1142 }
1143
1144 /*
1145  * Fill in aead givencrypt job descriptor
1146  */
1147 static void init_aead_giv_job(u32 *sh_desc, dma_addr_t ptr,
1148                               struct aead_edesc *edesc,
1149                               struct aead_request *req,
1150                               int contig)
1151 {
1152         struct crypto_aead *aead = crypto_aead_reqtfm(req);
1153         struct caam_ctx *ctx = crypto_aead_ctx(aead);
1154         int ivsize = crypto_aead_ivsize(aead);
1155         int authsize = ctx->authsize;
1156         u32 *desc = edesc->hw_desc;
1157         u32 out_options = 0, in_options;
1158         dma_addr_t dst_dma, src_dma;
1159         int len, sec4_sg_index = 0;
1160
1161 #ifdef DEBUG
1162         debug("assoclen %d cryptlen %d authsize %d\n",
1163               req->assoclen, req->cryptlen, authsize);
1164         print_hex_dump(KERN_ERR, "assoc  @"__stringify(__LINE__)": ",
1165                        DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->assoc),
1166                        req->assoclen , 1);
1167         print_hex_dump(KERN_ERR, "presciv@"__stringify(__LINE__)": ",
1168                        DUMP_PREFIX_ADDRESS, 16, 4, req->iv, ivsize, 1);
1169         print_hex_dump(KERN_ERR, "src    @"__stringify(__LINE__)": ",
1170                        DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src),
1171                         edesc->src_nents > 1 ? 100 : req->cryptlen, 1);
1172         print_hex_dump(KERN_ERR, "shrdesc@"__stringify(__LINE__)": ",
1173                        DUMP_PREFIX_ADDRESS, 16, 4, sh_desc,
1174                        desc_bytes(sh_desc), 1);
1175 #endif
1176
1177         len = desc_len(sh_desc);
1178         init_job_desc_shared(desc, ptr, len, HDR_SHARE_DEFER | HDR_REVERSE);
1179
1180         if (contig & GIV_SRC_CONTIG) {
1181                 src_dma = sg_dma_address(req->assoc);
1182                 in_options = 0;
1183         } else {
1184                 src_dma = edesc->sec4_sg_dma;
1185                 sec4_sg_index += edesc->assoc_nents + 1 + edesc->src_nents;
1186                 in_options = LDST_SGF;
1187         }
1188         append_seq_in_ptr(desc, src_dma, req->assoclen + ivsize + req->cryptlen,
1189                           in_options);
1190
1191         if (contig & GIV_DST_CONTIG) {
1192                 dst_dma = edesc->iv_dma;
1193         } else {
1194                 if (likely(req->src == req->dst)) {
1195                         dst_dma = src_dma + sizeof(struct sec4_sg_entry) *
1196                                   edesc->assoc_nents;
1197                         out_options = LDST_SGF;
1198                 } else {
1199                         dst_dma = edesc->sec4_sg_dma +
1200                                   sec4_sg_index *
1201                                   sizeof(struct sec4_sg_entry);
1202                         out_options = LDST_SGF;
1203                 }
1204         }
1205
1206         append_seq_out_ptr(desc, dst_dma, ivsize + req->cryptlen + authsize,
1207                            out_options);
1208 }
1209
1210 /*
1211  * Fill in ablkcipher job descriptor
1212  */
1213 static void init_ablkcipher_job(u32 *sh_desc, dma_addr_t ptr,
1214                                 struct ablkcipher_edesc *edesc,
1215                                 struct ablkcipher_request *req,
1216                                 bool iv_contig)
1217 {
1218         struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req);
1219         int ivsize = crypto_ablkcipher_ivsize(ablkcipher);
1220         u32 *desc = edesc->hw_desc;
1221         u32 out_options = 0, in_options;
1222         dma_addr_t dst_dma, src_dma;
1223         int len, sec4_sg_index = 0;
1224
1225 #ifdef DEBUG
1226         print_hex_dump(KERN_ERR, "presciv@"__stringify(__LINE__)": ",
1227                        DUMP_PREFIX_ADDRESS, 16, 4, req->info,
1228                        ivsize, 1);
1229         print_hex_dump(KERN_ERR, "src    @"__stringify(__LINE__)": ",
1230                        DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src),
1231                        edesc->src_nents ? 100 : req->nbytes, 1);
1232 #endif
1233
1234         len = desc_len(sh_desc);
1235         init_job_desc_shared(desc, ptr, len, HDR_SHARE_DEFER | HDR_REVERSE);
1236
1237         if (iv_contig) {
1238                 src_dma = edesc->iv_dma;
1239                 in_options = 0;
1240         } else {
1241                 src_dma = edesc->sec4_sg_dma;
1242                 sec4_sg_index += (iv_contig ? 0 : 1) + edesc->src_nents;
1243                 in_options = LDST_SGF;
1244         }
1245         append_seq_in_ptr(desc, src_dma, req->nbytes + ivsize, in_options);
1246
1247         if (likely(req->src == req->dst)) {
1248                 if (!edesc->src_nents && iv_contig) {
1249                         dst_dma = sg_dma_address(req->src);
1250                 } else {
1251                         dst_dma = edesc->sec4_sg_dma +
1252                                 sizeof(struct sec4_sg_entry);
1253                         out_options = LDST_SGF;
1254                 }
1255         } else {
1256                 if (!edesc->dst_nents) {
1257                         dst_dma = sg_dma_address(req->dst);
1258                 } else {
1259                         dst_dma = edesc->sec4_sg_dma +
1260                                 sec4_sg_index * sizeof(struct sec4_sg_entry);
1261                         out_options = LDST_SGF;
1262                 }
1263         }
1264         append_seq_out_ptr(desc, dst_dma, req->nbytes, out_options);
1265 }
1266
1267 /*
1268  * allocate and map the aead extended descriptor
1269  */
1270 static struct aead_edesc *aead_edesc_alloc(struct aead_request *req,
1271                                            int desc_bytes, bool *all_contig_ptr,
1272                                            bool encrypt)
1273 {
1274         struct crypto_aead *aead = crypto_aead_reqtfm(req);
1275         struct caam_ctx *ctx = crypto_aead_ctx(aead);
1276         struct device *jrdev = ctx->jrdev;
1277         gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG |
1278                        CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC;
1279         int assoc_nents, src_nents, dst_nents = 0;
1280         struct aead_edesc *edesc;
1281         dma_addr_t iv_dma = 0;
1282         int sgc;
1283         bool all_contig = true;
1284         bool assoc_chained = false, src_chained = false, dst_chained = false;
1285         int ivsize = crypto_aead_ivsize(aead);
1286         int sec4_sg_index, sec4_sg_len = 0, sec4_sg_bytes;
1287         unsigned int authsize = ctx->authsize;
1288
1289         assoc_nents = sg_count(req->assoc, req->assoclen, &assoc_chained);
1290
1291         if (unlikely(req->dst != req->src)) {
1292                 src_nents = sg_count(req->src, req->cryptlen, &src_chained);
1293                 dst_nents = sg_count(req->dst,
1294                                      req->cryptlen +
1295                                         (encrypt ? authsize : (-authsize)),
1296                                      &dst_chained);
1297         } else {
1298                 src_nents = sg_count(req->src,
1299                                      req->cryptlen +
1300                                         (encrypt ? authsize : 0),
1301                                      &src_chained);
1302         }
1303
1304         sgc = dma_map_sg_chained(jrdev, req->assoc, assoc_nents ? : 1,
1305                                  DMA_TO_DEVICE, assoc_chained);
1306         if (likely(req->src == req->dst)) {
1307                 sgc = dma_map_sg_chained(jrdev, req->src, src_nents ? : 1,
1308                                          DMA_BIDIRECTIONAL, src_chained);
1309         } else {
1310                 sgc = dma_map_sg_chained(jrdev, req->src, src_nents ? : 1,
1311                                          DMA_TO_DEVICE, src_chained);
1312                 sgc = dma_map_sg_chained(jrdev, req->dst, dst_nents ? : 1,
1313                                          DMA_FROM_DEVICE, dst_chained);
1314         }
1315
1316         /* Check if data are contiguous */
1317         iv_dma = dma_map_single(jrdev, req->iv, ivsize, DMA_TO_DEVICE);
1318         if (assoc_nents || sg_dma_address(req->assoc) + req->assoclen !=
1319             iv_dma || src_nents || iv_dma + ivsize !=
1320             sg_dma_address(req->src)) {
1321                 all_contig = false;
1322                 assoc_nents = assoc_nents ? : 1;
1323                 src_nents = src_nents ? : 1;
1324                 sec4_sg_len = assoc_nents + 1 + src_nents;
1325         }
1326         sec4_sg_len += dst_nents;
1327
1328         sec4_sg_bytes = sec4_sg_len * sizeof(struct sec4_sg_entry);
1329
1330         /* allocate space for base edesc and hw desc commands, link tables */
1331         edesc = kmalloc(sizeof(struct aead_edesc) + desc_bytes +
1332                         sec4_sg_bytes, GFP_DMA | flags);
1333         if (!edesc) {
1334                 dev_err(jrdev, "could not allocate extended descriptor\n");
1335                 return ERR_PTR(-ENOMEM);
1336         }
1337
1338         edesc->assoc_nents = assoc_nents;
1339         edesc->assoc_chained = assoc_chained;
1340         edesc->src_nents = src_nents;
1341         edesc->src_chained = src_chained;
1342         edesc->dst_nents = dst_nents;
1343         edesc->dst_chained = dst_chained;
1344         edesc->iv_dma = iv_dma;
1345         edesc->sec4_sg_bytes = sec4_sg_bytes;
1346         edesc->sec4_sg = (void *)edesc + sizeof(struct aead_edesc) +
1347                          desc_bytes;
1348         edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg,
1349                                             sec4_sg_bytes, DMA_TO_DEVICE);
1350         *all_contig_ptr = all_contig;
1351
1352         sec4_sg_index = 0;
1353         if (!all_contig) {
1354                 sg_to_sec4_sg(req->assoc,
1355                               (assoc_nents ? : 1),
1356                               edesc->sec4_sg +
1357                               sec4_sg_index, 0);
1358                 sec4_sg_index += assoc_nents ? : 1;
1359                 dma_to_sec4_sg_one(edesc->sec4_sg + sec4_sg_index,
1360                                    iv_dma, ivsize, 0);
1361                 sec4_sg_index += 1;
1362                 sg_to_sec4_sg_last(req->src,
1363                                    (src_nents ? : 1),
1364                                    edesc->sec4_sg +
1365                                    sec4_sg_index, 0);
1366                 sec4_sg_index += src_nents ? : 1;
1367         }
1368         if (dst_nents) {
1369                 sg_to_sec4_sg_last(req->dst, dst_nents,
1370                                    edesc->sec4_sg + sec4_sg_index, 0);
1371         }
1372
1373         return edesc;
1374 }
1375
1376 static int aead_encrypt(struct aead_request *req)
1377 {
1378         struct aead_edesc *edesc;
1379         struct crypto_aead *aead = crypto_aead_reqtfm(req);
1380         struct caam_ctx *ctx = crypto_aead_ctx(aead);
1381         struct device *jrdev = ctx->jrdev;
1382         bool all_contig;
1383         u32 *desc;
1384         int ret = 0;
1385
1386         /* allocate extended descriptor */
1387         edesc = aead_edesc_alloc(req, DESC_JOB_IO_LEN *
1388                                  CAAM_CMD_SZ, &all_contig, true);
1389         if (IS_ERR(edesc))
1390                 return PTR_ERR(edesc);
1391
1392         /* Create and submit job descriptor */
1393         init_aead_job(ctx->sh_desc_enc, ctx->sh_desc_enc_dma, edesc, req,
1394                       all_contig, true);
1395 #ifdef DEBUG
1396         print_hex_dump(KERN_ERR, "aead jobdesc@"__stringify(__LINE__)": ",
1397                        DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc,
1398                        desc_bytes(edesc->hw_desc), 1);
1399 #endif
1400
1401         desc = edesc->hw_desc;
1402         ret = caam_jr_enqueue(jrdev, desc, aead_encrypt_done, req);
1403         if (!ret) {
1404                 ret = -EINPROGRESS;
1405         } else {
1406                 aead_unmap(jrdev, edesc, req);
1407                 kfree(edesc);
1408         }
1409
1410         return ret;
1411 }
1412
1413 static int aead_decrypt(struct aead_request *req)
1414 {
1415         struct aead_edesc *edesc;
1416         struct crypto_aead *aead = crypto_aead_reqtfm(req);
1417         struct caam_ctx *ctx = crypto_aead_ctx(aead);
1418         struct device *jrdev = ctx->jrdev;
1419         bool all_contig;
1420         u32 *desc;
1421         int ret = 0;
1422
1423         /* allocate extended descriptor */
1424         edesc = aead_edesc_alloc(req, DESC_JOB_IO_LEN *
1425                                  CAAM_CMD_SZ, &all_contig, false);
1426         if (IS_ERR(edesc))
1427                 return PTR_ERR(edesc);
1428
1429 #ifdef DEBUG
1430         print_hex_dump(KERN_ERR, "dec src@"__stringify(__LINE__)": ",
1431                        DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src),
1432                        req->cryptlen, 1);
1433 #endif
1434
1435         /* Create and submit job descriptor*/
1436         init_aead_job(ctx->sh_desc_dec,
1437                       ctx->sh_desc_dec_dma, edesc, req, all_contig, false);
1438 #ifdef DEBUG
1439         print_hex_dump(KERN_ERR, "aead jobdesc@"__stringify(__LINE__)": ",
1440                        DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc,
1441                        desc_bytes(edesc->hw_desc), 1);
1442 #endif
1443
1444         desc = edesc->hw_desc;
1445         ret = caam_jr_enqueue(jrdev, desc, aead_decrypt_done, req);
1446         if (!ret) {
1447                 ret = -EINPROGRESS;
1448         } else {
1449                 aead_unmap(jrdev, edesc, req);
1450                 kfree(edesc);
1451         }
1452
1453         return ret;
1454 }
1455
1456 /*
1457  * allocate and map the aead extended descriptor for aead givencrypt
1458  */
1459 static struct aead_edesc *aead_giv_edesc_alloc(struct aead_givcrypt_request
1460                                                *greq, int desc_bytes,
1461                                                u32 *contig_ptr)
1462 {
1463         struct aead_request *req = &greq->areq;
1464         struct crypto_aead *aead = crypto_aead_reqtfm(req);
1465         struct caam_ctx *ctx = crypto_aead_ctx(aead);
1466         struct device *jrdev = ctx->jrdev;
1467         gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG |
1468                        CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC;
1469         int assoc_nents, src_nents, dst_nents = 0;
1470         struct aead_edesc *edesc;
1471         dma_addr_t iv_dma = 0;
1472         int sgc;
1473         u32 contig = GIV_SRC_CONTIG | GIV_DST_CONTIG;
1474         int ivsize = crypto_aead_ivsize(aead);
1475         bool assoc_chained = false, src_chained = false, dst_chained = false;
1476         int sec4_sg_index, sec4_sg_len = 0, sec4_sg_bytes;
1477
1478         assoc_nents = sg_count(req->assoc, req->assoclen, &assoc_chained);
1479         src_nents = sg_count(req->src, req->cryptlen, &src_chained);
1480
1481         if (unlikely(req->dst != req->src))
1482                 dst_nents = sg_count(req->dst, req->cryptlen + ctx->authsize,
1483                                      &dst_chained);
1484
1485         sgc = dma_map_sg_chained(jrdev, req->assoc, assoc_nents ? : 1,
1486                                  DMA_TO_DEVICE, assoc_chained);
1487         if (likely(req->src == req->dst)) {
1488                 sgc = dma_map_sg_chained(jrdev, req->src, src_nents ? : 1,
1489                                          DMA_BIDIRECTIONAL, src_chained);
1490         } else {
1491                 sgc = dma_map_sg_chained(jrdev, req->src, src_nents ? : 1,
1492                                          DMA_TO_DEVICE, src_chained);
1493                 sgc = dma_map_sg_chained(jrdev, req->dst, dst_nents ? : 1,
1494                                          DMA_FROM_DEVICE, dst_chained);
1495         }
1496
1497         /* Check if data are contiguous */
1498         iv_dma = dma_map_single(jrdev, greq->giv, ivsize, DMA_TO_DEVICE);
1499         if (assoc_nents || sg_dma_address(req->assoc) + req->assoclen !=
1500             iv_dma || src_nents || iv_dma + ivsize != sg_dma_address(req->src))
1501                 contig &= ~GIV_SRC_CONTIG;
1502         if (dst_nents || iv_dma + ivsize != sg_dma_address(req->dst))
1503                 contig &= ~GIV_DST_CONTIG;
1504         if (unlikely(req->src != req->dst)) {
1505                 dst_nents = dst_nents ? : 1;
1506                 sec4_sg_len += 1;
1507         }
1508         if (!(contig & GIV_SRC_CONTIG)) {
1509                 assoc_nents = assoc_nents ? : 1;
1510                 src_nents = src_nents ? : 1;
1511                 sec4_sg_len += assoc_nents + 1 + src_nents;
1512                 if (likely(req->src == req->dst))
1513                         contig &= ~GIV_DST_CONTIG;
1514         }
1515         sec4_sg_len += dst_nents;
1516
1517         sec4_sg_bytes = sec4_sg_len * sizeof(struct sec4_sg_entry);
1518
1519         /* allocate space for base edesc and hw desc commands, link tables */
1520         edesc = kmalloc(sizeof(struct aead_edesc) + desc_bytes +
1521                         sec4_sg_bytes, GFP_DMA | flags);
1522         if (!edesc) {
1523                 dev_err(jrdev, "could not allocate extended descriptor\n");
1524                 return ERR_PTR(-ENOMEM);
1525         }
1526
1527         edesc->assoc_nents = assoc_nents;
1528         edesc->assoc_chained = assoc_chained;
1529         edesc->src_nents = src_nents;
1530         edesc->src_chained = src_chained;
1531         edesc->dst_nents = dst_nents;
1532         edesc->dst_chained = dst_chained;
1533         edesc->iv_dma = iv_dma;
1534         edesc->sec4_sg_bytes = sec4_sg_bytes;
1535         edesc->sec4_sg = (void *)edesc + sizeof(struct aead_edesc) +
1536                          desc_bytes;
1537         edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg,
1538                                             sec4_sg_bytes, DMA_TO_DEVICE);
1539         *contig_ptr = contig;
1540
1541         sec4_sg_index = 0;
1542         if (!(contig & GIV_SRC_CONTIG)) {
1543                 sg_to_sec4_sg(req->assoc, assoc_nents,
1544                               edesc->sec4_sg +
1545                               sec4_sg_index, 0);
1546                 sec4_sg_index += assoc_nents;
1547                 dma_to_sec4_sg_one(edesc->sec4_sg + sec4_sg_index,
1548                                    iv_dma, ivsize, 0);
1549                 sec4_sg_index += 1;
1550                 sg_to_sec4_sg_last(req->src, src_nents,
1551                                    edesc->sec4_sg +
1552                                    sec4_sg_index, 0);
1553                 sec4_sg_index += src_nents;
1554         }
1555         if (unlikely(req->src != req->dst && !(contig & GIV_DST_CONTIG))) {
1556                 dma_to_sec4_sg_one(edesc->sec4_sg + sec4_sg_index,
1557                                    iv_dma, ivsize, 0);
1558                 sec4_sg_index += 1;
1559                 sg_to_sec4_sg_last(req->dst, dst_nents,
1560                                    edesc->sec4_sg + sec4_sg_index, 0);
1561         }
1562
1563         return edesc;
1564 }
1565
1566 static int aead_givencrypt(struct aead_givcrypt_request *areq)
1567 {
1568         struct aead_request *req = &areq->areq;
1569         struct aead_edesc *edesc;
1570         struct crypto_aead *aead = crypto_aead_reqtfm(req);
1571         struct caam_ctx *ctx = crypto_aead_ctx(aead);
1572         struct device *jrdev = ctx->jrdev;
1573         u32 contig;
1574         u32 *desc;
1575         int ret = 0;
1576
1577         /* allocate extended descriptor */
1578         edesc = aead_giv_edesc_alloc(areq, DESC_JOB_IO_LEN *
1579                                      CAAM_CMD_SZ, &contig);
1580
1581         if (IS_ERR(edesc))
1582                 return PTR_ERR(edesc);
1583
1584 #ifdef DEBUG
1585         print_hex_dump(KERN_ERR, "giv src@"__stringify(__LINE__)": ",
1586                        DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src),
1587                        req->cryptlen, 1);
1588 #endif
1589
1590         /* Create and submit job descriptor*/
1591         init_aead_giv_job(ctx->sh_desc_givenc,
1592                           ctx->sh_desc_givenc_dma, edesc, req, contig);
1593 #ifdef DEBUG
1594         print_hex_dump(KERN_ERR, "aead jobdesc@"__stringify(__LINE__)": ",
1595                        DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc,
1596                        desc_bytes(edesc->hw_desc), 1);
1597 #endif
1598
1599         desc = edesc->hw_desc;
1600         ret = caam_jr_enqueue(jrdev, desc, aead_encrypt_done, req);
1601         if (!ret) {
1602                 ret = -EINPROGRESS;
1603         } else {
1604                 aead_unmap(jrdev, edesc, req);
1605                 kfree(edesc);
1606         }
1607
1608         return ret;
1609 }
1610
1611 static int aead_null_givencrypt(struct aead_givcrypt_request *areq)
1612 {
1613         return aead_encrypt(&areq->areq);
1614 }
1615
1616 /*
1617  * allocate and map the ablkcipher extended descriptor for ablkcipher
1618  */
1619 static struct ablkcipher_edesc *ablkcipher_edesc_alloc(struct ablkcipher_request
1620                                                        *req, int desc_bytes,
1621                                                        bool *iv_contig_out)
1622 {
1623         struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req);
1624         struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher);
1625         struct device *jrdev = ctx->jrdev;
1626         gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG |
1627                                           CRYPTO_TFM_REQ_MAY_SLEEP)) ?
1628                        GFP_KERNEL : GFP_ATOMIC;
1629         int src_nents, dst_nents = 0, sec4_sg_bytes;
1630         struct ablkcipher_edesc *edesc;
1631         dma_addr_t iv_dma = 0;
1632         bool iv_contig = false;
1633         int sgc;
1634         int ivsize = crypto_ablkcipher_ivsize(ablkcipher);
1635         bool src_chained = false, dst_chained = false;
1636         int sec4_sg_index;
1637
1638         src_nents = sg_count(req->src, req->nbytes, &src_chained);
1639
1640         if (req->dst != req->src)
1641                 dst_nents = sg_count(req->dst, req->nbytes, &dst_chained);
1642
1643         if (likely(req->src == req->dst)) {
1644                 sgc = dma_map_sg_chained(jrdev, req->src, src_nents ? : 1,
1645                                          DMA_BIDIRECTIONAL, src_chained);
1646         } else {
1647                 sgc = dma_map_sg_chained(jrdev, req->src, src_nents ? : 1,
1648                                          DMA_TO_DEVICE, src_chained);
1649                 sgc = dma_map_sg_chained(jrdev, req->dst, dst_nents ? : 1,
1650                                          DMA_FROM_DEVICE, dst_chained);
1651         }
1652
1653         /*
1654          * Check if iv can be contiguous with source and destination.
1655          * If so, include it. If not, create scatterlist.
1656          */
1657         iv_dma = dma_map_single(jrdev, req->info, ivsize, DMA_TO_DEVICE);
1658         if (!src_nents && iv_dma + ivsize == sg_dma_address(req->src))
1659                 iv_contig = true;
1660         else
1661                 src_nents = src_nents ? : 1;
1662         sec4_sg_bytes = ((iv_contig ? 0 : 1) + src_nents + dst_nents) *
1663                         sizeof(struct sec4_sg_entry);
1664
1665         /* allocate space for base edesc and hw desc commands, link tables */
1666         edesc = kmalloc(sizeof(struct ablkcipher_edesc) + desc_bytes +
1667                         sec4_sg_bytes, GFP_DMA | flags);
1668         if (!edesc) {
1669                 dev_err(jrdev, "could not allocate extended descriptor\n");
1670                 return ERR_PTR(-ENOMEM);
1671         }
1672
1673         edesc->src_nents = src_nents;
1674         edesc->src_chained = src_chained;
1675         edesc->dst_nents = dst_nents;
1676         edesc->dst_chained = dst_chained;
1677         edesc->sec4_sg_bytes = sec4_sg_bytes;
1678         edesc->sec4_sg = (void *)edesc + sizeof(struct ablkcipher_edesc) +
1679                          desc_bytes;
1680
1681         sec4_sg_index = 0;
1682         if (!iv_contig) {
1683                 dma_to_sec4_sg_one(edesc->sec4_sg, iv_dma, ivsize, 0);
1684                 sg_to_sec4_sg_last(req->src, src_nents,
1685                                    edesc->sec4_sg + 1, 0);
1686                 sec4_sg_index += 1 + src_nents;
1687         }
1688
1689         if (dst_nents) {
1690                 sg_to_sec4_sg_last(req->dst, dst_nents,
1691                         edesc->sec4_sg + sec4_sg_index, 0);
1692         }
1693
1694         edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg,
1695                                             sec4_sg_bytes, DMA_TO_DEVICE);
1696         edesc->iv_dma = iv_dma;
1697
1698 #ifdef DEBUG
1699         print_hex_dump(KERN_ERR, "ablkcipher sec4_sg@"__stringify(__LINE__)": ",
1700                        DUMP_PREFIX_ADDRESS, 16, 4, edesc->sec4_sg,
1701                        sec4_sg_bytes, 1);
1702 #endif
1703
1704         *iv_contig_out = iv_contig;
1705         return edesc;
1706 }
1707
1708 static int ablkcipher_encrypt(struct ablkcipher_request *req)
1709 {
1710         struct ablkcipher_edesc *edesc;
1711         struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req);
1712         struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher);
1713         struct device *jrdev = ctx->jrdev;
1714         bool iv_contig;
1715         u32 *desc;
1716         int ret = 0;
1717
1718         /* allocate extended descriptor */
1719         edesc = ablkcipher_edesc_alloc(req, DESC_JOB_IO_LEN *
1720                                        CAAM_CMD_SZ, &iv_contig);
1721         if (IS_ERR(edesc))
1722                 return PTR_ERR(edesc);
1723
1724         /* Create and submit job descriptor*/
1725         init_ablkcipher_job(ctx->sh_desc_enc,
1726                 ctx->sh_desc_enc_dma, edesc, req, iv_contig);
1727 #ifdef DEBUG
1728         print_hex_dump(KERN_ERR, "ablkcipher jobdesc@"__stringify(__LINE__)": ",
1729                        DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc,
1730                        desc_bytes(edesc->hw_desc), 1);
1731 #endif
1732         desc = edesc->hw_desc;
1733         ret = caam_jr_enqueue(jrdev, desc, ablkcipher_encrypt_done, req);
1734
1735         if (!ret) {
1736                 ret = -EINPROGRESS;
1737         } else {
1738                 ablkcipher_unmap(jrdev, edesc, req);
1739                 kfree(edesc);
1740         }
1741
1742         return ret;
1743 }
1744
1745 static int ablkcipher_decrypt(struct ablkcipher_request *req)
1746 {
1747         struct ablkcipher_edesc *edesc;
1748         struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req);
1749         struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher);
1750         struct device *jrdev = ctx->jrdev;
1751         bool iv_contig;
1752         u32 *desc;
1753         int ret = 0;
1754
1755         /* allocate extended descriptor */
1756         edesc = ablkcipher_edesc_alloc(req, DESC_JOB_IO_LEN *
1757                                        CAAM_CMD_SZ, &iv_contig);
1758         if (IS_ERR(edesc))
1759                 return PTR_ERR(edesc);
1760
1761         /* Create and submit job descriptor*/
1762         init_ablkcipher_job(ctx->sh_desc_dec,
1763                 ctx->sh_desc_dec_dma, edesc, req, iv_contig);
1764         desc = edesc->hw_desc;
1765 #ifdef DEBUG
1766         print_hex_dump(KERN_ERR, "ablkcipher jobdesc@"__stringify(__LINE__)": ",
1767                        DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc,
1768                        desc_bytes(edesc->hw_desc), 1);
1769 #endif
1770
1771         ret = caam_jr_enqueue(jrdev, desc, ablkcipher_decrypt_done, req);
1772         if (!ret) {
1773                 ret = -EINPROGRESS;
1774         } else {
1775                 ablkcipher_unmap(jrdev, edesc, req);
1776                 kfree(edesc);
1777         }
1778
1779         return ret;
1780 }
1781
1782 #define template_aead           template_u.aead
1783 #define template_ablkcipher     template_u.ablkcipher
1784 struct caam_alg_template {
1785         char name[CRYPTO_MAX_ALG_NAME];
1786         char driver_name[CRYPTO_MAX_ALG_NAME];
1787         unsigned int blocksize;
1788         u32 type;
1789         union {
1790                 struct ablkcipher_alg ablkcipher;
1791                 struct aead_alg aead;
1792                 struct blkcipher_alg blkcipher;
1793                 struct cipher_alg cipher;
1794                 struct compress_alg compress;
1795                 struct rng_alg rng;
1796         } template_u;
1797         u32 class1_alg_type;
1798         u32 class2_alg_type;
1799         u32 alg_op;
1800 };
1801
1802 static struct caam_alg_template driver_algs[] = {
1803         /* single-pass ipsec_esp descriptor */
1804         {
1805                 .name = "authenc(hmac(md5),ecb(cipher_null))",
1806                 .driver_name = "authenc-hmac-md5-ecb-cipher_null-caam",
1807                 .blocksize = NULL_BLOCK_SIZE,
1808                 .type = CRYPTO_ALG_TYPE_AEAD,
1809                 .template_aead = {
1810                         .setkey = aead_setkey,
1811                         .setauthsize = aead_setauthsize,
1812                         .encrypt = aead_encrypt,
1813                         .decrypt = aead_decrypt,
1814                         .givencrypt = aead_null_givencrypt,
1815                         .geniv = "<built-in>",
1816                         .ivsize = NULL_IV_SIZE,
1817                         .maxauthsize = MD5_DIGEST_SIZE,
1818                         },
1819                 .class1_alg_type = 0,
1820                 .class2_alg_type = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC_PRECOMP,
1821                 .alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC,
1822         },
1823         {
1824                 .name = "authenc(hmac(sha1),ecb(cipher_null))",
1825                 .driver_name = "authenc-hmac-sha1-ecb-cipher_null-caam",
1826                 .blocksize = NULL_BLOCK_SIZE,
1827                 .type = CRYPTO_ALG_TYPE_AEAD,
1828                 .template_aead = {
1829                         .setkey = aead_setkey,
1830                         .setauthsize = aead_setauthsize,
1831                         .encrypt = aead_encrypt,
1832                         .decrypt = aead_decrypt,
1833                         .givencrypt = aead_null_givencrypt,
1834                         .geniv = "<built-in>",
1835                         .ivsize = NULL_IV_SIZE,
1836                         .maxauthsize = SHA1_DIGEST_SIZE,
1837                         },
1838                 .class1_alg_type = 0,
1839                 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC_PRECOMP,
1840                 .alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC,
1841         },
1842         {
1843                 .name = "authenc(hmac(sha224),ecb(cipher_null))",
1844                 .driver_name = "authenc-hmac-sha224-ecb-cipher_null-caam",
1845                 .blocksize = NULL_BLOCK_SIZE,
1846                 .type = CRYPTO_ALG_TYPE_AEAD,
1847                 .template_aead = {
1848                         .setkey = aead_setkey,
1849                         .setauthsize = aead_setauthsize,
1850                         .encrypt = aead_encrypt,
1851                         .decrypt = aead_decrypt,
1852                         .givencrypt = aead_null_givencrypt,
1853                         .geniv = "<built-in>",
1854                         .ivsize = NULL_IV_SIZE,
1855                         .maxauthsize = SHA224_DIGEST_SIZE,
1856                         },
1857                 .class1_alg_type = 0,
1858                 .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
1859                                    OP_ALG_AAI_HMAC_PRECOMP,
1860                 .alg_op = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC,
1861         },
1862         {
1863                 .name = "authenc(hmac(sha256),ecb(cipher_null))",
1864                 .driver_name = "authenc-hmac-sha256-ecb-cipher_null-caam",
1865                 .blocksize = NULL_BLOCK_SIZE,
1866                 .type = CRYPTO_ALG_TYPE_AEAD,
1867                 .template_aead = {
1868                         .setkey = aead_setkey,
1869                         .setauthsize = aead_setauthsize,
1870                         .encrypt = aead_encrypt,
1871                         .decrypt = aead_decrypt,
1872                         .givencrypt = aead_null_givencrypt,
1873                         .geniv = "<built-in>",
1874                         .ivsize = NULL_IV_SIZE,
1875                         .maxauthsize = SHA256_DIGEST_SIZE,
1876                         },
1877                 .class1_alg_type = 0,
1878                 .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
1879                                    OP_ALG_AAI_HMAC_PRECOMP,
1880                 .alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC,
1881         },
1882         {
1883                 .name = "authenc(hmac(sha384),ecb(cipher_null))",
1884                 .driver_name = "authenc-hmac-sha384-ecb-cipher_null-caam",
1885                 .blocksize = NULL_BLOCK_SIZE,
1886                 .type = CRYPTO_ALG_TYPE_AEAD,
1887                 .template_aead = {
1888                         .setkey = aead_setkey,
1889                         .setauthsize = aead_setauthsize,
1890                         .encrypt = aead_encrypt,
1891                         .decrypt = aead_decrypt,
1892                         .givencrypt = aead_null_givencrypt,
1893                         .geniv = "<built-in>",
1894                         .ivsize = NULL_IV_SIZE,
1895                         .maxauthsize = SHA384_DIGEST_SIZE,
1896                         },
1897                 .class1_alg_type = 0,
1898                 .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
1899                                    OP_ALG_AAI_HMAC_PRECOMP,
1900                 .alg_op = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC,
1901         },
1902         {
1903                 .name = "authenc(hmac(sha512),ecb(cipher_null))",
1904                 .driver_name = "authenc-hmac-sha512-ecb-cipher_null-caam",
1905                 .blocksize = NULL_BLOCK_SIZE,
1906                 .type = CRYPTO_ALG_TYPE_AEAD,
1907                 .template_aead = {
1908                         .setkey = aead_setkey,
1909                         .setauthsize = aead_setauthsize,
1910                         .encrypt = aead_encrypt,
1911                         .decrypt = aead_decrypt,
1912                         .givencrypt = aead_null_givencrypt,
1913                         .geniv = "<built-in>",
1914                         .ivsize = NULL_IV_SIZE,
1915                         .maxauthsize = SHA512_DIGEST_SIZE,
1916                         },
1917                 .class1_alg_type = 0,
1918                 .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
1919                                    OP_ALG_AAI_HMAC_PRECOMP,
1920                 .alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC,
1921         },
1922         {
1923                 .name = "authenc(hmac(md5),cbc(aes))",
1924                 .driver_name = "authenc-hmac-md5-cbc-aes-caam",
1925                 .blocksize = AES_BLOCK_SIZE,
1926                 .type = CRYPTO_ALG_TYPE_AEAD,
1927                 .template_aead = {
1928                         .setkey = aead_setkey,
1929                         .setauthsize = aead_setauthsize,
1930                         .encrypt = aead_encrypt,
1931                         .decrypt = aead_decrypt,
1932                         .givencrypt = aead_givencrypt,
1933                         .geniv = "<built-in>",
1934                         .ivsize = AES_BLOCK_SIZE,
1935                         .maxauthsize = MD5_DIGEST_SIZE,
1936                         },
1937                 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
1938                 .class2_alg_type = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC_PRECOMP,
1939                 .alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC,
1940         },
1941         {
1942                 .name = "authenc(hmac(sha1),cbc(aes))",
1943                 .driver_name = "authenc-hmac-sha1-cbc-aes-caam",
1944                 .blocksize = AES_BLOCK_SIZE,
1945                 .type = CRYPTO_ALG_TYPE_AEAD,
1946                 .template_aead = {
1947                         .setkey = aead_setkey,
1948                         .setauthsize = aead_setauthsize,
1949                         .encrypt = aead_encrypt,
1950                         .decrypt = aead_decrypt,
1951                         .givencrypt = aead_givencrypt,
1952                         .geniv = "<built-in>",
1953                         .ivsize = AES_BLOCK_SIZE,
1954                         .maxauthsize = SHA1_DIGEST_SIZE,
1955                         },
1956                 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
1957                 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC_PRECOMP,
1958                 .alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC,
1959         },
1960         {
1961                 .name = "authenc(hmac(sha224),cbc(aes))",
1962                 .driver_name = "authenc-hmac-sha224-cbc-aes-caam",
1963                 .blocksize = AES_BLOCK_SIZE,
1964                 .type = CRYPTO_ALG_TYPE_AEAD,
1965                 .template_aead = {
1966                         .setkey = aead_setkey,
1967                         .setauthsize = aead_setauthsize,
1968                         .encrypt = aead_encrypt,
1969                         .decrypt = aead_decrypt,
1970                         .givencrypt = aead_givencrypt,
1971                         .geniv = "<built-in>",
1972                         .ivsize = AES_BLOCK_SIZE,
1973                         .maxauthsize = SHA224_DIGEST_SIZE,
1974                         },
1975                 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
1976                 .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
1977                                    OP_ALG_AAI_HMAC_PRECOMP,
1978                 .alg_op = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC,
1979         },
1980         {
1981                 .name = "authenc(hmac(sha256),cbc(aes))",
1982                 .driver_name = "authenc-hmac-sha256-cbc-aes-caam",
1983                 .blocksize = AES_BLOCK_SIZE,
1984                 .type = CRYPTO_ALG_TYPE_AEAD,
1985                 .template_aead = {
1986                         .setkey = aead_setkey,
1987                         .setauthsize = aead_setauthsize,
1988                         .encrypt = aead_encrypt,
1989                         .decrypt = aead_decrypt,
1990                         .givencrypt = aead_givencrypt,
1991                         .geniv = "<built-in>",
1992                         .ivsize = AES_BLOCK_SIZE,
1993                         .maxauthsize = SHA256_DIGEST_SIZE,
1994                         },
1995                 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
1996                 .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
1997                                    OP_ALG_AAI_HMAC_PRECOMP,
1998                 .alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC,
1999         },
2000         {
2001                 .name = "authenc(hmac(sha384),cbc(aes))",
2002                 .driver_name = "authenc-hmac-sha384-cbc-aes-caam",
2003                 .blocksize = AES_BLOCK_SIZE,
2004                 .type = CRYPTO_ALG_TYPE_AEAD,
2005                 .template_aead = {
2006                         .setkey = aead_setkey,
2007                         .setauthsize = aead_setauthsize,
2008                         .encrypt = aead_encrypt,
2009                         .decrypt = aead_decrypt,
2010                         .givencrypt = aead_givencrypt,
2011                         .geniv = "<built-in>",
2012                         .ivsize = AES_BLOCK_SIZE,
2013                         .maxauthsize = SHA384_DIGEST_SIZE,
2014                         },
2015                 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
2016                 .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
2017                                    OP_ALG_AAI_HMAC_PRECOMP,
2018                 .alg_op = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC,
2019         },
2020
2021         {
2022                 .name = "authenc(hmac(sha512),cbc(aes))",
2023                 .driver_name = "authenc-hmac-sha512-cbc-aes-caam",
2024                 .blocksize = AES_BLOCK_SIZE,
2025                 .type = CRYPTO_ALG_TYPE_AEAD,
2026                 .template_aead = {
2027                         .setkey = aead_setkey,
2028                         .setauthsize = aead_setauthsize,
2029                         .encrypt = aead_encrypt,
2030                         .decrypt = aead_decrypt,
2031                         .givencrypt = aead_givencrypt,
2032                         .geniv = "<built-in>",
2033                         .ivsize = AES_BLOCK_SIZE,
2034                         .maxauthsize = SHA512_DIGEST_SIZE,
2035                         },
2036                 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
2037                 .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
2038                                    OP_ALG_AAI_HMAC_PRECOMP,
2039                 .alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC,
2040         },
2041         {
2042                 .name = "authenc(hmac(md5),cbc(des3_ede))",
2043                 .driver_name = "authenc-hmac-md5-cbc-des3_ede-caam",
2044                 .blocksize = DES3_EDE_BLOCK_SIZE,
2045                 .type = CRYPTO_ALG_TYPE_AEAD,
2046                 .template_aead = {
2047                         .setkey = aead_setkey,
2048                         .setauthsize = aead_setauthsize,
2049                         .encrypt = aead_encrypt,
2050                         .decrypt = aead_decrypt,
2051                         .givencrypt = aead_givencrypt,
2052                         .geniv = "<built-in>",
2053                         .ivsize = DES3_EDE_BLOCK_SIZE,
2054                         .maxauthsize = MD5_DIGEST_SIZE,
2055                         },
2056                 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2057                 .class2_alg_type = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC_PRECOMP,
2058                 .alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC,
2059         },
2060         {
2061                 .name = "authenc(hmac(sha1),cbc(des3_ede))",
2062                 .driver_name = "authenc-hmac-sha1-cbc-des3_ede-caam",
2063                 .blocksize = DES3_EDE_BLOCK_SIZE,
2064                 .type = CRYPTO_ALG_TYPE_AEAD,
2065                 .template_aead = {
2066                         .setkey = aead_setkey,
2067                         .setauthsize = aead_setauthsize,
2068                         .encrypt = aead_encrypt,
2069                         .decrypt = aead_decrypt,
2070                         .givencrypt = aead_givencrypt,
2071                         .geniv = "<built-in>",
2072                         .ivsize = DES3_EDE_BLOCK_SIZE,
2073                         .maxauthsize = SHA1_DIGEST_SIZE,
2074                         },
2075                 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2076                 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC_PRECOMP,
2077                 .alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC,
2078         },
2079         {
2080                 .name = "authenc(hmac(sha224),cbc(des3_ede))",
2081                 .driver_name = "authenc-hmac-sha224-cbc-des3_ede-caam",
2082                 .blocksize = DES3_EDE_BLOCK_SIZE,
2083                 .type = CRYPTO_ALG_TYPE_AEAD,
2084                 .template_aead = {
2085                         .setkey = aead_setkey,
2086                         .setauthsize = aead_setauthsize,
2087                         .encrypt = aead_encrypt,
2088                         .decrypt = aead_decrypt,
2089                         .givencrypt = aead_givencrypt,
2090                         .geniv = "<built-in>",
2091                         .ivsize = DES3_EDE_BLOCK_SIZE,
2092                         .maxauthsize = SHA224_DIGEST_SIZE,
2093                         },
2094                 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2095                 .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
2096                                    OP_ALG_AAI_HMAC_PRECOMP,
2097                 .alg_op = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC,
2098         },
2099         {
2100                 .name = "authenc(hmac(sha256),cbc(des3_ede))",
2101                 .driver_name = "authenc-hmac-sha256-cbc-des3_ede-caam",
2102                 .blocksize = DES3_EDE_BLOCK_SIZE,
2103                 .type = CRYPTO_ALG_TYPE_AEAD,
2104                 .template_aead = {
2105                         .setkey = aead_setkey,
2106                         .setauthsize = aead_setauthsize,
2107                         .encrypt = aead_encrypt,
2108                         .decrypt = aead_decrypt,
2109                         .givencrypt = aead_givencrypt,
2110                         .geniv = "<built-in>",
2111                         .ivsize = DES3_EDE_BLOCK_SIZE,
2112                         .maxauthsize = SHA256_DIGEST_SIZE,
2113                         },
2114                 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2115                 .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
2116                                    OP_ALG_AAI_HMAC_PRECOMP,
2117                 .alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC,
2118         },
2119         {
2120                 .name = "authenc(hmac(sha384),cbc(des3_ede))",
2121                 .driver_name = "authenc-hmac-sha384-cbc-des3_ede-caam",
2122                 .blocksize = DES3_EDE_BLOCK_SIZE,
2123                 .type = CRYPTO_ALG_TYPE_AEAD,
2124                 .template_aead = {
2125                         .setkey = aead_setkey,
2126                         .setauthsize = aead_setauthsize,
2127                         .encrypt = aead_encrypt,
2128                         .decrypt = aead_decrypt,
2129                         .givencrypt = aead_givencrypt,
2130                         .geniv = "<built-in>",
2131                         .ivsize = DES3_EDE_BLOCK_SIZE,
2132                         .maxauthsize = SHA384_DIGEST_SIZE,
2133                         },
2134                 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2135                 .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
2136                                    OP_ALG_AAI_HMAC_PRECOMP,
2137                 .alg_op = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC,
2138         },
2139         {
2140                 .name = "authenc(hmac(sha512),cbc(des3_ede))",
2141                 .driver_name = "authenc-hmac-sha512-cbc-des3_ede-caam",
2142                 .blocksize = DES3_EDE_BLOCK_SIZE,
2143                 .type = CRYPTO_ALG_TYPE_AEAD,
2144                 .template_aead = {
2145                         .setkey = aead_setkey,
2146                         .setauthsize = aead_setauthsize,
2147                         .encrypt = aead_encrypt,
2148                         .decrypt = aead_decrypt,
2149                         .givencrypt = aead_givencrypt,
2150                         .geniv = "<built-in>",
2151                         .ivsize = DES3_EDE_BLOCK_SIZE,
2152                         .maxauthsize = SHA512_DIGEST_SIZE,
2153                         },
2154                 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2155                 .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
2156                                    OP_ALG_AAI_HMAC_PRECOMP,
2157                 .alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC,
2158         },
2159         {
2160                 .name = "authenc(hmac(md5),cbc(des))",
2161                 .driver_name = "authenc-hmac-md5-cbc-des-caam",
2162                 .blocksize = DES_BLOCK_SIZE,
2163                 .type = CRYPTO_ALG_TYPE_AEAD,
2164                 .template_aead = {
2165                         .setkey = aead_setkey,
2166                         .setauthsize = aead_setauthsize,
2167                         .encrypt = aead_encrypt,
2168                         .decrypt = aead_decrypt,
2169                         .givencrypt = aead_givencrypt,
2170                         .geniv = "<built-in>",
2171                         .ivsize = DES_BLOCK_SIZE,
2172                         .maxauthsize = MD5_DIGEST_SIZE,
2173                         },
2174                 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2175                 .class2_alg_type = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC_PRECOMP,
2176                 .alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC,
2177         },
2178         {
2179                 .name = "authenc(hmac(sha1),cbc(des))",
2180                 .driver_name = "authenc-hmac-sha1-cbc-des-caam",
2181                 .blocksize = DES_BLOCK_SIZE,
2182                 .type = CRYPTO_ALG_TYPE_AEAD,
2183                 .template_aead = {
2184                         .setkey = aead_setkey,
2185                         .setauthsize = aead_setauthsize,
2186                         .encrypt = aead_encrypt,
2187                         .decrypt = aead_decrypt,
2188                         .givencrypt = aead_givencrypt,
2189                         .geniv = "<built-in>",
2190                         .ivsize = DES_BLOCK_SIZE,
2191                         .maxauthsize = SHA1_DIGEST_SIZE,
2192                         },
2193                 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2194                 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC_PRECOMP,
2195                 .alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC,
2196         },
2197         {
2198                 .name = "authenc(hmac(sha224),cbc(des))",
2199                 .driver_name = "authenc-hmac-sha224-cbc-des-caam",
2200                 .blocksize = DES_BLOCK_SIZE,
2201                 .type = CRYPTO_ALG_TYPE_AEAD,
2202                 .template_aead = {
2203                         .setkey = aead_setkey,
2204                         .setauthsize = aead_setauthsize,
2205                         .encrypt = aead_encrypt,
2206                         .decrypt = aead_decrypt,
2207                         .givencrypt = aead_givencrypt,
2208                         .geniv = "<built-in>",
2209                         .ivsize = DES_BLOCK_SIZE,
2210                         .maxauthsize = SHA224_DIGEST_SIZE,
2211                         },
2212                 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2213                 .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
2214                                    OP_ALG_AAI_HMAC_PRECOMP,
2215                 .alg_op = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC,
2216         },
2217         {
2218                 .name = "authenc(hmac(sha256),cbc(des))",
2219                 .driver_name = "authenc-hmac-sha256-cbc-des-caam",
2220                 .blocksize = DES_BLOCK_SIZE,
2221                 .type = CRYPTO_ALG_TYPE_AEAD,
2222                 .template_aead = {
2223                         .setkey = aead_setkey,
2224                         .setauthsize = aead_setauthsize,
2225                         .encrypt = aead_encrypt,
2226                         .decrypt = aead_decrypt,
2227                         .givencrypt = aead_givencrypt,
2228                         .geniv = "<built-in>",
2229                         .ivsize = DES_BLOCK_SIZE,
2230                         .maxauthsize = SHA256_DIGEST_SIZE,
2231                         },
2232                 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2233                 .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
2234                                    OP_ALG_AAI_HMAC_PRECOMP,
2235                 .alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC,
2236         },
2237         {
2238                 .name = "authenc(hmac(sha384),cbc(des))",
2239                 .driver_name = "authenc-hmac-sha384-cbc-des-caam",
2240                 .blocksize = DES_BLOCK_SIZE,
2241                 .type = CRYPTO_ALG_TYPE_AEAD,
2242                 .template_aead = {
2243                         .setkey = aead_setkey,
2244                         .setauthsize = aead_setauthsize,
2245                         .encrypt = aead_encrypt,
2246                         .decrypt = aead_decrypt,
2247                         .givencrypt = aead_givencrypt,
2248                         .geniv = "<built-in>",
2249                         .ivsize = DES_BLOCK_SIZE,
2250                         .maxauthsize = SHA384_DIGEST_SIZE,
2251                         },
2252                 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2253                 .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
2254                                    OP_ALG_AAI_HMAC_PRECOMP,
2255                 .alg_op = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC,
2256         },
2257         {
2258                 .name = "authenc(hmac(sha512),cbc(des))",
2259                 .driver_name = "authenc-hmac-sha512-cbc-des-caam",
2260                 .blocksize = DES_BLOCK_SIZE,
2261                 .type = CRYPTO_ALG_TYPE_AEAD,
2262                 .template_aead = {
2263                         .setkey = aead_setkey,
2264                         .setauthsize = aead_setauthsize,
2265                         .encrypt = aead_encrypt,
2266                         .decrypt = aead_decrypt,
2267                         .givencrypt = aead_givencrypt,
2268                         .geniv = "<built-in>",
2269                         .ivsize = DES_BLOCK_SIZE,
2270                         .maxauthsize = SHA512_DIGEST_SIZE,
2271                         },
2272                 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2273                 .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
2274                                    OP_ALG_AAI_HMAC_PRECOMP,
2275                 .alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC,
2276         },
2277         /* ablkcipher descriptor */
2278         {
2279                 .name = "cbc(aes)",
2280                 .driver_name = "cbc-aes-caam",
2281                 .blocksize = AES_BLOCK_SIZE,
2282                 .type = CRYPTO_ALG_TYPE_ABLKCIPHER,
2283                 .template_ablkcipher = {
2284                         .setkey = ablkcipher_setkey,
2285                         .encrypt = ablkcipher_encrypt,
2286                         .decrypt = ablkcipher_decrypt,
2287                         .geniv = "eseqiv",
2288                         .min_keysize = AES_MIN_KEY_SIZE,
2289                         .max_keysize = AES_MAX_KEY_SIZE,
2290                         .ivsize = AES_BLOCK_SIZE,
2291                         },
2292                 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
2293         },
2294         {
2295                 .name = "cbc(des3_ede)",
2296                 .driver_name = "cbc-3des-caam",
2297                 .blocksize = DES3_EDE_BLOCK_SIZE,
2298                 .type = CRYPTO_ALG_TYPE_ABLKCIPHER,
2299                 .template_ablkcipher = {
2300                         .setkey = ablkcipher_setkey,
2301                         .encrypt = ablkcipher_encrypt,
2302                         .decrypt = ablkcipher_decrypt,
2303                         .geniv = "eseqiv",
2304                         .min_keysize = DES3_EDE_KEY_SIZE,
2305                         .max_keysize = DES3_EDE_KEY_SIZE,
2306                         .ivsize = DES3_EDE_BLOCK_SIZE,
2307                         },
2308                 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2309         },
2310         {
2311                 .name = "cbc(des)",
2312                 .driver_name = "cbc-des-caam",
2313                 .blocksize = DES_BLOCK_SIZE,
2314                 .type = CRYPTO_ALG_TYPE_ABLKCIPHER,
2315                 .template_ablkcipher = {
2316                         .setkey = ablkcipher_setkey,
2317                         .encrypt = ablkcipher_encrypt,
2318                         .decrypt = ablkcipher_decrypt,
2319                         .geniv = "eseqiv",
2320                         .min_keysize = DES_KEY_SIZE,
2321                         .max_keysize = DES_KEY_SIZE,
2322                         .ivsize = DES_BLOCK_SIZE,
2323                         },
2324                 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2325         }
2326 };
2327
2328 struct caam_crypto_alg {
2329         struct list_head entry;
2330         int class1_alg_type;
2331         int class2_alg_type;
2332         int alg_op;
2333         struct crypto_alg crypto_alg;
2334 };
2335
2336 static int caam_cra_init(struct crypto_tfm *tfm)
2337 {
2338         struct crypto_alg *alg = tfm->__crt_alg;
2339         struct caam_crypto_alg *caam_alg =
2340                  container_of(alg, struct caam_crypto_alg, crypto_alg);
2341         struct caam_ctx *ctx = crypto_tfm_ctx(tfm);
2342
2343         ctx->jrdev = caam_jr_alloc();
2344         if (IS_ERR(ctx->jrdev)) {
2345                 pr_err("Job Ring Device allocation for transform failed\n");
2346                 return PTR_ERR(ctx->jrdev);
2347         }
2348
2349         /* copy descriptor header template value */
2350         ctx->class1_alg_type = OP_TYPE_CLASS1_ALG | caam_alg->class1_alg_type;
2351         ctx->class2_alg_type = OP_TYPE_CLASS2_ALG | caam_alg->class2_alg_type;
2352         ctx->alg_op = OP_TYPE_CLASS2_ALG | caam_alg->alg_op;
2353
2354         return 0;
2355 }
2356
2357 static void caam_cra_exit(struct crypto_tfm *tfm)
2358 {
2359         struct caam_ctx *ctx = crypto_tfm_ctx(tfm);
2360
2361         if (ctx->sh_desc_enc_dma &&
2362             !dma_mapping_error(ctx->jrdev, ctx->sh_desc_enc_dma))
2363                 dma_unmap_single(ctx->jrdev, ctx->sh_desc_enc_dma,
2364                                  desc_bytes(ctx->sh_desc_enc), DMA_TO_DEVICE);
2365         if (ctx->sh_desc_dec_dma &&
2366             !dma_mapping_error(ctx->jrdev, ctx->sh_desc_dec_dma))
2367                 dma_unmap_single(ctx->jrdev, ctx->sh_desc_dec_dma,
2368                                  desc_bytes(ctx->sh_desc_dec), DMA_TO_DEVICE);
2369         if (ctx->sh_desc_givenc_dma &&
2370             !dma_mapping_error(ctx->jrdev, ctx->sh_desc_givenc_dma))
2371                 dma_unmap_single(ctx->jrdev, ctx->sh_desc_givenc_dma,
2372                                  desc_bytes(ctx->sh_desc_givenc),
2373                                  DMA_TO_DEVICE);
2374         if (ctx->key_dma &&
2375             !dma_mapping_error(ctx->jrdev, ctx->key_dma))
2376                 dma_unmap_single(ctx->jrdev, ctx->key_dma,
2377                                  ctx->enckeylen + ctx->split_key_pad_len,
2378                                  DMA_TO_DEVICE);
2379
2380         caam_jr_free(ctx->jrdev);
2381 }
2382
2383 static void __exit caam_algapi_exit(void)
2384 {
2385
2386         struct caam_crypto_alg *t_alg, *n;
2387
2388         if (!alg_list.next)
2389                 return;
2390
2391         list_for_each_entry_safe(t_alg, n, &alg_list, entry) {
2392                 crypto_unregister_alg(&t_alg->crypto_alg);
2393                 list_del(&t_alg->entry);
2394                 kfree(t_alg);
2395         }
2396 }
2397
2398 static struct caam_crypto_alg *caam_alg_alloc(struct caam_alg_template
2399                                               *template)
2400 {
2401         struct caam_crypto_alg *t_alg;
2402         struct crypto_alg *alg;
2403
2404         t_alg = kzalloc(sizeof(struct caam_crypto_alg), GFP_KERNEL);
2405         if (!t_alg) {
2406                 pr_err("failed to allocate t_alg\n");
2407                 return ERR_PTR(-ENOMEM);
2408         }
2409
2410         alg = &t_alg->crypto_alg;
2411
2412         snprintf(alg->cra_name, CRYPTO_MAX_ALG_NAME, "%s", template->name);
2413         snprintf(alg->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s",
2414                  template->driver_name);
2415         alg->cra_module = THIS_MODULE;
2416         alg->cra_init = caam_cra_init;
2417         alg->cra_exit = caam_cra_exit;
2418         alg->cra_priority = CAAM_CRA_PRIORITY;
2419         alg->cra_blocksize = template->blocksize;
2420         alg->cra_alignmask = 0;
2421         alg->cra_ctxsize = sizeof(struct caam_ctx);
2422         alg->cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY |
2423                          template->type;
2424         switch (template->type) {
2425         case CRYPTO_ALG_TYPE_ABLKCIPHER:
2426                 alg->cra_type = &crypto_ablkcipher_type;
2427                 alg->cra_ablkcipher = template->template_ablkcipher;
2428                 break;
2429         case CRYPTO_ALG_TYPE_AEAD:
2430                 alg->cra_type = &crypto_aead_type;
2431                 alg->cra_aead = template->template_aead;
2432                 break;
2433         }
2434
2435         t_alg->class1_alg_type = template->class1_alg_type;
2436         t_alg->class2_alg_type = template->class2_alg_type;
2437         t_alg->alg_op = template->alg_op;
2438
2439         return t_alg;
2440 }
2441
2442 static int __init caam_algapi_init(void)
2443 {
2444         int i = 0, err = 0;
2445
2446         INIT_LIST_HEAD(&alg_list);
2447
2448         /* register crypto algorithms the device supports */
2449         for (i = 0; i < ARRAY_SIZE(driver_algs); i++) {
2450                 /* TODO: check if h/w supports alg */
2451                 struct caam_crypto_alg *t_alg;
2452
2453                 t_alg = caam_alg_alloc(&driver_algs[i]);
2454                 if (IS_ERR(t_alg)) {
2455                         err = PTR_ERR(t_alg);
2456                         pr_warn("%s alg allocation failed\n",
2457                                 driver_algs[i].driver_name);
2458                         continue;
2459                 }
2460
2461                 err = crypto_register_alg(&t_alg->crypto_alg);
2462                 if (err) {
2463                         pr_warn("%s alg registration failed\n",
2464                                 t_alg->crypto_alg.cra_driver_name);
2465                         kfree(t_alg);
2466                 } else
2467                         list_add_tail(&t_alg->entry, &alg_list);
2468         }
2469         if (!list_empty(&alg_list))
2470                 pr_info("caam algorithms registered in /proc/crypto\n");
2471
2472         return err;
2473 }
2474
2475 module_init(caam_algapi_init);
2476 module_exit(caam_algapi_exit);
2477
2478 MODULE_LICENSE("GPL");
2479 MODULE_DESCRIPTION("FSL CAAM support for crypto API");
2480 MODULE_AUTHOR("Freescale Semiconductor - NMG/STC");