drivers/crypto/talitos.c

   1 /*
   2  * talitos - Freescale Integrated Security Engine (SEC) device driver
   3  *
   4  * Copyright (c) 2008-2011 Freescale Semiconductor, Inc.
   5  *
   6  * Scatterlist Crypto API glue code copied from files with the following:
   7  * Copyright (c) 2006-2007 Herbert Xu <herbert@gondor.apana.org.au>
   8  *
   9  * Crypto algorithm registration code copied from hifn driver:
  10  * 2007+ Copyright (c) Evgeniy Polyakov <johnpol@2ka.mipt.ru>
  11  * All rights reserved.
  12  *
  13  * This program is free software; you can redistribute it and/or modify
  14  * it under the terms of the GNU General Public License as published by
  15  * the Free Software Foundation; either version 2 of the License, or
  16  * (at your option) any later version.
  17  *
  18  * This program is distributed in the hope that it will be useful,
  19  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  20  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  21  * GNU General Public License for more details.
  22  *
  23  * You should have received a copy of the GNU General Public License
  24  * along with this program; if not, write to the Free Software
  25  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  26  */
  27
  28 #include <linux/kernel.h>
  29 #include <linux/module.h>
  30 #include <linux/mod_devicetable.h>
  31 #include <linux/device.h>
  32 #include <linux/interrupt.h>
  33 #include <linux/crypto.h>
  34 #include <linux/hw_random.h>
  35 #include <linux/of_address.h>
  36 #include <linux/of_irq.h>
  37 #include <linux/of_platform.h>
  38 #include <linux/dma-mapping.h>
  39 #include <linux/io.h>
  40 #include <linux/spinlock.h>
  41 #include <linux/rtnetlink.h>
  42 #include <linux/slab.h>
  43
  44 #include <crypto/algapi.h>
  45 #include <crypto/aes.h>
  46 #include <crypto/des.h>
  47 #include <crypto/sha.h>
  48 #include <crypto/md5.h>
  49 #include <crypto/aead.h>
  50 #include <crypto/authenc.h>
  51 #include <crypto/skcipher.h>
  52 #include <crypto/hash.h>
  53 #include <crypto/internal/hash.h>
  54 #include <crypto/scatterwalk.h>
  55
  56 #include "talitos.h"
  57
  58 static void to_talitos_ptr(struct talitos_ptr *ptr, dma_addr_t dma_addr,
  59                            bool is_sec1)
  60 {
  61         ptr->ptr = cpu_to_be32(lower_32_bits(dma_addr));
  62         if (!is_sec1)
  63                 ptr->eptr = upper_32_bits(dma_addr);
  64 }
  65
  66 static void to_talitos_ptr_len(struct talitos_ptr *ptr, unsigned short len,
  67                                bool is_sec1)
  68 {
  69         if (is_sec1) {
  70                 ptr->res = 0;
  71                 ptr->len1 = cpu_to_be16(len);
  72         } else {
  73                 ptr->len = cpu_to_be16(len);
  74         }
  75 }
  76
  77 static unsigned short from_talitos_ptr_len(struct talitos_ptr *ptr,
  78                                            bool is_sec1)
  79 {
  80         if (is_sec1)
  81                 return be16_to_cpu(ptr->len1);
  82         else
  83                 return be16_to_cpu(ptr->len);
  84 }
  85
  86 static void to_talitos_ptr_extent_clear(struct talitos_ptr *ptr, bool is_sec1)
  87 {
  88         if (!is_sec1)
  89                 ptr->j_extent = 0;
  90 }
  91
  92 /*
  93  * map virtual single (contiguous) pointer to h/w descriptor pointer
  94  */
  95 static void map_single_talitos_ptr(struct device *dev,
  96                                    struct talitos_ptr *ptr,
  97                                    unsigned short len, void *data,
  98                                    enum dma_data_direction dir)
  99 {
 100         dma_addr_t dma_addr = dma_map_single(dev, data, len, dir);
 101         struct talitos_private *priv = dev_get_drvdata(dev);
 102         bool is_sec1 = has_ftr_sec1(priv);
 103
 104         to_talitos_ptr_len(ptr, len, is_sec1);
 105         to_talitos_ptr(ptr, dma_addr, is_sec1);
 106         to_talitos_ptr_extent_clear(ptr, is_sec1);
 107 }
 108
 109 /*
 110  * unmap bus single (contiguous) h/w descriptor pointer
 111  */
 112 static void unmap_single_talitos_ptr(struct device *dev,
 113                                      struct talitos_ptr *ptr,
 114                                      enum dma_data_direction dir)
 115 {
 116         struct talitos_private *priv = dev_get_drvdata(dev);
 117         bool is_sec1 = has_ftr_sec1(priv);
 118
 119         dma_unmap_single(dev, be32_to_cpu(ptr->ptr),
 120                          from_talitos_ptr_len(ptr, is_sec1), dir);
 121 }
 122
 123 static int reset_channel(struct device *dev, int ch)
 124 {
 125         struct talitos_private *priv = dev_get_drvdata(dev);
 126         unsigned int timeout = TALITOS_TIMEOUT;
 127
 128         setbits32(priv->chan[ch].reg + TALITOS_CCCR, TALITOS_CCCR_RESET);
 129
 130         while ((in_be32(priv->chan[ch].reg + TALITOS_CCCR) & TALITOS_CCCR_RESET)
 131                && --timeout)
 132                 cpu_relax();
 133
 134         if (timeout == 0) {
 135                 dev_err(dev, "failed to reset channel %d\n", ch);
 136                 return -EIO;
 137         }
 138
 139         /* set 36-bit addressing, done writeback enable and done IRQ enable */
 140         setbits32(priv->chan[ch].reg + TALITOS_CCCR_LO, TALITOS_CCCR_LO_EAE |
 141                   TALITOS_CCCR_LO_CDWE | TALITOS_CCCR_LO_CDIE);
 142
 143         /* and ICCR writeback, if available */
 144         if (priv->features & TALITOS_FTR_HW_AUTH_CHECK)
 145                 setbits32(priv->chan[ch].reg + TALITOS_CCCR_LO,
 146                           TALITOS_CCCR_LO_IWSE);
 147
 148         return 0;
 149 }
 150
 151 static int reset_device(struct device *dev)
 152 {
 153         struct talitos_private *priv = dev_get_drvdata(dev);
 154         unsigned int timeout = TALITOS_TIMEOUT;
 155         u32 mcr = TALITOS_MCR_SWR;
 156
 157         setbits32(priv->reg + TALITOS_MCR, mcr);
 158
 159         while ((in_be32(priv->reg + TALITOS_MCR) & TALITOS_MCR_SWR)
 160                && --timeout)
 161                 cpu_relax();
 162
 163         if (priv->irq[1]) {
 164                 mcr = TALITOS_MCR_RCA1 | TALITOS_MCR_RCA3;
 165                 setbits32(priv->reg + TALITOS_MCR, mcr);
 166         }
 167
 168         if (timeout == 0) {
 169                 dev_err(dev, "failed to reset device\n");
 170                 return -EIO;
 171         }
 172
 173         return 0;
 174 }
 175
 176 /*
 177  * Reset and initialize the device
 178  */
 179 static int init_device(struct device *dev)
 180 {
 181         struct talitos_private *priv = dev_get_drvdata(dev);
 182         int ch, err;
 183
 184         /*
 185          * Master reset
 186          * errata documentation: warning: certain SEC interrupts
 187          * are not fully cleared by writing the MCR:SWR bit,
 188          * set bit twice to completely reset
 189          */
 190         err = reset_device(dev);
 191         if (err)
 192                 return err;
 193
 194         err = reset_device(dev);
 195         if (err)
 196                 return err;
 197
 198         /* reset channels */
 199         for (ch = 0; ch < priv->num_channels; ch++) {
 200                 err = reset_channel(dev, ch);
 201                 if (err)
 202                         return err;
 203         }
 204
 205         /* enable channel done and error interrupts */
 206         setbits32(priv->reg + TALITOS_IMR, TALITOS_IMR_INIT);
 207         setbits32(priv->reg + TALITOS_IMR_LO, TALITOS_IMR_LO_INIT);
 208
 209         /* disable integrity check error interrupts (use writeback instead) */
 210         if (priv->features & TALITOS_FTR_HW_AUTH_CHECK)
 211                 setbits32(priv->reg + TALITOS_MDEUICR_LO,
 212                           TALITOS_MDEUICR_LO_ICE);
 213
 214         return 0;
 215 }
 216
 217 /**
 218  * talitos_submit - submits a descriptor to the device for processing
 219  * @dev:        the SEC device to be used
 220  * @ch:         the SEC device channel to be used
 221  * @desc:       the descriptor to be processed by the device
 222  * @callback:   whom to call when processing is complete
 223  * @context:    a handle for use by caller (optional)
 224  *
 225  * desc must contain valid dma-mapped (bus physical) address pointers.
 226  * callback must check err and feedback in descriptor header
 227  * for device processing status.
 228  */
 229 int talitos_submit(struct device *dev, int ch, struct talitos_desc *desc,
 230                    void (*callback)(struct device *dev,
 231                                     struct talitos_desc *desc,
 232                                     void *context, int error),
 233                    void *context)
 234 {
 235         struct talitos_private *priv = dev_get_drvdata(dev);
 236         struct talitos_request *request;
 237         unsigned long flags;
 238         int head;
 239         bool is_sec1 = has_ftr_sec1(priv);
 240
 241         spin_lock_irqsave(&priv->chan[ch].head_lock, flags);
 242
 243         if (!atomic_inc_not_zero(&priv->chan[ch].submit_count)) {
 244                 /* h/w fifo is full */
 245                 spin_unlock_irqrestore(&priv->chan[ch].head_lock, flags);
 246                 return -EAGAIN;
 247         }
 248
 249         head = priv->chan[ch].head;
 250         request = &priv->chan[ch].fifo[head];
 251
 252         /* map descriptor and save caller data */
 253         if (is_sec1) {
 254                 desc->hdr1 = desc->hdr;
 255                 desc->next_desc = 0;
 256                 request->dma_desc = dma_map_single(dev, &desc->hdr1,
 257                                                    TALITOS_DESC_SIZE,
 258                                                    DMA_BIDIRECTIONAL);
 259         } else {
 260                 request->dma_desc = dma_map_single(dev, desc,
 261                                                    TALITOS_DESC_SIZE,
 262                                                    DMA_BIDIRECTIONAL);
 263         }
 264         request->callback = callback;
 265         request->context = context;
 266
 267         /* increment fifo head */
 268         priv->chan[ch].head = (priv->chan[ch].head + 1) & (priv->fifo_len - 1);
 269
 270         smp_wmb();
 271         request->desc = desc;
 272
 273         /* GO! */
 274         wmb();
 275         out_be32(priv->chan[ch].reg + TALITOS_FF,
 276                  upper_32_bits(request->dma_desc));
 277         out_be32(priv->chan[ch].reg + TALITOS_FF_LO,
 278                  lower_32_bits(request->dma_desc));
 279
 280         spin_unlock_irqrestore(&priv->chan[ch].head_lock, flags);
 281
 282         return -EINPROGRESS;
 283 }
 284 EXPORT_SYMBOL(talitos_submit);
 285
 286 /*
 287  * process what was done, notify callback of error if not
 288  */
 289 static void flush_channel(struct device *dev, int ch, int error, int reset_ch)
 290 {
 291         struct talitos_private *priv = dev_get_drvdata(dev);
 292         struct talitos_request *request, saved_req;
 293         unsigned long flags;
 294         int tail, status;
 295         bool is_sec1 = has_ftr_sec1(priv);
 296
 297         spin_lock_irqsave(&priv->chan[ch].tail_lock, flags);
 298
 299         tail = priv->chan[ch].tail;
 300         while (priv->chan[ch].fifo[tail].desc) {
 301                 __be32 hdr;
 302
 303                 request = &priv->chan[ch].fifo[tail];
 304
 305                 /* descriptors with their done bits set don't get the error */
 306                 rmb();
 307                 hdr = is_sec1 ? request->desc->hdr1 : request->desc->hdr;
 308
 309                 if ((hdr & DESC_HDR_DONE) == DESC_HDR_DONE)
 310                         status = 0;
 311                 else
 312                         if (!error)
 313                                 break;
 314                         else
 315                                 status = error;
 316
 317                 dma_unmap_single(dev, request->dma_desc,
 318                                  TALITOS_DESC_SIZE,
 319                                  DMA_BIDIRECTIONAL);
 320
 321                 /* copy entries so we can call callback outside lock */
 322                 saved_req.desc = request->desc;
 323                 saved_req.callback = request->callback;
 324                 saved_req.context = request->context;
 325
 326                 /* release request entry in fifo */
 327                 smp_wmb();
 328                 request->desc = NULL;
 329
 330                 /* increment fifo tail */
 331                 priv->chan[ch].tail = (tail + 1) & (priv->fifo_len - 1);
 332
 333                 spin_unlock_irqrestore(&priv->chan[ch].tail_lock, flags);
 334
 335                 atomic_dec(&priv->chan[ch].submit_count);
 336
 337                 saved_req.callback(dev, saved_req.desc, saved_req.context,
 338                                    status);
 339                 /* channel may resume processing in single desc error case */
 340                 if (error && !reset_ch && status == error)
 341                         return;
 342                 spin_lock_irqsave(&priv->chan[ch].tail_lock, flags);
 343                 tail = priv->chan[ch].tail;
 344         }
 345
 346         spin_unlock_irqrestore(&priv->chan[ch].tail_lock, flags);
 347 }
 348
 349 /*
 350  * process completed requests for channels that have done status
 351  */
 352 #define DEF_TALITOS_DONE(name, ch_done_mask)                            \
 353 static void talitos_done_##name(unsigned long data)                     \
 354 {                                                                       \
 355         struct device *dev = (struct device *)data;                     \
 356         struct talitos_private *priv = dev_get_drvdata(dev);            \
 357         unsigned long flags;                                            \
 358                                                                         \
 359         if (ch_done_mask & 1)                                           \
 360                 flush_channel(dev, 0, 0, 0);                            \
 361         if (priv->num_channels == 1)                                    \
 362                 goto out;                                               \
 363         if (ch_done_mask & (1 << 2))                                    \
 364                 flush_channel(dev, 1, 0, 0);                            \
 365         if (ch_done_mask & (1 << 4))                                    \
 366                 flush_channel(dev, 2, 0, 0);                            \
 367         if (ch_done_mask & (1 << 6))                                    \
 368                 flush_channel(dev, 3, 0, 0);                            \
 369                                                                         \
 370 out:                                                                    \
 371         /* At this point, all completed channels have been processed */ \
 372         /* Unmask done interrupts for channels completed later on. */   \
 373         spin_lock_irqsave(&priv->reg_lock, flags);                      \
 374         setbits32(priv->reg + TALITOS_IMR, ch_done_mask);               \
 375         setbits32(priv->reg + TALITOS_IMR_LO, TALITOS_IMR_LO_INIT);     \
 376         spin_unlock_irqrestore(&priv->reg_lock, flags);                 \
 377 }
 378 DEF_TALITOS_DONE(4ch, TALITOS_ISR_4CHDONE)
 379 DEF_TALITOS_DONE(ch0_2, TALITOS_ISR_CH_0_2_DONE)
 380 DEF_TALITOS_DONE(ch1_3, TALITOS_ISR_CH_1_3_DONE)
 381
 382 /*
 383  * locate current (offending) descriptor
 384  */
 385 static u32 current_desc_hdr(struct device *dev, int ch)
 386 {
 387         struct talitos_private *priv = dev_get_drvdata(dev);
 388         int tail, iter;
 389         dma_addr_t cur_desc;
 390
 391         cur_desc = ((u64)in_be32(priv->chan[ch].reg + TALITOS_CDPR)) << 32;
 392         cur_desc |= in_be32(priv->chan[ch].reg + TALITOS_CDPR_LO);
 393
 394         if (!cur_desc) {
 395                 dev_err(dev, "CDPR is NULL, giving up search for offending descriptor\n");
 396                 return 0;
 397         }
 398
 399         tail = priv->chan[ch].tail;
 400
 401         iter = tail;
 402         while (priv->chan[ch].fifo[iter].dma_desc != cur_desc) {
 403                 iter = (iter + 1) & (priv->fifo_len - 1);
 404                 if (iter == tail) {
 405                         dev_err(dev, "couldn't locate current descriptor\n");
 406                         return 0;
 407                 }
 408         }
 409
 410         return priv->chan[ch].fifo[iter].desc->hdr;
 411 }
 412
 413 /*
 414  * user diagnostics; report root cause of error based on execution unit status
 415  */
 416 static void report_eu_error(struct device *dev, int ch, u32 desc_hdr)
 417 {
 418         struct talitos_private *priv = dev_get_drvdata(dev);
 419         int i;
 420
 421         if (!desc_hdr)
 422                 desc_hdr = in_be32(priv->chan[ch].reg + TALITOS_DESCBUF);
 423
 424         switch (desc_hdr & DESC_HDR_SEL0_MASK) {
 425         case DESC_HDR_SEL0_AFEU:
 426                 dev_err(dev, "AFEUISR 0x%08x_%08x\n",
 427                         in_be32(priv->reg + TALITOS_AFEUISR),
 428                         in_be32(priv->reg + TALITOS_AFEUISR_LO));
 429                 break;
 430         case DESC_HDR_SEL0_DEU:
 431                 dev_err(dev, "DEUISR 0x%08x_%08x\n",
 432                         in_be32(priv->reg + TALITOS_DEUISR),
 433                         in_be32(priv->reg + TALITOS_DEUISR_LO));
 434                 break;
 435         case DESC_HDR_SEL0_MDEUA:
 436         case DESC_HDR_SEL0_MDEUB:
 437                 dev_err(dev, "MDEUISR 0x%08x_%08x\n",
 438                         in_be32(priv->reg + TALITOS_MDEUISR),
 439                         in_be32(priv->reg + TALITOS_MDEUISR_LO));
 440                 break;
 441         case DESC_HDR_SEL0_RNG:
 442                 dev_err(dev, "RNGUISR 0x%08x_%08x\n",
 443                         in_be32(priv->reg + TALITOS_RNGUISR),
 444                         in_be32(priv->reg + TALITOS_RNGUISR_LO));
 445                 break;
 446         case DESC_HDR_SEL0_PKEU:
 447                 dev_err(dev, "PKEUISR 0x%08x_%08x\n",
 448                         in_be32(priv->reg + TALITOS_PKEUISR),
 449                         in_be32(priv->reg + TALITOS_PKEUISR_LO));
 450                 break;
 451         case DESC_HDR_SEL0_AESU:
 452                 dev_err(dev, "AESUISR 0x%08x_%08x\n",
 453                         in_be32(priv->reg + TALITOS_AESUISR),
 454                         in_be32(priv->reg + TALITOS_AESUISR_LO));
 455                 break;
 456         case DESC_HDR_SEL0_CRCU:
 457                 dev_err(dev, "CRCUISR 0x%08x_%08x\n",
 458                         in_be32(priv->reg + TALITOS_CRCUISR),
 459                         in_be32(priv->reg + TALITOS_CRCUISR_LO));
 460                 break;
 461         case DESC_HDR_SEL0_KEU:
 462                 dev_err(dev, "KEUISR 0x%08x_%08x\n",
 463                         in_be32(priv->reg + TALITOS_KEUISR),
 464                         in_be32(priv->reg + TALITOS_KEUISR_LO));
 465                 break;
 466         }
 467
 468         switch (desc_hdr & DESC_HDR_SEL1_MASK) {
 469         case DESC_HDR_SEL1_MDEUA:
 470         case DESC_HDR_SEL1_MDEUB:
 471                 dev_err(dev, "MDEUISR 0x%08x_%08x\n",
 472                         in_be32(priv->reg + TALITOS_MDEUISR),
 473                         in_be32(priv->reg + TALITOS_MDEUISR_LO));
 474                 break;
 475         case DESC_HDR_SEL1_CRCU:
 476                 dev_err(dev, "CRCUISR 0x%08x_%08x\n",
 477                         in_be32(priv->reg + TALITOS_CRCUISR),
 478                         in_be32(priv->reg + TALITOS_CRCUISR_LO));
 479                 break;
 480         }
 481
 482         for (i = 0; i < 8; i++)
 483                 dev_err(dev, "DESCBUF 0x%08x_%08x\n",
 484                         in_be32(priv->chan[ch].reg + TALITOS_DESCBUF + 8*i),
 485                         in_be32(priv->chan[ch].reg + TALITOS_DESCBUF_LO + 8*i));
 486 }
 487
 488 /*
 489  * recover from error interrupts
 490  */
 491 static void talitos_error(struct device *dev, u32 isr, u32 isr_lo)
 492 {
 493         struct talitos_private *priv = dev_get_drvdata(dev);
 494         unsigned int timeout = TALITOS_TIMEOUT;
 495         int ch, error, reset_dev = 0, reset_ch = 0;
 496         u32 v, v_lo;
 497
 498         for (ch = 0; ch < priv->num_channels; ch++) {
 499                 /* skip channels without errors */
 500                 if (!(isr & (1 << (ch * 2 + 1))))
 501                         continue;
 502
 503                 error = -EINVAL;
 504
 505                 v = in_be32(priv->chan[ch].reg + TALITOS_CCPSR);
 506                 v_lo = in_be32(priv->chan[ch].reg + TALITOS_CCPSR_LO);
 507
 508                 if (v_lo & TALITOS_CCPSR_LO_DOF) {
 509                         dev_err(dev, "double fetch fifo overflow error\n");
 510                         error = -EAGAIN;
 511                         reset_ch = 1;
 512                 }
 513                 if (v_lo & TALITOS_CCPSR_LO_SOF) {
 514                         /* h/w dropped descriptor */
 515                         dev_err(dev, "single fetch fifo overflow error\n");
 516                         error = -EAGAIN;
 517                 }
 518                 if (v_lo & TALITOS_CCPSR_LO_MDTE)
 519                         dev_err(dev, "master data transfer error\n");
 520                 if (v_lo & TALITOS_CCPSR_LO_SGDLZ)
 521                         dev_err(dev, "s/g data length zero error\n");
 522                 if (v_lo & TALITOS_CCPSR_LO_FPZ)
 523                         dev_err(dev, "fetch pointer zero error\n");
 524                 if (v_lo & TALITOS_CCPSR_LO_IDH)
 525                         dev_err(dev, "illegal descriptor header error\n");
 526                 if (v_lo & TALITOS_CCPSR_LO_IEU)
 527                         dev_err(dev, "invalid execution unit error\n");
 528                 if (v_lo & TALITOS_CCPSR_LO_EU)
 529                         report_eu_error(dev, ch, current_desc_hdr(dev, ch));
 530                 if (v_lo & TALITOS_CCPSR_LO_GB)
 531                         dev_err(dev, "gather boundary error\n");
 532                 if (v_lo & TALITOS_CCPSR_LO_GRL)
 533                         dev_err(dev, "gather return/length error\n");
 534                 if (v_lo & TALITOS_CCPSR_LO_SB)
 535                         dev_err(dev, "scatter boundary error\n");
 536                 if (v_lo & TALITOS_CCPSR_LO_SRL)
 537                         dev_err(dev, "scatter return/length error\n");
 538
 539                 flush_channel(dev, ch, error, reset_ch);
 540
 541                 if (reset_ch) {
 542                         reset_channel(dev, ch);
 543                 } else {
 544                         setbits32(priv->chan[ch].reg + TALITOS_CCCR,
 545                                   TALITOS_CCCR_CONT);
 546                         setbits32(priv->chan[ch].reg + TALITOS_CCCR_LO, 0);
 547                         while ((in_be32(priv->chan[ch].reg + TALITOS_CCCR) &
 548                                TALITOS_CCCR_CONT) && --timeout)
 549                                 cpu_relax();
 550                         if (timeout == 0) {
 551                                 dev_err(dev, "failed to restart channel %d\n",
 552                                         ch);
 553                                 reset_dev = 1;
 554                         }
 555                 }
 556         }
 557         if (reset_dev || isr & ~TALITOS_ISR_4CHERR || isr_lo) {
 558                 dev_err(dev, "done overflow, internal time out, or rngu error: "
 559                         "ISR 0x%08x_%08x\n", isr, isr_lo);
 560
 561                 /* purge request queues */
 562                 for (ch = 0; ch < priv->num_channels; ch++)
 563                         flush_channel(dev, ch, -EIO, 1);
 564
 565                 /* reset and reinitialize the device */
 566                 init_device(dev);
 567         }
 568 }
 569
 570 #define DEF_TALITOS_INTERRUPT(name, ch_done_mask, ch_err_mask, tlet)           \
 571 static irqreturn_t talitos_interrupt_##name(int irq, void *data)               \
 572 {                                                                              \
 573         struct device *dev = data;                                             \
 574         struct talitos_private *priv = dev_get_drvdata(dev);                   \
 575         u32 isr, isr_lo;                                                       \
 576         unsigned long flags;                                                   \
 577                                                                                \
 578         spin_lock_irqsave(&priv->reg_lock, flags);                             \
 579         isr = in_be32(priv->reg + TALITOS_ISR);                                \
 580         isr_lo = in_be32(priv->reg + TALITOS_ISR_LO);                          \
 581         /* Acknowledge interrupt */                                            \
 582         out_be32(priv->reg + TALITOS_ICR, isr & (ch_done_mask | ch_err_mask)); \
 583         out_be32(priv->reg + TALITOS_ICR_LO, isr_lo);                          \
 584                                                                                \
 585         if (unlikely(isr & ch_err_mask || isr_lo)) {                           \
 586                 spin_unlock_irqrestore(&priv->reg_lock, flags);                \
 587                 talitos_error(dev, isr & ch_err_mask, isr_lo);                 \
 588         }                                                                      \
 589         else {                                                                 \
 590                 if (likely(isr & ch_done_mask)) {                              \
 591                         /* mask further done interrupts. */                    \
 592                         clrbits32(priv->reg + TALITOS_IMR, ch_done_mask);      \
 593                         /* done_task will unmask done interrupts at exit */    \
 594                         tasklet_schedule(&priv->done_task[tlet]);              \
 595                 }                                                              \
 596                 spin_unlock_irqrestore(&priv->reg_lock, flags);                \
 597         }                                                                      \
 598                                                                                \
 599         return (isr & (ch_done_mask | ch_err_mask) || isr_lo) ? IRQ_HANDLED :  \
 600                                                                 IRQ_NONE;      \
 601 }
 602 DEF_TALITOS_INTERRUPT(4ch, TALITOS_ISR_4CHDONE, TALITOS_ISR_4CHERR, 0)
 603 DEF_TALITOS_INTERRUPT(ch0_2, TALITOS_ISR_CH_0_2_DONE, TALITOS_ISR_CH_0_2_ERR, 0)
 604 DEF_TALITOS_INTERRUPT(ch1_3, TALITOS_ISR_CH_1_3_DONE, TALITOS_ISR_CH_1_3_ERR, 1)
 605
 606 /*
 607  * hwrng
 608  */
 609 static int talitos_rng_data_present(struct hwrng *rng, int wait)
 610 {
 611         struct device *dev = (struct device *)rng->priv;
 612         struct talitos_private *priv = dev_get_drvdata(dev);
 613         u32 ofl;
 614         int i;
 615
 616         for (i = 0; i < 20; i++) {
 617                 ofl = in_be32(priv->reg + TALITOS_RNGUSR_LO) &
 618                       TALITOS_RNGUSR_LO_OFL;
 619                 if (ofl || !wait)
 620                         break;
 621                 udelay(10);
 622         }
 623
 624         return !!ofl;
 625 }
 626
 627 static int talitos_rng_data_read(struct hwrng *rng, u32 *data)
 628 {
 629         struct device *dev = (struct device *)rng->priv;
 630         struct talitos_private *priv = dev_get_drvdata(dev);
 631
 632         /* rng fifo requires 64-bit accesses */
 633         *data = in_be32(priv->reg + TALITOS_RNGU_FIFO);
 634         *data = in_be32(priv->reg + TALITOS_RNGU_FIFO_LO);
 635
 636         return sizeof(u32);
 637 }
 638
 639 static int talitos_rng_init(struct hwrng *rng)
 640 {
 641         struct device *dev = (struct device *)rng->priv;
 642         struct talitos_private *priv = dev_get_drvdata(dev);
 643         unsigned int timeout = TALITOS_TIMEOUT;
 644
 645         setbits32(priv->reg + TALITOS_RNGURCR_LO, TALITOS_RNGURCR_LO_SR);
 646         while (!(in_be32(priv->reg + TALITOS_RNGUSR_LO) & TALITOS_RNGUSR_LO_RD)
 647                && --timeout)
 648                 cpu_relax();
 649         if (timeout == 0) {
 650                 dev_err(dev, "failed to reset rng hw\n");
 651                 return -ENODEV;
 652         }
 653
 654         /* start generating */
 655         setbits32(priv->reg + TALITOS_RNGUDSR_LO, 0);
 656
 657         return 0;
 658 }
 659
 660 static int talitos_register_rng(struct device *dev)
 661 {
 662         struct talitos_private *priv = dev_get_drvdata(dev);
 663
 664         priv->rng.name          = dev_driver_string(dev),
 665         priv->rng.init          = talitos_rng_init,
 666         priv->rng.data_present  = talitos_rng_data_present,
 667         priv->rng.data_read     = talitos_rng_data_read,
 668         priv->rng.priv          = (unsigned long)dev;
 669
 670         return hwrng_register(&priv->rng);
 671 }
 672
 673 static void talitos_unregister_rng(struct device *dev)
 674 {
 675         struct talitos_private *priv = dev_get_drvdata(dev);
 676
 677         hwrng_unregister(&priv->rng);
 678 }
 679
 680 /*
 681  * crypto alg
 682  */
 683 #define TALITOS_CRA_PRIORITY            3000
 684 #define TALITOS_MAX_KEY_SIZE            96
 685 #define TALITOS_MAX_IV_LENGTH           16 /* max of AES_BLOCK_SIZE, DES3_EDE_BLOCK_SIZE */
 686
 687 struct talitos_ctx {
 688         struct device *dev;
 689         int ch;
 690         __be32 desc_hdr_template;
 691         u8 key[TALITOS_MAX_KEY_SIZE];
 692         u8 iv[TALITOS_MAX_IV_LENGTH];
 693         unsigned int keylen;
 694         unsigned int enckeylen;
 695         unsigned int authkeylen;
 696         unsigned int authsize;
 697 };
 698
 699 #define HASH_MAX_BLOCK_SIZE             SHA512_BLOCK_SIZE
 700 #define TALITOS_MDEU_MAX_CONTEXT_SIZE   TALITOS_MDEU_CONTEXT_SIZE_SHA384_SHA512
 701
 702 struct talitos_ahash_req_ctx {
 703         u32 hw_context[TALITOS_MDEU_MAX_CONTEXT_SIZE / sizeof(u32)];
 704         unsigned int hw_context_size;
 705         u8 buf[HASH_MAX_BLOCK_SIZE];
 706         u8 bufnext[HASH_MAX_BLOCK_SIZE];
 707         unsigned int swinit;
 708         unsigned int first;
 709         unsigned int last;
 710         unsigned int to_hash_later;
 711         u64 nbuf;
 712         struct scatterlist bufsl[2];
 713         struct scatterlist *psrc;
 714 };
 715
 716 static int aead_setauthsize(struct crypto_aead *authenc,
 717                             unsigned int authsize)
 718 {
 719         struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
 720
 721         ctx->authsize = authsize;
 722
 723         return 0;
 724 }
 725
 726 static int aead_setkey(struct crypto_aead *authenc,
 727                        const u8 *key, unsigned int keylen)
 728 {
 729         struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
 730         struct crypto_authenc_keys keys;
 731
 732         if (crypto_authenc_extractkeys(&keys, key, keylen) != 0)
 733                 goto badkey;
 734
 735         if (keys.authkeylen + keys.enckeylen > TALITOS_MAX_KEY_SIZE)
 736                 goto badkey;
 737
 738         memcpy(ctx->key, keys.authkey, keys.authkeylen);
 739         memcpy(&ctx->key[keys.authkeylen], keys.enckey, keys.enckeylen);
 740
 741         ctx->keylen = keys.authkeylen + keys.enckeylen;
 742         ctx->enckeylen = keys.enckeylen;
 743         ctx->authkeylen = keys.authkeylen;
 744
 745         return 0;
 746
 747 badkey:
 748         crypto_aead_set_flags(authenc, CRYPTO_TFM_RES_BAD_KEY_LEN);
 749         return -EINVAL;
 750 }
 751
 752 /*
 753  * talitos_edesc - s/w-extended descriptor
 754  * @assoc_nents: number of segments in associated data scatterlist
 755  * @src_nents: number of segments in input scatterlist
 756  * @dst_nents: number of segments in output scatterlist
 757  * @assoc_chained: whether assoc is chained or not
 758  * @src_chained: whether src is chained or not
 759  * @dst_chained: whether dst is chained or not
 760  * @iv_dma: dma address of iv for checking continuity and link table
 761  * @dma_len: length of dma mapped link_tbl space
 762  * @dma_link_tbl: bus physical address of link_tbl
 763  * @desc: h/w descriptor
 764  * @link_tbl: input and output h/w link tables (if {src,dst}_nents > 1)
 765  *
 766  * if decrypting (with authcheck), or either one of src_nents or dst_nents
 767  * is greater than 1, an integrity check value is concatenated to the end
 768  * of link_tbl data
 769  */
 770 struct talitos_edesc {
 771         int assoc_nents;
 772         int src_nents;
 773         int dst_nents;
 774         bool assoc_chained;
 775         bool src_chained;
 776         bool dst_chained;
 777         dma_addr_t iv_dma;
 778         int dma_len;
 779         dma_addr_t dma_link_tbl;
 780         struct talitos_desc desc;
 781         struct talitos_ptr link_tbl[0];
 782 };
 783
 784 static int talitos_map_sg(struct device *dev, struct scatterlist *sg,
 785                           unsigned int nents, enum dma_data_direction dir,
 786                           bool chained)
 787 {
 788         if (unlikely(chained))
 789                 while (sg) {
 790                         dma_map_sg(dev, sg, 1, dir);
 791                         sg = sg_next(sg);
 792                 }
 793         else
 794                 dma_map_sg(dev, sg, nents, dir);
 795         return nents;
 796 }
 797
 798 static void talitos_unmap_sg_chain(struct device *dev, struct scatterlist *sg,
 799                                    enum dma_data_direction dir)
 800 {
 801         while (sg) {
 802                 dma_unmap_sg(dev, sg, 1, dir);
 803                 sg = sg_next(sg);
 804         }
 805 }
 806
 807 static void talitos_sg_unmap(struct device *dev,
 808                              struct talitos_edesc *edesc,
 809                              struct scatterlist *src,
 810                              struct scatterlist *dst)
 811 {
 812         unsigned int src_nents = edesc->src_nents ? : 1;
 813         unsigned int dst_nents = edesc->dst_nents ? : 1;
 814
 815         if (src != dst) {
 816                 if (edesc->src_chained)
 817                         talitos_unmap_sg_chain(dev, src, DMA_TO_DEVICE);
 818                 else
 819                         dma_unmap_sg(dev, src, src_nents, DMA_TO_DEVICE);
 820
 821                 if (dst) {
 822                         if (edesc->dst_chained)
 823                                 talitos_unmap_sg_chain(dev, dst,
 824                                                        DMA_FROM_DEVICE);
 825                         else
 826                                 dma_unmap_sg(dev, dst, dst_nents,
 827                                              DMA_FROM_DEVICE);
 828                 }
 829         } else
 830                 if (edesc->src_chained)
 831                         talitos_unmap_sg_chain(dev, src, DMA_BIDIRECTIONAL);
 832                 else
 833                         dma_unmap_sg(dev, src, src_nents, DMA_BIDIRECTIONAL);
 834 }
 835
 836 static void ipsec_esp_unmap(struct device *dev,
 837                             struct talitos_edesc *edesc,
 838                             struct aead_request *areq)
 839 {
 840         unmap_single_talitos_ptr(dev, &edesc->desc.ptr[6], DMA_FROM_DEVICE);
 841         unmap_single_talitos_ptr(dev, &edesc->desc.ptr[3], DMA_TO_DEVICE);
 842         unmap_single_talitos_ptr(dev, &edesc->desc.ptr[2], DMA_TO_DEVICE);
 843         unmap_single_talitos_ptr(dev, &edesc->desc.ptr[0], DMA_TO_DEVICE);
 844
 845         if (edesc->assoc_chained)
 846                 talitos_unmap_sg_chain(dev, areq->assoc, DMA_TO_DEVICE);
 847         else if (areq->assoclen)
 848                 /* assoc_nents counts also for IV in non-contiguous cases */
 849                 dma_unmap_sg(dev, areq->assoc,
 850                              edesc->assoc_nents ? edesc->assoc_nents - 1 : 1,
 851                              DMA_TO_DEVICE);
 852
 853         talitos_sg_unmap(dev, edesc, areq->src, areq->dst);
 854
 855         if (edesc->dma_len)
 856                 dma_unmap_single(dev, edesc->dma_link_tbl, edesc->dma_len,
 857                                  DMA_BIDIRECTIONAL);
 858 }
 859
 860 /*
 861  * ipsec_esp descriptor callbacks
 862  */
 863 static void ipsec_esp_encrypt_done(struct device *dev,
 864                                    struct talitos_desc *desc, void *context,
 865                                    int err)
 866 {
 867         struct aead_request *areq = context;
 868         struct crypto_aead *authenc = crypto_aead_reqtfm(areq);
 869         struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
 870         struct talitos_edesc *edesc;
 871         struct scatterlist *sg;
 872         void *icvdata;
 873
 874         edesc = container_of(desc, struct talitos_edesc, desc);
 875
 876         ipsec_esp_unmap(dev, edesc, areq);
 877
 878         /* copy the generated ICV to dst */
 879         if (edesc->dst_nents) {
 880                 icvdata = &edesc->link_tbl[edesc->src_nents +
 881                                            edesc->dst_nents + 2 +
 882                                            edesc->assoc_nents];
 883                 sg = sg_last(areq->dst, edesc->dst_nents);
 884                 memcpy((char *)sg_virt(sg) + sg->length - ctx->authsize,
 885                        icvdata, ctx->authsize);
 886         }
 887
 888         kfree(edesc);
 889
 890         aead_request_complete(areq, err);
 891 }
 892
 893 static void ipsec_esp_decrypt_swauth_done(struct device *dev,
 894                                           struct talitos_desc *desc,
 895                                           void *context, int err)
 896 {
 897         struct aead_request *req = context;
 898         struct crypto_aead *authenc = crypto_aead_reqtfm(req);
 899         struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
 900         struct talitos_edesc *edesc;
 901         struct scatterlist *sg;
 902         void *icvdata;
 903
 904         edesc = container_of(desc, struct talitos_edesc, desc);
 905
 906         ipsec_esp_unmap(dev, edesc, req);
 907
 908         if (!err) {
 909                 /* auth check */
 910                 if (edesc->dma_len)
 911                         icvdata = &edesc->link_tbl[edesc->src_nents +
 912                                                    edesc->dst_nents + 2 +
 913                                                    edesc->assoc_nents];
 914                 else
 915                         icvdata = &edesc->link_tbl[0];
 916
 917                 sg = sg_last(req->dst, edesc->dst_nents ? : 1);
 918                 err = memcmp(icvdata, (char *)sg_virt(sg) + sg->length -
 919                              ctx->authsize, ctx->authsize) ? -EBADMSG : 0;
 920         }
 921
 922         kfree(edesc);
 923
 924         aead_request_complete(req, err);
 925 }
 926
 927 static void ipsec_esp_decrypt_hwauth_done(struct device *dev,
 928                                           struct talitos_desc *desc,
 929                                           void *context, int err)
 930 {
 931         struct aead_request *req = context;
 932         struct talitos_edesc *edesc;
 933
 934         edesc = container_of(desc, struct talitos_edesc, desc);
 935
 936         ipsec_esp_unmap(dev, edesc, req);
 937
 938         /* check ICV auth status */
 939         if (!err && ((desc->hdr_lo & DESC_HDR_LO_ICCR1_MASK) !=
 940                      DESC_HDR_LO_ICCR1_PASS))
 941                 err = -EBADMSG;
 942
 943         kfree(edesc);
 944
 945         aead_request_complete(req, err);
 946 }
 947
 948 /*
 949  * convert scatterlist to SEC h/w link table format
 950  * stop at cryptlen bytes
 951  */
 952 static int sg_to_link_tbl(struct scatterlist *sg, int sg_count,
 953                            int cryptlen, struct talitos_ptr *link_tbl_ptr)
 954 {
 955         int n_sg = sg_count;
 956
 957         while (n_sg--) {
 958                 to_talitos_ptr(link_tbl_ptr, sg_dma_address(sg), 0);
 959                 link_tbl_ptr->len = cpu_to_be16(sg_dma_len(sg));
 960                 link_tbl_ptr->j_extent = 0;
 961                 link_tbl_ptr++;
 962                 cryptlen -= sg_dma_len(sg);
 963                 sg = sg_next(sg);
 964         }
 965
 966         /* adjust (decrease) last one (or two) entry's len to cryptlen */
 967         link_tbl_ptr--;
 968         while (be16_to_cpu(link_tbl_ptr->len) <= (-cryptlen)) {
 969                 /* Empty this entry, and move to previous one */
 970                 cryptlen += be16_to_cpu(link_tbl_ptr->len);
 971                 link_tbl_ptr->len = 0;
 972                 sg_count--;
 973                 link_tbl_ptr--;
 974         }
 975         be16_add_cpu(&link_tbl_ptr->len, cryptlen);
 976
 977         /* tag end of link table */
 978         link_tbl_ptr->j_extent = DESC_PTR_LNKTBL_RETURN;
 979
 980         return sg_count;
 981 }
 982
 983 /*
 984  * fill in and submit ipsec_esp descriptor
 985  */
 986 static int ipsec_esp(struct talitos_edesc *edesc, struct aead_request *areq,
 987                      u64 seq, void (*callback) (struct device *dev,
 988                                                 struct talitos_desc *desc,
 989                                                 void *context, int error))
 990 {
 991         struct crypto_aead *aead = crypto_aead_reqtfm(areq);
 992         struct talitos_ctx *ctx = crypto_aead_ctx(aead);
 993         struct device *dev = ctx->dev;
 994         struct talitos_desc *desc = &edesc->desc;
 995         unsigned int cryptlen = areq->cryptlen;
 996         unsigned int authsize = ctx->authsize;
 997         unsigned int ivsize = crypto_aead_ivsize(aead);
 998         int sg_count, ret;
 999         int sg_link_tbl_len;
1000
1001         /* hmac key */
1002         map_single_talitos_ptr(dev, &desc->ptr[0], ctx->authkeylen, &ctx->key,
1003                                DMA_TO_DEVICE);
1004
1005         /* hmac data */
1006         desc->ptr[1].len = cpu_to_be16(areq->assoclen + ivsize);
1007         if (edesc->assoc_nents) {
1008                 int tbl_off = edesc->src_nents + edesc->dst_nents + 2;
1009                 struct talitos_ptr *tbl_ptr = &edesc->link_tbl[tbl_off];
1010
1011                 to_talitos_ptr(&desc->ptr[1], edesc->dma_link_tbl + tbl_off *
1012                                sizeof(struct talitos_ptr), 0);
1013                 desc->ptr[1].j_extent = DESC_PTR_LNKTBL_JUMP;
1014
1015                 /* assoc_nents - 1 entries for assoc, 1 for IV */
1016                 sg_count = sg_to_link_tbl(areq->assoc, edesc->assoc_nents - 1,
1017                                           areq->assoclen, tbl_ptr);
1018
1019                 /* add IV to link table */
1020                 tbl_ptr += sg_count - 1;
1021                 tbl_ptr->j_extent = 0;
1022                 tbl_ptr++;
1023                 to_talitos_ptr(tbl_ptr, edesc->iv_dma, 0);
1024                 tbl_ptr->len = cpu_to_be16(ivsize);
1025                 tbl_ptr->j_extent = DESC_PTR_LNKTBL_RETURN;
1026
1027                 dma_sync_single_for_device(dev, edesc->dma_link_tbl,
1028                                            edesc->dma_len, DMA_BIDIRECTIONAL);
1029         } else {
1030                 if (areq->assoclen)
1031                         to_talitos_ptr(&desc->ptr[1],
1032                                        sg_dma_address(areq->assoc), 0);
1033                 else
1034                         to_talitos_ptr(&desc->ptr[1], edesc->iv_dma, 0);
1035                 desc->ptr[1].j_extent = 0;
1036         }
1037
1038         /* cipher iv */
1039         to_talitos_ptr(&desc->ptr[2], edesc->iv_dma, 0);
1040         desc->ptr[2].len = cpu_to_be16(ivsize);
1041         desc->ptr[2].j_extent = 0;
1042         /* Sync needed for the aead_givencrypt case */
1043         dma_sync_single_for_device(dev, edesc->iv_dma, ivsize, DMA_TO_DEVICE);
1044
1045         /* cipher key */
1046         map_single_talitos_ptr(dev, &desc->ptr[3], ctx->enckeylen,
1047                                (char *)&ctx->key + ctx->authkeylen,
1048                                DMA_TO_DEVICE);
1049
1050         /*
1051          * cipher in
1052          * map and adjust cipher len to aead request cryptlen.
1053          * extent is bytes of HMAC postpended to ciphertext,
1054          * typically 12 for ipsec
1055          */
1056         desc->ptr[4].len = cpu_to_be16(cryptlen);
1057         desc->ptr[4].j_extent = authsize;
1058
1059         sg_count = talitos_map_sg(dev, areq->src, edesc->src_nents ? : 1,
1060                                   (areq->src == areq->dst) ? DMA_BIDIRECTIONAL
1061                                                            : DMA_TO_DEVICE,
1062                                   edesc->src_chained);
1063
1064         if (sg_count == 1) {
1065                 to_talitos_ptr(&desc->ptr[4], sg_dma_address(areq->src), 0);
1066         } else {
1067                 sg_link_tbl_len = cryptlen;
1068
1069                 if (edesc->desc.hdr & DESC_HDR_MODE1_MDEU_CICV)
1070                         sg_link_tbl_len = cryptlen + authsize;
1071
1072                 sg_count = sg_to_link_tbl(areq->src, sg_count, sg_link_tbl_len,
1073                                           &edesc->link_tbl[0]);
1074                 if (sg_count > 1) {
1075                         desc->ptr[4].j_extent |= DESC_PTR_LNKTBL_JUMP;
1076                         to_talitos_ptr(&desc->ptr[4], edesc->dma_link_tbl, 0);
1077                         dma_sync_single_for_device(dev, edesc->dma_link_tbl,
1078                                                    edesc->dma_len,
1079                                                    DMA_BIDIRECTIONAL);
1080                 } else {
1081                         /* Only one segment now, so no link tbl needed */
1082                         to_talitos_ptr(&desc->ptr[4],
1083                                        sg_dma_address(areq->src), 0);
1084                 }
1085         }
1086
1087         /* cipher out */
1088         desc->ptr[5].len = cpu_to_be16(cryptlen);
1089         desc->ptr[5].j_extent = authsize;
1090
1091         if (areq->src != areq->dst)
1092                 sg_count = talitos_map_sg(dev, areq->dst,
1093                                           edesc->dst_nents ? : 1,
1094                                           DMA_FROM_DEVICE, edesc->dst_chained);
1095
1096         if (sg_count == 1) {
1097                 to_talitos_ptr(&desc->ptr[5], sg_dma_address(areq->dst), 0);
1098         } else {
1099                 int tbl_off = edesc->src_nents + 1;
1100                 struct talitos_ptr *tbl_ptr = &edesc->link_tbl[tbl_off];
1101
1102                 to_talitos_ptr(&desc->ptr[5], edesc->dma_link_tbl +
1103                                tbl_off * sizeof(struct talitos_ptr), 0);
1104                 sg_count = sg_to_link_tbl(areq->dst, sg_count, cryptlen,
1105                                           tbl_ptr);
1106
1107                 /* Add an entry to the link table for ICV data */
1108                 tbl_ptr += sg_count - 1;
1109                 tbl_ptr->j_extent = 0;
1110                 tbl_ptr++;
1111                 tbl_ptr->j_extent = DESC_PTR_LNKTBL_RETURN;
1112                 tbl_ptr->len = cpu_to_be16(authsize);
1113
1114                 /* icv data follows link tables */
1115                 to_talitos_ptr(tbl_ptr, edesc->dma_link_tbl +
1116                                (tbl_off + edesc->dst_nents + 1 +
1117                                 edesc->assoc_nents) *
1118                                sizeof(struct talitos_ptr), 0);
1119                 desc->ptr[5].j_extent |= DESC_PTR_LNKTBL_JUMP;
1120                 dma_sync_single_for_device(ctx->dev, edesc->dma_link_tbl,
1121                                            edesc->dma_len, DMA_BIDIRECTIONAL);
1122         }
1123
1124         /* iv out */
1125         map_single_talitos_ptr(dev, &desc->ptr[6], ivsize, ctx->iv,
1126                                DMA_FROM_DEVICE);
1127
1128         ret = talitos_submit(dev, ctx->ch, desc, callback, areq);
1129         if (ret != -EINPROGRESS) {
1130                 ipsec_esp_unmap(dev, edesc, areq);
1131                 kfree(edesc);
1132         }
1133         return ret;
1134 }
1135
1136 /*
1137  * derive number of elements in scatterlist
1138  */
1139 static int sg_count(struct scatterlist *sg_list, int nbytes, bool *chained)
1140 {
1141         struct scatterlist *sg = sg_list;
1142         int sg_nents = 0;
1143
1144         *chained = false;
1145         while (nbytes > 0) {
1146                 sg_nents++;
1147                 nbytes -= sg->length;
1148                 if (!sg_is_last(sg) && (sg + 1)->length == 0)
1149                         *chained = true;
1150                 sg = sg_next(sg);
1151         }
1152
1153         return sg_nents;
1154 }
1155
1156 /*
1157  * allocate and map the extended descriptor
1158  */
1159 static struct talitos_edesc *talitos_edesc_alloc(struct device *dev,
1160                                                  struct scatterlist *assoc,
1161                                                  struct scatterlist *src,
1162                                                  struct scatterlist *dst,
1163                                                  u8 *iv,
1164                                                  unsigned int assoclen,
1165                                                  unsigned int cryptlen,
1166                                                  unsigned int authsize,
1167                                                  unsigned int ivsize,
1168                                                  int icv_stashing,
1169                                                  u32 cryptoflags,
1170                                                  bool encrypt)
1171 {
1172         struct talitos_edesc *edesc;
1173         int assoc_nents = 0, src_nents, dst_nents, alloc_len, dma_len;
1174         bool assoc_chained = false, src_chained = false, dst_chained = false;
1175         dma_addr_t iv_dma = 0;
1176         gfp_t flags = cryptoflags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL :
1177                       GFP_ATOMIC;
1178
1179         if (cryptlen + authsize > TALITOS_MAX_DATA_LEN) {
1180                 dev_err(dev, "length exceeds h/w max limit\n");
1181                 return ERR_PTR(-EINVAL);
1182         }
1183
1184         if (ivsize)
1185                 iv_dma = dma_map_single(dev, iv, ivsize, DMA_TO_DEVICE);
1186
1187         if (assoclen) {
1188                 /*
1189                  * Currently it is assumed that iv is provided whenever assoc
1190                  * is.
1191                  */
1192                 BUG_ON(!iv);
1193
1194                 assoc_nents = sg_count(assoc, assoclen, &assoc_chained);
1195                 talitos_map_sg(dev, assoc, assoc_nents, DMA_TO_DEVICE,
1196                                assoc_chained);
1197                 assoc_nents = (assoc_nents == 1) ? 0 : assoc_nents;
1198
1199                 if (assoc_nents || sg_dma_address(assoc) + assoclen != iv_dma)
1200                         assoc_nents = assoc_nents ? assoc_nents + 1 : 2;
1201         }
1202
1203         if (!dst || dst == src) {
1204                 src_nents = sg_count(src, cryptlen + authsize, &src_chained);
1205                 src_nents = (src_nents == 1) ? 0 : src_nents;
1206                 dst_nents = dst ? src_nents : 0;
1207         } else { /* dst && dst != src*/
1208                 src_nents = sg_count(src, cryptlen + (encrypt ? 0 : authsize),
1209                                      &src_chained);
1210                 src_nents = (src_nents == 1) ? 0 : src_nents;
1211                 dst_nents = sg_count(dst, cryptlen + (encrypt ? authsize : 0),
1212                                      &dst_chained);
1213                 dst_nents = (dst_nents == 1) ? 0 : dst_nents;
1214         }
1215
1216         /*
1217          * allocate space for base edesc plus the link tables,
1218          * allowing for two separate entries for ICV and generated ICV (+ 2),
1219          * and the ICV data itself
1220          */
1221         alloc_len = sizeof(struct talitos_edesc);
1222         if (assoc_nents || src_nents || dst_nents) {
1223                 dma_len = (src_nents + dst_nents + 2 + assoc_nents) *
1224                           sizeof(struct talitos_ptr) + authsize;
1225                 alloc_len += dma_len;
1226         } else {
1227                 dma_len = 0;
1228                 alloc_len += icv_stashing ? authsize : 0;
1229         }
1230
1231         edesc = kmalloc(alloc_len, GFP_DMA | flags);
1232         if (!edesc) {
1233                 if (assoc_chained)
1234                         talitos_unmap_sg_chain(dev, assoc, DMA_TO_DEVICE);
1235                 else if (assoclen)
1236                         dma_unmap_sg(dev, assoc,
1237                                      assoc_nents ? assoc_nents - 1 : 1,
1238                                      DMA_TO_DEVICE);
1239
1240                 if (iv_dma)
1241                         dma_unmap_single(dev, iv_dma, ivsize, DMA_TO_DEVICE);
1242
1243                 dev_err(dev, "could not allocate edescriptor\n");
1244                 return ERR_PTR(-ENOMEM);
1245         }
1246
1247         edesc->assoc_nents = assoc_nents;
1248         edesc->src_nents = src_nents;
1249         edesc->dst_nents = dst_nents;
1250         edesc->assoc_chained = assoc_chained;
1251         edesc->src_chained = src_chained;
1252         edesc->dst_chained = dst_chained;
1253         edesc->iv_dma = iv_dma;
1254         edesc->dma_len = dma_len;
1255         if (dma_len)
1256                 edesc->dma_link_tbl = dma_map_single(dev, &edesc->link_tbl[0],
1257                                                      edesc->dma_len,
1258                                                      DMA_BIDIRECTIONAL);
1259
1260         return edesc;
1261 }
1262
1263 static struct talitos_edesc *aead_edesc_alloc(struct aead_request *areq, u8 *iv,
1264                                               int icv_stashing, bool encrypt)
1265 {
1266         struct crypto_aead *authenc = crypto_aead_reqtfm(areq);
1267         struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
1268         unsigned int ivsize = crypto_aead_ivsize(authenc);
1269
1270         return talitos_edesc_alloc(ctx->dev, areq->assoc, areq->src, areq->dst,
1271                                    iv, areq->assoclen, areq->cryptlen,
1272                                    ctx->authsize, ivsize, icv_stashing,
1273                                    areq->base.flags, encrypt);
1274 }
1275
1276 static int aead_encrypt(struct aead_request *req)
1277 {
1278         struct crypto_aead *authenc = crypto_aead_reqtfm(req);
1279         struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
1280         struct talitos_edesc *edesc;
1281
1282         /* allocate extended descriptor */
1283         edesc = aead_edesc_alloc(req, req->iv, 0, true);
1284         if (IS_ERR(edesc))
1285                 return PTR_ERR(edesc);
1286
1287         /* set encrypt */
1288         edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_MODE0_ENCRYPT;
1289
1290         return ipsec_esp(edesc, req, 0, ipsec_esp_encrypt_done);
1291 }
1292
1293 static int aead_decrypt(struct aead_request *req)
1294 {
1295         struct crypto_aead *authenc = crypto_aead_reqtfm(req);
1296         struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
1297         unsigned int authsize = ctx->authsize;
1298         struct talitos_private *priv = dev_get_drvdata(ctx->dev);
1299         struct talitos_edesc *edesc;
1300         struct scatterlist *sg;
1301         void *icvdata;
1302
1303         req->cryptlen -= authsize;
1304
1305         /* allocate extended descriptor */
1306         edesc = aead_edesc_alloc(req, req->iv, 1, false);
1307         if (IS_ERR(edesc))
1308                 return PTR_ERR(edesc);
1309
1310         if ((priv->features & TALITOS_FTR_HW_AUTH_CHECK) &&
1311             ((!edesc->src_nents && !edesc->dst_nents) ||
1312              priv->features & TALITOS_FTR_SRC_LINK_TBL_LEN_INCLUDES_EXTENT)) {
1313
1314                 /* decrypt and check the ICV */
1315                 edesc->desc.hdr = ctx->desc_hdr_template |
1316                                   DESC_HDR_DIR_INBOUND |
1317                                   DESC_HDR_MODE1_MDEU_CICV;
1318
1319                 /* reset integrity check result bits */
1320                 edesc->desc.hdr_lo = 0;
1321
1322                 return ipsec_esp(edesc, req, 0, ipsec_esp_decrypt_hwauth_done);
1323         }
1324
1325         /* Have to check the ICV with software */
1326         edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_DIR_INBOUND;
1327
1328         /* stash incoming ICV for later cmp with ICV generated by the h/w */
1329         if (edesc->dma_len)
1330                 icvdata = &edesc->link_tbl[edesc->src_nents +
1331                                            edesc->dst_nents + 2 +
1332                                            edesc->assoc_nents];
1333         else
1334                 icvdata = &edesc->link_tbl[0];
1335
1336         sg = sg_last(req->src, edesc->src_nents ? : 1);
1337
1338         memcpy(icvdata, (char *)sg_virt(sg) + sg->length - ctx->authsize,
1339                ctx->authsize);
1340
1341         return ipsec_esp(edesc, req, 0, ipsec_esp_decrypt_swauth_done);
1342 }
1343
1344 static int aead_givencrypt(struct aead_givcrypt_request *req)
1345 {
1346         struct aead_request *areq = &req->areq;
1347         struct crypto_aead *authenc = crypto_aead_reqtfm(areq);
1348         struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
1349         struct talitos_edesc *edesc;
1350
1351         /* allocate extended descriptor */
1352         edesc = aead_edesc_alloc(areq, req->giv, 0, true);
1353         if (IS_ERR(edesc))
1354                 return PTR_ERR(edesc);
1355
1356         /* set encrypt */
1357         edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_MODE0_ENCRYPT;
1358
1359         memcpy(req->giv, ctx->iv, crypto_aead_ivsize(authenc));
1360         /* avoid consecutive packets going out with same IV */
1361         *(__be64 *)req->giv ^= cpu_to_be64(req->seq);
1362
1363         return ipsec_esp(edesc, areq, req->seq, ipsec_esp_encrypt_done);
1364 }
1365
1366 static int ablkcipher_setkey(struct crypto_ablkcipher *cipher,
1367                              const u8 *key, unsigned int keylen)
1368 {
1369         struct talitos_ctx *ctx = crypto_ablkcipher_ctx(cipher);
1370
1371         memcpy(&ctx->key, key, keylen);
1372         ctx->keylen = keylen;
1373
1374         return 0;
1375 }
1376
1377 static void unmap_sg_talitos_ptr(struct device *dev, struct scatterlist *src,
1378                                  struct scatterlist *dst, unsigned int len,
1379                                  struct talitos_edesc *edesc)
1380 {
1381         talitos_sg_unmap(dev, edesc, src, dst);
1382 }
1383
1384 static void common_nonsnoop_unmap(struct device *dev,
1385                                   struct talitos_edesc *edesc,
1386                                   struct ablkcipher_request *areq)
1387 {
1388         unmap_single_talitos_ptr(dev, &edesc->desc.ptr[5], DMA_FROM_DEVICE);
1389
1390         unmap_sg_talitos_ptr(dev, areq->src, areq->dst, areq->nbytes, edesc);
1391         unmap_single_talitos_ptr(dev, &edesc->desc.ptr[2], DMA_TO_DEVICE);
1392         unmap_single_talitos_ptr(dev, &edesc->desc.ptr[1], DMA_TO_DEVICE);
1393
1394         if (edesc->dma_len)
1395                 dma_unmap_single(dev, edesc->dma_link_tbl, edesc->dma_len,
1396                                  DMA_BIDIRECTIONAL);
1397 }
1398
1399 static void ablkcipher_done(struct device *dev,
1400                             struct talitos_desc *desc, void *context,
1401                             int err)
1402 {
1403         struct ablkcipher_request *areq = context;
1404         struct talitos_edesc *edesc;
1405
1406         edesc = container_of(desc, struct talitos_edesc, desc);
1407
1408         common_nonsnoop_unmap(dev, edesc, areq);
1409
1410         kfree(edesc);
1411
1412         areq->base.complete(&areq->base, err);
1413 }
1414
1415 int map_sg_in_talitos_ptr(struct device *dev, struct scatterlist *src,
1416                           unsigned int len, struct talitos_edesc *edesc,
1417                           enum dma_data_direction dir, struct talitos_ptr *ptr)
1418 {
1419         int sg_count;
1420         struct talitos_private *priv = dev_get_drvdata(dev);
1421         bool is_sec1 = has_ftr_sec1(priv);
1422
1423         to_talitos_ptr_len(ptr, len, is_sec1);
1424         to_talitos_ptr_extent_clear(ptr, is_sec1);
1425
1426         sg_count = talitos_map_sg(dev, src, edesc->src_nents ? : 1, dir,
1427                                   edesc->src_chained);
1428
1429         if (sg_count == 1) {
1430                 to_talitos_ptr(ptr, sg_dma_address(src), is_sec1);
1431         } else {
1432                 sg_count = sg_to_link_tbl(src, sg_count, len,
1433                                           &edesc->link_tbl[0]);
1434                 if (sg_count > 1) {
1435                         to_talitos_ptr(ptr, edesc->dma_link_tbl, 0);
1436                         ptr->j_extent |= DESC_PTR_LNKTBL_JUMP;
1437                         dma_sync_single_for_device(dev, edesc->dma_link_tbl,
1438                                                    edesc->dma_len,
1439                                                    DMA_BIDIRECTIONAL);
1440                 } else {
1441                         /* Only one segment now, so no link tbl needed */
1442                         to_talitos_ptr(ptr, sg_dma_address(src), is_sec1);
1443                 }
1444         }
1445         return sg_count;
1446 }
1447
1448 void map_sg_out_talitos_ptr(struct device *dev, struct scatterlist *dst,
1449                             unsigned int len, struct talitos_edesc *edesc,
1450                             enum dma_data_direction dir,
1451                             struct talitos_ptr *ptr, int sg_count)
1452 {
1453         struct talitos_private *priv = dev_get_drvdata(dev);
1454         bool is_sec1 = has_ftr_sec1(priv);
1455
1456         to_talitos_ptr_len(ptr, len, is_sec1);
1457         to_talitos_ptr_extent_clear(ptr, is_sec1);
1458
1459         if (dir != DMA_NONE)
1460                 sg_count = talitos_map_sg(dev, dst, edesc->dst_nents ? : 1,
1461                                           dir, edesc->dst_chained);
1462
1463         if (sg_count == 1) {
1464                 to_talitos_ptr(ptr, sg_dma_address(dst), is_sec1);
1465         } else {
1466                 struct talitos_ptr *link_tbl_ptr =
1467                         &edesc->link_tbl[edesc->src_nents + 1];
1468
1469                 to_talitos_ptr(ptr, edesc->dma_link_tbl +
1470                                               (edesc->src_nents + 1) *
1471                                               sizeof(struct talitos_ptr), 0);
1472                 ptr->j_extent |= DESC_PTR_LNKTBL_JUMP;
1473                 sg_count = sg_to_link_tbl(dst, sg_count, len, link_tbl_ptr);
1474                 dma_sync_single_for_device(dev, edesc->dma_link_tbl,
1475                                            edesc->dma_len, DMA_BIDIRECTIONAL);
1476         }
1477 }
1478
1479 static int common_nonsnoop(struct talitos_edesc *edesc,
1480                            struct ablkcipher_request *areq,
1481                            void (*callback) (struct device *dev,
1482                                              struct talitos_desc *desc,
1483                                              void *context, int error))
1484 {
1485         struct crypto_ablkcipher *cipher = crypto_ablkcipher_reqtfm(areq);
1486         struct talitos_ctx *ctx = crypto_ablkcipher_ctx(cipher);
1487         struct device *dev = ctx->dev;
1488         struct talitos_desc *desc = &edesc->desc;
1489         unsigned int cryptlen = areq->nbytes;
1490         unsigned int ivsize = crypto_ablkcipher_ivsize(cipher);
1491         int sg_count, ret;
1492         struct talitos_private *priv = dev_get_drvdata(dev);
1493         bool is_sec1 = has_ftr_sec1(priv);
1494
1495         /* first DWORD empty */
1496         desc->ptr[0] = zero_entry;
1497
1498         /* cipher iv */
1499         to_talitos_ptr(&desc->ptr[1], edesc->iv_dma, is_sec1);
1500         to_talitos_ptr_len(&desc->ptr[1], ivsize, is_sec1);
1501         to_talitos_ptr_extent_clear(&desc->ptr[1], is_sec1);
1502
1503         /* cipher key */
1504         map_single_talitos_ptr(dev, &desc->ptr[2], ctx->keylen,
1505                                (char *)&ctx->key, DMA_TO_DEVICE);
1506
1507         /*
1508          * cipher in
1509          */
1510         sg_count = map_sg_in_talitos_ptr(dev, areq->src, cryptlen, edesc,
1511                                          (areq->src == areq->dst) ?
1512                                           DMA_BIDIRECTIONAL : DMA_TO_DEVICE,
1513                                           &desc->ptr[3]);
1514
1515         /* cipher out */
1516         map_sg_out_talitos_ptr(dev, areq->dst, cryptlen, edesc,
1517                                (areq->src == areq->dst) ? DMA_NONE
1518                                                         : DMA_FROM_DEVICE,
1519                                &desc->ptr[4], sg_count);
1520
1521         /* iv out */
1522         map_single_talitos_ptr(dev, &desc->ptr[5], ivsize, ctx->iv,
1523                                DMA_FROM_DEVICE);
1524
1525         /* last DWORD empty */
1526         desc->ptr[6] = zero_entry;
1527
1528         ret = talitos_submit(dev, ctx->ch, desc, callback, areq);
1529         if (ret != -EINPROGRESS) {
1530                 common_nonsnoop_unmap(dev, edesc, areq);
1531                 kfree(edesc);
1532         }
1533         return ret;
1534 }
1535
1536 static struct talitos_edesc *ablkcipher_edesc_alloc(struct ablkcipher_request *
1537                                                     areq, bool encrypt)
1538 {
1539         struct crypto_ablkcipher *cipher = crypto_ablkcipher_reqtfm(areq);
1540         struct talitos_ctx *ctx = crypto_ablkcipher_ctx(cipher);
1541         unsigned int ivsize = crypto_ablkcipher_ivsize(cipher);
1542
1543         return talitos_edesc_alloc(ctx->dev, NULL, areq->src, areq->dst,
1544                                    areq->info, 0, areq->nbytes, 0, ivsize, 0,
1545                                    areq->base.flags, encrypt);
1546 }
1547
1548 static int ablkcipher_encrypt(struct ablkcipher_request *areq)
1549 {
1550         struct crypto_ablkcipher *cipher = crypto_ablkcipher_reqtfm(areq);
1551         struct talitos_ctx *ctx = crypto_ablkcipher_ctx(cipher);
1552         struct talitos_edesc *edesc;
1553
1554         /* allocate extended descriptor */
1555         edesc = ablkcipher_edesc_alloc(areq, true);
1556         if (IS_ERR(edesc))
1557                 return PTR_ERR(edesc);
1558
1559         /* set encrypt */
1560         edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_MODE0_ENCRYPT;
1561
1562         return common_nonsnoop(edesc, areq, ablkcipher_done);
1563 }
1564
1565 static int ablkcipher_decrypt(struct ablkcipher_request *areq)
1566 {
1567         struct crypto_ablkcipher *cipher = crypto_ablkcipher_reqtfm(areq);
1568         struct talitos_ctx *ctx = crypto_ablkcipher_ctx(cipher);
1569         struct talitos_edesc *edesc;
1570
1571         /* allocate extended descriptor */
1572         edesc = ablkcipher_edesc_alloc(areq, false);
1573         if (IS_ERR(edesc))
1574                 return PTR_ERR(edesc);
1575
1576         edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_DIR_INBOUND;
1577
1578         return common_nonsnoop(edesc, areq, ablkcipher_done);
1579 }
1580
1581 static void common_nonsnoop_hash_unmap(struct device *dev,
1582                                        struct talitos_edesc *edesc,
1583                                        struct ahash_request *areq)
1584 {
1585         struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1586         struct talitos_private *priv = dev_get_drvdata(dev);
1587         bool is_sec1 = has_ftr_sec1(priv);
1588
1589         unmap_single_talitos_ptr(dev, &edesc->desc.ptr[5], DMA_FROM_DEVICE);
1590
1591         unmap_sg_talitos_ptr(dev, req_ctx->psrc, NULL, 0, edesc);
1592
1593         /* When using hashctx-in, must unmap it. */
1594         if (from_talitos_ptr_len(&edesc->desc.ptr[1], is_sec1))
1595                 unmap_single_talitos_ptr(dev, &edesc->desc.ptr[1],
1596                                          DMA_TO_DEVICE);
1597
1598         if (from_talitos_ptr_len(&edesc->desc.ptr[2], is_sec1))
1599                 unmap_single_talitos_ptr(dev, &edesc->desc.ptr[2],
1600                                          DMA_TO_DEVICE);
1601
1602         if (edesc->dma_len)
1603                 dma_unmap_single(dev, edesc->dma_link_tbl, edesc->dma_len,
1604                                  DMA_BIDIRECTIONAL);
1605
1606 }
1607
1608 static void ahash_done(struct device *dev,
1609                        struct talitos_desc *desc, void *context,
1610                        int err)
1611 {
1612         struct ahash_request *areq = context;
1613         struct talitos_edesc *edesc =
1614                  container_of(desc, struct talitos_edesc, desc);
1615         struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1616
1617         if (!req_ctx->last && req_ctx->to_hash_later) {
1618                 /* Position any partial block for next update/final/finup */
1619                 memcpy(req_ctx->buf, req_ctx->bufnext, req_ctx->to_hash_later);
1620                 req_ctx->nbuf = req_ctx->to_hash_later;
1621         }
1622         common_nonsnoop_hash_unmap(dev, edesc, areq);
1623
1624         kfree(edesc);
1625
1626         areq->base.complete(&areq->base, err);
1627 }
1628
1629 static int common_nonsnoop_hash(struct talitos_edesc *edesc,
1630                                 struct ahash_request *areq, unsigned int length,
1631                                 void (*callback) (struct device *dev,
1632                                                   struct talitos_desc *desc,
1633                                                   void *context, int error))
1634 {
1635         struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
1636         struct talitos_ctx *ctx = crypto_ahash_ctx(tfm);
1637         struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1638         struct device *dev = ctx->dev;
1639         struct talitos_desc *desc = &edesc->desc;
1640         int ret;
1641         struct talitos_private *priv = dev_get_drvdata(dev);
1642         bool is_sec1 = has_ftr_sec1(priv);
1643
1644         /* first DWORD empty */
1645         desc->ptr[0] = zero_entry;
1646
1647         /* hash context in */
1648         if (!req_ctx->first || req_ctx->swinit) {
1649                 map_single_talitos_ptr(dev, &desc->ptr[1],
1650                                        req_ctx->hw_context_size,
1651                                        (char *)req_ctx->hw_context,
1652                                        DMA_TO_DEVICE);
1653                 req_ctx->swinit = 0;
1654         } else {
1655                 desc->ptr[1] = zero_entry;
1656                 /* Indicate next op is not the first. */
1657                 req_ctx->first = 0;
1658         }
1659
1660         /* HMAC key */
1661         if (ctx->keylen)
1662                 map_single_talitos_ptr(dev, &desc->ptr[2], ctx->keylen,
1663                                        (char *)&ctx->key, DMA_TO_DEVICE);
1664         else
1665                 desc->ptr[2] = zero_entry;
1666
1667         /*
1668          * data in
1669          */
1670         map_sg_in_talitos_ptr(dev, req_ctx->psrc, length, edesc,
1671                               DMA_TO_DEVICE, &desc->ptr[3]);
1672
1673         /* fifth DWORD empty */
1674         desc->ptr[4] = zero_entry;
1675
1676         /* hash/HMAC out -or- hash context out */
1677         if (req_ctx->last)
1678                 map_single_talitos_ptr(dev, &desc->ptr[5],
1679                                        crypto_ahash_digestsize(tfm),
1680                                        areq->result, DMA_FROM_DEVICE);
1681         else
1682                 map_single_talitos_ptr(dev, &desc->ptr[5],
1683                                        req_ctx->hw_context_size,
1684                                        req_ctx->hw_context, DMA_FROM_DEVICE);
1685
1686         /* last DWORD empty */
1687         desc->ptr[6] = zero_entry;
1688
1689         ret = talitos_submit(dev, ctx->ch, desc, callback, areq);
1690         if (ret != -EINPROGRESS) {
1691                 common_nonsnoop_hash_unmap(dev, edesc, areq);
1692                 kfree(edesc);
1693         }
1694         return ret;
1695 }
1696
1697 static struct talitos_edesc *ahash_edesc_alloc(struct ahash_request *areq,
1698                                                unsigned int nbytes)
1699 {
1700         struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
1701         struct talitos_ctx *ctx = crypto_ahash_ctx(tfm);
1702         struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1703
1704         return talitos_edesc_alloc(ctx->dev, NULL, req_ctx->psrc, NULL, NULL, 0,
1705                                    nbytes, 0, 0, 0, areq->base.flags, false);
1706 }
1707
1708 static int ahash_init(struct ahash_request *areq)
1709 {
1710         struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
1711         struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1712
1713         /* Initialize the context */
1714         req_ctx->nbuf = 0;
1715         req_ctx->first = 1; /* first indicates h/w must init its context */
1716         req_ctx->swinit = 0; /* assume h/w init of context */
1717         req_ctx->hw_context_size =
1718                 (crypto_ahash_digestsize(tfm) <= SHA256_DIGEST_SIZE)
1719                         ? TALITOS_MDEU_CONTEXT_SIZE_MD5_SHA1_SHA256
1720                         : TALITOS_MDEU_CONTEXT_SIZE_SHA384_SHA512;
1721
1722         return 0;
1723 }
1724
1725 /*
1726  * on h/w without explicit sha224 support, we initialize h/w context
1727  * manually with sha224 constants, and tell it to run sha256.
1728  */
1729 static int ahash_init_sha224_swinit(struct ahash_request *areq)
1730 {
1731         struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1732
1733         ahash_init(areq);
1734         req_ctx->swinit = 1;/* prevent h/w initting context with sha256 values*/
1735
1736         req_ctx->hw_context[0] = SHA224_H0;
1737         req_ctx->hw_context[1] = SHA224_H1;
1738         req_ctx->hw_context[2] = SHA224_H2;
1739         req_ctx->hw_context[3] = SHA224_H3;
1740         req_ctx->hw_context[4] = SHA224_H4;
1741         req_ctx->hw_context[5] = SHA224_H5;
1742         req_ctx->hw_context[6] = SHA224_H6;
1743         req_ctx->hw_context[7] = SHA224_H7;
1744
1745         /* init 64-bit count */
1746         req_ctx->hw_context[8] = 0;
1747         req_ctx->hw_context[9] = 0;
1748
1749         return 0;
1750 }
1751
1752 static int ahash_process_req(struct ahash_request *areq, unsigned int nbytes)
1753 {
1754         struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
1755         struct talitos_ctx *ctx = crypto_ahash_ctx(tfm);
1756         struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1757         struct talitos_edesc *edesc;
1758         unsigned int blocksize =
1759                         crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm));
1760         unsigned int nbytes_to_hash;
1761         unsigned int to_hash_later;
1762         unsigned int nsg;
1763         bool chained;
1764
1765         if (!req_ctx->last && (nbytes + req_ctx->nbuf <= blocksize)) {
1766                 /* Buffer up to one whole block */
1767                 sg_copy_to_buffer(areq->src,
1768                                   sg_count(areq->src, nbytes, &chained),
1769                                   req_ctx->buf + req_ctx->nbuf, nbytes);
1770                 req_ctx->nbuf += nbytes;
1771                 return 0;
1772         }
1773
1774         /* At least (blocksize + 1) bytes are available to hash */
1775         nbytes_to_hash = nbytes + req_ctx->nbuf;
1776         to_hash_later = nbytes_to_hash & (blocksize - 1);
1777
1778         if (req_ctx->last)
1779                 to_hash_later = 0;
1780         else if (to_hash_later)
1781                 /* There is a partial block. Hash the full block(s) now */
1782                 nbytes_to_hash -= to_hash_later;
1783         else {
1784                 /* Keep one block buffered */
1785                 nbytes_to_hash -= blocksize;
1786                 to_hash_later = blocksize;
1787         }
1788
1789         /* Chain in any previously buffered data */
1790         if (req_ctx->nbuf) {
1791                 nsg = (req_ctx->nbuf < nbytes_to_hash) ? 2 : 1;
1792                 sg_init_table(req_ctx->bufsl, nsg);
1793                 sg_set_buf(req_ctx->bufsl, req_ctx->buf, req_ctx->nbuf);
1794                 if (nsg > 1)
1795                         scatterwalk_sg_chain(req_ctx->bufsl, 2, areq->src);
1796                 req_ctx->psrc = req_ctx->bufsl;
1797         } else
1798                 req_ctx->psrc = areq->src;
1799
1800         if (to_hash_later) {
1801                 int nents = sg_count(areq->src, nbytes, &chained);
1802                 sg_pcopy_to_buffer(areq->src, nents,
1803                                       req_ctx->bufnext,
1804                                       to_hash_later,
1805                                       nbytes - to_hash_later);
1806         }
1807         req_ctx->to_hash_later = to_hash_later;
1808
1809         /* Allocate extended descriptor */
1810         edesc = ahash_edesc_alloc(areq, nbytes_to_hash);
1811         if (IS_ERR(edesc))
1812                 return PTR_ERR(edesc);
1813
1814         edesc->desc.hdr = ctx->desc_hdr_template;
1815
1816         /* On last one, request SEC to pad; otherwise continue */
1817         if (req_ctx->last)
1818                 edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_PAD;
1819         else
1820                 edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_CONT;
1821
1822         /* request SEC to INIT hash. */
1823         if (req_ctx->first && !req_ctx->swinit)
1824                 edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_INIT;
1825
1826         /* When the tfm context has a keylen, it's an HMAC.
1827          * A first or last (ie. not middle) descriptor must request HMAC.
1828          */
1829         if (ctx->keylen && (req_ctx->first || req_ctx->last))
1830                 edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_HMAC;
1831
1832         return common_nonsnoop_hash(edesc, areq, nbytes_to_hash,
1833                                     ahash_done);
1834 }
1835
1836 static int ahash_update(struct ahash_request *areq)
1837 {
1838         struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1839
1840         req_ctx->last = 0;
1841
1842         return ahash_process_req(areq, areq->nbytes);
1843 }
1844
1845 static int ahash_final(struct ahash_request *areq)
1846 {
1847         struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1848
1849         req_ctx->last = 1;
1850
1851         return ahash_process_req(areq, 0);
1852 }
1853
1854 static int ahash_finup(struct ahash_request *areq)
1855 {
1856         struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1857
1858         req_ctx->last = 1;
1859
1860         return ahash_process_req(areq, areq->nbytes);
1861 }
1862
1863 static int ahash_digest(struct ahash_request *areq)
1864 {
1865         struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1866         struct crypto_ahash *ahash = crypto_ahash_reqtfm(areq);
1867
1868         ahash->init(areq);
1869         req_ctx->last = 1;
1870
1871         return ahash_process_req(areq, areq->nbytes);
1872 }
1873
1874 struct keyhash_result {
1875         struct completion completion;
1876         int err;
1877 };
1878
1879 static void keyhash_complete(struct crypto_async_request *req, int err)
1880 {
1881         struct keyhash_result *res = req->data;
1882
1883         if (err == -EINPROGRESS)
1884                 return;
1885
1886         res->err = err;
1887         complete(&res->completion);
1888 }
1889
1890 static int keyhash(struct crypto_ahash *tfm, const u8 *key, unsigned int keylen,
1891                    u8 *hash)
1892 {
1893         struct talitos_ctx *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm));
1894
1895         struct scatterlist sg[1];
1896         struct ahash_request *req;
1897         struct keyhash_result hresult;
1898         int ret;
1899
1900         init_completion(&hresult.completion);
1901
1902         req = ahash_request_alloc(tfm, GFP_KERNEL);
1903         if (!req)
1904                 return -ENOMEM;
1905
1906         /* Keep tfm keylen == 0 during hash of the long key */
1907         ctx->keylen = 0;
1908         ahash_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,
1909                                    keyhash_complete, &hresult);
1910
1911         sg_init_one(&sg[0], key, keylen);
1912
1913         ahash_request_set_crypt(req, sg, hash, keylen);
1914         ret = crypto_ahash_digest(req);
1915         switch (ret) {
1916         case 0:
1917                 break;
1918         case -EINPROGRESS:
1919         case -EBUSY:
1920                 ret = wait_for_completion_interruptible(
1921                         &hresult.completion);
1922                 if (!ret)
1923                         ret = hresult.err;
1924                 break;
1925         default:
1926                 break;
1927         }
1928         ahash_request_free(req);
1929
1930         return ret;
1931 }
1932
1933 static int ahash_setkey(struct crypto_ahash *tfm, const u8 *key,
1934                         unsigned int keylen)
1935 {
1936         struct talitos_ctx *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm));
1937         unsigned int blocksize =
1938                         crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm));
1939         unsigned int digestsize = crypto_ahash_digestsize(tfm);
1940         unsigned int keysize = keylen;
1941         u8 hash[SHA512_DIGEST_SIZE];
1942         int ret;
1943
1944         if (keylen <= blocksize)
1945                 memcpy(ctx->key, key, keysize);
1946         else {
1947                 /* Must get the hash of the long key */
1948                 ret = keyhash(tfm, key, keylen, hash);
1949
1950                 if (ret) {
1951                         crypto_ahash_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
1952                         return -EINVAL;
1953                 }
1954
1955                 keysize = digestsize;
1956                 memcpy(ctx->key, hash, digestsize);
1957         }
1958
1959         ctx->keylen = keysize;
1960
1961         return 0;
1962 }
1963
1964
1965 struct talitos_alg_template {
1966         u32 type;
1967         union {
1968                 struct crypto_alg crypto;
1969                 struct ahash_alg hash;
1970         } alg;
1971         __be32 desc_hdr_template;
1972 };
1973
1974 static struct talitos_alg_template driver_algs[] = {
1975         /* AEAD algorithms.  These use a single-pass ipsec_esp descriptor */
1976         {       .type = CRYPTO_ALG_TYPE_AEAD,
1977                 .alg.crypto = {
1978                         .cra_name = "authenc(hmac(sha1),cbc(aes))",
1979                         .cra_driver_name = "authenc-hmac-sha1-cbc-aes-talitos",
1980                         .cra_blocksize = AES_BLOCK_SIZE,
1981                         .cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC,
1982                         .cra_aead = {
1983                                 .ivsize = AES_BLOCK_SIZE,
1984                                 .maxauthsize = SHA1_DIGEST_SIZE,
1985                         }
1986                 },
1987                 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
1988                                      DESC_HDR_SEL0_AESU |
1989                                      DESC_HDR_MODE0_AESU_CBC |
1990                                      DESC_HDR_SEL1_MDEUA |
1991                                      DESC_HDR_MODE1_MDEU_INIT |
1992                                      DESC_HDR_MODE1_MDEU_PAD |
1993                                      DESC_HDR_MODE1_MDEU_SHA1_HMAC,
1994         },
1995         {       .type = CRYPTO_ALG_TYPE_AEAD,
1996                 .alg.crypto = {
1997                         .cra_name = "authenc(hmac(sha1),cbc(des3_ede))",
1998                         .cra_driver_name = "authenc-hmac-sha1-cbc-3des-talitos",
1999                         .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2000                         .cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC,
2001                         .cra_aead = {
2002                                 .ivsize = DES3_EDE_BLOCK_SIZE,
2003                                 .maxauthsize = SHA1_DIGEST_SIZE,
2004                         }
2005                 },
2006                 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2007                                      DESC_HDR_SEL0_DEU |
2008                                      DESC_HDR_MODE0_DEU_CBC |
2009                                      DESC_HDR_MODE0_DEU_3DES |
2010                                      DESC_HDR_SEL1_MDEUA |
2011                                      DESC_HDR_MODE1_MDEU_INIT |
2012                                      DESC_HDR_MODE1_MDEU_PAD |
2013                                      DESC_HDR_MODE1_MDEU_SHA1_HMAC,
2014         },
2015         {       .type = CRYPTO_ALG_TYPE_AEAD,
2016                 .alg.crypto = {
2017                         .cra_name = "authenc(hmac(sha224),cbc(aes))",
2018                         .cra_driver_name = "authenc-hmac-sha224-cbc-aes-talitos",
2019                         .cra_blocksize = AES_BLOCK_SIZE,
2020                         .cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC,
2021                         .cra_aead = {
2022                                 .ivsize = AES_BLOCK_SIZE,
2023                                 .maxauthsize = SHA224_DIGEST_SIZE,
2024                         }
2025                 },
2026                 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2027                                      DESC_HDR_SEL0_AESU |
2028                                      DESC_HDR_MODE0_AESU_CBC |
2029                                      DESC_HDR_SEL1_MDEUA |
2030                                      DESC_HDR_MODE1_MDEU_INIT |
2031                                      DESC_HDR_MODE1_MDEU_PAD |
2032                                      DESC_HDR_MODE1_MDEU_SHA224_HMAC,
2033         },
2034         {       .type = CRYPTO_ALG_TYPE_AEAD,
2035                 .alg.crypto = {
2036                         .cra_name = "authenc(hmac(sha224),cbc(des3_ede))",
2037                         .cra_driver_name = "authenc-hmac-sha224-cbc-3des-talitos",
2038                         .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2039                         .cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC,
2040                         .cra_aead = {
2041                                 .ivsize = DES3_EDE_BLOCK_SIZE,
2042                                 .maxauthsize = SHA224_DIGEST_SIZE,
2043                         }
2044                 },
2045                 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2046                                      DESC_HDR_SEL0_DEU |
2047                                      DESC_HDR_MODE0_DEU_CBC |
2048                                      DESC_HDR_MODE0_DEU_3DES |
2049                                      DESC_HDR_SEL1_MDEUA |
2050                                      DESC_HDR_MODE1_MDEU_INIT |
2051                                      DESC_HDR_MODE1_MDEU_PAD |
2052                                      DESC_HDR_MODE1_MDEU_SHA224_HMAC,
2053         },
2054         {       .type = CRYPTO_ALG_TYPE_AEAD,
2055                 .alg.crypto = {
2056                         .cra_name = "authenc(hmac(sha256),cbc(aes))",
2057                         .cra_driver_name = "authenc-hmac-sha256-cbc-aes-talitos",
2058                         .cra_blocksize = AES_BLOCK_SIZE,
2059                         .cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC,
2060                         .cra_aead = {
2061                                 .ivsize = AES_BLOCK_SIZE,
2062                                 .maxauthsize = SHA256_DIGEST_SIZE,
2063                         }
2064                 },
2065                 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2066                                      DESC_HDR_SEL0_AESU |
2067                                      DESC_HDR_MODE0_AESU_CBC |
2068                                      DESC_HDR_SEL1_MDEUA |
2069                                      DESC_HDR_MODE1_MDEU_INIT |
2070                                      DESC_HDR_MODE1_MDEU_PAD |
2071                                      DESC_HDR_MODE1_MDEU_SHA256_HMAC,
2072         },
2073         {       .type = CRYPTO_ALG_TYPE_AEAD,
2074                 .alg.crypto = {
2075                         .cra_name = "authenc(hmac(sha256),cbc(des3_ede))",
2076                         .cra_driver_name = "authenc-hmac-sha256-cbc-3des-talitos",
2077                         .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2078                         .cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC,
2079                         .cra_aead = {
2080                                 .ivsize = DES3_EDE_BLOCK_SIZE,
2081                                 .maxauthsize = SHA256_DIGEST_SIZE,
2082                         }
2083                 },
2084                 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2085                                      DESC_HDR_SEL0_DEU |
2086                                      DESC_HDR_MODE0_DEU_CBC |
2087                                      DESC_HDR_MODE0_DEU_3DES |
2088                                      DESC_HDR_SEL1_MDEUA |
2089                                      DESC_HDR_MODE1_MDEU_INIT |
2090                                      DESC_HDR_MODE1_MDEU_PAD |
2091                                      DESC_HDR_MODE1_MDEU_SHA256_HMAC,
2092         },
2093         {       .type = CRYPTO_ALG_TYPE_AEAD,
2094                 .alg.crypto = {
2095                         .cra_name = "authenc(hmac(sha384),cbc(aes))",
2096                         .cra_driver_name = "authenc-hmac-sha384-cbc-aes-talitos",
2097                         .cra_blocksize = AES_BLOCK_SIZE,
2098                         .cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC,
2099                         .cra_aead = {
2100                                 .ivsize = AES_BLOCK_SIZE,
2101                                 .maxauthsize = SHA384_DIGEST_SIZE,
2102                         }
2103                 },
2104                 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2105                                      DESC_HDR_SEL0_AESU |
2106                                      DESC_HDR_MODE0_AESU_CBC |
2107                                      DESC_HDR_SEL1_MDEUB |
2108                                      DESC_HDR_MODE1_MDEU_INIT |
2109                                      DESC_HDR_MODE1_MDEU_PAD |
2110                                      DESC_HDR_MODE1_MDEUB_SHA384_HMAC,
2111         },
2112         {       .type = CRYPTO_ALG_TYPE_AEAD,
2113                 .alg.crypto = {
2114                         .cra_name = "authenc(hmac(sha384),cbc(des3_ede))",
2115                         .cra_driver_name = "authenc-hmac-sha384-cbc-3des-talitos",
2116                         .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2117                         .cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC,
2118                         .cra_aead = {
2119                                 .ivsize = DES3_EDE_BLOCK_SIZE,
2120                                 .maxauthsize = SHA384_DIGEST_SIZE,
2121                         }
2122                 },
2123                 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2124                                      DESC_HDR_SEL0_DEU |
2125                                      DESC_HDR_MODE0_DEU_CBC |
2126                                      DESC_HDR_MODE0_DEU_3DES |
2127                                      DESC_HDR_SEL1_MDEUB |
2128                                      DESC_HDR_MODE1_MDEU_INIT |
2129                                      DESC_HDR_MODE1_MDEU_PAD |
2130                                      DESC_HDR_MODE1_MDEUB_SHA384_HMAC,
2131         },
2132         {       .type = CRYPTO_ALG_TYPE_AEAD,
2133                 .alg.crypto = {
2134                         .cra_name = "authenc(hmac(sha512),cbc(aes))",
2135                         .cra_driver_name = "authenc-hmac-sha512-cbc-aes-talitos",
2136                         .cra_blocksize = AES_BLOCK_SIZE,
2137                         .cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC,
2138                         .cra_aead = {
2139                                 .ivsize = AES_BLOCK_SIZE,
2140                                 .maxauthsize = SHA512_DIGEST_SIZE,
2141                         }
2142                 },
2143                 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2144                                      DESC_HDR_SEL0_AESU |
2145                                      DESC_HDR_MODE0_AESU_CBC |
2146                                      DESC_HDR_SEL1_MDEUB |
2147                                      DESC_HDR_MODE1_MDEU_INIT |
2148                                      DESC_HDR_MODE1_MDEU_PAD |
2149                                      DESC_HDR_MODE1_MDEUB_SHA512_HMAC,
2150         },
2151         {       .type = CRYPTO_ALG_TYPE_AEAD,
2152                 .alg.crypto = {
2153                         .cra_name = "authenc(hmac(sha512),cbc(des3_ede))",
2154                         .cra_driver_name = "authenc-hmac-sha512-cbc-3des-talitos",
2155                         .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2156                         .cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC,
2157                         .cra_aead = {
2158                                 .ivsize = DES3_EDE_BLOCK_SIZE,
2159                                 .maxauthsize = SHA512_DIGEST_SIZE,
2160                         }
2161                 },
2162                 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2163                                      DESC_HDR_SEL0_DEU |
2164                                      DESC_HDR_MODE0_DEU_CBC |
2165                                      DESC_HDR_MODE0_DEU_3DES |
2166                                      DESC_HDR_SEL1_MDEUB |
2167                                      DESC_HDR_MODE1_MDEU_INIT |
2168                                      DESC_HDR_MODE1_MDEU_PAD |
2169                                      DESC_HDR_MODE1_MDEUB_SHA512_HMAC,
2170         },
2171         {       .type = CRYPTO_ALG_TYPE_AEAD,
2172                 .alg.crypto = {
2173                         .cra_name = "authenc(hmac(md5),cbc(aes))",
2174                         .cra_driver_name = "authenc-hmac-md5-cbc-aes-talitos",
2175                         .cra_blocksize = AES_BLOCK_SIZE,
2176                         .cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC,
2177                         .cra_aead = {
2178                                 .ivsize = AES_BLOCK_SIZE,
2179                                 .maxauthsize = MD5_DIGEST_SIZE,
2180                         }
2181                 },
2182                 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2183                                      DESC_HDR_SEL0_AESU |
2184                                      DESC_HDR_MODE0_AESU_CBC |
2185                                      DESC_HDR_SEL1_MDEUA |
2186                                      DESC_HDR_MODE1_MDEU_INIT |
2187                                      DESC_HDR_MODE1_MDEU_PAD |
2188                                      DESC_HDR_MODE1_MDEU_MD5_HMAC,
2189         },
2190         {       .type = CRYPTO_ALG_TYPE_AEAD,
2191                 .alg.crypto = {
2192                         .cra_name = "authenc(hmac(md5),cbc(des3_ede))",
2193                         .cra_driver_name = "authenc-hmac-md5-cbc-3des-talitos",
2194                         .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2195                         .cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC,
2196                         .cra_aead = {
2197                                 .ivsize = DES3_EDE_BLOCK_SIZE,
2198                                 .maxauthsize = MD5_DIGEST_SIZE,
2199                         }
2200                 },
2201                 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2202                                      DESC_HDR_SEL0_DEU |
2203                                      DESC_HDR_MODE0_DEU_CBC |
2204                                      DESC_HDR_MODE0_DEU_3DES |
2205                                      DESC_HDR_SEL1_MDEUA |
2206                                      DESC_HDR_MODE1_MDEU_INIT |
2207                                      DESC_HDR_MODE1_MDEU_PAD |
2208                                      DESC_HDR_MODE1_MDEU_MD5_HMAC,
2209         },
2210         /* ABLKCIPHER algorithms. */
2211         {       .type = CRYPTO_ALG_TYPE_ABLKCIPHER,
2212                 .alg.crypto = {
2213                         .cra_name = "cbc(aes)",
2214                         .cra_driver_name = "cbc-aes-talitos",
2215                         .cra_blocksize = AES_BLOCK_SIZE,
2216                         .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
2217                                      CRYPTO_ALG_ASYNC,
2218                         .cra_ablkcipher = {
2219                                 .min_keysize = AES_MIN_KEY_SIZE,
2220                                 .max_keysize = AES_MAX_KEY_SIZE,
2221                                 .ivsize = AES_BLOCK_SIZE,
2222                         }
2223                 },
2224                 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2225                                      DESC_HDR_SEL0_AESU |
2226                                      DESC_HDR_MODE0_AESU_CBC,
2227         },
2228         {       .type = CRYPTO_ALG_TYPE_ABLKCIPHER,
2229                 .alg.crypto = {
2230                         .cra_name = "cbc(des3_ede)",
2231                         .cra_driver_name = "cbc-3des-talitos",
2232                         .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2233                         .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
2234                                      CRYPTO_ALG_ASYNC,
2235                         .cra_ablkcipher = {
2236                                 .min_keysize = DES3_EDE_KEY_SIZE,
2237                                 .max_keysize = DES3_EDE_KEY_SIZE,
2238                                 .ivsize = DES3_EDE_BLOCK_SIZE,
2239                         }
2240                 },
2241                 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2242                                      DESC_HDR_SEL0_DEU |
2243                                      DESC_HDR_MODE0_DEU_CBC |
2244                                      DESC_HDR_MODE0_DEU_3DES,
2245         },
2246         /* AHASH algorithms. */
2247         {       .type = CRYPTO_ALG_TYPE_AHASH,
2248                 .alg.hash = {
2249                         .halg.digestsize = MD5_DIGEST_SIZE,
2250                         .halg.base = {
2251                                 .cra_name = "md5",
2252                                 .cra_driver_name = "md5-talitos",
2253                                 .cra_blocksize = MD5_HMAC_BLOCK_SIZE,
2254                                 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2255                                              CRYPTO_ALG_ASYNC,
2256                         }
2257                 },
2258                 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2259                                      DESC_HDR_SEL0_MDEUA |
2260                                      DESC_HDR_MODE0_MDEU_MD5,
2261         },
2262         {       .type = CRYPTO_ALG_TYPE_AHASH,
2263                 .alg.hash = {
2264                         .halg.digestsize = SHA1_DIGEST_SIZE,
2265                         .halg.base = {
2266                                 .cra_name = "sha1",
2267                                 .cra_driver_name = "sha1-talitos",
2268                                 .cra_blocksize = SHA1_BLOCK_SIZE,
2269                                 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2270                                              CRYPTO_ALG_ASYNC,
2271                         }
2272                 },
2273                 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2274                                      DESC_HDR_SEL0_MDEUA |
2275                                      DESC_HDR_MODE0_MDEU_SHA1,
2276         },
2277         {       .type = CRYPTO_ALG_TYPE_AHASH,
2278                 .alg.hash = {
2279                         .halg.digestsize = SHA224_DIGEST_SIZE,
2280                         .halg.base = {
2281                                 .cra_name = "sha224",
2282                                 .cra_driver_name = "sha224-talitos",
2283                                 .cra_blocksize = SHA224_BLOCK_SIZE,
2284                                 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2285                                              CRYPTO_ALG_ASYNC,
2286                         }
2287                 },
2288                 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2289                                      DESC_HDR_SEL0_MDEUA |
2290                                      DESC_HDR_MODE0_MDEU_SHA224,
2291         },
2292         {       .type = CRYPTO_ALG_TYPE_AHASH,
2293                 .alg.hash = {
2294                         .halg.digestsize = SHA256_DIGEST_SIZE,
2295                         .halg.base = {
2296                                 .cra_name = "sha256",
2297                                 .cra_driver_name = "sha256-talitos",
2298                                 .cra_blocksize = SHA256_BLOCK_SIZE,
2299                                 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2300                                              CRYPTO_ALG_ASYNC,
2301                         }
2302                 },
2303                 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2304                                      DESC_HDR_SEL0_MDEUA |
2305                                      DESC_HDR_MODE0_MDEU_SHA256,
2306         },
2307         {       .type = CRYPTO_ALG_TYPE_AHASH,
2308                 .alg.hash = {
2309                         .halg.digestsize = SHA384_DIGEST_SIZE,
2310                         .halg.base = {
2311                                 .cra_name = "sha384",
2312                                 .cra_driver_name = "sha384-talitos",
2313                                 .cra_blocksize = SHA384_BLOCK_SIZE,
2314                                 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2315                                              CRYPTO_ALG_ASYNC,
2316                         }
2317                 },
2318                 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2319                                      DESC_HDR_SEL0_MDEUB |
2320                                      DESC_HDR_MODE0_MDEUB_SHA384,
2321         },
2322         {       .type = CRYPTO_ALG_TYPE_AHASH,
2323                 .alg.hash = {
2324                         .halg.digestsize = SHA512_DIGEST_SIZE,
2325                         .halg.base = {
2326                                 .cra_name = "sha512",
2327                                 .cra_driver_name = "sha512-talitos",
2328                                 .cra_blocksize = SHA512_BLOCK_SIZE,
2329                                 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2330                                              CRYPTO_ALG_ASYNC,
2331                         }
2332                 },
2333                 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2334                                      DESC_HDR_SEL0_MDEUB |
2335                                      DESC_HDR_MODE0_MDEUB_SHA512,
2336         },
2337         {       .type = CRYPTO_ALG_TYPE_AHASH,
2338                 .alg.hash = {
2339                         .halg.digestsize = MD5_DIGEST_SIZE,
2340                         .halg.base = {
2341                                 .cra_name = "hmac(md5)",
2342                                 .cra_driver_name = "hmac-md5-talitos",
2343                                 .cra_blocksize = MD5_HMAC_BLOCK_SIZE,
2344                                 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2345                                              CRYPTO_ALG_ASYNC,
2346                         }
2347                 },
2348                 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2349                                      DESC_HDR_SEL0_MDEUA |
2350                                      DESC_HDR_MODE0_MDEU_MD5,
2351         },
2352         {       .type = CRYPTO_ALG_TYPE_AHASH,
2353                 .alg.hash = {
2354                         .halg.digestsize = SHA1_DIGEST_SIZE,
2355                         .halg.base = {
2356                                 .cra_name = "hmac(sha1)",
2357                                 .cra_driver_name = "hmac-sha1-talitos",
2358                                 .cra_blocksize = SHA1_BLOCK_SIZE,
2359                                 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2360                                              CRYPTO_ALG_ASYNC,
2361                         }
2362                 },
2363                 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2364                                      DESC_HDR_SEL0_MDEUA |
2365                                      DESC_HDR_MODE0_MDEU_SHA1,
2366         },
2367         {       .type = CRYPTO_ALG_TYPE_AHASH,
2368                 .alg.hash = {
2369                         .halg.digestsize = SHA224_DIGEST_SIZE,
2370                         .halg.base = {
2371                                 .cra_name = "hmac(sha224)",
2372                                 .cra_driver_name = "hmac-sha224-talitos",
2373                                 .cra_blocksize = SHA224_BLOCK_SIZE,
2374                                 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2375                                              CRYPTO_ALG_ASYNC,
2376                         }
2377                 },
2378                 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2379                                      DESC_HDR_SEL0_MDEUA |
2380                                      DESC_HDR_MODE0_MDEU_SHA224,
2381         },
2382         {       .type = CRYPTO_ALG_TYPE_AHASH,
2383                 .alg.hash = {
2384                         .halg.digestsize = SHA256_DIGEST_SIZE,
2385                         .halg.base = {
2386                                 .cra_name = "hmac(sha256)",
2387                                 .cra_driver_name = "hmac-sha256-talitos",
2388                                 .cra_blocksize = SHA256_BLOCK_SIZE,
2389                                 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2390                                              CRYPTO_ALG_ASYNC,
2391                         }
2392                 },
2393                 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2394                                      DESC_HDR_SEL0_MDEUA |
2395                                      DESC_HDR_MODE0_MDEU_SHA256,
2396         },
2397         {       .type = CRYPTO_ALG_TYPE_AHASH,
2398                 .alg.hash = {
2399                         .halg.digestsize = SHA384_DIGEST_SIZE,
2400                         .halg.base = {
2401                                 .cra_name = "hmac(sha384)",
2402                                 .cra_driver_name = "hmac-sha384-talitos",
2403                                 .cra_blocksize = SHA384_BLOCK_SIZE,
2404                                 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2405                                              CRYPTO_ALG_ASYNC,
2406                         }
2407                 },
2408                 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2409                                      DESC_HDR_SEL0_MDEUB |
2410                                      DESC_HDR_MODE0_MDEUB_SHA384,
2411         },
2412         {       .type = CRYPTO_ALG_TYPE_AHASH,
2413                 .alg.hash = {
2414                         .halg.digestsize = SHA512_DIGEST_SIZE,
2415                         .halg.base = {
2416                                 .cra_name = "hmac(sha512)",
2417                                 .cra_driver_name = "hmac-sha512-talitos",
2418                                 .cra_blocksize = SHA512_BLOCK_SIZE,
2419                                 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2420                                              CRYPTO_ALG_ASYNC,
2421                         }
2422                 },
2423                 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2424                                      DESC_HDR_SEL0_MDEUB |
2425                                      DESC_HDR_MODE0_MDEUB_SHA512,
2426         }
2427 };
2428
2429 struct talitos_crypto_alg {
2430         struct list_head entry;
2431         struct device *dev;
2432         struct talitos_alg_template algt;
2433 };
2434
2435 static int talitos_cra_init(struct crypto_tfm *tfm)
2436 {
2437         struct crypto_alg *alg = tfm->__crt_alg;
2438         struct talitos_crypto_alg *talitos_alg;
2439         struct talitos_ctx *ctx = crypto_tfm_ctx(tfm);
2440         struct talitos_private *priv;
2441
2442         if ((alg->cra_flags & CRYPTO_ALG_TYPE_MASK) == CRYPTO_ALG_TYPE_AHASH)
2443                 talitos_alg = container_of(__crypto_ahash_alg(alg),
2444                                            struct talitos_crypto_alg,
2445                                            algt.alg.hash);
2446         else
2447                 talitos_alg = container_of(alg, struct talitos_crypto_alg,
2448                                            algt.alg.crypto);
2449
2450         /* update context with ptr to dev */
2451         ctx->dev = talitos_alg->dev;
2452
2453         /* assign SEC channel to tfm in round-robin fashion */
2454         priv = dev_get_drvdata(ctx->dev);
2455         ctx->ch = atomic_inc_return(&priv->last_chan) &
2456                   (priv->num_channels - 1);
2457
2458         /* copy descriptor header template value */
2459         ctx->desc_hdr_template = talitos_alg->algt.desc_hdr_template;
2460
2461         /* select done notification */
2462         ctx->desc_hdr_template |= DESC_HDR_DONE_NOTIFY;
2463
2464         return 0;
2465 }
2466
2467 static int talitos_cra_init_aead(struct crypto_tfm *tfm)
2468 {
2469         struct talitos_ctx *ctx = crypto_tfm_ctx(tfm);
2470
2471         talitos_cra_init(tfm);
2472
2473         /* random first IV */
2474         get_random_bytes(ctx->iv, TALITOS_MAX_IV_LENGTH);
2475
2476         return 0;
2477 }
2478
2479 static int talitos_cra_init_ahash(struct crypto_tfm *tfm)
2480 {
2481         struct talitos_ctx *ctx = crypto_tfm_ctx(tfm);
2482
2483         talitos_cra_init(tfm);
2484
2485         ctx->keylen = 0;
2486         crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
2487                                  sizeof(struct talitos_ahash_req_ctx));
2488
2489         return 0;
2490 }
2491
2492 /*
2493  * given the alg's descriptor header template, determine whether descriptor
2494  * type and primary/secondary execution units required match the hw
2495  * capabilities description provided in the device tree node.
2496  */
2497 static int hw_supports(struct device *dev, __be32 desc_hdr_template)
2498 {
2499         struct talitos_private *priv = dev_get_drvdata(dev);
2500         int ret;
2501
2502         ret = (1 << DESC_TYPE(desc_hdr_template) & priv->desc_types) &&
2503               (1 << PRIMARY_EU(desc_hdr_template) & priv->exec_units);
2504
2505         if (SECONDARY_EU(desc_hdr_template))
2506                 ret = ret && (1 << SECONDARY_EU(desc_hdr_template)
2507                               & priv->exec_units);
2508
2509         return ret;
2510 }
2511
2512 static int talitos_remove(struct platform_device *ofdev)
2513 {
2514         struct device *dev = &ofdev->dev;
2515         struct talitos_private *priv = dev_get_drvdata(dev);
2516         struct talitos_crypto_alg *t_alg, *n;
2517         int i;
2518
2519         list_for_each_entry_safe(t_alg, n, &priv->alg_list, entry) {
2520                 switch (t_alg->algt.type) {
2521                 case CRYPTO_ALG_TYPE_ABLKCIPHER:
2522                 case CRYPTO_ALG_TYPE_AEAD:
2523                         crypto_unregister_alg(&t_alg->algt.alg.crypto);
2524                         break;
2525                 case CRYPTO_ALG_TYPE_AHASH:
2526                         crypto_unregister_ahash(&t_alg->algt.alg.hash);
2527                         break;
2528                 }
2529                 list_del(&t_alg->entry);
2530                 kfree(t_alg);
2531         }
2532
2533         if (hw_supports(dev, DESC_HDR_SEL0_RNG))
2534                 talitos_unregister_rng(dev);
2535
2536         for (i = 0; i < priv->num_channels; i++)
2537                 kfree(priv->chan[i].fifo);
2538
2539         kfree(priv->chan);
2540
2541         for (i = 0; i < 2; i++)
2542                 if (priv->irq[i]) {
2543                         free_irq(priv->irq[i], dev);
2544                         irq_dispose_mapping(priv->irq[i]);
2545                 }
2546
2547         tasklet_kill(&priv->done_task[0]);
2548         if (priv->irq[1])
2549                 tasklet_kill(&priv->done_task[1]);
2550
2551         iounmap(priv->reg);
2552
2553         kfree(priv);
2554
2555         return 0;
2556 }
2557
2558 static struct talitos_crypto_alg *talitos_alg_alloc(struct device *dev,
2559                                                     struct talitos_alg_template
2560                                                            *template)
2561 {
2562         struct talitos_private *priv = dev_get_drvdata(dev);
2563         struct talitos_crypto_alg *t_alg;
2564         struct crypto_alg *alg;
2565
2566         t_alg = kzalloc(sizeof(struct talitos_crypto_alg), GFP_KERNEL);
2567         if (!t_alg)
2568                 return ERR_PTR(-ENOMEM);
2569
2570         t_alg->algt = *template;
2571
2572         switch (t_alg->algt.type) {
2573         case CRYPTO_ALG_TYPE_ABLKCIPHER:
2574                 alg = &t_alg->algt.alg.crypto;
2575                 alg->cra_init = talitos_cra_init;
2576                 alg->cra_type = &crypto_ablkcipher_type;
2577                 alg->cra_ablkcipher.setkey = ablkcipher_setkey;
2578                 alg->cra_ablkcipher.encrypt = ablkcipher_encrypt;
2579                 alg->cra_ablkcipher.decrypt = ablkcipher_decrypt;
2580                 alg->cra_ablkcipher.geniv = "eseqiv";
2581                 break;
2582         case CRYPTO_ALG_TYPE_AEAD:
2583                 alg = &t_alg->algt.alg.crypto;
2584                 alg->cra_init = talitos_cra_init_aead;
2585                 alg->cra_type = &crypto_aead_type;
2586                 alg->cra_aead.setkey = aead_setkey;
2587                 alg->cra_aead.setauthsize = aead_setauthsize;
2588                 alg->cra_aead.encrypt = aead_encrypt;
2589                 alg->cra_aead.decrypt = aead_decrypt;
2590                 alg->cra_aead.givencrypt = aead_givencrypt;
2591                 alg->cra_aead.geniv = "<built-in>";
2592                 break;
2593         case CRYPTO_ALG_TYPE_AHASH:
2594                 alg = &t_alg->algt.alg.hash.halg.base;
2595                 alg->cra_init = talitos_cra_init_ahash;
2596                 alg->cra_type = &crypto_ahash_type;
2597                 t_alg->algt.alg.hash.init = ahash_init;
2598                 t_alg->algt.alg.hash.update = ahash_update;
2599                 t_alg->algt.alg.hash.final = ahash_final;
2600                 t_alg->algt.alg.hash.finup = ahash_finup;
2601                 t_alg->algt.alg.hash.digest = ahash_digest;
2602                 t_alg->algt.alg.hash.setkey = ahash_setkey;
2603
2604                 if (!(priv->features & TALITOS_FTR_HMAC_OK) &&
2605                     !strncmp(alg->cra_name, "hmac", 4)) {
2606                         kfree(t_alg);
2607                         return ERR_PTR(-ENOTSUPP);
2608                 }
2609                 if (!(priv->features & TALITOS_FTR_SHA224_HWINIT) &&
2610                     (!strcmp(alg->cra_name, "sha224") ||
2611                      !strcmp(alg->cra_name, "hmac(sha224)"))) {
2612                         t_alg->algt.alg.hash.init = ahash_init_sha224_swinit;
2613                         t_alg->algt.desc_hdr_template =
2614                                         DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2615                                         DESC_HDR_SEL0_MDEUA |
2616                                         DESC_HDR_MODE0_MDEU_SHA256;
2617                 }
2618                 break;
2619         default:
2620                 dev_err(dev, "unknown algorithm type %d\n", t_alg->algt.type);
2621                 return ERR_PTR(-EINVAL);
2622         }
2623
2624         alg->cra_module = THIS_MODULE;
2625         alg->cra_priority = TALITOS_CRA_PRIORITY;
2626         alg->cra_alignmask = 0;
2627         alg->cra_ctxsize = sizeof(struct talitos_ctx);
2628         alg->cra_flags |= CRYPTO_ALG_KERN_DRIVER_ONLY;
2629
2630         t_alg->dev = dev;
2631
2632         return t_alg;
2633 }
2634
2635 static int talitos_probe_irq(struct platform_device *ofdev)
2636 {
2637         struct device *dev = &ofdev->dev;
2638         struct device_node *np = ofdev->dev.of_node;
2639         struct talitos_private *priv = dev_get_drvdata(dev);
2640         int err;
2641
2642         priv->irq[0] = irq_of_parse_and_map(np, 0);
2643         if (!priv->irq[0]) {
2644                 dev_err(dev, "failed to map irq\n");
2645                 return -EINVAL;
2646         }
2647
2648         priv->irq[1] = irq_of_parse_and_map(np, 1);
2649
2650         /* get the primary irq line */
2651         if (!priv->irq[1]) {
2652                 err = request_irq(priv->irq[0], talitos_interrupt_4ch, 0,
2653                                   dev_driver_string(dev), dev);
2654                 goto primary_out;
2655         }
2656
2657         err = request_irq(priv->irq[0], talitos_interrupt_ch0_2, 0,
2658                           dev_driver_string(dev), dev);
2659         if (err)
2660                 goto primary_out;
2661
2662         /* get the secondary irq line */
2663         err = request_irq(priv->irq[1], talitos_interrupt_ch1_3, 0,
2664                           dev_driver_string(dev), dev);
2665         if (err) {
2666                 dev_err(dev, "failed to request secondary irq\n");
2667                 irq_dispose_mapping(priv->irq[1]);
2668                 priv->irq[1] = 0;
2669         }
2670
2671         return err;
2672
2673 primary_out:
2674         if (err) {
2675                 dev_err(dev, "failed to request primary irq\n");
2676                 irq_dispose_mapping(priv->irq[0]);
2677                 priv->irq[0] = 0;
2678         }
2679
2680         return err;
2681 }
2682
2683 static int talitos_probe(struct platform_device *ofdev)
2684 {
2685         struct device *dev = &ofdev->dev;
2686         struct device_node *np = ofdev->dev.of_node;
2687         struct talitos_private *priv;
2688         const unsigned int *prop;
2689         int i, err;
2690
2691         priv = kzalloc(sizeof(struct talitos_private), GFP_KERNEL);
2692         if (!priv)
2693                 return -ENOMEM;
2694
2695         INIT_LIST_HEAD(&priv->alg_list);
2696
2697         dev_set_drvdata(dev, priv);
2698
2699         priv->ofdev = ofdev;
2700
2701         spin_lock_init(&priv->reg_lock);
2702
2703         err = talitos_probe_irq(ofdev);
2704         if (err)
2705                 goto err_out;
2706
2707         if (!priv->irq[1]) {
2708                 tasklet_init(&priv->done_task[0], talitos_done_4ch,
2709                              (unsigned long)dev);
2710         } else {
2711                 tasklet_init(&priv->done_task[0], talitos_done_ch0_2,
2712                              (unsigned long)dev);
2713                 tasklet_init(&priv->done_task[1], talitos_done_ch1_3,
2714                              (unsigned long)dev);
2715         }
2716
2717         priv->reg = of_iomap(np, 0);
2718         if (!priv->reg) {
2719                 dev_err(dev, "failed to of_iomap\n");
2720                 err = -ENOMEM;
2721                 goto err_out;
2722         }
2723
2724         /* get SEC version capabilities from device tree */
2725         prop = of_get_property(np, "fsl,num-channels", NULL);
2726         if (prop)
2727                 priv->num_channels = *prop;
2728
2729         prop = of_get_property(np, "fsl,channel-fifo-len", NULL);
2730         if (prop)
2731                 priv->chfifo_len = *prop;
2732
2733         prop = of_get_property(np, "fsl,exec-units-mask", NULL);
2734         if (prop)
2735                 priv->exec_units = *prop;
2736
2737         prop = of_get_property(np, "fsl,descriptor-types-mask", NULL);
2738         if (prop)
2739                 priv->desc_types = *prop;
2740
2741         if (!is_power_of_2(priv->num_channels) || !priv->chfifo_len ||
2742             !priv->exec_units || !priv->desc_types) {
2743                 dev_err(dev, "invalid property data in device tree node\n");
2744                 err = -EINVAL;
2745                 goto err_out;
2746         }
2747
2748         if (of_device_is_compatible(np, "fsl,sec3.0"))
2749                 priv->features |= TALITOS_FTR_SRC_LINK_TBL_LEN_INCLUDES_EXTENT;
2750
2751         if (of_device_is_compatible(np, "fsl,sec2.1"))
2752                 priv->features |= TALITOS_FTR_HW_AUTH_CHECK |
2753                                   TALITOS_FTR_SHA224_HWINIT |
2754                                   TALITOS_FTR_HMAC_OK;
2755
2756         if (of_device_is_compatible(np, "fsl,sec1.0"))
2757                 priv->features |= TALITOS_FTR_SEC1;
2758
2759         priv->chan = kzalloc(sizeof(struct talitos_channel) *
2760                              priv->num_channels, GFP_KERNEL);
2761         if (!priv->chan) {
2762                 dev_err(dev, "failed to allocate channel management space\n");
2763                 err = -ENOMEM;
2764                 goto err_out;
2765         }
2766
2767         priv->fifo_len = roundup_pow_of_two(priv->chfifo_len);
2768
2769         for (i = 0; i < priv->num_channels; i++) {
2770                 priv->chan[i].reg = priv->reg + TALITOS_CH_STRIDE * (i + 1);
2771                 if (!priv->irq[1] || !(i & 1))
2772                         priv->chan[i].reg += TALITOS_CH_BASE_OFFSET;
2773
2774                 spin_lock_init(&priv->chan[i].head_lock);
2775                 spin_lock_init(&priv->chan[i].tail_lock);
2776
2777                 priv->chan[i].fifo = kzalloc(sizeof(struct talitos_request) *
2778                                              priv->fifo_len, GFP_KERNEL);
2779                 if (!priv->chan[i].fifo) {
2780                         dev_err(dev, "failed to allocate request fifo %d\n", i);
2781                         err = -ENOMEM;
2782                         goto err_out;
2783                 }
2784
2785                 atomic_set(&priv->chan[i].submit_count,
2786                            -(priv->chfifo_len - 1));
2787         }
2788
2789         dma_set_mask(dev, DMA_BIT_MASK(36));
2790
2791         /* reset and initialize the h/w */
2792         err = init_device(dev);
2793         if (err) {
2794                 dev_err(dev, "failed to initialize device\n");
2795                 goto err_out;
2796         }
2797
2798         /* register the RNG, if available */
2799         if (hw_supports(dev, DESC_HDR_SEL0_RNG)) {
2800                 err = talitos_register_rng(dev);
2801                 if (err) {
2802                         dev_err(dev, "failed to register hwrng: %d\n", err);
2803                         goto err_out;
2804                 } else
2805                         dev_info(dev, "hwrng\n");
2806         }
2807
2808         /* register crypto algorithms the device supports */
2809         for (i = 0; i < ARRAY_SIZE(driver_algs); i++) {
2810                 if (hw_supports(dev, driver_algs[i].desc_hdr_template)) {
2811                         struct talitos_crypto_alg *t_alg;
2812                         char *name = NULL;
2813
2814                         t_alg = talitos_alg_alloc(dev, &driver_algs[i]);
2815                         if (IS_ERR(t_alg)) {
2816                                 err = PTR_ERR(t_alg);
2817                                 if (err == -ENOTSUPP)
2818                                         continue;
2819                                 goto err_out;
2820                         }
2821
2822                         switch (t_alg->algt.type) {
2823                         case CRYPTO_ALG_TYPE_ABLKCIPHER:
2824                         case CRYPTO_ALG_TYPE_AEAD:
2825                                 err = crypto_register_alg(
2826                                                 &t_alg->algt.alg.crypto);
2827                                 name = t_alg->algt.alg.crypto.cra_driver_name;
2828                                 break;
2829                         case CRYPTO_ALG_TYPE_AHASH:
2830                                 err = crypto_register_ahash(
2831                                                 &t_alg->algt.alg.hash);
2832                                 name =
2833                                  t_alg->algt.alg.hash.halg.base.cra_driver_name;
2834                                 break;
2835                         }
2836                         if (err) {
2837                                 dev_err(dev, "%s alg registration failed\n",
2838                                         name);
2839                                 kfree(t_alg);
2840                         } else
2841                                 list_add_tail(&t_alg->entry, &priv->alg_list);
2842                 }
2843         }
2844         if (!list_empty(&priv->alg_list))
2845                 dev_info(dev, "%s algorithms registered in /proc/crypto\n",
2846                          (char *)of_get_property(np, "compatible", NULL));
2847
2848         return 0;
2849
2850 err_out:
2851         talitos_remove(ofdev);
2852
2853         return err;
2854 }
2855
2856 static const struct of_device_id talitos_match[] = {
2857         {
2858                 .compatible = "fsl,sec2.0",
2859         },
2860         {},
2861 };
2862 MODULE_DEVICE_TABLE(of, talitos_match);
2863
2864 static struct platform_driver talitos_driver = {
2865         .driver = {
2866                 .name = "talitos",
2867                 .of_match_table = talitos_match,
2868         },
2869         .probe = talitos_probe,
2870         .remove = talitos_remove,
2871 };
2872
2873 module_platform_driver(talitos_driver);
2874
2875 MODULE_LICENSE("GPL");
2876 MODULE_AUTHOR("Kim Phillips <kim.phillips@freescale.com>");
2877 MODULE_DESCRIPTION("Freescale integrated security engine (SEC) driver");