#define DEFAULT_TIMEOUT_INTERVAL HZ
-#define FLAGS_FIRST 0x0001
#define FLAGS_FINUP 0x0002
#define FLAGS_FINAL 0x0004
-#define FLAGS_FAST 0x0008
+#define FLAGS_SG 0x0008
#define FLAGS_SHA1 0x0010
#define FLAGS_DMA_ACTIVE 0x0020
#define FLAGS_OUTPUT_READY 0x0040
u32 *hash = (u32 *)ctx->digest;
int i;
+ /* MD5 is almost unused. So copy sha1 size to reduce code */
+ for (i = 0; i < SHA1_DIGEST_SIZE / sizeof(u32); i++) {
+ if (out)
+ hash[i] = omap_sham_read(ctx->dd,
+ SHA_REG_DIGEST(i));
+ else
+ omap_sham_write(ctx->dd,
+ SHA_REG_DIGEST(i), hash[i]);
+ }
+}
+
+static void omap_sham_copy_ready_hash(struct ahash_request *req)
+{
+ struct omap_sham_reqctx *ctx = ahash_request_ctx(req);
+ u32 *in = (u32 *)ctx->digest;
+ u32 *hash = (u32 *)req->result;
+ int i;
+
+ if (!hash)
+ return;
+
if (likely(ctx->flags & FLAGS_SHA1)) {
/* SHA1 results are in big endian */
for (i = 0; i < SHA1_DIGEST_SIZE / sizeof(u32); i++)
- if (out)
- hash[i] = be32_to_cpu(omap_sham_read(ctx->dd,
- SHA_REG_DIGEST(i)));
- else
- omap_sham_write(ctx->dd, SHA_REG_DIGEST(i),
- cpu_to_be32(hash[i]));
+ hash[i] = be32_to_cpu(in[i]);
} else {
/* MD5 results are in little endian */
for (i = 0; i < MD5_DIGEST_SIZE / sizeof(u32); i++)
- if (out)
- hash[i] = le32_to_cpu(omap_sham_read(ctx->dd,
- SHA_REG_DIGEST(i)));
- else
- omap_sham_write(ctx->dd, SHA_REG_DIGEST(i),
- cpu_to_le32(hash[i]));
+ hash[i] = le32_to_cpu(in[i]);
}
}
return -EINVAL;
}
+ ctx->flags &= ~FLAGS_SG;
+
/* next call does not fail... so no unmap in the case of error */
return omap_sham_xmit_dma(dd, ctx->dma_addr, length, final);
}
unsigned int final;
size_t count;
- if (!ctx->total)
- return 0;
-
omap_sham_append_sg(ctx);
final = (ctx->flags & FLAGS_FINUP) && !ctx->total;
return 0;
}
-static int omap_sham_update_dma_fast(struct omap_sham_dev *dd)
+/* Start address alignment */
+#define SG_AA(sg) (IS_ALIGNED(sg->offset, sizeof(u32)))
+/* SHA1 block size alignment */
+#define SG_SA(sg) (IS_ALIGNED(sg->length, SHA1_MD5_BLOCK_SIZE))
+
+static int omap_sham_update_dma_start(struct omap_sham_dev *dd)
{
struct omap_sham_reqctx *ctx = ahash_request_ctx(dd->req);
- unsigned int length;
+ unsigned int length, final, tail;
+ struct scatterlist *sg;
+
+ if (!ctx->total)
+ return 0;
+
+ if (ctx->bufcnt || ctx->offset)
+ return omap_sham_update_dma_slow(dd);
+
+ dev_dbg(dd->dev, "fast: digcnt: %d, bufcnt: %u, total: %u\n",
+ ctx->digcnt, ctx->bufcnt, ctx->total);
- ctx->flags |= FLAGS_FAST;
+ sg = ctx->sg;
- length = min(ctx->total, sg_dma_len(ctx->sg));
- ctx->total = length;
+ if (!SG_AA(sg))
+ return omap_sham_update_dma_slow(dd);
+
+ if (!sg_is_last(sg) && !SG_SA(sg))
+ /* size is not SHA1_BLOCK_SIZE aligned */
+ return omap_sham_update_dma_slow(dd);
+
+ length = min(ctx->total, sg->length);
+
+ if (sg_is_last(sg)) {
+ if (!(ctx->flags & FLAGS_FINUP)) {
+ /* not last sg must be SHA1_MD5_BLOCK_SIZE aligned */
+ tail = length & (SHA1_MD5_BLOCK_SIZE - 1);
+ /* without finup() we need one block to close hash */
+ if (!tail)
+ tail = SHA1_MD5_BLOCK_SIZE;
+ length -= tail;
+ }
+ }
if (!dma_map_sg(dd->dev, ctx->sg, 1, DMA_TO_DEVICE)) {
dev_err(dd->dev, "dma_map_sg error\n");
return -EINVAL;
}
+ ctx->flags |= FLAGS_SG;
+
ctx->total -= length;
+ ctx->offset = length; /* offset where to start slow */
+
+ final = (ctx->flags & FLAGS_FINUP) && !ctx->total;
/* next call does not fail... so no unmap in the case of error */
- return omap_sham_xmit_dma(dd, sg_dma_address(ctx->sg), length, 1);
+ return omap_sham_xmit_dma(dd, sg_dma_address(ctx->sg), length, final);
}
static int omap_sham_update_cpu(struct omap_sham_dev *dd)
struct omap_sham_reqctx *ctx = ahash_request_ctx(dd->req);
omap_stop_dma(dd->dma_lch);
- if (ctx->flags & FLAGS_FAST)
+ if (ctx->flags & FLAGS_SG) {
dma_unmap_sg(dd->dev, ctx->sg, 1, DMA_TO_DEVICE);
- else
+ if (ctx->sg->length == ctx->offset) {
+ ctx->sg = sg_next(ctx->sg);
+ if (ctx->sg)
+ ctx->offset = 0;
+ }
+ } else {
dma_unmap_single(dd->dev, ctx->dma_addr, ctx->buflen,
DMA_TO_DEVICE);
+ }
return 0;
}
spin_unlock_irqrestore(&dd->lock, flags);
if (ctx->digcnt)
- memcpy(req->result, ctx->digest, (ctx->flags & FLAGS_SHA1) ?
- SHA1_DIGEST_SIZE : MD5_DIGEST_SIZE);
+ omap_sham_copy_ready_hash(req);
dev_dbg(dd->dev, "digcnt: %d, bufcnt: %d\n", ctx->digcnt, ctx->bufcnt);
}
ctx->flags = 0;
- ctx->flags |= FLAGS_FIRST;
-
dev_dbg(dd->dev, "init: digest size: %d\n",
crypto_ahash_digestsize(tfm));
if (ctx->flags & FLAGS_CPU)
err = omap_sham_update_cpu(dd);
- else if (ctx->flags & FLAGS_FAST)
- err = omap_sham_update_dma_fast(dd);
else
- err = omap_sham_update_dma_slow(dd);
+ err = omap_sham_update_dma_start(dd);
/* wait for dma completion before can take more data */
dev_dbg(dd->dev, "update: err: %d, digcnt: %d\n", err, ctx->digcnt);
static int omap_sham_handle_queue(struct omap_sham_dev *dd,
struct ahash_request *req)
{
- struct crypto_async_request *async_req, *backlog = 0;
+ struct crypto_async_request *async_req, *backlog;
struct omap_sham_reqctx *ctx;
struct ahash_request *prev_req;
unsigned long flags;
spin_unlock_irqrestore(&dd->lock, flags);
return ret;
}
+ backlog = crypto_get_backlog(&dd->queue);
async_req = crypto_dequeue_request(&dd->queue);
- if (async_req) {
+ if (async_req)
dd->flags |= FLAGS_BUSY;
- backlog = crypto_get_backlog(&dd->queue);
- }
spin_unlock_irqrestore(&dd->lock, flags);
if (!async_req)
*/
omap_sham_append_sg(ctx);
return 0;
- } else if (ctx->bufcnt + ctx->total <= 64) {
+ } else if (ctx->bufcnt + ctx->total <= SHA1_MD5_BLOCK_SIZE) {
+ /*
+ * faster to use CPU for short transfers
+ */
ctx->flags |= FLAGS_CPU;
- } else if (!ctx->bufcnt && sg_is_last(ctx->sg)) {
- /* may be can use faster functions */
- int aligned = IS_ALIGNED((u32)ctx->sg->offset,
- sizeof(u32));
-
- if (aligned && (ctx->flags & FLAGS_FIRST))
- /* digest: first and final */
- ctx->flags |= FLAGS_FAST;
-
- ctx->flags &= ~FLAGS_FIRST;
}
- } else if (ctx->bufcnt + ctx->total <= ctx->buflen) {
- /* if not finaup -> not fast */
+ } else if (ctx->bufcnt + ctx->total < ctx->buflen) {
omap_sham_append_sg(ctx);
return 0;
}
dd->flags &= ~FLAGS_DMA_ACTIVE;
omap_sham_update_dma_stop(dd);
if (!dd->err)
- err = omap_sham_update_dma_slow(dd);
+ err = omap_sham_update_dma_start(dd);
}
err = dd->err ? : err;
projects / firefly-linux-kernel-4.4.55.git / blobdiff