crypto: caam - chaining support

support chained scatterlists for aead, ablkcipher and ahash.

Signed-off-by: Yuan Kang <Yuan.Kang@freescale.com>

- fix dma unmap leak
- un-unlikely src == dst, due to experience with AF_ALG

Signed-off-by: Kudupudi Ugendreshwar <B38865@freescale.com>
Signed-off-by: Kim Phillips <kim.phillips@freescale.com>
Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>

1 files changed, 38 insertions, 15 deletions

diff --git a/drivers/crypto/caam/caamhash.c b/drivers/crypto/caam/caamhash.c
index 7dcf28f4814..895aaf2bca9 100644
--- a/drivers/crypto/caam/caamhash.c
+++ b/drivers/crypto/caam/caamhash.c
@@ -175,9 +175,10 @@ static inline dma_addr_t buf_map_to_sec4_sg(struct device *jrdev,
 /* Map req->src and put it in link table */
 static inline void src_map_to_sec4_sg(struct device *jrdev,
 				      struct scatterlist *src, int src_nents,
-				      struct sec4_sg_entry *sec4_sg)
+				      struct sec4_sg_entry *sec4_sg,
+				      bool chained)
 {
-	dma_map_sg(jrdev, src, src_nents, DMA_TO_DEVICE);
+	dma_map_sg_chained(jrdev, src, src_nents, DMA_TO_DEVICE, chained);
 	sg_to_sec4_sg_last(src, src_nents, sec4_sg, 0);
 }
 
@@ -563,6 +564,7 @@ badkey:
  * ahash_edesc - s/w-extended ahash descriptor
  * @dst_dma: physical mapped address of req->result
  * @sec4_sg_dma: physical mapped address of h/w link table
+ * @chained: if source is chained
  * @src_nents: number of segments in input scatterlist
  * @sec4_sg_bytes: length of dma mapped sec4_sg space
  * @sec4_sg: pointer to h/w link table
@@ -571,6 +573,7 @@ badkey:
 struct ahash_edesc {
 	dma_addr_t dst_dma;
 	dma_addr_t sec4_sg_dma;
+	bool chained;
 	int src_nents;
 	int sec4_sg_bytes;
 	struct sec4_sg_entry *sec4_sg;
@@ -582,7 +585,8 @@ static inline void ahash_unmap(struct device *dev,
 			struct ahash_request *req, int dst_len)
 {
 	if (edesc->src_nents)
-		dma_unmap_sg(dev, req->src, edesc->src_nents, DMA_TO_DEVICE);
+		dma_unmap_sg_chained(dev, req->src, edesc->src_nents,
+				     DMA_TO_DEVICE, edesc->chained);
 	if (edesc->dst_dma)
 		dma_unmap_single(dev, edesc->dst_dma, dst_len, DMA_FROM_DEVICE);
 
@@ -775,6 +779,7 @@ static int ahash_update_ctx(struct ahash_request *req)
 	dma_addr_t ptr = ctx->sh_desc_update_dma;
 	int src_nents, sec4_sg_bytes, sec4_sg_src_index;
 	struct ahash_edesc *edesc;
+	bool chained = false;
 	int ret = 0;
 	int sh_len;
 
@@ -783,7 +788,8 @@ static int ahash_update_ctx(struct ahash_request *req)
 	to_hash = in_len - *next_buflen;
 
 	if (to_hash) {
-		src_nents = __sg_count(req->src, req->nbytes - (*next_buflen));
+		src_nents = __sg_count(req->src, req->nbytes - (*next_buflen),
+				       &chained);
 		sec4_sg_src_index = 1 + (*buflen ? 1 : 0);
 		sec4_sg_bytes = (sec4_sg_src_index + src_nents) *
 				 sizeof(struct sec4_sg_entry);
@@ -801,6 +807,7 @@ static int ahash_update_ctx(struct ahash_request *req)
 		}
 
 		edesc->src_nents = src_nents;
+		edesc->chained = chained;
 		edesc->sec4_sg_bytes = sec4_sg_bytes;
 		edesc->sec4_sg = (void *)edesc + sizeof(struct ahash_edesc) +
 				 DESC_JOB_IO_LEN;
@@ -818,7 +825,8 @@ static int ahash_update_ctx(struct ahash_request *req)
 
 		if (src_nents) {
 			src_map_to_sec4_sg(jrdev, req->src, src_nents,
-					   edesc->sec4_sg + sec4_sg_src_index);
+					   edesc->sec4_sg + sec4_sg_src_index,
+					   chained);
 			if (*next_buflen) {
 				sg_copy_part(next_buf, req->src, to_hash -
 					     *buflen, req->nbytes);
@@ -958,10 +966,11 @@ static int ahash_finup_ctx(struct ahash_request *req)
 	int src_nents;
 	int digestsize = crypto_ahash_digestsize(ahash);
 	struct ahash_edesc *edesc;
+	bool chained = false;
 	int ret = 0;
 	int sh_len;
 
-	src_nents = __sg_count(req->src, req->nbytes);
+	src_nents = __sg_count(req->src, req->nbytes, &chained);
 	sec4_sg_src_index = 1 + (buflen ? 1 : 0);
 	sec4_sg_bytes = (sec4_sg_src_index + src_nents) *
 			 sizeof(struct sec4_sg_entry);
@@ -979,6 +988,7 @@ static int ahash_finup_ctx(struct ahash_request *req)
 	init_job_desc_shared(desc, ptr, sh_len, HDR_SHARE_DEFER | HDR_REVERSE);
 
 	edesc->src_nents = src_nents;
+	edesc->chained = chained;
 	edesc->sec4_sg_bytes = sec4_sg_bytes;
 	edesc->sec4_sg = (void *)edesc + sizeof(struct ahash_edesc) +
 			 DESC_JOB_IO_LEN;
@@ -993,7 +1003,7 @@ static int ahash_finup_ctx(struct ahash_request *req)
 						last_buflen);
 
 	src_map_to_sec4_sg(jrdev, req->src, src_nents, edesc->sec4_sg +
-			   sec4_sg_src_index);
+			   sec4_sg_src_index, chained);
 
 	append_seq_in_ptr(desc, edesc->sec4_sg_dma, ctx->ctx_len +
 			       buflen + req->nbytes, LDST_SGF);
@@ -1030,12 +1040,14 @@ static int ahash_digest(struct ahash_request *req)
 	int src_nents, sec4_sg_bytes;
 	dma_addr_t src_dma;
 	struct ahash_edesc *edesc;
+	bool chained = false;
 	int ret = 0;
 	u32 options;
 	int sh_len;
 
-	src_nents = sg_count(req->src, req->nbytes);
-	dma_map_sg(jrdev, req->src, src_nents ? : 1, DMA_TO_DEVICE);
+	src_nents = sg_count(req->src, req->nbytes, &chained);
+	dma_map_sg_chained(jrdev, req->src, src_nents ? : 1, DMA_TO_DEVICE,
+			   chained);
 	sec4_sg_bytes = src_nents * sizeof(struct sec4_sg_entry);
 
 	/* allocate space for base edesc and hw desc commands, link tables */
@@ -1050,6 +1062,7 @@ static int ahash_digest(struct ahash_request *req)
 	edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg,
 					    sec4_sg_bytes, DMA_TO_DEVICE);
 	edesc->src_nents = src_nents;
+	edesc->chained = chained;
 
 	sh_len = desc_len(sh_desc);
 	desc = edesc->hw_desc;
@@ -1157,6 +1170,7 @@ static int ahash_update_no_ctx(struct ahash_request *req)
 	struct ahash_edesc *edesc;
 	u32 *desc, *sh_desc = ctx->sh_desc_update_first;
 	dma_addr_t ptr = ctx->sh_desc_update_first_dma;
+	bool chained = false;
 	int ret = 0;
 	int sh_len;
 
@@ -1164,7 +1178,8 @@ static int ahash_update_no_ctx(struct ahash_request *req)
 	to_hash = in_len - *next_buflen;
 
 	if (to_hash) {
-		src_nents = __sg_count(req->src, req->nbytes - (*next_buflen));
+		src_nents = __sg_count(req->src, req->nbytes - (*next_buflen),
+				       &chained);
 		sec4_sg_bytes = (1 + src_nents) *
 				sizeof(struct sec4_sg_entry);
 
@@ -1181,6 +1196,7 @@ static int ahash_update_no_ctx(struct ahash_request *req)
 		}
 
 		edesc->src_nents = src_nents;
+		edesc->chained = chained;
 		edesc->sec4_sg_bytes = sec4_sg_bytes;
 		edesc->sec4_sg = (void *)edesc + sizeof(struct ahash_edesc) +
 				 DESC_JOB_IO_LEN;
@@ -1191,7 +1207,7 @@ static int ahash_update_no_ctx(struct ahash_request *req)
 		state->buf_dma = buf_map_to_sec4_sg(jrdev, edesc->sec4_sg,
 						    buf, *buflen);
 		src_map_to_sec4_sg(jrdev, req->src, src_nents,
-				   edesc->sec4_sg + 1);
+				   edesc->sec4_sg + 1, chained);
 		if (*next_buflen) {
 			sg_copy_part(next_buf, req->src, to_hash - *buflen,
 				    req->nbytes);
@@ -1258,10 +1274,11 @@ static int ahash_finup_no_ctx(struct ahash_request *req)
 	int sec4_sg_bytes, sec4_sg_src_index, src_nents;
 	int digestsize = crypto_ahash_digestsize(ahash);
 	struct ahash_edesc *edesc;
+	bool chained = false;
 	int sh_len;
 	int ret = 0;
 
-	src_nents = __sg_count(req->src, req->nbytes);
+	src_nents = __sg_count(req->src, req->nbytes, &chained);
 	sec4_sg_src_index = 2;
 	sec4_sg_bytes = (sec4_sg_src_index + src_nents) *
 			 sizeof(struct sec4_sg_entry);
@@ -1279,6 +1296,7 @@ static int ahash_finup_no_ctx(struct ahash_request *req)
 	init_job_desc_shared(desc, ptr, sh_len, HDR_SHARE_DEFER | HDR_REVERSE);
 
 	edesc->src_nents = src_nents;
+	edesc->chained = chained;
 	edesc->sec4_sg_bytes = sec4_sg_bytes;
 	edesc->sec4_sg = (void *)edesc + sizeof(struct ahash_edesc) +
 			 DESC_JOB_IO_LEN;
@@ -1289,7 +1307,8 @@ static int ahash_finup_no_ctx(struct ahash_request *req)
 						state->buf_dma, buflen,
 						last_buflen);
 
-	src_map_to_sec4_sg(jrdev, req->src, src_nents, edesc->sec4_sg + 1);
+	src_map_to_sec4_sg(jrdev, req->src, src_nents, edesc->sec4_sg + 1,
+			   chained);
 
 	append_seq_in_ptr(desc, edesc->sec4_sg_dma, buflen +
 			       req->nbytes, LDST_SGF);
@@ -1332,6 +1351,7 @@ static int ahash_update_first(struct ahash_request *req)
 	dma_addr_t src_dma;
 	u32 options;
 	struct ahash_edesc *edesc;
+	bool chained = false;
 	int ret = 0;
 	int sh_len;
 
@@ -1340,8 +1360,10 @@ static int ahash_update_first(struct ahash_request *req)
 	to_hash = req->nbytes - *next_buflen;
 
 	if (to_hash) {
-		src_nents = sg_count(req->src, req->nbytes - (*next_buflen));
-		dma_map_sg(jrdev, req->src, src_nents ? : 1, DMA_TO_DEVICE);
+		src_nents = sg_count(req->src, req->nbytes - (*next_buflen),
+				     &chained);
+		dma_map_sg_chained(jrdev, req->src, src_nents ? : 1,
+				   DMA_TO_DEVICE, chained);
 		sec4_sg_bytes = src_nents * sizeof(struct sec4_sg_entry);
 
 		/*
@@ -1357,6 +1379,7 @@ static int ahash_update_first(struct ahash_request *req)
 		}
 
 		edesc->src_nents = src_nents;
+		edesc->chained = chained;
 		edesc->sec4_sg_bytes = sec4_sg_bytes;
 		edesc->sec4_sg = (void *)edesc + sizeof(struct ahash_edesc) +
 				 DESC_JOB_IO_LEN;