From a44b56cc69a5b017f396ed1daeaef6a63e086656 Mon Sep 17 00:00:00 2001 From: Evgeniy Polyakov Date: Sun, 12 Oct 2008 20:14:15 +0800 Subject: crypto: hifn - Disable driver when physical size exceeds 32 bits On Tue, Sep 23, 2008 at 08:06:32PM +0200, Dimitri Puzin (max@psycast.de) wrote: > With this patch applied it still doesn't work as expected. The overflow > messages are gone however syslog shows > [ 120.924266] hifn0: abort: c: 0, s: 1, d: 0, r: 0. > when doing cryptsetup luksFormat as in original e-mail. At this point > cryptsetup hangs and can't be killed with -SIGKILL. I've attached > SysRq-t dump of this condition. Yes, I was wrong with the patch: HIFN does not support 64-bit addresses afaics. Attached patch should not allow HIFN to be registered on 64-bit arch, so crypto layer will fallback to the software algorithms. Signed-off-by: Evgeniy Polyakov Signed-off-by: Herbert Xu --- drivers/crypto/hifn_795x.c | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'drivers') diff --git a/drivers/crypto/hifn_795x.c b/drivers/crypto/hifn_795x.c index 4d22b21bd3e..2969ea99f7c 100644 --- a/drivers/crypto/hifn_795x.c +++ b/drivers/crypto/hifn_795x.c @@ -2782,6 +2782,11 @@ static int __devinit hifn_init(void) unsigned int freq; int err; + if (sizeof(dma_addr_t) > 4) { + printk(KERN_INFO "HIFN supports only 32-bit addresses.\n"); + return -EINVAL; + } + if (strncmp(hifn_pll_ref, "ext", 3) && strncmp(hifn_pll_ref, "pci", 3)) { printk(KERN_ERR "hifn795x: invalid hifn_pll_ref clock, " -- cgit v1.2.3 From 40405f10b853ef404b395c6febca71b0de185490 Mon Sep 17 00:00:00 2001 From: Kim Phillips Date: Sun, 12 Oct 2008 20:19:35 +0800 Subject: crypto: talitos - Pass correct interrupt status to error handler Since we ack early, the re-read interrupt status in talitos_error may be already updated with a new value. Pass the error ISR value directly in order to report and handle the error based on the correct error status. Also remove unused error tasklet. Signed-off-by: Kim Phillips Signed-off-by: Lee Nipper Signed-off-by: Herbert Xu --- drivers/crypto/talitos.c | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) (limited to 'drivers') diff --git a/drivers/crypto/talitos.c b/drivers/crypto/talitos.c index 24607669a52..c429f684c79 100644 --- a/drivers/crypto/talitos.c +++ b/drivers/crypto/talitos.c @@ -127,7 +127,6 @@ struct talitos_private { /* request callback tasklet */ struct tasklet_struct done_task; - struct tasklet_struct error_task; /* list of registered algorithms */ struct list_head alg_list; @@ -469,16 +468,13 @@ static void report_eu_error(struct device *dev, int ch, struct talitos_desc *des /* * recover from error interrupts */ -static void talitos_error(unsigned long data) +static void talitos_error(unsigned long data, u32 isr, u32 isr_lo) { struct device *dev = (struct device *)data; struct talitos_private *priv = dev_get_drvdata(dev); unsigned int timeout = TALITOS_TIMEOUT; int ch, error, reset_dev = 0, reset_ch = 0; - u32 isr, isr_lo, v, v_lo; - - isr = in_be32(priv->reg + TALITOS_ISR); - isr_lo = in_be32(priv->reg + TALITOS_ISR_LO); + u32 v, v_lo; for (ch = 0; ch < priv->num_channels; ch++) { /* skip channels without errors */ @@ -566,7 +562,7 @@ static irqreturn_t talitos_interrupt(int irq, void *data) out_be32(priv->reg + TALITOS_ICR_LO, isr_lo); if (unlikely((isr & ~TALITOS_ISR_CHDONE) || isr_lo)) - talitos_error((unsigned long)data); + talitos_error((unsigned long)data, isr, isr_lo); else if (likely(isr & TALITOS_ISR_CHDONE)) tasklet_schedule(&priv->done_task); @@ -1391,7 +1387,6 @@ static int talitos_remove(struct of_device *ofdev) } tasklet_kill(&priv->done_task); - tasklet_kill(&priv->error_task); iounmap(priv->reg); @@ -1454,7 +1449,6 @@ static int talitos_probe(struct of_device *ofdev, INIT_LIST_HEAD(&priv->alg_list); tasklet_init(&priv->done_task, talitos_done, (unsigned long)dev); - tasklet_init(&priv->error_task, talitos_error, (unsigned long)dev); priv->irq = irq_of_parse_and_map(np, 0); -- cgit v1.2.3 From 1c2e8811eea5f0c5da3213ea206c2864fa4614fd Mon Sep 17 00:00:00 2001 From: Lee Nipper Date: Sun, 12 Oct 2008 20:29:34 +0800 Subject: crypto: talitos - Implement done interrupt mitigation In talitos_interrupt, upon one done interrupt, mask further done interrupts, and ack only any error interrupt. In talitos_done, unmask done interrupts after completing processing. In flush_channel, ack each done channel processed. Keep done overflow interrupts masked because even though each pkt is ack'ed, a few done overflows still occur. Signed-off-by: Lee Nipper Signed-off-by: Kim Phillips Signed-off-by: Herbert Xu --- drivers/crypto/talitos.c | 30 ++++++++++++++++++++++-------- drivers/crypto/talitos.h | 3 ++- 2 files changed, 24 insertions(+), 9 deletions(-) (limited to 'drivers') diff --git a/drivers/crypto/talitos.c b/drivers/crypto/talitos.c index c429f684c79..b5c2c9340a9 100644 --- a/drivers/crypto/talitos.c +++ b/drivers/crypto/talitos.c @@ -319,9 +319,11 @@ static void flush_channel(struct device *dev, int ch, int error, int reset_ch) /* descriptors with their done bits set don't get the error */ rmb(); - if ((request->desc->hdr & DESC_HDR_DONE) == DESC_HDR_DONE) + if ((request->desc->hdr & DESC_HDR_DONE) == DESC_HDR_DONE) { status = 0; - else + /* Ack each pkt completed on channel */ + out_be32(priv->reg + TALITOS_ICR, (1 << (ch * 2))); + } else if (!error) break; else @@ -369,6 +371,11 @@ static void talitos_done(unsigned long data) for (ch = 0; ch < priv->num_channels; ch++) flush_channel(dev, ch, 0, 0); + + /* At this point, all completed channels have been processed. + * Unmask done interrupts for channels completed later on. + */ + setbits32(priv->reg + TALITOS_IMR, TALITOS_IMR_DONE); } /* @@ -557,15 +564,22 @@ static irqreturn_t talitos_interrupt(int irq, void *data) isr = in_be32(priv->reg + TALITOS_ISR); isr_lo = in_be32(priv->reg + TALITOS_ISR_LO); - /* ack */ - out_be32(priv->reg + TALITOS_ICR, isr); - out_be32(priv->reg + TALITOS_ICR_LO, isr_lo); + if (unlikely((isr & ~TALITOS_ISR_CHDONE) || isr_lo)) { + /* + * Acknowledge error interrupts here. + * Done interrupts are ack'ed as part of done_task. + */ + out_be32(priv->reg + TALITOS_ICR, isr); + out_be32(priv->reg + TALITOS_ICR_LO, isr_lo); - if (unlikely((isr & ~TALITOS_ISR_CHDONE) || isr_lo)) talitos_error((unsigned long)data, isr, isr_lo); - else - if (likely(isr & TALITOS_ISR_CHDONE)) + } else + if (likely(isr & TALITOS_ISR_CHDONE)) { + /* mask further done interrupts. */ + clrbits32(priv->reg + TALITOS_IMR, TALITOS_IMR_DONE); + /* done_task will unmask done interrupts at exit */ tasklet_schedule(&priv->done_task); + } return (isr || isr_lo) ? IRQ_HANDLED : IRQ_NONE; } diff --git a/drivers/crypto/talitos.h b/drivers/crypto/talitos.h index c48a405abf7..e6b87770df0 100644 --- a/drivers/crypto/talitos.h +++ b/drivers/crypto/talitos.h @@ -37,7 +37,8 @@ #define TALITOS_MCR_LO 0x1038 #define TALITOS_MCR_SWR 0x1 /* s/w reset */ #define TALITOS_IMR 0x1008 /* interrupt mask register */ -#define TALITOS_IMR_INIT 0x10fff /* enable channel IRQs */ +#define TALITOS_IMR_INIT 0x100ff /* enable channel IRQs */ +#define TALITOS_IMR_DONE 0x00055 /* done IRQs */ #define TALITOS_IMR_LO 0x100C #define TALITOS_IMR_LO_INIT 0x20000 /* allow RNGU error IRQs */ #define TALITOS_ISR 0x1010 /* interrupt status register */ -- cgit v1.2.3 From fe5720e2b7c1e8ff95d4bf18329517cf64ad1d70 Mon Sep 17 00:00:00 2001 From: Kim Phillips Date: Sun, 12 Oct 2008 20:33:14 +0800 Subject: crypto: talitos - Perform auth check in h/w if on sec 2.1 and above SEC version 2.1 and above adds the capability to do the IPSec ICV memcmp in h/w. Results of the cmp are written back in the descriptor header, along with the done status. A new callback is added that checks these ICCR bits instead of performing the memcmp on the core, and is enabled by h/w capability. Signed-off-by: Kim Phillips After testing on different parts, another condition was added before using h/w auth check because different SEC revisions require different handling. The SEC 3.0 allows a more flexible link table where the auth data can span separate link table entries. The SEC 2.4/2.1 does not support this case. So a test was added in the decrypt routine for a fragmented case; the h/w auth check is disallowed for revisions not having the extent in the link table; in this case the hw auth check is done by software. A portion of a previous change for SEC 3.0 link table handling was removed since it became dead code with the hw auth check supported. This seems to be the best compromise for using hw auth check on supporting SEC revisions; it keeps the link table logic simpler for the fragmented cases. Signed-off-by: Lee Nipper Signed-off-by: Herbert Xu --- drivers/crypto/talitos.c | 127 +++++++++++++++++++++++++++++++---------------- drivers/crypto/talitos.h | 8 +++ 2 files changed, 91 insertions(+), 44 deletions(-) (limited to 'drivers') diff --git a/drivers/crypto/talitos.c b/drivers/crypto/talitos.c index b5c2c9340a9..16c97caa17a 100644 --- a/drivers/crypto/talitos.c +++ b/drivers/crypto/talitos.c @@ -137,6 +137,7 @@ struct talitos_private { /* .features flag */ #define TALITOS_FTR_SRC_LINK_TBL_LEN_INCLUDES_EXTENT 0x00000001 +#define TALITOS_FTR_HW_AUTH_CHECK 0x00000002 /* * map virtual single (contiguous) pointer to h/w descriptor pointer @@ -183,6 +184,11 @@ static int reset_channel(struct device *dev, int ch) setbits32(priv->reg + TALITOS_CCCR_LO(ch), TALITOS_CCCR_LO_CDWE | TALITOS_CCCR_LO_CDIE); + /* and ICCR writeback, if available */ + if (priv->features & TALITOS_FTR_HW_AUTH_CHECK) + setbits32(priv->reg + TALITOS_CCCR_LO(ch), + TALITOS_CCCR_LO_IWSE); + return 0; } @@ -238,6 +244,11 @@ static int init_device(struct device *dev) setbits32(priv->reg + TALITOS_IMR, TALITOS_IMR_INIT); setbits32(priv->reg + TALITOS_IMR_LO, TALITOS_IMR_LO_INIT); + /* disable integrity check error interrupts (use writeback instead) */ + if (priv->features & TALITOS_FTR_HW_AUTH_CHECK) + setbits32(priv->reg + TALITOS_MDEUICR_LO, + TALITOS_MDEUICR_LO_ICE); + return 0; } @@ -375,7 +386,8 @@ static void talitos_done(unsigned long data) /* At this point, all completed channels have been processed. * Unmask done interrupts for channels completed later on. */ - setbits32(priv->reg + TALITOS_IMR, TALITOS_IMR_DONE); + setbits32(priv->reg + TALITOS_IMR, TALITOS_IMR_INIT); + setbits32(priv->reg + TALITOS_IMR_LO, TALITOS_IMR_LO_INIT); } /* @@ -812,7 +824,7 @@ static void ipsec_esp_encrypt_done(struct device *dev, aead_request_complete(areq, err); } -static void ipsec_esp_decrypt_done(struct device *dev, +static void ipsec_esp_decrypt_swauth_done(struct device *dev, struct talitos_desc *desc, void *context, int err) { @@ -844,6 +856,27 @@ static void ipsec_esp_decrypt_done(struct device *dev, aead_request_complete(req, err); } +static void ipsec_esp_decrypt_hwauth_done(struct device *dev, + struct talitos_desc *desc, void *context, + int err) +{ + struct aead_request *req = context; + struct ipsec_esp_edesc *edesc = + container_of(desc, struct ipsec_esp_edesc, desc); + + ipsec_esp_unmap(dev, edesc, req); + + /* check ICV auth status */ + if (!err) + if ((desc->hdr_lo & DESC_HDR_LO_ICCR1_MASK) != + DESC_HDR_LO_ICCR1_PASS) + err = -EBADMSG; + + kfree(edesc); + + aead_request_complete(req, err); +} + /* * convert scatterlist to SEC h/w link table format * stop at cryptlen bytes @@ -897,6 +930,7 @@ static int ipsec_esp(struct ipsec_esp_edesc *edesc, struct aead_request *areq, unsigned int authsize = ctx->authsize; unsigned int ivsize; int sg_count, ret; + int sg_link_tbl_len; /* hmac key */ map_single_talitos_ptr(dev, &desc->ptr[0], ctx->authkeylen, &ctx->key, @@ -934,33 +968,19 @@ static int ipsec_esp(struct ipsec_esp_edesc *edesc, struct aead_request *areq, if (sg_count == 1) { desc->ptr[4].ptr = cpu_to_be32(sg_dma_address(areq->src)); } else { - sg_count = sg_to_link_tbl(areq->src, sg_count, cryptlen, + sg_link_tbl_len = cryptlen; + + if ((edesc->desc.hdr & DESC_HDR_MODE1_MDEU_CICV) && + (edesc->desc.hdr & DESC_HDR_MODE0_ENCRYPT) == 0) { + sg_link_tbl_len = cryptlen + authsize; + } + sg_count = sg_to_link_tbl(areq->src, sg_count, sg_link_tbl_len, &edesc->link_tbl[0]); if (sg_count > 1) { - struct talitos_ptr *link_tbl_ptr = - &edesc->link_tbl[sg_count-1]; - struct scatterlist *sg; - struct talitos_private *priv = dev_get_drvdata(dev); - desc->ptr[4].j_extent |= DESC_PTR_LNKTBL_JUMP; desc->ptr[4].ptr = cpu_to_be32(edesc->dma_link_tbl); dma_sync_single_for_device(ctx->dev, edesc->dma_link_tbl, edesc->dma_len, DMA_BIDIRECTIONAL); - /* If necessary for this SEC revision, - * add a link table entry for ICV. - */ - if ((priv->features & - TALITOS_FTR_SRC_LINK_TBL_LEN_INCLUDES_EXTENT) && - (edesc->desc.hdr & DESC_HDR_MODE0_ENCRYPT) == 0) { - link_tbl_ptr->j_extent = 0; - link_tbl_ptr++; - link_tbl_ptr->j_extent = DESC_PTR_LNKTBL_RETURN; - link_tbl_ptr->len = cpu_to_be16(authsize); - sg = sg_last(areq->src, edesc->src_nents ? : 1); - link_tbl_ptr->ptr = cpu_to_be32( - (char *)sg_dma_address(sg) - + sg->length - authsize); - } } else { /* Only one segment now, so no link tbl needed */ desc->ptr[4].ptr = cpu_to_be32(sg_dma_address(areq->src)); @@ -985,13 +1005,9 @@ static int ipsec_esp(struct ipsec_esp_edesc *edesc, struct aead_request *areq, desc->ptr[5].ptr = cpu_to_be32((struct talitos_ptr *) edesc->dma_link_tbl + edesc->src_nents + 1); - if (areq->src == areq->dst) { - memcpy(link_tbl_ptr, &edesc->link_tbl[0], - edesc->src_nents * sizeof(struct talitos_ptr)); - } else { - sg_count = sg_to_link_tbl(areq->dst, sg_count, cryptlen, - link_tbl_ptr); - } + sg_count = sg_to_link_tbl(areq->dst, sg_count, cryptlen, + link_tbl_ptr); + /* Add an entry to the link table for ICV data */ link_tbl_ptr += sg_count - 1; link_tbl_ptr->j_extent = 0; @@ -1116,11 +1132,14 @@ static int aead_authenc_encrypt(struct aead_request *req) return ipsec_esp(edesc, req, NULL, 0, ipsec_esp_encrypt_done); } + + static int aead_authenc_decrypt(struct aead_request *req) { struct crypto_aead *authenc = crypto_aead_reqtfm(req); struct talitos_ctx *ctx = crypto_aead_ctx(authenc); unsigned int authsize = ctx->authsize; + struct talitos_private *priv = dev_get_drvdata(ctx->dev); struct ipsec_esp_edesc *edesc; struct scatterlist *sg; void *icvdata; @@ -1132,22 +1151,39 @@ static int aead_authenc_decrypt(struct aead_request *req) if (IS_ERR(edesc)) return PTR_ERR(edesc); - /* stash incoming ICV for later cmp with ICV generated by the h/w */ - if (edesc->dma_len) - icvdata = &edesc->link_tbl[edesc->src_nents + - edesc->dst_nents + 2]; - else - icvdata = &edesc->link_tbl[0]; + if ((priv->features & TALITOS_FTR_HW_AUTH_CHECK) && + (((!edesc->src_nents && !edesc->dst_nents) || + priv->features & TALITOS_FTR_SRC_LINK_TBL_LEN_INCLUDES_EXTENT))) { - sg = sg_last(req->src, edesc->src_nents ? : 1); + /* decrypt and check the ICV */ + edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_DIR_INBOUND | + DESC_HDR_MODE1_MDEU_CICV; - memcpy(icvdata, (char *)sg_virt(sg) + sg->length - ctx->authsize, - ctx->authsize); + /* reset integrity check result bits */ + edesc->desc.hdr_lo = 0; - /* decrypt */ - edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_DIR_INBOUND; + return ipsec_esp(edesc, req, NULL, 0, ipsec_esp_decrypt_hwauth_done); - return ipsec_esp(edesc, req, NULL, 0, ipsec_esp_decrypt_done); + } else { + + /* Have to check the ICV with software */ + + edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_DIR_INBOUND; + + /* stash incoming ICV for later cmp with ICV generated by the h/w */ + if (edesc->dma_len) + icvdata = &edesc->link_tbl[edesc->src_nents + + edesc->dst_nents + 2]; + else + icvdata = &edesc->link_tbl[0]; + + sg = sg_last(req->src, edesc->src_nents ? : 1); + + memcpy(icvdata, (char *)sg_virt(sg) + sg->length - ctx->authsize, + ctx->authsize); + + return ipsec_esp(edesc, req, NULL, 0, ipsec_esp_decrypt_swauth_done); + } } static int aead_authenc_givencrypt( @@ -1460,10 +1496,10 @@ static int talitos_probe(struct of_device *ofdev, priv->ofdev = ofdev; - INIT_LIST_HEAD(&priv->alg_list); - tasklet_init(&priv->done_task, talitos_done, (unsigned long)dev); + INIT_LIST_HEAD(&priv->alg_list); + priv->irq = irq_of_parse_and_map(np, 0); if (priv->irq == NO_IRQ) { @@ -1516,6 +1552,9 @@ static int talitos_probe(struct of_device *ofdev, if (of_device_is_compatible(np, "fsl,sec3.0")) priv->features |= TALITOS_FTR_SRC_LINK_TBL_LEN_INCLUDES_EXTENT; + if (of_device_is_compatible(np, "fsl,sec2.1")) + priv->features |= TALITOS_FTR_HW_AUTH_CHECK; + priv->head_lock = kmalloc(sizeof(spinlock_t) * priv->num_channels, GFP_KERNEL); priv->tail_lock = kmalloc(sizeof(spinlock_t) * priv->num_channels, diff --git a/drivers/crypto/talitos.h b/drivers/crypto/talitos.h index e6b87770df0..125560a1d5f 100644 --- a/drivers/crypto/talitos.h +++ b/drivers/crypto/talitos.h @@ -56,6 +56,7 @@ #define TALITOS_CCCR_CONT 0x2 /* channel continue */ #define TALITOS_CCCR_RESET 0x1 /* channel reset */ #define TALITOS_CCCR_LO(ch) (ch * TALITOS_CH_STRIDE + 0x110c) +#define TALITOS_CCCR_LO_IWSE 0x80 /* chan. ICCR writeback enab. */ #define TALITOS_CCCR_LO_CDWE 0x10 /* chan. done writeback enab. */ #define TALITOS_CCCR_LO_NT 0x4 /* notification type */ #define TALITOS_CCCR_LO_CDIE 0x2 /* channel done IRQ enable */ @@ -103,6 +104,9 @@ #define TALITOS_AESUISR_LO 0x4034 #define TALITOS_MDEUISR 0x6030 /* message digest unit */ #define TALITOS_MDEUISR_LO 0x6034 +#define TALITOS_MDEUICR 0x6038 /* interrupt control */ +#define TALITOS_MDEUICR_LO 0x603c +#define TALITOS_MDEUICR_LO_ICE 0x4000 /* integrity check IRQ enable */ #define TALITOS_AFEUISR 0x8030 /* arc4 unit */ #define TALITOS_AFEUISR_LO 0x8034 #define TALITOS_RNGUISR 0xa030 /* random number unit */ @@ -131,6 +135,9 @@ /* written back when done */ #define DESC_HDR_DONE __constant_cpu_to_be32(0xff000000) +#define DESC_HDR_LO_ICCR1_MASK __constant_cpu_to_be32(0x00180000) +#define DESC_HDR_LO_ICCR1_PASS __constant_cpu_to_be32(0x00080000) +#define DESC_HDR_LO_ICCR1_FAIL __constant_cpu_to_be32(0x00100000) /* primary execution unit select */ #define DESC_HDR_SEL0_MASK __constant_cpu_to_be32(0xf0000000) @@ -169,6 +176,7 @@ #define DESC_HDR_SEL1_CRCU __constant_cpu_to_be32(0x00080000) /* secondary execution unit mode (MODE1) and derivatives */ +#define DESC_HDR_MODE1_MDEU_CICV __constant_cpu_to_be32(0x00004000) #define DESC_HDR_MODE1_MDEU_INIT __constant_cpu_to_be32(0x00001000) #define DESC_HDR_MODE1_MDEU_HMAC __constant_cpu_to_be32(0x00000800) #define DESC_HDR_MODE1_MDEU_PAD __constant_cpu_to_be32(0x00000400) -- cgit v1.2.3 From 4b24ea971a93f5d0bec34bf7bfd0939f70cfaae6 Mon Sep 17 00:00:00 2001 From: Vishnu Suresh Date: Mon, 20 Oct 2008 21:06:18 +0800 Subject: crypto: talitos - Preempt overflow interrupts off-by-one fix In commit ec6644d6325b5a38525f1d5b20fd4bf7db05cf2a "crypto: talitos - Preempt overflow interrupts", the test in atomic_inc_not_zero was interpreted by the author to be applied after the increment operation (not before). This off-by-one fix prevents overflow error interrupts from occurring when requests are frequent and large enough to do so. Signed-off-by: Vishnu Suresh Signed-off-by: Kim Phillips Signed-off-by: Herbert Xu --- drivers/crypto/talitos.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'drivers') diff --git a/drivers/crypto/talitos.c b/drivers/crypto/talitos.c index 16c97caa17a..c3df3b1df85 100644 --- a/drivers/crypto/talitos.c +++ b/drivers/crypto/talitos.c @@ -1598,7 +1598,7 @@ static int talitos_probe(struct of_device *ofdev, goto err_out; } for (i = 0; i < priv->num_channels; i++) - atomic_set(&priv->submit_count[i], -priv->chfifo_len); + atomic_set(&priv->submit_count[i], -(priv->chfifo_len - 1)); priv->head = kzalloc(sizeof(int) * priv->num_channels, GFP_KERNEL); priv->tail = kzalloc(sizeof(int) * priv->num_channels, GFP_KERNEL); -- cgit v1.2.3 From 420a4b20c504e4674bf253601f793bdb254f99fa Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Sun, 31 Aug 2008 15:58:45 +1000 Subject: crypto: padlock - Avoid resetting cword on successive operations Resetting the control word is quite expensive. Fortunately this isn't an issue for the common operations such as CBC and ECB as the whole operation is done through a single call. However, modes such as LRW and XTS have to call padlock over and over again for one operation which really hurts if each call resets the control word. This patch uses an idea by Sebastian Siewior to store the last control word used on a CPU and only reset the control word if that changes. Note that any task switch automatically resets the control word so we only need to be accurate with regard to the stored control word when no task switches occur. Signed-off-by: Herbert Xu --- drivers/crypto/padlock-aes.c | 52 +++++++++++++++++++++++++++++++++++--------- 1 file changed, 42 insertions(+), 10 deletions(-) (limited to 'drivers') diff --git a/drivers/crypto/padlock-aes.c b/drivers/crypto/padlock-aes.c index bf2917d197a..856b3cc2558 100644 --- a/drivers/crypto/padlock-aes.c +++ b/drivers/crypto/padlock-aes.c @@ -15,6 +15,8 @@ #include #include #include +#include +#include #include #include #include "padlock.h" @@ -49,6 +51,8 @@ struct aes_ctx { u32 *D; }; +static DEFINE_PER_CPU(struct cword *, last_cword); + /* Tells whether the ACE is capable to generate the extended key for a given key_len. */ static inline int @@ -89,6 +93,7 @@ static int aes_set_key(struct crypto_tfm *tfm, const u8 *in_key, const __le32 *key = (const __le32 *)in_key; u32 *flags = &tfm->crt_flags; struct crypto_aes_ctx gen_aes; + int cpu; if (key_len % 8) { *flags |= CRYPTO_TFM_RES_BAD_KEY_LEN; @@ -118,7 +123,7 @@ static int aes_set_key(struct crypto_tfm *tfm, const u8 *in_key, /* Don't generate extended keys if the hardware can do it. */ if (aes_hw_extkey_available(key_len)) - return 0; + goto ok; ctx->D = ctx->d_data; ctx->cword.encrypt.keygen = 1; @@ -131,15 +136,30 @@ static int aes_set_key(struct crypto_tfm *tfm, const u8 *in_key, memcpy(ctx->E, gen_aes.key_enc, AES_MAX_KEYLENGTH); memcpy(ctx->D, gen_aes.key_dec, AES_MAX_KEYLENGTH); + +ok: + for_each_online_cpu(cpu) + if (&ctx->cword.encrypt == per_cpu(last_cword, cpu) || + &ctx->cword.decrypt == per_cpu(last_cword, cpu)) + per_cpu(last_cword, cpu) = NULL; + return 0; } /* ====== Encryption/decryption routines ====== */ /* These are the real call to PadLock. */ -static inline void padlock_reset_key(void) +static inline void padlock_reset_key(struct cword *cword) +{ + int cpu = raw_smp_processor_id(); + + if (cword != per_cpu(last_cword, cpu)) + asm volatile ("pushfl; popfl"); +} + +static inline void padlock_store_cword(struct cword *cword) { - asm volatile ("pushfl; popfl"); + per_cpu(last_cword, raw_smp_processor_id()) = cword; } /* @@ -149,7 +169,7 @@ static inline void padlock_reset_key(void) */ static inline void padlock_xcrypt(const u8 *input, u8 *output, void *key, - void *control_word) + struct cword *control_word) { asm volatile (".byte 0xf3,0x0f,0xa7,0xc8" /* rep xcryptecb */ : "+S"(input), "+D"(output) @@ -213,22 +233,24 @@ static void aes_encrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in) { struct aes_ctx *ctx = aes_ctx(tfm); int ts_state; - padlock_reset_key(); + padlock_reset_key(&ctx->cword.encrypt); ts_state = irq_ts_save(); aes_crypt(in, out, ctx->E, &ctx->cword.encrypt); irq_ts_restore(ts_state); + padlock_store_cword(&ctx->cword.encrypt); } static void aes_decrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in) { struct aes_ctx *ctx = aes_ctx(tfm); int ts_state; - padlock_reset_key(); + padlock_reset_key(&ctx->cword.encrypt); ts_state = irq_ts_save(); aes_crypt(in, out, ctx->D, &ctx->cword.decrypt); irq_ts_restore(ts_state); + padlock_store_cword(&ctx->cword.encrypt); } static struct crypto_alg aes_alg = { @@ -261,7 +283,7 @@ static int ecb_aes_encrypt(struct blkcipher_desc *desc, int err; int ts_state; - padlock_reset_key(); + padlock_reset_key(&ctx->cword.encrypt); blkcipher_walk_init(&walk, dst, src, nbytes); err = blkcipher_walk_virt(desc, &walk); @@ -276,6 +298,8 @@ static int ecb_aes_encrypt(struct blkcipher_desc *desc, } irq_ts_restore(ts_state); + padlock_store_cword(&ctx->cword.encrypt); + return err; } @@ -288,7 +312,7 @@ static int ecb_aes_decrypt(struct blkcipher_desc *desc, int err; int ts_state; - padlock_reset_key(); + padlock_reset_key(&ctx->cword.decrypt); blkcipher_walk_init(&walk, dst, src, nbytes); err = blkcipher_walk_virt(desc, &walk); @@ -302,6 +326,9 @@ static int ecb_aes_decrypt(struct blkcipher_desc *desc, err = blkcipher_walk_done(desc, &walk, nbytes); } irq_ts_restore(ts_state); + + padlock_store_cword(&ctx->cword.encrypt); + return err; } @@ -336,7 +363,7 @@ static int cbc_aes_encrypt(struct blkcipher_desc *desc, int err; int ts_state; - padlock_reset_key(); + padlock_reset_key(&ctx->cword.encrypt); blkcipher_walk_init(&walk, dst, src, nbytes); err = blkcipher_walk_virt(desc, &walk); @@ -353,6 +380,8 @@ static int cbc_aes_encrypt(struct blkcipher_desc *desc, } irq_ts_restore(ts_state); + padlock_store_cword(&ctx->cword.decrypt); + return err; } @@ -365,7 +394,7 @@ static int cbc_aes_decrypt(struct blkcipher_desc *desc, int err; int ts_state; - padlock_reset_key(); + padlock_reset_key(&ctx->cword.encrypt); blkcipher_walk_init(&walk, dst, src, nbytes); err = blkcipher_walk_virt(desc, &walk); @@ -380,6 +409,9 @@ static int cbc_aes_decrypt(struct blkcipher_desc *desc, } irq_ts_restore(ts_state); + + padlock_store_cword(&ctx->cword.encrypt); + return err; } -- cgit v1.2.3 From 75741a034024f146ba5431602f3ad33a5df8363c Mon Sep 17 00:00:00 2001 From: Patrick McHardy Date: Mon, 24 Nov 2008 21:59:25 +0800 Subject: crypto: hifn_795x - Fix DMA setup Signed-off-by: Patrick McHardy Signed-off-by: Evgeniy Polyakov Signed-off-by: Herbert Xu --- drivers/crypto/hifn_795x.c | 165 ++++++++++++++++++++++----------------------- 1 file changed, 81 insertions(+), 84 deletions(-) (limited to 'drivers') diff --git a/drivers/crypto/hifn_795x.c b/drivers/crypto/hifn_795x.c index 2969ea99f7c..27b8af983aa 100644 --- a/drivers/crypto/hifn_795x.c +++ b/drivers/crypto/hifn_795x.c @@ -662,7 +662,6 @@ struct hifn_context unsigned int keysize, ivsize; u8 op, type, mode, unused; struct ablkcipher_walk walk; - atomic_t sg_num; }; #define crypto_alg_to_hifn(a) container_of(a, struct hifn_crypto_alg, alg) @@ -1284,7 +1283,7 @@ err_out: } static int hifn_setup_src_desc(struct hifn_device *dev, struct page *page, - unsigned int offset, unsigned int size) + unsigned int offset, unsigned int size, int last) { struct hifn_dma *dma = (struct hifn_dma *)dev->desc_virt; int idx; @@ -1296,12 +1295,12 @@ static int hifn_setup_src_desc(struct hifn_device *dev, struct page *page, dma->srcr[idx].p = __cpu_to_le32(addr); dma->srcr[idx].l = __cpu_to_le32(size | HIFN_D_VALID | - HIFN_D_MASKDONEIRQ | HIFN_D_LAST); + HIFN_D_MASKDONEIRQ | (last ? HIFN_D_LAST : 0)); if (++idx == HIFN_D_SRC_RSIZE) { dma->srcr[idx].l = __cpu_to_le32(HIFN_D_VALID | - HIFN_D_JUMP | - HIFN_D_MASKDONEIRQ | HIFN_D_LAST); + HIFN_D_JUMP | HIFN_D_MASKDONEIRQ | + (last ? HIFN_D_LAST : 0)); idx = 0; } @@ -1342,7 +1341,7 @@ static void hifn_setup_res_desc(struct hifn_device *dev) } static void hifn_setup_dst_desc(struct hifn_device *dev, struct page *page, - unsigned offset, unsigned size) + unsigned offset, unsigned size, int last) { struct hifn_dma *dma = (struct hifn_dma *)dev->desc_virt; int idx; @@ -1353,12 +1352,12 @@ static void hifn_setup_dst_desc(struct hifn_device *dev, struct page *page, idx = dma->dsti; dma->dstr[idx].p = __cpu_to_le32(addr); dma->dstr[idx].l = __cpu_to_le32(size | HIFN_D_VALID | - HIFN_D_MASKDONEIRQ | HIFN_D_LAST); + HIFN_D_MASKDONEIRQ | (last ? HIFN_D_LAST : 0)); if (++idx == HIFN_D_DST_RSIZE) { dma->dstr[idx].l = __cpu_to_le32(HIFN_D_VALID | HIFN_D_JUMP | HIFN_D_MASKDONEIRQ | - HIFN_D_LAST); + (last ? HIFN_D_LAST : 0)); idx = 0; } dma->dsti = idx; @@ -1370,16 +1369,47 @@ static void hifn_setup_dst_desc(struct hifn_device *dev, struct page *page, } } -static int hifn_setup_dma(struct hifn_device *dev, struct page *spage, unsigned int soff, - struct page *dpage, unsigned int doff, unsigned int nbytes, void *priv, - struct hifn_context *ctx) +static int hifn_setup_dma(struct hifn_device *dev, struct hifn_context *ctx, + struct scatterlist *src, struct scatterlist *dst, + unsigned int nbytes, void *priv) { - dprintk("%s: spage: %p, soffset: %u, dpage: %p, doffset: %u, nbytes: %u, priv: %p, ctx: %p.\n", + struct scatterlist *t; + struct page *spage, *dpage; + unsigned int soff, doff; + unsigned int n, len; + + t = &ctx->walk.cache[0]; + n = nbytes; + while (n) { + if (t->length) { + spage = dpage = sg_page(t); + soff = doff = 0; + len = t->length; + } else { + spage = sg_page(src); + soff = src->offset; + + dpage = sg_page(dst); + doff = dst->offset; + + len = dst->length; + } + len = min(len, n); + + dprintk("%s: spage: %p, soffset: %u, dpage: %p, doffset: %u, " + "nbytes: %u, priv: %p, ctx: %p.\n", dev->name, spage, soff, dpage, doff, nbytes, priv, ctx); - hifn_setup_src_desc(dev, spage, soff, nbytes); + hifn_setup_src_desc(dev, spage, soff, len, n - len == 0); + hifn_setup_dst_desc(dev, dpage, doff, len, n - len == 0); + + src++; + dst++; + t++; + n -= len; + } + hifn_setup_cmd_desc(dev, ctx, priv, nbytes); - hifn_setup_dst_desc(dev, dpage, doff, nbytes); hifn_setup_res_desc(dev); return 0; } @@ -1560,11 +1590,10 @@ static int hifn_setup_session(struct ablkcipher_request *req) { struct hifn_context *ctx = crypto_tfm_ctx(req->base.tfm); struct hifn_device *dev = ctx->dev; - struct page *spage, *dpage; - unsigned long soff, doff, dlen, flags; - unsigned int nbytes = req->nbytes, idx = 0, len; + unsigned long dlen, flags; + unsigned int nbytes = req->nbytes, idx = 0; int err = -EINVAL, sg_num; - struct scatterlist *src, *dst, *t; + struct scatterlist *dst; if (ctx->iv && !ctx->ivsize && ctx->mode != ACRYPTO_MODE_ECB) goto err_out_exit; @@ -1589,15 +1618,11 @@ static int hifn_setup_session(struct ablkcipher_request *req) return err; } - nbytes = req->nbytes; - idx = 0; - sg_num = ablkcipher_walk(req, &ctx->walk); if (sg_num < 0) { err = sg_num; goto err_out_exit; } - atomic_set(&ctx->sg_num, sg_num); spin_lock_irqsave(&dev->lock, flags); if (dev->started + sg_num > HIFN_QUEUE_LENGTH) { @@ -1606,36 +1631,11 @@ static int hifn_setup_session(struct ablkcipher_request *req) } dev->snum++; - dev->started += sg_num; - - while (nbytes) { - src = &req->src[idx]; - dst = &req->dst[idx]; - t = &ctx->walk.cache[idx]; - - if (t->length) { - spage = dpage = sg_page(t); - soff = doff = 0; - len = t->length; - } else { - spage = sg_page(src); - soff = src->offset; - - dpage = sg_page(dst); - doff = dst->offset; - - len = dst->length; - } - - idx++; + dev->started++; - err = hifn_setup_dma(dev, spage, soff, dpage, doff, nbytes, - req, ctx); - if (err) - goto err_out; - - nbytes -= min(len, nbytes); - } + err = hifn_setup_dma(dev, ctx, req->src, req->dst, req->nbytes, req); + if (err) + goto err_out; dev->active = HIFN_DEFAULT_ACTIVE_NUM; spin_unlock_irqrestore(&dev->lock, flags); @@ -1665,6 +1665,7 @@ static int hifn_test(struct hifn_device *dev, int encdec, u8 snum) 0xEF, 0x8A, 0x2C, 0x3B, 0x88, 0x4C, 0xFA, 0x59, 0xCA, 0x34, 0x2B, 0x2E}; + struct scatterlist sg; memset(src, 0, sizeof(src)); memset(ctx.key, 0, sizeof(ctx.key)); @@ -1676,12 +1677,11 @@ static int hifn_test(struct hifn_device *dev, int encdec, u8 snum) ctx.op = (encdec)?ACRYPTO_OP_ENCRYPT:ACRYPTO_OP_DECRYPT; ctx.mode = ACRYPTO_MODE_ECB; ctx.type = ACRYPTO_TYPE_AES_128; - atomic_set(&ctx.sg_num, 1); + ctx.walk.cache[0].length = 0; - err = hifn_setup_dma(dev, - virt_to_page(src), offset_in_page(src), - virt_to_page(src), offset_in_page(src), - sizeof(src), NULL, &ctx); + sg_init_one(&sg, &src, sizeof(src)); + + err = hifn_setup_dma(dev, &ctx, &sg, &sg, sizeof(src), NULL); if (err) goto err_out; @@ -1772,53 +1772,50 @@ static void hifn_process_ready(struct ablkcipher_request *req, int error) dprintk("%s: req: %p, ctx: %p.\n", __func__, req, ctx); dev = ctx->dev; - dprintk("%s: req: %p, started: %d, sg_num: %d.\n", - __func__, req, dev->started, atomic_read(&ctx->sg_num)); + dprintk("%s: req: %p, started: %d.\n", __func__, req, dev->started); if (--dev->started < 0) BUG(); - if (atomic_dec_and_test(&ctx->sg_num)) { + if (ctx->walk.flags & ASYNC_FLAGS_MISALIGNED) { unsigned int nbytes = req->nbytes; int idx = 0, err; struct scatterlist *dst, *t; void *saddr; - if (ctx->walk.flags & ASYNC_FLAGS_MISALIGNED) { - while (nbytes) { - t = &ctx->walk.cache[idx]; - dst = &req->dst[idx]; + while (nbytes) { + t = &ctx->walk.cache[idx]; + dst = &req->dst[idx]; - dprintk("\n%s: sg_page(t): %p, t->length: %u, " - "sg_page(dst): %p, dst->length: %u, " - "nbytes: %u.\n", - __func__, sg_page(t), t->length, - sg_page(dst), dst->length, nbytes); - - if (!t->length) { - nbytes -= min(dst->length, nbytes); - idx++; - continue; - } + dprintk("\n%s: sg_page(t): %p, t->length: %u, " + "sg_page(dst): %p, dst->length: %u, " + "nbytes: %u.\n", + __func__, sg_page(t), t->length, + sg_page(dst), dst->length, nbytes); - saddr = kmap_atomic(sg_page(t), KM_IRQ1); + if (!t->length) { + nbytes -= min(dst->length, nbytes); + idx++; + continue; + } - err = ablkcipher_get(saddr, &t->length, t->offset, - dst, nbytes, &nbytes); - if (err < 0) { - kunmap_atomic(saddr, KM_IRQ1); - break; - } + saddr = kmap_atomic(sg_page(t), KM_IRQ1); - idx += err; + err = ablkcipher_get(saddr, &t->length, t->offset, + dst, nbytes, &nbytes); + if (err < 0) { kunmap_atomic(saddr, KM_IRQ1); + break; } - ablkcipher_walk_exit(&ctx->walk); + idx += err; + kunmap_atomic(saddr, KM_IRQ1); } - req->base.complete(&req->base, error); + ablkcipher_walk_exit(&ctx->walk); } + + req->base.complete(&req->base, error); } static void hifn_check_for_completion(struct hifn_device *dev, int error) -- cgit v1.2.3 From 3416158680d5ba6fdb939e5bb52a8eba1ad5c027 Mon Sep 17 00:00:00 2001 From: Patrick McHardy Date: Mon, 24 Nov 2008 22:00:49 +0800 Subject: crypto: hifn_795x - Don't copy src sg list Signed-off-by: Patrick McHardy Signed-off-by: Evgeniy Polyakov Signed-off-by: Herbert Xu --- drivers/crypto/hifn_795x.c | 75 ++++++++++++++++++++-------------------------- 1 file changed, 33 insertions(+), 42 deletions(-) (limited to 'drivers') diff --git a/drivers/crypto/hifn_795x.c b/drivers/crypto/hifn_795x.c index 27b8af983aa..2b4940bc835 100644 --- a/drivers/crypto/hifn_795x.c +++ b/drivers/crypto/hifn_795x.c @@ -1378,32 +1378,40 @@ static int hifn_setup_dma(struct hifn_device *dev, struct hifn_context *ctx, unsigned int soff, doff; unsigned int n, len; + n = nbytes; + while (n) { + spage = sg_page(src); + soff = src->offset; + len = min(src->length, n); + + dprintk("%s: spage: %p, soffset: %u, nbytes: %u, " + "priv: %p, ctx: %p.\n", + dev->name, spage, soff, nbytes, priv, ctx); + hifn_setup_src_desc(dev, spage, soff, len, n - len == 0); + + src++; + n -= len; + } + t = &ctx->walk.cache[0]; n = nbytes; while (n) { if (t->length) { - spage = dpage = sg_page(t); - soff = doff = 0; + dpage = sg_page(t); + doff = 0; len = t->length; } else { - spage = sg_page(src); - soff = src->offset; - dpage = sg_page(dst); doff = dst->offset; - len = dst->length; } len = min(len, n); - dprintk("%s: spage: %p, soffset: %u, dpage: %p, doffset: %u, " - "nbytes: %u, priv: %p, ctx: %p.\n", - dev->name, spage, soff, dpage, doff, nbytes, priv, ctx); - - hifn_setup_src_desc(dev, spage, soff, len, n - len == 0); + dprintk("%s: dpage: %p, doffset: %u, nbytes: %u, " + "priv: %p, ctx: %p.\n", + dev->name, dpage, doff, nbytes, priv, ctx); hifn_setup_dst_desc(dev, dpage, doff, len, n - len == 0); - src++; dst++; t++; n -= len; @@ -1454,32 +1462,26 @@ static void ablkcipher_walk_exit(struct ablkcipher_walk *w) w->num = 0; } -static int ablkcipher_add(void *daddr, unsigned int *drestp, struct scatterlist *src, +static int ablkcipher_add(unsigned int *drestp, struct scatterlist *dst, unsigned int size, unsigned int *nbytesp) { unsigned int copy, drest = *drestp, nbytes = *nbytesp; int idx = 0; - void *saddr; if (drest < size || size > nbytes) return -EINVAL; while (size) { - copy = min(drest, min(size, src->length)); - - saddr = kmap_atomic(sg_page(src), KM_SOFTIRQ1); - memcpy(daddr, saddr + src->offset, copy); - kunmap_atomic(saddr, KM_SOFTIRQ1); + copy = min(drest, min(size, dst->length)); size -= copy; drest -= copy; nbytes -= copy; - daddr += copy; dprintk("%s: copy: %u, size: %u, drest: %u, nbytes: %u.\n", __func__, copy, size, drest, nbytes); - src++; + dst++; idx++; } @@ -1492,8 +1494,7 @@ static int ablkcipher_add(void *daddr, unsigned int *drestp, struct scatterlist static int ablkcipher_walk(struct ablkcipher_request *req, struct ablkcipher_walk *w) { - struct scatterlist *src, *dst, *t; - void *daddr; + struct scatterlist *dst, *t; unsigned int nbytes = req->nbytes, offset, copy, diff; int idx, tidx, err; @@ -1503,26 +1504,22 @@ static int ablkcipher_walk(struct ablkcipher_request *req, if (idx >= w->num && (w->flags & ASYNC_FLAGS_MISALIGNED)) return -EINVAL; - src = &req->src[idx]; dst = &req->dst[idx]; - dprintk("\n%s: slen: %u, dlen: %u, soff: %u, doff: %u, offset: %u, " - "nbytes: %u.\n", - __func__, src->length, dst->length, src->offset, - dst->offset, offset, nbytes); + dprintk("\n%s: dlen: %u, doff: %u, offset: %u, nbytes: %u.\n", + __func__, dst->length, dst->offset, offset, nbytes); if (!IS_ALIGNED(dst->offset, HIFN_D_DST_DALIGN) || !IS_ALIGNED(dst->length, HIFN_D_DST_DALIGN) || offset) { - unsigned slen = min(src->length - offset, nbytes); + unsigned slen = min(dst->length - offset, nbytes); unsigned dlen = PAGE_SIZE; t = &w->cache[idx]; - daddr = kmap_atomic(sg_page(t), KM_SOFTIRQ0); - err = ablkcipher_add(daddr, &dlen, src, slen, &nbytes); + err = ablkcipher_add(&dlen, dst, slen, &nbytes); if (err < 0) - goto err_out_unmap; + return err; idx += err; @@ -1558,21 +1555,19 @@ static int ablkcipher_walk(struct ablkcipher_request *req, } else { copy += diff + nbytes; - src = &req->src[idx]; + dst = &req->dst[idx]; - err = ablkcipher_add(daddr + slen, &dlen, src, nbytes, &nbytes); + err = ablkcipher_add(&dlen, dst, nbytes, &nbytes); if (err < 0) - goto err_out_unmap; + return err; idx += err; } t->length = copy; t->offset = offset; - - kunmap_atomic(daddr, KM_SOFTIRQ0); } else { - nbytes -= min(src->length, nbytes); + nbytes -= min(dst->length, nbytes); idx++; } @@ -1580,10 +1575,6 @@ static int ablkcipher_walk(struct ablkcipher_request *req, } return tidx; - -err_out_unmap: - kunmap_atomic(daddr, KM_SOFTIRQ0); - return err; } static int hifn_setup_session(struct ablkcipher_request *req) -- cgit v1.2.3 From 5df4c0c671bbb425e4a77dde5f51869aeebebd67 Mon Sep 17 00:00:00 2001 From: Patrick McHardy Date: Mon, 24 Nov 2008 22:01:42 +0800 Subject: crypto: hifn_795x - Fix request context corruption HIFN uses the transform context to store per-request data, which breaks when more than one request is outstanding. Move per request members from struct hifn_context to a new struct hifn_request_context and convert the code to use this. Signed-off-by: Patrick McHardy Signed-off-by: Evgeniy Polyakov Signed-off-by: Herbert Xu --- drivers/crypto/hifn_795x.c | 100 +++++++++++++++++++++++++-------------------- 1 file changed, 56 insertions(+), 44 deletions(-) (limited to 'drivers') diff --git a/drivers/crypto/hifn_795x.c b/drivers/crypto/hifn_795x.c index 2b4940bc835..97a77d7baca 100644 --- a/drivers/crypto/hifn_795x.c +++ b/drivers/crypto/hifn_795x.c @@ -657,9 +657,15 @@ struct ablkcipher_walk struct hifn_context { - u8 key[HIFN_MAX_CRYPT_KEY_LENGTH], *iv; + u8 key[HIFN_MAX_CRYPT_KEY_LENGTH]; struct hifn_device *dev; - unsigned int keysize, ivsize; + unsigned int keysize; +}; + +struct hifn_request_context +{ + u8 *iv; + unsigned int ivsize; u8 op, type, mode, unused; struct ablkcipher_walk walk; }; @@ -1167,7 +1173,8 @@ static int hifn_setup_crypto_command(struct hifn_device *dev, } static int hifn_setup_cmd_desc(struct hifn_device *dev, - struct hifn_context *ctx, void *priv, unsigned int nbytes) + struct hifn_context *ctx, struct hifn_request_context *rctx, + void *priv, unsigned int nbytes) { struct hifn_dma *dma = (struct hifn_dma *)dev->desc_virt; int cmd_len, sa_idx; @@ -1178,7 +1185,7 @@ static int hifn_setup_cmd_desc(struct hifn_device *dev, buf_pos = buf = dma->command_bufs[dma->cmdi]; mask = 0; - switch (ctx->op) { + switch (rctx->op) { case ACRYPTO_OP_DECRYPT: mask = HIFN_BASE_CMD_CRYPT | HIFN_BASE_CMD_DECODE; break; @@ -1195,15 +1202,15 @@ static int hifn_setup_cmd_desc(struct hifn_device *dev, buf_pos += hifn_setup_base_command(dev, buf_pos, nbytes, nbytes, mask, dev->snum); - if (ctx->op == ACRYPTO_OP_ENCRYPT || ctx->op == ACRYPTO_OP_DECRYPT) { + if (rctx->op == ACRYPTO_OP_ENCRYPT || rctx->op == ACRYPTO_OP_DECRYPT) { u16 md = 0; if (ctx->keysize) md |= HIFN_CRYPT_CMD_NEW_KEY; - if (ctx->iv && ctx->mode != ACRYPTO_MODE_ECB) + if (rctx->iv && rctx->mode != ACRYPTO_MODE_ECB) md |= HIFN_CRYPT_CMD_NEW_IV; - switch (ctx->mode) { + switch (rctx->mode) { case ACRYPTO_MODE_ECB: md |= HIFN_CRYPT_CMD_MODE_ECB; break; @@ -1220,7 +1227,7 @@ static int hifn_setup_cmd_desc(struct hifn_device *dev, goto err_out; } - switch (ctx->type) { + switch (rctx->type) { case ACRYPTO_TYPE_AES_128: if (ctx->keysize != 16) goto err_out; @@ -1255,7 +1262,7 @@ static int hifn_setup_cmd_desc(struct hifn_device *dev, buf_pos += hifn_setup_crypto_command(dev, buf_pos, nbytes, nbytes, ctx->key, ctx->keysize, - ctx->iv, ctx->ivsize, md); + rctx->iv, rctx->ivsize, md); } dev->sa[sa_idx] = priv; @@ -1265,7 +1272,7 @@ static int hifn_setup_cmd_desc(struct hifn_device *dev, HIFN_D_LAST | HIFN_D_MASKDONEIRQ); if (++dma->cmdi == HIFN_D_CMD_RSIZE) { - dma->cmdr[dma->cmdi].l = __cpu_to_le32(HIFN_MAX_COMMAND | + dma->cmdr[dma->cmdi].l = __cpu_to_le32( HIFN_D_VALID | HIFN_D_LAST | HIFN_D_MASKDONEIRQ | HIFN_D_JUMP); dma->cmdi = 0; @@ -1369,7 +1376,8 @@ static void hifn_setup_dst_desc(struct hifn_device *dev, struct page *page, } } -static int hifn_setup_dma(struct hifn_device *dev, struct hifn_context *ctx, +static int hifn_setup_dma(struct hifn_device *dev, + struct hifn_context *ctx, struct hifn_request_context *rctx, struct scatterlist *src, struct scatterlist *dst, unsigned int nbytes, void *priv) { @@ -1385,18 +1393,18 @@ static int hifn_setup_dma(struct hifn_device *dev, struct hifn_context *ctx, len = min(src->length, n); dprintk("%s: spage: %p, soffset: %u, nbytes: %u, " - "priv: %p, ctx: %p.\n", - dev->name, spage, soff, nbytes, priv, ctx); + "priv: %p, rctx: %p.\n", + dev->name, spage, soff, nbytes, priv, rctx); hifn_setup_src_desc(dev, spage, soff, len, n - len == 0); src++; n -= len; } - t = &ctx->walk.cache[0]; + t = &rctx->walk.cache[0]; n = nbytes; while (n) { - if (t->length) { + if (t->length && rctx->walk.flags & ASYNC_FLAGS_MISALIGNED) { dpage = sg_page(t); doff = 0; len = t->length; @@ -1408,8 +1416,8 @@ static int hifn_setup_dma(struct hifn_device *dev, struct hifn_context *ctx, len = min(len, n); dprintk("%s: dpage: %p, doffset: %u, nbytes: %u, " - "priv: %p, ctx: %p.\n", - dev->name, dpage, doff, nbytes, priv, ctx); + "priv: %p, rctx: %p.\n", + dev->name, dpage, doff, nbytes, priv, rctx); hifn_setup_dst_desc(dev, dpage, doff, len, n - len == 0); dst++; @@ -1417,7 +1425,7 @@ static int hifn_setup_dma(struct hifn_device *dev, struct hifn_context *ctx, n -= len; } - hifn_setup_cmd_desc(dev, ctx, priv, nbytes); + hifn_setup_cmd_desc(dev, ctx, rctx, priv, nbytes); hifn_setup_res_desc(dev); return 0; } @@ -1580,16 +1588,17 @@ static int ablkcipher_walk(struct ablkcipher_request *req, static int hifn_setup_session(struct ablkcipher_request *req) { struct hifn_context *ctx = crypto_tfm_ctx(req->base.tfm); + struct hifn_request_context *rctx = ablkcipher_request_ctx(req); struct hifn_device *dev = ctx->dev; unsigned long dlen, flags; unsigned int nbytes = req->nbytes, idx = 0; int err = -EINVAL, sg_num; struct scatterlist *dst; - if (ctx->iv && !ctx->ivsize && ctx->mode != ACRYPTO_MODE_ECB) + if (rctx->iv && !rctx->ivsize && rctx->mode != ACRYPTO_MODE_ECB) goto err_out_exit; - ctx->walk.flags = 0; + rctx->walk.flags = 0; while (nbytes) { dst = &req->dst[idx]; @@ -1597,19 +1606,19 @@ static int hifn_setup_session(struct ablkcipher_request *req) if (!IS_ALIGNED(dst->offset, HIFN_D_DST_DALIGN) || !IS_ALIGNED(dlen, HIFN_D_DST_DALIGN)) - ctx->walk.flags |= ASYNC_FLAGS_MISALIGNED; + rctx->walk.flags |= ASYNC_FLAGS_MISALIGNED; nbytes -= dlen; idx++; } - if (ctx->walk.flags & ASYNC_FLAGS_MISALIGNED) { - err = ablkcipher_walk_init(&ctx->walk, idx, GFP_ATOMIC); + if (rctx->walk.flags & ASYNC_FLAGS_MISALIGNED) { + err = ablkcipher_walk_init(&rctx->walk, idx, GFP_ATOMIC); if (err < 0) return err; } - sg_num = ablkcipher_walk(req, &ctx->walk); + sg_num = ablkcipher_walk(req, &rctx->walk); if (sg_num < 0) { err = sg_num; goto err_out_exit; @@ -1624,7 +1633,7 @@ static int hifn_setup_session(struct ablkcipher_request *req) dev->snum++; dev->started++; - err = hifn_setup_dma(dev, ctx, req->src, req->dst, req->nbytes, req); + err = hifn_setup_dma(dev, ctx, rctx, req->src, req->dst, req->nbytes, req); if (err) goto err_out; @@ -1639,9 +1648,9 @@ err_out_exit: if (err) dprintk("%s: iv: %p [%d], key: %p [%d], mode: %u, op: %u, " "type: %u, err: %d.\n", - dev->name, ctx->iv, ctx->ivsize, + dev->name, rctx->iv, rctx->ivsize, ctx->key, ctx->keysize, - ctx->mode, ctx->op, ctx->type, err); + rctx->mode, rctx->op, rctx->type, err); return err; } @@ -1651,6 +1660,7 @@ static int hifn_test(struct hifn_device *dev, int encdec, u8 snum) int n, err; u8 src[16]; struct hifn_context ctx; + struct hifn_request_context rctx; u8 fips_aes_ecb_from_zero[16] = { 0x66, 0xE9, 0x4B, 0xD4, 0xEF, 0x8A, 0x2C, 0x3B, @@ -1663,16 +1673,16 @@ static int hifn_test(struct hifn_device *dev, int encdec, u8 snum) ctx.dev = dev; ctx.keysize = 16; - ctx.ivsize = 0; - ctx.iv = NULL; - ctx.op = (encdec)?ACRYPTO_OP_ENCRYPT:ACRYPTO_OP_DECRYPT; - ctx.mode = ACRYPTO_MODE_ECB; - ctx.type = ACRYPTO_TYPE_AES_128; - ctx.walk.cache[0].length = 0; + rctx.ivsize = 0; + rctx.iv = NULL; + rctx.op = (encdec)?ACRYPTO_OP_ENCRYPT:ACRYPTO_OP_DECRYPT; + rctx.mode = ACRYPTO_MODE_ECB; + rctx.type = ACRYPTO_TYPE_AES_128; + rctx.walk.cache[0].length = 0; sg_init_one(&sg, &src, sizeof(src)); - err = hifn_setup_dma(dev, &ctx, &sg, &sg, sizeof(src), NULL); + err = hifn_setup_dma(dev, &ctx, &rctx, &sg, &sg, sizeof(src), NULL); if (err) goto err_out; @@ -1758,9 +1768,10 @@ static int ablkcipher_get(void *saddr, unsigned int *srestp, unsigned int offset static void hifn_process_ready(struct ablkcipher_request *req, int error) { struct hifn_context *ctx = crypto_tfm_ctx(req->base.tfm); + struct hifn_request_context *rctx = ablkcipher_request_ctx(req); struct hifn_device *dev; - dprintk("%s: req: %p, ctx: %p.\n", __func__, req, ctx); + dprintk("%s: req: %p, ctx: %p rctx: %p.\n", __func__, req, ctx, rctx); dev = ctx->dev; dprintk("%s: req: %p, started: %d.\n", __func__, req, dev->started); @@ -1768,14 +1779,14 @@ static void hifn_process_ready(struct ablkcipher_request *req, int error) if (--dev->started < 0) BUG(); - if (ctx->walk.flags & ASYNC_FLAGS_MISALIGNED) { + if (rctx->walk.flags & ASYNC_FLAGS_MISALIGNED) { unsigned int nbytes = req->nbytes; int idx = 0, err; struct scatterlist *dst, *t; void *saddr; while (nbytes) { - t = &ctx->walk.cache[idx]; + t = &rctx->walk.cache[idx]; dst = &req->dst[idx]; dprintk("\n%s: sg_page(t): %p, t->length: %u, " @@ -1803,7 +1814,7 @@ static void hifn_process_ready(struct ablkcipher_request *req, int error) kunmap_atomic(saddr, KM_IRQ1); } - ablkcipher_walk_exit(&ctx->walk); + ablkcipher_walk_exit(&rctx->walk); } req->base.complete(&req->base, error); @@ -2109,6 +2120,7 @@ static int hifn_setup_crypto_req(struct ablkcipher_request *req, u8 op, u8 type, u8 mode) { struct hifn_context *ctx = crypto_tfm_ctx(req->base.tfm); + struct hifn_request_context *rctx = ablkcipher_request_ctx(req); unsigned ivsize; ivsize = crypto_ablkcipher_ivsize(crypto_ablkcipher_reqtfm(req)); @@ -2129,11 +2141,11 @@ static int hifn_setup_crypto_req(struct ablkcipher_request *req, u8 op, type = ACRYPTO_TYPE_AES_256; } - ctx->op = op; - ctx->mode = mode; - ctx->type = type; - ctx->iv = req->info; - ctx->ivsize = ivsize; + rctx->op = op; + rctx->mode = mode; + rctx->type = type; + rctx->iv = req->info; + rctx->ivsize = ivsize; /* * HEAVY TODO: needs to kick Herbert XU to write documentation. @@ -2484,7 +2496,7 @@ static int hifn_cra_init(struct crypto_tfm *tfm) struct hifn_context *ctx = crypto_tfm_ctx(tfm); ctx->dev = ha->dev; - + tfm->crt_ablkcipher.reqsize = sizeof(struct hifn_request_context); return 0; } -- cgit v1.2.3 From ed4f92e3723a7d73bcd179ba09529094752063b2 Mon Sep 17 00:00:00 2001 From: Patrick McHardy Date: Mon, 24 Nov 2008 22:02:55 +0800 Subject: crypto: hifn_795x - Fix queue processing Signed-off-by: Patrick McHardy Signed-off-by: Evgeniy Polyakov Signed-off-by: Herbert Xu --- drivers/crypto/hifn_795x.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) (limited to 'drivers') diff --git a/drivers/crypto/hifn_795x.c b/drivers/crypto/hifn_795x.c index 97a77d7baca..ce3a27ecfaa 100644 --- a/drivers/crypto/hifn_795x.c +++ b/drivers/crypto/hifn_795x.c @@ -2158,7 +2158,7 @@ static int hifn_setup_crypto_req(struct ablkcipher_request *req, u8 op, static int hifn_process_queue(struct hifn_device *dev) { - struct crypto_async_request *async_req; + struct crypto_async_request *async_req, *backlog; struct hifn_context *ctx; struct ablkcipher_request *req; unsigned long flags; @@ -2166,12 +2166,16 @@ static int hifn_process_queue(struct hifn_device *dev) while (dev->started < HIFN_QUEUE_LENGTH) { spin_lock_irqsave(&dev->lock, flags); + backlog = crypto_get_backlog(&dev->queue); async_req = crypto_dequeue_request(&dev->queue); spin_unlock_irqrestore(&dev->lock, flags); if (!async_req) break; + if (backlog) + backlog->complete(backlog, -EINPROGRESS); + ctx = crypto_tfm_ctx(async_req->tfm); req = container_of(async_req, struct ablkcipher_request, base); @@ -2575,6 +2579,9 @@ static void hifn_tasklet_callback(unsigned long data) * context or update is atomic (like setting dev->sa[i] to NULL). */ hifn_check_for_completion(dev, 0); + + if (dev->started < HIFN_QUEUE_LENGTH && dev->queue.qlen) + hifn_process_queue(dev); } static int hifn_probe(struct pci_dev *pdev, const struct pci_device_id *id) -- cgit v1.2.3 From 3ec858de6468e810953e41b8c154a0d605b25d68 Mon Sep 17 00:00:00 2001 From: Patrick McHardy Date: Mon, 24 Nov 2008 22:03:37 +0800 Subject: crypto: hifn_795x - Remove some unused cruft Signed-off-by: Patrick McHardy Signed-off-by: Evgeniy Polyakov Signed-off-by: Herbert Xu --- drivers/crypto/hifn_795x.c | 32 +------------------------------- 1 file changed, 1 insertion(+), 31 deletions(-) (limited to 'drivers') diff --git a/drivers/crypto/hifn_795x.c b/drivers/crypto/hifn_795x.c index ce3a27ecfaa..a03fe571265 100644 --- a/drivers/crypto/hifn_795x.c +++ b/drivers/crypto/hifn_795x.c @@ -38,9 +38,6 @@ #include -#undef dprintk - -#define HIFN_TEST //#define HIFN_DEBUG #ifdef HIFN_DEBUG @@ -406,8 +403,6 @@ struct hifn_dma { u8 command_bufs[HIFN_D_CMD_RSIZE][HIFN_MAX_COMMAND]; u8 result_bufs[HIFN_D_CMD_RSIZE][HIFN_MAX_RESULT]; - u64 test_src, test_dst; - /* * Our current positions for insertion and removal from the descriptor * rings. @@ -434,9 +429,6 @@ struct hifn_device struct pci_dev *pdev; void __iomem *bar[3]; - unsigned long result_mem; - dma_addr_t dst; - void *desc_virt; dma_addr_t desc_dma; @@ -446,8 +438,6 @@ struct hifn_device spinlock_t lock; - void *priv; - u32 flags; int active, started; struct delayed_work work; @@ -2638,22 +2628,11 @@ static int hifn_probe(struct pci_dev *pdev, const struct pci_device_id *id) goto err_out_unmap_bars; } - dev->result_mem = __get_free_pages(GFP_KERNEL, HIFN_MAX_RESULT_ORDER); - if (!dev->result_mem) { - dprintk("Failed to allocate %d pages for result_mem.\n", - HIFN_MAX_RESULT_ORDER); - goto err_out_unmap_bars; - } - memset((void *)dev->result_mem, 0, PAGE_SIZE*(1<dst = pci_map_single(pdev, (void *)dev->result_mem, - PAGE_SIZE << HIFN_MAX_RESULT_ORDER, PCI_DMA_FROMDEVICE); - dev->desc_virt = pci_alloc_consistent(pdev, sizeof(struct hifn_dma), &dev->desc_dma); if (!dev->desc_virt) { dprintk("Failed to allocate descriptor rings.\n"); - goto err_out_free_result_pages; + goto err_out_unmap_bars; } memset(dev->desc_virt, 0, sizeof(struct hifn_dma)); @@ -2713,11 +2692,6 @@ err_out_free_desc: pci_free_consistent(pdev, sizeof(struct hifn_dma), dev->desc_virt, dev->desc_dma); -err_out_free_result_pages: - pci_unmap_single(pdev, dev->dst, PAGE_SIZE << HIFN_MAX_RESULT_ORDER, - PCI_DMA_FROMDEVICE); - free_pages(dev->result_mem, HIFN_MAX_RESULT_ORDER); - err_out_unmap_bars: for (i=0; i<3; ++i) if (dev->bar[i]) @@ -2755,10 +2729,6 @@ static void hifn_remove(struct pci_dev *pdev) pci_free_consistent(pdev, sizeof(struct hifn_dma), dev->desc_virt, dev->desc_dma); - pci_unmap_single(pdev, dev->dst, - PAGE_SIZE << HIFN_MAX_RESULT_ORDER, - PCI_DMA_FROMDEVICE); - free_pages(dev->result_mem, HIFN_MAX_RESULT_ORDER); for (i=0; i<3; ++i) if (dev->bar[i]) iounmap(dev->bar[i]); -- cgit v1.2.3 From d6a10c84a2a2b08bdd637852c9cc42c41e109b25 Mon Sep 17 00:00:00 2001 From: Evgeniy Polyakov Date: Mon, 24 Nov 2008 22:04:39 +0800 Subject: crypto: hifn_795x - Fix queue management Fix queue management. Change ring size and perform its check not one after another descriptor, but using stored pointers to the last checked descriptors. Signed-off-by: Evgeniy Polyakov Signed-off-by: Herbert Xu --- drivers/crypto/hifn_795x.c | 174 +++++++++++++++++++++------------------------ 1 file changed, 81 insertions(+), 93 deletions(-) (limited to 'drivers') diff --git a/drivers/crypto/hifn_795x.c b/drivers/crypto/hifn_795x.c index a03fe571265..ab4f94bec4f 100644 --- a/drivers/crypto/hifn_795x.c +++ b/drivers/crypto/hifn_795x.c @@ -360,14 +360,14 @@ static atomic_t hifn_dev_number; #define HIFN_NAMESIZE 32 #define HIFN_MAX_RESULT_ORDER 5 -#define HIFN_D_CMD_RSIZE 24*4 -#define HIFN_D_SRC_RSIZE 80*4 -#define HIFN_D_DST_RSIZE 80*4 -#define HIFN_D_RES_RSIZE 24*4 +#define HIFN_D_CMD_RSIZE 24*1 +#define HIFN_D_SRC_RSIZE 80*1 +#define HIFN_D_DST_RSIZE 80*1 +#define HIFN_D_RES_RSIZE 24*1 #define HIFN_D_DST_DALIGN 4 -#define HIFN_QUEUE_LENGTH HIFN_D_CMD_RSIZE-1 +#define HIFN_QUEUE_LENGTH (HIFN_D_CMD_RSIZE - 1) #define AES_MIN_KEY_SIZE 16 #define AES_MAX_KEY_SIZE 32 @@ -1256,6 +1256,7 @@ static int hifn_setup_cmd_desc(struct hifn_device *dev, } dev->sa[sa_idx] = priv; + dev->started++; cmd_len = buf_pos - buf; dma->cmdr[dma->cmdi].l = __cpu_to_le32(cmd_len | HIFN_D_VALID | @@ -1382,9 +1383,6 @@ static int hifn_setup_dma(struct hifn_device *dev, soff = src->offset; len = min(src->length, n); - dprintk("%s: spage: %p, soffset: %u, nbytes: %u, " - "priv: %p, rctx: %p.\n", - dev->name, spage, soff, nbytes, priv, rctx); hifn_setup_src_desc(dev, spage, soff, len, n - len == 0); src++; @@ -1405,9 +1403,6 @@ static int hifn_setup_dma(struct hifn_device *dev, } len = min(len, n); - dprintk("%s: dpage: %p, doffset: %u, nbytes: %u, " - "priv: %p, rctx: %p.\n", - dev->name, dpage, doff, nbytes, priv, rctx); hifn_setup_dst_desc(dev, dpage, doff, len, n - len == 0); dst++; @@ -1620,13 +1615,12 @@ static int hifn_setup_session(struct ablkcipher_request *req) goto err_out; } - dev->snum++; - dev->started++; - err = hifn_setup_dma(dev, ctx, rctx, req->src, req->dst, req->nbytes, req); if (err) goto err_out; + dev->snum++; + dev->active = HIFN_DEFAULT_ACTIVE_NUM; spin_unlock_irqrestore(&dev->lock, flags); @@ -1635,12 +1629,13 @@ static int hifn_setup_session(struct ablkcipher_request *req) err_out: spin_unlock_irqrestore(&dev->lock, flags); err_out_exit: - if (err) - dprintk("%s: iv: %p [%d], key: %p [%d], mode: %u, op: %u, " + if (err) { + printk("%s: iv: %p [%d], key: %p [%d], mode: %u, op: %u, " "type: %u, err: %d.\n", dev->name, rctx->iv, rctx->ivsize, ctx->key, ctx->keysize, rctx->mode, rctx->op, rctx->type, err); + } return err; } @@ -1676,6 +1671,7 @@ static int hifn_test(struct hifn_device *dev, int encdec, u8 snum) if (err) goto err_out; + dev->started = 0; msleep(200); dprintk("%s: decoded: ", dev->name); @@ -1702,6 +1698,7 @@ static int hifn_start_device(struct hifn_device *dev) { int err; + dev->started = dev->active = 0; hifn_reset_dma(dev, 1); err = hifn_enable_crypto(dev); @@ -1755,19 +1752,22 @@ static int ablkcipher_get(void *saddr, unsigned int *srestp, unsigned int offset return idx; } -static void hifn_process_ready(struct ablkcipher_request *req, int error) +static inline void hifn_complete_sa(struct hifn_device *dev, int i) { - struct hifn_context *ctx = crypto_tfm_ctx(req->base.tfm); - struct hifn_request_context *rctx = ablkcipher_request_ctx(req); - struct hifn_device *dev; - - dprintk("%s: req: %p, ctx: %p rctx: %p.\n", __func__, req, ctx, rctx); + unsigned long flags; - dev = ctx->dev; - dprintk("%s: req: %p, started: %d.\n", __func__, req, dev->started); + spin_lock_irqsave(&dev->lock, flags); + dev->sa[i] = NULL; + dev->started--; + if (dev->started < 0) + printk("%s: started: %d.\n", __func__, dev->started); + spin_unlock_irqrestore(&dev->lock, flags); + BUG_ON(dev->started < 0); +} - if (--dev->started < 0) - BUG(); +static void hifn_process_ready(struct ablkcipher_request *req, int error) +{ + struct hifn_request_context *rctx = ablkcipher_request_ctx(req); if (rctx->walk.flags & ASYNC_FLAGS_MISALIGNED) { unsigned int nbytes = req->nbytes; @@ -1810,33 +1810,7 @@ static void hifn_process_ready(struct ablkcipher_request *req, int error) req->base.complete(&req->base, error); } -static void hifn_check_for_completion(struct hifn_device *dev, int error) -{ - int i; - struct hifn_dma *dma = (struct hifn_dma *)dev->desc_virt; - - for (i=0; iresr[i]; - - if (!(d->l & __cpu_to_le32(HIFN_D_VALID)) && dev->sa[i]) { - dev->success++; - dev->reset = 0; - hifn_process_ready(dev->sa[i], error); - dev->sa[i] = NULL; - } - - if (d->l & __cpu_to_le32(HIFN_D_DESTOVER | HIFN_D_OVER)) - if (printk_ratelimit()) - printk("%s: overflow detected [d: %u, o: %u] " - "at %d resr: l: %08x, p: %08x.\n", - dev->name, - !!(d->l & __cpu_to_le32(HIFN_D_DESTOVER)), - !!(d->l & __cpu_to_le32(HIFN_D_OVER)), - i, d->l, d->p); - } -} - -static void hifn_clear_rings(struct hifn_device *dev) +static void hifn_clear_rings(struct hifn_device *dev, int error) { struct hifn_dma *dma = (struct hifn_dma *)dev->desc_virt; int i, u; @@ -1853,21 +1827,26 @@ static void hifn_clear_rings(struct hifn_device *dev) if (dma->resr[i].l & __cpu_to_le32(HIFN_D_VALID)) break; - if (i != HIFN_D_RES_RSIZE) - u--; + if (dev->sa[i]) { + dev->success++; + dev->reset = 0; + hifn_process_ready(dev->sa[i], error); + hifn_complete_sa(dev, i); + } - if (++i == (HIFN_D_RES_RSIZE + 1)) + if (++i == HIFN_D_RES_RSIZE) i = 0; + u--; } dma->resk = i; dma->resu = u; i = dma->srck; u = dma->srcu; while (u != 0) { - if (i == HIFN_D_SRC_RSIZE) - i = 0; if (dma->srcr[i].l & __cpu_to_le32(HIFN_D_VALID)) break; - i++, u--; + if (++i == HIFN_D_SRC_RSIZE) + i = 0; + u--; } dma->srck = i; dma->srcu = u; @@ -1875,20 +1854,19 @@ static void hifn_clear_rings(struct hifn_device *dev) while (u != 0) { if (dma->cmdr[i].l & __cpu_to_le32(HIFN_D_VALID)) break; - if (i != HIFN_D_CMD_RSIZE) - u--; - if (++i == (HIFN_D_CMD_RSIZE + 1)) + if (++i == HIFN_D_CMD_RSIZE) i = 0; + u--; } dma->cmdk = i; dma->cmdu = u; i = dma->dstk; u = dma->dstu; while (u != 0) { - if (i == HIFN_D_DST_RSIZE) - i = 0; if (dma->dstr[i].l & __cpu_to_le32(HIFN_D_VALID)) break; - i++, u--; + if (++i == HIFN_D_DST_RSIZE) + i = 0; + u--; } dma->dstk = i; dma->dstu = u; @@ -1933,30 +1911,39 @@ static void hifn_work(struct work_struct *work) } else dev->active--; - if (dev->prev_success == dev->success && dev->started) + if ((dev->prev_success == dev->success) && dev->started) reset = 1; dev->prev_success = dev->success; spin_unlock_irqrestore(&dev->lock, flags); if (reset) { - dprintk("%s: r: %08x, active: %d, started: %d, " - "success: %lu: reset: %d.\n", - dev->name, r, dev->active, dev->started, - dev->success, reset); - if (++dev->reset >= 5) { - dprintk("%s: really hard reset.\n", dev->name); + int i; + struct hifn_dma *dma = (struct hifn_dma *)dev->desc_virt; + + printk("%s: r: %08x, active: %d, started: %d, " + "success: %lu: qlen: %u/%u, reset: %d.\n", + dev->name, r, dev->active, dev->started, + dev->success, dev->queue.qlen, dev->queue.max_qlen, + reset); + + printk("%s: res: ", __func__); + for (i=0; iresr[i].l, dev->sa[i]); + if (dev->sa[i]) { + hifn_process_ready(dev->sa[i], -ENODEV); + hifn_complete_sa(dev, i); + } + } + printk("\n"); + hifn_reset_dma(dev, 1); hifn_stop_device(dev); hifn_start_device(dev); dev->reset = 0; } - spin_lock_irqsave(&dev->lock, flags); - hifn_check_for_completion(dev, -EBUSY); - hifn_clear_rings(dev); - dev->started = 0; - spin_unlock_irqrestore(&dev->lock, flags); + tasklet_schedule(&dev->tasklet); } schedule_delayed_work(&dev->work, HZ); @@ -1973,8 +1960,8 @@ static irqreturn_t hifn_interrupt(int irq, void *data) dprintk("%s: 1 dmacsr: %08x, dmareg: %08x, res: %08x [%d], " "i: %d.%d.%d.%d, u: %d.%d.%d.%d.\n", dev->name, dmacsr, dev->dmareg, dmacsr & dev->dmareg, dma->cmdi, - dma->cmdu, dma->srcu, dma->dstu, dma->resu, - dma->cmdi, dma->srci, dma->dsti, dma->resi); + dma->cmdi, dma->srci, dma->dsti, dma->resi, + dma->cmdu, dma->srcu, dma->dstu, dma->resu); if ((dmacsr & dev->dmareg) == 0) return IRQ_NONE; @@ -1991,11 +1978,10 @@ static irqreturn_t hifn_interrupt(int irq, void *data) if (restart) { u32 puisr = hifn_read_0(dev, HIFN_0_PUISR); - if (printk_ratelimit()) - printk("%s: overflow: r: %d, d: %d, puisr: %08x, d: %u.\n", - dev->name, !!(dmacsr & HIFN_DMACSR_R_OVER), - !!(dmacsr & HIFN_DMACSR_D_OVER), - puisr, !!(puisr & HIFN_PUISR_DSTOVER)); + printk(KERN_WARNING "%s: overflow: r: %d, d: %d, puisr: %08x, d: %u.\n", + dev->name, !!(dmacsr & HIFN_DMACSR_R_OVER), + !!(dmacsr & HIFN_DMACSR_D_OVER), + puisr, !!(puisr & HIFN_PUISR_DSTOVER)); if (!!(puisr & HIFN_PUISR_DSTOVER)) hifn_write_0(dev, HIFN_0_PUISR, HIFN_PUISR_DSTOVER); hifn_write_1(dev, HIFN_1_DMA_CSR, dmacsr & (HIFN_DMACSR_R_OVER | @@ -2005,12 +1991,11 @@ static irqreturn_t hifn_interrupt(int irq, void *data) restart = dmacsr & (HIFN_DMACSR_C_ABORT | HIFN_DMACSR_S_ABORT | HIFN_DMACSR_D_ABORT | HIFN_DMACSR_R_ABORT); if (restart) { - if (printk_ratelimit()) - printk("%s: abort: c: %d, s: %d, d: %d, r: %d.\n", - dev->name, !!(dmacsr & HIFN_DMACSR_C_ABORT), - !!(dmacsr & HIFN_DMACSR_S_ABORT), - !!(dmacsr & HIFN_DMACSR_D_ABORT), - !!(dmacsr & HIFN_DMACSR_R_ABORT)); + printk(KERN_WARNING "%s: abort: c: %d, s: %d, d: %d, r: %d.\n", + dev->name, !!(dmacsr & HIFN_DMACSR_C_ABORT), + !!(dmacsr & HIFN_DMACSR_S_ABORT), + !!(dmacsr & HIFN_DMACSR_D_ABORT), + !!(dmacsr & HIFN_DMACSR_R_ABORT)); hifn_reset_dma(dev, 1); hifn_init_dma(dev); hifn_init_registers(dev); @@ -2023,7 +2008,6 @@ static irqreturn_t hifn_interrupt(int irq, void *data) } tasklet_schedule(&dev->tasklet); - hifn_clear_rings(dev); return IRQ_HANDLED; } @@ -2037,21 +2021,25 @@ static void hifn_flush(struct hifn_device *dev) struct hifn_dma *dma = (struct hifn_dma *)dev->desc_virt; int i; - spin_lock_irqsave(&dev->lock, flags); for (i=0; iresr[i]; if (dev->sa[i]) { hifn_process_ready(dev->sa[i], (d->l & __cpu_to_le32(HIFN_D_VALID))?-ENODEV:0); + hifn_complete_sa(dev, i); } } + spin_lock_irqsave(&dev->lock, flags); while ((async_req = crypto_dequeue_request(&dev->queue))) { ctx = crypto_tfm_ctx(async_req->tfm); req = container_of(async_req, struct ablkcipher_request, base); + spin_unlock_irqrestore(&dev->lock, flags); hifn_process_ready(req, -ENODEV); + + spin_lock_irqsave(&dev->lock, flags); } spin_unlock_irqrestore(&dev->lock, flags); } @@ -2568,7 +2556,7 @@ static void hifn_tasklet_callback(unsigned long data) * (like dev->success), but they are used in process * context or update is atomic (like setting dev->sa[i] to NULL). */ - hifn_check_for_completion(dev, 0); + hifn_clear_rings(dev, 0); if (dev->started < HIFN_QUEUE_LENGTH && dev->queue.qlen) hifn_process_queue(dev); -- cgit v1.2.3 From 5f459f0adc7581da01cdd9b7f84db14a4559a390 Mon Sep 17 00:00:00 2001 From: Evgeniy Polyakov Date: Tue, 25 Nov 2008 23:17:10 +0800 Subject: crypto: hifn_795x - Use softirq kernel mapping in bh context Use KM_SOFTIRQ instead of KM_IRQ in tasklet context. Added bug_on on input no-page condition. Signed-off-by: Evgeniy Polyakov Signed-off-by: Herbert Xu --- drivers/crypto/hifn_795x.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) (limited to 'drivers') diff --git a/drivers/crypto/hifn_795x.c b/drivers/crypto/hifn_795x.c index ab4f94bec4f..0c79fe7f156 100644 --- a/drivers/crypto/hifn_795x.c +++ b/drivers/crypto/hifn_795x.c @@ -1393,10 +1393,12 @@ static int hifn_setup_dma(struct hifn_device *dev, n = nbytes; while (n) { if (t->length && rctx->walk.flags & ASYNC_FLAGS_MISALIGNED) { + BUG_ON(!sg_page(t)); dpage = sg_page(t); doff = 0; len = t->length; } else { + BUG_ON(!sg_page(dst)); dpage = sg_page(dst); doff = dst->offset; len = dst->length; @@ -1791,17 +1793,17 @@ static void hifn_process_ready(struct ablkcipher_request *req, int error) continue; } - saddr = kmap_atomic(sg_page(t), KM_IRQ1); + saddr = kmap_atomic(sg_page(t), KM_SOFTIRQ0); err = ablkcipher_get(saddr, &t->length, t->offset, dst, nbytes, &nbytes); if (err < 0) { - kunmap_atomic(saddr, KM_IRQ1); + kunmap_atomic(saddr, KM_SOFTIRQ0); break; } idx += err; - kunmap_atomic(saddr, KM_IRQ1); + kunmap_atomic(saddr, KM_SOFTIRQ0); } ablkcipher_walk_exit(&rctx->walk); -- cgit v1.2.3 From dad3df2044b78ba68a92bf78e38a408bab80ff61 Mon Sep 17 00:00:00 2001 From: Harvey Harrison Date: Fri, 28 Nov 2008 20:49:19 +0800 Subject: crypto: remove uses of __constant_{endian} helpers Base versions handle constant folding just fine. Signed-off-by: Harvey Harrison Signed-off-by: Herbert Xu --- drivers/crypto/talitos.h | 82 ++++++++++++++++++++++++------------------------ 1 file changed, 41 insertions(+), 41 deletions(-) (limited to 'drivers') diff --git a/drivers/crypto/talitos.h b/drivers/crypto/talitos.h index 125560a1d5f..575981f0cfd 100644 --- a/drivers/crypto/talitos.h +++ b/drivers/crypto/talitos.h @@ -134,34 +134,34 @@ */ /* written back when done */ -#define DESC_HDR_DONE __constant_cpu_to_be32(0xff000000) -#define DESC_HDR_LO_ICCR1_MASK __constant_cpu_to_be32(0x00180000) -#define DESC_HDR_LO_ICCR1_PASS __constant_cpu_to_be32(0x00080000) -#define DESC_HDR_LO_ICCR1_FAIL __constant_cpu_to_be32(0x00100000) +#define DESC_HDR_DONE cpu_to_be32(0xff000000) +#define DESC_HDR_LO_ICCR1_MASK cpu_to_be32(0x00180000) +#define DESC_HDR_LO_ICCR1_PASS cpu_to_be32(0x00080000) +#define DESC_HDR_LO_ICCR1_FAIL cpu_to_be32(0x00100000) /* primary execution unit select */ -#define DESC_HDR_SEL0_MASK __constant_cpu_to_be32(0xf0000000) -#define DESC_HDR_SEL0_AFEU __constant_cpu_to_be32(0x10000000) -#define DESC_HDR_SEL0_DEU __constant_cpu_to_be32(0x20000000) -#define DESC_HDR_SEL0_MDEUA __constant_cpu_to_be32(0x30000000) -#define DESC_HDR_SEL0_MDEUB __constant_cpu_to_be32(0xb0000000) -#define DESC_HDR_SEL0_RNG __constant_cpu_to_be32(0x40000000) -#define DESC_HDR_SEL0_PKEU __constant_cpu_to_be32(0x50000000) -#define DESC_HDR_SEL0_AESU __constant_cpu_to_be32(0x60000000) -#define DESC_HDR_SEL0_KEU __constant_cpu_to_be32(0x70000000) -#define DESC_HDR_SEL0_CRCU __constant_cpu_to_be32(0x80000000) +#define DESC_HDR_SEL0_MASK cpu_to_be32(0xf0000000) +#define DESC_HDR_SEL0_AFEU cpu_to_be32(0x10000000) +#define DESC_HDR_SEL0_DEU cpu_to_be32(0x20000000) +#define DESC_HDR_SEL0_MDEUA cpu_to_be32(0x30000000) +#define DESC_HDR_SEL0_MDEUB cpu_to_be32(0xb0000000) +#define DESC_HDR_SEL0_RNG cpu_to_be32(0x40000000) +#define DESC_HDR_SEL0_PKEU cpu_to_be32(0x50000000) +#define DESC_HDR_SEL0_AESU cpu_to_be32(0x60000000) +#define DESC_HDR_SEL0_KEU cpu_to_be32(0x70000000) +#define DESC_HDR_SEL0_CRCU cpu_to_be32(0x80000000) /* primary execution unit mode (MODE0) and derivatives */ -#define DESC_HDR_MODE0_ENCRYPT __constant_cpu_to_be32(0x00100000) -#define DESC_HDR_MODE0_AESU_CBC __constant_cpu_to_be32(0x00200000) -#define DESC_HDR_MODE0_DEU_CBC __constant_cpu_to_be32(0x00400000) -#define DESC_HDR_MODE0_DEU_3DES __constant_cpu_to_be32(0x00200000) -#define DESC_HDR_MODE0_MDEU_INIT __constant_cpu_to_be32(0x01000000) -#define DESC_HDR_MODE0_MDEU_HMAC __constant_cpu_to_be32(0x00800000) -#define DESC_HDR_MODE0_MDEU_PAD __constant_cpu_to_be32(0x00400000) -#define DESC_HDR_MODE0_MDEU_MD5 __constant_cpu_to_be32(0x00200000) -#define DESC_HDR_MODE0_MDEU_SHA256 __constant_cpu_to_be32(0x00100000) -#define DESC_HDR_MODE0_MDEU_SHA1 __constant_cpu_to_be32(0x00000000) +#define DESC_HDR_MODE0_ENCRYPT cpu_to_be32(0x00100000) +#define DESC_HDR_MODE0_AESU_CBC cpu_to_be32(0x00200000) +#define DESC_HDR_MODE0_DEU_CBC cpu_to_be32(0x00400000) +#define DESC_HDR_MODE0_DEU_3DES cpu_to_be32(0x00200000) +#define DESC_HDR_MODE0_MDEU_INIT cpu_to_be32(0x01000000) +#define DESC_HDR_MODE0_MDEU_HMAC cpu_to_be32(0x00800000) +#define DESC_HDR_MODE0_MDEU_PAD cpu_to_be32(0x00400000) +#define DESC_HDR_MODE0_MDEU_MD5 cpu_to_be32(0x00200000) +#define DESC_HDR_MODE0_MDEU_SHA256 cpu_to_be32(0x00100000) +#define DESC_HDR_MODE0_MDEU_SHA1 cpu_to_be32(0x00000000) #define DESC_HDR_MODE0_MDEU_MD5_HMAC (DESC_HDR_MODE0_MDEU_MD5 | \ DESC_HDR_MODE0_MDEU_HMAC) #define DESC_HDR_MODE0_MDEU_SHA256_HMAC (DESC_HDR_MODE0_MDEU_SHA256 | \ @@ -170,19 +170,19 @@ DESC_HDR_MODE0_MDEU_HMAC) /* secondary execution unit select (SEL1) */ -#define DESC_HDR_SEL1_MASK __constant_cpu_to_be32(0x000f0000) -#define DESC_HDR_SEL1_MDEUA __constant_cpu_to_be32(0x00030000) -#define DESC_HDR_SEL1_MDEUB __constant_cpu_to_be32(0x000b0000) -#define DESC_HDR_SEL1_CRCU __constant_cpu_to_be32(0x00080000) +#define DESC_HDR_SEL1_MASK cpu_to_be32(0x000f0000) +#define DESC_HDR_SEL1_MDEUA cpu_to_be32(0x00030000) +#define DESC_HDR_SEL1_MDEUB cpu_to_be32(0x000b0000) +#define DESC_HDR_SEL1_CRCU cpu_to_be32(0x00080000) /* secondary execution unit mode (MODE1) and derivatives */ -#define DESC_HDR_MODE1_MDEU_CICV __constant_cpu_to_be32(0x00004000) -#define DESC_HDR_MODE1_MDEU_INIT __constant_cpu_to_be32(0x00001000) -#define DESC_HDR_MODE1_MDEU_HMAC __constant_cpu_to_be32(0x00000800) -#define DESC_HDR_MODE1_MDEU_PAD __constant_cpu_to_be32(0x00000400) -#define DESC_HDR_MODE1_MDEU_MD5 __constant_cpu_to_be32(0x00000200) -#define DESC_HDR_MODE1_MDEU_SHA256 __constant_cpu_to_be32(0x00000100) -#define DESC_HDR_MODE1_MDEU_SHA1 __constant_cpu_to_be32(0x00000000) +#define DESC_HDR_MODE1_MDEU_CICV cpu_to_be32(0x00004000) +#define DESC_HDR_MODE1_MDEU_INIT cpu_to_be32(0x00001000) +#define DESC_HDR_MODE1_MDEU_HMAC cpu_to_be32(0x00000800) +#define DESC_HDR_MODE1_MDEU_PAD cpu_to_be32(0x00000400) +#define DESC_HDR_MODE1_MDEU_MD5 cpu_to_be32(0x00000200) +#define DESC_HDR_MODE1_MDEU_SHA256 cpu_to_be32(0x00000100) +#define DESC_HDR_MODE1_MDEU_SHA1 cpu_to_be32(0x00000000) #define DESC_HDR_MODE1_MDEU_MD5_HMAC (DESC_HDR_MODE1_MDEU_MD5 | \ DESC_HDR_MODE1_MDEU_HMAC) #define DESC_HDR_MODE1_MDEU_SHA256_HMAC (DESC_HDR_MODE1_MDEU_SHA256 | \ @@ -191,16 +191,16 @@ DESC_HDR_MODE1_MDEU_HMAC) /* direction of overall data flow (DIR) */ -#define DESC_HDR_DIR_INBOUND __constant_cpu_to_be32(0x00000002) +#define DESC_HDR_DIR_INBOUND cpu_to_be32(0x00000002) /* request done notification (DN) */ -#define DESC_HDR_DONE_NOTIFY __constant_cpu_to_be32(0x00000001) +#define DESC_HDR_DONE_NOTIFY cpu_to_be32(0x00000001) /* descriptor types */ -#define DESC_HDR_TYPE_AESU_CTR_NONSNOOP __constant_cpu_to_be32(0 << 3) -#define DESC_HDR_TYPE_IPSEC_ESP __constant_cpu_to_be32(1 << 3) -#define DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU __constant_cpu_to_be32(2 << 3) -#define DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU __constant_cpu_to_be32(4 << 3) +#define DESC_HDR_TYPE_AESU_CTR_NONSNOOP cpu_to_be32(0 << 3) +#define DESC_HDR_TYPE_IPSEC_ESP cpu_to_be32(1 << 3) +#define DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU cpu_to_be32(2 << 3) +#define DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU cpu_to_be32(4 << 3) /* link table extent field bits */ #define DESC_PTR_LNKTBL_JUMP 0x80 -- cgit v1.2.3 From ca38a814c6f86db0aa58884a31093d4f096e20aa Mon Sep 17 00:00:00 2001 From: Lee Nipper Date: Sat, 20 Dec 2008 17:09:25 +1100 Subject: crypto: talitos - Ack done interrupt in isr instead of tasklet Previous commit for interrupt mitigation moved the done interrupt acknowlegement from the isr to the talitos_done tasklet. This patch moves the done interrupt acknowledgement back into the isr so that done interrupts will always be acknowledged. This covers the case for acknowledging interrupts for channel done processing that has actually already been completed by the tasklet prior to fielding a pending interrupt. Signed-off-by: Lee Nipper Signed-off-by: Kim Phillips Signed-off-by: Herbert Xu --- drivers/crypto/talitos.c | 20 +++++++------------- 1 file changed, 7 insertions(+), 13 deletions(-) (limited to 'drivers') diff --git a/drivers/crypto/talitos.c b/drivers/crypto/talitos.c index c3df3b1df85..a3918c16b3d 100644 --- a/drivers/crypto/talitos.c +++ b/drivers/crypto/talitos.c @@ -330,11 +330,9 @@ static void flush_channel(struct device *dev, int ch, int error, int reset_ch) /* descriptors with their done bits set don't get the error */ rmb(); - if ((request->desc->hdr & DESC_HDR_DONE) == DESC_HDR_DONE) { + if ((request->desc->hdr & DESC_HDR_DONE) == DESC_HDR_DONE) status = 0; - /* Ack each pkt completed on channel */ - out_be32(priv->reg + TALITOS_ICR, (1 << (ch * 2))); - } else + else if (!error) break; else @@ -575,17 +573,13 @@ static irqreturn_t talitos_interrupt(int irq, void *data) isr = in_be32(priv->reg + TALITOS_ISR); isr_lo = in_be32(priv->reg + TALITOS_ISR_LO); + /* Acknowledge interrupt */ + out_be32(priv->reg + TALITOS_ICR, isr); + out_be32(priv->reg + TALITOS_ICR_LO, isr_lo); - if (unlikely((isr & ~TALITOS_ISR_CHDONE) || isr_lo)) { - /* - * Acknowledge error interrupts here. - * Done interrupts are ack'ed as part of done_task. - */ - out_be32(priv->reg + TALITOS_ICR, isr); - out_be32(priv->reg + TALITOS_ICR_LO, isr_lo); - + if (unlikely((isr & ~TALITOS_ISR_CHDONE) || isr_lo)) talitos_error((unsigned long)data, isr, isr_lo); - } else + else if (likely(isr & TALITOS_ISR_CHDONE)) { /* mask further done interrupts. */ clrbits32(priv->reg + TALITOS_IMR, TALITOS_IMR_DONE); -- cgit v1.2.3