Lines Matching full:engine
81 struct spacc_engine *engine; member
96 struct spacc_engine *engine; member
139 struct spacc_engine *engine; member
147 struct spacc_engine *engine; member
187 static inline int spacc_fifo_cmd_full(struct spacc_engine *engine) in spacc_fifo_cmd_full() argument
189 u32 fifo_stat = readl(engine->regs + SPA_FIFO_STAT_REG_OFFSET); in spacc_fifo_cmd_full()
205 return is_cipher_ctx ? ctx->engine->cipher_ctx_base + in spacc_ctx_page_addr()
206 (indx * ctx->engine->cipher_pg_sz) : in spacc_ctx_page_addr()
207 ctx->engine->hash_key_base + (indx * ctx->engine->hash_pg_sz); in spacc_ctx_page_addr()
241 unsigned indx = ctx->engine->next_ctx++; in spacc_load_ctx()
247 ctx->engine->next_ctx &= ctx->engine->fifo_sz - 1; in spacc_load_ctx()
252 ctx->engine->regs + SPA_KEY_SZ_REG_OFFSET); in spacc_load_ctx()
257 ctx->engine->regs + SPA_KEY_SZ_REG_OFFSET); in spacc_load_ctx()
274 static struct spacc_ddt *spacc_sg_to_ddt(struct spacc_engine *engine, in spacc_sg_to_ddt() argument
288 dev_err(engine->dev, "Invalid numbers of SG.\n"); in spacc_sg_to_ddt()
291 mapped_ents = dma_map_sg(engine->dev, payload, nents, dir); in spacc_sg_to_ddt()
296 ddt = dma_pool_alloc(engine->req_pool, GFP_ATOMIC, ddt_phys); in spacc_sg_to_ddt()
307 dma_unmap_sg(engine->dev, payload, nents, dir); in spacc_sg_to_ddt()
315 struct spacc_engine *engine = req->engine; in spacc_aead_make_ddts() local
328 dev_err(engine->dev, "Invalid numbers of src SG.\n"); in spacc_aead_make_ddts()
338 dev_err(engine->dev, "Invalid numbers of dst SG.\n"); in spacc_aead_make_ddts()
345 src_ddt = dma_pool_alloc(engine->req_pool, GFP_ATOMIC, &req->src_addr); in spacc_aead_make_ddts()
349 dst_ddt = dma_pool_alloc(engine->req_pool, GFP_ATOMIC, &req->dst_addr); in spacc_aead_make_ddts()
357 src_ents = dma_map_sg(engine->dev, areq->src, src_nents, in spacc_aead_make_ddts()
362 dst_ents = dma_map_sg(engine->dev, areq->dst, dst_nents, in spacc_aead_make_ddts()
366 dma_unmap_sg(engine->dev, areq->src, src_nents, in spacc_aead_make_ddts()
371 src_ents = dma_map_sg(engine->dev, areq->src, src_nents, in spacc_aead_make_ddts()
404 dma_pool_free(engine->req_pool, dst_ddt, req->dst_addr); in spacc_aead_make_ddts()
406 dma_pool_free(engine->req_pool, src_ddt, req->src_addr); in spacc_aead_make_ddts()
419 struct spacc_engine *engine = aead_ctx->generic.engine; in spacc_aead_free_ddts() local
424 dev_err(engine->dev, "Invalid numbers of src SG.\n"); in spacc_aead_free_ddts()
429 dma_unmap_sg(engine->dev, areq->src, nents, DMA_TO_DEVICE); in spacc_aead_free_ddts()
432 dev_err(engine->dev, "Invalid numbers of dst SG.\n"); in spacc_aead_free_ddts()
435 dma_unmap_sg(engine->dev, areq->dst, nents, DMA_FROM_DEVICE); in spacc_aead_free_ddts()
437 dma_unmap_sg(engine->dev, areq->src, nents, DMA_BIDIRECTIONAL); in spacc_aead_free_ddts()
439 dma_pool_free(engine->req_pool, req->src_ddt, req->src_addr); in spacc_aead_free_ddts()
440 dma_pool_free(engine->req_pool, req->dst_ddt, req->dst_addr); in spacc_aead_free_ddts()
450 dev_err(req->engine->dev, "Invalid numbers of SG.\n"); in spacc_free_ddt()
454 dma_unmap_sg(req->engine->dev, payload, nents, dir); in spacc_free_ddt()
455 dma_pool_free(req->engine->req_pool, ddt, ddt_addr); in spacc_free_ddt()
561 struct spacc_engine *engine = ctx->generic.engine; in spacc_aead_submit() local
570 writel(req->src_addr, engine->regs + SPA_SRC_PTR_REG_OFFSET); in spacc_aead_submit()
571 writel(req->dst_addr, engine->regs + SPA_DST_PTR_REG_OFFSET); in spacc_aead_submit()
572 writel(0, engine->regs + SPA_OFFSET_REG_OFFSET); in spacc_aead_submit()
584 writel(proc_len, engine->regs + SPA_PROC_LEN_REG_OFFSET); in spacc_aead_submit()
585 writel(assoc_len, engine->regs + SPA_AAD_LEN_REG_OFFSET); in spacc_aead_submit()
586 writel(authsize, engine->regs + SPA_ICV_LEN_REG_OFFSET); in spacc_aead_submit()
587 writel(0, engine->regs + SPA_ICV_OFFSET_REG_OFFSET); in spacc_aead_submit()
588 writel(0, engine->regs + SPA_AUX_INFO_REG_OFFSET); in spacc_aead_submit()
597 mod_timer(&engine->packet_timeout, jiffies + PACKET_TIMEOUT); in spacc_aead_submit()
599 writel(ctrl, engine->regs + SPA_CTRL_REG_OFFSET); in spacc_aead_submit()
606 static void spacc_push(struct spacc_engine *engine) in spacc_push() argument
610 while (!list_empty(&engine->pending) && in spacc_push()
611 engine->in_flight + 1 <= engine->fifo_sz) { in spacc_push()
613 ++engine->in_flight; in spacc_push()
614 req = list_first_entry(&engine->pending, struct spacc_req, in spacc_push()
616 list_move_tail(&req->list, &engine->in_progress); in spacc_push()
623 * Setup an AEAD request for processing. This will configure the engine, load
631 struct spacc_engine *engine = to_spacc_aead(alg)->engine; in spacc_aead_setup() local
639 dev_req->engine = engine; in spacc_aead_setup()
650 spin_lock_irqsave(&engine->hw_lock, flags); in spacc_aead_setup()
651 if (unlikely(spacc_fifo_cmd_full(engine)) || in spacc_aead_setup()
652 engine->in_flight + 1 > engine->fifo_sz) { in spacc_aead_setup()
655 spin_unlock_irqrestore(&engine->hw_lock, flags); in spacc_aead_setup()
658 list_add_tail(&dev_req->list, &engine->pending); in spacc_aead_setup()
660 list_add_tail(&dev_req->list, &engine->pending); in spacc_aead_setup()
661 spacc_push(engine); in spacc_aead_setup()
663 spin_unlock_irqrestore(&engine->hw_lock, flags); in spacc_aead_setup()
698 struct spacc_engine *engine = spacc_alg->engine; in spacc_aead_cra_init() local
701 ctx->generic.engine = engine; in spacc_aead_cra_init()
784 * IPSec engine only supports 128 and 256 bit AES keys. If we get a in spacc_aes_setkey()
871 struct spacc_engine *engine = ctx->generic.engine; in spacc_ablk_submit() local
878 writel(req->src_addr, engine->regs + SPA_SRC_PTR_REG_OFFSET); in spacc_ablk_submit()
879 writel(req->dst_addr, engine->regs + SPA_DST_PTR_REG_OFFSET); in spacc_ablk_submit()
880 writel(0, engine->regs + SPA_OFFSET_REG_OFFSET); in spacc_ablk_submit()
882 writel(ablk_req->cryptlen, engine->regs + SPA_PROC_LEN_REG_OFFSET); in spacc_ablk_submit()
883 writel(0, engine->regs + SPA_ICV_OFFSET_REG_OFFSET); in spacc_ablk_submit()
884 writel(0, engine->regs + SPA_AUX_INFO_REG_OFFSET); in spacc_ablk_submit()
885 writel(0, engine->regs + SPA_AAD_LEN_REG_OFFSET); in spacc_ablk_submit()
891 mod_timer(&engine->packet_timeout, jiffies + PACKET_TIMEOUT); in spacc_ablk_submit()
893 writel(ctrl, engine->regs + SPA_CTRL_REG_OFFSET); in spacc_ablk_submit()
928 struct spacc_engine *engine = to_spacc_skcipher(alg)->engine; in spacc_ablk_setup() local
935 dev_req->engine = engine; in spacc_ablk_setup()
943 * Create the DDT's for the engine. If we share the same source and in spacc_ablk_setup()
947 dev_req->src_ddt = spacc_sg_to_ddt(engine, req->src, in spacc_ablk_setup()
952 dev_req->dst_ddt = spacc_sg_to_ddt(engine, req->dst, in spacc_ablk_setup()
957 dev_req->dst_ddt = spacc_sg_to_ddt(engine, req->dst, in spacc_ablk_setup()
967 spin_lock_irqsave(&engine->hw_lock, flags); in spacc_ablk_setup()
969 * Check if the engine will accept the operation now. If it won't then in spacc_ablk_setup()
973 if (unlikely(spacc_fifo_cmd_full(engine)) || in spacc_ablk_setup()
974 engine->in_flight + 1 > engine->fifo_sz) { in spacc_ablk_setup()
977 spin_unlock_irqrestore(&engine->hw_lock, flags); in spacc_ablk_setup()
980 list_add_tail(&dev_req->list, &engine->pending); in spacc_ablk_setup()
982 list_add_tail(&dev_req->list, &engine->pending); in spacc_ablk_setup()
983 spacc_push(engine); in spacc_ablk_setup()
985 spin_unlock_irqrestore(&engine->hw_lock, flags); in spacc_ablk_setup()
1006 struct spacc_engine *engine = spacc_alg->engine; in spacc_ablk_init_tfm() local
1009 ctx->generic.engine = engine; in spacc_ablk_init_tfm()
1014 dev_warn(engine->dev, "failed to allocate fallback for %s\n", in spacc_ablk_init_tfm()
1057 static inline int spacc_fifo_stat_empty(struct spacc_engine *engine) in spacc_fifo_stat_empty() argument
1059 return readl(engine->regs + SPA_FIFO_STAT_REG_OFFSET) & in spacc_fifo_stat_empty()
1063 static void spacc_process_done(struct spacc_engine *engine) in spacc_process_done() argument
1068 spin_lock_irqsave(&engine->hw_lock, flags); in spacc_process_done()
1070 while (!spacc_fifo_stat_empty(engine)) { in spacc_process_done()
1071 req = list_first_entry(&engine->in_progress, struct spacc_req, in spacc_process_done()
1073 list_move_tail(&req->list, &engine->completed); in spacc_process_done()
1074 --engine->in_flight; in spacc_process_done()
1077 writel(~0, engine->regs + SPA_STAT_POP_REG_OFFSET); in spacc_process_done()
1078 req->result = (readl(engine->regs + SPA_STATUS_REG_OFFSET) & in spacc_process_done()
1092 dev_warn(engine->dev, in spacc_process_done()
1098 dev_warn(engine->dev, in spacc_process_done()
1106 tasklet_schedule(&engine->complete); in spacc_process_done()
1108 spin_unlock_irqrestore(&engine->hw_lock, flags); in spacc_process_done()
1113 struct spacc_engine *engine = (struct spacc_engine *)dev; in spacc_spacc_irq() local
1114 u32 spacc_irq_stat = readl(engine->regs + SPA_IRQ_STAT_REG_OFFSET); in spacc_spacc_irq()
1116 writel(spacc_irq_stat, engine->regs + SPA_IRQ_STAT_REG_OFFSET); in spacc_spacc_irq()
1117 spacc_process_done(engine); in spacc_spacc_irq()
1124 struct spacc_engine *engine = from_timer(engine, t, packet_timeout); in spacc_packet_timeout() local
1126 spacc_process_done(engine); in spacc_packet_timeout()
1141 struct spacc_engine *engine = (struct spacc_engine *)data; in spacc_spacc_complete() local
1146 spin_lock_irqsave(&engine->hw_lock, flags); in spacc_spacc_complete()
1148 list_splice_init(&engine->completed, &completed); in spacc_spacc_complete()
1149 spacc_push(engine); in spacc_spacc_complete()
1150 if (engine->in_flight) in spacc_spacc_complete()
1151 mod_timer(&engine->packet_timeout, jiffies + PACKET_TIMEOUT); in spacc_spacc_complete()
1153 spin_unlock_irqrestore(&engine->hw_lock, flags); in spacc_spacc_complete()
1164 struct spacc_engine *engine = dev_get_drvdata(dev); in spacc_suspend() local
1171 clk_disable(engine->clk); in spacc_suspend()
1178 struct spacc_engine *engine = dev_get_drvdata(dev); in spacc_resume() local
1180 return clk_enable(engine->clk); in spacc_resume()
1198 struct spacc_engine *engine = spacc_dev_to_engine(dev); in spacc_stat_irq_thresh_show() local
1200 return snprintf(buf, PAGE_SIZE, "%u\n", engine->stat_irq_thresh); in spacc_stat_irq_thresh_show()
1207 struct spacc_engine *engine = spacc_dev_to_engine(dev); in spacc_stat_irq_thresh_store() local
1213 thresh = clamp(thresh, 1UL, engine->fifo_sz - 1); in spacc_stat_irq_thresh_store()
1215 engine->stat_irq_thresh = thresh; in spacc_stat_irq_thresh_store()
1216 writel(engine->stat_irq_thresh << SPA_IRQ_CTRL_STAT_CNT_OFFSET, in spacc_stat_irq_thresh_store()
1217 engine->regs + SPA_IRQ_CTRL_REG_OFFSET); in spacc_stat_irq_thresh_store()
1614 struct spacc_engine *engine = devm_kzalloc(&pdev->dev, sizeof(*engine), in spacc_probe() local
1616 if (!engine) in spacc_probe()
1620 engine->max_ctxs = SPACC_CRYPTO_IPSEC_MAX_CTXS; in spacc_probe()
1621 engine->cipher_pg_sz = SPACC_CRYPTO_IPSEC_CIPHER_PG_SZ; in spacc_probe()
1622 engine->hash_pg_sz = SPACC_CRYPTO_IPSEC_HASH_PG_SZ; in spacc_probe()
1623 engine->fifo_sz = SPACC_CRYPTO_IPSEC_FIFO_SZ; in spacc_probe()
1624 engine->algs = ipsec_engine_algs; in spacc_probe()
1625 engine->num_algs = ARRAY_SIZE(ipsec_engine_algs); in spacc_probe()
1626 engine->aeads = ipsec_engine_aeads; in spacc_probe()
1627 engine->num_aeads = ARRAY_SIZE(ipsec_engine_aeads); in spacc_probe()
1629 engine->max_ctxs = SPACC_CRYPTO_L2_MAX_CTXS; in spacc_probe()
1630 engine->cipher_pg_sz = SPACC_CRYPTO_L2_CIPHER_PG_SZ; in spacc_probe()
1631 engine->hash_pg_sz = SPACC_CRYPTO_L2_HASH_PG_SZ; in spacc_probe()
1632 engine->fifo_sz = SPACC_CRYPTO_L2_FIFO_SZ; in spacc_probe()
1633 engine->algs = l2_engine_algs; in spacc_probe()
1634 engine->num_algs = ARRAY_SIZE(l2_engine_algs); in spacc_probe()
1639 engine->name = dev_name(&pdev->dev); in spacc_probe()
1641 engine->regs = devm_platform_ioremap_resource(pdev, 0); in spacc_probe()
1642 if (IS_ERR(engine->regs)) in spacc_probe()
1643 return PTR_ERR(engine->regs); in spacc_probe()
1647 dev_err(&pdev->dev, "no memory/irq resource for engine\n"); in spacc_probe()
1651 tasklet_init(&engine->complete, spacc_spacc_complete, in spacc_probe()
1652 (unsigned long)engine); in spacc_probe()
1655 &engine->complete); in spacc_probe()
1660 engine->name, engine)) { in spacc_probe()
1661 dev_err(engine->dev, "failed to request IRQ\n"); in spacc_probe()
1665 engine->dev = &pdev->dev; in spacc_probe()
1666 engine->cipher_ctx_base = engine->regs + SPA_CIPH_KEY_BASE_REG_OFFSET; in spacc_probe()
1667 engine->hash_key_base = engine->regs + SPA_HASH_KEY_BASE_REG_OFFSET; in spacc_probe()
1669 engine->req_pool = dmam_pool_create(engine->name, engine->dev, in spacc_probe()
1671 if (!engine->req_pool) in spacc_probe()
1674 spin_lock_init(&engine->hw_lock); in spacc_probe()
1676 engine->clk = clk_get(&pdev->dev, "ref"); in spacc_probe()
1677 if (IS_ERR(engine->clk)) { in spacc_probe()
1679 return PTR_ERR(engine->clk); in spacc_probe()
1682 if (clk_prepare_enable(engine->clk)) { in spacc_probe()
1693 engine->stat_irq_thresh = (engine->fifo_sz / 2); in spacc_probe()
1704 writel(engine->stat_irq_thresh << SPA_IRQ_CTRL_STAT_CNT_OFFSET, in spacc_probe()
1705 engine->regs + SPA_IRQ_CTRL_REG_OFFSET); in spacc_probe()
1707 engine->regs + SPA_IRQ_EN_REG_OFFSET); in spacc_probe()
1709 timer_setup(&engine->packet_timeout, spacc_packet_timeout, 0); in spacc_probe()
1711 INIT_LIST_HEAD(&engine->pending); in spacc_probe()
1712 INIT_LIST_HEAD(&engine->completed); in spacc_probe()
1713 INIT_LIST_HEAD(&engine->in_progress); in spacc_probe()
1714 engine->in_flight = 0; in spacc_probe()
1716 platform_set_drvdata(pdev, engine); in spacc_probe()
1719 INIT_LIST_HEAD(&engine->registered_algs); in spacc_probe()
1720 for (i = 0; i < engine->num_algs; ++i) { in spacc_probe()
1721 engine->algs[i].engine = engine; in spacc_probe()
1722 err = crypto_register_skcipher(&engine->algs[i].alg); in spacc_probe()
1724 list_add_tail(&engine->algs[i].entry, in spacc_probe()
1725 &engine->registered_algs); in spacc_probe()
1729 dev_err(engine->dev, "failed to register alg \"%s\"\n", in spacc_probe()
1730 engine->algs[i].alg.base.cra_name); in spacc_probe()
1732 dev_dbg(engine->dev, "registered alg \"%s\"\n", in spacc_probe()
1733 engine->algs[i].alg.base.cra_name); in spacc_probe()
1736 INIT_LIST_HEAD(&engine->registered_aeads); in spacc_probe()
1737 for (i = 0; i < engine->num_aeads; ++i) { in spacc_probe()
1738 engine->aeads[i].engine = engine; in spacc_probe()
1739 err = crypto_register_aead(&engine->aeads[i].alg); in spacc_probe()
1741 list_add_tail(&engine->aeads[i].entry, in spacc_probe()
1742 &engine->registered_aeads); in spacc_probe()
1746 dev_err(engine->dev, "failed to register alg \"%s\"\n", in spacc_probe()
1747 engine->aeads[i].alg.base.cra_name); in spacc_probe()
1749 dev_dbg(engine->dev, "registered alg \"%s\"\n", in spacc_probe()
1750 engine->aeads[i].alg.base.cra_name); in spacc_probe()
1756 del_timer_sync(&engine->packet_timeout); in spacc_probe()
1759 clk_disable_unprepare(engine->clk); in spacc_probe()
1761 clk_put(engine->clk); in spacc_probe()
1770 struct spacc_engine *engine = platform_get_drvdata(pdev); in spacc_remove() local
1772 del_timer_sync(&engine->packet_timeout); in spacc_remove()
1775 list_for_each_entry_safe(aead, an, &engine->registered_aeads, entry) { in spacc_remove()
1780 list_for_each_entry_safe(alg, next, &engine->registered_algs, entry) { in spacc_remove()
1785 clk_disable_unprepare(engine->clk); in spacc_remove()
1786 clk_put(engine->clk); in spacc_remove()