107 	struct mtk_cryp *cryp;  member
153 static inline u32 mtk_aes_read(struct mtk_cryp *cryp, u32 offset)  in mtk_aes_read()  argument
155 	return readl_relaxed(cryp->base + offset);  in mtk_aes_read()
158 static inline void mtk_aes_write(struct mtk_cryp *cryp,  in mtk_aes_write()  argument
161 	writel_relaxed(value, cryp->base + offset);  in mtk_aes_write()
166 	struct mtk_cryp *cryp = NULL;  in mtk_aes_find_dev()  local
170 	if (!ctx->cryp) {  in mtk_aes_find_dev()
172 			cryp = tmp;  in mtk_aes_find_dev()
175 		ctx->cryp = cryp;  in mtk_aes_find_dev()
177 		cryp = ctx->cryp;  in mtk_aes_find_dev()
181 	return cryp;  in mtk_aes_find_dev()
245 static inline int mtk_aes_complete(struct mtk_cryp *cryp,  in mtk_aes_complete()  argument
260 static int mtk_aes_xmit(struct mtk_cryp *cryp, struct mtk_aes_rec *aes)  in mtk_aes_xmit()  argument
262 	struct mtk_ring *ring = cryp->ring[aes->id];  in mtk_aes_xmit()
316 	mtk_aes_write(cryp, RDR_PREP_COUNT(aes->id), MTK_DESC_CNT(dlen));  in mtk_aes_xmit()
317 	mtk_aes_write(cryp, CDR_PREP_COUNT(aes->id), MTK_DESC_CNT(slen));  in mtk_aes_xmit()
322 static void mtk_aes_unmap(struct mtk_cryp *cryp, struct mtk_aes_rec *aes)  in mtk_aes_unmap()  argument
326 	dma_unmap_single(cryp->dev, ctx->ct_dma, sizeof(ctx->info),  in mtk_aes_unmap()
330 		dma_unmap_sg(cryp->dev, aes->src.sg, aes->src.nents,  in mtk_aes_unmap()
336 		dma_unmap_sg(cryp->dev, aes->dst.sg, aes->dst.nents,  in mtk_aes_unmap()
342 		dma_unmap_sg(cryp->dev, aes->src.sg, aes->src.nents,  in mtk_aes_unmap()
354 static int mtk_aes_map(struct mtk_cryp *cryp, struct mtk_aes_rec *aes)  in mtk_aes_map()  argument
359 	ctx->ct_dma = dma_map_single(cryp->dev, info, sizeof(*info),  in mtk_aes_map()
361 	if (unlikely(dma_mapping_error(cryp->dev, ctx->ct_dma)))  in mtk_aes_map()
367 		aes->src.sg_len = dma_map_sg(cryp->dev, aes->src.sg,  in mtk_aes_map()
374 		aes->src.sg_len = dma_map_sg(cryp->dev, aes->src.sg,  in mtk_aes_map()
379 		aes->dst.sg_len = dma_map_sg(cryp->dev, aes->dst.sg,  in mtk_aes_map()
382 			dma_unmap_sg(cryp->dev, aes->src.sg, aes->src.nents,  in mtk_aes_map()
388 	return mtk_aes_xmit(cryp, aes);  in mtk_aes_map()
391 	dma_unmap_single(cryp->dev, ctx->ct_dma, sizeof(*info), DMA_TO_DEVICE);  in mtk_aes_map()
393 	return mtk_aes_complete(cryp, aes, -EINVAL);  in mtk_aes_map()
397 static void mtk_aes_info_init(struct mtk_cryp *cryp, struct mtk_aes_rec *aes,  in mtk_aes_info_init()  argument
446 static int mtk_aes_dma(struct mtk_cryp *cryp, struct mtk_aes_rec *aes,  in mtk_aes_dma()  argument
468 			return mtk_aes_complete(cryp, aes, -ENOMEM);  in mtk_aes_dma()
487 	mtk_aes_info_init(cryp, aes, len + padlen);  in mtk_aes_dma()
489 	return mtk_aes_map(cryp, aes);  in mtk_aes_dma()
492 static int mtk_aes_handle_queue(struct mtk_cryp *cryp, u8 id,  in mtk_aes_handle_queue()  argument
495 	struct mtk_aes_rec *aes = cryp->aes[id];  in mtk_aes_handle_queue()
527 	return ctx->start(cryp, aes);  in mtk_aes_handle_queue()
530 static int mtk_aes_transfer_complete(struct mtk_cryp *cryp,  in mtk_aes_transfer_complete()  argument
533 	return mtk_aes_complete(cryp, aes, 0);  in mtk_aes_transfer_complete()
536 static int mtk_aes_start(struct mtk_cryp *cryp, struct mtk_aes_rec *aes)  in mtk_aes_start()  argument
544 	return mtk_aes_dma(cryp, aes, req->src, req->dst, req->cryptlen);  in mtk_aes_start()
553 static int mtk_aes_ctr_transfer(struct mtk_cryp *cryp, struct mtk_aes_rec *aes)  in mtk_aes_ctr_transfer()  argument
566 		return mtk_aes_transfer_complete(cryp, aes);  in mtk_aes_ctr_transfer()
599 	return mtk_aes_dma(cryp, aes, src, dst, datalen);  in mtk_aes_ctr_transfer()
602 static int mtk_aes_ctr_start(struct mtk_cryp *cryp, struct mtk_aes_rec *aes)  in mtk_aes_ctr_start()  argument
615 	return mtk_aes_ctr_transfer(cryp, aes);  in mtk_aes_ctr_start()
650 	struct mtk_cryp *cryp;  in mtk_aes_crypt()  local
652 	cryp = mtk_aes_find_dev(ctx);  in mtk_aes_crypt()
653 	if (!cryp)  in mtk_aes_crypt()
659 	return mtk_aes_handle_queue(cryp, !(mode & AES_FLAGS_ENCRYPT),  in mtk_aes_crypt()
831 static int mtk_aes_gcm_tag_verify(struct mtk_cryp *cryp,  in mtk_aes_gcm_tag_verify()  argument
834 	__le32 status = cryp->ring[aes->id]->res_prev->ct;  in mtk_aes_gcm_tag_verify()
836 	return mtk_aes_complete(cryp, aes, (status & AES_AUTH_TAG_ERR) ?  in mtk_aes_gcm_tag_verify()
841 static void mtk_aes_gcm_info_init(struct mtk_cryp *cryp,  in mtk_aes_gcm_info_init()  argument
879 static int mtk_aes_gcm_dma(struct mtk_cryp *cryp, struct mtk_aes_rec *aes,  in mtk_aes_gcm_dma()  argument
897 			return mtk_aes_complete(cryp, aes, -ENOMEM);  in mtk_aes_gcm_dma()
916 	mtk_aes_gcm_info_init(cryp, aes, len);  in mtk_aes_gcm_dma()
918 	return mtk_aes_map(cryp, aes);  in mtk_aes_gcm_dma()
922 static int mtk_aes_gcm_start(struct mtk_cryp *cryp, struct mtk_aes_rec *aes)  in mtk_aes_gcm_start()  argument
944 	return mtk_aes_gcm_dma(cryp, aes, req->src, req->dst, len);  in mtk_aes_gcm_start()
952 	struct mtk_cryp *cryp;  in mtk_aes_gcm_crypt()  local
955 	cryp = mtk_aes_find_dev(ctx);  in mtk_aes_gcm_crypt()
956 	if (!cryp)  in mtk_aes_gcm_crypt()
968 	return mtk_aes_handle_queue(cryp, enc, &req->base);  in mtk_aes_gcm_crypt()
1087 	mtk_aes_handle_queue(aes->cryp, aes->id, NULL);  in mtk_aes_queue_task()
1093 	struct mtk_cryp *cryp = aes->cryp;  in mtk_aes_done_task()  local
1095 	mtk_aes_unmap(cryp, aes);  in mtk_aes_done_task()
1096 	aes->resume(cryp, aes);  in mtk_aes_done_task()
1102 	struct mtk_cryp *cryp = aes->cryp;  in mtk_aes_irq()  local
1103 	u32 val = mtk_aes_read(cryp, RDR_STAT(aes->id));  in mtk_aes_irq()
1105 	mtk_aes_write(cryp, RDR_STAT(aes->id), val);  in mtk_aes_irq()
1108 		mtk_aes_write(cryp, RDR_PROC_COUNT(aes->id), MTK_CNT_RST);  in mtk_aes_irq()
1109 		mtk_aes_write(cryp, RDR_THRESH(aes->id),  in mtk_aes_irq()
1114 		dev_warn(cryp->dev, "AES interrupt when no active requests.\n");  in mtk_aes_irq()
1125 static int mtk_aes_record_init(struct mtk_cryp *cryp)  in mtk_aes_record_init()  argument
1127 	struct mtk_aes_rec **aes = cryp->aes;  in mtk_aes_record_init()
1140 		aes[i]->cryp = cryp;  in mtk_aes_record_init()
1166 static void mtk_aes_record_free(struct mtk_cryp *cryp)  in mtk_aes_record_free()  argument
1171 		tasklet_kill(&cryp->aes[i]->done_task);  in mtk_aes_record_free()
1172 		tasklet_kill(&cryp->aes[i]->queue_task);  in mtk_aes_record_free()
1174 		free_page((unsigned long)cryp->aes[i]->buf);  in mtk_aes_record_free()
1175 		kfree(cryp->aes[i]);  in mtk_aes_record_free()
1212 int mtk_cipher_alg_register(struct mtk_cryp *cryp)  in mtk_cipher_alg_register()  argument
1216 	INIT_LIST_HEAD(&cryp->aes_list);  in mtk_cipher_alg_register()
1219 	ret = mtk_aes_record_init(cryp);  in mtk_cipher_alg_register()
1223 	ret = devm_request_irq(cryp->dev, cryp->irq[MTK_RING0], mtk_aes_irq,  in mtk_cipher_alg_register()
1224 			       0, "mtk-aes", cryp->aes[0]);  in mtk_cipher_alg_register()
1226 		dev_err(cryp->dev, "unable to request AES irq.\n");  in mtk_cipher_alg_register()
1230 	ret = devm_request_irq(cryp->dev, cryp->irq[MTK_RING1], mtk_aes_irq,  in mtk_cipher_alg_register()
1231 			       0, "mtk-aes", cryp->aes[1]);  in mtk_cipher_alg_register()
1233 		dev_err(cryp->dev, "unable to request AES irq.\n");  in mtk_cipher_alg_register()
1238 	mtk_aes_write(cryp, AIC_ENABLE_SET(MTK_RING0), MTK_IRQ_RDR0);  in mtk_cipher_alg_register()
1239 	mtk_aes_write(cryp, AIC_ENABLE_SET(MTK_RING1), MTK_IRQ_RDR1);  in mtk_cipher_alg_register()
1242 	list_add_tail(&cryp->aes_list, &mtk_aes.dev_list);  in mtk_cipher_alg_register()
1253 	list_del(&cryp->aes_list);  in mtk_cipher_alg_register()
1256 	mtk_aes_record_free(cryp);  in mtk_cipher_alg_register()
1259 	dev_err(cryp->dev, "mtk-aes initialization failed.\n");  in mtk_cipher_alg_register()
1263 void mtk_cipher_alg_release(struct mtk_cryp *cryp)  in mtk_cipher_alg_release()  argument
1266 	list_del(&cryp->aes_list);  in mtk_cipher_alg_release()
1270 	mtk_aes_record_free(cryp);  in mtk_cipher_alg_release()