107 	struct device *jrdev;  member
125 	void (*ahash_op_done)(struct device *jrdev, u32 *desc, u32 err,
146 static inline int map_seq_out_ptr_ctx(u32 *desc, struct device *jrdev,  in map_seq_out_ptr_ctx()  argument
151 	state->ctx_dma = dma_map_single(jrdev, state->caam_ctx,  in map_seq_out_ptr_ctx()
153 	if (dma_mapping_error(jrdev, state->ctx_dma)) {  in map_seq_out_ptr_ctx()
154 		dev_err(jrdev, "unable to map ctx\n");  in map_seq_out_ptr_ctx()
165 static inline int buf_map_to_sec4_sg(struct device *jrdev,  in buf_map_to_sec4_sg()  argument
174 	state->buf_dma = dma_map_single(jrdev, state->buf, buflen,  in buf_map_to_sec4_sg()
176 	if (dma_mapping_error(jrdev, state->buf_dma)) {  in buf_map_to_sec4_sg()
177 		dev_err(jrdev, "unable to map buf\n");  in buf_map_to_sec4_sg()
188 static inline int ctx_map_to_sec4_sg(struct device *jrdev,  in ctx_map_to_sec4_sg()  argument
193 	state->ctx_dma = dma_map_single(jrdev, state->caam_ctx, ctx_len, flag);  in ctx_map_to_sec4_sg()
194 	if (dma_mapping_error(jrdev, state->ctx_dma)) {  in ctx_map_to_sec4_sg()
195 		dev_err(jrdev, "unable to map ctx\n");  in ctx_map_to_sec4_sg()
209 	struct device *jrdev = ctx->jrdev;  in ahash_set_sh_desc()  local
210 	struct caam_drv_private *ctrlpriv = dev_get_drvdata(jrdev->parent);  in ahash_set_sh_desc()
219 	dma_sync_single_for_device(jrdev, ctx->sh_desc_update_dma,  in ahash_set_sh_desc()
230 	dma_sync_single_for_device(jrdev, ctx->sh_desc_update_first_dma,  in ahash_set_sh_desc()
240 	dma_sync_single_for_device(jrdev, ctx->sh_desc_fin_dma,  in ahash_set_sh_desc()
251 	dma_sync_single_for_device(jrdev, ctx->sh_desc_digest_dma,  in ahash_set_sh_desc()
265 	struct device *jrdev = ctx->jrdev;  in axcbc_set_sh_desc()  local
272 	dma_sync_single_for_device(jrdev, ctx->sh_desc_update_dma,  in axcbc_set_sh_desc()
282 	dma_sync_single_for_device(jrdev, ctx->sh_desc_fin_dma,  in axcbc_set_sh_desc()
295 	dma_sync_single_for_device(jrdev, ctx->sh_desc_update_first_dma,  in axcbc_set_sh_desc()
305 	dma_sync_single_for_device(jrdev, ctx->sh_desc_digest_dma,  in axcbc_set_sh_desc()
317 	struct device *jrdev = ctx->jrdev;  in acmac_set_sh_desc()  local
324 	dma_sync_single_for_device(jrdev, ctx->sh_desc_update_dma,  in acmac_set_sh_desc()
334 	dma_sync_single_for_device(jrdev, ctx->sh_desc_fin_dma,  in acmac_set_sh_desc()
344 	dma_sync_single_for_device(jrdev, ctx->sh_desc_update_first_dma,  in acmac_set_sh_desc()
354 	dma_sync_single_for_device(jrdev, ctx->sh_desc_digest_dma,  in acmac_set_sh_desc()
367 	struct device *jrdev = ctx->jrdev;  in hash_digest_key()  local
379 	key_dma = dma_map_single(jrdev, key, *keylen, DMA_BIDIRECTIONAL);  in hash_digest_key()
380 	if (dma_mapping_error(jrdev, key_dma)) {  in hash_digest_key()
381 		dev_err(jrdev, "unable to map key memory\n");  in hash_digest_key()
405 	ret = caam_jr_enqueue(jrdev, desc, split_key_done, &result);  in hash_digest_key()
415 	dma_unmap_single(jrdev, key_dma, *keylen, DMA_BIDIRECTIONAL);  in hash_digest_key()
428 	struct device *jrdev = ctx->jrdev;  in ahash_setkey()  local
431 	struct caam_drv_private *ctrlpriv = dev_get_drvdata(ctx->jrdev->parent);  in ahash_setkey()
435 	dev_dbg(jrdev, "keylen %d\n", keylen);  in ahash_setkey()
476 			dma_sync_single_for_device(ctx->jrdev,  in ahash_setkey()
481 		ret = gen_split_key(ctx->jrdev, ctx->key, &ctx->adata, key,  in ahash_setkey()
498 	struct device *jrdev = ctx->jrdev;  in axcbc_setkey()  local
504 	dma_sync_single_for_device(jrdev, ctx->adata.key_dma, keylen,  in axcbc_setkey()
585 static inline void ahash_done_cpy(struct device *jrdev, u32 *desc, u32 err,  in ahash_done_cpy()  argument
589 	struct caam_drv_private_jr *jrp = dev_get_drvdata(jrdev);  in ahash_done_cpy()
598 	dev_dbg(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);  in ahash_done_cpy()
604 		ecode = caam_jr_strstatus(jrdev, err);  in ahash_done_cpy()
606 	ahash_unmap_ctx(jrdev, edesc, req, digestsize, dir);  in ahash_done_cpy()
624 static void ahash_done(struct device *jrdev, u32 *desc, u32 err,  in ahash_done()  argument
627 	ahash_done_cpy(jrdev, desc, err, context, DMA_FROM_DEVICE);  in ahash_done()
630 static void ahash_done_ctx_src(struct device *jrdev, u32 *desc, u32 err,  in ahash_done_ctx_src()  argument
633 	ahash_done_cpy(jrdev, desc, err, context, DMA_BIDIRECTIONAL);  in ahash_done_ctx_src()
636 static inline void ahash_done_switch(struct device *jrdev, u32 *desc, u32 err,  in ahash_done_switch()  argument
640 	struct caam_drv_private_jr *jrp = dev_get_drvdata(jrdev);  in ahash_done_switch()
649 	dev_dbg(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);  in ahash_done_switch()
654 		ecode = caam_jr_strstatus(jrdev, err);  in ahash_done_switch()
656 	ahash_unmap_ctx(jrdev, edesc, req, ctx->ctx_len, dir);  in ahash_done_switch()
687 static void ahash_done_bi(struct device *jrdev, u32 *desc, u32 err,  in ahash_done_bi()  argument
690 	ahash_done_switch(jrdev, desc, err, context, DMA_BIDIRECTIONAL);  in ahash_done_bi()
693 static void ahash_done_ctx_dst(struct device *jrdev, u32 *desc, u32 err,  in ahash_done_ctx_dst()  argument
696 	ahash_done_switch(jrdev, desc, err, context, DMA_FROM_DEVICE);  in ahash_done_ctx_dst()
741 		src_dma = dma_map_single(ctx->jrdev, sg, sgsize, DMA_TO_DEVICE);  in ahash_edesc_add_src()
742 		if (dma_mapping_error(ctx->jrdev, src_dma)) {  in ahash_edesc_add_src()
743 			dev_err(ctx->jrdev, "unable to map S/G table\n");  in ahash_edesc_add_src()
766 	struct device *jrdev = ctx->jrdev;  in ahash_do_one_req()  local
772 	ret = caam_jr_enqueue(jrdev, desc, state->ahash_op_done, req);  in ahash_do_one_req()
778 		ahash_unmap(jrdev, state->edesc, req, 0);  in ahash_do_one_req()
787 static int ahash_enqueue_req(struct device *jrdev,  in ahash_enqueue_req()  argument
788 			     void (*cbk)(struct device *jrdev, u32 *desc,  in ahash_enqueue_req()  argument
793 	struct caam_drv_private_jr *jrpriv = dev_get_drvdata(jrdev);  in ahash_enqueue_req()
810 		ret = caam_jr_enqueue(jrdev, desc, cbk, req);  in ahash_enqueue_req()
813 		ahash_unmap_ctx(jrdev, edesc, req, dst_len, dir);  in ahash_enqueue_req()
826 	struct device *jrdev = ctx->jrdev;  in ahash_update_ctx()  local
857 			dev_err(jrdev, "Invalid number of src SG.\n");  in ahash_update_ctx()
862 			mapped_nents = dma_map_sg(jrdev, req->src, src_nents,  in ahash_update_ctx()
865 				dev_err(jrdev, "unable to DMA map source\n");  in ahash_update_ctx()
883 			dma_unmap_sg(jrdev, req->src, src_nents, DMA_TO_DEVICE);  in ahash_update_ctx()
890 		ret = ctx_map_to_sec4_sg(jrdev, state, ctx->ctx_len,  in ahash_update_ctx()
895 		ret = buf_map_to_sec4_sg(jrdev, edesc->sec4_sg + 1, state);  in ahash_update_ctx()
909 		edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg,  in ahash_update_ctx()
912 		if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) {  in ahash_update_ctx()
913 			dev_err(jrdev, "unable to map S/G table\n");  in ahash_update_ctx()
927 		ret = ahash_enqueue_req(jrdev, ahash_done_bi, req,  in ahash_update_ctx()
941 	ahash_unmap_ctx(jrdev, edesc, req, ctx->ctx_len, DMA_BIDIRECTIONAL);  in ahash_update_ctx()
951 	struct device *jrdev = ctx->jrdev;  in ahash_final_ctx()  local
972 	ret = ctx_map_to_sec4_sg(jrdev, state, ctx->ctx_len,  in ahash_final_ctx()
977 	ret = buf_map_to_sec4_sg(jrdev, edesc->sec4_sg + 1, state);  in ahash_final_ctx()
983 	edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg,  in ahash_final_ctx()
985 	if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) {  in ahash_final_ctx()
986 		dev_err(jrdev, "unable to map S/G table\n");  in ahash_final_ctx()
999 	return ahash_enqueue_req(jrdev, ahash_done_ctx_src, req,  in ahash_final_ctx()
1002 	ahash_unmap_ctx(jrdev, edesc, req, digestsize, DMA_BIDIRECTIONAL);  in ahash_final_ctx()
1012 	struct device *jrdev = ctx->jrdev;  in ahash_finup_ctx()  local
1023 		dev_err(jrdev, "Invalid number of src SG.\n");  in ahash_finup_ctx()
1028 		mapped_nents = dma_map_sg(jrdev, req->src, src_nents,  in ahash_finup_ctx()
1031 			dev_err(jrdev, "unable to DMA map source\n");  in ahash_finup_ctx()
1044 		dma_unmap_sg(jrdev, req->src, src_nents, DMA_TO_DEVICE);  in ahash_finup_ctx()
1052 	ret = ctx_map_to_sec4_sg(jrdev, state, ctx->ctx_len,  in ahash_finup_ctx()
1057 	ret = buf_map_to_sec4_sg(jrdev, edesc->sec4_sg + 1, state);  in ahash_finup_ctx()
1073 	return ahash_enqueue_req(jrdev, ahash_done_ctx_src, req,  in ahash_finup_ctx()
1076 	ahash_unmap_ctx(jrdev, edesc, req, digestsize, DMA_BIDIRECTIONAL);  in ahash_finup_ctx()
1086 	struct device *jrdev = ctx->jrdev;  in ahash_digest()  local
1097 		dev_err(jrdev, "Invalid number of src SG.\n");  in ahash_digest()
1102 		mapped_nents = dma_map_sg(jrdev, req->src, src_nents,  in ahash_digest()
1105 			dev_err(jrdev, "unable to map source for DMA\n");  in ahash_digest()
1116 		dma_unmap_sg(jrdev, req->src, src_nents, DMA_TO_DEVICE);  in ahash_digest()
1125 		ahash_unmap(jrdev, edesc, req, digestsize);  in ahash_digest()
1132 	ret = map_seq_out_ptr_ctx(desc, jrdev, state, digestsize);  in ahash_digest()
1134 		ahash_unmap(jrdev, edesc, req, digestsize);  in ahash_digest()
1143 	return ahash_enqueue_req(jrdev, ahash_done, req, digestsize,  in ahash_digest()
1153 	struct device *jrdev = ctx->jrdev;  in ahash_final_no_ctx()  local
1170 		state->buf_dma = dma_map_single(jrdev, buf, buflen,  in ahash_final_no_ctx()
1172 		if (dma_mapping_error(jrdev, state->buf_dma)) {  in ahash_final_no_ctx()
1173 			dev_err(jrdev, "unable to map src\n");  in ahash_final_no_ctx()
1180 	ret = map_seq_out_ptr_ctx(desc, jrdev, state, digestsize);  in ahash_final_no_ctx()
1188 	return ahash_enqueue_req(jrdev, ahash_done, req,  in ahash_final_no_ctx()
1191 	ahash_unmap(jrdev, edesc, req, digestsize);  in ahash_final_no_ctx()
1202 	struct device *jrdev = ctx->jrdev;  in ahash_update_no_ctx()  local
1233 			dev_err(jrdev, "Invalid number of src SG.\n");  in ahash_update_no_ctx()
1238 			mapped_nents = dma_map_sg(jrdev, req->src, src_nents,  in ahash_update_no_ctx()
1241 				dev_err(jrdev, "unable to DMA map source\n");  in ahash_update_no_ctx()
1259 			dma_unmap_sg(jrdev, req->src, src_nents, DMA_TO_DEVICE);  in ahash_update_no_ctx()
1266 		ret = buf_map_to_sec4_sg(jrdev, edesc->sec4_sg, state);  in ahash_update_no_ctx()
1274 		edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg,  in ahash_update_no_ctx()
1277 		if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) {  in ahash_update_no_ctx()
1278 			dev_err(jrdev, "unable to map S/G table\n");  in ahash_update_no_ctx()
1285 		ret = map_seq_out_ptr_ctx(desc, jrdev, state, ctx->ctx_len);  in ahash_update_no_ctx()
1293 		ret = ahash_enqueue_req(jrdev, ahash_done_ctx_dst, req,  in ahash_update_no_ctx()
1312 	ahash_unmap_ctx(jrdev, edesc, req, ctx->ctx_len, DMA_TO_DEVICE);  in ahash_update_no_ctx()
1323 	struct device *jrdev = ctx->jrdev;  in ahash_finup_no_ctx()  local
1333 		dev_err(jrdev, "Invalid number of src SG.\n");  in ahash_finup_no_ctx()
1338 		mapped_nents = dma_map_sg(jrdev, req->src, src_nents,  in ahash_finup_no_ctx()
1341 			dev_err(jrdev, "unable to DMA map source\n");  in ahash_finup_no_ctx()
1356 		dma_unmap_sg(jrdev, req->src, src_nents, DMA_TO_DEVICE);  in ahash_finup_no_ctx()
1365 	ret = buf_map_to_sec4_sg(jrdev, edesc->sec4_sg, state);  in ahash_finup_no_ctx()
1372 		dev_err(jrdev, "unable to map S/G table\n");  in ahash_finup_no_ctx()
1376 	ret = map_seq_out_ptr_ctx(desc, jrdev, state, digestsize);  in ahash_finup_no_ctx()
1384 	return ahash_enqueue_req(jrdev, ahash_done, req,  in ahash_finup_no_ctx()
1387 	ahash_unmap(jrdev, edesc, req, digestsize);  in ahash_finup_no_ctx()
1399 	struct device *jrdev = ctx->jrdev;  in ahash_update_first()  local
1428 			dev_err(jrdev, "Invalid number of src SG.\n");  in ahash_update_first()
1433 			mapped_nents = dma_map_sg(jrdev, req->src, src_nents,  in ahash_update_first()
1436 				dev_err(jrdev, "unable to map source for DMA\n");  in ahash_update_first()
1452 			dma_unmap_sg(jrdev, req->src, src_nents, DMA_TO_DEVICE);  in ahash_update_first()
1465 		ret = map_seq_out_ptr_ctx(desc, jrdev, state, ctx->ctx_len);  in ahash_update_first()
1473 		ret = ahash_enqueue_req(jrdev, ahash_done_ctx_dst, req,  in ahash_update_first()
1495 	ahash_unmap_ctx(jrdev, edesc, req, ctx->ctx_len, DMA_TO_DEVICE);  in ahash_update_first()
1789 	ctx->jrdev = caam_jr_alloc();  in caam_hash_cra_init()
1790 	if (IS_ERR(ctx->jrdev)) {  in caam_hash_cra_init()
1792 		return PTR_ERR(ctx->jrdev);  in caam_hash_cra_init()
1795 	priv = dev_get_drvdata(ctx->jrdev->parent);  in caam_hash_cra_init()
1822 		ctx->adata.key_dma = dma_map_single_attrs(ctx->jrdev, ctx->key,  in caam_hash_cra_init()
1826 		if (dma_mapping_error(ctx->jrdev, ctx->adata.key_dma)) {  in caam_hash_cra_init()
1827 			dev_err(ctx->jrdev, "unable to map key\n");  in caam_hash_cra_init()
1828 			caam_jr_free(ctx->jrdev);  in caam_hash_cra_init()
1833 	dma_addr = dma_map_single_attrs(ctx->jrdev, ctx->sh_desc_update,  in caam_hash_cra_init()
1837 	if (dma_mapping_error(ctx->jrdev, dma_addr)) {  in caam_hash_cra_init()
1838 		dev_err(ctx->jrdev, "unable to map shared descriptors\n");  in caam_hash_cra_init()
1841 			dma_unmap_single_attrs(ctx->jrdev, ctx->adata.key_dma,  in caam_hash_cra_init()
1846 		caam_jr_free(ctx->jrdev);  in caam_hash_cra_init()
1875 	dma_unmap_single_attrs(ctx->jrdev, ctx->sh_desc_update_dma,  in caam_hash_cra_exit()
1880 		dma_unmap_single_attrs(ctx->jrdev, ctx->adata.key_dma,  in caam_hash_cra_exit()
1883 	caam_jr_free(ctx->jrdev);  in caam_hash_cra_exit()