13 #include "jh7110-cryp.h"
52 static inline int starfive_aes_wait_busy(struct starfive_cryp_dev *cryp)  in starfive_aes_wait_busy()  argument
56 	return readl_relaxed_poll_timeout(cryp->base + STARFIVE_AES_CSR, status,  in starfive_aes_wait_busy()
60 static inline int starfive_aes_wait_keydone(struct starfive_cryp_dev *cryp)  in starfive_aes_wait_keydone()  argument
64 	return readl_relaxed_poll_timeout(cryp->base + STARFIVE_AES_CSR, status,  in starfive_aes_wait_keydone()
68 static inline int starfive_aes_wait_gcmdone(struct starfive_cryp_dev *cryp)  in starfive_aes_wait_gcmdone()  argument
72 	return readl_relaxed_poll_timeout(cryp->base + STARFIVE_AES_CSR, status,  in starfive_aes_wait_gcmdone()
76 static inline int is_gcm(struct starfive_cryp_dev *cryp)  in is_gcm()  argument
78 	return (cryp->flags & FLG_MODE_MASK) == STARFIVE_AES_MODE_GCM;  in is_gcm()
81 static inline int is_encrypt(struct starfive_cryp_dev *cryp)  in is_encrypt()  argument
83 	return cryp->flags & FLG_ENCRYPT;  in is_encrypt()
88 	struct starfive_cryp_dev *cryp = ctx->cryp;  in starfive_aes_aead_hw_start()  local
93 		value = readl(ctx->cryp->base + STARFIVE_AES_CSR);  in starfive_aes_aead_hw_start()
95 		writel(value, cryp->base + STARFIVE_AES_CSR);  in starfive_aes_aead_hw_start()
96 		starfive_aes_wait_gcmdone(cryp);  in starfive_aes_aead_hw_start()
99 		value = readl(ctx->cryp->base + STARFIVE_AES_CSR);  in starfive_aes_aead_hw_start()
101 		writel(value, cryp->base + STARFIVE_AES_CSR);  in starfive_aes_aead_hw_start()
108 	struct starfive_cryp_dev *cryp = ctx->cryp;  in starfive_aes_set_ivlen()  local
110 	if (is_gcm(cryp))  in starfive_aes_set_ivlen()
111 		writel(GCM_AES_IV_SIZE, cryp->base + STARFIVE_AES_IVLEN);  in starfive_aes_set_ivlen()
113 		writel(AES_BLOCK_SIZE, cryp->base + STARFIVE_AES_IVLEN);  in starfive_aes_set_ivlen()
118 	struct starfive_cryp_dev *cryp = ctx->cryp;  in starfive_aes_set_alen()  local
120 	writel(upper_32_bits(cryp->assoclen), cryp->base + STARFIVE_AES_ALEN0);  in starfive_aes_set_alen()
121 	writel(lower_32_bits(cryp->assoclen), cryp->base + STARFIVE_AES_ALEN1);  in starfive_aes_set_alen()
126 	struct starfive_cryp_dev *cryp = ctx->cryp;  in starfive_aes_set_mlen()  local
128 	writel(upper_32_bits(cryp->total_in), cryp->base + STARFIVE_AES_MLEN0);  in starfive_aes_set_mlen()
129 	writel(lower_32_bits(cryp->total_in), cryp->base + STARFIVE_AES_MLEN1);  in starfive_aes_set_mlen()
143 	struct starfive_cryp_dev *cryp = ctx->cryp;  in starfive_aes_write_iv()  local
145 	writel(iv[0], cryp->base + STARFIVE_AES_IV0);  in starfive_aes_write_iv()
146 	writel(iv[1], cryp->base + STARFIVE_AES_IV1);  in starfive_aes_write_iv()
147 	writel(iv[2], cryp->base + STARFIVE_AES_IV2);  in starfive_aes_write_iv()
149 	if (is_gcm(cryp)) {  in starfive_aes_write_iv()
150 		if (starfive_aes_wait_gcmdone(cryp))  in starfive_aes_write_iv()
156 	writel(iv[3], cryp->base + STARFIVE_AES_IV3);  in starfive_aes_write_iv()
161 static inline void starfive_aes_get_iv(struct starfive_cryp_dev *cryp, u32 *iv)  in starfive_aes_get_iv()  argument
163 	iv[0] = readl(cryp->base + STARFIVE_AES_IV0);  in starfive_aes_get_iv()
164 	iv[1] = readl(cryp->base + STARFIVE_AES_IV1);  in starfive_aes_get_iv()
165 	iv[2] = readl(cryp->base + STARFIVE_AES_IV2);  in starfive_aes_get_iv()
166 	iv[3] = readl(cryp->base + STARFIVE_AES_IV3);  in starfive_aes_get_iv()
171 	struct starfive_cryp_dev *cryp = ctx->cryp;  in starfive_aes_write_nonce()  local
173 	writel(nonce[0], cryp->base + STARFIVE_AES_NONCE0);  in starfive_aes_write_nonce()
174 	writel(nonce[1], cryp->base + STARFIVE_AES_NONCE1);  in starfive_aes_write_nonce()
175 	writel(nonce[2], cryp->base + STARFIVE_AES_NONCE2);  in starfive_aes_write_nonce()
176 	writel(nonce[3], cryp->base + STARFIVE_AES_NONCE3);  in starfive_aes_write_nonce()
181 	struct starfive_cryp_dev *cryp = ctx->cryp;  in starfive_aes_write_key()  local
185 		writel(key[0], cryp->base + STARFIVE_AES_KEY0);  in starfive_aes_write_key()
186 		writel(key[1], cryp->base + STARFIVE_AES_KEY1);  in starfive_aes_write_key()
187 		writel(key[2], cryp->base + STARFIVE_AES_KEY2);  in starfive_aes_write_key()
188 		writel(key[3], cryp->base + STARFIVE_AES_KEY3);  in starfive_aes_write_key()
192 		writel(key[4], cryp->base + STARFIVE_AES_KEY4);  in starfive_aes_write_key()
193 		writel(key[5], cryp->base + STARFIVE_AES_KEY5);  in starfive_aes_write_key()
197 		writel(key[6], cryp->base + STARFIVE_AES_KEY6);  in starfive_aes_write_key()
198 		writel(key[7], cryp->base + STARFIVE_AES_KEY7);  in starfive_aes_write_key()
201 	if (starfive_aes_wait_keydone(cryp))  in starfive_aes_write_key()
209 	struct starfive_cryp_dev *cryp = ctx->cryp;  in starfive_aes_ccm_init()  local
213 	memcpy(iv, cryp->req.areq->iv, AES_BLOCK_SIZE);  in starfive_aes_ccm_init()
219 	b0[0] |= (8 * ((cryp->authsize - 2) / 2));  in starfive_aes_ccm_init()
221 	if (cryp->assoclen)  in starfive_aes_ccm_init()
224 	textlen = cryp->total_in;  in starfive_aes_ccm_init()
237 	struct starfive_cryp_dev *cryp = ctx->cryp;  in starfive_aes_hw_init()  local
243 	writel(rctx->csr.aes.v, cryp->base + STARFIVE_AES_CSR);  in starfive_aes_hw_init()
246 	hw_mode = cryp->flags & FLG_MODE_MASK;  in starfive_aes_hw_init()
263 	rctx->csr.aes.cmode = !is_encrypt(cryp);  in starfive_aes_hw_init()
267 	if (cryp->side_chan) {  in starfive_aes_hw_init()
272 	writel(rctx->csr.aes.v, cryp->base + STARFIVE_AES_CSR);  in starfive_aes_hw_init()
274 	cryp->err = starfive_aes_write_key(ctx);  in starfive_aes_hw_init()
275 	if (cryp->err)  in starfive_aes_hw_init()
276 		return cryp->err;  in starfive_aes_hw_init()
284 		starfive_aes_write_iv(ctx, (void *)cryp->req.areq->iv);  in starfive_aes_hw_init()
294 		starfive_aes_write_iv(ctx, (void *)cryp->req.sreq->iv);  in starfive_aes_hw_init()
300 	return cryp->err;  in starfive_aes_hw_init()
303 static int starfive_aes_read_authtag(struct starfive_cryp_dev *cryp)  in starfive_aes_read_authtag()  argument
307 	if (starfive_aes_wait_busy(cryp))  in starfive_aes_read_authtag()
308 		return dev_err_probe(cryp->dev, -ETIMEDOUT,  in starfive_aes_read_authtag()
313 	if (is_gcm(cryp))  in starfive_aes_read_authtag()
315 			cryp->tag_out[i] = readl(cryp->base + start_addr);  in starfive_aes_read_authtag()
318 			cryp->tag_out[i] = readl(cryp->base + STARFIVE_AES_AESDIO0R);  in starfive_aes_read_authtag()
320 	if (is_encrypt(cryp)) {  in starfive_aes_read_authtag()
321 		scatterwalk_copychunks(cryp->tag_out, &cryp->out_walk, cryp->authsize, 1);  in starfive_aes_read_authtag()
323 		scatterwalk_copychunks(cryp->tag_in, &cryp->in_walk, cryp->authsize, 0);  in starfive_aes_read_authtag()
325 		if (crypto_memneq(cryp->tag_in, cryp->tag_out, cryp->authsize))  in starfive_aes_read_authtag()
326 			return dev_err_probe(cryp->dev, -EBADMSG, "Failed tag verification\n");  in starfive_aes_read_authtag()
332 static void starfive_aes_finish_req(struct starfive_cryp_dev *cryp)  in starfive_aes_finish_req()  argument
335 	int err = cryp->err;  in starfive_aes_finish_req()
337 	if (!err && cryp->authsize)  in starfive_aes_finish_req()
338 		err = starfive_aes_read_authtag(cryp);  in starfive_aes_finish_req()
340 	if (!err && ((cryp->flags & FLG_MODE_MASK) == STARFIVE_AES_MODE_CBC ||  in starfive_aes_finish_req()
341 		     (cryp->flags & FLG_MODE_MASK) == STARFIVE_AES_MODE_CTR))  in starfive_aes_finish_req()
342 		starfive_aes_get_iv(cryp, (void *)cryp->req.sreq->iv);  in starfive_aes_finish_req()
347 	writel(csr.v, cryp->base + STARFIVE_AES_CSR);  in starfive_aes_finish_req()
349 	if (cryp->authsize)  in starfive_aes_finish_req()
350 		crypto_finalize_aead_request(cryp->engine, cryp->req.areq, err);  in starfive_aes_finish_req()
352 		crypto_finalize_skcipher_request(cryp->engine, cryp->req.sreq,  in starfive_aes_finish_req()
358 	struct starfive_cryp_dev *cryp = (struct starfive_cryp_dev *)param;  in starfive_aes_done_task()  local
364 		block[i] = readl(cryp->base + STARFIVE_AES_AESDIO0R);  in starfive_aes_done_task()
366 	scatterwalk_copychunks(block, &cryp->out_walk, min_t(size_t, AES_BLOCK_SIZE,  in starfive_aes_done_task()
367 							     cryp->total_out), 1);  in starfive_aes_done_task()
369 	cryp->total_out -= min_t(size_t, AES_BLOCK_SIZE, cryp->total_out);  in starfive_aes_done_task()
371 	if (!cryp->total_out) {  in starfive_aes_done_task()
372 		starfive_aes_finish_req(cryp);  in starfive_aes_done_task()
377 	scatterwalk_copychunks(block, &cryp->in_walk, min_t(size_t, AES_BLOCK_SIZE,  in starfive_aes_done_task()
378 							    cryp->total_in), 0);  in starfive_aes_done_task()
379 	cryp->total_in -= min_t(size_t, AES_BLOCK_SIZE, cryp->total_in);  in starfive_aes_done_task()
382 		writel(block[i], cryp->base + STARFIVE_AES_AESDIO0R);  in starfive_aes_done_task()
384 	stat = readl(cryp->base + STARFIVE_IE_MASK_OFFSET);  in starfive_aes_done_task()
386 	writel(stat, cryp->base + STARFIVE_IE_MASK_OFFSET);  in starfive_aes_done_task()
391 	struct starfive_cryp_dev *cryp = ctx->cryp;  in starfive_aes_gcm_write_adata()  local
396 	total_len = ALIGN(cryp->assoclen, AES_BLOCK_SIZE) / sizeof(unsigned int);  in starfive_aes_gcm_write_adata()
400 		writel(*buffer, cryp->base + STARFIVE_AES_NONCE0);  in starfive_aes_gcm_write_adata()
402 		writel(*buffer, cryp->base + STARFIVE_AES_NONCE1);  in starfive_aes_gcm_write_adata()
404 		writel(*buffer, cryp->base + STARFIVE_AES_NONCE2);  in starfive_aes_gcm_write_adata()
406 		writel(*buffer, cryp->base + STARFIVE_AES_NONCE3);  in starfive_aes_gcm_write_adata()
410 	if (starfive_aes_wait_gcmdone(cryp))  in starfive_aes_gcm_write_adata()
411 		return dev_err_probe(cryp->dev, -ETIMEDOUT,  in starfive_aes_gcm_write_adata()
419 	struct starfive_cryp_dev *cryp = ctx->cryp;  in starfive_aes_ccm_write_adata()  local
425 	total_len = cryp->assoclen;  in starfive_aes_ccm_write_adata()
428 	writeb(*ci, cryp->base + STARFIVE_AES_AESDIO0R);  in starfive_aes_ccm_write_adata()
430 	writeb(*ci, cryp->base + STARFIVE_AES_AESDIO0R);  in starfive_aes_ccm_write_adata()
436 		writel(*buffer, cryp->base + STARFIVE_AES_AESDIO0R);  in starfive_aes_ccm_write_adata()
442 			writel(*buffer, cryp->base + STARFIVE_AES_AESDIO0R);  in starfive_aes_ccm_write_adata()
447 	if (starfive_aes_wait_busy(cryp))  in starfive_aes_ccm_write_adata()
448 		return dev_err_probe(cryp->dev, -ETIMEDOUT,  in starfive_aes_ccm_write_adata()
459 	struct starfive_cryp_dev *cryp;  in starfive_aes_prepare_req()  local
467 	cryp = ctx->cryp;  in starfive_aes_prepare_req()
471 		cryp->req.sreq = req;  in starfive_aes_prepare_req()
472 		cryp->total_in = req->cryptlen;  in starfive_aes_prepare_req()
473 		cryp->total_out = req->cryptlen;  in starfive_aes_prepare_req()
474 		cryp->assoclen = 0;  in starfive_aes_prepare_req()
475 		cryp->authsize = 0;  in starfive_aes_prepare_req()
477 		cryp->req.areq = areq;  in starfive_aes_prepare_req()
478 		cryp->assoclen = areq->assoclen;  in starfive_aes_prepare_req()
479 		cryp->authsize = crypto_aead_authsize(crypto_aead_reqtfm(areq));  in starfive_aes_prepare_req()
480 		if (is_encrypt(cryp)) {  in starfive_aes_prepare_req()
481 			cryp->total_in = areq->cryptlen;  in starfive_aes_prepare_req()
482 			cryp->total_out = areq->cryptlen;  in starfive_aes_prepare_req()
484 			cryp->total_in = areq->cryptlen - cryp->authsize;  in starfive_aes_prepare_req()
485 			cryp->total_out = cryp->total_in;  in starfive_aes_prepare_req()
490 	scatterwalk_start(&cryp->in_walk, rctx->in_sg);  in starfive_aes_prepare_req()
493 	scatterwalk_start(&cryp->out_walk, rctx->out_sg);  in starfive_aes_prepare_req()
495 	if (cryp->assoclen) {  in starfive_aes_prepare_req()
496 		rctx->adata = kzalloc(cryp->assoclen + AES_BLOCK_SIZE, GFP_KERNEL);  in starfive_aes_prepare_req()
498 			return dev_err_probe(cryp->dev, -ENOMEM,  in starfive_aes_prepare_req()
501 		scatterwalk_copychunks(rctx->adata, &cryp->in_walk, cryp->assoclen, 0);  in starfive_aes_prepare_req()
502 		scatterwalk_copychunks(NULL, &cryp->out_walk, cryp->assoclen, 2);  in starfive_aes_prepare_req()
516 	struct starfive_cryp_dev *cryp = ctx->cryp;  in starfive_aes_do_one_req()  local
530 	scatterwalk_copychunks(block, &cryp->in_walk, min_t(size_t, AES_BLOCK_SIZE,  in starfive_aes_do_one_req()
531 							    cryp->total_in), 0);  in starfive_aes_do_one_req()
532 	cryp->total_in -= min_t(size_t, AES_BLOCK_SIZE, cryp->total_in);  in starfive_aes_do_one_req()
535 		writel(block[i], cryp->base + STARFIVE_AES_AESDIO0R);  in starfive_aes_do_one_req()
537 	stat = readl(cryp->base + STARFIVE_IE_MASK_OFFSET);  in starfive_aes_do_one_req()
539 	writel(stat, cryp->base + STARFIVE_IE_MASK_OFFSET);  in starfive_aes_do_one_req()
548 	ctx->cryp = starfive_cryp_find_dev(ctx);  in starfive_aes_init_tfm()
549 	if (!ctx->cryp)  in starfive_aes_init_tfm()
564 	struct starfive_cryp_dev *cryp = ctx->cryp;  in starfive_aes_aead_do_one_req()  local
577 	if (!cryp->assoclen)  in starfive_aes_aead_do_one_req()
580 	if ((cryp->flags & FLG_MODE_MASK) == STARFIVE_AES_MODE_CCM)  in starfive_aes_aead_do_one_req()
581 		cryp->err = starfive_aes_ccm_write_adata(ctx);  in starfive_aes_aead_do_one_req()
583 		cryp->err = starfive_aes_gcm_write_adata(ctx);  in starfive_aes_aead_do_one_req()
587 	if (cryp->err)  in starfive_aes_aead_do_one_req()
588 		return cryp->err;  in starfive_aes_aead_do_one_req()
591 	if (!cryp->total_in)  in starfive_aes_aead_do_one_req()
598 	scatterwalk_copychunks(block, &cryp->in_walk, min_t(size_t, AES_BLOCK_SIZE,  in starfive_aes_aead_do_one_req()
599 							    cryp->total_in), 0);  in starfive_aes_aead_do_one_req()
600 	cryp->total_in -= min_t(size_t, AES_BLOCK_SIZE, cryp->total_in);  in starfive_aes_aead_do_one_req()
603 		writel(block[i], cryp->base + STARFIVE_AES_AESDIO0R);  in starfive_aes_aead_do_one_req()
605 	stat = readl(cryp->base + STARFIVE_IE_MASK_OFFSET);  in starfive_aes_aead_do_one_req()
607 	writel(stat, cryp->base + STARFIVE_IE_MASK_OFFSET);  in starfive_aes_aead_do_one_req()
612 	starfive_aes_finish_req(cryp);  in starfive_aes_aead_do_one_req()
619 	struct starfive_cryp_dev *cryp = ctx->cryp;  in starfive_aes_aead_init_tfm()  local
623 	ctx->cryp = starfive_cryp_find_dev(ctx);  in starfive_aes_aead_init_tfm()
624 	if (!ctx->cryp)  in starfive_aes_aead_init_tfm()
631 			return dev_err_probe(cryp->dev, PTR_ERR(ctx->aead_fbk),  in starfive_aes_aead_init_tfm()
653 	struct starfive_cryp_dev *cryp = ctx->cryp;  in starfive_aes_crypt()  local
656 	cryp->flags = flags;  in starfive_aes_crypt()
658 	if ((cryp->flags & FLG_MODE_MASK) == STARFIVE_AES_MODE_ECB ||  in starfive_aes_crypt()
659 	    (cryp->flags & FLG_MODE_MASK) == STARFIVE_AES_MODE_CBC)  in starfive_aes_crypt()
663 	return crypto_transfer_skcipher_request_to_engine(cryp->engine, req);  in starfive_aes_crypt()
669 	struct starfive_cryp_dev *cryp = ctx->cryp;  in starfive_aes_aead_crypt()  local
671 	cryp->flags = flags;  in starfive_aes_aead_crypt()
677 	if (ctx->aead_fbk && !is_encrypt(cryp)) {  in starfive_aes_aead_crypt()
690 	return crypto_transfer_aead_request_to_engine(cryp->engine, req);  in starfive_aes_aead_crypt()