Lines Matching full:cryp
5 * Ux500 support taken from snippets in the old Ux500 cryp driver
30 #define DRIVER_NAME "stm32-cryp"
179 struct stm32_cryp *cryp; member
243 static inline bool is_aes(struct stm32_cryp *cryp) in is_aes() argument
245 return cryp->flags & FLG_AES; in is_aes()
248 static inline bool is_des(struct stm32_cryp *cryp) in is_des() argument
250 return cryp->flags & FLG_DES; in is_des()
253 static inline bool is_tdes(struct stm32_cryp *cryp) in is_tdes() argument
255 return cryp->flags & FLG_TDES; in is_tdes()
258 static inline bool is_ecb(struct stm32_cryp *cryp) in is_ecb() argument
260 return cryp->flags & FLG_ECB; in is_ecb()
263 static inline bool is_cbc(struct stm32_cryp *cryp) in is_cbc() argument
265 return cryp->flags & FLG_CBC; in is_cbc()
268 static inline bool is_ctr(struct stm32_cryp *cryp) in is_ctr() argument
270 return cryp->flags & FLG_CTR; in is_ctr()
273 static inline bool is_gcm(struct stm32_cryp *cryp) in is_gcm() argument
275 return cryp->flags & FLG_GCM; in is_gcm()
278 static inline bool is_ccm(struct stm32_cryp *cryp) in is_ccm() argument
280 return cryp->flags & FLG_CCM; in is_ccm()
283 static inline bool is_encrypt(struct stm32_cryp *cryp) in is_encrypt() argument
285 return cryp->flags & FLG_ENCRYPT; in is_encrypt()
288 static inline bool is_decrypt(struct stm32_cryp *cryp) in is_decrypt() argument
290 return !is_encrypt(cryp); in is_decrypt()
293 static inline u32 stm32_cryp_read(struct stm32_cryp *cryp, u32 ofst) in stm32_cryp_read() argument
295 return readl_relaxed(cryp->regs + ofst); in stm32_cryp_read()
298 static inline void stm32_cryp_write(struct stm32_cryp *cryp, u32 ofst, u32 val) in stm32_cryp_write() argument
300 writel_relaxed(val, cryp->regs + ofst); in stm32_cryp_write()
303 static inline int stm32_cryp_wait_busy(struct stm32_cryp *cryp) in stm32_cryp_wait_busy() argument
307 return readl_relaxed_poll_timeout(cryp->regs + cryp->caps->sr, status, in stm32_cryp_wait_busy()
311 static inline void stm32_cryp_enable(struct stm32_cryp *cryp) in stm32_cryp_enable() argument
313 writel_relaxed(readl_relaxed(cryp->regs + cryp->caps->cr) | CR_CRYPEN, in stm32_cryp_enable()
314 cryp->regs + cryp->caps->cr); in stm32_cryp_enable()
317 static inline int stm32_cryp_wait_enable(struct stm32_cryp *cryp) in stm32_cryp_wait_enable() argument
321 return readl_relaxed_poll_timeout(cryp->regs + cryp->caps->cr, status, in stm32_cryp_wait_enable()
325 static inline int stm32_cryp_wait_input(struct stm32_cryp *cryp) in stm32_cryp_wait_input() argument
329 return readl_relaxed_poll_timeout_atomic(cryp->regs + cryp->caps->sr, status, in stm32_cryp_wait_input()
333 static inline int stm32_cryp_wait_output(struct stm32_cryp *cryp) in stm32_cryp_wait_output() argument
337 return readl_relaxed_poll_timeout_atomic(cryp->regs + cryp->caps->sr, status, in stm32_cryp_wait_output()
341 static inline void stm32_cryp_key_read_enable(struct stm32_cryp *cryp) in stm32_cryp_key_read_enable() argument
343 writel_relaxed(readl_relaxed(cryp->regs + cryp->caps->cr) | CR_KEYRDEN, in stm32_cryp_key_read_enable()
344 cryp->regs + cryp->caps->cr); in stm32_cryp_key_read_enable()
347 static inline void stm32_cryp_key_read_disable(struct stm32_cryp *cryp) in stm32_cryp_key_read_disable() argument
349 writel_relaxed(readl_relaxed(cryp->regs + cryp->caps->cr) & ~CR_KEYRDEN, in stm32_cryp_key_read_disable()
350 cryp->regs + cryp->caps->cr); in stm32_cryp_key_read_disable()
353 static void stm32_cryp_irq_read_data(struct stm32_cryp *cryp);
354 static void stm32_cryp_irq_write_data(struct stm32_cryp *cryp);
355 static void stm32_cryp_irq_write_gcmccm_header(struct stm32_cryp *cryp);
356 static int stm32_cryp_read_auth_tag(struct stm32_cryp *cryp);
357 static void stm32_cryp_finish_req(struct stm32_cryp *cryp, int err);
358 static int stm32_cryp_dma_start(struct stm32_cryp *cryp);
359 static int stm32_cryp_it_start(struct stm32_cryp *cryp);
363 struct stm32_cryp *tmp, *cryp = NULL; in stm32_cryp_find_dev() local
366 if (!ctx->cryp) { in stm32_cryp_find_dev()
368 cryp = tmp; in stm32_cryp_find_dev()
371 ctx->cryp = cryp; in stm32_cryp_find_dev()
373 cryp = ctx->cryp; in stm32_cryp_find_dev()
378 return cryp; in stm32_cryp_find_dev()
381 static void stm32_cryp_hw_write_iv(struct stm32_cryp *cryp, __be32 *iv) in stm32_cryp_hw_write_iv() argument
386 stm32_cryp_write(cryp, cryp->caps->iv0l, be32_to_cpu(*iv++)); in stm32_cryp_hw_write_iv()
387 stm32_cryp_write(cryp, cryp->caps->iv0r, be32_to_cpu(*iv++)); in stm32_cryp_hw_write_iv()
389 if (is_aes(cryp)) { in stm32_cryp_hw_write_iv()
390 stm32_cryp_write(cryp, cryp->caps->iv1l, be32_to_cpu(*iv++)); in stm32_cryp_hw_write_iv()
391 stm32_cryp_write(cryp, cryp->caps->iv1r, be32_to_cpu(*iv++)); in stm32_cryp_hw_write_iv()
395 static void stm32_cryp_get_iv(struct stm32_cryp *cryp) in stm32_cryp_get_iv() argument
397 struct skcipher_request *req = cryp->req; in stm32_cryp_get_iv()
403 if (cryp->caps->iv_protection) in stm32_cryp_get_iv()
404 stm32_cryp_key_read_enable(cryp); in stm32_cryp_get_iv()
406 *tmp++ = cpu_to_be32(stm32_cryp_read(cryp, cryp->caps->iv0l)); in stm32_cryp_get_iv()
407 *tmp++ = cpu_to_be32(stm32_cryp_read(cryp, cryp->caps->iv0r)); in stm32_cryp_get_iv()
409 if (is_aes(cryp)) { in stm32_cryp_get_iv()
410 *tmp++ = cpu_to_be32(stm32_cryp_read(cryp, cryp->caps->iv1l)); in stm32_cryp_get_iv()
411 *tmp++ = cpu_to_be32(stm32_cryp_read(cryp, cryp->caps->iv1r)); in stm32_cryp_get_iv()
414 if (cryp->caps->iv_protection) in stm32_cryp_get_iv()
415 stm32_cryp_key_read_disable(cryp); in stm32_cryp_get_iv()
537 static u32 stm32_cryp_get_hw_mode(struct stm32_cryp *cryp) in stm32_cryp_get_hw_mode() argument
539 if (is_aes(cryp) && is_ecb(cryp)) in stm32_cryp_get_hw_mode()
542 if (is_aes(cryp) && is_cbc(cryp)) in stm32_cryp_get_hw_mode()
545 if (is_aes(cryp) && is_ctr(cryp)) in stm32_cryp_get_hw_mode()
548 if (is_aes(cryp) && is_gcm(cryp)) in stm32_cryp_get_hw_mode()
551 if (is_aes(cryp) && is_ccm(cryp)) in stm32_cryp_get_hw_mode()
554 if (is_des(cryp) && is_ecb(cryp)) in stm32_cryp_get_hw_mode()
557 if (is_des(cryp) && is_cbc(cryp)) in stm32_cryp_get_hw_mode()
560 if (is_tdes(cryp) && is_ecb(cryp)) in stm32_cryp_get_hw_mode()
563 if (is_tdes(cryp) && is_cbc(cryp)) in stm32_cryp_get_hw_mode()
566 dev_err(cryp->dev, "Unknown mode\n"); in stm32_cryp_get_hw_mode()
570 static unsigned int stm32_cryp_get_input_text_len(struct stm32_cryp *cryp) in stm32_cryp_get_input_text_len() argument
572 return is_encrypt(cryp) ? cryp->areq->cryptlen : in stm32_cryp_get_input_text_len()
573 cryp->areq->cryptlen - cryp->authsize; in stm32_cryp_get_input_text_len()
576 static int stm32_cryp_gcm_init(struct stm32_cryp *cryp, u32 cfg) in stm32_cryp_gcm_init() argument
582 memcpy(iv, cryp->areq->iv, 12); in stm32_cryp_gcm_init()
584 cryp->gcm_ctr = GCM_CTR_INIT; in stm32_cryp_gcm_init()
585 stm32_cryp_hw_write_iv(cryp, iv); in stm32_cryp_gcm_init()
587 stm32_cryp_write(cryp, cryp->caps->cr, cfg | CR_PH_INIT | CR_CRYPEN); in stm32_cryp_gcm_init()
590 ret = stm32_cryp_wait_enable(cryp); in stm32_cryp_gcm_init()
592 dev_err(cryp->dev, "Timeout (gcm init)\n"); in stm32_cryp_gcm_init()
597 if (cryp->areq->assoclen) { in stm32_cryp_gcm_init()
599 stm32_cryp_write(cryp, cryp->caps->cr, cfg); in stm32_cryp_gcm_init()
600 } else if (stm32_cryp_get_input_text_len(cryp)) { in stm32_cryp_gcm_init()
602 stm32_cryp_write(cryp, cryp->caps->cr, cfg); in stm32_cryp_gcm_init()
608 static void stm32_crypt_gcmccm_end_header(struct stm32_cryp *cryp) in stm32_crypt_gcmccm_end_header() argument
614 if (!cryp->header_in) { in stm32_crypt_gcmccm_end_header()
616 err = stm32_cryp_wait_busy(cryp); in stm32_crypt_gcmccm_end_header()
618 dev_err(cryp->dev, "Timeout (gcm/ccm header)\n"); in stm32_crypt_gcmccm_end_header()
619 stm32_cryp_write(cryp, cryp->caps->imsc, 0); in stm32_crypt_gcmccm_end_header()
620 stm32_cryp_finish_req(cryp, err); in stm32_crypt_gcmccm_end_header()
624 if (stm32_cryp_get_input_text_len(cryp)) { in stm32_crypt_gcmccm_end_header()
626 cfg = stm32_cryp_read(cryp, cryp->caps->cr); in stm32_crypt_gcmccm_end_header()
628 stm32_cryp_write(cryp, cryp->caps->cr, cfg); in stm32_crypt_gcmccm_end_header()
632 stm32_cryp_write(cryp, cryp->caps->cr, cfg); in stm32_crypt_gcmccm_end_header()
643 static void stm32_cryp_write_ccm_first_header(struct stm32_cryp *cryp) in stm32_cryp_write_ccm_first_header() argument
647 u32 alen = cryp->areq->assoclen; in stm32_cryp_write_ccm_first_header()
669 memcpy_from_scatterwalk((char *)block + len, &cryp->in_walk, written); in stm32_cryp_write_ccm_first_header()
671 writesl(cryp->regs + cryp->caps->din, block, AES_BLOCK_32); in stm32_cryp_write_ccm_first_header()
673 cryp->header_in -= written; in stm32_cryp_write_ccm_first_header()
675 stm32_crypt_gcmccm_end_header(cryp); in stm32_cryp_write_ccm_first_header()
678 static int stm32_cryp_ccm_init(struct stm32_cryp *cryp, u32 cfg) in stm32_cryp_ccm_init() argument
688 memcpy(iv, cryp->areq->iv, AES_BLOCK_SIZE); in stm32_cryp_ccm_init()
691 stm32_cryp_hw_write_iv(cryp, (__be32 *)iv); in stm32_cryp_ccm_init()
696 b0[0] |= (8 * ((cryp->authsize - 2) / 2)); in stm32_cryp_ccm_init()
698 if (cryp->areq->assoclen) in stm32_cryp_ccm_init()
701 textlen = stm32_cryp_get_input_text_len(cryp); in stm32_cryp_ccm_init()
707 stm32_cryp_write(cryp, cryp->caps->cr, cfg | CR_PH_INIT | CR_CRYPEN); in stm32_cryp_ccm_init()
716 if (!cryp->caps->padding_wa) in stm32_cryp_ccm_init()
718 stm32_cryp_write(cryp, cryp->caps->din, xd); in stm32_cryp_ccm_init()
722 ret = stm32_cryp_wait_enable(cryp); in stm32_cryp_ccm_init()
724 dev_err(cryp->dev, "Timeout (ccm init)\n"); in stm32_cryp_ccm_init()
729 if (cryp->areq->assoclen) { in stm32_cryp_ccm_init()
731 stm32_cryp_write(cryp, cryp->caps->cr, cfg); in stm32_cryp_ccm_init()
734 stm32_cryp_write_ccm_first_header(cryp); in stm32_cryp_ccm_init()
735 } else if (stm32_cryp_get_input_text_len(cryp)) { in stm32_cryp_ccm_init()
737 stm32_cryp_write(cryp, cryp->caps->cr, cfg); in stm32_cryp_ccm_init()
743 static int stm32_cryp_hw_init(struct stm32_cryp *cryp) in stm32_cryp_hw_init() argument
748 pm_runtime_get_sync(cryp->dev); in stm32_cryp_hw_init()
751 stm32_cryp_write(cryp, cryp->caps->imsc, 0); in stm32_cryp_hw_init()
756 switch (cryp->ctx->keylen) { in stm32_cryp_hw_init()
771 hw_mode = stm32_cryp_get_hw_mode(cryp); in stm32_cryp_hw_init()
776 if (is_decrypt(cryp) && in stm32_cryp_hw_init()
779 if (cryp->caps->kp_mode) in stm32_cryp_hw_init()
780 stm32_cryp_write(cryp, cryp->caps->cr, in stm32_cryp_hw_init()
783 stm32_cryp_write(cryp, in stm32_cryp_hw_init()
784 cryp->caps->cr, cfg | CR_AES_ECB | CR_KSE); in stm32_cryp_hw_init()
787 stm32_cryp_hw_write_key(cryp); in stm32_cryp_hw_init()
790 stm32_cryp_enable(cryp); in stm32_cryp_hw_init()
792 ret = stm32_cryp_wait_busy(cryp); in stm32_cryp_hw_init()
794 dev_err(cryp->dev, "Timeout (key preparation)\n"); in stm32_cryp_hw_init()
801 stm32_cryp_write(cryp, cryp->caps->cr, cfg); in stm32_cryp_hw_init()
804 if (is_decrypt(cryp)) in stm32_cryp_hw_init()
808 stm32_cryp_write(cryp, cryp->caps->cr, cfg); in stm32_cryp_hw_init()
811 stm32_cryp_hw_write_key(cryp); in stm32_cryp_hw_init()
819 ret = stm32_cryp_ccm_init(cryp, cfg); in stm32_cryp_hw_init()
821 ret = stm32_cryp_gcm_init(cryp, cfg); in stm32_cryp_hw_init()
832 stm32_cryp_hw_write_iv(cryp, (__be32 *)cryp->req->iv); in stm32_cryp_hw_init()
840 stm32_cryp_enable(cryp); in stm32_cryp_hw_init()
845 static void stm32_cryp_finish_req(struct stm32_cryp *cryp, int err) in stm32_cryp_finish_req() argument
847 if (!err && (is_gcm(cryp) || is_ccm(cryp))) in stm32_cryp_finish_req()
849 err = stm32_cryp_read_auth_tag(cryp); in stm32_cryp_finish_req()
851 if (!err && (!(is_gcm(cryp) || is_ccm(cryp) || is_ecb(cryp)))) in stm32_cryp_finish_req()
852 stm32_cryp_get_iv(cryp); in stm32_cryp_finish_req()
854 pm_runtime_mark_last_busy(cryp->dev); in stm32_cryp_finish_req()
855 pm_runtime_put_autosuspend(cryp->dev); in stm32_cryp_finish_req()
857 if (is_gcm(cryp) || is_ccm(cryp)) in stm32_cryp_finish_req()
858 crypto_finalize_aead_request(cryp->engine, cryp->areq, err); in stm32_cryp_finish_req()
860 crypto_finalize_skcipher_request(cryp->engine, cryp->req, err); in stm32_cryp_finish_req()
865 struct stm32_cryp *cryp = (struct stm32_cryp *)param; in stm32_cryp_header_dma_callback() local
869 dma_unmap_sg(cryp->dev, cryp->header_sg, cryp->header_sg_len, DMA_TO_DEVICE); in stm32_cryp_header_dma_callback()
871 reg = stm32_cryp_read(cryp, cryp->caps->dmacr); in stm32_cryp_header_dma_callback()
872 stm32_cryp_write(cryp, cryp->caps->dmacr, reg & ~(DMACR_DOEN | DMACR_DIEN)); in stm32_cryp_header_dma_callback()
874 kfree(cryp->header_sg); in stm32_cryp_header_dma_callback()
876 reg = stm32_cryp_read(cryp, cryp->caps->cr); in stm32_cryp_header_dma_callback()
878 if (cryp->header_in) { in stm32_cryp_header_dma_callback()
879 stm32_cryp_write(cryp, cryp->caps->cr, reg | CR_CRYPEN); in stm32_cryp_header_dma_callback()
881 ret = stm32_cryp_wait_input(cryp); in stm32_cryp_header_dma_callback()
883 dev_err(cryp->dev, "input header ready timeout after dma\n"); in stm32_cryp_header_dma_callback()
884 stm32_cryp_finish_req(cryp, ret); in stm32_cryp_header_dma_callback()
887 stm32_cryp_irq_write_gcmccm_header(cryp); in stm32_cryp_header_dma_callback()
888 WARN_ON(cryp->header_in); in stm32_cryp_header_dma_callback()
891 if (stm32_cryp_get_input_text_len(cryp)) { in stm32_cryp_header_dma_callback()
893 reg = stm32_cryp_read(cryp, cryp->caps->cr); in stm32_cryp_header_dma_callback()
894 stm32_cryp_write(cryp, cryp->caps->cr, reg & ~CR_CRYPEN); in stm32_cryp_header_dma_callback()
898 stm32_cryp_write(cryp, cryp->caps->cr, reg); in stm32_cryp_header_dma_callback()
900 if (cryp->flags & FLG_IN_OUT_DMA) { in stm32_cryp_header_dma_callback()
901 ret = stm32_cryp_dma_start(cryp); in stm32_cryp_header_dma_callback()
903 stm32_cryp_finish_req(cryp, ret); in stm32_cryp_header_dma_callback()
905 stm32_cryp_it_start(cryp); in stm32_cryp_header_dma_callback()
912 stm32_cryp_finish_req(cryp, 0); in stm32_cryp_header_dma_callback()
918 struct stm32_cryp *cryp = (struct stm32_cryp *)param; in stm32_cryp_dma_callback() local
922 complete(&cryp->dma_completion); /* completion to indicate no timeout */ in stm32_cryp_dma_callback()
924 dma_sync_sg_for_device(cryp->dev, cryp->out_sg, cryp->out_sg_len, DMA_FROM_DEVICE); in stm32_cryp_dma_callback()
926 if (cryp->in_sg != cryp->out_sg) in stm32_cryp_dma_callback()
927 dma_unmap_sg(cryp->dev, cryp->in_sg, cryp->in_sg_len, DMA_TO_DEVICE); in stm32_cryp_dma_callback()
929 dma_unmap_sg(cryp->dev, cryp->out_sg, cryp->out_sg_len, DMA_FROM_DEVICE); in stm32_cryp_dma_callback()
931 reg = stm32_cryp_read(cryp, cryp->caps->dmacr); in stm32_cryp_dma_callback()
932 stm32_cryp_write(cryp, cryp->caps->dmacr, reg & ~(DMACR_DOEN | DMACR_DIEN)); in stm32_cryp_dma_callback()
934 reg = stm32_cryp_read(cryp, cryp->caps->cr); in stm32_cryp_dma_callback()
936 if (is_gcm(cryp) || is_ccm(cryp)) { in stm32_cryp_dma_callback()
937 kfree(cryp->in_sg); in stm32_cryp_dma_callback()
938 kfree(cryp->out_sg); in stm32_cryp_dma_callback()
940 if (cryp->in_sg != cryp->req->src) in stm32_cryp_dma_callback()
941 kfree(cryp->in_sg); in stm32_cryp_dma_callback()
942 if (cryp->out_sg != cryp->req->dst) in stm32_cryp_dma_callback()
943 kfree(cryp->out_sg); in stm32_cryp_dma_callback()
946 if (cryp->payload_in) { in stm32_cryp_dma_callback()
947 stm32_cryp_write(cryp, cryp->caps->cr, reg | CR_CRYPEN); in stm32_cryp_dma_callback()
949 ret = stm32_cryp_wait_input(cryp); in stm32_cryp_dma_callback()
951 dev_err(cryp->dev, "input ready timeout after dma\n"); in stm32_cryp_dma_callback()
952 stm32_cryp_finish_req(cryp, ret); in stm32_cryp_dma_callback()
955 stm32_cryp_irq_write_data(cryp); in stm32_cryp_dma_callback()
957 ret = stm32_cryp_wait_output(cryp); in stm32_cryp_dma_callback()
959 dev_err(cryp->dev, "output ready timeout after dma\n"); in stm32_cryp_dma_callback()
960 stm32_cryp_finish_req(cryp, ret); in stm32_cryp_dma_callback()
963 stm32_cryp_irq_read_data(cryp); in stm32_cryp_dma_callback()
966 stm32_cryp_finish_req(cryp, 0); in stm32_cryp_dma_callback()
969 static int stm32_cryp_header_dma_start(struct stm32_cryp *cryp) in stm32_cryp_header_dma_start() argument
976 ret = dma_map_sg(cryp->dev, cryp->header_sg, cryp->header_sg_len, DMA_TO_DEVICE); in stm32_cryp_header_dma_start()
978 dev_err(cryp->dev, "dma_map_sg() error\n"); in stm32_cryp_header_dma_start()
982 dma_sync_sg_for_device(cryp->dev, cryp->header_sg, cryp->header_sg_len, DMA_TO_DEVICE); in stm32_cryp_header_dma_start()
984 tx_in = dmaengine_prep_slave_sg(cryp->dma_lch_in, cryp->header_sg, cryp->header_sg_len, in stm32_cryp_header_dma_start()
987 dev_err(cryp->dev, "IN prep_slave_sg() failed\n"); in stm32_cryp_header_dma_start()
991 tx_in->callback_param = cryp; in stm32_cryp_header_dma_start()
995 align_size = ALIGN_DOWN(cryp->header_in, cryp->hw_blocksize); in stm32_cryp_header_dma_start()
996 scatterwalk_skip(&cryp->in_walk, align_size); in stm32_cryp_header_dma_start()
997 cryp->header_in -= align_size; in stm32_cryp_header_dma_start()
1001 dev_err(cryp->dev, "DMA in submit failed\n"); in stm32_cryp_header_dma_start()
1004 dma_async_issue_pending(cryp->dma_lch_in); in stm32_cryp_header_dma_start()
1006 reg = stm32_cryp_read(cryp, cryp->caps->dmacr); in stm32_cryp_header_dma_start()
1007 stm32_cryp_write(cryp, cryp->caps->dmacr, reg | DMACR_DIEN); in stm32_cryp_header_dma_start()
1012 static int stm32_cryp_dma_start(struct stm32_cryp *cryp) in stm32_cryp_dma_start() argument
1019 if (cryp->in_sg != cryp->out_sg) { in stm32_cryp_dma_start()
1020 ret = dma_map_sg(cryp->dev, cryp->in_sg, cryp->in_sg_len, DMA_TO_DEVICE); in stm32_cryp_dma_start()
1022 dev_err(cryp->dev, "dma_map_sg() error\n"); in stm32_cryp_dma_start()
1027 ret = dma_map_sg(cryp->dev, cryp->out_sg, cryp->out_sg_len, DMA_FROM_DEVICE); in stm32_cryp_dma_start()
1029 dev_err(cryp->dev, "dma_map_sg() error\n"); in stm32_cryp_dma_start()
1033 dma_sync_sg_for_device(cryp->dev, cryp->in_sg, cryp->in_sg_len, DMA_TO_DEVICE); in stm32_cryp_dma_start()
1035 tx_in = dmaengine_prep_slave_sg(cryp->dma_lch_in, cryp->in_sg, cryp->in_sg_len, in stm32_cryp_dma_start()
1038 dev_err(cryp->dev, "IN prep_slave_sg() failed\n"); in stm32_cryp_dma_start()
1043 tx_in->callback_param = cryp; in stm32_cryp_dma_start()
1046 tx_out = dmaengine_prep_slave_sg(cryp->dma_lch_out, cryp->out_sg, cryp->out_sg_len, in stm32_cryp_dma_start()
1049 dev_err(cryp->dev, "OUT prep_slave_sg() failed\n"); in stm32_cryp_dma_start()
1053 reinit_completion(&cryp->dma_completion); in stm32_cryp_dma_start()
1055 tx_out->callback_param = cryp; in stm32_cryp_dma_start()
1058 align_size = ALIGN_DOWN(cryp->payload_in, cryp->hw_blocksize); in stm32_cryp_dma_start()
1059 scatterwalk_skip(&cryp->in_walk, align_size); in stm32_cryp_dma_start()
1060 cryp->payload_in -= align_size; in stm32_cryp_dma_start()
1064 dev_err(cryp->dev, "DMA in submit failed\n"); in stm32_cryp_dma_start()
1067 dma_async_issue_pending(cryp->dma_lch_in); in stm32_cryp_dma_start()
1070 scatterwalk_skip(&cryp->out_walk, align_size); in stm32_cryp_dma_start()
1071 cryp->payload_out -= align_size; in stm32_cryp_dma_start()
1074 dev_err(cryp->dev, "DMA out submit failed\n"); in stm32_cryp_dma_start()
1077 dma_async_issue_pending(cryp->dma_lch_out); in stm32_cryp_dma_start()
1079 reg = stm32_cryp_read(cryp, cryp->caps->dmacr); in stm32_cryp_dma_start()
1080 stm32_cryp_write(cryp, cryp->caps->dmacr, reg | DMACR_DOEN | DMACR_DIEN); in stm32_cryp_dma_start()
1082 if (!wait_for_completion_timeout(&cryp->dma_completion, msecs_to_jiffies(1000))) { in stm32_cryp_dma_start()
1083 dev_err(cryp->dev, "DMA out timed out\n"); in stm32_cryp_dma_start()
1084 dmaengine_terminate_sync(cryp->dma_lch_out); in stm32_cryp_dma_start()
1091 static int stm32_cryp_it_start(struct stm32_cryp *cryp) in stm32_cryp_it_start() argument
1094 stm32_cryp_write(cryp, cryp->caps->imsc, IMSCR_IN | IMSCR_OUT); in stm32_cryp_it_start()
1122 struct stm32_cryp *cryp = stm32_cryp_find_dev(ctx); in stm32_cryp_crypt() local
1124 if (!cryp) in stm32_cryp_crypt()
1129 return crypto_transfer_skcipher_request_to_engine(cryp->engine, req); in stm32_cryp_crypt()
1136 struct stm32_cryp *cryp = stm32_cryp_find_dev(ctx); in stm32_cryp_aead_crypt() local
1138 if (!cryp) in stm32_cryp_aead_crypt()
1143 return crypto_transfer_aead_request_to_engine(cryp->engine, req); in stm32_cryp_aead_crypt()
1446 static enum stm32_dma_mode stm32_cryp_dma_check(struct stm32_cryp *cryp, struct scatterlist *in_sg, in stm32_cryp_dma_check() argument
1451 if (!is_aes(cryp)) in stm32_cryp_dma_check()
1454 if (!cryp->dma_lch_in || !cryp->dma_lch_out) in stm32_cryp_dma_check()
1457 ret = stm32_cryp_dma_check_sg(in_sg, cryp->payload_in, AES_BLOCK_SIZE); in stm32_cryp_dma_check()
1461 ret = stm32_cryp_dma_check_sg(out_sg, cryp->payload_out, AES_BLOCK_SIZE); in stm32_cryp_dma_check()
1466 if (is_aes(cryp) && is_ctr(cryp)) { in stm32_cryp_dma_check()
1470 memcpy(&iv3, &cryp->req->iv[3 * sizeof(u32)], sizeof(iv3)); in stm32_cryp_dma_check()
1472 if ((c + cryp->payload_in) < cryp->payload_in) in stm32_cryp_dma_check()
1477 if (is_aes(cryp) && is_ctr(cryp) && ret == DMA_NEED_SG_TRUNC) in stm32_cryp_dma_check()
1542 static int stm32_cryp_cipher_prepare(struct stm32_cryp *cryp, struct scatterlist *in_sg, in stm32_cryp_cipher_prepare() argument
1548 cryp->dma_mode = stm32_cryp_dma_check(cryp, in_sg, out_sg); in stm32_cryp_cipher_prepare()
1550 scatterwalk_start(&cryp->in_walk, in_sg); in stm32_cryp_cipher_prepare()
1551 scatterwalk_start(&cryp->out_walk, out_sg); in stm32_cryp_cipher_prepare()
1553 if (cryp->dma_mode == NO_DMA) { in stm32_cryp_cipher_prepare()
1554 cryp->flags &= ~FLG_IN_OUT_DMA; in stm32_cryp_cipher_prepare()
1556 if (is_ctr(cryp)) in stm32_cryp_cipher_prepare()
1557 memset(cryp->last_ctr, 0, sizeof(cryp->last_ctr)); in stm32_cryp_cipher_prepare()
1559 } else if (cryp->dma_mode == DMA_NEED_SG_TRUNC) { in stm32_cryp_cipher_prepare()
1561 cryp->flags |= FLG_IN_OUT_DMA; in stm32_cryp_cipher_prepare()
1563 align_size = ALIGN_DOWN(cryp->payload_in, cryp->hw_blocksize); in stm32_cryp_cipher_prepare()
1564 ret = stm32_cryp_truncate_sg(&cryp->in_sg, &cryp->in_sg_len, in_sg, 0, align_size); in stm32_cryp_cipher_prepare()
1568 ret = stm32_cryp_truncate_sg(&cryp->out_sg, &cryp->out_sg_len, out_sg, 0, in stm32_cryp_cipher_prepare()
1571 kfree(cryp->in_sg); in stm32_cryp_cipher_prepare()
1575 cryp->flags |= FLG_IN_OUT_DMA; in stm32_cryp_cipher_prepare()
1577 cryp->in_sg = in_sg; in stm32_cryp_cipher_prepare()
1578 cryp->out_sg = out_sg; in stm32_cryp_cipher_prepare()
1580 ret = sg_nents_for_len(cryp->in_sg, cryp->payload_in); in stm32_cryp_cipher_prepare()
1583 cryp->in_sg_len = (size_t)ret; in stm32_cryp_cipher_prepare()
1585 ret = sg_nents_for_len(out_sg, cryp->payload_out); in stm32_cryp_cipher_prepare()
1588 cryp->out_sg_len = (size_t)ret; in stm32_cryp_cipher_prepare()
1594 static int stm32_cryp_aead_prepare(struct stm32_cryp *cryp, struct scatterlist *in_sg, in stm32_cryp_aead_prepare() argument
1601 cryp->header_sg = NULL; in stm32_cryp_aead_prepare()
1602 cryp->in_sg = NULL; in stm32_cryp_aead_prepare()
1603 cryp->out_sg = NULL; in stm32_cryp_aead_prepare()
1605 if (!cryp->dma_lch_in || !cryp->dma_lch_out) { in stm32_cryp_aead_prepare()
1606 cryp->dma_mode = NO_DMA; in stm32_cryp_aead_prepare()
1607 cryp->flags &= ~(FLG_IN_OUT_DMA | FLG_HEADER_DMA); in stm32_cryp_aead_prepare()
1613 skip = cryp->areq->assoclen - cryp->header_in; in stm32_cryp_aead_prepare()
1615 align_size = ALIGN_DOWN(cryp->header_in, cryp->hw_blocksize); in stm32_cryp_aead_prepare()
1616 ret = stm32_cryp_truncate_sg(&cryp->header_sg, &cryp->header_sg_len, in_sg, skip, in stm32_cryp_aead_prepare()
1621 ret = stm32_cryp_dma_check_sg(cryp->header_sg, align_size, AES_BLOCK_SIZE); in stm32_cryp_aead_prepare()
1624 kfree(cryp->header_sg); in stm32_cryp_aead_prepare()
1625 cryp->header_sg = NULL; in stm32_cryp_aead_prepare()
1627 cryp->flags &= ~FLG_HEADER_DMA; in stm32_cryp_aead_prepare()
1629 cryp->flags |= FLG_HEADER_DMA; in stm32_cryp_aead_prepare()
1633 skip = cryp->areq->assoclen; in stm32_cryp_aead_prepare()
1634 align_size = ALIGN_DOWN(cryp->payload_in, cryp->hw_blocksize); in stm32_cryp_aead_prepare()
1635 ret = stm32_cryp_truncate_sg(&cryp->in_sg, &cryp->in_sg_len, in_sg, skip, align_size); in stm32_cryp_aead_prepare()
1637 kfree(cryp->header_sg); in stm32_cryp_aead_prepare()
1642 ret = stm32_cryp_truncate_sg(&cryp->out_sg, &cryp->out_sg_len, out_sg, skip, align_size); in stm32_cryp_aead_prepare()
1644 kfree(cryp->header_sg); in stm32_cryp_aead_prepare()
1645 kfree(cryp->in_sg); in stm32_cryp_aead_prepare()
1649 ret = stm32_cryp_dma_check_sg(cryp->in_sg, align_size, AES_BLOCK_SIZE); in stm32_cryp_aead_prepare()
1650 ret2 = stm32_cryp_dma_check_sg(cryp->out_sg, align_size, AES_BLOCK_SIZE); in stm32_cryp_aead_prepare()
1652 kfree(cryp->in_sg); in stm32_cryp_aead_prepare()
1653 cryp->in_sg = NULL; in stm32_cryp_aead_prepare()
1655 kfree(cryp->out_sg); in stm32_cryp_aead_prepare()
1656 cryp->out_sg = NULL; in stm32_cryp_aead_prepare()
1658 cryp->flags &= ~FLG_IN_OUT_DMA; in stm32_cryp_aead_prepare()
1660 cryp->flags |= FLG_IN_OUT_DMA; in stm32_cryp_aead_prepare()
1670 struct stm32_cryp *cryp; in stm32_cryp_prepare_req() local
1681 cryp = ctx->cryp; in stm32_cryp_prepare_req()
1686 cryp->flags = (cryp->flags & ~FLG_MODE_MASK) | rctx->mode; in stm32_cryp_prepare_req()
1687 cryp->hw_blocksize = is_aes(cryp) ? AES_BLOCK_SIZE : DES_BLOCK_SIZE; in stm32_cryp_prepare_req()
1688 cryp->ctx = ctx; in stm32_cryp_prepare_req()
1691 cryp->req = req; in stm32_cryp_prepare_req()
1692 cryp->areq = NULL; in stm32_cryp_prepare_req()
1693 cryp->header_in = 0; in stm32_cryp_prepare_req()
1694 cryp->payload_in = req->cryptlen; in stm32_cryp_prepare_req()
1695 cryp->payload_out = req->cryptlen; in stm32_cryp_prepare_req()
1696 cryp->authsize = 0; in stm32_cryp_prepare_req()
1701 ret = stm32_cryp_cipher_prepare(cryp, in_sg, out_sg); in stm32_cryp_prepare_req()
1705 ret = stm32_cryp_hw_init(cryp); in stm32_cryp_prepare_req()
1723 cryp->areq = areq; in stm32_cryp_prepare_req()
1724 cryp->req = NULL; in stm32_cryp_prepare_req()
1725 cryp->authsize = crypto_aead_authsize(crypto_aead_reqtfm(areq)); in stm32_cryp_prepare_req()
1726 if (is_encrypt(cryp)) { in stm32_cryp_prepare_req()
1727 cryp->payload_in = areq->cryptlen; in stm32_cryp_prepare_req()
1728 cryp->header_in = areq->assoclen; in stm32_cryp_prepare_req()
1729 cryp->payload_out = areq->cryptlen; in stm32_cryp_prepare_req()
1731 cryp->payload_in = areq->cryptlen - cryp->authsize; in stm32_cryp_prepare_req()
1732 cryp->header_in = areq->assoclen; in stm32_cryp_prepare_req()
1733 cryp->payload_out = cryp->payload_in; in stm32_cryp_prepare_req()
1739 scatterwalk_start(&cryp->in_walk, in_sg); in stm32_cryp_prepare_req()
1741 scatterwalk_start_at_pos(&cryp->out_walk, out_sg, in stm32_cryp_prepare_req()
1744 ret = stm32_cryp_hw_init(cryp); in stm32_cryp_prepare_req()
1748 ret = stm32_cryp_aead_prepare(cryp, in_sg, out_sg); in stm32_cryp_prepare_req()
1761 struct stm32_cryp *cryp = ctx->cryp; in stm32_cryp_cipher_one_req() local
1764 if (!cryp) in stm32_cryp_cipher_one_req()
1771 if (cryp->flags & FLG_IN_OUT_DMA) in stm32_cryp_cipher_one_req()
1772 ret = stm32_cryp_dma_start(cryp); in stm32_cryp_cipher_one_req()
1774 ret = stm32_cryp_it_start(cryp); in stm32_cryp_cipher_one_req()
1777 stm32_cryp_finish_req(cryp, ret); in stm32_cryp_cipher_one_req()
1787 struct stm32_cryp *cryp = ctx->cryp; in stm32_cryp_aead_one_req() local
1790 if (!cryp) in stm32_cryp_aead_one_req()
1797 if (!stm32_cryp_get_input_text_len(cryp) && !cryp->header_in && in stm32_cryp_aead_one_req()
1798 !(cryp->flags & FLG_HEADER_DMA)) { in stm32_cryp_aead_one_req()
1800 stm32_cryp_finish_req(cryp, 0); in stm32_cryp_aead_one_req()
1804 if (cryp->flags & FLG_HEADER_DMA) in stm32_cryp_aead_one_req()
1805 return stm32_cryp_header_dma_start(cryp); in stm32_cryp_aead_one_req()
1807 if (!cryp->header_in && cryp->flags & FLG_IN_OUT_DMA) in stm32_cryp_aead_one_req()
1808 return stm32_cryp_dma_start(cryp); in stm32_cryp_aead_one_req()
1810 return stm32_cryp_it_start(cryp); in stm32_cryp_aead_one_req()
1813 static int stm32_cryp_read_auth_tag(struct stm32_cryp *cryp) in stm32_cryp_read_auth_tag() argument
1820 cfg = stm32_cryp_read(cryp, cryp->caps->cr); in stm32_cryp_read_auth_tag()
1827 stm32_cryp_write(cryp, cryp->caps->cr, cfg); in stm32_cryp_read_auth_tag()
1829 if (is_gcm(cryp)) { in stm32_cryp_read_auth_tag()
1831 size_bit = cryp->areq->assoclen * 8; in stm32_cryp_read_auth_tag()
1832 if (cryp->caps->swap_final) in stm32_cryp_read_auth_tag()
1835 stm32_cryp_write(cryp, cryp->caps->din, 0); in stm32_cryp_read_auth_tag()
1836 stm32_cryp_write(cryp, cryp->caps->din, size_bit); in stm32_cryp_read_auth_tag()
1838 size_bit = is_encrypt(cryp) ? cryp->areq->cryptlen : in stm32_cryp_read_auth_tag()
1839 cryp->areq->cryptlen - cryp->authsize; in stm32_cryp_read_auth_tag()
1841 if (cryp->caps->swap_final) in stm32_cryp_read_auth_tag()
1844 stm32_cryp_write(cryp, cryp->caps->din, 0); in stm32_cryp_read_auth_tag()
1845 stm32_cryp_write(cryp, cryp->caps->din, size_bit); in stm32_cryp_read_auth_tag()
1852 memcpy(iv, cryp->areq->iv, AES_BLOCK_SIZE); in stm32_cryp_read_auth_tag()
1858 if (!cryp->caps->padding_wa) in stm32_cryp_read_auth_tag()
1860 stm32_cryp_write(cryp, cryp->caps->din, xiv); in stm32_cryp_read_auth_tag()
1865 ret = stm32_cryp_wait_output(cryp); in stm32_cryp_read_auth_tag()
1867 dev_err(cryp->dev, "Timeout (read tag)\n"); in stm32_cryp_read_auth_tag()
1871 if (is_encrypt(cryp)) { in stm32_cryp_read_auth_tag()
1875 readsl(cryp->regs + cryp->caps->dout, out_tag, AES_BLOCK_32); in stm32_cryp_read_auth_tag()
1876 memcpy_to_scatterwalk(&cryp->out_walk, out_tag, cryp->authsize); in stm32_cryp_read_auth_tag()
1881 memcpy_from_scatterwalk(in_tag, &cryp->in_walk, cryp->authsize); in stm32_cryp_read_auth_tag()
1882 readsl(cryp->regs + cryp->caps->dout, out_tag, AES_BLOCK_32); in stm32_cryp_read_auth_tag()
1884 if (crypto_memneq(in_tag, out_tag, cryp->authsize)) in stm32_cryp_read_auth_tag()
1888 /* Disable cryp */ in stm32_cryp_read_auth_tag()
1890 stm32_cryp_write(cryp, cryp->caps->cr, cfg); in stm32_cryp_read_auth_tag()
1895 static void stm32_cryp_check_ctr_counter(struct stm32_cryp *cryp) in stm32_cryp_check_ctr_counter() argument
1899 if (unlikely(cryp->last_ctr[3] == cpu_to_be32(0xFFFFFFFF))) { in stm32_cryp_check_ctr_counter()
1904 crypto_inc((u8 *)cryp->last_ctr, sizeof(cryp->last_ctr)); in stm32_cryp_check_ctr_counter()
1906 cr = stm32_cryp_read(cryp, cryp->caps->cr); in stm32_cryp_check_ctr_counter()
1907 stm32_cryp_write(cryp, cryp->caps->cr, cr & ~CR_CRYPEN); in stm32_cryp_check_ctr_counter()
1909 stm32_cryp_hw_write_iv(cryp, cryp->last_ctr); in stm32_cryp_check_ctr_counter()
1911 stm32_cryp_write(cryp, cryp->caps->cr, cr); in stm32_cryp_check_ctr_counter()
1915 cryp->last_ctr[0] = cpu_to_be32(stm32_cryp_read(cryp, cryp->caps->iv0l)); in stm32_cryp_check_ctr_counter()
1916 cryp->last_ctr[1] = cpu_to_be32(stm32_cryp_read(cryp, cryp->caps->iv0r)); in stm32_cryp_check_ctr_counter()
1917 cryp->last_ctr[2] = cpu_to_be32(stm32_cryp_read(cryp, cryp->caps->iv1l)); in stm32_cryp_check_ctr_counter()
1918 cryp->last_ctr[3] = cpu_to_be32(stm32_cryp_read(cryp, cryp->caps->iv1r)); in stm32_cryp_check_ctr_counter()
1921 static void stm32_cryp_irq_read_data(struct stm32_cryp *cryp) in stm32_cryp_irq_read_data() argument
1925 readsl(cryp->regs + cryp->caps->dout, block, cryp->hw_blocksize / sizeof(u32)); in stm32_cryp_irq_read_data()
1926 memcpy_to_scatterwalk(&cryp->out_walk, block, min_t(size_t, cryp->hw_blocksize, in stm32_cryp_irq_read_data()
1927 cryp->payload_out)); in stm32_cryp_irq_read_data()
1928 cryp->payload_out -= min_t(size_t, cryp->hw_blocksize, in stm32_cryp_irq_read_data()
1929 cryp->payload_out); in stm32_cryp_irq_read_data()
1932 static void stm32_cryp_irq_write_block(struct stm32_cryp *cryp) in stm32_cryp_irq_write_block() argument
1936 memcpy_from_scatterwalk(block, &cryp->in_walk, min_t(size_t, cryp->hw_blocksize, in stm32_cryp_irq_write_block()
1937 cryp->payload_in)); in stm32_cryp_irq_write_block()
1938 writesl(cryp->regs + cryp->caps->din, block, cryp->hw_blocksize / sizeof(u32)); in stm32_cryp_irq_write_block()
1939 cryp->payload_in -= min_t(size_t, cryp->hw_blocksize, cryp->payload_in); in stm32_cryp_irq_write_block()
1942 static void stm32_cryp_irq_write_gcm_padded_data(struct stm32_cryp *cryp) in stm32_cryp_irq_write_gcm_padded_data() argument
1951 stm32_cryp_write(cryp, cryp->caps->imsc, 0); in stm32_cryp_irq_write_gcm_padded_data()
1952 cfg = stm32_cryp_read(cryp, cryp->caps->cr); in stm32_cryp_irq_write_gcm_padded_data()
1954 stm32_cryp_write(cryp, cryp->caps->cr, cfg); in stm32_cryp_irq_write_gcm_padded_data()
1957 stm32_cryp_write(cryp, cryp->caps->iv1r, cryp->gcm_ctr - 2); in stm32_cryp_irq_write_gcm_padded_data()
1962 stm32_cryp_write(cryp, cryp->caps->cr, cfg); in stm32_cryp_irq_write_gcm_padded_data()
1966 stm32_cryp_write(cryp, cryp->caps->cr, cfg); in stm32_cryp_irq_write_gcm_padded_data()
1969 stm32_cryp_irq_write_block(cryp); in stm32_cryp_irq_write_gcm_padded_data()
1971 err = stm32_cryp_wait_output(cryp); in stm32_cryp_irq_write_gcm_padded_data()
1973 dev_err(cryp->dev, "Timeout (write gcm last data)\n"); in stm32_cryp_irq_write_gcm_padded_data()
1974 return stm32_cryp_finish_req(cryp, err); in stm32_cryp_irq_write_gcm_padded_data()
1982 readsl(cryp->regs + cryp->caps->dout, block, cryp->hw_blocksize / sizeof(u32)); in stm32_cryp_irq_write_gcm_padded_data()
1984 memcpy_to_scatterwalk(&cryp->out_walk, block, min_t(size_t, cryp->hw_blocksize, in stm32_cryp_irq_write_gcm_padded_data()
1985 cryp->payload_out)); in stm32_cryp_irq_write_gcm_padded_data()
1986 cryp->payload_out -= min_t(size_t, cryp->hw_blocksize, in stm32_cryp_irq_write_gcm_padded_data()
1987 cryp->payload_out); in stm32_cryp_irq_write_gcm_padded_data()
1992 stm32_cryp_write(cryp, cryp->caps->cr, cfg); in stm32_cryp_irq_write_gcm_padded_data()
1997 stm32_cryp_write(cryp, cryp->caps->cr, cfg); in stm32_cryp_irq_write_gcm_padded_data()
2000 writesl(cryp->regs + cryp->caps->din, block, AES_BLOCK_32); in stm32_cryp_irq_write_gcm_padded_data()
2003 err = stm32_cryp_wait_output(cryp); in stm32_cryp_irq_write_gcm_padded_data()
2005 dev_err(cryp->dev, "Timeout (write gcm padded data)\n"); in stm32_cryp_irq_write_gcm_padded_data()
2006 return stm32_cryp_finish_req(cryp, err); in stm32_cryp_irq_write_gcm_padded_data()
2010 stm32_cryp_read(cryp, cryp->caps->dout); in stm32_cryp_irq_write_gcm_padded_data()
2013 stm32_cryp_finish_req(cryp, 0); in stm32_cryp_irq_write_gcm_padded_data()
2016 static void stm32_cryp_irq_set_npblb(struct stm32_cryp *cryp) in stm32_cryp_irq_set_npblb() argument
2021 cfg = stm32_cryp_read(cryp, cryp->caps->cr); in stm32_cryp_irq_set_npblb()
2023 stm32_cryp_write(cryp, cryp->caps->cr, cfg); in stm32_cryp_irq_set_npblb()
2025 cfg |= (cryp->hw_blocksize - cryp->payload_in) << CR_NBPBL_SHIFT; in stm32_cryp_irq_set_npblb()
2027 stm32_cryp_write(cryp, cryp->caps->cr, cfg); in stm32_cryp_irq_set_npblb()
2030 static void stm32_cryp_irq_write_ccm_padded_data(struct stm32_cryp *cryp) in stm32_cryp_irq_write_ccm_padded_data() argument
2041 stm32_cryp_write(cryp, cryp->caps->imsc, 0); in stm32_cryp_irq_write_ccm_padded_data()
2043 cfg = stm32_cryp_read(cryp, cryp->caps->cr); in stm32_cryp_irq_write_ccm_padded_data()
2045 stm32_cryp_write(cryp, cryp->caps->cr, cfg); in stm32_cryp_irq_write_ccm_padded_data()
2048 iv1tmp = stm32_cryp_read(cryp, CRYP_CSGCMCCM0R + 7 * 4); in stm32_cryp_irq_write_ccm_padded_data()
2052 cstmp1[i] = stm32_cryp_read(cryp, CRYP_CSGCMCCM0R + i * 4); in stm32_cryp_irq_write_ccm_padded_data()
2055 stm32_cryp_write(cryp, cryp->caps->iv1r, iv1tmp); in stm32_cryp_irq_write_ccm_padded_data()
2060 stm32_cryp_write(cryp, cryp->caps->cr, cfg); in stm32_cryp_irq_write_ccm_padded_data()
2064 stm32_cryp_write(cryp, cryp->caps->cr, cfg); in stm32_cryp_irq_write_ccm_padded_data()
2067 stm32_cryp_irq_write_block(cryp); in stm32_cryp_irq_write_ccm_padded_data()
2069 err = stm32_cryp_wait_output(cryp); in stm32_cryp_irq_write_ccm_padded_data()
2071 dev_err(cryp->dev, "Timeout (write ccm padded data)\n"); in stm32_cryp_irq_write_ccm_padded_data()
2072 return stm32_cryp_finish_req(cryp, err); in stm32_cryp_irq_write_ccm_padded_data()
2080 readsl(cryp->regs + cryp->caps->dout, block, cryp->hw_blocksize / sizeof(u32)); in stm32_cryp_irq_write_ccm_padded_data()
2082 memcpy_to_scatterwalk(&cryp->out_walk, block, min_t(size_t, cryp->hw_blocksize, in stm32_cryp_irq_write_ccm_padded_data()
2083 cryp->payload_out)); in stm32_cryp_irq_write_ccm_padded_data()
2084 cryp->payload_out -= min_t(size_t, cryp->hw_blocksize, cryp->payload_out); in stm32_cryp_irq_write_ccm_padded_data()
2088 cstmp2[i] = stm32_cryp_read(cryp, CRYP_CSGCMCCM0R + i * 4); in stm32_cryp_irq_write_ccm_padded_data()
2093 stm32_cryp_write(cryp, cryp->caps->cr, cfg); in stm32_cryp_irq_write_ccm_padded_data()
2098 stm32_cryp_write(cryp, cryp->caps->cr, cfg); in stm32_cryp_irq_write_ccm_padded_data()
2104 stm32_cryp_write(cryp, cryp->caps->din, block[i]); in stm32_cryp_irq_write_ccm_padded_data()
2108 err = stm32_cryp_wait_busy(cryp); in stm32_cryp_irq_write_ccm_padded_data()
2110 dev_err(cryp->dev, "Timeout (write ccm padded data)\n"); in stm32_cryp_irq_write_ccm_padded_data()
2113 stm32_cryp_finish_req(cryp, err); in stm32_cryp_irq_write_ccm_padded_data()
2116 static void stm32_cryp_irq_write_data(struct stm32_cryp *cryp) in stm32_cryp_irq_write_data() argument
2118 if (unlikely(!cryp->payload_in)) { in stm32_cryp_irq_write_data()
2119 dev_warn(cryp->dev, "No more data to process\n"); in stm32_cryp_irq_write_data()
2123 if (unlikely(cryp->payload_in < AES_BLOCK_SIZE && in stm32_cryp_irq_write_data()
2124 (stm32_cryp_get_hw_mode(cryp) == CR_AES_GCM) && in stm32_cryp_irq_write_data()
2125 is_encrypt(cryp))) { in stm32_cryp_irq_write_data()
2127 if (cryp->caps->padding_wa) { in stm32_cryp_irq_write_data()
2129 stm32_cryp_irq_write_gcm_padded_data(cryp); in stm32_cryp_irq_write_data()
2134 stm32_cryp_irq_set_npblb(cryp); in stm32_cryp_irq_write_data()
2137 if (unlikely((cryp->payload_in < AES_BLOCK_SIZE) && in stm32_cryp_irq_write_data()
2138 (stm32_cryp_get_hw_mode(cryp) == CR_AES_CCM) && in stm32_cryp_irq_write_data()
2139 is_decrypt(cryp))) { in stm32_cryp_irq_write_data()
2141 if (cryp->caps->padding_wa) { in stm32_cryp_irq_write_data()
2143 stm32_cryp_irq_write_ccm_padded_data(cryp); in stm32_cryp_irq_write_data()
2148 stm32_cryp_irq_set_npblb(cryp); in stm32_cryp_irq_write_data()
2151 if (is_aes(cryp) && is_ctr(cryp)) in stm32_cryp_irq_write_data()
2152 stm32_cryp_check_ctr_counter(cryp); in stm32_cryp_irq_write_data()
2154 stm32_cryp_irq_write_block(cryp); in stm32_cryp_irq_write_data()
2157 static void stm32_cryp_irq_write_gcmccm_header(struct stm32_cryp *cryp) in stm32_cryp_irq_write_gcmccm_header() argument
2162 written = min_t(size_t, AES_BLOCK_SIZE, cryp->header_in); in stm32_cryp_irq_write_gcmccm_header()
2164 memcpy_from_scatterwalk(block, &cryp->in_walk, written); in stm32_cryp_irq_write_gcmccm_header()
2166 writesl(cryp->regs + cryp->caps->din, block, AES_BLOCK_32); in stm32_cryp_irq_write_gcmccm_header()
2168 cryp->header_in -= written; in stm32_cryp_irq_write_gcmccm_header()
2170 stm32_crypt_gcmccm_end_header(cryp); in stm32_cryp_irq_write_gcmccm_header()
2175 struct stm32_cryp *cryp = arg; in stm32_cryp_irq_thread() local
2177 u32 it_mask = stm32_cryp_read(cryp, cryp->caps->imsc); in stm32_cryp_irq_thread()
2179 if (cryp->irq_status & MISR_OUT) in stm32_cryp_irq_thread()
2181 stm32_cryp_irq_read_data(cryp); in stm32_cryp_irq_thread()
2183 if (cryp->irq_status & MISR_IN) { in stm32_cryp_irq_thread()
2184 if (is_gcm(cryp) || is_ccm(cryp)) { in stm32_cryp_irq_thread()
2185 ph = stm32_cryp_read(cryp, cryp->caps->cr) & CR_PH_MASK; in stm32_cryp_irq_thread()
2188 stm32_cryp_irq_write_gcmccm_header(cryp); in stm32_cryp_irq_thread()
2191 stm32_cryp_irq_write_data(cryp); in stm32_cryp_irq_thread()
2192 if (is_gcm(cryp)) in stm32_cryp_irq_thread()
2193 cryp->gcm_ctr++; in stm32_cryp_irq_thread()
2196 stm32_cryp_irq_write_data(cryp); in stm32_cryp_irq_thread()
2201 if (!cryp->payload_in && !cryp->header_in) in stm32_cryp_irq_thread()
2203 if (!cryp->payload_out) in stm32_cryp_irq_thread()
2205 stm32_cryp_write(cryp, cryp->caps->imsc, it_mask); in stm32_cryp_irq_thread()
2207 if (!cryp->payload_in && !cryp->header_in && !cryp->payload_out) { in stm32_cryp_irq_thread()
2209 stm32_cryp_finish_req(cryp, 0); in stm32_cryp_irq_thread()
2218 struct stm32_cryp *cryp = arg; in stm32_cryp_irq() local
2220 cryp->irq_status = stm32_cryp_read(cryp, cryp->caps->mis); in stm32_cryp_irq()
2225 static int stm32_cryp_dma_init(struct stm32_cryp *cryp) in stm32_cryp_dma_init() argument
2234 dma_conf.dst_addr = cryp->phys_base + cryp->caps->din; in stm32_cryp_dma_init()
2239 chan = dma_request_chan(cryp->dev, "in"); in stm32_cryp_dma_init()
2243 cryp->dma_lch_in = chan; in stm32_cryp_dma_init()
2244 ret = dmaengine_slave_config(cryp->dma_lch_in, &dma_conf); in stm32_cryp_dma_init()
2246 dma_release_channel(cryp->dma_lch_in); in stm32_cryp_dma_init()
2247 cryp->dma_lch_in = NULL; in stm32_cryp_dma_init()
2248 dev_err(cryp->dev, "Couldn't configure DMA in slave.\n"); in stm32_cryp_dma_init()
2255 dma_conf.src_addr = cryp->phys_base + cryp->caps->dout; in stm32_cryp_dma_init()
2260 chan = dma_request_chan(cryp->dev, "out"); in stm32_cryp_dma_init()
2262 dma_release_channel(cryp->dma_lch_in); in stm32_cryp_dma_init()
2263 cryp->dma_lch_in = NULL; in stm32_cryp_dma_init()
2267 cryp->dma_lch_out = chan; in stm32_cryp_dma_init()
2269 ret = dmaengine_slave_config(cryp->dma_lch_out, &dma_conf); in stm32_cryp_dma_init()
2271 dma_release_channel(cryp->dma_lch_out); in stm32_cryp_dma_init()
2272 cryp->dma_lch_out = NULL; in stm32_cryp_dma_init()
2273 dev_err(cryp->dev, "Couldn't configure DMA out slave.\n"); in stm32_cryp_dma_init()
2274 dma_release_channel(cryp->dma_lch_in); in stm32_cryp_dma_init()
2275 cryp->dma_lch_in = NULL; in stm32_cryp_dma_init()
2279 init_completion(&cryp->dma_completion); in stm32_cryp_dma_init()
2564 { .compatible = "stericsson,ux500-cryp", .data = &ux500_data},
2565 { .compatible = "st,stm32f756-cryp", .data = &f7_data},
2566 { .compatible = "st,stm32mp1-cryp", .data = &mp1_data},
2574 struct stm32_cryp *cryp; in stm32_cryp_probe() local
2578 cryp = devm_kzalloc(dev, sizeof(*cryp), GFP_KERNEL); in stm32_cryp_probe()
2579 if (!cryp) in stm32_cryp_probe()
2582 cryp->caps = of_device_get_match_data(dev); in stm32_cryp_probe()
2583 if (!cryp->caps) in stm32_cryp_probe()
2586 cryp->dev = dev; in stm32_cryp_probe()
2588 cryp->regs = devm_platform_ioremap_resource(pdev, 0); in stm32_cryp_probe()
2589 if (IS_ERR(cryp->regs)) in stm32_cryp_probe()
2590 return PTR_ERR(cryp->regs); in stm32_cryp_probe()
2592 cryp->phys_base = platform_get_resource(pdev, IORESOURCE_MEM, 0)->start; in stm32_cryp_probe()
2600 dev_name(dev), cryp); in stm32_cryp_probe()
2606 cryp->clk = devm_clk_get(dev, NULL); in stm32_cryp_probe()
2607 if (IS_ERR(cryp->clk)) { in stm32_cryp_probe()
2608 dev_err_probe(dev, PTR_ERR(cryp->clk), "Could not get clock\n"); in stm32_cryp_probe()
2610 return PTR_ERR(cryp->clk); in stm32_cryp_probe()
2613 ret = clk_prepare_enable(cryp->clk); in stm32_cryp_probe()
2615 dev_err(cryp->dev, "Failed to enable clock\n"); in stm32_cryp_probe()
2637 platform_set_drvdata(pdev, cryp); in stm32_cryp_probe()
2639 ret = stm32_cryp_dma_init(cryp); in stm32_cryp_probe()
2651 list_add(&cryp->list, &cryp_list.dev_list); in stm32_cryp_probe()
2655 cryp->engine = crypto_engine_alloc_init(dev, 1); in stm32_cryp_probe()
2656 if (!cryp->engine) { in stm32_cryp_probe()
2662 ret = crypto_engine_start(cryp->engine); in stm32_cryp_probe()
2674 if (cryp->caps->aeads_support) { in stm32_cryp_probe()
2690 crypto_engine_exit(cryp->engine); in stm32_cryp_probe()
2693 list_del(&cryp->list); in stm32_cryp_probe()
2696 if (cryp->dma_lch_in) in stm32_cryp_probe()
2697 dma_release_channel(cryp->dma_lch_in); in stm32_cryp_probe()
2698 if (cryp->dma_lch_out) in stm32_cryp_probe()
2699 dma_release_channel(cryp->dma_lch_out); in stm32_cryp_probe()
2705 clk_disable_unprepare(cryp->clk); in stm32_cryp_probe()
2712 struct stm32_cryp *cryp = platform_get_drvdata(pdev); in stm32_cryp_remove() local
2715 ret = pm_runtime_get_sync(cryp->dev); in stm32_cryp_remove()
2717 if (cryp->caps->aeads_support) in stm32_cryp_remove()
2721 crypto_engine_exit(cryp->engine); in stm32_cryp_remove()
2724 list_del(&cryp->list); in stm32_cryp_remove()
2727 if (cryp->dma_lch_in) in stm32_cryp_remove()
2728 dma_release_channel(cryp->dma_lch_in); in stm32_cryp_remove()
2730 if (cryp->dma_lch_out) in stm32_cryp_remove()
2731 dma_release_channel(cryp->dma_lch_out); in stm32_cryp_remove()
2733 pm_runtime_disable(cryp->dev); in stm32_cryp_remove()
2734 pm_runtime_put_noidle(cryp->dev); in stm32_cryp_remove()
2737 clk_disable_unprepare(cryp->clk); in stm32_cryp_remove()
2743 struct stm32_cryp *cryp = dev_get_drvdata(dev); in stm32_cryp_runtime_suspend() local
2745 clk_disable_unprepare(cryp->clk); in stm32_cryp_runtime_suspend()
2752 struct stm32_cryp *cryp = dev_get_drvdata(dev); in stm32_cryp_runtime_resume() local
2755 ret = clk_prepare_enable(cryp->clk); in stm32_cryp_runtime_resume()
2757 dev_err(cryp->dev, "Failed to prepare_enable clock\n"); in stm32_cryp_runtime_resume()
2785 MODULE_DESCRIPTION("STMicrolectronics STM32 CRYP hardware driver");