170 	dev->pdr = dma_alloc_coherent(dev->core_dev->device,  in crypto4xx_build_pdr()
178 		dma_free_coherent(dev->core_dev->device,  in crypto4xx_build_pdr()
184 	dev->shadow_sa_pool = dma_alloc_coherent(dev->core_dev->device,  in crypto4xx_build_pdr()
191 	dev->shadow_sr_pool = dma_alloc_coherent(dev->core_dev->device,  in crypto4xx_build_pdr()
218 		dma_free_coherent(dev->core_dev->device,  in crypto4xx_destroy_pdr()
223 		dma_free_coherent(dev->core_dev->device,  in crypto4xx_destroy_pdr()
228 		dma_free_coherent(dev->core_dev->device,  in crypto4xx_destroy_pdr()
257 	spin_lock_irqsave(&dev->core_dev->lock, flags);  in crypto4xx_put_pd_to_pdr()
265 	spin_unlock_irqrestore(&dev->core_dev->lock, flags);  in crypto4xx_put_pd_to_pdr()
277 	dev->gdr = dma_alloc_coherent(dev->core_dev->device,  in crypto4xx_build_gdr()
289 		dma_free_coherent(dev->core_dev->device,  in crypto4xx_destroy_gdr()
324 	spin_lock_irqsave(&dev->core_dev->lock, flags);  in crypto4xx_put_gd_to_gdr()
326 		spin_unlock_irqrestore(&dev->core_dev->lock, flags);  in crypto4xx_put_gd_to_gdr()
335 	spin_unlock_irqrestore(&dev->core_dev->lock, flags);  in crypto4xx_put_gd_to_gdr()
358 		dma_alloc_coherent(dev->core_dev->device,  in crypto4xx_build_sdr()
365 	dev->sdr = dma_alloc_coherent(dev->core_dev->device,  in crypto4xx_build_sdr()
382 		dma_free_coherent(dev->core_dev->device,  in crypto4xx_destroy_sdr()
387 		dma_free_coherent(dev->core_dev->device,  in crypto4xx_destroy_sdr()
423 	spin_lock_irqsave(&dev->core_dev->lock, flags);  in crypto4xx_put_sd_to_sdr()
425 		spin_unlock_irqrestore(&dev->core_dev->lock, flags);  in crypto4xx_put_sd_to_sdr()
432 	spin_unlock_irqrestore(&dev->core_dev->lock, flags);  in crypto4xx_put_sd_to_sdr()
520 		dma_unmap_page(dev->core_dev->device, pd->dest, dst->length,  in crypto4xx_cipher_done()
556 		dma_unmap_page(dev->core_dev->device, pd->dest, dst->length,  in crypto4xx_aead_done()
618 static void crypto4xx_stop_all(struct crypto4xx_core_device *core_dev)  in crypto4xx_stop_all()  argument
620 	crypto4xx_destroy_pdr(core_dev->dev);  in crypto4xx_stop_all()
621 	crypto4xx_destroy_gdr(core_dev->dev);  in crypto4xx_stop_all()
622 	crypto4xx_destroy_sdr(core_dev->dev);  in crypto4xx_stop_all()
687 		dev_err(dev->core_dev->device, "Invalid number of src SG.\n");  in crypto4xx_build_pd()
718 	spin_lock_irqsave(&dev->core_dev->lock, flags);  in crypto4xx_build_pd()
739 			spin_unlock_irqrestore(&dev->core_dev->lock, flags);  in crypto4xx_build_pd()
747 			spin_unlock_irqrestore(&dev->core_dev->lock, flags);  in crypto4xx_build_pd()
756 			spin_unlock_irqrestore(&dev->core_dev->lock, flags);  in crypto4xx_build_pd()
766 		spin_unlock_irqrestore(&dev->core_dev->lock, flags);  in crypto4xx_build_pd()
769 	spin_unlock_irqrestore(&dev->core_dev->lock, flags);  in crypto4xx_build_pd()
808 			gd->ptr = dma_map_page(dev->core_dev->device,  in crypto4xx_build_pd()
822 		pd->src = (u32)dma_map_page(dev->core_dev->device, sg_page(src),  in crypto4xx_build_pd()
841 		pd->dest = (u32)dma_map_page(dev->core_dev->device,  in crypto4xx_build_pd()
1031 	struct crypto4xx_core_device *core_dev = dev_get_drvdata(dev);  in crypto4xx_bh_tasklet_cb()  local
1034 	u32 tail = core_dev->dev->pdr_tail;  in crypto4xx_bh_tasklet_cb()
1035 	u32 head = core_dev->dev->pdr_head;  in crypto4xx_bh_tasklet_cb()
1038 		pd_uinfo = &core_dev->dev->pdr_uinfo[tail];  in crypto4xx_bh_tasklet_cb()
1039 		pd = &core_dev->dev->pdr[tail];  in crypto4xx_bh_tasklet_cb()
1044 			crypto4xx_pd_done(core_dev->dev, tail);  in crypto4xx_bh_tasklet_cb()
1045 			tail = crypto4xx_put_pd_to_pdr(core_dev->dev, tail);  in crypto4xx_bh_tasklet_cb()
1060 	struct crypto4xx_core_device *core_dev = dev_get_drvdata(dev);  in crypto4xx_interrupt_handler()  local
1062 	writel(clr_val, core_dev->dev->ce_base + CRYPTO4XX_INT_CLR);  in crypto4xx_interrupt_handler()
1063 	tasklet_schedule(&core_dev->tasklet);  in crypto4xx_interrupt_handler()
1129 	mutex_lock(&dev->core_dev->rng_lock);  in crypto4xx_prng_generate()
1131 	mutex_unlock(&dev->core_dev->rng_lock);  in crypto4xx_prng_generate()
1292 	struct crypto4xx_core_device *core_dev;  in crypto4xx_probe()  local
1327 	core_dev = devm_kzalloc(  in crypto4xx_probe()
1329 	if (!core_dev)  in crypto4xx_probe()
1332 	dev_set_drvdata(dev, core_dev);  in crypto4xx_probe()
1333 	core_dev->ofdev = ofdev;  in crypto4xx_probe()
1334 	core_dev->dev = devm_kzalloc(  in crypto4xx_probe()
1336 	if (!core_dev->dev)  in crypto4xx_probe()
1353 	core_dev->dev->core_dev = core_dev;  in crypto4xx_probe()
1354 	core_dev->dev->is_revb = is_revb;  in crypto4xx_probe()
1355 	core_dev->device = dev;  in crypto4xx_probe()
1356 	rc = devm_mutex_init(&ofdev->dev, &core_dev->rng_lock);  in crypto4xx_probe()
1359 	spin_lock_init(&core_dev->lock);  in crypto4xx_probe()
1360 	INIT_LIST_HEAD(&core_dev->dev->alg_list);  in crypto4xx_probe()
1361 	ratelimit_default_init(&core_dev->dev->aead_ratelimit);  in crypto4xx_probe()
1362 	rc = crypto4xx_build_sdr(core_dev->dev);  in crypto4xx_probe()
1365 	rc = crypto4xx_build_pdr(core_dev->dev);  in crypto4xx_probe()
1369 	rc = crypto4xx_build_gdr(core_dev->dev);  in crypto4xx_probe()
1374 	tasklet_init(&core_dev->tasklet, crypto4xx_bh_tasklet_cb,  in crypto4xx_probe()
1377 	core_dev->dev->ce_base = devm_platform_ioremap_resource(ofdev, 0);  in crypto4xx_probe()
1378 	if (IS_ERR(core_dev->dev->ce_base)) {  in crypto4xx_probe()
1380 		rc = PTR_ERR(core_dev->dev->ce_base);  in crypto4xx_probe()
1385 	core_dev->irq = irq_of_parse_and_map(ofdev->dev.of_node, 0);  in crypto4xx_probe()
1386 	rc = devm_request_irq(&ofdev->dev, core_dev->irq,  in crypto4xx_probe()
1394 	crypto4xx_hw_init(core_dev->dev);  in crypto4xx_probe()
1397 	rc = crypto4xx_register_alg(core_dev->dev, crypto4xx_alg,  in crypto4xx_probe()
1402 	ppc4xx_trng_probe(core_dev);  in crypto4xx_probe()
1406 	tasklet_kill(&core_dev->tasklet);  in crypto4xx_probe()
1408 	crypto4xx_destroy_sdr(core_dev->dev);  in crypto4xx_probe()
1409 	crypto4xx_destroy_gdr(core_dev->dev);  in crypto4xx_probe()
1410 	crypto4xx_destroy_pdr(core_dev->dev);  in crypto4xx_probe()
1417 	struct crypto4xx_core_device *core_dev = dev_get_drvdata(dev);  in crypto4xx_remove()  local
1419 	ppc4xx_trng_remove(core_dev);  in crypto4xx_remove()
1421 	tasklet_kill(&core_dev->tasklet);  in crypto4xx_remove()
1423 	crypto4xx_unregister_alg(core_dev->dev);  in crypto4xx_remove()
1425 	crypto4xx_stop_all(core_dev);  in crypto4xx_remove()