Lines Matching refs:cdd
88 struct cppi41_dd *cdd; member
254 static struct cppi41_channel *desc_to_chan(struct cppi41_dd *cdd, u32 desc) in desc_to_chan() argument
262 if (!((desc >= cdd->descs_phys) && in desc_to_chan()
263 (desc < (cdd->descs_phys + descs_size)))) { in desc_to_chan()
267 desc_num = (desc - cdd->descs_phys) / sizeof(struct cppi41_desc); in desc_to_chan()
269 c = cdd->chan_busy[desc_num]; in desc_to_chan()
270 cdd->chan_busy[desc_num] = NULL; in desc_to_chan()
273 pm_runtime_put(cdd->ddev.dev); in desc_to_chan()
293 static u32 cppi41_pop_desc(struct cppi41_dd *cdd, unsigned queue_num) in cppi41_pop_desc() argument
297 desc = cppi_readl(cdd->qmgr_mem + QMGR_QUEUE_D(queue_num)); in cppi41_pop_desc()
304 struct cppi41_dd *cdd = data; in cppi41_irq() local
305 u16 first_completion_queue = cdd->first_completion_queue; in cppi41_irq()
306 u16 qmgr_num_pend = cdd->qmgr_num_pend; in cppi41_irq()
315 val = cppi_readl(cdd->qmgr_mem + QMGR_PEND(i)); in cppi41_irq()
336 WARN_ON(cdd->is_suspended); in cppi41_irq()
341 desc = cppi41_pop_desc(cdd, q_num); in cppi41_irq()
342 c = desc_to_chan(cdd, desc); in cppi41_irq()
374 struct cppi41_dd *cdd = c->cdd; in cppi41_dma_alloc_chan_resources() local
377 error = pm_runtime_get_sync(cdd->ddev.dev); in cppi41_dma_alloc_chan_resources()
379 dev_err(cdd->ddev.dev, "%s pm runtime get: %i\n", in cppi41_dma_alloc_chan_resources()
381 pm_runtime_put_noidle(cdd->ddev.dev); in cppi41_dma_alloc_chan_resources()
393 pm_runtime_put_autosuspend(cdd->ddev.dev); in cppi41_dma_alloc_chan_resources()
401 struct cppi41_dd *cdd = c->cdd; in cppi41_dma_free_chan_resources() local
404 error = pm_runtime_get_sync(cdd->ddev.dev); in cppi41_dma_free_chan_resources()
406 pm_runtime_put_noidle(cdd->ddev.dev); in cppi41_dma_free_chan_resources()
411 WARN_ON(!list_empty(&cdd->pending)); in cppi41_dma_free_chan_resources()
413 pm_runtime_put_autosuspend(cdd->ddev.dev); in cppi41_dma_free_chan_resources()
431 struct cppi41_dd *cdd = c->cdd; in push_desc_queue() local
461 pm_runtime_get(cdd->ddev.dev); in push_desc_queue()
464 desc_num = (desc_phys - cdd->descs_phys) / sizeof(struct cppi41_desc); in push_desc_queue()
465 WARN_ON(cdd->chan_busy[desc_num]); in push_desc_queue()
466 cdd->chan_busy[desc_num] = c; in push_desc_queue()
470 cppi_writel(reg, cdd->qmgr_mem + QMGR_QUEUE_D(c->q_num)); in push_desc_queue()
478 static void cppi41_run_queue(struct cppi41_dd *cdd) in cppi41_run_queue() argument
482 list_for_each_entry_safe(c, _c, &cdd->pending, node) { in cppi41_run_queue()
491 struct cppi41_dd *cdd = c->cdd; in cppi41_dma_issue_pending() local
495 error = pm_runtime_get(cdd->ddev.dev); in cppi41_dma_issue_pending()
497 pm_runtime_put_noidle(cdd->ddev.dev); in cppi41_dma_issue_pending()
498 dev_err(cdd->ddev.dev, "Failed to pm_runtime_get: %i\n", in cppi41_dma_issue_pending()
504 spin_lock_irqsave(&cdd->lock, flags); in cppi41_dma_issue_pending()
505 list_add_tail(&c->node, &cdd->pending); in cppi41_dma_issue_pending()
506 if (!cdd->is_suspended) in cppi41_dma_issue_pending()
507 cppi41_run_queue(cdd); in cppi41_dma_issue_pending()
508 spin_unlock_irqrestore(&cdd->lock, flags); in cppi41_dma_issue_pending()
510 pm_runtime_put_autosuspend(cdd->ddev.dev); in cppi41_dma_issue_pending()
587 struct cppi41_dd *cdd = c->cdd; in cppi41_dma_prep_slave_sg() local
593 error = pm_runtime_get(cdd->ddev.dev); in cppi41_dma_prep_slave_sg()
595 pm_runtime_put_noidle(cdd->ddev.dev); in cppi41_dma_prep_slave_sg()
600 if (cdd->is_suspended) in cppi41_dma_prep_slave_sg()
627 pm_runtime_put_autosuspend(cdd->ddev.dev); in cppi41_dma_prep_slave_sg()
640 struct cppi41_dd *cdd = c->cdd; in cppi41_tear_down_chan() local
646 td = cdd->cd; in cppi41_tear_down_chan()
647 td += cdd->first_td_desc; in cppi41_tear_down_chan()
649 td_desc_phys = cdd->descs_phys; in cppi41_tear_down_chan()
650 td_desc_phys += cdd->first_td_desc * sizeof(struct cppi41_desc); in cppi41_tear_down_chan()
658 cppi_writel(reg, cdd->qmgr_mem + in cppi41_tear_down_chan()
659 QMGR_QUEUE_D(cdd->td_queue.submit)); in cppi41_tear_down_chan()
665 reg |= cdd->td_queue.complete; in cppi41_tear_down_chan()
675 desc_phys = cppi41_pop_desc(cdd, cdd->td_queue.complete); in cppi41_tear_down_chan()
677 desc_phys = cppi41_pop_desc(cdd, c->q_comp_num); in cppi41_tear_down_chan()
713 desc_phys = cppi41_pop_desc(cdd, c->q_num); in cppi41_tear_down_chan()
715 desc_phys = cppi41_pop_desc(cdd, c->q_comp_num); in cppi41_tear_down_chan()
735 struct cppi41_dd *cdd = c->cdd; in cppi41_stop_chan() local
741 desc_num = (desc_phys - cdd->descs_phys) / sizeof(struct cppi41_desc); in cppi41_stop_chan()
742 if (!cdd->chan_busy[desc_num]) { in cppi41_stop_chan()
750 list_for_each_entry_safe(cc, _ct, &cdd->pending, node) { in cppi41_stop_chan()
763 WARN_ON(!cdd->chan_busy[desc_num]); in cppi41_stop_chan()
764 cdd->chan_busy[desc_num] = NULL; in cppi41_stop_chan()
767 pm_runtime_put(cdd->ddev.dev); in cppi41_stop_chan()
772 static int cppi41_add_chans(struct device *dev, struct cppi41_dd *cdd) in cppi41_add_chans() argument
776 u32 n_chans = cdd->n_chans; in cppi41_add_chans()
791 cchan->cdd = cdd; in cppi41_add_chans()
793 cchan->gcr_reg = cdd->ctrl_mem + DMA_TXGCR(i >> 1); in cppi41_add_chans()
796 cchan->gcr_reg = cdd->ctrl_mem + DMA_RXGCR(i >> 1); in cppi41_add_chans()
800 cchan->desc = &cdd->cd[i]; in cppi41_add_chans()
801 cchan->desc_phys = cdd->descs_phys; in cppi41_add_chans()
803 cchan->chan.device = &cdd->ddev; in cppi41_add_chans()
804 list_add_tail(&cchan->chan.device_node, &cdd->ddev.channels); in cppi41_add_chans()
806 cdd->first_td_desc = n_chans; in cppi41_add_chans()
811 static void purge_descs(struct device *dev, struct cppi41_dd *cdd) in purge_descs() argument
820 cppi_writel(0, cdd->qmgr_mem + QMGR_MEMBASE(i)); in purge_descs()
821 cppi_writel(0, cdd->qmgr_mem + QMGR_MEMCTRL(i)); in purge_descs()
823 dma_free_coherent(dev, mem_decs, cdd->cd, in purge_descs()
824 cdd->descs_phys); in purge_descs()
828 static void disable_sched(struct cppi41_dd *cdd) in disable_sched() argument
830 cppi_writel(0, cdd->sched_mem + DMA_SCHED_CTRL); in disable_sched()
833 static void deinit_cppi41(struct device *dev, struct cppi41_dd *cdd) in deinit_cppi41() argument
835 disable_sched(cdd); in deinit_cppi41()
837 purge_descs(dev, cdd); in deinit_cppi41()
839 cppi_writel(0, cdd->qmgr_mem + QMGR_LRAM0_BASE); in deinit_cppi41()
840 cppi_writel(0, cdd->qmgr_mem + QMGR_LRAM0_BASE); in deinit_cppi41()
841 dma_free_coherent(dev, QMGR_SCRATCH_SIZE, cdd->qmgr_scratch, in deinit_cppi41()
842 cdd->scratch_phys); in deinit_cppi41()
845 static int init_descs(struct device *dev, struct cppi41_dd *cdd) in init_descs() argument
869 cdd->cd = dma_alloc_coherent(dev, mem_decs, in init_descs()
870 &cdd->descs_phys, GFP_KERNEL); in init_descs()
871 if (!cdd->cd) in init_descs()
874 cppi_writel(cdd->descs_phys, cdd->qmgr_mem + QMGR_MEMBASE(i)); in init_descs()
875 cppi_writel(reg, cdd->qmgr_mem + QMGR_MEMCTRL(i)); in init_descs()
882 static void init_sched(struct cppi41_dd *cdd) in init_sched() argument
889 cppi_writel(0, cdd->sched_mem + DMA_SCHED_CTRL); in init_sched()
890 for (ch = 0; ch < cdd->n_chans; ch += 2) { in init_sched()
897 cppi_writel(reg, cdd->sched_mem + DMA_SCHED_WORD(word)); in init_sched()
900 reg = cdd->n_chans * 2 - 1; in init_sched()
902 cppi_writel(reg, cdd->sched_mem + DMA_SCHED_CTRL); in init_sched()
905 static int init_cppi41(struct device *dev, struct cppi41_dd *cdd) in init_cppi41() argument
910 cdd->qmgr_scratch = dma_alloc_coherent(dev, QMGR_SCRATCH_SIZE, in init_cppi41()
911 &cdd->scratch_phys, GFP_KERNEL); in init_cppi41()
912 if (!cdd->qmgr_scratch) in init_cppi41()
915 cppi_writel(cdd->scratch_phys, cdd->qmgr_mem + QMGR_LRAM0_BASE); in init_cppi41()
916 cppi_writel(TOTAL_DESCS_NUM, cdd->qmgr_mem + QMGR_LRAM_SIZE); in init_cppi41()
917 cppi_writel(0, cdd->qmgr_mem + QMGR_LRAM1_BASE); in init_cppi41()
919 ret = init_descs(dev, cdd); in init_cppi41()
923 cppi_writel(cdd->td_queue.submit, cdd->ctrl_mem + DMA_TDFDQ); in init_cppi41()
924 init_sched(cdd); in init_cppi41()
928 deinit_cppi41(dev, cdd); in init_cppi41()
945 struct cppi41_dd *cdd; in cpp41_dma_filter_fn() local
959 cdd = cchan->cdd; in cpp41_dma_filter_fn()
961 queues = cdd->queues_tx; in cpp41_dma_filter_fn()
963 queues = cdd->queues_rx; in cpp41_dma_filter_fn()
1035 struct cppi41_dd *cdd; in cppi41_dma_probe() local
1046 cdd = devm_kzalloc(&pdev->dev, sizeof(*cdd), GFP_KERNEL); in cppi41_dma_probe()
1047 if (!cdd) in cppi41_dma_probe()
1050 dma_cap_set(DMA_SLAVE, cdd->ddev.cap_mask); in cppi41_dma_probe()
1051 cdd->ddev.device_alloc_chan_resources = cppi41_dma_alloc_chan_resources; in cppi41_dma_probe()
1052 cdd->ddev.device_free_chan_resources = cppi41_dma_free_chan_resources; in cppi41_dma_probe()
1053 cdd->ddev.device_tx_status = cppi41_dma_tx_status; in cppi41_dma_probe()
1054 cdd->ddev.device_issue_pending = cppi41_dma_issue_pending; in cppi41_dma_probe()
1055 cdd->ddev.device_prep_slave_sg = cppi41_dma_prep_slave_sg; in cppi41_dma_probe()
1056 cdd->ddev.device_terminate_all = cppi41_stop_chan; in cppi41_dma_probe()
1057 cdd->ddev.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV); in cppi41_dma_probe()
1058 cdd->ddev.src_addr_widths = CPPI41_DMA_BUSWIDTHS; in cppi41_dma_probe()
1059 cdd->ddev.dst_addr_widths = CPPI41_DMA_BUSWIDTHS; in cppi41_dma_probe()
1060 cdd->ddev.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST; in cppi41_dma_probe()
1061 cdd->ddev.dev = dev; in cppi41_dma_probe()
1062 INIT_LIST_HEAD(&cdd->ddev.channels); in cppi41_dma_probe()
1063 cpp41_dma_info.dma_cap = cdd->ddev.cap_mask; in cppi41_dma_probe()
1070 cdd->ctrl_mem = devm_platform_ioremap_resource(pdev, index); in cppi41_dma_probe()
1071 if (IS_ERR(cdd->ctrl_mem)) in cppi41_dma_probe()
1072 return PTR_ERR(cdd->ctrl_mem); in cppi41_dma_probe()
1074 cdd->sched_mem = devm_platform_ioremap_resource(pdev, index + 1); in cppi41_dma_probe()
1075 if (IS_ERR(cdd->sched_mem)) in cppi41_dma_probe()
1076 return PTR_ERR(cdd->sched_mem); in cppi41_dma_probe()
1078 cdd->qmgr_mem = devm_platform_ioremap_resource(pdev, index + 2); in cppi41_dma_probe()
1079 if (IS_ERR(cdd->qmgr_mem)) in cppi41_dma_probe()
1080 return PTR_ERR(cdd->qmgr_mem); in cppi41_dma_probe()
1082 spin_lock_init(&cdd->lock); in cppi41_dma_probe()
1083 INIT_LIST_HEAD(&cdd->pending); in cppi41_dma_probe()
1085 platform_set_drvdata(pdev, cdd); in cppi41_dma_probe()
1094 cdd->queues_rx = glue_info->queues_rx; in cppi41_dma_probe()
1095 cdd->queues_tx = glue_info->queues_tx; in cppi41_dma_probe()
1096 cdd->td_queue = glue_info->td_queue; in cppi41_dma_probe()
1097 cdd->qmgr_num_pend = glue_info->qmgr_num_pend; in cppi41_dma_probe()
1098 cdd->first_completion_queue = glue_info->first_completion_queue; in cppi41_dma_probe()
1102 "dma-channels", &cdd->n_chans); in cppi41_dma_probe()
1105 "#dma-channels", &cdd->n_chans); in cppi41_dma_probe()
1109 ret = init_cppi41(dev, cdd); in cppi41_dma_probe()
1113 ret = cppi41_add_chans(dev, cdd); in cppi41_dma_probe()
1124 dev_name(dev), cdd); in cppi41_dma_probe()
1127 cdd->irq = irq; in cppi41_dma_probe()
1129 ret = dma_async_device_register(&cdd->ddev); in cppi41_dma_probe()
1142 dma_async_device_unregister(&cdd->ddev); in cppi41_dma_probe()
1144 deinit_cppi41(dev, cdd); in cppi41_dma_probe()
1156 struct cppi41_dd *cdd = platform_get_drvdata(pdev); in cppi41_dma_remove() local
1164 dma_async_device_unregister(&cdd->ddev); in cppi41_dma_remove()
1166 devm_free_irq(&pdev->dev, cdd->irq, cdd); in cppi41_dma_remove()
1167 deinit_cppi41(&pdev->dev, cdd); in cppi41_dma_remove()
1175 struct cppi41_dd *cdd = dev_get_drvdata(dev); in cppi41_suspend() local
1177 cdd->dma_tdfdq = cppi_readl(cdd->ctrl_mem + DMA_TDFDQ); in cppi41_suspend()
1178 disable_sched(cdd); in cppi41_suspend()
1185 struct cppi41_dd *cdd = dev_get_drvdata(dev); in cppi41_resume() local
1190 cppi_writel(cdd->descs_phys, cdd->qmgr_mem + QMGR_MEMBASE(i)); in cppi41_resume()
1192 list_for_each_entry(c, &cdd->ddev.channels, chan.device_node) in cppi41_resume()
1196 init_sched(cdd); in cppi41_resume()
1198 cppi_writel(cdd->dma_tdfdq, cdd->ctrl_mem + DMA_TDFDQ); in cppi41_resume()
1199 cppi_writel(cdd->scratch_phys, cdd->qmgr_mem + QMGR_LRAM0_BASE); in cppi41_resume()
1200 cppi_writel(QMGR_SCRATCH_SIZE, cdd->qmgr_mem + QMGR_LRAM_SIZE); in cppi41_resume()
1201 cppi_writel(0, cdd->qmgr_mem + QMGR_LRAM1_BASE); in cppi41_resume()
1208 struct cppi41_dd *cdd = dev_get_drvdata(dev); in cppi41_runtime_suspend() local
1211 spin_lock_irqsave(&cdd->lock, flags); in cppi41_runtime_suspend()
1212 cdd->is_suspended = true; in cppi41_runtime_suspend()
1213 WARN_ON(!list_empty(&cdd->pending)); in cppi41_runtime_suspend()
1214 spin_unlock_irqrestore(&cdd->lock, flags); in cppi41_runtime_suspend()
1221 struct cppi41_dd *cdd = dev_get_drvdata(dev); in cppi41_runtime_resume() local
1224 spin_lock_irqsave(&cdd->lock, flags); in cppi41_runtime_resume()
1225 cdd->is_suspended = false; in cppi41_runtime_resume()
1226 cppi41_run_queue(cdd); in cppi41_runtime_resume()
1227 spin_unlock_irqrestore(&cdd->lock, flags); in cppi41_runtime_resume()