dma/dw/core.c

80 	struct dw_dma *dw = to_dw_dma(dwc->chan.device);
84 	desc = dma_pool_zalloc(dw->desc_pool, GFP_ATOMIC, &phys);
99 	struct dw_dma *dw = to_dw_dma(dwc->chan.device);
107 		dma_pool_free(dw->desc_pool, child, child->txd.phys);
111 	dma_pool_free(dw->desc_pool, desc, desc->txd.phys);
117 	struct dw_dma *dw = to_dw_dma(dwc->chan.device);
119 	dw->initialize_chan(dwc);
122 	channel_set_bit(dw, MASK.XFER, dwc->mask);
123 	channel_set_bit(dw, MASK.ERROR, dwc->mask);
139 static inline void dwc_chan_disable(struct dw_dma *dw, struct dw_dma_chan *dwc)
141 	channel_clear_bit(dw, CH_EN, dwc->mask);
142 	while (dma_readl(dw, CH_EN) & dwc->mask)
152 	struct dw_dma	*dw = to_dw_dma(dwc->chan.device);
165 	channel_set_bit(dw, CH_EN, dwc->mask);
174 	struct dw_dma	*dw = to_dw_dma(dwc->chan.device);
179 	if (dma_readl(dw, CH_EN) & dwc->mask) {
214 	channel_set_bit(dw, CH_EN, dwc->mask);
260 static void dwc_complete_all(struct dw_dma *dw, struct dw_dma_chan *dwc)
267 	if (dma_readl(dw, CH_EN) & dwc->mask) {
272 		dwc_chan_disable(dw, dwc);
291 	struct dw_dma *dw = to_dw_dma(dwc->chan.device);
295 	return dw->block2bytes(dwc, ctlhi, ctllo >> 4 & 7);
298 static void dwc_scan_descriptors(struct dw_dma *dw, struct dw_dma_chan *dwc)
308 	status_xfer = dma_readl(dw, RAW.XFER);
312 		dma_writel(dw, CLEAR.XFER, dwc->mask);
346 		dwc_complete_all(dw, dwc);
405 	dwc_chan_disable(dw, dwc);
421 static void dwc_handle_error(struct dw_dma *dw, struct dw_dma_chan *dwc)
427 	dwc_scan_descriptors(dw, dwc);
441 	dma_writel(dw, CLEAR.ERROR, dwc->mask);
466 	struct dw_dma *dw = from_tasklet(dw, t, tasklet);
472 	status_xfer = dma_readl(dw, RAW.XFER);
473 	status_err = dma_readl(dw, RAW.ERROR);
475 	dev_vdbg(dw->dma.dev, "%s: status_err=%x\n", __func__, status_err);
477 	for (i = 0; i < dw->dma.chancnt; i++) {
478 		dwc = &dw->chan[i];
480 			dev_vdbg(dw->dma.dev, "Cyclic xfer is not implemented\n");
482 			dwc_handle_error(dw, dwc);
484 			dwc_scan_descriptors(dw, dwc);
488 	channel_set_bit(dw, MASK.XFER, dw->all_chan_mask);
489 	channel_set_bit(dw, MASK.ERROR, dw->all_chan_mask);
494 	struct dw_dma *dw = dev_id;
498 	if (!dw->in_use)
501 	status = dma_readl(dw, STATUS_INT);
502 	dev_vdbg(dw->dma.dev, "%s: status=0x%x\n", __func__, status);
512 	channel_clear_bit(dw, MASK.XFER, dw->all_chan_mask);
513 	channel_clear_bit(dw, MASK.BLOCK, dw->all_chan_mask);
514 	channel_clear_bit(dw, MASK.ERROR, dw->all_chan_mask);
516 	status = dma_readl(dw, STATUS_INT);
518 		dev_err(dw->dma.dev,
523 		channel_clear_bit(dw, MASK.XFER, (1 << 8) - 1);
524 		channel_clear_bit(dw, MASK.BLOCK, (1 << 8) - 1);
525 		channel_clear_bit(dw, MASK.SRC_TRAN, (1 << 8) - 1);
526 		channel_clear_bit(dw, MASK.DST_TRAN, (1 << 8) - 1);
527 		channel_clear_bit(dw, MASK.ERROR, (1 << 8) - 1);
530 	tasklet_schedule(&dw->tasklet);
542 	struct dw_dma		*dw = to_dw_dma(chan->device);
551 	unsigned int		data_width = dw->pdata->data_width[m_master];
568 	ctllo = dw->prepare_ctllo(dwc)
581 		ctlhi = dw->bytes2block(dwc, len - offset, src_width, &xfer_count);
620 	struct dw_dma		*dw = to_dw_dma(chan->device);
646 		ctllo = dw->prepare_ctllo(dwc)
669 			ctlhi = dw->bytes2block(dwc, len, mem_width, &dlen);
696 		ctllo = dw->prepare_ctllo(dwc)
717 			ctlhi = dw->bytes2block(dwc, len, reg_width, &dlen);
802 	struct dw_dma *dw = to_dw_dma(chan->device);
812 	max_width = dw->pdata->data_width[dwc->dws.p_master];
834 	struct dw_dma *dw = to_dw_dma(chan->device);
837 	mem_width = dw->pdata->data_width[dwc->dws.m_master];
890 	struct dw_dma *dw = to_dw_dma(dwc->chan.device);
893 	dw->suspend_chan(dwc, drain);
915 	struct dw_dma *dw = to_dw_dma(dwc->chan.device);
917 	dw->resume_chan(dwc, drain);
940 	struct dw_dma		*dw = to_dw_dma(chan->device);
951 	dwc_chan_disable(dw, dwc);
1042 void do_dw_dma_off(struct dw_dma *dw)
1044 	dma_writel(dw, CFG, 0);
1046 	channel_clear_bit(dw, MASK.XFER, dw->all_chan_mask);
1047 	channel_clear_bit(dw, MASK.BLOCK, dw->all_chan_mask);
1048 	channel_clear_bit(dw, MASK.SRC_TRAN, dw->all_chan_mask);
1049 	channel_clear_bit(dw, MASK.DST_TRAN, dw->all_chan_mask);
1050 	channel_clear_bit(dw, MASK.ERROR, dw->all_chan_mask);
1052 	while (dma_readl(dw, CFG) & DW_CFG_DMA_EN)
1056 void do_dw_dma_on(struct dw_dma *dw)
1058 	dma_writel(dw, CFG, DW_CFG_DMA_EN);
1064 	struct dw_dma		*dw = to_dw_dma(chan->device);
1069 	if (dma_readl(dw, CH_EN) & dwc->mask) {
1091 	if (!dw->in_use)
1092 		do_dw_dma_on(dw);
1093 	dw->in_use |= dwc->mask;
1101 	struct dw_dma		*dw = to_dw_dma(chan->device);
1118 	channel_clear_bit(dw, MASK.XFER, dwc->mask);
1119 	channel_clear_bit(dw, MASK.BLOCK, dwc->mask);
1120 	channel_clear_bit(dw, MASK.ERROR, dwc->mask);
1125 	dw->in_use &= ~dwc->mask;
1126 	if (!dw->in_use)
1127 		do_dw_dma_off(dw);
1153 	struct dw_dma *dw = chip->dw;
1160 	dw->pdata = devm_kzalloc(chip->dev, sizeof(*dw->pdata), GFP_KERNEL);
1161 	if (!dw->pdata)
1164 	dw->regs = chip->regs;
1169 		dw_params = dma_readl(dw, DW_PARAMS);
1179 		pdata = dw->pdata;
1188 		pdata->block_size = dma_readl(dw, MAX_BLK_SIZE);
1197 		memcpy(dw->pdata, chip->pdata, sizeof(*dw->pdata));
1200 		pdata = dw->pdata;
1203 	dw->chan = devm_kcalloc(chip->dev, pdata->nr_channels, sizeof(*dw->chan),
1205 	if (!dw->chan) {
1211 	dw->all_chan_mask = (1 << pdata->nr_channels) - 1;
1214 	dw->disable(dw);
1217 	dw->set_device_name(dw, chip->id);
1220 	dw->desc_pool = dmam_pool_create(dw->name, chip->dev,
1222 	if (!dw->desc_pool) {
1228 	tasklet_setup(&dw->tasklet, dw_dma_tasklet);
1231 			  dw->name, dw);
1235 	INIT_LIST_HEAD(&dw->dma.channels);
1237 		struct dw_dma_chan	*dwc = &dw->chan[i];
1239 		dwc->chan.device = &dw->dma;
1243 					&dw->dma.channels);
1245 			list_add(&dwc->chan.device_node, &dw->dma.channels);
1253 		dwc->ch_regs = &__dw_regs(dw)->CHAN[i];
1260 		channel_clear_bit(dw, CH_EN, dwc->mask);
1267 			void __iomem *addr = &__dw_regs(dw)->DWC_PARAMS[r];
1301 	dma_writel(dw, CLEAR.XFER, dw->all_chan_mask);
1302 	dma_writel(dw, CLEAR.BLOCK, dw->all_chan_mask);
1303 	dma_writel(dw, CLEAR.SRC_TRAN, dw->all_chan_mask);
1304 	dma_writel(dw, CLEAR.DST_TRAN, dw->all_chan_mask);
1305 	dma_writel(dw, CLEAR.ERROR, dw->all_chan_mask);
1308 	dma_cap_set(DMA_SLAVE, dw->dma.cap_mask);
1309 	dma_cap_set(DMA_PRIVATE, dw->dma.cap_mask);
1310 	dma_cap_set(DMA_MEMCPY, dw->dma.cap_mask);
1312 	dw->dma.dev = chip->dev;
1313 	dw->dma.device_alloc_chan_resources = dwc_alloc_chan_resources;
1314 	dw->dma.device_free_chan_resources = dwc_free_chan_resources;
1316 	dw->dma.device_prep_dma_memcpy = dwc_prep_dma_memcpy;
1317 	dw->dma.device_prep_slave_sg = dwc_prep_slave_sg;
1319 	dw->dma.device_caps = dwc_caps;
1320 	dw->dma.device_config = dwc_config;
1321 	dw->dma.device_pause = dwc_pause;
1322 	dw->dma.device_resume = dwc_resume;
1323 	dw->dma.device_terminate_all = dwc_terminate_all;
1325 	dw->dma.device_tx_status = dwc_tx_status;
1326 	dw->dma.device_issue_pending = dwc_issue_pending;
1329 	dw->dma.min_burst = DW_DMA_MIN_BURST;
1330 	dw->dma.max_burst = DW_DMA_MAX_BURST;
1331 	dw->dma.src_addr_widths = DW_DMA_BUSWIDTHS;
1332 	dw->dma.dst_addr_widths = DW_DMA_BUSWIDTHS;
1333 	dw->dma.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV) |
1335 	dw->dma.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST;
1342 	dma_set_max_seg_size(dw->dma.dev, dw->chan[0].block_size);
1344 	ret = dma_async_device_register(&dw->dma);
1356 	free_irq(chip->irq, dw);
1364 	struct dw_dma		*dw = chip->dw;
1369 	do_dw_dma_off(dw);
1370 	dma_async_device_unregister(&dw->dma);
1372 	free_irq(chip->irq, dw);
1373 	tasklet_kill(&dw->tasklet);
1375 	list_for_each_entry_safe(dwc, _dwc, &dw->dma.channels,
1378 		channel_clear_bit(dw, CH_EN, dwc->mask);
1387 	struct dw_dma *dw = chip->dw;
1389 	dw->disable(dw);
1396 	struct dw_dma *dw = chip->dw;
1398 	dw->enable(dw);