Lines Matching +full:dma +full:- +full:maxburst
1 // SPDX-License-Identifier: GPL-2.0-only
3 * Core driver for the Intel integrated DMA 64-bit
12 #include <linux/dma-mapping.h>
19 #include <linux/dma/idma64.h>
26 /* ---------------------------------------------------------------------- */
30 return &chan->dev->device; in chan2dev()
33 /* ---------------------------------------------------------------------- */
41 channel_clear_bit(idma64, MASK(XFER), idma64->all_chan_mask); in idma64_off()
42 channel_clear_bit(idma64, MASK(BLOCK), idma64->all_chan_mask); in idma64_off()
43 channel_clear_bit(idma64, MASK(SRC_TRAN), idma64->all_chan_mask); in idma64_off()
44 channel_clear_bit(idma64, MASK(DST_TRAN), idma64->all_chan_mask); in idma64_off()
45 channel_clear_bit(idma64, MASK(ERROR), idma64->all_chan_mask); in idma64_off()
49 } while (dma_readl(idma64, CFG) & IDMA64_CFG_DMA_EN && --count); in idma64_off()
57 /* ---------------------------------------------------------------------- */
71 channel_set_bit(idma64, MASK(XFER), idma64c->mask); in idma64_chan_init()
72 channel_set_bit(idma64, MASK(ERROR), idma64c->mask); in idma64_chan_init()
77 * The iDMA is turned off in ->probe() and looses context during system in idma64_chan_init()
86 channel_clear_bit(idma64, CH_EN, idma64c->mask); in idma64_chan_stop()
91 struct idma64_desc *desc = idma64c->desc; in idma64_chan_start()
92 struct idma64_hw_desc *hw = &desc->hw[0]; in idma64_chan_start()
100 channel_writeq(idma64c, LLP, hw->llp); in idma64_chan_start()
102 channel_set_bit(idma64, CH_EN, idma64c->mask); in idma64_chan_start()
107 struct idma64 *idma64 = to_idma64(idma64c->vchan.chan.device); in idma64_stop_transfer()
114 struct idma64 *idma64 = to_idma64(idma64c->vchan.chan.device); in idma64_start_transfer()
118 vdesc = vchan_next_desc(&idma64c->vchan); in idma64_start_transfer()
120 idma64c->desc = NULL; in idma64_start_transfer()
124 list_del(&vdesc->node); in idma64_start_transfer()
125 idma64c->desc = to_idma64_desc(vdesc); in idma64_start_transfer()
134 /* ---------------------------------------------------------------------- */
139 struct idma64_chan *idma64c = &idma64->chan[c]; in idma64_chan_irq()
142 spin_lock(&idma64c->vchan.lock); in idma64_chan_irq()
143 desc = idma64c->desc; in idma64_chan_irq()
146 dma_writel(idma64, CLEAR(ERROR), idma64c->mask); in idma64_chan_irq()
147 desc->status = DMA_ERROR; in idma64_chan_irq()
149 dma_writel(idma64, CLEAR(XFER), idma64c->mask); in idma64_chan_irq()
150 desc->status = DMA_COMPLETE; in idma64_chan_irq()
151 vchan_cookie_complete(&desc->vdesc); in idma64_chan_irq()
155 /* idma64_start_transfer() updates idma64c->desc */ in idma64_chan_irq()
156 if (idma64c->desc == NULL || desc->status == DMA_ERROR) in idma64_chan_irq()
159 spin_unlock(&idma64c->vchan.lock); in idma64_chan_irq()
170 dev_vdbg(idma64->dma.dev, "%s: status=%#x\n", __func__, status); in idma64_irq()
172 /* Check if we have any interrupt from the DMA controller */ in idma64_irq()
179 for (i = 0; i < idma64->dma.chancnt; i++) in idma64_irq()
185 /* ---------------------------------------------------------------------- */
195 desc->hw = kcalloc(ndesc, sizeof(*desc->hw), GFP_NOWAIT); in idma64_alloc_desc()
196 if (!desc->hw) { in idma64_alloc_desc()
209 if (desc->ndesc) { in idma64_desc_free()
210 unsigned int i = desc->ndesc; in idma64_desc_free()
213 hw = &desc->hw[--i]; in idma64_desc_free()
214 dma_pool_free(idma64c->pool, hw->lli, hw->llp); in idma64_desc_free()
218 kfree(desc->hw); in idma64_desc_free()
224 struct idma64_chan *idma64c = to_idma64_chan(vdesc->tx.chan); in idma64_vdesc_free()
233 struct idma64_lli *lli = hw->lli; in idma64_hw_desc_fill()
235 u32 ctlhi = IDMA64C_CTLH_BLOCK_TS(hw->len); in idma64_hw_desc_fill()
240 sar = hw->phys; in idma64_hw_desc_fill()
241 dar = config->dst_addr; in idma64_hw_desc_fill()
244 src_width = __ffs(sar | hw->len | 4); in idma64_hw_desc_fill()
245 dst_width = __ffs(config->dst_addr_width); in idma64_hw_desc_fill()
247 sar = config->src_addr; in idma64_hw_desc_fill()
248 dar = hw->phys; in idma64_hw_desc_fill()
251 src_width = __ffs(config->src_addr_width); in idma64_hw_desc_fill()
252 dst_width = __ffs(dar | hw->len | 4); in idma64_hw_desc_fill()
255 lli->sar = sar; in idma64_hw_desc_fill()
256 lli->dar = dar; in idma64_hw_desc_fill()
258 lli->ctlhi = ctlhi; in idma64_hw_desc_fill()
259 lli->ctllo = ctllo | in idma64_hw_desc_fill()
260 IDMA64C_CTLL_SRC_MSIZE(config->src_maxburst) | in idma64_hw_desc_fill()
261 IDMA64C_CTLL_DST_MSIZE(config->dst_maxburst) | in idma64_hw_desc_fill()
265 lli->llp = llp; in idma64_hw_desc_fill()
271 struct dma_slave_config *config = &idma64c->config; in idma64_desc_fill()
272 unsigned int i = desc->ndesc; in idma64_desc_fill()
273 struct idma64_hw_desc *hw = &desc->hw[i - 1]; in idma64_desc_fill()
274 struct idma64_lli *lli = hw->lli; in idma64_desc_fill()
279 hw = &desc->hw[--i]; in idma64_desc_fill()
280 idma64_hw_desc_fill(hw, config, desc->direction, llp); in idma64_desc_fill()
281 llp = hw->llp; in idma64_desc_fill()
282 desc->length += hw->len; in idma64_desc_fill()
286 lli->ctllo |= IDMA64C_CTLL_INT_EN; in idma64_desc_fill()
289 lli->ctllo &= ~(IDMA64C_CTLL_LLP_S_EN | IDMA64C_CTLL_LLP_D_EN); in idma64_desc_fill()
307 struct idma64_hw_desc *hw = &desc->hw[i]; in idma64_prep_slave_sg()
309 /* Allocate DMA capable memory for hardware descriptor */ in idma64_prep_slave_sg()
310 hw->lli = dma_pool_alloc(idma64c->pool, GFP_NOWAIT, &hw->llp); in idma64_prep_slave_sg()
311 if (!hw->lli) { in idma64_prep_slave_sg()
312 desc->ndesc = i; in idma64_prep_slave_sg()
317 hw->phys = sg_dma_address(sg); in idma64_prep_slave_sg()
318 hw->len = sg_dma_len(sg); in idma64_prep_slave_sg()
321 desc->ndesc = sg_len; in idma64_prep_slave_sg()
322 desc->direction = direction; in idma64_prep_slave_sg()
323 desc->status = DMA_IN_PROGRESS; in idma64_prep_slave_sg()
326 return vchan_tx_prep(&idma64c->vchan, &desc->vdesc, flags); in idma64_prep_slave_sg()
334 spin_lock_irqsave(&idma64c->vchan.lock, flags); in idma64_issue_pending()
335 if (vchan_issue_pending(&idma64c->vchan) && !idma64c->desc) in idma64_issue_pending()
337 spin_unlock_irqrestore(&idma64c->vchan.lock, flags); in idma64_issue_pending()
342 struct idma64_desc *desc = idma64c->desc; in idma64_active_desc_size()
344 size_t bytes = desc->length; in idma64_active_desc_size()
350 hw = &desc->hw[i]; in idma64_active_desc_size()
351 if (hw->llp == llp) in idma64_active_desc_size()
353 bytes -= hw->len; in idma64_active_desc_size()
354 } while (++i < desc->ndesc); in idma64_active_desc_size()
360 bytes += desc->hw[--i].len; in idma64_active_desc_size()
362 return bytes - IDMA64C_CTLH_BLOCK_TS(ctlhi); in idma64_active_desc_size()
378 spin_lock_irqsave(&idma64c->vchan.lock, flags); in idma64_tx_status()
379 vdesc = vchan_find_desc(&idma64c->vchan, cookie); in idma64_tx_status()
380 if (idma64c->desc && cookie == idma64c->desc->vdesc.tx.cookie) { in idma64_tx_status()
383 status = idma64c->desc->status; in idma64_tx_status()
385 bytes = to_idma64_desc(vdesc)->length; in idma64_tx_status()
388 spin_unlock_irqrestore(&idma64c->vchan.lock, flags); in idma64_tx_status()
393 static void convert_burst(u32 *maxburst) in convert_burst() argument
395 if (*maxburst) in convert_burst()
396 *maxburst = __fls(*maxburst); in convert_burst()
398 *maxburst = 0; in convert_burst()
406 memcpy(&idma64c->config, config, sizeof(idma64c->config)); in idma64_slave_config()
408 convert_burst(&idma64c->config.src_maxburst); in idma64_slave_config()
409 convert_burst(&idma64c->config.dst_maxburst); in idma64_slave_config()
429 } while (!(cfglo & IDMA64C_CFGL_FIFO_EMPTY) && --count); in idma64_chan_deactivate()
445 spin_lock_irqsave(&idma64c->vchan.lock, flags); in idma64_pause()
446 if (idma64c->desc && idma64c->desc->status == DMA_IN_PROGRESS) { in idma64_pause()
448 idma64c->desc->status = DMA_PAUSED; in idma64_pause()
450 spin_unlock_irqrestore(&idma64c->vchan.lock, flags); in idma64_pause()
460 spin_lock_irqsave(&idma64c->vchan.lock, flags); in idma64_resume()
461 if (idma64c->desc && idma64c->desc->status == DMA_PAUSED) { in idma64_resume()
462 idma64c->desc->status = DMA_IN_PROGRESS; in idma64_resume()
465 spin_unlock_irqrestore(&idma64c->vchan.lock, flags); in idma64_resume()
476 spin_lock_irqsave(&idma64c->vchan.lock, flags); in idma64_terminate_all()
479 if (idma64c->desc) { in idma64_terminate_all()
480 idma64_vdesc_free(&idma64c->desc->vdesc); in idma64_terminate_all()
481 idma64c->desc = NULL; in idma64_terminate_all()
483 vchan_get_all_descriptors(&idma64c->vchan, &head); in idma64_terminate_all()
484 spin_unlock_irqrestore(&idma64c->vchan.lock, flags); in idma64_terminate_all()
486 vchan_dma_desc_free_list(&idma64c->vchan, &head); in idma64_terminate_all()
494 vchan_synchronize(&idma64c->vchan); in idma64_synchronize()
502 idma64c->pool = dma_pool_create(dev_name(chan2dev(chan)), in idma64_alloc_chan_resources()
503 chan->device->dev, in idma64_alloc_chan_resources()
505 if (!idma64c->pool) { in idma64_alloc_chan_resources()
507 return -ENOMEM; in idma64_alloc_chan_resources()
518 dma_pool_destroy(idma64c->pool); in idma64_free_chan_resources()
519 idma64c->pool = NULL; in idma64_free_chan_resources()
522 /* ---------------------------------------------------------------------- */
536 idma64 = devm_kzalloc(chip->dev, sizeof(*idma64), GFP_KERNEL); in idma64_probe()
538 return -ENOMEM; in idma64_probe()
540 idma64->regs = chip->regs; in idma64_probe()
541 chip->idma64 = idma64; in idma64_probe()
543 idma64->chan = devm_kcalloc(chip->dev, nr_chan, sizeof(*idma64->chan), in idma64_probe()
545 if (!idma64->chan) in idma64_probe()
546 return -ENOMEM; in idma64_probe()
548 idma64->all_chan_mask = (1 << nr_chan) - 1; in idma64_probe()
553 ret = devm_request_irq(chip->dev, chip->irq, idma64_irq, IRQF_SHARED, in idma64_probe()
554 dev_name(chip->dev), idma64); in idma64_probe()
558 INIT_LIST_HEAD(&idma64->dma.channels); in idma64_probe()
560 struct idma64_chan *idma64c = &idma64->chan[i]; in idma64_probe()
562 idma64c->vchan.desc_free = idma64_vdesc_free; in idma64_probe()
563 vchan_init(&idma64c->vchan, &idma64->dma); in idma64_probe()
565 idma64c->regs = idma64->regs + i * IDMA64_CH_LENGTH; in idma64_probe()
566 idma64c->mask = BIT(i); in idma64_probe()
569 dma_cap_set(DMA_SLAVE, idma64->dma.cap_mask); in idma64_probe()
570 dma_cap_set(DMA_PRIVATE, idma64->dma.cap_mask); in idma64_probe()
572 idma64->dma.device_alloc_chan_resources = idma64_alloc_chan_resources; in idma64_probe()
573 idma64->dma.device_free_chan_resources = idma64_free_chan_resources; in idma64_probe()
575 idma64->dma.device_prep_slave_sg = idma64_prep_slave_sg; in idma64_probe()
577 idma64->dma.device_issue_pending = idma64_issue_pending; in idma64_probe()
578 idma64->dma.device_tx_status = idma64_tx_status; in idma64_probe()
580 idma64->dma.device_config = idma64_slave_config; in idma64_probe()
581 idma64->dma.device_pause = idma64_pause; in idma64_probe()
582 idma64->dma.device_resume = idma64_resume; in idma64_probe()
583 idma64->dma.device_terminate_all = idma64_terminate_all; in idma64_probe()
584 idma64->dma.device_synchronize = idma64_synchronize; in idma64_probe()
586 idma64->dma.src_addr_widths = IDMA64_BUSWIDTHS; in idma64_probe()
587 idma64->dma.dst_addr_widths = IDMA64_BUSWIDTHS; in idma64_probe()
588 idma64->dma.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV); in idma64_probe()
589 idma64->dma.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST; in idma64_probe()
591 idma64->dma.dev = chip->sysdev; in idma64_probe()
593 dma_set_max_seg_size(idma64->dma.dev, IDMA64C_CTLH_BLOCK_TS_MASK); in idma64_probe()
595 ret = dma_async_device_register(&idma64->dma); in idma64_probe()
599 dev_info(chip->dev, "Found Intel integrated DMA 64-bit\n"); in idma64_probe()
605 struct idma64 *idma64 = chip->idma64; in idma64_remove()
608 dma_async_device_unregister(&idma64->dma); in idma64_remove()
614 devm_free_irq(chip->dev, chip->irq, idma64); in idma64_remove()
616 for (i = 0; i < idma64->dma.chancnt; i++) { in idma64_remove()
617 struct idma64_chan *idma64c = &idma64->chan[i]; in idma64_remove()
619 tasklet_kill(&idma64c->vchan.task); in idma64_remove()
625 /* ---------------------------------------------------------------------- */
630 struct device *dev = &pdev->dev; in idma64_platform_probe()
631 struct device *sysdev = dev->parent; in idma64_platform_probe()
637 return -ENOMEM; in idma64_platform_probe()
639 chip->irq = platform_get_irq(pdev, 0); in idma64_platform_probe()
640 if (chip->irq < 0) in idma64_platform_probe()
641 return chip->irq; in idma64_platform_probe()
644 chip->regs = devm_ioremap_resource(dev, mem); in idma64_platform_probe()
645 if (IS_ERR(chip->regs)) in idma64_platform_probe()
646 return PTR_ERR(chip->regs); in idma64_platform_probe()
652 chip->dev = dev; in idma64_platform_probe()
653 chip->sysdev = sysdev; in idma64_platform_probe()
676 idma64_off(chip->idma64); in idma64_pm_suspend()
684 idma64_on(chip->idma64); in idma64_pm_resume()