Lines Matching +full:dma +full:- +full:maxburst
1 // SPDX-License-Identifier: GPL-2.0-only
3 * Core driver for the Intel integrated DMA 64-bit
12 #include <linux/dma-mapping.h>
19 #include <linux/dma/idma64.h>
26 /* ---------------------------------------------------------------------- */
30 return &chan->dev->device; in chan2dev()
33 /* ---------------------------------------------------------------------- */
41 channel_clear_bit(idma64, MASK(XFER), idma64->all_chan_mask); in idma64_off()
42 channel_clear_bit(idma64, MASK(BLOCK), idma64->all_chan_mask); in idma64_off()
43 channel_clear_bit(idma64, MASK(SRC_TRAN), idma64->all_chan_mask); in idma64_off()
44 channel_clear_bit(idma64, MASK(DST_TRAN), idma64->all_chan_mask); in idma64_off()
45 channel_clear_bit(idma64, MASK(ERROR), idma64->all_chan_mask); in idma64_off()
49 } while (dma_readl(idma64, CFG) & IDMA64_CFG_DMA_EN && --count); in idma64_off()
57 /* ---------------------------------------------------------------------- */
71 channel_set_bit(idma64, MASK(XFER), idma64c->mask); in idma64_chan_init()
72 channel_set_bit(idma64, MASK(ERROR), idma64c->mask); in idma64_chan_init()
77 * The iDMA is turned off in ->probe() and looses context during system in idma64_chan_init()
86 channel_clear_bit(idma64, CH_EN, idma64c->mask); in idma64_chan_stop()
91 struct idma64_desc *desc = idma64c->desc; in idma64_chan_start()
92 struct idma64_hw_desc *hw = &desc->hw[0]; in idma64_chan_start()
100 channel_writeq(idma64c, LLP, hw->llp); in idma64_chan_start()
102 channel_set_bit(idma64, CH_EN, idma64c->mask); in idma64_chan_start()
107 struct idma64 *idma64 = to_idma64(idma64c->vchan.chan.device); in idma64_stop_transfer()
114 struct idma64 *idma64 = to_idma64(idma64c->vchan.chan.device); in idma64_start_transfer()
118 vdesc = vchan_next_desc(&idma64c->vchan); in idma64_start_transfer()
120 idma64c->desc = NULL; in idma64_start_transfer()
124 list_del(&vdesc->node); in idma64_start_transfer()
125 idma64c->desc = to_idma64_desc(vdesc); in idma64_start_transfer()
134 /* ---------------------------------------------------------------------- */
139 struct idma64_chan *idma64c = &idma64->chan[c]; in idma64_chan_irq()
143 stat = this_cpu_ptr(idma64c->vchan.chan.local); in idma64_chan_irq()
145 spin_lock(&idma64c->vchan.lock); in idma64_chan_irq()
146 desc = idma64c->desc; in idma64_chan_irq()
149 dma_writel(idma64, CLEAR(ERROR), idma64c->mask); in idma64_chan_irq()
150 desc->status = DMA_ERROR; in idma64_chan_irq()
152 dma_writel(idma64, CLEAR(XFER), idma64c->mask); in idma64_chan_irq()
153 desc->status = DMA_COMPLETE; in idma64_chan_irq()
154 vchan_cookie_complete(&desc->vdesc); in idma64_chan_irq()
155 stat->bytes_transferred += desc->length; in idma64_chan_irq()
159 /* idma64_start_transfer() updates idma64c->desc */ in idma64_chan_irq()
160 if (idma64c->desc == NULL || desc->status == DMA_ERROR) in idma64_chan_irq()
163 spin_unlock(&idma64c->vchan.lock); in idma64_chan_irq()
174 dev_vdbg(idma64->dma.dev, "%s: status=%#x\n", __func__, status); in idma64_irq()
176 /* Check if we have any interrupt from the DMA controller */ in idma64_irq()
183 for (i = 0; i < idma64->dma.chancnt; i++) in idma64_irq()
189 /* ---------------------------------------------------------------------- */
199 desc->hw = kcalloc(ndesc, sizeof(*desc->hw), GFP_NOWAIT); in idma64_alloc_desc()
200 if (!desc->hw) { in idma64_alloc_desc()
213 if (desc->ndesc) { in idma64_desc_free()
214 unsigned int i = desc->ndesc; in idma64_desc_free()
217 hw = &desc->hw[--i]; in idma64_desc_free()
218 dma_pool_free(idma64c->pool, hw->lli, hw->llp); in idma64_desc_free()
222 kfree(desc->hw); in idma64_desc_free()
228 struct idma64_chan *idma64c = to_idma64_chan(vdesc->tx.chan); in idma64_vdesc_free()
237 struct idma64_lli *lli = hw->lli; in idma64_hw_desc_fill()
239 u32 ctlhi = IDMA64C_CTLH_BLOCK_TS(hw->len); in idma64_hw_desc_fill()
244 sar = hw->phys; in idma64_hw_desc_fill()
245 dar = config->dst_addr; in idma64_hw_desc_fill()
248 src_width = __ffs(sar | hw->len | 4); in idma64_hw_desc_fill()
249 dst_width = __ffs(config->dst_addr_width); in idma64_hw_desc_fill()
251 sar = config->src_addr; in idma64_hw_desc_fill()
252 dar = hw->phys; in idma64_hw_desc_fill()
255 src_width = __ffs(config->src_addr_width); in idma64_hw_desc_fill()
256 dst_width = __ffs(dar | hw->len | 4); in idma64_hw_desc_fill()
259 lli->sar = sar; in idma64_hw_desc_fill()
260 lli->dar = dar; in idma64_hw_desc_fill()
262 lli->ctlhi = ctlhi; in idma64_hw_desc_fill()
263 lli->ctllo = ctllo | in idma64_hw_desc_fill()
264 IDMA64C_CTLL_SRC_MSIZE(config->src_maxburst) | in idma64_hw_desc_fill()
265 IDMA64C_CTLL_DST_MSIZE(config->dst_maxburst) | in idma64_hw_desc_fill()
269 lli->llp = llp; in idma64_hw_desc_fill()
275 struct dma_slave_config *config = &idma64c->config; in idma64_desc_fill()
276 unsigned int i = desc->ndesc; in idma64_desc_fill()
277 struct idma64_hw_desc *hw = &desc->hw[i - 1]; in idma64_desc_fill()
278 struct idma64_lli *lli = hw->lli; in idma64_desc_fill()
283 hw = &desc->hw[--i]; in idma64_desc_fill()
284 idma64_hw_desc_fill(hw, config, desc->direction, llp); in idma64_desc_fill()
285 llp = hw->llp; in idma64_desc_fill()
286 desc->length += hw->len; in idma64_desc_fill()
290 lli->ctllo |= IDMA64C_CTLL_INT_EN; in idma64_desc_fill()
293 lli->ctllo &= ~(IDMA64C_CTLL_LLP_S_EN | IDMA64C_CTLL_LLP_D_EN); in idma64_desc_fill()
311 struct idma64_hw_desc *hw = &desc->hw[i]; in idma64_prep_slave_sg()
313 /* Allocate DMA capable memory for hardware descriptor */ in idma64_prep_slave_sg()
314 hw->lli = dma_pool_alloc(idma64c->pool, GFP_NOWAIT, &hw->llp); in idma64_prep_slave_sg()
315 if (!hw->lli) { in idma64_prep_slave_sg()
316 desc->ndesc = i; in idma64_prep_slave_sg()
321 hw->phys = sg_dma_address(sg); in idma64_prep_slave_sg()
322 hw->len = sg_dma_len(sg); in idma64_prep_slave_sg()
325 desc->ndesc = sg_len; in idma64_prep_slave_sg()
326 desc->direction = direction; in idma64_prep_slave_sg()
327 desc->status = DMA_IN_PROGRESS; in idma64_prep_slave_sg()
330 return vchan_tx_prep(&idma64c->vchan, &desc->vdesc, flags); in idma64_prep_slave_sg()
338 spin_lock_irqsave(&idma64c->vchan.lock, flags); in idma64_issue_pending()
339 if (vchan_issue_pending(&idma64c->vchan) && !idma64c->desc) in idma64_issue_pending()
341 spin_unlock_irqrestore(&idma64c->vchan.lock, flags); in idma64_issue_pending()
346 struct idma64_desc *desc = idma64c->desc; in idma64_active_desc_size()
348 size_t bytes = desc->length; in idma64_active_desc_size()
354 hw = &desc->hw[i]; in idma64_active_desc_size()
355 if (hw->llp == llp) in idma64_active_desc_size()
357 bytes -= hw->len; in idma64_active_desc_size()
358 } while (++i < desc->ndesc); in idma64_active_desc_size()
364 bytes += desc->hw[--i].len; in idma64_active_desc_size()
366 return bytes - IDMA64C_CTLH_BLOCK_TS(ctlhi); in idma64_active_desc_size()
382 spin_lock_irqsave(&idma64c->vchan.lock, flags); in idma64_tx_status()
383 vdesc = vchan_find_desc(&idma64c->vchan, cookie); in idma64_tx_status()
384 if (idma64c->desc && cookie == idma64c->desc->vdesc.tx.cookie) { in idma64_tx_status()
387 status = idma64c->desc->status; in idma64_tx_status()
389 bytes = to_idma64_desc(vdesc)->length; in idma64_tx_status()
392 spin_unlock_irqrestore(&idma64c->vchan.lock, flags); in idma64_tx_status()
397 static void convert_burst(u32 *maxburst) in convert_burst() argument
399 if (*maxburst) in convert_burst()
400 *maxburst = __fls(*maxburst); in convert_burst()
402 *maxburst = 0; in convert_burst()
410 memcpy(&idma64c->config, config, sizeof(idma64c->config)); in idma64_slave_config()
412 convert_burst(&idma64c->config.src_maxburst); in idma64_slave_config()
413 convert_burst(&idma64c->config.dst_maxburst); in idma64_slave_config()
433 } while (!(cfglo & IDMA64C_CFGL_FIFO_EMPTY) && --count); in idma64_chan_deactivate()
449 spin_lock_irqsave(&idma64c->vchan.lock, flags); in idma64_pause()
450 if (idma64c->desc && idma64c->desc->status == DMA_IN_PROGRESS) { in idma64_pause()
452 idma64c->desc->status = DMA_PAUSED; in idma64_pause()
454 spin_unlock_irqrestore(&idma64c->vchan.lock, flags); in idma64_pause()
464 spin_lock_irqsave(&idma64c->vchan.lock, flags); in idma64_resume()
465 if (idma64c->desc && idma64c->desc->status == DMA_PAUSED) { in idma64_resume()
466 idma64c->desc->status = DMA_IN_PROGRESS; in idma64_resume()
469 spin_unlock_irqrestore(&idma64c->vchan.lock, flags); in idma64_resume()
480 spin_lock_irqsave(&idma64c->vchan.lock, flags); in idma64_terminate_all()
483 if (idma64c->desc) { in idma64_terminate_all()
484 idma64_vdesc_free(&idma64c->desc->vdesc); in idma64_terminate_all()
485 idma64c->desc = NULL; in idma64_terminate_all()
487 vchan_get_all_descriptors(&idma64c->vchan, &head); in idma64_terminate_all()
488 spin_unlock_irqrestore(&idma64c->vchan.lock, flags); in idma64_terminate_all()
490 vchan_dma_desc_free_list(&idma64c->vchan, &head); in idma64_terminate_all()
498 vchan_synchronize(&idma64c->vchan); in idma64_synchronize()
506 idma64c->pool = dma_pool_create(dev_name(chan2dev(chan)), in idma64_alloc_chan_resources()
507 chan->device->dev, in idma64_alloc_chan_resources()
509 if (!idma64c->pool) { in idma64_alloc_chan_resources()
511 return -ENOMEM; in idma64_alloc_chan_resources()
522 dma_pool_destroy(idma64c->pool); in idma64_free_chan_resources()
523 idma64c->pool = NULL; in idma64_free_chan_resources()
526 /* ---------------------------------------------------------------------- */
540 idma64 = devm_kzalloc(chip->dev, sizeof(*idma64), GFP_KERNEL); in idma64_probe()
542 return -ENOMEM; in idma64_probe()
544 idma64->regs = chip->regs; in idma64_probe()
545 chip->idma64 = idma64; in idma64_probe()
547 idma64->chan = devm_kcalloc(chip->dev, nr_chan, sizeof(*idma64->chan), in idma64_probe()
549 if (!idma64->chan) in idma64_probe()
550 return -ENOMEM; in idma64_probe()
552 idma64->all_chan_mask = (1 << nr_chan) - 1; in idma64_probe()
557 ret = devm_request_irq(chip->dev, chip->irq, idma64_irq, IRQF_SHARED, in idma64_probe()
558 dev_name(chip->dev), idma64); in idma64_probe()
562 INIT_LIST_HEAD(&idma64->dma.channels); in idma64_probe()
564 struct idma64_chan *idma64c = &idma64->chan[i]; in idma64_probe()
566 idma64c->vchan.desc_free = idma64_vdesc_free; in idma64_probe()
567 vchan_init(&idma64c->vchan, &idma64->dma); in idma64_probe()
569 idma64c->regs = idma64->regs + i * IDMA64_CH_LENGTH; in idma64_probe()
570 idma64c->mask = BIT(i); in idma64_probe()
573 dma_cap_set(DMA_SLAVE, idma64->dma.cap_mask); in idma64_probe()
574 dma_cap_set(DMA_PRIVATE, idma64->dma.cap_mask); in idma64_probe()
576 idma64->dma.device_alloc_chan_resources = idma64_alloc_chan_resources; in idma64_probe()
577 idma64->dma.device_free_chan_resources = idma64_free_chan_resources; in idma64_probe()
579 idma64->dma.device_prep_slave_sg = idma64_prep_slave_sg; in idma64_probe()
581 idma64->dma.device_issue_pending = idma64_issue_pending; in idma64_probe()
582 idma64->dma.device_tx_status = idma64_tx_status; in idma64_probe()
584 idma64->dma.device_config = idma64_slave_config; in idma64_probe()
585 idma64->dma.device_pause = idma64_pause; in idma64_probe()
586 idma64->dma.device_resume = idma64_resume; in idma64_probe()
587 idma64->dma.device_terminate_all = idma64_terminate_all; in idma64_probe()
588 idma64->dma.device_synchronize = idma64_synchronize; in idma64_probe()
590 idma64->dma.src_addr_widths = IDMA64_BUSWIDTHS; in idma64_probe()
591 idma64->dma.dst_addr_widths = IDMA64_BUSWIDTHS; in idma64_probe()
592 idma64->dma.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV); in idma64_probe()
593 idma64->dma.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST; in idma64_probe()
595 idma64->dma.dev = chip->sysdev; in idma64_probe()
597 dma_set_max_seg_size(idma64->dma.dev, IDMA64C_CTLH_BLOCK_TS_MASK); in idma64_probe()
599 ret = dma_async_device_register(&idma64->dma); in idma64_probe()
603 dev_info(chip->dev, "Found Intel integrated DMA 64-bit\n"); in idma64_probe()
609 struct idma64 *idma64 = chip->idma64; in idma64_remove()
612 dma_async_device_unregister(&idma64->dma); in idma64_remove()
618 devm_free_irq(chip->dev, chip->irq, idma64); in idma64_remove()
620 for (i = 0; i < idma64->dma.chancnt; i++) { in idma64_remove()
621 struct idma64_chan *idma64c = &idma64->chan[i]; in idma64_remove()
623 tasklet_kill(&idma64c->vchan.task); in idma64_remove()
627 /* ---------------------------------------------------------------------- */
632 struct device *dev = &pdev->dev; in idma64_platform_probe()
633 struct device *sysdev = dev->parent; in idma64_platform_probe()
638 return -ENOMEM; in idma64_platform_probe()
640 chip->irq = platform_get_irq(pdev, 0); in idma64_platform_probe()
641 if (chip->irq < 0) in idma64_platform_probe()
642 return chip->irq; in idma64_platform_probe()
644 chip->regs = devm_platform_ioremap_resource(pdev, 0); in idma64_platform_probe()
645 if (IS_ERR(chip->regs)) in idma64_platform_probe()
646 return PTR_ERR(chip->regs); in idma64_platform_probe()
652 chip->dev = dev; in idma64_platform_probe()
653 chip->sysdev = sysdev; in idma64_platform_probe()
674 idma64_off(chip->idma64); in idma64_pm_suspend()
682 idma64_on(chip->idma64); in idma64_pm_resume()