Lines Matching +full:g +full:- +full:np +full:- +full:tx +full:- +full:fifo +full:- +full:size
1 // SPDX-License-Identifier: GPL-2.0-or-later
4 * Copyright (c) 2010 ST-Ericsson SA
10 * Documentation: ARM DDI 0196G == PL080
27 * - CH_CONFIG register at different offset,
28 * - separate CH_CONTROL2 register for transfer size,
29 * - bigger maximum transfer size,
30 * - 8-word aligned LLI, instead of 4-word, due to extra CCTL2 word,
31 * - no support for peripheral flow control.
41 * For peripherals with a FIFO:
42 * Source burst size == half the depth of the peripheral FIFO
43 * Destination burst size == the depth of the peripheral FIFO
45 * (Bursts are irrelevant for mem to mem transfers - there are no burst
51 * - DMAC flow control: the transfer size defines the number of transfers
58 * - Peripheral flow control: the transfer size is ignored (and should be
70 #include <linux/dma-mapping.h>
83 #include "virt-dma.h"
97 * struct vendor_data - vendor-specific config parameters for PL08x derivatives
106 * register and LLI word for transfer size.
108 * @max_transfer_size: the maximum single element transfer size for this
123 * struct pl08x_bus_data - information of source or destination
135 #define IS_BUS_ALIGNED(bus) IS_ALIGNED((bus)->addr, (bus)->buswidth)
138 * struct pl08x_phy_chan - holder for the physical channels
146 * @reg_busy: if the variant has a special per-channel busy register,
151 * @locked: channel unavailable for the system, e.g. dedicated to secure
173 * struct pl08x_sg - structure containing data per sg
187 * struct pl08x_txd - wrapper for struct dma_async_tx_descriptor
215 * enum pl08x_dma_chan_state - holds the PL08x specific virtual channel
233 * struct pl08x_dma_chan - this structure wraps a DMA ENGINE channel
263 * struct pl08x_driver_data - the local state holder for the PL08x
320 static inline struct pl08x_txd *to_pl08x_txd(struct dma_async_tx_descriptor *tx) in to_pl08x_txd() argument
322 return container_of(tx, struct pl08x_txd, vd.tx); in to_pl08x_txd()
335 const struct pl08x_platform_data *pd = plchan->host->pd; in pl08x_request_mux()
338 if (plchan->mux_use++ == 0 && pd->get_xfer_signal) { in pl08x_request_mux()
339 ret = pd->get_xfer_signal(plchan->cd); in pl08x_request_mux()
341 plchan->mux_use = 0; in pl08x_request_mux()
345 plchan->signal = ret; in pl08x_request_mux()
352 const struct pl08x_platform_data *pd = plchan->host->pd; in pl08x_release_mux()
354 if (plchan->signal >= 0) { in pl08x_release_mux()
355 WARN_ON(plchan->mux_use == 0); in pl08x_release_mux()
357 if (--plchan->mux_use == 0 && pd->put_xfer_signal) { in pl08x_release_mux()
358 pd->put_xfer_signal(plchan->cd, plchan->signal); in pl08x_release_mux()
359 plchan->signal = -1; in pl08x_release_mux()
374 if (ch->reg_busy) { in pl08x_phy_channel_busy()
375 val = readl(ch->reg_busy); in pl08x_phy_channel_busy()
376 return !!(val & BIT(ch->id)); in pl08x_phy_channel_busy()
378 val = readl(ch->reg_config); in pl08x_phy_channel_busy()
383 * pl08x_write_lli() - Write an LLI into the DMA controller.
388 * SRC, DST, LLI and control registers. On FTDMAC020 also the SIZE
394 if (pl08x->vd->pl080s) in pl08x_write_lli()
395 dev_vdbg(&pl08x->adev->dev, in pl08x_write_lli()
398 phychan->id, lli[PL080_LLI_SRC], lli[PL080_LLI_DST], in pl08x_write_lli()
402 dev_vdbg(&pl08x->adev->dev, in pl08x_write_lli()
405 phychan->id, lli[PL080_LLI_SRC], lli[PL080_LLI_DST], in pl08x_write_lli()
408 writel_relaxed(lli[PL080_LLI_SRC], phychan->reg_src); in pl08x_write_lli()
409 writel_relaxed(lli[PL080_LLI_DST], phychan->reg_dst); in pl08x_write_lli()
410 writel_relaxed(lli[PL080_LLI_LLI], phychan->reg_lli); in pl08x_write_lli()
414 * and the CCTL register which is split in CSR and SIZE registers. in pl08x_write_lli()
416 * the CSR and SIZE registers. in pl08x_write_lli()
418 if (phychan->ftdmac020) { in pl08x_write_lli()
422 /* Write the transfer size (12 bits) to the size register */ in pl08x_write_lli()
424 phychan->base + FTDMAC020_CH_SIZE); in pl08x_write_lli()
429 * Bit 28: TC_MSK - mask on all except last LLI in pl08x_write_lli()
440 (FTDMAC020_LLI_SRC_WIDTH_SHIFT - in pl08x_write_lli()
443 (FTDMAC020_LLI_DST_WIDTH_SHIFT - in pl08x_write_lli()
446 (FTDMAC020_LLI_SRCAD_CTL_SHIFT - in pl08x_write_lli()
449 (FTDMAC020_LLI_DSTAD_CTL_SHIFT - in pl08x_write_lli()
463 switch (pl08x->pd->memcpy_burst_size) { in pl08x_write_lli()
500 if (pl08x->pd->memcpy_prot_buff) in pl08x_write_lli()
502 if (pl08x->pd->memcpy_prot_cache) in pl08x_write_lli()
507 writel_relaxed(val, phychan->reg_control); in pl08x_write_lli()
510 writel_relaxed(lli[PL080_LLI_CCTL], phychan->reg_control); in pl08x_write_lli()
514 if (pl08x->vd->pl080s) in pl08x_write_lli()
516 phychan->base + PL080S_CH_CONTROL2); in pl08x_write_lli()
518 writel(ccfg, phychan->reg_config); in pl08x_write_lli()
529 struct pl08x_driver_data *pl08x = plchan->host; in pl08x_start_next_txd()
530 struct pl08x_phy_chan *phychan = plchan->phychan; in pl08x_start_next_txd()
531 struct virt_dma_desc *vd = vchan_next_desc(&plchan->vc); in pl08x_start_next_txd()
532 struct pl08x_txd *txd = to_pl08x_txd(&vd->tx); in pl08x_start_next_txd()
535 list_del(&txd->vd.node); in pl08x_start_next_txd()
537 plchan->at = txd; in pl08x_start_next_txd()
543 pl08x_write_lli(pl08x, phychan, &txd->llis_va[0], txd->ccfg); in pl08x_start_next_txd()
547 while (readl(pl08x->base + PL080_EN_CHAN) & BIT(phychan->id)) in pl08x_start_next_txd()
551 if (phychan->ftdmac020) { in pl08x_start_next_txd()
552 val = readl(phychan->reg_config); in pl08x_start_next_txd()
554 val = readl(phychan->reg_config); in pl08x_start_next_txd()
556 val = readl(phychan->reg_control); in pl08x_start_next_txd()
558 val = readl(phychan->reg_control); in pl08x_start_next_txd()
561 phychan->reg_control); in pl08x_start_next_txd()
563 val = readl(phychan->reg_config); in pl08x_start_next_txd()
566 val = readl(phychan->reg_config); in pl08x_start_next_txd()
568 writel(val | PL080_CONFIG_ENABLE, phychan->reg_config); in pl08x_start_next_txd()
575 * For M->P transfers, pause the DMAC first and then stop the peripheral -
576 * the FIFO can only drain if the peripheral is still requesting data.
577 * (note: this can still timeout if the DMAC FIFO never drains of data.)
579 * For P->M transfers, disable the peripheral first to stop it filling
580 * the DMAC FIFO, and then pause the DMAC.
587 if (ch->ftdmac020) { in pl08x_pause_phy_chan()
589 val = readl(ch->reg_control); in pl08x_pause_phy_chan()
591 writel(val, ch->reg_control); in pl08x_pause_phy_chan()
595 /* Set the HALT bit and wait for the FIFO to drain */ in pl08x_pause_phy_chan()
596 val = readl(ch->reg_config); in pl08x_pause_phy_chan()
598 writel(val, ch->reg_config); in pl08x_pause_phy_chan()
601 for (timeout = 1000; timeout; timeout--) { in pl08x_pause_phy_chan()
607 pr_err("pl08x: channel%u timeout waiting for pause\n", ch->id); in pl08x_pause_phy_chan()
615 if (ch->ftdmac020) { in pl08x_resume_phy_chan()
616 val = readl(ch->reg_control); in pl08x_resume_phy_chan()
618 writel(val, ch->reg_control); in pl08x_resume_phy_chan()
623 val = readl(ch->reg_config); in pl08x_resume_phy_chan()
625 writel(val, ch->reg_config); in pl08x_resume_phy_chan()
629 * pl08x_terminate_phy_chan() stops the channel, clears the FIFO and
631 * an on-going transfer, but as a method of shutting down a channel
640 if (ch->ftdmac020) { in pl08x_terminate_phy_chan()
642 val = readl(ch->reg_config); in pl08x_terminate_phy_chan()
646 writel(val, ch->reg_config); in pl08x_terminate_phy_chan()
649 val = readl(ch->reg_control); in pl08x_terminate_phy_chan()
652 writel(val, ch->reg_control); in pl08x_terminate_phy_chan()
655 writel(BIT(ch->id) | BIT(ch->id + 16), in pl08x_terminate_phy_chan()
656 pl08x->base + PL080_ERR_CLEAR); in pl08x_terminate_phy_chan()
657 writel(BIT(ch->id), pl08x->base + PL080_TC_CLEAR); in pl08x_terminate_phy_chan()
662 val = readl(ch->reg_config); in pl08x_terminate_phy_chan()
665 writel(val, ch->reg_config); in pl08x_terminate_phy_chan()
667 writel(BIT(ch->id), pl08x->base + PL080_ERR_CLEAR); in pl08x_terminate_phy_chan()
668 writel(BIT(ch->id), pl08x->base + PL080_TC_CLEAR); in pl08x_terminate_phy_chan()
676 if (ch->ftdmac020) { in get_bytes_in_phy_channel()
677 bytes = readl(ch->base + FTDMAC020_CH_SIZE); in get_bytes_in_phy_channel()
679 val = readl(ch->reg_control); in get_bytes_in_phy_channel()
682 } else if (ch->pl080s) { in get_bytes_in_phy_channel()
683 val = readl(ch->base + PL080S_CH_CONTROL2); in get_bytes_in_phy_channel()
686 val = readl(ch->reg_control); in get_bytes_in_phy_channel()
691 val = readl(ch->reg_control); in get_bytes_in_phy_channel()
716 if (ch->ftdmac020) { in get_bytes_in_lli()
723 } else if (ch->pl080s) { in get_bytes_in_lli()
755 struct pl08x_driver_data *pl08x = plchan->host; in pl08x_getbytes_chan()
764 ch = plchan->phychan; in pl08x_getbytes_chan()
765 txd = plchan->at; in pl08x_getbytes_chan()
774 clli = readl(ch->reg_lli) & ~PL080_LLI_LM_AHB2; in pl08x_getbytes_chan()
782 llis_va = txd->llis_va; in pl08x_getbytes_chan()
783 llis_bus = txd->llis_bus; in pl08x_getbytes_chan()
785 llis_max_words = pl08x->lli_words * MAX_NUM_TSFR_LLIS; in pl08x_getbytes_chan()
790 * Locate the next LLI - as this is an array, in pl08x_getbytes_chan()
793 llis_va += (clli - llis_bus) / sizeof(u32); in pl08x_getbytes_chan()
797 for (; llis_va < llis_va_limit; llis_va += pl08x->lli_words) { in pl08x_getbytes_chan()
825 for (i = 0; i < pl08x->vd->channels; i++) { in pl08x_get_phy_channel()
826 ch = &pl08x->phy_chans[i]; in pl08x_get_phy_channel()
828 spin_lock_irqsave(&ch->lock, flags); in pl08x_get_phy_channel()
830 if (!ch->locked && !ch->serving) { in pl08x_get_phy_channel()
831 ch->serving = virt_chan; in pl08x_get_phy_channel()
832 spin_unlock_irqrestore(&ch->lock, flags); in pl08x_get_phy_channel()
836 spin_unlock_irqrestore(&ch->lock, flags); in pl08x_get_phy_channel()
839 if (i == pl08x->vd->channels) { in pl08x_get_phy_channel()
851 ch->serving = NULL; in pl08x_put_phy_channel()
861 struct pl08x_driver_data *pl08x = plchan->host; in pl08x_phy_alloc_and_start()
866 dev_dbg(&pl08x->adev->dev, "no physical channel available for xfer on %s\n", plchan->name); in pl08x_phy_alloc_and_start()
867 plchan->state = PL08X_CHAN_WAITING; in pl08x_phy_alloc_and_start()
868 plchan->waiting_at = jiffies; in pl08x_phy_alloc_and_start()
872 dev_dbg(&pl08x->adev->dev, "allocated physical channel %d for xfer on %s\n", in pl08x_phy_alloc_and_start()
873 ch->id, plchan->name); in pl08x_phy_alloc_and_start()
875 plchan->phychan = ch; in pl08x_phy_alloc_and_start()
876 plchan->state = PL08X_CHAN_RUNNING; in pl08x_phy_alloc_and_start()
883 struct pl08x_driver_data *pl08x = plchan->host; in pl08x_phy_reassign_start()
885 dev_dbg(&pl08x->adev->dev, "reassigned physical channel %d for xfer on %s\n", in pl08x_phy_reassign_start()
886 ch->id, plchan->name); in pl08x_phy_reassign_start()
891 * that this will only be called when it _already_ is non-NULL. in pl08x_phy_reassign_start()
893 ch->serving = plchan; in pl08x_phy_reassign_start()
894 plchan->phychan = ch; in pl08x_phy_reassign_start()
895 plchan->state = PL08X_CHAN_RUNNING; in pl08x_phy_reassign_start()
905 struct pl08x_driver_data *pl08x = plchan->host; in pl08x_phy_free()
917 list_for_each_entry(p, &pl08x->memcpy.channels, vc.chan.device_node) in pl08x_phy_free()
918 if (p->state == PL08X_CHAN_WAITING && in pl08x_phy_free()
919 p->waiting_at <= waiting_at) { in pl08x_phy_free()
921 waiting_at = p->waiting_at; in pl08x_phy_free()
924 if (!next && pl08x->has_slave) { in pl08x_phy_free()
925 list_for_each_entry(p, &pl08x->slave.channels, vc.chan.device_node) in pl08x_phy_free()
926 if (p->state == PL08X_CHAN_WAITING && in pl08x_phy_free()
927 p->waiting_at <= waiting_at) { in pl08x_phy_free()
929 waiting_at = p->waiting_at; in pl08x_phy_free()
934 pl08x_terminate_phy_chan(pl08x, plchan->phychan); in pl08x_phy_free()
943 spin_lock(&next->vc.lock); in pl08x_phy_free()
944 /* Re-check the state now that we have the lock */ in pl08x_phy_free()
945 success = next->state == PL08X_CHAN_WAITING; in pl08x_phy_free()
947 pl08x_phy_reassign_start(plchan->phychan, next); in pl08x_phy_free()
948 spin_unlock(&next->vc.lock); in pl08x_phy_free()
955 pl08x_put_phy_channel(pl08x, plchan->phychan); in pl08x_phy_free()
958 plchan->phychan = NULL; in pl08x_phy_free()
959 plchan->state = PL08X_CHAN_IDLE; in pl08x_phy_free()
973 if (pl08x->vd->ftdmac020) { in pl08x_get_bytes_for_lli()
1011 * Remove all src, dst and transfer size bits, then set the in pl08x_lli_control_bits()
1012 * width and size according to the parameters. The bit offsets in pl08x_lli_control_bits()
1015 if (pl08x->vd->ftdmac020) { in pl08x_lli_control_bits()
1120 * - prefers the destination bus if both available
1121 * - prefers bus with fixed address (i.e. peripheral)
1133 * The FTDMAC020 only supports memory-to-memory transfer, so in pl08x_choose_master_bus()
1136 if (pl08x->vd->ftdmac020) { in pl08x_choose_master_bus()
1149 *mbus = &bd->dstbus; in pl08x_choose_master_bus()
1150 *sbus = &bd->srcbus; in pl08x_choose_master_bus()
1152 *mbus = &bd->srcbus; in pl08x_choose_master_bus()
1153 *sbus = &bd->dstbus; in pl08x_choose_master_bus()
1155 if (bd->dstbus.buswidth >= bd->srcbus.buswidth) { in pl08x_choose_master_bus()
1156 *mbus = &bd->dstbus; in pl08x_choose_master_bus()
1157 *sbus = &bd->srcbus; in pl08x_choose_master_bus()
1159 *mbus = &bd->srcbus; in pl08x_choose_master_bus()
1160 *sbus = &bd->dstbus; in pl08x_choose_master_bus()
1172 u32 offset = num_llis * pl08x->lli_words; in pl08x_fill_lli_for_desc()
1173 u32 *llis_va = bd->txd->llis_va + offset; in pl08x_fill_lli_for_desc()
1174 dma_addr_t llis_bus = bd->txd->llis_bus; in pl08x_fill_lli_for_desc()
1179 offset += pl08x->lli_words; in pl08x_fill_lli_for_desc()
1181 llis_va[PL080_LLI_SRC] = bd->srcbus.addr; in pl08x_fill_lli_for_desc()
1182 llis_va[PL080_LLI_DST] = bd->dstbus.addr; in pl08x_fill_lli_for_desc()
1184 llis_va[PL080_LLI_LLI] |= bd->lli_bus; in pl08x_fill_lli_for_desc()
1186 if (pl08x->vd->pl080s) in pl08x_fill_lli_for_desc()
1189 if (pl08x->vd->ftdmac020) { in pl08x_fill_lli_for_desc()
1191 bd->srcbus.addr += len; in pl08x_fill_lli_for_desc()
1192 bd->dstbus.addr += len; in pl08x_fill_lli_for_desc()
1195 bd->srcbus.addr += len; in pl08x_fill_lli_for_desc()
1197 bd->dstbus.addr += len; in pl08x_fill_lli_for_desc()
1200 BUG_ON(bd->remainder < len); in pl08x_fill_lli_for_desc()
1202 bd->remainder -= len; in pl08x_fill_lli_for_desc()
1220 if (pl08x->vd->pl080s) { in pl08x_dump_lli()
1221 dev_vdbg(&pl08x->adev->dev, in pl08x_dump_lli()
1222 "%-3s %-9s %-10s %-10s %-10s %-10s %s\n", in pl08x_dump_lli()
1225 dev_vdbg(&pl08x->adev->dev, in pl08x_dump_lli()
1231 llis_va += pl08x->lli_words; in pl08x_dump_lli()
1234 dev_vdbg(&pl08x->adev->dev, in pl08x_dump_lli()
1235 "%-3s %-9s %-10s %-10s %-10s %s\n", in pl08x_dump_lli()
1238 dev_vdbg(&pl08x->adev->dev, in pl08x_dump_lli()
1243 llis_va += pl08x->lli_words; in pl08x_dump_lli()
1268 txd->llis_va = dma_pool_alloc(pl08x->pool, GFP_NOWAIT, &txd->llis_bus); in pl08x_fill_llis_for_desc()
1269 if (!txd->llis_va) { in pl08x_fill_llis_for_desc()
1270 dev_err(&pl08x->adev->dev, "%s no memory for llis\n", __func__); in pl08x_fill_llis_for_desc()
1275 bd.lli_bus = (pl08x->lli_buses & PL08X_AHB2) ? PL080_LLI_LM_AHB2 : 0; in pl08x_fill_llis_for_desc()
1276 cctl = txd->cctl; in pl08x_fill_llis_for_desc()
1284 list_for_each_entry(dsg, &txd->dsg_list, node) { in pl08x_fill_llis_for_desc()
1286 cctl = txd->cctl; in pl08x_fill_llis_for_desc()
1288 bd.srcbus.addr = dsg->src_addr; in pl08x_fill_llis_for_desc()
1289 bd.dstbus.addr = dsg->dst_addr; in pl08x_fill_llis_for_desc()
1290 bd.remainder = dsg->len; in pl08x_fill_llis_for_desc()
1296 dev_vdbg(&pl08x->adev->dev, in pl08x_fill_llis_for_desc()
1305 dev_vdbg(&pl08x->adev->dev, "mbus=%s sbus=%s\n", in pl08x_fill_llis_for_desc()
1312 * - flow controller is peripheral. in pl08x_fill_llis_for_desc()
1313 * - src.addr is aligned to src.width in pl08x_fill_llis_for_desc()
1314 * - dst.addr is aligned to dst.width in pl08x_fill_llis_for_desc()
1318 * - Memory addresses are contiguous and are not scattered. in pl08x_fill_llis_for_desc()
1324 * - Memory addresses are scattered and are not contiguous. in pl08x_fill_llis_for_desc()
1333 /* FTDMAC020 only does memory-to-memory */ in pl08x_fill_llis_for_desc()
1334 if (pl08x->vd->ftdmac020) in pl08x_fill_llis_for_desc()
1337 fc = (txd->ccfg & PL080_CONFIG_FLOW_CONTROL_MASK) >> in pl08x_fill_llis_for_desc()
1341 dev_err(&pl08x->adev->dev, "%s sg len can't be zero", in pl08x_fill_llis_for_desc()
1348 dev_err(&pl08x->adev->dev, in pl08x_fill_llis_for_desc()
1365 * - Less than a bus width available in pl08x_fill_llis_for_desc()
1366 * - until master bus is aligned in pl08x_fill_llis_for_desc()
1368 if (bd.remainder < mbus->buswidth) in pl08x_fill_llis_for_desc()
1371 early_bytes = mbus->buswidth - in pl08x_fill_llis_for_desc()
1372 (mbus->addr & (mbus->buswidth - 1)); in pl08x_fill_llis_for_desc()
1373 if ((bd.remainder - early_bytes) < mbus->buswidth) in pl08x_fill_llis_for_desc()
1378 dev_vdbg(&pl08x->adev->dev, in pl08x_fill_llis_for_desc()
1388 * - if slave is not then we must set its width down in pl08x_fill_llis_for_desc()
1391 dev_dbg(&pl08x->adev->dev, in pl08x_fill_llis_for_desc()
1395 sbus->buswidth = 1; in pl08x_fill_llis_for_desc()
1403 pl08x->vd->max_transfer_size; in pl08x_fill_llis_for_desc()
1404 dev_vdbg(&pl08x->adev->dev, in pl08x_fill_llis_for_desc()
1412 while (bd.remainder > (mbus->buswidth - 1)) { in pl08x_fill_llis_for_desc()
1423 * Calculate actual transfer size in relation to in pl08x_fill_llis_for_desc()
1425 * highest bus width - 1 in pl08x_fill_llis_for_desc()
1427 width = max(mbus->buswidth, sbus->buswidth); in pl08x_fill_llis_for_desc()
1431 dev_vdbg(&pl08x->adev->dev, in pl08x_fill_llis_for_desc()
1433 "size 0x%08zx (remainder 0x%08zx)\n", in pl08x_fill_llis_for_desc()
1448 dev_vdbg(&pl08x->adev->dev, in pl08x_fill_llis_for_desc()
1456 if (total_bytes != dsg->len) { in pl08x_fill_llis_for_desc()
1457 dev_err(&pl08x->adev->dev, in pl08x_fill_llis_for_desc()
1458 "%s size of encoded lli:s don't match total txd, transferred 0x%08zx from size 0x%08zx\n", in pl08x_fill_llis_for_desc()
1459 __func__, total_bytes, dsg->len); in pl08x_fill_llis_for_desc()
1464 dev_err(&pl08x->adev->dev, in pl08x_fill_llis_for_desc()
1471 llis_va = txd->llis_va; in pl08x_fill_llis_for_desc()
1472 last_lli = llis_va + (num_llis - 1) * pl08x->lli_words; in pl08x_fill_llis_for_desc()
1474 if (txd->cyclic) { in pl08x_fill_llis_for_desc()
1476 last_lli[PL080_LLI_LLI] = txd->llis_bus | bd.lli_bus; in pl08x_fill_llis_for_desc()
1481 if (pl08x->vd->ftdmac020) in pl08x_fill_llis_for_desc()
1497 if (txd->llis_va) in pl08x_free_txd()
1498 dma_pool_free(pl08x->pool, txd->llis_va, txd->llis_bus); in pl08x_free_txd()
1500 list_for_each_entry_safe(dsg, _dsg, &txd->dsg_list, node) { in pl08x_free_txd()
1501 list_del(&dsg->node); in pl08x_free_txd()
1510 struct pl08x_txd *txd = to_pl08x_txd(&vd->tx); in pl08x_desc_free()
1511 struct pl08x_dma_chan *plchan = to_pl08x_chan(vd->tx.chan); in pl08x_desc_free()
1513 dma_descriptor_unmap(&vd->tx); in pl08x_desc_free()
1514 if (!txd->done) in pl08x_desc_free()
1517 pl08x_free_txd(plchan->host, txd); in pl08x_desc_free()
1525 vchan_get_all_descriptors(&plchan->vc, &head); in pl08x_free_txd_list()
1526 vchan_dma_desc_free_list(&plchan->vc, &head); in pl08x_free_txd_list()
1569 if (plchan->state == PL08X_CHAN_PAUSED) in pl08x_dma_tx_status()
1574 spin_lock_irqsave(&plchan->vc.lock, flags); in pl08x_dma_tx_status()
1577 vd = vchan_find_desc(&plchan->vc, cookie); in pl08x_dma_tx_status()
1580 struct pl08x_txd *txd = to_pl08x_txd(&vd->tx); in pl08x_dma_tx_status()
1583 list_for_each_entry(dsg, &txd->dsg_list, node) in pl08x_dma_tx_status()
1584 bytes += dsg->len; in pl08x_dma_tx_status()
1589 spin_unlock_irqrestore(&plchan->vc.lock, flags); in pl08x_dma_tx_status()
1597 if (plchan->state == PL08X_CHAN_PAUSED && ret == DMA_IN_PROGRESS) in pl08x_dma_tx_status()
1679 /* Access the cell in privileged mode, non-bufferable, non-cacheable */ in pl08x_cctl()
1725 if (plchan->cd->single) in pl08x_get_cctl()
1744 spin_lock_irqsave(&plchan->vc.lock, flags); in pl08x_issue_pending()
1745 if (vchan_issue_pending(&plchan->vc)) { in pl08x_issue_pending()
1746 if (!plchan->phychan && plchan->state != PL08X_CHAN_WAITING) in pl08x_issue_pending()
1749 spin_unlock_irqrestore(&plchan->vc.lock, flags); in pl08x_issue_pending()
1757 INIT_LIST_HEAD(&txd->dsg_list); in pl08x_get_txd()
1766 switch (pl08x->pd->memcpy_burst_size) { in pl08x_memcpy_cctl()
1768 dev_err(&pl08x->adev->dev, in pl08x_memcpy_cctl()
1769 "illegal burst size for memcpy, set to 1\n"); in pl08x_memcpy_cctl()
1805 switch (pl08x->pd->memcpy_bus_width) { in pl08x_memcpy_cctl()
1807 dev_err(&pl08x->adev->dev, in pl08x_memcpy_cctl()
1825 if (pl08x->pd->memcpy_prot_buff) in pl08x_memcpy_cctl()
1827 if (pl08x->pd->memcpy_prot_cache) in pl08x_memcpy_cctl()
1836 if (pl08x->vd->dualmaster) in pl08x_memcpy_cctl()
1838 pl08x->mem_buses, in pl08x_memcpy_cctl()
1839 pl08x->mem_buses); in pl08x_memcpy_cctl()
1849 switch (pl08x->pd->memcpy_bus_width) { in pl08x_ftdmac020_memcpy_cctl()
1851 dev_err(&pl08x->adev->dev, in pl08x_ftdmac020_memcpy_cctl()
1878 if (pl08x->vd->dualmaster) in pl08x_ftdmac020_memcpy_cctl()
1880 pl08x->mem_buses, in pl08x_ftdmac020_memcpy_cctl()
1881 pl08x->mem_buses); in pl08x_ftdmac020_memcpy_cctl()
1894 struct pl08x_driver_data *pl08x = plchan->host; in pl08x_prep_dma_memcpy()
1901 dev_err(&pl08x->adev->dev, in pl08x_prep_dma_memcpy()
1911 list_add_tail(&dsg->node, &txd->dsg_list); in pl08x_prep_dma_memcpy()
1913 dsg->src_addr = src; in pl08x_prep_dma_memcpy()
1914 dsg->dst_addr = dest; in pl08x_prep_dma_memcpy()
1915 dsg->len = len; in pl08x_prep_dma_memcpy()
1916 if (pl08x->vd->ftdmac020) { in pl08x_prep_dma_memcpy()
1918 txd->ccfg = 0; in pl08x_prep_dma_memcpy()
1919 txd->cctl = pl08x_ftdmac020_memcpy_cctl(pl08x); in pl08x_prep_dma_memcpy()
1921 txd->ccfg = PL080_CONFIG_ERR_IRQ_MASK | in pl08x_prep_dma_memcpy()
1924 txd->cctl = pl08x_memcpy_cctl(pl08x); in pl08x_prep_dma_memcpy()
1927 ret = pl08x_fill_llis_for_desc(plchan->host, txd); in pl08x_prep_dma_memcpy()
1933 return vchan_tx_prep(&plchan->vc, &txd->vd, flags); in pl08x_prep_dma_memcpy()
1942 struct pl08x_driver_data *pl08x = plchan->host; in pl08x_init_txd()
1951 dev_err(&pl08x->adev->dev, "%s no txd\n", __func__); in pl08x_init_txd()
1962 *slave_addr = plchan->cfg.dst_addr; in pl08x_init_txd()
1963 addr_width = plchan->cfg.dst_addr_width; in pl08x_init_txd()
1964 maxburst = plchan->cfg.dst_maxburst; in pl08x_init_txd()
1965 src_buses = pl08x->mem_buses; in pl08x_init_txd()
1966 dst_buses = plchan->cd->periph_buses; in pl08x_init_txd()
1969 *slave_addr = plchan->cfg.src_addr; in pl08x_init_txd()
1970 addr_width = plchan->cfg.src_addr_width; in pl08x_init_txd()
1971 maxburst = plchan->cfg.src_maxburst; in pl08x_init_txd()
1972 src_buses = plchan->cd->periph_buses; in pl08x_init_txd()
1973 dst_buses = pl08x->mem_buses; in pl08x_init_txd()
1976 dev_err(&pl08x->adev->dev, in pl08x_init_txd()
1984 dev_err(&pl08x->adev->dev, in pl08x_init_txd()
1989 txd->cctl = cctl | pl08x_select_bus(false, src_buses, dst_buses); in pl08x_init_txd()
1991 if (plchan->cfg.device_fc) in pl08x_init_txd()
1998 txd->ccfg = PL080_CONFIG_ERR_IRQ_MASK | in pl08x_init_txd()
2005 dev_dbg(&pl08x->adev->dev, in pl08x_init_txd()
2007 plchan->name); in pl08x_init_txd()
2011 dev_dbg(&pl08x->adev->dev, "allocated DMA request signal %d for xfer on %s\n", in pl08x_init_txd()
2012 plchan->signal, plchan->name); in pl08x_init_txd()
2016 txd->ccfg |= plchan->signal << PL080_CONFIG_DST_SEL_SHIFT; in pl08x_init_txd()
2018 txd->ccfg |= plchan->signal << PL080_CONFIG_SRC_SEL_SHIFT; in pl08x_init_txd()
2033 return -ENOMEM; in pl08x_tx_add_sg()
2035 list_add_tail(&dsg->node, &txd->dsg_list); in pl08x_tx_add_sg()
2037 dsg->len = len; in pl08x_tx_add_sg()
2039 dsg->src_addr = buf_addr; in pl08x_tx_add_sg()
2040 dsg->dst_addr = slave_addr; in pl08x_tx_add_sg()
2042 dsg->src_addr = slave_addr; in pl08x_tx_add_sg()
2043 dsg->dst_addr = buf_addr; in pl08x_tx_add_sg()
2055 struct pl08x_driver_data *pl08x = plchan->host; in pl08x_prep_slave_sg()
2061 dev_dbg(&pl08x->adev->dev, "%s prepare transaction of %d bytes from %s\n", in pl08x_prep_slave_sg()
2062 __func__, sg_dma_len(sgl), plchan->name); in pl08x_prep_slave_sg()
2075 dev_err(&pl08x->adev->dev, "%s no mem for pl080 sg\n", in pl08x_prep_slave_sg()
2081 ret = pl08x_fill_llis_for_desc(plchan->host, txd); in pl08x_prep_slave_sg()
2088 return vchan_tx_prep(&plchan->vc, &txd->vd, flags); in pl08x_prep_slave_sg()
2097 struct pl08x_driver_data *pl08x = plchan->host; in pl08x_prep_dma_cyclic()
2102 dev_dbg(&pl08x->adev->dev, in pl08x_prep_dma_cyclic()
2106 plchan->name); in pl08x_prep_dma_cyclic()
2112 txd->cyclic = true; in pl08x_prep_dma_cyclic()
2113 txd->cctl |= PL080_CONTROL_TC_IRQ_EN; in pl08x_prep_dma_cyclic()
2124 ret = pl08x_fill_llis_for_desc(plchan->host, txd); in pl08x_prep_dma_cyclic()
2131 return vchan_tx_prep(&plchan->vc, &txd->vd, flags); in pl08x_prep_dma_cyclic()
2138 struct pl08x_driver_data *pl08x = plchan->host; in pl08x_config()
2140 if (!plchan->slave) in pl08x_config()
2141 return -EINVAL; in pl08x_config()
2144 if (config->src_addr_width == DMA_SLAVE_BUSWIDTH_8_BYTES || in pl08x_config()
2145 config->dst_addr_width == DMA_SLAVE_BUSWIDTH_8_BYTES) in pl08x_config()
2146 return -EINVAL; in pl08x_config()
2148 if (config->device_fc && pl08x->vd->pl080s) { in pl08x_config()
2149 dev_err(&pl08x->adev->dev, in pl08x_config()
2152 return -EINVAL; in pl08x_config()
2155 plchan->cfg = *config; in pl08x_config()
2163 struct pl08x_driver_data *pl08x = plchan->host; in pl08x_terminate_all()
2166 spin_lock_irqsave(&plchan->vc.lock, flags); in pl08x_terminate_all()
2167 if (!plchan->phychan && !plchan->at) { in pl08x_terminate_all()
2168 spin_unlock_irqrestore(&plchan->vc.lock, flags); in pl08x_terminate_all()
2172 plchan->state = PL08X_CHAN_IDLE; in pl08x_terminate_all()
2174 if (plchan->phychan) { in pl08x_terminate_all()
2182 if (plchan->at) { in pl08x_terminate_all()
2183 vchan_terminate_vdesc(&plchan->at->vd); in pl08x_terminate_all()
2184 plchan->at = NULL; in pl08x_terminate_all()
2189 spin_unlock_irqrestore(&plchan->vc.lock, flags); in pl08x_terminate_all()
2198 vchan_synchronize(&plchan->vc); in pl08x_synchronize()
2210 spin_lock_irqsave(&plchan->vc.lock, flags); in pl08x_pause()
2211 if (!plchan->phychan && !plchan->at) { in pl08x_pause()
2212 spin_unlock_irqrestore(&plchan->vc.lock, flags); in pl08x_pause()
2216 pl08x_pause_phy_chan(plchan->phychan); in pl08x_pause()
2217 plchan->state = PL08X_CHAN_PAUSED; in pl08x_pause()
2219 spin_unlock_irqrestore(&plchan->vc.lock, flags); in pl08x_pause()
2233 spin_lock_irqsave(&plchan->vc.lock, flags); in pl08x_resume()
2234 if (!plchan->phychan && !plchan->at) { in pl08x_resume()
2235 spin_unlock_irqrestore(&plchan->vc.lock, flags); in pl08x_resume()
2239 pl08x_resume_phy_chan(plchan->phychan); in pl08x_resume()
2240 plchan->state = PL08X_CHAN_RUNNING; in pl08x_resume()
2242 spin_unlock_irqrestore(&plchan->vc.lock, flags); in pl08x_resume()
2253 if (chan->device->dev->driver != &pl08x_amba_driver.drv) in pl08x_filter_id()
2259 if (!strcmp(plchan->name, name)) in pl08x_filter_id()
2270 return plchan->cd == chan_id; in pl08x_filter_fn()
2282 if (pl08x->vd->nomadik) in pl08x_ensure_on()
2285 if (pl08x->vd->ftdmac020) { in pl08x_ensure_on()
2286 writel(PL080_CONFIG_ENABLE, pl08x->base + FTDMAC020_CSR); in pl08x_ensure_on()
2289 writel(PL080_CONFIG_ENABLE, pl08x->base + PL080_CONFIG); in pl08x_ensure_on()
2297 /* check & clear - ERR & TC interrupts */ in pl08x_irq()
2298 err = readl(pl08x->base + PL080_ERR_STATUS); in pl08x_irq()
2300 dev_err(&pl08x->adev->dev, "%s error interrupt, register value 0x%08x\n", in pl08x_irq()
2302 writel(err, pl08x->base + PL080_ERR_CLEAR); in pl08x_irq()
2304 tc = readl(pl08x->base + PL080_TC_STATUS); in pl08x_irq()
2306 writel(tc, pl08x->base + PL080_TC_CLEAR); in pl08x_irq()
2311 for (i = 0; i < pl08x->vd->channels; i++) { in pl08x_irq()
2314 struct pl08x_phy_chan *phychan = &pl08x->phy_chans[i]; in pl08x_irq()
2315 struct pl08x_dma_chan *plchan = phychan->serving; in pl08x_irq()
2316 struct pl08x_txd *tx; in pl08x_irq() local
2319 dev_err(&pl08x->adev->dev, in pl08x_irq()
2325 spin_lock(&plchan->vc.lock); in pl08x_irq()
2326 tx = plchan->at; in pl08x_irq()
2327 if (tx && tx->cyclic) { in pl08x_irq()
2328 vchan_cyclic_callback(&tx->vd); in pl08x_irq()
2329 } else if (tx) { in pl08x_irq()
2330 plchan->at = NULL; in pl08x_irq()
2336 tx->done = true; in pl08x_irq()
2337 vchan_cookie_complete(&tx->vd); in pl08x_irq()
2343 if (vchan_next_desc(&plchan->vc)) in pl08x_irq()
2348 spin_unlock(&plchan->vc.lock); in pl08x_irq()
2359 chan->slave = true; in pl08x_dma_slave_init()
2360 chan->name = chan->cd->bus_id; in pl08x_dma_slave_init()
2361 chan->cfg.src_addr = chan->cd->addr; in pl08x_dma_slave_init()
2362 chan->cfg.dst_addr = chan->cd->addr; in pl08x_dma_slave_init()
2375 INIT_LIST_HEAD(&dmadev->channels); in pl08x_dma_init_virtual_channels()
2385 return -ENOMEM; in pl08x_dma_init_virtual_channels()
2387 chan->host = pl08x; in pl08x_dma_init_virtual_channels()
2388 chan->state = PL08X_CHAN_IDLE; in pl08x_dma_init_virtual_channels()
2389 chan->signal = -1; in pl08x_dma_init_virtual_channels()
2392 chan->cd = &pl08x->pd->slave_channels[i]; in pl08x_dma_init_virtual_channels()
2398 chan->signal = i; in pl08x_dma_init_virtual_channels()
2401 chan->cd = kzalloc(sizeof(*chan->cd), GFP_KERNEL); in pl08x_dma_init_virtual_channels()
2402 if (!chan->cd) { in pl08x_dma_init_virtual_channels()
2404 return -ENOMEM; in pl08x_dma_init_virtual_channels()
2406 chan->cd->bus_id = "memcpy"; in pl08x_dma_init_virtual_channels()
2407 chan->cd->periph_buses = pl08x->pd->mem_buses; in pl08x_dma_init_virtual_channels()
2408 chan->name = kasprintf(GFP_KERNEL, "memcpy%d", i); in pl08x_dma_init_virtual_channels()
2409 if (!chan->name) { in pl08x_dma_init_virtual_channels()
2410 kfree(chan->cd); in pl08x_dma_init_virtual_channels()
2412 return -ENOMEM; in pl08x_dma_init_virtual_channels()
2415 dev_dbg(&pl08x->adev->dev, in pl08x_dma_init_virtual_channels()
2417 chan->name); in pl08x_dma_init_virtual_channels()
2419 chan->vc.desc_free = pl08x_desc_free; in pl08x_dma_init_virtual_channels()
2420 vchan_init(&chan->vc, dmadev); in pl08x_dma_init_virtual_channels()
2422 dev_info(&pl08x->adev->dev, "initialized %d virtual %s channels\n", in pl08x_dma_init_virtual_channels()
2433 next, &dmadev->channels, vc.chan.device_node) { in pl08x_free_virtual_channels()
2434 list_del(&chan->vc.chan.device_node); in pl08x_free_virtual_channels()
2459 struct pl08x_driver_data *pl08x = s->private; in pl08x_debugfs_show()
2467 seq_printf(s, "--------\t-----\n"); in pl08x_debugfs_show()
2468 for (i = 0; i < pl08x->vd->channels; i++) { in pl08x_debugfs_show()
2471 ch = &pl08x->phy_chans[i]; in pl08x_debugfs_show()
2473 spin_lock_irqsave(&ch->lock, flags); in pl08x_debugfs_show()
2474 virt_chan = ch->serving; in pl08x_debugfs_show()
2477 ch->id, in pl08x_debugfs_show()
2478 virt_chan ? virt_chan->name : "(none)", in pl08x_debugfs_show()
2479 ch->locked ? " LOCKED" : ""); in pl08x_debugfs_show()
2481 spin_unlock_irqrestore(&ch->lock, flags); in pl08x_debugfs_show()
2486 seq_printf(s, "--------\t------\n"); in pl08x_debugfs_show()
2487 list_for_each_entry(chan, &pl08x->memcpy.channels, vc.chan.device_node) { in pl08x_debugfs_show()
2488 seq_printf(s, "%s\t\t%s\n", chan->name, in pl08x_debugfs_show()
2489 pl08x_state_str(chan->state)); in pl08x_debugfs_show()
2492 if (pl08x->has_slave) { in pl08x_debugfs_show()
2495 seq_printf(s, "--------\t------\n"); in pl08x_debugfs_show()
2496 list_for_each_entry(chan, &pl08x->slave.channels, in pl08x_debugfs_show()
2498 seq_printf(s, "%s\t\t%s\n", chan->name, in pl08x_debugfs_show()
2499 pl08x_state_str(chan->state)); in pl08x_debugfs_show()
2511 debugfs_create_file(dev_name(&pl08x->adev->dev), S_IFREG | S_IRUGO, in init_pl08x_debugfs()
2528 if (!pl08x->has_slave) in pl08x_find_chan_id()
2531 list_for_each_entry(chan, &pl08x->slave.channels, vc.chan.device_node) { in pl08x_find_chan_id()
2532 if (chan->signal == id) in pl08x_find_chan_id()
2533 return &chan->vc.chan; in pl08x_find_chan_id()
2542 struct pl08x_driver_data *pl08x = ofdma->of_dma_data; in pl08x_of_xlate()
2549 if (dma_spec->args_count != 2) { in pl08x_of_xlate()
2550 dev_err(&pl08x->adev->dev, in pl08x_of_xlate()
2555 dma_chan = pl08x_find_chan_id(pl08x, dma_spec->args[0]); in pl08x_of_xlate()
2557 dev_err(&pl08x->adev->dev, in pl08x_of_xlate()
2563 dev_dbg(&pl08x->adev->dev, in pl08x_of_xlate()
2565 dma_spec->args[0]); in pl08x_of_xlate()
2568 plchan->cd->periph_buses = dma_spec->args[1]; in pl08x_of_xlate()
2574 struct device_node *np) in pl08x_of_probe() argument
2582 pd = devm_kzalloc(&adev->dev, sizeof(*pd), GFP_KERNEL); in pl08x_of_probe()
2584 return -ENOMEM; in pl08x_of_probe()
2587 if (of_property_read_bool(np, "lli-bus-interface-ahb1")) in pl08x_of_probe()
2588 pd->lli_buses |= PL08X_AHB1; in pl08x_of_probe()
2589 if (of_property_read_bool(np, "lli-bus-interface-ahb2")) in pl08x_of_probe()
2590 pd->lli_buses |= PL08X_AHB2; in pl08x_of_probe()
2591 if (!pd->lli_buses) { in pl08x_of_probe()
2592 dev_info(&adev->dev, "no bus masters for LLIs stated, assume all\n"); in pl08x_of_probe()
2593 pd->lli_buses |= PL08X_AHB1 | PL08X_AHB2; in pl08x_of_probe()
2597 if (of_property_read_bool(np, "mem-bus-interface-ahb1")) in pl08x_of_probe()
2598 pd->mem_buses |= PL08X_AHB1; in pl08x_of_probe()
2599 if (of_property_read_bool(np, "mem-bus-interface-ahb2")) in pl08x_of_probe()
2600 pd->mem_buses |= PL08X_AHB2; in pl08x_of_probe()
2601 if (!pd->mem_buses) { in pl08x_of_probe()
2602 dev_info(&adev->dev, "no bus masters for memory stated, assume all\n"); in pl08x_of_probe()
2603 pd->mem_buses |= PL08X_AHB1 | PL08X_AHB2; in pl08x_of_probe()
2607 ret = of_property_read_u32(np, "memcpy-burst-size", &val); in pl08x_of_probe()
2609 dev_info(&adev->dev, "no memcpy burst size specified, using 1 byte\n"); in pl08x_of_probe()
2614 dev_err(&adev->dev, "illegal burst size for memcpy, set to 1\n"); in pl08x_of_probe()
2617 pd->memcpy_burst_size = PL08X_BURST_SZ_1; in pl08x_of_probe()
2620 pd->memcpy_burst_size = PL08X_BURST_SZ_4; in pl08x_of_probe()
2623 pd->memcpy_burst_size = PL08X_BURST_SZ_8; in pl08x_of_probe()
2626 pd->memcpy_burst_size = PL08X_BURST_SZ_16; in pl08x_of_probe()
2629 pd->memcpy_burst_size = PL08X_BURST_SZ_32; in pl08x_of_probe()
2632 pd->memcpy_burst_size = PL08X_BURST_SZ_64; in pl08x_of_probe()
2635 pd->memcpy_burst_size = PL08X_BURST_SZ_128; in pl08x_of_probe()
2638 pd->memcpy_burst_size = PL08X_BURST_SZ_256; in pl08x_of_probe()
2642 ret = of_property_read_u32(np, "memcpy-bus-width", &val); in pl08x_of_probe()
2644 dev_info(&adev->dev, "no memcpy bus width specified, using 8 bits\n"); in pl08x_of_probe()
2649 dev_err(&adev->dev, "illegal bus width for memcpy, set to 8 bits\n"); in pl08x_of_probe()
2652 pd->memcpy_bus_width = PL08X_BUS_WIDTH_8_BITS; in pl08x_of_probe()
2655 pd->memcpy_bus_width = PL08X_BUS_WIDTH_16_BITS; in pl08x_of_probe()
2658 pd->memcpy_bus_width = PL08X_BUS_WIDTH_32_BITS; in pl08x_of_probe()
2668 if (pl08x->vd->signals) { in pl08x_of_probe()
2669 chanp = devm_kcalloc(&adev->dev, in pl08x_of_probe()
2670 pl08x->vd->signals, in pl08x_of_probe()
2674 return -ENOMEM; in pl08x_of_probe()
2676 pd->slave_channels = chanp; in pl08x_of_probe()
2677 for (i = 0; i < pl08x->vd->signals; i++) { in pl08x_of_probe()
2679 * chanp->periph_buses will be assigned at translation in pl08x_of_probe()
2681 chanp->bus_id = kasprintf(GFP_KERNEL, "slave%d", i); in pl08x_of_probe()
2684 pd->num_slave_channels = pl08x->vd->signals; in pl08x_of_probe()
2687 pl08x->pd = pd; in pl08x_of_probe()
2689 return of_dma_controller_register(adev->dev.of_node, pl08x_of_xlate, in pl08x_of_probe()
2695 struct device_node *np) in pl08x_of_probe() argument
2697 return -EINVAL; in pl08x_of_probe()
2704 struct vendor_data *vd = id->data; in pl08x_probe()
2705 struct device_node *np = adev->dev.of_node; in pl08x_probe() local
2715 ret = dma_set_mask_and_coherent(&adev->dev, DMA_BIT_MASK(32)); in pl08x_probe()
2722 ret = -ENOMEM; in pl08x_probe()
2727 pl08x->adev = adev; in pl08x_probe()
2728 pl08x->vd = vd; in pl08x_probe()
2730 pl08x->base = ioremap(adev->res.start, resource_size(&adev->res)); in pl08x_probe()
2731 if (!pl08x->base) { in pl08x_probe()
2732 ret = -ENOMEM; in pl08x_probe()
2736 if (vd->ftdmac020) { in pl08x_probe()
2739 val = readl(pl08x->base + FTDMAC020_REVISION); in pl08x_probe()
2740 dev_info(&pl08x->adev->dev, "FTDMAC020 %d.%d rel %d\n", in pl08x_probe()
2742 val = readl(pl08x->base + FTDMAC020_FEATURE); in pl08x_probe()
2743 dev_info(&pl08x->adev->dev, "FTDMAC020 %d channels, " in pl08x_probe()
2744 "%s built-in bridge, %s, %s linked lists\n", in pl08x_probe()
2752 dev_warn(&pl08x->adev->dev, in pl08x_probe()
2754 vd->channels = (val >> 12) & 0x0f; in pl08x_probe()
2755 vd->dualmaster = !!(val & BIT(9)); in pl08x_probe()
2759 dma_cap_set(DMA_MEMCPY, pl08x->memcpy.cap_mask); in pl08x_probe()
2760 pl08x->memcpy.dev = &adev->dev; in pl08x_probe()
2761 pl08x->memcpy.device_free_chan_resources = pl08x_free_chan_resources; in pl08x_probe()
2762 pl08x->memcpy.device_prep_dma_memcpy = pl08x_prep_dma_memcpy; in pl08x_probe()
2763 pl08x->memcpy.device_prep_dma_interrupt = pl08x_prep_dma_interrupt; in pl08x_probe()
2764 pl08x->memcpy.device_tx_status = pl08x_dma_tx_status; in pl08x_probe()
2765 pl08x->memcpy.device_issue_pending = pl08x_issue_pending; in pl08x_probe()
2766 pl08x->memcpy.device_config = pl08x_config; in pl08x_probe()
2767 pl08x->memcpy.device_pause = pl08x_pause; in pl08x_probe()
2768 pl08x->memcpy.device_resume = pl08x_resume; in pl08x_probe()
2769 pl08x->memcpy.device_terminate_all = pl08x_terminate_all; in pl08x_probe()
2770 pl08x->memcpy.device_synchronize = pl08x_synchronize; in pl08x_probe()
2771 pl08x->memcpy.src_addr_widths = PL80X_DMA_BUSWIDTHS; in pl08x_probe()
2772 pl08x->memcpy.dst_addr_widths = PL80X_DMA_BUSWIDTHS; in pl08x_probe()
2773 pl08x->memcpy.directions = BIT(DMA_MEM_TO_MEM); in pl08x_probe()
2774 pl08x->memcpy.residue_granularity = DMA_RESIDUE_GRANULARITY_SEGMENT; in pl08x_probe()
2775 if (vd->ftdmac020) in pl08x_probe()
2776 pl08x->memcpy.copy_align = DMAENGINE_ALIGN_4_BYTES; in pl08x_probe()
2783 if (vd->signals) { in pl08x_probe()
2784 pl08x->has_slave = true; in pl08x_probe()
2785 dma_cap_set(DMA_SLAVE, pl08x->slave.cap_mask); in pl08x_probe()
2786 dma_cap_set(DMA_CYCLIC, pl08x->slave.cap_mask); in pl08x_probe()
2787 pl08x->slave.dev = &adev->dev; in pl08x_probe()
2788 pl08x->slave.device_free_chan_resources = in pl08x_probe()
2790 pl08x->slave.device_prep_dma_interrupt = in pl08x_probe()
2792 pl08x->slave.device_tx_status = pl08x_dma_tx_status; in pl08x_probe()
2793 pl08x->slave.device_issue_pending = pl08x_issue_pending; in pl08x_probe()
2794 pl08x->slave.device_prep_slave_sg = pl08x_prep_slave_sg; in pl08x_probe()
2795 pl08x->slave.device_prep_dma_cyclic = pl08x_prep_dma_cyclic; in pl08x_probe()
2796 pl08x->slave.device_config = pl08x_config; in pl08x_probe()
2797 pl08x->slave.device_pause = pl08x_pause; in pl08x_probe()
2798 pl08x->slave.device_resume = pl08x_resume; in pl08x_probe()
2799 pl08x->slave.device_terminate_all = pl08x_terminate_all; in pl08x_probe()
2800 pl08x->slave.device_synchronize = pl08x_synchronize; in pl08x_probe()
2801 pl08x->slave.src_addr_widths = PL80X_DMA_BUSWIDTHS; in pl08x_probe()
2802 pl08x->slave.dst_addr_widths = PL80X_DMA_BUSWIDTHS; in pl08x_probe()
2803 pl08x->slave.directions = in pl08x_probe()
2805 pl08x->slave.residue_granularity = in pl08x_probe()
2810 pl08x->pd = dev_get_platdata(&adev->dev); in pl08x_probe()
2811 if (!pl08x->pd) { in pl08x_probe()
2812 if (np) { in pl08x_probe()
2813 ret = pl08x_of_probe(adev, pl08x, np); in pl08x_probe()
2817 dev_err(&adev->dev, "no platform data supplied\n"); in pl08x_probe()
2818 ret = -EINVAL; in pl08x_probe()
2822 pl08x->slave.filter.map = pl08x->pd->slave_map; in pl08x_probe()
2823 pl08x->slave.filter.mapcnt = pl08x->pd->slave_map_len; in pl08x_probe()
2824 pl08x->slave.filter.fn = pl08x_filter_fn; in pl08x_probe()
2828 pl08x->lli_buses = PL08X_AHB1; in pl08x_probe()
2829 pl08x->mem_buses = PL08X_AHB1; in pl08x_probe()
2830 if (pl08x->vd->dualmaster) { in pl08x_probe()
2831 pl08x->lli_buses = pl08x->pd->lli_buses; in pl08x_probe()
2832 pl08x->mem_buses = pl08x->pd->mem_buses; in pl08x_probe()
2835 if (vd->pl080s) in pl08x_probe()
2836 pl08x->lli_words = PL080S_LLI_WORDS; in pl08x_probe()
2838 pl08x->lli_words = PL080_LLI_WORDS; in pl08x_probe()
2839 tsfr_size = MAX_NUM_TSFR_LLIS * pl08x->lli_words * sizeof(u32); in pl08x_probe()
2841 /* A DMA memory pool for LLIs, align on 1-byte boundary */ in pl08x_probe()
2842 pl08x->pool = dma_pool_create(DRIVER_NAME, &pl08x->adev->dev, in pl08x_probe()
2844 if (!pl08x->pool) { in pl08x_probe()
2845 ret = -ENOMEM; in pl08x_probe()
2853 if (vd->ftdmac020) in pl08x_probe()
2854 /* This variant has error IRQs in bits 16-19 */ in pl08x_probe()
2855 writel(0x0000FFFF, pl08x->base + PL080_ERR_CLEAR); in pl08x_probe()
2857 writel(0x000000FF, pl08x->base + PL080_ERR_CLEAR); in pl08x_probe()
2858 writel(0x000000FF, pl08x->base + PL080_TC_CLEAR); in pl08x_probe()
2861 ret = request_irq(adev->irq[0], pl08x_irq, 0, DRIVER_NAME, pl08x); in pl08x_probe()
2863 dev_err(&adev->dev, "%s failed to request interrupt %d\n", in pl08x_probe()
2864 __func__, adev->irq[0]); in pl08x_probe()
2869 pl08x->phy_chans = kzalloc((vd->channels * sizeof(*pl08x->phy_chans)), in pl08x_probe()
2871 if (!pl08x->phy_chans) { in pl08x_probe()
2872 ret = -ENOMEM; in pl08x_probe()
2876 for (i = 0; i < vd->channels; i++) { in pl08x_probe()
2877 struct pl08x_phy_chan *ch = &pl08x->phy_chans[i]; in pl08x_probe()
2879 ch->id = i; in pl08x_probe()
2880 ch->base = pl08x->base + PL080_Cx_BASE(i); in pl08x_probe()
2881 if (vd->ftdmac020) { in pl08x_probe()
2883 ch->reg_busy = ch->base + FTDMAC020_CH_BUSY; in pl08x_probe()
2884 ch->reg_config = ch->base + FTDMAC020_CH_CFG; in pl08x_probe()
2885 ch->reg_control = ch->base + FTDMAC020_CH_CSR; in pl08x_probe()
2886 ch->reg_src = ch->base + FTDMAC020_CH_SRC_ADDR; in pl08x_probe()
2887 ch->reg_dst = ch->base + FTDMAC020_CH_DST_ADDR; in pl08x_probe()
2888 ch->reg_lli = ch->base + FTDMAC020_CH_LLP; in pl08x_probe()
2889 ch->ftdmac020 = true; in pl08x_probe()
2891 ch->reg_config = ch->base + vd->config_offset; in pl08x_probe()
2892 ch->reg_control = ch->base + PL080_CH_CONTROL; in pl08x_probe()
2893 ch->reg_src = ch->base + PL080_CH_SRC_ADDR; in pl08x_probe()
2894 ch->reg_dst = ch->base + PL080_CH_DST_ADDR; in pl08x_probe()
2895 ch->reg_lli = ch->base + PL080_CH_LLI; in pl08x_probe()
2897 if (vd->pl080s) in pl08x_probe()
2898 ch->pl080s = true; in pl08x_probe()
2900 spin_lock_init(&ch->lock); in pl08x_probe()
2907 if (vd->nomadik) { in pl08x_probe()
2910 val = readl(ch->reg_config); in pl08x_probe()
2912 dev_info(&adev->dev, "physical channel %d reserved for secure access only\n", i); in pl08x_probe()
2913 ch->locked = true; in pl08x_probe()
2917 dev_dbg(&adev->dev, "physical channel %d is %s\n", in pl08x_probe()
2922 ret = pl08x_dma_init_virtual_channels(pl08x, &pl08x->memcpy, in pl08x_probe()
2923 pl08x->vd->channels, false); in pl08x_probe()
2925 dev_warn(&pl08x->adev->dev, in pl08x_probe()
2926 "%s failed to enumerate memcpy channels - %d\n", in pl08x_probe()
2932 if (pl08x->has_slave) { in pl08x_probe()
2933 ret = pl08x_dma_init_virtual_channels(pl08x, &pl08x->slave, in pl08x_probe()
2934 pl08x->pd->num_slave_channels, true); in pl08x_probe()
2936 dev_warn(&pl08x->adev->dev, in pl08x_probe()
2937 "%s failed to enumerate slave channels - %d\n", in pl08x_probe()
2943 ret = dma_async_device_register(&pl08x->memcpy); in pl08x_probe()
2945 dev_warn(&pl08x->adev->dev, in pl08x_probe()
2946 "%s failed to register memcpy as an async device - %d\n", in pl08x_probe()
2951 if (pl08x->has_slave) { in pl08x_probe()
2952 ret = dma_async_device_register(&pl08x->slave); in pl08x_probe()
2954 dev_warn(&pl08x->adev->dev, in pl08x_probe()
2955 "%s failed to register slave as an async device - %d\n", in pl08x_probe()
2963 dev_info(&pl08x->adev->dev, "DMA: PL%03x%s rev%u at 0x%08llx irq %d\n", in pl08x_probe()
2964 amba_part(adev), pl08x->vd->pl080s ? "s" : "", amba_rev(adev), in pl08x_probe()
2965 (unsigned long long)adev->res.start, adev->irq[0]); in pl08x_probe()
2970 dma_async_device_unregister(&pl08x->memcpy); in pl08x_probe()
2972 if (pl08x->has_slave) in pl08x_probe()
2973 pl08x_free_virtual_channels(&pl08x->slave); in pl08x_probe()
2975 pl08x_free_virtual_channels(&pl08x->memcpy); in pl08x_probe()
2977 kfree(pl08x->phy_chans); in pl08x_probe()
2979 free_irq(adev->irq[0], pl08x); in pl08x_probe()
2981 dma_pool_destroy(pl08x->pool); in pl08x_probe()
2984 iounmap(pl08x->base); in pl08x_probe()