253 		struct omap_dmadev *od = to_omap_dma_dev(vd->tx.chan->device);  in omap_dma_desc_free()  local
258 				dma_pool_free(od->desc_pool, d->sg[i].t2_desc,  in omap_dma_desc_free()
351 static void omap_dma_glbl_write(struct omap_dmadev *od, unsigned reg, unsigned val)  in omap_dma_glbl_write()  argument
353 	const struct omap_dma_reg *r = od->reg_map + reg;  in omap_dma_glbl_write()
357 	omap_dma_write(val, r->type, od->base + r->offset);  in omap_dma_glbl_write()
360 static unsigned omap_dma_glbl_read(struct omap_dmadev *od, unsigned reg)  in omap_dma_glbl_read()  argument
362 	const struct omap_dma_reg *r = od->reg_map + reg;  in omap_dma_glbl_read()
366 	return omap_dma_read(r->type, od->base + r->offset);  in omap_dma_glbl_read()
401 static void omap_dma_clear_lch(struct omap_dmadev *od, int lch)  in omap_dma_clear_lch()  argument
406 	c = od->lch_map[lch];  in omap_dma_clear_lch()
410 	for (i = CSDP; i <= od->cfg->lch_end; i++)  in omap_dma_clear_lch()
414 static void omap_dma_assign(struct omap_dmadev *od, struct omap_chan *c,  in omap_dma_assign()  argument
417 	c->channel_base = od->base + od->plat->channel_stride * lch;  in omap_dma_assign()
419 	od->lch_map[lch] = c;  in omap_dma_assign()
424 	struct omap_dmadev *od = to_omap_dma_dev(c->vc.chan.device);  in omap_dma_start()  local
427 	if (__dma_omap15xx(od->plat->dma_attr))  in omap_dma_start()
449 	} else if (od->ll123_supported) {  in omap_dma_start()
487 	struct omap_dmadev *od = to_omap_dma_dev(c->vc.chan.device);  in omap_dma_stop()  local
496 	if (od->plat->errata & DMA_ERRATA_i541 && val & CCR_TRIGGER_SRC) {  in omap_dma_stop()
499 		sysconfig = omap_dma_glbl_read(od, OCP_SYSCONFIG);  in omap_dma_stop()
502 		omap_dma_glbl_write(od, OCP_SYSCONFIG, val);  in omap_dma_stop()
511 		omap_dma_glbl_write(od, OCP_SYSCONFIG, sysconfig);  in omap_dma_stop()
525 	if (!__dma_omap15xx(od->plat->dma_attr) && c->cyclic) {  in omap_dma_stop()
633 	struct omap_dmadev *od = devid;  in omap_dma_irq()  local
636 	spin_lock(&od->irq_lock);  in omap_dma_irq()
638 	status = omap_dma_glbl_read(od, IRQSTATUS_L1);  in omap_dma_irq()
639 	status &= od->irq_enable_mask;  in omap_dma_irq()
641 		spin_unlock(&od->irq_lock);  in omap_dma_irq()
653 		c = od->lch_map[channel];  in omap_dma_irq()
656 			dev_err(od->ddev.dev, "invalid channel %u\n", channel);  in omap_dma_irq()
661 		omap_dma_glbl_write(od, IRQSTATUS_L1, mask);  in omap_dma_irq()
666 	spin_unlock(&od->irq_lock);  in omap_dma_irq()
671 static int omap_dma_get_lch(struct omap_dmadev *od, int *lch)  in omap_dma_get_lch()  argument
675 	mutex_lock(&od->lch_lock);  in omap_dma_get_lch()
676 	channel = find_first_zero_bit(od->lch_bitmap, od->lch_count);  in omap_dma_get_lch()
677 	if (channel >= od->lch_count)  in omap_dma_get_lch()
679 	set_bit(channel, od->lch_bitmap);  in omap_dma_get_lch()
680 	mutex_unlock(&od->lch_lock);  in omap_dma_get_lch()
682 	omap_dma_clear_lch(od, channel);  in omap_dma_get_lch()
688 	mutex_unlock(&od->lch_lock);  in omap_dma_get_lch()
694 static void omap_dma_put_lch(struct omap_dmadev *od, int lch)  in omap_dma_put_lch()  argument
696 	omap_dma_clear_lch(od, lch);  in omap_dma_put_lch()
697 	mutex_lock(&od->lch_lock);  in omap_dma_put_lch()
698 	clear_bit(lch, od->lch_bitmap);  in omap_dma_put_lch()
699 	mutex_unlock(&od->lch_lock);  in omap_dma_put_lch()
704 	struct omap_dmadev *od = to_omap_dma_dev(chan->device);  in omap_dma_alloc_chan_resources()  local
706 	struct device *dev = od->ddev.dev;  in omap_dma_alloc_chan_resources()
709 	if (od->legacy) {  in omap_dma_alloc_chan_resources()
713 		ret = omap_dma_get_lch(od, &c->dma_ch);  in omap_dma_alloc_chan_resources()
719 		omap_dma_assign(od, c, c->dma_ch);  in omap_dma_alloc_chan_resources()
721 		if (!od->legacy) {  in omap_dma_alloc_chan_resources()
724 			spin_lock_irq(&od->irq_lock);  in omap_dma_alloc_chan_resources()
726 			omap_dma_glbl_write(od, IRQSTATUS_L1, val);  in omap_dma_alloc_chan_resources()
727 			od->irq_enable_mask |= val;  in omap_dma_alloc_chan_resources()
728 			omap_dma_glbl_write(od, IRQENABLE_L1, od->irq_enable_mask);  in omap_dma_alloc_chan_resources()
730 			val = omap_dma_glbl_read(od, IRQENABLE_L0);  in omap_dma_alloc_chan_resources()
732 			omap_dma_glbl_write(od, IRQENABLE_L0, val);  in omap_dma_alloc_chan_resources()
733 			spin_unlock_irq(&od->irq_lock);  in omap_dma_alloc_chan_resources()
738 		if (__dma_omap16xx(od->plat->dma_attr)) {  in omap_dma_alloc_chan_resources()
749 	if (od->plat->errata & DMA_ERRATA_IFRAME_BUFFERING)  in omap_dma_alloc_chan_resources()
757 	struct omap_dmadev *od = to_omap_dma_dev(chan->device);  in omap_dma_free_chan_resources()  local
760 	if (!od->legacy) {  in omap_dma_free_chan_resources()
761 		spin_lock_irq(&od->irq_lock);  in omap_dma_free_chan_resources()
762 		od->irq_enable_mask &= ~BIT(c->dma_ch);  in omap_dma_free_chan_resources()
763 		omap_dma_glbl_write(od, IRQENABLE_L1, od->irq_enable_mask);  in omap_dma_free_chan_resources()
764 		spin_unlock_irq(&od->irq_lock);  in omap_dma_free_chan_resources()
768 	od->lch_map[c->dma_ch] = NULL;  in omap_dma_free_chan_resources()
771 	if (od->legacy)  in omap_dma_free_chan_resources()
774 		omap_dma_put_lch(od, c->dma_ch);  in omap_dma_free_chan_resources()
776 	dev_dbg(od->ddev.dev, "freeing channel %u used for %u\n", c->dma_ch,  in omap_dma_free_chan_resources()
820 	struct omap_dmadev *od = to_omap_dma_dev(c->vc.chan.device);  in omap_dma_chan_read_3_3()  local
824 	if (val == 0 && od->plat->errata & DMA_ERRATA_3_3)  in omap_dma_chan_read_3_3()
832 	struct omap_dmadev *od = to_omap_dma_dev(c->vc.chan.device);  in omap_dma_get_src_pos()  local
835 	if (__dma_omap15xx(od->plat->dma_attr)) {  in omap_dma_get_src_pos()
858 	struct omap_dmadev *od = to_omap_dma_dev(c->vc.chan.device);  in omap_dma_get_dst_pos()  local
861 	if (__dma_omap15xx(od->plat->dma_attr)) {  in omap_dma_get_dst_pos()
958 	struct omap_dmadev *od = to_omap_dma_dev(chan->device);  in omap_dma_prep_slave_sg()  local
1078 	if (od->plat->errata & DMA_ERRATA_PARALLEL_CHANNELS)  in omap_dma_prep_slave_sg()
1094 		d->using_ll = od->ll123_supported;  in omap_dma_prep_slave_sg()
1104 			osg->t2_desc = dma_pool_alloc(od->desc_pool, GFP_ATOMIC,  in omap_dma_prep_slave_sg()
1126 				dma_pool_free(od->desc_pool, osg->t2_desc,  in omap_dma_prep_slave_sg()
1140 	struct omap_dmadev *od = to_omap_dma_dev(chan->device);  in omap_dma_prep_dma_cyclic()  local
1227 	if (__dma_omap15xx(od->plat->dma_attr))  in omap_dma_prep_dma_cyclic()
1424 	struct omap_dmadev *od = to_omap_dma_dev(chan->device);  in omap_dma_pause()  local
1429 	spin_lock_irqsave(&od->irq_lock, flags);  in omap_dma_pause()
1469 	spin_unlock_irqrestore(&od->irq_lock, flags);  in omap_dma_pause()
1477 	struct omap_dmadev *od = to_omap_dma_dev(chan->device);  in omap_dma_resume()  local
1481 	spin_lock_irqsave(&od->irq_lock, flags);  in omap_dma_resume()
1493 	spin_unlock_irqrestore(&od->irq_lock, flags);  in omap_dma_resume()
1498 static int omap_dma_chan_init(struct omap_dmadev *od)  in omap_dma_chan_init()  argument
1506 	c->reg_map = od->reg_map;  in omap_dma_chan_init()
1508 	vchan_init(&c->vc, &od->ddev);  in omap_dma_chan_init()
1513 static void omap_dma_free(struct omap_dmadev *od)  in omap_dma_free()  argument
1515 	while (!list_empty(&od->ddev.channels)) {  in omap_dma_free()
1516 		struct omap_chan *c = list_first_entry(&od->ddev.channels,  in omap_dma_free()
1526 static bool omap_dma_busy(struct omap_dmadev *od)  in omap_dma_busy()  argument
1532 		lch = find_next_bit(od->lch_bitmap, od->lch_count, lch + 1);  in omap_dma_busy()
1533 		if (lch >= od->lch_count)  in omap_dma_busy()
1535 		c = od->lch_map[lch];  in omap_dma_busy()
1549 	struct omap_dmadev *od;  in omap_dma_busy_notifier()  local
1551 	od = container_of(nb, struct omap_dmadev, nb);  in omap_dma_busy_notifier()
1555 		if (omap_dma_busy(od))  in omap_dma_busy_notifier()
1571 static void omap_dma_context_save(struct omap_dmadev *od)  in omap_dma_context_save()  argument
1573 	od->context.irqenable_l0 = omap_dma_glbl_read(od, IRQENABLE_L0);  in omap_dma_context_save()
1574 	od->context.irqenable_l1 = omap_dma_glbl_read(od, IRQENABLE_L1);  in omap_dma_context_save()
1575 	od->context.ocp_sysconfig = omap_dma_glbl_read(od, OCP_SYSCONFIG);  in omap_dma_context_save()
1576 	od->context.gcr = omap_dma_glbl_read(od, GCR);  in omap_dma_context_save()
1579 static void omap_dma_context_restore(struct omap_dmadev *od)  in omap_dma_context_restore()  argument
1583 	omap_dma_glbl_write(od, GCR, od->context.gcr);  in omap_dma_context_restore()
1584 	omap_dma_glbl_write(od, OCP_SYSCONFIG, od->context.ocp_sysconfig);  in omap_dma_context_restore()
1585 	omap_dma_glbl_write(od, IRQENABLE_L0, od->context.irqenable_l0);  in omap_dma_context_restore()
1586 	omap_dma_glbl_write(od, IRQENABLE_L1, od->context.irqenable_l1);  in omap_dma_context_restore()
1589 	if (od->plat->errata & DMA_ROMCODE_BUG)  in omap_dma_context_restore()
1590 		omap_dma_glbl_write(od, IRQSTATUS_L0, 0);  in omap_dma_context_restore()
1593 	for (i = 0; i < od->lch_count; i++)  in omap_dma_context_restore()
1594 		omap_dma_clear_lch(od, i);  in omap_dma_context_restore()
1601 	struct omap_dmadev *od;  in omap_dma_context_notifier()  local
1603 	od = container_of(nb, struct omap_dmadev, nb);  in omap_dma_context_notifier()
1607 		if (omap_dma_busy(od))  in omap_dma_context_notifier()
1609 		omap_dma_context_save(od);  in omap_dma_context_notifier()
1613 		omap_dma_context_restore(od);  in omap_dma_context_notifier()
1620 static void omap_dma_init_gcr(struct omap_dmadev *od, int arb_rate,  in omap_dma_init_gcr()  argument
1626 	if (!od->cfg->rw_priority)  in omap_dma_init_gcr()
1638 	omap_dma_glbl_write(od, GCR, val);  in omap_dma_init_gcr()
1654 	struct omap_dmadev *od;  in omap_dma_probe()  local
1659 	od = devm_kzalloc(&pdev->dev, sizeof(*od), GFP_KERNEL);  in omap_dma_probe()
1660 	if (!od)  in omap_dma_probe()
1664 	od->base = devm_ioremap_resource(&pdev->dev, res);  in omap_dma_probe()
1665 	if (IS_ERR(od->base))  in omap_dma_probe()
1666 		return PTR_ERR(od->base);  in omap_dma_probe()
1670 		od->cfg = conf;  in omap_dma_probe()
1671 		od->plat = dev_get_platdata(&pdev->dev);  in omap_dma_probe()
1672 		if (!od->plat) {  in omap_dma_probe()
1677 		od->cfg = &default_cfg;  in omap_dma_probe()
1679 		od->plat = omap_get_plat_info();  in omap_dma_probe()
1680 		if (!od->plat)  in omap_dma_probe()
1684 	od->reg_map = od->plat->reg_map;  in omap_dma_probe()
1686 	dma_cap_set(DMA_SLAVE, od->ddev.cap_mask);  in omap_dma_probe()
1687 	dma_cap_set(DMA_CYCLIC, od->ddev.cap_mask);  in omap_dma_probe()
1688 	dma_cap_set(DMA_MEMCPY, od->ddev.cap_mask);  in omap_dma_probe()
1689 	dma_cap_set(DMA_INTERLEAVE, od->ddev.cap_mask);  in omap_dma_probe()
1690 	od->ddev.device_alloc_chan_resources = omap_dma_alloc_chan_resources;  in omap_dma_probe()
1691 	od->ddev.device_free_chan_resources = omap_dma_free_chan_resources;  in omap_dma_probe()
1692 	od->ddev.device_tx_status = omap_dma_tx_status;  in omap_dma_probe()
1693 	od->ddev.device_issue_pending = omap_dma_issue_pending;  in omap_dma_probe()
1694 	od->ddev.device_prep_slave_sg = omap_dma_prep_slave_sg;  in omap_dma_probe()
1695 	od->ddev.device_prep_dma_cyclic = omap_dma_prep_dma_cyclic;  in omap_dma_probe()
1696 	od->ddev.device_prep_dma_memcpy = omap_dma_prep_dma_memcpy;  in omap_dma_probe()
1697 	od->ddev.device_prep_interleaved_dma = omap_dma_prep_dma_interleaved;  in omap_dma_probe()
1698 	od->ddev.device_config = omap_dma_slave_config;  in omap_dma_probe()
1699 	od->ddev.device_pause = omap_dma_pause;  in omap_dma_probe()
1700 	od->ddev.device_resume = omap_dma_resume;  in omap_dma_probe()
1701 	od->ddev.device_terminate_all = omap_dma_terminate_all;  in omap_dma_probe()
1702 	od->ddev.device_synchronize = omap_dma_synchronize;  in omap_dma_probe()
1703 	od->ddev.src_addr_widths = OMAP_DMA_BUSWIDTHS;  in omap_dma_probe()
1704 	od->ddev.dst_addr_widths = OMAP_DMA_BUSWIDTHS;  in omap_dma_probe()
1705 	od->ddev.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV);  in omap_dma_probe()
1706 	if (__dma_omap15xx(od->plat->dma_attr))  in omap_dma_probe()
1707 		od->ddev.residue_granularity =  in omap_dma_probe()
1710 		od->ddev.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST;  in omap_dma_probe()
1711 	od->ddev.max_burst = SZ_16M - 1; /* CCEN: 24bit unsigned */  in omap_dma_probe()
1712 	od->ddev.dev = &pdev->dev;  in omap_dma_probe()
1713 	INIT_LIST_HEAD(&od->ddev.channels);  in omap_dma_probe()
1714 	mutex_init(&od->lch_lock);  in omap_dma_probe()
1715 	spin_lock_init(&od->lock);  in omap_dma_probe()
1716 	spin_lock_init(&od->irq_lock);  in omap_dma_probe()
1719 	od->dma_requests = OMAP_SDMA_REQUESTS;  in omap_dma_probe()
1722 						      &od->dma_requests)) {  in omap_dma_probe()
1730 		od->lch_count = od->plat->dma_attr->lch_count;  in omap_dma_probe()
1731 		if (unlikely(!od->lch_count))  in omap_dma_probe()
1732 			od->lch_count = OMAP_SDMA_CHANNELS;  in omap_dma_probe()
1734 					&od->lch_count)) {  in omap_dma_probe()
1738 		od->lch_count = OMAP_SDMA_CHANNELS;  in omap_dma_probe()
1747 		bitmap_from_arr32(od->lch_bitmap, &val, od->lch_count);  in omap_dma_probe()
1749 	if (od->plat->dma_attr->dev_caps & HS_CHANNELS_RESERVED)  in omap_dma_probe()
1750 		bitmap_set(od->lch_bitmap, 0, 2);  in omap_dma_probe()
1752 	od->lch_map = devm_kcalloc(&pdev->dev, od->lch_count,  in omap_dma_probe()
1753 				   sizeof(*od->lch_map),  in omap_dma_probe()
1755 	if (!od->lch_map)  in omap_dma_probe()
1758 	for (i = 0; i < od->dma_requests; i++) {  in omap_dma_probe()
1759 		rc = omap_dma_chan_init(od);  in omap_dma_probe()
1761 			omap_dma_free(od);  in omap_dma_probe()
1769 		od->legacy = true;  in omap_dma_probe()
1772 		od->irq_enable_mask = 0;  in omap_dma_probe()
1773 		omap_dma_glbl_write(od, IRQENABLE_L1, 0);  in omap_dma_probe()
1776 				      IRQF_SHARED, "omap-dma-engine", od);  in omap_dma_probe()
1778 			omap_dma_free(od);  in omap_dma_probe()
1783 	if (omap_dma_glbl_read(od, CAPS_0) & CAPS_0_SUPPORT_LL123)  in omap_dma_probe()
1784 		od->ll123_supported = true;  in omap_dma_probe()
1786 	od->ddev.filter.map = od->plat->slave_map;  in omap_dma_probe()
1787 	od->ddev.filter.mapcnt = od->plat->slavecnt;  in omap_dma_probe()
1788 	od->ddev.filter.fn = omap_dma_filter_fn;  in omap_dma_probe()
1790 	if (od->ll123_supported) {  in omap_dma_probe()
1791 		od->desc_pool = dma_pool_create(dev_name(&pdev->dev),  in omap_dma_probe()
1795 		if (!od->desc_pool) {  in omap_dma_probe()
1798 			od->ll123_supported = false;  in omap_dma_probe()
1802 	rc = dma_async_device_register(&od->ddev);  in omap_dma_probe()
1806 		omap_dma_free(od);  in omap_dma_probe()
1810 	platform_set_drvdata(pdev, od);  in omap_dma_probe()
1813 		omap_dma_info.dma_cap = od->ddev.cap_mask;  in omap_dma_probe()
1820 			dma_async_device_unregister(&od->ddev);  in omap_dma_probe()
1821 			omap_dma_free(od);  in omap_dma_probe()
1825 	omap_dma_init_gcr(od, DMA_DEFAULT_ARB_RATE, DMA_DEFAULT_FIFO_DEPTH, 0);  in omap_dma_probe()
1827 	if (od->cfg->needs_busy_check) {  in omap_dma_probe()
1828 		od->nb.notifier_call = omap_dma_busy_notifier;  in omap_dma_probe()
1829 		cpu_pm_register_notifier(&od->nb);  in omap_dma_probe()
1830 	} else if (od->cfg->may_lose_context) {  in omap_dma_probe()
1831 		od->nb.notifier_call = omap_dma_context_notifier;  in omap_dma_probe()
1832 		cpu_pm_register_notifier(&od->nb);  in omap_dma_probe()
1836 		 od->ll123_supported ? " (LinkedList1/2/3 supported)" : "");  in omap_dma_probe()
1843 	struct omap_dmadev *od = platform_get_drvdata(pdev);  in omap_dma_remove()  local
1846 	if (od->cfg->may_lose_context)  in omap_dma_remove()
1847 		cpu_pm_unregister_notifier(&od->nb);  in omap_dma_remove()
1853 	devm_free_irq(&pdev->dev, irq, od);  in omap_dma_remove()
1855 	dma_async_device_unregister(&od->ddev);  in omap_dma_remove()
1857 	if (!od->legacy) {  in omap_dma_remove()
1859 		omap_dma_glbl_write(od, IRQENABLE_L0, 0);  in omap_dma_remove()
1862 	if (od->ll123_supported)  in omap_dma_remove()
1863 		dma_pool_destroy(od->desc_pool);  in omap_dma_remove()
1865 	omap_dma_free(od);  in omap_dma_remove()
1925 		struct omap_dmadev *od = to_omap_dma_dev(chan->device);  in omap_dma_filter_fn()  local
1929 		if (req <= od->dma_requests) {  in omap_dma_filter_fn()