Lines Matching +full:free +full:- +full:running
4 * Copyright (c) 2015-2017, The Linux Foundation. All rights reserved.
6 * This program is free software; you can redistribute it and/or modify
8 * only version 2 as published by the Free Software Foundation.
24 * written by Hongjun Chen <hong-jun.chen@freescale.com>.
29 * This program is free software; you can redistribute it and/or modify it
30 * under the terms of the GNU General Public License as published by the Free
46 #include <linux/dma-mapping.h>
101 INIT_LIST_HEAD(&dmadev->ddev.channels); in hidma_free()
116 struct dma_device *ddev = mchan->chan.device; in hidma_process_completed()
128 spin_lock_irqsave(&mchan->lock, irqflags); in hidma_process_completed()
129 list_splice_tail_init(&mchan->completed, &list); in hidma_process_completed()
130 spin_unlock_irqrestore(&mchan->lock, irqflags); in hidma_process_completed()
138 desc = &mdesc->desc; in hidma_process_completed()
139 last_cookie = desc->cookie; in hidma_process_completed()
141 llstat = hidma_ll_status(mdma->lldev, mdesc->tre_ch); in hidma_process_completed()
143 spin_lock_irqsave(&mchan->lock, irqflags); in hidma_process_completed()
145 mchan->last_success = last_cookie; in hidma_process_completed()
152 spin_unlock_irqrestore(&mchan->lock, irqflags); in hidma_process_completed()
158 spin_lock_irqsave(&mchan->lock, irqflags); in hidma_process_completed()
159 list_move(&mdesc->node, &mchan->free); in hidma_process_completed()
160 spin_unlock_irqrestore(&mchan->lock, irqflags); in hidma_process_completed()
174 struct hidma_chan *mchan = to_hidma_chan(mdesc->desc.chan); in hidma_callback()
175 struct dma_device *ddev = mchan->chan.device; in hidma_callback()
180 spin_lock_irqsave(&mchan->lock, irqflags); in hidma_callback()
181 if (mdesc->node.next) { in hidma_callback()
183 list_move_tail(&mdesc->node, &mchan->completed); in hidma_callback()
186 /* calculate the next running descriptor */ in hidma_callback()
187 mchan->running = list_first_entry(&mchan->active, in hidma_callback()
190 spin_unlock_irqrestore(&mchan->lock, irqflags); in hidma_callback()
195 pm_runtime_mark_last_busy(dmadev->ddev.dev); in hidma_callback()
196 pm_runtime_put_autosuspend(dmadev->ddev.dev); in hidma_callback()
205 mchan = devm_kzalloc(dmadev->ddev.dev, sizeof(*mchan), GFP_KERNEL); in hidma_chan_init()
207 return -ENOMEM; in hidma_chan_init()
209 ddev = &dmadev->ddev; in hidma_chan_init()
210 mchan->dma_sig = dma_sig; in hidma_chan_init()
211 mchan->dmadev = dmadev; in hidma_chan_init()
212 mchan->chan.device = ddev; in hidma_chan_init()
213 dma_cookie_init(&mchan->chan); in hidma_chan_init()
215 INIT_LIST_HEAD(&mchan->free); in hidma_chan_init()
216 INIT_LIST_HEAD(&mchan->prepared); in hidma_chan_init()
217 INIT_LIST_HEAD(&mchan->active); in hidma_chan_init()
218 INIT_LIST_HEAD(&mchan->completed); in hidma_chan_init()
219 INIT_LIST_HEAD(&mchan->queued); in hidma_chan_init()
221 spin_lock_init(&mchan->lock); in hidma_chan_init()
222 list_add_tail(&mchan->chan.device_node, &ddev->channels); in hidma_chan_init()
223 dmadev->ddev.chancnt++; in hidma_chan_init()
231 pm_runtime_get_sync(dmadev->ddev.dev); in hidma_issue_task()
232 hidma_ll_start(dmadev->lldev); in hidma_issue_task()
238 struct hidma_dev *dmadev = mchan->dmadev; in hidma_issue_pending()
243 spin_lock_irqsave(&mchan->lock, flags); in hidma_issue_pending()
244 list_for_each_entry_safe(qdesc, next, &mchan->queued, node) { in hidma_issue_pending()
245 hidma_ll_queue_request(dmadev->lldev, qdesc->tre_ch); in hidma_issue_pending()
246 list_move_tail(&qdesc->node, &mchan->active); in hidma_issue_pending()
249 if (!mchan->running) { in hidma_issue_pending()
250 struct hidma_desc *desc = list_first_entry(&mchan->active, in hidma_issue_pending()
253 mchan->running = desc; in hidma_issue_pending()
255 spin_unlock_irqrestore(&mchan->lock, flags); in hidma_issue_pending()
258 status = pm_runtime_get(dmadev->ddev.dev); in hidma_issue_pending()
260 tasklet_schedule(&dmadev->task); in hidma_issue_pending()
262 hidma_ll_start(dmadev->lldev); in hidma_issue_pending()
289 is_success = hidma_txn_is_success(cookie, mchan->last_success, in hidma_tx_status()
290 dmach->cookie); in hidma_tx_status()
294 if (mchan->paused && (ret == DMA_IN_PROGRESS)) { in hidma_tx_status()
298 spin_lock_irqsave(&mchan->lock, flags); in hidma_tx_status()
299 if (mchan->running) in hidma_tx_status()
300 runcookie = mchan->running->desc.cookie; in hidma_tx_status()
302 runcookie = -EINVAL; in hidma_tx_status()
307 spin_unlock_irqrestore(&mchan->lock, flags); in hidma_tx_status()
319 struct hidma_chan *mchan = to_hidma_chan(txd->chan); in hidma_tx_submit()
320 struct hidma_dev *dmadev = mchan->dmadev; in hidma_tx_submit()
325 pm_runtime_get_sync(dmadev->ddev.dev); in hidma_tx_submit()
326 if (!hidma_ll_isenabled(dmadev->lldev)) { in hidma_tx_submit()
327 pm_runtime_mark_last_busy(dmadev->ddev.dev); in hidma_tx_submit()
328 pm_runtime_put_autosuspend(dmadev->ddev.dev); in hidma_tx_submit()
329 return -ENODEV; in hidma_tx_submit()
331 pm_runtime_mark_last_busy(dmadev->ddev.dev); in hidma_tx_submit()
332 pm_runtime_put_autosuspend(dmadev->ddev.dev); in hidma_tx_submit()
335 spin_lock_irqsave(&mchan->lock, irqflags); in hidma_tx_submit()
338 list_move_tail(&mdesc->node, &mchan->queued); in hidma_tx_submit()
343 spin_unlock_irqrestore(&mchan->lock, irqflags); in hidma_tx_submit()
351 struct hidma_dev *dmadev = mchan->dmadev; in hidma_alloc_chan_resources()
358 if (mchan->allocated) in hidma_alloc_chan_resources()
362 for (i = 0; i < dmadev->nr_descriptors; i++) { in hidma_alloc_chan_resources()
365 rc = -ENOMEM; in hidma_alloc_chan_resources()
368 dma_async_tx_descriptor_init(&mdesc->desc, dmach); in hidma_alloc_chan_resources()
369 mdesc->desc.tx_submit = hidma_tx_submit; in hidma_alloc_chan_resources()
371 rc = hidma_ll_request(dmadev->lldev, mchan->dma_sig, in hidma_alloc_chan_resources()
373 &mdesc->tre_ch); in hidma_alloc_chan_resources()
375 dev_err(dmach->device->dev, in hidma_alloc_chan_resources()
380 list_add_tail(&mdesc->node, &descs); in hidma_alloc_chan_resources()
386 hidma_ll_free(dmadev->lldev, mdesc->tre_ch); in hidma_alloc_chan_resources()
392 spin_lock_irqsave(&mchan->lock, irqflags); in hidma_alloc_chan_resources()
393 list_splice_tail_init(&descs, &mchan->free); in hidma_alloc_chan_resources()
394 mchan->allocated = true; in hidma_alloc_chan_resources()
395 spin_unlock_irqrestore(&mchan->lock, irqflags); in hidma_alloc_chan_resources()
405 struct hidma_dev *mdma = mchan->dmadev; in hidma_prep_dma_memcpy()
408 /* Get free descriptor */ in hidma_prep_dma_memcpy()
409 spin_lock_irqsave(&mchan->lock, irqflags); in hidma_prep_dma_memcpy()
410 if (!list_empty(&mchan->free)) { in hidma_prep_dma_memcpy()
411 mdesc = list_first_entry(&mchan->free, struct hidma_desc, node); in hidma_prep_dma_memcpy()
412 list_del(&mdesc->node); in hidma_prep_dma_memcpy()
414 spin_unlock_irqrestore(&mchan->lock, irqflags); in hidma_prep_dma_memcpy()
419 mdesc->desc.flags = flags; in hidma_prep_dma_memcpy()
420 hidma_ll_set_transfer_params(mdma->lldev, mdesc->tre_ch, in hidma_prep_dma_memcpy()
425 spin_lock_irqsave(&mchan->lock, irqflags); in hidma_prep_dma_memcpy()
426 list_add_tail(&mdesc->node, &mchan->prepared); in hidma_prep_dma_memcpy()
427 spin_unlock_irqrestore(&mchan->lock, irqflags); in hidma_prep_dma_memcpy()
429 return &mdesc->desc; in hidma_prep_dma_memcpy()
438 struct hidma_dev *mdma = mchan->dmadev; in hidma_prep_dma_memset()
441 /* Get free descriptor */ in hidma_prep_dma_memset()
442 spin_lock_irqsave(&mchan->lock, irqflags); in hidma_prep_dma_memset()
443 if (!list_empty(&mchan->free)) { in hidma_prep_dma_memset()
444 mdesc = list_first_entry(&mchan->free, struct hidma_desc, node); in hidma_prep_dma_memset()
445 list_del(&mdesc->node); in hidma_prep_dma_memset()
447 spin_unlock_irqrestore(&mchan->lock, irqflags); in hidma_prep_dma_memset()
452 mdesc->desc.flags = flags; in hidma_prep_dma_memset()
453 hidma_ll_set_transfer_params(mdma->lldev, mdesc->tre_ch, in hidma_prep_dma_memset()
458 spin_lock_irqsave(&mchan->lock, irqflags); in hidma_prep_dma_memset()
459 list_add_tail(&mdesc->node, &mchan->prepared); in hidma_prep_dma_memset()
460 spin_unlock_irqrestore(&mchan->lock, irqflags); in hidma_prep_dma_memset()
462 return &mdesc->desc; in hidma_prep_dma_memset()
468 struct hidma_dev *dmadev = to_hidma_dev(mchan->chan.device); in hidma_terminate_channel()
474 pm_runtime_get_sync(dmadev->ddev.dev); in hidma_terminate_channel()
478 spin_lock_irqsave(&mchan->lock, irqflags); in hidma_terminate_channel()
479 mchan->last_success = 0; in hidma_terminate_channel()
480 list_splice_init(&mchan->active, &list); in hidma_terminate_channel()
481 list_splice_init(&mchan->prepared, &list); in hidma_terminate_channel()
482 list_splice_init(&mchan->completed, &list); in hidma_terminate_channel()
483 list_splice_init(&mchan->queued, &list); in hidma_terminate_channel()
484 spin_unlock_irqrestore(&mchan->lock, irqflags); in hidma_terminate_channel()
487 rc = hidma_ll_disable(dmadev->lldev); in hidma_terminate_channel()
489 dev_err(dmadev->ddev.dev, "channel did not pause\n"); in hidma_terminate_channel()
495 struct dma_async_tx_descriptor *txd = &mdesc->desc; in hidma_terminate_channel()
502 list_move(&mdesc->node, &mchan->free); in hidma_terminate_channel()
505 rc = hidma_ll_enable(dmadev->lldev); in hidma_terminate_channel()
507 pm_runtime_mark_last_busy(dmadev->ddev.dev); in hidma_terminate_channel()
508 pm_runtime_put_autosuspend(dmadev->ddev.dev); in hidma_terminate_channel()
515 struct hidma_dev *dmadev = to_hidma_dev(mchan->chan.device); in hidma_terminate_all()
523 pm_runtime_get_sync(dmadev->ddev.dev); in hidma_terminate_all()
524 rc = hidma_ll_setup(dmadev->lldev); in hidma_terminate_all()
525 pm_runtime_mark_last_busy(dmadev->ddev.dev); in hidma_terminate_all()
526 pm_runtime_put_autosuspend(dmadev->ddev.dev); in hidma_terminate_all()
533 struct hidma_dev *mdma = mchan->dmadev; in hidma_free_chan_resources()
538 /* terminate running transactions and free descriptors */ in hidma_free_chan_resources()
541 spin_lock_irqsave(&mchan->lock, irqflags); in hidma_free_chan_resources()
544 list_splice_tail_init(&mchan->free, &descs); in hidma_free_chan_resources()
546 /* Free descriptors */ in hidma_free_chan_resources()
548 hidma_ll_free(mdma->lldev, mdesc->tre_ch); in hidma_free_chan_resources()
549 list_del(&mdesc->node); in hidma_free_chan_resources()
553 mchan->allocated = false; in hidma_free_chan_resources()
554 spin_unlock_irqrestore(&mchan->lock, irqflags); in hidma_free_chan_resources()
563 dmadev = to_hidma_dev(mchan->chan.device); in hidma_pause()
564 if (!mchan->paused) { in hidma_pause()
565 pm_runtime_get_sync(dmadev->ddev.dev); in hidma_pause()
566 if (hidma_ll_disable(dmadev->lldev)) in hidma_pause()
567 dev_warn(dmadev->ddev.dev, "channel did not stop\n"); in hidma_pause()
568 mchan->paused = true; in hidma_pause()
569 pm_runtime_mark_last_busy(dmadev->ddev.dev); in hidma_pause()
570 pm_runtime_put_autosuspend(dmadev->ddev.dev); in hidma_pause()
582 dmadev = to_hidma_dev(mchan->chan.device); in hidma_resume()
583 if (mchan->paused) { in hidma_resume()
584 pm_runtime_get_sync(dmadev->ddev.dev); in hidma_resume()
585 rc = hidma_ll_enable(dmadev->lldev); in hidma_resume()
587 mchan->paused = false; in hidma_resume()
589 dev_err(dmadev->ddev.dev, in hidma_resume()
591 pm_runtime_mark_last_busy(dmadev->ddev.dev); in hidma_resume()
592 pm_runtime_put_autosuspend(dmadev->ddev.dev); in hidma_resume()
615 1 << (chirq - dmadev->msi_virqbase)); in hidma_chirq_handler_msi()
626 if (strcmp(attr->attr.name, "chid") == 0) in hidma_show_values()
627 sprintf(buf, "%d\n", mdev->chidx); in hidma_show_values()
634 device_remove_file(dev->ddev.dev, dev->chid_attrs); in hidma_sysfs_uninit()
643 attrs = devm_kmalloc(dev->ddev.dev, sizeof(struct device_attribute), in hidma_create_sysfs_entry()
648 name_copy = devm_kstrdup(dev->ddev.dev, name, GFP_KERNEL); in hidma_create_sysfs_entry()
652 attrs->attr.name = name_copy; in hidma_create_sysfs_entry()
653 attrs->attr.mode = mode; in hidma_create_sysfs_entry()
654 attrs->show = hidma_show_values; in hidma_create_sysfs_entry()
655 sysfs_attr_init(&attrs->attr); in hidma_create_sysfs_entry()
662 dev->chid_attrs = hidma_create_sysfs_entry(dev, "chid", S_IRUGO); in hidma_sysfs_init()
663 if (!dev->chid_attrs) in hidma_sysfs_init()
664 return -ENOMEM; in hidma_sysfs_init()
666 return device_create_file(dev->ddev.dev, dev->chid_attrs); in hidma_sysfs_init()
675 if (!desc->platform.msi_index) { in hidma_write_msi_msg()
676 writel(msg->address_lo, dmadev->dev_evca + 0x118); in hidma_write_msi_msg()
677 writel(msg->address_hi, dmadev->dev_evca + 0x11C); in hidma_write_msi_msg()
678 writel(msg->data, dmadev->dev_evca + 0x120); in hidma_write_msi_msg()
686 struct device *dev = dmadev->ddev.dev; in hidma_free_msis()
689 /* free allocated MSI interrupts above */ in hidma_free_msis()
691 devm_free_irq(dev, desc->irq, &dmadev->lldev); in hidma_free_msis()
705 rc = platform_msi_domain_alloc_irqs(&pdev->dev, HIDMA_MSI_INTS, in hidma_request_msi()
710 for_each_msi_entry(desc, &pdev->dev) { in hidma_request_msi()
711 if (!desc->platform.msi_index) in hidma_request_msi()
712 dmadev->msi_virqbase = desc->irq; in hidma_request_msi()
714 rc = devm_request_irq(&pdev->dev, desc->irq, in hidma_request_msi()
716 0, "qcom-hidma-msi", in hidma_request_msi()
717 &dmadev->lldev); in hidma_request_msi()
725 /* free allocated MSI interrupts above */ in hidma_request_msi()
726 for_each_msi_entry(desc, &pdev->dev) { in hidma_request_msi()
729 devm_free_irq(&pdev->dev, desc->irq, in hidma_request_msi()
730 &dmadev->lldev); in hidma_request_msi()
733 /* Add callback to free MSIs on teardown */ in hidma_request_msi()
734 hidma_ll_setup_irq(dmadev->lldev, true); in hidma_request_msi()
738 dev_warn(&pdev->dev, in hidma_request_msi()
742 return -EINVAL; in hidma_request_msi()
765 pm_runtime_set_autosuspend_delay(&pdev->dev, HIDMA_AUTOSUSPEND_TIMEOUT); in hidma_probe()
766 pm_runtime_use_autosuspend(&pdev->dev); in hidma_probe()
767 pm_runtime_set_active(&pdev->dev); in hidma_probe()
768 pm_runtime_enable(&pdev->dev); in hidma_probe()
771 trca = devm_ioremap_resource(&pdev->dev, trca_resource); in hidma_probe()
773 rc = -ENOMEM; in hidma_probe()
778 evca = devm_ioremap_resource(&pdev->dev, evca_resource); in hidma_probe()
780 rc = -ENOMEM; in hidma_probe()
790 rc = -ENODEV; in hidma_probe()
794 dmadev = devm_kzalloc(&pdev->dev, sizeof(*dmadev), GFP_KERNEL); in hidma_probe()
796 rc = -ENOMEM; in hidma_probe()
800 INIT_LIST_HEAD(&dmadev->ddev.channels); in hidma_probe()
801 spin_lock_init(&dmadev->lock); in hidma_probe()
802 dmadev->ddev.dev = &pdev->dev; in hidma_probe()
803 pm_runtime_get_sync(dmadev->ddev.dev); in hidma_probe()
805 dma_cap_set(DMA_MEMCPY, dmadev->ddev.cap_mask); in hidma_probe()
806 dma_cap_set(DMA_MEMSET, dmadev->ddev.cap_mask); in hidma_probe()
807 if (WARN_ON(!pdev->dev.dma_mask)) { in hidma_probe()
808 rc = -ENXIO; in hidma_probe()
812 dmadev->dev_evca = evca; in hidma_probe()
813 dmadev->evca_resource = evca_resource; in hidma_probe()
814 dmadev->dev_trca = trca; in hidma_probe()
815 dmadev->trca_resource = trca_resource; in hidma_probe()
816 dmadev->ddev.device_prep_dma_memcpy = hidma_prep_dma_memcpy; in hidma_probe()
817 dmadev->ddev.device_prep_dma_memset = hidma_prep_dma_memset; in hidma_probe()
818 dmadev->ddev.device_alloc_chan_resources = hidma_alloc_chan_resources; in hidma_probe()
819 dmadev->ddev.device_free_chan_resources = hidma_free_chan_resources; in hidma_probe()
820 dmadev->ddev.device_tx_status = hidma_tx_status; in hidma_probe()
821 dmadev->ddev.device_issue_pending = hidma_issue_pending; in hidma_probe()
822 dmadev->ddev.device_pause = hidma_pause; in hidma_probe()
823 dmadev->ddev.device_resume = hidma_resume; in hidma_probe()
824 dmadev->ddev.device_terminate_all = hidma_terminate_all; in hidma_probe()
825 dmadev->ddev.copy_align = 8; in hidma_probe()
831 msi = hidma_test_capability(&pdev->dev, HIDMA_MSI_CAP); in hidma_probe()
832 device_property_read_u32(&pdev->dev, "desc-count", in hidma_probe()
833 &dmadev->nr_descriptors); in hidma_probe()
836 dev_info(&pdev->dev, "overriding number of descriptors as %d\n", in hidma_probe()
838 dmadev->nr_descriptors = nr_desc_prm; in hidma_probe()
841 if (!dmadev->nr_descriptors) in hidma_probe()
842 dmadev->nr_descriptors = HIDMA_NR_DEFAULT_DESC; in hidma_probe()
844 if (hidma_test_capability(&pdev->dev, HIDMA_IDENTITY_CAP)) in hidma_probe()
845 dmadev->chidx = readl(dmadev->dev_trca + 0x40); in hidma_probe()
847 dmadev->chidx = readl(dmadev->dev_trca + 0x28); in hidma_probe()
850 rc = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); in hidma_probe()
852 dev_warn(&pdev->dev, "unable to set coherent mask to 64"); in hidma_probe()
853 rc = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)); in hidma_probe()
858 dmadev->lldev = hidma_ll_init(dmadev->ddev.dev, in hidma_probe()
859 dmadev->nr_descriptors, dmadev->dev_trca, in hidma_probe()
860 dmadev->dev_evca, dmadev->chidx); in hidma_probe()
861 if (!dmadev->lldev) { in hidma_probe()
862 rc = -EPROBE_DEFER; in hidma_probe()
871 hidma_ll_setup_irq(dmadev->lldev, false); in hidma_probe()
872 rc = devm_request_irq(&pdev->dev, chirq, hidma_chirq_handler, in hidma_probe()
873 0, "qcom-hidma", dmadev->lldev); in hidma_probe()
878 INIT_LIST_HEAD(&dmadev->ddev.channels); in hidma_probe()
883 rc = dma_async_device_register(&dmadev->ddev); in hidma_probe()
887 dmadev->irq = chirq; in hidma_probe()
888 tasklet_setup(&dmadev->task, hidma_issue_task); in hidma_probe()
891 dev_info(&pdev->dev, "HI-DMA engine driver registration complete\n"); in hidma_probe()
892 pm_runtime_mark_last_busy(dmadev->ddev.dev); in hidma_probe()
893 pm_runtime_put_autosuspend(dmadev->ddev.dev); in hidma_probe()
900 hidma_ll_uninit(dmadev->lldev); in hidma_probe()
905 pm_runtime_put_sync(&pdev->dev); in hidma_probe()
906 pm_runtime_disable(&pdev->dev); in hidma_probe()
914 dev_info(dmadev->ddev.dev, "HI-DMA engine shutdown\n"); in hidma_shutdown()
916 pm_runtime_get_sync(dmadev->ddev.dev); in hidma_shutdown()
917 if (hidma_ll_disable(dmadev->lldev)) in hidma_shutdown()
918 dev_warn(dmadev->ddev.dev, "channel did not stop\n"); in hidma_shutdown()
919 pm_runtime_mark_last_busy(dmadev->ddev.dev); in hidma_shutdown()
920 pm_runtime_put_autosuspend(dmadev->ddev.dev); in hidma_shutdown()
928 pm_runtime_get_sync(dmadev->ddev.dev); in hidma_remove()
929 dma_async_device_unregister(&dmadev->ddev); in hidma_remove()
930 if (!dmadev->lldev->msi_support) in hidma_remove()
931 devm_free_irq(dmadev->ddev.dev, dmadev->irq, dmadev->lldev); in hidma_remove()
935 tasklet_kill(&dmadev->task); in hidma_remove()
938 hidma_ll_uninit(dmadev->lldev); in hidma_remove()
941 dev_info(&pdev->dev, "HI-DMA engine removed\n"); in hidma_remove()
942 pm_runtime_put_sync_suspend(&pdev->dev); in hidma_remove()
943 pm_runtime_disable(&pdev->dev); in hidma_remove()
959 {.compatible = "qcom,hidma-1.0",},
960 {.compatible = "qcom,hidma-1.1", .data = (void *)(HIDMA_MSI_CAP),},
961 {.compatible = "qcom,hidma-1.2",