Lines Matching +full:map +full:- +full:to +full:- +full:dma +full:- +full:channel
1 // SPDX-License-Identifier: GPL-2.0
3 * Test driver to test endpoint functionality
18 #include <linux/pci-epc.h>
19 #include <linux/pci-epf.h>
99 epf_test->transfer_status = in pci_epf_test_dma_callback()
100 dmaengine_tx_status(epf_test->transfer_chan, in pci_epf_test_dma_callback()
101 epf_test->transfer_cookie, &state); in pci_epf_test_dma_callback()
102 if (epf_test->transfer_status == DMA_COMPLETE || in pci_epf_test_dma_callback()
103 epf_test->transfer_status == DMA_ERROR) in pci_epf_test_dma_callback()
104 complete(&epf_test->transfer_complete); in pci_epf_test_dma_callback()
108 * pci_epf_test_data_transfer() - Function that uses dmaengine API to transfer
112 * address given by pci_epc_mem_alloc_addr or DMA mapping APIs.
114 * address given by pci_epc_mem_alloc_addr or DMA mapping APIs.
117 * @dir: DMA transfer direction
119 * Function that uses dmaengine API to transfer data between PCIe EP and remote
121 * by pci_epc_mem_alloc_addr or the one obtained using DMA mapping APIs.
131 epf_test->dma_chan_tx : epf_test->dma_chan_rx; in pci_epf_test_data_transfer()
134 struct pci_epf *epf = epf_test->epf; in pci_epf_test_data_transfer()
137 struct device *dev = &epf->dev; in pci_epf_test_data_transfer()
141 dev_err(dev, "Invalid DMA memcpy channel\n"); in pci_epf_test_data_transfer()
142 return -EINVAL; in pci_epf_test_data_transfer()
145 if (epf_test->dma_private) { in pci_epf_test_data_transfer()
153 dev_err(dev, "DMA slave config fail\n"); in pci_epf_test_data_transfer()
154 return -EIO; in pci_epf_test_data_transfer()
164 dev_err(dev, "Failed to prepare DMA memcpy\n"); in pci_epf_test_data_transfer()
165 return -EIO; in pci_epf_test_data_transfer()
168 reinit_completion(&epf_test->transfer_complete); in pci_epf_test_data_transfer()
169 epf_test->transfer_chan = chan; in pci_epf_test_data_transfer()
170 tx->callback = pci_epf_test_dma_callback; in pci_epf_test_data_transfer()
171 tx->callback_param = epf_test; in pci_epf_test_data_transfer()
172 epf_test->transfer_cookie = dmaengine_submit(tx); in pci_epf_test_data_transfer()
174 ret = dma_submit_error(epf_test->transfer_cookie); in pci_epf_test_data_transfer()
176 dev_err(dev, "Failed to do DMA tx_submit %d\n", ret); in pci_epf_test_data_transfer()
181 ret = wait_for_completion_interruptible(&epf_test->transfer_complete); in pci_epf_test_data_transfer()
183 dev_err(dev, "DMA wait_for_completion interrupted\n"); in pci_epf_test_data_transfer()
187 if (epf_test->transfer_status == DMA_ERROR) { in pci_epf_test_data_transfer()
188 dev_err(dev, "DMA transfer failed\n"); in pci_epf_test_data_transfer()
189 ret = -EIO; in pci_epf_test_data_transfer()
211 return chan->device->dev == filter->dev in epf_dma_filter_fn()
212 && (filter->dma_mask & caps.directions); in epf_dma_filter_fn()
216 * pci_epf_test_init_dma_chan() - Function to initialize EPF test DMA channel
219 * Function to initialize EPF test DMA channel.
223 struct pci_epf *epf = epf_test->epf; in pci_epf_test_init_dma_chan()
224 struct device *dev = &epf->dev; in pci_epf_test_init_dma_chan()
230 filter.dev = epf->epc->dev.parent; in pci_epf_test_init_dma_chan()
237 dev_info(dev, "Failed to get private DMA rx channel. Falling back to generic one\n"); in pci_epf_test_init_dma_chan()
241 epf_test->dma_chan_rx = dma_chan; in pci_epf_test_init_dma_chan()
247 dev_info(dev, "Failed to get private DMA tx channel. Falling back to generic one\n"); in pci_epf_test_init_dma_chan()
251 epf_test->dma_chan_tx = dma_chan; in pci_epf_test_init_dma_chan()
252 epf_test->dma_private = true; in pci_epf_test_init_dma_chan()
254 init_completion(&epf_test->transfer_complete); in pci_epf_test_init_dma_chan()
259 dma_release_channel(epf_test->dma_chan_rx); in pci_epf_test_init_dma_chan()
260 epf_test->dma_chan_rx = NULL; in pci_epf_test_init_dma_chan()
269 if (ret != -EPROBE_DEFER) in pci_epf_test_init_dma_chan()
270 dev_err(dev, "Failed to get DMA channel\n"); in pci_epf_test_init_dma_chan()
273 init_completion(&epf_test->transfer_complete); in pci_epf_test_init_dma_chan()
275 epf_test->dma_chan_tx = epf_test->dma_chan_rx = dma_chan; in pci_epf_test_init_dma_chan()
281 * pci_epf_test_clean_dma_chan() - Function to cleanup EPF test DMA channel
284 * Helper to cleanup EPF test DMA channel.
288 if (!epf_test->dma_supported) in pci_epf_test_clean_dma_chan()
291 dma_release_channel(epf_test->dma_chan_tx); in pci_epf_test_clean_dma_chan()
292 if (epf_test->dma_chan_tx == epf_test->dma_chan_rx) { in pci_epf_test_clean_dma_chan()
293 epf_test->dma_chan_tx = NULL; in pci_epf_test_clean_dma_chan()
294 epf_test->dma_chan_rx = NULL; in pci_epf_test_clean_dma_chan()
298 dma_release_channel(epf_test->dma_chan_rx); in pci_epf_test_clean_dma_chan()
299 epf_test->dma_chan_rx = NULL; in pci_epf_test_clean_dma_chan()
305 struct timespec64 *end, bool dma) in pci_epf_test_print_rate() argument
315 dev_info(&epf_test->epf->dev, in pci_epf_test_print_rate()
316 "%s => Size: %llu B, DMA: %s, Time: %llu.%09u s, Rate: %llu KB/s\n", in pci_epf_test_print_rate()
317 op, size, dma ? "YES" : "NO", in pci_epf_test_print_rate()
326 struct pci_epf *epf = epf_test->epf; in pci_epf_test_copy()
327 struct pci_epc *epc = epf->epc; in pci_epf_test_copy()
328 struct device *dev = &epf->dev; in pci_epf_test_copy()
330 u64 src_addr = le64_to_cpu(reg->src_addr); in pci_epf_test_copy()
331 u64 dst_addr = le64_to_cpu(reg->dst_addr); in pci_epf_test_copy()
334 u32 flags = le32_to_cpu(reg->flags); in pci_epf_test_copy()
338 orig_size = copy_size = le32_to_cpu(reg->size); in pci_epf_test_copy()
341 if (!dma_has_cap(DMA_MEMCPY, epf_test->dma_chan_tx->device->cap_mask)) { in pci_epf_test_copy()
342 dev_err(dev, "DMA controller doesn't support MEMCPY\n"); in pci_epf_test_copy()
343 ret = -EINVAL; in pci_epf_test_copy()
349 ret = -ENOMEM; in pci_epf_test_copy()
356 ret = pci_epc_mem_map(epc, epf->func_no, epf->vfunc_no, in pci_epf_test_copy()
359 dev_err(dev, "Failed to map source address\n"); in pci_epf_test_copy()
364 ret = pci_epc_mem_map(epf->epc, epf->func_no, epf->vfunc_no, in pci_epf_test_copy()
367 dev_err(dev, "Failed to map destination address\n"); in pci_epf_test_copy()
369 pci_epc_mem_unmap(epc, epf->func_no, epf->vfunc_no, in pci_epf_test_copy()
392 copy_size -= map_size; in pci_epf_test_copy()
396 pci_epc_mem_unmap(epc, epf->func_no, epf->vfunc_no, &dst_map); in pci_epf_test_copy()
397 pci_epc_mem_unmap(epc, epf->func_no, epf->vfunc_no, &src_map); in pci_epf_test_copy()
406 pci_epc_mem_unmap(epc, epf->func_no, epf->vfunc_no, &dst_map); in pci_epf_test_copy()
407 pci_epc_mem_unmap(epc, epf->func_no, epf->vfunc_no, &src_map); in pci_epf_test_copy()
418 reg->status = cpu_to_le32(status); in pci_epf_test_copy()
427 struct pci_epc_map map; in pci_epf_test_read() local
430 struct pci_epf *epf = epf_test->epf; in pci_epf_test_read()
431 struct pci_epc *epc = epf->epc; in pci_epf_test_read()
432 struct device *dev = &epf->dev; in pci_epf_test_read()
433 struct device *dma_dev = epf->epc->dev.parent; in pci_epf_test_read()
434 u64 src_addr = le64_to_cpu(reg->src_addr); in pci_epf_test_read()
437 u32 flags = le32_to_cpu(reg->flags); in pci_epf_test_read()
438 u32 checksum = le32_to_cpu(reg->checksum); in pci_epf_test_read()
441 orig_size = src_size = le32_to_cpu(reg->size); in pci_epf_test_read()
445 ret = -ENOMEM; in pci_epf_test_read()
451 ret = pci_epc_mem_map(epc, epf->func_no, epf->vfunc_no, in pci_epf_test_read()
452 src_addr, src_size, &map); in pci_epf_test_read()
454 dev_err(dev, "Failed to map address\n"); in pci_epf_test_read()
459 map_size = map.pci_size; in pci_epf_test_read()
465 "Failed to map destination buffer addr\n"); in pci_epf_test_read()
466 ret = -ENOMEM; in pci_epf_test_read()
472 dst_phys_addr, map.phys_addr, in pci_epf_test_read()
485 memcpy_fromio(buf, map.virt_addr, map_size); in pci_epf_test_read()
489 src_size -= map_size; in pci_epf_test_read()
493 pci_epc_mem_unmap(epc, epf->func_no, epf->vfunc_no, &map); in pci_epf_test_read()
502 ret = -EIO; in pci_epf_test_read()
506 pci_epc_mem_unmap(epc, epf->func_no, epf->vfunc_no, &map); in pci_epf_test_read()
516 reg->status = cpu_to_le32(status); in pci_epf_test_read()
524 struct pci_epc_map map; in pci_epf_test_write() local
527 struct pci_epf *epf = epf_test->epf; in pci_epf_test_write()
528 struct pci_epc *epc = epf->epc; in pci_epf_test_write()
529 struct device *dev = &epf->dev; in pci_epf_test_write()
530 struct device *dma_dev = epf->epc->dev.parent; in pci_epf_test_write()
531 u64 dst_addr = le64_to_cpu(reg->dst_addr); in pci_epf_test_write()
534 u32 flags = le32_to_cpu(reg->flags); in pci_epf_test_write()
537 orig_size = dst_size = le32_to_cpu(reg->size); in pci_epf_test_write()
541 ret = -ENOMEM; in pci_epf_test_write()
545 reg->checksum = cpu_to_le32(crc32_le(~0, dst_buf, dst_size)); in pci_epf_test_write()
549 ret = pci_epc_mem_map(epc, epf->func_no, epf->vfunc_no, in pci_epf_test_write()
550 dst_addr, dst_size, &map); in pci_epf_test_write()
552 dev_err(dev, "Failed to map address\n"); in pci_epf_test_write()
557 map_size = map.pci_size; in pci_epf_test_write()
563 "Failed to map source buffer addr\n"); in pci_epf_test_write()
564 ret = -ENOMEM; in pci_epf_test_write()
571 map.phys_addr, src_phys_addr, in pci_epf_test_write()
585 memcpy_toio(map.virt_addr, buf, map_size); in pci_epf_test_write()
589 dst_size -= map_size; in pci_epf_test_write()
593 pci_epc_mem_unmap(epc, epf->func_no, epf->vfunc_no, &map); in pci_epf_test_write()
601 * wait 1ms inorder for the write to complete. Without this delay L3 in pci_epf_test_write()
608 pci_epc_mem_unmap(epc, epf->func_no, epf->vfunc_no, &map); in pci_epf_test_write()
618 reg->status = cpu_to_le32(status); in pci_epf_test_write()
624 struct pci_epf *epf = epf_test->epf; in pci_epf_test_raise_irq()
625 struct device *dev = &epf->dev; in pci_epf_test_raise_irq()
626 struct pci_epc *epc = epf->epc; in pci_epf_test_raise_irq()
627 u32 status = le32_to_cpu(reg->status); in pci_epf_test_raise_irq()
628 u32 irq_number = le32_to_cpu(reg->irq_number); in pci_epf_test_raise_irq()
629 u32 irq_type = le32_to_cpu(reg->irq_type); in pci_epf_test_raise_irq()
633 * Set the status before raising the IRQ to ensure that the host sees in pci_epf_test_raise_irq()
637 WRITE_ONCE(reg->status, cpu_to_le32(status)); in pci_epf_test_raise_irq()
641 pci_epc_raise_irq(epc, epf->func_no, epf->vfunc_no, in pci_epf_test_raise_irq()
645 count = pci_epc_get_msi(epc, epf->func_no, epf->vfunc_no); in pci_epf_test_raise_irq()
651 pci_epc_raise_irq(epc, epf->func_no, epf->vfunc_no, in pci_epf_test_raise_irq()
655 count = pci_epc_get_msix(epc, epf->func_no, epf->vfunc_no); in pci_epf_test_raise_irq()
657 dev_err(dev, "Invalid MSI-X IRQ number %d / %d\n", in pci_epf_test_raise_irq()
661 pci_epc_raise_irq(epc, epf->func_no, epf->vfunc_no, in pci_epf_test_raise_irq()
665 dev_err(dev, "Failed to raise IRQ, unknown type\n"); in pci_epf_test_raise_irq()
675 struct pci_epf *epf = epf_test->epf; in pci_epf_test_cmd_handler()
676 struct device *dev = &epf->dev; in pci_epf_test_cmd_handler()
677 enum pci_barno test_reg_bar = epf_test->test_reg_bar; in pci_epf_test_cmd_handler()
678 struct pci_epf_test_reg *reg = epf_test->reg[test_reg_bar]; in pci_epf_test_cmd_handler()
679 u32 irq_type = le32_to_cpu(reg->irq_type); in pci_epf_test_cmd_handler()
681 command = le32_to_cpu(READ_ONCE(reg->command)); in pci_epf_test_cmd_handler()
685 WRITE_ONCE(reg->command, 0); in pci_epf_test_cmd_handler()
686 WRITE_ONCE(reg->status, 0); in pci_epf_test_cmd_handler()
688 if ((le32_to_cpu(READ_ONCE(reg->flags)) & FLAG_USE_DMA) && in pci_epf_test_cmd_handler()
689 !epf_test->dma_supported) { in pci_epf_test_cmd_handler()
690 dev_err(dev, "Cannot transfer data using DMA\n"); in pci_epf_test_cmd_handler()
695 dev_err(dev, "Failed to detect IRQ type\n"); in pci_epf_test_cmd_handler()
723 queue_delayed_work(kpcitest_workqueue, &epf_test->cmd_handler, in pci_epf_test_cmd_handler()
730 struct pci_epc *epc = epf->epc; in pci_epf_test_set_bar()
731 struct device *dev = &epf->dev; in pci_epf_test_set_bar()
733 enum pci_barno test_reg_bar = epf_test->test_reg_bar; in pci_epf_test_set_bar()
736 if (!epf_test->reg[bar]) in pci_epf_test_set_bar()
739 ret = pci_epc_set_bar(epc, epf->func_no, epf->vfunc_no, in pci_epf_test_set_bar()
740 &epf->bar[bar]); in pci_epf_test_set_bar()
742 pci_epf_free_space(epf, epf_test->reg[bar], bar, in pci_epf_test_set_bar()
744 epf_test->reg[bar] = NULL; in pci_epf_test_set_bar()
745 dev_err(dev, "Failed to set BAR%d\n", bar); in pci_epf_test_set_bar()
757 struct pci_epc *epc = epf->epc; in pci_epf_test_clear_bar()
761 if (!epf_test->reg[bar]) in pci_epf_test_clear_bar()
764 pci_epc_clear_bar(epc, epf->func_no, epf->vfunc_no, in pci_epf_test_clear_bar()
765 &epf->bar[bar]); in pci_epf_test_clear_bar()
772 enum pci_barno test_reg_bar = epf_test->test_reg_bar; in pci_epf_test_set_capabilities()
773 struct pci_epf_test_reg *reg = epf_test->reg[test_reg_bar]; in pci_epf_test_set_capabilities()
774 struct pci_epc *epc = epf->epc; in pci_epf_test_set_capabilities()
777 if (epc->ops->align_addr) in pci_epf_test_set_capabilities()
780 if (epf_test->epc_features->msi_capable) in pci_epf_test_set_capabilities()
783 if (epf_test->epc_features->msix_capable) in pci_epf_test_set_capabilities()
786 if (epf_test->epc_features->intx_capable) in pci_epf_test_set_capabilities()
789 reg->caps = cpu_to_le32(caps); in pci_epf_test_set_capabilities()
795 struct pci_epf_header *header = epf->header; in pci_epf_test_epc_init()
796 const struct pci_epc_features *epc_features = epf_test->epc_features; in pci_epf_test_epc_init()
797 struct pci_epc *epc = epf->epc; in pci_epf_test_epc_init()
798 struct device *dev = &epf->dev; in pci_epf_test_epc_init()
802 epf_test->dma_supported = true; in pci_epf_test_epc_init()
806 epf_test->dma_supported = false; in pci_epf_test_epc_init()
808 if (epf->vfunc_no <= 1) { in pci_epf_test_epc_init()
809 ret = pci_epc_write_header(epc, epf->func_no, epf->vfunc_no, header); in pci_epf_test_epc_init()
822 if (epc_features->msi_capable) { in pci_epf_test_epc_init()
823 ret = pci_epc_set_msi(epc, epf->func_no, epf->vfunc_no, in pci_epf_test_epc_init()
824 epf->msi_interrupts); in pci_epf_test_epc_init()
831 if (epc_features->msix_capable) { in pci_epf_test_epc_init()
832 ret = pci_epc_set_msix(epc, epf->func_no, epf->vfunc_no, in pci_epf_test_epc_init()
833 epf->msix_interrupts, in pci_epf_test_epc_init()
834 epf_test->test_reg_bar, in pci_epf_test_epc_init()
835 epf_test->msix_table_offset); in pci_epf_test_epc_init()
837 dev_err(dev, "MSI-X configuration failed\n"); in pci_epf_test_epc_init()
842 linkup_notifier = epc_features->linkup_notifier; in pci_epf_test_epc_init()
844 queue_work(kpcitest_workqueue, &epf_test->cmd_handler.work); in pci_epf_test_epc_init()
853 cancel_delayed_work_sync(&epf_test->cmd_handler); in pci_epf_test_epc_deinit()
862 queue_delayed_work(kpcitest_workqueue, &epf_test->cmd_handler, in pci_epf_test_link_up()
872 cancel_delayed_work_sync(&epf_test->cmd_handler); in pci_epf_test_link_down()
887 struct device *dev = &epf->dev; in pci_epf_test_alloc_space()
892 enum pci_barno test_reg_bar = epf_test->test_reg_bar; in pci_epf_test_alloc_space()
894 const struct pci_epc_features *epc_features = epf_test->epc_features; in pci_epf_test_alloc_space()
899 if (epc_features->msix_capable) { in pci_epf_test_alloc_space()
900 msix_table_size = PCI_MSIX_ENTRY_SIZE * epf->msix_interrupts; in pci_epf_test_alloc_space()
901 epf_test->msix_table_offset = test_reg_bar_size; in pci_epf_test_alloc_space()
902 /* Align to QWORD or 8 Bytes */ in pci_epf_test_alloc_space()
903 pba_size = ALIGN(DIV_ROUND_UP(epf->msix_interrupts, 8), 8); in pci_epf_test_alloc_space()
910 dev_err(dev, "Failed to allocated register space\n"); in pci_epf_test_alloc_space()
911 return -ENOMEM; in pci_epf_test_alloc_space()
913 epf_test->reg[test_reg_bar] = base; in pci_epf_test_alloc_space()
926 dev_err(dev, "Failed to allocate space for BAR%d\n", in pci_epf_test_alloc_space()
928 epf_test->reg[bar] = base; in pci_epf_test_alloc_space()
940 if (!epf_test->reg[bar]) in pci_epf_test_free_space()
943 pci_epf_free_space(epf, epf_test->reg[bar], bar, in pci_epf_test_free_space()
945 epf_test->reg[bar] = NULL; in pci_epf_test_free_space()
955 struct pci_epc *epc = epf->epc; in pci_epf_test_bind()
958 return -EINVAL; in pci_epf_test_bind()
960 epc_features = pci_epc_get_features(epc, epf->func_no, epf->vfunc_no); in pci_epf_test_bind()
962 dev_err(&epf->dev, "epc_features not implemented\n"); in pci_epf_test_bind()
963 return -EOPNOTSUPP; in pci_epf_test_bind()
968 return -EINVAL; in pci_epf_test_bind()
970 epf_test->test_reg_bar = test_reg_bar; in pci_epf_test_bind()
971 epf_test->epc_features = epc_features; in pci_epf_test_bind()
983 struct pci_epc *epc = epf->epc; in pci_epf_test_unbind()
985 cancel_delayed_work_sync(&epf_test->cmd_handler); in pci_epf_test_unbind()
986 if (epc->init_complete) { in pci_epf_test_unbind()
1004 struct device *dev = &epf->dev; in pci_epf_test_probe()
1008 return -ENOMEM; in pci_epf_test_probe()
1010 epf->header = &test_header; in pci_epf_test_probe()
1011 epf_test->epf = epf; in pci_epf_test_probe()
1013 INIT_DELAYED_WORK(&epf_test->cmd_handler, pci_epf_test_cmd_handler); in pci_epf_test_probe()
1015 epf->event_ops = &pci_epf_test_event_ops; in pci_epf_test_probe()
1041 pr_err("Failed to allocate the kpcitest work queue\n"); in pci_epf_test_init()
1042 return -ENOMEM; in pci_epf_test_init()
1048 pr_err("Failed to register pci epf test driver --> %d\n", ret); in pci_epf_test_init()