1 // SPDX-License-Identifier: GPL-2.0-only
5 #include <linux/io-64-nonatomic-lo-hi.h>
44 	return oct_vdpa->oct_hw;  in vdpa_to_octep_hw()
54 	 * round-robin fashion.  in octep_vdpa_intr_handler()
57 	 * 0 -> 0, 8, 16, 24, 32, 40, 48, 56;  in octep_vdpa_intr_handler()
58 	 * 1 -> 1, 9, 17, 25, 33, 41, 49, 57;  in octep_vdpa_intr_handler()
60 	 * 7 -> 7, 15, 23, 31, 39, 47, 55, 63;  in octep_vdpa_intr_handler()
63 	for (i = irq - oct_hw->irqs[0]; i < oct_hw->nr_vring; i += oct_hw->nb_irqs) {  in octep_vdpa_intr_handler()
64 		if (ioread8(oct_hw->vqs[i].cb_notify_addr)) {  in octep_vdpa_intr_handler()
66 			iowrite8(0, oct_hw->vqs[i].cb_notify_addr);  in octep_vdpa_intr_handler()
68 			if (likely(oct_hw->vqs[i].cb.callback))  in octep_vdpa_intr_handler()
69 				oct_hw->vqs[i].cb.callback(oct_hw->vqs[i].cb.private);  in octep_vdpa_intr_handler()
75 	if (unlikely(irq == oct_hw->irqs[0] && ioread8(oct_hw->isr))) {  in octep_vdpa_intr_handler()
76 		iowrite8(0, oct_hw->isr);  in octep_vdpa_intr_handler()
78 		if (oct_hw->config_cb.callback)  in octep_vdpa_intr_handler()
79 			oct_hw->config_cb.callback(oct_hw->config_cb.private);  in octep_vdpa_intr_handler()
87 	struct pci_dev *pdev = oct_hw->pdev;  in octep_free_irqs()
90 	if (!oct_hw->irqs)  in octep_free_irqs()
93 	for (irq = 0; irq < oct_hw->nb_irqs; irq++) {  in octep_free_irqs()
94 		if (!oct_hw->irqs[irq])  in octep_free_irqs()
97 		devm_free_irq(&pdev->dev, oct_hw->irqs[irq], oct_hw);  in octep_free_irqs()
101 	devm_kfree(&pdev->dev, oct_hw->irqs);  in octep_free_irqs()
102 	oct_hw->irqs = NULL;  in octep_free_irqs()
107 	struct pci_dev *pdev = oct_hw->pdev;  in octep_request_irqs()
110 	oct_hw->irqs = devm_kcalloc(&pdev->dev, oct_hw->nb_irqs, sizeof(int), GFP_KERNEL);  in octep_request_irqs()
111 	if (!oct_hw->irqs)  in octep_request_irqs()
112 		return -ENOMEM;  in octep_request_irqs()
114 	ret = pci_alloc_irq_vectors(pdev, 1, oct_hw->nb_irqs, PCI_IRQ_MSIX);  in octep_request_irqs()
116 		dev_err(&pdev->dev, "Failed to alloc msix vector");  in octep_request_irqs()
120 	for (idx = 0; idx < oct_hw->nb_irqs; idx++) {  in octep_request_irqs()
122 		ret = devm_request_irq(&pdev->dev, irq, octep_vdpa_intr_handler, 0,  in octep_request_irqs()
123 				       dev_name(&pdev->dev), oct_hw);  in octep_request_irqs()
125 			dev_err(&pdev->dev, "Failed to register interrupt handler\n");  in octep_request_irqs()
128 		oct_hw->irqs[idx] = irq;  in octep_request_irqs()
142 	return oct_hw->features;  in octep_vdpa_get_device_features()
154 		dev_warn(&oct_hw->pdev->dev,  in octep_vdpa_set_driver_features()
206 	for (qid = 0; qid < oct_hw->nr_vring; qid++) {  in octep_vdpa_reset()
207 		oct_hw->vqs[qid].cb.callback = NULL;  in octep_vdpa_reset()
208 		oct_hw->vqs[qid].cb.private = NULL;  in octep_vdpa_reset()
209 		oct_hw->config_cb.callback = NULL;  in octep_vdpa_reset()
210 		oct_hw->config_cb.private = NULL;  in octep_vdpa_reset()
247 	oct_hw->vqs[qid].cb = *cb;  in octep_vdpa_set_vq_cb()
264 static void octep_vdpa_set_vq_num(struct vdpa_device *vdpa_dev, u16 qid, u32 num)  in octep_vdpa_set_vq_num()  argument
268 	octep_set_vq_num(oct_hw, qid, num);  in octep_vdpa_set_vq_num()
293 	vp_iowrite32(data, oct_hw->vqs[idx].notify_addr);  in octep_vdpa_kick_vq_with_data()
300 	return vp_ioread8(&oct_hw->common_cfg->config_generation);  in octep_vdpa_get_generation()
307 	return oct_hw->dev_id;  in octep_vdpa_get_device_id()
324 	return oct_hw->config_size;  in octep_vdpa_get_config_size()
345 	oct_hw->config_cb.callback = cb->callback;  in octep_vdpa_set_config_cb()
346 	oct_hw->config_cb.private = cb->private;  in octep_vdpa_set_config_cb()
355 	area.addr = oct_hw->vqs[idx].notify_pa;  in octep_get_vq_notification()
396 		dev_err(&pdev->dev, "Failed to request BAR:%u region\n", bar);  in octep_iomap_region()
402 		dev_err(&pdev->dev, "Failed to iomap BAR:%u\n", bar);  in octep_iomap_region()
404 		ret = -ENOMEM;  in octep_iomap_region()
418 	struct pci_dev *pf_dev = octpf->pdev;  in octep_vdpa_pf_bar_shrink()
419 	struct resource *res = pf_dev->resource + PCI_STD_RESOURCES + 4;  in octep_vdpa_pf_bar_shrink()
422 	octpf->res.start = res->start;  in octep_vdpa_pf_bar_shrink()
423 	octpf->res.end = res->end;  in octep_vdpa_pf_bar_shrink()
424 	octpf->vf_base = res->start;  in octep_vdpa_pf_bar_shrink()
426 	bus_region.start = res->start;  in octep_vdpa_pf_bar_shrink()
427 	bus_region.end = res->start - 1;  in octep_vdpa_pf_bar_shrink()
429 	pcibios_bus_to_resource(pf_dev->bus, res, &bus_region);  in octep_vdpa_pf_bar_shrink()
434 	struct pci_dev *pf_dev = octpf->pdev;  in octep_vdpa_pf_bar_expand()
435 	struct resource *res = pf_dev->resource + PCI_STD_RESOURCES + 4;  in octep_vdpa_pf_bar_expand()
438 	bus_region.start = octpf->res.start;  in octep_vdpa_pf_bar_expand()
439 	bus_region.end = octpf->res.end;  in octep_vdpa_pf_bar_expand()
441 	pcibios_bus_to_resource(pf_dev->bus, res, &bus_region);  in octep_vdpa_pf_bar_expand()
450 	if (octpf->base[OCTEP_HW_CAPS_BAR])  in octep_vdpa_remove_pf()
451 		octep_iounmap_region(pdev, octpf->base, OCTEP_HW_CAPS_BAR);  in octep_vdpa_remove_pf()
453 	if (octpf->base[OCTEP_HW_MBOX_BAR])  in octep_vdpa_remove_pf()
454 		octep_iounmap_region(pdev, octpf->base, OCTEP_HW_MBOX_BAR);  in octep_vdpa_remove_pf()
461 	struct resource *vf_res = pdev->resource + PCI_STD_RESOURCES + 4;  in octep_vdpa_vf_bar_shrink()
472 	oct_hw = &mgmt_dev->oct_hw;  in octep_vdpa_remove_vf()
473 	status = atomic_read(&mgmt_dev->status);  in octep_vdpa_remove_vf()
474 	atomic_set(&mgmt_dev->status, OCTEP_VDPA_DEV_STATUS_UNINIT);  in octep_vdpa_remove_vf()
476 	cancel_work_sync(&mgmt_dev->setup_task);  in octep_vdpa_remove_vf()
478 		vdpa_mgmtdev_unregister(&mgmt_dev->mdev);  in octep_vdpa_remove_vf()
480 	if (oct_hw->base[OCTEP_HW_CAPS_BAR])  in octep_vdpa_remove_vf()
481 		octep_iounmap_region(pdev, oct_hw->base, OCTEP_HW_CAPS_BAR);  in octep_vdpa_remove_vf()
483 	if (oct_hw->base[OCTEP_HW_MBOX_BAR])  in octep_vdpa_remove_vf()
484 		octep_iounmap_region(pdev, oct_hw->base, OCTEP_HW_MBOX_BAR);  in octep_vdpa_remove_vf()
491 	if (pdev->is_virtfn)  in octep_vdpa_remove()
501 	struct octep_hw *oct_hw = &mgmt_dev->oct_hw;  in octep_vdpa_dev_add()
502 	struct pci_dev *pdev = oct_hw->pdev;  in octep_vdpa_dev_add()
508 	oct_vdpa = vdpa_alloc_device(struct octep_vdpa, vdpa, &pdev->dev, &octep_vdpa_ops, 1, 1,  in octep_vdpa_dev_add()
511 		dev_err(&pdev->dev, "Failed to allocate vDPA structure for octep vdpa device");  in octep_vdpa_dev_add()
515 	oct_vdpa->pdev = pdev;  in octep_vdpa_dev_add()
516 	oct_vdpa->vdpa.dma_dev = &pdev->dev;  in octep_vdpa_dev_add()
517 	oct_vdpa->vdpa.mdev = mdev;  in octep_vdpa_dev_add()
518 	oct_vdpa->oct_hw = oct_hw;  in octep_vdpa_dev_add()
519 	vdpa_dev = &oct_vdpa->vdpa;  in octep_vdpa_dev_add()
521 	device_features = oct_hw->features;  in octep_vdpa_dev_add()
522 	if (config->mask & BIT_ULL(VDPA_ATTR_DEV_FEATURES)) {  in octep_vdpa_dev_add()
523 		if (config->device_features & ~device_features) {  in octep_vdpa_dev_add()
524 …dev_err(&pdev->dev, "The provisioned features 0x%llx are not supported by this device with feature…  in octep_vdpa_dev_add()
525 				config->device_features, device_features);  in octep_vdpa_dev_add()
526 			ret = -EINVAL;  in octep_vdpa_dev_add()
529 		device_features &= config->device_features;  in octep_vdpa_dev_add()
532 	oct_hw->features = device_features;  in octep_vdpa_dev_add()
533 	dev_info(&pdev->dev, "Vdpa management device features : %llx\n", device_features);  in octep_vdpa_dev_add()
537 		dev_warn(mdev->device,  in octep_vdpa_dev_add()
544 		ret = dev_set_name(&vdpa_dev->dev, "%s", name);  in octep_vdpa_dev_add()
546 		ret = dev_set_name(&vdpa_dev->dev, "vdpa%u", vdpa_dev->index);  in octep_vdpa_dev_add()
548 	ret = _vdpa_register_device(&oct_vdpa->vdpa, oct_hw->nr_vring);  in octep_vdpa_dev_add()
550 		dev_err(&pdev->dev, "Failed to register to vDPA bus");  in octep_vdpa_dev_add()
556 	put_device(&oct_vdpa->vdpa.dev);  in octep_vdpa_dev_add()
591 	struct pci_dev *pdev = mgmt_dev->pdev;  in octep_vdpa_setup_task()
592 	struct device *dev = &pdev->dev;  in octep_vdpa_setup_task()
598 	oct_hw = &mgmt_dev->oct_hw;  in octep_vdpa_setup_task()
600 	atomic_set(&mgmt_dev->status, OCTEP_VDPA_DEV_STATUS_WAIT_FOR_BAR_INIT);  in octep_vdpa_setup_task()
605 		if (get_device_ready_status(oct_hw->base[OCTEP_HW_MBOX_BAR])) {  in octep_vdpa_setup_task()
606 			atomic_set(&mgmt_dev->status, OCTEP_VDPA_DEV_STATUS_INIT);  in octep_vdpa_setup_task()
610 		if (atomic_read(&mgmt_dev->status) >= OCTEP_VDPA_DEV_STATUS_READY) {  in octep_vdpa_setup_task()
618 	if (atomic_read(&mgmt_dev->status) != OCTEP_VDPA_DEV_STATUS_INIT) {  in octep_vdpa_setup_task()
623 	ret = octep_iomap_region(pdev, oct_hw->base, OCTEP_HW_CAPS_BAR);  in octep_vdpa_setup_task()
627 	val = readq(oct_hw->base[OCTEP_HW_MBOX_BAR] + OCTEP_VF_IN_CTRL(0));  in octep_vdpa_setup_task()
628 	oct_hw->nb_irqs = OCTEP_VF_IN_CTRL_RPVF(val);  in octep_vdpa_setup_task()
629 	if (!oct_hw->nb_irqs || oct_hw->nb_irqs > OCTEP_MAX_CB_INTR) {  in octep_vdpa_setup_task()
630 		dev_err(dev, "Invalid number of interrupts %d\n", oct_hw->nb_irqs);  in octep_vdpa_setup_task()
638 	mgmt_dev->mdev.ops = &octep_vdpa_mgmt_dev_ops;  in octep_vdpa_setup_task()
639 	mgmt_dev->mdev.id_table = id_table;  in octep_vdpa_setup_task()
640 	mgmt_dev->mdev.max_supported_vqs = oct_hw->nr_vring;  in octep_vdpa_setup_task()
641 	mgmt_dev->mdev.supported_features = oct_hw->features;  in octep_vdpa_setup_task()
642 	mgmt_dev->mdev.config_attr_mask = (1 << VDPA_ATTR_DEV_FEATURES);  in octep_vdpa_setup_task()
643 	mgmt_dev->mdev.device = dev;  in octep_vdpa_setup_task()
645 	ret = vdpa_mgmtdev_register(&mgmt_dev->mdev);  in octep_vdpa_setup_task()
651 	atomic_set(&mgmt_dev->status, OCTEP_VDPA_DEV_STATUS_READY);  in octep_vdpa_setup_task()
656 	octep_iounmap_region(pdev, oct_hw->base, OCTEP_HW_CAPS_BAR);  in octep_vdpa_setup_task()
657 	oct_hw->base[OCTEP_HW_CAPS_BAR] = NULL;  in octep_vdpa_setup_task()
663 	struct device *dev = &pdev->dev;  in octep_vdpa_probe_vf()
681 		return -ENOMEM;  in octep_vdpa_probe_vf()
683 	ret = octep_iomap_region(pdev, mgmt_dev->oct_hw.base, OCTEP_HW_MBOX_BAR);  in octep_vdpa_probe_vf()
687 	mgmt_dev->pdev = pdev;  in octep_vdpa_probe_vf()
690 	atomic_set(&mgmt_dev->status, OCTEP_VDPA_DEV_STATUS_ALLOC);  in octep_vdpa_probe_vf()
691 	INIT_WORK(&mgmt_dev->setup_task, octep_vdpa_setup_task);  in octep_vdpa_probe_vf()
692 	schedule_work(&mgmt_dev->setup_task);  in octep_vdpa_probe_vf()
693 	dev_info(&pdev->dev, "octep vdpa mgmt device setup task is queued\n");  in octep_vdpa_probe_vf()
700 	struct resource *vf_res = vf_dev->resource + PCI_STD_RESOURCES + 4;  in octep_vdpa_assign_barspace()
701 	struct resource *pf_res = pf_dev->resource + PCI_STD_RESOURCES + 4;  in octep_vdpa_assign_barspace()
705 	vf_res->name = pci_name(vf_dev);  in octep_vdpa_assign_barspace()
706 	vf_res->flags = pf_res->flags;  in octep_vdpa_assign_barspace()
707 	vf_res->parent = (pf_dev->resource + PCI_STD_RESOURCES)->parent;  in octep_vdpa_assign_barspace()
709 	bus_region.start = pf->vf_base + idx * pf->vf_stride;  in octep_vdpa_assign_barspace()
710 	bus_region.end = bus_region.start + pf->vf_stride - 1;  in octep_vdpa_assign_barspace()
711 	pcibios_bus_to_resource(vf_dev->bus, vf_res, &bus_region);  in octep_vdpa_assign_barspace()
717 	u8 __iomem *addr = pf->base[OCTEP_HW_MBOX_BAR];  in octep_sriov_enable()
727 	pf->enabled_vfs = num_vfs;  in octep_sriov_enable()
730 		if (vf_pdev->device != pf->vf_devid)  in octep_sriov_enable()
741 		for (i = 0; i < pf->enabled_vfs; i++)  in octep_sriov_enable()
756 	pf->enabled_vfs = 0;  in octep_sriov_disable()
773 	switch (pdev->device) {  in octep_get_vf_devid()
793 	u8 __iomem *addr = octpf->base[OCTEP_HW_MBOX_BAR];  in octep_vdpa_pf_setup()
794 	struct pci_dev *pdev = octpf->pdev;  in octep_vdpa_pf_setup()
801 		dev_info(&pdev->dev, "Total VFs are %d in PF sriov configuration\n", totalvfs);  in octep_vdpa_pf_setup()
805 	addr = octpf->base[OCTEP_HW_MBOX_BAR];  in octep_vdpa_pf_setup()
808 		dev_err(&pdev->dev, "Invalid device configuration\n");  in octep_vdpa_pf_setup()
809 		return -EINVAL;  in octep_vdpa_pf_setup()
814 	octpf->vf_stride = len / totalvfs;  in octep_vdpa_pf_setup()
815 	octpf->vf_devid = octep_get_vf_devid(pdev);  in octep_vdpa_pf_setup()
824 	struct device *dev = &pdev->dev;  in octep_vdpa_probe_pf()
841 		return -ENOMEM;  in octep_vdpa_probe_pf()
843 	ret = octep_iomap_region(pdev, octpf->base, OCTEP_HW_MBOX_BAR);  in octep_vdpa_probe_pf()
849 	octpf->pdev = pdev;  in octep_vdpa_probe_pf()
858 	octep_iounmap_region(pdev, octpf->base, OCTEP_HW_MBOX_BAR);  in octep_vdpa_probe_pf()
864 	if (pdev->is_virtfn)  in octep_vdpa_probe()