1 // SPDX-License-Identifier: GPL-2.0-or-later
3  * Virtio PCI driver - common functionality for all device versions
5  * This module allows virtio devices to be used over a virtual PCI device.
24 		 "Force legacy mode for transitional virtio 1 devices");
33 	if (vp_dev->intx_enabled)  in vp_synchronize_vectors()
34 		synchronize_irq(vp_dev->pci_dev->irq);  in vp_synchronize_vectors()
36 	for (i = 0; i < vp_dev->msix_vectors; ++i)  in vp_synchronize_vectors()
37 		synchronize_irq(pci_irq_vector(vp_dev->pci_dev, i));  in vp_synchronize_vectors()
45 	iowrite16(vq->index, (void __iomem *)vq->priv);  in vp_notify()
54 	virtio_config_changed(&vp_dev->vdev);  in vp_config_changed()
66 	spin_lock_irqsave(&vp_dev->lock, flags);  in vp_vring_interrupt()
67 	list_for_each_entry(info, &vp_dev->virtqueues, node) {  in vp_vring_interrupt()
68 		if (vring_interrupt(irq, info->vq) == IRQ_HANDLED)  in vp_vring_interrupt()
71 	spin_unlock_irqrestore(&vp_dev->lock, flags);  in vp_vring_interrupt()
89 	isr = ioread8(vp_dev->isr);  in vp_interrupt()
106 	const char *name = dev_name(&vp_dev->vdev.dev);  in vp_request_msix_vectors()
109 	int err = -ENOMEM;  in vp_request_msix_vectors()
111 	vp_dev->msix_vectors = nvectors;  in vp_request_msix_vectors()
113 	vp_dev->msix_names = kmalloc_array(nvectors,  in vp_request_msix_vectors()
114 					   sizeof(*vp_dev->msix_names),  in vp_request_msix_vectors()
116 	if (!vp_dev->msix_names)  in vp_request_msix_vectors()
118 	vp_dev->msix_affinity_masks  in vp_request_msix_vectors()
119 		= kcalloc(nvectors, sizeof(*vp_dev->msix_affinity_masks),  in vp_request_msix_vectors()
121 	if (!vp_dev->msix_affinity_masks)  in vp_request_msix_vectors()
124 		if (!alloc_cpumask_var(&vp_dev->msix_affinity_masks[i],  in vp_request_msix_vectors()
130 		desc->pre_vectors++; /* virtio config vector */  in vp_request_msix_vectors()
133 	err = pci_alloc_irq_vectors_affinity(vp_dev->pci_dev, nvectors,  in vp_request_msix_vectors()
137 	vp_dev->msix_enabled = 1;  in vp_request_msix_vectors()
140 	v = vp_dev->msix_used_vectors;  in vp_request_msix_vectors()
141 	snprintf(vp_dev->msix_names[v], sizeof *vp_dev->msix_names,  in vp_request_msix_vectors()
142 		 "%s-config", name);  in vp_request_msix_vectors()
143 	err = request_irq(pci_irq_vector(vp_dev->pci_dev, v),  in vp_request_msix_vectors()
144 			  vp_config_changed, 0, vp_dev->msix_names[v],  in vp_request_msix_vectors()
148 	++vp_dev->msix_used_vectors;  in vp_request_msix_vectors()
150 	v = vp_dev->config_vector(vp_dev, v);  in vp_request_msix_vectors()
153 		err = -EBUSY;  in vp_request_msix_vectors()
159 		v = vp_dev->msix_used_vectors;  in vp_request_msix_vectors()
160 		snprintf(vp_dev->msix_names[v], sizeof *vp_dev->msix_names,  in vp_request_msix_vectors()
161 			 "%s-virtqueues", name);  in vp_request_msix_vectors()
162 		err = request_irq(pci_irq_vector(vp_dev->pci_dev, v),  in vp_request_msix_vectors()
163 				  vp_vring_interrupt, 0, vp_dev->msix_names[v],  in vp_request_msix_vectors()
167 		++vp_dev->msix_used_vectors;  in vp_request_msix_vectors()
187 		return ERR_PTR(-ENOMEM);  in vp_setup_vq()
189 	vq = vp_dev->setup_vq(vp_dev, info, index, callback, name, ctx,  in vp_setup_vq()
194 	info->vq = vq;  in vp_setup_vq()
196 		spin_lock_irqsave(&vp_dev->lock, flags);  in vp_setup_vq()
197 		list_add(&info->node, &vp_dev->virtqueues);  in vp_setup_vq()
198 		spin_unlock_irqrestore(&vp_dev->lock, flags);  in vp_setup_vq()
200 		INIT_LIST_HEAD(&info->node);  in vp_setup_vq()
203 	vp_dev->vqs[index] = info;  in vp_setup_vq()
213 	struct virtio_pci_device *vp_dev = to_vp_device(vq->vdev);  in vp_del_vq()
214 	struct virtio_pci_vq_info *info = vp_dev->vqs[vq->index];  in vp_del_vq()
217 	spin_lock_irqsave(&vp_dev->lock, flags);  in vp_del_vq()
218 	list_del(&info->node);  in vp_del_vq()
219 	spin_unlock_irqrestore(&vp_dev->lock, flags);  in vp_del_vq()
221 	vp_dev->del_vq(info);  in vp_del_vq()
225 /* the config->del_vqs() implementation */
232 	list_for_each_entry_safe(vq, n, &vdev->vqs, list) {  in vp_del_vqs()
233 		if (vp_dev->per_vq_vectors) {  in vp_del_vqs()
234 			int v = vp_dev->vqs[vq->index]->msix_vector;  in vp_del_vqs()
237 				int irq = pci_irq_vector(vp_dev->pci_dev, v);  in vp_del_vqs()
245 	vp_dev->per_vq_vectors = false;  in vp_del_vqs()
247 	if (vp_dev->intx_enabled) {  in vp_del_vqs()
248 		free_irq(vp_dev->pci_dev->irq, vp_dev);  in vp_del_vqs()
249 		vp_dev->intx_enabled = 0;  in vp_del_vqs()
252 	for (i = 0; i < vp_dev->msix_used_vectors; ++i)  in vp_del_vqs()
253 		free_irq(pci_irq_vector(vp_dev->pci_dev, i), vp_dev);  in vp_del_vqs()
255 	if (vp_dev->msix_affinity_masks) {  in vp_del_vqs()
256 		for (i = 0; i < vp_dev->msix_vectors; i++)  in vp_del_vqs()
257 			if (vp_dev->msix_affinity_masks[i])  in vp_del_vqs()
258 				free_cpumask_var(vp_dev->msix_affinity_masks[i]);  in vp_del_vqs()
261 	if (vp_dev->msix_enabled) {  in vp_del_vqs()
263 		vp_dev->config_vector(vp_dev, VIRTIO_MSI_NO_VECTOR);  in vp_del_vqs()
265 		pci_free_irq_vectors(vp_dev->pci_dev);  in vp_del_vqs()
266 		vp_dev->msix_enabled = 0;  in vp_del_vqs()
269 	vp_dev->msix_vectors = 0;  in vp_del_vqs()
270 	vp_dev->msix_used_vectors = 0;  in vp_del_vqs()
271 	kfree(vp_dev->msix_names);  in vp_del_vqs()
272 	vp_dev->msix_names = NULL;  in vp_del_vqs()
273 	kfree(vp_dev->msix_affinity_masks);  in vp_del_vqs()
274 	vp_dev->msix_affinity_masks = NULL;  in vp_del_vqs()
275 	kfree(vp_dev->vqs);  in vp_del_vqs()
276 	vp_dev->vqs = NULL;  in vp_del_vqs()
289 	vp_dev->vqs = kcalloc(nvqs, sizeof(*vp_dev->vqs), GFP_KERNEL);  in vp_find_vqs_msix()
290 	if (!vp_dev->vqs)  in vp_find_vqs_msix()
291 		return -ENOMEM;  in vp_find_vqs_msix()
309 	vp_dev->per_vq_vectors = per_vq_vectors;  in vp_find_vqs_msix()
310 	allocated_vectors = vp_dev->msix_used_vectors;  in vp_find_vqs_msix()
319 		else if (vp_dev->per_vq_vectors)  in vp_find_vqs_msix()
331 		if (!vp_dev->per_vq_vectors || msix_vec == VIRTIO_MSI_NO_VECTOR)  in vp_find_vqs_msix()
334 		/* allocate per-vq irq if available and necessary */  in vp_find_vqs_msix()
335 		snprintf(vp_dev->msix_names[msix_vec],  in vp_find_vqs_msix()
336 			 sizeof *vp_dev->msix_names,  in vp_find_vqs_msix()
337 			 "%s-%s",  in vp_find_vqs_msix()
338 			 dev_name(&vp_dev->vdev.dev), names[i]);  in vp_find_vqs_msix()
339 		err = request_irq(pci_irq_vector(vp_dev->pci_dev, msix_vec),  in vp_find_vqs_msix()
341 				  vp_dev->msix_names[msix_vec],  in vp_find_vqs_msix()
360 	vp_dev->vqs = kcalloc(nvqs, sizeof(*vp_dev->vqs), GFP_KERNEL);  in vp_find_vqs_intx()
361 	if (!vp_dev->vqs)  in vp_find_vqs_intx()
362 		return -ENOMEM;  in vp_find_vqs_intx()
364 	err = request_irq(vp_dev->pci_dev->irq, vp_interrupt, IRQF_SHARED,  in vp_find_vqs_intx()
365 			dev_name(&vdev->dev), vp_dev);  in vp_find_vqs_intx()
369 	vp_dev->intx_enabled = 1;  in vp_find_vqs_intx()
370 	vp_dev->per_vq_vectors = false;  in vp_find_vqs_intx()
391 /* the config->find_vqs() implementation */
399 	/* Try MSI-X with one vector per queue. */  in vp_find_vqs()
403 	/* Fallback: MSI-X with one vector for config, one shared for queues. */  in vp_find_vqs()
415 	return pci_name(vp_dev->pci_dev);  in vp_bus_name()
419  * - force the affinity for per vq vector
420  * - OR over all affinities for shared MSI
421  * - ignore the affinity request if we're using INTX
425 	struct virtio_device *vdev = vq->vdev;  in vp_set_vq_affinity()
427 	struct virtio_pci_vq_info *info = vp_dev->vqs[vq->index];  in vp_set_vq_affinity()
431 	if (!vq->callback)  in vp_set_vq_affinity()
432 		return -EINVAL;  in vp_set_vq_affinity()
434 	if (vp_dev->msix_enabled) {  in vp_set_vq_affinity()
435 		mask = vp_dev->msix_affinity_masks[info->msix_vector];  in vp_set_vq_affinity()
436 		irq = pci_irq_vector(vp_dev->pci_dev, info->msix_vector);  in vp_set_vq_affinity()
451 	if (!vp_dev->per_vq_vectors ||  in vp_get_vq_affinity()
452 	    vp_dev->vqs[index]->msix_vector == VIRTIO_MSI_NO_VECTOR)  in vp_get_vq_affinity()
455 	return pci_irq_get_affinity(vp_dev->pci_dev,  in vp_get_vq_affinity()
456 				    vp_dev->vqs[index]->msix_vector);  in vp_get_vq_affinity()
466 	ret = virtio_device_freeze(&vp_dev->vdev);  in virtio_pci_freeze()
484 	return virtio_device_restore(&vp_dev->vdev);  in virtio_pci_restore()
499 MODULE_DEVICE_TABLE(pci, virtio_pci_id_table);
521 		return -ENOMEM;  in virtio_pci_probe()
524 	vp_dev->vdev.dev.parent = &pci_dev->dev;  in virtio_pci_probe()
525 	vp_dev->vdev.dev.release = virtio_pci_release_dev;  in virtio_pci_probe()
526 	vp_dev->pci_dev = pci_dev;  in virtio_pci_probe()
527 	INIT_LIST_HEAD(&vp_dev->virtqueues);  in virtio_pci_probe()
528 	spin_lock_init(&vp_dev->lock);  in virtio_pci_probe()
538 		if (rc == -ENODEV || rc == -ENOMEM)  in virtio_pci_probe()
544 		if (rc == -ENODEV)  in virtio_pci_probe()
552 	rc = register_virtio_device(&vp_dev->vdev);  in virtio_pci_probe()
560 	if (vp_dev->ioaddr)  in virtio_pci_probe()
568 		put_device(&vp_dev->vdev.dev);  in virtio_pci_probe()
577 	struct device *dev = get_device(&vp_dev->vdev.dev);  in virtio_pci_remove()
581 	unregister_virtio_device(&vp_dev->vdev);  in virtio_pci_remove()
583 	if (vp_dev->ioaddr)  in virtio_pci_remove()
595 	struct virtio_device *vdev = &vp_dev->vdev;  in virtio_pci_sriov_configure()
598 	if (!(vdev->config->get_status(vdev) & VIRTIO_CONFIG_S_DRIVER_OK))  in virtio_pci_sriov_configure()
599 		return -EBUSY;  in virtio_pci_sriov_configure()
602 		return -EINVAL;  in virtio_pci_sriov_configure()
605 		return -EPERM;  in virtio_pci_sriov_configure()
620 	.name		= "virtio-pci",
633 MODULE_DESCRIPTION("virtio-pci");