1 // SPDX-License-Identifier: GPL-2.0-or-later
3  * Virtio PCI driver - modern (virtio 1.0) device support
5  * This module allows virtio devices to be used over a virtual PCI device.
23  * Type-safe wrappers for io accesses.
27  * method, i.e. 32-bit accesses for 32-bit fields, 16-bit accesses
28  * for 16-bit fields and 8-bit accesses for 8-bit fields.
85 		dev_err(&dev->dev,  in map_capability()
91 	if (length - start < minlen) {  in map_capability()
92 		dev_err(&dev->dev,  in map_capability()
98 	length -= start;  in map_capability()
101 		dev_err(&dev->dev,  in map_capability()
102 			"virtio_pci: map wrap-around %u+%u\n",  in map_capability()
109 	if (offset & (align - 1)) {  in map_capability()
110 		dev_err(&dev->dev,  in map_capability()
124 		dev_err(&dev->dev,  in map_capability()
125 			"virtio_pci: map virtio %zu@%u "  in map_capability()
134 		dev_err(&dev->dev,  in map_capability()
135 			"virtio_pci: unable to map virtio %u@%u on bar %i\n",  in map_capability()
140 /* virtio config->get_features() implementation */
146 	vp_iowrite32(0, &vp_dev->common->device_feature_select);  in vp_get_features()
147 	features = vp_ioread32(&vp_dev->common->device_feature);  in vp_get_features()
148 	vp_iowrite32(1, &vp_dev->common->device_feature_select);  in vp_get_features()
149 	features |= ((u64)vp_ioread32(&vp_dev->common->device_feature) << 32);  in vp_get_features()
157 	struct pci_dev *pci_dev = vp_dev->pci_dev;  in vp_transport_features()
164 /* virtio config->finalize_features() implementation */
168 	u64 features = vdev->features;  in vp_finalize_features()
177 		dev_err(&vdev->dev, "virtio: device uses modern interface "  in vp_finalize_features()
179 		return -EINVAL;  in vp_finalize_features()
182 	vp_iowrite32(0, &vp_dev->common->guest_feature_select);  in vp_finalize_features()
183 	vp_iowrite32((u32)vdev->features, &vp_dev->common->guest_feature);  in vp_finalize_features()
184 	vp_iowrite32(1, &vp_dev->common->guest_feature_select);  in vp_finalize_features()
185 	vp_iowrite32(vdev->features >> 32, &vp_dev->common->guest_feature);  in vp_finalize_features()
190 /* virtio config->get() implementation */
199 	BUG_ON(offset + len > vp_dev->device_len);  in vp_get()
203 		b = ioread8(vp_dev->device + offset);  in vp_get()
207 		w = cpu_to_le16(ioread16(vp_dev->device + offset));  in vp_get()
211 		l = cpu_to_le32(ioread32(vp_dev->device + offset));  in vp_get()
215 		l = cpu_to_le32(ioread32(vp_dev->device + offset));  in vp_get()
217 		l = cpu_to_le32(ioread32(vp_dev->device + offset + sizeof l));  in vp_get()
225 /* the config->set() implementation.  it's symmetric to the config->get()
235 	BUG_ON(offset + len > vp_dev->device_len);  in vp_set()
240 		iowrite8(b, vp_dev->device + offset);  in vp_set()
244 		iowrite16(le16_to_cpu(w), vp_dev->device + offset);  in vp_set()
248 		iowrite32(le32_to_cpu(l), vp_dev->device + offset);  in vp_set()
252 		iowrite32(le32_to_cpu(l), vp_dev->device + offset);  in vp_set()
254 		iowrite32(le32_to_cpu(l), vp_dev->device + offset + sizeof l);  in vp_set()
264 	return vp_ioread8(&vp_dev->common->config_generation);  in vp_generation()
267 /* config->{get,set}_status() implementations */
271 	return vp_ioread8(&vp_dev->common->device_status);  in vp_get_status()
279 	vp_iowrite8(status, &vp_dev->common->device_status);  in vp_set_status()
286 	vp_iowrite8(0, &vp_dev->common->device_status);  in vp_reset()
290 	 * including MSI-X interrupts, if any.  in vp_reset()
292 	while (vp_ioread8(&vp_dev->common->device_status))  in vp_reset()
301 	vp_iowrite16(vector, &vp_dev->common->msix_config);  in vp_config_vector()
304 	return vp_ioread16(&vp_dev->common->msix_config);  in vp_config_vector()
315 	struct virtio_pci_common_cfg __iomem *cfg = vp_dev->common;  in setup_vq()
320 	if (index >= vp_ioread16(&cfg->num_queues))  in setup_vq()
321 		return ERR_PTR(-ENOENT);  in setup_vq()
324 	vp_iowrite16(index, &cfg->queue_select);  in setup_vq()
327 	num = vp_ioread16(&cfg->queue_size);  in setup_vq()
328 	if (!num || vp_ioread16(&cfg->queue_enable))  in setup_vq()
329 		return ERR_PTR(-ENOENT);  in setup_vq()
331 	if (num & (num - 1)) {  in setup_vq()
332 		dev_warn(&vp_dev->pci_dev->dev, "bad queue size %u", num);  in setup_vq()
333 		return ERR_PTR(-EINVAL);  in setup_vq()
337 	off = vp_ioread16(&cfg->queue_notify_off);  in setup_vq()
339 	info->msix_vector = msix_vec;  in setup_vq()
343 				    SMP_CACHE_BYTES, &vp_dev->vdev,  in setup_vq()
347 		return ERR_PTR(-ENOMEM);  in setup_vq()
350 	vp_iowrite16(virtqueue_get_vring_size(vq), &cfg->queue_size);  in setup_vq()
352 			     &cfg->queue_desc_lo, &cfg->queue_desc_hi);  in setup_vq()
354 			     &cfg->queue_avail_lo, &cfg->queue_avail_hi);  in setup_vq()
356 			     &cfg->queue_used_lo, &cfg->queue_used_hi);  in setup_vq()
358 	if (vp_dev->notify_base) {  in setup_vq()
360 		if ((u64)off * vp_dev->notify_offset_multiplier + 2  in setup_vq()
361 		    > vp_dev->notify_len) {  in setup_vq()
362 			dev_warn(&vp_dev->pci_dev->dev,  in setup_vq()
365 				 off, vp_dev->notify_offset_multiplier,  in setup_vq()
366 				 index, vp_dev->notify_len);  in setup_vq()
367 			err = -EINVAL;  in setup_vq()
370 		vq->priv = (void __force *)vp_dev->notify_base +  in setup_vq()
371 			off * vp_dev->notify_offset_multiplier;  in setup_vq()
373 		vq->priv = (void __force *)map_capability(vp_dev->pci_dev,  in setup_vq()
374 					  vp_dev->notify_map_cap, 2, 2,  in setup_vq()
375 					  off * vp_dev->notify_offset_multiplier, 2,  in setup_vq()
379 	if (!vq->priv) {  in setup_vq()
380 		err = -ENOMEM;  in setup_vq()
385 		vp_iowrite16(msix_vec, &cfg->queue_msix_vector);  in setup_vq()
386 		msix_vec = vp_ioread16(&cfg->queue_msix_vector);  in setup_vq()
388 			err = -EBUSY;  in setup_vq()
396 	if (!vp_dev->notify_base)  in setup_vq()
397 		pci_iounmap(vp_dev->pci_dev, (void __iomem __force *)vq->priv);  in setup_vq()
419 	list_for_each_entry(vq, &vdev->vqs, list) {  in vp_modern_find_vqs()
420 		vp_iowrite16(vq->index, &vp_dev->common->queue_select);  in vp_modern_find_vqs()
421 		vp_iowrite16(1, &vp_dev->common->queue_enable);  in vp_modern_find_vqs()
429 	struct virtqueue *vq = info->vq;  in del_vq()
430 	struct virtio_pci_device *vp_dev = to_vp_device(vq->vdev);  in del_vq()
432 	vp_iowrite16(vq->index, &vp_dev->common->queue_select);  in del_vq()
434 	if (vp_dev->msix_enabled) {  in del_vq()
436 			     &vp_dev->common->queue_msix_vector);  in del_vq()
438 		vp_ioread16(&vp_dev->common->queue_msix_vector);  in del_vq()
441 	if (!vp_dev->notify_base)  in del_vq()
442 		pci_iounmap(vp_dev->pci_dev, (void __force __iomem *)vq->priv);  in del_vq()
466 			dev_err(&dev->dev, "%s: shm cap with bad size offset:"  in virtio_pci_find_shm_cap()
510 	struct pci_dev *pci_dev = vp_dev->pci_dev;  in vp_get_shm_region()
523 		dev_err(&pci_dev->dev, "%s: cap offset+len overflow detected\n",  in vp_get_shm_region()
529 		dev_err(&pci_dev->dev, "%s: bar shorter than cap offset+len\n",  in vp_get_shm_region()
534 	region->len = len;  in vp_get_shm_region()
535 	region->addr = (u64) phys_addr + offset;  in vp_get_shm_region()
575  * virtio_pci_find_capability - walk capabilities to find device info.
576  * @dev: the pci device
677 /* the PCI probing function */
680 	struct pci_dev *pci_dev = vp_dev->pci_dev;  in virtio_pci_modern_probe()
688 	if (pci_dev->device < 0x1000 || pci_dev->device > 0x107f)  in virtio_pci_modern_probe()
689 		return -ENODEV;  in virtio_pci_modern_probe()
691 	if (pci_dev->device < 0x1040) {  in virtio_pci_modern_probe()
692 		/* Transitional devices: use the PCI subsystem device id as  in virtio_pci_modern_probe()
693 		 * virtio device id, same as legacy driver always did.  in virtio_pci_modern_probe()
695 		vp_dev->vdev.id.device = pci_dev->subsystem_device;  in virtio_pci_modern_probe()
697 		/* Modern devices: simply use PCI device id, but start from 0x1040. */  in virtio_pci_modern_probe()
698 		vp_dev->vdev.id.device = pci_dev->device - 0x1040;  in virtio_pci_modern_probe()
700 	vp_dev->vdev.id.vendor = pci_dev->subsystem_vendor;  in virtio_pci_modern_probe()
705 					    &vp_dev->modern_bars);  in virtio_pci_modern_probe()
707 		dev_info(&pci_dev->dev,  in virtio_pci_modern_probe()
709 		return -ENODEV;  in virtio_pci_modern_probe()
715 					 &vp_dev->modern_bars);  in virtio_pci_modern_probe()
718 					    &vp_dev->modern_bars);  in virtio_pci_modern_probe()
720 		dev_err(&pci_dev->dev,  in virtio_pci_modern_probe()
723 		return -EINVAL;  in virtio_pci_modern_probe()
726 	err = dma_set_mask_and_coherent(&pci_dev->dev, DMA_BIT_MASK(64));  in virtio_pci_modern_probe()
728 		err = dma_set_mask_and_coherent(&pci_dev->dev,  in virtio_pci_modern_probe()
731 …dev_warn(&pci_dev->dev, "Failed to enable 64-bit or 32-bit DMA.  Trying to continue, but this migh…  in virtio_pci_modern_probe()
734 	 * device-specific configuration.  in virtio_pci_modern_probe()
738 					    &vp_dev->modern_bars);  in virtio_pci_modern_probe()
740 	err = pci_request_selected_regions(pci_dev, vp_dev->modern_bars,  in virtio_pci_modern_probe()
741 					   "virtio-pci-modern");  in virtio_pci_modern_probe()
745 	err = -EINVAL;  in virtio_pci_modern_probe()
746 	vp_dev->common = map_capability(pci_dev, common,  in virtio_pci_modern_probe()
750 	if (!vp_dev->common)  in virtio_pci_modern_probe()
752 	vp_dev->isr = map_capability(pci_dev, isr, sizeof(u8), 1,  in virtio_pci_modern_probe()
755 	if (!vp_dev->isr)  in virtio_pci_modern_probe()
762 			      &vp_dev->notify_offset_multiplier);  in virtio_pci_modern_probe()
779 		vp_dev->notify_base = map_capability(pci_dev, notify, 2, 2,  in virtio_pci_modern_probe()
781 						     &vp_dev->notify_len);  in virtio_pci_modern_probe()
782 		if (!vp_dev->notify_base)  in virtio_pci_modern_probe()
785 		vp_dev->notify_map_cap = notify;  in virtio_pci_modern_probe()
792 		vp_dev->device = map_capability(pci_dev, device, 0, 4,  in virtio_pci_modern_probe()
794 						&vp_dev->device_len);  in virtio_pci_modern_probe()
795 		if (!vp_dev->device)  in virtio_pci_modern_probe()
798 		vp_dev->vdev.config = &virtio_pci_config_ops;  in virtio_pci_modern_probe()
800 		vp_dev->vdev.config = &virtio_pci_config_nodev_ops;  in virtio_pci_modern_probe()
803 	vp_dev->config_vector = vp_config_vector;  in virtio_pci_modern_probe()
804 	vp_dev->setup_vq = setup_vq;  in virtio_pci_modern_probe()
805 	vp_dev->del_vq = del_vq;  in virtio_pci_modern_probe()
810 	if (vp_dev->notify_base)  in virtio_pci_modern_probe()
811 		pci_iounmap(pci_dev, vp_dev->notify_base);  in virtio_pci_modern_probe()
813 	pci_iounmap(pci_dev, vp_dev->isr);  in virtio_pci_modern_probe()
815 	pci_iounmap(pci_dev, vp_dev->common);  in virtio_pci_modern_probe()
822 	struct pci_dev *pci_dev = vp_dev->pci_dev;  in virtio_pci_modern_remove()
824 	if (vp_dev->device)  in virtio_pci_modern_remove()
825 		pci_iounmap(pci_dev, vp_dev->device);  in virtio_pci_modern_remove()
826 	if (vp_dev->notify_base)  in virtio_pci_modern_remove()
827 		pci_iounmap(pci_dev, vp_dev->notify_base);  in virtio_pci_modern_remove()
828 	pci_iounmap(pci_dev, vp_dev->isr);  in virtio_pci_modern_remove()
829 	pci_iounmap(pci_dev, vp_dev->common);  in virtio_pci_modern_remove()
830 	pci_release_selected_regions(pci_dev, vp_dev->modern_bars);  in virtio_pci_modern_remove()