1416b2c2dSAsias He #include "kvm/virtio-blk.h" 2b30d05adSPekka Enberg 331638bcaSCyrill Gorcunov #include "kvm/virtio-pci-dev.h" 45a24a9f2SPekka Enberg #include "kvm/disk-image.h" 539d6af07SAsias He #include "kvm/virtio.h" 64ef0f4d6SPekka Enberg #include "kvm/mutex.h" 7fe99fd4eSPekka Enberg #include "kvm/util.h" 88b1ff07eSPekka Enberg #include "kvm/kvm.h" 9b30d05adSPekka Enberg #include "kvm/pci.h" 10fb0957f2SSasha Levin #include "kvm/threadpool.h" 11ec75b82fSSasha Levin #include "kvm/ioeventfd.h" 12404d164bSSasha Levin #include "kvm/guest_compat.h" 13427948d5SSasha Levin #include "kvm/virtio-pci.h" 14b30d05adSPekka Enberg 1520c64ecaSPekka Enberg #include <linux/virtio_ring.h> 1620c64ecaSPekka Enberg #include <linux/virtio_blk.h> 170528c2a7SPekka Enberg 18427948d5SSasha Levin #include <linux/kernel.h> 19ebe9ac19SSasha Levin #include <linux/list.h> 203fdf659dSSasha Levin #include <linux/types.h> 210528c2a7SPekka Enberg #include <pthread.h> 224155ba8cSPekka Enberg 234749e795SSasha Levin #define VIRTIO_BLK_MAX_DEV 4 2410eca11dSPekka Enberg #define NUM_VIRT_QUEUES 1 2510eca11dSPekka Enberg 2603110ff3SAsias He #define VIRTIO_BLK_QUEUE_SIZE 128 273d7831a1SAsias He /* 283d7831a1SAsias He * the header and status consume too entries 293d7831a1SAsias He */ 303d7831a1SAsias He #define DISK_SEG_MAX (VIRTIO_BLK_QUEUE_SIZE - 2) 3110eca11dSPekka Enberg 32fe2a70d1SSasha Levin struct blk_dev_job { 334749e795SSasha Levin struct virt_queue *vq; 34fe2a70d1SSasha Levin struct blk_dev *bdev; 3569971b13SSasha Levin struct iovec iov[VIRTIO_BLK_QUEUE_SIZE]; 3669971b13SSasha Levin u16 out, in, head; 37df0c7f57SSasha Levin struct thread_pool__job job_id; 384749e795SSasha Levin }; 394749e795SSasha Levin 40fe2a70d1SSasha Levin struct blk_dev { 410528c2a7SPekka Enberg pthread_mutex_t mutex; 42ebe9ac19SSasha Levin struct list_head list; 430528c2a7SPekka Enberg 44427948d5SSasha Levin struct virtio_pci vpci; 4540ce993fSPekka Enberg struct virtio_blk_config blk_config; 4638605e1cSSasha Levin struct disk_image *disk; 47427948d5SSasha Levin u32 features; 4810eca11dSPekka Enberg 4945e47970SAsias He struct virt_queue vqs[NUM_VIRT_QUEUES]; 5069971b13SSasha Levin struct blk_dev_job jobs[VIRTIO_BLK_QUEUE_SIZE]; 5169971b13SSasha Levin u16 job_idx; 52fbc2fbf9SPekka Enberg }; 53fbc2fbf9SPekka Enberg 54ebe9ac19SSasha Levin static LIST_HEAD(bdevs); 55312c62d1SSasha Levin static int compat_id; 5640ce993fSPekka Enberg 5769971b13SSasha Levin static void virtio_blk_do_io_request(struct kvm *kvm, void *param) 584155ba8cSPekka Enberg { 594155ba8cSPekka Enberg struct virtio_blk_outhdr *req; 603fdf659dSSasha Levin u8 *status; 6169971b13SSasha Levin ssize_t block_cnt; 6269971b13SSasha Levin struct blk_dev_job *job; 6369971b13SSasha Levin struct blk_dev *bdev; 6469971b13SSasha Levin struct virt_queue *queue; 6569971b13SSasha Levin struct iovec *iov; 6669971b13SSasha Levin u16 out, in, head; 674155ba8cSPekka Enberg 6869971b13SSasha Levin block_cnt = -1; 6969971b13SSasha Levin job = param; 7069971b13SSasha Levin bdev = job->bdev; 7169971b13SSasha Levin queue = job->vq; 7269971b13SSasha Levin iov = job->iov; 7369971b13SSasha Levin out = job->out; 7469971b13SSasha Levin in = job->in; 7569971b13SSasha Levin head = job->head; 7645e47970SAsias He req = iov[0].iov_base; 7703110ff3SAsias He 78258dd093SPekka Enberg switch (req->type) { 7903110ff3SAsias He case VIRTIO_BLK_T_IN: 80b8861977SAsias He block_cnt = disk_image__read(bdev->disk, req->sector, iov + 1, in + out - 2); 81258dd093SPekka Enberg break; 8203110ff3SAsias He case VIRTIO_BLK_T_OUT: 83b8861977SAsias He block_cnt = disk_image__write(bdev->disk, req->sector, iov + 1, in + out - 2); 84258dd093SPekka Enberg break; 8529084a74SPrasad Joshi case VIRTIO_BLK_T_FLUSH: 8629084a74SPrasad Joshi block_cnt = disk_image__flush(bdev->disk); 8729084a74SPrasad Joshi break; 88ff6462e8SSasha Levin case VIRTIO_BLK_T_GET_ID: 89ff6462e8SSasha Levin block_cnt = VIRTIO_BLK_ID_BYTES; 90ff6462e8SSasha Levin disk_image__get_serial(bdev->disk, (iov + 1)->iov_base, &block_cnt); 91ff6462e8SSasha Levin break; 92258dd093SPekka Enberg default: 934542f276SCyrill Gorcunov pr_warning("request type %d", req->type); 9470b53f25SSasha Levin block_cnt = -1; 95407475bfSPekka Enberg break; 9603110ff3SAsias He } 9703110ff3SAsias He 9845e47970SAsias He /* status */ 9945e47970SAsias He status = iov[out + in - 1].iov_base; 10070b53f25SSasha Levin *status = (block_cnt < 0) ? VIRTIO_BLK_S_IOERR : VIRTIO_BLK_S_OK; 10103110ff3SAsias He 10269971b13SSasha Levin mutex_lock(&bdev->mutex); 10345e47970SAsias He virt_queue__set_used_elem(queue, head, block_cnt); 10469971b13SSasha Levin mutex_unlock(&bdev->mutex); 1054155ba8cSPekka Enberg 106427948d5SSasha Levin virtio_pci__signal_vq(kvm, &bdev->vpci, queue - bdev->vqs); 1074155ba8cSPekka Enberg } 1084155ba8cSPekka Enberg 10969971b13SSasha Levin static void virtio_blk_do_io(struct kvm *kvm, struct virt_queue *vq, struct blk_dev *bdev) 11045e47970SAsias He { 11169971b13SSasha Levin while (virt_queue__available(vq)) { 11269971b13SSasha Levin struct blk_dev_job *job = &bdev->jobs[bdev->job_idx++ % VIRTIO_BLK_QUEUE_SIZE]; 113407475bfSPekka Enberg 11469971b13SSasha Levin *job = (struct blk_dev_job) { 11569971b13SSasha Levin .vq = vq, 11669971b13SSasha Levin .bdev = bdev, 11769971b13SSasha Levin }; 11869971b13SSasha Levin job->head = virt_queue__get_iov(vq, job->iov, &job->out, &job->in, kvm); 11945e47970SAsias He 12069971b13SSasha Levin thread_pool__init_job(&job->job_id, kvm, virtio_blk_do_io_request, job); 12169971b13SSasha Levin thread_pool__do_job(&job->job_id); 12269971b13SSasha Levin } 1234baf6f73SSasha Levin } 1240528c2a7SPekka Enberg 125427948d5SSasha Levin static void set_config(struct kvm *kvm, void *dev, u8 data, u32 offset) 126427948d5SSasha Levin { 127427948d5SSasha Levin struct blk_dev *bdev = dev; 128427948d5SSasha Levin 129427948d5SSasha Levin ((u8 *)(&bdev->blk_config))[offset] = data; 130427948d5SSasha Levin } 131427948d5SSasha Levin 132427948d5SSasha Levin static u8 get_config(struct kvm *kvm, void *dev, u32 offset) 133427948d5SSasha Levin { 134427948d5SSasha Levin struct blk_dev *bdev = dev; 135427948d5SSasha Levin 136427948d5SSasha Levin return ((u8 *)(&bdev->blk_config))[offset]; 137427948d5SSasha Levin } 138427948d5SSasha Levin 139427948d5SSasha Levin static u32 get_host_features(struct kvm *kvm, void *dev) 140427948d5SSasha Levin { 141427948d5SSasha Levin return 1UL << VIRTIO_BLK_F_SEG_MAX | 1UL << VIRTIO_BLK_F_FLUSH; 142427948d5SSasha Levin } 143427948d5SSasha Levin 144427948d5SSasha Levin static void set_guest_features(struct kvm *kvm, void *dev, u32 features) 145427948d5SSasha Levin { 146427948d5SSasha Levin struct blk_dev *bdev = dev; 147427948d5SSasha Levin 148427948d5SSasha Levin bdev->features = features; 149427948d5SSasha Levin } 150427948d5SSasha Levin 151427948d5SSasha Levin static int init_vq(struct kvm *kvm, void *dev, u32 vq, u32 pfn) 152427948d5SSasha Levin { 153427948d5SSasha Levin struct blk_dev *bdev = dev; 154427948d5SSasha Levin struct virt_queue *queue; 155427948d5SSasha Levin void *p; 156427948d5SSasha Levin 157312c62d1SSasha Levin compat__remove_message(compat_id); 158427948d5SSasha Levin 159427948d5SSasha Levin queue = &bdev->vqs[vq]; 160427948d5SSasha Levin queue->pfn = pfn; 161427948d5SSasha Levin p = guest_pfn_to_host(kvm, queue->pfn); 162427948d5SSasha Levin 163427948d5SSasha Levin vring_init(&queue->vring, VIRTIO_BLK_QUEUE_SIZE, p, VIRTIO_PCI_VRING_ALIGN); 164427948d5SSasha Levin 165427948d5SSasha Levin return 0; 166427948d5SSasha Levin } 167427948d5SSasha Levin 168427948d5SSasha Levin static int notify_vq(struct kvm *kvm, void *dev, u32 vq) 169427948d5SSasha Levin { 170427948d5SSasha Levin struct blk_dev *bdev = dev; 171427948d5SSasha Levin 172427948d5SSasha Levin virtio_blk_do_io(kvm, &bdev->vqs[vq], bdev); 173427948d5SSasha Levin 174427948d5SSasha Levin return 0; 175427948d5SSasha Levin } 176427948d5SSasha Levin 177427948d5SSasha Levin static int get_pfn_vq(struct kvm *kvm, void *dev, u32 vq) 178427948d5SSasha Levin { 179427948d5SSasha Levin struct blk_dev *bdev = dev; 180427948d5SSasha Levin 181427948d5SSasha Levin return bdev->vqs[vq].pfn; 182427948d5SSasha Levin } 183427948d5SSasha Levin 184427948d5SSasha Levin static int get_size_vq(struct kvm *kvm, void *dev, u32 vq) 185427948d5SSasha Levin { 186427948d5SSasha Levin return VIRTIO_BLK_QUEUE_SIZE; 187427948d5SSasha Levin } 188427948d5SSasha Levin 18943835ac9SSasha Levin void virtio_blk__init(struct kvm *kvm, struct disk_image *disk) 1904749e795SSasha Levin { 191fe2a70d1SSasha Levin struct blk_dev *bdev; 1924749e795SSasha Levin 1934749e795SSasha Levin if (!disk) 1944749e795SSasha Levin return; 1954749e795SSasha Levin 196ebe9ac19SSasha Levin bdev = calloc(1, sizeof(struct blk_dev)); 197ebe9ac19SSasha Levin if (bdev == NULL) 198fe2a70d1SSasha Levin die("Failed allocating bdev"); 1994749e795SSasha Levin 200fe2a70d1SSasha Levin *bdev = (struct blk_dev) { 2014749e795SSasha Levin .mutex = PTHREAD_MUTEX_INITIALIZER, 2024749e795SSasha Levin .disk = disk, 2034749e795SSasha Levin .blk_config = (struct virtio_blk_config) { 2044749e795SSasha Levin .capacity = disk->size / SECTOR_SIZE, 2053d7831a1SAsias He .seg_max = DISK_SEG_MAX, 2064749e795SSasha Levin }, 207427948d5SSasha Levin }; 208427948d5SSasha Levin 209427948d5SSasha Levin virtio_pci__init(kvm, &bdev->vpci, bdev, PCI_DEVICE_ID_VIRTIO_BLK, VIRTIO_ID_BLOCK); 210427948d5SSasha Levin bdev->vpci.ops = (struct virtio_pci_ops) { 211427948d5SSasha Levin .set_config = set_config, 212427948d5SSasha Levin .get_config = get_config, 213427948d5SSasha Levin .get_host_features = get_host_features, 214427948d5SSasha Levin .set_guest_features = set_guest_features, 215427948d5SSasha Levin .init_vq = init_vq, 216427948d5SSasha Levin .notify_vq = notify_vq, 217427948d5SSasha Levin .get_pfn_vq = get_pfn_vq, 218427948d5SSasha Levin .get_size_vq = get_size_vq, 219b30d05adSPekka Enberg }; 220b30d05adSPekka Enberg 221ebe9ac19SSasha Levin list_add_tail(&bdev->list, &bdevs); 222ebe9ac19SSasha Levin 223312c62d1SSasha Levin if (compat_id != -1) 224312c62d1SSasha Levin compat_id = compat__add_message("virtio-blk device was not detected", 225404d164bSSasha Levin "While you have requested a virtio-blk device, " 226*fc835ab3SSasha Levin "the guest kernel did not initialize it.\n" 227*fc835ab3SSasha Levin "Please make sure that the guest kernel was " 228*fc835ab3SSasha Levin "compiled with CONFIG_VIRTIO_BLK=y enabled " 229*fc835ab3SSasha Levin "in its .config"); 230b30d05adSPekka Enberg } 231bcb6aacaSPrasad Joshi 232bcb6aacaSPrasad Joshi void virtio_blk__init_all(struct kvm *kvm) 233bcb6aacaSPrasad Joshi { 234bcb6aacaSPrasad Joshi int i; 235bcb6aacaSPrasad Joshi 236bcb6aacaSPrasad Joshi for (i = 0; i < kvm->nr_disks; i++) 237bcb6aacaSPrasad Joshi virtio_blk__init(kvm, kvm->disks[i]); 238bcb6aacaSPrasad Joshi } 239a0a1e3c2SPrasad Joshi 240a0a1e3c2SPrasad Joshi void virtio_blk__delete_all(struct kvm *kvm) 241a0a1e3c2SPrasad Joshi { 242ebe9ac19SSasha Levin while (!list_empty(&bdevs)) { 243ebe9ac19SSasha Levin struct blk_dev *bdev; 244a0a1e3c2SPrasad Joshi 245ebe9ac19SSasha Levin bdev = list_first_entry(&bdevs, struct blk_dev, list); 246ebe9ac19SSasha Levin list_del(&bdev->list); 247ebe9ac19SSasha Levin free(bdev); 248ebe9ac19SSasha Levin } 249a0a1e3c2SPrasad Joshi } 250