1416b2c2dSAsias He #include "kvm/virtio-blk.h" 2b30d05adSPekka Enberg 331638bcaSCyrill Gorcunov #include "kvm/virtio-pci-dev.h" 45a24a9f2SPekka Enberg #include "kvm/disk-image.h" 54ef0f4d6SPekka Enberg #include "kvm/mutex.h" 6fe99fd4eSPekka Enberg #include "kvm/util.h" 78b1ff07eSPekka Enberg #include "kvm/kvm.h" 8b30d05adSPekka Enberg #include "kvm/pci.h" 9fb0957f2SSasha Levin #include "kvm/threadpool.h" 10ec75b82fSSasha Levin #include "kvm/ioeventfd.h" 11404d164bSSasha Levin #include "kvm/guest_compat.h" 12427948d5SSasha Levin #include "kvm/virtio-pci.h" 13f41a132bSSasha Levin #include "kvm/virtio.h" 14b30d05adSPekka Enberg 1520c64ecaSPekka Enberg #include <linux/virtio_ring.h> 1620c64ecaSPekka Enberg #include <linux/virtio_blk.h> 17427948d5SSasha Levin #include <linux/kernel.h> 18ebe9ac19SSasha Levin #include <linux/list.h> 193fdf659dSSasha Levin #include <linux/types.h> 200528c2a7SPekka Enberg #include <pthread.h> 214155ba8cSPekka Enberg 224749e795SSasha Levin #define VIRTIO_BLK_MAX_DEV 4 2310eca11dSPekka Enberg 243d7831a1SAsias He /* 253d7831a1SAsias He * the header and status consume too entries 263d7831a1SAsias He */ 273d7831a1SAsias He #define DISK_SEG_MAX (VIRTIO_BLK_QUEUE_SIZE - 2) 284059ad8bSAsias He #define VIRTIO_BLK_QUEUE_SIZE 256 29f41a132bSSasha Levin #define NUM_VIRT_QUEUES 1 3010eca11dSPekka Enberg 318b52f877SSasha Levin struct blk_dev_req { 324749e795SSasha Levin struct virt_queue *vq; 33fe2a70d1SSasha Levin struct blk_dev *bdev; 3469971b13SSasha Levin struct iovec iov[VIRTIO_BLK_QUEUE_SIZE]; 3569971b13SSasha Levin u16 out, in, head; 368b52f877SSasha Levin struct kvm *kvm; 374749e795SSasha Levin }; 384749e795SSasha Levin 39fe2a70d1SSasha Levin struct blk_dev { 40d3476f7dSSasha Levin struct mutex mutex; 418b52f877SSasha Levin 42ebe9ac19SSasha Levin struct list_head list; 430528c2a7SPekka Enberg 4402eca50cSAsias He struct virtio_device vdev; 4540ce993fSPekka Enberg struct virtio_blk_config blk_config; 4638605e1cSSasha Levin struct disk_image *disk; 47427948d5SSasha Levin u32 features; 4810eca11dSPekka Enberg 4945e47970SAsias He struct virt_queue vqs[NUM_VIRT_QUEUES]; 508b52f877SSasha Levin struct blk_dev_req reqs[VIRTIO_BLK_QUEUE_SIZE]; 515ac1178bSAsias He 525ac1178bSAsias He pthread_t io_thread; 535ac1178bSAsias He int io_efd; 545ac1178bSAsias He 555ac1178bSAsias He struct kvm *kvm; 56fbc2fbf9SPekka Enberg }; 57fbc2fbf9SPekka Enberg 58ebe9ac19SSasha Levin static LIST_HEAD(bdevs); 59bdbbcb63SAsias He static int compat_id = -1; 6040ce993fSPekka Enberg 618b52f877SSasha Levin void virtio_blk_complete(void *param, long len) 628b52f877SSasha Levin { 638b52f877SSasha Levin struct blk_dev_req *req = param; 648b52f877SSasha Levin struct blk_dev *bdev = req->bdev; 658b52f877SSasha Levin int queueid = req->vq - bdev->vqs; 663fdf659dSSasha Levin u8 *status; 678b52f877SSasha Levin 688b52f877SSasha Levin /* status */ 698b52f877SSasha Levin status = req->iov[req->out + req->in - 1].iov_base; 708b52f877SSasha Levin *status = (len < 0) ? VIRTIO_BLK_S_IOERR : VIRTIO_BLK_S_OK; 718b52f877SSasha Levin 728b52f877SSasha Levin mutex_lock(&bdev->mutex); 738b52f877SSasha Levin virt_queue__set_used_elem(req->vq, req->head, len); 748b52f877SSasha Levin mutex_unlock(&bdev->mutex); 758b52f877SSasha Levin 767ab3d207SSasha Levin if (virtio_queue__should_signal(&bdev->vqs[queueid])) 7702eca50cSAsias He bdev->vdev.ops->signal_vq(req->kvm, &bdev->vdev, queueid); 788b52f877SSasha Levin } 798b52f877SSasha Levin 8001dafc9eSMarc Zyngier static void virtio_blk_do_io_request(struct kvm *kvm, struct virt_queue *vq, struct blk_dev_req *req) 818b52f877SSasha Levin { 828b52f877SSasha Levin struct virtio_blk_outhdr *req_hdr; 8369971b13SSasha Levin ssize_t block_cnt; 8469971b13SSasha Levin struct blk_dev *bdev; 8569971b13SSasha Levin struct iovec *iov; 86f41a132bSSasha Levin u16 out, in; 8701dafc9eSMarc Zyngier u32 type; 8801dafc9eSMarc Zyngier u64 sector; 894155ba8cSPekka Enberg 9069971b13SSasha Levin block_cnt = -1; 918b52f877SSasha Levin bdev = req->bdev; 928b52f877SSasha Levin iov = req->iov; 938b52f877SSasha Levin out = req->out; 948b52f877SSasha Levin in = req->in; 958b52f877SSasha Levin req_hdr = iov[0].iov_base; 9603110ff3SAsias He 9701dafc9eSMarc Zyngier type = virtio_guest_to_host_u32(vq, req_hdr->type); 9801dafc9eSMarc Zyngier sector = virtio_guest_to_host_u64(vq, req_hdr->sector); 9901dafc9eSMarc Zyngier 10001dafc9eSMarc Zyngier switch (type) { 10103110ff3SAsias He case VIRTIO_BLK_T_IN: 10201dafc9eSMarc Zyngier block_cnt = disk_image__read(bdev->disk, sector, 10334239c78SAsias He iov + 1, in + out - 2, req); 104258dd093SPekka Enberg break; 10503110ff3SAsias He case VIRTIO_BLK_T_OUT: 10601dafc9eSMarc Zyngier block_cnt = disk_image__write(bdev->disk, sector, 10734239c78SAsias He iov + 1, in + out - 2, req); 108258dd093SPekka Enberg break; 10929084a74SPrasad Joshi case VIRTIO_BLK_T_FLUSH: 11029084a74SPrasad Joshi block_cnt = disk_image__flush(bdev->disk); 111fb434ac3SSasha Levin virtio_blk_complete(req, block_cnt); 11229084a74SPrasad Joshi break; 113ff6462e8SSasha Levin case VIRTIO_BLK_T_GET_ID: 114ff6462e8SSasha Levin block_cnt = VIRTIO_BLK_ID_BYTES; 11534239c78SAsias He disk_image__get_serial(bdev->disk, 11634239c78SAsias He (iov + 1)->iov_base, &block_cnt); 117fb434ac3SSasha Levin virtio_blk_complete(req, block_cnt); 118ff6462e8SSasha Levin break; 119258dd093SPekka Enberg default: 12001dafc9eSMarc Zyngier pr_warning("request type %d", type); 12170b53f25SSasha Levin block_cnt = -1; 122407475bfSPekka Enberg break; 12303110ff3SAsias He } 1244155ba8cSPekka Enberg } 1254155ba8cSPekka Enberg 12669971b13SSasha Levin static void virtio_blk_do_io(struct kvm *kvm, struct virt_queue *vq, struct blk_dev *bdev) 12745e47970SAsias He { 1282fddfdb5SAsias He struct blk_dev_req *req; 1292fddfdb5SAsias He u16 head; 130407475bfSPekka Enberg 1312fddfdb5SAsias He while (virt_queue__available(vq)) { 1322fddfdb5SAsias He head = virt_queue__pop(vq); 1332fddfdb5SAsias He req = &bdev->reqs[head]; 13434239c78SAsias He req->head = virt_queue__get_head_iov(vq, req->iov, &req->out, 13534239c78SAsias He &req->in, head, kvm); 1362fddfdb5SAsias He req->vq = vq; 13745e47970SAsias He 13801dafc9eSMarc Zyngier virtio_blk_do_io_request(kvm, vq, req); 13969971b13SSasha Levin } 1404baf6f73SSasha Levin } 1410528c2a7SPekka Enberg 142c5ae742bSSasha Levin static u8 *get_config(struct kvm *kvm, void *dev) 143427948d5SSasha Levin { 144427948d5SSasha Levin struct blk_dev *bdev = dev; 145427948d5SSasha Levin 146c5ae742bSSasha Levin return ((u8 *)(&bdev->blk_config)); 147427948d5SSasha Levin } 148427948d5SSasha Levin 149427948d5SSasha Levin static u32 get_host_features(struct kvm *kvm, void *dev) 150427948d5SSasha Levin { 1517ab3d207SSasha Levin return 1UL << VIRTIO_BLK_F_SEG_MAX 1527ab3d207SSasha Levin | 1UL << VIRTIO_BLK_F_FLUSH 153754c8ce3SSasha Levin | 1UL << VIRTIO_RING_F_EVENT_IDX 154754c8ce3SSasha Levin | 1UL << VIRTIO_RING_F_INDIRECT_DESC; 155427948d5SSasha Levin } 156427948d5SSasha Levin 157427948d5SSasha Levin static void set_guest_features(struct kvm *kvm, void *dev, u32 features) 158427948d5SSasha Levin { 159427948d5SSasha Levin struct blk_dev *bdev = dev; 16001dafc9eSMarc Zyngier struct virtio_blk_config *conf = &bdev->blk_config; 16101dafc9eSMarc Zyngier struct virtio_blk_geometry *geo = &conf->geometry; 162427948d5SSasha Levin 163427948d5SSasha Levin bdev->features = features; 16401dafc9eSMarc Zyngier 16501dafc9eSMarc Zyngier conf->capacity = virtio_host_to_guest_u64(&bdev->vdev, conf->capacity); 16601dafc9eSMarc Zyngier conf->size_max = virtio_host_to_guest_u32(&bdev->vdev, conf->size_max); 16701dafc9eSMarc Zyngier conf->seg_max = virtio_host_to_guest_u32(&bdev->vdev, conf->seg_max); 16801dafc9eSMarc Zyngier 16901dafc9eSMarc Zyngier /* Geometry */ 17001dafc9eSMarc Zyngier geo->cylinders = virtio_host_to_guest_u16(&bdev->vdev, geo->cylinders); 17101dafc9eSMarc Zyngier 17201dafc9eSMarc Zyngier conf->blk_size = virtio_host_to_guest_u32(&bdev->vdev, conf->blk_size); 17301dafc9eSMarc Zyngier conf->min_io_size = virtio_host_to_guest_u16(&bdev->vdev, conf->min_io_size); 17401dafc9eSMarc Zyngier conf->opt_io_size = virtio_host_to_guest_u32(&bdev->vdev, conf->opt_io_size); 175427948d5SSasha Levin } 176427948d5SSasha Levin 17795242e44SJean-Philippe Brucker static void notify_status(struct kvm *kvm, void *dev, u32 status) 17895242e44SJean-Philippe Brucker { 17995242e44SJean-Philippe Brucker } 18095242e44SJean-Philippe Brucker 1815ac1178bSAsias He static void *virtio_blk_thread(void *dev) 1825ac1178bSAsias He { 1835ac1178bSAsias He struct blk_dev *bdev = dev; 1845ac1178bSAsias He u64 data; 185a7aa454eSSasha Levin int r; 1865ac1178bSAsias He 187a4d8c55eSSasha Levin kvm__set_thread_name("virtio-blk-io"); 188a4d8c55eSSasha Levin 1895ac1178bSAsias He while (1) { 190a7aa454eSSasha Levin r = read(bdev->io_efd, &data, sizeof(u64)); 191a7aa454eSSasha Levin if (r < 0) 192a7aa454eSSasha Levin continue; 1935ac1178bSAsias He virtio_blk_do_io(bdev->kvm, &bdev->vqs[0], bdev); 1945ac1178bSAsias He } 1955ac1178bSAsias He 1965ac1178bSAsias He pthread_exit(NULL); 1975ac1178bSAsias He return NULL; 1985ac1178bSAsias He } 1995ac1178bSAsias He 200*6730b51fSJean-Philippe Brucker static int init_vq(struct kvm *kvm, void *dev, u32 vq, u32 page_size, u32 align, 201*6730b51fSJean-Philippe Brucker u32 pfn) 202*6730b51fSJean-Philippe Brucker { 203*6730b51fSJean-Philippe Brucker unsigned int i; 204*6730b51fSJean-Philippe Brucker struct blk_dev *bdev = dev; 205*6730b51fSJean-Philippe Brucker struct virt_queue *queue; 206*6730b51fSJean-Philippe Brucker void *p; 207*6730b51fSJean-Philippe Brucker 208*6730b51fSJean-Philippe Brucker compat__remove_message(compat_id); 209*6730b51fSJean-Philippe Brucker 210*6730b51fSJean-Philippe Brucker queue = &bdev->vqs[vq]; 211*6730b51fSJean-Philippe Brucker queue->pfn = pfn; 212*6730b51fSJean-Philippe Brucker p = virtio_get_vq(kvm, queue->pfn, page_size); 213*6730b51fSJean-Philippe Brucker 214*6730b51fSJean-Philippe Brucker vring_init(&queue->vring, VIRTIO_BLK_QUEUE_SIZE, p, align); 215*6730b51fSJean-Philippe Brucker virtio_init_device_vq(&bdev->vdev, queue); 216*6730b51fSJean-Philippe Brucker 217*6730b51fSJean-Philippe Brucker if (vq != 0) 218*6730b51fSJean-Philippe Brucker return 0; 219*6730b51fSJean-Philippe Brucker 220*6730b51fSJean-Philippe Brucker for (i = 0; i < ARRAY_SIZE(bdev->reqs); i++) { 221*6730b51fSJean-Philippe Brucker bdev->reqs[i] = (struct blk_dev_req) { 222*6730b51fSJean-Philippe Brucker .bdev = bdev, 223*6730b51fSJean-Philippe Brucker .kvm = kvm, 224*6730b51fSJean-Philippe Brucker }; 225*6730b51fSJean-Philippe Brucker } 226*6730b51fSJean-Philippe Brucker 227*6730b51fSJean-Philippe Brucker mutex_init(&bdev->mutex); 228*6730b51fSJean-Philippe Brucker bdev->io_efd = eventfd(0, 0); 229*6730b51fSJean-Philippe Brucker if (bdev->io_efd < 0) 230*6730b51fSJean-Philippe Brucker return -errno; 231*6730b51fSJean-Philippe Brucker 232*6730b51fSJean-Philippe Brucker if (pthread_create(&bdev->io_thread, NULL, virtio_blk_thread, bdev)) 233*6730b51fSJean-Philippe Brucker return -errno; 234*6730b51fSJean-Philippe Brucker 235*6730b51fSJean-Philippe Brucker return 0; 236*6730b51fSJean-Philippe Brucker } 237*6730b51fSJean-Philippe Brucker 238*6730b51fSJean-Philippe Brucker static void exit_vq(struct kvm *kvm, void *dev, u32 vq) 239*6730b51fSJean-Philippe Brucker { 240*6730b51fSJean-Philippe Brucker struct blk_dev *bdev = dev; 241*6730b51fSJean-Philippe Brucker 242*6730b51fSJean-Philippe Brucker if (vq != 0) 243*6730b51fSJean-Philippe Brucker return; 244*6730b51fSJean-Philippe Brucker 245*6730b51fSJean-Philippe Brucker close(bdev->io_efd); 246*6730b51fSJean-Philippe Brucker pthread_cancel(bdev->io_thread); 247*6730b51fSJean-Philippe Brucker pthread_join(bdev->io_thread, NULL); 248*6730b51fSJean-Philippe Brucker } 249*6730b51fSJean-Philippe Brucker 250427948d5SSasha Levin static int notify_vq(struct kvm *kvm, void *dev, u32 vq) 251427948d5SSasha Levin { 252427948d5SSasha Levin struct blk_dev *bdev = dev; 2535ac1178bSAsias He u64 data = 1; 254a7aa454eSSasha Levin int r; 255427948d5SSasha Levin 256a7aa454eSSasha Levin r = write(bdev->io_efd, &data, sizeof(data)); 257a7aa454eSSasha Levin if (r < 0) 258a7aa454eSSasha Levin return r; 259427948d5SSasha Levin 260427948d5SSasha Levin return 0; 261427948d5SSasha Levin } 262427948d5SSasha Levin 26353fbb17bSJean-Philippe Brucker static struct virt_queue *get_vq(struct kvm *kvm, void *dev, u32 vq) 264427948d5SSasha Levin { 265427948d5SSasha Levin struct blk_dev *bdev = dev; 266427948d5SSasha Levin 26753fbb17bSJean-Philippe Brucker return &bdev->vqs[vq]; 268427948d5SSasha Levin } 269427948d5SSasha Levin 270427948d5SSasha Levin static int get_size_vq(struct kvm *kvm, void *dev, u32 vq) 271427948d5SSasha Levin { 272ffcc904aSAsias He /* FIXME: dynamic */ 273427948d5SSasha Levin return VIRTIO_BLK_QUEUE_SIZE; 274427948d5SSasha Levin } 275427948d5SSasha Levin 276ffcc904aSAsias He static int set_size_vq(struct kvm *kvm, void *dev, u32 vq, int size) 277ffcc904aSAsias He { 278ffcc904aSAsias He /* FIXME: dynamic */ 279ffcc904aSAsias He return size; 280ffcc904aSAsias He } 281ffcc904aSAsias He 282b98ac591SJean-Philippe Brucker static int get_vq_count(struct kvm *kvm, void *dev) 283b98ac591SJean-Philippe Brucker { 284b98ac591SJean-Philippe Brucker return NUM_VIRT_QUEUES; 285b98ac591SJean-Philippe Brucker } 286b98ac591SJean-Philippe Brucker 28715542babSAndre Przywara static struct virtio_ops blk_dev_virtio_ops = { 2881c47ce69SSasha Levin .get_config = get_config, 2891c47ce69SSasha Levin .get_host_features = get_host_features, 2901c47ce69SSasha Levin .set_guest_features = set_guest_features, 291b98ac591SJean-Philippe Brucker .get_vq_count = get_vq_count, 2921c47ce69SSasha Levin .init_vq = init_vq, 293*6730b51fSJean-Philippe Brucker .exit_vq = exit_vq, 29495242e44SJean-Philippe Brucker .notify_status = notify_status, 2951c47ce69SSasha Levin .notify_vq = notify_vq, 29653fbb17bSJean-Philippe Brucker .get_vq = get_vq, 2971c47ce69SSasha Levin .get_size_vq = get_size_vq, 298ffcc904aSAsias He .set_size_vq = set_size_vq, 2991c47ce69SSasha Levin }; 3001c47ce69SSasha Levin 3019f9207c5SSasha Levin static int virtio_blk__init_one(struct kvm *kvm, struct disk_image *disk) 3024749e795SSasha Levin { 303fe2a70d1SSasha Levin struct blk_dev *bdev; 3044749e795SSasha Levin 3054749e795SSasha Levin if (!disk) 3069f9207c5SSasha Levin return -EINVAL; 3074749e795SSasha Levin 308ebe9ac19SSasha Levin bdev = calloc(1, sizeof(struct blk_dev)); 309ebe9ac19SSasha Levin if (bdev == NULL) 3109f9207c5SSasha Levin return -ENOMEM; 3114749e795SSasha Levin 312fe2a70d1SSasha Levin *bdev = (struct blk_dev) { 3134749e795SSasha Levin .disk = disk, 3144749e795SSasha Levin .blk_config = (struct virtio_blk_config) { 3154749e795SSasha Levin .capacity = disk->size / SECTOR_SIZE, 3163d7831a1SAsias He .seg_max = DISK_SEG_MAX, 3174749e795SSasha Levin }, 3185ac1178bSAsias He .kvm = kvm, 319427948d5SSasha Levin }; 320427948d5SSasha Levin 32102eca50cSAsias He virtio_init(kvm, bdev, &bdev->vdev, &blk_dev_virtio_ops, 322d97dadecSWill Deacon VIRTIO_DEFAULT_TRANS(kvm), PCI_DEVICE_ID_VIRTIO_BLK, 323ae06ce71SWill Deacon VIRTIO_ID_BLOCK, PCI_CLASS_BLK); 324b30d05adSPekka Enberg 325ebe9ac19SSasha Levin list_add_tail(&bdev->list, &bdevs); 326ebe9ac19SSasha Levin 327fb434ac3SSasha Levin disk_image__set_callback(bdev->disk, virtio_blk_complete); 328fb434ac3SSasha Levin 329d278197dSAsias He if (compat_id == -1) 33052f34d2cSAsias He compat_id = virtio_compat_add_message("virtio-blk", "CONFIG_VIRTIO_BLK"); 3315ac1178bSAsias He 3329f9207c5SSasha Levin return 0; 333b30d05adSPekka Enberg } 334bcb6aacaSPrasad Joshi 3359f9207c5SSasha Levin static int virtio_blk__exit_one(struct kvm *kvm, struct blk_dev *bdev) 336bcb6aacaSPrasad Joshi { 3379f9207c5SSasha Levin list_del(&bdev->list); 3389f9207c5SSasha Levin free(bdev); 339bcb6aacaSPrasad Joshi 3409f9207c5SSasha Levin return 0; 341bcb6aacaSPrasad Joshi } 342a0a1e3c2SPrasad Joshi 3439f9207c5SSasha Levin int virtio_blk__init(struct kvm *kvm) 3449f9207c5SSasha Levin { 3459f9207c5SSasha Levin int i, r = 0; 3469f9207c5SSasha Levin 3479f9207c5SSasha Levin for (i = 0; i < kvm->nr_disks; i++) { 348a67da3beSAsias He if (kvm->disks[i]->wwpn) 349a67da3beSAsias He continue; 3509f9207c5SSasha Levin r = virtio_blk__init_one(kvm, kvm->disks[i]); 3519f9207c5SSasha Levin if (r < 0) 3529f9207c5SSasha Levin goto cleanup; 3539f9207c5SSasha Levin } 3549f9207c5SSasha Levin 3559f9207c5SSasha Levin return 0; 3569f9207c5SSasha Levin cleanup: 3579f9207c5SSasha Levin return virtio_blk__exit(kvm); 3589f9207c5SSasha Levin } 35949a8afd1SSasha Levin virtio_dev_init(virtio_blk__init); 3609f9207c5SSasha Levin 3619f9207c5SSasha Levin int virtio_blk__exit(struct kvm *kvm) 362a0a1e3c2SPrasad Joshi { 363ebe9ac19SSasha Levin while (!list_empty(&bdevs)) { 364ebe9ac19SSasha Levin struct blk_dev *bdev; 365a0a1e3c2SPrasad Joshi 366ebe9ac19SSasha Levin bdev = list_first_entry(&bdevs, struct blk_dev, list); 3679f9207c5SSasha Levin virtio_blk__exit_one(kvm, bdev); 368ebe9ac19SSasha Levin } 3699f9207c5SSasha Levin 3709f9207c5SSasha Levin return 0; 371a0a1e3c2SPrasad Joshi } 37249a8afd1SSasha Levin virtio_dev_exit(virtio_blk__exit); 373