1 #include <linux/virtio_ring.h> 2 #include <linux/types.h> 3 #include <sys/uio.h> 4 #include <stdlib.h> 5 6 #include "kvm/guest_compat.h" 7 #include "kvm/barrier.h" 8 #include "kvm/virtio.h" 9 #include "kvm/virtio-pci.h" 10 #include "kvm/virtio-mmio.h" 11 #include "kvm/util.h" 12 #include "kvm/kvm.h" 13 14 15 const char* virtio_trans_name(enum virtio_trans trans) 16 { 17 if (trans == VIRTIO_PCI || trans == VIRTIO_PCI_LEGACY) 18 return "pci"; 19 else if (trans == VIRTIO_MMIO || trans == VIRTIO_MMIO_LEGACY) 20 return "mmio"; 21 return "unknown"; 22 } 23 24 int virtio_transport_parser(const struct option *opt, const char *arg, int unset) 25 { 26 enum virtio_trans *type = opt->value; 27 struct kvm *kvm; 28 29 if (!strcmp(opt->long_name, "virtio-transport")) { 30 if (!strcmp(arg, "pci")) { 31 *type = VIRTIO_PCI; 32 } else if (!strcmp(arg, "pci-legacy")) { 33 *type = VIRTIO_PCI_LEGACY; 34 #if defined(CONFIG_ARM) || defined(CONFIG_ARM64) || defined(CONFIG_RISCV) 35 } else if (!strcmp(arg, "mmio")) { 36 *type = VIRTIO_MMIO; 37 } else if (!strcmp(arg, "mmio-legacy")) { 38 *type = VIRTIO_MMIO_LEGACY; 39 #endif 40 } else { 41 pr_err("virtio-transport: unknown type \"%s\"\n", arg); 42 return -1; 43 } 44 } else if (!strcmp(opt->long_name, "virtio-legacy")) { 45 *type = VIRTIO_PCI_LEGACY; 46 } else if (!strcmp(opt->long_name, "force-pci")) { 47 kvm = opt->ptr; 48 kvm->cfg.virtio_transport = VIRTIO_PCI; 49 } 50 51 return 0; 52 } 53 54 void virt_queue__used_idx_advance(struct virt_queue *queue, u16 jump) 55 { 56 u16 idx = virtio_guest_to_host_u16(queue, queue->vring.used->idx); 57 58 /* 59 * Use wmb to assure that used elem was updated with head and len. 60 * We need a wmb here since we can't advance idx unless we're ready 61 * to pass the used element to the guest. 62 */ 63 wmb(); 64 idx += jump; 65 queue->vring.used->idx = virtio_host_to_guest_u16(queue, idx); 66 } 67 68 struct vring_used_elem * 69 virt_queue__set_used_elem_no_update(struct virt_queue *queue, u32 head, 70 u32 len, u16 offset) 71 { 72 struct vring_used_elem *used_elem; 73 u16 idx = virtio_guest_to_host_u16(queue, queue->vring.used->idx); 74 75 idx += offset; 76 used_elem = &queue->vring.used->ring[idx % queue->vring.num]; 77 used_elem->id = virtio_host_to_guest_u32(queue, head); 78 used_elem->len = virtio_host_to_guest_u32(queue, len); 79 80 return used_elem; 81 } 82 83 struct vring_used_elem *virt_queue__set_used_elem(struct virt_queue *queue, u32 head, u32 len) 84 { 85 struct vring_used_elem *used_elem; 86 87 used_elem = virt_queue__set_used_elem_no_update(queue, head, len, 0); 88 virt_queue__used_idx_advance(queue, 1); 89 90 return used_elem; 91 } 92 93 static inline bool virt_desc__test_flag(struct virt_queue *vq, 94 struct vring_desc *desc, u16 flag) 95 { 96 return !!(virtio_guest_to_host_u16(vq, desc->flags) & flag); 97 } 98 99 /* 100 * Each buffer in the virtqueues is actually a chain of descriptors. This 101 * function returns the next descriptor in the chain, or max if we're at the 102 * end. 103 */ 104 static unsigned next_desc(struct virt_queue *vq, struct vring_desc *desc, 105 unsigned int i, unsigned int max) 106 { 107 unsigned int next; 108 109 /* If this descriptor says it doesn't chain, we're done. */ 110 if (!virt_desc__test_flag(vq, &desc[i], VRING_DESC_F_NEXT)) 111 return max; 112 113 next = virtio_guest_to_host_u16(vq, desc[i].next); 114 115 /* Ensure they're not leading us off end of descriptors. */ 116 return min(next, max); 117 } 118 119 u16 virt_queue__get_head_iov(struct virt_queue *vq, struct iovec iov[], u16 *out, u16 *in, u16 head, struct kvm *kvm) 120 { 121 struct vring_desc *desc; 122 u16 idx; 123 u16 max; 124 125 idx = head; 126 *out = *in = 0; 127 max = vq->vring.num; 128 desc = vq->vring.desc; 129 130 if (virt_desc__test_flag(vq, &desc[idx], VRING_DESC_F_INDIRECT)) { 131 max = virtio_guest_to_host_u32(vq, desc[idx].len) / sizeof(struct vring_desc); 132 desc = guest_flat_to_host(kvm, virtio_guest_to_host_u64(vq, desc[idx].addr)); 133 idx = 0; 134 } 135 136 do { 137 /* Grab the first descriptor, and check it's OK. */ 138 iov[*out + *in].iov_len = virtio_guest_to_host_u32(vq, desc[idx].len); 139 iov[*out + *in].iov_base = guest_flat_to_host(kvm, 140 virtio_guest_to_host_u64(vq, desc[idx].addr)); 141 /* If this is an input descriptor, increment that count. */ 142 if (virt_desc__test_flag(vq, &desc[idx], VRING_DESC_F_WRITE)) 143 (*in)++; 144 else 145 (*out)++; 146 } while ((idx = next_desc(vq, desc, idx, max)) != max); 147 148 return head; 149 } 150 151 u16 virt_queue__get_iov(struct virt_queue *vq, struct iovec iov[], u16 *out, u16 *in, struct kvm *kvm) 152 { 153 u16 head; 154 155 head = virt_queue__pop(vq); 156 157 return virt_queue__get_head_iov(vq, iov, out, in, head, kvm); 158 } 159 160 /* in and out are relative to guest */ 161 u16 virt_queue__get_inout_iov(struct kvm *kvm, struct virt_queue *queue, 162 struct iovec in_iov[], struct iovec out_iov[], 163 u16 *in, u16 *out) 164 { 165 struct vring_desc *desc; 166 u16 head, idx; 167 168 idx = head = virt_queue__pop(queue); 169 *out = *in = 0; 170 do { 171 u64 addr; 172 desc = virt_queue__get_desc(queue, idx); 173 addr = virtio_guest_to_host_u64(queue, desc->addr); 174 if (virt_desc__test_flag(queue, desc, VRING_DESC_F_WRITE)) { 175 in_iov[*in].iov_base = guest_flat_to_host(kvm, addr); 176 in_iov[*in].iov_len = virtio_guest_to_host_u32(queue, desc->len); 177 (*in)++; 178 } else { 179 out_iov[*out].iov_base = guest_flat_to_host(kvm, addr); 180 out_iov[*out].iov_len = virtio_guest_to_host_u32(queue, desc->len); 181 (*out)++; 182 } 183 if (virt_desc__test_flag(queue, desc, VRING_DESC_F_NEXT)) 184 idx = virtio_guest_to_host_u16(queue, desc->next); 185 else 186 break; 187 } while (1); 188 189 return head; 190 } 191 192 void virtio_init_device_vq(struct kvm *kvm, struct virtio_device *vdev, 193 struct virt_queue *vq, size_t nr_descs) 194 { 195 struct vring_addr *addr = &vq->vring_addr; 196 197 vq->endian = vdev->endian; 198 vq->use_event_idx = (vdev->features & (1UL << VIRTIO_RING_F_EVENT_IDX)); 199 vq->enabled = true; 200 201 if (addr->legacy) { 202 unsigned long base = (u64)addr->pfn * addr->pgsize; 203 void *p = guest_flat_to_host(kvm, base); 204 205 vring_init(&vq->vring, nr_descs, p, addr->align); 206 } else { 207 u64 desc = (u64)addr->desc_hi << 32 | addr->desc_lo; 208 u64 avail = (u64)addr->avail_hi << 32 | addr->avail_lo; 209 u64 used = (u64)addr->used_hi << 32 | addr->used_lo; 210 211 vq->vring = (struct vring) { 212 .desc = guest_flat_to_host(kvm, desc), 213 .used = guest_flat_to_host(kvm, used), 214 .avail = guest_flat_to_host(kvm, avail), 215 .num = nr_descs, 216 }; 217 } 218 } 219 220 void virtio_exit_vq(struct kvm *kvm, struct virtio_device *vdev, 221 void *dev, int num) 222 { 223 struct virt_queue *vq = vdev->ops->get_vq(kvm, dev, num); 224 225 if (vq->enabled && vdev->ops->exit_vq) 226 vdev->ops->exit_vq(kvm, dev, num); 227 memset(vq, 0, sizeof(*vq)); 228 } 229 230 int virtio__get_dev_specific_field(int offset, bool msix, u32 *config_off) 231 { 232 if (msix) { 233 if (offset < 4) 234 return VIRTIO_PCI_O_MSIX; 235 else 236 offset -= 4; 237 } 238 239 *config_off = offset; 240 241 return VIRTIO_PCI_O_CONFIG; 242 } 243 244 bool virtio_queue__should_signal(struct virt_queue *vq) 245 { 246 u16 old_idx, new_idx, event_idx; 247 248 /* 249 * Use mb to assure used idx has been increased before we signal the 250 * guest, and we don't read a stale value for used_event. Without a mb 251 * here we might not send a notification that we need to send, or the 252 * guest may ignore the queue since it won't see an updated idx. 253 */ 254 mb(); 255 256 if (!vq->use_event_idx) { 257 /* 258 * When VIRTIO_RING_F_EVENT_IDX isn't negotiated, interrupt the 259 * guest if it didn't explicitly request to be left alone. 260 */ 261 return !(virtio_guest_to_host_u16(vq, vq->vring.avail->flags) & 262 VRING_AVAIL_F_NO_INTERRUPT); 263 } 264 265 old_idx = vq->last_used_signalled; 266 new_idx = virtio_guest_to_host_u16(vq, vq->vring.used->idx); 267 event_idx = virtio_guest_to_host_u16(vq, vring_used_event(&vq->vring)); 268 269 if (vring_need_event(event_idx, new_idx, old_idx)) { 270 vq->last_used_signalled = new_idx; 271 return true; 272 } 273 274 return false; 275 } 276 277 void virtio_set_guest_features(struct kvm *kvm, struct virtio_device *vdev, 278 void *dev, u64 features) 279 { 280 /* TODO: fail negotiation if features & ~host_features */ 281 282 vdev->features |= features; 283 } 284 285 void virtio_notify_status(struct kvm *kvm, struct virtio_device *vdev, 286 void *dev, u8 status) 287 { 288 u32 ext_status = status; 289 290 vdev->status &= ~VIRTIO_CONFIG_S_MASK; 291 vdev->status |= status; 292 293 /* Add a few hints to help devices */ 294 if ((status & VIRTIO_CONFIG_S_DRIVER_OK) && 295 !(vdev->status & VIRTIO__STATUS_START)) { 296 vdev->status |= VIRTIO__STATUS_START; 297 ext_status |= VIRTIO__STATUS_START; 298 299 } else if (!status && (vdev->status & VIRTIO__STATUS_START)) { 300 vdev->status &= ~VIRTIO__STATUS_START; 301 ext_status |= VIRTIO__STATUS_STOP; 302 303 /* 304 * Reset virtqueues and stop all traffic now, so that the device 305 * can safely reset the backend in notify_status(). 306 */ 307 vdev->ops->reset(kvm, vdev); 308 } 309 if (!status) 310 ext_status |= VIRTIO__STATUS_CONFIG; 311 312 if (vdev->ops->notify_status) 313 vdev->ops->notify_status(kvm, dev, ext_status); 314 } 315 316 bool virtio_access_config(struct kvm *kvm, struct virtio_device *vdev, 317 void *dev, unsigned long offset, void *data, 318 size_t size, bool is_write) 319 { 320 void *in, *out, *config; 321 size_t config_size = vdev->ops->get_config_size(kvm, dev); 322 323 if (WARN_ONCE(offset + size > config_size, 324 "Config access offset (%lu) is beyond config size (%zu)\n", 325 offset, config_size)) 326 return false; 327 328 config = vdev->ops->get_config(kvm, dev) + offset; 329 330 in = is_write ? data : config; 331 out = is_write ? config : data; 332 333 switch (size) { 334 case 1: 335 *(u8 *)out = *(u8 *)in; 336 break; 337 case 2: 338 *(u16 *)out = *(u16 *)in; 339 break; 340 case 4: 341 *(u32 *)out = *(u32 *)in; 342 break; 343 case 8: 344 *(u64 *)out = *(u64 *)in; 345 break; 346 default: 347 WARN_ONCE(1, "%s: invalid access size\n", __func__); 348 return false; 349 } 350 351 return true; 352 } 353 354 int virtio_init(struct kvm *kvm, void *dev, struct virtio_device *vdev, 355 struct virtio_ops *ops, enum virtio_trans trans, 356 int device_id, int subsys_id, int class) 357 { 358 void *virtio; 359 int r; 360 361 switch (trans) { 362 case VIRTIO_PCI_LEGACY: 363 vdev->legacy = true; 364 /* fall through */ 365 case VIRTIO_PCI: 366 virtio = calloc(sizeof(struct virtio_pci), 1); 367 if (!virtio) 368 return -ENOMEM; 369 vdev->virtio = virtio; 370 vdev->ops = ops; 371 vdev->ops->signal_vq = virtio_pci__signal_vq; 372 vdev->ops->signal_config = virtio_pci__signal_config; 373 vdev->ops->init = virtio_pci__init; 374 vdev->ops->exit = virtio_pci__exit; 375 vdev->ops->reset = virtio_pci__reset; 376 r = vdev->ops->init(kvm, dev, vdev, device_id, subsys_id, class); 377 break; 378 case VIRTIO_MMIO_LEGACY: 379 vdev->legacy = true; 380 /* fall through */ 381 case VIRTIO_MMIO: 382 virtio = calloc(sizeof(struct virtio_mmio), 1); 383 if (!virtio) 384 return -ENOMEM; 385 vdev->virtio = virtio; 386 vdev->ops = ops; 387 vdev->ops->signal_vq = virtio_mmio_signal_vq; 388 vdev->ops->signal_config = virtio_mmio_signal_config; 389 vdev->ops->init = virtio_mmio_init; 390 vdev->ops->exit = virtio_mmio_exit; 391 vdev->ops->reset = virtio_mmio_reset; 392 r = vdev->ops->init(kvm, dev, vdev, device_id, subsys_id, class); 393 break; 394 default: 395 r = -1; 396 }; 397 398 return r; 399 } 400 401 int virtio_compat_add_message(const char *device, const char *config) 402 { 403 int len = 1024; 404 int compat_id; 405 char *title; 406 char *desc; 407 408 title = malloc(len); 409 if (!title) 410 return -ENOMEM; 411 412 desc = malloc(len); 413 if (!desc) { 414 free(title); 415 return -ENOMEM; 416 } 417 418 snprintf(title, len, "%s device was not detected.", device); 419 snprintf(desc, len, "While you have requested a %s device, " 420 "the guest kernel did not initialize it.\n" 421 "\tPlease make sure that the guest kernel was " 422 "compiled with %s=y enabled in .config.", 423 device, config); 424 425 compat_id = compat__add_message(title, desc); 426 427 free(desc); 428 free(title); 429 430 return compat_id; 431 } 432