Lines Matching +full:- +full:- +full:-
2 * QEMU Hyper-V VMBus
4 * Copyright (c) 2017-2018 Virtuozzo International GmbH.
7 * See the COPYING file in the top-level directory.
11 #include "qemu/error-report.h"
12 #include "qemu/main-loop.h"
16 #include "hw/qdev-properties.h"
17 #include "hw/qdev-properties-system.h"
20 #include "hw/hyperv/vmbus-bridge.h"
70 * are non-contiguous and may belong to different memory regions.
162 * guest->host notifications, either sent directly or dispatched via
169 * SINT route to signal with host->guest notifications; may be shared with
182 * Hyper-V spec mandates that every message port has 16 buffers, which means
185 * For outgoing (i.e. host->guest) messages there's no queue; the VMBus just
191 /* Hyper-V devices never use channel #0. Must be something special. */
194 #define VMBUS_CHANID_COUNT (HV_EVENT_FLAGS_COUNT - VMBUS_FIRST_CHANID)
201 * vmbus_run and vmbus_msg_cb (the latter is called when the host->guest
246 * guest->host notifications for older VMBus, to be dispatched via
254 return gpadl->seen_gfns == gpadl->num_gfns; in gpadl_full()
262 gpadl->id = id; in create_gpadl()
263 gpadl->child_relid = child_relid; in create_gpadl()
264 gpadl->num_gfns = num_gfns; in create_gpadl()
265 gpadl->gfns = g_new(uint64_t, num_gfns); in create_gpadl()
266 QTAILQ_INSERT_HEAD(&vmbus->gpadl_list, gpadl, link); in create_gpadl()
267 gpadl->vmbus = vmbus; in create_gpadl()
268 gpadl->refcount = 1; in create_gpadl()
274 QTAILQ_REMOVE(&gpadl->vmbus->gpadl_list, gpadl, link); in free_gpadl()
275 g_free(gpadl->gfns); in free_gpadl()
282 QTAILQ_FOREACH(gpadl, &vmbus->gpadl_list, link) { in find_gpadl()
283 if (gpadl->id == gpadl_id) { in find_gpadl()
292 VMBusGpadl *gpadl = find_gpadl(chan->vmbus, gpadl_id); in vmbus_get_gpadl()
296 gpadl->refcount++; in vmbus_get_gpadl()
305 if (--gpadl->refcount) { in vmbus_put_gpadl()
313 return gpadl->num_gfns * TARGET_PAGE_SIZE; in vmbus_gpadl_len()
319 iter->gpadl = gpadl; in gpadl_iter_init()
320 iter->as = as; in gpadl_iter_init()
321 iter->dir = dir; in gpadl_iter_init()
322 iter->active = false; in gpadl_iter_init()
327 uint32_t map_start_in_page = (uintptr_t)iter->map & ~TARGET_PAGE_MASK; in gpadl_iter_cache_unmap()
328 uint32_t io_end_in_page = ((iter->last_off - 1) & ~TARGET_PAGE_MASK) + 1; in gpadl_iter_cache_unmap()
330 /* mapping is only done to do non-zero amount of i/o */ in gpadl_iter_cache_unmap()
331 assert(iter->last_off > 0); in gpadl_iter_cache_unmap()
334 dma_memory_unmap(iter->as, iter->map, TARGET_PAGE_SIZE - map_start_in_page, in gpadl_iter_cache_unmap()
335 iter->dir, io_end_in_page - map_start_in_page); in gpadl_iter_cache_unmap()
340 * The direction of the copy is determined by @iter->dir.
349 assert(iter->active); in gpadl_iter_io()
352 uint32_t off_in_page = iter->off & ~TARGET_PAGE_MASK; in gpadl_iter_io()
353 uint32_t pgleft = TARGET_PAGE_SIZE - off_in_page; in gpadl_iter_io()
358 if (iter->map) { in gpadl_iter_io()
360 (uintptr_t)iter->map & ~TARGET_PAGE_MASK; in gpadl_iter_io()
361 uint32_t off_base = iter->off & ~TARGET_PAGE_MASK; in gpadl_iter_io()
362 uint32_t mapped_base = (iter->last_off - 1) & ~TARGET_PAGE_MASK; in gpadl_iter_io()
365 iter->map = NULL; in gpadl_iter_io()
369 if (!iter->map) { in gpadl_iter_io()
372 uint32_t idx = iter->off >> TARGET_PAGE_BITS; in gpadl_iter_io()
373 assert(idx < iter->gpadl->num_gfns); in gpadl_iter_io()
375 maddr = (iter->gpadl->gfns[idx] << TARGET_PAGE_BITS) | off_in_page; in gpadl_iter_io()
377 iter->map = dma_memory_map(iter->as, maddr, &mlen, iter->dir, in gpadl_iter_io()
380 dma_memory_unmap(iter->as, iter->map, mlen, iter->dir, 0); in gpadl_iter_io()
381 iter->map = NULL; in gpadl_iter_io()
382 return -EFAULT; in gpadl_iter_io()
386 p = (void *)(uintptr_t)(((uintptr_t)iter->map & TARGET_PAGE_MASK) | in gpadl_iter_io()
388 if (iter->dir == DMA_DIRECTION_FROM_DEVICE) { in gpadl_iter_io()
395 len -= cplen; in gpadl_iter_io()
396 iter->off += cplen; in gpadl_iter_io()
397 iter->last_off = iter->off; in gpadl_iter_io()
410 assert(iter->active); in gpadl_iter_seek()
411 iter->off = new_off; in gpadl_iter_seek()
420 assert(!iter->active); in gpadl_iter_start_io()
422 iter->map = NULL; in gpadl_iter_start_io()
423 iter->active = true; in gpadl_iter_start_io()
432 assert(iter->active); in gpadl_iter_end_io()
434 if (iter->map) { in gpadl_iter_end_io()
438 iter->active = false; in gpadl_iter_end_io()
451 gpadl_iter_init(&iter, gpadl, chan->dev->dma_as, in vmbus_iov_to_gpadl()
471 QEMUSGList *sgl = &req->sgl; in vmbus_map_sgl()
472 ScatterGatherEntry *sg = sgl->sg; in vmbus_map_sgl()
474 for (i = 0; i < sgl->nsg; i++) { in vmbus_map_sgl()
478 off -= sg[i].len; in vmbus_map_sgl()
480 for (; len && i < sgl->nsg; i++) { in vmbus_map_sgl()
481 dma_addr_t mlen = MIN(sg[i].len - off, len); in vmbus_map_sgl()
483 len -= mlen; in vmbus_map_sgl()
491 ret = -ENOBUFS; in vmbus_map_sgl()
495 iov[ret_cnt].iov_base = dma_memory_map(sgl->as, a, &l, dir, in vmbus_map_sgl()
498 ret = -EFAULT; in vmbus_map_sgl()
503 mlen -= l; in vmbus_map_sgl()
516 QEMUSGList *sgl = &req->sgl; in vmbus_unmap_sgl()
521 dma_memory_unmap(sgl->as, iov[i].iov_base, iov[i].iov_len, dir, acsd); in vmbus_unmap_sgl()
522 accessed -= acsd; in vmbus_unmap_sgl()
550 idx -= len; in rb_idx_wrap()
557 * @allow_catchup - whether @idx1 may catch up @idx2; e.g. read index may catch
563 return rb_idx_wrap(idx2 + len - idx1 - !allow_catchup, len); in rb_idx_delta()
571 rb = dma_memory_map(ringbuf->as, ringbuf->rb_addr, &mlen, in ringbuf_map_hdr()
574 dma_memory_unmap(ringbuf->as, rb, mlen, in ringbuf_map_hdr()
586 dma_memory_unmap(ringbuf->as, rb, sizeof(*rb), DMA_DIRECTION_FROM_DEVICE, in ringbuf_unmap_hdr()
594 ringbuf->as = as; in ringbuf_init_common()
595 ringbuf->rb_addr = gpadl->gfns[begin] << TARGET_PAGE_BITS; in ringbuf_init_common()
596 ringbuf->base = (begin + 1) << TARGET_PAGE_BITS; in ringbuf_init_common()
597 ringbuf->len = (end - begin - 1) << TARGET_PAGE_BITS; in ringbuf_init_common()
598 gpadl_iter_init(&ringbuf->iter, gpadl, as, dir); in ringbuf_init_common()
604 VMBusSendRingBuf *send_ringbuf = &chan->send_ringbuf; in ringbufs_init()
605 VMBusRecvRingBuf *recv_ringbuf = &chan->recv_ringbuf; in ringbufs_init()
607 if (chan->ringbuf_send_offset <= 1 || in ringbufs_init()
608 chan->gpadl->num_gfns <= chan->ringbuf_send_offset + 1) { in ringbufs_init()
609 return -EINVAL; in ringbufs_init()
612 ringbuf_init_common(&recv_ringbuf->common, chan->gpadl, chan->dev->dma_as, in ringbufs_init()
613 DMA_DIRECTION_TO_DEVICE, 0, chan->ringbuf_send_offset); in ringbufs_init()
614 ringbuf_init_common(&send_ringbuf->common, chan->gpadl, chan->dev->dma_as, in ringbufs_init()
615 DMA_DIRECTION_FROM_DEVICE, chan->ringbuf_send_offset, in ringbufs_init()
616 chan->gpadl->num_gfns); in ringbufs_init()
617 send_ringbuf->wanted = 0; in ringbufs_init()
618 send_ringbuf->reserved = 0; in ringbufs_init()
620 rb = ringbuf_map_hdr(&recv_ringbuf->common); in ringbufs_init()
622 return -EFAULT; in ringbufs_init()
624 recv_ringbuf->rd_idx = recv_ringbuf->last_rd_idx = rb->read_index; in ringbufs_init()
625 ringbuf_unmap_hdr(&recv_ringbuf->common, rb, false); in ringbufs_init()
627 rb = ringbuf_map_hdr(&send_ringbuf->common); in ringbufs_init()
629 return -EFAULT; in ringbufs_init()
631 send_ringbuf->wr_idx = send_ringbuf->last_wr_idx = rb->write_index; in ringbufs_init()
632 send_ringbuf->last_seen_rd_idx = rb->read_index; in ringbufs_init()
633 rb->feature_bits |= VMBUS_RING_BUFFER_FEAT_PENDING_SZ; in ringbufs_init()
634 ringbuf_unmap_hdr(&send_ringbuf->common, rb, true); in ringbufs_init()
636 if (recv_ringbuf->rd_idx >= recv_ringbuf->common.len || in ringbufs_init()
637 send_ringbuf->wr_idx >= send_ringbuf->common.len) { in ringbufs_init()
638 return -EOVERFLOW; in ringbufs_init()
645 * Perform io between the GPADL-backed ringbuffer @ringbuf and @buf, wrapping
653 uint32_t remain = ringbuf->len + ringbuf->base - ringbuf->iter.off; in ringbuf_io()
656 ret1 = gpadl_iter_io(&ringbuf->iter, buf, remain); in ringbuf_io()
660 gpadl_iter_seek(&ringbuf->iter, ringbuf->base); in ringbuf_io()
662 len -= remain; in ringbuf_io()
664 ret2 = gpadl_iter_io(&ringbuf->iter, buf, len); in ringbuf_io()
679 gpadl_iter_seek(&ringbuf->iter, in ringbuf_seek()
680 ringbuf->base + rb_idx_wrap(new_off, ringbuf->len)); in ringbuf_seek()
685 return ringbuf->iter.off - ringbuf->base; in ringbuf_tell()
690 gpadl_iter_start_io(&ringbuf->iter); in ringbuf_start_io()
695 gpadl_iter_end_io(&ringbuf->iter); in ringbuf_end_io()
700 return chan->dev; in vmbus_channel_device()
705 if (chan_idx >= dev->num_channels) { in vmbus_device_channel()
708 return &dev->channels[chan_idx]; in vmbus_device_channel()
713 return chan - chan->dev->channels; in vmbus_channel_idx()
718 event_notifier_set(&chan->notifier); in vmbus_channel_notify_host()
723 return chan->is_open; in vmbus_channel_is_open()
728 * The notification is done by signaling a dedicated per-channel SynIC event
737 hwaddr addr = chan->vmbus->int_page_gpa; in vmbus_channel_notify_guest()
740 trace_vmbus_channel_notify_guest(chan->id); in vmbus_channel_notify_guest()
743 return hyperv_set_event_flag(chan->notify_route, chan->id); in vmbus_channel_notify_guest()
748 res = -ENXIO; in vmbus_channel_notify_guest()
752 idx = BIT_WORD(chan->id); in vmbus_channel_notify_guest()
753 mask = BIT_MASK(chan->id); in vmbus_channel_notify_guest()
755 res = hyperv_sint_route_set_sint(chan->notify_route); in vmbus_channel_notify_guest()
769 hdr->offset_qwords = sizeof(*hdr) / sizeof(uint64_t) + in vmbus_pkt_hdr_set_offsets()
771 hdr->len_qwords = hdr->offset_qwords + in vmbus_pkt_hdr_set_offsets()
773 return hdr->len_qwords * sizeof(uint64_t) + VMBUS_PKT_TRAILER; in vmbus_pkt_hdr_set_offsets()
785 * smp_mb [B] * read/write out-of-band data
786 * read/write out-of-band data * smp_mb [B]
798 if (ringbuf->last_seen_rd_idx >= ringbuf->common.len) { in ringbuf_send_avail()
801 return rb_idx_delta(ringbuf->wr_idx, ringbuf->last_seen_rd_idx, in ringbuf_send_avail()
802 ringbuf->common.len, false); in ringbuf_send_avail()
807 VMBusSendRingBuf *ringbuf = &chan->send_ringbuf; in ringbuf_send_update_idx()
811 written = rb_idx_delta(ringbuf->last_wr_idx, ringbuf->wr_idx, in ringbuf_send_update_idx()
812 ringbuf->common.len, true); in ringbuf_send_update_idx()
817 rb = ringbuf_map_hdr(&ringbuf->common); in ringbuf_send_update_idx()
819 return -EFAULT; in ringbuf_send_update_idx()
822 ringbuf->reserved -= written; in ringbuf_send_update_idx()
826 rb->write_index = ringbuf->wr_idx; in ringbuf_send_update_idx()
833 if (ringbuf->wanted) { in ringbuf_send_update_idx()
835 assert(ringbuf->wanted < written); in ringbuf_send_update_idx()
836 ringbuf->wanted -= written; in ringbuf_send_update_idx()
839 rb->pending_send_sz = ringbuf->wanted; in ringbuf_send_update_idx()
844 ringbuf->last_seen_rd_idx = rb->read_index; in ringbuf_send_update_idx()
848 * notification, so re-check the blocking condition, and, if it's no longer in ringbuf_send_update_idx()
852 if (ringbuf_send_avail(ringbuf) >= ringbuf->wanted) { in ringbuf_send_update_idx()
857 if (rb->interrupt_mask) { in ringbuf_send_update_idx()
867 if (rb_idx_delta(ringbuf->last_seen_rd_idx, ringbuf->wr_idx, in ringbuf_send_update_idx()
868 ringbuf->common.len, true) > written) { in ringbuf_send_update_idx()
874 ringbuf_unmap_hdr(&ringbuf->common, rb, true); in ringbuf_send_update_idx()
875 ringbuf->last_wr_idx = ringbuf->wr_idx; in ringbuf_send_update_idx()
882 VMBusSendRingBuf *ringbuf = &chan->send_ringbuf; in vmbus_channel_reserve()
885 uint32_t needed = ringbuf->reserved + in vmbus_channel_reserve()
893 rb = ringbuf_map_hdr(&ringbuf->common); in vmbus_channel_reserve()
895 return -EFAULT; in vmbus_channel_reserve()
899 ringbuf->last_seen_rd_idx = rb->read_index; in vmbus_channel_reserve()
905 rb->pending_send_sz = needed; in vmbus_channel_reserve()
909 * seeing updated pending_send_sz, so re-read read_index (preventing in vmbus_channel_reserve()
913 ringbuf->last_seen_rd_idx = rb->read_index; in vmbus_channel_reserve()
920 ringbuf->reserved = needed; in vmbus_channel_reserve()
924 if (ringbuf->wanted) { in vmbus_channel_reserve()
926 rb = ringbuf_map_hdr(&ringbuf->common); in vmbus_channel_reserve()
928 /* failure to clear pending_send_sz is non-fatal */ in vmbus_channel_reserve()
933 rb->pending_send_sz = 0; in vmbus_channel_reserve()
941 ringbuf_unmap_hdr(&ringbuf->common, rb, ringbuf->wanted == needed); in vmbus_channel_reserve()
943 ringbuf->wanted = needed; in vmbus_channel_reserve()
944 return needed ? -ENOSPC : 0; in vmbus_channel_reserve()
955 VMBusSendRingBuf *ringbuf = &chan->send_ringbuf; in vmbus_channel_send()
958 return -EINVAL; in vmbus_channel_send()
966 assert(totlen <= ringbuf->reserved); in vmbus_channel_send()
968 ringbuf_start_io(&ringbuf->common); in vmbus_channel_send()
969 ringbuf_seek(&ringbuf->common, ringbuf->wr_idx); in vmbus_channel_send()
970 ret = ringbuf_io(&ringbuf->common, &hdr, sizeof(hdr)); in vmbus_channel_send()
976 ret = ringbuf_io(&ringbuf->common, desc, desclen); in vmbus_channel_send()
980 ringbuf_seek(&ringbuf->common, in vmbus_channel_send()
981 ringbuf->wr_idx + hdr.offset_qwords * sizeof(uint64_t)); in vmbus_channel_send()
983 ret = ringbuf_io(&ringbuf->common, msg, msglen); in vmbus_channel_send()
987 ringbuf_seek(&ringbuf->common, ringbuf->wr_idx + totlen); in vmbus_channel_send()
988 ringbuf->wr_idx = ringbuf_tell(&ringbuf->common); in vmbus_channel_send()
991 ringbuf_end_io(&ringbuf->common); in vmbus_channel_send()
1001 assert(req->need_comp); in vmbus_channel_send_completion()
1002 return vmbus_channel_send(req->chan, VMBUS_PACKET_COMP, NULL, 0, in vmbus_channel_send_completion()
1003 msg, msglen, false, req->transaction_id); in vmbus_channel_send_completion()
1016 return -EIO; in sgl_from_gpa_ranges()
1022 len -= sizeof(hdr); in sgl_from_gpa_ranges()
1024 num = (len - hdr.rangecount * sizeof(vmbus_gpa_range)) / sizeof(uint64_t); in sgl_from_gpa_ranges()
1026 return -EIO; in sgl_from_gpa_ranges()
1028 qemu_sglist_init(sgl, DEVICE(dev), num, ringbuf->as); in sgl_from_gpa_ranges()
1030 for (; hdr.rangecount; hdr.rangecount--) { in sgl_from_gpa_ranges()
1040 len -= sizeof(range); in sgl_from_gpa_ranges()
1049 TARGET_PAGE_SIZE - range.byte_offset); in sgl_from_gpa_ranges()
1058 len -= sizeof(uint64_t); in sgl_from_gpa_ranges()
1061 range.byte_count -= plen; in sgl_from_gpa_ranges()
1064 /* consecutive fragments - join */ in sgl_from_gpa_ranges()
1083 ret = -EIO; in sgl_from_gpa_ranges()
1095 uint32_t msgoff = QEMU_ALIGN_UP(size, __alignof__(*req->msg)); in vmbus_alloc_req()
1099 req->chan = chan; in vmbus_alloc_req()
1100 req->pkt_type = pkt_type; in vmbus_alloc_req()
1101 req->msg = (void *)req + msgoff; in vmbus_alloc_req()
1102 req->msglen = msglen; in vmbus_alloc_req()
1103 req->transaction_id = transaction_id; in vmbus_alloc_req()
1104 req->need_comp = need_comp; in vmbus_alloc_req()
1110 VMBusRecvRingBuf *ringbuf = &chan->recv_ringbuf; in vmbus_channel_recv_start()
1113 rb = ringbuf_map_hdr(&ringbuf->common); in vmbus_channel_recv_start()
1115 return -EFAULT; in vmbus_channel_recv_start()
1117 ringbuf->last_seen_wr_idx = rb->write_index; in vmbus_channel_recv_start()
1118 ringbuf_unmap_hdr(&ringbuf->common, rb, false); in vmbus_channel_recv_start()
1120 if (ringbuf->last_seen_wr_idx >= ringbuf->common.len) { in vmbus_channel_recv_start()
1121 return -EOVERFLOW; in vmbus_channel_recv_start()
1131 VMBusRecvRingBuf *ringbuf = &chan->recv_ringbuf; in vmbus_channel_recv_peek()
1140 avail = rb_idx_delta(ringbuf->rd_idx, ringbuf->last_seen_wr_idx, in vmbus_channel_recv_peek()
1141 ringbuf->common.len, true); in vmbus_channel_recv_peek()
1146 ringbuf_seek(&ringbuf->common, ringbuf->rd_idx); in vmbus_channel_recv_peek()
1147 if (ringbuf_io(&ringbuf->common, &hdr, sizeof(hdr)) < 0) { in vmbus_channel_recv_peek()
1163 msglen = pktlen - msgoff; in vmbus_channel_recv_peek()
1170 desclen = msgoff - sizeof(hdr); in vmbus_channel_recv_peek()
1171 if (sgl_from_gpa_ranges(&req->sgl, chan->dev, &ringbuf->common, in vmbus_channel_recv_peek()
1185 ringbuf_seek(&ringbuf->common, ringbuf->rd_idx + msgoff); in vmbus_channel_recv_peek()
1186 if (ringbuf_io(&ringbuf->common, req->msg, msglen) < 0) { in vmbus_channel_recv_peek()
1189 ringbuf_seek(&ringbuf->common, ringbuf->rd_idx + totlen); in vmbus_channel_recv_peek()
1199 VMBusRecvRingBuf *ringbuf = &chan->recv_ringbuf; in vmbus_channel_recv_pop()
1200 ringbuf->rd_idx = ringbuf_tell(&ringbuf->common); in vmbus_channel_recv_pop()
1205 VMBusRecvRingBuf *ringbuf = &chan->recv_ringbuf; in vmbus_channel_recv_done()
1209 read = rb_idx_delta(ringbuf->last_rd_idx, ringbuf->rd_idx, in vmbus_channel_recv_done()
1210 ringbuf->common.len, true); in vmbus_channel_recv_done()
1215 rb = ringbuf_map_hdr(&ringbuf->common); in vmbus_channel_recv_done()
1217 return -EFAULT; in vmbus_channel_recv_done()
1222 rb->read_index = ringbuf->rd_idx; in vmbus_channel_recv_done()
1227 if (rb->interrupt_mask) { in vmbus_channel_recv_done()
1231 if (rb->feature_bits & VMBUS_RING_BUFFER_FEAT_PENDING_SZ) { in vmbus_channel_recv_done()
1233 uint32_t wanted = rb->pending_send_sz; in vmbus_channel_recv_done()
1241 wr_idx = rb->write_index; in vmbus_channel_recv_done()
1243 wr_avail = rb_idx_delta(wr_idx, ringbuf->rd_idx, ringbuf->common.len, in vmbus_channel_recv_done()
1258 ringbuf_unmap_hdr(&ringbuf->common, rb, true); in vmbus_channel_recv_done()
1259 ringbuf->last_rd_idx = ringbuf->rd_idx; in vmbus_channel_recv_done()
1271 if (r->sgl.dev) { in vmbus_free_req()
1272 qemu_sglist_destroy(&r->sgl); in vmbus_free_req()
1288 VMBusRecvRingBuf *ringbuf = &chan->recv_ringbuf; in channel_event_cb()
1289 ringbuf_start_io(&ringbuf->common); in channel_event_cb()
1290 chan->notify_cb(chan); in channel_event_cb()
1291 ringbuf_end_io(&ringbuf->common); in channel_event_cb()
1300 ret = find_next_zero_bit(vmbus->chanid_bitmap, VMBUS_CHANID_COUNT, 0); in alloc_chan_id()
1302 return -ENOMEM; in alloc_chan_id()
1309 return test_and_set_bit(chan->id - VMBUS_FIRST_CHANID, in register_chan_id()
1310 chan->vmbus->chanid_bitmap) ? -EEXIST : 0; in register_chan_id()
1315 clear_bit(chan->id - VMBUS_FIRST_CHANID, chan->vmbus->chanid_bitmap); in unregister_chan_id()
1320 return VMBUS_CHAN_CONNECTION_OFFSET + chan->id; in chan_connection_id()
1328 chan->dev = dev; in init_channel()
1329 chan->notify_cb = vdc->chan_notify_cb; in init_channel()
1330 chan->subchan_idx = idx; in init_channel()
1331 chan->vmbus = vmbus; in init_channel()
1338 chan->id = res; in init_channel()
1347 QTAILQ_INSERT_TAIL(&vmbus->channel_list, chan, link); in init_channel()
1352 assert(chan->state == VMCHAN_INIT); in deinit_channel()
1353 QTAILQ_REMOVE(&chan->vmbus->channel_list, chan, link); in deinit_channel()
1363 dev->num_channels = vdc->num_channels ? vdc->num_channels(dev) : 1; in create_channels()
1364 if (dev->num_channels < 1) { in create_channels()
1365 error_setg(errp, "invalid #channels: %u", dev->num_channels); in create_channels()
1369 dev->channels = g_new0(VMBusChannel, dev->num_channels); in create_channels()
1370 for (i = 0; i < dev->num_channels; i++) { in create_channels()
1371 init_channel(vmbus, dev, vdc, &dev->channels[i], i, &err); in create_channels()
1380 while (i--) { in create_channels()
1381 deinit_channel(&dev->channels[i]); in create_channels()
1389 for (i = 0; i < dev->num_channels; i++) { in free_channels()
1390 deinit_channel(&dev->channels[i]); in free_channels()
1392 g_free(dev->channels); in free_channels()
1399 if (vp_index == vmbus->target_vp) { in make_sint_route()
1400 hyperv_sint_route_ref(vmbus->sint_route); in make_sint_route()
1401 return vmbus->sint_route; in make_sint_route()
1404 QTAILQ_FOREACH(chan, &vmbus->channel_list, link) { in make_sint_route()
1405 if (chan->target_vp == vp_index && vmbus_channel_is_open(chan)) { in make_sint_route()
1406 hyperv_sint_route_ref(chan->notify_route); in make_sint_route()
1407 return chan->notify_route; in make_sint_route()
1416 VMBusDeviceClass *vdc = VMBUS_DEVICE_GET_CLASS(chan->dev); in open_channel()
1418 chan->gpadl = vmbus_get_gpadl(chan, chan->ringbuf_gpadl); in open_channel()
1419 if (!chan->gpadl) { in open_channel()
1427 if (event_notifier_init(&chan->notifier, 0)) { in open_channel()
1431 event_notifier_set_handler(&chan->notifier, channel_event_cb); in open_channel()
1434 &chan->notifier)) { in open_channel()
1438 chan->notify_route = make_sint_route(chan->vmbus, chan->target_vp); in open_channel()
1439 if (!chan->notify_route) { in open_channel()
1443 if (vdc->open_channel && vdc->open_channel(chan)) { in open_channel()
1447 chan->is_open = true; in open_channel()
1451 hyperv_sint_route_unref(chan->notify_route); in open_channel()
1455 event_notifier_set_handler(&chan->notifier, NULL); in open_channel()
1456 event_notifier_cleanup(&chan->notifier); in open_channel()
1458 vmbus_put_gpadl(chan->gpadl); in open_channel()
1463 VMBusDeviceClass *vdc = VMBUS_DEVICE_GET_CLASS(chan->dev); in close_channel()
1465 if (!chan->is_open) { in close_channel()
1469 if (vdc->close_channel) { in close_channel()
1470 vdc->close_channel(chan); in close_channel()
1473 hyperv_sint_route_unref(chan->notify_route); in close_channel()
1475 event_notifier_set_handler(&chan->notifier, NULL); in close_channel()
1476 event_notifier_cleanup(&chan->notifier); in close_channel()
1477 vmbus_put_gpadl(chan->gpadl); in close_channel()
1478 chan->is_open = false; in close_channel()
1509 QTAILQ_FOREACH(chan, &vmbus->channel_list, link) { in find_channel()
1510 if (chan->id == id) { in find_channel()
1523 qemu_mutex_lock(&vmbus->rx_queue_lock); in enqueue_incoming_message()
1525 if (vmbus->rx_queue_size == HV_MSG_QUEUE_LEN) { in enqueue_incoming_message()
1526 ret = -ENOBUFS; in enqueue_incoming_message()
1530 prev_size = vmbus->rx_queue_size; in enqueue_incoming_message()
1531 idx = (vmbus->rx_queue_head + vmbus->rx_queue_size) % HV_MSG_QUEUE_LEN; in enqueue_incoming_message()
1532 memcpy(&vmbus->rx_queue[idx], msg, sizeof(*msg)); in enqueue_incoming_message()
1533 vmbus->rx_queue_size++; in enqueue_incoming_message()
1540 qemu_mutex_unlock(&vmbus->rx_queue_lock); in enqueue_incoming_message()
1550 if (msg->message_type != HV_MESSAGE_VMBUS) { in vmbus_recv_message()
1554 if (msg->payload_size < sizeof(struct vmbus_message_header)) { in vmbus_recv_message()
1558 vmbus_msg = (struct vmbus_message_header *)msg->payload; in vmbus_recv_message()
1560 trace_vmbus_recv_message(vmbus_msg->message_type, msg->payload_size); in vmbus_recv_message()
1562 if (vmbus_msg->message_type == VMBUS_MSG_INVALID || in vmbus_recv_message()
1563 vmbus_msg->message_type >= VMBUS_MSG_COUNT) { in vmbus_recv_message()
1565 vmbus_msg->message_type); in vmbus_recv_message()
1577 return vmbus->version > 0 && vmbus->version <= VMBUS_VERSION_CURRENT; in vmbus_initialized()
1592 assert(!vmbus->msg_in_progress); in post_msg()
1596 vmbus->msg_in_progress = true; in post_msg()
1598 trace_vmbus_post_msg(((struct vmbus_message_header *)msgdata)->message_type, in post_msg()
1604 ret = hyperv_post_msg(vmbus->sint_route, &msg); in post_msg()
1605 if (ret == 0 || ret == -EAGAIN) { in post_msg()
1615 if (vmbus->target_vp != (uint32_t)-1) { in vmbus_init()
1616 vmbus->sint_route = hyperv_sint_route_new(vmbus->target_vp, VMBUS_SINT, in vmbus_init()
1618 if (!vmbus->sint_route) { in vmbus_init()
1620 return -ENOMEM; in vmbus_init()
1631 QTAILQ_FOREACH_SAFE(gpadl, &vmbus->gpadl_list, link, tmp_gpadl) { in vmbus_deinit()
1632 if (gpadl->state == VMGPADL_TORNDOWN) { in vmbus_deinit()
1638 QTAILQ_FOREACH(chan, &vmbus->channel_list, link) { in vmbus_deinit()
1639 chan->offer_state = VMOFFER_INIT; in vmbus_deinit()
1642 hyperv_sint_route_unref(vmbus->sint_route); in vmbus_deinit()
1643 vmbus->sint_route = NULL; in vmbus_deinit()
1644 vmbus->int_page_gpa = 0; in vmbus_deinit()
1645 vmbus->target_vp = (uint32_t)-1; in vmbus_deinit()
1646 vmbus->version = 0; in vmbus_deinit()
1647 vmbus->state = VMBUS_LISTEN; in vmbus_deinit()
1648 vmbus->msg_in_progress = false; in vmbus_deinit()
1659 trace_vmbus_initiate_contact(msg->version_requested >> 16, in handle_initiate_contact()
1660 msg->version_requested & 0xffff, in handle_initiate_contact()
1661 msg->target_vcpu, msg->monitor_page1, in handle_initiate_contact()
1662 msg->monitor_page2, msg->interrupt_page); in handle_initiate_contact()
1666 * Useful, in particular, with vmbus-aware BIOS which can't shut vmbus down in handle_initiate_contact()
1671 vmbus->target_vp = msg->target_vcpu; in handle_initiate_contact()
1672 vmbus->version = msg->version_requested; in handle_initiate_contact()
1673 if (vmbus->version < VMBUS_VERSION_WIN8) { in handle_initiate_contact()
1675 vmbus->int_page_gpa = msg->interrupt_page; in handle_initiate_contact()
1677 vmbus->state = VMBUS_HANDSHAKE; in handle_initiate_contact()
1704 QTAILQ_FOREACH(chan, &vmbus->channel_list, link) { in handle_request_offers()
1705 if (chan->offer_state == VMOFFER_INIT) { in handle_request_offers()
1706 chan->offer_state = VMOFFER_SENDING; in handle_request_offers()
1711 vmbus->state = VMBUS_OFFER; in handle_request_offers()
1721 QTAILQ_FOREACH(chan, &vmbus->channel_list, link) { in send_offer()
1722 if (chan->offer_state == VMOFFER_SENDING) { in send_offer()
1723 VMBusDeviceClass *vdc = VMBUS_DEVICE_GET_CLASS(chan->dev); in send_offer()
1724 /* Hyper-V wants LE GUIDs */ in send_offer()
1725 QemuUUID classid = qemu_uuid_bswap(vdc->classid); in send_offer()
1726 QemuUUID instanceid = qemu_uuid_bswap(chan->dev->instanceid); in send_offer()
1729 .child_relid = chan->id, in send_offer()
1731 .channel_flags = vdc->channel_flags, in send_offer()
1732 .mmio_size_mb = vdc->mmio_size_mb, in send_offer()
1740 trace_vmbus_send_offer(chan->id, chan->dev); in send_offer()
1756 QTAILQ_FOREACH(chan, &vmbus->channel_list, link) { in complete_offer()
1757 if (chan->offer_state == VMOFFER_SENDING) { in complete_offer()
1758 chan->offer_state = VMOFFER_SENT; in complete_offer()
1770 QTAILQ_FOREACH(chan, &vmbus->channel_list, link) { in complete_offer()
1771 if (chan->offer_state == VMOFFER_INIT) { in complete_offer()
1772 chan->offer_state = VMOFFER_SENDING; in complete_offer()
1791 if (msglen < sizeof(*msg) + sizeof(msg->range[0]) || in handle_gpadl_header()
1796 num_gfns = (msg->range_buflen - msg->rangecount * sizeof(msg->range[0])) / in handle_gpadl_header()
1797 sizeof(msg->range[0].pfn_array[0]); in handle_gpadl_header()
1799 trace_vmbus_gpadl_header(msg->gpadl_id, num_gfns); in handle_gpadl_header()
1804 * single-range page-aligned GPADLs have been observed so just ignore in handle_gpadl_header()
1807 if (msg->rangecount != 1 || msg->range[0].byte_offset || in handle_gpadl_header()
1808 (msg->range[0].byte_count != (num_gfns << TARGET_PAGE_BITS))) { in handle_gpadl_header()
1813 if (find_gpadl(vmbus, msg->gpadl_id)) { in handle_gpadl_header()
1817 gpadl = create_gpadl(vmbus, msg->gpadl_id, msg->child_relid, num_gfns); in handle_gpadl_header()
1820 (void *)&msg->range[0].pfn_array[i + 1] <= (void *)msg + msglen; in handle_gpadl_header()
1822 gpadl->gfns[gpadl->seen_gfns++] = msg->range[0].pfn_array[i]; in handle_gpadl_header()
1826 vmbus->state = VMBUS_CREATE_GPADL; in handle_gpadl_header()
1840 trace_vmbus_gpadl_body(msg->gpadl_id); in handle_gpadl_body()
1842 gpadl = find_gpadl(vmbus, msg->gpadl_id); in handle_gpadl_body()
1847 num_gfns_left = gpadl->num_gfns - gpadl->seen_gfns; in handle_gpadl_body()
1851 (void *)&msg->pfn_array[i + 1] <= (void *)msg + msglen; i++) { in handle_gpadl_body()
1852 gpadl->gfns[gpadl->seen_gfns++] = msg->pfn_array[i]; in handle_gpadl_body()
1856 vmbus->state = VMBUS_CREATE_GPADL; in handle_gpadl_body()
1864 QTAILQ_FOREACH(gpadl, &vmbus->gpadl_list, link) { in send_create_gpadl()
1865 if (gpadl_full(gpadl) && gpadl->state == VMGPADL_INIT) { in send_create_gpadl()
1868 .gpadl_id = gpadl->id, in send_create_gpadl()
1869 .child_relid = gpadl->child_relid, in send_create_gpadl()
1872 trace_vmbus_gpadl_created(gpadl->id); in send_create_gpadl()
1885 QTAILQ_FOREACH(gpadl, &vmbus->gpadl_list, link) { in complete_create_gpadl()
1886 if (gpadl_full(gpadl) && gpadl->state == VMGPADL_INIT) { in complete_create_gpadl()
1887 gpadl->state = VMGPADL_ALIVE; in complete_create_gpadl()
1906 trace_vmbus_gpadl_teardown(msg->gpadl_id); in handle_gpadl_teardown()
1908 gpadl = find_gpadl(vmbus, msg->gpadl_id); in handle_gpadl_teardown()
1909 if (!gpadl || gpadl->state == VMGPADL_TORNDOWN) { in handle_gpadl_teardown()
1913 gpadl->state = VMGPADL_TEARINGDOWN; in handle_gpadl_teardown()
1914 vmbus->state = VMBUS_TEARDOWN_GPADL; in handle_gpadl_teardown()
1921 QTAILQ_FOREACH(gpadl, &vmbus->gpadl_list, link) { in send_teardown_gpadl()
1922 if (gpadl->state == VMGPADL_TEARINGDOWN) { in send_teardown_gpadl()
1925 .gpadl_id = gpadl->id, in send_teardown_gpadl()
1928 trace_vmbus_gpadl_torndown(gpadl->id); in send_teardown_gpadl()
1941 QTAILQ_FOREACH(gpadl, &vmbus->gpadl_list, link) { in complete_teardown_gpadl()
1942 if (gpadl->state == VMGPADL_TEARINGDOWN) { in complete_teardown_gpadl()
1943 gpadl->state = VMGPADL_TORNDOWN; in complete_teardown_gpadl()
1961 trace_vmbus_open_channel(msg->child_relid, msg->ring_buffer_gpadl_id, in handle_open_channel()
1962 msg->target_vp); in handle_open_channel()
1963 chan = find_channel(vmbus, msg->child_relid); in handle_open_channel()
1964 if (!chan || chan->state != VMCHAN_INIT) { in handle_open_channel()
1968 chan->ringbuf_gpadl = msg->ring_buffer_gpadl_id; in handle_open_channel()
1969 chan->ringbuf_send_offset = msg->ring_buffer_offset; in handle_open_channel()
1970 chan->target_vp = msg->target_vp; in handle_open_channel()
1971 chan->open_id = msg->open_id; in handle_open_channel()
1975 chan->state = VMCHAN_OPENING; in handle_open_channel()
1976 vmbus->state = VMBUS_OPEN_CHANNEL; in handle_open_channel()
1983 QTAILQ_FOREACH(chan, &vmbus->channel_list, link) { in send_open_channel()
1984 if (chan->state == VMCHAN_OPENING) { in send_open_channel()
1987 .child_relid = chan->id, in send_open_channel()
1988 .open_id = chan->open_id, in send_open_channel()
1992 trace_vmbus_channel_open(chan->id, msg.status); in send_open_channel()
2005 QTAILQ_FOREACH(chan, &vmbus->channel_list, link) { in complete_open_channel()
2006 if (chan->state == VMCHAN_OPENING) { in complete_open_channel()
2008 chan->state = VMCHAN_OPEN; in complete_open_channel()
2016 chan->state = VMCHAN_INIT; in complete_open_channel()
2029 for (i = 0; i < vdev->num_channels; i++) { in vdev_reset_on_close()
2030 if (vmbus_channel_is_open(&vdev->channels[i])) { in vdev_reset_on_close()
2035 /* all channels closed -- reset device */ in vdev_reset_on_close()
2048 trace_vmbus_close_channel(msg->child_relid); in handle_close_channel()
2050 chan = find_channel(vmbus, msg->child_relid); in handle_close_channel()
2056 chan->state = VMCHAN_INIT; in handle_close_channel()
2058 vdev_reset_on_close(chan->dev); in handle_close_channel()
2063 vmbus->state = VMBUS_UNLOAD; in handle_unload()
2072 qemu_mutex_lock(&vmbus->rx_queue_lock); in send_unload()
2073 vmbus->rx_queue_size = 0; in send_unload()
2074 qemu_mutex_unlock(&vmbus->rx_queue_lock); in send_unload()
2092 qemu_mutex_lock(&vmbus->rx_queue_lock); in process_message()
2094 if (!vmbus->rx_queue_size) { in process_message()
2098 hv_msg = &vmbus->rx_queue[vmbus->rx_queue_head]; in process_message()
2099 msglen = hv_msg->payload_size; in process_message()
2103 msgdata = hv_msg->payload; in process_message()
2106 trace_vmbus_process_incoming_message(msg->message_type); in process_message()
2108 switch (msg->message_type) { in process_message()
2134 error_report("unknown message type %#x", msg->message_type); in process_message()
2139 vmbus->rx_queue_size--; in process_message()
2140 vmbus->rx_queue_head++; in process_message()
2141 vmbus->rx_queue_head %= HV_MSG_QUEUE_LEN; in process_message()
2145 qemu_mutex_unlock(&vmbus->rx_queue_lock); in process_message()
2163 if (vmbus->msg_in_progress) { in vmbus_do_run()
2167 assert(vmbus->state < VMBUS_STATE_MAX); in vmbus_do_run()
2168 assert(state_runner[vmbus->state].run); in vmbus_do_run()
2169 state_runner[vmbus->state].run(vmbus); in vmbus_do_run()
2177 if (vmbus->in_progress) { in vmbus_run()
2181 vmbus->in_progress = true; in vmbus_run()
2188 vmbus->in_progress = false; in vmbus_run()
2196 assert(vmbus->msg_in_progress); in vmbus_msg_cb()
2200 if (status == -EAGAIN) { in vmbus_msg_cb()
2210 assert(vmbus->state < VMBUS_STATE_MAX); in vmbus_msg_cb()
2211 complete = state_runner[vmbus->state].complete; in vmbus_msg_cb()
2213 vmbus->state = VMBUS_LISTEN; in vmbus_msg_cb()
2216 vmbus->msg_in_progress = false; in vmbus_msg_cb()
2239 if (!vmbus->int_page_gpa) { in vmbus_signal_event()
2243 addr = vmbus->int_page_gpa + TARGET_PAGE_SIZE / 2; in vmbus_signal_event()
2250 QTAILQ_FOREACH(chan, &vmbus->channel_list, link) { in vmbus_signal_event()
2251 if (bitmap_test_and_clear_atomic(int_map, chan->id, 1)) { in vmbus_signal_event()
2273 assert(!qemu_uuid_is_null(&vdev->instanceid)); in vmbus_dev_realize()
2275 if (!qemu_uuid_is_null(&vdc->instanceid)) { in vmbus_dev_realize()
2277 if (!qemu_uuid_is_equal(&vdev->instanceid, &vdc->instanceid)) { in vmbus_dev_realize()
2284 QTAILQ_FOREACH(child, &BUS(vmbus)->children, sibling) { in vmbus_dev_realize()
2285 VMBusDevice *child_dev = VMBUS_DEVICE(child->child); in vmbus_dev_realize()
2291 if (qemu_uuid_is_equal(&child_dev->instanceid, &vdev->instanceid)) { in vmbus_dev_realize()
2292 qemu_uuid_unparse(&vdev->instanceid, idstr); in vmbus_dev_realize()
2298 vdev->dma_as = &address_space_memory; in vmbus_dev_realize()
2305 if (vdc->vmdev_realize) { in vmbus_dev_realize()
2306 vdc->vmdev_realize(vdev, &err); in vmbus_dev_realize()
2325 if (vdev->channels) { in vmbus_dev_reset()
2326 for (i = 0; i < vdev->num_channels; i++) { in vmbus_dev_reset()
2327 VMBusChannel *chan = &vdev->channels[i]; in vmbus_dev_reset()
2329 chan->state = VMCHAN_INIT; in vmbus_dev_reset()
2333 if (vdc->vmdev_reset) { in vmbus_dev_reset()
2334 vdc->vmdev_reset(vdev); in vmbus_dev_reset()
2343 if (vdc->vmdev_unrealize) { in vmbus_dev_unrealize()
2344 vdc->vmdev_unrealize(vdev); in vmbus_dev_unrealize()
2358 kdev->bus_type = TYPE_VMBUS; in vmbus_dev_class_init()
2359 kdev->realize = vmbus_dev_realize; in vmbus_dev_class_init()
2360 kdev->unrealize = vmbus_dev_unrealize; in vmbus_dev_class_init()
2369 if (!qemu_uuid_is_null(&vdc->instanceid)) { in vmbus_dev_instance_init()
2371 vdev->instanceid = vdc->instanceid; in vmbus_dev_instance_init()
2405 qemu_mutex_init(&vmbus->rx_queue_lock); in vmbus_realize()
2407 QTAILQ_INIT(&vmbus->gpadl_list); in vmbus_realize()
2408 QTAILQ_INIT(&vmbus->channel_list); in vmbus_realize()
2417 ret = event_notifier_init(&vmbus->notifier, 0); in vmbus_realize()
2423 event_notifier_set_handler(&vmbus->notifier, vmbus_signal_event); in vmbus_realize()
2425 &vmbus->notifier); in vmbus_realize()
2434 event_notifier_cleanup(&vmbus->notifier); in vmbus_realize()
2438 qemu_mutex_destroy(&vmbus->rx_queue_lock); in vmbus_realize()
2447 event_notifier_cleanup(&vmbus->notifier); in vmbus_unrealize()
2449 qemu_mutex_destroy(&vmbus->rx_queue_lock); in vmbus_unrealize()
2460 return qdev_get_dev_path(bus->parent); in vmbus_get_dev_path()
2468 qemu_uuid_unparse(&vdev->instanceid, uuid); in vmbus_get_fw_dev_path()
2477 k->get_dev_path = vmbus_get_dev_path; in vmbus_class_init()
2478 k->get_fw_dev_path = vmbus_get_fw_dev_path; in vmbus_class_init()
2479 k->realize = vmbus_realize; in vmbus_class_init()
2480 k->unrealize = vmbus_unrealize; in vmbus_class_init()
2481 rc->phases.hold = vmbus_reset_hold; in vmbus_class_init()
2493 QTAILQ_FOREACH(chan, &vmbus->channel_list, link) { in vmbus_pre_load()
2511 QTAILQ_FOREACH(gpadl, &vmbus->gpadl_list, link) { in vmbus_post_load()
2512 gpadl->vmbus = vmbus; in vmbus_post_load()
2513 gpadl->refcount = 1; in vmbus_post_load()
2520 QTAILQ_FOREACH(chan, &vmbus->channel_list, link) { in vmbus_post_load()
2522 if (chan->state == VMCHAN_OPENING || chan->state == VMCHAN_OPEN) { in vmbus_post_load()
2526 if (chan->state != VMCHAN_OPEN) { in vmbus_post_load()
2532 return -1; in vmbus_post_load()
2536 hyperv_sint_route_set_sint(chan->notify_route); in vmbus_post_load()
2564 return vmbus->rx_queue_size; in vmbus_rx_queue_needed()
2626 error_report("VMBus requires usable Hyper-V SynIC and VP_INDEX"); in vmbus_bridge_realize()
2631 warn_report("VMBus enabled without the recommended set of Hyper-V features: " in vmbus_bridge_realize()
2632 "hv-stimer, hv-vapic and hv-runtime. " in vmbus_bridge_realize()
2636 bridge->bus = VMBUS(qbus_new(TYPE_VMBUS, dev, "vmbus")); in vmbus_bridge_realize()
2664 k->realize = vmbus_bridge_realize; in vmbus_bridge_class_init()
2665 k->fw_name = "vmbus"; in vmbus_bridge_class_init()
2666 sk->explicit_ofw_unit_address = vmbus_bridge_ofw_unit_address; in vmbus_bridge_class_init()
2667 set_bit(DEVICE_CATEGORY_BRIDGE, k->categories); in vmbus_bridge_class_init()
2668 k->vmsd = &vmstate_vmbus_bridge; in vmbus_bridge_class_init()
2671 k->user_creatable = true; in vmbus_bridge_class_init()