1 /* 2 * Virtio Network Device 3 * 4 * Copyright IBM, Corp. 2007 5 * 6 * Authors: 7 * Anthony Liguori <aliguori@us.ibm.com> 8 * 9 * This work is licensed under the terms of the GNU GPL, version 2. See 10 * the COPYING file in the top-level directory. 11 * 12 */ 13 14 #include "qemu/osdep.h" 15 #include "qemu/atomic.h" 16 #include "qemu/iov.h" 17 #include "qemu/main-loop.h" 18 #include "qemu/module.h" 19 #include "hw/virtio/virtio.h" 20 #include "net/net.h" 21 #include "net/checksum.h" 22 #include "net/tap.h" 23 #include "qemu/error-report.h" 24 #include "qemu/timer.h" 25 #include "qemu/option.h" 26 #include "qemu/option_int.h" 27 #include "qemu/config-file.h" 28 #include "qapi/qmp/qdict.h" 29 #include "hw/virtio/virtio-net.h" 30 #include "net/vhost_net.h" 31 #include "net/announce.h" 32 #include "hw/virtio/virtio-bus.h" 33 #include "qapi/error.h" 34 #include "qapi/qapi-events-net.h" 35 #include "hw/qdev-properties.h" 36 #include "qapi/qapi-types-migration.h" 37 #include "qapi/qapi-events-migration.h" 38 #include "hw/virtio/virtio-access.h" 39 #include "migration/misc.h" 40 #include "standard-headers/linux/ethtool.h" 41 #include "sysemu/sysemu.h" 42 #include "trace.h" 43 #include "monitor/qdev.h" 44 #include "hw/pci/pci.h" 45 #include "net_rx_pkt.h" 46 #include "hw/virtio/vhost.h" 47 #include "sysemu/qtest.h" 48 49 #define VIRTIO_NET_VM_VERSION 11 50 51 #define MAC_TABLE_ENTRIES 64 52 #define MAX_VLAN (1 << 12) /* Per 802.1Q definition */ 53 54 /* previously fixed value */ 55 #define VIRTIO_NET_RX_QUEUE_DEFAULT_SIZE 256 56 #define VIRTIO_NET_TX_QUEUE_DEFAULT_SIZE 256 57 58 /* for now, only allow larger queue_pairs; with virtio-1, guest can downsize */ 59 #define VIRTIO_NET_RX_QUEUE_MIN_SIZE VIRTIO_NET_RX_QUEUE_DEFAULT_SIZE 60 #define VIRTIO_NET_TX_QUEUE_MIN_SIZE VIRTIO_NET_TX_QUEUE_DEFAULT_SIZE 61 62 #define VIRTIO_NET_IP4_ADDR_SIZE 8 /* ipv4 saddr + daddr */ 63 64 #define VIRTIO_NET_TCP_FLAG 0x3F 65 #define VIRTIO_NET_TCP_HDR_LENGTH 0xF000 66 67 /* IPv4 max payload, 16 bits in the header */ 68 #define VIRTIO_NET_MAX_IP4_PAYLOAD (65535 - sizeof(struct ip_header)) 69 #define VIRTIO_NET_MAX_TCP_PAYLOAD 65535 70 71 /* header length value in ip header without option */ 72 #define VIRTIO_NET_IP4_HEADER_LENGTH 5 73 74 #define VIRTIO_NET_IP6_ADDR_SIZE 32 /* ipv6 saddr + daddr */ 75 #define VIRTIO_NET_MAX_IP6_PAYLOAD VIRTIO_NET_MAX_TCP_PAYLOAD 76 77 /* Purge coalesced packets timer interval, This value affects the performance 78 a lot, and should be tuned carefully, '300000'(300us) is the recommended 79 value to pass the WHQL test, '50000' can gain 2x netperf throughput with 80 tso/gso/gro 'off'. */ 81 #define VIRTIO_NET_RSC_DEFAULT_INTERVAL 300000 82 83 #define VIRTIO_NET_RSS_SUPPORTED_HASHES (VIRTIO_NET_RSS_HASH_TYPE_IPv4 | \ 84 VIRTIO_NET_RSS_HASH_TYPE_TCPv4 | \ 85 VIRTIO_NET_RSS_HASH_TYPE_UDPv4 | \ 86 VIRTIO_NET_RSS_HASH_TYPE_IPv6 | \ 87 VIRTIO_NET_RSS_HASH_TYPE_TCPv6 | \ 88 VIRTIO_NET_RSS_HASH_TYPE_UDPv6 | \ 89 VIRTIO_NET_RSS_HASH_TYPE_IP_EX | \ 90 VIRTIO_NET_RSS_HASH_TYPE_TCP_EX | \ 91 VIRTIO_NET_RSS_HASH_TYPE_UDP_EX) 92 93 static const VirtIOFeature feature_sizes[] = { 94 {.flags = 1ULL << VIRTIO_NET_F_MAC, 95 .end = endof(struct virtio_net_config, mac)}, 96 {.flags = 1ULL << VIRTIO_NET_F_STATUS, 97 .end = endof(struct virtio_net_config, status)}, 98 {.flags = 1ULL << VIRTIO_NET_F_MQ, 99 .end = endof(struct virtio_net_config, max_virtqueue_pairs)}, 100 {.flags = 1ULL << VIRTIO_NET_F_MTU, 101 .end = endof(struct virtio_net_config, mtu)}, 102 {.flags = 1ULL << VIRTIO_NET_F_SPEED_DUPLEX, 103 .end = endof(struct virtio_net_config, duplex)}, 104 {.flags = (1ULL << VIRTIO_NET_F_RSS) | (1ULL << VIRTIO_NET_F_HASH_REPORT), 105 .end = endof(struct virtio_net_config, supported_hash_types)}, 106 {} 107 }; 108 109 static VirtIONetQueue *virtio_net_get_subqueue(NetClientState *nc) 110 { 111 VirtIONet *n = qemu_get_nic_opaque(nc); 112 113 return &n->vqs[nc->queue_index]; 114 } 115 116 static int vq2q(int queue_index) 117 { 118 return queue_index / 2; 119 } 120 121 /* TODO 122 * - we could suppress RX interrupt if we were so inclined. 123 */ 124 125 static void virtio_net_get_config(VirtIODevice *vdev, uint8_t *config) 126 { 127 VirtIONet *n = VIRTIO_NET(vdev); 128 struct virtio_net_config netcfg; 129 NetClientState *nc = qemu_get_queue(n->nic); 130 static const MACAddr zero = { .a = { 0, 0, 0, 0, 0, 0 } }; 131 132 int ret = 0; 133 memset(&netcfg, 0 , sizeof(struct virtio_net_config)); 134 virtio_stw_p(vdev, &netcfg.status, n->status); 135 virtio_stw_p(vdev, &netcfg.max_virtqueue_pairs, n->max_queue_pairs); 136 virtio_stw_p(vdev, &netcfg.mtu, n->net_conf.mtu); 137 memcpy(netcfg.mac, n->mac, ETH_ALEN); 138 virtio_stl_p(vdev, &netcfg.speed, n->net_conf.speed); 139 netcfg.duplex = n->net_conf.duplex; 140 netcfg.rss_max_key_size = VIRTIO_NET_RSS_MAX_KEY_SIZE; 141 virtio_stw_p(vdev, &netcfg.rss_max_indirection_table_length, 142 virtio_host_has_feature(vdev, VIRTIO_NET_F_RSS) ? 143 VIRTIO_NET_RSS_MAX_TABLE_LEN : 1); 144 virtio_stl_p(vdev, &netcfg.supported_hash_types, 145 VIRTIO_NET_RSS_SUPPORTED_HASHES); 146 memcpy(config, &netcfg, n->config_size); 147 148 /* 149 * Is this VDPA? No peer means not VDPA: there's no way to 150 * disconnect/reconnect a VDPA peer. 151 */ 152 if (nc->peer && nc->peer->info->type == NET_CLIENT_DRIVER_VHOST_VDPA) { 153 ret = vhost_net_get_config(get_vhost_net(nc->peer), (uint8_t *)&netcfg, 154 n->config_size); 155 if (ret != -1) { 156 /* 157 * Some NIC/kernel combinations present 0 as the mac address. As 158 * that is not a legal address, try to proceed with the 159 * address from the QEMU command line in the hope that the 160 * address has been configured correctly elsewhere - just not 161 * reported by the device. 162 */ 163 if (memcmp(&netcfg.mac, &zero, sizeof(zero)) == 0) { 164 info_report("Zero hardware mac address detected. Ignoring."); 165 memcpy(netcfg.mac, n->mac, ETH_ALEN); 166 } 167 memcpy(config, &netcfg, n->config_size); 168 } 169 } 170 } 171 172 static void virtio_net_set_config(VirtIODevice *vdev, const uint8_t *config) 173 { 174 VirtIONet *n = VIRTIO_NET(vdev); 175 struct virtio_net_config netcfg = {}; 176 NetClientState *nc = qemu_get_queue(n->nic); 177 178 memcpy(&netcfg, config, n->config_size); 179 180 if (!virtio_vdev_has_feature(vdev, VIRTIO_NET_F_CTRL_MAC_ADDR) && 181 !virtio_vdev_has_feature(vdev, VIRTIO_F_VERSION_1) && 182 memcmp(netcfg.mac, n->mac, ETH_ALEN)) { 183 memcpy(n->mac, netcfg.mac, ETH_ALEN); 184 qemu_format_nic_info_str(qemu_get_queue(n->nic), n->mac); 185 } 186 187 /* 188 * Is this VDPA? No peer means not VDPA: there's no way to 189 * disconnect/reconnect a VDPA peer. 190 */ 191 if (nc->peer && nc->peer->info->type == NET_CLIENT_DRIVER_VHOST_VDPA) { 192 vhost_net_set_config(get_vhost_net(nc->peer), 193 (uint8_t *)&netcfg, 0, n->config_size, 194 VHOST_SET_CONFIG_TYPE_MASTER); 195 } 196 } 197 198 static bool virtio_net_started(VirtIONet *n, uint8_t status) 199 { 200 VirtIODevice *vdev = VIRTIO_DEVICE(n); 201 return (status & VIRTIO_CONFIG_S_DRIVER_OK) && 202 (n->status & VIRTIO_NET_S_LINK_UP) && vdev->vm_running; 203 } 204 205 static void virtio_net_announce_notify(VirtIONet *net) 206 { 207 VirtIODevice *vdev = VIRTIO_DEVICE(net); 208 trace_virtio_net_announce_notify(); 209 210 net->status |= VIRTIO_NET_S_ANNOUNCE; 211 virtio_notify_config(vdev); 212 } 213 214 static void virtio_net_announce_timer(void *opaque) 215 { 216 VirtIONet *n = opaque; 217 trace_virtio_net_announce_timer(n->announce_timer.round); 218 219 n->announce_timer.round--; 220 virtio_net_announce_notify(n); 221 } 222 223 static void virtio_net_announce(NetClientState *nc) 224 { 225 VirtIONet *n = qemu_get_nic_opaque(nc); 226 VirtIODevice *vdev = VIRTIO_DEVICE(n); 227 228 /* 229 * Make sure the virtio migration announcement timer isn't running 230 * If it is, let it trigger announcement so that we do not cause 231 * confusion. 232 */ 233 if (n->announce_timer.round) { 234 return; 235 } 236 237 if (virtio_vdev_has_feature(vdev, VIRTIO_NET_F_GUEST_ANNOUNCE) && 238 virtio_vdev_has_feature(vdev, VIRTIO_NET_F_CTRL_VQ)) { 239 virtio_net_announce_notify(n); 240 } 241 } 242 243 static void virtio_net_vhost_status(VirtIONet *n, uint8_t status) 244 { 245 VirtIODevice *vdev = VIRTIO_DEVICE(n); 246 NetClientState *nc = qemu_get_queue(n->nic); 247 int queue_pairs = n->multiqueue ? n->max_queue_pairs : 1; 248 int cvq = n->max_ncs - n->max_queue_pairs; 249 250 if (!get_vhost_net(nc->peer)) { 251 return; 252 } 253 254 if ((virtio_net_started(n, status) && !nc->peer->link_down) == 255 !!n->vhost_started) { 256 return; 257 } 258 if (!n->vhost_started) { 259 int r, i; 260 261 if (n->needs_vnet_hdr_swap) { 262 error_report("backend does not support %s vnet headers; " 263 "falling back on userspace virtio", 264 virtio_is_big_endian(vdev) ? "BE" : "LE"); 265 return; 266 } 267 268 /* Any packets outstanding? Purge them to avoid touching rings 269 * when vhost is running. 270 */ 271 for (i = 0; i < queue_pairs; i++) { 272 NetClientState *qnc = qemu_get_subqueue(n->nic, i); 273 274 /* Purge both directions: TX and RX. */ 275 qemu_net_queue_purge(qnc->peer->incoming_queue, qnc); 276 qemu_net_queue_purge(qnc->incoming_queue, qnc->peer); 277 } 278 279 if (virtio_has_feature(vdev->guest_features, VIRTIO_NET_F_MTU)) { 280 r = vhost_net_set_mtu(get_vhost_net(nc->peer), n->net_conf.mtu); 281 if (r < 0) { 282 error_report("%uBytes MTU not supported by the backend", 283 n->net_conf.mtu); 284 285 return; 286 } 287 } 288 289 n->vhost_started = 1; 290 r = vhost_net_start(vdev, n->nic->ncs, queue_pairs, cvq); 291 if (r < 0) { 292 error_report("unable to start vhost net: %d: " 293 "falling back on userspace virtio", -r); 294 n->vhost_started = 0; 295 } 296 } else { 297 vhost_net_stop(vdev, n->nic->ncs, queue_pairs, cvq); 298 n->vhost_started = 0; 299 } 300 } 301 302 static int virtio_net_set_vnet_endian_one(VirtIODevice *vdev, 303 NetClientState *peer, 304 bool enable) 305 { 306 if (virtio_is_big_endian(vdev)) { 307 return qemu_set_vnet_be(peer, enable); 308 } else { 309 return qemu_set_vnet_le(peer, enable); 310 } 311 } 312 313 static bool virtio_net_set_vnet_endian(VirtIODevice *vdev, NetClientState *ncs, 314 int queue_pairs, bool enable) 315 { 316 int i; 317 318 for (i = 0; i < queue_pairs; i++) { 319 if (virtio_net_set_vnet_endian_one(vdev, ncs[i].peer, enable) < 0 && 320 enable) { 321 while (--i >= 0) { 322 virtio_net_set_vnet_endian_one(vdev, ncs[i].peer, false); 323 } 324 325 return true; 326 } 327 } 328 329 return false; 330 } 331 332 static void virtio_net_vnet_endian_status(VirtIONet *n, uint8_t status) 333 { 334 VirtIODevice *vdev = VIRTIO_DEVICE(n); 335 int queue_pairs = n->multiqueue ? n->max_queue_pairs : 1; 336 337 if (virtio_net_started(n, status)) { 338 /* Before using the device, we tell the network backend about the 339 * endianness to use when parsing vnet headers. If the backend 340 * can't do it, we fallback onto fixing the headers in the core 341 * virtio-net code. 342 */ 343 n->needs_vnet_hdr_swap = virtio_net_set_vnet_endian(vdev, n->nic->ncs, 344 queue_pairs, true); 345 } else if (virtio_net_started(n, vdev->status)) { 346 /* After using the device, we need to reset the network backend to 347 * the default (guest native endianness), otherwise the guest may 348 * lose network connectivity if it is rebooted into a different 349 * endianness. 350 */ 351 virtio_net_set_vnet_endian(vdev, n->nic->ncs, queue_pairs, false); 352 } 353 } 354 355 static void virtio_net_drop_tx_queue_data(VirtIODevice *vdev, VirtQueue *vq) 356 { 357 unsigned int dropped = virtqueue_drop_all(vq); 358 if (dropped) { 359 virtio_notify(vdev, vq); 360 } 361 } 362 363 static void virtio_net_set_status(struct VirtIODevice *vdev, uint8_t status) 364 { 365 VirtIONet *n = VIRTIO_NET(vdev); 366 VirtIONetQueue *q; 367 int i; 368 uint8_t queue_status; 369 370 virtio_net_vnet_endian_status(n, status); 371 virtio_net_vhost_status(n, status); 372 373 for (i = 0; i < n->max_queue_pairs; i++) { 374 NetClientState *ncs = qemu_get_subqueue(n->nic, i); 375 bool queue_started; 376 q = &n->vqs[i]; 377 378 if ((!n->multiqueue && i != 0) || i >= n->curr_queue_pairs) { 379 queue_status = 0; 380 } else { 381 queue_status = status; 382 } 383 queue_started = 384 virtio_net_started(n, queue_status) && !n->vhost_started; 385 386 if (queue_started) { 387 qemu_flush_queued_packets(ncs); 388 } 389 390 if (!q->tx_waiting) { 391 continue; 392 } 393 394 if (queue_started) { 395 if (q->tx_timer) { 396 timer_mod(q->tx_timer, 397 qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) + n->tx_timeout); 398 } else { 399 qemu_bh_schedule(q->tx_bh); 400 } 401 } else { 402 if (q->tx_timer) { 403 timer_del(q->tx_timer); 404 } else { 405 qemu_bh_cancel(q->tx_bh); 406 } 407 if ((n->status & VIRTIO_NET_S_LINK_UP) == 0 && 408 (queue_status & VIRTIO_CONFIG_S_DRIVER_OK) && 409 vdev->vm_running) { 410 /* if tx is waiting we are likely have some packets in tx queue 411 * and disabled notification */ 412 q->tx_waiting = 0; 413 virtio_queue_set_notification(q->tx_vq, 1); 414 virtio_net_drop_tx_queue_data(vdev, q->tx_vq); 415 } 416 } 417 } 418 } 419 420 static void virtio_net_set_link_status(NetClientState *nc) 421 { 422 VirtIONet *n = qemu_get_nic_opaque(nc); 423 VirtIODevice *vdev = VIRTIO_DEVICE(n); 424 uint16_t old_status = n->status; 425 426 if (nc->link_down) 427 n->status &= ~VIRTIO_NET_S_LINK_UP; 428 else 429 n->status |= VIRTIO_NET_S_LINK_UP; 430 431 if (n->status != old_status) 432 virtio_notify_config(vdev); 433 434 virtio_net_set_status(vdev, vdev->status); 435 } 436 437 static void rxfilter_notify(NetClientState *nc) 438 { 439 VirtIONet *n = qemu_get_nic_opaque(nc); 440 441 if (nc->rxfilter_notify_enabled) { 442 char *path = object_get_canonical_path(OBJECT(n->qdev)); 443 qapi_event_send_nic_rx_filter_changed(!!n->netclient_name, 444 n->netclient_name, path); 445 g_free(path); 446 447 /* disable event notification to avoid events flooding */ 448 nc->rxfilter_notify_enabled = 0; 449 } 450 } 451 452 static intList *get_vlan_table(VirtIONet *n) 453 { 454 intList *list; 455 int i, j; 456 457 list = NULL; 458 for (i = 0; i < MAX_VLAN >> 5; i++) { 459 for (j = 0; n->vlans[i] && j <= 0x1f; j++) { 460 if (n->vlans[i] & (1U << j)) { 461 QAPI_LIST_PREPEND(list, (i << 5) + j); 462 } 463 } 464 } 465 466 return list; 467 } 468 469 static RxFilterInfo *virtio_net_query_rxfilter(NetClientState *nc) 470 { 471 VirtIONet *n = qemu_get_nic_opaque(nc); 472 VirtIODevice *vdev = VIRTIO_DEVICE(n); 473 RxFilterInfo *info; 474 strList *str_list; 475 int i; 476 477 info = g_malloc0(sizeof(*info)); 478 info->name = g_strdup(nc->name); 479 info->promiscuous = n->promisc; 480 481 if (n->nouni) { 482 info->unicast = RX_STATE_NONE; 483 } else if (n->alluni) { 484 info->unicast = RX_STATE_ALL; 485 } else { 486 info->unicast = RX_STATE_NORMAL; 487 } 488 489 if (n->nomulti) { 490 info->multicast = RX_STATE_NONE; 491 } else if (n->allmulti) { 492 info->multicast = RX_STATE_ALL; 493 } else { 494 info->multicast = RX_STATE_NORMAL; 495 } 496 497 info->broadcast_allowed = n->nobcast; 498 info->multicast_overflow = n->mac_table.multi_overflow; 499 info->unicast_overflow = n->mac_table.uni_overflow; 500 501 info->main_mac = qemu_mac_strdup_printf(n->mac); 502 503 str_list = NULL; 504 for (i = 0; i < n->mac_table.first_multi; i++) { 505 QAPI_LIST_PREPEND(str_list, 506 qemu_mac_strdup_printf(n->mac_table.macs + i * ETH_ALEN)); 507 } 508 info->unicast_table = str_list; 509 510 str_list = NULL; 511 for (i = n->mac_table.first_multi; i < n->mac_table.in_use; i++) { 512 QAPI_LIST_PREPEND(str_list, 513 qemu_mac_strdup_printf(n->mac_table.macs + i * ETH_ALEN)); 514 } 515 info->multicast_table = str_list; 516 info->vlan_table = get_vlan_table(n); 517 518 if (!virtio_vdev_has_feature(vdev, VIRTIO_NET_F_CTRL_VLAN)) { 519 info->vlan = RX_STATE_ALL; 520 } else if (!info->vlan_table) { 521 info->vlan = RX_STATE_NONE; 522 } else { 523 info->vlan = RX_STATE_NORMAL; 524 } 525 526 /* enable event notification after query */ 527 nc->rxfilter_notify_enabled = 1; 528 529 return info; 530 } 531 532 static void virtio_net_reset(VirtIODevice *vdev) 533 { 534 VirtIONet *n = VIRTIO_NET(vdev); 535 int i; 536 537 /* Reset back to compatibility mode */ 538 n->promisc = 1; 539 n->allmulti = 0; 540 n->alluni = 0; 541 n->nomulti = 0; 542 n->nouni = 0; 543 n->nobcast = 0; 544 /* multiqueue is disabled by default */ 545 n->curr_queue_pairs = 1; 546 timer_del(n->announce_timer.tm); 547 n->announce_timer.round = 0; 548 n->status &= ~VIRTIO_NET_S_ANNOUNCE; 549 550 /* Flush any MAC and VLAN filter table state */ 551 n->mac_table.in_use = 0; 552 n->mac_table.first_multi = 0; 553 n->mac_table.multi_overflow = 0; 554 n->mac_table.uni_overflow = 0; 555 memset(n->mac_table.macs, 0, MAC_TABLE_ENTRIES * ETH_ALEN); 556 memcpy(&n->mac[0], &n->nic->conf->macaddr, sizeof(n->mac)); 557 qemu_format_nic_info_str(qemu_get_queue(n->nic), n->mac); 558 memset(n->vlans, 0, MAX_VLAN >> 3); 559 560 /* Flush any async TX */ 561 for (i = 0; i < n->max_queue_pairs; i++) { 562 NetClientState *nc = qemu_get_subqueue(n->nic, i); 563 564 if (nc->peer) { 565 qemu_flush_or_purge_queued_packets(nc->peer, true); 566 assert(!virtio_net_get_subqueue(nc)->async_tx.elem); 567 } 568 } 569 } 570 571 static void peer_test_vnet_hdr(VirtIONet *n) 572 { 573 NetClientState *nc = qemu_get_queue(n->nic); 574 if (!nc->peer) { 575 return; 576 } 577 578 n->has_vnet_hdr = qemu_has_vnet_hdr(nc->peer); 579 } 580 581 static int peer_has_vnet_hdr(VirtIONet *n) 582 { 583 return n->has_vnet_hdr; 584 } 585 586 static int peer_has_ufo(VirtIONet *n) 587 { 588 if (!peer_has_vnet_hdr(n)) 589 return 0; 590 591 n->has_ufo = qemu_has_ufo(qemu_get_queue(n->nic)->peer); 592 593 return n->has_ufo; 594 } 595 596 static void virtio_net_set_mrg_rx_bufs(VirtIONet *n, int mergeable_rx_bufs, 597 int version_1, int hash_report) 598 { 599 int i; 600 NetClientState *nc; 601 602 n->mergeable_rx_bufs = mergeable_rx_bufs; 603 604 if (version_1) { 605 n->guest_hdr_len = hash_report ? 606 sizeof(struct virtio_net_hdr_v1_hash) : 607 sizeof(struct virtio_net_hdr_mrg_rxbuf); 608 n->rss_data.populate_hash = !!hash_report; 609 } else { 610 n->guest_hdr_len = n->mergeable_rx_bufs ? 611 sizeof(struct virtio_net_hdr_mrg_rxbuf) : 612 sizeof(struct virtio_net_hdr); 613 } 614 615 for (i = 0; i < n->max_queue_pairs; i++) { 616 nc = qemu_get_subqueue(n->nic, i); 617 618 if (peer_has_vnet_hdr(n) && 619 qemu_has_vnet_hdr_len(nc->peer, n->guest_hdr_len)) { 620 qemu_set_vnet_hdr_len(nc->peer, n->guest_hdr_len); 621 n->host_hdr_len = n->guest_hdr_len; 622 } 623 } 624 } 625 626 static int virtio_net_max_tx_queue_size(VirtIONet *n) 627 { 628 NetClientState *peer = n->nic_conf.peers.ncs[0]; 629 630 /* 631 * Backends other than vhost-user or vhost-vdpa don't support max queue 632 * size. 633 */ 634 if (!peer) { 635 return VIRTIO_NET_TX_QUEUE_DEFAULT_SIZE; 636 } 637 638 switch(peer->info->type) { 639 case NET_CLIENT_DRIVER_VHOST_USER: 640 case NET_CLIENT_DRIVER_VHOST_VDPA: 641 return VIRTQUEUE_MAX_SIZE; 642 default: 643 return VIRTIO_NET_TX_QUEUE_DEFAULT_SIZE; 644 }; 645 } 646 647 static int peer_attach(VirtIONet *n, int index) 648 { 649 NetClientState *nc = qemu_get_subqueue(n->nic, index); 650 651 if (!nc->peer) { 652 return 0; 653 } 654 655 if (nc->peer->info->type == NET_CLIENT_DRIVER_VHOST_USER) { 656 vhost_set_vring_enable(nc->peer, 1); 657 } 658 659 if (nc->peer->info->type != NET_CLIENT_DRIVER_TAP) { 660 return 0; 661 } 662 663 if (n->max_queue_pairs == 1) { 664 return 0; 665 } 666 667 return tap_enable(nc->peer); 668 } 669 670 static int peer_detach(VirtIONet *n, int index) 671 { 672 NetClientState *nc = qemu_get_subqueue(n->nic, index); 673 674 if (!nc->peer) { 675 return 0; 676 } 677 678 if (nc->peer->info->type == NET_CLIENT_DRIVER_VHOST_USER) { 679 vhost_set_vring_enable(nc->peer, 0); 680 } 681 682 if (nc->peer->info->type != NET_CLIENT_DRIVER_TAP) { 683 return 0; 684 } 685 686 return tap_disable(nc->peer); 687 } 688 689 static void virtio_net_set_queue_pairs(VirtIONet *n) 690 { 691 int i; 692 int r; 693 694 if (n->nic->peer_deleted) { 695 return; 696 } 697 698 for (i = 0; i < n->max_queue_pairs; i++) { 699 if (i < n->curr_queue_pairs) { 700 r = peer_attach(n, i); 701 assert(!r); 702 } else { 703 r = peer_detach(n, i); 704 assert(!r); 705 } 706 } 707 } 708 709 static void virtio_net_set_multiqueue(VirtIONet *n, int multiqueue); 710 711 static uint64_t virtio_net_get_features(VirtIODevice *vdev, uint64_t features, 712 Error **errp) 713 { 714 VirtIONet *n = VIRTIO_NET(vdev); 715 NetClientState *nc = qemu_get_queue(n->nic); 716 717 /* Firstly sync all virtio-net possible supported features */ 718 features |= n->host_features; 719 720 virtio_add_feature(&features, VIRTIO_NET_F_MAC); 721 722 if (!peer_has_vnet_hdr(n)) { 723 virtio_clear_feature(&features, VIRTIO_NET_F_CSUM); 724 virtio_clear_feature(&features, VIRTIO_NET_F_HOST_TSO4); 725 virtio_clear_feature(&features, VIRTIO_NET_F_HOST_TSO6); 726 virtio_clear_feature(&features, VIRTIO_NET_F_HOST_ECN); 727 728 virtio_clear_feature(&features, VIRTIO_NET_F_GUEST_CSUM); 729 virtio_clear_feature(&features, VIRTIO_NET_F_GUEST_TSO4); 730 virtio_clear_feature(&features, VIRTIO_NET_F_GUEST_TSO6); 731 virtio_clear_feature(&features, VIRTIO_NET_F_GUEST_ECN); 732 733 virtio_clear_feature(&features, VIRTIO_NET_F_HASH_REPORT); 734 } 735 736 if (!peer_has_vnet_hdr(n) || !peer_has_ufo(n)) { 737 virtio_clear_feature(&features, VIRTIO_NET_F_GUEST_UFO); 738 virtio_clear_feature(&features, VIRTIO_NET_F_HOST_UFO); 739 } 740 741 if (!get_vhost_net(nc->peer)) { 742 return features; 743 } 744 745 if (!ebpf_rss_is_loaded(&n->ebpf_rss)) { 746 virtio_clear_feature(&features, VIRTIO_NET_F_RSS); 747 } 748 features = vhost_net_get_features(get_vhost_net(nc->peer), features); 749 vdev->backend_features = features; 750 751 if (n->mtu_bypass_backend && 752 (n->host_features & 1ULL << VIRTIO_NET_F_MTU)) { 753 features |= (1ULL << VIRTIO_NET_F_MTU); 754 } 755 756 return features; 757 } 758 759 static uint64_t virtio_net_bad_features(VirtIODevice *vdev) 760 { 761 uint64_t features = 0; 762 763 /* Linux kernel 2.6.25. It understood MAC (as everyone must), 764 * but also these: */ 765 virtio_add_feature(&features, VIRTIO_NET_F_MAC); 766 virtio_add_feature(&features, VIRTIO_NET_F_CSUM); 767 virtio_add_feature(&features, VIRTIO_NET_F_HOST_TSO4); 768 virtio_add_feature(&features, VIRTIO_NET_F_HOST_TSO6); 769 virtio_add_feature(&features, VIRTIO_NET_F_HOST_ECN); 770 771 return features; 772 } 773 774 static void virtio_net_apply_guest_offloads(VirtIONet *n) 775 { 776 qemu_set_offload(qemu_get_queue(n->nic)->peer, 777 !!(n->curr_guest_offloads & (1ULL << VIRTIO_NET_F_GUEST_CSUM)), 778 !!(n->curr_guest_offloads & (1ULL << VIRTIO_NET_F_GUEST_TSO4)), 779 !!(n->curr_guest_offloads & (1ULL << VIRTIO_NET_F_GUEST_TSO6)), 780 !!(n->curr_guest_offloads & (1ULL << VIRTIO_NET_F_GUEST_ECN)), 781 !!(n->curr_guest_offloads & (1ULL << VIRTIO_NET_F_GUEST_UFO))); 782 } 783 784 static uint64_t virtio_net_guest_offloads_by_features(uint32_t features) 785 { 786 static const uint64_t guest_offloads_mask = 787 (1ULL << VIRTIO_NET_F_GUEST_CSUM) | 788 (1ULL << VIRTIO_NET_F_GUEST_TSO4) | 789 (1ULL << VIRTIO_NET_F_GUEST_TSO6) | 790 (1ULL << VIRTIO_NET_F_GUEST_ECN) | 791 (1ULL << VIRTIO_NET_F_GUEST_UFO); 792 793 return guest_offloads_mask & features; 794 } 795 796 static inline uint64_t virtio_net_supported_guest_offloads(VirtIONet *n) 797 { 798 VirtIODevice *vdev = VIRTIO_DEVICE(n); 799 return virtio_net_guest_offloads_by_features(vdev->guest_features); 800 } 801 802 typedef struct { 803 VirtIONet *n; 804 DeviceState *dev; 805 } FailoverDevice; 806 807 /** 808 * Set the failover primary device 809 * 810 * @opaque: FailoverId to setup 811 * @opts: opts for device we are handling 812 * @errp: returns an error if this function fails 813 */ 814 static int failover_set_primary(DeviceState *dev, void *opaque) 815 { 816 FailoverDevice *fdev = opaque; 817 PCIDevice *pci_dev = (PCIDevice *) 818 object_dynamic_cast(OBJECT(dev), TYPE_PCI_DEVICE); 819 820 if (!pci_dev) { 821 return 0; 822 } 823 824 if (!g_strcmp0(pci_dev->failover_pair_id, fdev->n->netclient_name)) { 825 fdev->dev = dev; 826 return 1; 827 } 828 829 return 0; 830 } 831 832 /** 833 * Find the primary device for this failover virtio-net 834 * 835 * @n: VirtIONet device 836 * @errp: returns an error if this function fails 837 */ 838 static DeviceState *failover_find_primary_device(VirtIONet *n) 839 { 840 FailoverDevice fdev = { 841 .n = n, 842 }; 843 844 qbus_walk_children(sysbus_get_default(), failover_set_primary, NULL, 845 NULL, NULL, &fdev); 846 return fdev.dev; 847 } 848 849 static void failover_add_primary(VirtIONet *n, Error **errp) 850 { 851 Error *err = NULL; 852 DeviceState *dev = failover_find_primary_device(n); 853 854 if (dev) { 855 return; 856 } 857 858 if (!n->primary_opts) { 859 error_setg(errp, "Primary device not found"); 860 error_append_hint(errp, "Virtio-net failover will not work. Make " 861 "sure primary device has parameter" 862 " failover_pair_id=%s\n", n->netclient_name); 863 return; 864 } 865 866 dev = qdev_device_add_from_qdict(n->primary_opts, 867 n->primary_opts_from_json, 868 &err); 869 if (err) { 870 qobject_unref(n->primary_opts); 871 n->primary_opts = NULL; 872 } else { 873 object_unref(OBJECT(dev)); 874 } 875 error_propagate(errp, err); 876 } 877 878 static void virtio_net_set_features(VirtIODevice *vdev, uint64_t features) 879 { 880 VirtIONet *n = VIRTIO_NET(vdev); 881 Error *err = NULL; 882 int i; 883 884 if (n->mtu_bypass_backend && 885 !virtio_has_feature(vdev->backend_features, VIRTIO_NET_F_MTU)) { 886 features &= ~(1ULL << VIRTIO_NET_F_MTU); 887 } 888 889 virtio_net_set_multiqueue(n, 890 virtio_has_feature(features, VIRTIO_NET_F_RSS) || 891 virtio_has_feature(features, VIRTIO_NET_F_MQ)); 892 893 virtio_net_set_mrg_rx_bufs(n, 894 virtio_has_feature(features, 895 VIRTIO_NET_F_MRG_RXBUF), 896 virtio_has_feature(features, 897 VIRTIO_F_VERSION_1), 898 virtio_has_feature(features, 899 VIRTIO_NET_F_HASH_REPORT)); 900 901 n->rsc4_enabled = virtio_has_feature(features, VIRTIO_NET_F_RSC_EXT) && 902 virtio_has_feature(features, VIRTIO_NET_F_GUEST_TSO4); 903 n->rsc6_enabled = virtio_has_feature(features, VIRTIO_NET_F_RSC_EXT) && 904 virtio_has_feature(features, VIRTIO_NET_F_GUEST_TSO6); 905 n->rss_data.redirect = virtio_has_feature(features, VIRTIO_NET_F_RSS); 906 907 if (n->has_vnet_hdr) { 908 n->curr_guest_offloads = 909 virtio_net_guest_offloads_by_features(features); 910 virtio_net_apply_guest_offloads(n); 911 } 912 913 for (i = 0; i < n->max_queue_pairs; i++) { 914 NetClientState *nc = qemu_get_subqueue(n->nic, i); 915 916 if (!get_vhost_net(nc->peer)) { 917 continue; 918 } 919 vhost_net_ack_features(get_vhost_net(nc->peer), features); 920 } 921 922 if (virtio_has_feature(features, VIRTIO_NET_F_CTRL_VLAN)) { 923 memset(n->vlans, 0, MAX_VLAN >> 3); 924 } else { 925 memset(n->vlans, 0xff, MAX_VLAN >> 3); 926 } 927 928 if (virtio_has_feature(features, VIRTIO_NET_F_STANDBY)) { 929 qapi_event_send_failover_negotiated(n->netclient_name); 930 qatomic_set(&n->failover_primary_hidden, false); 931 failover_add_primary(n, &err); 932 if (err) { 933 if (!qtest_enabled()) { 934 warn_report_err(err); 935 } else { 936 error_free(err); 937 } 938 } 939 } 940 } 941 942 static int virtio_net_handle_rx_mode(VirtIONet *n, uint8_t cmd, 943 struct iovec *iov, unsigned int iov_cnt) 944 { 945 uint8_t on; 946 size_t s; 947 NetClientState *nc = qemu_get_queue(n->nic); 948 949 s = iov_to_buf(iov, iov_cnt, 0, &on, sizeof(on)); 950 if (s != sizeof(on)) { 951 return VIRTIO_NET_ERR; 952 } 953 954 if (cmd == VIRTIO_NET_CTRL_RX_PROMISC) { 955 n->promisc = on; 956 } else if (cmd == VIRTIO_NET_CTRL_RX_ALLMULTI) { 957 n->allmulti = on; 958 } else if (cmd == VIRTIO_NET_CTRL_RX_ALLUNI) { 959 n->alluni = on; 960 } else if (cmd == VIRTIO_NET_CTRL_RX_NOMULTI) { 961 n->nomulti = on; 962 } else if (cmd == VIRTIO_NET_CTRL_RX_NOUNI) { 963 n->nouni = on; 964 } else if (cmd == VIRTIO_NET_CTRL_RX_NOBCAST) { 965 n->nobcast = on; 966 } else { 967 return VIRTIO_NET_ERR; 968 } 969 970 rxfilter_notify(nc); 971 972 return VIRTIO_NET_OK; 973 } 974 975 static int virtio_net_handle_offloads(VirtIONet *n, uint8_t cmd, 976 struct iovec *iov, unsigned int iov_cnt) 977 { 978 VirtIODevice *vdev = VIRTIO_DEVICE(n); 979 uint64_t offloads; 980 size_t s; 981 982 if (!virtio_vdev_has_feature(vdev, VIRTIO_NET_F_CTRL_GUEST_OFFLOADS)) { 983 return VIRTIO_NET_ERR; 984 } 985 986 s = iov_to_buf(iov, iov_cnt, 0, &offloads, sizeof(offloads)); 987 if (s != sizeof(offloads)) { 988 return VIRTIO_NET_ERR; 989 } 990 991 if (cmd == VIRTIO_NET_CTRL_GUEST_OFFLOADS_SET) { 992 uint64_t supported_offloads; 993 994 offloads = virtio_ldq_p(vdev, &offloads); 995 996 if (!n->has_vnet_hdr) { 997 return VIRTIO_NET_ERR; 998 } 999 1000 n->rsc4_enabled = virtio_has_feature(offloads, VIRTIO_NET_F_RSC_EXT) && 1001 virtio_has_feature(offloads, VIRTIO_NET_F_GUEST_TSO4); 1002 n->rsc6_enabled = virtio_has_feature(offloads, VIRTIO_NET_F_RSC_EXT) && 1003 virtio_has_feature(offloads, VIRTIO_NET_F_GUEST_TSO6); 1004 virtio_clear_feature(&offloads, VIRTIO_NET_F_RSC_EXT); 1005 1006 supported_offloads = virtio_net_supported_guest_offloads(n); 1007 if (offloads & ~supported_offloads) { 1008 return VIRTIO_NET_ERR; 1009 } 1010 1011 n->curr_guest_offloads = offloads; 1012 virtio_net_apply_guest_offloads(n); 1013 1014 return VIRTIO_NET_OK; 1015 } else { 1016 return VIRTIO_NET_ERR; 1017 } 1018 } 1019 1020 static int virtio_net_handle_mac(VirtIONet *n, uint8_t cmd, 1021 struct iovec *iov, unsigned int iov_cnt) 1022 { 1023 VirtIODevice *vdev = VIRTIO_DEVICE(n); 1024 struct virtio_net_ctrl_mac mac_data; 1025 size_t s; 1026 NetClientState *nc = qemu_get_queue(n->nic); 1027 1028 if (cmd == VIRTIO_NET_CTRL_MAC_ADDR_SET) { 1029 if (iov_size(iov, iov_cnt) != sizeof(n->mac)) { 1030 return VIRTIO_NET_ERR; 1031 } 1032 s = iov_to_buf(iov, iov_cnt, 0, &n->mac, sizeof(n->mac)); 1033 assert(s == sizeof(n->mac)); 1034 qemu_format_nic_info_str(qemu_get_queue(n->nic), n->mac); 1035 rxfilter_notify(nc); 1036 1037 return VIRTIO_NET_OK; 1038 } 1039 1040 if (cmd != VIRTIO_NET_CTRL_MAC_TABLE_SET) { 1041 return VIRTIO_NET_ERR; 1042 } 1043 1044 int in_use = 0; 1045 int first_multi = 0; 1046 uint8_t uni_overflow = 0; 1047 uint8_t multi_overflow = 0; 1048 uint8_t *macs = g_malloc0(MAC_TABLE_ENTRIES * ETH_ALEN); 1049 1050 s = iov_to_buf(iov, iov_cnt, 0, &mac_data.entries, 1051 sizeof(mac_data.entries)); 1052 mac_data.entries = virtio_ldl_p(vdev, &mac_data.entries); 1053 if (s != sizeof(mac_data.entries)) { 1054 goto error; 1055 } 1056 iov_discard_front(&iov, &iov_cnt, s); 1057 1058 if (mac_data.entries * ETH_ALEN > iov_size(iov, iov_cnt)) { 1059 goto error; 1060 } 1061 1062 if (mac_data.entries <= MAC_TABLE_ENTRIES) { 1063 s = iov_to_buf(iov, iov_cnt, 0, macs, 1064 mac_data.entries * ETH_ALEN); 1065 if (s != mac_data.entries * ETH_ALEN) { 1066 goto error; 1067 } 1068 in_use += mac_data.entries; 1069 } else { 1070 uni_overflow = 1; 1071 } 1072 1073 iov_discard_front(&iov, &iov_cnt, mac_data.entries * ETH_ALEN); 1074 1075 first_multi = in_use; 1076 1077 s = iov_to_buf(iov, iov_cnt, 0, &mac_data.entries, 1078 sizeof(mac_data.entries)); 1079 mac_data.entries = virtio_ldl_p(vdev, &mac_data.entries); 1080 if (s != sizeof(mac_data.entries)) { 1081 goto error; 1082 } 1083 1084 iov_discard_front(&iov, &iov_cnt, s); 1085 1086 if (mac_data.entries * ETH_ALEN != iov_size(iov, iov_cnt)) { 1087 goto error; 1088 } 1089 1090 if (mac_data.entries <= MAC_TABLE_ENTRIES - in_use) { 1091 s = iov_to_buf(iov, iov_cnt, 0, &macs[in_use * ETH_ALEN], 1092 mac_data.entries * ETH_ALEN); 1093 if (s != mac_data.entries * ETH_ALEN) { 1094 goto error; 1095 } 1096 in_use += mac_data.entries; 1097 } else { 1098 multi_overflow = 1; 1099 } 1100 1101 n->mac_table.in_use = in_use; 1102 n->mac_table.first_multi = first_multi; 1103 n->mac_table.uni_overflow = uni_overflow; 1104 n->mac_table.multi_overflow = multi_overflow; 1105 memcpy(n->mac_table.macs, macs, MAC_TABLE_ENTRIES * ETH_ALEN); 1106 g_free(macs); 1107 rxfilter_notify(nc); 1108 1109 return VIRTIO_NET_OK; 1110 1111 error: 1112 g_free(macs); 1113 return VIRTIO_NET_ERR; 1114 } 1115 1116 static int virtio_net_handle_vlan_table(VirtIONet *n, uint8_t cmd, 1117 struct iovec *iov, unsigned int iov_cnt) 1118 { 1119 VirtIODevice *vdev = VIRTIO_DEVICE(n); 1120 uint16_t vid; 1121 size_t s; 1122 NetClientState *nc = qemu_get_queue(n->nic); 1123 1124 s = iov_to_buf(iov, iov_cnt, 0, &vid, sizeof(vid)); 1125 vid = virtio_lduw_p(vdev, &vid); 1126 if (s != sizeof(vid)) { 1127 return VIRTIO_NET_ERR; 1128 } 1129 1130 if (vid >= MAX_VLAN) 1131 return VIRTIO_NET_ERR; 1132 1133 if (cmd == VIRTIO_NET_CTRL_VLAN_ADD) 1134 n->vlans[vid >> 5] |= (1U << (vid & 0x1f)); 1135 else if (cmd == VIRTIO_NET_CTRL_VLAN_DEL) 1136 n->vlans[vid >> 5] &= ~(1U << (vid & 0x1f)); 1137 else 1138 return VIRTIO_NET_ERR; 1139 1140 rxfilter_notify(nc); 1141 1142 return VIRTIO_NET_OK; 1143 } 1144 1145 static int virtio_net_handle_announce(VirtIONet *n, uint8_t cmd, 1146 struct iovec *iov, unsigned int iov_cnt) 1147 { 1148 trace_virtio_net_handle_announce(n->announce_timer.round); 1149 if (cmd == VIRTIO_NET_CTRL_ANNOUNCE_ACK && 1150 n->status & VIRTIO_NET_S_ANNOUNCE) { 1151 n->status &= ~VIRTIO_NET_S_ANNOUNCE; 1152 if (n->announce_timer.round) { 1153 qemu_announce_timer_step(&n->announce_timer); 1154 } 1155 return VIRTIO_NET_OK; 1156 } else { 1157 return VIRTIO_NET_ERR; 1158 } 1159 } 1160 1161 static void virtio_net_detach_epbf_rss(VirtIONet *n); 1162 1163 static void virtio_net_disable_rss(VirtIONet *n) 1164 { 1165 if (n->rss_data.enabled) { 1166 trace_virtio_net_rss_disable(); 1167 } 1168 n->rss_data.enabled = false; 1169 1170 virtio_net_detach_epbf_rss(n); 1171 } 1172 1173 static bool virtio_net_attach_ebpf_to_backend(NICState *nic, int prog_fd) 1174 { 1175 NetClientState *nc = qemu_get_peer(qemu_get_queue(nic), 0); 1176 if (nc == NULL || nc->info->set_steering_ebpf == NULL) { 1177 return false; 1178 } 1179 1180 return nc->info->set_steering_ebpf(nc, prog_fd); 1181 } 1182 1183 static void rss_data_to_rss_config(struct VirtioNetRssData *data, 1184 struct EBPFRSSConfig *config) 1185 { 1186 config->redirect = data->redirect; 1187 config->populate_hash = data->populate_hash; 1188 config->hash_types = data->hash_types; 1189 config->indirections_len = data->indirections_len; 1190 config->default_queue = data->default_queue; 1191 } 1192 1193 static bool virtio_net_attach_epbf_rss(VirtIONet *n) 1194 { 1195 struct EBPFRSSConfig config = {}; 1196 1197 if (!ebpf_rss_is_loaded(&n->ebpf_rss)) { 1198 return false; 1199 } 1200 1201 rss_data_to_rss_config(&n->rss_data, &config); 1202 1203 if (!ebpf_rss_set_all(&n->ebpf_rss, &config, 1204 n->rss_data.indirections_table, n->rss_data.key)) { 1205 return false; 1206 } 1207 1208 if (!virtio_net_attach_ebpf_to_backend(n->nic, n->ebpf_rss.program_fd)) { 1209 return false; 1210 } 1211 1212 return true; 1213 } 1214 1215 static void virtio_net_detach_epbf_rss(VirtIONet *n) 1216 { 1217 virtio_net_attach_ebpf_to_backend(n->nic, -1); 1218 } 1219 1220 static bool virtio_net_load_ebpf(VirtIONet *n) 1221 { 1222 if (!virtio_net_attach_ebpf_to_backend(n->nic, -1)) { 1223 /* backend does't support steering ebpf */ 1224 return false; 1225 } 1226 1227 return ebpf_rss_load(&n->ebpf_rss); 1228 } 1229 1230 static void virtio_net_unload_ebpf(VirtIONet *n) 1231 { 1232 virtio_net_attach_ebpf_to_backend(n->nic, -1); 1233 ebpf_rss_unload(&n->ebpf_rss); 1234 } 1235 1236 static uint16_t virtio_net_handle_rss(VirtIONet *n, 1237 struct iovec *iov, 1238 unsigned int iov_cnt, 1239 bool do_rss) 1240 { 1241 VirtIODevice *vdev = VIRTIO_DEVICE(n); 1242 struct virtio_net_rss_config cfg; 1243 size_t s, offset = 0, size_get; 1244 uint16_t queue_pairs, i; 1245 struct { 1246 uint16_t us; 1247 uint8_t b; 1248 } QEMU_PACKED temp; 1249 const char *err_msg = ""; 1250 uint32_t err_value = 0; 1251 1252 if (do_rss && !virtio_vdev_has_feature(vdev, VIRTIO_NET_F_RSS)) { 1253 err_msg = "RSS is not negotiated"; 1254 goto error; 1255 } 1256 if (!do_rss && !virtio_vdev_has_feature(vdev, VIRTIO_NET_F_HASH_REPORT)) { 1257 err_msg = "Hash report is not negotiated"; 1258 goto error; 1259 } 1260 size_get = offsetof(struct virtio_net_rss_config, indirection_table); 1261 s = iov_to_buf(iov, iov_cnt, offset, &cfg, size_get); 1262 if (s != size_get) { 1263 err_msg = "Short command buffer"; 1264 err_value = (uint32_t)s; 1265 goto error; 1266 } 1267 n->rss_data.hash_types = virtio_ldl_p(vdev, &cfg.hash_types); 1268 n->rss_data.indirections_len = 1269 virtio_lduw_p(vdev, &cfg.indirection_table_mask); 1270 n->rss_data.indirections_len++; 1271 if (!do_rss) { 1272 n->rss_data.indirections_len = 1; 1273 } 1274 if (!is_power_of_2(n->rss_data.indirections_len)) { 1275 err_msg = "Invalid size of indirection table"; 1276 err_value = n->rss_data.indirections_len; 1277 goto error; 1278 } 1279 if (n->rss_data.indirections_len > VIRTIO_NET_RSS_MAX_TABLE_LEN) { 1280 err_msg = "Too large indirection table"; 1281 err_value = n->rss_data.indirections_len; 1282 goto error; 1283 } 1284 n->rss_data.default_queue = do_rss ? 1285 virtio_lduw_p(vdev, &cfg.unclassified_queue) : 0; 1286 if (n->rss_data.default_queue >= n->max_queue_pairs) { 1287 err_msg = "Invalid default queue"; 1288 err_value = n->rss_data.default_queue; 1289 goto error; 1290 } 1291 offset += size_get; 1292 size_get = sizeof(uint16_t) * n->rss_data.indirections_len; 1293 g_free(n->rss_data.indirections_table); 1294 n->rss_data.indirections_table = g_malloc(size_get); 1295 if (!n->rss_data.indirections_table) { 1296 err_msg = "Can't allocate indirections table"; 1297 err_value = n->rss_data.indirections_len; 1298 goto error; 1299 } 1300 s = iov_to_buf(iov, iov_cnt, offset, 1301 n->rss_data.indirections_table, size_get); 1302 if (s != size_get) { 1303 err_msg = "Short indirection table buffer"; 1304 err_value = (uint32_t)s; 1305 goto error; 1306 } 1307 for (i = 0; i < n->rss_data.indirections_len; ++i) { 1308 uint16_t val = n->rss_data.indirections_table[i]; 1309 n->rss_data.indirections_table[i] = virtio_lduw_p(vdev, &val); 1310 } 1311 offset += size_get; 1312 size_get = sizeof(temp); 1313 s = iov_to_buf(iov, iov_cnt, offset, &temp, size_get); 1314 if (s != size_get) { 1315 err_msg = "Can't get queue_pairs"; 1316 err_value = (uint32_t)s; 1317 goto error; 1318 } 1319 queue_pairs = do_rss ? virtio_lduw_p(vdev, &temp.us) : n->curr_queue_pairs; 1320 if (queue_pairs == 0 || queue_pairs > n->max_queue_pairs) { 1321 err_msg = "Invalid number of queue_pairs"; 1322 err_value = queue_pairs; 1323 goto error; 1324 } 1325 if (temp.b > VIRTIO_NET_RSS_MAX_KEY_SIZE) { 1326 err_msg = "Invalid key size"; 1327 err_value = temp.b; 1328 goto error; 1329 } 1330 if (!temp.b && n->rss_data.hash_types) { 1331 err_msg = "No key provided"; 1332 err_value = 0; 1333 goto error; 1334 } 1335 if (!temp.b && !n->rss_data.hash_types) { 1336 virtio_net_disable_rss(n); 1337 return queue_pairs; 1338 } 1339 offset += size_get; 1340 size_get = temp.b; 1341 s = iov_to_buf(iov, iov_cnt, offset, n->rss_data.key, size_get); 1342 if (s != size_get) { 1343 err_msg = "Can get key buffer"; 1344 err_value = (uint32_t)s; 1345 goto error; 1346 } 1347 n->rss_data.enabled = true; 1348 1349 if (!n->rss_data.populate_hash) { 1350 if (!virtio_net_attach_epbf_rss(n)) { 1351 /* EBPF must be loaded for vhost */ 1352 if (get_vhost_net(qemu_get_queue(n->nic)->peer)) { 1353 warn_report("Can't load eBPF RSS for vhost"); 1354 goto error; 1355 } 1356 /* fallback to software RSS */ 1357 warn_report("Can't load eBPF RSS - fallback to software RSS"); 1358 n->rss_data.enabled_software_rss = true; 1359 } 1360 } else { 1361 /* use software RSS for hash populating */ 1362 /* and detach eBPF if was loaded before */ 1363 virtio_net_detach_epbf_rss(n); 1364 n->rss_data.enabled_software_rss = true; 1365 } 1366 1367 trace_virtio_net_rss_enable(n->rss_data.hash_types, 1368 n->rss_data.indirections_len, 1369 temp.b); 1370 return queue_pairs; 1371 error: 1372 trace_virtio_net_rss_error(err_msg, err_value); 1373 virtio_net_disable_rss(n); 1374 return 0; 1375 } 1376 1377 static int virtio_net_handle_mq(VirtIONet *n, uint8_t cmd, 1378 struct iovec *iov, unsigned int iov_cnt) 1379 { 1380 VirtIODevice *vdev = VIRTIO_DEVICE(n); 1381 uint16_t queue_pairs; 1382 1383 virtio_net_disable_rss(n); 1384 if (cmd == VIRTIO_NET_CTRL_MQ_HASH_CONFIG) { 1385 queue_pairs = virtio_net_handle_rss(n, iov, iov_cnt, false); 1386 return queue_pairs ? VIRTIO_NET_OK : VIRTIO_NET_ERR; 1387 } 1388 if (cmd == VIRTIO_NET_CTRL_MQ_RSS_CONFIG) { 1389 queue_pairs = virtio_net_handle_rss(n, iov, iov_cnt, true); 1390 } else if (cmd == VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET) { 1391 struct virtio_net_ctrl_mq mq; 1392 size_t s; 1393 if (!virtio_vdev_has_feature(vdev, VIRTIO_NET_F_MQ)) { 1394 return VIRTIO_NET_ERR; 1395 } 1396 s = iov_to_buf(iov, iov_cnt, 0, &mq, sizeof(mq)); 1397 if (s != sizeof(mq)) { 1398 return VIRTIO_NET_ERR; 1399 } 1400 queue_pairs = virtio_lduw_p(vdev, &mq.virtqueue_pairs); 1401 1402 } else { 1403 return VIRTIO_NET_ERR; 1404 } 1405 1406 if (queue_pairs < VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MIN || 1407 queue_pairs > VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MAX || 1408 queue_pairs > n->max_queue_pairs || 1409 !n->multiqueue) { 1410 return VIRTIO_NET_ERR; 1411 } 1412 1413 n->curr_queue_pairs = queue_pairs; 1414 /* stop the backend before changing the number of queue_pairs to avoid handling a 1415 * disabled queue */ 1416 virtio_net_set_status(vdev, vdev->status); 1417 virtio_net_set_queue_pairs(n); 1418 1419 return VIRTIO_NET_OK; 1420 } 1421 1422 static void virtio_net_handle_ctrl(VirtIODevice *vdev, VirtQueue *vq) 1423 { 1424 VirtIONet *n = VIRTIO_NET(vdev); 1425 struct virtio_net_ctrl_hdr ctrl; 1426 virtio_net_ctrl_ack status = VIRTIO_NET_ERR; 1427 VirtQueueElement *elem; 1428 size_t s; 1429 struct iovec *iov, *iov2; 1430 unsigned int iov_cnt; 1431 1432 for (;;) { 1433 elem = virtqueue_pop(vq, sizeof(VirtQueueElement)); 1434 if (!elem) { 1435 break; 1436 } 1437 if (iov_size(elem->in_sg, elem->in_num) < sizeof(status) || 1438 iov_size(elem->out_sg, elem->out_num) < sizeof(ctrl)) { 1439 virtio_error(vdev, "virtio-net ctrl missing headers"); 1440 virtqueue_detach_element(vq, elem, 0); 1441 g_free(elem); 1442 break; 1443 } 1444 1445 iov_cnt = elem->out_num; 1446 iov2 = iov = g_memdup(elem->out_sg, sizeof(struct iovec) * elem->out_num); 1447 s = iov_to_buf(iov, iov_cnt, 0, &ctrl, sizeof(ctrl)); 1448 iov_discard_front(&iov, &iov_cnt, sizeof(ctrl)); 1449 if (s != sizeof(ctrl)) { 1450 status = VIRTIO_NET_ERR; 1451 } else if (ctrl.class == VIRTIO_NET_CTRL_RX) { 1452 status = virtio_net_handle_rx_mode(n, ctrl.cmd, iov, iov_cnt); 1453 } else if (ctrl.class == VIRTIO_NET_CTRL_MAC) { 1454 status = virtio_net_handle_mac(n, ctrl.cmd, iov, iov_cnt); 1455 } else if (ctrl.class == VIRTIO_NET_CTRL_VLAN) { 1456 status = virtio_net_handle_vlan_table(n, ctrl.cmd, iov, iov_cnt); 1457 } else if (ctrl.class == VIRTIO_NET_CTRL_ANNOUNCE) { 1458 status = virtio_net_handle_announce(n, ctrl.cmd, iov, iov_cnt); 1459 } else if (ctrl.class == VIRTIO_NET_CTRL_MQ) { 1460 status = virtio_net_handle_mq(n, ctrl.cmd, iov, iov_cnt); 1461 } else if (ctrl.class == VIRTIO_NET_CTRL_GUEST_OFFLOADS) { 1462 status = virtio_net_handle_offloads(n, ctrl.cmd, iov, iov_cnt); 1463 } 1464 1465 s = iov_from_buf(elem->in_sg, elem->in_num, 0, &status, sizeof(status)); 1466 assert(s == sizeof(status)); 1467 1468 virtqueue_push(vq, elem, sizeof(status)); 1469 virtio_notify(vdev, vq); 1470 g_free(iov2); 1471 g_free(elem); 1472 } 1473 } 1474 1475 /* RX */ 1476 1477 static void virtio_net_handle_rx(VirtIODevice *vdev, VirtQueue *vq) 1478 { 1479 VirtIONet *n = VIRTIO_NET(vdev); 1480 int queue_index = vq2q(virtio_get_queue_index(vq)); 1481 1482 qemu_flush_queued_packets(qemu_get_subqueue(n->nic, queue_index)); 1483 } 1484 1485 static bool virtio_net_can_receive(NetClientState *nc) 1486 { 1487 VirtIONet *n = qemu_get_nic_opaque(nc); 1488 VirtIODevice *vdev = VIRTIO_DEVICE(n); 1489 VirtIONetQueue *q = virtio_net_get_subqueue(nc); 1490 1491 if (!vdev->vm_running) { 1492 return false; 1493 } 1494 1495 if (nc->queue_index >= n->curr_queue_pairs) { 1496 return false; 1497 } 1498 1499 if (!virtio_queue_ready(q->rx_vq) || 1500 !(vdev->status & VIRTIO_CONFIG_S_DRIVER_OK)) { 1501 return false; 1502 } 1503 1504 return true; 1505 } 1506 1507 static int virtio_net_has_buffers(VirtIONetQueue *q, int bufsize) 1508 { 1509 VirtIONet *n = q->n; 1510 if (virtio_queue_empty(q->rx_vq) || 1511 (n->mergeable_rx_bufs && 1512 !virtqueue_avail_bytes(q->rx_vq, bufsize, 0))) { 1513 virtio_queue_set_notification(q->rx_vq, 1); 1514 1515 /* To avoid a race condition where the guest has made some buffers 1516 * available after the above check but before notification was 1517 * enabled, check for available buffers again. 1518 */ 1519 if (virtio_queue_empty(q->rx_vq) || 1520 (n->mergeable_rx_bufs && 1521 !virtqueue_avail_bytes(q->rx_vq, bufsize, 0))) { 1522 return 0; 1523 } 1524 } 1525 1526 virtio_queue_set_notification(q->rx_vq, 0); 1527 return 1; 1528 } 1529 1530 static void virtio_net_hdr_swap(VirtIODevice *vdev, struct virtio_net_hdr *hdr) 1531 { 1532 virtio_tswap16s(vdev, &hdr->hdr_len); 1533 virtio_tswap16s(vdev, &hdr->gso_size); 1534 virtio_tswap16s(vdev, &hdr->csum_start); 1535 virtio_tswap16s(vdev, &hdr->csum_offset); 1536 } 1537 1538 /* dhclient uses AF_PACKET but doesn't pass auxdata to the kernel so 1539 * it never finds out that the packets don't have valid checksums. This 1540 * causes dhclient to get upset. Fedora's carried a patch for ages to 1541 * fix this with Xen but it hasn't appeared in an upstream release of 1542 * dhclient yet. 1543 * 1544 * To avoid breaking existing guests, we catch udp packets and add 1545 * checksums. This is terrible but it's better than hacking the guest 1546 * kernels. 1547 * 1548 * N.B. if we introduce a zero-copy API, this operation is no longer free so 1549 * we should provide a mechanism to disable it to avoid polluting the host 1550 * cache. 1551 */ 1552 static void work_around_broken_dhclient(struct virtio_net_hdr *hdr, 1553 uint8_t *buf, size_t size) 1554 { 1555 if ((hdr->flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) && /* missing csum */ 1556 (size > 27 && size < 1500) && /* normal sized MTU */ 1557 (buf[12] == 0x08 && buf[13] == 0x00) && /* ethertype == IPv4 */ 1558 (buf[23] == 17) && /* ip.protocol == UDP */ 1559 (buf[34] == 0 && buf[35] == 67)) { /* udp.srcport == bootps */ 1560 net_checksum_calculate(buf, size, CSUM_UDP); 1561 hdr->flags &= ~VIRTIO_NET_HDR_F_NEEDS_CSUM; 1562 } 1563 } 1564 1565 static void receive_header(VirtIONet *n, const struct iovec *iov, int iov_cnt, 1566 const void *buf, size_t size) 1567 { 1568 if (n->has_vnet_hdr) { 1569 /* FIXME this cast is evil */ 1570 void *wbuf = (void *)buf; 1571 work_around_broken_dhclient(wbuf, wbuf + n->host_hdr_len, 1572 size - n->host_hdr_len); 1573 1574 if (n->needs_vnet_hdr_swap) { 1575 virtio_net_hdr_swap(VIRTIO_DEVICE(n), wbuf); 1576 } 1577 iov_from_buf(iov, iov_cnt, 0, buf, sizeof(struct virtio_net_hdr)); 1578 } else { 1579 struct virtio_net_hdr hdr = { 1580 .flags = 0, 1581 .gso_type = VIRTIO_NET_HDR_GSO_NONE 1582 }; 1583 iov_from_buf(iov, iov_cnt, 0, &hdr, sizeof hdr); 1584 } 1585 } 1586 1587 static int receive_filter(VirtIONet *n, const uint8_t *buf, int size) 1588 { 1589 static const uint8_t bcast[] = {0xff, 0xff, 0xff, 0xff, 0xff, 0xff}; 1590 static const uint8_t vlan[] = {0x81, 0x00}; 1591 uint8_t *ptr = (uint8_t *)buf; 1592 int i; 1593 1594 if (n->promisc) 1595 return 1; 1596 1597 ptr += n->host_hdr_len; 1598 1599 if (!memcmp(&ptr[12], vlan, sizeof(vlan))) { 1600 int vid = lduw_be_p(ptr + 14) & 0xfff; 1601 if (!(n->vlans[vid >> 5] & (1U << (vid & 0x1f)))) 1602 return 0; 1603 } 1604 1605 if (ptr[0] & 1) { // multicast 1606 if (!memcmp(ptr, bcast, sizeof(bcast))) { 1607 return !n->nobcast; 1608 } else if (n->nomulti) { 1609 return 0; 1610 } else if (n->allmulti || n->mac_table.multi_overflow) { 1611 return 1; 1612 } 1613 1614 for (i = n->mac_table.first_multi; i < n->mac_table.in_use; i++) { 1615 if (!memcmp(ptr, &n->mac_table.macs[i * ETH_ALEN], ETH_ALEN)) { 1616 return 1; 1617 } 1618 } 1619 } else { // unicast 1620 if (n->nouni) { 1621 return 0; 1622 } else if (n->alluni || n->mac_table.uni_overflow) { 1623 return 1; 1624 } else if (!memcmp(ptr, n->mac, ETH_ALEN)) { 1625 return 1; 1626 } 1627 1628 for (i = 0; i < n->mac_table.first_multi; i++) { 1629 if (!memcmp(ptr, &n->mac_table.macs[i * ETH_ALEN], ETH_ALEN)) { 1630 return 1; 1631 } 1632 } 1633 } 1634 1635 return 0; 1636 } 1637 1638 static uint8_t virtio_net_get_hash_type(bool isip4, 1639 bool isip6, 1640 bool isudp, 1641 bool istcp, 1642 uint32_t types) 1643 { 1644 if (isip4) { 1645 if (istcp && (types & VIRTIO_NET_RSS_HASH_TYPE_TCPv4)) { 1646 return NetPktRssIpV4Tcp; 1647 } 1648 if (isudp && (types & VIRTIO_NET_RSS_HASH_TYPE_UDPv4)) { 1649 return NetPktRssIpV4Udp; 1650 } 1651 if (types & VIRTIO_NET_RSS_HASH_TYPE_IPv4) { 1652 return NetPktRssIpV4; 1653 } 1654 } else if (isip6) { 1655 uint32_t mask = VIRTIO_NET_RSS_HASH_TYPE_TCP_EX | 1656 VIRTIO_NET_RSS_HASH_TYPE_TCPv6; 1657 1658 if (istcp && (types & mask)) { 1659 return (types & VIRTIO_NET_RSS_HASH_TYPE_TCP_EX) ? 1660 NetPktRssIpV6TcpEx : NetPktRssIpV6Tcp; 1661 } 1662 mask = VIRTIO_NET_RSS_HASH_TYPE_UDP_EX | VIRTIO_NET_RSS_HASH_TYPE_UDPv6; 1663 if (isudp && (types & mask)) { 1664 return (types & VIRTIO_NET_RSS_HASH_TYPE_UDP_EX) ? 1665 NetPktRssIpV6UdpEx : NetPktRssIpV6Udp; 1666 } 1667 mask = VIRTIO_NET_RSS_HASH_TYPE_IP_EX | VIRTIO_NET_RSS_HASH_TYPE_IPv6; 1668 if (types & mask) { 1669 return (types & VIRTIO_NET_RSS_HASH_TYPE_IP_EX) ? 1670 NetPktRssIpV6Ex : NetPktRssIpV6; 1671 } 1672 } 1673 return 0xff; 1674 } 1675 1676 static void virtio_set_packet_hash(const uint8_t *buf, uint8_t report, 1677 uint32_t hash) 1678 { 1679 struct virtio_net_hdr_v1_hash *hdr = (void *)buf; 1680 hdr->hash_value = hash; 1681 hdr->hash_report = report; 1682 } 1683 1684 static int virtio_net_process_rss(NetClientState *nc, const uint8_t *buf, 1685 size_t size) 1686 { 1687 VirtIONet *n = qemu_get_nic_opaque(nc); 1688 unsigned int index = nc->queue_index, new_index = index; 1689 struct NetRxPkt *pkt = n->rx_pkt; 1690 uint8_t net_hash_type; 1691 uint32_t hash; 1692 bool isip4, isip6, isudp, istcp; 1693 static const uint8_t reports[NetPktRssIpV6UdpEx + 1] = { 1694 VIRTIO_NET_HASH_REPORT_IPv4, 1695 VIRTIO_NET_HASH_REPORT_TCPv4, 1696 VIRTIO_NET_HASH_REPORT_TCPv6, 1697 VIRTIO_NET_HASH_REPORT_IPv6, 1698 VIRTIO_NET_HASH_REPORT_IPv6_EX, 1699 VIRTIO_NET_HASH_REPORT_TCPv6_EX, 1700 VIRTIO_NET_HASH_REPORT_UDPv4, 1701 VIRTIO_NET_HASH_REPORT_UDPv6, 1702 VIRTIO_NET_HASH_REPORT_UDPv6_EX 1703 }; 1704 1705 net_rx_pkt_set_protocols(pkt, buf + n->host_hdr_len, 1706 size - n->host_hdr_len); 1707 net_rx_pkt_get_protocols(pkt, &isip4, &isip6, &isudp, &istcp); 1708 if (isip4 && (net_rx_pkt_get_ip4_info(pkt)->fragment)) { 1709 istcp = isudp = false; 1710 } 1711 if (isip6 && (net_rx_pkt_get_ip6_info(pkt)->fragment)) { 1712 istcp = isudp = false; 1713 } 1714 net_hash_type = virtio_net_get_hash_type(isip4, isip6, isudp, istcp, 1715 n->rss_data.hash_types); 1716 if (net_hash_type > NetPktRssIpV6UdpEx) { 1717 if (n->rss_data.populate_hash) { 1718 virtio_set_packet_hash(buf, VIRTIO_NET_HASH_REPORT_NONE, 0); 1719 } 1720 return n->rss_data.redirect ? n->rss_data.default_queue : -1; 1721 } 1722 1723 hash = net_rx_pkt_calc_rss_hash(pkt, net_hash_type, n->rss_data.key); 1724 1725 if (n->rss_data.populate_hash) { 1726 virtio_set_packet_hash(buf, reports[net_hash_type], hash); 1727 } 1728 1729 if (n->rss_data.redirect) { 1730 new_index = hash & (n->rss_data.indirections_len - 1); 1731 new_index = n->rss_data.indirections_table[new_index]; 1732 } 1733 1734 return (index == new_index) ? -1 : new_index; 1735 } 1736 1737 static ssize_t virtio_net_receive_rcu(NetClientState *nc, const uint8_t *buf, 1738 size_t size, bool no_rss) 1739 { 1740 VirtIONet *n = qemu_get_nic_opaque(nc); 1741 VirtIONetQueue *q = virtio_net_get_subqueue(nc); 1742 VirtIODevice *vdev = VIRTIO_DEVICE(n); 1743 VirtQueueElement *elems[VIRTQUEUE_MAX_SIZE]; 1744 size_t lens[VIRTQUEUE_MAX_SIZE]; 1745 struct iovec mhdr_sg[VIRTQUEUE_MAX_SIZE]; 1746 struct virtio_net_hdr_mrg_rxbuf mhdr; 1747 unsigned mhdr_cnt = 0; 1748 size_t offset, i, guest_offset, j; 1749 ssize_t err; 1750 1751 if (!virtio_net_can_receive(nc)) { 1752 return -1; 1753 } 1754 1755 if (!no_rss && n->rss_data.enabled && n->rss_data.enabled_software_rss) { 1756 int index = virtio_net_process_rss(nc, buf, size); 1757 if (index >= 0) { 1758 NetClientState *nc2 = qemu_get_subqueue(n->nic, index); 1759 return virtio_net_receive_rcu(nc2, buf, size, true); 1760 } 1761 } 1762 1763 /* hdr_len refers to the header we supply to the guest */ 1764 if (!virtio_net_has_buffers(q, size + n->guest_hdr_len - n->host_hdr_len)) { 1765 return 0; 1766 } 1767 1768 if (!receive_filter(n, buf, size)) 1769 return size; 1770 1771 offset = i = 0; 1772 1773 while (offset < size) { 1774 VirtQueueElement *elem; 1775 int len, total; 1776 const struct iovec *sg; 1777 1778 total = 0; 1779 1780 if (i == VIRTQUEUE_MAX_SIZE) { 1781 virtio_error(vdev, "virtio-net unexpected long buffer chain"); 1782 err = size; 1783 goto err; 1784 } 1785 1786 elem = virtqueue_pop(q->rx_vq, sizeof(VirtQueueElement)); 1787 if (!elem) { 1788 if (i) { 1789 virtio_error(vdev, "virtio-net unexpected empty queue: " 1790 "i %zd mergeable %d offset %zd, size %zd, " 1791 "guest hdr len %zd, host hdr len %zd " 1792 "guest features 0x%" PRIx64, 1793 i, n->mergeable_rx_bufs, offset, size, 1794 n->guest_hdr_len, n->host_hdr_len, 1795 vdev->guest_features); 1796 } 1797 err = -1; 1798 goto err; 1799 } 1800 1801 if (elem->in_num < 1) { 1802 virtio_error(vdev, 1803 "virtio-net receive queue contains no in buffers"); 1804 virtqueue_detach_element(q->rx_vq, elem, 0); 1805 g_free(elem); 1806 err = -1; 1807 goto err; 1808 } 1809 1810 sg = elem->in_sg; 1811 if (i == 0) { 1812 assert(offset == 0); 1813 if (n->mergeable_rx_bufs) { 1814 mhdr_cnt = iov_copy(mhdr_sg, ARRAY_SIZE(mhdr_sg), 1815 sg, elem->in_num, 1816 offsetof(typeof(mhdr), num_buffers), 1817 sizeof(mhdr.num_buffers)); 1818 } 1819 1820 receive_header(n, sg, elem->in_num, buf, size); 1821 if (n->rss_data.populate_hash) { 1822 offset = sizeof(mhdr); 1823 iov_from_buf(sg, elem->in_num, offset, 1824 buf + offset, n->host_hdr_len - sizeof(mhdr)); 1825 } 1826 offset = n->host_hdr_len; 1827 total += n->guest_hdr_len; 1828 guest_offset = n->guest_hdr_len; 1829 } else { 1830 guest_offset = 0; 1831 } 1832 1833 /* copy in packet. ugh */ 1834 len = iov_from_buf(sg, elem->in_num, guest_offset, 1835 buf + offset, size - offset); 1836 total += len; 1837 offset += len; 1838 /* If buffers can't be merged, at this point we 1839 * must have consumed the complete packet. 1840 * Otherwise, drop it. */ 1841 if (!n->mergeable_rx_bufs && offset < size) { 1842 virtqueue_unpop(q->rx_vq, elem, total); 1843 g_free(elem); 1844 err = size; 1845 goto err; 1846 } 1847 1848 elems[i] = elem; 1849 lens[i] = total; 1850 i++; 1851 } 1852 1853 if (mhdr_cnt) { 1854 virtio_stw_p(vdev, &mhdr.num_buffers, i); 1855 iov_from_buf(mhdr_sg, mhdr_cnt, 1856 0, 1857 &mhdr.num_buffers, sizeof mhdr.num_buffers); 1858 } 1859 1860 for (j = 0; j < i; j++) { 1861 /* signal other side */ 1862 virtqueue_fill(q->rx_vq, elems[j], lens[j], j); 1863 g_free(elems[j]); 1864 } 1865 1866 virtqueue_flush(q->rx_vq, i); 1867 virtio_notify(vdev, q->rx_vq); 1868 1869 return size; 1870 1871 err: 1872 for (j = 0; j < i; j++) { 1873 g_free(elems[j]); 1874 } 1875 1876 return err; 1877 } 1878 1879 static ssize_t virtio_net_do_receive(NetClientState *nc, const uint8_t *buf, 1880 size_t size) 1881 { 1882 RCU_READ_LOCK_GUARD(); 1883 1884 return virtio_net_receive_rcu(nc, buf, size, false); 1885 } 1886 1887 static void virtio_net_rsc_extract_unit4(VirtioNetRscChain *chain, 1888 const uint8_t *buf, 1889 VirtioNetRscUnit *unit) 1890 { 1891 uint16_t ip_hdrlen; 1892 struct ip_header *ip; 1893 1894 ip = (struct ip_header *)(buf + chain->n->guest_hdr_len 1895 + sizeof(struct eth_header)); 1896 unit->ip = (void *)ip; 1897 ip_hdrlen = (ip->ip_ver_len & 0xF) << 2; 1898 unit->ip_plen = &ip->ip_len; 1899 unit->tcp = (struct tcp_header *)(((uint8_t *)unit->ip) + ip_hdrlen); 1900 unit->tcp_hdrlen = (htons(unit->tcp->th_offset_flags) & 0xF000) >> 10; 1901 unit->payload = htons(*unit->ip_plen) - ip_hdrlen - unit->tcp_hdrlen; 1902 } 1903 1904 static void virtio_net_rsc_extract_unit6(VirtioNetRscChain *chain, 1905 const uint8_t *buf, 1906 VirtioNetRscUnit *unit) 1907 { 1908 struct ip6_header *ip6; 1909 1910 ip6 = (struct ip6_header *)(buf + chain->n->guest_hdr_len 1911 + sizeof(struct eth_header)); 1912 unit->ip = ip6; 1913 unit->ip_plen = &(ip6->ip6_ctlun.ip6_un1.ip6_un1_plen); 1914 unit->tcp = (struct tcp_header *)(((uint8_t *)unit->ip) 1915 + sizeof(struct ip6_header)); 1916 unit->tcp_hdrlen = (htons(unit->tcp->th_offset_flags) & 0xF000) >> 10; 1917 1918 /* There is a difference between payload lenght in ipv4 and v6, 1919 ip header is excluded in ipv6 */ 1920 unit->payload = htons(*unit->ip_plen) - unit->tcp_hdrlen; 1921 } 1922 1923 static size_t virtio_net_rsc_drain_seg(VirtioNetRscChain *chain, 1924 VirtioNetRscSeg *seg) 1925 { 1926 int ret; 1927 struct virtio_net_hdr_v1 *h; 1928 1929 h = (struct virtio_net_hdr_v1 *)seg->buf; 1930 h->flags = 0; 1931 h->gso_type = VIRTIO_NET_HDR_GSO_NONE; 1932 1933 if (seg->is_coalesced) { 1934 h->rsc.segments = seg->packets; 1935 h->rsc.dup_acks = seg->dup_ack; 1936 h->flags = VIRTIO_NET_HDR_F_RSC_INFO; 1937 if (chain->proto == ETH_P_IP) { 1938 h->gso_type = VIRTIO_NET_HDR_GSO_TCPV4; 1939 } else { 1940 h->gso_type = VIRTIO_NET_HDR_GSO_TCPV6; 1941 } 1942 } 1943 1944 ret = virtio_net_do_receive(seg->nc, seg->buf, seg->size); 1945 QTAILQ_REMOVE(&chain->buffers, seg, next); 1946 g_free(seg->buf); 1947 g_free(seg); 1948 1949 return ret; 1950 } 1951 1952 static void virtio_net_rsc_purge(void *opq) 1953 { 1954 VirtioNetRscSeg *seg, *rn; 1955 VirtioNetRscChain *chain = (VirtioNetRscChain *)opq; 1956 1957 QTAILQ_FOREACH_SAFE(seg, &chain->buffers, next, rn) { 1958 if (virtio_net_rsc_drain_seg(chain, seg) == 0) { 1959 chain->stat.purge_failed++; 1960 continue; 1961 } 1962 } 1963 1964 chain->stat.timer++; 1965 if (!QTAILQ_EMPTY(&chain->buffers)) { 1966 timer_mod(chain->drain_timer, 1967 qemu_clock_get_ns(QEMU_CLOCK_HOST) + chain->n->rsc_timeout); 1968 } 1969 } 1970 1971 static void virtio_net_rsc_cleanup(VirtIONet *n) 1972 { 1973 VirtioNetRscChain *chain, *rn_chain; 1974 VirtioNetRscSeg *seg, *rn_seg; 1975 1976 QTAILQ_FOREACH_SAFE(chain, &n->rsc_chains, next, rn_chain) { 1977 QTAILQ_FOREACH_SAFE(seg, &chain->buffers, next, rn_seg) { 1978 QTAILQ_REMOVE(&chain->buffers, seg, next); 1979 g_free(seg->buf); 1980 g_free(seg); 1981 } 1982 1983 timer_free(chain->drain_timer); 1984 QTAILQ_REMOVE(&n->rsc_chains, chain, next); 1985 g_free(chain); 1986 } 1987 } 1988 1989 static void virtio_net_rsc_cache_buf(VirtioNetRscChain *chain, 1990 NetClientState *nc, 1991 const uint8_t *buf, size_t size) 1992 { 1993 uint16_t hdr_len; 1994 VirtioNetRscSeg *seg; 1995 1996 hdr_len = chain->n->guest_hdr_len; 1997 seg = g_malloc(sizeof(VirtioNetRscSeg)); 1998 seg->buf = g_malloc(hdr_len + sizeof(struct eth_header) 1999 + sizeof(struct ip6_header) + VIRTIO_NET_MAX_TCP_PAYLOAD); 2000 memcpy(seg->buf, buf, size); 2001 seg->size = size; 2002 seg->packets = 1; 2003 seg->dup_ack = 0; 2004 seg->is_coalesced = 0; 2005 seg->nc = nc; 2006 2007 QTAILQ_INSERT_TAIL(&chain->buffers, seg, next); 2008 chain->stat.cache++; 2009 2010 switch (chain->proto) { 2011 case ETH_P_IP: 2012 virtio_net_rsc_extract_unit4(chain, seg->buf, &seg->unit); 2013 break; 2014 case ETH_P_IPV6: 2015 virtio_net_rsc_extract_unit6(chain, seg->buf, &seg->unit); 2016 break; 2017 default: 2018 g_assert_not_reached(); 2019 } 2020 } 2021 2022 static int32_t virtio_net_rsc_handle_ack(VirtioNetRscChain *chain, 2023 VirtioNetRscSeg *seg, 2024 const uint8_t *buf, 2025 struct tcp_header *n_tcp, 2026 struct tcp_header *o_tcp) 2027 { 2028 uint32_t nack, oack; 2029 uint16_t nwin, owin; 2030 2031 nack = htonl(n_tcp->th_ack); 2032 nwin = htons(n_tcp->th_win); 2033 oack = htonl(o_tcp->th_ack); 2034 owin = htons(o_tcp->th_win); 2035 2036 if ((nack - oack) >= VIRTIO_NET_MAX_TCP_PAYLOAD) { 2037 chain->stat.ack_out_of_win++; 2038 return RSC_FINAL; 2039 } else if (nack == oack) { 2040 /* duplicated ack or window probe */ 2041 if (nwin == owin) { 2042 /* duplicated ack, add dup ack count due to whql test up to 1 */ 2043 chain->stat.dup_ack++; 2044 return RSC_FINAL; 2045 } else { 2046 /* Coalesce window update */ 2047 o_tcp->th_win = n_tcp->th_win; 2048 chain->stat.win_update++; 2049 return RSC_COALESCE; 2050 } 2051 } else { 2052 /* pure ack, go to 'C', finalize*/ 2053 chain->stat.pure_ack++; 2054 return RSC_FINAL; 2055 } 2056 } 2057 2058 static int32_t virtio_net_rsc_coalesce_data(VirtioNetRscChain *chain, 2059 VirtioNetRscSeg *seg, 2060 const uint8_t *buf, 2061 VirtioNetRscUnit *n_unit) 2062 { 2063 void *data; 2064 uint16_t o_ip_len; 2065 uint32_t nseq, oseq; 2066 VirtioNetRscUnit *o_unit; 2067 2068 o_unit = &seg->unit; 2069 o_ip_len = htons(*o_unit->ip_plen); 2070 nseq = htonl(n_unit->tcp->th_seq); 2071 oseq = htonl(o_unit->tcp->th_seq); 2072 2073 /* out of order or retransmitted. */ 2074 if ((nseq - oseq) > VIRTIO_NET_MAX_TCP_PAYLOAD) { 2075 chain->stat.data_out_of_win++; 2076 return RSC_FINAL; 2077 } 2078 2079 data = ((uint8_t *)n_unit->tcp) + n_unit->tcp_hdrlen; 2080 if (nseq == oseq) { 2081 if ((o_unit->payload == 0) && n_unit->payload) { 2082 /* From no payload to payload, normal case, not a dup ack or etc */ 2083 chain->stat.data_after_pure_ack++; 2084 goto coalesce; 2085 } else { 2086 return virtio_net_rsc_handle_ack(chain, seg, buf, 2087 n_unit->tcp, o_unit->tcp); 2088 } 2089 } else if ((nseq - oseq) != o_unit->payload) { 2090 /* Not a consistent packet, out of order */ 2091 chain->stat.data_out_of_order++; 2092 return RSC_FINAL; 2093 } else { 2094 coalesce: 2095 if ((o_ip_len + n_unit->payload) > chain->max_payload) { 2096 chain->stat.over_size++; 2097 return RSC_FINAL; 2098 } 2099 2100 /* Here comes the right data, the payload length in v4/v6 is different, 2101 so use the field value to update and record the new data len */ 2102 o_unit->payload += n_unit->payload; /* update new data len */ 2103 2104 /* update field in ip header */ 2105 *o_unit->ip_plen = htons(o_ip_len + n_unit->payload); 2106 2107 /* Bring 'PUSH' big, the whql test guide says 'PUSH' can be coalesced 2108 for windows guest, while this may change the behavior for linux 2109 guest (only if it uses RSC feature). */ 2110 o_unit->tcp->th_offset_flags = n_unit->tcp->th_offset_flags; 2111 2112 o_unit->tcp->th_ack = n_unit->tcp->th_ack; 2113 o_unit->tcp->th_win = n_unit->tcp->th_win; 2114 2115 memmove(seg->buf + seg->size, data, n_unit->payload); 2116 seg->size += n_unit->payload; 2117 seg->packets++; 2118 chain->stat.coalesced++; 2119 return RSC_COALESCE; 2120 } 2121 } 2122 2123 static int32_t virtio_net_rsc_coalesce4(VirtioNetRscChain *chain, 2124 VirtioNetRscSeg *seg, 2125 const uint8_t *buf, size_t size, 2126 VirtioNetRscUnit *unit) 2127 { 2128 struct ip_header *ip1, *ip2; 2129 2130 ip1 = (struct ip_header *)(unit->ip); 2131 ip2 = (struct ip_header *)(seg->unit.ip); 2132 if ((ip1->ip_src ^ ip2->ip_src) || (ip1->ip_dst ^ ip2->ip_dst) 2133 || (unit->tcp->th_sport ^ seg->unit.tcp->th_sport) 2134 || (unit->tcp->th_dport ^ seg->unit.tcp->th_dport)) { 2135 chain->stat.no_match++; 2136 return RSC_NO_MATCH; 2137 } 2138 2139 return virtio_net_rsc_coalesce_data(chain, seg, buf, unit); 2140 } 2141 2142 static int32_t virtio_net_rsc_coalesce6(VirtioNetRscChain *chain, 2143 VirtioNetRscSeg *seg, 2144 const uint8_t *buf, size_t size, 2145 VirtioNetRscUnit *unit) 2146 { 2147 struct ip6_header *ip1, *ip2; 2148 2149 ip1 = (struct ip6_header *)(unit->ip); 2150 ip2 = (struct ip6_header *)(seg->unit.ip); 2151 if (memcmp(&ip1->ip6_src, &ip2->ip6_src, sizeof(struct in6_address)) 2152 || memcmp(&ip1->ip6_dst, &ip2->ip6_dst, sizeof(struct in6_address)) 2153 || (unit->tcp->th_sport ^ seg->unit.tcp->th_sport) 2154 || (unit->tcp->th_dport ^ seg->unit.tcp->th_dport)) { 2155 chain->stat.no_match++; 2156 return RSC_NO_MATCH; 2157 } 2158 2159 return virtio_net_rsc_coalesce_data(chain, seg, buf, unit); 2160 } 2161 2162 /* Packets with 'SYN' should bypass, other flag should be sent after drain 2163 * to prevent out of order */ 2164 static int virtio_net_rsc_tcp_ctrl_check(VirtioNetRscChain *chain, 2165 struct tcp_header *tcp) 2166 { 2167 uint16_t tcp_hdr; 2168 uint16_t tcp_flag; 2169 2170 tcp_flag = htons(tcp->th_offset_flags); 2171 tcp_hdr = (tcp_flag & VIRTIO_NET_TCP_HDR_LENGTH) >> 10; 2172 tcp_flag &= VIRTIO_NET_TCP_FLAG; 2173 if (tcp_flag & TH_SYN) { 2174 chain->stat.tcp_syn++; 2175 return RSC_BYPASS; 2176 } 2177 2178 if (tcp_flag & (TH_FIN | TH_URG | TH_RST | TH_ECE | TH_CWR)) { 2179 chain->stat.tcp_ctrl_drain++; 2180 return RSC_FINAL; 2181 } 2182 2183 if (tcp_hdr > sizeof(struct tcp_header)) { 2184 chain->stat.tcp_all_opt++; 2185 return RSC_FINAL; 2186 } 2187 2188 return RSC_CANDIDATE; 2189 } 2190 2191 static size_t virtio_net_rsc_do_coalesce(VirtioNetRscChain *chain, 2192 NetClientState *nc, 2193 const uint8_t *buf, size_t size, 2194 VirtioNetRscUnit *unit) 2195 { 2196 int ret; 2197 VirtioNetRscSeg *seg, *nseg; 2198 2199 if (QTAILQ_EMPTY(&chain->buffers)) { 2200 chain->stat.empty_cache++; 2201 virtio_net_rsc_cache_buf(chain, nc, buf, size); 2202 timer_mod(chain->drain_timer, 2203 qemu_clock_get_ns(QEMU_CLOCK_HOST) + chain->n->rsc_timeout); 2204 return size; 2205 } 2206 2207 QTAILQ_FOREACH_SAFE(seg, &chain->buffers, next, nseg) { 2208 if (chain->proto == ETH_P_IP) { 2209 ret = virtio_net_rsc_coalesce4(chain, seg, buf, size, unit); 2210 } else { 2211 ret = virtio_net_rsc_coalesce6(chain, seg, buf, size, unit); 2212 } 2213 2214 if (ret == RSC_FINAL) { 2215 if (virtio_net_rsc_drain_seg(chain, seg) == 0) { 2216 /* Send failed */ 2217 chain->stat.final_failed++; 2218 return 0; 2219 } 2220 2221 /* Send current packet */ 2222 return virtio_net_do_receive(nc, buf, size); 2223 } else if (ret == RSC_NO_MATCH) { 2224 continue; 2225 } else { 2226 /* Coalesced, mark coalesced flag to tell calc cksum for ipv4 */ 2227 seg->is_coalesced = 1; 2228 return size; 2229 } 2230 } 2231 2232 chain->stat.no_match_cache++; 2233 virtio_net_rsc_cache_buf(chain, nc, buf, size); 2234 return size; 2235 } 2236 2237 /* Drain a connection data, this is to avoid out of order segments */ 2238 static size_t virtio_net_rsc_drain_flow(VirtioNetRscChain *chain, 2239 NetClientState *nc, 2240 const uint8_t *buf, size_t size, 2241 uint16_t ip_start, uint16_t ip_size, 2242 uint16_t tcp_port) 2243 { 2244 VirtioNetRscSeg *seg, *nseg; 2245 uint32_t ppair1, ppair2; 2246 2247 ppair1 = *(uint32_t *)(buf + tcp_port); 2248 QTAILQ_FOREACH_SAFE(seg, &chain->buffers, next, nseg) { 2249 ppair2 = *(uint32_t *)(seg->buf + tcp_port); 2250 if (memcmp(buf + ip_start, seg->buf + ip_start, ip_size) 2251 || (ppair1 != ppair2)) { 2252 continue; 2253 } 2254 if (virtio_net_rsc_drain_seg(chain, seg) == 0) { 2255 chain->stat.drain_failed++; 2256 } 2257 2258 break; 2259 } 2260 2261 return virtio_net_do_receive(nc, buf, size); 2262 } 2263 2264 static int32_t virtio_net_rsc_sanity_check4(VirtioNetRscChain *chain, 2265 struct ip_header *ip, 2266 const uint8_t *buf, size_t size) 2267 { 2268 uint16_t ip_len; 2269 2270 /* Not an ipv4 packet */ 2271 if (((ip->ip_ver_len & 0xF0) >> 4) != IP_HEADER_VERSION_4) { 2272 chain->stat.ip_option++; 2273 return RSC_BYPASS; 2274 } 2275 2276 /* Don't handle packets with ip option */ 2277 if ((ip->ip_ver_len & 0xF) != VIRTIO_NET_IP4_HEADER_LENGTH) { 2278 chain->stat.ip_option++; 2279 return RSC_BYPASS; 2280 } 2281 2282 if (ip->ip_p != IPPROTO_TCP) { 2283 chain->stat.bypass_not_tcp++; 2284 return RSC_BYPASS; 2285 } 2286 2287 /* Don't handle packets with ip fragment */ 2288 if (!(htons(ip->ip_off) & IP_DF)) { 2289 chain->stat.ip_frag++; 2290 return RSC_BYPASS; 2291 } 2292 2293 /* Don't handle packets with ecn flag */ 2294 if (IPTOS_ECN(ip->ip_tos)) { 2295 chain->stat.ip_ecn++; 2296 return RSC_BYPASS; 2297 } 2298 2299 ip_len = htons(ip->ip_len); 2300 if (ip_len < (sizeof(struct ip_header) + sizeof(struct tcp_header)) 2301 || ip_len > (size - chain->n->guest_hdr_len - 2302 sizeof(struct eth_header))) { 2303 chain->stat.ip_hacked++; 2304 return RSC_BYPASS; 2305 } 2306 2307 return RSC_CANDIDATE; 2308 } 2309 2310 static size_t virtio_net_rsc_receive4(VirtioNetRscChain *chain, 2311 NetClientState *nc, 2312 const uint8_t *buf, size_t size) 2313 { 2314 int32_t ret; 2315 uint16_t hdr_len; 2316 VirtioNetRscUnit unit; 2317 2318 hdr_len = ((VirtIONet *)(chain->n))->guest_hdr_len; 2319 2320 if (size < (hdr_len + sizeof(struct eth_header) + sizeof(struct ip_header) 2321 + sizeof(struct tcp_header))) { 2322 chain->stat.bypass_not_tcp++; 2323 return virtio_net_do_receive(nc, buf, size); 2324 } 2325 2326 virtio_net_rsc_extract_unit4(chain, buf, &unit); 2327 if (virtio_net_rsc_sanity_check4(chain, unit.ip, buf, size) 2328 != RSC_CANDIDATE) { 2329 return virtio_net_do_receive(nc, buf, size); 2330 } 2331 2332 ret = virtio_net_rsc_tcp_ctrl_check(chain, unit.tcp); 2333 if (ret == RSC_BYPASS) { 2334 return virtio_net_do_receive(nc, buf, size); 2335 } else if (ret == RSC_FINAL) { 2336 return virtio_net_rsc_drain_flow(chain, nc, buf, size, 2337 ((hdr_len + sizeof(struct eth_header)) + 12), 2338 VIRTIO_NET_IP4_ADDR_SIZE, 2339 hdr_len + sizeof(struct eth_header) + sizeof(struct ip_header)); 2340 } 2341 2342 return virtio_net_rsc_do_coalesce(chain, nc, buf, size, &unit); 2343 } 2344 2345 static int32_t virtio_net_rsc_sanity_check6(VirtioNetRscChain *chain, 2346 struct ip6_header *ip6, 2347 const uint8_t *buf, size_t size) 2348 { 2349 uint16_t ip_len; 2350 2351 if (((ip6->ip6_ctlun.ip6_un1.ip6_un1_flow & 0xF0) >> 4) 2352 != IP_HEADER_VERSION_6) { 2353 return RSC_BYPASS; 2354 } 2355 2356 /* Both option and protocol is checked in this */ 2357 if (ip6->ip6_ctlun.ip6_un1.ip6_un1_nxt != IPPROTO_TCP) { 2358 chain->stat.bypass_not_tcp++; 2359 return RSC_BYPASS; 2360 } 2361 2362 ip_len = htons(ip6->ip6_ctlun.ip6_un1.ip6_un1_plen); 2363 if (ip_len < sizeof(struct tcp_header) || 2364 ip_len > (size - chain->n->guest_hdr_len - sizeof(struct eth_header) 2365 - sizeof(struct ip6_header))) { 2366 chain->stat.ip_hacked++; 2367 return RSC_BYPASS; 2368 } 2369 2370 /* Don't handle packets with ecn flag */ 2371 if (IP6_ECN(ip6->ip6_ctlun.ip6_un3.ip6_un3_ecn)) { 2372 chain->stat.ip_ecn++; 2373 return RSC_BYPASS; 2374 } 2375 2376 return RSC_CANDIDATE; 2377 } 2378 2379 static size_t virtio_net_rsc_receive6(void *opq, NetClientState *nc, 2380 const uint8_t *buf, size_t size) 2381 { 2382 int32_t ret; 2383 uint16_t hdr_len; 2384 VirtioNetRscChain *chain; 2385 VirtioNetRscUnit unit; 2386 2387 chain = (VirtioNetRscChain *)opq; 2388 hdr_len = ((VirtIONet *)(chain->n))->guest_hdr_len; 2389 2390 if (size < (hdr_len + sizeof(struct eth_header) + sizeof(struct ip6_header) 2391 + sizeof(tcp_header))) { 2392 return virtio_net_do_receive(nc, buf, size); 2393 } 2394 2395 virtio_net_rsc_extract_unit6(chain, buf, &unit); 2396 if (RSC_CANDIDATE != virtio_net_rsc_sanity_check6(chain, 2397 unit.ip, buf, size)) { 2398 return virtio_net_do_receive(nc, buf, size); 2399 } 2400 2401 ret = virtio_net_rsc_tcp_ctrl_check(chain, unit.tcp); 2402 if (ret == RSC_BYPASS) { 2403 return virtio_net_do_receive(nc, buf, size); 2404 } else if (ret == RSC_FINAL) { 2405 return virtio_net_rsc_drain_flow(chain, nc, buf, size, 2406 ((hdr_len + sizeof(struct eth_header)) + 8), 2407 VIRTIO_NET_IP6_ADDR_SIZE, 2408 hdr_len + sizeof(struct eth_header) 2409 + sizeof(struct ip6_header)); 2410 } 2411 2412 return virtio_net_rsc_do_coalesce(chain, nc, buf, size, &unit); 2413 } 2414 2415 static VirtioNetRscChain *virtio_net_rsc_lookup_chain(VirtIONet *n, 2416 NetClientState *nc, 2417 uint16_t proto) 2418 { 2419 VirtioNetRscChain *chain; 2420 2421 if ((proto != (uint16_t)ETH_P_IP) && (proto != (uint16_t)ETH_P_IPV6)) { 2422 return NULL; 2423 } 2424 2425 QTAILQ_FOREACH(chain, &n->rsc_chains, next) { 2426 if (chain->proto == proto) { 2427 return chain; 2428 } 2429 } 2430 2431 chain = g_malloc(sizeof(*chain)); 2432 chain->n = n; 2433 chain->proto = proto; 2434 if (proto == (uint16_t)ETH_P_IP) { 2435 chain->max_payload = VIRTIO_NET_MAX_IP4_PAYLOAD; 2436 chain->gso_type = VIRTIO_NET_HDR_GSO_TCPV4; 2437 } else { 2438 chain->max_payload = VIRTIO_NET_MAX_IP6_PAYLOAD; 2439 chain->gso_type = VIRTIO_NET_HDR_GSO_TCPV6; 2440 } 2441 chain->drain_timer = timer_new_ns(QEMU_CLOCK_HOST, 2442 virtio_net_rsc_purge, chain); 2443 memset(&chain->stat, 0, sizeof(chain->stat)); 2444 2445 QTAILQ_INIT(&chain->buffers); 2446 QTAILQ_INSERT_TAIL(&n->rsc_chains, chain, next); 2447 2448 return chain; 2449 } 2450 2451 static ssize_t virtio_net_rsc_receive(NetClientState *nc, 2452 const uint8_t *buf, 2453 size_t size) 2454 { 2455 uint16_t proto; 2456 VirtioNetRscChain *chain; 2457 struct eth_header *eth; 2458 VirtIONet *n; 2459 2460 n = qemu_get_nic_opaque(nc); 2461 if (size < (n->host_hdr_len + sizeof(struct eth_header))) { 2462 return virtio_net_do_receive(nc, buf, size); 2463 } 2464 2465 eth = (struct eth_header *)(buf + n->guest_hdr_len); 2466 proto = htons(eth->h_proto); 2467 2468 chain = virtio_net_rsc_lookup_chain(n, nc, proto); 2469 if (chain) { 2470 chain->stat.received++; 2471 if (proto == (uint16_t)ETH_P_IP && n->rsc4_enabled) { 2472 return virtio_net_rsc_receive4(chain, nc, buf, size); 2473 } else if (proto == (uint16_t)ETH_P_IPV6 && n->rsc6_enabled) { 2474 return virtio_net_rsc_receive6(chain, nc, buf, size); 2475 } 2476 } 2477 return virtio_net_do_receive(nc, buf, size); 2478 } 2479 2480 static ssize_t virtio_net_receive(NetClientState *nc, const uint8_t *buf, 2481 size_t size) 2482 { 2483 VirtIONet *n = qemu_get_nic_opaque(nc); 2484 if ((n->rsc4_enabled || n->rsc6_enabled)) { 2485 return virtio_net_rsc_receive(nc, buf, size); 2486 } else { 2487 return virtio_net_do_receive(nc, buf, size); 2488 } 2489 } 2490 2491 static int32_t virtio_net_flush_tx(VirtIONetQueue *q); 2492 2493 static void virtio_net_tx_complete(NetClientState *nc, ssize_t len) 2494 { 2495 VirtIONet *n = qemu_get_nic_opaque(nc); 2496 VirtIONetQueue *q = virtio_net_get_subqueue(nc); 2497 VirtIODevice *vdev = VIRTIO_DEVICE(n); 2498 2499 virtqueue_push(q->tx_vq, q->async_tx.elem, 0); 2500 virtio_notify(vdev, q->tx_vq); 2501 2502 g_free(q->async_tx.elem); 2503 q->async_tx.elem = NULL; 2504 2505 virtio_queue_set_notification(q->tx_vq, 1); 2506 virtio_net_flush_tx(q); 2507 } 2508 2509 /* TX */ 2510 static int32_t virtio_net_flush_tx(VirtIONetQueue *q) 2511 { 2512 VirtIONet *n = q->n; 2513 VirtIODevice *vdev = VIRTIO_DEVICE(n); 2514 VirtQueueElement *elem; 2515 int32_t num_packets = 0; 2516 int queue_index = vq2q(virtio_get_queue_index(q->tx_vq)); 2517 if (!(vdev->status & VIRTIO_CONFIG_S_DRIVER_OK)) { 2518 return num_packets; 2519 } 2520 2521 if (q->async_tx.elem) { 2522 virtio_queue_set_notification(q->tx_vq, 0); 2523 return num_packets; 2524 } 2525 2526 for (;;) { 2527 ssize_t ret; 2528 unsigned int out_num; 2529 struct iovec sg[VIRTQUEUE_MAX_SIZE], sg2[VIRTQUEUE_MAX_SIZE + 1], *out_sg; 2530 struct virtio_net_hdr_mrg_rxbuf mhdr; 2531 2532 elem = virtqueue_pop(q->tx_vq, sizeof(VirtQueueElement)); 2533 if (!elem) { 2534 break; 2535 } 2536 2537 out_num = elem->out_num; 2538 out_sg = elem->out_sg; 2539 if (out_num < 1) { 2540 virtio_error(vdev, "virtio-net header not in first element"); 2541 virtqueue_detach_element(q->tx_vq, elem, 0); 2542 g_free(elem); 2543 return -EINVAL; 2544 } 2545 2546 if (n->has_vnet_hdr) { 2547 if (iov_to_buf(out_sg, out_num, 0, &mhdr, n->guest_hdr_len) < 2548 n->guest_hdr_len) { 2549 virtio_error(vdev, "virtio-net header incorrect"); 2550 virtqueue_detach_element(q->tx_vq, elem, 0); 2551 g_free(elem); 2552 return -EINVAL; 2553 } 2554 if (n->needs_vnet_hdr_swap) { 2555 virtio_net_hdr_swap(vdev, (void *) &mhdr); 2556 sg2[0].iov_base = &mhdr; 2557 sg2[0].iov_len = n->guest_hdr_len; 2558 out_num = iov_copy(&sg2[1], ARRAY_SIZE(sg2) - 1, 2559 out_sg, out_num, 2560 n->guest_hdr_len, -1); 2561 if (out_num == VIRTQUEUE_MAX_SIZE) { 2562 goto drop; 2563 } 2564 out_num += 1; 2565 out_sg = sg2; 2566 } 2567 } 2568 /* 2569 * If host wants to see the guest header as is, we can 2570 * pass it on unchanged. Otherwise, copy just the parts 2571 * that host is interested in. 2572 */ 2573 assert(n->host_hdr_len <= n->guest_hdr_len); 2574 if (n->host_hdr_len != n->guest_hdr_len) { 2575 unsigned sg_num = iov_copy(sg, ARRAY_SIZE(sg), 2576 out_sg, out_num, 2577 0, n->host_hdr_len); 2578 sg_num += iov_copy(sg + sg_num, ARRAY_SIZE(sg) - sg_num, 2579 out_sg, out_num, 2580 n->guest_hdr_len, -1); 2581 out_num = sg_num; 2582 out_sg = sg; 2583 } 2584 2585 ret = qemu_sendv_packet_async(qemu_get_subqueue(n->nic, queue_index), 2586 out_sg, out_num, virtio_net_tx_complete); 2587 if (ret == 0) { 2588 virtio_queue_set_notification(q->tx_vq, 0); 2589 q->async_tx.elem = elem; 2590 return -EBUSY; 2591 } 2592 2593 drop: 2594 virtqueue_push(q->tx_vq, elem, 0); 2595 virtio_notify(vdev, q->tx_vq); 2596 g_free(elem); 2597 2598 if (++num_packets >= n->tx_burst) { 2599 break; 2600 } 2601 } 2602 return num_packets; 2603 } 2604 2605 static void virtio_net_handle_tx_timer(VirtIODevice *vdev, VirtQueue *vq) 2606 { 2607 VirtIONet *n = VIRTIO_NET(vdev); 2608 VirtIONetQueue *q = &n->vqs[vq2q(virtio_get_queue_index(vq))]; 2609 2610 if (unlikely((n->status & VIRTIO_NET_S_LINK_UP) == 0)) { 2611 virtio_net_drop_tx_queue_data(vdev, vq); 2612 return; 2613 } 2614 2615 /* This happens when device was stopped but VCPU wasn't. */ 2616 if (!vdev->vm_running) { 2617 q->tx_waiting = 1; 2618 return; 2619 } 2620 2621 if (q->tx_waiting) { 2622 virtio_queue_set_notification(vq, 1); 2623 timer_del(q->tx_timer); 2624 q->tx_waiting = 0; 2625 if (virtio_net_flush_tx(q) == -EINVAL) { 2626 return; 2627 } 2628 } else { 2629 timer_mod(q->tx_timer, 2630 qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) + n->tx_timeout); 2631 q->tx_waiting = 1; 2632 virtio_queue_set_notification(vq, 0); 2633 } 2634 } 2635 2636 static void virtio_net_handle_tx_bh(VirtIODevice *vdev, VirtQueue *vq) 2637 { 2638 VirtIONet *n = VIRTIO_NET(vdev); 2639 VirtIONetQueue *q = &n->vqs[vq2q(virtio_get_queue_index(vq))]; 2640 2641 if (unlikely((n->status & VIRTIO_NET_S_LINK_UP) == 0)) { 2642 virtio_net_drop_tx_queue_data(vdev, vq); 2643 return; 2644 } 2645 2646 if (unlikely(q->tx_waiting)) { 2647 return; 2648 } 2649 q->tx_waiting = 1; 2650 /* This happens when device was stopped but VCPU wasn't. */ 2651 if (!vdev->vm_running) { 2652 return; 2653 } 2654 virtio_queue_set_notification(vq, 0); 2655 qemu_bh_schedule(q->tx_bh); 2656 } 2657 2658 static void virtio_net_tx_timer(void *opaque) 2659 { 2660 VirtIONetQueue *q = opaque; 2661 VirtIONet *n = q->n; 2662 VirtIODevice *vdev = VIRTIO_DEVICE(n); 2663 /* This happens when device was stopped but BH wasn't. */ 2664 if (!vdev->vm_running) { 2665 /* Make sure tx waiting is set, so we'll run when restarted. */ 2666 assert(q->tx_waiting); 2667 return; 2668 } 2669 2670 q->tx_waiting = 0; 2671 2672 /* Just in case the driver is not ready on more */ 2673 if (!(vdev->status & VIRTIO_CONFIG_S_DRIVER_OK)) { 2674 return; 2675 } 2676 2677 virtio_queue_set_notification(q->tx_vq, 1); 2678 virtio_net_flush_tx(q); 2679 } 2680 2681 static void virtio_net_tx_bh(void *opaque) 2682 { 2683 VirtIONetQueue *q = opaque; 2684 VirtIONet *n = q->n; 2685 VirtIODevice *vdev = VIRTIO_DEVICE(n); 2686 int32_t ret; 2687 2688 /* This happens when device was stopped but BH wasn't. */ 2689 if (!vdev->vm_running) { 2690 /* Make sure tx waiting is set, so we'll run when restarted. */ 2691 assert(q->tx_waiting); 2692 return; 2693 } 2694 2695 q->tx_waiting = 0; 2696 2697 /* Just in case the driver is not ready on more */ 2698 if (unlikely(!(vdev->status & VIRTIO_CONFIG_S_DRIVER_OK))) { 2699 return; 2700 } 2701 2702 ret = virtio_net_flush_tx(q); 2703 if (ret == -EBUSY || ret == -EINVAL) { 2704 return; /* Notification re-enable handled by tx_complete or device 2705 * broken */ 2706 } 2707 2708 /* If we flush a full burst of packets, assume there are 2709 * more coming and immediately reschedule */ 2710 if (ret >= n->tx_burst) { 2711 qemu_bh_schedule(q->tx_bh); 2712 q->tx_waiting = 1; 2713 return; 2714 } 2715 2716 /* If less than a full burst, re-enable notification and flush 2717 * anything that may have come in while we weren't looking. If 2718 * we find something, assume the guest is still active and reschedule */ 2719 virtio_queue_set_notification(q->tx_vq, 1); 2720 ret = virtio_net_flush_tx(q); 2721 if (ret == -EINVAL) { 2722 return; 2723 } else if (ret > 0) { 2724 virtio_queue_set_notification(q->tx_vq, 0); 2725 qemu_bh_schedule(q->tx_bh); 2726 q->tx_waiting = 1; 2727 } 2728 } 2729 2730 static void virtio_net_add_queue(VirtIONet *n, int index) 2731 { 2732 VirtIODevice *vdev = VIRTIO_DEVICE(n); 2733 2734 n->vqs[index].rx_vq = virtio_add_queue(vdev, n->net_conf.rx_queue_size, 2735 virtio_net_handle_rx); 2736 2737 if (n->net_conf.tx && !strcmp(n->net_conf.tx, "timer")) { 2738 n->vqs[index].tx_vq = 2739 virtio_add_queue(vdev, n->net_conf.tx_queue_size, 2740 virtio_net_handle_tx_timer); 2741 n->vqs[index].tx_timer = timer_new_ns(QEMU_CLOCK_VIRTUAL, 2742 virtio_net_tx_timer, 2743 &n->vqs[index]); 2744 } else { 2745 n->vqs[index].tx_vq = 2746 virtio_add_queue(vdev, n->net_conf.tx_queue_size, 2747 virtio_net_handle_tx_bh); 2748 n->vqs[index].tx_bh = qemu_bh_new(virtio_net_tx_bh, &n->vqs[index]); 2749 } 2750 2751 n->vqs[index].tx_waiting = 0; 2752 n->vqs[index].n = n; 2753 } 2754 2755 static void virtio_net_del_queue(VirtIONet *n, int index) 2756 { 2757 VirtIODevice *vdev = VIRTIO_DEVICE(n); 2758 VirtIONetQueue *q = &n->vqs[index]; 2759 NetClientState *nc = qemu_get_subqueue(n->nic, index); 2760 2761 qemu_purge_queued_packets(nc); 2762 2763 virtio_del_queue(vdev, index * 2); 2764 if (q->tx_timer) { 2765 timer_free(q->tx_timer); 2766 q->tx_timer = NULL; 2767 } else { 2768 qemu_bh_delete(q->tx_bh); 2769 q->tx_bh = NULL; 2770 } 2771 q->tx_waiting = 0; 2772 virtio_del_queue(vdev, index * 2 + 1); 2773 } 2774 2775 static void virtio_net_change_num_queue_pairs(VirtIONet *n, int new_max_queue_pairs) 2776 { 2777 VirtIODevice *vdev = VIRTIO_DEVICE(n); 2778 int old_num_queues = virtio_get_num_queues(vdev); 2779 int new_num_queues = new_max_queue_pairs * 2 + 1; 2780 int i; 2781 2782 assert(old_num_queues >= 3); 2783 assert(old_num_queues % 2 == 1); 2784 2785 if (old_num_queues == new_num_queues) { 2786 return; 2787 } 2788 2789 /* 2790 * We always need to remove and add ctrl vq if 2791 * old_num_queues != new_num_queues. Remove ctrl_vq first, 2792 * and then we only enter one of the following two loops. 2793 */ 2794 virtio_del_queue(vdev, old_num_queues - 1); 2795 2796 for (i = new_num_queues - 1; i < old_num_queues - 1; i += 2) { 2797 /* new_num_queues < old_num_queues */ 2798 virtio_net_del_queue(n, i / 2); 2799 } 2800 2801 for (i = old_num_queues - 1; i < new_num_queues - 1; i += 2) { 2802 /* new_num_queues > old_num_queues */ 2803 virtio_net_add_queue(n, i / 2); 2804 } 2805 2806 /* add ctrl_vq last */ 2807 n->ctrl_vq = virtio_add_queue(vdev, 64, virtio_net_handle_ctrl); 2808 } 2809 2810 static void virtio_net_set_multiqueue(VirtIONet *n, int multiqueue) 2811 { 2812 int max = multiqueue ? n->max_queue_pairs : 1; 2813 2814 n->multiqueue = multiqueue; 2815 virtio_net_change_num_queue_pairs(n, max); 2816 2817 virtio_net_set_queue_pairs(n); 2818 } 2819 2820 static int virtio_net_post_load_device(void *opaque, int version_id) 2821 { 2822 VirtIONet *n = opaque; 2823 VirtIODevice *vdev = VIRTIO_DEVICE(n); 2824 int i, link_down; 2825 2826 trace_virtio_net_post_load_device(); 2827 virtio_net_set_mrg_rx_bufs(n, n->mergeable_rx_bufs, 2828 virtio_vdev_has_feature(vdev, 2829 VIRTIO_F_VERSION_1), 2830 virtio_vdev_has_feature(vdev, 2831 VIRTIO_NET_F_HASH_REPORT)); 2832 2833 /* MAC_TABLE_ENTRIES may be different from the saved image */ 2834 if (n->mac_table.in_use > MAC_TABLE_ENTRIES) { 2835 n->mac_table.in_use = 0; 2836 } 2837 2838 if (!virtio_vdev_has_feature(vdev, VIRTIO_NET_F_CTRL_GUEST_OFFLOADS)) { 2839 n->curr_guest_offloads = virtio_net_supported_guest_offloads(n); 2840 } 2841 2842 /* 2843 * curr_guest_offloads will be later overwritten by the 2844 * virtio_set_features_nocheck call done from the virtio_load. 2845 * Here we make sure it is preserved and restored accordingly 2846 * in the virtio_net_post_load_virtio callback. 2847 */ 2848 n->saved_guest_offloads = n->curr_guest_offloads; 2849 2850 virtio_net_set_queue_pairs(n); 2851 2852 /* Find the first multicast entry in the saved MAC filter */ 2853 for (i = 0; i < n->mac_table.in_use; i++) { 2854 if (n->mac_table.macs[i * ETH_ALEN] & 1) { 2855 break; 2856 } 2857 } 2858 n->mac_table.first_multi = i; 2859 2860 /* nc.link_down can't be migrated, so infer link_down according 2861 * to link status bit in n->status */ 2862 link_down = (n->status & VIRTIO_NET_S_LINK_UP) == 0; 2863 for (i = 0; i < n->max_queue_pairs; i++) { 2864 qemu_get_subqueue(n->nic, i)->link_down = link_down; 2865 } 2866 2867 if (virtio_vdev_has_feature(vdev, VIRTIO_NET_F_GUEST_ANNOUNCE) && 2868 virtio_vdev_has_feature(vdev, VIRTIO_NET_F_CTRL_VQ)) { 2869 qemu_announce_timer_reset(&n->announce_timer, migrate_announce_params(), 2870 QEMU_CLOCK_VIRTUAL, 2871 virtio_net_announce_timer, n); 2872 if (n->announce_timer.round) { 2873 timer_mod(n->announce_timer.tm, 2874 qemu_clock_get_ms(n->announce_timer.type)); 2875 } else { 2876 qemu_announce_timer_del(&n->announce_timer, false); 2877 } 2878 } 2879 2880 if (n->rss_data.enabled) { 2881 n->rss_data.enabled_software_rss = n->rss_data.populate_hash; 2882 if (!n->rss_data.populate_hash) { 2883 if (!virtio_net_attach_epbf_rss(n)) { 2884 if (get_vhost_net(qemu_get_queue(n->nic)->peer)) { 2885 warn_report("Can't post-load eBPF RSS for vhost"); 2886 } else { 2887 warn_report("Can't post-load eBPF RSS - " 2888 "fallback to software RSS"); 2889 n->rss_data.enabled_software_rss = true; 2890 } 2891 } 2892 } 2893 2894 trace_virtio_net_rss_enable(n->rss_data.hash_types, 2895 n->rss_data.indirections_len, 2896 sizeof(n->rss_data.key)); 2897 } else { 2898 trace_virtio_net_rss_disable(); 2899 } 2900 return 0; 2901 } 2902 2903 static int virtio_net_post_load_virtio(VirtIODevice *vdev) 2904 { 2905 VirtIONet *n = VIRTIO_NET(vdev); 2906 /* 2907 * The actual needed state is now in saved_guest_offloads, 2908 * see virtio_net_post_load_device for detail. 2909 * Restore it back and apply the desired offloads. 2910 */ 2911 n->curr_guest_offloads = n->saved_guest_offloads; 2912 if (peer_has_vnet_hdr(n)) { 2913 virtio_net_apply_guest_offloads(n); 2914 } 2915 2916 return 0; 2917 } 2918 2919 /* tx_waiting field of a VirtIONetQueue */ 2920 static const VMStateDescription vmstate_virtio_net_queue_tx_waiting = { 2921 .name = "virtio-net-queue-tx_waiting", 2922 .fields = (VMStateField[]) { 2923 VMSTATE_UINT32(tx_waiting, VirtIONetQueue), 2924 VMSTATE_END_OF_LIST() 2925 }, 2926 }; 2927 2928 static bool max_queue_pairs_gt_1(void *opaque, int version_id) 2929 { 2930 return VIRTIO_NET(opaque)->max_queue_pairs > 1; 2931 } 2932 2933 static bool has_ctrl_guest_offloads(void *opaque, int version_id) 2934 { 2935 return virtio_vdev_has_feature(VIRTIO_DEVICE(opaque), 2936 VIRTIO_NET_F_CTRL_GUEST_OFFLOADS); 2937 } 2938 2939 static bool mac_table_fits(void *opaque, int version_id) 2940 { 2941 return VIRTIO_NET(opaque)->mac_table.in_use <= MAC_TABLE_ENTRIES; 2942 } 2943 2944 static bool mac_table_doesnt_fit(void *opaque, int version_id) 2945 { 2946 return !mac_table_fits(opaque, version_id); 2947 } 2948 2949 /* This temporary type is shared by all the WITH_TMP methods 2950 * although only some fields are used by each. 2951 */ 2952 struct VirtIONetMigTmp { 2953 VirtIONet *parent; 2954 VirtIONetQueue *vqs_1; 2955 uint16_t curr_queue_pairs_1; 2956 uint8_t has_ufo; 2957 uint32_t has_vnet_hdr; 2958 }; 2959 2960 /* The 2nd and subsequent tx_waiting flags are loaded later than 2961 * the 1st entry in the queue_pairs and only if there's more than one 2962 * entry. We use the tmp mechanism to calculate a temporary 2963 * pointer and count and also validate the count. 2964 */ 2965 2966 static int virtio_net_tx_waiting_pre_save(void *opaque) 2967 { 2968 struct VirtIONetMigTmp *tmp = opaque; 2969 2970 tmp->vqs_1 = tmp->parent->vqs + 1; 2971 tmp->curr_queue_pairs_1 = tmp->parent->curr_queue_pairs - 1; 2972 if (tmp->parent->curr_queue_pairs == 0) { 2973 tmp->curr_queue_pairs_1 = 0; 2974 } 2975 2976 return 0; 2977 } 2978 2979 static int virtio_net_tx_waiting_pre_load(void *opaque) 2980 { 2981 struct VirtIONetMigTmp *tmp = opaque; 2982 2983 /* Reuse the pointer setup from save */ 2984 virtio_net_tx_waiting_pre_save(opaque); 2985 2986 if (tmp->parent->curr_queue_pairs > tmp->parent->max_queue_pairs) { 2987 error_report("virtio-net: curr_queue_pairs %x > max_queue_pairs %x", 2988 tmp->parent->curr_queue_pairs, tmp->parent->max_queue_pairs); 2989 2990 return -EINVAL; 2991 } 2992 2993 return 0; /* all good */ 2994 } 2995 2996 static const VMStateDescription vmstate_virtio_net_tx_waiting = { 2997 .name = "virtio-net-tx_waiting", 2998 .pre_load = virtio_net_tx_waiting_pre_load, 2999 .pre_save = virtio_net_tx_waiting_pre_save, 3000 .fields = (VMStateField[]) { 3001 VMSTATE_STRUCT_VARRAY_POINTER_UINT16(vqs_1, struct VirtIONetMigTmp, 3002 curr_queue_pairs_1, 3003 vmstate_virtio_net_queue_tx_waiting, 3004 struct VirtIONetQueue), 3005 VMSTATE_END_OF_LIST() 3006 }, 3007 }; 3008 3009 /* the 'has_ufo' flag is just tested; if the incoming stream has the 3010 * flag set we need to check that we have it 3011 */ 3012 static int virtio_net_ufo_post_load(void *opaque, int version_id) 3013 { 3014 struct VirtIONetMigTmp *tmp = opaque; 3015 3016 if (tmp->has_ufo && !peer_has_ufo(tmp->parent)) { 3017 error_report("virtio-net: saved image requires TUN_F_UFO support"); 3018 return -EINVAL; 3019 } 3020 3021 return 0; 3022 } 3023 3024 static int virtio_net_ufo_pre_save(void *opaque) 3025 { 3026 struct VirtIONetMigTmp *tmp = opaque; 3027 3028 tmp->has_ufo = tmp->parent->has_ufo; 3029 3030 return 0; 3031 } 3032 3033 static const VMStateDescription vmstate_virtio_net_has_ufo = { 3034 .name = "virtio-net-ufo", 3035 .post_load = virtio_net_ufo_post_load, 3036 .pre_save = virtio_net_ufo_pre_save, 3037 .fields = (VMStateField[]) { 3038 VMSTATE_UINT8(has_ufo, struct VirtIONetMigTmp), 3039 VMSTATE_END_OF_LIST() 3040 }, 3041 }; 3042 3043 /* the 'has_vnet_hdr' flag is just tested; if the incoming stream has the 3044 * flag set we need to check that we have it 3045 */ 3046 static int virtio_net_vnet_post_load(void *opaque, int version_id) 3047 { 3048 struct VirtIONetMigTmp *tmp = opaque; 3049 3050 if (tmp->has_vnet_hdr && !peer_has_vnet_hdr(tmp->parent)) { 3051 error_report("virtio-net: saved image requires vnet_hdr=on"); 3052 return -EINVAL; 3053 } 3054 3055 return 0; 3056 } 3057 3058 static int virtio_net_vnet_pre_save(void *opaque) 3059 { 3060 struct VirtIONetMigTmp *tmp = opaque; 3061 3062 tmp->has_vnet_hdr = tmp->parent->has_vnet_hdr; 3063 3064 return 0; 3065 } 3066 3067 static const VMStateDescription vmstate_virtio_net_has_vnet = { 3068 .name = "virtio-net-vnet", 3069 .post_load = virtio_net_vnet_post_load, 3070 .pre_save = virtio_net_vnet_pre_save, 3071 .fields = (VMStateField[]) { 3072 VMSTATE_UINT32(has_vnet_hdr, struct VirtIONetMigTmp), 3073 VMSTATE_END_OF_LIST() 3074 }, 3075 }; 3076 3077 static bool virtio_net_rss_needed(void *opaque) 3078 { 3079 return VIRTIO_NET(opaque)->rss_data.enabled; 3080 } 3081 3082 static const VMStateDescription vmstate_virtio_net_rss = { 3083 .name = "virtio-net-device/rss", 3084 .version_id = 1, 3085 .minimum_version_id = 1, 3086 .needed = virtio_net_rss_needed, 3087 .fields = (VMStateField[]) { 3088 VMSTATE_BOOL(rss_data.enabled, VirtIONet), 3089 VMSTATE_BOOL(rss_data.redirect, VirtIONet), 3090 VMSTATE_BOOL(rss_data.populate_hash, VirtIONet), 3091 VMSTATE_UINT32(rss_data.hash_types, VirtIONet), 3092 VMSTATE_UINT16(rss_data.indirections_len, VirtIONet), 3093 VMSTATE_UINT16(rss_data.default_queue, VirtIONet), 3094 VMSTATE_UINT8_ARRAY(rss_data.key, VirtIONet, 3095 VIRTIO_NET_RSS_MAX_KEY_SIZE), 3096 VMSTATE_VARRAY_UINT16_ALLOC(rss_data.indirections_table, VirtIONet, 3097 rss_data.indirections_len, 0, 3098 vmstate_info_uint16, uint16_t), 3099 VMSTATE_END_OF_LIST() 3100 }, 3101 }; 3102 3103 static const VMStateDescription vmstate_virtio_net_device = { 3104 .name = "virtio-net-device", 3105 .version_id = VIRTIO_NET_VM_VERSION, 3106 .minimum_version_id = VIRTIO_NET_VM_VERSION, 3107 .post_load = virtio_net_post_load_device, 3108 .fields = (VMStateField[]) { 3109 VMSTATE_UINT8_ARRAY(mac, VirtIONet, ETH_ALEN), 3110 VMSTATE_STRUCT_POINTER(vqs, VirtIONet, 3111 vmstate_virtio_net_queue_tx_waiting, 3112 VirtIONetQueue), 3113 VMSTATE_UINT32(mergeable_rx_bufs, VirtIONet), 3114 VMSTATE_UINT16(status, VirtIONet), 3115 VMSTATE_UINT8(promisc, VirtIONet), 3116 VMSTATE_UINT8(allmulti, VirtIONet), 3117 VMSTATE_UINT32(mac_table.in_use, VirtIONet), 3118 3119 /* Guarded pair: If it fits we load it, else we throw it away 3120 * - can happen if source has a larger MAC table.; post-load 3121 * sets flags in this case. 3122 */ 3123 VMSTATE_VBUFFER_MULTIPLY(mac_table.macs, VirtIONet, 3124 0, mac_table_fits, mac_table.in_use, 3125 ETH_ALEN), 3126 VMSTATE_UNUSED_VARRAY_UINT32(VirtIONet, mac_table_doesnt_fit, 0, 3127 mac_table.in_use, ETH_ALEN), 3128 3129 /* Note: This is an array of uint32's that's always been saved as a 3130 * buffer; hold onto your endiannesses; it's actually used as a bitmap 3131 * but based on the uint. 3132 */ 3133 VMSTATE_BUFFER_POINTER_UNSAFE(vlans, VirtIONet, 0, MAX_VLAN >> 3), 3134 VMSTATE_WITH_TMP(VirtIONet, struct VirtIONetMigTmp, 3135 vmstate_virtio_net_has_vnet), 3136 VMSTATE_UINT8(mac_table.multi_overflow, VirtIONet), 3137 VMSTATE_UINT8(mac_table.uni_overflow, VirtIONet), 3138 VMSTATE_UINT8(alluni, VirtIONet), 3139 VMSTATE_UINT8(nomulti, VirtIONet), 3140 VMSTATE_UINT8(nouni, VirtIONet), 3141 VMSTATE_UINT8(nobcast, VirtIONet), 3142 VMSTATE_WITH_TMP(VirtIONet, struct VirtIONetMigTmp, 3143 vmstate_virtio_net_has_ufo), 3144 VMSTATE_SINGLE_TEST(max_queue_pairs, VirtIONet, max_queue_pairs_gt_1, 0, 3145 vmstate_info_uint16_equal, uint16_t), 3146 VMSTATE_UINT16_TEST(curr_queue_pairs, VirtIONet, max_queue_pairs_gt_1), 3147 VMSTATE_WITH_TMP(VirtIONet, struct VirtIONetMigTmp, 3148 vmstate_virtio_net_tx_waiting), 3149 VMSTATE_UINT64_TEST(curr_guest_offloads, VirtIONet, 3150 has_ctrl_guest_offloads), 3151 VMSTATE_END_OF_LIST() 3152 }, 3153 .subsections = (const VMStateDescription * []) { 3154 &vmstate_virtio_net_rss, 3155 NULL 3156 } 3157 }; 3158 3159 static NetClientInfo net_virtio_info = { 3160 .type = NET_CLIENT_DRIVER_NIC, 3161 .size = sizeof(NICState), 3162 .can_receive = virtio_net_can_receive, 3163 .receive = virtio_net_receive, 3164 .link_status_changed = virtio_net_set_link_status, 3165 .query_rx_filter = virtio_net_query_rxfilter, 3166 .announce = virtio_net_announce, 3167 }; 3168 3169 static bool virtio_net_guest_notifier_pending(VirtIODevice *vdev, int idx) 3170 { 3171 VirtIONet *n = VIRTIO_NET(vdev); 3172 NetClientState *nc = qemu_get_subqueue(n->nic, vq2q(idx)); 3173 assert(n->vhost_started); 3174 return vhost_net_virtqueue_pending(get_vhost_net(nc->peer), idx); 3175 } 3176 3177 static void virtio_net_guest_notifier_mask(VirtIODevice *vdev, int idx, 3178 bool mask) 3179 { 3180 VirtIONet *n = VIRTIO_NET(vdev); 3181 NetClientState *nc = qemu_get_subqueue(n->nic, vq2q(idx)); 3182 assert(n->vhost_started); 3183 vhost_net_virtqueue_mask(get_vhost_net(nc->peer), 3184 vdev, idx, mask); 3185 } 3186 3187 static void virtio_net_set_config_size(VirtIONet *n, uint64_t host_features) 3188 { 3189 virtio_add_feature(&host_features, VIRTIO_NET_F_MAC); 3190 3191 n->config_size = virtio_feature_get_config_size(feature_sizes, 3192 host_features); 3193 } 3194 3195 void virtio_net_set_netclient_name(VirtIONet *n, const char *name, 3196 const char *type) 3197 { 3198 /* 3199 * The name can be NULL, the netclient name will be type.x. 3200 */ 3201 assert(type != NULL); 3202 3203 g_free(n->netclient_name); 3204 g_free(n->netclient_type); 3205 n->netclient_name = g_strdup(name); 3206 n->netclient_type = g_strdup(type); 3207 } 3208 3209 static bool failover_unplug_primary(VirtIONet *n, DeviceState *dev) 3210 { 3211 HotplugHandler *hotplug_ctrl; 3212 PCIDevice *pci_dev; 3213 Error *err = NULL; 3214 3215 hotplug_ctrl = qdev_get_hotplug_handler(dev); 3216 if (hotplug_ctrl) { 3217 pci_dev = PCI_DEVICE(dev); 3218 pci_dev->partially_hotplugged = true; 3219 hotplug_handler_unplug_request(hotplug_ctrl, dev, &err); 3220 if (err) { 3221 error_report_err(err); 3222 return false; 3223 } 3224 } else { 3225 return false; 3226 } 3227 return true; 3228 } 3229 3230 static bool failover_replug_primary(VirtIONet *n, DeviceState *dev, 3231 Error **errp) 3232 { 3233 Error *err = NULL; 3234 HotplugHandler *hotplug_ctrl; 3235 PCIDevice *pdev = PCI_DEVICE(dev); 3236 BusState *primary_bus; 3237 3238 if (!pdev->partially_hotplugged) { 3239 return true; 3240 } 3241 primary_bus = dev->parent_bus; 3242 if (!primary_bus) { 3243 error_setg(errp, "virtio_net: couldn't find primary bus"); 3244 return false; 3245 } 3246 qdev_set_parent_bus(dev, primary_bus, &error_abort); 3247 qatomic_set(&n->failover_primary_hidden, false); 3248 hotplug_ctrl = qdev_get_hotplug_handler(dev); 3249 if (hotplug_ctrl) { 3250 hotplug_handler_pre_plug(hotplug_ctrl, dev, &err); 3251 if (err) { 3252 goto out; 3253 } 3254 hotplug_handler_plug(hotplug_ctrl, dev, &err); 3255 } 3256 pdev->partially_hotplugged = false; 3257 3258 out: 3259 error_propagate(errp, err); 3260 return !err; 3261 } 3262 3263 static void virtio_net_handle_migration_primary(VirtIONet *n, MigrationState *s) 3264 { 3265 bool should_be_hidden; 3266 Error *err = NULL; 3267 DeviceState *dev = failover_find_primary_device(n); 3268 3269 if (!dev) { 3270 return; 3271 } 3272 3273 should_be_hidden = qatomic_read(&n->failover_primary_hidden); 3274 3275 if (migration_in_setup(s) && !should_be_hidden) { 3276 if (failover_unplug_primary(n, dev)) { 3277 vmstate_unregister(VMSTATE_IF(dev), qdev_get_vmsd(dev), dev); 3278 qapi_event_send_unplug_primary(dev->id); 3279 qatomic_set(&n->failover_primary_hidden, true); 3280 } else { 3281 warn_report("couldn't unplug primary device"); 3282 } 3283 } else if (migration_has_failed(s)) { 3284 /* We already unplugged the device let's plug it back */ 3285 if (!failover_replug_primary(n, dev, &err)) { 3286 if (err) { 3287 error_report_err(err); 3288 } 3289 } 3290 } 3291 } 3292 3293 static void virtio_net_migration_state_notifier(Notifier *notifier, void *data) 3294 { 3295 MigrationState *s = data; 3296 VirtIONet *n = container_of(notifier, VirtIONet, migration_state); 3297 virtio_net_handle_migration_primary(n, s); 3298 } 3299 3300 static bool failover_hide_primary_device(DeviceListener *listener, 3301 const QDict *device_opts, 3302 bool from_json, 3303 Error **errp) 3304 { 3305 VirtIONet *n = container_of(listener, VirtIONet, primary_listener); 3306 const char *standby_id; 3307 3308 if (!device_opts) { 3309 return false; 3310 } 3311 3312 if (!qdict_haskey(device_opts, "failover_pair_id")) { 3313 return false; 3314 } 3315 3316 if (!qdict_haskey(device_opts, "id")) { 3317 error_setg(errp, "Device with failover_pair_id needs to have id"); 3318 return false; 3319 } 3320 3321 standby_id = qdict_get_str(device_opts, "failover_pair_id"); 3322 if (g_strcmp0(standby_id, n->netclient_name) != 0) { 3323 return false; 3324 } 3325 3326 /* 3327 * The hide helper can be called several times for a given device. 3328 * Check there is only one primary for a virtio-net device but 3329 * don't duplicate the qdict several times if it's called for the same 3330 * device. 3331 */ 3332 if (n->primary_opts) { 3333 const char *old, *new; 3334 /* devices with failover_pair_id always have an id */ 3335 old = qdict_get_str(n->primary_opts, "id"); 3336 new = qdict_get_str(device_opts, "id"); 3337 if (strcmp(old, new) != 0) { 3338 error_setg(errp, "Cannot attach more than one primary device to " 3339 "'%s': '%s' and '%s'", n->netclient_name, old, new); 3340 return false; 3341 } 3342 } else { 3343 n->primary_opts = qdict_clone_shallow(device_opts); 3344 n->primary_opts_from_json = from_json; 3345 } 3346 3347 /* failover_primary_hidden is set during feature negotiation */ 3348 return qatomic_read(&n->failover_primary_hidden); 3349 } 3350 3351 static void virtio_net_device_realize(DeviceState *dev, Error **errp) 3352 { 3353 VirtIODevice *vdev = VIRTIO_DEVICE(dev); 3354 VirtIONet *n = VIRTIO_NET(dev); 3355 NetClientState *nc; 3356 int i; 3357 3358 if (n->net_conf.mtu) { 3359 n->host_features |= (1ULL << VIRTIO_NET_F_MTU); 3360 } 3361 3362 if (n->net_conf.duplex_str) { 3363 if (strncmp(n->net_conf.duplex_str, "half", 5) == 0) { 3364 n->net_conf.duplex = DUPLEX_HALF; 3365 } else if (strncmp(n->net_conf.duplex_str, "full", 5) == 0) { 3366 n->net_conf.duplex = DUPLEX_FULL; 3367 } else { 3368 error_setg(errp, "'duplex' must be 'half' or 'full'"); 3369 return; 3370 } 3371 n->host_features |= (1ULL << VIRTIO_NET_F_SPEED_DUPLEX); 3372 } else { 3373 n->net_conf.duplex = DUPLEX_UNKNOWN; 3374 } 3375 3376 if (n->net_conf.speed < SPEED_UNKNOWN) { 3377 error_setg(errp, "'speed' must be between 0 and INT_MAX"); 3378 return; 3379 } 3380 if (n->net_conf.speed >= 0) { 3381 n->host_features |= (1ULL << VIRTIO_NET_F_SPEED_DUPLEX); 3382 } 3383 3384 if (n->failover) { 3385 n->primary_listener.hide_device = failover_hide_primary_device; 3386 qatomic_set(&n->failover_primary_hidden, true); 3387 device_listener_register(&n->primary_listener); 3388 n->migration_state.notify = virtio_net_migration_state_notifier; 3389 add_migration_state_change_notifier(&n->migration_state); 3390 n->host_features |= (1ULL << VIRTIO_NET_F_STANDBY); 3391 } 3392 3393 virtio_net_set_config_size(n, n->host_features); 3394 virtio_init(vdev, "virtio-net", VIRTIO_ID_NET, n->config_size); 3395 3396 /* 3397 * We set a lower limit on RX queue size to what it always was. 3398 * Guests that want a smaller ring can always resize it without 3399 * help from us (using virtio 1 and up). 3400 */ 3401 if (n->net_conf.rx_queue_size < VIRTIO_NET_RX_QUEUE_MIN_SIZE || 3402 n->net_conf.rx_queue_size > VIRTQUEUE_MAX_SIZE || 3403 !is_power_of_2(n->net_conf.rx_queue_size)) { 3404 error_setg(errp, "Invalid rx_queue_size (= %" PRIu16 "), " 3405 "must be a power of 2 between %d and %d.", 3406 n->net_conf.rx_queue_size, VIRTIO_NET_RX_QUEUE_MIN_SIZE, 3407 VIRTQUEUE_MAX_SIZE); 3408 virtio_cleanup(vdev); 3409 return; 3410 } 3411 3412 if (n->net_conf.tx_queue_size < VIRTIO_NET_TX_QUEUE_MIN_SIZE || 3413 n->net_conf.tx_queue_size > VIRTQUEUE_MAX_SIZE || 3414 !is_power_of_2(n->net_conf.tx_queue_size)) { 3415 error_setg(errp, "Invalid tx_queue_size (= %" PRIu16 "), " 3416 "must be a power of 2 between %d and %d", 3417 n->net_conf.tx_queue_size, VIRTIO_NET_TX_QUEUE_MIN_SIZE, 3418 VIRTQUEUE_MAX_SIZE); 3419 virtio_cleanup(vdev); 3420 return; 3421 } 3422 3423 n->max_ncs = MAX(n->nic_conf.peers.queues, 1); 3424 3425 /* 3426 * Figure out the datapath queue pairs since the backend could 3427 * provide control queue via peers as well. 3428 */ 3429 if (n->nic_conf.peers.queues) { 3430 for (i = 0; i < n->max_ncs; i++) { 3431 if (n->nic_conf.peers.ncs[i]->is_datapath) { 3432 ++n->max_queue_pairs; 3433 } 3434 } 3435 } 3436 n->max_queue_pairs = MAX(n->max_queue_pairs, 1); 3437 3438 if (n->max_queue_pairs * 2 + 1 > VIRTIO_QUEUE_MAX) { 3439 error_setg(errp, "Invalid number of queue pairs (= %" PRIu32 "), " 3440 "must be a positive integer less than %d.", 3441 n->max_queue_pairs, (VIRTIO_QUEUE_MAX - 1) / 2); 3442 virtio_cleanup(vdev); 3443 return; 3444 } 3445 n->vqs = g_malloc0(sizeof(VirtIONetQueue) * n->max_queue_pairs); 3446 n->curr_queue_pairs = 1; 3447 n->tx_timeout = n->net_conf.txtimer; 3448 3449 if (n->net_conf.tx && strcmp(n->net_conf.tx, "timer") 3450 && strcmp(n->net_conf.tx, "bh")) { 3451 warn_report("virtio-net: " 3452 "Unknown option tx=%s, valid options: \"timer\" \"bh\"", 3453 n->net_conf.tx); 3454 error_printf("Defaulting to \"bh\""); 3455 } 3456 3457 n->net_conf.tx_queue_size = MIN(virtio_net_max_tx_queue_size(n), 3458 n->net_conf.tx_queue_size); 3459 3460 for (i = 0; i < n->max_queue_pairs; i++) { 3461 virtio_net_add_queue(n, i); 3462 } 3463 3464 n->ctrl_vq = virtio_add_queue(vdev, 64, virtio_net_handle_ctrl); 3465 qemu_macaddr_default_if_unset(&n->nic_conf.macaddr); 3466 memcpy(&n->mac[0], &n->nic_conf.macaddr, sizeof(n->mac)); 3467 n->status = VIRTIO_NET_S_LINK_UP; 3468 qemu_announce_timer_reset(&n->announce_timer, migrate_announce_params(), 3469 QEMU_CLOCK_VIRTUAL, 3470 virtio_net_announce_timer, n); 3471 n->announce_timer.round = 0; 3472 3473 if (n->netclient_type) { 3474 /* 3475 * Happen when virtio_net_set_netclient_name has been called. 3476 */ 3477 n->nic = qemu_new_nic(&net_virtio_info, &n->nic_conf, 3478 n->netclient_type, n->netclient_name, n); 3479 } else { 3480 n->nic = qemu_new_nic(&net_virtio_info, &n->nic_conf, 3481 object_get_typename(OBJECT(dev)), dev->id, n); 3482 } 3483 3484 for (i = 0; i < n->max_queue_pairs; i++) { 3485 n->nic->ncs[i].do_not_pad = true; 3486 } 3487 3488 peer_test_vnet_hdr(n); 3489 if (peer_has_vnet_hdr(n)) { 3490 for (i = 0; i < n->max_queue_pairs; i++) { 3491 qemu_using_vnet_hdr(qemu_get_subqueue(n->nic, i)->peer, true); 3492 } 3493 n->host_hdr_len = sizeof(struct virtio_net_hdr); 3494 } else { 3495 n->host_hdr_len = 0; 3496 } 3497 3498 qemu_format_nic_info_str(qemu_get_queue(n->nic), n->nic_conf.macaddr.a); 3499 3500 n->vqs[0].tx_waiting = 0; 3501 n->tx_burst = n->net_conf.txburst; 3502 virtio_net_set_mrg_rx_bufs(n, 0, 0, 0); 3503 n->promisc = 1; /* for compatibility */ 3504 3505 n->mac_table.macs = g_malloc0(MAC_TABLE_ENTRIES * ETH_ALEN); 3506 3507 n->vlans = g_malloc0(MAX_VLAN >> 3); 3508 3509 nc = qemu_get_queue(n->nic); 3510 nc->rxfilter_notify_enabled = 1; 3511 3512 if (nc->peer && nc->peer->info->type == NET_CLIENT_DRIVER_VHOST_VDPA) { 3513 struct virtio_net_config netcfg = {}; 3514 memcpy(&netcfg.mac, &n->nic_conf.macaddr, ETH_ALEN); 3515 vhost_net_set_config(get_vhost_net(nc->peer), 3516 (uint8_t *)&netcfg, 0, ETH_ALEN, VHOST_SET_CONFIG_TYPE_MASTER); 3517 } 3518 QTAILQ_INIT(&n->rsc_chains); 3519 n->qdev = dev; 3520 3521 net_rx_pkt_init(&n->rx_pkt, false); 3522 3523 if (virtio_has_feature(n->host_features, VIRTIO_NET_F_RSS)) { 3524 virtio_net_load_ebpf(n); 3525 } 3526 } 3527 3528 static void virtio_net_device_unrealize(DeviceState *dev) 3529 { 3530 VirtIODevice *vdev = VIRTIO_DEVICE(dev); 3531 VirtIONet *n = VIRTIO_NET(dev); 3532 int i, max_queue_pairs; 3533 3534 if (virtio_has_feature(n->host_features, VIRTIO_NET_F_RSS)) { 3535 virtio_net_unload_ebpf(n); 3536 } 3537 3538 /* This will stop vhost backend if appropriate. */ 3539 virtio_net_set_status(vdev, 0); 3540 3541 g_free(n->netclient_name); 3542 n->netclient_name = NULL; 3543 g_free(n->netclient_type); 3544 n->netclient_type = NULL; 3545 3546 g_free(n->mac_table.macs); 3547 g_free(n->vlans); 3548 3549 if (n->failover) { 3550 qobject_unref(n->primary_opts); 3551 device_listener_unregister(&n->primary_listener); 3552 remove_migration_state_change_notifier(&n->migration_state); 3553 } else { 3554 assert(n->primary_opts == NULL); 3555 } 3556 3557 max_queue_pairs = n->multiqueue ? n->max_queue_pairs : 1; 3558 for (i = 0; i < max_queue_pairs; i++) { 3559 virtio_net_del_queue(n, i); 3560 } 3561 /* delete also control vq */ 3562 virtio_del_queue(vdev, max_queue_pairs * 2); 3563 qemu_announce_timer_del(&n->announce_timer, false); 3564 g_free(n->vqs); 3565 qemu_del_nic(n->nic); 3566 virtio_net_rsc_cleanup(n); 3567 g_free(n->rss_data.indirections_table); 3568 net_rx_pkt_uninit(n->rx_pkt); 3569 virtio_cleanup(vdev); 3570 } 3571 3572 static void virtio_net_instance_init(Object *obj) 3573 { 3574 VirtIONet *n = VIRTIO_NET(obj); 3575 3576 /* 3577 * The default config_size is sizeof(struct virtio_net_config). 3578 * Can be overriden with virtio_net_set_config_size. 3579 */ 3580 n->config_size = sizeof(struct virtio_net_config); 3581 device_add_bootindex_property(obj, &n->nic_conf.bootindex, 3582 "bootindex", "/ethernet-phy@0", 3583 DEVICE(n)); 3584 3585 ebpf_rss_init(&n->ebpf_rss); 3586 } 3587 3588 static int virtio_net_pre_save(void *opaque) 3589 { 3590 VirtIONet *n = opaque; 3591 3592 /* At this point, backend must be stopped, otherwise 3593 * it might keep writing to memory. */ 3594 assert(!n->vhost_started); 3595 3596 return 0; 3597 } 3598 3599 static bool primary_unplug_pending(void *opaque) 3600 { 3601 DeviceState *dev = opaque; 3602 DeviceState *primary; 3603 VirtIODevice *vdev = VIRTIO_DEVICE(dev); 3604 VirtIONet *n = VIRTIO_NET(vdev); 3605 3606 if (!virtio_vdev_has_feature(vdev, VIRTIO_NET_F_STANDBY)) { 3607 return false; 3608 } 3609 primary = failover_find_primary_device(n); 3610 return primary ? primary->pending_deleted_event : false; 3611 } 3612 3613 static bool dev_unplug_pending(void *opaque) 3614 { 3615 DeviceState *dev = opaque; 3616 VirtioDeviceClass *vdc = VIRTIO_DEVICE_GET_CLASS(dev); 3617 3618 return vdc->primary_unplug_pending(dev); 3619 } 3620 3621 static const VMStateDescription vmstate_virtio_net = { 3622 .name = "virtio-net", 3623 .minimum_version_id = VIRTIO_NET_VM_VERSION, 3624 .version_id = VIRTIO_NET_VM_VERSION, 3625 .fields = (VMStateField[]) { 3626 VMSTATE_VIRTIO_DEVICE, 3627 VMSTATE_END_OF_LIST() 3628 }, 3629 .pre_save = virtio_net_pre_save, 3630 .dev_unplug_pending = dev_unplug_pending, 3631 }; 3632 3633 static Property virtio_net_properties[] = { 3634 DEFINE_PROP_BIT64("csum", VirtIONet, host_features, 3635 VIRTIO_NET_F_CSUM, true), 3636 DEFINE_PROP_BIT64("guest_csum", VirtIONet, host_features, 3637 VIRTIO_NET_F_GUEST_CSUM, true), 3638 DEFINE_PROP_BIT64("gso", VirtIONet, host_features, VIRTIO_NET_F_GSO, true), 3639 DEFINE_PROP_BIT64("guest_tso4", VirtIONet, host_features, 3640 VIRTIO_NET_F_GUEST_TSO4, true), 3641 DEFINE_PROP_BIT64("guest_tso6", VirtIONet, host_features, 3642 VIRTIO_NET_F_GUEST_TSO6, true), 3643 DEFINE_PROP_BIT64("guest_ecn", VirtIONet, host_features, 3644 VIRTIO_NET_F_GUEST_ECN, true), 3645 DEFINE_PROP_BIT64("guest_ufo", VirtIONet, host_features, 3646 VIRTIO_NET_F_GUEST_UFO, true), 3647 DEFINE_PROP_BIT64("guest_announce", VirtIONet, host_features, 3648 VIRTIO_NET_F_GUEST_ANNOUNCE, true), 3649 DEFINE_PROP_BIT64("host_tso4", VirtIONet, host_features, 3650 VIRTIO_NET_F_HOST_TSO4, true), 3651 DEFINE_PROP_BIT64("host_tso6", VirtIONet, host_features, 3652 VIRTIO_NET_F_HOST_TSO6, true), 3653 DEFINE_PROP_BIT64("host_ecn", VirtIONet, host_features, 3654 VIRTIO_NET_F_HOST_ECN, true), 3655 DEFINE_PROP_BIT64("host_ufo", VirtIONet, host_features, 3656 VIRTIO_NET_F_HOST_UFO, true), 3657 DEFINE_PROP_BIT64("mrg_rxbuf", VirtIONet, host_features, 3658 VIRTIO_NET_F_MRG_RXBUF, true), 3659 DEFINE_PROP_BIT64("status", VirtIONet, host_features, 3660 VIRTIO_NET_F_STATUS, true), 3661 DEFINE_PROP_BIT64("ctrl_vq", VirtIONet, host_features, 3662 VIRTIO_NET_F_CTRL_VQ, true), 3663 DEFINE_PROP_BIT64("ctrl_rx", VirtIONet, host_features, 3664 VIRTIO_NET_F_CTRL_RX, true), 3665 DEFINE_PROP_BIT64("ctrl_vlan", VirtIONet, host_features, 3666 VIRTIO_NET_F_CTRL_VLAN, true), 3667 DEFINE_PROP_BIT64("ctrl_rx_extra", VirtIONet, host_features, 3668 VIRTIO_NET_F_CTRL_RX_EXTRA, true), 3669 DEFINE_PROP_BIT64("ctrl_mac_addr", VirtIONet, host_features, 3670 VIRTIO_NET_F_CTRL_MAC_ADDR, true), 3671 DEFINE_PROP_BIT64("ctrl_guest_offloads", VirtIONet, host_features, 3672 VIRTIO_NET_F_CTRL_GUEST_OFFLOADS, true), 3673 DEFINE_PROP_BIT64("mq", VirtIONet, host_features, VIRTIO_NET_F_MQ, false), 3674 DEFINE_PROP_BIT64("rss", VirtIONet, host_features, 3675 VIRTIO_NET_F_RSS, false), 3676 DEFINE_PROP_BIT64("hash", VirtIONet, host_features, 3677 VIRTIO_NET_F_HASH_REPORT, false), 3678 DEFINE_PROP_BIT64("guest_rsc_ext", VirtIONet, host_features, 3679 VIRTIO_NET_F_RSC_EXT, false), 3680 DEFINE_PROP_UINT32("rsc_interval", VirtIONet, rsc_timeout, 3681 VIRTIO_NET_RSC_DEFAULT_INTERVAL), 3682 DEFINE_NIC_PROPERTIES(VirtIONet, nic_conf), 3683 DEFINE_PROP_UINT32("x-txtimer", VirtIONet, net_conf.txtimer, 3684 TX_TIMER_INTERVAL), 3685 DEFINE_PROP_INT32("x-txburst", VirtIONet, net_conf.txburst, TX_BURST), 3686 DEFINE_PROP_STRING("tx", VirtIONet, net_conf.tx), 3687 DEFINE_PROP_UINT16("rx_queue_size", VirtIONet, net_conf.rx_queue_size, 3688 VIRTIO_NET_RX_QUEUE_DEFAULT_SIZE), 3689 DEFINE_PROP_UINT16("tx_queue_size", VirtIONet, net_conf.tx_queue_size, 3690 VIRTIO_NET_TX_QUEUE_DEFAULT_SIZE), 3691 DEFINE_PROP_UINT16("host_mtu", VirtIONet, net_conf.mtu, 0), 3692 DEFINE_PROP_BOOL("x-mtu-bypass-backend", VirtIONet, mtu_bypass_backend, 3693 true), 3694 DEFINE_PROP_INT32("speed", VirtIONet, net_conf.speed, SPEED_UNKNOWN), 3695 DEFINE_PROP_STRING("duplex", VirtIONet, net_conf.duplex_str), 3696 DEFINE_PROP_BOOL("failover", VirtIONet, failover, false), 3697 DEFINE_PROP_END_OF_LIST(), 3698 }; 3699 3700 static void virtio_net_class_init(ObjectClass *klass, void *data) 3701 { 3702 DeviceClass *dc = DEVICE_CLASS(klass); 3703 VirtioDeviceClass *vdc = VIRTIO_DEVICE_CLASS(klass); 3704 3705 device_class_set_props(dc, virtio_net_properties); 3706 dc->vmsd = &vmstate_virtio_net; 3707 set_bit(DEVICE_CATEGORY_NETWORK, dc->categories); 3708 vdc->realize = virtio_net_device_realize; 3709 vdc->unrealize = virtio_net_device_unrealize; 3710 vdc->get_config = virtio_net_get_config; 3711 vdc->set_config = virtio_net_set_config; 3712 vdc->get_features = virtio_net_get_features; 3713 vdc->set_features = virtio_net_set_features; 3714 vdc->bad_features = virtio_net_bad_features; 3715 vdc->reset = virtio_net_reset; 3716 vdc->set_status = virtio_net_set_status; 3717 vdc->guest_notifier_mask = virtio_net_guest_notifier_mask; 3718 vdc->guest_notifier_pending = virtio_net_guest_notifier_pending; 3719 vdc->legacy_features |= (0x1 << VIRTIO_NET_F_GSO); 3720 vdc->post_load = virtio_net_post_load_virtio; 3721 vdc->vmsd = &vmstate_virtio_net_device; 3722 vdc->primary_unplug_pending = primary_unplug_pending; 3723 } 3724 3725 static const TypeInfo virtio_net_info = { 3726 .name = TYPE_VIRTIO_NET, 3727 .parent = TYPE_VIRTIO_DEVICE, 3728 .instance_size = sizeof(VirtIONet), 3729 .instance_init = virtio_net_instance_init, 3730 .class_init = virtio_net_class_init, 3731 }; 3732 3733 static void virtio_register_types(void) 3734 { 3735 type_register_static(&virtio_net_info); 3736 } 3737 3738 type_init(virtio_register_types) 3739