xref: /qemu/hw/net/virtio-net.c (revision c41481af9a5d0d463607cc45b45c510875570817)
1 /*
2  * Virtio Network Device
3  *
4  * Copyright IBM, Corp. 2007
5  *
6  * Authors:
7  *  Anthony Liguori   <aliguori@us.ibm.com>
8  *
9  * This work is licensed under the terms of the GNU GPL, version 2.  See
10  * the COPYING file in the top-level directory.
11  *
12  */
13 
14 #include "qemu/osdep.h"
15 #include "qemu/atomic.h"
16 #include "qemu/iov.h"
17 #include "qemu/main-loop.h"
18 #include "qemu/module.h"
19 #include "hw/virtio/virtio.h"
20 #include "net/net.h"
21 #include "net/checksum.h"
22 #include "net/tap.h"
23 #include "qemu/error-report.h"
24 #include "qemu/timer.h"
25 #include "qemu/option.h"
26 #include "qemu/option_int.h"
27 #include "qemu/config-file.h"
28 #include "qapi/qmp/qdict.h"
29 #include "hw/virtio/virtio-net.h"
30 #include "net/vhost_net.h"
31 #include "net/announce.h"
32 #include "hw/virtio/virtio-bus.h"
33 #include "qapi/error.h"
34 #include "qapi/qapi-events-net.h"
35 #include "hw/qdev-properties.h"
36 #include "qapi/qapi-types-migration.h"
37 #include "qapi/qapi-events-migration.h"
38 #include "hw/virtio/virtio-access.h"
39 #include "migration/misc.h"
40 #include "standard-headers/linux/ethtool.h"
41 #include "sysemu/sysemu.h"
42 #include "trace.h"
43 #include "monitor/qdev.h"
44 #include "hw/pci/pci.h"
45 #include "net_rx_pkt.h"
46 #include "hw/virtio/vhost.h"
47 #include "sysemu/qtest.h"
48 
49 #define VIRTIO_NET_VM_VERSION    11
50 
51 #define MAC_TABLE_ENTRIES    64
52 #define MAX_VLAN    (1 << 12)   /* Per 802.1Q definition */
53 
54 /* previously fixed value */
55 #define VIRTIO_NET_RX_QUEUE_DEFAULT_SIZE 256
56 #define VIRTIO_NET_TX_QUEUE_DEFAULT_SIZE 256
57 
58 /* for now, only allow larger queue_pairs; with virtio-1, guest can downsize */
59 #define VIRTIO_NET_RX_QUEUE_MIN_SIZE VIRTIO_NET_RX_QUEUE_DEFAULT_SIZE
60 #define VIRTIO_NET_TX_QUEUE_MIN_SIZE VIRTIO_NET_TX_QUEUE_DEFAULT_SIZE
61 
62 #define VIRTIO_NET_IP4_ADDR_SIZE   8        /* ipv4 saddr + daddr */
63 
64 #define VIRTIO_NET_TCP_FLAG         0x3F
65 #define VIRTIO_NET_TCP_HDR_LENGTH   0xF000
66 
67 /* IPv4 max payload, 16 bits in the header */
68 #define VIRTIO_NET_MAX_IP4_PAYLOAD (65535 - sizeof(struct ip_header))
69 #define VIRTIO_NET_MAX_TCP_PAYLOAD 65535
70 
71 /* header length value in ip header without option */
72 #define VIRTIO_NET_IP4_HEADER_LENGTH 5
73 
74 #define VIRTIO_NET_IP6_ADDR_SIZE   32      /* ipv6 saddr + daddr */
75 #define VIRTIO_NET_MAX_IP6_PAYLOAD VIRTIO_NET_MAX_TCP_PAYLOAD
76 
77 /* Purge coalesced packets timer interval, This value affects the performance
78    a lot, and should be tuned carefully, '300000'(300us) is the recommended
79    value to pass the WHQL test, '50000' can gain 2x netperf throughput with
80    tso/gso/gro 'off'. */
81 #define VIRTIO_NET_RSC_DEFAULT_INTERVAL 300000
82 
83 #define VIRTIO_NET_RSS_SUPPORTED_HASHES (VIRTIO_NET_RSS_HASH_TYPE_IPv4 | \
84                                          VIRTIO_NET_RSS_HASH_TYPE_TCPv4 | \
85                                          VIRTIO_NET_RSS_HASH_TYPE_UDPv4 | \
86                                          VIRTIO_NET_RSS_HASH_TYPE_IPv6 | \
87                                          VIRTIO_NET_RSS_HASH_TYPE_TCPv6 | \
88                                          VIRTIO_NET_RSS_HASH_TYPE_UDPv6 | \
89                                          VIRTIO_NET_RSS_HASH_TYPE_IP_EX | \
90                                          VIRTIO_NET_RSS_HASH_TYPE_TCP_EX | \
91                                          VIRTIO_NET_RSS_HASH_TYPE_UDP_EX)
92 
93 static const VirtIOFeature feature_sizes[] = {
94     {.flags = 1ULL << VIRTIO_NET_F_MAC,
95      .end = endof(struct virtio_net_config, mac)},
96     {.flags = 1ULL << VIRTIO_NET_F_STATUS,
97      .end = endof(struct virtio_net_config, status)},
98     {.flags = 1ULL << VIRTIO_NET_F_MQ,
99      .end = endof(struct virtio_net_config, max_virtqueue_pairs)},
100     {.flags = 1ULL << VIRTIO_NET_F_MTU,
101      .end = endof(struct virtio_net_config, mtu)},
102     {.flags = 1ULL << VIRTIO_NET_F_SPEED_DUPLEX,
103      .end = endof(struct virtio_net_config, duplex)},
104     {.flags = (1ULL << VIRTIO_NET_F_RSS) | (1ULL << VIRTIO_NET_F_HASH_REPORT),
105      .end = endof(struct virtio_net_config, supported_hash_types)},
106     {}
107 };
108 
109 static VirtIONetQueue *virtio_net_get_subqueue(NetClientState *nc)
110 {
111     VirtIONet *n = qemu_get_nic_opaque(nc);
112 
113     return &n->vqs[nc->queue_index];
114 }
115 
116 static int vq2q(int queue_index)
117 {
118     return queue_index / 2;
119 }
120 
121 /* TODO
122  * - we could suppress RX interrupt if we were so inclined.
123  */
124 
125 static void virtio_net_get_config(VirtIODevice *vdev, uint8_t *config)
126 {
127     VirtIONet *n = VIRTIO_NET(vdev);
128     struct virtio_net_config netcfg;
129     NetClientState *nc = qemu_get_queue(n->nic);
130     static const MACAddr zero = { .a = { 0, 0, 0, 0, 0, 0 } };
131 
132     int ret = 0;
133     memset(&netcfg, 0 , sizeof(struct virtio_net_config));
134     virtio_stw_p(vdev, &netcfg.status, n->status);
135     virtio_stw_p(vdev, &netcfg.max_virtqueue_pairs, n->max_queue_pairs);
136     virtio_stw_p(vdev, &netcfg.mtu, n->net_conf.mtu);
137     memcpy(netcfg.mac, n->mac, ETH_ALEN);
138     virtio_stl_p(vdev, &netcfg.speed, n->net_conf.speed);
139     netcfg.duplex = n->net_conf.duplex;
140     netcfg.rss_max_key_size = VIRTIO_NET_RSS_MAX_KEY_SIZE;
141     virtio_stw_p(vdev, &netcfg.rss_max_indirection_table_length,
142                  virtio_host_has_feature(vdev, VIRTIO_NET_F_RSS) ?
143                  VIRTIO_NET_RSS_MAX_TABLE_LEN : 1);
144     virtio_stl_p(vdev, &netcfg.supported_hash_types,
145                  VIRTIO_NET_RSS_SUPPORTED_HASHES);
146     memcpy(config, &netcfg, n->config_size);
147 
148     /*
149      * Is this VDPA? No peer means not VDPA: there's no way to
150      * disconnect/reconnect a VDPA peer.
151      */
152     if (nc->peer && nc->peer->info->type == NET_CLIENT_DRIVER_VHOST_VDPA) {
153         ret = vhost_net_get_config(get_vhost_net(nc->peer), (uint8_t *)&netcfg,
154                                    n->config_size);
155         if (ret != -1) {
156             /*
157              * Some NIC/kernel combinations present 0 as the mac address.  As
158              * that is not a legal address, try to proceed with the
159              * address from the QEMU command line in the hope that the
160              * address has been configured correctly elsewhere - just not
161              * reported by the device.
162              */
163             if (memcmp(&netcfg.mac, &zero, sizeof(zero)) == 0) {
164                 info_report("Zero hardware mac address detected. Ignoring.");
165                 memcpy(netcfg.mac, n->mac, ETH_ALEN);
166             }
167             memcpy(config, &netcfg, n->config_size);
168         }
169     }
170 }
171 
172 static void virtio_net_set_config(VirtIODevice *vdev, const uint8_t *config)
173 {
174     VirtIONet *n = VIRTIO_NET(vdev);
175     struct virtio_net_config netcfg = {};
176     NetClientState *nc = qemu_get_queue(n->nic);
177 
178     memcpy(&netcfg, config, n->config_size);
179 
180     if (!virtio_vdev_has_feature(vdev, VIRTIO_NET_F_CTRL_MAC_ADDR) &&
181         !virtio_vdev_has_feature(vdev, VIRTIO_F_VERSION_1) &&
182         memcmp(netcfg.mac, n->mac, ETH_ALEN)) {
183         memcpy(n->mac, netcfg.mac, ETH_ALEN);
184         qemu_format_nic_info_str(qemu_get_queue(n->nic), n->mac);
185     }
186 
187     /*
188      * Is this VDPA? No peer means not VDPA: there's no way to
189      * disconnect/reconnect a VDPA peer.
190      */
191     if (nc->peer && nc->peer->info->type == NET_CLIENT_DRIVER_VHOST_VDPA) {
192         vhost_net_set_config(get_vhost_net(nc->peer),
193                              (uint8_t *)&netcfg, 0, n->config_size,
194                              VHOST_SET_CONFIG_TYPE_MASTER);
195       }
196 }
197 
198 static bool virtio_net_started(VirtIONet *n, uint8_t status)
199 {
200     VirtIODevice *vdev = VIRTIO_DEVICE(n);
201     return (status & VIRTIO_CONFIG_S_DRIVER_OK) &&
202         (n->status & VIRTIO_NET_S_LINK_UP) && vdev->vm_running;
203 }
204 
205 static void virtio_net_announce_notify(VirtIONet *net)
206 {
207     VirtIODevice *vdev = VIRTIO_DEVICE(net);
208     trace_virtio_net_announce_notify();
209 
210     net->status |= VIRTIO_NET_S_ANNOUNCE;
211     virtio_notify_config(vdev);
212 }
213 
214 static void virtio_net_announce_timer(void *opaque)
215 {
216     VirtIONet *n = opaque;
217     trace_virtio_net_announce_timer(n->announce_timer.round);
218 
219     n->announce_timer.round--;
220     virtio_net_announce_notify(n);
221 }
222 
223 static void virtio_net_announce(NetClientState *nc)
224 {
225     VirtIONet *n = qemu_get_nic_opaque(nc);
226     VirtIODevice *vdev = VIRTIO_DEVICE(n);
227 
228     /*
229      * Make sure the virtio migration announcement timer isn't running
230      * If it is, let it trigger announcement so that we do not cause
231      * confusion.
232      */
233     if (n->announce_timer.round) {
234         return;
235     }
236 
237     if (virtio_vdev_has_feature(vdev, VIRTIO_NET_F_GUEST_ANNOUNCE) &&
238         virtio_vdev_has_feature(vdev, VIRTIO_NET_F_CTRL_VQ)) {
239             virtio_net_announce_notify(n);
240     }
241 }
242 
243 static void virtio_net_vhost_status(VirtIONet *n, uint8_t status)
244 {
245     VirtIODevice *vdev = VIRTIO_DEVICE(n);
246     NetClientState *nc = qemu_get_queue(n->nic);
247     int queue_pairs = n->multiqueue ? n->max_queue_pairs : 1;
248     int cvq = n->max_ncs - n->max_queue_pairs;
249 
250     if (!get_vhost_net(nc->peer)) {
251         return;
252     }
253 
254     if ((virtio_net_started(n, status) && !nc->peer->link_down) ==
255         !!n->vhost_started) {
256         return;
257     }
258     if (!n->vhost_started) {
259         int r, i;
260 
261         if (n->needs_vnet_hdr_swap) {
262             error_report("backend does not support %s vnet headers; "
263                          "falling back on userspace virtio",
264                          virtio_is_big_endian(vdev) ? "BE" : "LE");
265             return;
266         }
267 
268         /* Any packets outstanding? Purge them to avoid touching rings
269          * when vhost is running.
270          */
271         for (i = 0;  i < queue_pairs; i++) {
272             NetClientState *qnc = qemu_get_subqueue(n->nic, i);
273 
274             /* Purge both directions: TX and RX. */
275             qemu_net_queue_purge(qnc->peer->incoming_queue, qnc);
276             qemu_net_queue_purge(qnc->incoming_queue, qnc->peer);
277         }
278 
279         if (virtio_has_feature(vdev->guest_features, VIRTIO_NET_F_MTU)) {
280             r = vhost_net_set_mtu(get_vhost_net(nc->peer), n->net_conf.mtu);
281             if (r < 0) {
282                 error_report("%uBytes MTU not supported by the backend",
283                              n->net_conf.mtu);
284 
285                 return;
286             }
287         }
288 
289         n->vhost_started = 1;
290         r = vhost_net_start(vdev, n->nic->ncs, queue_pairs, cvq);
291         if (r < 0) {
292             error_report("unable to start vhost net: %d: "
293                          "falling back on userspace virtio", -r);
294             n->vhost_started = 0;
295         }
296     } else {
297         vhost_net_stop(vdev, n->nic->ncs, queue_pairs, cvq);
298         n->vhost_started = 0;
299     }
300 }
301 
302 static int virtio_net_set_vnet_endian_one(VirtIODevice *vdev,
303                                           NetClientState *peer,
304                                           bool enable)
305 {
306     if (virtio_is_big_endian(vdev)) {
307         return qemu_set_vnet_be(peer, enable);
308     } else {
309         return qemu_set_vnet_le(peer, enable);
310     }
311 }
312 
313 static bool virtio_net_set_vnet_endian(VirtIODevice *vdev, NetClientState *ncs,
314                                        int queue_pairs, bool enable)
315 {
316     int i;
317 
318     for (i = 0; i < queue_pairs; i++) {
319         if (virtio_net_set_vnet_endian_one(vdev, ncs[i].peer, enable) < 0 &&
320             enable) {
321             while (--i >= 0) {
322                 virtio_net_set_vnet_endian_one(vdev, ncs[i].peer, false);
323             }
324 
325             return true;
326         }
327     }
328 
329     return false;
330 }
331 
332 static void virtio_net_vnet_endian_status(VirtIONet *n, uint8_t status)
333 {
334     VirtIODevice *vdev = VIRTIO_DEVICE(n);
335     int queue_pairs = n->multiqueue ? n->max_queue_pairs : 1;
336 
337     if (virtio_net_started(n, status)) {
338         /* Before using the device, we tell the network backend about the
339          * endianness to use when parsing vnet headers. If the backend
340          * can't do it, we fallback onto fixing the headers in the core
341          * virtio-net code.
342          */
343         n->needs_vnet_hdr_swap = virtio_net_set_vnet_endian(vdev, n->nic->ncs,
344                                                             queue_pairs, true);
345     } else if (virtio_net_started(n, vdev->status)) {
346         /* After using the device, we need to reset the network backend to
347          * the default (guest native endianness), otherwise the guest may
348          * lose network connectivity if it is rebooted into a different
349          * endianness.
350          */
351         virtio_net_set_vnet_endian(vdev, n->nic->ncs, queue_pairs, false);
352     }
353 }
354 
355 static void virtio_net_drop_tx_queue_data(VirtIODevice *vdev, VirtQueue *vq)
356 {
357     unsigned int dropped = virtqueue_drop_all(vq);
358     if (dropped) {
359         virtio_notify(vdev, vq);
360     }
361 }
362 
363 static void virtio_net_set_status(struct VirtIODevice *vdev, uint8_t status)
364 {
365     VirtIONet *n = VIRTIO_NET(vdev);
366     VirtIONetQueue *q;
367     int i;
368     uint8_t queue_status;
369 
370     virtio_net_vnet_endian_status(n, status);
371     virtio_net_vhost_status(n, status);
372 
373     for (i = 0; i < n->max_queue_pairs; i++) {
374         NetClientState *ncs = qemu_get_subqueue(n->nic, i);
375         bool queue_started;
376         q = &n->vqs[i];
377 
378         if ((!n->multiqueue && i != 0) || i >= n->curr_queue_pairs) {
379             queue_status = 0;
380         } else {
381             queue_status = status;
382         }
383         queue_started =
384             virtio_net_started(n, queue_status) && !n->vhost_started;
385 
386         if (queue_started) {
387             qemu_flush_queued_packets(ncs);
388         }
389 
390         if (!q->tx_waiting) {
391             continue;
392         }
393 
394         if (queue_started) {
395             if (q->tx_timer) {
396                 timer_mod(q->tx_timer,
397                                qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) + n->tx_timeout);
398             } else {
399                 qemu_bh_schedule(q->tx_bh);
400             }
401         } else {
402             if (q->tx_timer) {
403                 timer_del(q->tx_timer);
404             } else {
405                 qemu_bh_cancel(q->tx_bh);
406             }
407             if ((n->status & VIRTIO_NET_S_LINK_UP) == 0 &&
408                 (queue_status & VIRTIO_CONFIG_S_DRIVER_OK) &&
409                 vdev->vm_running) {
410                 /* if tx is waiting we are likely have some packets in tx queue
411                  * and disabled notification */
412                 q->tx_waiting = 0;
413                 virtio_queue_set_notification(q->tx_vq, 1);
414                 virtio_net_drop_tx_queue_data(vdev, q->tx_vq);
415             }
416         }
417     }
418 }
419 
420 static void virtio_net_set_link_status(NetClientState *nc)
421 {
422     VirtIONet *n = qemu_get_nic_opaque(nc);
423     VirtIODevice *vdev = VIRTIO_DEVICE(n);
424     uint16_t old_status = n->status;
425 
426     if (nc->link_down)
427         n->status &= ~VIRTIO_NET_S_LINK_UP;
428     else
429         n->status |= VIRTIO_NET_S_LINK_UP;
430 
431     if (n->status != old_status)
432         virtio_notify_config(vdev);
433 
434     virtio_net_set_status(vdev, vdev->status);
435 }
436 
437 static void rxfilter_notify(NetClientState *nc)
438 {
439     VirtIONet *n = qemu_get_nic_opaque(nc);
440 
441     if (nc->rxfilter_notify_enabled) {
442         char *path = object_get_canonical_path(OBJECT(n->qdev));
443         qapi_event_send_nic_rx_filter_changed(!!n->netclient_name,
444                                               n->netclient_name, path);
445         g_free(path);
446 
447         /* disable event notification to avoid events flooding */
448         nc->rxfilter_notify_enabled = 0;
449     }
450 }
451 
452 static intList *get_vlan_table(VirtIONet *n)
453 {
454     intList *list;
455     int i, j;
456 
457     list = NULL;
458     for (i = 0; i < MAX_VLAN >> 5; i++) {
459         for (j = 0; n->vlans[i] && j <= 0x1f; j++) {
460             if (n->vlans[i] & (1U << j)) {
461                 QAPI_LIST_PREPEND(list, (i << 5) + j);
462             }
463         }
464     }
465 
466     return list;
467 }
468 
469 static RxFilterInfo *virtio_net_query_rxfilter(NetClientState *nc)
470 {
471     VirtIONet *n = qemu_get_nic_opaque(nc);
472     VirtIODevice *vdev = VIRTIO_DEVICE(n);
473     RxFilterInfo *info;
474     strList *str_list;
475     int i;
476 
477     info = g_malloc0(sizeof(*info));
478     info->name = g_strdup(nc->name);
479     info->promiscuous = n->promisc;
480 
481     if (n->nouni) {
482         info->unicast = RX_STATE_NONE;
483     } else if (n->alluni) {
484         info->unicast = RX_STATE_ALL;
485     } else {
486         info->unicast = RX_STATE_NORMAL;
487     }
488 
489     if (n->nomulti) {
490         info->multicast = RX_STATE_NONE;
491     } else if (n->allmulti) {
492         info->multicast = RX_STATE_ALL;
493     } else {
494         info->multicast = RX_STATE_NORMAL;
495     }
496 
497     info->broadcast_allowed = n->nobcast;
498     info->multicast_overflow = n->mac_table.multi_overflow;
499     info->unicast_overflow = n->mac_table.uni_overflow;
500 
501     info->main_mac = qemu_mac_strdup_printf(n->mac);
502 
503     str_list = NULL;
504     for (i = 0; i < n->mac_table.first_multi; i++) {
505         QAPI_LIST_PREPEND(str_list,
506                       qemu_mac_strdup_printf(n->mac_table.macs + i * ETH_ALEN));
507     }
508     info->unicast_table = str_list;
509 
510     str_list = NULL;
511     for (i = n->mac_table.first_multi; i < n->mac_table.in_use; i++) {
512         QAPI_LIST_PREPEND(str_list,
513                       qemu_mac_strdup_printf(n->mac_table.macs + i * ETH_ALEN));
514     }
515     info->multicast_table = str_list;
516     info->vlan_table = get_vlan_table(n);
517 
518     if (!virtio_vdev_has_feature(vdev, VIRTIO_NET_F_CTRL_VLAN)) {
519         info->vlan = RX_STATE_ALL;
520     } else if (!info->vlan_table) {
521         info->vlan = RX_STATE_NONE;
522     } else {
523         info->vlan = RX_STATE_NORMAL;
524     }
525 
526     /* enable event notification after query */
527     nc->rxfilter_notify_enabled = 1;
528 
529     return info;
530 }
531 
532 static void virtio_net_reset(VirtIODevice *vdev)
533 {
534     VirtIONet *n = VIRTIO_NET(vdev);
535     int i;
536 
537     /* Reset back to compatibility mode */
538     n->promisc = 1;
539     n->allmulti = 0;
540     n->alluni = 0;
541     n->nomulti = 0;
542     n->nouni = 0;
543     n->nobcast = 0;
544     /* multiqueue is disabled by default */
545     n->curr_queue_pairs = 1;
546     timer_del(n->announce_timer.tm);
547     n->announce_timer.round = 0;
548     n->status &= ~VIRTIO_NET_S_ANNOUNCE;
549 
550     /* Flush any MAC and VLAN filter table state */
551     n->mac_table.in_use = 0;
552     n->mac_table.first_multi = 0;
553     n->mac_table.multi_overflow = 0;
554     n->mac_table.uni_overflow = 0;
555     memset(n->mac_table.macs, 0, MAC_TABLE_ENTRIES * ETH_ALEN);
556     memcpy(&n->mac[0], &n->nic->conf->macaddr, sizeof(n->mac));
557     qemu_format_nic_info_str(qemu_get_queue(n->nic), n->mac);
558     memset(n->vlans, 0, MAX_VLAN >> 3);
559 
560     /* Flush any async TX */
561     for (i = 0;  i < n->max_queue_pairs; i++) {
562         NetClientState *nc = qemu_get_subqueue(n->nic, i);
563 
564         if (nc->peer) {
565             qemu_flush_or_purge_queued_packets(nc->peer, true);
566             assert(!virtio_net_get_subqueue(nc)->async_tx.elem);
567         }
568     }
569 }
570 
571 static void peer_test_vnet_hdr(VirtIONet *n)
572 {
573     NetClientState *nc = qemu_get_queue(n->nic);
574     if (!nc->peer) {
575         return;
576     }
577 
578     n->has_vnet_hdr = qemu_has_vnet_hdr(nc->peer);
579 }
580 
581 static int peer_has_vnet_hdr(VirtIONet *n)
582 {
583     return n->has_vnet_hdr;
584 }
585 
586 static int peer_has_ufo(VirtIONet *n)
587 {
588     if (!peer_has_vnet_hdr(n))
589         return 0;
590 
591     n->has_ufo = qemu_has_ufo(qemu_get_queue(n->nic)->peer);
592 
593     return n->has_ufo;
594 }
595 
596 static void virtio_net_set_mrg_rx_bufs(VirtIONet *n, int mergeable_rx_bufs,
597                                        int version_1, int hash_report)
598 {
599     int i;
600     NetClientState *nc;
601 
602     n->mergeable_rx_bufs = mergeable_rx_bufs;
603 
604     if (version_1) {
605         n->guest_hdr_len = hash_report ?
606             sizeof(struct virtio_net_hdr_v1_hash) :
607             sizeof(struct virtio_net_hdr_mrg_rxbuf);
608         n->rss_data.populate_hash = !!hash_report;
609     } else {
610         n->guest_hdr_len = n->mergeable_rx_bufs ?
611             sizeof(struct virtio_net_hdr_mrg_rxbuf) :
612             sizeof(struct virtio_net_hdr);
613     }
614 
615     for (i = 0; i < n->max_queue_pairs; i++) {
616         nc = qemu_get_subqueue(n->nic, i);
617 
618         if (peer_has_vnet_hdr(n) &&
619             qemu_has_vnet_hdr_len(nc->peer, n->guest_hdr_len)) {
620             qemu_set_vnet_hdr_len(nc->peer, n->guest_hdr_len);
621             n->host_hdr_len = n->guest_hdr_len;
622         }
623     }
624 }
625 
626 static int virtio_net_max_tx_queue_size(VirtIONet *n)
627 {
628     NetClientState *peer = n->nic_conf.peers.ncs[0];
629 
630     /*
631      * Backends other than vhost-user or vhost-vdpa don't support max queue
632      * size.
633      */
634     if (!peer) {
635         return VIRTIO_NET_TX_QUEUE_DEFAULT_SIZE;
636     }
637 
638     switch(peer->info->type) {
639     case NET_CLIENT_DRIVER_VHOST_USER:
640     case NET_CLIENT_DRIVER_VHOST_VDPA:
641         return VIRTQUEUE_MAX_SIZE;
642     default:
643         return VIRTIO_NET_TX_QUEUE_DEFAULT_SIZE;
644     };
645 }
646 
647 static int peer_attach(VirtIONet *n, int index)
648 {
649     NetClientState *nc = qemu_get_subqueue(n->nic, index);
650 
651     if (!nc->peer) {
652         return 0;
653     }
654 
655     if (nc->peer->info->type == NET_CLIENT_DRIVER_VHOST_USER) {
656         vhost_set_vring_enable(nc->peer, 1);
657     }
658 
659     if (nc->peer->info->type != NET_CLIENT_DRIVER_TAP) {
660         return 0;
661     }
662 
663     if (n->max_queue_pairs == 1) {
664         return 0;
665     }
666 
667     return tap_enable(nc->peer);
668 }
669 
670 static int peer_detach(VirtIONet *n, int index)
671 {
672     NetClientState *nc = qemu_get_subqueue(n->nic, index);
673 
674     if (!nc->peer) {
675         return 0;
676     }
677 
678     if (nc->peer->info->type == NET_CLIENT_DRIVER_VHOST_USER) {
679         vhost_set_vring_enable(nc->peer, 0);
680     }
681 
682     if (nc->peer->info->type !=  NET_CLIENT_DRIVER_TAP) {
683         return 0;
684     }
685 
686     return tap_disable(nc->peer);
687 }
688 
689 static void virtio_net_set_queue_pairs(VirtIONet *n)
690 {
691     int i;
692     int r;
693 
694     if (n->nic->peer_deleted) {
695         return;
696     }
697 
698     for (i = 0; i < n->max_queue_pairs; i++) {
699         if (i < n->curr_queue_pairs) {
700             r = peer_attach(n, i);
701             assert(!r);
702         } else {
703             r = peer_detach(n, i);
704             assert(!r);
705         }
706     }
707 }
708 
709 static void virtio_net_set_multiqueue(VirtIONet *n, int multiqueue);
710 
711 static uint64_t virtio_net_get_features(VirtIODevice *vdev, uint64_t features,
712                                         Error **errp)
713 {
714     VirtIONet *n = VIRTIO_NET(vdev);
715     NetClientState *nc = qemu_get_queue(n->nic);
716 
717     /* Firstly sync all virtio-net possible supported features */
718     features |= n->host_features;
719 
720     virtio_add_feature(&features, VIRTIO_NET_F_MAC);
721 
722     if (!peer_has_vnet_hdr(n)) {
723         virtio_clear_feature(&features, VIRTIO_NET_F_CSUM);
724         virtio_clear_feature(&features, VIRTIO_NET_F_HOST_TSO4);
725         virtio_clear_feature(&features, VIRTIO_NET_F_HOST_TSO6);
726         virtio_clear_feature(&features, VIRTIO_NET_F_HOST_ECN);
727 
728         virtio_clear_feature(&features, VIRTIO_NET_F_GUEST_CSUM);
729         virtio_clear_feature(&features, VIRTIO_NET_F_GUEST_TSO4);
730         virtio_clear_feature(&features, VIRTIO_NET_F_GUEST_TSO6);
731         virtio_clear_feature(&features, VIRTIO_NET_F_GUEST_ECN);
732 
733         virtio_clear_feature(&features, VIRTIO_NET_F_HASH_REPORT);
734     }
735 
736     if (!peer_has_vnet_hdr(n) || !peer_has_ufo(n)) {
737         virtio_clear_feature(&features, VIRTIO_NET_F_GUEST_UFO);
738         virtio_clear_feature(&features, VIRTIO_NET_F_HOST_UFO);
739     }
740 
741     if (!get_vhost_net(nc->peer)) {
742         return features;
743     }
744 
745     if (!ebpf_rss_is_loaded(&n->ebpf_rss)) {
746         virtio_clear_feature(&features, VIRTIO_NET_F_RSS);
747     }
748     features = vhost_net_get_features(get_vhost_net(nc->peer), features);
749     vdev->backend_features = features;
750 
751     if (n->mtu_bypass_backend &&
752             (n->host_features & 1ULL << VIRTIO_NET_F_MTU)) {
753         features |= (1ULL << VIRTIO_NET_F_MTU);
754     }
755 
756     return features;
757 }
758 
759 static uint64_t virtio_net_bad_features(VirtIODevice *vdev)
760 {
761     uint64_t features = 0;
762 
763     /* Linux kernel 2.6.25.  It understood MAC (as everyone must),
764      * but also these: */
765     virtio_add_feature(&features, VIRTIO_NET_F_MAC);
766     virtio_add_feature(&features, VIRTIO_NET_F_CSUM);
767     virtio_add_feature(&features, VIRTIO_NET_F_HOST_TSO4);
768     virtio_add_feature(&features, VIRTIO_NET_F_HOST_TSO6);
769     virtio_add_feature(&features, VIRTIO_NET_F_HOST_ECN);
770 
771     return features;
772 }
773 
774 static void virtio_net_apply_guest_offloads(VirtIONet *n)
775 {
776     qemu_set_offload(qemu_get_queue(n->nic)->peer,
777             !!(n->curr_guest_offloads & (1ULL << VIRTIO_NET_F_GUEST_CSUM)),
778             !!(n->curr_guest_offloads & (1ULL << VIRTIO_NET_F_GUEST_TSO4)),
779             !!(n->curr_guest_offloads & (1ULL << VIRTIO_NET_F_GUEST_TSO6)),
780             !!(n->curr_guest_offloads & (1ULL << VIRTIO_NET_F_GUEST_ECN)),
781             !!(n->curr_guest_offloads & (1ULL << VIRTIO_NET_F_GUEST_UFO)));
782 }
783 
784 static uint64_t virtio_net_guest_offloads_by_features(uint32_t features)
785 {
786     static const uint64_t guest_offloads_mask =
787         (1ULL << VIRTIO_NET_F_GUEST_CSUM) |
788         (1ULL << VIRTIO_NET_F_GUEST_TSO4) |
789         (1ULL << VIRTIO_NET_F_GUEST_TSO6) |
790         (1ULL << VIRTIO_NET_F_GUEST_ECN)  |
791         (1ULL << VIRTIO_NET_F_GUEST_UFO);
792 
793     return guest_offloads_mask & features;
794 }
795 
796 static inline uint64_t virtio_net_supported_guest_offloads(VirtIONet *n)
797 {
798     VirtIODevice *vdev = VIRTIO_DEVICE(n);
799     return virtio_net_guest_offloads_by_features(vdev->guest_features);
800 }
801 
802 typedef struct {
803     VirtIONet *n;
804     DeviceState *dev;
805 } FailoverDevice;
806 
807 /**
808  * Set the failover primary device
809  *
810  * @opaque: FailoverId to setup
811  * @opts: opts for device we are handling
812  * @errp: returns an error if this function fails
813  */
814 static int failover_set_primary(DeviceState *dev, void *opaque)
815 {
816     FailoverDevice *fdev = opaque;
817     PCIDevice *pci_dev = (PCIDevice *)
818         object_dynamic_cast(OBJECT(dev), TYPE_PCI_DEVICE);
819 
820     if (!pci_dev) {
821         return 0;
822     }
823 
824     if (!g_strcmp0(pci_dev->failover_pair_id, fdev->n->netclient_name)) {
825         fdev->dev = dev;
826         return 1;
827     }
828 
829     return 0;
830 }
831 
832 /**
833  * Find the primary device for this failover virtio-net
834  *
835  * @n: VirtIONet device
836  * @errp: returns an error if this function fails
837  */
838 static DeviceState *failover_find_primary_device(VirtIONet *n)
839 {
840     FailoverDevice fdev = {
841         .n = n,
842     };
843 
844     qbus_walk_children(sysbus_get_default(), failover_set_primary, NULL,
845                        NULL, NULL, &fdev);
846     return fdev.dev;
847 }
848 
849 static void failover_add_primary(VirtIONet *n, Error **errp)
850 {
851     Error *err = NULL;
852     DeviceState *dev = failover_find_primary_device(n);
853 
854     if (dev) {
855         return;
856     }
857 
858     if (!n->primary_opts) {
859         error_setg(errp, "Primary device not found");
860         error_append_hint(errp, "Virtio-net failover will not work. Make "
861                           "sure primary device has parameter"
862                           " failover_pair_id=%s\n", n->netclient_name);
863         return;
864     }
865 
866     dev = qdev_device_add_from_qdict(n->primary_opts,
867                                      n->primary_opts_from_json,
868                                      &err);
869     if (err) {
870         qobject_unref(n->primary_opts);
871         n->primary_opts = NULL;
872     } else {
873         object_unref(OBJECT(dev));
874     }
875     error_propagate(errp, err);
876 }
877 
878 static void virtio_net_set_features(VirtIODevice *vdev, uint64_t features)
879 {
880     VirtIONet *n = VIRTIO_NET(vdev);
881     Error *err = NULL;
882     int i;
883 
884     if (n->mtu_bypass_backend &&
885             !virtio_has_feature(vdev->backend_features, VIRTIO_NET_F_MTU)) {
886         features &= ~(1ULL << VIRTIO_NET_F_MTU);
887     }
888 
889     virtio_net_set_multiqueue(n,
890                               virtio_has_feature(features, VIRTIO_NET_F_RSS) ||
891                               virtio_has_feature(features, VIRTIO_NET_F_MQ));
892 
893     virtio_net_set_mrg_rx_bufs(n,
894                                virtio_has_feature(features,
895                                                   VIRTIO_NET_F_MRG_RXBUF),
896                                virtio_has_feature(features,
897                                                   VIRTIO_F_VERSION_1),
898                                virtio_has_feature(features,
899                                                   VIRTIO_NET_F_HASH_REPORT));
900 
901     n->rsc4_enabled = virtio_has_feature(features, VIRTIO_NET_F_RSC_EXT) &&
902         virtio_has_feature(features, VIRTIO_NET_F_GUEST_TSO4);
903     n->rsc6_enabled = virtio_has_feature(features, VIRTIO_NET_F_RSC_EXT) &&
904         virtio_has_feature(features, VIRTIO_NET_F_GUEST_TSO6);
905     n->rss_data.redirect = virtio_has_feature(features, VIRTIO_NET_F_RSS);
906 
907     if (n->has_vnet_hdr) {
908         n->curr_guest_offloads =
909             virtio_net_guest_offloads_by_features(features);
910         virtio_net_apply_guest_offloads(n);
911     }
912 
913     for (i = 0;  i < n->max_queue_pairs; i++) {
914         NetClientState *nc = qemu_get_subqueue(n->nic, i);
915 
916         if (!get_vhost_net(nc->peer)) {
917             continue;
918         }
919         vhost_net_ack_features(get_vhost_net(nc->peer), features);
920     }
921 
922     if (virtio_has_feature(features, VIRTIO_NET_F_CTRL_VLAN)) {
923         memset(n->vlans, 0, MAX_VLAN >> 3);
924     } else {
925         memset(n->vlans, 0xff, MAX_VLAN >> 3);
926     }
927 
928     if (virtio_has_feature(features, VIRTIO_NET_F_STANDBY)) {
929         qapi_event_send_failover_negotiated(n->netclient_name);
930         qatomic_set(&n->failover_primary_hidden, false);
931         failover_add_primary(n, &err);
932         if (err) {
933             if (!qtest_enabled()) {
934                 warn_report_err(err);
935             } else {
936                 error_free(err);
937             }
938         }
939     }
940 }
941 
942 static int virtio_net_handle_rx_mode(VirtIONet *n, uint8_t cmd,
943                                      struct iovec *iov, unsigned int iov_cnt)
944 {
945     uint8_t on;
946     size_t s;
947     NetClientState *nc = qemu_get_queue(n->nic);
948 
949     s = iov_to_buf(iov, iov_cnt, 0, &on, sizeof(on));
950     if (s != sizeof(on)) {
951         return VIRTIO_NET_ERR;
952     }
953 
954     if (cmd == VIRTIO_NET_CTRL_RX_PROMISC) {
955         n->promisc = on;
956     } else if (cmd == VIRTIO_NET_CTRL_RX_ALLMULTI) {
957         n->allmulti = on;
958     } else if (cmd == VIRTIO_NET_CTRL_RX_ALLUNI) {
959         n->alluni = on;
960     } else if (cmd == VIRTIO_NET_CTRL_RX_NOMULTI) {
961         n->nomulti = on;
962     } else if (cmd == VIRTIO_NET_CTRL_RX_NOUNI) {
963         n->nouni = on;
964     } else if (cmd == VIRTIO_NET_CTRL_RX_NOBCAST) {
965         n->nobcast = on;
966     } else {
967         return VIRTIO_NET_ERR;
968     }
969 
970     rxfilter_notify(nc);
971 
972     return VIRTIO_NET_OK;
973 }
974 
975 static int virtio_net_handle_offloads(VirtIONet *n, uint8_t cmd,
976                                      struct iovec *iov, unsigned int iov_cnt)
977 {
978     VirtIODevice *vdev = VIRTIO_DEVICE(n);
979     uint64_t offloads;
980     size_t s;
981 
982     if (!virtio_vdev_has_feature(vdev, VIRTIO_NET_F_CTRL_GUEST_OFFLOADS)) {
983         return VIRTIO_NET_ERR;
984     }
985 
986     s = iov_to_buf(iov, iov_cnt, 0, &offloads, sizeof(offloads));
987     if (s != sizeof(offloads)) {
988         return VIRTIO_NET_ERR;
989     }
990 
991     if (cmd == VIRTIO_NET_CTRL_GUEST_OFFLOADS_SET) {
992         uint64_t supported_offloads;
993 
994         offloads = virtio_ldq_p(vdev, &offloads);
995 
996         if (!n->has_vnet_hdr) {
997             return VIRTIO_NET_ERR;
998         }
999 
1000         n->rsc4_enabled = virtio_has_feature(offloads, VIRTIO_NET_F_RSC_EXT) &&
1001             virtio_has_feature(offloads, VIRTIO_NET_F_GUEST_TSO4);
1002         n->rsc6_enabled = virtio_has_feature(offloads, VIRTIO_NET_F_RSC_EXT) &&
1003             virtio_has_feature(offloads, VIRTIO_NET_F_GUEST_TSO6);
1004         virtio_clear_feature(&offloads, VIRTIO_NET_F_RSC_EXT);
1005 
1006         supported_offloads = virtio_net_supported_guest_offloads(n);
1007         if (offloads & ~supported_offloads) {
1008             return VIRTIO_NET_ERR;
1009         }
1010 
1011         n->curr_guest_offloads = offloads;
1012         virtio_net_apply_guest_offloads(n);
1013 
1014         return VIRTIO_NET_OK;
1015     } else {
1016         return VIRTIO_NET_ERR;
1017     }
1018 }
1019 
1020 static int virtio_net_handle_mac(VirtIONet *n, uint8_t cmd,
1021                                  struct iovec *iov, unsigned int iov_cnt)
1022 {
1023     VirtIODevice *vdev = VIRTIO_DEVICE(n);
1024     struct virtio_net_ctrl_mac mac_data;
1025     size_t s;
1026     NetClientState *nc = qemu_get_queue(n->nic);
1027 
1028     if (cmd == VIRTIO_NET_CTRL_MAC_ADDR_SET) {
1029         if (iov_size(iov, iov_cnt) != sizeof(n->mac)) {
1030             return VIRTIO_NET_ERR;
1031         }
1032         s = iov_to_buf(iov, iov_cnt, 0, &n->mac, sizeof(n->mac));
1033         assert(s == sizeof(n->mac));
1034         qemu_format_nic_info_str(qemu_get_queue(n->nic), n->mac);
1035         rxfilter_notify(nc);
1036 
1037         return VIRTIO_NET_OK;
1038     }
1039 
1040     if (cmd != VIRTIO_NET_CTRL_MAC_TABLE_SET) {
1041         return VIRTIO_NET_ERR;
1042     }
1043 
1044     int in_use = 0;
1045     int first_multi = 0;
1046     uint8_t uni_overflow = 0;
1047     uint8_t multi_overflow = 0;
1048     uint8_t *macs = g_malloc0(MAC_TABLE_ENTRIES * ETH_ALEN);
1049 
1050     s = iov_to_buf(iov, iov_cnt, 0, &mac_data.entries,
1051                    sizeof(mac_data.entries));
1052     mac_data.entries = virtio_ldl_p(vdev, &mac_data.entries);
1053     if (s != sizeof(mac_data.entries)) {
1054         goto error;
1055     }
1056     iov_discard_front(&iov, &iov_cnt, s);
1057 
1058     if (mac_data.entries * ETH_ALEN > iov_size(iov, iov_cnt)) {
1059         goto error;
1060     }
1061 
1062     if (mac_data.entries <= MAC_TABLE_ENTRIES) {
1063         s = iov_to_buf(iov, iov_cnt, 0, macs,
1064                        mac_data.entries * ETH_ALEN);
1065         if (s != mac_data.entries * ETH_ALEN) {
1066             goto error;
1067         }
1068         in_use += mac_data.entries;
1069     } else {
1070         uni_overflow = 1;
1071     }
1072 
1073     iov_discard_front(&iov, &iov_cnt, mac_data.entries * ETH_ALEN);
1074 
1075     first_multi = in_use;
1076 
1077     s = iov_to_buf(iov, iov_cnt, 0, &mac_data.entries,
1078                    sizeof(mac_data.entries));
1079     mac_data.entries = virtio_ldl_p(vdev, &mac_data.entries);
1080     if (s != sizeof(mac_data.entries)) {
1081         goto error;
1082     }
1083 
1084     iov_discard_front(&iov, &iov_cnt, s);
1085 
1086     if (mac_data.entries * ETH_ALEN != iov_size(iov, iov_cnt)) {
1087         goto error;
1088     }
1089 
1090     if (mac_data.entries <= MAC_TABLE_ENTRIES - in_use) {
1091         s = iov_to_buf(iov, iov_cnt, 0, &macs[in_use * ETH_ALEN],
1092                        mac_data.entries * ETH_ALEN);
1093         if (s != mac_data.entries * ETH_ALEN) {
1094             goto error;
1095         }
1096         in_use += mac_data.entries;
1097     } else {
1098         multi_overflow = 1;
1099     }
1100 
1101     n->mac_table.in_use = in_use;
1102     n->mac_table.first_multi = first_multi;
1103     n->mac_table.uni_overflow = uni_overflow;
1104     n->mac_table.multi_overflow = multi_overflow;
1105     memcpy(n->mac_table.macs, macs, MAC_TABLE_ENTRIES * ETH_ALEN);
1106     g_free(macs);
1107     rxfilter_notify(nc);
1108 
1109     return VIRTIO_NET_OK;
1110 
1111 error:
1112     g_free(macs);
1113     return VIRTIO_NET_ERR;
1114 }
1115 
1116 static int virtio_net_handle_vlan_table(VirtIONet *n, uint8_t cmd,
1117                                         struct iovec *iov, unsigned int iov_cnt)
1118 {
1119     VirtIODevice *vdev = VIRTIO_DEVICE(n);
1120     uint16_t vid;
1121     size_t s;
1122     NetClientState *nc = qemu_get_queue(n->nic);
1123 
1124     s = iov_to_buf(iov, iov_cnt, 0, &vid, sizeof(vid));
1125     vid = virtio_lduw_p(vdev, &vid);
1126     if (s != sizeof(vid)) {
1127         return VIRTIO_NET_ERR;
1128     }
1129 
1130     if (vid >= MAX_VLAN)
1131         return VIRTIO_NET_ERR;
1132 
1133     if (cmd == VIRTIO_NET_CTRL_VLAN_ADD)
1134         n->vlans[vid >> 5] |= (1U << (vid & 0x1f));
1135     else if (cmd == VIRTIO_NET_CTRL_VLAN_DEL)
1136         n->vlans[vid >> 5] &= ~(1U << (vid & 0x1f));
1137     else
1138         return VIRTIO_NET_ERR;
1139 
1140     rxfilter_notify(nc);
1141 
1142     return VIRTIO_NET_OK;
1143 }
1144 
1145 static int virtio_net_handle_announce(VirtIONet *n, uint8_t cmd,
1146                                       struct iovec *iov, unsigned int iov_cnt)
1147 {
1148     trace_virtio_net_handle_announce(n->announce_timer.round);
1149     if (cmd == VIRTIO_NET_CTRL_ANNOUNCE_ACK &&
1150         n->status & VIRTIO_NET_S_ANNOUNCE) {
1151         n->status &= ~VIRTIO_NET_S_ANNOUNCE;
1152         if (n->announce_timer.round) {
1153             qemu_announce_timer_step(&n->announce_timer);
1154         }
1155         return VIRTIO_NET_OK;
1156     } else {
1157         return VIRTIO_NET_ERR;
1158     }
1159 }
1160 
1161 static void virtio_net_detach_epbf_rss(VirtIONet *n);
1162 
1163 static void virtio_net_disable_rss(VirtIONet *n)
1164 {
1165     if (n->rss_data.enabled) {
1166         trace_virtio_net_rss_disable();
1167     }
1168     n->rss_data.enabled = false;
1169 
1170     virtio_net_detach_epbf_rss(n);
1171 }
1172 
1173 static bool virtio_net_attach_ebpf_to_backend(NICState *nic, int prog_fd)
1174 {
1175     NetClientState *nc = qemu_get_peer(qemu_get_queue(nic), 0);
1176     if (nc == NULL || nc->info->set_steering_ebpf == NULL) {
1177         return false;
1178     }
1179 
1180     return nc->info->set_steering_ebpf(nc, prog_fd);
1181 }
1182 
1183 static void rss_data_to_rss_config(struct VirtioNetRssData *data,
1184                                    struct EBPFRSSConfig *config)
1185 {
1186     config->redirect = data->redirect;
1187     config->populate_hash = data->populate_hash;
1188     config->hash_types = data->hash_types;
1189     config->indirections_len = data->indirections_len;
1190     config->default_queue = data->default_queue;
1191 }
1192 
1193 static bool virtio_net_attach_epbf_rss(VirtIONet *n)
1194 {
1195     struct EBPFRSSConfig config = {};
1196 
1197     if (!ebpf_rss_is_loaded(&n->ebpf_rss)) {
1198         return false;
1199     }
1200 
1201     rss_data_to_rss_config(&n->rss_data, &config);
1202 
1203     if (!ebpf_rss_set_all(&n->ebpf_rss, &config,
1204                           n->rss_data.indirections_table, n->rss_data.key)) {
1205         return false;
1206     }
1207 
1208     if (!virtio_net_attach_ebpf_to_backend(n->nic, n->ebpf_rss.program_fd)) {
1209         return false;
1210     }
1211 
1212     return true;
1213 }
1214 
1215 static void virtio_net_detach_epbf_rss(VirtIONet *n)
1216 {
1217     virtio_net_attach_ebpf_to_backend(n->nic, -1);
1218 }
1219 
1220 static bool virtio_net_load_ebpf(VirtIONet *n)
1221 {
1222     if (!virtio_net_attach_ebpf_to_backend(n->nic, -1)) {
1223         /* backend does't support steering ebpf */
1224         return false;
1225     }
1226 
1227     return ebpf_rss_load(&n->ebpf_rss);
1228 }
1229 
1230 static void virtio_net_unload_ebpf(VirtIONet *n)
1231 {
1232     virtio_net_attach_ebpf_to_backend(n->nic, -1);
1233     ebpf_rss_unload(&n->ebpf_rss);
1234 }
1235 
1236 static uint16_t virtio_net_handle_rss(VirtIONet *n,
1237                                       struct iovec *iov,
1238                                       unsigned int iov_cnt,
1239                                       bool do_rss)
1240 {
1241     VirtIODevice *vdev = VIRTIO_DEVICE(n);
1242     struct virtio_net_rss_config cfg;
1243     size_t s, offset = 0, size_get;
1244     uint16_t queue_pairs, i;
1245     struct {
1246         uint16_t us;
1247         uint8_t b;
1248     } QEMU_PACKED temp;
1249     const char *err_msg = "";
1250     uint32_t err_value = 0;
1251 
1252     if (do_rss && !virtio_vdev_has_feature(vdev, VIRTIO_NET_F_RSS)) {
1253         err_msg = "RSS is not negotiated";
1254         goto error;
1255     }
1256     if (!do_rss && !virtio_vdev_has_feature(vdev, VIRTIO_NET_F_HASH_REPORT)) {
1257         err_msg = "Hash report is not negotiated";
1258         goto error;
1259     }
1260     size_get = offsetof(struct virtio_net_rss_config, indirection_table);
1261     s = iov_to_buf(iov, iov_cnt, offset, &cfg, size_get);
1262     if (s != size_get) {
1263         err_msg = "Short command buffer";
1264         err_value = (uint32_t)s;
1265         goto error;
1266     }
1267     n->rss_data.hash_types = virtio_ldl_p(vdev, &cfg.hash_types);
1268     n->rss_data.indirections_len =
1269         virtio_lduw_p(vdev, &cfg.indirection_table_mask);
1270     n->rss_data.indirections_len++;
1271     if (!do_rss) {
1272         n->rss_data.indirections_len = 1;
1273     }
1274     if (!is_power_of_2(n->rss_data.indirections_len)) {
1275         err_msg = "Invalid size of indirection table";
1276         err_value = n->rss_data.indirections_len;
1277         goto error;
1278     }
1279     if (n->rss_data.indirections_len > VIRTIO_NET_RSS_MAX_TABLE_LEN) {
1280         err_msg = "Too large indirection table";
1281         err_value = n->rss_data.indirections_len;
1282         goto error;
1283     }
1284     n->rss_data.default_queue = do_rss ?
1285         virtio_lduw_p(vdev, &cfg.unclassified_queue) : 0;
1286     if (n->rss_data.default_queue >= n->max_queue_pairs) {
1287         err_msg = "Invalid default queue";
1288         err_value = n->rss_data.default_queue;
1289         goto error;
1290     }
1291     offset += size_get;
1292     size_get = sizeof(uint16_t) * n->rss_data.indirections_len;
1293     g_free(n->rss_data.indirections_table);
1294     n->rss_data.indirections_table = g_malloc(size_get);
1295     if (!n->rss_data.indirections_table) {
1296         err_msg = "Can't allocate indirections table";
1297         err_value = n->rss_data.indirections_len;
1298         goto error;
1299     }
1300     s = iov_to_buf(iov, iov_cnt, offset,
1301                    n->rss_data.indirections_table, size_get);
1302     if (s != size_get) {
1303         err_msg = "Short indirection table buffer";
1304         err_value = (uint32_t)s;
1305         goto error;
1306     }
1307     for (i = 0; i < n->rss_data.indirections_len; ++i) {
1308         uint16_t val = n->rss_data.indirections_table[i];
1309         n->rss_data.indirections_table[i] = virtio_lduw_p(vdev, &val);
1310     }
1311     offset += size_get;
1312     size_get = sizeof(temp);
1313     s = iov_to_buf(iov, iov_cnt, offset, &temp, size_get);
1314     if (s != size_get) {
1315         err_msg = "Can't get queue_pairs";
1316         err_value = (uint32_t)s;
1317         goto error;
1318     }
1319     queue_pairs = do_rss ? virtio_lduw_p(vdev, &temp.us) : n->curr_queue_pairs;
1320     if (queue_pairs == 0 || queue_pairs > n->max_queue_pairs) {
1321         err_msg = "Invalid number of queue_pairs";
1322         err_value = queue_pairs;
1323         goto error;
1324     }
1325     if (temp.b > VIRTIO_NET_RSS_MAX_KEY_SIZE) {
1326         err_msg = "Invalid key size";
1327         err_value = temp.b;
1328         goto error;
1329     }
1330     if (!temp.b && n->rss_data.hash_types) {
1331         err_msg = "No key provided";
1332         err_value = 0;
1333         goto error;
1334     }
1335     if (!temp.b && !n->rss_data.hash_types) {
1336         virtio_net_disable_rss(n);
1337         return queue_pairs;
1338     }
1339     offset += size_get;
1340     size_get = temp.b;
1341     s = iov_to_buf(iov, iov_cnt, offset, n->rss_data.key, size_get);
1342     if (s != size_get) {
1343         err_msg = "Can get key buffer";
1344         err_value = (uint32_t)s;
1345         goto error;
1346     }
1347     n->rss_data.enabled = true;
1348 
1349     if (!n->rss_data.populate_hash) {
1350         if (!virtio_net_attach_epbf_rss(n)) {
1351             /* EBPF must be loaded for vhost */
1352             if (get_vhost_net(qemu_get_queue(n->nic)->peer)) {
1353                 warn_report("Can't load eBPF RSS for vhost");
1354                 goto error;
1355             }
1356             /* fallback to software RSS */
1357             warn_report("Can't load eBPF RSS - fallback to software RSS");
1358             n->rss_data.enabled_software_rss = true;
1359         }
1360     } else {
1361         /* use software RSS for hash populating */
1362         /* and detach eBPF if was loaded before */
1363         virtio_net_detach_epbf_rss(n);
1364         n->rss_data.enabled_software_rss = true;
1365     }
1366 
1367     trace_virtio_net_rss_enable(n->rss_data.hash_types,
1368                                 n->rss_data.indirections_len,
1369                                 temp.b);
1370     return queue_pairs;
1371 error:
1372     trace_virtio_net_rss_error(err_msg, err_value);
1373     virtio_net_disable_rss(n);
1374     return 0;
1375 }
1376 
1377 static int virtio_net_handle_mq(VirtIONet *n, uint8_t cmd,
1378                                 struct iovec *iov, unsigned int iov_cnt)
1379 {
1380     VirtIODevice *vdev = VIRTIO_DEVICE(n);
1381     uint16_t queue_pairs;
1382 
1383     virtio_net_disable_rss(n);
1384     if (cmd == VIRTIO_NET_CTRL_MQ_HASH_CONFIG) {
1385         queue_pairs = virtio_net_handle_rss(n, iov, iov_cnt, false);
1386         return queue_pairs ? VIRTIO_NET_OK : VIRTIO_NET_ERR;
1387     }
1388     if (cmd == VIRTIO_NET_CTRL_MQ_RSS_CONFIG) {
1389         queue_pairs = virtio_net_handle_rss(n, iov, iov_cnt, true);
1390     } else if (cmd == VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET) {
1391         struct virtio_net_ctrl_mq mq;
1392         size_t s;
1393         if (!virtio_vdev_has_feature(vdev, VIRTIO_NET_F_MQ)) {
1394             return VIRTIO_NET_ERR;
1395         }
1396         s = iov_to_buf(iov, iov_cnt, 0, &mq, sizeof(mq));
1397         if (s != sizeof(mq)) {
1398             return VIRTIO_NET_ERR;
1399         }
1400         queue_pairs = virtio_lduw_p(vdev, &mq.virtqueue_pairs);
1401 
1402     } else {
1403         return VIRTIO_NET_ERR;
1404     }
1405 
1406     if (queue_pairs < VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MIN ||
1407         queue_pairs > VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MAX ||
1408         queue_pairs > n->max_queue_pairs ||
1409         !n->multiqueue) {
1410         return VIRTIO_NET_ERR;
1411     }
1412 
1413     n->curr_queue_pairs = queue_pairs;
1414     /* stop the backend before changing the number of queue_pairs to avoid handling a
1415      * disabled queue */
1416     virtio_net_set_status(vdev, vdev->status);
1417     virtio_net_set_queue_pairs(n);
1418 
1419     return VIRTIO_NET_OK;
1420 }
1421 
1422 static void virtio_net_handle_ctrl(VirtIODevice *vdev, VirtQueue *vq)
1423 {
1424     VirtIONet *n = VIRTIO_NET(vdev);
1425     struct virtio_net_ctrl_hdr ctrl;
1426     virtio_net_ctrl_ack status = VIRTIO_NET_ERR;
1427     VirtQueueElement *elem;
1428     size_t s;
1429     struct iovec *iov, *iov2;
1430     unsigned int iov_cnt;
1431 
1432     for (;;) {
1433         elem = virtqueue_pop(vq, sizeof(VirtQueueElement));
1434         if (!elem) {
1435             break;
1436         }
1437         if (iov_size(elem->in_sg, elem->in_num) < sizeof(status) ||
1438             iov_size(elem->out_sg, elem->out_num) < sizeof(ctrl)) {
1439             virtio_error(vdev, "virtio-net ctrl missing headers");
1440             virtqueue_detach_element(vq, elem, 0);
1441             g_free(elem);
1442             break;
1443         }
1444 
1445         iov_cnt = elem->out_num;
1446         iov2 = iov = g_memdup(elem->out_sg, sizeof(struct iovec) * elem->out_num);
1447         s = iov_to_buf(iov, iov_cnt, 0, &ctrl, sizeof(ctrl));
1448         iov_discard_front(&iov, &iov_cnt, sizeof(ctrl));
1449         if (s != sizeof(ctrl)) {
1450             status = VIRTIO_NET_ERR;
1451         } else if (ctrl.class == VIRTIO_NET_CTRL_RX) {
1452             status = virtio_net_handle_rx_mode(n, ctrl.cmd, iov, iov_cnt);
1453         } else if (ctrl.class == VIRTIO_NET_CTRL_MAC) {
1454             status = virtio_net_handle_mac(n, ctrl.cmd, iov, iov_cnt);
1455         } else if (ctrl.class == VIRTIO_NET_CTRL_VLAN) {
1456             status = virtio_net_handle_vlan_table(n, ctrl.cmd, iov, iov_cnt);
1457         } else if (ctrl.class == VIRTIO_NET_CTRL_ANNOUNCE) {
1458             status = virtio_net_handle_announce(n, ctrl.cmd, iov, iov_cnt);
1459         } else if (ctrl.class == VIRTIO_NET_CTRL_MQ) {
1460             status = virtio_net_handle_mq(n, ctrl.cmd, iov, iov_cnt);
1461         } else if (ctrl.class == VIRTIO_NET_CTRL_GUEST_OFFLOADS) {
1462             status = virtio_net_handle_offloads(n, ctrl.cmd, iov, iov_cnt);
1463         }
1464 
1465         s = iov_from_buf(elem->in_sg, elem->in_num, 0, &status, sizeof(status));
1466         assert(s == sizeof(status));
1467 
1468         virtqueue_push(vq, elem, sizeof(status));
1469         virtio_notify(vdev, vq);
1470         g_free(iov2);
1471         g_free(elem);
1472     }
1473 }
1474 
1475 /* RX */
1476 
1477 static void virtio_net_handle_rx(VirtIODevice *vdev, VirtQueue *vq)
1478 {
1479     VirtIONet *n = VIRTIO_NET(vdev);
1480     int queue_index = vq2q(virtio_get_queue_index(vq));
1481 
1482     qemu_flush_queued_packets(qemu_get_subqueue(n->nic, queue_index));
1483 }
1484 
1485 static bool virtio_net_can_receive(NetClientState *nc)
1486 {
1487     VirtIONet *n = qemu_get_nic_opaque(nc);
1488     VirtIODevice *vdev = VIRTIO_DEVICE(n);
1489     VirtIONetQueue *q = virtio_net_get_subqueue(nc);
1490 
1491     if (!vdev->vm_running) {
1492         return false;
1493     }
1494 
1495     if (nc->queue_index >= n->curr_queue_pairs) {
1496         return false;
1497     }
1498 
1499     if (!virtio_queue_ready(q->rx_vq) ||
1500         !(vdev->status & VIRTIO_CONFIG_S_DRIVER_OK)) {
1501         return false;
1502     }
1503 
1504     return true;
1505 }
1506 
1507 static int virtio_net_has_buffers(VirtIONetQueue *q, int bufsize)
1508 {
1509     VirtIONet *n = q->n;
1510     if (virtio_queue_empty(q->rx_vq) ||
1511         (n->mergeable_rx_bufs &&
1512          !virtqueue_avail_bytes(q->rx_vq, bufsize, 0))) {
1513         virtio_queue_set_notification(q->rx_vq, 1);
1514 
1515         /* To avoid a race condition where the guest has made some buffers
1516          * available after the above check but before notification was
1517          * enabled, check for available buffers again.
1518          */
1519         if (virtio_queue_empty(q->rx_vq) ||
1520             (n->mergeable_rx_bufs &&
1521              !virtqueue_avail_bytes(q->rx_vq, bufsize, 0))) {
1522             return 0;
1523         }
1524     }
1525 
1526     virtio_queue_set_notification(q->rx_vq, 0);
1527     return 1;
1528 }
1529 
1530 static void virtio_net_hdr_swap(VirtIODevice *vdev, struct virtio_net_hdr *hdr)
1531 {
1532     virtio_tswap16s(vdev, &hdr->hdr_len);
1533     virtio_tswap16s(vdev, &hdr->gso_size);
1534     virtio_tswap16s(vdev, &hdr->csum_start);
1535     virtio_tswap16s(vdev, &hdr->csum_offset);
1536 }
1537 
1538 /* dhclient uses AF_PACKET but doesn't pass auxdata to the kernel so
1539  * it never finds out that the packets don't have valid checksums.  This
1540  * causes dhclient to get upset.  Fedora's carried a patch for ages to
1541  * fix this with Xen but it hasn't appeared in an upstream release of
1542  * dhclient yet.
1543  *
1544  * To avoid breaking existing guests, we catch udp packets and add
1545  * checksums.  This is terrible but it's better than hacking the guest
1546  * kernels.
1547  *
1548  * N.B. if we introduce a zero-copy API, this operation is no longer free so
1549  * we should provide a mechanism to disable it to avoid polluting the host
1550  * cache.
1551  */
1552 static void work_around_broken_dhclient(struct virtio_net_hdr *hdr,
1553                                         uint8_t *buf, size_t size)
1554 {
1555     if ((hdr->flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) && /* missing csum */
1556         (size > 27 && size < 1500) && /* normal sized MTU */
1557         (buf[12] == 0x08 && buf[13] == 0x00) && /* ethertype == IPv4 */
1558         (buf[23] == 17) && /* ip.protocol == UDP */
1559         (buf[34] == 0 && buf[35] == 67)) { /* udp.srcport == bootps */
1560         net_checksum_calculate(buf, size, CSUM_UDP);
1561         hdr->flags &= ~VIRTIO_NET_HDR_F_NEEDS_CSUM;
1562     }
1563 }
1564 
1565 static void receive_header(VirtIONet *n, const struct iovec *iov, int iov_cnt,
1566                            const void *buf, size_t size)
1567 {
1568     if (n->has_vnet_hdr) {
1569         /* FIXME this cast is evil */
1570         void *wbuf = (void *)buf;
1571         work_around_broken_dhclient(wbuf, wbuf + n->host_hdr_len,
1572                                     size - n->host_hdr_len);
1573 
1574         if (n->needs_vnet_hdr_swap) {
1575             virtio_net_hdr_swap(VIRTIO_DEVICE(n), wbuf);
1576         }
1577         iov_from_buf(iov, iov_cnt, 0, buf, sizeof(struct virtio_net_hdr));
1578     } else {
1579         struct virtio_net_hdr hdr = {
1580             .flags = 0,
1581             .gso_type = VIRTIO_NET_HDR_GSO_NONE
1582         };
1583         iov_from_buf(iov, iov_cnt, 0, &hdr, sizeof hdr);
1584     }
1585 }
1586 
1587 static int receive_filter(VirtIONet *n, const uint8_t *buf, int size)
1588 {
1589     static const uint8_t bcast[] = {0xff, 0xff, 0xff, 0xff, 0xff, 0xff};
1590     static const uint8_t vlan[] = {0x81, 0x00};
1591     uint8_t *ptr = (uint8_t *)buf;
1592     int i;
1593 
1594     if (n->promisc)
1595         return 1;
1596 
1597     ptr += n->host_hdr_len;
1598 
1599     if (!memcmp(&ptr[12], vlan, sizeof(vlan))) {
1600         int vid = lduw_be_p(ptr + 14) & 0xfff;
1601         if (!(n->vlans[vid >> 5] & (1U << (vid & 0x1f))))
1602             return 0;
1603     }
1604 
1605     if (ptr[0] & 1) { // multicast
1606         if (!memcmp(ptr, bcast, sizeof(bcast))) {
1607             return !n->nobcast;
1608         } else if (n->nomulti) {
1609             return 0;
1610         } else if (n->allmulti || n->mac_table.multi_overflow) {
1611             return 1;
1612         }
1613 
1614         for (i = n->mac_table.first_multi; i < n->mac_table.in_use; i++) {
1615             if (!memcmp(ptr, &n->mac_table.macs[i * ETH_ALEN], ETH_ALEN)) {
1616                 return 1;
1617             }
1618         }
1619     } else { // unicast
1620         if (n->nouni) {
1621             return 0;
1622         } else if (n->alluni || n->mac_table.uni_overflow) {
1623             return 1;
1624         } else if (!memcmp(ptr, n->mac, ETH_ALEN)) {
1625             return 1;
1626         }
1627 
1628         for (i = 0; i < n->mac_table.first_multi; i++) {
1629             if (!memcmp(ptr, &n->mac_table.macs[i * ETH_ALEN], ETH_ALEN)) {
1630                 return 1;
1631             }
1632         }
1633     }
1634 
1635     return 0;
1636 }
1637 
1638 static uint8_t virtio_net_get_hash_type(bool isip4,
1639                                         bool isip6,
1640                                         bool isudp,
1641                                         bool istcp,
1642                                         uint32_t types)
1643 {
1644     if (isip4) {
1645         if (istcp && (types & VIRTIO_NET_RSS_HASH_TYPE_TCPv4)) {
1646             return NetPktRssIpV4Tcp;
1647         }
1648         if (isudp && (types & VIRTIO_NET_RSS_HASH_TYPE_UDPv4)) {
1649             return NetPktRssIpV4Udp;
1650         }
1651         if (types & VIRTIO_NET_RSS_HASH_TYPE_IPv4) {
1652             return NetPktRssIpV4;
1653         }
1654     } else if (isip6) {
1655         uint32_t mask = VIRTIO_NET_RSS_HASH_TYPE_TCP_EX |
1656                         VIRTIO_NET_RSS_HASH_TYPE_TCPv6;
1657 
1658         if (istcp && (types & mask)) {
1659             return (types & VIRTIO_NET_RSS_HASH_TYPE_TCP_EX) ?
1660                 NetPktRssIpV6TcpEx : NetPktRssIpV6Tcp;
1661         }
1662         mask = VIRTIO_NET_RSS_HASH_TYPE_UDP_EX | VIRTIO_NET_RSS_HASH_TYPE_UDPv6;
1663         if (isudp && (types & mask)) {
1664             return (types & VIRTIO_NET_RSS_HASH_TYPE_UDP_EX) ?
1665                 NetPktRssIpV6UdpEx : NetPktRssIpV6Udp;
1666         }
1667         mask = VIRTIO_NET_RSS_HASH_TYPE_IP_EX | VIRTIO_NET_RSS_HASH_TYPE_IPv6;
1668         if (types & mask) {
1669             return (types & VIRTIO_NET_RSS_HASH_TYPE_IP_EX) ?
1670                 NetPktRssIpV6Ex : NetPktRssIpV6;
1671         }
1672     }
1673     return 0xff;
1674 }
1675 
1676 static void virtio_set_packet_hash(const uint8_t *buf, uint8_t report,
1677                                    uint32_t hash)
1678 {
1679     struct virtio_net_hdr_v1_hash *hdr = (void *)buf;
1680     hdr->hash_value = hash;
1681     hdr->hash_report = report;
1682 }
1683 
1684 static int virtio_net_process_rss(NetClientState *nc, const uint8_t *buf,
1685                                   size_t size)
1686 {
1687     VirtIONet *n = qemu_get_nic_opaque(nc);
1688     unsigned int index = nc->queue_index, new_index = index;
1689     struct NetRxPkt *pkt = n->rx_pkt;
1690     uint8_t net_hash_type;
1691     uint32_t hash;
1692     bool isip4, isip6, isudp, istcp;
1693     static const uint8_t reports[NetPktRssIpV6UdpEx + 1] = {
1694         VIRTIO_NET_HASH_REPORT_IPv4,
1695         VIRTIO_NET_HASH_REPORT_TCPv4,
1696         VIRTIO_NET_HASH_REPORT_TCPv6,
1697         VIRTIO_NET_HASH_REPORT_IPv6,
1698         VIRTIO_NET_HASH_REPORT_IPv6_EX,
1699         VIRTIO_NET_HASH_REPORT_TCPv6_EX,
1700         VIRTIO_NET_HASH_REPORT_UDPv4,
1701         VIRTIO_NET_HASH_REPORT_UDPv6,
1702         VIRTIO_NET_HASH_REPORT_UDPv6_EX
1703     };
1704 
1705     net_rx_pkt_set_protocols(pkt, buf + n->host_hdr_len,
1706                              size - n->host_hdr_len);
1707     net_rx_pkt_get_protocols(pkt, &isip4, &isip6, &isudp, &istcp);
1708     if (isip4 && (net_rx_pkt_get_ip4_info(pkt)->fragment)) {
1709         istcp = isudp = false;
1710     }
1711     if (isip6 && (net_rx_pkt_get_ip6_info(pkt)->fragment)) {
1712         istcp = isudp = false;
1713     }
1714     net_hash_type = virtio_net_get_hash_type(isip4, isip6, isudp, istcp,
1715                                              n->rss_data.hash_types);
1716     if (net_hash_type > NetPktRssIpV6UdpEx) {
1717         if (n->rss_data.populate_hash) {
1718             virtio_set_packet_hash(buf, VIRTIO_NET_HASH_REPORT_NONE, 0);
1719         }
1720         return n->rss_data.redirect ? n->rss_data.default_queue : -1;
1721     }
1722 
1723     hash = net_rx_pkt_calc_rss_hash(pkt, net_hash_type, n->rss_data.key);
1724 
1725     if (n->rss_data.populate_hash) {
1726         virtio_set_packet_hash(buf, reports[net_hash_type], hash);
1727     }
1728 
1729     if (n->rss_data.redirect) {
1730         new_index = hash & (n->rss_data.indirections_len - 1);
1731         new_index = n->rss_data.indirections_table[new_index];
1732     }
1733 
1734     return (index == new_index) ? -1 : new_index;
1735 }
1736 
1737 static ssize_t virtio_net_receive_rcu(NetClientState *nc, const uint8_t *buf,
1738                                       size_t size, bool no_rss)
1739 {
1740     VirtIONet *n = qemu_get_nic_opaque(nc);
1741     VirtIONetQueue *q = virtio_net_get_subqueue(nc);
1742     VirtIODevice *vdev = VIRTIO_DEVICE(n);
1743     VirtQueueElement *elems[VIRTQUEUE_MAX_SIZE];
1744     size_t lens[VIRTQUEUE_MAX_SIZE];
1745     struct iovec mhdr_sg[VIRTQUEUE_MAX_SIZE];
1746     struct virtio_net_hdr_mrg_rxbuf mhdr;
1747     unsigned mhdr_cnt = 0;
1748     size_t offset, i, guest_offset, j;
1749     ssize_t err;
1750 
1751     if (!virtio_net_can_receive(nc)) {
1752         return -1;
1753     }
1754 
1755     if (!no_rss && n->rss_data.enabled && n->rss_data.enabled_software_rss) {
1756         int index = virtio_net_process_rss(nc, buf, size);
1757         if (index >= 0) {
1758             NetClientState *nc2 = qemu_get_subqueue(n->nic, index);
1759             return virtio_net_receive_rcu(nc2, buf, size, true);
1760         }
1761     }
1762 
1763     /* hdr_len refers to the header we supply to the guest */
1764     if (!virtio_net_has_buffers(q, size + n->guest_hdr_len - n->host_hdr_len)) {
1765         return 0;
1766     }
1767 
1768     if (!receive_filter(n, buf, size))
1769         return size;
1770 
1771     offset = i = 0;
1772 
1773     while (offset < size) {
1774         VirtQueueElement *elem;
1775         int len, total;
1776         const struct iovec *sg;
1777 
1778         total = 0;
1779 
1780         if (i == VIRTQUEUE_MAX_SIZE) {
1781             virtio_error(vdev, "virtio-net unexpected long buffer chain");
1782             err = size;
1783             goto err;
1784         }
1785 
1786         elem = virtqueue_pop(q->rx_vq, sizeof(VirtQueueElement));
1787         if (!elem) {
1788             if (i) {
1789                 virtio_error(vdev, "virtio-net unexpected empty queue: "
1790                              "i %zd mergeable %d offset %zd, size %zd, "
1791                              "guest hdr len %zd, host hdr len %zd "
1792                              "guest features 0x%" PRIx64,
1793                              i, n->mergeable_rx_bufs, offset, size,
1794                              n->guest_hdr_len, n->host_hdr_len,
1795                              vdev->guest_features);
1796             }
1797             err = -1;
1798             goto err;
1799         }
1800 
1801         if (elem->in_num < 1) {
1802             virtio_error(vdev,
1803                          "virtio-net receive queue contains no in buffers");
1804             virtqueue_detach_element(q->rx_vq, elem, 0);
1805             g_free(elem);
1806             err = -1;
1807             goto err;
1808         }
1809 
1810         sg = elem->in_sg;
1811         if (i == 0) {
1812             assert(offset == 0);
1813             if (n->mergeable_rx_bufs) {
1814                 mhdr_cnt = iov_copy(mhdr_sg, ARRAY_SIZE(mhdr_sg),
1815                                     sg, elem->in_num,
1816                                     offsetof(typeof(mhdr), num_buffers),
1817                                     sizeof(mhdr.num_buffers));
1818             }
1819 
1820             receive_header(n, sg, elem->in_num, buf, size);
1821             if (n->rss_data.populate_hash) {
1822                 offset = sizeof(mhdr);
1823                 iov_from_buf(sg, elem->in_num, offset,
1824                              buf + offset, n->host_hdr_len - sizeof(mhdr));
1825             }
1826             offset = n->host_hdr_len;
1827             total += n->guest_hdr_len;
1828             guest_offset = n->guest_hdr_len;
1829         } else {
1830             guest_offset = 0;
1831         }
1832 
1833         /* copy in packet.  ugh */
1834         len = iov_from_buf(sg, elem->in_num, guest_offset,
1835                            buf + offset, size - offset);
1836         total += len;
1837         offset += len;
1838         /* If buffers can't be merged, at this point we
1839          * must have consumed the complete packet.
1840          * Otherwise, drop it. */
1841         if (!n->mergeable_rx_bufs && offset < size) {
1842             virtqueue_unpop(q->rx_vq, elem, total);
1843             g_free(elem);
1844             err = size;
1845             goto err;
1846         }
1847 
1848         elems[i] = elem;
1849         lens[i] = total;
1850         i++;
1851     }
1852 
1853     if (mhdr_cnt) {
1854         virtio_stw_p(vdev, &mhdr.num_buffers, i);
1855         iov_from_buf(mhdr_sg, mhdr_cnt,
1856                      0,
1857                      &mhdr.num_buffers, sizeof mhdr.num_buffers);
1858     }
1859 
1860     for (j = 0; j < i; j++) {
1861         /* signal other side */
1862         virtqueue_fill(q->rx_vq, elems[j], lens[j], j);
1863         g_free(elems[j]);
1864     }
1865 
1866     virtqueue_flush(q->rx_vq, i);
1867     virtio_notify(vdev, q->rx_vq);
1868 
1869     return size;
1870 
1871 err:
1872     for (j = 0; j < i; j++) {
1873         g_free(elems[j]);
1874     }
1875 
1876     return err;
1877 }
1878 
1879 static ssize_t virtio_net_do_receive(NetClientState *nc, const uint8_t *buf,
1880                                   size_t size)
1881 {
1882     RCU_READ_LOCK_GUARD();
1883 
1884     return virtio_net_receive_rcu(nc, buf, size, false);
1885 }
1886 
1887 static void virtio_net_rsc_extract_unit4(VirtioNetRscChain *chain,
1888                                          const uint8_t *buf,
1889                                          VirtioNetRscUnit *unit)
1890 {
1891     uint16_t ip_hdrlen;
1892     struct ip_header *ip;
1893 
1894     ip = (struct ip_header *)(buf + chain->n->guest_hdr_len
1895                               + sizeof(struct eth_header));
1896     unit->ip = (void *)ip;
1897     ip_hdrlen = (ip->ip_ver_len & 0xF) << 2;
1898     unit->ip_plen = &ip->ip_len;
1899     unit->tcp = (struct tcp_header *)(((uint8_t *)unit->ip) + ip_hdrlen);
1900     unit->tcp_hdrlen = (htons(unit->tcp->th_offset_flags) & 0xF000) >> 10;
1901     unit->payload = htons(*unit->ip_plen) - ip_hdrlen - unit->tcp_hdrlen;
1902 }
1903 
1904 static void virtio_net_rsc_extract_unit6(VirtioNetRscChain *chain,
1905                                          const uint8_t *buf,
1906                                          VirtioNetRscUnit *unit)
1907 {
1908     struct ip6_header *ip6;
1909 
1910     ip6 = (struct ip6_header *)(buf + chain->n->guest_hdr_len
1911                                  + sizeof(struct eth_header));
1912     unit->ip = ip6;
1913     unit->ip_plen = &(ip6->ip6_ctlun.ip6_un1.ip6_un1_plen);
1914     unit->tcp = (struct tcp_header *)(((uint8_t *)unit->ip)
1915                                         + sizeof(struct ip6_header));
1916     unit->tcp_hdrlen = (htons(unit->tcp->th_offset_flags) & 0xF000) >> 10;
1917 
1918     /* There is a difference between payload lenght in ipv4 and v6,
1919        ip header is excluded in ipv6 */
1920     unit->payload = htons(*unit->ip_plen) - unit->tcp_hdrlen;
1921 }
1922 
1923 static size_t virtio_net_rsc_drain_seg(VirtioNetRscChain *chain,
1924                                        VirtioNetRscSeg *seg)
1925 {
1926     int ret;
1927     struct virtio_net_hdr_v1 *h;
1928 
1929     h = (struct virtio_net_hdr_v1 *)seg->buf;
1930     h->flags = 0;
1931     h->gso_type = VIRTIO_NET_HDR_GSO_NONE;
1932 
1933     if (seg->is_coalesced) {
1934         h->rsc.segments = seg->packets;
1935         h->rsc.dup_acks = seg->dup_ack;
1936         h->flags = VIRTIO_NET_HDR_F_RSC_INFO;
1937         if (chain->proto == ETH_P_IP) {
1938             h->gso_type = VIRTIO_NET_HDR_GSO_TCPV4;
1939         } else {
1940             h->gso_type = VIRTIO_NET_HDR_GSO_TCPV6;
1941         }
1942     }
1943 
1944     ret = virtio_net_do_receive(seg->nc, seg->buf, seg->size);
1945     QTAILQ_REMOVE(&chain->buffers, seg, next);
1946     g_free(seg->buf);
1947     g_free(seg);
1948 
1949     return ret;
1950 }
1951 
1952 static void virtio_net_rsc_purge(void *opq)
1953 {
1954     VirtioNetRscSeg *seg, *rn;
1955     VirtioNetRscChain *chain = (VirtioNetRscChain *)opq;
1956 
1957     QTAILQ_FOREACH_SAFE(seg, &chain->buffers, next, rn) {
1958         if (virtio_net_rsc_drain_seg(chain, seg) == 0) {
1959             chain->stat.purge_failed++;
1960             continue;
1961         }
1962     }
1963 
1964     chain->stat.timer++;
1965     if (!QTAILQ_EMPTY(&chain->buffers)) {
1966         timer_mod(chain->drain_timer,
1967               qemu_clock_get_ns(QEMU_CLOCK_HOST) + chain->n->rsc_timeout);
1968     }
1969 }
1970 
1971 static void virtio_net_rsc_cleanup(VirtIONet *n)
1972 {
1973     VirtioNetRscChain *chain, *rn_chain;
1974     VirtioNetRscSeg *seg, *rn_seg;
1975 
1976     QTAILQ_FOREACH_SAFE(chain, &n->rsc_chains, next, rn_chain) {
1977         QTAILQ_FOREACH_SAFE(seg, &chain->buffers, next, rn_seg) {
1978             QTAILQ_REMOVE(&chain->buffers, seg, next);
1979             g_free(seg->buf);
1980             g_free(seg);
1981         }
1982 
1983         timer_free(chain->drain_timer);
1984         QTAILQ_REMOVE(&n->rsc_chains, chain, next);
1985         g_free(chain);
1986     }
1987 }
1988 
1989 static void virtio_net_rsc_cache_buf(VirtioNetRscChain *chain,
1990                                      NetClientState *nc,
1991                                      const uint8_t *buf, size_t size)
1992 {
1993     uint16_t hdr_len;
1994     VirtioNetRscSeg *seg;
1995 
1996     hdr_len = chain->n->guest_hdr_len;
1997     seg = g_malloc(sizeof(VirtioNetRscSeg));
1998     seg->buf = g_malloc(hdr_len + sizeof(struct eth_header)
1999         + sizeof(struct ip6_header) + VIRTIO_NET_MAX_TCP_PAYLOAD);
2000     memcpy(seg->buf, buf, size);
2001     seg->size = size;
2002     seg->packets = 1;
2003     seg->dup_ack = 0;
2004     seg->is_coalesced = 0;
2005     seg->nc = nc;
2006 
2007     QTAILQ_INSERT_TAIL(&chain->buffers, seg, next);
2008     chain->stat.cache++;
2009 
2010     switch (chain->proto) {
2011     case ETH_P_IP:
2012         virtio_net_rsc_extract_unit4(chain, seg->buf, &seg->unit);
2013         break;
2014     case ETH_P_IPV6:
2015         virtio_net_rsc_extract_unit6(chain, seg->buf, &seg->unit);
2016         break;
2017     default:
2018         g_assert_not_reached();
2019     }
2020 }
2021 
2022 static int32_t virtio_net_rsc_handle_ack(VirtioNetRscChain *chain,
2023                                          VirtioNetRscSeg *seg,
2024                                          const uint8_t *buf,
2025                                          struct tcp_header *n_tcp,
2026                                          struct tcp_header *o_tcp)
2027 {
2028     uint32_t nack, oack;
2029     uint16_t nwin, owin;
2030 
2031     nack = htonl(n_tcp->th_ack);
2032     nwin = htons(n_tcp->th_win);
2033     oack = htonl(o_tcp->th_ack);
2034     owin = htons(o_tcp->th_win);
2035 
2036     if ((nack - oack) >= VIRTIO_NET_MAX_TCP_PAYLOAD) {
2037         chain->stat.ack_out_of_win++;
2038         return RSC_FINAL;
2039     } else if (nack == oack) {
2040         /* duplicated ack or window probe */
2041         if (nwin == owin) {
2042             /* duplicated ack, add dup ack count due to whql test up to 1 */
2043             chain->stat.dup_ack++;
2044             return RSC_FINAL;
2045         } else {
2046             /* Coalesce window update */
2047             o_tcp->th_win = n_tcp->th_win;
2048             chain->stat.win_update++;
2049             return RSC_COALESCE;
2050         }
2051     } else {
2052         /* pure ack, go to 'C', finalize*/
2053         chain->stat.pure_ack++;
2054         return RSC_FINAL;
2055     }
2056 }
2057 
2058 static int32_t virtio_net_rsc_coalesce_data(VirtioNetRscChain *chain,
2059                                             VirtioNetRscSeg *seg,
2060                                             const uint8_t *buf,
2061                                             VirtioNetRscUnit *n_unit)
2062 {
2063     void *data;
2064     uint16_t o_ip_len;
2065     uint32_t nseq, oseq;
2066     VirtioNetRscUnit *o_unit;
2067 
2068     o_unit = &seg->unit;
2069     o_ip_len = htons(*o_unit->ip_plen);
2070     nseq = htonl(n_unit->tcp->th_seq);
2071     oseq = htonl(o_unit->tcp->th_seq);
2072 
2073     /* out of order or retransmitted. */
2074     if ((nseq - oseq) > VIRTIO_NET_MAX_TCP_PAYLOAD) {
2075         chain->stat.data_out_of_win++;
2076         return RSC_FINAL;
2077     }
2078 
2079     data = ((uint8_t *)n_unit->tcp) + n_unit->tcp_hdrlen;
2080     if (nseq == oseq) {
2081         if ((o_unit->payload == 0) && n_unit->payload) {
2082             /* From no payload to payload, normal case, not a dup ack or etc */
2083             chain->stat.data_after_pure_ack++;
2084             goto coalesce;
2085         } else {
2086             return virtio_net_rsc_handle_ack(chain, seg, buf,
2087                                              n_unit->tcp, o_unit->tcp);
2088         }
2089     } else if ((nseq - oseq) != o_unit->payload) {
2090         /* Not a consistent packet, out of order */
2091         chain->stat.data_out_of_order++;
2092         return RSC_FINAL;
2093     } else {
2094 coalesce:
2095         if ((o_ip_len + n_unit->payload) > chain->max_payload) {
2096             chain->stat.over_size++;
2097             return RSC_FINAL;
2098         }
2099 
2100         /* Here comes the right data, the payload length in v4/v6 is different,
2101            so use the field value to update and record the new data len */
2102         o_unit->payload += n_unit->payload; /* update new data len */
2103 
2104         /* update field in ip header */
2105         *o_unit->ip_plen = htons(o_ip_len + n_unit->payload);
2106 
2107         /* Bring 'PUSH' big, the whql test guide says 'PUSH' can be coalesced
2108            for windows guest, while this may change the behavior for linux
2109            guest (only if it uses RSC feature). */
2110         o_unit->tcp->th_offset_flags = n_unit->tcp->th_offset_flags;
2111 
2112         o_unit->tcp->th_ack = n_unit->tcp->th_ack;
2113         o_unit->tcp->th_win = n_unit->tcp->th_win;
2114 
2115         memmove(seg->buf + seg->size, data, n_unit->payload);
2116         seg->size += n_unit->payload;
2117         seg->packets++;
2118         chain->stat.coalesced++;
2119         return RSC_COALESCE;
2120     }
2121 }
2122 
2123 static int32_t virtio_net_rsc_coalesce4(VirtioNetRscChain *chain,
2124                                         VirtioNetRscSeg *seg,
2125                                         const uint8_t *buf, size_t size,
2126                                         VirtioNetRscUnit *unit)
2127 {
2128     struct ip_header *ip1, *ip2;
2129 
2130     ip1 = (struct ip_header *)(unit->ip);
2131     ip2 = (struct ip_header *)(seg->unit.ip);
2132     if ((ip1->ip_src ^ ip2->ip_src) || (ip1->ip_dst ^ ip2->ip_dst)
2133         || (unit->tcp->th_sport ^ seg->unit.tcp->th_sport)
2134         || (unit->tcp->th_dport ^ seg->unit.tcp->th_dport)) {
2135         chain->stat.no_match++;
2136         return RSC_NO_MATCH;
2137     }
2138 
2139     return virtio_net_rsc_coalesce_data(chain, seg, buf, unit);
2140 }
2141 
2142 static int32_t virtio_net_rsc_coalesce6(VirtioNetRscChain *chain,
2143                                         VirtioNetRscSeg *seg,
2144                                         const uint8_t *buf, size_t size,
2145                                         VirtioNetRscUnit *unit)
2146 {
2147     struct ip6_header *ip1, *ip2;
2148 
2149     ip1 = (struct ip6_header *)(unit->ip);
2150     ip2 = (struct ip6_header *)(seg->unit.ip);
2151     if (memcmp(&ip1->ip6_src, &ip2->ip6_src, sizeof(struct in6_address))
2152         || memcmp(&ip1->ip6_dst, &ip2->ip6_dst, sizeof(struct in6_address))
2153         || (unit->tcp->th_sport ^ seg->unit.tcp->th_sport)
2154         || (unit->tcp->th_dport ^ seg->unit.tcp->th_dport)) {
2155             chain->stat.no_match++;
2156             return RSC_NO_MATCH;
2157     }
2158 
2159     return virtio_net_rsc_coalesce_data(chain, seg, buf, unit);
2160 }
2161 
2162 /* Packets with 'SYN' should bypass, other flag should be sent after drain
2163  * to prevent out of order */
2164 static int virtio_net_rsc_tcp_ctrl_check(VirtioNetRscChain *chain,
2165                                          struct tcp_header *tcp)
2166 {
2167     uint16_t tcp_hdr;
2168     uint16_t tcp_flag;
2169 
2170     tcp_flag = htons(tcp->th_offset_flags);
2171     tcp_hdr = (tcp_flag & VIRTIO_NET_TCP_HDR_LENGTH) >> 10;
2172     tcp_flag &= VIRTIO_NET_TCP_FLAG;
2173     if (tcp_flag & TH_SYN) {
2174         chain->stat.tcp_syn++;
2175         return RSC_BYPASS;
2176     }
2177 
2178     if (tcp_flag & (TH_FIN | TH_URG | TH_RST | TH_ECE | TH_CWR)) {
2179         chain->stat.tcp_ctrl_drain++;
2180         return RSC_FINAL;
2181     }
2182 
2183     if (tcp_hdr > sizeof(struct tcp_header)) {
2184         chain->stat.tcp_all_opt++;
2185         return RSC_FINAL;
2186     }
2187 
2188     return RSC_CANDIDATE;
2189 }
2190 
2191 static size_t virtio_net_rsc_do_coalesce(VirtioNetRscChain *chain,
2192                                          NetClientState *nc,
2193                                          const uint8_t *buf, size_t size,
2194                                          VirtioNetRscUnit *unit)
2195 {
2196     int ret;
2197     VirtioNetRscSeg *seg, *nseg;
2198 
2199     if (QTAILQ_EMPTY(&chain->buffers)) {
2200         chain->stat.empty_cache++;
2201         virtio_net_rsc_cache_buf(chain, nc, buf, size);
2202         timer_mod(chain->drain_timer,
2203               qemu_clock_get_ns(QEMU_CLOCK_HOST) + chain->n->rsc_timeout);
2204         return size;
2205     }
2206 
2207     QTAILQ_FOREACH_SAFE(seg, &chain->buffers, next, nseg) {
2208         if (chain->proto == ETH_P_IP) {
2209             ret = virtio_net_rsc_coalesce4(chain, seg, buf, size, unit);
2210         } else {
2211             ret = virtio_net_rsc_coalesce6(chain, seg, buf, size, unit);
2212         }
2213 
2214         if (ret == RSC_FINAL) {
2215             if (virtio_net_rsc_drain_seg(chain, seg) == 0) {
2216                 /* Send failed */
2217                 chain->stat.final_failed++;
2218                 return 0;
2219             }
2220 
2221             /* Send current packet */
2222             return virtio_net_do_receive(nc, buf, size);
2223         } else if (ret == RSC_NO_MATCH) {
2224             continue;
2225         } else {
2226             /* Coalesced, mark coalesced flag to tell calc cksum for ipv4 */
2227             seg->is_coalesced = 1;
2228             return size;
2229         }
2230     }
2231 
2232     chain->stat.no_match_cache++;
2233     virtio_net_rsc_cache_buf(chain, nc, buf, size);
2234     return size;
2235 }
2236 
2237 /* Drain a connection data, this is to avoid out of order segments */
2238 static size_t virtio_net_rsc_drain_flow(VirtioNetRscChain *chain,
2239                                         NetClientState *nc,
2240                                         const uint8_t *buf, size_t size,
2241                                         uint16_t ip_start, uint16_t ip_size,
2242                                         uint16_t tcp_port)
2243 {
2244     VirtioNetRscSeg *seg, *nseg;
2245     uint32_t ppair1, ppair2;
2246 
2247     ppair1 = *(uint32_t *)(buf + tcp_port);
2248     QTAILQ_FOREACH_SAFE(seg, &chain->buffers, next, nseg) {
2249         ppair2 = *(uint32_t *)(seg->buf + tcp_port);
2250         if (memcmp(buf + ip_start, seg->buf + ip_start, ip_size)
2251             || (ppair1 != ppair2)) {
2252             continue;
2253         }
2254         if (virtio_net_rsc_drain_seg(chain, seg) == 0) {
2255             chain->stat.drain_failed++;
2256         }
2257 
2258         break;
2259     }
2260 
2261     return virtio_net_do_receive(nc, buf, size);
2262 }
2263 
2264 static int32_t virtio_net_rsc_sanity_check4(VirtioNetRscChain *chain,
2265                                             struct ip_header *ip,
2266                                             const uint8_t *buf, size_t size)
2267 {
2268     uint16_t ip_len;
2269 
2270     /* Not an ipv4 packet */
2271     if (((ip->ip_ver_len & 0xF0) >> 4) != IP_HEADER_VERSION_4) {
2272         chain->stat.ip_option++;
2273         return RSC_BYPASS;
2274     }
2275 
2276     /* Don't handle packets with ip option */
2277     if ((ip->ip_ver_len & 0xF) != VIRTIO_NET_IP4_HEADER_LENGTH) {
2278         chain->stat.ip_option++;
2279         return RSC_BYPASS;
2280     }
2281 
2282     if (ip->ip_p != IPPROTO_TCP) {
2283         chain->stat.bypass_not_tcp++;
2284         return RSC_BYPASS;
2285     }
2286 
2287     /* Don't handle packets with ip fragment */
2288     if (!(htons(ip->ip_off) & IP_DF)) {
2289         chain->stat.ip_frag++;
2290         return RSC_BYPASS;
2291     }
2292 
2293     /* Don't handle packets with ecn flag */
2294     if (IPTOS_ECN(ip->ip_tos)) {
2295         chain->stat.ip_ecn++;
2296         return RSC_BYPASS;
2297     }
2298 
2299     ip_len = htons(ip->ip_len);
2300     if (ip_len < (sizeof(struct ip_header) + sizeof(struct tcp_header))
2301         || ip_len > (size - chain->n->guest_hdr_len -
2302                      sizeof(struct eth_header))) {
2303         chain->stat.ip_hacked++;
2304         return RSC_BYPASS;
2305     }
2306 
2307     return RSC_CANDIDATE;
2308 }
2309 
2310 static size_t virtio_net_rsc_receive4(VirtioNetRscChain *chain,
2311                                       NetClientState *nc,
2312                                       const uint8_t *buf, size_t size)
2313 {
2314     int32_t ret;
2315     uint16_t hdr_len;
2316     VirtioNetRscUnit unit;
2317 
2318     hdr_len = ((VirtIONet *)(chain->n))->guest_hdr_len;
2319 
2320     if (size < (hdr_len + sizeof(struct eth_header) + sizeof(struct ip_header)
2321         + sizeof(struct tcp_header))) {
2322         chain->stat.bypass_not_tcp++;
2323         return virtio_net_do_receive(nc, buf, size);
2324     }
2325 
2326     virtio_net_rsc_extract_unit4(chain, buf, &unit);
2327     if (virtio_net_rsc_sanity_check4(chain, unit.ip, buf, size)
2328         != RSC_CANDIDATE) {
2329         return virtio_net_do_receive(nc, buf, size);
2330     }
2331 
2332     ret = virtio_net_rsc_tcp_ctrl_check(chain, unit.tcp);
2333     if (ret == RSC_BYPASS) {
2334         return virtio_net_do_receive(nc, buf, size);
2335     } else if (ret == RSC_FINAL) {
2336         return virtio_net_rsc_drain_flow(chain, nc, buf, size,
2337                 ((hdr_len + sizeof(struct eth_header)) + 12),
2338                 VIRTIO_NET_IP4_ADDR_SIZE,
2339                 hdr_len + sizeof(struct eth_header) + sizeof(struct ip_header));
2340     }
2341 
2342     return virtio_net_rsc_do_coalesce(chain, nc, buf, size, &unit);
2343 }
2344 
2345 static int32_t virtio_net_rsc_sanity_check6(VirtioNetRscChain *chain,
2346                                             struct ip6_header *ip6,
2347                                             const uint8_t *buf, size_t size)
2348 {
2349     uint16_t ip_len;
2350 
2351     if (((ip6->ip6_ctlun.ip6_un1.ip6_un1_flow & 0xF0) >> 4)
2352         != IP_HEADER_VERSION_6) {
2353         return RSC_BYPASS;
2354     }
2355 
2356     /* Both option and protocol is checked in this */
2357     if (ip6->ip6_ctlun.ip6_un1.ip6_un1_nxt != IPPROTO_TCP) {
2358         chain->stat.bypass_not_tcp++;
2359         return RSC_BYPASS;
2360     }
2361 
2362     ip_len = htons(ip6->ip6_ctlun.ip6_un1.ip6_un1_plen);
2363     if (ip_len < sizeof(struct tcp_header) ||
2364         ip_len > (size - chain->n->guest_hdr_len - sizeof(struct eth_header)
2365                   - sizeof(struct ip6_header))) {
2366         chain->stat.ip_hacked++;
2367         return RSC_BYPASS;
2368     }
2369 
2370     /* Don't handle packets with ecn flag */
2371     if (IP6_ECN(ip6->ip6_ctlun.ip6_un3.ip6_un3_ecn)) {
2372         chain->stat.ip_ecn++;
2373         return RSC_BYPASS;
2374     }
2375 
2376     return RSC_CANDIDATE;
2377 }
2378 
2379 static size_t virtio_net_rsc_receive6(void *opq, NetClientState *nc,
2380                                       const uint8_t *buf, size_t size)
2381 {
2382     int32_t ret;
2383     uint16_t hdr_len;
2384     VirtioNetRscChain *chain;
2385     VirtioNetRscUnit unit;
2386 
2387     chain = (VirtioNetRscChain *)opq;
2388     hdr_len = ((VirtIONet *)(chain->n))->guest_hdr_len;
2389 
2390     if (size < (hdr_len + sizeof(struct eth_header) + sizeof(struct ip6_header)
2391         + sizeof(tcp_header))) {
2392         return virtio_net_do_receive(nc, buf, size);
2393     }
2394 
2395     virtio_net_rsc_extract_unit6(chain, buf, &unit);
2396     if (RSC_CANDIDATE != virtio_net_rsc_sanity_check6(chain,
2397                                                  unit.ip, buf, size)) {
2398         return virtio_net_do_receive(nc, buf, size);
2399     }
2400 
2401     ret = virtio_net_rsc_tcp_ctrl_check(chain, unit.tcp);
2402     if (ret == RSC_BYPASS) {
2403         return virtio_net_do_receive(nc, buf, size);
2404     } else if (ret == RSC_FINAL) {
2405         return virtio_net_rsc_drain_flow(chain, nc, buf, size,
2406                 ((hdr_len + sizeof(struct eth_header)) + 8),
2407                 VIRTIO_NET_IP6_ADDR_SIZE,
2408                 hdr_len + sizeof(struct eth_header)
2409                 + sizeof(struct ip6_header));
2410     }
2411 
2412     return virtio_net_rsc_do_coalesce(chain, nc, buf, size, &unit);
2413 }
2414 
2415 static VirtioNetRscChain *virtio_net_rsc_lookup_chain(VirtIONet *n,
2416                                                       NetClientState *nc,
2417                                                       uint16_t proto)
2418 {
2419     VirtioNetRscChain *chain;
2420 
2421     if ((proto != (uint16_t)ETH_P_IP) && (proto != (uint16_t)ETH_P_IPV6)) {
2422         return NULL;
2423     }
2424 
2425     QTAILQ_FOREACH(chain, &n->rsc_chains, next) {
2426         if (chain->proto == proto) {
2427             return chain;
2428         }
2429     }
2430 
2431     chain = g_malloc(sizeof(*chain));
2432     chain->n = n;
2433     chain->proto = proto;
2434     if (proto == (uint16_t)ETH_P_IP) {
2435         chain->max_payload = VIRTIO_NET_MAX_IP4_PAYLOAD;
2436         chain->gso_type = VIRTIO_NET_HDR_GSO_TCPV4;
2437     } else {
2438         chain->max_payload = VIRTIO_NET_MAX_IP6_PAYLOAD;
2439         chain->gso_type = VIRTIO_NET_HDR_GSO_TCPV6;
2440     }
2441     chain->drain_timer = timer_new_ns(QEMU_CLOCK_HOST,
2442                                       virtio_net_rsc_purge, chain);
2443     memset(&chain->stat, 0, sizeof(chain->stat));
2444 
2445     QTAILQ_INIT(&chain->buffers);
2446     QTAILQ_INSERT_TAIL(&n->rsc_chains, chain, next);
2447 
2448     return chain;
2449 }
2450 
2451 static ssize_t virtio_net_rsc_receive(NetClientState *nc,
2452                                       const uint8_t *buf,
2453                                       size_t size)
2454 {
2455     uint16_t proto;
2456     VirtioNetRscChain *chain;
2457     struct eth_header *eth;
2458     VirtIONet *n;
2459 
2460     n = qemu_get_nic_opaque(nc);
2461     if (size < (n->host_hdr_len + sizeof(struct eth_header))) {
2462         return virtio_net_do_receive(nc, buf, size);
2463     }
2464 
2465     eth = (struct eth_header *)(buf + n->guest_hdr_len);
2466     proto = htons(eth->h_proto);
2467 
2468     chain = virtio_net_rsc_lookup_chain(n, nc, proto);
2469     if (chain) {
2470         chain->stat.received++;
2471         if (proto == (uint16_t)ETH_P_IP && n->rsc4_enabled) {
2472             return virtio_net_rsc_receive4(chain, nc, buf, size);
2473         } else if (proto == (uint16_t)ETH_P_IPV6 && n->rsc6_enabled) {
2474             return virtio_net_rsc_receive6(chain, nc, buf, size);
2475         }
2476     }
2477     return virtio_net_do_receive(nc, buf, size);
2478 }
2479 
2480 static ssize_t virtio_net_receive(NetClientState *nc, const uint8_t *buf,
2481                                   size_t size)
2482 {
2483     VirtIONet *n = qemu_get_nic_opaque(nc);
2484     if ((n->rsc4_enabled || n->rsc6_enabled)) {
2485         return virtio_net_rsc_receive(nc, buf, size);
2486     } else {
2487         return virtio_net_do_receive(nc, buf, size);
2488     }
2489 }
2490 
2491 static int32_t virtio_net_flush_tx(VirtIONetQueue *q);
2492 
2493 static void virtio_net_tx_complete(NetClientState *nc, ssize_t len)
2494 {
2495     VirtIONet *n = qemu_get_nic_opaque(nc);
2496     VirtIONetQueue *q = virtio_net_get_subqueue(nc);
2497     VirtIODevice *vdev = VIRTIO_DEVICE(n);
2498 
2499     virtqueue_push(q->tx_vq, q->async_tx.elem, 0);
2500     virtio_notify(vdev, q->tx_vq);
2501 
2502     g_free(q->async_tx.elem);
2503     q->async_tx.elem = NULL;
2504 
2505     virtio_queue_set_notification(q->tx_vq, 1);
2506     virtio_net_flush_tx(q);
2507 }
2508 
2509 /* TX */
2510 static int32_t virtio_net_flush_tx(VirtIONetQueue *q)
2511 {
2512     VirtIONet *n = q->n;
2513     VirtIODevice *vdev = VIRTIO_DEVICE(n);
2514     VirtQueueElement *elem;
2515     int32_t num_packets = 0;
2516     int queue_index = vq2q(virtio_get_queue_index(q->tx_vq));
2517     if (!(vdev->status & VIRTIO_CONFIG_S_DRIVER_OK)) {
2518         return num_packets;
2519     }
2520 
2521     if (q->async_tx.elem) {
2522         virtio_queue_set_notification(q->tx_vq, 0);
2523         return num_packets;
2524     }
2525 
2526     for (;;) {
2527         ssize_t ret;
2528         unsigned int out_num;
2529         struct iovec sg[VIRTQUEUE_MAX_SIZE], sg2[VIRTQUEUE_MAX_SIZE + 1], *out_sg;
2530         struct virtio_net_hdr_mrg_rxbuf mhdr;
2531 
2532         elem = virtqueue_pop(q->tx_vq, sizeof(VirtQueueElement));
2533         if (!elem) {
2534             break;
2535         }
2536 
2537         out_num = elem->out_num;
2538         out_sg = elem->out_sg;
2539         if (out_num < 1) {
2540             virtio_error(vdev, "virtio-net header not in first element");
2541             virtqueue_detach_element(q->tx_vq, elem, 0);
2542             g_free(elem);
2543             return -EINVAL;
2544         }
2545 
2546         if (n->has_vnet_hdr) {
2547             if (iov_to_buf(out_sg, out_num, 0, &mhdr, n->guest_hdr_len) <
2548                 n->guest_hdr_len) {
2549                 virtio_error(vdev, "virtio-net header incorrect");
2550                 virtqueue_detach_element(q->tx_vq, elem, 0);
2551                 g_free(elem);
2552                 return -EINVAL;
2553             }
2554             if (n->needs_vnet_hdr_swap) {
2555                 virtio_net_hdr_swap(vdev, (void *) &mhdr);
2556                 sg2[0].iov_base = &mhdr;
2557                 sg2[0].iov_len = n->guest_hdr_len;
2558                 out_num = iov_copy(&sg2[1], ARRAY_SIZE(sg2) - 1,
2559                                    out_sg, out_num,
2560                                    n->guest_hdr_len, -1);
2561                 if (out_num == VIRTQUEUE_MAX_SIZE) {
2562                     goto drop;
2563                 }
2564                 out_num += 1;
2565                 out_sg = sg2;
2566             }
2567         }
2568         /*
2569          * If host wants to see the guest header as is, we can
2570          * pass it on unchanged. Otherwise, copy just the parts
2571          * that host is interested in.
2572          */
2573         assert(n->host_hdr_len <= n->guest_hdr_len);
2574         if (n->host_hdr_len != n->guest_hdr_len) {
2575             unsigned sg_num = iov_copy(sg, ARRAY_SIZE(sg),
2576                                        out_sg, out_num,
2577                                        0, n->host_hdr_len);
2578             sg_num += iov_copy(sg + sg_num, ARRAY_SIZE(sg) - sg_num,
2579                              out_sg, out_num,
2580                              n->guest_hdr_len, -1);
2581             out_num = sg_num;
2582             out_sg = sg;
2583         }
2584 
2585         ret = qemu_sendv_packet_async(qemu_get_subqueue(n->nic, queue_index),
2586                                       out_sg, out_num, virtio_net_tx_complete);
2587         if (ret == 0) {
2588             virtio_queue_set_notification(q->tx_vq, 0);
2589             q->async_tx.elem = elem;
2590             return -EBUSY;
2591         }
2592 
2593 drop:
2594         virtqueue_push(q->tx_vq, elem, 0);
2595         virtio_notify(vdev, q->tx_vq);
2596         g_free(elem);
2597 
2598         if (++num_packets >= n->tx_burst) {
2599             break;
2600         }
2601     }
2602     return num_packets;
2603 }
2604 
2605 static void virtio_net_handle_tx_timer(VirtIODevice *vdev, VirtQueue *vq)
2606 {
2607     VirtIONet *n = VIRTIO_NET(vdev);
2608     VirtIONetQueue *q = &n->vqs[vq2q(virtio_get_queue_index(vq))];
2609 
2610     if (unlikely((n->status & VIRTIO_NET_S_LINK_UP) == 0)) {
2611         virtio_net_drop_tx_queue_data(vdev, vq);
2612         return;
2613     }
2614 
2615     /* This happens when device was stopped but VCPU wasn't. */
2616     if (!vdev->vm_running) {
2617         q->tx_waiting = 1;
2618         return;
2619     }
2620 
2621     if (q->tx_waiting) {
2622         virtio_queue_set_notification(vq, 1);
2623         timer_del(q->tx_timer);
2624         q->tx_waiting = 0;
2625         if (virtio_net_flush_tx(q) == -EINVAL) {
2626             return;
2627         }
2628     } else {
2629         timer_mod(q->tx_timer,
2630                        qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) + n->tx_timeout);
2631         q->tx_waiting = 1;
2632         virtio_queue_set_notification(vq, 0);
2633     }
2634 }
2635 
2636 static void virtio_net_handle_tx_bh(VirtIODevice *vdev, VirtQueue *vq)
2637 {
2638     VirtIONet *n = VIRTIO_NET(vdev);
2639     VirtIONetQueue *q = &n->vqs[vq2q(virtio_get_queue_index(vq))];
2640 
2641     if (unlikely((n->status & VIRTIO_NET_S_LINK_UP) == 0)) {
2642         virtio_net_drop_tx_queue_data(vdev, vq);
2643         return;
2644     }
2645 
2646     if (unlikely(q->tx_waiting)) {
2647         return;
2648     }
2649     q->tx_waiting = 1;
2650     /* This happens when device was stopped but VCPU wasn't. */
2651     if (!vdev->vm_running) {
2652         return;
2653     }
2654     virtio_queue_set_notification(vq, 0);
2655     qemu_bh_schedule(q->tx_bh);
2656 }
2657 
2658 static void virtio_net_tx_timer(void *opaque)
2659 {
2660     VirtIONetQueue *q = opaque;
2661     VirtIONet *n = q->n;
2662     VirtIODevice *vdev = VIRTIO_DEVICE(n);
2663     /* This happens when device was stopped but BH wasn't. */
2664     if (!vdev->vm_running) {
2665         /* Make sure tx waiting is set, so we'll run when restarted. */
2666         assert(q->tx_waiting);
2667         return;
2668     }
2669 
2670     q->tx_waiting = 0;
2671 
2672     /* Just in case the driver is not ready on more */
2673     if (!(vdev->status & VIRTIO_CONFIG_S_DRIVER_OK)) {
2674         return;
2675     }
2676 
2677     virtio_queue_set_notification(q->tx_vq, 1);
2678     virtio_net_flush_tx(q);
2679 }
2680 
2681 static void virtio_net_tx_bh(void *opaque)
2682 {
2683     VirtIONetQueue *q = opaque;
2684     VirtIONet *n = q->n;
2685     VirtIODevice *vdev = VIRTIO_DEVICE(n);
2686     int32_t ret;
2687 
2688     /* This happens when device was stopped but BH wasn't. */
2689     if (!vdev->vm_running) {
2690         /* Make sure tx waiting is set, so we'll run when restarted. */
2691         assert(q->tx_waiting);
2692         return;
2693     }
2694 
2695     q->tx_waiting = 0;
2696 
2697     /* Just in case the driver is not ready on more */
2698     if (unlikely(!(vdev->status & VIRTIO_CONFIG_S_DRIVER_OK))) {
2699         return;
2700     }
2701 
2702     ret = virtio_net_flush_tx(q);
2703     if (ret == -EBUSY || ret == -EINVAL) {
2704         return; /* Notification re-enable handled by tx_complete or device
2705                  * broken */
2706     }
2707 
2708     /* If we flush a full burst of packets, assume there are
2709      * more coming and immediately reschedule */
2710     if (ret >= n->tx_burst) {
2711         qemu_bh_schedule(q->tx_bh);
2712         q->tx_waiting = 1;
2713         return;
2714     }
2715 
2716     /* If less than a full burst, re-enable notification and flush
2717      * anything that may have come in while we weren't looking.  If
2718      * we find something, assume the guest is still active and reschedule */
2719     virtio_queue_set_notification(q->tx_vq, 1);
2720     ret = virtio_net_flush_tx(q);
2721     if (ret == -EINVAL) {
2722         return;
2723     } else if (ret > 0) {
2724         virtio_queue_set_notification(q->tx_vq, 0);
2725         qemu_bh_schedule(q->tx_bh);
2726         q->tx_waiting = 1;
2727     }
2728 }
2729 
2730 static void virtio_net_add_queue(VirtIONet *n, int index)
2731 {
2732     VirtIODevice *vdev = VIRTIO_DEVICE(n);
2733 
2734     n->vqs[index].rx_vq = virtio_add_queue(vdev, n->net_conf.rx_queue_size,
2735                                            virtio_net_handle_rx);
2736 
2737     if (n->net_conf.tx && !strcmp(n->net_conf.tx, "timer")) {
2738         n->vqs[index].tx_vq =
2739             virtio_add_queue(vdev, n->net_conf.tx_queue_size,
2740                              virtio_net_handle_tx_timer);
2741         n->vqs[index].tx_timer = timer_new_ns(QEMU_CLOCK_VIRTUAL,
2742                                               virtio_net_tx_timer,
2743                                               &n->vqs[index]);
2744     } else {
2745         n->vqs[index].tx_vq =
2746             virtio_add_queue(vdev, n->net_conf.tx_queue_size,
2747                              virtio_net_handle_tx_bh);
2748         n->vqs[index].tx_bh = qemu_bh_new(virtio_net_tx_bh, &n->vqs[index]);
2749     }
2750 
2751     n->vqs[index].tx_waiting = 0;
2752     n->vqs[index].n = n;
2753 }
2754 
2755 static void virtio_net_del_queue(VirtIONet *n, int index)
2756 {
2757     VirtIODevice *vdev = VIRTIO_DEVICE(n);
2758     VirtIONetQueue *q = &n->vqs[index];
2759     NetClientState *nc = qemu_get_subqueue(n->nic, index);
2760 
2761     qemu_purge_queued_packets(nc);
2762 
2763     virtio_del_queue(vdev, index * 2);
2764     if (q->tx_timer) {
2765         timer_free(q->tx_timer);
2766         q->tx_timer = NULL;
2767     } else {
2768         qemu_bh_delete(q->tx_bh);
2769         q->tx_bh = NULL;
2770     }
2771     q->tx_waiting = 0;
2772     virtio_del_queue(vdev, index * 2 + 1);
2773 }
2774 
2775 static void virtio_net_change_num_queue_pairs(VirtIONet *n, int new_max_queue_pairs)
2776 {
2777     VirtIODevice *vdev = VIRTIO_DEVICE(n);
2778     int old_num_queues = virtio_get_num_queues(vdev);
2779     int new_num_queues = new_max_queue_pairs * 2 + 1;
2780     int i;
2781 
2782     assert(old_num_queues >= 3);
2783     assert(old_num_queues % 2 == 1);
2784 
2785     if (old_num_queues == new_num_queues) {
2786         return;
2787     }
2788 
2789     /*
2790      * We always need to remove and add ctrl vq if
2791      * old_num_queues != new_num_queues. Remove ctrl_vq first,
2792      * and then we only enter one of the following two loops.
2793      */
2794     virtio_del_queue(vdev, old_num_queues - 1);
2795 
2796     for (i = new_num_queues - 1; i < old_num_queues - 1; i += 2) {
2797         /* new_num_queues < old_num_queues */
2798         virtio_net_del_queue(n, i / 2);
2799     }
2800 
2801     for (i = old_num_queues - 1; i < new_num_queues - 1; i += 2) {
2802         /* new_num_queues > old_num_queues */
2803         virtio_net_add_queue(n, i / 2);
2804     }
2805 
2806     /* add ctrl_vq last */
2807     n->ctrl_vq = virtio_add_queue(vdev, 64, virtio_net_handle_ctrl);
2808 }
2809 
2810 static void virtio_net_set_multiqueue(VirtIONet *n, int multiqueue)
2811 {
2812     int max = multiqueue ? n->max_queue_pairs : 1;
2813 
2814     n->multiqueue = multiqueue;
2815     virtio_net_change_num_queue_pairs(n, max);
2816 
2817     virtio_net_set_queue_pairs(n);
2818 }
2819 
2820 static int virtio_net_post_load_device(void *opaque, int version_id)
2821 {
2822     VirtIONet *n = opaque;
2823     VirtIODevice *vdev = VIRTIO_DEVICE(n);
2824     int i, link_down;
2825 
2826     trace_virtio_net_post_load_device();
2827     virtio_net_set_mrg_rx_bufs(n, n->mergeable_rx_bufs,
2828                                virtio_vdev_has_feature(vdev,
2829                                                        VIRTIO_F_VERSION_1),
2830                                virtio_vdev_has_feature(vdev,
2831                                                        VIRTIO_NET_F_HASH_REPORT));
2832 
2833     /* MAC_TABLE_ENTRIES may be different from the saved image */
2834     if (n->mac_table.in_use > MAC_TABLE_ENTRIES) {
2835         n->mac_table.in_use = 0;
2836     }
2837 
2838     if (!virtio_vdev_has_feature(vdev, VIRTIO_NET_F_CTRL_GUEST_OFFLOADS)) {
2839         n->curr_guest_offloads = virtio_net_supported_guest_offloads(n);
2840     }
2841 
2842     /*
2843      * curr_guest_offloads will be later overwritten by the
2844      * virtio_set_features_nocheck call done from the virtio_load.
2845      * Here we make sure it is preserved and restored accordingly
2846      * in the virtio_net_post_load_virtio callback.
2847      */
2848     n->saved_guest_offloads = n->curr_guest_offloads;
2849 
2850     virtio_net_set_queue_pairs(n);
2851 
2852     /* Find the first multicast entry in the saved MAC filter */
2853     for (i = 0; i < n->mac_table.in_use; i++) {
2854         if (n->mac_table.macs[i * ETH_ALEN] & 1) {
2855             break;
2856         }
2857     }
2858     n->mac_table.first_multi = i;
2859 
2860     /* nc.link_down can't be migrated, so infer link_down according
2861      * to link status bit in n->status */
2862     link_down = (n->status & VIRTIO_NET_S_LINK_UP) == 0;
2863     for (i = 0; i < n->max_queue_pairs; i++) {
2864         qemu_get_subqueue(n->nic, i)->link_down = link_down;
2865     }
2866 
2867     if (virtio_vdev_has_feature(vdev, VIRTIO_NET_F_GUEST_ANNOUNCE) &&
2868         virtio_vdev_has_feature(vdev, VIRTIO_NET_F_CTRL_VQ)) {
2869         qemu_announce_timer_reset(&n->announce_timer, migrate_announce_params(),
2870                                   QEMU_CLOCK_VIRTUAL,
2871                                   virtio_net_announce_timer, n);
2872         if (n->announce_timer.round) {
2873             timer_mod(n->announce_timer.tm,
2874                       qemu_clock_get_ms(n->announce_timer.type));
2875         } else {
2876             qemu_announce_timer_del(&n->announce_timer, false);
2877         }
2878     }
2879 
2880     if (n->rss_data.enabled) {
2881         n->rss_data.enabled_software_rss = n->rss_data.populate_hash;
2882         if (!n->rss_data.populate_hash) {
2883             if (!virtio_net_attach_epbf_rss(n)) {
2884                 if (get_vhost_net(qemu_get_queue(n->nic)->peer)) {
2885                     warn_report("Can't post-load eBPF RSS for vhost");
2886                 } else {
2887                     warn_report("Can't post-load eBPF RSS - "
2888                                 "fallback to software RSS");
2889                     n->rss_data.enabled_software_rss = true;
2890                 }
2891             }
2892         }
2893 
2894         trace_virtio_net_rss_enable(n->rss_data.hash_types,
2895                                     n->rss_data.indirections_len,
2896                                     sizeof(n->rss_data.key));
2897     } else {
2898         trace_virtio_net_rss_disable();
2899     }
2900     return 0;
2901 }
2902 
2903 static int virtio_net_post_load_virtio(VirtIODevice *vdev)
2904 {
2905     VirtIONet *n = VIRTIO_NET(vdev);
2906     /*
2907      * The actual needed state is now in saved_guest_offloads,
2908      * see virtio_net_post_load_device for detail.
2909      * Restore it back and apply the desired offloads.
2910      */
2911     n->curr_guest_offloads = n->saved_guest_offloads;
2912     if (peer_has_vnet_hdr(n)) {
2913         virtio_net_apply_guest_offloads(n);
2914     }
2915 
2916     return 0;
2917 }
2918 
2919 /* tx_waiting field of a VirtIONetQueue */
2920 static const VMStateDescription vmstate_virtio_net_queue_tx_waiting = {
2921     .name = "virtio-net-queue-tx_waiting",
2922     .fields = (VMStateField[]) {
2923         VMSTATE_UINT32(tx_waiting, VirtIONetQueue),
2924         VMSTATE_END_OF_LIST()
2925    },
2926 };
2927 
2928 static bool max_queue_pairs_gt_1(void *opaque, int version_id)
2929 {
2930     return VIRTIO_NET(opaque)->max_queue_pairs > 1;
2931 }
2932 
2933 static bool has_ctrl_guest_offloads(void *opaque, int version_id)
2934 {
2935     return virtio_vdev_has_feature(VIRTIO_DEVICE(opaque),
2936                                    VIRTIO_NET_F_CTRL_GUEST_OFFLOADS);
2937 }
2938 
2939 static bool mac_table_fits(void *opaque, int version_id)
2940 {
2941     return VIRTIO_NET(opaque)->mac_table.in_use <= MAC_TABLE_ENTRIES;
2942 }
2943 
2944 static bool mac_table_doesnt_fit(void *opaque, int version_id)
2945 {
2946     return !mac_table_fits(opaque, version_id);
2947 }
2948 
2949 /* This temporary type is shared by all the WITH_TMP methods
2950  * although only some fields are used by each.
2951  */
2952 struct VirtIONetMigTmp {
2953     VirtIONet      *parent;
2954     VirtIONetQueue *vqs_1;
2955     uint16_t        curr_queue_pairs_1;
2956     uint8_t         has_ufo;
2957     uint32_t        has_vnet_hdr;
2958 };
2959 
2960 /* The 2nd and subsequent tx_waiting flags are loaded later than
2961  * the 1st entry in the queue_pairs and only if there's more than one
2962  * entry.  We use the tmp mechanism to calculate a temporary
2963  * pointer and count and also validate the count.
2964  */
2965 
2966 static int virtio_net_tx_waiting_pre_save(void *opaque)
2967 {
2968     struct VirtIONetMigTmp *tmp = opaque;
2969 
2970     tmp->vqs_1 = tmp->parent->vqs + 1;
2971     tmp->curr_queue_pairs_1 = tmp->parent->curr_queue_pairs - 1;
2972     if (tmp->parent->curr_queue_pairs == 0) {
2973         tmp->curr_queue_pairs_1 = 0;
2974     }
2975 
2976     return 0;
2977 }
2978 
2979 static int virtio_net_tx_waiting_pre_load(void *opaque)
2980 {
2981     struct VirtIONetMigTmp *tmp = opaque;
2982 
2983     /* Reuse the pointer setup from save */
2984     virtio_net_tx_waiting_pre_save(opaque);
2985 
2986     if (tmp->parent->curr_queue_pairs > tmp->parent->max_queue_pairs) {
2987         error_report("virtio-net: curr_queue_pairs %x > max_queue_pairs %x",
2988             tmp->parent->curr_queue_pairs, tmp->parent->max_queue_pairs);
2989 
2990         return -EINVAL;
2991     }
2992 
2993     return 0; /* all good */
2994 }
2995 
2996 static const VMStateDescription vmstate_virtio_net_tx_waiting = {
2997     .name      = "virtio-net-tx_waiting",
2998     .pre_load  = virtio_net_tx_waiting_pre_load,
2999     .pre_save  = virtio_net_tx_waiting_pre_save,
3000     .fields    = (VMStateField[]) {
3001         VMSTATE_STRUCT_VARRAY_POINTER_UINT16(vqs_1, struct VirtIONetMigTmp,
3002                                      curr_queue_pairs_1,
3003                                      vmstate_virtio_net_queue_tx_waiting,
3004                                      struct VirtIONetQueue),
3005         VMSTATE_END_OF_LIST()
3006     },
3007 };
3008 
3009 /* the 'has_ufo' flag is just tested; if the incoming stream has the
3010  * flag set we need to check that we have it
3011  */
3012 static int virtio_net_ufo_post_load(void *opaque, int version_id)
3013 {
3014     struct VirtIONetMigTmp *tmp = opaque;
3015 
3016     if (tmp->has_ufo && !peer_has_ufo(tmp->parent)) {
3017         error_report("virtio-net: saved image requires TUN_F_UFO support");
3018         return -EINVAL;
3019     }
3020 
3021     return 0;
3022 }
3023 
3024 static int virtio_net_ufo_pre_save(void *opaque)
3025 {
3026     struct VirtIONetMigTmp *tmp = opaque;
3027 
3028     tmp->has_ufo = tmp->parent->has_ufo;
3029 
3030     return 0;
3031 }
3032 
3033 static const VMStateDescription vmstate_virtio_net_has_ufo = {
3034     .name      = "virtio-net-ufo",
3035     .post_load = virtio_net_ufo_post_load,
3036     .pre_save  = virtio_net_ufo_pre_save,
3037     .fields    = (VMStateField[]) {
3038         VMSTATE_UINT8(has_ufo, struct VirtIONetMigTmp),
3039         VMSTATE_END_OF_LIST()
3040     },
3041 };
3042 
3043 /* the 'has_vnet_hdr' flag is just tested; if the incoming stream has the
3044  * flag set we need to check that we have it
3045  */
3046 static int virtio_net_vnet_post_load(void *opaque, int version_id)
3047 {
3048     struct VirtIONetMigTmp *tmp = opaque;
3049 
3050     if (tmp->has_vnet_hdr && !peer_has_vnet_hdr(tmp->parent)) {
3051         error_report("virtio-net: saved image requires vnet_hdr=on");
3052         return -EINVAL;
3053     }
3054 
3055     return 0;
3056 }
3057 
3058 static int virtio_net_vnet_pre_save(void *opaque)
3059 {
3060     struct VirtIONetMigTmp *tmp = opaque;
3061 
3062     tmp->has_vnet_hdr = tmp->parent->has_vnet_hdr;
3063 
3064     return 0;
3065 }
3066 
3067 static const VMStateDescription vmstate_virtio_net_has_vnet = {
3068     .name      = "virtio-net-vnet",
3069     .post_load = virtio_net_vnet_post_load,
3070     .pre_save  = virtio_net_vnet_pre_save,
3071     .fields    = (VMStateField[]) {
3072         VMSTATE_UINT32(has_vnet_hdr, struct VirtIONetMigTmp),
3073         VMSTATE_END_OF_LIST()
3074     },
3075 };
3076 
3077 static bool virtio_net_rss_needed(void *opaque)
3078 {
3079     return VIRTIO_NET(opaque)->rss_data.enabled;
3080 }
3081 
3082 static const VMStateDescription vmstate_virtio_net_rss = {
3083     .name      = "virtio-net-device/rss",
3084     .version_id = 1,
3085     .minimum_version_id = 1,
3086     .needed = virtio_net_rss_needed,
3087     .fields = (VMStateField[]) {
3088         VMSTATE_BOOL(rss_data.enabled, VirtIONet),
3089         VMSTATE_BOOL(rss_data.redirect, VirtIONet),
3090         VMSTATE_BOOL(rss_data.populate_hash, VirtIONet),
3091         VMSTATE_UINT32(rss_data.hash_types, VirtIONet),
3092         VMSTATE_UINT16(rss_data.indirections_len, VirtIONet),
3093         VMSTATE_UINT16(rss_data.default_queue, VirtIONet),
3094         VMSTATE_UINT8_ARRAY(rss_data.key, VirtIONet,
3095                             VIRTIO_NET_RSS_MAX_KEY_SIZE),
3096         VMSTATE_VARRAY_UINT16_ALLOC(rss_data.indirections_table, VirtIONet,
3097                                     rss_data.indirections_len, 0,
3098                                     vmstate_info_uint16, uint16_t),
3099         VMSTATE_END_OF_LIST()
3100     },
3101 };
3102 
3103 static const VMStateDescription vmstate_virtio_net_device = {
3104     .name = "virtio-net-device",
3105     .version_id = VIRTIO_NET_VM_VERSION,
3106     .minimum_version_id = VIRTIO_NET_VM_VERSION,
3107     .post_load = virtio_net_post_load_device,
3108     .fields = (VMStateField[]) {
3109         VMSTATE_UINT8_ARRAY(mac, VirtIONet, ETH_ALEN),
3110         VMSTATE_STRUCT_POINTER(vqs, VirtIONet,
3111                                vmstate_virtio_net_queue_tx_waiting,
3112                                VirtIONetQueue),
3113         VMSTATE_UINT32(mergeable_rx_bufs, VirtIONet),
3114         VMSTATE_UINT16(status, VirtIONet),
3115         VMSTATE_UINT8(promisc, VirtIONet),
3116         VMSTATE_UINT8(allmulti, VirtIONet),
3117         VMSTATE_UINT32(mac_table.in_use, VirtIONet),
3118 
3119         /* Guarded pair: If it fits we load it, else we throw it away
3120          * - can happen if source has a larger MAC table.; post-load
3121          *  sets flags in this case.
3122          */
3123         VMSTATE_VBUFFER_MULTIPLY(mac_table.macs, VirtIONet,
3124                                 0, mac_table_fits, mac_table.in_use,
3125                                  ETH_ALEN),
3126         VMSTATE_UNUSED_VARRAY_UINT32(VirtIONet, mac_table_doesnt_fit, 0,
3127                                      mac_table.in_use, ETH_ALEN),
3128 
3129         /* Note: This is an array of uint32's that's always been saved as a
3130          * buffer; hold onto your endiannesses; it's actually used as a bitmap
3131          * but based on the uint.
3132          */
3133         VMSTATE_BUFFER_POINTER_UNSAFE(vlans, VirtIONet, 0, MAX_VLAN >> 3),
3134         VMSTATE_WITH_TMP(VirtIONet, struct VirtIONetMigTmp,
3135                          vmstate_virtio_net_has_vnet),
3136         VMSTATE_UINT8(mac_table.multi_overflow, VirtIONet),
3137         VMSTATE_UINT8(mac_table.uni_overflow, VirtIONet),
3138         VMSTATE_UINT8(alluni, VirtIONet),
3139         VMSTATE_UINT8(nomulti, VirtIONet),
3140         VMSTATE_UINT8(nouni, VirtIONet),
3141         VMSTATE_UINT8(nobcast, VirtIONet),
3142         VMSTATE_WITH_TMP(VirtIONet, struct VirtIONetMigTmp,
3143                          vmstate_virtio_net_has_ufo),
3144         VMSTATE_SINGLE_TEST(max_queue_pairs, VirtIONet, max_queue_pairs_gt_1, 0,
3145                             vmstate_info_uint16_equal, uint16_t),
3146         VMSTATE_UINT16_TEST(curr_queue_pairs, VirtIONet, max_queue_pairs_gt_1),
3147         VMSTATE_WITH_TMP(VirtIONet, struct VirtIONetMigTmp,
3148                          vmstate_virtio_net_tx_waiting),
3149         VMSTATE_UINT64_TEST(curr_guest_offloads, VirtIONet,
3150                             has_ctrl_guest_offloads),
3151         VMSTATE_END_OF_LIST()
3152    },
3153     .subsections = (const VMStateDescription * []) {
3154         &vmstate_virtio_net_rss,
3155         NULL
3156     }
3157 };
3158 
3159 static NetClientInfo net_virtio_info = {
3160     .type = NET_CLIENT_DRIVER_NIC,
3161     .size = sizeof(NICState),
3162     .can_receive = virtio_net_can_receive,
3163     .receive = virtio_net_receive,
3164     .link_status_changed = virtio_net_set_link_status,
3165     .query_rx_filter = virtio_net_query_rxfilter,
3166     .announce = virtio_net_announce,
3167 };
3168 
3169 static bool virtio_net_guest_notifier_pending(VirtIODevice *vdev, int idx)
3170 {
3171     VirtIONet *n = VIRTIO_NET(vdev);
3172     NetClientState *nc = qemu_get_subqueue(n->nic, vq2q(idx));
3173     assert(n->vhost_started);
3174     return vhost_net_virtqueue_pending(get_vhost_net(nc->peer), idx);
3175 }
3176 
3177 static void virtio_net_guest_notifier_mask(VirtIODevice *vdev, int idx,
3178                                            bool mask)
3179 {
3180     VirtIONet *n = VIRTIO_NET(vdev);
3181     NetClientState *nc = qemu_get_subqueue(n->nic, vq2q(idx));
3182     assert(n->vhost_started);
3183     vhost_net_virtqueue_mask(get_vhost_net(nc->peer),
3184                              vdev, idx, mask);
3185 }
3186 
3187 static void virtio_net_set_config_size(VirtIONet *n, uint64_t host_features)
3188 {
3189     virtio_add_feature(&host_features, VIRTIO_NET_F_MAC);
3190 
3191     n->config_size = virtio_feature_get_config_size(feature_sizes,
3192                                                     host_features);
3193 }
3194 
3195 void virtio_net_set_netclient_name(VirtIONet *n, const char *name,
3196                                    const char *type)
3197 {
3198     /*
3199      * The name can be NULL, the netclient name will be type.x.
3200      */
3201     assert(type != NULL);
3202 
3203     g_free(n->netclient_name);
3204     g_free(n->netclient_type);
3205     n->netclient_name = g_strdup(name);
3206     n->netclient_type = g_strdup(type);
3207 }
3208 
3209 static bool failover_unplug_primary(VirtIONet *n, DeviceState *dev)
3210 {
3211     HotplugHandler *hotplug_ctrl;
3212     PCIDevice *pci_dev;
3213     Error *err = NULL;
3214 
3215     hotplug_ctrl = qdev_get_hotplug_handler(dev);
3216     if (hotplug_ctrl) {
3217         pci_dev = PCI_DEVICE(dev);
3218         pci_dev->partially_hotplugged = true;
3219         hotplug_handler_unplug_request(hotplug_ctrl, dev, &err);
3220         if (err) {
3221             error_report_err(err);
3222             return false;
3223         }
3224     } else {
3225         return false;
3226     }
3227     return true;
3228 }
3229 
3230 static bool failover_replug_primary(VirtIONet *n, DeviceState *dev,
3231                                     Error **errp)
3232 {
3233     Error *err = NULL;
3234     HotplugHandler *hotplug_ctrl;
3235     PCIDevice *pdev = PCI_DEVICE(dev);
3236     BusState *primary_bus;
3237 
3238     if (!pdev->partially_hotplugged) {
3239         return true;
3240     }
3241     primary_bus = dev->parent_bus;
3242     if (!primary_bus) {
3243         error_setg(errp, "virtio_net: couldn't find primary bus");
3244         return false;
3245     }
3246     qdev_set_parent_bus(dev, primary_bus, &error_abort);
3247     qatomic_set(&n->failover_primary_hidden, false);
3248     hotplug_ctrl = qdev_get_hotplug_handler(dev);
3249     if (hotplug_ctrl) {
3250         hotplug_handler_pre_plug(hotplug_ctrl, dev, &err);
3251         if (err) {
3252             goto out;
3253         }
3254         hotplug_handler_plug(hotplug_ctrl, dev, &err);
3255     }
3256     pdev->partially_hotplugged = false;
3257 
3258 out:
3259     error_propagate(errp, err);
3260     return !err;
3261 }
3262 
3263 static void virtio_net_handle_migration_primary(VirtIONet *n, MigrationState *s)
3264 {
3265     bool should_be_hidden;
3266     Error *err = NULL;
3267     DeviceState *dev = failover_find_primary_device(n);
3268 
3269     if (!dev) {
3270         return;
3271     }
3272 
3273     should_be_hidden = qatomic_read(&n->failover_primary_hidden);
3274 
3275     if (migration_in_setup(s) && !should_be_hidden) {
3276         if (failover_unplug_primary(n, dev)) {
3277             vmstate_unregister(VMSTATE_IF(dev), qdev_get_vmsd(dev), dev);
3278             qapi_event_send_unplug_primary(dev->id);
3279             qatomic_set(&n->failover_primary_hidden, true);
3280         } else {
3281             warn_report("couldn't unplug primary device");
3282         }
3283     } else if (migration_has_failed(s)) {
3284         /* We already unplugged the device let's plug it back */
3285         if (!failover_replug_primary(n, dev, &err)) {
3286             if (err) {
3287                 error_report_err(err);
3288             }
3289         }
3290     }
3291 }
3292 
3293 static void virtio_net_migration_state_notifier(Notifier *notifier, void *data)
3294 {
3295     MigrationState *s = data;
3296     VirtIONet *n = container_of(notifier, VirtIONet, migration_state);
3297     virtio_net_handle_migration_primary(n, s);
3298 }
3299 
3300 static bool failover_hide_primary_device(DeviceListener *listener,
3301                                          const QDict *device_opts,
3302                                          bool from_json,
3303                                          Error **errp)
3304 {
3305     VirtIONet *n = container_of(listener, VirtIONet, primary_listener);
3306     const char *standby_id;
3307 
3308     if (!device_opts) {
3309         return false;
3310     }
3311 
3312     if (!qdict_haskey(device_opts, "failover_pair_id")) {
3313         return false;
3314     }
3315 
3316     if (!qdict_haskey(device_opts, "id")) {
3317         error_setg(errp, "Device with failover_pair_id needs to have id");
3318         return false;
3319     }
3320 
3321     standby_id = qdict_get_str(device_opts, "failover_pair_id");
3322     if (g_strcmp0(standby_id, n->netclient_name) != 0) {
3323         return false;
3324     }
3325 
3326     /*
3327      * The hide helper can be called several times for a given device.
3328      * Check there is only one primary for a virtio-net device but
3329      * don't duplicate the qdict several times if it's called for the same
3330      * device.
3331      */
3332     if (n->primary_opts) {
3333         const char *old, *new;
3334         /* devices with failover_pair_id always have an id */
3335         old = qdict_get_str(n->primary_opts, "id");
3336         new = qdict_get_str(device_opts, "id");
3337         if (strcmp(old, new) != 0) {
3338             error_setg(errp, "Cannot attach more than one primary device to "
3339                        "'%s': '%s' and '%s'", n->netclient_name, old, new);
3340             return false;
3341         }
3342     } else {
3343         n->primary_opts = qdict_clone_shallow(device_opts);
3344         n->primary_opts_from_json = from_json;
3345     }
3346 
3347     /* failover_primary_hidden is set during feature negotiation */
3348     return qatomic_read(&n->failover_primary_hidden);
3349 }
3350 
3351 static void virtio_net_device_realize(DeviceState *dev, Error **errp)
3352 {
3353     VirtIODevice *vdev = VIRTIO_DEVICE(dev);
3354     VirtIONet *n = VIRTIO_NET(dev);
3355     NetClientState *nc;
3356     int i;
3357 
3358     if (n->net_conf.mtu) {
3359         n->host_features |= (1ULL << VIRTIO_NET_F_MTU);
3360     }
3361 
3362     if (n->net_conf.duplex_str) {
3363         if (strncmp(n->net_conf.duplex_str, "half", 5) == 0) {
3364             n->net_conf.duplex = DUPLEX_HALF;
3365         } else if (strncmp(n->net_conf.duplex_str, "full", 5) == 0) {
3366             n->net_conf.duplex = DUPLEX_FULL;
3367         } else {
3368             error_setg(errp, "'duplex' must be 'half' or 'full'");
3369             return;
3370         }
3371         n->host_features |= (1ULL << VIRTIO_NET_F_SPEED_DUPLEX);
3372     } else {
3373         n->net_conf.duplex = DUPLEX_UNKNOWN;
3374     }
3375 
3376     if (n->net_conf.speed < SPEED_UNKNOWN) {
3377         error_setg(errp, "'speed' must be between 0 and INT_MAX");
3378         return;
3379     }
3380     if (n->net_conf.speed >= 0) {
3381         n->host_features |= (1ULL << VIRTIO_NET_F_SPEED_DUPLEX);
3382     }
3383 
3384     if (n->failover) {
3385         n->primary_listener.hide_device = failover_hide_primary_device;
3386         qatomic_set(&n->failover_primary_hidden, true);
3387         device_listener_register(&n->primary_listener);
3388         n->migration_state.notify = virtio_net_migration_state_notifier;
3389         add_migration_state_change_notifier(&n->migration_state);
3390         n->host_features |= (1ULL << VIRTIO_NET_F_STANDBY);
3391     }
3392 
3393     virtio_net_set_config_size(n, n->host_features);
3394     virtio_init(vdev, "virtio-net", VIRTIO_ID_NET, n->config_size);
3395 
3396     /*
3397      * We set a lower limit on RX queue size to what it always was.
3398      * Guests that want a smaller ring can always resize it without
3399      * help from us (using virtio 1 and up).
3400      */
3401     if (n->net_conf.rx_queue_size < VIRTIO_NET_RX_QUEUE_MIN_SIZE ||
3402         n->net_conf.rx_queue_size > VIRTQUEUE_MAX_SIZE ||
3403         !is_power_of_2(n->net_conf.rx_queue_size)) {
3404         error_setg(errp, "Invalid rx_queue_size (= %" PRIu16 "), "
3405                    "must be a power of 2 between %d and %d.",
3406                    n->net_conf.rx_queue_size, VIRTIO_NET_RX_QUEUE_MIN_SIZE,
3407                    VIRTQUEUE_MAX_SIZE);
3408         virtio_cleanup(vdev);
3409         return;
3410     }
3411 
3412     if (n->net_conf.tx_queue_size < VIRTIO_NET_TX_QUEUE_MIN_SIZE ||
3413         n->net_conf.tx_queue_size > VIRTQUEUE_MAX_SIZE ||
3414         !is_power_of_2(n->net_conf.tx_queue_size)) {
3415         error_setg(errp, "Invalid tx_queue_size (= %" PRIu16 "), "
3416                    "must be a power of 2 between %d and %d",
3417                    n->net_conf.tx_queue_size, VIRTIO_NET_TX_QUEUE_MIN_SIZE,
3418                    VIRTQUEUE_MAX_SIZE);
3419         virtio_cleanup(vdev);
3420         return;
3421     }
3422 
3423     n->max_ncs = MAX(n->nic_conf.peers.queues, 1);
3424 
3425     /*
3426      * Figure out the datapath queue pairs since the backend could
3427      * provide control queue via peers as well.
3428      */
3429     if (n->nic_conf.peers.queues) {
3430         for (i = 0; i < n->max_ncs; i++) {
3431             if (n->nic_conf.peers.ncs[i]->is_datapath) {
3432                 ++n->max_queue_pairs;
3433             }
3434         }
3435     }
3436     n->max_queue_pairs = MAX(n->max_queue_pairs, 1);
3437 
3438     if (n->max_queue_pairs * 2 + 1 > VIRTIO_QUEUE_MAX) {
3439         error_setg(errp, "Invalid number of queue pairs (= %" PRIu32 "), "
3440                    "must be a positive integer less than %d.",
3441                    n->max_queue_pairs, (VIRTIO_QUEUE_MAX - 1) / 2);
3442         virtio_cleanup(vdev);
3443         return;
3444     }
3445     n->vqs = g_malloc0(sizeof(VirtIONetQueue) * n->max_queue_pairs);
3446     n->curr_queue_pairs = 1;
3447     n->tx_timeout = n->net_conf.txtimer;
3448 
3449     if (n->net_conf.tx && strcmp(n->net_conf.tx, "timer")
3450                        && strcmp(n->net_conf.tx, "bh")) {
3451         warn_report("virtio-net: "
3452                     "Unknown option tx=%s, valid options: \"timer\" \"bh\"",
3453                     n->net_conf.tx);
3454         error_printf("Defaulting to \"bh\"");
3455     }
3456 
3457     n->net_conf.tx_queue_size = MIN(virtio_net_max_tx_queue_size(n),
3458                                     n->net_conf.tx_queue_size);
3459 
3460     for (i = 0; i < n->max_queue_pairs; i++) {
3461         virtio_net_add_queue(n, i);
3462     }
3463 
3464     n->ctrl_vq = virtio_add_queue(vdev, 64, virtio_net_handle_ctrl);
3465     qemu_macaddr_default_if_unset(&n->nic_conf.macaddr);
3466     memcpy(&n->mac[0], &n->nic_conf.macaddr, sizeof(n->mac));
3467     n->status = VIRTIO_NET_S_LINK_UP;
3468     qemu_announce_timer_reset(&n->announce_timer, migrate_announce_params(),
3469                               QEMU_CLOCK_VIRTUAL,
3470                               virtio_net_announce_timer, n);
3471     n->announce_timer.round = 0;
3472 
3473     if (n->netclient_type) {
3474         /*
3475          * Happen when virtio_net_set_netclient_name has been called.
3476          */
3477         n->nic = qemu_new_nic(&net_virtio_info, &n->nic_conf,
3478                               n->netclient_type, n->netclient_name, n);
3479     } else {
3480         n->nic = qemu_new_nic(&net_virtio_info, &n->nic_conf,
3481                               object_get_typename(OBJECT(dev)), dev->id, n);
3482     }
3483 
3484     for (i = 0; i < n->max_queue_pairs; i++) {
3485         n->nic->ncs[i].do_not_pad = true;
3486     }
3487 
3488     peer_test_vnet_hdr(n);
3489     if (peer_has_vnet_hdr(n)) {
3490         for (i = 0; i < n->max_queue_pairs; i++) {
3491             qemu_using_vnet_hdr(qemu_get_subqueue(n->nic, i)->peer, true);
3492         }
3493         n->host_hdr_len = sizeof(struct virtio_net_hdr);
3494     } else {
3495         n->host_hdr_len = 0;
3496     }
3497 
3498     qemu_format_nic_info_str(qemu_get_queue(n->nic), n->nic_conf.macaddr.a);
3499 
3500     n->vqs[0].tx_waiting = 0;
3501     n->tx_burst = n->net_conf.txburst;
3502     virtio_net_set_mrg_rx_bufs(n, 0, 0, 0);
3503     n->promisc = 1; /* for compatibility */
3504 
3505     n->mac_table.macs = g_malloc0(MAC_TABLE_ENTRIES * ETH_ALEN);
3506 
3507     n->vlans = g_malloc0(MAX_VLAN >> 3);
3508 
3509     nc = qemu_get_queue(n->nic);
3510     nc->rxfilter_notify_enabled = 1;
3511 
3512    if (nc->peer && nc->peer->info->type == NET_CLIENT_DRIVER_VHOST_VDPA) {
3513         struct virtio_net_config netcfg = {};
3514         memcpy(&netcfg.mac, &n->nic_conf.macaddr, ETH_ALEN);
3515         vhost_net_set_config(get_vhost_net(nc->peer),
3516             (uint8_t *)&netcfg, 0, ETH_ALEN, VHOST_SET_CONFIG_TYPE_MASTER);
3517     }
3518     QTAILQ_INIT(&n->rsc_chains);
3519     n->qdev = dev;
3520 
3521     net_rx_pkt_init(&n->rx_pkt, false);
3522 
3523     if (virtio_has_feature(n->host_features, VIRTIO_NET_F_RSS)) {
3524         virtio_net_load_ebpf(n);
3525     }
3526 }
3527 
3528 static void virtio_net_device_unrealize(DeviceState *dev)
3529 {
3530     VirtIODevice *vdev = VIRTIO_DEVICE(dev);
3531     VirtIONet *n = VIRTIO_NET(dev);
3532     int i, max_queue_pairs;
3533 
3534     if (virtio_has_feature(n->host_features, VIRTIO_NET_F_RSS)) {
3535         virtio_net_unload_ebpf(n);
3536     }
3537 
3538     /* This will stop vhost backend if appropriate. */
3539     virtio_net_set_status(vdev, 0);
3540 
3541     g_free(n->netclient_name);
3542     n->netclient_name = NULL;
3543     g_free(n->netclient_type);
3544     n->netclient_type = NULL;
3545 
3546     g_free(n->mac_table.macs);
3547     g_free(n->vlans);
3548 
3549     if (n->failover) {
3550         qobject_unref(n->primary_opts);
3551         device_listener_unregister(&n->primary_listener);
3552         remove_migration_state_change_notifier(&n->migration_state);
3553     } else {
3554         assert(n->primary_opts == NULL);
3555     }
3556 
3557     max_queue_pairs = n->multiqueue ? n->max_queue_pairs : 1;
3558     for (i = 0; i < max_queue_pairs; i++) {
3559         virtio_net_del_queue(n, i);
3560     }
3561     /* delete also control vq */
3562     virtio_del_queue(vdev, max_queue_pairs * 2);
3563     qemu_announce_timer_del(&n->announce_timer, false);
3564     g_free(n->vqs);
3565     qemu_del_nic(n->nic);
3566     virtio_net_rsc_cleanup(n);
3567     g_free(n->rss_data.indirections_table);
3568     net_rx_pkt_uninit(n->rx_pkt);
3569     virtio_cleanup(vdev);
3570 }
3571 
3572 static void virtio_net_instance_init(Object *obj)
3573 {
3574     VirtIONet *n = VIRTIO_NET(obj);
3575 
3576     /*
3577      * The default config_size is sizeof(struct virtio_net_config).
3578      * Can be overriden with virtio_net_set_config_size.
3579      */
3580     n->config_size = sizeof(struct virtio_net_config);
3581     device_add_bootindex_property(obj, &n->nic_conf.bootindex,
3582                                   "bootindex", "/ethernet-phy@0",
3583                                   DEVICE(n));
3584 
3585     ebpf_rss_init(&n->ebpf_rss);
3586 }
3587 
3588 static int virtio_net_pre_save(void *opaque)
3589 {
3590     VirtIONet *n = opaque;
3591 
3592     /* At this point, backend must be stopped, otherwise
3593      * it might keep writing to memory. */
3594     assert(!n->vhost_started);
3595 
3596     return 0;
3597 }
3598 
3599 static bool primary_unplug_pending(void *opaque)
3600 {
3601     DeviceState *dev = opaque;
3602     DeviceState *primary;
3603     VirtIODevice *vdev = VIRTIO_DEVICE(dev);
3604     VirtIONet *n = VIRTIO_NET(vdev);
3605 
3606     if (!virtio_vdev_has_feature(vdev, VIRTIO_NET_F_STANDBY)) {
3607         return false;
3608     }
3609     primary = failover_find_primary_device(n);
3610     return primary ? primary->pending_deleted_event : false;
3611 }
3612 
3613 static bool dev_unplug_pending(void *opaque)
3614 {
3615     DeviceState *dev = opaque;
3616     VirtioDeviceClass *vdc = VIRTIO_DEVICE_GET_CLASS(dev);
3617 
3618     return vdc->primary_unplug_pending(dev);
3619 }
3620 
3621 static const VMStateDescription vmstate_virtio_net = {
3622     .name = "virtio-net",
3623     .minimum_version_id = VIRTIO_NET_VM_VERSION,
3624     .version_id = VIRTIO_NET_VM_VERSION,
3625     .fields = (VMStateField[]) {
3626         VMSTATE_VIRTIO_DEVICE,
3627         VMSTATE_END_OF_LIST()
3628     },
3629     .pre_save = virtio_net_pre_save,
3630     .dev_unplug_pending = dev_unplug_pending,
3631 };
3632 
3633 static Property virtio_net_properties[] = {
3634     DEFINE_PROP_BIT64("csum", VirtIONet, host_features,
3635                     VIRTIO_NET_F_CSUM, true),
3636     DEFINE_PROP_BIT64("guest_csum", VirtIONet, host_features,
3637                     VIRTIO_NET_F_GUEST_CSUM, true),
3638     DEFINE_PROP_BIT64("gso", VirtIONet, host_features, VIRTIO_NET_F_GSO, true),
3639     DEFINE_PROP_BIT64("guest_tso4", VirtIONet, host_features,
3640                     VIRTIO_NET_F_GUEST_TSO4, true),
3641     DEFINE_PROP_BIT64("guest_tso6", VirtIONet, host_features,
3642                     VIRTIO_NET_F_GUEST_TSO6, true),
3643     DEFINE_PROP_BIT64("guest_ecn", VirtIONet, host_features,
3644                     VIRTIO_NET_F_GUEST_ECN, true),
3645     DEFINE_PROP_BIT64("guest_ufo", VirtIONet, host_features,
3646                     VIRTIO_NET_F_GUEST_UFO, true),
3647     DEFINE_PROP_BIT64("guest_announce", VirtIONet, host_features,
3648                     VIRTIO_NET_F_GUEST_ANNOUNCE, true),
3649     DEFINE_PROP_BIT64("host_tso4", VirtIONet, host_features,
3650                     VIRTIO_NET_F_HOST_TSO4, true),
3651     DEFINE_PROP_BIT64("host_tso6", VirtIONet, host_features,
3652                     VIRTIO_NET_F_HOST_TSO6, true),
3653     DEFINE_PROP_BIT64("host_ecn", VirtIONet, host_features,
3654                     VIRTIO_NET_F_HOST_ECN, true),
3655     DEFINE_PROP_BIT64("host_ufo", VirtIONet, host_features,
3656                     VIRTIO_NET_F_HOST_UFO, true),
3657     DEFINE_PROP_BIT64("mrg_rxbuf", VirtIONet, host_features,
3658                     VIRTIO_NET_F_MRG_RXBUF, true),
3659     DEFINE_PROP_BIT64("status", VirtIONet, host_features,
3660                     VIRTIO_NET_F_STATUS, true),
3661     DEFINE_PROP_BIT64("ctrl_vq", VirtIONet, host_features,
3662                     VIRTIO_NET_F_CTRL_VQ, true),
3663     DEFINE_PROP_BIT64("ctrl_rx", VirtIONet, host_features,
3664                     VIRTIO_NET_F_CTRL_RX, true),
3665     DEFINE_PROP_BIT64("ctrl_vlan", VirtIONet, host_features,
3666                     VIRTIO_NET_F_CTRL_VLAN, true),
3667     DEFINE_PROP_BIT64("ctrl_rx_extra", VirtIONet, host_features,
3668                     VIRTIO_NET_F_CTRL_RX_EXTRA, true),
3669     DEFINE_PROP_BIT64("ctrl_mac_addr", VirtIONet, host_features,
3670                     VIRTIO_NET_F_CTRL_MAC_ADDR, true),
3671     DEFINE_PROP_BIT64("ctrl_guest_offloads", VirtIONet, host_features,
3672                     VIRTIO_NET_F_CTRL_GUEST_OFFLOADS, true),
3673     DEFINE_PROP_BIT64("mq", VirtIONet, host_features, VIRTIO_NET_F_MQ, false),
3674     DEFINE_PROP_BIT64("rss", VirtIONet, host_features,
3675                     VIRTIO_NET_F_RSS, false),
3676     DEFINE_PROP_BIT64("hash", VirtIONet, host_features,
3677                     VIRTIO_NET_F_HASH_REPORT, false),
3678     DEFINE_PROP_BIT64("guest_rsc_ext", VirtIONet, host_features,
3679                     VIRTIO_NET_F_RSC_EXT, false),
3680     DEFINE_PROP_UINT32("rsc_interval", VirtIONet, rsc_timeout,
3681                        VIRTIO_NET_RSC_DEFAULT_INTERVAL),
3682     DEFINE_NIC_PROPERTIES(VirtIONet, nic_conf),
3683     DEFINE_PROP_UINT32("x-txtimer", VirtIONet, net_conf.txtimer,
3684                        TX_TIMER_INTERVAL),
3685     DEFINE_PROP_INT32("x-txburst", VirtIONet, net_conf.txburst, TX_BURST),
3686     DEFINE_PROP_STRING("tx", VirtIONet, net_conf.tx),
3687     DEFINE_PROP_UINT16("rx_queue_size", VirtIONet, net_conf.rx_queue_size,
3688                        VIRTIO_NET_RX_QUEUE_DEFAULT_SIZE),
3689     DEFINE_PROP_UINT16("tx_queue_size", VirtIONet, net_conf.tx_queue_size,
3690                        VIRTIO_NET_TX_QUEUE_DEFAULT_SIZE),
3691     DEFINE_PROP_UINT16("host_mtu", VirtIONet, net_conf.mtu, 0),
3692     DEFINE_PROP_BOOL("x-mtu-bypass-backend", VirtIONet, mtu_bypass_backend,
3693                      true),
3694     DEFINE_PROP_INT32("speed", VirtIONet, net_conf.speed, SPEED_UNKNOWN),
3695     DEFINE_PROP_STRING("duplex", VirtIONet, net_conf.duplex_str),
3696     DEFINE_PROP_BOOL("failover", VirtIONet, failover, false),
3697     DEFINE_PROP_END_OF_LIST(),
3698 };
3699 
3700 static void virtio_net_class_init(ObjectClass *klass, void *data)
3701 {
3702     DeviceClass *dc = DEVICE_CLASS(klass);
3703     VirtioDeviceClass *vdc = VIRTIO_DEVICE_CLASS(klass);
3704 
3705     device_class_set_props(dc, virtio_net_properties);
3706     dc->vmsd = &vmstate_virtio_net;
3707     set_bit(DEVICE_CATEGORY_NETWORK, dc->categories);
3708     vdc->realize = virtio_net_device_realize;
3709     vdc->unrealize = virtio_net_device_unrealize;
3710     vdc->get_config = virtio_net_get_config;
3711     vdc->set_config = virtio_net_set_config;
3712     vdc->get_features = virtio_net_get_features;
3713     vdc->set_features = virtio_net_set_features;
3714     vdc->bad_features = virtio_net_bad_features;
3715     vdc->reset = virtio_net_reset;
3716     vdc->set_status = virtio_net_set_status;
3717     vdc->guest_notifier_mask = virtio_net_guest_notifier_mask;
3718     vdc->guest_notifier_pending = virtio_net_guest_notifier_pending;
3719     vdc->legacy_features |= (0x1 << VIRTIO_NET_F_GSO);
3720     vdc->post_load = virtio_net_post_load_virtio;
3721     vdc->vmsd = &vmstate_virtio_net_device;
3722     vdc->primary_unplug_pending = primary_unplug_pending;
3723 }
3724 
3725 static const TypeInfo virtio_net_info = {
3726     .name = TYPE_VIRTIO_NET,
3727     .parent = TYPE_VIRTIO_DEVICE,
3728     .instance_size = sizeof(VirtIONet),
3729     .instance_init = virtio_net_instance_init,
3730     .class_init = virtio_net_class_init,
3731 };
3732 
3733 static void virtio_register_types(void)
3734 {
3735     type_register_static(&virtio_net_info);
3736 }
3737 
3738 type_init(virtio_register_types)
3739