xref: /cloud-hypervisor/virtio-devices/src/net.rs (revision 9af2968a7dc47b89bf07ea9dc5e735084efcfa3a)
1 // Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.
2 // SPDX-License-Identifier: Apache-2.0
3 //
4 // Portions Copyright 2017 The Chromium OS Authors. All rights reserved.
5 // Use of this source code is governed by a BSD-style license that can be
6 // found in the THIRD-PARTY file.
7 
8 use super::Error as DeviceError;
9 use super::{
10     ActivateError, ActivateResult, EpollHelper, EpollHelperError, EpollHelperHandler, Queue,
11     RateLimiterConfig, VirtioCommon, VirtioDevice, VirtioDeviceType, VirtioInterruptType,
12     EPOLL_HELPER_EVENT_LAST,
13 };
14 use crate::seccomp_filters::{get_seccomp_filter, Thread};
15 use crate::GuestMemoryMmap;
16 use crate::VirtioInterrupt;
17 use net_util::CtrlQueue;
18 use net_util::{
19     build_net_config_space, build_net_config_space_with_mq, open_tap,
20     virtio_features_to_tap_offload, MacAddr, NetCounters, NetQueuePair, OpenTapError, RxVirtio,
21     Tap, TapError, TxVirtio, VirtioNetConfig,
22 };
23 use seccomp::{SeccompAction, SeccompFilter};
24 use std::net::Ipv4Addr;
25 use std::num::Wrapping;
26 use std::os::unix::io::{AsRawFd, RawFd};
27 use std::result;
28 use std::sync::atomic::{AtomicBool, Ordering};
29 use std::sync::{Arc, Barrier};
30 use std::thread;
31 use std::vec::Vec;
32 use std::{collections::HashMap, convert::TryInto};
33 use versionize::{VersionMap, Versionize, VersionizeResult};
34 use versionize_derive::Versionize;
35 use virtio_bindings::bindings::virtio_net::*;
36 use virtio_bindings::bindings::virtio_ring::VIRTIO_RING_F_EVENT_IDX;
37 use vm_memory::{ByteValued, GuestAddressSpace, GuestMemoryAtomic};
38 use vm_migration::VersionMapped;
39 use vm_migration::{Migratable, MigratableError, Pausable, Snapshot, Snapshottable, Transportable};
40 use vmm_sys_util::eventfd::EventFd;
41 
42 /// Control queue
43 // Event available on the control queue.
44 const CTRL_QUEUE_EVENT: u16 = EPOLL_HELPER_EVENT_LAST + 1;
45 
46 pub struct NetCtrlEpollHandler {
47     pub mem: GuestMemoryAtomic<GuestMemoryMmap>,
48     pub kill_evt: EventFd,
49     pub pause_evt: EventFd,
50     pub ctrl_q: CtrlQueue,
51     pub queue_evt: EventFd,
52     pub queue: Queue,
53 }
54 
55 impl NetCtrlEpollHandler {
56     pub fn run_ctrl(
57         &mut self,
58         paused: Arc<AtomicBool>,
59         paused_sync: Arc<Barrier>,
60     ) -> std::result::Result<(), EpollHelperError> {
61         let mut helper = EpollHelper::new(&self.kill_evt, &self.pause_evt)?;
62         helper.add_event(self.queue_evt.as_raw_fd(), CTRL_QUEUE_EVENT)?;
63         helper.run(paused, paused_sync, self)?;
64 
65         Ok(())
66     }
67 }
68 
69 impl EpollHelperHandler for NetCtrlEpollHandler {
70     fn handle_event(&mut self, _helper: &mut EpollHelper, event: &epoll::Event) -> bool {
71         let ev_type = event.data as u16;
72         match ev_type {
73             CTRL_QUEUE_EVENT => {
74                 let mem = self.mem.memory();
75                 if let Err(e) = self.queue_evt.read() {
76                     error!("failed to get ctl queue event: {:?}", e);
77                     return true;
78                 }
79                 if let Err(e) = self.ctrl_q.process(&mem, &mut self.queue) {
80                     error!("failed to process ctrl queue: {:?}", e);
81                     return true;
82                 }
83             }
84             _ => {
85                 error!("Unknown event for virtio-net");
86                 return true;
87             }
88         }
89 
90         false
91     }
92 }
93 
94 /// Rx/Tx queue pair
95 // The guest has made a buffer available to receive a frame into.
96 pub const RX_QUEUE_EVENT: u16 = EPOLL_HELPER_EVENT_LAST + 1;
97 // The transmit queue has a frame that is ready to send from the guest.
98 pub const TX_QUEUE_EVENT: u16 = EPOLL_HELPER_EVENT_LAST + 2;
99 // A frame is available for reading from the tap device to receive in the guest.
100 pub const RX_TAP_EVENT: u16 = EPOLL_HELPER_EVENT_LAST + 3;
101 // The TAP can be written to. Used after an EAGAIN error to retry TX.
102 pub const TX_TAP_EVENT: u16 = EPOLL_HELPER_EVENT_LAST + 4;
103 // New 'wake up' event from the rx rate limiter
104 pub const RX_RATE_LIMITER_EVENT: u16 = EPOLL_HELPER_EVENT_LAST + 5;
105 // New 'wake up' event from the tx rate limiter
106 pub const TX_RATE_LIMITER_EVENT: u16 = EPOLL_HELPER_EVENT_LAST + 6;
107 
108 #[derive(Debug)]
109 pub enum Error {
110     /// Failed to open taps.
111     OpenTap(OpenTapError),
112 
113     // Using existing tap
114     TapError(TapError),
115 
116     // Error calling dup() on tap fd
117     DuplicateTapFd(std::io::Error),
118 }
119 
120 pub type Result<T> = result::Result<T, Error>;
121 
122 struct NetEpollHandler {
123     net: NetQueuePair,
124     interrupt_cb: Arc<dyn VirtioInterrupt>,
125     kill_evt: EventFd,
126     pause_evt: EventFd,
127     queue_pair: Vec<Queue>,
128     queue_evt_pair: Vec<EventFd>,
129     // Always generate interrupts until the driver has signalled to the device.
130     // This mitigates a problem with interrupts from tap events being "lost" upon
131     // a restore as the vCPU thread isn't ready to handle the interrupt. This causes
132     // issues when combined with VIRTIO_RING_F_EVENT_IDX interrupt suppression.
133     driver_awake: bool,
134 }
135 
136 impl NetEpollHandler {
137     fn signal_used_queue(&self, queue: &Queue) -> result::Result<(), DeviceError> {
138         self.interrupt_cb
139             .trigger(&VirtioInterruptType::Queue, Some(queue))
140             .map_err(|e| {
141                 error!("Failed to signal used queue: {:?}", e);
142                 DeviceError::FailedSignalingUsedQueue(e)
143             })
144     }
145 
146     fn handle_rx_event(&mut self) -> result::Result<(), DeviceError> {
147         let queue_evt = &self.queue_evt_pair[0];
148         if let Err(e) = queue_evt.read() {
149             error!("Failed to get rx queue event: {:?}", e);
150         }
151 
152         self.net.rx_desc_avail = true;
153 
154         let rate_limit_reached = self
155             .net
156             .rx_rate_limiter
157             .as_ref()
158             .map_or(false, |r| r.is_blocked());
159 
160         // Start to listen on RX_TAP_EVENT only when the rate limit is not reached
161         if !self.net.rx_tap_listening && !rate_limit_reached {
162             net_util::register_listener(
163                 self.net.epoll_fd.unwrap(),
164                 self.net.tap.as_raw_fd(),
165                 epoll::Events::EPOLLIN,
166                 u64::from(self.net.tap_rx_event_id),
167             )
168             .map_err(DeviceError::IoError)?;
169             self.net.rx_tap_listening = true;
170         }
171 
172         Ok(())
173     }
174 
175     fn process_tx(&mut self) -> result::Result<(), DeviceError> {
176         if self
177             .net
178             .process_tx(&mut self.queue_pair[1])
179             .map_err(DeviceError::NetQueuePair)?
180             || !self.driver_awake
181         {
182             self.signal_used_queue(&self.queue_pair[1])?;
183             debug!("Signalling TX queue");
184         } else {
185             debug!("Not signalling TX queue");
186         }
187         Ok(())
188     }
189 
190     fn handle_tx_event(&mut self) -> result::Result<(), DeviceError> {
191         let rate_limit_reached = self
192             .net
193             .tx_rate_limiter
194             .as_ref()
195             .map_or(false, |r| r.is_blocked());
196 
197         if !rate_limit_reached {
198             self.process_tx()?;
199         }
200 
201         Ok(())
202     }
203 
204     fn handle_rx_tap_event(&mut self) -> result::Result<(), DeviceError> {
205         if self
206             .net
207             .process_rx(&mut self.queue_pair[0])
208             .map_err(DeviceError::NetQueuePair)?
209             || !self.driver_awake
210         {
211             self.signal_used_queue(&self.queue_pair[0])?;
212             debug!("Signalling RX queue");
213         } else {
214             debug!("Not signalling RX queue");
215         }
216         Ok(())
217     }
218 
219     fn run(
220         &mut self,
221         paused: Arc<AtomicBool>,
222         paused_sync: Arc<Barrier>,
223     ) -> result::Result<(), EpollHelperError> {
224         let mut helper = EpollHelper::new(&self.kill_evt, &self.pause_evt)?;
225         helper.add_event(self.queue_evt_pair[0].as_raw_fd(), RX_QUEUE_EVENT)?;
226         helper.add_event(self.queue_evt_pair[1].as_raw_fd(), TX_QUEUE_EVENT)?;
227         if let Some(rate_limiter) = &self.net.rx_rate_limiter {
228             helper.add_event(rate_limiter.as_raw_fd(), RX_RATE_LIMITER_EVENT)?;
229         }
230         if let Some(rate_limiter) = &self.net.tx_rate_limiter {
231             helper.add_event(rate_limiter.as_raw_fd(), TX_RATE_LIMITER_EVENT)?;
232         }
233 
234         // If there are some already available descriptors on the RX queue,
235         // then we can start the thread while listening onto the TAP.
236         if self.queue_pair[0]
237             .available_descriptors(&self.net.mem.as_ref().unwrap().memory())
238             .unwrap()
239         {
240             helper.add_event(self.net.tap.as_raw_fd(), RX_TAP_EVENT)?;
241             self.net.rx_tap_listening = true;
242             info!("Listener registered at start");
243         }
244 
245         // The NetQueuePair needs the epoll fd.
246         self.net.epoll_fd = Some(helper.as_raw_fd());
247 
248         helper.run(paused, paused_sync, self)?;
249 
250         Ok(())
251     }
252 }
253 
254 impl EpollHelperHandler for NetEpollHandler {
255     fn handle_event(&mut self, _helper: &mut EpollHelper, event: &epoll::Event) -> bool {
256         let ev_type = event.data as u16;
257         match ev_type {
258             RX_QUEUE_EVENT => {
259                 self.driver_awake = true;
260                 if let Err(e) = self.handle_rx_event() {
261                     error!("Error processing RX queue: {:?}", e);
262                     return true;
263                 }
264             }
265             TX_QUEUE_EVENT => {
266                 let queue_evt = &self.queue_evt_pair[1];
267                 if let Err(e) = queue_evt.read() {
268                     error!("Failed to get tx queue event: {:?}", e);
269                 }
270                 self.driver_awake = true;
271                 if let Err(e) = self.handle_tx_event() {
272                     error!("Error processing TX queue: {:?}", e);
273                     return true;
274                 }
275             }
276             TX_TAP_EVENT => {
277                 if let Err(e) = self.handle_tx_event() {
278                     error!("Error processing TX queue (TAP event): {:?}", e);
279                     return true;
280                 }
281             }
282             RX_TAP_EVENT => {
283                 if let Err(e) = self.handle_rx_tap_event() {
284                     error!("Error processing tap queue: {:?}", e);
285                     return true;
286                 }
287             }
288             RX_RATE_LIMITER_EVENT => {
289                 if let Some(rate_limiter) = &mut self.net.rx_rate_limiter {
290                     // Upon rate limiter event, call the rate limiter handler and register the
291                     // TAP fd for further processing if some RX buffers are available
292                     match rate_limiter.event_handler() {
293                         Ok(_) => {
294                             if !self.net.rx_tap_listening && self.net.rx_desc_avail {
295                                 if let Err(e) = net_util::register_listener(
296                                     self.net.epoll_fd.unwrap(),
297                                     self.net.tap.as_raw_fd(),
298                                     epoll::Events::EPOLLIN,
299                                     u64::from(self.net.tap_rx_event_id),
300                                 ) {
301                                     error!("Error register_listener with `RX_RATE_LIMITER_EVENT`: {:?}", e);
302                                     return true;
303                                 }
304                                 self.net.rx_tap_listening = true;
305                             }
306                         }
307                         Err(e) => {
308                             error!("Error from 'rate_limiter.event_handler()': {:?}", e);
309                             return true;
310                         }
311                     }
312                 } else {
313                     error!("Unexpected RX_RATE_LIMITER_EVENT");
314                     return true;
315                 }
316             }
317             TX_RATE_LIMITER_EVENT => {
318                 if let Some(rate_limiter) = &mut self.net.tx_rate_limiter {
319                     // Upon rate limiter event, call the rate limiter handler
320                     // and restart processing the queue.
321                     match rate_limiter.event_handler() {
322                         Ok(_) => {
323                             self.driver_awake = true;
324                             if let Err(e) = self.process_tx() {
325                                 error!("Error processing TX queue: {:?}", e);
326                                 return true;
327                             }
328                         }
329                         Err(e) => {
330                             error!("Error from 'rate_limiter.event_handler()': {:?}", e);
331                             return true;
332                         }
333                     }
334                 } else {
335                     error!("Unexpected TX_RATE_LIMITER_EVENT");
336                     return true;
337                 }
338             }
339             _ => {
340                 error!("Unknown event: {}", ev_type);
341                 return true;
342             }
343         }
344         false
345     }
346 }
347 
348 pub struct Net {
349     common: VirtioCommon,
350     id: String,
351     taps: Vec<Tap>,
352     config: VirtioNetConfig,
353     ctrl_queue_epoll_thread: Option<thread::JoinHandle<()>>,
354     counters: NetCounters,
355     seccomp_action: SeccompAction,
356     rate_limiter_config: Option<RateLimiterConfig>,
357 }
358 
359 #[derive(Versionize)]
360 pub struct NetState {
361     pub avail_features: u64,
362     pub acked_features: u64,
363     pub config: VirtioNetConfig,
364     pub queue_size: Vec<u16>,
365 }
366 
367 impl VersionMapped for NetState {}
368 
369 impl Net {
370     /// Create a new virtio network device with the given TAP interface.
371     #[allow(clippy::too_many_arguments)]
372     pub fn new_with_tap(
373         id: String,
374         taps: Vec<Tap>,
375         guest_mac: Option<MacAddr>,
376         iommu: bool,
377         num_queues: usize,
378         queue_size: u16,
379         seccomp_action: SeccompAction,
380         rate_limiter_config: Option<RateLimiterConfig>,
381     ) -> Result<Self> {
382         let mut avail_features = 1 << VIRTIO_NET_F_CSUM
383             | 1 << VIRTIO_NET_F_CTRL_GUEST_OFFLOADS
384             | 1 << VIRTIO_NET_F_GUEST_CSUM
385             | 1 << VIRTIO_NET_F_GUEST_ECN
386             | 1 << VIRTIO_NET_F_GUEST_TSO4
387             | 1 << VIRTIO_NET_F_GUEST_TSO6
388             | 1 << VIRTIO_NET_F_GUEST_UFO
389             | 1 << VIRTIO_NET_F_HOST_ECN
390             | 1 << VIRTIO_NET_F_HOST_TSO4
391             | 1 << VIRTIO_NET_F_HOST_TSO6
392             | 1 << VIRTIO_NET_F_HOST_UFO
393             | 1 << VIRTIO_RING_F_EVENT_IDX
394             | 1 << VIRTIO_F_VERSION_1;
395 
396         if iommu {
397             avail_features |= 1u64 << VIRTIO_F_IOMMU_PLATFORM;
398         }
399 
400         avail_features |= 1 << VIRTIO_NET_F_CTRL_VQ;
401         let queue_num = num_queues + 1;
402 
403         let mut config = VirtioNetConfig::default();
404         if let Some(mac) = guest_mac {
405             build_net_config_space(&mut config, mac, num_queues, &mut avail_features);
406         } else {
407             build_net_config_space_with_mq(&mut config, num_queues, &mut avail_features);
408         }
409 
410         Ok(Net {
411             common: VirtioCommon {
412                 device_type: VirtioDeviceType::Net as u32,
413                 avail_features,
414                 queue_sizes: vec![queue_size; queue_num],
415                 paused_sync: Some(Arc::new(Barrier::new((num_queues / 2) + 1))),
416                 min_queues: 2,
417                 ..Default::default()
418             },
419             id,
420             taps,
421             config,
422             ctrl_queue_epoll_thread: None,
423             counters: NetCounters::default(),
424             seccomp_action,
425             rate_limiter_config,
426         })
427     }
428 
429     /// Create a new virtio network device with the given IP address and
430     /// netmask.
431     #[allow(clippy::too_many_arguments)]
432     pub fn new(
433         id: String,
434         if_name: Option<&str>,
435         ip_addr: Option<Ipv4Addr>,
436         netmask: Option<Ipv4Addr>,
437         guest_mac: Option<MacAddr>,
438         host_mac: &mut Option<MacAddr>,
439         iommu: bool,
440         num_queues: usize,
441         queue_size: u16,
442         seccomp_action: SeccompAction,
443         rate_limiter_config: Option<RateLimiterConfig>,
444     ) -> Result<Self> {
445         let taps = open_tap(if_name, ip_addr, netmask, host_mac, num_queues / 2, None)
446             .map_err(Error::OpenTap)?;
447 
448         Self::new_with_tap(
449             id,
450             taps,
451             guest_mac,
452             iommu,
453             num_queues,
454             queue_size,
455             seccomp_action,
456             rate_limiter_config,
457         )
458     }
459 
460     pub fn from_tap_fds(
461         id: String,
462         fds: &[RawFd],
463         guest_mac: Option<MacAddr>,
464         iommu: bool,
465         queue_size: u16,
466         seccomp_action: SeccompAction,
467         rate_limiter_config: Option<RateLimiterConfig>,
468     ) -> Result<Self> {
469         let mut taps: Vec<Tap> = Vec::new();
470         let num_queue_pairs = fds.len();
471 
472         for fd in fds.iter() {
473             // Duplicate so that it can survive reboots
474             let fd = unsafe { libc::dup(*fd) };
475             if fd < 0 {
476                 return Err(Error::DuplicateTapFd(std::io::Error::last_os_error()));
477             }
478             let tap = Tap::from_tap_fd(fd, num_queue_pairs).map_err(Error::TapError)?;
479             taps.push(tap);
480         }
481 
482         Self::new_with_tap(
483             id,
484             taps,
485             guest_mac,
486             iommu,
487             num_queue_pairs * 2,
488             queue_size,
489             seccomp_action,
490             rate_limiter_config,
491         )
492     }
493 
494     fn state(&self) -> NetState {
495         NetState {
496             avail_features: self.common.avail_features,
497             acked_features: self.common.acked_features,
498             config: self.config,
499             queue_size: self.common.queue_sizes.clone(),
500         }
501     }
502 
503     fn set_state(&mut self, state: &NetState) {
504         self.common.avail_features = state.avail_features;
505         self.common.acked_features = state.acked_features;
506         self.config = state.config;
507         self.common.queue_sizes = state.queue_size.clone();
508     }
509 }
510 
511 impl Drop for Net {
512     fn drop(&mut self) {
513         if let Some(kill_evt) = self.common.kill_evt.take() {
514             // Ignore the result because there is nothing we can do about it.
515             let _ = kill_evt.write(1);
516         }
517     }
518 }
519 
520 impl VirtioDevice for Net {
521     fn device_type(&self) -> u32 {
522         self.common.device_type
523     }
524 
525     fn queue_max_sizes(&self) -> &[u16] {
526         &self.common.queue_sizes
527     }
528 
529     fn features(&self) -> u64 {
530         self.common.avail_features
531     }
532 
533     fn ack_features(&mut self, value: u64) {
534         self.common.ack_features(value)
535     }
536 
537     fn read_config(&self, offset: u64, data: &mut [u8]) {
538         self.read_config_from_slice(self.config.as_slice(), offset, data);
539     }
540 
541     fn activate(
542         &mut self,
543         mem: GuestMemoryAtomic<GuestMemoryMmap>,
544         interrupt_cb: Arc<dyn VirtioInterrupt>,
545         mut queues: Vec<Queue>,
546         mut queue_evts: Vec<EventFd>,
547     ) -> ActivateResult {
548         self.common.activate(&queues, &queue_evts, &interrupt_cb)?;
549 
550         let queue_num = queues.len();
551         if self.common.feature_acked(VIRTIO_NET_F_CTRL_VQ.into()) && queue_num % 2 != 0 {
552             let cvq_queue = queues.remove(queue_num - 1);
553             let cvq_queue_evt = queue_evts.remove(queue_num - 1);
554 
555             let (kill_evt, pause_evt) = self.common.dup_eventfds();
556             let mut ctrl_handler = NetCtrlEpollHandler {
557                 mem: mem.clone(),
558                 kill_evt,
559                 pause_evt,
560                 ctrl_q: CtrlQueue::new(self.taps.clone()),
561                 queue: cvq_queue,
562                 queue_evt: cvq_queue_evt,
563             };
564 
565             let paused = self.common.paused.clone();
566             // Let's update the barrier as we need 1 for each RX/TX pair +
567             // 1 for the control queue + 1 for the main thread signalling
568             // the pause.
569             self.common.paused_sync = Some(Arc::new(Barrier::new(self.taps.len() + 2)));
570             let paused_sync = self.common.paused_sync.clone();
571 
572             // Retrieve seccomp filter for virtio_net_ctl thread
573             let virtio_net_ctl_seccomp_filter =
574                 get_seccomp_filter(&self.seccomp_action, Thread::VirtioNetCtl)
575                     .map_err(ActivateError::CreateSeccompFilter)?;
576             thread::Builder::new()
577                 .name(format!("{}_ctrl", self.id))
578                 .spawn(move || {
579                     if let Err(e) = SeccompFilter::apply(virtio_net_ctl_seccomp_filter) {
580                         error!("Error applying seccomp filter: {:?}", e);
581                     } else if let Err(e) = ctrl_handler.run_ctrl(paused, paused_sync.unwrap()) {
582                         error!("Error running worker: {:?}", e);
583                     }
584                 })
585                 .map(|thread| self.ctrl_queue_epoll_thread = Some(thread))
586                 .map_err(|e| {
587                     error!("failed to clone queue EventFd: {}", e);
588                     ActivateError::BadActivate
589                 })?;
590         }
591 
592         let event_idx = self.common.feature_acked(VIRTIO_RING_F_EVENT_IDX.into());
593 
594         let mut epoll_threads = Vec::new();
595         let mut taps = self.taps.clone();
596         for i in 0..queues.len() / 2 {
597             let rx = RxVirtio::new();
598             let tx = TxVirtio::new();
599             let rx_tap_listening = false;
600 
601             let mut queue_pair = vec![queues.remove(0), queues.remove(0)];
602             queue_pair[0].set_event_idx(event_idx);
603             queue_pair[1].set_event_idx(event_idx);
604 
605             let queue_evt_pair = vec![queue_evts.remove(0), queue_evts.remove(0)];
606 
607             let (kill_evt, pause_evt) = self.common.dup_eventfds();
608 
609             let rx_rate_limiter: Option<rate_limiter::RateLimiter> = self
610                 .rate_limiter_config
611                 .map(RateLimiterConfig::try_into)
612                 .transpose()
613                 .map_err(ActivateError::CreateRateLimiter)?;
614 
615             let tx_rate_limiter: Option<rate_limiter::RateLimiter> = self
616                 .rate_limiter_config
617                 .map(RateLimiterConfig::try_into)
618                 .transpose()
619                 .map_err(ActivateError::CreateRateLimiter)?;
620 
621             let tap = taps.remove(0);
622             tap.set_offload(virtio_features_to_tap_offload(self.common.acked_features))
623                 .map_err(|e| {
624                     error!("Error programming tap offload: {:?}", e);
625                     ActivateError::BadActivate
626                 })?;
627 
628             let mut handler = NetEpollHandler {
629                 net: NetQueuePair {
630                     mem: Some(mem.clone()),
631                     tap_for_write_epoll: tap.clone(),
632                     tap,
633                     rx,
634                     tx,
635                     epoll_fd: None,
636                     rx_tap_listening,
637                     tx_tap_listening: false,
638                     counters: self.counters.clone(),
639                     tap_rx_event_id: RX_TAP_EVENT,
640                     tap_tx_event_id: TX_TAP_EVENT,
641                     rx_desc_avail: false,
642                     rx_rate_limiter,
643                     tx_rate_limiter,
644                 },
645                 queue_pair,
646                 queue_evt_pair,
647                 interrupt_cb: interrupt_cb.clone(),
648                 kill_evt,
649                 pause_evt,
650                 driver_awake: false,
651             };
652 
653             let paused = self.common.paused.clone();
654             let paused_sync = self.common.paused_sync.clone();
655             // Retrieve seccomp filter for virtio_net thread
656             let virtio_net_seccomp_filter =
657                 get_seccomp_filter(&self.seccomp_action, Thread::VirtioNet)
658                     .map_err(ActivateError::CreateSeccompFilter)?;
659             thread::Builder::new()
660                 .name(format!("{}_qp{}", self.id.clone(), i))
661                 .spawn(move || {
662                     if let Err(e) = SeccompFilter::apply(virtio_net_seccomp_filter) {
663                         error!("Error applying seccomp filter: {:?}", e);
664                     } else if let Err(e) = handler.run(paused, paused_sync.unwrap()) {
665                         error!("Error running worker: {:?}", e);
666                     }
667                 })
668                 .map(|thread| epoll_threads.push(thread))
669                 .map_err(|e| {
670                     error!("failed to clone queue EventFd: {}", e);
671                     ActivateError::BadActivate
672                 })?;
673         }
674 
675         self.common.epoll_threads = Some(epoll_threads);
676 
677         event!("virtio-device", "activated", "id", &self.id);
678         Ok(())
679     }
680 
681     fn reset(&mut self) -> Option<Arc<dyn VirtioInterrupt>> {
682         let result = self.common.reset();
683         event!("virtio-device", "reset", "id", &self.id);
684         result
685     }
686 
687     fn counters(&self) -> Option<HashMap<&'static str, Wrapping<u64>>> {
688         let mut counters = HashMap::new();
689 
690         counters.insert(
691             "rx_bytes",
692             Wrapping(self.counters.rx_bytes.load(Ordering::Acquire)),
693         );
694         counters.insert(
695             "rx_frames",
696             Wrapping(self.counters.rx_frames.load(Ordering::Acquire)),
697         );
698         counters.insert(
699             "tx_bytes",
700             Wrapping(self.counters.tx_bytes.load(Ordering::Acquire)),
701         );
702         counters.insert(
703             "tx_frames",
704             Wrapping(self.counters.tx_frames.load(Ordering::Acquire)),
705         );
706 
707         Some(counters)
708     }
709 }
710 
711 impl Pausable for Net {
712     fn pause(&mut self) -> result::Result<(), MigratableError> {
713         self.common.pause()
714     }
715 
716     fn resume(&mut self) -> result::Result<(), MigratableError> {
717         self.common.resume()?;
718 
719         if let Some(ctrl_queue_epoll_thread) = &self.ctrl_queue_epoll_thread {
720             ctrl_queue_epoll_thread.thread().unpark();
721         }
722         Ok(())
723     }
724 }
725 
726 impl Snapshottable for Net {
727     fn id(&self) -> String {
728         self.id.clone()
729     }
730 
731     fn snapshot(&mut self) -> std::result::Result<Snapshot, MigratableError> {
732         Snapshot::new_from_versioned_state(&self.id, &self.state())
733     }
734 
735     fn restore(&mut self, snapshot: Snapshot) -> std::result::Result<(), MigratableError> {
736         self.set_state(&snapshot.to_versioned_state(&self.id)?);
737         Ok(())
738     }
739 }
740 impl Transportable for Net {}
741 impl Migratable for Net {}
742