1 // Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 // SPDX-License-Identifier: Apache-2.0 3 // 4 // Portions Copyright 2017 The Chromium OS Authors. All rights reserved. 5 // Use of this source code is governed by a BSD-style license that can be 6 // found in the THIRD-PARTY file. 7 8 use super::Error as DeviceError; 9 use super::{ 10 ActivateError, ActivateResult, EpollHelper, EpollHelperError, EpollHelperHandler, 11 RateLimiterConfig, VirtioCommon, VirtioDevice, VirtioDeviceType, VirtioInterruptType, 12 EPOLL_HELPER_EVENT_LAST, 13 }; 14 use crate::seccomp_filters::Thread; 15 use crate::thread_helper::spawn_virtio_thread; 16 use crate::GuestMemoryMmap; 17 use crate::VirtioInterrupt; 18 use anyhow::anyhow; 19 use net_util::CtrlQueue; 20 use net_util::{ 21 build_net_config_space, build_net_config_space_with_mq, open_tap, 22 virtio_features_to_tap_offload, MacAddr, NetCounters, NetQueuePair, OpenTapError, RxVirtio, 23 Tap, TapError, TxVirtio, VirtioNetConfig, 24 }; 25 use seccompiler::SeccompAction; 26 use std::net::Ipv4Addr; 27 use std::num::Wrapping; 28 use std::ops::Deref; 29 use std::os::unix::io::{AsRawFd, RawFd}; 30 use std::result; 31 use std::sync::atomic::{AtomicBool, Ordering}; 32 use std::sync::{Arc, Barrier}; 33 use std::thread; 34 use std::vec::Vec; 35 use std::{collections::HashMap, convert::TryInto}; 36 use thiserror::Error; 37 use versionize::{VersionMap, Versionize, VersionizeResult}; 38 use versionize_derive::Versionize; 39 use virtio_bindings::bindings::virtio_net::*; 40 use virtio_bindings::bindings::virtio_ring::VIRTIO_RING_F_EVENT_IDX; 41 use virtio_queue::{Queue, QueueT}; 42 use vm_memory::{ByteValued, GuestAddressSpace, GuestMemoryAtomic}; 43 use vm_migration::VersionMapped; 44 use vm_migration::{Migratable, MigratableError, Pausable, Snapshot, Snapshottable, Transportable}; 45 use vm_virtio::AccessPlatform; 46 use vmm_sys_util::eventfd::EventFd; 47 48 /// Control queue 49 // Event available on the control queue. 50 const CTRL_QUEUE_EVENT: u16 = EPOLL_HELPER_EVENT_LAST + 1; 51 52 // Following the VIRTIO specification, the MTU should be at least 1280. 53 pub const MIN_MTU: u16 = 1280; 54 55 pub struct NetCtrlEpollHandler { 56 pub mem: GuestMemoryAtomic<GuestMemoryMmap>, 57 pub kill_evt: EventFd, 58 pub pause_evt: EventFd, 59 pub ctrl_q: CtrlQueue, 60 pub queue_evt: EventFd, 61 pub queue: Queue, 62 pub access_platform: Option<Arc<dyn AccessPlatform>>, 63 pub interrupt_cb: Arc<dyn VirtioInterrupt>, 64 pub queue_index: u16, 65 } 66 67 impl NetCtrlEpollHandler { 68 fn signal_used_queue(&self, queue_index: u16) -> result::Result<(), DeviceError> { 69 self.interrupt_cb 70 .trigger(VirtioInterruptType::Queue(queue_index)) 71 .map_err(|e| { 72 error!("Failed to signal used queue: {:?}", e); 73 DeviceError::FailedSignalingUsedQueue(e) 74 }) 75 } 76 77 pub fn run_ctrl( 78 &mut self, 79 paused: Arc<AtomicBool>, 80 paused_sync: Arc<Barrier>, 81 ) -> std::result::Result<(), EpollHelperError> { 82 let mut helper = EpollHelper::new(&self.kill_evt, &self.pause_evt)?; 83 helper.add_event(self.queue_evt.as_raw_fd(), CTRL_QUEUE_EVENT)?; 84 helper.run(paused, paused_sync, self)?; 85 86 Ok(()) 87 } 88 } 89 90 impl EpollHelperHandler for NetCtrlEpollHandler { 91 fn handle_event( 92 &mut self, 93 _helper: &mut EpollHelper, 94 event: &epoll::Event, 95 ) -> result::Result<(), EpollHelperError> { 96 let ev_type = event.data as u16; 97 match ev_type { 98 CTRL_QUEUE_EVENT => { 99 let mem = self.mem.memory(); 100 self.queue_evt.read().map_err(|e| { 101 EpollHelperError::HandleEvent(anyhow!( 102 "Failed to get control queue event: {:?}", 103 e 104 )) 105 })?; 106 self.ctrl_q 107 .process(mem.deref(), &mut self.queue, self.access_platform.as_ref()) 108 .map_err(|e| { 109 EpollHelperError::HandleEvent(anyhow!( 110 "Failed to process control queue: {:?}", 111 e 112 )) 113 })?; 114 match self.queue.needs_notification(mem.deref()) { 115 Ok(true) => { 116 self.signal_used_queue(self.queue_index).map_err(|e| { 117 EpollHelperError::HandleEvent(anyhow!( 118 "Error signalling that control queue was used: {:?}", 119 e 120 )) 121 })?; 122 } 123 Ok(false) => {} 124 Err(e) => { 125 return Err(EpollHelperError::HandleEvent(anyhow!( 126 "Error getting notification state of control queue: {}", 127 e 128 ))); 129 } 130 }; 131 } 132 _ => { 133 return Err(EpollHelperError::HandleEvent(anyhow!( 134 "Unknown event for virtio-net control queue" 135 ))); 136 } 137 } 138 139 Ok(()) 140 } 141 } 142 143 /// Rx/Tx queue pair 144 // The guest has made a buffer available to receive a frame into. 145 pub const RX_QUEUE_EVENT: u16 = EPOLL_HELPER_EVENT_LAST + 1; 146 // The transmit queue has a frame that is ready to send from the guest. 147 pub const TX_QUEUE_EVENT: u16 = EPOLL_HELPER_EVENT_LAST + 2; 148 // A frame is available for reading from the tap device to receive in the guest. 149 pub const RX_TAP_EVENT: u16 = EPOLL_HELPER_EVENT_LAST + 3; 150 // The TAP can be written to. Used after an EAGAIN error to retry TX. 151 pub const TX_TAP_EVENT: u16 = EPOLL_HELPER_EVENT_LAST + 4; 152 // New 'wake up' event from the rx rate limiter 153 pub const RX_RATE_LIMITER_EVENT: u16 = EPOLL_HELPER_EVENT_LAST + 5; 154 // New 'wake up' event from the tx rate limiter 155 pub const TX_RATE_LIMITER_EVENT: u16 = EPOLL_HELPER_EVENT_LAST + 6; 156 157 #[derive(Error, Debug)] 158 pub enum Error { 159 #[error("Failed to open taps: {0}")] 160 OpenTap(OpenTapError), 161 #[error("Using existing tap: {0}")] 162 TapError(TapError), 163 #[error("Error calling dup() on tap fd: {0}")] 164 DuplicateTapFd(std::io::Error), 165 } 166 167 pub type Result<T> = result::Result<T, Error>; 168 169 struct NetEpollHandler { 170 net: NetQueuePair, 171 mem: GuestMemoryAtomic<GuestMemoryMmap>, 172 interrupt_cb: Arc<dyn VirtioInterrupt>, 173 kill_evt: EventFd, 174 pause_evt: EventFd, 175 queue_index_base: u16, 176 queue_pair: (Queue, Queue), 177 queue_evt_pair: (EventFd, EventFd), 178 // Always generate interrupts until the driver has signalled to the device. 179 // This mitigates a problem with interrupts from tap events being "lost" upon 180 // a restore as the vCPU thread isn't ready to handle the interrupt. This causes 181 // issues when combined with VIRTIO_RING_F_EVENT_IDX interrupt suppression. 182 driver_awake: bool, 183 } 184 185 impl NetEpollHandler { 186 fn signal_used_queue(&self, queue_index: u16) -> result::Result<(), DeviceError> { 187 self.interrupt_cb 188 .trigger(VirtioInterruptType::Queue(queue_index)) 189 .map_err(|e| { 190 error!("Failed to signal used queue: {:?}", e); 191 DeviceError::FailedSignalingUsedQueue(e) 192 }) 193 } 194 195 fn handle_rx_event(&mut self) -> result::Result<(), DeviceError> { 196 let queue_evt = &self.queue_evt_pair.0; 197 if let Err(e) = queue_evt.read() { 198 error!("Failed to get rx queue event: {:?}", e); 199 } 200 201 self.net.rx_desc_avail = true; 202 203 let rate_limit_reached = self 204 .net 205 .rx_rate_limiter 206 .as_ref() 207 .map_or(false, |r| r.is_blocked()); 208 209 // Start to listen on RX_TAP_EVENT only when the rate limit is not reached 210 if !self.net.rx_tap_listening && !rate_limit_reached { 211 net_util::register_listener( 212 self.net.epoll_fd.unwrap(), 213 self.net.tap.as_raw_fd(), 214 epoll::Events::EPOLLIN, 215 u64::from(self.net.tap_rx_event_id), 216 ) 217 .map_err(DeviceError::IoError)?; 218 self.net.rx_tap_listening = true; 219 } 220 221 Ok(()) 222 } 223 224 fn process_tx(&mut self) -> result::Result<(), DeviceError> { 225 if self 226 .net 227 .process_tx(&self.mem.memory(), &mut self.queue_pair.1) 228 .map_err(DeviceError::NetQueuePair)? 229 || !self.driver_awake 230 { 231 self.signal_used_queue(self.queue_index_base + 1)?; 232 debug!("Signalling TX queue"); 233 } else { 234 debug!("Not signalling TX queue"); 235 } 236 Ok(()) 237 } 238 239 fn handle_tx_event(&mut self) -> result::Result<(), DeviceError> { 240 let rate_limit_reached = self 241 .net 242 .tx_rate_limiter 243 .as_ref() 244 .map_or(false, |r| r.is_blocked()); 245 246 if !rate_limit_reached { 247 self.process_tx()?; 248 } 249 250 Ok(()) 251 } 252 253 fn handle_rx_tap_event(&mut self) -> result::Result<(), DeviceError> { 254 if self 255 .net 256 .process_rx(&self.mem.memory(), &mut self.queue_pair.0) 257 .map_err(DeviceError::NetQueuePair)? 258 || !self.driver_awake 259 { 260 self.signal_used_queue(self.queue_index_base)?; 261 debug!("Signalling RX queue"); 262 } else { 263 debug!("Not signalling RX queue"); 264 } 265 Ok(()) 266 } 267 268 fn run( 269 &mut self, 270 paused: Arc<AtomicBool>, 271 paused_sync: Arc<Barrier>, 272 ) -> result::Result<(), EpollHelperError> { 273 let mut helper = EpollHelper::new(&self.kill_evt, &self.pause_evt)?; 274 helper.add_event(self.queue_evt_pair.0.as_raw_fd(), RX_QUEUE_EVENT)?; 275 helper.add_event(self.queue_evt_pair.1.as_raw_fd(), TX_QUEUE_EVENT)?; 276 if let Some(rate_limiter) = &self.net.rx_rate_limiter { 277 helper.add_event(rate_limiter.as_raw_fd(), RX_RATE_LIMITER_EVENT)?; 278 } 279 if let Some(rate_limiter) = &self.net.tx_rate_limiter { 280 helper.add_event(rate_limiter.as_raw_fd(), TX_RATE_LIMITER_EVENT)?; 281 } 282 283 let mem = self.mem.memory(); 284 // If there are some already available descriptors on the RX queue, 285 // then we can start the thread while listening onto the TAP. 286 if self 287 .queue_pair 288 .0 289 .used_idx(mem.deref(), Ordering::Acquire) 290 .map_err(EpollHelperError::QueueRingIndex)? 291 < self 292 .queue_pair 293 .0 294 .avail_idx(mem.deref(), Ordering::Acquire) 295 .map_err(EpollHelperError::QueueRingIndex)? 296 { 297 helper.add_event(self.net.tap.as_raw_fd(), RX_TAP_EVENT)?; 298 self.net.rx_tap_listening = true; 299 info!("Listener registered at start"); 300 } 301 302 // The NetQueuePair needs the epoll fd. 303 self.net.epoll_fd = Some(helper.as_raw_fd()); 304 305 helper.run(paused, paused_sync, self)?; 306 307 Ok(()) 308 } 309 } 310 311 impl EpollHelperHandler for NetEpollHandler { 312 fn handle_event( 313 &mut self, 314 _helper: &mut EpollHelper, 315 event: &epoll::Event, 316 ) -> result::Result<(), EpollHelperError> { 317 let ev_type = event.data as u16; 318 match ev_type { 319 RX_QUEUE_EVENT => { 320 self.driver_awake = true; 321 self.handle_rx_event().map_err(|e| { 322 EpollHelperError::HandleEvent(anyhow!("Error processing RX queue: {:?}", e)) 323 })?; 324 } 325 TX_QUEUE_EVENT => { 326 let queue_evt = &self.queue_evt_pair.1; 327 if let Err(e) = queue_evt.read() { 328 error!("Failed to get tx queue event: {:?}", e); 329 } 330 self.driver_awake = true; 331 self.handle_tx_event().map_err(|e| { 332 EpollHelperError::HandleEvent(anyhow!("Error processing TX queue: {:?}", e)) 333 })?; 334 } 335 TX_TAP_EVENT => { 336 self.handle_tx_event().map_err(|e| { 337 EpollHelperError::HandleEvent(anyhow!( 338 "Error processing TX queue (TAP event): {:?}", 339 e 340 )) 341 })?; 342 } 343 RX_TAP_EVENT => { 344 self.handle_rx_tap_event().map_err(|e| { 345 EpollHelperError::HandleEvent(anyhow!("Error processing tap queue: {:?}", e)) 346 })?; 347 } 348 RX_RATE_LIMITER_EVENT => { 349 if let Some(rate_limiter) = &mut self.net.rx_rate_limiter { 350 // Upon rate limiter event, call the rate limiter handler and register the 351 // TAP fd for further processing if some RX buffers are available 352 rate_limiter.event_handler().map_err(|e| { 353 EpollHelperError::HandleEvent(anyhow!( 354 "Error from 'rate_limiter.event_handler()': {:?}", 355 e 356 )) 357 })?; 358 359 if !self.net.rx_tap_listening && self.net.rx_desc_avail { 360 net_util::register_listener( 361 self.net.epoll_fd.unwrap(), 362 self.net.tap.as_raw_fd(), 363 epoll::Events::EPOLLIN, 364 u64::from(self.net.tap_rx_event_id), 365 ) 366 .map_err(|e| { 367 EpollHelperError::HandleEvent(anyhow!( 368 "Error register_listener with `RX_RATE_LIMITER_EVENT`: {:?}", 369 e 370 )) 371 })?; 372 373 self.net.rx_tap_listening = true; 374 } 375 } else { 376 return Err(EpollHelperError::HandleEvent(anyhow!( 377 "Unexpected RX_RATE_LIMITER_EVENT" 378 ))); 379 } 380 } 381 TX_RATE_LIMITER_EVENT => { 382 if let Some(rate_limiter) = &mut self.net.tx_rate_limiter { 383 // Upon rate limiter event, call the rate limiter handler 384 // and restart processing the queue. 385 rate_limiter.event_handler().map_err(|e| { 386 EpollHelperError::HandleEvent(anyhow!( 387 "Error from 'rate_limiter.event_handler()': {:?}", 388 e 389 )) 390 })?; 391 392 self.driver_awake = true; 393 self.process_tx().map_err(|e| { 394 EpollHelperError::HandleEvent(anyhow!("Error processing TX queue: {:?}", e)) 395 })?; 396 } else { 397 return Err(EpollHelperError::HandleEvent(anyhow!( 398 "Unexpected TX_RATE_LIMITER_EVENT" 399 ))); 400 } 401 } 402 _ => { 403 return Err(EpollHelperError::HandleEvent(anyhow!( 404 "Unexpected event: {}", 405 ev_type 406 ))); 407 } 408 } 409 Ok(()) 410 } 411 } 412 413 pub struct Net { 414 common: VirtioCommon, 415 id: String, 416 taps: Vec<Tap>, 417 config: VirtioNetConfig, 418 ctrl_queue_epoll_thread: Option<thread::JoinHandle<()>>, 419 counters: NetCounters, 420 seccomp_action: SeccompAction, 421 rate_limiter_config: Option<RateLimiterConfig>, 422 exit_evt: EventFd, 423 } 424 425 #[derive(Versionize)] 426 pub struct NetState { 427 pub avail_features: u64, 428 pub acked_features: u64, 429 pub config: VirtioNetConfig, 430 pub queue_size: Vec<u16>, 431 } 432 433 impl VersionMapped for NetState {} 434 435 impl Net { 436 /// Create a new virtio network device with the given TAP interface. 437 #[allow(clippy::too_many_arguments)] 438 pub fn new_with_tap( 439 id: String, 440 taps: Vec<Tap>, 441 guest_mac: Option<MacAddr>, 442 iommu: bool, 443 num_queues: usize, 444 queue_size: u16, 445 seccomp_action: SeccompAction, 446 rate_limiter_config: Option<RateLimiterConfig>, 447 exit_evt: EventFd, 448 state: Option<NetState>, 449 ) -> Result<Self> { 450 assert!(!taps.is_empty()); 451 452 let mtu = taps[0].mtu().map_err(Error::TapError)? as u16; 453 454 let (avail_features, acked_features, config, queue_sizes, paused) = 455 if let Some(state) = state { 456 info!("Restoring virtio-net {}", id); 457 ( 458 state.avail_features, 459 state.acked_features, 460 state.config, 461 state.queue_size, 462 true, 463 ) 464 } else { 465 let mut avail_features = 1 << VIRTIO_NET_F_CSUM 466 | 1 << VIRTIO_NET_F_CTRL_GUEST_OFFLOADS 467 | 1 << VIRTIO_NET_F_GUEST_CSUM 468 | 1 << VIRTIO_NET_F_GUEST_ECN 469 | 1 << VIRTIO_NET_F_GUEST_TSO4 470 | 1 << VIRTIO_NET_F_GUEST_TSO6 471 | 1 << VIRTIO_NET_F_GUEST_UFO 472 | 1 << VIRTIO_NET_F_HOST_ECN 473 | 1 << VIRTIO_NET_F_HOST_TSO4 474 | 1 << VIRTIO_NET_F_HOST_TSO6 475 | 1 << VIRTIO_NET_F_HOST_UFO 476 | 1 << VIRTIO_NET_F_MTU 477 | 1 << VIRTIO_RING_F_EVENT_IDX 478 | 1 << VIRTIO_F_VERSION_1; 479 480 if iommu { 481 avail_features |= 1u64 << VIRTIO_F_IOMMU_PLATFORM; 482 } 483 484 avail_features |= 1 << VIRTIO_NET_F_CTRL_VQ; 485 let queue_num = num_queues + 1; 486 487 let mut config = VirtioNetConfig::default(); 488 if let Some(mac) = guest_mac { 489 build_net_config_space( 490 &mut config, 491 mac, 492 num_queues, 493 Some(mtu), 494 &mut avail_features, 495 ); 496 } else { 497 build_net_config_space_with_mq( 498 &mut config, 499 num_queues, 500 Some(mtu), 501 &mut avail_features, 502 ); 503 } 504 505 ( 506 avail_features, 507 0, 508 config, 509 vec![queue_size; queue_num], 510 false, 511 ) 512 }; 513 514 Ok(Net { 515 common: VirtioCommon { 516 device_type: VirtioDeviceType::Net as u32, 517 avail_features, 518 acked_features, 519 queue_sizes, 520 paused_sync: Some(Arc::new(Barrier::new((num_queues / 2) + 1))), 521 min_queues: 2, 522 paused: Arc::new(AtomicBool::new(paused)), 523 ..Default::default() 524 }, 525 id, 526 taps, 527 config, 528 ctrl_queue_epoll_thread: None, 529 counters: NetCounters::default(), 530 seccomp_action, 531 rate_limiter_config, 532 exit_evt, 533 }) 534 } 535 536 /// Create a new virtio network device with the given IP address and 537 /// netmask. 538 #[allow(clippy::too_many_arguments)] 539 pub fn new( 540 id: String, 541 if_name: Option<&str>, 542 ip_addr: Option<Ipv4Addr>, 543 netmask: Option<Ipv4Addr>, 544 guest_mac: Option<MacAddr>, 545 host_mac: &mut Option<MacAddr>, 546 mtu: Option<u16>, 547 iommu: bool, 548 num_queues: usize, 549 queue_size: u16, 550 seccomp_action: SeccompAction, 551 rate_limiter_config: Option<RateLimiterConfig>, 552 exit_evt: EventFd, 553 state: Option<NetState>, 554 ) -> Result<Self> { 555 let taps = open_tap( 556 if_name, 557 ip_addr, 558 netmask, 559 host_mac, 560 mtu, 561 num_queues / 2, 562 None, 563 ) 564 .map_err(Error::OpenTap)?; 565 566 Self::new_with_tap( 567 id, 568 taps, 569 guest_mac, 570 iommu, 571 num_queues, 572 queue_size, 573 seccomp_action, 574 rate_limiter_config, 575 exit_evt, 576 state, 577 ) 578 } 579 580 #[allow(clippy::too_many_arguments)] 581 pub fn from_tap_fds( 582 id: String, 583 fds: &[RawFd], 584 guest_mac: Option<MacAddr>, 585 mtu: Option<u16>, 586 iommu: bool, 587 queue_size: u16, 588 seccomp_action: SeccompAction, 589 rate_limiter_config: Option<RateLimiterConfig>, 590 exit_evt: EventFd, 591 state: Option<NetState>, 592 ) -> Result<Self> { 593 let mut taps: Vec<Tap> = Vec::new(); 594 let num_queue_pairs = fds.len(); 595 596 for fd in fds.iter() { 597 // Duplicate so that it can survive reboots 598 // SAFETY: FFI call to dup. Trivially safe. 599 let fd = unsafe { libc::dup(*fd) }; 600 if fd < 0 { 601 return Err(Error::DuplicateTapFd(std::io::Error::last_os_error())); 602 } 603 let tap = Tap::from_tap_fd(fd, num_queue_pairs).map_err(Error::TapError)?; 604 taps.push(tap); 605 } 606 607 assert!(!taps.is_empty()); 608 609 if let Some(mtu) = mtu { 610 taps[0].set_mtu(mtu as i32).map_err(Error::TapError)?; 611 } 612 613 Self::new_with_tap( 614 id, 615 taps, 616 guest_mac, 617 iommu, 618 num_queue_pairs * 2, 619 queue_size, 620 seccomp_action, 621 rate_limiter_config, 622 exit_evt, 623 state, 624 ) 625 } 626 627 fn state(&self) -> NetState { 628 NetState { 629 avail_features: self.common.avail_features, 630 acked_features: self.common.acked_features, 631 config: self.config, 632 queue_size: self.common.queue_sizes.clone(), 633 } 634 } 635 636 #[cfg(fuzzing)] 637 pub fn wait_for_epoll_threads(&mut self) { 638 self.common.wait_for_epoll_threads(); 639 } 640 } 641 642 impl Drop for Net { 643 fn drop(&mut self) { 644 if let Some(kill_evt) = self.common.kill_evt.take() { 645 // Ignore the result because there is nothing we can do about it. 646 let _ = kill_evt.write(1); 647 } 648 // Needed to ensure all references to tap FDs are dropped (#4868) 649 self.common.wait_for_epoll_threads(); 650 } 651 } 652 653 impl VirtioDevice for Net { 654 fn device_type(&self) -> u32 { 655 self.common.device_type 656 } 657 658 fn queue_max_sizes(&self) -> &[u16] { 659 &self.common.queue_sizes 660 } 661 662 fn features(&self) -> u64 { 663 self.common.avail_features 664 } 665 666 fn ack_features(&mut self, value: u64) { 667 self.common.ack_features(value) 668 } 669 670 fn read_config(&self, offset: u64, data: &mut [u8]) { 671 self.read_config_from_slice(self.config.as_slice(), offset, data); 672 } 673 674 fn activate( 675 &mut self, 676 mem: GuestMemoryAtomic<GuestMemoryMmap>, 677 interrupt_cb: Arc<dyn VirtioInterrupt>, 678 mut queues: Vec<(usize, Queue, EventFd)>, 679 ) -> ActivateResult { 680 self.common.activate(&queues, &interrupt_cb)?; 681 682 let num_queues = queues.len(); 683 let event_idx = self.common.feature_acked(VIRTIO_RING_F_EVENT_IDX.into()); 684 if self.common.feature_acked(VIRTIO_NET_F_CTRL_VQ.into()) && num_queues % 2 != 0 { 685 let ctrl_queue_index = num_queues - 1; 686 let (_, mut ctrl_queue, ctrl_queue_evt) = queues.remove(ctrl_queue_index); 687 688 ctrl_queue.set_event_idx(event_idx); 689 690 let (kill_evt, pause_evt) = self.common.dup_eventfds(); 691 let mut ctrl_handler = NetCtrlEpollHandler { 692 mem: mem.clone(), 693 kill_evt, 694 pause_evt, 695 ctrl_q: CtrlQueue::new(self.taps.clone()), 696 queue: ctrl_queue, 697 queue_evt: ctrl_queue_evt, 698 access_platform: self.common.access_platform.clone(), 699 queue_index: ctrl_queue_index as u16, 700 interrupt_cb: interrupt_cb.clone(), 701 }; 702 703 let paused = self.common.paused.clone(); 704 // Let's update the barrier as we need 1 for each RX/TX pair + 705 // 1 for the control queue + 1 for the main thread signalling 706 // the pause. 707 self.common.paused_sync = Some(Arc::new(Barrier::new(self.taps.len() + 2))); 708 let paused_sync = self.common.paused_sync.clone(); 709 710 let mut epoll_threads = Vec::new(); 711 spawn_virtio_thread( 712 &format!("{}_ctrl", &self.id), 713 &self.seccomp_action, 714 Thread::VirtioNetCtl, 715 &mut epoll_threads, 716 &self.exit_evt, 717 move || ctrl_handler.run_ctrl(paused, paused_sync.unwrap()), 718 )?; 719 self.ctrl_queue_epoll_thread = Some(epoll_threads.remove(0)); 720 } 721 722 let mut epoll_threads = Vec::new(); 723 let mut taps = self.taps.clone(); 724 for i in 0..queues.len() / 2 { 725 let rx = RxVirtio::new(); 726 let tx = TxVirtio::new(); 727 let rx_tap_listening = false; 728 729 let (_, queue_0, queue_evt_0) = queues.remove(0); 730 let (_, queue_1, queue_evt_1) = queues.remove(0); 731 let mut queue_pair = (queue_0, queue_1); 732 queue_pair.0.set_event_idx(event_idx); 733 queue_pair.1.set_event_idx(event_idx); 734 735 let queue_evt_pair = (queue_evt_0, queue_evt_1); 736 737 let (kill_evt, pause_evt) = self.common.dup_eventfds(); 738 739 let rx_rate_limiter: Option<rate_limiter::RateLimiter> = self 740 .rate_limiter_config 741 .map(RateLimiterConfig::try_into) 742 .transpose() 743 .map_err(ActivateError::CreateRateLimiter)?; 744 745 let tx_rate_limiter: Option<rate_limiter::RateLimiter> = self 746 .rate_limiter_config 747 .map(RateLimiterConfig::try_into) 748 .transpose() 749 .map_err(ActivateError::CreateRateLimiter)?; 750 751 let tap = taps.remove(0); 752 #[cfg(not(fuzzing))] 753 tap.set_offload(virtio_features_to_tap_offload(self.common.acked_features)) 754 .map_err(|e| { 755 error!("Error programming tap offload: {:?}", e); 756 ActivateError::BadActivate 757 })?; 758 759 let mut handler = NetEpollHandler { 760 net: NetQueuePair { 761 tap_for_write_epoll: tap.clone(), 762 tap, 763 rx, 764 tx, 765 epoll_fd: None, 766 rx_tap_listening, 767 tx_tap_listening: false, 768 counters: self.counters.clone(), 769 tap_rx_event_id: RX_TAP_EVENT, 770 tap_tx_event_id: TX_TAP_EVENT, 771 rx_desc_avail: false, 772 rx_rate_limiter, 773 tx_rate_limiter, 774 access_platform: self.common.access_platform.clone(), 775 }, 776 mem: mem.clone(), 777 queue_index_base: (i * 2) as u16, 778 queue_pair, 779 queue_evt_pair, 780 interrupt_cb: interrupt_cb.clone(), 781 kill_evt, 782 pause_evt, 783 driver_awake: false, 784 }; 785 786 let paused = self.common.paused.clone(); 787 let paused_sync = self.common.paused_sync.clone(); 788 789 spawn_virtio_thread( 790 &format!("{}_qp{}", self.id.clone(), i), 791 &self.seccomp_action, 792 Thread::VirtioNet, 793 &mut epoll_threads, 794 &self.exit_evt, 795 move || handler.run(paused, paused_sync.unwrap()), 796 )?; 797 } 798 799 self.common.epoll_threads = Some(epoll_threads); 800 801 event!("virtio-device", "activated", "id", &self.id); 802 Ok(()) 803 } 804 805 fn reset(&mut self) -> Option<Arc<dyn VirtioInterrupt>> { 806 let result = self.common.reset(); 807 event!("virtio-device", "reset", "id", &self.id); 808 result 809 } 810 811 fn counters(&self) -> Option<HashMap<&'static str, Wrapping<u64>>> { 812 let mut counters = HashMap::new(); 813 814 counters.insert( 815 "rx_bytes", 816 Wrapping(self.counters.rx_bytes.load(Ordering::Acquire)), 817 ); 818 counters.insert( 819 "rx_frames", 820 Wrapping(self.counters.rx_frames.load(Ordering::Acquire)), 821 ); 822 counters.insert( 823 "tx_bytes", 824 Wrapping(self.counters.tx_bytes.load(Ordering::Acquire)), 825 ); 826 counters.insert( 827 "tx_frames", 828 Wrapping(self.counters.tx_frames.load(Ordering::Acquire)), 829 ); 830 831 Some(counters) 832 } 833 834 fn set_access_platform(&mut self, access_platform: Arc<dyn AccessPlatform>) { 835 self.common.set_access_platform(access_platform) 836 } 837 } 838 839 impl Pausable for Net { 840 fn pause(&mut self) -> result::Result<(), MigratableError> { 841 self.common.pause() 842 } 843 844 fn resume(&mut self) -> result::Result<(), MigratableError> { 845 self.common.resume()?; 846 847 if let Some(ctrl_queue_epoll_thread) = &self.ctrl_queue_epoll_thread { 848 ctrl_queue_epoll_thread.thread().unpark(); 849 } 850 Ok(()) 851 } 852 } 853 854 impl Snapshottable for Net { 855 fn id(&self) -> String { 856 self.id.clone() 857 } 858 859 fn snapshot(&mut self) -> std::result::Result<Snapshot, MigratableError> { 860 Snapshot::new_from_versioned_state(&self.state()) 861 } 862 } 863 impl Transportable for Net {} 864 impl Migratable for Net {} 865