1 // Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 // SPDX-License-Identifier: Apache-2.0 3 // 4 // Portions Copyright 2017 The Chromium OS Authors. All rights reserved. 5 // Use of this source code is governed by a BSD-style license that can be 6 // found in the THIRD-PARTY file. 7 8 use std::collections::HashMap; 9 use std::net::Ipv4Addr; 10 use std::num::Wrapping; 11 use std::ops::Deref; 12 use std::os::unix::io::{AsRawFd, RawFd}; 13 use std::sync::atomic::{AtomicBool, Ordering}; 14 use std::sync::{Arc, Barrier}; 15 use std::{result, thread}; 16 17 use anyhow::anyhow; 18 #[cfg(not(fuzzing))] 19 use net_util::virtio_features_to_tap_offload; 20 use net_util::{ 21 build_net_config_space, build_net_config_space_with_mq, open_tap, CtrlQueue, MacAddr, 22 NetCounters, NetQueuePair, OpenTapError, RxVirtio, Tap, TapError, TxVirtio, VirtioNetConfig, 23 }; 24 use seccompiler::SeccompAction; 25 use serde::{Deserialize, Serialize}; 26 use thiserror::Error; 27 use virtio_bindings::virtio_config::*; 28 use virtio_bindings::virtio_net::*; 29 use virtio_bindings::virtio_ring::VIRTIO_RING_F_EVENT_IDX; 30 use virtio_queue::{Queue, QueueT}; 31 use vm_memory::{ByteValued, GuestAddressSpace, GuestMemoryAtomic}; 32 use vm_migration::{Migratable, MigratableError, Pausable, Snapshot, Snapshottable, Transportable}; 33 use vm_virtio::AccessPlatform; 34 use vmm_sys_util::eventfd::EventFd; 35 36 use super::{ 37 ActivateError, ActivateResult, EpollHelper, EpollHelperError, EpollHelperHandler, 38 Error as DeviceError, RateLimiterConfig, VirtioCommon, VirtioDevice, VirtioDeviceType, 39 VirtioInterruptType, EPOLL_HELPER_EVENT_LAST, 40 }; 41 use crate::seccomp_filters::Thread; 42 use crate::thread_helper::spawn_virtio_thread; 43 use crate::{GuestMemoryMmap, VirtioInterrupt}; 44 45 /// Control queue 46 // Event available on the control queue. 47 const CTRL_QUEUE_EVENT: u16 = EPOLL_HELPER_EVENT_LAST + 1; 48 49 // Following the VIRTIO specification, the MTU should be at least 1280. 50 pub const MIN_MTU: u16 = 1280; 51 52 pub struct NetCtrlEpollHandler { 53 pub mem: GuestMemoryAtomic<GuestMemoryMmap>, 54 pub kill_evt: EventFd, 55 pub pause_evt: EventFd, 56 pub ctrl_q: CtrlQueue, 57 pub queue_evt: EventFd, 58 pub queue: Queue, 59 pub access_platform: Option<Arc<dyn AccessPlatform>>, 60 pub interrupt_cb: Arc<dyn VirtioInterrupt>, 61 pub queue_index: u16, 62 } 63 64 impl NetCtrlEpollHandler { 65 fn signal_used_queue(&self, queue_index: u16) -> result::Result<(), DeviceError> { 66 self.interrupt_cb 67 .trigger(VirtioInterruptType::Queue(queue_index)) 68 .map_err(|e| { 69 error!("Failed to signal used queue: {:?}", e); 70 DeviceError::FailedSignalingUsedQueue(e) 71 }) 72 } 73 74 pub fn run_ctrl( 75 &mut self, 76 paused: Arc<AtomicBool>, 77 paused_sync: Arc<Barrier>, 78 ) -> std::result::Result<(), EpollHelperError> { 79 let mut helper = EpollHelper::new(&self.kill_evt, &self.pause_evt)?; 80 helper.add_event(self.queue_evt.as_raw_fd(), CTRL_QUEUE_EVENT)?; 81 helper.run(paused, paused_sync, self)?; 82 83 Ok(()) 84 } 85 } 86 87 impl EpollHelperHandler for NetCtrlEpollHandler { 88 fn handle_event( 89 &mut self, 90 _helper: &mut EpollHelper, 91 event: &epoll::Event, 92 ) -> result::Result<(), EpollHelperError> { 93 let ev_type = event.data as u16; 94 match ev_type { 95 CTRL_QUEUE_EVENT => { 96 let mem = self.mem.memory(); 97 self.queue_evt.read().map_err(|e| { 98 EpollHelperError::HandleEvent(anyhow!( 99 "Failed to get control queue event: {:?}", 100 e 101 )) 102 })?; 103 self.ctrl_q 104 .process(mem.deref(), &mut self.queue, self.access_platform.as_ref()) 105 .map_err(|e| { 106 EpollHelperError::HandleEvent(anyhow!( 107 "Failed to process control queue: {:?}", 108 e 109 )) 110 })?; 111 match self.queue.needs_notification(mem.deref()) { 112 Ok(true) => { 113 self.signal_used_queue(self.queue_index).map_err(|e| { 114 EpollHelperError::HandleEvent(anyhow!( 115 "Error signalling that control queue was used: {:?}", 116 e 117 )) 118 })?; 119 } 120 Ok(false) => {} 121 Err(e) => { 122 return Err(EpollHelperError::HandleEvent(anyhow!( 123 "Error getting notification state of control queue: {}", 124 e 125 ))); 126 } 127 }; 128 } 129 _ => { 130 return Err(EpollHelperError::HandleEvent(anyhow!( 131 "Unknown event for virtio-net control queue" 132 ))); 133 } 134 } 135 136 Ok(()) 137 } 138 } 139 140 /// Rx/Tx queue pair 141 // The guest has made a buffer available to receive a frame into. 142 pub const RX_QUEUE_EVENT: u16 = EPOLL_HELPER_EVENT_LAST + 1; 143 // The transmit queue has a frame that is ready to send from the guest. 144 pub const TX_QUEUE_EVENT: u16 = EPOLL_HELPER_EVENT_LAST + 2; 145 // A frame is available for reading from the tap device to receive in the guest. 146 pub const RX_TAP_EVENT: u16 = EPOLL_HELPER_EVENT_LAST + 3; 147 // The TAP can be written to. Used after an EAGAIN error to retry TX. 148 pub const TX_TAP_EVENT: u16 = EPOLL_HELPER_EVENT_LAST + 4; 149 // New 'wake up' event from the rx rate limiter 150 pub const RX_RATE_LIMITER_EVENT: u16 = EPOLL_HELPER_EVENT_LAST + 5; 151 // New 'wake up' event from the tx rate limiter 152 pub const TX_RATE_LIMITER_EVENT: u16 = EPOLL_HELPER_EVENT_LAST + 6; 153 154 #[derive(Error, Debug)] 155 pub enum Error { 156 #[error("Failed to open taps: {0}")] 157 OpenTap(OpenTapError), 158 #[error("Using existing tap: {0}")] 159 TapError(TapError), 160 #[error("Error calling dup() on tap fd: {0}")] 161 DuplicateTapFd(std::io::Error), 162 } 163 164 pub type Result<T> = result::Result<T, Error>; 165 166 struct NetEpollHandler { 167 net: NetQueuePair, 168 mem: GuestMemoryAtomic<GuestMemoryMmap>, 169 interrupt_cb: Arc<dyn VirtioInterrupt>, 170 kill_evt: EventFd, 171 pause_evt: EventFd, 172 queue_index_base: u16, 173 queue_pair: (Queue, Queue), 174 queue_evt_pair: (EventFd, EventFd), 175 // Always generate interrupts until the driver has signalled to the device. 176 // This mitigates a problem with interrupts from tap events being "lost" upon 177 // a restore as the vCPU thread isn't ready to handle the interrupt. This causes 178 // issues when combined with VIRTIO_RING_F_EVENT_IDX interrupt suppression. 179 driver_awake: bool, 180 } 181 182 impl NetEpollHandler { 183 fn signal_used_queue(&self, queue_index: u16) -> result::Result<(), DeviceError> { 184 self.interrupt_cb 185 .trigger(VirtioInterruptType::Queue(queue_index)) 186 .map_err(|e| { 187 error!("Failed to signal used queue: {:?}", e); 188 DeviceError::FailedSignalingUsedQueue(e) 189 }) 190 } 191 192 fn handle_rx_event(&mut self) -> result::Result<(), DeviceError> { 193 let queue_evt = &self.queue_evt_pair.0; 194 if let Err(e) = queue_evt.read() { 195 error!("Failed to get rx queue event: {:?}", e); 196 } 197 198 self.net.rx_desc_avail = true; 199 200 let rate_limit_reached = self 201 .net 202 .rx_rate_limiter 203 .as_ref() 204 .is_some_and(|r| r.is_blocked()); 205 206 // Start to listen on RX_TAP_EVENT only when the rate limit is not reached 207 if !self.net.rx_tap_listening && !rate_limit_reached { 208 net_util::register_listener( 209 self.net.epoll_fd.unwrap(), 210 self.net.tap.as_raw_fd(), 211 epoll::Events::EPOLLIN, 212 u64::from(self.net.tap_rx_event_id), 213 ) 214 .map_err(DeviceError::IoError)?; 215 self.net.rx_tap_listening = true; 216 } 217 218 Ok(()) 219 } 220 221 fn process_tx(&mut self) -> result::Result<(), DeviceError> { 222 if self 223 .net 224 .process_tx(&self.mem.memory(), &mut self.queue_pair.1) 225 .map_err(DeviceError::NetQueuePair)? 226 || !self.driver_awake 227 { 228 self.signal_used_queue(self.queue_index_base + 1)?; 229 debug!("Signalling TX queue"); 230 } else { 231 debug!("Not signalling TX queue"); 232 } 233 Ok(()) 234 } 235 236 fn handle_tx_event(&mut self) -> result::Result<(), DeviceError> { 237 let rate_limit_reached = self 238 .net 239 .tx_rate_limiter 240 .as_ref() 241 .is_some_and(|r| r.is_blocked()); 242 243 if !rate_limit_reached { 244 self.process_tx()?; 245 } 246 247 Ok(()) 248 } 249 250 fn handle_rx_tap_event(&mut self) -> result::Result<(), DeviceError> { 251 if self 252 .net 253 .process_rx(&self.mem.memory(), &mut self.queue_pair.0) 254 .map_err(DeviceError::NetQueuePair)? 255 || !self.driver_awake 256 { 257 self.signal_used_queue(self.queue_index_base)?; 258 debug!("Signalling RX queue"); 259 } else { 260 debug!("Not signalling RX queue"); 261 } 262 Ok(()) 263 } 264 265 fn run( 266 &mut self, 267 paused: Arc<AtomicBool>, 268 paused_sync: Arc<Barrier>, 269 ) -> result::Result<(), EpollHelperError> { 270 let mut helper = EpollHelper::new(&self.kill_evt, &self.pause_evt)?; 271 helper.add_event(self.queue_evt_pair.0.as_raw_fd(), RX_QUEUE_EVENT)?; 272 helper.add_event(self.queue_evt_pair.1.as_raw_fd(), TX_QUEUE_EVENT)?; 273 if let Some(rate_limiter) = &self.net.rx_rate_limiter { 274 helper.add_event(rate_limiter.as_raw_fd(), RX_RATE_LIMITER_EVENT)?; 275 } 276 if let Some(rate_limiter) = &self.net.tx_rate_limiter { 277 helper.add_event(rate_limiter.as_raw_fd(), TX_RATE_LIMITER_EVENT)?; 278 } 279 280 let mem = self.mem.memory(); 281 // If there are some already available descriptors on the RX queue, 282 // then we can start the thread while listening onto the TAP. 283 if self 284 .queue_pair 285 .0 286 .used_idx(mem.deref(), Ordering::Acquire) 287 .map_err(EpollHelperError::QueueRingIndex)? 288 < self 289 .queue_pair 290 .0 291 .avail_idx(mem.deref(), Ordering::Acquire) 292 .map_err(EpollHelperError::QueueRingIndex)? 293 { 294 helper.add_event(self.net.tap.as_raw_fd(), RX_TAP_EVENT)?; 295 self.net.rx_tap_listening = true; 296 info!("Listener registered at start"); 297 } 298 299 // The NetQueuePair needs the epoll fd. 300 self.net.epoll_fd = Some(helper.as_raw_fd()); 301 302 helper.run(paused, paused_sync, self)?; 303 304 Ok(()) 305 } 306 } 307 308 impl EpollHelperHandler for NetEpollHandler { 309 fn handle_event( 310 &mut self, 311 _helper: &mut EpollHelper, 312 event: &epoll::Event, 313 ) -> result::Result<(), EpollHelperError> { 314 let ev_type = event.data as u16; 315 match ev_type { 316 RX_QUEUE_EVENT => { 317 self.driver_awake = true; 318 self.handle_rx_event().map_err(|e| { 319 EpollHelperError::HandleEvent(anyhow!("Error processing RX queue: {:?}", e)) 320 })?; 321 } 322 TX_QUEUE_EVENT => { 323 let queue_evt = &self.queue_evt_pair.1; 324 if let Err(e) = queue_evt.read() { 325 error!("Failed to get tx queue event: {:?}", e); 326 } 327 self.driver_awake = true; 328 self.handle_tx_event().map_err(|e| { 329 EpollHelperError::HandleEvent(anyhow!("Error processing TX queue: {:?}", e)) 330 })?; 331 } 332 TX_TAP_EVENT => { 333 self.handle_tx_event().map_err(|e| { 334 EpollHelperError::HandleEvent(anyhow!( 335 "Error processing TX queue (TAP event): {:?}", 336 e 337 )) 338 })?; 339 } 340 RX_TAP_EVENT => { 341 self.handle_rx_tap_event().map_err(|e| { 342 EpollHelperError::HandleEvent(anyhow!("Error processing tap queue: {:?}", e)) 343 })?; 344 } 345 RX_RATE_LIMITER_EVENT => { 346 if let Some(rate_limiter) = &mut self.net.rx_rate_limiter { 347 // Upon rate limiter event, call the rate limiter handler and register the 348 // TAP fd for further processing if some RX buffers are available 349 rate_limiter.event_handler().map_err(|e| { 350 EpollHelperError::HandleEvent(anyhow!( 351 "Error from 'rate_limiter.event_handler()': {:?}", 352 e 353 )) 354 })?; 355 356 if !self.net.rx_tap_listening && self.net.rx_desc_avail { 357 net_util::register_listener( 358 self.net.epoll_fd.unwrap(), 359 self.net.tap.as_raw_fd(), 360 epoll::Events::EPOLLIN, 361 u64::from(self.net.tap_rx_event_id), 362 ) 363 .map_err(|e| { 364 EpollHelperError::HandleEvent(anyhow!( 365 "Error register_listener with `RX_RATE_LIMITER_EVENT`: {:?}", 366 e 367 )) 368 })?; 369 370 self.net.rx_tap_listening = true; 371 } 372 } else { 373 return Err(EpollHelperError::HandleEvent(anyhow!( 374 "Unexpected RX_RATE_LIMITER_EVENT" 375 ))); 376 } 377 } 378 TX_RATE_LIMITER_EVENT => { 379 if let Some(rate_limiter) = &mut self.net.tx_rate_limiter { 380 // Upon rate limiter event, call the rate limiter handler 381 // and restart processing the queue. 382 rate_limiter.event_handler().map_err(|e| { 383 EpollHelperError::HandleEvent(anyhow!( 384 "Error from 'rate_limiter.event_handler()': {:?}", 385 e 386 )) 387 })?; 388 389 self.driver_awake = true; 390 self.process_tx().map_err(|e| { 391 EpollHelperError::HandleEvent(anyhow!("Error processing TX queue: {:?}", e)) 392 })?; 393 } else { 394 return Err(EpollHelperError::HandleEvent(anyhow!( 395 "Unexpected TX_RATE_LIMITER_EVENT" 396 ))); 397 } 398 } 399 _ => { 400 return Err(EpollHelperError::HandleEvent(anyhow!( 401 "Unexpected event: {}", 402 ev_type 403 ))); 404 } 405 } 406 Ok(()) 407 } 408 } 409 410 pub struct Net { 411 common: VirtioCommon, 412 id: String, 413 taps: Vec<Tap>, 414 config: VirtioNetConfig, 415 ctrl_queue_epoll_thread: Option<thread::JoinHandle<()>>, 416 counters: NetCounters, 417 seccomp_action: SeccompAction, 418 rate_limiter_config: Option<RateLimiterConfig>, 419 exit_evt: EventFd, 420 } 421 422 #[derive(Serialize, Deserialize)] 423 pub struct NetState { 424 pub avail_features: u64, 425 pub acked_features: u64, 426 pub config: VirtioNetConfig, 427 pub queue_size: Vec<u16>, 428 } 429 430 impl Net { 431 /// Create a new virtio network device with the given TAP interface. 432 #[allow(clippy::too_many_arguments)] 433 pub fn new_with_tap( 434 id: String, 435 taps: Vec<Tap>, 436 guest_mac: Option<MacAddr>, 437 iommu: bool, 438 num_queues: usize, 439 queue_size: u16, 440 seccomp_action: SeccompAction, 441 rate_limiter_config: Option<RateLimiterConfig>, 442 exit_evt: EventFd, 443 state: Option<NetState>, 444 offload_tso: bool, 445 offload_ufo: bool, 446 offload_csum: bool, 447 ) -> Result<Self> { 448 assert!(!taps.is_empty()); 449 450 let mtu = taps[0].mtu().map_err(Error::TapError)? as u16; 451 452 let (avail_features, acked_features, config, queue_sizes, paused) = 453 if let Some(state) = state { 454 info!("Restoring virtio-net {}", id); 455 ( 456 state.avail_features, 457 state.acked_features, 458 state.config, 459 state.queue_size, 460 true, 461 ) 462 } else { 463 let mut avail_features = 464 1 << VIRTIO_NET_F_MTU | 1 << VIRTIO_RING_F_EVENT_IDX | 1 << VIRTIO_F_VERSION_1; 465 466 if iommu { 467 avail_features |= 1u64 << VIRTIO_F_IOMMU_PLATFORM; 468 } 469 470 // Configure TSO/UFO features when hardware checksum offload is enabled. 471 if offload_csum { 472 avail_features |= 1 << VIRTIO_NET_F_CSUM 473 | 1 << VIRTIO_NET_F_GUEST_CSUM 474 | 1 << VIRTIO_NET_F_CTRL_GUEST_OFFLOADS; 475 476 if offload_tso { 477 avail_features |= 1 << VIRTIO_NET_F_HOST_ECN 478 | 1 << VIRTIO_NET_F_HOST_TSO4 479 | 1 << VIRTIO_NET_F_HOST_TSO6 480 | 1 << VIRTIO_NET_F_GUEST_ECN 481 | 1 << VIRTIO_NET_F_GUEST_TSO4 482 | 1 << VIRTIO_NET_F_GUEST_TSO6; 483 } 484 485 if offload_ufo { 486 avail_features |= 1 << VIRTIO_NET_F_HOST_UFO | 1 << VIRTIO_NET_F_GUEST_UFO; 487 } 488 } 489 490 avail_features |= 1 << VIRTIO_NET_F_CTRL_VQ; 491 let queue_num = num_queues + 1; 492 493 let mut config = VirtioNetConfig::default(); 494 if let Some(mac) = guest_mac { 495 build_net_config_space( 496 &mut config, 497 mac, 498 num_queues, 499 Some(mtu), 500 &mut avail_features, 501 ); 502 } else { 503 build_net_config_space_with_mq( 504 &mut config, 505 num_queues, 506 Some(mtu), 507 &mut avail_features, 508 ); 509 } 510 511 ( 512 avail_features, 513 0, 514 config, 515 vec![queue_size; queue_num], 516 false, 517 ) 518 }; 519 520 Ok(Net { 521 common: VirtioCommon { 522 device_type: VirtioDeviceType::Net as u32, 523 avail_features, 524 acked_features, 525 queue_sizes, 526 paused_sync: Some(Arc::new(Barrier::new((num_queues / 2) + 1))), 527 min_queues: 2, 528 paused: Arc::new(AtomicBool::new(paused)), 529 ..Default::default() 530 }, 531 id, 532 taps, 533 config, 534 ctrl_queue_epoll_thread: None, 535 counters: NetCounters::default(), 536 seccomp_action, 537 rate_limiter_config, 538 exit_evt, 539 }) 540 } 541 542 /// Create a new virtio network device with the given IP address and 543 /// netmask. 544 #[allow(clippy::too_many_arguments)] 545 pub fn new( 546 id: String, 547 if_name: Option<&str>, 548 ip_addr: Option<Ipv4Addr>, 549 netmask: Option<Ipv4Addr>, 550 guest_mac: Option<MacAddr>, 551 host_mac: &mut Option<MacAddr>, 552 mtu: Option<u16>, 553 iommu: bool, 554 num_queues: usize, 555 queue_size: u16, 556 seccomp_action: SeccompAction, 557 rate_limiter_config: Option<RateLimiterConfig>, 558 exit_evt: EventFd, 559 state: Option<NetState>, 560 offload_tso: bool, 561 offload_ufo: bool, 562 offload_csum: bool, 563 ) -> Result<Self> { 564 let taps = open_tap( 565 if_name, 566 ip_addr, 567 netmask, 568 host_mac, 569 mtu, 570 num_queues / 2, 571 None, 572 ) 573 .map_err(Error::OpenTap)?; 574 575 Self::new_with_tap( 576 id, 577 taps, 578 guest_mac, 579 iommu, 580 num_queues, 581 queue_size, 582 seccomp_action, 583 rate_limiter_config, 584 exit_evt, 585 state, 586 offload_tso, 587 offload_ufo, 588 offload_csum, 589 ) 590 } 591 592 #[allow(clippy::too_many_arguments)] 593 pub fn from_tap_fds( 594 id: String, 595 fds: &[RawFd], 596 guest_mac: Option<MacAddr>, 597 mtu: Option<u16>, 598 iommu: bool, 599 queue_size: u16, 600 seccomp_action: SeccompAction, 601 rate_limiter_config: Option<RateLimiterConfig>, 602 exit_evt: EventFd, 603 state: Option<NetState>, 604 offload_tso: bool, 605 offload_ufo: bool, 606 offload_csum: bool, 607 ) -> Result<Self> { 608 let mut taps: Vec<Tap> = Vec::new(); 609 let num_queue_pairs = fds.len(); 610 611 for fd in fds.iter() { 612 // Duplicate so that it can survive reboots 613 // SAFETY: FFI call to dup. Trivially safe. 614 let fd = unsafe { libc::dup(*fd) }; 615 if fd < 0 { 616 return Err(Error::DuplicateTapFd(std::io::Error::last_os_error())); 617 } 618 let tap = Tap::from_tap_fd(fd, num_queue_pairs).map_err(Error::TapError)?; 619 taps.push(tap); 620 } 621 622 assert!(!taps.is_empty()); 623 624 if let Some(mtu) = mtu { 625 taps[0].set_mtu(mtu as i32).map_err(Error::TapError)?; 626 } 627 628 Self::new_with_tap( 629 id, 630 taps, 631 guest_mac, 632 iommu, 633 num_queue_pairs * 2, 634 queue_size, 635 seccomp_action, 636 rate_limiter_config, 637 exit_evt, 638 state, 639 offload_tso, 640 offload_ufo, 641 offload_csum, 642 ) 643 } 644 645 fn state(&self) -> NetState { 646 NetState { 647 avail_features: self.common.avail_features, 648 acked_features: self.common.acked_features, 649 config: self.config, 650 queue_size: self.common.queue_sizes.clone(), 651 } 652 } 653 654 #[cfg(fuzzing)] 655 pub fn wait_for_epoll_threads(&mut self) { 656 self.common.wait_for_epoll_threads(); 657 } 658 } 659 660 impl Drop for Net { 661 fn drop(&mut self) { 662 if let Some(kill_evt) = self.common.kill_evt.take() { 663 // Ignore the result because there is nothing we can do about it. 664 let _ = kill_evt.write(1); 665 } 666 // Needed to ensure all references to tap FDs are dropped (#4868) 667 self.common.wait_for_epoll_threads(); 668 if let Some(thread) = self.ctrl_queue_epoll_thread.take() { 669 if let Err(e) = thread.join() { 670 error!("Error joining thread: {:?}", e); 671 } 672 } 673 } 674 } 675 676 impl VirtioDevice for Net { 677 fn device_type(&self) -> u32 { 678 self.common.device_type 679 } 680 681 fn queue_max_sizes(&self) -> &[u16] { 682 &self.common.queue_sizes 683 } 684 685 fn features(&self) -> u64 { 686 self.common.avail_features 687 } 688 689 fn ack_features(&mut self, value: u64) { 690 self.common.ack_features(value) 691 } 692 693 fn read_config(&self, offset: u64, data: &mut [u8]) { 694 self.read_config_from_slice(self.config.as_slice(), offset, data); 695 } 696 697 fn activate( 698 &mut self, 699 mem: GuestMemoryAtomic<GuestMemoryMmap>, 700 interrupt_cb: Arc<dyn VirtioInterrupt>, 701 mut queues: Vec<(usize, Queue, EventFd)>, 702 ) -> ActivateResult { 703 self.common.activate(&queues, &interrupt_cb)?; 704 705 let num_queues = queues.len(); 706 let event_idx = self.common.feature_acked(VIRTIO_RING_F_EVENT_IDX.into()); 707 if self.common.feature_acked(VIRTIO_NET_F_CTRL_VQ.into()) && num_queues % 2 != 0 { 708 let ctrl_queue_index = num_queues - 1; 709 let (_, mut ctrl_queue, ctrl_queue_evt) = queues.remove(ctrl_queue_index); 710 711 ctrl_queue.set_event_idx(event_idx); 712 713 let (kill_evt, pause_evt) = self.common.dup_eventfds(); 714 let mut ctrl_handler = NetCtrlEpollHandler { 715 mem: mem.clone(), 716 kill_evt, 717 pause_evt, 718 ctrl_q: CtrlQueue::new(self.taps.clone()), 719 queue: ctrl_queue, 720 queue_evt: ctrl_queue_evt, 721 access_platform: self.common.access_platform.clone(), 722 queue_index: ctrl_queue_index as u16, 723 interrupt_cb: interrupt_cb.clone(), 724 }; 725 726 let paused = self.common.paused.clone(); 727 // Let's update the barrier as we need 1 for each RX/TX pair + 728 // 1 for the control queue + 1 for the main thread signalling 729 // the pause. 730 self.common.paused_sync = Some(Arc::new(Barrier::new(self.taps.len() + 2))); 731 let paused_sync = self.common.paused_sync.clone(); 732 733 let mut epoll_threads = Vec::new(); 734 spawn_virtio_thread( 735 &format!("{}_ctrl", &self.id), 736 &self.seccomp_action, 737 Thread::VirtioNetCtl, 738 &mut epoll_threads, 739 &self.exit_evt, 740 move || ctrl_handler.run_ctrl(paused, paused_sync.unwrap()), 741 )?; 742 self.ctrl_queue_epoll_thread = Some(epoll_threads.remove(0)); 743 } 744 745 let mut epoll_threads = Vec::new(); 746 let mut taps = self.taps.clone(); 747 for i in 0..queues.len() / 2 { 748 let rx = RxVirtio::new(); 749 let tx = TxVirtio::new(); 750 let rx_tap_listening = false; 751 752 let (_, queue_0, queue_evt_0) = queues.remove(0); 753 let (_, queue_1, queue_evt_1) = queues.remove(0); 754 let mut queue_pair = (queue_0, queue_1); 755 queue_pair.0.set_event_idx(event_idx); 756 queue_pair.1.set_event_idx(event_idx); 757 758 let queue_evt_pair = (queue_evt_0, queue_evt_1); 759 760 let (kill_evt, pause_evt) = self.common.dup_eventfds(); 761 762 let rx_rate_limiter: Option<rate_limiter::RateLimiter> = self 763 .rate_limiter_config 764 .map(RateLimiterConfig::try_into) 765 .transpose() 766 .map_err(ActivateError::CreateRateLimiter)?; 767 768 let tx_rate_limiter: Option<rate_limiter::RateLimiter> = self 769 .rate_limiter_config 770 .map(RateLimiterConfig::try_into) 771 .transpose() 772 .map_err(ActivateError::CreateRateLimiter)?; 773 774 let tap = taps.remove(0); 775 #[cfg(not(fuzzing))] 776 tap.set_offload(virtio_features_to_tap_offload(self.common.acked_features)) 777 .map_err(|e| { 778 error!("Error programming tap offload: {:?}", e); 779 ActivateError::BadActivate 780 })?; 781 782 let mut handler = NetEpollHandler { 783 net: NetQueuePair { 784 tap_for_write_epoll: tap.clone(), 785 tap, 786 rx, 787 tx, 788 epoll_fd: None, 789 rx_tap_listening, 790 tx_tap_listening: false, 791 counters: self.counters.clone(), 792 tap_rx_event_id: RX_TAP_EVENT, 793 tap_tx_event_id: TX_TAP_EVENT, 794 rx_desc_avail: false, 795 rx_rate_limiter, 796 tx_rate_limiter, 797 access_platform: self.common.access_platform.clone(), 798 }, 799 mem: mem.clone(), 800 queue_index_base: (i * 2) as u16, 801 queue_pair, 802 queue_evt_pair, 803 interrupt_cb: interrupt_cb.clone(), 804 kill_evt, 805 pause_evt, 806 driver_awake: false, 807 }; 808 809 let paused = self.common.paused.clone(); 810 let paused_sync = self.common.paused_sync.clone(); 811 812 spawn_virtio_thread( 813 &format!("{}_qp{}", self.id.clone(), i), 814 &self.seccomp_action, 815 Thread::VirtioNet, 816 &mut epoll_threads, 817 &self.exit_evt, 818 move || handler.run(paused, paused_sync.unwrap()), 819 )?; 820 } 821 822 self.common.epoll_threads = Some(epoll_threads); 823 824 event!("virtio-device", "activated", "id", &self.id); 825 Ok(()) 826 } 827 828 fn reset(&mut self) -> Option<Arc<dyn VirtioInterrupt>> { 829 let result = self.common.reset(); 830 event!("virtio-device", "reset", "id", &self.id); 831 result 832 } 833 834 fn counters(&self) -> Option<HashMap<&'static str, Wrapping<u64>>> { 835 let mut counters = HashMap::new(); 836 837 counters.insert( 838 "rx_bytes", 839 Wrapping(self.counters.rx_bytes.load(Ordering::Acquire)), 840 ); 841 counters.insert( 842 "rx_frames", 843 Wrapping(self.counters.rx_frames.load(Ordering::Acquire)), 844 ); 845 counters.insert( 846 "tx_bytes", 847 Wrapping(self.counters.tx_bytes.load(Ordering::Acquire)), 848 ); 849 counters.insert( 850 "tx_frames", 851 Wrapping(self.counters.tx_frames.load(Ordering::Acquire)), 852 ); 853 854 Some(counters) 855 } 856 857 fn set_access_platform(&mut self, access_platform: Arc<dyn AccessPlatform>) { 858 self.common.set_access_platform(access_platform) 859 } 860 } 861 862 impl Pausable for Net { 863 fn pause(&mut self) -> result::Result<(), MigratableError> { 864 self.common.pause() 865 } 866 867 fn resume(&mut self) -> result::Result<(), MigratableError> { 868 self.common.resume()?; 869 870 if let Some(ctrl_queue_epoll_thread) = &self.ctrl_queue_epoll_thread { 871 ctrl_queue_epoll_thread.thread().unpark(); 872 } 873 Ok(()) 874 } 875 } 876 877 impl Snapshottable for Net { 878 fn id(&self) -> String { 879 self.id.clone() 880 } 881 882 fn snapshot(&mut self) -> std::result::Result<Snapshot, MigratableError> { 883 Snapshot::new_from_state(&self.state()) 884 } 885 } 886 impl Transportable for Net {} 887 impl Migratable for Net {} 888