1 // Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 // SPDX-License-Identifier: Apache-2.0 3 // 4 // Portions Copyright 2017 The Chromium OS Authors. All rights reserved. 5 // Use of this source code is governed by a BSD-style license that can be 6 // found in the THIRD-PARTY file. 7 8 use super::Error as DeviceError; 9 use super::{ 10 ActivateError, ActivateResult, EpollHelper, EpollHelperError, EpollHelperHandler, 11 RateLimiterConfig, VirtioCommon, VirtioDevice, VirtioDeviceType, VirtioInterruptType, 12 EPOLL_HELPER_EVENT_LAST, 13 }; 14 use crate::seccomp_filters::Thread; 15 use crate::thread_helper::spawn_virtio_thread; 16 use crate::GuestMemoryMmap; 17 use crate::VirtioInterrupt; 18 use anyhow::anyhow; 19 #[cfg(not(fuzzing))] 20 use net_util::virtio_features_to_tap_offload; 21 use net_util::CtrlQueue; 22 use net_util::{ 23 build_net_config_space, build_net_config_space_with_mq, open_tap, MacAddr, NetCounters, 24 NetQueuePair, OpenTapError, RxVirtio, Tap, TapError, TxVirtio, VirtioNetConfig, 25 }; 26 use seccompiler::SeccompAction; 27 use serde::{Deserialize, Serialize}; 28 use std::collections::HashMap; 29 use std::net::Ipv4Addr; 30 use std::num::Wrapping; 31 use std::ops::Deref; 32 use std::os::unix::io::{AsRawFd, RawFd}; 33 use std::result; 34 use std::sync::atomic::{AtomicBool, Ordering}; 35 use std::sync::{Arc, Barrier}; 36 use std::thread; 37 use thiserror::Error; 38 use virtio_bindings::virtio_config::*; 39 use virtio_bindings::virtio_net::*; 40 use virtio_bindings::virtio_ring::VIRTIO_RING_F_EVENT_IDX; 41 use virtio_queue::{Queue, QueueT}; 42 use vm_memory::{ByteValued, GuestAddressSpace, GuestMemoryAtomic}; 43 use vm_migration::{Migratable, MigratableError, Pausable, Snapshot, Snapshottable, Transportable}; 44 use vm_virtio::AccessPlatform; 45 use vmm_sys_util::eventfd::EventFd; 46 47 /// Control queue 48 // Event available on the control queue. 49 const CTRL_QUEUE_EVENT: u16 = EPOLL_HELPER_EVENT_LAST + 1; 50 51 // Following the VIRTIO specification, the MTU should be at least 1280. 52 pub const MIN_MTU: u16 = 1280; 53 54 pub struct NetCtrlEpollHandler { 55 pub mem: GuestMemoryAtomic<GuestMemoryMmap>, 56 pub kill_evt: EventFd, 57 pub pause_evt: EventFd, 58 pub ctrl_q: CtrlQueue, 59 pub queue_evt: EventFd, 60 pub queue: Queue, 61 pub access_platform: Option<Arc<dyn AccessPlatform>>, 62 pub interrupt_cb: Arc<dyn VirtioInterrupt>, 63 pub queue_index: u16, 64 } 65 66 impl NetCtrlEpollHandler { 67 fn signal_used_queue(&self, queue_index: u16) -> result::Result<(), DeviceError> { 68 self.interrupt_cb 69 .trigger(VirtioInterruptType::Queue(queue_index)) 70 .map_err(|e| { 71 error!("Failed to signal used queue: {:?}", e); 72 DeviceError::FailedSignalingUsedQueue(e) 73 }) 74 } 75 76 pub fn run_ctrl( 77 &mut self, 78 paused: Arc<AtomicBool>, 79 paused_sync: Arc<Barrier>, 80 ) -> std::result::Result<(), EpollHelperError> { 81 let mut helper = EpollHelper::new(&self.kill_evt, &self.pause_evt)?; 82 helper.add_event(self.queue_evt.as_raw_fd(), CTRL_QUEUE_EVENT)?; 83 helper.run(paused, paused_sync, self)?; 84 85 Ok(()) 86 } 87 } 88 89 impl EpollHelperHandler for NetCtrlEpollHandler { 90 fn handle_event( 91 &mut self, 92 _helper: &mut EpollHelper, 93 event: &epoll::Event, 94 ) -> result::Result<(), EpollHelperError> { 95 let ev_type = event.data as u16; 96 match ev_type { 97 CTRL_QUEUE_EVENT => { 98 let mem = self.mem.memory(); 99 self.queue_evt.read().map_err(|e| { 100 EpollHelperError::HandleEvent(anyhow!( 101 "Failed to get control queue event: {:?}", 102 e 103 )) 104 })?; 105 self.ctrl_q 106 .process(mem.deref(), &mut self.queue, self.access_platform.as_ref()) 107 .map_err(|e| { 108 EpollHelperError::HandleEvent(anyhow!( 109 "Failed to process control queue: {:?}", 110 e 111 )) 112 })?; 113 match self.queue.needs_notification(mem.deref()) { 114 Ok(true) => { 115 self.signal_used_queue(self.queue_index).map_err(|e| { 116 EpollHelperError::HandleEvent(anyhow!( 117 "Error signalling that control queue was used: {:?}", 118 e 119 )) 120 })?; 121 } 122 Ok(false) => {} 123 Err(e) => { 124 return Err(EpollHelperError::HandleEvent(anyhow!( 125 "Error getting notification state of control queue: {}", 126 e 127 ))); 128 } 129 }; 130 } 131 _ => { 132 return Err(EpollHelperError::HandleEvent(anyhow!( 133 "Unknown event for virtio-net control queue" 134 ))); 135 } 136 } 137 138 Ok(()) 139 } 140 } 141 142 /// Rx/Tx queue pair 143 // The guest has made a buffer available to receive a frame into. 144 pub const RX_QUEUE_EVENT: u16 = EPOLL_HELPER_EVENT_LAST + 1; 145 // The transmit queue has a frame that is ready to send from the guest. 146 pub const TX_QUEUE_EVENT: u16 = EPOLL_HELPER_EVENT_LAST + 2; 147 // A frame is available for reading from the tap device to receive in the guest. 148 pub const RX_TAP_EVENT: u16 = EPOLL_HELPER_EVENT_LAST + 3; 149 // The TAP can be written to. Used after an EAGAIN error to retry TX. 150 pub const TX_TAP_EVENT: u16 = EPOLL_HELPER_EVENT_LAST + 4; 151 // New 'wake up' event from the rx rate limiter 152 pub const RX_RATE_LIMITER_EVENT: u16 = EPOLL_HELPER_EVENT_LAST + 5; 153 // New 'wake up' event from the tx rate limiter 154 pub const TX_RATE_LIMITER_EVENT: u16 = EPOLL_HELPER_EVENT_LAST + 6; 155 156 #[derive(Error, Debug)] 157 pub enum Error { 158 #[error("Failed to open taps: {0}")] 159 OpenTap(OpenTapError), 160 #[error("Using existing tap: {0}")] 161 TapError(TapError), 162 #[error("Error calling dup() on tap fd: {0}")] 163 DuplicateTapFd(std::io::Error), 164 } 165 166 pub type Result<T> = result::Result<T, Error>; 167 168 struct NetEpollHandler { 169 net: NetQueuePair, 170 mem: GuestMemoryAtomic<GuestMemoryMmap>, 171 interrupt_cb: Arc<dyn VirtioInterrupt>, 172 kill_evt: EventFd, 173 pause_evt: EventFd, 174 queue_index_base: u16, 175 queue_pair: (Queue, Queue), 176 queue_evt_pair: (EventFd, EventFd), 177 // Always generate interrupts until the driver has signalled to the device. 178 // This mitigates a problem with interrupts from tap events being "lost" upon 179 // a restore as the vCPU thread isn't ready to handle the interrupt. This causes 180 // issues when combined with VIRTIO_RING_F_EVENT_IDX interrupt suppression. 181 driver_awake: bool, 182 } 183 184 impl NetEpollHandler { 185 fn signal_used_queue(&self, queue_index: u16) -> result::Result<(), DeviceError> { 186 self.interrupt_cb 187 .trigger(VirtioInterruptType::Queue(queue_index)) 188 .map_err(|e| { 189 error!("Failed to signal used queue: {:?}", e); 190 DeviceError::FailedSignalingUsedQueue(e) 191 }) 192 } 193 194 fn handle_rx_event(&mut self) -> result::Result<(), DeviceError> { 195 let queue_evt = &self.queue_evt_pair.0; 196 if let Err(e) = queue_evt.read() { 197 error!("Failed to get rx queue event: {:?}", e); 198 } 199 200 self.net.rx_desc_avail = true; 201 202 let rate_limit_reached = self 203 .net 204 .rx_rate_limiter 205 .as_ref() 206 .map_or(false, |r| r.is_blocked()); 207 208 // Start to listen on RX_TAP_EVENT only when the rate limit is not reached 209 if !self.net.rx_tap_listening && !rate_limit_reached { 210 net_util::register_listener( 211 self.net.epoll_fd.unwrap(), 212 self.net.tap.as_raw_fd(), 213 epoll::Events::EPOLLIN, 214 u64::from(self.net.tap_rx_event_id), 215 ) 216 .map_err(DeviceError::IoError)?; 217 self.net.rx_tap_listening = true; 218 } 219 220 Ok(()) 221 } 222 223 fn process_tx(&mut self) -> result::Result<(), DeviceError> { 224 if self 225 .net 226 .process_tx(&self.mem.memory(), &mut self.queue_pair.1) 227 .map_err(DeviceError::NetQueuePair)? 228 || !self.driver_awake 229 { 230 self.signal_used_queue(self.queue_index_base + 1)?; 231 debug!("Signalling TX queue"); 232 } else { 233 debug!("Not signalling TX queue"); 234 } 235 Ok(()) 236 } 237 238 fn handle_tx_event(&mut self) -> result::Result<(), DeviceError> { 239 let rate_limit_reached = self 240 .net 241 .tx_rate_limiter 242 .as_ref() 243 .map_or(false, |r| r.is_blocked()); 244 245 if !rate_limit_reached { 246 self.process_tx()?; 247 } 248 249 Ok(()) 250 } 251 252 fn handle_rx_tap_event(&mut self) -> result::Result<(), DeviceError> { 253 if self 254 .net 255 .process_rx(&self.mem.memory(), &mut self.queue_pair.0) 256 .map_err(DeviceError::NetQueuePair)? 257 || !self.driver_awake 258 { 259 self.signal_used_queue(self.queue_index_base)?; 260 debug!("Signalling RX queue"); 261 } else { 262 debug!("Not signalling RX queue"); 263 } 264 Ok(()) 265 } 266 267 fn run( 268 &mut self, 269 paused: Arc<AtomicBool>, 270 paused_sync: Arc<Barrier>, 271 ) -> result::Result<(), EpollHelperError> { 272 let mut helper = EpollHelper::new(&self.kill_evt, &self.pause_evt)?; 273 helper.add_event(self.queue_evt_pair.0.as_raw_fd(), RX_QUEUE_EVENT)?; 274 helper.add_event(self.queue_evt_pair.1.as_raw_fd(), TX_QUEUE_EVENT)?; 275 if let Some(rate_limiter) = &self.net.rx_rate_limiter { 276 helper.add_event(rate_limiter.as_raw_fd(), RX_RATE_LIMITER_EVENT)?; 277 } 278 if let Some(rate_limiter) = &self.net.tx_rate_limiter { 279 helper.add_event(rate_limiter.as_raw_fd(), TX_RATE_LIMITER_EVENT)?; 280 } 281 282 let mem = self.mem.memory(); 283 // If there are some already available descriptors on the RX queue, 284 // then we can start the thread while listening onto the TAP. 285 if self 286 .queue_pair 287 .0 288 .used_idx(mem.deref(), Ordering::Acquire) 289 .map_err(EpollHelperError::QueueRingIndex)? 290 < self 291 .queue_pair 292 .0 293 .avail_idx(mem.deref(), Ordering::Acquire) 294 .map_err(EpollHelperError::QueueRingIndex)? 295 { 296 helper.add_event(self.net.tap.as_raw_fd(), RX_TAP_EVENT)?; 297 self.net.rx_tap_listening = true; 298 info!("Listener registered at start"); 299 } 300 301 // The NetQueuePair needs the epoll fd. 302 self.net.epoll_fd = Some(helper.as_raw_fd()); 303 304 helper.run(paused, paused_sync, self)?; 305 306 Ok(()) 307 } 308 } 309 310 impl EpollHelperHandler for NetEpollHandler { 311 fn handle_event( 312 &mut self, 313 _helper: &mut EpollHelper, 314 event: &epoll::Event, 315 ) -> result::Result<(), EpollHelperError> { 316 let ev_type = event.data as u16; 317 match ev_type { 318 RX_QUEUE_EVENT => { 319 self.driver_awake = true; 320 self.handle_rx_event().map_err(|e| { 321 EpollHelperError::HandleEvent(anyhow!("Error processing RX queue: {:?}", e)) 322 })?; 323 } 324 TX_QUEUE_EVENT => { 325 let queue_evt = &self.queue_evt_pair.1; 326 if let Err(e) = queue_evt.read() { 327 error!("Failed to get tx queue event: {:?}", e); 328 } 329 self.driver_awake = true; 330 self.handle_tx_event().map_err(|e| { 331 EpollHelperError::HandleEvent(anyhow!("Error processing TX queue: {:?}", e)) 332 })?; 333 } 334 TX_TAP_EVENT => { 335 self.handle_tx_event().map_err(|e| { 336 EpollHelperError::HandleEvent(anyhow!( 337 "Error processing TX queue (TAP event): {:?}", 338 e 339 )) 340 })?; 341 } 342 RX_TAP_EVENT => { 343 self.handle_rx_tap_event().map_err(|e| { 344 EpollHelperError::HandleEvent(anyhow!("Error processing tap queue: {:?}", e)) 345 })?; 346 } 347 RX_RATE_LIMITER_EVENT => { 348 if let Some(rate_limiter) = &mut self.net.rx_rate_limiter { 349 // Upon rate limiter event, call the rate limiter handler and register the 350 // TAP fd for further processing if some RX buffers are available 351 rate_limiter.event_handler().map_err(|e| { 352 EpollHelperError::HandleEvent(anyhow!( 353 "Error from 'rate_limiter.event_handler()': {:?}", 354 e 355 )) 356 })?; 357 358 if !self.net.rx_tap_listening && self.net.rx_desc_avail { 359 net_util::register_listener( 360 self.net.epoll_fd.unwrap(), 361 self.net.tap.as_raw_fd(), 362 epoll::Events::EPOLLIN, 363 u64::from(self.net.tap_rx_event_id), 364 ) 365 .map_err(|e| { 366 EpollHelperError::HandleEvent(anyhow!( 367 "Error register_listener with `RX_RATE_LIMITER_EVENT`: {:?}", 368 e 369 )) 370 })?; 371 372 self.net.rx_tap_listening = true; 373 } 374 } else { 375 return Err(EpollHelperError::HandleEvent(anyhow!( 376 "Unexpected RX_RATE_LIMITER_EVENT" 377 ))); 378 } 379 } 380 TX_RATE_LIMITER_EVENT => { 381 if let Some(rate_limiter) = &mut self.net.tx_rate_limiter { 382 // Upon rate limiter event, call the rate limiter handler 383 // and restart processing the queue. 384 rate_limiter.event_handler().map_err(|e| { 385 EpollHelperError::HandleEvent(anyhow!( 386 "Error from 'rate_limiter.event_handler()': {:?}", 387 e 388 )) 389 })?; 390 391 self.driver_awake = true; 392 self.process_tx().map_err(|e| { 393 EpollHelperError::HandleEvent(anyhow!("Error processing TX queue: {:?}", e)) 394 })?; 395 } else { 396 return Err(EpollHelperError::HandleEvent(anyhow!( 397 "Unexpected TX_RATE_LIMITER_EVENT" 398 ))); 399 } 400 } 401 _ => { 402 return Err(EpollHelperError::HandleEvent(anyhow!( 403 "Unexpected event: {}", 404 ev_type 405 ))); 406 } 407 } 408 Ok(()) 409 } 410 } 411 412 pub struct Net { 413 common: VirtioCommon, 414 id: String, 415 taps: Vec<Tap>, 416 config: VirtioNetConfig, 417 ctrl_queue_epoll_thread: Option<thread::JoinHandle<()>>, 418 counters: NetCounters, 419 seccomp_action: SeccompAction, 420 rate_limiter_config: Option<RateLimiterConfig>, 421 exit_evt: EventFd, 422 } 423 424 #[derive(Serialize, Deserialize)] 425 pub struct NetState { 426 pub avail_features: u64, 427 pub acked_features: u64, 428 pub config: VirtioNetConfig, 429 pub queue_size: Vec<u16>, 430 } 431 432 impl Net { 433 /// Create a new virtio network device with the given TAP interface. 434 #[allow(clippy::too_many_arguments)] 435 pub fn new_with_tap( 436 id: String, 437 taps: Vec<Tap>, 438 guest_mac: Option<MacAddr>, 439 iommu: bool, 440 num_queues: usize, 441 queue_size: u16, 442 seccomp_action: SeccompAction, 443 rate_limiter_config: Option<RateLimiterConfig>, 444 exit_evt: EventFd, 445 state: Option<NetState>, 446 offload_tso: bool, 447 offload_ufo: bool, 448 offload_csum: bool, 449 ) -> Result<Self> { 450 assert!(!taps.is_empty()); 451 452 let mtu = taps[0].mtu().map_err(Error::TapError)? as u16; 453 454 let (avail_features, acked_features, config, queue_sizes, paused) = 455 if let Some(state) = state { 456 info!("Restoring virtio-net {}", id); 457 ( 458 state.avail_features, 459 state.acked_features, 460 state.config, 461 state.queue_size, 462 true, 463 ) 464 } else { 465 let mut avail_features = 466 1 << VIRTIO_NET_F_MTU | 1 << VIRTIO_RING_F_EVENT_IDX | 1 << VIRTIO_F_VERSION_1; 467 468 if iommu { 469 avail_features |= 1u64 << VIRTIO_F_IOMMU_PLATFORM; 470 } 471 472 // Configure TSO/UFO features when hardware checksum offload is enabled. 473 if offload_csum { 474 avail_features |= 1 << VIRTIO_NET_F_CSUM 475 | 1 << VIRTIO_NET_F_GUEST_CSUM 476 | 1 << VIRTIO_NET_F_CTRL_GUEST_OFFLOADS; 477 478 if offload_tso { 479 avail_features |= 1 << VIRTIO_NET_F_HOST_ECN 480 | 1 << VIRTIO_NET_F_HOST_TSO4 481 | 1 << VIRTIO_NET_F_HOST_TSO6 482 | 1 << VIRTIO_NET_F_GUEST_ECN 483 | 1 << VIRTIO_NET_F_GUEST_TSO4 484 | 1 << VIRTIO_NET_F_GUEST_TSO6; 485 } 486 487 if offload_ufo { 488 avail_features |= 1 << VIRTIO_NET_F_HOST_UFO | 1 << VIRTIO_NET_F_GUEST_UFO; 489 } 490 } 491 492 avail_features |= 1 << VIRTIO_NET_F_CTRL_VQ; 493 let queue_num = num_queues + 1; 494 495 let mut config = VirtioNetConfig::default(); 496 if let Some(mac) = guest_mac { 497 build_net_config_space( 498 &mut config, 499 mac, 500 num_queues, 501 Some(mtu), 502 &mut avail_features, 503 ); 504 } else { 505 build_net_config_space_with_mq( 506 &mut config, 507 num_queues, 508 Some(mtu), 509 &mut avail_features, 510 ); 511 } 512 513 ( 514 avail_features, 515 0, 516 config, 517 vec![queue_size; queue_num], 518 false, 519 ) 520 }; 521 522 Ok(Net { 523 common: VirtioCommon { 524 device_type: VirtioDeviceType::Net as u32, 525 avail_features, 526 acked_features, 527 queue_sizes, 528 paused_sync: Some(Arc::new(Barrier::new((num_queues / 2) + 1))), 529 min_queues: 2, 530 paused: Arc::new(AtomicBool::new(paused)), 531 ..Default::default() 532 }, 533 id, 534 taps, 535 config, 536 ctrl_queue_epoll_thread: None, 537 counters: NetCounters::default(), 538 seccomp_action, 539 rate_limiter_config, 540 exit_evt, 541 }) 542 } 543 544 /// Create a new virtio network device with the given IP address and 545 /// netmask. 546 #[allow(clippy::too_many_arguments)] 547 pub fn new( 548 id: String, 549 if_name: Option<&str>, 550 ip_addr: Option<Ipv4Addr>, 551 netmask: Option<Ipv4Addr>, 552 guest_mac: Option<MacAddr>, 553 host_mac: &mut Option<MacAddr>, 554 mtu: Option<u16>, 555 iommu: bool, 556 num_queues: usize, 557 queue_size: u16, 558 seccomp_action: SeccompAction, 559 rate_limiter_config: Option<RateLimiterConfig>, 560 exit_evt: EventFd, 561 state: Option<NetState>, 562 offload_tso: bool, 563 offload_ufo: bool, 564 offload_csum: bool, 565 ) -> Result<Self> { 566 let taps = open_tap( 567 if_name, 568 ip_addr, 569 netmask, 570 host_mac, 571 mtu, 572 num_queues / 2, 573 None, 574 ) 575 .map_err(Error::OpenTap)?; 576 577 Self::new_with_tap( 578 id, 579 taps, 580 guest_mac, 581 iommu, 582 num_queues, 583 queue_size, 584 seccomp_action, 585 rate_limiter_config, 586 exit_evt, 587 state, 588 offload_tso, 589 offload_ufo, 590 offload_csum, 591 ) 592 } 593 594 #[allow(clippy::too_many_arguments)] 595 pub fn from_tap_fds( 596 id: String, 597 fds: &[RawFd], 598 guest_mac: Option<MacAddr>, 599 mtu: Option<u16>, 600 iommu: bool, 601 queue_size: u16, 602 seccomp_action: SeccompAction, 603 rate_limiter_config: Option<RateLimiterConfig>, 604 exit_evt: EventFd, 605 state: Option<NetState>, 606 offload_tso: bool, 607 offload_ufo: bool, 608 offload_csum: bool, 609 ) -> Result<Self> { 610 let mut taps: Vec<Tap> = Vec::new(); 611 let num_queue_pairs = fds.len(); 612 613 for fd in fds.iter() { 614 // Duplicate so that it can survive reboots 615 // SAFETY: FFI call to dup. Trivially safe. 616 let fd = unsafe { libc::dup(*fd) }; 617 if fd < 0 { 618 return Err(Error::DuplicateTapFd(std::io::Error::last_os_error())); 619 } 620 let tap = Tap::from_tap_fd(fd, num_queue_pairs).map_err(Error::TapError)?; 621 taps.push(tap); 622 } 623 624 assert!(!taps.is_empty()); 625 626 if let Some(mtu) = mtu { 627 taps[0].set_mtu(mtu as i32).map_err(Error::TapError)?; 628 } 629 630 Self::new_with_tap( 631 id, 632 taps, 633 guest_mac, 634 iommu, 635 num_queue_pairs * 2, 636 queue_size, 637 seccomp_action, 638 rate_limiter_config, 639 exit_evt, 640 state, 641 offload_tso, 642 offload_ufo, 643 offload_csum, 644 ) 645 } 646 647 fn state(&self) -> NetState { 648 NetState { 649 avail_features: self.common.avail_features, 650 acked_features: self.common.acked_features, 651 config: self.config, 652 queue_size: self.common.queue_sizes.clone(), 653 } 654 } 655 656 #[cfg(fuzzing)] 657 pub fn wait_for_epoll_threads(&mut self) { 658 self.common.wait_for_epoll_threads(); 659 } 660 } 661 662 impl Drop for Net { 663 fn drop(&mut self) { 664 if let Some(kill_evt) = self.common.kill_evt.take() { 665 // Ignore the result because there is nothing we can do about it. 666 let _ = kill_evt.write(1); 667 } 668 // Needed to ensure all references to tap FDs are dropped (#4868) 669 self.common.wait_for_epoll_threads(); 670 if let Some(thread) = self.ctrl_queue_epoll_thread.take() { 671 if let Err(e) = thread.join() { 672 error!("Error joining thread: {:?}", e); 673 } 674 } 675 } 676 } 677 678 impl VirtioDevice for Net { 679 fn device_type(&self) -> u32 { 680 self.common.device_type 681 } 682 683 fn queue_max_sizes(&self) -> &[u16] { 684 &self.common.queue_sizes 685 } 686 687 fn features(&self) -> u64 { 688 self.common.avail_features 689 } 690 691 fn ack_features(&mut self, value: u64) { 692 self.common.ack_features(value) 693 } 694 695 fn read_config(&self, offset: u64, data: &mut [u8]) { 696 self.read_config_from_slice(self.config.as_slice(), offset, data); 697 } 698 699 fn activate( 700 &mut self, 701 mem: GuestMemoryAtomic<GuestMemoryMmap>, 702 interrupt_cb: Arc<dyn VirtioInterrupt>, 703 mut queues: Vec<(usize, Queue, EventFd)>, 704 ) -> ActivateResult { 705 self.common.activate(&queues, &interrupt_cb)?; 706 707 let num_queues = queues.len(); 708 let event_idx = self.common.feature_acked(VIRTIO_RING_F_EVENT_IDX.into()); 709 if self.common.feature_acked(VIRTIO_NET_F_CTRL_VQ.into()) && num_queues % 2 != 0 { 710 let ctrl_queue_index = num_queues - 1; 711 let (_, mut ctrl_queue, ctrl_queue_evt) = queues.remove(ctrl_queue_index); 712 713 ctrl_queue.set_event_idx(event_idx); 714 715 let (kill_evt, pause_evt) = self.common.dup_eventfds(); 716 let mut ctrl_handler = NetCtrlEpollHandler { 717 mem: mem.clone(), 718 kill_evt, 719 pause_evt, 720 ctrl_q: CtrlQueue::new(self.taps.clone()), 721 queue: ctrl_queue, 722 queue_evt: ctrl_queue_evt, 723 access_platform: self.common.access_platform.clone(), 724 queue_index: ctrl_queue_index as u16, 725 interrupt_cb: interrupt_cb.clone(), 726 }; 727 728 let paused = self.common.paused.clone(); 729 // Let's update the barrier as we need 1 for each RX/TX pair + 730 // 1 for the control queue + 1 for the main thread signalling 731 // the pause. 732 self.common.paused_sync = Some(Arc::new(Barrier::new(self.taps.len() + 2))); 733 let paused_sync = self.common.paused_sync.clone(); 734 735 let mut epoll_threads = Vec::new(); 736 spawn_virtio_thread( 737 &format!("{}_ctrl", &self.id), 738 &self.seccomp_action, 739 Thread::VirtioNetCtl, 740 &mut epoll_threads, 741 &self.exit_evt, 742 move || ctrl_handler.run_ctrl(paused, paused_sync.unwrap()), 743 )?; 744 self.ctrl_queue_epoll_thread = Some(epoll_threads.remove(0)); 745 } 746 747 let mut epoll_threads = Vec::new(); 748 let mut taps = self.taps.clone(); 749 for i in 0..queues.len() / 2 { 750 let rx = RxVirtio::new(); 751 let tx = TxVirtio::new(); 752 let rx_tap_listening = false; 753 754 let (_, queue_0, queue_evt_0) = queues.remove(0); 755 let (_, queue_1, queue_evt_1) = queues.remove(0); 756 let mut queue_pair = (queue_0, queue_1); 757 queue_pair.0.set_event_idx(event_idx); 758 queue_pair.1.set_event_idx(event_idx); 759 760 let queue_evt_pair = (queue_evt_0, queue_evt_1); 761 762 let (kill_evt, pause_evt) = self.common.dup_eventfds(); 763 764 let rx_rate_limiter: Option<rate_limiter::RateLimiter> = self 765 .rate_limiter_config 766 .map(RateLimiterConfig::try_into) 767 .transpose() 768 .map_err(ActivateError::CreateRateLimiter)?; 769 770 let tx_rate_limiter: Option<rate_limiter::RateLimiter> = self 771 .rate_limiter_config 772 .map(RateLimiterConfig::try_into) 773 .transpose() 774 .map_err(ActivateError::CreateRateLimiter)?; 775 776 let tap = taps.remove(0); 777 #[cfg(not(fuzzing))] 778 tap.set_offload(virtio_features_to_tap_offload(self.common.acked_features)) 779 .map_err(|e| { 780 error!("Error programming tap offload: {:?}", e); 781 ActivateError::BadActivate 782 })?; 783 784 let mut handler = NetEpollHandler { 785 net: NetQueuePair { 786 tap_for_write_epoll: tap.clone(), 787 tap, 788 rx, 789 tx, 790 epoll_fd: None, 791 rx_tap_listening, 792 tx_tap_listening: false, 793 counters: self.counters.clone(), 794 tap_rx_event_id: RX_TAP_EVENT, 795 tap_tx_event_id: TX_TAP_EVENT, 796 rx_desc_avail: false, 797 rx_rate_limiter, 798 tx_rate_limiter, 799 access_platform: self.common.access_platform.clone(), 800 }, 801 mem: mem.clone(), 802 queue_index_base: (i * 2) as u16, 803 queue_pair, 804 queue_evt_pair, 805 interrupt_cb: interrupt_cb.clone(), 806 kill_evt, 807 pause_evt, 808 driver_awake: false, 809 }; 810 811 let paused = self.common.paused.clone(); 812 let paused_sync = self.common.paused_sync.clone(); 813 814 spawn_virtio_thread( 815 &format!("{}_qp{}", self.id.clone(), i), 816 &self.seccomp_action, 817 Thread::VirtioNet, 818 &mut epoll_threads, 819 &self.exit_evt, 820 move || handler.run(paused, paused_sync.unwrap()), 821 )?; 822 } 823 824 self.common.epoll_threads = Some(epoll_threads); 825 826 event!("virtio-device", "activated", "id", &self.id); 827 Ok(()) 828 } 829 830 fn reset(&mut self) -> Option<Arc<dyn VirtioInterrupt>> { 831 let result = self.common.reset(); 832 event!("virtio-device", "reset", "id", &self.id); 833 result 834 } 835 836 fn counters(&self) -> Option<HashMap<&'static str, Wrapping<u64>>> { 837 let mut counters = HashMap::new(); 838 839 counters.insert( 840 "rx_bytes", 841 Wrapping(self.counters.rx_bytes.load(Ordering::Acquire)), 842 ); 843 counters.insert( 844 "rx_frames", 845 Wrapping(self.counters.rx_frames.load(Ordering::Acquire)), 846 ); 847 counters.insert( 848 "tx_bytes", 849 Wrapping(self.counters.tx_bytes.load(Ordering::Acquire)), 850 ); 851 counters.insert( 852 "tx_frames", 853 Wrapping(self.counters.tx_frames.load(Ordering::Acquire)), 854 ); 855 856 Some(counters) 857 } 858 859 fn set_access_platform(&mut self, access_platform: Arc<dyn AccessPlatform>) { 860 self.common.set_access_platform(access_platform) 861 } 862 } 863 864 impl Pausable for Net { 865 fn pause(&mut self) -> result::Result<(), MigratableError> { 866 self.common.pause() 867 } 868 869 fn resume(&mut self) -> result::Result<(), MigratableError> { 870 self.common.resume()?; 871 872 if let Some(ctrl_queue_epoll_thread) = &self.ctrl_queue_epoll_thread { 873 ctrl_queue_epoll_thread.thread().unpark(); 874 } 875 Ok(()) 876 } 877 } 878 879 impl Snapshottable for Net { 880 fn id(&self) -> String { 881 self.id.clone() 882 } 883 884 fn snapshot(&mut self) -> std::result::Result<Snapshot, MigratableError> { 885 Snapshot::new_from_state(&self.state()) 886 } 887 } 888 impl Transportable for Net {} 889 impl Migratable for Net {} 890