1 // Copyright 2019 Intel Corporation. All Rights Reserved. 2 // SPDX-License-Identifier: Apache-2.0 3 4 use std::io; 5 use std::ops::Deref; 6 use std::os::unix::io::AsRawFd; 7 use std::sync::{atomic::AtomicBool, Arc, Barrier, Mutex}; 8 9 use anyhow::anyhow; 10 use serde::{Deserialize, Serialize}; 11 use thiserror::Error; 12 use vhost::vhost_user::message::{ 13 VhostUserInflight, VhostUserProtocolFeatures, VhostUserVirtioFeatures, 14 }; 15 use vhost::vhost_user::{FrontendReqHandler, VhostUserFrontendReqHandler}; 16 use vhost::Error as VhostError; 17 use virtio_queue::Error as QueueError; 18 use virtio_queue::Queue; 19 use vm_memory::{ 20 mmap::MmapRegionError, Address, Error as MmapError, GuestAddressSpace, GuestMemory, 21 GuestMemoryAtomic, 22 }; 23 use vm_migration::{protocol::MemoryRangeTable, MigratableError, Snapshot}; 24 use vmm_sys_util::eventfd::EventFd; 25 use vu_common_ctrl::VhostUserHandle; 26 27 use crate::{ 28 ActivateError, EpollHelper, EpollHelperError, EpollHelperHandler, GuestMemoryMmap, 29 GuestRegionMmap, VirtioInterrupt, EPOLL_HELPER_EVENT_LAST, VIRTIO_F_IN_ORDER, 30 VIRTIO_F_NOTIFICATION_DATA, VIRTIO_F_ORDER_PLATFORM, VIRTIO_F_RING_EVENT_IDX, 31 VIRTIO_F_RING_INDIRECT_DESC, VIRTIO_F_VERSION_1, 32 }; 33 34 pub mod blk; 35 pub mod fs; 36 pub mod net; 37 pub mod vu_common_ctrl; 38 39 pub use self::blk::Blk; 40 pub use self::fs::*; 41 pub use self::net::Net; 42 pub use self::vu_common_ctrl::VhostUserConfig; 43 44 #[derive(Error, Debug)] 45 pub enum Error { 46 #[error("Failed accepting connection: {0}")] 47 AcceptConnection(io::Error), 48 #[error("Invalid available address")] 49 AvailAddress, 50 #[error("Queue number is not correct")] 51 BadQueueNum, 52 #[error("Failed binding vhost-user socket: {0}")] 53 BindSocket(io::Error), 54 #[error("Creating kill eventfd failed: {0}")] 55 CreateKillEventFd(io::Error), 56 #[error("Cloning kill eventfd failed: {0}")] 57 CloneKillEventFd(io::Error), 58 #[error("Invalid descriptor table address")] 59 DescriptorTableAddress, 60 #[error("Signal used queue failed: {0}")] 61 FailedSignalingUsedQueue(io::Error), 62 #[error("Failed to read vhost eventfd: {0}")] 63 MemoryRegions(MmapError), 64 #[error("Failed removing socket path: {0}")] 65 RemoveSocketPath(io::Error), 66 #[error("Failed to create frontend: {0}")] 67 VhostUserCreateFrontend(VhostError), 68 #[error("Failed to open vhost device: {0}")] 69 VhostUserOpen(VhostError), 70 #[error("Connection to socket failed")] 71 VhostUserConnect, 72 #[error("Get features failed: {0}")] 73 VhostUserGetFeatures(VhostError), 74 #[error("Get queue max number failed: {0}")] 75 VhostUserGetQueueMaxNum(VhostError), 76 #[error("Get protocol features failed: {0}")] 77 VhostUserGetProtocolFeatures(VhostError), 78 #[error("Get vring base failed: {0}")] 79 VhostUserGetVringBase(VhostError), 80 #[error("Vhost-user Backend not support vhost-user protocol")] 81 VhostUserProtocolNotSupport, 82 #[error("Set owner failed: {0}")] 83 VhostUserSetOwner(VhostError), 84 #[error("Reset owner failed: {0}")] 85 VhostUserResetOwner(VhostError), 86 #[error("Set features failed: {0}")] 87 VhostUserSetFeatures(VhostError), 88 #[error("Set protocol features failed: {0}")] 89 VhostUserSetProtocolFeatures(VhostError), 90 #[error("Set mem table failed: {0}")] 91 VhostUserSetMemTable(VhostError), 92 #[error("Set vring num failed: {0}")] 93 VhostUserSetVringNum(VhostError), 94 #[error("Set vring addr failed: {0}")] 95 VhostUserSetVringAddr(VhostError), 96 #[error("Set vring base failed: {0}")] 97 VhostUserSetVringBase(VhostError), 98 #[error("Set vring call failed: {0}")] 99 VhostUserSetVringCall(VhostError), 100 #[error("Set vring kick failed: {0}")] 101 VhostUserSetVringKick(VhostError), 102 #[error("Set vring enable failed: {0}")] 103 VhostUserSetVringEnable(VhostError), 104 #[error("Failed to create vhost eventfd: {0}")] 105 VhostIrqCreate(io::Error), 106 #[error("Failed to read vhost eventfd: {0}")] 107 VhostIrqRead(io::Error), 108 #[error("Failed to read vhost eventfd: {0}")] 109 VhostUserMemoryRegion(MmapError), 110 #[error("Failed to create the frontend request handler from backend: {0}")] 111 FrontendReqHandlerCreation(vhost::vhost_user::Error), 112 #[error("Set backend request fd failed: {0}")] 113 VhostUserSetBackendRequestFd(vhost::Error), 114 #[error("Add memory region failed: {0}")] 115 VhostUserAddMemReg(VhostError), 116 #[error("Failed getting the configuration: {0}")] 117 VhostUserGetConfig(VhostError), 118 #[error("Failed setting the configuration: {0}")] 119 VhostUserSetConfig(VhostError), 120 #[error("Failed getting inflight shm log: {0}")] 121 VhostUserGetInflight(VhostError), 122 #[error("Failed setting inflight shm log: {0}")] 123 VhostUserSetInflight(VhostError), 124 #[error("Failed setting the log base: {0}")] 125 VhostUserSetLogBase(VhostError), 126 #[error("Invalid used address")] 127 UsedAddress, 128 #[error("Invalid features provided from vhost-user backend")] 129 InvalidFeatures, 130 #[error("Missing file descriptor for the region")] 131 MissingRegionFd, 132 #[error("Missing IrqFd")] 133 MissingIrqFd, 134 #[error("Failed getting the available index: {0}")] 135 GetAvailableIndex(QueueError), 136 #[error("Migration is not supported by this vhost-user device")] 137 MigrationNotSupported, 138 #[error("Failed creating memfd: {0}")] 139 MemfdCreate(io::Error), 140 #[error("Failed truncating the file size to the expected size: {0}")] 141 SetFileSize(io::Error), 142 #[error("Failed to set the seals on the file: {0}")] 143 SetSeals(io::Error), 144 #[error("Failed creating new mmap region: {0}")] 145 NewMmapRegion(MmapRegionError), 146 #[error("Could not find the shm log region")] 147 MissingShmLogRegion, 148 } 149 type Result<T> = std::result::Result<T, Error>; 150 151 pub const DEFAULT_VIRTIO_FEATURES: u64 = 1 << VIRTIO_F_RING_INDIRECT_DESC 152 | 1 << VIRTIO_F_RING_EVENT_IDX 153 | 1 << VIRTIO_F_VERSION_1 154 | 1 << VIRTIO_F_IN_ORDER 155 | 1 << VIRTIO_F_ORDER_PLATFORM 156 | 1 << VIRTIO_F_NOTIFICATION_DATA 157 | VhostUserVirtioFeatures::PROTOCOL_FEATURES.bits(); 158 159 const HUP_CONNECTION_EVENT: u16 = EPOLL_HELPER_EVENT_LAST + 1; 160 const BACKEND_REQ_EVENT: u16 = EPOLL_HELPER_EVENT_LAST + 2; 161 162 #[derive(Default)] 163 pub struct Inflight { 164 pub info: VhostUserInflight, 165 pub fd: Option<std::fs::File>, 166 } 167 168 pub struct VhostUserEpollHandler<S: VhostUserFrontendReqHandler> { 169 pub vu: Arc<Mutex<VhostUserHandle>>, 170 pub mem: GuestMemoryAtomic<GuestMemoryMmap>, 171 pub kill_evt: EventFd, 172 pub pause_evt: EventFd, 173 pub queues: Vec<(usize, Queue, EventFd)>, 174 pub virtio_interrupt: Arc<dyn VirtioInterrupt>, 175 pub acked_features: u64, 176 pub acked_protocol_features: u64, 177 pub socket_path: String, 178 pub server: bool, 179 pub backend_req_handler: Option<FrontendReqHandler<S>>, 180 pub inflight: Option<Inflight>, 181 } 182 183 impl<S: VhostUserFrontendReqHandler> VhostUserEpollHandler<S> { 184 pub fn run( 185 &mut self, 186 paused: Arc<AtomicBool>, 187 paused_sync: Arc<Barrier>, 188 ) -> std::result::Result<(), EpollHelperError> { 189 let mut helper = EpollHelper::new(&self.kill_evt, &self.pause_evt)?; 190 helper.add_event_custom( 191 self.vu.lock().unwrap().socket_handle().as_raw_fd(), 192 HUP_CONNECTION_EVENT, 193 epoll::Events::EPOLLHUP, 194 )?; 195 196 if let Some(backend_req_handler) = &self.backend_req_handler { 197 helper.add_event(backend_req_handler.as_raw_fd(), BACKEND_REQ_EVENT)?; 198 } 199 200 helper.run(paused, paused_sync, self)?; 201 202 Ok(()) 203 } 204 205 fn reconnect(&mut self, helper: &mut EpollHelper) -> std::result::Result<(), EpollHelperError> { 206 helper.del_event_custom( 207 self.vu.lock().unwrap().socket_handle().as_raw_fd(), 208 HUP_CONNECTION_EVENT, 209 epoll::Events::EPOLLHUP, 210 )?; 211 212 let mut vhost_user = VhostUserHandle::connect_vhost_user( 213 self.server, 214 &self.socket_path, 215 self.queues.len() as u64, 216 true, 217 ) 218 .map_err(|e| { 219 EpollHelperError::IoError(std::io::Error::new( 220 std::io::ErrorKind::Other, 221 format!("failed connecting vhost-user backend {e:?}"), 222 )) 223 })?; 224 225 // Initialize the backend 226 vhost_user 227 .reinitialize_vhost_user( 228 self.mem.memory().deref(), 229 self.queues 230 .iter() 231 .map(|(i, q, e)| (*i, vm_virtio::clone_queue(q), e.try_clone().unwrap())) 232 .collect(), 233 &self.virtio_interrupt, 234 self.acked_features, 235 self.acked_protocol_features, 236 &self.backend_req_handler, 237 self.inflight.as_mut(), 238 ) 239 .map_err(|e| { 240 EpollHelperError::IoError(std::io::Error::new( 241 std::io::ErrorKind::Other, 242 format!("failed reconnecting vhost-user backend: {e:?}"), 243 )) 244 })?; 245 246 helper.add_event_custom( 247 vhost_user.socket_handle().as_raw_fd(), 248 HUP_CONNECTION_EVENT, 249 epoll::Events::EPOLLHUP, 250 )?; 251 252 // Update vhost-user reference 253 let mut vu = self.vu.lock().unwrap(); 254 *vu = vhost_user; 255 256 Ok(()) 257 } 258 } 259 260 impl<S: VhostUserFrontendReqHandler> EpollHelperHandler for VhostUserEpollHandler<S> { 261 fn handle_event( 262 &mut self, 263 helper: &mut EpollHelper, 264 event: &epoll::Event, 265 ) -> std::result::Result<(), EpollHelperError> { 266 let ev_type = event.data as u16; 267 match ev_type { 268 HUP_CONNECTION_EVENT => { 269 self.reconnect(helper).map_err(|e| { 270 EpollHelperError::HandleEvent(anyhow!( 271 "failed to reconnect vhost-user backend: {:?}", 272 e 273 )) 274 })?; 275 } 276 BACKEND_REQ_EVENT => { 277 if let Some(backend_req_handler) = self.backend_req_handler.as_mut() { 278 backend_req_handler.handle_request().map_err(|e| { 279 EpollHelperError::HandleEvent(anyhow!( 280 "Failed to handle request from vhost-user backend: {:?}", 281 e 282 )) 283 })?; 284 } 285 } 286 _ => { 287 return Err(EpollHelperError::HandleEvent(anyhow!( 288 "Unknown event for vhost-user thread" 289 ))); 290 } 291 } 292 293 Ok(()) 294 } 295 } 296 297 #[derive(Default)] 298 pub struct VhostUserCommon { 299 pub vu: Option<Arc<Mutex<VhostUserHandle>>>, 300 pub acked_protocol_features: u64, 301 pub socket_path: String, 302 pub vu_num_queues: usize, 303 pub migration_started: bool, 304 pub server: bool, 305 } 306 307 impl VhostUserCommon { 308 #[allow(clippy::too_many_arguments)] 309 pub fn activate<T: VhostUserFrontendReqHandler>( 310 &mut self, 311 mem: GuestMemoryAtomic<GuestMemoryMmap>, 312 queues: Vec<(usize, Queue, EventFd)>, 313 interrupt_cb: Arc<dyn VirtioInterrupt>, 314 acked_features: u64, 315 backend_req_handler: Option<FrontendReqHandler<T>>, 316 kill_evt: EventFd, 317 pause_evt: EventFd, 318 ) -> std::result::Result<VhostUserEpollHandler<T>, ActivateError> { 319 let mut inflight: Option<Inflight> = 320 if self.acked_protocol_features & VhostUserProtocolFeatures::INFLIGHT_SHMFD.bits() != 0 321 { 322 Some(Inflight::default()) 323 } else { 324 None 325 }; 326 327 if self.vu.is_none() { 328 error!("Missing vhost-user handle"); 329 return Err(ActivateError::BadActivate); 330 } 331 let vu = self.vu.as_ref().unwrap(); 332 vu.lock() 333 .unwrap() 334 .setup_vhost_user( 335 &mem.memory(), 336 queues 337 .iter() 338 .map(|(i, q, e)| (*i, vm_virtio::clone_queue(q), e.try_clone().unwrap())) 339 .collect(), 340 &interrupt_cb, 341 acked_features, 342 &backend_req_handler, 343 inflight.as_mut(), 344 ) 345 .map_err(ActivateError::VhostUserSetup)?; 346 347 Ok(VhostUserEpollHandler { 348 vu: vu.clone(), 349 mem, 350 kill_evt, 351 pause_evt, 352 queues, 353 virtio_interrupt: interrupt_cb, 354 acked_features, 355 acked_protocol_features: self.acked_protocol_features, 356 socket_path: self.socket_path.clone(), 357 server: self.server, 358 backend_req_handler, 359 inflight, 360 }) 361 } 362 363 pub fn restore_backend_connection(&mut self, acked_features: u64) -> Result<()> { 364 let mut vu = VhostUserHandle::connect_vhost_user( 365 self.server, 366 &self.socket_path, 367 self.vu_num_queues as u64, 368 false, 369 )?; 370 371 vu.set_protocol_features_vhost_user(acked_features, self.acked_protocol_features)?; 372 373 self.vu = Some(Arc::new(Mutex::new(vu))); 374 375 Ok(()) 376 } 377 378 pub fn shutdown(&mut self) { 379 if let Some(vu) = &self.vu { 380 // SAFETY: trivially safe 381 let _ = unsafe { libc::close(vu.lock().unwrap().socket_handle().as_raw_fd()) }; 382 } 383 384 // Remove socket path if needed 385 if self.server { 386 let _ = std::fs::remove_file(&self.socket_path); 387 } 388 } 389 390 pub fn add_memory_region( 391 &mut self, 392 guest_memory: &Option<GuestMemoryAtomic<GuestMemoryMmap>>, 393 region: &Arc<GuestRegionMmap>, 394 ) -> std::result::Result<(), crate::Error> { 395 if let Some(vu) = &self.vu { 396 if self.acked_protocol_features & VhostUserProtocolFeatures::CONFIGURE_MEM_SLOTS.bits() 397 != 0 398 { 399 return vu 400 .lock() 401 .unwrap() 402 .add_memory_region(region) 403 .map_err(crate::Error::VhostUserAddMemoryRegion); 404 } else if let Some(guest_memory) = guest_memory { 405 return vu 406 .lock() 407 .unwrap() 408 .update_mem_table(guest_memory.memory().deref()) 409 .map_err(crate::Error::VhostUserUpdateMemory); 410 } 411 } 412 Ok(()) 413 } 414 415 pub fn pause(&mut self) -> std::result::Result<(), MigratableError> { 416 if let Some(vu) = &self.vu { 417 vu.lock().unwrap().pause_vhost_user().map_err(|e| { 418 MigratableError::Pause(anyhow!("Error pausing vhost-user backend: {:?}", e)) 419 }) 420 } else { 421 Ok(()) 422 } 423 } 424 425 pub fn resume(&mut self) -> std::result::Result<(), MigratableError> { 426 if let Some(vu) = &self.vu { 427 vu.lock().unwrap().resume_vhost_user().map_err(|e| { 428 MigratableError::Resume(anyhow!("Error resuming vhost-user backend: {:?}", e)) 429 }) 430 } else { 431 Ok(()) 432 } 433 } 434 435 pub fn snapshot<'a, T>(&mut self, state: &T) -> std::result::Result<Snapshot, MigratableError> 436 where 437 T: Serialize + Deserialize<'a>, 438 { 439 let snapshot = Snapshot::new_from_state(state)?; 440 441 if self.migration_started { 442 self.shutdown(); 443 } 444 445 Ok(snapshot) 446 } 447 448 pub fn start_dirty_log( 449 &mut self, 450 guest_memory: &Option<GuestMemoryAtomic<GuestMemoryMmap>>, 451 ) -> std::result::Result<(), MigratableError> { 452 if let Some(vu) = &self.vu { 453 if let Some(guest_memory) = guest_memory { 454 let last_ram_addr = guest_memory.memory().last_addr().raw_value(); 455 vu.lock() 456 .unwrap() 457 .start_dirty_log(last_ram_addr) 458 .map_err(|e| { 459 MigratableError::StartDirtyLog(anyhow!( 460 "Error starting migration for vhost-user backend: {:?}", 461 e 462 )) 463 }) 464 } else { 465 Err(MigratableError::StartDirtyLog(anyhow!( 466 "Missing guest memory" 467 ))) 468 } 469 } else { 470 Ok(()) 471 } 472 } 473 474 pub fn stop_dirty_log(&mut self) -> std::result::Result<(), MigratableError> { 475 if let Some(vu) = &self.vu { 476 vu.lock().unwrap().stop_dirty_log().map_err(|e| { 477 MigratableError::StopDirtyLog(anyhow!( 478 "Error stopping migration for vhost-user backend: {:?}", 479 e 480 )) 481 }) 482 } else { 483 Ok(()) 484 } 485 } 486 487 pub fn dirty_log( 488 &mut self, 489 guest_memory: &Option<GuestMemoryAtomic<GuestMemoryMmap>>, 490 ) -> std::result::Result<MemoryRangeTable, MigratableError> { 491 if let Some(vu) = &self.vu { 492 if let Some(guest_memory) = guest_memory { 493 let last_ram_addr = guest_memory.memory().last_addr().raw_value(); 494 vu.lock().unwrap().dirty_log(last_ram_addr).map_err(|e| { 495 MigratableError::DirtyLog(anyhow!( 496 "Error retrieving dirty ranges from vhost-user backend: {:?}", 497 e 498 )) 499 }) 500 } else { 501 Err(MigratableError::DirtyLog(anyhow!("Missing guest memory"))) 502 } 503 } else { 504 Ok(MemoryRangeTable::default()) 505 } 506 } 507 508 pub fn start_migration(&mut self) -> std::result::Result<(), MigratableError> { 509 self.migration_started = true; 510 Ok(()) 511 } 512 513 pub fn complete_migration( 514 &mut self, 515 kill_evt: Option<EventFd>, 516 ) -> std::result::Result<(), MigratableError> { 517 self.migration_started = false; 518 519 // Make sure the device thread is killed in order to prevent from 520 // reconnections to the socket. 521 if let Some(kill_evt) = kill_evt { 522 kill_evt.write(1).map_err(|e| { 523 MigratableError::CompleteMigration(anyhow!( 524 "Error killing vhost-user thread: {:?}", 525 e 526 )) 527 })?; 528 } 529 530 // Drop the vhost-user handler to avoid further calls to fail because 531 // the connection with the backend has been closed. 532 self.vu = None; 533 534 Ok(()) 535 } 536 } 537