1 // Copyright 2019 Intel Corporation. All Rights Reserved. 2 // SPDX-License-Identifier: Apache-2.0 3 4 use std::io; 5 use std::ops::Deref; 6 use std::os::unix::io::AsRawFd; 7 use std::sync::atomic::AtomicBool; 8 use std::sync::{Arc, Barrier, Mutex}; 9 10 use anyhow::anyhow; 11 use serde::{Deserialize, Serialize}; 12 use thiserror::Error; 13 use vhost::vhost_user::message::{ 14 VhostUserInflight, VhostUserProtocolFeatures, VhostUserVirtioFeatures, 15 }; 16 use vhost::vhost_user::{FrontendReqHandler, VhostUserFrontendReqHandler}; 17 use vhost::Error as VhostError; 18 use virtio_queue::{Error as QueueError, Queue}; 19 use vm_memory::mmap::MmapRegionError; 20 use vm_memory::{Address, Error as MmapError, GuestAddressSpace, GuestMemory, GuestMemoryAtomic}; 21 use vm_migration::protocol::MemoryRangeTable; 22 use vm_migration::{MigratableError, Snapshot}; 23 use vmm_sys_util::eventfd::EventFd; 24 use vu_common_ctrl::VhostUserHandle; 25 26 use crate::{ 27 ActivateError, EpollHelper, EpollHelperError, EpollHelperHandler, GuestMemoryMmap, 28 GuestRegionMmap, VirtioInterrupt, EPOLL_HELPER_EVENT_LAST, VIRTIO_F_IN_ORDER, 29 VIRTIO_F_NOTIFICATION_DATA, VIRTIO_F_ORDER_PLATFORM, VIRTIO_F_RING_EVENT_IDX, 30 VIRTIO_F_RING_INDIRECT_DESC, VIRTIO_F_VERSION_1, 31 }; 32 33 pub mod blk; 34 pub mod fs; 35 pub mod net; 36 pub mod vu_common_ctrl; 37 38 pub use self::blk::Blk; 39 pub use self::fs::*; 40 pub use self::net::Net; 41 pub use self::vu_common_ctrl::VhostUserConfig; 42 43 #[derive(Error, Debug)] 44 pub enum Error { 45 #[error("Failed accepting connection: {0}")] 46 AcceptConnection(io::Error), 47 #[error("Invalid available address")] 48 AvailAddress, 49 #[error("Queue number is not correct")] 50 BadQueueNum, 51 #[error("Failed binding vhost-user socket: {0}")] 52 BindSocket(io::Error), 53 #[error("Creating kill eventfd failed: {0}")] 54 CreateKillEventFd(io::Error), 55 #[error("Cloning kill eventfd failed: {0}")] 56 CloneKillEventFd(io::Error), 57 #[error("Invalid descriptor table address")] 58 DescriptorTableAddress, 59 #[error("Signal used queue failed: {0}")] 60 FailedSignalingUsedQueue(io::Error), 61 #[error("Failed to read vhost eventfd: {0}")] 62 MemoryRegions(MmapError), 63 #[error("Failed removing socket path: {0}")] 64 RemoveSocketPath(io::Error), 65 #[error("Failed to create frontend: {0}")] 66 VhostUserCreateFrontend(VhostError), 67 #[error("Failed to open vhost device: {0}")] 68 VhostUserOpen(VhostError), 69 #[error("Connection to socket failed")] 70 VhostUserConnect, 71 #[error("Get features failed: {0}")] 72 VhostUserGetFeatures(VhostError), 73 #[error("Get queue max number failed: {0}")] 74 VhostUserGetQueueMaxNum(VhostError), 75 #[error("Get protocol features failed: {0}")] 76 VhostUserGetProtocolFeatures(VhostError), 77 #[error("Get vring base failed: {0}")] 78 VhostUserGetVringBase(VhostError), 79 #[error("Vhost-user Backend not support vhost-user protocol")] 80 VhostUserProtocolNotSupport, 81 #[error("Set owner failed: {0}")] 82 VhostUserSetOwner(VhostError), 83 #[error("Reset owner failed: {0}")] 84 VhostUserResetOwner(VhostError), 85 #[error("Set features failed: {0}")] 86 VhostUserSetFeatures(VhostError), 87 #[error("Set protocol features failed: {0}")] 88 VhostUserSetProtocolFeatures(VhostError), 89 #[error("Set mem table failed: {0}")] 90 VhostUserSetMemTable(VhostError), 91 #[error("Set vring num failed: {0}")] 92 VhostUserSetVringNum(VhostError), 93 #[error("Set vring addr failed: {0}")] 94 VhostUserSetVringAddr(VhostError), 95 #[error("Set vring base failed: {0}")] 96 VhostUserSetVringBase(VhostError), 97 #[error("Set vring call failed: {0}")] 98 VhostUserSetVringCall(VhostError), 99 #[error("Set vring kick failed: {0}")] 100 VhostUserSetVringKick(VhostError), 101 #[error("Set vring enable failed: {0}")] 102 VhostUserSetVringEnable(VhostError), 103 #[error("Failed to create vhost eventfd: {0}")] 104 VhostIrqCreate(io::Error), 105 #[error("Failed to read vhost eventfd: {0}")] 106 VhostIrqRead(io::Error), 107 #[error("Failed to read vhost eventfd: {0}")] 108 VhostUserMemoryRegion(MmapError), 109 #[error("Failed to create the frontend request handler from backend: {0}")] 110 FrontendReqHandlerCreation(vhost::vhost_user::Error), 111 #[error("Set backend request fd failed: {0}")] 112 VhostUserSetBackendRequestFd(vhost::Error), 113 #[error("Add memory region failed: {0}")] 114 VhostUserAddMemReg(VhostError), 115 #[error("Failed getting the configuration: {0}")] 116 VhostUserGetConfig(VhostError), 117 #[error("Failed setting the configuration: {0}")] 118 VhostUserSetConfig(VhostError), 119 #[error("Failed getting inflight shm log: {0}")] 120 VhostUserGetInflight(VhostError), 121 #[error("Failed setting inflight shm log: {0}")] 122 VhostUserSetInflight(VhostError), 123 #[error("Failed setting the log base: {0}")] 124 VhostUserSetLogBase(VhostError), 125 #[error("Invalid used address")] 126 UsedAddress, 127 #[error("Invalid features provided from vhost-user backend")] 128 InvalidFeatures, 129 #[error("Missing file descriptor for the region")] 130 MissingRegionFd, 131 #[error("Missing IrqFd")] 132 MissingIrqFd, 133 #[error("Failed getting the available index: {0}")] 134 GetAvailableIndex(QueueError), 135 #[error("Migration is not supported by this vhost-user device")] 136 MigrationNotSupported, 137 #[error("Failed creating memfd: {0}")] 138 MemfdCreate(io::Error), 139 #[error("Failed truncating the file size to the expected size: {0}")] 140 SetFileSize(io::Error), 141 #[error("Failed to set the seals on the file: {0}")] 142 SetSeals(io::Error), 143 #[error("Failed creating new mmap region: {0}")] 144 NewMmapRegion(MmapRegionError), 145 #[error("Could not find the shm log region")] 146 MissingShmLogRegion, 147 } 148 type Result<T> = std::result::Result<T, Error>; 149 150 pub const DEFAULT_VIRTIO_FEATURES: u64 = 1 << VIRTIO_F_RING_INDIRECT_DESC 151 | 1 << VIRTIO_F_RING_EVENT_IDX 152 | 1 << VIRTIO_F_VERSION_1 153 | 1 << VIRTIO_F_IN_ORDER 154 | 1 << VIRTIO_F_ORDER_PLATFORM 155 | 1 << VIRTIO_F_NOTIFICATION_DATA 156 | VhostUserVirtioFeatures::PROTOCOL_FEATURES.bits(); 157 158 const HUP_CONNECTION_EVENT: u16 = EPOLL_HELPER_EVENT_LAST + 1; 159 const BACKEND_REQ_EVENT: u16 = EPOLL_HELPER_EVENT_LAST + 2; 160 161 #[derive(Default)] 162 pub struct Inflight { 163 pub info: VhostUserInflight, 164 pub fd: Option<std::fs::File>, 165 } 166 167 pub struct VhostUserEpollHandler<S: VhostUserFrontendReqHandler> { 168 pub vu: Arc<Mutex<VhostUserHandle>>, 169 pub mem: GuestMemoryAtomic<GuestMemoryMmap>, 170 pub kill_evt: EventFd, 171 pub pause_evt: EventFd, 172 pub queues: Vec<(usize, Queue, EventFd)>, 173 pub virtio_interrupt: Arc<dyn VirtioInterrupt>, 174 pub acked_features: u64, 175 pub acked_protocol_features: u64, 176 pub socket_path: String, 177 pub server: bool, 178 pub backend_req_handler: Option<FrontendReqHandler<S>>, 179 pub inflight: Option<Inflight>, 180 } 181 182 impl<S: VhostUserFrontendReqHandler> VhostUserEpollHandler<S> { 183 pub fn run( 184 &mut self, 185 paused: Arc<AtomicBool>, 186 paused_sync: Arc<Barrier>, 187 ) -> std::result::Result<(), EpollHelperError> { 188 let mut helper = EpollHelper::new(&self.kill_evt, &self.pause_evt)?; 189 helper.add_event_custom( 190 self.vu.lock().unwrap().socket_handle().as_raw_fd(), 191 HUP_CONNECTION_EVENT, 192 epoll::Events::EPOLLHUP, 193 )?; 194 195 if let Some(backend_req_handler) = &self.backend_req_handler { 196 helper.add_event(backend_req_handler.as_raw_fd(), BACKEND_REQ_EVENT)?; 197 } 198 199 helper.run(paused, paused_sync, self)?; 200 201 Ok(()) 202 } 203 204 fn reconnect(&mut self, helper: &mut EpollHelper) -> std::result::Result<(), EpollHelperError> { 205 helper.del_event_custom( 206 self.vu.lock().unwrap().socket_handle().as_raw_fd(), 207 HUP_CONNECTION_EVENT, 208 epoll::Events::EPOLLHUP, 209 )?; 210 211 let mut vhost_user = VhostUserHandle::connect_vhost_user( 212 self.server, 213 &self.socket_path, 214 self.queues.len() as u64, 215 true, 216 ) 217 .map_err(|e| { 218 EpollHelperError::IoError(std::io::Error::new( 219 std::io::ErrorKind::Other, 220 format!("failed connecting vhost-user backend {e:?}"), 221 )) 222 })?; 223 224 // Initialize the backend 225 vhost_user 226 .reinitialize_vhost_user( 227 self.mem.memory().deref(), 228 self.queues 229 .iter() 230 .map(|(i, q, e)| (*i, vm_virtio::clone_queue(q), e.try_clone().unwrap())) 231 .collect(), 232 &self.virtio_interrupt, 233 self.acked_features, 234 self.acked_protocol_features, 235 &self.backend_req_handler, 236 self.inflight.as_mut(), 237 ) 238 .map_err(|e| { 239 EpollHelperError::IoError(std::io::Error::new( 240 std::io::ErrorKind::Other, 241 format!("failed reconnecting vhost-user backend: {e:?}"), 242 )) 243 })?; 244 245 helper.add_event_custom( 246 vhost_user.socket_handle().as_raw_fd(), 247 HUP_CONNECTION_EVENT, 248 epoll::Events::EPOLLHUP, 249 )?; 250 251 // Update vhost-user reference 252 let mut vu = self.vu.lock().unwrap(); 253 *vu = vhost_user; 254 255 Ok(()) 256 } 257 } 258 259 impl<S: VhostUserFrontendReqHandler> EpollHelperHandler for VhostUserEpollHandler<S> { 260 fn handle_event( 261 &mut self, 262 helper: &mut EpollHelper, 263 event: &epoll::Event, 264 ) -> std::result::Result<(), EpollHelperError> { 265 let ev_type = event.data as u16; 266 match ev_type { 267 HUP_CONNECTION_EVENT => { 268 self.reconnect(helper).map_err(|e| { 269 EpollHelperError::HandleEvent(anyhow!( 270 "failed to reconnect vhost-user backend: {:?}", 271 e 272 )) 273 })?; 274 } 275 BACKEND_REQ_EVENT => { 276 if let Some(backend_req_handler) = self.backend_req_handler.as_mut() { 277 backend_req_handler.handle_request().map_err(|e| { 278 EpollHelperError::HandleEvent(anyhow!( 279 "Failed to handle request from vhost-user backend: {:?}", 280 e 281 )) 282 })?; 283 } 284 } 285 _ => { 286 return Err(EpollHelperError::HandleEvent(anyhow!( 287 "Unknown event for vhost-user thread" 288 ))); 289 } 290 } 291 292 Ok(()) 293 } 294 } 295 296 #[derive(Default)] 297 pub struct VhostUserCommon { 298 pub vu: Option<Arc<Mutex<VhostUserHandle>>>, 299 pub acked_protocol_features: u64, 300 pub socket_path: String, 301 pub vu_num_queues: usize, 302 pub migration_started: bool, 303 pub server: bool, 304 } 305 306 impl VhostUserCommon { 307 #[allow(clippy::too_many_arguments)] 308 pub fn activate<T: VhostUserFrontendReqHandler>( 309 &mut self, 310 mem: GuestMemoryAtomic<GuestMemoryMmap>, 311 queues: Vec<(usize, Queue, EventFd)>, 312 interrupt_cb: Arc<dyn VirtioInterrupt>, 313 acked_features: u64, 314 backend_req_handler: Option<FrontendReqHandler<T>>, 315 kill_evt: EventFd, 316 pause_evt: EventFd, 317 ) -> std::result::Result<VhostUserEpollHandler<T>, ActivateError> { 318 let mut inflight: Option<Inflight> = 319 if self.acked_protocol_features & VhostUserProtocolFeatures::INFLIGHT_SHMFD.bits() != 0 320 { 321 Some(Inflight::default()) 322 } else { 323 None 324 }; 325 326 if self.vu.is_none() { 327 error!("Missing vhost-user handle"); 328 return Err(ActivateError::BadActivate); 329 } 330 let vu = self.vu.as_ref().unwrap(); 331 vu.lock() 332 .unwrap() 333 .setup_vhost_user( 334 &mem.memory(), 335 queues 336 .iter() 337 .map(|(i, q, e)| (*i, vm_virtio::clone_queue(q), e.try_clone().unwrap())) 338 .collect(), 339 &interrupt_cb, 340 acked_features, 341 &backend_req_handler, 342 inflight.as_mut(), 343 ) 344 .map_err(ActivateError::VhostUserSetup)?; 345 346 Ok(VhostUserEpollHandler { 347 vu: vu.clone(), 348 mem, 349 kill_evt, 350 pause_evt, 351 queues, 352 virtio_interrupt: interrupt_cb, 353 acked_features, 354 acked_protocol_features: self.acked_protocol_features, 355 socket_path: self.socket_path.clone(), 356 server: self.server, 357 backend_req_handler, 358 inflight, 359 }) 360 } 361 362 pub fn restore_backend_connection(&mut self, acked_features: u64) -> Result<()> { 363 let mut vu = VhostUserHandle::connect_vhost_user( 364 self.server, 365 &self.socket_path, 366 self.vu_num_queues as u64, 367 false, 368 )?; 369 370 vu.set_protocol_features_vhost_user(acked_features, self.acked_protocol_features)?; 371 372 self.vu = Some(Arc::new(Mutex::new(vu))); 373 374 Ok(()) 375 } 376 377 pub fn shutdown(&mut self) { 378 if let Some(vu) = &self.vu { 379 // SAFETY: trivially safe 380 let _ = unsafe { libc::close(vu.lock().unwrap().socket_handle().as_raw_fd()) }; 381 } 382 383 // Remove socket path if needed 384 if self.server { 385 let _ = std::fs::remove_file(&self.socket_path); 386 } 387 } 388 389 pub fn add_memory_region( 390 &mut self, 391 guest_memory: &Option<GuestMemoryAtomic<GuestMemoryMmap>>, 392 region: &Arc<GuestRegionMmap>, 393 ) -> std::result::Result<(), crate::Error> { 394 if let Some(vu) = &self.vu { 395 if self.acked_protocol_features & VhostUserProtocolFeatures::CONFIGURE_MEM_SLOTS.bits() 396 != 0 397 { 398 return vu 399 .lock() 400 .unwrap() 401 .add_memory_region(region) 402 .map_err(crate::Error::VhostUserAddMemoryRegion); 403 } else if let Some(guest_memory) = guest_memory { 404 return vu 405 .lock() 406 .unwrap() 407 .update_mem_table(guest_memory.memory().deref()) 408 .map_err(crate::Error::VhostUserUpdateMemory); 409 } 410 } 411 Ok(()) 412 } 413 414 pub fn pause(&mut self) -> std::result::Result<(), MigratableError> { 415 if let Some(vu) = &self.vu { 416 vu.lock().unwrap().pause_vhost_user().map_err(|e| { 417 MigratableError::Pause(anyhow!("Error pausing vhost-user backend: {:?}", e)) 418 }) 419 } else { 420 Ok(()) 421 } 422 } 423 424 pub fn resume(&mut self) -> std::result::Result<(), MigratableError> { 425 if let Some(vu) = &self.vu { 426 vu.lock().unwrap().resume_vhost_user().map_err(|e| { 427 MigratableError::Resume(anyhow!("Error resuming vhost-user backend: {:?}", e)) 428 }) 429 } else { 430 Ok(()) 431 } 432 } 433 434 pub fn snapshot<'a, T>(&mut self, state: &T) -> std::result::Result<Snapshot, MigratableError> 435 where 436 T: Serialize + Deserialize<'a>, 437 { 438 let snapshot = Snapshot::new_from_state(state)?; 439 440 if self.migration_started { 441 self.shutdown(); 442 } 443 444 Ok(snapshot) 445 } 446 447 pub fn start_dirty_log( 448 &mut self, 449 guest_memory: &Option<GuestMemoryAtomic<GuestMemoryMmap>>, 450 ) -> std::result::Result<(), MigratableError> { 451 if let Some(vu) = &self.vu { 452 if let Some(guest_memory) = guest_memory { 453 let last_ram_addr = guest_memory.memory().last_addr().raw_value(); 454 vu.lock() 455 .unwrap() 456 .start_dirty_log(last_ram_addr) 457 .map_err(|e| { 458 MigratableError::StartDirtyLog(anyhow!( 459 "Error starting migration for vhost-user backend: {:?}", 460 e 461 )) 462 }) 463 } else { 464 Err(MigratableError::StartDirtyLog(anyhow!( 465 "Missing guest memory" 466 ))) 467 } 468 } else { 469 Ok(()) 470 } 471 } 472 473 pub fn stop_dirty_log(&mut self) -> std::result::Result<(), MigratableError> { 474 if let Some(vu) = &self.vu { 475 vu.lock().unwrap().stop_dirty_log().map_err(|e| { 476 MigratableError::StopDirtyLog(anyhow!( 477 "Error stopping migration for vhost-user backend: {:?}", 478 e 479 )) 480 }) 481 } else { 482 Ok(()) 483 } 484 } 485 486 pub fn dirty_log( 487 &mut self, 488 guest_memory: &Option<GuestMemoryAtomic<GuestMemoryMmap>>, 489 ) -> std::result::Result<MemoryRangeTable, MigratableError> { 490 if let Some(vu) = &self.vu { 491 if let Some(guest_memory) = guest_memory { 492 let last_ram_addr = guest_memory.memory().last_addr().raw_value(); 493 vu.lock().unwrap().dirty_log(last_ram_addr).map_err(|e| { 494 MigratableError::DirtyLog(anyhow!( 495 "Error retrieving dirty ranges from vhost-user backend: {:?}", 496 e 497 )) 498 }) 499 } else { 500 Err(MigratableError::DirtyLog(anyhow!("Missing guest memory"))) 501 } 502 } else { 503 Ok(MemoryRangeTable::default()) 504 } 505 } 506 507 pub fn start_migration(&mut self) -> std::result::Result<(), MigratableError> { 508 self.migration_started = true; 509 Ok(()) 510 } 511 512 pub fn complete_migration( 513 &mut self, 514 kill_evt: Option<EventFd>, 515 ) -> std::result::Result<(), MigratableError> { 516 self.migration_started = false; 517 518 // Make sure the device thread is killed in order to prevent from 519 // reconnections to the socket. 520 if let Some(kill_evt) = kill_evt { 521 kill_evt.write(1).map_err(|e| { 522 MigratableError::CompleteMigration(anyhow!( 523 "Error killing vhost-user thread: {:?}", 524 e 525 )) 526 })?; 527 } 528 529 // Drop the vhost-user handler to avoid further calls to fail because 530 // the connection with the backend has been closed. 531 self.vu = None; 532 533 Ok(()) 534 } 535 } 536