1 // Copyright 2019 Intel Corporation. All Rights Reserved. 2 // SPDX-License-Identifier: Apache-2.0 3 4 use crate::{ 5 ActivateError, EpollHelper, EpollHelperError, EpollHelperHandler, GuestMemoryMmap, 6 GuestRegionMmap, VirtioInterrupt, EPOLL_HELPER_EVENT_LAST, VIRTIO_F_IN_ORDER, 7 VIRTIO_F_NOTIFICATION_DATA, VIRTIO_F_ORDER_PLATFORM, VIRTIO_F_RING_EVENT_IDX, 8 VIRTIO_F_RING_INDIRECT_DESC, VIRTIO_F_VERSION_1, 9 }; 10 use anyhow::anyhow; 11 use std::io; 12 use std::ops::Deref; 13 use std::os::unix::io::AsRawFd; 14 use std::sync::{atomic::AtomicBool, Arc, Barrier, Mutex}; 15 use thiserror::Error; 16 use versionize::Versionize; 17 use vhost::vhost_user::message::{ 18 VhostUserInflight, VhostUserProtocolFeatures, VhostUserVirtioFeatures, 19 }; 20 use vhost::vhost_user::{MasterReqHandler, VhostUserMasterReqHandler}; 21 use vhost::Error as VhostError; 22 use virtio_queue::Error as QueueError; 23 use virtio_queue::Queue; 24 use vm_memory::{ 25 mmap::MmapRegionError, Address, Error as MmapError, GuestAddressSpace, GuestMemory, 26 GuestMemoryAtomic, 27 }; 28 use vm_migration::{protocol::MemoryRangeTable, MigratableError, Snapshot, VersionMapped}; 29 use vmm_sys_util::eventfd::EventFd; 30 use vu_common_ctrl::VhostUserHandle; 31 32 pub mod blk; 33 pub mod fs; 34 pub mod net; 35 pub mod vu_common_ctrl; 36 37 pub use self::blk::Blk; 38 pub use self::fs::*; 39 pub use self::net::Net; 40 pub use self::vu_common_ctrl::VhostUserConfig; 41 42 #[derive(Error, Debug)] 43 pub enum Error { 44 #[error("Failed accepting connection: {0}")] 45 AcceptConnection(io::Error), 46 #[error("Invalid available address")] 47 AvailAddress, 48 #[error("Queue number is not correct")] 49 BadQueueNum, 50 #[error("Failed binding vhost-user socket: {0}")] 51 BindSocket(io::Error), 52 #[error("Creating kill eventfd failed: {0}")] 53 CreateKillEventFd(io::Error), 54 #[error("Cloning kill eventfd failed: {0}")] 55 CloneKillEventFd(io::Error), 56 #[error("Invalid descriptor table address")] 57 DescriptorTableAddress, 58 #[error("Signal used queue failed: {0}")] 59 FailedSignalingUsedQueue(io::Error), 60 #[error("Failed to read vhost eventfd: {0}")] 61 MemoryRegions(MmapError), 62 #[error("Failed removing socket path: {0}")] 63 RemoveSocketPath(io::Error), 64 #[error("Failed to create master: {0}")] 65 VhostUserCreateMaster(VhostError), 66 #[error("Failed to open vhost device: {0}")] 67 VhostUserOpen(VhostError), 68 #[error("Connection to socket failed")] 69 VhostUserConnect, 70 #[error("Get features failed: {0}")] 71 VhostUserGetFeatures(VhostError), 72 #[error("Get queue max number failed: {0}")] 73 VhostUserGetQueueMaxNum(VhostError), 74 #[error("Get protocol features failed: {0}")] 75 VhostUserGetProtocolFeatures(VhostError), 76 #[error("Get vring base failed: {0}")] 77 VhostUserGetVringBase(VhostError), 78 #[error("Vhost-user Backend not support vhost-user protocol")] 79 VhostUserProtocolNotSupport, 80 #[error("Set owner failed: {0}")] 81 VhostUserSetOwner(VhostError), 82 #[error("Reset owner failed: {0}")] 83 VhostUserResetOwner(VhostError), 84 #[error("Set features failed: {0}")] 85 VhostUserSetFeatures(VhostError), 86 #[error("Set protocol features failed: {0}")] 87 VhostUserSetProtocolFeatures(VhostError), 88 #[error("Set mem table failed: {0}")] 89 VhostUserSetMemTable(VhostError), 90 #[error("Set vring num failed: {0}")] 91 VhostUserSetVringNum(VhostError), 92 #[error("Set vring addr failed: {0}")] 93 VhostUserSetVringAddr(VhostError), 94 #[error("Set vring base failed: {0}")] 95 VhostUserSetVringBase(VhostError), 96 #[error("Set vring call failed: {0}")] 97 VhostUserSetVringCall(VhostError), 98 #[error("Set vring kick failed: {0}")] 99 VhostUserSetVringKick(VhostError), 100 #[error("Set vring enable failed: {0}")] 101 VhostUserSetVringEnable(VhostError), 102 #[error("Failed to create vhost eventfd: {0}")] 103 VhostIrqCreate(io::Error), 104 #[error("Failed to read vhost eventfd: {0}")] 105 VhostIrqRead(io::Error), 106 #[error("Failed to read vhost eventfd: {0}")] 107 VhostUserMemoryRegion(MmapError), 108 #[error("Failed to create the master request handler from slave: {0}")] 109 MasterReqHandlerCreation(vhost::vhost_user::Error), 110 #[error("Set slave request fd failed: {0}")] 111 VhostUserSetSlaveRequestFd(vhost::Error), 112 #[error("Add memory region failed: {0}")] 113 VhostUserAddMemReg(VhostError), 114 #[error("Failed getting the configuration: {0}")] 115 VhostUserGetConfig(VhostError), 116 #[error("Failed setting the configuration: {0}")] 117 VhostUserSetConfig(VhostError), 118 #[error("Failed getting inflight shm log: {0}")] 119 VhostUserGetInflight(VhostError), 120 #[error("Failed setting inflight shm log: {0}")] 121 VhostUserSetInflight(VhostError), 122 #[error("Failed setting the log base: {0}")] 123 VhostUserSetLogBase(VhostError), 124 #[error("Invalid used address")] 125 UsedAddress, 126 #[error("Invalid features provided from vhost-user backe")] 127 InvalidFeatures, 128 #[error("Missing file descriptor for the region")] 129 MissingRegionFd, 130 #[error("Missing IrqFd")] 131 MissingIrqFd, 132 #[error("Failed getting the available index: {0}")] 133 GetAvailableIndex(QueueError), 134 #[error("Migration is not supported by this vhost-user device")] 135 MigrationNotSupported, 136 #[error("Failed creating memfd: {0}")] 137 MemfdCreate(io::Error), 138 #[error("Failed truncating the file size to the expected size: {0}")] 139 SetFileSize(io::Error), 140 #[error("Failed to set the seals on the file: {0}")] 141 SetSeals(io::Error), 142 #[error("Failed creating new mmap region: {0}")] 143 NewMmapRegion(MmapRegionError), 144 #[error("Could not find the shm log region")] 145 MissingShmLogRegion, 146 } 147 type Result<T> = std::result::Result<T, Error>; 148 149 pub const DEFAULT_VIRTIO_FEATURES: u64 = 1 << VIRTIO_F_RING_INDIRECT_DESC 150 | 1 << VIRTIO_F_RING_EVENT_IDX 151 | 1 << VIRTIO_F_VERSION_1 152 | 1 << VIRTIO_F_IN_ORDER 153 | 1 << VIRTIO_F_ORDER_PLATFORM 154 | 1 << VIRTIO_F_NOTIFICATION_DATA 155 | VhostUserVirtioFeatures::PROTOCOL_FEATURES.bits(); 156 157 const HUP_CONNECTION_EVENT: u16 = EPOLL_HELPER_EVENT_LAST + 1; 158 const SLAVE_REQ_EVENT: u16 = EPOLL_HELPER_EVENT_LAST + 2; 159 160 #[derive(Default)] 161 pub struct Inflight { 162 pub info: VhostUserInflight, 163 pub fd: Option<std::fs::File>, 164 } 165 166 pub struct VhostUserEpollHandler<S: VhostUserMasterReqHandler> { 167 pub vu: Arc<Mutex<VhostUserHandle>>, 168 pub mem: GuestMemoryAtomic<GuestMemoryMmap>, 169 pub kill_evt: EventFd, 170 pub pause_evt: EventFd, 171 pub queues: Vec<(usize, Queue, EventFd)>, 172 pub virtio_interrupt: Arc<dyn VirtioInterrupt>, 173 pub acked_features: u64, 174 pub acked_protocol_features: u64, 175 pub socket_path: String, 176 pub server: bool, 177 pub slave_req_handler: Option<MasterReqHandler<S>>, 178 pub inflight: Option<Inflight>, 179 } 180 181 impl<S: VhostUserMasterReqHandler> VhostUserEpollHandler<S> { 182 pub fn run( 183 &mut self, 184 paused: Arc<AtomicBool>, 185 paused_sync: Arc<Barrier>, 186 ) -> std::result::Result<(), EpollHelperError> { 187 let mut helper = EpollHelper::new(&self.kill_evt, &self.pause_evt)?; 188 helper.add_event_custom( 189 self.vu.lock().unwrap().socket_handle().as_raw_fd(), 190 HUP_CONNECTION_EVENT, 191 epoll::Events::EPOLLHUP, 192 )?; 193 194 if let Some(slave_req_handler) = &self.slave_req_handler { 195 helper.add_event(slave_req_handler.as_raw_fd(), SLAVE_REQ_EVENT)?; 196 } 197 198 helper.run(paused, paused_sync, self)?; 199 200 Ok(()) 201 } 202 203 fn reconnect(&mut self, helper: &mut EpollHelper) -> std::result::Result<(), EpollHelperError> { 204 helper.del_event_custom( 205 self.vu.lock().unwrap().socket_handle().as_raw_fd(), 206 HUP_CONNECTION_EVENT, 207 epoll::Events::EPOLLHUP, 208 )?; 209 210 let mut vhost_user = VhostUserHandle::connect_vhost_user( 211 self.server, 212 &self.socket_path, 213 self.queues.len() as u64, 214 true, 215 ) 216 .map_err(|e| { 217 EpollHelperError::IoError(std::io::Error::new( 218 std::io::ErrorKind::Other, 219 format!("failed connecting vhost-user backend {:?}", e), 220 )) 221 })?; 222 223 // Initialize the backend 224 vhost_user 225 .reinitialize_vhost_user( 226 self.mem.memory().deref(), 227 self.queues 228 .iter() 229 .map(|(i, q, e)| (*i, vm_virtio::clone_queue(q), e.try_clone().unwrap())) 230 .collect(), 231 &self.virtio_interrupt, 232 self.acked_features, 233 self.acked_protocol_features, 234 &self.slave_req_handler, 235 self.inflight.as_mut(), 236 ) 237 .map_err(|e| { 238 EpollHelperError::IoError(std::io::Error::new( 239 std::io::ErrorKind::Other, 240 format!("failed reconnecting vhost-user backend{:?}", e), 241 )) 242 })?; 243 244 helper.add_event_custom( 245 vhost_user.socket_handle().as_raw_fd(), 246 HUP_CONNECTION_EVENT, 247 epoll::Events::EPOLLHUP, 248 )?; 249 250 // Update vhost-user reference 251 let mut vu = self.vu.lock().unwrap(); 252 *vu = vhost_user; 253 254 Ok(()) 255 } 256 } 257 258 impl<S: VhostUserMasterReqHandler> EpollHelperHandler for VhostUserEpollHandler<S> { 259 fn handle_event( 260 &mut self, 261 helper: &mut EpollHelper, 262 event: &epoll::Event, 263 ) -> std::result::Result<(), EpollHelperError> { 264 let ev_type = event.data as u16; 265 match ev_type { 266 HUP_CONNECTION_EVENT => { 267 self.reconnect(helper).map_err(|e| { 268 EpollHelperError::HandleEvent(anyhow!( 269 "failed to reconnect vhost-user backend: {:?}", 270 e 271 )) 272 })?; 273 } 274 SLAVE_REQ_EVENT => { 275 if let Some(slave_req_handler) = self.slave_req_handler.as_mut() { 276 slave_req_handler.handle_request().map_err(|e| { 277 EpollHelperError::HandleEvent(anyhow!( 278 "Failed to handle request from vhost-user backend: {:?}", 279 e 280 )) 281 })?; 282 } 283 } 284 _ => { 285 return Err(EpollHelperError::HandleEvent(anyhow!( 286 "Unknown event for vhost-user thread" 287 ))); 288 } 289 } 290 291 Ok(()) 292 } 293 } 294 295 #[derive(Default)] 296 pub struct VhostUserCommon { 297 pub vu: Option<Arc<Mutex<VhostUserHandle>>>, 298 pub acked_protocol_features: u64, 299 pub socket_path: String, 300 pub vu_num_queues: usize, 301 pub migration_started: bool, 302 pub server: bool, 303 } 304 305 impl VhostUserCommon { 306 #[allow(clippy::too_many_arguments)] 307 pub fn activate<T: VhostUserMasterReqHandler>( 308 &mut self, 309 mem: GuestMemoryAtomic<GuestMemoryMmap>, 310 queues: Vec<(usize, Queue, EventFd)>, 311 interrupt_cb: Arc<dyn VirtioInterrupt>, 312 acked_features: u64, 313 slave_req_handler: Option<MasterReqHandler<T>>, 314 kill_evt: EventFd, 315 pause_evt: EventFd, 316 ) -> std::result::Result<VhostUserEpollHandler<T>, ActivateError> { 317 let mut inflight: Option<Inflight> = 318 if self.acked_protocol_features & VhostUserProtocolFeatures::INFLIGHT_SHMFD.bits() != 0 319 { 320 Some(Inflight::default()) 321 } else { 322 None 323 }; 324 325 if self.vu.is_none() { 326 error!("Missing vhost-user handle"); 327 return Err(ActivateError::BadActivate); 328 } 329 let vu = self.vu.as_ref().unwrap(); 330 vu.lock() 331 .unwrap() 332 .setup_vhost_user( 333 &mem.memory(), 334 queues 335 .iter() 336 .map(|(i, q, e)| (*i, vm_virtio::clone_queue(q), e.try_clone().unwrap())) 337 .collect(), 338 &interrupt_cb, 339 acked_features, 340 &slave_req_handler, 341 inflight.as_mut(), 342 ) 343 .map_err(ActivateError::VhostUserBlkSetup)?; 344 345 Ok(VhostUserEpollHandler { 346 vu: vu.clone(), 347 mem, 348 kill_evt, 349 pause_evt, 350 queues, 351 virtio_interrupt: interrupt_cb, 352 acked_features, 353 acked_protocol_features: self.acked_protocol_features, 354 socket_path: self.socket_path.clone(), 355 server: self.server, 356 slave_req_handler, 357 inflight, 358 }) 359 } 360 361 pub fn restore_backend_connection(&mut self, acked_features: u64) -> Result<()> { 362 let mut vu = VhostUserHandle::connect_vhost_user( 363 self.server, 364 &self.socket_path, 365 self.vu_num_queues as u64, 366 false, 367 )?; 368 369 vu.set_protocol_features_vhost_user(acked_features, self.acked_protocol_features)?; 370 371 self.vu = Some(Arc::new(Mutex::new(vu))); 372 373 Ok(()) 374 } 375 376 pub fn shutdown(&mut self) { 377 if let Some(vu) = &self.vu { 378 // SAFETY: trivially safe 379 let _ = unsafe { libc::close(vu.lock().unwrap().socket_handle().as_raw_fd()) }; 380 } 381 382 // Remove socket path if needed 383 if self.server { 384 let _ = std::fs::remove_file(&self.socket_path); 385 } 386 } 387 388 pub fn add_memory_region( 389 &mut self, 390 guest_memory: &Option<GuestMemoryAtomic<GuestMemoryMmap>>, 391 region: &Arc<GuestRegionMmap>, 392 ) -> std::result::Result<(), crate::Error> { 393 if let Some(vu) = &self.vu { 394 if self.acked_protocol_features & VhostUserProtocolFeatures::CONFIGURE_MEM_SLOTS.bits() 395 != 0 396 { 397 return vu 398 .lock() 399 .unwrap() 400 .add_memory_region(region) 401 .map_err(crate::Error::VhostUserAddMemoryRegion); 402 } else if let Some(guest_memory) = guest_memory { 403 return vu 404 .lock() 405 .unwrap() 406 .update_mem_table(guest_memory.memory().deref()) 407 .map_err(crate::Error::VhostUserUpdateMemory); 408 } 409 } 410 Ok(()) 411 } 412 413 pub fn pause(&mut self) -> std::result::Result<(), MigratableError> { 414 if let Some(vu) = &self.vu { 415 vu.lock().unwrap().pause_vhost_user().map_err(|e| { 416 MigratableError::Pause(anyhow!("Error pausing vhost-user-blk backend: {:?}", e)) 417 }) 418 } else { 419 Ok(()) 420 } 421 } 422 423 pub fn resume(&mut self) -> std::result::Result<(), MigratableError> { 424 if let Some(vu) = &self.vu { 425 vu.lock().unwrap().resume_vhost_user().map_err(|e| { 426 MigratableError::Resume(anyhow!("Error resuming vhost-user-blk backend: {:?}", e)) 427 }) 428 } else { 429 Ok(()) 430 } 431 } 432 433 pub fn snapshot<T>( 434 &mut self, 435 id: &str, 436 state: &T, 437 ) -> std::result::Result<Snapshot, MigratableError> 438 where 439 T: Versionize + VersionMapped, 440 { 441 let snapshot = Snapshot::new_from_versioned_state(id, state)?; 442 443 if self.migration_started { 444 self.shutdown(); 445 } 446 447 Ok(snapshot) 448 } 449 450 pub fn start_dirty_log( 451 &mut self, 452 guest_memory: &Option<GuestMemoryAtomic<GuestMemoryMmap>>, 453 ) -> std::result::Result<(), MigratableError> { 454 if let Some(vu) = &self.vu { 455 if let Some(guest_memory) = guest_memory { 456 let last_ram_addr = guest_memory.memory().last_addr().raw_value(); 457 vu.lock() 458 .unwrap() 459 .start_dirty_log(last_ram_addr) 460 .map_err(|e| { 461 MigratableError::StartDirtyLog(anyhow!( 462 "Error starting migration for vhost-user backend: {:?}", 463 e 464 )) 465 }) 466 } else { 467 Err(MigratableError::StartDirtyLog(anyhow!( 468 "Missing guest memory" 469 ))) 470 } 471 } else { 472 Ok(()) 473 } 474 } 475 476 pub fn stop_dirty_log(&mut self) -> std::result::Result<(), MigratableError> { 477 if let Some(vu) = &self.vu { 478 vu.lock().unwrap().stop_dirty_log().map_err(|e| { 479 MigratableError::StopDirtyLog(anyhow!( 480 "Error stopping migration for vhost-user backend: {:?}", 481 e 482 )) 483 }) 484 } else { 485 Ok(()) 486 } 487 } 488 489 pub fn dirty_log( 490 &mut self, 491 guest_memory: &Option<GuestMemoryAtomic<GuestMemoryMmap>>, 492 ) -> std::result::Result<MemoryRangeTable, MigratableError> { 493 if let Some(vu) = &self.vu { 494 if let Some(guest_memory) = guest_memory { 495 let last_ram_addr = guest_memory.memory().last_addr().raw_value(); 496 vu.lock().unwrap().dirty_log(last_ram_addr).map_err(|e| { 497 MigratableError::DirtyLog(anyhow!( 498 "Error retrieving dirty ranges from vhost-user backend: {:?}", 499 e 500 )) 501 }) 502 } else { 503 Err(MigratableError::DirtyLog(anyhow!("Missing guest memory"))) 504 } 505 } else { 506 Ok(MemoryRangeTable::default()) 507 } 508 } 509 510 pub fn start_migration(&mut self) -> std::result::Result<(), MigratableError> { 511 self.migration_started = true; 512 Ok(()) 513 } 514 515 pub fn complete_migration( 516 &mut self, 517 kill_evt: Option<EventFd>, 518 ) -> std::result::Result<(), MigratableError> { 519 self.migration_started = false; 520 521 // Make sure the device thread is killed in order to prevent from 522 // reconnections to the socket. 523 if let Some(kill_evt) = kill_evt { 524 kill_evt.write(1).map_err(|e| { 525 MigratableError::CompleteMigration(anyhow!( 526 "Error killing vhost-user thread: {:?}", 527 e 528 )) 529 })?; 530 } 531 532 // Drop the vhost-user handler to avoid further calls to fail because 533 // the connection with the backend has been closed. 534 self.vu = None; 535 536 Ok(()) 537 } 538 } 539