1 // Copyright 2019 Intel Corporation. All Rights Reserved. 2 // SPDX-License-Identifier: Apache-2.0 3 4 use std::io; 5 use std::ops::Deref; 6 use std::os::unix::io::AsRawFd; 7 use std::sync::atomic::AtomicBool; 8 use std::sync::{Arc, Barrier, Mutex}; 9 10 use anyhow::anyhow; 11 use serde::{Deserialize, Serialize}; 12 use thiserror::Error; 13 use vhost::vhost_user::message::{ 14 VhostUserInflight, VhostUserProtocolFeatures, VhostUserVirtioFeatures, 15 }; 16 use vhost::vhost_user::{FrontendReqHandler, VhostUserFrontendReqHandler}; 17 use vhost::Error as VhostError; 18 use virtio_queue::{Error as QueueError, Queue}; 19 use vm_memory::mmap::MmapRegionError; 20 use vm_memory::{Address, Error as MmapError, GuestAddressSpace, GuestMemory, GuestMemoryAtomic}; 21 use vm_migration::protocol::MemoryRangeTable; 22 use vm_migration::{MigratableError, Snapshot}; 23 use vmm_sys_util::eventfd::EventFd; 24 use vu_common_ctrl::VhostUserHandle; 25 26 use crate::{ 27 ActivateError, EpollHelper, EpollHelperError, EpollHelperHandler, GuestMemoryMmap, 28 GuestRegionMmap, VirtioInterrupt, EPOLL_HELPER_EVENT_LAST, VIRTIO_F_IN_ORDER, 29 VIRTIO_F_NOTIFICATION_DATA, VIRTIO_F_ORDER_PLATFORM, VIRTIO_F_RING_EVENT_IDX, 30 VIRTIO_F_RING_INDIRECT_DESC, VIRTIO_F_VERSION_1, 31 }; 32 33 pub mod blk; 34 pub mod fs; 35 pub mod net; 36 pub mod vu_common_ctrl; 37 38 pub use self::blk::Blk; 39 pub use self::fs::*; 40 pub use self::net::Net; 41 pub use self::vu_common_ctrl::VhostUserConfig; 42 43 #[derive(Error, Debug)] 44 pub enum Error { 45 #[error("Failed accepting connection")] 46 AcceptConnection(#[source] io::Error), 47 #[error("Invalid available address")] 48 AvailAddress, 49 #[error("Queue number is not correct")] 50 BadQueueNum, 51 #[error("Failed binding vhost-user socket")] 52 BindSocket(#[source] io::Error), 53 #[error("Creating kill eventfd failed")] 54 CreateKillEventFd(#[source] io::Error), 55 #[error("Cloning kill eventfd failed")] 56 CloneKillEventFd(#[source] io::Error), 57 #[error("Invalid descriptor table address")] 58 DescriptorTableAddress, 59 #[error("Signal used queue failed")] 60 FailedSignalingUsedQueue(#[source] io::Error), 61 #[error("Failed to read vhost eventfd")] 62 MemoryRegions(#[source] MmapError), 63 #[error("Failed removing socket path")] 64 RemoveSocketPath(#[source] io::Error), 65 #[error("Failed to create frontend")] 66 VhostUserCreateFrontend(#[source] VhostError), 67 #[error("Failed to open vhost device")] 68 VhostUserOpen(#[source] VhostError), 69 #[error("Connection to socket failed")] 70 VhostUserConnect, 71 #[error("Get features failed")] 72 VhostUserGetFeatures(#[source] VhostError), 73 #[error("Get queue max number failed")] 74 VhostUserGetQueueMaxNum(#[source] VhostError), 75 #[error("Get protocol features failed")] 76 VhostUserGetProtocolFeatures(#[source] VhostError), 77 #[error("Get vring base failed")] 78 VhostUserGetVringBase(#[source] VhostError), 79 #[error("Vhost-user Backend not support vhost-user protocol")] 80 VhostUserProtocolNotSupport, 81 #[error("Set owner failed")] 82 VhostUserSetOwner(#[source] VhostError), 83 #[error("Reset owner failed")] 84 VhostUserResetOwner(#[source] VhostError), 85 #[error("Set features failed")] 86 VhostUserSetFeatures(#[source] VhostError), 87 #[error("Set protocol features failed")] 88 VhostUserSetProtocolFeatures(#[source] VhostError), 89 #[error("Set mem table failed")] 90 VhostUserSetMemTable(#[source] VhostError), 91 #[error("Set vring num failed")] 92 VhostUserSetVringNum(#[source] VhostError), 93 #[error("Set vring addr failed")] 94 VhostUserSetVringAddr(#[source] VhostError), 95 #[error("Set vring base failed")] 96 VhostUserSetVringBase(#[source] VhostError), 97 #[error("Set vring call failed")] 98 VhostUserSetVringCall(#[source] VhostError), 99 #[error("Set vring kick failed")] 100 VhostUserSetVringKick(#[source] VhostError), 101 #[error("Set vring enable failed")] 102 VhostUserSetVringEnable(#[source] VhostError), 103 #[error("Failed to create vhost eventfd")] 104 VhostIrqCreate(#[source] io::Error), 105 #[error("Failed to read vhost eventfd")] 106 VhostIrqRead(#[source] io::Error), 107 #[error("Failed to read vhost eventfd")] 108 VhostUserMemoryRegion(#[source] MmapError), 109 #[error("Failed to create the frontend request handler from backend")] 110 FrontendReqHandlerCreation(#[source] vhost::vhost_user::Error), 111 #[error("Set backend request fd failed")] 112 VhostUserSetBackendRequestFd(#[source] vhost::Error), 113 #[error("Add memory region failed")] 114 VhostUserAddMemReg(#[source] VhostError), 115 #[error("Failed getting the configuration")] 116 VhostUserGetConfig(#[source] VhostError), 117 #[error("Failed setting the configuration")] 118 VhostUserSetConfig(#[source] VhostError), 119 #[error("Failed getting inflight shm log")] 120 VhostUserGetInflight(#[source] VhostError), 121 #[error("Failed setting inflight shm log")] 122 VhostUserSetInflight(#[source] VhostError), 123 #[error("Failed setting the log base")] 124 VhostUserSetLogBase(#[source] VhostError), 125 #[error("Invalid used address")] 126 UsedAddress, 127 #[error("Invalid features provided from vhost-user backend")] 128 InvalidFeatures, 129 #[error("Missing file descriptor for the region")] 130 MissingRegionFd, 131 #[error("Missing IrqFd")] 132 MissingIrqFd, 133 #[error("Failed getting the available index")] 134 GetAvailableIndex(#[source] QueueError), 135 #[error("Migration is not supported by this vhost-user device")] 136 MigrationNotSupported, 137 #[error("Failed creating memfd")] 138 MemfdCreate(#[source] io::Error), 139 #[error("Failed truncating the file size to the expected size")] 140 SetFileSize(#[source] io::Error), 141 #[error("Failed to set the seals on the file")] 142 SetSeals(#[source] io::Error), 143 #[error("Failed creating new mmap region")] 144 NewMmapRegion(#[source] MmapRegionError), 145 #[error("Could not find the shm log region")] 146 MissingShmLogRegion, 147 } 148 type Result<T> = std::result::Result<T, Error>; 149 150 pub const DEFAULT_VIRTIO_FEATURES: u64 = (1 << VIRTIO_F_RING_INDIRECT_DESC) 151 | (1 << VIRTIO_F_RING_EVENT_IDX) 152 | (1 << VIRTIO_F_VERSION_1) 153 | (1 << VIRTIO_F_IN_ORDER) 154 | (1 << VIRTIO_F_ORDER_PLATFORM) 155 | (1 << VIRTIO_F_NOTIFICATION_DATA) 156 | VhostUserVirtioFeatures::PROTOCOL_FEATURES.bits(); 157 158 const HUP_CONNECTION_EVENT: u16 = EPOLL_HELPER_EVENT_LAST + 1; 159 const BACKEND_REQ_EVENT: u16 = EPOLL_HELPER_EVENT_LAST + 2; 160 161 #[derive(Default)] 162 pub struct Inflight { 163 pub info: VhostUserInflight, 164 pub fd: Option<std::fs::File>, 165 } 166 167 pub struct VhostUserEpollHandler<S: VhostUserFrontendReqHandler> { 168 pub vu: Arc<Mutex<VhostUserHandle>>, 169 pub mem: GuestMemoryAtomic<GuestMemoryMmap>, 170 pub kill_evt: EventFd, 171 pub pause_evt: EventFd, 172 pub queues: Vec<(usize, Queue, EventFd)>, 173 pub virtio_interrupt: Arc<dyn VirtioInterrupt>, 174 pub acked_features: u64, 175 pub acked_protocol_features: u64, 176 pub socket_path: String, 177 pub server: bool, 178 pub backend_req_handler: Option<FrontendReqHandler<S>>, 179 pub inflight: Option<Inflight>, 180 } 181 182 impl<S: VhostUserFrontendReqHandler> VhostUserEpollHandler<S> { 183 pub fn run( 184 &mut self, 185 paused: Arc<AtomicBool>, 186 paused_sync: Arc<Barrier>, 187 ) -> std::result::Result<(), EpollHelperError> { 188 let mut helper = EpollHelper::new(&self.kill_evt, &self.pause_evt)?; 189 helper.add_event_custom( 190 self.vu.lock().unwrap().socket_handle().as_raw_fd(), 191 HUP_CONNECTION_EVENT, 192 epoll::Events::EPOLLHUP, 193 )?; 194 195 if let Some(backend_req_handler) = &self.backend_req_handler { 196 helper.add_event(backend_req_handler.as_raw_fd(), BACKEND_REQ_EVENT)?; 197 } 198 199 helper.run(paused, paused_sync, self)?; 200 201 Ok(()) 202 } 203 204 fn reconnect(&mut self, helper: &mut EpollHelper) -> std::result::Result<(), EpollHelperError> { 205 helper.del_event_custom( 206 self.vu.lock().unwrap().socket_handle().as_raw_fd(), 207 HUP_CONNECTION_EVENT, 208 epoll::Events::EPOLLHUP, 209 )?; 210 211 let mut vhost_user = VhostUserHandle::connect_vhost_user( 212 self.server, 213 &self.socket_path, 214 self.queues.len() as u64, 215 true, 216 ) 217 .map_err(|e| { 218 EpollHelperError::IoError(std::io::Error::other(format!( 219 "failed connecting vhost-user backend {e:?}" 220 ))) 221 })?; 222 223 // Initialize the backend 224 vhost_user 225 .reinitialize_vhost_user( 226 self.mem.memory().deref(), 227 self.queues 228 .iter() 229 .map(|(i, q, e)| (*i, vm_virtio::clone_queue(q), e.try_clone().unwrap())) 230 .collect(), 231 &self.virtio_interrupt, 232 self.acked_features, 233 self.acked_protocol_features, 234 &self.backend_req_handler, 235 self.inflight.as_mut(), 236 ) 237 .map_err(|e| { 238 EpollHelperError::IoError(std::io::Error::other(format!( 239 "failed reconnecting vhost-user backend: {e:?}" 240 ))) 241 })?; 242 243 helper.add_event_custom( 244 vhost_user.socket_handle().as_raw_fd(), 245 HUP_CONNECTION_EVENT, 246 epoll::Events::EPOLLHUP, 247 )?; 248 249 // Update vhost-user reference 250 let mut vu = self.vu.lock().unwrap(); 251 *vu = vhost_user; 252 253 Ok(()) 254 } 255 } 256 257 impl<S: VhostUserFrontendReqHandler> EpollHelperHandler for VhostUserEpollHandler<S> { 258 fn handle_event( 259 &mut self, 260 helper: &mut EpollHelper, 261 event: &epoll::Event, 262 ) -> std::result::Result<(), EpollHelperError> { 263 let ev_type = event.data as u16; 264 match ev_type { 265 HUP_CONNECTION_EVENT => { 266 self.reconnect(helper).map_err(|e| { 267 EpollHelperError::HandleEvent(anyhow!( 268 "failed to reconnect vhost-user backend: {:?}", 269 e 270 )) 271 })?; 272 } 273 BACKEND_REQ_EVENT => { 274 if let Some(backend_req_handler) = self.backend_req_handler.as_mut() { 275 backend_req_handler.handle_request().map_err(|e| { 276 EpollHelperError::HandleEvent(anyhow!( 277 "Failed to handle request from vhost-user backend: {:?}", 278 e 279 )) 280 })?; 281 } 282 } 283 _ => { 284 return Err(EpollHelperError::HandleEvent(anyhow!( 285 "Unknown event for vhost-user thread" 286 ))); 287 } 288 } 289 290 Ok(()) 291 } 292 } 293 294 #[derive(Default)] 295 pub struct VhostUserCommon { 296 pub vu: Option<Arc<Mutex<VhostUserHandle>>>, 297 pub acked_protocol_features: u64, 298 pub socket_path: String, 299 pub vu_num_queues: usize, 300 pub migration_started: bool, 301 pub server: bool, 302 } 303 304 impl VhostUserCommon { 305 #[allow(clippy::too_many_arguments)] 306 pub fn activate<T: VhostUserFrontendReqHandler>( 307 &mut self, 308 mem: GuestMemoryAtomic<GuestMemoryMmap>, 309 queues: Vec<(usize, Queue, EventFd)>, 310 interrupt_cb: Arc<dyn VirtioInterrupt>, 311 acked_features: u64, 312 backend_req_handler: Option<FrontendReqHandler<T>>, 313 kill_evt: EventFd, 314 pause_evt: EventFd, 315 ) -> std::result::Result<VhostUserEpollHandler<T>, ActivateError> { 316 let mut inflight: Option<Inflight> = 317 if self.acked_protocol_features & VhostUserProtocolFeatures::INFLIGHT_SHMFD.bits() != 0 318 { 319 Some(Inflight::default()) 320 } else { 321 None 322 }; 323 324 if self.vu.is_none() { 325 error!("Missing vhost-user handle"); 326 return Err(ActivateError::BadActivate); 327 } 328 let vu = self.vu.as_ref().unwrap(); 329 vu.lock() 330 .unwrap() 331 .setup_vhost_user( 332 &mem.memory(), 333 queues 334 .iter() 335 .map(|(i, q, e)| (*i, vm_virtio::clone_queue(q), e.try_clone().unwrap())) 336 .collect(), 337 &interrupt_cb, 338 acked_features, 339 &backend_req_handler, 340 inflight.as_mut(), 341 ) 342 .map_err(ActivateError::VhostUserSetup)?; 343 344 Ok(VhostUserEpollHandler { 345 vu: vu.clone(), 346 mem, 347 kill_evt, 348 pause_evt, 349 queues, 350 virtio_interrupt: interrupt_cb, 351 acked_features, 352 acked_protocol_features: self.acked_protocol_features, 353 socket_path: self.socket_path.clone(), 354 server: self.server, 355 backend_req_handler, 356 inflight, 357 }) 358 } 359 360 pub fn restore_backend_connection(&mut self, acked_features: u64) -> Result<()> { 361 let mut vu = VhostUserHandle::connect_vhost_user( 362 self.server, 363 &self.socket_path, 364 self.vu_num_queues as u64, 365 false, 366 )?; 367 368 vu.set_protocol_features_vhost_user(acked_features, self.acked_protocol_features)?; 369 370 self.vu = Some(Arc::new(Mutex::new(vu))); 371 372 Ok(()) 373 } 374 375 pub fn shutdown(&mut self) { 376 if let Some(vu) = &self.vu { 377 // SAFETY: trivially safe 378 let _ = unsafe { libc::close(vu.lock().unwrap().socket_handle().as_raw_fd()) }; 379 } 380 381 // Remove socket path if needed 382 if self.server { 383 let _ = std::fs::remove_file(&self.socket_path); 384 } 385 } 386 387 pub fn add_memory_region( 388 &mut self, 389 guest_memory: &Option<GuestMemoryAtomic<GuestMemoryMmap>>, 390 region: &Arc<GuestRegionMmap>, 391 ) -> std::result::Result<(), crate::Error> { 392 if let Some(vu) = &self.vu { 393 if self.acked_protocol_features & VhostUserProtocolFeatures::CONFIGURE_MEM_SLOTS.bits() 394 != 0 395 { 396 return vu 397 .lock() 398 .unwrap() 399 .add_memory_region(region) 400 .map_err(crate::Error::VhostUserAddMemoryRegion); 401 } else if let Some(guest_memory) = guest_memory { 402 return vu 403 .lock() 404 .unwrap() 405 .update_mem_table(guest_memory.memory().deref()) 406 .map_err(crate::Error::VhostUserUpdateMemory); 407 } 408 } 409 Ok(()) 410 } 411 412 pub fn pause(&mut self) -> std::result::Result<(), MigratableError> { 413 if let Some(vu) = &self.vu { 414 vu.lock().unwrap().pause_vhost_user().map_err(|e| { 415 MigratableError::Pause(anyhow!("Error pausing vhost-user backend: {:?}", e)) 416 }) 417 } else { 418 Ok(()) 419 } 420 } 421 422 pub fn resume(&mut self) -> std::result::Result<(), MigratableError> { 423 if let Some(vu) = &self.vu { 424 vu.lock().unwrap().resume_vhost_user().map_err(|e| { 425 MigratableError::Resume(anyhow!("Error resuming vhost-user backend: {:?}", e)) 426 }) 427 } else { 428 Ok(()) 429 } 430 } 431 432 pub fn snapshot<'a, T>(&mut self, state: &T) -> std::result::Result<Snapshot, MigratableError> 433 where 434 T: Serialize + Deserialize<'a>, 435 { 436 let snapshot = Snapshot::new_from_state(state)?; 437 438 if self.migration_started { 439 self.shutdown(); 440 } 441 442 Ok(snapshot) 443 } 444 445 pub fn start_dirty_log( 446 &mut self, 447 guest_memory: &Option<GuestMemoryAtomic<GuestMemoryMmap>>, 448 ) -> std::result::Result<(), MigratableError> { 449 if let Some(vu) = &self.vu { 450 if let Some(guest_memory) = guest_memory { 451 let last_ram_addr = guest_memory.memory().last_addr().raw_value(); 452 vu.lock() 453 .unwrap() 454 .start_dirty_log(last_ram_addr) 455 .map_err(|e| { 456 MigratableError::StartDirtyLog(anyhow!( 457 "Error starting migration for vhost-user backend: {:?}", 458 e 459 )) 460 }) 461 } else { 462 Err(MigratableError::StartDirtyLog(anyhow!( 463 "Missing guest memory" 464 ))) 465 } 466 } else { 467 Ok(()) 468 } 469 } 470 471 pub fn stop_dirty_log(&mut self) -> std::result::Result<(), MigratableError> { 472 if let Some(vu) = &self.vu { 473 vu.lock().unwrap().stop_dirty_log().map_err(|e| { 474 MigratableError::StopDirtyLog(anyhow!( 475 "Error stopping migration for vhost-user backend: {:?}", 476 e 477 )) 478 }) 479 } else { 480 Ok(()) 481 } 482 } 483 484 pub fn dirty_log( 485 &mut self, 486 guest_memory: &Option<GuestMemoryAtomic<GuestMemoryMmap>>, 487 ) -> std::result::Result<MemoryRangeTable, MigratableError> { 488 if let Some(vu) = &self.vu { 489 if let Some(guest_memory) = guest_memory { 490 let last_ram_addr = guest_memory.memory().last_addr().raw_value(); 491 vu.lock().unwrap().dirty_log(last_ram_addr).map_err(|e| { 492 MigratableError::DirtyLog(anyhow!( 493 "Error retrieving dirty ranges from vhost-user backend: {:?}", 494 e 495 )) 496 }) 497 } else { 498 Err(MigratableError::DirtyLog(anyhow!("Missing guest memory"))) 499 } 500 } else { 501 Ok(MemoryRangeTable::default()) 502 } 503 } 504 505 pub fn start_migration(&mut self) -> std::result::Result<(), MigratableError> { 506 self.migration_started = true; 507 Ok(()) 508 } 509 510 pub fn complete_migration( 511 &mut self, 512 kill_evt: Option<EventFd>, 513 ) -> std::result::Result<(), MigratableError> { 514 self.migration_started = false; 515 516 // Make sure the device thread is killed in order to prevent from 517 // reconnections to the socket. 518 if let Some(kill_evt) = kill_evt { 519 kill_evt.write(1).map_err(|e| { 520 MigratableError::CompleteMigration(anyhow!( 521 "Error killing vhost-user thread: {:?}", 522 e 523 )) 524 })?; 525 } 526 527 // Drop the vhost-user handler to avoid further calls to fail because 528 // the connection with the backend has been closed. 529 self.vu = None; 530 531 Ok(()) 532 } 533 } 534