1 // Copyright 2019 Intel Corporation. All Rights Reserved. 2 // SPDX-License-Identifier: Apache-2.0 3 4 use crate::{ 5 EpollHelper, EpollHelperError, EpollHelperHandler, GuestMemoryMmap, Queue, VirtioInterrupt, 6 EPOLL_HELPER_EVENT_LAST, VIRTIO_F_IN_ORDER, VIRTIO_F_NOTIFICATION_DATA, 7 VIRTIO_F_ORDER_PLATFORM, VIRTIO_F_RING_EVENT_IDX, VIRTIO_F_RING_INDIRECT_DESC, 8 VIRTIO_F_VERSION_1, 9 }; 10 use std::io; 11 use std::ops::Deref; 12 use std::os::unix::io::AsRawFd; 13 use std::sync::{atomic::AtomicBool, Arc, Barrier, Mutex}; 14 use vhost::vhost_user::message::{VhostUserInflight, VhostUserVirtioFeatures}; 15 use vhost::vhost_user::{MasterReqHandler, VhostUserMasterReqHandler}; 16 use vhost::Error as VhostError; 17 use vm_memory::{mmap::MmapRegionError, Error as MmapError, GuestAddressSpace, GuestMemoryAtomic}; 18 use vm_virtio::Error as VirtioError; 19 use vmm_sys_util::eventfd::EventFd; 20 use vu_common_ctrl::VhostUserHandle; 21 22 pub mod blk; 23 pub mod fs; 24 pub mod net; 25 pub mod vu_common_ctrl; 26 27 pub use self::blk::Blk; 28 pub use self::fs::*; 29 pub use self::net::Net; 30 pub use self::vu_common_ctrl::VhostUserConfig; 31 32 #[derive(Debug)] 33 pub enum Error { 34 /// Failed accepting connection. 35 AcceptConnection(io::Error), 36 /// Invalid available address. 37 AvailAddress, 38 /// Queue number is not correct 39 BadQueueNum, 40 /// Failed binding vhost-user socket. 41 BindSocket(io::Error), 42 /// Creating kill eventfd failed. 43 CreateKillEventFd(io::Error), 44 /// Cloning kill eventfd failed. 45 CloneKillEventFd(io::Error), 46 /// Invalid descriptor table address. 47 DescriptorTableAddress, 48 /// Signal used queue failed. 49 FailedSignalingUsedQueue(io::Error), 50 /// Failed to read vhost eventfd. 51 MemoryRegions(MmapError), 52 /// Failed removing socket path 53 RemoveSocketPath(io::Error), 54 /// Failed to create master. 55 VhostUserCreateMaster(VhostError), 56 /// Failed to open vhost device. 57 VhostUserOpen(VhostError), 58 /// Connection to socket failed. 59 VhostUserConnect, 60 /// Get features failed. 61 VhostUserGetFeatures(VhostError), 62 /// Get queue max number failed. 63 VhostUserGetQueueMaxNum(VhostError), 64 /// Get protocol features failed. 65 VhostUserGetProtocolFeatures(VhostError), 66 /// Get vring base failed. 67 VhostUserGetVringBase(VhostError), 68 /// Vhost-user Backend not support vhost-user protocol. 69 VhostUserProtocolNotSupport, 70 /// Set owner failed. 71 VhostUserSetOwner(VhostError), 72 /// Reset owner failed. 73 VhostUserResetOwner(VhostError), 74 /// Set features failed. 75 VhostUserSetFeatures(VhostError), 76 /// Set protocol features failed. 77 VhostUserSetProtocolFeatures(VhostError), 78 /// Set mem table failed. 79 VhostUserSetMemTable(VhostError), 80 /// Set vring num failed. 81 VhostUserSetVringNum(VhostError), 82 /// Set vring addr failed. 83 VhostUserSetVringAddr(VhostError), 84 /// Set vring base failed. 85 VhostUserSetVringBase(VhostError), 86 /// Set vring call failed. 87 VhostUserSetVringCall(VhostError), 88 /// Set vring kick failed. 89 VhostUserSetVringKick(VhostError), 90 /// Set vring enable failed. 91 VhostUserSetVringEnable(VhostError), 92 /// Failed to create vhost eventfd. 93 VhostIrqCreate(io::Error), 94 /// Failed to read vhost eventfd. 95 VhostIrqRead(io::Error), 96 /// Failed to read vhost eventfd. 97 VhostUserMemoryRegion(MmapError), 98 /// Failed to create the master request handler from slave. 99 MasterReqHandlerCreation(vhost::vhost_user::Error), 100 /// Set slave request fd failed. 101 VhostUserSetSlaveRequestFd(vhost::Error), 102 /// Add memory region failed. 103 VhostUserAddMemReg(VhostError), 104 /// Failed getting the configuration. 105 VhostUserGetConfig(VhostError), 106 /// Failed setting the configuration. 107 VhostUserSetConfig(VhostError), 108 /// Failed getting inflight shm log. 109 VhostUserGetInflight(VhostError), 110 /// Failed setting inflight shm log. 111 VhostUserSetInflight(VhostError), 112 /// Failed setting the log base. 113 VhostUserSetLogBase(VhostError), 114 /// Invalid used address. 115 UsedAddress, 116 /// Invalid features provided from vhost-user backend 117 InvalidFeatures, 118 /// Missing file descriptor for the region. 119 MissingRegionFd, 120 /// Missing IrqFd 121 MissingIrqFd, 122 /// Failed getting the available index. 123 GetAvailableIndex(VirtioError), 124 /// Migration is not supported by this vhost-user device. 125 MigrationNotSupported, 126 /// Failed creating memfd. 127 MemfdCreate(io::Error), 128 /// Failed truncating the file size to the expected size. 129 SetFileSize(io::Error), 130 /// Failed to set the seals on the file. 131 SetSeals(io::Error), 132 /// Failed creating new mmap region 133 NewMmapRegion(MmapRegionError), 134 /// Could not find the shm log region 135 MissingShmLogRegion, 136 } 137 type Result<T> = std::result::Result<T, Error>; 138 139 pub const DEFAULT_VIRTIO_FEATURES: u64 = 1 << VIRTIO_F_RING_INDIRECT_DESC 140 | 1 << VIRTIO_F_RING_EVENT_IDX 141 | 1 << VIRTIO_F_VERSION_1 142 | 1 << VIRTIO_F_IN_ORDER 143 | 1 << VIRTIO_F_ORDER_PLATFORM 144 | 1 << VIRTIO_F_NOTIFICATION_DATA 145 | VhostUserVirtioFeatures::PROTOCOL_FEATURES.bits(); 146 147 const HUP_CONNECTION_EVENT: u16 = EPOLL_HELPER_EVENT_LAST + 1; 148 const SLAVE_REQ_EVENT: u16 = EPOLL_HELPER_EVENT_LAST + 2; 149 150 #[derive(Default)] 151 pub struct Inflight { 152 pub info: VhostUserInflight, 153 pub fd: Option<std::fs::File>, 154 } 155 156 pub struct VhostUserEpollHandler<S: VhostUserMasterReqHandler> { 157 pub vu: Arc<Mutex<VhostUserHandle>>, 158 pub mem: GuestMemoryAtomic<GuestMemoryMmap>, 159 pub kill_evt: EventFd, 160 pub pause_evt: EventFd, 161 pub queues: Vec<Queue>, 162 pub queue_evts: Vec<EventFd>, 163 pub virtio_interrupt: Arc<dyn VirtioInterrupt>, 164 pub acked_features: u64, 165 pub acked_protocol_features: u64, 166 pub socket_path: String, 167 pub server: bool, 168 pub slave_req_handler: Option<MasterReqHandler<S>>, 169 pub inflight: Option<Inflight>, 170 } 171 172 impl<S: VhostUserMasterReqHandler> VhostUserEpollHandler<S> { 173 pub fn run( 174 &mut self, 175 paused: Arc<AtomicBool>, 176 paused_sync: Arc<Barrier>, 177 ) -> std::result::Result<(), EpollHelperError> { 178 let mut helper = EpollHelper::new(&self.kill_evt, &self.pause_evt)?; 179 helper.add_event_custom( 180 self.vu.lock().unwrap().socket_handle().as_raw_fd(), 181 HUP_CONNECTION_EVENT, 182 epoll::Events::EPOLLHUP, 183 )?; 184 185 if let Some(slave_req_handler) = &self.slave_req_handler { 186 helper.add_event(slave_req_handler.as_raw_fd(), SLAVE_REQ_EVENT)?; 187 } 188 189 helper.run(paused, paused_sync, self)?; 190 191 Ok(()) 192 } 193 194 fn reconnect(&mut self, helper: &mut EpollHelper) -> std::result::Result<(), EpollHelperError> { 195 helper.del_event_custom( 196 self.vu.lock().unwrap().socket_handle().as_raw_fd(), 197 HUP_CONNECTION_EVENT, 198 epoll::Events::EPOLLHUP, 199 )?; 200 201 let mut vhost_user = VhostUserHandle::connect_vhost_user( 202 self.server, 203 &self.socket_path, 204 self.queues.len() as u64, 205 true, 206 ) 207 .map_err(|e| { 208 EpollHelperError::IoError(std::io::Error::new( 209 std::io::ErrorKind::Other, 210 format!("failed connecting vhost-user backend{:?}", e), 211 )) 212 })?; 213 214 // Initialize the backend 215 vhost_user 216 .reinitialize_vhost_user( 217 self.mem.memory().deref(), 218 self.queues.clone(), 219 self.queue_evts 220 .iter() 221 .map(|q| q.try_clone().unwrap()) 222 .collect(), 223 &self.virtio_interrupt, 224 self.acked_features, 225 self.acked_protocol_features, 226 &self.slave_req_handler, 227 self.inflight.as_mut(), 228 ) 229 .map_err(|e| { 230 EpollHelperError::IoError(std::io::Error::new( 231 std::io::ErrorKind::Other, 232 format!("failed reconnecting vhost-user backend{:?}", e), 233 )) 234 })?; 235 236 helper.add_event_custom( 237 vhost_user.socket_handle().as_raw_fd(), 238 HUP_CONNECTION_EVENT, 239 epoll::Events::EPOLLHUP, 240 )?; 241 242 // Update vhost-user reference 243 let mut vu = self.vu.lock().unwrap(); 244 *vu = vhost_user; 245 246 Ok(()) 247 } 248 } 249 250 impl<S: VhostUserMasterReqHandler> EpollHelperHandler for VhostUserEpollHandler<S> { 251 fn handle_event(&mut self, helper: &mut EpollHelper, event: &epoll::Event) -> bool { 252 let ev_type = event.data as u16; 253 match ev_type { 254 HUP_CONNECTION_EVENT => { 255 if let Err(e) = self.reconnect(helper) { 256 error!("failed to reconnect vhost-user backend: {:?}", e); 257 return true; 258 } 259 } 260 SLAVE_REQ_EVENT => { 261 if let Some(slave_req_handler) = self.slave_req_handler.as_mut() { 262 if let Err(e) = slave_req_handler.handle_request() { 263 error!("Failed to handle request from vhost-user backend: {:?}", e); 264 return true; 265 } 266 } 267 } 268 _ => { 269 error!("Unknown event for vhost-user thread"); 270 return true; 271 } 272 } 273 274 false 275 } 276 } 277