1 // Copyright 2019 Intel Corporation. All Rights Reserved. 2 // SPDX-License-Identifier: Apache-2.0 3 4 use crate::{ 5 EpollHelper, EpollHelperError, EpollHelperHandler, GuestMemoryMmap, Queue, VirtioInterrupt, 6 EPOLL_HELPER_EVENT_LAST, VIRTIO_F_IN_ORDER, VIRTIO_F_NOTIFICATION_DATA, 7 VIRTIO_F_ORDER_PLATFORM, VIRTIO_F_RING_EVENT_IDX, VIRTIO_F_RING_INDIRECT_DESC, 8 VIRTIO_F_VERSION_1, 9 }; 10 use std::io; 11 use std::ops::Deref; 12 use std::os::unix::io::AsRawFd; 13 use std::sync::{atomic::AtomicBool, Arc, Barrier, Mutex}; 14 use vhost::vhost_user::message::{VhostUserInflight, VhostUserVirtioFeatures}; 15 use vhost::vhost_user::{Master, MasterReqHandler, VhostUserMasterReqHandler}; 16 use vhost::Error as VhostError; 17 use vm_memory::{Error as MmapError, GuestAddressSpace, GuestMemoryAtomic}; 18 use vm_virtio::Error as VirtioError; 19 use vmm_sys_util::eventfd::EventFd; 20 use vu_common_ctrl::{connect_vhost_user, reinitialize_vhost_user}; 21 22 pub mod blk; 23 pub mod fs; 24 pub mod net; 25 pub mod vu_common_ctrl; 26 27 pub use self::blk::Blk; 28 pub use self::fs::*; 29 pub use self::net::Net; 30 pub use self::vu_common_ctrl::VhostUserConfig; 31 32 #[derive(Debug)] 33 pub enum Error { 34 /// Failed accepting connection. 35 AcceptConnection(io::Error), 36 /// Invalid available address. 37 AvailAddress, 38 /// Queue number is not correct 39 BadQueueNum, 40 /// Failed binding vhost-user socket. 41 BindSocket(io::Error), 42 /// Creating kill eventfd failed. 43 CreateKillEventFd(io::Error), 44 /// Cloning kill eventfd failed. 45 CloneKillEventFd(io::Error), 46 /// Invalid descriptor table address. 47 DescriptorTableAddress, 48 /// Signal used queue failed. 49 FailedSignalingUsedQueue(io::Error), 50 /// Failed to read vhost eventfd. 51 MemoryRegions(MmapError), 52 /// Failed removing socket path 53 RemoveSocketPath(io::Error), 54 /// Failed to create master. 55 VhostUserCreateMaster(VhostError), 56 /// Failed to open vhost device. 57 VhostUserOpen(VhostError), 58 /// Connection to socket failed. 59 VhostUserConnect, 60 /// Get features failed. 61 VhostUserGetFeatures(VhostError), 62 /// Get queue max number failed. 63 VhostUserGetQueueMaxNum(VhostError), 64 /// Get protocol features failed. 65 VhostUserGetProtocolFeatures(VhostError), 66 /// Vhost-user Backend not support vhost-user protocol. 67 VhostUserProtocolNotSupport, 68 /// Set owner failed. 69 VhostUserSetOwner(VhostError), 70 /// Reset owner failed. 71 VhostUserResetOwner(VhostError), 72 /// Set features failed. 73 VhostUserSetFeatures(VhostError), 74 /// Set protocol features failed. 75 VhostUserSetProtocolFeatures(VhostError), 76 /// Set mem table failed. 77 VhostUserSetMemTable(VhostError), 78 /// Set vring num failed. 79 VhostUserSetVringNum(VhostError), 80 /// Set vring addr failed. 81 VhostUserSetVringAddr(VhostError), 82 /// Set vring base failed. 83 VhostUserSetVringBase(VhostError), 84 /// Set vring call failed. 85 VhostUserSetVringCall(VhostError), 86 /// Set vring kick failed. 87 VhostUserSetVringKick(VhostError), 88 /// Set vring enable failed. 89 VhostUserSetVringEnable(VhostError), 90 /// Failed to create vhost eventfd. 91 VhostIrqCreate(io::Error), 92 /// Failed to read vhost eventfd. 93 VhostIrqRead(io::Error), 94 /// Failed to read vhost eventfd. 95 VhostUserMemoryRegion(MmapError), 96 /// Failed to create the master request handler from slave. 97 MasterReqHandlerCreation(vhost::vhost_user::Error), 98 /// Set slave request fd failed. 99 VhostUserSetSlaveRequestFd(vhost::Error), 100 /// Add memory region failed. 101 VhostUserAddMemReg(VhostError), 102 /// Failed getting the configuration. 103 VhostUserGetConfig(VhostError), 104 /// Failed setting the configuration. 105 VhostUserSetConfig(VhostError), 106 /// Failed getting inflight shm log. 107 VhostUserGetInflight(VhostError), 108 /// Failed setting inflight shm log. 109 VhostUserSetInflight(VhostError), 110 /// Invalid used address. 111 UsedAddress, 112 /// Invalid features provided from vhost-user backend 113 InvalidFeatures, 114 /// Missing file descriptor for the region. 115 MissingRegionFd, 116 /// Missing IrqFd 117 MissingIrqFd, 118 /// Failed getting the available index. 119 GetAvailableIndex(VirtioError), 120 } 121 type Result<T> = std::result::Result<T, Error>; 122 123 pub const DEFAULT_VIRTIO_FEATURES: u64 = 1 << VIRTIO_F_RING_INDIRECT_DESC 124 | 1 << VIRTIO_F_RING_EVENT_IDX 125 | 1 << VIRTIO_F_VERSION_1 126 | 1 << VIRTIO_F_IN_ORDER 127 | 1 << VIRTIO_F_ORDER_PLATFORM 128 | 1 << VIRTIO_F_NOTIFICATION_DATA 129 | VhostUserVirtioFeatures::PROTOCOL_FEATURES.bits(); 130 131 const HUP_CONNECTION_EVENT: u16 = EPOLL_HELPER_EVENT_LAST + 1; 132 const SLAVE_REQ_EVENT: u16 = EPOLL_HELPER_EVENT_LAST + 2; 133 134 #[derive(Default)] 135 pub struct Inflight { 136 pub info: VhostUserInflight, 137 pub fd: Option<std::fs::File>, 138 } 139 140 pub struct VhostUserEpollHandler<S: VhostUserMasterReqHandler> { 141 pub vu: Arc<Mutex<Master>>, 142 pub mem: GuestMemoryAtomic<GuestMemoryMmap>, 143 pub kill_evt: EventFd, 144 pub pause_evt: EventFd, 145 pub queues: Vec<Queue>, 146 pub queue_evts: Vec<EventFd>, 147 pub virtio_interrupt: Arc<dyn VirtioInterrupt>, 148 pub acked_features: u64, 149 pub acked_protocol_features: u64, 150 pub socket_path: String, 151 pub server: bool, 152 pub slave_req_handler: Option<MasterReqHandler<S>>, 153 pub inflight: Option<Inflight>, 154 } 155 156 impl<S: VhostUserMasterReqHandler> VhostUserEpollHandler<S> { 157 pub fn run( 158 &mut self, 159 paused: Arc<AtomicBool>, 160 paused_sync: Arc<Barrier>, 161 ) -> std::result::Result<(), EpollHelperError> { 162 let mut helper = EpollHelper::new(&self.kill_evt, &self.pause_evt)?; 163 helper.add_event_custom( 164 self.vu.lock().unwrap().as_raw_fd(), 165 HUP_CONNECTION_EVENT, 166 epoll::Events::EPOLLHUP, 167 )?; 168 169 if let Some(slave_req_handler) = &self.slave_req_handler { 170 helper.add_event(slave_req_handler.as_raw_fd(), SLAVE_REQ_EVENT)?; 171 } 172 173 helper.run(paused, paused_sync, self)?; 174 175 Ok(()) 176 } 177 178 fn reconnect(&mut self, helper: &mut EpollHelper) -> std::result::Result<(), EpollHelperError> { 179 helper.del_event_custom( 180 self.vu.lock().unwrap().as_raw_fd(), 181 HUP_CONNECTION_EVENT, 182 epoll::Events::EPOLLHUP, 183 )?; 184 185 let mut vhost_user = connect_vhost_user( 186 self.server, 187 &self.socket_path, 188 self.queues.len() as u64, 189 true, 190 ) 191 .map_err(|e| { 192 EpollHelperError::IoError(std::io::Error::new( 193 std::io::ErrorKind::Other, 194 format!("failed connecting vhost-user backend{:?}", e), 195 )) 196 })?; 197 198 // Initialize the backend 199 reinitialize_vhost_user( 200 &mut vhost_user, 201 self.mem.memory().deref(), 202 self.queues.clone(), 203 self.queue_evts 204 .iter() 205 .map(|q| q.try_clone().unwrap()) 206 .collect(), 207 &self.virtio_interrupt, 208 self.acked_features, 209 self.acked_protocol_features, 210 &self.slave_req_handler, 211 self.inflight.as_mut(), 212 ) 213 .map_err(|e| { 214 EpollHelperError::IoError(std::io::Error::new( 215 std::io::ErrorKind::Other, 216 format!("failed reconnecting vhost-user backend{:?}", e), 217 )) 218 })?; 219 220 helper.add_event_custom( 221 vhost_user.as_raw_fd(), 222 HUP_CONNECTION_EVENT, 223 epoll::Events::EPOLLHUP, 224 )?; 225 226 // Update vhost-user reference 227 let mut vu = self.vu.lock().unwrap(); 228 *vu = vhost_user; 229 230 Ok(()) 231 } 232 } 233 234 impl<S: VhostUserMasterReqHandler> EpollHelperHandler for VhostUserEpollHandler<S> { 235 fn handle_event(&mut self, helper: &mut EpollHelper, event: &epoll::Event) -> bool { 236 let ev_type = event.data as u16; 237 match ev_type { 238 HUP_CONNECTION_EVENT => { 239 if let Err(e) = self.reconnect(helper) { 240 error!("failed to reconnect vhost-user backend: {:?}", e); 241 return true; 242 } 243 } 244 SLAVE_REQ_EVENT => { 245 if let Some(slave_req_handler) = self.slave_req_handler.as_mut() { 246 if let Err(e) = slave_req_handler.handle_request() { 247 error!("Failed to handle request from vhost-user backend: {:?}", e); 248 return true; 249 } 250 } 251 } 252 _ => { 253 error!("Unknown event for vhost-user thread"); 254 return true; 255 } 256 } 257 258 false 259 } 260 } 261