1 // Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 // 3 // Portions Copyright 2017 The Chromium OS Authors. All rights reserved. 4 // Use of this source code is governed by a BSD-style license that can be 5 // found in the LICENSE-BSD-3-Clause file. 6 // 7 // Copyright © 2020 Intel Corporation 8 // 9 // SPDX-License-Identifier: Apache-2.0 AND BSD-3-Clause 10 11 use std::fs::File; 12 use std::os::unix::io::{AsRawFd, FromRawFd, RawFd}; 13 use std::sync::atomic::{AtomicBool, Ordering}; 14 use std::sync::{Arc, Barrier}; 15 use std::thread; 16 use thiserror::Error; 17 use vmm_sys_util::eventfd::EventFd; 18 19 pub struct EpollHelper { 20 pause_evt: EventFd, 21 epoll_file: File, 22 } 23 24 #[derive(Error, Debug)] 25 pub enum EpollHelperError { 26 #[error("Failed to create Fd: {0}")] 27 CreateFd(std::io::Error), 28 #[error("Failed to epoll_ctl: {0}")] 29 Ctl(std::io::Error), 30 #[error("IO error: {0}")] 31 IoError(std::io::Error), 32 #[error("Failed to epoll_wait: {0}")] 33 Wait(std::io::Error), 34 #[error("Failed to get virtio-queue index: {0}")] 35 QueueRingIndex(virtio_queue::Error), 36 #[error("Failed to handle virtio device events: {0}")] 37 HandleEvent(anyhow::Error), 38 #[error("Failed to handle timeout: {0}")] 39 HandleTimeout(anyhow::Error), 40 } 41 42 pub const EPOLL_HELPER_EVENT_PAUSE: u16 = 0; 43 pub const EPOLL_HELPER_EVENT_KILL: u16 = 1; 44 pub const EPOLL_HELPER_EVENT_LAST: u16 = 15; 45 46 pub trait EpollHelperHandler { 47 // Handle one event at a time. The EpollHelper iterates over a list of 48 // events that have been returned by epoll_wait(). For each event, the 49 // current method is invoked to let the implementation decide how to process 50 // the incoming event. 51 fn handle_event( 52 &mut self, 53 helper: &mut EpollHelper, 54 event: &epoll::Event, 55 ) -> Result<(), EpollHelperError>; 56 57 // This method is only invoked if the EpollHelper was configured to call 58 // epoll_wait() with a valid timeout (different from -1), meaning the call 59 // won't block forever. When the timeout is reached, and if no even has been 60 // triggered, this function will be called to let the implementation decide 61 // how to interpret such situation. By default, it provides a no-op 62 // implementation. 63 fn handle_timeout(&mut self, _helper: &mut EpollHelper) -> Result<(), EpollHelperError> { 64 Ok(()) 65 } 66 67 // In some situations, it might be useful to know the full list of events 68 // triggered while waiting on epoll_wait(). And having this list provided 69 // prior to the iterations over each event might help make some informed 70 // decisions. This function should not replace handle_event(), otherwise it 71 // would completely defeat the purpose of having the loop being factorized 72 // through the EpollHelper structure. 73 fn event_list( 74 &mut self, 75 _helper: &mut EpollHelper, 76 _events: &[epoll::Event], 77 ) -> Result<(), EpollHelperError> { 78 Ok(()) 79 } 80 } 81 82 impl EpollHelper { 83 pub fn new( 84 kill_evt: &EventFd, 85 pause_evt: &EventFd, 86 ) -> std::result::Result<Self, EpollHelperError> { 87 // Create the epoll file descriptor 88 let epoll_fd = epoll::create(true).map_err(EpollHelperError::CreateFd)?; 89 // Use 'File' to enforce closing on 'epoll_fd' 90 // SAFETY: epoll_fd is a valid fd 91 let epoll_file = unsafe { File::from_raw_fd(epoll_fd) }; 92 93 let mut helper = Self { 94 pause_evt: pause_evt.try_clone().unwrap(), 95 epoll_file, 96 }; 97 98 helper.add_event(kill_evt.as_raw_fd(), EPOLL_HELPER_EVENT_KILL)?; 99 helper.add_event(pause_evt.as_raw_fd(), EPOLL_HELPER_EVENT_PAUSE)?; 100 Ok(helper) 101 } 102 103 pub fn add_event(&mut self, fd: RawFd, id: u16) -> std::result::Result<(), EpollHelperError> { 104 self.add_event_custom(fd, id, epoll::Events::EPOLLIN) 105 } 106 107 pub fn add_event_custom( 108 &mut self, 109 fd: RawFd, 110 id: u16, 111 evts: epoll::Events, 112 ) -> std::result::Result<(), EpollHelperError> { 113 epoll::ctl( 114 self.epoll_file.as_raw_fd(), 115 epoll::ControlOptions::EPOLL_CTL_ADD, 116 fd, 117 epoll::Event::new(evts, id.into()), 118 ) 119 .map_err(EpollHelperError::Ctl) 120 } 121 122 pub fn mod_event_custom( 123 &mut self, 124 fd: RawFd, 125 id: u16, 126 evts: epoll::Events, 127 ) -> std::result::Result<(), EpollHelperError> { 128 epoll::ctl( 129 self.epoll_file.as_raw_fd(), 130 epoll::ControlOptions::EPOLL_CTL_MOD, 131 fd, 132 epoll::Event::new(evts, id.into()), 133 ) 134 .map_err(EpollHelperError::Ctl) 135 } 136 137 pub fn del_event_custom( 138 &mut self, 139 fd: RawFd, 140 id: u16, 141 evts: epoll::Events, 142 ) -> std::result::Result<(), EpollHelperError> { 143 epoll::ctl( 144 self.epoll_file.as_raw_fd(), 145 epoll::ControlOptions::EPOLL_CTL_DEL, 146 fd, 147 epoll::Event::new(evts, id.into()), 148 ) 149 .map_err(EpollHelperError::Ctl) 150 } 151 152 pub fn run( 153 &mut self, 154 paused: Arc<AtomicBool>, 155 paused_sync: Arc<Barrier>, 156 handler: &mut dyn EpollHelperHandler, 157 ) -> std::result::Result<(), EpollHelperError> { 158 self.run_with_timeout(paused, paused_sync, handler, -1, false) 159 } 160 161 #[cfg(not(fuzzing))] 162 pub fn run_with_timeout( 163 &mut self, 164 paused: Arc<AtomicBool>, 165 paused_sync: Arc<Barrier>, 166 handler: &mut dyn EpollHelperHandler, 167 timeout: i32, 168 enable_event_list: bool, 169 ) -> std::result::Result<(), EpollHelperError> { 170 const EPOLL_EVENTS_LEN: usize = 100; 171 let mut events = vec![epoll::Event::new(epoll::Events::empty(), 0); EPOLL_EVENTS_LEN]; 172 173 // Before jumping into the epoll loop, check if the device is expected 174 // to be in a paused state. This is helpful for the restore code path 175 // as the device thread should not start processing anything before the 176 // device has been resumed. 177 while paused.load(Ordering::SeqCst) { 178 thread::park(); 179 } 180 181 loop { 182 let num_events = 183 match epoll::wait(self.epoll_file.as_raw_fd(), timeout, &mut events[..]) { 184 Ok(res) => res, 185 Err(e) => { 186 if e.kind() == std::io::ErrorKind::Interrupted { 187 // It's well defined from the epoll_wait() syscall 188 // documentation that the epoll loop can be interrupted 189 // before any of the requested events occurred or the 190 // timeout expired. In both those cases, epoll_wait() 191 // returns an error of type EINTR, but this should not 192 // be considered as a regular error. Instead it is more 193 // appropriate to retry, by calling into epoll_wait(). 194 continue; 195 } 196 return Err(EpollHelperError::Wait(e)); 197 } 198 }; 199 200 if num_events == 0 { 201 // This case happens when the timeout is reached before any of 202 // the registered events is triggered. 203 handler.handle_timeout(self)?; 204 continue; 205 } 206 207 if enable_event_list { 208 handler.event_list(self, &events[..num_events])?; 209 } 210 211 for event in events.iter().take(num_events) { 212 let ev_type = event.data as u16; 213 214 match ev_type { 215 EPOLL_HELPER_EVENT_KILL => { 216 info!("KILL_EVENT received, stopping epoll loop"); 217 return Ok(()); 218 } 219 EPOLL_HELPER_EVENT_PAUSE => { 220 info!("PAUSE_EVENT received, pausing epoll loop"); 221 222 // Acknowledge the pause is effective by using the 223 // paused_sync barrier. 224 paused_sync.wait(); 225 226 // We loop here to handle spurious park() returns. 227 // Until we have not resumed, the paused boolean will 228 // be true. 229 while paused.load(Ordering::SeqCst) { 230 thread::park(); 231 } 232 233 // Drain pause event after the device has been resumed. 234 // This ensures the pause event has been seen by each 235 // thread related to this virtio device. 236 let _ = self.pause_evt.read(); 237 } 238 _ => { 239 handler.handle_event(self, event)?; 240 } 241 } 242 } 243 } 244 } 245 246 #[cfg(fuzzing)] 247 // Require to have a 'queue_evt' being kicked before calling 248 // and return when no epoll events are active 249 pub fn run_with_timeout( 250 &mut self, 251 paused: Arc<AtomicBool>, 252 paused_sync: Arc<Barrier>, 253 handler: &mut dyn EpollHelperHandler, 254 _timeout: i32, 255 _enable_event_list: bool, 256 ) -> std::result::Result<(), EpollHelperError> { 257 const EPOLL_EVENTS_LEN: usize = 100; 258 let mut events = vec![epoll::Event::new(epoll::Events::empty(), 0); EPOLL_EVENTS_LEN]; 259 260 loop { 261 let num_events = match epoll::wait(self.epoll_file.as_raw_fd(), 0, &mut events[..]) { 262 Ok(res) => res, 263 Err(e) => { 264 if e.kind() == std::io::ErrorKind::Interrupted { 265 // It's well defined from the epoll_wait() syscall 266 // documentation that the epoll loop can be interrupted 267 // before any of the requested events occurred or the 268 // timeout expired. In both those cases, epoll_wait() 269 // returns an error of type EINTR, but this should not 270 // be considered as a regular error. Instead it is more 271 // appropriate to retry, by calling into epoll_wait(). 272 continue; 273 } 274 return Err(EpollHelperError::Wait(e)); 275 } 276 }; 277 278 // Return when no epoll events are active 279 if num_events == 0 { 280 return Ok(()); 281 } 282 283 for event in events.iter().take(num_events) { 284 let ev_type = event.data as u16; 285 286 match ev_type { 287 EPOLL_HELPER_EVENT_KILL => { 288 info!("KILL_EVENT received, stopping epoll loop"); 289 return Ok(()); 290 } 291 EPOLL_HELPER_EVENT_PAUSE => { 292 info!("PAUSE_EVENT received, pausing epoll loop"); 293 294 // Acknowledge the pause is effective by using the 295 // paused_sync barrier. 296 paused_sync.wait(); 297 298 // We loop here to handle spurious park() returns. 299 // Until we have not resumed, the paused boolean will 300 // be true. 301 while paused.load(Ordering::SeqCst) { 302 thread::park(); 303 } 304 305 // Drain pause event after the device has been resumed. 306 // This ensures the pause event has been seen by each 307 // thread related to this virtio device. 308 let _ = self.pause_evt.read(); 309 } 310 _ => { 311 handler.handle_event(self, event)?; 312 } 313 } 314 } 315 } 316 } 317 } 318 319 impl AsRawFd for EpollHelper { 320 fn as_raw_fd(&self) -> RawFd { 321 self.epoll_file.as_raw_fd() 322 } 323 } 324