1 // Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 // 3 // Portions Copyright 2017 The Chromium OS Authors. All rights reserved. 4 // Use of this source code is governed by a BSD-style license that can be 5 // found in the LICENSE-BSD-3-Clause file. 6 // 7 // Copyright © 2020 Intel Corporation 8 // 9 // SPDX-License-Identifier: Apache-2.0 AND BSD-3-Clause 10 11 use std::fs::File; 12 use std::os::unix::io::{AsRawFd, FromRawFd, RawFd}; 13 use std::sync::atomic::{AtomicBool, Ordering}; 14 use std::sync::{Arc, Barrier}; 15 use std::thread; 16 use thiserror::Error; 17 use vmm_sys_util::eventfd::EventFd; 18 19 pub struct EpollHelper { 20 pause_evt: EventFd, 21 epoll_file: File, 22 } 23 24 #[derive(Error, Debug)] 25 pub enum EpollHelperError { 26 #[error("Failed to create Fd: {0}")] 27 CreateFd(std::io::Error), 28 #[error("Failed to epoll_ctl: {0}")] 29 Ctl(std::io::Error), 30 #[error("IO error: {0}")] 31 IoError(std::io::Error), 32 #[error("Failed to epoll_wait: {0}")] 33 Wait(std::io::Error), 34 #[error("Failed to get virtio-queue index: {0}")] 35 QueueRingIndex(virtio_queue::Error), 36 #[error("Failed to handle virtio device events: {0}")] 37 HandleEvent(anyhow::Error), 38 #[error("Failed to handle timeout: {0}")] 39 HandleTimeout(anyhow::Error), 40 } 41 42 pub const EPOLL_HELPER_EVENT_PAUSE: u16 = 0; 43 pub const EPOLL_HELPER_EVENT_KILL: u16 = 1; 44 pub const EPOLL_HELPER_EVENT_LAST: u16 = 15; 45 46 pub trait EpollHelperHandler { 47 // Handle one event at a time. The EpollHelper iterates over a list of 48 // events that have been returned by epoll_wait(). For each event, the 49 // current method is invoked to let the implementation decide how to process 50 // the incoming event. 51 fn handle_event( 52 &mut self, 53 helper: &mut EpollHelper, 54 event: &epoll::Event, 55 ) -> Result<(), EpollHelperError>; 56 57 // This method is only invoked if the EpollHelper was configured to call 58 // epoll_wait() with a valid timeout (different from -1), meaning the call 59 // won't block forever. When the timeout is reached, and if no even has been 60 // triggered, this function will be called to let the implementation decide 61 // how to interpret such situation. By default, it provides a no-op 62 // implementation. 63 fn handle_timeout(&mut self, _helper: &mut EpollHelper) -> Result<(), EpollHelperError> { 64 Ok(()) 65 } 66 67 // In some situations, it might be useful to know the full list of events 68 // triggered while waiting on epoll_wait(). And having this list provided 69 // prior to the iterations over each event might help make some informed 70 // decisions. This function should not replace handle_event(), otherwise it 71 // would completely defeat the purpose of having the loop being factorized 72 // through the EpollHelper structure. 73 fn event_list( 74 &mut self, 75 _helper: &mut EpollHelper, 76 _events: &[epoll::Event], 77 ) -> Result<(), EpollHelperError> { 78 Ok(()) 79 } 80 } 81 82 impl EpollHelper { 83 pub fn new( 84 kill_evt: &EventFd, 85 pause_evt: &EventFd, 86 ) -> std::result::Result<Self, EpollHelperError> { 87 // Create the epoll file descriptor 88 let epoll_fd = epoll::create(true).map_err(EpollHelperError::CreateFd)?; 89 // Use 'File' to enforce closing on 'epoll_fd' 90 let epoll_file = unsafe { File::from_raw_fd(epoll_fd) }; 91 92 let mut helper = Self { 93 pause_evt: pause_evt.try_clone().unwrap(), 94 epoll_file, 95 }; 96 97 helper.add_event(kill_evt.as_raw_fd(), EPOLL_HELPER_EVENT_KILL)?; 98 helper.add_event(pause_evt.as_raw_fd(), EPOLL_HELPER_EVENT_PAUSE)?; 99 Ok(helper) 100 } 101 102 pub fn add_event(&mut self, fd: RawFd, id: u16) -> std::result::Result<(), EpollHelperError> { 103 self.add_event_custom(fd, id, epoll::Events::EPOLLIN) 104 } 105 106 pub fn add_event_custom( 107 &mut self, 108 fd: RawFd, 109 id: u16, 110 evts: epoll::Events, 111 ) -> std::result::Result<(), EpollHelperError> { 112 epoll::ctl( 113 self.epoll_file.as_raw_fd(), 114 epoll::ControlOptions::EPOLL_CTL_ADD, 115 fd, 116 epoll::Event::new(evts, id.into()), 117 ) 118 .map_err(EpollHelperError::Ctl) 119 } 120 121 pub fn mod_event_custom( 122 &mut self, 123 fd: RawFd, 124 id: u16, 125 evts: epoll::Events, 126 ) -> std::result::Result<(), EpollHelperError> { 127 epoll::ctl( 128 self.epoll_file.as_raw_fd(), 129 epoll::ControlOptions::EPOLL_CTL_MOD, 130 fd, 131 epoll::Event::new(evts, id.into()), 132 ) 133 .map_err(EpollHelperError::Ctl) 134 } 135 136 pub fn del_event_custom( 137 &mut self, 138 fd: RawFd, 139 id: u16, 140 evts: epoll::Events, 141 ) -> std::result::Result<(), EpollHelperError> { 142 epoll::ctl( 143 self.epoll_file.as_raw_fd(), 144 epoll::ControlOptions::EPOLL_CTL_DEL, 145 fd, 146 epoll::Event::new(evts, id.into()), 147 ) 148 .map_err(EpollHelperError::Ctl) 149 } 150 151 #[cfg(not(fuzzing))] 152 pub fn run( 153 &mut self, 154 paused: Arc<AtomicBool>, 155 paused_sync: Arc<Barrier>, 156 handler: &mut dyn EpollHelperHandler, 157 ) -> std::result::Result<(), EpollHelperError> { 158 self.run_with_timeout(paused, paused_sync, handler, -1, false) 159 } 160 161 pub fn run_with_timeout( 162 &mut self, 163 paused: Arc<AtomicBool>, 164 paused_sync: Arc<Barrier>, 165 handler: &mut dyn EpollHelperHandler, 166 timeout: i32, 167 enable_event_list: bool, 168 ) -> std::result::Result<(), EpollHelperError> { 169 const EPOLL_EVENTS_LEN: usize = 100; 170 let mut events = vec![epoll::Event::new(epoll::Events::empty(), 0); EPOLL_EVENTS_LEN]; 171 172 // Before jumping into the epoll loop, check if the device is expected 173 // to be in a paused state. This is helpful for the restore code path 174 // as the device thread should not start processing anything before the 175 // device has been resumed. 176 while paused.load(Ordering::SeqCst) { 177 thread::park(); 178 } 179 180 loop { 181 let num_events = 182 match epoll::wait(self.epoll_file.as_raw_fd(), timeout, &mut events[..]) { 183 Ok(res) => res, 184 Err(e) => { 185 if e.kind() == std::io::ErrorKind::Interrupted { 186 // It's well defined from the epoll_wait() syscall 187 // documentation that the epoll loop can be interrupted 188 // before any of the requested events occurred or the 189 // timeout expired. In both those cases, epoll_wait() 190 // returns an error of type EINTR, but this should not 191 // be considered as a regular error. Instead it is more 192 // appropriate to retry, by calling into epoll_wait(). 193 continue; 194 } 195 return Err(EpollHelperError::Wait(e)); 196 } 197 }; 198 199 if num_events == 0 { 200 // This case happens when the timeout is reached before any of 201 // the registered events is triggered. 202 handler.handle_timeout(self)?; 203 continue; 204 } 205 206 if enable_event_list { 207 handler.event_list(self, &events[..num_events])?; 208 } 209 210 for event in events.iter().take(num_events) { 211 let ev_type = event.data as u16; 212 213 match ev_type { 214 EPOLL_HELPER_EVENT_KILL => { 215 info!("KILL_EVENT received, stopping epoll loop"); 216 return Ok(()); 217 } 218 EPOLL_HELPER_EVENT_PAUSE => { 219 info!("PAUSE_EVENT received, pausing epoll loop"); 220 221 // Acknowledge the pause is effective by using the 222 // paused_sync barrier. 223 paused_sync.wait(); 224 225 // We loop here to handle spurious park() returns. 226 // Until we have not resumed, the paused boolean will 227 // be true. 228 while paused.load(Ordering::SeqCst) { 229 thread::park(); 230 } 231 232 // Drain pause event after the device has been resumed. 233 // This ensures the pause event has been seen by each 234 // thread related to this virtio device. 235 let _ = self.pause_evt.read(); 236 } 237 _ => { 238 handler.handle_event(self, event)?; 239 } 240 } 241 } 242 } 243 } 244 245 #[cfg(fuzzing)] 246 // Require to have a 'queue_evt' being kicked before calling 247 // and return when no epoll events are active 248 pub fn run( 249 &mut self, 250 paused: Arc<AtomicBool>, 251 paused_sync: Arc<Barrier>, 252 handler: &mut dyn EpollHelperHandler, 253 ) -> std::result::Result<(), EpollHelperError> { 254 const EPOLL_EVENTS_LEN: usize = 100; 255 let mut events = vec![epoll::Event::new(epoll::Events::empty(), 0); EPOLL_EVENTS_LEN]; 256 257 loop { 258 let num_events = match epoll::wait(self.epoll_file.as_raw_fd(), 0, &mut events[..]) { 259 Ok(res) => res, 260 Err(e) => { 261 if e.kind() == std::io::ErrorKind::Interrupted { 262 // It's well defined from the epoll_wait() syscall 263 // documentation that the epoll loop can be interrupted 264 // before any of the requested events occurred or the 265 // timeout expired. In both those cases, epoll_wait() 266 // returns an error of type EINTR, but this should not 267 // be considered as a regular error. Instead it is more 268 // appropriate to retry, by calling into epoll_wait(). 269 continue; 270 } 271 return Err(EpollHelperError::Wait(e)); 272 } 273 }; 274 275 // Return when no epoll events are active 276 if num_events == 0 { 277 return Ok(()); 278 } 279 280 for event in events.iter().take(num_events) { 281 let ev_type = event.data as u16; 282 283 match ev_type { 284 EPOLL_HELPER_EVENT_KILL => { 285 info!("KILL_EVENT received, stopping epoll loop"); 286 return Ok(()); 287 } 288 EPOLL_HELPER_EVENT_PAUSE => { 289 info!("PAUSE_EVENT received, pausing epoll loop"); 290 291 // Acknowledge the pause is effective by using the 292 // paused_sync barrier. 293 paused_sync.wait(); 294 295 // We loop here to handle spurious park() returns. 296 // Until we have not resumed, the paused boolean will 297 // be true. 298 while paused.load(Ordering::SeqCst) { 299 thread::park(); 300 } 301 302 // Drain pause event after the device has been resumed. 303 // This ensures the pause event has been seen by each 304 // thread related to this virtio device. 305 let _ = self.pause_evt.read(); 306 } 307 _ => { 308 handler.handle_event(self, event)?; 309 } 310 } 311 } 312 } 313 } 314 } 315 316 impl AsRawFd for EpollHelper { 317 fn as_raw_fd(&self) -> RawFd { 318 self.epoll_file.as_raw_fd() 319 } 320 } 321