1 // Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 // 3 // Portions Copyright 2017 The Chromium OS Authors. All rights reserved. 4 // Use of this source code is governed by a BSD-style license that can be 5 // found in the LICENSE-BSD-3-Clause file. 6 // 7 // Copyright © 2020 Intel Corporation 8 // 9 // SPDX-License-Identifier: Apache-2.0 AND BSD-3-Clause 10 11 use std::fs::File; 12 use std::os::unix::io::{AsRawFd, FromRawFd, RawFd}; 13 use std::sync::atomic::{AtomicBool, Ordering}; 14 use std::sync::{Arc, Barrier}; 15 use std::thread; 16 17 use thiserror::Error; 18 use vmm_sys_util::eventfd::EventFd; 19 20 pub struct EpollHelper { 21 pause_evt: EventFd, 22 epoll_file: File, 23 } 24 25 #[derive(Error, Debug)] 26 pub enum EpollHelperError { 27 #[error("Failed to create Fd")] 28 CreateFd(#[source] std::io::Error), 29 #[error("Failed to epoll_ctl")] 30 Ctl(#[source] std::io::Error), 31 #[error("IO error")] 32 IoError(#[source] std::io::Error), 33 #[error("Failed to epoll_wait")] 34 Wait(#[source] std::io::Error), 35 #[error("Failed to get virtio-queue index")] 36 QueueRingIndex(#[source] virtio_queue::Error), 37 #[error("Failed to handle virtio device events")] 38 HandleEvent(#[source] anyhow::Error), 39 #[error("Failed to handle timeout")] 40 HandleTimeout(#[source] anyhow::Error), 41 } 42 43 pub const EPOLL_HELPER_EVENT_PAUSE: u16 = 0; 44 pub const EPOLL_HELPER_EVENT_KILL: u16 = 1; 45 pub const EPOLL_HELPER_EVENT_LAST: u16 = 15; 46 47 pub trait EpollHelperHandler { 48 // Handle one event at a time. The EpollHelper iterates over a list of 49 // events that have been returned by epoll_wait(). For each event, the 50 // current method is invoked to let the implementation decide how to process 51 // the incoming event. 52 fn handle_event( 53 &mut self, 54 helper: &mut EpollHelper, 55 event: &epoll::Event, 56 ) -> Result<(), EpollHelperError>; 57 58 // This method is only invoked if the EpollHelper was configured to call 59 // epoll_wait() with a valid timeout (different from -1), meaning the call 60 // won't block forever. When the timeout is reached, and if no even has been 61 // triggered, this function will be called to let the implementation decide 62 // how to interpret such situation. By default, it provides a no-op 63 // implementation. 64 fn handle_timeout(&mut self, _helper: &mut EpollHelper) -> Result<(), EpollHelperError> { 65 Ok(()) 66 } 67 68 // In some situations, it might be useful to know the full list of events 69 // triggered while waiting on epoll_wait(). And having this list provided 70 // prior to the iterations over each event might help make some informed 71 // decisions. This function should not replace handle_event(), otherwise it 72 // would completely defeat the purpose of having the loop being factorized 73 // through the EpollHelper structure. 74 fn event_list( 75 &mut self, 76 _helper: &mut EpollHelper, 77 _events: &[epoll::Event], 78 ) -> Result<(), EpollHelperError> { 79 Ok(()) 80 } 81 } 82 83 impl EpollHelper { 84 pub fn new( 85 kill_evt: &EventFd, 86 pause_evt: &EventFd, 87 ) -> std::result::Result<Self, EpollHelperError> { 88 // Create the epoll file descriptor 89 let epoll_fd = epoll::create(true).map_err(EpollHelperError::CreateFd)?; 90 // Use 'File' to enforce closing on 'epoll_fd' 91 // SAFETY: epoll_fd is a valid fd 92 let epoll_file = unsafe { File::from_raw_fd(epoll_fd) }; 93 94 let mut helper = Self { 95 pause_evt: pause_evt.try_clone().unwrap(), 96 epoll_file, 97 }; 98 99 helper.add_event(kill_evt.as_raw_fd(), EPOLL_HELPER_EVENT_KILL)?; 100 helper.add_event(pause_evt.as_raw_fd(), EPOLL_HELPER_EVENT_PAUSE)?; 101 Ok(helper) 102 } 103 104 pub fn add_event(&mut self, fd: RawFd, id: u16) -> std::result::Result<(), EpollHelperError> { 105 self.add_event_custom(fd, id, epoll::Events::EPOLLIN) 106 } 107 108 pub fn add_event_custom( 109 &mut self, 110 fd: RawFd, 111 id: u16, 112 evts: epoll::Events, 113 ) -> std::result::Result<(), EpollHelperError> { 114 epoll::ctl( 115 self.epoll_file.as_raw_fd(), 116 epoll::ControlOptions::EPOLL_CTL_ADD, 117 fd, 118 epoll::Event::new(evts, id.into()), 119 ) 120 .map_err(EpollHelperError::Ctl) 121 } 122 123 pub fn mod_event_custom( 124 &mut self, 125 fd: RawFd, 126 id: u16, 127 evts: epoll::Events, 128 ) -> std::result::Result<(), EpollHelperError> { 129 epoll::ctl( 130 self.epoll_file.as_raw_fd(), 131 epoll::ControlOptions::EPOLL_CTL_MOD, 132 fd, 133 epoll::Event::new(evts, id.into()), 134 ) 135 .map_err(EpollHelperError::Ctl) 136 } 137 138 pub fn del_event_custom( 139 &mut self, 140 fd: RawFd, 141 id: u16, 142 evts: epoll::Events, 143 ) -> std::result::Result<(), EpollHelperError> { 144 epoll::ctl( 145 self.epoll_file.as_raw_fd(), 146 epoll::ControlOptions::EPOLL_CTL_DEL, 147 fd, 148 epoll::Event::new(evts, id.into()), 149 ) 150 .map_err(EpollHelperError::Ctl) 151 } 152 153 pub fn run( 154 &mut self, 155 paused: Arc<AtomicBool>, 156 paused_sync: Arc<Barrier>, 157 handler: &mut dyn EpollHelperHandler, 158 ) -> std::result::Result<(), EpollHelperError> { 159 self.run_with_timeout(paused, paused_sync, handler, -1, false) 160 } 161 162 #[cfg(not(fuzzing))] 163 pub fn run_with_timeout( 164 &mut self, 165 paused: Arc<AtomicBool>, 166 paused_sync: Arc<Barrier>, 167 handler: &mut dyn EpollHelperHandler, 168 timeout: i32, 169 enable_event_list: bool, 170 ) -> std::result::Result<(), EpollHelperError> { 171 const EPOLL_EVENTS_LEN: usize = 100; 172 let mut events = vec![epoll::Event::new(epoll::Events::empty(), 0); EPOLL_EVENTS_LEN]; 173 174 // Before jumping into the epoll loop, check if the device is expected 175 // to be in a paused state. This is helpful for the restore code path 176 // as the device thread should not start processing anything before the 177 // device has been resumed. 178 while paused.load(Ordering::SeqCst) { 179 thread::park(); 180 } 181 182 loop { 183 let num_events = 184 match epoll::wait(self.epoll_file.as_raw_fd(), timeout, &mut events[..]) { 185 Ok(res) => res, 186 Err(e) => { 187 if e.kind() == std::io::ErrorKind::Interrupted { 188 // It's well defined from the epoll_wait() syscall 189 // documentation that the epoll loop can be interrupted 190 // before any of the requested events occurred or the 191 // timeout expired. In both those cases, epoll_wait() 192 // returns an error of type EINTR, but this should not 193 // be considered as a regular error. Instead it is more 194 // appropriate to retry, by calling into epoll_wait(). 195 continue; 196 } 197 return Err(EpollHelperError::Wait(e)); 198 } 199 }; 200 201 if num_events == 0 { 202 // This case happens when the timeout is reached before any of 203 // the registered events is triggered. 204 handler.handle_timeout(self)?; 205 continue; 206 } 207 208 if enable_event_list { 209 handler.event_list(self, &events[..num_events])?; 210 } 211 212 for event in events.iter().take(num_events) { 213 let ev_type = event.data as u16; 214 215 match ev_type { 216 EPOLL_HELPER_EVENT_KILL => { 217 info!("KILL_EVENT received, stopping epoll loop"); 218 return Ok(()); 219 } 220 EPOLL_HELPER_EVENT_PAUSE => { 221 info!("PAUSE_EVENT received, pausing epoll loop"); 222 223 // Acknowledge the pause is effective by using the 224 // paused_sync barrier. 225 paused_sync.wait(); 226 227 // We loop here to handle spurious park() returns. 228 // Until we have not resumed, the paused boolean will 229 // be true. 230 while paused.load(Ordering::SeqCst) { 231 thread::park(); 232 } 233 234 // Drain pause event after the device has been resumed. 235 // This ensures the pause event has been seen by each 236 // thread related to this virtio device. 237 let _ = self.pause_evt.read(); 238 } 239 _ => { 240 handler.handle_event(self, event)?; 241 } 242 } 243 } 244 } 245 } 246 247 #[cfg(fuzzing)] 248 // Require to have a 'queue_evt' being kicked before calling 249 // and return when no epoll events are active 250 pub fn run_with_timeout( 251 &mut self, 252 paused: Arc<AtomicBool>, 253 paused_sync: Arc<Barrier>, 254 handler: &mut dyn EpollHelperHandler, 255 _timeout: i32, 256 _enable_event_list: bool, 257 ) -> std::result::Result<(), EpollHelperError> { 258 const EPOLL_EVENTS_LEN: usize = 100; 259 let mut events = vec![epoll::Event::new(epoll::Events::empty(), 0); EPOLL_EVENTS_LEN]; 260 261 loop { 262 let num_events = match epoll::wait(self.epoll_file.as_raw_fd(), 0, &mut events[..]) { 263 Ok(res) => res, 264 Err(e) => { 265 if e.kind() == std::io::ErrorKind::Interrupted { 266 // It's well defined from the epoll_wait() syscall 267 // documentation that the epoll loop can be interrupted 268 // before any of the requested events occurred or the 269 // timeout expired. In both those cases, epoll_wait() 270 // returns an error of type EINTR, but this should not 271 // be considered as a regular error. Instead it is more 272 // appropriate to retry, by calling into epoll_wait(). 273 continue; 274 } 275 return Err(EpollHelperError::Wait(e)); 276 } 277 }; 278 279 // Return when no epoll events are active 280 if num_events == 0 { 281 return Ok(()); 282 } 283 284 for event in events.iter().take(num_events) { 285 let ev_type = event.data as u16; 286 287 match ev_type { 288 EPOLL_HELPER_EVENT_KILL => { 289 info!("KILL_EVENT received, stopping epoll loop"); 290 return Ok(()); 291 } 292 EPOLL_HELPER_EVENT_PAUSE => { 293 info!("PAUSE_EVENT received, pausing epoll loop"); 294 295 // Acknowledge the pause is effective by using the 296 // paused_sync barrier. 297 paused_sync.wait(); 298 299 // We loop here to handle spurious park() returns. 300 // Until we have not resumed, the paused boolean will 301 // be true. 302 while paused.load(Ordering::SeqCst) { 303 thread::park(); 304 } 305 306 // Drain pause event after the device has been resumed. 307 // This ensures the pause event has been seen by each 308 // thread related to this virtio device. 309 let _ = self.pause_evt.read(); 310 } 311 _ => { 312 handler.handle_event(self, event)?; 313 } 314 } 315 } 316 } 317 } 318 } 319 320 impl AsRawFd for EpollHelper { 321 fn as_raw_fd(&self) -> RawFd { 322 self.epoll_file.as_raw_fd() 323 } 324 } 325