182464347SSamuel Ortiz // Copyright 2018 The Chromium OS Authors. All rights reserved. 282464347SSamuel Ortiz // Use of this source code is governed by a BSD-style license that can be 3040ea543SSamuel Ortiz // found in the LICENSE-BSD-3-Clause file. 4040ea543SSamuel Ortiz // 5040ea543SSamuel Ortiz // Copyright © 2019 Intel Corporation 6040ea543SSamuel Ortiz // 7040ea543SSamuel Ortiz // SPDX-License-Identifier: Apache-2.0 AND BSD-3-Clause 882464347SSamuel Ortiz 9dd54883aSRob Bradford use std::collections::HashMap; 10d2625408SRob Bradford use std::io::Write; 11dd54883aSRob Bradford use std::num::Wrapping; 1261e57e1cSRuoqing He use std::sync::atomic::{AtomicBool, Ordering}; 1361e57e1cSRuoqing He use std::sync::{Arc, Barrier}; 1434875249SRob Bradford use std::thread; 1588a9f799SRob Bradford 1688a9f799SRob Bradford use libc::EFD_NONBLOCK; 178eed276dSSebastien Boeuf use virtio_queue::Queue; 18b5bcdbafSBo Chen use vm_memory::{GuestAddress, GuestMemoryAtomic, GuestUsize}; 1934875249SRob Bradford use vm_migration::{MigratableError, Pausable}; 2061e57e1cSRuoqing He use vm_virtio::{AccessPlatform, VirtioDeviceType}; 219caad739SRob Bradford use vmm_sys_util::eventfd::EventFd; 2282464347SSamuel Ortiz 2388a9f799SRob Bradford use crate::{ 2488a9f799SRob Bradford ActivateError, ActivateResult, Error, GuestMemoryMmap, GuestRegionMmap, 2588a9f799SRob Bradford VIRTIO_F_RING_INDIRECT_DESC, 2688a9f799SRob Bradford }; 2788a9f799SRob Bradford 2898d7955eSSebastien Boeuf pub enum VirtioInterruptType { 2998d7955eSSebastien Boeuf Config, 30de3e003eSSebastien Boeuf Queue(u16), 3198d7955eSSebastien Boeuf } 3298d7955eSSebastien Boeuf 33c396bacaSSebastien Boeuf pub trait VirtioInterrupt: Send + Sync { trigger(&self, int_type: VirtioInterruptType) -> std::result::Result<(), std::io::Error>34de3e003eSSebastien Boeuf fn trigger(&self, int_type: VirtioInterruptType) -> std::result::Result<(), std::io::Error>; notifier(&self, _int_type: VirtioInterruptType) -> Option<EventFd>35de3e003eSSebastien Boeuf fn notifier(&self, _int_type: VirtioInterruptType) -> Option<EventFd> { 361f029dd2SSebastien Boeuf None 371f029dd2SSebastien Boeuf } 38c396bacaSSebastien Boeuf } 39d3c7b455SSebastien Boeuf 40e2b38cc0SSebastien Boeuf #[derive(Clone)] 413fb0a02fSSebastien Boeuf pub struct UserspaceMapping { 423fb0a02fSSebastien Boeuf pub host_addr: u64, 433fb0a02fSSebastien Boeuf pub mem_slot: u32, 443fb0a02fSSebastien Boeuf pub addr: GuestAddress, 453fb0a02fSSebastien Boeuf pub len: GuestUsize, 463fb0a02fSSebastien Boeuf pub mergeable: bool, 473fb0a02fSSebastien Boeuf } 483fb0a02fSSebastien Boeuf 493fb0a02fSSebastien Boeuf #[derive(Clone)] 50e2b38cc0SSebastien Boeuf pub struct VirtioSharedMemory { 51e2b38cc0SSebastien Boeuf pub offset: u64, 52e2b38cc0SSebastien Boeuf pub len: u64, 53e2b38cc0SSebastien Boeuf } 54e2b38cc0SSebastien Boeuf 55e2b38cc0SSebastien Boeuf #[derive(Clone)] 56e2b38cc0SSebastien Boeuf pub struct VirtioSharedMemoryList { 57d35e775eSSebastien Boeuf pub host_addr: u64, 58d35e775eSSebastien Boeuf pub mem_slot: u32, 59e2b38cc0SSebastien Boeuf pub addr: GuestAddress, 60e2b38cc0SSebastien Boeuf pub len: GuestUsize, 61e2b38cc0SSebastien Boeuf pub region_list: Vec<VirtioSharedMemory>, 62e2b38cc0SSebastien Boeuf } 63e2b38cc0SSebastien Boeuf 6482464347SSamuel Ortiz /// Trait for virtio devices to be driven by a virtio transport. 6582464347SSamuel Ortiz /// 6682464347SSamuel Ortiz /// The lifecycle of a virtio device is to be moved to a virtio transport, which will then query the 6782464347SSamuel Ortiz /// device. Once the guest driver has configured the device, `VirtioDevice::activate` will be called 6882464347SSamuel Ortiz /// and all the events, memory, and queues for device operation will be moved into the device. 6982464347SSamuel Ortiz /// Optionally, a virtio device can implement device reset in which it returns said resources and 7082464347SSamuel Ortiz /// resets its internal. 7182464347SSamuel Ortiz pub trait VirtioDevice: Send { 7282464347SSamuel Ortiz /// The virtio device type. device_type(&self) -> u327382464347SSamuel Ortiz fn device_type(&self) -> u32; 7482464347SSamuel Ortiz 7582464347SSamuel Ortiz /// The maximum size of each queue that this device supports. queue_max_sizes(&self) -> &[u16]7682464347SSamuel Ortiz fn queue_max_sizes(&self) -> &[u16]; 7782464347SSamuel Ortiz 7814eddf72SCathy Zhang /// The set of feature bits that this device supports. features(&self) -> u647914eddf72SCathy Zhang fn features(&self) -> u64 { 8082464347SSamuel Ortiz 0 8182464347SSamuel Ortiz } 8282464347SSamuel Ortiz 8382464347SSamuel Ortiz /// Acknowledges that this set of features should be enabled. ack_features(&mut self, value: u64)8414eddf72SCathy Zhang fn ack_features(&mut self, value: u64) { 8514eddf72SCathy Zhang let _ = value; 8614eddf72SCathy Zhang } 8782464347SSamuel Ortiz 8882464347SSamuel Ortiz /// Reads this device configuration space at `offset`. read_config(&self, _offset: u64, _data: &mut [u8])896ba1c431SRob Bradford fn read_config(&self, _offset: u64, _data: &mut [u8]) { 906ba1c431SRob Bradford warn!( 916ba1c431SRob Bradford "No readable configuration fields for {}", 926ba1c431SRob Bradford VirtioDeviceType::from(self.device_type()) 936ba1c431SRob Bradford ); 946ba1c431SRob Bradford } 9582464347SSamuel Ortiz 9682464347SSamuel Ortiz /// Writes to this device configuration space at `offset`. write_config(&mut self, _offset: u64, _data: &[u8])976ba1c431SRob Bradford fn write_config(&mut self, _offset: u64, _data: &[u8]) { 986ba1c431SRob Bradford warn!( 996ba1c431SRob Bradford "No writable configuration fields for {}", 1006ba1c431SRob Bradford VirtioDeviceType::from(self.device_type()) 1016ba1c431SRob Bradford ); 1026ba1c431SRob Bradford } 10382464347SSamuel Ortiz 10482464347SSamuel Ortiz /// Activates this device for real usage. activate( &mut self, mem: GuestMemoryAtomic<GuestMemoryMmap>, interrupt_evt: Arc<dyn VirtioInterrupt>, queues: Vec<(usize, Queue, EventFd)>, ) -> ActivateResult10582464347SSamuel Ortiz fn activate( 10682464347SSamuel Ortiz &mut self, 107793d4e7bSSebastien Boeuf mem: GuestMemoryAtomic<GuestMemoryMmap>, 108c396bacaSSebastien Boeuf interrupt_evt: Arc<dyn VirtioInterrupt>, 109a423bf13SSebastien Boeuf queues: Vec<(usize, Queue, EventFd)>, 11082464347SSamuel Ortiz ) -> ActivateResult; 11182464347SSamuel Ortiz 11282464347SSamuel Ortiz /// Optionally deactivates this device and returns ownership of the guest memory map, interrupt 11382464347SSamuel Ortiz /// event, and queue events. reset(&mut self) -> Option<Arc<dyn VirtioInterrupt>>11423f9ec50SRob Bradford fn reset(&mut self) -> Option<Arc<dyn VirtioInterrupt>> { 11582464347SSamuel Ortiz None 11682464347SSamuel Ortiz } 11782464347SSamuel Ortiz 118e2b38cc0SSebastien Boeuf /// Returns the list of shared memory regions required by the device. get_shm_regions(&self) -> Option<VirtioSharedMemoryList>119e2b38cc0SSebastien Boeuf fn get_shm_regions(&self) -> Option<VirtioSharedMemoryList> { 120e2b38cc0SSebastien Boeuf None 121e2b38cc0SSebastien Boeuf } 1220acb1e32SSebastien Boeuf 123d35e775eSSebastien Boeuf /// Updates the list of shared memory regions required by the device. set_shm_regions( &mut self, _shm_regions: VirtioSharedMemoryList, ) -> std::result::Result<(), Error>124d35e775eSSebastien Boeuf fn set_shm_regions( 125d35e775eSSebastien Boeuf &mut self, 126d35e775eSSebastien Boeuf _shm_regions: VirtioSharedMemoryList, 127d35e775eSSebastien Boeuf ) -> std::result::Result<(), Error> { 128d35e775eSSebastien Boeuf std::unimplemented!() 129d35e775eSSebastien Boeuf } 130d35e775eSSebastien Boeuf 131545ea9eaSRob Bradford /// Some devices may need to do some explicit shutdown work. This method 132545ea9eaSRob Bradford /// may be implemented to do this. The VMM should call shutdown() on 133545ea9eaSRob Bradford /// every device as part of shutting down the VM. Acting on the device 134545ea9eaSRob Bradford /// after a shutdown() can lead to unpredictable results. shutdown(&mut self)135545ea9eaSRob Bradford fn shutdown(&mut self) {} 136bc874a9bSSebastien Boeuf add_memory_region( &mut self, _region: &Arc<GuestRegionMmap>, ) -> std::result::Result<(), Error>1379e53efa3SSebastien Boeuf fn add_memory_region( 1389e53efa3SSebastien Boeuf &mut self, 1399e53efa3SSebastien Boeuf _region: &Arc<GuestRegionMmap>, 1409e53efa3SSebastien Boeuf ) -> std::result::Result<(), Error> { 1419e53efa3SSebastien Boeuf Ok(()) 1429e53efa3SSebastien Boeuf } 1439e53efa3SSebastien Boeuf 1443fb0a02fSSebastien Boeuf /// Returns the list of userspace mappings associated with this device. userspace_mappings(&self) -> Vec<UserspaceMapping>1453fb0a02fSSebastien Boeuf fn userspace_mappings(&self) -> Vec<UserspaceMapping> { 1463fb0a02fSSebastien Boeuf Vec::new() 1473fb0a02fSSebastien Boeuf } 148dd54883aSRob Bradford 149dd54883aSRob Bradford /// Return the counters that this device exposes counters(&self) -> Option<HashMap<&'static str, Wrapping<u64>>>150dd54883aSRob Bradford fn counters(&self) -> Option<HashMap<&'static str, Wrapping<u64>>> { 151dd54883aSRob Bradford None 152dd54883aSRob Bradford } 153d2625408SRob Bradford 154d2625408SRob Bradford /// Helper to allow common implementation of read_config read_config_from_slice(&self, config: &[u8], offset: u64, mut data: &mut [u8])155d2625408SRob Bradford fn read_config_from_slice(&self, config: &[u8], offset: u64, mut data: &mut [u8]) { 156d2625408SRob Bradford let config_len = config.len() as u64; 157d2625408SRob Bradford let data_len = data.len() as u64; 158d2625408SRob Bradford if offset + data_len > config_len { 159d2625408SRob Bradford error!( 160d2625408SRob Bradford "Out-of-bound access to configuration: config_len = {} offset = {:x} length = {} for {}", 161d2625408SRob Bradford config_len, 162d2625408SRob Bradford offset, 163d2625408SRob Bradford data_len, 164d2625408SRob Bradford self.device_type() 165d2625408SRob Bradford ); 166d2625408SRob Bradford return; 167d2625408SRob Bradford } 168d2625408SRob Bradford if let Some(end) = offset.checked_add(data.len() as u64) { 169d2625408SRob Bradford data.write_all(&config[offset as usize..std::cmp::min(end, config_len) as usize]) 170d2625408SRob Bradford .unwrap(); 171d2625408SRob Bradford } 172d2625408SRob Bradford } 173c75f8b2fSHui Zhu 17475b9e70eSSebastien Boeuf /// Set the access platform trait to let the device perform address 17575b9e70eSSebastien Boeuf /// translations if needed. set_access_platform(&mut self, _access_platform: Arc<dyn AccessPlatform>)17675b9e70eSSebastien Boeuf fn set_access_platform(&mut self, _access_platform: Arc<dyn AccessPlatform>) {} 1770acb1e32SSebastien Boeuf } 1780acb1e32SSebastien Boeuf 17960c8a72eSBo Chen /// Trait to define address translation for devices managed by virtio-iommu 18060c8a72eSBo Chen /// 1810acb1e32SSebastien Boeuf /// Trait providing address translation the same way a physical DMA remapping 1820acb1e32SSebastien Boeuf /// table would provide translation between an IOVA and a physical address. 1830acb1e32SSebastien Boeuf /// The goal of this trait is to be used by virtio devices to perform the 1840acb1e32SSebastien Boeuf /// address translation before they try to read from the guest physical address. 1850acb1e32SSebastien Boeuf /// On the other side, the implementation itself should be provided by the code 1860acb1e32SSebastien Boeuf /// emulating the IOMMU for the guest. 187439823e7SWei Liu pub trait DmaRemapping { 188059e787cSSebastien Boeuf /// Provide a way to translate GVA address ranges into GPAs. translate_gva(&self, id: u32, addr: u64) -> std::result::Result<u64, std::io::Error>189059e787cSSebastien Boeuf fn translate_gva(&self, id: u32, addr: u64) -> std::result::Result<u64, std::io::Error>; 1901cdf8b23SSebastien Boeuf /// Provide a way to translate GPA address ranges into GVAs. translate_gpa(&self, id: u32, addr: u64) -> std::result::Result<u64, std::io::Error>1911cdf8b23SSebastien Boeuf fn translate_gpa(&self, id: u32, addr: u64) -> std::result::Result<u64, std::io::Error>; 19282464347SSamuel Ortiz } 193dae0b2efSSamuel Ortiz 194081c8979SRob Bradford /// Structure to handle device state common to all devices 195a9a13846SRob Bradford #[derive(Default)] 196081c8979SRob Bradford pub struct VirtioCommon { 197081c8979SRob Bradford pub avail_features: u64, 198081c8979SRob Bradford pub acked_features: u64, 19934875249SRob Bradford pub kill_evt: Option<EventFd>, 20034875249SRob Bradford pub interrupt_cb: Option<Arc<dyn VirtioInterrupt>>, 20134875249SRob Bradford pub pause_evt: Option<EventFd>, 20234875249SRob Bradford pub paused: Arc<AtomicBool>, 20334875249SRob Bradford pub paused_sync: Option<Arc<Barrier>>, 20434875249SRob Bradford pub epoll_threads: Option<Vec<thread::JoinHandle<()>>>, 20534875249SRob Bradford pub queue_sizes: Vec<u16>, 20634875249SRob Bradford pub device_type: u32, 207c90f77e3SRob Bradford pub min_queues: u16, 20875b9e70eSSebastien Boeuf pub access_platform: Option<Arc<dyn AccessPlatform>>, 209081c8979SRob Bradford } 210081c8979SRob Bradford 211081c8979SRob Bradford impl VirtioCommon { feature_acked(&self, feature: u64) -> bool212081c8979SRob Bradford pub fn feature_acked(&self, feature: u64) -> bool { 213*2624f17fSRob Bradford self.acked_features & (1 << feature) == 1 << feature 214081c8979SRob Bradford } 215081c8979SRob Bradford ack_features(&mut self, value: u64)216081c8979SRob Bradford pub fn ack_features(&mut self, value: u64) { 217081c8979SRob Bradford let mut v = value; 218081c8979SRob Bradford // Check if the guest is ACK'ing a feature that we didn't claim to have. 219081c8979SRob Bradford let unrequested_features = v & !self.avail_features; 220081c8979SRob Bradford if unrequested_features != 0 { 221081c8979SRob Bradford warn!("Received acknowledge request for unknown feature."); 222081c8979SRob Bradford 223081c8979SRob Bradford // Don't count these features as acked. 224081c8979SRob Bradford v &= !unrequested_features; 225081c8979SRob Bradford } 226081c8979SRob Bradford self.acked_features |= v; 227081c8979SRob Bradford } 22834875249SRob Bradford activate( &mut self, queues: &[(usize, Queue, EventFd)], interrupt_cb: &Arc<dyn VirtioInterrupt>, ) -> ActivateResult22934875249SRob Bradford pub fn activate( 23034875249SRob Bradford &mut self, 231a423bf13SSebastien Boeuf queues: &[(usize, Queue, EventFd)], 23234875249SRob Bradford interrupt_cb: &Arc<dyn VirtioInterrupt>, 23334875249SRob Bradford ) -> ActivateResult { 234c90f77e3SRob Bradford if queues.len() < self.min_queues.into() { 235c90f77e3SRob Bradford error!( 2361adcf522SYi Wang "Number of enabled queues lower than min: {} vs {}", 237c90f77e3SRob Bradford queues.len(), 238c90f77e3SRob Bradford self.min_queues 239c90f77e3SRob Bradford ); 240c90f77e3SRob Bradford return Err(ActivateError::BadActivate); 241c90f77e3SRob Bradford } 242c90f77e3SRob Bradford 24334875249SRob Bradford let kill_evt = EventFd::new(EFD_NONBLOCK).map_err(|e| { 24434875249SRob Bradford error!("failed creating kill EventFd: {}", e); 24534875249SRob Bradford ActivateError::BadActivate 24634875249SRob Bradford })?; 24734875249SRob Bradford self.kill_evt = Some(kill_evt); 24834875249SRob Bradford 24934875249SRob Bradford let pause_evt = EventFd::new(EFD_NONBLOCK).map_err(|e| { 25034875249SRob Bradford error!("failed creating pause EventFd: {}", e); 25134875249SRob Bradford ActivateError::BadActivate 25234875249SRob Bradford })?; 25334875249SRob Bradford self.pause_evt = Some(pause_evt); 25434875249SRob Bradford 25534875249SRob Bradford // Save the interrupt EventFD as we need to return it on reset 25634875249SRob Bradford // but clone it to pass into the thread. 25734875249SRob Bradford self.interrupt_cb = Some(interrupt_cb.clone()); 25834875249SRob Bradford 25934875249SRob Bradford Ok(()) 26034875249SRob Bradford } 26134875249SRob Bradford reset(&mut self) -> Option<Arc<dyn VirtioInterrupt>>26223f9ec50SRob Bradford pub fn reset(&mut self) -> Option<Arc<dyn VirtioInterrupt>> { 26334875249SRob Bradford // We first must resume the virtio thread if it was paused. 26434875249SRob Bradford if self.pause_evt.take().is_some() { 26534875249SRob Bradford self.resume().ok()?; 26634875249SRob Bradford } 26734875249SRob Bradford 26834875249SRob Bradford if let Some(kill_evt) = self.kill_evt.take() { 26934875249SRob Bradford // Ignore the result because there is nothing we can do about it. 27034875249SRob Bradford let _ = kill_evt.write(1); 27134875249SRob Bradford } 27234875249SRob Bradford 27322649c4aSRob Bradford if let Some(mut threads) = self.epoll_threads.take() { 27422649c4aSRob Bradford for t in threads.drain(..) { 27522649c4aSRob Bradford if let Err(e) = t.join() { 27622649c4aSRob Bradford error!("Error joining thread: {:?}", e); 27722649c4aSRob Bradford } 27822649c4aSRob Bradford } 27922649c4aSRob Bradford } 28022649c4aSRob Bradford 28123f9ec50SRob Bradford // Return the interrupt 28223f9ec50SRob Bradford Some(self.interrupt_cb.take().unwrap()) 28334875249SRob Bradford } 284280bef83SRob Bradford 285a9924df2SBo Chen // Wait for the worker thread to finish and return wait_for_epoll_threads(&mut self)286194b59f4SRob Bradford pub fn wait_for_epoll_threads(&mut self) { 287a9924df2SBo Chen if let Some(mut threads) = self.epoll_threads.take() { 288a9924df2SBo Chen for t in threads.drain(..) { 289a9924df2SBo Chen if let Err(e) = t.join() { 290a9924df2SBo Chen error!("Error joining thread: {:?}", e); 291a9924df2SBo Chen } 292a9924df2SBo Chen } 293a9924df2SBo Chen } 294a9924df2SBo Chen } 295a9924df2SBo Chen dup_eventfds(&self) -> (EventFd, EventFd)296280bef83SRob Bradford pub fn dup_eventfds(&self) -> (EventFd, EventFd) { 297280bef83SRob Bradford ( 298280bef83SRob Bradford self.kill_evt.as_ref().unwrap().try_clone().unwrap(), 299280bef83SRob Bradford self.pause_evt.as_ref().unwrap().try_clone().unwrap(), 300280bef83SRob Bradford ) 301280bef83SRob Bradford } 30275b9e70eSSebastien Boeuf set_access_platform(&mut self, access_platform: Arc<dyn AccessPlatform>)30375b9e70eSSebastien Boeuf pub fn set_access_platform(&mut self, access_platform: Arc<dyn AccessPlatform>) { 30475b9e70eSSebastien Boeuf self.access_platform = Some(access_platform); 30575b9e70eSSebastien Boeuf // Indirect descriptors feature is not supported when the device 30675b9e70eSSebastien Boeuf // requires the addresses held by the descriptors to be translated. 30775b9e70eSSebastien Boeuf self.avail_features &= !(1 << VIRTIO_F_RING_INDIRECT_DESC); 30875b9e70eSSebastien Boeuf } 30934875249SRob Bradford } 31034875249SRob Bradford 31134875249SRob Bradford impl Pausable for VirtioCommon { pause(&mut self) -> std::result::Result<(), MigratableError>31234875249SRob Bradford fn pause(&mut self) -> std::result::Result<(), MigratableError> { 313e475b12cSRob Bradford info!( 31434875249SRob Bradford "Pausing virtio-{}", 31534875249SRob Bradford VirtioDeviceType::from(self.device_type) 31634875249SRob Bradford ); 31734875249SRob Bradford self.paused.store(true, Ordering::SeqCst); 31834875249SRob Bradford if let Some(pause_evt) = &self.pause_evt { 31934875249SRob Bradford pause_evt 32034875249SRob Bradford .write(1) 32134875249SRob Bradford .map_err(|e| MigratableError::Pause(e.into()))?; 32234875249SRob Bradford 32334875249SRob Bradford // Wait for all threads to acknowledge the pause before going 32434875249SRob Bradford // any further. This is exclusively performed when pause_evt 32534875249SRob Bradford // eventfd is Some(), as this means the virtio device has been 32634875249SRob Bradford // activated. One specific case where the device can be paused 32734875249SRob Bradford // while it hasn't been yet activated is snapshot/restore. 32834875249SRob Bradford self.paused_sync.as_ref().unwrap().wait(); 32934875249SRob Bradford } 33034875249SRob Bradford 33134875249SRob Bradford Ok(()) 33234875249SRob Bradford } 33334875249SRob Bradford resume(&mut self) -> std::result::Result<(), MigratableError>33434875249SRob Bradford fn resume(&mut self) -> std::result::Result<(), MigratableError> { 335e475b12cSRob Bradford info!( 33634875249SRob Bradford "Resuming virtio-{}", 33734875249SRob Bradford VirtioDeviceType::from(self.device_type) 33834875249SRob Bradford ); 33934875249SRob Bradford self.paused.store(false, Ordering::SeqCst); 34034875249SRob Bradford if let Some(epoll_threads) = &self.epoll_threads { 34134875249SRob Bradford for t in epoll_threads.iter() { 34234875249SRob Bradford t.thread().unpark(); 34334875249SRob Bradford } 34434875249SRob Bradford } 34534875249SRob Bradford 34634875249SRob Bradford Ok(()) 34734875249SRob Bradford } 348081c8979SRob Bradford } 349