1 // Copyright © 2021 Intel Corporation 2 // 3 // SPDX-License-Identifier: Apache-2.0 4 // 5 6 use crate::vfio::{Interrupt, Vfio, VfioCommon, VfioError}; 7 use crate::{BarReprogrammingParams, PciBarRegionType, VfioPciError}; 8 use crate::{ 9 PciClassCode, PciConfiguration, PciDevice, PciDeviceError, PciHeaderType, PciSubclass, 10 }; 11 use hypervisor::HypervisorVmError; 12 use std::any::Any; 13 use std::os::unix::prelude::AsRawFd; 14 use std::path::Path; 15 use std::ptr::null_mut; 16 use std::sync::{Arc, Barrier, Mutex}; 17 use std::u32; 18 use thiserror::Error; 19 use vfio_bindings::bindings::vfio::*; 20 use vfio_ioctls::VfioIrq; 21 use vfio_user::{Client, Error as VfioUserError}; 22 use vm_allocator::SystemAllocator; 23 use vm_device::interrupt::{InterruptManager, InterruptSourceGroup, MsiIrqGroupConfig}; 24 use vm_device::BusDevice; 25 use vm_memory::bitmap::AtomicBitmap; 26 use vm_memory::{Address, GuestAddress, GuestMemoryRegion, GuestRegionMmap, GuestUsize}; 27 use vmm_sys_util::eventfd::EventFd; 28 29 pub struct VfioUserPciDevice { 30 vm: Arc<dyn hypervisor::Vm>, 31 client: Arc<Mutex<Client>>, 32 vfio_wrapper: VfioUserClientWrapper, 33 common: VfioCommon, 34 } 35 36 #[derive(Error, Debug)] 37 pub enum VfioUserPciDeviceError { 38 #[error("Client error: {0}")] 39 Client(#[source] VfioUserError), 40 #[error("Failed to map VFIO PCI region into guest: {0}")] 41 MapRegionGuest(#[source] HypervisorVmError), 42 #[error("Failed to DMA map: {0}")] 43 DmaMap(#[source] VfioUserError), 44 #[error("Failed to DMA unmap: {0}")] 45 DmaUnmap(#[source] VfioUserError), 46 #[error("Failed to initialize legacy interrupts: {0}")] 47 InitializeLegacyInterrupts(#[source] VfioPciError), 48 } 49 50 #[derive(Copy, Clone)] 51 enum PciVfioUserSubclass { 52 VfioUserSubclass = 0xff, 53 } 54 55 impl PciSubclass for PciVfioUserSubclass { 56 fn get_register_value(&self) -> u8 { 57 *self as u8 58 } 59 } 60 61 impl VfioUserPciDevice { 62 pub fn new( 63 vm: &Arc<dyn hypervisor::Vm>, 64 path: &Path, 65 msi_interrupt_manager: &Arc<dyn InterruptManager<GroupConfig = MsiIrqGroupConfig>>, 66 legacy_interrupt_group: Option<Arc<dyn InterruptSourceGroup>>, 67 ) -> Result<Self, VfioUserPciDeviceError> { 68 let mut client = Client::new(path).map_err(VfioUserPciDeviceError::Client)?; 69 70 // This is used for the BAR and capabilities only 71 let configuration = PciConfiguration::new( 72 0, 73 0, 74 0, 75 PciClassCode::Other, 76 &PciVfioUserSubclass::VfioUserSubclass, 77 None, 78 PciHeaderType::Device, 79 0, 80 0, 81 None, 82 ); 83 if client.resettable() { 84 client.reset().map_err(VfioUserPciDeviceError::Client)?; 85 } 86 87 let client = Arc::new(Mutex::new(client)); 88 89 let vfio_wrapper = VfioUserClientWrapper { 90 client: client.clone(), 91 }; 92 93 let mut common = VfioCommon { 94 mmio_regions: Vec::new(), 95 configuration, 96 interrupt: Interrupt { 97 intx: None, 98 msi: None, 99 msix: None, 100 }, 101 }; 102 103 common.parse_capabilities(msi_interrupt_manager, &vfio_wrapper); 104 common 105 .initialize_legacy_interrupt(legacy_interrupt_group, &vfio_wrapper) 106 .map_err(VfioUserPciDeviceError::InitializeLegacyInterrupts)?; 107 108 Ok(Self { 109 vm: vm.clone(), 110 client, 111 vfio_wrapper, 112 common, 113 }) 114 } 115 } 116 117 impl BusDevice for VfioUserPciDevice { 118 fn read(&mut self, base: u64, offset: u64, data: &mut [u8]) { 119 self.read_bar(base, offset, data) 120 } 121 122 fn write(&mut self, base: u64, offset: u64, data: &[u8]) -> Option<Arc<Barrier>> { 123 self.write_bar(base, offset, data) 124 } 125 } 126 127 #[repr(u32)] 128 #[derive(Debug, PartialEq, Eq, PartialOrd, Ord)] 129 #[allow(dead_code)] 130 enum Regions { 131 Bar0, 132 Bar1, 133 Bar2, 134 Bar3, 135 Bar4, 136 Bar5, 137 Rom, 138 Config, 139 Vga, 140 Migration, 141 } 142 143 struct VfioUserClientWrapper { 144 client: Arc<Mutex<Client>>, 145 } 146 147 impl Vfio for VfioUserClientWrapper { 148 fn region_read(&self, index: u32, offset: u64, data: &mut [u8]) { 149 self.client 150 .lock() 151 .unwrap() 152 .region_read(index, offset, data) 153 .ok(); 154 } 155 156 fn region_write(&self, index: u32, offset: u64, data: &[u8]) { 157 self.client 158 .lock() 159 .unwrap() 160 .region_write(index, offset, data) 161 .ok(); 162 } 163 164 fn get_irq_info(&self, irq_index: u32) -> Option<VfioIrq> { 165 self.client 166 .lock() 167 .unwrap() 168 .get_irq_info(irq_index) 169 .ok() 170 .map(|i| VfioIrq { 171 index: i.index, 172 flags: i.flags, 173 count: i.count, 174 }) 175 } 176 177 fn enable_irq(&self, irq_index: u32, event_fds: Vec<&EventFd>) -> Result<(), VfioError> { 178 info!( 179 "Enabling IRQ {:x} number of fds = {:?}", 180 irq_index, 181 event_fds.len() 182 ); 183 let fds: Vec<i32> = event_fds.iter().map(|e| e.as_raw_fd()).collect(); 184 185 self.client 186 .lock() 187 .unwrap() 188 .set_irqs( 189 irq_index, 190 VFIO_IRQ_SET_DATA_EVENTFD | VFIO_IRQ_SET_ACTION_TRIGGER, 191 0, 192 event_fds.len() as u32, 193 &fds, 194 ) 195 .map_err(VfioError::VfioUser) 196 } 197 198 fn disable_irq(&self, irq_index: u32) -> Result<(), VfioError> { 199 info!("Disabling IRQ {:x}", irq_index); 200 self.client 201 .lock() 202 .unwrap() 203 .set_irqs( 204 irq_index, 205 VFIO_IRQ_SET_DATA_NONE | VFIO_IRQ_SET_ACTION_TRIGGER, 206 0, 207 0, 208 &[], 209 ) 210 .map_err(VfioError::VfioUser) 211 } 212 213 fn unmask_irq(&self, irq_index: u32) -> Result<(), VfioError> { 214 info!("Unmasking IRQ {:x}", irq_index); 215 self.client 216 .lock() 217 .unwrap() 218 .set_irqs( 219 irq_index, 220 VFIO_IRQ_SET_DATA_NONE | VFIO_IRQ_SET_ACTION_UNMASK, 221 0, 222 1, 223 &[], 224 ) 225 .map_err(VfioError::VfioUser) 226 } 227 } 228 229 impl PciDevice for VfioUserPciDevice { 230 fn allocate_bars( 231 &mut self, 232 allocator: &mut SystemAllocator, 233 ) -> Result<Vec<(GuestAddress, GuestUsize, PciBarRegionType)>, PciDeviceError> { 234 self.common.allocate_bars(allocator, &self.vfio_wrapper) 235 } 236 237 fn free_bars(&mut self, allocator: &mut SystemAllocator) -> Result<(), PciDeviceError> { 238 self.common.free_bars(allocator) 239 } 240 241 fn as_any(&mut self) -> &mut dyn Any { 242 self 243 } 244 245 fn detect_bar_reprogramming( 246 &mut self, 247 reg_idx: usize, 248 data: &[u8], 249 ) -> Option<BarReprogrammingParams> { 250 self.common 251 .configuration 252 .detect_bar_reprogramming(reg_idx, data) 253 } 254 255 fn write_config_register( 256 &mut self, 257 reg_idx: usize, 258 offset: u64, 259 data: &[u8], 260 ) -> Option<Arc<Barrier>> { 261 self.common 262 .write_config_register(reg_idx, offset, data, &self.vfio_wrapper) 263 } 264 265 fn read_config_register(&mut self, reg_idx: usize) -> u32 { 266 self.common 267 .read_config_register(reg_idx, &self.vfio_wrapper) 268 } 269 270 fn read_bar(&mut self, base: u64, offset: u64, data: &mut [u8]) { 271 self.common.read_bar(base, offset, data, &self.vfio_wrapper) 272 } 273 274 fn write_bar(&mut self, base: u64, offset: u64, data: &[u8]) -> Option<Arc<Barrier>> { 275 self.common 276 .write_bar(base, offset, data, &self.vfio_wrapper) 277 } 278 279 fn move_bar(&mut self, old_base: u64, new_base: u64) -> Result<(), std::io::Error> { 280 info!("Moving BAR 0x{:x} -> 0x{:x}", old_base, new_base); 281 for mmio_region in self.common.mmio_regions.iter_mut() { 282 if mmio_region.start.raw_value() == old_base { 283 mmio_region.start = GuestAddress(new_base); 284 285 if let Some(mem_slot) = mmio_region.mem_slot { 286 if let Some(host_addr) = mmio_region.host_addr { 287 // Remove original region 288 let old_region = self.vm.make_user_memory_region( 289 mem_slot, 290 old_base, 291 mmio_region.length as u64, 292 host_addr as u64, 293 false, 294 false, 295 ); 296 297 self.vm 298 .remove_user_memory_region(old_region) 299 .map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e))?; 300 301 let new_region = self.vm.make_user_memory_region( 302 mem_slot, 303 new_base, 304 mmio_region.length as u64, 305 host_addr as u64, 306 false, 307 false, 308 ); 309 310 self.vm 311 .create_user_memory_region(new_region) 312 .map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e))?; 313 } 314 } 315 info!("Moved bar 0x{:x} -> 0x{:x}", old_base, new_base); 316 } 317 } 318 319 Ok(()) 320 } 321 } 322 323 impl VfioUserPciDevice { 324 pub fn map_mmio_regions<F>( 325 &mut self, 326 vm: &Arc<dyn hypervisor::Vm>, 327 mem_slot: F, 328 ) -> Result<(), VfioUserPciDeviceError> 329 where 330 F: Fn() -> u32, 331 { 332 for mmio_region in &mut self.common.mmio_regions { 333 let region_flags = self 334 .client 335 .lock() 336 .unwrap() 337 .region(mmio_region.index) 338 .unwrap() 339 .flags; 340 let file_offset = self 341 .client 342 .lock() 343 .unwrap() 344 .region(mmio_region.index) 345 .unwrap() 346 .file_offset 347 .clone(); 348 349 if region_flags & VFIO_REGION_INFO_FLAG_MMAP != 0 { 350 let mut prot = 0; 351 if region_flags & VFIO_REGION_INFO_FLAG_READ != 0 { 352 prot |= libc::PROT_READ; 353 } 354 if region_flags & VFIO_REGION_INFO_FLAG_WRITE != 0 { 355 prot |= libc::PROT_WRITE; 356 } 357 358 let host_addr = unsafe { 359 libc::mmap( 360 null_mut(), 361 mmio_region.length as usize, 362 prot, 363 libc::MAP_SHARED, 364 file_offset.as_ref().unwrap().file().as_raw_fd(), 365 file_offset.as_ref().unwrap().start() as libc::off_t, 366 ) 367 }; 368 369 if host_addr == libc::MAP_FAILED { 370 error!( 371 "Could not mmap regions, error:{}", 372 std::io::Error::last_os_error() 373 ); 374 continue; 375 } 376 377 let slot = mem_slot(); 378 let mem_region = vm.make_user_memory_region( 379 slot, 380 mmio_region.start.0, 381 mmio_region.length as u64, 382 host_addr as u64, 383 false, 384 false, 385 ); 386 387 vm.create_user_memory_region(mem_region) 388 .map_err(VfioUserPciDeviceError::MapRegionGuest)?; 389 390 mmio_region.mem_slot = Some(slot); 391 mmio_region.host_addr = Some(host_addr as u64); 392 mmio_region.mmap_size = Some(mmio_region.length as usize); 393 } 394 } 395 396 Ok(()) 397 } 398 399 pub fn unmap_mmio_regions(&mut self) { 400 for mmio_region in self.common.mmio_regions.iter() { 401 if let (Some(host_addr), Some(mmap_size), Some(mem_slot)) = ( 402 mmio_region.host_addr, 403 mmio_region.mmap_size, 404 mmio_region.mem_slot, 405 ) { 406 // Remove region 407 let r = self.vm.make_user_memory_region( 408 mem_slot, 409 mmio_region.start.raw_value(), 410 mmap_size as u64, 411 host_addr as u64, 412 false, 413 false, 414 ); 415 416 if let Err(e) = self.vm.remove_user_memory_region(r) { 417 error!("Could not remove the userspace memory region: {}", e); 418 } 419 420 let ret = unsafe { libc::munmap(host_addr as *mut libc::c_void, mmap_size) }; 421 if ret != 0 { 422 error!( 423 "Could not unmap region {}, error:{}", 424 mmio_region.index, 425 std::io::Error::last_os_error() 426 ); 427 } 428 } 429 } 430 } 431 432 pub fn dma_map( 433 &mut self, 434 region: &GuestRegionMmap<AtomicBitmap>, 435 ) -> Result<(), VfioUserPciDeviceError> { 436 let (fd, offset) = match region.file_offset() { 437 Some(_file_offset) => (_file_offset.file().as_raw_fd(), _file_offset.start()), 438 None => return Ok(()), 439 }; 440 441 self.client 442 .lock() 443 .unwrap() 444 .dma_map( 445 offset, 446 region.start_addr().raw_value(), 447 region.len() as u64, 448 fd, 449 ) 450 .map_err(VfioUserPciDeviceError::DmaMap) 451 } 452 453 pub fn dma_unmap( 454 &mut self, 455 region: &GuestRegionMmap<AtomicBitmap>, 456 ) -> Result<(), VfioUserPciDeviceError> { 457 self.client 458 .lock() 459 .unwrap() 460 .dma_unmap(region.start_addr().raw_value(), region.len() as u64) 461 .map_err(VfioUserPciDeviceError::DmaUnmap) 462 } 463 } 464 465 impl Drop for VfioUserPciDevice { 466 fn drop(&mut self) { 467 self.unmap_mmio_regions(); 468 469 if let Some(msix) = &self.common.interrupt.msix { 470 if msix.bar.enabled() { 471 self.common.disable_msix(&self.vfio_wrapper); 472 } 473 } 474 475 if let Some(msi) = &self.common.interrupt.msi { 476 if msi.cfg.enabled() { 477 self.common.disable_msi(&self.vfio_wrapper) 478 } 479 } 480 481 if self.common.interrupt.intx_in_use() { 482 self.common.disable_intx(&self.vfio_wrapper); 483 } 484 } 485 } 486