1 // Copyright © 2021 Intel Corporation 2 // 3 // SPDX-License-Identifier: Apache-2.0 4 // 5 6 use crate::vfio::{Interrupt, Vfio, VfioCommon, VfioError}; 7 use crate::{BarReprogrammingParams, PciBarRegionType, VfioPciError}; 8 use crate::{ 9 PciBdf, PciClassCode, PciConfiguration, PciDevice, PciDeviceError, PciHeaderType, PciSubclass, 10 }; 11 use hypervisor::HypervisorVmError; 12 use std::any::Any; 13 use std::os::unix::prelude::AsRawFd; 14 use std::ptr::null_mut; 15 use std::sync::{Arc, Barrier, Mutex}; 16 use std::u32; 17 use thiserror::Error; 18 use vfio_bindings::bindings::vfio::*; 19 use vfio_ioctls::VfioIrq; 20 use vfio_user::{Client, Error as VfioUserError}; 21 use vm_allocator::{AddressAllocator, SystemAllocator}; 22 use vm_device::dma_mapping::ExternalDmaMapping; 23 use vm_device::interrupt::{InterruptManager, InterruptSourceGroup, MsiIrqGroupConfig}; 24 use vm_device::BusDevice; 25 use vm_memory::bitmap::AtomicBitmap; 26 use vm_memory::{ 27 Address, GuestAddress, GuestAddressSpace, GuestMemory, GuestMemoryRegion, GuestRegionMmap, 28 GuestUsize, 29 }; 30 use vmm_sys_util::eventfd::EventFd; 31 32 pub struct VfioUserPciDevice { 33 vm: Arc<dyn hypervisor::Vm>, 34 client: Arc<Mutex<Client>>, 35 vfio_wrapper: VfioUserClientWrapper, 36 common: VfioCommon, 37 } 38 39 #[derive(Error, Debug)] 40 pub enum VfioUserPciDeviceError { 41 #[error("Client error: {0}")] 42 Client(#[source] VfioUserError), 43 #[error("Failed to map VFIO PCI region into guest: {0}")] 44 MapRegionGuest(#[source] HypervisorVmError), 45 #[error("Failed to DMA map: {0}")] 46 DmaMap(#[source] VfioUserError), 47 #[error("Failed to DMA unmap: {0}")] 48 DmaUnmap(#[source] VfioUserError), 49 #[error("Failed to initialize legacy interrupts: {0}")] 50 InitializeLegacyInterrupts(#[source] VfioPciError), 51 } 52 53 #[derive(Copy, Clone)] 54 enum PciVfioUserSubclass { 55 VfioUserSubclass = 0xff, 56 } 57 58 impl PciSubclass for PciVfioUserSubclass { 59 fn get_register_value(&self) -> u8 { 60 *self as u8 61 } 62 } 63 64 impl VfioUserPciDevice { 65 pub fn new( 66 vm: &Arc<dyn hypervisor::Vm>, 67 client: Arc<Mutex<Client>>, 68 msi_interrupt_manager: &Arc<dyn InterruptManager<GroupConfig = MsiIrqGroupConfig>>, 69 legacy_interrupt_group: Option<Arc<dyn InterruptSourceGroup>>, 70 bdf: PciBdf, 71 ) -> Result<Self, VfioUserPciDeviceError> { 72 // This is used for the BAR and capabilities only 73 let configuration = PciConfiguration::new( 74 0, 75 0, 76 0, 77 PciClassCode::Other, 78 &PciVfioUserSubclass::VfioUserSubclass, 79 None, 80 PciHeaderType::Device, 81 0, 82 0, 83 None, 84 ); 85 let resettable = client.lock().unwrap().resettable(); 86 if resettable { 87 client 88 .lock() 89 .unwrap() 90 .reset() 91 .map_err(VfioUserPciDeviceError::Client)?; 92 } 93 94 let vfio_wrapper = VfioUserClientWrapper { 95 client: client.clone(), 96 }; 97 98 let mut common = VfioCommon { 99 mmio_regions: Vec::new(), 100 configuration, 101 interrupt: Interrupt { 102 intx: None, 103 msi: None, 104 msix: None, 105 }, 106 }; 107 108 common.parse_capabilities(msi_interrupt_manager, &vfio_wrapper, bdf); 109 common 110 .initialize_legacy_interrupt(legacy_interrupt_group, &vfio_wrapper) 111 .map_err(VfioUserPciDeviceError::InitializeLegacyInterrupts)?; 112 113 Ok(Self { 114 vm: vm.clone(), 115 client, 116 vfio_wrapper, 117 common, 118 }) 119 } 120 121 pub fn map_mmio_regions<F>( 122 &mut self, 123 vm: &Arc<dyn hypervisor::Vm>, 124 mem_slot: F, 125 ) -> Result<(), VfioUserPciDeviceError> 126 where 127 F: Fn() -> u32, 128 { 129 for mmio_region in &mut self.common.mmio_regions { 130 let region_flags = self 131 .client 132 .lock() 133 .unwrap() 134 .region(mmio_region.index) 135 .unwrap() 136 .flags; 137 let file_offset = self 138 .client 139 .lock() 140 .unwrap() 141 .region(mmio_region.index) 142 .unwrap() 143 .file_offset 144 .clone(); 145 146 if region_flags & VFIO_REGION_INFO_FLAG_MMAP != 0 { 147 let mut prot = 0; 148 if region_flags & VFIO_REGION_INFO_FLAG_READ != 0 { 149 prot |= libc::PROT_READ; 150 } 151 if region_flags & VFIO_REGION_INFO_FLAG_WRITE != 0 { 152 prot |= libc::PROT_WRITE; 153 } 154 155 let host_addr = unsafe { 156 libc::mmap( 157 null_mut(), 158 mmio_region.length as usize, 159 prot, 160 libc::MAP_SHARED, 161 file_offset.as_ref().unwrap().file().as_raw_fd(), 162 file_offset.as_ref().unwrap().start() as libc::off_t, 163 ) 164 }; 165 166 if host_addr == libc::MAP_FAILED { 167 error!( 168 "Could not mmap regions, error:{}", 169 std::io::Error::last_os_error() 170 ); 171 continue; 172 } 173 174 let slot = mem_slot(); 175 let mem_region = vm.make_user_memory_region( 176 slot, 177 mmio_region.start.0, 178 mmio_region.length as u64, 179 host_addr as u64, 180 false, 181 false, 182 ); 183 184 vm.create_user_memory_region(mem_region) 185 .map_err(VfioUserPciDeviceError::MapRegionGuest)?; 186 187 mmio_region.mem_slot = Some(slot); 188 mmio_region.host_addr = Some(host_addr as u64); 189 mmio_region.mmap_size = Some(mmio_region.length as usize); 190 } 191 } 192 193 Ok(()) 194 } 195 196 pub fn unmap_mmio_regions(&mut self) { 197 for mmio_region in self.common.mmio_regions.iter() { 198 if let (Some(host_addr), Some(mmap_size), Some(mem_slot)) = ( 199 mmio_region.host_addr, 200 mmio_region.mmap_size, 201 mmio_region.mem_slot, 202 ) { 203 // Remove region 204 let r = self.vm.make_user_memory_region( 205 mem_slot, 206 mmio_region.start.raw_value(), 207 mmap_size as u64, 208 host_addr as u64, 209 false, 210 false, 211 ); 212 213 if let Err(e) = self.vm.remove_user_memory_region(r) { 214 error!("Could not remove the userspace memory region: {}", e); 215 } 216 217 let ret = unsafe { libc::munmap(host_addr as *mut libc::c_void, mmap_size) }; 218 if ret != 0 { 219 error!( 220 "Could not unmap region {}, error:{}", 221 mmio_region.index, 222 std::io::Error::last_os_error() 223 ); 224 } 225 } 226 } 227 } 228 229 pub fn dma_map( 230 &mut self, 231 region: &GuestRegionMmap<AtomicBitmap>, 232 ) -> Result<(), VfioUserPciDeviceError> { 233 let (fd, offset) = match region.file_offset() { 234 Some(_file_offset) => (_file_offset.file().as_raw_fd(), _file_offset.start()), 235 None => return Ok(()), 236 }; 237 238 self.client 239 .lock() 240 .unwrap() 241 .dma_map( 242 offset, 243 region.start_addr().raw_value(), 244 region.len() as u64, 245 fd, 246 ) 247 .map_err(VfioUserPciDeviceError::DmaMap) 248 } 249 250 pub fn dma_unmap( 251 &mut self, 252 region: &GuestRegionMmap<AtomicBitmap>, 253 ) -> Result<(), VfioUserPciDeviceError> { 254 self.client 255 .lock() 256 .unwrap() 257 .dma_unmap(region.start_addr().raw_value(), region.len() as u64) 258 .map_err(VfioUserPciDeviceError::DmaUnmap) 259 } 260 } 261 262 impl BusDevice for VfioUserPciDevice { 263 fn read(&mut self, base: u64, offset: u64, data: &mut [u8]) { 264 self.read_bar(base, offset, data) 265 } 266 267 fn write(&mut self, base: u64, offset: u64, data: &[u8]) -> Option<Arc<Barrier>> { 268 self.write_bar(base, offset, data) 269 } 270 } 271 272 #[repr(u32)] 273 #[derive(Debug, PartialEq, Eq, PartialOrd, Ord)] 274 #[allow(dead_code)] 275 enum Regions { 276 Bar0, 277 Bar1, 278 Bar2, 279 Bar3, 280 Bar4, 281 Bar5, 282 Rom, 283 Config, 284 Vga, 285 Migration, 286 } 287 288 struct VfioUserClientWrapper { 289 client: Arc<Mutex<Client>>, 290 } 291 292 impl Vfio for VfioUserClientWrapper { 293 fn region_read(&self, index: u32, offset: u64, data: &mut [u8]) { 294 self.client 295 .lock() 296 .unwrap() 297 .region_read(index, offset, data) 298 .ok(); 299 } 300 301 fn region_write(&self, index: u32, offset: u64, data: &[u8]) { 302 self.client 303 .lock() 304 .unwrap() 305 .region_write(index, offset, data) 306 .ok(); 307 } 308 309 fn get_irq_info(&self, irq_index: u32) -> Option<VfioIrq> { 310 self.client 311 .lock() 312 .unwrap() 313 .get_irq_info(irq_index) 314 .ok() 315 .map(|i| VfioIrq { 316 index: i.index, 317 flags: i.flags, 318 count: i.count, 319 }) 320 } 321 322 fn enable_irq(&self, irq_index: u32, event_fds: Vec<&EventFd>) -> Result<(), VfioError> { 323 info!( 324 "Enabling IRQ {:x} number of fds = {:?}", 325 irq_index, 326 event_fds.len() 327 ); 328 let fds: Vec<i32> = event_fds.iter().map(|e| e.as_raw_fd()).collect(); 329 330 // Batch into blocks of 16 fds as sendmsg() has a size limit 331 let mut sent_fds = 0; 332 let num_fds = event_fds.len() as u32; 333 while sent_fds < num_fds { 334 let remaining_fds = num_fds - sent_fds; 335 let count = if remaining_fds > 16 { 336 16 337 } else { 338 remaining_fds 339 }; 340 341 self.client 342 .lock() 343 .unwrap() 344 .set_irqs( 345 irq_index, 346 VFIO_IRQ_SET_DATA_EVENTFD | VFIO_IRQ_SET_ACTION_TRIGGER, 347 sent_fds, 348 count, 349 &fds[sent_fds as usize..(sent_fds + count) as usize], 350 ) 351 .map_err(VfioError::VfioUser)?; 352 353 sent_fds += count; 354 } 355 356 Ok(()) 357 } 358 359 fn disable_irq(&self, irq_index: u32) -> Result<(), VfioError> { 360 info!("Disabling IRQ {:x}", irq_index); 361 self.client 362 .lock() 363 .unwrap() 364 .set_irqs( 365 irq_index, 366 VFIO_IRQ_SET_DATA_NONE | VFIO_IRQ_SET_ACTION_TRIGGER, 367 0, 368 0, 369 &[], 370 ) 371 .map_err(VfioError::VfioUser) 372 } 373 374 fn unmask_irq(&self, irq_index: u32) -> Result<(), VfioError> { 375 info!("Unmasking IRQ {:x}", irq_index); 376 self.client 377 .lock() 378 .unwrap() 379 .set_irqs( 380 irq_index, 381 VFIO_IRQ_SET_DATA_NONE | VFIO_IRQ_SET_ACTION_UNMASK, 382 0, 383 1, 384 &[], 385 ) 386 .map_err(VfioError::VfioUser) 387 } 388 } 389 390 impl PciDevice for VfioUserPciDevice { 391 fn allocate_bars( 392 &mut self, 393 allocator: &Arc<Mutex<SystemAllocator>>, 394 mmio_allocator: &mut AddressAllocator, 395 ) -> Result<Vec<(GuestAddress, GuestUsize, PciBarRegionType)>, PciDeviceError> { 396 self.common 397 .allocate_bars(allocator, mmio_allocator, &self.vfio_wrapper) 398 } 399 400 fn free_bars( 401 &mut self, 402 allocator: &mut SystemAllocator, 403 mmio_allocator: &mut AddressAllocator, 404 ) -> Result<(), PciDeviceError> { 405 self.common.free_bars(allocator, mmio_allocator) 406 } 407 408 fn as_any(&mut self) -> &mut dyn Any { 409 self 410 } 411 412 fn detect_bar_reprogramming( 413 &mut self, 414 reg_idx: usize, 415 data: &[u8], 416 ) -> Option<BarReprogrammingParams> { 417 self.common 418 .configuration 419 .detect_bar_reprogramming(reg_idx, data) 420 } 421 422 fn write_config_register( 423 &mut self, 424 reg_idx: usize, 425 offset: u64, 426 data: &[u8], 427 ) -> Option<Arc<Barrier>> { 428 self.common 429 .write_config_register(reg_idx, offset, data, &self.vfio_wrapper) 430 } 431 432 fn read_config_register(&mut self, reg_idx: usize) -> u32 { 433 self.common 434 .read_config_register(reg_idx, &self.vfio_wrapper) 435 } 436 437 fn read_bar(&mut self, base: u64, offset: u64, data: &mut [u8]) { 438 self.common.read_bar(base, offset, data, &self.vfio_wrapper) 439 } 440 441 fn write_bar(&mut self, base: u64, offset: u64, data: &[u8]) -> Option<Arc<Barrier>> { 442 self.common 443 .write_bar(base, offset, data, &self.vfio_wrapper) 444 } 445 446 fn move_bar(&mut self, old_base: u64, new_base: u64) -> Result<(), std::io::Error> { 447 info!("Moving BAR 0x{:x} -> 0x{:x}", old_base, new_base); 448 for mmio_region in self.common.mmio_regions.iter_mut() { 449 if mmio_region.start.raw_value() == old_base { 450 mmio_region.start = GuestAddress(new_base); 451 452 if let Some(mem_slot) = mmio_region.mem_slot { 453 if let Some(host_addr) = mmio_region.host_addr { 454 // Remove original region 455 let old_region = self.vm.make_user_memory_region( 456 mem_slot, 457 old_base, 458 mmio_region.length as u64, 459 host_addr as u64, 460 false, 461 false, 462 ); 463 464 self.vm 465 .remove_user_memory_region(old_region) 466 .map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e))?; 467 468 let new_region = self.vm.make_user_memory_region( 469 mem_slot, 470 new_base, 471 mmio_region.length as u64, 472 host_addr as u64, 473 false, 474 false, 475 ); 476 477 self.vm 478 .create_user_memory_region(new_region) 479 .map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e))?; 480 } 481 } 482 info!("Moved bar 0x{:x} -> 0x{:x}", old_base, new_base); 483 } 484 } 485 486 Ok(()) 487 } 488 } 489 490 impl Drop for VfioUserPciDevice { 491 fn drop(&mut self) { 492 self.unmap_mmio_regions(); 493 494 if let Some(msix) = &self.common.interrupt.msix { 495 if msix.bar.enabled() { 496 self.common.disable_msix(&self.vfio_wrapper); 497 } 498 } 499 500 if let Some(msi) = &self.common.interrupt.msi { 501 if msi.cfg.enabled() { 502 self.common.disable_msi(&self.vfio_wrapper) 503 } 504 } 505 506 if self.common.interrupt.intx_in_use() { 507 self.common.disable_intx(&self.vfio_wrapper); 508 } 509 510 if let Err(e) = self.client.lock().unwrap().shutdown() { 511 error!("Failed shutting down vfio-user client: {}", e); 512 } 513 } 514 } 515 516 pub struct VfioUserDmaMapping<M: GuestAddressSpace> { 517 client: Arc<Mutex<Client>>, 518 memory: Arc<M>, 519 } 520 521 impl<M: GuestAddressSpace> VfioUserDmaMapping<M> { 522 pub fn new(client: Arc<Mutex<Client>>, memory: Arc<M>) -> Self { 523 Self { client, memory } 524 } 525 } 526 527 impl<M: GuestAddressSpace + Sync + Send> ExternalDmaMapping for VfioUserDmaMapping<M> { 528 fn map(&self, iova: u64, gpa: u64, size: u64) -> std::result::Result<(), std::io::Error> { 529 let mem = self.memory.memory(); 530 let guest_addr = GuestAddress(gpa); 531 let region = mem.find_region(guest_addr); 532 533 if let Some(region) = region { 534 let file_offset = region.file_offset().unwrap(); 535 let offset = (GuestAddress(gpa).checked_offset_from(region.start_addr())).unwrap() 536 + file_offset.start(); 537 538 self.client 539 .lock() 540 .unwrap() 541 .dma_map(offset, iova, size, file_offset.file().as_raw_fd()) 542 .map_err(|e| { 543 std::io::Error::new( 544 std::io::ErrorKind::Other, 545 format!("Error mapping region: {}", e), 546 ) 547 }) 548 } else { 549 return Err(std::io::Error::new( 550 std::io::ErrorKind::Other, 551 format!("Region not found for 0x{:x}", gpa), 552 )); 553 } 554 } 555 556 fn unmap(&self, iova: u64, size: u64) -> std::result::Result<(), std::io::Error> { 557 self.client 558 .lock() 559 .unwrap() 560 .dma_unmap(iova, size) 561 .map_err(|e| { 562 std::io::Error::new( 563 std::io::ErrorKind::Other, 564 format!("Error unmapping region: {}", e), 565 ) 566 }) 567 } 568 } 569