1 // Copyright © 2021 Intel Corporation 2 // 3 // SPDX-License-Identifier: Apache-2.0 4 // 5 6 use crate::vfio::{UserMemoryRegion, Vfio, VfioCommon, VfioError, VFIO_COMMON_ID}; 7 use crate::{BarReprogrammingParams, PciBarConfiguration, VfioPciError}; 8 use crate::{PciBdf, PciDevice, PciDeviceError, PciSubclass}; 9 use hypervisor::HypervisorVmError; 10 use std::any::Any; 11 use std::os::unix::prelude::AsRawFd; 12 use std::ptr::null_mut; 13 use std::sync::{Arc, Barrier, Mutex}; 14 use std::u32; 15 use thiserror::Error; 16 use vfio_bindings::bindings::vfio::*; 17 use vfio_ioctls::VfioIrq; 18 use vfio_user::{Client, Error as VfioUserError}; 19 use vm_allocator::{AddressAllocator, SystemAllocator}; 20 use vm_device::dma_mapping::ExternalDmaMapping; 21 use vm_device::interrupt::{InterruptManager, InterruptSourceGroup, MsiIrqGroupConfig}; 22 use vm_device::{BusDevice, Resource}; 23 use vm_memory::bitmap::AtomicBitmap; 24 use vm_memory::{ 25 Address, GuestAddress, GuestAddressSpace, GuestMemory, GuestMemoryRegion, GuestRegionMmap, 26 }; 27 use vm_migration::{Migratable, MigratableError, Pausable, Snapshot, Snapshottable, Transportable}; 28 use vmm_sys_util::eventfd::EventFd; 29 30 pub struct VfioUserPciDevice { 31 id: String, 32 vm: Arc<dyn hypervisor::Vm>, 33 client: Arc<Mutex<Client>>, 34 common: VfioCommon, 35 memory_slot: Arc<dyn Fn() -> u32 + Send + Sync>, 36 } 37 38 #[derive(Error, Debug)] 39 pub enum VfioUserPciDeviceError { 40 #[error("Client error: {0}")] 41 Client(#[source] VfioUserError), 42 #[error("Failed to map VFIO PCI region into guest: {0}")] 43 MapRegionGuest(#[source] HypervisorVmError), 44 #[error("Failed to DMA map: {0}")] 45 DmaMap(#[source] VfioUserError), 46 #[error("Failed to DMA unmap: {0}")] 47 DmaUnmap(#[source] VfioUserError), 48 #[error("Failed to initialize legacy interrupts: {0}")] 49 InitializeLegacyInterrupts(#[source] VfioPciError), 50 #[error("Failed to create VfioCommon: {0}")] 51 CreateVfioCommon(#[source] VfioPciError), 52 } 53 54 #[derive(Copy, Clone)] 55 enum PciVfioUserSubclass { 56 VfioUserSubclass = 0xff, 57 } 58 59 impl PciSubclass for PciVfioUserSubclass { 60 fn get_register_value(&self) -> u8 { 61 *self as u8 62 } 63 } 64 65 impl VfioUserPciDevice { 66 #[allow(clippy::too_many_arguments)] 67 pub fn new( 68 id: String, 69 vm: &Arc<dyn hypervisor::Vm>, 70 client: Arc<Mutex<Client>>, 71 msi_interrupt_manager: Arc<dyn InterruptManager<GroupConfig = MsiIrqGroupConfig>>, 72 legacy_interrupt_group: Option<Arc<dyn InterruptSourceGroup>>, 73 bdf: PciBdf, 74 memory_slot: Arc<dyn Fn() -> u32 + Send + Sync>, 75 snapshot: Option<Snapshot>, 76 ) -> Result<Self, VfioUserPciDeviceError> { 77 let resettable = client.lock().unwrap().resettable(); 78 if resettable { 79 client 80 .lock() 81 .unwrap() 82 .reset() 83 .map_err(VfioUserPciDeviceError::Client)?; 84 } 85 86 let vfio_wrapper = VfioUserClientWrapper { 87 client: client.clone(), 88 }; 89 90 let common = VfioCommon::new( 91 msi_interrupt_manager, 92 legacy_interrupt_group, 93 Arc::new(vfio_wrapper) as Arc<dyn Vfio>, 94 &PciVfioUserSubclass::VfioUserSubclass, 95 bdf, 96 vm_migration::snapshot_from_id(snapshot.as_ref(), VFIO_COMMON_ID), 97 None, 98 ) 99 .map_err(VfioUserPciDeviceError::CreateVfioCommon)?; 100 101 Ok(Self { 102 id, 103 vm: vm.clone(), 104 client, 105 common, 106 memory_slot, 107 }) 108 } 109 110 pub fn map_mmio_regions(&mut self) -> Result<(), VfioUserPciDeviceError> { 111 for mmio_region in &mut self.common.mmio_regions { 112 let region_flags = self 113 .client 114 .lock() 115 .unwrap() 116 .region(mmio_region.index) 117 .unwrap() 118 .flags; 119 let file_offset = self 120 .client 121 .lock() 122 .unwrap() 123 .region(mmio_region.index) 124 .unwrap() 125 .file_offset 126 .clone(); 127 128 let sparse_areas = self 129 .client 130 .lock() 131 .unwrap() 132 .region(mmio_region.index) 133 .unwrap() 134 .sparse_areas 135 .clone(); 136 137 if region_flags & VFIO_REGION_INFO_FLAG_MMAP != 0 { 138 let mut prot = 0; 139 if region_flags & VFIO_REGION_INFO_FLAG_READ != 0 { 140 prot |= libc::PROT_READ; 141 } 142 if region_flags & VFIO_REGION_INFO_FLAG_WRITE != 0 { 143 prot |= libc::PROT_WRITE; 144 } 145 146 let mmaps = if sparse_areas.is_empty() { 147 vec![vfio_region_sparse_mmap_area { 148 offset: 0, 149 size: mmio_region.length, 150 }] 151 } else { 152 sparse_areas 153 }; 154 155 for s in mmaps.iter() { 156 // SAFETY: FFI call with correct arguments 157 let host_addr = unsafe { 158 libc::mmap( 159 null_mut(), 160 s.size as usize, 161 prot, 162 libc::MAP_SHARED, 163 file_offset.as_ref().unwrap().file().as_raw_fd(), 164 file_offset.as_ref().unwrap().start() as libc::off_t 165 + s.offset as libc::off_t, 166 ) 167 }; 168 169 if host_addr == libc::MAP_FAILED { 170 error!( 171 "Could not mmap regions, error:{}", 172 std::io::Error::last_os_error() 173 ); 174 continue; 175 } 176 177 let user_memory_region = UserMemoryRegion { 178 slot: (self.memory_slot)(), 179 start: mmio_region.start.0 + s.offset, 180 size: s.size, 181 host_addr: host_addr as u64, 182 }; 183 184 mmio_region.user_memory_regions.push(user_memory_region); 185 186 let mem_region = self.vm.make_user_memory_region( 187 user_memory_region.slot, 188 user_memory_region.start, 189 user_memory_region.size, 190 user_memory_region.host_addr, 191 false, 192 false, 193 ); 194 195 self.vm 196 .create_user_memory_region(mem_region) 197 .map_err(VfioUserPciDeviceError::MapRegionGuest)?; 198 } 199 } 200 } 201 202 Ok(()) 203 } 204 205 pub fn unmap_mmio_regions(&mut self) { 206 for mmio_region in self.common.mmio_regions.iter() { 207 for user_memory_region in mmio_region.user_memory_regions.iter() { 208 // Remove region 209 let r = self.vm.make_user_memory_region( 210 user_memory_region.slot, 211 user_memory_region.start, 212 user_memory_region.size, 213 user_memory_region.host_addr, 214 false, 215 false, 216 ); 217 218 if let Err(e) = self.vm.remove_user_memory_region(r) { 219 error!("Could not remove the userspace memory region: {}", e); 220 } 221 222 // Remove mmaps 223 // SAFETY: FFI call with correct arguments 224 let ret = unsafe { 225 libc::munmap( 226 user_memory_region.host_addr as *mut libc::c_void, 227 user_memory_region.size as usize, 228 ) 229 }; 230 if ret != 0 { 231 error!( 232 "Could not unmap region {}, error:{}", 233 mmio_region.index, 234 std::io::Error::last_os_error() 235 ); 236 } 237 } 238 } 239 } 240 241 pub fn dma_map( 242 &mut self, 243 region: &GuestRegionMmap<AtomicBitmap>, 244 ) -> Result<(), VfioUserPciDeviceError> { 245 let (fd, offset) = match region.file_offset() { 246 Some(_file_offset) => (_file_offset.file().as_raw_fd(), _file_offset.start()), 247 None => return Ok(()), 248 }; 249 250 self.client 251 .lock() 252 .unwrap() 253 .dma_map(offset, region.start_addr().raw_value(), region.len(), fd) 254 .map_err(VfioUserPciDeviceError::DmaMap) 255 } 256 257 pub fn dma_unmap( 258 &mut self, 259 region: &GuestRegionMmap<AtomicBitmap>, 260 ) -> Result<(), VfioUserPciDeviceError> { 261 self.client 262 .lock() 263 .unwrap() 264 .dma_unmap(region.start_addr().raw_value(), region.len()) 265 .map_err(VfioUserPciDeviceError::DmaUnmap) 266 } 267 } 268 269 impl BusDevice for VfioUserPciDevice { 270 fn read(&mut self, base: u64, offset: u64, data: &mut [u8]) { 271 self.read_bar(base, offset, data) 272 } 273 274 fn write(&mut self, base: u64, offset: u64, data: &[u8]) -> Option<Arc<Barrier>> { 275 self.write_bar(base, offset, data) 276 } 277 } 278 279 #[repr(u32)] 280 #[derive(Debug, PartialEq, Eq, PartialOrd, Ord)] 281 #[allow(dead_code)] 282 enum Regions { 283 Bar0, 284 Bar1, 285 Bar2, 286 Bar3, 287 Bar4, 288 Bar5, 289 Rom, 290 Config, 291 Vga, 292 Migration, 293 } 294 295 struct VfioUserClientWrapper { 296 client: Arc<Mutex<Client>>, 297 } 298 299 impl Vfio for VfioUserClientWrapper { 300 fn region_read(&self, index: u32, offset: u64, data: &mut [u8]) { 301 self.client 302 .lock() 303 .unwrap() 304 .region_read(index, offset, data) 305 .ok(); 306 } 307 308 fn region_write(&self, index: u32, offset: u64, data: &[u8]) { 309 self.client 310 .lock() 311 .unwrap() 312 .region_write(index, offset, data) 313 .ok(); 314 } 315 316 fn get_irq_info(&self, irq_index: u32) -> Option<VfioIrq> { 317 self.client 318 .lock() 319 .unwrap() 320 .get_irq_info(irq_index) 321 .ok() 322 .map(|i| VfioIrq { 323 index: i.index, 324 flags: i.flags, 325 count: i.count, 326 }) 327 } 328 329 fn enable_irq(&self, irq_index: u32, event_fds: Vec<&EventFd>) -> Result<(), VfioError> { 330 info!( 331 "Enabling IRQ {:x} number of fds = {:?}", 332 irq_index, 333 event_fds.len() 334 ); 335 let fds: Vec<i32> = event_fds.iter().map(|e| e.as_raw_fd()).collect(); 336 337 // Batch into blocks of 16 fds as sendmsg() has a size limit 338 let mut sent_fds = 0; 339 let num_fds = event_fds.len() as u32; 340 while sent_fds < num_fds { 341 let remaining_fds = num_fds - sent_fds; 342 let count = if remaining_fds > 16 { 343 16 344 } else { 345 remaining_fds 346 }; 347 348 self.client 349 .lock() 350 .unwrap() 351 .set_irqs( 352 irq_index, 353 VFIO_IRQ_SET_DATA_EVENTFD | VFIO_IRQ_SET_ACTION_TRIGGER, 354 sent_fds, 355 count, 356 &fds[sent_fds as usize..(sent_fds + count) as usize], 357 ) 358 .map_err(VfioError::VfioUser)?; 359 360 sent_fds += count; 361 } 362 363 Ok(()) 364 } 365 366 fn disable_irq(&self, irq_index: u32) -> Result<(), VfioError> { 367 info!("Disabling IRQ {:x}", irq_index); 368 self.client 369 .lock() 370 .unwrap() 371 .set_irqs( 372 irq_index, 373 VFIO_IRQ_SET_DATA_NONE | VFIO_IRQ_SET_ACTION_TRIGGER, 374 0, 375 0, 376 &[], 377 ) 378 .map_err(VfioError::VfioUser) 379 } 380 381 fn unmask_irq(&self, irq_index: u32) -> Result<(), VfioError> { 382 info!("Unmasking IRQ {:x}", irq_index); 383 self.client 384 .lock() 385 .unwrap() 386 .set_irqs( 387 irq_index, 388 VFIO_IRQ_SET_DATA_NONE | VFIO_IRQ_SET_ACTION_UNMASK, 389 0, 390 1, 391 &[], 392 ) 393 .map_err(VfioError::VfioUser) 394 } 395 } 396 397 impl PciDevice for VfioUserPciDevice { 398 fn allocate_bars( 399 &mut self, 400 allocator: &Arc<Mutex<SystemAllocator>>, 401 mmio32_allocator: &mut AddressAllocator, 402 mmio64_allocator: &mut AddressAllocator, 403 resources: Option<Vec<Resource>>, 404 ) -> Result<Vec<PciBarConfiguration>, PciDeviceError> { 405 self.common 406 .allocate_bars(allocator, mmio32_allocator, mmio64_allocator, resources) 407 } 408 409 fn free_bars( 410 &mut self, 411 allocator: &mut SystemAllocator, 412 mmio32_allocator: &mut AddressAllocator, 413 mmio64_allocator: &mut AddressAllocator, 414 ) -> Result<(), PciDeviceError> { 415 self.common 416 .free_bars(allocator, mmio32_allocator, mmio64_allocator) 417 } 418 419 fn as_any(&mut self) -> &mut dyn Any { 420 self 421 } 422 423 fn detect_bar_reprogramming( 424 &mut self, 425 reg_idx: usize, 426 data: &[u8], 427 ) -> Option<BarReprogrammingParams> { 428 self.common 429 .configuration 430 .detect_bar_reprogramming(reg_idx, data) 431 } 432 433 fn write_config_register( 434 &mut self, 435 reg_idx: usize, 436 offset: u64, 437 data: &[u8], 438 ) -> Option<Arc<Barrier>> { 439 self.common.write_config_register(reg_idx, offset, data) 440 } 441 442 fn read_config_register(&mut self, reg_idx: usize) -> u32 { 443 self.common.read_config_register(reg_idx) 444 } 445 446 fn read_bar(&mut self, base: u64, offset: u64, data: &mut [u8]) { 447 self.common.read_bar(base, offset, data) 448 } 449 450 fn write_bar(&mut self, base: u64, offset: u64, data: &[u8]) -> Option<Arc<Barrier>> { 451 self.common.write_bar(base, offset, data) 452 } 453 454 fn move_bar(&mut self, old_base: u64, new_base: u64) -> Result<(), std::io::Error> { 455 info!("Moving BAR 0x{:x} -> 0x{:x}", old_base, new_base); 456 for mmio_region in self.common.mmio_regions.iter_mut() { 457 if mmio_region.start.raw_value() == old_base { 458 mmio_region.start = GuestAddress(new_base); 459 460 for user_memory_region in mmio_region.user_memory_regions.iter_mut() { 461 // Remove old region 462 let old_region = self.vm.make_user_memory_region( 463 user_memory_region.slot, 464 user_memory_region.start, 465 user_memory_region.size, 466 user_memory_region.host_addr, 467 false, 468 false, 469 ); 470 471 self.vm 472 .remove_user_memory_region(old_region) 473 .map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e))?; 474 475 // Update the user memory region with the correct start address. 476 if new_base > old_base { 477 user_memory_region.start += new_base - old_base; 478 } else { 479 user_memory_region.start -= old_base - new_base; 480 } 481 482 // Insert new region 483 let new_region = self.vm.make_user_memory_region( 484 user_memory_region.slot, 485 user_memory_region.start, 486 user_memory_region.size, 487 user_memory_region.host_addr, 488 false, 489 false, 490 ); 491 492 self.vm 493 .create_user_memory_region(new_region) 494 .map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e))?; 495 } 496 info!("Moved bar 0x{:x} -> 0x{:x}", old_base, new_base); 497 } 498 } 499 500 Ok(()) 501 } 502 503 fn id(&self) -> Option<String> { 504 Some(self.id.clone()) 505 } 506 } 507 508 impl Drop for VfioUserPciDevice { 509 fn drop(&mut self) { 510 self.unmap_mmio_regions(); 511 512 if let Some(msix) = &self.common.interrupt.msix { 513 if msix.bar.enabled() { 514 self.common.disable_msix(); 515 } 516 } 517 518 if let Some(msi) = &self.common.interrupt.msi { 519 if msi.cfg.enabled() { 520 self.common.disable_msi() 521 } 522 } 523 524 if self.common.interrupt.intx_in_use() { 525 self.common.disable_intx(); 526 } 527 528 if let Err(e) = self.client.lock().unwrap().shutdown() { 529 error!("Failed shutting down vfio-user client: {}", e); 530 } 531 } 532 } 533 534 impl Pausable for VfioUserPciDevice {} 535 536 impl Snapshottable for VfioUserPciDevice { 537 fn id(&self) -> String { 538 self.id.clone() 539 } 540 541 fn snapshot(&mut self) -> std::result::Result<Snapshot, MigratableError> { 542 let mut vfio_pci_dev_snapshot = Snapshot::default(); 543 544 // Snapshot VfioCommon 545 vfio_pci_dev_snapshot.add_snapshot(self.common.id(), self.common.snapshot()?); 546 547 Ok(vfio_pci_dev_snapshot) 548 } 549 } 550 impl Transportable for VfioUserPciDevice {} 551 impl Migratable for VfioUserPciDevice {} 552 553 pub struct VfioUserDmaMapping<M: GuestAddressSpace> { 554 client: Arc<Mutex<Client>>, 555 memory: Arc<M>, 556 } 557 558 impl<M: GuestAddressSpace> VfioUserDmaMapping<M> { 559 pub fn new(client: Arc<Mutex<Client>>, memory: Arc<M>) -> Self { 560 Self { client, memory } 561 } 562 } 563 564 impl<M: GuestAddressSpace + Sync + Send> ExternalDmaMapping for VfioUserDmaMapping<M> { 565 fn map(&self, iova: u64, gpa: u64, size: u64) -> std::result::Result<(), std::io::Error> { 566 let mem = self.memory.memory(); 567 let guest_addr = GuestAddress(gpa); 568 let region = mem.find_region(guest_addr); 569 570 if let Some(region) = region { 571 let file_offset = region.file_offset().unwrap(); 572 let offset = (GuestAddress(gpa).checked_offset_from(region.start_addr())).unwrap() 573 + file_offset.start(); 574 575 self.client 576 .lock() 577 .unwrap() 578 .dma_map(offset, iova, size, file_offset.file().as_raw_fd()) 579 .map_err(|e| { 580 std::io::Error::new( 581 std::io::ErrorKind::Other, 582 format!("Error mapping region: {e}"), 583 ) 584 }) 585 } else { 586 Err(std::io::Error::new( 587 std::io::ErrorKind::Other, 588 format!("Region not found for 0x{gpa:x}"), 589 )) 590 } 591 } 592 593 fn unmap(&self, iova: u64, size: u64) -> std::result::Result<(), std::io::Error> { 594 self.client 595 .lock() 596 .unwrap() 597 .dma_unmap(iova, size) 598 .map_err(|e| { 599 std::io::Error::new( 600 std::io::ErrorKind::Other, 601 format!("Error unmapping region: {e}"), 602 ) 603 }) 604 } 605 } 606