1 // Copyright © 2021 Intel Corporation 2 // 3 // SPDX-License-Identifier: Apache-2.0 4 // 5 6 use std::any::Any; 7 use std::os::unix::prelude::AsRawFd; 8 use std::ptr::null_mut; 9 use std::sync::{Arc, Barrier, Mutex}; 10 11 use hypervisor::HypervisorVmError; 12 use thiserror::Error; 13 use vfio_bindings::bindings::vfio::*; 14 use vfio_ioctls::VfioIrq; 15 use vfio_user::{Client, Error as VfioUserError}; 16 use vm_allocator::{AddressAllocator, SystemAllocator}; 17 use vm_device::dma_mapping::ExternalDmaMapping; 18 use vm_device::interrupt::{InterruptManager, InterruptSourceGroup, MsiIrqGroupConfig}; 19 use vm_device::{BusDevice, Resource}; 20 use vm_memory::bitmap::AtomicBitmap; 21 use vm_memory::{ 22 Address, GuestAddress, GuestAddressSpace, GuestMemory, GuestMemoryRegion, GuestRegionMmap, 23 }; 24 use vm_migration::{Migratable, MigratableError, Pausable, Snapshot, Snapshottable, Transportable}; 25 use vmm_sys_util::eventfd::EventFd; 26 27 use crate::vfio::{UserMemoryRegion, Vfio, VfioCommon, VfioError, VFIO_COMMON_ID}; 28 use crate::{ 29 BarReprogrammingParams, PciBarConfiguration, PciBdf, PciDevice, PciDeviceError, PciSubclass, 30 VfioPciError, 31 }; 32 33 pub struct VfioUserPciDevice { 34 id: String, 35 vm: Arc<dyn hypervisor::Vm>, 36 client: Arc<Mutex<Client>>, 37 common: VfioCommon, 38 memory_slot: Arc<dyn Fn() -> u32 + Send + Sync>, 39 } 40 41 #[derive(Error, Debug)] 42 pub enum VfioUserPciDeviceError { 43 #[error("Client error: {0}")] 44 Client(#[source] VfioUserError), 45 #[error("Failed to map VFIO PCI region into guest: {0}")] 46 MapRegionGuest(#[source] HypervisorVmError), 47 #[error("Failed to DMA map: {0}")] 48 DmaMap(#[source] VfioUserError), 49 #[error("Failed to DMA unmap: {0}")] 50 DmaUnmap(#[source] VfioUserError), 51 #[error("Failed to initialize legacy interrupts: {0}")] 52 InitializeLegacyInterrupts(#[source] VfioPciError), 53 #[error("Failed to create VfioCommon: {0}")] 54 CreateVfioCommon(#[source] VfioPciError), 55 } 56 57 #[derive(Copy, Clone)] 58 enum PciVfioUserSubclass { 59 VfioUserSubclass = 0xff, 60 } 61 62 impl PciSubclass for PciVfioUserSubclass { 63 fn get_register_value(&self) -> u8 { 64 *self as u8 65 } 66 } 67 68 impl VfioUserPciDevice { 69 #[allow(clippy::too_many_arguments)] 70 pub fn new( 71 id: String, 72 vm: &Arc<dyn hypervisor::Vm>, 73 client: Arc<Mutex<Client>>, 74 msi_interrupt_manager: Arc<dyn InterruptManager<GroupConfig = MsiIrqGroupConfig>>, 75 legacy_interrupt_group: Option<Arc<dyn InterruptSourceGroup>>, 76 bdf: PciBdf, 77 memory_slot: Arc<dyn Fn() -> u32 + Send + Sync>, 78 snapshot: Option<Snapshot>, 79 ) -> Result<Self, VfioUserPciDeviceError> { 80 let resettable = client.lock().unwrap().resettable(); 81 if resettable { 82 client 83 .lock() 84 .unwrap() 85 .reset() 86 .map_err(VfioUserPciDeviceError::Client)?; 87 } 88 89 let vfio_wrapper = VfioUserClientWrapper { 90 client: client.clone(), 91 }; 92 93 let common = VfioCommon::new( 94 msi_interrupt_manager, 95 legacy_interrupt_group, 96 Arc::new(vfio_wrapper) as Arc<dyn Vfio>, 97 &PciVfioUserSubclass::VfioUserSubclass, 98 bdf, 99 vm_migration::snapshot_from_id(snapshot.as_ref(), VFIO_COMMON_ID), 100 None, 101 ) 102 .map_err(VfioUserPciDeviceError::CreateVfioCommon)?; 103 104 Ok(Self { 105 id, 106 vm: vm.clone(), 107 client, 108 common, 109 memory_slot, 110 }) 111 } 112 113 pub fn map_mmio_regions(&mut self) -> Result<(), VfioUserPciDeviceError> { 114 for mmio_region in &mut self.common.mmio_regions { 115 let region_flags = self 116 .client 117 .lock() 118 .unwrap() 119 .region(mmio_region.index) 120 .unwrap() 121 .flags; 122 let file_offset = self 123 .client 124 .lock() 125 .unwrap() 126 .region(mmio_region.index) 127 .unwrap() 128 .file_offset 129 .clone(); 130 131 let sparse_areas = self 132 .client 133 .lock() 134 .unwrap() 135 .region(mmio_region.index) 136 .unwrap() 137 .sparse_areas 138 .clone(); 139 140 if region_flags & VFIO_REGION_INFO_FLAG_MMAP != 0 { 141 let mut prot = 0; 142 if region_flags & VFIO_REGION_INFO_FLAG_READ != 0 { 143 prot |= libc::PROT_READ; 144 } 145 if region_flags & VFIO_REGION_INFO_FLAG_WRITE != 0 { 146 prot |= libc::PROT_WRITE; 147 } 148 149 let mmaps = if sparse_areas.is_empty() { 150 vec![vfio_region_sparse_mmap_area { 151 offset: 0, 152 size: mmio_region.length, 153 }] 154 } else { 155 sparse_areas 156 }; 157 158 for s in mmaps.iter() { 159 // SAFETY: FFI call with correct arguments 160 let host_addr = unsafe { 161 libc::mmap( 162 null_mut(), 163 s.size as usize, 164 prot, 165 libc::MAP_SHARED, 166 file_offset.as_ref().unwrap().file().as_raw_fd(), 167 file_offset.as_ref().unwrap().start() as libc::off_t 168 + s.offset as libc::off_t, 169 ) 170 }; 171 172 if host_addr == libc::MAP_FAILED { 173 error!( 174 "Could not mmap regions, error:{}", 175 std::io::Error::last_os_error() 176 ); 177 continue; 178 } 179 180 let user_memory_region = UserMemoryRegion { 181 slot: (self.memory_slot)(), 182 start: mmio_region.start.0 + s.offset, 183 size: s.size, 184 host_addr: host_addr as u64, 185 }; 186 187 mmio_region.user_memory_regions.push(user_memory_region); 188 189 let mem_region = self.vm.make_user_memory_region( 190 user_memory_region.slot, 191 user_memory_region.start, 192 user_memory_region.size, 193 user_memory_region.host_addr, 194 false, 195 false, 196 ); 197 198 self.vm 199 .create_user_memory_region(mem_region) 200 .map_err(VfioUserPciDeviceError::MapRegionGuest)?; 201 } 202 } 203 } 204 205 Ok(()) 206 } 207 208 pub fn unmap_mmio_regions(&mut self) { 209 for mmio_region in self.common.mmio_regions.iter() { 210 for user_memory_region in mmio_region.user_memory_regions.iter() { 211 // Remove region 212 let r = self.vm.make_user_memory_region( 213 user_memory_region.slot, 214 user_memory_region.start, 215 user_memory_region.size, 216 user_memory_region.host_addr, 217 false, 218 false, 219 ); 220 221 if let Err(e) = self.vm.remove_user_memory_region(r) { 222 error!("Could not remove the userspace memory region: {}", e); 223 } 224 225 // Remove mmaps 226 // SAFETY: FFI call with correct arguments 227 let ret = unsafe { 228 libc::munmap( 229 user_memory_region.host_addr as *mut libc::c_void, 230 user_memory_region.size as usize, 231 ) 232 }; 233 if ret != 0 { 234 error!( 235 "Could not unmap region {}, error:{}", 236 mmio_region.index, 237 std::io::Error::last_os_error() 238 ); 239 } 240 } 241 } 242 } 243 244 pub fn dma_map( 245 &mut self, 246 region: &GuestRegionMmap<AtomicBitmap>, 247 ) -> Result<(), VfioUserPciDeviceError> { 248 let (fd, offset) = match region.file_offset() { 249 Some(_file_offset) => (_file_offset.file().as_raw_fd(), _file_offset.start()), 250 None => return Ok(()), 251 }; 252 253 self.client 254 .lock() 255 .unwrap() 256 .dma_map(offset, region.start_addr().raw_value(), region.len(), fd) 257 .map_err(VfioUserPciDeviceError::DmaMap) 258 } 259 260 pub fn dma_unmap( 261 &mut self, 262 region: &GuestRegionMmap<AtomicBitmap>, 263 ) -> Result<(), VfioUserPciDeviceError> { 264 self.client 265 .lock() 266 .unwrap() 267 .dma_unmap(region.start_addr().raw_value(), region.len()) 268 .map_err(VfioUserPciDeviceError::DmaUnmap) 269 } 270 } 271 272 impl BusDevice for VfioUserPciDevice { 273 fn read(&mut self, base: u64, offset: u64, data: &mut [u8]) { 274 self.read_bar(base, offset, data) 275 } 276 277 fn write(&mut self, base: u64, offset: u64, data: &[u8]) -> Option<Arc<Barrier>> { 278 self.write_bar(base, offset, data) 279 } 280 } 281 282 #[repr(u32)] 283 #[derive(Debug, PartialEq, Eq, PartialOrd, Ord)] 284 #[allow(dead_code)] 285 enum Regions { 286 Bar0, 287 Bar1, 288 Bar2, 289 Bar3, 290 Bar4, 291 Bar5, 292 Rom, 293 Config, 294 Vga, 295 Migration, 296 } 297 298 struct VfioUserClientWrapper { 299 client: Arc<Mutex<Client>>, 300 } 301 302 impl Vfio for VfioUserClientWrapper { 303 fn region_read(&self, index: u32, offset: u64, data: &mut [u8]) { 304 self.client 305 .lock() 306 .unwrap() 307 .region_read(index, offset, data) 308 .ok(); 309 } 310 311 fn region_write(&self, index: u32, offset: u64, data: &[u8]) { 312 self.client 313 .lock() 314 .unwrap() 315 .region_write(index, offset, data) 316 .ok(); 317 } 318 319 fn get_irq_info(&self, irq_index: u32) -> Option<VfioIrq> { 320 self.client 321 .lock() 322 .unwrap() 323 .get_irq_info(irq_index) 324 .ok() 325 .map(|i| VfioIrq { 326 index: i.index, 327 flags: i.flags, 328 count: i.count, 329 }) 330 } 331 332 fn enable_irq(&self, irq_index: u32, event_fds: Vec<&EventFd>) -> Result<(), VfioError> { 333 info!( 334 "Enabling IRQ {:x} number of fds = {:?}", 335 irq_index, 336 event_fds.len() 337 ); 338 let fds: Vec<i32> = event_fds.iter().map(|e| e.as_raw_fd()).collect(); 339 340 // Batch into blocks of 16 fds as sendmsg() has a size limit 341 let mut sent_fds = 0; 342 let num_fds = event_fds.len() as u32; 343 while sent_fds < num_fds { 344 let remaining_fds = num_fds - sent_fds; 345 let count = if remaining_fds > 16 { 346 16 347 } else { 348 remaining_fds 349 }; 350 351 self.client 352 .lock() 353 .unwrap() 354 .set_irqs( 355 irq_index, 356 VFIO_IRQ_SET_DATA_EVENTFD | VFIO_IRQ_SET_ACTION_TRIGGER, 357 sent_fds, 358 count, 359 &fds[sent_fds as usize..(sent_fds + count) as usize], 360 ) 361 .map_err(VfioError::VfioUser)?; 362 363 sent_fds += count; 364 } 365 366 Ok(()) 367 } 368 369 fn disable_irq(&self, irq_index: u32) -> Result<(), VfioError> { 370 info!("Disabling IRQ {:x}", irq_index); 371 self.client 372 .lock() 373 .unwrap() 374 .set_irqs( 375 irq_index, 376 VFIO_IRQ_SET_DATA_NONE | VFIO_IRQ_SET_ACTION_TRIGGER, 377 0, 378 0, 379 &[], 380 ) 381 .map_err(VfioError::VfioUser) 382 } 383 384 fn unmask_irq(&self, irq_index: u32) -> Result<(), VfioError> { 385 info!("Unmasking IRQ {:x}", irq_index); 386 self.client 387 .lock() 388 .unwrap() 389 .set_irqs( 390 irq_index, 391 VFIO_IRQ_SET_DATA_NONE | VFIO_IRQ_SET_ACTION_UNMASK, 392 0, 393 1, 394 &[], 395 ) 396 .map_err(VfioError::VfioUser) 397 } 398 } 399 400 impl PciDevice for VfioUserPciDevice { 401 fn allocate_bars( 402 &mut self, 403 allocator: &Arc<Mutex<SystemAllocator>>, 404 mmio32_allocator: &mut AddressAllocator, 405 mmio64_allocator: &mut AddressAllocator, 406 resources: Option<Vec<Resource>>, 407 ) -> Result<Vec<PciBarConfiguration>, PciDeviceError> { 408 self.common 409 .allocate_bars(allocator, mmio32_allocator, mmio64_allocator, resources) 410 } 411 412 fn free_bars( 413 &mut self, 414 allocator: &mut SystemAllocator, 415 mmio32_allocator: &mut AddressAllocator, 416 mmio64_allocator: &mut AddressAllocator, 417 ) -> Result<(), PciDeviceError> { 418 self.common 419 .free_bars(allocator, mmio32_allocator, mmio64_allocator) 420 } 421 422 fn as_any(&mut self) -> &mut dyn Any { 423 self 424 } 425 426 fn detect_bar_reprogramming( 427 &mut self, 428 reg_idx: usize, 429 data: &[u8], 430 ) -> Option<BarReprogrammingParams> { 431 self.common 432 .configuration 433 .detect_bar_reprogramming(reg_idx, data) 434 } 435 436 fn write_config_register( 437 &mut self, 438 reg_idx: usize, 439 offset: u64, 440 data: &[u8], 441 ) -> Option<Arc<Barrier>> { 442 self.common.write_config_register(reg_idx, offset, data) 443 } 444 445 fn read_config_register(&mut self, reg_idx: usize) -> u32 { 446 self.common.read_config_register(reg_idx) 447 } 448 449 fn read_bar(&mut self, base: u64, offset: u64, data: &mut [u8]) { 450 self.common.read_bar(base, offset, data) 451 } 452 453 fn write_bar(&mut self, base: u64, offset: u64, data: &[u8]) -> Option<Arc<Barrier>> { 454 self.common.write_bar(base, offset, data) 455 } 456 457 fn move_bar(&mut self, old_base: u64, new_base: u64) -> Result<(), std::io::Error> { 458 info!("Moving BAR 0x{:x} -> 0x{:x}", old_base, new_base); 459 for mmio_region in self.common.mmio_regions.iter_mut() { 460 if mmio_region.start.raw_value() == old_base { 461 mmio_region.start = GuestAddress(new_base); 462 463 for user_memory_region in mmio_region.user_memory_regions.iter_mut() { 464 // Remove old region 465 let old_region = self.vm.make_user_memory_region( 466 user_memory_region.slot, 467 user_memory_region.start, 468 user_memory_region.size, 469 user_memory_region.host_addr, 470 false, 471 false, 472 ); 473 474 self.vm 475 .remove_user_memory_region(old_region) 476 .map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e))?; 477 478 // Update the user memory region with the correct start address. 479 if new_base > old_base { 480 user_memory_region.start += new_base - old_base; 481 } else { 482 user_memory_region.start -= old_base - new_base; 483 } 484 485 // Insert new region 486 let new_region = self.vm.make_user_memory_region( 487 user_memory_region.slot, 488 user_memory_region.start, 489 user_memory_region.size, 490 user_memory_region.host_addr, 491 false, 492 false, 493 ); 494 495 self.vm 496 .create_user_memory_region(new_region) 497 .map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e))?; 498 } 499 info!("Moved bar 0x{:x} -> 0x{:x}", old_base, new_base); 500 } 501 } 502 503 Ok(()) 504 } 505 506 fn id(&self) -> Option<String> { 507 Some(self.id.clone()) 508 } 509 } 510 511 impl Drop for VfioUserPciDevice { 512 fn drop(&mut self) { 513 self.unmap_mmio_regions(); 514 515 if let Some(msix) = &self.common.interrupt.msix { 516 if msix.bar.enabled() { 517 self.common.disable_msix(); 518 } 519 } 520 521 if let Some(msi) = &self.common.interrupt.msi { 522 if msi.cfg.enabled() { 523 self.common.disable_msi() 524 } 525 } 526 527 if self.common.interrupt.intx_in_use() { 528 self.common.disable_intx(); 529 } 530 531 if let Err(e) = self.client.lock().unwrap().shutdown() { 532 error!("Failed shutting down vfio-user client: {}", e); 533 } 534 } 535 } 536 537 impl Pausable for VfioUserPciDevice {} 538 539 impl Snapshottable for VfioUserPciDevice { 540 fn id(&self) -> String { 541 self.id.clone() 542 } 543 544 fn snapshot(&mut self) -> std::result::Result<Snapshot, MigratableError> { 545 let mut vfio_pci_dev_snapshot = Snapshot::default(); 546 547 // Snapshot VfioCommon 548 vfio_pci_dev_snapshot.add_snapshot(self.common.id(), self.common.snapshot()?); 549 550 Ok(vfio_pci_dev_snapshot) 551 } 552 } 553 impl Transportable for VfioUserPciDevice {} 554 impl Migratable for VfioUserPciDevice {} 555 556 pub struct VfioUserDmaMapping<M: GuestAddressSpace> { 557 client: Arc<Mutex<Client>>, 558 memory: Arc<M>, 559 } 560 561 impl<M: GuestAddressSpace> VfioUserDmaMapping<M> { 562 pub fn new(client: Arc<Mutex<Client>>, memory: Arc<M>) -> Self { 563 Self { client, memory } 564 } 565 } 566 567 impl<M: GuestAddressSpace + Sync + Send> ExternalDmaMapping for VfioUserDmaMapping<M> { 568 fn map(&self, iova: u64, gpa: u64, size: u64) -> std::result::Result<(), std::io::Error> { 569 let mem = self.memory.memory(); 570 let guest_addr = GuestAddress(gpa); 571 let region = mem.find_region(guest_addr); 572 573 if let Some(region) = region { 574 let file_offset = region.file_offset().unwrap(); 575 let offset = (GuestAddress(gpa).checked_offset_from(region.start_addr())).unwrap() 576 + file_offset.start(); 577 578 self.client 579 .lock() 580 .unwrap() 581 .dma_map(offset, iova, size, file_offset.file().as_raw_fd()) 582 .map_err(|e| { 583 std::io::Error::new( 584 std::io::ErrorKind::Other, 585 format!("Error mapping region: {e}"), 586 ) 587 }) 588 } else { 589 Err(std::io::Error::new( 590 std::io::ErrorKind::Other, 591 format!("Region not found for 0x{gpa:x}"), 592 )) 593 } 594 } 595 596 fn unmap(&self, iova: u64, size: u64) -> std::result::Result<(), std::io::Error> { 597 self.client 598 .lock() 599 .unwrap() 600 .dma_unmap(iova, size) 601 .map_err(|e| { 602 std::io::Error::new( 603 std::io::ErrorKind::Other, 604 format!("Error unmapping region: {e}"), 605 ) 606 }) 607 } 608 } 609