Lines Matching +full:add +full:- +full:user +full:- +full:device
3 // SPDX-License-Identifier: Apache-2.0 OR BSD-3-Clause
50 #[error("Failed to create user memory region")]
52 #[error("Failed to DMA map: {0} for device {1} (guest BDF: {2})")]
54 #[error("Failed to DMA unmap: {0} for device {1} (guest BDF: {2})")]
60 #[error("Failed to enable MSI-x")]
86 fn get_register_value(&self) -> u8 { in get_register_value()
121 fn update(&mut self, offset: u64, data: &[u8]) -> Option<InterruptUpdateAction> { in update()
155 fn update(&mut self, offset: u64, data: &[u8]) -> Option<InterruptUpdateAction> { in update()
176 fn table_accessed(&self, bar_index: u32, offset: u64) -> bool { in table_accessed()
192 fn update_msi(&mut self, offset: u64, data: &[u8]) -> Option<InterruptUpdateAction> { in update_msi()
201 fn update_msix(&mut self, offset: u64, data: &[u8]) -> Option<InterruptUpdateAction> { in update_msix()
210 fn accessed(&self, offset: u64) -> Option<(PciCapabilityId, u64)> { in accessed()
231 fn msix_table_accessed(&self, bar_index: u32, offset: u64) -> bool { in msix_table_accessed()
241 let offset = offset - u64::from(msix.cap.table_offset()); in msix_write_table()
248 let offset = offset - u64::from(msix.cap.table_offset()); in msix_read_table()
253 pub(crate) fn intx_in_use(&self) -> bool { in intx_in_use()
280 fn check_range(&self, guest_addr: u64, size: u64) -> bool; in check_range()
281 fn find_user_address(&self, guest_addr: u64) -> Result<u64, io::Error>; in find_user_address()
286 fn check_range(&self, guest_addr: u64, size: u64) -> bool { in check_range()
301 // Locate the user region address for a guest address within all mmio regions
302 fn find_user_address(&self, guest_addr: u64) -> Result<u64, io::Error> { in find_user_address()
308 return Ok(user_region.host_addr + (guest_addr - user_region.start)); in find_user_address()
314 "unable to find user address: 0x{guest_addr:x}" in find_user_address()
323 #[error("VFIO user error")]
328 fn read_config_byte(&self, offset: u32) -> u8 { in read_config_byte()
334 fn read_config_word(&self, offset: u32) -> u16 { in read_config_word()
340 fn read_config_dword(&self, offset: u32) -> u32 { in read_config_dword()
359 fn enable_msi(&self, fds: Vec<&EventFd>) -> Result<(), VfioError> { in enable_msi()
363 fn disable_msi(&self) -> Result<(), VfioError> { in disable_msi()
367 fn enable_msix(&self, fds: Vec<&EventFd>) -> Result<(), VfioError> { in enable_msix()
371 fn disable_msix(&self) -> Result<(), VfioError> { in disable_msix()
383 fn get_irq_info(&self, _irq_index: u32) -> Option<VfioIrq> { in get_irq_info()
387 fn enable_irq(&self, _irq_index: u32, _event_fds: Vec<&EventFd>) -> Result<(), VfioError> { in enable_irq()
391 fn disable_irq(&self, _irq_index: u32) -> Result<(), VfioError> { in disable_irq()
395 fn unmask_irq(&self, _irq_index: u32) -> Result<(), VfioError> { in unmask_irq()
401 device: Arc<VfioDevice>, field
405 fn new(device: Arc<VfioDevice>) -> Self { in new()
406 Self { device } in new()
412 self.device.region_read(index, data, offset) in region_read()
416 self.device.region_write(index, data, offset) in region_write()
419 fn get_irq_info(&self, irq_index: u32) -> Option<VfioIrq> { in get_irq_info()
420 self.device.get_irq_info(irq_index).copied() in get_irq_info()
423 fn enable_irq(&self, irq_index: u32, event_fds: Vec<&EventFd>) -> Result<(), VfioError> { in enable_irq()
424 self.device in enable_irq()
429 fn disable_irq(&self, irq_index: u32) -> Result<(), VfioError> { in disable_irq()
430 self.device in disable_irq()
435 fn unmask_irq(&self, irq_index: u32) -> Result<(), VfioError> { in unmask_irq()
436 self.device in unmask_irq()
474 ) -> Result<Self, VfioPciError> { in new()
490 PciHeaderType::Device, in new()
549 fn fixup_msix_region(&mut self, bar_id: u32, region_size: u64) -> u64 { in fixup_msix_region()
565 msix_cap.pba_set_offset((size / 2 + pba_offset - table_offset) as u32); in fixup_msix_region()
569 // MSI-X not supported for this device in fixup_msix_region()
582 ) -> Result<Vec<PciBarConfiguration>, PciDeviceError> { in allocate_bars()
636 // Is this a 64-bit BAR? in allocate_bars()
657 … // And read back BAR value. The device will write zeros for bits it doesn't care about in allocate_bars()
673 // Invert bits and add 1 to calculate size in allocate_bars()
679 // Query size of upper BAR of 64-bit BAR in allocate_bars()
696 // Invert and add 1 to to find size in allocate_bars()
709 // Invert and add 1 to to find size in allocate_bars()
790 ) -> Result<(), PciDeviceError> { in free_bars()
807 fn parse_msix_capabilities(&mut self, cap: u8) -> MsixCap { in parse_msix_capabilities()
852 fn parse_msi_capabilities(&mut self, cap: u8) -> u16 { in parse_msi_capabilities()
874 /// Returns true, if the device claims to have a PCI capability list.
875 fn has_capabilities(&self) -> bool { in has_capabilities()
880 fn get_msix_cap_idx(&self) -> Option<usize> { in get_msix_cap_idx()
930 // Parse capability only if the VFIO device in parse_capabilities()
941 // Parse capability only if the VFIO device in parse_capabilities()
942 // supports MSI-X. in parse_capabilities()
956 // Break out of the loop, if we either find the end or we have a broken device. This in parse_capabilities()
957 // doesn't handle all cases where a device might send us in a loop here, but it in parse_capabilities()
958 // handles case of a device returning 0xFF instead of implementing a real in parse_capabilities()
1040 pub(crate) fn enable_intx(&mut self) -> Result<(), VfioPciError> { in enable_intx()
1070 pub(crate) fn enable_msi(&self) -> Result<(), VfioPciError> { in enable_msi()
1095 pub(crate) fn enable_msix(&self) -> Result<(), VfioPciError> { in enable_msix()
1116 error!("Could not disable MSI-X: {}", e); in disable_msix()
1120 fn initialize_legacy_interrupt(&mut self) -> Result<(), VfioPciError> { in initialize_legacy_interrupt()
1141 fn update_msi_capabilities(&mut self, offset: u64, data: &[u8]) -> Result<(), VfioPciError> { in update_msi_capabilities()
1159 fn update_msix_capabilities(&mut self, offset: u64, data: &[u8]) -> Result<(), VfioPciError> { in update_msix_capabilities()
1162 // Disable INTx before we can enable MSI-X in update_msix_capabilities()
1167 // Fallback onto INTx when disabling MSI-X in update_msix_capabilities()
1177 fn find_region(&self, addr: u64) -> Option<MmioRegion> { in find_region()
1191 let offset = addr - region.start.raw_value(); in read_bar()
1215 ) -> Option<Arc<Barrier>> { in write_bar()
1218 let offset = addr - region.start.raw_value(); in write_bar()
1220 // If the MSI-X table is written to, we need to update our cache. in write_bar()
1245 ) -> (Vec<BarReprogrammingParams>, Option<Arc<Barrier>>) { in write_config_register()
1248 // VFIO device. in write_config_register()
1264 // If the MSI or MSI-X capabilities are accessed, we need to in write_config_register()
1267 // trigger a VFIO MSI or MSI-X toggle. in write_config_register()
1269 let cap_offset: u64 = reg - cap_base + offset; in write_config_register()
1278 error!("Could not update MSI-X capabilities: {}", e); in write_config_register()
1285 // Make sure to write to the device's PCI config space after MSI/MSI-X in write_config_register()
1292 // to the device region to update the MSI Enable bit. in write_config_register()
1317 pub(crate) fn read_config_register(&mut self, reg_idx: usize) -> u32 { in read_config_register()
1320 // use that and not the VFIO device BARs as it does not map in read_config_register()
1337 // Since we don't support passing multi-functions devices, we should in read_config_register()
1338 // mask the multi-function bit, bit 7 of the Header Type byte on the in read_config_register()
1346 // The config register read comes from the VFIO device itself. in read_config_register()
1356 fn state(&self) -> VfioCommonState { in state()
1384 ) -> Result<(), VfioPciError> { in set_state()
1413 fn id(&self) -> String { in id()
1417 fn snapshot(&mut self) -> std::result::Result<Snapshot, MigratableError> { in snapshot()
1428 // Snapshot MSI-X in snapshot()
1437 /// VfioPciDevice represents a VFIO PCI device.
1440 /// A VfioPciDevice is bound to a VfioDevice and is also a PCI device.
1446 device: Arc<VfioDevice>, field
1456 /// Constructs a new Vfio Pci device for the given Vfio device
1461 device: VfioDevice, in new()
1471 ) -> Result<Self, VfioPciError> { in new()
1472 let device = Arc::new(device); in new() localVariable
1473 device.reset(); in new()
1475 let vfio_wrapper = VfioDeviceWrapper::new(Arc::clone(&device)); in new()
1490 device, in new()
1502 pub fn iommu_attached(&self) -> bool { in iommu_attached()
1512 ) -> Result<Vec<VfioRegionSparseMmapArea>, VfioPciError> { in generate_sparse_areas()
1532 // In case the region contains the MSI-X vectors table or in generate_sparse_areas()
1533 // the MSI-X PBA table, we must calculate the subregions in generate_sparse_areas()
1536 // to these MSI-X specific ranges. If these region don't align in generate_sparse_areas()
1563 size: range_offset - current_offset, in generate_sparse_areas()
1572 size: region_size - current_offset, in generate_sparse_areas()
1595 /// * `vm` - The VM object. It is used to set the VFIO MMIO regions
1596 /// as user memory regions.
1597 /// * `mem_slot` - The closure to return a memory slot.
1598 pub fn map_mmio_regions(&mut self) -> Result<(), VfioPciError> { in map_mmio_regions()
1599 let fd = self.device.as_raw_fd(); in map_mmio_regions()
1602 let region_flags = self.device.get_region_flags(region.index); in map_mmio_regions()
1614 self.device.get_region_caps(region.index) in map_mmio_regions()
1619 // Don't try to mmap the region if it contains MSI-X table or in map_mmio_regions()
1620 // MSI-X PBA subregion, and if we couldn't find MSIX_MAPPABLE in map_mmio_regions()
1630 let mmap_size = self.device.get_region_size(region.index); in map_mmio_regions()
1631 let mmap_offset = self.device.get_region_offset(region.index); in map_mmio_regions()
1766 pub fn dma_map(&self, iova: u64, size: u64, user_addr: u64) -> Result<(), VfioPciError> { in dma_map()
1776 pub fn dma_unmap(&self, iova: u64, size: u64) -> Result<(), VfioPciError> { in dma_unmap()
1786 pub fn mmio_regions(&self) -> Vec<MmioRegion> { in mmio_regions()
1818 fn write(&mut self, base: u64, offset: u64, data: &[u8]) -> Option<Arc<Barrier>> { in write()
1823 // Offset of the 16-bit status register in the PCI configuration space.
1837 // 64-bit memory bar flag.
1843 // Number of BARs for a PCI device
1859 ) -> Result<Vec<PciBarConfiguration>, PciDeviceError> { in allocate_bars()
1869 ) -> Result<(), PciDeviceError> { in free_bars()
1879 ) -> (Vec<BarReprogrammingParams>, Option<Arc<Barrier>>) { in write_config_register()
1883 fn read_config_register(&mut self, reg_idx: usize) -> u32 { in read_config_register()
1891 fn write_bar(&mut self, base: u64, offset: u64, data: &[u8]) -> Option<Arc<Barrier>> { in write_bar()
1895 fn move_bar(&mut self, old_base: u64, new_base: u64) -> Result<(), io::Error> { in move_bar()
1932 // Update the user memory region with the correct start address. in move_bar()
1934 user_memory_region.start += new_base - old_base; in move_bar()
1936 user_memory_region.start -= old_base - new_base; in move_bar()
1979 fn as_any_mut(&mut self) -> &mut dyn Any { in as_any_mut()
1983 fn id(&self) -> Option<String> { in id()
1991 fn id(&self) -> String { in id()
1995 fn snapshot(&mut self) -> std::result::Result<Snapshot, MigratableError> { in snapshot()
2026 ) -> Self { in new()
2036 fn map(&self, iova: u64, gpa: u64, size: u64) -> std::result::Result<(), io::Error> { in map()
2044 … format!("unable to retrieve user address for gpa 0x{gpa:x} from guest memory region: {e}") in map()
2066 fn unmap(&self, iova: u64, size: u64) -> std::result::Result<(), io::Error> { in unmap()