1f40adff2SSebastien Boeuf // Copyright © 2019 Intel Corporation
2f40adff2SSebastien Boeuf //
3f40adff2SSebastien Boeuf // SPDX-License-Identifier: Apache-2.0 AND BSD-3-Clause
4f40adff2SSebastien Boeuf
5f40adff2SSebastien Boeuf use std::collections::BTreeMap;
6f40adff2SSebastien Boeuf use std::mem::size_of;
7b5d64be4SRob Bradford use std::os::unix::io::AsRawFd;
8f38360deSSebastien Boeuf use std::sync::atomic::{AtomicBool, Ordering};
9e9b1ad95SSebastien Boeuf use std::sync::{Arc, Barrier, Mutex, RwLock};
1061e57e1cSRuoqing He use std::{io, result};
1188a9f799SRob Bradford
1288a9f799SRob Bradford use anyhow::anyhow;
1388a9f799SRob Bradford use seccompiler::SeccompAction;
1488a9f799SRob Bradford use serde::{Deserialize, Serialize};
1584105992SBo Chen use thiserror::Error;
1687f57f7cSSebastien Boeuf use virtio_queue::{DescriptorChain, Queue, QueueT};
17aee11558SSebastien Boeuf use vm_device::dma_mapping::ExternalDmaMapping;
180162d73eSSebastien Boeuf use vm_memory::{
19a423bf13SSebastien Boeuf Address, ByteValued, Bytes, GuestAddress, GuestAddressSpace, GuestMemoryAtomic,
20a423bf13SSebastien Boeuf GuestMemoryError, GuestMemoryLoadGuard,
210162d73eSSebastien Boeuf };
226f5d4702SRob Bradford use vm_migration::{Migratable, MigratableError, Pausable, Snapshot, Snapshottable, Transportable};
238eed276dSSebastien Boeuf use vm_virtio::AccessPlatform;
24f40adff2SSebastien Boeuf use vmm_sys_util::eventfd::EventFd;
25f40adff2SSebastien Boeuf
2688a9f799SRob Bradford use super::{
2761e57e1cSRuoqing He ActivateResult, EpollHelper, EpollHelperError, EpollHelperHandler, Error as DeviceError,
2861e57e1cSRuoqing He VirtioCommon, VirtioDevice, VirtioDeviceType, EPOLL_HELPER_EVENT_LAST, VIRTIO_F_VERSION_1,
2988a9f799SRob Bradford };
3088a9f799SRob Bradford use crate::seccomp_filters::Thread;
3188a9f799SRob Bradford use crate::thread_helper::spawn_virtio_thread;
3261e57e1cSRuoqing He use crate::{DmaRemapping, GuestMemoryMmap, VirtioInterrupt, VirtioInterruptType};
3388a9f799SRob Bradford
34f40adff2SSebastien Boeuf /// Queues sizes
35f40adff2SSebastien Boeuf const QUEUE_SIZE: u16 = 256;
36f40adff2SSebastien Boeuf const NUM_QUEUES: usize = 2;
37f40adff2SSebastien Boeuf const QUEUE_SIZES: &[u16] = &[QUEUE_SIZE; NUM_QUEUES];
38f40adff2SSebastien Boeuf
39f40adff2SSebastien Boeuf /// New descriptors are pending on the request queue.
40f40adff2SSebastien Boeuf /// "requestq" is meant to be used anytime an action is required to be
41f40adff2SSebastien Boeuf /// performed on behalf of the guest driver.
42b5d64be4SRob Bradford const REQUEST_Q_EVENT: u16 = EPOLL_HELPER_EVENT_LAST + 1;
43f40adff2SSebastien Boeuf /// New descriptors are pending on the event queue.
44f40adff2SSebastien Boeuf /// "eventq" lets the device report any fault or other asynchronous event to
45f40adff2SSebastien Boeuf /// the guest driver.
469c658e21SBo Chen const _EVENT_Q_EVENT: u16 = EPOLL_HELPER_EVENT_LAST + 2;
47f40adff2SSebastien Boeuf
48e1822cfdSSebastien Boeuf /// PROBE properties size.
49e1822cfdSSebastien Boeuf /// This is the minimal size to provide at least one RESV_MEM property.
50e1822cfdSSebastien Boeuf /// Because virtio-iommu expects one MSI reserved region, we must provide it,
51e1822cfdSSebastien Boeuf /// otherwise the driver in the guest will define a predefined one between
52e1822cfdSSebastien Boeuf /// 0x8000000 and 0x80FFFFF, which is only relevant for ARM architecture, but
53e1822cfdSSebastien Boeuf /// will conflict with x86.
54e1822cfdSSebastien Boeuf const PROBE_PROP_SIZE: u32 =
55e1822cfdSSebastien Boeuf (size_of::<VirtioIommuProbeProperty>() + size_of::<VirtioIommuProbeResvMem>()) as u32;
56e1822cfdSSebastien Boeuf
57f40adff2SSebastien Boeuf /// Virtio IOMMU features
58f40adff2SSebastien Boeuf #[allow(unused)]
59f40adff2SSebastien Boeuf const VIRTIO_IOMMU_F_INPUT_RANGE: u32 = 0;
60f40adff2SSebastien Boeuf #[allow(unused)]
61a6fe4aa7SSebastien Boeuf const VIRTIO_IOMMU_F_DOMAIN_RANGE: u32 = 1;
62f40adff2SSebastien Boeuf #[allow(unused)]
63f40adff2SSebastien Boeuf const VIRTIO_IOMMU_F_MAP_UNMAP: u32 = 2;
64f40adff2SSebastien Boeuf #[allow(unused)]
65f40adff2SSebastien Boeuf const VIRTIO_IOMMU_F_BYPASS: u32 = 3;
66f40adff2SSebastien Boeuf const VIRTIO_IOMMU_F_PROBE: u32 = 4;
670c73ff81SSebastien Boeuf #[allow(unused)]
680c73ff81SSebastien Boeuf const VIRTIO_IOMMU_F_MMIO: u32 = 5;
69a6fe4aa7SSebastien Boeuf const VIRTIO_IOMMU_F_BYPASS_CONFIG: u32 = 6;
70f40adff2SSebastien Boeuf
71efbafdf9SSebastien Boeuf // Support 2MiB and 4KiB page sizes.
72efbafdf9SSebastien Boeuf const VIRTIO_IOMMU_PAGE_SIZE_MASK: u64 = (2 << 20) | (4 << 10);
73f40adff2SSebastien Boeuf
74f40adff2SSebastien Boeuf #[derive(Copy, Clone, Debug, Default)]
75778c05d6SWei Liu #[repr(C, packed)]
7684f0f332SRob Bradford #[allow(dead_code)]
77e1822cfdSSebastien Boeuf struct VirtioIommuRange32 {
78e1822cfdSSebastien Boeuf start: u32,
79e1822cfdSSebastien Boeuf end: u32,
80e1822cfdSSebastien Boeuf }
81e1822cfdSSebastien Boeuf
82e1822cfdSSebastien Boeuf #[derive(Copy, Clone, Debug, Default)]
83778c05d6SWei Liu #[repr(C, packed)]
8484f0f332SRob Bradford #[allow(dead_code)]
85e1822cfdSSebastien Boeuf struct VirtioIommuRange64 {
86f40adff2SSebastien Boeuf start: u64,
87f40adff2SSebastien Boeuf end: u64,
88f40adff2SSebastien Boeuf }
89f40adff2SSebastien Boeuf
90e1822cfdSSebastien Boeuf #[derive(Copy, Clone, Debug, Default)]
91778c05d6SWei Liu #[repr(C, packed)]
9284f0f332SRob Bradford #[allow(dead_code)]
93f40adff2SSebastien Boeuf struct VirtioIommuConfig {
94f40adff2SSebastien Boeuf page_size_mask: u64,
95e1822cfdSSebastien Boeuf input_range: VirtioIommuRange64,
96e1822cfdSSebastien Boeuf domain_range: VirtioIommuRange32,
97f40adff2SSebastien Boeuf probe_size: u32,
98a6fe4aa7SSebastien Boeuf bypass: u8,
9984f0f332SRob Bradford _reserved: [u8; 7],
100f40adff2SSebastien Boeuf }
101f40adff2SSebastien Boeuf
102f40adff2SSebastien Boeuf /// Virtio IOMMU request type
103f40adff2SSebastien Boeuf const VIRTIO_IOMMU_T_ATTACH: u8 = 1;
104f40adff2SSebastien Boeuf const VIRTIO_IOMMU_T_DETACH: u8 = 2;
105f40adff2SSebastien Boeuf const VIRTIO_IOMMU_T_MAP: u8 = 3;
106f40adff2SSebastien Boeuf const VIRTIO_IOMMU_T_UNMAP: u8 = 4;
107f40adff2SSebastien Boeuf const VIRTIO_IOMMU_T_PROBE: u8 = 5;
108f40adff2SSebastien Boeuf
109f40adff2SSebastien Boeuf #[derive(Copy, Clone, Debug, Default)]
110778c05d6SWei Liu #[repr(C, packed)]
111f40adff2SSebastien Boeuf struct VirtioIommuReqHead {
112f40adff2SSebastien Boeuf type_: u8,
11384f0f332SRob Bradford _reserved: [u8; 3],
114f40adff2SSebastien Boeuf }
115f40adff2SSebastien Boeuf
116f40adff2SSebastien Boeuf /// Virtio IOMMU request status
117f40adff2SSebastien Boeuf const VIRTIO_IOMMU_S_OK: u8 = 0;
118f40adff2SSebastien Boeuf #[allow(unused)]
119f40adff2SSebastien Boeuf const VIRTIO_IOMMU_S_IOERR: u8 = 1;
120f40adff2SSebastien Boeuf #[allow(unused)]
121f40adff2SSebastien Boeuf const VIRTIO_IOMMU_S_UNSUPP: u8 = 2;
122f40adff2SSebastien Boeuf #[allow(unused)]
123f40adff2SSebastien Boeuf const VIRTIO_IOMMU_S_DEVERR: u8 = 3;
124f40adff2SSebastien Boeuf #[allow(unused)]
125f40adff2SSebastien Boeuf const VIRTIO_IOMMU_S_INVAL: u8 = 4;
126f40adff2SSebastien Boeuf #[allow(unused)]
127f40adff2SSebastien Boeuf const VIRTIO_IOMMU_S_RANGE: u8 = 5;
128f40adff2SSebastien Boeuf #[allow(unused)]
129f40adff2SSebastien Boeuf const VIRTIO_IOMMU_S_NOENT: u8 = 6;
130f40adff2SSebastien Boeuf #[allow(unused)]
131f40adff2SSebastien Boeuf const VIRTIO_IOMMU_S_FAULT: u8 = 7;
132a6fe4aa7SSebastien Boeuf #[allow(unused)]
133a6fe4aa7SSebastien Boeuf const VIRTIO_IOMMU_S_NOMEM: u8 = 8;
134f40adff2SSebastien Boeuf
135f40adff2SSebastien Boeuf #[derive(Copy, Clone, Debug, Default)]
136778c05d6SWei Liu #[repr(C, packed)]
13784f0f332SRob Bradford #[allow(dead_code)]
138f40adff2SSebastien Boeuf struct VirtioIommuReqTail {
139f40adff2SSebastien Boeuf status: u8,
14084f0f332SRob Bradford _reserved: [u8; 3],
141f40adff2SSebastien Boeuf }
142f40adff2SSebastien Boeuf
143f40adff2SSebastien Boeuf /// ATTACH request
144f40adff2SSebastien Boeuf #[derive(Copy, Clone, Debug, Default)]
145778c05d6SWei Liu #[repr(C, packed)]
146f40adff2SSebastien Boeuf struct VirtioIommuReqAttach {
147f40adff2SSebastien Boeuf domain: u32,
148f40adff2SSebastien Boeuf endpoint: u32,
149bbd0667bSSebastien Boeuf flags: u32,
150bbd0667bSSebastien Boeuf _reserved: [u8; 4],
151f40adff2SSebastien Boeuf }
152f40adff2SSebastien Boeuf
153bbd0667bSSebastien Boeuf const VIRTIO_IOMMU_ATTACH_F_BYPASS: u32 = 1;
154bbd0667bSSebastien Boeuf
155f40adff2SSebastien Boeuf /// DETACH request
156f40adff2SSebastien Boeuf #[derive(Copy, Clone, Debug, Default)]
157778c05d6SWei Liu #[repr(C, packed)]
158f40adff2SSebastien Boeuf struct VirtioIommuReqDetach {
159f40adff2SSebastien Boeuf domain: u32,
160f40adff2SSebastien Boeuf endpoint: u32,
16184f0f332SRob Bradford _reserved: [u8; 8],
162f40adff2SSebastien Boeuf }
163f40adff2SSebastien Boeuf
164f40adff2SSebastien Boeuf /// Virtio IOMMU request MAP flags
165f40adff2SSebastien Boeuf #[allow(unused)]
166f40adff2SSebastien Boeuf const VIRTIO_IOMMU_MAP_F_READ: u32 = 1;
167f40adff2SSebastien Boeuf #[allow(unused)]
168f40adff2SSebastien Boeuf const VIRTIO_IOMMU_MAP_F_WRITE: u32 = 1 << 1;
169f40adff2SSebastien Boeuf #[allow(unused)]
170a6fe4aa7SSebastien Boeuf const VIRTIO_IOMMU_MAP_F_MMIO: u32 = 1 << 2;
171f40adff2SSebastien Boeuf #[allow(unused)]
172a6fe4aa7SSebastien Boeuf const VIRTIO_IOMMU_MAP_F_MASK: u32 =
173a6fe4aa7SSebastien Boeuf VIRTIO_IOMMU_MAP_F_READ | VIRTIO_IOMMU_MAP_F_WRITE | VIRTIO_IOMMU_MAP_F_MMIO;
174f40adff2SSebastien Boeuf
175f40adff2SSebastien Boeuf /// MAP request
176f40adff2SSebastien Boeuf #[derive(Copy, Clone, Debug, Default)]
177778c05d6SWei Liu #[repr(C, packed)]
178f40adff2SSebastien Boeuf struct VirtioIommuReqMap {
179f40adff2SSebastien Boeuf domain: u32,
180f40adff2SSebastien Boeuf virt_start: u64,
181f40adff2SSebastien Boeuf virt_end: u64,
182f40adff2SSebastien Boeuf phys_start: u64,
18384f0f332SRob Bradford _flags: u32,
184f40adff2SSebastien Boeuf }
185f40adff2SSebastien Boeuf
186f40adff2SSebastien Boeuf /// UNMAP request
187f40adff2SSebastien Boeuf #[derive(Copy, Clone, Debug, Default)]
188778c05d6SWei Liu #[repr(C, packed)]
189f40adff2SSebastien Boeuf struct VirtioIommuReqUnmap {
190f40adff2SSebastien Boeuf domain: u32,
191f40adff2SSebastien Boeuf virt_start: u64,
192f40adff2SSebastien Boeuf virt_end: u64,
19384f0f332SRob Bradford _reserved: [u8; 4],
194f40adff2SSebastien Boeuf }
195f40adff2SSebastien Boeuf
196f40adff2SSebastien Boeuf /// Virtio IOMMU request PROBE types
197f40adff2SSebastien Boeuf #[allow(unused)]
198e1822cfdSSebastien Boeuf const VIRTIO_IOMMU_PROBE_T_NONE: u16 = 0;
199e1822cfdSSebastien Boeuf const VIRTIO_IOMMU_PROBE_T_RESV_MEM: u16 = 1;
200a6fe4aa7SSebastien Boeuf #[allow(unused)]
201a6fe4aa7SSebastien Boeuf const VIRTIO_IOMMU_PROBE_T_MASK: u16 = 0xfff;
202f40adff2SSebastien Boeuf
203f40adff2SSebastien Boeuf /// PROBE request
204f40adff2SSebastien Boeuf #[derive(Copy, Clone, Debug, Default)]
205778c05d6SWei Liu #[repr(C, packed)]
20684f0f332SRob Bradford #[allow(dead_code)]
207f40adff2SSebastien Boeuf struct VirtioIommuReqProbe {
208f40adff2SSebastien Boeuf endpoint: u32,
20984f0f332SRob Bradford _reserved: [u64; 8],
210f40adff2SSebastien Boeuf }
211f40adff2SSebastien Boeuf
212f40adff2SSebastien Boeuf #[derive(Copy, Clone, Debug, Default)]
213778c05d6SWei Liu #[repr(C, packed)]
21484f0f332SRob Bradford #[allow(dead_code)]
215f40adff2SSebastien Boeuf struct VirtioIommuProbeProperty {
216f40adff2SSebastien Boeuf type_: u16,
217f40adff2SSebastien Boeuf length: u16,
218f40adff2SSebastien Boeuf }
219f40adff2SSebastien Boeuf
220f40adff2SSebastien Boeuf /// Virtio IOMMU request PROBE property RESV_MEM subtypes
221f40adff2SSebastien Boeuf #[allow(unused)]
222e1822cfdSSebastien Boeuf const VIRTIO_IOMMU_RESV_MEM_T_RESERVED: u8 = 0;
223e1822cfdSSebastien Boeuf const VIRTIO_IOMMU_RESV_MEM_T_MSI: u8 = 1;
224f40adff2SSebastien Boeuf
225f40adff2SSebastien Boeuf #[derive(Copy, Clone, Debug, Default)]
226778c05d6SWei Liu #[repr(C, packed)]
22784f0f332SRob Bradford #[allow(dead_code)]
228f40adff2SSebastien Boeuf struct VirtioIommuProbeResvMem {
229f40adff2SSebastien Boeuf subtype: u8,
23084f0f332SRob Bradford _reserved: [u8; 3],
231f40adff2SSebastien Boeuf start: u64,
232f40adff2SSebastien Boeuf end: u64,
233f40adff2SSebastien Boeuf }
234f40adff2SSebastien Boeuf
235f40adff2SSebastien Boeuf /// Virtio IOMMU fault flags
236f40adff2SSebastien Boeuf #[allow(unused)]
237f40adff2SSebastien Boeuf const VIRTIO_IOMMU_FAULT_F_READ: u32 = 1;
238f40adff2SSebastien Boeuf #[allow(unused)]
239f40adff2SSebastien Boeuf const VIRTIO_IOMMU_FAULT_F_WRITE: u32 = 1 << 1;
240f40adff2SSebastien Boeuf #[allow(unused)]
241f40adff2SSebastien Boeuf const VIRTIO_IOMMU_FAULT_F_EXEC: u32 = 1 << 2;
242f40adff2SSebastien Boeuf #[allow(unused)]
243f40adff2SSebastien Boeuf const VIRTIO_IOMMU_FAULT_F_ADDRESS: u32 = 1 << 8;
244f40adff2SSebastien Boeuf
245f40adff2SSebastien Boeuf /// Virtio IOMMU fault reasons
246f40adff2SSebastien Boeuf #[allow(unused)]
247f40adff2SSebastien Boeuf const VIRTIO_IOMMU_FAULT_R_UNKNOWN: u32 = 0;
248f40adff2SSebastien Boeuf #[allow(unused)]
249f40adff2SSebastien Boeuf const VIRTIO_IOMMU_FAULT_R_DOMAIN: u32 = 1;
250f40adff2SSebastien Boeuf #[allow(unused)]
251f40adff2SSebastien Boeuf const VIRTIO_IOMMU_FAULT_R_MAPPING: u32 = 2;
252f40adff2SSebastien Boeuf
253f40adff2SSebastien Boeuf /// Fault reporting through eventq
254f40adff2SSebastien Boeuf #[allow(unused)]
255f40adff2SSebastien Boeuf #[derive(Copy, Clone, Debug, Default)]
256778c05d6SWei Liu #[repr(C, packed)]
257f40adff2SSebastien Boeuf struct VirtioIommuFault {
258f40adff2SSebastien Boeuf reason: u8,
259f40adff2SSebastien Boeuf reserved: [u8; 3],
260f40adff2SSebastien Boeuf flags: u32,
261f40adff2SSebastien Boeuf endpoint: u32,
262a6fe4aa7SSebastien Boeuf reserved2: [u8; 4],
263f40adff2SSebastien Boeuf address: u64,
264f40adff2SSebastien Boeuf }
265f40adff2SSebastien Boeuf
266c45d24dfSWei Liu // SAFETY: data structure only contain integers and have no implicit padding
26731b3871eSWei Liu unsafe impl ByteValued for VirtioIommuRange32 {}
268c45d24dfSWei Liu // SAFETY: data structure only contain integers and have no implicit padding
26931b3871eSWei Liu unsafe impl ByteValued for VirtioIommuRange64 {}
270c45d24dfSWei Liu // SAFETY: data structure only contain integers and have no implicit padding
27131b3871eSWei Liu unsafe impl ByteValued for VirtioIommuConfig {}
272c45d24dfSWei Liu // SAFETY: data structure only contain integers and have no implicit padding
27331b3871eSWei Liu unsafe impl ByteValued for VirtioIommuReqHead {}
274c45d24dfSWei Liu // SAFETY: data structure only contain integers and have no implicit padding
27531b3871eSWei Liu unsafe impl ByteValued for VirtioIommuReqTail {}
276c45d24dfSWei Liu // SAFETY: data structure only contain integers and have no implicit padding
27731b3871eSWei Liu unsafe impl ByteValued for VirtioIommuReqAttach {}
278c45d24dfSWei Liu // SAFETY: data structure only contain integers and have no implicit padding
27931b3871eSWei Liu unsafe impl ByteValued for VirtioIommuReqDetach {}
280c45d24dfSWei Liu // SAFETY: data structure only contain integers and have no implicit padding
28131b3871eSWei Liu unsafe impl ByteValued for VirtioIommuReqMap {}
282c45d24dfSWei Liu // SAFETY: data structure only contain integers and have no implicit padding
28331b3871eSWei Liu unsafe impl ByteValued for VirtioIommuReqUnmap {}
284c45d24dfSWei Liu // SAFETY: data structure only contain integers and have no implicit padding
28531b3871eSWei Liu unsafe impl ByteValued for VirtioIommuReqProbe {}
286c45d24dfSWei Liu // SAFETY: data structure only contain integers and have no implicit padding
28731b3871eSWei Liu unsafe impl ByteValued for VirtioIommuProbeProperty {}
288c45d24dfSWei Liu // SAFETY: data structure only contain integers and have no implicit padding
28931b3871eSWei Liu unsafe impl ByteValued for VirtioIommuProbeResvMem {}
290c45d24dfSWei Liu // SAFETY: data structure only contain integers and have no implicit padding
291f40adff2SSebastien Boeuf unsafe impl ByteValued for VirtioIommuFault {}
292f40adff2SSebastien Boeuf
29384105992SBo Chen #[derive(Error, Debug)]
294f40adff2SSebastien Boeuf enum Error {
295*8e2973feSPhilipp Schuster #[error("Guest gave us bad memory addresses")]
296a2123439SPhilipp Schuster GuestMemory(#[source] GuestMemoryError),
2972af2cc53SBo Chen #[error("Guest gave us a write only descriptor that protocol says to read from")]
298f40adff2SSebastien Boeuf UnexpectedWriteOnlyDescriptor,
2992af2cc53SBo Chen #[error("Guest gave us a read only descriptor that protocol says to write to")]
300f40adff2SSebastien Boeuf UnexpectedReadOnlyDescriptor,
3012af2cc53SBo Chen #[error("Guest gave us too few descriptors in a descriptor chain")]
302f40adff2SSebastien Boeuf DescriptorChainTooShort,
3032af2cc53SBo Chen #[error("Guest gave us a buffer that was too short to use")]
304f40adff2SSebastien Boeuf BufferLengthTooSmall,
3052af2cc53SBo Chen #[error("Guest sent us invalid request")]
306f40adff2SSebastien Boeuf InvalidRequest,
3072af2cc53SBo Chen #[error("Guest sent us invalid ATTACH request")]
308f40adff2SSebastien Boeuf InvalidAttachRequest,
3092af2cc53SBo Chen #[error("Guest sent us invalid DETACH request")]
310f40adff2SSebastien Boeuf InvalidDetachRequest,
3112af2cc53SBo Chen #[error("Guest sent us invalid MAP request")]
312f40adff2SSebastien Boeuf InvalidMapRequest,
3132af2cc53SBo Chen #[error("Invalid to map because the domain is in bypass mode")]
314bbd0667bSSebastien Boeuf InvalidMapRequestBypassDomain,
3152af2cc53SBo Chen #[error("Invalid to map because the domain is missing")]
316bbd0667bSSebastien Boeuf InvalidMapRequestMissingDomain,
3172af2cc53SBo Chen #[error("Guest sent us invalid UNMAP request")]
318f40adff2SSebastien Boeuf InvalidUnmapRequest,
3192af2cc53SBo Chen #[error("Invalid to unmap because the domain is in bypass mode")]
320bbd0667bSSebastien Boeuf InvalidUnmapRequestBypassDomain,
3212af2cc53SBo Chen #[error("Invalid to unmap because the domain is missing")]
322bbd0667bSSebastien Boeuf InvalidUnmapRequestMissingDomain,
3232af2cc53SBo Chen #[error("Guest sent us invalid PROBE request")]
324f40adff2SSebastien Boeuf InvalidProbeRequest,
325*8e2973feSPhilipp Schuster #[error("Failed to performing external mapping")]
326a2123439SPhilipp Schuster ExternalMapping(#[source] io::Error),
327*8e2973feSPhilipp Schuster #[error("Failed to performing external unmapping")]
328a2123439SPhilipp Schuster ExternalUnmapping(#[source] io::Error),
329*8e2973feSPhilipp Schuster #[error("Failed adding used index")]
330a2123439SPhilipp Schuster QueueAddUsed(#[source] virtio_queue::Error),
331f40adff2SSebastien Boeuf }
332f40adff2SSebastien Boeuf
333e1822cfdSSebastien Boeuf struct Request {}
334f40adff2SSebastien Boeuf
335f40adff2SSebastien Boeuf impl Request {
336c65ead5dSSebastien Boeuf // Parse the available vring buffer. Based on the hashmap table of external
337c65ead5dSSebastien Boeuf // mappings required from various devices such as VFIO or vhost-user ones,
338c65ead5dSSebastien Boeuf // this function might update the hashmap table of external mappings per
339c65ead5dSSebastien Boeuf // domain.
340c65ead5dSSebastien Boeuf // Basically, the VMM knows about the device_id <=> mapping relationship
341c65ead5dSSebastien Boeuf // before running the VM, but at runtime, a new domain <=> mapping hashmap
342c65ead5dSSebastien Boeuf // is created based on the information provided from the guest driver for
343c65ead5dSSebastien Boeuf // virtio-iommu (giving the link device_id <=> domain).
parse( desc_chain: &mut DescriptorChain<GuestMemoryLoadGuard<GuestMemoryMmap>>, mapping: &Arc<IommuMapping>, ext_mapping: &BTreeMap<u32, Arc<dyn ExternalDmaMapping>>, msi_iova_space: (u64, u64), ) -> result::Result<usize, Error>344f40adff2SSebastien Boeuf fn parse(
3450162d73eSSebastien Boeuf desc_chain: &mut DescriptorChain<GuestMemoryLoadGuard<GuestMemoryMmap>>,
346f40adff2SSebastien Boeuf mapping: &Arc<IommuMapping>,
347c65ead5dSSebastien Boeuf ext_mapping: &BTreeMap<u32, Arc<dyn ExternalDmaMapping>>,
348b3fa5654SMichael Zhao msi_iova_space: (u64, u64),
349e1822cfdSSebastien Boeuf ) -> result::Result<usize, Error> {
3500249e864SSebastien Boeuf let desc = desc_chain
3510249e864SSebastien Boeuf .next()
3520249e864SSebastien Boeuf .ok_or(Error::DescriptorChainTooShort)
353b7512263SWei Liu .inspect_err(|_| {
3540249e864SSebastien Boeuf error!("Missing head descriptor");
3550249e864SSebastien Boeuf })?;
3560249e864SSebastien Boeuf
3570249e864SSebastien Boeuf // The descriptor contains the request type which MUST be readable.
3580249e864SSebastien Boeuf if desc.is_write_only() {
359f40adff2SSebastien Boeuf return Err(Error::UnexpectedWriteOnlyDescriptor);
360f40adff2SSebastien Boeuf }
361f40adff2SSebastien Boeuf
3620249e864SSebastien Boeuf if (desc.len() as usize) < size_of::<VirtioIommuReqHead>() {
363f40adff2SSebastien Boeuf return Err(Error::InvalidRequest);
364f40adff2SSebastien Boeuf }
365f40adff2SSebastien Boeuf
3660249e864SSebastien Boeuf let req_head: VirtioIommuReqHead = desc_chain
3670249e864SSebastien Boeuf .memory()
3680249e864SSebastien Boeuf .read_obj(desc.addr())
3690249e864SSebastien Boeuf .map_err(Error::GuestMemory)?;
370f40adff2SSebastien Boeuf let req_offset = size_of::<VirtioIommuReqHead>();
3710249e864SSebastien Boeuf let desc_size_left = (desc.len() as usize) - req_offset;
3720249e864SSebastien Boeuf let req_addr = if let Some(addr) = desc.addr().checked_add(req_offset as u64) {
373f40adff2SSebastien Boeuf addr
374f40adff2SSebastien Boeuf } else {
375f40adff2SSebastien Boeuf return Err(Error::InvalidRequest);
376f40adff2SSebastien Boeuf };
377f40adff2SSebastien Boeuf
378b3fa5654SMichael Zhao let (msi_iova_start, msi_iova_end) = msi_iova_space;
379b3fa5654SMichael Zhao
380e1822cfdSSebastien Boeuf // Create the reply
381e1822cfdSSebastien Boeuf let mut reply: Vec<u8> = Vec::new();
3826df8f0bbSSebastien Boeuf let mut status = VIRTIO_IOMMU_S_OK;
3836df8f0bbSSebastien Boeuf let mut hdr_len = 0;
384e1822cfdSSebastien Boeuf
3856df8f0bbSSebastien Boeuf let result = (|| {
3866df8f0bbSSebastien Boeuf match req_head.type_ {
387f40adff2SSebastien Boeuf VIRTIO_IOMMU_T_ATTACH => {
388f40adff2SSebastien Boeuf if desc_size_left != size_of::<VirtioIommuReqAttach>() {
3896df8f0bbSSebastien Boeuf status = VIRTIO_IOMMU_S_INVAL;
390f40adff2SSebastien Boeuf return Err(Error::InvalidAttachRequest);
391f40adff2SSebastien Boeuf }
392f40adff2SSebastien Boeuf
3930249e864SSebastien Boeuf let req: VirtioIommuReqAttach = desc_chain
3940249e864SSebastien Boeuf .memory()
395f40adff2SSebastien Boeuf .read_obj(req_addr as GuestAddress)
396f40adff2SSebastien Boeuf .map_err(Error::GuestMemory)?;
3972fc4de6cSRob Bradford debug!("Attach request 0x{:x?}", req);
398f40adff2SSebastien Boeuf
399f40adff2SSebastien Boeuf // Copy the value to use it as a proper reference.
400bbd0667bSSebastien Boeuf let domain_id = req.domain;
401c65ead5dSSebastien Boeuf let endpoint = req.endpoint;
402bbd0667bSSebastien Boeuf let bypass =
403bbd0667bSSebastien Boeuf (req.flags & VIRTIO_IOMMU_ATTACH_F_BYPASS) == VIRTIO_IOMMU_ATTACH_F_BYPASS;
404f40adff2SSebastien Boeuf
4053fa02b34SAndrew Carp let mut old_domain_id = domain_id;
4063fa02b34SAndrew Carp if let Some(&id) = mapping.endpoints.read().unwrap().get(&endpoint) {
4073fa02b34SAndrew Carp old_domain_id = id;
4083fa02b34SAndrew Carp }
4093fa02b34SAndrew Carp
4103fa02b34SAndrew Carp if old_domain_id != domain_id {
4113fa02b34SAndrew Carp detach_endpoint_from_domain(endpoint, old_domain_id, mapping, ext_mapping)?;
4123fa02b34SAndrew Carp }
4133fa02b34SAndrew Carp
414f40adff2SSebastien Boeuf // Add endpoint associated with specific domain
415bbd0667bSSebastien Boeuf mapping
416bbd0667bSSebastien Boeuf .endpoints
417bbd0667bSSebastien Boeuf .write()
418bbd0667bSSebastien Boeuf .unwrap()
419bbd0667bSSebastien Boeuf .insert(endpoint, domain_id);
420c65ead5dSSebastien Boeuf
4215668f02eSAndrew Carp // If any other mappings exist in the domain for other containers,
4225668f02eSAndrew Carp // make sure to issue these mappings for the new endpoint/container
4235668f02eSAndrew Carp if let Some(domain_mappings) = &mapping.domains.read().unwrap().get(&domain_id)
4245668f02eSAndrew Carp {
4255668f02eSAndrew Carp if let Some(ext_map) = ext_mapping.get(&endpoint) {
4265668f02eSAndrew Carp for (virt_start, addr_map) in &domain_mappings.mappings {
4275668f02eSAndrew Carp ext_map
4285668f02eSAndrew Carp .map(*virt_start, addr_map.gpa, addr_map.size)
4295668f02eSAndrew Carp .map_err(Error::ExternalUnmapping)?;
4305668f02eSAndrew Carp }
4315668f02eSAndrew Carp }
4325668f02eSAndrew Carp }
4335668f02eSAndrew Carp
434f40adff2SSebastien Boeuf // Add new domain with no mapping if the entry didn't exist yet
435b40633f9SSebastien Boeuf let mut domains = mapping.domains.write().unwrap();
436bbd0667bSSebastien Boeuf let domain = Domain {
437bbd0667bSSebastien Boeuf mappings: BTreeMap::new(),
438bbd0667bSSebastien Boeuf bypass,
439bbd0667bSSebastien Boeuf };
440bbd0667bSSebastien Boeuf domains.entry(domain_id).or_insert_with(|| domain);
441f40adff2SSebastien Boeuf }
442f40adff2SSebastien Boeuf VIRTIO_IOMMU_T_DETACH => {
443f40adff2SSebastien Boeuf if desc_size_left != size_of::<VirtioIommuReqDetach>() {
4446df8f0bbSSebastien Boeuf status = VIRTIO_IOMMU_S_INVAL;
445f40adff2SSebastien Boeuf return Err(Error::InvalidDetachRequest);
446f40adff2SSebastien Boeuf }
447f40adff2SSebastien Boeuf
4480249e864SSebastien Boeuf let req: VirtioIommuReqDetach = desc_chain
4490249e864SSebastien Boeuf .memory()
450f40adff2SSebastien Boeuf .read_obj(req_addr as GuestAddress)
451f40adff2SSebastien Boeuf .map_err(Error::GuestMemory)?;
4522fc4de6cSRob Bradford debug!("Detach request 0x{:x?}", req);
453f40adff2SSebastien Boeuf
454f40adff2SSebastien Boeuf // Copy the value to use it as a proper reference.
455bbd0667bSSebastien Boeuf let domain_id = req.domain;
456f40adff2SSebastien Boeuf let endpoint = req.endpoint;
457f40adff2SSebastien Boeuf
458f40adff2SSebastien Boeuf // Remove endpoint associated with specific domain
4593fa02b34SAndrew Carp detach_endpoint_from_domain(endpoint, domain_id, mapping, ext_mapping)?;
460f40adff2SSebastien Boeuf }
461f40adff2SSebastien Boeuf VIRTIO_IOMMU_T_MAP => {
462f40adff2SSebastien Boeuf if desc_size_left != size_of::<VirtioIommuReqMap>() {
4636df8f0bbSSebastien Boeuf status = VIRTIO_IOMMU_S_INVAL;
464f40adff2SSebastien Boeuf return Err(Error::InvalidMapRequest);
465f40adff2SSebastien Boeuf }
466f40adff2SSebastien Boeuf
4670249e864SSebastien Boeuf let req: VirtioIommuReqMap = desc_chain
4680249e864SSebastien Boeuf .memory()
469f40adff2SSebastien Boeuf .read_obj(req_addr as GuestAddress)
470f40adff2SSebastien Boeuf .map_err(Error::GuestMemory)?;
4712fc4de6cSRob Bradford debug!("Map request 0x{:x?}", req);
472f40adff2SSebastien Boeuf
473f40adff2SSebastien Boeuf // Copy the value to use it as a proper reference.
474bbd0667bSSebastien Boeuf let domain_id = req.domain;
475bbd0667bSSebastien Boeuf
476bbd0667bSSebastien Boeuf if let Some(domain) = mapping.domains.read().unwrap().get(&domain_id) {
477bbd0667bSSebastien Boeuf if domain.bypass {
4786df8f0bbSSebastien Boeuf status = VIRTIO_IOMMU_S_INVAL;
479bbd0667bSSebastien Boeuf return Err(Error::InvalidMapRequestBypassDomain);
480bbd0667bSSebastien Boeuf }
481bbd0667bSSebastien Boeuf } else {
4826df8f0bbSSebastien Boeuf status = VIRTIO_IOMMU_S_INVAL;
483bbd0667bSSebastien Boeuf return Err(Error::InvalidMapRequestMissingDomain);
484bbd0667bSSebastien Boeuf }
485bbd0667bSSebastien Boeuf
486fc8facddSSebastien Boeuf // Find the list of endpoints attached to the given domain.
487fc8facddSSebastien Boeuf let endpoints: Vec<u32> = mapping
488fc8facddSSebastien Boeuf .endpoints
489fc8facddSSebastien Boeuf .write()
490fc8facddSSebastien Boeuf .unwrap()
491fc8facddSSebastien Boeuf .iter()
492bbd0667bSSebastien Boeuf .filter(|(_, &d)| d == domain_id)
493fc8facddSSebastien Boeuf .map(|(&e, _)| e)
494fc8facddSSebastien Boeuf .collect();
495f40adff2SSebastien Boeuf
4963fa02b34SAndrew Carp // For viommu all endpoints receive their own VFIO container, as a result
4973fa02b34SAndrew Carp // Each endpoint within the domain needs to be separately mapped, as the
4983fa02b34SAndrew Carp // mapping is done on a per-container level, not a per-domain level
499fc8facddSSebastien Boeuf for endpoint in endpoints {
500fc8facddSSebastien Boeuf if let Some(ext_map) = ext_mapping.get(&endpoint) {
501c65ead5dSSebastien Boeuf let size = req.virt_end - req.virt_start + 1;
502c65ead5dSSebastien Boeuf ext_map
503c65ead5dSSebastien Boeuf .map(req.virt_start, req.phys_start, size)
504c65ead5dSSebastien Boeuf .map_err(Error::ExternalMapping)?;
505c65ead5dSSebastien Boeuf }
506fc8facddSSebastien Boeuf }
507c65ead5dSSebastien Boeuf
508f40adff2SSebastien Boeuf // Add new mapping associated with the domain
509bbd0667bSSebastien Boeuf mapping
510bbd0667bSSebastien Boeuf .domains
511bbd0667bSSebastien Boeuf .write()
512bbd0667bSSebastien Boeuf .unwrap()
513bbd0667bSSebastien Boeuf .get_mut(&domain_id)
514bbd0667bSSebastien Boeuf .unwrap()
515bbd0667bSSebastien Boeuf .mappings
516bbd0667bSSebastien Boeuf .insert(
517f40adff2SSebastien Boeuf req.virt_start,
518f40adff2SSebastien Boeuf Mapping {
519f40adff2SSebastien Boeuf gpa: req.phys_start,
520f40adff2SSebastien Boeuf size: req.virt_end - req.virt_start + 1,
521f40adff2SSebastien Boeuf },
522f40adff2SSebastien Boeuf );
523f40adff2SSebastien Boeuf }
524f40adff2SSebastien Boeuf VIRTIO_IOMMU_T_UNMAP => {
525f40adff2SSebastien Boeuf if desc_size_left != size_of::<VirtioIommuReqUnmap>() {
5266df8f0bbSSebastien Boeuf status = VIRTIO_IOMMU_S_INVAL;
527f40adff2SSebastien Boeuf return Err(Error::InvalidUnmapRequest);
528f40adff2SSebastien Boeuf }
529f40adff2SSebastien Boeuf
5300249e864SSebastien Boeuf let req: VirtioIommuReqUnmap = desc_chain
5310249e864SSebastien Boeuf .memory()
532f40adff2SSebastien Boeuf .read_obj(req_addr as GuestAddress)
533f40adff2SSebastien Boeuf .map_err(Error::GuestMemory)?;
5342fc4de6cSRob Bradford debug!("Unmap request 0x{:x?}", req);
535f40adff2SSebastien Boeuf
536f40adff2SSebastien Boeuf // Copy the value to use it as a proper reference.
537bbd0667bSSebastien Boeuf let domain_id = req.domain;
538f40adff2SSebastien Boeuf let virt_start = req.virt_start;
539bbd0667bSSebastien Boeuf
540bbd0667bSSebastien Boeuf if let Some(domain) = mapping.domains.read().unwrap().get(&domain_id) {
541bbd0667bSSebastien Boeuf if domain.bypass {
5426df8f0bbSSebastien Boeuf status = VIRTIO_IOMMU_S_INVAL;
543bbd0667bSSebastien Boeuf return Err(Error::InvalidUnmapRequestBypassDomain);
544bbd0667bSSebastien Boeuf }
545bbd0667bSSebastien Boeuf } else {
5466df8f0bbSSebastien Boeuf status = VIRTIO_IOMMU_S_INVAL;
547bbd0667bSSebastien Boeuf return Err(Error::InvalidUnmapRequestMissingDomain);
548bbd0667bSSebastien Boeuf }
549bbd0667bSSebastien Boeuf
550fc8facddSSebastien Boeuf // Find the list of endpoints attached to the given domain.
551fc8facddSSebastien Boeuf let endpoints: Vec<u32> = mapping
552fc8facddSSebastien Boeuf .endpoints
553fc8facddSSebastien Boeuf .write()
554fc8facddSSebastien Boeuf .unwrap()
555fc8facddSSebastien Boeuf .iter()
556bbd0667bSSebastien Boeuf .filter(|(_, &d)| d == domain_id)
557fc8facddSSebastien Boeuf .map(|(&e, _)| e)
558fc8facddSSebastien Boeuf .collect();
559f40adff2SSebastien Boeuf
560c65ead5dSSebastien Boeuf // Trigger external unmapping if necessary.
561fc8facddSSebastien Boeuf for endpoint in endpoints {
562fc8facddSSebastien Boeuf if let Some(ext_map) = ext_mapping.get(&endpoint) {
563c65ead5dSSebastien Boeuf let size = req.virt_end - virt_start + 1;
564c65ead5dSSebastien Boeuf ext_map
565c65ead5dSSebastien Boeuf .unmap(virt_start, size)
566c65ead5dSSebastien Boeuf .map_err(Error::ExternalUnmapping)?;
567c65ead5dSSebastien Boeuf }
568fc8facddSSebastien Boeuf }
569c65ead5dSSebastien Boeuf
570fbdc5d44SAndrew Carp // Remove all mappings associated with the domain within the requested range
571bbd0667bSSebastien Boeuf mapping
572bbd0667bSSebastien Boeuf .domains
573bbd0667bSSebastien Boeuf .write()
574bbd0667bSSebastien Boeuf .unwrap()
575bbd0667bSSebastien Boeuf .get_mut(&domain_id)
576bbd0667bSSebastien Boeuf .unwrap()
577bbd0667bSSebastien Boeuf .mappings
578fbdc5d44SAndrew Carp .retain(|&x, _| (x < req.virt_start || x > req.virt_end));
579f40adff2SSebastien Boeuf }
580f40adff2SSebastien Boeuf VIRTIO_IOMMU_T_PROBE => {
581f40adff2SSebastien Boeuf if desc_size_left != size_of::<VirtioIommuReqProbe>() {
5826df8f0bbSSebastien Boeuf status = VIRTIO_IOMMU_S_INVAL;
583f40adff2SSebastien Boeuf return Err(Error::InvalidProbeRequest);
584f40adff2SSebastien Boeuf }
585f40adff2SSebastien Boeuf
5860249e864SSebastien Boeuf let req: VirtioIommuReqProbe = desc_chain
5870249e864SSebastien Boeuf .memory()
588f40adff2SSebastien Boeuf .read_obj(req_addr as GuestAddress)
589f40adff2SSebastien Boeuf .map_err(Error::GuestMemory)?;
5902fc4de6cSRob Bradford debug!("Probe request 0x{:x?}", req);
591f40adff2SSebastien Boeuf
592e1822cfdSSebastien Boeuf let probe_prop = VirtioIommuProbeProperty {
593e1822cfdSSebastien Boeuf type_: VIRTIO_IOMMU_PROBE_T_RESV_MEM,
594e1822cfdSSebastien Boeuf length: size_of::<VirtioIommuProbeResvMem>() as u16,
595e1822cfdSSebastien Boeuf };
596e1822cfdSSebastien Boeuf reply.extend_from_slice(probe_prop.as_slice());
597e1822cfdSSebastien Boeuf
598e1822cfdSSebastien Boeuf let resv_mem = VirtioIommuProbeResvMem {
599e1822cfdSSebastien Boeuf subtype: VIRTIO_IOMMU_RESV_MEM_T_MSI,
600b3fa5654SMichael Zhao start: msi_iova_start,
601b3fa5654SMichael Zhao end: msi_iova_end,
602e1822cfdSSebastien Boeuf ..Default::default()
603e1822cfdSSebastien Boeuf };
604e1822cfdSSebastien Boeuf reply.extend_from_slice(resv_mem.as_slice());
605e1822cfdSSebastien Boeuf
6066df8f0bbSSebastien Boeuf hdr_len = PROBE_PROP_SIZE;
607f40adff2SSebastien Boeuf }
6086df8f0bbSSebastien Boeuf _ => {
6096df8f0bbSSebastien Boeuf status = VIRTIO_IOMMU_S_INVAL;
6106df8f0bbSSebastien Boeuf return Err(Error::InvalidRequest);
6116df8f0bbSSebastien Boeuf }
6126df8f0bbSSebastien Boeuf }
6136df8f0bbSSebastien Boeuf Ok(())
6146df8f0bbSSebastien Boeuf })();
615f40adff2SSebastien Boeuf
6160249e864SSebastien Boeuf let status_desc = desc_chain.next().ok_or(Error::DescriptorChainTooShort)?;
617f40adff2SSebastien Boeuf
618f40adff2SSebastien Boeuf // The status MUST always be writable
619f40adff2SSebastien Boeuf if !status_desc.is_write_only() {
620f40adff2SSebastien Boeuf return Err(Error::UnexpectedReadOnlyDescriptor);
621f40adff2SSebastien Boeuf }
622f40adff2SSebastien Boeuf
6230249e864SSebastien Boeuf if status_desc.len() < hdr_len + size_of::<VirtioIommuReqTail>() as u32 {
624f40adff2SSebastien Boeuf return Err(Error::BufferLengthTooSmall);
625f40adff2SSebastien Boeuf }
626f40adff2SSebastien Boeuf
627e1822cfdSSebastien Boeuf let tail = VirtioIommuReqTail {
6286df8f0bbSSebastien Boeuf status,
629e1822cfdSSebastien Boeuf ..Default::default()
630e1822cfdSSebastien Boeuf };
631e1822cfdSSebastien Boeuf reply.extend_from_slice(tail.as_slice());
632e1822cfdSSebastien Boeuf
6336df8f0bbSSebastien Boeuf // Make sure we return the result of the request to the guest before
6346df8f0bbSSebastien Boeuf // we return a potential error internally.
6350249e864SSebastien Boeuf desc_chain
6360249e864SSebastien Boeuf .memory()
6370249e864SSebastien Boeuf .write_slice(reply.as_slice(), status_desc.addr())
638e1822cfdSSebastien Boeuf .map_err(Error::GuestMemory)?;
639e1822cfdSSebastien Boeuf
6406df8f0bbSSebastien Boeuf // Return the error if the result was not Ok().
6416df8f0bbSSebastien Boeuf result?;
6426df8f0bbSSebastien Boeuf
643e1822cfdSSebastien Boeuf Ok((hdr_len as usize) + size_of::<VirtioIommuReqTail>())
644f40adff2SSebastien Boeuf }
645f40adff2SSebastien Boeuf }
646f40adff2SSebastien Boeuf
detach_endpoint_from_domain( endpoint: u32, domain_id: u32, mapping: &Arc<IommuMapping>, ext_mapping: &BTreeMap<u32, Arc<dyn ExternalDmaMapping>>, ) -> result::Result<(), Error>6473fa02b34SAndrew Carp fn detach_endpoint_from_domain(
6483fa02b34SAndrew Carp endpoint: u32,
6493fa02b34SAndrew Carp domain_id: u32,
6503fa02b34SAndrew Carp mapping: &Arc<IommuMapping>,
6513fa02b34SAndrew Carp ext_mapping: &BTreeMap<u32, Arc<dyn ExternalDmaMapping>>,
6523fa02b34SAndrew Carp ) -> result::Result<(), Error> {
6533fa02b34SAndrew Carp // Remove endpoint associated with specific domain
6543fa02b34SAndrew Carp mapping.endpoints.write().unwrap().remove(&endpoint);
6553fa02b34SAndrew Carp
6563fa02b34SAndrew Carp // Trigger external unmapping for the endpoint if necessary.
6573fa02b34SAndrew Carp if let Some(domain_mappings) = &mapping.domains.read().unwrap().get(&domain_id) {
6583fa02b34SAndrew Carp if let Some(ext_map) = ext_mapping.get(&endpoint) {
6593fa02b34SAndrew Carp for (virt_start, addr_map) in &domain_mappings.mappings {
6603fa02b34SAndrew Carp ext_map
6613fa02b34SAndrew Carp .unmap(*virt_start, addr_map.size)
6623fa02b34SAndrew Carp .map_err(Error::ExternalUnmapping)?;
6633fa02b34SAndrew Carp }
6643fa02b34SAndrew Carp }
6653fa02b34SAndrew Carp }
6663fa02b34SAndrew Carp
6673fa02b34SAndrew Carp if mapping
6683fa02b34SAndrew Carp .endpoints
6693fa02b34SAndrew Carp .write()
6703fa02b34SAndrew Carp .unwrap()
6713fa02b34SAndrew Carp .iter()
6723fa02b34SAndrew Carp .filter(|(_, &d)| d == domain_id)
6733fa02b34SAndrew Carp .count()
6743fa02b34SAndrew Carp == 0
6753fa02b34SAndrew Carp {
6763fa02b34SAndrew Carp mapping.domains.write().unwrap().remove(&domain_id);
6773fa02b34SAndrew Carp }
6783fa02b34SAndrew Carp
6793fa02b34SAndrew Carp Ok(())
6803fa02b34SAndrew Carp }
6813fa02b34SAndrew Carp
682f40adff2SSebastien Boeuf struct IommuEpollHandler {
683a423bf13SSebastien Boeuf mem: GuestMemoryAtomic<GuestMemoryMmap>,
684710a860eSBo Chen request_queue: Queue,
685710a860eSBo Chen _event_queue: Queue,
686c396bacaSSebastien Boeuf interrupt_cb: Arc<dyn VirtioInterrupt>,
687710a860eSBo Chen request_queue_evt: EventFd,
6889c658e21SBo Chen _event_queue_evt: EventFd,
689f40adff2SSebastien Boeuf kill_evt: EventFd,
690dae0b2efSSamuel Ortiz pause_evt: EventFd,
691f40adff2SSebastien Boeuf mapping: Arc<IommuMapping>,
692e9b1ad95SSebastien Boeuf ext_mapping: Arc<Mutex<BTreeMap<u32, Arc<dyn ExternalDmaMapping>>>>,
693b3fa5654SMichael Zhao msi_iova_space: (u64, u64),
694f40adff2SSebastien Boeuf }
695f40adff2SSebastien Boeuf
696f40adff2SSebastien Boeuf impl IommuEpollHandler {
request_queue(&mut self) -> Result<bool, Error>6975b706422SBo Chen fn request_queue(&mut self) -> Result<bool, Error> {
698a4859ffeSSebastien Boeuf let mut used_descs = false;
699710a860eSBo Chen while let Some(mut desc_chain) = self.request_queue.pop_descriptor_chain(self.mem.memory())
700710a860eSBo Chen {
7015b706422SBo Chen let len = Request::parse(
7020249e864SSebastien Boeuf &mut desc_chain,
703c65ead5dSSebastien Boeuf &self.mapping,
704e9b1ad95SSebastien Boeuf &self.ext_mapping.lock().unwrap(),
705b3fa5654SMichael Zhao self.msi_iova_space,
7065b706422SBo Chen )?;
707f40adff2SSebastien Boeuf
708710a860eSBo Chen self.request_queue
7095b706422SBo Chen .add_used(desc_chain.memory(), desc_chain.head_index(), len as u32)
7105b706422SBo Chen .map_err(Error::QueueAddUsed)?;
7115b706422SBo Chen
712a4859ffeSSebastien Boeuf used_descs = true;
713f40adff2SSebastien Boeuf }
714f40adff2SSebastien Boeuf
7155b706422SBo Chen Ok(used_descs)
716f40adff2SSebastien Boeuf }
717f40adff2SSebastien Boeuf
signal_used_queue(&self, queue_index: u16) -> result::Result<(), DeviceError>718de3e003eSSebastien Boeuf fn signal_used_queue(&self, queue_index: u16) -> result::Result<(), DeviceError> {
719c396bacaSSebastien Boeuf self.interrupt_cb
720de3e003eSSebastien Boeuf .trigger(VirtioInterruptType::Queue(queue_index))
721c396bacaSSebastien Boeuf .map_err(|e| {
722f40adff2SSebastien Boeuf error!("Failed to signal used queue: {:?}", e);
723f40adff2SSebastien Boeuf DeviceError::FailedSignalingUsedQueue(e)
724f40adff2SSebastien Boeuf })
725f40adff2SSebastien Boeuf }
726f40adff2SSebastien Boeuf
run( &mut self, paused: Arc<AtomicBool>, paused_sync: Arc<Barrier>, ) -> result::Result<(), EpollHelperError>727aa57762cSSebastien Boeuf fn run(
728aa57762cSSebastien Boeuf &mut self,
729aa57762cSSebastien Boeuf paused: Arc<AtomicBool>,
730aa57762cSSebastien Boeuf paused_sync: Arc<Barrier>,
731aa57762cSSebastien Boeuf ) -> result::Result<(), EpollHelperError> {
732b5d64be4SRob Bradford let mut helper = EpollHelper::new(&self.kill_evt, &self.pause_evt)?;
733710a860eSBo Chen helper.add_event(self.request_queue_evt.as_raw_fd(), REQUEST_Q_EVENT)?;
734aa57762cSSebastien Boeuf helper.run(paused, paused_sync, self)?;
735f40adff2SSebastien Boeuf
736b5d64be4SRob Bradford Ok(())
737b5d64be4SRob Bradford }
738e382dc66SSebastien Boeuf }
739e382dc66SSebastien Boeuf
740b5d64be4SRob Bradford impl EpollHelperHandler for IommuEpollHandler {
handle_event( &mut self, _helper: &mut EpollHelper, event: &epoll::Event, ) -> result::Result<(), EpollHelperError>741b1752994SBo Chen fn handle_event(
742b1752994SBo Chen &mut self,
743b1752994SBo Chen _helper: &mut EpollHelper,
744b1752994SBo Chen event: &epoll::Event,
745b1752994SBo Chen ) -> result::Result<(), EpollHelperError> {
74601e7bd72SSebastien Boeuf let ev_type = event.data as u16;
74701e7bd72SSebastien Boeuf match ev_type {
748f40adff2SSebastien Boeuf REQUEST_Q_EVENT => {
749710a860eSBo Chen self.request_queue_evt.read().map_err(|e| {
750b1752994SBo Chen EpollHelperError::HandleEvent(anyhow!("Failed to get queue event: {:?}", e))
751b1752994SBo Chen })?;
752b1752994SBo Chen
7535b706422SBo Chen let needs_notification = self.request_queue().map_err(|e| {
7545b706422SBo Chen EpollHelperError::HandleEvent(anyhow!(
7555b706422SBo Chen "Failed to process request queue : {:?}",
7565b706422SBo Chen e
7575b706422SBo Chen ))
7585b706422SBo Chen })?;
7595b706422SBo Chen if needs_notification {
760b1752994SBo Chen self.signal_used_queue(0).map_err(|e| {
761b1752994SBo Chen EpollHelperError::HandleEvent(anyhow!(
762b1752994SBo Chen "Failed to signal used queue: {:?}",
763b1752994SBo Chen e
764b1752994SBo Chen ))
765b1752994SBo Chen })?;
766f40adff2SSebastien Boeuf }
767f40adff2SSebastien Boeuf }
768f40adff2SSebastien Boeuf _ => {
769b1752994SBo Chen return Err(EpollHelperError::HandleEvent(anyhow!(
770b1752994SBo Chen "Unexpected event: {}",
771b1752994SBo Chen ev_type
772b1752994SBo Chen )));
773f40adff2SSebastien Boeuf }
774f40adff2SSebastien Boeuf }
775b1752994SBo Chen Ok(())
776f40adff2SSebastien Boeuf }
777f40adff2SSebastien Boeuf }
778f40adff2SSebastien Boeuf
77910ab87d6SRob Bradford #[derive(Clone, Copy, Debug, Serialize, Deserialize)]
780f40adff2SSebastien Boeuf struct Mapping {
781f40adff2SSebastien Boeuf gpa: u64,
782f40adff2SSebastien Boeuf size: u64,
783f40adff2SSebastien Boeuf }
784f40adff2SSebastien Boeuf
785bbd0667bSSebastien Boeuf #[derive(Clone, Debug)]
786b40633f9SSebastien Boeuf struct Domain {
787b40633f9SSebastien Boeuf mappings: BTreeMap<u64, Mapping>,
788bbd0667bSSebastien Boeuf bypass: bool,
789b40633f9SSebastien Boeuf }
790b40633f9SSebastien Boeuf
7910249e864SSebastien Boeuf #[derive(Debug)]
792f40adff2SSebastien Boeuf pub struct IommuMapping {
793f40adff2SSebastien Boeuf // Domain related to an endpoint.
794f40adff2SSebastien Boeuf endpoints: Arc<RwLock<BTreeMap<u32, u32>>>,
795b40633f9SSebastien Boeuf // Information related to each domain.
796b40633f9SSebastien Boeuf domains: Arc<RwLock<BTreeMap<u32, Domain>>>,
797f38360deSSebastien Boeuf // Global flag indicating if endpoints that are not attached to any domain
798f38360deSSebastien Boeuf // are in bypass mode.
799f38360deSSebastien Boeuf bypass: AtomicBool,
800f40adff2SSebastien Boeuf }
801f40adff2SSebastien Boeuf
802f40adff2SSebastien Boeuf impl DmaRemapping for IommuMapping {
translate_gva(&self, id: u32, addr: u64) -> std::result::Result<u64, std::io::Error>803059e787cSSebastien Boeuf fn translate_gva(&self, id: u32, addr: u64) -> std::result::Result<u64, std::io::Error> {
8041cdf8b23SSebastien Boeuf debug!("Translate GVA addr 0x{:x}", addr);
805b40633f9SSebastien Boeuf if let Some(domain_id) = self.endpoints.read().unwrap().get(&id) {
806b40633f9SSebastien Boeuf if let Some(domain) = self.domains.read().unwrap().get(domain_id) {
807bbd0667bSSebastien Boeuf // Directly return identity mapping in case the domain is in
808bbd0667bSSebastien Boeuf // bypass mode.
809bbd0667bSSebastien Boeuf if domain.bypass {
810bbd0667bSSebastien Boeuf return Ok(addr);
811bbd0667bSSebastien Boeuf }
812bbd0667bSSebastien Boeuf
81303eeb36bSRob Bradford for (&key, &value) in domain.mappings.iter() {
814f40adff2SSebastien Boeuf if addr >= key && addr < key + value.size {
815f40adff2SSebastien Boeuf let new_addr = addr - key + value.gpa;
8161cdf8b23SSebastien Boeuf debug!("Into GPA addr 0x{:x}", new_addr);
8171cdf8b23SSebastien Boeuf return Ok(new_addr);
8181cdf8b23SSebastien Boeuf }
8191cdf8b23SSebastien Boeuf }
8201cdf8b23SSebastien Boeuf }
821f38360deSSebastien Boeuf } else if self.bypass.load(Ordering::Acquire) {
822f38360deSSebastien Boeuf return Ok(addr);
8231cdf8b23SSebastien Boeuf }
8241cdf8b23SSebastien Boeuf
825ea4693a0SJinank Jain Err(io::Error::other(format!(
826ea4693a0SJinank Jain "failed to translate GVA addr 0x{addr:x}"
827ea4693a0SJinank Jain )))
8281cdf8b23SSebastien Boeuf }
8291cdf8b23SSebastien Boeuf
translate_gpa(&self, id: u32, addr: u64) -> std::result::Result<u64, std::io::Error>8301cdf8b23SSebastien Boeuf fn translate_gpa(&self, id: u32, addr: u64) -> std::result::Result<u64, std::io::Error> {
8311cdf8b23SSebastien Boeuf debug!("Translate GPA addr 0x{:x}", addr);
832b40633f9SSebastien Boeuf if let Some(domain_id) = self.endpoints.read().unwrap().get(&id) {
833b40633f9SSebastien Boeuf if let Some(domain) = self.domains.read().unwrap().get(domain_id) {
834bbd0667bSSebastien Boeuf // Directly return identity mapping in case the domain is in
835bbd0667bSSebastien Boeuf // bypass mode.
836bbd0667bSSebastien Boeuf if domain.bypass {
837bbd0667bSSebastien Boeuf return Ok(addr);
838bbd0667bSSebastien Boeuf }
839bbd0667bSSebastien Boeuf
840b40633f9SSebastien Boeuf for (&key, &value) in domain.mappings.iter() {
8411cdf8b23SSebastien Boeuf if addr >= value.gpa && addr < value.gpa + value.size {
8421cdf8b23SSebastien Boeuf let new_addr = addr - value.gpa + key;
8431cdf8b23SSebastien Boeuf debug!("Into GVA addr 0x{:x}", new_addr);
844f40adff2SSebastien Boeuf return Ok(new_addr);
845f40adff2SSebastien Boeuf }
846f40adff2SSebastien Boeuf }
847f40adff2SSebastien Boeuf }
848f38360deSSebastien Boeuf } else if self.bypass.load(Ordering::Acquire) {
849f38360deSSebastien Boeuf return Ok(addr);
850f40adff2SSebastien Boeuf }
851f40adff2SSebastien Boeuf
852ea4693a0SJinank Jain Err(io::Error::other(format!(
853ea4693a0SJinank Jain "failed to translate GPA addr 0x{addr:x}"
854ea4693a0SJinank Jain )))
855f40adff2SSebastien Boeuf }
856f40adff2SSebastien Boeuf }
857f40adff2SSebastien Boeuf
8580249e864SSebastien Boeuf #[derive(Debug)]
8590249e864SSebastien Boeuf pub struct AccessPlatformMapping {
8600249e864SSebastien Boeuf id: u32,
8610249e864SSebastien Boeuf mapping: Arc<IommuMapping>,
8620249e864SSebastien Boeuf }
8630249e864SSebastien Boeuf
8640249e864SSebastien Boeuf impl AccessPlatformMapping {
new(id: u32, mapping: Arc<IommuMapping>) -> Self8650249e864SSebastien Boeuf pub fn new(id: u32, mapping: Arc<IommuMapping>) -> Self {
8660249e864SSebastien Boeuf AccessPlatformMapping { id, mapping }
8670249e864SSebastien Boeuf }
8680249e864SSebastien Boeuf }
8690249e864SSebastien Boeuf
8700249e864SSebastien Boeuf impl AccessPlatform for AccessPlatformMapping {
translate_gva(&self, base: u64, _size: u64) -> std::result::Result<u64, std::io::Error>871059e787cSSebastien Boeuf fn translate_gva(&self, base: u64, _size: u64) -> std::result::Result<u64, std::io::Error> {
872059e787cSSebastien Boeuf self.mapping.translate_gva(self.id, base)
8730249e864SSebastien Boeuf }
translate_gpa(&self, base: u64, _size: u64) -> std::result::Result<u64, std::io::Error>8741cdf8b23SSebastien Boeuf fn translate_gpa(&self, base: u64, _size: u64) -> std::result::Result<u64, std::io::Error> {
8751cdf8b23SSebastien Boeuf self.mapping.translate_gpa(self.id, base)
8761cdf8b23SSebastien Boeuf }
8770249e864SSebastien Boeuf }
8780249e864SSebastien Boeuf
879f40adff2SSebastien Boeuf pub struct Iommu {
8809fc8b6d2SRob Bradford common: VirtioCommon,
88155687157SSebastien Boeuf id: String,
882f40adff2SSebastien Boeuf config: VirtioIommuConfig,
883f40adff2SSebastien Boeuf mapping: Arc<IommuMapping>,
884e9b1ad95SSebastien Boeuf ext_mapping: Arc<Mutex<BTreeMap<u32, Arc<dyn ExternalDmaMapping>>>>,
88545392366SBo Chen seccomp_action: SeccompAction,
886687d646cSRob Bradford exit_evt: EventFd,
887b3fa5654SMichael Zhao msi_iova_space: (u64, u64),
888f40adff2SSebastien Boeuf }
889f40adff2SSebastien Boeuf
890bbd0667bSSebastien Boeuf type EndpointsState = Vec<(u32, u32)>;
891bbd0667bSSebastien Boeuf type DomainsState = Vec<(u32, (Vec<(u64, Mapping)>, bool))>;
892bbd0667bSSebastien Boeuf
89310ab87d6SRob Bradford #[derive(Serialize, Deserialize)]
8941f0e5eb6SSebastien Boeuf pub struct IommuState {
89502cbea54SSebastien Boeuf avail_features: u64,
89602cbea54SSebastien Boeuf acked_features: u64,
897bbd0667bSSebastien Boeuf endpoints: EndpointsState,
898bbd0667bSSebastien Boeuf domains: DomainsState,
89902cbea54SSebastien Boeuf }
90002cbea54SSebastien Boeuf
901f40adff2SSebastien Boeuf impl Iommu {
new( id: String, seccomp_action: SeccompAction, exit_evt: EventFd, msi_iova_space: (u64, u64), address_width_bits: u8, state: Option<IommuState>, ) -> io::Result<(Self, Arc<IommuMapping>)>902687d646cSRob Bradford pub fn new(
903687d646cSRob Bradford id: String,
904687d646cSRob Bradford seccomp_action: SeccompAction,
905687d646cSRob Bradford exit_evt: EventFd,
906b3fa5654SMichael Zhao msi_iova_space: (u64, u64),
90727fda753SNikolay Edigaryev address_width_bits: u8,
9081f0e5eb6SSebastien Boeuf state: Option<IommuState>,
909687d646cSRob Bradford ) -> io::Result<(Self, Arc<IommuMapping>)> {
91027fda753SNikolay Edigaryev let (mut avail_features, acked_features, endpoints, domains, paused) =
911b62a40efSSebastien Boeuf if let Some(state) = state {
9121f0e5eb6SSebastien Boeuf info!("Restoring virtio-iommu {}", id);
9131f0e5eb6SSebastien Boeuf (
9141f0e5eb6SSebastien Boeuf state.avail_features,
9151f0e5eb6SSebastien Boeuf state.acked_features,
9161f0e5eb6SSebastien Boeuf state.endpoints.into_iter().collect(),
9171f0e5eb6SSebastien Boeuf state
9181f0e5eb6SSebastien Boeuf .domains
9191f0e5eb6SSebastien Boeuf .into_iter()
9201f0e5eb6SSebastien Boeuf .map(|(k, v)| {
9211f0e5eb6SSebastien Boeuf (
9221f0e5eb6SSebastien Boeuf k,
9231f0e5eb6SSebastien Boeuf Domain {
9241f0e5eb6SSebastien Boeuf mappings: v.0.into_iter().collect(),
9251f0e5eb6SSebastien Boeuf bypass: v.1,
9261f0e5eb6SSebastien Boeuf },
9271f0e5eb6SSebastien Boeuf )
9281f0e5eb6SSebastien Boeuf })
9291f0e5eb6SSebastien Boeuf .collect(),
930b62a40efSSebastien Boeuf true,
9311f0e5eb6SSebastien Boeuf )
9321f0e5eb6SSebastien Boeuf } else {
9332624f17fSRob Bradford let avail_features = (1u64 << VIRTIO_F_VERSION_1)
9342624f17fSRob Bradford | (1u64 << VIRTIO_IOMMU_F_MAP_UNMAP)
9352624f17fSRob Bradford | (1u64 << VIRTIO_IOMMU_F_PROBE)
9362624f17fSRob Bradford | (1u64 << VIRTIO_IOMMU_F_BYPASS_CONFIG);
9371f0e5eb6SSebastien Boeuf
938b62a40efSSebastien Boeuf (avail_features, 0, BTreeMap::new(), BTreeMap::new(), false)
9391f0e5eb6SSebastien Boeuf };
9401f0e5eb6SSebastien Boeuf
94127fda753SNikolay Edigaryev let mut config = VirtioIommuConfig {
942f40adff2SSebastien Boeuf page_size_mask: VIRTIO_IOMMU_PAGE_SIZE_MASK,
943e1822cfdSSebastien Boeuf probe_size: PROBE_PROP_SIZE,
944f40adff2SSebastien Boeuf ..Default::default()
945f40adff2SSebastien Boeuf };
946f40adff2SSebastien Boeuf
94727fda753SNikolay Edigaryev if address_width_bits < 64 {
94827fda753SNikolay Edigaryev avail_features |= 1u64 << VIRTIO_IOMMU_F_INPUT_RANGE;
94927fda753SNikolay Edigaryev config.input_range = VirtioIommuRange64 {
95027fda753SNikolay Edigaryev start: 0,
95127fda753SNikolay Edigaryev end: (1u64 << address_width_bits) - 1,
95227fda753SNikolay Edigaryev }
95327fda753SNikolay Edigaryev }
95427fda753SNikolay Edigaryev
955f40adff2SSebastien Boeuf let mapping = Arc::new(IommuMapping {
9561f0e5eb6SSebastien Boeuf endpoints: Arc::new(RwLock::new(endpoints)),
9571f0e5eb6SSebastien Boeuf domains: Arc::new(RwLock::new(domains)),
958f38360deSSebastien Boeuf bypass: AtomicBool::new(true),
959f40adff2SSebastien Boeuf });
960f40adff2SSebastien Boeuf
961f40adff2SSebastien Boeuf Ok((
962f40adff2SSebastien Boeuf Iommu {
96355687157SSebastien Boeuf id,
9649fc8b6d2SRob Bradford common: VirtioCommon {
965aa34d545SRob Bradford device_type: VirtioDeviceType::Iommu as u32,
966376babb2SRob Bradford queue_sizes: QUEUE_SIZES.to_vec(),
9671f0e5eb6SSebastien Boeuf avail_features,
9681f0e5eb6SSebastien Boeuf acked_features,
969376babb2SRob Bradford paused_sync: Some(Arc::new(Barrier::new(2))),
97083f22ac7SBo Chen min_queues: NUM_QUEUES as u16,
971b62a40efSSebastien Boeuf paused: Arc::new(AtomicBool::new(paused)),
972a9a13846SRob Bradford ..Default::default()
9739fc8b6d2SRob Bradford },
974f40adff2SSebastien Boeuf config,
975f40adff2SSebastien Boeuf mapping: mapping.clone(),
976e9b1ad95SSebastien Boeuf ext_mapping: Arc::new(Mutex::new(BTreeMap::new())),
97745392366SBo Chen seccomp_action,
978687d646cSRob Bradford exit_evt,
979b3fa5654SMichael Zhao msi_iova_space,
980f40adff2SSebastien Boeuf },
981f40adff2SSebastien Boeuf mapping,
982f40adff2SSebastien Boeuf ))
983f40adff2SSebastien Boeuf }
984c65ead5dSSebastien Boeuf
state(&self) -> IommuState98502cbea54SSebastien Boeuf fn state(&self) -> IommuState {
98602cbea54SSebastien Boeuf IommuState {
9879fc8b6d2SRob Bradford avail_features: self.common.avail_features,
9889fc8b6d2SRob Bradford acked_features: self.common.acked_features,
989dab1cab4SRob Bradford endpoints: self
990dab1cab4SRob Bradford .mapping
991dab1cab4SRob Bradford .endpoints
992dab1cab4SRob Bradford .read()
993dab1cab4SRob Bradford .unwrap()
994dab1cab4SRob Bradford .clone()
995dab1cab4SRob Bradford .into_iter()
996dab1cab4SRob Bradford .collect(),
997b40633f9SSebastien Boeuf domains: self
998dab1cab4SRob Bradford .mapping
999b40633f9SSebastien Boeuf .domains
1000dab1cab4SRob Bradford .read()
1001dab1cab4SRob Bradford .unwrap()
1002dab1cab4SRob Bradford .clone()
1003dab1cab4SRob Bradford .into_iter()
1004bbd0667bSSebastien Boeuf .map(|(k, v)| (k, (v.mappings.into_iter().collect(), v.bypass)))
1005dab1cab4SRob Bradford .collect(),
100602cbea54SSebastien Boeuf }
100702cbea54SSebastien Boeuf }
100802cbea54SSebastien Boeuf
update_bypass(&mut self)1009f38360deSSebastien Boeuf fn update_bypass(&mut self) {
1010f38360deSSebastien Boeuf // Use bypass from config if VIRTIO_IOMMU_F_BYPASS_CONFIG has been negotiated
1011f38360deSSebastien Boeuf if !self
1012f38360deSSebastien Boeuf .common
1013f38360deSSebastien Boeuf .feature_acked(VIRTIO_IOMMU_F_BYPASS_CONFIG.into())
1014f38360deSSebastien Boeuf {
1015f38360deSSebastien Boeuf return;
1016f38360deSSebastien Boeuf }
1017f38360deSSebastien Boeuf
1018f38360deSSebastien Boeuf let bypass = self.config.bypass == 1;
1019f38360deSSebastien Boeuf info!("Updating bypass mode to {}", bypass);
1020f38360deSSebastien Boeuf self.mapping.bypass.store(bypass, Ordering::Release);
1021f38360deSSebastien Boeuf }
1022f38360deSSebastien Boeuf
add_external_mapping(&mut self, device_id: u32, mapping: Arc<dyn ExternalDmaMapping>)1023c65ead5dSSebastien Boeuf pub fn add_external_mapping(&mut self, device_id: u32, mapping: Arc<dyn ExternalDmaMapping>) {
1024e9b1ad95SSebastien Boeuf self.ext_mapping.lock().unwrap().insert(device_id, mapping);
1025c65ead5dSSebastien Boeuf }
1026fdecd94bSBo Chen
1027fdecd94bSBo Chen #[cfg(fuzzing)]
wait_for_epoll_threads(&mut self)1028fdecd94bSBo Chen pub fn wait_for_epoll_threads(&mut self) {
1029fdecd94bSBo Chen self.common.wait_for_epoll_threads();
1030fdecd94bSBo Chen }
1031f40adff2SSebastien Boeuf }
1032f40adff2SSebastien Boeuf
1033f40adff2SSebastien Boeuf impl Drop for Iommu {
drop(&mut self)1034f40adff2SSebastien Boeuf fn drop(&mut self) {
1035376babb2SRob Bradford if let Some(kill_evt) = self.common.kill_evt.take() {
1036f40adff2SSebastien Boeuf // Ignore the result because there is nothing we can do about it.
1037f40adff2SSebastien Boeuf let _ = kill_evt.write(1);
1038f40adff2SSebastien Boeuf }
1039ad6c0ee5SPhilipp Schuster self.common.wait_for_epoll_threads();
1040f40adff2SSebastien Boeuf }
1041f40adff2SSebastien Boeuf }
1042f40adff2SSebastien Boeuf
1043f40adff2SSebastien Boeuf impl VirtioDevice for Iommu {
device_type(&self) -> u321044f40adff2SSebastien Boeuf fn device_type(&self) -> u32 {
1045376babb2SRob Bradford self.common.device_type
1046f40adff2SSebastien Boeuf }
1047f40adff2SSebastien Boeuf
queue_max_sizes(&self) -> &[u16]1048f40adff2SSebastien Boeuf fn queue_max_sizes(&self) -> &[u16] {
1049376babb2SRob Bradford &self.common.queue_sizes
1050f40adff2SSebastien Boeuf }
1051f40adff2SSebastien Boeuf
features(&self) -> u64105214eddf72SCathy Zhang fn features(&self) -> u64 {
10539fc8b6d2SRob Bradford self.common.avail_features
1054f40adff2SSebastien Boeuf }
1055f40adff2SSebastien Boeuf
ack_features(&mut self, value: u64)105614eddf72SCathy Zhang fn ack_features(&mut self, value: u64) {
10579fc8b6d2SRob Bradford self.common.ack_features(value)
1058f40adff2SSebastien Boeuf }
1059f40adff2SSebastien Boeuf
read_config(&self, offset: u64, data: &mut [u8])1060751a3020SRob Bradford fn read_config(&self, offset: u64, data: &mut [u8]) {
1061a6fe4aa7SSebastien Boeuf self.read_config_from_slice(self.config.as_slice(), offset, data);
1062f40adff2SSebastien Boeuf }
1063f40adff2SSebastien Boeuf
write_config(&mut self, offset: u64, data: &[u8])1064f38360deSSebastien Boeuf fn write_config(&mut self, offset: u64, data: &[u8]) {
1065f38360deSSebastien Boeuf // The "bypass" field is the only mutable field
1066f38360deSSebastien Boeuf let bypass_offset =
1067f38360deSSebastien Boeuf (&self.config.bypass as *const _ as u64) - (&self.config as *const _ as u64);
1068f38360deSSebastien Boeuf if offset != bypass_offset || data.len() != std::mem::size_of_val(&self.config.bypass) {
1069f38360deSSebastien Boeuf error!(
1070f38360deSSebastien Boeuf "Attempt to write to read-only field: offset {:x} length {}",
1071f38360deSSebastien Boeuf offset,
1072f38360deSSebastien Boeuf data.len()
1073f38360deSSebastien Boeuf );
1074f38360deSSebastien Boeuf return;
1075f38360deSSebastien Boeuf }
1076f38360deSSebastien Boeuf
1077f38360deSSebastien Boeuf self.config.bypass = data[0];
1078f38360deSSebastien Boeuf
1079f38360deSSebastien Boeuf self.update_bypass();
1080f38360deSSebastien Boeuf }
1081f38360deSSebastien Boeuf
activate( &mut self, mem: GuestMemoryAtomic<GuestMemoryMmap>, interrupt_cb: Arc<dyn VirtioInterrupt>, mut queues: Vec<(usize, Queue, EventFd)>, ) -> ActivateResult1082f40adff2SSebastien Boeuf fn activate(
1083f40adff2SSebastien Boeuf &mut self,
1084a423bf13SSebastien Boeuf mem: GuestMemoryAtomic<GuestMemoryMmap>,
1085c396bacaSSebastien Boeuf interrupt_cb: Arc<dyn VirtioInterrupt>,
1086710a860eSBo Chen mut queues: Vec<(usize, Queue, EventFd)>,
1087f40adff2SSebastien Boeuf ) -> ActivateResult {
10883f62a172SSebastien Boeuf self.common.activate(&queues, &interrupt_cb)?;
1089280bef83SRob Bradford let (kill_evt, pause_evt) = self.common.dup_eventfds();
10903f62a172SSebastien Boeuf
1091710a860eSBo Chen let (_, request_queue, request_queue_evt) = queues.remove(0);
10929c658e21SBo Chen let (_, _event_queue, _event_queue_evt) = queues.remove(0);
10933f62a172SSebastien Boeuf
1094f40adff2SSebastien Boeuf let mut handler = IommuEpollHandler {
1095a423bf13SSebastien Boeuf mem,
1096710a860eSBo Chen request_queue,
1097710a860eSBo Chen _event_queue,
1098f40adff2SSebastien Boeuf interrupt_cb,
1099710a860eSBo Chen request_queue_evt,
11009c658e21SBo Chen _event_queue_evt,
1101f40adff2SSebastien Boeuf kill_evt,
1102dae0b2efSSamuel Ortiz pause_evt,
1103f40adff2SSebastien Boeuf mapping: self.mapping.clone(),
1104c65ead5dSSebastien Boeuf ext_mapping: self.ext_mapping.clone(),
1105b3fa5654SMichael Zhao msi_iova_space: self.msi_iova_space,
1106f40adff2SSebastien Boeuf };
1107f40adff2SSebastien Boeuf
1108376babb2SRob Bradford let paused = self.common.paused.clone();
1109376babb2SRob Bradford let paused_sync = self.common.paused_sync.clone();
1110f648f285SSamuel Ortiz let mut epoll_threads = Vec::new();
111154e523c3SRob Bradford spawn_virtio_thread(
111254e523c3SRob Bradford &self.id,
111354e523c3SRob Bradford &self.seccomp_action,
111454e523c3SRob Bradford Thread::VirtioIommu,
111554e523c3SRob Bradford &mut epoll_threads,
1116687d646cSRob Bradford &self.exit_evt,
1117df5b803aSBo Chen move || handler.run(paused, paused_sync.unwrap()),
111854e523c3SRob Bradford )?;
1119f40adff2SSebastien Boeuf
1120376babb2SRob Bradford self.common.epoll_threads = Some(epoll_threads);
1121f648f285SSamuel Ortiz
1122c89095abSRob Bradford event!("virtio-device", "activated", "id", &self.id);
1123f40adff2SSebastien Boeuf Ok(())
1124f40adff2SSebastien Boeuf }
1125f40adff2SSebastien Boeuf
reset(&mut self) -> Option<Arc<dyn VirtioInterrupt>>112623f9ec50SRob Bradford fn reset(&mut self) -> Option<Arc<dyn VirtioInterrupt>> {
1127c89095abSRob Bradford let result = self.common.reset();
1128c89095abSRob Bradford event!("virtio-device", "reset", "id", &self.id);
1129c89095abSRob Bradford result
1130f40adff2SSebastien Boeuf }
1131f40adff2SSebastien Boeuf }
1132dae0b2efSSamuel Ortiz
1133376babb2SRob Bradford impl Pausable for Iommu {
pause(&mut self) -> result::Result<(), MigratableError>1134376babb2SRob Bradford fn pause(&mut self) -> result::Result<(), MigratableError> {
1135376babb2SRob Bradford self.common.pause()
1136376babb2SRob Bradford }
1137376babb2SRob Bradford
resume(&mut self) -> result::Result<(), MigratableError>1138376babb2SRob Bradford fn resume(&mut self) -> result::Result<(), MigratableError> {
1139376babb2SRob Bradford self.common.resume()
1140376babb2SRob Bradford }
1141376babb2SRob Bradford }
1142376babb2SRob Bradford
114355687157SSebastien Boeuf impl Snapshottable for Iommu {
id(&self) -> String114455687157SSebastien Boeuf fn id(&self) -> String {
114555687157SSebastien Boeuf self.id.clone()
114655687157SSebastien Boeuf }
114702cbea54SSebastien Boeuf
snapshot(&mut self) -> std::result::Result<Snapshot, MigratableError>1148871138d5SSebastien Boeuf fn snapshot(&mut self) -> std::result::Result<Snapshot, MigratableError> {
114910ab87d6SRob Bradford Snapshot::new_from_state(&self.state())
115002cbea54SSebastien Boeuf }
115155687157SSebastien Boeuf }
11521b1a2175SSamuel Ortiz impl Transportable for Iommu {}
1153dae0b2efSSamuel Ortiz impl Migratable for Iommu {}
1154