1 /* SPDX-License-Identifier: GPL-2.0-only */
2 /*
3 * Copyright (C) 2021 Intel Corporation
4 * Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES
5 */
6 #ifndef __LINUX_IOMMUFD_H
7 #define __LINUX_IOMMUFD_H
8
9 #include <linux/err.h>
10 #include <linux/errno.h>
11 #include <linux/iommu.h>
12 #include <linux/refcount.h>
13 #include <linux/types.h>
14 #include <linux/xarray.h>
15 #include <uapi/linux/iommufd.h>
16
17 struct device;
18 struct file;
19 struct iommu_group;
20 struct iommu_user_data;
21 struct iommu_user_data_array;
22 struct iommufd_access;
23 struct iommufd_ctx;
24 struct iommufd_device;
25 struct iommufd_viommu_ops;
26 struct page;
27
28 enum iommufd_object_type {
29 IOMMUFD_OBJ_NONE,
30 IOMMUFD_OBJ_ANY = IOMMUFD_OBJ_NONE,
31 IOMMUFD_OBJ_DEVICE,
32 IOMMUFD_OBJ_HWPT_PAGING,
33 IOMMUFD_OBJ_HWPT_NESTED,
34 IOMMUFD_OBJ_IOAS,
35 IOMMUFD_OBJ_ACCESS,
36 IOMMUFD_OBJ_FAULT,
37 IOMMUFD_OBJ_VIOMMU,
38 IOMMUFD_OBJ_VDEVICE,
39 IOMMUFD_OBJ_VEVENTQ,
40 IOMMUFD_OBJ_HW_QUEUE,
41 #ifdef CONFIG_IOMMUFD_TEST
42 IOMMUFD_OBJ_SELFTEST,
43 #endif
44 IOMMUFD_OBJ_MAX,
45 };
46
47 /* Base struct for all objects with a userspace ID handle. */
48 struct iommufd_object {
49 /*
50 * Destroy will sleep and wait for wait_cnt to go to zero. This allows
51 * concurrent users of the ID to reliably avoid causing a spurious
52 * destroy failure. Incrementing this count should either be short
53 * lived or be revoked and blocked during pre_destroy().
54 */
55 refcount_t wait_cnt;
56 refcount_t users;
57 enum iommufd_object_type type;
58 unsigned int id;
59 };
60
61 struct iommufd_device *iommufd_device_bind(struct iommufd_ctx *ictx,
62 struct device *dev, u32 *id);
63 void iommufd_device_unbind(struct iommufd_device *idev);
64
65 int iommufd_device_attach(struct iommufd_device *idev, ioasid_t pasid,
66 u32 *pt_id);
67 int iommufd_device_replace(struct iommufd_device *idev, ioasid_t pasid,
68 u32 *pt_id);
69 void iommufd_device_detach(struct iommufd_device *idev, ioasid_t pasid);
70
71 struct iommufd_ctx *iommufd_device_to_ictx(struct iommufd_device *idev);
72 u32 iommufd_device_to_id(struct iommufd_device *idev);
73
74 struct iommufd_access_ops {
75 u8 needs_pin_pages : 1;
76 void (*unmap)(void *data, unsigned long iova, unsigned long length);
77 };
78
79 enum {
80 IOMMUFD_ACCESS_RW_READ = 0,
81 IOMMUFD_ACCESS_RW_WRITE = 1 << 0,
82 /* Set if the caller is in a kthread then rw will use kthread_use_mm() */
83 IOMMUFD_ACCESS_RW_KTHREAD = 1 << 1,
84
85 /* Only for use by selftest */
86 __IOMMUFD_ACCESS_RW_SLOW_PATH = 1 << 2,
87 };
88
89 struct iommufd_access *
90 iommufd_access_create(struct iommufd_ctx *ictx,
91 const struct iommufd_access_ops *ops, void *data, u32 *id);
92 void iommufd_access_destroy(struct iommufd_access *access);
93 int iommufd_access_attach(struct iommufd_access *access, u32 ioas_id);
94 int iommufd_access_replace(struct iommufd_access *access, u32 ioas_id);
95 void iommufd_access_detach(struct iommufd_access *access);
96
97 void iommufd_ctx_get(struct iommufd_ctx *ictx);
98
99 struct iommufd_viommu {
100 struct iommufd_object obj;
101 struct iommufd_ctx *ictx;
102 struct iommu_device *iommu_dev;
103 struct iommufd_hwpt_paging *hwpt;
104
105 const struct iommufd_viommu_ops *ops;
106
107 struct xarray vdevs;
108 struct list_head veventqs;
109 struct rw_semaphore veventqs_rwsem;
110
111 enum iommu_viommu_type type;
112 };
113
114 struct iommufd_vdevice {
115 struct iommufd_object obj;
116 struct iommufd_viommu *viommu;
117 struct iommufd_device *idev;
118
119 /*
120 * Virtual device ID per vIOMMU, e.g. vSID of ARM SMMUv3, vDeviceID of
121 * AMD IOMMU, and vRID of Intel VT-d
122 */
123 u64 virt_id;
124
125 /* Clean up all driver-specific parts of an iommufd_vdevice */
126 void (*destroy)(struct iommufd_vdevice *vdev);
127 };
128
129 struct iommufd_hw_queue {
130 struct iommufd_object obj;
131 struct iommufd_viommu *viommu;
132 struct iommufd_access *access;
133
134 u64 base_addr; /* in guest physical address space */
135 size_t length;
136
137 enum iommu_hw_queue_type type;
138
139 /* Clean up all driver-specific parts of an iommufd_hw_queue */
140 void (*destroy)(struct iommufd_hw_queue *hw_queue);
141 };
142
143 /**
144 * struct iommufd_viommu_ops - vIOMMU specific operations
145 * @destroy: Clean up all driver-specific parts of an iommufd_viommu. The memory
146 * of the vIOMMU will be free-ed by iommufd core after calling this op
147 * @alloc_domain_nested: Allocate a IOMMU_DOMAIN_NESTED on a vIOMMU that holds a
148 * nesting parent domain (IOMMU_DOMAIN_PAGING). @user_data
149 * must be defined in include/uapi/linux/iommufd.h.
150 * It must fully initialize the new iommu_domain before
151 * returning. Upon failure, ERR_PTR must be returned.
152 * @cache_invalidate: Flush hardware cache used by a vIOMMU. It can be used for
153 * any IOMMU hardware specific cache: TLB and device cache.
154 * The @array passes in the cache invalidation requests, in
155 * form of a driver data structure. A driver must update the
156 * array->entry_num to report the number of handled requests.
157 * The data structure of the array entry must be defined in
158 * include/uapi/linux/iommufd.h
159 * @vdevice_size: Size of the driver-defined vDEVICE structure per this vIOMMU
160 * @vdevice_init: Initialize the driver-level structure of a vDEVICE object, or
161 * related HW procedure. @vdev is already initialized by iommufd
162 * core: vdev->dev and vdev->viommu pointers; vdev->id carries a
163 * per-vIOMMU virtual ID (refer to struct iommu_vdevice_alloc in
164 * include/uapi/linux/iommufd.h)
165 * If driver has a deinit function to revert what vdevice_init op
166 * does, it should set it to the @vdev->destroy function pointer
167 * @get_hw_queue_size: Get the size of a driver-defined HW queue structure for a
168 * given @viommu corresponding to @queue_type. Driver should
169 * return 0 if HW queue aren't supported accordingly. It is
170 * required for driver to use the HW_QUEUE_STRUCT_SIZE macro
171 * to sanitize the driver-level HW queue structure related
172 * to the core one
173 * @hw_queue_init_phys: Initialize the driver-level structure of a HW queue that
174 * is initialized with its core-level structure that holds
175 * all the info about a guest queue memory.
176 * Driver providing this op indicates that HW accesses the
177 * guest queue memory via physical addresses.
178 * @index carries the logical HW QUEUE ID per vIOMMU in a
179 * guest VM, for a multi-queue model. @base_addr_pa carries
180 * the physical location of the guest queue
181 * If driver has a deinit function to revert what this op
182 * does, it should set it to the @hw_queue->destroy pointer
183 */
184 struct iommufd_viommu_ops {
185 void (*destroy)(struct iommufd_viommu *viommu);
186 struct iommu_domain *(*alloc_domain_nested)(
187 struct iommufd_viommu *viommu, u32 flags,
188 const struct iommu_user_data *user_data);
189 int (*cache_invalidate)(struct iommufd_viommu *viommu,
190 struct iommu_user_data_array *array);
191 const size_t vdevice_size;
192 int (*vdevice_init)(struct iommufd_vdevice *vdev);
193 size_t (*get_hw_queue_size)(struct iommufd_viommu *viommu,
194 enum iommu_hw_queue_type queue_type);
195 /* AMD's HW will add hw_queue_init simply using @hw_queue->base_addr */
196 int (*hw_queue_init_phys)(struct iommufd_hw_queue *hw_queue, u32 index,
197 phys_addr_t base_addr_pa);
198 };
199
200 #if IS_ENABLED(CONFIG_IOMMUFD)
201 struct iommufd_ctx *iommufd_ctx_from_file(struct file *file);
202 struct iommufd_ctx *iommufd_ctx_from_fd(int fd);
203 void iommufd_ctx_put(struct iommufd_ctx *ictx);
204 bool iommufd_ctx_has_group(struct iommufd_ctx *ictx, struct iommu_group *group);
205
206 int iommufd_access_pin_pages(struct iommufd_access *access, unsigned long iova,
207 unsigned long length, struct page **out_pages,
208 unsigned int flags);
209 void iommufd_access_unpin_pages(struct iommufd_access *access,
210 unsigned long iova, unsigned long length);
211 int iommufd_access_rw(struct iommufd_access *access, unsigned long iova,
212 void *data, size_t len, unsigned int flags);
213 int iommufd_vfio_compat_ioas_get_id(struct iommufd_ctx *ictx, u32 *out_ioas_id);
214 int iommufd_vfio_compat_ioas_create(struct iommufd_ctx *ictx);
215 int iommufd_vfio_compat_set_no_iommu(struct iommufd_ctx *ictx);
216 #else /* !CONFIG_IOMMUFD */
iommufd_ctx_from_file(struct file * file)217 static inline struct iommufd_ctx *iommufd_ctx_from_file(struct file *file)
218 {
219 return ERR_PTR(-EOPNOTSUPP);
220 }
221
iommufd_ctx_put(struct iommufd_ctx * ictx)222 static inline void iommufd_ctx_put(struct iommufd_ctx *ictx)
223 {
224 }
225
iommufd_access_pin_pages(struct iommufd_access * access,unsigned long iova,unsigned long length,struct page ** out_pages,unsigned int flags)226 static inline int iommufd_access_pin_pages(struct iommufd_access *access,
227 unsigned long iova,
228 unsigned long length,
229 struct page **out_pages,
230 unsigned int flags)
231 {
232 return -EOPNOTSUPP;
233 }
234
iommufd_access_unpin_pages(struct iommufd_access * access,unsigned long iova,unsigned long length)235 static inline void iommufd_access_unpin_pages(struct iommufd_access *access,
236 unsigned long iova,
237 unsigned long length)
238 {
239 }
240
iommufd_access_rw(struct iommufd_access * access,unsigned long iova,void * data,size_t len,unsigned int flags)241 static inline int iommufd_access_rw(struct iommufd_access *access,
242 unsigned long iova, void *data, size_t len,
243 unsigned int flags)
244 {
245 return -EOPNOTSUPP;
246 }
247
iommufd_vfio_compat_ioas_create(struct iommufd_ctx * ictx)248 static inline int iommufd_vfio_compat_ioas_create(struct iommufd_ctx *ictx)
249 {
250 return -EOPNOTSUPP;
251 }
252
iommufd_vfio_compat_set_no_iommu(struct iommufd_ctx * ictx)253 static inline int iommufd_vfio_compat_set_no_iommu(struct iommufd_ctx *ictx)
254 {
255 return -EOPNOTSUPP;
256 }
257 #endif /* CONFIG_IOMMUFD */
258
259 #if IS_ENABLED(CONFIG_IOMMUFD_DRIVER_CORE)
260 int _iommufd_object_depend(struct iommufd_object *obj_dependent,
261 struct iommufd_object *obj_depended);
262 void _iommufd_object_undepend(struct iommufd_object *obj_dependent,
263 struct iommufd_object *obj_depended);
264 int _iommufd_alloc_mmap(struct iommufd_ctx *ictx, struct iommufd_object *owner,
265 phys_addr_t mmio_addr, size_t length,
266 unsigned long *offset);
267 void _iommufd_destroy_mmap(struct iommufd_ctx *ictx,
268 struct iommufd_object *owner, unsigned long offset);
269 struct device *iommufd_vdevice_to_device(struct iommufd_vdevice *vdev);
270 struct device *iommufd_viommu_find_dev(struct iommufd_viommu *viommu,
271 unsigned long vdev_id);
272 int iommufd_viommu_get_vdev_id(struct iommufd_viommu *viommu,
273 struct device *dev, unsigned long *vdev_id);
274 int iommufd_viommu_report_event(struct iommufd_viommu *viommu,
275 enum iommu_veventq_type type, void *event_data,
276 size_t data_len);
277 #else /* !CONFIG_IOMMUFD_DRIVER_CORE */
_iommufd_object_depend(struct iommufd_object * obj_dependent,struct iommufd_object * obj_depended)278 static inline int _iommufd_object_depend(struct iommufd_object *obj_dependent,
279 struct iommufd_object *obj_depended)
280 {
281 return -EOPNOTSUPP;
282 }
283
284 static inline void
_iommufd_object_undepend(struct iommufd_object * obj_dependent,struct iommufd_object * obj_depended)285 _iommufd_object_undepend(struct iommufd_object *obj_dependent,
286 struct iommufd_object *obj_depended)
287 {
288 }
289
_iommufd_alloc_mmap(struct iommufd_ctx * ictx,struct iommufd_object * owner,phys_addr_t mmio_addr,size_t length,unsigned long * offset)290 static inline int _iommufd_alloc_mmap(struct iommufd_ctx *ictx,
291 struct iommufd_object *owner,
292 phys_addr_t mmio_addr, size_t length,
293 unsigned long *offset)
294 {
295 return -EOPNOTSUPP;
296 }
297
_iommufd_destroy_mmap(struct iommufd_ctx * ictx,struct iommufd_object * owner,unsigned long offset)298 static inline void _iommufd_destroy_mmap(struct iommufd_ctx *ictx,
299 struct iommufd_object *owner,
300 unsigned long offset)
301 {
302 }
303
304 static inline struct device *
iommufd_vdevice_to_device(struct iommufd_vdevice * vdev)305 iommufd_vdevice_to_device(struct iommufd_vdevice *vdev)
306 {
307 return NULL;
308 }
309
310 static inline struct device *
iommufd_viommu_find_dev(struct iommufd_viommu * viommu,unsigned long vdev_id)311 iommufd_viommu_find_dev(struct iommufd_viommu *viommu, unsigned long vdev_id)
312 {
313 return NULL;
314 }
315
iommufd_viommu_get_vdev_id(struct iommufd_viommu * viommu,struct device * dev,unsigned long * vdev_id)316 static inline int iommufd_viommu_get_vdev_id(struct iommufd_viommu *viommu,
317 struct device *dev,
318 unsigned long *vdev_id)
319 {
320 return -ENOENT;
321 }
322
iommufd_viommu_report_event(struct iommufd_viommu * viommu,enum iommu_veventq_type type,void * event_data,size_t data_len)323 static inline int iommufd_viommu_report_event(struct iommufd_viommu *viommu,
324 enum iommu_veventq_type type,
325 void *event_data, size_t data_len)
326 {
327 return -EOPNOTSUPP;
328 }
329 #endif /* CONFIG_IOMMUFD_DRIVER_CORE */
330
331 #define VIOMMU_STRUCT_SIZE(drv_struct, member) \
332 (sizeof(drv_struct) + \
333 BUILD_BUG_ON_ZERO(offsetof(drv_struct, member)) + \
334 BUILD_BUG_ON_ZERO(!__same_type(struct iommufd_viommu, \
335 ((drv_struct *)NULL)->member)))
336
337 #define VDEVICE_STRUCT_SIZE(drv_struct, member) \
338 (sizeof(drv_struct) + \
339 BUILD_BUG_ON_ZERO(offsetof(drv_struct, member)) + \
340 BUILD_BUG_ON_ZERO(!__same_type(struct iommufd_vdevice, \
341 ((drv_struct *)NULL)->member)))
342
343 #define HW_QUEUE_STRUCT_SIZE(drv_struct, member) \
344 (sizeof(drv_struct) + \
345 BUILD_BUG_ON_ZERO(offsetof(drv_struct, member)) + \
346 BUILD_BUG_ON_ZERO(!__same_type(struct iommufd_hw_queue, \
347 ((drv_struct *)NULL)->member)))
348
349 /*
350 * Helpers for IOMMU driver to build/destroy a dependency between two sibling
351 * structures created by one of the allocators above
352 */
353 #define iommufd_hw_queue_depend(dependent, depended, member) \
354 ({ \
355 int ret = -EINVAL; \
356 \
357 static_assert(__same_type(struct iommufd_hw_queue, \
358 dependent->member)); \
359 static_assert(__same_type(typeof(*dependent), *depended)); \
360 if (!WARN_ON_ONCE(dependent->member.viommu != \
361 depended->member.viommu)) \
362 ret = _iommufd_object_depend(&dependent->member.obj, \
363 &depended->member.obj); \
364 ret; \
365 })
366
367 #define iommufd_hw_queue_undepend(dependent, depended, member) \
368 ({ \
369 static_assert(__same_type(struct iommufd_hw_queue, \
370 dependent->member)); \
371 static_assert(__same_type(typeof(*dependent), *depended)); \
372 WARN_ON_ONCE(dependent->member.viommu != \
373 depended->member.viommu); \
374 _iommufd_object_undepend(&dependent->member.obj, \
375 &depended->member.obj); \
376 })
377
378 /*
379 * Helpers for IOMMU driver to alloc/destroy an mmapable area for a structure.
380 *
381 * To support an mmappable MMIO region, kernel driver must first register it to
382 * iommufd core to allocate an @offset, during a driver-structure initialization
383 * (e.g. viommu_init op). Then, it should report to user space this @offset and
384 * the @length of the MMIO region for mmap syscall.
385 */
iommufd_viommu_alloc_mmap(struct iommufd_viommu * viommu,phys_addr_t mmio_addr,size_t length,unsigned long * offset)386 static inline int iommufd_viommu_alloc_mmap(struct iommufd_viommu *viommu,
387 phys_addr_t mmio_addr,
388 size_t length,
389 unsigned long *offset)
390 {
391 return _iommufd_alloc_mmap(viommu->ictx, &viommu->obj, mmio_addr,
392 length, offset);
393 }
394
iommufd_viommu_destroy_mmap(struct iommufd_viommu * viommu,unsigned long offset)395 static inline void iommufd_viommu_destroy_mmap(struct iommufd_viommu *viommu,
396 unsigned long offset)
397 {
398 _iommufd_destroy_mmap(viommu->ictx, &viommu->obj, offset);
399 }
400 #endif
401