1 /* SPDX-License-Identifier: GPL-2.0-only */
2 /*
3 * Copyright (C) 2007-2008 Advanced Micro Devices, Inc.
4 * Author: Joerg Roedel <joerg.roedel@amd.com>
5 */
6
7 #ifndef __LINUX_IOMMU_H
8 #define __LINUX_IOMMU_H
9
10 #include <linux/scatterlist.h>
11 #include <linux/device.h>
12 #include <linux/types.h>
13 #include <linux/errno.h>
14 #include <linux/err.h>
15 #include <linux/of.h>
16 #include <linux/iova_bitmap.h>
17 #include <uapi/linux/iommu.h>
18
19 #define IOMMU_READ (1 << 0)
20 #define IOMMU_WRITE (1 << 1)
21 #define IOMMU_CACHE (1 << 2) /* DMA cache coherency */
22 #define IOMMU_NOEXEC (1 << 3)
23 #define IOMMU_MMIO (1 << 4) /* e.g. things like MSI doorbells */
24 /*
25 * Where the bus hardware includes a privilege level as part of its access type
26 * markings, and certain devices are capable of issuing transactions marked as
27 * either 'supervisor' or 'user', the IOMMU_PRIV flag requests that the other
28 * given permission flags only apply to accesses at the higher privilege level,
29 * and that unprivileged transactions should have as little access as possible.
30 * This would usually imply the same permissions as kernel mappings on the CPU,
31 * if the IOMMU page table format is equivalent.
32 */
33 #define IOMMU_PRIV (1 << 5)
34
35 struct iommu_ops;
36 struct iommu_group;
37 struct bus_type;
38 struct device;
39 struct iommu_domain;
40 struct iommu_domain_ops;
41 struct iommu_dirty_ops;
42 struct notifier_block;
43 struct iommu_sva;
44 struct iommu_fault_event;
45 struct iommu_dma_cookie;
46
47 /* iommu fault flags */
48 #define IOMMU_FAULT_READ 0x0
49 #define IOMMU_FAULT_WRITE 0x1
50
51 typedef int (*iommu_fault_handler_t)(struct iommu_domain *,
52 struct device *, unsigned long, int, void *);
53 typedef int (*iommu_dev_fault_handler_t)(struct iommu_fault *, void *);
54
55 struct iommu_domain_geometry {
56 dma_addr_t aperture_start; /* First address that can be mapped */
57 dma_addr_t aperture_end; /* Last address that can be mapped */
58 bool force_aperture; /* DMA only allowed in mappable range? */
59 };
60
61 /* Domain feature flags */
62 #define __IOMMU_DOMAIN_PAGING (1U << 0) /* Support for iommu_map/unmap */
63 #define __IOMMU_DOMAIN_DMA_API (1U << 1) /* Domain for use in DMA-API
64 implementation */
65 #define __IOMMU_DOMAIN_PT (1U << 2) /* Domain is identity mapped */
66 #define __IOMMU_DOMAIN_DMA_FQ (1U << 3) /* DMA-API uses flush queue */
67
68 #define __IOMMU_DOMAIN_SVA (1U << 4) /* Shared process address space */
69 #define __IOMMU_DOMAIN_PLATFORM (1U << 5)
70
71 #define __IOMMU_DOMAIN_NESTED (1U << 6) /* User-managed address space nested
72 on a stage-2 translation */
73
74 #define IOMMU_DOMAIN_ALLOC_FLAGS ~__IOMMU_DOMAIN_DMA_FQ
75 /*
76 * This are the possible domain-types
77 *
78 * IOMMU_DOMAIN_BLOCKED - All DMA is blocked, can be used to isolate
79 * devices
80 * IOMMU_DOMAIN_IDENTITY - DMA addresses are system physical addresses
81 * IOMMU_DOMAIN_UNMANAGED - DMA mappings managed by IOMMU-API user, used
82 * for VMs
83 * IOMMU_DOMAIN_DMA - Internally used for DMA-API implementations.
84 * This flag allows IOMMU drivers to implement
85 * certain optimizations for these domains
86 * IOMMU_DOMAIN_DMA_FQ - As above, but definitely using batched TLB
87 * invalidation.
88 * IOMMU_DOMAIN_SVA - DMA addresses are shared process addresses
89 * represented by mm_struct's.
90 * IOMMU_DOMAIN_PLATFORM - Legacy domain for drivers that do their own
91 * dma_api stuff. Do not use in new drivers.
92 */
93 #define IOMMU_DOMAIN_BLOCKED (0U)
94 #define IOMMU_DOMAIN_IDENTITY (__IOMMU_DOMAIN_PT)
95 #define IOMMU_DOMAIN_UNMANAGED (__IOMMU_DOMAIN_PAGING)
96 #define IOMMU_DOMAIN_DMA (__IOMMU_DOMAIN_PAGING | \
97 __IOMMU_DOMAIN_DMA_API)
98 #define IOMMU_DOMAIN_DMA_FQ (__IOMMU_DOMAIN_PAGING | \
99 __IOMMU_DOMAIN_DMA_API | \
100 __IOMMU_DOMAIN_DMA_FQ)
101 #define IOMMU_DOMAIN_SVA (__IOMMU_DOMAIN_SVA)
102 #define IOMMU_DOMAIN_PLATFORM (__IOMMU_DOMAIN_PLATFORM)
103 #define IOMMU_DOMAIN_NESTED (__IOMMU_DOMAIN_NESTED)
104
105 struct iommu_domain {
106 unsigned type;
107 const struct iommu_domain_ops *ops;
108 const struct iommu_dirty_ops *dirty_ops;
109 const struct iommu_ops *owner; /* Whose domain_alloc we came from */
110 unsigned long pgsize_bitmap; /* Bitmap of page sizes in use */
111 struct iommu_domain_geometry geometry;
112 struct iommu_dma_cookie *iova_cookie;
113 enum iommu_page_response_code (*iopf_handler)(struct iommu_fault *fault,
114 void *data);
115 void *fault_data;
116 union {
117 struct {
118 iommu_fault_handler_t handler;
119 void *handler_token;
120 };
121 struct { /* IOMMU_DOMAIN_SVA */
122 struct mm_struct *mm;
123 int users;
124 /*
125 * Next iommu_domain in mm->iommu_mm->sva-domains list
126 * protected by iommu_sva_lock.
127 */
128 struct list_head next;
129 };
130 };
131 };
132
iommu_is_dma_domain(struct iommu_domain * domain)133 static inline bool iommu_is_dma_domain(struct iommu_domain *domain)
134 {
135 return domain->type & __IOMMU_DOMAIN_DMA_API;
136 }
137
138 enum iommu_cap {
139 IOMMU_CAP_CACHE_COHERENCY, /* IOMMU_CACHE is supported */
140 IOMMU_CAP_NOEXEC, /* IOMMU_NOEXEC flag */
141 IOMMU_CAP_PRE_BOOT_PROTECTION, /* Firmware says it used the IOMMU for
142 DMA protection and we should too */
143 /*
144 * Per-device flag indicating if enforce_cache_coherency() will work on
145 * this device.
146 */
147 IOMMU_CAP_ENFORCE_CACHE_COHERENCY,
148 /*
149 * IOMMU driver does not issue TLB maintenance during .unmap, so can
150 * usefully support the non-strict DMA flush queue.
151 */
152 IOMMU_CAP_DEFERRED_FLUSH,
153 IOMMU_CAP_DIRTY_TRACKING, /* IOMMU supports dirty tracking */
154 };
155
156 /* These are the possible reserved region types */
157 enum iommu_resv_type {
158 /* Memory regions which must be mapped 1:1 at all times */
159 IOMMU_RESV_DIRECT,
160 /*
161 * Memory regions which are advertised to be 1:1 but are
162 * commonly considered relaxable in some conditions,
163 * for instance in device assignment use case (USB, Graphics)
164 */
165 IOMMU_RESV_DIRECT_RELAXABLE,
166 /* Arbitrary "never map this or give it to a device" address ranges */
167 IOMMU_RESV_RESERVED,
168 /* Hardware MSI region (untranslated) */
169 IOMMU_RESV_MSI,
170 /* Software-managed MSI translation window */
171 IOMMU_RESV_SW_MSI,
172 };
173
174 /**
175 * struct iommu_resv_region - descriptor for a reserved memory region
176 * @list: Linked list pointers
177 * @start: System physical start address of the region
178 * @length: Length of the region in bytes
179 * @prot: IOMMU Protection flags (READ/WRITE/...)
180 * @type: Type of the reserved region
181 * @free: Callback to free associated memory allocations
182 */
183 struct iommu_resv_region {
184 struct list_head list;
185 phys_addr_t start;
186 size_t length;
187 int prot;
188 enum iommu_resv_type type;
189 void (*free)(struct device *dev, struct iommu_resv_region *region);
190 };
191
192 struct iommu_iort_rmr_data {
193 struct iommu_resv_region rr;
194
195 /* Stream IDs associated with IORT RMR entry */
196 const u32 *sids;
197 u32 num_sids;
198 };
199
200 /**
201 * enum iommu_dev_features - Per device IOMMU features
202 * @IOMMU_DEV_FEAT_SVA: Shared Virtual Addresses
203 * @IOMMU_DEV_FEAT_IOPF: I/O Page Faults such as PRI or Stall. Generally
204 * enabling %IOMMU_DEV_FEAT_SVA requires
205 * %IOMMU_DEV_FEAT_IOPF, but some devices manage I/O Page
206 * Faults themselves instead of relying on the IOMMU. When
207 * supported, this feature must be enabled before and
208 * disabled after %IOMMU_DEV_FEAT_SVA.
209 *
210 * Device drivers enable a feature using iommu_dev_enable_feature().
211 */
212 enum iommu_dev_features {
213 IOMMU_DEV_FEAT_SVA,
214 IOMMU_DEV_FEAT_IOPF,
215 };
216
217 #define IOMMU_NO_PASID (0U) /* Reserved for DMA w/o PASID */
218 #define IOMMU_FIRST_GLOBAL_PASID (1U) /*starting range for allocation */
219 #define IOMMU_PASID_INVALID (-1U)
220 typedef unsigned int ioasid_t;
221
222 #ifdef CONFIG_IOMMU_API
223
224 /**
225 * struct iommu_iotlb_gather - Range information for a pending IOTLB flush
226 *
227 * @start: IOVA representing the start of the range to be flushed
228 * @end: IOVA representing the end of the range to be flushed (inclusive)
229 * @pgsize: The interval at which to perform the flush
230 * @freelist: Removed pages to free after sync
231 * @queued: Indicates that the flush will be queued
232 *
233 * This structure is intended to be updated by multiple calls to the
234 * ->unmap() function in struct iommu_ops before eventually being passed
235 * into ->iotlb_sync(). Drivers can add pages to @freelist to be freed after
236 * ->iotlb_sync() or ->iotlb_flush_all() have cleared all cached references to
237 * them. @queued is set to indicate when ->iotlb_flush_all() will be called
238 * later instead of ->iotlb_sync(), so drivers may optimise accordingly.
239 */
240 struct iommu_iotlb_gather {
241 unsigned long start;
242 unsigned long end;
243 size_t pgsize;
244 struct list_head freelist;
245 bool queued;
246 };
247
248 /**
249 * struct iommu_dirty_bitmap - Dirty IOVA bitmap state
250 * @bitmap: IOVA bitmap
251 * @gather: Range information for a pending IOTLB flush
252 */
253 struct iommu_dirty_bitmap {
254 struct iova_bitmap *bitmap;
255 struct iommu_iotlb_gather *gather;
256 };
257
258 /* Read but do not clear any dirty bits */
259 #define IOMMU_DIRTY_NO_CLEAR (1 << 0)
260
261 /**
262 * struct iommu_dirty_ops - domain specific dirty tracking operations
263 * @set_dirty_tracking: Enable or Disable dirty tracking on the iommu domain
264 * @read_and_clear_dirty: Walk IOMMU page tables for dirtied PTEs marshalled
265 * into a bitmap, with a bit represented as a page.
266 * Reads the dirty PTE bits and clears it from IO
267 * pagetables.
268 */
269 struct iommu_dirty_ops {
270 int (*set_dirty_tracking)(struct iommu_domain *domain, bool enabled);
271 int (*read_and_clear_dirty)(struct iommu_domain *domain,
272 unsigned long iova, size_t size,
273 unsigned long flags,
274 struct iommu_dirty_bitmap *dirty);
275 };
276
277 /**
278 * struct iommu_user_data - iommu driver specific user space data info
279 * @type: The data type of the user buffer
280 * @uptr: Pointer to the user buffer for copy_from_user()
281 * @len: The length of the user buffer in bytes
282 *
283 * A user space data is an uAPI that is defined in include/uapi/linux/iommufd.h
284 * @type, @uptr and @len should be just copied from an iommufd core uAPI struct.
285 */
286 struct iommu_user_data {
287 unsigned int type;
288 void __user *uptr;
289 size_t len;
290 };
291
292 /**
293 * struct iommu_user_data_array - iommu driver specific user space data array
294 * @type: The data type of all the entries in the user buffer array
295 * @uptr: Pointer to the user buffer array
296 * @entry_len: The fixed-width length of an entry in the array, in bytes
297 * @entry_num: The number of total entries in the array
298 *
299 * The user buffer includes an array of requests with format defined in
300 * include/uapi/linux/iommufd.h
301 */
302 struct iommu_user_data_array {
303 unsigned int type;
304 void __user *uptr;
305 size_t entry_len;
306 u32 entry_num;
307 };
308
309 /**
310 * __iommu_copy_struct_from_user - Copy iommu driver specific user space data
311 * @dst_data: Pointer to an iommu driver specific user data that is defined in
312 * include/uapi/linux/iommufd.h
313 * @src_data: Pointer to a struct iommu_user_data for user space data info
314 * @data_type: The data type of the @dst_data. Must match with @src_data.type
315 * @data_len: Length of current user data structure, i.e. sizeof(struct _dst)
316 * @min_len: Initial length of user data structure for backward compatibility.
317 * This should be offsetofend using the last member in the user data
318 * struct that was initially added to include/uapi/linux/iommufd.h
319 */
__iommu_copy_struct_from_user(void * dst_data,const struct iommu_user_data * src_data,unsigned int data_type,size_t data_len,size_t min_len)320 static inline int __iommu_copy_struct_from_user(
321 void *dst_data, const struct iommu_user_data *src_data,
322 unsigned int data_type, size_t data_len, size_t min_len)
323 {
324 if (src_data->type != data_type)
325 return -EINVAL;
326 if (WARN_ON(!dst_data || !src_data))
327 return -EINVAL;
328 if (src_data->len < min_len || data_len < src_data->len)
329 return -EINVAL;
330 return copy_struct_from_user(dst_data, data_len, src_data->uptr,
331 src_data->len);
332 }
333
334 /**
335 * iommu_copy_struct_from_user - Copy iommu driver specific user space data
336 * @kdst: Pointer to an iommu driver specific user data that is defined in
337 * include/uapi/linux/iommufd.h
338 * @user_data: Pointer to a struct iommu_user_data for user space data info
339 * @data_type: The data type of the @kdst. Must match with @user_data->type
340 * @min_last: The last memember of the data structure @kdst points in the
341 * initial version.
342 * Return 0 for success, otherwise -error.
343 */
344 #define iommu_copy_struct_from_user(kdst, user_data, data_type, min_last) \
345 __iommu_copy_struct_from_user(kdst, user_data, data_type, \
346 sizeof(*kdst), \
347 offsetofend(typeof(*kdst), min_last))
348
349 /**
350 * __iommu_copy_struct_from_user_array - Copy iommu driver specific user space
351 * data from an iommu_user_data_array
352 * @dst_data: Pointer to an iommu driver specific user data that is defined in
353 * include/uapi/linux/iommufd.h
354 * @src_array: Pointer to a struct iommu_user_data_array for a user space array
355 * @data_type: The data type of the @dst_data. Must match with @src_array.type
356 * @index: Index to the location in the array to copy user data from
357 * @data_len: Length of current user data structure, i.e. sizeof(struct _dst)
358 * @min_len: Initial length of user data structure for backward compatibility.
359 * This should be offsetofend using the last member in the user data
360 * struct that was initially added to include/uapi/linux/iommufd.h
361 */
__iommu_copy_struct_from_user_array(void * dst_data,const struct iommu_user_data_array * src_array,unsigned int data_type,unsigned int index,size_t data_len,size_t min_len)362 static inline int __iommu_copy_struct_from_user_array(
363 void *dst_data, const struct iommu_user_data_array *src_array,
364 unsigned int data_type, unsigned int index, size_t data_len,
365 size_t min_len)
366 {
367 struct iommu_user_data src_data;
368
369 if (WARN_ON(!src_array || index >= src_array->entry_num))
370 return -EINVAL;
371 if (!src_array->entry_num)
372 return -EINVAL;
373 src_data.uptr = src_array->uptr + src_array->entry_len * index;
374 src_data.len = src_array->entry_len;
375 src_data.type = src_array->type;
376
377 return __iommu_copy_struct_from_user(dst_data, &src_data, data_type,
378 data_len, min_len);
379 }
380
381 /**
382 * iommu_copy_struct_from_user_array - Copy iommu driver specific user space
383 * data from an iommu_user_data_array
384 * @kdst: Pointer to an iommu driver specific user data that is defined in
385 * include/uapi/linux/iommufd.h
386 * @user_array: Pointer to a struct iommu_user_data_array for a user space
387 * array
388 * @data_type: The data type of the @kdst. Must match with @user_array->type
389 * @index: Index to the location in the array to copy user data from
390 * @min_last: The last member of the data structure @kdst points in the
391 * initial version.
392 * Return 0 for success, otherwise -error.
393 */
394 #define iommu_copy_struct_from_user_array(kdst, user_array, data_type, index, \
395 min_last) \
396 __iommu_copy_struct_from_user_array( \
397 kdst, user_array, data_type, index, sizeof(*(kdst)), \
398 offsetofend(typeof(*(kdst)), min_last))
399
400 /**
401 * struct iommu_ops - iommu ops and capabilities
402 * @capable: check capability
403 * @hw_info: report iommu hardware information. The data buffer returned by this
404 * op is allocated in the iommu driver and freed by the caller after
405 * use. The information type is one of enum iommu_hw_info_type defined
406 * in include/uapi/linux/iommufd.h.
407 * @domain_alloc: allocate and return an iommu domain if success. Otherwise
408 * NULL is returned. The domain is not fully initialized until
409 * the caller iommu_domain_alloc() returns.
410 * @domain_alloc_user: Allocate an iommu domain corresponding to the input
411 * parameters as defined in include/uapi/linux/iommufd.h.
412 * Unlike @domain_alloc, it is called only by IOMMUFD and
413 * must fully initialize the new domain before return.
414 * Upon success, if the @user_data is valid and the @parent
415 * points to a kernel-managed domain, the new domain must be
416 * IOMMU_DOMAIN_NESTED type; otherwise, the @parent must be
417 * NULL while the @user_data can be optionally provided, the
418 * new domain must support __IOMMU_DOMAIN_PAGING.
419 * Upon failure, ERR_PTR must be returned.
420 * @domain_alloc_paging: Allocate an iommu_domain that can be used for
421 * UNMANAGED, DMA, and DMA_FQ domain types.
422 * @probe_device: Add device to iommu driver handling
423 * @release_device: Remove device from iommu driver handling
424 * @probe_finalize: Do final setup work after the device is added to an IOMMU
425 * group and attached to the groups domain
426 * @device_group: find iommu group for a particular device
427 * @get_resv_regions: Request list of reserved regions for a device
428 * @of_xlate: add OF master IDs to iommu grouping
429 * @is_attach_deferred: Check if domain attach should be deferred from iommu
430 * driver init to device driver init (default no)
431 * @dev_enable/disable_feat: per device entries to enable/disable
432 * iommu specific features.
433 * @page_response: handle page request response
434 * @def_domain_type: device default domain type, return value:
435 * - IOMMU_DOMAIN_IDENTITY: must use an identity domain
436 * - IOMMU_DOMAIN_DMA: must use a dma domain
437 * - 0: use the default setting
438 * @default_domain_ops: the default ops for domains
439 * @remove_dev_pasid: Remove any translation configurations of a specific
440 * pasid, so that any DMA transactions with this pasid
441 * will be blocked by the hardware.
442 * @pgsize_bitmap: bitmap of all possible supported page sizes
443 * @owner: Driver module providing these ops
444 * @identity_domain: An always available, always attachable identity
445 * translation.
446 * @blocked_domain: An always available, always attachable blocking
447 * translation.
448 * @default_domain: If not NULL this will always be set as the default domain.
449 * This should be an IDENTITY/BLOCKED/PLATFORM domain.
450 * Do not use in new drivers.
451 */
452 struct iommu_ops {
453 bool (*capable)(struct device *dev, enum iommu_cap);
454 void *(*hw_info)(struct device *dev, u32 *length, u32 *type);
455
456 /* Domain allocation and freeing by the iommu driver */
457 struct iommu_domain *(*domain_alloc)(unsigned iommu_domain_type);
458 struct iommu_domain *(*domain_alloc_user)(
459 struct device *dev, u32 flags, struct iommu_domain *parent,
460 const struct iommu_user_data *user_data);
461 struct iommu_domain *(*domain_alloc_paging)(struct device *dev);
462
463 struct iommu_device *(*probe_device)(struct device *dev);
464 void (*release_device)(struct device *dev);
465 void (*probe_finalize)(struct device *dev);
466 struct iommu_group *(*device_group)(struct device *dev);
467
468 /* Request/Free a list of reserved regions for a device */
469 void (*get_resv_regions)(struct device *dev, struct list_head *list);
470
471 int (*of_xlate)(struct device *dev, struct of_phandle_args *args);
472 bool (*is_attach_deferred)(struct device *dev);
473
474 /* Per device IOMMU features */
475 int (*dev_enable_feat)(struct device *dev, enum iommu_dev_features f);
476 int (*dev_disable_feat)(struct device *dev, enum iommu_dev_features f);
477
478 int (*page_response)(struct device *dev,
479 struct iommu_fault_event *evt,
480 struct iommu_page_response *msg);
481
482 int (*def_domain_type)(struct device *dev);
483 void (*remove_dev_pasid)(struct device *dev, ioasid_t pasid);
484
485 const struct iommu_domain_ops *default_domain_ops;
486 unsigned long pgsize_bitmap;
487 struct module *owner;
488 struct iommu_domain *identity_domain;
489 struct iommu_domain *blocked_domain;
490 struct iommu_domain *default_domain;
491 };
492
493 /**
494 * struct iommu_domain_ops - domain specific operations
495 * @attach_dev: attach an iommu domain to a device
496 * Return:
497 * * 0 - success
498 * * EINVAL - can indicate that device and domain are incompatible due to
499 * some previous configuration of the domain, in which case the
500 * driver shouldn't log an error, since it is legitimate for a
501 * caller to test reuse of existing domains. Otherwise, it may
502 * still represent some other fundamental problem
503 * * ENOMEM - out of memory
504 * * ENOSPC - non-ENOMEM type of resource allocation failures
505 * * EBUSY - device is attached to a domain and cannot be changed
506 * * ENODEV - device specific errors, not able to be attached
507 * * <others> - treated as ENODEV by the caller. Use is discouraged
508 * @set_dev_pasid: set an iommu domain to a pasid of device
509 * @map_pages: map a physically contiguous set of pages of the same size to
510 * an iommu domain.
511 * @unmap_pages: unmap a number of pages of the same size from an iommu domain
512 * @flush_iotlb_all: Synchronously flush all hardware TLBs for this domain
513 * @iotlb_sync_map: Sync mappings created recently using @map to the hardware
514 * @iotlb_sync: Flush all queued ranges from the hardware TLBs and empty flush
515 * queue
516 * @cache_invalidate_user: Flush hardware cache for user space IO page table.
517 * The @domain must be IOMMU_DOMAIN_NESTED. The @array
518 * passes in the cache invalidation requests, in form
519 * of a driver data structure. The driver must update
520 * array->entry_num to report the number of handled
521 * invalidation requests. The driver data structure
522 * must be defined in include/uapi/linux/iommufd.h
523 * @iova_to_phys: translate iova to physical address
524 * @enforce_cache_coherency: Prevent any kind of DMA from bypassing IOMMU_CACHE,
525 * including no-snoop TLPs on PCIe or other platform
526 * specific mechanisms.
527 * @enable_nesting: Enable nesting
528 * @set_pgtable_quirks: Set io page table quirks (IO_PGTABLE_QUIRK_*)
529 * @free: Release the domain after use.
530 */
531 struct iommu_domain_ops {
532 int (*attach_dev)(struct iommu_domain *domain, struct device *dev);
533 int (*set_dev_pasid)(struct iommu_domain *domain, struct device *dev,
534 ioasid_t pasid);
535
536 int (*map_pages)(struct iommu_domain *domain, unsigned long iova,
537 phys_addr_t paddr, size_t pgsize, size_t pgcount,
538 int prot, gfp_t gfp, size_t *mapped);
539 size_t (*unmap_pages)(struct iommu_domain *domain, unsigned long iova,
540 size_t pgsize, size_t pgcount,
541 struct iommu_iotlb_gather *iotlb_gather);
542
543 void (*flush_iotlb_all)(struct iommu_domain *domain);
544 int (*iotlb_sync_map)(struct iommu_domain *domain, unsigned long iova,
545 size_t size);
546 void (*iotlb_sync)(struct iommu_domain *domain,
547 struct iommu_iotlb_gather *iotlb_gather);
548 int (*cache_invalidate_user)(struct iommu_domain *domain,
549 struct iommu_user_data_array *array);
550
551 phys_addr_t (*iova_to_phys)(struct iommu_domain *domain,
552 dma_addr_t iova);
553
554 bool (*enforce_cache_coherency)(struct iommu_domain *domain);
555 int (*enable_nesting)(struct iommu_domain *domain);
556 int (*set_pgtable_quirks)(struct iommu_domain *domain,
557 unsigned long quirks);
558
559 void (*free)(struct iommu_domain *domain);
560 };
561
562 /**
563 * struct iommu_device - IOMMU core representation of one IOMMU hardware
564 * instance
565 * @list: Used by the iommu-core to keep a list of registered iommus
566 * @ops: iommu-ops for talking to this iommu
567 * @dev: struct device for sysfs handling
568 * @singleton_group: Used internally for drivers that have only one group
569 * @max_pasids: number of supported PASIDs
570 */
571 struct iommu_device {
572 struct list_head list;
573 const struct iommu_ops *ops;
574 struct fwnode_handle *fwnode;
575 struct device *dev;
576 struct iommu_group *singleton_group;
577 u32 max_pasids;
578 };
579
580 /**
581 * struct iommu_fault_event - Generic fault event
582 *
583 * Can represent recoverable faults such as a page requests or
584 * unrecoverable faults such as DMA or IRQ remapping faults.
585 *
586 * @fault: fault descriptor
587 * @list: pending fault event list, used for tracking responses
588 */
589 struct iommu_fault_event {
590 struct iommu_fault fault;
591 struct list_head list;
592 };
593
594 /**
595 * struct iommu_fault_param - per-device IOMMU fault data
596 * @handler: Callback function to handle IOMMU faults at device level
597 * @data: handler private data
598 * @faults: holds the pending faults which needs response
599 * @lock: protect pending faults list
600 */
601 struct iommu_fault_param {
602 iommu_dev_fault_handler_t handler;
603 void *data;
604 struct list_head faults;
605 struct mutex lock;
606 };
607
608 /**
609 * struct dev_iommu - Collection of per-device IOMMU data
610 *
611 * @fault_param: IOMMU detected device fault reporting data
612 * @iopf_param: I/O Page Fault queue and data
613 * @fwspec: IOMMU fwspec data
614 * @iommu_dev: IOMMU device this device is linked to
615 * @priv: IOMMU Driver private data
616 * @max_pasids: number of PASIDs this device can consume
617 * @attach_deferred: the dma domain attachment is deferred
618 * @pci_32bit_workaround: Limit DMA allocations to 32-bit IOVAs
619 * @require_direct: device requires IOMMU_RESV_DIRECT regions
620 * @shadow_on_flush: IOTLB flushes are used to sync shadow tables
621 *
622 * TODO: migrate other per device data pointers under iommu_dev_data, e.g.
623 * struct iommu_group *iommu_group;
624 */
625 struct dev_iommu {
626 struct mutex lock;
627 struct iommu_fault_param *fault_param;
628 struct iopf_device_param *iopf_param;
629 struct iommu_fwspec *fwspec;
630 struct iommu_device *iommu_dev;
631 void *priv;
632 u32 max_pasids;
633 u32 attach_deferred:1;
634 u32 pci_32bit_workaround:1;
635 u32 require_direct:1;
636 u32 shadow_on_flush:1;
637 };
638
639 int iommu_device_register(struct iommu_device *iommu,
640 const struct iommu_ops *ops,
641 struct device *hwdev);
642 void iommu_device_unregister(struct iommu_device *iommu);
643 int iommu_device_sysfs_add(struct iommu_device *iommu,
644 struct device *parent,
645 const struct attribute_group **groups,
646 const char *fmt, ...) __printf(4, 5);
647 void iommu_device_sysfs_remove(struct iommu_device *iommu);
648 int iommu_device_link(struct iommu_device *iommu, struct device *link);
649 void iommu_device_unlink(struct iommu_device *iommu, struct device *link);
650 int iommu_deferred_attach(struct device *dev, struct iommu_domain *domain);
651
dev_to_iommu_device(struct device * dev)652 static inline struct iommu_device *dev_to_iommu_device(struct device *dev)
653 {
654 return (struct iommu_device *)dev_get_drvdata(dev);
655 }
656
iommu_iotlb_gather_init(struct iommu_iotlb_gather * gather)657 static inline void iommu_iotlb_gather_init(struct iommu_iotlb_gather *gather)
658 {
659 *gather = (struct iommu_iotlb_gather) {
660 .start = ULONG_MAX,
661 .freelist = LIST_HEAD_INIT(gather->freelist),
662 };
663 }
664
665 extern int bus_iommu_probe(const struct bus_type *bus);
666 extern bool iommu_present(const struct bus_type *bus);
667 extern bool device_iommu_capable(struct device *dev, enum iommu_cap cap);
668 extern bool iommu_group_has_isolated_msi(struct iommu_group *group);
669 extern struct iommu_domain *iommu_domain_alloc(const struct bus_type *bus);
670 extern void iommu_domain_free(struct iommu_domain *domain);
671 extern int iommu_attach_device(struct iommu_domain *domain,
672 struct device *dev);
673 extern void iommu_detach_device(struct iommu_domain *domain,
674 struct device *dev);
675 extern int iommu_sva_unbind_gpasid(struct iommu_domain *domain,
676 struct device *dev, ioasid_t pasid);
677 extern struct iommu_domain *iommu_get_domain_for_dev(struct device *dev);
678 extern struct iommu_domain *iommu_get_dma_domain(struct device *dev);
679 extern int iommu_map(struct iommu_domain *domain, unsigned long iova,
680 phys_addr_t paddr, size_t size, int prot, gfp_t gfp);
681 extern size_t iommu_unmap(struct iommu_domain *domain, unsigned long iova,
682 size_t size);
683 extern size_t iommu_unmap_fast(struct iommu_domain *domain,
684 unsigned long iova, size_t size,
685 struct iommu_iotlb_gather *iotlb_gather);
686 extern ssize_t iommu_map_sg(struct iommu_domain *domain, unsigned long iova,
687 struct scatterlist *sg, unsigned int nents,
688 int prot, gfp_t gfp);
689 extern phys_addr_t iommu_iova_to_phys(struct iommu_domain *domain, dma_addr_t iova);
690 extern void iommu_set_fault_handler(struct iommu_domain *domain,
691 iommu_fault_handler_t handler, void *token);
692
693 extern void iommu_get_resv_regions(struct device *dev, struct list_head *list);
694 extern void iommu_put_resv_regions(struct device *dev, struct list_head *list);
695 extern void iommu_set_default_passthrough(bool cmd_line);
696 extern void iommu_set_default_translated(bool cmd_line);
697 extern bool iommu_default_passthrough(void);
698 extern struct iommu_resv_region *
699 iommu_alloc_resv_region(phys_addr_t start, size_t length, int prot,
700 enum iommu_resv_type type, gfp_t gfp);
701 extern int iommu_get_group_resv_regions(struct iommu_group *group,
702 struct list_head *head);
703
704 extern int iommu_attach_group(struct iommu_domain *domain,
705 struct iommu_group *group);
706 extern void iommu_detach_group(struct iommu_domain *domain,
707 struct iommu_group *group);
708 extern struct iommu_group *iommu_group_alloc(void);
709 extern void *iommu_group_get_iommudata(struct iommu_group *group);
710 extern void iommu_group_set_iommudata(struct iommu_group *group,
711 void *iommu_data,
712 void (*release)(void *iommu_data));
713 extern int iommu_group_set_name(struct iommu_group *group, const char *name);
714 extern int iommu_group_add_device(struct iommu_group *group,
715 struct device *dev);
716 extern void iommu_group_remove_device(struct device *dev);
717 extern int iommu_group_for_each_dev(struct iommu_group *group, void *data,
718 int (*fn)(struct device *, void *));
719 extern struct iommu_group *iommu_group_get(struct device *dev);
720 extern struct iommu_group *iommu_group_ref_get(struct iommu_group *group);
721 extern void iommu_group_put(struct iommu_group *group);
722 extern int iommu_register_device_fault_handler(struct device *dev,
723 iommu_dev_fault_handler_t handler,
724 void *data);
725
726 extern int iommu_unregister_device_fault_handler(struct device *dev);
727
728 extern int iommu_report_device_fault(struct device *dev,
729 struct iommu_fault_event *evt);
730 extern int iommu_page_response(struct device *dev,
731 struct iommu_page_response *msg);
732
733 extern int iommu_group_id(struct iommu_group *group);
734 extern struct iommu_domain *iommu_group_default_domain(struct iommu_group *);
735
736 int iommu_enable_nesting(struct iommu_domain *domain);
737 int iommu_set_pgtable_quirks(struct iommu_domain *domain,
738 unsigned long quirks);
739
740 void iommu_set_dma_strict(void);
741
742 extern int report_iommu_fault(struct iommu_domain *domain, struct device *dev,
743 unsigned long iova, int flags);
744
iommu_flush_iotlb_all(struct iommu_domain * domain)745 static inline void iommu_flush_iotlb_all(struct iommu_domain *domain)
746 {
747 if (domain->ops->flush_iotlb_all)
748 domain->ops->flush_iotlb_all(domain);
749 }
750
iommu_iotlb_sync(struct iommu_domain * domain,struct iommu_iotlb_gather * iotlb_gather)751 static inline void iommu_iotlb_sync(struct iommu_domain *domain,
752 struct iommu_iotlb_gather *iotlb_gather)
753 {
754 if (domain->ops->iotlb_sync)
755 domain->ops->iotlb_sync(domain, iotlb_gather);
756
757 iommu_iotlb_gather_init(iotlb_gather);
758 }
759
760 /**
761 * iommu_iotlb_gather_is_disjoint - Checks whether a new range is disjoint
762 *
763 * @gather: TLB gather data
764 * @iova: start of page to invalidate
765 * @size: size of page to invalidate
766 *
767 * Helper for IOMMU drivers to check whether a new range and the gathered range
768 * are disjoint. For many IOMMUs, flushing the IOMMU in this case is better
769 * than merging the two, which might lead to unnecessary invalidations.
770 */
771 static inline
iommu_iotlb_gather_is_disjoint(struct iommu_iotlb_gather * gather,unsigned long iova,size_t size)772 bool iommu_iotlb_gather_is_disjoint(struct iommu_iotlb_gather *gather,
773 unsigned long iova, size_t size)
774 {
775 unsigned long start = iova, end = start + size - 1;
776
777 return gather->end != 0 &&
778 (end + 1 < gather->start || start > gather->end + 1);
779 }
780
781
782 /**
783 * iommu_iotlb_gather_add_range - Gather for address-based TLB invalidation
784 * @gather: TLB gather data
785 * @iova: start of page to invalidate
786 * @size: size of page to invalidate
787 *
788 * Helper for IOMMU drivers to build arbitrarily-sized invalidation commands
789 * where only the address range matters, and simply minimising intermediate
790 * syncs is preferred.
791 */
iommu_iotlb_gather_add_range(struct iommu_iotlb_gather * gather,unsigned long iova,size_t size)792 static inline void iommu_iotlb_gather_add_range(struct iommu_iotlb_gather *gather,
793 unsigned long iova, size_t size)
794 {
795 unsigned long end = iova + size - 1;
796
797 if (gather->start > iova)
798 gather->start = iova;
799 if (gather->end < end)
800 gather->end = end;
801 }
802
803 /**
804 * iommu_iotlb_gather_add_page - Gather for page-based TLB invalidation
805 * @domain: IOMMU domain to be invalidated
806 * @gather: TLB gather data
807 * @iova: start of page to invalidate
808 * @size: size of page to invalidate
809 *
810 * Helper for IOMMU drivers to build invalidation commands based on individual
811 * pages, or with page size/table level hints which cannot be gathered if they
812 * differ.
813 */
iommu_iotlb_gather_add_page(struct iommu_domain * domain,struct iommu_iotlb_gather * gather,unsigned long iova,size_t size)814 static inline void iommu_iotlb_gather_add_page(struct iommu_domain *domain,
815 struct iommu_iotlb_gather *gather,
816 unsigned long iova, size_t size)
817 {
818 /*
819 * If the new page is disjoint from the current range or is mapped at
820 * a different granularity, then sync the TLB so that the gather
821 * structure can be rewritten.
822 */
823 if ((gather->pgsize && gather->pgsize != size) ||
824 iommu_iotlb_gather_is_disjoint(gather, iova, size))
825 iommu_iotlb_sync(domain, gather);
826
827 gather->pgsize = size;
828 iommu_iotlb_gather_add_range(gather, iova, size);
829 }
830
iommu_iotlb_gather_queued(struct iommu_iotlb_gather * gather)831 static inline bool iommu_iotlb_gather_queued(struct iommu_iotlb_gather *gather)
832 {
833 return gather && gather->queued;
834 }
835
iommu_dirty_bitmap_init(struct iommu_dirty_bitmap * dirty,struct iova_bitmap * bitmap,struct iommu_iotlb_gather * gather)836 static inline void iommu_dirty_bitmap_init(struct iommu_dirty_bitmap *dirty,
837 struct iova_bitmap *bitmap,
838 struct iommu_iotlb_gather *gather)
839 {
840 if (gather)
841 iommu_iotlb_gather_init(gather);
842
843 dirty->bitmap = bitmap;
844 dirty->gather = gather;
845 }
846
iommu_dirty_bitmap_record(struct iommu_dirty_bitmap * dirty,unsigned long iova,unsigned long length)847 static inline void iommu_dirty_bitmap_record(struct iommu_dirty_bitmap *dirty,
848 unsigned long iova,
849 unsigned long length)
850 {
851 if (dirty->bitmap)
852 iova_bitmap_set(dirty->bitmap, iova, length);
853
854 if (dirty->gather)
855 iommu_iotlb_gather_add_range(dirty->gather, iova, length);
856 }
857
858 /* PCI device grouping function */
859 extern struct iommu_group *pci_device_group(struct device *dev);
860 /* Generic device grouping function */
861 extern struct iommu_group *generic_device_group(struct device *dev);
862 /* FSL-MC device grouping function */
863 struct iommu_group *fsl_mc_device_group(struct device *dev);
864 extern struct iommu_group *generic_single_device_group(struct device *dev);
865
866 /**
867 * struct iommu_fwspec - per-device IOMMU instance data
868 * @ops: ops for this device's IOMMU
869 * @iommu_fwnode: firmware handle for this device's IOMMU
870 * @flags: IOMMU_FWSPEC_* flags
871 * @num_ids: number of associated device IDs
872 * @ids: IDs which this device may present to the IOMMU
873 *
874 * Note that the IDs (and any other information, really) stored in this structure should be
875 * considered private to the IOMMU device driver and are not to be used directly by IOMMU
876 * consumers.
877 */
878 struct iommu_fwspec {
879 const struct iommu_ops *ops;
880 struct fwnode_handle *iommu_fwnode;
881 u32 flags;
882 unsigned int num_ids;
883 u32 ids[];
884 };
885
886 /* ATS is supported */
887 #define IOMMU_FWSPEC_PCI_RC_ATS (1 << 0)
888
889 /**
890 * struct iommu_sva - handle to a device-mm bond
891 */
892 struct iommu_sva {
893 struct device *dev;
894 struct iommu_domain *domain;
895 struct list_head handle_item;
896 refcount_t users;
897 };
898
899 struct iommu_mm_data {
900 u32 pasid;
901 struct list_head sva_domains;
902 struct list_head sva_handles;
903 };
904
905 int iommu_fwspec_init(struct device *dev, struct fwnode_handle *iommu_fwnode,
906 const struct iommu_ops *ops);
907 void iommu_fwspec_free(struct device *dev);
908 int iommu_fwspec_add_ids(struct device *dev, u32 *ids, int num_ids);
909 const struct iommu_ops *iommu_ops_from_fwnode(struct fwnode_handle *fwnode);
910
dev_iommu_fwspec_get(struct device * dev)911 static inline struct iommu_fwspec *dev_iommu_fwspec_get(struct device *dev)
912 {
913 if (dev->iommu)
914 return dev->iommu->fwspec;
915 else
916 return NULL;
917 }
918
dev_iommu_fwspec_set(struct device * dev,struct iommu_fwspec * fwspec)919 static inline void dev_iommu_fwspec_set(struct device *dev,
920 struct iommu_fwspec *fwspec)
921 {
922 dev->iommu->fwspec = fwspec;
923 }
924
dev_iommu_priv_get(struct device * dev)925 static inline void *dev_iommu_priv_get(struct device *dev)
926 {
927 if (dev->iommu)
928 return dev->iommu->priv;
929 else
930 return NULL;
931 }
932
933 void dev_iommu_priv_set(struct device *dev, void *priv);
934
935 extern struct mutex iommu_probe_device_lock;
936 int iommu_probe_device(struct device *dev);
937
938 int iommu_dev_enable_feature(struct device *dev, enum iommu_dev_features f);
939 int iommu_dev_disable_feature(struct device *dev, enum iommu_dev_features f);
940
941 int iommu_device_use_default_domain(struct device *dev);
942 void iommu_device_unuse_default_domain(struct device *dev);
943
944 int iommu_group_claim_dma_owner(struct iommu_group *group, void *owner);
945 void iommu_group_release_dma_owner(struct iommu_group *group);
946 bool iommu_group_dma_owner_claimed(struct iommu_group *group);
947
948 int iommu_device_claim_dma_owner(struct device *dev, void *owner);
949 void iommu_device_release_dma_owner(struct device *dev);
950
951 struct iommu_domain *iommu_sva_domain_alloc(struct device *dev,
952 struct mm_struct *mm);
953 int iommu_attach_device_pasid(struct iommu_domain *domain,
954 struct device *dev, ioasid_t pasid);
955 void iommu_detach_device_pasid(struct iommu_domain *domain,
956 struct device *dev, ioasid_t pasid);
957 struct iommu_domain *
958 iommu_get_domain_for_dev_pasid(struct device *dev, ioasid_t pasid,
959 unsigned int type);
960 ioasid_t iommu_alloc_global_pasid(struct device *dev);
961 void iommu_free_global_pasid(ioasid_t pasid);
962 #else /* CONFIG_IOMMU_API */
963
964 struct iommu_ops {};
965 struct iommu_group {};
966 struct iommu_fwspec {};
967 struct iommu_device {};
968 struct iommu_fault_param {};
969 struct iommu_iotlb_gather {};
970 struct iommu_dirty_bitmap {};
971 struct iommu_dirty_ops {};
972
iommu_present(const struct bus_type * bus)973 static inline bool iommu_present(const struct bus_type *bus)
974 {
975 return false;
976 }
977
device_iommu_capable(struct device * dev,enum iommu_cap cap)978 static inline bool device_iommu_capable(struct device *dev, enum iommu_cap cap)
979 {
980 return false;
981 }
982
iommu_domain_alloc(const struct bus_type * bus)983 static inline struct iommu_domain *iommu_domain_alloc(const struct bus_type *bus)
984 {
985 return NULL;
986 }
987
iommu_domain_free(struct iommu_domain * domain)988 static inline void iommu_domain_free(struct iommu_domain *domain)
989 {
990 }
991
iommu_attach_device(struct iommu_domain * domain,struct device * dev)992 static inline int iommu_attach_device(struct iommu_domain *domain,
993 struct device *dev)
994 {
995 return -ENODEV;
996 }
997
iommu_detach_device(struct iommu_domain * domain,struct device * dev)998 static inline void iommu_detach_device(struct iommu_domain *domain,
999 struct device *dev)
1000 {
1001 }
1002
iommu_get_domain_for_dev(struct device * dev)1003 static inline struct iommu_domain *iommu_get_domain_for_dev(struct device *dev)
1004 {
1005 return NULL;
1006 }
1007
iommu_map(struct iommu_domain * domain,unsigned long iova,phys_addr_t paddr,size_t size,int prot,gfp_t gfp)1008 static inline int iommu_map(struct iommu_domain *domain, unsigned long iova,
1009 phys_addr_t paddr, size_t size, int prot, gfp_t gfp)
1010 {
1011 return -ENODEV;
1012 }
1013
iommu_unmap(struct iommu_domain * domain,unsigned long iova,size_t size)1014 static inline size_t iommu_unmap(struct iommu_domain *domain,
1015 unsigned long iova, size_t size)
1016 {
1017 return 0;
1018 }
1019
iommu_unmap_fast(struct iommu_domain * domain,unsigned long iova,int gfp_order,struct iommu_iotlb_gather * iotlb_gather)1020 static inline size_t iommu_unmap_fast(struct iommu_domain *domain,
1021 unsigned long iova, int gfp_order,
1022 struct iommu_iotlb_gather *iotlb_gather)
1023 {
1024 return 0;
1025 }
1026
iommu_map_sg(struct iommu_domain * domain,unsigned long iova,struct scatterlist * sg,unsigned int nents,int prot,gfp_t gfp)1027 static inline ssize_t iommu_map_sg(struct iommu_domain *domain,
1028 unsigned long iova, struct scatterlist *sg,
1029 unsigned int nents, int prot, gfp_t gfp)
1030 {
1031 return -ENODEV;
1032 }
1033
iommu_flush_iotlb_all(struct iommu_domain * domain)1034 static inline void iommu_flush_iotlb_all(struct iommu_domain *domain)
1035 {
1036 }
1037
iommu_iotlb_sync(struct iommu_domain * domain,struct iommu_iotlb_gather * iotlb_gather)1038 static inline void iommu_iotlb_sync(struct iommu_domain *domain,
1039 struct iommu_iotlb_gather *iotlb_gather)
1040 {
1041 }
1042
iommu_iova_to_phys(struct iommu_domain * domain,dma_addr_t iova)1043 static inline phys_addr_t iommu_iova_to_phys(struct iommu_domain *domain, dma_addr_t iova)
1044 {
1045 return 0;
1046 }
1047
iommu_set_fault_handler(struct iommu_domain * domain,iommu_fault_handler_t handler,void * token)1048 static inline void iommu_set_fault_handler(struct iommu_domain *domain,
1049 iommu_fault_handler_t handler, void *token)
1050 {
1051 }
1052
iommu_get_resv_regions(struct device * dev,struct list_head * list)1053 static inline void iommu_get_resv_regions(struct device *dev,
1054 struct list_head *list)
1055 {
1056 }
1057
iommu_put_resv_regions(struct device * dev,struct list_head * list)1058 static inline void iommu_put_resv_regions(struct device *dev,
1059 struct list_head *list)
1060 {
1061 }
1062
iommu_get_group_resv_regions(struct iommu_group * group,struct list_head * head)1063 static inline int iommu_get_group_resv_regions(struct iommu_group *group,
1064 struct list_head *head)
1065 {
1066 return -ENODEV;
1067 }
1068
iommu_set_default_passthrough(bool cmd_line)1069 static inline void iommu_set_default_passthrough(bool cmd_line)
1070 {
1071 }
1072
iommu_set_default_translated(bool cmd_line)1073 static inline void iommu_set_default_translated(bool cmd_line)
1074 {
1075 }
1076
iommu_default_passthrough(void)1077 static inline bool iommu_default_passthrough(void)
1078 {
1079 return true;
1080 }
1081
iommu_attach_group(struct iommu_domain * domain,struct iommu_group * group)1082 static inline int iommu_attach_group(struct iommu_domain *domain,
1083 struct iommu_group *group)
1084 {
1085 return -ENODEV;
1086 }
1087
iommu_detach_group(struct iommu_domain * domain,struct iommu_group * group)1088 static inline void iommu_detach_group(struct iommu_domain *domain,
1089 struct iommu_group *group)
1090 {
1091 }
1092
iommu_group_alloc(void)1093 static inline struct iommu_group *iommu_group_alloc(void)
1094 {
1095 return ERR_PTR(-ENODEV);
1096 }
1097
iommu_group_get_iommudata(struct iommu_group * group)1098 static inline void *iommu_group_get_iommudata(struct iommu_group *group)
1099 {
1100 return NULL;
1101 }
1102
iommu_group_set_iommudata(struct iommu_group * group,void * iommu_data,void (* release)(void * iommu_data))1103 static inline void iommu_group_set_iommudata(struct iommu_group *group,
1104 void *iommu_data,
1105 void (*release)(void *iommu_data))
1106 {
1107 }
1108
iommu_group_set_name(struct iommu_group * group,const char * name)1109 static inline int iommu_group_set_name(struct iommu_group *group,
1110 const char *name)
1111 {
1112 return -ENODEV;
1113 }
1114
iommu_group_add_device(struct iommu_group * group,struct device * dev)1115 static inline int iommu_group_add_device(struct iommu_group *group,
1116 struct device *dev)
1117 {
1118 return -ENODEV;
1119 }
1120
iommu_group_remove_device(struct device * dev)1121 static inline void iommu_group_remove_device(struct device *dev)
1122 {
1123 }
1124
iommu_group_for_each_dev(struct iommu_group * group,void * data,int (* fn)(struct device *,void *))1125 static inline int iommu_group_for_each_dev(struct iommu_group *group,
1126 void *data,
1127 int (*fn)(struct device *, void *))
1128 {
1129 return -ENODEV;
1130 }
1131
iommu_group_get(struct device * dev)1132 static inline struct iommu_group *iommu_group_get(struct device *dev)
1133 {
1134 return NULL;
1135 }
1136
iommu_group_put(struct iommu_group * group)1137 static inline void iommu_group_put(struct iommu_group *group)
1138 {
1139 }
1140
1141 static inline
iommu_register_device_fault_handler(struct device * dev,iommu_dev_fault_handler_t handler,void * data)1142 int iommu_register_device_fault_handler(struct device *dev,
1143 iommu_dev_fault_handler_t handler,
1144 void *data)
1145 {
1146 return -ENODEV;
1147 }
1148
iommu_unregister_device_fault_handler(struct device * dev)1149 static inline int iommu_unregister_device_fault_handler(struct device *dev)
1150 {
1151 return 0;
1152 }
1153
1154 static inline
iommu_report_device_fault(struct device * dev,struct iommu_fault_event * evt)1155 int iommu_report_device_fault(struct device *dev, struct iommu_fault_event *evt)
1156 {
1157 return -ENODEV;
1158 }
1159
iommu_page_response(struct device * dev,struct iommu_page_response * msg)1160 static inline int iommu_page_response(struct device *dev,
1161 struct iommu_page_response *msg)
1162 {
1163 return -ENODEV;
1164 }
1165
iommu_group_id(struct iommu_group * group)1166 static inline int iommu_group_id(struct iommu_group *group)
1167 {
1168 return -ENODEV;
1169 }
1170
iommu_set_pgtable_quirks(struct iommu_domain * domain,unsigned long quirks)1171 static inline int iommu_set_pgtable_quirks(struct iommu_domain *domain,
1172 unsigned long quirks)
1173 {
1174 return 0;
1175 }
1176
iommu_device_register(struct iommu_device * iommu,const struct iommu_ops * ops,struct device * hwdev)1177 static inline int iommu_device_register(struct iommu_device *iommu,
1178 const struct iommu_ops *ops,
1179 struct device *hwdev)
1180 {
1181 return -ENODEV;
1182 }
1183
dev_to_iommu_device(struct device * dev)1184 static inline struct iommu_device *dev_to_iommu_device(struct device *dev)
1185 {
1186 return NULL;
1187 }
1188
iommu_iotlb_gather_init(struct iommu_iotlb_gather * gather)1189 static inline void iommu_iotlb_gather_init(struct iommu_iotlb_gather *gather)
1190 {
1191 }
1192
iommu_iotlb_gather_add_page(struct iommu_domain * domain,struct iommu_iotlb_gather * gather,unsigned long iova,size_t size)1193 static inline void iommu_iotlb_gather_add_page(struct iommu_domain *domain,
1194 struct iommu_iotlb_gather *gather,
1195 unsigned long iova, size_t size)
1196 {
1197 }
1198
iommu_iotlb_gather_queued(struct iommu_iotlb_gather * gather)1199 static inline bool iommu_iotlb_gather_queued(struct iommu_iotlb_gather *gather)
1200 {
1201 return false;
1202 }
1203
iommu_dirty_bitmap_init(struct iommu_dirty_bitmap * dirty,struct iova_bitmap * bitmap,struct iommu_iotlb_gather * gather)1204 static inline void iommu_dirty_bitmap_init(struct iommu_dirty_bitmap *dirty,
1205 struct iova_bitmap *bitmap,
1206 struct iommu_iotlb_gather *gather)
1207 {
1208 }
1209
iommu_dirty_bitmap_record(struct iommu_dirty_bitmap * dirty,unsigned long iova,unsigned long length)1210 static inline void iommu_dirty_bitmap_record(struct iommu_dirty_bitmap *dirty,
1211 unsigned long iova,
1212 unsigned long length)
1213 {
1214 }
1215
iommu_device_unregister(struct iommu_device * iommu)1216 static inline void iommu_device_unregister(struct iommu_device *iommu)
1217 {
1218 }
1219
iommu_device_sysfs_add(struct iommu_device * iommu,struct device * parent,const struct attribute_group ** groups,const char * fmt,...)1220 static inline int iommu_device_sysfs_add(struct iommu_device *iommu,
1221 struct device *parent,
1222 const struct attribute_group **groups,
1223 const char *fmt, ...)
1224 {
1225 return -ENODEV;
1226 }
1227
iommu_device_sysfs_remove(struct iommu_device * iommu)1228 static inline void iommu_device_sysfs_remove(struct iommu_device *iommu)
1229 {
1230 }
1231
iommu_device_link(struct device * dev,struct device * link)1232 static inline int iommu_device_link(struct device *dev, struct device *link)
1233 {
1234 return -EINVAL;
1235 }
1236
iommu_device_unlink(struct device * dev,struct device * link)1237 static inline void iommu_device_unlink(struct device *dev, struct device *link)
1238 {
1239 }
1240
iommu_fwspec_init(struct device * dev,struct fwnode_handle * iommu_fwnode,const struct iommu_ops * ops)1241 static inline int iommu_fwspec_init(struct device *dev,
1242 struct fwnode_handle *iommu_fwnode,
1243 const struct iommu_ops *ops)
1244 {
1245 return -ENODEV;
1246 }
1247
iommu_fwspec_free(struct device * dev)1248 static inline void iommu_fwspec_free(struct device *dev)
1249 {
1250 }
1251
iommu_fwspec_add_ids(struct device * dev,u32 * ids,int num_ids)1252 static inline int iommu_fwspec_add_ids(struct device *dev, u32 *ids,
1253 int num_ids)
1254 {
1255 return -ENODEV;
1256 }
1257
1258 static inline
iommu_ops_from_fwnode(struct fwnode_handle * fwnode)1259 const struct iommu_ops *iommu_ops_from_fwnode(struct fwnode_handle *fwnode)
1260 {
1261 return NULL;
1262 }
1263
1264 static inline int
iommu_dev_enable_feature(struct device * dev,enum iommu_dev_features feat)1265 iommu_dev_enable_feature(struct device *dev, enum iommu_dev_features feat)
1266 {
1267 return -ENODEV;
1268 }
1269
1270 static inline int
iommu_dev_disable_feature(struct device * dev,enum iommu_dev_features feat)1271 iommu_dev_disable_feature(struct device *dev, enum iommu_dev_features feat)
1272 {
1273 return -ENODEV;
1274 }
1275
dev_iommu_fwspec_get(struct device * dev)1276 static inline struct iommu_fwspec *dev_iommu_fwspec_get(struct device *dev)
1277 {
1278 return NULL;
1279 }
1280
iommu_device_use_default_domain(struct device * dev)1281 static inline int iommu_device_use_default_domain(struct device *dev)
1282 {
1283 return 0;
1284 }
1285
iommu_device_unuse_default_domain(struct device * dev)1286 static inline void iommu_device_unuse_default_domain(struct device *dev)
1287 {
1288 }
1289
1290 static inline int
iommu_group_claim_dma_owner(struct iommu_group * group,void * owner)1291 iommu_group_claim_dma_owner(struct iommu_group *group, void *owner)
1292 {
1293 return -ENODEV;
1294 }
1295
iommu_group_release_dma_owner(struct iommu_group * group)1296 static inline void iommu_group_release_dma_owner(struct iommu_group *group)
1297 {
1298 }
1299
iommu_group_dma_owner_claimed(struct iommu_group * group)1300 static inline bool iommu_group_dma_owner_claimed(struct iommu_group *group)
1301 {
1302 return false;
1303 }
1304
iommu_device_release_dma_owner(struct device * dev)1305 static inline void iommu_device_release_dma_owner(struct device *dev)
1306 {
1307 }
1308
iommu_device_claim_dma_owner(struct device * dev,void * owner)1309 static inline int iommu_device_claim_dma_owner(struct device *dev, void *owner)
1310 {
1311 return -ENODEV;
1312 }
1313
1314 static inline struct iommu_domain *
iommu_sva_domain_alloc(struct device * dev,struct mm_struct * mm)1315 iommu_sva_domain_alloc(struct device *dev, struct mm_struct *mm)
1316 {
1317 return NULL;
1318 }
1319
iommu_attach_device_pasid(struct iommu_domain * domain,struct device * dev,ioasid_t pasid)1320 static inline int iommu_attach_device_pasid(struct iommu_domain *domain,
1321 struct device *dev, ioasid_t pasid)
1322 {
1323 return -ENODEV;
1324 }
1325
iommu_detach_device_pasid(struct iommu_domain * domain,struct device * dev,ioasid_t pasid)1326 static inline void iommu_detach_device_pasid(struct iommu_domain *domain,
1327 struct device *dev, ioasid_t pasid)
1328 {
1329 }
1330
1331 static inline struct iommu_domain *
iommu_get_domain_for_dev_pasid(struct device * dev,ioasid_t pasid,unsigned int type)1332 iommu_get_domain_for_dev_pasid(struct device *dev, ioasid_t pasid,
1333 unsigned int type)
1334 {
1335 return NULL;
1336 }
1337
iommu_alloc_global_pasid(struct device * dev)1338 static inline ioasid_t iommu_alloc_global_pasid(struct device *dev)
1339 {
1340 return IOMMU_PASID_INVALID;
1341 }
1342
iommu_free_global_pasid(ioasid_t pasid)1343 static inline void iommu_free_global_pasid(ioasid_t pasid) {}
1344 #endif /* CONFIG_IOMMU_API */
1345
1346 /**
1347 * iommu_map_sgtable - Map the given buffer to the IOMMU domain
1348 * @domain: The IOMMU domain to perform the mapping
1349 * @iova: The start address to map the buffer
1350 * @sgt: The sg_table object describing the buffer
1351 * @prot: IOMMU protection bits
1352 *
1353 * Creates a mapping at @iova for the buffer described by a scatterlist
1354 * stored in the given sg_table object in the provided IOMMU domain.
1355 */
iommu_map_sgtable(struct iommu_domain * domain,unsigned long iova,struct sg_table * sgt,int prot)1356 static inline ssize_t iommu_map_sgtable(struct iommu_domain *domain,
1357 unsigned long iova, struct sg_table *sgt, int prot)
1358 {
1359 return iommu_map_sg(domain, iova, sgt->sgl, sgt->orig_nents, prot,
1360 GFP_KERNEL);
1361 }
1362
1363 #ifdef CONFIG_IOMMU_DEBUGFS
1364 extern struct dentry *iommu_debugfs_dir;
1365 void iommu_debugfs_setup(void);
1366 #else
iommu_debugfs_setup(void)1367 static inline void iommu_debugfs_setup(void) {}
1368 #endif
1369
1370 #ifdef CONFIG_IOMMU_DMA
1371 #include <linux/msi.h>
1372
1373 /* Setup call for arch DMA mapping code */
1374 void iommu_setup_dma_ops(struct device *dev, u64 dma_base, u64 dma_limit);
1375
1376 int iommu_get_msi_cookie(struct iommu_domain *domain, dma_addr_t base);
1377
1378 int iommu_dma_prepare_msi(struct msi_desc *desc, phys_addr_t msi_addr);
1379 void iommu_dma_compose_msi_msg(struct msi_desc *desc, struct msi_msg *msg);
1380
1381 #else /* CONFIG_IOMMU_DMA */
1382
1383 struct msi_desc;
1384 struct msi_msg;
1385
iommu_setup_dma_ops(struct device * dev,u64 dma_base,u64 dma_limit)1386 static inline void iommu_setup_dma_ops(struct device *dev, u64 dma_base, u64 dma_limit)
1387 {
1388 }
1389
iommu_get_msi_cookie(struct iommu_domain * domain,dma_addr_t base)1390 static inline int iommu_get_msi_cookie(struct iommu_domain *domain, dma_addr_t base)
1391 {
1392 return -ENODEV;
1393 }
1394
iommu_dma_prepare_msi(struct msi_desc * desc,phys_addr_t msi_addr)1395 static inline int iommu_dma_prepare_msi(struct msi_desc *desc, phys_addr_t msi_addr)
1396 {
1397 return 0;
1398 }
1399
iommu_dma_compose_msi_msg(struct msi_desc * desc,struct msi_msg * msg)1400 static inline void iommu_dma_compose_msi_msg(struct msi_desc *desc, struct msi_msg *msg)
1401 {
1402 }
1403
1404 #endif /* CONFIG_IOMMU_DMA */
1405
1406 /*
1407 * Newer generations of Tegra SoCs require devices' stream IDs to be directly programmed into
1408 * some registers. These are always paired with a Tegra SMMU or ARM SMMU, for which the contents
1409 * of the struct iommu_fwspec are known. Use this helper to formalize access to these internals.
1410 */
1411 #define TEGRA_STREAM_ID_BYPASS 0x7f
1412
tegra_dev_iommu_get_stream_id(struct device * dev,u32 * stream_id)1413 static inline bool tegra_dev_iommu_get_stream_id(struct device *dev, u32 *stream_id)
1414 {
1415 #ifdef CONFIG_IOMMU_API
1416 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
1417
1418 if (fwspec && fwspec->num_ids == 1) {
1419 *stream_id = fwspec->ids[0] & 0xffff;
1420 return true;
1421 }
1422 #endif
1423
1424 return false;
1425 }
1426
1427 #ifdef CONFIG_IOMMU_MM_DATA
mm_pasid_init(struct mm_struct * mm)1428 static inline void mm_pasid_init(struct mm_struct *mm)
1429 {
1430 /*
1431 * During dup_mm(), a new mm will be memcpy'd from an old one and that makes
1432 * the new mm and the old one point to a same iommu_mm instance. When either
1433 * one of the two mms gets released, the iommu_mm instance is freed, leaving
1434 * the other mm running into a use-after-free/double-free problem. To avoid
1435 * the problem, zeroing the iommu_mm pointer of a new mm is needed here.
1436 */
1437 mm->iommu_mm = NULL;
1438 }
1439
mm_valid_pasid(struct mm_struct * mm)1440 static inline bool mm_valid_pasid(struct mm_struct *mm)
1441 {
1442 return READ_ONCE(mm->iommu_mm);
1443 }
1444
mm_get_enqcmd_pasid(struct mm_struct * mm)1445 static inline u32 mm_get_enqcmd_pasid(struct mm_struct *mm)
1446 {
1447 struct iommu_mm_data *iommu_mm = READ_ONCE(mm->iommu_mm);
1448
1449 if (!iommu_mm)
1450 return IOMMU_PASID_INVALID;
1451 return iommu_mm->pasid;
1452 }
1453
1454 void mm_pasid_drop(struct mm_struct *mm);
1455 struct iommu_sva *iommu_sva_bind_device(struct device *dev,
1456 struct mm_struct *mm);
1457 void iommu_sva_unbind_device(struct iommu_sva *handle);
1458 u32 iommu_sva_get_pasid(struct iommu_sva *handle);
1459 #else
1460 static inline struct iommu_sva *
iommu_sva_bind_device(struct device * dev,struct mm_struct * mm)1461 iommu_sva_bind_device(struct device *dev, struct mm_struct *mm)
1462 {
1463 return NULL;
1464 }
1465
iommu_sva_unbind_device(struct iommu_sva * handle)1466 static inline void iommu_sva_unbind_device(struct iommu_sva *handle)
1467 {
1468 }
1469
iommu_sva_get_pasid(struct iommu_sva * handle)1470 static inline u32 iommu_sva_get_pasid(struct iommu_sva *handle)
1471 {
1472 return IOMMU_PASID_INVALID;
1473 }
mm_pasid_init(struct mm_struct * mm)1474 static inline void mm_pasid_init(struct mm_struct *mm) {}
mm_valid_pasid(struct mm_struct * mm)1475 static inline bool mm_valid_pasid(struct mm_struct *mm) { return false; }
1476
mm_get_enqcmd_pasid(struct mm_struct * mm)1477 static inline u32 mm_get_enqcmd_pasid(struct mm_struct *mm)
1478 {
1479 return IOMMU_PASID_INVALID;
1480 }
1481
mm_pasid_drop(struct mm_struct * mm)1482 static inline void mm_pasid_drop(struct mm_struct *mm) {}
1483 #endif /* CONFIG_IOMMU_SVA */
1484
1485 #endif /* __LINUX_IOMMU_H */
1486