Lines Matching +full:iommu +full:- +full:secure +full:- +full:id
1 // SPDX-License-Identifier: GPL-2.0-only
2 /* Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES
4 #include <linux/iommu.h>
6 #include <linux/pci-ats.h>
10 #include "../iommu-priv.h"
31 WARN_ON(!xa_empty(&igroup->pasid_attach)); in iommufd_group_release()
33 xa_cmpxchg(&igroup->ictx->groups, iommu_group_id(igroup->group), igroup, in iommufd_group_release()
35 iommu_group_put(igroup->group); in iommufd_group_release()
36 mutex_destroy(&igroup->lock); in iommufd_group_release()
42 kref_put(&group->ref, iommufd_group_release); in iommufd_put_group()
51 * group ID's cannot be re-used until the group is put back which does in iommufd_group_try_get()
54 if (WARN_ON(igroup->group != group)) in iommufd_group_try_get()
56 return kref_get_unless_zero(&igroup->ref); in iommufd_group_try_get()
61 * parallel xarray indexed by iommu_group id to hold this instead of putting it
73 unsigned int id; in iommufd_get_group() local
77 return ERR_PTR(-ENODEV); in iommufd_get_group()
79 id = iommu_group_id(group); in iommufd_get_group()
81 xa_lock(&ictx->groups); in iommufd_get_group()
82 igroup = xa_load(&ictx->groups, id); in iommufd_get_group()
84 xa_unlock(&ictx->groups); in iommufd_get_group()
88 xa_unlock(&ictx->groups); in iommufd_get_group()
93 return ERR_PTR(-ENOMEM); in iommufd_get_group()
96 kref_init(&new_igroup->ref); in iommufd_get_group()
97 mutex_init(&new_igroup->lock); in iommufd_get_group()
98 xa_init(&new_igroup->pasid_attach); in iommufd_get_group()
99 new_igroup->sw_msi_start = PHYS_ADDR_MAX; in iommufd_get_group()
101 new_igroup->group = group; in iommufd_get_group()
107 new_igroup->ictx = ictx; in iommufd_get_group()
114 xa_lock(&ictx->groups); in iommufd_get_group()
116 igroup = __xa_cmpxchg(&ictx->groups, id, cur_igroup, new_igroup, in iommufd_get_group()
119 xa_unlock(&ictx->groups); in iommufd_get_group()
126 xa_unlock(&ictx->groups); in iommufd_get_group()
132 xa_unlock(&ictx->groups); in iommufd_get_group()
145 iommu_device_release_dma_owner(idev->dev); in iommufd_device_destroy()
146 iommufd_put_group(idev->igroup); in iommufd_device_destroy()
147 if (!iommufd_selftest_is_mock_dev(idev->dev)) in iommufd_device_destroy()
148 iommufd_ctx_put(idev->ictx); in iommufd_device_destroy()
152 * iommufd_device_bind - Bind a physical device to an iommu fd
155 * @id: Output ID number to return to userspace for this device
168 struct device *dev, u32 *id) in iommufd_device_bind() argument
179 return ERR_PTR(-EINVAL); in iommufd_device_bind()
187 * allowed if the module parameter is set. Secure/Isolated means that a in iommufd_device_bind()
192 !iommu_group_has_isolated_msi(igroup->group)) { in iommufd_device_bind()
194 rc = -EPERM; in iommufd_device_bind()
200 "MSI interrupts are not secure, they cannot be isolated by the platform. " in iommufd_device_bind()
214 idev->ictx = ictx; in iommufd_device_bind()
217 idev->dev = dev; in iommufd_device_bind()
218 idev->enforce_cache_coherency = in iommufd_device_bind()
221 refcount_inc(&idev->obj.users); in iommufd_device_bind()
223 idev->igroup = igroup; in iommufd_device_bind()
224 mutex_init(&idev->iopf_lock); in iommufd_device_bind()
232 iommufd_object_finalize(ictx, &idev->obj); in iommufd_device_bind()
233 *id = idev->obj.id; in iommufd_device_bind()
245 * iommufd_ctx_has_group - True if any device within the group is bound
261 xa_lock(&ictx->objects); in iommufd_ctx_has_group()
262 xa_for_each(&ictx->objects, index, obj) { in iommufd_ctx_has_group()
263 if (obj->type == IOMMUFD_OBJ_DEVICE && in iommufd_ctx_has_group()
265 ->igroup->group == group) { in iommufd_ctx_has_group()
266 xa_unlock(&ictx->objects); in iommufd_ctx_has_group()
270 xa_unlock(&ictx->objects); in iommufd_ctx_has_group()
276 * iommufd_device_unbind - Undo iommufd_device_bind()
286 iommufd_object_destroy_user(idev->ictx, &idev->obj); in iommufd_device_unbind()
292 return idev->ictx; in iommufd_device_to_ictx()
298 return idev->obj.id; in iommufd_device_to_id()
310 lockdep_assert_held(&igroup->lock); in iommufd_group_device_num()
312 attach = xa_load(&igroup->pasid_attach, pasid); in iommufd_group_device_num()
314 xa_for_each(&attach->device_array, index, idev) in iommufd_group_device_num()
323 struct iommufd_ctx *ictx = igroup->ictx; in iommufd_group_setup_msi()
326 if (igroup->sw_msi_start == PHYS_ADDR_MAX) in iommufd_group_setup_msi()
332 guard(mutex)(&ictx->sw_msi_lock); in iommufd_group_setup_msi()
333 list_for_each_entry(cur, &ictx->sw_msi_list, sw_msi_item) { in iommufd_group_setup_msi()
336 if (cur->sw_msi_start != igroup->sw_msi_start || in iommufd_group_setup_msi()
337 !test_bit(cur->id, igroup->required_sw_msi.bitmap)) in iommufd_group_setup_msi()
358 lockdep_assert_held(&igroup->lock); in iommufd_group_first_attach()
359 return !xa_load(&igroup->pasid_attach, pasid); in iommufd_group_first_attach()
366 struct iommufd_group *igroup = idev->igroup; in iommufd_device_attach_reserved_iova()
369 lockdep_assert_held(&igroup->lock); in iommufd_device_attach_reserved_iova()
371 rc = iopt_table_enforce_dev_resv_regions(&hwpt_paging->ioas->iopt, in iommufd_device_attach_reserved_iova()
372 idev->dev, in iommufd_device_attach_reserved_iova()
373 &igroup->sw_msi_start); in iommufd_device_attach_reserved_iova()
380 iopt_remove_reserved_iova(&hwpt_paging->ioas->iopt, in iommufd_device_attach_reserved_iova()
381 idev->dev); in iommufd_device_attach_reserved_iova()
395 attach = xa_load(&idev->igroup->pasid_attach, pasid); in iommufd_device_is_attached()
396 return xa_load(&attach->device_array, idev->obj.id); in iommufd_device_is_attached()
403 struct iommufd_group *igroup = idev->igroup; in iommufd_hwpt_pasid_compat()
405 lockdep_assert_held(&igroup->lock); in iommufd_hwpt_pasid_compat()
410 if (!hwpt->pasid_compat && in iommufd_hwpt_pasid_compat()
411 xa_find_after(&igroup->pasid_attach, in iommufd_hwpt_pasid_compat()
413 return -EINVAL; in iommufd_hwpt_pasid_compat()
417 if (!hwpt->pasid_compat) in iommufd_hwpt_pasid_compat()
418 return -EINVAL; in iommufd_hwpt_pasid_compat()
420 attach = xa_load(&igroup->pasid_attach, IOMMU_NO_PASID); in iommufd_hwpt_pasid_compat()
421 if (attach && attach->hwpt && !attach->hwpt->pasid_compat) in iommufd_hwpt_pasid_compat()
422 return -EINVAL; in iommufd_hwpt_pasid_compat()
441 return -ENOMEM; in iommufd_hwpt_attach_device()
443 if (hwpt->fault) { in iommufd_hwpt_attach_device()
449 handle->idev = idev; in iommufd_hwpt_attach_device()
451 rc = iommu_attach_group_handle(hwpt->domain, idev->igroup->group, in iommufd_hwpt_attach_device()
452 &handle->handle); in iommufd_hwpt_attach_device()
454 rc = iommu_attach_device_pasid(hwpt->domain, idev->dev, pasid, in iommufd_hwpt_attach_device()
455 &handle->handle); in iommufd_hwpt_attach_device()
462 if (hwpt->fault) in iommufd_hwpt_attach_device()
474 lockdep_assert_held(&idev->igroup->lock); in iommufd_device_get_attach_handle()
477 iommu_attach_handle_get(idev->igroup->group, pasid, 0); in iommufd_device_get_attach_handle()
491 iommu_detach_group_handle(hwpt->domain, idev->igroup->group); in iommufd_hwpt_detach_device()
493 iommu_detach_device_pasid(hwpt->domain, idev->dev, pasid); in iommufd_hwpt_detach_device()
495 if (hwpt->fault) { in iommufd_hwpt_detach_device()
518 return -ENOMEM; in iommufd_hwpt_replace_device()
520 if (hwpt->fault && !old->fault) { in iommufd_hwpt_replace_device()
526 handle->idev = idev; in iommufd_hwpt_replace_device()
528 rc = iommu_replace_group_handle(idev->igroup->group, in iommufd_hwpt_replace_device()
529 hwpt->domain, &handle->handle); in iommufd_hwpt_replace_device()
531 rc = iommu_replace_device_pasid(hwpt->domain, idev->dev, in iommufd_hwpt_replace_device()
532 pasid, &handle->handle); in iommufd_hwpt_replace_device()
536 if (old->fault) { in iommufd_hwpt_replace_device()
538 if (!hwpt->fault) in iommufd_hwpt_replace_device()
546 if (hwpt->fault && !old->fault) in iommufd_hwpt_replace_device()
558 struct iommufd_group *igroup = idev->igroup; in iommufd_hw_pagetable_attach()
563 mutex_lock(&igroup->lock); in iommufd_hw_pagetable_attach()
565 attach = xa_cmpxchg(&igroup->pasid_attach, pasid, NULL, in iommufd_hw_pagetable_attach()
575 rc = -ENOMEM; in iommufd_hw_pagetable_attach()
578 xa_init(&attach->device_array); in iommufd_hw_pagetable_attach()
581 old_hwpt = attach->hwpt; in iommufd_hw_pagetable_attach()
583 rc = xa_insert(&attach->device_array, idev->obj.id, XA_ZERO_ENTRY, in iommufd_hw_pagetable_attach()
586 WARN_ON(rc == -EBUSY && !old_hwpt); in iommufd_hw_pagetable_attach()
591 rc = -EINVAL; in iommufd_hw_pagetable_attach()
604 * should attach every device individually to the hwpt as the per-device in iommufd_hw_pagetable_attach()
612 attach->hwpt = hwpt; in iommufd_hw_pagetable_attach()
613 WARN_ON(xa_is_err(xa_store(&igroup->pasid_attach, pasid, attach, in iommufd_hw_pagetable_attach()
616 refcount_inc(&hwpt->obj.users); in iommufd_hw_pagetable_attach()
617 WARN_ON(xa_is_err(xa_store(&attach->device_array, idev->obj.id, in iommufd_hw_pagetable_attach()
619 mutex_unlock(&igroup->lock); in iommufd_hw_pagetable_attach()
623 iopt_remove_reserved_iova(&hwpt_paging->ioas->iopt, idev->dev); in iommufd_hw_pagetable_attach()
625 xa_release(&attach->device_array, idev->obj.id); in iommufd_hw_pagetable_attach()
631 xa_release(&igroup->pasid_attach, pasid); in iommufd_hw_pagetable_attach()
633 mutex_unlock(&igroup->lock); in iommufd_hw_pagetable_attach()
640 struct iommufd_group *igroup = idev->igroup; in iommufd_hw_pagetable_detach()
645 mutex_lock(&igroup->lock); in iommufd_hw_pagetable_detach()
646 attach = xa_load(&igroup->pasid_attach, pasid); in iommufd_hw_pagetable_detach()
648 mutex_unlock(&igroup->lock); in iommufd_hw_pagetable_detach()
652 hwpt = attach->hwpt; in iommufd_hw_pagetable_detach()
655 xa_erase(&attach->device_array, idev->obj.id); in iommufd_hw_pagetable_detach()
656 if (xa_empty(&attach->device_array)) { in iommufd_hw_pagetable_detach()
658 xa_erase(&igroup->pasid_attach, pasid); in iommufd_hw_pagetable_detach()
662 iopt_remove_reserved_iova(&hwpt_paging->ioas->iopt, idev->dev); in iommufd_hw_pagetable_detach()
663 mutex_unlock(&igroup->lock); in iommufd_hw_pagetable_detach()
689 lockdep_assert_held(&igroup->lock); in iommufd_group_remove_reserved_iova()
691 attach = xa_load(&igroup->pasid_attach, IOMMU_NO_PASID); in iommufd_group_remove_reserved_iova()
692 xa_for_each(&attach->device_array, index, cur) in iommufd_group_remove_reserved_iova()
693 iopt_remove_reserved_iova(&hwpt_paging->ioas->iopt, cur->dev); in iommufd_group_remove_reserved_iova()
706 lockdep_assert_held(&igroup->lock); in iommufd_group_do_replace_reserved_iova()
708 attach = xa_load(&igroup->pasid_attach, IOMMU_NO_PASID); in iommufd_group_do_replace_reserved_iova()
709 old_hwpt_paging = find_hwpt_paging(attach->hwpt); in iommufd_group_do_replace_reserved_iova()
710 if (!old_hwpt_paging || hwpt_paging->ioas != old_hwpt_paging->ioas) { in iommufd_group_do_replace_reserved_iova()
711 xa_for_each(&attach->device_array, index, cur) { in iommufd_group_do_replace_reserved_iova()
713 &hwpt_paging->ioas->iopt, cur->dev, NULL); in iommufd_group_do_replace_reserved_iova()
736 struct iommufd_group *igroup = idev->igroup; in iommufd_device_do_replace()
742 mutex_lock(&igroup->lock); in iommufd_device_do_replace()
744 attach = xa_load(&igroup->pasid_attach, pasid); in iommufd_device_do_replace()
746 rc = -EINVAL; in iommufd_device_do_replace()
750 old_hwpt = attach->hwpt; in iommufd_device_do_replace()
752 WARN_ON(!old_hwpt || xa_empty(&attach->device_array)); in iommufd_device_do_replace()
755 rc = -EINVAL; in iommufd_device_do_replace()
760 mutex_unlock(&igroup->lock); in iommufd_device_do_replace()
776 (!hwpt_paging || hwpt_paging->ioas != old_hwpt_paging->ioas)) in iommufd_device_do_replace()
779 attach->hwpt = hwpt; in iommufd_device_do_replace()
786 refcount_add(num_devices, &hwpt->obj.users); in iommufd_device_do_replace()
788 WARN_ON(refcount_sub_and_test(num_devices - 1, in iommufd_device_do_replace()
789 &old_hwpt->obj.users)); in iommufd_device_do_replace()
790 mutex_unlock(&igroup->lock); in iommufd_device_do_replace()
798 mutex_unlock(&igroup->lock); in iommufd_device_do_replace()
833 mutex_lock(&ioas->mutex); in iommufd_device_auto_get_domain()
834 list_for_each_entry(hwpt_paging, &ioas->hwpt_list, hwpt_item) { in iommufd_device_auto_get_domain()
835 if (!hwpt_paging->auto_domain) in iommufd_device_auto_get_domain()
838 hwpt = &hwpt_paging->common; in iommufd_device_auto_get_domain()
839 if (!iommufd_lock_obj(&hwpt->obj)) in iommufd_device_auto_get_domain()
843 iommufd_put_object(idev->ictx, &hwpt->obj); in iommufd_device_auto_get_domain()
845 * -EINVAL means the domain is incompatible with the in iommufd_device_auto_get_domain()
850 if (PTR_ERR(destroy_hwpt) == -EINVAL) in iommufd_device_auto_get_domain()
854 *pt_id = hwpt->obj.id; in iommufd_device_auto_get_domain()
855 iommufd_put_object(idev->ictx, &hwpt->obj); in iommufd_device_auto_get_domain()
859 hwpt_paging = iommufd_hwpt_paging_alloc(idev->ictx, ioas, idev, pasid, in iommufd_device_auto_get_domain()
865 hwpt = &hwpt_paging->common; in iommufd_device_auto_get_domain()
875 hwpt_paging->auto_domain = true; in iommufd_device_auto_get_domain()
876 *pt_id = hwpt->obj.id; in iommufd_device_auto_get_domain()
878 iommufd_object_finalize(idev->ictx, &hwpt->obj); in iommufd_device_auto_get_domain()
879 mutex_unlock(&ioas->mutex); in iommufd_device_auto_get_domain()
883 iommufd_object_abort_and_destroy(idev->ictx, &hwpt->obj); in iommufd_device_auto_get_domain()
885 mutex_unlock(&ioas->mutex); in iommufd_device_auto_get_domain()
896 pt_obj = iommufd_get_object(idev->ictx, *pt_id, IOMMUFD_OBJ_ANY); in iommufd_device_change_pt()
900 switch (pt_obj->type) { in iommufd_device_change_pt()
922 destroy_hwpt = ERR_PTR(-EINVAL); in iommufd_device_change_pt()
925 iommufd_put_object(idev->ictx, pt_obj); in iommufd_device_change_pt()
929 iommufd_hw_pagetable_put(idev->ictx, destroy_hwpt); in iommufd_device_change_pt()
933 iommufd_put_object(idev->ictx, pt_obj); in iommufd_device_change_pt()
938 * iommufd_device_attach - Connect a device/pasid to an iommu_domain
942 * Output the IOMMUFD_OBJ_HWPT_PAGING ID
962 * Pairs with iommufd_device_detach() - catches caller bugs attempting in iommufd_device_attach()
965 refcount_inc(&idev->obj.users); in iommufd_device_attach()
971 * iommufd_device_replace - Change the device/pasid's iommu_domain
975 * Output the IOMMUFD_OBJ_HWPT_PAGING ID
982 * If it fails then no change is made to the attachment. The iommu driver may
996 * iommufd_device_detach - Disconnect a device/device to an iommu_domain
1011 iommufd_hw_pagetable_put(idev->ictx, hwpt); in iommufd_device_detach()
1012 refcount_dec(&idev->obj.users); in iommufd_device_detach()
1018 * a valid cur_ioas (access->ioas). A caller passing in a valid new_ioas should
1024 u32 iopt_access_list_id = access->iopt_access_list_id; in iommufd_access_change_ioas()
1025 struct iommufd_ioas *cur_ioas = access->ioas; in iommufd_access_change_ioas()
1028 lockdep_assert_held(&access->ioas_lock); in iommufd_access_change_ioas()
1031 if (cur_ioas != access->ioas_unpin) in iommufd_access_change_ioas()
1032 return -EBUSY; in iommufd_access_change_ioas()
1039 * iommufd_access_unpin_pages() can continue using access->ioas_unpin. in iommufd_access_change_ioas()
1041 access->ioas = NULL; in iommufd_access_change_ioas()
1044 rc = iopt_add_access(&new_ioas->iopt, access); in iommufd_access_change_ioas()
1046 access->ioas = cur_ioas; in iommufd_access_change_ioas()
1049 refcount_inc(&new_ioas->obj.users); in iommufd_access_change_ioas()
1053 if (access->ops->unmap) { in iommufd_access_change_ioas()
1054 mutex_unlock(&access->ioas_lock); in iommufd_access_change_ioas()
1055 access->ops->unmap(access->data, 0, ULONG_MAX); in iommufd_access_change_ioas()
1056 mutex_lock(&access->ioas_lock); in iommufd_access_change_ioas()
1058 iopt_remove_access(&cur_ioas->iopt, access, iopt_access_list_id); in iommufd_access_change_ioas()
1059 refcount_dec(&cur_ioas->obj.users); in iommufd_access_change_ioas()
1062 access->ioas = new_ioas; in iommufd_access_change_ioas()
1063 access->ioas_unpin = new_ioas; in iommufd_access_change_ioas()
1068 static int iommufd_access_change_ioas_id(struct iommufd_access *access, u32 id) in iommufd_access_change_ioas_id() argument
1070 struct iommufd_ioas *ioas = iommufd_get_ioas(access->ictx, id); in iommufd_access_change_ioas_id()
1076 iommufd_put_object(access->ictx, &ioas->obj); in iommufd_access_change_ioas_id()
1085 mutex_lock(&access->ioas_lock); in iommufd_access_destroy_object()
1086 if (access->ioas) in iommufd_access_destroy_object()
1088 mutex_unlock(&access->ioas_lock); in iommufd_access_destroy_object()
1089 iommufd_ctx_put(access->ictx); in iommufd_access_destroy_object()
1093 * iommufd_access_create - Create an iommufd_access
1097 * @id: Output ID number to return to userspace for this access
1107 const struct iommufd_access_ops *ops, void *data, u32 *id) in iommufd_access_create() argument
1119 access->data = data; in iommufd_access_create()
1120 access->ops = ops; in iommufd_access_create()
1122 if (ops->needs_pin_pages) in iommufd_access_create()
1123 access->iova_alignment = PAGE_SIZE; in iommufd_access_create()
1125 access->iova_alignment = 1; in iommufd_access_create()
1128 refcount_inc(&access->obj.users); in iommufd_access_create()
1129 access->ictx = ictx; in iommufd_access_create()
1131 iommufd_object_finalize(ictx, &access->obj); in iommufd_access_create()
1132 *id = access->obj.id; in iommufd_access_create()
1133 mutex_init(&access->ioas_lock); in iommufd_access_create()
1139 * iommufd_access_destroy - Destroy an iommufd_access
1146 iommufd_object_destroy_user(access->ictx, &access->obj); in iommufd_access_destroy()
1152 mutex_lock(&access->ioas_lock); in iommufd_access_detach()
1153 if (WARN_ON(!access->ioas)) { in iommufd_access_detach()
1154 mutex_unlock(&access->ioas_lock); in iommufd_access_detach()
1158 mutex_unlock(&access->ioas_lock); in iommufd_access_detach()
1166 mutex_lock(&access->ioas_lock); in iommufd_access_attach()
1167 if (WARN_ON(access->ioas)) { in iommufd_access_attach()
1168 mutex_unlock(&access->ioas_lock); in iommufd_access_attach()
1169 return -EINVAL; in iommufd_access_attach()
1173 mutex_unlock(&access->ioas_lock); in iommufd_access_attach()
1182 mutex_lock(&access->ioas_lock); in iommufd_access_replace()
1183 if (!access->ioas) { in iommufd_access_replace()
1184 mutex_unlock(&access->ioas_lock); in iommufd_access_replace()
1185 return -ENOENT; in iommufd_access_replace()
1188 mutex_unlock(&access->ioas_lock); in iommufd_access_replace()
1194 * iommufd_access_notify_unmap - Notify users of an iopt to stop using it
1217 xa_lock(&ioas->iopt.access_list); in iommufd_access_notify_unmap()
1218 xa_for_each(&ioas->iopt.access_list, index, access) { in iommufd_access_notify_unmap()
1219 if (!iommufd_lock_obj(&access->obj)) in iommufd_access_notify_unmap()
1221 xa_unlock(&ioas->iopt.access_list); in iommufd_access_notify_unmap()
1223 access->ops->unmap(access->data, iova, length); in iommufd_access_notify_unmap()
1225 iommufd_put_object(access->ictx, &access->obj); in iommufd_access_notify_unmap()
1226 xa_lock(&ioas->iopt.access_list); in iommufd_access_notify_unmap()
1228 xa_unlock(&ioas->iopt.access_list); in iommufd_access_notify_unmap()
1232 * iommufd_access_unpin_pages() - Undo iommufd_access_pin_pages
1249 WARN_ON(check_add_overflow(iova, length - 1, &last_iova))) in iommufd_access_unpin_pages()
1252 mutex_lock(&access->ioas_lock); in iommufd_access_unpin_pages()
1257 if (WARN_ON(!access->ioas_unpin)) { in iommufd_access_unpin_pages()
1258 mutex_unlock(&access->ioas_lock); in iommufd_access_unpin_pages()
1261 iopt = &access->ioas_unpin->iopt; in iommufd_access_unpin_pages()
1263 down_read(&iopt->iova_rwsem); in iommufd_access_unpin_pages()
1271 up_read(&iopt->iova_rwsem); in iommufd_access_unpin_pages()
1272 mutex_unlock(&access->ioas_lock); in iommufd_access_unpin_pages()
1278 if (iopt_area_start_byte(iter->area, iter->cur_iova) % PAGE_SIZE) in iopt_area_contig_is_aligned()
1282 (iopt_area_start_byte(iter->area, iopt_area_last_iova(iter->area)) % in iopt_area_contig_is_aligned()
1283 PAGE_SIZE) != (PAGE_SIZE - 1)) in iopt_area_contig_is_aligned()
1291 return area->iommu_prot & IOMMU_WRITE; in check_area_prot()
1292 return area->iommu_prot & IOMMU_READ; in check_area_prot()
1296 * iommufd_access_pin_pages() - Return a list of pages under the iova
1326 WARN_ON(access->iova_alignment != PAGE_SIZE || !access->ops->unmap)) in iommufd_access_pin_pages()
1327 return -EINVAL; in iommufd_access_pin_pages()
1330 return -EINVAL; in iommufd_access_pin_pages()
1331 if (check_add_overflow(iova, length - 1, &last_iova)) in iommufd_access_pin_pages()
1332 return -EOVERFLOW; in iommufd_access_pin_pages()
1334 mutex_lock(&access->ioas_lock); in iommufd_access_pin_pages()
1335 if (!access->ioas) { in iommufd_access_pin_pages()
1336 mutex_unlock(&access->ioas_lock); in iommufd_access_pin_pages()
1337 return -ENOENT; in iommufd_access_pin_pages()
1339 iopt = &access->ioas->iopt; in iommufd_access_pin_pages()
1341 down_read(&iopt->iova_rwsem); in iommufd_access_pin_pages()
1348 if (area->prevent_access || in iommufd_access_pin_pages()
1350 rc = -EINVAL; in iommufd_access_pin_pages()
1355 rc = -EPERM; in iommufd_access_pin_pages()
1363 out_pages += last_index - index + 1; in iommufd_access_pin_pages()
1366 rc = -ENOENT; in iommufd_access_pin_pages()
1370 up_read(&iopt->iova_rwsem); in iommufd_access_pin_pages()
1371 mutex_unlock(&access->ioas_lock); in iommufd_access_pin_pages()
1376 last_iova = iter.cur_iova - 1; in iommufd_access_pin_pages()
1385 up_read(&iopt->iova_rwsem); in iommufd_access_pin_pages()
1386 mutex_unlock(&access->ioas_lock); in iommufd_access_pin_pages()
1392 * iommufd_access_rw - Read or write data under the iova
1410 int rc = -EINVAL; in iommufd_access_rw()
1413 return -EINVAL; in iommufd_access_rw()
1414 if (check_add_overflow(iova, length - 1, &last_iova)) in iommufd_access_rw()
1415 return -EOVERFLOW; in iommufd_access_rw()
1417 mutex_lock(&access->ioas_lock); in iommufd_access_rw()
1418 if (!access->ioas) { in iommufd_access_rw()
1419 mutex_unlock(&access->ioas_lock); in iommufd_access_rw()
1420 return -ENOENT; in iommufd_access_rw()
1422 iopt = &access->ioas->iopt; in iommufd_access_rw()
1424 down_read(&iopt->iova_rwsem); in iommufd_access_rw()
1427 unsigned long bytes = (last - iter.cur_iova) + 1; in iommufd_access_rw()
1429 if (area->prevent_access) { in iommufd_access_rw()
1430 rc = -EINVAL; in iommufd_access_rw()
1435 rc = -EPERM; in iommufd_access_rw()
1440 area->pages, iopt_area_start_byte(area, iter.cur_iova), in iommufd_access_rw()
1447 rc = -ENOENT; in iommufd_access_rw()
1449 up_read(&iopt->iova_rwsem); in iommufd_access_rw()
1450 mutex_unlock(&access->ioas_lock); in iommufd_access_rw()
1457 struct iommu_hw_info *cmd = ucmd->cmd; in iommufd_get_hw_info()
1458 void __user *user_ptr = u64_to_user_ptr(cmd->data_uptr); in iommufd_get_hw_info()
1466 if (cmd->flags || cmd->__reserved[0] || cmd->__reserved[1] || in iommufd_get_hw_info()
1467 cmd->__reserved[2]) in iommufd_get_hw_info()
1468 return -EOPNOTSUPP; in iommufd_get_hw_info()
1470 idev = iommufd_get_device(ucmd, cmd->dev_id); in iommufd_get_hw_info()
1474 ops = dev_iommu_ops(idev->dev); in iommufd_get_hw_info()
1475 if (ops->hw_info) { in iommufd_get_hw_info()
1476 data = ops->hw_info(idev->dev, &data_len, &cmd->out_data_type); in iommufd_get_hw_info()
1486 if (WARN_ON_ONCE(cmd->out_data_type == in iommufd_get_hw_info()
1488 rc = -ENODEV; in iommufd_get_hw_info()
1492 cmd->out_data_type = IOMMU_HW_INFO_TYPE_NONE; in iommufd_get_hw_info()
1497 copy_len = min(cmd->data_len, data_len); in iommufd_get_hw_info()
1499 rc = -EFAULT; in iommufd_get_hw_info()
1507 if (copy_len < cmd->data_len) { in iommufd_get_hw_info()
1508 if (clear_user(user_ptr + copy_len, cmd->data_len - copy_len)) { in iommufd_get_hw_info()
1509 rc = -EFAULT; in iommufd_get_hw_info()
1518 cmd->data_len = data_len; in iommufd_get_hw_info()
1520 cmd->out_capabilities = 0; in iommufd_get_hw_info()
1521 if (device_iommu_capable(idev->dev, IOMMU_CAP_DIRTY_TRACKING)) in iommufd_get_hw_info()
1522 cmd->out_capabilities |= IOMMU_HW_CAP_DIRTY_TRACKING; in iommufd_get_hw_info()
1524 cmd->out_max_pasid_log2 = 0; in iommufd_get_hw_info()
1526 * Currently, all iommu drivers enable PASID in the probe_device() in iommufd_get_hw_info()
1527 * op if iommu and device supports it. So the max_pasids stored in in iommufd_get_hw_info()
1528 * dev->iommu indicates both PASID support and enable status. A in iommufd_get_hw_info()
1529 * non-zero dev->iommu->max_pasids means PASID is supported and in iommufd_get_hw_info()
1533 if (idev->dev->iommu->max_pasids) { in iommufd_get_hw_info()
1534 cmd->out_max_pasid_log2 = ilog2(idev->dev->iommu->max_pasids); in iommufd_get_hw_info()
1536 if (dev_is_pci(idev->dev)) { in iommufd_get_hw_info()
1537 struct pci_dev *pdev = to_pci_dev(idev->dev); in iommufd_get_hw_info()
1546 cmd->out_capabilities |= in iommufd_get_hw_info()
1549 cmd->out_capabilities |= in iommufd_get_hw_info()
1558 iommufd_put_object(ucmd->ictx, &idev->obj); in iommufd_get_hw_info()