68 	struct iommufd_fault *fault = hwpt->fault;  in iommufd_auto_response_faults()  local
73 	if (!fault)  in iommufd_auto_response_faults()
77 	mutex_lock(&fault->mutex);  in iommufd_auto_response_faults()
78 	spin_lock(&fault->common.lock);  in iommufd_auto_response_faults()
79 	list_for_each_entry_safe(group, next, &fault->common.deliver, node) {  in iommufd_auto_response_faults()
84 	spin_unlock(&fault->common.lock);  in iommufd_auto_response_faults()
92 	xa_for_each(&fault->response, index, group) {  in iommufd_auto_response_faults()
95 		xa_erase(&fault->response, index);  in iommufd_auto_response_faults()
99 	mutex_unlock(&fault->mutex);  in iommufd_auto_response_faults()
106 	struct iommufd_fault *fault = eventq_to_fault(eventq);  in iommufd_fault_destroy()  local
116 	list_for_each_entry_safe(group, next, &fault->common.deliver, node) {  in iommufd_fault_destroy()
121 	xa_for_each(&fault->response, index, group) {  in iommufd_fault_destroy()
122 		xa_erase(&fault->response, index);  in iommufd_fault_destroy()
126 	xa_destroy(&fault->response);  in iommufd_fault_destroy()
127 	mutex_destroy(&fault->mutex);  in iommufd_fault_destroy()
130 static void iommufd_compose_fault_message(struct iommu_fault *fault,  in iommufd_compose_fault_message()  argument
135 	hwpt_fault->flags = fault->prm.flags;  in iommufd_compose_fault_message()
137 	hwpt_fault->pasid = fault->prm.pasid;  in iommufd_compose_fault_message()
138 	hwpt_fault->grpid = fault->prm.grpid;  in iommufd_compose_fault_message()
139 	hwpt_fault->perm = fault->prm.perm;  in iommufd_compose_fault_message()
140 	hwpt_fault->addr = fault->prm.addr;  in iommufd_compose_fault_message()
145 /* Fetch the first node out of the fault->deliver list */
147 iommufd_fault_deliver_fetch(struct iommufd_fault *fault)  in iommufd_fault_deliver_fetch()  argument
149 	struct list_head *list = &fault->common.deliver;  in iommufd_fault_deliver_fetch()
152 	spin_lock(&fault->common.lock);  in iommufd_fault_deliver_fetch()
157 	spin_unlock(&fault->common.lock);  in iommufd_fault_deliver_fetch()
161 /* Restore a node back to the head of the fault->deliver list */
162 static void iommufd_fault_deliver_restore(struct iommufd_fault *fault,  in iommufd_fault_deliver_restore()  argument
165 	spin_lock(&fault->common.lock);  in iommufd_fault_deliver_restore()
166 	list_add(&group->node, &fault->common.deliver);  in iommufd_fault_deliver_restore()
167 	spin_unlock(&fault->common.lock);  in iommufd_fault_deliver_restore()
175 	struct iommufd_fault *fault = eventq_to_fault(eventq);  in iommufd_fault_fops_read()  local
186 	mutex_lock(&fault->mutex);  in iommufd_fault_fops_read()
187 	while ((group = iommufd_fault_deliver_fetch(fault))) {  in iommufd_fault_fops_read()
190 			iommufd_fault_deliver_restore(fault, group);  in iommufd_fault_fops_read()
194 		rc = xa_alloc(&fault->response, &group->cookie, group,  in iommufd_fault_fops_read()
197 			iommufd_fault_deliver_restore(fault, group);  in iommufd_fault_fops_read()
203 			iommufd_compose_fault_message(&iopf->fault,  in iommufd_fault_fops_read()
207 				xa_erase(&fault->response, group->cookie);  in iommufd_fault_fops_read()
208 				iommufd_fault_deliver_restore(fault, group);  in iommufd_fault_fops_read()
215 	mutex_unlock(&fault->mutex);  in iommufd_fault_fops_read()
225 	struct iommufd_fault *fault = eventq_to_fault(eventq);  in iommufd_fault_fops_write()  local
234 	mutex_lock(&fault->mutex);  in iommufd_fault_fops_write()
250 		group = xa_erase(&fault->response, response.cookie);  in iommufd_fault_fops_write()
260 	mutex_unlock(&fault->mutex);  in iommufd_fault_fops_write()
469 	struct iommufd_fault *fault;  in iommufd_fault_alloc()  local
476 	fault = __iommufd_object_alloc(ucmd->ictx, fault, IOMMUFD_OBJ_FAULT,  in iommufd_fault_alloc()
478 	if (IS_ERR(fault))  in iommufd_fault_alloc()
479 		return PTR_ERR(fault);  in iommufd_fault_alloc()
481 	xa_init_flags(&fault->response, XA_FLAGS_ALLOC1);  in iommufd_fault_alloc()
482 	mutex_init(&fault->mutex);  in iommufd_fault_alloc()
484 	fdno = iommufd_eventq_init(&fault->common, "[iommufd-pgfault]",  in iommufd_fault_alloc()
491 	cmd->out_fault_id = fault->common.obj.id;  in iommufd_fault_alloc()
497 	iommufd_object_finalize(ucmd->ictx, &fault->common.obj);  in iommufd_fault_alloc()
499 	fd_install(fdno, fault->common.filep);  in iommufd_fault_alloc()
504 	fput(fault->common.filep);  in iommufd_fault_alloc()
506 	iommufd_object_abort_and_destroy(ucmd->ictx, &fault->common.obj);  in iommufd_fault_alloc()
514 	struct iommufd_fault *fault;  in iommufd_fault_iopf_handler()  local
517 	fault = hwpt->fault;  in iommufd_fault_iopf_handler()
519 	spin_lock(&fault->common.lock);  in iommufd_fault_iopf_handler()
520 	list_add_tail(&group->node, &fault->common.deliver);  in iommufd_fault_iopf_handler()
521 	spin_unlock(&fault->common.lock);  in iommufd_fault_iopf_handler()
523 	wake_up_interruptible(&fault->common.wait_queue);  in iommufd_fault_iopf_handler()