15 #include <linux/msi.h>
23  * struct msi_ctrl - MSI internal management control structure
28  *		than the range due to PCI/multi-MSI.
87 	struct msi_device_data *md = dev->msi.data;  in msi_insert_desc()
123  * msi_domain_insert_msi_desc - Allocate and initialize a MSI descriptor and
128  * @init_desc:	Pointer to an MSI descriptor to initialize the new descriptor
137 	lockdep_assert_held(&dev->msi.data->mutex);  in msi_domain_insert_msi_desc()
168 			 (dev->msi.domain &&  in msi_ctrl_valid()
169 			  !dev->msi.data->__domains[ctrl->domid].domain)))  in msi_ctrl_valid()
186 	lockdep_assert_held(&dev->msi.data->mutex);  in msi_domain_free_descs()
191 	xa = &dev->msi.data->__domains[ctrl->domid].store;  in msi_domain_free_descs()
203  * msi_domain_free_msi_descs_range - Free a range of MSI descriptors of a device in an irqdomain
222  * msi_domain_add_simple_msi_descs - Allocate and initialize MSI descriptors
234 	lockdep_assert_held(&dev->msi.data->mutex);  in msi_domain_add_simple_msi_descs()
279 	dev->msi.data = NULL;  in msi_device_data_release()
283  * msi_setup_device_data - Setup MSI device data
284  * @dev:	Device for which MSI device data should be set up
288  * This can be called more than once for @dev. If the MSI device data is
297 	if (dev->msi.data)  in msi_setup_device_data()
314 	 * If @dev::msi::domain is set and is a global MSI domain, copy the  in msi_setup_device_data()
317 	 * architecture specific PCI/MSI support working.  in msi_setup_device_data()
319 	if (dev->msi.domain && !irq_domain_is_msi_parent(dev->msi.domain))  in msi_setup_device_data()
320 		md->__domains[MSI_DEFAULT_DOMAIN].domain = dev->msi.domain;  in msi_setup_device_data()
323 	dev->msi.data = md;  in msi_setup_device_data()
329  * msi_lock_descs - Lock the MSI descriptor storage of a device
334 	mutex_lock(&dev->msi.data->mutex);  in msi_lock_descs()
339  * msi_unlock_descs - Unlock the MSI descriptor storage of a device
345 	dev->msi.data->__iter_idx = MSI_XA_MAX_INDEX;  in msi_unlock_descs()
346 	mutex_unlock(&dev->msi.data->mutex);  in msi_unlock_descs()
365  * msi_domain_first_desc - Get the first MSI descriptor of an irqdomain associated to a device
370  * Must be called with the MSI descriptor mutex held, i.e. msi_lock_descs()
373  * Return: Pointer to the first MSI descriptor matching the search
379 	struct msi_device_data *md = dev->msi.data;  in msi_domain_first_desc()
392  * msi_next_desc - Get the next MSI descriptor of a device
400  * to be done within the same MSI mutex held region.
402  * Return: Pointer to the next MSI descriptor matching the search
408 	struct msi_device_data *md = dev->msi.data;  in msi_next_desc()
424  * msi_domain_get_virq - Lookup the Linux interrupt number for a MSI index on a interrupt domain
427  * @index:	MSI interrupt index to look for (0-based)
438 	if (!dev->msi.data)  in msi_domain_get_virq()
444 	/* This check is only valid for the PCI default MSI domain */  in msi_domain_get_virq()
449 	xa = &dev->msi.data->__domains[domid].store;  in msi_domain_get_virq()
453 		 * PCI-MSI has only one descriptor for multiple interrupts.  in msi_domain_get_virq()
454 		 * PCI-MSIX and platform MSI use a descriptor per  in msi_domain_get_virq()
488 	/* MSI vs. MSIX is per device not per interrupt */  in msi_mode_show()
491 	return sysfs_emit(buf, "%s\n", is_msix ? "msix" : "msi");  in msi_mode_show()
588 	lockdep_assert_held(&dev->msi.data->mutex);  in msi_get_device_domain()
593 	domain = dev->msi.data->__domains[domid].domain;  in msi_get_device_domain()
628 	 * If the MSI provider has messed with the second message and  in msi_check_level()
637  * msi_domain_set_affinity - Generic affinity setter function for MSI domains
642  * Intended to be used by MSI interrupt controllers which are
840  * msi_create_irq_domain - Create an MSI interrupt domain
842  * @info:	MSI domain info
855  * msi_parent_init_dev_msi_info - Delegate initialization of device MSI info down
861  * @msi_child_info:	The MSI domain info of the IRQ_DOMAIN_FLAG_MSI_DEVICE
866  * This is the most complex problem of per device MSI domains and the
896  * msi_create_device_irq_domain - Create a device MSI interrupt domain
899  * @template:		MSI domain info bundle used as template
900  * @hwsize:		Maximum number of MSI table entries (0 if unknown or unlimited)
912  * The domain name and the irq chip name for a MSI device domain are
915  * $PREFIX:   Optional prefix provided by the underlying MSI parent domain
924  * PCI-MSI-0000:00:1c.0     0-edge  Parent domain has no prefix
925  * IR-PCI-MSI-0000:00:1c.4  0-edge  Same with interrupt remapping prefix 'IR-'
928  * IR-PCI-MSIX-0000:3d:00.0 1-edge  the real MSI-X index on that device
935  * The domain pointer is stored in @dev::msi::data::__irqdomains[]. All
939  * in the context of @dev::msi::data freeing, but it can also be
947 	struct irq_domain *domain, *parent = dev->msi.domain;  in msi_create_device_irq_domain()
993 	dev->msi.data->__domains[domid].domain = domain;  in msi_create_device_irq_domain()
1007  * msi_remove_device_irq_domain - Free a device MSI interrupt domain
1024 	dev->msi.data->__domains[domid].domain = NULL;  in msi_remove_device_irq_domain()
1095 	xa = &dev->msi.data->__domains[ctrl.domid].store;  in msi_domain_populate_irqs()
1136 	xa = &dev->msi.data->__domains[ctrl.domid].store;  in msi_domain_depopulate_descs()
1144  * dummy vector to the device. If the PCI/MSI device does not support
1150  * used. For now reservation mode is restricted to PCI/MSI.
1175 	 * Checking the first MSI descriptor is sufficient. MSIX supports  in msi_check_reservation_mode()
1176 	 * masking and MSI does so when the can_mask attribute is set.  in msi_check_reservation_mode()
1197 	/* Let a failed PCI multi MSI allocation retry */  in msi_handle_pci_fail()
1250 	struct xarray *xa = &dev->msi.data->__domains[ctrl->domid].store;  in __msi_domain_alloc_irqs()
1265 	 * the MSI entries before the PCI layer enables MSI in the  in __msi_domain_alloc_irqs()
1266 	 * card. Otherwise the card latches a random msi message.  in __msi_domain_alloc_irqs()
1361  * msi_domain_alloc_irqs_range_locked - Allocate interrupts from a MSI interrupt domain
1369  * pair. Use this for MSI irqdomains which implement their own descriptor
1388  * msi_domain_alloc_irqs_range - Allocate interrupts from a MSI interrupt domain
1409  * msi_domain_alloc_irqs_all_locked - Allocate all interrupts from a MSI interrupt domain
1416  * This function scans all MSI descriptors of the MSI domain and allocates interrupts
1417  * for all unassigned ones. That function is to be used for MSI domain usage where
1418  * the descriptor allocation is handled at the call site, e.g. PCI/MSI[X].
1435  * msi_domain_alloc_irq_at - Allocate an interrupt from a MSI interrupt domain at
1446  *		Must be NULL for MSI-X allocations
1448  * This requires a MSI interrupt domain which lets the core code manage the
1449  * MSI descriptors.
1509 	struct xarray *xa = &dev->msi.data->__domains[ctrl->domid].store;  in __msi_domain_free_irqs()
1517 		/* Only handle MSI entries which have an interrupt associated */  in __msi_domain_free_irqs()
1564  * msi_domain_free_irqs_range_locked - Free a range of interrupts from a MSI interrupt domain
1584  * msi_domain_free_irqs_range - Free a range of interrupts from a MSI interrupt domain
1601  * msi_domain_free_irqs_all_locked - Free all interrupts from a MSI interrupt domain
1608  * pair. Use this for MSI irqdomains which implement their own vector
1618  * msi_domain_free_irqs_all - Free all interrupts from a MSI interrupt domain
1632  * msi_get_domain_info - Get the MSI interrupt domain info for @domain
1643  * msi_device_has_isolated_msi - True if the device has isolated MSI
1646  * Isolated MSI means that HW modeled by an irq_domain on the path from the
1647  * initiating device to the CPU will validate that the MSI message specifies an
1650  * Currently authorization means the MSI vector is one assigned to the device.
1652  * This is interesting for securing VFIO use cases where a rouge MSI (eg created