1 // SPDX-License-Identifier: GPL-2.0
26  * struct msi_device_data - MSI per device data
40  * struct msi_ctrl - MSI internal management control structure
45  *		than the range due to PCI/multi-MSI.
55 #define MSI_XA_MAX_INDEX	(ULONG_MAX - 1)
65  * msi_alloc_desc - Allocate an initialized msi_desc
83 	desc->dev = dev;  in msi_alloc_desc()
84 	desc->nvec_used = nvec;  in msi_alloc_desc()
86 		desc->affinity = kmemdup_array(affinity, nvec, sizeof(*desc->affinity), GFP_KERNEL);  in msi_alloc_desc()
87 		if (!desc->affinity) {  in msi_alloc_desc()
97 	kfree(desc->affinity);  in msi_free_desc()
104 	struct msi_device_data *md = dev->msi.data;  in msi_insert_desc()
105 	struct xarray *xa = &md->__domains[domid].store;  in msi_insert_desc()
112 		struct xa_limit limit = { .min = 0, .max = hwsize - 1 };  in msi_insert_desc()
120 		desc->msi_index = index;  in msi_insert_desc()
124 			ret = -ERANGE;  in msi_insert_desc()
128 		desc->msi_index = index;  in msi_insert_desc()
140  * msi_domain_insert_msi_desc - Allocate and initialize a MSI descriptor and
141  *				insert it at @init_desc->msi_index
154 	lockdep_assert_held(&dev->msi.data->mutex);  in msi_domain_insert_msi_desc()
156 	desc = msi_alloc_desc(dev, init_desc->nvec_used, init_desc->affinity);  in msi_domain_insert_msi_desc()
158 		return -ENOMEM;  in msi_domain_insert_msi_desc()
161 	desc->pci = init_desc->pci;  in msi_domain_insert_msi_desc()
163 	return msi_insert_desc(dev, desc, domid, init_desc->msi_index);  in msi_domain_insert_msi_desc()
172 		return !desc->irq;  in msi_desc_match()
174 		return !!desc->irq;  in msi_desc_match()
184 	if (WARN_ON_ONCE(ctrl->domid >= MSI_MAX_DEVICE_IRQDOMAINS ||  in msi_ctrl_valid()
185 			 (dev->msi.domain &&  in msi_ctrl_valid()
186 			  !dev->msi.data->__domains[ctrl->domid].domain)))  in msi_ctrl_valid()
189 	hwsize = msi_domain_get_hwsize(dev, ctrl->domid);  in msi_ctrl_valid()
190 	if (WARN_ON_ONCE(ctrl->first > ctrl->last ||  in msi_ctrl_valid()
191 			 ctrl->first >= hwsize ||  in msi_ctrl_valid()
192 			 ctrl->last >= hwsize))  in msi_ctrl_valid()
203 	lockdep_assert_held(&dev->msi.data->mutex);  in msi_domain_free_descs()
208 	xa = &dev->msi.data->__domains[ctrl->domid].store;  in msi_domain_free_descs()
209 	xa_for_each_range(xa, idx, desc, ctrl->first, ctrl->last) {  in msi_domain_free_descs()
220  * msi_domain_free_msi_descs_range - Free a range of MSI descriptors of a device in an irqdomain
239  * msi_domain_add_simple_msi_descs - Allocate and initialize MSI descriptors
251 	lockdep_assert_held(&dev->msi.data->mutex);  in msi_domain_add_simple_msi_descs()
254 		return -EINVAL;  in msi_domain_add_simple_msi_descs()
256 	for (idx = ctrl->first; idx <= ctrl->last; idx++) {  in msi_domain_add_simple_msi_descs()
260 		ret = msi_insert_desc(dev, desc, ctrl->domid, idx);  in msi_domain_add_simple_msi_descs()
267 	ret = -ENOMEM;  in msi_domain_add_simple_msi_descs()
275 	*msg = entry->msg;  in __get_cached_msi_msg()
293 		WARN_ON_ONCE(!xa_empty(&md->__domains[i].store));  in msi_device_data_release()
294 		xa_destroy(&md->__domains[i].store);  in msi_device_data_release()
296 	dev->msi.data = NULL;  in msi_device_data_release()
300  * msi_setup_device_data - Setup MSI device data
314 	if (dev->msi.data)  in msi_setup_device_data()
319 		return -ENOMEM;  in msi_setup_device_data()
328 		xa_init_flags(&md->__domains[i].store, XA_FLAGS_ALLOC);  in msi_setup_device_data()
336 	if (dev->msi.domain && !irq_domain_is_msi_parent(dev->msi.domain))  in msi_setup_device_data()
337 		md->__domains[MSI_DEFAULT_DOMAIN].domain = dev->msi.domain;  in msi_setup_device_data()
339 	mutex_init(&md->mutex);  in msi_setup_device_data()
340 	dev->msi.data = md;  in msi_setup_device_data()
346  * msi_lock_descs - Lock the MSI descriptor storage of a device
351 	mutex_lock(&dev->msi.data->mutex);  in msi_lock_descs()
356  * msi_unlock_descs - Unlock the MSI descriptor storage of a device
362 	dev->msi.data->__iter_idx = MSI_XA_MAX_INDEX;  in msi_unlock_descs()
363 	mutex_unlock(&dev->msi.data->mutex);  in msi_unlock_descs()
370 	struct xarray *xa = &md->__domains[domid].store;  in msi_find_desc()
373 	xa_for_each_start(xa, md->__iter_idx, desc, md->__iter_idx) {  in msi_find_desc()
377 	md->__iter_idx = MSI_XA_MAX_INDEX;  in msi_find_desc()
382  * msi_domain_first_desc - Get the first MSI descriptor of an irqdomain associated to a device
396 	struct msi_device_data *md = dev->msi.data;  in msi_domain_first_desc()
401 	lockdep_assert_held(&md->mutex);  in msi_domain_first_desc()
403 	md->__iter_idx = 0;  in msi_domain_first_desc()
409  * msi_next_desc - Get the next MSI descriptor of a device
425 	struct msi_device_data *md = dev->msi.data;  in msi_next_desc()
430 	lockdep_assert_held(&md->mutex);  in msi_next_desc()
432 	if (md->__iter_idx >= (unsigned long)MSI_MAX_INDEX)  in msi_next_desc()
435 	md->__iter_idx++;  in msi_next_desc()
441  * msi_domain_get_virq - Lookup the Linux interrupt number for a MSI index on a interrupt domain
444  * @index:	MSI interrupt index to look for (0-based)
455 	if (!dev->msi.data)  in msi_domain_get_virq()
463 		pcimsi = to_pci_dev(dev)->msi_enabled;  in msi_domain_get_virq()
466 	xa = &dev->msi.data->__domains[domid].store;  in msi_domain_get_virq()
468 	if (desc && desc->irq) {  in msi_domain_get_virq()
470 		 * PCI-MSI has only one descriptor for multiple interrupts.  in msi_domain_get_virq()
471 		 * PCI-MSIX and platform MSI use a descriptor per  in msi_domain_get_virq()
475 			if (index < desc->nvec_used)  in msi_domain_get_virq()
476 				ret = desc->irq + index;  in msi_domain_get_virq()
478 			ret = desc->irq;  in msi_domain_get_virq()
506 	bool is_msix = dev_is_pci(dev) ? to_pci_dev(dev)->msix_enabled : false;  in msi_mode_show()
513 	struct device_attribute *attrs = desc->sysfs_attrs;  in msi_sysfs_remove_desc()
519 	desc->sysfs_attrs = NULL;  in msi_sysfs_remove_desc()
520 	for (i = 0; i < desc->nvec_used; i++) {  in msi_sysfs_remove_desc()
522 			sysfs_remove_file_from_group(&dev->kobj, &attrs[i].attr, msi_irqs_group.name);  in msi_sysfs_remove_desc()
533 	attrs = kcalloc(desc->nvec_used, sizeof(*attrs), GFP_KERNEL);  in msi_sysfs_populate_desc()
535 		return -ENOMEM;  in msi_sysfs_populate_desc()
537 	desc->sysfs_attrs = attrs;  in msi_sysfs_populate_desc()
538 	for (i = 0; i < desc->nvec_used; i++) {  in msi_sysfs_populate_desc()
540 		attrs[i].attr.name = kasprintf(GFP_KERNEL, "%d", desc->irq + i);  in msi_sysfs_populate_desc()
542 			ret = -ENOMEM;  in msi_sysfs_populate_desc()
549 		ret = sysfs_add_file_to_group(&dev->kobj, &attrs[i].attr, msi_irqs_group.name);  in msi_sysfs_populate_desc()
564  * msi_device_populate_sysfs - Populate msi_irqs sysfs entries for a device
573 		if (desc->sysfs_attrs)  in msi_device_populate_sysfs()
583  * msi_device_destroy_sysfs - Destroy msi_irqs sysfs entries for a device
605 	lockdep_assert_held(&dev->msi.data->mutex);  in msi_get_device_domain()
610 	domain = dev->msi.data->__domains[domid].domain;  in msi_get_device_domain()
627 		info = domain->host_data;  in msi_domain_get_hwsize()
628 		return info->hwsize;  in msi_domain_get_hwsize()
637 	data->chip->irq_write_msi_msg(data, msg);  in irq_chip_write_msi_msg()
642 	struct msi_domain_info *info = domain->host_data;  in msi_check_level()
646 	 * not advertized that it is level-capable, signal the breakage.  in msi_check_level()
648 	WARN_ON(!((info->flags & MSI_FLAG_LEVEL_CAPABLE) &&  in msi_check_level()
649 		  (info->chip->flags & IRQCHIP_SUPPORTS_LEVEL_MSI)) &&  in msi_check_level()
654  * msi_domain_set_affinity - Generic affinity setter function for MSI domains
667 	struct irq_data *parent = irq_data->parent_data;  in msi_domain_set_affinity()
671 	ret = parent->chip->irq_set_affinity(parent, mask, force);  in msi_domain_set_affinity()
674 		msi_check_level(irq_data->domain, msg);  in msi_domain_set_affinity()
687 	msi_check_level(irq_data->domain, msg);  in msi_domain_activate()
704 	struct msi_domain_info *info = domain->host_data;  in msi_domain_alloc()
705 	struct msi_domain_ops *ops = info->ops;  in msi_domain_alloc()
706 	irq_hw_number_t hwirq = ops->get_hwirq(info, arg);  in msi_domain_alloc()
710 		return -EEXIST;  in msi_domain_alloc()
712 	if (domain->parent) {  in msi_domain_alloc()
719 		ret = ops->msi_init(domain, info, virq + i, hwirq + i, arg);  in msi_domain_alloc()
721 			if (ops->msi_free) {  in msi_domain_alloc()
722 				for (i--; i >= 0; i--)  in msi_domain_alloc()
723 					ops->msi_free(domain, info, virq + i);  in msi_domain_alloc()
736 	struct msi_domain_info *info = domain->host_data;  in msi_domain_free()
739 	if (info->ops->msi_free) {  in msi_domain_free()
741 			info->ops->msi_free(domain, info, virq + i);  in msi_domain_free()
749 	struct msi_domain_info *info = domain->host_data;  in msi_domain_translate()
755 	if (!info->ops->msi_translate)  in msi_domain_translate()
756 		return -ENOTSUPP;  in msi_domain_translate()
757 	return info->ops->msi_translate(domain, fwspec, hwirq, type);  in msi_domain_translate()
769 	seq_printf(m, "\n%*saddress_hi: 0x%08x", ind + 1, "", desc->msg.address_hi);  in msi_domain_debug_show()
770 	seq_printf(m, "\n%*saddress_lo: 0x%08x", ind + 1, "", desc->msg.address_lo);  in msi_domain_debug_show()
771 	seq_printf(m, "\n%*smsg_data:   0x%08x\n", ind + 1, "", desc->msg.data);  in msi_domain_debug_show()
789 	return arg->hwirq;  in msi_domain_ops_get_hwirq()
802 	arg->desc = desc;  in msi_domain_ops_set_desc()
810 	irq_domain_set_hwirq_and_chip(domain, virq, hwirq, info->chip,  in msi_domain_ops_init()
811 				      info->chip_data);  in msi_domain_ops_init()
812 	if (info->handler && info->handler_name) {  in msi_domain_ops_init()
813 		__irq_set_handler(virq, info->handler, 0, info->handler_name);  in msi_domain_ops_init()
814 		if (info->handler_data)  in msi_domain_ops_init()
815 			irq_set_handler_data(virq, info->handler_data);  in msi_domain_ops_init()
829 	struct msi_domain_ops *ops = info->ops;  in msi_domain_update_dom_ops()
832 		info->ops = &msi_domain_ops_default;  in msi_domain_update_dom_ops()
836 	if (!(info->flags & MSI_FLAG_USE_DEF_DOM_OPS))  in msi_domain_update_dom_ops()
839 	if (ops->get_hwirq == NULL)  in msi_domain_update_dom_ops()
840 		ops->get_hwirq = msi_domain_ops_default.get_hwirq;  in msi_domain_update_dom_ops()
841 	if (ops->msi_init == NULL)  in msi_domain_update_dom_ops()
842 		ops->msi_init = msi_domain_ops_default.msi_init;  in msi_domain_update_dom_ops()
843 	if (ops->msi_prepare == NULL)  in msi_domain_update_dom_ops()
844 		ops->msi_prepare = msi_domain_ops_default.msi_prepare;  in msi_domain_update_dom_ops()
845 	if (ops->set_desc == NULL)  in msi_domain_update_dom_ops()
846 		ops->set_desc = msi_domain_ops_default.set_desc;  in msi_domain_update_dom_ops()
851 	struct irq_chip *chip = info->chip;  in msi_domain_update_chip_ops()
853 	BUG_ON(!chip || !chip->irq_mask || !chip->irq_unmask);  in msi_domain_update_chip_ops()
854 	if (!chip->irq_set_affinity && !(info->flags & MSI_FLAG_NO_AFFINITY))  in msi_domain_update_chip_ops()
855 		chip->irq_set_affinity = msi_domain_set_affinity;  in msi_domain_update_chip_ops()
865 	if (info->hwsize > MSI_XA_DOMAIN_SIZE)  in __msi_create_irq_domain()
873 	if (!info->hwsize)  in __msi_create_irq_domain()
874 		info->hwsize = MSI_XA_DOMAIN_SIZE;  in __msi_create_irq_domain()
877 	if (info->flags & MSI_FLAG_USE_DEF_CHIP_OPS)  in __msi_create_irq_domain()
884 		irq_domain_update_bus_token(domain, info->bus_token);  in __msi_create_irq_domain()
885 		if (info->flags & MSI_FLAG_PARENT_PM_DEV)  in __msi_create_irq_domain()
886 			domain->pm_dev = parent->pm_dev;  in __msi_create_irq_domain()
893  * msi_create_irq_domain - Create an MSI interrupt domain
908  * msi_parent_init_dev_msi_info - Delegate initialization of device MSI info down
925  * That's trivial for a simple parent->child relationship, but it gets
926  * interesting with an intermediate domain: root->parent->child.  The
938 	struct irq_domain *parent = domain->parent;  in msi_parent_init_dev_msi_info()
940 	if (WARN_ON_ONCE(!parent || !parent->msi_parent_ops ||  in msi_parent_init_dev_msi_info()
941 			 !parent->msi_parent_ops->init_dev_msi_info))  in msi_parent_init_dev_msi_info()
944 	return parent->msi_parent_ops->init_dev_msi_info(dev, parent, msi_parent_domain,  in msi_parent_init_dev_msi_info()
949  * msi_create_device_irq_domain - Create a device MSI interrupt domain
966  * composed by: "$(PREFIX)$(CHIPNAME)-$(DEVNAME)"
977  * PCI-MSI-0000:00:1c.0     0-edge  Parent domain has no prefix
978  * IR-PCI-MSI-0000:00:1c.4  0-edge  Same with interrupt remapping prefix 'IR-'
980  * IR-PCI-MSIX-0000:3d:00.0 0-edge  Hardware interrupt numbers reflect
981  * IR-PCI-MSIX-0000:3d:00.0 1-edge  the real MSI-X index on that device
982  * IR-PCI-MSIX-0000:3d:00.0 2-edge
1000 	struct irq_domain *domain, *parent = dev->msi.domain;  in msi_create_device_irq_domain()
1015 	bundle->info.hwsize = hwsize;  in msi_create_device_irq_domain()
1016 	bundle->info.chip = &bundle->chip;  in msi_create_device_irq_domain()
1017 	bundle->info.ops = &bundle->ops;  in msi_create_device_irq_domain()
1018 	bundle->info.data = domain_data;  in msi_create_device_irq_domain()
1019 	bundle->info.chip_data = chip_data;  in msi_create_device_irq_domain()
1021 	pops = parent->msi_parent_ops;  in msi_create_device_irq_domain()
1022 	snprintf(bundle->name, sizeof(bundle->name), "%s%s-%s",  in msi_create_device_irq_domain()
1023 		 pops->prefix ? : "", bundle->chip.name, dev_name(dev));  in msi_create_device_irq_domain()
1024 	bundle->chip.name = bundle->name;  in msi_create_device_irq_domain()
1027 	 * Using the device firmware node is required for wire to MSI  in msi_create_device_irq_domain()
1034 	if (bundle->info.flags & MSI_FLAG_USE_DEV_FWNODE)  in msi_create_device_irq_domain()
1035 		fwnode = dev->fwnode;  in msi_create_device_irq_domain()
1037 		fwnode = fwnalloced = irq_domain_alloc_named_fwnode(bundle->name);  in msi_create_device_irq_domain()
1050 	if (!pops->init_dev_msi_info(dev, parent, parent, &bundle->info))  in msi_create_device_irq_domain()
1053 	domain = __msi_create_irq_domain(fwnode, &bundle->info, IRQ_DOMAIN_FLAG_MSI_DEVICE, parent);  in msi_create_device_irq_domain()
1057 	domain->dev = dev;  in msi_create_device_irq_domain()
1058 	dev->msi.data->__domains[domid].domain = domain;  in msi_create_device_irq_domain()
1072  * msi_remove_device_irq_domain - Free a device MSI interrupt domain
1089 	dev->msi.data->__domains[domid].domain = NULL;  in msi_remove_device_irq_domain()
1090 	info = domain->host_data;  in msi_remove_device_irq_domain()
1092 		fwnode = domain->fwnode;  in msi_remove_device_irq_domain()
1102  * msi_match_device_irq_domain - Match a device irq domain against a bus token
1119 		info = domain->host_data;  in msi_match_device_irq_domain()
1120 		ret = info->bus_token == bus_token;  in msi_match_device_irq_domain()
1129 	struct msi_domain_info *info = domain->host_data;  in msi_domain_prepare_irqs()
1130 	struct msi_domain_ops *ops = info->ops;  in msi_domain_prepare_irqs()
1132 	return ops->msi_prepare(domain, dev, nvec, arg);  in msi_domain_prepare_irqs()
1152 	switch(domain->bus_token) {  in msi_check_reservation_mode()
1162 	if (!(info->flags & MSI_FLAG_MUST_REACTIVATE))  in msi_check_reservation_mode()
1165 	if (info->flags & MSI_FLAG_NO_MASK)  in msi_check_reservation_mode()
1173 	return desc->pci.msi_attrib.is_msix || desc->pci.msi_attrib.can_mask;  in msi_check_reservation_mode()
1179 	switch(domain->bus_token) {  in msi_handle_pci_fail()
1188 		return -ENOSPC;  in msi_handle_pci_fail()
1192 	if (desc->nvec_used > 1)  in msi_handle_pci_fail()
1196 	return allocated ? allocated : -ENOSPC;  in msi_handle_pci_fail()
1215 		 * different way by using a catch-all vector.  in msi_init_virq()
1244 	struct xarray *xa = &dev->msi.data->__domains[ctrl->domid].store;  in __msi_domain_alloc_irqs()
1245 	struct msi_domain_info *info = domain->host_data;  in __msi_domain_alloc_irqs()
1246 	struct msi_domain_ops *ops = info->ops;  in __msi_domain_alloc_irqs()
1253 	ret = msi_domain_prepare_irqs(domain, dev, ctrl->nirqs, &arg);  in __msi_domain_alloc_irqs()
1262 	if (info->flags & MSI_FLAG_ACTIVATE_EARLY)  in __msi_domain_alloc_irqs()
1272 	xa_for_each_range(xa, idx, desc, ctrl->first, ctrl->last) {  in __msi_domain_alloc_irqs()
1276 		/* This should return -ECONFUSED... */  in __msi_domain_alloc_irqs()
1277 		if (WARN_ON_ONCE(allocated >= ctrl->nirqs))  in __msi_domain_alloc_irqs()
1278 			return -EINVAL;  in __msi_domain_alloc_irqs()
1280 		if (ops->prepare_desc)  in __msi_domain_alloc_irqs()
1281 			ops->prepare_desc(domain, &arg, desc);  in __msi_domain_alloc_irqs()
1283 		ops->set_desc(&arg, desc);  in __msi_domain_alloc_irqs()
1285 		virq = __irq_domain_alloc_irqs(domain, -1, desc->nvec_used,  in __msi_domain_alloc_irqs()
1287 					       desc->affinity);  in __msi_domain_alloc_irqs()
1291 		for (i = 0; i < desc->nvec_used; i++) {  in __msi_domain_alloc_irqs()
1298 		if (info->flags & MSI_FLAG_DEV_SYSFS) {  in __msi_domain_alloc_irqs()
1312 	if (!(info->flags & MSI_FLAG_ALLOC_SIMPLE_MSI_DESCS))  in msi_domain_alloc_simple_msi_descs()
1326 		return -EINVAL;  in __msi_domain_alloc_locked()
1328 	domain = msi_get_device_domain(dev, ctrl->domid);  in __msi_domain_alloc_locked()
1330 		return -ENODEV;  in __msi_domain_alloc_locked()
1332 	info = domain->host_data;  in __msi_domain_alloc_locked()
1338 	ops = info->ops;  in __msi_domain_alloc_locked()
1339 	if (ops->domain_alloc_irqs)  in __msi_domain_alloc_locked()
1340 		return ops->domain_alloc_irqs(domain, dev, ctrl->nirqs);  in __msi_domain_alloc_locked()
1355  * msi_domain_alloc_irqs_range_locked - Allocate interrupts from a MSI interrupt domain
1375 		.nirqs	= last + 1 - first,  in msi_domain_alloc_irqs_range_locked()
1382  * msi_domain_alloc_irqs_range - Allocate interrupts from a MSI interrupt domain
1404  * msi_domain_alloc_irqs_all_locked - Allocate all interrupts from a MSI interrupt domain
1422 		.last	= msi_domain_get_hwsize(dev, domid) - 1,  in msi_domain_alloc_irqs_all_locked()
1442 		map.index = -ENODEV;  in __msi_domain_alloc_irq_at()
1448 		map.index = -ENOMEM;  in __msi_domain_alloc_irq_at()
1453 		desc->data.icookie = *icookie;  in __msi_domain_alloc_irq_at()
1461 	ctrl.first = ctrl.last = desc->msi_index;  in __msi_domain_alloc_irq_at()
1468 		map.index = desc->msi_index;  in __msi_domain_alloc_irq_at()
1469 		map.virq = desc->irq;  in __msi_domain_alloc_irq_at()
1475  * msi_domain_alloc_irq_at - Allocate an interrupt from a MSI interrupt domain at
1476  *			     a given index - or at the next free index
1485  *		non-NULL the content of the cookie is stored in msi_desc::data.
1486  *		Must be NULL for MSI-X allocations
1512  * msi_device_domain_alloc_wired - Allocate a "wired" interrupt on @domain
1517  * This weirdness supports wire to MSI controllers like MBIGEN.
1520  * irq_create_fwspec_mapping(). As the wire to MSI domain is sparse, but
1521  * sized in firmware, the hardware interrupt number cannot be used as MSI
1537 	struct device *dev = domain->dev;  in msi_device_domain_alloc_wired()
1540 	if (WARN_ON_ONCE(!dev || domain->bus_token != DOMAIN_BUS_WIRED_TO_MSI))  in msi_device_domain_alloc_wired()
1541 		return -EINVAL;  in msi_device_domain_alloc_wired()
1547 		map.index = -EINVAL;  in msi_device_domain_alloc_wired()
1558 	struct xarray *xa = &dev->msi.data->__domains[ctrl->domid].store;  in __msi_domain_free_irqs()
1559 	struct msi_domain_info *info = domain->host_data;  in __msi_domain_free_irqs()
1565 	xa_for_each_range(xa, idx, desc, ctrl->first, ctrl->last) {  in __msi_domain_free_irqs()
1571 		for (i = 0; i < desc->nvec_used; i++) {  in __msi_domain_free_irqs()
1572 			irqd = irq_domain_get_irq_data(domain, desc->irq + i);  in __msi_domain_free_irqs()
1577 		irq_domain_free_irqs(desc->irq, desc->nvec_used);  in __msi_domain_free_irqs()
1578 		if (info->flags & MSI_FLAG_DEV_SYSFS)  in __msi_domain_free_irqs()
1580 		desc->irq = 0;  in __msi_domain_free_irqs()
1593 	domain = msi_get_device_domain(dev, ctrl->domid);  in msi_domain_free_locked()
1597 	info = domain->host_data;  in msi_domain_free_locked()
1598 	ops = info->ops;  in msi_domain_free_locked()
1600 	if (ops->domain_free_irqs)  in msi_domain_free_locked()
1601 		ops->domain_free_irqs(domain, dev);  in msi_domain_free_locked()
1605 	if (ops->msi_post_free)  in msi_domain_free_locked()
1606 		ops->msi_post_free(domain, dev);  in msi_domain_free_locked()
1608 	if (info->flags & MSI_FLAG_FREE_MSI_DESCS)  in msi_domain_free_locked()
1613  * msi_domain_free_irqs_range_locked - Free a range of interrupts from a MSI interrupt domain
1633  * msi_domain_free_irqs_range - Free a range of interrupts from a MSI interrupt domain
1651  * msi_domain_free_irqs_all_locked - Free all interrupts from a MSI interrupt domain
1664 					  msi_domain_get_hwsize(dev, domid) - 1);  in msi_domain_free_irqs_all_locked()
1668  * msi_domain_free_irqs_all - Free all interrupts from a MSI interrupt domain
1682  * msi_device_domain_free_wired - Free a wired interrupt in @domain
1692 	struct device *dev = domain->dev;  in msi_device_domain_free_wired()
1694 	if (WARN_ON_ONCE(!dev || !desc || domain->bus_token != DOMAIN_BUS_WIRED_TO_MSI))  in msi_device_domain_free_wired()
1699 		msi_domain_free_irqs_range_locked(dev, MSI_DEFAULT_DOMAIN, desc->msi_index,  in msi_device_domain_free_wired()
1700 						  desc->msi_index);  in msi_device_domain_free_wired()
1706  * msi_get_domain_info - Get the MSI interrupt domain info for @domain
1709  * Return: the pointer to the msi_domain_info stored in @domain->host_data.
1713 	return (struct msi_domain_info *)domain->host_data;  in msi_get_domain_info()
1717  * msi_device_has_isolated_msi - True if the device has isolated MSI
1736 	for (; domain; domain = domain->parent)  in msi_device_has_isolated_msi()
1737 		if (domain->flags & IRQ_DOMAIN_FLAG_ISOLATED_MSI)  in msi_device_has_isolated_msi()