1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * PCI Message Signaled Interrupt (MSI) - irqdomain support 4 */ 5 #include <linux/acpi_iort.h> 6 #include <linux/irqdomain.h> 7 #include <linux/of_irq.h> 8 9 #include "msi.h" 10 11 int pci_msi_setup_msi_irqs(struct pci_dev *dev, int nvec, int type) 12 { 13 struct irq_domain *domain; 14 15 domain = dev_get_msi_domain(&dev->dev); 16 if (domain && irq_domain_is_hierarchy(domain)) 17 return msi_domain_alloc_irqs_all_locked(&dev->dev, MSI_DEFAULT_DOMAIN, nvec); 18 19 return pci_msi_legacy_setup_msi_irqs(dev, nvec, type); 20 } 21 22 void pci_msi_teardown_msi_irqs(struct pci_dev *dev) 23 { 24 struct irq_domain *domain; 25 26 domain = dev_get_msi_domain(&dev->dev); 27 if (domain && irq_domain_is_hierarchy(domain)) { 28 msi_domain_free_irqs_all_locked(&dev->dev, MSI_DEFAULT_DOMAIN); 29 } else { 30 pci_msi_legacy_teardown_msi_irqs(dev); 31 msi_free_msi_descs(&dev->dev); 32 } 33 } 34 35 /** 36 * pci_msi_domain_write_msg - Helper to write MSI message to PCI config space 37 * @irq_data: Pointer to interrupt data of the MSI interrupt 38 * @msg: Pointer to the message 39 */ 40 static void pci_msi_domain_write_msg(struct irq_data *irq_data, struct msi_msg *msg) 41 { 42 struct msi_desc *desc = irq_data_get_msi_desc(irq_data); 43 44 /* 45 * For MSI-X desc->irq is always equal to irq_data->irq. For 46 * MSI only the first interrupt of MULTI MSI passes the test. 47 */ 48 if (desc->irq == irq_data->irq) 49 __pci_write_msi_msg(desc, msg); 50 } 51 52 /** 53 * pci_msi_domain_calc_hwirq - Generate a unique ID for an MSI source 54 * @desc: Pointer to the MSI descriptor 55 * 56 * The ID number is only used within the irqdomain. 57 */ 58 static irq_hw_number_t pci_msi_domain_calc_hwirq(struct msi_desc *desc) 59 { 60 struct pci_dev *dev = msi_desc_to_pci_dev(desc); 61 62 return (irq_hw_number_t)desc->msi_index | 63 pci_dev_id(dev) << 11 | 64 ((irq_hw_number_t)(pci_domain_nr(dev->bus) & 0xFFFFFFFF)) << 27; 65 } 66 67 static void pci_msi_domain_set_desc(msi_alloc_info_t *arg, 68 struct msi_desc *desc) 69 { 70 arg->desc = desc; 71 arg->hwirq = pci_msi_domain_calc_hwirq(desc); 72 } 73 74 static struct msi_domain_ops pci_msi_domain_ops_default = { 75 .set_desc = pci_msi_domain_set_desc, 76 }; 77 78 static void pci_msi_domain_update_dom_ops(struct msi_domain_info *info) 79 { 80 struct msi_domain_ops *ops = info->ops; 81 82 if (ops == NULL) { 83 info->ops = &pci_msi_domain_ops_default; 84 } else { 85 if (ops->set_desc == NULL) 86 ops->set_desc = pci_msi_domain_set_desc; 87 } 88 } 89 90 static void pci_msi_domain_update_chip_ops(struct msi_domain_info *info) 91 { 92 struct irq_chip *chip = info->chip; 93 94 BUG_ON(!chip); 95 if (!chip->irq_write_msi_msg) 96 chip->irq_write_msi_msg = pci_msi_domain_write_msg; 97 if (!chip->irq_mask) 98 chip->irq_mask = pci_msi_mask_irq; 99 if (!chip->irq_unmask) 100 chip->irq_unmask = pci_msi_unmask_irq; 101 } 102 103 /** 104 * pci_msi_create_irq_domain - Create a MSI interrupt domain 105 * @fwnode: Optional fwnode of the interrupt controller 106 * @info: MSI domain info 107 * @parent: Parent irq domain 108 * 109 * Updates the domain and chip ops and creates a MSI interrupt domain. 110 * 111 * Returns: 112 * A domain pointer or NULL in case of failure. 113 */ 114 struct irq_domain *pci_msi_create_irq_domain(struct fwnode_handle *fwnode, 115 struct msi_domain_info *info, 116 struct irq_domain *parent) 117 { 118 if (WARN_ON(info->flags & MSI_FLAG_LEVEL_CAPABLE)) 119 info->flags &= ~MSI_FLAG_LEVEL_CAPABLE; 120 121 if (info->flags & MSI_FLAG_USE_DEF_DOM_OPS) 122 pci_msi_domain_update_dom_ops(info); 123 if (info->flags & MSI_FLAG_USE_DEF_CHIP_OPS) 124 pci_msi_domain_update_chip_ops(info); 125 126 /* Let the core code free MSI descriptors when freeing interrupts */ 127 info->flags |= MSI_FLAG_FREE_MSI_DESCS; 128 129 info->flags |= MSI_FLAG_ACTIVATE_EARLY | MSI_FLAG_DEV_SYSFS; 130 if (IS_ENABLED(CONFIG_GENERIC_IRQ_RESERVATION_MODE)) 131 info->flags |= MSI_FLAG_MUST_REACTIVATE; 132 133 /* PCI-MSI is oneshot-safe */ 134 info->chip->flags |= IRQCHIP_ONESHOT_SAFE; 135 /* Let the core update the bus token */ 136 info->bus_token = DOMAIN_BUS_PCI_MSI; 137 138 return msi_create_irq_domain(fwnode, info, parent); 139 } 140 EXPORT_SYMBOL_GPL(pci_msi_create_irq_domain); 141 142 /* 143 * Per device MSI[-X] domain functionality 144 */ 145 static void pci_device_domain_set_desc(msi_alloc_info_t *arg, struct msi_desc *desc) 146 { 147 arg->desc = desc; 148 arg->hwirq = desc->msi_index; 149 } 150 151 static __always_inline void cond_mask_parent(struct irq_data *data) 152 { 153 struct msi_domain_info *info = data->domain->host_data; 154 155 if (unlikely(info->flags & MSI_FLAG_PCI_MSI_MASK_PARENT)) 156 irq_chip_mask_parent(data); 157 } 158 159 static __always_inline void cond_unmask_parent(struct irq_data *data) 160 { 161 struct msi_domain_info *info = data->domain->host_data; 162 163 if (unlikely(info->flags & MSI_FLAG_PCI_MSI_MASK_PARENT)) 164 irq_chip_unmask_parent(data); 165 } 166 167 static void pci_irq_mask_msi(struct irq_data *data) 168 { 169 struct msi_desc *desc = irq_data_get_msi_desc(data); 170 171 pci_msi_mask(desc, BIT(data->irq - desc->irq)); 172 cond_mask_parent(data); 173 } 174 175 static void pci_irq_unmask_msi(struct irq_data *data) 176 { 177 struct msi_desc *desc = irq_data_get_msi_desc(data); 178 179 cond_unmask_parent(data); 180 pci_msi_unmask(desc, BIT(data->irq - desc->irq)); 181 } 182 183 #ifdef CONFIG_GENERIC_IRQ_RESERVATION_MODE 184 # define MSI_REACTIVATE MSI_FLAG_MUST_REACTIVATE 185 #else 186 # define MSI_REACTIVATE 0 187 #endif 188 189 #define MSI_COMMON_FLAGS (MSI_FLAG_FREE_MSI_DESCS | \ 190 MSI_FLAG_ACTIVATE_EARLY | \ 191 MSI_FLAG_DEV_SYSFS | \ 192 MSI_REACTIVATE) 193 194 static const struct msi_domain_template pci_msi_template = { 195 .chip = { 196 .name = "PCI-MSI", 197 .irq_mask = pci_irq_mask_msi, 198 .irq_unmask = pci_irq_unmask_msi, 199 .irq_write_msi_msg = pci_msi_domain_write_msg, 200 .flags = IRQCHIP_ONESHOT_SAFE, 201 }, 202 203 .ops = { 204 .set_desc = pci_device_domain_set_desc, 205 }, 206 207 .info = { 208 .flags = MSI_COMMON_FLAGS | MSI_FLAG_MULTI_PCI_MSI, 209 .bus_token = DOMAIN_BUS_PCI_DEVICE_MSI, 210 }, 211 }; 212 213 static void pci_irq_mask_msix(struct irq_data *data) 214 { 215 pci_msix_mask(irq_data_get_msi_desc(data)); 216 cond_mask_parent(data); 217 } 218 219 static void pci_irq_unmask_msix(struct irq_data *data) 220 { 221 cond_unmask_parent(data); 222 pci_msix_unmask(irq_data_get_msi_desc(data)); 223 } 224 225 static void pci_msix_prepare_desc(struct irq_domain *domain, msi_alloc_info_t *arg, 226 struct msi_desc *desc) 227 { 228 /* Don't fiddle with preallocated MSI descriptors */ 229 if (!desc->pci.mask_base) 230 msix_prepare_msi_desc(to_pci_dev(desc->dev), desc); 231 } 232 233 static const struct msi_domain_template pci_msix_template = { 234 .chip = { 235 .name = "PCI-MSIX", 236 .irq_mask = pci_irq_mask_msix, 237 .irq_unmask = pci_irq_unmask_msix, 238 .irq_write_msi_msg = pci_msi_domain_write_msg, 239 .flags = IRQCHIP_ONESHOT_SAFE, 240 }, 241 242 .ops = { 243 .prepare_desc = pci_msix_prepare_desc, 244 .set_desc = pci_device_domain_set_desc, 245 }, 246 247 .info = { 248 .flags = MSI_COMMON_FLAGS | MSI_FLAG_PCI_MSIX | 249 MSI_FLAG_PCI_MSIX_ALLOC_DYN, 250 .bus_token = DOMAIN_BUS_PCI_DEVICE_MSIX, 251 }, 252 }; 253 254 static bool pci_match_device_domain(struct pci_dev *pdev, enum irq_domain_bus_token bus_token) 255 { 256 return msi_match_device_irq_domain(&pdev->dev, MSI_DEFAULT_DOMAIN, bus_token); 257 } 258 259 static bool pci_create_device_domain(struct pci_dev *pdev, const struct msi_domain_template *tmpl, 260 unsigned int hwsize) 261 { 262 struct irq_domain *domain = dev_get_msi_domain(&pdev->dev); 263 264 if (!domain || !irq_domain_is_msi_parent(domain)) 265 return true; 266 267 return msi_create_device_irq_domain(&pdev->dev, MSI_DEFAULT_DOMAIN, tmpl, 268 hwsize, NULL, NULL); 269 } 270 271 /** 272 * pci_setup_msi_device_domain - Setup a device MSI interrupt domain 273 * @pdev: The PCI device to create the domain on 274 * @hwsize: The maximum number of MSI vectors 275 * 276 * Return: 277 * True when: 278 * - The device does not have a MSI parent irq domain associated, 279 * which keeps the legacy architecture specific and the global 280 * PCI/MSI domain models working 281 * - The MSI domain exists already 282 * - The MSI domain was successfully allocated 283 * False when: 284 * - MSI-X is enabled 285 * - The domain creation fails. 286 * 287 * The created MSI domain is preserved until: 288 * - The device is removed 289 * - MSI is disabled and a MSI-X domain is created 290 */ 291 bool pci_setup_msi_device_domain(struct pci_dev *pdev, unsigned int hwsize) 292 { 293 if (WARN_ON_ONCE(pdev->msix_enabled)) 294 return false; 295 296 if (pci_match_device_domain(pdev, DOMAIN_BUS_PCI_DEVICE_MSI)) 297 return true; 298 if (pci_match_device_domain(pdev, DOMAIN_BUS_PCI_DEVICE_MSIX)) 299 msi_remove_device_irq_domain(&pdev->dev, MSI_DEFAULT_DOMAIN); 300 301 return pci_create_device_domain(pdev, &pci_msi_template, hwsize); 302 } 303 304 /** 305 * pci_setup_msix_device_domain - Setup a device MSI-X interrupt domain 306 * @pdev: The PCI device to create the domain on 307 * @hwsize: The size of the MSI-X vector table 308 * 309 * Return: 310 * True when: 311 * - The device does not have a MSI parent irq domain associated, 312 * which keeps the legacy architecture specific and the global 313 * PCI/MSI domain models working 314 * - The MSI-X domain exists already 315 * - The MSI-X domain was successfully allocated 316 * False when: 317 * - MSI is enabled 318 * - The domain creation fails. 319 * 320 * The created MSI-X domain is preserved until: 321 * - The device is removed 322 * - MSI-X is disabled and a MSI domain is created 323 */ 324 bool pci_setup_msix_device_domain(struct pci_dev *pdev, unsigned int hwsize) 325 { 326 if (WARN_ON_ONCE(pdev->msi_enabled)) 327 return false; 328 329 if (pci_match_device_domain(pdev, DOMAIN_BUS_PCI_DEVICE_MSIX)) 330 return true; 331 if (pci_match_device_domain(pdev, DOMAIN_BUS_PCI_DEVICE_MSI)) 332 msi_remove_device_irq_domain(&pdev->dev, MSI_DEFAULT_DOMAIN); 333 334 return pci_create_device_domain(pdev, &pci_msix_template, hwsize); 335 } 336 337 /** 338 * pci_msi_domain_supports - Check for support of a particular feature flag 339 * @pdev: The PCI device to operate on 340 * @feature_mask: The feature mask to check for (full match) 341 * @mode: If ALLOW_LEGACY this grants the feature when there is no irq domain 342 * associated to the device. If DENY_LEGACY the lack of an irq domain 343 * makes the feature unsupported 344 */ 345 bool pci_msi_domain_supports(struct pci_dev *pdev, unsigned int feature_mask, 346 enum support_mode mode) 347 { 348 struct msi_domain_info *info; 349 struct irq_domain *domain; 350 unsigned int supported; 351 352 domain = dev_get_msi_domain(&pdev->dev); 353 354 if (!domain || !irq_domain_is_hierarchy(domain)) { 355 if (IS_ENABLED(CONFIG_PCI_MSI_ARCH_FALLBACKS)) 356 return mode == ALLOW_LEGACY; 357 return false; 358 } 359 360 if (!irq_domain_is_msi_parent(domain)) { 361 /* 362 * For "global" PCI/MSI interrupt domains the associated 363 * msi_domain_info::flags is the authoritative source of 364 * information. 365 */ 366 info = domain->host_data; 367 supported = info->flags; 368 } else { 369 /* 370 * For MSI parent domains the supported feature set 371 * is available in the parent ops. This makes checks 372 * possible before actually instantiating the 373 * per device domain because the parent is never 374 * expanding the PCI/MSI functionality. 375 */ 376 supported = domain->msi_parent_ops->supported_flags; 377 } 378 379 return (supported & feature_mask) == feature_mask; 380 } 381 382 /* 383 * Users of the generic MSI infrastructure expect a device to have a single ID, 384 * so with DMA aliases we have to pick the least-worst compromise. Devices with 385 * DMA phantom functions tend to still emit MSIs from the real function number, 386 * so we ignore those and only consider topological aliases where either the 387 * alias device or RID appears on a different bus number. We also make the 388 * reasonable assumption that bridges are walked in an upstream direction (so 389 * the last one seen wins), and the much braver assumption that the most likely 390 * case is that of PCI->PCIe so we should always use the alias RID. This echoes 391 * the logic from intel_irq_remapping's set_msi_sid(), which presumably works 392 * well enough in practice; in the face of the horrible PCIe<->PCI-X conditions 393 * for taking ownership all we can really do is close our eyes and hope... 394 */ 395 static int get_msi_id_cb(struct pci_dev *pdev, u16 alias, void *data) 396 { 397 u32 *pa = data; 398 u8 bus = PCI_BUS_NUM(*pa); 399 400 if (pdev->bus->number != bus || PCI_BUS_NUM(alias) != bus) 401 *pa = alias; 402 403 return 0; 404 } 405 406 /** 407 * pci_msi_domain_get_msi_rid - Get the MSI requester id (RID) 408 * @domain: The interrupt domain 409 * @pdev: The PCI device. 410 * 411 * The RID for a device is formed from the alias, with a firmware 412 * supplied mapping applied 413 * 414 * Returns: The RID. 415 */ 416 u32 pci_msi_domain_get_msi_rid(struct irq_domain *domain, struct pci_dev *pdev) 417 { 418 struct device_node *of_node; 419 u32 rid = pci_dev_id(pdev); 420 421 pci_for_each_dma_alias(pdev, get_msi_id_cb, &rid); 422 423 of_node = irq_domain_get_of_node(domain); 424 rid = of_node ? of_msi_map_id(&pdev->dev, of_node, rid) : 425 iort_msi_map_id(&pdev->dev, rid); 426 427 return rid; 428 } 429 430 /** 431 * pci_msi_get_device_domain - Get the MSI domain for a given PCI device 432 * @pdev: The PCI device 433 * 434 * Use the firmware data to find a device-specific MSI domain 435 * (i.e. not one that is set as a default). 436 * 437 * Returns: The corresponding MSI domain or NULL if none has been found. 438 */ 439 struct irq_domain *pci_msi_get_device_domain(struct pci_dev *pdev) 440 { 441 struct irq_domain *dom; 442 u32 rid = pci_dev_id(pdev); 443 444 pci_for_each_dma_alias(pdev, get_msi_id_cb, &rid); 445 dom = of_msi_map_get_device_domain(&pdev->dev, rid, DOMAIN_BUS_PCI_MSI); 446 if (!dom) 447 dom = iort_get_device_domain(&pdev->dev, rid, 448 DOMAIN_BUS_PCI_MSI); 449 return dom; 450 } 451