1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * PCI MSI/MSI-X — Exported APIs for device drivers 4 * 5 * Copyright (C) 2003-2004 Intel 6 * Copyright (C) Tom Long Nguyen (tom.l.nguyen@intel.com) 7 * Copyright (C) 2016 Christoph Hellwig. 8 * Copyright (C) 2022 Linutronix GmbH 9 */ 10 11 #include <linux/export.h> 12 #include <linux/irq.h> 13 14 #include "msi.h" 15 16 /** 17 * pci_enable_msi() - Enable MSI interrupt mode on device 18 * @dev: the PCI device to operate on 19 * 20 * Legacy device driver API to enable MSI interrupts mode on device and 21 * allocate a single interrupt vector. On success, the allocated vector 22 * Linux IRQ will be saved at @dev->irq. The driver must invoke 23 * pci_disable_msi() on cleanup. 24 * 25 * NOTE: The newer pci_alloc_irq_vectors() / pci_free_irq_vectors() API 26 * pair should, in general, be used instead. 27 * 28 * Return: 0 on success, errno otherwise 29 */ 30 int pci_enable_msi(struct pci_dev *dev) 31 { 32 int rc = __pci_enable_msi_range(dev, 1, 1, NULL); 33 if (rc < 0) 34 return rc; 35 return 0; 36 } 37 EXPORT_SYMBOL(pci_enable_msi); 38 39 /** 40 * pci_disable_msi() - Disable MSI interrupt mode on device 41 * @dev: the PCI device to operate on 42 * 43 * Legacy device driver API to disable MSI interrupt mode on device, 44 * free earlier allocated interrupt vectors, and restore INTx emulation. 45 * The PCI device Linux IRQ (@dev->irq) is restored to its default 46 * pin-assertion IRQ. This is the cleanup pair of pci_enable_msi(). 47 * 48 * NOTE: The newer pci_alloc_irq_vectors() / pci_free_irq_vectors() API 49 * pair should, in general, be used instead. 50 */ 51 void pci_disable_msi(struct pci_dev *dev) 52 { 53 if (!pci_msi_enabled() || !dev || !dev->msi_enabled) 54 return; 55 56 guard(msi_descs_lock)(&dev->dev); 57 pci_msi_shutdown(dev); 58 pci_free_msi_irqs(dev); 59 } 60 EXPORT_SYMBOL(pci_disable_msi); 61 62 /** 63 * pci_msix_vec_count() - Get number of MSI-X interrupt vectors on device 64 * @dev: the PCI device to operate on 65 * 66 * Return: number of MSI-X interrupt vectors available on this device 67 * (i.e., the device's MSI-X capability structure "table size"), -EINVAL 68 * if the device is not MSI-X capable, other errnos otherwise. 69 */ 70 int pci_msix_vec_count(struct pci_dev *dev) 71 { 72 u16 control; 73 74 if (!dev->msix_cap) 75 return -EINVAL; 76 77 pci_read_config_word(dev, dev->msix_cap + PCI_MSIX_FLAGS, &control); 78 return msix_table_size(control); 79 } 80 EXPORT_SYMBOL(pci_msix_vec_count); 81 82 /** 83 * pci_enable_msix_range() - Enable MSI-X interrupt mode on device 84 * @dev: the PCI device to operate on 85 * @entries: input/output parameter, array of MSI-X configuration entries 86 * @minvec: minimum required number of MSI-X vectors 87 * @maxvec: maximum desired number of MSI-X vectors 88 * 89 * Legacy device driver API to enable MSI-X interrupt mode on device and 90 * configure its MSI-X capability structure as appropriate. The passed 91 * @entries array must have each of its members "entry" field set to a 92 * desired (valid) MSI-X vector number, where the range of valid MSI-X 93 * vector numbers can be queried through pci_msix_vec_count(). If 94 * successful, the driver must invoke pci_disable_msix() on cleanup. 95 * 96 * NOTE: The newer pci_alloc_irq_vectors() / pci_free_irq_vectors() API 97 * pair should, in general, be used instead. 98 * 99 * Return: number of MSI-X vectors allocated (which might be smaller 100 * than @maxvecs), where Linux IRQ numbers for such allocated vectors 101 * are saved back in the @entries array elements' "vector" field. Return 102 * -ENOSPC if less than @minvecs interrupt vectors are available. 103 * Return -EINVAL if one of the passed @entries members "entry" field 104 * was invalid or a duplicate, or if plain MSI interrupts mode was 105 * earlier enabled on device. Return other errnos otherwise. 106 */ 107 int pci_enable_msix_range(struct pci_dev *dev, struct msix_entry *entries, 108 int minvec, int maxvec) 109 { 110 return __pci_enable_msix_range(dev, entries, minvec, maxvec, NULL, 0); 111 } 112 EXPORT_SYMBOL(pci_enable_msix_range); 113 114 /** 115 * pci_msix_can_alloc_dyn - Query whether dynamic allocation after enabling 116 * MSI-X is supported 117 * 118 * @dev: PCI device to operate on 119 * 120 * Return: True if supported, false otherwise 121 */ 122 bool pci_msix_can_alloc_dyn(struct pci_dev *dev) 123 { 124 if (!dev->msix_cap) 125 return false; 126 127 return pci_msi_domain_supports(dev, MSI_FLAG_PCI_MSIX_ALLOC_DYN, DENY_LEGACY); 128 } 129 EXPORT_SYMBOL_GPL(pci_msix_can_alloc_dyn); 130 131 /** 132 * pci_msix_alloc_irq_at - Allocate an MSI-X interrupt after enabling MSI-X 133 * at a given MSI-X vector index or any free vector index 134 * 135 * @dev: PCI device to operate on 136 * @index: Index to allocate. If @index == MSI_ANY_INDEX this allocates 137 * the next free index in the MSI-X table 138 * @affdesc: Optional pointer to an affinity descriptor structure. NULL otherwise 139 * 140 * Return: A struct msi_map 141 * 142 * On success msi_map::index contains the allocated index (>= 0) and 143 * msi_map::virq contains the allocated Linux interrupt number (> 0). 144 * 145 * On fail msi_map::index contains the error code and msi_map::virq 146 * is set to 0. 147 */ 148 struct msi_map pci_msix_alloc_irq_at(struct pci_dev *dev, unsigned int index, 149 const struct irq_affinity_desc *affdesc) 150 { 151 struct msi_map map = { .index = -ENOTSUPP }; 152 153 if (!dev->msix_enabled) 154 return map; 155 156 if (!pci_msix_can_alloc_dyn(dev)) 157 return map; 158 159 return msi_domain_alloc_irq_at(&dev->dev, MSI_DEFAULT_DOMAIN, index, affdesc, NULL); 160 } 161 EXPORT_SYMBOL_GPL(pci_msix_alloc_irq_at); 162 163 /** 164 * pci_msix_free_irq - Free an interrupt on a PCI/MSI-X interrupt domain 165 * 166 * @dev: The PCI device to operate on 167 * @map: A struct msi_map describing the interrupt to free 168 * 169 * Undo an interrupt vector allocation. Does not disable MSI-X. 170 */ 171 void pci_msix_free_irq(struct pci_dev *dev, struct msi_map map) 172 { 173 if (WARN_ON_ONCE(map.index < 0 || map.virq <= 0)) 174 return; 175 if (WARN_ON_ONCE(!pci_msix_can_alloc_dyn(dev))) 176 return; 177 msi_domain_free_irqs_range(&dev->dev, MSI_DEFAULT_DOMAIN, map.index, map.index); 178 } 179 EXPORT_SYMBOL_GPL(pci_msix_free_irq); 180 181 /** 182 * pci_disable_msix() - Disable MSI-X interrupt mode on device 183 * @dev: the PCI device to operate on 184 * 185 * Legacy device driver API to disable MSI-X interrupt mode on device, 186 * free earlier-allocated interrupt vectors, and restore INTx. 187 * The PCI device Linux IRQ (@dev->irq) is restored to its default pin 188 * assertion IRQ. This is the cleanup pair of pci_enable_msix_range(). 189 * 190 * NOTE: The newer pci_alloc_irq_vectors() / pci_free_irq_vectors() API 191 * pair should, in general, be used instead. 192 */ 193 void pci_disable_msix(struct pci_dev *dev) 194 { 195 if (!pci_msi_enabled() || !dev || !dev->msix_enabled) 196 return; 197 198 guard(msi_descs_lock)(&dev->dev); 199 pci_msix_shutdown(dev); 200 pci_free_msi_irqs(dev); 201 } 202 EXPORT_SYMBOL(pci_disable_msix); 203 204 /** 205 * pci_alloc_irq_vectors() - Allocate multiple device interrupt vectors 206 * @dev: the PCI device to operate on 207 * @min_vecs: minimum required number of vectors (must be >= 1) 208 * @max_vecs: maximum desired number of vectors 209 * @flags: One or more of: 210 * 211 * * %PCI_IRQ_MSIX Allow trying MSI-X vector allocations 212 * * %PCI_IRQ_MSI Allow trying MSI vector allocations 213 * 214 * * %PCI_IRQ_INTX Allow trying INTx interrupts, if and 215 * only if @min_vecs == 1 216 * 217 * * %PCI_IRQ_AFFINITY Auto-manage IRQs affinity by spreading 218 * the vectors around available CPUs 219 * 220 * Allocate up to @max_vecs interrupt vectors on device. MSI-X irq 221 * vector allocation has a higher precedence over plain MSI, which has a 222 * higher precedence over legacy INTx emulation. 223 * 224 * Upon a successful allocation, the caller should use pci_irq_vector() 225 * to get the Linux IRQ number to be passed to request_threaded_irq(). 226 * The driver must call pci_free_irq_vectors() on cleanup. 227 * 228 * Return: number of allocated vectors (which might be smaller than 229 * @max_vecs), -ENOSPC if less than @min_vecs interrupt vectors are 230 * available, other errnos otherwise. 231 */ 232 int pci_alloc_irq_vectors(struct pci_dev *dev, unsigned int min_vecs, 233 unsigned int max_vecs, unsigned int flags) 234 { 235 return pci_alloc_irq_vectors_affinity(dev, min_vecs, max_vecs, 236 flags, NULL); 237 } 238 EXPORT_SYMBOL(pci_alloc_irq_vectors); 239 240 /** 241 * pci_alloc_irq_vectors_affinity() - Allocate multiple device interrupt 242 * vectors with affinity requirements 243 * @dev: the PCI device to operate on 244 * @min_vecs: minimum required number of vectors (must be >= 1) 245 * @max_vecs: maximum desired number of vectors 246 * @flags: allocation flags, as in pci_alloc_irq_vectors() 247 * @affd: affinity requirements (can be %NULL). 248 * 249 * Same as pci_alloc_irq_vectors(), but with the extra @affd parameter. 250 * Check that function docs, and &struct irq_affinity, for more details. 251 */ 252 int pci_alloc_irq_vectors_affinity(struct pci_dev *dev, unsigned int min_vecs, 253 unsigned int max_vecs, unsigned int flags, 254 struct irq_affinity *affd) 255 { 256 struct irq_affinity msi_default_affd = {0}; 257 int nvecs = -ENOSPC; 258 259 if (flags & PCI_IRQ_AFFINITY) { 260 if (!affd) 261 affd = &msi_default_affd; 262 } else { 263 if (WARN_ON(affd)) 264 affd = NULL; 265 } 266 267 if (flags & PCI_IRQ_MSIX) { 268 nvecs = __pci_enable_msix_range(dev, NULL, min_vecs, max_vecs, 269 affd, flags); 270 if (nvecs > 0) 271 return nvecs; 272 } 273 274 if (flags & PCI_IRQ_MSI) { 275 nvecs = __pci_enable_msi_range(dev, min_vecs, max_vecs, affd); 276 if (nvecs > 0) 277 return nvecs; 278 } 279 280 /* use INTx IRQ if allowed */ 281 if (flags & PCI_IRQ_INTX) { 282 if (min_vecs == 1 && dev->irq) { 283 /* 284 * Invoke the affinity spreading logic to ensure that 285 * the device driver can adjust queue configuration 286 * for the single interrupt case. 287 */ 288 if (affd) 289 irq_create_affinity_masks(1, affd); 290 pci_intx(dev, 1); 291 return 1; 292 } 293 } 294 295 return nvecs; 296 } 297 EXPORT_SYMBOL(pci_alloc_irq_vectors_affinity); 298 299 /** 300 * pci_irq_vector() - Get Linux IRQ number of a device interrupt vector 301 * @dev: the PCI device to operate on 302 * @nr: device-relative interrupt vector index (0-based); has different 303 * meanings, depending on interrupt mode: 304 * 305 * * MSI-X the index in the MSI-X vector table 306 * * MSI the index of the enabled MSI vectors 307 * * INTx must be 0 308 * 309 * Return: the Linux IRQ number, or -EINVAL if @nr is out of range 310 */ 311 int pci_irq_vector(struct pci_dev *dev, unsigned int nr) 312 { 313 unsigned int irq; 314 315 if (!dev->msi_enabled && !dev->msix_enabled) 316 return !nr ? dev->irq : -EINVAL; 317 318 irq = msi_get_virq(&dev->dev, nr); 319 return irq ? irq : -EINVAL; 320 } 321 EXPORT_SYMBOL(pci_irq_vector); 322 323 /** 324 * pci_irq_get_affinity() - Get a device interrupt vector affinity 325 * @dev: the PCI device to operate on 326 * @nr: device-relative interrupt vector index (0-based); has different 327 * meanings, depending on interrupt mode: 328 * 329 * * MSI-X the index in the MSI-X vector table 330 * * MSI the index of the enabled MSI vectors 331 * * INTx must be 0 332 * 333 * Return: MSI/MSI-X vector affinity, NULL if @nr is out of range or if 334 * the MSI(-X) vector was allocated without explicit affinity 335 * requirements (e.g., by pci_enable_msi(), pci_enable_msix_range(), or 336 * pci_alloc_irq_vectors() without the %PCI_IRQ_AFFINITY flag). Return a 337 * generic set of CPU IDs representing all possible CPUs available 338 * during system boot if the device is in legacy INTx mode. 339 */ 340 const struct cpumask *pci_irq_get_affinity(struct pci_dev *dev, int nr) 341 { 342 int idx, irq = pci_irq_vector(dev, nr); 343 struct msi_desc *desc; 344 345 if (WARN_ON_ONCE(irq <= 0)) 346 return NULL; 347 348 desc = irq_get_msi_desc(irq); 349 /* Non-MSI does not have the information handy */ 350 if (!desc) 351 return cpu_possible_mask; 352 353 /* MSI[X] interrupts can be allocated without affinity descriptor */ 354 if (!desc->affinity) 355 return NULL; 356 357 /* 358 * MSI has a mask array in the descriptor. 359 * MSI-X has a single mask. 360 */ 361 idx = dev->msi_enabled ? nr : 0; 362 return &desc->affinity[idx].mask; 363 } 364 EXPORT_SYMBOL(pci_irq_get_affinity); 365 366 /** 367 * pci_free_irq_vectors() - Free previously allocated IRQs for a device 368 * @dev: the PCI device to operate on 369 * 370 * Undo the interrupt vector allocations and possible device MSI/MSI-X 371 * enablement earlier done through pci_alloc_irq_vectors_affinity() or 372 * pci_alloc_irq_vectors(). 373 */ 374 void pci_free_irq_vectors(struct pci_dev *dev) 375 { 376 pci_disable_msix(dev); 377 pci_disable_msi(dev); 378 } 379 EXPORT_SYMBOL(pci_free_irq_vectors); 380 381 /** 382 * pci_restore_msi_state() - Restore cached MSI(-X) state on device 383 * @dev: the PCI device to operate on 384 * 385 * Write the Linux-cached MSI(-X) state back on device. This is 386 * typically useful upon system resume, or after an error-recovery PCI 387 * adapter reset. 388 */ 389 void pci_restore_msi_state(struct pci_dev *dev) 390 { 391 __pci_restore_msi_state(dev); 392 __pci_restore_msix_state(dev); 393 } 394 EXPORT_SYMBOL_GPL(pci_restore_msi_state); 395 396 /** 397 * pci_msi_enabled() - Are MSI(-X) interrupts enabled system-wide? 398 * 399 * Return: true if MSI has not been globally disabled through ACPI FADT, 400 * PCI bridge quirks, or the "pci=nomsi" kernel command-line option. 401 */ 402 bool pci_msi_enabled(void) 403 { 404 return pci_msi_enable; 405 } 406 EXPORT_SYMBOL(pci_msi_enabled); 407