1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright (c) Meta Platforms, Inc. and affiliates. */ 3 4 #include <linux/pci.h> 5 #include <linux/types.h> 6 7 #include "fbnic.h" 8 #include "fbnic_netdev.h" 9 #include "fbnic_txrx.h" 10 11 static irqreturn_t fbnic_fw_msix_intr(int __always_unused irq, void *data) 12 { 13 struct fbnic_dev *fbd = (struct fbnic_dev *)data; 14 15 fbnic_mbx_poll(fbd); 16 17 fbnic_wr32(fbd, FBNIC_INTR_MASK_CLEAR(0), 1u << FBNIC_FW_MSIX_ENTRY); 18 19 return IRQ_HANDLED; 20 } 21 22 static int __fbnic_fw_enable_mbx(struct fbnic_dev *fbd, int vector) 23 { 24 int err; 25 26 /* Initialize mailbox and attempt to poll it into ready state */ 27 fbnic_mbx_init(fbd); 28 err = fbnic_mbx_poll_tx_ready(fbd); 29 if (err) { 30 dev_warn(fbd->dev, "FW mailbox did not enter ready state\n"); 31 return err; 32 } 33 34 /* Enable interrupt and unmask the vector */ 35 enable_irq(vector); 36 fbnic_wr32(fbd, FBNIC_INTR_MASK_CLEAR(0), 1u << FBNIC_FW_MSIX_ENTRY); 37 38 return 0; 39 } 40 41 /** 42 * fbnic_fw_request_mbx - Configure and initialize Firmware Mailbox 43 * @fbd: Pointer to device to initialize 44 * 45 * This function will allocate the IRQ and then reinitialize the mailbox 46 * starting communication between the host and firmware. 47 * 48 * Return: non-zero on failure. 49 **/ 50 int fbnic_fw_request_mbx(struct fbnic_dev *fbd) 51 { 52 struct pci_dev *pdev = to_pci_dev(fbd->dev); 53 int vector, err; 54 55 WARN_ON(fbd->fw_msix_vector); 56 57 vector = pci_irq_vector(pdev, FBNIC_FW_MSIX_ENTRY); 58 if (vector < 0) 59 return vector; 60 61 /* Request the IRQ for FW Mailbox vector. */ 62 err = request_threaded_irq(vector, NULL, &fbnic_fw_msix_intr, 63 IRQF_ONESHOT | IRQF_NO_AUTOEN, 64 dev_name(fbd->dev), fbd); 65 if (err) 66 return err; 67 68 /* Initialize mailbox and attempt to poll it into ready state */ 69 err = __fbnic_fw_enable_mbx(fbd, vector); 70 if (err) 71 free_irq(vector, fbd); 72 73 fbd->fw_msix_vector = vector; 74 75 return err; 76 } 77 78 /** 79 * fbnic_fw_disable_mbx - Temporarily place mailbox in standby state 80 * @fbd: Pointer to device 81 * 82 * Shutdown the mailbox by notifying the firmware to stop sending us logs, mask 83 * and synchronize the IRQ, and then clean up the rings. 84 **/ 85 static void fbnic_fw_disable_mbx(struct fbnic_dev *fbd) 86 { 87 /* Disable interrupt and synchronize the IRQ */ 88 disable_irq(fbd->fw_msix_vector); 89 90 /* Mask the vector */ 91 fbnic_wr32(fbd, FBNIC_INTR_MASK_SET(0), 1u << FBNIC_FW_MSIX_ENTRY); 92 93 /* Make sure disabling logs message is sent, must be done here to 94 * avoid risk of completing without a running interrupt. 95 */ 96 fbnic_mbx_flush_tx(fbd); 97 fbnic_mbx_clean(fbd); 98 } 99 100 /** 101 * fbnic_fw_free_mbx - Disable mailbox and place it in standby state 102 * @fbd: Pointer to device to disable 103 * 104 * This function will disable the mailbox interrupt, free any messages still 105 * in the mailbox and place it into a disabled state. The firmware is 106 * expected to see the update and assume that the host is in the reset state. 107 **/ 108 void fbnic_fw_free_mbx(struct fbnic_dev *fbd) 109 { 110 /* Vector has already been freed */ 111 if (!fbd->fw_msix_vector) 112 return; 113 114 fbnic_fw_disable_mbx(fbd); 115 116 /* Free the vector */ 117 free_irq(fbd->fw_msix_vector, fbd); 118 fbd->fw_msix_vector = 0; 119 } 120 121 static irqreturn_t fbnic_mac_msix_intr(int __always_unused irq, void *data) 122 { 123 struct fbnic_dev *fbd = data; 124 struct fbnic_net *fbn; 125 126 if (fbd->mac->get_link_event(fbd) == FBNIC_LINK_EVENT_NONE) { 127 fbnic_wr32(fbd, FBNIC_INTR_MASK_CLEAR(0), 128 1u << FBNIC_PCS_MSIX_ENTRY); 129 return IRQ_HANDLED; 130 } 131 132 fbn = netdev_priv(fbd->netdev); 133 134 /* Record link down events */ 135 if (!fbd->mac->get_link(fbd, fbn->aui, fbn->fec)) 136 phylink_pcs_change(fbn->pcs, false); 137 138 return IRQ_HANDLED; 139 } 140 141 /** 142 * fbnic_mac_request_irq - Configure the MAC to enable it to advertise link 143 * @fbd: Pointer to device to initialize 144 * 145 * This function provides basic bringup for the MAC/PHY IRQ. For now the IRQ 146 * will remain disabled until we start the MAC/PCS/PHY logic via phylink. 147 * 148 * Return: non-zero on failure. 149 **/ 150 int fbnic_mac_request_irq(struct fbnic_dev *fbd) 151 { 152 struct pci_dev *pdev = to_pci_dev(fbd->dev); 153 int vector, err; 154 155 WARN_ON(fbd->mac_msix_vector); 156 157 vector = pci_irq_vector(pdev, FBNIC_PCS_MSIX_ENTRY); 158 if (vector < 0) 159 return vector; 160 161 /* Request the IRQ for PCS link vector. 162 * Map PCS cause to it, and unmask it 163 */ 164 err = request_irq(vector, &fbnic_mac_msix_intr, 0, 165 fbd->netdev->name, fbd); 166 if (err) 167 return err; 168 169 /* Map and enable interrupt, unmask vector after link is configured */ 170 fbnic_wr32(fbd, FBNIC_INTR_MSIX_CTRL(FBNIC_INTR_MSIX_CTRL_PCS_IDX), 171 FBNIC_PCS_MSIX_ENTRY | FBNIC_INTR_MSIX_CTRL_ENABLE); 172 173 fbnic_wr32(fbd, FBNIC_INTR_MSIX_CTRL(FBNIC_INTR_MSIX_CTRL_RXB_IDX), 0); 174 175 fbd->mac_msix_vector = vector; 176 177 return 0; 178 } 179 180 /** 181 * fbnic_mac_free_irq - Teardown the MAC IRQ to prepare for stopping 182 * @fbd: Pointer to device that is stopping 183 * 184 * This function undoes the work done in fbnic_mac_request_irq and prepares 185 * the device to no longer receive traffic on the host interface. 186 **/ 187 void fbnic_mac_free_irq(struct fbnic_dev *fbd) 188 { 189 /* Vector has already been freed */ 190 if (!fbd->mac_msix_vector) 191 return; 192 193 /* Disable interrupt */ 194 fbnic_wr32(fbd, FBNIC_INTR_MSIX_CTRL(FBNIC_INTR_MSIX_CTRL_PCS_IDX), 195 FBNIC_PCS_MSIX_ENTRY); 196 fbnic_wrfl(fbd); 197 198 /* Synchronize IRQ to prevent race that would unmask vector */ 199 synchronize_irq(fbd->mac_msix_vector); 200 201 /* Mask the vector */ 202 fbnic_wr32(fbd, FBNIC_INTR_MASK_SET(0), 1u << FBNIC_PCS_MSIX_ENTRY); 203 204 /* Free the vector */ 205 free_irq(fbd->mac_msix_vector, fbd); 206 fbd->mac_msix_vector = 0; 207 } 208 209 void fbnic_synchronize_irq(struct fbnic_dev *fbd, int nr) 210 { 211 struct pci_dev *pdev = to_pci_dev(fbd->dev); 212 int irq = pci_irq_vector(pdev, nr); 213 214 if (irq < 0) 215 return; 216 217 synchronize_irq(irq); 218 } 219 220 int fbnic_request_irq(struct fbnic_dev *fbd, int nr, irq_handler_t handler, 221 unsigned long flags, const char *name, void *data) 222 { 223 struct pci_dev *pdev = to_pci_dev(fbd->dev); 224 int irq = pci_irq_vector(pdev, nr); 225 226 if (irq < 0) 227 return irq; 228 229 return request_irq(irq, handler, flags, name, data); 230 } 231 232 void fbnic_free_irq(struct fbnic_dev *fbd, int nr, void *data) 233 { 234 struct pci_dev *pdev = to_pci_dev(fbd->dev); 235 int irq = pci_irq_vector(pdev, nr); 236 237 if (irq < 0) 238 return; 239 240 free_irq(irq, data); 241 } 242 243 struct fbnic_msix_test_data { 244 struct fbnic_dev *fbd; 245 unsigned long test_msix_status[BITS_TO_LONGS(FBNIC_MAX_MSIX_VECS)]; 246 int irq_vector[FBNIC_MAX_MSIX_VECS]; 247 }; 248 249 static irqreturn_t fbnic_irq_test(int irq, void *data) 250 { 251 struct fbnic_msix_test_data *test_data = data; 252 struct fbnic_dev *fbd = test_data->fbd; 253 int i; 254 255 for (i = fbd->num_irqs; i--;) { 256 if (test_data->irq_vector[i] == irq) { 257 set_bit(i, test_data->test_msix_status); 258 break; 259 } 260 } 261 262 return IRQ_HANDLED; 263 } 264 265 /** 266 * fbnic_msix_test - Verify behavior of NIC interrupts 267 * @fbd: device to test 268 * 269 * This function is meant to test the global interrupt registers and the 270 * PCIe IP MSI-X functionality. It essentially goes through and tests 271 * various combinations of the set, clear, and mask bits in order to 272 * verify the behavior is as we expect it to be from the driver. 273 * 274 * Return: See enum fbnic_msix_self_test_codes 275 **/ 276 enum fbnic_msix_self_test_codes fbnic_msix_test(struct fbnic_dev *fbd) 277 { 278 enum fbnic_msix_self_test_codes result = FBNIC_TEST_MSIX_SUCCESS; 279 struct pci_dev *pdev = to_pci_dev(fbd->dev); 280 struct fbnic_msix_test_data *test_data; 281 u32 mask = 0; 282 int i; 283 284 /* Allocate bitmap and IRQ vector table */ 285 test_data = kzalloc_obj(*test_data, GFP_KERNEL); 286 287 /* memory allocation failure */ 288 if (!test_data) 289 return FBNIC_TEST_MSIX_NOMEM; 290 291 /* Initialize test data */ 292 test_data->fbd = fbd; 293 294 for (i = FBNIC_NON_NAPI_VECTORS; i < fbd->num_irqs; i++) { 295 /* Add IRQ to vector table so it can be found */ 296 test_data->irq_vector[i] = pci_irq_vector(pdev, i); 297 298 /* Enable the interrupt */ 299 if (!fbnic_request_irq(fbd, i, fbnic_irq_test, 0, 300 fbd->netdev->name, test_data)) 301 continue; 302 303 while (i-- > FBNIC_NON_NAPI_VECTORS) 304 fbnic_free_irq(fbd, i, test_data); 305 kfree(test_data); 306 307 /* IRQ request failure */ 308 return FBNIC_TEST_MSIX_IRQ_REQ_FAIL; 309 } 310 311 /* Test each bit individually */ 312 for (i = FBNIC_NON_NAPI_VECTORS; i < fbd->num_irqs; i++) { 313 mask = 1U << (i % 32); 314 315 /* Start with mask set and interrupt cleared */ 316 fbnic_wr32(fbd, FBNIC_INTR_MASK_SET(i / 32), mask); 317 fbnic_wrfl(fbd); 318 fbnic_wr32(fbd, FBNIC_INTR_CLEAR(i / 32), mask); 319 fbnic_wrfl(fbd); 320 321 /* masking failure to prevent interrupt */ 322 result = FBNIC_TEST_MSIX_MASK; 323 324 fbnic_wr32(fbd, FBNIC_INTR_SET(i / 32), mask); 325 fbnic_wrfl(fbd); 326 usleep_range(10000, 11000); 327 328 if (test_bit(i, test_data->test_msix_status)) 329 break; 330 331 /* unmasking failure w/ sw status set */ 332 result = FBNIC_TEST_MSIX_UNMASK; 333 334 fbnic_wr32(fbd, FBNIC_INTR_MASK_CLEAR(i / 32), mask); 335 fbnic_wrfl(fbd); 336 usleep_range(10000, 11000); 337 338 if (!test_bit(i, test_data->test_msix_status)) 339 break; 340 341 /* interrupt when clearing mask */ 342 result = FBNIC_TEST_MSIX_IRQ_CLEAR; 343 344 clear_bit(i, test_data->test_msix_status); 345 fbnic_wr32(fbd, FBNIC_INTR_MASK_CLEAR(i / 32), mask); 346 fbnic_wrfl(fbd); 347 usleep_range(10000, 11000); 348 349 if (test_bit(i, test_data->test_msix_status)) 350 break; 351 352 /* interrupt not triggering when not masked */ 353 result = FBNIC_TEST_MSIX_NO_INTERRUPT; 354 355 fbnic_wr32(fbd, FBNIC_INTR_SET(i / 32), mask); 356 fbnic_wrfl(fbd); 357 usleep_range(10000, 11000); 358 359 if (!test_bit(i, test_data->test_msix_status)) 360 break; 361 362 /* status not cleared, or mask not set */ 363 result = FBNIC_TEST_MSIX_NO_CLEAR_OR_MASK; 364 if (mask & fbnic_rd32(fbd, FBNIC_INTR_STATUS(i / 32))) 365 break; 366 if (!(mask & fbnic_rd32(fbd, FBNIC_INTR_MASK(i / 32)))) 367 break; 368 369 /* Result = 0 - Success */ 370 result = FBNIC_TEST_MSIX_SUCCESS; 371 372 clear_bit(i, test_data->test_msix_status); 373 } 374 375 if (i < fbd->num_irqs) { 376 fbnic_wr32(fbd, FBNIC_INTR_MASK_SET(i / 32), mask); 377 fbnic_wrfl(fbd); 378 fbnic_wr32(fbd, FBNIC_INTR_CLEAR(i / 32), mask); 379 fbnic_wrfl(fbd); 380 clear_bit(i, test_data->test_msix_status); 381 } 382 383 for (i = FBNIC_NON_NAPI_VECTORS; i < fbd->num_irqs; i++) { 384 /* Test for bits set after testing */ 385 if (test_bit(i, test_data->test_msix_status)) 386 result = FBNIC_TEST_MSIX_BITS_SET_AFTER_TEST; 387 388 /* Free IRQ */ 389 fbnic_free_irq(fbd, i, test_data); 390 } 391 392 kfree(test_data); 393 394 return result; 395 } 396 397 void fbnic_napi_name_irqs(struct fbnic_dev *fbd) 398 { 399 unsigned int i; 400 401 for (i = 0; i < ARRAY_SIZE(fbd->napi_irq); i++) 402 snprintf(fbd->napi_irq[i].name, 403 sizeof(fbd->napi_irq[i].name), 404 "%s-TxRx-%u", fbd->netdev->name, i); 405 } 406 407 int fbnic_napi_request_irq(struct fbnic_dev *fbd, 408 struct fbnic_napi_vector *nv) 409 { 410 struct fbnic_net *fbn = netdev_priv(fbd->netdev); 411 int i = fbnic_napi_idx(nv); 412 int err; 413 414 if (!fbd->napi_irq[i].users) { 415 err = fbnic_request_irq(fbd, nv->v_idx, 416 fbnic_msix_clean_rings, 0, 417 fbd->napi_irq[i].name, 418 &fbn->napi[i]); 419 if (err) 420 return err; 421 } 422 423 fbd->napi_irq[i].users++; 424 return 0; 425 } 426 427 void fbnic_napi_free_irq(struct fbnic_dev *fbd, 428 struct fbnic_napi_vector *nv) 429 { 430 struct fbnic_net *fbn = netdev_priv(fbd->netdev); 431 int i = fbnic_napi_idx(nv); 432 433 if (--fbd->napi_irq[i].users) 434 return; 435 436 fbnic_free_irq(fbd, nv->v_idx, &fbn->napi[i]); 437 } 438 439 void fbnic_free_irqs(struct fbnic_dev *fbd) 440 { 441 struct pci_dev *pdev = to_pci_dev(fbd->dev); 442 443 fbd->num_irqs = 0; 444 445 pci_free_irq_vectors(pdev); 446 } 447 448 int fbnic_alloc_irqs(struct fbnic_dev *fbd) 449 { 450 unsigned int wanted_irqs = FBNIC_NON_NAPI_VECTORS; 451 struct pci_dev *pdev = to_pci_dev(fbd->dev); 452 int num_irqs; 453 454 wanted_irqs += min_t(unsigned int, num_online_cpus(), FBNIC_MAX_RXQS); 455 num_irqs = pci_alloc_irq_vectors(pdev, FBNIC_NON_NAPI_VECTORS + 1, 456 wanted_irqs, PCI_IRQ_MSIX); 457 if (num_irqs < 0) { 458 dev_err(fbd->dev, "Failed to allocate MSI-X entries\n"); 459 return num_irqs; 460 } 461 462 if (num_irqs < wanted_irqs) 463 dev_warn(fbd->dev, "Allocated %d IRQs, expected %d\n", 464 num_irqs, wanted_irqs); 465 466 fbd->num_irqs = num_irqs; 467 468 return 0; 469 } 470