1 // SPDX-License-Identifier: GPL-2.0-or-later
3  * PCI Error Recovery Driver for RPA-compliant PPC64 platform.
17 #include <asm/ppc-pci.h>
18 #include <asm/pci-bridge.h>
56 		return "need reset";  in pci_ers_result_name()
79 	return !edev || (edev->mode & EEH_DEV_REMOVED);  in eeh_dev_removed()
84 	if (!edev->pdev)  in eeh_edev_actionable()
86 	if (edev->pdev->error_state == pci_channel_io_perm_failure)  in eeh_edev_actionable()
90 	if (eeh_pe_passed(edev->pe))  in eeh_edev_actionable()
97  * eeh_pcid_get - Get the PCI device driver
102  * of the PCI device driver to prevent that being unloaded on
107 	if (!pdev || !pdev->driver)  in eeh_pcid_get()
110 	if (!try_module_get(pdev->driver->driver.owner))  in eeh_pcid_get()
113 	return pdev->driver;  in eeh_pcid_get()
117  * eeh_pcid_put - Dereference on the PCI device driver
120  * The function is called to do dereference on the PCI device
125 	if (!pdev || !pdev->driver)  in eeh_pcid_put()
128 	module_put(pdev->driver->driver.owner);  in eeh_pcid_put()
132  * eeh_disable_irq - Disable interrupt for the recovering device
137  * device. If the device has enabled MSI or MSI-X interrupt, we needn't
139  * devices encountering EEH errors, which includes MSI or MSI-X.
143 	/* Don't disable MSI and MSI-X interrupts. They are  in eeh_disable_irq()
147 	if (edev->pdev->msi_enabled || edev->pdev->msix_enabled)  in eeh_disable_irq()
150 	if (!irq_has_action(edev->pdev->irq))  in eeh_disable_irq()
153 	edev->mode |= EEH_DEV_IRQ_DISABLED;  in eeh_disable_irq()
154 	disable_irq_nosync(edev->pdev->irq);  in eeh_disable_irq()
158  * eeh_enable_irq - Enable interrupt for the recovering device
166 	if ((edev->mode) & EEH_DEV_IRQ_DISABLED) {  in eeh_enable_irq()
167 		edev->mode &= ~EEH_DEV_IRQ_DISABLED;  in eeh_enable_irq()
188 		if (irqd_irq_disabled(irq_get_irq_data(edev->pdev->irq)))  in eeh_enable_irq()
189 			enable_irq(edev->pdev->irq);  in eeh_enable_irq()
201 	 * We cannot access the config space on some adapters.  in eeh_dev_save_state()
207 	if (edev->pe && (edev->pe->state & EEH_PE_CFG_RESTRICTED))  in eeh_dev_save_state()
225 				edev->pdev->error_state = s;  in eeh_set_channel_state()
238 			if (!eeh_pcid_get(edev->pdev))  in eeh_set_irq_state()
246 			eeh_pcid_put(edev->pdev);  in eeh_set_irq_state()
262 	pdev = edev->pdev;  in eeh_pe_report_edev()
264 		get_device(&pdev->dev);  in eeh_pe_report_edev()
270 	device_lock(&pdev->dev);  in eeh_pe_report_edev()
276 		else if (!driver->err_handler)  in eeh_pe_report_edev()
278 		else if (edev->mode & EEH_DEV_NO_HANDLER)  in eeh_pe_report_edev()
283 				      driver->name,  in eeh_pe_report_edev()
293 			      !eeh_dev_removed(edev), !eeh_pe_passed(edev->pe));  in eeh_pe_report_edev()
295 	device_unlock(&pdev->dev);  in eeh_pe_report_edev()
296 	if (edev->pdev != pdev)  in eeh_pe_report_edev()
298 	put_device(&pdev->dev);  in eeh_pe_report_edev()
318  * eeh_report_error - Report pci error to each device driver
330 	if (!driver->err_handler->error_detected)  in eeh_report_error()
333 	eeh_edev_info(edev, "Invoking %s->error_detected(IO frozen)",  in eeh_report_error()
334 		      driver->name);  in eeh_report_error()
335 	rc = driver->err_handler->error_detected(pdev, pci_channel_io_frozen);  in eeh_report_error()
337 	edev->in_error = true;  in eeh_report_error()
343  * eeh_report_mmio_enabled - Tell drivers that MMIO has been enabled
354 	if (!driver->err_handler->mmio_enabled)  in eeh_report_mmio_enabled()
356 	eeh_edev_info(edev, "Invoking %s->mmio_enabled()", driver->name);  in eeh_report_mmio_enabled()
357 	return driver->err_handler->mmio_enabled(pdev);  in eeh_report_mmio_enabled()
361  * eeh_report_reset - Tell device that slot has been reset
365  * This routine must be called while EEH tries to reset particular
367  * some actions, usually to save data the driver needs so that the
374 	if (!driver->err_handler->slot_reset || !edev->in_error)  in eeh_report_reset()
376 	eeh_edev_info(edev, "Invoking %s->slot_reset()", driver->name);  in eeh_report_reset()
377 	return driver->err_handler->slot_reset(pdev);  in eeh_report_reset()
389 	 * the blocked config space on some adapters. We have  in eeh_dev_restore_state()
393 	if (edev->pe && (edev->pe->state & EEH_PE_CFG_RESTRICTED)) {  in eeh_dev_restore_state()
394 		if (list_is_last(&edev->entry, &edev->pe->edevs))  in eeh_dev_restore_state()
395 			eeh_pe_restore_bars(edev->pe);  in eeh_dev_restore_state()
408  * eeh_report_resume - Tell device to resume normal operations
413  * could resume so that the device driver can do some initialization
420 	if (!driver->err_handler->resume || !edev->in_error)  in eeh_report_resume()
423 	eeh_edev_info(edev, "Invoking %s->resume()", driver->name);  in eeh_report_resume()
424 	driver->err_handler->resume(pdev);  in eeh_report_resume()
426 	pci_uevent_ers(edev->pdev, PCI_ERS_RESULT_RECOVERED);  in eeh_report_resume()
428 	if (eeh_ops->notify_resume)  in eeh_report_resume()
429 		eeh_ops->notify_resume(edev);  in eeh_report_resume()
435  * eeh_report_failure - Tell device driver that device is dead.
440  * dead, and that no further recovery attempts will be made on it.
448 	if (!driver->err_handler->error_detected)  in eeh_report_failure()
451 	eeh_edev_info(edev, "Invoking %s->error_detected(permanent failure)",  in eeh_report_failure()
452 		      driver->name);  in eeh_report_failure()
453 	rc = driver->err_handler->error_detected(pdev,  in eeh_report_failure()
465 	if (!(edev->physfn)) {  in eeh_add_virt_device()
472 		if (driver->err_handler) {  in eeh_add_virt_device()
480 	pci_iov_add_virtfn(edev->physfn, edev->vf_index);  in eeh_add_virt_device()
499 	    (dev->hdr_type == PCI_HEADER_TYPE_BRIDGE))  in eeh_rmv_device()
505 			if (driver->err_handler &&  in eeh_rmv_device()
506 			    driver->err_handler->error_detected &&  in eeh_rmv_device()
507 			    driver->err_handler->slot_reset) {  in eeh_rmv_device()
518 	edev->mode |= EEH_DEV_DISCONNECTED;  in eeh_rmv_device()
520 		rmv_data->removed_dev_count++;  in eeh_rmv_device()
522 	if (edev->physfn) {  in eeh_rmv_device()
524 		pci_iov_remove_virtfn(edev->physfn, edev->vf_index);  in eeh_rmv_device()
525 		edev->pdev = NULL;  in eeh_rmv_device()
528 			list_add(&edev->rmv_entry, &rmv_data->removed_vf_list);  in eeh_rmv_device()
541 		if (!(edev->mode & EEH_DEV_DISCONNECTED))  in eeh_pe_detach_dev()
544 		edev->mode &= ~(EEH_DEV_DISCONNECTED | EEH_DEV_IRQ_DISABLED);  in eeh_pe_detach_dev()
555  * PE reset (for 3 times), we try to clear the frozen state
569 				return -EIO;  in eeh_clear_pe_frozen_state()
581 	if (pe->state & EEH_PE_RECOVERING)  in eeh_pe_reset_and_recover()
590 	/* Issue reset */  in eeh_pe_reset_and_recover()
614  * eeh_reset_device - Perform actual reset of a pci slot
620  * This routine must be called to do reset on the indicated PE.
621  * During the reset, udev might be invoked because those affected
638 	cnt = pe->freeze_count;  in eeh_reset_device()
639 	tstamp = pe->tstamp;  in eeh_reset_device()
648 	if (any_passed || driver_eeh_aware || (pe->type & EEH_PE_VF)) {  in eeh_reset_device()
657 	 * Reset the pci controller. (Asserts RST#; resets config space).  in eeh_reset_device()
659 	 * up if the reset failed for some reason.  in eeh_reset_device()
661 	 * During the reset, it's very dangerous to have uncontrolled PCI  in eeh_reset_device()
672 	eeh_ops->configure_bridge(pe);  in eeh_reset_device()
682 	/* Give the system 5 seconds to finish running the user-space  in eeh_reset_device()
688 	if (!driver_eeh_aware || rmv_data->removed_dev_count) {  in eeh_reset_device()
698 		edev = list_first_entry(&pe->edevs, struct eeh_dev, entry);  in eeh_reset_device()
700 		if (pe->type & EEH_PE_VF) {  in eeh_reset_device()
710 	pe->tstamp = tstamp;  in eeh_reset_device()
711 	pe->freeze_count = cnt;  in eeh_reset_device()
718  * to come back on line, in seconds.
725  * NB: This needs to be recursive to ensure the leaf PEs get removed
734 	list_for_each_entry_safe(child_pe, tmp, &pe->child_list, child)  in eeh_pe_cleanup()
737 	if (pe->state & EEH_PE_KEEP)  in eeh_pe_cleanup()
740 	if (!(pe->state & EEH_PE_INVALID))  in eeh_pe_cleanup()
743 	if (list_empty(&pe->edevs) && list_empty(&pe->child_list)) {  in eeh_pe_cleanup()
744 		list_del(&pe->child);  in eeh_pe_cleanup()
750  * eeh_check_slot_presence - Check if a device is still present in a slot
771 	if (pdev->error_state == pci_channel_io_perm_failure)  in eeh_slot_presence_check()
774 	slot = pdev->slot;  in eeh_slot_presence_check()
775 	if (!slot || !slot->hotplug)  in eeh_slot_presence_check()
778 	ops = slot->hotplug->ops;  in eeh_slot_presence_check()
779 	if (!ops || !ops->get_adapter_status)  in eeh_slot_presence_check()
783 	if (ops->set_attention_status)  in eeh_slot_presence_check()
784 		ops->set_attention_status(slot->hotplug, 1);  in eeh_slot_presence_check()
786 	rc = ops->get_adapter_status(slot->hotplug, &state);  in eeh_slot_presence_check()
801 	if (pdev->error_state == pci_channel_io_perm_failure)  in eeh_clear_slot_attention()
804 	slot = pdev->slot;  in eeh_clear_slot_attention()
805 	if (!slot || !slot->hotplug)  in eeh_clear_slot_attention()
808 	ops = slot->hotplug->ops;  in eeh_clear_slot_attention()
809 	if (!ops || !ops->set_attention_status)  in eeh_clear_slot_attention()
812 	ops->set_attention_status(slot->hotplug, 0);  in eeh_clear_slot_attention()
816  * eeh_handle_normal_event - Handle EEH events on a specific PE
817  * @pe: EEH PE - which should not be used after we return, as it may
823  * While PHB detects address or data parity errors on particular PCI
827  * #PERR or other misc PCI-related errors also can trigger EEH errors.
849 		pr_err("%s: Cannot find PCI bus for PHB#%x-PE#%x\n",  in eeh_handle_normal_event()
850 			__func__, pe->phb->global_number, pe->addr);  in eeh_handle_normal_event()
855 	 * When devices are hot-removed we might get an EEH due to  in eeh_handle_normal_event()
865 			if (eeh_slot_presence_check(edev->pdev))  in eeh_handle_normal_event()
869 		pr_debug("EEH: Frozen PHB#%x-PE#%x is empty!\n",  in eeh_handle_normal_event()
870 			pe->phb->global_number, pe->addr);  in eeh_handle_normal_event()
875 	if (pe->type & EEH_PE_PHB) {  in eeh_handle_normal_event()
877 			pe->phb->global_number, eeh_pe_loc_get(pe));  in eeh_handle_normal_event()
879 		struct eeh_pe *phb_pe = eeh_phb_pe_get(pe->phb);  in eeh_handle_normal_event()
881 		pr_err("EEH: Recovering PHB#%x-PE#%x\n",  in eeh_handle_normal_event()
882 		       pe->phb->global_number, pe->addr);  in eeh_handle_normal_event()
892 	if (pe->trace_entries) {  in eeh_handle_normal_event()
893 		void **ptrs = (void **) pe->stack_trace;  in eeh_handle_normal_event()
896 		pr_err("EEH: Frozen PHB#%x-PE#%x detected\n",  in eeh_handle_normal_event()
897 		       pe->phb->global_number, pe->addr);  in eeh_handle_normal_event()
901 		for (i = 0; i < pe->trace_entries; i++)  in eeh_handle_normal_event()
904 		pe->trace_entries = 0;  in eeh_handle_normal_event()
909 	pe->freeze_count++;  in eeh_handle_normal_event()
910 	if (pe->freeze_count > eeh_max_freezes) {  in eeh_handle_normal_event()
911 …pr_err("EEH: PHB#%x-PE#%x has failed %d times in the last hour and has been permanently disabled.\…  in eeh_handle_normal_event()
912 		       pe->phb->global_number, pe->addr,  in eeh_handle_normal_event()
913 		       pe->freeze_count);  in eeh_handle_normal_event()
919 			edev->mode &= ~EEH_DEV_NO_HANDLER;  in eeh_handle_normal_event()
922 	 * a reset sequence, giving each an opportunity to do what it needs  in eeh_handle_normal_event()
923 	 * to accomplish the reset.  Each child gets a report of the  in eeh_handle_normal_event()
924 	 * status ... if any child can't handle the reset, then the entire  in eeh_handle_normal_event()
927 	 * When the PHB is fenced, we have to issue a reset to recover from  in eeh_handle_normal_event()
933 			pe->freeze_count, eeh_max_freezes);  in eeh_handle_normal_event()
939 		if ((pe->type & EEH_PE_PHB) &&  in eeh_handle_normal_event()
965 	/* If all device drivers were EEH-unaware, then shut  in eeh_handle_normal_event()
970 		pr_info("EEH: Reset with hotplug activity\n");  in eeh_handle_normal_event()
973 			pr_warn("%s: Unable to reset, err=%d\n",  in eeh_handle_normal_event()
979 	/* If all devices reported they can proceed, then re-enable MMIO */  in eeh_handle_normal_event()
989 			pr_info("EEH: Notify device drivers to resume I/O\n");  in eeh_handle_normal_event()
995 	/* If all devices reported they can proceed, then re-enable DMA */  in eeh_handle_normal_event()
1006 			 * We didn't do PE reset for the case. The PE  in eeh_handle_normal_event()
1015 	/* If any device called out for a reset, then reset the slot */  in eeh_handle_normal_event()
1017 		pr_info("EEH: Reset without hotplug activity\n");  in eeh_handle_normal_event()
1020 			pr_warn("%s: Cannot reset, err=%d\n",  in eeh_handle_normal_event()
1041 			list_del(&edev->rmv_entry);  in eeh_handle_normal_event()
1044 		/* Tell all device drivers that they can resume operations */  in eeh_handle_normal_event()
1045 		pr_info("EEH: Notify device driver to resume\n");  in eeh_handle_normal_event()
1048 		eeh_pe_report("resume", pe, eeh_report_resume, NULL);  in eeh_handle_normal_event()
1051 				edev->mode &= ~EEH_DEV_NO_HANDLER;  in eeh_handle_normal_event()
1052 				edev->in_error = false;  in eeh_handle_normal_event()
1059 		 * About 90% of all real-life EEH failures in the field  in eeh_handle_normal_event()
1063 		pr_err("EEH: Unable to recover from failure from PHB#%x-PE#%x.\n"  in eeh_handle_normal_event()
1065 			pe->phb->global_number, pe->addr);  in eeh_handle_normal_event()
1083 		if (pe->type & EEH_PE_VF) {  in eeh_handle_normal_event()
1108 			eeh_clear_slot_attention(edev->pdev);  in eeh_handle_normal_event()
1114  * eeh_handle_special_event - Handle EEH events without a specific failing PE
1131 		rc = eeh_ops->next_error(&pe);  in eeh_handle_special_event()
1187 					edev->mode &= ~EEH_DEV_NO_HANDLER;  in eeh_handle_special_event()
1200 				    !(phb_pe->state & EEH_PE_ISOLATED) ||  in eeh_handle_special_event()
1201 				    (phb_pe->state & EEH_PE_RECOVERING))  in eeh_handle_special_event()
1207 					       "PHB#%x-PE#%x\n",  in eeh_handle_special_event()
1209 					       pe->phb->global_number,  in eeh_handle_special_event()
1210 					       pe->addr);  in eeh_handle_special_event()