[deliverable/linux.git] / arch / powerpc / platforms / pseries / eeh_driver.c

/*
 * PCI Error Recovery Driver for RPA-compliant PPC64 platform.
 * Copyright (C) 2004, 2005 Linas Vepstas <linas@linas.org>
 *
 * All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or (at
 * your option) any later version.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or
 * NON INFRINGEMENT.  See the GNU General Public License for more
 * details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
 *
 * Send feedback to <linas@us.ibm.com>
 *
 */
#include <linux/delay.h>
#include <linux/irq.h>
#include <linux/interrupt.h>
#include <linux/notifier.h>
#include <linux/pci.h>
#include <asm/eeh.h>
#include <asm/eeh_event.h>
#include <asm/ppc-pci.h>
#include <asm/pci-bridge.h>
#include <asm/prom.h>
#include <asm/rtas.h>


static inline const char * pcid_name (struct pci_dev *pdev)
{
	if (pdev->dev.driver)
		return pdev->dev.driver->name;
	return "";
}

#ifdef DEBUG
static void print_device_node_tree (struct pci_dn *pdn, int dent)
{
	int i;
	if (!pdn) return;
	for (i=0;i<dent; i++)
		printk(" ");
	printk("dn=%s mode=%x \tcfg_addr=%x pe_addr=%x \tfull=%s\n",
		pdn->node->name, pdn->eeh_mode, pdn->eeh_config_addr,
		pdn->eeh_pe_config_addr, pdn->node->full_name);
	dent += 3;
	struct device_node *pc = pdn->node->child;
	while (pc) {
		print_device_node_tree(PCI_DN(pc), dent);
		pc = pc->sibling;
	}
}
#endif

/** 
 * irq_in_use - return true if this irq is being used 
 */
static int irq_in_use(unsigned int irq)
{
	int rc = 0;
	unsigned long flags;
   struct irq_desc *desc = irq_desc + irq;

	spin_lock_irqsave(&desc->lock, flags);
	if (desc->action)
		rc = 1;
	spin_unlock_irqrestore(&desc->lock, flags);
	return rc;
}

/* ------------------------------------------------------- */
/** eeh_report_error - report an EEH error to each device,
 *  collect up and merge the device responses.
 */

static void eeh_report_error(struct pci_dev *dev, void *userdata)
{
	enum pcierr_result rc, *res = userdata;
	struct pci_driver *driver = dev->driver;

	dev->error_state = pci_channel_io_frozen;

	if (!driver)
		return;

	if (irq_in_use (dev->irq)) {
		struct device_node *dn = pci_device_to_OF_node(dev);
		PCI_DN(dn)->eeh_mode |= EEH_MODE_IRQ_DISABLED;
		disable_irq_nosync(dev->irq);
	}
	if (!driver->err_handler)
		return;
	if (!driver->err_handler->error_detected)
		return;

	rc = driver->err_handler->error_detected (dev, pci_channel_io_frozen);
	if (*res == PCIERR_RESULT_NONE) *res = rc;
	if (*res == PCIERR_RESULT_NEED_RESET) return;
	if (*res == PCIERR_RESULT_DISCONNECT &&
	     rc == PCIERR_RESULT_NEED_RESET) *res = rc;
}

/** eeh_report_reset -- tell this device that the pci slot
 *  has been reset.
 */

static void eeh_report_reset(struct pci_dev *dev, void *userdata)
{
	struct pci_driver *driver = dev->driver;
	struct device_node *dn = pci_device_to_OF_node(dev);

	if (!driver)
		return;

	if ((PCI_DN(dn)->eeh_mode) & EEH_MODE_IRQ_DISABLED) {
		PCI_DN(dn)->eeh_mode &= ~EEH_MODE_IRQ_DISABLED;
		enable_irq(dev->irq);
	}
	if (!driver->err_handler)
		return;
	if (!driver->err_handler->slot_reset)
		return;

	driver->err_handler->slot_reset(dev);
}

static void eeh_report_resume(struct pci_dev *dev, void *userdata)
{
	struct pci_driver *driver = dev->driver;

	dev->error_state = pci_channel_io_normal;

	if (!driver)
		return;
	if (!driver->err_handler)
		return;
	if (!driver->err_handler->resume)
		return;

	driver->err_handler->resume(dev);
}

static void eeh_report_failure(struct pci_dev *dev, void *userdata)
{
	struct pci_driver *driver = dev->driver;

	dev->error_state = pci_channel_io_perm_failure;

	if (!driver)
		return;

	if (irq_in_use (dev->irq)) {
		struct device_node *dn = pci_device_to_OF_node(dev);
		PCI_DN(dn)->eeh_mode |= EEH_MODE_IRQ_DISABLED;
		disable_irq_nosync(dev->irq);
	}
	if (!driver->err_handler)
		return;
	if (!driver->err_handler->error_detected)
		return;
	driver->err_handler->error_detected(dev, pci_channel_io_perm_failure);
}

/* ------------------------------------------------------- */
/**
 * handle_eeh_events -- reset a PCI device after hard lockup.
 *
 * pSeries systems will isolate a PCI slot if the PCI-Host
 * bridge detects address or data parity errors, DMA's
 * occuring to wild addresses (which usually happen due to
 * bugs in device drivers or in PCI adapter firmware).
 * Slot isolations also occur if #SERR, #PERR or other misc
 * PCI-related errors are detected.
 *
 * Recovery process consists of unplugging the device driver
 * (which generated hotplug events to userspace), then issuing
 * a PCI #RST to the device, then reconfiguring the PCI config
 * space for all bridges & devices under this slot, and then
 * finally restarting the device drivers (which cause a second
 * set of hotplug events to go out to userspace).
 */

/**
 * eeh_reset_device() -- perform actual reset of a pci slot
 * Args: bus: pointer to the pci bus structure corresponding
 *            to the isolated slot. A non-null value will
 *            cause all devices under the bus to be removed
 *            and then re-added.
 *     pe_dn: pointer to a "Partionable Endpoint" device node.
 *            This is the top-level structure on which pci
 *            bus resets can be performed.
 */

static int eeh_reset_device (struct pci_dn *pe_dn, struct pci_bus *bus)
{
	int rc;
	if (bus)
		pcibios_remove_pci_devices(bus);

	/* Reset the pci controller. (Asserts RST#; resets config space).
	 * Reconfigure bridges and devices. Don't try to bring the system
	 * up if the reset failed for some reason. */
	rc = rtas_set_slot_reset(pe_dn);
	if (rc)
		return rc;

 	/* New-style config addrs might be shared across multiple devices,
 	 * Walk over all functions on this device */
 	if (pe_dn->eeh_pe_config_addr) {
 		struct device_node *pe = pe_dn->node;
 		pe = pe->parent->child;
 		while (pe) {
 			struct pci_dn *ppe = PCI_DN(pe);
 			if (pe_dn->eeh_pe_config_addr == ppe->eeh_pe_config_addr) {
 				rtas_configure_bridge(ppe);
 				eeh_restore_bars(ppe);
 			}
 			pe = pe->sibling;
 		}
 	} else {
 		rtas_configure_bridge(pe_dn);
 		eeh_restore_bars(pe_dn);
 	}

	/* Give the system 5 seconds to finish running the user-space
	 * hotplug shutdown scripts, e.g. ifdown for ethernet.  Yes, 
	 * this is a hack, but if we don't do this, and try to bring 
	 * the device up before the scripts have taken it down, 
	 * potentially weird things happen.
	 */
	if (bus) {
		ssleep (5);
		pcibios_add_pci_devices(bus);
	}

	return 0;
}

/* The longest amount of time to wait for a pci device
 * to come back on line, in seconds.
 */
#define MAX_WAIT_FOR_RECOVERY 15

void handle_eeh_events (struct eeh_event *event)
{
	struct device_node *frozen_dn;
	struct pci_dn *frozen_pdn;
	struct pci_bus *frozen_bus;
	int rc = 0;

	frozen_dn = find_device_pe(event->dn);
	frozen_bus = pcibios_find_pci_bus(frozen_dn);

	if (!frozen_dn) {
		printk(KERN_ERR "EEH: Error: Cannot find partition endpoint for %s\n",
		        pci_name(event->dev));
		return;
	}

	/* There are two different styles for coming up with the PE.
	 * In the old style, it was the highest EEH-capable device
	 * which was always an EADS pci bridge.  In the new style,
	 * there might not be any EADS bridges, and even when there are,
	 * the firmware marks them as "EEH incapable". So another
	 * two-step is needed to find the pci bus.. */
	if (!frozen_bus)
		frozen_bus = pcibios_find_pci_bus (frozen_dn->parent);

	if (!frozen_bus) {
		printk(KERN_ERR "EEH: Cannot find PCI bus for %s\n",
		        frozen_dn->full_name);
		return;
	}

#if 0
	/* We may get "permanent failure" messages on empty slots.
	 * These are false alarms. Empty slots have no child dn. */
	if ((event->state == pci_channel_io_perm_failure) && (frozen_device == NULL))
		return;
#endif

	frozen_pdn = PCI_DN(frozen_dn);
	frozen_pdn->eeh_freeze_count++;
	
	if (frozen_pdn->eeh_freeze_count > EEH_MAX_ALLOWED_FREEZES)
		goto hard_fail;

	/* If the reset state is a '5' and the time to reset is 0 (infinity)
	 * or is more then 15 seconds, then mark this as a permanent failure.
	 */
	if ((event->state == pci_channel_io_perm_failure) &&
	    ((event->time_unavail <= 0) ||
	     (event->time_unavail > MAX_WAIT_FOR_RECOVERY*1000)))
		goto hard_fail;

	eeh_slot_error_detail(frozen_pdn, 1 /* Temporary Error */);
	printk(KERN_WARNING
	   "EEH: This PCI device has failed %d times since last reboot: %s - %s\n",
		frozen_pdn->eeh_freeze_count,
		pci_name (frozen_pdn->pcidev), 
		pcid_name(frozen_pdn->pcidev));

	/* Walk the various device drivers attached to this slot through
	 * a reset sequence, giving each an opportunity to do what it needs
	 * to accomplish the reset.  Each child gets a report of the
	 * status ... if any child can't handle the reset, then the entire
	 * slot is dlpar removed and added.
	 */
	enum pcierr_result result = PCIERR_RESULT_NONE;
	pci_walk_bus(frozen_bus, eeh_report_error, &result);

	/* If all device drivers were EEH-unaware, then shut
	 * down all of the device drivers, and hope they
	 * go down willingly, without panicing the system.
	 */
	if (result == PCIERR_RESULT_NONE) {
		rc = eeh_reset_device(frozen_pdn, frozen_bus);
		if (rc)
			goto hard_fail;
	}

	/* If any device called out for a reset, then reset the slot */
	if (result == PCIERR_RESULT_NEED_RESET) {
		rc = eeh_reset_device(frozen_pdn, NULL);
		if (rc)
			goto hard_fail;
		pci_walk_bus(frozen_bus, eeh_report_reset, 0);
	}

	/* If all devices reported they can proceed, the re-enable PIO */
	if (result == PCIERR_RESULT_CAN_RECOVER) {
		/* XXX Not supported; we brute-force reset the device */
		rc = eeh_reset_device(frozen_pdn, NULL);
		if (rc)
			goto hard_fail;
		pci_walk_bus(frozen_bus, eeh_report_reset, 0);
	}

	/* Tell all device drivers that they can resume operations */
	pci_walk_bus(frozen_bus, eeh_report_resume, 0);

	return;
	
hard_fail:
	/*
	 * About 90% of all real-life EEH failures in the field
	 * are due to poorly seated PCI cards. Only 10% or so are
	 * due to actual, failed cards.
	 */
	printk(KERN_ERR
	   "EEH: PCI device %s - %s has failed %d times \n"
	   "and has been permanently disabled.  Please try reseating\n"
	   "this device or replacing it.\n",
		pci_name (frozen_pdn->pcidev), 
		pcid_name(frozen_pdn->pcidev), 
		frozen_pdn->eeh_freeze_count);

	eeh_slot_error_detail(frozen_pdn, 2 /* Permanent Error */);

	/* Notify all devices that they're about to go down. */
	pci_walk_bus(frozen_bus, eeh_report_failure, 0);

	/* Shut down the device drivers for good. */
	pcibios_remove_pci_devices(frozen_bus);
}

/* ---------- end of file ---------- */
Commit	Line	Data
77bd7415 LV	1	/*
	2	* PCI Error Recovery Driver for RPA-compliant PPC64 platform.
	3	* Copyright (C) 2004, 2005 Linas Vepstas <linas@linas.org>
	4	*
	5	* All rights reserved.
	6	*
	7	* This program is free software; you can redistribute it and/or modify
	8	* it under the terms of the GNU General Public License as published by
	9	* the Free Software Foundation; either version 2 of the License, or (at
	10	* your option) any later version.
	11	*
	12	* This program is distributed in the hope that it will be useful, but
	13	* WITHOUT ANY WARRANTY; without even the implied warranty of
	14	* MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or
	15	* NON INFRINGEMENT. See the GNU General Public License for more
	16	* details.
	17	*
	18	* You should have received a copy of the GNU General Public License
	19	* along with this program; if not, write to the Free Software
	20	* Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
	21	*
	22	* Send feedback to <linas@us.ibm.com>
	23	*
	24	*/
	25	#include <linux/delay.h>
	26	#include <linux/irq.h>
	27	#include <linux/interrupt.h>
	28	#include <linux/notifier.h>
	29	#include <linux/pci.h>
	30	#include <asm/eeh.h>
	31	#include <asm/eeh_event.h>
	32	#include <asm/ppc-pci.h>
	33	#include <asm/pci-bridge.h>
	34	#include <asm/prom.h>
	35	#include <asm/rtas.h>
	36
	37
	38	static inline const char * pcid_name (struct pci_dev *pdev)
	39	{
	40	if (pdev->dev.driver)
	41	return pdev->dev.driver->name;
	42	return "";
	43	}
	44
77bd7415 LV	45	#ifdef DEBUG
	46	static void print_device_node_tree (struct pci_dn *pdn, int dent)
	47	{
	48	int i;
	49	if (!pdn) return;
	50	for (i=0;i<dent; i++)
	51	printk(" ");
	52	printk("dn=%s mode=%x \tcfg_addr=%x pe_addr=%x \tfull=%s\n",
	53	pdn->node->name, pdn->eeh_mode, pdn->eeh_config_addr,
	54	pdn->eeh_pe_config_addr, pdn->node->full_name);
	55	dent += 3;
	56	struct device_node *pc = pdn->node->child;
	57	while (pc) {
	58	print_device_node_tree(PCI_DN(pc), dent);
	59	pc = pc->sibling;
	60	}
	61	}
	62	#endif
	63
	64	/**
	65	* irq_in_use - return true if this irq is being used
	66	*/
	67	static int irq_in_use(unsigned int irq)
	68	{
	69	int rc = 0;
	70	unsigned long flags;
	71	struct irq_desc *desc = irq_desc + irq;
	72
	73	spin_lock_irqsave(&desc->lock, flags);
	74	if (desc->action)
	75	rc = 1;
	76	spin_unlock_irqrestore(&desc->lock, flags);
	77	return rc;
	78	}
	79
	80	/* ------------------------------------------------------- */
	81	/** eeh_report_error - report an EEH error to each device,
	82	* collect up and merge the device responses.
	83	*/
	84
	85	static void eeh_report_error(struct pci_dev dev, void userdata)
	86	{
	87	enum pcierr_result rc, *res = userdata;
	88	struct pci_driver *driver = dev->driver;
	89
	90	dev->error_state = pci_channel_io_frozen;
	91
	92	if (!driver)
	93	return;
	94
	95	if (irq_in_use (dev->irq)) {
	96	struct device_node *dn = pci_device_to_OF_node(dev);
	97	PCI_DN(dn)->eeh_mode \|= EEH_MODE_IRQ_DISABLED;
	98	disable_irq_nosync(dev->irq);
	99	}
	100	if (!driver->err_handler)
	101	return;
	102	if (!driver->err_handler->error_detected)
	103	return;
	104
	105	rc = driver->err_handler->error_detected (dev, pci_channel_io_frozen);
	106	if (res == PCIERR_RESULT_NONE) res = rc;
	107	if (*res == PCIERR_RESULT_NEED_RESET) return;
	108	if (*res == PCIERR_RESULT_DISCONNECT &&
109	rc == PCIERR_RESULT_NEED_RESET) *res = rc;
110	}
111
112	/** eeh_report_reset -- tell this device that the pci slot
113	* has been reset.
114	*/
115
116	static void eeh_report_reset(struct pci_dev dev, void userdata)
117	{
118	struct pci_driver *driver = dev->driver;
119	struct device_node *dn = pci_device_to_OF_node(dev);
120
121	if (!driver)
122	return;
123
124	if ((PCI_DN(dn)->eeh_mode) & EEH_MODE_IRQ_DISABLED) {
125	PCI_DN(dn)->eeh_mode &= ~EEH_MODE_IRQ_DISABLED;
126	enable_irq(dev->irq);
127	}
128	if (!driver->err_handler)
129	return;
130	if (!driver->err_handler->slot_reset)
131	return;
132
133	driver->err_handler->slot_reset(dev);
134	}
135
136	static void eeh_report_resume(struct pci_dev dev, void userdata)
137	{
138	struct pci_driver *driver = dev->driver;
139
140	dev->error_state = pci_channel_io_normal;
141
142	if (!driver)
143	return;
144	if (!driver->err_handler)
145	return;
146	if (!driver->err_handler->resume)
147	return;
148
149	driver->err_handler->resume(dev);
150	}
151
152	static void eeh_report_failure(struct pci_dev dev, void userdata)
153	{
154	struct pci_driver *driver = dev->driver;
155
156	dev->error_state = pci_channel_io_perm_failure;
157
158	if (!driver)
159	return;
160
161	if (irq_in_use (dev->irq)) {
162	struct device_node *dn = pci_device_to_OF_node(dev);
163	PCI_DN(dn)->eeh_mode \|= EEH_MODE_IRQ_DISABLED;
164	disable_irq_nosync(dev->irq);
165	}
166	if (!driver->err_handler)
167	return;
168	if (!driver->err_handler->error_detected)
169	return;
170	driver->err_handler->error_detected(dev, pci_channel_io_perm_failure);
171	}
172
173	/* ------------------------------------------------------- */
174	/**
175	* handle_eeh_events -- reset a PCI device after hard lockup.
176	*
177	* pSeries systems will isolate a PCI slot if the PCI-Host
178	* bridge detects address or data parity errors, DMA's
179	* occuring to wild addresses (which usually happen due to
180	* bugs in device drivers or in PCI adapter firmware).
181	* Slot isolations also occur if #SERR, #PERR or other misc
182	* PCI-related errors are detected.
183	*
184	* Recovery process consists of unplugging the device driver
185	* (which generated hotplug events to userspace), then issuing
186	* a PCI #RST to the device, then reconfiguring the PCI config
187	* space for all bridges & devices under this slot, and then
188	* finally restarting the device drivers (which cause a second
189	* set of hotplug events to go out to userspace).
190	*/
191
192	/**
193	* eeh_reset_device() -- perform actual reset of a pci slot
194	* Args: bus: pointer to the pci bus structure corresponding
195	* to the isolated slot. A non-null value will
196	* cause all devices under the bus to be removed
197	* and then re-added.
198	* pe_dn: pointer to a "Partionable Endpoint" device node.
199	* This is the top-level structure on which pci
200	* bus resets can be performed.
201	*/
202
b6495c0c	203	static int eeh_reset_device (struct pci_dn pe_dn, struct pci_bus bus)
77bd7415	204	{
b6495c0c	205	int rc;
77bd7415 LV	206	if (bus)
	207	pcibios_remove_pci_devices(bus);
	208
	209	/* Reset the pci controller. (Asserts RST#; resets config space).
b6495c0c LV	210	* Reconfigure bridges and devices. Don't try to bring the system
	211	* up if the reset failed for some reason. */
	212	rc = rtas_set_slot_reset(pe_dn);
	213	if (rc)
	214	return rc;
77bd7415	215
3914ac7b LV	216	/* New-style config addrs might be shared across multiple devices,
	217	* Walk over all functions on this device */
	218	if (pe_dn->eeh_pe_config_addr) {
	219	struct device_node *pe = pe_dn->node;
	220	pe = pe->parent->child;
	221	while (pe) {
	222	struct pci_dn *ppe = PCI_DN(pe);
	223	if (pe_dn->eeh_pe_config_addr == ppe->eeh_pe_config_addr) {
	224	rtas_configure_bridge(ppe);
	225	eeh_restore_bars(ppe);
	226	}
	227	pe = pe->sibling;
	228	}
	229	} else {
	230	rtas_configure_bridge(pe_dn);
	231	eeh_restore_bars(pe_dn);
	232	}
77bd7415 LV	233
	234	/* Give the system 5 seconds to finish running the user-space
	235	* hotplug shutdown scripts, e.g. ifdown for ethernet. Yes,
	236	* this is a hack, but if we don't do this, and try to bring
	237	* the device up before the scripts have taken it down,
	238	* potentially weird things happen.
	239	*/
	240	if (bus) {
	241	ssleep (5);
	242	pcibios_add_pci_devices(bus);
	243	}
b6495c0c LV	244
b6495c0c LV	245	return 0;
77bd7415 LV	246	}
	247
	248	/* The longest amount of time to wait for a pci device
	249	* to come back on line, in seconds.
	250	*/
	251	#define MAX_WAIT_FOR_RECOVERY 15
	252
	253	void handle_eeh_events (struct eeh_event *event)
	254	{
	255	struct device_node *frozen_dn;
	256	struct pci_dn *frozen_pdn;
	257	struct pci_bus *frozen_bus;
b6495c0c	258	int rc = 0;
77bd7415 LV	259
	260	frozen_dn = find_device_pe(event->dn);
	261	frozen_bus = pcibios_find_pci_bus(frozen_dn);
	262
	263	if (!frozen_dn) {
	264	printk(KERN_ERR "EEH: Error: Cannot find partition endpoint for %s\n",
	265	pci_name(event->dev));
	266	return;
	267	}
	268
	269	/* There are two different styles for coming up with the PE.
	270	* In the old style, it was the highest EEH-capable device
	271	* which was always an EADS pci bridge. In the new style,
	272	* there might not be any EADS bridges, and even when there are,
	273	* the firmware marks them as "EEH incapable". So another
	274	* two-step is needed to find the pci bus.. */
	275	if (!frozen_bus)
	276	frozen_bus = pcibios_find_pci_bus (frozen_dn->parent);
	277
	278	if (!frozen_bus) {
	279	printk(KERN_ERR "EEH: Cannot find PCI bus for %s\n",
	280	frozen_dn->full_name);
	281	return;
	282	}
	283
	284	#if 0
	285	/* We may get "permanent failure" messages on empty slots.
	286	* These are false alarms. Empty slots have no child dn. */
	287	if ((event->state == pci_channel_io_perm_failure) && (frozen_device == NULL))
	288	return;
	289	#endif
	290
	291	frozen_pdn = PCI_DN(frozen_dn);
	292	frozen_pdn->eeh_freeze_count++;
	293
	294	if (frozen_pdn->eeh_freeze_count > EEH_MAX_ALLOWED_FREEZES)
b6495c0c	295	goto hard_fail;
77bd7415 LV	296
	297	/* If the reset state is a '5' and the time to reset is 0 (infinity)
	298	* or is more then 15 seconds, then mark this as a permanent failure.
	299	*/
	300	if ((event->state == pci_channel_io_perm_failure) &&
	301	((event->time_unavail <= 0) \|\|
	302	(event->time_unavail > MAX_WAIT_FOR_RECOVERY*1000)))
b6495c0c	303	goto hard_fail;
77bd7415 LV	304
	305	eeh_slot_error_detail(frozen_pdn, 1 /* Temporary Error */);
	306	printk(KERN_WARNING
	307	"EEH: This PCI device has failed %d times since last reboot: %s - %s\n",
	308	frozen_pdn->eeh_freeze_count,
	309	pci_name (frozen_pdn->pcidev),
	310	pcid_name(frozen_pdn->pcidev));
	311
	312	/* Walk the various device drivers attached to this slot through
	313	* a reset sequence, giving each an opportunity to do what it needs
	314	* to accomplish the reset. Each child gets a report of the
	315	* status ... if any child can't handle the reset, then the entire
	316	* slot is dlpar removed and added.
	317	*/
	318	enum pcierr_result result = PCIERR_RESULT_NONE;
	319	pci_walk_bus(frozen_bus, eeh_report_error, &result);
	320
	321	/* If all device drivers were EEH-unaware, then shut
	322	* down all of the device drivers, and hope they
	323	* go down willingly, without panicing the system.
	324	*/
	325	if (result == PCIERR_RESULT_NONE) {
b6495c0c LV	326	rc = eeh_reset_device(frozen_pdn, frozen_bus);
	327	if (rc)
	328	goto hard_fail;
77bd7415 LV	329	}
	330
	331	/* If any device called out for a reset, then reset the slot */
	332	if (result == PCIERR_RESULT_NEED_RESET) {
b6495c0c LV	333	rc = eeh_reset_device(frozen_pdn, NULL);
	334	if (rc)
	335	goto hard_fail;
77bd7415 LV	336	pci_walk_bus(frozen_bus, eeh_report_reset, 0);
	337	}
	338
	339	/* If all devices reported they can proceed, the re-enable PIO */
	340	if (result == PCIERR_RESULT_CAN_RECOVER) {
	341	/* XXX Not supported; we brute-force reset the device */
b6495c0c LV	342	rc = eeh_reset_device(frozen_pdn, NULL);
	343	if (rc)
	344	goto hard_fail;
77bd7415 LV	345	pci_walk_bus(frozen_bus, eeh_report_reset, 0);
	346	}
	347
	348	/* Tell all device drivers that they can resume operations */
	349	pci_walk_bus(frozen_bus, eeh_report_resume, 0);
b6495c0c LV	350
	351	return;
	352
	353	hard_fail:
	354	/*
	355	* About 90% of all real-life EEH failures in the field
	356	* are due to poorly seated PCI cards. Only 10% or so are
	357	* due to actual, failed cards.
	358	*/
	359	printk(KERN_ERR
	360	"EEH: PCI device %s - %s has failed %d times \n"
	361	"and has been permanently disabled. Please try reseating\n"
	362	"this device or replacing it.\n",
	363	pci_name (frozen_pdn->pcidev),
	364	pcid_name(frozen_pdn->pcidev),
	365	frozen_pdn->eeh_freeze_count);
	366
	367	eeh_slot_error_detail(frozen_pdn, 2 /* Permanent Error */);
	368
	369	/* Notify all devices that they're about to go down. */
	370	pci_walk_bus(frozen_bus, eeh_report_failure, 0);
	371
	372	/* Shut down the device drivers for good. */
	373	pcibios_remove_pci_devices(frozen_bus);
77bd7415 LV	374	}
	375
	376	/* ---------- end of file ---------- */