[deliverable/linux.git] / arch / mips / pci / msi-xlp.c

/*
 * Copyright (c) 2003-2012 Broadcom Corporation
 * All Rights Reserved
 *
 * This software is available to you under a choice of one of two
 * licenses.  You may choose to be licensed under the terms of the GNU
 * General Public License (GPL) Version 2, available from the file
 * COPYING in the main directory of this source tree, or the Broadcom
 * license below:
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 *
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in
 *    the documentation and/or other materials provided with the
 *    distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY BROADCOM ``AS IS'' AND ANY EXPRESS OR
 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED. IN NO EVENT SHALL BROADCOM OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
 * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
 * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
 * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */

#include <linux/types.h>
#include <linux/pci.h>
#include <linux/kernel.h>
#include <linux/init.h>
#include <linux/msi.h>
#include <linux/mm.h>
#include <linux/irq.h>
#include <linux/irqdesc.h>
#include <linux/console.h>

#include <asm/io.h>

#include <asm/netlogic/interrupt.h>
#include <asm/netlogic/haldefs.h>
#include <asm/netlogic/common.h>
#include <asm/netlogic/mips-extns.h>

#include <asm/netlogic/xlp-hal/iomap.h>
#include <asm/netlogic/xlp-hal/xlp.h>
#include <asm/netlogic/xlp-hal/pic.h>
#include <asm/netlogic/xlp-hal/pcibus.h>
#include <asm/netlogic/xlp-hal/bridge.h>

#define XLP_MSIVEC_PER_LINK	32
#define XLP_MSIXVEC_TOTAL	(cpu_is_xlp9xx() ? 128 : 32)
#define XLP_MSIXVEC_PER_LINK	(cpu_is_xlp9xx() ? 32 : 8)

/* 128 MSI irqs per node, mapped starting at NLM_MSI_VEC_BASE */
static inline int nlm_link_msiirq(int link, int msivec)
{
	return NLM_MSI_VEC_BASE + link * XLP_MSIVEC_PER_LINK + msivec;
}

/* get the link MSI vector from irq number */
static inline int nlm_irq_msivec(int irq)
{
	return (irq - NLM_MSI_VEC_BASE) % XLP_MSIVEC_PER_LINK;
}

/* get the link from the irq number */
static inline int nlm_irq_msilink(int irq)
{
	int total_msivec = XLP_MSIVEC_PER_LINK * PCIE_NLINKS;

	return ((irq - NLM_MSI_VEC_BASE) % total_msivec) /
		XLP_MSIVEC_PER_LINK;
}

/*
 * For XLP 8xx/4xx/3xx/2xx, only 32 MSI-X vectors are possible because
 * there are only 32 PIC interrupts for MSI. We split them statically
 * and use 8 MSI-X vectors per link - this keeps the allocation and
 * lookup simple.
 * On XLP 9xx, there are 32 vectors per link, and the interrupts are
 * not routed thru PIC, so we can use all 128 MSI-X vectors.
 */
static inline int nlm_link_msixirq(int link, int bit)
{
	return NLM_MSIX_VEC_BASE + link * XLP_MSIXVEC_PER_LINK + bit;
}

/* get the link MSI vector from irq number */
static inline int nlm_irq_msixvec(int irq)
{
	return (irq - NLM_MSIX_VEC_BASE) % XLP_MSIXVEC_TOTAL;
}

/* get the link from MSIX vec */
static inline int nlm_irq_msixlink(int msixvec)
{
	return msixvec / XLP_MSIXVEC_PER_LINK;
}

/*
 * Per link MSI and MSI-X information, set as IRQ handler data for
 * MSI and MSI-X interrupts.
 */
struct xlp_msi_data {
	struct nlm_soc_info *node;
	uint64_t	lnkbase;
	uint32_t	msi_enabled_mask;
	uint32_t	msi_alloc_mask;
	uint32_t	msix_alloc_mask;
	spinlock_t	msi_lock;
};

/*
 * MSI Chip definitions
 *
 * On XLP, there is a PIC interrupt associated with each PCIe link on the
 * chip (which appears as a PCI bridge to us). This gives us 32 MSI irqa
 * per link and 128 overall.
 *
 * When a device connected to the link raises a MSI interrupt, we get a
 * link interrupt and we then have to look at PCIE_MSI_STATUS register at
 * the bridge to map it to the IRQ
 */
static void xlp_msi_enable(struct irq_data *d)
{
	struct xlp_msi_data *md = irq_data_get_irq_chip_data(d);
	unsigned long flags;
	int vec;

	vec = nlm_irq_msivec(d->irq);
	spin_lock_irqsave(&md->msi_lock, flags);
	md->msi_enabled_mask |= 1u << vec;
	if (cpu_is_xlp9xx())
		nlm_write_reg(md->lnkbase, PCIE_9XX_MSI_EN,
				md->msi_enabled_mask);
	else
		nlm_write_reg(md->lnkbase, PCIE_MSI_EN, md->msi_enabled_mask);
	spin_unlock_irqrestore(&md->msi_lock, flags);
}

static void xlp_msi_disable(struct irq_data *d)
{
	struct xlp_msi_data *md = irq_data_get_irq_chip_data(d);
	unsigned long flags;
	int vec;

	vec = nlm_irq_msivec(d->irq);
	spin_lock_irqsave(&md->msi_lock, flags);
	md->msi_enabled_mask &= ~(1u << vec);
	if (cpu_is_xlp9xx())
		nlm_write_reg(md->lnkbase, PCIE_9XX_MSI_EN,
				md->msi_enabled_mask);
	else
		nlm_write_reg(md->lnkbase, PCIE_MSI_EN, md->msi_enabled_mask);
	spin_unlock_irqrestore(&md->msi_lock, flags);
}

static void xlp_msi_mask_ack(struct irq_data *d)
{
	struct xlp_msi_data *md = irq_data_get_irq_chip_data(d);
	int link, vec;

	link = nlm_irq_msilink(d->irq);
	vec = nlm_irq_msivec(d->irq);
	xlp_msi_disable(d);

	/* Ack MSI on bridge */
	if (cpu_is_xlp9xx())
		nlm_write_reg(md->lnkbase, PCIE_9XX_MSI_STATUS, 1u << vec);
	else
		nlm_write_reg(md->lnkbase, PCIE_MSI_STATUS, 1u << vec);

}

static struct irq_chip xlp_msi_chip = {
	.name		= "XLP-MSI",
	.irq_enable	= xlp_msi_enable,
	.irq_disable	= xlp_msi_disable,
	.irq_mask_ack	= xlp_msi_mask_ack,
	.irq_unmask	= xlp_msi_enable,
};

/*
 * XLP8XX/4XX/3XX/2XX:
 * The MSI-X interrupt handling is different from MSI, there are 32 MSI-X
 * interrupts generated by the PIC and each of these correspond to a MSI-X
 * vector (0-31) that can be assigned.
 *
 * We divide the MSI-X vectors to 8 per link and do a per-link allocation
 *
 * XLP9XX:
 * 32 MSI-X vectors are available per link, and the interrupts are not routed
 * thru the PIC. PIC ack not needed.
 *
 * Enable and disable done using standard MSI functions.
 */
static void xlp_msix_mask_ack(struct irq_data *d)
{
	struct xlp_msi_data *md;
	int link, msixvec;
	uint32_t status_reg, bit;

	msixvec = nlm_irq_msixvec(d->irq);
	link = nlm_irq_msixlink(msixvec);
	pci_msi_mask_irq(d);
	md = irq_data_get_irq_chip_data(d);

	/* Ack MSI on bridge */
	if (cpu_is_xlp9xx()) {
		status_reg = PCIE_9XX_MSIX_STATUSX(link);
		bit = msixvec % XLP_MSIXVEC_PER_LINK;
	} else {
		status_reg = PCIE_MSIX_STATUS;
		bit = msixvec;
	}
	nlm_write_reg(md->lnkbase, status_reg, 1u << bit);

	if (!cpu_is_xlp9xx())
		nlm_pic_ack(md->node->picbase,
				PIC_IRT_PCIE_MSIX_INDEX(msixvec));
}

static struct irq_chip xlp_msix_chip = {
	.name		= "XLP-MSIX",
	.irq_enable	= pci_msi_unmask_irq,
	.irq_disable	= pci_msi_mask_irq,
	.irq_mask_ack	= xlp_msix_mask_ack,
	.irq_unmask	= pci_msi_unmask_irq,
};

void arch_teardown_msi_irq(unsigned int irq)
{
}

/*
 * Setup a PCIe link for MSI.  By default, the links are in
 * legacy interrupt mode.  We will switch them to MSI mode
 * at the first MSI request.
 */
static void xlp_config_link_msi(uint64_t lnkbase, int lirq, uint64_t msiaddr)
{
	u32 val;

	if (cpu_is_xlp9xx()) {
		val = nlm_read_reg(lnkbase, PCIE_9XX_INT_EN0);
		if ((val & 0x200) == 0) {
			val |= 0x200;		/* MSI Interrupt enable */
			nlm_write_reg(lnkbase, PCIE_9XX_INT_EN0, val);
		}
	} else {
		val = nlm_read_reg(lnkbase, PCIE_INT_EN0);
		if ((val & 0x200) == 0) {
			val |= 0x200;
			nlm_write_reg(lnkbase, PCIE_INT_EN0, val);
		}
	}

	val = nlm_read_reg(lnkbase, 0x1);	/* CMD */
	if ((val & 0x0400) == 0) {
		val |= 0x0400;
		nlm_write_reg(lnkbase, 0x1, val);
	}

	/* Update IRQ in the PCI irq reg */
	val = nlm_read_pci_reg(lnkbase, 0xf);
	val &= ~0x1fu;
	val |= (1 << 8) | lirq;
	nlm_write_pci_reg(lnkbase, 0xf, val);

	/* MSI addr */
	nlm_write_reg(lnkbase, PCIE_BRIDGE_MSI_ADDRH, msiaddr >> 32);
	nlm_write_reg(lnkbase, PCIE_BRIDGE_MSI_ADDRL, msiaddr & 0xffffffff);

	/* MSI cap for bridge */
	val = nlm_read_reg(lnkbase, PCIE_BRIDGE_MSI_CAP);
	if ((val & (1 << 16)) == 0) {
		val |= 0xb << 16;		/* mmc32, msi enable */
		nlm_write_reg(lnkbase, PCIE_BRIDGE_MSI_CAP, val);
	}
}

/*
 * Allocate a MSI vector on a link
 */
static int xlp_setup_msi(uint64_t lnkbase, int node, int link,
	struct msi_desc *desc)
{
	struct xlp_msi_data *md;
	struct msi_msg msg;
	unsigned long flags;
	int msivec, irt, lirq, xirq, ret;
	uint64_t msiaddr;

	/* Get MSI data for the link */
	lirq = PIC_PCIE_LINK_MSI_IRQ(link);
	xirq = nlm_irq_to_xirq(node, nlm_link_msiirq(link, 0));
	md = irq_get_chip_data(xirq);
	msiaddr = MSI_LINK_ADDR(node, link);

	spin_lock_irqsave(&md->msi_lock, flags);
	if (md->msi_alloc_mask == 0) {
		xlp_config_link_msi(lnkbase, lirq, msiaddr);
		/* switch the link IRQ to MSI range */
		if (cpu_is_xlp9xx())
			irt = PIC_9XX_IRT_PCIE_LINK_INDEX(link);
		else
			irt = PIC_IRT_PCIE_LINK_INDEX(link);
		nlm_setup_pic_irq(node, lirq, lirq, irt);
		nlm_pic_init_irt(nlm_get_node(node)->picbase, irt, lirq,
				 node * nlm_threads_per_node(), 1 /*en */);
	}

	/* allocate a MSI vec, and tell the bridge about it */
	msivec = fls(md->msi_alloc_mask);
	if (msivec == XLP_MSIVEC_PER_LINK) {
		spin_unlock_irqrestore(&md->msi_lock, flags);
		return -ENOMEM;
	}
	md->msi_alloc_mask |= (1u << msivec);
	spin_unlock_irqrestore(&md->msi_lock, flags);

	msg.address_hi = msiaddr >> 32;
	msg.address_lo = msiaddr & 0xffffffff;
	msg.data = 0xc00 | msivec;

	xirq = xirq + msivec;		/* msi mapped to global irq space */
	ret = irq_set_msi_desc(xirq, desc);
	if (ret < 0)
		return ret;

	pci_write_msi_msg(xirq, &msg);
	return 0;
}

/*
 * Switch a link to MSI-X mode
 */
static void xlp_config_link_msix(uint64_t lnkbase, int lirq, uint64_t msixaddr)
{
	u32 val;

	val = nlm_read_reg(lnkbase, 0x2C);
	if ((val & 0x80000000U) == 0) {
		val |= 0x80000000U;
		nlm_write_reg(lnkbase, 0x2C, val);
	}

	if (cpu_is_xlp9xx()) {
		val = nlm_read_reg(lnkbase, PCIE_9XX_INT_EN0);
		if ((val & 0x200) == 0) {
			val |= 0x200;		/* MSI Interrupt enable */
			nlm_write_reg(lnkbase, PCIE_9XX_INT_EN0, val);
		}
	} else {
		val = nlm_read_reg(lnkbase, PCIE_INT_EN0);
		if ((val & 0x200) == 0) {
			val |= 0x200;		/* MSI Interrupt enable */
			nlm_write_reg(lnkbase, PCIE_INT_EN0, val);
		}
	}

	val = nlm_read_reg(lnkbase, 0x1);	/* CMD */
	if ((val & 0x0400) == 0) {
		val |= 0x0400;
		nlm_write_reg(lnkbase, 0x1, val);
	}

	/* Update IRQ in the PCI irq reg */
	val = nlm_read_pci_reg(lnkbase, 0xf);
	val &= ~0x1fu;
	val |= (1 << 8) | lirq;
	nlm_write_pci_reg(lnkbase, 0xf, val);

	if (cpu_is_xlp9xx()) {
		/* MSI-X addresses */
		nlm_write_reg(lnkbase, PCIE_9XX_BRIDGE_MSIX_ADDR_BASE,
				msixaddr >> 8);
		nlm_write_reg(lnkbase, PCIE_9XX_BRIDGE_MSIX_ADDR_LIMIT,
				(msixaddr + MSI_ADDR_SZ) >> 8);
	} else {
		/* MSI-X addresses */
		nlm_write_reg(lnkbase, PCIE_BRIDGE_MSIX_ADDR_BASE,
				msixaddr >> 8);
		nlm_write_reg(lnkbase, PCIE_BRIDGE_MSIX_ADDR_LIMIT,
				(msixaddr + MSI_ADDR_SZ) >> 8);
	}
}

/*
 *  Allocate a MSI-X vector
 */
static int xlp_setup_msix(uint64_t lnkbase, int node, int link,
	struct msi_desc *desc)
{
	struct xlp_msi_data *md;
	struct msi_msg msg;
	unsigned long flags;
	int t, msixvec, lirq, xirq, ret;
	uint64_t msixaddr;

	/* Get MSI data for the link */
	lirq = PIC_PCIE_MSIX_IRQ(link);
	xirq = nlm_irq_to_xirq(node, nlm_link_msixirq(link, 0));
	md = irq_get_chip_data(xirq);
	msixaddr = MSIX_LINK_ADDR(node, link);

	spin_lock_irqsave(&md->msi_lock, flags);
	/* switch the PCIe link to MSI-X mode at the first alloc */
	if (md->msix_alloc_mask == 0)
		xlp_config_link_msix(lnkbase, lirq, msixaddr);

	/* allocate a MSI-X vec, and tell the bridge about it */
	t = fls(md->msix_alloc_mask);
	if (t == XLP_MSIXVEC_PER_LINK) {
		spin_unlock_irqrestore(&md->msi_lock, flags);
		return -ENOMEM;
	}
	md->msix_alloc_mask |= (1u << t);
	spin_unlock_irqrestore(&md->msi_lock, flags);

	xirq += t;
	msixvec = nlm_irq_msixvec(xirq);

	msg.address_hi = msixaddr >> 32;
	msg.address_lo = msixaddr & 0xffffffff;
	msg.data = 0xc00 | msixvec;

	ret = irq_set_msi_desc(xirq, desc);
	if (ret < 0)
		return ret;

	pci_write_msi_msg(xirq, &msg);
	return 0;
}

int arch_setup_msi_irq(struct pci_dev *dev, struct msi_desc *desc)
{
	struct pci_dev *lnkdev;
	uint64_t lnkbase;
	int node, link, slot;

	lnkdev = xlp_get_pcie_link(dev);
	if (lnkdev == NULL) {
		dev_err(&dev->dev, "Could not find bridge\n");
		return 1;
	}
	slot = PCI_SLOT(lnkdev->devfn);
	link = PCI_FUNC(lnkdev->devfn);
	node = slot / 8;
	lnkbase = nlm_get_pcie_base(node, link);

	if (desc->msi_attrib.is_msix)
		return xlp_setup_msix(lnkbase, node, link, desc);
	else
		return xlp_setup_msi(lnkbase, node, link, desc);
}

void __init xlp_init_node_msi_irqs(int node, int link)
{
	struct nlm_soc_info *nodep;
	struct xlp_msi_data *md;
	int irq, i, irt, msixvec, val;

	pr_info("[%d %d] Init node PCI IRT\n", node, link);
	nodep = nlm_get_node(node);

	/* Alloc an MSI block for the link */
	md = kzalloc(sizeof(*md), GFP_KERNEL);
	spin_lock_init(&md->msi_lock);
	md->msi_enabled_mask = 0;
	md->msi_alloc_mask = 0;
	md->msix_alloc_mask = 0;
	md->node = nodep;
	md->lnkbase = nlm_get_pcie_base(node, link);

	/* extended space for MSI interrupts */
	irq = nlm_irq_to_xirq(node, nlm_link_msiirq(link, 0));
	for (i = irq; i < irq + XLP_MSIVEC_PER_LINK; i++) {
		irq_set_chip_and_handler(i, &xlp_msi_chip, handle_level_irq);
		irq_set_chip_data(i, md);
	}

	for (i = 0; i < XLP_MSIXVEC_PER_LINK ; i++) {
		if (cpu_is_xlp9xx()) {
			val = ((node * nlm_threads_per_node()) << 7 |
				PIC_PCIE_MSIX_IRQ(link) << 1 | 0 << 0);
			nlm_write_pcie_reg(md->lnkbase, PCIE_9XX_MSIX_VECX(i +
					(link * XLP_MSIXVEC_PER_LINK)), val);
		} else {
			/* Initialize MSI-X irts to generate one interrupt
			 * per link
			 */
			msixvec = link * XLP_MSIXVEC_PER_LINK + i;
			irt = PIC_IRT_PCIE_MSIX_INDEX(msixvec);
			nlm_pic_init_irt(nodep->picbase, irt,
					PIC_PCIE_MSIX_IRQ(link),
					node * nlm_threads_per_node(), 1);
		}

		/* Initialize MSI-X extended irq space for the link  */
		irq = nlm_irq_to_xirq(node, nlm_link_msixirq(link, i));
		irq_set_chip_and_handler(irq, &xlp_msix_chip, handle_level_irq);
		irq_set_chip_data(irq, md);
	}
}

void nlm_dispatch_msi(int node, int lirq)
{
	struct xlp_msi_data *md;
	int link, i, irqbase;
	u32 status;

	link = lirq - PIC_PCIE_LINK_MSI_IRQ_BASE;
	irqbase = nlm_irq_to_xirq(node, nlm_link_msiirq(link, 0));
	md = irq_get_chip_data(irqbase);
	if (cpu_is_xlp9xx())
		status = nlm_read_reg(md->lnkbase, PCIE_9XX_MSI_STATUS) &
						md->msi_enabled_mask;
	else
		status = nlm_read_reg(md->lnkbase, PCIE_MSI_STATUS) &
						md->msi_enabled_mask;
	while (status) {
		i = __ffs(status);
		do_IRQ(irqbase + i);
		status &= status - 1;
	}

	/* Ack at eirr and PIC */
	ack_c0_eirr(PIC_PCIE_LINK_MSI_IRQ(link));
	if (cpu_is_xlp9xx())
		nlm_pic_ack(md->node->picbase,
				PIC_9XX_IRT_PCIE_LINK_INDEX(link));
	else
		nlm_pic_ack(md->node->picbase, PIC_IRT_PCIE_LINK_INDEX(link));
}

void nlm_dispatch_msix(int node, int lirq)
{
	struct xlp_msi_data *md;
	int link, i, irqbase;
	u32 status;

	link = lirq - PIC_PCIE_MSIX_IRQ_BASE;
	irqbase = nlm_irq_to_xirq(node, nlm_link_msixirq(link, 0));
	md = irq_get_chip_data(irqbase);
	if (cpu_is_xlp9xx())
		status = nlm_read_reg(md->lnkbase, PCIE_9XX_MSIX_STATUSX(link));
	else
		status = nlm_read_reg(md->lnkbase, PCIE_MSIX_STATUS);

	/* narrow it down to the MSI-x vectors for our link */
	if (!cpu_is_xlp9xx())
		status = (status >> (link * XLP_MSIXVEC_PER_LINK)) &
			((1 << XLP_MSIXVEC_PER_LINK) - 1);

	while (status) {
		i = __ffs(status);
		do_IRQ(irqbase + i);
		status &= status - 1;
	}
	/* Ack at eirr and PIC */
	ack_c0_eirr(PIC_PCIE_MSIX_IRQ(link));
}
Commit	Line	Data
c24a8a7a J	1	/*
	2	* Copyright (c) 2003-2012 Broadcom Corporation
	3	* All Rights Reserved
	4	*
	5	* This software is available to you under a choice of one of two
	6	* licenses. You may choose to be licensed under the terms of the GNU
	7	* General Public License (GPL) Version 2, available from the file
	8	* COPYING in the main directory of this source tree, or the Broadcom
	9	* license below:
	10	*
	11	* Redistribution and use in source and binary forms, with or without
	12	* modification, are permitted provided that the following conditions
	13	* are met:
	14	*
	15	* 1. Redistributions of source code must retain the above copyright
	16	* notice, this list of conditions and the following disclaimer.
	17	* 2. Redistributions in binary form must reproduce the above copyright
	18	* notice, this list of conditions and the following disclaimer in
	19	* the documentation and/or other materials provided with the
	20	* distribution.
	21	*
	22	* THIS SOFTWARE IS PROVIDED BY BROADCOM ``AS IS'' AND ANY EXPRESS OR
	23	* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
	24	* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
	25	* ARE DISCLAIMED. IN NO EVENT SHALL BROADCOM OR CONTRIBUTORS BE LIABLE
	26	* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
	27	* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
	28	* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
	29	* BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
	30	* WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
	31	* OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
	32	* IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
	33	*/
	34
	35	#include <linux/types.h>
	36	#include <linux/pci.h>
	37	#include <linux/kernel.h>
	38	#include <linux/init.h>
	39	#include <linux/msi.h>
	40	#include <linux/mm.h>
	41	#include <linux/irq.h>
	42	#include <linux/irqdesc.h>
	43	#include <linux/console.h>
	44
	45	#include <asm/io.h>
	46
	47	#include <asm/netlogic/interrupt.h>
	48	#include <asm/netlogic/haldefs.h>
	49	#include <asm/netlogic/common.h>
	50	#include <asm/netlogic/mips-extns.h>
	51
	52	#include <asm/netlogic/xlp-hal/iomap.h>
	53	#include <asm/netlogic/xlp-hal/xlp.h>
	54	#include <asm/netlogic/xlp-hal/pic.h>
	55	#include <asm/netlogic/xlp-hal/pcibus.h>
	56	#include <asm/netlogic/xlp-hal/bridge.h>
	57
	58	#define XLP_MSIVEC_PER_LINK 32
d66f3f0e GR	59	#define XLP_MSIXVEC_TOTAL (cpu_is_xlp9xx() ? 128 : 32)
d66f3f0e GR	60	#define XLP_MSIXVEC_PER_LINK (cpu_is_xlp9xx() ? 32 : 8)
c24a8a7a J	61
	62	/* 128 MSI irqs per node, mapped starting at NLM_MSI_VEC_BASE */
	63	static inline int nlm_link_msiirq(int link, int msivec)
	64	{
	65	return NLM_MSI_VEC_BASE + link * XLP_MSIVEC_PER_LINK + msivec;
	66	}
	67
d66f3f0e	68	/* get the link MSI vector from irq number */
c24a8a7a J	69	static inline int nlm_irq_msivec(int irq)
c24a8a7a J	70	{
d66f3f0e	71	return (irq - NLM_MSI_VEC_BASE) % XLP_MSIVEC_PER_LINK;
c24a8a7a J	72	}
c24a8a7a J	73
d66f3f0e	74	/* get the link from the irq number */
c24a8a7a J	75	static inline int nlm_irq_msilink(int irq)
c24a8a7a J	76	{
d66f3f0e GR	77	int total_msivec = XLP_MSIVEC_PER_LINK * PCIE_NLINKS;
	78
	79	return ((irq - NLM_MSI_VEC_BASE) % total_msivec) /
	80	XLP_MSIVEC_PER_LINK;
c24a8a7a J	81	}
	82
	83	/*
d66f3f0e GR	84	* For XLP 8xx/4xx/3xx/2xx, only 32 MSI-X vectors are possible because
	85	* there are only 32 PIC interrupts for MSI. We split them statically
	86	* and use 8 MSI-X vectors per link - this keeps the allocation and
	87	* lookup simple.
	88	* On XLP 9xx, there are 32 vectors per link, and the interrupts are
	89	* not routed thru PIC, so we can use all 128 MSI-X vectors.
c24a8a7a J	90	*/
	91	static inline int nlm_link_msixirq(int link, int bit)
	92	{
	93	return NLM_MSIX_VEC_BASE + link * XLP_MSIXVEC_PER_LINK + bit;
	94	}
	95
d66f3f0e	96	/* get the link MSI vector from irq number */
c24a8a7a J	97	static inline int nlm_irq_msixvec(int irq)
c24a8a7a J	98	{
d66f3f0e	99	return (irq - NLM_MSIX_VEC_BASE) % XLP_MSIXVEC_TOTAL;
c24a8a7a J	100	}
c24a8a7a J	101
d66f3f0e GR	102	/* get the link from MSIX vec */
d66f3f0e GR	103	static inline int nlm_irq_msixlink(int msixvec)
c24a8a7a	104	{
d66f3f0e	105	return msixvec / XLP_MSIXVEC_PER_LINK;
c24a8a7a J	106	}
	107
	108	/*
	109	* Per link MSI and MSI-X information, set as IRQ handler data for
	110	* MSI and MSI-X interrupts.
	111	*/
	112	struct xlp_msi_data {
	113	struct nlm_soc_info *node;
	114	uint64_t lnkbase;
	115	uint32_t msi_enabled_mask;
	116	uint32_t msi_alloc_mask;
	117	uint32_t msix_alloc_mask;
	118	spinlock_t msi_lock;
	119	};
	120
	121	/*
	122	* MSI Chip definitions
	123	*
	124	* On XLP, there is a PIC interrupt associated with each PCIe link on the
	125	* chip (which appears as a PCI bridge to us). This gives us 32 MSI irqa
	126	* per link and 128 overall.
	127	*
	128	* When a device connected to the link raises a MSI interrupt, we get a
	129	* link interrupt and we then have to look at PCIE_MSI_STATUS register at
	130	* the bridge to map it to the IRQ
	131	*/
	132	static void xlp_msi_enable(struct irq_data *d)
	133	{
325f0a18	134	struct xlp_msi_data *md = irq_data_get_irq_chip_data(d);
c24a8a7a J	135	unsigned long flags;
	136	int vec;
	137
	138	vec = nlm_irq_msivec(d->irq);
	139	spin_lock_irqsave(&md->msi_lock, flags);
	140	md->msi_enabled_mask \|= 1u << vec;
d66f3f0e GR	141	if (cpu_is_xlp9xx())
	142	nlm_write_reg(md->lnkbase, PCIE_9XX_MSI_EN,
	143	md->msi_enabled_mask);
	144	else
	145	nlm_write_reg(md->lnkbase, PCIE_MSI_EN, md->msi_enabled_mask);
c24a8a7a J	146	spin_unlock_irqrestore(&md->msi_lock, flags);
	147	}
	148
	149	static void xlp_msi_disable(struct irq_data *d)
	150	{
325f0a18	151	struct xlp_msi_data *md = irq_data_get_irq_chip_data(d);
c24a8a7a J	152	unsigned long flags;
	153	int vec;
	154
	155	vec = nlm_irq_msivec(d->irq);
	156	spin_lock_irqsave(&md->msi_lock, flags);
	157	md->msi_enabled_mask &= ~(1u << vec);
d66f3f0e GR	158	if (cpu_is_xlp9xx())
	159	nlm_write_reg(md->lnkbase, PCIE_9XX_MSI_EN,
	160	md->msi_enabled_mask);
	161	else
	162	nlm_write_reg(md->lnkbase, PCIE_MSI_EN, md->msi_enabled_mask);
c24a8a7a J	163	spin_unlock_irqrestore(&md->msi_lock, flags);
	164	}
	165
	166	static void xlp_msi_mask_ack(struct irq_data *d)
	167	{
325f0a18	168	struct xlp_msi_data *md = irq_data_get_irq_chip_data(d);
c24a8a7a J	169	int link, vec;
	170
	171	link = nlm_irq_msilink(d->irq);
	172	vec = nlm_irq_msivec(d->irq);
	173	xlp_msi_disable(d);
	174
	175	/* Ack MSI on bridge */
d66f3f0e GR	176	if (cpu_is_xlp9xx())
	177	nlm_write_reg(md->lnkbase, PCIE_9XX_MSI_STATUS, 1u << vec);
	178	else
	179	nlm_write_reg(md->lnkbase, PCIE_MSI_STATUS, 1u << vec);
c24a8a7a	180
c24a8a7a J	181	}
	182
	183	static struct irq_chip xlp_msi_chip = {
	184	.name = "XLP-MSI",
	185	.irq_enable = xlp_msi_enable,
	186	.irq_disable = xlp_msi_disable,
	187	.irq_mask_ack = xlp_msi_mask_ack,
	188	.irq_unmask = xlp_msi_enable,
	189	};
	190
	191	/*
d66f3f0e GR	192	* XLP8XX/4XX/3XX/2XX:
	193	* The MSI-X interrupt handling is different from MSI, there are 32 MSI-X
	194	* interrupts generated by the PIC and each of these correspond to a MSI-X
	195	* vector (0-31) that can be assigned.
c24a8a7a	196	*
d66f3f0e GR	197	* We divide the MSI-X vectors to 8 per link and do a per-link allocation
	198	*
	199	* XLP9XX:
	200	* 32 MSI-X vectors are available per link, and the interrupts are not routed
	201	* thru the PIC. PIC ack not needed.
c24a8a7a J	202	*
	203	* Enable and disable done using standard MSI functions.
	204	*/
	205	static void xlp_msix_mask_ack(struct irq_data *d)
	206	{
d66f3f0e	207	struct xlp_msi_data *md;
c24a8a7a	208	int link, msixvec;
d66f3f0e	209	uint32_t status_reg, bit;
c24a8a7a J	210
c24a8a7a J	211	msixvec = nlm_irq_msixvec(d->irq);
d66f3f0e	212	link = nlm_irq_msixlink(msixvec);
280510f1	213	pci_msi_mask_irq(d);
325f0a18	214	md = irq_data_get_irq_chip_data(d);
c24a8a7a J	215
c24a8a7a J	216	/* Ack MSI on bridge */
d66f3f0e GR	217	if (cpu_is_xlp9xx()) {
	218	status_reg = PCIE_9XX_MSIX_STATUSX(link);
	219	bit = msixvec % XLP_MSIXVEC_PER_LINK;
	220	} else {
	221	status_reg = PCIE_MSIX_STATUS;
	222	bit = msixvec;
	223	}
	224	nlm_write_reg(md->lnkbase, status_reg, 1u << bit);
c24a8a7a	225
d66f3f0e GR	226	if (!cpu_is_xlp9xx())
	227	nlm_pic_ack(md->node->picbase,
	228	PIC_IRT_PCIE_MSIX_INDEX(msixvec));
c24a8a7a J	229	}
	230
	231	static struct irq_chip xlp_msix_chip = {
	232	.name = "XLP-MSIX",
280510f1 TG	233	.irq_enable = pci_msi_unmask_irq,
280510f1 TG	234	.irq_disable = pci_msi_mask_irq,
c24a8a7a	235	.irq_mask_ack = xlp_msix_mask_ack,
280510f1	236	.irq_unmask = pci_msi_unmask_irq,
c24a8a7a J	237	};
c24a8a7a J	238
c24a8a7a J	239	void arch_teardown_msi_irq(unsigned int irq)
c24a8a7a J	240	{
c24a8a7a J	241	}
	242
	243	/*
	244	* Setup a PCIe link for MSI. By default, the links are in
	245	* legacy interrupt mode. We will switch them to MSI mode
	246	* at the first MSI request.
	247	*/
	248	static void xlp_config_link_msi(uint64_t lnkbase, int lirq, uint64_t msiaddr)
	249	{
	250	u32 val;
	251
d66f3f0e GR	252	if (cpu_is_xlp9xx()) {
	253	val = nlm_read_reg(lnkbase, PCIE_9XX_INT_EN0);
	254	if ((val & 0x200) == 0) {
	255	val \|= 0x200; /* MSI Interrupt enable */
	256	nlm_write_reg(lnkbase, PCIE_9XX_INT_EN0, val);
	257	}
	258	} else {
	259	val = nlm_read_reg(lnkbase, PCIE_INT_EN0);
	260	if ((val & 0x200) == 0) {
	261	val \|= 0x200;
	262	nlm_write_reg(lnkbase, PCIE_INT_EN0, val);
	263	}
c24a8a7a J	264	}
	265
	266	val = nlm_read_reg(lnkbase, 0x1); /* CMD */
	267	if ((val & 0x0400) == 0) {
	268	val \|= 0x0400;
	269	nlm_write_reg(lnkbase, 0x1, val);
	270	}
	271
	272	/* Update IRQ in the PCI irq reg */
	273	val = nlm_read_pci_reg(lnkbase, 0xf);
	274	val &= ~0x1fu;
	275	val \|= (1 << 8) \| lirq;
	276	nlm_write_pci_reg(lnkbase, 0xf, val);
	277
	278	/* MSI addr */
	279	nlm_write_reg(lnkbase, PCIE_BRIDGE_MSI_ADDRH, msiaddr >> 32);
	280	nlm_write_reg(lnkbase, PCIE_BRIDGE_MSI_ADDRL, msiaddr & 0xffffffff);
	281
	282	/* MSI cap for bridge */
	283	val = nlm_read_reg(lnkbase, PCIE_BRIDGE_MSI_CAP);
	284	if ((val & (1 << 16)) == 0) {
	285	val \|= 0xb << 16; /* mmc32, msi enable */
	286	nlm_write_reg(lnkbase, PCIE_BRIDGE_MSI_CAP, val);
	287	}
	288	}
	289
	290	/*
	291	* Allocate a MSI vector on a link
	292	*/
	293	static int xlp_setup_msi(uint64_t lnkbase, int node, int link,
	294	struct msi_desc *desc)
	295	{
	296	struct xlp_msi_data *md;
	297	struct msi_msg msg;
	298	unsigned long flags;
	299	int msivec, irt, lirq, xirq, ret;
	300	uint64_t msiaddr;
	301
	302	/* Get MSI data for the link */
	303	lirq = PIC_PCIE_LINK_MSI_IRQ(link);
	304	xirq = nlm_irq_to_xirq(node, nlm_link_msiirq(link, 0));
325f0a18	305	md = irq_get_chip_data(xirq);
c24a8a7a J	306	msiaddr = MSI_LINK_ADDR(node, link);
	307
	308	spin_lock_irqsave(&md->msi_lock, flags);
	309	if (md->msi_alloc_mask == 0) {
c24a8a7a	310	xlp_config_link_msi(lnkbase, lirq, msiaddr);
d66f3f0e GR	311	/* switch the link IRQ to MSI range */
	312	if (cpu_is_xlp9xx())
	313	irt = PIC_9XX_IRT_PCIE_LINK_INDEX(link);
	314	else
	315	irt = PIC_IRT_PCIE_LINK_INDEX(link);
c24a8a7a J	316	nlm_setup_pic_irq(node, lirq, lirq, irt);
c24a8a7a J	317	nlm_pic_init_irt(nlm_get_node(node)->picbase, irt, lirq,
98d4884c	318	node * nlm_threads_per_node(), 1 /en /);
c24a8a7a J	319	}
	320
	321	/* allocate a MSI vec, and tell the bridge about it */
	322	msivec = fls(md->msi_alloc_mask);
	323	if (msivec == XLP_MSIVEC_PER_LINK) {
	324	spin_unlock_irqrestore(&md->msi_lock, flags);
	325	return -ENOMEM;
	326	}
	327	md->msi_alloc_mask \|= (1u << msivec);
	328	spin_unlock_irqrestore(&md->msi_lock, flags);
	329
	330	msg.address_hi = msiaddr >> 32;
	331	msg.address_lo = msiaddr & 0xffffffff;
	332	msg.data = 0xc00 \| msivec;
	333
	334	xirq = xirq + msivec; /* msi mapped to global irq space */
	335	ret = irq_set_msi_desc(xirq, desc);
465665f7	336	if (ret < 0)
c24a8a7a	337	return ret;
c24a8a7a	338
83a18912	339	pci_write_msi_msg(xirq, &msg);
c24a8a7a J	340	return 0;
	341	}
	342
	343	/*
	344	* Switch a link to MSI-X mode
	345	*/
	346	static void xlp_config_link_msix(uint64_t lnkbase, int lirq, uint64_t msixaddr)
	347	{
	348	u32 val;
	349
	350	val = nlm_read_reg(lnkbase, 0x2C);
	351	if ((val & 0x80000000U) == 0) {
	352	val \|= 0x80000000U;
	353	nlm_write_reg(lnkbase, 0x2C, val);
	354	}
d66f3f0e GR	355
	356	if (cpu_is_xlp9xx()) {
	357	val = nlm_read_reg(lnkbase, PCIE_9XX_INT_EN0);
	358	if ((val & 0x200) == 0) {
	359	val \|= 0x200; /* MSI Interrupt enable */
	360	nlm_write_reg(lnkbase, PCIE_9XX_INT_EN0, val);
	361	}
	362	} else {
	363	val = nlm_read_reg(lnkbase, PCIE_INT_EN0);
	364	if ((val & 0x200) == 0) {
	365	val \|= 0x200; /* MSI Interrupt enable */
	366	nlm_write_reg(lnkbase, PCIE_INT_EN0, val);
	367	}
c24a8a7a J	368	}
	369
	370	val = nlm_read_reg(lnkbase, 0x1); /* CMD */
	371	if ((val & 0x0400) == 0) {
	372	val \|= 0x0400;
	373	nlm_write_reg(lnkbase, 0x1, val);
	374	}
	375
	376	/* Update IRQ in the PCI irq reg */
	377	val = nlm_read_pci_reg(lnkbase, 0xf);
	378	val &= ~0x1fu;
	379	val \|= (1 << 8) \| lirq;
	380	nlm_write_pci_reg(lnkbase, 0xf, val);
	381
d66f3f0e GR	382	if (cpu_is_xlp9xx()) {
	383	/* MSI-X addresses */
	384	nlm_write_reg(lnkbase, PCIE_9XX_BRIDGE_MSIX_ADDR_BASE,
	385	msixaddr >> 8);
	386	nlm_write_reg(lnkbase, PCIE_9XX_BRIDGE_MSIX_ADDR_LIMIT,
	387	(msixaddr + MSI_ADDR_SZ) >> 8);
	388	} else {
	389	/* MSI-X addresses */
	390	nlm_write_reg(lnkbase, PCIE_BRIDGE_MSIX_ADDR_BASE,
	391	msixaddr >> 8);
	392	nlm_write_reg(lnkbase, PCIE_BRIDGE_MSIX_ADDR_LIMIT,
	393	(msixaddr + MSI_ADDR_SZ) >> 8);
	394	}
c24a8a7a J	395	}
	396
	397	/*
	398	* Allocate a MSI-X vector
	399	*/
	400	static int xlp_setup_msix(uint64_t lnkbase, int node, int link,
	401	struct msi_desc *desc)
	402	{
	403	struct xlp_msi_data *md;
	404	struct msi_msg msg;
	405	unsigned long flags;
	406	int t, msixvec, lirq, xirq, ret;
	407	uint64_t msixaddr;
	408
	409	/* Get MSI data for the link */
	410	lirq = PIC_PCIE_MSIX_IRQ(link);
	411	xirq = nlm_irq_to_xirq(node, nlm_link_msixirq(link, 0));
325f0a18	412	md = irq_get_chip_data(xirq);
c24a8a7a J	413	msixaddr = MSIX_LINK_ADDR(node, link);
	414
	415	spin_lock_irqsave(&md->msi_lock, flags);
	416	/* switch the PCIe link to MSI-X mode at the first alloc */
	417	if (md->msix_alloc_mask == 0)
	418	xlp_config_link_msix(lnkbase, lirq, msixaddr);
	419
	420	/* allocate a MSI-X vec, and tell the bridge about it */
	421	t = fls(md->msix_alloc_mask);
	422	if (t == XLP_MSIXVEC_PER_LINK) {
	423	spin_unlock_irqrestore(&md->msi_lock, flags);
	424	return -ENOMEM;
	425	}
	426	md->msix_alloc_mask \|= (1u << t);
	427	spin_unlock_irqrestore(&md->msi_lock, flags);
	428
	429	xirq += t;
	430	msixvec = nlm_irq_msixvec(xirq);
d66f3f0e	431
c24a8a7a J	432	msg.address_hi = msixaddr >> 32;
	433	msg.address_lo = msixaddr & 0xffffffff;
	434	msg.data = 0xc00 \| msixvec;
	435
	436	ret = irq_set_msi_desc(xirq, desc);
d5a238df	437	if (ret < 0)
c24a8a7a	438	return ret;
c24a8a7a	439
83a18912	440	pci_write_msi_msg(xirq, &msg);
c24a8a7a J	441	return 0;
	442	}
	443
	444	int arch_setup_msi_irq(struct pci_dev dev, struct msi_desc desc)
	445	{
	446	struct pci_dev *lnkdev;
	447	uint64_t lnkbase;
	448	int node, link, slot;
	449
	450	lnkdev = xlp_get_pcie_link(dev);
	451	if (lnkdev == NULL) {
	452	dev_err(&dev->dev, "Could not find bridge\n");
	453	return 1;
	454	}
	455	slot = PCI_SLOT(lnkdev->devfn);
	456	link = PCI_FUNC(lnkdev->devfn);
	457	node = slot / 8;
	458	lnkbase = nlm_get_pcie_base(node, link);
	459
	460	if (desc->msi_attrib.is_msix)
	461	return xlp_setup_msix(lnkbase, node, link, desc);
	462	else
	463	return xlp_setup_msi(lnkbase, node, link, desc);
	464	}
	465
	466	void __init xlp_init_node_msi_irqs(int node, int link)
	467	{
	468	struct nlm_soc_info *nodep;
	469	struct xlp_msi_data *md;
d66f3f0e	470	int irq, i, irt, msixvec, val;
c24a8a7a J	471
	472	pr_info("[%d %d] Init node PCI IRT\n", node, link);
	473	nodep = nlm_get_node(node);
	474
	475	/* Alloc an MSI block for the link */
	476	md = kzalloc(sizeof(*md), GFP_KERNEL);
	477	spin_lock_init(&md->msi_lock);
	478	md->msi_enabled_mask = 0;
	479	md->msi_alloc_mask = 0;
	480	md->msix_alloc_mask = 0;
	481	md->node = nodep;
	482	md->lnkbase = nlm_get_pcie_base(node, link);
	483
	484	/* extended space for MSI interrupts */
	485	irq = nlm_irq_to_xirq(node, nlm_link_msiirq(link, 0));
	486	for (i = irq; i < irq + XLP_MSIVEC_PER_LINK; i++) {
	487	irq_set_chip_and_handler(i, &xlp_msi_chip, handle_level_irq);
325f0a18	488	irq_set_chip_data(i, md);
c24a8a7a J	489	}
c24a8a7a J	490
d66f3f0e GR	491	for (i = 0; i < XLP_MSIXVEC_PER_LINK ; i++) {
	492	if (cpu_is_xlp9xx()) {
	493	val = ((node * nlm_threads_per_node()) << 7 \|
	494	PIC_PCIE_MSIX_IRQ(link) << 1 \| 0 << 0);
	495	nlm_write_pcie_reg(md->lnkbase, PCIE_9XX_MSIX_VECX(i +
	496	(link * XLP_MSIXVEC_PER_LINK)), val);
	497	} else {
	498	/* Initialize MSI-X irts to generate one interrupt
	499	* per link
	500	*/
	501	msixvec = link * XLP_MSIXVEC_PER_LINK + i;
	502	irt = PIC_IRT_PCIE_MSIX_INDEX(msixvec);
	503	nlm_pic_init_irt(nodep->picbase, irt,
	504	PIC_PCIE_MSIX_IRQ(link),
	505	node * nlm_threads_per_node(), 1);
	506	}
c24a8a7a J	507
	508	/* Initialize MSI-X extended irq space for the link */
	509	irq = nlm_irq_to_xirq(node, nlm_link_msixirq(link, i));
	510	irq_set_chip_and_handler(irq, &xlp_msix_chip, handle_level_irq);
325f0a18	511	irq_set_chip_data(irq, md);
c24a8a7a	512	}
c24a8a7a J	513	}
	514
	515	void nlm_dispatch_msi(int node, int lirq)
	516	{
	517	struct xlp_msi_data *md;
	518	int link, i, irqbase;
	519	u32 status;
	520
	521	link = lirq - PIC_PCIE_LINK_MSI_IRQ_BASE;
	522	irqbase = nlm_irq_to_xirq(node, nlm_link_msiirq(link, 0));
325f0a18	523	md = irq_get_chip_data(irqbase);
d66f3f0e GR	524	if (cpu_is_xlp9xx())
	525	status = nlm_read_reg(md->lnkbase, PCIE_9XX_MSI_STATUS) &
	526	md->msi_enabled_mask;
	527	else
	528	status = nlm_read_reg(md->lnkbase, PCIE_MSI_STATUS) &
c24a8a7a J	529	md->msi_enabled_mask;
	530	while (status) {
	531	i = __ffs(status);
	532	do_IRQ(irqbase + i);
	533	status &= status - 1;
	534	}
53f67697 J	535
	536	/* Ack at eirr and PIC */
	537	ack_c0_eirr(PIC_PCIE_LINK_MSI_IRQ(link));
	538	if (cpu_is_xlp9xx())
	539	nlm_pic_ack(md->node->picbase,
	540	PIC_9XX_IRT_PCIE_LINK_INDEX(link));
	541	else
	542	nlm_pic_ack(md->node->picbase, PIC_IRT_PCIE_LINK_INDEX(link));
c24a8a7a J	543	}
	544
	545	void nlm_dispatch_msix(int node, int lirq)
	546	{
	547	struct xlp_msi_data *md;
	548	int link, i, irqbase;
	549	u32 status;
	550
	551	link = lirq - PIC_PCIE_MSIX_IRQ_BASE;
	552	irqbase = nlm_irq_to_xirq(node, nlm_link_msixirq(link, 0));
325f0a18	553	md = irq_get_chip_data(irqbase);
d66f3f0e GR	554	if (cpu_is_xlp9xx())
	555	status = nlm_read_reg(md->lnkbase, PCIE_9XX_MSIX_STATUSX(link));
	556	else
	557	status = nlm_read_reg(md->lnkbase, PCIE_MSIX_STATUS);
c24a8a7a J	558
c24a8a7a J	559	/* narrow it down to the MSI-x vectors for our link */
d66f3f0e GR	560	if (!cpu_is_xlp9xx())
d66f3f0e GR	561	status = (status >> (link * XLP_MSIXVEC_PER_LINK)) &
c24a8a7a J	562	((1 << XLP_MSIXVEC_PER_LINK) - 1);
	563
	564	while (status) {
	565	i = __ffs(status);
	566	do_IRQ(irqbase + i);
	567	status &= status - 1;
	568	}
53f67697 J	569	/* Ack at eirr and PIC */
53f67697 J	570	ack_c0_eirr(PIC_PCIE_MSIX_IRQ(link));
c24a8a7a	571	}