[deliverable/linux.git] / drivers / dma / ioat / dca.c

/*
 * Intel I/OAT DMA Linux driver
 * Copyright(c) 2007 - 2009 Intel Corporation.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms and conditions of the GNU General Public License,
 * version 2, as published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
 * more details.
 *
 * The full GNU General Public License is included in this distribution in
 * the file called "COPYING".
 *
 */

#include <linux/kernel.h>
#include <linux/pci.h>
#include <linux/smp.h>
#include <linux/interrupt.h>
#include <linux/dca.h>

/* either a kernel change is needed, or we need something like this in kernel */
#ifndef CONFIG_SMP
#include <asm/smp.h>
#undef cpu_physical_id
#define cpu_physical_id(cpu) (cpuid_ebx(1) >> 24)
#endif

#include "dma.h"
#include "registers.h"
#include "dma_v2.h"

/*
 * Bit 7 of a tag map entry is the "valid" bit, if it is set then bits 0:6
 * contain the bit number of the APIC ID to map into the DCA tag.  If the valid
 * bit is not set, then the value must be 0 or 1 and defines the bit in the tag.
 */
#define DCA_TAG_MAP_VALID 0x80

#define DCA3_TAG_MAP_BIT_TO_INV 0x80
#define DCA3_TAG_MAP_BIT_TO_SEL 0x40
#define DCA3_TAG_MAP_LITERAL_VAL 0x1

#define DCA_TAG_MAP_MASK 0xDF

/* expected tag map bytes for I/OAT ver.2 */
#define DCA2_TAG_MAP_BYTE0 0x80
#define DCA2_TAG_MAP_BYTE1 0x0
#define DCA2_TAG_MAP_BYTE2 0x81
#define DCA2_TAG_MAP_BYTE3 0x82
#define DCA2_TAG_MAP_BYTE4 0x82

/* verify if tag map matches expected values */
static inline int dca2_tag_map_valid(u8 *tag_map)
{
	return ((tag_map[0] == DCA2_TAG_MAP_BYTE0) &&
		(tag_map[1] == DCA2_TAG_MAP_BYTE1) &&
		(tag_map[2] == DCA2_TAG_MAP_BYTE2) &&
		(tag_map[3] == DCA2_TAG_MAP_BYTE3) &&
		(tag_map[4] == DCA2_TAG_MAP_BYTE4));
}

/*
 * "Legacy" DCA systems do not implement the DCA register set in the
 * I/OAT device.  Software needs direct support for their tag mappings.
 */

#define APICID_BIT(x)		(DCA_TAG_MAP_VALID | (x))
#define IOAT_TAG_MAP_LEN	8

/* pack PCI B/D/F into a u16 */
static inline u16 dcaid_from_pcidev(struct pci_dev *pci)
{
	return (pci->bus->number << 8) | pci->devfn;
}

static int dca_enabled_in_bios(struct pci_dev *pdev)
{
	/* CPUID level 9 returns DCA configuration */
	/* Bit 0 indicates DCA enabled by the BIOS */
	unsigned long cpuid_level_9;
	int res;

	cpuid_level_9 = cpuid_eax(9);
	res = test_bit(0, &cpuid_level_9);
	if (!res)
		dev_dbg(&pdev->dev, "DCA is disabled in BIOS\n");

	return res;
}

int system_has_dca_enabled(struct pci_dev *pdev)
{
	if (boot_cpu_has(X86_FEATURE_DCA))
		return dca_enabled_in_bios(pdev);

	dev_dbg(&pdev->dev, "boot cpu doesn't have X86_FEATURE_DCA\n");
	return 0;
}

struct ioat_dca_slot {
	struct pci_dev *pdev;	/* requester device */
	u16 rid;		/* requester id, as used by IOAT */
};

#define IOAT_DCA_MAX_REQ 6
#define IOAT3_DCA_MAX_REQ 2

struct ioat_dca_priv {
	void __iomem		*iobase;
	void __iomem		*dca_base;
	int			 max_requesters;
	int			 requester_count;
	u8			 tag_map[IOAT_TAG_MAP_LEN];
	struct ioat_dca_slot 	 req_slots[0];
};

static int ioat_dca_dev_managed(struct dca_provider *dca,
				struct device *dev)
{
	struct ioat_dca_priv *ioatdca = dca_priv(dca);
	struct pci_dev *pdev;
	int i;

	pdev = to_pci_dev(dev);
	for (i = 0; i < ioatdca->max_requesters; i++) {
		if (ioatdca->req_slots[i].pdev == pdev)
			return 1;
	}
	return 0;
}

static int ioat3_dca_add_requester(struct dca_provider *dca, struct device *dev)
{
	struct ioat_dca_priv *ioatdca = dca_priv(dca);
	struct pci_dev *pdev;
	int i;
	u16 id;
	u16 global_req_table;

	/* This implementation only supports PCI-Express */
	if (!dev_is_pci(dev))
		return -ENODEV;
	pdev = to_pci_dev(dev);
	id = dcaid_from_pcidev(pdev);

	if (ioatdca->requester_count == ioatdca->max_requesters)
		return -ENODEV;

	for (i = 0; i < ioatdca->max_requesters; i++) {
		if (ioatdca->req_slots[i].pdev == NULL) {
			/* found an empty slot */
			ioatdca->requester_count++;
			ioatdca->req_slots[i].pdev = pdev;
			ioatdca->req_slots[i].rid = id;
			global_req_table =
			      readw(ioatdca->dca_base + IOAT3_DCA_GREQID_OFFSET);
			writel(id | IOAT_DCA_GREQID_VALID,
			       ioatdca->iobase + global_req_table + (i * 4));
			return i;
		}
	}
	/* Error, ioatdma->requester_count is out of whack */
	return -EFAULT;
}

static int ioat3_dca_remove_requester(struct dca_provider *dca,
				      struct device *dev)
{
	struct ioat_dca_priv *ioatdca = dca_priv(dca);
	struct pci_dev *pdev;
	int i;
	u16 global_req_table;

	/* This implementation only supports PCI-Express */
	if (!dev_is_pci(dev))
		return -ENODEV;
	pdev = to_pci_dev(dev);

	for (i = 0; i < ioatdca->max_requesters; i++) {
		if (ioatdca->req_slots[i].pdev == pdev) {
			global_req_table =
			      readw(ioatdca->dca_base + IOAT3_DCA_GREQID_OFFSET);
			writel(0, ioatdca->iobase + global_req_table + (i * 4));
			ioatdca->req_slots[i].pdev = NULL;
			ioatdca->req_slots[i].rid = 0;
			ioatdca->requester_count--;
			return i;
		}
	}
	return -ENODEV;
}

static u8 ioat3_dca_get_tag(struct dca_provider *dca,
			    struct device *dev,
			    int cpu)
{
	u8 tag;

	struct ioat_dca_priv *ioatdca = dca_priv(dca);
	int i, apic_id, bit, value;
	u8 entry;

	tag = 0;
	apic_id = cpu_physical_id(cpu);

	for (i = 0; i < IOAT_TAG_MAP_LEN; i++) {
		entry = ioatdca->tag_map[i];
		if (entry & DCA3_TAG_MAP_BIT_TO_SEL) {
			bit = entry &
				~(DCA3_TAG_MAP_BIT_TO_SEL | DCA3_TAG_MAP_BIT_TO_INV);
			value = (apic_id & (1 << bit)) ? 1 : 0;
		} else if (entry & DCA3_TAG_MAP_BIT_TO_INV) {
			bit = entry & ~DCA3_TAG_MAP_BIT_TO_INV;
			value = (apic_id & (1 << bit)) ? 0 : 1;
		} else {
			value = (entry & DCA3_TAG_MAP_LITERAL_VAL) ? 1 : 0;
		}
		tag |= (value << i);
	}

	return tag;
}

static struct dca_ops ioat3_dca_ops = {
	.add_requester		= ioat3_dca_add_requester,
	.remove_requester	= ioat3_dca_remove_requester,
	.get_tag		= ioat3_dca_get_tag,
	.dev_managed		= ioat_dca_dev_managed,
};

static int ioat3_dca_count_dca_slots(void *iobase, u16 dca_offset)
{
	int slots = 0;
	u32 req;
	u16 global_req_table;

	global_req_table = readw(iobase + dca_offset + IOAT3_DCA_GREQID_OFFSET);
	if (global_req_table == 0)
		return 0;

	do {
		req = readl(iobase + global_req_table + (slots * sizeof(u32)));
		slots++;
	} while ((req & IOAT_DCA_GREQID_LASTID) == 0);

	return slots;
}

static inline int dca3_tag_map_invalid(u8 *tag_map)
{
	/*
	 * If the tag map is not programmed by the BIOS the default is:
	 * 0x80 0x80 0x80 0x80 0x80 0x00 0x00 0x00
	 *
	 * This an invalid map and will result in only 2 possible tags
	 * 0x1F and 0x00.  0x00 is an invalid DCA tag so we know that
	 * this entire definition is invalid.
	 */
	return ((tag_map[0] == DCA_TAG_MAP_VALID) &&
		(tag_map[1] == DCA_TAG_MAP_VALID) &&
		(tag_map[2] == DCA_TAG_MAP_VALID) &&
		(tag_map[3] == DCA_TAG_MAP_VALID) &&
		(tag_map[4] == DCA_TAG_MAP_VALID));
}

struct dca_provider *ioat3_dca_init(struct pci_dev *pdev, void __iomem *iobase)
{
	struct dca_provider *dca;
	struct ioat_dca_priv *ioatdca;
	int slots;
	int i;
	int err;
	u16 dca_offset;
	u16 csi_fsb_control;
	u16 pcie_control;
	u8 bit;

	union {
		u64 full;
		struct {
			u32 low;
			u32 high;
		};
	} tag_map;

	if (!system_has_dca_enabled(pdev))
		return NULL;

	dca_offset = readw(iobase + IOAT_DCAOFFSET_OFFSET);
	if (dca_offset == 0)
		return NULL;

	slots = ioat3_dca_count_dca_slots(iobase, dca_offset);
	if (slots == 0)
		return NULL;

	dca = alloc_dca_provider(&ioat3_dca_ops,
				 sizeof(*ioatdca)
				      + (sizeof(struct ioat_dca_slot) * slots));
	if (!dca)
		return NULL;

	ioatdca = dca_priv(dca);
	ioatdca->iobase = iobase;
	ioatdca->dca_base = iobase + dca_offset;
	ioatdca->max_requesters = slots;

	/* some bios might not know to turn these on */
	csi_fsb_control = readw(ioatdca->dca_base + IOAT3_CSI_CONTROL_OFFSET);
	if ((csi_fsb_control & IOAT3_CSI_CONTROL_PREFETCH) == 0) {
		csi_fsb_control |= IOAT3_CSI_CONTROL_PREFETCH;
		writew(csi_fsb_control,
		       ioatdca->dca_base + IOAT3_CSI_CONTROL_OFFSET);
	}
	pcie_control = readw(ioatdca->dca_base + IOAT3_PCI_CONTROL_OFFSET);
	if ((pcie_control & IOAT3_PCI_CONTROL_MEMWR) == 0) {
		pcie_control |= IOAT3_PCI_CONTROL_MEMWR;
		writew(pcie_control,
		       ioatdca->dca_base + IOAT3_PCI_CONTROL_OFFSET);
	}


	/* TODO version, compatibility and configuration checks */

	/* copy out the APIC to DCA tag map */
	tag_map.low =
		readl(ioatdca->dca_base + IOAT3_APICID_TAG_MAP_OFFSET_LOW);
	tag_map.high =
		readl(ioatdca->dca_base + IOAT3_APICID_TAG_MAP_OFFSET_HIGH);
	for (i = 0; i < 8; i++) {
		bit = tag_map.full >> (8 * i);
		ioatdca->tag_map[i] = bit & DCA_TAG_MAP_MASK;
	}

	if (dca3_tag_map_invalid(ioatdca->tag_map)) {
		WARN_TAINT_ONCE(1, TAINT_FIRMWARE_WORKAROUND,
				"%s %s: APICID_TAG_MAP set incorrectly by BIOS, disabling DCA\n",
				dev_driver_string(&pdev->dev),
				dev_name(&pdev->dev));
		free_dca_provider(dca);
		return NULL;
	}

	err = register_dca_provider(dca, &pdev->dev);
	if (err) {
		free_dca_provider(dca);
		return NULL;
	}

	return dca;
}
Commit	Line	Data
2ed6dc34 SN	1	/*
2ed6dc34 SN	2	* Intel I/OAT DMA Linux driver
211a22ce	3	* Copyright(c) 2007 - 2009 Intel Corporation.
2ed6dc34 SN	4	*
	5	* This program is free software; you can redistribute it and/or modify it
	6	* under the terms and conditions of the GNU General Public License,
	7	* version 2, as published by the Free Software Foundation.
	8	*
	9	* This program is distributed in the hope that it will be useful, but WITHOUT
	10	* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
	11	* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
	12	* more details.
	13	*
2ed6dc34 SN	14	* The full GNU General Public License is included in this distribution in
	15	* the file called "COPYING".
	16	*
	17	*/
	18
	19	#include <linux/kernel.h>
	20	#include <linux/pci.h>
	21	#include <linux/smp.h>
	22	#include <linux/interrupt.h>
	23	#include <linux/dca.h>
	24
	25	/* either a kernel change is needed, or we need something like this in kernel */
	26	#ifndef CONFIG_SMP
	27	#include <asm/smp.h>
	28	#undef cpu_physical_id
	29	#define cpu_physical_id(cpu) (cpuid_ebx(1) >> 24)
	30	#endif
	31
584ec227 DW	32	#include "dma.h"
584ec227 DW	33	#include "registers.h"
2358b820	34	#include "dma_v2.h"
2ed6dc34 SN	35
2ed6dc34 SN	36	/*
7f1b358a	37	* Bit 7 of a tag map entry is the "valid" bit, if it is set then bits 0:6
2ed6dc34 SN	38	* contain the bit number of the APIC ID to map into the DCA tag. If the valid
	39	* bit is not set, then the value must be 0 or 1 and defines the bit in the tag.
	40	*/
	41	#define DCA_TAG_MAP_VALID 0x80
	42
7f1b358a MS	43	#define DCA3_TAG_MAP_BIT_TO_INV 0x80
	44	#define DCA3_TAG_MAP_BIT_TO_SEL 0x40
	45	#define DCA3_TAG_MAP_LITERAL_VAL 0x1
	46
	47	#define DCA_TAG_MAP_MASK 0xDF
	48
49bc4636 MS	49	/* expected tag map bytes for I/OAT ver.2 */
	50	#define DCA2_TAG_MAP_BYTE0 0x80
	51	#define DCA2_TAG_MAP_BYTE1 0x0
	52	#define DCA2_TAG_MAP_BYTE2 0x81
	53	#define DCA2_TAG_MAP_BYTE3 0x82
	54	#define DCA2_TAG_MAP_BYTE4 0x82
	55
	56	/* verify if tag map matches expected values */
	57	static inline int dca2_tag_map_valid(u8 *tag_map)
	58	{
	59	return ((tag_map[0] == DCA2_TAG_MAP_BYTE0) &&
	60	(tag_map[1] == DCA2_TAG_MAP_BYTE1) &&
	61	(tag_map[2] == DCA2_TAG_MAP_BYTE2) &&
	62	(tag_map[3] == DCA2_TAG_MAP_BYTE3) &&
	63	(tag_map[4] == DCA2_TAG_MAP_BYTE4));
	64	}
	65
2ed6dc34 SN	66	/*
	67	* "Legacy" DCA systems do not implement the DCA register set in the
	68	* I/OAT device. Software needs direct support for their tag mappings.
	69	*/
	70
	71	#define APICID_BIT(x) (DCA_TAG_MAP_VALID \| (x))
	72	#define IOAT_TAG_MAP_LEN 8
	73
2ed6dc34 SN	74	/* pack PCI B/D/F into a u16 */
	75	static inline u16 dcaid_from_pcidev(struct pci_dev *pci)
	76	{
	77	return (pci->bus->number << 8) \| pci->devfn;
	78	}
	79
5149fd01	80	static int dca_enabled_in_bios(struct pci_dev *pdev)
2ed6dc34 SN	81	{
	82	/* CPUID level 9 returns DCA configuration */
	83	/* Bit 0 indicates DCA enabled by the BIOS */
	84	unsigned long cpuid_level_9;
	85	int res;
	86
	87	cpuid_level_9 = cpuid_eax(9);
	88	res = test_bit(0, &cpuid_level_9);
	89	if (!res)
e22dde99	90	dev_dbg(&pdev->dev, "DCA is disabled in BIOS\n");
2ed6dc34 SN	91
	92	return res;
	93	}
	94
228c4f5c	95	int system_has_dca_enabled(struct pci_dev *pdev)
2ed6dc34 SN	96	{
2ed6dc34 SN	97	if (boot_cpu_has(X86_FEATURE_DCA))
5149fd01	98	return dca_enabled_in_bios(pdev);
2ed6dc34	99
e22dde99	100	dev_dbg(&pdev->dev, "boot cpu doesn't have X86_FEATURE_DCA\n");
2ed6dc34 SN	101	return 0;
	102	}
	103
	104	struct ioat_dca_slot {
	105	struct pci_dev pdev; / requester device */
	106	u16 rid; /* requester id, as used by IOAT */
	107	};
	108
	109	#define IOAT_DCA_MAX_REQ 6
7f1b358a	110	#define IOAT3_DCA_MAX_REQ 2
2ed6dc34 SN	111
	112	struct ioat_dca_priv {
	113	void __iomem *iobase;
53a0c98e	114	void __iomem *dca_base;
2ed6dc34 SN	115	int max_requesters;
	116	int requester_count;
	117	u8 tag_map[IOAT_TAG_MAP_LEN];
	118	struct ioat_dca_slot req_slots[0];
	119	};
	120
7f1b358a MS	121	static int ioat_dca_dev_managed(struct dca_provider *dca,
	122	struct device *dev)
	123	{
	124	struct ioat_dca_priv *ioatdca = dca_priv(dca);
	125	struct pci_dev *pdev;
	126	int i;
	127
	128	pdev = to_pci_dev(dev);
	129	for (i = 0; i < ioatdca->max_requesters; i++) {
	130	if (ioatdca->req_slots[i].pdev == pdev)
	131	return 1;
	132	}
	133	return 0;
	134	}
	135
7f1b358a MS	136	static int ioat3_dca_add_requester(struct dca_provider dca, struct device dev)
	137	{
	138	struct ioat_dca_priv *ioatdca = dca_priv(dca);
	139	struct pci_dev *pdev;
	140	int i;
	141	u16 id;
	142	u16 global_req_table;
	143
	144	/* This implementation only supports PCI-Express */
1fde2548	145	if (!dev_is_pci(dev))
7f1b358a MS	146	return -ENODEV;
	147	pdev = to_pci_dev(dev);
	148	id = dcaid_from_pcidev(pdev);
	149
	150	if (ioatdca->requester_count == ioatdca->max_requesters)
	151	return -ENODEV;
	152
	153	for (i = 0; i < ioatdca->max_requesters; i++) {
	154	if (ioatdca->req_slots[i].pdev == NULL) {
	155	/* found an empty slot */
	156	ioatdca->requester_count++;
	157	ioatdca->req_slots[i].pdev = pdev;
	158	ioatdca->req_slots[i].rid = id;
	159	global_req_table =
	160	readw(ioatdca->dca_base + IOAT3_DCA_GREQID_OFFSET);
	161	writel(id \| IOAT_DCA_GREQID_VALID,
	162	ioatdca->iobase + global_req_table + (i * 4));
	163	return i;
	164	}
	165	}
	166	/* Error, ioatdma->requester_count is out of whack */
	167	return -EFAULT;
	168	}
	169
	170	static int ioat3_dca_remove_requester(struct dca_provider *dca,
	171	struct device *dev)
	172	{
	173	struct ioat_dca_priv *ioatdca = dca_priv(dca);
	174	struct pci_dev *pdev;
	175	int i;
	176	u16 global_req_table;
	177
	178	/* This implementation only supports PCI-Express */
1fde2548	179	if (!dev_is_pci(dev))
7f1b358a MS	180	return -ENODEV;
	181	pdev = to_pci_dev(dev);
	182
	183	for (i = 0; i < ioatdca->max_requesters; i++) {
	184	if (ioatdca->req_slots[i].pdev == pdev) {
	185	global_req_table =
	186	readw(ioatdca->dca_base + IOAT3_DCA_GREQID_OFFSET);
	187	writel(0, ioatdca->iobase + global_req_table + (i * 4));
	188	ioatdca->req_slots[i].pdev = NULL;
	189	ioatdca->req_slots[i].rid = 0;
	190	ioatdca->requester_count--;
	191	return i;
	192	}
	193	}
	194	return -ENODEV;
	195	}
	196
	197	static u8 ioat3_dca_get_tag(struct dca_provider *dca,
	198	struct device *dev,
	199	int cpu)
	200	{
	201	u8 tag;
	202
	203	struct ioat_dca_priv *ioatdca = dca_priv(dca);
	204	int i, apic_id, bit, value;
	205	u8 entry;
	206
	207	tag = 0;
	208	apic_id = cpu_physical_id(cpu);
	209
	210	for (i = 0; i < IOAT_TAG_MAP_LEN; i++) {
	211	entry = ioatdca->tag_map[i];
	212	if (entry & DCA3_TAG_MAP_BIT_TO_SEL) {
	213	bit = entry &
	214	~(DCA3_TAG_MAP_BIT_TO_SEL \| DCA3_TAG_MAP_BIT_TO_INV);
	215	value = (apic_id & (1 << bit)) ? 1 : 0;
	216	} else if (entry & DCA3_TAG_MAP_BIT_TO_INV) {
	217	bit = entry & ~DCA3_TAG_MAP_BIT_TO_INV;
	218	value = (apic_id & (1 << bit)) ? 0 : 1;
	219	} else {
	220	value = (entry & DCA3_TAG_MAP_LITERAL_VAL) ? 1 : 0;
	221	}
	222	tag \|= (value << i);
	223	}
	224
	225	return tag;
	226	}
	227
	228	static struct dca_ops ioat3_dca_ops = {
	229	.add_requester = ioat3_dca_add_requester,
	230	.remove_requester = ioat3_dca_remove_requester,
	231	.get_tag = ioat3_dca_get_tag,
	232	.dev_managed = ioat_dca_dev_managed,
	233	};
	234
	235	static int ioat3_dca_count_dca_slots(void *iobase, u16 dca_offset)
	236	{
	237	int slots = 0;
	238	u32 req;
	239	u16 global_req_table;
	240
	241	global_req_table = readw(iobase + dca_offset + IOAT3_DCA_GREQID_OFFSET);
	242	if (global_req_table == 0)
	243	return 0;
244
245	do {
246	req = readl(iobase + global_req_table + (slots * sizeof(u32)));
247	slots++;
248	} while ((req & IOAT_DCA_GREQID_LASTID) == 0);
249
250	return slots;
251	}
252
07bd34db AD	253	static inline int dca3_tag_map_invalid(u8 *tag_map)
	254	{
	255	/*
	256	* If the tag map is not programmed by the BIOS the default is:
	257	* 0x80 0x80 0x80 0x80 0x80 0x00 0x00 0x00
	258	*
	259	* This an invalid map and will result in only 2 possible tags
	260	* 0x1F and 0x00. 0x00 is an invalid DCA tag so we know that
	261	* this entire definition is invalid.
	262	*/
	263	return ((tag_map[0] == DCA_TAG_MAP_VALID) &&
	264	(tag_map[1] == DCA_TAG_MAP_VALID) &&
	265	(tag_map[2] == DCA_TAG_MAP_VALID) &&
	266	(tag_map[3] == DCA_TAG_MAP_VALID) &&
	267	(tag_map[4] == DCA_TAG_MAP_VALID));
	268	}
	269
4bf27b8b	270	struct dca_provider ioat3_dca_init(struct pci_dev pdev, void __iomem *iobase)
7f1b358a MS	271	{
	272	struct dca_provider *dca;
	273	struct ioat_dca_priv *ioatdca;
	274	int slots;
	275	int i;
	276	int err;
	277	u16 dca_offset;
	278	u16 csi_fsb_control;
	279	u16 pcie_control;
	280	u8 bit;
	281
	282	union {
	283	u64 full;
	284	struct {
	285	u32 low;
	286	u32 high;
	287	};
	288	} tag_map;
	289
	290	if (!system_has_dca_enabled(pdev))
	291	return NULL;
	292
	293	dca_offset = readw(iobase + IOAT_DCAOFFSET_OFFSET);
	294	if (dca_offset == 0)
	295	return NULL;
	296
	297	slots = ioat3_dca_count_dca_slots(iobase, dca_offset);
	298	if (slots == 0)
	299	return NULL;
	300
	301	dca = alloc_dca_provider(&ioat3_dca_ops,
	302	sizeof(*ioatdca)
	303	+ (sizeof(struct ioat_dca_slot) * slots));
	304	if (!dca)
	305	return NULL;
	306
	307	ioatdca = dca_priv(dca);
	308	ioatdca->iobase = iobase;
	309	ioatdca->dca_base = iobase + dca_offset;
	310	ioatdca->max_requesters = slots;
	311
	312	/* some bios might not know to turn these on */
	313	csi_fsb_control = readw(ioatdca->dca_base + IOAT3_CSI_CONTROL_OFFSET);
	314	if ((csi_fsb_control & IOAT3_CSI_CONTROL_PREFETCH) == 0) {
	315	csi_fsb_control \|= IOAT3_CSI_CONTROL_PREFETCH;
	316	writew(csi_fsb_control,
	317	ioatdca->dca_base + IOAT3_CSI_CONTROL_OFFSET);
	318	}
	319	pcie_control = readw(ioatdca->dca_base + IOAT3_PCI_CONTROL_OFFSET);
	320	if ((pcie_control & IOAT3_PCI_CONTROL_MEMWR) == 0) {
	321	pcie_control \|= IOAT3_PCI_CONTROL_MEMWR;
	322	writew(pcie_control,
	323	ioatdca->dca_base + IOAT3_PCI_CONTROL_OFFSET);
	324	}
	325
	326
	327	/* TODO version, compatibility and configuration checks */
	328
	329	/* copy out the APIC to DCA tag map */
	330	tag_map.low =
	331	readl(ioatdca->dca_base + IOAT3_APICID_TAG_MAP_OFFSET_LOW);
	332	tag_map.high =
	333	readl(ioatdca->dca_base + IOAT3_APICID_TAG_MAP_OFFSET_HIGH);
	334	for (i = 0; i < 8; i++) {
335	bit = tag_map.full >> (8 * i);
336	ioatdca->tag_map[i] = bit & DCA_TAG_MAP_MASK;
337	}
338
07bd34db	339	if (dca3_tag_map_invalid(ioatdca->tag_map)) {
f3c78f85 AD	340	WARN_TAINT_ONCE(1, TAINT_FIRMWARE_WORKAROUND,
	341	"%s %s: APICID_TAG_MAP set incorrectly by BIOS, disabling DCA\n",
	342	dev_driver_string(&pdev->dev),
	343	dev_name(&pdev->dev));
07bd34db AD	344	free_dca_provider(dca);
	345	return NULL;
	346	}
	347
7f1b358a MS	348	err = register_dca_provider(dca, &pdev->dev);
	349	if (err) {
	350	free_dca_provider(dca);
	351	return NULL;
	352	}
	353
	354	return dca;
	355	}