[deliverable/linux.git] / arch / x86 / kernel / pci-dma.c

#include <linux/dma-mapping.h>
#include <linux/dmar.h>
#include <linux/bootmem.h>
#include <linux/pci.h>

#include <asm/proto.h>
#include <asm/dma.h>
#include <asm/gart.h>
#include <asm/calgary.h>

int forbid_dac __read_mostly;
EXPORT_SYMBOL(forbid_dac);

const struct dma_mapping_ops *dma_ops;
EXPORT_SYMBOL(dma_ops);

static int iommu_sac_force __read_mostly;

#ifdef CONFIG_IOMMU_DEBUG
int panic_on_overflow __read_mostly = 1;
int force_iommu __read_mostly = 1;
#else
int panic_on_overflow __read_mostly = 0;
int force_iommu __read_mostly = 0;
#endif

int iommu_merge __read_mostly = 0;

int no_iommu __read_mostly;
/* Set this to 1 if there is a HW IOMMU in the system */
int iommu_detected __read_mostly = 0;

/* This tells the BIO block layer to assume merging. Default to off
   because we cannot guarantee merging later. */
int iommu_bio_merge __read_mostly = 0;
EXPORT_SYMBOL(iommu_bio_merge);

dma_addr_t bad_dma_address __read_mostly = 0;
EXPORT_SYMBOL(bad_dma_address);

/* Dummy device used for NULL arguments (normally ISA). Better would
   be probably a smaller DMA mask, but this is bug-to-bug compatible
   to older i386. */
struct device fallback_dev = {
	.bus_id = "fallback device",
	.coherent_dma_mask = DMA_32BIT_MASK,
	.dma_mask = &fallback_dev.coherent_dma_mask,
};

int dma_set_mask(struct device *dev, u64 mask)
{
	if (!dev->dma_mask || !dma_supported(dev, mask))
		return -EIO;

	*dev->dma_mask = mask;

	return 0;
}
EXPORT_SYMBOL(dma_set_mask);

#ifdef CONFIG_X86_64
static __initdata void *dma32_bootmem_ptr;
static unsigned long dma32_bootmem_size __initdata = (128ULL<<20);

static int __init parse_dma32_size_opt(char *p)
{
	if (!p)
		return -EINVAL;
	dma32_bootmem_size = memparse(p, &p);
	return 0;
}
early_param("dma32_size", parse_dma32_size_opt);

void __init dma32_reserve_bootmem(void)
{
	unsigned long size, align;
	if (end_pfn <= MAX_DMA32_PFN)
		return;

	align = 64ULL<<20;
	size = round_up(dma32_bootmem_size, align);
	dma32_bootmem_ptr = __alloc_bootmem_nopanic(size, align,
				 __pa(MAX_DMA_ADDRESS));
	if (dma32_bootmem_ptr)
		dma32_bootmem_size = size;
	else
		dma32_bootmem_size = 0;
}
static void __init dma32_free_bootmem(void)
{
	int node;

	if (end_pfn <= MAX_DMA32_PFN)
		return;

	if (!dma32_bootmem_ptr)
		return;

	for_each_online_node(node)
		free_bootmem_node(NODE_DATA(node), __pa(dma32_bootmem_ptr),
				  dma32_bootmem_size);

	dma32_bootmem_ptr = NULL;
	dma32_bootmem_size = 0;
}

void __init pci_iommu_alloc(void)
{
	/* free the range so iommu could get some range less than 4G */
	dma32_free_bootmem();
	/*
	 * The order of these functions is important for
	 * fall-back/fail-over reasons
	 */
#ifdef CONFIG_GART_IOMMU
	gart_iommu_hole_init();
#endif

#ifdef CONFIG_CALGARY_IOMMU
	detect_calgary();
#endif

	detect_intel_iommu();

#ifdef CONFIG_SWIOTLB
	pci_swiotlb_init();
#endif
}
#endif

/*
 * See <Documentation/x86_64/boot-options.txt> for the iommu kernel parameter
 * documentation.
 */
static __init int iommu_setup(char *p)
{
	iommu_merge = 1;

	if (!p)
		return -EINVAL;

	while (*p) {
		if (!strncmp(p, "off", 3))
			no_iommu = 1;
		/* gart_parse_options has more force support */
		if (!strncmp(p, "force", 5))
			force_iommu = 1;
		if (!strncmp(p, "noforce", 7)) {
			iommu_merge = 0;
			force_iommu = 0;
		}

		if (!strncmp(p, "biomerge", 8)) {
			iommu_bio_merge = 4096;
			iommu_merge = 1;
			force_iommu = 1;
		}
		if (!strncmp(p, "panic", 5))
			panic_on_overflow = 1;
		if (!strncmp(p, "nopanic", 7))
			panic_on_overflow = 0;
		if (!strncmp(p, "merge", 5)) {
			iommu_merge = 1;
			force_iommu = 1;
		}
		if (!strncmp(p, "nomerge", 7))
			iommu_merge = 0;
		if (!strncmp(p, "forcesac", 8))
			iommu_sac_force = 1;
		if (!strncmp(p, "allowdac", 8))
			forbid_dac = 0;
		if (!strncmp(p, "nodac", 5))
			forbid_dac = -1;
		if (!strncmp(p, "usedac", 6)) {
			forbid_dac = -1;
			return 1;
		}
#ifdef CONFIG_SWIOTLB
		if (!strncmp(p, "soft", 4))
			swiotlb = 1;
#endif

#ifdef CONFIG_GART_IOMMU
		gart_parse_options(p);
#endif

#ifdef CONFIG_CALGARY_IOMMU
		if (!strncmp(p, "calgary", 7))
			use_calgary = 1;
#endif /* CONFIG_CALGARY_IOMMU */

		p += strcspn(p, ",");
		if (*p == ',')
			++p;
	}
	return 0;
}
early_param("iommu", iommu_setup);

#ifdef CONFIG_X86_32
int dma_declare_coherent_memory(struct device *dev, dma_addr_t bus_addr,
				dma_addr_t device_addr, size_t size, int flags)
{
	void __iomem *mem_base = NULL;
	int pages = size >> PAGE_SHIFT;
	int bitmap_size = BITS_TO_LONGS(pages) * sizeof(long);

	if ((flags & (DMA_MEMORY_MAP | DMA_MEMORY_IO)) == 0)
		goto out;
	if (!size)
		goto out;
	if (dev->dma_mem)
		goto out;

	/* FIXME: this routine just ignores DMA_MEMORY_INCLUDES_CHILDREN */

	mem_base = ioremap(bus_addr, size);
	if (!mem_base)
		goto out;

	dev->dma_mem = kzalloc(sizeof(struct dma_coherent_mem), GFP_KERNEL);
	if (!dev->dma_mem)
		goto out;
	dev->dma_mem->bitmap = kzalloc(bitmap_size, GFP_KERNEL);
	if (!dev->dma_mem->bitmap)
		goto free1_out;

	dev->dma_mem->virt_base = mem_base;
	dev->dma_mem->device_base = device_addr;
	dev->dma_mem->size = pages;
	dev->dma_mem->flags = flags;

	if (flags & DMA_MEMORY_MAP)
		return DMA_MEMORY_MAP;

	return DMA_MEMORY_IO;

 free1_out:
	kfree(dev->dma_mem);
 out:
	if (mem_base)
		iounmap(mem_base);
	return 0;
}
EXPORT_SYMBOL(dma_declare_coherent_memory);

void dma_release_declared_memory(struct device *dev)
{
	struct dma_coherent_mem *mem = dev->dma_mem;

	if (!mem)
		return;
	dev->dma_mem = NULL;
	iounmap(mem->virt_base);
	kfree(mem->bitmap);
	kfree(mem);
}
EXPORT_SYMBOL(dma_release_declared_memory);

void *dma_mark_declared_memory_occupied(struct device *dev,
					dma_addr_t device_addr, size_t size)
{
	struct dma_coherent_mem *mem = dev->dma_mem;
	int pos, err;
	int pages = (size + (device_addr & ~PAGE_MASK) + PAGE_SIZE - 1);

	pages >>= PAGE_SHIFT;

	if (!mem)
		return ERR_PTR(-EINVAL);

	pos = (device_addr - mem->device_base) >> PAGE_SHIFT;
	err = bitmap_allocate_region(mem->bitmap, pos, get_order(pages));
	if (err != 0)
		return ERR_PTR(err);
	return mem->virt_base + (pos << PAGE_SHIFT);
}
EXPORT_SYMBOL(dma_mark_declared_memory_occupied);

static int dma_alloc_from_coherent_mem(struct device *dev, ssize_t size,
				       dma_addr_t *dma_handle, void **ret)
{
	struct dma_coherent_mem *mem = dev ? dev->dma_mem : NULL;
	int order = get_order(size);

	if (mem) {
		int page = bitmap_find_free_region(mem->bitmap, mem->size,
						     order);
		if (page >= 0) {
			*dma_handle = mem->device_base + (page << PAGE_SHIFT);
			*ret = mem->virt_base + (page << PAGE_SHIFT);
			memset(*ret, 0, size);
		}
		if (mem->flags & DMA_MEMORY_EXCLUSIVE)
			*ret = NULL;
	}
	return (mem != NULL);
}

static int dma_release_coherent(struct device *dev, int order, void *vaddr)
{
	struct dma_coherent_mem *mem = dev ? dev->dma_mem : NULL;

	if (mem && vaddr >= mem->virt_base && vaddr <
		   (mem->virt_base + (mem->size << PAGE_SHIFT))) {
		int page = (vaddr - mem->virt_base) >> PAGE_SHIFT;

		bitmap_release_region(mem->bitmap, page, order);
		return 1;
	}
	return 0;
}
#else
#define dma_alloc_from_coherent_mem(dev, size, handle, ret) (0)
#define dma_release_coherent(dev, order, vaddr) (0)
#endif /* CONFIG_X86_32 */

int dma_supported(struct device *dev, u64 mask)
{
#ifdef CONFIG_PCI
	if (mask > 0xffffffff && forbid_dac > 0) {
		printk(KERN_INFO "PCI: Disallowing DAC for device %s\n",
				 dev->bus_id);
		return 0;
	}
#endif

	if (dma_ops->dma_supported)
		return dma_ops->dma_supported(dev, mask);

	/* Copied from i386. Doesn't make much sense, because it will
	   only work for pci_alloc_coherent.
	   The caller just has to use GFP_DMA in this case. */
	if (mask < DMA_24BIT_MASK)
		return 0;

	/* Tell the device to use SAC when IOMMU force is on.  This
	   allows the driver to use cheaper accesses in some cases.

	   Problem with this is that if we overflow the IOMMU area and
	   return DAC as fallback address the device may not handle it
	   correctly.

	   As a special case some controllers have a 39bit address
	   mode that is as efficient as 32bit (aic79xx). Don't force
	   SAC for these.  Assume all masks <= 40 bits are of this
	   type. Normally this doesn't make any difference, but gives
	   more gentle handling of IOMMU overflow. */
	if (iommu_sac_force && (mask >= DMA_40BIT_MASK)) {
		printk(KERN_INFO "%s: Force SAC with mask %Lx\n",
				 dev->bus_id, mask);
		return 0;
	}

	return 1;
}
EXPORT_SYMBOL(dma_supported);

/* Allocate DMA memory on node near device */
noinline struct page *
dma_alloc_pages(struct device *dev, gfp_t gfp, unsigned order)
{
	int node;

	node = dev_to_node(dev);

	return alloc_pages_node(node, gfp, order);
}

/*
 * Allocate memory for a coherent mapping.
 */
void *
dma_alloc_coherent(struct device *dev, size_t size, dma_addr_t *dma_handle,
		   gfp_t gfp)
{
	void *memory = NULL;
	struct page *page;
	unsigned long dma_mask = 0;
	dma_addr_t bus;

	/* ignore region specifiers */
	gfp &= ~(__GFP_DMA | __GFP_HIGHMEM | __GFP_DMA32);

	if (dma_alloc_from_coherent_mem(dev, size, dma_handle, &memory))
		return memory;

	if (!dev) {
		dev = &fallback_dev;
		gfp |= GFP_DMA;
	}
	dma_mask = dev->coherent_dma_mask;
	if (dma_mask == 0)
		dma_mask = (gfp & GFP_DMA) ? DMA_24BIT_MASK : DMA_32BIT_MASK;

	/* Device not DMA able */
	if (dev->dma_mask == NULL)
		return NULL;

	/* Don't invoke OOM killer */
	gfp |= __GFP_NORETRY;

#ifdef CONFIG_X86_64
	/* Why <=? Even when the mask is smaller than 4GB it is often
	   larger than 16MB and in this case we have a chance of
	   finding fitting memory in the next higher zone first. If
	   not retry with true GFP_DMA. -AK */
	if (dma_mask <= DMA_32BIT_MASK && !(gfp & GFP_DMA))
		gfp |= GFP_DMA32;
#endif

 again:
	page = dma_alloc_pages(dev, gfp, get_order(size));
	if (page == NULL)
		return NULL;

	{
		int high, mmu;
		bus = page_to_phys(page);
		memory = page_address(page);
		high = (bus + size) >= dma_mask;
		mmu = high;
		if (force_iommu && !(gfp & GFP_DMA))
			mmu = 1;
		else if (high) {
			free_pages((unsigned long)memory,
				   get_order(size));

			/* Don't use the 16MB ZONE_DMA unless absolutely
			   needed. It's better to use remapping first. */
			if (dma_mask < DMA_32BIT_MASK && !(gfp & GFP_DMA)) {
				gfp = (gfp & ~GFP_DMA32) | GFP_DMA;
				goto again;
			}

			/* Let low level make its own zone decisions */
			gfp &= ~(GFP_DMA32|GFP_DMA);

			if (dma_ops->alloc_coherent)
				return dma_ops->alloc_coherent(dev, size,
							   dma_handle, gfp);
			return NULL;
		}

		memset(memory, 0, size);
		if (!mmu) {
			*dma_handle = bus;
			return memory;
		}
	}

	if (dma_ops->alloc_coherent) {
		free_pages((unsigned long)memory, get_order(size));
		gfp &= ~(GFP_DMA|GFP_DMA32);
		return dma_ops->alloc_coherent(dev, size, dma_handle, gfp);
	}

	if (dma_ops->map_simple) {
		*dma_handle = dma_ops->map_simple(dev, virt_to_phys(memory),
					      size,
					      PCI_DMA_BIDIRECTIONAL);
		if (*dma_handle != bad_dma_address)
			return memory;
	}

	if (panic_on_overflow)
		panic("dma_alloc_coherent: IOMMU overflow by %lu bytes\n",
		      (unsigned long)size);
	free_pages((unsigned long)memory, get_order(size));
	return NULL;
}
EXPORT_SYMBOL(dma_alloc_coherent);

/*
 * Unmap coherent memory.
 * The caller must ensure that the device has finished accessing the mapping.
 */
void dma_free_coherent(struct device *dev, size_t size,
			 void *vaddr, dma_addr_t bus)
{
	int order = get_order(size);
	WARN_ON(irqs_disabled());	/* for portability */
	if (dma_release_coherent(dev, order, vaddr))
		return;
	if (dma_ops->unmap_single)
		dma_ops->unmap_single(dev, bus, size, 0);
	free_pages((unsigned long)vaddr, order);
}
EXPORT_SYMBOL(dma_free_coherent);

static int __init pci_iommu_init(void)
{
#ifdef CONFIG_CALGARY_IOMMU
	calgary_iommu_init();
#endif

	intel_iommu_init();

#ifdef CONFIG_GART_IOMMU
	gart_iommu_init();
#endif

	no_iommu_init();
	return 0;
}

void pci_iommu_shutdown(void)
{
	gart_iommu_shutdown();
}
/* Must execute after PCI subsystem */
fs_initcall(pci_iommu_init);

#ifdef CONFIG_PCI
/* Many VIA bridges seem to corrupt data for DAC. Disable it here */

static __devinit void via_no_dac(struct pci_dev *dev)
{
	if ((dev->class >> 8) == PCI_CLASS_BRIDGE_PCI && forbid_dac == 0) {
		printk(KERN_INFO "PCI: VIA PCI bridge detected."
				 "Disabling DAC.\n");
		forbid_dac = 1;
	}
}
DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_VIA, PCI_ANY_ID, via_no_dac);
#endif
Commit	Line	Data
459121c9	1	#include <linux/dma-mapping.h>
cb5867a5	2	#include <linux/dmar.h>
116890d5	3	#include <linux/bootmem.h>
bca5c096	4	#include <linux/pci.h>
cb5867a5	5
116890d5 GC	6	#include <asm/proto.h>
116890d5 GC	7	#include <asm/dma.h>
cb5867a5 GC	8	#include <asm/gart.h>
cb5867a5 GC	9	#include <asm/calgary.h>
459121c9	10
bca5c096 GC	11	int forbid_dac __read_mostly;
	12	EXPORT_SYMBOL(forbid_dac);
	13
85c246ee GC	14	const struct dma_mapping_ops *dma_ops;
	15	EXPORT_SYMBOL(dma_ops);
	16
b4cdc430	17	static int iommu_sac_force __read_mostly;
8e0c3797	18
f9c258de GC	19	#ifdef CONFIG_IOMMU_DEBUG
	20	int panic_on_overflow __read_mostly = 1;
	21	int force_iommu __read_mostly = 1;
	22	#else
	23	int panic_on_overflow __read_mostly = 0;
	24	int force_iommu __read_mostly = 0;
	25	#endif
	26
fae9a0d8 GC	27	int iommu_merge __read_mostly = 0;
	28
	29	int no_iommu __read_mostly;
	30	/* Set this to 1 if there is a HW IOMMU in the system */
	31	int iommu_detected __read_mostly = 0;
	32
	33	/* This tells the BIO block layer to assume merging. Default to off
	34	because we cannot guarantee merging later. */
	35	int iommu_bio_merge __read_mostly = 0;
	36	EXPORT_SYMBOL(iommu_bio_merge);
	37
cac67877 GC	38	dma_addr_t bad_dma_address __read_mostly = 0;
cac67877 GC	39	EXPORT_SYMBOL(bad_dma_address);
fae9a0d8	40
098cb7f2 GC	41	/* Dummy device used for NULL arguments (normally ISA). Better would
	42	be probably a smaller DMA mask, but this is bug-to-bug compatible
	43	to older i386. */
	44	struct device fallback_dev = {
	45	.bus_id = "fallback device",
	46	.coherent_dma_mask = DMA_32BIT_MASK,
	47	.dma_mask = &fallback_dev.coherent_dma_mask,
	48	};
	49
459121c9 GC	50	int dma_set_mask(struct device *dev, u64 mask)
	51	{
	52	if (!dev->dma_mask \|\| !dma_supported(dev, mask))
	53	return -EIO;
	54
	55	*dev->dma_mask = mask;
	56
	57	return 0;
	58	}
	59	EXPORT_SYMBOL(dma_set_mask);
	60
116890d5 GC	61	#ifdef CONFIG_X86_64
	62	static __initdata void *dma32_bootmem_ptr;
	63	static unsigned long dma32_bootmem_size __initdata = (128ULL<<20);
	64
	65	static int __init parse_dma32_size_opt(char *p)
	66	{
	67	if (!p)
	68	return -EINVAL;
	69	dma32_bootmem_size = memparse(p, &p);
	70	return 0;
	71	}
	72	early_param("dma32_size", parse_dma32_size_opt);
	73
	74	void __init dma32_reserve_bootmem(void)
	75	{
	76	unsigned long size, align;
	77	if (end_pfn <= MAX_DMA32_PFN)
	78	return;
	79
	80	align = 64ULL<<20;
	81	size = round_up(dma32_bootmem_size, align);
	82	dma32_bootmem_ptr = __alloc_bootmem_nopanic(size, align,
	83	__pa(MAX_DMA_ADDRESS));
	84	if (dma32_bootmem_ptr)
	85	dma32_bootmem_size = size;
	86	else
	87	dma32_bootmem_size = 0;
	88	}
	89	static void __init dma32_free_bootmem(void)
	90	{
	91	int node;
	92
	93	if (end_pfn <= MAX_DMA32_PFN)
	94	return;
	95
	96	if (!dma32_bootmem_ptr)
	97	return;
	98
	99	for_each_online_node(node)
	100	free_bootmem_node(NODE_DATA(node), __pa(dma32_bootmem_ptr),
	101	dma32_bootmem_size);
	102
	103	dma32_bootmem_ptr = NULL;
	104	dma32_bootmem_size = 0;
	105	}
	106
	107	void __init pci_iommu_alloc(void)
	108	{
	109	/* free the range so iommu could get some range less than 4G */
	110	dma32_free_bootmem();
	111	/*
	112	* The order of these functions is important for
	113	* fall-back/fail-over reasons
	114	*/
	115	#ifdef CONFIG_GART_IOMMU
	116	gart_iommu_hole_init();
	117	#endif
	118
	119	#ifdef CONFIG_CALGARY_IOMMU
	120	detect_calgary();
	121	#endif
	122
	123	detect_intel_iommu();
	124
125	#ifdef CONFIG_SWIOTLB
126	pci_swiotlb_init();
127	#endif
128	}
129	#endif
130
fae9a0d8 GC	131	/*
	132	* See <Documentation/x86_64/boot-options.txt> for the iommu kernel parameter
	133	* documentation.
	134	*/
	135	static __init int iommu_setup(char *p)
	136	{
	137	iommu_merge = 1;
	138
	139	if (!p)
	140	return -EINVAL;
	141
	142	while (*p) {
	143	if (!strncmp(p, "off", 3))
	144	no_iommu = 1;
	145	/* gart_parse_options has more force support */
	146	if (!strncmp(p, "force", 5))
	147	force_iommu = 1;
	148	if (!strncmp(p, "noforce", 7)) {
	149	iommu_merge = 0;
	150	force_iommu = 0;
	151	}
	152
	153	if (!strncmp(p, "biomerge", 8)) {
	154	iommu_bio_merge = 4096;
	155	iommu_merge = 1;
	156	force_iommu = 1;
	157	}
	158	if (!strncmp(p, "panic", 5))
	159	panic_on_overflow = 1;
	160	if (!strncmp(p, "nopanic", 7))
	161	panic_on_overflow = 0;
	162	if (!strncmp(p, "merge", 5)) {
	163	iommu_merge = 1;
	164	force_iommu = 1;
	165	}
	166	if (!strncmp(p, "nomerge", 7))
	167	iommu_merge = 0;
	168	if (!strncmp(p, "forcesac", 8))
	169	iommu_sac_force = 1;
	170	if (!strncmp(p, "allowdac", 8))
	171	forbid_dac = 0;
	172	if (!strncmp(p, "nodac", 5))
	173	forbid_dac = -1;
	174	if (!strncmp(p, "usedac", 6)) {
	175	forbid_dac = -1;
	176	return 1;
	177	}
	178	#ifdef CONFIG_SWIOTLB
	179	if (!strncmp(p, "soft", 4))
	180	swiotlb = 1;
	181	#endif
	182
	183	#ifdef CONFIG_GART_IOMMU
	184	gart_parse_options(p);
	185	#endif
	186
	187	#ifdef CONFIG_CALGARY_IOMMU
	188	if (!strncmp(p, "calgary", 7))
	189	use_calgary = 1;
	190	#endif /* CONFIG_CALGARY_IOMMU */
	191
	192	p += strcspn(p, ",");
	193	if (*p == ',')
	194	++p;
195	}
196	return 0;
197	}
198	early_param("iommu", iommu_setup);
199
8e8edc64 GC	200	#ifdef CONFIG_X86_32
	201	int dma_declare_coherent_memory(struct device *dev, dma_addr_t bus_addr,
	202	dma_addr_t device_addr, size_t size, int flags)
	203	{
	204	void __iomem *mem_base = NULL;
	205	int pages = size >> PAGE_SHIFT;
	206	int bitmap_size = BITS_TO_LONGS(pages) * sizeof(long);
	207
	208	if ((flags & (DMA_MEMORY_MAP \| DMA_MEMORY_IO)) == 0)
	209	goto out;
	210	if (!size)
	211	goto out;
	212	if (dev->dma_mem)
	213	goto out;
	214
	215	/* FIXME: this routine just ignores DMA_MEMORY_INCLUDES_CHILDREN */
	216
	217	mem_base = ioremap(bus_addr, size);
	218	if (!mem_base)
	219	goto out;
	220
	221	dev->dma_mem = kzalloc(sizeof(struct dma_coherent_mem), GFP_KERNEL);
	222	if (!dev->dma_mem)
	223	goto out;
	224	dev->dma_mem->bitmap = kzalloc(bitmap_size, GFP_KERNEL);
	225	if (!dev->dma_mem->bitmap)
	226	goto free1_out;
	227
	228	dev->dma_mem->virt_base = mem_base;
	229	dev->dma_mem->device_base = device_addr;
	230	dev->dma_mem->size = pages;
	231	dev->dma_mem->flags = flags;
	232
	233	if (flags & DMA_MEMORY_MAP)
	234	return DMA_MEMORY_MAP;
	235
	236	return DMA_MEMORY_IO;
	237
	238	free1_out:
	239	kfree(dev->dma_mem);
	240	out:
	241	if (mem_base)
	242	iounmap(mem_base);
	243	return 0;
	244	}
	245	EXPORT_SYMBOL(dma_declare_coherent_memory);
	246
	247	void dma_release_declared_memory(struct device *dev)
	248	{
	249	struct dma_coherent_mem *mem = dev->dma_mem;
	250
	251	if (!mem)
	252	return;
	253	dev->dma_mem = NULL;
	254	iounmap(mem->virt_base);
	255	kfree(mem->bitmap);
	256	kfree(mem);
	257	}
	258	EXPORT_SYMBOL(dma_release_declared_memory);
	259
	260	void dma_mark_declared_memory_occupied(struct device dev,
	261	dma_addr_t device_addr, size_t size)
	262	{
	263	struct dma_coherent_mem *mem = dev->dma_mem;
264	int pos, err;
265	int pages = (size + (device_addr & ~PAGE_MASK) + PAGE_SIZE - 1);
266
267	pages >>= PAGE_SHIFT;
268
269	if (!mem)
270	return ERR_PTR(-EINVAL);
271
272	pos = (device_addr - mem->device_base) >> PAGE_SHIFT;
273	err = bitmap_allocate_region(mem->bitmap, pos, get_order(pages));
274	if (err != 0)
275	return ERR_PTR(err);
276	return mem->virt_base + (pos << PAGE_SHIFT);
277	}
278	EXPORT_SYMBOL(dma_mark_declared_memory_occupied);
098cb7f2 GC	279
	280	static int dma_alloc_from_coherent_mem(struct device *dev, ssize_t size,
	281	dma_addr_t dma_handle, void *ret)
	282	{
	283	struct dma_coherent_mem *mem = dev ? dev->dma_mem : NULL;
	284	int order = get_order(size);
	285
	286	if (mem) {
	287	int page = bitmap_find_free_region(mem->bitmap, mem->size,
	288	order);
	289	if (page >= 0) {
	290	*dma_handle = mem->device_base + (page << PAGE_SHIFT);
	291	*ret = mem->virt_base + (page << PAGE_SHIFT);
	292	memset(*ret, 0, size);
	293	}
	294	if (mem->flags & DMA_MEMORY_EXCLUSIVE)
	295	*ret = NULL;
	296	}
	297	return (mem != NULL);
	298	}
	299
	300	static int dma_release_coherent(struct device dev, int order, void vaddr)
	301	{
	302	struct dma_coherent_mem *mem = dev ? dev->dma_mem : NULL;
	303
	304	if (mem && vaddr >= mem->virt_base && vaddr <
	305	(mem->virt_base + (mem->size << PAGE_SHIFT))) {
	306	int page = (vaddr - mem->virt_base) >> PAGE_SHIFT;
	307
	308	bitmap_release_region(mem->bitmap, page, order);
	309	return 1;
	310	}
	311	return 0;
	312	}
	313	#else
	314	#define dma_alloc_from_coherent_mem(dev, size, handle, ret) (0)
	315	#define dma_release_coherent(dev, order, vaddr) (0)
8e8edc64 GC	316	#endif /* CONFIG_X86_32 */
8e8edc64 GC	317
8e0c3797 GC	318	int dma_supported(struct device *dev, u64 mask)
	319	{
	320	#ifdef CONFIG_PCI
	321	if (mask > 0xffffffff && forbid_dac > 0) {
	322	printk(KERN_INFO "PCI: Disallowing DAC for device %s\n",
	323	dev->bus_id);
	324	return 0;
	325	}
	326	#endif
	327
	328	if (dma_ops->dma_supported)
	329	return dma_ops->dma_supported(dev, mask);
	330
	331	/* Copied from i386. Doesn't make much sense, because it will
	332	only work for pci_alloc_coherent.
	333	The caller just has to use GFP_DMA in this case. */
	334	if (mask < DMA_24BIT_MASK)
	335	return 0;
	336
	337	/* Tell the device to use SAC when IOMMU force is on. This
	338	allows the driver to use cheaper accesses in some cases.
	339
	340	Problem with this is that if we overflow the IOMMU area and
	341	return DAC as fallback address the device may not handle it
	342	correctly.
	343
	344	As a special case some controllers have a 39bit address
	345	mode that is as efficient as 32bit (aic79xx). Don't force
	346	SAC for these. Assume all masks <= 40 bits are of this
	347	type. Normally this doesn't make any difference, but gives
	348	more gentle handling of IOMMU overflow. */
	349	if (iommu_sac_force && (mask >= DMA_40BIT_MASK)) {
	350	printk(KERN_INFO "%s: Force SAC with mask %Lx\n",
	351	dev->bus_id, mask);
	352	return 0;
	353	}
	354
	355	return 1;
	356	}
	357	EXPORT_SYMBOL(dma_supported);
	358
098cb7f2 GC	359	/* Allocate DMA memory on node near device */
	360	noinline struct page *
	361	dma_alloc_pages(struct device *dev, gfp_t gfp, unsigned order)
	362	{
	363	int node;
	364
	365	node = dev_to_node(dev);
	366
	367	return alloc_pages_node(node, gfp, order);
	368	}
	369
	370	/*
	371	* Allocate memory for a coherent mapping.
	372	*/
	373	void *
	374	dma_alloc_coherent(struct device dev, size_t size, dma_addr_t dma_handle,
	375	gfp_t gfp)
	376	{
	377	void *memory = NULL;
	378	struct page *page;
	379	unsigned long dma_mask = 0;
	380	dma_addr_t bus;
	381
	382	/* ignore region specifiers */
	383	gfp &= ~(__GFP_DMA \| __GFP_HIGHMEM \| __GFP_DMA32);
	384
	385	if (dma_alloc_from_coherent_mem(dev, size, dma_handle, &memory))
	386	return memory;
	387
4a367f3a	388	if (!dev) {
098cb7f2	389	dev = &fallback_dev;
4a367f3a TI	390	gfp \|= GFP_DMA;
4a367f3a TI	391	}
098cb7f2 GC	392	dma_mask = dev->coherent_dma_mask;
098cb7f2 GC	393	if (dma_mask == 0)
4a367f3a	394	dma_mask = (gfp & GFP_DMA) ? DMA_24BIT_MASK : DMA_32BIT_MASK;
098cb7f2 GC	395
	396	/* Device not DMA able */
	397	if (dev->dma_mask == NULL)
	398	return NULL;
	399
	400	/* Don't invoke OOM killer */
	401	gfp \|= __GFP_NORETRY;
	402
	403	#ifdef CONFIG_X86_64
	404	/* Why <=? Even when the mask is smaller than 4GB it is often
	405	larger than 16MB and in this case we have a chance of
	406	finding fitting memory in the next higher zone first. If
	407	not retry with true GFP_DMA. -AK */
4a367f3a	408	if (dma_mask <= DMA_32BIT_MASK && !(gfp & GFP_DMA))
098cb7f2 GC	409	gfp \|= GFP_DMA32;
	410	#endif
	411
	412	again:
	413	page = dma_alloc_pages(dev, gfp, get_order(size));
	414	if (page == NULL)
	415	return NULL;
	416
	417	{
	418	int high, mmu;
	419	bus = page_to_phys(page);
	420	memory = page_address(page);
	421	high = (bus + size) >= dma_mask;
	422	mmu = high;
	423	if (force_iommu && !(gfp & GFP_DMA))
	424	mmu = 1;
	425	else if (high) {
	426	free_pages((unsigned long)memory,
	427	get_order(size));
	428
	429	/* Don't use the 16MB ZONE_DMA unless absolutely
	430	needed. It's better to use remapping first. */
	431	if (dma_mask < DMA_32BIT_MASK && !(gfp & GFP_DMA)) {
	432	gfp = (gfp & ~GFP_DMA32) \| GFP_DMA;
	433	goto again;
	434	}
	435
	436	/* Let low level make its own zone decisions */
	437	gfp &= ~(GFP_DMA32\|GFP_DMA);
	438
	439	if (dma_ops->alloc_coherent)
	440	return dma_ops->alloc_coherent(dev, size,
	441	dma_handle, gfp);
	442	return NULL;
	443	}
	444
	445	memset(memory, 0, size);
	446	if (!mmu) {
	447	*dma_handle = bus;
	448	return memory;
	449	}
	450	}
	451
	452	if (dma_ops->alloc_coherent) {
	453	free_pages((unsigned long)memory, get_order(size));
	454	gfp &= ~(GFP_DMA\|GFP_DMA32);
	455	return dma_ops->alloc_coherent(dev, size, dma_handle, gfp);
	456	}
	457
	458	if (dma_ops->map_simple) {
	459	*dma_handle = dma_ops->map_simple(dev, virt_to_phys(memory),
	460	size,
	461	PCI_DMA_BIDIRECTIONAL);
	462	if (*dma_handle != bad_dma_address)
	463	return memory;
	464	}
	465
	466	if (panic_on_overflow)
	467	panic("dma_alloc_coherent: IOMMU overflow by %lu bytes\n",
	468	(unsigned long)size);
	469	free_pages((unsigned long)memory, get_order(size));
	470	return NULL;
	471	}
	472	EXPORT_SYMBOL(dma_alloc_coherent);
473
474	/*
475	* Unmap coherent memory.
476	* The caller must ensure that the device has finished accessing the mapping.
477	*/
478	void dma_free_coherent(struct device *dev, size_t size,
479	void *vaddr, dma_addr_t bus)
480	{
481	int order = get_order(size);
482	WARN_ON(irqs_disabled()); /* for portability */
483	if (dma_release_coherent(dev, order, vaddr))
484	return;
485	if (dma_ops->unmap_single)
486	dma_ops->unmap_single(dev, bus, size, 0);
487	free_pages((unsigned long)vaddr, order);
488	}
489	EXPORT_SYMBOL(dma_free_coherent);
8e0c3797	490
cb5867a5 GC	491	static int __init pci_iommu_init(void)
	492	{
	493	#ifdef CONFIG_CALGARY_IOMMU
	494	calgary_iommu_init();
	495	#endif
	496
	497	intel_iommu_init();
	498
	499	#ifdef CONFIG_GART_IOMMU
	500	gart_iommu_init();
	501	#endif
459121c9	502
cb5867a5 GC	503	no_iommu_init();
	504	return 0;
	505	}
	506
	507	void pci_iommu_shutdown(void)
	508	{
	509	gart_iommu_shutdown();
	510	}
	511	/* Must execute after PCI subsystem */
	512	fs_initcall(pci_iommu_init);
bca5c096 GC	513
	514	#ifdef CONFIG_PCI
	515	/* Many VIA bridges seem to corrupt data for DAC. Disable it here */
	516
	517	static __devinit void via_no_dac(struct pci_dev *dev)
	518	{
	519	if ((dev->class >> 8) == PCI_CLASS_BRIDGE_PCI && forbid_dac == 0) {
	520	printk(KERN_INFO "PCI: VIA PCI bridge detected."
	521	"Disabling DAC.\n");
	522	forbid_dac = 1;
	523	}
	524	}
	525	DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_VIA, PCI_ANY_ID, via_no_dac);
	526	#endif