[deliverable/linux.git] / arch / arm64 / mm / dma-mapping.c

/*
 * SWIOTLB-based DMA API implementation
 *
 * Copyright (C) 2012 ARM Ltd.
 * Author: Catalin Marinas <catalin.marinas@arm.com>
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License version 2 as
 * published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */

#include <linux/gfp.h>
#include <linux/export.h>
#include <linux/slab.h>
#include <linux/dma-mapping.h>
#include <linux/dma-contiguous.h>
#include <linux/of.h>
#include <linux/platform_device.h>
#include <linux/vmalloc.h>
#include <linux/swiotlb.h>
#include <linux/amba/bus.h>

#include <asm/cacheflush.h>

struct dma_map_ops *dma_ops;
EXPORT_SYMBOL(dma_ops);

static pgprot_t __get_dma_pgprot(struct dma_attrs *attrs, pgprot_t prot,
				 bool coherent)
{
	if (!coherent || dma_get_attr(DMA_ATTR_WRITE_COMBINE, attrs))
		return pgprot_writecombine(prot);
	return prot;
}

static void *__dma_alloc_coherent(struct device *dev, size_t size,
				  dma_addr_t *dma_handle, gfp_t flags,
				  struct dma_attrs *attrs)
{
	if (dev == NULL) {
		WARN_ONCE(1, "Use an actual device structure for DMA allocation\n");
		return NULL;
	}

	if (IS_ENABLED(CONFIG_ZONE_DMA) &&
	    dev->coherent_dma_mask <= DMA_BIT_MASK(32))
		flags |= GFP_DMA;
	if (IS_ENABLED(CONFIG_DMA_CMA)) {
		struct page *page;

		size = PAGE_ALIGN(size);
		page = dma_alloc_from_contiguous(dev, size >> PAGE_SHIFT,
							get_order(size));
		if (!page)
			return NULL;

		*dma_handle = phys_to_dma(dev, page_to_phys(page));
		return page_address(page);
	} else {
		return swiotlb_alloc_coherent(dev, size, dma_handle, flags);
	}
}

static void __dma_free_coherent(struct device *dev, size_t size,
				void *vaddr, dma_addr_t dma_handle,
				struct dma_attrs *attrs)
{
	if (dev == NULL) {
		WARN_ONCE(1, "Use an actual device structure for DMA allocation\n");
		return;
	}

	if (IS_ENABLED(CONFIG_DMA_CMA)) {
		phys_addr_t paddr = dma_to_phys(dev, dma_handle);

		dma_release_from_contiguous(dev,
					phys_to_page(paddr),
					size >> PAGE_SHIFT);
	} else {
		swiotlb_free_coherent(dev, size, vaddr, dma_handle);
	}
}

static void *__dma_alloc_noncoherent(struct device *dev, size_t size,
				     dma_addr_t *dma_handle, gfp_t flags,
				     struct dma_attrs *attrs)
{
	struct page *page, **map;
	void *ptr, *coherent_ptr;
	int order, i;

	size = PAGE_ALIGN(size);
	order = get_order(size);

	ptr = __dma_alloc_coherent(dev, size, dma_handle, flags, attrs);
	if (!ptr)
		goto no_mem;
	map = kmalloc(sizeof(struct page *) << order, flags & ~GFP_DMA);
	if (!map)
		goto no_map;

	/* remove any dirty cache lines on the kernel alias */
	__dma_flush_range(ptr, ptr + size);

	/* create a coherent mapping */
	page = virt_to_page(ptr);
	for (i = 0; i < (size >> PAGE_SHIFT); i++)
		map[i] = page + i;
	coherent_ptr = vmap(map, size >> PAGE_SHIFT, VM_MAP,
			    __get_dma_pgprot(attrs, pgprot_default, false));
	kfree(map);
	if (!coherent_ptr)
		goto no_map;

	return coherent_ptr;

no_map:
	__dma_free_coherent(dev, size, ptr, *dma_handle, attrs);
no_mem:
	*dma_handle = ~0;
	return NULL;
}

static void __dma_free_noncoherent(struct device *dev, size_t size,
				   void *vaddr, dma_addr_t dma_handle,
				   struct dma_attrs *attrs)
{
	void *swiotlb_addr = phys_to_virt(dma_to_phys(dev, dma_handle));

	vunmap(vaddr);
	__dma_free_coherent(dev, size, swiotlb_addr, dma_handle, attrs);
}

static dma_addr_t __swiotlb_map_page(struct device *dev, struct page *page,
				     unsigned long offset, size_t size,
				     enum dma_data_direction dir,
				     struct dma_attrs *attrs)
{
	dma_addr_t dev_addr;

	dev_addr = swiotlb_map_page(dev, page, offset, size, dir, attrs);
	__dma_map_area(phys_to_virt(dma_to_phys(dev, dev_addr)), size, dir);

	return dev_addr;
}


static void __swiotlb_unmap_page(struct device *dev, dma_addr_t dev_addr,
				 size_t size, enum dma_data_direction dir,
				 struct dma_attrs *attrs)
{
	__dma_unmap_area(phys_to_virt(dma_to_phys(dev, dev_addr)), size, dir);
	swiotlb_unmap_page(dev, dev_addr, size, dir, attrs);
}

static int __swiotlb_map_sg_attrs(struct device *dev, struct scatterlist *sgl,
				  int nelems, enum dma_data_direction dir,
				  struct dma_attrs *attrs)
{
	struct scatterlist *sg;
	int i, ret;

	ret = swiotlb_map_sg_attrs(dev, sgl, nelems, dir, attrs);
	for_each_sg(sgl, sg, ret, i)
		__dma_map_area(phys_to_virt(dma_to_phys(dev, sg->dma_address)),
			       sg->length, dir);

	return ret;
}

static void __swiotlb_unmap_sg_attrs(struct device *dev,
				     struct scatterlist *sgl, int nelems,
				     enum dma_data_direction dir,
				     struct dma_attrs *attrs)
{
	struct scatterlist *sg;
	int i;

	for_each_sg(sgl, sg, nelems, i)
		__dma_unmap_area(phys_to_virt(dma_to_phys(dev, sg->dma_address)),
				 sg->length, dir);
	swiotlb_unmap_sg_attrs(dev, sgl, nelems, dir, attrs);
}

static void __swiotlb_sync_single_for_cpu(struct device *dev,
					  dma_addr_t dev_addr, size_t size,
					  enum dma_data_direction dir)
{
	__dma_unmap_area(phys_to_virt(dma_to_phys(dev, dev_addr)), size, dir);
	swiotlb_sync_single_for_cpu(dev, dev_addr, size, dir);
}

static void __swiotlb_sync_single_for_device(struct device *dev,
					     dma_addr_t dev_addr, size_t size,
					     enum dma_data_direction dir)
{
	swiotlb_sync_single_for_device(dev, dev_addr, size, dir);
	__dma_map_area(phys_to_virt(dma_to_phys(dev, dev_addr)), size, dir);
}

static void __swiotlb_sync_sg_for_cpu(struct device *dev,
				      struct scatterlist *sgl, int nelems,
				      enum dma_data_direction dir)
{
	struct scatterlist *sg;
	int i;

	for_each_sg(sgl, sg, nelems, i)
		__dma_unmap_area(phys_to_virt(dma_to_phys(dev, sg->dma_address)),
				 sg->length, dir);
	swiotlb_sync_sg_for_cpu(dev, sgl, nelems, dir);
}

static void __swiotlb_sync_sg_for_device(struct device *dev,
					 struct scatterlist *sgl, int nelems,
					 enum dma_data_direction dir)
{
	struct scatterlist *sg;
	int i;

	swiotlb_sync_sg_for_device(dev, sgl, nelems, dir);
	for_each_sg(sgl, sg, nelems, i)
		__dma_map_area(phys_to_virt(dma_to_phys(dev, sg->dma_address)),
			       sg->length, dir);
}

/* vma->vm_page_prot must be set appropriately before calling this function */
static int __dma_common_mmap(struct device *dev, struct vm_area_struct *vma,
			     void *cpu_addr, dma_addr_t dma_addr, size_t size)
{
	int ret = -ENXIO;
	unsigned long nr_vma_pages = (vma->vm_end - vma->vm_start) >>
					PAGE_SHIFT;
	unsigned long nr_pages = PAGE_ALIGN(size) >> PAGE_SHIFT;
	unsigned long pfn = dma_to_phys(dev, dma_addr) >> PAGE_SHIFT;
	unsigned long off = vma->vm_pgoff;

	if (dma_mmap_from_coherent(dev, vma, cpu_addr, size, &ret))
		return ret;

	if (off < nr_pages && nr_vma_pages <= (nr_pages - off)) {
		ret = remap_pfn_range(vma, vma->vm_start,
				      pfn + off,
				      vma->vm_end - vma->vm_start,
				      vma->vm_page_prot);
	}

	return ret;
}

static int __swiotlb_mmap_noncoherent(struct device *dev,
		struct vm_area_struct *vma,
		void *cpu_addr, dma_addr_t dma_addr, size_t size,
		struct dma_attrs *attrs)
{
	vma->vm_page_prot = __get_dma_pgprot(attrs, vma->vm_page_prot, false);
	return __dma_common_mmap(dev, vma, cpu_addr, dma_addr, size);
}

static int __swiotlb_mmap_coherent(struct device *dev,
		struct vm_area_struct *vma,
		void *cpu_addr, dma_addr_t dma_addr, size_t size,
		struct dma_attrs *attrs)
{
	/* Just use whatever page_prot attributes were specified */
	return __dma_common_mmap(dev, vma, cpu_addr, dma_addr, size);
}

struct dma_map_ops noncoherent_swiotlb_dma_ops = {
	.alloc = __dma_alloc_noncoherent,
	.free = __dma_free_noncoherent,
	.mmap = __swiotlb_mmap_noncoherent,
	.map_page = __swiotlb_map_page,
	.unmap_page = __swiotlb_unmap_page,
	.map_sg = __swiotlb_map_sg_attrs,
	.unmap_sg = __swiotlb_unmap_sg_attrs,
	.sync_single_for_cpu = __swiotlb_sync_single_for_cpu,
	.sync_single_for_device = __swiotlb_sync_single_for_device,
	.sync_sg_for_cpu = __swiotlb_sync_sg_for_cpu,
	.sync_sg_for_device = __swiotlb_sync_sg_for_device,
	.dma_supported = swiotlb_dma_supported,
	.mapping_error = swiotlb_dma_mapping_error,
};
EXPORT_SYMBOL(noncoherent_swiotlb_dma_ops);

struct dma_map_ops coherent_swiotlb_dma_ops = {
	.alloc = __dma_alloc_coherent,
	.free = __dma_free_coherent,
	.mmap = __swiotlb_mmap_coherent,
	.map_page = swiotlb_map_page,
	.unmap_page = swiotlb_unmap_page,
	.map_sg = swiotlb_map_sg_attrs,
	.unmap_sg = swiotlb_unmap_sg_attrs,
	.sync_single_for_cpu = swiotlb_sync_single_for_cpu,
	.sync_single_for_device = swiotlb_sync_single_for_device,
	.sync_sg_for_cpu = swiotlb_sync_sg_for_cpu,
	.sync_sg_for_device = swiotlb_sync_sg_for_device,
	.dma_supported = swiotlb_dma_supported,
	.mapping_error = swiotlb_dma_mapping_error,
};
EXPORT_SYMBOL(coherent_swiotlb_dma_ops);

static int dma_bus_notifier(struct notifier_block *nb,
			    unsigned long event, void *_dev)
{
	struct device *dev = _dev;

	if (event != BUS_NOTIFY_ADD_DEVICE)
		return NOTIFY_DONE;

	if (of_property_read_bool(dev->of_node, "dma-coherent"))
		set_dma_ops(dev, &coherent_swiotlb_dma_ops);

	return NOTIFY_OK;
}

static struct notifier_block platform_bus_nb = {
	.notifier_call = dma_bus_notifier,
};

static struct notifier_block amba_bus_nb = {
	.notifier_call = dma_bus_notifier,
};

extern int swiotlb_late_init_with_default_size(size_t default_size);

static int __init swiotlb_late_init(void)
{
	size_t swiotlb_size = min(SZ_64M, MAX_ORDER_NR_PAGES << PAGE_SHIFT);

	/*
	 * These must be registered before of_platform_populate().
	 */
	bus_register_notifier(&platform_bus_type, &platform_bus_nb);
	bus_register_notifier(&amba_bustype, &amba_bus_nb);

	dma_ops = &noncoherent_swiotlb_dma_ops;

	return swiotlb_late_init_with_default_size(swiotlb_size);
}
arch_initcall(swiotlb_late_init);

#define PREALLOC_DMA_DEBUG_ENTRIES	4096

static int __init dma_debug_do_init(void)
{
	dma_debug_init(PREALLOC_DMA_DEBUG_ENTRIES);
	return 0;
}
fs_initcall(dma_debug_do_init);
Commit	Line	Data
09b55412 CM	1	/*
	2	* SWIOTLB-based DMA API implementation
	3	*
	4	* Copyright (C) 2012 ARM Ltd.
	5	* Author: Catalin Marinas <catalin.marinas@arm.com>
	6	*
	7	* This program is free software; you can redistribute it and/or modify
	8	* it under the terms of the GNU General Public License version 2 as
	9	* published by the Free Software Foundation.
	10	*
	11	* This program is distributed in the hope that it will be useful,
	12	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	13	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	14	* GNU General Public License for more details.
	15	*
	16	* You should have received a copy of the GNU General Public License
	17	* along with this program. If not, see <http://www.gnu.org/licenses/>.
	18	*/
	19
	20	#include <linux/gfp.h>
	21	#include <linux/export.h>
	22	#include <linux/slab.h>
	23	#include <linux/dma-mapping.h>
6ac2104d	24	#include <linux/dma-contiguous.h>
6ecba8eb CM	25	#include <linux/of.h>
6ecba8eb CM	26	#include <linux/platform_device.h>
09b55412 CM	27	#include <linux/vmalloc.h>
09b55412 CM	28	#include <linux/swiotlb.h>
6ecba8eb	29	#include <linux/amba/bus.h>
09b55412 CM	30
	31	#include <asm/cacheflush.h>
	32
	33	struct dma_map_ops *dma_ops;
	34	EXPORT_SYMBOL(dma_ops);
	35
214fdbe7 LA	36	static pgprot_t __get_dma_pgprot(struct dma_attrs *attrs, pgprot_t prot,
	37	bool coherent)
	38	{
196adf2f	39	if (!coherent \|\| dma_get_attr(DMA_ATTR_WRITE_COMBINE, attrs))
214fdbe7	40	return pgprot_writecombine(prot);
214fdbe7 LA	41	return prot;
	42	}
	43
bb10eb7b RH	44	static void __dma_alloc_coherent(struct device dev, size_t size,
	45	dma_addr_t *dma_handle, gfp_t flags,
	46	struct dma_attrs *attrs)
09b55412	47	{
c666e8d5 LA	48	if (dev == NULL) {
	49	WARN_ONCE(1, "Use an actual device structure for DMA allocation\n");
	50	return NULL;
	51	}
	52
19e7640d	53	if (IS_ENABLED(CONFIG_ZONE_DMA) &&
09b55412	54	dev->coherent_dma_mask <= DMA_BIT_MASK(32))
19e7640d	55	flags \|= GFP_DMA;
6ac2104d LA	56	if (IS_ENABLED(CONFIG_DMA_CMA)) {
	57	struct page *page;
	58
ccc9e244	59	size = PAGE_ALIGN(size);
6ac2104d LA	60	page = dma_alloc_from_contiguous(dev, size >> PAGE_SHIFT,
	61	get_order(size));
	62	if (!page)
	63	return NULL;
	64
	65	*dma_handle = phys_to_dma(dev, page_to_phys(page));
	66	return page_address(page);
	67	} else {
	68	return swiotlb_alloc_coherent(dev, size, dma_handle, flags);
	69	}
09b55412 CM	70	}
09b55412 CM	71
bb10eb7b RH	72	static void __dma_free_coherent(struct device *dev, size_t size,
	73	void *vaddr, dma_addr_t dma_handle,
	74	struct dma_attrs *attrs)
09b55412	75	{
c666e8d5 LA	76	if (dev == NULL) {
	77	WARN_ONCE(1, "Use an actual device structure for DMA allocation\n");
	78	return;
	79	}
	80
6ac2104d LA	81	if (IS_ENABLED(CONFIG_DMA_CMA)) {
	82	phys_addr_t paddr = dma_to_phys(dev, dma_handle);
	83
	84	dma_release_from_contiguous(dev,
	85	phys_to_page(paddr),
	86	size >> PAGE_SHIFT);
	87	} else {
	88	swiotlb_free_coherent(dev, size, vaddr, dma_handle);
	89	}
09b55412 CM	90	}
09b55412 CM	91
7363590d CM	92	static void __dma_alloc_noncoherent(struct device dev, size_t size,
	93	dma_addr_t *dma_handle, gfp_t flags,
	94	struct dma_attrs *attrs)
	95	{
	96	struct page page, *map;
	97	void ptr, coherent_ptr;
	98	int order, i;
	99
	100	size = PAGE_ALIGN(size);
	101	order = get_order(size);
	102
	103	ptr = __dma_alloc_coherent(dev, size, dma_handle, flags, attrs);
	104	if (!ptr)
	105	goto no_mem;
	106	map = kmalloc(sizeof(struct page *) << order, flags & ~GFP_DMA);
	107	if (!map)
	108	goto no_map;
	109
	110	/* remove any dirty cache lines on the kernel alias */
	111	__dma_flush_range(ptr, ptr + size);
	112
	113	/* create a coherent mapping */
	114	page = virt_to_page(ptr);
	115	for (i = 0; i < (size >> PAGE_SHIFT); i++)
	116	map[i] = page + i;
	117	coherent_ptr = vmap(map, size >> PAGE_SHIFT, VM_MAP,
214fdbe7	118	__get_dma_pgprot(attrs, pgprot_default, false));
7363590d CM	119	kfree(map);
	120	if (!coherent_ptr)
	121	goto no_map;
	122
	123	return coherent_ptr;
	124
	125	no_map:
	126	__dma_free_coherent(dev, size, ptr, *dma_handle, attrs);
	127	no_mem:
	128	*dma_handle = ~0;
	129	return NULL;
	130	}
	131
	132	static void __dma_free_noncoherent(struct device *dev, size_t size,
	133	void *vaddr, dma_addr_t dma_handle,
	134	struct dma_attrs *attrs)
	135	{
	136	void *swiotlb_addr = phys_to_virt(dma_to_phys(dev, dma_handle));
	137
	138	vunmap(vaddr);
	139	__dma_free_coherent(dev, size, swiotlb_addr, dma_handle, attrs);
	140	}
	141
	142	static dma_addr_t __swiotlb_map_page(struct device dev, struct page page,
	143	unsigned long offset, size_t size,
	144	enum dma_data_direction dir,
	145	struct dma_attrs *attrs)
	146	{
	147	dma_addr_t dev_addr;
	148
	149	dev_addr = swiotlb_map_page(dev, page, offset, size, dir, attrs);
	150	__dma_map_area(phys_to_virt(dma_to_phys(dev, dev_addr)), size, dir);
	151
	152	return dev_addr;
	153	}
	154
	155
	156	static void __swiotlb_unmap_page(struct device *dev, dma_addr_t dev_addr,
	157	size_t size, enum dma_data_direction dir,
	158	struct dma_attrs *attrs)
	159	{
	160	__dma_unmap_area(phys_to_virt(dma_to_phys(dev, dev_addr)), size, dir);
	161	swiotlb_unmap_page(dev, dev_addr, size, dir, attrs);
	162	}
	163
	164	static int __swiotlb_map_sg_attrs(struct device dev, struct scatterlist sgl,
	165	int nelems, enum dma_data_direction dir,
	166	struct dma_attrs *attrs)
	167	{
	168	struct scatterlist *sg;
	169	int i, ret;
	170
	171	ret = swiotlb_map_sg_attrs(dev, sgl, nelems, dir, attrs);
	172	for_each_sg(sgl, sg, ret, i)
	173	__dma_map_area(phys_to_virt(dma_to_phys(dev, sg->dma_address)),
	174	sg->length, dir);
	175
	176	return ret;
	177	}
	178
	179	static void __swiotlb_unmap_sg_attrs(struct device *dev,
	180	struct scatterlist *sgl, int nelems,
	181	enum dma_data_direction dir,
	182	struct dma_attrs *attrs)
183	{
184	struct scatterlist *sg;
185	int i;
186
187	for_each_sg(sgl, sg, nelems, i)
188	__dma_unmap_area(phys_to_virt(dma_to_phys(dev, sg->dma_address)),
189	sg->length, dir);
190	swiotlb_unmap_sg_attrs(dev, sgl, nelems, dir, attrs);
191	}
192
193	static void __swiotlb_sync_single_for_cpu(struct device *dev,
194	dma_addr_t dev_addr, size_t size,
195	enum dma_data_direction dir)
196	{
197	__dma_unmap_area(phys_to_virt(dma_to_phys(dev, dev_addr)), size, dir);
198	swiotlb_sync_single_for_cpu(dev, dev_addr, size, dir);
199	}
200
201	static void __swiotlb_sync_single_for_device(struct device *dev,
202	dma_addr_t dev_addr, size_t size,
203	enum dma_data_direction dir)
204	{
205	swiotlb_sync_single_for_device(dev, dev_addr, size, dir);
206	__dma_map_area(phys_to_virt(dma_to_phys(dev, dev_addr)), size, dir);
207	}
208
209	static void __swiotlb_sync_sg_for_cpu(struct device *dev,
210	struct scatterlist *sgl, int nelems,
211	enum dma_data_direction dir)
212	{
213	struct scatterlist *sg;
214	int i;
215
216	for_each_sg(sgl, sg, nelems, i)
217	__dma_unmap_area(phys_to_virt(dma_to_phys(dev, sg->dma_address)),
218	sg->length, dir);
219	swiotlb_sync_sg_for_cpu(dev, sgl, nelems, dir);
220	}
221
222	static void __swiotlb_sync_sg_for_device(struct device *dev,
223	struct scatterlist *sgl, int nelems,
224	enum dma_data_direction dir)
225	{
226	struct scatterlist *sg;
227	int i;
228
229	swiotlb_sync_sg_for_device(dev, sgl, nelems, dir);
230	for_each_sg(sgl, sg, nelems, i)
231	__dma_map_area(phys_to_virt(dma_to_phys(dev, sg->dma_address)),
232	sg->length, dir);
233	}
234
6e8d7968 LA	235	/* vma->vm_page_prot must be set appropriately before calling this function */
	236	static int __dma_common_mmap(struct device dev, struct vm_area_struct vma,
	237	void *cpu_addr, dma_addr_t dma_addr, size_t size)
	238	{
	239	int ret = -ENXIO;
	240	unsigned long nr_vma_pages = (vma->vm_end - vma->vm_start) >>
	241	PAGE_SHIFT;
	242	unsigned long nr_pages = PAGE_ALIGN(size) >> PAGE_SHIFT;
	243	unsigned long pfn = dma_to_phys(dev, dma_addr) >> PAGE_SHIFT;
	244	unsigned long off = vma->vm_pgoff;
	245
	246	if (dma_mmap_from_coherent(dev, vma, cpu_addr, size, &ret))
	247	return ret;
	248
	249	if (off < nr_pages && nr_vma_pages <= (nr_pages - off)) {
	250	ret = remap_pfn_range(vma, vma->vm_start,
	251	pfn + off,
	252	vma->vm_end - vma->vm_start,
	253	vma->vm_page_prot);
	254	}
	255
	256	return ret;
	257	}
	258
	259	static int __swiotlb_mmap_noncoherent(struct device *dev,
	260	struct vm_area_struct *vma,
	261	void *cpu_addr, dma_addr_t dma_addr, size_t size,
	262	struct dma_attrs *attrs)
	263	{
214fdbe7	264	vma->vm_page_prot = __get_dma_pgprot(attrs, vma->vm_page_prot, false);
6e8d7968 LA	265	return __dma_common_mmap(dev, vma, cpu_addr, dma_addr, size);
	266	}
	267
	268	static int __swiotlb_mmap_coherent(struct device *dev,
	269	struct vm_area_struct *vma,
	270	void *cpu_addr, dma_addr_t dma_addr, size_t size,
	271	struct dma_attrs *attrs)
	272	{
	273	/* Just use whatever page_prot attributes were specified */
	274	return __dma_common_mmap(dev, vma, cpu_addr, dma_addr, size);
	275	}
	276
7363590d CM	277	struct dma_map_ops noncoherent_swiotlb_dma_ops = {
	278	.alloc = __dma_alloc_noncoherent,
	279	.free = __dma_free_noncoherent,
6e8d7968	280	.mmap = __swiotlb_mmap_noncoherent,
7363590d CM	281	.map_page = __swiotlb_map_page,
	282	.unmap_page = __swiotlb_unmap_page,
	283	.map_sg = __swiotlb_map_sg_attrs,
	284	.unmap_sg = __swiotlb_unmap_sg_attrs,
	285	.sync_single_for_cpu = __swiotlb_sync_single_for_cpu,
	286	.sync_single_for_device = __swiotlb_sync_single_for_device,
	287	.sync_sg_for_cpu = __swiotlb_sync_sg_for_cpu,
	288	.sync_sg_for_device = __swiotlb_sync_sg_for_device,
	289	.dma_supported = swiotlb_dma_supported,
	290	.mapping_error = swiotlb_dma_mapping_error,
	291	};
	292	EXPORT_SYMBOL(noncoherent_swiotlb_dma_ops);
	293
	294	struct dma_map_ops coherent_swiotlb_dma_ops = {
bb10eb7b RH	295	.alloc = __dma_alloc_coherent,
bb10eb7b RH	296	.free = __dma_free_coherent,
6e8d7968	297	.mmap = __swiotlb_mmap_coherent,
09b55412 CM	298	.map_page = swiotlb_map_page,
	299	.unmap_page = swiotlb_unmap_page,
	300	.map_sg = swiotlb_map_sg_attrs,
	301	.unmap_sg = swiotlb_unmap_sg_attrs,
	302	.sync_single_for_cpu = swiotlb_sync_single_for_cpu,
	303	.sync_single_for_device = swiotlb_sync_single_for_device,
	304	.sync_sg_for_cpu = swiotlb_sync_sg_for_cpu,
	305	.sync_sg_for_device = swiotlb_sync_sg_for_device,
	306	.dma_supported = swiotlb_dma_supported,
	307	.mapping_error = swiotlb_dma_mapping_error,
	308	};
7363590d	309	EXPORT_SYMBOL(coherent_swiotlb_dma_ops);
09b55412	310
6ecba8eb CM	311	static int dma_bus_notifier(struct notifier_block *nb,
	312	unsigned long event, void *_dev)
	313	{
	314	struct device *dev = _dev;
	315
	316	if (event != BUS_NOTIFY_ADD_DEVICE)
	317	return NOTIFY_DONE;
	318
	319	if (of_property_read_bool(dev->of_node, "dma-coherent"))
	320	set_dma_ops(dev, &coherent_swiotlb_dma_ops);
	321
	322	return NOTIFY_OK;
	323	}
	324
	325	static struct notifier_block platform_bus_nb = {
	326	.notifier_call = dma_bus_notifier,
	327	};
	328
	329	static struct notifier_block amba_bus_nb = {
	330	.notifier_call = dma_bus_notifier,
	331	};
	332
3690951f CM	333	extern int swiotlb_late_init_with_default_size(size_t default_size);
	334
	335	static int __init swiotlb_late_init(void)
09b55412	336	{
3690951f CM	337	size_t swiotlb_size = min(SZ_64M, MAX_ORDER_NR_PAGES << PAGE_SHIFT);
3690951f CM	338
6ecba8eb CM	339	/*
	340	* These must be registered before of_platform_populate().
	341	*/
	342	bus_register_notifier(&platform_bus_type, &platform_bus_nb);
	343	bus_register_notifier(&amba_bustype, &amba_bus_nb);
	344
c7a4a765	345	dma_ops = &noncoherent_swiotlb_dma_ops;
3690951f CM	346
3690951f CM	347	return swiotlb_late_init_with_default_size(swiotlb_size);
09b55412	348	}
6ecba8eb	349	arch_initcall(swiotlb_late_init);
09b55412 CM	350
	351	#define PREALLOC_DMA_DEBUG_ENTRIES 4096
	352
	353	static int __init dma_debug_do_init(void)
	354	{
	355	dma_debug_init(PREALLOC_DMA_DEBUG_ENTRIES);
	356	return 0;
	357	}
	358	fs_initcall(dma_debug_do_init);