2 * linux/arch/arm/mm/dma-mapping.c
4 * Copyright (C) 2000-2004 Russell King
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License version 2 as
8 * published by the Free Software Foundation.
10 * DMA uncached mapping support.
12 #include <linux/module.h>
14 #include <linux/gfp.h>
15 #include <linux/errno.h>
16 #include <linux/list.h>
17 #include <linux/init.h>
18 #include <linux/device.h>
19 #include <linux/dma-mapping.h>
20 #include <linux/highmem.h>
21 #include <linux/slab.h>
23 #include <asm/memory.h>
24 #include <asm/highmem.h>
25 #include <asm/cacheflush.h>
26 #include <asm/tlbflush.h>
27 #include <asm/sizes.h>
28 #include <asm/mach/arch.h>
32 static u64
get_coherent_dma_mask(struct device
*dev
)
34 u64 mask
= (u64
)arm_dma_limit
;
37 mask
= dev
->coherent_dma_mask
;
40 * Sanity check the DMA mask - it must be non-zero, and
41 * must be able to be satisfied by a DMA allocation.
44 dev_warn(dev
, "coherent DMA mask is unset\n");
48 if ((~mask
) & (u64
)arm_dma_limit
) {
49 dev_warn(dev
, "coherent DMA mask %#llx is smaller "
50 "than system GFP_DMA mask %#llx\n",
51 mask
, (u64
)arm_dma_limit
);
60 * Allocate a DMA buffer for 'dev' of size 'size' using the
61 * specified gfp mask. Note that 'size' must be page aligned.
63 static struct page
*__dma_alloc_buffer(struct device
*dev
, size_t size
, gfp_t gfp
)
65 unsigned long order
= get_order(size
);
66 struct page
*page
, *p
, *e
;
68 u64 mask
= get_coherent_dma_mask(dev
);
70 #ifdef CONFIG_DMA_API_DEBUG
71 u64 limit
= (mask
+ 1) & ~mask
;
72 if (limit
&& size
>= limit
) {
73 dev_warn(dev
, "coherent allocation too big (requested %#x mask %#llx)\n",
82 if (mask
< 0xffffffffULL
)
85 page
= alloc_pages(gfp
, order
);
90 * Now split the huge page and free the excess pages
92 split_page(page
, order
);
93 for (p
= page
+ (size
>> PAGE_SHIFT
), e
= page
+ (1 << order
); p
< e
; p
++)
97 * Ensure that the allocated pages are zeroed, and that any data
98 * lurking in the kernel direct-mapped region is invalidated.
100 ptr
= page_address(page
);
101 memset(ptr
, 0, size
);
102 dmac_flush_range(ptr
, ptr
+ size
);
103 outer_flush_range(__pa(ptr
), __pa(ptr
) + size
);
109 * Free a DMA buffer. 'size' must be page aligned.
111 static void __dma_free_buffer(struct page
*page
, size_t size
)
113 struct page
*e
= page
+ (size
>> PAGE_SHIFT
);
123 #define CONSISTENT_OFFSET(x) (((unsigned long)(x) - consistent_base) >> PAGE_SHIFT)
124 #define CONSISTENT_PTE_INDEX(x) (((unsigned long)(x) - consistent_base) >> PMD_SHIFT)
127 * These are the page tables (2MB each) covering uncached, DMA consistent allocations
129 static pte_t
**consistent_pte
;
131 #define DEFAULT_CONSISTENT_DMA_SIZE SZ_2M
133 unsigned long consistent_base
= CONSISTENT_END
- DEFAULT_CONSISTENT_DMA_SIZE
;
135 void __init
init_consistent_dma_size(unsigned long size
)
137 unsigned long base
= CONSISTENT_END
- ALIGN(size
, SZ_2M
);
139 BUG_ON(consistent_pte
); /* Check we're called before DMA region init */
140 BUG_ON(base
< VMALLOC_END
);
142 /* Grow region to accommodate specified size */
143 if (base
< consistent_base
)
144 consistent_base
= base
;
147 #include "vmregion.h"
149 static struct arm_vmregion_head consistent_head
= {
150 .vm_lock
= __SPIN_LOCK_UNLOCKED(&consistent_head
.vm_lock
),
151 .vm_list
= LIST_HEAD_INIT(consistent_head
.vm_list
),
152 .vm_end
= CONSISTENT_END
,
155 #ifdef CONFIG_HUGETLB_PAGE
156 #error ARM Coherent DMA allocator does not (yet) support huge TLB
160 * Initialise the consistent memory allocation.
162 static int __init
consistent_init(void)
170 unsigned long base
= consistent_base
;
171 unsigned long num_ptes
= (CONSISTENT_END
- base
) >> PMD_SHIFT
;
173 consistent_pte
= kmalloc(num_ptes
* sizeof(pte_t
), GFP_KERNEL
);
174 if (!consistent_pte
) {
175 pr_err("%s: no memory\n", __func__
);
179 pr_debug("DMA memory: 0x%08lx - 0x%08lx:\n", base
, CONSISTENT_END
);
180 consistent_head
.vm_start
= base
;
183 pgd
= pgd_offset(&init_mm
, base
);
185 pud
= pud_alloc(&init_mm
, pgd
, base
);
187 pr_err("%s: no pud tables\n", __func__
);
192 pmd
= pmd_alloc(&init_mm
, pud
, base
);
194 pr_err("%s: no pmd tables\n", __func__
);
198 WARN_ON(!pmd_none(*pmd
));
200 pte
= pte_alloc_kernel(pmd
, base
);
202 pr_err("%s: no pte tables\n", __func__
);
207 consistent_pte
[i
++] = pte
;
209 } while (base
< CONSISTENT_END
);
214 core_initcall(consistent_init
);
217 __dma_alloc_remap(struct page
*page
, size_t size
, gfp_t gfp
, pgprot_t prot
,
220 struct arm_vmregion
*c
;
224 if (!consistent_pte
) {
225 pr_err("%s: not initialised\n", __func__
);
231 * Align the virtual region allocation - maximum alignment is
232 * a section size, minimum is a page size. This helps reduce
233 * fragmentation of the DMA space, and also prevents allocations
234 * smaller than a section from crossing a section boundary.
237 if (bit
> SECTION_SHIFT
)
242 * Allocate a virtual address in the consistent mapping region.
244 c
= arm_vmregion_alloc(&consistent_head
, align
, size
,
245 gfp
& ~(__GFP_DMA
| __GFP_HIGHMEM
), caller
);
248 int idx
= CONSISTENT_PTE_INDEX(c
->vm_start
);
249 u32 off
= CONSISTENT_OFFSET(c
->vm_start
) & (PTRS_PER_PTE
-1);
251 pte
= consistent_pte
[idx
] + off
;
255 BUG_ON(!pte_none(*pte
));
257 set_pte_ext(pte
, mk_pte(page
, prot
), 0);
261 if (off
>= PTRS_PER_PTE
) {
263 pte
= consistent_pte
[++idx
];
265 } while (size
-= PAGE_SIZE
);
269 return (void *)c
->vm_start
;
274 static void __dma_free_remap(void *cpu_addr
, size_t size
)
276 struct arm_vmregion
*c
;
282 c
= arm_vmregion_find_remove(&consistent_head
, (unsigned long)cpu_addr
);
284 pr_err("%s: trying to free invalid coherent area: %p\n",
290 if ((c
->vm_end
- c
->vm_start
) != size
) {
291 pr_err("%s: freeing wrong coherent size (%ld != %d)\n",
292 __func__
, c
->vm_end
- c
->vm_start
, size
);
294 size
= c
->vm_end
- c
->vm_start
;
297 idx
= CONSISTENT_PTE_INDEX(c
->vm_start
);
298 off
= CONSISTENT_OFFSET(c
->vm_start
) & (PTRS_PER_PTE
-1);
299 ptep
= consistent_pte
[idx
] + off
;
302 pte_t pte
= ptep_get_and_clear(&init_mm
, addr
, ptep
);
307 if (off
>= PTRS_PER_PTE
) {
309 ptep
= consistent_pte
[++idx
];
312 if (pte_none(pte
) || !pte_present(pte
))
313 pr_crit("%s: bad page in kernel page table\n",
315 } while (size
-= PAGE_SIZE
);
317 flush_tlb_kernel_range(c
->vm_start
, c
->vm_end
);
319 arm_vmregion_free(&consistent_head
, c
);
322 #else /* !CONFIG_MMU */
324 #define __dma_alloc_remap(page, size, gfp, prot, c) page_address(page)
325 #define __dma_free_remap(addr, size) do { } while (0)
327 #endif /* CONFIG_MMU */
330 __dma_alloc(struct device
*dev
, size_t size
, dma_addr_t
*handle
, gfp_t gfp
,
331 pgprot_t prot
, const void *caller
)
337 * Following is a work-around (a.k.a. hack) to prevent pages
338 * with __GFP_COMP being passed to split_page() which cannot
339 * handle them. The real problem is that this flag probably
340 * should be 0 on ARM as it is not supported on this
341 * platform; see CONFIG_HUGETLBFS.
343 gfp
&= ~(__GFP_COMP
);
345 *handle
= DMA_ERROR_CODE
;
346 size
= PAGE_ALIGN(size
);
348 page
= __dma_alloc_buffer(dev
, size
, gfp
);
352 if (!arch_is_coherent())
353 addr
= __dma_alloc_remap(page
, size
, gfp
, prot
, caller
);
355 addr
= page_address(page
);
358 *handle
= pfn_to_dma(dev
, page_to_pfn(page
));
360 __dma_free_buffer(page
, size
);
366 * Allocate DMA-coherent memory space and return both the kernel remapped
367 * virtual and bus address for that space.
370 dma_alloc_coherent(struct device
*dev
, size_t size
, dma_addr_t
*handle
, gfp_t gfp
)
374 if (dma_alloc_from_coherent(dev
, size
, handle
, &memory
))
377 return __dma_alloc(dev
, size
, handle
, gfp
,
378 pgprot_dmacoherent(pgprot_kernel
),
379 __builtin_return_address(0));
381 EXPORT_SYMBOL(dma_alloc_coherent
);
384 * Allocate a writecombining region, in much the same way as
385 * dma_alloc_coherent above.
388 dma_alloc_writecombine(struct device
*dev
, size_t size
, dma_addr_t
*handle
, gfp_t gfp
)
390 return __dma_alloc(dev
, size
, handle
, gfp
,
391 pgprot_writecombine(pgprot_kernel
),
392 __builtin_return_address(0));
394 EXPORT_SYMBOL(dma_alloc_writecombine
);
396 static int dma_mmap(struct device
*dev
, struct vm_area_struct
*vma
,
397 void *cpu_addr
, dma_addr_t dma_addr
, size_t size
)
401 unsigned long user_size
, kern_size
;
402 struct arm_vmregion
*c
;
404 if (dma_mmap_from_coherent(dev
, vma
, cpu_addr
, size
, &ret
))
407 user_size
= (vma
->vm_end
- vma
->vm_start
) >> PAGE_SHIFT
;
409 c
= arm_vmregion_find(&consistent_head
, (unsigned long)cpu_addr
);
411 unsigned long off
= vma
->vm_pgoff
;
413 kern_size
= (c
->vm_end
- c
->vm_start
) >> PAGE_SHIFT
;
415 if (off
< kern_size
&&
416 user_size
<= (kern_size
- off
)) {
417 ret
= remap_pfn_range(vma
, vma
->vm_start
,
418 page_to_pfn(c
->vm_pages
) + off
,
419 user_size
<< PAGE_SHIFT
,
423 #endif /* CONFIG_MMU */
428 int dma_mmap_coherent(struct device
*dev
, struct vm_area_struct
*vma
,
429 void *cpu_addr
, dma_addr_t dma_addr
, size_t size
)
431 vma
->vm_page_prot
= pgprot_dmacoherent(vma
->vm_page_prot
);
432 return dma_mmap(dev
, vma
, cpu_addr
, dma_addr
, size
);
434 EXPORT_SYMBOL(dma_mmap_coherent
);
436 int dma_mmap_writecombine(struct device
*dev
, struct vm_area_struct
*vma
,
437 void *cpu_addr
, dma_addr_t dma_addr
, size_t size
)
439 vma
->vm_page_prot
= pgprot_writecombine(vma
->vm_page_prot
);
440 return dma_mmap(dev
, vma
, cpu_addr
, dma_addr
, size
);
442 EXPORT_SYMBOL(dma_mmap_writecombine
);
445 * free a page as defined by the above mapping.
446 * Must not be called with IRQs disabled.
448 void dma_free_coherent(struct device
*dev
, size_t size
, void *cpu_addr
, dma_addr_t handle
)
450 WARN_ON(irqs_disabled());
452 if (dma_release_from_coherent(dev
, get_order(size
), cpu_addr
))
455 size
= PAGE_ALIGN(size
);
457 if (!arch_is_coherent())
458 __dma_free_remap(cpu_addr
, size
);
460 __dma_free_buffer(pfn_to_page(dma_to_pfn(dev
, handle
)), size
);
462 EXPORT_SYMBOL(dma_free_coherent
);
465 * Make an area consistent for devices.
466 * Note: Drivers should NOT use this function directly, as it will break
467 * platforms with CONFIG_DMABOUNCE.
468 * Use the driver DMA support - see dma-mapping.h (dma_sync_*)
470 void ___dma_single_cpu_to_dev(const void *kaddr
, size_t size
,
471 enum dma_data_direction dir
)
475 BUG_ON(!virt_addr_valid(kaddr
) || !virt_addr_valid(kaddr
+ size
- 1));
477 dmac_map_area(kaddr
, size
, dir
);
480 if (dir
== DMA_FROM_DEVICE
) {
481 outer_inv_range(paddr
, paddr
+ size
);
483 outer_clean_range(paddr
, paddr
+ size
);
485 /* FIXME: non-speculating: flush on bidirectional mappings? */
487 EXPORT_SYMBOL(___dma_single_cpu_to_dev
);
489 void ___dma_single_dev_to_cpu(const void *kaddr
, size_t size
,
490 enum dma_data_direction dir
)
492 BUG_ON(!virt_addr_valid(kaddr
) || !virt_addr_valid(kaddr
+ size
- 1));
494 /* FIXME: non-speculating: not required */
495 /* don't bother invalidating if DMA to device */
496 if (dir
!= DMA_TO_DEVICE
) {
497 unsigned long paddr
= __pa(kaddr
);
498 outer_inv_range(paddr
, paddr
+ size
);
501 dmac_unmap_area(kaddr
, size
, dir
);
503 EXPORT_SYMBOL(___dma_single_dev_to_cpu
);
505 static void dma_cache_maint_page(struct page
*page
, unsigned long offset
,
506 size_t size
, enum dma_data_direction dir
,
507 void (*op
)(const void *, size_t, int))
510 * A single sg entry may refer to multiple physically contiguous
511 * pages. But we still need to process highmem pages individually.
512 * If highmem is not configured then the bulk of this loop gets
520 if (PageHighMem(page
)) {
521 if (len
+ offset
> PAGE_SIZE
) {
522 if (offset
>= PAGE_SIZE
) {
523 page
+= offset
/ PAGE_SIZE
;
526 len
= PAGE_SIZE
- offset
;
528 vaddr
= kmap_high_get(page
);
533 } else if (cache_is_vipt()) {
534 /* unmapped pages might still be cached */
535 vaddr
= kmap_atomic(page
);
536 op(vaddr
+ offset
, len
, dir
);
537 kunmap_atomic(vaddr
);
540 vaddr
= page_address(page
) + offset
;
549 void ___dma_page_cpu_to_dev(struct page
*page
, unsigned long off
,
550 size_t size
, enum dma_data_direction dir
)
554 dma_cache_maint_page(page
, off
, size
, dir
, dmac_map_area
);
556 paddr
= page_to_phys(page
) + off
;
557 if (dir
== DMA_FROM_DEVICE
) {
558 outer_inv_range(paddr
, paddr
+ size
);
560 outer_clean_range(paddr
, paddr
+ size
);
562 /* FIXME: non-speculating: flush on bidirectional mappings? */
564 EXPORT_SYMBOL(___dma_page_cpu_to_dev
);
566 void ___dma_page_dev_to_cpu(struct page
*page
, unsigned long off
,
567 size_t size
, enum dma_data_direction dir
)
569 unsigned long paddr
= page_to_phys(page
) + off
;
571 /* FIXME: non-speculating: not required */
572 /* don't bother invalidating if DMA to device */
573 if (dir
!= DMA_TO_DEVICE
)
574 outer_inv_range(paddr
, paddr
+ size
);
576 dma_cache_maint_page(page
, off
, size
, dir
, dmac_unmap_area
);
579 * Mark the D-cache clean for this page to avoid extra flushing.
581 if (dir
!= DMA_TO_DEVICE
&& off
== 0 && size
>= PAGE_SIZE
)
582 set_bit(PG_dcache_clean
, &page
->flags
);
584 EXPORT_SYMBOL(___dma_page_dev_to_cpu
);
587 * dma_map_sg - map a set of SG buffers for streaming mode DMA
588 * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices
589 * @sg: list of buffers
590 * @nents: number of buffers to map
591 * @dir: DMA transfer direction
593 * Map a set of buffers described by scatterlist in streaming mode for DMA.
594 * This is the scatter-gather version of the dma_map_single interface.
595 * Here the scatter gather list elements are each tagged with the
596 * appropriate dma address and length. They are obtained via
597 * sg_dma_{address,length}.
599 * Device ownership issues as mentioned for dma_map_single are the same
602 int dma_map_sg(struct device
*dev
, struct scatterlist
*sg
, int nents
,
603 enum dma_data_direction dir
)
605 struct scatterlist
*s
;
608 BUG_ON(!valid_dma_direction(dir
));
610 for_each_sg(sg
, s
, nents
, i
) {
611 s
->dma_address
= __dma_map_page(dev
, sg_page(s
), s
->offset
,
613 if (dma_mapping_error(dev
, s
->dma_address
))
616 debug_dma_map_sg(dev
, sg
, nents
, nents
, dir
);
620 for_each_sg(sg
, s
, i
, j
)
621 __dma_unmap_page(dev
, sg_dma_address(s
), sg_dma_len(s
), dir
);
624 EXPORT_SYMBOL(dma_map_sg
);
627 * dma_unmap_sg - unmap a set of SG buffers mapped by dma_map_sg
628 * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices
629 * @sg: list of buffers
630 * @nents: number of buffers to unmap (same as was passed to dma_map_sg)
631 * @dir: DMA transfer direction (same as was passed to dma_map_sg)
633 * Unmap a set of streaming mode DMA translations. Again, CPU access
634 * rules concerning calls here are the same as for dma_unmap_single().
636 void dma_unmap_sg(struct device
*dev
, struct scatterlist
*sg
, int nents
,
637 enum dma_data_direction dir
)
639 struct scatterlist
*s
;
642 debug_dma_unmap_sg(dev
, sg
, nents
, dir
);
644 for_each_sg(sg
, s
, nents
, i
)
645 __dma_unmap_page(dev
, sg_dma_address(s
), sg_dma_len(s
), dir
);
647 EXPORT_SYMBOL(dma_unmap_sg
);
650 * dma_sync_sg_for_cpu
651 * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices
652 * @sg: list of buffers
653 * @nents: number of buffers to map (returned from dma_map_sg)
654 * @dir: DMA transfer direction (same as was passed to dma_map_sg)
656 void dma_sync_sg_for_cpu(struct device
*dev
, struct scatterlist
*sg
,
657 int nents
, enum dma_data_direction dir
)
659 struct scatterlist
*s
;
662 for_each_sg(sg
, s
, nents
, i
) {
663 if (!dmabounce_sync_for_cpu(dev
, sg_dma_address(s
), 0,
667 __dma_page_dev_to_cpu(sg_page(s
), s
->offset
,
671 debug_dma_sync_sg_for_cpu(dev
, sg
, nents
, dir
);
673 EXPORT_SYMBOL(dma_sync_sg_for_cpu
);
676 * dma_sync_sg_for_device
677 * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices
678 * @sg: list of buffers
679 * @nents: number of buffers to map (returned from dma_map_sg)
680 * @dir: DMA transfer direction (same as was passed to dma_map_sg)
682 void dma_sync_sg_for_device(struct device
*dev
, struct scatterlist
*sg
,
683 int nents
, enum dma_data_direction dir
)
685 struct scatterlist
*s
;
688 for_each_sg(sg
, s
, nents
, i
) {
689 if (!dmabounce_sync_for_device(dev
, sg_dma_address(s
), 0,
693 __dma_page_cpu_to_dev(sg_page(s
), s
->offset
,
697 debug_dma_sync_sg_for_device(dev
, sg
, nents
, dir
);
699 EXPORT_SYMBOL(dma_sync_sg_for_device
);
702 * Return whether the given device DMA address mask can be supported
703 * properly. For example, if your device can only drive the low 24-bits
704 * during bus mastering, then you would pass 0x00ffffff as the mask
707 int dma_supported(struct device
*dev
, u64 mask
)
709 if (mask
< (u64
)arm_dma_limit
)
713 EXPORT_SYMBOL(dma_supported
);
715 int dma_set_mask(struct device
*dev
, u64 dma_mask
)
717 if (!dev
->dma_mask
|| !dma_supported(dev
, dma_mask
))
720 #ifndef CONFIG_DMABOUNCE
721 *dev
->dma_mask
= dma_mask
;
726 EXPORT_SYMBOL(dma_set_mask
);
728 #define PREALLOC_DMA_DEBUG_ENTRIES 4096
730 static int __init
dma_debug_do_init(void)
733 arm_vmregion_create_proc("dma-mappings", &consistent_head
);
735 dma_debug_init(PREALLOC_DMA_DEBUG_ENTRIES
);
738 fs_initcall(dma_debug_do_init
);