2 * Coherent per-device memory handling.
5 #include <linux/slab.h>
6 #include <linux/kernel.h>
7 #include <linux/module.h>
8 #include <linux/dma-mapping.h>
10 struct dma_coherent_mem
{
12 dma_addr_t device_base
;
13 unsigned long pfn_base
;
16 unsigned long *bitmap
;
20 static bool dma_init_coherent_memory(
21 phys_addr_t phys_addr
, dma_addr_t device_addr
, size_t size
, int flags
,
22 struct dma_coherent_mem
**mem
)
24 struct dma_coherent_mem
*dma_mem
= NULL
;
25 void __iomem
*mem_base
= NULL
;
26 int pages
= size
>> PAGE_SHIFT
;
27 int bitmap_size
= BITS_TO_LONGS(pages
) * sizeof(long);
29 if ((flags
& (DMA_MEMORY_MAP
| DMA_MEMORY_IO
)) == 0)
34 mem_base
= ioremap(phys_addr
, size
);
38 dma_mem
= kzalloc(sizeof(struct dma_coherent_mem
), GFP_KERNEL
);
41 dma_mem
->bitmap
= kzalloc(bitmap_size
, GFP_KERNEL
);
45 dma_mem
->virt_base
= mem_base
;
46 dma_mem
->device_base
= device_addr
;
47 dma_mem
->pfn_base
= PFN_DOWN(phys_addr
);
48 dma_mem
->size
= pages
;
49 dma_mem
->flags
= flags
;
50 spin_lock_init(&dma_mem
->spinlock
);
62 static void dma_release_coherent_memory(struct dma_coherent_mem
*mem
)
66 iounmap(mem
->virt_base
);
71 static int dma_assign_coherent_memory(struct device
*dev
,
72 struct dma_coherent_mem
*mem
)
78 /* FIXME: this routine just ignores DMA_MEMORY_INCLUDES_CHILDREN */
83 int dma_declare_coherent_memory(struct device
*dev
, phys_addr_t phys_addr
,
84 dma_addr_t device_addr
, size_t size
, int flags
)
86 struct dma_coherent_mem
*mem
;
88 if (!dma_init_coherent_memory(phys_addr
, device_addr
, size
, flags
,
92 if (dma_assign_coherent_memory(dev
, mem
) == 0)
93 return flags
& DMA_MEMORY_MAP
? DMA_MEMORY_MAP
: DMA_MEMORY_IO
;
95 dma_release_coherent_memory(mem
);
98 EXPORT_SYMBOL(dma_declare_coherent_memory
);
100 void dma_release_declared_memory(struct device
*dev
)
102 struct dma_coherent_mem
*mem
= dev
->dma_mem
;
106 dma_release_coherent_memory(mem
);
109 EXPORT_SYMBOL(dma_release_declared_memory
);
111 void *dma_mark_declared_memory_occupied(struct device
*dev
,
112 dma_addr_t device_addr
, size_t size
)
114 struct dma_coherent_mem
*mem
= dev
->dma_mem
;
118 size
+= device_addr
& ~PAGE_MASK
;
121 return ERR_PTR(-EINVAL
);
123 spin_lock_irqsave(&mem
->spinlock
, flags
);
124 pos
= (device_addr
- mem
->device_base
) >> PAGE_SHIFT
;
125 err
= bitmap_allocate_region(mem
->bitmap
, pos
, get_order(size
));
126 spin_unlock_irqrestore(&mem
->spinlock
, flags
);
130 return mem
->virt_base
+ (pos
<< PAGE_SHIFT
);
132 EXPORT_SYMBOL(dma_mark_declared_memory_occupied
);
135 * dma_alloc_from_coherent() - try to allocate memory from the per-device coherent area
137 * @dev: device from which we allocate memory
138 * @size: size of requested memory area
139 * @dma_handle: This will be filled with the correct dma handle
140 * @ret: This pointer will be filled with the virtual address
143 * This function should be only called from per-arch dma_alloc_coherent()
144 * to support allocation from per-device coherent memory pools.
146 * Returns 0 if dma_alloc_coherent should continue with allocating from
147 * generic memory areas, or !0 if dma_alloc_coherent should return @ret.
149 int dma_alloc_from_coherent(struct device
*dev
, ssize_t size
,
150 dma_addr_t
*dma_handle
, void **ret
)
152 struct dma_coherent_mem
*mem
;
153 int order
= get_order(size
);
164 spin_lock_irqsave(&mem
->spinlock
, flags
);
166 if (unlikely(size
> (mem
->size
<< PAGE_SHIFT
)))
169 pageno
= bitmap_find_free_region(mem
->bitmap
, mem
->size
, order
);
170 if (unlikely(pageno
< 0))
174 * Memory was found in the per-device area.
176 *dma_handle
= mem
->device_base
+ (pageno
<< PAGE_SHIFT
);
177 *ret
= mem
->virt_base
+ (pageno
<< PAGE_SHIFT
);
178 memset(*ret
, 0, size
);
179 spin_unlock_irqrestore(&mem
->spinlock
, flags
);
184 spin_unlock_irqrestore(&mem
->spinlock
, flags
);
186 * In the case where the allocation can not be satisfied from the
187 * per-device area, try to fall back to generic memory if the
188 * constraints allow it.
190 return mem
->flags
& DMA_MEMORY_EXCLUSIVE
;
192 EXPORT_SYMBOL(dma_alloc_from_coherent
);
195 * dma_release_from_coherent() - try to free the memory allocated from per-device coherent memory pool
196 * @dev: device from which the memory was allocated
197 * @order: the order of pages allocated
198 * @vaddr: virtual address of allocated pages
200 * This checks whether the memory was allocated from the per-device
201 * coherent memory pool and if so, releases that memory.
203 * Returns 1 if we correctly released the memory, or 0 if
204 * dma_release_coherent() should proceed with releasing memory from
207 int dma_release_from_coherent(struct device
*dev
, int order
, void *vaddr
)
209 struct dma_coherent_mem
*mem
= dev
? dev
->dma_mem
: NULL
;
211 if (mem
&& vaddr
>= mem
->virt_base
&& vaddr
<
212 (mem
->virt_base
+ (mem
->size
<< PAGE_SHIFT
))) {
213 int page
= (vaddr
- mem
->virt_base
) >> PAGE_SHIFT
;
216 spin_lock_irqsave(&mem
->spinlock
, flags
);
217 bitmap_release_region(mem
->bitmap
, page
, order
);
218 spin_unlock_irqrestore(&mem
->spinlock
, flags
);
223 EXPORT_SYMBOL(dma_release_from_coherent
);
226 * dma_mmap_from_coherent() - try to mmap the memory allocated from
227 * per-device coherent memory pool to userspace
228 * @dev: device from which the memory was allocated
229 * @vma: vm_area for the userspace memory
230 * @vaddr: cpu address returned by dma_alloc_from_coherent
231 * @size: size of the memory buffer allocated by dma_alloc_from_coherent
232 * @ret: result from remap_pfn_range()
234 * This checks whether the memory was allocated from the per-device
235 * coherent memory pool and if so, maps that memory to the provided vma.
237 * Returns 1 if we correctly mapped the memory, or 0 if the caller should
238 * proceed with mapping memory from generic pools.
240 int dma_mmap_from_coherent(struct device
*dev
, struct vm_area_struct
*vma
,
241 void *vaddr
, size_t size
, int *ret
)
243 struct dma_coherent_mem
*mem
= dev
? dev
->dma_mem
: NULL
;
245 if (mem
&& vaddr
>= mem
->virt_base
&& vaddr
+ size
<=
246 (mem
->virt_base
+ (mem
->size
<< PAGE_SHIFT
))) {
247 unsigned long off
= vma
->vm_pgoff
;
248 int start
= (vaddr
- mem
->virt_base
) >> PAGE_SHIFT
;
249 int user_count
= (vma
->vm_end
- vma
->vm_start
) >> PAGE_SHIFT
;
250 int count
= size
>> PAGE_SHIFT
;
253 if (off
< count
&& user_count
<= count
- off
) {
254 unsigned long pfn
= mem
->pfn_base
+ start
+ off
;
255 *ret
= remap_pfn_range(vma
, vma
->vm_start
, pfn
,
256 user_count
<< PAGE_SHIFT
,
263 EXPORT_SYMBOL(dma_mmap_from_coherent
);
266 * Support for reserved memory regions defined in device tree
268 #ifdef CONFIG_OF_RESERVED_MEM
269 #include <linux/of.h>
270 #include <linux/of_fdt.h>
271 #include <linux/of_reserved_mem.h>
273 static int rmem_dma_device_init(struct reserved_mem
*rmem
, struct device
*dev
)
275 struct dma_coherent_mem
*mem
= rmem
->priv
;
278 !dma_init_coherent_memory(rmem
->base
, rmem
->base
, rmem
->size
,
279 DMA_MEMORY_MAP
| DMA_MEMORY_EXCLUSIVE
,
281 pr_err("Reserved memory: failed to init DMA memory pool at %pa, size %ld MiB\n",
282 &rmem
->base
, (unsigned long)rmem
->size
/ SZ_1M
);
286 dma_assign_coherent_memory(dev
, mem
);
290 static void rmem_dma_device_release(struct reserved_mem
*rmem
,
296 static const struct reserved_mem_ops rmem_dma_ops
= {
297 .device_init
= rmem_dma_device_init
,
298 .device_release
= rmem_dma_device_release
,
301 static int __init
rmem_dma_setup(struct reserved_mem
*rmem
)
303 unsigned long node
= rmem
->fdt_node
;
305 if (of_get_flat_dt_prop(node
, "reusable", NULL
))
309 if (!of_get_flat_dt_prop(node
, "no-map", NULL
)) {
310 pr_err("Reserved memory: regions without no-map are not yet supported\n");
315 rmem
->ops
= &rmem_dma_ops
;
316 pr_info("Reserved memory: created DMA memory pool at %pa, size %ld MiB\n",
317 &rmem
->base
, (unsigned long)rmem
->size
/ SZ_1M
);
320 RESERVEDMEM_OF_DECLARE(dma
, "shared-dma-pool", rmem_dma_setup
);