Commit | Line | Data |
---|---|---|
ee7e5516 DB |
1 | /* |
2 | * Coherent per-device memory handling. | |
3 | * Borrowed from i386 | |
4 | */ | |
5a0e3ad6 | 5 | #include <linux/slab.h> |
ee7e5516 | 6 | #include <linux/kernel.h> |
08a999ce | 7 | #include <linux/module.h> |
ee7e5516 DB |
8 | #include <linux/dma-mapping.h> |
9 | ||
10 | struct dma_coherent_mem { | |
11 | void *virt_base; | |
ed1d218c | 12 | dma_addr_t device_base; |
88a984ba | 13 | unsigned long pfn_base; |
ee7e5516 DB |
14 | int size; |
15 | int flags; | |
16 | unsigned long *bitmap; | |
7bfa5ab6 | 17 | spinlock_t spinlock; |
ee7e5516 DB |
18 | }; |
19 | ||
7bfa5ab6 MS |
20 | static int dma_init_coherent_memory(phys_addr_t phys_addr, dma_addr_t device_addr, |
21 | size_t size, int flags, | |
22 | struct dma_coherent_mem **mem) | |
ee7e5516 | 23 | { |
7bfa5ab6 | 24 | struct dma_coherent_mem *dma_mem = NULL; |
ee7e5516 DB |
25 | void __iomem *mem_base = NULL; |
26 | int pages = size >> PAGE_SHIFT; | |
27 | int bitmap_size = BITS_TO_LONGS(pages) * sizeof(long); | |
28 | ||
29 | if ((flags & (DMA_MEMORY_MAP | DMA_MEMORY_IO)) == 0) | |
30 | goto out; | |
31 | if (!size) | |
32 | goto out; | |
ee7e5516 | 33 | |
88a984ba | 34 | mem_base = ioremap(phys_addr, size); |
ee7e5516 DB |
35 | if (!mem_base) |
36 | goto out; | |
37 | ||
7bfa5ab6 MS |
38 | dma_mem = kzalloc(sizeof(struct dma_coherent_mem), GFP_KERNEL); |
39 | if (!dma_mem) | |
ee7e5516 | 40 | goto out; |
7bfa5ab6 MS |
41 | dma_mem->bitmap = kzalloc(bitmap_size, GFP_KERNEL); |
42 | if (!dma_mem->bitmap) | |
43 | goto out; | |
44 | ||
45 | dma_mem->virt_base = mem_base; | |
46 | dma_mem->device_base = device_addr; | |
47 | dma_mem->pfn_base = PFN_DOWN(phys_addr); | |
48 | dma_mem->size = pages; | |
49 | dma_mem->flags = flags; | |
50 | spin_lock_init(&dma_mem->spinlock); | |
ee7e5516 | 51 | |
7bfa5ab6 | 52 | *mem = dma_mem; |
ee7e5516 DB |
53 | |
54 | if (flags & DMA_MEMORY_MAP) | |
55 | return DMA_MEMORY_MAP; | |
56 | ||
57 | return DMA_MEMORY_IO; | |
58 | ||
7bfa5ab6 MS |
59 | out: |
60 | kfree(dma_mem); | |
ee7e5516 DB |
61 | if (mem_base) |
62 | iounmap(mem_base); | |
63 | return 0; | |
64 | } | |
7bfa5ab6 MS |
65 | |
66 | static void dma_release_coherent_memory(struct dma_coherent_mem *mem) | |
67 | { | |
68 | if (!mem) | |
69 | return; | |
70 | iounmap(mem->virt_base); | |
71 | kfree(mem->bitmap); | |
72 | kfree(mem); | |
73 | } | |
74 | ||
75 | static int dma_assign_coherent_memory(struct device *dev, | |
76 | struct dma_coherent_mem *mem) | |
77 | { | |
78 | if (dev->dma_mem) | |
79 | return -EBUSY; | |
80 | ||
81 | dev->dma_mem = mem; | |
82 | /* FIXME: this routine just ignores DMA_MEMORY_INCLUDES_CHILDREN */ | |
83 | ||
84 | return 0; | |
85 | } | |
86 | ||
87 | int dma_declare_coherent_memory(struct device *dev, phys_addr_t phys_addr, | |
88 | dma_addr_t device_addr, size_t size, int flags) | |
89 | { | |
90 | struct dma_coherent_mem *mem; | |
91 | int ret; | |
92 | ||
93 | ret = dma_init_coherent_memory(phys_addr, device_addr, size, flags, | |
94 | &mem); | |
95 | if (ret == 0) | |
96 | return 0; | |
97 | ||
98 | if (dma_assign_coherent_memory(dev, mem) == 0) | |
99 | return ret; | |
100 | ||
101 | dma_release_coherent_memory(mem); | |
102 | return 0; | |
103 | } | |
ee7e5516 DB |
104 | EXPORT_SYMBOL(dma_declare_coherent_memory); |
105 | ||
106 | void dma_release_declared_memory(struct device *dev) | |
107 | { | |
108 | struct dma_coherent_mem *mem = dev->dma_mem; | |
109 | ||
110 | if (!mem) | |
111 | return; | |
7bfa5ab6 | 112 | dma_release_coherent_memory(mem); |
ee7e5516 | 113 | dev->dma_mem = NULL; |
ee7e5516 DB |
114 | } |
115 | EXPORT_SYMBOL(dma_release_declared_memory); | |
116 | ||
117 | void *dma_mark_declared_memory_occupied(struct device *dev, | |
118 | dma_addr_t device_addr, size_t size) | |
119 | { | |
120 | struct dma_coherent_mem *mem = dev->dma_mem; | |
7bfa5ab6 | 121 | unsigned long flags; |
ee7e5516 | 122 | int pos, err; |
ee7e5516 | 123 | |
d2dc1f4a | 124 | size += device_addr & ~PAGE_MASK; |
ee7e5516 DB |
125 | |
126 | if (!mem) | |
127 | return ERR_PTR(-EINVAL); | |
128 | ||
7bfa5ab6 | 129 | spin_lock_irqsave(&mem->spinlock, flags); |
ee7e5516 | 130 | pos = (device_addr - mem->device_base) >> PAGE_SHIFT; |
d2dc1f4a | 131 | err = bitmap_allocate_region(mem->bitmap, pos, get_order(size)); |
7bfa5ab6 MS |
132 | spin_unlock_irqrestore(&mem->spinlock, flags); |
133 | ||
ee7e5516 DB |
134 | if (err != 0) |
135 | return ERR_PTR(err); | |
136 | return mem->virt_base + (pos << PAGE_SHIFT); | |
137 | } | |
138 | EXPORT_SYMBOL(dma_mark_declared_memory_occupied); | |
139 | ||
b6d4f7e3 | 140 | /** |
cb3952bf | 141 | * dma_alloc_from_coherent() - try to allocate memory from the per-device coherent area |
b6d4f7e3 DB |
142 | * |
143 | * @dev: device from which we allocate memory | |
144 | * @size: size of requested memory area | |
145 | * @dma_handle: This will be filled with the correct dma handle | |
146 | * @ret: This pointer will be filled with the virtual address | |
0609697e | 147 | * to allocated area. |
b6d4f7e3 | 148 | * |
cb3952bf | 149 | * This function should be only called from per-arch dma_alloc_coherent() |
b6d4f7e3 DB |
150 | * to support allocation from per-device coherent memory pools. |
151 | * | |
152 | * Returns 0 if dma_alloc_coherent should continue with allocating from | |
cb3952bf | 153 | * generic memory areas, or !0 if dma_alloc_coherent should return @ret. |
b6d4f7e3 | 154 | */ |
ee7e5516 DB |
155 | int dma_alloc_from_coherent(struct device *dev, ssize_t size, |
156 | dma_addr_t *dma_handle, void **ret) | |
157 | { | |
eccd83e1 | 158 | struct dma_coherent_mem *mem; |
ee7e5516 | 159 | int order = get_order(size); |
7bfa5ab6 | 160 | unsigned long flags; |
eccd83e1 | 161 | int pageno; |
ee7e5516 | 162 | |
eccd83e1 AM |
163 | if (!dev) |
164 | return 0; | |
165 | mem = dev->dma_mem; | |
166 | if (!mem) | |
167 | return 0; | |
0609697e PM |
168 | |
169 | *ret = NULL; | |
7bfa5ab6 | 170 | spin_lock_irqsave(&mem->spinlock, flags); |
0609697e | 171 | |
cdf57cab | 172 | if (unlikely(size > (mem->size << PAGE_SHIFT))) |
0609697e | 173 | goto err; |
eccd83e1 AM |
174 | |
175 | pageno = bitmap_find_free_region(mem->bitmap, mem->size, order); | |
0609697e PM |
176 | if (unlikely(pageno < 0)) |
177 | goto err; | |
178 | ||
179 | /* | |
180 | * Memory was found in the per-device area. | |
181 | */ | |
182 | *dma_handle = mem->device_base + (pageno << PAGE_SHIFT); | |
183 | *ret = mem->virt_base + (pageno << PAGE_SHIFT); | |
184 | memset(*ret, 0, size); | |
7bfa5ab6 | 185 | spin_unlock_irqrestore(&mem->spinlock, flags); |
0609697e | 186 | |
eccd83e1 | 187 | return 1; |
0609697e PM |
188 | |
189 | err: | |
7bfa5ab6 | 190 | spin_unlock_irqrestore(&mem->spinlock, flags); |
0609697e PM |
191 | /* |
192 | * In the case where the allocation can not be satisfied from the | |
193 | * per-device area, try to fall back to generic memory if the | |
194 | * constraints allow it. | |
195 | */ | |
196 | return mem->flags & DMA_MEMORY_EXCLUSIVE; | |
ee7e5516 | 197 | } |
a38409fb | 198 | EXPORT_SYMBOL(dma_alloc_from_coherent); |
ee7e5516 | 199 | |
b6d4f7e3 | 200 | /** |
cb3952bf | 201 | * dma_release_from_coherent() - try to free the memory allocated from per-device coherent memory pool |
b6d4f7e3 DB |
202 | * @dev: device from which the memory was allocated |
203 | * @order: the order of pages allocated | |
204 | * @vaddr: virtual address of allocated pages | |
205 | * | |
206 | * This checks whether the memory was allocated from the per-device | |
207 | * coherent memory pool and if so, releases that memory. | |
208 | * | |
209 | * Returns 1 if we correctly released the memory, or 0 if | |
cb3952bf | 210 | * dma_release_coherent() should proceed with releasing memory from |
b6d4f7e3 DB |
211 | * generic pools. |
212 | */ | |
ee7e5516 DB |
213 | int dma_release_from_coherent(struct device *dev, int order, void *vaddr) |
214 | { | |
215 | struct dma_coherent_mem *mem = dev ? dev->dma_mem : NULL; | |
216 | ||
217 | if (mem && vaddr >= mem->virt_base && vaddr < | |
218 | (mem->virt_base + (mem->size << PAGE_SHIFT))) { | |
219 | int page = (vaddr - mem->virt_base) >> PAGE_SHIFT; | |
7bfa5ab6 | 220 | unsigned long flags; |
ee7e5516 | 221 | |
7bfa5ab6 | 222 | spin_lock_irqsave(&mem->spinlock, flags); |
ee7e5516 | 223 | bitmap_release_region(mem->bitmap, page, order); |
7bfa5ab6 | 224 | spin_unlock_irqrestore(&mem->spinlock, flags); |
ee7e5516 DB |
225 | return 1; |
226 | } | |
227 | return 0; | |
228 | } | |
a38409fb | 229 | EXPORT_SYMBOL(dma_release_from_coherent); |
bca0fa5f MS |
230 | |
231 | /** | |
232 | * dma_mmap_from_coherent() - try to mmap the memory allocated from | |
233 | * per-device coherent memory pool to userspace | |
234 | * @dev: device from which the memory was allocated | |
235 | * @vma: vm_area for the userspace memory | |
236 | * @vaddr: cpu address returned by dma_alloc_from_coherent | |
237 | * @size: size of the memory buffer allocated by dma_alloc_from_coherent | |
6e7b4a59 | 238 | * @ret: result from remap_pfn_range() |
bca0fa5f MS |
239 | * |
240 | * This checks whether the memory was allocated from the per-device | |
241 | * coherent memory pool and if so, maps that memory to the provided vma. | |
242 | * | |
ba4d93bc LP |
243 | * Returns 1 if we correctly mapped the memory, or 0 if the caller should |
244 | * proceed with mapping memory from generic pools. | |
bca0fa5f MS |
245 | */ |
246 | int dma_mmap_from_coherent(struct device *dev, struct vm_area_struct *vma, | |
247 | void *vaddr, size_t size, int *ret) | |
248 | { | |
249 | struct dma_coherent_mem *mem = dev ? dev->dma_mem : NULL; | |
250 | ||
251 | if (mem && vaddr >= mem->virt_base && vaddr + size <= | |
252 | (mem->virt_base + (mem->size << PAGE_SHIFT))) { | |
253 | unsigned long off = vma->vm_pgoff; | |
254 | int start = (vaddr - mem->virt_base) >> PAGE_SHIFT; | |
255 | int user_count = (vma->vm_end - vma->vm_start) >> PAGE_SHIFT; | |
256 | int count = size >> PAGE_SHIFT; | |
257 | ||
258 | *ret = -ENXIO; | |
259 | if (off < count && user_count <= count - off) { | |
88a984ba | 260 | unsigned long pfn = mem->pfn_base + start + off; |
bca0fa5f MS |
261 | *ret = remap_pfn_range(vma, vma->vm_start, pfn, |
262 | user_count << PAGE_SHIFT, | |
263 | vma->vm_page_prot); | |
264 | } | |
265 | return 1; | |
266 | } | |
267 | return 0; | |
268 | } | |
269 | EXPORT_SYMBOL(dma_mmap_from_coherent); | |
7bfa5ab6 MS |
270 | |
271 | /* | |
272 | * Support for reserved memory regions defined in device tree | |
273 | */ | |
274 | #ifdef CONFIG_OF_RESERVED_MEM | |
275 | #include <linux/of.h> | |
276 | #include <linux/of_fdt.h> | |
277 | #include <linux/of_reserved_mem.h> | |
278 | ||
279 | static int rmem_dma_device_init(struct reserved_mem *rmem, struct device *dev) | |
280 | { | |
281 | struct dma_coherent_mem *mem = rmem->priv; | |
282 | ||
283 | if (!mem && | |
284 | dma_init_coherent_memory(rmem->base, rmem->base, rmem->size, | |
285 | DMA_MEMORY_MAP | DMA_MEMORY_EXCLUSIVE, | |
286 | &mem) != DMA_MEMORY_MAP) { | |
287 | pr_err("Reserved memory: failed to init DMA memory pool at %pa, size %ld MiB\n", | |
288 | &rmem->base, (unsigned long)rmem->size / SZ_1M); | |
289 | return -ENODEV; | |
290 | } | |
291 | rmem->priv = mem; | |
292 | dma_assign_coherent_memory(dev, mem); | |
293 | return 0; | |
294 | } | |
295 | ||
296 | static void rmem_dma_device_release(struct reserved_mem *rmem, | |
297 | struct device *dev) | |
298 | { | |
299 | dev->dma_mem = NULL; | |
300 | } | |
301 | ||
302 | static const struct reserved_mem_ops rmem_dma_ops = { | |
303 | .device_init = rmem_dma_device_init, | |
304 | .device_release = rmem_dma_device_release, | |
305 | }; | |
306 | ||
307 | static int __init rmem_dma_setup(struct reserved_mem *rmem) | |
308 | { | |
309 | unsigned long node = rmem->fdt_node; | |
310 | ||
311 | if (of_get_flat_dt_prop(node, "reusable", NULL)) | |
312 | return -EINVAL; | |
313 | ||
314 | #ifdef CONFIG_ARM | |
315 | if (!of_get_flat_dt_prop(node, "no-map", NULL)) { | |
316 | pr_err("Reserved memory: regions without no-map are not yet supported\n"); | |
317 | return -EINVAL; | |
318 | } | |
319 | #endif | |
320 | ||
321 | rmem->ops = &rmem_dma_ops; | |
322 | pr_info("Reserved memory: created DMA memory pool at %pa, size %ld MiB\n", | |
323 | &rmem->base, (unsigned long)rmem->size / SZ_1M); | |
324 | return 0; | |
325 | } | |
326 | RESERVEDMEM_OF_DECLARE(dma, "shared-dma-pool", rmem_dma_setup); | |
327 | #endif |