2 * omap iommu: simple virtual address space management
4 * Copyright (C) 2008-2009 Nokia Corporation
6 * Written by Hiroshi DOYU <Hiroshi.DOYU@nokia.com>
8 * This program is free software; you can redistribute it and/or modify
9 * it under the terms of the GNU General Public License version 2 as
10 * published by the Free Software Foundation.
13 #include <linux/module.h>
14 #include <linux/err.h>
15 #include <linux/slab.h>
16 #include <linux/vmalloc.h>
17 #include <linux/device.h>
18 #include <linux/scatterlist.h>
19 #include <linux/iommu.h>
21 #include <asm/cacheflush.h>
22 #include <asm/mach/map.h>
24 #include <plat/iommu.h>
25 #include <plat/iovmm.h>
27 #include <plat/iopgtable.h>
29 static struct kmem_cache
*iovm_area_cachep
;
31 /* return the offset of the first scatterlist entry in a sg table */
32 static unsigned int sgtable_offset(const struct sg_table
*sgt
)
34 if (!sgt
|| !sgt
->nents
)
37 return sgt
->sgl
->offset
;
40 /* return total bytes of sg buffers */
41 static size_t sgtable_len(const struct sg_table
*sgt
)
43 unsigned int i
, total
= 0;
44 struct scatterlist
*sg
;
49 for_each_sg(sgt
->sgl
, sg
, sgt
->nents
, i
) {
52 bytes
= sg
->length
+ sg
->offset
;
54 if (!iopgsz_ok(bytes
)) {
55 pr_err("%s: sg[%d] not iommu pagesize(%u %u)\n",
56 __func__
, i
, bytes
, sg
->offset
);
60 if (i
&& sg
->offset
) {
61 pr_err("%s: sg[%d] offset not allowed in internal "
62 "entries\n", __func__
, i
);
71 #define sgtable_ok(x) (!!sgtable_len(x))
73 static unsigned max_alignment(u32 addr
)
76 unsigned pagesize
[] = { SZ_16M
, SZ_1M
, SZ_64K
, SZ_4K
, };
77 for (i
= 0; i
< ARRAY_SIZE(pagesize
) && addr
& (pagesize
[i
] - 1); i
++)
79 return (i
< ARRAY_SIZE(pagesize
)) ? pagesize
[i
] : 0;
83 * calculate the optimal number sg elements from total bytes based on
86 static unsigned sgtable_nents(size_t bytes
, u32 da
, u32 pa
)
88 unsigned nr_entries
= 0, ent_sz
;
90 if (!IS_ALIGNED(bytes
, PAGE_SIZE
)) {
91 pr_err("%s: wrong size %08x\n", __func__
, bytes
);
96 ent_sz
= max_alignment(da
| pa
);
97 ent_sz
= min_t(unsigned, ent_sz
, iopgsz_max(bytes
));
107 /* allocate and initialize sg_table header(a kind of 'superblock') */
108 static struct sg_table
*sgtable_alloc(const size_t bytes
, u32 flags
,
111 unsigned int nr_entries
;
113 struct sg_table
*sgt
;
116 return ERR_PTR(-EINVAL
);
118 if (!IS_ALIGNED(bytes
, PAGE_SIZE
))
119 return ERR_PTR(-EINVAL
);
121 if (flags
& IOVMF_LINEAR
) {
122 nr_entries
= sgtable_nents(bytes
, da
, pa
);
124 return ERR_PTR(-EINVAL
);
126 nr_entries
= bytes
/ PAGE_SIZE
;
128 sgt
= kzalloc(sizeof(*sgt
), GFP_KERNEL
);
130 return ERR_PTR(-ENOMEM
);
132 err
= sg_alloc_table(sgt
, nr_entries
, GFP_KERNEL
);
138 pr_debug("%s: sgt:%p(%d entries)\n", __func__
, sgt
, nr_entries
);
143 /* free sg_table header(a kind of superblock) */
144 static void sgtable_free(struct sg_table
*sgt
)
152 pr_debug("%s: sgt:%p\n", __func__
, sgt
);
155 /* map 'sglist' to a contiguous mpu virtual area and return 'va' */
156 static void *vmap_sg(const struct sg_table
*sgt
)
161 struct scatterlist
*sg
;
162 struct vm_struct
*new;
163 const struct mem_type
*mtype
;
165 mtype
= get_mem_type(MT_DEVICE
);
167 return ERR_PTR(-EINVAL
);
169 total
= sgtable_len(sgt
);
171 return ERR_PTR(-EINVAL
);
173 new = __get_vm_area(total
, VM_IOREMAP
, VMALLOC_START
, VMALLOC_END
);
175 return ERR_PTR(-ENOMEM
);
178 for_each_sg(sgt
->sgl
, sg
, sgt
->nents
, i
) {
183 pa
= sg_phys(sg
) - sg
->offset
;
184 bytes
= sg
->length
+ sg
->offset
;
186 BUG_ON(bytes
!= PAGE_SIZE
);
188 err
= ioremap_page(va
, pa
, mtype
);
195 flush_cache_vmap((unsigned long)new->addr
,
196 (unsigned long)(new->addr
+ total
));
200 WARN_ON(1); /* FIXME: cleanup some mpu mappings */
202 return ERR_PTR(-EAGAIN
);
205 static inline void vunmap_sg(const void *va
)
210 static struct iovm_struct
*__find_iovm_area(struct omap_iommu
*obj
,
213 struct iovm_struct
*tmp
;
215 list_for_each_entry(tmp
, &obj
->mmap
, list
) {
216 if ((da
>= tmp
->da_start
) && (da
< tmp
->da_end
)) {
219 len
= tmp
->da_end
- tmp
->da_start
;
221 dev_dbg(obj
->dev
, "%s: %08x-%08x-%08x(%x) %08x\n",
222 __func__
, tmp
->da_start
, da
, tmp
->da_end
, len
,
233 * omap_find_iovm_area - find iovma which includes @da
234 * @dev: client device
235 * @da: iommu device virtual address
237 * Find the existing iovma starting at @da
239 struct iovm_struct
*omap_find_iovm_area(struct device
*dev
, u32 da
)
241 struct omap_iommu
*obj
= dev_to_omap_iommu(dev
);
242 struct iovm_struct
*area
;
244 mutex_lock(&obj
->mmap_lock
);
245 area
= __find_iovm_area(obj
, da
);
246 mutex_unlock(&obj
->mmap_lock
);
250 EXPORT_SYMBOL_GPL(omap_find_iovm_area
);
253 * This finds the hole(area) which fits the requested address and len
254 * in iovmas mmap, and returns the new allocated iovma.
256 static struct iovm_struct
*alloc_iovm_area(struct omap_iommu
*obj
, u32 da
,
257 size_t bytes
, u32 flags
)
259 struct iovm_struct
*new, *tmp
;
260 u32 start
, prev_end
, alignment
;
263 return ERR_PTR(-EINVAL
);
266 alignment
= PAGE_SIZE
;
268 if (~flags
& IOVMF_DA_FIXED
) {
269 /* Don't map address 0 */
270 start
= obj
->da_start
? obj
->da_start
: alignment
;
272 if (flags
& IOVMF_LINEAR
)
273 alignment
= iopgsz_max(bytes
);
274 start
= roundup(start
, alignment
);
275 } else if (start
< obj
->da_start
|| start
> obj
->da_end
||
276 obj
->da_end
- start
< bytes
) {
277 return ERR_PTR(-EINVAL
);
281 if (list_empty(&obj
->mmap
))
285 list_for_each_entry(tmp
, &obj
->mmap
, list
) {
287 if (prev_end
> start
)
290 if (tmp
->da_start
> start
&& (tmp
->da_start
- start
) >= bytes
)
293 if (tmp
->da_end
>= start
&& ~flags
& IOVMF_DA_FIXED
)
294 start
= roundup(tmp
->da_end
+ 1, alignment
);
296 prev_end
= tmp
->da_end
;
299 if ((start
>= prev_end
) && (obj
->da_end
- start
>= bytes
))
302 dev_dbg(obj
->dev
, "%s: no space to fit %08x(%x) flags: %08x\n",
303 __func__
, da
, bytes
, flags
);
305 return ERR_PTR(-EINVAL
);
308 new = kmem_cache_zalloc(iovm_area_cachep
, GFP_KERNEL
);
310 return ERR_PTR(-ENOMEM
);
313 new->da_start
= start
;
314 new->da_end
= start
+ bytes
;
318 * keep ascending order of iovmas
321 list_add_tail(&new->list
, &tmp
->list
);
323 list_add(&new->list
, &obj
->mmap
);
325 dev_dbg(obj
->dev
, "%s: found %08x-%08x-%08x(%x) %08x\n",
326 __func__
, new->da_start
, start
, new->da_end
, bytes
, flags
);
331 static void free_iovm_area(struct omap_iommu
*obj
, struct iovm_struct
*area
)
335 BUG_ON(!obj
|| !area
);
337 bytes
= area
->da_end
- area
->da_start
;
339 dev_dbg(obj
->dev
, "%s: %08x-%08x(%x) %08x\n",
340 __func__
, area
->da_start
, area
->da_end
, bytes
, area
->flags
);
342 list_del(&area
->list
);
343 kmem_cache_free(iovm_area_cachep
, area
);
347 * omap_da_to_va - convert (d) to (v)
348 * @dev: client device
349 * @da: iommu device virtual address
350 * @va: mpu virtual address
352 * Returns mpu virtual addr which corresponds to a given device virtual addr
354 void *omap_da_to_va(struct device
*dev
, u32 da
)
356 struct omap_iommu
*obj
= dev_to_omap_iommu(dev
);
358 struct iovm_struct
*area
;
360 mutex_lock(&obj
->mmap_lock
);
362 area
= __find_iovm_area(obj
, da
);
364 dev_dbg(obj
->dev
, "%s: no da area(%08x)\n", __func__
, da
);
369 mutex_unlock(&obj
->mmap_lock
);
373 EXPORT_SYMBOL_GPL(omap_da_to_va
);
375 static void sgtable_fill_vmalloc(struct sg_table
*sgt
, void *_va
)
378 struct scatterlist
*sg
;
382 for_each_sg(sgt
->sgl
, sg
, sgt
->nents
, i
) {
384 const size_t bytes
= PAGE_SIZE
;
387 * iommu 'superpage' isn't supported with 'omap_iommu_vmalloc()'
389 pg
= vmalloc_to_page(va
);
391 sg_set_page(sg
, pg
, bytes
, 0);
396 va_end
= _va
+ PAGE_SIZE
* i
;
399 static inline void sgtable_drain_vmalloc(struct sg_table
*sgt
)
402 * Actually this is not necessary at all, just exists for
403 * consistency of the code readability.
408 /* create 'da' <-> 'pa' mapping from 'sgt' */
409 static int map_iovm_area(struct iommu_domain
*domain
, struct iovm_struct
*new,
410 const struct sg_table
*sgt
, u32 flags
)
414 struct scatterlist
*sg
;
415 u32 da
= new->da_start
;
420 BUG_ON(!sgtable_ok(sgt
));
422 for_each_sg(sgt
->sgl
, sg
, sgt
->nents
, i
) {
426 pa
= sg_phys(sg
) - sg
->offset
;
427 bytes
= sg
->length
+ sg
->offset
;
429 flags
&= ~IOVMF_PGSZ_MASK
;
431 if (bytes_to_iopgsz(bytes
) < 0)
434 pr_debug("%s: [%d] %08x %08x(%x)\n", __func__
,
437 err
= iommu_map(domain
, da
, pa
, bytes
, flags
);
448 for_each_sg(sgt
->sgl
, sg
, i
, j
) {
451 bytes
= sg
->length
+ sg
->offset
;
453 /* ignore failures.. we're already handling one */
454 iommu_unmap(domain
, da
, bytes
);
461 /* release 'da' <-> 'pa' mapping */
462 static void unmap_iovm_area(struct iommu_domain
*domain
, struct omap_iommu
*obj
,
463 struct iovm_struct
*area
)
466 size_t total
= area
->da_end
- area
->da_start
;
467 const struct sg_table
*sgt
= area
->sgt
;
468 struct scatterlist
*sg
;
472 BUG_ON(!sgtable_ok(sgt
));
473 BUG_ON((!total
) || !IS_ALIGNED(total
, PAGE_SIZE
));
475 start
= area
->da_start
;
476 for_each_sg(sgt
->sgl
, sg
, sgt
->nents
, i
) {
479 bytes
= sg
->length
+ sg
->offset
;
481 unmapped
= iommu_unmap(domain
, start
, bytes
);
482 if (unmapped
< bytes
)
485 dev_dbg(obj
->dev
, "%s: unmap %08x(%x) %08x\n",
486 __func__
, start
, bytes
, area
->flags
);
488 BUG_ON(!IS_ALIGNED(bytes
, PAGE_SIZE
));
496 /* template function for all unmapping */
497 static struct sg_table
*unmap_vm_area(struct iommu_domain
*domain
,
498 struct omap_iommu
*obj
, const u32 da
,
499 void (*fn
)(const void *), u32 flags
)
501 struct sg_table
*sgt
= NULL
;
502 struct iovm_struct
*area
;
504 if (!IS_ALIGNED(da
, PAGE_SIZE
)) {
505 dev_err(obj
->dev
, "%s: alignment err(%08x)\n", __func__
, da
);
509 mutex_lock(&obj
->mmap_lock
);
511 area
= __find_iovm_area(obj
, da
);
513 dev_dbg(obj
->dev
, "%s: no da area(%08x)\n", __func__
, da
);
517 if ((area
->flags
& flags
) != flags
) {
518 dev_err(obj
->dev
, "%s: wrong flags(%08x)\n", __func__
,
522 sgt
= (struct sg_table
*)area
->sgt
;
524 unmap_iovm_area(domain
, obj
, area
);
528 dev_dbg(obj
->dev
, "%s: %08x-%08x-%08x(%x) %08x\n", __func__
,
529 area
->da_start
, da
, area
->da_end
,
530 area
->da_end
- area
->da_start
, area
->flags
);
532 free_iovm_area(obj
, area
);
534 mutex_unlock(&obj
->mmap_lock
);
539 static u32
map_iommu_region(struct iommu_domain
*domain
, struct omap_iommu
*obj
,
540 u32 da
, const struct sg_table
*sgt
, void *va
,
541 size_t bytes
, u32 flags
)
544 struct iovm_struct
*new;
546 mutex_lock(&obj
->mmap_lock
);
548 new = alloc_iovm_area(obj
, da
, bytes
, flags
);
551 goto err_alloc_iovma
;
556 if (map_iovm_area(domain
, new, sgt
, new->flags
))
559 mutex_unlock(&obj
->mmap_lock
);
561 dev_dbg(obj
->dev
, "%s: da:%08x(%x) flags:%08x va:%p\n",
562 __func__
, new->da_start
, bytes
, new->flags
, va
);
564 return new->da_start
;
567 free_iovm_area(obj
, new);
569 mutex_unlock(&obj
->mmap_lock
);
574 __iommu_vmap(struct iommu_domain
*domain
, struct omap_iommu
*obj
,
575 u32 da
, const struct sg_table
*sgt
,
576 void *va
, size_t bytes
, u32 flags
)
578 return map_iommu_region(domain
, obj
, da
, sgt
, va
, bytes
, flags
);
582 * omap_iommu_vmap - (d)-(p)-(v) address mapper
583 * @domain: iommu domain
584 * @dev: client device
585 * @sgt: address of scatter gather table
586 * @flags: iovma and page property
588 * Creates 1-n-1 mapping with given @sgt and returns @da.
589 * All @sgt element must be io page size aligned.
591 u32
omap_iommu_vmap(struct iommu_domain
*domain
, struct device
*dev
, u32 da
,
592 const struct sg_table
*sgt
, u32 flags
)
594 struct omap_iommu
*obj
= dev_to_omap_iommu(dev
);
598 if (!obj
|| !obj
->dev
|| !sgt
)
601 bytes
= sgtable_len(sgt
);
604 bytes
= PAGE_ALIGN(bytes
);
606 if (flags
& IOVMF_MMIO
) {
612 flags
|= IOVMF_DISCONT
;
615 da
= __iommu_vmap(domain
, obj
, da
, sgt
, va
, bytes
, flags
);
616 if (IS_ERR_VALUE(da
))
619 return da
+ sgtable_offset(sgt
);
621 EXPORT_SYMBOL_GPL(omap_iommu_vmap
);
624 * omap_iommu_vunmap - release virtual mapping obtained by 'omap_iommu_vmap()'
625 * @domain: iommu domain
626 * @dev: client device
627 * @da: iommu device virtual address
629 * Free the iommu virtually contiguous memory area starting at
630 * @da, which was returned by 'omap_iommu_vmap()'.
633 omap_iommu_vunmap(struct iommu_domain
*domain
, struct device
*dev
, u32 da
)
635 struct omap_iommu
*obj
= dev_to_omap_iommu(dev
);
636 struct sg_table
*sgt
;
638 * 'sgt' is allocated before 'omap_iommu_vmalloc()' is called.
639 * Just returns 'sgt' to the caller to free
642 sgt
= unmap_vm_area(domain
, obj
, da
, vunmap_sg
,
643 IOVMF_DISCONT
| IOVMF_MMIO
);
645 dev_dbg(obj
->dev
, "%s: No sgt\n", __func__
);
648 EXPORT_SYMBOL_GPL(omap_iommu_vunmap
);
651 * omap_iommu_vmalloc - (d)-(p)-(v) address allocator and mapper
652 * @dev: client device
653 * @da: contiguous iommu virtual memory
654 * @bytes: allocation size
655 * @flags: iovma and page property
657 * Allocate @bytes linearly and creates 1-n-1 mapping and returns
658 * @da again, which might be adjusted if 'IOVMF_DA_FIXED' is not set.
661 omap_iommu_vmalloc(struct iommu_domain
*domain
, struct device
*dev
, u32 da
,
662 size_t bytes
, u32 flags
)
664 struct omap_iommu
*obj
= dev_to_omap_iommu(dev
);
666 struct sg_table
*sgt
;
668 if (!obj
|| !obj
->dev
|| !bytes
)
671 bytes
= PAGE_ALIGN(bytes
);
677 flags
|= IOVMF_DISCONT
;
678 flags
|= IOVMF_ALLOC
;
680 sgt
= sgtable_alloc(bytes
, flags
, da
, 0);
685 sgtable_fill_vmalloc(sgt
, va
);
687 da
= __iommu_vmap(domain
, obj
, da
, sgt
, va
, bytes
, flags
);
688 if (IS_ERR_VALUE(da
))
694 sgtable_drain_vmalloc(sgt
);
700 EXPORT_SYMBOL_GPL(omap_iommu_vmalloc
);
703 * omap_iommu_vfree - release memory allocated by 'omap_iommu_vmalloc()'
704 * @dev: client device
705 * @da: iommu device virtual address
707 * Frees the iommu virtually continuous memory area starting at
708 * @da, as obtained from 'omap_iommu_vmalloc()'.
710 void omap_iommu_vfree(struct iommu_domain
*domain
, struct device
*dev
,
713 struct omap_iommu
*obj
= dev_to_omap_iommu(dev
);
714 struct sg_table
*sgt
;
716 sgt
= unmap_vm_area(domain
, obj
, da
, vfree
,
717 IOVMF_DISCONT
| IOVMF_ALLOC
);
719 dev_dbg(obj
->dev
, "%s: No sgt\n", __func__
);
722 EXPORT_SYMBOL_GPL(omap_iommu_vfree
);
724 static int __init
iovmm_init(void)
726 const unsigned long flags
= SLAB_HWCACHE_ALIGN
;
727 struct kmem_cache
*p
;
729 p
= kmem_cache_create("iovm_area_cache", sizeof(struct iovm_struct
), 0,
733 iovm_area_cachep
= p
;
737 module_init(iovmm_init
);
739 static void __exit
iovmm_exit(void)
741 kmem_cache_destroy(iovm_area_cachep
);
743 module_exit(iovmm_exit
);
745 MODULE_DESCRIPTION("omap iommu: simple virtual address space management");
746 MODULE_AUTHOR("Hiroshi DOYU <Hiroshi.DOYU@nokia.com>");
747 MODULE_LICENSE("GPL v2");
This page took 0.045602 seconds and 5 git commands to generate.