2 * omap iommu: tlb and pagetable primitives
4 * Copyright (C) 2008-2010 Nokia Corporation
6 * Written by Hiroshi DOYU <Hiroshi.DOYU@nokia.com>,
7 * Paul Mundt and Toshihiro Kobayashi
9 * This program is free software; you can redistribute it and/or modify
10 * it under the terms of the GNU General Public License version 2 as
11 * published by the Free Software Foundation.
14 #include <linux/err.h>
15 #include <linux/module.h>
16 #include <linux/slab.h>
17 #include <linux/interrupt.h>
18 #include <linux/ioport.h>
19 #include <linux/platform_device.h>
20 #include <linux/iommu.h>
21 #include <linux/omap-iommu.h>
22 #include <linux/mutex.h>
23 #include <linux/spinlock.h>
25 #include <linux/pm_runtime.h>
27 #include <linux/of_iommu.h>
28 #include <linux/of_irq.h>
29 #include <linux/of_platform.h>
31 #include <asm/cacheflush.h>
33 #include <linux/platform_data/iommu-omap.h>
35 #include "omap-iopgtable.h"
36 #include "omap-iommu.h"
38 #define to_iommu(dev) \
39 ((struct omap_iommu *)platform_get_drvdata(to_platform_device(dev)))
41 #define for_each_iotlb_cr(obj, n, __i, cr) \
43 (__i < (n)) && (cr = __iotlb_read_cr((obj), __i), true); \
46 /* bitmap of the page sizes currently supported */
47 #define OMAP_IOMMU_PGSIZES (SZ_4K | SZ_64K | SZ_1M | SZ_16M)
50 * struct omap_iommu_domain - omap iommu domain
51 * @pgtable: the page table
52 * @iommu_dev: an omap iommu device attached to this domain. only a single
53 * iommu device can be attached for now.
54 * @dev: Device using this domain.
55 * @lock: domain lock, should be taken when attaching/detaching
57 struct omap_iommu_domain
{
59 struct omap_iommu
*iommu_dev
;
62 struct iommu_domain domain
;
65 #define MMU_LOCK_BASE_SHIFT 10
66 #define MMU_LOCK_BASE_MASK (0x1f << MMU_LOCK_BASE_SHIFT)
67 #define MMU_LOCK_BASE(x) \
68 ((x & MMU_LOCK_BASE_MASK) >> MMU_LOCK_BASE_SHIFT)
70 #define MMU_LOCK_VICT_SHIFT 4
71 #define MMU_LOCK_VICT_MASK (0x1f << MMU_LOCK_VICT_SHIFT)
72 #define MMU_LOCK_VICT(x) \
73 ((x & MMU_LOCK_VICT_MASK) >> MMU_LOCK_VICT_SHIFT)
80 static struct platform_driver omap_iommu_driver
;
81 static struct kmem_cache
*iopte_cachep
;
84 * to_omap_domain - Get struct omap_iommu_domain from generic iommu_domain
85 * @dom: generic iommu domain handle
87 static struct omap_iommu_domain
*to_omap_domain(struct iommu_domain
*dom
)
89 return container_of(dom
, struct omap_iommu_domain
, domain
);
93 * omap_iommu_save_ctx - Save registers for pm off-mode support
96 void omap_iommu_save_ctx(struct device
*dev
)
98 struct omap_iommu
*obj
= dev_to_omap_iommu(dev
);
102 for (i
= 0; i
< (MMU_REG_SIZE
/ sizeof(u32
)); i
++) {
103 p
[i
] = iommu_read_reg(obj
, i
* sizeof(u32
));
104 dev_dbg(obj
->dev
, "%s\t[%02d] %08x\n", __func__
, i
, p
[i
]);
107 EXPORT_SYMBOL_GPL(omap_iommu_save_ctx
);
110 * omap_iommu_restore_ctx - Restore registers for pm off-mode support
111 * @dev: client device
113 void omap_iommu_restore_ctx(struct device
*dev
)
115 struct omap_iommu
*obj
= dev_to_omap_iommu(dev
);
119 for (i
= 0; i
< (MMU_REG_SIZE
/ sizeof(u32
)); i
++) {
120 iommu_write_reg(obj
, p
[i
], i
* sizeof(u32
));
121 dev_dbg(obj
->dev
, "%s\t[%02d] %08x\n", __func__
, i
, p
[i
]);
124 EXPORT_SYMBOL_GPL(omap_iommu_restore_ctx
);
126 static void __iommu_set_twl(struct omap_iommu
*obj
, bool on
)
128 u32 l
= iommu_read_reg(obj
, MMU_CNTL
);
131 iommu_write_reg(obj
, MMU_IRQ_TWL_MASK
, MMU_IRQENABLE
);
133 iommu_write_reg(obj
, MMU_IRQ_TLB_MISS_MASK
, MMU_IRQENABLE
);
137 l
|= (MMU_CNTL_MMU_EN
| MMU_CNTL_TWL_EN
);
139 l
|= (MMU_CNTL_MMU_EN
);
141 iommu_write_reg(obj
, l
, MMU_CNTL
);
144 static int omap2_iommu_enable(struct omap_iommu
*obj
)
148 if (!obj
->iopgd
|| !IS_ALIGNED((u32
)obj
->iopgd
, SZ_16K
))
151 pa
= virt_to_phys(obj
->iopgd
);
152 if (!IS_ALIGNED(pa
, SZ_16K
))
155 l
= iommu_read_reg(obj
, MMU_REVISION
);
156 dev_info(obj
->dev
, "%s: version %d.%d\n", obj
->name
,
157 (l
>> 4) & 0xf, l
& 0xf);
159 iommu_write_reg(obj
, pa
, MMU_TTB
);
161 if (obj
->has_bus_err_back
)
162 iommu_write_reg(obj
, MMU_GP_REG_BUS_ERR_BACK_EN
, MMU_GP_REG
);
164 __iommu_set_twl(obj
, true);
169 static void omap2_iommu_disable(struct omap_iommu
*obj
)
171 u32 l
= iommu_read_reg(obj
, MMU_CNTL
);
174 iommu_write_reg(obj
, l
, MMU_CNTL
);
176 dev_dbg(obj
->dev
, "%s is shutting down\n", obj
->name
);
179 static int iommu_enable(struct omap_iommu
*obj
)
182 struct platform_device
*pdev
= to_platform_device(obj
->dev
);
183 struct iommu_platform_data
*pdata
= dev_get_platdata(&pdev
->dev
);
185 if (pdata
&& pdata
->deassert_reset
) {
186 err
= pdata
->deassert_reset(pdev
, pdata
->reset_name
);
188 dev_err(obj
->dev
, "deassert_reset failed: %d\n", err
);
193 pm_runtime_get_sync(obj
->dev
);
195 err
= omap2_iommu_enable(obj
);
200 static void iommu_disable(struct omap_iommu
*obj
)
202 struct platform_device
*pdev
= to_platform_device(obj
->dev
);
203 struct iommu_platform_data
*pdata
= dev_get_platdata(&pdev
->dev
);
205 omap2_iommu_disable(obj
);
207 pm_runtime_put_sync(obj
->dev
);
209 if (pdata
&& pdata
->assert_reset
)
210 pdata
->assert_reset(pdev
, pdata
->reset_name
);
216 static inline int iotlb_cr_valid(struct cr_regs
*cr
)
221 return cr
->cam
& MMU_CAM_V
;
224 static u32
iotlb_cr_to_virt(struct cr_regs
*cr
)
226 u32 page_size
= cr
->cam
& MMU_CAM_PGSZ_MASK
;
227 u32 mask
= get_cam_va_mask(cr
->cam
& page_size
);
229 return cr
->cam
& mask
;
232 static u32
get_iopte_attr(struct iotlb_entry
*e
)
236 attr
= e
->mixed
<< 5;
238 attr
|= e
->elsz
>> 3;
239 attr
<<= (((e
->pgsz
== MMU_CAM_PGSZ_4K
) ||
240 (e
->pgsz
== MMU_CAM_PGSZ_64K
)) ? 0 : 6);
244 static u32
iommu_report_fault(struct omap_iommu
*obj
, u32
*da
)
246 u32 status
, fault_addr
;
248 status
= iommu_read_reg(obj
, MMU_IRQSTATUS
);
249 status
&= MMU_IRQ_MASK
;
255 fault_addr
= iommu_read_reg(obj
, MMU_FAULT_AD
);
258 iommu_write_reg(obj
, status
, MMU_IRQSTATUS
);
263 static void iotlb_lock_get(struct omap_iommu
*obj
, struct iotlb_lock
*l
)
267 val
= iommu_read_reg(obj
, MMU_LOCK
);
269 l
->base
= MMU_LOCK_BASE(val
);
270 l
->vict
= MMU_LOCK_VICT(val
);
274 static void iotlb_lock_set(struct omap_iommu
*obj
, struct iotlb_lock
*l
)
278 val
= (l
->base
<< MMU_LOCK_BASE_SHIFT
);
279 val
|= (l
->vict
<< MMU_LOCK_VICT_SHIFT
);
281 iommu_write_reg(obj
, val
, MMU_LOCK
);
284 static void iotlb_read_cr(struct omap_iommu
*obj
, struct cr_regs
*cr
)
286 cr
->cam
= iommu_read_reg(obj
, MMU_READ_CAM
);
287 cr
->ram
= iommu_read_reg(obj
, MMU_READ_RAM
);
290 static void iotlb_load_cr(struct omap_iommu
*obj
, struct cr_regs
*cr
)
292 iommu_write_reg(obj
, cr
->cam
| MMU_CAM_V
, MMU_CAM
);
293 iommu_write_reg(obj
, cr
->ram
, MMU_RAM
);
295 iommu_write_reg(obj
, 1, MMU_FLUSH_ENTRY
);
296 iommu_write_reg(obj
, 1, MMU_LD_TLB
);
299 /* only used in iotlb iteration for-loop */
300 static struct cr_regs
__iotlb_read_cr(struct omap_iommu
*obj
, int n
)
305 iotlb_lock_get(obj
, &l
);
307 iotlb_lock_set(obj
, &l
);
308 iotlb_read_cr(obj
, &cr
);
313 #ifdef PREFETCH_IOTLB
314 static struct cr_regs
*iotlb_alloc_cr(struct omap_iommu
*obj
,
315 struct iotlb_entry
*e
)
322 if (e
->da
& ~(get_cam_va_mask(e
->pgsz
))) {
323 dev_err(obj
->dev
, "%s:\twrong alignment: %08x\n", __func__
,
325 return ERR_PTR(-EINVAL
);
328 cr
= kmalloc(sizeof(*cr
), GFP_KERNEL
);
330 return ERR_PTR(-ENOMEM
);
332 cr
->cam
= (e
->da
& MMU_CAM_VATAG_MASK
) | e
->prsvd
| e
->pgsz
| e
->valid
;
333 cr
->ram
= e
->pa
| e
->endian
| e
->elsz
| e
->mixed
;
339 * load_iotlb_entry - Set an iommu tlb entry
341 * @e: an iommu tlb entry info
343 static int load_iotlb_entry(struct omap_iommu
*obj
, struct iotlb_entry
*e
)
349 if (!obj
|| !obj
->nr_tlb_entries
|| !e
)
352 pm_runtime_get_sync(obj
->dev
);
354 iotlb_lock_get(obj
, &l
);
355 if (l
.base
== obj
->nr_tlb_entries
) {
356 dev_warn(obj
->dev
, "%s: preserve entries full\n", __func__
);
364 for_each_iotlb_cr(obj
, obj
->nr_tlb_entries
, i
, tmp
)
365 if (!iotlb_cr_valid(&tmp
))
368 if (i
== obj
->nr_tlb_entries
) {
369 dev_dbg(obj
->dev
, "%s: full: no entry\n", __func__
);
374 iotlb_lock_get(obj
, &l
);
377 iotlb_lock_set(obj
, &l
);
380 cr
= iotlb_alloc_cr(obj
, e
);
382 pm_runtime_put_sync(obj
->dev
);
386 iotlb_load_cr(obj
, cr
);
391 /* increment victim for next tlb load */
392 if (++l
.vict
== obj
->nr_tlb_entries
)
394 iotlb_lock_set(obj
, &l
);
396 pm_runtime_put_sync(obj
->dev
);
400 #else /* !PREFETCH_IOTLB */
402 static int load_iotlb_entry(struct omap_iommu
*obj
, struct iotlb_entry
*e
)
407 #endif /* !PREFETCH_IOTLB */
409 static int prefetch_iotlb_entry(struct omap_iommu
*obj
, struct iotlb_entry
*e
)
411 return load_iotlb_entry(obj
, e
);
415 * flush_iotlb_page - Clear an iommu tlb entry
417 * @da: iommu device virtual address
419 * Clear an iommu tlb entry which includes 'da' address.
421 static void flush_iotlb_page(struct omap_iommu
*obj
, u32 da
)
426 pm_runtime_get_sync(obj
->dev
);
428 for_each_iotlb_cr(obj
, obj
->nr_tlb_entries
, i
, cr
) {
432 if (!iotlb_cr_valid(&cr
))
435 start
= iotlb_cr_to_virt(&cr
);
436 bytes
= iopgsz_to_bytes(cr
.cam
& 3);
438 if ((start
<= da
) && (da
< start
+ bytes
)) {
439 dev_dbg(obj
->dev
, "%s: %08x<=%08x(%x)\n",
440 __func__
, start
, da
, bytes
);
441 iotlb_load_cr(obj
, &cr
);
442 iommu_write_reg(obj
, 1, MMU_FLUSH_ENTRY
);
446 pm_runtime_put_sync(obj
->dev
);
448 if (i
== obj
->nr_tlb_entries
)
449 dev_dbg(obj
->dev
, "%s: no page for %08x\n", __func__
, da
);
453 * flush_iotlb_all - Clear all iommu tlb entries
456 static void flush_iotlb_all(struct omap_iommu
*obj
)
460 pm_runtime_get_sync(obj
->dev
);
464 iotlb_lock_set(obj
, &l
);
466 iommu_write_reg(obj
, 1, MMU_GFLUSH
);
468 pm_runtime_put_sync(obj
->dev
);
471 #ifdef CONFIG_OMAP_IOMMU_DEBUG
473 #define pr_reg(name) \
476 const char *str = "%20s: %08x\n"; \
477 const int maxcol = 32; \
478 bytes = snprintf(p, maxcol, str, __stringify(name), \
479 iommu_read_reg(obj, MMU_##name)); \
487 omap2_iommu_dump_ctx(struct omap_iommu
*obj
, char *buf
, ssize_t len
)
506 pr_reg(EMU_FAULT_AD
);
511 ssize_t
omap_iommu_dump_ctx(struct omap_iommu
*obj
, char *buf
, ssize_t bytes
)
516 pm_runtime_get_sync(obj
->dev
);
518 bytes
= omap2_iommu_dump_ctx(obj
, buf
, bytes
);
520 pm_runtime_put_sync(obj
->dev
);
526 __dump_tlb_entries(struct omap_iommu
*obj
, struct cr_regs
*crs
, int num
)
529 struct iotlb_lock saved
;
531 struct cr_regs
*p
= crs
;
533 pm_runtime_get_sync(obj
->dev
);
534 iotlb_lock_get(obj
, &saved
);
536 for_each_iotlb_cr(obj
, num
, i
, tmp
) {
537 if (!iotlb_cr_valid(&tmp
))
542 iotlb_lock_set(obj
, &saved
);
543 pm_runtime_put_sync(obj
->dev
);
549 * iotlb_dump_cr - Dump an iommu tlb entry into buf
551 * @cr: contents of cam and ram register
552 * @buf: output buffer
554 static ssize_t
iotlb_dump_cr(struct omap_iommu
*obj
, struct cr_regs
*cr
,
559 /* FIXME: Need more detail analysis of cam/ram */
560 p
+= sprintf(p
, "%08x %08x %01x\n", cr
->cam
, cr
->ram
,
561 (cr
->cam
& MMU_CAM_P
) ? 1 : 0);
567 * omap_dump_tlb_entries - dump cr arrays to given buffer
569 * @buf: output buffer
571 size_t omap_dump_tlb_entries(struct omap_iommu
*obj
, char *buf
, ssize_t bytes
)
577 num
= bytes
/ sizeof(*cr
);
578 num
= min(obj
->nr_tlb_entries
, num
);
580 cr
= kcalloc(num
, sizeof(*cr
), GFP_KERNEL
);
584 num
= __dump_tlb_entries(obj
, cr
, num
);
585 for (i
= 0; i
< num
; i
++)
586 p
+= iotlb_dump_cr(obj
, cr
+ i
, p
);
592 #endif /* CONFIG_OMAP_IOMMU_DEBUG */
595 * H/W pagetable operations
597 static void flush_iopgd_range(u32
*first
, u32
*last
)
599 /* FIXME: L2 cache should be taken care of if it exists */
601 asm("mcr p15, 0, %0, c7, c10, 1 @ flush_pgd"
603 first
+= L1_CACHE_BYTES
/ sizeof(*first
);
604 } while (first
<= last
);
607 static void flush_iopte_range(u32
*first
, u32
*last
)
609 /* FIXME: L2 cache should be taken care of if it exists */
611 asm("mcr p15, 0, %0, c7, c10, 1 @ flush_pte"
613 first
+= L1_CACHE_BYTES
/ sizeof(*first
);
614 } while (first
<= last
);
617 static void iopte_free(u32
*iopte
)
619 /* Note: freed iopte's must be clean ready for re-use */
621 kmem_cache_free(iopte_cachep
, iopte
);
624 static u32
*iopte_alloc(struct omap_iommu
*obj
, u32
*iopgd
, u32 da
)
628 /* a table has already existed */
633 * do the allocation outside the page table lock
635 spin_unlock(&obj
->page_table_lock
);
636 iopte
= kmem_cache_zalloc(iopte_cachep
, GFP_KERNEL
);
637 spin_lock(&obj
->page_table_lock
);
641 return ERR_PTR(-ENOMEM
);
643 *iopgd
= virt_to_phys(iopte
) | IOPGD_TABLE
;
644 flush_iopgd_range(iopgd
, iopgd
);
646 dev_vdbg(obj
->dev
, "%s: a new pte:%p\n", __func__
, iopte
);
648 /* We raced, free the reduniovant table */
653 iopte
= iopte_offset(iopgd
, da
);
656 "%s: da:%08x pgd:%p *pgd:%08x pte:%p *pte:%08x\n",
657 __func__
, da
, iopgd
, *iopgd
, iopte
, *iopte
);
662 static int iopgd_alloc_section(struct omap_iommu
*obj
, u32 da
, u32 pa
, u32 prot
)
664 u32
*iopgd
= iopgd_offset(obj
, da
);
666 if ((da
| pa
) & ~IOSECTION_MASK
) {
667 dev_err(obj
->dev
, "%s: %08x:%08x should aligned on %08lx\n",
668 __func__
, da
, pa
, IOSECTION_SIZE
);
672 *iopgd
= (pa
& IOSECTION_MASK
) | prot
| IOPGD_SECTION
;
673 flush_iopgd_range(iopgd
, iopgd
);
677 static int iopgd_alloc_super(struct omap_iommu
*obj
, u32 da
, u32 pa
, u32 prot
)
679 u32
*iopgd
= iopgd_offset(obj
, da
);
682 if ((da
| pa
) & ~IOSUPER_MASK
) {
683 dev_err(obj
->dev
, "%s: %08x:%08x should aligned on %08lx\n",
684 __func__
, da
, pa
, IOSUPER_SIZE
);
688 for (i
= 0; i
< 16; i
++)
689 *(iopgd
+ i
) = (pa
& IOSUPER_MASK
) | prot
| IOPGD_SUPER
;
690 flush_iopgd_range(iopgd
, iopgd
+ 15);
694 static int iopte_alloc_page(struct omap_iommu
*obj
, u32 da
, u32 pa
, u32 prot
)
696 u32
*iopgd
= iopgd_offset(obj
, da
);
697 u32
*iopte
= iopte_alloc(obj
, iopgd
, da
);
700 return PTR_ERR(iopte
);
702 *iopte
= (pa
& IOPAGE_MASK
) | prot
| IOPTE_SMALL
;
703 flush_iopte_range(iopte
, iopte
);
705 dev_vdbg(obj
->dev
, "%s: da:%08x pa:%08x pte:%p *pte:%08x\n",
706 __func__
, da
, pa
, iopte
, *iopte
);
711 static int iopte_alloc_large(struct omap_iommu
*obj
, u32 da
, u32 pa
, u32 prot
)
713 u32
*iopgd
= iopgd_offset(obj
, da
);
714 u32
*iopte
= iopte_alloc(obj
, iopgd
, da
);
717 if ((da
| pa
) & ~IOLARGE_MASK
) {
718 dev_err(obj
->dev
, "%s: %08x:%08x should aligned on %08lx\n",
719 __func__
, da
, pa
, IOLARGE_SIZE
);
724 return PTR_ERR(iopte
);
726 for (i
= 0; i
< 16; i
++)
727 *(iopte
+ i
) = (pa
& IOLARGE_MASK
) | prot
| IOPTE_LARGE
;
728 flush_iopte_range(iopte
, iopte
+ 15);
733 iopgtable_store_entry_core(struct omap_iommu
*obj
, struct iotlb_entry
*e
)
735 int (*fn
)(struct omap_iommu
*, u32
, u32
, u32
);
743 case MMU_CAM_PGSZ_16M
:
744 fn
= iopgd_alloc_super
;
746 case MMU_CAM_PGSZ_1M
:
747 fn
= iopgd_alloc_section
;
749 case MMU_CAM_PGSZ_64K
:
750 fn
= iopte_alloc_large
;
752 case MMU_CAM_PGSZ_4K
:
753 fn
= iopte_alloc_page
;
761 prot
= get_iopte_attr(e
);
763 spin_lock(&obj
->page_table_lock
);
764 err
= fn(obj
, e
->da
, e
->pa
, prot
);
765 spin_unlock(&obj
->page_table_lock
);
771 * omap_iopgtable_store_entry - Make an iommu pte entry
773 * @e: an iommu tlb entry info
776 omap_iopgtable_store_entry(struct omap_iommu
*obj
, struct iotlb_entry
*e
)
780 flush_iotlb_page(obj
, e
->da
);
781 err
= iopgtable_store_entry_core(obj
, e
);
783 prefetch_iotlb_entry(obj
, e
);
788 * iopgtable_lookup_entry - Lookup an iommu pte entry
790 * @da: iommu device virtual address
791 * @ppgd: iommu pgd entry pointer to be returned
792 * @ppte: iommu pte entry pointer to be returned
795 iopgtable_lookup_entry(struct omap_iommu
*obj
, u32 da
, u32
**ppgd
, u32
**ppte
)
797 u32
*iopgd
, *iopte
= NULL
;
799 iopgd
= iopgd_offset(obj
, da
);
803 if (iopgd_is_table(*iopgd
))
804 iopte
= iopte_offset(iopgd
, da
);
810 static size_t iopgtable_clear_entry_core(struct omap_iommu
*obj
, u32 da
)
813 u32
*iopgd
= iopgd_offset(obj
, da
);
819 if (iopgd_is_table(*iopgd
)) {
821 u32
*iopte
= iopte_offset(iopgd
, da
);
824 if (*iopte
& IOPTE_LARGE
) {
826 /* rewind to the 1st entry */
827 iopte
= iopte_offset(iopgd
, (da
& IOLARGE_MASK
));
830 memset(iopte
, 0, nent
* sizeof(*iopte
));
831 flush_iopte_range(iopte
, iopte
+ (nent
- 1) * sizeof(*iopte
));
834 * do table walk to check if this table is necessary or not
836 iopte
= iopte_offset(iopgd
, 0);
837 for (i
= 0; i
< PTRS_PER_IOPTE
; i
++)
842 nent
= 1; /* for the next L1 entry */
845 if ((*iopgd
& IOPGD_SUPER
) == IOPGD_SUPER
) {
847 /* rewind to the 1st entry */
848 iopgd
= iopgd_offset(obj
, (da
& IOSUPER_MASK
));
852 memset(iopgd
, 0, nent
* sizeof(*iopgd
));
853 flush_iopgd_range(iopgd
, iopgd
+ (nent
- 1) * sizeof(*iopgd
));
859 * iopgtable_clear_entry - Remove an iommu pte entry
861 * @da: iommu device virtual address
863 static size_t iopgtable_clear_entry(struct omap_iommu
*obj
, u32 da
)
867 spin_lock(&obj
->page_table_lock
);
869 bytes
= iopgtable_clear_entry_core(obj
, da
);
870 flush_iotlb_page(obj
, da
);
872 spin_unlock(&obj
->page_table_lock
);
877 static void iopgtable_clear_entry_all(struct omap_iommu
*obj
)
881 spin_lock(&obj
->page_table_lock
);
883 for (i
= 0; i
< PTRS_PER_IOPGD
; i
++) {
887 da
= i
<< IOPGD_SHIFT
;
888 iopgd
= iopgd_offset(obj
, da
);
893 if (iopgd_is_table(*iopgd
))
894 iopte_free(iopte_offset(iopgd
, 0));
897 flush_iopgd_range(iopgd
, iopgd
);
900 flush_iotlb_all(obj
);
902 spin_unlock(&obj
->page_table_lock
);
906 * Device IOMMU generic operations
908 static irqreturn_t
iommu_fault_handler(int irq
, void *data
)
912 struct omap_iommu
*obj
= data
;
913 struct iommu_domain
*domain
= obj
->domain
;
914 struct omap_iommu_domain
*omap_domain
= to_omap_domain(domain
);
916 if (!omap_domain
->iommu_dev
)
919 errs
= iommu_report_fault(obj
, &da
);
923 /* Fault callback or TLB/PTE Dynamic loading */
924 if (!report_iommu_fault(domain
, obj
->dev
, da
, 0))
929 iopgd
= iopgd_offset(obj
, da
);
931 if (!iopgd_is_table(*iopgd
)) {
932 dev_err(obj
->dev
, "%s: errs:0x%08x da:0x%08x pgd:0x%p *pgd:px%08x\n",
933 obj
->name
, errs
, da
, iopgd
, *iopgd
);
937 iopte
= iopte_offset(iopgd
, da
);
939 dev_err(obj
->dev
, "%s: errs:0x%08x da:0x%08x pgd:0x%p *pgd:0x%08x pte:0x%p *pte:0x%08x\n",
940 obj
->name
, errs
, da
, iopgd
, *iopgd
, iopte
, *iopte
);
945 static int device_match_by_alias(struct device
*dev
, void *data
)
947 struct omap_iommu
*obj
= to_iommu(dev
);
948 const char *name
= data
;
950 pr_debug("%s: %s %s\n", __func__
, obj
->name
, name
);
952 return strcmp(obj
->name
, name
) == 0;
956 * omap_iommu_attach() - attach iommu device to an iommu domain
957 * @name: name of target omap iommu device
960 static struct omap_iommu
*omap_iommu_attach(const char *name
, u32
*iopgd
)
964 struct omap_iommu
*obj
;
966 dev
= driver_find_device(&omap_iommu_driver
.driver
, NULL
,
968 device_match_by_alias
);
970 return ERR_PTR(-ENODEV
);
974 spin_lock(&obj
->iommu_lock
);
977 err
= iommu_enable(obj
);
980 flush_iotlb_all(obj
);
982 spin_unlock(&obj
->iommu_lock
);
984 dev_dbg(obj
->dev
, "%s: %s\n", __func__
, obj
->name
);
988 spin_unlock(&obj
->iommu_lock
);
993 * omap_iommu_detach - release iommu device
996 static void omap_iommu_detach(struct omap_iommu
*obj
)
998 if (!obj
|| IS_ERR(obj
))
1001 spin_lock(&obj
->iommu_lock
);
1006 spin_unlock(&obj
->iommu_lock
);
1008 dev_dbg(obj
->dev
, "%s: %s\n", __func__
, obj
->name
);
1012 * OMAP Device MMU(IOMMU) detection
1014 static int omap_iommu_probe(struct platform_device
*pdev
)
1018 struct omap_iommu
*obj
;
1019 struct resource
*res
;
1020 struct iommu_platform_data
*pdata
= dev_get_platdata(&pdev
->dev
);
1021 struct device_node
*of
= pdev
->dev
.of_node
;
1023 obj
= devm_kzalloc(&pdev
->dev
, sizeof(*obj
) + MMU_REG_SIZE
, GFP_KERNEL
);
1028 obj
->name
= dev_name(&pdev
->dev
);
1029 obj
->nr_tlb_entries
= 32;
1030 err
= of_property_read_u32(of
, "ti,#tlb-entries",
1031 &obj
->nr_tlb_entries
);
1032 if (err
&& err
!= -EINVAL
)
1034 if (obj
->nr_tlb_entries
!= 32 && obj
->nr_tlb_entries
!= 8)
1036 if (of_find_property(of
, "ti,iommu-bus-err-back", NULL
))
1037 obj
->has_bus_err_back
= MMU_GP_REG_BUS_ERR_BACK_EN
;
1039 obj
->nr_tlb_entries
= pdata
->nr_tlb_entries
;
1040 obj
->name
= pdata
->name
;
1043 obj
->dev
= &pdev
->dev
;
1044 obj
->ctx
= (void *)obj
+ sizeof(*obj
);
1046 spin_lock_init(&obj
->iommu_lock
);
1047 spin_lock_init(&obj
->page_table_lock
);
1049 res
= platform_get_resource(pdev
, IORESOURCE_MEM
, 0);
1050 obj
->regbase
= devm_ioremap_resource(obj
->dev
, res
);
1051 if (IS_ERR(obj
->regbase
))
1052 return PTR_ERR(obj
->regbase
);
1054 irq
= platform_get_irq(pdev
, 0);
1058 err
= devm_request_irq(obj
->dev
, irq
, iommu_fault_handler
, IRQF_SHARED
,
1059 dev_name(obj
->dev
), obj
);
1062 platform_set_drvdata(pdev
, obj
);
1064 pm_runtime_irq_safe(obj
->dev
);
1065 pm_runtime_enable(obj
->dev
);
1067 omap_iommu_debugfs_add(obj
);
1069 dev_info(&pdev
->dev
, "%s registered\n", obj
->name
);
1073 static int omap_iommu_remove(struct platform_device
*pdev
)
1075 struct omap_iommu
*obj
= platform_get_drvdata(pdev
);
1077 iopgtable_clear_entry_all(obj
);
1078 omap_iommu_debugfs_remove(obj
);
1080 pm_runtime_disable(obj
->dev
);
1082 dev_info(&pdev
->dev
, "%s removed\n", obj
->name
);
1086 static const struct of_device_id omap_iommu_of_match
[] = {
1087 { .compatible
= "ti,omap2-iommu" },
1088 { .compatible
= "ti,omap4-iommu" },
1089 { .compatible
= "ti,dra7-iommu" },
1092 MODULE_DEVICE_TABLE(of
, omap_iommu_of_match
);
1094 static struct platform_driver omap_iommu_driver
= {
1095 .probe
= omap_iommu_probe
,
1096 .remove
= omap_iommu_remove
,
1098 .name
= "omap-iommu",
1099 .of_match_table
= of_match_ptr(omap_iommu_of_match
),
1103 static void iopte_cachep_ctor(void *iopte
)
1105 clean_dcache_area(iopte
, IOPTE_TABLE_SIZE
);
1108 static u32
iotlb_init_entry(struct iotlb_entry
*e
, u32 da
, u32 pa
, int pgsz
)
1110 memset(e
, 0, sizeof(*e
));
1114 e
->valid
= MMU_CAM_V
;
1116 e
->endian
= MMU_RAM_ENDIAN_LITTLE
;
1117 e
->elsz
= MMU_RAM_ELSZ_8
;
1120 return iopgsz_to_bytes(e
->pgsz
);
1123 static int omap_iommu_map(struct iommu_domain
*domain
, unsigned long da
,
1124 phys_addr_t pa
, size_t bytes
, int prot
)
1126 struct omap_iommu_domain
*omap_domain
= to_omap_domain(domain
);
1127 struct omap_iommu
*oiommu
= omap_domain
->iommu_dev
;
1128 struct device
*dev
= oiommu
->dev
;
1129 struct iotlb_entry e
;
1133 omap_pgsz
= bytes_to_iopgsz(bytes
);
1134 if (omap_pgsz
< 0) {
1135 dev_err(dev
, "invalid size to map: %d\n", bytes
);
1139 dev_dbg(dev
, "mapping da 0x%lx to pa %pa size 0x%x\n", da
, &pa
, bytes
);
1141 iotlb_init_entry(&e
, da
, pa
, omap_pgsz
);
1143 ret
= omap_iopgtable_store_entry(oiommu
, &e
);
1145 dev_err(dev
, "omap_iopgtable_store_entry failed: %d\n", ret
);
1150 static size_t omap_iommu_unmap(struct iommu_domain
*domain
, unsigned long da
,
1153 struct omap_iommu_domain
*omap_domain
= to_omap_domain(domain
);
1154 struct omap_iommu
*oiommu
= omap_domain
->iommu_dev
;
1155 struct device
*dev
= oiommu
->dev
;
1157 dev_dbg(dev
, "unmapping da 0x%lx size %u\n", da
, size
);
1159 return iopgtable_clear_entry(oiommu
, da
);
1163 omap_iommu_attach_dev(struct iommu_domain
*domain
, struct device
*dev
)
1165 struct omap_iommu_domain
*omap_domain
= to_omap_domain(domain
);
1166 struct omap_iommu
*oiommu
;
1167 struct omap_iommu_arch_data
*arch_data
= dev
->archdata
.iommu
;
1170 if (!arch_data
|| !arch_data
->name
) {
1171 dev_err(dev
, "device doesn't have an associated iommu\n");
1175 spin_lock(&omap_domain
->lock
);
1177 /* only a single device is supported per domain for now */
1178 if (omap_domain
->iommu_dev
) {
1179 dev_err(dev
, "iommu domain is already attached\n");
1184 /* get a handle to and enable the omap iommu */
1185 oiommu
= omap_iommu_attach(arch_data
->name
, omap_domain
->pgtable
);
1186 if (IS_ERR(oiommu
)) {
1187 ret
= PTR_ERR(oiommu
);
1188 dev_err(dev
, "can't get omap iommu: %d\n", ret
);
1192 omap_domain
->iommu_dev
= arch_data
->iommu_dev
= oiommu
;
1193 omap_domain
->dev
= dev
;
1194 oiommu
->domain
= domain
;
1197 spin_unlock(&omap_domain
->lock
);
1201 static void _omap_iommu_detach_dev(struct omap_iommu_domain
*omap_domain
,
1204 struct omap_iommu
*oiommu
= dev_to_omap_iommu(dev
);
1205 struct omap_iommu_arch_data
*arch_data
= dev
->archdata
.iommu
;
1207 /* only a single device is supported per domain for now */
1208 if (omap_domain
->iommu_dev
!= oiommu
) {
1209 dev_err(dev
, "invalid iommu device\n");
1213 iopgtable_clear_entry_all(oiommu
);
1215 omap_iommu_detach(oiommu
);
1217 omap_domain
->iommu_dev
= arch_data
->iommu_dev
= NULL
;
1218 omap_domain
->dev
= NULL
;
1219 oiommu
->domain
= NULL
;
1222 static void omap_iommu_detach_dev(struct iommu_domain
*domain
,
1225 struct omap_iommu_domain
*omap_domain
= to_omap_domain(domain
);
1227 spin_lock(&omap_domain
->lock
);
1228 _omap_iommu_detach_dev(omap_domain
, dev
);
1229 spin_unlock(&omap_domain
->lock
);
1232 static struct iommu_domain
*omap_iommu_domain_alloc(unsigned type
)
1234 struct omap_iommu_domain
*omap_domain
;
1236 if (type
!= IOMMU_DOMAIN_UNMANAGED
)
1239 omap_domain
= kzalloc(sizeof(*omap_domain
), GFP_KERNEL
);
1241 pr_err("kzalloc failed\n");
1245 omap_domain
->pgtable
= kzalloc(IOPGD_TABLE_SIZE
, GFP_KERNEL
);
1246 if (!omap_domain
->pgtable
) {
1247 pr_err("kzalloc failed\n");
1252 * should never fail, but please keep this around to ensure
1253 * we keep the hardware happy
1255 BUG_ON(!IS_ALIGNED((long)omap_domain
->pgtable
, IOPGD_TABLE_SIZE
));
1257 clean_dcache_area(omap_domain
->pgtable
, IOPGD_TABLE_SIZE
);
1258 spin_lock_init(&omap_domain
->lock
);
1260 omap_domain
->domain
.geometry
.aperture_start
= 0;
1261 omap_domain
->domain
.geometry
.aperture_end
= (1ULL << 32) - 1;
1262 omap_domain
->domain
.geometry
.force_aperture
= true;
1264 return &omap_domain
->domain
;
1272 static void omap_iommu_domain_free(struct iommu_domain
*domain
)
1274 struct omap_iommu_domain
*omap_domain
= to_omap_domain(domain
);
1277 * An iommu device is still attached
1278 * (currently, only one device can be attached) ?
1280 if (omap_domain
->iommu_dev
)
1281 _omap_iommu_detach_dev(omap_domain
, omap_domain
->dev
);
1283 kfree(omap_domain
->pgtable
);
1287 static phys_addr_t
omap_iommu_iova_to_phys(struct iommu_domain
*domain
,
1290 struct omap_iommu_domain
*omap_domain
= to_omap_domain(domain
);
1291 struct omap_iommu
*oiommu
= omap_domain
->iommu_dev
;
1292 struct device
*dev
= oiommu
->dev
;
1294 phys_addr_t ret
= 0;
1296 iopgtable_lookup_entry(oiommu
, da
, &pgd
, &pte
);
1299 if (iopte_is_small(*pte
))
1300 ret
= omap_iommu_translate(*pte
, da
, IOPTE_MASK
);
1301 else if (iopte_is_large(*pte
))
1302 ret
= omap_iommu_translate(*pte
, da
, IOLARGE_MASK
);
1304 dev_err(dev
, "bogus pte 0x%x, da 0x%llx", *pte
,
1305 (unsigned long long)da
);
1307 if (iopgd_is_section(*pgd
))
1308 ret
= omap_iommu_translate(*pgd
, da
, IOSECTION_MASK
);
1309 else if (iopgd_is_super(*pgd
))
1310 ret
= omap_iommu_translate(*pgd
, da
, IOSUPER_MASK
);
1312 dev_err(dev
, "bogus pgd 0x%x, da 0x%llx", *pgd
,
1313 (unsigned long long)da
);
1319 static int omap_iommu_add_device(struct device
*dev
)
1321 struct omap_iommu_arch_data
*arch_data
;
1322 struct device_node
*np
;
1323 struct platform_device
*pdev
;
1326 * Allocate the archdata iommu structure for DT-based devices.
1328 * TODO: Simplify this when removing non-DT support completely from the
1334 np
= of_parse_phandle(dev
->of_node
, "iommus", 0);
1338 pdev
= of_find_device_by_node(np
);
1339 if (WARN_ON(!pdev
)) {
1344 arch_data
= kzalloc(sizeof(*arch_data
), GFP_KERNEL
);
1350 arch_data
->name
= kstrdup(dev_name(&pdev
->dev
), GFP_KERNEL
);
1351 dev
->archdata
.iommu
= arch_data
;
1358 static void omap_iommu_remove_device(struct device
*dev
)
1360 struct omap_iommu_arch_data
*arch_data
= dev
->archdata
.iommu
;
1362 if (!dev
->of_node
|| !arch_data
)
1365 kfree(arch_data
->name
);
1369 static const struct iommu_ops omap_iommu_ops
= {
1370 .domain_alloc
= omap_iommu_domain_alloc
,
1371 .domain_free
= omap_iommu_domain_free
,
1372 .attach_dev
= omap_iommu_attach_dev
,
1373 .detach_dev
= omap_iommu_detach_dev
,
1374 .map
= omap_iommu_map
,
1375 .unmap
= omap_iommu_unmap
,
1376 .map_sg
= default_iommu_map_sg
,
1377 .iova_to_phys
= omap_iommu_iova_to_phys
,
1378 .add_device
= omap_iommu_add_device
,
1379 .remove_device
= omap_iommu_remove_device
,
1380 .pgsize_bitmap
= OMAP_IOMMU_PGSIZES
,
1383 static int __init
omap_iommu_init(void)
1385 struct kmem_cache
*p
;
1386 const unsigned long flags
= SLAB_HWCACHE_ALIGN
;
1387 size_t align
= 1 << 10; /* L2 pagetable alignement */
1388 struct device_node
*np
;
1390 np
= of_find_matching_node(NULL
, omap_iommu_of_match
);
1396 p
= kmem_cache_create("iopte_cache", IOPTE_TABLE_SIZE
, align
, flags
,
1402 bus_set_iommu(&platform_bus_type
, &omap_iommu_ops
);
1404 omap_iommu_debugfs_init();
1406 return platform_driver_register(&omap_iommu_driver
);
1408 /* must be ready before omap3isp is probed */
1409 subsys_initcall(omap_iommu_init
);
1411 static void __exit
omap_iommu_exit(void)
1413 kmem_cache_destroy(iopte_cachep
);
1415 platform_driver_unregister(&omap_iommu_driver
);
1417 omap_iommu_debugfs_exit();
1419 module_exit(omap_iommu_exit
);
1421 MODULE_DESCRIPTION("omap iommu: tlb and pagetable primitives");
1422 MODULE_ALIAS("platform:omap-iommu");
1423 MODULE_AUTHOR("Hiroshi DOYU, Paul Mundt and Toshihiro Kobayashi");
1424 MODULE_LICENSE("GPL v2");