4 * Copyright (C) 2014 Renesas Electronics Corporation
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License as published by
8 * the Free Software Foundation; version 2 of the License.
11 #include <linux/delay.h>
12 #include <linux/dma-mapping.h>
13 #include <linux/err.h>
14 #include <linux/export.h>
15 #include <linux/interrupt.h>
17 #include <linux/iommu.h>
18 #include <linux/module.h>
20 #include <linux/platform_device.h>
21 #include <linux/sizes.h>
22 #include <linux/slab.h>
24 #include <asm/dma-iommu.h>
25 #include <asm/pgalloc.h>
27 struct ipmmu_vmsa_device
{
30 struct list_head list
;
32 unsigned int num_utlbs
;
34 struct dma_iommu_mapping
*mapping
;
37 struct ipmmu_vmsa_domain
{
38 struct ipmmu_vmsa_device
*mmu
;
39 struct iommu_domain
*io_domain
;
41 unsigned int context_id
;
42 spinlock_t lock
; /* Protects mappings */
46 struct ipmmu_vmsa_archdata
{
47 struct ipmmu_vmsa_device
*mmu
;
49 unsigned int num_utlbs
;
52 static DEFINE_SPINLOCK(ipmmu_devices_lock
);
53 static LIST_HEAD(ipmmu_devices
);
55 #define TLB_LOOP_TIMEOUT 100 /* 100us */
57 /* -----------------------------------------------------------------------------
58 * Registers Definition
61 #define IM_NS_ALIAS_OFFSET 0x800
63 #define IM_CTX_SIZE 0x40
66 #define IMCTR_TRE (1 << 17)
67 #define IMCTR_AFE (1 << 16)
68 #define IMCTR_RTSEL_MASK (3 << 4)
69 #define IMCTR_RTSEL_SHIFT 4
70 #define IMCTR_TREN (1 << 3)
71 #define IMCTR_INTEN (1 << 2)
72 #define IMCTR_FLUSH (1 << 1)
73 #define IMCTR_MMUEN (1 << 0)
77 #define IMTTBCR 0x0008
78 #define IMTTBCR_EAE (1 << 31)
79 #define IMTTBCR_PMB (1 << 30)
80 #define IMTTBCR_SH1_NON_SHAREABLE (0 << 28)
81 #define IMTTBCR_SH1_OUTER_SHAREABLE (2 << 28)
82 #define IMTTBCR_SH1_INNER_SHAREABLE (3 << 28)
83 #define IMTTBCR_SH1_MASK (3 << 28)
84 #define IMTTBCR_ORGN1_NC (0 << 26)
85 #define IMTTBCR_ORGN1_WB_WA (1 << 26)
86 #define IMTTBCR_ORGN1_WT (2 << 26)
87 #define IMTTBCR_ORGN1_WB (3 << 26)
88 #define IMTTBCR_ORGN1_MASK (3 << 26)
89 #define IMTTBCR_IRGN1_NC (0 << 24)
90 #define IMTTBCR_IRGN1_WB_WA (1 << 24)
91 #define IMTTBCR_IRGN1_WT (2 << 24)
92 #define IMTTBCR_IRGN1_WB (3 << 24)
93 #define IMTTBCR_IRGN1_MASK (3 << 24)
94 #define IMTTBCR_TSZ1_MASK (7 << 16)
95 #define IMTTBCR_TSZ1_SHIFT 16
96 #define IMTTBCR_SH0_NON_SHAREABLE (0 << 12)
97 #define IMTTBCR_SH0_OUTER_SHAREABLE (2 << 12)
98 #define IMTTBCR_SH0_INNER_SHAREABLE (3 << 12)
99 #define IMTTBCR_SH0_MASK (3 << 12)
100 #define IMTTBCR_ORGN0_NC (0 << 10)
101 #define IMTTBCR_ORGN0_WB_WA (1 << 10)
102 #define IMTTBCR_ORGN0_WT (2 << 10)
103 #define IMTTBCR_ORGN0_WB (3 << 10)
104 #define IMTTBCR_ORGN0_MASK (3 << 10)
105 #define IMTTBCR_IRGN0_NC (0 << 8)
106 #define IMTTBCR_IRGN0_WB_WA (1 << 8)
107 #define IMTTBCR_IRGN0_WT (2 << 8)
108 #define IMTTBCR_IRGN0_WB (3 << 8)
109 #define IMTTBCR_IRGN0_MASK (3 << 8)
110 #define IMTTBCR_SL0_LVL_2 (0 << 4)
111 #define IMTTBCR_SL0_LVL_1 (1 << 4)
112 #define IMTTBCR_TSZ0_MASK (7 << 0)
113 #define IMTTBCR_TSZ0_SHIFT O
115 #define IMBUSCR 0x000c
116 #define IMBUSCR_DVM (1 << 2)
117 #define IMBUSCR_BUSSEL_SYS (0 << 0)
118 #define IMBUSCR_BUSSEL_CCI (1 << 0)
119 #define IMBUSCR_BUSSEL_IMCAAR (2 << 0)
120 #define IMBUSCR_BUSSEL_CCI_IMCAAR (3 << 0)
121 #define IMBUSCR_BUSSEL_MASK (3 << 0)
123 #define IMTTLBR0 0x0010
124 #define IMTTUBR0 0x0014
125 #define IMTTLBR1 0x0018
126 #define IMTTUBR1 0x001c
129 #define IMSTR_ERRLVL_MASK (3 << 12)
130 #define IMSTR_ERRLVL_SHIFT 12
131 #define IMSTR_ERRCODE_TLB_FORMAT (1 << 8)
132 #define IMSTR_ERRCODE_ACCESS_PERM (4 << 8)
133 #define IMSTR_ERRCODE_SECURE_ACCESS (5 << 8)
134 #define IMSTR_ERRCODE_MASK (7 << 8)
135 #define IMSTR_MHIT (1 << 4)
136 #define IMSTR_ABORT (1 << 2)
137 #define IMSTR_PF (1 << 1)
138 #define IMSTR_TF (1 << 0)
140 #define IMMAIR0 0x0028
141 #define IMMAIR1 0x002c
142 #define IMMAIR_ATTR_MASK 0xff
143 #define IMMAIR_ATTR_DEVICE 0x04
144 #define IMMAIR_ATTR_NC 0x44
145 #define IMMAIR_ATTR_WBRWA 0xff
146 #define IMMAIR_ATTR_SHIFT(n) ((n) << 3)
147 #define IMMAIR_ATTR_IDX_NC 0
148 #define IMMAIR_ATTR_IDX_WBRWA 1
149 #define IMMAIR_ATTR_IDX_DEV 2
153 #define IMPCTR 0x0200
154 #define IMPSTR 0x0208
155 #define IMPEAR 0x020c
156 #define IMPMBA(n) (0x0280 + ((n) * 4))
157 #define IMPMBD(n) (0x02c0 + ((n) * 4))
159 #define IMUCTR(n) (0x0300 + ((n) * 16))
160 #define IMUCTR_FIXADDEN (1 << 31)
161 #define IMUCTR_FIXADD_MASK (0xff << 16)
162 #define IMUCTR_FIXADD_SHIFT 16
163 #define IMUCTR_TTSEL_MMU(n) ((n) << 4)
164 #define IMUCTR_TTSEL_PMB (8 << 4)
165 #define IMUCTR_TTSEL_MASK (15 << 4)
166 #define IMUCTR_FLUSH (1 << 1)
167 #define IMUCTR_MMUEN (1 << 0)
169 #define IMUASID(n) (0x0308 + ((n) * 16))
170 #define IMUASID_ASID8_MASK (0xff << 8)
171 #define IMUASID_ASID8_SHIFT 8
172 #define IMUASID_ASID0_MASK (0xff << 0)
173 #define IMUASID_ASID0_SHIFT 0
175 /* -----------------------------------------------------------------------------
180 * VMSA states in section B3.6.3 "Control of Secure or Non-secure memory access,
181 * Long-descriptor format" that the NStable bit being set in a table descriptor
182 * will result in the NStable and NS bits of all child entries being ignored and
183 * considered as being set. The IPMMU seems not to comply with this, as it
184 * generates a secure access page fault if any of the NStable and NS bits isn't
185 * set when running in non-secure mode.
188 #define PMD_NSTABLE (_AT(pmdval_t, 1) << 63)
191 #define ARM_VMSA_PTE_XN (((pteval_t)3) << 53)
192 #define ARM_VMSA_PTE_CONT (((pteval_t)1) << 52)
193 #define ARM_VMSA_PTE_AF (((pteval_t)1) << 10)
194 #define ARM_VMSA_PTE_SH_NS (((pteval_t)0) << 8)
195 #define ARM_VMSA_PTE_SH_OS (((pteval_t)2) << 8)
196 #define ARM_VMSA_PTE_SH_IS (((pteval_t)3) << 8)
197 #define ARM_VMSA_PTE_SH_MASK (((pteval_t)3) << 8)
198 #define ARM_VMSA_PTE_NS (((pteval_t)1) << 5)
199 #define ARM_VMSA_PTE_PAGE (((pteval_t)3) << 0)
202 #define ARM_VMSA_PTE_nG (((pteval_t)1) << 11)
203 #define ARM_VMSA_PTE_AP_UNPRIV (((pteval_t)1) << 6)
204 #define ARM_VMSA_PTE_AP_RDONLY (((pteval_t)2) << 6)
205 #define ARM_VMSA_PTE_AP_MASK (((pteval_t)3) << 6)
206 #define ARM_VMSA_PTE_ATTRINDX_MASK (((pteval_t)3) << 2)
207 #define ARM_VMSA_PTE_ATTRINDX_SHIFT 2
209 #define ARM_VMSA_PTE_ATTRS_MASK \
210 (ARM_VMSA_PTE_XN | ARM_VMSA_PTE_CONT | ARM_VMSA_PTE_nG | \
211 ARM_VMSA_PTE_AF | ARM_VMSA_PTE_SH_MASK | ARM_VMSA_PTE_AP_MASK | \
212 ARM_VMSA_PTE_NS | ARM_VMSA_PTE_ATTRINDX_MASK)
214 #define ARM_VMSA_PTE_CONT_ENTRIES 16
215 #define ARM_VMSA_PTE_CONT_SIZE (PAGE_SIZE * ARM_VMSA_PTE_CONT_ENTRIES)
217 #define IPMMU_PTRS_PER_PTE 512
218 #define IPMMU_PTRS_PER_PMD 512
219 #define IPMMU_PTRS_PER_PGD 4
221 /* -----------------------------------------------------------------------------
225 static u32
ipmmu_read(struct ipmmu_vmsa_device
*mmu
, unsigned int offset
)
227 return ioread32(mmu
->base
+ offset
);
230 static void ipmmu_write(struct ipmmu_vmsa_device
*mmu
, unsigned int offset
,
233 iowrite32(data
, mmu
->base
+ offset
);
236 static u32
ipmmu_ctx_read(struct ipmmu_vmsa_domain
*domain
, unsigned int reg
)
238 return ipmmu_read(domain
->mmu
, domain
->context_id
* IM_CTX_SIZE
+ reg
);
241 static void ipmmu_ctx_write(struct ipmmu_vmsa_domain
*domain
, unsigned int reg
,
244 ipmmu_write(domain
->mmu
, domain
->context_id
* IM_CTX_SIZE
+ reg
, data
);
247 /* -----------------------------------------------------------------------------
248 * TLB and microTLB Management
251 /* Wait for any pending TLB invalidations to complete */
252 static void ipmmu_tlb_sync(struct ipmmu_vmsa_domain
*domain
)
254 unsigned int count
= 0;
256 while (ipmmu_ctx_read(domain
, IMCTR
) & IMCTR_FLUSH
) {
258 if (++count
== TLB_LOOP_TIMEOUT
) {
259 dev_err_ratelimited(domain
->mmu
->dev
,
260 "TLB sync timed out -- MMU may be deadlocked\n");
267 static void ipmmu_tlb_invalidate(struct ipmmu_vmsa_domain
*domain
)
271 reg
= ipmmu_ctx_read(domain
, IMCTR
);
273 ipmmu_ctx_write(domain
, IMCTR
, reg
);
275 ipmmu_tlb_sync(domain
);
279 * Enable MMU translation for the microTLB.
281 static void ipmmu_utlb_enable(struct ipmmu_vmsa_domain
*domain
,
284 struct ipmmu_vmsa_device
*mmu
= domain
->mmu
;
287 * TODO: Reference-count the microTLB as several bus masters can be
288 * connected to the same microTLB.
291 /* TODO: What should we set the ASID to ? */
292 ipmmu_write(mmu
, IMUASID(utlb
), 0);
293 /* TODO: Do we need to flush the microTLB ? */
294 ipmmu_write(mmu
, IMUCTR(utlb
),
295 IMUCTR_TTSEL_MMU(domain
->context_id
) | IMUCTR_FLUSH
|
300 * Disable MMU translation for the microTLB.
302 static void ipmmu_utlb_disable(struct ipmmu_vmsa_domain
*domain
,
305 struct ipmmu_vmsa_device
*mmu
= domain
->mmu
;
307 ipmmu_write(mmu
, IMUCTR(utlb
), 0);
310 static void ipmmu_flush_pgtable(struct ipmmu_vmsa_device
*mmu
, void *addr
,
313 unsigned long offset
= (unsigned long)addr
& ~PAGE_MASK
;
316 * TODO: Add support for coherent walk through CCI with DVM and remove
319 dma_map_page(mmu
->dev
, virt_to_page(addr
), offset
, size
, DMA_TO_DEVICE
);
322 /* -----------------------------------------------------------------------------
323 * Domain/Context Management
326 static int ipmmu_domain_init_context(struct ipmmu_vmsa_domain
*domain
)
332 * TODO: When adding support for multiple contexts, find an unused
335 domain
->context_id
= 0;
338 ipmmu_flush_pgtable(domain
->mmu
, domain
->pgd
,
339 IPMMU_PTRS_PER_PGD
* sizeof(*domain
->pgd
));
340 ttbr
= __pa(domain
->pgd
);
341 ipmmu_ctx_write(domain
, IMTTLBR0
, ttbr
);
342 ipmmu_ctx_write(domain
, IMTTUBR0
, ttbr
>> 32);
346 * We use long descriptors with inner-shareable WBWA tables and allocate
347 * the whole 32-bit VA space to TTBR0.
349 ipmmu_ctx_write(domain
, IMTTBCR
, IMTTBCR_EAE
|
350 IMTTBCR_SH0_INNER_SHAREABLE
| IMTTBCR_ORGN0_WB_WA
|
351 IMTTBCR_IRGN0_WB_WA
| IMTTBCR_SL0_LVL_1
);
355 * We need three attributes only, non-cacheable, write-back read/write
356 * allocate and device memory.
358 reg
= (IMMAIR_ATTR_NC
<< IMMAIR_ATTR_SHIFT(IMMAIR_ATTR_IDX_NC
))
359 | (IMMAIR_ATTR_WBRWA
<< IMMAIR_ATTR_SHIFT(IMMAIR_ATTR_IDX_WBRWA
))
360 | (IMMAIR_ATTR_DEVICE
<< IMMAIR_ATTR_SHIFT(IMMAIR_ATTR_IDX_DEV
));
361 ipmmu_ctx_write(domain
, IMMAIR0
, reg
);
364 ipmmu_ctx_write(domain
, IMBUSCR
,
365 ipmmu_ctx_read(domain
, IMBUSCR
) &
366 ~(IMBUSCR_DVM
| IMBUSCR_BUSSEL_MASK
));
370 * Clear all interrupt flags.
372 ipmmu_ctx_write(domain
, IMSTR
, ipmmu_ctx_read(domain
, IMSTR
));
376 * Enable the MMU and interrupt generation. The long-descriptor
377 * translation table format doesn't use TEX remapping. Don't enable AF
378 * software management as we have no use for it. Flush the TLB as
379 * required when modifying the context registers.
381 ipmmu_ctx_write(domain
, IMCTR
, IMCTR_INTEN
| IMCTR_FLUSH
| IMCTR_MMUEN
);
386 static void ipmmu_domain_destroy_context(struct ipmmu_vmsa_domain
*domain
)
389 * Disable the context. Flush the TLB as required when modifying the
392 * TODO: Is TLB flush really needed ?
394 ipmmu_ctx_write(domain
, IMCTR
, IMCTR_FLUSH
);
395 ipmmu_tlb_sync(domain
);
398 /* -----------------------------------------------------------------------------
402 static irqreturn_t
ipmmu_domain_irq(struct ipmmu_vmsa_domain
*domain
)
404 const u32 err_mask
= IMSTR_MHIT
| IMSTR_ABORT
| IMSTR_PF
| IMSTR_TF
;
405 struct ipmmu_vmsa_device
*mmu
= domain
->mmu
;
409 status
= ipmmu_ctx_read(domain
, IMSTR
);
410 if (!(status
& err_mask
))
413 iova
= ipmmu_ctx_read(domain
, IMEAR
);
416 * Clear the error status flags. Unlike traditional interrupt flag
417 * registers that must be cleared by writing 1, this status register
418 * seems to require 0. The error address register must be read before,
419 * otherwise its value will be 0.
421 ipmmu_ctx_write(domain
, IMSTR
, 0);
423 /* Log fatal errors. */
424 if (status
& IMSTR_MHIT
)
425 dev_err_ratelimited(mmu
->dev
, "Multiple TLB hits @0x%08x\n",
427 if (status
& IMSTR_ABORT
)
428 dev_err_ratelimited(mmu
->dev
, "Page Table Walk Abort @0x%08x\n",
431 if (!(status
& (IMSTR_PF
| IMSTR_TF
)))
435 * Try to handle page faults and translation faults.
437 * TODO: We need to look up the faulty device based on the I/O VA. Use
438 * the IOMMU device for now.
440 if (!report_iommu_fault(domain
->io_domain
, mmu
->dev
, iova
, 0))
443 dev_err_ratelimited(mmu
->dev
,
444 "Unhandled fault: status 0x%08x iova 0x%08x\n",
450 static irqreturn_t
ipmmu_irq(int irq
, void *dev
)
452 struct ipmmu_vmsa_device
*mmu
= dev
;
453 struct iommu_domain
*io_domain
;
454 struct ipmmu_vmsa_domain
*domain
;
459 io_domain
= mmu
->mapping
->domain
;
460 domain
= io_domain
->priv
;
462 return ipmmu_domain_irq(domain
);
465 /* -----------------------------------------------------------------------------
466 * Page Table Management
469 #define pud_pgtable(pud) pfn_to_page(__phys_to_pfn(pud_val(pud) & PHYS_MASK))
471 static void ipmmu_free_ptes(pmd_t
*pmd
)
473 pgtable_t table
= pmd_pgtable(*pmd
);
477 static void ipmmu_free_pmds(pud_t
*pud
)
479 pmd_t
*pmd
= pmd_offset(pud
, 0);
483 for (i
= 0; i
< IPMMU_PTRS_PER_PMD
; ++i
) {
484 if (!pmd_table(*pmd
))
487 ipmmu_free_ptes(pmd
);
491 table
= pud_pgtable(*pud
);
495 static void ipmmu_free_pgtables(struct ipmmu_vmsa_domain
*domain
)
497 pgd_t
*pgd
, *pgd_base
= domain
->pgd
;
501 * Recursively free the page tables for this domain. We don't care about
502 * speculative TLB filling, because the TLB will be nuked next time this
503 * context bank is re-allocated and no devices currently map to these
507 for (i
= 0; i
< IPMMU_PTRS_PER_PGD
; ++i
) {
510 ipmmu_free_pmds((pud_t
*)pgd
);
518 * We can't use the (pgd|pud|pmd|pte)_populate or the set_(pgd|pud|pmd|pte)
519 * functions as they would flush the CPU TLB.
522 static pte_t
*ipmmu_alloc_pte(struct ipmmu_vmsa_device
*mmu
, pmd_t
*pmd
,
528 return pte_offset_kernel(pmd
, iova
);
530 pte
= (pte_t
*)get_zeroed_page(GFP_ATOMIC
);
534 ipmmu_flush_pgtable(mmu
, pte
, PAGE_SIZE
);
535 *pmd
= __pmd(__pa(pte
) | PMD_NSTABLE
| PMD_TYPE_TABLE
);
536 ipmmu_flush_pgtable(mmu
, pmd
, sizeof(*pmd
));
538 return pte
+ pte_index(iova
);
541 static pmd_t
*ipmmu_alloc_pmd(struct ipmmu_vmsa_device
*mmu
, pgd_t
*pgd
,
544 pud_t
*pud
= (pud_t
*)pgd
;
548 return pmd_offset(pud
, iova
);
550 pmd
= (pmd_t
*)get_zeroed_page(GFP_ATOMIC
);
554 ipmmu_flush_pgtable(mmu
, pmd
, PAGE_SIZE
);
555 *pud
= __pud(__pa(pmd
) | PMD_NSTABLE
| PMD_TYPE_TABLE
);
556 ipmmu_flush_pgtable(mmu
, pud
, sizeof(*pud
));
558 return pmd
+ pmd_index(iova
);
561 static u64
ipmmu_page_prot(unsigned int prot
, u64 type
)
563 u64 pgprot
= ARM_VMSA_PTE_nG
| ARM_VMSA_PTE_AF
564 | ARM_VMSA_PTE_SH_IS
| ARM_VMSA_PTE_AP_UNPRIV
565 | ARM_VMSA_PTE_NS
| type
;
567 if (!(prot
& IOMMU_WRITE
) && (prot
& IOMMU_READ
))
568 pgprot
|= ARM_VMSA_PTE_AP_RDONLY
;
570 if (prot
& IOMMU_CACHE
)
571 pgprot
|= IMMAIR_ATTR_IDX_WBRWA
<< ARM_VMSA_PTE_ATTRINDX_SHIFT
;
573 if (prot
& IOMMU_NOEXEC
)
574 pgprot
|= ARM_VMSA_PTE_XN
;
575 else if (!(prot
& (IOMMU_READ
| IOMMU_WRITE
)))
576 /* If no access create a faulting entry to avoid TLB fills. */
577 pgprot
&= ~ARM_VMSA_PTE_PAGE
;
582 static int ipmmu_alloc_init_pte(struct ipmmu_vmsa_device
*mmu
, pmd_t
*pmd
,
583 unsigned long iova
, unsigned long pfn
,
584 size_t size
, int prot
)
586 pteval_t pteval
= ipmmu_page_prot(prot
, ARM_VMSA_PTE_PAGE
);
587 unsigned int num_ptes
= 1;
591 pte
= ipmmu_alloc_pte(mmu
, pmd
, iova
);
598 * Install the page table entries. We can be called both for a single
599 * page or for a block of 16 physically contiguous pages. In the latter
600 * case set the PTE contiguous hint.
602 if (size
== SZ_64K
) {
603 pteval
|= ARM_VMSA_PTE_CONT
;
604 num_ptes
= ARM_VMSA_PTE_CONT_ENTRIES
;
607 for (i
= num_ptes
; i
; --i
)
608 *pte
++ = pfn_pte(pfn
++, __pgprot(pteval
));
610 ipmmu_flush_pgtable(mmu
, start
, sizeof(*pte
) * num_ptes
);
615 static int ipmmu_alloc_init_pmd(struct ipmmu_vmsa_device
*mmu
, pmd_t
*pmd
,
616 unsigned long iova
, unsigned long pfn
,
619 pmdval_t pmdval
= ipmmu_page_prot(prot
, PMD_TYPE_SECT
);
621 *pmd
= pfn_pmd(pfn
, __pgprot(pmdval
));
622 ipmmu_flush_pgtable(mmu
, pmd
, sizeof(*pmd
));
627 static int ipmmu_create_mapping(struct ipmmu_vmsa_domain
*domain
,
628 unsigned long iova
, phys_addr_t paddr
,
629 size_t size
, int prot
)
631 struct ipmmu_vmsa_device
*mmu
= domain
->mmu
;
632 pgd_t
*pgd
= domain
->pgd
;
641 if (size
& ~PAGE_MASK
)
644 if (paddr
& ~((1ULL << 40) - 1))
647 pfn
= __phys_to_pfn(paddr
);
648 pgd
+= pgd_index(iova
);
650 /* Update the page tables. */
651 spin_lock_irqsave(&domain
->lock
, flags
);
653 pmd
= ipmmu_alloc_pmd(mmu
, pgd
, iova
);
661 ret
= ipmmu_alloc_init_pmd(mmu
, pmd
, iova
, pfn
, prot
);
665 ret
= ipmmu_alloc_init_pte(mmu
, pmd
, iova
, pfn
, size
, prot
);
673 spin_unlock_irqrestore(&domain
->lock
, flags
);
676 ipmmu_tlb_invalidate(domain
);
681 static void ipmmu_clear_pud(struct ipmmu_vmsa_device
*mmu
, pud_t
*pud
)
683 pgtable_t table
= pud_pgtable(*pud
);
687 ipmmu_flush_pgtable(mmu
, pud
, sizeof(*pud
));
689 /* Free the page table. */
693 static void ipmmu_clear_pmd(struct ipmmu_vmsa_device
*mmu
, pud_t
*pud
,
701 ipmmu_flush_pgtable(mmu
, pmd
, sizeof(*pmd
));
703 /* Free the page table. */
704 if (pmd_table(pmdval
)) {
705 pgtable_t table
= pmd_pgtable(pmdval
);
710 /* Check whether the PUD is still needed. */
711 pmd
= pmd_offset(pud
, 0);
712 for (i
= 0; i
< IPMMU_PTRS_PER_PMD
; ++i
) {
713 if (!pmd_none(pmd
[i
]))
717 /* Clear the parent PUD. */
718 ipmmu_clear_pud(mmu
, pud
);
721 static void ipmmu_clear_pte(struct ipmmu_vmsa_device
*mmu
, pud_t
*pud
,
722 pmd_t
*pmd
, pte_t
*pte
, unsigned int num_ptes
)
727 for (i
= num_ptes
; i
; --i
)
730 ipmmu_flush_pgtable(mmu
, pte
, sizeof(*pte
) * num_ptes
);
732 /* Check whether the PMD is still needed. */
733 pte
= pte_offset_kernel(pmd
, 0);
734 for (i
= 0; i
< IPMMU_PTRS_PER_PTE
; ++i
) {
735 if (!pte_none(pte
[i
]))
739 /* Clear the parent PMD. */
740 ipmmu_clear_pmd(mmu
, pud
, pmd
);
743 static int ipmmu_split_pmd(struct ipmmu_vmsa_device
*mmu
, pmd_t
*pmd
)
750 pte
= (pte_t
*)get_zeroed_page(GFP_ATOMIC
);
754 /* Copy the PMD attributes. */
755 pteval
= (pmd_val(*pmd
) & ARM_VMSA_PTE_ATTRS_MASK
)
756 | ARM_VMSA_PTE_CONT
| ARM_VMSA_PTE_PAGE
;
761 for (i
= IPMMU_PTRS_PER_PTE
; i
; --i
)
762 *pte
++ = pfn_pte(pfn
++, __pgprot(pteval
));
764 ipmmu_flush_pgtable(mmu
, start
, PAGE_SIZE
);
765 *pmd
= __pmd(__pa(start
) | PMD_NSTABLE
| PMD_TYPE_TABLE
);
766 ipmmu_flush_pgtable(mmu
, pmd
, sizeof(*pmd
));
771 static void ipmmu_split_pte(struct ipmmu_vmsa_device
*mmu
, pte_t
*pte
)
775 for (i
= ARM_VMSA_PTE_CONT_ENTRIES
; i
; --i
)
776 pte
[i
-1] = __pte(pte_val(*pte
) & ~ARM_VMSA_PTE_CONT
);
778 ipmmu_flush_pgtable(mmu
, pte
, sizeof(*pte
) * ARM_VMSA_PTE_CONT_ENTRIES
);
781 static int ipmmu_clear_mapping(struct ipmmu_vmsa_domain
*domain
,
782 unsigned long iova
, size_t size
)
784 struct ipmmu_vmsa_device
*mmu
= domain
->mmu
;
786 pgd_t
*pgd
= domain
->pgd
;
794 if (size
& ~PAGE_MASK
)
797 pgd
+= pgd_index(iova
);
800 spin_lock_irqsave(&domain
->lock
, flags
);
802 /* If there's no PUD or PMD we're done. */
806 pmd
= pmd_offset(pud
, iova
);
811 * When freeing a 2MB block just clear the PMD. In the unlikely case the
812 * block is mapped as individual pages this will free the corresponding
816 ipmmu_clear_pmd(mmu
, pud
, pmd
);
821 * If the PMD has been mapped as a section remap it as pages to allow
822 * freeing individual pages.
825 ipmmu_split_pmd(mmu
, pmd
);
827 pte
= pte_offset_kernel(pmd
, iova
);
830 * When freeing a 64kB block just clear the PTE entries. We don't have
831 * to care about the contiguous hint of the surrounding entries.
833 if (size
== SZ_64K
) {
834 ipmmu_clear_pte(mmu
, pud
, pmd
, pte
, ARM_VMSA_PTE_CONT_ENTRIES
);
839 * If the PTE has been mapped with the contiguous hint set remap it and
840 * its surrounding PTEs to allow unmapping a single page.
842 if (pte_val(*pte
) & ARM_VMSA_PTE_CONT
)
843 ipmmu_split_pte(mmu
, pte
);
846 ipmmu_clear_pte(mmu
, pud
, pmd
, pte
, 1);
849 spin_unlock_irqrestore(&domain
->lock
, flags
);
851 ipmmu_tlb_invalidate(domain
);
856 /* -----------------------------------------------------------------------------
860 static int ipmmu_domain_init(struct iommu_domain
*io_domain
)
862 struct ipmmu_vmsa_domain
*domain
;
864 domain
= kzalloc(sizeof(*domain
), GFP_KERNEL
);
868 spin_lock_init(&domain
->lock
);
870 domain
->pgd
= kzalloc(IPMMU_PTRS_PER_PGD
* sizeof(pgd_t
), GFP_KERNEL
);
876 io_domain
->priv
= domain
;
877 domain
->io_domain
= io_domain
;
882 static void ipmmu_domain_destroy(struct iommu_domain
*io_domain
)
884 struct ipmmu_vmsa_domain
*domain
= io_domain
->priv
;
887 * Free the domain resources. We assume that all devices have already
890 ipmmu_domain_destroy_context(domain
);
891 ipmmu_free_pgtables(domain
);
895 static int ipmmu_attach_device(struct iommu_domain
*io_domain
,
898 struct ipmmu_vmsa_archdata
*archdata
= dev
->archdata
.iommu
;
899 struct ipmmu_vmsa_device
*mmu
= archdata
->mmu
;
900 struct ipmmu_vmsa_domain
*domain
= io_domain
->priv
;
906 dev_err(dev
, "Cannot attach to IPMMU\n");
910 spin_lock_irqsave(&domain
->lock
, flags
);
913 /* The domain hasn't been used yet, initialize it. */
915 ret
= ipmmu_domain_init_context(domain
);
916 } else if (domain
->mmu
!= mmu
) {
918 * Something is wrong, we can't attach two devices using
919 * different IOMMUs to the same domain.
921 dev_err(dev
, "Can't attach IPMMU %s to domain on IPMMU %s\n",
922 dev_name(mmu
->dev
), dev_name(domain
->mmu
->dev
));
926 spin_unlock_irqrestore(&domain
->lock
, flags
);
931 for (i
= 0; i
< archdata
->num_utlbs
; ++i
)
932 ipmmu_utlb_enable(domain
, archdata
->utlbs
[i
]);
937 static void ipmmu_detach_device(struct iommu_domain
*io_domain
,
940 struct ipmmu_vmsa_archdata
*archdata
= dev
->archdata
.iommu
;
941 struct ipmmu_vmsa_domain
*domain
= io_domain
->priv
;
944 for (i
= 0; i
< archdata
->num_utlbs
; ++i
)
945 ipmmu_utlb_disable(domain
, archdata
->utlbs
[i
]);
948 * TODO: Optimize by disabling the context when no device is attached.
952 static int ipmmu_map(struct iommu_domain
*io_domain
, unsigned long iova
,
953 phys_addr_t paddr
, size_t size
, int prot
)
955 struct ipmmu_vmsa_domain
*domain
= io_domain
->priv
;
960 return ipmmu_create_mapping(domain
, iova
, paddr
, size
, prot
);
963 static size_t ipmmu_unmap(struct iommu_domain
*io_domain
, unsigned long iova
,
966 struct ipmmu_vmsa_domain
*domain
= io_domain
->priv
;
969 ret
= ipmmu_clear_mapping(domain
, iova
, size
);
970 return ret
? 0 : size
;
973 static phys_addr_t
ipmmu_iova_to_phys(struct iommu_domain
*io_domain
,
976 struct ipmmu_vmsa_domain
*domain
= io_domain
->priv
;
982 /* TODO: Is locking needed ? */
987 pgd
= *(domain
->pgd
+ pgd_index(iova
));
991 pud
= *pud_offset(&pgd
, iova
);
995 pmd
= *pmd_offset(&pud
, iova
);
1000 return __pfn_to_phys(pmd_pfn(pmd
)) | (iova
& ~PMD_MASK
);
1002 pte
= *(pmd_page_vaddr(pmd
) + pte_index(iova
));
1006 return __pfn_to_phys(pte_pfn(pte
)) | (iova
& ~PAGE_MASK
);
1009 static int ipmmu_find_utlbs(struct ipmmu_vmsa_device
*mmu
, struct device
*dev
,
1010 unsigned int **_utlbs
)
1012 unsigned int *utlbs
;
1016 count
= of_count_phandle_with_args(dev
->of_node
, "iommus",
1021 utlbs
= kcalloc(count
, sizeof(*utlbs
), GFP_KERNEL
);
1025 for (i
= 0; i
< count
; ++i
) {
1026 struct of_phandle_args args
;
1029 ret
= of_parse_phandle_with_args(dev
->of_node
, "iommus",
1030 "#iommu-cells", i
, &args
);
1034 of_node_put(args
.np
);
1036 if (args
.np
!= mmu
->dev
->of_node
|| args
.args_count
!= 1)
1039 utlbs
[i
] = args
.args
[0];
1051 static int ipmmu_add_device(struct device
*dev
)
1053 struct ipmmu_vmsa_archdata
*archdata
;
1054 struct ipmmu_vmsa_device
*mmu
;
1055 struct iommu_group
*group
= NULL
;
1056 unsigned int *utlbs
= NULL
;
1061 if (dev
->archdata
.iommu
) {
1062 dev_warn(dev
, "IOMMU driver already assigned to device %s\n",
1067 /* Find the master corresponding to the device. */
1068 spin_lock(&ipmmu_devices_lock
);
1070 list_for_each_entry(mmu
, &ipmmu_devices
, list
) {
1071 num_utlbs
= ipmmu_find_utlbs(mmu
, dev
, &utlbs
);
1074 * TODO Take a reference to the MMU to protect
1075 * against device removal.
1081 spin_unlock(&ipmmu_devices_lock
);
1086 for (i
= 0; i
< num_utlbs
; ++i
) {
1087 if (utlbs
[i
] >= mmu
->num_utlbs
) {
1093 /* Create a device group and add the device to it. */
1094 group
= iommu_group_alloc();
1095 if (IS_ERR(group
)) {
1096 dev_err(dev
, "Failed to allocate IOMMU group\n");
1097 ret
= PTR_ERR(group
);
1101 ret
= iommu_group_add_device(group
, dev
);
1102 iommu_group_put(group
);
1105 dev_err(dev
, "Failed to add device to IPMMU group\n");
1110 archdata
= kzalloc(sizeof(*archdata
), GFP_KERNEL
);
1116 archdata
->mmu
= mmu
;
1117 archdata
->utlbs
= utlbs
;
1118 archdata
->num_utlbs
= num_utlbs
;
1119 dev
->archdata
.iommu
= archdata
;
1122 * Create the ARM mapping, used by the ARM DMA mapping core to allocate
1123 * VAs. This will allocate a corresponding IOMMU domain.
1126 * - Create one mapping per context (TLB).
1127 * - Make the mapping size configurable ? We currently use a 2GB mapping
1128 * at a 1GB offset to ensure that NULL VAs will fault.
1130 if (!mmu
->mapping
) {
1131 struct dma_iommu_mapping
*mapping
;
1133 mapping
= arm_iommu_create_mapping(&platform_bus_type
,
1135 if (IS_ERR(mapping
)) {
1136 dev_err(mmu
->dev
, "failed to create ARM IOMMU mapping\n");
1137 ret
= PTR_ERR(mapping
);
1141 mmu
->mapping
= mapping
;
1144 /* Attach the ARM VA mapping to the device. */
1145 ret
= arm_iommu_attach_device(dev
, mmu
->mapping
);
1147 dev_err(dev
, "Failed to attach device to VA mapping\n");
1154 arm_iommu_release_mapping(mmu
->mapping
);
1156 kfree(dev
->archdata
.iommu
);
1159 dev
->archdata
.iommu
= NULL
;
1161 if (!IS_ERR_OR_NULL(group
))
1162 iommu_group_remove_device(dev
);
1167 static void ipmmu_remove_device(struct device
*dev
)
1169 struct ipmmu_vmsa_archdata
*archdata
= dev
->archdata
.iommu
;
1171 arm_iommu_detach_device(dev
);
1172 iommu_group_remove_device(dev
);
1174 kfree(archdata
->utlbs
);
1177 dev
->archdata
.iommu
= NULL
;
1180 static const struct iommu_ops ipmmu_ops
= {
1181 .domain_init
= ipmmu_domain_init
,
1182 .domain_destroy
= ipmmu_domain_destroy
,
1183 .attach_dev
= ipmmu_attach_device
,
1184 .detach_dev
= ipmmu_detach_device
,
1186 .unmap
= ipmmu_unmap
,
1187 .map_sg
= default_iommu_map_sg
,
1188 .iova_to_phys
= ipmmu_iova_to_phys
,
1189 .add_device
= ipmmu_add_device
,
1190 .remove_device
= ipmmu_remove_device
,
1191 .pgsize_bitmap
= SZ_2M
| SZ_64K
| SZ_4K
,
1194 /* -----------------------------------------------------------------------------
1195 * Probe/remove and init
1198 static void ipmmu_device_reset(struct ipmmu_vmsa_device
*mmu
)
1202 /* Disable all contexts. */
1203 for (i
= 0; i
< 4; ++i
)
1204 ipmmu_write(mmu
, i
* IM_CTX_SIZE
+ IMCTR
, 0);
1207 static int ipmmu_probe(struct platform_device
*pdev
)
1209 struct ipmmu_vmsa_device
*mmu
;
1210 struct resource
*res
;
1214 if (!IS_ENABLED(CONFIG_OF
) && !pdev
->dev
.platform_data
) {
1215 dev_err(&pdev
->dev
, "missing platform data\n");
1219 mmu
= devm_kzalloc(&pdev
->dev
, sizeof(*mmu
), GFP_KERNEL
);
1221 dev_err(&pdev
->dev
, "cannot allocate device data\n");
1225 mmu
->dev
= &pdev
->dev
;
1226 mmu
->num_utlbs
= 32;
1228 /* Map I/O memory and request IRQ. */
1229 res
= platform_get_resource(pdev
, IORESOURCE_MEM
, 0);
1230 mmu
->base
= devm_ioremap_resource(&pdev
->dev
, res
);
1231 if (IS_ERR(mmu
->base
))
1232 return PTR_ERR(mmu
->base
);
1235 * The IPMMU has two register banks, for secure and non-secure modes.
1236 * The bank mapped at the beginning of the IPMMU address space
1237 * corresponds to the running mode of the CPU. When running in secure
1238 * mode the non-secure register bank is also available at an offset.
1240 * Secure mode operation isn't clearly documented and is thus currently
1241 * not implemented in the driver. Furthermore, preliminary tests of
1242 * non-secure operation with the main register bank were not successful.
1243 * Offset the registers base unconditionally to point to the non-secure
1244 * alias space for now.
1246 mmu
->base
+= IM_NS_ALIAS_OFFSET
;
1248 irq
= platform_get_irq(pdev
, 0);
1250 dev_err(&pdev
->dev
, "no IRQ found\n");
1254 ret
= devm_request_irq(&pdev
->dev
, irq
, ipmmu_irq
, 0,
1255 dev_name(&pdev
->dev
), mmu
);
1257 dev_err(&pdev
->dev
, "failed to request IRQ %d\n", irq
);
1261 ipmmu_device_reset(mmu
);
1264 * We can't create the ARM mapping here as it requires the bus to have
1265 * an IOMMU, which only happens when bus_set_iommu() is called in
1266 * ipmmu_init() after the probe function returns.
1269 spin_lock(&ipmmu_devices_lock
);
1270 list_add(&mmu
->list
, &ipmmu_devices
);
1271 spin_unlock(&ipmmu_devices_lock
);
1273 platform_set_drvdata(pdev
, mmu
);
1278 static int ipmmu_remove(struct platform_device
*pdev
)
1280 struct ipmmu_vmsa_device
*mmu
= platform_get_drvdata(pdev
);
1282 spin_lock(&ipmmu_devices_lock
);
1283 list_del(&mmu
->list
);
1284 spin_unlock(&ipmmu_devices_lock
);
1286 arm_iommu_release_mapping(mmu
->mapping
);
1288 ipmmu_device_reset(mmu
);
1293 static const struct of_device_id ipmmu_of_ids
[] = {
1294 { .compatible
= "renesas,ipmmu-vmsa", },
1297 static struct platform_driver ipmmu_driver
= {
1299 .name
= "ipmmu-vmsa",
1300 .of_match_table
= of_match_ptr(ipmmu_of_ids
),
1302 .probe
= ipmmu_probe
,
1303 .remove
= ipmmu_remove
,
1306 static int __init
ipmmu_init(void)
1310 ret
= platform_driver_register(&ipmmu_driver
);
1314 if (!iommu_present(&platform_bus_type
))
1315 bus_set_iommu(&platform_bus_type
, &ipmmu_ops
);
1320 static void __exit
ipmmu_exit(void)
1322 return platform_driver_unregister(&ipmmu_driver
);
1325 subsys_initcall(ipmmu_init
);
1326 module_exit(ipmmu_exit
);
1328 MODULE_DESCRIPTION("IOMMU API for Renesas VMSA-compatible IPMMU");
1329 MODULE_AUTHOR("Laurent Pinchart <laurent.pinchart@ideasonboard.com>");
1330 MODULE_LICENSE("GPL v2");