2 * Copyright (c) 2006, Intel Corporation.
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms and conditions of the GNU General Public License,
6 * version 2, as published by the Free Software Foundation.
8 * This program is distributed in the hope it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
13 * You should have received a copy of the GNU General Public License along with
14 * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
15 * Place - Suite 330, Boston, MA 02111-1307 USA.
17 * Copyright (C) 2006-2008 Intel Corporation
18 * Author: Ashok Raj <ashok.raj@intel.com>
19 * Author: Shaohua Li <shaohua.li@intel.com>
20 * Author: Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>
21 * Author: Fenghua Yu <fenghua.yu@intel.com>
24 #include <linux/init.h>
25 #include <linux/bitmap.h>
26 #include <linux/debugfs.h>
27 #include <linux/slab.h>
28 #include <linux/irq.h>
29 #include <linux/interrupt.h>
30 #include <linux/spinlock.h>
31 #include <linux/pci.h>
32 #include <linux/dmar.h>
33 #include <linux/dma-mapping.h>
34 #include <linux/mempool.h>
35 #include <linux/timer.h>
36 #include <linux/iova.h>
37 #include <linux/intel-iommu.h>
38 #include <asm/cacheflush.h>
39 #include <asm/iommu.h>
42 #define ROOT_SIZE VTD_PAGE_SIZE
43 #define CONTEXT_SIZE VTD_PAGE_SIZE
45 #define IS_GFX_DEVICE(pdev) ((pdev->class >> 16) == PCI_BASE_CLASS_DISPLAY)
46 #define IS_ISA_DEVICE(pdev) ((pdev->class >> 8) == PCI_CLASS_BRIDGE_ISA)
48 #define IOAPIC_RANGE_START (0xfee00000)
49 #define IOAPIC_RANGE_END (0xfeefffff)
50 #define IOVA_START_ADDR (0x1000)
52 #define DEFAULT_DOMAIN_ADDRESS_WIDTH 48
54 #define DOMAIN_MAX_ADDR(gaw) ((((u64)1) << gaw) - 1)
56 #define IOVA_PFN(addr) ((addr) >> PAGE_SHIFT)
57 #define DMA_32BIT_PFN IOVA_PFN(DMA_32BIT_MASK)
58 #define DMA_64BIT_PFN IOVA_PFN(DMA_64BIT_MASK)
63 * 12-63: Context Ptr (12 - (haw-1))
70 #define ROOT_ENTRY_NR (VTD_PAGE_SIZE/sizeof(struct root_entry))
71 static inline bool root_present(struct root_entry
*root
)
73 return (root
->val
& 1);
75 static inline void set_root_present(struct root_entry
*root
)
79 static inline void set_root_value(struct root_entry
*root
, unsigned long value
)
81 root
->val
|= value
& VTD_PAGE_MASK
;
84 static inline struct context_entry
*
85 get_context_addr_from_root(struct root_entry
*root
)
87 return (struct context_entry
*)
88 (root_present(root
)?phys_to_virt(
89 root
->val
& VTD_PAGE_MASK
) :
96 * 1: fault processing disable
97 * 2-3: translation type
98 * 12-63: address space root
104 struct context_entry
{
108 #define context_present(c) ((c).lo & 1)
109 #define context_fault_disable(c) (((c).lo >> 1) & 1)
110 #define context_translation_type(c) (((c).lo >> 2) & 3)
111 #define context_address_root(c) ((c).lo & VTD_PAGE_MASK)
112 #define context_address_width(c) ((c).hi & 7)
113 #define context_domain_id(c) (((c).hi >> 8) & ((1 << 16) - 1))
115 #define context_set_present(c) do {(c).lo |= 1;} while (0)
116 #define context_set_fault_enable(c) \
117 do {(c).lo &= (((u64)-1) << 2) | 1;} while (0)
118 #define context_set_translation_type(c, val) \
120 (c).lo &= (((u64)-1) << 4) | 3; \
121 (c).lo |= ((val) & 3) << 2; \
123 #define CONTEXT_TT_MULTI_LEVEL 0
124 #define context_set_address_root(c, val) \
125 do {(c).lo |= (val) & VTD_PAGE_MASK; } while (0)
126 #define context_set_address_width(c, val) do {(c).hi |= (val) & 7;} while (0)
127 #define context_set_domain_id(c, val) \
128 do {(c).hi |= ((val) & ((1 << 16) - 1)) << 8;} while (0)
129 #define context_clear_entry(c) do {(c).lo = 0; (c).hi = 0;} while (0)
131 static void flush_unmaps_timeout(unsigned long data
);
133 DEFINE_TIMER(unmap_timer
, flush_unmaps_timeout
, 0, 0);
135 #define HIGH_WATER_MARK 250
136 struct deferred_flush_tables
{
138 struct iova
*iova
[HIGH_WATER_MARK
];
139 struct dmar_domain
*domain
[HIGH_WATER_MARK
];
142 static struct deferred_flush_tables
*deferred_flush
;
144 /* bitmap for indexing intel_iommus */
145 static int g_num_of_iommus
;
147 static DEFINE_SPINLOCK(async_umap_flush_lock
);
148 static LIST_HEAD(unmaps_to_do
);
151 static long list_size
;
153 static void domain_remove_dev_info(struct dmar_domain
*domain
);
156 static int __initdata dmar_map_gfx
= 1;
157 static int dmar_forcedac
;
158 static int intel_iommu_strict
;
160 #define DUMMY_DEVICE_DOMAIN_INFO ((struct device_domain_info *)(-1))
161 static DEFINE_SPINLOCK(device_domain_lock
);
162 static LIST_HEAD(device_domain_list
);
164 static int __init
intel_iommu_setup(char *str
)
169 if (!strncmp(str
, "off", 3)) {
171 printk(KERN_INFO
"Intel-IOMMU: disabled\n");
172 } else if (!strncmp(str
, "igfx_off", 8)) {
175 "Intel-IOMMU: disable GFX device mapping\n");
176 } else if (!strncmp(str
, "forcedac", 8)) {
178 "Intel-IOMMU: Forcing DAC for PCI devices\n");
180 } else if (!strncmp(str
, "strict", 6)) {
182 "Intel-IOMMU: disable batched IOTLB flush\n");
183 intel_iommu_strict
= 1;
186 str
+= strcspn(str
, ",");
192 __setup("intel_iommu=", intel_iommu_setup
);
194 static struct kmem_cache
*iommu_domain_cache
;
195 static struct kmem_cache
*iommu_devinfo_cache
;
196 static struct kmem_cache
*iommu_iova_cache
;
198 static inline void *iommu_kmem_cache_alloc(struct kmem_cache
*cachep
)
203 /* trying to avoid low memory issues */
204 flags
= current
->flags
& PF_MEMALLOC
;
205 current
->flags
|= PF_MEMALLOC
;
206 vaddr
= kmem_cache_alloc(cachep
, GFP_ATOMIC
);
207 current
->flags
&= (~PF_MEMALLOC
| flags
);
212 static inline void *alloc_pgtable_page(void)
217 /* trying to avoid low memory issues */
218 flags
= current
->flags
& PF_MEMALLOC
;
219 current
->flags
|= PF_MEMALLOC
;
220 vaddr
= (void *)get_zeroed_page(GFP_ATOMIC
);
221 current
->flags
&= (~PF_MEMALLOC
| flags
);
225 static inline void free_pgtable_page(void *vaddr
)
227 free_page((unsigned long)vaddr
);
230 static inline void *alloc_domain_mem(void)
232 return iommu_kmem_cache_alloc(iommu_domain_cache
);
235 static void free_domain_mem(void *vaddr
)
237 kmem_cache_free(iommu_domain_cache
, vaddr
);
240 static inline void * alloc_devinfo_mem(void)
242 return iommu_kmem_cache_alloc(iommu_devinfo_cache
);
245 static inline void free_devinfo_mem(void *vaddr
)
247 kmem_cache_free(iommu_devinfo_cache
, vaddr
);
250 struct iova
*alloc_iova_mem(void)
252 return iommu_kmem_cache_alloc(iommu_iova_cache
);
255 void free_iova_mem(struct iova
*iova
)
257 kmem_cache_free(iommu_iova_cache
, iova
);
260 /* Gets context entry for a given bus and devfn */
261 static struct context_entry
* device_to_context_entry(struct intel_iommu
*iommu
,
264 struct root_entry
*root
;
265 struct context_entry
*context
;
266 unsigned long phy_addr
;
269 spin_lock_irqsave(&iommu
->lock
, flags
);
270 root
= &iommu
->root_entry
[bus
];
271 context
= get_context_addr_from_root(root
);
273 context
= (struct context_entry
*)alloc_pgtable_page();
275 spin_unlock_irqrestore(&iommu
->lock
, flags
);
278 __iommu_flush_cache(iommu
, (void *)context
, CONTEXT_SIZE
);
279 phy_addr
= virt_to_phys((void *)context
);
280 set_root_value(root
, phy_addr
);
281 set_root_present(root
);
282 __iommu_flush_cache(iommu
, root
, sizeof(*root
));
284 spin_unlock_irqrestore(&iommu
->lock
, flags
);
285 return &context
[devfn
];
288 static int device_context_mapped(struct intel_iommu
*iommu
, u8 bus
, u8 devfn
)
290 struct root_entry
*root
;
291 struct context_entry
*context
;
295 spin_lock_irqsave(&iommu
->lock
, flags
);
296 root
= &iommu
->root_entry
[bus
];
297 context
= get_context_addr_from_root(root
);
302 ret
= context_present(context
[devfn
]);
304 spin_unlock_irqrestore(&iommu
->lock
, flags
);
308 static void clear_context_table(struct intel_iommu
*iommu
, u8 bus
, u8 devfn
)
310 struct root_entry
*root
;
311 struct context_entry
*context
;
314 spin_lock_irqsave(&iommu
->lock
, flags
);
315 root
= &iommu
->root_entry
[bus
];
316 context
= get_context_addr_from_root(root
);
318 context_clear_entry(context
[devfn
]);
319 __iommu_flush_cache(iommu
, &context
[devfn
], \
322 spin_unlock_irqrestore(&iommu
->lock
, flags
);
325 static void free_context_table(struct intel_iommu
*iommu
)
327 struct root_entry
*root
;
330 struct context_entry
*context
;
332 spin_lock_irqsave(&iommu
->lock
, flags
);
333 if (!iommu
->root_entry
) {
336 for (i
= 0; i
< ROOT_ENTRY_NR
; i
++) {
337 root
= &iommu
->root_entry
[i
];
338 context
= get_context_addr_from_root(root
);
340 free_pgtable_page(context
);
342 free_pgtable_page(iommu
->root_entry
);
343 iommu
->root_entry
= NULL
;
345 spin_unlock_irqrestore(&iommu
->lock
, flags
);
348 /* page table handling */
349 #define LEVEL_STRIDE (9)
350 #define LEVEL_MASK (((u64)1 << LEVEL_STRIDE) - 1)
352 static inline int agaw_to_level(int agaw
)
357 static inline int agaw_to_width(int agaw
)
359 return 30 + agaw
* LEVEL_STRIDE
;
363 static inline int width_to_agaw(int width
)
365 return (width
- 30) / LEVEL_STRIDE
;
368 static inline unsigned int level_to_offset_bits(int level
)
370 return (12 + (level
- 1) * LEVEL_STRIDE
);
373 static inline int address_level_offset(u64 addr
, int level
)
375 return ((addr
>> level_to_offset_bits(level
)) & LEVEL_MASK
);
378 static inline u64
level_mask(int level
)
380 return ((u64
)-1 << level_to_offset_bits(level
));
383 static inline u64
level_size(int level
)
385 return ((u64
)1 << level_to_offset_bits(level
));
388 static inline u64
align_to_level(u64 addr
, int level
)
390 return ((addr
+ level_size(level
) - 1) & level_mask(level
));
393 static struct dma_pte
* addr_to_dma_pte(struct dmar_domain
*domain
, u64 addr
)
395 int addr_width
= agaw_to_width(domain
->agaw
);
396 struct dma_pte
*parent
, *pte
= NULL
;
397 int level
= agaw_to_level(domain
->agaw
);
401 BUG_ON(!domain
->pgd
);
403 addr
&= (((u64
)1) << addr_width
) - 1;
404 parent
= domain
->pgd
;
406 spin_lock_irqsave(&domain
->mapping_lock
, flags
);
410 offset
= address_level_offset(addr
, level
);
411 pte
= &parent
[offset
];
415 if (!dma_pte_present(*pte
)) {
416 tmp_page
= alloc_pgtable_page();
419 spin_unlock_irqrestore(&domain
->mapping_lock
,
423 __iommu_flush_cache(domain
->iommu
, tmp_page
,
425 dma_set_pte_addr(*pte
, virt_to_phys(tmp_page
));
427 * high level table always sets r/w, last level page
428 * table control read/write
430 dma_set_pte_readable(*pte
);
431 dma_set_pte_writable(*pte
);
432 __iommu_flush_cache(domain
->iommu
, pte
, sizeof(*pte
));
434 parent
= phys_to_virt(dma_pte_addr(*pte
));
438 spin_unlock_irqrestore(&domain
->mapping_lock
, flags
);
442 /* return address's pte at specific level */
443 static struct dma_pte
*dma_addr_level_pte(struct dmar_domain
*domain
, u64 addr
,
446 struct dma_pte
*parent
, *pte
= NULL
;
447 int total
= agaw_to_level(domain
->agaw
);
450 parent
= domain
->pgd
;
451 while (level
<= total
) {
452 offset
= address_level_offset(addr
, total
);
453 pte
= &parent
[offset
];
457 if (!dma_pte_present(*pte
))
459 parent
= phys_to_virt(dma_pte_addr(*pte
));
465 /* clear one page's page table */
466 static void dma_pte_clear_one(struct dmar_domain
*domain
, u64 addr
)
468 struct dma_pte
*pte
= NULL
;
470 /* get last level pte */
471 pte
= dma_addr_level_pte(domain
, addr
, 1);
475 __iommu_flush_cache(domain
->iommu
, pte
, sizeof(*pte
));
479 /* clear last level pte, a tlb flush should be followed */
480 static void dma_pte_clear_range(struct dmar_domain
*domain
, u64 start
, u64 end
)
482 int addr_width
= agaw_to_width(domain
->agaw
);
484 start
&= (((u64
)1) << addr_width
) - 1;
485 end
&= (((u64
)1) << addr_width
) - 1;
486 /* in case it's partial page */
487 start
= PAGE_ALIGN(start
);
490 /* we don't need lock here, nobody else touches the iova range */
491 while (start
< end
) {
492 dma_pte_clear_one(domain
, start
);
493 start
+= VTD_PAGE_SIZE
;
497 /* free page table pages. last level pte should already be cleared */
498 static void dma_pte_free_pagetable(struct dmar_domain
*domain
,
501 int addr_width
= agaw_to_width(domain
->agaw
);
503 int total
= agaw_to_level(domain
->agaw
);
507 start
&= (((u64
)1) << addr_width
) - 1;
508 end
&= (((u64
)1) << addr_width
) - 1;
510 /* we don't need lock here, nobody else touches the iova range */
512 while (level
<= total
) {
513 tmp
= align_to_level(start
, level
);
514 if (tmp
>= end
|| (tmp
+ level_size(level
) > end
))
518 pte
= dma_addr_level_pte(domain
, tmp
, level
);
521 phys_to_virt(dma_pte_addr(*pte
)));
523 __iommu_flush_cache(domain
->iommu
,
526 tmp
+= level_size(level
);
531 if (start
== 0 && end
>= ((((u64
)1) << addr_width
) - 1)) {
532 free_pgtable_page(domain
->pgd
);
538 static int iommu_alloc_root_entry(struct intel_iommu
*iommu
)
540 struct root_entry
*root
;
543 root
= (struct root_entry
*)alloc_pgtable_page();
547 __iommu_flush_cache(iommu
, root
, ROOT_SIZE
);
549 spin_lock_irqsave(&iommu
->lock
, flags
);
550 iommu
->root_entry
= root
;
551 spin_unlock_irqrestore(&iommu
->lock
, flags
);
556 static void iommu_set_root_entry(struct intel_iommu
*iommu
)
562 addr
= iommu
->root_entry
;
564 spin_lock_irqsave(&iommu
->register_lock
, flag
);
565 dmar_writeq(iommu
->reg
+ DMAR_RTADDR_REG
, virt_to_phys(addr
));
567 cmd
= iommu
->gcmd
| DMA_GCMD_SRTP
;
568 writel(cmd
, iommu
->reg
+ DMAR_GCMD_REG
);
570 /* Make sure hardware complete it */
571 IOMMU_WAIT_OP(iommu
, DMAR_GSTS_REG
,
572 readl
, (sts
& DMA_GSTS_RTPS
), sts
);
574 spin_unlock_irqrestore(&iommu
->register_lock
, flag
);
577 static void iommu_flush_write_buffer(struct intel_iommu
*iommu
)
582 if (!cap_rwbf(iommu
->cap
))
584 val
= iommu
->gcmd
| DMA_GCMD_WBF
;
586 spin_lock_irqsave(&iommu
->register_lock
, flag
);
587 writel(val
, iommu
->reg
+ DMAR_GCMD_REG
);
589 /* Make sure hardware complete it */
590 IOMMU_WAIT_OP(iommu
, DMAR_GSTS_REG
,
591 readl
, (!(val
& DMA_GSTS_WBFS
)), val
);
593 spin_unlock_irqrestore(&iommu
->register_lock
, flag
);
596 /* return value determine if we need a write buffer flush */
597 static int __iommu_flush_context(struct intel_iommu
*iommu
,
598 u16 did
, u16 source_id
, u8 function_mask
, u64 type
,
599 int non_present_entry_flush
)
605 * In the non-present entry flush case, if hardware doesn't cache
606 * non-present entry we do nothing and if hardware cache non-present
607 * entry, we flush entries of domain 0 (the domain id is used to cache
608 * any non-present entries)
610 if (non_present_entry_flush
) {
611 if (!cap_caching_mode(iommu
->cap
))
618 case DMA_CCMD_GLOBAL_INVL
:
619 val
= DMA_CCMD_GLOBAL_INVL
;
621 case DMA_CCMD_DOMAIN_INVL
:
622 val
= DMA_CCMD_DOMAIN_INVL
|DMA_CCMD_DID(did
);
624 case DMA_CCMD_DEVICE_INVL
:
625 val
= DMA_CCMD_DEVICE_INVL
|DMA_CCMD_DID(did
)
626 | DMA_CCMD_SID(source_id
) | DMA_CCMD_FM(function_mask
);
633 spin_lock_irqsave(&iommu
->register_lock
, flag
);
634 dmar_writeq(iommu
->reg
+ DMAR_CCMD_REG
, val
);
636 /* Make sure hardware complete it */
637 IOMMU_WAIT_OP(iommu
, DMAR_CCMD_REG
,
638 dmar_readq
, (!(val
& DMA_CCMD_ICC
)), val
);
640 spin_unlock_irqrestore(&iommu
->register_lock
, flag
);
642 /* flush context entry will implicitly flush write buffer */
646 /* return value determine if we need a write buffer flush */
647 static int __iommu_flush_iotlb(struct intel_iommu
*iommu
, u16 did
,
648 u64 addr
, unsigned int size_order
, u64 type
,
649 int non_present_entry_flush
)
651 int tlb_offset
= ecap_iotlb_offset(iommu
->ecap
);
652 u64 val
= 0, val_iva
= 0;
656 * In the non-present entry flush case, if hardware doesn't cache
657 * non-present entry we do nothing and if hardware cache non-present
658 * entry, we flush entries of domain 0 (the domain id is used to cache
659 * any non-present entries)
661 if (non_present_entry_flush
) {
662 if (!cap_caching_mode(iommu
->cap
))
669 case DMA_TLB_GLOBAL_FLUSH
:
670 /* global flush doesn't need set IVA_REG */
671 val
= DMA_TLB_GLOBAL_FLUSH
|DMA_TLB_IVT
;
673 case DMA_TLB_DSI_FLUSH
:
674 val
= DMA_TLB_DSI_FLUSH
|DMA_TLB_IVT
|DMA_TLB_DID(did
);
676 case DMA_TLB_PSI_FLUSH
:
677 val
= DMA_TLB_PSI_FLUSH
|DMA_TLB_IVT
|DMA_TLB_DID(did
);
678 /* Note: always flush non-leaf currently */
679 val_iva
= size_order
| addr
;
684 /* Note: set drain read/write */
687 * This is probably to be super secure.. Looks like we can
688 * ignore it without any impact.
690 if (cap_read_drain(iommu
->cap
))
691 val
|= DMA_TLB_READ_DRAIN
;
693 if (cap_write_drain(iommu
->cap
))
694 val
|= DMA_TLB_WRITE_DRAIN
;
696 spin_lock_irqsave(&iommu
->register_lock
, flag
);
697 /* Note: Only uses first TLB reg currently */
699 dmar_writeq(iommu
->reg
+ tlb_offset
, val_iva
);
700 dmar_writeq(iommu
->reg
+ tlb_offset
+ 8, val
);
702 /* Make sure hardware complete it */
703 IOMMU_WAIT_OP(iommu
, tlb_offset
+ 8,
704 dmar_readq
, (!(val
& DMA_TLB_IVT
)), val
);
706 spin_unlock_irqrestore(&iommu
->register_lock
, flag
);
708 /* check IOTLB invalidation granularity */
709 if (DMA_TLB_IAIG(val
) == 0)
710 printk(KERN_ERR
"IOMMU: flush IOTLB failed\n");
711 if (DMA_TLB_IAIG(val
) != DMA_TLB_IIRG(type
))
712 pr_debug("IOMMU: tlb flush request %Lx, actual %Lx\n",
713 (unsigned long long)DMA_TLB_IIRG(type
),
714 (unsigned long long)DMA_TLB_IAIG(val
));
715 /* flush iotlb entry will implicitly flush write buffer */
719 static int iommu_flush_iotlb_psi(struct intel_iommu
*iommu
, u16 did
,
720 u64 addr
, unsigned int pages
, int non_present_entry_flush
)
724 BUG_ON(addr
& (~VTD_PAGE_MASK
));
727 /* Fallback to domain selective flush if no PSI support */
728 if (!cap_pgsel_inv(iommu
->cap
))
729 return iommu
->flush
.flush_iotlb(iommu
, did
, 0, 0,
731 non_present_entry_flush
);
734 * PSI requires page size to be 2 ^ x, and the base address is naturally
735 * aligned to the size
737 mask
= ilog2(__roundup_pow_of_two(pages
));
738 /* Fallback to domain selective flush if size is too big */
739 if (mask
> cap_max_amask_val(iommu
->cap
))
740 return iommu
->flush
.flush_iotlb(iommu
, did
, 0, 0,
741 DMA_TLB_DSI_FLUSH
, non_present_entry_flush
);
743 return iommu
->flush
.flush_iotlb(iommu
, did
, addr
, mask
,
745 non_present_entry_flush
);
748 static void iommu_disable_protect_mem_regions(struct intel_iommu
*iommu
)
753 spin_lock_irqsave(&iommu
->register_lock
, flags
);
754 pmen
= readl(iommu
->reg
+ DMAR_PMEN_REG
);
755 pmen
&= ~DMA_PMEN_EPM
;
756 writel(pmen
, iommu
->reg
+ DMAR_PMEN_REG
);
758 /* wait for the protected region status bit to clear */
759 IOMMU_WAIT_OP(iommu
, DMAR_PMEN_REG
,
760 readl
, !(pmen
& DMA_PMEN_PRS
), pmen
);
762 spin_unlock_irqrestore(&iommu
->register_lock
, flags
);
765 static int iommu_enable_translation(struct intel_iommu
*iommu
)
770 spin_lock_irqsave(&iommu
->register_lock
, flags
);
771 writel(iommu
->gcmd
|DMA_GCMD_TE
, iommu
->reg
+ DMAR_GCMD_REG
);
773 /* Make sure hardware complete it */
774 IOMMU_WAIT_OP(iommu
, DMAR_GSTS_REG
,
775 readl
, (sts
& DMA_GSTS_TES
), sts
);
777 iommu
->gcmd
|= DMA_GCMD_TE
;
778 spin_unlock_irqrestore(&iommu
->register_lock
, flags
);
782 static int iommu_disable_translation(struct intel_iommu
*iommu
)
787 spin_lock_irqsave(&iommu
->register_lock
, flag
);
788 iommu
->gcmd
&= ~DMA_GCMD_TE
;
789 writel(iommu
->gcmd
, iommu
->reg
+ DMAR_GCMD_REG
);
791 /* Make sure hardware complete it */
792 IOMMU_WAIT_OP(iommu
, DMAR_GSTS_REG
,
793 readl
, (!(sts
& DMA_GSTS_TES
)), sts
);
795 spin_unlock_irqrestore(&iommu
->register_lock
, flag
);
799 /* iommu interrupt handling. Most stuff are MSI-like. */
801 static const char *fault_reason_strings
[] =
804 "Present bit in root entry is clear",
805 "Present bit in context entry is clear",
806 "Invalid context entry",
807 "Access beyond MGAW",
808 "PTE Write access is not set",
809 "PTE Read access is not set",
810 "Next page table ptr is invalid",
811 "Root table address invalid",
812 "Context table ptr is invalid",
813 "non-zero reserved fields in RTP",
814 "non-zero reserved fields in CTP",
815 "non-zero reserved fields in PTE",
817 #define MAX_FAULT_REASON_IDX (ARRAY_SIZE(fault_reason_strings) - 1)
819 const char *dmar_get_fault_reason(u8 fault_reason
)
821 if (fault_reason
> MAX_FAULT_REASON_IDX
)
824 return fault_reason_strings
[fault_reason
];
827 void dmar_msi_unmask(unsigned int irq
)
829 struct intel_iommu
*iommu
= get_irq_data(irq
);
833 spin_lock_irqsave(&iommu
->register_lock
, flag
);
834 writel(0, iommu
->reg
+ DMAR_FECTL_REG
);
835 /* Read a reg to force flush the post write */
836 readl(iommu
->reg
+ DMAR_FECTL_REG
);
837 spin_unlock_irqrestore(&iommu
->register_lock
, flag
);
840 void dmar_msi_mask(unsigned int irq
)
843 struct intel_iommu
*iommu
= get_irq_data(irq
);
846 spin_lock_irqsave(&iommu
->register_lock
, flag
);
847 writel(DMA_FECTL_IM
, iommu
->reg
+ DMAR_FECTL_REG
);
848 /* Read a reg to force flush the post write */
849 readl(iommu
->reg
+ DMAR_FECTL_REG
);
850 spin_unlock_irqrestore(&iommu
->register_lock
, flag
);
853 void dmar_msi_write(int irq
, struct msi_msg
*msg
)
855 struct intel_iommu
*iommu
= get_irq_data(irq
);
858 spin_lock_irqsave(&iommu
->register_lock
, flag
);
859 writel(msg
->data
, iommu
->reg
+ DMAR_FEDATA_REG
);
860 writel(msg
->address_lo
, iommu
->reg
+ DMAR_FEADDR_REG
);
861 writel(msg
->address_hi
, iommu
->reg
+ DMAR_FEUADDR_REG
);
862 spin_unlock_irqrestore(&iommu
->register_lock
, flag
);
865 void dmar_msi_read(int irq
, struct msi_msg
*msg
)
867 struct intel_iommu
*iommu
= get_irq_data(irq
);
870 spin_lock_irqsave(&iommu
->register_lock
, flag
);
871 msg
->data
= readl(iommu
->reg
+ DMAR_FEDATA_REG
);
872 msg
->address_lo
= readl(iommu
->reg
+ DMAR_FEADDR_REG
);
873 msg
->address_hi
= readl(iommu
->reg
+ DMAR_FEUADDR_REG
);
874 spin_unlock_irqrestore(&iommu
->register_lock
, flag
);
877 static int iommu_page_fault_do_one(struct intel_iommu
*iommu
, int type
,
878 u8 fault_reason
, u16 source_id
, unsigned long long addr
)
882 reason
= dmar_get_fault_reason(fault_reason
);
885 "DMAR:[%s] Request device [%02x:%02x.%d] "
887 "DMAR:[fault reason %02d] %s\n",
888 (type
? "DMA Read" : "DMA Write"),
889 (source_id
>> 8), PCI_SLOT(source_id
& 0xFF),
890 PCI_FUNC(source_id
& 0xFF), addr
, fault_reason
, reason
);
894 #define PRIMARY_FAULT_REG_LEN (16)
895 static irqreturn_t
iommu_page_fault(int irq
, void *dev_id
)
897 struct intel_iommu
*iommu
= dev_id
;
898 int reg
, fault_index
;
902 spin_lock_irqsave(&iommu
->register_lock
, flag
);
903 fault_status
= readl(iommu
->reg
+ DMAR_FSTS_REG
);
905 /* TBD: ignore advanced fault log currently */
906 if (!(fault_status
& DMA_FSTS_PPF
))
909 fault_index
= dma_fsts_fault_record_index(fault_status
);
910 reg
= cap_fault_reg_offset(iommu
->cap
);
918 /* highest 32 bits */
919 data
= readl(iommu
->reg
+ reg
+
920 fault_index
* PRIMARY_FAULT_REG_LEN
+ 12);
921 if (!(data
& DMA_FRCD_F
))
924 fault_reason
= dma_frcd_fault_reason(data
);
925 type
= dma_frcd_type(data
);
927 data
= readl(iommu
->reg
+ reg
+
928 fault_index
* PRIMARY_FAULT_REG_LEN
+ 8);
929 source_id
= dma_frcd_source_id(data
);
931 guest_addr
= dmar_readq(iommu
->reg
+ reg
+
932 fault_index
* PRIMARY_FAULT_REG_LEN
);
933 guest_addr
= dma_frcd_page_addr(guest_addr
);
934 /* clear the fault */
935 writel(DMA_FRCD_F
, iommu
->reg
+ reg
+
936 fault_index
* PRIMARY_FAULT_REG_LEN
+ 12);
938 spin_unlock_irqrestore(&iommu
->register_lock
, flag
);
940 iommu_page_fault_do_one(iommu
, type
, fault_reason
,
941 source_id
, guest_addr
);
944 if (fault_index
> cap_num_fault_regs(iommu
->cap
))
946 spin_lock_irqsave(&iommu
->register_lock
, flag
);
949 /* clear primary fault overflow */
950 fault_status
= readl(iommu
->reg
+ DMAR_FSTS_REG
);
951 if (fault_status
& DMA_FSTS_PFO
)
952 writel(DMA_FSTS_PFO
, iommu
->reg
+ DMAR_FSTS_REG
);
954 spin_unlock_irqrestore(&iommu
->register_lock
, flag
);
958 int dmar_set_interrupt(struct intel_iommu
*iommu
)
964 printk(KERN_ERR
"IOMMU: no free vectors\n");
968 set_irq_data(irq
, iommu
);
971 ret
= arch_setup_dmar_msi(irq
);
973 set_irq_data(irq
, NULL
);
979 /* Force fault register is cleared */
980 iommu_page_fault(irq
, iommu
);
982 ret
= request_irq(irq
, iommu_page_fault
, 0, iommu
->name
, iommu
);
984 printk(KERN_ERR
"IOMMU: can't request irq\n");
988 static int iommu_init_domains(struct intel_iommu
*iommu
)
990 unsigned long ndomains
;
991 unsigned long nlongs
;
993 ndomains
= cap_ndoms(iommu
->cap
);
994 pr_debug("Number of Domains supportd <%ld>\n", ndomains
);
995 nlongs
= BITS_TO_LONGS(ndomains
);
997 /* TBD: there might be 64K domains,
998 * consider other allocation for future chip
1000 iommu
->domain_ids
= kcalloc(nlongs
, sizeof(unsigned long), GFP_KERNEL
);
1001 if (!iommu
->domain_ids
) {
1002 printk(KERN_ERR
"Allocating domain id array failed\n");
1005 iommu
->domains
= kcalloc(ndomains
, sizeof(struct dmar_domain
*),
1007 if (!iommu
->domains
) {
1008 printk(KERN_ERR
"Allocating domain array failed\n");
1009 kfree(iommu
->domain_ids
);
1013 spin_lock_init(&iommu
->lock
);
1016 * if Caching mode is set, then invalid translations are tagged
1017 * with domainid 0. Hence we need to pre-allocate it.
1019 if (cap_caching_mode(iommu
->cap
))
1020 set_bit(0, iommu
->domain_ids
);
1025 static void domain_exit(struct dmar_domain
*domain
);
1027 void free_dmar_iommu(struct intel_iommu
*iommu
)
1029 struct dmar_domain
*domain
;
1032 i
= find_first_bit(iommu
->domain_ids
, cap_ndoms(iommu
->cap
));
1033 for (; i
< cap_ndoms(iommu
->cap
); ) {
1034 domain
= iommu
->domains
[i
];
1035 clear_bit(i
, iommu
->domain_ids
);
1036 domain_exit(domain
);
1037 i
= find_next_bit(iommu
->domain_ids
,
1038 cap_ndoms(iommu
->cap
), i
+1);
1041 if (iommu
->gcmd
& DMA_GCMD_TE
)
1042 iommu_disable_translation(iommu
);
1045 set_irq_data(iommu
->irq
, NULL
);
1046 /* This will mask the irq */
1047 free_irq(iommu
->irq
, iommu
);
1048 destroy_irq(iommu
->irq
);
1051 kfree(iommu
->domains
);
1052 kfree(iommu
->domain_ids
);
1054 /* free context mapping */
1055 free_context_table(iommu
);
1058 static struct dmar_domain
* iommu_alloc_domain(struct intel_iommu
*iommu
)
1061 unsigned long ndomains
;
1062 struct dmar_domain
*domain
;
1063 unsigned long flags
;
1065 domain
= alloc_domain_mem();
1069 ndomains
= cap_ndoms(iommu
->cap
);
1071 spin_lock_irqsave(&iommu
->lock
, flags
);
1072 num
= find_first_zero_bit(iommu
->domain_ids
, ndomains
);
1073 if (num
>= ndomains
) {
1074 spin_unlock_irqrestore(&iommu
->lock
, flags
);
1075 free_domain_mem(domain
);
1076 printk(KERN_ERR
"IOMMU: no free domain ids\n");
1080 set_bit(num
, iommu
->domain_ids
);
1082 domain
->iommu
= iommu
;
1083 iommu
->domains
[num
] = domain
;
1084 spin_unlock_irqrestore(&iommu
->lock
, flags
);
1089 static void iommu_free_domain(struct dmar_domain
*domain
)
1091 unsigned long flags
;
1093 spin_lock_irqsave(&domain
->iommu
->lock
, flags
);
1094 clear_bit(domain
->id
, domain
->iommu
->domain_ids
);
1095 spin_unlock_irqrestore(&domain
->iommu
->lock
, flags
);
1098 static struct iova_domain reserved_iova_list
;
1099 static struct lock_class_key reserved_alloc_key
;
1100 static struct lock_class_key reserved_rbtree_key
;
1102 static void dmar_init_reserved_ranges(void)
1104 struct pci_dev
*pdev
= NULL
;
1109 init_iova_domain(&reserved_iova_list
, DMA_32BIT_PFN
);
1111 lockdep_set_class(&reserved_iova_list
.iova_alloc_lock
,
1112 &reserved_alloc_key
);
1113 lockdep_set_class(&reserved_iova_list
.iova_rbtree_lock
,
1114 &reserved_rbtree_key
);
1116 /* IOAPIC ranges shouldn't be accessed by DMA */
1117 iova
= reserve_iova(&reserved_iova_list
, IOVA_PFN(IOAPIC_RANGE_START
),
1118 IOVA_PFN(IOAPIC_RANGE_END
));
1120 printk(KERN_ERR
"Reserve IOAPIC range failed\n");
1122 /* Reserve all PCI MMIO to avoid peer-to-peer access */
1123 for_each_pci_dev(pdev
) {
1126 for (i
= 0; i
< PCI_NUM_RESOURCES
; i
++) {
1127 r
= &pdev
->resource
[i
];
1128 if (!r
->flags
|| !(r
->flags
& IORESOURCE_MEM
))
1132 size
= r
->end
- addr
;
1133 size
= PAGE_ALIGN(size
);
1134 iova
= reserve_iova(&reserved_iova_list
, IOVA_PFN(addr
),
1135 IOVA_PFN(size
+ addr
) - 1);
1137 printk(KERN_ERR
"Reserve iova failed\n");
1143 static void domain_reserve_special_ranges(struct dmar_domain
*domain
)
1145 copy_reserved_iova(&reserved_iova_list
, &domain
->iovad
);
1148 static inline int guestwidth_to_adjustwidth(int gaw
)
1151 int r
= (gaw
- 12) % 9;
1162 static int domain_init(struct dmar_domain
*domain
, int guest_width
)
1164 struct intel_iommu
*iommu
;
1165 int adjust_width
, agaw
;
1166 unsigned long sagaw
;
1168 init_iova_domain(&domain
->iovad
, DMA_32BIT_PFN
);
1169 spin_lock_init(&domain
->mapping_lock
);
1171 domain_reserve_special_ranges(domain
);
1173 /* calculate AGAW */
1174 iommu
= domain
->iommu
;
1175 if (guest_width
> cap_mgaw(iommu
->cap
))
1176 guest_width
= cap_mgaw(iommu
->cap
);
1177 domain
->gaw
= guest_width
;
1178 adjust_width
= guestwidth_to_adjustwidth(guest_width
);
1179 agaw
= width_to_agaw(adjust_width
);
1180 sagaw
= cap_sagaw(iommu
->cap
);
1181 if (!test_bit(agaw
, &sagaw
)) {
1182 /* hardware doesn't support it, choose a bigger one */
1183 pr_debug("IOMMU: hardware doesn't support agaw %d\n", agaw
);
1184 agaw
= find_next_bit(&sagaw
, 5, agaw
);
1188 domain
->agaw
= agaw
;
1189 INIT_LIST_HEAD(&domain
->devices
);
1191 /* always allocate the top pgd */
1192 domain
->pgd
= (struct dma_pte
*)alloc_pgtable_page();
1195 __iommu_flush_cache(iommu
, domain
->pgd
, PAGE_SIZE
);
1199 static void domain_exit(struct dmar_domain
*domain
)
1203 /* Domain 0 is reserved, so dont process it */
1207 domain_remove_dev_info(domain
);
1209 put_iova_domain(&domain
->iovad
);
1210 end
= DOMAIN_MAX_ADDR(domain
->gaw
);
1211 end
= end
& (~PAGE_MASK
);
1214 dma_pte_clear_range(domain
, 0, end
);
1216 /* free page tables */
1217 dma_pte_free_pagetable(domain
, 0, end
);
1219 iommu_free_domain(domain
);
1220 free_domain_mem(domain
);
1223 static int domain_context_mapping_one(struct dmar_domain
*domain
,
1226 struct context_entry
*context
;
1227 struct intel_iommu
*iommu
= domain
->iommu
;
1228 unsigned long flags
;
1230 pr_debug("Set context mapping for %02x:%02x.%d\n",
1231 bus
, PCI_SLOT(devfn
), PCI_FUNC(devfn
));
1232 BUG_ON(!domain
->pgd
);
1233 context
= device_to_context_entry(iommu
, bus
, devfn
);
1236 spin_lock_irqsave(&iommu
->lock
, flags
);
1237 if (context_present(*context
)) {
1238 spin_unlock_irqrestore(&iommu
->lock
, flags
);
1242 context_set_domain_id(*context
, domain
->id
);
1243 context_set_address_width(*context
, domain
->agaw
);
1244 context_set_address_root(*context
, virt_to_phys(domain
->pgd
));
1245 context_set_translation_type(*context
, CONTEXT_TT_MULTI_LEVEL
);
1246 context_set_fault_enable(*context
);
1247 context_set_present(*context
);
1248 __iommu_flush_cache(iommu
, context
, sizeof(*context
));
1250 /* it's a non-present to present mapping */
1251 if (iommu
->flush
.flush_context(iommu
, domain
->id
,
1252 (((u16
)bus
) << 8) | devfn
, DMA_CCMD_MASK_NOBIT
,
1253 DMA_CCMD_DEVICE_INVL
, 1))
1254 iommu_flush_write_buffer(iommu
);
1256 iommu
->flush
.flush_iotlb(iommu
, 0, 0, 0, DMA_TLB_DSI_FLUSH
, 0);
1258 spin_unlock_irqrestore(&iommu
->lock
, flags
);
1263 domain_context_mapping(struct dmar_domain
*domain
, struct pci_dev
*pdev
)
1266 struct pci_dev
*tmp
, *parent
;
1268 ret
= domain_context_mapping_one(domain
, pdev
->bus
->number
,
1273 /* dependent device mapping */
1274 tmp
= pci_find_upstream_pcie_bridge(pdev
);
1277 /* Secondary interface's bus number and devfn 0 */
1278 parent
= pdev
->bus
->self
;
1279 while (parent
!= tmp
) {
1280 ret
= domain_context_mapping_one(domain
, parent
->bus
->number
,
1284 parent
= parent
->bus
->self
;
1286 if (tmp
->is_pcie
) /* this is a PCIE-to-PCI bridge */
1287 return domain_context_mapping_one(domain
,
1288 tmp
->subordinate
->number
, 0);
1289 else /* this is a legacy PCI bridge */
1290 return domain_context_mapping_one(domain
,
1291 tmp
->bus
->number
, tmp
->devfn
);
1294 static int domain_context_mapped(struct dmar_domain
*domain
,
1295 struct pci_dev
*pdev
)
1298 struct pci_dev
*tmp
, *parent
;
1300 ret
= device_context_mapped(domain
->iommu
,
1301 pdev
->bus
->number
, pdev
->devfn
);
1304 /* dependent device mapping */
1305 tmp
= pci_find_upstream_pcie_bridge(pdev
);
1308 /* Secondary interface's bus number and devfn 0 */
1309 parent
= pdev
->bus
->self
;
1310 while (parent
!= tmp
) {
1311 ret
= device_context_mapped(domain
->iommu
, parent
->bus
->number
,
1315 parent
= parent
->bus
->self
;
1318 return device_context_mapped(domain
->iommu
,
1319 tmp
->subordinate
->number
, 0);
1321 return device_context_mapped(domain
->iommu
,
1322 tmp
->bus
->number
, tmp
->devfn
);
1326 domain_page_mapping(struct dmar_domain
*domain
, dma_addr_t iova
,
1327 u64 hpa
, size_t size
, int prot
)
1329 u64 start_pfn
, end_pfn
;
1330 struct dma_pte
*pte
;
1332 int addr_width
= agaw_to_width(domain
->agaw
);
1334 hpa
&= (((u64
)1) << addr_width
) - 1;
1336 if ((prot
& (DMA_PTE_READ
|DMA_PTE_WRITE
)) == 0)
1339 start_pfn
= ((u64
)hpa
) >> VTD_PAGE_SHIFT
;
1340 end_pfn
= (VTD_PAGE_ALIGN(((u64
)hpa
) + size
)) >> VTD_PAGE_SHIFT
;
1342 while (start_pfn
< end_pfn
) {
1343 pte
= addr_to_dma_pte(domain
, iova
+ VTD_PAGE_SIZE
* index
);
1346 /* We don't need lock here, nobody else
1347 * touches the iova range
1349 BUG_ON(dma_pte_addr(*pte
));
1350 dma_set_pte_addr(*pte
, start_pfn
<< VTD_PAGE_SHIFT
);
1351 dma_set_pte_prot(*pte
, prot
);
1352 __iommu_flush_cache(domain
->iommu
, pte
, sizeof(*pte
));
1359 static void detach_domain_for_dev(struct dmar_domain
*domain
, u8 bus
, u8 devfn
)
1361 clear_context_table(domain
->iommu
, bus
, devfn
);
1362 domain
->iommu
->flush
.flush_context(domain
->iommu
, 0, 0, 0,
1363 DMA_CCMD_GLOBAL_INVL
, 0);
1364 domain
->iommu
->flush
.flush_iotlb(domain
->iommu
, 0, 0, 0,
1365 DMA_TLB_GLOBAL_FLUSH
, 0);
1368 static void domain_remove_dev_info(struct dmar_domain
*domain
)
1370 struct device_domain_info
*info
;
1371 unsigned long flags
;
1373 spin_lock_irqsave(&device_domain_lock
, flags
);
1374 while (!list_empty(&domain
->devices
)) {
1375 info
= list_entry(domain
->devices
.next
,
1376 struct device_domain_info
, link
);
1377 list_del(&info
->link
);
1378 list_del(&info
->global
);
1380 info
->dev
->dev
.archdata
.iommu
= NULL
;
1381 spin_unlock_irqrestore(&device_domain_lock
, flags
);
1383 detach_domain_for_dev(info
->domain
, info
->bus
, info
->devfn
);
1384 free_devinfo_mem(info
);
1386 spin_lock_irqsave(&device_domain_lock
, flags
);
1388 spin_unlock_irqrestore(&device_domain_lock
, flags
);
1393 * Note: we use struct pci_dev->dev.archdata.iommu stores the info
1395 static struct dmar_domain
*
1396 find_domain(struct pci_dev
*pdev
)
1398 struct device_domain_info
*info
;
1400 /* No lock here, assumes no domain exit in normal case */
1401 info
= pdev
->dev
.archdata
.iommu
;
1403 return info
->domain
;
1407 /* domain is initialized */
1408 static struct dmar_domain
*get_domain_for_dev(struct pci_dev
*pdev
, int gaw
)
1410 struct dmar_domain
*domain
, *found
= NULL
;
1411 struct intel_iommu
*iommu
;
1412 struct dmar_drhd_unit
*drhd
;
1413 struct device_domain_info
*info
, *tmp
;
1414 struct pci_dev
*dev_tmp
;
1415 unsigned long flags
;
1416 int bus
= 0, devfn
= 0;
1418 domain
= find_domain(pdev
);
1422 dev_tmp
= pci_find_upstream_pcie_bridge(pdev
);
1424 if (dev_tmp
->is_pcie
) {
1425 bus
= dev_tmp
->subordinate
->number
;
1428 bus
= dev_tmp
->bus
->number
;
1429 devfn
= dev_tmp
->devfn
;
1431 spin_lock_irqsave(&device_domain_lock
, flags
);
1432 list_for_each_entry(info
, &device_domain_list
, global
) {
1433 if (info
->bus
== bus
&& info
->devfn
== devfn
) {
1434 found
= info
->domain
;
1438 spin_unlock_irqrestore(&device_domain_lock
, flags
);
1439 /* pcie-pci bridge already has a domain, uses it */
1446 /* Allocate new domain for the device */
1447 drhd
= dmar_find_matched_drhd_unit(pdev
);
1449 printk(KERN_ERR
"IOMMU: can't find DMAR for device %s\n",
1453 iommu
= drhd
->iommu
;
1455 domain
= iommu_alloc_domain(iommu
);
1459 if (domain_init(domain
, gaw
)) {
1460 domain_exit(domain
);
1464 /* register pcie-to-pci device */
1466 info
= alloc_devinfo_mem();
1468 domain_exit(domain
);
1472 info
->devfn
= devfn
;
1474 info
->domain
= domain
;
1475 /* This domain is shared by devices under p2p bridge */
1476 domain
->flags
|= DOMAIN_FLAG_MULTIPLE_DEVICES
;
1478 /* pcie-to-pci bridge already has a domain, uses it */
1480 spin_lock_irqsave(&device_domain_lock
, flags
);
1481 list_for_each_entry(tmp
, &device_domain_list
, global
) {
1482 if (tmp
->bus
== bus
&& tmp
->devfn
== devfn
) {
1483 found
= tmp
->domain
;
1488 free_devinfo_mem(info
);
1489 domain_exit(domain
);
1492 list_add(&info
->link
, &domain
->devices
);
1493 list_add(&info
->global
, &device_domain_list
);
1495 spin_unlock_irqrestore(&device_domain_lock
, flags
);
1499 info
= alloc_devinfo_mem();
1502 info
->bus
= pdev
->bus
->number
;
1503 info
->devfn
= pdev
->devfn
;
1505 info
->domain
= domain
;
1506 spin_lock_irqsave(&device_domain_lock
, flags
);
1507 /* somebody is fast */
1508 found
= find_domain(pdev
);
1509 if (found
!= NULL
) {
1510 spin_unlock_irqrestore(&device_domain_lock
, flags
);
1511 if (found
!= domain
) {
1512 domain_exit(domain
);
1515 free_devinfo_mem(info
);
1518 list_add(&info
->link
, &domain
->devices
);
1519 list_add(&info
->global
, &device_domain_list
);
1520 pdev
->dev
.archdata
.iommu
= info
;
1521 spin_unlock_irqrestore(&device_domain_lock
, flags
);
1524 /* recheck it here, maybe others set it */
1525 return find_domain(pdev
);
1528 static int iommu_prepare_identity_map(struct pci_dev
*pdev
,
1529 unsigned long long start
,
1530 unsigned long long end
)
1532 struct dmar_domain
*domain
;
1534 unsigned long long base
;
1538 "IOMMU: Setting identity map for device %s [0x%Lx - 0x%Lx]\n",
1539 pci_name(pdev
), start
, end
);
1540 /* page table init */
1541 domain
= get_domain_for_dev(pdev
, DEFAULT_DOMAIN_ADDRESS_WIDTH
);
1545 /* The address might not be aligned */
1546 base
= start
& PAGE_MASK
;
1548 size
= PAGE_ALIGN(size
);
1549 if (!reserve_iova(&domain
->iovad
, IOVA_PFN(base
),
1550 IOVA_PFN(base
+ size
) - 1)) {
1551 printk(KERN_ERR
"IOMMU: reserve iova failed\n");
1556 pr_debug("Mapping reserved region %lx@%llx for %s\n",
1557 size
, base
, pci_name(pdev
));
1559 * RMRR range might have overlap with physical memory range,
1562 dma_pte_clear_range(domain
, base
, base
+ size
);
1564 ret
= domain_page_mapping(domain
, base
, base
, size
,
1565 DMA_PTE_READ
|DMA_PTE_WRITE
);
1569 /* context entry init */
1570 ret
= domain_context_mapping(domain
, pdev
);
1574 domain_exit(domain
);
1579 static inline int iommu_prepare_rmrr_dev(struct dmar_rmrr_unit
*rmrr
,
1580 struct pci_dev
*pdev
)
1582 if (pdev
->dev
.archdata
.iommu
== DUMMY_DEVICE_DOMAIN_INFO
)
1584 return iommu_prepare_identity_map(pdev
, rmrr
->base_address
,
1585 rmrr
->end_address
+ 1);
1588 #ifdef CONFIG_DMAR_GFX_WA
1589 struct iommu_prepare_data
{
1590 struct pci_dev
*pdev
;
1594 static int __init
iommu_prepare_work_fn(unsigned long start_pfn
,
1595 unsigned long end_pfn
, void *datax
)
1597 struct iommu_prepare_data
*data
;
1599 data
= (struct iommu_prepare_data
*)datax
;
1601 data
->ret
= iommu_prepare_identity_map(data
->pdev
,
1602 start_pfn
<<PAGE_SHIFT
, end_pfn
<<PAGE_SHIFT
);
1607 static int __init
iommu_prepare_with_active_regions(struct pci_dev
*pdev
)
1610 struct iommu_prepare_data data
;
1615 for_each_online_node(nid
) {
1616 work_with_active_regions(nid
, iommu_prepare_work_fn
, &data
);
1623 static void __init
iommu_prepare_gfx_mapping(void)
1625 struct pci_dev
*pdev
= NULL
;
1628 for_each_pci_dev(pdev
) {
1629 if (pdev
->dev
.archdata
.iommu
== DUMMY_DEVICE_DOMAIN_INFO
||
1630 !IS_GFX_DEVICE(pdev
))
1632 printk(KERN_INFO
"IOMMU: gfx device %s 1-1 mapping\n",
1634 ret
= iommu_prepare_with_active_regions(pdev
);
1636 printk(KERN_ERR
"IOMMU: mapping reserved region failed\n");
1641 #ifdef CONFIG_DMAR_FLOPPY_WA
1642 static inline void iommu_prepare_isa(void)
1644 struct pci_dev
*pdev
;
1647 pdev
= pci_get_class(PCI_CLASS_BRIDGE_ISA
<< 8, NULL
);
1651 printk(KERN_INFO
"IOMMU: Prepare 0-16M unity mapping for LPC\n");
1652 ret
= iommu_prepare_identity_map(pdev
, 0, 16*1024*1024);
1655 printk("IOMMU: Failed to create 0-64M identity map, "
1656 "floppy might not work\n");
1660 static inline void iommu_prepare_isa(void)
1664 #endif /* !CONFIG_DMAR_FLPY_WA */
1666 static int __init
init_dmars(void)
1668 struct dmar_drhd_unit
*drhd
;
1669 struct dmar_rmrr_unit
*rmrr
;
1670 struct pci_dev
*pdev
;
1671 struct intel_iommu
*iommu
;
1672 int i
, ret
, unit
= 0;
1677 * initialize and program root entry to not present
1680 for_each_drhd_unit(drhd
) {
1683 * lock not needed as this is only incremented in the single
1684 * threaded kernel __init code path all other access are read
1689 deferred_flush
= kzalloc(g_num_of_iommus
*
1690 sizeof(struct deferred_flush_tables
), GFP_KERNEL
);
1691 if (!deferred_flush
) {
1696 for_each_drhd_unit(drhd
) {
1700 iommu
= drhd
->iommu
;
1702 ret
= iommu_init_domains(iommu
);
1708 * we could share the same root & context tables
1709 * amoung all IOMMU's. Need to Split it later.
1711 ret
= iommu_alloc_root_entry(iommu
);
1713 printk(KERN_ERR
"IOMMU: allocate root entry failed\n");
1718 for_each_drhd_unit(drhd
) {
1722 iommu
= drhd
->iommu
;
1723 if (dmar_enable_qi(iommu
)) {
1725 * Queued Invalidate not enabled, use Register Based
1728 iommu
->flush
.flush_context
= __iommu_flush_context
;
1729 iommu
->flush
.flush_iotlb
= __iommu_flush_iotlb
;
1730 printk(KERN_INFO
"IOMMU 0x%Lx: using Register based "
1732 (unsigned long long)drhd
->reg_base_addr
);
1734 iommu
->flush
.flush_context
= qi_flush_context
;
1735 iommu
->flush
.flush_iotlb
= qi_flush_iotlb
;
1736 printk(KERN_INFO
"IOMMU 0x%Lx: using Queued "
1738 (unsigned long long)drhd
->reg_base_addr
);
1744 * for each dev attached to rmrr
1746 * locate drhd for dev, alloc domain for dev
1747 * allocate free domain
1748 * allocate page table entries for rmrr
1749 * if context not allocated for bus
1750 * allocate and init context
1751 * set present in root table for this bus
1752 * init context with domain, translation etc
1756 for_each_rmrr_units(rmrr
) {
1757 for (i
= 0; i
< rmrr
->devices_cnt
; i
++) {
1758 pdev
= rmrr
->devices
[i
];
1759 /* some BIOS lists non-exist devices in DMAR table */
1762 ret
= iommu_prepare_rmrr_dev(rmrr
, pdev
);
1765 "IOMMU: mapping reserved region failed\n");
1769 iommu_prepare_gfx_mapping();
1771 iommu_prepare_isa();
1776 * global invalidate context cache
1777 * global invalidate iotlb
1778 * enable translation
1780 for_each_drhd_unit(drhd
) {
1783 iommu
= drhd
->iommu
;
1784 sprintf (iommu
->name
, "dmar%d", unit
++);
1786 iommu_flush_write_buffer(iommu
);
1788 ret
= dmar_set_interrupt(iommu
);
1792 iommu_set_root_entry(iommu
);
1794 iommu
->flush
.flush_context(iommu
, 0, 0, 0, DMA_CCMD_GLOBAL_INVL
,
1796 iommu
->flush
.flush_iotlb(iommu
, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH
,
1798 iommu_disable_protect_mem_regions(iommu
);
1800 ret
= iommu_enable_translation(iommu
);
1807 for_each_drhd_unit(drhd
) {
1810 iommu
= drhd
->iommu
;
1816 static inline u64
aligned_size(u64 host_addr
, size_t size
)
1819 addr
= (host_addr
& (~PAGE_MASK
)) + size
;
1820 return PAGE_ALIGN(addr
);
1824 iommu_alloc_iova(struct dmar_domain
*domain
, size_t size
, u64 end
)
1828 /* Make sure it's in range */
1829 end
= min_t(u64
, DOMAIN_MAX_ADDR(domain
->gaw
), end
);
1830 if (!size
|| (IOVA_START_ADDR
+ size
> end
))
1833 piova
= alloc_iova(&domain
->iovad
,
1834 size
>> PAGE_SHIFT
, IOVA_PFN(end
), 1);
1838 static struct iova
*
1839 __intel_alloc_iova(struct device
*dev
, struct dmar_domain
*domain
,
1840 size_t size
, u64 dma_mask
)
1842 struct pci_dev
*pdev
= to_pci_dev(dev
);
1843 struct iova
*iova
= NULL
;
1845 if (dma_mask
<= DMA_32BIT_MASK
|| dmar_forcedac
)
1846 iova
= iommu_alloc_iova(domain
, size
, dma_mask
);
1849 * First try to allocate an io virtual address in
1850 * DMA_32BIT_MASK and if that fails then try allocating
1853 iova
= iommu_alloc_iova(domain
, size
, DMA_32BIT_MASK
);
1855 iova
= iommu_alloc_iova(domain
, size
, dma_mask
);
1859 printk(KERN_ERR
"Allocating iova for %s failed", pci_name(pdev
));
1866 static struct dmar_domain
*
1867 get_valid_domain_for_dev(struct pci_dev
*pdev
)
1869 struct dmar_domain
*domain
;
1872 domain
= get_domain_for_dev(pdev
,
1873 DEFAULT_DOMAIN_ADDRESS_WIDTH
);
1876 "Allocating domain for %s failed", pci_name(pdev
));
1880 /* make sure context mapping is ok */
1881 if (unlikely(!domain_context_mapped(domain
, pdev
))) {
1882 ret
= domain_context_mapping(domain
, pdev
);
1885 "Domain context map for %s failed",
1894 static dma_addr_t
__intel_map_single(struct device
*hwdev
, phys_addr_t paddr
,
1895 size_t size
, int dir
, u64 dma_mask
)
1897 struct pci_dev
*pdev
= to_pci_dev(hwdev
);
1898 struct dmar_domain
*domain
;
1899 phys_addr_t start_paddr
;
1904 BUG_ON(dir
== DMA_NONE
);
1905 if (pdev
->dev
.archdata
.iommu
== DUMMY_DEVICE_DOMAIN_INFO
)
1908 domain
= get_valid_domain_for_dev(pdev
);
1912 size
= aligned_size((u64
)paddr
, size
);
1914 iova
= __intel_alloc_iova(hwdev
, domain
, size
, pdev
->dma_mask
);
1918 start_paddr
= (phys_addr_t
)iova
->pfn_lo
<< PAGE_SHIFT
;
1921 * Check if DMAR supports zero-length reads on write only
1924 if (dir
== DMA_TO_DEVICE
|| dir
== DMA_BIDIRECTIONAL
|| \
1925 !cap_zlr(domain
->iommu
->cap
))
1926 prot
|= DMA_PTE_READ
;
1927 if (dir
== DMA_FROM_DEVICE
|| dir
== DMA_BIDIRECTIONAL
)
1928 prot
|= DMA_PTE_WRITE
;
1930 * paddr - (paddr + size) might be partial page, we should map the whole
1931 * page. Note: if two part of one page are separately mapped, we
1932 * might have two guest_addr mapping to the same host paddr, but this
1933 * is not a big problem
1935 ret
= domain_page_mapping(domain
, start_paddr
,
1936 ((u64
)paddr
) & PAGE_MASK
, size
, prot
);
1940 /* it's a non-present to present mapping */
1941 ret
= iommu_flush_iotlb_psi(domain
->iommu
, domain
->id
,
1942 start_paddr
, size
>> VTD_PAGE_SHIFT
, 1);
1944 iommu_flush_write_buffer(domain
->iommu
);
1946 return start_paddr
+ ((u64
)paddr
& (~PAGE_MASK
));
1950 __free_iova(&domain
->iovad
, iova
);
1951 printk(KERN_ERR
"Device %s request: %lx@%llx dir %d --- failed\n",
1952 pci_name(pdev
), size
, (unsigned long long)paddr
, dir
);
1956 dma_addr_t
intel_map_single(struct device
*hwdev
, phys_addr_t paddr
,
1957 size_t size
, int dir
)
1959 return __intel_map_single(hwdev
, paddr
, size
, dir
,
1960 to_pci_dev(hwdev
)->dma_mask
);
1963 static void flush_unmaps(void)
1969 /* just flush them all */
1970 for (i
= 0; i
< g_num_of_iommus
; i
++) {
1971 if (deferred_flush
[i
].next
) {
1972 struct intel_iommu
*iommu
=
1973 deferred_flush
[i
].domain
[0]->iommu
;
1975 iommu
->flush
.flush_iotlb(iommu
, 0, 0, 0,
1976 DMA_TLB_GLOBAL_FLUSH
, 0);
1977 for (j
= 0; j
< deferred_flush
[i
].next
; j
++) {
1978 __free_iova(&deferred_flush
[i
].domain
[j
]->iovad
,
1979 deferred_flush
[i
].iova
[j
]);
1981 deferred_flush
[i
].next
= 0;
1988 static void flush_unmaps_timeout(unsigned long data
)
1990 unsigned long flags
;
1992 spin_lock_irqsave(&async_umap_flush_lock
, flags
);
1994 spin_unlock_irqrestore(&async_umap_flush_lock
, flags
);
1997 static void add_unmap(struct dmar_domain
*dom
, struct iova
*iova
)
1999 unsigned long flags
;
2002 spin_lock_irqsave(&async_umap_flush_lock
, flags
);
2003 if (list_size
== HIGH_WATER_MARK
)
2006 iommu_id
= dom
->iommu
->seq_id
;
2008 next
= deferred_flush
[iommu_id
].next
;
2009 deferred_flush
[iommu_id
].domain
[next
] = dom
;
2010 deferred_flush
[iommu_id
].iova
[next
] = iova
;
2011 deferred_flush
[iommu_id
].next
++;
2014 mod_timer(&unmap_timer
, jiffies
+ msecs_to_jiffies(10));
2018 spin_unlock_irqrestore(&async_umap_flush_lock
, flags
);
2021 void intel_unmap_single(struct device
*dev
, dma_addr_t dev_addr
, size_t size
,
2024 struct pci_dev
*pdev
= to_pci_dev(dev
);
2025 struct dmar_domain
*domain
;
2026 unsigned long start_addr
;
2029 if (pdev
->dev
.archdata
.iommu
== DUMMY_DEVICE_DOMAIN_INFO
)
2031 domain
= find_domain(pdev
);
2034 iova
= find_iova(&domain
->iovad
, IOVA_PFN(dev_addr
));
2038 start_addr
= iova
->pfn_lo
<< PAGE_SHIFT
;
2039 size
= aligned_size((u64
)dev_addr
, size
);
2041 pr_debug("Device %s unmapping: %lx@%llx\n",
2042 pci_name(pdev
), size
, (unsigned long long)start_addr
);
2044 /* clear the whole page */
2045 dma_pte_clear_range(domain
, start_addr
, start_addr
+ size
);
2046 /* free page tables */
2047 dma_pte_free_pagetable(domain
, start_addr
, start_addr
+ size
);
2048 if (intel_iommu_strict
) {
2049 if (iommu_flush_iotlb_psi(domain
->iommu
,
2050 domain
->id
, start_addr
, size
>> VTD_PAGE_SHIFT
, 0))
2051 iommu_flush_write_buffer(domain
->iommu
);
2053 __free_iova(&domain
->iovad
, iova
);
2055 add_unmap(domain
, iova
);
2057 * queue up the release of the unmap to save the 1/6th of the
2058 * cpu used up by the iotlb flush operation...
2063 void *intel_alloc_coherent(struct device
*hwdev
, size_t size
,
2064 dma_addr_t
*dma_handle
, gfp_t flags
)
2069 size
= PAGE_ALIGN(size
);
2070 order
= get_order(size
);
2071 flags
&= ~(GFP_DMA
| GFP_DMA32
);
2073 vaddr
= (void *)__get_free_pages(flags
, order
);
2076 memset(vaddr
, 0, size
);
2078 *dma_handle
= __intel_map_single(hwdev
, virt_to_bus(vaddr
), size
,
2080 hwdev
->coherent_dma_mask
);
2083 free_pages((unsigned long)vaddr
, order
);
2087 void intel_free_coherent(struct device
*hwdev
, size_t size
, void *vaddr
,
2088 dma_addr_t dma_handle
)
2092 size
= PAGE_ALIGN(size
);
2093 order
= get_order(size
);
2095 intel_unmap_single(hwdev
, dma_handle
, size
, DMA_BIDIRECTIONAL
);
2096 free_pages((unsigned long)vaddr
, order
);
2099 #define SG_ENT_VIRT_ADDRESS(sg) (sg_virt((sg)))
2101 void intel_unmap_sg(struct device
*hwdev
, struct scatterlist
*sglist
,
2102 int nelems
, int dir
)
2105 struct pci_dev
*pdev
= to_pci_dev(hwdev
);
2106 struct dmar_domain
*domain
;
2107 unsigned long start_addr
;
2111 struct scatterlist
*sg
;
2113 if (pdev
->dev
.archdata
.iommu
== DUMMY_DEVICE_DOMAIN_INFO
)
2116 domain
= find_domain(pdev
);
2118 iova
= find_iova(&domain
->iovad
, IOVA_PFN(sglist
[0].dma_address
));
2121 for_each_sg(sglist
, sg
, nelems
, i
) {
2122 addr
= SG_ENT_VIRT_ADDRESS(sg
);
2123 size
+= aligned_size((u64
)addr
, sg
->length
);
2126 start_addr
= iova
->pfn_lo
<< PAGE_SHIFT
;
2128 /* clear the whole page */
2129 dma_pte_clear_range(domain
, start_addr
, start_addr
+ size
);
2130 /* free page tables */
2131 dma_pte_free_pagetable(domain
, start_addr
, start_addr
+ size
);
2133 if (iommu_flush_iotlb_psi(domain
->iommu
, domain
->id
, start_addr
,
2134 size
>> VTD_PAGE_SHIFT
, 0))
2135 iommu_flush_write_buffer(domain
->iommu
);
2138 __free_iova(&domain
->iovad
, iova
);
2141 static int intel_nontranslate_map_sg(struct device
*hddev
,
2142 struct scatterlist
*sglist
, int nelems
, int dir
)
2145 struct scatterlist
*sg
;
2147 for_each_sg(sglist
, sg
, nelems
, i
) {
2148 BUG_ON(!sg_page(sg
));
2149 sg
->dma_address
= virt_to_bus(SG_ENT_VIRT_ADDRESS(sg
));
2150 sg
->dma_length
= sg
->length
;
2155 int intel_map_sg(struct device
*hwdev
, struct scatterlist
*sglist
, int nelems
,
2160 struct pci_dev
*pdev
= to_pci_dev(hwdev
);
2161 struct dmar_domain
*domain
;
2165 struct iova
*iova
= NULL
;
2167 struct scatterlist
*sg
;
2168 unsigned long start_addr
;
2170 BUG_ON(dir
== DMA_NONE
);
2171 if (pdev
->dev
.archdata
.iommu
== DUMMY_DEVICE_DOMAIN_INFO
)
2172 return intel_nontranslate_map_sg(hwdev
, sglist
, nelems
, dir
);
2174 domain
= get_valid_domain_for_dev(pdev
);
2178 for_each_sg(sglist
, sg
, nelems
, i
) {
2179 addr
= SG_ENT_VIRT_ADDRESS(sg
);
2180 addr
= (void *)virt_to_phys(addr
);
2181 size
+= aligned_size((u64
)addr
, sg
->length
);
2184 iova
= __intel_alloc_iova(hwdev
, domain
, size
, pdev
->dma_mask
);
2186 sglist
->dma_length
= 0;
2191 * Check if DMAR supports zero-length reads on write only
2194 if (dir
== DMA_TO_DEVICE
|| dir
== DMA_BIDIRECTIONAL
|| \
2195 !cap_zlr(domain
->iommu
->cap
))
2196 prot
|= DMA_PTE_READ
;
2197 if (dir
== DMA_FROM_DEVICE
|| dir
== DMA_BIDIRECTIONAL
)
2198 prot
|= DMA_PTE_WRITE
;
2200 start_addr
= iova
->pfn_lo
<< PAGE_SHIFT
;
2202 for_each_sg(sglist
, sg
, nelems
, i
) {
2203 addr
= SG_ENT_VIRT_ADDRESS(sg
);
2204 addr
= (void *)virt_to_phys(addr
);
2205 size
= aligned_size((u64
)addr
, sg
->length
);
2206 ret
= domain_page_mapping(domain
, start_addr
+ offset
,
2207 ((u64
)addr
) & PAGE_MASK
,
2210 /* clear the page */
2211 dma_pte_clear_range(domain
, start_addr
,
2212 start_addr
+ offset
);
2213 /* free page tables */
2214 dma_pte_free_pagetable(domain
, start_addr
,
2215 start_addr
+ offset
);
2217 __free_iova(&domain
->iovad
, iova
);
2220 sg
->dma_address
= start_addr
+ offset
+
2221 ((u64
)addr
& (~PAGE_MASK
));
2222 sg
->dma_length
= sg
->length
;
2226 /* it's a non-present to present mapping */
2227 if (iommu_flush_iotlb_psi(domain
->iommu
, domain
->id
,
2228 start_addr
, offset
>> VTD_PAGE_SHIFT
, 1))
2229 iommu_flush_write_buffer(domain
->iommu
);
2233 static struct dma_mapping_ops intel_dma_ops
= {
2234 .alloc_coherent
= intel_alloc_coherent
,
2235 .free_coherent
= intel_free_coherent
,
2236 .map_single
= intel_map_single
,
2237 .unmap_single
= intel_unmap_single
,
2238 .map_sg
= intel_map_sg
,
2239 .unmap_sg
= intel_unmap_sg
,
2242 static inline int iommu_domain_cache_init(void)
2246 iommu_domain_cache
= kmem_cache_create("iommu_domain",
2247 sizeof(struct dmar_domain
),
2252 if (!iommu_domain_cache
) {
2253 printk(KERN_ERR
"Couldn't create iommu_domain cache\n");
2260 static inline int iommu_devinfo_cache_init(void)
2264 iommu_devinfo_cache
= kmem_cache_create("iommu_devinfo",
2265 sizeof(struct device_domain_info
),
2269 if (!iommu_devinfo_cache
) {
2270 printk(KERN_ERR
"Couldn't create devinfo cache\n");
2277 static inline int iommu_iova_cache_init(void)
2281 iommu_iova_cache
= kmem_cache_create("iommu_iova",
2282 sizeof(struct iova
),
2286 if (!iommu_iova_cache
) {
2287 printk(KERN_ERR
"Couldn't create iova cache\n");
2294 static int __init
iommu_init_mempool(void)
2297 ret
= iommu_iova_cache_init();
2301 ret
= iommu_domain_cache_init();
2305 ret
= iommu_devinfo_cache_init();
2309 kmem_cache_destroy(iommu_domain_cache
);
2311 kmem_cache_destroy(iommu_iova_cache
);
2316 static void __init
iommu_exit_mempool(void)
2318 kmem_cache_destroy(iommu_devinfo_cache
);
2319 kmem_cache_destroy(iommu_domain_cache
);
2320 kmem_cache_destroy(iommu_iova_cache
);
2324 static void __init
init_no_remapping_devices(void)
2326 struct dmar_drhd_unit
*drhd
;
2328 for_each_drhd_unit(drhd
) {
2329 if (!drhd
->include_all
) {
2331 for (i
= 0; i
< drhd
->devices_cnt
; i
++)
2332 if (drhd
->devices
[i
] != NULL
)
2334 /* ignore DMAR unit if no pci devices exist */
2335 if (i
== drhd
->devices_cnt
)
2343 for_each_drhd_unit(drhd
) {
2345 if (drhd
->ignored
|| drhd
->include_all
)
2348 for (i
= 0; i
< drhd
->devices_cnt
; i
++)
2349 if (drhd
->devices
[i
] &&
2350 !IS_GFX_DEVICE(drhd
->devices
[i
]))
2353 if (i
< drhd
->devices_cnt
)
2356 /* bypass IOMMU if it is just for gfx devices */
2358 for (i
= 0; i
< drhd
->devices_cnt
; i
++) {
2359 if (!drhd
->devices
[i
])
2361 drhd
->devices
[i
]->dev
.archdata
.iommu
= DUMMY_DEVICE_DOMAIN_INFO
;
2366 int __init
intel_iommu_init(void)
2370 if (dmar_table_init())
2373 if (dmar_dev_scope_init())
2377 * Check the need for DMA-remapping initialization now.
2378 * Above initialization will also be used by Interrupt-remapping.
2380 if (no_iommu
|| swiotlb
|| dmar_disabled
)
2383 iommu_init_mempool();
2384 dmar_init_reserved_ranges();
2386 init_no_remapping_devices();
2390 printk(KERN_ERR
"IOMMU: dmar init failed\n");
2391 put_iova_domain(&reserved_iova_list
);
2392 iommu_exit_mempool();
2396 "PCI-DMA: Intel(R) Virtualization Technology for Directed I/O\n");
2398 init_timer(&unmap_timer
);
2400 dma_ops
= &intel_dma_ops
;
2404 void intel_iommu_domain_exit(struct dmar_domain
*domain
)
2408 /* Domain 0 is reserved, so dont process it */
2412 end
= DOMAIN_MAX_ADDR(domain
->gaw
);
2413 end
= end
& (~VTD_PAGE_MASK
);
2416 dma_pte_clear_range(domain
, 0, end
);
2418 /* free page tables */
2419 dma_pte_free_pagetable(domain
, 0, end
);
2421 iommu_free_domain(domain
);
2422 free_domain_mem(domain
);
2424 EXPORT_SYMBOL_GPL(intel_iommu_domain_exit
);
2426 struct dmar_domain
*intel_iommu_domain_alloc(struct pci_dev
*pdev
)
2428 struct dmar_drhd_unit
*drhd
;
2429 struct dmar_domain
*domain
;
2430 struct intel_iommu
*iommu
;
2432 drhd
= dmar_find_matched_drhd_unit(pdev
);
2434 printk(KERN_ERR
"intel_iommu_domain_alloc: drhd == NULL\n");
2438 iommu
= drhd
->iommu
;
2441 "intel_iommu_domain_alloc: iommu == NULL\n");
2444 domain
= iommu_alloc_domain(iommu
);
2447 "intel_iommu_domain_alloc: domain == NULL\n");
2450 if (domain_init(domain
, DEFAULT_DOMAIN_ADDRESS_WIDTH
)) {
2452 "intel_iommu_domain_alloc: domain_init() failed\n");
2453 intel_iommu_domain_exit(domain
);
2458 EXPORT_SYMBOL_GPL(intel_iommu_domain_alloc
);
2460 int intel_iommu_context_mapping(
2461 struct dmar_domain
*domain
, struct pci_dev
*pdev
)
2464 rc
= domain_context_mapping(domain
, pdev
);
2467 EXPORT_SYMBOL_GPL(intel_iommu_context_mapping
);
2469 int intel_iommu_page_mapping(
2470 struct dmar_domain
*domain
, dma_addr_t iova
,
2471 u64 hpa
, size_t size
, int prot
)
2474 rc
= domain_page_mapping(domain
, iova
, hpa
, size
, prot
);
2477 EXPORT_SYMBOL_GPL(intel_iommu_page_mapping
);
2479 void intel_iommu_detach_dev(struct dmar_domain
*domain
, u8 bus
, u8 devfn
)
2481 detach_domain_for_dev(domain
, bus
, devfn
);
2483 EXPORT_SYMBOL_GPL(intel_iommu_detach_dev
);
2485 struct dmar_domain
*
2486 intel_iommu_find_domain(struct pci_dev
*pdev
)
2488 return find_domain(pdev
);
2490 EXPORT_SYMBOL_GPL(intel_iommu_find_domain
);
2492 int intel_iommu_found(void)
2494 return g_num_of_iommus
;
2496 EXPORT_SYMBOL_GPL(intel_iommu_found
);
2498 u64
intel_iommu_iova_to_pfn(struct dmar_domain
*domain
, u64 iova
)
2500 struct dma_pte
*pte
;
2504 pte
= addr_to_dma_pte(domain
, iova
);
2507 pfn
= dma_pte_addr(*pte
);
2509 return pfn
>> VTD_PAGE_SHIFT
;
2511 EXPORT_SYMBOL_GPL(intel_iommu_iova_to_pfn
);