projects / deliverable / linux.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
intel-iommu: move root entry defs from dma_remapping.h
[deliverable/linux.git] / drivers / pci / intel-iommu.c
1 /*
2 * Copyright (c) 2006, Intel Corporation.
3 *
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms and conditions of the GNU General Public License,
6 * version 2, as published by the Free Software Foundation.
7 *
8 * This program is distributed in the hope it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
11 * more details.
12 *
13 * You should have received a copy of the GNU General Public License along with
14 * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
15 * Place - Suite 330, Boston, MA 02111-1307 USA.
16 *
17 * Copyright (C) 2006-2008 Intel Corporation
18 * Author: Ashok Raj <ashok.raj@intel.com>
19 * Author: Shaohua Li <shaohua.li@intel.com>
20 * Author: Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>
21 * Author: Fenghua Yu <fenghua.yu@intel.com>
22 */
23
24 #include <linux/init.h>
25 #include <linux/bitmap.h>
26 #include <linux/debugfs.h>
27 #include <linux/slab.h>
28 #include <linux/irq.h>
29 #include <linux/interrupt.h>
30 #include <linux/spinlock.h>
31 #include <linux/pci.h>
32 #include <linux/dmar.h>
33 #include <linux/dma-mapping.h>
34 #include <linux/mempool.h>
35 #include <linux/timer.h>
36 #include <linux/iova.h>
37 #include <linux/intel-iommu.h>
38 #include <asm/cacheflush.h>
39 #include <asm/iommu.h>
40 #include "pci.h"
41
42 #define ROOT_SIZE VTD_PAGE_SIZE
43 #define CONTEXT_SIZE VTD_PAGE_SIZE
44
45 #define IS_GFX_DEVICE(pdev) ((pdev->class >> 16) == PCI_BASE_CLASS_DISPLAY)
46 #define IS_ISA_DEVICE(pdev) ((pdev->class >> 8) == PCI_CLASS_BRIDGE_ISA)
47
48 #define IOAPIC_RANGE_START (0xfee00000)
49 #define IOAPIC_RANGE_END (0xfeefffff)
50 #define IOVA_START_ADDR (0x1000)
51
52 #define DEFAULT_DOMAIN_ADDRESS_WIDTH 48
53
54 #define DOMAIN_MAX_ADDR(gaw) ((((u64)1) << gaw) - 1)
55
56 #define IOVA_PFN(addr) ((addr) >> PAGE_SHIFT)
57 #define DMA_32BIT_PFN IOVA_PFN(DMA_32BIT_MASK)
58 #define DMA_64BIT_PFN IOVA_PFN(DMA_64BIT_MASK)
59
60 /*
61 * 0: Present
62 * 1-11: Reserved
63 * 12-63: Context Ptr (12 - (haw-1))
64 * 64-127: Reserved
65 */
66 struct root_entry {
67 u64 val;
68 u64 rsvd1;
69 };
70 #define ROOT_ENTRY_NR (VTD_PAGE_SIZE/sizeof(struct root_entry))
71 static inline bool root_present(struct root_entry *root)
72 {
73 return (root->val & 1);
74 }
75 static inline void set_root_present(struct root_entry *root)
76 {
77 root->val |= 1;
78 }
79 static inline void set_root_value(struct root_entry *root, unsigned long value)
80 {
81 root->val |= value & VTD_PAGE_MASK;
82 }
83
84 static inline struct context_entry *
85 get_context_addr_from_root(struct root_entry *root)
86 {
87 return (struct context_entry *)
88 (root_present(root)?phys_to_virt(
89 root->val & VTD_PAGE_MASK) :
90 NULL);
91 }
92
93 static void flush_unmaps_timeout(unsigned long data);
94
95 DEFINE_TIMER(unmap_timer, flush_unmaps_timeout, 0, 0);
96
97 #define HIGH_WATER_MARK 250
98 struct deferred_flush_tables {
99 int next;
100 struct iova *iova[HIGH_WATER_MARK];
101 struct dmar_domain *domain[HIGH_WATER_MARK];
102 };
103
104 static struct deferred_flush_tables *deferred_flush;
105
106 /* bitmap for indexing intel_iommus */
107 static int g_num_of_iommus;
108
109 static DEFINE_SPINLOCK(async_umap_flush_lock);
110 static LIST_HEAD(unmaps_to_do);
111
112 static int timer_on;
113 static long list_size;
114
115 static void domain_remove_dev_info(struct dmar_domain *domain);
116
117 int dmar_disabled;
118 static int __initdata dmar_map_gfx = 1;
119 static int dmar_forcedac;
120 static int intel_iommu_strict;
121
122 #define DUMMY_DEVICE_DOMAIN_INFO ((struct device_domain_info *)(-1))
123 static DEFINE_SPINLOCK(device_domain_lock);
124 static LIST_HEAD(device_domain_list);
125
126 static int __init intel_iommu_setup(char *str)
127 {
128 if (!str)
129 return -EINVAL;
130 while (*str) {
131 if (!strncmp(str, "off", 3)) {
132 dmar_disabled = 1;
133 printk(KERN_INFO"Intel-IOMMU: disabled\n");
134 } else if (!strncmp(str, "igfx_off", 8)) {
135 dmar_map_gfx = 0;
136 printk(KERN_INFO
137 "Intel-IOMMU: disable GFX device mapping\n");
138 } else if (!strncmp(str, "forcedac", 8)) {
139 printk(KERN_INFO
140 "Intel-IOMMU: Forcing DAC for PCI devices\n");
141 dmar_forcedac = 1;
142 } else if (!strncmp(str, "strict", 6)) {
143 printk(KERN_INFO
144 "Intel-IOMMU: disable batched IOTLB flush\n");
145 intel_iommu_strict = 1;
146 }
147
148 str += strcspn(str, ",");
149 while (*str == ',')
150 str++;
151 }
152 return 0;
153 }
154 __setup("intel_iommu=", intel_iommu_setup);
155
156 static struct kmem_cache *iommu_domain_cache;
157 static struct kmem_cache *iommu_devinfo_cache;
158 static struct kmem_cache *iommu_iova_cache;
159
160 static inline void *iommu_kmem_cache_alloc(struct kmem_cache *cachep)
161 {
162 unsigned int flags;
163 void *vaddr;
164
165 /* trying to avoid low memory issues */
166 flags = current->flags & PF_MEMALLOC;
167 current->flags |= PF_MEMALLOC;
168 vaddr = kmem_cache_alloc(cachep, GFP_ATOMIC);
169 current->flags &= (~PF_MEMALLOC | flags);
170 return vaddr;
171 }
172
173
174 static inline void *alloc_pgtable_page(void)
175 {
176 unsigned int flags;
177 void *vaddr;
178
179 /* trying to avoid low memory issues */
180 flags = current->flags & PF_MEMALLOC;
181 current->flags |= PF_MEMALLOC;
182 vaddr = (void *)get_zeroed_page(GFP_ATOMIC);
183 current->flags &= (~PF_MEMALLOC | flags);
184 return vaddr;
185 }
186
187 static inline void free_pgtable_page(void *vaddr)
188 {
189 free_page((unsigned long)vaddr);
190 }
191
192 static inline void *alloc_domain_mem(void)
193 {
194 return iommu_kmem_cache_alloc(iommu_domain_cache);
195 }
196
197 static void free_domain_mem(void *vaddr)
198 {
199 kmem_cache_free(iommu_domain_cache, vaddr);
200 }
201
202 static inline void * alloc_devinfo_mem(void)
203 {
204 return iommu_kmem_cache_alloc(iommu_devinfo_cache);
205 }
206
207 static inline void free_devinfo_mem(void *vaddr)
208 {
209 kmem_cache_free(iommu_devinfo_cache, vaddr);
210 }
211
212 struct iova *alloc_iova_mem(void)
213 {
214 return iommu_kmem_cache_alloc(iommu_iova_cache);
215 }
216
217 void free_iova_mem(struct iova *iova)
218 {
219 kmem_cache_free(iommu_iova_cache, iova);
220 }
221
222 /* Gets context entry for a given bus and devfn */
223 static struct context_entry * device_to_context_entry(struct intel_iommu *iommu,
224 u8 bus, u8 devfn)
225 {
226 struct root_entry *root;
227 struct context_entry *context;
228 unsigned long phy_addr;
229 unsigned long flags;
230
231 spin_lock_irqsave(&iommu->lock, flags);
232 root = &iommu->root_entry[bus];
233 context = get_context_addr_from_root(root);
234 if (!context) {
235 context = (struct context_entry *)alloc_pgtable_page();
236 if (!context) {
237 spin_unlock_irqrestore(&iommu->lock, flags);
238 return NULL;
239 }
240 __iommu_flush_cache(iommu, (void *)context, CONTEXT_SIZE);
241 phy_addr = virt_to_phys((void *)context);
242 set_root_value(root, phy_addr);
243 set_root_present(root);
244 __iommu_flush_cache(iommu, root, sizeof(*root));
245 }
246 spin_unlock_irqrestore(&iommu->lock, flags);
247 return &context[devfn];
248 }
249
250 static int device_context_mapped(struct intel_iommu *iommu, u8 bus, u8 devfn)
251 {
252 struct root_entry *root;
253 struct context_entry *context;
254 int ret;
255 unsigned long flags;
256
257 spin_lock_irqsave(&iommu->lock, flags);
258 root = &iommu->root_entry[bus];
259 context = get_context_addr_from_root(root);
260 if (!context) {
261 ret = 0;
262 goto out;
263 }
264 ret = context_present(context[devfn]);
265 out:
266 spin_unlock_irqrestore(&iommu->lock, flags);
267 return ret;
268 }
269
270 static void clear_context_table(struct intel_iommu *iommu, u8 bus, u8 devfn)
271 {
272 struct root_entry *root;
273 struct context_entry *context;
274 unsigned long flags;
275
276 spin_lock_irqsave(&iommu->lock, flags);
277 root = &iommu->root_entry[bus];
278 context = get_context_addr_from_root(root);
279 if (context) {
280 context_clear_entry(context[devfn]);
281 __iommu_flush_cache(iommu, &context[devfn], \
282 sizeof(*context));
283 }
284 spin_unlock_irqrestore(&iommu->lock, flags);
285 }
286
287 static void free_context_table(struct intel_iommu *iommu)
288 {
289 struct root_entry *root;
290 int i;
291 unsigned long flags;
292 struct context_entry *context;
293
294 spin_lock_irqsave(&iommu->lock, flags);
295 if (!iommu->root_entry) {
296 goto out;
297 }
298 for (i = 0; i < ROOT_ENTRY_NR; i++) {
299 root = &iommu->root_entry[i];
300 context = get_context_addr_from_root(root);
301 if (context)
302 free_pgtable_page(context);
303 }
304 free_pgtable_page(iommu->root_entry);
305 iommu->root_entry = NULL;
306 out:
307 spin_unlock_irqrestore(&iommu->lock, flags);
308 }
309
310 /* page table handling */
311 #define LEVEL_STRIDE (9)
312 #define LEVEL_MASK (((u64)1 << LEVEL_STRIDE) - 1)
313
314 static inline int agaw_to_level(int agaw)
315 {
316 return agaw + 2;
317 }
318
319 static inline int agaw_to_width(int agaw)
320 {
321 return 30 + agaw * LEVEL_STRIDE;
322
323 }
324
325 static inline int width_to_agaw(int width)
326 {
327 return (width - 30) / LEVEL_STRIDE;
328 }
329
330 static inline unsigned int level_to_offset_bits(int level)
331 {
332 return (12 + (level - 1) * LEVEL_STRIDE);
333 }
334
335 static inline int address_level_offset(u64 addr, int level)
336 {
337 return ((addr >> level_to_offset_bits(level)) & LEVEL_MASK);
338 }
339
340 static inline u64 level_mask(int level)
341 {
342 return ((u64)-1 << level_to_offset_bits(level));
343 }
344
345 static inline u64 level_size(int level)
346 {
347 return ((u64)1 << level_to_offset_bits(level));
348 }
349
350 static inline u64 align_to_level(u64 addr, int level)
351 {
352 return ((addr + level_size(level) - 1) & level_mask(level));
353 }
354
355 static struct dma_pte * addr_to_dma_pte(struct dmar_domain *domain, u64 addr)
356 {
357 int addr_width = agaw_to_width(domain->agaw);
358 struct dma_pte *parent, *pte = NULL;
359 int level = agaw_to_level(domain->agaw);
360 int offset;
361 unsigned long flags;
362
363 BUG_ON(!domain->pgd);
364
365 addr &= (((u64)1) << addr_width) - 1;
366 parent = domain->pgd;
367
368 spin_lock_irqsave(&domain->mapping_lock, flags);
369 while (level > 0) {
370 void *tmp_page;
371
372 offset = address_level_offset(addr, level);
373 pte = &parent[offset];
374 if (level == 1)
375 break;
376
377 if (!dma_pte_present(*pte)) {
378 tmp_page = alloc_pgtable_page();
379
380 if (!tmp_page) {
381 spin_unlock_irqrestore(&domain->mapping_lock,
382 flags);
383 return NULL;
384 }
385 __iommu_flush_cache(domain->iommu, tmp_page,
386 PAGE_SIZE);
387 dma_set_pte_addr(*pte, virt_to_phys(tmp_page));
388 /*
389 * high level table always sets r/w, last level page
390 * table control read/write
391 */
392 dma_set_pte_readable(*pte);
393 dma_set_pte_writable(*pte);
394 __iommu_flush_cache(domain->iommu, pte, sizeof(*pte));
395 }
396 parent = phys_to_virt(dma_pte_addr(*pte));
397 level--;
398 }
399
400 spin_unlock_irqrestore(&domain->mapping_lock, flags);
401 return pte;
402 }
403
404 /* return address's pte at specific level */
405 static struct dma_pte *dma_addr_level_pte(struct dmar_domain *domain, u64 addr,
406 int level)
407 {
408 struct dma_pte *parent, *pte = NULL;
409 int total = agaw_to_level(domain->agaw);
410 int offset;
411
412 parent = domain->pgd;
413 while (level <= total) {
414 offset = address_level_offset(addr, total);
415 pte = &parent[offset];
416 if (level == total)
417 return pte;
418
419 if (!dma_pte_present(*pte))
420 break;
421 parent = phys_to_virt(dma_pte_addr(*pte));
422 total--;
423 }
424 return NULL;
425 }
426
427 /* clear one page's page table */
428 static void dma_pte_clear_one(struct dmar_domain *domain, u64 addr)
429 {
430 struct dma_pte *pte = NULL;
431
432 /* get last level pte */
433 pte = dma_addr_level_pte(domain, addr, 1);
434
435 if (pte) {
436 dma_clear_pte(*pte);
437 __iommu_flush_cache(domain->iommu, pte, sizeof(*pte));
438 }
439 }
440
441 /* clear last level pte, a tlb flush should be followed */
442 static void dma_pte_clear_range(struct dmar_domain *domain, u64 start, u64 end)
443 {
444 int addr_width = agaw_to_width(domain->agaw);
445
446 start &= (((u64)1) << addr_width) - 1;
447 end &= (((u64)1) << addr_width) - 1;
448 /* in case it's partial page */
449 start = PAGE_ALIGN(start);
450 end &= PAGE_MASK;
451
452 /* we don't need lock here, nobody else touches the iova range */
453 while (start < end) {
454 dma_pte_clear_one(domain, start);
455 start += VTD_PAGE_SIZE;
456 }
457 }
458
459 /* free page table pages. last level pte should already be cleared */
460 static void dma_pte_free_pagetable(struct dmar_domain *domain,
461 u64 start, u64 end)
462 {
463 int addr_width = agaw_to_width(domain->agaw);
464 struct dma_pte *pte;
465 int total = agaw_to_level(domain->agaw);
466 int level;
467 u64 tmp;
468
469 start &= (((u64)1) << addr_width) - 1;
470 end &= (((u64)1) << addr_width) - 1;
471
472 /* we don't need lock here, nobody else touches the iova range */
473 level = 2;
474 while (level <= total) {
475 tmp = align_to_level(start, level);
476 if (tmp >= end || (tmp + level_size(level) > end))
477 return;
478
479 while (tmp < end) {
480 pte = dma_addr_level_pte(domain, tmp, level);
481 if (pte) {
482 free_pgtable_page(
483 phys_to_virt(dma_pte_addr(*pte)));
484 dma_clear_pte(*pte);
485 __iommu_flush_cache(domain->iommu,
486 pte, sizeof(*pte));
487 }
488 tmp += level_size(level);
489 }
490 level++;
491 }
492 /* free pgd */
493 if (start == 0 && end >= ((((u64)1) << addr_width) - 1)) {
494 free_pgtable_page(domain->pgd);
495 domain->pgd = NULL;
496 }
497 }
498
499 /* iommu handling */
500 static int iommu_alloc_root_entry(struct intel_iommu *iommu)
501 {
502 struct root_entry *root;
503 unsigned long flags;
504
505 root = (struct root_entry *)alloc_pgtable_page();
506 if (!root)
507 return -ENOMEM;
508
509 __iommu_flush_cache(iommu, root, ROOT_SIZE);
510
511 spin_lock_irqsave(&iommu->lock, flags);
512 iommu->root_entry = root;
513 spin_unlock_irqrestore(&iommu->lock, flags);
514
515 return 0;
516 }
517
518 static void iommu_set_root_entry(struct intel_iommu *iommu)
519 {
520 void *addr;
521 u32 cmd, sts;
522 unsigned long flag;
523
524 addr = iommu->root_entry;
525
526 spin_lock_irqsave(&iommu->register_lock, flag);
527 dmar_writeq(iommu->reg + DMAR_RTADDR_REG, virt_to_phys(addr));
528
529 cmd = iommu->gcmd | DMA_GCMD_SRTP;
530 writel(cmd, iommu->reg + DMAR_GCMD_REG);
531
532 /* Make sure hardware complete it */
533 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
534 readl, (sts & DMA_GSTS_RTPS), sts);
535
536 spin_unlock_irqrestore(&iommu->register_lock, flag);
537 }
538
539 static void iommu_flush_write_buffer(struct intel_iommu *iommu)
540 {
541 u32 val;
542 unsigned long flag;
543
544 if (!cap_rwbf(iommu->cap))
545 return;
546 val = iommu->gcmd | DMA_GCMD_WBF;
547
548 spin_lock_irqsave(&iommu->register_lock, flag);
549 writel(val, iommu->reg + DMAR_GCMD_REG);
550
551 /* Make sure hardware complete it */
552 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
553 readl, (!(val & DMA_GSTS_WBFS)), val);
554
555 spin_unlock_irqrestore(&iommu->register_lock, flag);
556 }
557
558 /* return value determine if we need a write buffer flush */
559 static int __iommu_flush_context(struct intel_iommu *iommu,
560 u16 did, u16 source_id, u8 function_mask, u64 type,
561 int non_present_entry_flush)
562 {
563 u64 val = 0;
564 unsigned long flag;
565
566 /*
567 * In the non-present entry flush case, if hardware doesn't cache
568 * non-present entry we do nothing and if hardware cache non-present
569 * entry, we flush entries of domain 0 (the domain id is used to cache
570 * any non-present entries)
571 */
572 if (non_present_entry_flush) {
573 if (!cap_caching_mode(iommu->cap))
574 return 1;
575 else
576 did = 0;
577 }
578
579 switch (type) {
580 case DMA_CCMD_GLOBAL_INVL:
581 val = DMA_CCMD_GLOBAL_INVL;
582 break;
583 case DMA_CCMD_DOMAIN_INVL:
584 val = DMA_CCMD_DOMAIN_INVL|DMA_CCMD_DID(did);
585 break;
586 case DMA_CCMD_DEVICE_INVL:
587 val = DMA_CCMD_DEVICE_INVL|DMA_CCMD_DID(did)
588 | DMA_CCMD_SID(source_id) | DMA_CCMD_FM(function_mask);
589 break;
590 default:
591 BUG();
592 }
593 val |= DMA_CCMD_ICC;
594
595 spin_lock_irqsave(&iommu->register_lock, flag);
596 dmar_writeq(iommu->reg + DMAR_CCMD_REG, val);
597
598 /* Make sure hardware complete it */
599 IOMMU_WAIT_OP(iommu, DMAR_CCMD_REG,
600 dmar_readq, (!(val & DMA_CCMD_ICC)), val);
601
602 spin_unlock_irqrestore(&iommu->register_lock, flag);
603
604 /* flush context entry will implicitly flush write buffer */
605 return 0;
606 }
607
608 /* return value determine if we need a write buffer flush */
609 static int __iommu_flush_iotlb(struct intel_iommu *iommu, u16 did,
610 u64 addr, unsigned int size_order, u64 type,
611 int non_present_entry_flush)
612 {
613 int tlb_offset = ecap_iotlb_offset(iommu->ecap);
614 u64 val = 0, val_iva = 0;
615 unsigned long flag;
616
617 /*
618 * In the non-present entry flush case, if hardware doesn't cache
619 * non-present entry we do nothing and if hardware cache non-present
620 * entry, we flush entries of domain 0 (the domain id is used to cache
621 * any non-present entries)
622 */
623 if (non_present_entry_flush) {
624 if (!cap_caching_mode(iommu->cap))
625 return 1;
626 else
627 did = 0;
628 }
629
630 switch (type) {
631 case DMA_TLB_GLOBAL_FLUSH:
632 /* global flush doesn't need set IVA_REG */
633 val = DMA_TLB_GLOBAL_FLUSH|DMA_TLB_IVT;
634 break;
635 case DMA_TLB_DSI_FLUSH:
636 val = DMA_TLB_DSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
637 break;
638 case DMA_TLB_PSI_FLUSH:
639 val = DMA_TLB_PSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
640 /* Note: always flush non-leaf currently */
641 val_iva = size_order | addr;
642 break;
643 default:
644 BUG();
645 }
646 /* Note: set drain read/write */
647 #if 0
648 /*
649 * This is probably to be super secure.. Looks like we can
650 * ignore it without any impact.
651 */
652 if (cap_read_drain(iommu->cap))
653 val |= DMA_TLB_READ_DRAIN;
654 #endif
655 if (cap_write_drain(iommu->cap))
656 val |= DMA_TLB_WRITE_DRAIN;
657
658 spin_lock_irqsave(&iommu->register_lock, flag);
659 /* Note: Only uses first TLB reg currently */
660 if (val_iva)
661 dmar_writeq(iommu->reg + tlb_offset, val_iva);
662 dmar_writeq(iommu->reg + tlb_offset + 8, val);
663
664 /* Make sure hardware complete it */
665 IOMMU_WAIT_OP(iommu, tlb_offset + 8,
666 dmar_readq, (!(val & DMA_TLB_IVT)), val);
667
668 spin_unlock_irqrestore(&iommu->register_lock, flag);
669
670 /* check IOTLB invalidation granularity */
671 if (DMA_TLB_IAIG(val) == 0)
672 printk(KERN_ERR"IOMMU: flush IOTLB failed\n");
673 if (DMA_TLB_IAIG(val) != DMA_TLB_IIRG(type))
674 pr_debug("IOMMU: tlb flush request %Lx, actual %Lx\n",
675 (unsigned long long)DMA_TLB_IIRG(type),
676 (unsigned long long)DMA_TLB_IAIG(val));
677 /* flush iotlb entry will implicitly flush write buffer */
678 return 0;
679 }
680
681 static int iommu_flush_iotlb_psi(struct intel_iommu *iommu, u16 did,
682 u64 addr, unsigned int pages, int non_present_entry_flush)
683 {
684 unsigned int mask;
685
686 BUG_ON(addr & (~VTD_PAGE_MASK));
687 BUG_ON(pages == 0);
688
689 /* Fallback to domain selective flush if no PSI support */
690 if (!cap_pgsel_inv(iommu->cap))
691 return iommu->flush.flush_iotlb(iommu, did, 0, 0,
692 DMA_TLB_DSI_FLUSH,
693 non_present_entry_flush);
694
695 /*
696 * PSI requires page size to be 2 ^ x, and the base address is naturally
697 * aligned to the size
698 */
699 mask = ilog2(__roundup_pow_of_two(pages));
700 /* Fallback to domain selective flush if size is too big */
701 if (mask > cap_max_amask_val(iommu->cap))
702 return iommu->flush.flush_iotlb(iommu, did, 0, 0,
703 DMA_TLB_DSI_FLUSH, non_present_entry_flush);
704
705 return iommu->flush.flush_iotlb(iommu, did, addr, mask,
706 DMA_TLB_PSI_FLUSH,
707 non_present_entry_flush);
708 }
709
710 static void iommu_disable_protect_mem_regions(struct intel_iommu *iommu)
711 {
712 u32 pmen;
713 unsigned long flags;
714
715 spin_lock_irqsave(&iommu->register_lock, flags);
716 pmen = readl(iommu->reg + DMAR_PMEN_REG);
717 pmen &= ~DMA_PMEN_EPM;
718 writel(pmen, iommu->reg + DMAR_PMEN_REG);
719
720 /* wait for the protected region status bit to clear */
721 IOMMU_WAIT_OP(iommu, DMAR_PMEN_REG,
722 readl, !(pmen & DMA_PMEN_PRS), pmen);
723
724 spin_unlock_irqrestore(&iommu->register_lock, flags);
725 }
726
727 static int iommu_enable_translation(struct intel_iommu *iommu)
728 {
729 u32 sts;
730 unsigned long flags;
731
732 spin_lock_irqsave(&iommu->register_lock, flags);
733 writel(iommu->gcmd|DMA_GCMD_TE, iommu->reg + DMAR_GCMD_REG);
734
735 /* Make sure hardware complete it */
736 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
737 readl, (sts & DMA_GSTS_TES), sts);
738
739 iommu->gcmd |= DMA_GCMD_TE;
740 spin_unlock_irqrestore(&iommu->register_lock, flags);
741 return 0;
742 }
743
744 static int iommu_disable_translation(struct intel_iommu *iommu)
745 {
746 u32 sts;
747 unsigned long flag;
748
749 spin_lock_irqsave(&iommu->register_lock, flag);
750 iommu->gcmd &= ~DMA_GCMD_TE;
751 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
752
753 /* Make sure hardware complete it */
754 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
755 readl, (!(sts & DMA_GSTS_TES)), sts);
756
757 spin_unlock_irqrestore(&iommu->register_lock, flag);
758 return 0;
759 }
760
761 /* iommu interrupt handling. Most stuff are MSI-like. */
762
763 static const char *fault_reason_strings[] =
764 {
765 "Software",
766 "Present bit in root entry is clear",
767 "Present bit in context entry is clear",
768 "Invalid context entry",
769 "Access beyond MGAW",
770 "PTE Write access is not set",
771 "PTE Read access is not set",
772 "Next page table ptr is invalid",
773 "Root table address invalid",
774 "Context table ptr is invalid",
775 "non-zero reserved fields in RTP",
776 "non-zero reserved fields in CTP",
777 "non-zero reserved fields in PTE",
778 };
779 #define MAX_FAULT_REASON_IDX (ARRAY_SIZE(fault_reason_strings) - 1)
780
781 const char *dmar_get_fault_reason(u8 fault_reason)
782 {
783 if (fault_reason > MAX_FAULT_REASON_IDX)
784 return "Unknown";
785 else
786 return fault_reason_strings[fault_reason];
787 }
788
789 void dmar_msi_unmask(unsigned int irq)
790 {
791 struct intel_iommu *iommu = get_irq_data(irq);
792 unsigned long flag;
793
794 /* unmask it */
795 spin_lock_irqsave(&iommu->register_lock, flag);
796 writel(0, iommu->reg + DMAR_FECTL_REG);
797 /* Read a reg to force flush the post write */
798 readl(iommu->reg + DMAR_FECTL_REG);
799 spin_unlock_irqrestore(&iommu->register_lock, flag);
800 }
801
802 void dmar_msi_mask(unsigned int irq)
803 {
804 unsigned long flag;
805 struct intel_iommu *iommu = get_irq_data(irq);
806
807 /* mask it */
808 spin_lock_irqsave(&iommu->register_lock, flag);
809 writel(DMA_FECTL_IM, iommu->reg + DMAR_FECTL_REG);
810 /* Read a reg to force flush the post write */
811 readl(iommu->reg + DMAR_FECTL_REG);
812 spin_unlock_irqrestore(&iommu->register_lock, flag);
813 }
814
815 void dmar_msi_write(int irq, struct msi_msg *msg)
816 {
817 struct intel_iommu *iommu = get_irq_data(irq);
818 unsigned long flag;
819
820 spin_lock_irqsave(&iommu->register_lock, flag);
821 writel(msg->data, iommu->reg + DMAR_FEDATA_REG);
822 writel(msg->address_lo, iommu->reg + DMAR_FEADDR_REG);
823 writel(msg->address_hi, iommu->reg + DMAR_FEUADDR_REG);
824 spin_unlock_irqrestore(&iommu->register_lock, flag);
825 }
826
827 void dmar_msi_read(int irq, struct msi_msg *msg)
828 {
829 struct intel_iommu *iommu = get_irq_data(irq);
830 unsigned long flag;
831
832 spin_lock_irqsave(&iommu->register_lock, flag);
833 msg->data = readl(iommu->reg + DMAR_FEDATA_REG);
834 msg->address_lo = readl(iommu->reg + DMAR_FEADDR_REG);
835 msg->address_hi = readl(iommu->reg + DMAR_FEUADDR_REG);
836 spin_unlock_irqrestore(&iommu->register_lock, flag);
837 }
838
839 static int iommu_page_fault_do_one(struct intel_iommu *iommu, int type,
840 u8 fault_reason, u16 source_id, unsigned long long addr)
841 {
842 const char *reason;
843
844 reason = dmar_get_fault_reason(fault_reason);
845
846 printk(KERN_ERR
847 "DMAR:[%s] Request device [%02x:%02x.%d] "
848 "fault addr %llx \n"
849 "DMAR:[fault reason %02d] %s\n",
850 (type ? "DMA Read" : "DMA Write"),
851 (source_id >> 8), PCI_SLOT(source_id & 0xFF),
852 PCI_FUNC(source_id & 0xFF), addr, fault_reason, reason);
853 return 0;
854 }
855
856 #define PRIMARY_FAULT_REG_LEN (16)
857 static irqreturn_t iommu_page_fault(int irq, void *dev_id)
858 {
859 struct intel_iommu *iommu = dev_id;
860 int reg, fault_index;
861 u32 fault_status;
862 unsigned long flag;
863
864 spin_lock_irqsave(&iommu->register_lock, flag);
865 fault_status = readl(iommu->reg + DMAR_FSTS_REG);
866
867 /* TBD: ignore advanced fault log currently */
868 if (!(fault_status & DMA_FSTS_PPF))
869 goto clear_overflow;
870
871 fault_index = dma_fsts_fault_record_index(fault_status);
872 reg = cap_fault_reg_offset(iommu->cap);
873 while (1) {
874 u8 fault_reason;
875 u16 source_id;
876 u64 guest_addr;
877 int type;
878 u32 data;
879
880 /* highest 32 bits */
881 data = readl(iommu->reg + reg +
882 fault_index * PRIMARY_FAULT_REG_LEN + 12);
883 if (!(data & DMA_FRCD_F))
884 break;
885
886 fault_reason = dma_frcd_fault_reason(data);
887 type = dma_frcd_type(data);
888
889 data = readl(iommu->reg + reg +
890 fault_index * PRIMARY_FAULT_REG_LEN + 8);
891 source_id = dma_frcd_source_id(data);
892
893 guest_addr = dmar_readq(iommu->reg + reg +
894 fault_index * PRIMARY_FAULT_REG_LEN);
895 guest_addr = dma_frcd_page_addr(guest_addr);
896 /* clear the fault */
897 writel(DMA_FRCD_F, iommu->reg + reg +
898 fault_index * PRIMARY_FAULT_REG_LEN + 12);
899
900 spin_unlock_irqrestore(&iommu->register_lock, flag);
901
902 iommu_page_fault_do_one(iommu, type, fault_reason,
903 source_id, guest_addr);
904
905 fault_index++;
906 if (fault_index > cap_num_fault_regs(iommu->cap))
907 fault_index = 0;
908 spin_lock_irqsave(&iommu->register_lock, flag);
909 }
910 clear_overflow:
911 /* clear primary fault overflow */
912 fault_status = readl(iommu->reg + DMAR_FSTS_REG);
913 if (fault_status & DMA_FSTS_PFO)
914 writel(DMA_FSTS_PFO, iommu->reg + DMAR_FSTS_REG);
915
916 spin_unlock_irqrestore(&iommu->register_lock, flag);
917 return IRQ_HANDLED;
918 }
919
920 int dmar_set_interrupt(struct intel_iommu *iommu)
921 {
922 int irq, ret;
923
924 irq = create_irq();
925 if (!irq) {
926 printk(KERN_ERR "IOMMU: no free vectors\n");
927 return -EINVAL;
928 }
929
930 set_irq_data(irq, iommu);
931 iommu->irq = irq;
932
933 ret = arch_setup_dmar_msi(irq);
934 if (ret) {
935 set_irq_data(irq, NULL);
936 iommu->irq = 0;
937 destroy_irq(irq);
938 return 0;
939 }
940
941 /* Force fault register is cleared */
942 iommu_page_fault(irq, iommu);
943
944 ret = request_irq(irq, iommu_page_fault, 0, iommu->name, iommu);
945 if (ret)
946 printk(KERN_ERR "IOMMU: can't request irq\n");
947 return ret;
948 }
949
950 static int iommu_init_domains(struct intel_iommu *iommu)
951 {
952 unsigned long ndomains;
953 unsigned long nlongs;
954
955 ndomains = cap_ndoms(iommu->cap);
956 pr_debug("Number of Domains supportd <%ld>\n", ndomains);
957 nlongs = BITS_TO_LONGS(ndomains);
958
959 /* TBD: there might be 64K domains,
960 * consider other allocation for future chip
961 */
962 iommu->domain_ids = kcalloc(nlongs, sizeof(unsigned long), GFP_KERNEL);
963 if (!iommu->domain_ids) {
964 printk(KERN_ERR "Allocating domain id array failed\n");
965 return -ENOMEM;
966 }
967 iommu->domains = kcalloc(ndomains, sizeof(struct dmar_domain *),
968 GFP_KERNEL);
969 if (!iommu->domains) {
970 printk(KERN_ERR "Allocating domain array failed\n");
971 kfree(iommu->domain_ids);
972 return -ENOMEM;
973 }
974
975 spin_lock_init(&iommu->lock);
976
977 /*
978 * if Caching mode is set, then invalid translations are tagged
979 * with domainid 0. Hence we need to pre-allocate it.
980 */
981 if (cap_caching_mode(iommu->cap))
982 set_bit(0, iommu->domain_ids);
983 return 0;
984 }
985
986
987 static void domain_exit(struct dmar_domain *domain);
988
989 void free_dmar_iommu(struct intel_iommu *iommu)
990 {
991 struct dmar_domain *domain;
992 int i;
993
994 i = find_first_bit(iommu->domain_ids, cap_ndoms(iommu->cap));
995 for (; i < cap_ndoms(iommu->cap); ) {
996 domain = iommu->domains[i];
997 clear_bit(i, iommu->domain_ids);
998 domain_exit(domain);
999 i = find_next_bit(iommu->domain_ids,
1000 cap_ndoms(iommu->cap), i+1);
1001 }
1002
1003 if (iommu->gcmd & DMA_GCMD_TE)
1004 iommu_disable_translation(iommu);
1005
1006 if (iommu->irq) {
1007 set_irq_data(iommu->irq, NULL);
1008 /* This will mask the irq */
1009 free_irq(iommu->irq, iommu);
1010 destroy_irq(iommu->irq);
1011 }
1012
1013 kfree(iommu->domains);
1014 kfree(iommu->domain_ids);
1015
1016 /* free context mapping */
1017 free_context_table(iommu);
1018 }
1019
1020 static struct dmar_domain * iommu_alloc_domain(struct intel_iommu *iommu)
1021 {
1022 unsigned long num;
1023 unsigned long ndomains;
1024 struct dmar_domain *domain;
1025 unsigned long flags;
1026
1027 domain = alloc_domain_mem();
1028 if (!domain)
1029 return NULL;
1030
1031 ndomains = cap_ndoms(iommu->cap);
1032
1033 spin_lock_irqsave(&iommu->lock, flags);
1034 num = find_first_zero_bit(iommu->domain_ids, ndomains);
1035 if (num >= ndomains) {
1036 spin_unlock_irqrestore(&iommu->lock, flags);
1037 free_domain_mem(domain);
1038 printk(KERN_ERR "IOMMU: no free domain ids\n");
1039 return NULL;
1040 }
1041
1042 set_bit(num, iommu->domain_ids);
1043 domain->id = num;
1044 domain->iommu = iommu;
1045 iommu->domains[num] = domain;
1046 spin_unlock_irqrestore(&iommu->lock, flags);
1047
1048 return domain;
1049 }
1050
1051 static void iommu_free_domain(struct dmar_domain *domain)
1052 {
1053 unsigned long flags;
1054
1055 spin_lock_irqsave(&domain->iommu->lock, flags);
1056 clear_bit(domain->id, domain->iommu->domain_ids);
1057 spin_unlock_irqrestore(&domain->iommu->lock, flags);
1058 }
1059
1060 static struct iova_domain reserved_iova_list;
1061 static struct lock_class_key reserved_alloc_key;
1062 static struct lock_class_key reserved_rbtree_key;
1063
1064 static void dmar_init_reserved_ranges(void)
1065 {
1066 struct pci_dev *pdev = NULL;
1067 struct iova *iova;
1068 int i;
1069 u64 addr, size;
1070
1071 init_iova_domain(&reserved_iova_list, DMA_32BIT_PFN);
1072
1073 lockdep_set_class(&reserved_iova_list.iova_alloc_lock,
1074 &reserved_alloc_key);
1075 lockdep_set_class(&reserved_iova_list.iova_rbtree_lock,
1076 &reserved_rbtree_key);
1077
1078 /* IOAPIC ranges shouldn't be accessed by DMA */
1079 iova = reserve_iova(&reserved_iova_list, IOVA_PFN(IOAPIC_RANGE_START),
1080 IOVA_PFN(IOAPIC_RANGE_END));
1081 if (!iova)
1082 printk(KERN_ERR "Reserve IOAPIC range failed\n");
1083
1084 /* Reserve all PCI MMIO to avoid peer-to-peer access */
1085 for_each_pci_dev(pdev) {
1086 struct resource *r;
1087
1088 for (i = 0; i < PCI_NUM_RESOURCES; i++) {
1089 r = &pdev->resource[i];
1090 if (!r->flags || !(r->flags & IORESOURCE_MEM))
1091 continue;
1092 addr = r->start;
1093 addr &= PAGE_MASK;
1094 size = r->end - addr;
1095 size = PAGE_ALIGN(size);
1096 iova = reserve_iova(&reserved_iova_list, IOVA_PFN(addr),
1097 IOVA_PFN(size + addr) - 1);
1098 if (!iova)
1099 printk(KERN_ERR "Reserve iova failed\n");
1100 }
1101 }
1102
1103 }
1104
1105 static void domain_reserve_special_ranges(struct dmar_domain *domain)
1106 {
1107 copy_reserved_iova(&reserved_iova_list, &domain->iovad);
1108 }
1109
1110 static inline int guestwidth_to_adjustwidth(int gaw)
1111 {
1112 int agaw;
1113 int r = (gaw - 12) % 9;
1114
1115 if (r == 0)
1116 agaw = gaw;
1117 else
1118 agaw = gaw + 9 - r;
1119 if (agaw > 64)
1120 agaw = 64;
1121 return agaw;
1122 }
1123
1124 static int domain_init(struct dmar_domain *domain, int guest_width)
1125 {
1126 struct intel_iommu *iommu;
1127 int adjust_width, agaw;
1128 unsigned long sagaw;
1129
1130 init_iova_domain(&domain->iovad, DMA_32BIT_PFN);
1131 spin_lock_init(&domain->mapping_lock);
1132
1133 domain_reserve_special_ranges(domain);
1134
1135 /* calculate AGAW */
1136 iommu = domain->iommu;
1137 if (guest_width > cap_mgaw(iommu->cap))
1138 guest_width = cap_mgaw(iommu->cap);
1139 domain->gaw = guest_width;
1140 adjust_width = guestwidth_to_adjustwidth(guest_width);
1141 agaw = width_to_agaw(adjust_width);
1142 sagaw = cap_sagaw(iommu->cap);
1143 if (!test_bit(agaw, &sagaw)) {
1144 /* hardware doesn't support it, choose a bigger one */
1145 pr_debug("IOMMU: hardware doesn't support agaw %d\n", agaw);
1146 agaw = find_next_bit(&sagaw, 5, agaw);
1147 if (agaw >= 5)
1148 return -ENODEV;
1149 }
1150 domain->agaw = agaw;
1151 INIT_LIST_HEAD(&domain->devices);
1152
1153 /* always allocate the top pgd */
1154 domain->pgd = (struct dma_pte *)alloc_pgtable_page();
1155 if (!domain->pgd)
1156 return -ENOMEM;
1157 __iommu_flush_cache(iommu, domain->pgd, PAGE_SIZE);
1158 return 0;
1159 }
1160
1161 static void domain_exit(struct dmar_domain *domain)
1162 {
1163 u64 end;
1164
1165 /* Domain 0 is reserved, so dont process it */
1166 if (!domain)
1167 return;
1168
1169 domain_remove_dev_info(domain);
1170 /* destroy iovas */
1171 put_iova_domain(&domain->iovad);
1172 end = DOMAIN_MAX_ADDR(domain->gaw);
1173 end = end & (~PAGE_MASK);
1174
1175 /* clear ptes */
1176 dma_pte_clear_range(domain, 0, end);
1177
1178 /* free page tables */
1179 dma_pte_free_pagetable(domain, 0, end);
1180
1181 iommu_free_domain(domain);
1182 free_domain_mem(domain);
1183 }
1184
1185 static int domain_context_mapping_one(struct dmar_domain *domain,
1186 u8 bus, u8 devfn)
1187 {
1188 struct context_entry *context;
1189 struct intel_iommu *iommu = domain->iommu;
1190 unsigned long flags;
1191
1192 pr_debug("Set context mapping for %02x:%02x.%d\n",
1193 bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
1194 BUG_ON(!domain->pgd);
1195 context = device_to_context_entry(iommu, bus, devfn);
1196 if (!context)
1197 return -ENOMEM;
1198 spin_lock_irqsave(&iommu->lock, flags);
1199 if (context_present(*context)) {
1200 spin_unlock_irqrestore(&iommu->lock, flags);
1201 return 0;
1202 }
1203
1204 context_set_domain_id(*context, domain->id);
1205 context_set_address_width(*context, domain->agaw);
1206 context_set_address_root(*context, virt_to_phys(domain->pgd));
1207 context_set_translation_type(*context, CONTEXT_TT_MULTI_LEVEL);
1208 context_set_fault_enable(*context);
1209 context_set_present(*context);
1210 __iommu_flush_cache(iommu, context, sizeof(*context));
1211
1212 /* it's a non-present to present mapping */
1213 if (iommu->flush.flush_context(iommu, domain->id,
1214 (((u16)bus) << 8) | devfn, DMA_CCMD_MASK_NOBIT,
1215 DMA_CCMD_DEVICE_INVL, 1))
1216 iommu_flush_write_buffer(iommu);
1217 else
1218 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_DSI_FLUSH, 0);
1219
1220 spin_unlock_irqrestore(&iommu->lock, flags);
1221 return 0;
1222 }
1223
1224 static int
1225 domain_context_mapping(struct dmar_domain *domain, struct pci_dev *pdev)
1226 {
1227 int ret;
1228 struct pci_dev *tmp, *parent;
1229
1230 ret = domain_context_mapping_one(domain, pdev->bus->number,
1231 pdev->devfn);
1232 if (ret)
1233 return ret;
1234
1235 /* dependent device mapping */
1236 tmp = pci_find_upstream_pcie_bridge(pdev);
1237 if (!tmp)
1238 return 0;
1239 /* Secondary interface's bus number and devfn 0 */
1240 parent = pdev->bus->self;
1241 while (parent != tmp) {
1242 ret = domain_context_mapping_one(domain, parent->bus->number,
1243 parent->devfn);
1244 if (ret)
1245 return ret;
1246 parent = parent->bus->self;
1247 }
1248 if (tmp->is_pcie) /* this is a PCIE-to-PCI bridge */
1249 return domain_context_mapping_one(domain,
1250 tmp->subordinate->number, 0);
1251 else /* this is a legacy PCI bridge */
1252 return domain_context_mapping_one(domain,
1253 tmp->bus->number, tmp->devfn);
1254 }
1255
1256 static int domain_context_mapped(struct dmar_domain *domain,
1257 struct pci_dev *pdev)
1258 {
1259 int ret;
1260 struct pci_dev *tmp, *parent;
1261
1262 ret = device_context_mapped(domain->iommu,
1263 pdev->bus->number, pdev->devfn);
1264 if (!ret)
1265 return ret;
1266 /* dependent device mapping */
1267 tmp = pci_find_upstream_pcie_bridge(pdev);
1268 if (!tmp)
1269 return ret;
1270 /* Secondary interface's bus number and devfn 0 */
1271 parent = pdev->bus->self;
1272 while (parent != tmp) {
1273 ret = device_context_mapped(domain->iommu, parent->bus->number,
1274 parent->devfn);
1275 if (!ret)
1276 return ret;
1277 parent = parent->bus->self;
1278 }
1279 if (tmp->is_pcie)
1280 return device_context_mapped(domain->iommu,
1281 tmp->subordinate->number, 0);
1282 else
1283 return device_context_mapped(domain->iommu,
1284 tmp->bus->number, tmp->devfn);
1285 }
1286
1287 static int
1288 domain_page_mapping(struct dmar_domain *domain, dma_addr_t iova,
1289 u64 hpa, size_t size, int prot)
1290 {
1291 u64 start_pfn, end_pfn;
1292 struct dma_pte *pte;
1293 int index;
1294 int addr_width = agaw_to_width(domain->agaw);
1295
1296 hpa &= (((u64)1) << addr_width) - 1;
1297
1298 if ((prot & (DMA_PTE_READ|DMA_PTE_WRITE)) == 0)
1299 return -EINVAL;
1300 iova &= PAGE_MASK;
1301 start_pfn = ((u64)hpa) >> VTD_PAGE_SHIFT;
1302 end_pfn = (VTD_PAGE_ALIGN(((u64)hpa) + size)) >> VTD_PAGE_SHIFT;
1303 index = 0;
1304 while (start_pfn < end_pfn) {
1305 pte = addr_to_dma_pte(domain, iova + VTD_PAGE_SIZE * index);
1306 if (!pte)
1307 return -ENOMEM;
1308 /* We don't need lock here, nobody else
1309 * touches the iova range
1310 */
1311 BUG_ON(dma_pte_addr(*pte));
1312 dma_set_pte_addr(*pte, start_pfn << VTD_PAGE_SHIFT);
1313 dma_set_pte_prot(*pte, prot);
1314 __iommu_flush_cache(domain->iommu, pte, sizeof(*pte));
1315 start_pfn++;
1316 index++;
1317 }
1318 return 0;
1319 }
1320
1321 static void detach_domain_for_dev(struct dmar_domain *domain, u8 bus, u8 devfn)
1322 {
1323 clear_context_table(domain->iommu, bus, devfn);
1324 domain->iommu->flush.flush_context(domain->iommu, 0, 0, 0,
1325 DMA_CCMD_GLOBAL_INVL, 0);
1326 domain->iommu->flush.flush_iotlb(domain->iommu, 0, 0, 0,
1327 DMA_TLB_GLOBAL_FLUSH, 0);
1328 }
1329
1330 static void domain_remove_dev_info(struct dmar_domain *domain)
1331 {
1332 struct device_domain_info *info;
1333 unsigned long flags;
1334
1335 spin_lock_irqsave(&device_domain_lock, flags);
1336 while (!list_empty(&domain->devices)) {
1337 info = list_entry(domain->devices.next,
1338 struct device_domain_info, link);
1339 list_del(&info->link);
1340 list_del(&info->global);
1341 if (info->dev)
1342 info->dev->dev.archdata.iommu = NULL;
1343 spin_unlock_irqrestore(&device_domain_lock, flags);
1344
1345 detach_domain_for_dev(info->domain, info->bus, info->devfn);
1346 free_devinfo_mem(info);
1347
1348 spin_lock_irqsave(&device_domain_lock, flags);
1349 }
1350 spin_unlock_irqrestore(&device_domain_lock, flags);
1351 }
1352
1353 /*
1354 * find_domain
1355 * Note: we use struct pci_dev->dev.archdata.iommu stores the info
1356 */
1357 static struct dmar_domain *
1358 find_domain(struct pci_dev *pdev)
1359 {
1360 struct device_domain_info *info;
1361
1362 /* No lock here, assumes no domain exit in normal case */
1363 info = pdev->dev.archdata.iommu;
1364 if (info)
1365 return info->domain;
1366 return NULL;
1367 }
1368
1369 /* domain is initialized */
1370 static struct dmar_domain *get_domain_for_dev(struct pci_dev *pdev, int gaw)
1371 {
1372 struct dmar_domain *domain, *found = NULL;
1373 struct intel_iommu *iommu;
1374 struct dmar_drhd_unit *drhd;
1375 struct device_domain_info *info, *tmp;
1376 struct pci_dev *dev_tmp;
1377 unsigned long flags;
1378 int bus = 0, devfn = 0;
1379
1380 domain = find_domain(pdev);
1381 if (domain)
1382 return domain;
1383
1384 dev_tmp = pci_find_upstream_pcie_bridge(pdev);
1385 if (dev_tmp) {
1386 if (dev_tmp->is_pcie) {
1387 bus = dev_tmp->subordinate->number;
1388 devfn = 0;
1389 } else {
1390 bus = dev_tmp->bus->number;
1391 devfn = dev_tmp->devfn;
1392 }
1393 spin_lock_irqsave(&device_domain_lock, flags);
1394 list_for_each_entry(info, &device_domain_list, global) {
1395 if (info->bus == bus && info->devfn == devfn) {
1396 found = info->domain;
1397 break;
1398 }
1399 }
1400 spin_unlock_irqrestore(&device_domain_lock, flags);
1401 /* pcie-pci bridge already has a domain, uses it */
1402 if (found) {
1403 domain = found;
1404 goto found_domain;
1405 }
1406 }
1407
1408 /* Allocate new domain for the device */
1409 drhd = dmar_find_matched_drhd_unit(pdev);
1410 if (!drhd) {
1411 printk(KERN_ERR "IOMMU: can't find DMAR for device %s\n",
1412 pci_name(pdev));
1413 return NULL;
1414 }
1415 iommu = drhd->iommu;
1416
1417 domain = iommu_alloc_domain(iommu);
1418 if (!domain)
1419 goto error;
1420
1421 if (domain_init(domain, gaw)) {
1422 domain_exit(domain);
1423 goto error;
1424 }
1425
1426 /* register pcie-to-pci device */
1427 if (dev_tmp) {
1428 info = alloc_devinfo_mem();
1429 if (!info) {
1430 domain_exit(domain);
1431 goto error;
1432 }
1433 info->bus = bus;
1434 info->devfn = devfn;
1435 info->dev = NULL;
1436 info->domain = domain;
1437 /* This domain is shared by devices under p2p bridge */
1438 domain->flags |= DOMAIN_FLAG_MULTIPLE_DEVICES;
1439
1440 /* pcie-to-pci bridge already has a domain, uses it */
1441 found = NULL;
1442 spin_lock_irqsave(&device_domain_lock, flags);
1443 list_for_each_entry(tmp, &device_domain_list, global) {
1444 if (tmp->bus == bus && tmp->devfn == devfn) {
1445 found = tmp->domain;
1446 break;
1447 }
1448 }
1449 if (found) {
1450 free_devinfo_mem(info);
1451 domain_exit(domain);
1452 domain = found;
1453 } else {
1454 list_add(&info->link, &domain->devices);
1455 list_add(&info->global, &device_domain_list);
1456 }
1457 spin_unlock_irqrestore(&device_domain_lock, flags);
1458 }
1459
1460 found_domain:
1461 info = alloc_devinfo_mem();
1462 if (!info)
1463 goto error;
1464 info->bus = pdev->bus->number;
1465 info->devfn = pdev->devfn;
1466 info->dev = pdev;
1467 info->domain = domain;
1468 spin_lock_irqsave(&device_domain_lock, flags);
1469 /* somebody is fast */
1470 found = find_domain(pdev);
1471 if (found != NULL) {
1472 spin_unlock_irqrestore(&device_domain_lock, flags);
1473 if (found != domain) {
1474 domain_exit(domain);
1475 domain = found;
1476 }
1477 free_devinfo_mem(info);
1478 return domain;
1479 }
1480 list_add(&info->link, &domain->devices);
1481 list_add(&info->global, &device_domain_list);
1482 pdev->dev.archdata.iommu = info;
1483 spin_unlock_irqrestore(&device_domain_lock, flags);
1484 return domain;
1485 error:
1486 /* recheck it here, maybe others set it */
1487 return find_domain(pdev);
1488 }
1489
1490 static int iommu_prepare_identity_map(struct pci_dev *pdev,
1491 unsigned long long start,
1492 unsigned long long end)
1493 {
1494 struct dmar_domain *domain;
1495 unsigned long size;
1496 unsigned long long base;
1497 int ret;
1498
1499 printk(KERN_INFO
1500 "IOMMU: Setting identity map for device %s [0x%Lx - 0x%Lx]\n",
1501 pci_name(pdev), start, end);
1502 /* page table init */
1503 domain = get_domain_for_dev(pdev, DEFAULT_DOMAIN_ADDRESS_WIDTH);
1504 if (!domain)
1505 return -ENOMEM;
1506
1507 /* The address might not be aligned */
1508 base = start & PAGE_MASK;
1509 size = end - base;
1510 size = PAGE_ALIGN(size);
1511 if (!reserve_iova(&domain->iovad, IOVA_PFN(base),
1512 IOVA_PFN(base + size) - 1)) {
1513 printk(KERN_ERR "IOMMU: reserve iova failed\n");
1514 ret = -ENOMEM;
1515 goto error;
1516 }
1517
1518 pr_debug("Mapping reserved region %lx@%llx for %s\n",
1519 size, base, pci_name(pdev));
1520 /*
1521 * RMRR range might have overlap with physical memory range,
1522 * clear it first
1523 */
1524 dma_pte_clear_range(domain, base, base + size);
1525
1526 ret = domain_page_mapping(domain, base, base, size,
1527 DMA_PTE_READ|DMA_PTE_WRITE);
1528 if (ret)
1529 goto error;
1530
1531 /* context entry init */
1532 ret = domain_context_mapping(domain, pdev);
1533 if (!ret)
1534 return 0;
1535 error:
1536 domain_exit(domain);
1537 return ret;
1538
1539 }
1540
1541 static inline int iommu_prepare_rmrr_dev(struct dmar_rmrr_unit *rmrr,
1542 struct pci_dev *pdev)
1543 {
1544 if (pdev->dev.archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO)
1545 return 0;
1546 return iommu_prepare_identity_map(pdev, rmrr->base_address,
1547 rmrr->end_address + 1);
1548 }
1549
1550 #ifdef CONFIG_DMAR_GFX_WA
1551 struct iommu_prepare_data {
1552 struct pci_dev *pdev;
1553 int ret;
1554 };
1555
1556 static int __init iommu_prepare_work_fn(unsigned long start_pfn,
1557 unsigned long end_pfn, void *datax)
1558 {
1559 struct iommu_prepare_data *data;
1560
1561 data = (struct iommu_prepare_data *)datax;
1562
1563 data->ret = iommu_prepare_identity_map(data->pdev,
1564 start_pfn<<PAGE_SHIFT, end_pfn<<PAGE_SHIFT);
1565 return data->ret;
1566
1567 }
1568
1569 static int __init iommu_prepare_with_active_regions(struct pci_dev *pdev)
1570 {
1571 int nid;
1572 struct iommu_prepare_data data;
1573
1574 data.pdev = pdev;
1575 data.ret = 0;
1576
1577 for_each_online_node(nid) {
1578 work_with_active_regions(nid, iommu_prepare_work_fn, &data);
1579 if (data.ret)
1580 return data.ret;
1581 }
1582 return data.ret;
1583 }
1584
1585 static void __init iommu_prepare_gfx_mapping(void)
1586 {
1587 struct pci_dev *pdev = NULL;
1588 int ret;
1589
1590 for_each_pci_dev(pdev) {
1591 if (pdev->dev.archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO ||
1592 !IS_GFX_DEVICE(pdev))
1593 continue;
1594 printk(KERN_INFO "IOMMU: gfx device %s 1-1 mapping\n",
1595 pci_name(pdev));
1596 ret = iommu_prepare_with_active_regions(pdev);
1597 if (ret)
1598 printk(KERN_ERR "IOMMU: mapping reserved region failed\n");
1599 }
1600 }
1601 #endif
1602
1603 #ifdef CONFIG_DMAR_FLOPPY_WA
1604 static inline void iommu_prepare_isa(void)
1605 {
1606 struct pci_dev *pdev;
1607 int ret;
1608
1609 pdev = pci_get_class(PCI_CLASS_BRIDGE_ISA << 8, NULL);
1610 if (!pdev)
1611 return;
1612
1613 printk(KERN_INFO "IOMMU: Prepare 0-16M unity mapping for LPC\n");
1614 ret = iommu_prepare_identity_map(pdev, 0, 16*1024*1024);
1615
1616 if (ret)
1617 printk("IOMMU: Failed to create 0-64M identity map, "
1618 "floppy might not work\n");
1619
1620 }
1621 #else
1622 static inline void iommu_prepare_isa(void)
1623 {
1624 return;
1625 }
1626 #endif /* !CONFIG_DMAR_FLPY_WA */
1627
1628 static int __init init_dmars(void)
1629 {
1630 struct dmar_drhd_unit *drhd;
1631 struct dmar_rmrr_unit *rmrr;
1632 struct pci_dev *pdev;
1633 struct intel_iommu *iommu;
1634 int i, ret, unit = 0;
1635
1636 /*
1637 * for each drhd
1638 * allocate root
1639 * initialize and program root entry to not present
1640 * endfor
1641 */
1642 for_each_drhd_unit(drhd) {
1643 g_num_of_iommus++;
1644 /*
1645 * lock not needed as this is only incremented in the single
1646 * threaded kernel __init code path all other access are read
1647 * only
1648 */
1649 }
1650
1651 deferred_flush = kzalloc(g_num_of_iommus *
1652 sizeof(struct deferred_flush_tables), GFP_KERNEL);
1653 if (!deferred_flush) {
1654 ret = -ENOMEM;
1655 goto error;
1656 }
1657
1658 for_each_drhd_unit(drhd) {
1659 if (drhd->ignored)
1660 continue;
1661
1662 iommu = drhd->iommu;
1663
1664 ret = iommu_init_domains(iommu);
1665 if (ret)
1666 goto error;
1667
1668 /*
1669 * TBD:
1670 * we could share the same root & context tables
1671 * amoung all IOMMU's. Need to Split it later.
1672 */
1673 ret = iommu_alloc_root_entry(iommu);
1674 if (ret) {
1675 printk(KERN_ERR "IOMMU: allocate root entry failed\n");
1676 goto error;
1677 }
1678 }
1679
1680 for_each_drhd_unit(drhd) {
1681 if (drhd->ignored)
1682 continue;
1683
1684 iommu = drhd->iommu;
1685 if (dmar_enable_qi(iommu)) {
1686 /*
1687 * Queued Invalidate not enabled, use Register Based
1688 * Invalidate
1689 */
1690 iommu->flush.flush_context = __iommu_flush_context;
1691 iommu->flush.flush_iotlb = __iommu_flush_iotlb;
1692 printk(KERN_INFO "IOMMU 0x%Lx: using Register based "
1693 "invalidation\n",
1694 (unsigned long long)drhd->reg_base_addr);
1695 } else {
1696 iommu->flush.flush_context = qi_flush_context;
1697 iommu->flush.flush_iotlb = qi_flush_iotlb;
1698 printk(KERN_INFO "IOMMU 0x%Lx: using Queued "
1699 "invalidation\n",
1700 (unsigned long long)drhd->reg_base_addr);
1701 }
1702 }
1703
1704 /*
1705 * For each rmrr
1706 * for each dev attached to rmrr
1707 * do
1708 * locate drhd for dev, alloc domain for dev
1709 * allocate free domain
1710 * allocate page table entries for rmrr
1711 * if context not allocated for bus
1712 * allocate and init context
1713 * set present in root table for this bus
1714 * init context with domain, translation etc
1715 * endfor
1716 * endfor
1717 */
1718 for_each_rmrr_units(rmrr) {
1719 for (i = 0; i < rmrr->devices_cnt; i++) {
1720 pdev = rmrr->devices[i];
1721 /* some BIOS lists non-exist devices in DMAR table */
1722 if (!pdev)
1723 continue;
1724 ret = iommu_prepare_rmrr_dev(rmrr, pdev);
1725 if (ret)
1726 printk(KERN_ERR
1727 "IOMMU: mapping reserved region failed\n");
1728 }
1729 }
1730
1731 iommu_prepare_gfx_mapping();
1732
1733 iommu_prepare_isa();
1734
1735 /*
1736 * for each drhd
1737 * enable fault log
1738 * global invalidate context cache
1739 * global invalidate iotlb
1740 * enable translation
1741 */
1742 for_each_drhd_unit(drhd) {
1743 if (drhd->ignored)
1744 continue;
1745 iommu = drhd->iommu;
1746 sprintf (iommu->name, "dmar%d", unit++);
1747
1748 iommu_flush_write_buffer(iommu);
1749
1750 ret = dmar_set_interrupt(iommu);
1751 if (ret)
1752 goto error;
1753
1754 iommu_set_root_entry(iommu);
1755
1756 iommu->flush.flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL,
1757 0);
1758 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH,
1759 0);
1760 iommu_disable_protect_mem_regions(iommu);
1761
1762 ret = iommu_enable_translation(iommu);
1763 if (ret)
1764 goto error;
1765 }
1766
1767 return 0;
1768 error:
1769 for_each_drhd_unit(drhd) {
1770 if (drhd->ignored)
1771 continue;
1772 iommu = drhd->iommu;
1773 free_iommu(iommu);
1774 }
1775 return ret;
1776 }
1777
1778 static inline u64 aligned_size(u64 host_addr, size_t size)
1779 {
1780 u64 addr;
1781 addr = (host_addr & (~PAGE_MASK)) + size;
1782 return PAGE_ALIGN(addr);
1783 }
1784
1785 struct iova *
1786 iommu_alloc_iova(struct dmar_domain *domain, size_t size, u64 end)
1787 {
1788 struct iova *piova;
1789
1790 /* Make sure it's in range */
1791 end = min_t(u64, DOMAIN_MAX_ADDR(domain->gaw), end);
1792 if (!size || (IOVA_START_ADDR + size > end))
1793 return NULL;
1794
1795 piova = alloc_iova(&domain->iovad,
1796 size >> PAGE_SHIFT, IOVA_PFN(end), 1);
1797 return piova;
1798 }
1799
1800 static struct iova *
1801 __intel_alloc_iova(struct device *dev, struct dmar_domain *domain,
1802 size_t size, u64 dma_mask)
1803 {
1804 struct pci_dev *pdev = to_pci_dev(dev);
1805 struct iova *iova = NULL;
1806
1807 if (dma_mask <= DMA_32BIT_MASK || dmar_forcedac)
1808 iova = iommu_alloc_iova(domain, size, dma_mask);
1809 else {
1810 /*
1811 * First try to allocate an io virtual address in
1812 * DMA_32BIT_MASK and if that fails then try allocating
1813 * from higher range
1814 */
1815 iova = iommu_alloc_iova(domain, size, DMA_32BIT_MASK);
1816 if (!iova)
1817 iova = iommu_alloc_iova(domain, size, dma_mask);
1818 }
1819
1820 if (!iova) {
1821 printk(KERN_ERR"Allocating iova for %s failed", pci_name(pdev));
1822 return NULL;
1823 }
1824
1825 return iova;
1826 }
1827
1828 static struct dmar_domain *
1829 get_valid_domain_for_dev(struct pci_dev *pdev)
1830 {
1831 struct dmar_domain *domain;
1832 int ret;
1833
1834 domain = get_domain_for_dev(pdev,
1835 DEFAULT_DOMAIN_ADDRESS_WIDTH);
1836 if (!domain) {
1837 printk(KERN_ERR
1838 "Allocating domain for %s failed", pci_name(pdev));
1839 return NULL;
1840 }
1841
1842 /* make sure context mapping is ok */
1843 if (unlikely(!domain_context_mapped(domain, pdev))) {
1844 ret = domain_context_mapping(domain, pdev);
1845 if (ret) {
1846 printk(KERN_ERR
1847 "Domain context map for %s failed",
1848 pci_name(pdev));
1849 return NULL;
1850 }
1851 }
1852
1853 return domain;
1854 }
1855
1856 static dma_addr_t __intel_map_single(struct device *hwdev, phys_addr_t paddr,
1857 size_t size, int dir, u64 dma_mask)
1858 {
1859 struct pci_dev *pdev = to_pci_dev(hwdev);
1860 struct dmar_domain *domain;
1861 phys_addr_t start_paddr;
1862 struct iova *iova;
1863 int prot = 0;
1864 int ret;
1865
1866 BUG_ON(dir == DMA_NONE);
1867 if (pdev->dev.archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO)
1868 return paddr;
1869
1870 domain = get_valid_domain_for_dev(pdev);
1871 if (!domain)
1872 return 0;
1873
1874 size = aligned_size((u64)paddr, size);
1875
1876 iova = __intel_alloc_iova(hwdev, domain, size, pdev->dma_mask);
1877 if (!iova)
1878 goto error;
1879
1880 start_paddr = (phys_addr_t)iova->pfn_lo << PAGE_SHIFT;
1881
1882 /*
1883 * Check if DMAR supports zero-length reads on write only
1884 * mappings..
1885 */
1886 if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
1887 !cap_zlr(domain->iommu->cap))
1888 prot |= DMA_PTE_READ;
1889 if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
1890 prot |= DMA_PTE_WRITE;
1891 /*
1892 * paddr - (paddr + size) might be partial page, we should map the whole
1893 * page. Note: if two part of one page are separately mapped, we
1894 * might have two guest_addr mapping to the same host paddr, but this
1895 * is not a big problem
1896 */
1897 ret = domain_page_mapping(domain, start_paddr,
1898 ((u64)paddr) & PAGE_MASK, size, prot);
1899 if (ret)
1900 goto error;
1901
1902 /* it's a non-present to present mapping */
1903 ret = iommu_flush_iotlb_psi(domain->iommu, domain->id,
1904 start_paddr, size >> VTD_PAGE_SHIFT, 1);
1905 if (ret)
1906 iommu_flush_write_buffer(domain->iommu);
1907
1908 return start_paddr + ((u64)paddr & (~PAGE_MASK));
1909
1910 error:
1911 if (iova)
1912 __free_iova(&domain->iovad, iova);
1913 printk(KERN_ERR"Device %s request: %lx@%llx dir %d --- failed\n",
1914 pci_name(pdev), size, (unsigned long long)paddr, dir);
1915 return 0;
1916 }
1917
1918 dma_addr_t intel_map_single(struct device *hwdev, phys_addr_t paddr,
1919 size_t size, int dir)
1920 {
1921 return __intel_map_single(hwdev, paddr, size, dir,
1922 to_pci_dev(hwdev)->dma_mask);
1923 }
1924
1925 static void flush_unmaps(void)
1926 {
1927 int i, j;
1928
1929 timer_on = 0;
1930
1931 /* just flush them all */
1932 for (i = 0; i < g_num_of_iommus; i++) {
1933 if (deferred_flush[i].next) {
1934 struct intel_iommu *iommu =
1935 deferred_flush[i].domain[0]->iommu;
1936
1937 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
1938 DMA_TLB_GLOBAL_FLUSH, 0);
1939 for (j = 0; j < deferred_flush[i].next; j++) {
1940 __free_iova(&deferred_flush[i].domain[j]->iovad,
1941 deferred_flush[i].iova[j]);
1942 }
1943 deferred_flush[i].next = 0;
1944 }
1945 }
1946
1947 list_size = 0;
1948 }
1949
1950 static void flush_unmaps_timeout(unsigned long data)
1951 {
1952 unsigned long flags;
1953
1954 spin_lock_irqsave(&async_umap_flush_lock, flags);
1955 flush_unmaps();
1956 spin_unlock_irqrestore(&async_umap_flush_lock, flags);
1957 }
1958
1959 static void add_unmap(struct dmar_domain *dom, struct iova *iova)
1960 {
1961 unsigned long flags;
1962 int next, iommu_id;
1963
1964 spin_lock_irqsave(&async_umap_flush_lock, flags);
1965 if (list_size == HIGH_WATER_MARK)
1966 flush_unmaps();
1967
1968 iommu_id = dom->iommu->seq_id;
1969
1970 next = deferred_flush[iommu_id].next;
1971 deferred_flush[iommu_id].domain[next] = dom;
1972 deferred_flush[iommu_id].iova[next] = iova;
1973 deferred_flush[iommu_id].next++;
1974
1975 if (!timer_on) {
1976 mod_timer(&unmap_timer, jiffies + msecs_to_jiffies(10));
1977 timer_on = 1;
1978 }
1979 list_size++;
1980 spin_unlock_irqrestore(&async_umap_flush_lock, flags);
1981 }
1982
1983 void intel_unmap_single(struct device *dev, dma_addr_t dev_addr, size_t size,
1984 int dir)
1985 {
1986 struct pci_dev *pdev = to_pci_dev(dev);
1987 struct dmar_domain *domain;
1988 unsigned long start_addr;
1989 struct iova *iova;
1990
1991 if (pdev->dev.archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO)
1992 return;
1993 domain = find_domain(pdev);
1994 BUG_ON(!domain);
1995
1996 iova = find_iova(&domain->iovad, IOVA_PFN(dev_addr));
1997 if (!iova)
1998 return;
1999
2000 start_addr = iova->pfn_lo << PAGE_SHIFT;
2001 size = aligned_size((u64)dev_addr, size);
2002
2003 pr_debug("Device %s unmapping: %lx@%llx\n",
2004 pci_name(pdev), size, (unsigned long long)start_addr);
2005
2006 /* clear the whole page */
2007 dma_pte_clear_range(domain, start_addr, start_addr + size);
2008 /* free page tables */
2009 dma_pte_free_pagetable(domain, start_addr, start_addr + size);
2010 if (intel_iommu_strict) {
2011 if (iommu_flush_iotlb_psi(domain->iommu,
2012 domain->id, start_addr, size >> VTD_PAGE_SHIFT, 0))
2013 iommu_flush_write_buffer(domain->iommu);
2014 /* free iova */
2015 __free_iova(&domain->iovad, iova);
2016 } else {
2017 add_unmap(domain, iova);
2018 /*
2019 * queue up the release of the unmap to save the 1/6th of the
2020 * cpu used up by the iotlb flush operation...
2021 */
2022 }
2023 }
2024
2025 void *intel_alloc_coherent(struct device *hwdev, size_t size,
2026 dma_addr_t *dma_handle, gfp_t flags)
2027 {
2028 void *vaddr;
2029 int order;
2030
2031 size = PAGE_ALIGN(size);
2032 order = get_order(size);
2033 flags &= ~(GFP_DMA | GFP_DMA32);
2034
2035 vaddr = (void *)__get_free_pages(flags, order);
2036 if (!vaddr)
2037 return NULL;
2038 memset(vaddr, 0, size);
2039
2040 *dma_handle = __intel_map_single(hwdev, virt_to_bus(vaddr), size,
2041 DMA_BIDIRECTIONAL,
2042 hwdev->coherent_dma_mask);
2043 if (*dma_handle)
2044 return vaddr;
2045 free_pages((unsigned long)vaddr, order);
2046 return NULL;
2047 }
2048
2049 void intel_free_coherent(struct device *hwdev, size_t size, void *vaddr,
2050 dma_addr_t dma_handle)
2051 {
2052 int order;
2053
2054 size = PAGE_ALIGN(size);
2055 order = get_order(size);
2056
2057 intel_unmap_single(hwdev, dma_handle, size, DMA_BIDIRECTIONAL);
2058 free_pages((unsigned long)vaddr, order);
2059 }
2060
2061 #define SG_ENT_VIRT_ADDRESS(sg) (sg_virt((sg)))
2062
2063 void intel_unmap_sg(struct device *hwdev, struct scatterlist *sglist,
2064 int nelems, int dir)
2065 {
2066 int i;
2067 struct pci_dev *pdev = to_pci_dev(hwdev);
2068 struct dmar_domain *domain;
2069 unsigned long start_addr;
2070 struct iova *iova;
2071 size_t size = 0;
2072 void *addr;
2073 struct scatterlist *sg;
2074
2075 if (pdev->dev.archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO)
2076 return;
2077
2078 domain = find_domain(pdev);
2079
2080 iova = find_iova(&domain->iovad, IOVA_PFN(sglist[0].dma_address));
2081 if (!iova)
2082 return;
2083 for_each_sg(sglist, sg, nelems, i) {
2084 addr = SG_ENT_VIRT_ADDRESS(sg);
2085 size += aligned_size((u64)addr, sg->length);
2086 }
2087
2088 start_addr = iova->pfn_lo << PAGE_SHIFT;
2089
2090 /* clear the whole page */
2091 dma_pte_clear_range(domain, start_addr, start_addr + size);
2092 /* free page tables */
2093 dma_pte_free_pagetable(domain, start_addr, start_addr + size);
2094
2095 if (iommu_flush_iotlb_psi(domain->iommu, domain->id, start_addr,
2096 size >> VTD_PAGE_SHIFT, 0))
2097 iommu_flush_write_buffer(domain->iommu);
2098
2099 /* free iova */
2100 __free_iova(&domain->iovad, iova);
2101 }
2102
2103 static int intel_nontranslate_map_sg(struct device *hddev,
2104 struct scatterlist *sglist, int nelems, int dir)
2105 {
2106 int i;
2107 struct scatterlist *sg;
2108
2109 for_each_sg(sglist, sg, nelems, i) {
2110 BUG_ON(!sg_page(sg));
2111 sg->dma_address = virt_to_bus(SG_ENT_VIRT_ADDRESS(sg));
2112 sg->dma_length = sg->length;
2113 }
2114 return nelems;
2115 }
2116
2117 int intel_map_sg(struct device *hwdev, struct scatterlist *sglist, int nelems,
2118 int dir)
2119 {
2120 void *addr;
2121 int i;
2122 struct pci_dev *pdev = to_pci_dev(hwdev);
2123 struct dmar_domain *domain;
2124 size_t size = 0;
2125 int prot = 0;
2126 size_t offset = 0;
2127 struct iova *iova = NULL;
2128 int ret;
2129 struct scatterlist *sg;
2130 unsigned long start_addr;
2131
2132 BUG_ON(dir == DMA_NONE);
2133 if (pdev->dev.archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO)
2134 return intel_nontranslate_map_sg(hwdev, sglist, nelems, dir);
2135
2136 domain = get_valid_domain_for_dev(pdev);
2137 if (!domain)
2138 return 0;
2139
2140 for_each_sg(sglist, sg, nelems, i) {
2141 addr = SG_ENT_VIRT_ADDRESS(sg);
2142 addr = (void *)virt_to_phys(addr);
2143 size += aligned_size((u64)addr, sg->length);
2144 }
2145
2146 iova = __intel_alloc_iova(hwdev, domain, size, pdev->dma_mask);
2147 if (!iova) {
2148 sglist->dma_length = 0;
2149 return 0;
2150 }
2151
2152 /*
2153 * Check if DMAR supports zero-length reads on write only
2154 * mappings..
2155 */
2156 if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
2157 !cap_zlr(domain->iommu->cap))
2158 prot |= DMA_PTE_READ;
2159 if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
2160 prot |= DMA_PTE_WRITE;
2161
2162 start_addr = iova->pfn_lo << PAGE_SHIFT;
2163 offset = 0;
2164 for_each_sg(sglist, sg, nelems, i) {
2165 addr = SG_ENT_VIRT_ADDRESS(sg);
2166 addr = (void *)virt_to_phys(addr);
2167 size = aligned_size((u64)addr, sg->length);
2168 ret = domain_page_mapping(domain, start_addr + offset,
2169 ((u64)addr) & PAGE_MASK,
2170 size, prot);
2171 if (ret) {
2172 /* clear the page */
2173 dma_pte_clear_range(domain, start_addr,
2174 start_addr + offset);
2175 /* free page tables */
2176 dma_pte_free_pagetable(domain, start_addr,
2177 start_addr + offset);
2178 /* free iova */
2179 __free_iova(&domain->iovad, iova);
2180 return 0;
2181 }
2182 sg->dma_address = start_addr + offset +
2183 ((u64)addr & (~PAGE_MASK));
2184 sg->dma_length = sg->length;
2185 offset += size;
2186 }
2187
2188 /* it's a non-present to present mapping */
2189 if (iommu_flush_iotlb_psi(domain->iommu, domain->id,
2190 start_addr, offset >> VTD_PAGE_SHIFT, 1))
2191 iommu_flush_write_buffer(domain->iommu);
2192 return nelems;
2193 }
2194
2195 static struct dma_mapping_ops intel_dma_ops = {
2196 .alloc_coherent = intel_alloc_coherent,
2197 .free_coherent = intel_free_coherent,
2198 .map_single = intel_map_single,
2199 .unmap_single = intel_unmap_single,
2200 .map_sg = intel_map_sg,
2201 .unmap_sg = intel_unmap_sg,
2202 };
2203
2204 static inline int iommu_domain_cache_init(void)
2205 {
2206 int ret = 0;
2207
2208 iommu_domain_cache = kmem_cache_create("iommu_domain",
2209 sizeof(struct dmar_domain),
2210 0,
2211 SLAB_HWCACHE_ALIGN,
2212
2213 NULL);
2214 if (!iommu_domain_cache) {
2215 printk(KERN_ERR "Couldn't create iommu_domain cache\n");
2216 ret = -ENOMEM;
2217 }
2218
2219 return ret;
2220 }
2221
2222 static inline int iommu_devinfo_cache_init(void)
2223 {
2224 int ret = 0;
2225
2226 iommu_devinfo_cache = kmem_cache_create("iommu_devinfo",
2227 sizeof(struct device_domain_info),
2228 0,
2229 SLAB_HWCACHE_ALIGN,
2230 NULL);
2231 if (!iommu_devinfo_cache) {
2232 printk(KERN_ERR "Couldn't create devinfo cache\n");
2233 ret = -ENOMEM;
2234 }
2235
2236 return ret;
2237 }
2238
2239 static inline int iommu_iova_cache_init(void)
2240 {
2241 int ret = 0;
2242
2243 iommu_iova_cache = kmem_cache_create("iommu_iova",
2244 sizeof(struct iova),
2245 0,
2246 SLAB_HWCACHE_ALIGN,
2247 NULL);
2248 if (!iommu_iova_cache) {
2249 printk(KERN_ERR "Couldn't create iova cache\n");
2250 ret = -ENOMEM;
2251 }
2252
2253 return ret;
2254 }
2255
2256 static int __init iommu_init_mempool(void)
2257 {
2258 int ret;
2259 ret = iommu_iova_cache_init();
2260 if (ret)
2261 return ret;
2262
2263 ret = iommu_domain_cache_init();
2264 if (ret)
2265 goto domain_error;
2266
2267 ret = iommu_devinfo_cache_init();
2268 if (!ret)
2269 return ret;
2270
2271 kmem_cache_destroy(iommu_domain_cache);
2272 domain_error:
2273 kmem_cache_destroy(iommu_iova_cache);
2274
2275 return -ENOMEM;
2276 }
2277
2278 static void __init iommu_exit_mempool(void)
2279 {
2280 kmem_cache_destroy(iommu_devinfo_cache);
2281 kmem_cache_destroy(iommu_domain_cache);
2282 kmem_cache_destroy(iommu_iova_cache);
2283
2284 }
2285
2286 static void __init init_no_remapping_devices(void)
2287 {
2288 struct dmar_drhd_unit *drhd;
2289
2290 for_each_drhd_unit(drhd) {
2291 if (!drhd->include_all) {
2292 int i;
2293 for (i = 0; i < drhd->devices_cnt; i++)
2294 if (drhd->devices[i] != NULL)
2295 break;
2296 /* ignore DMAR unit if no pci devices exist */
2297 if (i == drhd->devices_cnt)
2298 drhd->ignored = 1;
2299 }
2300 }
2301
2302 if (dmar_map_gfx)
2303 return;
2304
2305 for_each_drhd_unit(drhd) {
2306 int i;
2307 if (drhd->ignored || drhd->include_all)
2308 continue;
2309
2310 for (i = 0; i < drhd->devices_cnt; i++)
2311 if (drhd->devices[i] &&
2312 !IS_GFX_DEVICE(drhd->devices[i]))
2313 break;
2314
2315 if (i < drhd->devices_cnt)
2316 continue;
2317
2318 /* bypass IOMMU if it is just for gfx devices */
2319 drhd->ignored = 1;
2320 for (i = 0; i < drhd->devices_cnt; i++) {
2321 if (!drhd->devices[i])
2322 continue;
2323 drhd->devices[i]->dev.archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO;
2324 }
2325 }
2326 }
2327
2328 int __init intel_iommu_init(void)
2329 {
2330 int ret = 0;
2331
2332 if (dmar_table_init())
2333 return -ENODEV;
2334
2335 if (dmar_dev_scope_init())
2336 return -ENODEV;
2337
2338 /*
2339 * Check the need for DMA-remapping initialization now.
2340 * Above initialization will also be used by Interrupt-remapping.
2341 */
2342 if (no_iommu || swiotlb || dmar_disabled)
2343 return -ENODEV;
2344
2345 iommu_init_mempool();
2346 dmar_init_reserved_ranges();
2347
2348 init_no_remapping_devices();
2349
2350 ret = init_dmars();
2351 if (ret) {
2352 printk(KERN_ERR "IOMMU: dmar init failed\n");
2353 put_iova_domain(&reserved_iova_list);
2354 iommu_exit_mempool();
2355 return ret;
2356 }
2357 printk(KERN_INFO
2358 "PCI-DMA: Intel(R) Virtualization Technology for Directed I/O\n");
2359
2360 init_timer(&unmap_timer);
2361 force_iommu = 1;
2362 dma_ops = &intel_dma_ops;
2363 return 0;
2364 }
2365
2366 void intel_iommu_domain_exit(struct dmar_domain *domain)
2367 {
2368 u64 end;
2369
2370 /* Domain 0 is reserved, so dont process it */
2371 if (!domain)
2372 return;
2373
2374 end = DOMAIN_MAX_ADDR(domain->gaw);
2375 end = end & (~VTD_PAGE_MASK);
2376
2377 /* clear ptes */
2378 dma_pte_clear_range(domain, 0, end);
2379
2380 /* free page tables */
2381 dma_pte_free_pagetable(domain, 0, end);
2382
2383 iommu_free_domain(domain);
2384 free_domain_mem(domain);
2385 }
2386 EXPORT_SYMBOL_GPL(intel_iommu_domain_exit);
2387
2388 struct dmar_domain *intel_iommu_domain_alloc(struct pci_dev *pdev)
2389 {
2390 struct dmar_drhd_unit *drhd;
2391 struct dmar_domain *domain;
2392 struct intel_iommu *iommu;
2393
2394 drhd = dmar_find_matched_drhd_unit(pdev);
2395 if (!drhd) {
2396 printk(KERN_ERR "intel_iommu_domain_alloc: drhd == NULL\n");
2397 return NULL;
2398 }
2399
2400 iommu = drhd->iommu;
2401 if (!iommu) {
2402 printk(KERN_ERR
2403 "intel_iommu_domain_alloc: iommu == NULL\n");
2404 return NULL;
2405 }
2406 domain = iommu_alloc_domain(iommu);
2407 if (!domain) {
2408 printk(KERN_ERR
2409 "intel_iommu_domain_alloc: domain == NULL\n");
2410 return NULL;
2411 }
2412 if (domain_init(domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
2413 printk(KERN_ERR
2414 "intel_iommu_domain_alloc: domain_init() failed\n");
2415 intel_iommu_domain_exit(domain);
2416 return NULL;
2417 }
2418 return domain;
2419 }
2420 EXPORT_SYMBOL_GPL(intel_iommu_domain_alloc);
2421
2422 int intel_iommu_context_mapping(
2423 struct dmar_domain *domain, struct pci_dev *pdev)
2424 {
2425 int rc;
2426 rc = domain_context_mapping(domain, pdev);
2427 return rc;
2428 }
2429 EXPORT_SYMBOL_GPL(intel_iommu_context_mapping);
2430
2431 int intel_iommu_page_mapping(
2432 struct dmar_domain *domain, dma_addr_t iova,
2433 u64 hpa, size_t size, int prot)
2434 {
2435 int rc;
2436 rc = domain_page_mapping(domain, iova, hpa, size, prot);
2437 return rc;
2438 }
2439 EXPORT_SYMBOL_GPL(intel_iommu_page_mapping);
2440
2441 void intel_iommu_detach_dev(struct dmar_domain *domain, u8 bus, u8 devfn)
2442 {
2443 detach_domain_for_dev(domain, bus, devfn);
2444 }
2445 EXPORT_SYMBOL_GPL(intel_iommu_detach_dev);
2446
2447 struct dmar_domain *
2448 intel_iommu_find_domain(struct pci_dev *pdev)
2449 {
2450 return find_domain(pdev);
2451 }
2452 EXPORT_SYMBOL_GPL(intel_iommu_find_domain);
2453
2454 int intel_iommu_found(void)
2455 {
2456 return g_num_of_iommus;
2457 }
2458 EXPORT_SYMBOL_GPL(intel_iommu_found);
2459
2460 u64 intel_iommu_iova_to_pfn(struct dmar_domain *domain, u64 iova)
2461 {
2462 struct dma_pte *pte;
2463 u64 pfn;
2464
2465 pfn = 0;
2466 pte = addr_to_dma_pte(domain, iova);
2467
2468 if (pte)
2469 pfn = dma_pte_addr(*pte);
2470
2471 return pfn >> VTD_PAGE_SHIFT;
2472 }
2473 EXPORT_SYMBOL_GPL(intel_iommu_iova_to_pfn);
Linux kernel - Deliverables
This page took 0.079837 seconds and 6 git commands to generate.