projects / deliverable / linux.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
Merge remote-tracking branch 'vfio/next'
[deliverable/linux.git] / drivers / iommu / intel-iommu.c
1 /*
2 * Copyright © 2006-2014 Intel Corporation.
3 *
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms and conditions of the GNU General Public License,
6 * version 2, as published by the Free Software Foundation.
7 *
8 * This program is distributed in the hope it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
11 * more details.
12 *
13 * Authors: David Woodhouse <dwmw2@infradead.org>,
14 * Ashok Raj <ashok.raj@intel.com>,
15 * Shaohua Li <shaohua.li@intel.com>,
16 * Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>,
17 * Fenghua Yu <fenghua.yu@intel.com>
18 * Joerg Roedel <jroedel@suse.de>
19 */
20
21 #define pr_fmt(fmt) "DMAR: " fmt
22
23 #include <linux/init.h>
24 #include <linux/bitmap.h>
25 #include <linux/debugfs.h>
26 #include <linux/export.h>
27 #include <linux/slab.h>
28 #include <linux/irq.h>
29 #include <linux/interrupt.h>
30 #include <linux/spinlock.h>
31 #include <linux/pci.h>
32 #include <linux/dmar.h>
33 #include <linux/dma-mapping.h>
34 #include <linux/mempool.h>
35 #include <linux/memory.h>
36 #include <linux/cpu.h>
37 #include <linux/timer.h>
38 #include <linux/io.h>
39 #include <linux/iova.h>
40 #include <linux/iommu.h>
41 #include <linux/intel-iommu.h>
42 #include <linux/syscore_ops.h>
43 #include <linux/tboot.h>
44 #include <linux/dmi.h>
45 #include <linux/pci-ats.h>
46 #include <linux/memblock.h>
47 #include <linux/dma-contiguous.h>
48 #include <linux/crash_dump.h>
49 #include <asm/irq_remapping.h>
50 #include <asm/cacheflush.h>
51 #include <asm/iommu.h>
52
53 #include "irq_remapping.h"
54
55 #define ROOT_SIZE VTD_PAGE_SIZE
56 #define CONTEXT_SIZE VTD_PAGE_SIZE
57
58 #define IS_GFX_DEVICE(pdev) ((pdev->class >> 16) == PCI_BASE_CLASS_DISPLAY)
59 #define IS_USB_DEVICE(pdev) ((pdev->class >> 8) == PCI_CLASS_SERIAL_USB)
60 #define IS_ISA_DEVICE(pdev) ((pdev->class >> 8) == PCI_CLASS_BRIDGE_ISA)
61 #define IS_AZALIA(pdev) ((pdev)->vendor == 0x8086 && (pdev)->device == 0x3a3e)
62
63 #define IOAPIC_RANGE_START (0xfee00000)
64 #define IOAPIC_RANGE_END (0xfeefffff)
65 #define IOVA_START_ADDR (0x1000)
66
67 #define DEFAULT_DOMAIN_ADDRESS_WIDTH 48
68
69 #define MAX_AGAW_WIDTH 64
70 #define MAX_AGAW_PFN_WIDTH (MAX_AGAW_WIDTH - VTD_PAGE_SHIFT)
71
72 #define __DOMAIN_MAX_PFN(gaw) ((((uint64_t)1) << (gaw-VTD_PAGE_SHIFT)) - 1)
73 #define __DOMAIN_MAX_ADDR(gaw) ((((uint64_t)1) << gaw) - 1)
74
75 /* We limit DOMAIN_MAX_PFN to fit in an unsigned long, and DOMAIN_MAX_ADDR
76 to match. That way, we can use 'unsigned long' for PFNs with impunity. */
77 #define DOMAIN_MAX_PFN(gaw) ((unsigned long) min_t(uint64_t, \
78 __DOMAIN_MAX_PFN(gaw), (unsigned long)-1))
79 #define DOMAIN_MAX_ADDR(gaw) (((uint64_t)__DOMAIN_MAX_PFN(gaw)) << VTD_PAGE_SHIFT)
80
81 /* IO virtual address start page frame number */
82 #define IOVA_START_PFN (1)
83
84 #define IOVA_PFN(addr) ((addr) >> PAGE_SHIFT)
85 #define DMA_32BIT_PFN IOVA_PFN(DMA_BIT_MASK(32))
86 #define DMA_64BIT_PFN IOVA_PFN(DMA_BIT_MASK(64))
87
88 /* page table handling */
89 #define LEVEL_STRIDE (9)
90 #define LEVEL_MASK (((u64)1 << LEVEL_STRIDE) - 1)
91
92 /*
93 * This bitmap is used to advertise the page sizes our hardware support
94 * to the IOMMU core, which will then use this information to split
95 * physically contiguous memory regions it is mapping into page sizes
96 * that we support.
97 *
98 * Traditionally the IOMMU core just handed us the mappings directly,
99 * after making sure the size is an order of a 4KiB page and that the
100 * mapping has natural alignment.
101 *
102 * To retain this behavior, we currently advertise that we support
103 * all page sizes that are an order of 4KiB.
104 *
105 * If at some point we'd like to utilize the IOMMU core's new behavior,
106 * we could change this to advertise the real page sizes we support.
107 */
108 #define INTEL_IOMMU_PGSIZES (~0xFFFUL)
109
110 static inline int agaw_to_level(int agaw)
111 {
112 return agaw + 2;
113 }
114
115 static inline int agaw_to_width(int agaw)
116 {
117 return min_t(int, 30 + agaw * LEVEL_STRIDE, MAX_AGAW_WIDTH);
118 }
119
120 static inline int width_to_agaw(int width)
121 {
122 return DIV_ROUND_UP(width - 30, LEVEL_STRIDE);
123 }
124
125 static inline unsigned int level_to_offset_bits(int level)
126 {
127 return (level - 1) * LEVEL_STRIDE;
128 }
129
130 static inline int pfn_level_offset(unsigned long pfn, int level)
131 {
132 return (pfn >> level_to_offset_bits(level)) & LEVEL_MASK;
133 }
134
135 static inline unsigned long level_mask(int level)
136 {
137 return -1UL << level_to_offset_bits(level);
138 }
139
140 static inline unsigned long level_size(int level)
141 {
142 return 1UL << level_to_offset_bits(level);
143 }
144
145 static inline unsigned long align_to_level(unsigned long pfn, int level)
146 {
147 return (pfn + level_size(level) - 1) & level_mask(level);
148 }
149
150 static inline unsigned long lvl_to_nr_pages(unsigned int lvl)
151 {
152 return 1 << min_t(int, (lvl - 1) * LEVEL_STRIDE, MAX_AGAW_PFN_WIDTH);
153 }
154
155 /* VT-d pages must always be _smaller_ than MM pages. Otherwise things
156 are never going to work. */
157 static inline unsigned long dma_to_mm_pfn(unsigned long dma_pfn)
158 {
159 return dma_pfn >> (PAGE_SHIFT - VTD_PAGE_SHIFT);
160 }
161
162 static inline unsigned long mm_to_dma_pfn(unsigned long mm_pfn)
163 {
164 return mm_pfn << (PAGE_SHIFT - VTD_PAGE_SHIFT);
165 }
166 static inline unsigned long page_to_dma_pfn(struct page *pg)
167 {
168 return mm_to_dma_pfn(page_to_pfn(pg));
169 }
170 static inline unsigned long virt_to_dma_pfn(void *p)
171 {
172 return page_to_dma_pfn(virt_to_page(p));
173 }
174
175 /* global iommu list, set NULL for ignored DMAR units */
176 static struct intel_iommu **g_iommus;
177
178 static void __init check_tylersburg_isoch(void);
179 static int rwbf_quirk;
180
181 /*
182 * set to 1 to panic kernel if can't successfully enable VT-d
183 * (used when kernel is launched w/ TXT)
184 */
185 static int force_on = 0;
186
187 /*
188 * 0: Present
189 * 1-11: Reserved
190 * 12-63: Context Ptr (12 - (haw-1))
191 * 64-127: Reserved
192 */
193 struct root_entry {
194 u64 lo;
195 u64 hi;
196 };
197 #define ROOT_ENTRY_NR (VTD_PAGE_SIZE/sizeof(struct root_entry))
198
199 /*
200 * Take a root_entry and return the Lower Context Table Pointer (LCTP)
201 * if marked present.
202 */
203 static phys_addr_t root_entry_lctp(struct root_entry *re)
204 {
205 if (!(re->lo & 1))
206 return 0;
207
208 return re->lo & VTD_PAGE_MASK;
209 }
210
211 /*
212 * Take a root_entry and return the Upper Context Table Pointer (UCTP)
213 * if marked present.
214 */
215 static phys_addr_t root_entry_uctp(struct root_entry *re)
216 {
217 if (!(re->hi & 1))
218 return 0;
219
220 return re->hi & VTD_PAGE_MASK;
221 }
222 /*
223 * low 64 bits:
224 * 0: present
225 * 1: fault processing disable
226 * 2-3: translation type
227 * 12-63: address space root
228 * high 64 bits:
229 * 0-2: address width
230 * 3-6: aval
231 * 8-23: domain id
232 */
233 struct context_entry {
234 u64 lo;
235 u64 hi;
236 };
237
238 static inline void context_clear_pasid_enable(struct context_entry *context)
239 {
240 context->lo &= ~(1ULL << 11);
241 }
242
243 static inline bool context_pasid_enabled(struct context_entry *context)
244 {
245 return !!(context->lo & (1ULL << 11));
246 }
247
248 static inline void context_set_copied(struct context_entry *context)
249 {
250 context->hi |= (1ull << 3);
251 }
252
253 static inline bool context_copied(struct context_entry *context)
254 {
255 return !!(context->hi & (1ULL << 3));
256 }
257
258 static inline bool __context_present(struct context_entry *context)
259 {
260 return (context->lo & 1);
261 }
262
263 static inline bool context_present(struct context_entry *context)
264 {
265 return context_pasid_enabled(context) ?
266 __context_present(context) :
267 __context_present(context) && !context_copied(context);
268 }
269
270 static inline void context_set_present(struct context_entry *context)
271 {
272 context->lo |= 1;
273 }
274
275 static inline void context_set_fault_enable(struct context_entry *context)
276 {
277 context->lo &= (((u64)-1) << 2) | 1;
278 }
279
280 static inline void context_set_translation_type(struct context_entry *context,
281 unsigned long value)
282 {
283 context->lo &= (((u64)-1) << 4) | 3;
284 context->lo |= (value & 3) << 2;
285 }
286
287 static inline void context_set_address_root(struct context_entry *context,
288 unsigned long value)
289 {
290 context->lo &= ~VTD_PAGE_MASK;
291 context->lo |= value & VTD_PAGE_MASK;
292 }
293
294 static inline void context_set_address_width(struct context_entry *context,
295 unsigned long value)
296 {
297 context->hi |= value & 7;
298 }
299
300 static inline void context_set_domain_id(struct context_entry *context,
301 unsigned long value)
302 {
303 context->hi |= (value & ((1 << 16) - 1)) << 8;
304 }
305
306 static inline int context_domain_id(struct context_entry *c)
307 {
308 return((c->hi >> 8) & 0xffff);
309 }
310
311 static inline void context_clear_entry(struct context_entry *context)
312 {
313 context->lo = 0;
314 context->hi = 0;
315 }
316
317 /*
318 * 0: readable
319 * 1: writable
320 * 2-6: reserved
321 * 7: super page
322 * 8-10: available
323 * 11: snoop behavior
324 * 12-63: Host physcial address
325 */
326 struct dma_pte {
327 u64 val;
328 };
329
330 static inline void dma_clear_pte(struct dma_pte *pte)
331 {
332 pte->val = 0;
333 }
334
335 static inline u64 dma_pte_addr(struct dma_pte *pte)
336 {
337 #ifdef CONFIG_64BIT
338 return pte->val & VTD_PAGE_MASK;
339 #else
340 /* Must have a full atomic 64-bit read */
341 return __cmpxchg64(&pte->val, 0ULL, 0ULL) & VTD_PAGE_MASK;
342 #endif
343 }
344
345 static inline bool dma_pte_present(struct dma_pte *pte)
346 {
347 return (pte->val & 3) != 0;
348 }
349
350 static inline bool dma_pte_superpage(struct dma_pte *pte)
351 {
352 return (pte->val & DMA_PTE_LARGE_PAGE);
353 }
354
355 static inline int first_pte_in_page(struct dma_pte *pte)
356 {
357 return !((unsigned long)pte & ~VTD_PAGE_MASK);
358 }
359
360 /*
361 * This domain is a statically identity mapping domain.
362 * 1. This domain creats a static 1:1 mapping to all usable memory.
363 * 2. It maps to each iommu if successful.
364 * 3. Each iommu mapps to this domain if successful.
365 */
366 static struct dmar_domain *si_domain;
367 static int hw_pass_through = 1;
368
369 /*
370 * Domain represents a virtual machine, more than one devices
371 * across iommus may be owned in one domain, e.g. kvm guest.
372 */
373 #define DOMAIN_FLAG_VIRTUAL_MACHINE (1 << 0)
374
375 /* si_domain contains mulitple devices */
376 #define DOMAIN_FLAG_STATIC_IDENTITY (1 << 1)
377
378 #define for_each_domain_iommu(idx, domain) \
379 for (idx = 0; idx < g_num_of_iommus; idx++) \
380 if (domain->iommu_refcnt[idx])
381
382 struct dmar_domain {
383 int nid; /* node id */
384
385 unsigned iommu_refcnt[DMAR_UNITS_SUPPORTED];
386 /* Refcount of devices per iommu */
387
388
389 u16 iommu_did[DMAR_UNITS_SUPPORTED];
390 /* Domain ids per IOMMU. Use u16 since
391 * domain ids are 16 bit wide according
392 * to VT-d spec, section 9.3 */
393
394 bool has_iotlb_device;
395 struct list_head devices; /* all devices' list */
396 struct iova_domain iovad; /* iova's that belong to this domain */
397
398 struct dma_pte *pgd; /* virtual address */
399 int gaw; /* max guest address width */
400
401 /* adjusted guest address width, 0 is level 2 30-bit */
402 int agaw;
403
404 int flags; /* flags to find out type of domain */
405
406 int iommu_coherency;/* indicate coherency of iommu access */
407 int iommu_snooping; /* indicate snooping control feature*/
408 int iommu_count; /* reference count of iommu */
409 int iommu_superpage;/* Level of superpages supported:
410 0 == 4KiB (no superpages), 1 == 2MiB,
411 2 == 1GiB, 3 == 512GiB, 4 == 1TiB */
412 u64 max_addr; /* maximum mapped address */
413
414 struct iommu_domain domain; /* generic domain data structure for
415 iommu core */
416 };
417
418 /* PCI domain-device relationship */
419 struct device_domain_info {
420 struct list_head link; /* link to domain siblings */
421 struct list_head global; /* link to global list */
422 u8 bus; /* PCI bus number */
423 u8 devfn; /* PCI devfn number */
424 u8 pasid_supported:3;
425 u8 pasid_enabled:1;
426 u8 pri_supported:1;
427 u8 pri_enabled:1;
428 u8 ats_supported:1;
429 u8 ats_enabled:1;
430 u8 ats_qdep;
431 struct device *dev; /* it's NULL for PCIe-to-PCI bridge */
432 struct intel_iommu *iommu; /* IOMMU used by this device */
433 struct dmar_domain *domain; /* pointer to domain */
434 };
435
436 struct dmar_rmrr_unit {
437 struct list_head list; /* list of rmrr units */
438 struct acpi_dmar_header *hdr; /* ACPI header */
439 u64 base_address; /* reserved base address*/
440 u64 end_address; /* reserved end address */
441 struct dmar_dev_scope *devices; /* target devices */
442 int devices_cnt; /* target device count */
443 };
444
445 struct dmar_atsr_unit {
446 struct list_head list; /* list of ATSR units */
447 struct acpi_dmar_header *hdr; /* ACPI header */
448 struct dmar_dev_scope *devices; /* target devices */
449 int devices_cnt; /* target device count */
450 u8 include_all:1; /* include all ports */
451 };
452
453 static LIST_HEAD(dmar_atsr_units);
454 static LIST_HEAD(dmar_rmrr_units);
455
456 #define for_each_rmrr_units(rmrr) \
457 list_for_each_entry(rmrr, &dmar_rmrr_units, list)
458
459 static void flush_unmaps_timeout(unsigned long data);
460
461 struct deferred_flush_entry {
462 unsigned long iova_pfn;
463 unsigned long nrpages;
464 struct dmar_domain *domain;
465 struct page *freelist;
466 };
467
468 #define HIGH_WATER_MARK 250
469 struct deferred_flush_table {
470 int next;
471 struct deferred_flush_entry entries[HIGH_WATER_MARK];
472 };
473
474 struct deferred_flush_data {
475 spinlock_t lock;
476 int timer_on;
477 struct timer_list timer;
478 long size;
479 struct deferred_flush_table *tables;
480 };
481
482 DEFINE_PER_CPU(struct deferred_flush_data, deferred_flush);
483
484 /* bitmap for indexing intel_iommus */
485 static int g_num_of_iommus;
486
487 static void domain_exit(struct dmar_domain *domain);
488 static void domain_remove_dev_info(struct dmar_domain *domain);
489 static void dmar_remove_one_dev_info(struct dmar_domain *domain,
490 struct device *dev);
491 static void __dmar_remove_one_dev_info(struct device_domain_info *info);
492 static void domain_context_clear(struct intel_iommu *iommu,
493 struct device *dev);
494 static int domain_detach_iommu(struct dmar_domain *domain,
495 struct intel_iommu *iommu);
496
497 #ifdef CONFIG_INTEL_IOMMU_DEFAULT_ON
498 int dmar_disabled = 0;
499 #else
500 int dmar_disabled = 1;
501 #endif /*CONFIG_INTEL_IOMMU_DEFAULT_ON*/
502
503 int intel_iommu_enabled = 0;
504 EXPORT_SYMBOL_GPL(intel_iommu_enabled);
505
506 static int dmar_map_gfx = 1;
507 static int dmar_forcedac;
508 static int intel_iommu_strict;
509 static int intel_iommu_superpage = 1;
510 static int intel_iommu_ecs = 1;
511 static int intel_iommu_pasid28;
512 static int iommu_identity_mapping;
513
514 #define IDENTMAP_ALL 1
515 #define IDENTMAP_GFX 2
516 #define IDENTMAP_AZALIA 4
517
518 /* Broadwell and Skylake have broken ECS support — normal so-called "second
519 * level" translation of DMA requests-without-PASID doesn't actually happen
520 * unless you also set the NESTE bit in an extended context-entry. Which of
521 * course means that SVM doesn't work because it's trying to do nested
522 * translation of the physical addresses it finds in the process page tables,
523 * through the IOVA->phys mapping found in the "second level" page tables.
524 *
525 * The VT-d specification was retroactively changed to change the definition
526 * of the capability bits and pretend that Broadwell/Skylake never happened...
527 * but unfortunately the wrong bit was changed. It's ECS which is broken, but
528 * for some reason it was the PASID capability bit which was redefined (from
529 * bit 28 on BDW/SKL to bit 40 in future).
530 *
531 * So our test for ECS needs to eschew those implementations which set the old
532 * PASID capabiity bit 28, since those are the ones on which ECS is broken.
533 * Unless we are working around the 'pasid28' limitations, that is, by putting
534 * the device into passthrough mode for normal DMA and thus masking the bug.
535 */
536 #define ecs_enabled(iommu) (intel_iommu_ecs && ecap_ecs(iommu->ecap) && \
537 (intel_iommu_pasid28 || !ecap_broken_pasid(iommu->ecap)))
538 /* PASID support is thus enabled if ECS is enabled and *either* of the old
539 * or new capability bits are set. */
540 #define pasid_enabled(iommu) (ecs_enabled(iommu) && \
541 (ecap_pasid(iommu->ecap) || ecap_broken_pasid(iommu->ecap)))
542
543 int intel_iommu_gfx_mapped;
544 EXPORT_SYMBOL_GPL(intel_iommu_gfx_mapped);
545
546 #define DUMMY_DEVICE_DOMAIN_INFO ((struct device_domain_info *)(-1))
547 static DEFINE_SPINLOCK(device_domain_lock);
548 static LIST_HEAD(device_domain_list);
549
550 static const struct iommu_ops intel_iommu_ops;
551
552 static bool translation_pre_enabled(struct intel_iommu *iommu)
553 {
554 return (iommu->flags & VTD_FLAG_TRANS_PRE_ENABLED);
555 }
556
557 static void clear_translation_pre_enabled(struct intel_iommu *iommu)
558 {
559 iommu->flags &= ~VTD_FLAG_TRANS_PRE_ENABLED;
560 }
561
562 static void init_translation_status(struct intel_iommu *iommu)
563 {
564 u32 gsts;
565
566 gsts = readl(iommu->reg + DMAR_GSTS_REG);
567 if (gsts & DMA_GSTS_TES)
568 iommu->flags |= VTD_FLAG_TRANS_PRE_ENABLED;
569 }
570
571 /* Convert generic 'struct iommu_domain to private struct dmar_domain */
572 static struct dmar_domain *to_dmar_domain(struct iommu_domain *dom)
573 {
574 return container_of(dom, struct dmar_domain, domain);
575 }
576
577 static int __init intel_iommu_setup(char *str)
578 {
579 if (!str)
580 return -EINVAL;
581 while (*str) {
582 if (!strncmp(str, "on", 2)) {
583 dmar_disabled = 0;
584 pr_info("IOMMU enabled\n");
585 } else if (!strncmp(str, "off", 3)) {
586 dmar_disabled = 1;
587 pr_info("IOMMU disabled\n");
588 } else if (!strncmp(str, "igfx_off", 8)) {
589 dmar_map_gfx = 0;
590 pr_info("Disable GFX device mapping\n");
591 } else if (!strncmp(str, "forcedac", 8)) {
592 pr_info("Forcing DAC for PCI devices\n");
593 dmar_forcedac = 1;
594 } else if (!strncmp(str, "strict", 6)) {
595 pr_info("Disable batched IOTLB flush\n");
596 intel_iommu_strict = 1;
597 } else if (!strncmp(str, "sp_off", 6)) {
598 pr_info("Disable supported super page\n");
599 intel_iommu_superpage = 0;
600 } else if (!strncmp(str, "ecs_off", 7)) {
601 printk(KERN_INFO
602 "Intel-IOMMU: disable extended context table support\n");
603 intel_iommu_ecs = 0;
604 } else if (!strncmp(str, "pasid28", 7)) {
605 printk(KERN_INFO
606 "Intel-IOMMU: enable pre-production PASID support\n");
607 intel_iommu_pasid28 = 1;
608 iommu_identity_mapping |= IDENTMAP_GFX;
609 }
610
611 str += strcspn(str, ",");
612 while (*str == ',')
613 str++;
614 }
615 return 0;
616 }
617 __setup("intel_iommu=", intel_iommu_setup);
618
619 static struct kmem_cache *iommu_domain_cache;
620 static struct kmem_cache *iommu_devinfo_cache;
621
622 static struct dmar_domain* get_iommu_domain(struct intel_iommu *iommu, u16 did)
623 {
624 struct dmar_domain **domains;
625 int idx = did >> 8;
626
627 domains = iommu->domains[idx];
628 if (!domains)
629 return NULL;
630
631 return domains[did & 0xff];
632 }
633
634 static void set_iommu_domain(struct intel_iommu *iommu, u16 did,
635 struct dmar_domain *domain)
636 {
637 struct dmar_domain **domains;
638 int idx = did >> 8;
639
640 if (!iommu->domains[idx]) {
641 size_t size = 256 * sizeof(struct dmar_domain *);
642 iommu->domains[idx] = kzalloc(size, GFP_ATOMIC);
643 }
644
645 domains = iommu->domains[idx];
646 if (WARN_ON(!domains))
647 return;
648 else
649 domains[did & 0xff] = domain;
650 }
651
652 static inline void *alloc_pgtable_page(int node)
653 {
654 struct page *page;
655 void *vaddr = NULL;
656
657 page = alloc_pages_node(node, GFP_ATOMIC | __GFP_ZERO, 0);
658 if (page)
659 vaddr = page_address(page);
660 return vaddr;
661 }
662
663 static inline void free_pgtable_page(void *vaddr)
664 {
665 free_page((unsigned long)vaddr);
666 }
667
668 static inline void *alloc_domain_mem(void)
669 {
670 return kmem_cache_alloc(iommu_domain_cache, GFP_ATOMIC);
671 }
672
673 static void free_domain_mem(void *vaddr)
674 {
675 kmem_cache_free(iommu_domain_cache, vaddr);
676 }
677
678 static inline void * alloc_devinfo_mem(void)
679 {
680 return kmem_cache_alloc(iommu_devinfo_cache, GFP_ATOMIC);
681 }
682
683 static inline void free_devinfo_mem(void *vaddr)
684 {
685 kmem_cache_free(iommu_devinfo_cache, vaddr);
686 }
687
688 static inline int domain_type_is_vm(struct dmar_domain *domain)
689 {
690 return domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE;
691 }
692
693 static inline int domain_type_is_si(struct dmar_domain *domain)
694 {
695 return domain->flags & DOMAIN_FLAG_STATIC_IDENTITY;
696 }
697
698 static inline int domain_type_is_vm_or_si(struct dmar_domain *domain)
699 {
700 return domain->flags & (DOMAIN_FLAG_VIRTUAL_MACHINE |
701 DOMAIN_FLAG_STATIC_IDENTITY);
702 }
703
704 static inline int domain_pfn_supported(struct dmar_domain *domain,
705 unsigned long pfn)
706 {
707 int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
708
709 return !(addr_width < BITS_PER_LONG && pfn >> addr_width);
710 }
711
712 static int __iommu_calculate_agaw(struct intel_iommu *iommu, int max_gaw)
713 {
714 unsigned long sagaw;
715 int agaw = -1;
716
717 sagaw = cap_sagaw(iommu->cap);
718 for (agaw = width_to_agaw(max_gaw);
719 agaw >= 0; agaw--) {
720 if (test_bit(agaw, &sagaw))
721 break;
722 }
723
724 return agaw;
725 }
726
727 /*
728 * Calculate max SAGAW for each iommu.
729 */
730 int iommu_calculate_max_sagaw(struct intel_iommu *iommu)
731 {
732 return __iommu_calculate_agaw(iommu, MAX_AGAW_WIDTH);
733 }
734
735 /*
736 * calculate agaw for each iommu.
737 * "SAGAW" may be different across iommus, use a default agaw, and
738 * get a supported less agaw for iommus that don't support the default agaw.
739 */
740 int iommu_calculate_agaw(struct intel_iommu *iommu)
741 {
742 return __iommu_calculate_agaw(iommu, DEFAULT_DOMAIN_ADDRESS_WIDTH);
743 }
744
745 /* This functionin only returns single iommu in a domain */
746 static struct intel_iommu *domain_get_iommu(struct dmar_domain *domain)
747 {
748 int iommu_id;
749
750 /* si_domain and vm domain should not get here. */
751 BUG_ON(domain_type_is_vm_or_si(domain));
752 for_each_domain_iommu(iommu_id, domain)
753 break;
754
755 if (iommu_id < 0 || iommu_id >= g_num_of_iommus)
756 return NULL;
757
758 return g_iommus[iommu_id];
759 }
760
761 static void domain_update_iommu_coherency(struct dmar_domain *domain)
762 {
763 struct dmar_drhd_unit *drhd;
764 struct intel_iommu *iommu;
765 bool found = false;
766 int i;
767
768 domain->iommu_coherency = 1;
769
770 for_each_domain_iommu(i, domain) {
771 found = true;
772 if (!ecap_coherent(g_iommus[i]->ecap)) {
773 domain->iommu_coherency = 0;
774 break;
775 }
776 }
777 if (found)
778 return;
779
780 /* No hardware attached; use lowest common denominator */
781 rcu_read_lock();
782 for_each_active_iommu(iommu, drhd) {
783 if (!ecap_coherent(iommu->ecap)) {
784 domain->iommu_coherency = 0;
785 break;
786 }
787 }
788 rcu_read_unlock();
789 }
790
791 static int domain_update_iommu_snooping(struct intel_iommu *skip)
792 {
793 struct dmar_drhd_unit *drhd;
794 struct intel_iommu *iommu;
795 int ret = 1;
796
797 rcu_read_lock();
798 for_each_active_iommu(iommu, drhd) {
799 if (iommu != skip) {
800 if (!ecap_sc_support(iommu->ecap)) {
801 ret = 0;
802 break;
803 }
804 }
805 }
806 rcu_read_unlock();
807
808 return ret;
809 }
810
811 static int domain_update_iommu_superpage(struct intel_iommu *skip)
812 {
813 struct dmar_drhd_unit *drhd;
814 struct intel_iommu *iommu;
815 int mask = 0xf;
816
817 if (!intel_iommu_superpage) {
818 return 0;
819 }
820
821 /* set iommu_superpage to the smallest common denominator */
822 rcu_read_lock();
823 for_each_active_iommu(iommu, drhd) {
824 if (iommu != skip) {
825 mask &= cap_super_page_val(iommu->cap);
826 if (!mask)
827 break;
828 }
829 }
830 rcu_read_unlock();
831
832 return fls(mask);
833 }
834
835 /* Some capabilities may be different across iommus */
836 static void domain_update_iommu_cap(struct dmar_domain *domain)
837 {
838 domain_update_iommu_coherency(domain);
839 domain->iommu_snooping = domain_update_iommu_snooping(NULL);
840 domain->iommu_superpage = domain_update_iommu_superpage(NULL);
841 }
842
843 static inline struct context_entry *iommu_context_addr(struct intel_iommu *iommu,
844 u8 bus, u8 devfn, int alloc)
845 {
846 struct root_entry *root = &iommu->root_entry[bus];
847 struct context_entry *context;
848 u64 *entry;
849
850 entry = &root->lo;
851 if (ecs_enabled(iommu)) {
852 if (devfn >= 0x80) {
853 devfn -= 0x80;
854 entry = &root->hi;
855 }
856 devfn *= 2;
857 }
858 if (*entry & 1)
859 context = phys_to_virt(*entry & VTD_PAGE_MASK);
860 else {
861 unsigned long phy_addr;
862 if (!alloc)
863 return NULL;
864
865 context = alloc_pgtable_page(iommu->node);
866 if (!context)
867 return NULL;
868
869 __iommu_flush_cache(iommu, (void *)context, CONTEXT_SIZE);
870 phy_addr = virt_to_phys((void *)context);
871 *entry = phy_addr | 1;
872 __iommu_flush_cache(iommu, entry, sizeof(*entry));
873 }
874 return &context[devfn];
875 }
876
877 static int iommu_dummy(struct device *dev)
878 {
879 return dev->archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO;
880 }
881
882 static struct intel_iommu *device_to_iommu(struct device *dev, u8 *bus, u8 *devfn)
883 {
884 struct dmar_drhd_unit *drhd = NULL;
885 struct intel_iommu *iommu;
886 struct device *tmp;
887 struct pci_dev *ptmp, *pdev = NULL;
888 u16 segment = 0;
889 int i;
890
891 if (iommu_dummy(dev))
892 return NULL;
893
894 if (dev_is_pci(dev)) {
895 pdev = to_pci_dev(dev);
896 segment = pci_domain_nr(pdev->bus);
897 } else if (has_acpi_companion(dev))
898 dev = &ACPI_COMPANION(dev)->dev;
899
900 rcu_read_lock();
901 for_each_active_iommu(iommu, drhd) {
902 if (pdev && segment != drhd->segment)
903 continue;
904
905 for_each_active_dev_scope(drhd->devices,
906 drhd->devices_cnt, i, tmp) {
907 if (tmp == dev) {
908 *bus = drhd->devices[i].bus;
909 *devfn = drhd->devices[i].devfn;
910 goto out;
911 }
912
913 if (!pdev || !dev_is_pci(tmp))
914 continue;
915
916 ptmp = to_pci_dev(tmp);
917 if (ptmp->subordinate &&
918 ptmp->subordinate->number <= pdev->bus->number &&
919 ptmp->subordinate->busn_res.end >= pdev->bus->number)
920 goto got_pdev;
921 }
922
923 if (pdev && drhd->include_all) {
924 got_pdev:
925 *bus = pdev->bus->number;
926 *devfn = pdev->devfn;
927 goto out;
928 }
929 }
930 iommu = NULL;
931 out:
932 rcu_read_unlock();
933
934 return iommu;
935 }
936
937 static void domain_flush_cache(struct dmar_domain *domain,
938 void *addr, int size)
939 {
940 if (!domain->iommu_coherency)
941 clflush_cache_range(addr, size);
942 }
943
944 static int device_context_mapped(struct intel_iommu *iommu, u8 bus, u8 devfn)
945 {
946 struct context_entry *context;
947 int ret = 0;
948 unsigned long flags;
949
950 spin_lock_irqsave(&iommu->lock, flags);
951 context = iommu_context_addr(iommu, bus, devfn, 0);
952 if (context)
953 ret = context_present(context);
954 spin_unlock_irqrestore(&iommu->lock, flags);
955 return ret;
956 }
957
958 static void clear_context_table(struct intel_iommu *iommu, u8 bus, u8 devfn)
959 {
960 struct context_entry *context;
961 unsigned long flags;
962
963 spin_lock_irqsave(&iommu->lock, flags);
964 context = iommu_context_addr(iommu, bus, devfn, 0);
965 if (context) {
966 context_clear_entry(context);
967 __iommu_flush_cache(iommu, context, sizeof(*context));
968 }
969 spin_unlock_irqrestore(&iommu->lock, flags);
970 }
971
972 static void free_context_table(struct intel_iommu *iommu)
973 {
974 int i;
975 unsigned long flags;
976 struct context_entry *context;
977
978 spin_lock_irqsave(&iommu->lock, flags);
979 if (!iommu->root_entry) {
980 goto out;
981 }
982 for (i = 0; i < ROOT_ENTRY_NR; i++) {
983 context = iommu_context_addr(iommu, i, 0, 0);
984 if (context)
985 free_pgtable_page(context);
986
987 if (!ecs_enabled(iommu))
988 continue;
989
990 context = iommu_context_addr(iommu, i, 0x80, 0);
991 if (context)
992 free_pgtable_page(context);
993
994 }
995 free_pgtable_page(iommu->root_entry);
996 iommu->root_entry = NULL;
997 out:
998 spin_unlock_irqrestore(&iommu->lock, flags);
999 }
1000
1001 static struct dma_pte *pfn_to_dma_pte(struct dmar_domain *domain,
1002 unsigned long pfn, int *target_level)
1003 {
1004 struct dma_pte *parent, *pte = NULL;
1005 int level = agaw_to_level(domain->agaw);
1006 int offset;
1007
1008 BUG_ON(!domain->pgd);
1009
1010 if (!domain_pfn_supported(domain, pfn))
1011 /* Address beyond IOMMU's addressing capabilities. */
1012 return NULL;
1013
1014 parent = domain->pgd;
1015
1016 while (1) {
1017 void *tmp_page;
1018
1019 offset = pfn_level_offset(pfn, level);
1020 pte = &parent[offset];
1021 if (!*target_level && (dma_pte_superpage(pte) || !dma_pte_present(pte)))
1022 break;
1023 if (level == *target_level)
1024 break;
1025
1026 if (!dma_pte_present(pte)) {
1027 uint64_t pteval;
1028
1029 tmp_page = alloc_pgtable_page(domain->nid);
1030
1031 if (!tmp_page)
1032 return NULL;
1033
1034 domain_flush_cache(domain, tmp_page, VTD_PAGE_SIZE);
1035 pteval = ((uint64_t)virt_to_dma_pfn(tmp_page) << VTD_PAGE_SHIFT) | DMA_PTE_READ | DMA_PTE_WRITE;
1036 if (cmpxchg64(&pte->val, 0ULL, pteval))
1037 /* Someone else set it while we were thinking; use theirs. */
1038 free_pgtable_page(tmp_page);
1039 else
1040 domain_flush_cache(domain, pte, sizeof(*pte));
1041 }
1042 if (level == 1)
1043 break;
1044
1045 parent = phys_to_virt(dma_pte_addr(pte));
1046 level--;
1047 }
1048
1049 if (!*target_level)
1050 *target_level = level;
1051
1052 return pte;
1053 }
1054
1055
1056 /* return address's pte at specific level */
1057 static struct dma_pte *dma_pfn_level_pte(struct dmar_domain *domain,
1058 unsigned long pfn,
1059 int level, int *large_page)
1060 {
1061 struct dma_pte *parent, *pte = NULL;
1062 int total = agaw_to_level(domain->agaw);
1063 int offset;
1064
1065 parent = domain->pgd;
1066 while (level <= total) {
1067 offset = pfn_level_offset(pfn, total);
1068 pte = &parent[offset];
1069 if (level == total)
1070 return pte;
1071
1072 if (!dma_pte_present(pte)) {
1073 *large_page = total;
1074 break;
1075 }
1076
1077 if (dma_pte_superpage(pte)) {
1078 *large_page = total;
1079 return pte;
1080 }
1081
1082 parent = phys_to_virt(dma_pte_addr(pte));
1083 total--;
1084 }
1085 return NULL;
1086 }
1087
1088 /* clear last level pte, a tlb flush should be followed */
1089 static void dma_pte_clear_range(struct dmar_domain *domain,
1090 unsigned long start_pfn,
1091 unsigned long last_pfn)
1092 {
1093 unsigned int large_page = 1;
1094 struct dma_pte *first_pte, *pte;
1095
1096 BUG_ON(!domain_pfn_supported(domain, start_pfn));
1097 BUG_ON(!domain_pfn_supported(domain, last_pfn));
1098 BUG_ON(start_pfn > last_pfn);
1099
1100 /* we don't need lock here; nobody else touches the iova range */
1101 do {
1102 large_page = 1;
1103 first_pte = pte = dma_pfn_level_pte(domain, start_pfn, 1, &large_page);
1104 if (!pte) {
1105 start_pfn = align_to_level(start_pfn + 1, large_page + 1);
1106 continue;
1107 }
1108 do {
1109 dma_clear_pte(pte);
1110 start_pfn += lvl_to_nr_pages(large_page);
1111 pte++;
1112 } while (start_pfn <= last_pfn && !first_pte_in_page(pte));
1113
1114 domain_flush_cache(domain, first_pte,
1115 (void *)pte - (void *)first_pte);
1116
1117 } while (start_pfn && start_pfn <= last_pfn);
1118 }
1119
1120 static void dma_pte_free_level(struct dmar_domain *domain, int level,
1121 struct dma_pte *pte, unsigned long pfn,
1122 unsigned long start_pfn, unsigned long last_pfn)
1123 {
1124 pfn = max(start_pfn, pfn);
1125 pte = &pte[pfn_level_offset(pfn, level)];
1126
1127 do {
1128 unsigned long level_pfn;
1129 struct dma_pte *level_pte;
1130
1131 if (!dma_pte_present(pte) || dma_pte_superpage(pte))
1132 goto next;
1133
1134 level_pfn = pfn & level_mask(level - 1);
1135 level_pte = phys_to_virt(dma_pte_addr(pte));
1136
1137 if (level > 2)
1138 dma_pte_free_level(domain, level - 1, level_pte,
1139 level_pfn, start_pfn, last_pfn);
1140
1141 /* If range covers entire pagetable, free it */
1142 if (!(start_pfn > level_pfn ||
1143 last_pfn < level_pfn + level_size(level) - 1)) {
1144 dma_clear_pte(pte);
1145 domain_flush_cache(domain, pte, sizeof(*pte));
1146 free_pgtable_page(level_pte);
1147 }
1148 next:
1149 pfn += level_size(level);
1150 } while (!first_pte_in_page(++pte) && pfn <= last_pfn);
1151 }
1152
1153 /* clear last level (leaf) ptes and free page table pages. */
1154 static void dma_pte_free_pagetable(struct dmar_domain *domain,
1155 unsigned long start_pfn,
1156 unsigned long last_pfn)
1157 {
1158 BUG_ON(!domain_pfn_supported(domain, start_pfn));
1159 BUG_ON(!domain_pfn_supported(domain, last_pfn));
1160 BUG_ON(start_pfn > last_pfn);
1161
1162 dma_pte_clear_range(domain, start_pfn, last_pfn);
1163
1164 /* We don't need lock here; nobody else touches the iova range */
1165 dma_pte_free_level(domain, agaw_to_level(domain->agaw),
1166 domain->pgd, 0, start_pfn, last_pfn);
1167
1168 /* free pgd */
1169 if (start_pfn == 0 && last_pfn == DOMAIN_MAX_PFN(domain->gaw)) {
1170 free_pgtable_page(domain->pgd);
1171 domain->pgd = NULL;
1172 }
1173 }
1174
1175 /* When a page at a given level is being unlinked from its parent, we don't
1176 need to *modify* it at all. All we need to do is make a list of all the
1177 pages which can be freed just as soon as we've flushed the IOTLB and we
1178 know the hardware page-walk will no longer touch them.
1179 The 'pte' argument is the *parent* PTE, pointing to the page that is to
1180 be freed. */
1181 static struct page *dma_pte_list_pagetables(struct dmar_domain *domain,
1182 int level, struct dma_pte *pte,
1183 struct page *freelist)
1184 {
1185 struct page *pg;
1186
1187 pg = pfn_to_page(dma_pte_addr(pte) >> PAGE_SHIFT);
1188 pg->freelist = freelist;
1189 freelist = pg;
1190
1191 if (level == 1)
1192 return freelist;
1193
1194 pte = page_address(pg);
1195 do {
1196 if (dma_pte_present(pte) && !dma_pte_superpage(pte))
1197 freelist = dma_pte_list_pagetables(domain, level - 1,
1198 pte, freelist);
1199 pte++;
1200 } while (!first_pte_in_page(pte));
1201
1202 return freelist;
1203 }
1204
1205 static struct page *dma_pte_clear_level(struct dmar_domain *domain, int level,
1206 struct dma_pte *pte, unsigned long pfn,
1207 unsigned long start_pfn,
1208 unsigned long last_pfn,
1209 struct page *freelist)
1210 {
1211 struct dma_pte *first_pte = NULL, *last_pte = NULL;
1212
1213 pfn = max(start_pfn, pfn);
1214 pte = &pte[pfn_level_offset(pfn, level)];
1215
1216 do {
1217 unsigned long level_pfn;
1218
1219 if (!dma_pte_present(pte))
1220 goto next;
1221
1222 level_pfn = pfn & level_mask(level);
1223
1224 /* If range covers entire pagetable, free it */
1225 if (start_pfn <= level_pfn &&
1226 last_pfn >= level_pfn + level_size(level) - 1) {
1227 /* These suborbinate page tables are going away entirely. Don't
1228 bother to clear them; we're just going to *free* them. */
1229 if (level > 1 && !dma_pte_superpage(pte))
1230 freelist = dma_pte_list_pagetables(domain, level - 1, pte, freelist);
1231
1232 dma_clear_pte(pte);
1233 if (!first_pte)
1234 first_pte = pte;
1235 last_pte = pte;
1236 } else if (level > 1) {
1237 /* Recurse down into a level that isn't *entirely* obsolete */
1238 freelist = dma_pte_clear_level(domain, level - 1,
1239 phys_to_virt(dma_pte_addr(pte)),
1240 level_pfn, start_pfn, last_pfn,
1241 freelist);
1242 }
1243 next:
1244 pfn += level_size(level);
1245 } while (!first_pte_in_page(++pte) && pfn <= last_pfn);
1246
1247 if (first_pte)
1248 domain_flush_cache(domain, first_pte,
1249 (void *)++last_pte - (void *)first_pte);
1250
1251 return freelist;
1252 }
1253
1254 /* We can't just free the pages because the IOMMU may still be walking
1255 the page tables, and may have cached the intermediate levels. The
1256 pages can only be freed after the IOTLB flush has been done. */
1257 static struct page *domain_unmap(struct dmar_domain *domain,
1258 unsigned long start_pfn,
1259 unsigned long last_pfn)
1260 {
1261 struct page *freelist = NULL;
1262
1263 BUG_ON(!domain_pfn_supported(domain, start_pfn));
1264 BUG_ON(!domain_pfn_supported(domain, last_pfn));
1265 BUG_ON(start_pfn > last_pfn);
1266
1267 /* we don't need lock here; nobody else touches the iova range */
1268 freelist = dma_pte_clear_level(domain, agaw_to_level(domain->agaw),
1269 domain->pgd, 0, start_pfn, last_pfn, NULL);
1270
1271 /* free pgd */
1272 if (start_pfn == 0 && last_pfn == DOMAIN_MAX_PFN(domain->gaw)) {
1273 struct page *pgd_page = virt_to_page(domain->pgd);
1274 pgd_page->freelist = freelist;
1275 freelist = pgd_page;
1276
1277 domain->pgd = NULL;
1278 }
1279
1280 return freelist;
1281 }
1282
1283 static void dma_free_pagelist(struct page *freelist)
1284 {
1285 struct page *pg;
1286
1287 while ((pg = freelist)) {
1288 freelist = pg->freelist;
1289 free_pgtable_page(page_address(pg));
1290 }
1291 }
1292
1293 /* iommu handling */
1294 static int iommu_alloc_root_entry(struct intel_iommu *iommu)
1295 {
1296 struct root_entry *root;
1297 unsigned long flags;
1298
1299 root = (struct root_entry *)alloc_pgtable_page(iommu->node);
1300 if (!root) {
1301 pr_err("Allocating root entry for %s failed\n",
1302 iommu->name);
1303 return -ENOMEM;
1304 }
1305
1306 __iommu_flush_cache(iommu, root, ROOT_SIZE);
1307
1308 spin_lock_irqsave(&iommu->lock, flags);
1309 iommu->root_entry = root;
1310 spin_unlock_irqrestore(&iommu->lock, flags);
1311
1312 return 0;
1313 }
1314
1315 static void iommu_set_root_entry(struct intel_iommu *iommu)
1316 {
1317 u64 addr;
1318 u32 sts;
1319 unsigned long flag;
1320
1321 addr = virt_to_phys(iommu->root_entry);
1322 if (ecs_enabled(iommu))
1323 addr |= DMA_RTADDR_RTT;
1324
1325 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1326 dmar_writeq(iommu->reg + DMAR_RTADDR_REG, addr);
1327
1328 writel(iommu->gcmd | DMA_GCMD_SRTP, iommu->reg + DMAR_GCMD_REG);
1329
1330 /* Make sure hardware complete it */
1331 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1332 readl, (sts & DMA_GSTS_RTPS), sts);
1333
1334 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1335 }
1336
1337 static void iommu_flush_write_buffer(struct intel_iommu *iommu)
1338 {
1339 u32 val;
1340 unsigned long flag;
1341
1342 if (!rwbf_quirk && !cap_rwbf(iommu->cap))
1343 return;
1344
1345 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1346 writel(iommu->gcmd | DMA_GCMD_WBF, iommu->reg + DMAR_GCMD_REG);
1347
1348 /* Make sure hardware complete it */
1349 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1350 readl, (!(val & DMA_GSTS_WBFS)), val);
1351
1352 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1353 }
1354
1355 /* return value determine if we need a write buffer flush */
1356 static void __iommu_flush_context(struct intel_iommu *iommu,
1357 u16 did, u16 source_id, u8 function_mask,
1358 u64 type)
1359 {
1360 u64 val = 0;
1361 unsigned long flag;
1362
1363 switch (type) {
1364 case DMA_CCMD_GLOBAL_INVL:
1365 val = DMA_CCMD_GLOBAL_INVL;
1366 break;
1367 case DMA_CCMD_DOMAIN_INVL:
1368 val = DMA_CCMD_DOMAIN_INVL|DMA_CCMD_DID(did);
1369 break;
1370 case DMA_CCMD_DEVICE_INVL:
1371 val = DMA_CCMD_DEVICE_INVL|DMA_CCMD_DID(did)
1372 | DMA_CCMD_SID(source_id) | DMA_CCMD_FM(function_mask);
1373 break;
1374 default:
1375 BUG();
1376 }
1377 val |= DMA_CCMD_ICC;
1378
1379 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1380 dmar_writeq(iommu->reg + DMAR_CCMD_REG, val);
1381
1382 /* Make sure hardware complete it */
1383 IOMMU_WAIT_OP(iommu, DMAR_CCMD_REG,
1384 dmar_readq, (!(val & DMA_CCMD_ICC)), val);
1385
1386 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1387 }
1388
1389 /* return value determine if we need a write buffer flush */
1390 static void __iommu_flush_iotlb(struct intel_iommu *iommu, u16 did,
1391 u64 addr, unsigned int size_order, u64 type)
1392 {
1393 int tlb_offset = ecap_iotlb_offset(iommu->ecap);
1394 u64 val = 0, val_iva = 0;
1395 unsigned long flag;
1396
1397 switch (type) {
1398 case DMA_TLB_GLOBAL_FLUSH:
1399 /* global flush doesn't need set IVA_REG */
1400 val = DMA_TLB_GLOBAL_FLUSH|DMA_TLB_IVT;
1401 break;
1402 case DMA_TLB_DSI_FLUSH:
1403 val = DMA_TLB_DSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
1404 break;
1405 case DMA_TLB_PSI_FLUSH:
1406 val = DMA_TLB_PSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
1407 /* IH bit is passed in as part of address */
1408 val_iva = size_order | addr;
1409 break;
1410 default:
1411 BUG();
1412 }
1413 /* Note: set drain read/write */
1414 #if 0
1415 /*
1416 * This is probably to be super secure.. Looks like we can
1417 * ignore it without any impact.
1418 */
1419 if (cap_read_drain(iommu->cap))
1420 val |= DMA_TLB_READ_DRAIN;
1421 #endif
1422 if (cap_write_drain(iommu->cap))
1423 val |= DMA_TLB_WRITE_DRAIN;
1424
1425 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1426 /* Note: Only uses first TLB reg currently */
1427 if (val_iva)
1428 dmar_writeq(iommu->reg + tlb_offset, val_iva);
1429 dmar_writeq(iommu->reg + tlb_offset + 8, val);
1430
1431 /* Make sure hardware complete it */
1432 IOMMU_WAIT_OP(iommu, tlb_offset + 8,
1433 dmar_readq, (!(val & DMA_TLB_IVT)), val);
1434
1435 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1436
1437 /* check IOTLB invalidation granularity */
1438 if (DMA_TLB_IAIG(val) == 0)
1439 pr_err("Flush IOTLB failed\n");
1440 if (DMA_TLB_IAIG(val) != DMA_TLB_IIRG(type))
1441 pr_debug("TLB flush request %Lx, actual %Lx\n",
1442 (unsigned long long)DMA_TLB_IIRG(type),
1443 (unsigned long long)DMA_TLB_IAIG(val));
1444 }
1445
1446 static struct device_domain_info *
1447 iommu_support_dev_iotlb (struct dmar_domain *domain, struct intel_iommu *iommu,
1448 u8 bus, u8 devfn)
1449 {
1450 struct device_domain_info *info;
1451
1452 assert_spin_locked(&device_domain_lock);
1453
1454 if (!iommu->qi)
1455 return NULL;
1456
1457 list_for_each_entry(info, &domain->devices, link)
1458 if (info->iommu == iommu && info->bus == bus &&
1459 info->devfn == devfn) {
1460 if (info->ats_supported && info->dev)
1461 return info;
1462 break;
1463 }
1464
1465 return NULL;
1466 }
1467
1468 static void domain_update_iotlb(struct dmar_domain *domain)
1469 {
1470 struct device_domain_info *info;
1471 bool has_iotlb_device = false;
1472
1473 assert_spin_locked(&device_domain_lock);
1474
1475 list_for_each_entry(info, &domain->devices, link) {
1476 struct pci_dev *pdev;
1477
1478 if (!info->dev || !dev_is_pci(info->dev))
1479 continue;
1480
1481 pdev = to_pci_dev(info->dev);
1482 if (pdev->ats_enabled) {
1483 has_iotlb_device = true;
1484 break;
1485 }
1486 }
1487
1488 domain->has_iotlb_device = has_iotlb_device;
1489 }
1490
1491 static void iommu_enable_dev_iotlb(struct device_domain_info *info)
1492 {
1493 struct pci_dev *pdev;
1494
1495 assert_spin_locked(&device_domain_lock);
1496
1497 if (!info || !dev_is_pci(info->dev))
1498 return;
1499
1500 pdev = to_pci_dev(info->dev);
1501
1502 #ifdef CONFIG_INTEL_IOMMU_SVM
1503 /* The PCIe spec, in its wisdom, declares that the behaviour of
1504 the device if you enable PASID support after ATS support is
1505 undefined. So always enable PASID support on devices which
1506 have it, even if we can't yet know if we're ever going to
1507 use it. */
1508 if (info->pasid_supported && !pci_enable_pasid(pdev, info->pasid_supported & ~1))
1509 info->pasid_enabled = 1;
1510
1511 if (info->pri_supported && !pci_reset_pri(pdev) && !pci_enable_pri(pdev, 32))
1512 info->pri_enabled = 1;
1513 #endif
1514 if (info->ats_supported && !pci_enable_ats(pdev, VTD_PAGE_SHIFT)) {
1515 info->ats_enabled = 1;
1516 domain_update_iotlb(info->domain);
1517 info->ats_qdep = pci_ats_queue_depth(pdev);
1518 }
1519 }
1520
1521 static void iommu_disable_dev_iotlb(struct device_domain_info *info)
1522 {
1523 struct pci_dev *pdev;
1524
1525 assert_spin_locked(&device_domain_lock);
1526
1527 if (!dev_is_pci(info->dev))
1528 return;
1529
1530 pdev = to_pci_dev(info->dev);
1531
1532 if (info->ats_enabled) {
1533 pci_disable_ats(pdev);
1534 info->ats_enabled = 0;
1535 domain_update_iotlb(info->domain);
1536 }
1537 #ifdef CONFIG_INTEL_IOMMU_SVM
1538 if (info->pri_enabled) {
1539 pci_disable_pri(pdev);
1540 info->pri_enabled = 0;
1541 }
1542 if (info->pasid_enabled) {
1543 pci_disable_pasid(pdev);
1544 info->pasid_enabled = 0;
1545 }
1546 #endif
1547 }
1548
1549 static void iommu_flush_dev_iotlb(struct dmar_domain *domain,
1550 u64 addr, unsigned mask)
1551 {
1552 u16 sid, qdep;
1553 unsigned long flags;
1554 struct device_domain_info *info;
1555
1556 if (!domain->has_iotlb_device)
1557 return;
1558
1559 spin_lock_irqsave(&device_domain_lock, flags);
1560 list_for_each_entry(info, &domain->devices, link) {
1561 if (!info->ats_enabled)
1562 continue;
1563
1564 sid = info->bus << 8 | info->devfn;
1565 qdep = info->ats_qdep;
1566 qi_flush_dev_iotlb(info->iommu, sid, qdep, addr, mask);
1567 }
1568 spin_unlock_irqrestore(&device_domain_lock, flags);
1569 }
1570
1571 static void iommu_flush_iotlb_psi(struct intel_iommu *iommu,
1572 struct dmar_domain *domain,
1573 unsigned long pfn, unsigned int pages,
1574 int ih, int map)
1575 {
1576 unsigned int mask = ilog2(__roundup_pow_of_two(pages));
1577 uint64_t addr = (uint64_t)pfn << VTD_PAGE_SHIFT;
1578 u16 did = domain->iommu_did[iommu->seq_id];
1579
1580 BUG_ON(pages == 0);
1581
1582 if (ih)
1583 ih = 1 << 6;
1584 /*
1585 * Fallback to domain selective flush if no PSI support or the size is
1586 * too big.
1587 * PSI requires page size to be 2 ^ x, and the base address is naturally
1588 * aligned to the size
1589 */
1590 if (!cap_pgsel_inv(iommu->cap) || mask > cap_max_amask_val(iommu->cap))
1591 iommu->flush.flush_iotlb(iommu, did, 0, 0,
1592 DMA_TLB_DSI_FLUSH);
1593 else
1594 iommu->flush.flush_iotlb(iommu, did, addr | ih, mask,
1595 DMA_TLB_PSI_FLUSH);
1596
1597 /*
1598 * In caching mode, changes of pages from non-present to present require
1599 * flush. However, device IOTLB doesn't need to be flushed in this case.
1600 */
1601 if (!cap_caching_mode(iommu->cap) || !map)
1602 iommu_flush_dev_iotlb(get_iommu_domain(iommu, did),
1603 addr, mask);
1604 }
1605
1606 static void iommu_disable_protect_mem_regions(struct intel_iommu *iommu)
1607 {
1608 u32 pmen;
1609 unsigned long flags;
1610
1611 raw_spin_lock_irqsave(&iommu->register_lock, flags);
1612 pmen = readl(iommu->reg + DMAR_PMEN_REG);
1613 pmen &= ~DMA_PMEN_EPM;
1614 writel(pmen, iommu->reg + DMAR_PMEN_REG);
1615
1616 /* wait for the protected region status bit to clear */
1617 IOMMU_WAIT_OP(iommu, DMAR_PMEN_REG,
1618 readl, !(pmen & DMA_PMEN_PRS), pmen);
1619
1620 raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
1621 }
1622
1623 static void iommu_enable_translation(struct intel_iommu *iommu)
1624 {
1625 u32 sts;
1626 unsigned long flags;
1627
1628 raw_spin_lock_irqsave(&iommu->register_lock, flags);
1629 iommu->gcmd |= DMA_GCMD_TE;
1630 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1631
1632 /* Make sure hardware complete it */
1633 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1634 readl, (sts & DMA_GSTS_TES), sts);
1635
1636 raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
1637 }
1638
1639 static void iommu_disable_translation(struct intel_iommu *iommu)
1640 {
1641 u32 sts;
1642 unsigned long flag;
1643
1644 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1645 iommu->gcmd &= ~DMA_GCMD_TE;
1646 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1647
1648 /* Make sure hardware complete it */
1649 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1650 readl, (!(sts & DMA_GSTS_TES)), sts);
1651
1652 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1653 }
1654
1655
1656 static int iommu_init_domains(struct intel_iommu *iommu)
1657 {
1658 u32 ndomains, nlongs;
1659 size_t size;
1660
1661 ndomains = cap_ndoms(iommu->cap);
1662 pr_debug("%s: Number of Domains supported <%d>\n",
1663 iommu->name, ndomains);
1664 nlongs = BITS_TO_LONGS(ndomains);
1665
1666 spin_lock_init(&iommu->lock);
1667
1668 iommu->domain_ids = kcalloc(nlongs, sizeof(unsigned long), GFP_KERNEL);
1669 if (!iommu->domain_ids) {
1670 pr_err("%s: Allocating domain id array failed\n",
1671 iommu->name);
1672 return -ENOMEM;
1673 }
1674
1675 size = (ALIGN(ndomains, 256) >> 8) * sizeof(struct dmar_domain **);
1676 iommu->domains = kzalloc(size, GFP_KERNEL);
1677
1678 if (iommu->domains) {
1679 size = 256 * sizeof(struct dmar_domain *);
1680 iommu->domains[0] = kzalloc(size, GFP_KERNEL);
1681 }
1682
1683 if (!iommu->domains || !iommu->domains[0]) {
1684 pr_err("%s: Allocating domain array failed\n",
1685 iommu->name);
1686 kfree(iommu->domain_ids);
1687 kfree(iommu->domains);
1688 iommu->domain_ids = NULL;
1689 iommu->domains = NULL;
1690 return -ENOMEM;
1691 }
1692
1693
1694
1695 /*
1696 * If Caching mode is set, then invalid translations are tagged
1697 * with domain-id 0, hence we need to pre-allocate it. We also
1698 * use domain-id 0 as a marker for non-allocated domain-id, so
1699 * make sure it is not used for a real domain.
1700 */
1701 set_bit(0, iommu->domain_ids);
1702
1703 return 0;
1704 }
1705
1706 static void disable_dmar_iommu(struct intel_iommu *iommu)
1707 {
1708 struct device_domain_info *info, *tmp;
1709 unsigned long flags;
1710
1711 if (!iommu->domains || !iommu->domain_ids)
1712 return;
1713
1714 spin_lock_irqsave(&device_domain_lock, flags);
1715 list_for_each_entry_safe(info, tmp, &device_domain_list, global) {
1716 struct dmar_domain *domain;
1717
1718 if (info->iommu != iommu)
1719 continue;
1720
1721 if (!info->dev || !info->domain)
1722 continue;
1723
1724 domain = info->domain;
1725
1726 dmar_remove_one_dev_info(domain, info->dev);
1727
1728 if (!domain_type_is_vm_or_si(domain))
1729 domain_exit(domain);
1730 }
1731 spin_unlock_irqrestore(&device_domain_lock, flags);
1732
1733 if (iommu->gcmd & DMA_GCMD_TE)
1734 iommu_disable_translation(iommu);
1735 }
1736
1737 static void free_dmar_iommu(struct intel_iommu *iommu)
1738 {
1739 if ((iommu->domains) && (iommu->domain_ids)) {
1740 int elems = ALIGN(cap_ndoms(iommu->cap), 256) >> 8;
1741 int i;
1742
1743 for (i = 0; i < elems; i++)
1744 kfree(iommu->domains[i]);
1745 kfree(iommu->domains);
1746 kfree(iommu->domain_ids);
1747 iommu->domains = NULL;
1748 iommu->domain_ids = NULL;
1749 }
1750
1751 g_iommus[iommu->seq_id] = NULL;
1752
1753 /* free context mapping */
1754 free_context_table(iommu);
1755
1756 #ifdef CONFIG_INTEL_IOMMU_SVM
1757 if (pasid_enabled(iommu)) {
1758 if (ecap_prs(iommu->ecap))
1759 intel_svm_finish_prq(iommu);
1760 intel_svm_free_pasid_tables(iommu);
1761 }
1762 #endif
1763 }
1764
1765 static struct dmar_domain *alloc_domain(int flags)
1766 {
1767 struct dmar_domain *domain;
1768
1769 domain = alloc_domain_mem();
1770 if (!domain)
1771 return NULL;
1772
1773 memset(domain, 0, sizeof(*domain));
1774 domain->nid = -1;
1775 domain->flags = flags;
1776 domain->has_iotlb_device = false;
1777 INIT_LIST_HEAD(&domain->devices);
1778
1779 return domain;
1780 }
1781
1782 /* Must be called with iommu->lock */
1783 static int domain_attach_iommu(struct dmar_domain *domain,
1784 struct intel_iommu *iommu)
1785 {
1786 unsigned long ndomains;
1787 int num;
1788
1789 assert_spin_locked(&device_domain_lock);
1790 assert_spin_locked(&iommu->lock);
1791
1792 domain->iommu_refcnt[iommu->seq_id] += 1;
1793 domain->iommu_count += 1;
1794 if (domain->iommu_refcnt[iommu->seq_id] == 1) {
1795 ndomains = cap_ndoms(iommu->cap);
1796 num = find_first_zero_bit(iommu->domain_ids, ndomains);
1797
1798 if (num >= ndomains) {
1799 pr_err("%s: No free domain ids\n", iommu->name);
1800 domain->iommu_refcnt[iommu->seq_id] -= 1;
1801 domain->iommu_count -= 1;
1802 return -ENOSPC;
1803 }
1804
1805 set_bit(num, iommu->domain_ids);
1806 set_iommu_domain(iommu, num, domain);
1807
1808 domain->iommu_did[iommu->seq_id] = num;
1809 domain->nid = iommu->node;
1810
1811 domain_update_iommu_cap(domain);
1812 }
1813
1814 return 0;
1815 }
1816
1817 static int domain_detach_iommu(struct dmar_domain *domain,
1818 struct intel_iommu *iommu)
1819 {
1820 int num, count = INT_MAX;
1821
1822 assert_spin_locked(&device_domain_lock);
1823 assert_spin_locked(&iommu->lock);
1824
1825 domain->iommu_refcnt[iommu->seq_id] -= 1;
1826 count = --domain->iommu_count;
1827 if (domain->iommu_refcnt[iommu->seq_id] == 0) {
1828 num = domain->iommu_did[iommu->seq_id];
1829 clear_bit(num, iommu->domain_ids);
1830 set_iommu_domain(iommu, num, NULL);
1831
1832 domain_update_iommu_cap(domain);
1833 domain->iommu_did[iommu->seq_id] = 0;
1834 }
1835
1836 return count;
1837 }
1838
1839 static struct iova_domain reserved_iova_list;
1840 static struct lock_class_key reserved_rbtree_key;
1841
1842 static int dmar_init_reserved_ranges(void)
1843 {
1844 struct pci_dev *pdev = NULL;
1845 struct iova *iova;
1846 int i;
1847
1848 init_iova_domain(&reserved_iova_list, VTD_PAGE_SIZE, IOVA_START_PFN,
1849 DMA_32BIT_PFN);
1850
1851 lockdep_set_class(&reserved_iova_list.iova_rbtree_lock,
1852 &reserved_rbtree_key);
1853
1854 /* IOAPIC ranges shouldn't be accessed by DMA */
1855 iova = reserve_iova(&reserved_iova_list, IOVA_PFN(IOAPIC_RANGE_START),
1856 IOVA_PFN(IOAPIC_RANGE_END));
1857 if (!iova) {
1858 pr_err("Reserve IOAPIC range failed\n");
1859 return -ENODEV;
1860 }
1861
1862 /* Reserve all PCI MMIO to avoid peer-to-peer access */
1863 for_each_pci_dev(pdev) {
1864 struct resource *r;
1865
1866 for (i = 0; i < PCI_NUM_RESOURCES; i++) {
1867 r = &pdev->resource[i];
1868 if (!r->flags || !(r->flags & IORESOURCE_MEM))
1869 continue;
1870 iova = reserve_iova(&reserved_iova_list,
1871 IOVA_PFN(r->start),
1872 IOVA_PFN(r->end));
1873 if (!iova) {
1874 pr_err("Reserve iova failed\n");
1875 return -ENODEV;
1876 }
1877 }
1878 }
1879 return 0;
1880 }
1881
1882 static void domain_reserve_special_ranges(struct dmar_domain *domain)
1883 {
1884 copy_reserved_iova(&reserved_iova_list, &domain->iovad);
1885 }
1886
1887 static inline int guestwidth_to_adjustwidth(int gaw)
1888 {
1889 int agaw;
1890 int r = (gaw - 12) % 9;
1891
1892 if (r == 0)
1893 agaw = gaw;
1894 else
1895 agaw = gaw + 9 - r;
1896 if (agaw > 64)
1897 agaw = 64;
1898 return agaw;
1899 }
1900
1901 static int domain_init(struct dmar_domain *domain, struct intel_iommu *iommu,
1902 int guest_width)
1903 {
1904 int adjust_width, agaw;
1905 unsigned long sagaw;
1906
1907 init_iova_domain(&domain->iovad, VTD_PAGE_SIZE, IOVA_START_PFN,
1908 DMA_32BIT_PFN);
1909 domain_reserve_special_ranges(domain);
1910
1911 /* calculate AGAW */
1912 if (guest_width > cap_mgaw(iommu->cap))
1913 guest_width = cap_mgaw(iommu->cap);
1914 domain->gaw = guest_width;
1915 adjust_width = guestwidth_to_adjustwidth(guest_width);
1916 agaw = width_to_agaw(adjust_width);
1917 sagaw = cap_sagaw(iommu->cap);
1918 if (!test_bit(agaw, &sagaw)) {
1919 /* hardware doesn't support it, choose a bigger one */
1920 pr_debug("Hardware doesn't support agaw %d\n", agaw);
1921 agaw = find_next_bit(&sagaw, 5, agaw);
1922 if (agaw >= 5)
1923 return -ENODEV;
1924 }
1925 domain->agaw = agaw;
1926
1927 if (ecap_coherent(iommu->ecap))
1928 domain->iommu_coherency = 1;
1929 else
1930 domain->iommu_coherency = 0;
1931
1932 if (ecap_sc_support(iommu->ecap))
1933 domain->iommu_snooping = 1;
1934 else
1935 domain->iommu_snooping = 0;
1936
1937 if (intel_iommu_superpage)
1938 domain->iommu_superpage = fls(cap_super_page_val(iommu->cap));
1939 else
1940 domain->iommu_superpage = 0;
1941
1942 domain->nid = iommu->node;
1943
1944 /* always allocate the top pgd */
1945 domain->pgd = (struct dma_pte *)alloc_pgtable_page(domain->nid);
1946 if (!domain->pgd)
1947 return -ENOMEM;
1948 __iommu_flush_cache(iommu, domain->pgd, PAGE_SIZE);
1949 return 0;
1950 }
1951
1952 static void domain_exit(struct dmar_domain *domain)
1953 {
1954 struct page *freelist = NULL;
1955
1956 /* Domain 0 is reserved, so dont process it */
1957 if (!domain)
1958 return;
1959
1960 /* Flush any lazy unmaps that may reference this domain */
1961 if (!intel_iommu_strict) {
1962 int cpu;
1963
1964 for_each_possible_cpu(cpu)
1965 flush_unmaps_timeout(cpu);
1966 }
1967
1968 /* Remove associated devices and clear attached or cached domains */
1969 rcu_read_lock();
1970 domain_remove_dev_info(domain);
1971 rcu_read_unlock();
1972
1973 /* destroy iovas */
1974 put_iova_domain(&domain->iovad);
1975
1976 freelist = domain_unmap(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
1977
1978 dma_free_pagelist(freelist);
1979
1980 free_domain_mem(domain);
1981 }
1982
1983 static int domain_context_mapping_one(struct dmar_domain *domain,
1984 struct intel_iommu *iommu,
1985 u8 bus, u8 devfn)
1986 {
1987 u16 did = domain->iommu_did[iommu->seq_id];
1988 int translation = CONTEXT_TT_MULTI_LEVEL;
1989 struct device_domain_info *info = NULL;
1990 struct context_entry *context;
1991 unsigned long flags;
1992 struct dma_pte *pgd;
1993 int ret, agaw;
1994
1995 WARN_ON(did == 0);
1996
1997 if (hw_pass_through && domain_type_is_si(domain))
1998 translation = CONTEXT_TT_PASS_THROUGH;
1999
2000 pr_debug("Set context mapping for %02x:%02x.%d\n",
2001 bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
2002
2003 BUG_ON(!domain->pgd);
2004
2005 spin_lock_irqsave(&device_domain_lock, flags);
2006 spin_lock(&iommu->lock);
2007
2008 ret = -ENOMEM;
2009 context = iommu_context_addr(iommu, bus, devfn, 1);
2010 if (!context)
2011 goto out_unlock;
2012
2013 ret = 0;
2014 if (context_present(context))
2015 goto out_unlock;
2016
2017 pgd = domain->pgd;
2018
2019 context_clear_entry(context);
2020 context_set_domain_id(context, did);
2021
2022 /*
2023 * Skip top levels of page tables for iommu which has less agaw
2024 * than default. Unnecessary for PT mode.
2025 */
2026 if (translation != CONTEXT_TT_PASS_THROUGH) {
2027 for (agaw = domain->agaw; agaw != iommu->agaw; agaw--) {
2028 ret = -ENOMEM;
2029 pgd = phys_to_virt(dma_pte_addr(pgd));
2030 if (!dma_pte_present(pgd))
2031 goto out_unlock;
2032 }
2033
2034 info = iommu_support_dev_iotlb(domain, iommu, bus, devfn);
2035 if (info && info->ats_supported)
2036 translation = CONTEXT_TT_DEV_IOTLB;
2037 else
2038 translation = CONTEXT_TT_MULTI_LEVEL;
2039
2040 context_set_address_root(context, virt_to_phys(pgd));
2041 context_set_address_width(context, iommu->agaw);
2042 } else {
2043 /*
2044 * In pass through mode, AW must be programmed to
2045 * indicate the largest AGAW value supported by
2046 * hardware. And ASR is ignored by hardware.
2047 */
2048 context_set_address_width(context, iommu->msagaw);
2049 }
2050
2051 context_set_translation_type(context, translation);
2052 context_set_fault_enable(context);
2053 context_set_present(context);
2054 domain_flush_cache(domain, context, sizeof(*context));
2055
2056 /*
2057 * It's a non-present to present mapping. If hardware doesn't cache
2058 * non-present entry we only need to flush the write-buffer. If the
2059 * _does_ cache non-present entries, then it does so in the special
2060 * domain #0, which we have to flush:
2061 */
2062 if (cap_caching_mode(iommu->cap)) {
2063 iommu->flush.flush_context(iommu, 0,
2064 (((u16)bus) << 8) | devfn,
2065 DMA_CCMD_MASK_NOBIT,
2066 DMA_CCMD_DEVICE_INVL);
2067 iommu->flush.flush_iotlb(iommu, did, 0, 0, DMA_TLB_DSI_FLUSH);
2068 } else {
2069 iommu_flush_write_buffer(iommu);
2070 }
2071 iommu_enable_dev_iotlb(info);
2072
2073 ret = 0;
2074
2075 out_unlock:
2076 spin_unlock(&iommu->lock);
2077 spin_unlock_irqrestore(&device_domain_lock, flags);
2078
2079 return ret;
2080 }
2081
2082 struct domain_context_mapping_data {
2083 struct dmar_domain *domain;
2084 struct intel_iommu *iommu;
2085 };
2086
2087 static int domain_context_mapping_cb(struct pci_dev *pdev,
2088 u16 alias, void *opaque)
2089 {
2090 struct domain_context_mapping_data *data = opaque;
2091
2092 return domain_context_mapping_one(data->domain, data->iommu,
2093 PCI_BUS_NUM(alias), alias & 0xff);
2094 }
2095
2096 static int
2097 domain_context_mapping(struct dmar_domain *domain, struct device *dev)
2098 {
2099 struct intel_iommu *iommu;
2100 u8 bus, devfn;
2101 struct domain_context_mapping_data data;
2102
2103 iommu = device_to_iommu(dev, &bus, &devfn);
2104 if (!iommu)
2105 return -ENODEV;
2106
2107 if (!dev_is_pci(dev))
2108 return domain_context_mapping_one(domain, iommu, bus, devfn);
2109
2110 data.domain = domain;
2111 data.iommu = iommu;
2112
2113 return pci_for_each_dma_alias(to_pci_dev(dev),
2114 &domain_context_mapping_cb, &data);
2115 }
2116
2117 static int domain_context_mapped_cb(struct pci_dev *pdev,
2118 u16 alias, void *opaque)
2119 {
2120 struct intel_iommu *iommu = opaque;
2121
2122 return !device_context_mapped(iommu, PCI_BUS_NUM(alias), alias & 0xff);
2123 }
2124
2125 static int domain_context_mapped(struct device *dev)
2126 {
2127 struct intel_iommu *iommu;
2128 u8 bus, devfn;
2129
2130 iommu = device_to_iommu(dev, &bus, &devfn);
2131 if (!iommu)
2132 return -ENODEV;
2133
2134 if (!dev_is_pci(dev))
2135 return device_context_mapped(iommu, bus, devfn);
2136
2137 return !pci_for_each_dma_alias(to_pci_dev(dev),
2138 domain_context_mapped_cb, iommu);
2139 }
2140
2141 /* Returns a number of VTD pages, but aligned to MM page size */
2142 static inline unsigned long aligned_nrpages(unsigned long host_addr,
2143 size_t size)
2144 {
2145 host_addr &= ~PAGE_MASK;
2146 return PAGE_ALIGN(host_addr + size) >> VTD_PAGE_SHIFT;
2147 }
2148
2149 /* Return largest possible superpage level for a given mapping */
2150 static inline int hardware_largepage_caps(struct dmar_domain *domain,
2151 unsigned long iov_pfn,
2152 unsigned long phy_pfn,
2153 unsigned long pages)
2154 {
2155 int support, level = 1;
2156 unsigned long pfnmerge;
2157
2158 support = domain->iommu_superpage;
2159
2160 /* To use a large page, the virtual *and* physical addresses
2161 must be aligned to 2MiB/1GiB/etc. Lower bits set in either
2162 of them will mean we have to use smaller pages. So just
2163 merge them and check both at once. */
2164 pfnmerge = iov_pfn | phy_pfn;
2165
2166 while (support && !(pfnmerge & ~VTD_STRIDE_MASK)) {
2167 pages >>= VTD_STRIDE_SHIFT;
2168 if (!pages)
2169 break;
2170 pfnmerge >>= VTD_STRIDE_SHIFT;
2171 level++;
2172 support--;
2173 }
2174 return level;
2175 }
2176
2177 static int __domain_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
2178 struct scatterlist *sg, unsigned long phys_pfn,
2179 unsigned long nr_pages, int prot)
2180 {
2181 struct dma_pte *first_pte = NULL, *pte = NULL;
2182 phys_addr_t uninitialized_var(pteval);
2183 unsigned long sg_res = 0;
2184 unsigned int largepage_lvl = 0;
2185 unsigned long lvl_pages = 0;
2186
2187 BUG_ON(!domain_pfn_supported(domain, iov_pfn + nr_pages - 1));
2188
2189 if ((prot & (DMA_PTE_READ|DMA_PTE_WRITE)) == 0)
2190 return -EINVAL;
2191
2192 prot &= DMA_PTE_READ | DMA_PTE_WRITE | DMA_PTE_SNP;
2193
2194 if (!sg) {
2195 sg_res = nr_pages;
2196 pteval = ((phys_addr_t)phys_pfn << VTD_PAGE_SHIFT) | prot;
2197 }
2198
2199 while (nr_pages > 0) {
2200 uint64_t tmp;
2201
2202 if (!sg_res) {
2203 sg_res = aligned_nrpages(sg->offset, sg->length);
2204 sg->dma_address = ((dma_addr_t)iov_pfn << VTD_PAGE_SHIFT) + sg->offset;
2205 sg->dma_length = sg->length;
2206 pteval = page_to_phys(sg_page(sg)) | prot;
2207 phys_pfn = pteval >> VTD_PAGE_SHIFT;
2208 }
2209
2210 if (!pte) {
2211 largepage_lvl = hardware_largepage_caps(domain, iov_pfn, phys_pfn, sg_res);
2212
2213 first_pte = pte = pfn_to_dma_pte(domain, iov_pfn, &largepage_lvl);
2214 if (!pte)
2215 return -ENOMEM;
2216 /* It is large page*/
2217 if (largepage_lvl > 1) {
2218 unsigned long nr_superpages, end_pfn;
2219
2220 pteval |= DMA_PTE_LARGE_PAGE;
2221 lvl_pages = lvl_to_nr_pages(largepage_lvl);
2222
2223 nr_superpages = sg_res / lvl_pages;
2224 end_pfn = iov_pfn + nr_superpages * lvl_pages - 1;
2225
2226 /*
2227 * Ensure that old small page tables are
2228 * removed to make room for superpage(s).
2229 */
2230 dma_pte_free_pagetable(domain, iov_pfn, end_pfn);
2231 } else {
2232 pteval &= ~(uint64_t)DMA_PTE_LARGE_PAGE;
2233 }
2234
2235 }
2236 /* We don't need lock here, nobody else
2237 * touches the iova range
2238 */
2239 tmp = cmpxchg64_local(&pte->val, 0ULL, pteval);
2240 if (tmp) {
2241 static int dumps = 5;
2242 pr_crit("ERROR: DMA PTE for vPFN 0x%lx already set (to %llx not %llx)\n",
2243 iov_pfn, tmp, (unsigned long long)pteval);
2244 if (dumps) {
2245 dumps--;
2246 debug_dma_dump_mappings(NULL);
2247 }
2248 WARN_ON(1);
2249 }
2250
2251 lvl_pages = lvl_to_nr_pages(largepage_lvl);
2252
2253 BUG_ON(nr_pages < lvl_pages);
2254 BUG_ON(sg_res < lvl_pages);
2255
2256 nr_pages -= lvl_pages;
2257 iov_pfn += lvl_pages;
2258 phys_pfn += lvl_pages;
2259 pteval += lvl_pages * VTD_PAGE_SIZE;
2260 sg_res -= lvl_pages;
2261
2262 /* If the next PTE would be the first in a new page, then we
2263 need to flush the cache on the entries we've just written.
2264 And then we'll need to recalculate 'pte', so clear it and
2265 let it get set again in the if (!pte) block above.
2266
2267 If we're done (!nr_pages) we need to flush the cache too.
2268
2269 Also if we've been setting superpages, we may need to
2270 recalculate 'pte' and switch back to smaller pages for the
2271 end of the mapping, if the trailing size is not enough to
2272 use another superpage (i.e. sg_res < lvl_pages). */
2273 pte++;
2274 if (!nr_pages || first_pte_in_page(pte) ||
2275 (largepage_lvl > 1 && sg_res < lvl_pages)) {
2276 domain_flush_cache(domain, first_pte,
2277 (void *)pte - (void *)first_pte);
2278 pte = NULL;
2279 }
2280
2281 if (!sg_res && nr_pages)
2282 sg = sg_next(sg);
2283 }
2284 return 0;
2285 }
2286
2287 static inline int domain_sg_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
2288 struct scatterlist *sg, unsigned long nr_pages,
2289 int prot)
2290 {
2291 return __domain_mapping(domain, iov_pfn, sg, 0, nr_pages, prot);
2292 }
2293
2294 static inline int domain_pfn_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
2295 unsigned long phys_pfn, unsigned long nr_pages,
2296 int prot)
2297 {
2298 return __domain_mapping(domain, iov_pfn, NULL, phys_pfn, nr_pages, prot);
2299 }
2300
2301 static void domain_context_clear_one(struct intel_iommu *iommu, u8 bus, u8 devfn)
2302 {
2303 if (!iommu)
2304 return;
2305
2306 clear_context_table(iommu, bus, devfn);
2307 iommu->flush.flush_context(iommu, 0, 0, 0,
2308 DMA_CCMD_GLOBAL_INVL);
2309 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
2310 }
2311
2312 static inline void unlink_domain_info(struct device_domain_info *info)
2313 {
2314 assert_spin_locked(&device_domain_lock);
2315 list_del(&info->link);
2316 list_del(&info->global);
2317 if (info->dev)
2318 info->dev->archdata.iommu = NULL;
2319 }
2320
2321 static void domain_remove_dev_info(struct dmar_domain *domain)
2322 {
2323 struct device_domain_info *info, *tmp;
2324 unsigned long flags;
2325
2326 spin_lock_irqsave(&device_domain_lock, flags);
2327 list_for_each_entry_safe(info, tmp, &domain->devices, link)
2328 __dmar_remove_one_dev_info(info);
2329 spin_unlock_irqrestore(&device_domain_lock, flags);
2330 }
2331
2332 /*
2333 * find_domain
2334 * Note: we use struct device->archdata.iommu stores the info
2335 */
2336 static struct dmar_domain *find_domain(struct device *dev)
2337 {
2338 struct device_domain_info *info;
2339
2340 /* No lock here, assumes no domain exit in normal case */
2341 info = dev->archdata.iommu;
2342 if (info)
2343 return info->domain;
2344 return NULL;
2345 }
2346
2347 static inline struct device_domain_info *
2348 dmar_search_domain_by_dev_info(int segment, int bus, int devfn)
2349 {
2350 struct device_domain_info *info;
2351
2352 list_for_each_entry(info, &device_domain_list, global)
2353 if (info->iommu->segment == segment && info->bus == bus &&
2354 info->devfn == devfn)
2355 return info;
2356
2357 return NULL;
2358 }
2359
2360 static struct dmar_domain *dmar_insert_one_dev_info(struct intel_iommu *iommu,
2361 int bus, int devfn,
2362 struct device *dev,
2363 struct dmar_domain *domain)
2364 {
2365 struct dmar_domain *found = NULL;
2366 struct device_domain_info *info;
2367 unsigned long flags;
2368 int ret;
2369
2370 info = alloc_devinfo_mem();
2371 if (!info)
2372 return NULL;
2373
2374 info->bus = bus;
2375 info->devfn = devfn;
2376 info->ats_supported = info->pasid_supported = info->pri_supported = 0;
2377 info->ats_enabled = info->pasid_enabled = info->pri_enabled = 0;
2378 info->ats_qdep = 0;
2379 info->dev = dev;
2380 info->domain = domain;
2381 info->iommu = iommu;
2382
2383 if (dev && dev_is_pci(dev)) {
2384 struct pci_dev *pdev = to_pci_dev(info->dev);
2385
2386 if (ecap_dev_iotlb_support(iommu->ecap) &&
2387 pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_ATS) &&
2388 dmar_find_matched_atsr_unit(pdev))
2389 info->ats_supported = 1;
2390
2391 if (ecs_enabled(iommu)) {
2392 if (pasid_enabled(iommu)) {
2393 int features = pci_pasid_features(pdev);
2394 if (features >= 0)
2395 info->pasid_supported = features | 1;
2396 }
2397
2398 if (info->ats_supported && ecap_prs(iommu->ecap) &&
2399 pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PRI))
2400 info->pri_supported = 1;
2401 }
2402 }
2403
2404 spin_lock_irqsave(&device_domain_lock, flags);
2405 if (dev)
2406 found = find_domain(dev);
2407
2408 if (!found) {
2409 struct device_domain_info *info2;
2410 info2 = dmar_search_domain_by_dev_info(iommu->segment, bus, devfn);
2411 if (info2) {
2412 found = info2->domain;
2413 info2->dev = dev;
2414 }
2415 }
2416
2417 if (found) {
2418 spin_unlock_irqrestore(&device_domain_lock, flags);
2419 free_devinfo_mem(info);
2420 /* Caller must free the original domain */
2421 return found;
2422 }
2423
2424 spin_lock(&iommu->lock);
2425 ret = domain_attach_iommu(domain, iommu);
2426 spin_unlock(&iommu->lock);
2427
2428 if (ret) {
2429 spin_unlock_irqrestore(&device_domain_lock, flags);
2430 free_devinfo_mem(info);
2431 return NULL;
2432 }
2433
2434 list_add(&info->link, &domain->devices);
2435 list_add(&info->global, &device_domain_list);
2436 if (dev)
2437 dev->archdata.iommu = info;
2438 spin_unlock_irqrestore(&device_domain_lock, flags);
2439
2440 if (dev && domain_context_mapping(domain, dev)) {
2441 pr_err("Domain context map for %s failed\n", dev_name(dev));
2442 dmar_remove_one_dev_info(domain, dev);
2443 return NULL;
2444 }
2445
2446 return domain;
2447 }
2448
2449 static int get_last_alias(struct pci_dev *pdev, u16 alias, void *opaque)
2450 {
2451 *(u16 *)opaque = alias;
2452 return 0;
2453 }
2454
2455 static struct dmar_domain *find_or_alloc_domain(struct device *dev, int gaw)
2456 {
2457 struct device_domain_info *info = NULL;
2458 struct dmar_domain *domain = NULL;
2459 struct intel_iommu *iommu;
2460 u16 req_id, dma_alias;
2461 unsigned long flags;
2462 u8 bus, devfn;
2463
2464 iommu = device_to_iommu(dev, &bus, &devfn);
2465 if (!iommu)
2466 return NULL;
2467
2468 req_id = ((u16)bus << 8) | devfn;
2469
2470 if (dev_is_pci(dev)) {
2471 struct pci_dev *pdev = to_pci_dev(dev);
2472
2473 pci_for_each_dma_alias(pdev, get_last_alias, &dma_alias);
2474
2475 spin_lock_irqsave(&device_domain_lock, flags);
2476 info = dmar_search_domain_by_dev_info(pci_domain_nr(pdev->bus),
2477 PCI_BUS_NUM(dma_alias),
2478 dma_alias & 0xff);
2479 if (info) {
2480 iommu = info->iommu;
2481 domain = info->domain;
2482 }
2483 spin_unlock_irqrestore(&device_domain_lock, flags);
2484
2485 /* DMA alias already has a domain, use it */
2486 if (info)
2487 goto out;
2488 }
2489
2490 /* Allocate and initialize new domain for the device */
2491 domain = alloc_domain(0);
2492 if (!domain)
2493 return NULL;
2494 if (domain_init(domain, iommu, gaw)) {
2495 domain_exit(domain);
2496 return NULL;
2497 }
2498
2499 out:
2500
2501 return domain;
2502 }
2503
2504 static struct dmar_domain *set_domain_for_dev(struct device *dev,
2505 struct dmar_domain *domain)
2506 {
2507 struct intel_iommu *iommu;
2508 struct dmar_domain *tmp;
2509 u16 req_id, dma_alias;
2510 u8 bus, devfn;
2511
2512 iommu = device_to_iommu(dev, &bus, &devfn);
2513 if (!iommu)
2514 return NULL;
2515
2516 req_id = ((u16)bus << 8) | devfn;
2517
2518 if (dev_is_pci(dev)) {
2519 struct pci_dev *pdev = to_pci_dev(dev);
2520
2521 pci_for_each_dma_alias(pdev, get_last_alias, &dma_alias);
2522
2523 /* register PCI DMA alias device */
2524 if (req_id != dma_alias) {
2525 tmp = dmar_insert_one_dev_info(iommu, PCI_BUS_NUM(dma_alias),
2526 dma_alias & 0xff, NULL, domain);
2527
2528 if (!tmp || tmp != domain)
2529 return tmp;
2530 }
2531 }
2532
2533 tmp = dmar_insert_one_dev_info(iommu, bus, devfn, dev, domain);
2534 if (!tmp || tmp != domain)
2535 return tmp;
2536
2537 return domain;
2538 }
2539
2540 static struct dmar_domain *get_domain_for_dev(struct device *dev, int gaw)
2541 {
2542 struct dmar_domain *domain, *tmp;
2543
2544 domain = find_domain(dev);
2545 if (domain)
2546 goto out;
2547
2548 domain = find_or_alloc_domain(dev, gaw);
2549 if (!domain)
2550 goto out;
2551
2552 tmp = set_domain_for_dev(dev, domain);
2553 if (!tmp || domain != tmp) {
2554 domain_exit(domain);
2555 domain = tmp;
2556 }
2557
2558 out:
2559
2560 return domain;
2561 }
2562
2563 static int iommu_domain_identity_map(struct dmar_domain *domain,
2564 unsigned long long start,
2565 unsigned long long end)
2566 {
2567 unsigned long first_vpfn = start >> VTD_PAGE_SHIFT;
2568 unsigned long last_vpfn = end >> VTD_PAGE_SHIFT;
2569
2570 if (!reserve_iova(&domain->iovad, dma_to_mm_pfn(first_vpfn),
2571 dma_to_mm_pfn(last_vpfn))) {
2572 pr_err("Reserving iova failed\n");
2573 return -ENOMEM;
2574 }
2575
2576 pr_debug("Mapping reserved region %llx-%llx\n", start, end);
2577 /*
2578 * RMRR range might have overlap with physical memory range,
2579 * clear it first
2580 */
2581 dma_pte_clear_range(domain, first_vpfn, last_vpfn);
2582
2583 return domain_pfn_mapping(domain, first_vpfn, first_vpfn,
2584 last_vpfn - first_vpfn + 1,
2585 DMA_PTE_READ|DMA_PTE_WRITE);
2586 }
2587
2588 static int domain_prepare_identity_map(struct device *dev,
2589 struct dmar_domain *domain,
2590 unsigned long long start,
2591 unsigned long long end)
2592 {
2593 /* For _hardware_ passthrough, don't bother. But for software
2594 passthrough, we do it anyway -- it may indicate a memory
2595 range which is reserved in E820, so which didn't get set
2596 up to start with in si_domain */
2597 if (domain == si_domain && hw_pass_through) {
2598 pr_warn("Ignoring identity map for HW passthrough device %s [0x%Lx - 0x%Lx]\n",
2599 dev_name(dev), start, end);
2600 return 0;
2601 }
2602
2603 pr_info("Setting identity map for device %s [0x%Lx - 0x%Lx]\n",
2604 dev_name(dev), start, end);
2605
2606 if (end < start) {
2607 WARN(1, "Your BIOS is broken; RMRR ends before it starts!\n"
2608 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
2609 dmi_get_system_info(DMI_BIOS_VENDOR),
2610 dmi_get_system_info(DMI_BIOS_VERSION),
2611 dmi_get_system_info(DMI_PRODUCT_VERSION));
2612 return -EIO;
2613 }
2614
2615 if (end >> agaw_to_width(domain->agaw)) {
2616 WARN(1, "Your BIOS is broken; RMRR exceeds permitted address width (%d bits)\n"
2617 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
2618 agaw_to_width(domain->agaw),
2619 dmi_get_system_info(DMI_BIOS_VENDOR),
2620 dmi_get_system_info(DMI_BIOS_VERSION),
2621 dmi_get_system_info(DMI_PRODUCT_VERSION));
2622 return -EIO;
2623 }
2624
2625 return iommu_domain_identity_map(domain, start, end);
2626 }
2627
2628 static int iommu_prepare_identity_map(struct device *dev,
2629 unsigned long long start,
2630 unsigned long long end)
2631 {
2632 struct dmar_domain *domain;
2633 int ret;
2634
2635 domain = get_domain_for_dev(dev, DEFAULT_DOMAIN_ADDRESS_WIDTH);
2636 if (!domain)
2637 return -ENOMEM;
2638
2639 ret = domain_prepare_identity_map(dev, domain, start, end);
2640 if (ret)
2641 domain_exit(domain);
2642
2643 return ret;
2644 }
2645
2646 static inline int iommu_prepare_rmrr_dev(struct dmar_rmrr_unit *rmrr,
2647 struct device *dev)
2648 {
2649 if (dev->archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO)
2650 return 0;
2651 return iommu_prepare_identity_map(dev, rmrr->base_address,
2652 rmrr->end_address);
2653 }
2654
2655 #ifdef CONFIG_INTEL_IOMMU_FLOPPY_WA
2656 static inline void iommu_prepare_isa(void)
2657 {
2658 struct pci_dev *pdev;
2659 int ret;
2660
2661 pdev = pci_get_class(PCI_CLASS_BRIDGE_ISA << 8, NULL);
2662 if (!pdev)
2663 return;
2664
2665 pr_info("Prepare 0-16MiB unity mapping for LPC\n");
2666 ret = iommu_prepare_identity_map(&pdev->dev, 0, 16*1024*1024 - 1);
2667
2668 if (ret)
2669 pr_err("Failed to create 0-16MiB identity map - floppy might not work\n");
2670
2671 pci_dev_put(pdev);
2672 }
2673 #else
2674 static inline void iommu_prepare_isa(void)
2675 {
2676 return;
2677 }
2678 #endif /* !CONFIG_INTEL_IOMMU_FLPY_WA */
2679
2680 static int md_domain_init(struct dmar_domain *domain, int guest_width);
2681
2682 static int __init si_domain_init(int hw)
2683 {
2684 int nid, ret = 0;
2685
2686 si_domain = alloc_domain(DOMAIN_FLAG_STATIC_IDENTITY);
2687 if (!si_domain)
2688 return -EFAULT;
2689
2690 if (md_domain_init(si_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
2691 domain_exit(si_domain);
2692 return -EFAULT;
2693 }
2694
2695 pr_debug("Identity mapping domain allocated\n");
2696
2697 if (hw)
2698 return 0;
2699
2700 for_each_online_node(nid) {
2701 unsigned long start_pfn, end_pfn;
2702 int i;
2703
2704 for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, NULL) {
2705 ret = iommu_domain_identity_map(si_domain,
2706 PFN_PHYS(start_pfn), PFN_PHYS(end_pfn));
2707 if (ret)
2708 return ret;
2709 }
2710 }
2711
2712 return 0;
2713 }
2714
2715 static int identity_mapping(struct device *dev)
2716 {
2717 struct device_domain_info *info;
2718
2719 if (likely(!iommu_identity_mapping))
2720 return 0;
2721
2722 info = dev->archdata.iommu;
2723 if (info && info != DUMMY_DEVICE_DOMAIN_INFO)
2724 return (info->domain == si_domain);
2725
2726 return 0;
2727 }
2728
2729 static int domain_add_dev_info(struct dmar_domain *domain, struct device *dev)
2730 {
2731 struct dmar_domain *ndomain;
2732 struct intel_iommu *iommu;
2733 u8 bus, devfn;
2734
2735 iommu = device_to_iommu(dev, &bus, &devfn);
2736 if (!iommu)
2737 return -ENODEV;
2738
2739 ndomain = dmar_insert_one_dev_info(iommu, bus, devfn, dev, domain);
2740 if (ndomain != domain)
2741 return -EBUSY;
2742
2743 return 0;
2744 }
2745
2746 static bool device_has_rmrr(struct device *dev)
2747 {
2748 struct dmar_rmrr_unit *rmrr;
2749 struct device *tmp;
2750 int i;
2751
2752 rcu_read_lock();
2753 for_each_rmrr_units(rmrr) {
2754 /*
2755 * Return TRUE if this RMRR contains the device that
2756 * is passed in.
2757 */
2758 for_each_active_dev_scope(rmrr->devices,
2759 rmrr->devices_cnt, i, tmp)
2760 if (tmp == dev) {
2761 rcu_read_unlock();
2762 return true;
2763 }
2764 }
2765 rcu_read_unlock();
2766 return false;
2767 }
2768
2769 /*
2770 * There are a couple cases where we need to restrict the functionality of
2771 * devices associated with RMRRs. The first is when evaluating a device for
2772 * identity mapping because problems exist when devices are moved in and out
2773 * of domains and their respective RMRR information is lost. This means that
2774 * a device with associated RMRRs will never be in a "passthrough" domain.
2775 * The second is use of the device through the IOMMU API. This interface
2776 * expects to have full control of the IOVA space for the device. We cannot
2777 * satisfy both the requirement that RMRR access is maintained and have an
2778 * unencumbered IOVA space. We also have no ability to quiesce the device's
2779 * use of the RMRR space or even inform the IOMMU API user of the restriction.
2780 * We therefore prevent devices associated with an RMRR from participating in
2781 * the IOMMU API, which eliminates them from device assignment.
2782 *
2783 * In both cases we assume that PCI USB devices with RMRRs have them largely
2784 * for historical reasons and that the RMRR space is not actively used post
2785 * boot. This exclusion may change if vendors begin to abuse it.
2786 *
2787 * The same exception is made for graphics devices, with the requirement that
2788 * any use of the RMRR regions will be torn down before assigning the device
2789 * to a guest.
2790 */
2791 static bool device_is_rmrr_locked(struct device *dev)
2792 {
2793 if (!device_has_rmrr(dev))
2794 return false;
2795
2796 if (dev_is_pci(dev)) {
2797 struct pci_dev *pdev = to_pci_dev(dev);
2798
2799 if (IS_USB_DEVICE(pdev) || IS_GFX_DEVICE(pdev))
2800 return false;
2801 }
2802
2803 return true;
2804 }
2805
2806 static int iommu_should_identity_map(struct device *dev, int startup)
2807 {
2808
2809 if (dev_is_pci(dev)) {
2810 struct pci_dev *pdev = to_pci_dev(dev);
2811
2812 if (device_is_rmrr_locked(dev))
2813 return 0;
2814
2815 if ((iommu_identity_mapping & IDENTMAP_AZALIA) && IS_AZALIA(pdev))
2816 return 1;
2817
2818 if ((iommu_identity_mapping & IDENTMAP_GFX) && IS_GFX_DEVICE(pdev))
2819 return 1;
2820
2821 if (!(iommu_identity_mapping & IDENTMAP_ALL))
2822 return 0;
2823
2824 /*
2825 * We want to start off with all devices in the 1:1 domain, and
2826 * take them out later if we find they can't access all of memory.
2827 *
2828 * However, we can't do this for PCI devices behind bridges,
2829 * because all PCI devices behind the same bridge will end up
2830 * with the same source-id on their transactions.
2831 *
2832 * Practically speaking, we can't change things around for these
2833 * devices at run-time, because we can't be sure there'll be no
2834 * DMA transactions in flight for any of their siblings.
2835 *
2836 * So PCI devices (unless they're on the root bus) as well as
2837 * their parent PCI-PCI or PCIe-PCI bridges must be left _out_ of
2838 * the 1:1 domain, just in _case_ one of their siblings turns out
2839 * not to be able to map all of memory.
2840 */
2841 if (!pci_is_pcie(pdev)) {
2842 if (!pci_is_root_bus(pdev->bus))
2843 return 0;
2844 if (pdev->class >> 8 == PCI_CLASS_BRIDGE_PCI)
2845 return 0;
2846 } else if (pci_pcie_type(pdev) == PCI_EXP_TYPE_PCI_BRIDGE)
2847 return 0;
2848 } else {
2849 if (device_has_rmrr(dev))
2850 return 0;
2851 }
2852
2853 /*
2854 * At boot time, we don't yet know if devices will be 64-bit capable.
2855 * Assume that they will — if they turn out not to be, then we can
2856 * take them out of the 1:1 domain later.
2857 */
2858 if (!startup) {
2859 /*
2860 * If the device's dma_mask is less than the system's memory
2861 * size then this is not a candidate for identity mapping.
2862 */
2863 u64 dma_mask = *dev->dma_mask;
2864
2865 if (dev->coherent_dma_mask &&
2866 dev->coherent_dma_mask < dma_mask)
2867 dma_mask = dev->coherent_dma_mask;
2868
2869 return dma_mask >= dma_get_required_mask(dev);
2870 }
2871
2872 return 1;
2873 }
2874
2875 static int __init dev_prepare_static_identity_mapping(struct device *dev, int hw)
2876 {
2877 int ret;
2878
2879 if (!iommu_should_identity_map(dev, 1))
2880 return 0;
2881
2882 ret = domain_add_dev_info(si_domain, dev);
2883 if (!ret)
2884 pr_info("%s identity mapping for device %s\n",
2885 hw ? "Hardware" : "Software", dev_name(dev));
2886 else if (ret == -ENODEV)
2887 /* device not associated with an iommu */
2888 ret = 0;
2889
2890 return ret;
2891 }
2892
2893
2894 static int __init iommu_prepare_static_identity_mapping(int hw)
2895 {
2896 struct pci_dev *pdev = NULL;
2897 struct dmar_drhd_unit *drhd;
2898 struct intel_iommu *iommu;
2899 struct device *dev;
2900 int i;
2901 int ret = 0;
2902
2903 for_each_pci_dev(pdev) {
2904 ret = dev_prepare_static_identity_mapping(&pdev->dev, hw);
2905 if (ret)
2906 return ret;
2907 }
2908
2909 for_each_active_iommu(iommu, drhd)
2910 for_each_active_dev_scope(drhd->devices, drhd->devices_cnt, i, dev) {
2911 struct acpi_device_physical_node *pn;
2912 struct acpi_device *adev;
2913
2914 if (dev->bus != &acpi_bus_type)
2915 continue;
2916
2917 adev= to_acpi_device(dev);
2918 mutex_lock(&adev->physical_node_lock);
2919 list_for_each_entry(pn, &adev->physical_node_list, node) {
2920 ret = dev_prepare_static_identity_mapping(pn->dev, hw);
2921 if (ret)
2922 break;
2923 }
2924 mutex_unlock(&adev->physical_node_lock);
2925 if (ret)
2926 return ret;
2927 }
2928
2929 return 0;
2930 }
2931
2932 static void intel_iommu_init_qi(struct intel_iommu *iommu)
2933 {
2934 /*
2935 * Start from the sane iommu hardware state.
2936 * If the queued invalidation is already initialized by us
2937 * (for example, while enabling interrupt-remapping) then
2938 * we got the things already rolling from a sane state.
2939 */
2940 if (!iommu->qi) {
2941 /*
2942 * Clear any previous faults.
2943 */
2944 dmar_fault(-1, iommu);
2945 /*
2946 * Disable queued invalidation if supported and already enabled
2947 * before OS handover.
2948 */
2949 dmar_disable_qi(iommu);
2950 }
2951
2952 if (dmar_enable_qi(iommu)) {
2953 /*
2954 * Queued Invalidate not enabled, use Register Based Invalidate
2955 */
2956 iommu->flush.flush_context = __iommu_flush_context;
2957 iommu->flush.flush_iotlb = __iommu_flush_iotlb;
2958 pr_info("%s: Using Register based invalidation\n",
2959 iommu->name);
2960 } else {
2961 iommu->flush.flush_context = qi_flush_context;
2962 iommu->flush.flush_iotlb = qi_flush_iotlb;
2963 pr_info("%s: Using Queued invalidation\n", iommu->name);
2964 }
2965 }
2966
2967 static int copy_context_table(struct intel_iommu *iommu,
2968 struct root_entry *old_re,
2969 struct context_entry **tbl,
2970 int bus, bool ext)
2971 {
2972 int tbl_idx, pos = 0, idx, devfn, ret = 0, did;
2973 struct context_entry *new_ce = NULL, ce;
2974 struct context_entry *old_ce = NULL;
2975 struct root_entry re;
2976 phys_addr_t old_ce_phys;
2977
2978 tbl_idx = ext ? bus * 2 : bus;
2979 memcpy(&re, old_re, sizeof(re));
2980
2981 for (devfn = 0; devfn < 256; devfn++) {
2982 /* First calculate the correct index */
2983 idx = (ext ? devfn * 2 : devfn) % 256;
2984
2985 if (idx == 0) {
2986 /* First save what we may have and clean up */
2987 if (new_ce) {
2988 tbl[tbl_idx] = new_ce;
2989 __iommu_flush_cache(iommu, new_ce,
2990 VTD_PAGE_SIZE);
2991 pos = 1;
2992 }
2993
2994 if (old_ce)
2995 iounmap(old_ce);
2996
2997 ret = 0;
2998 if (devfn < 0x80)
2999 old_ce_phys = root_entry_lctp(&re);
3000 else
3001 old_ce_phys = root_entry_uctp(&re);
3002
3003 if (!old_ce_phys) {
3004 if (ext && devfn == 0) {
3005 /* No LCTP, try UCTP */
3006 devfn = 0x7f;
3007 continue;
3008 } else {
3009 goto out;
3010 }
3011 }
3012
3013 ret = -ENOMEM;
3014 old_ce = memremap(old_ce_phys, PAGE_SIZE,
3015 MEMREMAP_WB);
3016 if (!old_ce)
3017 goto out;
3018
3019 new_ce = alloc_pgtable_page(iommu->node);
3020 if (!new_ce)
3021 goto out_unmap;
3022
3023 ret = 0;
3024 }
3025
3026 /* Now copy the context entry */
3027 memcpy(&ce, old_ce + idx, sizeof(ce));
3028
3029 if (!__context_present(&ce))
3030 continue;
3031
3032 did = context_domain_id(&ce);
3033 if (did >= 0 && did < cap_ndoms(iommu->cap))
3034 set_bit(did, iommu->domain_ids);
3035
3036 /*
3037 * We need a marker for copied context entries. This
3038 * marker needs to work for the old format as well as
3039 * for extended context entries.
3040 *
3041 * Bit 67 of the context entry is used. In the old
3042 * format this bit is available to software, in the
3043 * extended format it is the PGE bit, but PGE is ignored
3044 * by HW if PASIDs are disabled (and thus still
3045 * available).
3046 *
3047 * So disable PASIDs first and then mark the entry
3048 * copied. This means that we don't copy PASID
3049 * translations from the old kernel, but this is fine as
3050 * faults there are not fatal.
3051 */
3052 context_clear_pasid_enable(&ce);
3053 context_set_copied(&ce);
3054
3055 new_ce[idx] = ce;
3056 }
3057
3058 tbl[tbl_idx + pos] = new_ce;
3059
3060 __iommu_flush_cache(iommu, new_ce, VTD_PAGE_SIZE);
3061
3062 out_unmap:
3063 memunmap(old_ce);
3064
3065 out:
3066 return ret;
3067 }
3068
3069 static int copy_translation_tables(struct intel_iommu *iommu)
3070 {
3071 struct context_entry **ctxt_tbls;
3072 struct root_entry *old_rt;
3073 phys_addr_t old_rt_phys;
3074 int ctxt_table_entries;
3075 unsigned long flags;
3076 u64 rtaddr_reg;
3077 int bus, ret;
3078 bool new_ext, ext;
3079
3080 rtaddr_reg = dmar_readq(iommu->reg + DMAR_RTADDR_REG);
3081 ext = !!(rtaddr_reg & DMA_RTADDR_RTT);
3082 new_ext = !!ecap_ecs(iommu->ecap);
3083
3084 /*
3085 * The RTT bit can only be changed when translation is disabled,
3086 * but disabling translation means to open a window for data
3087 * corruption. So bail out and don't copy anything if we would
3088 * have to change the bit.
3089 */
3090 if (new_ext != ext)
3091 return -EINVAL;
3092
3093 old_rt_phys = rtaddr_reg & VTD_PAGE_MASK;
3094 if (!old_rt_phys)
3095 return -EINVAL;
3096
3097 old_rt = memremap(old_rt_phys, PAGE_SIZE, MEMREMAP_WB);
3098 if (!old_rt)
3099 return -ENOMEM;
3100
3101 /* This is too big for the stack - allocate it from slab */
3102 ctxt_table_entries = ext ? 512 : 256;
3103 ret = -ENOMEM;
3104 ctxt_tbls = kzalloc(ctxt_table_entries * sizeof(void *), GFP_KERNEL);
3105 if (!ctxt_tbls)
3106 goto out_unmap;
3107
3108 for (bus = 0; bus < 256; bus++) {
3109 ret = copy_context_table(iommu, &old_rt[bus],
3110 ctxt_tbls, bus, ext);
3111 if (ret) {
3112 pr_err("%s: Failed to copy context table for bus %d\n",
3113 iommu->name, bus);
3114 continue;
3115 }
3116 }
3117
3118 spin_lock_irqsave(&iommu->lock, flags);
3119
3120 /* Context tables are copied, now write them to the root_entry table */
3121 for (bus = 0; bus < 256; bus++) {
3122 int idx = ext ? bus * 2 : bus;
3123 u64 val;
3124
3125 if (ctxt_tbls[idx]) {
3126 val = virt_to_phys(ctxt_tbls[idx]) | 1;
3127 iommu->root_entry[bus].lo = val;
3128 }
3129
3130 if (!ext || !ctxt_tbls[idx + 1])
3131 continue;
3132
3133 val = virt_to_phys(ctxt_tbls[idx + 1]) | 1;
3134 iommu->root_entry[bus].hi = val;
3135 }
3136
3137 spin_unlock_irqrestore(&iommu->lock, flags);
3138
3139 kfree(ctxt_tbls);
3140
3141 __iommu_flush_cache(iommu, iommu->root_entry, PAGE_SIZE);
3142
3143 ret = 0;
3144
3145 out_unmap:
3146 memunmap(old_rt);
3147
3148 return ret;
3149 }
3150
3151 static int __init init_dmars(void)
3152 {
3153 struct dmar_drhd_unit *drhd;
3154 struct dmar_rmrr_unit *rmrr;
3155 bool copied_tables = false;
3156 struct device *dev;
3157 struct intel_iommu *iommu;
3158 int i, ret, cpu;
3159
3160 /*
3161 * for each drhd
3162 * allocate root
3163 * initialize and program root entry to not present
3164 * endfor
3165 */
3166 for_each_drhd_unit(drhd) {
3167 /*
3168 * lock not needed as this is only incremented in the single
3169 * threaded kernel __init code path all other access are read
3170 * only
3171 */
3172 if (g_num_of_iommus < DMAR_UNITS_SUPPORTED) {
3173 g_num_of_iommus++;
3174 continue;
3175 }
3176 pr_err_once("Exceeded %d IOMMUs\n", DMAR_UNITS_SUPPORTED);
3177 }
3178
3179 /* Preallocate enough resources for IOMMU hot-addition */
3180 if (g_num_of_iommus < DMAR_UNITS_SUPPORTED)
3181 g_num_of_iommus = DMAR_UNITS_SUPPORTED;
3182
3183 g_iommus = kcalloc(g_num_of_iommus, sizeof(struct intel_iommu *),
3184 GFP_KERNEL);
3185 if (!g_iommus) {
3186 pr_err("Allocating global iommu array failed\n");
3187 ret = -ENOMEM;
3188 goto error;
3189 }
3190
3191 for_each_possible_cpu(cpu) {
3192 struct deferred_flush_data *dfd = per_cpu_ptr(&deferred_flush,
3193 cpu);
3194
3195 dfd->tables = kzalloc(g_num_of_iommus *
3196 sizeof(struct deferred_flush_table),
3197 GFP_KERNEL);
3198 if (!dfd->tables) {
3199 ret = -ENOMEM;
3200 goto free_g_iommus;
3201 }
3202
3203 spin_lock_init(&dfd->lock);
3204 setup_timer(&dfd->timer, flush_unmaps_timeout, cpu);
3205 }
3206
3207 for_each_active_iommu(iommu, drhd) {
3208 g_iommus[iommu->seq_id] = iommu;
3209
3210 intel_iommu_init_qi(iommu);
3211
3212 ret = iommu_init_domains(iommu);
3213 if (ret)
3214 goto free_iommu;
3215
3216 init_translation_status(iommu);
3217
3218 if (translation_pre_enabled(iommu) && !is_kdump_kernel()) {
3219 iommu_disable_translation(iommu);
3220 clear_translation_pre_enabled(iommu);
3221 pr_warn("Translation was enabled for %s but we are not in kdump mode\n",
3222 iommu->name);
3223 }
3224
3225 /*
3226 * TBD:
3227 * we could share the same root & context tables
3228 * among all IOMMU's. Need to Split it later.
3229 */
3230 ret = iommu_alloc_root_entry(iommu);
3231 if (ret)
3232 goto free_iommu;
3233
3234 if (translation_pre_enabled(iommu)) {
3235 pr_info("Translation already enabled - trying to copy translation structures\n");
3236
3237 ret = copy_translation_tables(iommu);
3238 if (ret) {
3239 /*
3240 * We found the IOMMU with translation
3241 * enabled - but failed to copy over the
3242 * old root-entry table. Try to proceed
3243 * by disabling translation now and
3244 * allocating a clean root-entry table.
3245 * This might cause DMAR faults, but
3246 * probably the dump will still succeed.
3247 */
3248 pr_err("Failed to copy translation tables from previous kernel for %s\n",
3249 iommu->name);
3250 iommu_disable_translation(iommu);
3251 clear_translation_pre_enabled(iommu);
3252 } else {
3253 pr_info("Copied translation tables from previous kernel for %s\n",
3254 iommu->name);
3255 copied_tables = true;
3256 }
3257 }
3258
3259 if (!ecap_pass_through(iommu->ecap))
3260 hw_pass_through = 0;
3261 #ifdef CONFIG_INTEL_IOMMU_SVM
3262 if (pasid_enabled(iommu))
3263 intel_svm_alloc_pasid_tables(iommu);
3264 #endif
3265 }
3266
3267 /*
3268 * Now that qi is enabled on all iommus, set the root entry and flush
3269 * caches. This is required on some Intel X58 chipsets, otherwise the
3270 * flush_context function will loop forever and the boot hangs.
3271 */
3272 for_each_active_iommu(iommu, drhd) {
3273 iommu_flush_write_buffer(iommu);
3274 iommu_set_root_entry(iommu);
3275 iommu->flush.flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL);
3276 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
3277 }
3278
3279 if (iommu_pass_through)
3280 iommu_identity_mapping |= IDENTMAP_ALL;
3281
3282 #ifdef CONFIG_INTEL_IOMMU_BROKEN_GFX_WA
3283 iommu_identity_mapping |= IDENTMAP_GFX;
3284 #endif
3285
3286 if (iommu_identity_mapping) {
3287 ret = si_domain_init(hw_pass_through);
3288 if (ret)
3289 goto free_iommu;
3290 }
3291
3292 check_tylersburg_isoch();
3293
3294 /*
3295 * If we copied translations from a previous kernel in the kdump
3296 * case, we can not assign the devices to domains now, as that
3297 * would eliminate the old mappings. So skip this part and defer
3298 * the assignment to device driver initialization time.
3299 */
3300 if (copied_tables)
3301 goto domains_done;
3302
3303 /*
3304 * If pass through is not set or not enabled, setup context entries for
3305 * identity mappings for rmrr, gfx, and isa and may fall back to static
3306 * identity mapping if iommu_identity_mapping is set.
3307 */
3308 if (iommu_identity_mapping) {
3309 ret = iommu_prepare_static_identity_mapping(hw_pass_through);
3310 if (ret) {
3311 pr_crit("Failed to setup IOMMU pass-through\n");
3312 goto free_iommu;
3313 }
3314 }
3315 /*
3316 * For each rmrr
3317 * for each dev attached to rmrr
3318 * do
3319 * locate drhd for dev, alloc domain for dev
3320 * allocate free domain
3321 * allocate page table entries for rmrr
3322 * if context not allocated for bus
3323 * allocate and init context
3324 * set present in root table for this bus
3325 * init context with domain, translation etc
3326 * endfor
3327 * endfor
3328 */
3329 pr_info("Setting RMRR:\n");
3330 for_each_rmrr_units(rmrr) {
3331 /* some BIOS lists non-exist devices in DMAR table. */
3332 for_each_active_dev_scope(rmrr->devices, rmrr->devices_cnt,
3333 i, dev) {
3334 ret = iommu_prepare_rmrr_dev(rmrr, dev);
3335 if (ret)
3336 pr_err("Mapping reserved region failed\n");
3337 }
3338 }
3339
3340 iommu_prepare_isa();
3341
3342 domains_done:
3343
3344 /*
3345 * for each drhd
3346 * enable fault log
3347 * global invalidate context cache
3348 * global invalidate iotlb
3349 * enable translation
3350 */
3351 for_each_iommu(iommu, drhd) {
3352 if (drhd->ignored) {
3353 /*
3354 * we always have to disable PMRs or DMA may fail on
3355 * this device
3356 */
3357 if (force_on)
3358 iommu_disable_protect_mem_regions(iommu);
3359 continue;
3360 }
3361
3362 iommu_flush_write_buffer(iommu);
3363
3364 #ifdef CONFIG_INTEL_IOMMU_SVM
3365 if (pasid_enabled(iommu) && ecap_prs(iommu->ecap)) {
3366 ret = intel_svm_enable_prq(iommu);
3367 if (ret)
3368 goto free_iommu;
3369 }
3370 #endif
3371 ret = dmar_set_interrupt(iommu);
3372 if (ret)
3373 goto free_iommu;
3374
3375 if (!translation_pre_enabled(iommu))
3376 iommu_enable_translation(iommu);
3377
3378 iommu_disable_protect_mem_regions(iommu);
3379 }
3380
3381 return 0;
3382
3383 free_iommu:
3384 for_each_active_iommu(iommu, drhd) {
3385 disable_dmar_iommu(iommu);
3386 free_dmar_iommu(iommu);
3387 }
3388 free_g_iommus:
3389 for_each_possible_cpu(cpu)
3390 kfree(per_cpu_ptr(&deferred_flush, cpu)->tables);
3391 kfree(g_iommus);
3392 error:
3393 return ret;
3394 }
3395
3396 /* This takes a number of _MM_ pages, not VTD pages */
3397 static unsigned long intel_alloc_iova(struct device *dev,
3398 struct dmar_domain *domain,
3399 unsigned long nrpages, uint64_t dma_mask)
3400 {
3401 unsigned long iova_pfn = 0;
3402
3403 /* Restrict dma_mask to the width that the iommu can handle */
3404 dma_mask = min_t(uint64_t, DOMAIN_MAX_ADDR(domain->gaw), dma_mask);
3405 /* Ensure we reserve the whole size-aligned region */
3406 nrpages = __roundup_pow_of_two(nrpages);
3407
3408 if (!dmar_forcedac && dma_mask > DMA_BIT_MASK(32)) {
3409 /*
3410 * First try to allocate an io virtual address in
3411 * DMA_BIT_MASK(32) and if that fails then try allocating
3412 * from higher range
3413 */
3414 iova_pfn = alloc_iova_fast(&domain->iovad, nrpages,
3415 IOVA_PFN(DMA_BIT_MASK(32)));
3416 if (iova_pfn)
3417 return iova_pfn;
3418 }
3419 iova_pfn = alloc_iova_fast(&domain->iovad, nrpages, IOVA_PFN(dma_mask));
3420 if (unlikely(!iova_pfn)) {
3421 pr_err("Allocating %ld-page iova for %s failed",
3422 nrpages, dev_name(dev));
3423 return 0;
3424 }
3425
3426 return iova_pfn;
3427 }
3428
3429 static struct dmar_domain *__get_valid_domain_for_dev(struct device *dev)
3430 {
3431 struct dmar_domain *domain, *tmp;
3432 struct dmar_rmrr_unit *rmrr;
3433 struct device *i_dev;
3434 int i, ret;
3435
3436 domain = find_domain(dev);
3437 if (domain)
3438 goto out;
3439
3440 domain = find_or_alloc_domain(dev, DEFAULT_DOMAIN_ADDRESS_WIDTH);
3441 if (!domain)
3442 goto out;
3443
3444 /* We have a new domain - setup possible RMRRs for the device */
3445 rcu_read_lock();
3446 for_each_rmrr_units(rmrr) {
3447 for_each_active_dev_scope(rmrr->devices, rmrr->devices_cnt,
3448 i, i_dev) {
3449 if (i_dev != dev)
3450 continue;
3451
3452 ret = domain_prepare_identity_map(dev, domain,
3453 rmrr->base_address,
3454 rmrr->end_address);
3455 if (ret)
3456 dev_err(dev, "Mapping reserved region failed\n");
3457 }
3458 }
3459 rcu_read_unlock();
3460
3461 tmp = set_domain_for_dev(dev, domain);
3462 if (!tmp || domain != tmp) {
3463 domain_exit(domain);
3464 domain = tmp;
3465 }
3466
3467 out:
3468
3469 if (!domain)
3470 pr_err("Allocating domain for %s failed\n", dev_name(dev));
3471
3472
3473 return domain;
3474 }
3475
3476 static inline struct dmar_domain *get_valid_domain_for_dev(struct device *dev)
3477 {
3478 struct device_domain_info *info;
3479
3480 /* No lock here, assumes no domain exit in normal case */
3481 info = dev->archdata.iommu;
3482 if (likely(info))
3483 return info->domain;
3484
3485 return __get_valid_domain_for_dev(dev);
3486 }
3487
3488 /* Check if the dev needs to go through non-identity map and unmap process.*/
3489 static int iommu_no_mapping(struct device *dev)
3490 {
3491 int found;
3492
3493 if (iommu_dummy(dev))
3494 return 1;
3495
3496 if (!iommu_identity_mapping)
3497 return 0;
3498
3499 found = identity_mapping(dev);
3500 if (found) {
3501 if (iommu_should_identity_map(dev, 0))
3502 return 1;
3503 else {
3504 /*
3505 * 32 bit DMA is removed from si_domain and fall back
3506 * to non-identity mapping.
3507 */
3508 dmar_remove_one_dev_info(si_domain, dev);
3509 pr_info("32bit %s uses non-identity mapping\n",
3510 dev_name(dev));
3511 return 0;
3512 }
3513 } else {
3514 /*
3515 * In case of a detached 64 bit DMA device from vm, the device
3516 * is put into si_domain for identity mapping.
3517 */
3518 if (iommu_should_identity_map(dev, 0)) {
3519 int ret;
3520 ret = domain_add_dev_info(si_domain, dev);
3521 if (!ret) {
3522 pr_info("64bit %s uses identity mapping\n",
3523 dev_name(dev));
3524 return 1;
3525 }
3526 }
3527 }
3528
3529 return 0;
3530 }
3531
3532 static dma_addr_t __intel_map_single(struct device *dev, phys_addr_t paddr,
3533 size_t size, int dir, u64 dma_mask)
3534 {
3535 struct dmar_domain *domain;
3536 phys_addr_t start_paddr;
3537 unsigned long iova_pfn;
3538 int prot = 0;
3539 int ret;
3540 struct intel_iommu *iommu;
3541 unsigned long paddr_pfn = paddr >> PAGE_SHIFT;
3542
3543 BUG_ON(dir == DMA_NONE);
3544
3545 if (iommu_no_mapping(dev))
3546 return paddr;
3547
3548 domain = get_valid_domain_for_dev(dev);
3549 if (!domain)
3550 return 0;
3551
3552 iommu = domain_get_iommu(domain);
3553 size = aligned_nrpages(paddr, size);
3554
3555 iova_pfn = intel_alloc_iova(dev, domain, dma_to_mm_pfn(size), dma_mask);
3556 if (!iova_pfn)
3557 goto error;
3558
3559 /*
3560 * Check if DMAR supports zero-length reads on write only
3561 * mappings..
3562 */
3563 if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
3564 !cap_zlr(iommu->cap))
3565 prot |= DMA_PTE_READ;
3566 if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
3567 prot |= DMA_PTE_WRITE;
3568 /*
3569 * paddr - (paddr + size) might be partial page, we should map the whole
3570 * page. Note: if two part of one page are separately mapped, we
3571 * might have two guest_addr mapping to the same host paddr, but this
3572 * is not a big problem
3573 */
3574 ret = domain_pfn_mapping(domain, mm_to_dma_pfn(iova_pfn),
3575 mm_to_dma_pfn(paddr_pfn), size, prot);
3576 if (ret)
3577 goto error;
3578
3579 /* it's a non-present to present mapping. Only flush if caching mode */
3580 if (cap_caching_mode(iommu->cap))
3581 iommu_flush_iotlb_psi(iommu, domain,
3582 mm_to_dma_pfn(iova_pfn),
3583 size, 0, 1);
3584 else
3585 iommu_flush_write_buffer(iommu);
3586
3587 start_paddr = (phys_addr_t)iova_pfn << PAGE_SHIFT;
3588 start_paddr += paddr & ~PAGE_MASK;
3589 return start_paddr;
3590
3591 error:
3592 if (iova_pfn)
3593 free_iova_fast(&domain->iovad, iova_pfn, dma_to_mm_pfn(size));
3594 pr_err("Device %s request: %zx@%llx dir %d --- failed\n",
3595 dev_name(dev), size, (unsigned long long)paddr, dir);
3596 return 0;
3597 }
3598
3599 static dma_addr_t intel_map_page(struct device *dev, struct page *page,
3600 unsigned long offset, size_t size,
3601 enum dma_data_direction dir,
3602 unsigned long attrs)
3603 {
3604 return __intel_map_single(dev, page_to_phys(page) + offset, size,
3605 dir, *dev->dma_mask);
3606 }
3607
3608 static void flush_unmaps(struct deferred_flush_data *flush_data)
3609 {
3610 int i, j;
3611
3612 flush_data->timer_on = 0;
3613
3614 /* just flush them all */
3615 for (i = 0; i < g_num_of_iommus; i++) {
3616 struct intel_iommu *iommu = g_iommus[i];
3617 struct deferred_flush_table *flush_table =
3618 &flush_data->tables[i];
3619 if (!iommu)
3620 continue;
3621
3622 if (!flush_table->next)
3623 continue;
3624
3625 /* In caching mode, global flushes turn emulation expensive */
3626 if (!cap_caching_mode(iommu->cap))
3627 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
3628 DMA_TLB_GLOBAL_FLUSH);
3629 for (j = 0; j < flush_table->next; j++) {
3630 unsigned long mask;
3631 struct deferred_flush_entry *entry =
3632 &flush_table->entries[j];
3633 unsigned long iova_pfn = entry->iova_pfn;
3634 unsigned long nrpages = entry->nrpages;
3635 struct dmar_domain *domain = entry->domain;
3636 struct page *freelist = entry->freelist;
3637
3638 /* On real hardware multiple invalidations are expensive */
3639 if (cap_caching_mode(iommu->cap))
3640 iommu_flush_iotlb_psi(iommu, domain,
3641 mm_to_dma_pfn(iova_pfn),
3642 nrpages, !freelist, 0);
3643 else {
3644 mask = ilog2(nrpages);
3645 iommu_flush_dev_iotlb(domain,
3646 (uint64_t)iova_pfn << PAGE_SHIFT, mask);
3647 }
3648 free_iova_fast(&domain->iovad, iova_pfn, nrpages);
3649 if (freelist)
3650 dma_free_pagelist(freelist);
3651 }
3652 flush_table->next = 0;
3653 }
3654
3655 flush_data->size = 0;
3656 }
3657
3658 static void flush_unmaps_timeout(unsigned long cpuid)
3659 {
3660 struct deferred_flush_data *flush_data = per_cpu_ptr(&deferred_flush, cpuid);
3661 unsigned long flags;
3662
3663 spin_lock_irqsave(&flush_data->lock, flags);
3664 flush_unmaps(flush_data);
3665 spin_unlock_irqrestore(&flush_data->lock, flags);
3666 }
3667
3668 static void add_unmap(struct dmar_domain *dom, unsigned long iova_pfn,
3669 unsigned long nrpages, struct page *freelist)
3670 {
3671 unsigned long flags;
3672 int entry_id, iommu_id;
3673 struct intel_iommu *iommu;
3674 struct deferred_flush_entry *entry;
3675 struct deferred_flush_data *flush_data;
3676 unsigned int cpuid;
3677
3678 cpuid = get_cpu();
3679 flush_data = per_cpu_ptr(&deferred_flush, cpuid);
3680
3681 /* Flush all CPUs' entries to avoid deferring too much. If
3682 * this becomes a bottleneck, can just flush us, and rely on
3683 * flush timer for the rest.
3684 */
3685 if (flush_data->size == HIGH_WATER_MARK) {
3686 int cpu;
3687
3688 for_each_online_cpu(cpu)
3689 flush_unmaps_timeout(cpu);
3690 }
3691
3692 spin_lock_irqsave(&flush_data->lock, flags);
3693
3694 iommu = domain_get_iommu(dom);
3695 iommu_id = iommu->seq_id;
3696
3697 entry_id = flush_data->tables[iommu_id].next;
3698 ++(flush_data->tables[iommu_id].next);
3699
3700 entry = &flush_data->tables[iommu_id].entries[entry_id];
3701 entry->domain = dom;
3702 entry->iova_pfn = iova_pfn;
3703 entry->nrpages = nrpages;
3704 entry->freelist = freelist;
3705
3706 if (!flush_data->timer_on) {
3707 mod_timer(&flush_data->timer, jiffies + msecs_to_jiffies(10));
3708 flush_data->timer_on = 1;
3709 }
3710 flush_data->size++;
3711 spin_unlock_irqrestore(&flush_data->lock, flags);
3712
3713 put_cpu();
3714 }
3715
3716 static void intel_unmap(struct device *dev, dma_addr_t dev_addr, size_t size)
3717 {
3718 struct dmar_domain *domain;
3719 unsigned long start_pfn, last_pfn;
3720 unsigned long nrpages;
3721 unsigned long iova_pfn;
3722 struct intel_iommu *iommu;
3723 struct page *freelist;
3724
3725 if (iommu_no_mapping(dev))
3726 return;
3727
3728 domain = find_domain(dev);
3729 BUG_ON(!domain);
3730
3731 iommu = domain_get_iommu(domain);
3732
3733 iova_pfn = IOVA_PFN(dev_addr);
3734
3735 nrpages = aligned_nrpages(dev_addr, size);
3736 start_pfn = mm_to_dma_pfn(iova_pfn);
3737 last_pfn = start_pfn + nrpages - 1;
3738
3739 pr_debug("Device %s unmapping: pfn %lx-%lx\n",
3740 dev_name(dev), start_pfn, last_pfn);
3741
3742 freelist = domain_unmap(domain, start_pfn, last_pfn);
3743
3744 if (intel_iommu_strict) {
3745 iommu_flush_iotlb_psi(iommu, domain, start_pfn,
3746 nrpages, !freelist, 0);
3747 /* free iova */
3748 free_iova_fast(&domain->iovad, iova_pfn, dma_to_mm_pfn(nrpages));
3749 dma_free_pagelist(freelist);
3750 } else {
3751 add_unmap(domain, iova_pfn, nrpages, freelist);
3752 /*
3753 * queue up the release of the unmap to save the 1/6th of the
3754 * cpu used up by the iotlb flush operation...
3755 */
3756 }
3757 }
3758
3759 static void intel_unmap_page(struct device *dev, dma_addr_t dev_addr,
3760 size_t size, enum dma_data_direction dir,
3761 unsigned long attrs)
3762 {
3763 intel_unmap(dev, dev_addr, size);
3764 }
3765
3766 static void *intel_alloc_coherent(struct device *dev, size_t size,
3767 dma_addr_t *dma_handle, gfp_t flags,
3768 unsigned long attrs)
3769 {
3770 struct page *page = NULL;
3771 int order;
3772
3773 size = PAGE_ALIGN(size);
3774 order = get_order(size);
3775
3776 if (!iommu_no_mapping(dev))
3777 flags &= ~(GFP_DMA | GFP_DMA32);
3778 else if (dev->coherent_dma_mask < dma_get_required_mask(dev)) {
3779 if (dev->coherent_dma_mask < DMA_BIT_MASK(32))
3780 flags |= GFP_DMA;
3781 else
3782 flags |= GFP_DMA32;
3783 }
3784
3785 if (gfpflags_allow_blocking(flags)) {
3786 unsigned int count = size >> PAGE_SHIFT;
3787
3788 page = dma_alloc_from_contiguous(dev, count, order);
3789 if (page && iommu_no_mapping(dev) &&
3790 page_to_phys(page) + size > dev->coherent_dma_mask) {
3791 dma_release_from_contiguous(dev, page, count);
3792 page = NULL;
3793 }
3794 }
3795
3796 if (!page)
3797 page = alloc_pages(flags, order);
3798 if (!page)
3799 return NULL;
3800 memset(page_address(page), 0, size);
3801
3802 *dma_handle = __intel_map_single(dev, page_to_phys(page), size,
3803 DMA_BIDIRECTIONAL,
3804 dev->coherent_dma_mask);
3805 if (*dma_handle)
3806 return page_address(page);
3807 if (!dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT))
3808 __free_pages(page, order);
3809
3810 return NULL;
3811 }
3812
3813 static void intel_free_coherent(struct device *dev, size_t size, void *vaddr,
3814 dma_addr_t dma_handle, unsigned long attrs)
3815 {
3816 int order;
3817 struct page *page = virt_to_page(vaddr);
3818
3819 size = PAGE_ALIGN(size);
3820 order = get_order(size);
3821
3822 intel_unmap(dev, dma_handle, size);
3823 if (!dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT))
3824 __free_pages(page, order);
3825 }
3826
3827 static void intel_unmap_sg(struct device *dev, struct scatterlist *sglist,
3828 int nelems, enum dma_data_direction dir,
3829 unsigned long attrs)
3830 {
3831 dma_addr_t startaddr = sg_dma_address(sglist) & PAGE_MASK;
3832 unsigned long nrpages = 0;
3833 struct scatterlist *sg;
3834 int i;
3835
3836 for_each_sg(sglist, sg, nelems, i) {
3837 nrpages += aligned_nrpages(sg_dma_address(sg), sg_dma_len(sg));
3838 }
3839
3840 intel_unmap(dev, startaddr, nrpages << VTD_PAGE_SHIFT);
3841 }
3842
3843 static int intel_nontranslate_map_sg(struct device *hddev,
3844 struct scatterlist *sglist, int nelems, int dir)
3845 {
3846 int i;
3847 struct scatterlist *sg;
3848
3849 for_each_sg(sglist, sg, nelems, i) {
3850 BUG_ON(!sg_page(sg));
3851 sg->dma_address = page_to_phys(sg_page(sg)) + sg->offset;
3852 sg->dma_length = sg->length;
3853 }
3854 return nelems;
3855 }
3856
3857 static int intel_map_sg(struct device *dev, struct scatterlist *sglist, int nelems,
3858 enum dma_data_direction dir, unsigned long attrs)
3859 {
3860 int i;
3861 struct dmar_domain *domain;
3862 size_t size = 0;
3863 int prot = 0;
3864 unsigned long iova_pfn;
3865 int ret;
3866 struct scatterlist *sg;
3867 unsigned long start_vpfn;
3868 struct intel_iommu *iommu;
3869
3870 BUG_ON(dir == DMA_NONE);
3871 if (iommu_no_mapping(dev))
3872 return intel_nontranslate_map_sg(dev, sglist, nelems, dir);
3873
3874 domain = get_valid_domain_for_dev(dev);
3875 if (!domain)
3876 return 0;
3877
3878 iommu = domain_get_iommu(domain);
3879
3880 for_each_sg(sglist, sg, nelems, i)
3881 size += aligned_nrpages(sg->offset, sg->length);
3882
3883 iova_pfn = intel_alloc_iova(dev, domain, dma_to_mm_pfn(size),
3884 *dev->dma_mask);
3885 if (!iova_pfn) {
3886 sglist->dma_length = 0;
3887 return 0;
3888 }
3889
3890 /*
3891 * Check if DMAR supports zero-length reads on write only
3892 * mappings..
3893 */
3894 if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
3895 !cap_zlr(iommu->cap))
3896 prot |= DMA_PTE_READ;
3897 if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
3898 prot |= DMA_PTE_WRITE;
3899
3900 start_vpfn = mm_to_dma_pfn(iova_pfn);
3901
3902 ret = domain_sg_mapping(domain, start_vpfn, sglist, size, prot);
3903 if (unlikely(ret)) {
3904 dma_pte_free_pagetable(domain, start_vpfn,
3905 start_vpfn + size - 1);
3906 free_iova_fast(&domain->iovad, iova_pfn, dma_to_mm_pfn(size));
3907 return 0;
3908 }
3909
3910 /* it's a non-present to present mapping. Only flush if caching mode */
3911 if (cap_caching_mode(iommu->cap))
3912 iommu_flush_iotlb_psi(iommu, domain, start_vpfn, size, 0, 1);
3913 else
3914 iommu_flush_write_buffer(iommu);
3915
3916 return nelems;
3917 }
3918
3919 static int intel_mapping_error(struct device *dev, dma_addr_t dma_addr)
3920 {
3921 return !dma_addr;
3922 }
3923
3924 struct dma_map_ops intel_dma_ops = {
3925 .alloc = intel_alloc_coherent,
3926 .free = intel_free_coherent,
3927 .map_sg = intel_map_sg,
3928 .unmap_sg = intel_unmap_sg,
3929 .map_page = intel_map_page,
3930 .unmap_page = intel_unmap_page,
3931 .mapping_error = intel_mapping_error,
3932 };
3933
3934 static inline int iommu_domain_cache_init(void)
3935 {
3936 int ret = 0;
3937
3938 iommu_domain_cache = kmem_cache_create("iommu_domain",
3939 sizeof(struct dmar_domain),
3940 0,
3941 SLAB_HWCACHE_ALIGN,
3942
3943 NULL);
3944 if (!iommu_domain_cache) {
3945 pr_err("Couldn't create iommu_domain cache\n");
3946 ret = -ENOMEM;
3947 }
3948
3949 return ret;
3950 }
3951
3952 static inline int iommu_devinfo_cache_init(void)
3953 {
3954 int ret = 0;
3955
3956 iommu_devinfo_cache = kmem_cache_create("iommu_devinfo",
3957 sizeof(struct device_domain_info),
3958 0,
3959 SLAB_HWCACHE_ALIGN,
3960 NULL);
3961 if (!iommu_devinfo_cache) {
3962 pr_err("Couldn't create devinfo cache\n");
3963 ret = -ENOMEM;
3964 }
3965
3966 return ret;
3967 }
3968
3969 static int __init iommu_init_mempool(void)
3970 {
3971 int ret;
3972 ret = iova_cache_get();
3973 if (ret)
3974 return ret;
3975
3976 ret = iommu_domain_cache_init();
3977 if (ret)
3978 goto domain_error;
3979
3980 ret = iommu_devinfo_cache_init();
3981 if (!ret)
3982 return ret;
3983
3984 kmem_cache_destroy(iommu_domain_cache);
3985 domain_error:
3986 iova_cache_put();
3987
3988 return -ENOMEM;
3989 }
3990
3991 static void __init iommu_exit_mempool(void)
3992 {
3993 kmem_cache_destroy(iommu_devinfo_cache);
3994 kmem_cache_destroy(iommu_domain_cache);
3995 iova_cache_put();
3996 }
3997
3998 static void quirk_ioat_snb_local_iommu(struct pci_dev *pdev)
3999 {
4000 struct dmar_drhd_unit *drhd;
4001 u32 vtbar;
4002 int rc;
4003
4004 /* We know that this device on this chipset has its own IOMMU.
4005 * If we find it under a different IOMMU, then the BIOS is lying
4006 * to us. Hope that the IOMMU for this device is actually
4007 * disabled, and it needs no translation...
4008 */
4009 rc = pci_bus_read_config_dword(pdev->bus, PCI_DEVFN(0, 0), 0xb0, &vtbar);
4010 if (rc) {
4011 /* "can't" happen */
4012 dev_info(&pdev->dev, "failed to run vt-d quirk\n");
4013 return;
4014 }
4015 vtbar &= 0xffff0000;
4016
4017 /* we know that the this iommu should be at offset 0xa000 from vtbar */
4018 drhd = dmar_find_matched_drhd_unit(pdev);
4019 if (WARN_TAINT_ONCE(!drhd || drhd->reg_base_addr - vtbar != 0xa000,
4020 TAINT_FIRMWARE_WORKAROUND,
4021 "BIOS assigned incorrect VT-d unit for Intel(R) QuickData Technology device\n"))
4022 pdev->dev.archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO;
4023 }
4024 DECLARE_PCI_FIXUP_ENABLE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB, quirk_ioat_snb_local_iommu);
4025
4026 static void __init init_no_remapping_devices(void)
4027 {
4028 struct dmar_drhd_unit *drhd;
4029 struct device *dev;
4030 int i;
4031
4032 for_each_drhd_unit(drhd) {
4033 if (!drhd->include_all) {
4034 for_each_active_dev_scope(drhd->devices,
4035 drhd->devices_cnt, i, dev)
4036 break;
4037 /* ignore DMAR unit if no devices exist */
4038 if (i == drhd->devices_cnt)
4039 drhd->ignored = 1;
4040 }
4041 }
4042
4043 for_each_active_drhd_unit(drhd) {
4044 if (drhd->include_all)
4045 continue;
4046
4047 for_each_active_dev_scope(drhd->devices,
4048 drhd->devices_cnt, i, dev)
4049 if (!dev_is_pci(dev) || !IS_GFX_DEVICE(to_pci_dev(dev)))
4050 break;
4051 if (i < drhd->devices_cnt)
4052 continue;
4053
4054 /* This IOMMU has *only* gfx devices. Either bypass it or
4055 set the gfx_mapped flag, as appropriate */
4056 if (dmar_map_gfx) {
4057 intel_iommu_gfx_mapped = 1;
4058 } else {
4059 drhd->ignored = 1;
4060 for_each_active_dev_scope(drhd->devices,
4061 drhd->devices_cnt, i, dev)
4062 dev->archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO;
4063 }
4064 }
4065 }
4066
4067 #ifdef CONFIG_SUSPEND
4068 static int init_iommu_hw(void)
4069 {
4070 struct dmar_drhd_unit *drhd;
4071 struct intel_iommu *iommu = NULL;
4072
4073 for_each_active_iommu(iommu, drhd)
4074 if (iommu->qi)
4075 dmar_reenable_qi(iommu);
4076
4077 for_each_iommu(iommu, drhd) {
4078 if (drhd->ignored) {
4079 /*
4080 * we always have to disable PMRs or DMA may fail on
4081 * this device
4082 */
4083 if (force_on)
4084 iommu_disable_protect_mem_regions(iommu);
4085 continue;
4086 }
4087
4088 iommu_flush_write_buffer(iommu);
4089
4090 iommu_set_root_entry(iommu);
4091
4092 iommu->flush.flush_context(iommu, 0, 0, 0,
4093 DMA_CCMD_GLOBAL_INVL);
4094 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
4095 iommu_enable_translation(iommu);
4096 iommu_disable_protect_mem_regions(iommu);
4097 }
4098
4099 return 0;
4100 }
4101
4102 static void iommu_flush_all(void)
4103 {
4104 struct dmar_drhd_unit *drhd;
4105 struct intel_iommu *iommu;
4106
4107 for_each_active_iommu(iommu, drhd) {
4108 iommu->flush.flush_context(iommu, 0, 0, 0,
4109 DMA_CCMD_GLOBAL_INVL);
4110 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
4111 DMA_TLB_GLOBAL_FLUSH);
4112 }
4113 }
4114
4115 static int iommu_suspend(void)
4116 {
4117 struct dmar_drhd_unit *drhd;
4118 struct intel_iommu *iommu = NULL;
4119 unsigned long flag;
4120
4121 for_each_active_iommu(iommu, drhd) {
4122 iommu->iommu_state = kzalloc(sizeof(u32) * MAX_SR_DMAR_REGS,
4123 GFP_ATOMIC);
4124 if (!iommu->iommu_state)
4125 goto nomem;
4126 }
4127
4128 iommu_flush_all();
4129
4130 for_each_active_iommu(iommu, drhd) {
4131 iommu_disable_translation(iommu);
4132
4133 raw_spin_lock_irqsave(&iommu->register_lock, flag);
4134
4135 iommu->iommu_state[SR_DMAR_FECTL_REG] =
4136 readl(iommu->reg + DMAR_FECTL_REG);
4137 iommu->iommu_state[SR_DMAR_FEDATA_REG] =
4138 readl(iommu->reg + DMAR_FEDATA_REG);
4139 iommu->iommu_state[SR_DMAR_FEADDR_REG] =
4140 readl(iommu->reg + DMAR_FEADDR_REG);
4141 iommu->iommu_state[SR_DMAR_FEUADDR_REG] =
4142 readl(iommu->reg + DMAR_FEUADDR_REG);
4143
4144 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
4145 }
4146 return 0;
4147
4148 nomem:
4149 for_each_active_iommu(iommu, drhd)
4150 kfree(iommu->iommu_state);
4151
4152 return -ENOMEM;
4153 }
4154
4155 static void iommu_resume(void)
4156 {
4157 struct dmar_drhd_unit *drhd;
4158 struct intel_iommu *iommu = NULL;
4159 unsigned long flag;
4160
4161 if (init_iommu_hw()) {
4162 if (force_on)
4163 panic("tboot: IOMMU setup failed, DMAR can not resume!\n");
4164 else
4165 WARN(1, "IOMMU setup failed, DMAR can not resume!\n");
4166 return;
4167 }
4168
4169 for_each_active_iommu(iommu, drhd) {
4170
4171 raw_spin_lock_irqsave(&iommu->register_lock, flag);
4172
4173 writel(iommu->iommu_state[SR_DMAR_FECTL_REG],
4174 iommu->reg + DMAR_FECTL_REG);
4175 writel(iommu->iommu_state[SR_DMAR_FEDATA_REG],
4176 iommu->reg + DMAR_FEDATA_REG);
4177 writel(iommu->iommu_state[SR_DMAR_FEADDR_REG],
4178 iommu->reg + DMAR_FEADDR_REG);
4179 writel(iommu->iommu_state[SR_DMAR_FEUADDR_REG],
4180 iommu->reg + DMAR_FEUADDR_REG);
4181
4182 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
4183 }
4184
4185 for_each_active_iommu(iommu, drhd)
4186 kfree(iommu->iommu_state);
4187 }
4188
4189 static struct syscore_ops iommu_syscore_ops = {
4190 .resume = iommu_resume,
4191 .suspend = iommu_suspend,
4192 };
4193
4194 static void __init init_iommu_pm_ops(void)
4195 {
4196 register_syscore_ops(&iommu_syscore_ops);
4197 }
4198
4199 #else
4200 static inline void init_iommu_pm_ops(void) {}
4201 #endif /* CONFIG_PM */
4202
4203
4204 int __init dmar_parse_one_rmrr(struct acpi_dmar_header *header, void *arg)
4205 {
4206 struct acpi_dmar_reserved_memory *rmrr;
4207 struct dmar_rmrr_unit *rmrru;
4208
4209 rmrru = kzalloc(sizeof(*rmrru), GFP_KERNEL);
4210 if (!rmrru)
4211 return -ENOMEM;
4212
4213 rmrru->hdr = header;
4214 rmrr = (struct acpi_dmar_reserved_memory *)header;
4215 rmrru->base_address = rmrr->base_address;
4216 rmrru->end_address = rmrr->end_address;
4217 rmrru->devices = dmar_alloc_dev_scope((void *)(rmrr + 1),
4218 ((void *)rmrr) + rmrr->header.length,
4219 &rmrru->devices_cnt);
4220 if (rmrru->devices_cnt && rmrru->devices == NULL) {
4221 kfree(rmrru);
4222 return -ENOMEM;
4223 }
4224
4225 list_add(&rmrru->list, &dmar_rmrr_units);
4226
4227 return 0;
4228 }
4229
4230 static struct dmar_atsr_unit *dmar_find_atsr(struct acpi_dmar_atsr *atsr)
4231 {
4232 struct dmar_atsr_unit *atsru;
4233 struct acpi_dmar_atsr *tmp;
4234
4235 list_for_each_entry_rcu(atsru, &dmar_atsr_units, list) {
4236 tmp = (struct acpi_dmar_atsr *)atsru->hdr;
4237 if (atsr->segment != tmp->segment)
4238 continue;
4239 if (atsr->header.length != tmp->header.length)
4240 continue;
4241 if (memcmp(atsr, tmp, atsr->header.length) == 0)
4242 return atsru;
4243 }
4244
4245 return NULL;
4246 }
4247
4248 int dmar_parse_one_atsr(struct acpi_dmar_header *hdr, void *arg)
4249 {
4250 struct acpi_dmar_atsr *atsr;
4251 struct dmar_atsr_unit *atsru;
4252
4253 if (system_state != SYSTEM_BOOTING && !intel_iommu_enabled)
4254 return 0;
4255
4256 atsr = container_of(hdr, struct acpi_dmar_atsr, header);
4257 atsru = dmar_find_atsr(atsr);
4258 if (atsru)
4259 return 0;
4260
4261 atsru = kzalloc(sizeof(*atsru) + hdr->length, GFP_KERNEL);
4262 if (!atsru)
4263 return -ENOMEM;
4264
4265 /*
4266 * If memory is allocated from slab by ACPI _DSM method, we need to
4267 * copy the memory content because the memory buffer will be freed
4268 * on return.
4269 */
4270 atsru->hdr = (void *)(atsru + 1);
4271 memcpy(atsru->hdr, hdr, hdr->length);
4272 atsru->include_all = atsr->flags & 0x1;
4273 if (!atsru->include_all) {
4274 atsru->devices = dmar_alloc_dev_scope((void *)(atsr + 1),
4275 (void *)atsr + atsr->header.length,
4276 &atsru->devices_cnt);
4277 if (atsru->devices_cnt && atsru->devices == NULL) {
4278 kfree(atsru);
4279 return -ENOMEM;
4280 }
4281 }
4282
4283 list_add_rcu(&atsru->list, &dmar_atsr_units);
4284
4285 return 0;
4286 }
4287
4288 static void intel_iommu_free_atsr(struct dmar_atsr_unit *atsru)
4289 {
4290 dmar_free_dev_scope(&atsru->devices, &atsru->devices_cnt);
4291 kfree(atsru);
4292 }
4293
4294 int dmar_release_one_atsr(struct acpi_dmar_header *hdr, void *arg)
4295 {
4296 struct acpi_dmar_atsr *atsr;
4297 struct dmar_atsr_unit *atsru;
4298
4299 atsr = container_of(hdr, struct acpi_dmar_atsr, header);
4300 atsru = dmar_find_atsr(atsr);
4301 if (atsru) {
4302 list_del_rcu(&atsru->list);
4303 synchronize_rcu();
4304 intel_iommu_free_atsr(atsru);
4305 }
4306
4307 return 0;
4308 }
4309
4310 int dmar_check_one_atsr(struct acpi_dmar_header *hdr, void *arg)
4311 {
4312 int i;
4313 struct device *dev;
4314 struct acpi_dmar_atsr *atsr;
4315 struct dmar_atsr_unit *atsru;
4316
4317 atsr = container_of(hdr, struct acpi_dmar_atsr, header);
4318 atsru = dmar_find_atsr(atsr);
4319 if (!atsru)
4320 return 0;
4321
4322 if (!atsru->include_all && atsru->devices && atsru->devices_cnt) {
4323 for_each_active_dev_scope(atsru->devices, atsru->devices_cnt,
4324 i, dev)
4325 return -EBUSY;
4326 }
4327
4328 return 0;
4329 }
4330
4331 static int intel_iommu_add(struct dmar_drhd_unit *dmaru)
4332 {
4333 int sp, ret = 0;
4334 struct intel_iommu *iommu = dmaru->iommu;
4335
4336 if (g_iommus[iommu->seq_id])
4337 return 0;
4338
4339 if (hw_pass_through && !ecap_pass_through(iommu->ecap)) {
4340 pr_warn("%s: Doesn't support hardware pass through.\n",
4341 iommu->name);
4342 return -ENXIO;
4343 }
4344 if (!ecap_sc_support(iommu->ecap) &&
4345 domain_update_iommu_snooping(iommu)) {
4346 pr_warn("%s: Doesn't support snooping.\n",
4347 iommu->name);
4348 return -ENXIO;
4349 }
4350 sp = domain_update_iommu_superpage(iommu) - 1;
4351 if (sp >= 0 && !(cap_super_page_val(iommu->cap) & (1 << sp))) {
4352 pr_warn("%s: Doesn't support large page.\n",
4353 iommu->name);
4354 return -ENXIO;
4355 }
4356
4357 /*
4358 * Disable translation if already enabled prior to OS handover.
4359 */
4360 if (iommu->gcmd & DMA_GCMD_TE)
4361 iommu_disable_translation(iommu);
4362
4363 g_iommus[iommu->seq_id] = iommu;
4364 ret = iommu_init_domains(iommu);
4365 if (ret == 0)
4366 ret = iommu_alloc_root_entry(iommu);
4367 if (ret)
4368 goto out;
4369
4370 #ifdef CONFIG_INTEL_IOMMU_SVM
4371 if (pasid_enabled(iommu))
4372 intel_svm_alloc_pasid_tables(iommu);
4373 #endif
4374
4375 if (dmaru->ignored) {
4376 /*
4377 * we always have to disable PMRs or DMA may fail on this device
4378 */
4379 if (force_on)
4380 iommu_disable_protect_mem_regions(iommu);
4381 return 0;
4382 }
4383
4384 intel_iommu_init_qi(iommu);
4385 iommu_flush_write_buffer(iommu);
4386
4387 #ifdef CONFIG_INTEL_IOMMU_SVM
4388 if (pasid_enabled(iommu) && ecap_prs(iommu->ecap)) {
4389 ret = intel_svm_enable_prq(iommu);
4390 if (ret)
4391 goto disable_iommu;
4392 }
4393 #endif
4394 ret = dmar_set_interrupt(iommu);
4395 if (ret)
4396 goto disable_iommu;
4397
4398 iommu_set_root_entry(iommu);
4399 iommu->flush.flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL);
4400 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
4401 iommu_enable_translation(iommu);
4402
4403 iommu_disable_protect_mem_regions(iommu);
4404 return 0;
4405
4406 disable_iommu:
4407 disable_dmar_iommu(iommu);
4408 out:
4409 free_dmar_iommu(iommu);
4410 return ret;
4411 }
4412
4413 int dmar_iommu_hotplug(struct dmar_drhd_unit *dmaru, bool insert)
4414 {
4415 int ret = 0;
4416 struct intel_iommu *iommu = dmaru->iommu;
4417
4418 if (!intel_iommu_enabled)
4419 return 0;
4420 if (iommu == NULL)
4421 return -EINVAL;
4422
4423 if (insert) {
4424 ret = intel_iommu_add(dmaru);
4425 } else {
4426 disable_dmar_iommu(iommu);
4427 free_dmar_iommu(iommu);
4428 }
4429
4430 return ret;
4431 }
4432
4433 static void intel_iommu_free_dmars(void)
4434 {
4435 struct dmar_rmrr_unit *rmrru, *rmrr_n;
4436 struct dmar_atsr_unit *atsru, *atsr_n;
4437
4438 list_for_each_entry_safe(rmrru, rmrr_n, &dmar_rmrr_units, list) {
4439 list_del(&rmrru->list);
4440 dmar_free_dev_scope(&rmrru->devices, &rmrru->devices_cnt);
4441 kfree(rmrru);
4442 }
4443
4444 list_for_each_entry_safe(atsru, atsr_n, &dmar_atsr_units, list) {
4445 list_del(&atsru->list);
4446 intel_iommu_free_atsr(atsru);
4447 }
4448 }
4449
4450 int dmar_find_matched_atsr_unit(struct pci_dev *dev)
4451 {
4452 int i, ret = 1;
4453 struct pci_bus *bus;
4454 struct pci_dev *bridge = NULL;
4455 struct device *tmp;
4456 struct acpi_dmar_atsr *atsr;
4457 struct dmar_atsr_unit *atsru;
4458
4459 dev = pci_physfn(dev);
4460 for (bus = dev->bus; bus; bus = bus->parent) {
4461 bridge = bus->self;
4462 /* If it's an integrated device, allow ATS */
4463 if (!bridge)
4464 return 1;
4465 /* Connected via non-PCIe: no ATS */
4466 if (!pci_is_pcie(bridge) ||
4467 pci_pcie_type(bridge) == PCI_EXP_TYPE_PCI_BRIDGE)
4468 return 0;
4469 /* If we found the root port, look it up in the ATSR */
4470 if (pci_pcie_type(bridge) == PCI_EXP_TYPE_ROOT_PORT)
4471 break;
4472 }
4473
4474 rcu_read_lock();
4475 list_for_each_entry_rcu(atsru, &dmar_atsr_units, list) {
4476 atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header);
4477 if (atsr->segment != pci_domain_nr(dev->bus))
4478 continue;
4479
4480 for_each_dev_scope(atsru->devices, atsru->devices_cnt, i, tmp)
4481 if (tmp == &bridge->dev)
4482 goto out;
4483
4484 if (atsru->include_all)
4485 goto out;
4486 }
4487 ret = 0;
4488 out:
4489 rcu_read_unlock();
4490
4491 return ret;
4492 }
4493
4494 int dmar_iommu_notify_scope_dev(struct dmar_pci_notify_info *info)
4495 {
4496 int ret = 0;
4497 struct dmar_rmrr_unit *rmrru;
4498 struct dmar_atsr_unit *atsru;
4499 struct acpi_dmar_atsr *atsr;
4500 struct acpi_dmar_reserved_memory *rmrr;
4501
4502 if (!intel_iommu_enabled && system_state != SYSTEM_BOOTING)
4503 return 0;
4504
4505 list_for_each_entry(rmrru, &dmar_rmrr_units, list) {
4506 rmrr = container_of(rmrru->hdr,
4507 struct acpi_dmar_reserved_memory, header);
4508 if (info->event == BUS_NOTIFY_ADD_DEVICE) {
4509 ret = dmar_insert_dev_scope(info, (void *)(rmrr + 1),
4510 ((void *)rmrr) + rmrr->header.length,
4511 rmrr->segment, rmrru->devices,
4512 rmrru->devices_cnt);
4513 if(ret < 0)
4514 return ret;
4515 } else if (info->event == BUS_NOTIFY_REMOVED_DEVICE) {
4516 dmar_remove_dev_scope(info, rmrr->segment,
4517 rmrru->devices, rmrru->devices_cnt);
4518 }
4519 }
4520
4521 list_for_each_entry(atsru, &dmar_atsr_units, list) {
4522 if (atsru->include_all)
4523 continue;
4524
4525 atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header);
4526 if (info->event == BUS_NOTIFY_ADD_DEVICE) {
4527 ret = dmar_insert_dev_scope(info, (void *)(atsr + 1),
4528 (void *)atsr + atsr->header.length,
4529 atsr->segment, atsru->devices,
4530 atsru->devices_cnt);
4531 if (ret > 0)
4532 break;
4533 else if(ret < 0)
4534 return ret;
4535 } else if (info->event == BUS_NOTIFY_REMOVED_DEVICE) {
4536 if (dmar_remove_dev_scope(info, atsr->segment,
4537 atsru->devices, atsru->devices_cnt))
4538 break;
4539 }
4540 }
4541
4542 return 0;
4543 }
4544
4545 /*
4546 * Here we only respond to action of unbound device from driver.
4547 *
4548 * Added device is not attached to its DMAR domain here yet. That will happen
4549 * when mapping the device to iova.
4550 */
4551 static int device_notifier(struct notifier_block *nb,
4552 unsigned long action, void *data)
4553 {
4554 struct device *dev = data;
4555 struct dmar_domain *domain;
4556
4557 if (iommu_dummy(dev))
4558 return 0;
4559
4560 if (action != BUS_NOTIFY_REMOVED_DEVICE)
4561 return 0;
4562
4563 domain = find_domain(dev);
4564 if (!domain)
4565 return 0;
4566
4567 dmar_remove_one_dev_info(domain, dev);
4568 if (!domain_type_is_vm_or_si(domain) && list_empty(&domain->devices))
4569 domain_exit(domain);
4570
4571 return 0;
4572 }
4573
4574 static struct notifier_block device_nb = {
4575 .notifier_call = device_notifier,
4576 };
4577
4578 static int intel_iommu_memory_notifier(struct notifier_block *nb,
4579 unsigned long val, void *v)
4580 {
4581 struct memory_notify *mhp = v;
4582 unsigned long long start, end;
4583 unsigned long start_vpfn, last_vpfn;
4584
4585 switch (val) {
4586 case MEM_GOING_ONLINE:
4587 start = mhp->start_pfn << PAGE_SHIFT;
4588 end = ((mhp->start_pfn + mhp->nr_pages) << PAGE_SHIFT) - 1;
4589 if (iommu_domain_identity_map(si_domain, start, end)) {
4590 pr_warn("Failed to build identity map for [%llx-%llx]\n",
4591 start, end);
4592 return NOTIFY_BAD;
4593 }
4594 break;
4595
4596 case MEM_OFFLINE:
4597 case MEM_CANCEL_ONLINE:
4598 start_vpfn = mm_to_dma_pfn(mhp->start_pfn);
4599 last_vpfn = mm_to_dma_pfn(mhp->start_pfn + mhp->nr_pages - 1);
4600 while (start_vpfn <= last_vpfn) {
4601 struct iova *iova;
4602 struct dmar_drhd_unit *drhd;
4603 struct intel_iommu *iommu;
4604 struct page *freelist;
4605
4606 iova = find_iova(&si_domain->iovad, start_vpfn);
4607 if (iova == NULL) {
4608 pr_debug("Failed get IOVA for PFN %lx\n",
4609 start_vpfn);
4610 break;
4611 }
4612
4613 iova = split_and_remove_iova(&si_domain->iovad, iova,
4614 start_vpfn, last_vpfn);
4615 if (iova == NULL) {
4616 pr_warn("Failed to split IOVA PFN [%lx-%lx]\n",
4617 start_vpfn, last_vpfn);
4618 return NOTIFY_BAD;
4619 }
4620
4621 freelist = domain_unmap(si_domain, iova->pfn_lo,
4622 iova->pfn_hi);
4623
4624 rcu_read_lock();
4625 for_each_active_iommu(iommu, drhd)
4626 iommu_flush_iotlb_psi(iommu, si_domain,
4627 iova->pfn_lo, iova_size(iova),
4628 !freelist, 0);
4629 rcu_read_unlock();
4630 dma_free_pagelist(freelist);
4631
4632 start_vpfn = iova->pfn_hi + 1;
4633 free_iova_mem(iova);
4634 }
4635 break;
4636 }
4637
4638 return NOTIFY_OK;
4639 }
4640
4641 static struct notifier_block intel_iommu_memory_nb = {
4642 .notifier_call = intel_iommu_memory_notifier,
4643 .priority = 0
4644 };
4645
4646 static void free_all_cpu_cached_iovas(unsigned int cpu)
4647 {
4648 int i;
4649
4650 for (i = 0; i < g_num_of_iommus; i++) {
4651 struct intel_iommu *iommu = g_iommus[i];
4652 struct dmar_domain *domain;
4653 int did;
4654
4655 if (!iommu)
4656 continue;
4657
4658 for (did = 0; did < cap_ndoms(iommu->cap); did++) {
4659 domain = get_iommu_domain(iommu, (u16)did);
4660
4661 if (!domain)
4662 continue;
4663 free_cpu_cached_iovas(cpu, &domain->iovad);
4664 }
4665 }
4666 }
4667
4668 static int intel_iommu_cpu_notifier(struct notifier_block *nfb,
4669 unsigned long action, void *v)
4670 {
4671 unsigned int cpu = (unsigned long)v;
4672
4673 switch (action) {
4674 case CPU_DEAD:
4675 case CPU_DEAD_FROZEN:
4676 free_all_cpu_cached_iovas(cpu);
4677 flush_unmaps_timeout(cpu);
4678 break;
4679 }
4680 return NOTIFY_OK;
4681 }
4682
4683 static struct notifier_block intel_iommu_cpu_nb = {
4684 .notifier_call = intel_iommu_cpu_notifier,
4685 };
4686
4687 static ssize_t intel_iommu_show_version(struct device *dev,
4688 struct device_attribute *attr,
4689 char *buf)
4690 {
4691 struct intel_iommu *iommu = dev_get_drvdata(dev);
4692 u32 ver = readl(iommu->reg + DMAR_VER_REG);
4693 return sprintf(buf, "%d:%d\n",
4694 DMAR_VER_MAJOR(ver), DMAR_VER_MINOR(ver));
4695 }
4696 static DEVICE_ATTR(version, S_IRUGO, intel_iommu_show_version, NULL);
4697
4698 static ssize_t intel_iommu_show_address(struct device *dev,
4699 struct device_attribute *attr,
4700 char *buf)
4701 {
4702 struct intel_iommu *iommu = dev_get_drvdata(dev);
4703 return sprintf(buf, "%llx\n", iommu->reg_phys);
4704 }
4705 static DEVICE_ATTR(address, S_IRUGO, intel_iommu_show_address, NULL);
4706
4707 static ssize_t intel_iommu_show_cap(struct device *dev,
4708 struct device_attribute *attr,
4709 char *buf)
4710 {
4711 struct intel_iommu *iommu = dev_get_drvdata(dev);
4712 return sprintf(buf, "%llx\n", iommu->cap);
4713 }
4714 static DEVICE_ATTR(cap, S_IRUGO, intel_iommu_show_cap, NULL);
4715
4716 static ssize_t intel_iommu_show_ecap(struct device *dev,
4717 struct device_attribute *attr,
4718 char *buf)
4719 {
4720 struct intel_iommu *iommu = dev_get_drvdata(dev);
4721 return sprintf(buf, "%llx\n", iommu->ecap);
4722 }
4723 static DEVICE_ATTR(ecap, S_IRUGO, intel_iommu_show_ecap, NULL);
4724
4725 static ssize_t intel_iommu_show_ndoms(struct device *dev,
4726 struct device_attribute *attr,
4727 char *buf)
4728 {
4729 struct intel_iommu *iommu = dev_get_drvdata(dev);
4730 return sprintf(buf, "%ld\n", cap_ndoms(iommu->cap));
4731 }
4732 static DEVICE_ATTR(domains_supported, S_IRUGO, intel_iommu_show_ndoms, NULL);
4733
4734 static ssize_t intel_iommu_show_ndoms_used(struct device *dev,
4735 struct device_attribute *attr,
4736 char *buf)
4737 {
4738 struct intel_iommu *iommu = dev_get_drvdata(dev);
4739 return sprintf(buf, "%d\n", bitmap_weight(iommu->domain_ids,
4740 cap_ndoms(iommu->cap)));
4741 }
4742 static DEVICE_ATTR(domains_used, S_IRUGO, intel_iommu_show_ndoms_used, NULL);
4743
4744 static struct attribute *intel_iommu_attrs[] = {
4745 &dev_attr_version.attr,
4746 &dev_attr_address.attr,
4747 &dev_attr_cap.attr,
4748 &dev_attr_ecap.attr,
4749 &dev_attr_domains_supported.attr,
4750 &dev_attr_domains_used.attr,
4751 NULL,
4752 };
4753
4754 static struct attribute_group intel_iommu_group = {
4755 .name = "intel-iommu",
4756 .attrs = intel_iommu_attrs,
4757 };
4758
4759 const struct attribute_group *intel_iommu_groups[] = {
4760 &intel_iommu_group,
4761 NULL,
4762 };
4763
4764 int __init intel_iommu_init(void)
4765 {
4766 int ret = -ENODEV;
4767 struct dmar_drhd_unit *drhd;
4768 struct intel_iommu *iommu;
4769
4770 /* VT-d is required for a TXT/tboot launch, so enforce that */
4771 force_on = tboot_force_iommu();
4772
4773 if (iommu_init_mempool()) {
4774 if (force_on)
4775 panic("tboot: Failed to initialize iommu memory\n");
4776 return -ENOMEM;
4777 }
4778
4779 down_write(&dmar_global_lock);
4780 if (dmar_table_init()) {
4781 if (force_on)
4782 panic("tboot: Failed to initialize DMAR table\n");
4783 goto out_free_dmar;
4784 }
4785
4786 if (dmar_dev_scope_init() < 0) {
4787 if (force_on)
4788 panic("tboot: Failed to initialize DMAR device scope\n");
4789 goto out_free_dmar;
4790 }
4791
4792 if (no_iommu || dmar_disabled)
4793 goto out_free_dmar;
4794
4795 if (list_empty(&dmar_rmrr_units))
4796 pr_info("No RMRR found\n");
4797
4798 if (list_empty(&dmar_atsr_units))
4799 pr_info("No ATSR found\n");
4800
4801 if (dmar_init_reserved_ranges()) {
4802 if (force_on)
4803 panic("tboot: Failed to reserve iommu ranges\n");
4804 goto out_free_reserved_range;
4805 }
4806
4807 init_no_remapping_devices();
4808
4809 ret = init_dmars();
4810 if (ret) {
4811 if (force_on)
4812 panic("tboot: Failed to initialize DMARs\n");
4813 pr_err("Initialization failed\n");
4814 goto out_free_reserved_range;
4815 }
4816 up_write(&dmar_global_lock);
4817 pr_info("Intel(R) Virtualization Technology for Directed I/O\n");
4818
4819 #ifdef CONFIG_SWIOTLB
4820 swiotlb = 0;
4821 #endif
4822 dma_ops = &intel_dma_ops;
4823
4824 init_iommu_pm_ops();
4825
4826 for_each_active_iommu(iommu, drhd)
4827 iommu->iommu_dev = iommu_device_create(NULL, iommu,
4828 intel_iommu_groups,
4829 "%s", iommu->name);
4830
4831 bus_set_iommu(&pci_bus_type, &intel_iommu_ops);
4832 bus_register_notifier(&pci_bus_type, &device_nb);
4833 if (si_domain && !hw_pass_through)
4834 register_memory_notifier(&intel_iommu_memory_nb);
4835 register_hotcpu_notifier(&intel_iommu_cpu_nb);
4836
4837 intel_iommu_enabled = 1;
4838
4839 return 0;
4840
4841 out_free_reserved_range:
4842 put_iova_domain(&reserved_iova_list);
4843 out_free_dmar:
4844 intel_iommu_free_dmars();
4845 up_write(&dmar_global_lock);
4846 iommu_exit_mempool();
4847 return ret;
4848 }
4849
4850 static int domain_context_clear_one_cb(struct pci_dev *pdev, u16 alias, void *opaque)
4851 {
4852 struct intel_iommu *iommu = opaque;
4853
4854 domain_context_clear_one(iommu, PCI_BUS_NUM(alias), alias & 0xff);
4855 return 0;
4856 }
4857
4858 /*
4859 * NB - intel-iommu lacks any sort of reference counting for the users of
4860 * dependent devices. If multiple endpoints have intersecting dependent
4861 * devices, unbinding the driver from any one of them will possibly leave
4862 * the others unable to operate.
4863 */
4864 static void domain_context_clear(struct intel_iommu *iommu, struct device *dev)
4865 {
4866 if (!iommu || !dev || !dev_is_pci(dev))
4867 return;
4868
4869 pci_for_each_dma_alias(to_pci_dev(dev), &domain_context_clear_one_cb, iommu);
4870 }
4871
4872 static void __dmar_remove_one_dev_info(struct device_domain_info *info)
4873 {
4874 struct intel_iommu *iommu;
4875 unsigned long flags;
4876
4877 assert_spin_locked(&device_domain_lock);
4878
4879 if (WARN_ON(!info))
4880 return;
4881
4882 iommu = info->iommu;
4883
4884 if (info->dev) {
4885 iommu_disable_dev_iotlb(info);
4886 domain_context_clear(iommu, info->dev);
4887 }
4888
4889 unlink_domain_info(info);
4890
4891 spin_lock_irqsave(&iommu->lock, flags);
4892 domain_detach_iommu(info->domain, iommu);
4893 spin_unlock_irqrestore(&iommu->lock, flags);
4894
4895 free_devinfo_mem(info);
4896 }
4897
4898 static void dmar_remove_one_dev_info(struct dmar_domain *domain,
4899 struct device *dev)
4900 {
4901 struct device_domain_info *info;
4902 unsigned long flags;
4903
4904 spin_lock_irqsave(&device_domain_lock, flags);
4905 info = dev->archdata.iommu;
4906 __dmar_remove_one_dev_info(info);
4907 spin_unlock_irqrestore(&device_domain_lock, flags);
4908 }
4909
4910 static int md_domain_init(struct dmar_domain *domain, int guest_width)
4911 {
4912 int adjust_width;
4913
4914 init_iova_domain(&domain->iovad, VTD_PAGE_SIZE, IOVA_START_PFN,
4915 DMA_32BIT_PFN);
4916 domain_reserve_special_ranges(domain);
4917
4918 /* calculate AGAW */
4919 domain->gaw = guest_width;
4920 adjust_width = guestwidth_to_adjustwidth(guest_width);
4921 domain->agaw = width_to_agaw(adjust_width);
4922
4923 domain->iommu_coherency = 0;
4924 domain->iommu_snooping = 0;
4925 domain->iommu_superpage = 0;
4926 domain->max_addr = 0;
4927
4928 /* always allocate the top pgd */
4929 domain->pgd = (struct dma_pte *)alloc_pgtable_page(domain->nid);
4930 if (!domain->pgd)
4931 return -ENOMEM;
4932 domain_flush_cache(domain, domain->pgd, PAGE_SIZE);
4933 return 0;
4934 }
4935
4936 static struct iommu_domain *intel_iommu_domain_alloc(unsigned type)
4937 {
4938 struct dmar_domain *dmar_domain;
4939 struct iommu_domain *domain;
4940
4941 if (type != IOMMU_DOMAIN_UNMANAGED)
4942 return NULL;
4943
4944 dmar_domain = alloc_domain(DOMAIN_FLAG_VIRTUAL_MACHINE);
4945 if (!dmar_domain) {
4946 pr_err("Can't allocate dmar_domain\n");
4947 return NULL;
4948 }
4949 if (md_domain_init(dmar_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
4950 pr_err("Domain initialization failed\n");
4951 domain_exit(dmar_domain);
4952 return NULL;
4953 }
4954 domain_update_iommu_cap(dmar_domain);
4955
4956 domain = &dmar_domain->domain;
4957 domain->geometry.aperture_start = 0;
4958 domain->geometry.aperture_end = __DOMAIN_MAX_ADDR(dmar_domain->gaw);
4959 domain->geometry.force_aperture = true;
4960
4961 return domain;
4962 }
4963
4964 static void intel_iommu_domain_free(struct iommu_domain *domain)
4965 {
4966 domain_exit(to_dmar_domain(domain));
4967 }
4968
4969 static int intel_iommu_attach_device(struct iommu_domain *domain,
4970 struct device *dev)
4971 {
4972 struct dmar_domain *dmar_domain = to_dmar_domain(domain);
4973 struct intel_iommu *iommu;
4974 int addr_width;
4975 u8 bus, devfn;
4976
4977 if (device_is_rmrr_locked(dev)) {
4978 dev_warn(dev, "Device is ineligible for IOMMU domain attach due to platform RMRR requirement. Contact your platform vendor.\n");
4979 return -EPERM;
4980 }
4981
4982 /* normally dev is not mapped */
4983 if (unlikely(domain_context_mapped(dev))) {
4984 struct dmar_domain *old_domain;
4985
4986 old_domain = find_domain(dev);
4987 if (old_domain) {
4988 rcu_read_lock();
4989 dmar_remove_one_dev_info(old_domain, dev);
4990 rcu_read_unlock();
4991
4992 if (!domain_type_is_vm_or_si(old_domain) &&
4993 list_empty(&old_domain->devices))
4994 domain_exit(old_domain);
4995 }
4996 }
4997
4998 iommu = device_to_iommu(dev, &bus, &devfn);
4999 if (!iommu)
5000 return -ENODEV;
5001
5002 /* check if this iommu agaw is sufficient for max mapped address */
5003 addr_width = agaw_to_width(iommu->agaw);
5004 if (addr_width > cap_mgaw(iommu->cap))
5005 addr_width = cap_mgaw(iommu->cap);
5006
5007 if (dmar_domain->max_addr > (1LL << addr_width)) {
5008 pr_err("%s: iommu width (%d) is not "
5009 "sufficient for the mapped address (%llx)\n",
5010 __func__, addr_width, dmar_domain->max_addr);
5011 return -EFAULT;
5012 }
5013 dmar_domain->gaw = addr_width;
5014
5015 /*
5016 * Knock out extra levels of page tables if necessary
5017 */
5018 while (iommu->agaw < dmar_domain->agaw) {
5019 struct dma_pte *pte;
5020
5021 pte = dmar_domain->pgd;
5022 if (dma_pte_present(pte)) {
5023 dmar_domain->pgd = (struct dma_pte *)
5024 phys_to_virt(dma_pte_addr(pte));
5025 free_pgtable_page(pte);
5026 }
5027 dmar_domain->agaw--;
5028 }
5029
5030 return domain_add_dev_info(dmar_domain, dev);
5031 }
5032
5033 static void intel_iommu_detach_device(struct iommu_domain *domain,
5034 struct device *dev)
5035 {
5036 dmar_remove_one_dev_info(to_dmar_domain(domain), dev);
5037 }
5038
5039 static int intel_iommu_map(struct iommu_domain *domain,
5040 unsigned long iova, phys_addr_t hpa,
5041 size_t size, int iommu_prot)
5042 {
5043 struct dmar_domain *dmar_domain = to_dmar_domain(domain);
5044 u64 max_addr;
5045 int prot = 0;
5046 int ret;
5047
5048 if (iommu_prot & IOMMU_READ)
5049 prot |= DMA_PTE_READ;
5050 if (iommu_prot & IOMMU_WRITE)
5051 prot |= DMA_PTE_WRITE;
5052 if ((iommu_prot & IOMMU_CACHE) && dmar_domain->iommu_snooping)
5053 prot |= DMA_PTE_SNP;
5054
5055 max_addr = iova + size;
5056 if (dmar_domain->max_addr < max_addr) {
5057 u64 end;
5058
5059 /* check if minimum agaw is sufficient for mapped address */
5060 end = __DOMAIN_MAX_ADDR(dmar_domain->gaw) + 1;
5061 if (end < max_addr) {
5062 pr_err("%s: iommu width (%d) is not "
5063 "sufficient for the mapped address (%llx)\n",
5064 __func__, dmar_domain->gaw, max_addr);
5065 return -EFAULT;
5066 }
5067 dmar_domain->max_addr = max_addr;
5068 }
5069 /* Round up size to next multiple of PAGE_SIZE, if it and
5070 the low bits of hpa would take us onto the next page */
5071 size = aligned_nrpages(hpa, size);
5072 ret = domain_pfn_mapping(dmar_domain, iova >> VTD_PAGE_SHIFT,
5073 hpa >> VTD_PAGE_SHIFT, size, prot);
5074 return ret;
5075 }
5076
5077 static size_t intel_iommu_unmap(struct iommu_domain *domain,
5078 unsigned long iova, size_t size)
5079 {
5080 struct dmar_domain *dmar_domain = to_dmar_domain(domain);
5081 struct page *freelist = NULL;
5082 struct intel_iommu *iommu;
5083 unsigned long start_pfn, last_pfn;
5084 unsigned int npages;
5085 int iommu_id, level = 0;
5086
5087 /* Cope with horrid API which requires us to unmap more than the
5088 size argument if it happens to be a large-page mapping. */
5089 BUG_ON(!pfn_to_dma_pte(dmar_domain, iova >> VTD_PAGE_SHIFT, &level));
5090
5091 if (size < VTD_PAGE_SIZE << level_to_offset_bits(level))
5092 size = VTD_PAGE_SIZE << level_to_offset_bits(level);
5093
5094 start_pfn = iova >> VTD_PAGE_SHIFT;
5095 last_pfn = (iova + size - 1) >> VTD_PAGE_SHIFT;
5096
5097 freelist = domain_unmap(dmar_domain, start_pfn, last_pfn);
5098
5099 npages = last_pfn - start_pfn + 1;
5100
5101 for_each_domain_iommu(iommu_id, dmar_domain) {
5102 iommu = g_iommus[iommu_id];
5103
5104 iommu_flush_iotlb_psi(g_iommus[iommu_id], dmar_domain,
5105 start_pfn, npages, !freelist, 0);
5106 }
5107
5108 dma_free_pagelist(freelist);
5109
5110 if (dmar_domain->max_addr == iova + size)
5111 dmar_domain->max_addr = iova;
5112
5113 return size;
5114 }
5115
5116 static phys_addr_t intel_iommu_iova_to_phys(struct iommu_domain *domain,
5117 dma_addr_t iova)
5118 {
5119 struct dmar_domain *dmar_domain = to_dmar_domain(domain);
5120 struct dma_pte *pte;
5121 int level = 0;
5122 u64 phys = 0;
5123
5124 pte = pfn_to_dma_pte(dmar_domain, iova >> VTD_PAGE_SHIFT, &level);
5125 if (pte)
5126 phys = dma_pte_addr(pte);
5127
5128 return phys;
5129 }
5130
5131 static bool intel_iommu_capable(enum iommu_cap cap)
5132 {
5133 if (cap == IOMMU_CAP_CACHE_COHERENCY)
5134 return domain_update_iommu_snooping(NULL) == 1;
5135 if (cap == IOMMU_CAP_INTR_REMAP)
5136 return irq_remapping_enabled == 1;
5137
5138 return false;
5139 }
5140
5141 static int intel_iommu_add_device(struct device *dev)
5142 {
5143 struct intel_iommu *iommu;
5144 struct iommu_group *group;
5145 u8 bus, devfn;
5146
5147 iommu = device_to_iommu(dev, &bus, &devfn);
5148 if (!iommu)
5149 return -ENODEV;
5150
5151 iommu_device_link(iommu->iommu_dev, dev);
5152
5153 group = iommu_group_get_for_dev(dev);
5154
5155 if (IS_ERR(group))
5156 return PTR_ERR(group);
5157
5158 iommu_group_put(group);
5159 return 0;
5160 }
5161
5162 static void intel_iommu_remove_device(struct device *dev)
5163 {
5164 struct intel_iommu *iommu;
5165 u8 bus, devfn;
5166
5167 iommu = device_to_iommu(dev, &bus, &devfn);
5168 if (!iommu)
5169 return;
5170
5171 iommu_group_remove_device(dev);
5172
5173 iommu_device_unlink(iommu->iommu_dev, dev);
5174 }
5175
5176 #ifdef CONFIG_INTEL_IOMMU_SVM
5177 int intel_iommu_enable_pasid(struct intel_iommu *iommu, struct intel_svm_dev *sdev)
5178 {
5179 struct device_domain_info *info;
5180 struct context_entry *context;
5181 struct dmar_domain *domain;
5182 unsigned long flags;
5183 u64 ctx_lo;
5184 int ret;
5185
5186 domain = get_valid_domain_for_dev(sdev->dev);
5187 if (!domain)
5188 return -EINVAL;
5189
5190 spin_lock_irqsave(&device_domain_lock, flags);
5191 spin_lock(&iommu->lock);
5192
5193 ret = -EINVAL;
5194 info = sdev->dev->archdata.iommu;
5195 if (!info || !info->pasid_supported)
5196 goto out;
5197
5198 context = iommu_context_addr(iommu, info->bus, info->devfn, 0);
5199 if (WARN_ON(!context))
5200 goto out;
5201
5202 ctx_lo = context[0].lo;
5203
5204 sdev->did = domain->iommu_did[iommu->seq_id];
5205 sdev->sid = PCI_DEVID(info->bus, info->devfn);
5206
5207 if (!(ctx_lo & CONTEXT_PASIDE)) {
5208 context[1].hi = (u64)virt_to_phys(iommu->pasid_state_table);
5209 context[1].lo = (u64)virt_to_phys(iommu->pasid_table) | ecap_pss(iommu->ecap);
5210 wmb();
5211 /* CONTEXT_TT_MULTI_LEVEL and CONTEXT_TT_DEV_IOTLB are both
5212 * extended to permit requests-with-PASID if the PASIDE bit
5213 * is set. which makes sense. For CONTEXT_TT_PASS_THROUGH,
5214 * however, the PASIDE bit is ignored and requests-with-PASID
5215 * are unconditionally blocked. Which makes less sense.
5216 * So convert from CONTEXT_TT_PASS_THROUGH to one of the new
5217 * "guest mode" translation types depending on whether ATS
5218 * is available or not. Annoyingly, we can't use the new
5219 * modes *unless* PASIDE is set. */
5220 if ((ctx_lo & CONTEXT_TT_MASK) == (CONTEXT_TT_PASS_THROUGH << 2)) {
5221 ctx_lo &= ~CONTEXT_TT_MASK;
5222 if (info->ats_supported)
5223 ctx_lo |= CONTEXT_TT_PT_PASID_DEV_IOTLB << 2;
5224 else
5225 ctx_lo |= CONTEXT_TT_PT_PASID << 2;
5226 }
5227 ctx_lo |= CONTEXT_PASIDE;
5228 if (iommu->pasid_state_table)
5229 ctx_lo |= CONTEXT_DINVE;
5230 if (info->pri_supported)
5231 ctx_lo |= CONTEXT_PRS;
5232 context[0].lo = ctx_lo;
5233 wmb();
5234 iommu->flush.flush_context(iommu, sdev->did, sdev->sid,
5235 DMA_CCMD_MASK_NOBIT,
5236 DMA_CCMD_DEVICE_INVL);
5237 }
5238
5239 /* Enable PASID support in the device, if it wasn't already */
5240 if (!info->pasid_enabled)
5241 iommu_enable_dev_iotlb(info);
5242
5243 if (info->ats_enabled) {
5244 sdev->dev_iotlb = 1;
5245 sdev->qdep = info->ats_qdep;
5246 if (sdev->qdep >= QI_DEV_EIOTLB_MAX_INVS)
5247 sdev->qdep = 0;
5248 }
5249 ret = 0;
5250
5251 out:
5252 spin_unlock(&iommu->lock);
5253 spin_unlock_irqrestore(&device_domain_lock, flags);
5254
5255 return ret;
5256 }
5257
5258 struct intel_iommu *intel_svm_device_to_iommu(struct device *dev)
5259 {
5260 struct intel_iommu *iommu;
5261 u8 bus, devfn;
5262
5263 if (iommu_dummy(dev)) {
5264 dev_warn(dev,
5265 "No IOMMU translation for device; cannot enable SVM\n");
5266 return NULL;
5267 }
5268
5269 iommu = device_to_iommu(dev, &bus, &devfn);
5270 if ((!iommu)) {
5271 dev_err(dev, "No IOMMU for device; cannot enable SVM\n");
5272 return NULL;
5273 }
5274
5275 if (!iommu->pasid_table) {
5276 dev_err(dev, "PASID not enabled on IOMMU; cannot enable SVM\n");
5277 return NULL;
5278 }
5279
5280 return iommu;
5281 }
5282 #endif /* CONFIG_INTEL_IOMMU_SVM */
5283
5284 static const struct iommu_ops intel_iommu_ops = {
5285 .capable = intel_iommu_capable,
5286 .domain_alloc = intel_iommu_domain_alloc,
5287 .domain_free = intel_iommu_domain_free,
5288 .attach_dev = intel_iommu_attach_device,
5289 .detach_dev = intel_iommu_detach_device,
5290 .map = intel_iommu_map,
5291 .unmap = intel_iommu_unmap,
5292 .map_sg = default_iommu_map_sg,
5293 .iova_to_phys = intel_iommu_iova_to_phys,
5294 .add_device = intel_iommu_add_device,
5295 .remove_device = intel_iommu_remove_device,
5296 .device_group = pci_device_group,
5297 .pgsize_bitmap = INTEL_IOMMU_PGSIZES,
5298 };
5299
5300 static void quirk_iommu_g4x_gfx(struct pci_dev *dev)
5301 {
5302 /* G4x/GM45 integrated gfx dmar support is totally busted. */
5303 pr_info("Disabling IOMMU for graphics on this chipset\n");
5304 dmar_map_gfx = 0;
5305 }
5306
5307 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2a40, quirk_iommu_g4x_gfx);
5308 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e00, quirk_iommu_g4x_gfx);
5309 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e10, quirk_iommu_g4x_gfx);
5310 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e20, quirk_iommu_g4x_gfx);
5311 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e30, quirk_iommu_g4x_gfx);
5312 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e40, quirk_iommu_g4x_gfx);
5313 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e90, quirk_iommu_g4x_gfx);
5314
5315 static void quirk_iommu_rwbf(struct pci_dev *dev)
5316 {
5317 /*
5318 * Mobile 4 Series Chipset neglects to set RWBF capability,
5319 * but needs it. Same seems to hold for the desktop versions.
5320 */
5321 pr_info("Forcing write-buffer flush capability\n");
5322 rwbf_quirk = 1;
5323 }
5324
5325 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2a40, quirk_iommu_rwbf);
5326 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e00, quirk_iommu_rwbf);
5327 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e10, quirk_iommu_rwbf);
5328 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e20, quirk_iommu_rwbf);
5329 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e30, quirk_iommu_rwbf);
5330 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e40, quirk_iommu_rwbf);
5331 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e90, quirk_iommu_rwbf);
5332
5333 #define GGC 0x52
5334 #define GGC_MEMORY_SIZE_MASK (0xf << 8)
5335 #define GGC_MEMORY_SIZE_NONE (0x0 << 8)
5336 #define GGC_MEMORY_SIZE_1M (0x1 << 8)
5337 #define GGC_MEMORY_SIZE_2M (0x3 << 8)
5338 #define GGC_MEMORY_VT_ENABLED (0x8 << 8)
5339 #define GGC_MEMORY_SIZE_2M_VT (0x9 << 8)
5340 #define GGC_MEMORY_SIZE_3M_VT (0xa << 8)
5341 #define GGC_MEMORY_SIZE_4M_VT (0xb << 8)
5342
5343 static void quirk_calpella_no_shadow_gtt(struct pci_dev *dev)
5344 {
5345 unsigned short ggc;
5346
5347 if (pci_read_config_word(dev, GGC, &ggc))
5348 return;
5349
5350 if (!(ggc & GGC_MEMORY_VT_ENABLED)) {
5351 pr_info("BIOS has allocated no shadow GTT; disabling IOMMU for graphics\n");
5352 dmar_map_gfx = 0;
5353 } else if (dmar_map_gfx) {
5354 /* we have to ensure the gfx device is idle before we flush */
5355 pr_info("Disabling batched IOTLB flush on Ironlake\n");
5356 intel_iommu_strict = 1;
5357 }
5358 }
5359 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0040, quirk_calpella_no_shadow_gtt);
5360 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0044, quirk_calpella_no_shadow_gtt);
5361 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0062, quirk_calpella_no_shadow_gtt);
5362 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x006a, quirk_calpella_no_shadow_gtt);
5363
5364 /* On Tylersburg chipsets, some BIOSes have been known to enable the
5365 ISOCH DMAR unit for the Azalia sound device, but not give it any
5366 TLB entries, which causes it to deadlock. Check for that. We do
5367 this in a function called from init_dmars(), instead of in a PCI
5368 quirk, because we don't want to print the obnoxious "BIOS broken"
5369 message if VT-d is actually disabled.
5370 */
5371 static void __init check_tylersburg_isoch(void)
5372 {
5373 struct pci_dev *pdev;
5374 uint32_t vtisochctrl;
5375
5376 /* If there's no Azalia in the system anyway, forget it. */
5377 pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x3a3e, NULL);
5378 if (!pdev)
5379 return;
5380 pci_dev_put(pdev);
5381
5382 /* System Management Registers. Might be hidden, in which case
5383 we can't do the sanity check. But that's OK, because the
5384 known-broken BIOSes _don't_ actually hide it, so far. */
5385 pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x342e, NULL);
5386 if (!pdev)
5387 return;
5388
5389 if (pci_read_config_dword(pdev, 0x188, &vtisochctrl)) {
5390 pci_dev_put(pdev);
5391 return;
5392 }
5393
5394 pci_dev_put(pdev);
5395
5396 /* If Azalia DMA is routed to the non-isoch DMAR unit, fine. */
5397 if (vtisochctrl & 1)
5398 return;
5399
5400 /* Drop all bits other than the number of TLB entries */
5401 vtisochctrl &= 0x1c;
5402
5403 /* If we have the recommended number of TLB entries (16), fine. */
5404 if (vtisochctrl == 0x10)
5405 return;
5406
5407 /* Zero TLB entries? You get to ride the short bus to school. */
5408 if (!vtisochctrl) {
5409 WARN(1, "Your BIOS is broken; DMA routed to ISOCH DMAR unit but no TLB space.\n"
5410 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
5411 dmi_get_system_info(DMI_BIOS_VENDOR),
5412 dmi_get_system_info(DMI_BIOS_VERSION),
5413 dmi_get_system_info(DMI_PRODUCT_VERSION));
5414 iommu_identity_mapping |= IDENTMAP_AZALIA;
5415 return;
5416 }
5417
5418 pr_warn("Recommended TLB entries for ISOCH unit is 16; your BIOS set %d\n",
5419 vtisochctrl);
5420 }
Linux kernel - Deliverables
This page took 0.262889 seconds and 5 git commands to generate.