projects / deliverable / linux.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
iommu/vt-d: fix wrong return value of dmar_table_init()
[deliverable/linux.git] / drivers / iommu / intel-iommu.c
CommitLineData
ba395927
KA		 1/*
2 * Copyright (c) 2006, Intel Corporation.
3 *
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms and conditions of the GNU General Public License,
6 * version 2, as published by the Free Software Foundation.
7 *
8 * This program is distributed in the hope it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
11 * more details.
12 *
13 * You should have received a copy of the GNU General Public License along with
14 * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
15 * Place - Suite 330, Boston, MA 02111-1307 USA.
16 *
98bcef56 17 * Copyright (C) 2006-2008 Intel Corporation
18 * Author: Ashok Raj <ashok.raj@intel.com>
19 * Author: Shaohua Li <shaohua.li@intel.com>
20 * Author: Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>
5b6985ce 21 * Author: Fenghua Yu <fenghua.yu@intel.com>
ba395927
KA		 22 */
23
24#include <linux/init.h>
25#include <linux/bitmap.h>
5e0d2a6f 26#include <linux/debugfs.h>
54485c30 27#include <linux/export.h>
ba395927
KA		 28#include <linux/slab.h>
29#include <linux/irq.h>
30#include <linux/interrupt.h>
ba395927
KA		 31#include <linux/spinlock.h>
32#include <linux/pci.h>
33#include <linux/dmar.h>
34#include <linux/dma-mapping.h>
35#include <linux/mempool.h>
5e0d2a6f 36#include <linux/timer.h>
38717946 37#include <linux/iova.h>
5d450806 38#include <linux/iommu.h>
38717946 39#include <linux/intel-iommu.h>
134fac3f 40#include <linux/syscore_ops.h>
69575d38 41#include <linux/tboot.h>
adb2fe02 42#include <linux/dmi.h>
5cdede24 43#include <linux/pci-ats.h>
0ee332c1 44#include <linux/memblock.h>
8a8f422d 45#include <asm/irq_remapping.h>
ba395927 46#include <asm/cacheflush.h>
46a7fa27 47#include <asm/iommu.h>
ba395927 48
078e1ee2 49#include "irq_remapping.h"
61e015ac 50#include "pci.h"
078e1ee2 51
5b6985ce
FY		 52#define ROOT_SIZE VTD_PAGE_SIZE
53#define CONTEXT_SIZE VTD_PAGE_SIZE
54
ba395927
KA		 55#define IS_GFX_DEVICE(pdev) ((pdev->class >> 16) == PCI_BASE_CLASS_DISPLAY)
56#define IS_ISA_DEVICE(pdev) ((pdev->class >> 8) == PCI_CLASS_BRIDGE_ISA)
e0fc7e0b 57#define IS_AZALIA(pdev) ((pdev)->vendor == 0x8086 && (pdev)->device == 0x3a3e)
ba395927
KA		 58
59#define IOAPIC_RANGE_START (0xfee00000)
60#define IOAPIC_RANGE_END (0xfeefffff)
61#define IOVA_START_ADDR (0x1000)
62
63#define DEFAULT_DOMAIN_ADDRESS_WIDTH 48
64
4ed0d3e6 65#define MAX_AGAW_WIDTH 64
5c645b35 66#define MAX_AGAW_PFN_WIDTH (MAX_AGAW_WIDTH - VTD_PAGE_SHIFT)
4ed0d3e6 67
2ebe3151
DW		 68#define __DOMAIN_MAX_PFN(gaw) ((((uint64_t)1) << (gaw-VTD_PAGE_SHIFT)) - 1)
69#define __DOMAIN_MAX_ADDR(gaw) ((((uint64_t)1) << gaw) - 1)
70
71/* We limit DOMAIN_MAX_PFN to fit in an unsigned long, and DOMAIN_MAX_ADDR
72 to match. That way, we can use 'unsigned long' for PFNs with impunity. */
73#define DOMAIN_MAX_PFN(gaw) ((unsigned long) min_t(uint64_t, \
74 __DOMAIN_MAX_PFN(gaw), (unsigned long)-1))
75#define DOMAIN_MAX_ADDR(gaw) (((uint64_t)__DOMAIN_MAX_PFN(gaw)) << VTD_PAGE_SHIFT)
ba395927 76
f27be03b 77#define IOVA_PFN(addr) ((addr) >> PAGE_SHIFT)
284901a9 78#define DMA_32BIT_PFN IOVA_PFN(DMA_BIT_MASK(32))
6a35528a 79#define DMA_64BIT_PFN IOVA_PFN(DMA_BIT_MASK(64))
5e0d2a6f 80
df08cdc7
AM		 81/* page table handling */
82#define LEVEL_STRIDE (9)
83#define LEVEL_MASK (((u64)1 << LEVEL_STRIDE) - 1)
84
6d1c56a9
OBC		 85/*
86 * This bitmap is used to advertise the page sizes our hardware support
87 * to the IOMMU core, which will then use this information to split
88 * physically contiguous memory regions it is mapping into page sizes
89 * that we support.
90 *
91 * Traditionally the IOMMU core just handed us the mappings directly,
92 * after making sure the size is an order of a 4KiB page and that the
93 * mapping has natural alignment.
94 *
95 * To retain this behavior, we currently advertise that we support
96 * all page sizes that are an order of 4KiB.
97 *
98 * If at some point we'd like to utilize the IOMMU core's new behavior,
99 * we could change this to advertise the real page sizes we support.
100 */
101#define INTEL_IOMMU_PGSIZES (~0xFFFUL)
102
df08cdc7
AM		 103static inline int agaw_to_level(int agaw)
104{
105 return agaw + 2;
106}
107
108static inline int agaw_to_width(int agaw)
109{
5c645b35 110 return min_t(int, 30 + agaw * LEVEL_STRIDE, MAX_AGAW_WIDTH);
df08cdc7
AM		 111}
112
113static inline int width_to_agaw(int width)
114{
5c645b35 115 return DIV_ROUND_UP(width - 30, LEVEL_STRIDE);
df08cdc7
AM		 116}
117
118static inline unsigned int level_to_offset_bits(int level)
119{
120 return (level - 1) * LEVEL_STRIDE;
121}
122
123static inline int pfn_level_offset(unsigned long pfn, int level)
124{
125 return (pfn >> level_to_offset_bits(level)) & LEVEL_MASK;
126}
127
128static inline unsigned long level_mask(int level)
129{
130 return -1UL << level_to_offset_bits(level);
131}
132
133static inline unsigned long level_size(int level)
134{
135 return 1UL << level_to_offset_bits(level);
136}
137
138static inline unsigned long align_to_level(unsigned long pfn, int level)
139{
140 return (pfn + level_size(level) - 1) & level_mask(level);
141}
fd18de50 142
6dd9a7c7
YS		 143static inline unsigned long lvl_to_nr_pages(unsigned int lvl)
144{
5c645b35 145 return 1 << min_t(int, (lvl - 1) * LEVEL_STRIDE, MAX_AGAW_PFN_WIDTH);
6dd9a7c7
YS		 146}
147
dd4e8319
DW		 148/* VT-d pages must always be _smaller_ than MM pages. Otherwise things
149 are never going to work. */
150static inline unsigned long dma_to_mm_pfn(unsigned long dma_pfn)
151{
152 return dma_pfn >> (PAGE_SHIFT - VTD_PAGE_SHIFT);
153}
154
155static inline unsigned long mm_to_dma_pfn(unsigned long mm_pfn)
156{
157 return mm_pfn << (PAGE_SHIFT - VTD_PAGE_SHIFT);
158}
159static inline unsigned long page_to_dma_pfn(struct page *pg)
160{
161 return mm_to_dma_pfn(page_to_pfn(pg));
162}
163static inline unsigned long virt_to_dma_pfn(void *p)
164{
165 return page_to_dma_pfn(virt_to_page(p));
166}
167
d9630fe9
WH		 168/* global iommu list, set NULL for ignored DMAR units */
169static struct intel_iommu **g_iommus;
170
e0fc7e0b 171static void __init check_tylersburg_isoch(void);
9af88143
DW		 172static int rwbf_quirk;
173
b779260b
JC		 174/*
175 * set to 1 to panic kernel if can't successfully enable VT-d
176 * (used when kernel is launched w/ TXT)
177 */
178static int force_on = 0;
179
46b08e1a
MM		 180/*
181 * 0: Present
182 * 1-11: Reserved
183 * 12-63: Context Ptr (12 - (haw-1))
184 * 64-127: Reserved
185 */
186struct root_entry {
187 u64 val;
188 u64 rsvd1;
189};
190#define ROOT_ENTRY_NR (VTD_PAGE_SIZE/sizeof(struct root_entry))
191static inline bool root_present(struct root_entry *root)
192{
193 return (root->val & 1);
194}
195static inline void set_root_present(struct root_entry *root)
196{
197 root->val |= 1;
198}
199static inline void set_root_value(struct root_entry *root, unsigned long value)
200{
201 root->val |= value & VTD_PAGE_MASK;
202}
203
204static inline struct context_entry *
205get_context_addr_from_root(struct root_entry *root)
206{
207 return (struct context_entry *)
208 (root_present(root)?phys_to_virt(
209 root->val & VTD_PAGE_MASK) :
210 NULL);
211}
212
7a8fc25e
MM		 213/*
214 * low 64 bits:
215 * 0: present
216 * 1: fault processing disable
217 * 2-3: translation type
218 * 12-63: address space root
219 * high 64 bits:
220 * 0-2: address width
221 * 3-6: aval
222 * 8-23: domain id
223 */
224struct context_entry {
225 u64 lo;
226 u64 hi;
227};
c07e7d21
MM		 228
229static inline bool context_present(struct context_entry *context)
230{
231 return (context->lo & 1);
232}
233static inline void context_set_present(struct context_entry *context)
234{
235 context->lo |= 1;
236}
237
238static inline void context_set_fault_enable(struct context_entry *context)
239{
240 context->lo &= (((u64)-1) << 2) | 1;
241}
242
c07e7d21
MM		 243static inline void context_set_translation_type(struct context_entry *context,
244 unsigned long value)
245{
246 context->lo &= (((u64)-1) << 4) | 3;
247 context->lo |= (value & 3) << 2;
248}
249
250static inline void context_set_address_root(struct context_entry *context,
251 unsigned long value)
252{
253 context->lo |= value & VTD_PAGE_MASK;
254}
255
256static inline void context_set_address_width(struct context_entry *context,
257 unsigned long value)
258{
259 context->hi |= value & 7;
260}
261
262static inline void context_set_domain_id(struct context_entry *context,
263 unsigned long value)
264{
265 context->hi |= (value & ((1 << 16) - 1)) << 8;
266}
267
268static inline void context_clear_entry(struct context_entry *context)
269{
270 context->lo = 0;
271 context->hi = 0;
272}
7a8fc25e 273
622ba12a
MM		 274/*
275 * 0: readable
276 * 1: writable
277 * 2-6: reserved
278 * 7: super page
9cf06697
SY		 279 * 8-10: available
280 * 11: snoop behavior
622ba12a
MM		 281 * 12-63: Host physcial address
282 */
283struct dma_pte {
284 u64 val;
285};
622ba12a 286
19c239ce
MM		 287static inline void dma_clear_pte(struct dma_pte *pte)
288{
289 pte->val = 0;
290}
291
19c239ce
MM		 292static inline u64 dma_pte_addr(struct dma_pte *pte)
293{
c85994e4
DW		 294#ifdef CONFIG_64BIT
295 return pte->val & VTD_PAGE_MASK;
296#else
297 /* Must have a full atomic 64-bit read */
1a8bd481 298 return __cmpxchg64(&pte->val, 0ULL, 0ULL) & VTD_PAGE_MASK;
c85994e4 299#endif
19c239ce
MM		 300}
301
19c239ce
MM		 302static inline bool dma_pte_present(struct dma_pte *pte)
303{
304 return (pte->val & 3) != 0;
305}
622ba12a 306
4399c8bf
AK		 307static inline bool dma_pte_superpage(struct dma_pte *pte)
308{
309 return (pte->val & (1 << 7));
310}
311
75e6bf96
DW		 312static inline int first_pte_in_page(struct dma_pte *pte)
313{
314 return !((unsigned long)pte & ~VTD_PAGE_MASK);
315}
316
2c2e2c38
FY		 317/*
318 * This domain is a statically identity mapping domain.
319 * 1. This domain creats a static 1:1 mapping to all usable memory.
320 * 2. It maps to each iommu if successful.
321 * 3. Each iommu mapps to this domain if successful.
322 */
19943b0e
DW		 323static struct dmar_domain *si_domain;
324static int hw_pass_through = 1;
2c2e2c38 325
3b5410e7 326/* devices under the same p2p bridge are owned in one domain */
cdc7b837 327#define DOMAIN_FLAG_P2P_MULTIPLE_DEVICES (1 << 0)
3b5410e7 328
1ce28feb
WH		 329/* domain represents a virtual machine, more than one devices
330 * across iommus may be owned in one domain, e.g. kvm guest.
331 */
332#define DOMAIN_FLAG_VIRTUAL_MACHINE (1 << 1)
333
2c2e2c38
FY		 334/* si_domain contains mulitple devices */
335#define DOMAIN_FLAG_STATIC_IDENTITY (1 << 2)
336
1b198bb0
MT		 337/* define the limit of IOMMUs supported in each domain */
338#ifdef CONFIG_X86
339# define IOMMU_UNITS_SUPPORTED MAX_IO_APICS
340#else
341# define IOMMU_UNITS_SUPPORTED 64
342#endif
343
99126f7c
MM		 344struct dmar_domain {
345 int id; /* domain id */
4c923d47 346 int nid; /* node id */
1b198bb0
MT		 347 DECLARE_BITMAP(iommu_bmp, IOMMU_UNITS_SUPPORTED);
348 /* bitmap of iommus this domain uses*/
99126f7c
MM		 349
350 struct list_head devices; /* all devices' list */
351 struct iova_domain iovad; /* iova's that belong to this domain */
352
353 struct dma_pte *pgd; /* virtual address */
99126f7c
MM		 354 int gaw; /* max guest address width */
355
356 /* adjusted guest address width, 0 is level 2 30-bit */
357 int agaw;
358
3b5410e7 359 int flags; /* flags to find out type of domain */
8e604097
WH		 360
361 int iommu_coherency;/* indicate coherency of iommu access */
58c610bd 362 int iommu_snooping; /* indicate snooping control feature*/
c7151a8d 363 int iommu_count; /* reference count of iommu */
6dd9a7c7
YS		 364 int iommu_superpage;/* Level of superpages supported:
365 0 == 4KiB (no superpages), 1 == 2MiB,
366 2 == 1GiB, 3 == 512GiB, 4 == 1TiB */
c7151a8d 367 spinlock_t iommu_lock; /* protect iommu set in domain */
fe40f1e0 368 u64 max_addr; /* maximum mapped address */
99126f7c
MM		 369};
370
a647dacb
MM		 371/* PCI domain-device relationship */
372struct device_domain_info {
373 struct list_head link; /* link to domain siblings */
374 struct list_head global; /* link to global list */
276dbf99
DW		 375 int segment; /* PCI domain */
376 u8 bus; /* PCI bus number */
a647dacb 377 u8 devfn; /* PCI devfn number */
45e829ea 378 struct pci_dev *dev; /* it's NULL for PCIe-to-PCI bridge */
93a23a72 379 struct intel_iommu *iommu; /* IOMMU used by this device */
a647dacb
MM		 380 struct dmar_domain *domain; /* pointer to domain */
381};
382
5e0d2a6f 383static void flush_unmaps_timeout(unsigned long data);
384
385DEFINE_TIMER(unmap_timer, flush_unmaps_timeout, 0, 0);
386
80b20dd8 387#define HIGH_WATER_MARK 250
388struct deferred_flush_tables {
389 int next;
390 struct iova *iova[HIGH_WATER_MARK];
391 struct dmar_domain *domain[HIGH_WATER_MARK];
392};
393
394static struct deferred_flush_tables *deferred_flush;
395
5e0d2a6f 396/* bitmap for indexing intel_iommus */
5e0d2a6f 397static int g_num_of_iommus;
398
399static DEFINE_SPINLOCK(async_umap_flush_lock);
400static LIST_HEAD(unmaps_to_do);
401
402static int timer_on;
403static long list_size;
5e0d2a6f 404
ba395927
KA		 405static void domain_remove_dev_info(struct dmar_domain *domain);
406
d3f13810 407#ifdef CONFIG_INTEL_IOMMU_DEFAULT_ON
0cd5c3c8
KM		 408int dmar_disabled = 0;
409#else
410int dmar_disabled = 1;
d3f13810 411#endif /*CONFIG_INTEL_IOMMU_DEFAULT_ON*/
0cd5c3c8 412
8bc1f85c
ED		 413int intel_iommu_enabled = 0;
414EXPORT_SYMBOL_GPL(intel_iommu_enabled);
415
2d9e667e 416static int dmar_map_gfx = 1;
7d3b03ce 417static int dmar_forcedac;
5e0d2a6f 418static int intel_iommu_strict;
6dd9a7c7 419static int intel_iommu_superpage = 1;
ba395927 420
c0771df8
DW		 421int intel_iommu_gfx_mapped;
422EXPORT_SYMBOL_GPL(intel_iommu_gfx_mapped);
423
ba395927
KA		 424#define DUMMY_DEVICE_DOMAIN_INFO ((struct device_domain_info *)(-1))
425static DEFINE_SPINLOCK(device_domain_lock);
426static LIST_HEAD(device_domain_list);
427
a8bcbb0d
JR		 428static struct iommu_ops intel_iommu_ops;
429
ba395927
KA		 430static int __init intel_iommu_setup(char *str)
431{
432 if (!str)
433 return -EINVAL;
434 while (*str) {
0cd5c3c8
KM		 435 if (!strncmp(str, "on", 2)) {
436 dmar_disabled = 0;
437 printk(KERN_INFO "Intel-IOMMU: enabled\n");
438 } else if (!strncmp(str, "off", 3)) {
ba395927 439 dmar_disabled = 1;
0cd5c3c8 440 printk(KERN_INFO "Intel-IOMMU: disabled\n");
ba395927
KA		 441 } else if (!strncmp(str, "igfx_off", 8)) {
442 dmar_map_gfx = 0;
443 printk(KERN_INFO
444 "Intel-IOMMU: disable GFX device mapping\n");
7d3b03ce 445 } else if (!strncmp(str, "forcedac", 8)) {
5e0d2a6f 446 printk(KERN_INFO
7d3b03ce
KA		 447 "Intel-IOMMU: Forcing DAC for PCI devices\n");
448 dmar_forcedac = 1;
5e0d2a6f 449 } else if (!strncmp(str, "strict", 6)) {
450 printk(KERN_INFO
451 "Intel-IOMMU: disable batched IOTLB flush\n");
452 intel_iommu_strict = 1;
6dd9a7c7
YS		 453 } else if (!strncmp(str, "sp_off", 6)) {
454 printk(KERN_INFO
455 "Intel-IOMMU: disable supported super page\n");
456 intel_iommu_superpage = 0;
ba395927
KA		 457 }
458
459 str += strcspn(str, ",");
460 while (*str == ',')
461 str++;
462 }
463 return 0;
464}
465__setup("intel_iommu=", intel_iommu_setup);
466
467static struct kmem_cache *iommu_domain_cache;
468static struct kmem_cache *iommu_devinfo_cache;
469static struct kmem_cache *iommu_iova_cache;
470
4c923d47 471static inline void *alloc_pgtable_page(int node)
eb3fa7cb 472{
4c923d47
SS		 473 struct page *page;
474 void *vaddr = NULL;
eb3fa7cb 475
4c923d47
SS		 476 page = alloc_pages_node(node, GFP_ATOMIC | __GFP_ZERO, 0);
477 if (page)
478 vaddr = page_address(page);
eb3fa7cb 479 return vaddr;
ba395927
KA		 480}
481
482static inline void free_pgtable_page(void *vaddr)
483{
484 free_page((unsigned long)vaddr);
485}
486
487static inline void *alloc_domain_mem(void)
488{
354bb65e 489 return kmem_cache_alloc(iommu_domain_cache, GFP_ATOMIC);
ba395927
KA		 490}
491
38717946 492static void free_domain_mem(void *vaddr)
ba395927
KA		 493{
494 kmem_cache_free(iommu_domain_cache, vaddr);
495}
496
497static inline void * alloc_devinfo_mem(void)
498{
354bb65e 499 return kmem_cache_alloc(iommu_devinfo_cache, GFP_ATOMIC);
ba395927
KA		 500}
501
502static inline void free_devinfo_mem(void *vaddr)
503{
504 kmem_cache_free(iommu_devinfo_cache, vaddr);
505}
506
507struct iova *alloc_iova_mem(void)
508{
354bb65e 509 return kmem_cache_alloc(iommu_iova_cache, GFP_ATOMIC);
ba395927
KA		 510}
511
512void free_iova_mem(struct iova *iova)
513{
514 kmem_cache_free(iommu_iova_cache, iova);
515}
516
1b573683 517
4ed0d3e6 518static int __iommu_calculate_agaw(struct intel_iommu *iommu, int max_gaw)
1b573683
WH		 519{
520 unsigned long sagaw;
521 int agaw = -1;
522
523 sagaw = cap_sagaw(iommu->cap);
4ed0d3e6 524 for (agaw = width_to_agaw(max_gaw);
1b573683
WH		 525 agaw >= 0; agaw--) {
526 if (test_bit(agaw, &sagaw))
527 break;
528 }
529
530 return agaw;
531}
532
4ed0d3e6
FY		 533/*
534 * Calculate max SAGAW for each iommu.
535 */
536int iommu_calculate_max_sagaw(struct intel_iommu *iommu)
537{
538 return __iommu_calculate_agaw(iommu, MAX_AGAW_WIDTH);
539}
540
541/*
542 * calculate agaw for each iommu.
543 * "SAGAW" may be different across iommus, use a default agaw, and
544 * get a supported less agaw for iommus that don't support the default agaw.
545 */
546int iommu_calculate_agaw(struct intel_iommu *iommu)
547{
548 return __iommu_calculate_agaw(iommu, DEFAULT_DOMAIN_ADDRESS_WIDTH);
549}
550
2c2e2c38 551/* This functionin only returns single iommu in a domain */
8c11e798
WH		 552static struct intel_iommu *domain_get_iommu(struct dmar_domain *domain)
553{
554 int iommu_id;
555
2c2e2c38 556 /* si_domain and vm domain should not get here. */
1ce28feb 557 BUG_ON(domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE);
2c2e2c38 558 BUG_ON(domain->flags & DOMAIN_FLAG_STATIC_IDENTITY);
1ce28feb 559
1b198bb0 560 iommu_id = find_first_bit(domain->iommu_bmp, g_num_of_iommus);
8c11e798
WH		 561 if (iommu_id < 0 || iommu_id >= g_num_of_iommus)
562 return NULL;
563
564 return g_iommus[iommu_id];
565}
566
8e604097
WH		 567static void domain_update_iommu_coherency(struct dmar_domain *domain)
568{
569 int i;
570
2e12bc29
AW		 571 i = find_first_bit(domain->iommu_bmp, g_num_of_iommus);
572
573 domain->iommu_coherency = i < g_num_of_iommus ? 1 : 0;
8e604097 574
1b198bb0 575 for_each_set_bit(i, domain->iommu_bmp, g_num_of_iommus) {
8e604097
WH		 576 if (!ecap_coherent(g_iommus[i]->ecap)) {
577 domain->iommu_coherency = 0;
578 break;
579 }
8e604097
WH		 580 }
581}
582
58c610bd
SY		 583static void domain_update_iommu_snooping(struct dmar_domain *domain)
584{
585 int i;
586
587 domain->iommu_snooping = 1;
588
1b198bb0 589 for_each_set_bit(i, domain->iommu_bmp, g_num_of_iommus) {
58c610bd
SY		 590 if (!ecap_sc_support(g_iommus[i]->ecap)) {
591 domain->iommu_snooping = 0;
592 break;
593 }
58c610bd
SY		 594 }
595}
596
6dd9a7c7
YS		 597static void domain_update_iommu_superpage(struct dmar_domain *domain)
598{
8140a95d
AK		 599 struct dmar_drhd_unit *drhd;
600 struct intel_iommu *iommu = NULL;
601 int mask = 0xf;
6dd9a7c7
YS		 602
603 if (!intel_iommu_superpage) {
604 domain->iommu_superpage = 0;
605 return;
606 }
607
8140a95d
AK		 608 /* set iommu_superpage to the smallest common denominator */
609 for_each_active_iommu(iommu, drhd) {
610 mask &= cap_super_page_val(iommu->cap);
6dd9a7c7
YS		 611 if (!mask) {
612 break;
613 }
614 }
615 domain->iommu_superpage = fls(mask);
616}
617
58c610bd
SY		 618/* Some capabilities may be different across iommus */
619static void domain_update_iommu_cap(struct dmar_domain *domain)
620{
621 domain_update_iommu_coherency(domain);
622 domain_update_iommu_snooping(domain);
6dd9a7c7 623 domain_update_iommu_superpage(domain);
58c610bd
SY		 624}
625
276dbf99 626static struct intel_iommu *device_to_iommu(int segment, u8 bus, u8 devfn)
c7151a8d
WH		 627{
628 struct dmar_drhd_unit *drhd = NULL;
629 int i;
630
7c919779 631 for_each_active_drhd_unit(drhd) {
276dbf99
DW		 632 if (segment != drhd->segment)
633 continue;
c7151a8d 634
924b6231 635 for (i = 0; i < drhd->devices_cnt; i++) {
288e4877
DH		 636 if (drhd->devices[i] &&
637 drhd->devices[i]->bus->number == bus &&
c7151a8d
WH		 638 drhd->devices[i]->devfn == devfn)
639 return drhd->iommu;
4958c5dc
DW		 640 if (drhd->devices[i] &&
641 drhd->devices[i]->subordinate &&
924b6231 642 drhd->devices[i]->subordinate->number <= bus &&
b918c62e 643 drhd->devices[i]->subordinate->busn_res.end >= bus)
924b6231
DW		 644 return drhd->iommu;
645 }
c7151a8d
WH		 646
647 if (drhd->include_all)
648 return drhd->iommu;
649 }
650
651 return NULL;
652}
653
5331fe6f
WH		 654static void domain_flush_cache(struct dmar_domain *domain,
655 void *addr, int size)
656{
657 if (!domain->iommu_coherency)
658 clflush_cache_range(addr, size);
659}
660
ba395927
KA		 661/* Gets context entry for a given bus and devfn */
662static struct context_entry * device_to_context_entry(struct intel_iommu *iommu,
663 u8 bus, u8 devfn)
664{
665 struct root_entry *root;
666 struct context_entry *context;
667 unsigned long phy_addr;
668 unsigned long flags;
669
670 spin_lock_irqsave(&iommu->lock, flags);
671 root = &iommu->root_entry[bus];
672 context = get_context_addr_from_root(root);
673 if (!context) {
4c923d47
SS		 674 context = (struct context_entry *)
675 alloc_pgtable_page(iommu->node);
ba395927
KA		 676 if (!context) {
677 spin_unlock_irqrestore(&iommu->lock, flags);
678 return NULL;
679 }
5b6985ce 680 __iommu_flush_cache(iommu, (void *)context, CONTEXT_SIZE);
ba395927
KA		 681 phy_addr = virt_to_phys((void *)context);
682 set_root_value(root, phy_addr);
683 set_root_present(root);
684 __iommu_flush_cache(iommu, root, sizeof(*root));
685 }
686 spin_unlock_irqrestore(&iommu->lock, flags);
687 return &context[devfn];
688}
689
690static int device_context_mapped(struct intel_iommu *iommu, u8 bus, u8 devfn)
691{
692 struct root_entry *root;
693 struct context_entry *context;
694 int ret;
695 unsigned long flags;
696
697 spin_lock_irqsave(&iommu->lock, flags);
698 root = &iommu->root_entry[bus];
699 context = get_context_addr_from_root(root);
700 if (!context) {
701 ret = 0;
702 goto out;
703 }
c07e7d21 704 ret = context_present(&context[devfn]);
ba395927
KA		 705out:
706 spin_unlock_irqrestore(&iommu->lock, flags);
707 return ret;
708}
709
710static void clear_context_table(struct intel_iommu *iommu, u8 bus, u8 devfn)
711{
712 struct root_entry *root;
713 struct context_entry *context;
714 unsigned long flags;
715
716 spin_lock_irqsave(&iommu->lock, flags);
717 root = &iommu->root_entry[bus];
718 context = get_context_addr_from_root(root);
719 if (context) {
c07e7d21 720 context_clear_entry(&context[devfn]);
ba395927
KA		 721 __iommu_flush_cache(iommu, &context[devfn], \
722 sizeof(*context));
723 }
724 spin_unlock_irqrestore(&iommu->lock, flags);
725}
726
727static void free_context_table(struct intel_iommu *iommu)
728{
729 struct root_entry *root;
730 int i;
731 unsigned long flags;
732 struct context_entry *context;
733
734 spin_lock_irqsave(&iommu->lock, flags);
735 if (!iommu->root_entry) {
736 goto out;
737 }
738 for (i = 0; i < ROOT_ENTRY_NR; i++) {
739 root = &iommu->root_entry[i];
740 context = get_context_addr_from_root(root);
741 if (context)
742 free_pgtable_page(context);
743 }
744 free_pgtable_page(iommu->root_entry);
745 iommu->root_entry = NULL;
746out:
747 spin_unlock_irqrestore(&iommu->lock, flags);
748}
749
b026fd28 750static struct dma_pte *pfn_to_dma_pte(struct dmar_domain *domain,
4399c8bf 751 unsigned long pfn, int target_level)
ba395927 752{
b026fd28 753 int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
ba395927
KA		 754 struct dma_pte *parent, *pte = NULL;
755 int level = agaw_to_level(domain->agaw);
4399c8bf 756 int offset;
ba395927
KA		 757
758 BUG_ON(!domain->pgd);
f9423606
JS		 759
760 if (addr_width < BITS_PER_LONG && pfn >> addr_width)
761 /* Address beyond IOMMU's addressing capabilities. */
762 return NULL;
763
ba395927
KA		 764 parent = domain->pgd;
765
ba395927
KA		 766 while (level > 0) {
767 void *tmp_page;
768
b026fd28 769 offset = pfn_level_offset(pfn, level);
ba395927 770 pte = &parent[offset];
4399c8bf 771 if (!target_level && (dma_pte_superpage(pte) || !dma_pte_present(pte)))
6dd9a7c7
YS		 772 break;
773 if (level == target_level)
ba395927
KA		 774 break;
775
19c239ce 776 if (!dma_pte_present(pte)) {
c85994e4
DW		 777 uint64_t pteval;
778
4c923d47 779 tmp_page = alloc_pgtable_page(domain->nid);
ba395927 780
206a73c1 781 if (!tmp_page)
ba395927 782 return NULL;
206a73c1 783
c85994e4 784 domain_flush_cache(domain, tmp_page, VTD_PAGE_SIZE);
64de5af0 785 pteval = ((uint64_t)virt_to_dma_pfn(tmp_page) << VTD_PAGE_SHIFT) | DMA_PTE_READ | DMA_PTE_WRITE;
c85994e4
DW		 786 if (cmpxchg64(&pte->val, 0ULL, pteval)) {
787 /* Someone else set it while we were thinking; use theirs. */
788 free_pgtable_page(tmp_page);
789 } else {
790 dma_pte_addr(pte);
791 domain_flush_cache(domain, pte, sizeof(*pte));
792 }
ba395927 793 }
19c239ce 794 parent = phys_to_virt(dma_pte_addr(pte));
ba395927
KA		 795 level--;
796 }
797
ba395927
KA		 798 return pte;
799}
800
6dd9a7c7 801
ba395927 802/* return address's pte at specific level */
90dcfb5e
DW		 803static struct dma_pte *dma_pfn_level_pte(struct dmar_domain *domain,
804 unsigned long pfn,
6dd9a7c7 805 int level, int *large_page)
ba395927
KA		 806{
807 struct dma_pte *parent, *pte = NULL;
808 int total = agaw_to_level(domain->agaw);
809 int offset;
810
811 parent = domain->pgd;
812 while (level <= total) {
90dcfb5e 813 offset = pfn_level_offset(pfn, total);
ba395927
KA		 814 pte = &parent[offset];
815 if (level == total)
816 return pte;
817
6dd9a7c7
YS		 818 if (!dma_pte_present(pte)) {
819 *large_page = total;
ba395927 820 break;
6dd9a7c7
YS		 821 }
822
823 if (pte->val & DMA_PTE_LARGE_PAGE) {
824 *large_page = total;
825 return pte;
826 }
827
19c239ce 828 parent = phys_to_virt(dma_pte_addr(pte));
ba395927
KA		 829 total--;
830 }
831 return NULL;
832}
833
ba395927 834/* clear last level pte, a tlb flush should be followed */
292827cb 835static int dma_pte_clear_range(struct dmar_domain *domain,
595badf5
DW		 836 unsigned long start_pfn,
837 unsigned long last_pfn)
ba395927 838{
04b18e65 839 int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
6dd9a7c7 840 unsigned int large_page = 1;
310a5ab9 841 struct dma_pte *first_pte, *pte;
66eae846 842
04b18e65 843 BUG_ON(addr_width < BITS_PER_LONG && start_pfn >> addr_width);
595badf5 844 BUG_ON(addr_width < BITS_PER_LONG && last_pfn >> addr_width);
59c36286 845 BUG_ON(start_pfn > last_pfn);
ba395927 846
04b18e65 847 /* we don't need lock here; nobody else touches the iova range */
59c36286 848 do {
6dd9a7c7
YS		 849 large_page = 1;
850 first_pte = pte = dma_pfn_level_pte(domain, start_pfn, 1, &large_page);
310a5ab9 851 if (!pte) {
6dd9a7c7 852 start_pfn = align_to_level(start_pfn + 1, large_page + 1);
310a5ab9
DW		 853 continue;
854 }
6dd9a7c7 855 do {
310a5ab9 856 dma_clear_pte(pte);
6dd9a7c7 857 start_pfn += lvl_to_nr_pages(large_page);
310a5ab9 858 pte++;
75e6bf96
DW		 859 } while (start_pfn <= last_pfn && !first_pte_in_page(pte));
860
310a5ab9
DW		 861 domain_flush_cache(domain, first_pte,
862 (void *)pte - (void *)first_pte);
59c36286
DW		 863
864 } while (start_pfn && start_pfn <= last_pfn);
292827cb 865
5c645b35 866 return min_t(int, (large_page - 1) * 9, MAX_AGAW_PFN_WIDTH);
ba395927
KA		 867}
868
3269ee0b
AW		 869static void dma_pte_free_level(struct dmar_domain *domain, int level,
870 struct dma_pte *pte, unsigned long pfn,
871 unsigned long start_pfn, unsigned long last_pfn)
872{
873 pfn = max(start_pfn, pfn);
874 pte = &pte[pfn_level_offset(pfn, level)];
875
876 do {
877 unsigned long level_pfn;
878 struct dma_pte *level_pte;
879
880 if (!dma_pte_present(pte) || dma_pte_superpage(pte))
881 goto next;
882
883 level_pfn = pfn & level_mask(level - 1);
884 level_pte = phys_to_virt(dma_pte_addr(pte));
885
886 if (level > 2)
887 dma_pte_free_level(domain, level - 1, level_pte,
888 level_pfn, start_pfn, last_pfn);
889
890 /* If range covers entire pagetable, free it */
891 if (!(start_pfn > level_pfn ||
892 last_pfn < level_pfn + level_size(level))) {
893 dma_clear_pte(pte);
894 domain_flush_cache(domain, pte, sizeof(*pte));
895 free_pgtable_page(level_pte);
896 }
897next:
898 pfn += level_size(level);
899 } while (!first_pte_in_page(++pte) && pfn <= last_pfn);
900}
901
ba395927
KA		 902/* free page table pages. last level pte should already be cleared */
903static void dma_pte_free_pagetable(struct dmar_domain *domain,
d794dc9b
DW		 904 unsigned long start_pfn,
905 unsigned long last_pfn)
ba395927 906{
6660c63a 907 int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
ba395927 908
6660c63a
DW		 909 BUG_ON(addr_width < BITS_PER_LONG && start_pfn >> addr_width);
910 BUG_ON(addr_width < BITS_PER_LONG && last_pfn >> addr_width);
59c36286 911 BUG_ON(start_pfn > last_pfn);
ba395927 912
f3a0a52f 913 /* We don't need lock here; nobody else touches the iova range */
3269ee0b
AW		 914 dma_pte_free_level(domain, agaw_to_level(domain->agaw),
915 domain->pgd, 0, start_pfn, last_pfn);
6660c63a 916
ba395927 917 /* free pgd */
d794dc9b 918 if (start_pfn == 0 && last_pfn == DOMAIN_MAX_PFN(domain->gaw)) {
ba395927
KA		 919 free_pgtable_page(domain->pgd);
920 domain->pgd = NULL;
921 }
922}
923
924/* iommu handling */
925static int iommu_alloc_root_entry(struct intel_iommu *iommu)
926{
927 struct root_entry *root;
928 unsigned long flags;
929
4c923d47 930 root = (struct root_entry *)alloc_pgtable_page(iommu->node);
ba395927
KA		 931 if (!root)
932 return -ENOMEM;
933
5b6985ce 934 __iommu_flush_cache(iommu, root, ROOT_SIZE);
ba395927
KA		 935
936 spin_lock_irqsave(&iommu->lock, flags);
937 iommu->root_entry = root;
938 spin_unlock_irqrestore(&iommu->lock, flags);
939
940 return 0;
941}
942
ba395927
KA		 943static void iommu_set_root_entry(struct intel_iommu *iommu)
944{
945 void *addr;
c416daa9 946 u32 sts;
ba395927
KA		 947 unsigned long flag;
948
949 addr = iommu->root_entry;
950
1f5b3c3f 951 raw_spin_lock_irqsave(&iommu->register_lock, flag);
ba395927
KA		 952 dmar_writeq(iommu->reg + DMAR_RTADDR_REG, virt_to_phys(addr));
953
c416daa9 954 writel(iommu->gcmd | DMA_GCMD_SRTP, iommu->reg + DMAR_GCMD_REG);
ba395927
KA		 955
956 /* Make sure hardware complete it */
957 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
c416daa9 958 readl, (sts & DMA_GSTS_RTPS), sts);
ba395927 959
1f5b3c3f 960 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
ba395927
KA		 961}
962
963static void iommu_flush_write_buffer(struct intel_iommu *iommu)
964{
965 u32 val;
966 unsigned long flag;
967
9af88143 968 if (!rwbf_quirk && !cap_rwbf(iommu->cap))
ba395927 969 return;
ba395927 970
1f5b3c3f 971 raw_spin_lock_irqsave(&iommu->register_lock, flag);
462b60f6 972 writel(iommu->gcmd | DMA_GCMD_WBF, iommu->reg + DMAR_GCMD_REG);
ba395927
KA		 973
974 /* Make sure hardware complete it */
975 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
c416daa9 976 readl, (!(val & DMA_GSTS_WBFS)), val);
ba395927 977
1f5b3c3f 978 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
ba395927
KA		 979}
980
981/* return value determine if we need a write buffer flush */
4c25a2c1
DW		 982static void __iommu_flush_context(struct intel_iommu *iommu,
983 u16 did, u16 source_id, u8 function_mask,
984 u64 type)
ba395927
KA		 985{
986 u64 val = 0;
987 unsigned long flag;
988
ba395927
KA		 989 switch (type) {
990 case DMA_CCMD_GLOBAL_INVL:
991 val = DMA_CCMD_GLOBAL_INVL;
992 break;
993 case DMA_CCMD_DOMAIN_INVL:
994 val = DMA_CCMD_DOMAIN_INVL|DMA_CCMD_DID(did);
995 break;
996 case DMA_CCMD_DEVICE_INVL:
997 val = DMA_CCMD_DEVICE_INVL|DMA_CCMD_DID(did)
998 | DMA_CCMD_SID(source_id) | DMA_CCMD_FM(function_mask);
999 break;
1000 default:
1001 BUG();
1002 }
1003 val |= DMA_CCMD_ICC;
1004
1f5b3c3f 1005 raw_spin_lock_irqsave(&iommu->register_lock, flag);
ba395927
KA		 1006 dmar_writeq(iommu->reg + DMAR_CCMD_REG, val);
1007
1008 /* Make sure hardware complete it */
1009 IOMMU_WAIT_OP(iommu, DMAR_CCMD_REG,
1010 dmar_readq, (!(val & DMA_CCMD_ICC)), val);
1011
1f5b3c3f 1012 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
ba395927
KA		 1013}
1014
ba395927 1015/* return value determine if we need a write buffer flush */
1f0ef2aa
DW		 1016static void __iommu_flush_iotlb(struct intel_iommu *iommu, u16 did,
1017 u64 addr, unsigned int size_order, u64 type)
ba395927
KA		 1018{
1019 int tlb_offset = ecap_iotlb_offset(iommu->ecap);
1020 u64 val = 0, val_iva = 0;
1021 unsigned long flag;
1022
ba395927
KA		 1023 switch (type) {
1024 case DMA_TLB_GLOBAL_FLUSH:
1025 /* global flush doesn't need set IVA_REG */
1026 val = DMA_TLB_GLOBAL_FLUSH|DMA_TLB_IVT;
1027 break;
1028 case DMA_TLB_DSI_FLUSH:
1029 val = DMA_TLB_DSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
1030 break;
1031 case DMA_TLB_PSI_FLUSH:
1032 val = DMA_TLB_PSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
1033 /* Note: always flush non-leaf currently */
1034 val_iva = size_order | addr;
1035 break;
1036 default:
1037 BUG();
1038 }
1039 /* Note: set drain read/write */
1040#if 0
1041 /*
1042 * This is probably to be super secure.. Looks like we can
1043 * ignore it without any impact.
1044 */
1045 if (cap_read_drain(iommu->cap))
1046 val |= DMA_TLB_READ_DRAIN;
1047#endif
1048 if (cap_write_drain(iommu->cap))
1049 val |= DMA_TLB_WRITE_DRAIN;
1050
1f5b3c3f 1051 raw_spin_lock_irqsave(&iommu->register_lock, flag);
ba395927
KA		 1052 /* Note: Only uses first TLB reg currently */
1053 if (val_iva)
1054 dmar_writeq(iommu->reg + tlb_offset, val_iva);
1055 dmar_writeq(iommu->reg + tlb_offset + 8, val);
1056
1057 /* Make sure hardware complete it */
1058 IOMMU_WAIT_OP(iommu, tlb_offset + 8,
1059 dmar_readq, (!(val & DMA_TLB_IVT)), val);
1060
1f5b3c3f 1061 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
ba395927
KA		 1062
1063 /* check IOTLB invalidation granularity */
1064 if (DMA_TLB_IAIG(val) == 0)
1065 printk(KERN_ERR"IOMMU: flush IOTLB failed\n");
1066 if (DMA_TLB_IAIG(val) != DMA_TLB_IIRG(type))
1067 pr_debug("IOMMU: tlb flush request %Lx, actual %Lx\n",
5b6985ce
FY		 1068 (unsigned long long)DMA_TLB_IIRG(type),
1069 (unsigned long long)DMA_TLB_IAIG(val));
ba395927
KA		 1070}
1071
93a23a72
YZ		 1072static struct device_domain_info *iommu_support_dev_iotlb(
1073 struct dmar_domain *domain, int segment, u8 bus, u8 devfn)
1074{
1075 int found = 0;
1076 unsigned long flags;
1077 struct device_domain_info *info;
1078 struct intel_iommu *iommu = device_to_iommu(segment, bus, devfn);
1079
1080 if (!ecap_dev_iotlb_support(iommu->ecap))
1081 return NULL;
1082
1083 if (!iommu->qi)
1084 return NULL;
1085
1086 spin_lock_irqsave(&device_domain_lock, flags);
1087 list_for_each_entry(info, &domain->devices, link)
1088 if (info->bus == bus && info->devfn == devfn) {
1089 found = 1;
1090 break;
1091 }
1092 spin_unlock_irqrestore(&device_domain_lock, flags);
1093
1094 if (!found || !info->dev)
1095 return NULL;
1096
1097 if (!pci_find_ext_capability(info->dev, PCI_EXT_CAP_ID_ATS))
1098 return NULL;
1099
1100 if (!dmar_find_matched_atsr_unit(info->dev))
1101 return NULL;
1102
1103 info->iommu = iommu;
1104
1105 return info;
1106}
1107
1108static void iommu_enable_dev_iotlb(struct device_domain_info *info)
ba395927 1109{
93a23a72
YZ		 1110 if (!info)
1111 return;
1112
1113 pci_enable_ats(info->dev, VTD_PAGE_SHIFT);
1114}
1115
1116static void iommu_disable_dev_iotlb(struct device_domain_info *info)
1117{
1118 if (!info->dev || !pci_ats_enabled(info->dev))
1119 return;
1120
1121 pci_disable_ats(info->dev);
1122}
1123
1124static void iommu_flush_dev_iotlb(struct dmar_domain *domain,
1125 u64 addr, unsigned mask)
1126{
1127 u16 sid, qdep;
1128 unsigned long flags;
1129 struct device_domain_info *info;
1130
1131 spin_lock_irqsave(&device_domain_lock, flags);
1132 list_for_each_entry(info, &domain->devices, link) {
1133 if (!info->dev || !pci_ats_enabled(info->dev))
1134 continue;
1135
1136 sid = info->bus << 8 | info->devfn;
1137 qdep = pci_ats_queue_depth(info->dev);
1138 qi_flush_dev_iotlb(info->iommu, sid, qdep, addr, mask);
1139 }
1140 spin_unlock_irqrestore(&device_domain_lock, flags);
1141}
1142
1f0ef2aa 1143static void iommu_flush_iotlb_psi(struct intel_iommu *iommu, u16 did,
82653633 1144 unsigned long pfn, unsigned int pages, int map)
ba395927 1145{
9dd2fe89 1146 unsigned int mask = ilog2(__roundup_pow_of_two(pages));
03d6a246 1147 uint64_t addr = (uint64_t)pfn << VTD_PAGE_SHIFT;
ba395927 1148
ba395927
KA		 1149 BUG_ON(pages == 0);
1150
ba395927 1151 /*
9dd2fe89
YZ		 1152 * Fallback to domain selective flush if no PSI support or the size is
1153 * too big.
ba395927
KA		 1154 * PSI requires page size to be 2 ^ x, and the base address is naturally
1155 * aligned to the size
1156 */
9dd2fe89
YZ		 1157 if (!cap_pgsel_inv(iommu->cap) || mask > cap_max_amask_val(iommu->cap))
1158 iommu->flush.flush_iotlb(iommu, did, 0, 0,
1f0ef2aa 1159 DMA_TLB_DSI_FLUSH);
9dd2fe89
YZ		 1160 else
1161 iommu->flush.flush_iotlb(iommu, did, addr, mask,
1162 DMA_TLB_PSI_FLUSH);
bf92df30
YZ		 1163
1164 /*
82653633
NA		 1165 * In caching mode, changes of pages from non-present to present require
1166 * flush. However, device IOTLB doesn't need to be flushed in this case.
bf92df30 1167 */
82653633 1168 if (!cap_caching_mode(iommu->cap) || !map)
93a23a72 1169 iommu_flush_dev_iotlb(iommu->domains[did], addr, mask);
ba395927
KA		 1170}
1171
f8bab735 1172static void iommu_disable_protect_mem_regions(struct intel_iommu *iommu)
1173{
1174 u32 pmen;
1175 unsigned long flags;
1176
1f5b3c3f 1177 raw_spin_lock_irqsave(&iommu->register_lock, flags);
f8bab735 1178 pmen = readl(iommu->reg + DMAR_PMEN_REG);
1179 pmen &= ~DMA_PMEN_EPM;
1180 writel(pmen, iommu->reg + DMAR_PMEN_REG);
1181
1182 /* wait for the protected region status bit to clear */
1183 IOMMU_WAIT_OP(iommu, DMAR_PMEN_REG,
1184 readl, !(pmen & DMA_PMEN_PRS), pmen);
1185
1f5b3c3f 1186 raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
f8bab735 1187}
1188
ba395927
KA		 1189static int iommu_enable_translation(struct intel_iommu *iommu)
1190{
1191 u32 sts;
1192 unsigned long flags;
1193
1f5b3c3f 1194 raw_spin_lock_irqsave(&iommu->register_lock, flags);
c416daa9
DW		 1195 iommu->gcmd |= DMA_GCMD_TE;
1196 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
ba395927
KA		 1197
1198 /* Make sure hardware complete it */
1199 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
c416daa9 1200 readl, (sts & DMA_GSTS_TES), sts);
ba395927 1201
1f5b3c3f 1202 raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
ba395927
KA		 1203 return 0;
1204}
1205
1206static int iommu_disable_translation(struct intel_iommu *iommu)
1207{
1208 u32 sts;
1209 unsigned long flag;
1210
1f5b3c3f 1211 raw_spin_lock_irqsave(&iommu->register_lock, flag);
ba395927
KA		 1212 iommu->gcmd &= ~DMA_GCMD_TE;
1213 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1214
1215 /* Make sure hardware complete it */
1216 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
c416daa9 1217 readl, (!(sts & DMA_GSTS_TES)), sts);
ba395927 1218
1f5b3c3f 1219 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
ba395927
KA		 1220 return 0;
1221}
1222
3460a6d9 1223
ba395927
KA		 1224static int iommu_init_domains(struct intel_iommu *iommu)
1225{
1226 unsigned long ndomains;
1227 unsigned long nlongs;
1228
1229 ndomains = cap_ndoms(iommu->cap);
852bdb04
JL		 1230 pr_debug("IOMMU%d: Number of Domains supported <%ld>\n",
1231 iommu->seq_id, ndomains);
ba395927
KA		 1232 nlongs = BITS_TO_LONGS(ndomains);
1233
94a91b50
DD		 1234 spin_lock_init(&iommu->lock);
1235
ba395927
KA		 1236 /* TBD: there might be 64K domains,
1237 * consider other allocation for future chip
1238 */
1239 iommu->domain_ids = kcalloc(nlongs, sizeof(unsigned long), GFP_KERNEL);
1240 if (!iommu->domain_ids) {
852bdb04
JL		 1241 pr_err("IOMMU%d: allocating domain id array failed\n",
1242 iommu->seq_id);
ba395927
KA		 1243 return -ENOMEM;
1244 }
1245 iommu->domains = kcalloc(ndomains, sizeof(struct dmar_domain *),
1246 GFP_KERNEL);
1247 if (!iommu->domains) {
852bdb04
JL		 1248 pr_err("IOMMU%d: allocating domain array failed\n",
1249 iommu->seq_id);
1250 kfree(iommu->domain_ids);
1251 iommu->domain_ids = NULL;
ba395927
KA		 1252 return -ENOMEM;
1253 }
1254
1255 /*
1256 * if Caching mode is set, then invalid translations are tagged
1257 * with domainid 0. Hence we need to pre-allocate it.
1258 */
1259 if (cap_caching_mode(iommu->cap))
1260 set_bit(0, iommu->domain_ids);
1261 return 0;
1262}
ba395927 1263
ba395927
KA		 1264
1265static void domain_exit(struct dmar_domain *domain);
5e98c4b1 1266static void vm_domain_exit(struct dmar_domain *domain);
e61d98d8 1267
a868e6b7 1268static void free_dmar_iommu(struct intel_iommu *iommu)
ba395927
KA		 1269{
1270 struct dmar_domain *domain;
5ced12af 1271 int i, count;
c7151a8d 1272 unsigned long flags;
ba395927 1273
94a91b50 1274 if ((iommu->domains) && (iommu->domain_ids)) {
a45946ab 1275 for_each_set_bit(i, iommu->domain_ids, cap_ndoms(iommu->cap)) {
94a91b50
DD		 1276 domain = iommu->domains[i];
1277 clear_bit(i, iommu->domain_ids);
1278
1279 spin_lock_irqsave(&domain->iommu_lock, flags);
5ced12af
JL		 1280 count = --domain->iommu_count;
1281 spin_unlock_irqrestore(&domain->iommu_lock, flags);
1282 if (count == 0) {
94a91b50
DD		 1283 if (domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE)
1284 vm_domain_exit(domain);
1285 else
1286 domain_exit(domain);
1287 }
5e98c4b1 1288 }
ba395927
KA		 1289 }
1290
1291 if (iommu->gcmd & DMA_GCMD_TE)
1292 iommu_disable_translation(iommu);
1293
ba395927
KA		 1294 kfree(iommu->domains);
1295 kfree(iommu->domain_ids);
a868e6b7
JL		 1296 iommu->domains = NULL;
1297 iommu->domain_ids = NULL;
ba395927 1298
d9630fe9
WH		 1299 g_iommus[iommu->seq_id] = NULL;
1300
1301 /* if all iommus are freed, free g_iommus */
1302 for (i = 0; i < g_num_of_iommus; i++) {
1303 if (g_iommus[i])
1304 break;
1305 }
1306
1307 if (i == g_num_of_iommus)
1308 kfree(g_iommus);
1309
ba395927
KA		 1310 /* free context mapping */
1311 free_context_table(iommu);
ba395927
KA		 1312}
1313
2c2e2c38 1314static struct dmar_domain *alloc_domain(void)
ba395927 1315{
ba395927 1316 struct dmar_domain *domain;
ba395927
KA		 1317
1318 domain = alloc_domain_mem();
1319 if (!domain)
1320 return NULL;
1321
4c923d47 1322 domain->nid = -1;
1b198bb0 1323 memset(domain->iommu_bmp, 0, sizeof(domain->iommu_bmp));
2c2e2c38
FY		 1324 domain->flags = 0;
1325
1326 return domain;
1327}
1328
1329static int iommu_attach_domain(struct dmar_domain *domain,
1330 struct intel_iommu *iommu)
1331{
1332 int num;
1333 unsigned long ndomains;
1334 unsigned long flags;
1335
ba395927
KA		 1336 ndomains = cap_ndoms(iommu->cap);
1337
1338 spin_lock_irqsave(&iommu->lock, flags);
2c2e2c38 1339
ba395927
KA		 1340 num = find_first_zero_bit(iommu->domain_ids, ndomains);
1341 if (num >= ndomains) {
1342 spin_unlock_irqrestore(&iommu->lock, flags);
ba395927 1343 printk(KERN_ERR "IOMMU: no free domain ids\n");
2c2e2c38 1344 return -ENOMEM;
ba395927
KA		 1345 }
1346
ba395927 1347 domain->id = num;
2c2e2c38 1348 set_bit(num, iommu->domain_ids);
1b198bb0 1349 set_bit(iommu->seq_id, domain->iommu_bmp);
ba395927
KA		 1350 iommu->domains[num] = domain;
1351 spin_unlock_irqrestore(&iommu->lock, flags);
1352
2c2e2c38 1353 return 0;
ba395927
KA		 1354}
1355
2c2e2c38
FY		 1356static void iommu_detach_domain(struct dmar_domain *domain,
1357 struct intel_iommu *iommu)
ba395927
KA		 1358{
1359 unsigned long flags;
2c2e2c38
FY		 1360 int num, ndomains;
1361 int found = 0;
ba395927 1362
8c11e798 1363 spin_lock_irqsave(&iommu->lock, flags);
2c2e2c38 1364 ndomains = cap_ndoms(iommu->cap);
a45946ab 1365 for_each_set_bit(num, iommu->domain_ids, ndomains) {
2c2e2c38
FY		 1366 if (iommu->domains[num] == domain) {
1367 found = 1;
1368 break;
1369 }
2c2e2c38
FY		 1370 }
1371
1372 if (found) {
1373 clear_bit(num, iommu->domain_ids);
1b198bb0 1374 clear_bit(iommu->seq_id, domain->iommu_bmp);
2c2e2c38
FY		 1375 iommu->domains[num] = NULL;
1376 }
8c11e798 1377 spin_unlock_irqrestore(&iommu->lock, flags);
ba395927
KA		 1378}
1379
1380static struct iova_domain reserved_iova_list;
8a443df4 1381static struct lock_class_key reserved_rbtree_key;
ba395927 1382
51a63e67 1383static int dmar_init_reserved_ranges(void)
ba395927
KA		 1384{
1385 struct pci_dev *pdev = NULL;
1386 struct iova *iova;
1387 int i;
ba395927 1388
f661197e 1389 init_iova_domain(&reserved_iova_list, DMA_32BIT_PFN);
ba395927 1390
8a443df4
MG		 1391 lockdep_set_class(&reserved_iova_list.iova_rbtree_lock,
1392 &reserved_rbtree_key);
1393
ba395927
KA		 1394 /* IOAPIC ranges shouldn't be accessed by DMA */
1395 iova = reserve_iova(&reserved_iova_list, IOVA_PFN(IOAPIC_RANGE_START),
1396 IOVA_PFN(IOAPIC_RANGE_END));
51a63e67 1397 if (!iova) {
ba395927 1398 printk(KERN_ERR "Reserve IOAPIC range failed\n");
51a63e67
JC		 1399 return -ENODEV;
1400 }
ba395927
KA		 1401
1402 /* Reserve all PCI MMIO to avoid peer-to-peer access */
1403 for_each_pci_dev(pdev) {
1404 struct resource *r;
1405
1406 for (i = 0; i < PCI_NUM_RESOURCES; i++) {
1407 r = &pdev->resource[i];
1408 if (!r->flags || !(r->flags & IORESOURCE_MEM))
1409 continue;
1a4a4551
DW		 1410 iova = reserve_iova(&reserved_iova_list,
1411 IOVA_PFN(r->start),
1412 IOVA_PFN(r->end));
51a63e67 1413 if (!iova) {
ba395927 1414 printk(KERN_ERR "Reserve iova failed\n");
51a63e67
JC		 1415 return -ENODEV;
1416 }
ba395927
KA		 1417 }
1418 }
51a63e67 1419 return 0;
ba395927
KA		 1420}
1421
1422static void domain_reserve_special_ranges(struct dmar_domain *domain)
1423{
1424 copy_reserved_iova(&reserved_iova_list, &domain->iovad);
1425}
1426
1427static inline int guestwidth_to_adjustwidth(int gaw)
1428{
1429 int agaw;
1430 int r = (gaw - 12) % 9;
1431
1432 if (r == 0)
1433 agaw = gaw;
1434 else
1435 agaw = gaw + 9 - r;
1436 if (agaw > 64)
1437 agaw = 64;
1438 return agaw;
1439}
1440
1441static int domain_init(struct dmar_domain *domain, int guest_width)
1442{
1443 struct intel_iommu *iommu;
1444 int adjust_width, agaw;
1445 unsigned long sagaw;
1446
f661197e 1447 init_iova_domain(&domain->iovad, DMA_32BIT_PFN);
c7151a8d 1448 spin_lock_init(&domain->iommu_lock);
ba395927
KA		 1449
1450 domain_reserve_special_ranges(domain);
1451
1452 /* calculate AGAW */
8c11e798 1453 iommu = domain_get_iommu(domain);
ba395927
KA		 1454 if (guest_width > cap_mgaw(iommu->cap))
1455 guest_width = cap_mgaw(iommu->cap);
1456 domain->gaw = guest_width;
1457 adjust_width = guestwidth_to_adjustwidth(guest_width);
1458 agaw = width_to_agaw(adjust_width);
1459 sagaw = cap_sagaw(iommu->cap);
1460 if (!test_bit(agaw, &sagaw)) {
1461 /* hardware doesn't support it, choose a bigger one */
1462 pr_debug("IOMMU: hardware doesn't support agaw %d\n", agaw);
1463 agaw = find_next_bit(&sagaw, 5, agaw);
1464 if (agaw >= 5)
1465 return -ENODEV;
1466 }
1467 domain->agaw = agaw;
1468 INIT_LIST_HEAD(&domain->devices);
1469
8e604097
WH		 1470 if (ecap_coherent(iommu->ecap))
1471 domain->iommu_coherency = 1;
1472 else
1473 domain->iommu_coherency = 0;
1474
58c610bd
SY		 1475 if (ecap_sc_support(iommu->ecap))
1476 domain->iommu_snooping = 1;
1477 else
1478 domain->iommu_snooping = 0;
1479
6dd9a7c7 1480 domain->iommu_superpage = fls(cap_super_page_val(iommu->cap));
c7151a8d 1481 domain->iommu_count = 1;
4c923d47 1482 domain->nid = iommu->node;
c7151a8d 1483
ba395927 1484 /* always allocate the top pgd */
4c923d47 1485 domain->pgd = (struct dma_pte *)alloc_pgtable_page(domain->nid);
ba395927
KA		 1486 if (!domain->pgd)
1487 return -ENOMEM;
5b6985ce 1488 __iommu_flush_cache(iommu, domain->pgd, PAGE_SIZE);
ba395927
KA		 1489 return 0;
1490}
1491
1492static void domain_exit(struct dmar_domain *domain)
1493{
2c2e2c38
FY		 1494 struct dmar_drhd_unit *drhd;
1495 struct intel_iommu *iommu;
ba395927
KA		 1496
1497 /* Domain 0 is reserved, so dont process it */
1498 if (!domain)
1499 return;
1500
7b668357
AW		 1501 /* Flush any lazy unmaps that may reference this domain */
1502 if (!intel_iommu_strict)
1503 flush_unmaps_timeout(0);
1504
ba395927
KA		 1505 domain_remove_dev_info(domain);
1506 /* destroy iovas */
1507 put_iova_domain(&domain->iovad);
ba395927
KA		 1508
1509 /* clear ptes */
595badf5 1510 dma_pte_clear_range(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
ba395927
KA		 1511
1512 /* free page tables */
d794dc9b 1513 dma_pte_free_pagetable(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
ba395927 1514
2c2e2c38 1515 for_each_active_iommu(iommu, drhd)
1b198bb0 1516 if (test_bit(iommu->seq_id, domain->iommu_bmp))
2c2e2c38
FY		 1517 iommu_detach_domain(domain, iommu);
1518
ba395927
KA		 1519 free_domain_mem(domain);
1520}
1521
4ed0d3e6
FY		 1522static int domain_context_mapping_one(struct dmar_domain *domain, int segment,
1523 u8 bus, u8 devfn, int translation)
ba395927
KA		 1524{
1525 struct context_entry *context;
ba395927 1526 unsigned long flags;
5331fe6f 1527 struct intel_iommu *iommu;
ea6606b0
WH		 1528 struct dma_pte *pgd;
1529 unsigned long num;
1530 unsigned long ndomains;
1531 int id;
1532 int agaw;
93a23a72 1533 struct device_domain_info *info = NULL;
ba395927
KA		 1534
1535 pr_debug("Set context mapping for %02x:%02x.%d\n",
1536 bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
4ed0d3e6 1537
ba395927 1538 BUG_ON(!domain->pgd);
4ed0d3e6
FY		 1539 BUG_ON(translation != CONTEXT_TT_PASS_THROUGH &&
1540 translation != CONTEXT_TT_MULTI_LEVEL);
5331fe6f 1541
276dbf99 1542 iommu = device_to_iommu(segment, bus, devfn);
5331fe6f
WH		 1543 if (!iommu)
1544 return -ENODEV;
1545
ba395927
KA		 1546 context = device_to_context_entry(iommu, bus, devfn);
1547 if (!context)
1548 return -ENOMEM;
1549 spin_lock_irqsave(&iommu->lock, flags);
c07e7d21 1550 if (context_present(context)) {
ba395927
KA		 1551 spin_unlock_irqrestore(&iommu->lock, flags);
1552 return 0;
1553 }
1554
ea6606b0
WH		 1555 id = domain->id;
1556 pgd = domain->pgd;
1557
2c2e2c38
FY		 1558 if (domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE ||
1559 domain->flags & DOMAIN_FLAG_STATIC_IDENTITY) {
ea6606b0
WH		 1560 int found = 0;
1561
1562 /* find an available domain id for this device in iommu */
1563 ndomains = cap_ndoms(iommu->cap);
a45946ab 1564 for_each_set_bit(num, iommu->domain_ids, ndomains) {
ea6606b0
WH		 1565 if (iommu->domains[num] == domain) {
1566 id = num;
1567 found = 1;
1568 break;
1569 }
ea6606b0
WH		 1570 }
1571
1572 if (found == 0) {
1573 num = find_first_zero_bit(iommu->domain_ids, ndomains);
1574 if (num >= ndomains) {
1575 spin_unlock_irqrestore(&iommu->lock, flags);
1576 printk(KERN_ERR "IOMMU: no free domain ids\n");
1577 return -EFAULT;
1578 }
1579
1580 set_bit(num, iommu->domain_ids);
1581 iommu->domains[num] = domain;
1582 id = num;
1583 }
1584
1585 /* Skip top levels of page tables for
1586 * iommu which has less agaw than default.
1672af11 1587 * Unnecessary for PT mode.
ea6606b0 1588 */
1672af11
CW		 1589 if (translation != CONTEXT_TT_PASS_THROUGH) {
1590 for (agaw = domain->agaw; agaw != iommu->agaw; agaw--) {
1591 pgd = phys_to_virt(dma_pte_addr(pgd));
1592 if (!dma_pte_present(pgd)) {
1593 spin_unlock_irqrestore(&iommu->lock, flags);
1594 return -ENOMEM;
1595 }
ea6606b0
WH		 1596 }
1597 }
1598 }
1599
1600 context_set_domain_id(context, id);
4ed0d3e6 1601
93a23a72
YZ		 1602 if (translation != CONTEXT_TT_PASS_THROUGH) {
1603 info = iommu_support_dev_iotlb(domain, segment, bus, devfn);
1604 translation = info ? CONTEXT_TT_DEV_IOTLB :
1605 CONTEXT_TT_MULTI_LEVEL;
1606 }
4ed0d3e6
FY		 1607 /*
1608 * In pass through mode, AW must be programmed to indicate the largest
1609 * AGAW value supported by hardware. And ASR is ignored by hardware.
1610 */
93a23a72 1611 if (unlikely(translation == CONTEXT_TT_PASS_THROUGH))
4ed0d3e6 1612 context_set_address_width(context, iommu->msagaw);
93a23a72
YZ		 1613 else {
1614 context_set_address_root(context, virt_to_phys(pgd));
1615 context_set_address_width(context, iommu->agaw);
1616 }
4ed0d3e6
FY		 1617
1618 context_set_translation_type(context, translation);
c07e7d21
MM		 1619 context_set_fault_enable(context);
1620 context_set_present(context);
5331fe6f 1621 domain_flush_cache(domain, context, sizeof(*context));
ba395927 1622
4c25a2c1
DW		 1623 /*
1624 * It's a non-present to present mapping. If hardware doesn't cache
1625 * non-present entry we only need to flush the write-buffer. If the
1626 * _does_ cache non-present entries, then it does so in the special
1627 * domain #0, which we have to flush:
1628 */
1629 if (cap_caching_mode(iommu->cap)) {
1630 iommu->flush.flush_context(iommu, 0,
1631 (((u16)bus) << 8) | devfn,
1632 DMA_CCMD_MASK_NOBIT,
1633 DMA_CCMD_DEVICE_INVL);
82653633 1634 iommu->flush.flush_iotlb(iommu, domain->id, 0, 0, DMA_TLB_DSI_FLUSH);
4c25a2c1 1635 } else {
ba395927 1636 iommu_flush_write_buffer(iommu);
4c25a2c1 1637 }
93a23a72 1638 iommu_enable_dev_iotlb(info);
ba395927 1639 spin_unlock_irqrestore(&iommu->lock, flags);
c7151a8d
WH		 1640
1641 spin_lock_irqsave(&domain->iommu_lock, flags);
1b198bb0 1642 if (!test_and_set_bit(iommu->seq_id, domain->iommu_bmp)) {
c7151a8d 1643 domain->iommu_count++;
4c923d47
SS		 1644 if (domain->iommu_count == 1)
1645 domain->nid = iommu->node;
58c610bd 1646 domain_update_iommu_cap(domain);
c7151a8d
WH		 1647 }
1648 spin_unlock_irqrestore(&domain->iommu_lock, flags);
ba395927
KA		 1649 return 0;
1650}
1651
1652static int
4ed0d3e6
FY		 1653domain_context_mapping(struct dmar_domain *domain, struct pci_dev *pdev,
1654 int translation)
ba395927
KA		 1655{
1656 int ret;
1657 struct pci_dev *tmp, *parent;
1658
276dbf99 1659 ret = domain_context_mapping_one(domain, pci_domain_nr(pdev->bus),
4ed0d3e6
FY		 1660 pdev->bus->number, pdev->devfn,
1661 translation);
ba395927
KA		 1662 if (ret)
1663 return ret;
1664
1665 /* dependent device mapping */
1666 tmp = pci_find_upstream_pcie_bridge(pdev);
1667 if (!tmp)
1668 return 0;
1669 /* Secondary interface's bus number and devfn 0 */
1670 parent = pdev->bus->self;
1671 while (parent != tmp) {
276dbf99
DW		 1672 ret = domain_context_mapping_one(domain,
1673 pci_domain_nr(parent->bus),
1674 parent->bus->number,
4ed0d3e6 1675 parent->devfn, translation);
ba395927
KA		 1676 if (ret)
1677 return ret;
1678 parent = parent->bus->self;
1679 }
45e829ea 1680 if (pci_is_pcie(tmp)) /* this is a PCIe-to-PCI bridge */
ba395927 1681 return domain_context_mapping_one(domain,
276dbf99 1682 pci_domain_nr(tmp->subordinate),
4ed0d3e6
FY		 1683 tmp->subordinate->number, 0,
1684 translation);
ba395927
KA		 1685 else /* this is a legacy PCI bridge */
1686 return domain_context_mapping_one(domain,
276dbf99
DW		 1687 pci_domain_nr(tmp->bus),
1688 tmp->bus->number,
4ed0d3e6
FY		 1689 tmp->devfn,
1690 translation);
ba395927
KA		 1691}
1692
5331fe6f 1693static int domain_context_mapped(struct pci_dev *pdev)
ba395927
KA		 1694{
1695 int ret;
1696 struct pci_dev *tmp, *parent;
5331fe6f
WH		 1697 struct intel_iommu *iommu;
1698
276dbf99
DW		 1699 iommu = device_to_iommu(pci_domain_nr(pdev->bus), pdev->bus->number,
1700 pdev->devfn);
5331fe6f
WH		 1701 if (!iommu)
1702 return -ENODEV;
ba395927 1703
276dbf99 1704 ret = device_context_mapped(iommu, pdev->bus->number, pdev->devfn);
ba395927
KA		 1705 if (!ret)
1706 return ret;
1707 /* dependent device mapping */
1708 tmp = pci_find_upstream_pcie_bridge(pdev);
1709 if (!tmp)
1710 return ret;
1711 /* Secondary interface's bus number and devfn 0 */
1712 parent = pdev->bus->self;
1713 while (parent != tmp) {
8c11e798 1714 ret = device_context_mapped(iommu, parent->bus->number,
276dbf99 1715 parent->devfn);
ba395927
KA		 1716 if (!ret)
1717 return ret;
1718 parent = parent->bus->self;
1719 }
5f4d91a1 1720 if (pci_is_pcie(tmp))
276dbf99
DW		 1721 return device_context_mapped(iommu, tmp->subordinate->number,
1722 0);
ba395927 1723 else
276dbf99
DW		 1724 return device_context_mapped(iommu, tmp->bus->number,
1725 tmp->devfn);
ba395927
KA		 1726}
1727
f532959b
FY		 1728/* Returns a number of VTD pages, but aligned to MM page size */
1729static inline unsigned long aligned_nrpages(unsigned long host_addr,
1730 size_t size)
1731{
1732 host_addr &= ~PAGE_MASK;
1733 return PAGE_ALIGN(host_addr + size) >> VTD_PAGE_SHIFT;
1734}
1735
6dd9a7c7
YS		 1736/* Return largest possible superpage level for a given mapping */
1737static inline int hardware_largepage_caps(struct dmar_domain *domain,
1738 unsigned long iov_pfn,
1739 unsigned long phy_pfn,
1740 unsigned long pages)
1741{
1742 int support, level = 1;
1743 unsigned long pfnmerge;
1744
1745 support = domain->iommu_superpage;
1746
1747 /* To use a large page, the virtual *and* physical addresses
1748 must be aligned to 2MiB/1GiB/etc. Lower bits set in either
1749 of them will mean we have to use smaller pages. So just
1750 merge them and check both at once. */
1751 pfnmerge = iov_pfn | phy_pfn;
1752
1753 while (support && !(pfnmerge & ~VTD_STRIDE_MASK)) {
1754 pages >>= VTD_STRIDE_SHIFT;
1755 if (!pages)
1756 break;
1757 pfnmerge >>= VTD_STRIDE_SHIFT;
1758 level++;
1759 support--;
1760 }
1761 return level;
1762}
1763
9051aa02
DW		 1764static int __domain_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
1765 struct scatterlist *sg, unsigned long phys_pfn,
1766 unsigned long nr_pages, int prot)
e1605495
DW		 1767{
1768 struct dma_pte *first_pte = NULL, *pte = NULL;
9051aa02 1769 phys_addr_t uninitialized_var(pteval);
e1605495 1770 int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
9051aa02 1771 unsigned long sg_res;
6dd9a7c7
YS		 1772 unsigned int largepage_lvl = 0;
1773 unsigned long lvl_pages = 0;
e1605495
DW		 1774
1775 BUG_ON(addr_width < BITS_PER_LONG && (iov_pfn + nr_pages - 1) >> addr_width);
1776
1777 if ((prot & (DMA_PTE_READ|DMA_PTE_WRITE)) == 0)
1778 return -EINVAL;
1779
1780 prot &= DMA_PTE_READ | DMA_PTE_WRITE | DMA_PTE_SNP;
1781
9051aa02
DW		 1782 if (sg)
1783 sg_res = 0;
1784 else {
1785 sg_res = nr_pages + 1;
1786 pteval = ((phys_addr_t)phys_pfn << VTD_PAGE_SHIFT) | prot;
1787 }
1788
6dd9a7c7 1789 while (nr_pages > 0) {
c85994e4
DW		 1790 uint64_t tmp;
1791
e1605495 1792 if (!sg_res) {
f532959b 1793 sg_res = aligned_nrpages(sg->offset, sg->length);
e1605495
DW		 1794 sg->dma_address = ((dma_addr_t)iov_pfn << VTD_PAGE_SHIFT) + sg->offset;
1795 sg->dma_length = sg->length;
1796 pteval = page_to_phys(sg_page(sg)) | prot;
6dd9a7c7 1797 phys_pfn = pteval >> VTD_PAGE_SHIFT;
e1605495 1798 }
6dd9a7c7 1799
e1605495 1800 if (!pte) {
6dd9a7c7
YS		 1801 largepage_lvl = hardware_largepage_caps(domain, iov_pfn, phys_pfn, sg_res);
1802
1803 first_pte = pte = pfn_to_dma_pte(domain, iov_pfn, largepage_lvl);
e1605495
DW		 1804 if (!pte)
1805 return -ENOMEM;
6dd9a7c7 1806 /* It is large page*/
6491d4d0 1807 if (largepage_lvl > 1) {
6dd9a7c7 1808 pteval |= DMA_PTE_LARGE_PAGE;
6491d4d0
WD		 1809 /* Ensure that old small page tables are removed to make room
1810 for superpage, if they exist. */
1811 dma_pte_clear_range(domain, iov_pfn,
1812 iov_pfn + lvl_to_nr_pages(largepage_lvl) - 1);
1813 dma_pte_free_pagetable(domain, iov_pfn,
1814 iov_pfn + lvl_to_nr_pages(largepage_lvl) - 1);
1815 } else {
6dd9a7c7 1816 pteval &= ~(uint64_t)DMA_PTE_LARGE_PAGE;
6491d4d0 1817 }
6dd9a7c7 1818
e1605495
DW		 1819 }
1820 /* We don't need lock here, nobody else
1821 * touches the iova range
1822 */
7766a3fb 1823 tmp = cmpxchg64_local(&pte->val, 0ULL, pteval);
c85994e4 1824 if (tmp) {
1bf20f0d 1825 static int dumps = 5;
c85994e4
DW		 1826 printk(KERN_CRIT "ERROR: DMA PTE for vPFN 0x%lx already set (to %llx not %llx)\n",
1827 iov_pfn, tmp, (unsigned long long)pteval);
1bf20f0d
DW		 1828 if (dumps) {
1829 dumps--;
1830 debug_dma_dump_mappings(NULL);
1831 }
1832 WARN_ON(1);
1833 }
6dd9a7c7
YS		 1834
1835 lvl_pages = lvl_to_nr_pages(largepage_lvl);
1836
1837 BUG_ON(nr_pages < lvl_pages);
1838 BUG_ON(sg_res < lvl_pages);
1839
1840 nr_pages -= lvl_pages;
1841 iov_pfn += lvl_pages;
1842 phys_pfn += lvl_pages;
1843 pteval += lvl_pages * VTD_PAGE_SIZE;
1844 sg_res -= lvl_pages;
1845
1846 /* If the next PTE would be the first in a new page, then we
1847 need to flush the cache on the entries we've just written.
1848 And then we'll need to recalculate 'pte', so clear it and
1849 let it get set again in the if (!pte) block above.
1850
1851 If we're done (!nr_pages) we need to flush the cache too.
1852
1853 Also if we've been setting superpages, we may need to
1854 recalculate 'pte' and switch back to smaller pages for the
1855 end of the mapping, if the trailing size is not enough to
1856 use another superpage (i.e. sg_res < lvl_pages). */
e1605495 1857 pte++;
6dd9a7c7
YS		 1858 if (!nr_pages || first_pte_in_page(pte) ||
1859 (largepage_lvl > 1 && sg_res < lvl_pages)) {
e1605495
DW		 1860 domain_flush_cache(domain, first_pte,
1861 (void *)pte - (void *)first_pte);
1862 pte = NULL;
1863 }
6dd9a7c7
YS		 1864
1865 if (!sg_res && nr_pages)
e1605495
DW		 1866 sg = sg_next(sg);
1867 }
1868 return 0;
1869}
1870
9051aa02
DW		 1871static inline int domain_sg_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
1872 struct scatterlist *sg, unsigned long nr_pages,
1873 int prot)
ba395927 1874{
9051aa02
DW		 1875 return __domain_mapping(domain, iov_pfn, sg, 0, nr_pages, prot);
1876}
6f6a00e4 1877
9051aa02
DW		 1878static inline int domain_pfn_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
1879 unsigned long phys_pfn, unsigned long nr_pages,
1880 int prot)
1881{
1882 return __domain_mapping(domain, iov_pfn, NULL, phys_pfn, nr_pages, prot);
ba395927
KA		 1883}
1884
c7151a8d 1885static void iommu_detach_dev(struct intel_iommu *iommu, u8 bus, u8 devfn)
ba395927 1886{
c7151a8d
WH		 1887 if (!iommu)
1888 return;
8c11e798
WH		 1889
1890 clear_context_table(iommu, bus, devfn);
1891 iommu->flush.flush_context(iommu, 0, 0, 0,
4c25a2c1 1892 DMA_CCMD_GLOBAL_INVL);
1f0ef2aa 1893 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
ba395927
KA		 1894}
1895
109b9b04
DW		 1896static inline void unlink_domain_info(struct device_domain_info *info)
1897{
1898 assert_spin_locked(&device_domain_lock);
1899 list_del(&info->link);
1900 list_del(&info->global);
1901 if (info->dev)
1902 info->dev->dev.archdata.iommu = NULL;
1903}
1904
ba395927
KA		 1905static void domain_remove_dev_info(struct dmar_domain *domain)
1906{
1907 struct device_domain_info *info;
1908 unsigned long flags;
c7151a8d 1909 struct intel_iommu *iommu;
ba395927
KA		 1910
1911 spin_lock_irqsave(&device_domain_lock, flags);
1912 while (!list_empty(&domain->devices)) {
1913 info = list_entry(domain->devices.next,
1914 struct device_domain_info, link);
109b9b04 1915 unlink_domain_info(info);
ba395927
KA		 1916 spin_unlock_irqrestore(&device_domain_lock, flags);
1917
93a23a72 1918 iommu_disable_dev_iotlb(info);
276dbf99 1919 iommu = device_to_iommu(info->segment, info->bus, info->devfn);
c7151a8d 1920 iommu_detach_dev(iommu, info->bus, info->devfn);
ba395927
KA		 1921 free_devinfo_mem(info);
1922
1923 spin_lock_irqsave(&device_domain_lock, flags);
1924 }
1925 spin_unlock_irqrestore(&device_domain_lock, flags);
1926}
1927
1928/*
1929 * find_domain
358dd8ac 1930 * Note: we use struct pci_dev->dev.archdata.iommu stores the info
ba395927 1931 */
38717946 1932static struct dmar_domain *
ba395927
KA		 1933find_domain(struct pci_dev *pdev)
1934{
1935 struct device_domain_info *info;
1936
1937 /* No lock here, assumes no domain exit in normal case */
358dd8ac 1938 info = pdev->dev.archdata.iommu;
ba395927
KA		 1939 if (info)
1940 return info->domain;
1941 return NULL;
1942}
1943
ba395927
KA		 1944/* domain is initialized */
1945static struct dmar_domain *get_domain_for_dev(struct pci_dev *pdev, int gaw)
1946{
1947 struct dmar_domain *domain, *found = NULL;
1948 struct intel_iommu *iommu;
1949 struct dmar_drhd_unit *drhd;
1950 struct device_domain_info *info, *tmp;
1951 struct pci_dev *dev_tmp;
1952 unsigned long flags;
1953 int bus = 0, devfn = 0;
276dbf99 1954 int segment;
2c2e2c38 1955 int ret;
ba395927
KA		 1956
1957 domain = find_domain(pdev);
1958 if (domain)
1959 return domain;
1960
276dbf99
DW		 1961 segment = pci_domain_nr(pdev->bus);
1962
ba395927
KA		 1963 dev_tmp = pci_find_upstream_pcie_bridge(pdev);
1964 if (dev_tmp) {
5f4d91a1 1965 if (pci_is_pcie(dev_tmp)) {
ba395927
KA		 1966 bus = dev_tmp->subordinate->number;
1967 devfn = 0;
1968 } else {
1969 bus = dev_tmp->bus->number;
1970 devfn = dev_tmp->devfn;
1971 }
1972 spin_lock_irqsave(&device_domain_lock, flags);
1973 list_for_each_entry(info, &device_domain_list, global) {
276dbf99
DW		 1974 if (info->segment == segment &&
1975 info->bus == bus && info->devfn == devfn) {
ba395927
KA		 1976 found = info->domain;
1977 break;
1978 }
1979 }
1980 spin_unlock_irqrestore(&device_domain_lock, flags);
1981 /* pcie-pci bridge already has a domain, uses it */
1982 if (found) {
1983 domain = found;
1984 goto found_domain;
1985 }
1986 }
1987
2c2e2c38
FY		 1988 domain = alloc_domain();
1989 if (!domain)
1990 goto error;
1991
ba395927
KA		 1992 /* Allocate new domain for the device */
1993 drhd = dmar_find_matched_drhd_unit(pdev);
1994 if (!drhd) {
1995 printk(KERN_ERR "IOMMU: can't find DMAR for device %s\n",
1996 pci_name(pdev));
d2900bd6 1997 free_domain_mem(domain);
ba395927
KA		 1998 return NULL;
1999 }
2000 iommu = drhd->iommu;
2001
2c2e2c38
FY		 2002 ret = iommu_attach_domain(domain, iommu);
2003 if (ret) {
2fe9723d 2004 free_domain_mem(domain);
ba395927 2005 goto error;
2c2e2c38 2006 }
ba395927
KA		 2007
2008 if (domain_init(domain, gaw)) {
2009 domain_exit(domain);
2010 goto error;
2011 }
2012
2013 /* register pcie-to-pci device */
2014 if (dev_tmp) {
2015 info = alloc_devinfo_mem();
2016 if (!info) {
2017 domain_exit(domain);
2018 goto error;
2019 }
276dbf99 2020 info->segment = segment;
ba395927
KA		 2021 info->bus = bus;
2022 info->devfn = devfn;
2023 info->dev = NULL;
2024 info->domain = domain;
2025 /* This domain is shared by devices under p2p bridge */
3b5410e7 2026 domain->flags |= DOMAIN_FLAG_P2P_MULTIPLE_DEVICES;
ba395927
KA		 2027
2028 /* pcie-to-pci bridge already has a domain, uses it */
2029 found = NULL;
2030 spin_lock_irqsave(&device_domain_lock, flags);
2031 list_for_each_entry(tmp, &device_domain_list, global) {
276dbf99
DW		 2032 if (tmp->segment == segment &&
2033 tmp->bus == bus && tmp->devfn == devfn) {
ba395927
KA		 2034 found = tmp->domain;
2035 break;
2036 }
2037 }
2038 if (found) {
00dfff77 2039 spin_unlock_irqrestore(&device_domain_lock, flags);
ba395927
KA		 2040 free_devinfo_mem(info);
2041 domain_exit(domain);
2042 domain = found;
2043 } else {
2044 list_add(&info->link, &domain->devices);
2045 list_add(&info->global, &device_domain_list);
00dfff77 2046 spin_unlock_irqrestore(&device_domain_lock, flags);
ba395927 2047 }
ba395927
KA		 2048 }
2049
2050found_domain:
2051 info = alloc_devinfo_mem();
2052 if (!info)
2053 goto error;
276dbf99 2054 info->segment = segment;
ba395927
KA		 2055 info->bus = pdev->bus->number;
2056 info->devfn = pdev->devfn;
2057 info->dev = pdev;
2058 info->domain = domain;
2059 spin_lock_irqsave(&device_domain_lock, flags);
2060 /* somebody is fast */
2061 found = find_domain(pdev);
2062 if (found != NULL) {
2063 spin_unlock_irqrestore(&device_domain_lock, flags);
2064 if (found != domain) {
2065 domain_exit(domain);
2066 domain = found;
2067 }
2068 free_devinfo_mem(info);
2069 return domain;
2070 }
2071 list_add(&info->link, &domain->devices);
2072 list_add(&info->global, &device_domain_list);
358dd8ac 2073 pdev->dev.archdata.iommu = info;
ba395927
KA		 2074 spin_unlock_irqrestore(&device_domain_lock, flags);
2075 return domain;
2076error:
2077 /* recheck it here, maybe others set it */
2078 return find_domain(pdev);
2079}
2080
2c2e2c38 2081static int iommu_identity_mapping;
e0fc7e0b
DW		 2082#define IDENTMAP_ALL 1
2083#define IDENTMAP_GFX 2
2084#define IDENTMAP_AZALIA 4
2c2e2c38 2085
b213203e
DW		 2086static int iommu_domain_identity_map(struct dmar_domain *domain,
2087 unsigned long long start,
2088 unsigned long long end)
ba395927 2089{
c5395d5c
DW		 2090 unsigned long first_vpfn = start >> VTD_PAGE_SHIFT;
2091 unsigned long last_vpfn = end >> VTD_PAGE_SHIFT;
2092
2093 if (!reserve_iova(&domain->iovad, dma_to_mm_pfn(first_vpfn),
2094 dma_to_mm_pfn(last_vpfn))) {
ba395927 2095 printk(KERN_ERR "IOMMU: reserve iova failed\n");
b213203e 2096 return -ENOMEM;
ba395927
KA		 2097 }
2098
c5395d5c
DW		 2099 pr_debug("Mapping reserved region %llx-%llx for domain %d\n",
2100 start, end, domain->id);
ba395927
KA		 2101 /*
2102 * RMRR range might have overlap with physical memory range,
2103 * clear it first
2104 */
c5395d5c 2105 dma_pte_clear_range(domain, first_vpfn, last_vpfn);
ba395927 2106
c5395d5c
DW		 2107 return domain_pfn_mapping(domain, first_vpfn, first_vpfn,
2108 last_vpfn - first_vpfn + 1,
61df7443 2109 DMA_PTE_READ|DMA_PTE_WRITE);
b213203e
DW		 2110}
2111
2112static int iommu_prepare_identity_map(struct pci_dev *pdev,
2113 unsigned long long start,
2114 unsigned long long end)
2115{
2116 struct dmar_domain *domain;
2117 int ret;
2118
c7ab48d2 2119 domain = get_domain_for_dev(pdev, DEFAULT_DOMAIN_ADDRESS_WIDTH);
b213203e
DW		 2120 if (!domain)
2121 return -ENOMEM;
2122
19943b0e
DW		 2123 /* For _hardware_ passthrough, don't bother. But for software
2124 passthrough, we do it anyway -- it may indicate a memory
2125 range which is reserved in E820, so which didn't get set
2126 up to start with in si_domain */
2127 if (domain == si_domain && hw_pass_through) {
2128 printk("Ignoring identity map for HW passthrough device %s [0x%Lx - 0x%Lx]\n",
2129 pci_name(pdev), start, end);
2130 return 0;
2131 }
2132
2133 printk(KERN_INFO
2134 "IOMMU: Setting identity map for device %s [0x%Lx - 0x%Lx]\n",
2135 pci_name(pdev), start, end);
2ff729f5 2136
5595b528
DW		 2137 if (end < start) {
2138 WARN(1, "Your BIOS is broken; RMRR ends before it starts!\n"
2139 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
2140 dmi_get_system_info(DMI_BIOS_VENDOR),
2141 dmi_get_system_info(DMI_BIOS_VERSION),
2142 dmi_get_system_info(DMI_PRODUCT_VERSION));
2143 ret = -EIO;
2144 goto error;
2145 }
2146
2ff729f5
DW		 2147 if (end >> agaw_to_width(domain->agaw)) {
2148 WARN(1, "Your BIOS is broken; RMRR exceeds permitted address width (%d bits)\n"
2149 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
2150 agaw_to_width(domain->agaw),
2151 dmi_get_system_info(DMI_BIOS_VENDOR),
2152 dmi_get_system_info(DMI_BIOS_VERSION),
2153 dmi_get_system_info(DMI_PRODUCT_VERSION));
2154 ret = -EIO;
2155 goto error;
2156 }
19943b0e 2157
b213203e 2158 ret = iommu_domain_identity_map(domain, start, end);
ba395927
KA		 2159 if (ret)
2160 goto error;
2161
2162 /* context entry init */
4ed0d3e6 2163 ret = domain_context_mapping(domain, pdev, CONTEXT_TT_MULTI_LEVEL);
b213203e
DW		 2164 if (ret)
2165 goto error;
2166
2167 return 0;
2168
2169 error:
ba395927
KA		 2170 domain_exit(domain);
2171 return ret;
ba395927
KA		 2172}
2173
2174static inline int iommu_prepare_rmrr_dev(struct dmar_rmrr_unit *rmrr,
2175 struct pci_dev *pdev)
2176{
358dd8ac 2177 if (pdev->dev.archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO)
ba395927
KA		 2178 return 0;
2179 return iommu_prepare_identity_map(pdev, rmrr->base_address,
70e535d1 2180 rmrr->end_address);
ba395927
KA		 2181}
2182
d3f13810 2183#ifdef CONFIG_INTEL_IOMMU_FLOPPY_WA
49a0429e
KA		 2184static inline void iommu_prepare_isa(void)
2185{
2186 struct pci_dev *pdev;
2187 int ret;
2188
2189 pdev = pci_get_class(PCI_CLASS_BRIDGE_ISA << 8, NULL);
2190 if (!pdev)
2191 return;
2192
c7ab48d2 2193 printk(KERN_INFO "IOMMU: Prepare 0-16MiB unity mapping for LPC\n");
70e535d1 2194 ret = iommu_prepare_identity_map(pdev, 0, 16*1024*1024 - 1);
49a0429e
KA		 2195
2196 if (ret)
c7ab48d2
DW		 2197 printk(KERN_ERR "IOMMU: Failed to create 0-16MiB identity map; "
2198 "floppy might not work\n");
49a0429e
KA		 2199
2200}
2201#else
2202static inline void iommu_prepare_isa(void)
2203{
2204 return;
2205}
d3f13810 2206#endif /* !CONFIG_INTEL_IOMMU_FLPY_WA */
49a0429e 2207
2c2e2c38 2208static int md_domain_init(struct dmar_domain *domain, int guest_width);
c7ab48d2 2209
071e1374 2210static int __init si_domain_init(int hw)
2c2e2c38
FY		 2211{
2212 struct dmar_drhd_unit *drhd;
2213 struct intel_iommu *iommu;
c7ab48d2 2214 int nid, ret = 0;
2c2e2c38
FY		 2215
2216 si_domain = alloc_domain();
2217 if (!si_domain)
2218 return -EFAULT;
2219
2c2e2c38
FY		 2220 for_each_active_iommu(iommu, drhd) {
2221 ret = iommu_attach_domain(si_domain, iommu);
2222 if (ret) {
2223 domain_exit(si_domain);
2224 return -EFAULT;
2225 }
2226 }
2227
2228 if (md_domain_init(si_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
2229 domain_exit(si_domain);
2230 return -EFAULT;
2231 }
2232
2233 si_domain->flags = DOMAIN_FLAG_STATIC_IDENTITY;
9544c003
JL		 2234 pr_debug("IOMMU: identity mapping domain is domain %d\n",
2235 si_domain->id);
2c2e2c38 2236
19943b0e
DW		 2237 if (hw)
2238 return 0;
2239
c7ab48d2 2240 for_each_online_node(nid) {
5dfe8660
TH		 2241 unsigned long start_pfn, end_pfn;
2242 int i;
2243
2244 for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, NULL) {
2245 ret = iommu_domain_identity_map(si_domain,
2246 PFN_PHYS(start_pfn), PFN_PHYS(end_pfn));
2247 if (ret)
2248 return ret;
2249 }
c7ab48d2
DW		 2250 }
2251
2c2e2c38
FY		 2252 return 0;
2253}
2254
2255static void domain_remove_one_dev_info(struct dmar_domain *domain,
2256 struct pci_dev *pdev);
2257static int identity_mapping(struct pci_dev *pdev)
2258{
2259 struct device_domain_info *info;
2260
2261 if (likely(!iommu_identity_mapping))
2262 return 0;
2263
cb452a40
MT		 2264 info = pdev->dev.archdata.iommu;
2265 if (info && info != DUMMY_DEVICE_DOMAIN_INFO)
2266 return (info->domain == si_domain);
2c2e2c38 2267
2c2e2c38
FY		 2268 return 0;
2269}
2270
2271static int domain_add_dev_info(struct dmar_domain *domain,
5fe60f4e
DW		 2272 struct pci_dev *pdev,
2273 int translation)
2c2e2c38
FY		 2274{
2275 struct device_domain_info *info;
2276 unsigned long flags;
5fe60f4e 2277 int ret;
2c2e2c38
FY		 2278
2279 info = alloc_devinfo_mem();
2280 if (!info)
2281 return -ENOMEM;
2282
2283 info->segment = pci_domain_nr(pdev->bus);
2284 info->bus = pdev->bus->number;
2285 info->devfn = pdev->devfn;
2286 info->dev = pdev;
2287 info->domain = domain;
2288
2289 spin_lock_irqsave(&device_domain_lock, flags);
2290 list_add(&info->link, &domain->devices);
2291 list_add(&info->global, &device_domain_list);
2292 pdev->dev.archdata.iommu = info;
2293 spin_unlock_irqrestore(&device_domain_lock, flags);
2294
e2ad23d0
DW		 2295 ret = domain_context_mapping(domain, pdev, translation);
2296 if (ret) {
2297 spin_lock_irqsave(&device_domain_lock, flags);
109b9b04 2298 unlink_domain_info(info);
e2ad23d0
DW		 2299 spin_unlock_irqrestore(&device_domain_lock, flags);
2300 free_devinfo_mem(info);
2301 return ret;
2302 }
2303
2c2e2c38
FY		 2304 return 0;
2305}
2306
ea2447f7
TM		 2307static bool device_has_rmrr(struct pci_dev *dev)
2308{
2309 struct dmar_rmrr_unit *rmrr;
2310 int i;
2311
2312 for_each_rmrr_units(rmrr) {
2313 for (i = 0; i < rmrr->devices_cnt; i++) {
2314 /*
2315 * Return TRUE if this RMRR contains the device that
2316 * is passed in.
2317 */
2318 if (rmrr->devices[i] == dev)
2319 return true;
2320 }
2321 }
2322 return false;
2323}
2324
6941af28
DW		 2325static int iommu_should_identity_map(struct pci_dev *pdev, int startup)
2326{
ea2447f7
TM		 2327
2328 /*
2329 * We want to prevent any device associated with an RMRR from
2330 * getting placed into the SI Domain. This is done because
2331 * problems exist when devices are moved in and out of domains
2332 * and their respective RMRR info is lost. We exempt USB devices
2333 * from this process due to their usage of RMRRs that are known
2334 * to not be needed after BIOS hand-off to OS.
2335 */
2336 if (device_has_rmrr(pdev) &&
2337 (pdev->class >> 8) != PCI_CLASS_SERIAL_USB)
2338 return 0;
2339
e0fc7e0b
DW		 2340 if ((iommu_identity_mapping & IDENTMAP_AZALIA) && IS_AZALIA(pdev))
2341 return 1;
2342
2343 if ((iommu_identity_mapping & IDENTMAP_GFX) && IS_GFX_DEVICE(pdev))
2344 return 1;
2345
2346 if (!(iommu_identity_mapping & IDENTMAP_ALL))
2347 return 0;
6941af28 2348
3dfc813d
DW		 2349 /*
2350 * We want to start off with all devices in the 1:1 domain, and
2351 * take them out later if we find they can't access all of memory.
2352 *
2353 * However, we can't do this for PCI devices behind bridges,
2354 * because all PCI devices behind the same bridge will end up
2355 * with the same source-id on their transactions.
2356 *
2357 * Practically speaking, we can't change things around for these
2358 * devices at run-time, because we can't be sure there'll be no
2359 * DMA transactions in flight for any of their siblings.
2360 *
2361 * So PCI devices (unless they're on the root bus) as well as
2362 * their parent PCI-PCI or PCIe-PCI bridges must be left _out_ of
2363 * the 1:1 domain, just in _case_ one of their siblings turns out
2364 * not to be able to map all of memory.
2365 */
5f4d91a1 2366 if (!pci_is_pcie(pdev)) {
3dfc813d
DW		 2367 if (!pci_is_root_bus(pdev->bus))
2368 return 0;
2369 if (pdev->class >> 8 == PCI_CLASS_BRIDGE_PCI)
2370 return 0;
62f87c0e 2371 } else if (pci_pcie_type(pdev) == PCI_EXP_TYPE_PCI_BRIDGE)
3dfc813d
DW		 2372 return 0;
2373
2374 /*
2375 * At boot time, we don't yet know if devices will be 64-bit capable.
2376 * Assume that they will -- if they turn out not to be, then we can
2377 * take them out of the 1:1 domain later.
2378 */
8fcc5372
CW		 2379 if (!startup) {
2380 /*
2381 * If the device's dma_mask is less than the system's memory
2382 * size then this is not a candidate for identity mapping.
2383 */
2384 u64 dma_mask = pdev->dma_mask;
2385
2386 if (pdev->dev.coherent_dma_mask &&
2387 pdev->dev.coherent_dma_mask < dma_mask)
2388 dma_mask = pdev->dev.coherent_dma_mask;
2389
2390 return dma_mask >= dma_get_required_mask(&pdev->dev);
2391 }
6941af28
DW		 2392
2393 return 1;
2394}
2395
071e1374 2396static int __init iommu_prepare_static_identity_mapping(int hw)
2c2e2c38 2397{
2c2e2c38
FY		 2398 struct pci_dev *pdev = NULL;
2399 int ret;
2400
19943b0e 2401 ret = si_domain_init(hw);
2c2e2c38
FY		 2402 if (ret)
2403 return -EFAULT;
2404
2c2e2c38 2405 for_each_pci_dev(pdev) {
6941af28 2406 if (iommu_should_identity_map(pdev, 1)) {
5fe60f4e 2407 ret = domain_add_dev_info(si_domain, pdev,
eae460b6
MT		 2408 hw ? CONTEXT_TT_PASS_THROUGH :
2409 CONTEXT_TT_MULTI_LEVEL);
2410 if (ret) {
2411 /* device not associated with an iommu */
2412 if (ret == -ENODEV)
2413 continue;
62edf5dc 2414 return ret;
eae460b6
MT		 2415 }
2416 pr_info("IOMMU: %s identity mapping for device %s\n",
2417 hw ? "hardware" : "software", pci_name(pdev));
62edf5dc 2418 }
2c2e2c38
FY		 2419 }
2420
2421 return 0;
2422}
2423
b779260b 2424static int __init init_dmars(void)
ba395927
KA		 2425{
2426 struct dmar_drhd_unit *drhd;
2427 struct dmar_rmrr_unit *rmrr;
2428 struct pci_dev *pdev;
2429 struct intel_iommu *iommu;
9d783ba0 2430 int i, ret;
2c2e2c38 2431
ba395927
KA		 2432 /*
2433 * for each drhd
2434 * allocate root
2435 * initialize and program root entry to not present
2436 * endfor
2437 */
2438 for_each_drhd_unit(drhd) {
5e0d2a6f 2439 /*
2440 * lock not needed as this is only incremented in the single
2441 * threaded kernel __init code path all other access are read
2442 * only
2443 */
1b198bb0
MT		 2444 if (g_num_of_iommus < IOMMU_UNITS_SUPPORTED) {
2445 g_num_of_iommus++;
2446 continue;
2447 }
2448 printk_once(KERN_ERR "intel-iommu: exceeded %d IOMMUs\n",
2449 IOMMU_UNITS_SUPPORTED);
5e0d2a6f 2450 }
2451
d9630fe9
WH		 2452 g_iommus = kcalloc(g_num_of_iommus, sizeof(struct intel_iommu *),
2453 GFP_KERNEL);
2454 if (!g_iommus) {
2455 printk(KERN_ERR "Allocating global iommu array failed\n");
2456 ret = -ENOMEM;
2457 goto error;
2458 }
2459
80b20dd8 2460 deferred_flush = kzalloc(g_num_of_iommus *
2461 sizeof(struct deferred_flush_tables), GFP_KERNEL);
2462 if (!deferred_flush) {
5e0d2a6f 2463 ret = -ENOMEM;
2464 goto error;
2465 }
2466
7c919779 2467 for_each_active_iommu(iommu, drhd) {
d9630fe9 2468 g_iommus[iommu->seq_id] = iommu;
ba395927 2469
e61d98d8
SS		 2470 ret = iommu_init_domains(iommu);
2471 if (ret)
2472 goto error;
2473
ba395927
KA		 2474 /*
2475 * TBD:
2476 * we could share the same root & context tables
25985edc 2477 * among all IOMMU's. Need to Split it later.
ba395927
KA		 2478 */
2479 ret = iommu_alloc_root_entry(iommu);
2480 if (ret) {
2481 printk(KERN_ERR "IOMMU: allocate root entry failed\n");
2482 goto error;
2483 }
4ed0d3e6 2484 if (!ecap_pass_through(iommu->ecap))
19943b0e 2485 hw_pass_through = 0;
ba395927
KA		 2486 }
2487
1531a6a6
SS		 2488 /*
2489 * Start from the sane iommu hardware state.
2490 */
7c919779 2491 for_each_active_iommu(iommu, drhd) {
1531a6a6
SS		 2492 /*
2493 * If the queued invalidation is already initialized by us
2494 * (for example, while enabling interrupt-remapping) then
2495 * we got the things already rolling from a sane state.
2496 */
2497 if (iommu->qi)
2498 continue;
2499
2500 /*
2501 * Clear any previous faults.
2502 */
2503 dmar_fault(-1, iommu);
2504 /*
2505 * Disable queued invalidation if supported and already enabled
2506 * before OS handover.
2507 */
2508 dmar_disable_qi(iommu);
2509 }
2510
7c919779 2511 for_each_active_iommu(iommu, drhd) {
a77b67d4
YS		 2512 if (dmar_enable_qi(iommu)) {
2513 /*
2514 * Queued Invalidate not enabled, use Register Based
2515 * Invalidate
2516 */
2517 iommu->flush.flush_context = __iommu_flush_context;
2518 iommu->flush.flush_iotlb = __iommu_flush_iotlb;
680a7524 2519 printk(KERN_INFO "IOMMU %d 0x%Lx: using Register based "
b4e0f9eb 2520 "invalidation\n",
680a7524 2521 iommu->seq_id,
b4e0f9eb 2522 (unsigned long long)drhd->reg_base_addr);
a77b67d4
YS		 2523 } else {
2524 iommu->flush.flush_context = qi_flush_context;
2525 iommu->flush.flush_iotlb = qi_flush_iotlb;
680a7524 2526 printk(KERN_INFO "IOMMU %d 0x%Lx: using Queued "
b4e0f9eb 2527 "invalidation\n",
680a7524 2528 iommu->seq_id,
b4e0f9eb 2529 (unsigned long long)drhd->reg_base_addr);
a77b67d4
YS		 2530 }
2531 }
2532
19943b0e 2533 if (iommu_pass_through)
e0fc7e0b
DW		 2534 iommu_identity_mapping |= IDENTMAP_ALL;
2535
d3f13810 2536#ifdef CONFIG_INTEL_IOMMU_BROKEN_GFX_WA
e0fc7e0b 2537 iommu_identity_mapping |= IDENTMAP_GFX;
19943b0e 2538#endif
e0fc7e0b
DW		 2539
2540 check_tylersburg_isoch();
2541
ba395927 2542 /*
19943b0e
DW		 2543 * If pass through is not set or not enabled, setup context entries for
2544 * identity mappings for rmrr, gfx, and isa and may fall back to static
2545 * identity mapping if iommu_identity_mapping is set.
ba395927 2546 */
19943b0e
DW		 2547 if (iommu_identity_mapping) {
2548 ret = iommu_prepare_static_identity_mapping(hw_pass_through);
4ed0d3e6 2549 if (ret) {
19943b0e
DW		 2550 printk(KERN_CRIT "Failed to setup IOMMU pass-through\n");
2551 goto error;
ba395927
KA		 2552 }
2553 }
ba395927 2554 /*
19943b0e
DW		 2555 * For each rmrr
2556 * for each dev attached to rmrr
2557 * do
2558 * locate drhd for dev, alloc domain for dev
2559 * allocate free domain
2560 * allocate page table entries for rmrr
2561 * if context not allocated for bus
2562 * allocate and init context
2563 * set present in root table for this bus
2564 * init context with domain, translation etc
2565 * endfor
2566 * endfor
ba395927 2567 */
19943b0e
DW		 2568 printk(KERN_INFO "IOMMU: Setting RMRR:\n");
2569 for_each_rmrr_units(rmrr) {
2570 for (i = 0; i < rmrr->devices_cnt; i++) {
2571 pdev = rmrr->devices[i];
2572 /*
2573 * some BIOS lists non-exist devices in DMAR
2574 * table.
2575 */
2576 if (!pdev)
2577 continue;
2578 ret = iommu_prepare_rmrr_dev(rmrr, pdev);
2579 if (ret)
2580 printk(KERN_ERR
2581 "IOMMU: mapping reserved region failed\n");
ba395927 2582 }
4ed0d3e6 2583 }
49a0429e 2584
19943b0e
DW		 2585 iommu_prepare_isa();
2586
ba395927
KA		 2587 /*
2588 * for each drhd
2589 * enable fault log
2590 * global invalidate context cache
2591 * global invalidate iotlb
2592 * enable translation
2593 */
7c919779 2594 for_each_iommu(iommu, drhd) {
51a63e67
JC		 2595 if (drhd->ignored) {
2596 /*
2597 * we always have to disable PMRs or DMA may fail on
2598 * this device
2599 */
2600 if (force_on)
7c919779 2601 iommu_disable_protect_mem_regions(iommu);
ba395927 2602 continue;
51a63e67 2603 }
ba395927
KA		 2604
2605 iommu_flush_write_buffer(iommu);
2606
3460a6d9
KA		 2607 ret = dmar_set_interrupt(iommu);
2608 if (ret)
2609 goto error;
2610
ba395927
KA		 2611 iommu_set_root_entry(iommu);
2612
4c25a2c1 2613 iommu->flush.flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL);
1f0ef2aa 2614 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
f8bab735 2615
ba395927
KA		 2616 ret = iommu_enable_translation(iommu);
2617 if (ret)
2618 goto error;
b94996c9
DW		 2619
2620 iommu_disable_protect_mem_regions(iommu);
ba395927
KA		 2621 }
2622
2623 return 0;
2624error:
7c919779 2625 for_each_active_iommu(iommu, drhd)
a868e6b7 2626 free_dmar_iommu(iommu);
d9630fe9 2627 kfree(g_iommus);
ba395927
KA		 2628 return ret;
2629}
2630
5a5e02a6 2631/* This takes a number of _MM_ pages, not VTD pages */
875764de
DW		 2632static struct iova *intel_alloc_iova(struct device *dev,
2633 struct dmar_domain *domain,
2634 unsigned long nrpages, uint64_t dma_mask)
ba395927 2635{
ba395927 2636 struct pci_dev *pdev = to_pci_dev(dev);
ba395927 2637 struct iova *iova = NULL;
ba395927 2638
875764de
DW		 2639 /* Restrict dma_mask to the width that the iommu can handle */
2640 dma_mask = min_t(uint64_t, DOMAIN_MAX_ADDR(domain->gaw), dma_mask);
2641
2642 if (!dmar_forcedac && dma_mask > DMA_BIT_MASK(32)) {
ba395927
KA		 2643 /*
2644 * First try to allocate an io virtual address in
284901a9 2645 * DMA_BIT_MASK(32) and if that fails then try allocating
3609801e 2646 * from higher range
ba395927 2647 */
875764de
DW		 2648 iova = alloc_iova(&domain->iovad, nrpages,
2649 IOVA_PFN(DMA_BIT_MASK(32)), 1);
2650 if (iova)
2651 return iova;
2652 }
2653 iova = alloc_iova(&domain->iovad, nrpages, IOVA_PFN(dma_mask), 1);
2654 if (unlikely(!iova)) {
2655 printk(KERN_ERR "Allocating %ld-page iova for %s failed",
2656 nrpages, pci_name(pdev));
f76aec76
KA		 2657 return NULL;
2658 }
2659
2660 return iova;
2661}
2662
147202aa 2663static struct dmar_domain *__get_valid_domain_for_dev(struct pci_dev *pdev)
f76aec76
KA		 2664{
2665 struct dmar_domain *domain;
2666 int ret;
2667
2668 domain = get_domain_for_dev(pdev,
2669 DEFAULT_DOMAIN_ADDRESS_WIDTH);
2670 if (!domain) {
2671 printk(KERN_ERR
2672 "Allocating domain for %s failed", pci_name(pdev));
4fe05bbc 2673 return NULL;
ba395927
KA		 2674 }
2675
2676 /* make sure context mapping is ok */
5331fe6f 2677 if (unlikely(!domain_context_mapped(pdev))) {
4ed0d3e6
FY		 2678 ret = domain_context_mapping(domain, pdev,
2679 CONTEXT_TT_MULTI_LEVEL);
f76aec76
KA		 2680 if (ret) {
2681 printk(KERN_ERR
2682 "Domain context map for %s failed",
2683 pci_name(pdev));
4fe05bbc 2684 return NULL;
f76aec76 2685 }
ba395927
KA		 2686 }
2687
f76aec76
KA		 2688 return domain;
2689}
2690
147202aa
DW		 2691static inline struct dmar_domain *get_valid_domain_for_dev(struct pci_dev *dev)
2692{
2693 struct device_domain_info *info;
2694
2695 /* No lock here, assumes no domain exit in normal case */
2696 info = dev->dev.archdata.iommu;
2697 if (likely(info))
2698 return info->domain;
2699
2700 return __get_valid_domain_for_dev(dev);
2701}
2702
2c2e2c38
FY		 2703static int iommu_dummy(struct pci_dev *pdev)
2704{
2705 return pdev->dev.archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO;
2706}
2707
2708/* Check if the pdev needs to go through non-identity map and unmap process.*/
73676832 2709static int iommu_no_mapping(struct device *dev)
2c2e2c38 2710{
73676832 2711 struct pci_dev *pdev;
2c2e2c38
FY		 2712 int found;
2713
dbad0864 2714 if (unlikely(!dev_is_pci(dev)))
73676832
DW		 2715 return 1;
2716
2717 pdev = to_pci_dev(dev);
1e4c64c4
DW		 2718 if (iommu_dummy(pdev))
2719 return 1;
2720
2c2e2c38 2721 if (!iommu_identity_mapping)
1e4c64c4 2722 return 0;
2c2e2c38
FY		 2723
2724 found = identity_mapping(pdev);
2725 if (found) {
6941af28 2726 if (iommu_should_identity_map(pdev, 0))
2c2e2c38
FY		 2727 return 1;
2728 else {
2729 /*
2730 * 32 bit DMA is removed from si_domain and fall back
2731 * to non-identity mapping.
2732 */
2733 domain_remove_one_dev_info(si_domain, pdev);
2734 printk(KERN_INFO "32bit %s uses non-identity mapping\n",
2735 pci_name(pdev));
2736 return 0;
2737 }
2738 } else {
2739 /*
2740 * In case of a detached 64 bit DMA device from vm, the device
2741 * is put into si_domain for identity mapping.
2742 */
6941af28 2743 if (iommu_should_identity_map(pdev, 0)) {
2c2e2c38 2744 int ret;
5fe60f4e
DW		 2745 ret = domain_add_dev_info(si_domain, pdev,
2746 hw_pass_through ?
2747 CONTEXT_TT_PASS_THROUGH :
2748 CONTEXT_TT_MULTI_LEVEL);
2c2e2c38
FY		 2749 if (!ret) {
2750 printk(KERN_INFO "64bit %s uses identity mapping\n",
2751 pci_name(pdev));
2752 return 1;
2753 }
2754 }
2755 }
2756
1e4c64c4 2757 return 0;
2c2e2c38
FY		 2758}
2759
bb9e6d65
FT		 2760static dma_addr_t __intel_map_single(struct device *hwdev, phys_addr_t paddr,
2761 size_t size, int dir, u64 dma_mask)
f76aec76
KA		 2762{
2763 struct pci_dev *pdev = to_pci_dev(hwdev);
f76aec76 2764 struct dmar_domain *domain;
5b6985ce 2765 phys_addr_t start_paddr;
f76aec76
KA		 2766 struct iova *iova;
2767 int prot = 0;
6865f0d1 2768 int ret;
8c11e798 2769 struct intel_iommu *iommu;
33041ec0 2770 unsigned long paddr_pfn = paddr >> PAGE_SHIFT;
f76aec76
KA		 2771
2772 BUG_ON(dir == DMA_NONE);
2c2e2c38 2773
73676832 2774 if (iommu_no_mapping(hwdev))
6865f0d1 2775 return paddr;
f76aec76
KA		 2776
2777 domain = get_valid_domain_for_dev(pdev);
2778 if (!domain)
2779 return 0;
2780
8c11e798 2781 iommu = domain_get_iommu(domain);
88cb6a74 2782 size = aligned_nrpages(paddr, size);
f76aec76 2783
c681d0ba 2784 iova = intel_alloc_iova(hwdev, domain, dma_to_mm_pfn(size), dma_mask);
f76aec76
KA		 2785 if (!iova)
2786 goto error;
2787
ba395927
KA		 2788 /*
2789 * Check if DMAR supports zero-length reads on write only
2790 * mappings..
2791 */
2792 if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
8c11e798 2793 !cap_zlr(iommu->cap))
ba395927
KA		 2794 prot |= DMA_PTE_READ;
2795 if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
2796 prot |= DMA_PTE_WRITE;
2797 /*
6865f0d1 2798 * paddr - (paddr + size) might be partial page, we should map the whole
ba395927 2799 * page. Note: if two part of one page are separately mapped, we
6865f0d1 2800 * might have two guest_addr mapping to the same host paddr, but this
ba395927
KA		 2801 * is not a big problem
2802 */
0ab36de2 2803 ret = domain_pfn_mapping(domain, mm_to_dma_pfn(iova->pfn_lo),
33041ec0 2804 mm_to_dma_pfn(paddr_pfn), size, prot);
ba395927
KA		 2805 if (ret)
2806 goto error;
2807
1f0ef2aa
DW		 2808 /* it's a non-present to present mapping. Only flush if caching mode */
2809 if (cap_caching_mode(iommu->cap))
82653633 2810 iommu_flush_iotlb_psi(iommu, domain->id, mm_to_dma_pfn(iova->pfn_lo), size, 1);
1f0ef2aa 2811 else
8c11e798 2812 iommu_flush_write_buffer(iommu);
f76aec76 2813
03d6a246
DW		 2814 start_paddr = (phys_addr_t)iova->pfn_lo << PAGE_SHIFT;
2815 start_paddr += paddr & ~PAGE_MASK;
2816 return start_paddr;
ba395927 2817
ba395927 2818error:
f76aec76
KA		 2819 if (iova)
2820 __free_iova(&domain->iovad, iova);
4cf2e75d 2821 printk(KERN_ERR"Device %s request: %zx@%llx dir %d --- failed\n",
5b6985ce 2822 pci_name(pdev), size, (unsigned long long)paddr, dir);
ba395927
KA		 2823 return 0;
2824}
2825
ffbbef5c
FT		 2826static dma_addr_t intel_map_page(struct device *dev, struct page *page,
2827 unsigned long offset, size_t size,
2828 enum dma_data_direction dir,
2829 struct dma_attrs *attrs)
bb9e6d65 2830{
ffbbef5c
FT		 2831 return __intel_map_single(dev, page_to_phys(page) + offset, size,
2832 dir, to_pci_dev(dev)->dma_mask);
bb9e6d65
FT		 2833}
2834
5e0d2a6f 2835static void flush_unmaps(void)
2836{
80b20dd8 2837 int i, j;
5e0d2a6f 2838
5e0d2a6f 2839 timer_on = 0;
2840
2841 /* just flush them all */
2842 for (i = 0; i < g_num_of_iommus; i++) {
a2bb8459
WH		 2843 struct intel_iommu *iommu = g_iommus[i];
2844 if (!iommu)
2845 continue;
c42d9f32 2846
9dd2fe89
YZ		 2847 if (!deferred_flush[i].next)
2848 continue;
2849
78d5f0f5
NA		 2850 /* In caching mode, global flushes turn emulation expensive */
2851 if (!cap_caching_mode(iommu->cap))
2852 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
93a23a72 2853 DMA_TLB_GLOBAL_FLUSH);
9dd2fe89 2854 for (j = 0; j < deferred_flush[i].next; j++) {
93a23a72
YZ		 2855 unsigned long mask;
2856 struct iova *iova = deferred_flush[i].iova[j];
78d5f0f5
NA		 2857 struct dmar_domain *domain = deferred_flush[i].domain[j];
2858
2859 /* On real hardware multiple invalidations are expensive */
2860 if (cap_caching_mode(iommu->cap))
2861 iommu_flush_iotlb_psi(iommu, domain->id,
2862 iova->pfn_lo, iova->pfn_hi - iova->pfn_lo + 1, 0);
2863 else {
2864 mask = ilog2(mm_to_dma_pfn(iova->pfn_hi - iova->pfn_lo + 1));
2865 iommu_flush_dev_iotlb(deferred_flush[i].domain[j],
2866 (uint64_t)iova->pfn_lo << PAGE_SHIFT, mask);
2867 }
93a23a72 2868 __free_iova(&deferred_flush[i].domain[j]->iovad, iova);
80b20dd8 2869 }
9dd2fe89 2870 deferred_flush[i].next = 0;
5e0d2a6f 2871 }
2872
5e0d2a6f 2873 list_size = 0;
5e0d2a6f 2874}
2875
2876static void flush_unmaps_timeout(unsigned long data)
2877{
80b20dd8 2878 unsigned long flags;
2879
2880 spin_lock_irqsave(&async_umap_flush_lock, flags);
5e0d2a6f 2881 flush_unmaps();
80b20dd8 2882 spin_unlock_irqrestore(&async_umap_flush_lock, flags);
5e0d2a6f 2883}
2884
2885static void add_unmap(struct dmar_domain *dom, struct iova *iova)
2886{
2887 unsigned long flags;
80b20dd8 2888 int next, iommu_id;
8c11e798 2889 struct intel_iommu *iommu;
5e0d2a6f 2890
2891 spin_lock_irqsave(&async_umap_flush_lock, flags);
80b20dd8 2892 if (list_size == HIGH_WATER_MARK)
2893 flush_unmaps();
2894
8c11e798
WH		 2895 iommu = domain_get_iommu(dom);
2896 iommu_id = iommu->seq_id;
c42d9f32 2897
80b20dd8 2898 next = deferred_flush[iommu_id].next;
2899 deferred_flush[iommu_id].domain[next] = dom;
2900 deferred_flush[iommu_id].iova[next] = iova;
2901 deferred_flush[iommu_id].next++;
5e0d2a6f 2902
2903 if (!timer_on) {
2904 mod_timer(&unmap_timer, jiffies + msecs_to_jiffies(10));
2905 timer_on = 1;
2906 }
2907 list_size++;
2908 spin_unlock_irqrestore(&async_umap_flush_lock, flags);
2909}
2910
ffbbef5c
FT		 2911static void intel_unmap_page(struct device *dev, dma_addr_t dev_addr,
2912 size_t size, enum dma_data_direction dir,
2913 struct dma_attrs *attrs)
ba395927 2914{
ba395927 2915 struct pci_dev *pdev = to_pci_dev(dev);
f76aec76 2916 struct dmar_domain *domain;
d794dc9b 2917 unsigned long start_pfn, last_pfn;
ba395927 2918 struct iova *iova;
8c11e798 2919 struct intel_iommu *iommu;
ba395927 2920
73676832 2921 if (iommu_no_mapping(dev))
f76aec76 2922 return;
2c2e2c38 2923
ba395927
KA		 2924 domain = find_domain(pdev);
2925 BUG_ON(!domain);
2926
8c11e798
WH		 2927 iommu = domain_get_iommu(domain);
2928
ba395927 2929 iova = find_iova(&domain->iovad, IOVA_PFN(dev_addr));
85b98276
DW		 2930 if (WARN_ONCE(!iova, "Driver unmaps unmatched page at PFN %llx\n",
2931 (unsigned long long)dev_addr))
ba395927 2932 return;
ba395927 2933
d794dc9b
DW		 2934 start_pfn = mm_to_dma_pfn(iova->pfn_lo);
2935 last_pfn = mm_to_dma_pfn(iova->pfn_hi + 1) - 1;
ba395927 2936
d794dc9b
DW		 2937 pr_debug("Device %s unmapping: pfn %lx-%lx\n",
2938 pci_name(pdev), start_pfn, last_pfn);
ba395927 2939
f76aec76 2940 /* clear the whole page */
d794dc9b
DW		 2941 dma_pte_clear_range(domain, start_pfn, last_pfn);
2942
f76aec76 2943 /* free page tables */
d794dc9b
DW		 2944 dma_pte_free_pagetable(domain, start_pfn, last_pfn);
2945
5e0d2a6f 2946 if (intel_iommu_strict) {
03d6a246 2947 iommu_flush_iotlb_psi(iommu, domain->id, start_pfn,
82653633 2948 last_pfn - start_pfn + 1, 0);
5e0d2a6f 2949 /* free iova */
2950 __free_iova(&domain->iovad, iova);
2951 } else {
2952 add_unmap(domain, iova);
2953 /*
2954 * queue up the release of the unmap to save the 1/6th of the
2955 * cpu used up by the iotlb flush operation...
2956 */
5e0d2a6f 2957 }
ba395927
KA		 2958}
2959
d7ab5c46 2960static void *intel_alloc_coherent(struct device *hwdev, size_t size,
baa676fc
AP		 2961 dma_addr_t *dma_handle, gfp_t flags,
2962 struct dma_attrs *attrs)
ba395927
KA		 2963{
2964 void *vaddr;
2965 int order;
2966
5b6985ce 2967 size = PAGE_ALIGN(size);
ba395927 2968 order = get_order(size);
e8bb910d
AW		 2969
2970 if (!iommu_no_mapping(hwdev))
2971 flags &= ~(GFP_DMA | GFP_DMA32);
2972 else if (hwdev->coherent_dma_mask < dma_get_required_mask(hwdev)) {
2973 if (hwdev->coherent_dma_mask < DMA_BIT_MASK(32))
2974 flags |= GFP_DMA;
2975 else
2976 flags |= GFP_DMA32;
2977 }
ba395927
KA		 2978
2979 vaddr = (void *)__get_free_pages(flags, order);
2980 if (!vaddr)
2981 return NULL;
2982 memset(vaddr, 0, size);
2983
bb9e6d65
FT		 2984 *dma_handle = __intel_map_single(hwdev, virt_to_bus(vaddr), size,
2985 DMA_BIDIRECTIONAL,
2986 hwdev->coherent_dma_mask);
ba395927
KA		 2987 if (*dma_handle)
2988 return vaddr;
2989 free_pages((unsigned long)vaddr, order);
2990 return NULL;
2991}
2992
d7ab5c46 2993static void intel_free_coherent(struct device *hwdev, size_t size, void *vaddr,
baa676fc 2994 dma_addr_t dma_handle, struct dma_attrs *attrs)
ba395927
KA		 2995{
2996 int order;
2997
5b6985ce 2998 size = PAGE_ALIGN(size);
ba395927
KA		 2999 order = get_order(size);
3000
0db9b7ae 3001 intel_unmap_page(hwdev, dma_handle, size, DMA_BIDIRECTIONAL, NULL);
ba395927
KA		 3002 free_pages((unsigned long)vaddr, order);
3003}
3004
d7ab5c46
FT		 3005static void intel_unmap_sg(struct device *hwdev, struct scatterlist *sglist,
3006 int nelems, enum dma_data_direction dir,
3007 struct dma_attrs *attrs)
ba395927 3008{
ba395927
KA		 3009 struct pci_dev *pdev = to_pci_dev(hwdev);
3010 struct dmar_domain *domain;
d794dc9b 3011 unsigned long start_pfn, last_pfn;
f76aec76 3012 struct iova *iova;
8c11e798 3013 struct intel_iommu *iommu;
ba395927 3014
73676832 3015 if (iommu_no_mapping(hwdev))
ba395927
KA		 3016 return;
3017
3018 domain = find_domain(pdev);
8c11e798
WH		 3019 BUG_ON(!domain);
3020
3021 iommu = domain_get_iommu(domain);
ba395927 3022
c03ab37c 3023 iova = find_iova(&domain->iovad, IOVA_PFN(sglist[0].dma_address));
85b98276
DW		 3024 if (WARN_ONCE(!iova, "Driver unmaps unmatched sglist at PFN %llx\n",
3025 (unsigned long long)sglist[0].dma_address))
f76aec76 3026 return;
f76aec76 3027
d794dc9b
DW		 3028 start_pfn = mm_to_dma_pfn(iova->pfn_lo);
3029 last_pfn = mm_to_dma_pfn(iova->pfn_hi + 1) - 1;
f76aec76
KA		 3030
3031 /* clear the whole page */
d794dc9b
DW		 3032 dma_pte_clear_range(domain, start_pfn, last_pfn);
3033
f76aec76 3034 /* free page tables */
d794dc9b 3035 dma_pte_free_pagetable(domain, start_pfn, last_pfn);
f76aec76 3036
acea0018
DW		 3037 if (intel_iommu_strict) {
3038 iommu_flush_iotlb_psi(iommu, domain->id, start_pfn,
82653633 3039 last_pfn - start_pfn + 1, 0);
acea0018
DW		 3040 /* free iova */
3041 __free_iova(&domain->iovad, iova);
3042 } else {
3043 add_unmap(domain, iova);
3044 /*
3045 * queue up the release of the unmap to save the 1/6th of the
3046 * cpu used up by the iotlb flush operation...
3047 */
3048 }
ba395927
KA		 3049}
3050
ba395927 3051static int intel_nontranslate_map_sg(struct device *hddev,
c03ab37c 3052 struct scatterlist *sglist, int nelems, int dir)
ba395927
KA		 3053{
3054 int i;
c03ab37c 3055 struct scatterlist *sg;
ba395927 3056
c03ab37c 3057 for_each_sg(sglist, sg, nelems, i) {
12d4d40e 3058 BUG_ON(!sg_page(sg));
4cf2e75d 3059 sg->dma_address = page_to_phys(sg_page(sg)) + sg->offset;
c03ab37c 3060 sg->dma_length = sg->length;
ba395927
KA		 3061 }
3062 return nelems;
3063}
3064
d7ab5c46
FT		 3065static int intel_map_sg(struct device *hwdev, struct scatterlist *sglist, int nelems,
3066 enum dma_data_direction dir, struct dma_attrs *attrs)
ba395927 3067{
ba395927 3068 int i;
ba395927
KA		 3069 struct pci_dev *pdev = to_pci_dev(hwdev);
3070 struct dmar_domain *domain;
f76aec76
KA		 3071 size_t size = 0;
3072 int prot = 0;
f76aec76
KA		 3073 struct iova *iova = NULL;
3074 int ret;
c03ab37c 3075 struct scatterlist *sg;
b536d24d 3076 unsigned long start_vpfn;
8c11e798 3077 struct intel_iommu *iommu;
ba395927
KA		 3078
3079 BUG_ON(dir == DMA_NONE);
73676832 3080 if (iommu_no_mapping(hwdev))
c03ab37c 3081 return intel_nontranslate_map_sg(hwdev, sglist, nelems, dir);
ba395927 3082
f76aec76
KA		 3083 domain = get_valid_domain_for_dev(pdev);
3084 if (!domain)
3085 return 0;
3086
8c11e798
WH		 3087 iommu = domain_get_iommu(domain);
3088
b536d24d 3089 for_each_sg(sglist, sg, nelems, i)
88cb6a74 3090 size += aligned_nrpages(sg->offset, sg->length);
f76aec76 3091
5a5e02a6
DW		 3092 iova = intel_alloc_iova(hwdev, domain, dma_to_mm_pfn(size),
3093 pdev->dma_mask);
f76aec76 3094 if (!iova) {
c03ab37c 3095 sglist->dma_length = 0;
f76aec76
KA		 3096 return 0;
3097 }
3098
3099 /*
3100 * Check if DMAR supports zero-length reads on write only
3101 * mappings..
3102 */
3103 if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
8c11e798 3104 !cap_zlr(iommu->cap))
f76aec76
KA		 3105 prot |= DMA_PTE_READ;
3106 if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
3107 prot |= DMA_PTE_WRITE;
3108
b536d24d 3109 start_vpfn = mm_to_dma_pfn(iova->pfn_lo);
e1605495 3110
f532959b 3111 ret = domain_sg_mapping(domain, start_vpfn, sglist, size, prot);
e1605495
DW		 3112 if (unlikely(ret)) {
3113 /* clear the page */
3114 dma_pte_clear_range(domain, start_vpfn,
3115 start_vpfn + size - 1);
3116 /* free page tables */
3117 dma_pte_free_pagetable(domain, start_vpfn,
3118 start_vpfn + size - 1);
3119 /* free iova */
3120 __free_iova(&domain->iovad, iova);
3121 return 0;
ba395927
KA		 3122 }
3123
1f0ef2aa
DW		 3124 /* it's a non-present to present mapping. Only flush if caching mode */
3125 if (cap_caching_mode(iommu->cap))
82653633 3126 iommu_flush_iotlb_psi(iommu, domain->id, start_vpfn, size, 1);
1f0ef2aa 3127 else
8c11e798 3128 iommu_flush_write_buffer(iommu);
1f0ef2aa 3129
ba395927
KA		 3130 return nelems;
3131}
3132
dfb805e8
FT		 3133static int intel_mapping_error(struct device *dev, dma_addr_t dma_addr)
3134{
3135 return !dma_addr;
3136}
3137
160c1d8e 3138struct dma_map_ops intel_dma_ops = {
baa676fc
AP		 3139 .alloc = intel_alloc_coherent,
3140 .free = intel_free_coherent,
ba395927
KA		 3141 .map_sg = intel_map_sg,
3142 .unmap_sg = intel_unmap_sg,
ffbbef5c
FT		 3143 .map_page = intel_map_page,
3144 .unmap_page = intel_unmap_page,
dfb805e8 3145 .mapping_error = intel_mapping_error,
ba395927
KA		 3146};
3147
3148static inline int iommu_domain_cache_init(void)
3149{
3150 int ret = 0;
3151
3152 iommu_domain_cache = kmem_cache_create("iommu_domain",
3153 sizeof(struct dmar_domain),
3154 0,
3155 SLAB_HWCACHE_ALIGN,
3156
3157 NULL);
3158 if (!iommu_domain_cache) {
3159 printk(KERN_ERR "Couldn't create iommu_domain cache\n");
3160 ret = -ENOMEM;
3161 }
3162
3163 return ret;
3164}
3165
3166static inline int iommu_devinfo_cache_init(void)
3167{
3168 int ret = 0;
3169
3170 iommu_devinfo_cache = kmem_cache_create("iommu_devinfo",
3171 sizeof(struct device_domain_info),
3172 0,
3173 SLAB_HWCACHE_ALIGN,
ba395927
KA		 3174 NULL);
3175 if (!iommu_devinfo_cache) {
3176 printk(KERN_ERR "Couldn't create devinfo cache\n");
3177 ret = -ENOMEM;
3178 }
3179
3180 return ret;
3181}
3182
3183static inline int iommu_iova_cache_init(void)
3184{
3185 int ret = 0;
3186
3187 iommu_iova_cache = kmem_cache_create("iommu_iova",
3188 sizeof(struct iova),
3189 0,
3190 SLAB_HWCACHE_ALIGN,
ba395927
KA		 3191 NULL);
3192 if (!iommu_iova_cache) {
3193 printk(KERN_ERR "Couldn't create iova cache\n");
3194 ret = -ENOMEM;
3195 }
3196
3197 return ret;
3198}
3199
3200static int __init iommu_init_mempool(void)
3201{
3202 int ret;
3203 ret = iommu_iova_cache_init();
3204 if (ret)
3205 return ret;
3206
3207 ret = iommu_domain_cache_init();
3208 if (ret)
3209 goto domain_error;
3210
3211 ret = iommu_devinfo_cache_init();
3212 if (!ret)
3213 return ret;
3214
3215 kmem_cache_destroy(iommu_domain_cache);
3216domain_error:
3217 kmem_cache_destroy(iommu_iova_cache);
3218
3219 return -ENOMEM;
3220}
3221
3222static void __init iommu_exit_mempool(void)
3223{
3224 kmem_cache_destroy(iommu_devinfo_cache);
3225 kmem_cache_destroy(iommu_domain_cache);
3226 kmem_cache_destroy(iommu_iova_cache);
3227
3228}
3229
556ab45f
DW		 3230static void quirk_ioat_snb_local_iommu(struct pci_dev *pdev)
3231{
3232 struct dmar_drhd_unit *drhd;
3233 u32 vtbar;
3234 int rc;
3235
3236 /* We know that this device on this chipset has its own IOMMU.
3237 * If we find it under a different IOMMU, then the BIOS is lying
3238 * to us. Hope that the IOMMU for this device is actually
3239 * disabled, and it needs no translation...
3240 */
3241 rc = pci_bus_read_config_dword(pdev->bus, PCI_DEVFN(0, 0), 0xb0, &vtbar);
3242 if (rc) {
3243 /* "can't" happen */
3244 dev_info(&pdev->dev, "failed to run vt-d quirk\n");
3245 return;
3246 }
3247 vtbar &= 0xffff0000;
3248
3249 /* we know that the this iommu should be at offset 0xa000 from vtbar */
3250 drhd = dmar_find_matched_drhd_unit(pdev);
3251 if (WARN_TAINT_ONCE(!drhd || drhd->reg_base_addr - vtbar != 0xa000,
3252 TAINT_FIRMWARE_WORKAROUND,
3253 "BIOS assigned incorrect VT-d unit for Intel(R) QuickData Technology device\n"))
3254 pdev->dev.archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO;
3255}
3256DECLARE_PCI_FIXUP_ENABLE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB, quirk_ioat_snb_local_iommu);
3257
ba395927
KA		 3258static void __init init_no_remapping_devices(void)
3259{
3260 struct dmar_drhd_unit *drhd;
3261
3262 for_each_drhd_unit(drhd) {
3263 if (!drhd->include_all) {
3264 int i;
3265 for (i = 0; i < drhd->devices_cnt; i++)
3266 if (drhd->devices[i] != NULL)
3267 break;
3268 /* ignore DMAR unit if no pci devices exist */
3269 if (i == drhd->devices_cnt)
3270 drhd->ignored = 1;
3271 }
3272 }
3273
7c919779 3274 for_each_active_drhd_unit(drhd) {
ba395927 3275 int i;
7c919779 3276 if (drhd->include_all)
ba395927
KA		 3277 continue;
3278
3279 for (i = 0; i < drhd->devices_cnt; i++)
3280 if (drhd->devices[i] &&
c0771df8 3281 !IS_GFX_DEVICE(drhd->devices[i]))
ba395927
KA		 3282 break;
3283
3284 if (i < drhd->devices_cnt)
3285 continue;
3286
c0771df8
DW		 3287 /* This IOMMU has *only* gfx devices. Either bypass it or
3288 set the gfx_mapped flag, as appropriate */
3289 if (dmar_map_gfx) {
3290 intel_iommu_gfx_mapped = 1;
3291 } else {
3292 drhd->ignored = 1;
3293 for (i = 0; i < drhd->devices_cnt; i++) {
3294 if (!drhd->devices[i])
3295 continue;
3296 drhd->devices[i]->dev.archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO;
3297 }
ba395927
KA		 3298 }
3299 }
3300}
3301
f59c7b69
FY		 3302#ifdef CONFIG_SUSPEND
3303static int init_iommu_hw(void)
3304{
3305 struct dmar_drhd_unit *drhd;
3306 struct intel_iommu *iommu = NULL;
3307
3308 for_each_active_iommu(iommu, drhd)
3309 if (iommu->qi)
3310 dmar_reenable_qi(iommu);
3311
b779260b
JC		 3312 for_each_iommu(iommu, drhd) {
3313 if (drhd->ignored) {
3314 /*
3315 * we always have to disable PMRs or DMA may fail on
3316 * this device
3317 */
3318 if (force_on)
3319 iommu_disable_protect_mem_regions(iommu);
3320 continue;
3321 }
3322
f59c7b69
FY		 3323 iommu_flush_write_buffer(iommu);
3324
3325 iommu_set_root_entry(iommu);
3326
3327 iommu->flush.flush_context(iommu, 0, 0, 0,
1f0ef2aa 3328 DMA_CCMD_GLOBAL_INVL);
f59c7b69 3329 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
1f0ef2aa 3330 DMA_TLB_GLOBAL_FLUSH);
b779260b
JC		 3331 if (iommu_enable_translation(iommu))
3332 return 1;
b94996c9 3333 iommu_disable_protect_mem_regions(iommu);
f59c7b69
FY		 3334 }
3335
3336 return 0;
3337}
3338
3339static void iommu_flush_all(void)
3340{
3341 struct dmar_drhd_unit *drhd;
3342 struct intel_iommu *iommu;
3343
3344 for_each_active_iommu(iommu, drhd) {
3345 iommu->flush.flush_context(iommu, 0, 0, 0,
1f0ef2aa 3346 DMA_CCMD_GLOBAL_INVL);
f59c7b69 3347 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
1f0ef2aa 3348 DMA_TLB_GLOBAL_FLUSH);
f59c7b69
FY		 3349 }
3350}
3351
134fac3f 3352static int iommu_suspend(void)
f59c7b69
FY		 3353{
3354 struct dmar_drhd_unit *drhd;
3355 struct intel_iommu *iommu = NULL;
3356 unsigned long flag;
3357
3358 for_each_active_iommu(iommu, drhd) {
3359 iommu->iommu_state = kzalloc(sizeof(u32) * MAX_SR_DMAR_REGS,
3360 GFP_ATOMIC);
3361 if (!iommu->iommu_state)
3362 goto nomem;
3363 }
3364
3365 iommu_flush_all();
3366
3367 for_each_active_iommu(iommu, drhd) {
3368 iommu_disable_translation(iommu);
3369
1f5b3c3f 3370 raw_spin_lock_irqsave(&iommu->register_lock, flag);
f59c7b69
FY		 3371
3372 iommu->iommu_state[SR_DMAR_FECTL_REG] =
3373 readl(iommu->reg + DMAR_FECTL_REG);
3374 iommu->iommu_state[SR_DMAR_FEDATA_REG] =
3375 readl(iommu->reg + DMAR_FEDATA_REG);
3376 iommu->iommu_state[SR_DMAR_FEADDR_REG] =
3377 readl(iommu->reg + DMAR_FEADDR_REG);
3378 iommu->iommu_state[SR_DMAR_FEUADDR_REG] =
3379 readl(iommu->reg + DMAR_FEUADDR_REG);
3380
1f5b3c3f 3381 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
f59c7b69
FY		 3382 }
3383 return 0;
3384
3385nomem:
3386 for_each_active_iommu(iommu, drhd)
3387 kfree(iommu->iommu_state);
3388
3389 return -ENOMEM;
3390}
3391
134fac3f 3392static void iommu_resume(void)
f59c7b69
FY		 3393{
3394 struct dmar_drhd_unit *drhd;
3395 struct intel_iommu *iommu = NULL;
3396 unsigned long flag;
3397
3398 if (init_iommu_hw()) {
b779260b
JC		 3399 if (force_on)
3400 panic("tboot: IOMMU setup failed, DMAR can not resume!\n");
3401 else
3402 WARN(1, "IOMMU setup failed, DMAR can not resume!\n");
134fac3f 3403 return;
f59c7b69
FY		 3404 }
3405
3406 for_each_active_iommu(iommu, drhd) {
3407
1f5b3c3f 3408 raw_spin_lock_irqsave(&iommu->register_lock, flag);
f59c7b69
FY		 3409
3410 writel(iommu->iommu_state[SR_DMAR_FECTL_REG],
3411 iommu->reg + DMAR_FECTL_REG);
3412 writel(iommu->iommu_state[SR_DMAR_FEDATA_REG],
3413 iommu->reg + DMAR_FEDATA_REG);
3414 writel(iommu->iommu_state[SR_DMAR_FEADDR_REG],
3415 iommu->reg + DMAR_FEADDR_REG);
3416 writel(iommu->iommu_state[SR_DMAR_FEUADDR_REG],
3417 iommu->reg + DMAR_FEUADDR_REG);
3418
1f5b3c3f 3419 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
f59c7b69
FY		 3420 }
3421
3422 for_each_active_iommu(iommu, drhd)
3423 kfree(iommu->iommu_state);
f59c7b69
FY		 3424}
3425
134fac3f 3426static struct syscore_ops iommu_syscore_ops = {
f59c7b69
FY		 3427 .resume = iommu_resume,
3428 .suspend = iommu_suspend,
3429};
3430
134fac3f 3431static void __init init_iommu_pm_ops(void)
f59c7b69 3432{
134fac3f 3433 register_syscore_ops(&iommu_syscore_ops);
f59c7b69
FY		 3434}
3435
3436#else
99592ba4 3437static inline void init_iommu_pm_ops(void) {}
f59c7b69
FY		 3438#endif /* CONFIG_PM */
3439
318fe7df
SS		 3440LIST_HEAD(dmar_rmrr_units);
3441
3442static void __init dmar_register_rmrr_unit(struct dmar_rmrr_unit *rmrr)
3443{
3444 list_add(&rmrr->list, &dmar_rmrr_units);
3445}
3446
3447
3448int __init dmar_parse_one_rmrr(struct acpi_dmar_header *header)
3449{
3450 struct acpi_dmar_reserved_memory *rmrr;
3451 struct dmar_rmrr_unit *rmrru;
3452
3453 rmrru = kzalloc(sizeof(*rmrru), GFP_KERNEL);
3454 if (!rmrru)
3455 return -ENOMEM;
3456
3457 rmrru->hdr = header;
3458 rmrr = (struct acpi_dmar_reserved_memory *)header;
3459 rmrru->base_address = rmrr->base_address;
3460 rmrru->end_address = rmrr->end_address;
3461
3462 dmar_register_rmrr_unit(rmrru);
3463 return 0;
3464}
3465
3466static int __init
3467rmrr_parse_dev(struct dmar_rmrr_unit *rmrru)
3468{
3469 struct acpi_dmar_reserved_memory *rmrr;
3470 int ret;
3471
3472 rmrr = (struct acpi_dmar_reserved_memory *) rmrru->hdr;
3473 ret = dmar_parse_dev_scope((void *)(rmrr + 1),
3474 ((void *)rmrr) + rmrr->header.length,
3475 &rmrru->devices_cnt, &rmrru->devices, rmrr->segment);
3476
3477 if (ret || (rmrru->devices_cnt == 0)) {
3478 list_del(&rmrru->list);
3479 kfree(rmrru);
3480 }
3481 return ret;
3482}
3483
3484static LIST_HEAD(dmar_atsr_units);
3485
3486int __init dmar_parse_one_atsr(struct acpi_dmar_header *hdr)
3487{
3488 struct acpi_dmar_atsr *atsr;
3489 struct dmar_atsr_unit *atsru;
3490
3491 atsr = container_of(hdr, struct acpi_dmar_atsr, header);
3492 atsru = kzalloc(sizeof(*atsru), GFP_KERNEL);
3493 if (!atsru)
3494 return -ENOMEM;
3495
3496 atsru->hdr = hdr;
3497 atsru->include_all = atsr->flags & 0x1;
3498
3499 list_add(&atsru->list, &dmar_atsr_units);
3500
3501 return 0;
3502}
3503
3504static int __init atsr_parse_dev(struct dmar_atsr_unit *atsru)
3505{
3506 int rc;
3507 struct acpi_dmar_atsr *atsr;
3508
3509 if (atsru->include_all)
3510 return 0;
3511
3512 atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header);
3513 rc = dmar_parse_dev_scope((void *)(atsr + 1),
3514 (void *)atsr + atsr->header.length,
3515 &atsru->devices_cnt, &atsru->devices,
3516 atsr->segment);
3517 if (rc || !atsru->devices_cnt) {
3518 list_del(&atsru->list);
3519 kfree(atsru);
3520 }
3521
3522 return rc;
3523}
3524
3525int dmar_find_matched_atsr_unit(struct pci_dev *dev)
3526{
3527 int i;
3528 struct pci_bus *bus;
3529 struct acpi_dmar_atsr *atsr;
3530 struct dmar_atsr_unit *atsru;
3531
3532 dev = pci_physfn(dev);
3533
3534 list_for_each_entry(atsru, &dmar_atsr_units, list) {
3535 atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header);
3536 if (atsr->segment == pci_domain_nr(dev->bus))
3537 goto found;
3538 }
3539
3540 return 0;
3541
3542found:
3543 for (bus = dev->bus; bus; bus = bus->parent) {
3544 struct pci_dev *bridge = bus->self;
3545
3546 if (!bridge || !pci_is_pcie(bridge) ||
62f87c0e 3547 pci_pcie_type(bridge) == PCI_EXP_TYPE_PCI_BRIDGE)
318fe7df
SS		 3548 return 0;
3549
62f87c0e 3550 if (pci_pcie_type(bridge) == PCI_EXP_TYPE_ROOT_PORT) {
318fe7df
SS		 3551 for (i = 0; i < atsru->devices_cnt; i++)
3552 if (atsru->devices[i] == bridge)
3553 return 1;
3554 break;
3555 }
3556 }
3557
3558 if (atsru->include_all)
3559 return 1;
3560
3561 return 0;
3562}
3563
c8f369ab 3564int __init dmar_parse_rmrr_atsr_dev(void)
318fe7df
SS		 3565{
3566 struct dmar_rmrr_unit *rmrr, *rmrr_n;
3567 struct dmar_atsr_unit *atsr, *atsr_n;
3568 int ret = 0;
3569
3570 list_for_each_entry_safe(rmrr, rmrr_n, &dmar_rmrr_units, list) {
3571 ret = rmrr_parse_dev(rmrr);
3572 if (ret)
3573 return ret;
3574 }
3575
3576 list_for_each_entry_safe(atsr, atsr_n, &dmar_atsr_units, list) {
3577 ret = atsr_parse_dev(atsr);
3578 if (ret)
3579 return ret;
3580 }
3581
3582 return ret;
3583}
3584
99dcaded
FY		 3585/*
3586 * Here we only respond to action of unbound device from driver.
3587 *
3588 * Added device is not attached to its DMAR domain here yet. That will happen
3589 * when mapping the device to iova.
3590 */
3591static int device_notifier(struct notifier_block *nb,
3592 unsigned long action, void *data)
3593{
3594 struct device *dev = data;
3595 struct pci_dev *pdev = to_pci_dev(dev);
3596 struct dmar_domain *domain;
3597
44cd613c
DW		 3598 if (iommu_no_mapping(dev))
3599 return 0;
3600
99dcaded
FY		 3601 domain = find_domain(pdev);
3602 if (!domain)
3603 return 0;
3604
a97590e5 3605 if (action == BUS_NOTIFY_UNBOUND_DRIVER && !iommu_pass_through) {
99dcaded
FY		 3606 domain_remove_one_dev_info(domain, pdev);
3607
a97590e5
AW		 3608 if (!(domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE) &&
3609 !(domain->flags & DOMAIN_FLAG_STATIC_IDENTITY) &&
3610 list_empty(&domain->devices))
3611 domain_exit(domain);
3612 }
3613
99dcaded
FY		 3614 return 0;
3615}
3616
3617static struct notifier_block device_nb = {
3618 .notifier_call = device_notifier,
3619};
3620
ba395927
KA		 3621int __init intel_iommu_init(void)
3622{
3623 int ret = 0;
3a93c841 3624 struct dmar_drhd_unit *drhd;
7c919779 3625 struct intel_iommu *iommu;
ba395927 3626
a59b50e9
JC		 3627 /* VT-d is required for a TXT/tboot launch, so enforce that */
3628 force_on = tboot_force_iommu();
3629
3630 if (dmar_table_init()) {
3631 if (force_on)
3632 panic("tboot: Failed to initialize DMAR table\n");
ba395927 3633 return -ENODEV;
a59b50e9 3634 }
ba395927 3635
3a93c841
TI		 3636 /*
3637 * Disable translation if already enabled prior to OS handover.
3638 */
7c919779 3639 for_each_active_iommu(iommu, drhd)
3a93c841
TI		 3640 if (iommu->gcmd & DMA_GCMD_TE)
3641 iommu_disable_translation(iommu);
3a93c841 3642
c2c7286a 3643 if (dmar_dev_scope_init() < 0) {
a59b50e9
JC		 3644 if (force_on)
3645 panic("tboot: Failed to initialize DMAR device scope\n");
1886e8a9 3646 return -ENODEV;
a59b50e9 3647 }
1886e8a9 3648
75f1cdf1 3649 if (no_iommu || dmar_disabled)
2ae21010
SS		 3650 return -ENODEV;
3651
51a63e67
JC		 3652 if (iommu_init_mempool()) {
3653 if (force_on)
3654 panic("tboot: Failed to initialize iommu memory\n");
3655 return -ENODEV;
3656 }
3657
318fe7df
SS		 3658 if (list_empty(&dmar_rmrr_units))
3659 printk(KERN_INFO "DMAR: No RMRR found\n");
3660
3661 if (list_empty(&dmar_atsr_units))
3662 printk(KERN_INFO "DMAR: No ATSR found\n");
3663
51a63e67
JC		 3664 if (dmar_init_reserved_ranges()) {
3665 if (force_on)
3666 panic("tboot: Failed to reserve iommu ranges\n");
3667 return -ENODEV;
3668 }
ba395927
KA		 3669
3670 init_no_remapping_devices();
3671
b779260b 3672 ret = init_dmars();
ba395927 3673 if (ret) {
a59b50e9
JC		 3674 if (force_on)
3675 panic("tboot: Failed to initialize DMARs\n");
ba395927
KA		 3676 printk(KERN_ERR "IOMMU: dmar init failed\n");
3677 put_iova_domain(&reserved_iova_list);
3678 iommu_exit_mempool();
3679 return ret;
3680 }
3681 printk(KERN_INFO
3682 "PCI-DMA: Intel(R) Virtualization Technology for Directed I/O\n");
3683
5e0d2a6f 3684 init_timer(&unmap_timer);
75f1cdf1
FT		 3685#ifdef CONFIG_SWIOTLB
3686 swiotlb = 0;
3687#endif
19943b0e 3688 dma_ops = &intel_dma_ops;
4ed0d3e6 3689
134fac3f 3690 init_iommu_pm_ops();
a8bcbb0d 3691
4236d97d 3692 bus_set_iommu(&pci_bus_type, &intel_iommu_ops);
a8bcbb0d 3693
99dcaded
FY		 3694 bus_register_notifier(&pci_bus_type, &device_nb);
3695
8bc1f85c
ED		 3696 intel_iommu_enabled = 1;
3697
ba395927
KA		 3698 return 0;
3699}
e820482c 3700
3199aa6b
HW		 3701static void iommu_detach_dependent_devices(struct intel_iommu *iommu,
3702 struct pci_dev *pdev)
3703{
3704 struct pci_dev *tmp, *parent;
3705
3706 if (!iommu || !pdev)
3707 return;
3708
3709 /* dependent device detach */
3710 tmp = pci_find_upstream_pcie_bridge(pdev);
3711 /* Secondary interface's bus number and devfn 0 */
3712 if (tmp) {
3713 parent = pdev->bus->self;
3714 while (parent != tmp) {
3715 iommu_detach_dev(iommu, parent->bus->number,
276dbf99 3716 parent->devfn);
3199aa6b
HW		 3717 parent = parent->bus->self;
3718 }
45e829ea 3719 if (pci_is_pcie(tmp)) /* this is a PCIe-to-PCI bridge */
3199aa6b
HW		 3720 iommu_detach_dev(iommu,
3721 tmp->subordinate->number, 0);
3722 else /* this is a legacy PCI bridge */
276dbf99
DW		 3723 iommu_detach_dev(iommu, tmp->bus->number,
3724 tmp->devfn);
3199aa6b
HW		 3725 }
3726}
3727
2c2e2c38 3728static void domain_remove_one_dev_info(struct dmar_domain *domain,
c7151a8d
WH		 3729 struct pci_dev *pdev)
3730{
bca2b916 3731 struct device_domain_info *info, *tmp;
c7151a8d
WH		 3732 struct intel_iommu *iommu;
3733 unsigned long flags;
3734 int found = 0;
c7151a8d 3735
276dbf99
DW		 3736 iommu = device_to_iommu(pci_domain_nr(pdev->bus), pdev->bus->number,
3737 pdev->devfn);
c7151a8d
WH		 3738 if (!iommu)
3739 return;
3740
3741 spin_lock_irqsave(&device_domain_lock, flags);
bca2b916 3742 list_for_each_entry_safe(info, tmp, &domain->devices, link) {
8519dc44
MH		 3743 if (info->segment == pci_domain_nr(pdev->bus) &&
3744 info->bus == pdev->bus->number &&
c7151a8d 3745 info->devfn == pdev->devfn) {
109b9b04 3746 unlink_domain_info(info);
c7151a8d
WH		 3747 spin_unlock_irqrestore(&device_domain_lock, flags);
3748
93a23a72 3749 iommu_disable_dev_iotlb(info);
c7151a8d 3750 iommu_detach_dev(iommu, info->bus, info->devfn);
3199aa6b 3751 iommu_detach_dependent_devices(iommu, pdev);
c7151a8d
WH		 3752 free_devinfo_mem(info);
3753
3754 spin_lock_irqsave(&device_domain_lock, flags);
3755
3756 if (found)
3757 break;
3758 else
3759 continue;
3760 }
3761
3762 /* if there is no other devices under the same iommu
3763 * owned by this domain, clear this iommu in iommu_bmp
3764 * update iommu count and coherency
3765 */
276dbf99
DW		 3766 if (iommu == device_to_iommu(info->segment, info->bus,
3767 info->devfn))
c7151a8d
WH		 3768 found = 1;
3769 }
3770
3e7abe25
RD		 3771 spin_unlock_irqrestore(&device_domain_lock, flags);
3772
c7151a8d
WH		 3773 if (found == 0) {
3774 unsigned long tmp_flags;
3775 spin_lock_irqsave(&domain->iommu_lock, tmp_flags);
1b198bb0 3776 clear_bit(iommu->seq_id, domain->iommu_bmp);
c7151a8d 3777 domain->iommu_count--;
58c610bd 3778 domain_update_iommu_cap(domain);
c7151a8d 3779 spin_unlock_irqrestore(&domain->iommu_lock, tmp_flags);
a97590e5 3780
9b4554b2
AW		 3781 if (!(domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE) &&
3782 !(domain->flags & DOMAIN_FLAG_STATIC_IDENTITY)) {
3783 spin_lock_irqsave(&iommu->lock, tmp_flags);
3784 clear_bit(domain->id, iommu->domain_ids);
3785 iommu->domains[domain->id] = NULL;
3786 spin_unlock_irqrestore(&iommu->lock, tmp_flags);
3787 }
c7151a8d 3788 }
c7151a8d
WH		 3789}
3790
3791static void vm_domain_remove_all_dev_info(struct dmar_domain *domain)
3792{
3793 struct device_domain_info *info;
3794 struct intel_iommu *iommu;
3795 unsigned long flags1, flags2;
3796
3797 spin_lock_irqsave(&device_domain_lock, flags1);
3798 while (!list_empty(&domain->devices)) {
3799 info = list_entry(domain->devices.next,
3800 struct device_domain_info, link);
109b9b04 3801 unlink_domain_info(info);
c7151a8d
WH		 3802 spin_unlock_irqrestore(&device_domain_lock, flags1);
3803
93a23a72 3804 iommu_disable_dev_iotlb(info);
276dbf99 3805 iommu = device_to_iommu(info->segment, info->bus, info->devfn);
c7151a8d 3806 iommu_detach_dev(iommu, info->bus, info->devfn);
3199aa6b 3807 iommu_detach_dependent_devices(iommu, info->dev);
c7151a8d
WH		 3808
3809 /* clear this iommu in iommu_bmp, update iommu count
58c610bd 3810 * and capabilities
c7151a8d
WH		 3811 */
3812 spin_lock_irqsave(&domain->iommu_lock, flags2);
3813 if (test_and_clear_bit(iommu->seq_id,
1b198bb0 3814 domain->iommu_bmp)) {
c7151a8d 3815 domain->iommu_count--;
58c610bd 3816 domain_update_iommu_cap(domain);
c7151a8d
WH		 3817 }
3818 spin_unlock_irqrestore(&domain->iommu_lock, flags2);
3819
3820 free_devinfo_mem(info);
3821 spin_lock_irqsave(&device_domain_lock, flags1);
3822 }
3823 spin_unlock_irqrestore(&device_domain_lock, flags1);
3824}
3825
5e98c4b1 3826/* domain id for virtual machine, it won't be set in context */
18d99165 3827static atomic_t vm_domid = ATOMIC_INIT(0);
5e98c4b1
WH		 3828
3829static struct dmar_domain *iommu_alloc_vm_domain(void)
3830{
3831 struct dmar_domain *domain;
3832
3833 domain = alloc_domain_mem();
3834 if (!domain)
3835 return NULL;
3836
18d99165 3837 domain->id = atomic_inc_return(&vm_domid);
4c923d47 3838 domain->nid = -1;
1b198bb0 3839 memset(domain->iommu_bmp, 0, sizeof(domain->iommu_bmp));
5e98c4b1
WH		 3840 domain->flags = DOMAIN_FLAG_VIRTUAL_MACHINE;
3841
3842 return domain;
3843}
3844
2c2e2c38 3845static int md_domain_init(struct dmar_domain *domain, int guest_width)
5e98c4b1
WH		 3846{
3847 int adjust_width;
3848
3849 init_iova_domain(&domain->iovad, DMA_32BIT_PFN);
5e98c4b1
WH		 3850 spin_lock_init(&domain->iommu_lock);
3851
3852 domain_reserve_special_ranges(domain);
3853
3854 /* calculate AGAW */
3855 domain->gaw = guest_width;
3856 adjust_width = guestwidth_to_adjustwidth(guest_width);
3857 domain->agaw = width_to_agaw(adjust_width);
3858
3859 INIT_LIST_HEAD(&domain->devices);
3860
3861 domain->iommu_count = 0;
3862 domain->iommu_coherency = 0;
c5b15255 3863 domain->iommu_snooping = 0;
6dd9a7c7 3864 domain->iommu_superpage = 0;
fe40f1e0 3865 domain->max_addr = 0;
4c923d47 3866 domain->nid = -1;
5e98c4b1
WH		 3867
3868 /* always allocate the top pgd */
4c923d47 3869 domain->pgd = (struct dma_pte *)alloc_pgtable_page(domain->nid);
5e98c4b1
WH		 3870 if (!domain->pgd)
3871 return -ENOMEM;
3872 domain_flush_cache(domain, domain->pgd, PAGE_SIZE);
3873 return 0;
3874}
3875
3876static void iommu_free_vm_domain(struct dmar_domain *domain)
3877{
3878 unsigned long flags;
3879 struct dmar_drhd_unit *drhd;
3880 struct intel_iommu *iommu;
3881 unsigned long i;
3882 unsigned long ndomains;
3883
7c919779 3884 for_each_active_iommu(iommu, drhd) {
5e98c4b1 3885 ndomains = cap_ndoms(iommu->cap);
a45946ab 3886 for_each_set_bit(i, iommu->domain_ids, ndomains) {
5e98c4b1
WH		 3887 if (iommu->domains[i] == domain) {
3888 spin_lock_irqsave(&iommu->lock, flags);
3889 clear_bit(i, iommu->domain_ids);
3890 iommu->domains[i] = NULL;
3891 spin_unlock_irqrestore(&iommu->lock, flags);
3892 break;
3893 }
5e98c4b1
WH		 3894 }
3895 }
3896}
3897
3898static void vm_domain_exit(struct dmar_domain *domain)
3899{
5e98c4b1
WH		 3900 /* Domain 0 is reserved, so dont process it */
3901 if (!domain)
3902 return;
3903
3904 vm_domain_remove_all_dev_info(domain);
3905 /* destroy iovas */
3906 put_iova_domain(&domain->iovad);
5e98c4b1
WH		 3907
3908 /* clear ptes */
595badf5 3909 dma_pte_clear_range(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
5e98c4b1
WH		 3910
3911 /* free page tables */
d794dc9b 3912 dma_pte_free_pagetable(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
5e98c4b1
WH		 3913
3914 iommu_free_vm_domain(domain);
3915 free_domain_mem(domain);
3916}
3917
5d450806 3918static int intel_iommu_domain_init(struct iommu_domain *domain)
38717946 3919{
5d450806 3920 struct dmar_domain *dmar_domain;
38717946 3921
5d450806
JR		 3922 dmar_domain = iommu_alloc_vm_domain();
3923 if (!dmar_domain) {
38717946 3924 printk(KERN_ERR
5d450806
JR		 3925 "intel_iommu_domain_init: dmar_domain == NULL\n");
3926 return -ENOMEM;
38717946 3927 }
2c2e2c38 3928 if (md_domain_init(dmar_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
38717946 3929 printk(KERN_ERR
5d450806
JR		 3930 "intel_iommu_domain_init() failed\n");
3931 vm_domain_exit(dmar_domain);
3932 return -ENOMEM;
38717946 3933 }
8140a95d 3934 domain_update_iommu_cap(dmar_domain);
5d450806 3935 domain->priv = dmar_domain;
faa3d6f5 3936
8a0e715b
JR		 3937 domain->geometry.aperture_start = 0;
3938 domain->geometry.aperture_end = __DOMAIN_MAX_ADDR(dmar_domain->gaw);
3939 domain->geometry.force_aperture = true;
3940
5d450806 3941 return 0;
38717946 3942}
38717946 3943
5d450806 3944static void intel_iommu_domain_destroy(struct iommu_domain *domain)
38717946 3945{
5d450806
JR		 3946 struct dmar_domain *dmar_domain = domain->priv;
3947
3948 domain->priv = NULL;
3949 vm_domain_exit(dmar_domain);
38717946 3950}
38717946 3951
4c5478c9
JR		 3952static int intel_iommu_attach_device(struct iommu_domain *domain,
3953 struct device *dev)
38717946 3954{
4c5478c9
JR		 3955 struct dmar_domain *dmar_domain = domain->priv;
3956 struct pci_dev *pdev = to_pci_dev(dev);
fe40f1e0
WH		 3957 struct intel_iommu *iommu;
3958 int addr_width;
faa3d6f5
WH		 3959
3960 /* normally pdev is not mapped */
3961 if (unlikely(domain_context_mapped(pdev))) {
3962 struct dmar_domain *old_domain;
3963
3964 old_domain = find_domain(pdev);
3965 if (old_domain) {
2c2e2c38
FY		 3966 if (dmar_domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE ||
3967 dmar_domain->flags & DOMAIN_FLAG_STATIC_IDENTITY)
3968 domain_remove_one_dev_info(old_domain, pdev);
faa3d6f5
WH		 3969 else
3970 domain_remove_dev_info(old_domain);
3971 }
3972 }
3973
276dbf99
DW		 3974 iommu = device_to_iommu(pci_domain_nr(pdev->bus), pdev->bus->number,
3975 pdev->devfn);
fe40f1e0
WH		 3976 if (!iommu)
3977 return -ENODEV;
3978
3979 /* check if this iommu agaw is sufficient for max mapped address */
3980 addr_width = agaw_to_width(iommu->agaw);
a99c47a2
TL		 3981 if (addr_width > cap_mgaw(iommu->cap))
3982 addr_width = cap_mgaw(iommu->cap);
3983
3984 if (dmar_domain->max_addr > (1LL << addr_width)) {
3985 printk(KERN_ERR "%s: iommu width (%d) is not "
fe40f1e0 3986 "sufficient for the mapped address (%llx)\n",
a99c47a2 3987 __func__, addr_width, dmar_domain->max_addr);
fe40f1e0
WH		 3988 return -EFAULT;
3989 }
a99c47a2
TL		 3990 dmar_domain->gaw = addr_width;
3991
3992 /*
3993 * Knock out extra levels of page tables if necessary
3994 */
3995 while (iommu->agaw < dmar_domain->agaw) {
3996 struct dma_pte *pte;
3997
3998 pte = dmar_domain->pgd;
3999 if (dma_pte_present(pte)) {
25cbff16
SY		 4000 dmar_domain->pgd = (struct dma_pte *)
4001 phys_to_virt(dma_pte_addr(pte));
7a661013 4002 free_pgtable_page(pte);
a99c47a2
TL		 4003 }
4004 dmar_domain->agaw--;
4005 }
fe40f1e0 4006
5fe60f4e 4007 return domain_add_dev_info(dmar_domain, pdev, CONTEXT_TT_MULTI_LEVEL);
38717946 4008}
38717946 4009
4c5478c9
JR		 4010static void intel_iommu_detach_device(struct iommu_domain *domain,
4011 struct device *dev)
38717946 4012{
4c5478c9
JR		 4013 struct dmar_domain *dmar_domain = domain->priv;
4014 struct pci_dev *pdev = to_pci_dev(dev);
4015
2c2e2c38 4016 domain_remove_one_dev_info(dmar_domain, pdev);
faa3d6f5 4017}
c7151a8d 4018
b146a1c9
JR		 4019static int intel_iommu_map(struct iommu_domain *domain,
4020 unsigned long iova, phys_addr_t hpa,
5009065d 4021 size_t size, int iommu_prot)
faa3d6f5 4022{
dde57a21 4023 struct dmar_domain *dmar_domain = domain->priv;
fe40f1e0 4024 u64 max_addr;
dde57a21 4025 int prot = 0;
faa3d6f5 4026 int ret;
fe40f1e0 4027
dde57a21
JR		 4028 if (iommu_prot & IOMMU_READ)
4029 prot |= DMA_PTE_READ;
4030 if (iommu_prot & IOMMU_WRITE)
4031 prot |= DMA_PTE_WRITE;
9cf06697
SY		 4032 if ((iommu_prot & IOMMU_CACHE) && dmar_domain->iommu_snooping)
4033 prot |= DMA_PTE_SNP;
dde57a21 4034
163cc52c 4035 max_addr = iova + size;
dde57a21 4036 if (dmar_domain->max_addr < max_addr) {
fe40f1e0
WH		 4037 u64 end;
4038
4039 /* check if minimum agaw is sufficient for mapped address */
8954da1f 4040 end = __DOMAIN_MAX_ADDR(dmar_domain->gaw) + 1;
fe40f1e0 4041 if (end < max_addr) {
8954da1f 4042 printk(KERN_ERR "%s: iommu width (%d) is not "
fe40f1e0 4043 "sufficient for the mapped address (%llx)\n",
8954da1f 4044 __func__, dmar_domain->gaw, max_addr);
fe40f1e0
WH		 4045 return -EFAULT;
4046 }
dde57a21 4047 dmar_domain->max_addr = max_addr;
fe40f1e0 4048 }
ad051221
DW		 4049 /* Round up size to next multiple of PAGE_SIZE, if it and
4050 the low bits of hpa would take us onto the next page */
88cb6a74 4051 size = aligned_nrpages(hpa, size);
ad051221
DW		 4052 ret = domain_pfn_mapping(dmar_domain, iova >> VTD_PAGE_SHIFT,
4053 hpa >> VTD_PAGE_SHIFT, size, prot);
faa3d6f5 4054 return ret;
38717946 4055}
38717946 4056
5009065d
OBC		 4057static size_t intel_iommu_unmap(struct iommu_domain *domain,
4058 unsigned long iova, size_t size)
38717946 4059{
dde57a21 4060 struct dmar_domain *dmar_domain = domain->priv;
292827cb 4061 int order;
4b99d352 4062
292827cb 4063 order = dma_pte_clear_range(dmar_domain, iova >> VTD_PAGE_SHIFT,
163cc52c 4064 (iova + size - 1) >> VTD_PAGE_SHIFT);
fe40f1e0 4065
163cc52c
DW		 4066 if (dmar_domain->max_addr == iova + size)
4067 dmar_domain->max_addr = iova;
b146a1c9 4068
5009065d 4069 return PAGE_SIZE << order;
38717946 4070}
38717946 4071
d14d6577 4072static phys_addr_t intel_iommu_iova_to_phys(struct iommu_domain *domain,
bb5547ac 4073 dma_addr_t iova)
38717946 4074{
d14d6577 4075 struct dmar_domain *dmar_domain = domain->priv;
38717946 4076 struct dma_pte *pte;
faa3d6f5 4077 u64 phys = 0;
38717946 4078
6dd9a7c7 4079 pte = pfn_to_dma_pte(dmar_domain, iova >> VTD_PAGE_SHIFT, 0);
38717946 4080 if (pte)
faa3d6f5 4081 phys = dma_pte_addr(pte);
38717946 4082
faa3d6f5 4083 return phys;
38717946 4084}
a8bcbb0d 4085
dbb9fd86
SY		 4086static int intel_iommu_domain_has_cap(struct iommu_domain *domain,
4087 unsigned long cap)
4088{
4089 struct dmar_domain *dmar_domain = domain->priv;
4090
4091 if (cap == IOMMU_CAP_CACHE_COHERENCY)
4092 return dmar_domain->iommu_snooping;
323f99cb 4093 if (cap == IOMMU_CAP_INTR_REMAP)
95a02e97 4094 return irq_remapping_enabled;
dbb9fd86
SY		 4095
4096 return 0;
4097}
4098
783f157b 4099#define REQ_ACS_FLAGS (PCI_ACS_SV | PCI_ACS_RR | PCI_ACS_CR | PCI_ACS_UF)
70ae6f0d 4100
abdfdde2
AW		 4101static int intel_iommu_add_device(struct device *dev)
4102{
4103 struct pci_dev *pdev = to_pci_dev(dev);
3da4af0a 4104 struct pci_dev *bridge, *dma_pdev = NULL;
abdfdde2
AW		 4105 struct iommu_group *group;
4106 int ret;
70ae6f0d 4107
abdfdde2
AW		 4108 if (!device_to_iommu(pci_domain_nr(pdev->bus),
4109 pdev->bus->number, pdev->devfn))
70ae6f0d
AW		 4110 return -ENODEV;
4111
4112 bridge = pci_find_upstream_pcie_bridge(pdev);
4113 if (bridge) {
abdfdde2
AW		 4114 if (pci_is_pcie(bridge))
4115 dma_pdev = pci_get_domain_bus_and_slot(
4116 pci_domain_nr(pdev->bus),
4117 bridge->subordinate->number, 0);
3da4af0a 4118 if (!dma_pdev)
abdfdde2
AW		 4119 dma_pdev = pci_dev_get(bridge);
4120 } else
4121 dma_pdev = pci_dev_get(pdev);
4122
a4ff1fc2 4123 /* Account for quirked devices */
783f157b
AW		 4124 swap_pci_ref(&dma_pdev, pci_get_dma_source(dma_pdev));
4125
a4ff1fc2
AW		 4126 /*
4127 * If it's a multifunction device that does not support our
c14d2690
AW		 4128 * required ACS flags, add to the same group as lowest numbered
4129 * function that also does not suport the required ACS flags.
a4ff1fc2 4130 */
783f157b 4131 if (dma_pdev->multifunction &&
c14d2690
AW		 4132 !pci_acs_enabled(dma_pdev, REQ_ACS_FLAGS)) {
4133 u8 i, slot = PCI_SLOT(dma_pdev->devfn);
4134
4135 for (i = 0; i < 8; i++) {
4136 struct pci_dev *tmp;
4137
4138 tmp = pci_get_slot(dma_pdev->bus, PCI_DEVFN(slot, i));
4139 if (!tmp)
4140 continue;
4141
4142 if (!pci_acs_enabled(tmp, REQ_ACS_FLAGS)) {
4143 swap_pci_ref(&dma_pdev, tmp);
4144 break;
4145 }
4146 pci_dev_put(tmp);
4147 }
4148 }
783f157b 4149
a4ff1fc2
AW		 4150 /*
4151 * Devices on the root bus go through the iommu. If that's not us,
4152 * find the next upstream device and test ACS up to the root bus.
4153 * Finding the next device may require skipping virtual buses.
4154 */
783f157b 4155 while (!pci_is_root_bus(dma_pdev->bus)) {
a4ff1fc2
AW		 4156 struct pci_bus *bus = dma_pdev->bus;
4157
4158 while (!bus->self) {
4159 if (!pci_is_root_bus(bus))
4160 bus = bus->parent;
4161 else
4162 goto root_bus;
4163 }
4164
4165 if (pci_acs_path_enabled(bus->self, NULL, REQ_ACS_FLAGS))
783f157b
AW		 4166 break;
4167
a4ff1fc2 4168 swap_pci_ref(&dma_pdev, pci_dev_get(bus->self));
783f157b
AW		 4169 }
4170
a4ff1fc2 4171root_bus:
abdfdde2
AW		 4172 group = iommu_group_get(&dma_pdev->dev);
4173 pci_dev_put(dma_pdev);
4174 if (!group) {
4175 group = iommu_group_alloc();
4176 if (IS_ERR(group))
4177 return PTR_ERR(group);
70ae6f0d
AW		 4178 }
4179
abdfdde2 4180 ret = iommu_group_add_device(group, dev);
bcb71abe 4181
abdfdde2
AW		 4182 iommu_group_put(group);
4183 return ret;
4184}
70ae6f0d 4185
abdfdde2
AW		 4186static void intel_iommu_remove_device(struct device *dev)
4187{
4188 iommu_group_remove_device(dev);
70ae6f0d
AW		 4189}
4190
a8bcbb0d
JR		 4191static struct iommu_ops intel_iommu_ops = {
4192 .domain_init = intel_iommu_domain_init,
4193 .domain_destroy = intel_iommu_domain_destroy,
4194 .attach_dev = intel_iommu_attach_device,
4195 .detach_dev = intel_iommu_detach_device,
b146a1c9
JR		 4196 .map = intel_iommu_map,
4197 .unmap = intel_iommu_unmap,
a8bcbb0d 4198 .iova_to_phys = intel_iommu_iova_to_phys,
dbb9fd86 4199 .domain_has_cap = intel_iommu_domain_has_cap,
abdfdde2
AW		 4200 .add_device = intel_iommu_add_device,
4201 .remove_device = intel_iommu_remove_device,
6d1c56a9 4202 .pgsize_bitmap = INTEL_IOMMU_PGSIZES,
a8bcbb0d 4203};
9af88143 4204
9452618e
DV		 4205static void quirk_iommu_g4x_gfx(struct pci_dev *dev)
4206{
4207 /* G4x/GM45 integrated gfx dmar support is totally busted. */
4208 printk(KERN_INFO "DMAR: Disabling IOMMU for graphics on this chipset\n");
4209 dmar_map_gfx = 0;
4210}
4211
4212DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2a40, quirk_iommu_g4x_gfx);
4213DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e00, quirk_iommu_g4x_gfx);
4214DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e10, quirk_iommu_g4x_gfx);
4215DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e20, quirk_iommu_g4x_gfx);
4216DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e30, quirk_iommu_g4x_gfx);
4217DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e40, quirk_iommu_g4x_gfx);
4218DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e90, quirk_iommu_g4x_gfx);
4219
d34d6517 4220static void quirk_iommu_rwbf(struct pci_dev *dev)
9af88143
DW		 4221{
4222 /*
4223 * Mobile 4 Series Chipset neglects to set RWBF capability,
210561ff 4224 * but needs it. Same seems to hold for the desktop versions.
9af88143
DW		 4225 */
4226 printk(KERN_INFO "DMAR: Forcing write-buffer flush capability\n");
4227 rwbf_quirk = 1;
4228}
4229
4230DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2a40, quirk_iommu_rwbf);
210561ff
DV		 4231DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e00, quirk_iommu_rwbf);
4232DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e10, quirk_iommu_rwbf);
4233DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e20, quirk_iommu_rwbf);
4234DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e30, quirk_iommu_rwbf);
4235DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e40, quirk_iommu_rwbf);
4236DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e90, quirk_iommu_rwbf);
e0fc7e0b 4237
eecfd57f
AJ		 4238#define GGC 0x52
4239#define GGC_MEMORY_SIZE_MASK (0xf << 8)
4240#define GGC_MEMORY_SIZE_NONE (0x0 << 8)
4241#define GGC_MEMORY_SIZE_1M (0x1 << 8)
4242#define GGC_MEMORY_SIZE_2M (0x3 << 8)
4243#define GGC_MEMORY_VT_ENABLED (0x8 << 8)
4244#define GGC_MEMORY_SIZE_2M_VT (0x9 << 8)
4245#define GGC_MEMORY_SIZE_3M_VT (0xa << 8)
4246#define GGC_MEMORY_SIZE_4M_VT (0xb << 8)
4247
d34d6517 4248static void quirk_calpella_no_shadow_gtt(struct pci_dev *dev)
9eecabcb
DW		 4249{
4250 unsigned short ggc;
4251
eecfd57f 4252 if (pci_read_config_word(dev, GGC, &ggc))
9eecabcb
DW		 4253 return;
4254
eecfd57f 4255 if (!(ggc & GGC_MEMORY_VT_ENABLED)) {
9eecabcb
DW		 4256 printk(KERN_INFO "DMAR: BIOS has allocated no shadow GTT; disabling IOMMU for graphics\n");
4257 dmar_map_gfx = 0;
6fbcfb3e
DW		 4258 } else if (dmar_map_gfx) {
4259 /* we have to ensure the gfx device is idle before we flush */
4260 printk(KERN_INFO "DMAR: Disabling batched IOTLB flush on Ironlake\n");
4261 intel_iommu_strict = 1;
4262 }
9eecabcb
DW		 4263}
4264DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0040, quirk_calpella_no_shadow_gtt);
4265DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0044, quirk_calpella_no_shadow_gtt);
4266DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0062, quirk_calpella_no_shadow_gtt);
4267DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x006a, quirk_calpella_no_shadow_gtt);
4268
e0fc7e0b
DW		 4269/* On Tylersburg chipsets, some BIOSes have been known to enable the
4270 ISOCH DMAR unit for the Azalia sound device, but not give it any
4271 TLB entries, which causes it to deadlock. Check for that. We do
4272 this in a function called from init_dmars(), instead of in a PCI
4273 quirk, because we don't want to print the obnoxious "BIOS broken"
4274 message if VT-d is actually disabled.
4275*/
4276static void __init check_tylersburg_isoch(void)
4277{
4278 struct pci_dev *pdev;
4279 uint32_t vtisochctrl;
4280
4281 /* If there's no Azalia in the system anyway, forget it. */
4282 pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x3a3e, NULL);
4283 if (!pdev)
4284 return;
4285 pci_dev_put(pdev);
4286
4287 /* System Management Registers. Might be hidden, in which case
4288 we can't do the sanity check. But that's OK, because the
4289 known-broken BIOSes _don't_ actually hide it, so far. */
4290 pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x342e, NULL);
4291 if (!pdev)
4292 return;
4293
4294 if (pci_read_config_dword(pdev, 0x188, &vtisochctrl)) {
4295 pci_dev_put(pdev);
4296 return;
4297 }
4298
4299 pci_dev_put(pdev);
4300
4301 /* If Azalia DMA is routed to the non-isoch DMAR unit, fine. */
4302 if (vtisochctrl & 1)
4303 return;
4304
4305 /* Drop all bits other than the number of TLB entries */
4306 vtisochctrl &= 0x1c;
4307
4308 /* If we have the recommended number of TLB entries (16), fine. */
4309 if (vtisochctrl == 0x10)
4310 return;
4311
4312 /* Zero TLB entries? You get to ride the short bus to school. */
4313 if (!vtisochctrl) {
4314 WARN(1, "Your BIOS is broken; DMA routed to ISOCH DMAR unit but no TLB space.\n"
4315 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
4316 dmi_get_system_info(DMI_BIOS_VENDOR),
4317 dmi_get_system_info(DMI_BIOS_VERSION),
4318 dmi_get_system_info(DMI_PRODUCT_VERSION));
4319 iommu_identity_mapping |= IDENTMAP_AZALIA;
4320 return;
4321 }
4322
4323 printk(KERN_WARNING "DMAR: Recommended TLB entries for ISOCH unit is 16; your BIOS set %d\n",
4324 vtisochctrl);
4325}
Linux kernel - Deliverables
This page took 0.924291 seconds and 5 git commands to generate.