projects / deliverable / linux.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
NVMe: per-cpu io queues
[deliverable/linux.git] / drivers / iommu / intel-iommu.c
CommitLineData
ba395927
KA		 1/*
2 * Copyright (c) 2006, Intel Corporation.
3 *
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms and conditions of the GNU General Public License,
6 * version 2, as published by the Free Software Foundation.
7 *
8 * This program is distributed in the hope it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
11 * more details.
12 *
13 * You should have received a copy of the GNU General Public License along with
14 * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
15 * Place - Suite 330, Boston, MA 02111-1307 USA.
16 *
98bcef56 17 * Copyright (C) 2006-2008 Intel Corporation
18 * Author: Ashok Raj <ashok.raj@intel.com>
19 * Author: Shaohua Li <shaohua.li@intel.com>
20 * Author: Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>
5b6985ce 21 * Author: Fenghua Yu <fenghua.yu@intel.com>
ba395927
KA		 22 */
23
24#include <linux/init.h>
25#include <linux/bitmap.h>
5e0d2a6f 26#include <linux/debugfs.h>
54485c30 27#include <linux/export.h>
ba395927
KA		 28#include <linux/slab.h>
29#include <linux/irq.h>
30#include <linux/interrupt.h>
ba395927
KA		 31#include <linux/spinlock.h>
32#include <linux/pci.h>
33#include <linux/dmar.h>
34#include <linux/dma-mapping.h>
35#include <linux/mempool.h>
5e0d2a6f 36#include <linux/timer.h>
38717946 37#include <linux/iova.h>
5d450806 38#include <linux/iommu.h>
38717946 39#include <linux/intel-iommu.h>
134fac3f 40#include <linux/syscore_ops.h>
69575d38 41#include <linux/tboot.h>
adb2fe02 42#include <linux/dmi.h>
5cdede24 43#include <linux/pci-ats.h>
0ee332c1 44#include <linux/memblock.h>
8a8f422d 45#include <asm/irq_remapping.h>
ba395927 46#include <asm/cacheflush.h>
46a7fa27 47#include <asm/iommu.h>
ba395927 48
078e1ee2 49#include "irq_remapping.h"
61e015ac 50#include "pci.h"
078e1ee2 51
5b6985ce
FY		 52#define ROOT_SIZE VTD_PAGE_SIZE
53#define CONTEXT_SIZE VTD_PAGE_SIZE
54
ba395927
KA		 55#define IS_GFX_DEVICE(pdev) ((pdev->class >> 16) == PCI_BASE_CLASS_DISPLAY)
56#define IS_ISA_DEVICE(pdev) ((pdev->class >> 8) == PCI_CLASS_BRIDGE_ISA)
e0fc7e0b 57#define IS_AZALIA(pdev) ((pdev)->vendor == 0x8086 && (pdev)->device == 0x3a3e)
ba395927
KA		 58
59#define IOAPIC_RANGE_START (0xfee00000)
60#define IOAPIC_RANGE_END (0xfeefffff)
61#define IOVA_START_ADDR (0x1000)
62
63#define DEFAULT_DOMAIN_ADDRESS_WIDTH 48
64
4ed0d3e6 65#define MAX_AGAW_WIDTH 64
5c645b35 66#define MAX_AGAW_PFN_WIDTH (MAX_AGAW_WIDTH - VTD_PAGE_SHIFT)
4ed0d3e6 67
2ebe3151
DW		 68#define __DOMAIN_MAX_PFN(gaw) ((((uint64_t)1) << (gaw-VTD_PAGE_SHIFT)) - 1)
69#define __DOMAIN_MAX_ADDR(gaw) ((((uint64_t)1) << gaw) - 1)
70
71/* We limit DOMAIN_MAX_PFN to fit in an unsigned long, and DOMAIN_MAX_ADDR
72 to match. That way, we can use 'unsigned long' for PFNs with impunity. */
73#define DOMAIN_MAX_PFN(gaw) ((unsigned long) min_t(uint64_t, \
74 __DOMAIN_MAX_PFN(gaw), (unsigned long)-1))
75#define DOMAIN_MAX_ADDR(gaw) (((uint64_t)__DOMAIN_MAX_PFN(gaw)) << VTD_PAGE_SHIFT)
ba395927 76
f27be03b 77#define IOVA_PFN(addr) ((addr) >> PAGE_SHIFT)
284901a9 78#define DMA_32BIT_PFN IOVA_PFN(DMA_BIT_MASK(32))
6a35528a 79#define DMA_64BIT_PFN IOVA_PFN(DMA_BIT_MASK(64))
5e0d2a6f 80
df08cdc7
AM		 81/* page table handling */
82#define LEVEL_STRIDE (9)
83#define LEVEL_MASK (((u64)1 << LEVEL_STRIDE) - 1)
84
6d1c56a9
OBC		 85/*
86 * This bitmap is used to advertise the page sizes our hardware support
87 * to the IOMMU core, which will then use this information to split
88 * physically contiguous memory regions it is mapping into page sizes
89 * that we support.
90 *
91 * Traditionally the IOMMU core just handed us the mappings directly,
92 * after making sure the size is an order of a 4KiB page and that the
93 * mapping has natural alignment.
94 *
95 * To retain this behavior, we currently advertise that we support
96 * all page sizes that are an order of 4KiB.
97 *
98 * If at some point we'd like to utilize the IOMMU core's new behavior,
99 * we could change this to advertise the real page sizes we support.
100 */
101#define INTEL_IOMMU_PGSIZES (~0xFFFUL)
102
df08cdc7
AM		 103static inline int agaw_to_level(int agaw)
104{
105 return agaw + 2;
106}
107
108static inline int agaw_to_width(int agaw)
109{
5c645b35 110 return min_t(int, 30 + agaw * LEVEL_STRIDE, MAX_AGAW_WIDTH);
df08cdc7
AM		 111}
112
113static inline int width_to_agaw(int width)
114{
5c645b35 115 return DIV_ROUND_UP(width - 30, LEVEL_STRIDE);
df08cdc7
AM		 116}
117
118static inline unsigned int level_to_offset_bits(int level)
119{
120 return (level - 1) * LEVEL_STRIDE;
121}
122
123static inline int pfn_level_offset(unsigned long pfn, int level)
124{
125 return (pfn >> level_to_offset_bits(level)) & LEVEL_MASK;
126}
127
128static inline unsigned long level_mask(int level)
129{
130 return -1UL << level_to_offset_bits(level);
131}
132
133static inline unsigned long level_size(int level)
134{
135 return 1UL << level_to_offset_bits(level);
136}
137
138static inline unsigned long align_to_level(unsigned long pfn, int level)
139{
140 return (pfn + level_size(level) - 1) & level_mask(level);
141}
fd18de50 142
6dd9a7c7
YS		 143static inline unsigned long lvl_to_nr_pages(unsigned int lvl)
144{
5c645b35 145 return 1 << min_t(int, (lvl - 1) * LEVEL_STRIDE, MAX_AGAW_PFN_WIDTH);
6dd9a7c7
YS		 146}
147
dd4e8319
DW		 148/* VT-d pages must always be _smaller_ than MM pages. Otherwise things
149 are never going to work. */
150static inline unsigned long dma_to_mm_pfn(unsigned long dma_pfn)
151{
152 return dma_pfn >> (PAGE_SHIFT - VTD_PAGE_SHIFT);
153}
154
155static inline unsigned long mm_to_dma_pfn(unsigned long mm_pfn)
156{
157 return mm_pfn << (PAGE_SHIFT - VTD_PAGE_SHIFT);
158}
159static inline unsigned long page_to_dma_pfn(struct page *pg)
160{
161 return mm_to_dma_pfn(page_to_pfn(pg));
162}
163static inline unsigned long virt_to_dma_pfn(void *p)
164{
165 return page_to_dma_pfn(virt_to_page(p));
166}
167
d9630fe9
WH		 168/* global iommu list, set NULL for ignored DMAR units */
169static struct intel_iommu **g_iommus;
170
e0fc7e0b 171static void __init check_tylersburg_isoch(void);
9af88143
DW		 172static int rwbf_quirk;
173
b779260b
JC		 174/*
175 * set to 1 to panic kernel if can't successfully enable VT-d
176 * (used when kernel is launched w/ TXT)
177 */
178static int force_on = 0;
179
46b08e1a
MM		 180/*
181 * 0: Present
182 * 1-11: Reserved
183 * 12-63: Context Ptr (12 - (haw-1))
184 * 64-127: Reserved
185 */
186struct root_entry {
187 u64 val;
188 u64 rsvd1;
189};
190#define ROOT_ENTRY_NR (VTD_PAGE_SIZE/sizeof(struct root_entry))
191static inline bool root_present(struct root_entry *root)
192{
193 return (root->val & 1);
194}
195static inline void set_root_present(struct root_entry *root)
196{
197 root->val |= 1;
198}
199static inline void set_root_value(struct root_entry *root, unsigned long value)
200{
201 root->val |= value & VTD_PAGE_MASK;
202}
203
204static inline struct context_entry *
205get_context_addr_from_root(struct root_entry *root)
206{
207 return (struct context_entry *)
208 (root_present(root)?phys_to_virt(
209 root->val & VTD_PAGE_MASK) :
210 NULL);
211}
212
7a8fc25e
MM		 213/*
214 * low 64 bits:
215 * 0: present
216 * 1: fault processing disable
217 * 2-3: translation type
218 * 12-63: address space root
219 * high 64 bits:
220 * 0-2: address width
221 * 3-6: aval
222 * 8-23: domain id
223 */
224struct context_entry {
225 u64 lo;
226 u64 hi;
227};
c07e7d21
MM		 228
229static inline bool context_present(struct context_entry *context)
230{
231 return (context->lo & 1);
232}
233static inline void context_set_present(struct context_entry *context)
234{
235 context->lo |= 1;
236}
237
238static inline void context_set_fault_enable(struct context_entry *context)
239{
240 context->lo &= (((u64)-1) << 2) | 1;
241}
242
c07e7d21
MM		 243static inline void context_set_translation_type(struct context_entry *context,
244 unsigned long value)
245{
246 context->lo &= (((u64)-1) << 4) | 3;
247 context->lo |= (value & 3) << 2;
248}
249
250static inline void context_set_address_root(struct context_entry *context,
251 unsigned long value)
252{
253 context->lo |= value & VTD_PAGE_MASK;
254}
255
256static inline void context_set_address_width(struct context_entry *context,
257 unsigned long value)
258{
259 context->hi |= value & 7;
260}
261
262static inline void context_set_domain_id(struct context_entry *context,
263 unsigned long value)
264{
265 context->hi |= (value & ((1 << 16) - 1)) << 8;
266}
267
268static inline void context_clear_entry(struct context_entry *context)
269{
270 context->lo = 0;
271 context->hi = 0;
272}
7a8fc25e 273
622ba12a
MM		 274/*
275 * 0: readable
276 * 1: writable
277 * 2-6: reserved
278 * 7: super page
9cf06697
SY		 279 * 8-10: available
280 * 11: snoop behavior
622ba12a
MM		 281 * 12-63: Host physcial address
282 */
283struct dma_pte {
284 u64 val;
285};
622ba12a 286
19c239ce
MM		 287static inline void dma_clear_pte(struct dma_pte *pte)
288{
289 pte->val = 0;
290}
291
19c239ce
MM		 292static inline u64 dma_pte_addr(struct dma_pte *pte)
293{
c85994e4
DW		 294#ifdef CONFIG_64BIT
295 return pte->val & VTD_PAGE_MASK;
296#else
297 /* Must have a full atomic 64-bit read */
1a8bd481 298 return __cmpxchg64(&pte->val, 0ULL, 0ULL) & VTD_PAGE_MASK;
c85994e4 299#endif
19c239ce
MM		 300}
301
19c239ce
MM		 302static inline bool dma_pte_present(struct dma_pte *pte)
303{
304 return (pte->val & 3) != 0;
305}
622ba12a 306
4399c8bf
AK		 307static inline bool dma_pte_superpage(struct dma_pte *pte)
308{
309 return (pte->val & (1 << 7));
310}
311
75e6bf96
DW		 312static inline int first_pte_in_page(struct dma_pte *pte)
313{
314 return !((unsigned long)pte & ~VTD_PAGE_MASK);
315}
316
2c2e2c38
FY		 317/*
318 * This domain is a statically identity mapping domain.
319 * 1. This domain creats a static 1:1 mapping to all usable memory.
320 * 2. It maps to each iommu if successful.
321 * 3. Each iommu mapps to this domain if successful.
322 */
19943b0e
DW		 323static struct dmar_domain *si_domain;
324static int hw_pass_through = 1;
2c2e2c38 325
3b5410e7 326/* devices under the same p2p bridge are owned in one domain */
cdc7b837 327#define DOMAIN_FLAG_P2P_MULTIPLE_DEVICES (1 << 0)
3b5410e7 328
1ce28feb
WH		 329/* domain represents a virtual machine, more than one devices
330 * across iommus may be owned in one domain, e.g. kvm guest.
331 */
332#define DOMAIN_FLAG_VIRTUAL_MACHINE (1 << 1)
333
2c2e2c38
FY		 334/* si_domain contains mulitple devices */
335#define DOMAIN_FLAG_STATIC_IDENTITY (1 << 2)
336
1b198bb0
MT		 337/* define the limit of IOMMUs supported in each domain */
338#ifdef CONFIG_X86
339# define IOMMU_UNITS_SUPPORTED MAX_IO_APICS
340#else
341# define IOMMU_UNITS_SUPPORTED 64
342#endif
343
99126f7c
MM		 344struct dmar_domain {
345 int id; /* domain id */
4c923d47 346 int nid; /* node id */
1b198bb0
MT		 347 DECLARE_BITMAP(iommu_bmp, IOMMU_UNITS_SUPPORTED);
348 /* bitmap of iommus this domain uses*/
99126f7c
MM		 349
350 struct list_head devices; /* all devices' list */
351 struct iova_domain iovad; /* iova's that belong to this domain */
352
353 struct dma_pte *pgd; /* virtual address */
99126f7c
MM		 354 int gaw; /* max guest address width */
355
356 /* adjusted guest address width, 0 is level 2 30-bit */
357 int agaw;
358
3b5410e7 359 int flags; /* flags to find out type of domain */
8e604097
WH		 360
361 int iommu_coherency;/* indicate coherency of iommu access */
58c610bd 362 int iommu_snooping; /* indicate snooping control feature*/
c7151a8d 363 int iommu_count; /* reference count of iommu */
6dd9a7c7
YS		 364 int iommu_superpage;/* Level of superpages supported:
365 0 == 4KiB (no superpages), 1 == 2MiB,
366 2 == 1GiB, 3 == 512GiB, 4 == 1TiB */
c7151a8d 367 spinlock_t iommu_lock; /* protect iommu set in domain */
fe40f1e0 368 u64 max_addr; /* maximum mapped address */
99126f7c
MM		 369};
370
a647dacb
MM		 371/* PCI domain-device relationship */
372struct device_domain_info {
373 struct list_head link; /* link to domain siblings */
374 struct list_head global; /* link to global list */
276dbf99
DW		 375 int segment; /* PCI domain */
376 u8 bus; /* PCI bus number */
a647dacb 377 u8 devfn; /* PCI devfn number */
45e829ea 378 struct pci_dev *dev; /* it's NULL for PCIe-to-PCI bridge */
93a23a72 379 struct intel_iommu *iommu; /* IOMMU used by this device */
a647dacb
MM		 380 struct dmar_domain *domain; /* pointer to domain */
381};
382
5e0d2a6f 383static void flush_unmaps_timeout(unsigned long data);
384
b707cb02 385static DEFINE_TIMER(unmap_timer, flush_unmaps_timeout, 0, 0);
5e0d2a6f 386
80b20dd8 387#define HIGH_WATER_MARK 250
388struct deferred_flush_tables {
389 int next;
390 struct iova *iova[HIGH_WATER_MARK];
391 struct dmar_domain *domain[HIGH_WATER_MARK];
392};
393
394static struct deferred_flush_tables *deferred_flush;
395
5e0d2a6f 396/* bitmap for indexing intel_iommus */
5e0d2a6f 397static int g_num_of_iommus;
398
399static DEFINE_SPINLOCK(async_umap_flush_lock);
400static LIST_HEAD(unmaps_to_do);
401
402static int timer_on;
403static long list_size;
5e0d2a6f 404
ba395927
KA		 405static void domain_remove_dev_info(struct dmar_domain *domain);
406
d3f13810 407#ifdef CONFIG_INTEL_IOMMU_DEFAULT_ON
0cd5c3c8
KM		 408int dmar_disabled = 0;
409#else
410int dmar_disabled = 1;
d3f13810 411#endif /*CONFIG_INTEL_IOMMU_DEFAULT_ON*/
0cd5c3c8 412
8bc1f85c
ED		 413int intel_iommu_enabled = 0;
414EXPORT_SYMBOL_GPL(intel_iommu_enabled);
415
2d9e667e 416static int dmar_map_gfx = 1;
7d3b03ce 417static int dmar_forcedac;
5e0d2a6f 418static int intel_iommu_strict;
6dd9a7c7 419static int intel_iommu_superpage = 1;
ba395927 420
c0771df8
DW		 421int intel_iommu_gfx_mapped;
422EXPORT_SYMBOL_GPL(intel_iommu_gfx_mapped);
423
ba395927
KA		 424#define DUMMY_DEVICE_DOMAIN_INFO ((struct device_domain_info *)(-1))
425static DEFINE_SPINLOCK(device_domain_lock);
426static LIST_HEAD(device_domain_list);
427
a8bcbb0d
JR		 428static struct iommu_ops intel_iommu_ops;
429
ba395927
KA		 430static int __init intel_iommu_setup(char *str)
431{
432 if (!str)
433 return -EINVAL;
434 while (*str) {
0cd5c3c8
KM		 435 if (!strncmp(str, "on", 2)) {
436 dmar_disabled = 0;
437 printk(KERN_INFO "Intel-IOMMU: enabled\n");
438 } else if (!strncmp(str, "off", 3)) {
ba395927 439 dmar_disabled = 1;
0cd5c3c8 440 printk(KERN_INFO "Intel-IOMMU: disabled\n");
ba395927
KA		 441 } else if (!strncmp(str, "igfx_off", 8)) {
442 dmar_map_gfx = 0;
443 printk(KERN_INFO
444 "Intel-IOMMU: disable GFX device mapping\n");
7d3b03ce 445 } else if (!strncmp(str, "forcedac", 8)) {
5e0d2a6f 446 printk(KERN_INFO
7d3b03ce
KA		 447 "Intel-IOMMU: Forcing DAC for PCI devices\n");
448 dmar_forcedac = 1;
5e0d2a6f 449 } else if (!strncmp(str, "strict", 6)) {
450 printk(KERN_INFO
451 "Intel-IOMMU: disable batched IOTLB flush\n");
452 intel_iommu_strict = 1;
6dd9a7c7
YS		 453 } else if (!strncmp(str, "sp_off", 6)) {
454 printk(KERN_INFO
455 "Intel-IOMMU: disable supported super page\n");
456 intel_iommu_superpage = 0;
ba395927
KA		 457 }
458
459 str += strcspn(str, ",");
460 while (*str == ',')
461 str++;
462 }
463 return 0;
464}
465__setup("intel_iommu=", intel_iommu_setup);
466
467static struct kmem_cache *iommu_domain_cache;
468static struct kmem_cache *iommu_devinfo_cache;
469static struct kmem_cache *iommu_iova_cache;
470
4c923d47 471static inline void *alloc_pgtable_page(int node)
eb3fa7cb 472{
4c923d47
SS		 473 struct page *page;
474 void *vaddr = NULL;
eb3fa7cb 475
4c923d47
SS		 476 page = alloc_pages_node(node, GFP_ATOMIC | __GFP_ZERO, 0);
477 if (page)
478 vaddr = page_address(page);
eb3fa7cb 479 return vaddr;
ba395927
KA		 480}
481
482static inline void free_pgtable_page(void *vaddr)
483{
484 free_page((unsigned long)vaddr);
485}
486
487static inline void *alloc_domain_mem(void)
488{
354bb65e 489 return kmem_cache_alloc(iommu_domain_cache, GFP_ATOMIC);
ba395927
KA		 490}
491
38717946 492static void free_domain_mem(void *vaddr)
ba395927
KA		 493{
494 kmem_cache_free(iommu_domain_cache, vaddr);
495}
496
497static inline void * alloc_devinfo_mem(void)
498{
354bb65e 499 return kmem_cache_alloc(iommu_devinfo_cache, GFP_ATOMIC);
ba395927
KA		 500}
501
502static inline void free_devinfo_mem(void *vaddr)
503{
504 kmem_cache_free(iommu_devinfo_cache, vaddr);
505}
506
507struct iova *alloc_iova_mem(void)
508{
354bb65e 509 return kmem_cache_alloc(iommu_iova_cache, GFP_ATOMIC);
ba395927
KA		 510}
511
512void free_iova_mem(struct iova *iova)
513{
514 kmem_cache_free(iommu_iova_cache, iova);
515}
516
1b573683 517
4ed0d3e6 518static int __iommu_calculate_agaw(struct intel_iommu *iommu, int max_gaw)
1b573683
WH		 519{
520 unsigned long sagaw;
521 int agaw = -1;
522
523 sagaw = cap_sagaw(iommu->cap);
4ed0d3e6 524 for (agaw = width_to_agaw(max_gaw);
1b573683
WH		 525 agaw >= 0; agaw--) {
526 if (test_bit(agaw, &sagaw))
527 break;
528 }
529
530 return agaw;
531}
532
4ed0d3e6
FY		 533/*
534 * Calculate max SAGAW for each iommu.
535 */
536int iommu_calculate_max_sagaw(struct intel_iommu *iommu)
537{
538 return __iommu_calculate_agaw(iommu, MAX_AGAW_WIDTH);
539}
540
541/*
542 * calculate agaw for each iommu.
543 * "SAGAW" may be different across iommus, use a default agaw, and
544 * get a supported less agaw for iommus that don't support the default agaw.
545 */
546int iommu_calculate_agaw(struct intel_iommu *iommu)
547{
548 return __iommu_calculate_agaw(iommu, DEFAULT_DOMAIN_ADDRESS_WIDTH);
549}
550
2c2e2c38 551/* This functionin only returns single iommu in a domain */
8c11e798
WH		 552static struct intel_iommu *domain_get_iommu(struct dmar_domain *domain)
553{
554 int iommu_id;
555
2c2e2c38 556 /* si_domain and vm domain should not get here. */
1ce28feb 557 BUG_ON(domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE);
2c2e2c38 558 BUG_ON(domain->flags & DOMAIN_FLAG_STATIC_IDENTITY);
1ce28feb 559
1b198bb0 560 iommu_id = find_first_bit(domain->iommu_bmp, g_num_of_iommus);
8c11e798
WH		 561 if (iommu_id < 0 || iommu_id >= g_num_of_iommus)
562 return NULL;
563
564 return g_iommus[iommu_id];
565}
566
8e604097
WH		 567static void domain_update_iommu_coherency(struct dmar_domain *domain)
568{
569 int i;
570
2e12bc29
AW		 571 i = find_first_bit(domain->iommu_bmp, g_num_of_iommus);
572
573 domain->iommu_coherency = i < g_num_of_iommus ? 1 : 0;
8e604097 574
1b198bb0 575 for_each_set_bit(i, domain->iommu_bmp, g_num_of_iommus) {
8e604097
WH		 576 if (!ecap_coherent(g_iommus[i]->ecap)) {
577 domain->iommu_coherency = 0;
578 break;
579 }
8e604097
WH		 580 }
581}
582
58c610bd
SY		 583static void domain_update_iommu_snooping(struct dmar_domain *domain)
584{
585 int i;
586
587 domain->iommu_snooping = 1;
588
1b198bb0 589 for_each_set_bit(i, domain->iommu_bmp, g_num_of_iommus) {
58c610bd
SY		 590 if (!ecap_sc_support(g_iommus[i]->ecap)) {
591 domain->iommu_snooping = 0;
592 break;
593 }
58c610bd
SY		 594 }
595}
596
6dd9a7c7
YS		 597static void domain_update_iommu_superpage(struct dmar_domain *domain)
598{
8140a95d
AK		 599 struct dmar_drhd_unit *drhd;
600 struct intel_iommu *iommu = NULL;
601 int mask = 0xf;
6dd9a7c7
YS		 602
603 if (!intel_iommu_superpage) {
604 domain->iommu_superpage = 0;
605 return;
606 }
607
8140a95d
AK		 608 /* set iommu_superpage to the smallest common denominator */
609 for_each_active_iommu(iommu, drhd) {
610 mask &= cap_super_page_val(iommu->cap);
6dd9a7c7
YS		 611 if (!mask) {
612 break;
613 }
614 }
615 domain->iommu_superpage = fls(mask);
616}
617
58c610bd
SY		 618/* Some capabilities may be different across iommus */
619static void domain_update_iommu_cap(struct dmar_domain *domain)
620{
621 domain_update_iommu_coherency(domain);
622 domain_update_iommu_snooping(domain);
6dd9a7c7 623 domain_update_iommu_superpage(domain);
58c610bd
SY		 624}
625
276dbf99 626static struct intel_iommu *device_to_iommu(int segment, u8 bus, u8 devfn)
c7151a8d
WH		 627{
628 struct dmar_drhd_unit *drhd = NULL;
629 int i;
630
7c919779 631 for_each_active_drhd_unit(drhd) {
276dbf99
DW		 632 if (segment != drhd->segment)
633 continue;
c7151a8d 634
924b6231 635 for (i = 0; i < drhd->devices_cnt; i++) {
288e4877
DH		 636 if (drhd->devices[i] &&
637 drhd->devices[i]->bus->number == bus &&
c7151a8d
WH		 638 drhd->devices[i]->devfn == devfn)
639 return drhd->iommu;
4958c5dc
DW		 640 if (drhd->devices[i] &&
641 drhd->devices[i]->subordinate &&
924b6231 642 drhd->devices[i]->subordinate->number <= bus &&
b918c62e 643 drhd->devices[i]->subordinate->busn_res.end >= bus)
924b6231
DW		 644 return drhd->iommu;
645 }
c7151a8d
WH		 646
647 if (drhd->include_all)
648 return drhd->iommu;
649 }
650
651 return NULL;
652}
653
5331fe6f
WH		 654static void domain_flush_cache(struct dmar_domain *domain,
655 void *addr, int size)
656{
657 if (!domain->iommu_coherency)
658 clflush_cache_range(addr, size);
659}
660
ba395927
KA		 661/* Gets context entry for a given bus and devfn */
662static struct context_entry * device_to_context_entry(struct intel_iommu *iommu,
663 u8 bus, u8 devfn)
664{
665 struct root_entry *root;
666 struct context_entry *context;
667 unsigned long phy_addr;
668 unsigned long flags;
669
670 spin_lock_irqsave(&iommu->lock, flags);
671 root = &iommu->root_entry[bus];
672 context = get_context_addr_from_root(root);
673 if (!context) {
4c923d47
SS		 674 context = (struct context_entry *)
675 alloc_pgtable_page(iommu->node);
ba395927
KA		 676 if (!context) {
677 spin_unlock_irqrestore(&iommu->lock, flags);
678 return NULL;
679 }
5b6985ce 680 __iommu_flush_cache(iommu, (void *)context, CONTEXT_SIZE);
ba395927
KA		 681 phy_addr = virt_to_phys((void *)context);
682 set_root_value(root, phy_addr);
683 set_root_present(root);
684 __iommu_flush_cache(iommu, root, sizeof(*root));
685 }
686 spin_unlock_irqrestore(&iommu->lock, flags);
687 return &context[devfn];
688}
689
690static int device_context_mapped(struct intel_iommu *iommu, u8 bus, u8 devfn)
691{
692 struct root_entry *root;
693 struct context_entry *context;
694 int ret;
695 unsigned long flags;
696
697 spin_lock_irqsave(&iommu->lock, flags);
698 root = &iommu->root_entry[bus];
699 context = get_context_addr_from_root(root);
700 if (!context) {
701 ret = 0;
702 goto out;
703 }
c07e7d21 704 ret = context_present(&context[devfn]);
ba395927
KA		 705out:
706 spin_unlock_irqrestore(&iommu->lock, flags);
707 return ret;
708}
709
710static void clear_context_table(struct intel_iommu *iommu, u8 bus, u8 devfn)
711{
712 struct root_entry *root;
713 struct context_entry *context;
714 unsigned long flags;
715
716 spin_lock_irqsave(&iommu->lock, flags);
717 root = &iommu->root_entry[bus];
718 context = get_context_addr_from_root(root);
719 if (context) {
c07e7d21 720 context_clear_entry(&context[devfn]);
ba395927
KA		 721 __iommu_flush_cache(iommu, &context[devfn], \
722 sizeof(*context));
723 }
724 spin_unlock_irqrestore(&iommu->lock, flags);
725}
726
727static void free_context_table(struct intel_iommu *iommu)
728{
729 struct root_entry *root;
730 int i;
731 unsigned long flags;
732 struct context_entry *context;
733
734 spin_lock_irqsave(&iommu->lock, flags);
735 if (!iommu->root_entry) {
736 goto out;
737 }
738 for (i = 0; i < ROOT_ENTRY_NR; i++) {
739 root = &iommu->root_entry[i];
740 context = get_context_addr_from_root(root);
741 if (context)
742 free_pgtable_page(context);
743 }
744 free_pgtable_page(iommu->root_entry);
745 iommu->root_entry = NULL;
746out:
747 spin_unlock_irqrestore(&iommu->lock, flags);
748}
749
b026fd28 750static struct dma_pte *pfn_to_dma_pte(struct dmar_domain *domain,
4399c8bf 751 unsigned long pfn, int target_level)
ba395927 752{
b026fd28 753 int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
ba395927
KA		 754 struct dma_pte *parent, *pte = NULL;
755 int level = agaw_to_level(domain->agaw);
4399c8bf 756 int offset;
ba395927
KA		 757
758 BUG_ON(!domain->pgd);
f9423606
JS		 759
760 if (addr_width < BITS_PER_LONG && pfn >> addr_width)
761 /* Address beyond IOMMU's addressing capabilities. */
762 return NULL;
763
ba395927
KA		 764 parent = domain->pgd;
765
ba395927
KA		 766 while (level > 0) {
767 void *tmp_page;
768
b026fd28 769 offset = pfn_level_offset(pfn, level);
ba395927 770 pte = &parent[offset];
4399c8bf 771 if (!target_level && (dma_pte_superpage(pte) || !dma_pte_present(pte)))
6dd9a7c7
YS		 772 break;
773 if (level == target_level)
ba395927
KA		 774 break;
775
19c239ce 776 if (!dma_pte_present(pte)) {
c85994e4
DW		 777 uint64_t pteval;
778
4c923d47 779 tmp_page = alloc_pgtable_page(domain->nid);
ba395927 780
206a73c1 781 if (!tmp_page)
ba395927 782 return NULL;
206a73c1 783
c85994e4 784 domain_flush_cache(domain, tmp_page, VTD_PAGE_SIZE);
64de5af0 785 pteval = ((uint64_t)virt_to_dma_pfn(tmp_page) << VTD_PAGE_SHIFT) | DMA_PTE_READ | DMA_PTE_WRITE;
c85994e4
DW		 786 if (cmpxchg64(&pte->val, 0ULL, pteval)) {
787 /* Someone else set it while we were thinking; use theirs. */
788 free_pgtable_page(tmp_page);
789 } else {
790 dma_pte_addr(pte);
791 domain_flush_cache(domain, pte, sizeof(*pte));
792 }
ba395927 793 }
19c239ce 794 parent = phys_to_virt(dma_pte_addr(pte));
ba395927
KA		 795 level--;
796 }
797
ba395927
KA		 798 return pte;
799}
800
6dd9a7c7 801
ba395927 802/* return address's pte at specific level */
90dcfb5e
DW		 803static struct dma_pte *dma_pfn_level_pte(struct dmar_domain *domain,
804 unsigned long pfn,
6dd9a7c7 805 int level, int *large_page)
ba395927
KA		 806{
807 struct dma_pte *parent, *pte = NULL;
808 int total = agaw_to_level(domain->agaw);
809 int offset;
810
811 parent = domain->pgd;
812 while (level <= total) {
90dcfb5e 813 offset = pfn_level_offset(pfn, total);
ba395927
KA		 814 pte = &parent[offset];
815 if (level == total)
816 return pte;
817
6dd9a7c7
YS		 818 if (!dma_pte_present(pte)) {
819 *large_page = total;
ba395927 820 break;
6dd9a7c7
YS		 821 }
822
823 if (pte->val & DMA_PTE_LARGE_PAGE) {
824 *large_page = total;
825 return pte;
826 }
827
19c239ce 828 parent = phys_to_virt(dma_pte_addr(pte));
ba395927
KA		 829 total--;
830 }
831 return NULL;
832}
833
ba395927 834/* clear last level pte, a tlb flush should be followed */
292827cb 835static int dma_pte_clear_range(struct dmar_domain *domain,
595badf5
DW		 836 unsigned long start_pfn,
837 unsigned long last_pfn)
ba395927 838{
04b18e65 839 int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
6dd9a7c7 840 unsigned int large_page = 1;
310a5ab9 841 struct dma_pte *first_pte, *pte;
66eae846 842
04b18e65 843 BUG_ON(addr_width < BITS_PER_LONG && start_pfn >> addr_width);
595badf5 844 BUG_ON(addr_width < BITS_PER_LONG && last_pfn >> addr_width);
59c36286 845 BUG_ON(start_pfn > last_pfn);
ba395927 846
04b18e65 847 /* we don't need lock here; nobody else touches the iova range */
59c36286 848 do {
6dd9a7c7
YS		 849 large_page = 1;
850 first_pte = pte = dma_pfn_level_pte(domain, start_pfn, 1, &large_page);
310a5ab9 851 if (!pte) {
6dd9a7c7 852 start_pfn = align_to_level(start_pfn + 1, large_page + 1);
310a5ab9
DW		 853 continue;
854 }
6dd9a7c7 855 do {
310a5ab9 856 dma_clear_pte(pte);
6dd9a7c7 857 start_pfn += lvl_to_nr_pages(large_page);
310a5ab9 858 pte++;
75e6bf96
DW		 859 } while (start_pfn <= last_pfn && !first_pte_in_page(pte));
860
310a5ab9
DW		 861 domain_flush_cache(domain, first_pte,
862 (void *)pte - (void *)first_pte);
59c36286
DW		 863
864 } while (start_pfn && start_pfn <= last_pfn);
292827cb 865
5c645b35 866 return min_t(int, (large_page - 1) * 9, MAX_AGAW_PFN_WIDTH);
ba395927
KA		 867}
868
3269ee0b
AW		 869static void dma_pte_free_level(struct dmar_domain *domain, int level,
870 struct dma_pte *pte, unsigned long pfn,
871 unsigned long start_pfn, unsigned long last_pfn)
872{
873 pfn = max(start_pfn, pfn);
874 pte = &pte[pfn_level_offset(pfn, level)];
875
876 do {
877 unsigned long level_pfn;
878 struct dma_pte *level_pte;
879
880 if (!dma_pte_present(pte) || dma_pte_superpage(pte))
881 goto next;
882
883 level_pfn = pfn & level_mask(level - 1);
884 level_pte = phys_to_virt(dma_pte_addr(pte));
885
886 if (level > 2)
887 dma_pte_free_level(domain, level - 1, level_pte,
888 level_pfn, start_pfn, last_pfn);
889
890 /* If range covers entire pagetable, free it */
891 if (!(start_pfn > level_pfn ||
08336fd2 892 last_pfn < level_pfn + level_size(level) - 1)) {
3269ee0b
AW		 893 dma_clear_pte(pte);
894 domain_flush_cache(domain, pte, sizeof(*pte));
895 free_pgtable_page(level_pte);
896 }
897next:
898 pfn += level_size(level);
899 } while (!first_pte_in_page(++pte) && pfn <= last_pfn);
900}
901
ba395927
KA		 902/* free page table pages. last level pte should already be cleared */
903static void dma_pte_free_pagetable(struct dmar_domain *domain,
d794dc9b
DW		 904 unsigned long start_pfn,
905 unsigned long last_pfn)
ba395927 906{
6660c63a 907 int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
ba395927 908
6660c63a
DW		 909 BUG_ON(addr_width < BITS_PER_LONG && start_pfn >> addr_width);
910 BUG_ON(addr_width < BITS_PER_LONG && last_pfn >> addr_width);
59c36286 911 BUG_ON(start_pfn > last_pfn);
ba395927 912
f3a0a52f 913 /* We don't need lock here; nobody else touches the iova range */
3269ee0b
AW		 914 dma_pte_free_level(domain, agaw_to_level(domain->agaw),
915 domain->pgd, 0, start_pfn, last_pfn);
6660c63a 916
ba395927 917 /* free pgd */
d794dc9b 918 if (start_pfn == 0 && last_pfn == DOMAIN_MAX_PFN(domain->gaw)) {
ba395927
KA		 919 free_pgtable_page(domain->pgd);
920 domain->pgd = NULL;
921 }
922}
923
924/* iommu handling */
925static int iommu_alloc_root_entry(struct intel_iommu *iommu)
926{
927 struct root_entry *root;
928 unsigned long flags;
929
4c923d47 930 root = (struct root_entry *)alloc_pgtable_page(iommu->node);
ba395927
KA		 931 if (!root)
932 return -ENOMEM;
933
5b6985ce 934 __iommu_flush_cache(iommu, root, ROOT_SIZE);
ba395927
KA		 935
936 spin_lock_irqsave(&iommu->lock, flags);
937 iommu->root_entry = root;
938 spin_unlock_irqrestore(&iommu->lock, flags);
939
940 return 0;
941}
942
ba395927
KA		 943static void iommu_set_root_entry(struct intel_iommu *iommu)
944{
945 void *addr;
c416daa9 946 u32 sts;
ba395927
KA		 947 unsigned long flag;
948
949 addr = iommu->root_entry;
950
1f5b3c3f 951 raw_spin_lock_irqsave(&iommu->register_lock, flag);
ba395927
KA		 952 dmar_writeq(iommu->reg + DMAR_RTADDR_REG, virt_to_phys(addr));
953
c416daa9 954 writel(iommu->gcmd | DMA_GCMD_SRTP, iommu->reg + DMAR_GCMD_REG);
ba395927
KA		 955
956 /* Make sure hardware complete it */
957 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
c416daa9 958 readl, (sts & DMA_GSTS_RTPS), sts);
ba395927 959
1f5b3c3f 960 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
ba395927
KA		 961}
962
963static void iommu_flush_write_buffer(struct intel_iommu *iommu)
964{
965 u32 val;
966 unsigned long flag;
967
9af88143 968 if (!rwbf_quirk && !cap_rwbf(iommu->cap))
ba395927 969 return;
ba395927 970
1f5b3c3f 971 raw_spin_lock_irqsave(&iommu->register_lock, flag);
462b60f6 972 writel(iommu->gcmd | DMA_GCMD_WBF, iommu->reg + DMAR_GCMD_REG);
ba395927
KA		 973
974 /* Make sure hardware complete it */
975 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
c416daa9 976 readl, (!(val & DMA_GSTS_WBFS)), val);
ba395927 977
1f5b3c3f 978 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
ba395927
KA		 979}
980
981/* return value determine if we need a write buffer flush */
4c25a2c1
DW		 982static void __iommu_flush_context(struct intel_iommu *iommu,
983 u16 did, u16 source_id, u8 function_mask,
984 u64 type)
ba395927
KA		 985{
986 u64 val = 0;
987 unsigned long flag;
988
ba395927
KA		 989 switch (type) {
990 case DMA_CCMD_GLOBAL_INVL:
991 val = DMA_CCMD_GLOBAL_INVL;
992 break;
993 case DMA_CCMD_DOMAIN_INVL:
994 val = DMA_CCMD_DOMAIN_INVL|DMA_CCMD_DID(did);
995 break;
996 case DMA_CCMD_DEVICE_INVL:
997 val = DMA_CCMD_DEVICE_INVL|DMA_CCMD_DID(did)
998 | DMA_CCMD_SID(source_id) | DMA_CCMD_FM(function_mask);
999 break;
1000 default:
1001 BUG();
1002 }
1003 val |= DMA_CCMD_ICC;
1004
1f5b3c3f 1005 raw_spin_lock_irqsave(&iommu->register_lock, flag);
ba395927
KA		 1006 dmar_writeq(iommu->reg + DMAR_CCMD_REG, val);
1007
1008 /* Make sure hardware complete it */
1009 IOMMU_WAIT_OP(iommu, DMAR_CCMD_REG,
1010 dmar_readq, (!(val & DMA_CCMD_ICC)), val);
1011
1f5b3c3f 1012 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
ba395927
KA		 1013}
1014
ba395927 1015/* return value determine if we need a write buffer flush */
1f0ef2aa
DW		 1016static void __iommu_flush_iotlb(struct intel_iommu *iommu, u16 did,
1017 u64 addr, unsigned int size_order, u64 type)
ba395927
KA		 1018{
1019 int tlb_offset = ecap_iotlb_offset(iommu->ecap);
1020 u64 val = 0, val_iva = 0;
1021 unsigned long flag;
1022
ba395927
KA		 1023 switch (type) {
1024 case DMA_TLB_GLOBAL_FLUSH:
1025 /* global flush doesn't need set IVA_REG */
1026 val = DMA_TLB_GLOBAL_FLUSH|DMA_TLB_IVT;
1027 break;
1028 case DMA_TLB_DSI_FLUSH:
1029 val = DMA_TLB_DSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
1030 break;
1031 case DMA_TLB_PSI_FLUSH:
1032 val = DMA_TLB_PSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
1033 /* Note: always flush non-leaf currently */
1034 val_iva = size_order | addr;
1035 break;
1036 default:
1037 BUG();
1038 }
1039 /* Note: set drain read/write */
1040#if 0
1041 /*
1042 * This is probably to be super secure.. Looks like we can
1043 * ignore it without any impact.
1044 */
1045 if (cap_read_drain(iommu->cap))
1046 val |= DMA_TLB_READ_DRAIN;
1047#endif
1048 if (cap_write_drain(iommu->cap))
1049 val |= DMA_TLB_WRITE_DRAIN;
1050
1f5b3c3f 1051 raw_spin_lock_irqsave(&iommu->register_lock, flag);
ba395927
KA		 1052 /* Note: Only uses first TLB reg currently */
1053 if (val_iva)
1054 dmar_writeq(iommu->reg + tlb_offset, val_iva);
1055 dmar_writeq(iommu->reg + tlb_offset + 8, val);
1056
1057 /* Make sure hardware complete it */
1058 IOMMU_WAIT_OP(iommu, tlb_offset + 8,
1059 dmar_readq, (!(val & DMA_TLB_IVT)), val);
1060
1f5b3c3f 1061 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
ba395927
KA		 1062
1063 /* check IOTLB invalidation granularity */
1064 if (DMA_TLB_IAIG(val) == 0)
1065 printk(KERN_ERR"IOMMU: flush IOTLB failed\n");
1066 if (DMA_TLB_IAIG(val) != DMA_TLB_IIRG(type))
1067 pr_debug("IOMMU: tlb flush request %Lx, actual %Lx\n",
5b6985ce
FY		 1068 (unsigned long long)DMA_TLB_IIRG(type),
1069 (unsigned long long)DMA_TLB_IAIG(val));
ba395927
KA		 1070}
1071
93a23a72
YZ		 1072static struct device_domain_info *iommu_support_dev_iotlb(
1073 struct dmar_domain *domain, int segment, u8 bus, u8 devfn)
1074{
1075 int found = 0;
1076 unsigned long flags;
1077 struct device_domain_info *info;
1078 struct intel_iommu *iommu = device_to_iommu(segment, bus, devfn);
1079
1080 if (!ecap_dev_iotlb_support(iommu->ecap))
1081 return NULL;
1082
1083 if (!iommu->qi)
1084 return NULL;
1085
1086 spin_lock_irqsave(&device_domain_lock, flags);
1087 list_for_each_entry(info, &domain->devices, link)
1088 if (info->bus == bus && info->devfn == devfn) {
1089 found = 1;
1090 break;
1091 }
1092 spin_unlock_irqrestore(&device_domain_lock, flags);
1093
1094 if (!found || !info->dev)
1095 return NULL;
1096
1097 if (!pci_find_ext_capability(info->dev, PCI_EXT_CAP_ID_ATS))
1098 return NULL;
1099
1100 if (!dmar_find_matched_atsr_unit(info->dev))
1101 return NULL;
1102
1103 info->iommu = iommu;
1104
1105 return info;
1106}
1107
1108static void iommu_enable_dev_iotlb(struct device_domain_info *info)
ba395927 1109{
93a23a72
YZ		 1110 if (!info)
1111 return;
1112
1113 pci_enable_ats(info->dev, VTD_PAGE_SHIFT);
1114}
1115
1116static void iommu_disable_dev_iotlb(struct device_domain_info *info)
1117{
1118 if (!info->dev || !pci_ats_enabled(info->dev))
1119 return;
1120
1121 pci_disable_ats(info->dev);
1122}
1123
1124static void iommu_flush_dev_iotlb(struct dmar_domain *domain,
1125 u64 addr, unsigned mask)
1126{
1127 u16 sid, qdep;
1128 unsigned long flags;
1129 struct device_domain_info *info;
1130
1131 spin_lock_irqsave(&device_domain_lock, flags);
1132 list_for_each_entry(info, &domain->devices, link) {
1133 if (!info->dev || !pci_ats_enabled(info->dev))
1134 continue;
1135
1136 sid = info->bus << 8 | info->devfn;
1137 qdep = pci_ats_queue_depth(info->dev);
1138 qi_flush_dev_iotlb(info->iommu, sid, qdep, addr, mask);
1139 }
1140 spin_unlock_irqrestore(&device_domain_lock, flags);
1141}
1142
1f0ef2aa 1143static void iommu_flush_iotlb_psi(struct intel_iommu *iommu, u16 did,
82653633 1144 unsigned long pfn, unsigned int pages, int map)
ba395927 1145{
9dd2fe89 1146 unsigned int mask = ilog2(__roundup_pow_of_two(pages));
03d6a246 1147 uint64_t addr = (uint64_t)pfn << VTD_PAGE_SHIFT;
ba395927 1148
ba395927
KA		 1149 BUG_ON(pages == 0);
1150
ba395927 1151 /*
9dd2fe89
YZ		 1152 * Fallback to domain selective flush if no PSI support or the size is
1153 * too big.
ba395927
KA		 1154 * PSI requires page size to be 2 ^ x, and the base address is naturally
1155 * aligned to the size
1156 */
9dd2fe89
YZ		 1157 if (!cap_pgsel_inv(iommu->cap) || mask > cap_max_amask_val(iommu->cap))
1158 iommu->flush.flush_iotlb(iommu, did, 0, 0,
1f0ef2aa 1159 DMA_TLB_DSI_FLUSH);
9dd2fe89
YZ		 1160 else
1161 iommu->flush.flush_iotlb(iommu, did, addr, mask,
1162 DMA_TLB_PSI_FLUSH);
bf92df30
YZ		 1163
1164 /*
82653633
NA		 1165 * In caching mode, changes of pages from non-present to present require
1166 * flush. However, device IOTLB doesn't need to be flushed in this case.
bf92df30 1167 */
82653633 1168 if (!cap_caching_mode(iommu->cap) || !map)
93a23a72 1169 iommu_flush_dev_iotlb(iommu->domains[did], addr, mask);
ba395927
KA		 1170}
1171
f8bab735 1172static void iommu_disable_protect_mem_regions(struct intel_iommu *iommu)
1173{
1174 u32 pmen;
1175 unsigned long flags;
1176
1f5b3c3f 1177 raw_spin_lock_irqsave(&iommu->register_lock, flags);
f8bab735 1178 pmen = readl(iommu->reg + DMAR_PMEN_REG);
1179 pmen &= ~DMA_PMEN_EPM;
1180 writel(pmen, iommu->reg + DMAR_PMEN_REG);
1181
1182 /* wait for the protected region status bit to clear */
1183 IOMMU_WAIT_OP(iommu, DMAR_PMEN_REG,
1184 readl, !(pmen & DMA_PMEN_PRS), pmen);
1185
1f5b3c3f 1186 raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
f8bab735 1187}
1188
ba395927
KA		 1189static int iommu_enable_translation(struct intel_iommu *iommu)
1190{
1191 u32 sts;
1192 unsigned long flags;
1193
1f5b3c3f 1194 raw_spin_lock_irqsave(&iommu->register_lock, flags);
c416daa9
DW		 1195 iommu->gcmd |= DMA_GCMD_TE;
1196 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
ba395927
KA		 1197
1198 /* Make sure hardware complete it */
1199 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
c416daa9 1200 readl, (sts & DMA_GSTS_TES), sts);
ba395927 1201
1f5b3c3f 1202 raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
ba395927
KA		 1203 return 0;
1204}
1205
1206static int iommu_disable_translation(struct intel_iommu *iommu)
1207{
1208 u32 sts;
1209 unsigned long flag;
1210
1f5b3c3f 1211 raw_spin_lock_irqsave(&iommu->register_lock, flag);
ba395927
KA		 1212 iommu->gcmd &= ~DMA_GCMD_TE;
1213 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1214
1215 /* Make sure hardware complete it */
1216 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
c416daa9 1217 readl, (!(sts & DMA_GSTS_TES)), sts);
ba395927 1218
1f5b3c3f 1219 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
ba395927
KA		 1220 return 0;
1221}
1222
3460a6d9 1223
ba395927
KA		 1224static int iommu_init_domains(struct intel_iommu *iommu)
1225{
1226 unsigned long ndomains;
1227 unsigned long nlongs;
1228
1229 ndomains = cap_ndoms(iommu->cap);
852bdb04
JL		 1230 pr_debug("IOMMU%d: Number of Domains supported <%ld>\n",
1231 iommu->seq_id, ndomains);
ba395927
KA		 1232 nlongs = BITS_TO_LONGS(ndomains);
1233
94a91b50
DD		 1234 spin_lock_init(&iommu->lock);
1235
ba395927
KA		 1236 /* TBD: there might be 64K domains,
1237 * consider other allocation for future chip
1238 */
1239 iommu->domain_ids = kcalloc(nlongs, sizeof(unsigned long), GFP_KERNEL);
1240 if (!iommu->domain_ids) {
852bdb04
JL		 1241 pr_err("IOMMU%d: allocating domain id array failed\n",
1242 iommu->seq_id);
ba395927
KA		 1243 return -ENOMEM;
1244 }
1245 iommu->domains = kcalloc(ndomains, sizeof(struct dmar_domain *),
1246 GFP_KERNEL);
1247 if (!iommu->domains) {
852bdb04
JL		 1248 pr_err("IOMMU%d: allocating domain array failed\n",
1249 iommu->seq_id);
1250 kfree(iommu->domain_ids);
1251 iommu->domain_ids = NULL;
ba395927
KA		 1252 return -ENOMEM;
1253 }
1254
1255 /*
1256 * if Caching mode is set, then invalid translations are tagged
1257 * with domainid 0. Hence we need to pre-allocate it.
1258 */
1259 if (cap_caching_mode(iommu->cap))
1260 set_bit(0, iommu->domain_ids);
1261 return 0;
1262}
ba395927 1263
ba395927
KA		 1264
1265static void domain_exit(struct dmar_domain *domain);
5e98c4b1 1266static void vm_domain_exit(struct dmar_domain *domain);
e61d98d8 1267
a868e6b7 1268static void free_dmar_iommu(struct intel_iommu *iommu)
ba395927
KA		 1269{
1270 struct dmar_domain *domain;
5ced12af 1271 int i, count;
c7151a8d 1272 unsigned long flags;
ba395927 1273
94a91b50 1274 if ((iommu->domains) && (iommu->domain_ids)) {
a45946ab 1275 for_each_set_bit(i, iommu->domain_ids, cap_ndoms(iommu->cap)) {
94a91b50
DD		 1276 domain = iommu->domains[i];
1277 clear_bit(i, iommu->domain_ids);
1278
1279 spin_lock_irqsave(&domain->iommu_lock, flags);
5ced12af
JL		 1280 count = --domain->iommu_count;
1281 spin_unlock_irqrestore(&domain->iommu_lock, flags);
1282 if (count == 0) {
94a91b50
DD		 1283 if (domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE)
1284 vm_domain_exit(domain);
1285 else
1286 domain_exit(domain);
1287 }
5e98c4b1 1288 }
ba395927
KA		 1289 }
1290
1291 if (iommu->gcmd & DMA_GCMD_TE)
1292 iommu_disable_translation(iommu);
1293
ba395927
KA		 1294 kfree(iommu->domains);
1295 kfree(iommu->domain_ids);
a868e6b7
JL		 1296 iommu->domains = NULL;
1297 iommu->domain_ids = NULL;
ba395927 1298
d9630fe9
WH		 1299 g_iommus[iommu->seq_id] = NULL;
1300
1301 /* if all iommus are freed, free g_iommus */
1302 for (i = 0; i < g_num_of_iommus; i++) {
1303 if (g_iommus[i])
1304 break;
1305 }
1306
1307 if (i == g_num_of_iommus)
1308 kfree(g_iommus);
1309
ba395927
KA		 1310 /* free context mapping */
1311 free_context_table(iommu);
ba395927
KA		 1312}
1313
2c2e2c38 1314static struct dmar_domain *alloc_domain(void)
ba395927 1315{
ba395927 1316 struct dmar_domain *domain;
ba395927
KA		 1317
1318 domain = alloc_domain_mem();
1319 if (!domain)
1320 return NULL;
1321
4c923d47 1322 domain->nid = -1;
1b198bb0 1323 memset(domain->iommu_bmp, 0, sizeof(domain->iommu_bmp));
2c2e2c38
FY		 1324 domain->flags = 0;
1325
1326 return domain;
1327}
1328
1329static int iommu_attach_domain(struct dmar_domain *domain,
1330 struct intel_iommu *iommu)
1331{
1332 int num;
1333 unsigned long ndomains;
1334 unsigned long flags;
1335
ba395927
KA		 1336 ndomains = cap_ndoms(iommu->cap);
1337
1338 spin_lock_irqsave(&iommu->lock, flags);
2c2e2c38 1339
ba395927
KA		 1340 num = find_first_zero_bit(iommu->domain_ids, ndomains);
1341 if (num >= ndomains) {
1342 spin_unlock_irqrestore(&iommu->lock, flags);
ba395927 1343 printk(KERN_ERR "IOMMU: no free domain ids\n");
2c2e2c38 1344 return -ENOMEM;
ba395927
KA		 1345 }
1346
ba395927 1347 domain->id = num;
2c2e2c38 1348 set_bit(num, iommu->domain_ids);
1b198bb0 1349 set_bit(iommu->seq_id, domain->iommu_bmp);
ba395927
KA		 1350 iommu->domains[num] = domain;
1351 spin_unlock_irqrestore(&iommu->lock, flags);
1352
2c2e2c38 1353 return 0;
ba395927
KA		 1354}
1355
2c2e2c38
FY		 1356static void iommu_detach_domain(struct dmar_domain *domain,
1357 struct intel_iommu *iommu)
ba395927
KA		 1358{
1359 unsigned long flags;
2c2e2c38
FY		 1360 int num, ndomains;
1361 int found = 0;
ba395927 1362
8c11e798 1363 spin_lock_irqsave(&iommu->lock, flags);
2c2e2c38 1364 ndomains = cap_ndoms(iommu->cap);
a45946ab 1365 for_each_set_bit(num, iommu->domain_ids, ndomains) {
2c2e2c38
FY		 1366 if (iommu->domains[num] == domain) {
1367 found = 1;
1368 break;
1369 }
2c2e2c38
FY		 1370 }
1371
1372 if (found) {
1373 clear_bit(num, iommu->domain_ids);
1b198bb0 1374 clear_bit(iommu->seq_id, domain->iommu_bmp);
2c2e2c38
FY		 1375 iommu->domains[num] = NULL;
1376 }
8c11e798 1377 spin_unlock_irqrestore(&iommu->lock, flags);
ba395927
KA		 1378}
1379
1380static struct iova_domain reserved_iova_list;
8a443df4 1381static struct lock_class_key reserved_rbtree_key;
ba395927 1382
51a63e67 1383static int dmar_init_reserved_ranges(void)
ba395927
KA		 1384{
1385 struct pci_dev *pdev = NULL;
1386 struct iova *iova;
1387 int i;
ba395927 1388
f661197e 1389 init_iova_domain(&reserved_iova_list, DMA_32BIT_PFN);
ba395927 1390
8a443df4
MG		 1391 lockdep_set_class(&reserved_iova_list.iova_rbtree_lock,
1392 &reserved_rbtree_key);
1393
ba395927
KA		 1394 /* IOAPIC ranges shouldn't be accessed by DMA */
1395 iova = reserve_iova(&reserved_iova_list, IOVA_PFN(IOAPIC_RANGE_START),
1396 IOVA_PFN(IOAPIC_RANGE_END));
51a63e67 1397 if (!iova) {
ba395927 1398 printk(KERN_ERR "Reserve IOAPIC range failed\n");
51a63e67
JC		 1399 return -ENODEV;
1400 }
ba395927
KA		 1401
1402 /* Reserve all PCI MMIO to avoid peer-to-peer access */
1403 for_each_pci_dev(pdev) {
1404 struct resource *r;
1405
1406 for (i = 0; i < PCI_NUM_RESOURCES; i++) {
1407 r = &pdev->resource[i];
1408 if (!r->flags || !(r->flags & IORESOURCE_MEM))
1409 continue;
1a4a4551
DW		 1410 iova = reserve_iova(&reserved_iova_list,
1411 IOVA_PFN(r->start),
1412 IOVA_PFN(r->end));
51a63e67 1413 if (!iova) {
ba395927 1414 printk(KERN_ERR "Reserve iova failed\n");
51a63e67
JC		 1415 return -ENODEV;
1416 }
ba395927
KA		 1417 }
1418 }
51a63e67 1419 return 0;
ba395927
KA		 1420}
1421
1422static void domain_reserve_special_ranges(struct dmar_domain *domain)
1423{
1424 copy_reserved_iova(&reserved_iova_list, &domain->iovad);
1425}
1426
1427static inline int guestwidth_to_adjustwidth(int gaw)
1428{
1429 int agaw;
1430 int r = (gaw - 12) % 9;
1431
1432 if (r == 0)
1433 agaw = gaw;
1434 else
1435 agaw = gaw + 9 - r;
1436 if (agaw > 64)
1437 agaw = 64;
1438 return agaw;
1439}
1440
1441static int domain_init(struct dmar_domain *domain, int guest_width)
1442{
1443 struct intel_iommu *iommu;
1444 int adjust_width, agaw;
1445 unsigned long sagaw;
1446
f661197e 1447 init_iova_domain(&domain->iovad, DMA_32BIT_PFN);
c7151a8d 1448 spin_lock_init(&domain->iommu_lock);
ba395927
KA		 1449
1450 domain_reserve_special_ranges(domain);
1451
1452 /* calculate AGAW */
8c11e798 1453 iommu = domain_get_iommu(domain);
ba395927
KA		 1454 if (guest_width > cap_mgaw(iommu->cap))
1455 guest_width = cap_mgaw(iommu->cap);
1456 domain->gaw = guest_width;
1457 adjust_width = guestwidth_to_adjustwidth(guest_width);
1458 agaw = width_to_agaw(adjust_width);
1459 sagaw = cap_sagaw(iommu->cap);
1460 if (!test_bit(agaw, &sagaw)) {
1461 /* hardware doesn't support it, choose a bigger one */
1462 pr_debug("IOMMU: hardware doesn't support agaw %d\n", agaw);
1463 agaw = find_next_bit(&sagaw, 5, agaw);
1464 if (agaw >= 5)
1465 return -ENODEV;
1466 }
1467 domain->agaw = agaw;
1468 INIT_LIST_HEAD(&domain->devices);
1469
8e604097
WH		 1470 if (ecap_coherent(iommu->ecap))
1471 domain->iommu_coherency = 1;
1472 else
1473 domain->iommu_coherency = 0;
1474
58c610bd
SY		 1475 if (ecap_sc_support(iommu->ecap))
1476 domain->iommu_snooping = 1;
1477 else
1478 domain->iommu_snooping = 0;
1479
6dd9a7c7 1480 domain->iommu_superpage = fls(cap_super_page_val(iommu->cap));
c7151a8d 1481 domain->iommu_count = 1;
4c923d47 1482 domain->nid = iommu->node;
c7151a8d 1483
ba395927 1484 /* always allocate the top pgd */
4c923d47 1485 domain->pgd = (struct dma_pte *)alloc_pgtable_page(domain->nid);
ba395927
KA		 1486 if (!domain->pgd)
1487 return -ENOMEM;
5b6985ce 1488 __iommu_flush_cache(iommu, domain->pgd, PAGE_SIZE);
ba395927
KA		 1489 return 0;
1490}
1491
1492static void domain_exit(struct dmar_domain *domain)
1493{
2c2e2c38
FY		 1494 struct dmar_drhd_unit *drhd;
1495 struct intel_iommu *iommu;
ba395927
KA		 1496
1497 /* Domain 0 is reserved, so dont process it */
1498 if (!domain)
1499 return;
1500
7b668357
AW		 1501 /* Flush any lazy unmaps that may reference this domain */
1502 if (!intel_iommu_strict)
1503 flush_unmaps_timeout(0);
1504
ba395927
KA		 1505 domain_remove_dev_info(domain);
1506 /* destroy iovas */
1507 put_iova_domain(&domain->iovad);
ba395927
KA		 1508
1509 /* clear ptes */
595badf5 1510 dma_pte_clear_range(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
ba395927
KA		 1511
1512 /* free page tables */
d794dc9b 1513 dma_pte_free_pagetable(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
ba395927 1514
2c2e2c38 1515 for_each_active_iommu(iommu, drhd)
1b198bb0 1516 if (test_bit(iommu->seq_id, domain->iommu_bmp))
2c2e2c38
FY		 1517 iommu_detach_domain(domain, iommu);
1518
ba395927
KA		 1519 free_domain_mem(domain);
1520}
1521
4ed0d3e6
FY		 1522static int domain_context_mapping_one(struct dmar_domain *domain, int segment,
1523 u8 bus, u8 devfn, int translation)
ba395927
KA		 1524{
1525 struct context_entry *context;
ba395927 1526 unsigned long flags;
5331fe6f 1527 struct intel_iommu *iommu;
ea6606b0
WH		 1528 struct dma_pte *pgd;
1529 unsigned long num;
1530 unsigned long ndomains;
1531 int id;
1532 int agaw;
93a23a72 1533 struct device_domain_info *info = NULL;
ba395927
KA		 1534
1535 pr_debug("Set context mapping for %02x:%02x.%d\n",
1536 bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
4ed0d3e6 1537
ba395927 1538 BUG_ON(!domain->pgd);
4ed0d3e6
FY		 1539 BUG_ON(translation != CONTEXT_TT_PASS_THROUGH &&
1540 translation != CONTEXT_TT_MULTI_LEVEL);
5331fe6f 1541
276dbf99 1542 iommu = device_to_iommu(segment, bus, devfn);
5331fe6f
WH		 1543 if (!iommu)
1544 return -ENODEV;
1545
ba395927
KA		 1546 context = device_to_context_entry(iommu, bus, devfn);
1547 if (!context)
1548 return -ENOMEM;
1549 spin_lock_irqsave(&iommu->lock, flags);
c07e7d21 1550 if (context_present(context)) {
ba395927
KA		 1551 spin_unlock_irqrestore(&iommu->lock, flags);
1552 return 0;
1553 }
1554
ea6606b0
WH		 1555 id = domain->id;
1556 pgd = domain->pgd;
1557
2c2e2c38
FY		 1558 if (domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE ||
1559 domain->flags & DOMAIN_FLAG_STATIC_IDENTITY) {
ea6606b0
WH		 1560 int found = 0;
1561
1562 /* find an available domain id for this device in iommu */
1563 ndomains = cap_ndoms(iommu->cap);
a45946ab 1564 for_each_set_bit(num, iommu->domain_ids, ndomains) {
ea6606b0
WH		 1565 if (iommu->domains[num] == domain) {
1566 id = num;
1567 found = 1;
1568 break;
1569 }
ea6606b0
WH		 1570 }
1571
1572 if (found == 0) {
1573 num = find_first_zero_bit(iommu->domain_ids, ndomains);
1574 if (num >= ndomains) {
1575 spin_unlock_irqrestore(&iommu->lock, flags);
1576 printk(KERN_ERR "IOMMU: no free domain ids\n");
1577 return -EFAULT;
1578 }
1579
1580 set_bit(num, iommu->domain_ids);
1581 iommu->domains[num] = domain;
1582 id = num;
1583 }
1584
1585 /* Skip top levels of page tables for
1586 * iommu which has less agaw than default.
1672af11 1587 * Unnecessary for PT mode.
ea6606b0 1588 */
1672af11
CW		 1589 if (translation != CONTEXT_TT_PASS_THROUGH) {
1590 for (agaw = domain->agaw; agaw != iommu->agaw; agaw--) {
1591 pgd = phys_to_virt(dma_pte_addr(pgd));
1592 if (!dma_pte_present(pgd)) {
1593 spin_unlock_irqrestore(&iommu->lock, flags);
1594 return -ENOMEM;
1595 }
ea6606b0
WH		 1596 }
1597 }
1598 }
1599
1600 context_set_domain_id(context, id);
4ed0d3e6 1601
93a23a72
YZ		 1602 if (translation != CONTEXT_TT_PASS_THROUGH) {
1603 info = iommu_support_dev_iotlb(domain, segment, bus, devfn);
1604 translation = info ? CONTEXT_TT_DEV_IOTLB :
1605 CONTEXT_TT_MULTI_LEVEL;
1606 }
4ed0d3e6
FY		 1607 /*
1608 * In pass through mode, AW must be programmed to indicate the largest
1609 * AGAW value supported by hardware. And ASR is ignored by hardware.
1610 */
93a23a72 1611 if (unlikely(translation == CONTEXT_TT_PASS_THROUGH))
4ed0d3e6 1612 context_set_address_width(context, iommu->msagaw);
93a23a72
YZ		 1613 else {
1614 context_set_address_root(context, virt_to_phys(pgd));
1615 context_set_address_width(context, iommu->agaw);
1616 }
4ed0d3e6
FY		 1617
1618 context_set_translation_type(context, translation);
c07e7d21
MM		 1619 context_set_fault_enable(context);
1620 context_set_present(context);
5331fe6f 1621 domain_flush_cache(domain, context, sizeof(*context));
ba395927 1622
4c25a2c1
DW		 1623 /*
1624 * It's a non-present to present mapping. If hardware doesn't cache
1625 * non-present entry we only need to flush the write-buffer. If the
1626 * _does_ cache non-present entries, then it does so in the special
1627 * domain #0, which we have to flush:
1628 */
1629 if (cap_caching_mode(iommu->cap)) {
1630 iommu->flush.flush_context(iommu, 0,
1631 (((u16)bus) << 8) | devfn,
1632 DMA_CCMD_MASK_NOBIT,
1633 DMA_CCMD_DEVICE_INVL);
82653633 1634 iommu->flush.flush_iotlb(iommu, domain->id, 0, 0, DMA_TLB_DSI_FLUSH);
4c25a2c1 1635 } else {
ba395927 1636 iommu_flush_write_buffer(iommu);
4c25a2c1 1637 }
93a23a72 1638 iommu_enable_dev_iotlb(info);
ba395927 1639 spin_unlock_irqrestore(&iommu->lock, flags);
c7151a8d
WH		 1640
1641 spin_lock_irqsave(&domain->iommu_lock, flags);
1b198bb0 1642 if (!test_and_set_bit(iommu->seq_id, domain->iommu_bmp)) {
c7151a8d 1643 domain->iommu_count++;
4c923d47
SS		 1644 if (domain->iommu_count == 1)
1645 domain->nid = iommu->node;
58c610bd 1646 domain_update_iommu_cap(domain);
c7151a8d
WH		 1647 }
1648 spin_unlock_irqrestore(&domain->iommu_lock, flags);
ba395927
KA		 1649 return 0;
1650}
1651
1652static int
4ed0d3e6
FY		 1653domain_context_mapping(struct dmar_domain *domain, struct pci_dev *pdev,
1654 int translation)
ba395927
KA		 1655{
1656 int ret;
1657 struct pci_dev *tmp, *parent;
1658
276dbf99 1659 ret = domain_context_mapping_one(domain, pci_domain_nr(pdev->bus),
4ed0d3e6
FY		 1660 pdev->bus->number, pdev->devfn,
1661 translation);
ba395927
KA		 1662 if (ret)
1663 return ret;
1664
1665 /* dependent device mapping */
1666 tmp = pci_find_upstream_pcie_bridge(pdev);
1667 if (!tmp)
1668 return 0;
1669 /* Secondary interface's bus number and devfn 0 */
1670 parent = pdev->bus->self;
1671 while (parent != tmp) {
276dbf99
DW		 1672 ret = domain_context_mapping_one(domain,
1673 pci_domain_nr(parent->bus),
1674 parent->bus->number,
4ed0d3e6 1675 parent->devfn, translation);
ba395927
KA		 1676 if (ret)
1677 return ret;
1678 parent = parent->bus->self;
1679 }
45e829ea 1680 if (pci_is_pcie(tmp)) /* this is a PCIe-to-PCI bridge */
ba395927 1681 return domain_context_mapping_one(domain,
276dbf99 1682 pci_domain_nr(tmp->subordinate),
4ed0d3e6
FY		 1683 tmp->subordinate->number, 0,
1684 translation);
ba395927
KA		 1685 else /* this is a legacy PCI bridge */
1686 return domain_context_mapping_one(domain,
276dbf99
DW		 1687 pci_domain_nr(tmp->bus),
1688 tmp->bus->number,
4ed0d3e6
FY		 1689 tmp->devfn,
1690 translation);
ba395927
KA		 1691}
1692
5331fe6f 1693static int domain_context_mapped(struct pci_dev *pdev)
ba395927
KA		 1694{
1695 int ret;
1696 struct pci_dev *tmp, *parent;
5331fe6f
WH		 1697 struct intel_iommu *iommu;
1698
276dbf99
DW		 1699 iommu = device_to_iommu(pci_domain_nr(pdev->bus), pdev->bus->number,
1700 pdev->devfn);
5331fe6f
WH		 1701 if (!iommu)
1702 return -ENODEV;
ba395927 1703
276dbf99 1704 ret = device_context_mapped(iommu, pdev->bus->number, pdev->devfn);
ba395927
KA		 1705 if (!ret)
1706 return ret;
1707 /* dependent device mapping */
1708 tmp = pci_find_upstream_pcie_bridge(pdev);
1709 if (!tmp)
1710 return ret;
1711 /* Secondary interface's bus number and devfn 0 */
1712 parent = pdev->bus->self;
1713 while (parent != tmp) {
8c11e798 1714 ret = device_context_mapped(iommu, parent->bus->number,
276dbf99 1715 parent->devfn);
ba395927
KA		 1716 if (!ret)
1717 return ret;
1718 parent = parent->bus->self;
1719 }
5f4d91a1 1720 if (pci_is_pcie(tmp))
276dbf99
DW		 1721 return device_context_mapped(iommu, tmp->subordinate->number,
1722 0);
ba395927 1723 else
276dbf99
DW		 1724 return device_context_mapped(iommu, tmp->bus->number,
1725 tmp->devfn);
ba395927
KA		 1726}
1727
f532959b
FY		 1728/* Returns a number of VTD pages, but aligned to MM page size */
1729static inline unsigned long aligned_nrpages(unsigned long host_addr,
1730 size_t size)
1731{
1732 host_addr &= ~PAGE_MASK;
1733 return PAGE_ALIGN(host_addr + size) >> VTD_PAGE_SHIFT;
1734}
1735
6dd9a7c7
YS		 1736/* Return largest possible superpage level for a given mapping */
1737static inline int hardware_largepage_caps(struct dmar_domain *domain,
1738 unsigned long iov_pfn,
1739 unsigned long phy_pfn,
1740 unsigned long pages)
1741{
1742 int support, level = 1;
1743 unsigned long pfnmerge;
1744
1745 support = domain->iommu_superpage;
1746
1747 /* To use a large page, the virtual *and* physical addresses
1748 must be aligned to 2MiB/1GiB/etc. Lower bits set in either
1749 of them will mean we have to use smaller pages. So just
1750 merge them and check both at once. */
1751 pfnmerge = iov_pfn | phy_pfn;
1752
1753 while (support && !(pfnmerge & ~VTD_STRIDE_MASK)) {
1754 pages >>= VTD_STRIDE_SHIFT;
1755 if (!pages)
1756 break;
1757 pfnmerge >>= VTD_STRIDE_SHIFT;
1758 level++;
1759 support--;
1760 }
1761 return level;
1762}
1763
9051aa02
DW		 1764static int __domain_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
1765 struct scatterlist *sg, unsigned long phys_pfn,
1766 unsigned long nr_pages, int prot)
e1605495
DW		 1767{
1768 struct dma_pte *first_pte = NULL, *pte = NULL;
9051aa02 1769 phys_addr_t uninitialized_var(pteval);
e1605495 1770 int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
9051aa02 1771 unsigned long sg_res;
6dd9a7c7
YS		 1772 unsigned int largepage_lvl = 0;
1773 unsigned long lvl_pages = 0;
e1605495
DW		 1774
1775 BUG_ON(addr_width < BITS_PER_LONG && (iov_pfn + nr_pages - 1) >> addr_width);
1776
1777 if ((prot & (DMA_PTE_READ|DMA_PTE_WRITE)) == 0)
1778 return -EINVAL;
1779
1780 prot &= DMA_PTE_READ | DMA_PTE_WRITE | DMA_PTE_SNP;
1781
9051aa02
DW		 1782 if (sg)
1783 sg_res = 0;
1784 else {
1785 sg_res = nr_pages + 1;
1786 pteval = ((phys_addr_t)phys_pfn << VTD_PAGE_SHIFT) | prot;
1787 }
1788
6dd9a7c7 1789 while (nr_pages > 0) {
c85994e4
DW		 1790 uint64_t tmp;
1791
e1605495 1792 if (!sg_res) {
f532959b 1793 sg_res = aligned_nrpages(sg->offset, sg->length);
e1605495
DW		 1794 sg->dma_address = ((dma_addr_t)iov_pfn << VTD_PAGE_SHIFT) + sg->offset;
1795 sg->dma_length = sg->length;
1796 pteval = page_to_phys(sg_page(sg)) | prot;
6dd9a7c7 1797 phys_pfn = pteval >> VTD_PAGE_SHIFT;
e1605495 1798 }
6dd9a7c7 1799
e1605495 1800 if (!pte) {
6dd9a7c7
YS		 1801 largepage_lvl = hardware_largepage_caps(domain, iov_pfn, phys_pfn, sg_res);
1802
1803 first_pte = pte = pfn_to_dma_pte(domain, iov_pfn, largepage_lvl);
e1605495
DW		 1804 if (!pte)
1805 return -ENOMEM;
6dd9a7c7 1806 /* It is large page*/
6491d4d0 1807 if (largepage_lvl > 1) {
6dd9a7c7 1808 pteval |= DMA_PTE_LARGE_PAGE;
6491d4d0
WD		 1809 /* Ensure that old small page tables are removed to make room
1810 for superpage, if they exist. */
1811 dma_pte_clear_range(domain, iov_pfn,
1812 iov_pfn + lvl_to_nr_pages(largepage_lvl) - 1);
1813 dma_pte_free_pagetable(domain, iov_pfn,
1814 iov_pfn + lvl_to_nr_pages(largepage_lvl) - 1);
1815 } else {
6dd9a7c7 1816 pteval &= ~(uint64_t)DMA_PTE_LARGE_PAGE;
6491d4d0 1817 }
6dd9a7c7 1818
e1605495
DW		 1819 }
1820 /* We don't need lock here, nobody else
1821 * touches the iova range
1822 */
7766a3fb 1823 tmp = cmpxchg64_local(&pte->val, 0ULL, pteval);
c85994e4 1824 if (tmp) {
1bf20f0d 1825 static int dumps = 5;
c85994e4
DW		 1826 printk(KERN_CRIT "ERROR: DMA PTE for vPFN 0x%lx already set (to %llx not %llx)\n",
1827 iov_pfn, tmp, (unsigned long long)pteval);
1bf20f0d
DW		 1828 if (dumps) {
1829 dumps--;
1830 debug_dma_dump_mappings(NULL);
1831 }
1832 WARN_ON(1);
1833 }
6dd9a7c7
YS		 1834
1835 lvl_pages = lvl_to_nr_pages(largepage_lvl);
1836
1837 BUG_ON(nr_pages < lvl_pages);
1838 BUG_ON(sg_res < lvl_pages);
1839
1840 nr_pages -= lvl_pages;
1841 iov_pfn += lvl_pages;
1842 phys_pfn += lvl_pages;
1843 pteval += lvl_pages * VTD_PAGE_SIZE;
1844 sg_res -= lvl_pages;
1845
1846 /* If the next PTE would be the first in a new page, then we
1847 need to flush the cache on the entries we've just written.
1848 And then we'll need to recalculate 'pte', so clear it and
1849 let it get set again in the if (!pte) block above.
1850
1851 If we're done (!nr_pages) we need to flush the cache too.
1852
1853 Also if we've been setting superpages, we may need to
1854 recalculate 'pte' and switch back to smaller pages for the
1855 end of the mapping, if the trailing size is not enough to
1856 use another superpage (i.e. sg_res < lvl_pages). */
e1605495 1857 pte++;
6dd9a7c7
YS		 1858 if (!nr_pages || first_pte_in_page(pte) ||
1859 (largepage_lvl > 1 && sg_res < lvl_pages)) {
e1605495
DW		 1860 domain_flush_cache(domain, first_pte,
1861 (void *)pte - (void *)first_pte);
1862 pte = NULL;
1863 }
6dd9a7c7
YS		 1864
1865 if (!sg_res && nr_pages)
e1605495
DW		 1866 sg = sg_next(sg);
1867 }
1868 return 0;
1869}
1870
9051aa02
DW		 1871static inline int domain_sg_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
1872 struct scatterlist *sg, unsigned long nr_pages,
1873 int prot)
ba395927 1874{
9051aa02
DW		 1875 return __domain_mapping(domain, iov_pfn, sg, 0, nr_pages, prot);
1876}
6f6a00e4 1877
9051aa02
DW		 1878static inline int domain_pfn_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
1879 unsigned long phys_pfn, unsigned long nr_pages,
1880 int prot)
1881{
1882 return __domain_mapping(domain, iov_pfn, NULL, phys_pfn, nr_pages, prot);
ba395927
KA		 1883}
1884
c7151a8d 1885static void iommu_detach_dev(struct intel_iommu *iommu, u8 bus, u8 devfn)
ba395927 1886{
c7151a8d
WH		 1887 if (!iommu)
1888 return;
8c11e798
WH		 1889
1890 clear_context_table(iommu, bus, devfn);
1891 iommu->flush.flush_context(iommu, 0, 0, 0,
4c25a2c1 1892 DMA_CCMD_GLOBAL_INVL);
1f0ef2aa 1893 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
ba395927
KA		 1894}
1895
109b9b04
DW		 1896static inline void unlink_domain_info(struct device_domain_info *info)
1897{
1898 assert_spin_locked(&device_domain_lock);
1899 list_del(&info->link);
1900 list_del(&info->global);
1901 if (info->dev)
1902 info->dev->dev.archdata.iommu = NULL;
1903}
1904
ba395927
KA		 1905static void domain_remove_dev_info(struct dmar_domain *domain)
1906{
1907 struct device_domain_info *info;
1908 unsigned long flags;
c7151a8d 1909 struct intel_iommu *iommu;
ba395927
KA		 1910
1911 spin_lock_irqsave(&device_domain_lock, flags);
1912 while (!list_empty(&domain->devices)) {
1913 info = list_entry(domain->devices.next,
1914 struct device_domain_info, link);
109b9b04 1915 unlink_domain_info(info);
ba395927
KA		 1916 spin_unlock_irqrestore(&device_domain_lock, flags);
1917
93a23a72 1918 iommu_disable_dev_iotlb(info);
276dbf99 1919 iommu = device_to_iommu(info->segment, info->bus, info->devfn);
c7151a8d 1920 iommu_detach_dev(iommu, info->bus, info->devfn);
ba395927
KA		 1921 free_devinfo_mem(info);
1922
1923 spin_lock_irqsave(&device_domain_lock, flags);
1924 }
1925 spin_unlock_irqrestore(&device_domain_lock, flags);
1926}
1927
1928/*
1929 * find_domain
358dd8ac 1930 * Note: we use struct pci_dev->dev.archdata.iommu stores the info
ba395927 1931 */
38717946 1932static struct dmar_domain *
ba395927
KA		 1933find_domain(struct pci_dev *pdev)
1934{
1935 struct device_domain_info *info;
1936
1937 /* No lock here, assumes no domain exit in normal case */
358dd8ac 1938 info = pdev->dev.archdata.iommu;
ba395927
KA		 1939 if (info)
1940 return info->domain;
1941 return NULL;
1942}
1943
ba395927
KA		 1944/* domain is initialized */
1945static struct dmar_domain *get_domain_for_dev(struct pci_dev *pdev, int gaw)
1946{
1947 struct dmar_domain *domain, *found = NULL;
1948 struct intel_iommu *iommu;
1949 struct dmar_drhd_unit *drhd;
1950 struct device_domain_info *info, *tmp;
1951 struct pci_dev *dev_tmp;
1952 unsigned long flags;
1953 int bus = 0, devfn = 0;
276dbf99 1954 int segment;
2c2e2c38 1955 int ret;
ba395927
KA		 1956
1957 domain = find_domain(pdev);
1958 if (domain)
1959 return domain;
1960
276dbf99
DW		 1961 segment = pci_domain_nr(pdev->bus);
1962
ba395927
KA		 1963 dev_tmp = pci_find_upstream_pcie_bridge(pdev);
1964 if (dev_tmp) {
5f4d91a1 1965 if (pci_is_pcie(dev_tmp)) {
ba395927
KA		 1966 bus = dev_tmp->subordinate->number;
1967 devfn = 0;
1968 } else {
1969 bus = dev_tmp->bus->number;
1970 devfn = dev_tmp->devfn;
1971 }
1972 spin_lock_irqsave(&device_domain_lock, flags);
1973 list_for_each_entry(info, &device_domain_list, global) {
276dbf99
DW		 1974 if (info->segment == segment &&
1975 info->bus == bus && info->devfn == devfn) {
ba395927
KA		 1976 found = info->domain;
1977 break;
1978 }
1979 }
1980 spin_unlock_irqrestore(&device_domain_lock, flags);
1981 /* pcie-pci bridge already has a domain, uses it */
1982 if (found) {
1983 domain = found;
1984 goto found_domain;
1985 }
1986 }
1987
2c2e2c38
FY		 1988 domain = alloc_domain();
1989 if (!domain)
1990 goto error;
1991
ba395927
KA		 1992 /* Allocate new domain for the device */
1993 drhd = dmar_find_matched_drhd_unit(pdev);
1994 if (!drhd) {
1995 printk(KERN_ERR "IOMMU: can't find DMAR for device %s\n",
1996 pci_name(pdev));
d2900bd6 1997 free_domain_mem(domain);
ba395927
KA		 1998 return NULL;
1999 }
2000 iommu = drhd->iommu;
2001
2c2e2c38
FY		 2002 ret = iommu_attach_domain(domain, iommu);
2003 if (ret) {
2fe9723d 2004 free_domain_mem(domain);
ba395927 2005 goto error;
2c2e2c38 2006 }
ba395927
KA		 2007
2008 if (domain_init(domain, gaw)) {
2009 domain_exit(domain);
2010 goto error;
2011 }
2012
2013 /* register pcie-to-pci device */
2014 if (dev_tmp) {
2015 info = alloc_devinfo_mem();
2016 if (!info) {
2017 domain_exit(domain);
2018 goto error;
2019 }
276dbf99 2020 info->segment = segment;
ba395927
KA		 2021 info->bus = bus;
2022 info->devfn = devfn;
2023 info->dev = NULL;
2024 info->domain = domain;
2025 /* This domain is shared by devices under p2p bridge */
3b5410e7 2026 domain->flags |= DOMAIN_FLAG_P2P_MULTIPLE_DEVICES;
ba395927
KA		 2027
2028 /* pcie-to-pci bridge already has a domain, uses it */
2029 found = NULL;
2030 spin_lock_irqsave(&device_domain_lock, flags);
2031 list_for_each_entry(tmp, &device_domain_list, global) {
276dbf99
DW		 2032 if (tmp->segment == segment &&
2033 tmp->bus == bus && tmp->devfn == devfn) {
ba395927
KA		 2034 found = tmp->domain;
2035 break;
2036 }
2037 }
2038 if (found) {
00dfff77 2039 spin_unlock_irqrestore(&device_domain_lock, flags);
ba395927
KA		 2040 free_devinfo_mem(info);
2041 domain_exit(domain);
2042 domain = found;
2043 } else {
2044 list_add(&info->link, &domain->devices);
2045 list_add(&info->global, &device_domain_list);
00dfff77 2046 spin_unlock_irqrestore(&device_domain_lock, flags);
ba395927 2047 }
ba395927
KA		 2048 }
2049
2050found_domain:
2051 info = alloc_devinfo_mem();
2052 if (!info)
2053 goto error;
276dbf99 2054 info->segment = segment;
ba395927
KA		 2055 info->bus = pdev->bus->number;
2056 info->devfn = pdev->devfn;
2057 info->dev = pdev;
2058 info->domain = domain;
2059 spin_lock_irqsave(&device_domain_lock, flags);
2060 /* somebody is fast */
2061 found = find_domain(pdev);
2062 if (found != NULL) {
2063 spin_unlock_irqrestore(&device_domain_lock, flags);
2064 if (found != domain) {
2065 domain_exit(domain);
2066 domain = found;
2067 }
2068 free_devinfo_mem(info);
2069 return domain;
2070 }
2071 list_add(&info->link, &domain->devices);
2072 list_add(&info->global, &device_domain_list);
358dd8ac 2073 pdev->dev.archdata.iommu = info;
ba395927
KA		 2074 spin_unlock_irqrestore(&device_domain_lock, flags);
2075 return domain;
2076error:
2077 /* recheck it here, maybe others set it */
2078 return find_domain(pdev);
2079}
2080
2c2e2c38 2081static int iommu_identity_mapping;
e0fc7e0b
DW		 2082#define IDENTMAP_ALL 1
2083#define IDENTMAP_GFX 2
2084#define IDENTMAP_AZALIA 4
2c2e2c38 2085
b213203e
DW		 2086static int iommu_domain_identity_map(struct dmar_domain *domain,
2087 unsigned long long start,
2088 unsigned long long end)
ba395927 2089{
c5395d5c
DW		 2090 unsigned long first_vpfn = start >> VTD_PAGE_SHIFT;
2091 unsigned long last_vpfn = end >> VTD_PAGE_SHIFT;
2092
2093 if (!reserve_iova(&domain->iovad, dma_to_mm_pfn(first_vpfn),
2094 dma_to_mm_pfn(last_vpfn))) {
ba395927 2095 printk(KERN_ERR "IOMMU: reserve iova failed\n");
b213203e 2096 return -ENOMEM;
ba395927
KA		 2097 }
2098
c5395d5c
DW		 2099 pr_debug("Mapping reserved region %llx-%llx for domain %d\n",
2100 start, end, domain->id);
ba395927
KA		 2101 /*
2102 * RMRR range might have overlap with physical memory range,
2103 * clear it first
2104 */
c5395d5c 2105 dma_pte_clear_range(domain, first_vpfn, last_vpfn);
ba395927 2106
c5395d5c
DW		 2107 return domain_pfn_mapping(domain, first_vpfn, first_vpfn,
2108 last_vpfn - first_vpfn + 1,
61df7443 2109 DMA_PTE_READ|DMA_PTE_WRITE);
b213203e
DW		 2110}
2111
2112static int iommu_prepare_identity_map(struct pci_dev *pdev,
2113 unsigned long long start,
2114 unsigned long long end)
2115{
2116 struct dmar_domain *domain;
2117 int ret;
2118
c7ab48d2 2119 domain = get_domain_for_dev(pdev, DEFAULT_DOMAIN_ADDRESS_WIDTH);
b213203e
DW		 2120 if (!domain)
2121 return -ENOMEM;
2122
19943b0e
DW		 2123 /* For _hardware_ passthrough, don't bother. But for software
2124 passthrough, we do it anyway -- it may indicate a memory
2125 range which is reserved in E820, so which didn't get set
2126 up to start with in si_domain */
2127 if (domain == si_domain && hw_pass_through) {
2128 printk("Ignoring identity map for HW passthrough device %s [0x%Lx - 0x%Lx]\n",
2129 pci_name(pdev), start, end);
2130 return 0;
2131 }
2132
2133 printk(KERN_INFO
2134 "IOMMU: Setting identity map for device %s [0x%Lx - 0x%Lx]\n",
2135 pci_name(pdev), start, end);
2ff729f5 2136
5595b528
DW		 2137 if (end < start) {
2138 WARN(1, "Your BIOS is broken; RMRR ends before it starts!\n"
2139 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
2140 dmi_get_system_info(DMI_BIOS_VENDOR),
2141 dmi_get_system_info(DMI_BIOS_VERSION),
2142 dmi_get_system_info(DMI_PRODUCT_VERSION));
2143 ret = -EIO;
2144 goto error;
2145 }
2146
2ff729f5
DW		 2147 if (end >> agaw_to_width(domain->agaw)) {
2148 WARN(1, "Your BIOS is broken; RMRR exceeds permitted address width (%d bits)\n"
2149 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
2150 agaw_to_width(domain->agaw),
2151 dmi_get_system_info(DMI_BIOS_VENDOR),
2152 dmi_get_system_info(DMI_BIOS_VERSION),
2153 dmi_get_system_info(DMI_PRODUCT_VERSION));
2154 ret = -EIO;
2155 goto error;
2156 }
19943b0e 2157
b213203e 2158 ret = iommu_domain_identity_map(domain, start, end);
ba395927
KA		 2159 if (ret)
2160 goto error;
2161
2162 /* context entry init */
4ed0d3e6 2163 ret = domain_context_mapping(domain, pdev, CONTEXT_TT_MULTI_LEVEL);
b213203e
DW		 2164 if (ret)
2165 goto error;
2166
2167 return 0;
2168
2169 error:
ba395927
KA		 2170 domain_exit(domain);
2171 return ret;
ba395927
KA		 2172}
2173
2174static inline int iommu_prepare_rmrr_dev(struct dmar_rmrr_unit *rmrr,
2175 struct pci_dev *pdev)
2176{
358dd8ac 2177 if (pdev->dev.archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO)
ba395927
KA		 2178 return 0;
2179 return iommu_prepare_identity_map(pdev, rmrr->base_address,
70e535d1 2180 rmrr->end_address);
ba395927
KA		 2181}
2182
d3f13810 2183#ifdef CONFIG_INTEL_IOMMU_FLOPPY_WA
49a0429e
KA		 2184static inline void iommu_prepare_isa(void)
2185{
2186 struct pci_dev *pdev;
2187 int ret;
2188
2189 pdev = pci_get_class(PCI_CLASS_BRIDGE_ISA << 8, NULL);
2190 if (!pdev)
2191 return;
2192
c7ab48d2 2193 printk(KERN_INFO "IOMMU: Prepare 0-16MiB unity mapping for LPC\n");
70e535d1 2194 ret = iommu_prepare_identity_map(pdev, 0, 16*1024*1024 - 1);
49a0429e
KA		 2195
2196 if (ret)
c7ab48d2
DW		 2197 printk(KERN_ERR "IOMMU: Failed to create 0-16MiB identity map; "
2198 "floppy might not work\n");
49a0429e
KA		 2199
2200}
2201#else
2202static inline void iommu_prepare_isa(void)
2203{
2204 return;
2205}
d3f13810 2206#endif /* !CONFIG_INTEL_IOMMU_FLPY_WA */
49a0429e 2207
2c2e2c38 2208static int md_domain_init(struct dmar_domain *domain, int guest_width);
c7ab48d2 2209
071e1374 2210static int __init si_domain_init(int hw)
2c2e2c38
FY		 2211{
2212 struct dmar_drhd_unit *drhd;
2213 struct intel_iommu *iommu;
c7ab48d2 2214 int nid, ret = 0;
2c2e2c38
FY		 2215
2216 si_domain = alloc_domain();
2217 if (!si_domain)
2218 return -EFAULT;
2219
2c2e2c38
FY		 2220 for_each_active_iommu(iommu, drhd) {
2221 ret = iommu_attach_domain(si_domain, iommu);
2222 if (ret) {
2223 domain_exit(si_domain);
2224 return -EFAULT;
2225 }
2226 }
2227
2228 if (md_domain_init(si_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
2229 domain_exit(si_domain);
2230 return -EFAULT;
2231 }
2232
2233 si_domain->flags = DOMAIN_FLAG_STATIC_IDENTITY;
9544c003
JL		 2234 pr_debug("IOMMU: identity mapping domain is domain %d\n",
2235 si_domain->id);
2c2e2c38 2236
19943b0e
DW		 2237 if (hw)
2238 return 0;
2239
c7ab48d2 2240 for_each_online_node(nid) {
5dfe8660
TH		 2241 unsigned long start_pfn, end_pfn;
2242 int i;
2243
2244 for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, NULL) {
2245 ret = iommu_domain_identity_map(si_domain,
2246 PFN_PHYS(start_pfn), PFN_PHYS(end_pfn));
2247 if (ret)
2248 return ret;
2249 }
c7ab48d2
DW		 2250 }
2251
2c2e2c38
FY		 2252 return 0;
2253}
2254
2255static void domain_remove_one_dev_info(struct dmar_domain *domain,
2256 struct pci_dev *pdev);
2257static int identity_mapping(struct pci_dev *pdev)
2258{
2259 struct device_domain_info *info;
2260
2261 if (likely(!iommu_identity_mapping))
2262 return 0;
2263
cb452a40
MT		 2264 info = pdev->dev.archdata.iommu;
2265 if (info && info != DUMMY_DEVICE_DOMAIN_INFO)
2266 return (info->domain == si_domain);
2c2e2c38 2267
2c2e2c38
FY		 2268 return 0;
2269}
2270
2271static int domain_add_dev_info(struct dmar_domain *domain,
5fe60f4e
DW		 2272 struct pci_dev *pdev,
2273 int translation)
2c2e2c38
FY		 2274{
2275 struct device_domain_info *info;
2276 unsigned long flags;
5fe60f4e 2277 int ret;
2c2e2c38
FY		 2278
2279 info = alloc_devinfo_mem();
2280 if (!info)
2281 return -ENOMEM;
2282
2283 info->segment = pci_domain_nr(pdev->bus);
2284 info->bus = pdev->bus->number;
2285 info->devfn = pdev->devfn;
2286 info->dev = pdev;
2287 info->domain = domain;
2288
2289 spin_lock_irqsave(&device_domain_lock, flags);
2290 list_add(&info->link, &domain->devices);
2291 list_add(&info->global, &device_domain_list);
2292 pdev->dev.archdata.iommu = info;
2293 spin_unlock_irqrestore(&device_domain_lock, flags);
2294
e2ad23d0
DW		 2295 ret = domain_context_mapping(domain, pdev, translation);
2296 if (ret) {
2297 spin_lock_irqsave(&device_domain_lock, flags);
109b9b04 2298 unlink_domain_info(info);
e2ad23d0
DW		 2299 spin_unlock_irqrestore(&device_domain_lock, flags);
2300 free_devinfo_mem(info);
2301 return ret;
2302 }
2303
2c2e2c38
FY		 2304 return 0;
2305}
2306
ea2447f7
TM		 2307static bool device_has_rmrr(struct pci_dev *dev)
2308{
2309 struct dmar_rmrr_unit *rmrr;
2310 int i;
2311
2312 for_each_rmrr_units(rmrr) {
2313 for (i = 0; i < rmrr->devices_cnt; i++) {
2314 /*
2315 * Return TRUE if this RMRR contains the device that
2316 * is passed in.
2317 */
2318 if (rmrr->devices[i] == dev)
2319 return true;
2320 }
2321 }
2322 return false;
2323}
2324
6941af28
DW		 2325static int iommu_should_identity_map(struct pci_dev *pdev, int startup)
2326{
ea2447f7
TM		 2327
2328 /*
2329 * We want to prevent any device associated with an RMRR from
2330 * getting placed into the SI Domain. This is done because
2331 * problems exist when devices are moved in and out of domains
2332 * and their respective RMRR info is lost. We exempt USB devices
2333 * from this process due to their usage of RMRRs that are known
2334 * to not be needed after BIOS hand-off to OS.
2335 */
2336 if (device_has_rmrr(pdev) &&
2337 (pdev->class >> 8) != PCI_CLASS_SERIAL_USB)
2338 return 0;
2339
e0fc7e0b
DW		 2340 if ((iommu_identity_mapping & IDENTMAP_AZALIA) && IS_AZALIA(pdev))
2341 return 1;
2342
2343 if ((iommu_identity_mapping & IDENTMAP_GFX) && IS_GFX_DEVICE(pdev))
2344 return 1;
2345
2346 if (!(iommu_identity_mapping & IDENTMAP_ALL))
2347 return 0;
6941af28 2348
3dfc813d
DW		 2349 /*
2350 * We want to start off with all devices in the 1:1 domain, and
2351 * take them out later if we find they can't access all of memory.
2352 *
2353 * However, we can't do this for PCI devices behind bridges,
2354 * because all PCI devices behind the same bridge will end up
2355 * with the same source-id on their transactions.
2356 *
2357 * Practically speaking, we can't change things around for these
2358 * devices at run-time, because we can't be sure there'll be no
2359 * DMA transactions in flight for any of their siblings.
2360 *
2361 * So PCI devices (unless they're on the root bus) as well as
2362 * their parent PCI-PCI or PCIe-PCI bridges must be left _out_ of
2363 * the 1:1 domain, just in _case_ one of their siblings turns out
2364 * not to be able to map all of memory.
2365 */
5f4d91a1 2366 if (!pci_is_pcie(pdev)) {
3dfc813d
DW		 2367 if (!pci_is_root_bus(pdev->bus))
2368 return 0;
2369 if (pdev->class >> 8 == PCI_CLASS_BRIDGE_PCI)
2370 return 0;
62f87c0e 2371 } else if (pci_pcie_type(pdev) == PCI_EXP_TYPE_PCI_BRIDGE)
3dfc813d
DW		 2372 return 0;
2373
2374 /*
2375 * At boot time, we don't yet know if devices will be 64-bit capable.
2376 * Assume that they will -- if they turn out not to be, then we can
2377 * take them out of the 1:1 domain later.
2378 */
8fcc5372
CW		 2379 if (!startup) {
2380 /*
2381 * If the device's dma_mask is less than the system's memory
2382 * size then this is not a candidate for identity mapping.
2383 */
2384 u64 dma_mask = pdev->dma_mask;
2385
2386 if (pdev->dev.coherent_dma_mask &&
2387 pdev->dev.coherent_dma_mask < dma_mask)
2388 dma_mask = pdev->dev.coherent_dma_mask;
2389
2390 return dma_mask >= dma_get_required_mask(&pdev->dev);
2391 }
6941af28
DW		 2392
2393 return 1;
2394}
2395
071e1374 2396static int __init iommu_prepare_static_identity_mapping(int hw)
2c2e2c38 2397{
2c2e2c38
FY		 2398 struct pci_dev *pdev = NULL;
2399 int ret;
2400
19943b0e 2401 ret = si_domain_init(hw);
2c2e2c38
FY		 2402 if (ret)
2403 return -EFAULT;
2404
2c2e2c38 2405 for_each_pci_dev(pdev) {
6941af28 2406 if (iommu_should_identity_map(pdev, 1)) {
5fe60f4e 2407 ret = domain_add_dev_info(si_domain, pdev,
eae460b6
MT		 2408 hw ? CONTEXT_TT_PASS_THROUGH :
2409 CONTEXT_TT_MULTI_LEVEL);
2410 if (ret) {
2411 /* device not associated with an iommu */
2412 if (ret == -ENODEV)
2413 continue;
62edf5dc 2414 return ret;
eae460b6
MT		 2415 }
2416 pr_info("IOMMU: %s identity mapping for device %s\n",
2417 hw ? "hardware" : "software", pci_name(pdev));
62edf5dc 2418 }
2c2e2c38
FY		 2419 }
2420
2421 return 0;
2422}
2423
b779260b 2424static int __init init_dmars(void)
ba395927
KA		 2425{
2426 struct dmar_drhd_unit *drhd;
2427 struct dmar_rmrr_unit *rmrr;
2428 struct pci_dev *pdev;
2429 struct intel_iommu *iommu;
9d783ba0 2430 int i, ret;
2c2e2c38 2431
ba395927
KA		 2432 /*
2433 * for each drhd
2434 * allocate root
2435 * initialize and program root entry to not present
2436 * endfor
2437 */
2438 for_each_drhd_unit(drhd) {
5e0d2a6f 2439 /*
2440 * lock not needed as this is only incremented in the single
2441 * threaded kernel __init code path all other access are read
2442 * only
2443 */
1b198bb0
MT		 2444 if (g_num_of_iommus < IOMMU_UNITS_SUPPORTED) {
2445 g_num_of_iommus++;
2446 continue;
2447 }
2448 printk_once(KERN_ERR "intel-iommu: exceeded %d IOMMUs\n",
2449 IOMMU_UNITS_SUPPORTED);
5e0d2a6f 2450 }
2451
d9630fe9
WH		 2452 g_iommus = kcalloc(g_num_of_iommus, sizeof(struct intel_iommu *),
2453 GFP_KERNEL);
2454 if (!g_iommus) {
2455 printk(KERN_ERR "Allocating global iommu array failed\n");
2456 ret = -ENOMEM;
2457 goto error;
2458 }
2459
80b20dd8 2460 deferred_flush = kzalloc(g_num_of_iommus *
2461 sizeof(struct deferred_flush_tables), GFP_KERNEL);
2462 if (!deferred_flush) {
5e0d2a6f 2463 ret = -ENOMEM;
2464 goto error;
2465 }
2466
7c919779 2467 for_each_active_iommu(iommu, drhd) {
d9630fe9 2468 g_iommus[iommu->seq_id] = iommu;
ba395927 2469
e61d98d8
SS		 2470 ret = iommu_init_domains(iommu);
2471 if (ret)
2472 goto error;
2473
ba395927
KA		 2474 /*
2475 * TBD:
2476 * we could share the same root & context tables
25985edc 2477 * among all IOMMU's. Need to Split it later.
ba395927
KA		 2478 */
2479 ret = iommu_alloc_root_entry(iommu);
2480 if (ret) {
2481 printk(KERN_ERR "IOMMU: allocate root entry failed\n");
2482 goto error;
2483 }
4ed0d3e6 2484 if (!ecap_pass_through(iommu->ecap))
19943b0e 2485 hw_pass_through = 0;
ba395927
KA		 2486 }
2487
1531a6a6
SS		 2488 /*
2489 * Start from the sane iommu hardware state.
2490 */
7c919779 2491 for_each_active_iommu(iommu, drhd) {
1531a6a6
SS		 2492 /*
2493 * If the queued invalidation is already initialized by us
2494 * (for example, while enabling interrupt-remapping) then
2495 * we got the things already rolling from a sane state.
2496 */
2497 if (iommu->qi)
2498 continue;
2499
2500 /*
2501 * Clear any previous faults.
2502 */
2503 dmar_fault(-1, iommu);
2504 /*
2505 * Disable queued invalidation if supported and already enabled
2506 * before OS handover.
2507 */
2508 dmar_disable_qi(iommu);
2509 }
2510
7c919779 2511 for_each_active_iommu(iommu, drhd) {
a77b67d4
YS		 2512 if (dmar_enable_qi(iommu)) {
2513 /*
2514 * Queued Invalidate not enabled, use Register Based
2515 * Invalidate
2516 */
2517 iommu->flush.flush_context = __iommu_flush_context;
2518 iommu->flush.flush_iotlb = __iommu_flush_iotlb;
680a7524 2519 printk(KERN_INFO "IOMMU %d 0x%Lx: using Register based "
b4e0f9eb 2520 "invalidation\n",
680a7524 2521 iommu->seq_id,
b4e0f9eb 2522 (unsigned long long)drhd->reg_base_addr);
a77b67d4
YS		 2523 } else {
2524 iommu->flush.flush_context = qi_flush_context;
2525 iommu->flush.flush_iotlb = qi_flush_iotlb;
680a7524 2526 printk(KERN_INFO "IOMMU %d 0x%Lx: using Queued "
b4e0f9eb 2527 "invalidation\n",
680a7524 2528 iommu->seq_id,
b4e0f9eb 2529 (unsigned long long)drhd->reg_base_addr);
a77b67d4
YS		 2530 }
2531 }
2532
19943b0e 2533 if (iommu_pass_through)
e0fc7e0b
DW		 2534 iommu_identity_mapping |= IDENTMAP_ALL;
2535
d3f13810 2536#ifdef CONFIG_INTEL_IOMMU_BROKEN_GFX_WA
e0fc7e0b 2537 iommu_identity_mapping |= IDENTMAP_GFX;
19943b0e 2538#endif
e0fc7e0b
DW		 2539
2540 check_tylersburg_isoch();
2541
ba395927 2542 /*
19943b0e
DW		 2543 * If pass through is not set or not enabled, setup context entries for
2544 * identity mappings for rmrr, gfx, and isa and may fall back to static
2545 * identity mapping if iommu_identity_mapping is set.
ba395927 2546 */
19943b0e
DW		 2547 if (iommu_identity_mapping) {
2548 ret = iommu_prepare_static_identity_mapping(hw_pass_through);
4ed0d3e6 2549 if (ret) {
19943b0e
DW		 2550 printk(KERN_CRIT "Failed to setup IOMMU pass-through\n");
2551 goto error;
ba395927
KA		 2552 }
2553 }
ba395927 2554 /*
19943b0e
DW		 2555 * For each rmrr
2556 * for each dev attached to rmrr
2557 * do
2558 * locate drhd for dev, alloc domain for dev
2559 * allocate free domain
2560 * allocate page table entries for rmrr
2561 * if context not allocated for bus
2562 * allocate and init context
2563 * set present in root table for this bus
2564 * init context with domain, translation etc
2565 * endfor
2566 * endfor
ba395927 2567 */
19943b0e
DW		 2568 printk(KERN_INFO "IOMMU: Setting RMRR:\n");
2569 for_each_rmrr_units(rmrr) {
2570 for (i = 0; i < rmrr->devices_cnt; i++) {
2571 pdev = rmrr->devices[i];
2572 /*
2573 * some BIOS lists non-exist devices in DMAR
2574 * table.
2575 */
2576 if (!pdev)
2577 continue;
2578 ret = iommu_prepare_rmrr_dev(rmrr, pdev);
2579 if (ret)
2580 printk(KERN_ERR
2581 "IOMMU: mapping reserved region failed\n");
ba395927 2582 }
4ed0d3e6 2583 }
49a0429e 2584
19943b0e
DW		 2585 iommu_prepare_isa();
2586
ba395927
KA		 2587 /*
2588 * for each drhd
2589 * enable fault log
2590 * global invalidate context cache
2591 * global invalidate iotlb
2592 * enable translation
2593 */
7c919779 2594 for_each_iommu(iommu, drhd) {
51a63e67
JC		 2595 if (drhd->ignored) {
2596 /*
2597 * we always have to disable PMRs or DMA may fail on
2598 * this device
2599 */
2600 if (force_on)
7c919779 2601 iommu_disable_protect_mem_regions(iommu);
ba395927 2602 continue;
51a63e67 2603 }
ba395927
KA		 2604
2605 iommu_flush_write_buffer(iommu);
2606
3460a6d9
KA		 2607 ret = dmar_set_interrupt(iommu);
2608 if (ret)
2609 goto error;
2610
ba395927
KA		 2611 iommu_set_root_entry(iommu);
2612
4c25a2c1 2613 iommu->flush.flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL);
1f0ef2aa 2614 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
f8bab735 2615
ba395927
KA		 2616 ret = iommu_enable_translation(iommu);
2617 if (ret)
2618 goto error;
b94996c9
DW		 2619
2620 iommu_disable_protect_mem_regions(iommu);
ba395927
KA		 2621 }
2622
2623 return 0;
2624error:
7c919779 2625 for_each_active_iommu(iommu, drhd)
a868e6b7 2626 free_dmar_iommu(iommu);
9bdc531e 2627 kfree(deferred_flush);
d9630fe9 2628 kfree(g_iommus);
ba395927
KA		 2629 return ret;
2630}
2631
5a5e02a6 2632/* This takes a number of _MM_ pages, not VTD pages */
875764de
DW		 2633static struct iova *intel_alloc_iova(struct device *dev,
2634 struct dmar_domain *domain,
2635 unsigned long nrpages, uint64_t dma_mask)
ba395927 2636{
ba395927 2637 struct pci_dev *pdev = to_pci_dev(dev);
ba395927 2638 struct iova *iova = NULL;
ba395927 2639
875764de
DW		 2640 /* Restrict dma_mask to the width that the iommu can handle */
2641 dma_mask = min_t(uint64_t, DOMAIN_MAX_ADDR(domain->gaw), dma_mask);
2642
2643 if (!dmar_forcedac && dma_mask > DMA_BIT_MASK(32)) {
ba395927
KA		 2644 /*
2645 * First try to allocate an io virtual address in
284901a9 2646 * DMA_BIT_MASK(32) and if that fails then try allocating
3609801e 2647 * from higher range
ba395927 2648 */
875764de
DW		 2649 iova = alloc_iova(&domain->iovad, nrpages,
2650 IOVA_PFN(DMA_BIT_MASK(32)), 1);
2651 if (iova)
2652 return iova;
2653 }
2654 iova = alloc_iova(&domain->iovad, nrpages, IOVA_PFN(dma_mask), 1);
2655 if (unlikely(!iova)) {
2656 printk(KERN_ERR "Allocating %ld-page iova for %s failed",
2657 nrpages, pci_name(pdev));
f76aec76
KA		 2658 return NULL;
2659 }
2660
2661 return iova;
2662}
2663
147202aa 2664static struct dmar_domain *__get_valid_domain_for_dev(struct pci_dev *pdev)
f76aec76
KA		 2665{
2666 struct dmar_domain *domain;
2667 int ret;
2668
2669 domain = get_domain_for_dev(pdev,
2670 DEFAULT_DOMAIN_ADDRESS_WIDTH);
2671 if (!domain) {
2672 printk(KERN_ERR
2673 "Allocating domain for %s failed", pci_name(pdev));
4fe05bbc 2674 return NULL;
ba395927
KA		 2675 }
2676
2677 /* make sure context mapping is ok */
5331fe6f 2678 if (unlikely(!domain_context_mapped(pdev))) {
4ed0d3e6
FY		 2679 ret = domain_context_mapping(domain, pdev,
2680 CONTEXT_TT_MULTI_LEVEL);
f76aec76
KA		 2681 if (ret) {
2682 printk(KERN_ERR
2683 "Domain context map for %s failed",
2684 pci_name(pdev));
4fe05bbc 2685 return NULL;
f76aec76 2686 }
ba395927
KA		 2687 }
2688
f76aec76
KA		 2689 return domain;
2690}
2691
147202aa
DW		 2692static inline struct dmar_domain *get_valid_domain_for_dev(struct pci_dev *dev)
2693{
2694 struct device_domain_info *info;
2695
2696 /* No lock here, assumes no domain exit in normal case */
2697 info = dev->dev.archdata.iommu;
2698 if (likely(info))
2699 return info->domain;
2700
2701 return __get_valid_domain_for_dev(dev);
2702}
2703
2c2e2c38
FY		 2704static int iommu_dummy(struct pci_dev *pdev)
2705{
2706 return pdev->dev.archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO;
2707}
2708
2709/* Check if the pdev needs to go through non-identity map and unmap process.*/
73676832 2710static int iommu_no_mapping(struct device *dev)
2c2e2c38 2711{
73676832 2712 struct pci_dev *pdev;
2c2e2c38
FY		 2713 int found;
2714
dbad0864 2715 if (unlikely(!dev_is_pci(dev)))
73676832
DW		 2716 return 1;
2717
2718 pdev = to_pci_dev(dev);
1e4c64c4
DW		 2719 if (iommu_dummy(pdev))
2720 return 1;
2721
2c2e2c38 2722 if (!iommu_identity_mapping)
1e4c64c4 2723 return 0;
2c2e2c38
FY		 2724
2725 found = identity_mapping(pdev);
2726 if (found) {
6941af28 2727 if (iommu_should_identity_map(pdev, 0))
2c2e2c38
FY		 2728 return 1;
2729 else {
2730 /*
2731 * 32 bit DMA is removed from si_domain and fall back
2732 * to non-identity mapping.
2733 */
2734 domain_remove_one_dev_info(si_domain, pdev);
2735 printk(KERN_INFO "32bit %s uses non-identity mapping\n",
2736 pci_name(pdev));
2737 return 0;
2738 }
2739 } else {
2740 /*
2741 * In case of a detached 64 bit DMA device from vm, the device
2742 * is put into si_domain for identity mapping.
2743 */
6941af28 2744 if (iommu_should_identity_map(pdev, 0)) {
2c2e2c38 2745 int ret;
5fe60f4e
DW		 2746 ret = domain_add_dev_info(si_domain, pdev,
2747 hw_pass_through ?
2748 CONTEXT_TT_PASS_THROUGH :
2749 CONTEXT_TT_MULTI_LEVEL);
2c2e2c38
FY		 2750 if (!ret) {
2751 printk(KERN_INFO "64bit %s uses identity mapping\n",
2752 pci_name(pdev));
2753 return 1;
2754 }
2755 }
2756 }
2757
1e4c64c4 2758 return 0;
2c2e2c38
FY		 2759}
2760
bb9e6d65
FT		 2761static dma_addr_t __intel_map_single(struct device *hwdev, phys_addr_t paddr,
2762 size_t size, int dir, u64 dma_mask)
f76aec76
KA		 2763{
2764 struct pci_dev *pdev = to_pci_dev(hwdev);
f76aec76 2765 struct dmar_domain *domain;
5b6985ce 2766 phys_addr_t start_paddr;
f76aec76
KA		 2767 struct iova *iova;
2768 int prot = 0;
6865f0d1 2769 int ret;
8c11e798 2770 struct intel_iommu *iommu;
33041ec0 2771 unsigned long paddr_pfn = paddr >> PAGE_SHIFT;
f76aec76
KA		 2772
2773 BUG_ON(dir == DMA_NONE);
2c2e2c38 2774
73676832 2775 if (iommu_no_mapping(hwdev))
6865f0d1 2776 return paddr;
f76aec76
KA		 2777
2778 domain = get_valid_domain_for_dev(pdev);
2779 if (!domain)
2780 return 0;
2781
8c11e798 2782 iommu = domain_get_iommu(domain);
88cb6a74 2783 size = aligned_nrpages(paddr, size);
f76aec76 2784
c681d0ba 2785 iova = intel_alloc_iova(hwdev, domain, dma_to_mm_pfn(size), dma_mask);
f76aec76
KA		 2786 if (!iova)
2787 goto error;
2788
ba395927
KA		 2789 /*
2790 * Check if DMAR supports zero-length reads on write only
2791 * mappings..
2792 */
2793 if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
8c11e798 2794 !cap_zlr(iommu->cap))
ba395927
KA		 2795 prot |= DMA_PTE_READ;
2796 if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
2797 prot |= DMA_PTE_WRITE;
2798 /*
6865f0d1 2799 * paddr - (paddr + size) might be partial page, we should map the whole
ba395927 2800 * page. Note: if two part of one page are separately mapped, we
6865f0d1 2801 * might have two guest_addr mapping to the same host paddr, but this
ba395927
KA		 2802 * is not a big problem
2803 */
0ab36de2 2804 ret = domain_pfn_mapping(domain, mm_to_dma_pfn(iova->pfn_lo),
33041ec0 2805 mm_to_dma_pfn(paddr_pfn), size, prot);
ba395927
KA		 2806 if (ret)
2807 goto error;
2808
1f0ef2aa
DW		 2809 /* it's a non-present to present mapping. Only flush if caching mode */
2810 if (cap_caching_mode(iommu->cap))
82653633 2811 iommu_flush_iotlb_psi(iommu, domain->id, mm_to_dma_pfn(iova->pfn_lo), size, 1);
1f0ef2aa 2812 else
8c11e798 2813 iommu_flush_write_buffer(iommu);
f76aec76 2814
03d6a246
DW		 2815 start_paddr = (phys_addr_t)iova->pfn_lo << PAGE_SHIFT;
2816 start_paddr += paddr & ~PAGE_MASK;
2817 return start_paddr;
ba395927 2818
ba395927 2819error:
f76aec76
KA		 2820 if (iova)
2821 __free_iova(&domain->iovad, iova);
4cf2e75d 2822 printk(KERN_ERR"Device %s request: %zx@%llx dir %d --- failed\n",
5b6985ce 2823 pci_name(pdev), size, (unsigned long long)paddr, dir);
ba395927
KA		 2824 return 0;
2825}
2826
ffbbef5c
FT		 2827static dma_addr_t intel_map_page(struct device *dev, struct page *page,
2828 unsigned long offset, size_t size,
2829 enum dma_data_direction dir,
2830 struct dma_attrs *attrs)
bb9e6d65 2831{
ffbbef5c
FT		 2832 return __intel_map_single(dev, page_to_phys(page) + offset, size,
2833 dir, to_pci_dev(dev)->dma_mask);
bb9e6d65
FT		 2834}
2835
5e0d2a6f 2836static void flush_unmaps(void)
2837{
80b20dd8 2838 int i, j;
5e0d2a6f 2839
5e0d2a6f 2840 timer_on = 0;
2841
2842 /* just flush them all */
2843 for (i = 0; i < g_num_of_iommus; i++) {
a2bb8459
WH		 2844 struct intel_iommu *iommu = g_iommus[i];
2845 if (!iommu)
2846 continue;
c42d9f32 2847
9dd2fe89
YZ		 2848 if (!deferred_flush[i].next)
2849 continue;
2850
78d5f0f5
NA		 2851 /* In caching mode, global flushes turn emulation expensive */
2852 if (!cap_caching_mode(iommu->cap))
2853 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
93a23a72 2854 DMA_TLB_GLOBAL_FLUSH);
9dd2fe89 2855 for (j = 0; j < deferred_flush[i].next; j++) {
93a23a72
YZ		 2856 unsigned long mask;
2857 struct iova *iova = deferred_flush[i].iova[j];
78d5f0f5
NA		 2858 struct dmar_domain *domain = deferred_flush[i].domain[j];
2859
2860 /* On real hardware multiple invalidations are expensive */
2861 if (cap_caching_mode(iommu->cap))
2862 iommu_flush_iotlb_psi(iommu, domain->id,
2863 iova->pfn_lo, iova->pfn_hi - iova->pfn_lo + 1, 0);
2864 else {
2865 mask = ilog2(mm_to_dma_pfn(iova->pfn_hi - iova->pfn_lo + 1));
2866 iommu_flush_dev_iotlb(deferred_flush[i].domain[j],
2867 (uint64_t)iova->pfn_lo << PAGE_SHIFT, mask);
2868 }
93a23a72 2869 __free_iova(&deferred_flush[i].domain[j]->iovad, iova);
80b20dd8 2870 }
9dd2fe89 2871 deferred_flush[i].next = 0;
5e0d2a6f 2872 }
2873
5e0d2a6f 2874 list_size = 0;
5e0d2a6f 2875}
2876
2877static void flush_unmaps_timeout(unsigned long data)
2878{
80b20dd8 2879 unsigned long flags;
2880
2881 spin_lock_irqsave(&async_umap_flush_lock, flags);
5e0d2a6f 2882 flush_unmaps();
80b20dd8 2883 spin_unlock_irqrestore(&async_umap_flush_lock, flags);
5e0d2a6f 2884}
2885
2886static void add_unmap(struct dmar_domain *dom, struct iova *iova)
2887{
2888 unsigned long flags;
80b20dd8 2889 int next, iommu_id;
8c11e798 2890 struct intel_iommu *iommu;
5e0d2a6f 2891
2892 spin_lock_irqsave(&async_umap_flush_lock, flags);
80b20dd8 2893 if (list_size == HIGH_WATER_MARK)
2894 flush_unmaps();
2895
8c11e798
WH		 2896 iommu = domain_get_iommu(dom);
2897 iommu_id = iommu->seq_id;
c42d9f32 2898
80b20dd8 2899 next = deferred_flush[iommu_id].next;
2900 deferred_flush[iommu_id].domain[next] = dom;
2901 deferred_flush[iommu_id].iova[next] = iova;
2902 deferred_flush[iommu_id].next++;
5e0d2a6f 2903
2904 if (!timer_on) {
2905 mod_timer(&unmap_timer, jiffies + msecs_to_jiffies(10));
2906 timer_on = 1;
2907 }
2908 list_size++;
2909 spin_unlock_irqrestore(&async_umap_flush_lock, flags);
2910}
2911
ffbbef5c
FT		 2912static void intel_unmap_page(struct device *dev, dma_addr_t dev_addr,
2913 size_t size, enum dma_data_direction dir,
2914 struct dma_attrs *attrs)
ba395927 2915{
ba395927 2916 struct pci_dev *pdev = to_pci_dev(dev);
f76aec76 2917 struct dmar_domain *domain;
d794dc9b 2918 unsigned long start_pfn, last_pfn;
ba395927 2919 struct iova *iova;
8c11e798 2920 struct intel_iommu *iommu;
ba395927 2921
73676832 2922 if (iommu_no_mapping(dev))
f76aec76 2923 return;
2c2e2c38 2924
ba395927
KA		 2925 domain = find_domain(pdev);
2926 BUG_ON(!domain);
2927
8c11e798
WH		 2928 iommu = domain_get_iommu(domain);
2929
ba395927 2930 iova = find_iova(&domain->iovad, IOVA_PFN(dev_addr));
85b98276
DW		 2931 if (WARN_ONCE(!iova, "Driver unmaps unmatched page at PFN %llx\n",
2932 (unsigned long long)dev_addr))
ba395927 2933 return;
ba395927 2934
d794dc9b
DW		 2935 start_pfn = mm_to_dma_pfn(iova->pfn_lo);
2936 last_pfn = mm_to_dma_pfn(iova->pfn_hi + 1) - 1;
ba395927 2937
d794dc9b
DW		 2938 pr_debug("Device %s unmapping: pfn %lx-%lx\n",
2939 pci_name(pdev), start_pfn, last_pfn);
ba395927 2940
f76aec76 2941 /* clear the whole page */
d794dc9b
DW		 2942 dma_pte_clear_range(domain, start_pfn, last_pfn);
2943
f76aec76 2944 /* free page tables */
d794dc9b
DW		 2945 dma_pte_free_pagetable(domain, start_pfn, last_pfn);
2946
5e0d2a6f 2947 if (intel_iommu_strict) {
03d6a246 2948 iommu_flush_iotlb_psi(iommu, domain->id, start_pfn,
82653633 2949 last_pfn - start_pfn + 1, 0);
5e0d2a6f 2950 /* free iova */
2951 __free_iova(&domain->iovad, iova);
2952 } else {
2953 add_unmap(domain, iova);
2954 /*
2955 * queue up the release of the unmap to save the 1/6th of the
2956 * cpu used up by the iotlb flush operation...
2957 */
5e0d2a6f 2958 }
ba395927
KA		 2959}
2960
d7ab5c46 2961static void *intel_alloc_coherent(struct device *hwdev, size_t size,
baa676fc
AP		 2962 dma_addr_t *dma_handle, gfp_t flags,
2963 struct dma_attrs *attrs)
ba395927
KA		 2964{
2965 void *vaddr;
2966 int order;
2967
5b6985ce 2968 size = PAGE_ALIGN(size);
ba395927 2969 order = get_order(size);
e8bb910d
AW		 2970
2971 if (!iommu_no_mapping(hwdev))
2972 flags &= ~(GFP_DMA | GFP_DMA32);
2973 else if (hwdev->coherent_dma_mask < dma_get_required_mask(hwdev)) {
2974 if (hwdev->coherent_dma_mask < DMA_BIT_MASK(32))
2975 flags |= GFP_DMA;
2976 else
2977 flags |= GFP_DMA32;
2978 }
ba395927
KA		 2979
2980 vaddr = (void *)__get_free_pages(flags, order);
2981 if (!vaddr)
2982 return NULL;
2983 memset(vaddr, 0, size);
2984
bb9e6d65
FT		 2985 *dma_handle = __intel_map_single(hwdev, virt_to_bus(vaddr), size,
2986 DMA_BIDIRECTIONAL,
2987 hwdev->coherent_dma_mask);
ba395927
KA		 2988 if (*dma_handle)
2989 return vaddr;
2990 free_pages((unsigned long)vaddr, order);
2991 return NULL;
2992}
2993
d7ab5c46 2994static void intel_free_coherent(struct device *hwdev, size_t size, void *vaddr,
baa676fc 2995 dma_addr_t dma_handle, struct dma_attrs *attrs)
ba395927
KA		 2996{
2997 int order;
2998
5b6985ce 2999 size = PAGE_ALIGN(size);
ba395927
KA		 3000 order = get_order(size);
3001
0db9b7ae 3002 intel_unmap_page(hwdev, dma_handle, size, DMA_BIDIRECTIONAL, NULL);
ba395927
KA		 3003 free_pages((unsigned long)vaddr, order);
3004}
3005
d7ab5c46
FT		 3006static void intel_unmap_sg(struct device *hwdev, struct scatterlist *sglist,
3007 int nelems, enum dma_data_direction dir,
3008 struct dma_attrs *attrs)
ba395927 3009{
ba395927
KA		 3010 struct pci_dev *pdev = to_pci_dev(hwdev);
3011 struct dmar_domain *domain;
d794dc9b 3012 unsigned long start_pfn, last_pfn;
f76aec76 3013 struct iova *iova;
8c11e798 3014 struct intel_iommu *iommu;
ba395927 3015
73676832 3016 if (iommu_no_mapping(hwdev))
ba395927
KA		 3017 return;
3018
3019 domain = find_domain(pdev);
8c11e798
WH		 3020 BUG_ON(!domain);
3021
3022 iommu = domain_get_iommu(domain);
ba395927 3023
c03ab37c 3024 iova = find_iova(&domain->iovad, IOVA_PFN(sglist[0].dma_address));
85b98276
DW		 3025 if (WARN_ONCE(!iova, "Driver unmaps unmatched sglist at PFN %llx\n",
3026 (unsigned long long)sglist[0].dma_address))
f76aec76 3027 return;
f76aec76 3028
d794dc9b
DW		 3029 start_pfn = mm_to_dma_pfn(iova->pfn_lo);
3030 last_pfn = mm_to_dma_pfn(iova->pfn_hi + 1) - 1;
f76aec76
KA		 3031
3032 /* clear the whole page */
d794dc9b
DW		 3033 dma_pte_clear_range(domain, start_pfn, last_pfn);
3034
f76aec76 3035 /* free page tables */
d794dc9b 3036 dma_pte_free_pagetable(domain, start_pfn, last_pfn);
f76aec76 3037
acea0018
DW		 3038 if (intel_iommu_strict) {
3039 iommu_flush_iotlb_psi(iommu, domain->id, start_pfn,
82653633 3040 last_pfn - start_pfn + 1, 0);
acea0018
DW		 3041 /* free iova */
3042 __free_iova(&domain->iovad, iova);
3043 } else {
3044 add_unmap(domain, iova);
3045 /*
3046 * queue up the release of the unmap to save the 1/6th of the
3047 * cpu used up by the iotlb flush operation...
3048 */
3049 }
ba395927
KA		 3050}
3051
ba395927 3052static int intel_nontranslate_map_sg(struct device *hddev,
c03ab37c 3053 struct scatterlist *sglist, int nelems, int dir)
ba395927
KA		 3054{
3055 int i;
c03ab37c 3056 struct scatterlist *sg;
ba395927 3057
c03ab37c 3058 for_each_sg(sglist, sg, nelems, i) {
12d4d40e 3059 BUG_ON(!sg_page(sg));
4cf2e75d 3060 sg->dma_address = page_to_phys(sg_page(sg)) + sg->offset;
c03ab37c 3061 sg->dma_length = sg->length;
ba395927
KA		 3062 }
3063 return nelems;
3064}
3065
d7ab5c46
FT		 3066static int intel_map_sg(struct device *hwdev, struct scatterlist *sglist, int nelems,
3067 enum dma_data_direction dir, struct dma_attrs *attrs)
ba395927 3068{
ba395927 3069 int i;
ba395927
KA		 3070 struct pci_dev *pdev = to_pci_dev(hwdev);
3071 struct dmar_domain *domain;
f76aec76
KA		 3072 size_t size = 0;
3073 int prot = 0;
f76aec76
KA		 3074 struct iova *iova = NULL;
3075 int ret;
c03ab37c 3076 struct scatterlist *sg;
b536d24d 3077 unsigned long start_vpfn;
8c11e798 3078 struct intel_iommu *iommu;
ba395927
KA		 3079
3080 BUG_ON(dir == DMA_NONE);
73676832 3081 if (iommu_no_mapping(hwdev))
c03ab37c 3082 return intel_nontranslate_map_sg(hwdev, sglist, nelems, dir);
ba395927 3083
f76aec76
KA		 3084 domain = get_valid_domain_for_dev(pdev);
3085 if (!domain)
3086 return 0;
3087
8c11e798
WH		 3088 iommu = domain_get_iommu(domain);
3089
b536d24d 3090 for_each_sg(sglist, sg, nelems, i)
88cb6a74 3091 size += aligned_nrpages(sg->offset, sg->length);
f76aec76 3092
5a5e02a6
DW		 3093 iova = intel_alloc_iova(hwdev, domain, dma_to_mm_pfn(size),
3094 pdev->dma_mask);
f76aec76 3095 if (!iova) {
c03ab37c 3096 sglist->dma_length = 0;
f76aec76
KA		 3097 return 0;
3098 }
3099
3100 /*
3101 * Check if DMAR supports zero-length reads on write only
3102 * mappings..
3103 */
3104 if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
8c11e798 3105 !cap_zlr(iommu->cap))
f76aec76
KA		 3106 prot |= DMA_PTE_READ;
3107 if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
3108 prot |= DMA_PTE_WRITE;
3109
b536d24d 3110 start_vpfn = mm_to_dma_pfn(iova->pfn_lo);
e1605495 3111
f532959b 3112 ret = domain_sg_mapping(domain, start_vpfn, sglist, size, prot);
e1605495
DW		 3113 if (unlikely(ret)) {
3114 /* clear the page */
3115 dma_pte_clear_range(domain, start_vpfn,
3116 start_vpfn + size - 1);
3117 /* free page tables */
3118 dma_pte_free_pagetable(domain, start_vpfn,
3119 start_vpfn + size - 1);
3120 /* free iova */
3121 __free_iova(&domain->iovad, iova);
3122 return 0;
ba395927
KA		 3123 }
3124
1f0ef2aa
DW		 3125 /* it's a non-present to present mapping. Only flush if caching mode */
3126 if (cap_caching_mode(iommu->cap))
82653633 3127 iommu_flush_iotlb_psi(iommu, domain->id, start_vpfn, size, 1);
1f0ef2aa 3128 else
8c11e798 3129 iommu_flush_write_buffer(iommu);
1f0ef2aa 3130
ba395927
KA		 3131 return nelems;
3132}
3133
dfb805e8
FT		 3134static int intel_mapping_error(struct device *dev, dma_addr_t dma_addr)
3135{
3136 return !dma_addr;
3137}
3138
160c1d8e 3139struct dma_map_ops intel_dma_ops = {
baa676fc
AP		 3140 .alloc = intel_alloc_coherent,
3141 .free = intel_free_coherent,
ba395927
KA		 3142 .map_sg = intel_map_sg,
3143 .unmap_sg = intel_unmap_sg,
ffbbef5c
FT		 3144 .map_page = intel_map_page,
3145 .unmap_page = intel_unmap_page,
dfb805e8 3146 .mapping_error = intel_mapping_error,
ba395927
KA		 3147};
3148
3149static inline int iommu_domain_cache_init(void)
3150{
3151 int ret = 0;
3152
3153 iommu_domain_cache = kmem_cache_create("iommu_domain",
3154 sizeof(struct dmar_domain),
3155 0,
3156 SLAB_HWCACHE_ALIGN,
3157
3158 NULL);
3159 if (!iommu_domain_cache) {
3160 printk(KERN_ERR "Couldn't create iommu_domain cache\n");
3161 ret = -ENOMEM;
3162 }
3163
3164 return ret;
3165}
3166
3167static inline int iommu_devinfo_cache_init(void)
3168{
3169 int ret = 0;
3170
3171 iommu_devinfo_cache = kmem_cache_create("iommu_devinfo",
3172 sizeof(struct device_domain_info),
3173 0,
3174 SLAB_HWCACHE_ALIGN,
ba395927
KA		 3175 NULL);
3176 if (!iommu_devinfo_cache) {
3177 printk(KERN_ERR "Couldn't create devinfo cache\n");
3178 ret = -ENOMEM;
3179 }
3180
3181 return ret;
3182}
3183
3184static inline int iommu_iova_cache_init(void)
3185{
3186 int ret = 0;
3187
3188 iommu_iova_cache = kmem_cache_create("iommu_iova",
3189 sizeof(struct iova),
3190 0,
3191 SLAB_HWCACHE_ALIGN,
ba395927
KA		 3192 NULL);
3193 if (!iommu_iova_cache) {
3194 printk(KERN_ERR "Couldn't create iova cache\n");
3195 ret = -ENOMEM;
3196 }
3197
3198 return ret;
3199}
3200
3201static int __init iommu_init_mempool(void)
3202{
3203 int ret;
3204 ret = iommu_iova_cache_init();
3205 if (ret)
3206 return ret;
3207
3208 ret = iommu_domain_cache_init();
3209 if (ret)
3210 goto domain_error;
3211
3212 ret = iommu_devinfo_cache_init();
3213 if (!ret)
3214 return ret;
3215
3216 kmem_cache_destroy(iommu_domain_cache);
3217domain_error:
3218 kmem_cache_destroy(iommu_iova_cache);
3219
3220 return -ENOMEM;
3221}
3222
3223static void __init iommu_exit_mempool(void)
3224{
3225 kmem_cache_destroy(iommu_devinfo_cache);
3226 kmem_cache_destroy(iommu_domain_cache);
3227 kmem_cache_destroy(iommu_iova_cache);
3228
3229}
3230
556ab45f
DW		 3231static void quirk_ioat_snb_local_iommu(struct pci_dev *pdev)
3232{
3233 struct dmar_drhd_unit *drhd;
3234 u32 vtbar;
3235 int rc;
3236
3237 /* We know that this device on this chipset has its own IOMMU.
3238 * If we find it under a different IOMMU, then the BIOS is lying
3239 * to us. Hope that the IOMMU for this device is actually
3240 * disabled, and it needs no translation...
3241 */
3242 rc = pci_bus_read_config_dword(pdev->bus, PCI_DEVFN(0, 0), 0xb0, &vtbar);
3243 if (rc) {
3244 /* "can't" happen */
3245 dev_info(&pdev->dev, "failed to run vt-d quirk\n");
3246 return;
3247 }
3248 vtbar &= 0xffff0000;
3249
3250 /* we know that the this iommu should be at offset 0xa000 from vtbar */
3251 drhd = dmar_find_matched_drhd_unit(pdev);
3252 if (WARN_TAINT_ONCE(!drhd || drhd->reg_base_addr - vtbar != 0xa000,
3253 TAINT_FIRMWARE_WORKAROUND,
3254 "BIOS assigned incorrect VT-d unit for Intel(R) QuickData Technology device\n"))
3255 pdev->dev.archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO;
3256}
3257DECLARE_PCI_FIXUP_ENABLE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB, quirk_ioat_snb_local_iommu);
3258
ba395927
KA		 3259static void __init init_no_remapping_devices(void)
3260{
3261 struct dmar_drhd_unit *drhd;
3262
3263 for_each_drhd_unit(drhd) {
3264 if (!drhd->include_all) {
3265 int i;
3266 for (i = 0; i < drhd->devices_cnt; i++)
3267 if (drhd->devices[i] != NULL)
3268 break;
3269 /* ignore DMAR unit if no pci devices exist */
3270 if (i == drhd->devices_cnt)
3271 drhd->ignored = 1;
3272 }
3273 }
3274
7c919779 3275 for_each_active_drhd_unit(drhd) {
ba395927 3276 int i;
7c919779 3277 if (drhd->include_all)
ba395927
KA		 3278 continue;
3279
3280 for (i = 0; i < drhd->devices_cnt; i++)
3281 if (drhd->devices[i] &&
c0771df8 3282 !IS_GFX_DEVICE(drhd->devices[i]))
ba395927
KA		 3283 break;
3284
3285 if (i < drhd->devices_cnt)
3286 continue;
3287
c0771df8
DW		 3288 /* This IOMMU has *only* gfx devices. Either bypass it or
3289 set the gfx_mapped flag, as appropriate */
3290 if (dmar_map_gfx) {
3291 intel_iommu_gfx_mapped = 1;
3292 } else {
3293 drhd->ignored = 1;
3294 for (i = 0; i < drhd->devices_cnt; i++) {
3295 if (!drhd->devices[i])
3296 continue;
3297 drhd->devices[i]->dev.archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO;
3298 }
ba395927
KA		 3299 }
3300 }
3301}
3302
f59c7b69
FY		 3303#ifdef CONFIG_SUSPEND
3304static int init_iommu_hw(void)
3305{
3306 struct dmar_drhd_unit *drhd;
3307 struct intel_iommu *iommu = NULL;
3308
3309 for_each_active_iommu(iommu, drhd)
3310 if (iommu->qi)
3311 dmar_reenable_qi(iommu);
3312
b779260b
JC		 3313 for_each_iommu(iommu, drhd) {
3314 if (drhd->ignored) {
3315 /*
3316 * we always have to disable PMRs or DMA may fail on
3317 * this device
3318 */
3319 if (force_on)
3320 iommu_disable_protect_mem_regions(iommu);
3321 continue;
3322 }
3323
f59c7b69
FY		 3324 iommu_flush_write_buffer(iommu);
3325
3326 iommu_set_root_entry(iommu);
3327
3328 iommu->flush.flush_context(iommu, 0, 0, 0,
1f0ef2aa 3329 DMA_CCMD_GLOBAL_INVL);
f59c7b69 3330 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
1f0ef2aa 3331 DMA_TLB_GLOBAL_FLUSH);
b779260b
JC		 3332 if (iommu_enable_translation(iommu))
3333 return 1;
b94996c9 3334 iommu_disable_protect_mem_regions(iommu);
f59c7b69
FY		 3335 }
3336
3337 return 0;
3338}
3339
3340static void iommu_flush_all(void)
3341{
3342 struct dmar_drhd_unit *drhd;
3343 struct intel_iommu *iommu;
3344
3345 for_each_active_iommu(iommu, drhd) {
3346 iommu->flush.flush_context(iommu, 0, 0, 0,
1f0ef2aa 3347 DMA_CCMD_GLOBAL_INVL);
f59c7b69 3348 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
1f0ef2aa 3349 DMA_TLB_GLOBAL_FLUSH);
f59c7b69
FY		 3350 }
3351}
3352
134fac3f 3353static int iommu_suspend(void)
f59c7b69
FY		 3354{
3355 struct dmar_drhd_unit *drhd;
3356 struct intel_iommu *iommu = NULL;
3357 unsigned long flag;
3358
3359 for_each_active_iommu(iommu, drhd) {
3360 iommu->iommu_state = kzalloc(sizeof(u32) * MAX_SR_DMAR_REGS,
3361 GFP_ATOMIC);
3362 if (!iommu->iommu_state)
3363 goto nomem;
3364 }
3365
3366 iommu_flush_all();
3367
3368 for_each_active_iommu(iommu, drhd) {
3369 iommu_disable_translation(iommu);
3370
1f5b3c3f 3371 raw_spin_lock_irqsave(&iommu->register_lock, flag);
f59c7b69
FY		 3372
3373 iommu->iommu_state[SR_DMAR_FECTL_REG] =
3374 readl(iommu->reg + DMAR_FECTL_REG);
3375 iommu->iommu_state[SR_DMAR_FEDATA_REG] =
3376 readl(iommu->reg + DMAR_FEDATA_REG);
3377 iommu->iommu_state[SR_DMAR_FEADDR_REG] =
3378 readl(iommu->reg + DMAR_FEADDR_REG);
3379 iommu->iommu_state[SR_DMAR_FEUADDR_REG] =
3380 readl(iommu->reg + DMAR_FEUADDR_REG);
3381
1f5b3c3f 3382 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
f59c7b69
FY		 3383 }
3384 return 0;
3385
3386nomem:
3387 for_each_active_iommu(iommu, drhd)
3388 kfree(iommu->iommu_state);
3389
3390 return -ENOMEM;
3391}
3392
134fac3f 3393static void iommu_resume(void)
f59c7b69
FY		 3394{
3395 struct dmar_drhd_unit *drhd;
3396 struct intel_iommu *iommu = NULL;
3397 unsigned long flag;
3398
3399 if (init_iommu_hw()) {
b779260b
JC		 3400 if (force_on)
3401 panic("tboot: IOMMU setup failed, DMAR can not resume!\n");
3402 else
3403 WARN(1, "IOMMU setup failed, DMAR can not resume!\n");
134fac3f 3404 return;
f59c7b69
FY		 3405 }
3406
3407 for_each_active_iommu(iommu, drhd) {
3408
1f5b3c3f 3409 raw_spin_lock_irqsave(&iommu->register_lock, flag);
f59c7b69
FY		 3410
3411 writel(iommu->iommu_state[SR_DMAR_FECTL_REG],
3412 iommu->reg + DMAR_FECTL_REG);
3413 writel(iommu->iommu_state[SR_DMAR_FEDATA_REG],
3414 iommu->reg + DMAR_FEDATA_REG);
3415 writel(iommu->iommu_state[SR_DMAR_FEADDR_REG],
3416 iommu->reg + DMAR_FEADDR_REG);
3417 writel(iommu->iommu_state[SR_DMAR_FEUADDR_REG],
3418 iommu->reg + DMAR_FEUADDR_REG);
3419
1f5b3c3f 3420 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
f59c7b69
FY		 3421 }
3422
3423 for_each_active_iommu(iommu, drhd)
3424 kfree(iommu->iommu_state);
f59c7b69
FY		 3425}
3426
134fac3f 3427static struct syscore_ops iommu_syscore_ops = {
f59c7b69
FY		 3428 .resume = iommu_resume,
3429 .suspend = iommu_suspend,
3430};
3431
134fac3f 3432static void __init init_iommu_pm_ops(void)
f59c7b69 3433{
134fac3f 3434 register_syscore_ops(&iommu_syscore_ops);
f59c7b69
FY		 3435}
3436
3437#else
99592ba4 3438static inline void init_iommu_pm_ops(void) {}
f59c7b69
FY		 3439#endif /* CONFIG_PM */
3440
318fe7df
SS		 3441LIST_HEAD(dmar_rmrr_units);
3442
3443static void __init dmar_register_rmrr_unit(struct dmar_rmrr_unit *rmrr)
3444{
3445 list_add(&rmrr->list, &dmar_rmrr_units);
3446}
3447
3448
3449int __init dmar_parse_one_rmrr(struct acpi_dmar_header *header)
3450{
3451 struct acpi_dmar_reserved_memory *rmrr;
3452 struct dmar_rmrr_unit *rmrru;
3453
3454 rmrru = kzalloc(sizeof(*rmrru), GFP_KERNEL);
3455 if (!rmrru)
3456 return -ENOMEM;
3457
3458 rmrru->hdr = header;
3459 rmrr = (struct acpi_dmar_reserved_memory *)header;
3460 rmrru->base_address = rmrr->base_address;
3461 rmrru->end_address = rmrr->end_address;
3462
3463 dmar_register_rmrr_unit(rmrru);
3464 return 0;
3465}
3466
3467static int __init
3468rmrr_parse_dev(struct dmar_rmrr_unit *rmrru)
3469{
3470 struct acpi_dmar_reserved_memory *rmrr;
318fe7df
SS		 3471
3472 rmrr = (struct acpi_dmar_reserved_memory *) rmrru->hdr;
9bdc531e
JL		 3473 return dmar_parse_dev_scope((void *)(rmrr + 1),
3474 ((void *)rmrr) + rmrr->header.length,
3475 &rmrru->devices_cnt, &rmrru->devices,
3476 rmrr->segment);
318fe7df
SS		 3477}
3478
3479static LIST_HEAD(dmar_atsr_units);
3480
3481int __init dmar_parse_one_atsr(struct acpi_dmar_header *hdr)
3482{
3483 struct acpi_dmar_atsr *atsr;
3484 struct dmar_atsr_unit *atsru;
3485
3486 atsr = container_of(hdr, struct acpi_dmar_atsr, header);
3487 atsru = kzalloc(sizeof(*atsru), GFP_KERNEL);
3488 if (!atsru)
3489 return -ENOMEM;
3490
3491 atsru->hdr = hdr;
3492 atsru->include_all = atsr->flags & 0x1;
3493
3494 list_add(&atsru->list, &dmar_atsr_units);
3495
3496 return 0;
3497}
3498
3499static int __init atsr_parse_dev(struct dmar_atsr_unit *atsru)
3500{
318fe7df
SS		 3501 struct acpi_dmar_atsr *atsr;
3502
3503 if (atsru->include_all)
3504 return 0;
3505
3506 atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header);
9bdc531e
JL		 3507 return dmar_parse_dev_scope((void *)(atsr + 1),
3508 (void *)atsr + atsr->header.length,
3509 &atsru->devices_cnt, &atsru->devices,
3510 atsr->segment);
3511}
3512
3513static void intel_iommu_free_atsr(struct dmar_atsr_unit *atsru)
3514{
3515 dmar_free_dev_scope(&atsru->devices, &atsru->devices_cnt);
3516 kfree(atsru);
3517}
3518
3519static void intel_iommu_free_dmars(void)
3520{
3521 struct dmar_rmrr_unit *rmrru, *rmrr_n;
3522 struct dmar_atsr_unit *atsru, *atsr_n;
3523
3524 list_for_each_entry_safe(rmrru, rmrr_n, &dmar_rmrr_units, list) {
3525 list_del(&rmrru->list);
3526 dmar_free_dev_scope(&rmrru->devices, &rmrru->devices_cnt);
3527 kfree(rmrru);
318fe7df
SS		 3528 }
3529
9bdc531e
JL		 3530 list_for_each_entry_safe(atsru, atsr_n, &dmar_atsr_units, list) {
3531 list_del(&atsru->list);
3532 intel_iommu_free_atsr(atsru);
3533 }
318fe7df
SS		 3534}
3535
3536int dmar_find_matched_atsr_unit(struct pci_dev *dev)
3537{
3538 int i;
3539 struct pci_bus *bus;
3540 struct acpi_dmar_atsr *atsr;
3541 struct dmar_atsr_unit *atsru;
3542
3543 dev = pci_physfn(dev);
3544
3545 list_for_each_entry(atsru, &dmar_atsr_units, list) {
3546 atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header);
3547 if (atsr->segment == pci_domain_nr(dev->bus))
3548 goto found;
3549 }
3550
3551 return 0;
3552
3553found:
3554 for (bus = dev->bus; bus; bus = bus->parent) {
3555 struct pci_dev *bridge = bus->self;
3556
3557 if (!bridge || !pci_is_pcie(bridge) ||
62f87c0e 3558 pci_pcie_type(bridge) == PCI_EXP_TYPE_PCI_BRIDGE)
318fe7df
SS		 3559 return 0;
3560
62f87c0e 3561 if (pci_pcie_type(bridge) == PCI_EXP_TYPE_ROOT_PORT) {
318fe7df
SS		 3562 for (i = 0; i < atsru->devices_cnt; i++)
3563 if (atsru->devices[i] == bridge)
3564 return 1;
3565 break;
3566 }
3567 }
3568
3569 if (atsru->include_all)
3570 return 1;
3571
3572 return 0;
3573}
3574
c8f369ab 3575int __init dmar_parse_rmrr_atsr_dev(void)
318fe7df 3576{
9bdc531e
JL		 3577 struct dmar_rmrr_unit *rmrr;
3578 struct dmar_atsr_unit *atsr;
318fe7df
SS		 3579 int ret = 0;
3580
9bdc531e 3581 list_for_each_entry(rmrr, &dmar_rmrr_units, list) {
318fe7df
SS		 3582 ret = rmrr_parse_dev(rmrr);
3583 if (ret)
3584 return ret;
3585 }
3586
9bdc531e 3587 list_for_each_entry(atsr, &dmar_atsr_units, list) {
318fe7df
SS		 3588 ret = atsr_parse_dev(atsr);
3589 if (ret)
3590 return ret;
3591 }
3592
3593 return ret;
3594}
3595
99dcaded
FY		 3596/*
3597 * Here we only respond to action of unbound device from driver.
3598 *
3599 * Added device is not attached to its DMAR domain here yet. That will happen
3600 * when mapping the device to iova.
3601 */
3602static int device_notifier(struct notifier_block *nb,
3603 unsigned long action, void *data)
3604{
3605 struct device *dev = data;
3606 struct pci_dev *pdev = to_pci_dev(dev);
3607 struct dmar_domain *domain;
3608
44cd613c
DW		 3609 if (iommu_no_mapping(dev))
3610 return 0;
3611
99dcaded
FY		 3612 domain = find_domain(pdev);
3613 if (!domain)
3614 return 0;
3615
a97590e5 3616 if (action == BUS_NOTIFY_UNBOUND_DRIVER && !iommu_pass_through) {
99dcaded
FY		 3617 domain_remove_one_dev_info(domain, pdev);
3618
a97590e5
AW		 3619 if (!(domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE) &&
3620 !(domain->flags & DOMAIN_FLAG_STATIC_IDENTITY) &&
3621 list_empty(&domain->devices))
3622 domain_exit(domain);
3623 }
3624
99dcaded
FY		 3625 return 0;
3626}
3627
3628static struct notifier_block device_nb = {
3629 .notifier_call = device_notifier,
3630};
3631
ba395927
KA		 3632int __init intel_iommu_init(void)
3633{
9bdc531e 3634 int ret = -ENODEV;
3a93c841 3635 struct dmar_drhd_unit *drhd;
7c919779 3636 struct intel_iommu *iommu;
ba395927 3637
a59b50e9
JC		 3638 /* VT-d is required for a TXT/tboot launch, so enforce that */
3639 force_on = tboot_force_iommu();
3640
3641 if (dmar_table_init()) {
3642 if (force_on)
3643 panic("tboot: Failed to initialize DMAR table\n");
9bdc531e 3644 goto out_free_dmar;
a59b50e9 3645 }
ba395927 3646
3a93c841
TI		 3647 /*
3648 * Disable translation if already enabled prior to OS handover.
3649 */
7c919779 3650 for_each_active_iommu(iommu, drhd)
3a93c841
TI		 3651 if (iommu->gcmd & DMA_GCMD_TE)
3652 iommu_disable_translation(iommu);
3a93c841 3653
c2c7286a 3654 if (dmar_dev_scope_init() < 0) {
a59b50e9
JC		 3655 if (force_on)
3656 panic("tboot: Failed to initialize DMAR device scope\n");
9bdc531e 3657 goto out_free_dmar;
a59b50e9 3658 }
1886e8a9 3659
75f1cdf1 3660 if (no_iommu || dmar_disabled)
9bdc531e 3661 goto out_free_dmar;
2ae21010 3662
51a63e67
JC		 3663 if (iommu_init_mempool()) {
3664 if (force_on)
3665 panic("tboot: Failed to initialize iommu memory\n");
9bdc531e 3666 goto out_free_dmar;
51a63e67
JC		 3667 }
3668
318fe7df
SS		 3669 if (list_empty(&dmar_rmrr_units))
3670 printk(KERN_INFO "DMAR: No RMRR found\n");
3671
3672 if (list_empty(&dmar_atsr_units))
3673 printk(KERN_INFO "DMAR: No ATSR found\n");
3674
51a63e67
JC		 3675 if (dmar_init_reserved_ranges()) {
3676 if (force_on)
3677 panic("tboot: Failed to reserve iommu ranges\n");
9bdc531e 3678 goto out_free_mempool;
51a63e67 3679 }
ba395927
KA		 3680
3681 init_no_remapping_devices();
3682
b779260b 3683 ret = init_dmars();
ba395927 3684 if (ret) {
a59b50e9
JC		 3685 if (force_on)
3686 panic("tboot: Failed to initialize DMARs\n");
ba395927 3687 printk(KERN_ERR "IOMMU: dmar init failed\n");
9bdc531e 3688 goto out_free_reserved_range;
ba395927
KA		 3689 }
3690 printk(KERN_INFO
3691 "PCI-DMA: Intel(R) Virtualization Technology for Directed I/O\n");
3692
5e0d2a6f 3693 init_timer(&unmap_timer);
75f1cdf1
FT		 3694#ifdef CONFIG_SWIOTLB
3695 swiotlb = 0;
3696#endif
19943b0e 3697 dma_ops = &intel_dma_ops;
4ed0d3e6 3698
134fac3f 3699 init_iommu_pm_ops();
a8bcbb0d 3700
4236d97d 3701 bus_set_iommu(&pci_bus_type, &intel_iommu_ops);
a8bcbb0d 3702
99dcaded
FY		 3703 bus_register_notifier(&pci_bus_type, &device_nb);
3704
8bc1f85c
ED		 3705 intel_iommu_enabled = 1;
3706
ba395927 3707 return 0;
9bdc531e
JL		 3708
3709out_free_reserved_range:
3710 put_iova_domain(&reserved_iova_list);
3711out_free_mempool:
3712 iommu_exit_mempool();
3713out_free_dmar:
3714 intel_iommu_free_dmars();
3715 return ret;
ba395927 3716}
e820482c 3717
3199aa6b
HW		 3718static void iommu_detach_dependent_devices(struct intel_iommu *iommu,
3719 struct pci_dev *pdev)
3720{
3721 struct pci_dev *tmp, *parent;
3722
3723 if (!iommu || !pdev)
3724 return;
3725
3726 /* dependent device detach */
3727 tmp = pci_find_upstream_pcie_bridge(pdev);
3728 /* Secondary interface's bus number and devfn 0 */
3729 if (tmp) {
3730 parent = pdev->bus->self;
3731 while (parent != tmp) {
3732 iommu_detach_dev(iommu, parent->bus->number,
276dbf99 3733 parent->devfn);
3199aa6b
HW		 3734 parent = parent->bus->self;
3735 }
45e829ea 3736 if (pci_is_pcie(tmp)) /* this is a PCIe-to-PCI bridge */
3199aa6b
HW		 3737 iommu_detach_dev(iommu,
3738 tmp->subordinate->number, 0);
3739 else /* this is a legacy PCI bridge */
276dbf99
DW		 3740 iommu_detach_dev(iommu, tmp->bus->number,
3741 tmp->devfn);
3199aa6b
HW		 3742 }
3743}
3744
2c2e2c38 3745static void domain_remove_one_dev_info(struct dmar_domain *domain,
c7151a8d
WH		 3746 struct pci_dev *pdev)
3747{
bca2b916 3748 struct device_domain_info *info, *tmp;
c7151a8d
WH		 3749 struct intel_iommu *iommu;
3750 unsigned long flags;
3751 int found = 0;
c7151a8d 3752
276dbf99
DW		 3753 iommu = device_to_iommu(pci_domain_nr(pdev->bus), pdev->bus->number,
3754 pdev->devfn);
c7151a8d
WH		 3755 if (!iommu)
3756 return;
3757
3758 spin_lock_irqsave(&device_domain_lock, flags);
bca2b916 3759 list_for_each_entry_safe(info, tmp, &domain->devices, link) {
8519dc44
MH		 3760 if (info->segment == pci_domain_nr(pdev->bus) &&
3761 info->bus == pdev->bus->number &&
c7151a8d 3762 info->devfn == pdev->devfn) {
109b9b04 3763 unlink_domain_info(info);
c7151a8d
WH		 3764 spin_unlock_irqrestore(&device_domain_lock, flags);
3765
93a23a72 3766 iommu_disable_dev_iotlb(info);
c7151a8d 3767 iommu_detach_dev(iommu, info->bus, info->devfn);
3199aa6b 3768 iommu_detach_dependent_devices(iommu, pdev);
c7151a8d
WH		 3769 free_devinfo_mem(info);
3770
3771 spin_lock_irqsave(&device_domain_lock, flags);
3772
3773 if (found)
3774 break;
3775 else
3776 continue;
3777 }
3778
3779 /* if there is no other devices under the same iommu
3780 * owned by this domain, clear this iommu in iommu_bmp
3781 * update iommu count and coherency
3782 */
276dbf99
DW		 3783 if (iommu == device_to_iommu(info->segment, info->bus,
3784 info->devfn))
c7151a8d
WH		 3785 found = 1;
3786 }
3787
3e7abe25
RD		 3788 spin_unlock_irqrestore(&device_domain_lock, flags);
3789
c7151a8d
WH		 3790 if (found == 0) {
3791 unsigned long tmp_flags;
3792 spin_lock_irqsave(&domain->iommu_lock, tmp_flags);
1b198bb0 3793 clear_bit(iommu->seq_id, domain->iommu_bmp);
c7151a8d 3794 domain->iommu_count--;
58c610bd 3795 domain_update_iommu_cap(domain);
c7151a8d 3796 spin_unlock_irqrestore(&domain->iommu_lock, tmp_flags);
a97590e5 3797
9b4554b2
AW		 3798 if (!(domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE) &&
3799 !(domain->flags & DOMAIN_FLAG_STATIC_IDENTITY)) {
3800 spin_lock_irqsave(&iommu->lock, tmp_flags);
3801 clear_bit(domain->id, iommu->domain_ids);
3802 iommu->domains[domain->id] = NULL;
3803 spin_unlock_irqrestore(&iommu->lock, tmp_flags);
3804 }
c7151a8d 3805 }
c7151a8d
WH		 3806}
3807
3808static void vm_domain_remove_all_dev_info(struct dmar_domain *domain)
3809{
3810 struct device_domain_info *info;
3811 struct intel_iommu *iommu;
3812 unsigned long flags1, flags2;
3813
3814 spin_lock_irqsave(&device_domain_lock, flags1);
3815 while (!list_empty(&domain->devices)) {
3816 info = list_entry(domain->devices.next,
3817 struct device_domain_info, link);
109b9b04 3818 unlink_domain_info(info);
c7151a8d
WH		 3819 spin_unlock_irqrestore(&device_domain_lock, flags1);
3820
93a23a72 3821 iommu_disable_dev_iotlb(info);
276dbf99 3822 iommu = device_to_iommu(info->segment, info->bus, info->devfn);
c7151a8d 3823 iommu_detach_dev(iommu, info->bus, info->devfn);
3199aa6b 3824 iommu_detach_dependent_devices(iommu, info->dev);
c7151a8d
WH		 3825
3826 /* clear this iommu in iommu_bmp, update iommu count
58c610bd 3827 * and capabilities
c7151a8d
WH		 3828 */
3829 spin_lock_irqsave(&domain->iommu_lock, flags2);
3830 if (test_and_clear_bit(iommu->seq_id,
1b198bb0 3831 domain->iommu_bmp)) {
c7151a8d 3832 domain->iommu_count--;
58c610bd 3833 domain_update_iommu_cap(domain);
c7151a8d
WH		 3834 }
3835 spin_unlock_irqrestore(&domain->iommu_lock, flags2);
3836
3837 free_devinfo_mem(info);
3838 spin_lock_irqsave(&device_domain_lock, flags1);
3839 }
3840 spin_unlock_irqrestore(&device_domain_lock, flags1);
3841}
3842
5e98c4b1 3843/* domain id for virtual machine, it won't be set in context */
18d99165 3844static atomic_t vm_domid = ATOMIC_INIT(0);
5e98c4b1
WH		 3845
3846static struct dmar_domain *iommu_alloc_vm_domain(void)
3847{
3848 struct dmar_domain *domain;
3849
3850 domain = alloc_domain_mem();
3851 if (!domain)
3852 return NULL;
3853
18d99165 3854 domain->id = atomic_inc_return(&vm_domid);
4c923d47 3855 domain->nid = -1;
1b198bb0 3856 memset(domain->iommu_bmp, 0, sizeof(domain->iommu_bmp));
5e98c4b1
WH		 3857 domain->flags = DOMAIN_FLAG_VIRTUAL_MACHINE;
3858
3859 return domain;
3860}
3861
2c2e2c38 3862static int md_domain_init(struct dmar_domain *domain, int guest_width)
5e98c4b1
WH		 3863{
3864 int adjust_width;
3865
3866 init_iova_domain(&domain->iovad, DMA_32BIT_PFN);
5e98c4b1
WH		 3867 spin_lock_init(&domain->iommu_lock);
3868
3869 domain_reserve_special_ranges(domain);
3870
3871 /* calculate AGAW */
3872 domain->gaw = guest_width;
3873 adjust_width = guestwidth_to_adjustwidth(guest_width);
3874 domain->agaw = width_to_agaw(adjust_width);
3875
3876 INIT_LIST_HEAD(&domain->devices);
3877
3878 domain->iommu_count = 0;
3879 domain->iommu_coherency = 0;
c5b15255 3880 domain->iommu_snooping = 0;
6dd9a7c7 3881 domain->iommu_superpage = 0;
fe40f1e0 3882 domain->max_addr = 0;
4c923d47 3883 domain->nid = -1;
5e98c4b1
WH		 3884
3885 /* always allocate the top pgd */
4c923d47 3886 domain->pgd = (struct dma_pte *)alloc_pgtable_page(domain->nid);
5e98c4b1
WH		 3887 if (!domain->pgd)
3888 return -ENOMEM;
3889 domain_flush_cache(domain, domain->pgd, PAGE_SIZE);
3890 return 0;
3891}
3892
3893static void iommu_free_vm_domain(struct dmar_domain *domain)
3894{
3895 unsigned long flags;
3896 struct dmar_drhd_unit *drhd;
3897 struct intel_iommu *iommu;
3898 unsigned long i;
3899 unsigned long ndomains;
3900
7c919779 3901 for_each_active_iommu(iommu, drhd) {
5e98c4b1 3902 ndomains = cap_ndoms(iommu->cap);
a45946ab 3903 for_each_set_bit(i, iommu->domain_ids, ndomains) {
5e98c4b1
WH		 3904 if (iommu->domains[i] == domain) {
3905 spin_lock_irqsave(&iommu->lock, flags);
3906 clear_bit(i, iommu->domain_ids);
3907 iommu->domains[i] = NULL;
3908 spin_unlock_irqrestore(&iommu->lock, flags);
3909 break;
3910 }
5e98c4b1
WH		 3911 }
3912 }
3913}
3914
3915static void vm_domain_exit(struct dmar_domain *domain)
3916{
5e98c4b1
WH		 3917 /* Domain 0 is reserved, so dont process it */
3918 if (!domain)
3919 return;
3920
3921 vm_domain_remove_all_dev_info(domain);
3922 /* destroy iovas */
3923 put_iova_domain(&domain->iovad);
5e98c4b1
WH		 3924
3925 /* clear ptes */
595badf5 3926 dma_pte_clear_range(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
5e98c4b1
WH		 3927
3928 /* free page tables */
d794dc9b 3929 dma_pte_free_pagetable(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
5e98c4b1
WH		 3930
3931 iommu_free_vm_domain(domain);
3932 free_domain_mem(domain);
3933}
3934
5d450806 3935static int intel_iommu_domain_init(struct iommu_domain *domain)
38717946 3936{
5d450806 3937 struct dmar_domain *dmar_domain;
38717946 3938
5d450806
JR		 3939 dmar_domain = iommu_alloc_vm_domain();
3940 if (!dmar_domain) {
38717946 3941 printk(KERN_ERR
5d450806
JR		 3942 "intel_iommu_domain_init: dmar_domain == NULL\n");
3943 return -ENOMEM;
38717946 3944 }
2c2e2c38 3945 if (md_domain_init(dmar_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
38717946 3946 printk(KERN_ERR
5d450806
JR		 3947 "intel_iommu_domain_init() failed\n");
3948 vm_domain_exit(dmar_domain);
3949 return -ENOMEM;
38717946 3950 }
8140a95d 3951 domain_update_iommu_cap(dmar_domain);
5d450806 3952 domain->priv = dmar_domain;
faa3d6f5 3953
8a0e715b
JR		 3954 domain->geometry.aperture_start = 0;
3955 domain->geometry.aperture_end = __DOMAIN_MAX_ADDR(dmar_domain->gaw);
3956 domain->geometry.force_aperture = true;
3957
5d450806 3958 return 0;
38717946 3959}
38717946 3960
5d450806 3961static void intel_iommu_domain_destroy(struct iommu_domain *domain)
38717946 3962{
5d450806
JR		 3963 struct dmar_domain *dmar_domain = domain->priv;
3964
3965 domain->priv = NULL;
3966 vm_domain_exit(dmar_domain);
38717946 3967}
38717946 3968
4c5478c9
JR		 3969static int intel_iommu_attach_device(struct iommu_domain *domain,
3970 struct device *dev)
38717946 3971{
4c5478c9
JR		 3972 struct dmar_domain *dmar_domain = domain->priv;
3973 struct pci_dev *pdev = to_pci_dev(dev);
fe40f1e0
WH		 3974 struct intel_iommu *iommu;
3975 int addr_width;
faa3d6f5
WH		 3976
3977 /* normally pdev is not mapped */
3978 if (unlikely(domain_context_mapped(pdev))) {
3979 struct dmar_domain *old_domain;
3980
3981 old_domain = find_domain(pdev);
3982 if (old_domain) {
2c2e2c38
FY		 3983 if (dmar_domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE ||
3984 dmar_domain->flags & DOMAIN_FLAG_STATIC_IDENTITY)
3985 domain_remove_one_dev_info(old_domain, pdev);
faa3d6f5
WH		 3986 else
3987 domain_remove_dev_info(old_domain);
3988 }
3989 }
3990
276dbf99
DW		 3991 iommu = device_to_iommu(pci_domain_nr(pdev->bus), pdev->bus->number,
3992 pdev->devfn);
fe40f1e0
WH		 3993 if (!iommu)
3994 return -ENODEV;
3995
3996 /* check if this iommu agaw is sufficient for max mapped address */
3997 addr_width = agaw_to_width(iommu->agaw);
a99c47a2
TL		 3998 if (addr_width > cap_mgaw(iommu->cap))
3999 addr_width = cap_mgaw(iommu->cap);
4000
4001 if (dmar_domain->max_addr > (1LL << addr_width)) {
4002 printk(KERN_ERR "%s: iommu width (%d) is not "
fe40f1e0 4003 "sufficient for the mapped address (%llx)\n",
a99c47a2 4004 __func__, addr_width, dmar_domain->max_addr);
fe40f1e0
WH		 4005 return -EFAULT;
4006 }
a99c47a2
TL		 4007 dmar_domain->gaw = addr_width;
4008
4009 /*
4010 * Knock out extra levels of page tables if necessary
4011 */
4012 while (iommu->agaw < dmar_domain->agaw) {
4013 struct dma_pte *pte;
4014
4015 pte = dmar_domain->pgd;
4016 if (dma_pte_present(pte)) {
25cbff16
SY		 4017 dmar_domain->pgd = (struct dma_pte *)
4018 phys_to_virt(dma_pte_addr(pte));
7a661013 4019 free_pgtable_page(pte);
a99c47a2
TL		 4020 }
4021 dmar_domain->agaw--;
4022 }
fe40f1e0 4023
5fe60f4e 4024 return domain_add_dev_info(dmar_domain, pdev, CONTEXT_TT_MULTI_LEVEL);
38717946 4025}
38717946 4026
4c5478c9
JR		 4027static void intel_iommu_detach_device(struct iommu_domain *domain,
4028 struct device *dev)
38717946 4029{
4c5478c9
JR		 4030 struct dmar_domain *dmar_domain = domain->priv;
4031 struct pci_dev *pdev = to_pci_dev(dev);
4032
2c2e2c38 4033 domain_remove_one_dev_info(dmar_domain, pdev);
faa3d6f5 4034}
c7151a8d 4035
b146a1c9
JR		 4036static int intel_iommu_map(struct iommu_domain *domain,
4037 unsigned long iova, phys_addr_t hpa,
5009065d 4038 size_t size, int iommu_prot)
faa3d6f5 4039{
dde57a21 4040 struct dmar_domain *dmar_domain = domain->priv;
fe40f1e0 4041 u64 max_addr;
dde57a21 4042 int prot = 0;
faa3d6f5 4043 int ret;
fe40f1e0 4044
dde57a21
JR		 4045 if (iommu_prot & IOMMU_READ)
4046 prot |= DMA_PTE_READ;
4047 if (iommu_prot & IOMMU_WRITE)
4048 prot |= DMA_PTE_WRITE;
9cf06697
SY		 4049 if ((iommu_prot & IOMMU_CACHE) && dmar_domain->iommu_snooping)
4050 prot |= DMA_PTE_SNP;
dde57a21 4051
163cc52c 4052 max_addr = iova + size;
dde57a21 4053 if (dmar_domain->max_addr < max_addr) {
fe40f1e0
WH		 4054 u64 end;
4055
4056 /* check if minimum agaw is sufficient for mapped address */
8954da1f 4057 end = __DOMAIN_MAX_ADDR(dmar_domain->gaw) + 1;
fe40f1e0 4058 if (end < max_addr) {
8954da1f 4059 printk(KERN_ERR "%s: iommu width (%d) is not "
fe40f1e0 4060 "sufficient for the mapped address (%llx)\n",
8954da1f 4061 __func__, dmar_domain->gaw, max_addr);
fe40f1e0
WH		 4062 return -EFAULT;
4063 }
dde57a21 4064 dmar_domain->max_addr = max_addr;
fe40f1e0 4065 }
ad051221
DW		 4066 /* Round up size to next multiple of PAGE_SIZE, if it and
4067 the low bits of hpa would take us onto the next page */
88cb6a74 4068 size = aligned_nrpages(hpa, size);
ad051221
DW		 4069 ret = domain_pfn_mapping(dmar_domain, iova >> VTD_PAGE_SHIFT,
4070 hpa >> VTD_PAGE_SHIFT, size, prot);
faa3d6f5 4071 return ret;
38717946 4072}
38717946 4073
5009065d
OBC		 4074static size_t intel_iommu_unmap(struct iommu_domain *domain,
4075 unsigned long iova, size_t size)
38717946 4076{
dde57a21 4077 struct dmar_domain *dmar_domain = domain->priv;
292827cb 4078 int order;
4b99d352 4079
292827cb 4080 order = dma_pte_clear_range(dmar_domain, iova >> VTD_PAGE_SHIFT,
163cc52c 4081 (iova + size - 1) >> VTD_PAGE_SHIFT);
fe40f1e0 4082
163cc52c
DW		 4083 if (dmar_domain->max_addr == iova + size)
4084 dmar_domain->max_addr = iova;
b146a1c9 4085
5009065d 4086 return PAGE_SIZE << order;
38717946 4087}
38717946 4088
d14d6577 4089static phys_addr_t intel_iommu_iova_to_phys(struct iommu_domain *domain,
bb5547ac 4090 dma_addr_t iova)
38717946 4091{
d14d6577 4092 struct dmar_domain *dmar_domain = domain->priv;
38717946 4093 struct dma_pte *pte;
faa3d6f5 4094 u64 phys = 0;
38717946 4095
6dd9a7c7 4096 pte = pfn_to_dma_pte(dmar_domain, iova >> VTD_PAGE_SHIFT, 0);
38717946 4097 if (pte)
faa3d6f5 4098 phys = dma_pte_addr(pte);
38717946 4099
faa3d6f5 4100 return phys;
38717946 4101}
a8bcbb0d 4102
dbb9fd86
SY		 4103static int intel_iommu_domain_has_cap(struct iommu_domain *domain,
4104 unsigned long cap)
4105{
4106 struct dmar_domain *dmar_domain = domain->priv;
4107
4108 if (cap == IOMMU_CAP_CACHE_COHERENCY)
4109 return dmar_domain->iommu_snooping;
323f99cb 4110 if (cap == IOMMU_CAP_INTR_REMAP)
95a02e97 4111 return irq_remapping_enabled;
dbb9fd86
SY		 4112
4113 return 0;
4114}
4115
783f157b 4116#define REQ_ACS_FLAGS (PCI_ACS_SV | PCI_ACS_RR | PCI_ACS_CR | PCI_ACS_UF)
70ae6f0d 4117
abdfdde2
AW		 4118static int intel_iommu_add_device(struct device *dev)
4119{
4120 struct pci_dev *pdev = to_pci_dev(dev);
3da4af0a 4121 struct pci_dev *bridge, *dma_pdev = NULL;
abdfdde2
AW		 4122 struct iommu_group *group;
4123 int ret;
70ae6f0d 4124
abdfdde2
AW		 4125 if (!device_to_iommu(pci_domain_nr(pdev->bus),
4126 pdev->bus->number, pdev->devfn))
70ae6f0d
AW		 4127 return -ENODEV;
4128
4129 bridge = pci_find_upstream_pcie_bridge(pdev);
4130 if (bridge) {
abdfdde2
AW		 4131 if (pci_is_pcie(bridge))
4132 dma_pdev = pci_get_domain_bus_and_slot(
4133 pci_domain_nr(pdev->bus),
4134 bridge->subordinate->number, 0);
3da4af0a 4135 if (!dma_pdev)
abdfdde2
AW		 4136 dma_pdev = pci_dev_get(bridge);
4137 } else
4138 dma_pdev = pci_dev_get(pdev);
4139
a4ff1fc2 4140 /* Account for quirked devices */
783f157b
AW		 4141 swap_pci_ref(&dma_pdev, pci_get_dma_source(dma_pdev));
4142
a4ff1fc2
AW		 4143 /*
4144 * If it's a multifunction device that does not support our
c14d2690
AW		 4145 * required ACS flags, add to the same group as lowest numbered
4146 * function that also does not suport the required ACS flags.
a4ff1fc2 4147 */
783f157b 4148 if (dma_pdev->multifunction &&
c14d2690
AW		 4149 !pci_acs_enabled(dma_pdev, REQ_ACS_FLAGS)) {
4150 u8 i, slot = PCI_SLOT(dma_pdev->devfn);
4151
4152 for (i = 0; i < 8; i++) {
4153 struct pci_dev *tmp;
4154
4155 tmp = pci_get_slot(dma_pdev->bus, PCI_DEVFN(slot, i));
4156 if (!tmp)
4157 continue;
4158
4159 if (!pci_acs_enabled(tmp, REQ_ACS_FLAGS)) {
4160 swap_pci_ref(&dma_pdev, tmp);
4161 break;
4162 }
4163 pci_dev_put(tmp);
4164 }
4165 }
783f157b 4166
a4ff1fc2
AW		 4167 /*
4168 * Devices on the root bus go through the iommu. If that's not us,
4169 * find the next upstream device and test ACS up to the root bus.
4170 * Finding the next device may require skipping virtual buses.
4171 */
783f157b 4172 while (!pci_is_root_bus(dma_pdev->bus)) {
a4ff1fc2
AW		 4173 struct pci_bus *bus = dma_pdev->bus;
4174
4175 while (!bus->self) {
4176 if (!pci_is_root_bus(bus))
4177 bus = bus->parent;
4178 else
4179 goto root_bus;
4180 }
4181
4182 if (pci_acs_path_enabled(bus->self, NULL, REQ_ACS_FLAGS))
783f157b
AW		 4183 break;
4184
a4ff1fc2 4185 swap_pci_ref(&dma_pdev, pci_dev_get(bus->self));
783f157b
AW		 4186 }
4187
a4ff1fc2 4188root_bus:
abdfdde2
AW		 4189 group = iommu_group_get(&dma_pdev->dev);
4190 pci_dev_put(dma_pdev);
4191 if (!group) {
4192 group = iommu_group_alloc();
4193 if (IS_ERR(group))
4194 return PTR_ERR(group);
70ae6f0d
AW		 4195 }
4196
abdfdde2 4197 ret = iommu_group_add_device(group, dev);
bcb71abe 4198
abdfdde2
AW		 4199 iommu_group_put(group);
4200 return ret;
4201}
70ae6f0d 4202
abdfdde2
AW		 4203static void intel_iommu_remove_device(struct device *dev)
4204{
4205 iommu_group_remove_device(dev);
70ae6f0d
AW		 4206}
4207
a8bcbb0d
JR		 4208static struct iommu_ops intel_iommu_ops = {
4209 .domain_init = intel_iommu_domain_init,
4210 .domain_destroy = intel_iommu_domain_destroy,
4211 .attach_dev = intel_iommu_attach_device,
4212 .detach_dev = intel_iommu_detach_device,
b146a1c9
JR		 4213 .map = intel_iommu_map,
4214 .unmap = intel_iommu_unmap,
a8bcbb0d 4215 .iova_to_phys = intel_iommu_iova_to_phys,
dbb9fd86 4216 .domain_has_cap = intel_iommu_domain_has_cap,
abdfdde2
AW		 4217 .add_device = intel_iommu_add_device,
4218 .remove_device = intel_iommu_remove_device,
6d1c56a9 4219 .pgsize_bitmap = INTEL_IOMMU_PGSIZES,
a8bcbb0d 4220};
9af88143 4221
9452618e
DV		 4222static void quirk_iommu_g4x_gfx(struct pci_dev *dev)
4223{
4224 /* G4x/GM45 integrated gfx dmar support is totally busted. */
4225 printk(KERN_INFO "DMAR: Disabling IOMMU for graphics on this chipset\n");
4226 dmar_map_gfx = 0;
4227}
4228
4229DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2a40, quirk_iommu_g4x_gfx);
4230DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e00, quirk_iommu_g4x_gfx);
4231DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e10, quirk_iommu_g4x_gfx);
4232DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e20, quirk_iommu_g4x_gfx);
4233DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e30, quirk_iommu_g4x_gfx);
4234DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e40, quirk_iommu_g4x_gfx);
4235DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e90, quirk_iommu_g4x_gfx);
4236
d34d6517 4237static void quirk_iommu_rwbf(struct pci_dev *dev)
9af88143
DW		 4238{
4239 /*
4240 * Mobile 4 Series Chipset neglects to set RWBF capability,
210561ff 4241 * but needs it. Same seems to hold for the desktop versions.
9af88143
DW		 4242 */
4243 printk(KERN_INFO "DMAR: Forcing write-buffer flush capability\n");
4244 rwbf_quirk = 1;
4245}
4246
4247DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2a40, quirk_iommu_rwbf);
210561ff
DV		 4248DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e00, quirk_iommu_rwbf);
4249DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e10, quirk_iommu_rwbf);
4250DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e20, quirk_iommu_rwbf);
4251DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e30, quirk_iommu_rwbf);
4252DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e40, quirk_iommu_rwbf);
4253DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e90, quirk_iommu_rwbf);
e0fc7e0b 4254
eecfd57f
AJ		 4255#define GGC 0x52
4256#define GGC_MEMORY_SIZE_MASK (0xf << 8)
4257#define GGC_MEMORY_SIZE_NONE (0x0 << 8)
4258#define GGC_MEMORY_SIZE_1M (0x1 << 8)
4259#define GGC_MEMORY_SIZE_2M (0x3 << 8)
4260#define GGC_MEMORY_VT_ENABLED (0x8 << 8)
4261#define GGC_MEMORY_SIZE_2M_VT (0x9 << 8)
4262#define GGC_MEMORY_SIZE_3M_VT (0xa << 8)
4263#define GGC_MEMORY_SIZE_4M_VT (0xb << 8)
4264
d34d6517 4265static void quirk_calpella_no_shadow_gtt(struct pci_dev *dev)
9eecabcb
DW		 4266{
4267 unsigned short ggc;
4268
eecfd57f 4269 if (pci_read_config_word(dev, GGC, &ggc))
9eecabcb
DW		 4270 return;
4271
eecfd57f 4272 if (!(ggc & GGC_MEMORY_VT_ENABLED)) {
9eecabcb
DW		 4273 printk(KERN_INFO "DMAR: BIOS has allocated no shadow GTT; disabling IOMMU for graphics\n");
4274 dmar_map_gfx = 0;
6fbcfb3e
DW		 4275 } else if (dmar_map_gfx) {
4276 /* we have to ensure the gfx device is idle before we flush */
4277 printk(KERN_INFO "DMAR: Disabling batched IOTLB flush on Ironlake\n");
4278 intel_iommu_strict = 1;
4279 }
9eecabcb
DW		 4280}
4281DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0040, quirk_calpella_no_shadow_gtt);
4282DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0044, quirk_calpella_no_shadow_gtt);
4283DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0062, quirk_calpella_no_shadow_gtt);
4284DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x006a, quirk_calpella_no_shadow_gtt);
4285
e0fc7e0b
DW		 4286/* On Tylersburg chipsets, some BIOSes have been known to enable the
4287 ISOCH DMAR unit for the Azalia sound device, but not give it any
4288 TLB entries, which causes it to deadlock. Check for that. We do
4289 this in a function called from init_dmars(), instead of in a PCI
4290 quirk, because we don't want to print the obnoxious "BIOS broken"
4291 message if VT-d is actually disabled.
4292*/
4293static void __init check_tylersburg_isoch(void)
4294{
4295 struct pci_dev *pdev;
4296 uint32_t vtisochctrl;
4297
4298 /* If there's no Azalia in the system anyway, forget it. */
4299 pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x3a3e, NULL);
4300 if (!pdev)
4301 return;
4302 pci_dev_put(pdev);
4303
4304 /* System Management Registers. Might be hidden, in which case
4305 we can't do the sanity check. But that's OK, because the
4306 known-broken BIOSes _don't_ actually hide it, so far. */
4307 pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x342e, NULL);
4308 if (!pdev)
4309 return;
4310
4311 if (pci_read_config_dword(pdev, 0x188, &vtisochctrl)) {
4312 pci_dev_put(pdev);
4313 return;
4314 }
4315
4316 pci_dev_put(pdev);
4317
4318 /* If Azalia DMA is routed to the non-isoch DMAR unit, fine. */
4319 if (vtisochctrl & 1)
4320 return;
4321
4322 /* Drop all bits other than the number of TLB entries */
4323 vtisochctrl &= 0x1c;
4324
4325 /* If we have the recommended number of TLB entries (16), fine. */
4326 if (vtisochctrl == 0x10)
4327 return;
4328
4329 /* Zero TLB entries? You get to ride the short bus to school. */
4330 if (!vtisochctrl) {
4331 WARN(1, "Your BIOS is broken; DMA routed to ISOCH DMAR unit but no TLB space.\n"
4332 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
4333 dmi_get_system_info(DMI_BIOS_VENDOR),
4334 dmi_get_system_info(DMI_BIOS_VERSION),
4335 dmi_get_system_info(DMI_PRODUCT_VERSION));
4336 iommu_identity_mapping |= IDENTMAP_AZALIA;
4337 return;
4338 }
4339
4340 printk(KERN_WARNING "DMAR: Recommended TLB entries for ISOCH unit is 16; your BIOS set %d\n",
4341 vtisochctrl);
4342}
Linux kernel - Deliverables
This page took 0.924063 seconds and 5 git commands to generate.