projects / deliverable / linux.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
iommu/vt-d: Add initial support for PASID tables
[deliverable/linux.git] / drivers / iommu / intel-iommu.c
CommitLineData
ba395927 1/*
ea8ea460 2 * Copyright © 2006-2014 Intel Corporation.
ba395927
KA		 3 *
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms and conditions of the GNU General Public License,
6 * version 2, as published by the Free Software Foundation.
7 *
8 * This program is distributed in the hope it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
11 * more details.
12 *
ea8ea460
DW		 13 * Authors: David Woodhouse <dwmw2@infradead.org>,
14 * Ashok Raj <ashok.raj@intel.com>,
15 * Shaohua Li <shaohua.li@intel.com>,
16 * Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>,
17 * Fenghua Yu <fenghua.yu@intel.com>
9f10e5bf 18 * Joerg Roedel <jroedel@suse.de>
ba395927
KA		 19 */
20
9f10e5bf
JR		 21#define pr_fmt(fmt) "DMAR: " fmt
22
ba395927
KA		 23#include <linux/init.h>
24#include <linux/bitmap.h>
5e0d2a6f 25#include <linux/debugfs.h>
54485c30 26#include <linux/export.h>
ba395927
KA		 27#include <linux/slab.h>
28#include <linux/irq.h>
29#include <linux/interrupt.h>
ba395927
KA		 30#include <linux/spinlock.h>
31#include <linux/pci.h>
32#include <linux/dmar.h>
33#include <linux/dma-mapping.h>
34#include <linux/mempool.h>
75f05569 35#include <linux/memory.h>
5e0d2a6f 36#include <linux/timer.h>
38717946 37#include <linux/iova.h>
5d450806 38#include <linux/iommu.h>
38717946 39#include <linux/intel-iommu.h>
134fac3f 40#include <linux/syscore_ops.h>
69575d38 41#include <linux/tboot.h>
adb2fe02 42#include <linux/dmi.h>
5cdede24 43#include <linux/pci-ats.h>
0ee332c1 44#include <linux/memblock.h>
36746436 45#include <linux/dma-contiguous.h>
091d42e4 46#include <linux/crash_dump.h>
8a8f422d 47#include <asm/irq_remapping.h>
ba395927 48#include <asm/cacheflush.h>
46a7fa27 49#include <asm/iommu.h>
ba395927 50
078e1ee2
JR		 51#include "irq_remapping.h"
52
5b6985ce
FY		 53#define ROOT_SIZE VTD_PAGE_SIZE
54#define CONTEXT_SIZE VTD_PAGE_SIZE
55
ba395927 56#define IS_GFX_DEVICE(pdev) ((pdev->class >> 16) == PCI_BASE_CLASS_DISPLAY)
18436afd 57#define IS_USB_DEVICE(pdev) ((pdev->class >> 8) == PCI_CLASS_SERIAL_USB)
ba395927 58#define IS_ISA_DEVICE(pdev) ((pdev->class >> 8) == PCI_CLASS_BRIDGE_ISA)
e0fc7e0b 59#define IS_AZALIA(pdev) ((pdev)->vendor == 0x8086 && (pdev)->device == 0x3a3e)
ba395927
KA		 60
61#define IOAPIC_RANGE_START (0xfee00000)
62#define IOAPIC_RANGE_END (0xfeefffff)
63#define IOVA_START_ADDR (0x1000)
64
65#define DEFAULT_DOMAIN_ADDRESS_WIDTH 48
66
4ed0d3e6 67#define MAX_AGAW_WIDTH 64
5c645b35 68#define MAX_AGAW_PFN_WIDTH (MAX_AGAW_WIDTH - VTD_PAGE_SHIFT)
4ed0d3e6 69
2ebe3151
DW		 70#define __DOMAIN_MAX_PFN(gaw) ((((uint64_t)1) << (gaw-VTD_PAGE_SHIFT)) - 1)
71#define __DOMAIN_MAX_ADDR(gaw) ((((uint64_t)1) << gaw) - 1)
72
73/* We limit DOMAIN_MAX_PFN to fit in an unsigned long, and DOMAIN_MAX_ADDR
74 to match. That way, we can use 'unsigned long' for PFNs with impunity. */
75#define DOMAIN_MAX_PFN(gaw) ((unsigned long) min_t(uint64_t, \
76 __DOMAIN_MAX_PFN(gaw), (unsigned long)-1))
77#define DOMAIN_MAX_ADDR(gaw) (((uint64_t)__DOMAIN_MAX_PFN(gaw)) << VTD_PAGE_SHIFT)
ba395927 78
1b722500
RM		 79/* IO virtual address start page frame number */
80#define IOVA_START_PFN (1)
81
f27be03b 82#define IOVA_PFN(addr) ((addr) >> PAGE_SHIFT)
284901a9 83#define DMA_32BIT_PFN IOVA_PFN(DMA_BIT_MASK(32))
6a35528a 84#define DMA_64BIT_PFN IOVA_PFN(DMA_BIT_MASK(64))
5e0d2a6f 85
df08cdc7
AM		 86/* page table handling */
87#define LEVEL_STRIDE (9)
88#define LEVEL_MASK (((u64)1 << LEVEL_STRIDE) - 1)
89
6d1c56a9
OBC		 90/*
91 * This bitmap is used to advertise the page sizes our hardware support
92 * to the IOMMU core, which will then use this information to split
93 * physically contiguous memory regions it is mapping into page sizes
94 * that we support.
95 *
96 * Traditionally the IOMMU core just handed us the mappings directly,
97 * after making sure the size is an order of a 4KiB page and that the
98 * mapping has natural alignment.
99 *
100 * To retain this behavior, we currently advertise that we support
101 * all page sizes that are an order of 4KiB.
102 *
103 * If at some point we'd like to utilize the IOMMU core's new behavior,
104 * we could change this to advertise the real page sizes we support.
105 */
106#define INTEL_IOMMU_PGSIZES (~0xFFFUL)
107
df08cdc7
AM		 108static inline int agaw_to_level(int agaw)
109{
110 return agaw + 2;
111}
112
113static inline int agaw_to_width(int agaw)
114{
5c645b35 115 return min_t(int, 30 + agaw * LEVEL_STRIDE, MAX_AGAW_WIDTH);
df08cdc7
AM		 116}
117
118static inline int width_to_agaw(int width)
119{
5c645b35 120 return DIV_ROUND_UP(width - 30, LEVEL_STRIDE);
df08cdc7
AM		 121}
122
123static inline unsigned int level_to_offset_bits(int level)
124{
125 return (level - 1) * LEVEL_STRIDE;
126}
127
128static inline int pfn_level_offset(unsigned long pfn, int level)
129{
130 return (pfn >> level_to_offset_bits(level)) & LEVEL_MASK;
131}
132
133static inline unsigned long level_mask(int level)
134{
135 return -1UL << level_to_offset_bits(level);
136}
137
138static inline unsigned long level_size(int level)
139{
140 return 1UL << level_to_offset_bits(level);
141}
142
143static inline unsigned long align_to_level(unsigned long pfn, int level)
144{
145 return (pfn + level_size(level) - 1) & level_mask(level);
146}
fd18de50 147
6dd9a7c7
YS		 148static inline unsigned long lvl_to_nr_pages(unsigned int lvl)
149{
5c645b35 150 return 1 << min_t(int, (lvl - 1) * LEVEL_STRIDE, MAX_AGAW_PFN_WIDTH);
6dd9a7c7
YS		 151}
152
dd4e8319
DW		 153/* VT-d pages must always be _smaller_ than MM pages. Otherwise things
154 are never going to work. */
155static inline unsigned long dma_to_mm_pfn(unsigned long dma_pfn)
156{
157 return dma_pfn >> (PAGE_SHIFT - VTD_PAGE_SHIFT);
158}
159
160static inline unsigned long mm_to_dma_pfn(unsigned long mm_pfn)
161{
162 return mm_pfn << (PAGE_SHIFT - VTD_PAGE_SHIFT);
163}
164static inline unsigned long page_to_dma_pfn(struct page *pg)
165{
166 return mm_to_dma_pfn(page_to_pfn(pg));
167}
168static inline unsigned long virt_to_dma_pfn(void *p)
169{
170 return page_to_dma_pfn(virt_to_page(p));
171}
172
d9630fe9
WH		 173/* global iommu list, set NULL for ignored DMAR units */
174static struct intel_iommu **g_iommus;
175
e0fc7e0b 176static void __init check_tylersburg_isoch(void);
9af88143
DW		 177static int rwbf_quirk;
178
b779260b
JC		 179/*
180 * set to 1 to panic kernel if can't successfully enable VT-d
181 * (used when kernel is launched w/ TXT)
182 */
183static int force_on = 0;
184
46b08e1a
MM		 185/*
186 * 0: Present
187 * 1-11: Reserved
188 * 12-63: Context Ptr (12 - (haw-1))
189 * 64-127: Reserved
190 */
191struct root_entry {
03ecc32c
DW		 192 u64 lo;
193 u64 hi;
46b08e1a
MM		 194};
195#define ROOT_ENTRY_NR (VTD_PAGE_SIZE/sizeof(struct root_entry))
46b08e1a 196
091d42e4
JR		 197/*
198 * Take a root_entry and return the Lower Context Table Pointer (LCTP)
199 * if marked present.
200 */
201static phys_addr_t root_entry_lctp(struct root_entry *re)
202{
203 if (!(re->lo & 1))
204 return 0;
205
206 return re->lo & VTD_PAGE_MASK;
207}
208
209/*
210 * Take a root_entry and return the Upper Context Table Pointer (UCTP)
211 * if marked present.
212 */
213static phys_addr_t root_entry_uctp(struct root_entry *re)
214{
215 if (!(re->hi & 1))
216 return 0;
46b08e1a 217
091d42e4
JR		 218 return re->hi & VTD_PAGE_MASK;
219}
7a8fc25e
MM		 220/*
221 * low 64 bits:
222 * 0: present
223 * 1: fault processing disable
224 * 2-3: translation type
225 * 12-63: address space root
226 * high 64 bits:
227 * 0-2: address width
228 * 3-6: aval
229 * 8-23: domain id
230 */
231struct context_entry {
232 u64 lo;
233 u64 hi;
234};
c07e7d21 235
cf484d0e
JR		 236static inline void context_clear_pasid_enable(struct context_entry *context)
237{
238 context->lo &= ~(1ULL << 11);
239}
240
241static inline bool context_pasid_enabled(struct context_entry *context)
242{
243 return !!(context->lo & (1ULL << 11));
244}
245
246static inline void context_set_copied(struct context_entry *context)
247{
248 context->hi |= (1ull << 3);
249}
250
251static inline bool context_copied(struct context_entry *context)
252{
253 return !!(context->hi & (1ULL << 3));
254}
255
256static inline bool __context_present(struct context_entry *context)
c07e7d21
MM		 257{
258 return (context->lo & 1);
259}
cf484d0e
JR		 260
261static inline bool context_present(struct context_entry *context)
262{
263 return context_pasid_enabled(context) ?
264 __context_present(context) :
265 __context_present(context) && !context_copied(context);
266}
267
c07e7d21
MM		 268static inline void context_set_present(struct context_entry *context)
269{
270 context->lo |= 1;
271}
272
273static inline void context_set_fault_enable(struct context_entry *context)
274{
275 context->lo &= (((u64)-1) << 2) | 1;
276}
277
c07e7d21
MM		 278static inline void context_set_translation_type(struct context_entry *context,
279 unsigned long value)
280{
281 context->lo &= (((u64)-1) << 4) | 3;
282 context->lo |= (value & 3) << 2;
283}
284
285static inline void context_set_address_root(struct context_entry *context,
286 unsigned long value)
287{
1a2262f9 288 context->lo &= ~VTD_PAGE_MASK;
c07e7d21
MM		 289 context->lo |= value & VTD_PAGE_MASK;
290}
291
292static inline void context_set_address_width(struct context_entry *context,
293 unsigned long value)
294{
295 context->hi |= value & 7;
296}
297
298static inline void context_set_domain_id(struct context_entry *context,
299 unsigned long value)
300{
301 context->hi |= (value & ((1 << 16) - 1)) << 8;
302}
303
dbcd861f
JR		 304static inline int context_domain_id(struct context_entry *c)
305{
306 return((c->hi >> 8) & 0xffff);
307}
308
c07e7d21
MM		 309static inline void context_clear_entry(struct context_entry *context)
310{
311 context->lo = 0;
312 context->hi = 0;
313}
7a8fc25e 314
622ba12a
MM		 315/*
316 * 0: readable
317 * 1: writable
318 * 2-6: reserved
319 * 7: super page
9cf06697
SY		 320 * 8-10: available
321 * 11: snoop behavior
622ba12a
MM		 322 * 12-63: Host physcial address
323 */
324struct dma_pte {
325 u64 val;
326};
622ba12a 327
19c239ce
MM		 328static inline void dma_clear_pte(struct dma_pte *pte)
329{
330 pte->val = 0;
331}
332
19c239ce
MM		 333static inline u64 dma_pte_addr(struct dma_pte *pte)
334{
c85994e4
DW		 335#ifdef CONFIG_64BIT
336 return pte->val & VTD_PAGE_MASK;
337#else
338 /* Must have a full atomic 64-bit read */
1a8bd481 339 return __cmpxchg64(&pte->val, 0ULL, 0ULL) & VTD_PAGE_MASK;
c85994e4 340#endif
19c239ce
MM		 341}
342
19c239ce
MM		 343static inline bool dma_pte_present(struct dma_pte *pte)
344{
345 return (pte->val & 3) != 0;
346}
622ba12a 347
4399c8bf
AK		 348static inline bool dma_pte_superpage(struct dma_pte *pte)
349{
c3c75eb7 350 return (pte->val & DMA_PTE_LARGE_PAGE);
4399c8bf
AK		 351}
352
75e6bf96
DW		 353static inline int first_pte_in_page(struct dma_pte *pte)
354{
355 return !((unsigned long)pte & ~VTD_PAGE_MASK);
356}
357
2c2e2c38
FY		 358/*
359 * This domain is a statically identity mapping domain.
360 * 1. This domain creats a static 1:1 mapping to all usable memory.
361 * 2. It maps to each iommu if successful.
362 * 3. Each iommu mapps to this domain if successful.
363 */
19943b0e
DW		 364static struct dmar_domain *si_domain;
365static int hw_pass_through = 1;
2c2e2c38 366
28ccce0d
JR		 367/*
368 * Domain represents a virtual machine, more than one devices
1ce28feb
WH		 369 * across iommus may be owned in one domain, e.g. kvm guest.
370 */
ab8dfe25 371#define DOMAIN_FLAG_VIRTUAL_MACHINE (1 << 0)
1ce28feb 372
2c2e2c38 373/* si_domain contains mulitple devices */
ab8dfe25 374#define DOMAIN_FLAG_STATIC_IDENTITY (1 << 1)
2c2e2c38 375
29a27719
JR		 376#define for_each_domain_iommu(idx, domain) \
377 for (idx = 0; idx < g_num_of_iommus; idx++) \
378 if (domain->iommu_refcnt[idx])
379
99126f7c 380struct dmar_domain {
4c923d47 381 int nid; /* node id */
29a27719
JR		 382
383 unsigned iommu_refcnt[DMAR_UNITS_SUPPORTED];
384 /* Refcount of devices per iommu */
385
99126f7c 386
c0e8a6c8
JR		 387 u16 iommu_did[DMAR_UNITS_SUPPORTED];
388 /* Domain ids per IOMMU. Use u16 since
389 * domain ids are 16 bit wide according
390 * to VT-d spec, section 9.3 */
99126f7c 391
00a77deb 392 struct list_head devices; /* all devices' list */
99126f7c
MM		 393 struct iova_domain iovad; /* iova's that belong to this domain */
394
395 struct dma_pte *pgd; /* virtual address */
99126f7c
MM		 396 int gaw; /* max guest address width */
397
398 /* adjusted guest address width, 0 is level 2 30-bit */
399 int agaw;
400
3b5410e7 401 int flags; /* flags to find out type of domain */
8e604097
WH		 402
403 int iommu_coherency;/* indicate coherency of iommu access */
58c610bd 404 int iommu_snooping; /* indicate snooping control feature*/
c7151a8d 405 int iommu_count; /* reference count of iommu */
6dd9a7c7
YS		 406 int iommu_superpage;/* Level of superpages supported:
407 0 == 4KiB (no superpages), 1 == 2MiB,
408 2 == 1GiB, 3 == 512GiB, 4 == 1TiB */
fe40f1e0 409 u64 max_addr; /* maximum mapped address */
00a77deb
JR		 410
411 struct iommu_domain domain; /* generic domain data structure for
412 iommu core */
99126f7c
MM		 413};
414
a647dacb
MM		 415/* PCI domain-device relationship */
416struct device_domain_info {
417 struct list_head link; /* link to domain siblings */
418 struct list_head global; /* link to global list */
276dbf99 419 u8 bus; /* PCI bus number */
a647dacb 420 u8 devfn; /* PCI devfn number */
fb0cc3aa
BH		 421 struct {
422 u8 enabled:1;
423 u8 qdep;
424 } ats; /* ATS state */
0bcb3e28 425 struct device *dev; /* it's NULL for PCIe-to-PCI bridge */
93a23a72 426 struct intel_iommu *iommu; /* IOMMU used by this device */
a647dacb
MM		 427 struct dmar_domain *domain; /* pointer to domain */
428};
429
b94e4117
JL		 430struct dmar_rmrr_unit {
431 struct list_head list; /* list of rmrr units */
432 struct acpi_dmar_header *hdr; /* ACPI header */
433 u64 base_address; /* reserved base address*/
434 u64 end_address; /* reserved end address */
832bd858 435 struct dmar_dev_scope *devices; /* target devices */
b94e4117
JL		 436 int devices_cnt; /* target device count */
437};
438
439struct dmar_atsr_unit {
440 struct list_head list; /* list of ATSR units */
441 struct acpi_dmar_header *hdr; /* ACPI header */
832bd858 442 struct dmar_dev_scope *devices; /* target devices */
b94e4117
JL		 443 int devices_cnt; /* target device count */
444 u8 include_all:1; /* include all ports */
445};
446
447static LIST_HEAD(dmar_atsr_units);
448static LIST_HEAD(dmar_rmrr_units);
449
450#define for_each_rmrr_units(rmrr) \
451 list_for_each_entry(rmrr, &dmar_rmrr_units, list)
452
5e0d2a6f 453static void flush_unmaps_timeout(unsigned long data);
454
b707cb02 455static DEFINE_TIMER(unmap_timer, flush_unmaps_timeout, 0, 0);
5e0d2a6f 456
80b20dd8 457#define HIGH_WATER_MARK 250
458struct deferred_flush_tables {
459 int next;
460 struct iova *iova[HIGH_WATER_MARK];
461 struct dmar_domain *domain[HIGH_WATER_MARK];
ea8ea460 462 struct page *freelist[HIGH_WATER_MARK];
80b20dd8 463};
464
465static struct deferred_flush_tables *deferred_flush;
466
5e0d2a6f 467/* bitmap for indexing intel_iommus */
5e0d2a6f 468static int g_num_of_iommus;
469
470static DEFINE_SPINLOCK(async_umap_flush_lock);
471static LIST_HEAD(unmaps_to_do);
472
473static int timer_on;
474static long list_size;
5e0d2a6f 475
92d03cc8 476static void domain_exit(struct dmar_domain *domain);
ba395927 477static void domain_remove_dev_info(struct dmar_domain *domain);
e6de0f8d
JR		 478static void dmar_remove_one_dev_info(struct dmar_domain *domain,
479 struct device *dev);
127c7615 480static void __dmar_remove_one_dev_info(struct device_domain_info *info);
2452d9db
JR		 481static void domain_context_clear(struct intel_iommu *iommu,
482 struct device *dev);
2a46ddf7
JL		 483static int domain_detach_iommu(struct dmar_domain *domain,
484 struct intel_iommu *iommu);
ba395927 485
d3f13810 486#ifdef CONFIG_INTEL_IOMMU_DEFAULT_ON
0cd5c3c8
KM		 487int dmar_disabled = 0;
488#else
489int dmar_disabled = 1;
d3f13810 490#endif /*CONFIG_INTEL_IOMMU_DEFAULT_ON*/
0cd5c3c8 491
8bc1f85c
ED		 492int intel_iommu_enabled = 0;
493EXPORT_SYMBOL_GPL(intel_iommu_enabled);
494
2d9e667e 495static int dmar_map_gfx = 1;
7d3b03ce 496static int dmar_forcedac;
5e0d2a6f 497static int intel_iommu_strict;
6dd9a7c7 498static int intel_iommu_superpage = 1;
c83b2f20 499static int intel_iommu_ecs = 1;
ae853ddb
DW		 500static int intel_iommu_pasid28;
501static int iommu_identity_mapping;
502
503#define IDENTMAP_ALL 1
504#define IDENTMAP_GFX 2
505#define IDENTMAP_AZALIA 4
c83b2f20
DW		 506
507/* We only actually use ECS when PASID support (on the new bit 40)
508 * is also advertised. Some early implementations — the ones with
509 * PASID support on bit 28 — have issues even when we *only* use
510 * extended root/context tables. */
ae853ddb
DW		 511#define pasid_enabled(iommu) (ecap_pasid(iommu->ecap) || \
512 (intel_iommu_pasid28 && ecap_broken_pasid(iommu->ecap)))
c83b2f20 513#define ecs_enabled(iommu) (intel_iommu_ecs && ecap_ecs(iommu->ecap) && \
ae853ddb 514 pasid_enabled(iommu))
ba395927 515
c0771df8
DW		 516int intel_iommu_gfx_mapped;
517EXPORT_SYMBOL_GPL(intel_iommu_gfx_mapped);
518
ba395927
KA		 519#define DUMMY_DEVICE_DOMAIN_INFO ((struct device_domain_info *)(-1))
520static DEFINE_SPINLOCK(device_domain_lock);
521static LIST_HEAD(device_domain_list);
522
b22f6434 523static const struct iommu_ops intel_iommu_ops;
a8bcbb0d 524
4158c2ec
JR		 525static bool translation_pre_enabled(struct intel_iommu *iommu)
526{
527 return (iommu->flags & VTD_FLAG_TRANS_PRE_ENABLED);
528}
529
091d42e4
JR		 530static void clear_translation_pre_enabled(struct intel_iommu *iommu)
531{
532 iommu->flags &= ~VTD_FLAG_TRANS_PRE_ENABLED;
533}
534
4158c2ec
JR		 535static void init_translation_status(struct intel_iommu *iommu)
536{
537 u32 gsts;
538
539 gsts = readl(iommu->reg + DMAR_GSTS_REG);
540 if (gsts & DMA_GSTS_TES)
541 iommu->flags |= VTD_FLAG_TRANS_PRE_ENABLED;
542}
543
00a77deb
JR		 544/* Convert generic 'struct iommu_domain to private struct dmar_domain */
545static struct dmar_domain *to_dmar_domain(struct iommu_domain *dom)
546{
547 return container_of(dom, struct dmar_domain, domain);
548}
549
ba395927
KA		 550static int __init intel_iommu_setup(char *str)
551{
552 if (!str)
553 return -EINVAL;
554 while (*str) {
0cd5c3c8
KM		 555 if (!strncmp(str, "on", 2)) {
556 dmar_disabled = 0;
9f10e5bf 557 pr_info("IOMMU enabled\n");
0cd5c3c8 558 } else if (!strncmp(str, "off", 3)) {
ba395927 559 dmar_disabled = 1;
9f10e5bf 560 pr_info("IOMMU disabled\n");
ba395927
KA		 561 } else if (!strncmp(str, "igfx_off", 8)) {
562 dmar_map_gfx = 0;
9f10e5bf 563 pr_info("Disable GFX device mapping\n");
7d3b03ce 564 } else if (!strncmp(str, "forcedac", 8)) {
9f10e5bf 565 pr_info("Forcing DAC for PCI devices\n");
7d3b03ce 566 dmar_forcedac = 1;
5e0d2a6f 567 } else if (!strncmp(str, "strict", 6)) {
9f10e5bf 568 pr_info("Disable batched IOTLB flush\n");
5e0d2a6f 569 intel_iommu_strict = 1;
6dd9a7c7 570 } else if (!strncmp(str, "sp_off", 6)) {
9f10e5bf 571 pr_info("Disable supported super page\n");
6dd9a7c7 572 intel_iommu_superpage = 0;
c83b2f20
DW		 573 } else if (!strncmp(str, "ecs_off", 7)) {
574 printk(KERN_INFO
575 "Intel-IOMMU: disable extended context table support\n");
576 intel_iommu_ecs = 0;
ae853ddb
DW		 577 } else if (!strncmp(str, "pasid28", 7)) {
578 printk(KERN_INFO
579 "Intel-IOMMU: enable pre-production PASID support\n");
580 intel_iommu_pasid28 = 1;
581 iommu_identity_mapping |= IDENTMAP_GFX;
ba395927
KA		 582 }
583
584 str += strcspn(str, ",");
585 while (*str == ',')
586 str++;
587 }
588 return 0;
589}
590__setup("intel_iommu=", intel_iommu_setup);
591
592static struct kmem_cache *iommu_domain_cache;
593static struct kmem_cache *iommu_devinfo_cache;
ba395927 594
9452d5bf
JR		 595static struct dmar_domain* get_iommu_domain(struct intel_iommu *iommu, u16 did)
596{
8bf47816
JR		 597 struct dmar_domain **domains;
598 int idx = did >> 8;
599
600 domains = iommu->domains[idx];
601 if (!domains)
602 return NULL;
603
604 return domains[did & 0xff];
9452d5bf
JR		 605}
606
607static void set_iommu_domain(struct intel_iommu *iommu, u16 did,
608 struct dmar_domain *domain)
609{
8bf47816
JR		 610 struct dmar_domain **domains;
611 int idx = did >> 8;
612
613 if (!iommu->domains[idx]) {
614 size_t size = 256 * sizeof(struct dmar_domain *);
615 iommu->domains[idx] = kzalloc(size, GFP_ATOMIC);
616 }
617
618 domains = iommu->domains[idx];
619 if (WARN_ON(!domains))
620 return;
621 else
622 domains[did & 0xff] = domain;
9452d5bf
JR		 623}
624
4c923d47 625static inline void *alloc_pgtable_page(int node)
eb3fa7cb 626{
4c923d47
SS		 627 struct page *page;
628 void *vaddr = NULL;
eb3fa7cb 629
4c923d47
SS		 630 page = alloc_pages_node(node, GFP_ATOMIC | __GFP_ZERO, 0);
631 if (page)
632 vaddr = page_address(page);
eb3fa7cb 633 return vaddr;
ba395927
KA		 634}
635
636static inline void free_pgtable_page(void *vaddr)
637{
638 free_page((unsigned long)vaddr);
639}
640
641static inline void *alloc_domain_mem(void)
642{
354bb65e 643 return kmem_cache_alloc(iommu_domain_cache, GFP_ATOMIC);
ba395927
KA		 644}
645
38717946 646static void free_domain_mem(void *vaddr)
ba395927
KA		 647{
648 kmem_cache_free(iommu_domain_cache, vaddr);
649}
650
651static inline void * alloc_devinfo_mem(void)
652{
354bb65e 653 return kmem_cache_alloc(iommu_devinfo_cache, GFP_ATOMIC);
ba395927
KA		 654}
655
656static inline void free_devinfo_mem(void *vaddr)
657{
658 kmem_cache_free(iommu_devinfo_cache, vaddr);
659}
660
ab8dfe25
JL		 661static inline int domain_type_is_vm(struct dmar_domain *domain)
662{
663 return domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE;
664}
665
28ccce0d
JR		 666static inline int domain_type_is_si(struct dmar_domain *domain)
667{
668 return domain->flags & DOMAIN_FLAG_STATIC_IDENTITY;
669}
670
ab8dfe25
JL		 671static inline int domain_type_is_vm_or_si(struct dmar_domain *domain)
672{
673 return domain->flags & (DOMAIN_FLAG_VIRTUAL_MACHINE |
674 DOMAIN_FLAG_STATIC_IDENTITY);
675}
1b573683 676
162d1b10
JL		 677static inline int domain_pfn_supported(struct dmar_domain *domain,
678 unsigned long pfn)
679{
680 int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
681
682 return !(addr_width < BITS_PER_LONG && pfn >> addr_width);
683}
684
4ed0d3e6 685static int __iommu_calculate_agaw(struct intel_iommu *iommu, int max_gaw)
1b573683
WH		 686{
687 unsigned long sagaw;
688 int agaw = -1;
689
690 sagaw = cap_sagaw(iommu->cap);
4ed0d3e6 691 for (agaw = width_to_agaw(max_gaw);
1b573683
WH		 692 agaw >= 0; agaw--) {
693 if (test_bit(agaw, &sagaw))
694 break;
695 }
696
697 return agaw;
698}
699
4ed0d3e6
FY		 700/*
701 * Calculate max SAGAW for each iommu.
702 */
703int iommu_calculate_max_sagaw(struct intel_iommu *iommu)
704{
705 return __iommu_calculate_agaw(iommu, MAX_AGAW_WIDTH);
706}
707
708/*
709 * calculate agaw for each iommu.
710 * "SAGAW" may be different across iommus, use a default agaw, and
711 * get a supported less agaw for iommus that don't support the default agaw.
712 */
713int iommu_calculate_agaw(struct intel_iommu *iommu)
714{
715 return __iommu_calculate_agaw(iommu, DEFAULT_DOMAIN_ADDRESS_WIDTH);
716}
717
2c2e2c38 718/* This functionin only returns single iommu in a domain */
8c11e798
WH		 719static struct intel_iommu *domain_get_iommu(struct dmar_domain *domain)
720{
721 int iommu_id;
722
2c2e2c38 723 /* si_domain and vm domain should not get here. */
ab8dfe25 724 BUG_ON(domain_type_is_vm_or_si(domain));
29a27719
JR		 725 for_each_domain_iommu(iommu_id, domain)
726 break;
727
8c11e798
WH		 728 if (iommu_id < 0 || iommu_id >= g_num_of_iommus)
729 return NULL;
730
731 return g_iommus[iommu_id];
732}
733
8e604097
WH		 734static void domain_update_iommu_coherency(struct dmar_domain *domain)
735{
d0501960
DW		 736 struct dmar_drhd_unit *drhd;
737 struct intel_iommu *iommu;
2f119c78
QL		 738 bool found = false;
739 int i;
2e12bc29 740
d0501960 741 domain->iommu_coherency = 1;
8e604097 742
29a27719 743 for_each_domain_iommu(i, domain) {
2f119c78 744 found = true;
8e604097
WH		 745 if (!ecap_coherent(g_iommus[i]->ecap)) {
746 domain->iommu_coherency = 0;
747 break;
748 }
8e604097 749 }
d0501960
DW		 750 if (found)
751 return;
752
753 /* No hardware attached; use lowest common denominator */
754 rcu_read_lock();
755 for_each_active_iommu(iommu, drhd) {
756 if (!ecap_coherent(iommu->ecap)) {
757 domain->iommu_coherency = 0;
758 break;
759 }
760 }
761 rcu_read_unlock();
8e604097
WH		 762}
763
161f6934 764static int domain_update_iommu_snooping(struct intel_iommu *skip)
58c610bd 765{
161f6934
JL		 766 struct dmar_drhd_unit *drhd;
767 struct intel_iommu *iommu;
768 int ret = 1;
58c610bd 769
161f6934
JL		 770 rcu_read_lock();
771 for_each_active_iommu(iommu, drhd) {
772 if (iommu != skip) {
773 if (!ecap_sc_support(iommu->ecap)) {
774 ret = 0;
775 break;
776 }
58c610bd 777 }
58c610bd 778 }
161f6934
JL		 779 rcu_read_unlock();
780
781 return ret;
58c610bd
SY		 782}
783
161f6934 784static int domain_update_iommu_superpage(struct intel_iommu *skip)
6dd9a7c7 785{
8140a95d 786 struct dmar_drhd_unit *drhd;
161f6934 787 struct intel_iommu *iommu;
8140a95d 788 int mask = 0xf;
6dd9a7c7
YS		 789
790 if (!intel_iommu_superpage) {
161f6934 791 return 0;
6dd9a7c7
YS		 792 }
793
8140a95d 794 /* set iommu_superpage to the smallest common denominator */
0e242612 795 rcu_read_lock();
8140a95d 796 for_each_active_iommu(iommu, drhd) {
161f6934
JL		 797 if (iommu != skip) {
798 mask &= cap_super_page_val(iommu->cap);
799 if (!mask)
800 break;
6dd9a7c7
YS		 801 }
802 }
0e242612
JL		 803 rcu_read_unlock();
804
161f6934 805 return fls(mask);
6dd9a7c7
YS		 806}
807
58c610bd
SY		 808/* Some capabilities may be different across iommus */
809static void domain_update_iommu_cap(struct dmar_domain *domain)
810{
811 domain_update_iommu_coherency(domain);
161f6934
JL		 812 domain->iommu_snooping = domain_update_iommu_snooping(NULL);
813 domain->iommu_superpage = domain_update_iommu_superpage(NULL);
58c610bd
SY		 814}
815
03ecc32c
DW		 816static inline struct context_entry *iommu_context_addr(struct intel_iommu *iommu,
817 u8 bus, u8 devfn, int alloc)
818{
819 struct root_entry *root = &iommu->root_entry[bus];
820 struct context_entry *context;
821 u64 *entry;
822
4df4eab1 823 entry = &root->lo;
c83b2f20 824 if (ecs_enabled(iommu)) {
03ecc32c
DW		 825 if (devfn >= 0x80) {
826 devfn -= 0x80;
827 entry = &root->hi;
828 }
829 devfn *= 2;
830 }
03ecc32c
DW		 831 if (*entry & 1)
832 context = phys_to_virt(*entry & VTD_PAGE_MASK);
833 else {
834 unsigned long phy_addr;
835 if (!alloc)
836 return NULL;
837
838 context = alloc_pgtable_page(iommu->node);
839 if (!context)
840 return NULL;
841
842 __iommu_flush_cache(iommu, (void *)context, CONTEXT_SIZE);
843 phy_addr = virt_to_phys((void *)context);
844 *entry = phy_addr | 1;
845 __iommu_flush_cache(iommu, entry, sizeof(*entry));
846 }
847 return &context[devfn];
848}
849
4ed6a540
DW		 850static int iommu_dummy(struct device *dev)
851{
852 return dev->archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO;
853}
854
156baca8 855static struct intel_iommu *device_to_iommu(struct device *dev, u8 *bus, u8 *devfn)
c7151a8d
WH		 856{
857 struct dmar_drhd_unit *drhd = NULL;
b683b230 858 struct intel_iommu *iommu;
156baca8
DW		 859 struct device *tmp;
860 struct pci_dev *ptmp, *pdev = NULL;
aa4d066a 861 u16 segment = 0;
c7151a8d
WH		 862 int i;
863
4ed6a540
DW		 864 if (iommu_dummy(dev))
865 return NULL;
866
156baca8
DW		 867 if (dev_is_pci(dev)) {
868 pdev = to_pci_dev(dev);
869 segment = pci_domain_nr(pdev->bus);
ca5b74d2 870 } else if (has_acpi_companion(dev))
156baca8
DW		 871 dev = &ACPI_COMPANION(dev)->dev;
872
0e242612 873 rcu_read_lock();
b683b230 874 for_each_active_iommu(iommu, drhd) {
156baca8 875 if (pdev && segment != drhd->segment)
276dbf99 876 continue;
c7151a8d 877
b683b230 878 for_each_active_dev_scope(drhd->devices,
156baca8
DW		 879 drhd->devices_cnt, i, tmp) {
880 if (tmp == dev) {
881 *bus = drhd->devices[i].bus;
882 *devfn = drhd->devices[i].devfn;
b683b230 883 goto out;
156baca8
DW		 884 }
885
886 if (!pdev || !dev_is_pci(tmp))
887 continue;
888
889 ptmp = to_pci_dev(tmp);
890 if (ptmp->subordinate &&
891 ptmp->subordinate->number <= pdev->bus->number &&
892 ptmp->subordinate->busn_res.end >= pdev->bus->number)
893 goto got_pdev;
924b6231 894 }
c7151a8d 895
156baca8
DW		 896 if (pdev && drhd->include_all) {
897 got_pdev:
898 *bus = pdev->bus->number;
899 *devfn = pdev->devfn;
b683b230 900 goto out;
156baca8 901 }
c7151a8d 902 }
b683b230 903 iommu = NULL;
156baca8 904 out:
0e242612 905 rcu_read_unlock();
c7151a8d 906
b683b230 907 return iommu;
c7151a8d
WH		 908}
909
5331fe6f
WH		 910static void domain_flush_cache(struct dmar_domain *domain,
911 void *addr, int size)
912{
913 if (!domain->iommu_coherency)
914 clflush_cache_range(addr, size);
915}
916
ba395927
KA		 917static int device_context_mapped(struct intel_iommu *iommu, u8 bus, u8 devfn)
918{
ba395927 919 struct context_entry *context;
03ecc32c 920 int ret = 0;
ba395927
KA		 921 unsigned long flags;
922
923 spin_lock_irqsave(&iommu->lock, flags);
03ecc32c
DW		 924 context = iommu_context_addr(iommu, bus, devfn, 0);
925 if (context)
926 ret = context_present(context);
ba395927
KA		 927 spin_unlock_irqrestore(&iommu->lock, flags);
928 return ret;
929}
930
931static void clear_context_table(struct intel_iommu *iommu, u8 bus, u8 devfn)
932{
ba395927
KA		 933 struct context_entry *context;
934 unsigned long flags;
935
936 spin_lock_irqsave(&iommu->lock, flags);
03ecc32c 937 context = iommu_context_addr(iommu, bus, devfn, 0);
ba395927 938 if (context) {
03ecc32c
DW		 939 context_clear_entry(context);
940 __iommu_flush_cache(iommu, context, sizeof(*context));
ba395927
KA		 941 }
942 spin_unlock_irqrestore(&iommu->lock, flags);
943}
944
945static void free_context_table(struct intel_iommu *iommu)
946{
ba395927
KA		 947 int i;
948 unsigned long flags;
949 struct context_entry *context;
950
951 spin_lock_irqsave(&iommu->lock, flags);
952 if (!iommu->root_entry) {
953 goto out;
954 }
955 for (i = 0; i < ROOT_ENTRY_NR; i++) {
03ecc32c 956 context = iommu_context_addr(iommu, i, 0, 0);
ba395927
KA		 957 if (context)
958 free_pgtable_page(context);
03ecc32c 959
c83b2f20 960 if (!ecs_enabled(iommu))
03ecc32c
DW		 961 continue;
962
963 context = iommu_context_addr(iommu, i, 0x80, 0);
964 if (context)
965 free_pgtable_page(context);
966
ba395927
KA		 967 }
968 free_pgtable_page(iommu->root_entry);
969 iommu->root_entry = NULL;
970out:
971 spin_unlock_irqrestore(&iommu->lock, flags);
972}
973
b026fd28 974static struct dma_pte *pfn_to_dma_pte(struct dmar_domain *domain,
5cf0a76f 975 unsigned long pfn, int *target_level)
ba395927 976{
ba395927
KA		 977 struct dma_pte *parent, *pte = NULL;
978 int level = agaw_to_level(domain->agaw);
4399c8bf 979 int offset;
ba395927
KA		 980
981 BUG_ON(!domain->pgd);
f9423606 982
162d1b10 983 if (!domain_pfn_supported(domain, pfn))
f9423606
JS		 984 /* Address beyond IOMMU's addressing capabilities. */
985 return NULL;
986
ba395927
KA		 987 parent = domain->pgd;
988
5cf0a76f 989 while (1) {
ba395927
KA		 990 void *tmp_page;
991
b026fd28 992 offset = pfn_level_offset(pfn, level);
ba395927 993 pte = &parent[offset];
5cf0a76f 994 if (!*target_level && (dma_pte_superpage(pte) || !dma_pte_present(pte)))
6dd9a7c7 995 break;
5cf0a76f 996 if (level == *target_level)
ba395927
KA		 997 break;
998
19c239ce 999 if (!dma_pte_present(pte)) {
c85994e4
DW		 1000 uint64_t pteval;
1001
4c923d47 1002 tmp_page = alloc_pgtable_page(domain->nid);
ba395927 1003
206a73c1 1004 if (!tmp_page)
ba395927 1005 return NULL;
206a73c1 1006
c85994e4 1007 domain_flush_cache(domain, tmp_page, VTD_PAGE_SIZE);
64de5af0 1008 pteval = ((uint64_t)virt_to_dma_pfn(tmp_page) << VTD_PAGE_SHIFT) | DMA_PTE_READ | DMA_PTE_WRITE;
effad4b5 1009 if (cmpxchg64(&pte->val, 0ULL, pteval))
c85994e4
DW		 1010 /* Someone else set it while we were thinking; use theirs. */
1011 free_pgtable_page(tmp_page);
effad4b5 1012 else
c85994e4 1013 domain_flush_cache(domain, pte, sizeof(*pte));
ba395927 1014 }
5cf0a76f
DW		 1015 if (level == 1)
1016 break;
1017
19c239ce 1018 parent = phys_to_virt(dma_pte_addr(pte));
ba395927
KA		 1019 level--;
1020 }
1021
5cf0a76f
DW		 1022 if (!*target_level)
1023 *target_level = level;
1024
ba395927
KA		 1025 return pte;
1026}
1027
6dd9a7c7 1028
ba395927 1029/* return address's pte at specific level */
90dcfb5e
DW		 1030static struct dma_pte *dma_pfn_level_pte(struct dmar_domain *domain,
1031 unsigned long pfn,
6dd9a7c7 1032 int level, int *large_page)
ba395927
KA		 1033{
1034 struct dma_pte *parent, *pte = NULL;
1035 int total = agaw_to_level(domain->agaw);
1036 int offset;
1037
1038 parent = domain->pgd;
1039 while (level <= total) {
90dcfb5e 1040 offset = pfn_level_offset(pfn, total);
ba395927
KA		 1041 pte = &parent[offset];
1042 if (level == total)
1043 return pte;
1044
6dd9a7c7
YS		 1045 if (!dma_pte_present(pte)) {
1046 *large_page = total;
ba395927 1047 break;
6dd9a7c7
YS		 1048 }
1049
e16922af 1050 if (dma_pte_superpage(pte)) {
6dd9a7c7
YS		 1051 *large_page = total;
1052 return pte;
1053 }
1054
19c239ce 1055 parent = phys_to_virt(dma_pte_addr(pte));
ba395927
KA		 1056 total--;
1057 }
1058 return NULL;
1059}
1060
ba395927 1061/* clear last level pte, a tlb flush should be followed */
5cf0a76f 1062static void dma_pte_clear_range(struct dmar_domain *domain,
595badf5
DW		 1063 unsigned long start_pfn,
1064 unsigned long last_pfn)
ba395927 1065{
6dd9a7c7 1066 unsigned int large_page = 1;
310a5ab9 1067 struct dma_pte *first_pte, *pte;
66eae846 1068
162d1b10
JL		 1069 BUG_ON(!domain_pfn_supported(domain, start_pfn));
1070 BUG_ON(!domain_pfn_supported(domain, last_pfn));
59c36286 1071 BUG_ON(start_pfn > last_pfn);
ba395927 1072
04b18e65 1073 /* we don't need lock here; nobody else touches the iova range */
59c36286 1074 do {
6dd9a7c7
YS		 1075 large_page = 1;
1076 first_pte = pte = dma_pfn_level_pte(domain, start_pfn, 1, &large_page);
310a5ab9 1077 if (!pte) {
6dd9a7c7 1078 start_pfn = align_to_level(start_pfn + 1, large_page + 1);
310a5ab9
DW		 1079 continue;
1080 }
6dd9a7c7 1081 do {
310a5ab9 1082 dma_clear_pte(pte);
6dd9a7c7 1083 start_pfn += lvl_to_nr_pages(large_page);
310a5ab9 1084 pte++;
75e6bf96
DW		 1085 } while (start_pfn <= last_pfn && !first_pte_in_page(pte));
1086
310a5ab9
DW		 1087 domain_flush_cache(domain, first_pte,
1088 (void *)pte - (void *)first_pte);
59c36286
DW		 1089
1090 } while (start_pfn && start_pfn <= last_pfn);
ba395927
KA		 1091}
1092
3269ee0b
AW		 1093static void dma_pte_free_level(struct dmar_domain *domain, int level,
1094 struct dma_pte *pte, unsigned long pfn,
1095 unsigned long start_pfn, unsigned long last_pfn)
1096{
1097 pfn = max(start_pfn, pfn);
1098 pte = &pte[pfn_level_offset(pfn, level)];
1099
1100 do {
1101 unsigned long level_pfn;
1102 struct dma_pte *level_pte;
1103
1104 if (!dma_pte_present(pte) || dma_pte_superpage(pte))
1105 goto next;
1106
1107 level_pfn = pfn & level_mask(level - 1);
1108 level_pte = phys_to_virt(dma_pte_addr(pte));
1109
1110 if (level > 2)
1111 dma_pte_free_level(domain, level - 1, level_pte,
1112 level_pfn, start_pfn, last_pfn);
1113
1114 /* If range covers entire pagetable, free it */
1115 if (!(start_pfn > level_pfn ||
08336fd2 1116 last_pfn < level_pfn + level_size(level) - 1)) {
3269ee0b
AW		 1117 dma_clear_pte(pte);
1118 domain_flush_cache(domain, pte, sizeof(*pte));
1119 free_pgtable_page(level_pte);
1120 }
1121next:
1122 pfn += level_size(level);
1123 } while (!first_pte_in_page(++pte) && pfn <= last_pfn);
1124}
1125
ba395927
KA		 1126/* free page table pages. last level pte should already be cleared */
1127static void dma_pte_free_pagetable(struct dmar_domain *domain,
d794dc9b
DW		 1128 unsigned long start_pfn,
1129 unsigned long last_pfn)
ba395927 1130{
162d1b10
JL		 1131 BUG_ON(!domain_pfn_supported(domain, start_pfn));
1132 BUG_ON(!domain_pfn_supported(domain, last_pfn));
59c36286 1133 BUG_ON(start_pfn > last_pfn);
ba395927 1134
d41a4adb
JL		 1135 dma_pte_clear_range(domain, start_pfn, last_pfn);
1136
f3a0a52f 1137 /* We don't need lock here; nobody else touches the iova range */
3269ee0b
AW		 1138 dma_pte_free_level(domain, agaw_to_level(domain->agaw),
1139 domain->pgd, 0, start_pfn, last_pfn);
6660c63a 1140
ba395927 1141 /* free pgd */
d794dc9b 1142 if (start_pfn == 0 && last_pfn == DOMAIN_MAX_PFN(domain->gaw)) {
ba395927
KA		 1143 free_pgtable_page(domain->pgd);
1144 domain->pgd = NULL;
1145 }
1146}
1147
ea8ea460
DW		 1148/* When a page at a given level is being unlinked from its parent, we don't
1149 need to *modify* it at all. All we need to do is make a list of all the
1150 pages which can be freed just as soon as we've flushed the IOTLB and we
1151 know the hardware page-walk will no longer touch them.
1152 The 'pte' argument is the *parent* PTE, pointing to the page that is to
1153 be freed. */
1154static struct page *dma_pte_list_pagetables(struct dmar_domain *domain,
1155 int level, struct dma_pte *pte,
1156 struct page *freelist)
1157{
1158 struct page *pg;
1159
1160 pg = pfn_to_page(dma_pte_addr(pte) >> PAGE_SHIFT);
1161 pg->freelist = freelist;
1162 freelist = pg;
1163
1164 if (level == 1)
1165 return freelist;
1166
adeb2590
JL		 1167 pte = page_address(pg);
1168 do {
ea8ea460
DW		 1169 if (dma_pte_present(pte) && !dma_pte_superpage(pte))
1170 freelist = dma_pte_list_pagetables(domain, level - 1,
1171 pte, freelist);
adeb2590
JL		 1172 pte++;
1173 } while (!first_pte_in_page(pte));
ea8ea460
DW		 1174
1175 return freelist;
1176}
1177
1178static struct page *dma_pte_clear_level(struct dmar_domain *domain, int level,
1179 struct dma_pte *pte, unsigned long pfn,
1180 unsigned long start_pfn,
1181 unsigned long last_pfn,
1182 struct page *freelist)
1183{
1184 struct dma_pte *first_pte = NULL, *last_pte = NULL;
1185
1186 pfn = max(start_pfn, pfn);
1187 pte = &pte[pfn_level_offset(pfn, level)];
1188
1189 do {
1190 unsigned long level_pfn;
1191
1192 if (!dma_pte_present(pte))
1193 goto next;
1194
1195 level_pfn = pfn & level_mask(level);
1196
1197 /* If range covers entire pagetable, free it */
1198 if (start_pfn <= level_pfn &&
1199 last_pfn >= level_pfn + level_size(level) - 1) {
1200 /* These suborbinate page tables are going away entirely. Don't
1201 bother to clear them; we're just going to *free* them. */
1202 if (level > 1 && !dma_pte_superpage(pte))
1203 freelist = dma_pte_list_pagetables(domain, level - 1, pte, freelist);
1204
1205 dma_clear_pte(pte);
1206 if (!first_pte)
1207 first_pte = pte;
1208 last_pte = pte;
1209 } else if (level > 1) {
1210 /* Recurse down into a level that isn't *entirely* obsolete */
1211 freelist = dma_pte_clear_level(domain, level - 1,
1212 phys_to_virt(dma_pte_addr(pte)),
1213 level_pfn, start_pfn, last_pfn,
1214 freelist);
1215 }
1216next:
1217 pfn += level_size(level);
1218 } while (!first_pte_in_page(++pte) && pfn <= last_pfn);
1219
1220 if (first_pte)
1221 domain_flush_cache(domain, first_pte,
1222 (void *)++last_pte - (void *)first_pte);
1223
1224 return freelist;
1225}
1226
1227/* We can't just free the pages because the IOMMU may still be walking
1228 the page tables, and may have cached the intermediate levels. The
1229 pages can only be freed after the IOTLB flush has been done. */
b690420a
JR		 1230static struct page *domain_unmap(struct dmar_domain *domain,
1231 unsigned long start_pfn,
1232 unsigned long last_pfn)
ea8ea460 1233{
ea8ea460
DW		 1234 struct page *freelist = NULL;
1235
162d1b10
JL		 1236 BUG_ON(!domain_pfn_supported(domain, start_pfn));
1237 BUG_ON(!domain_pfn_supported(domain, last_pfn));
ea8ea460
DW		 1238 BUG_ON(start_pfn > last_pfn);
1239
1240 /* we don't need lock here; nobody else touches the iova range */
1241 freelist = dma_pte_clear_level(domain, agaw_to_level(domain->agaw),
1242 domain->pgd, 0, start_pfn, last_pfn, NULL);
1243
1244 /* free pgd */
1245 if (start_pfn == 0 && last_pfn == DOMAIN_MAX_PFN(domain->gaw)) {
1246 struct page *pgd_page = virt_to_page(domain->pgd);
1247 pgd_page->freelist = freelist;
1248 freelist = pgd_page;
1249
1250 domain->pgd = NULL;
1251 }
1252
1253 return freelist;
1254}
1255
b690420a 1256static void dma_free_pagelist(struct page *freelist)
ea8ea460
DW		 1257{
1258 struct page *pg;
1259
1260 while ((pg = freelist)) {
1261 freelist = pg->freelist;
1262 free_pgtable_page(page_address(pg));
1263 }
1264}
1265
ba395927
KA		 1266/* iommu handling */
1267static int iommu_alloc_root_entry(struct intel_iommu *iommu)
1268{
1269 struct root_entry *root;
1270 unsigned long flags;
1271
4c923d47 1272 root = (struct root_entry *)alloc_pgtable_page(iommu->node);
ffebeb46 1273 if (!root) {
9f10e5bf 1274 pr_err("Allocating root entry for %s failed\n",
ffebeb46 1275 iommu->name);
ba395927 1276 return -ENOMEM;
ffebeb46 1277 }
ba395927 1278
5b6985ce 1279 __iommu_flush_cache(iommu, root, ROOT_SIZE);
ba395927
KA		 1280
1281 spin_lock_irqsave(&iommu->lock, flags);
1282 iommu->root_entry = root;
1283 spin_unlock_irqrestore(&iommu->lock, flags);
1284
1285 return 0;
1286}
1287
ba395927
KA		 1288static void iommu_set_root_entry(struct intel_iommu *iommu)
1289{
03ecc32c 1290 u64 addr;
c416daa9 1291 u32 sts;
ba395927
KA		 1292 unsigned long flag;
1293
03ecc32c 1294 addr = virt_to_phys(iommu->root_entry);
c83b2f20 1295 if (ecs_enabled(iommu))
03ecc32c 1296 addr |= DMA_RTADDR_RTT;
ba395927 1297
1f5b3c3f 1298 raw_spin_lock_irqsave(&iommu->register_lock, flag);
03ecc32c 1299 dmar_writeq(iommu->reg + DMAR_RTADDR_REG, addr);
ba395927 1300
c416daa9 1301 writel(iommu->gcmd | DMA_GCMD_SRTP, iommu->reg + DMAR_GCMD_REG);
ba395927
KA		 1302
1303 /* Make sure hardware complete it */
1304 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
c416daa9 1305 readl, (sts & DMA_GSTS_RTPS), sts);
ba395927 1306
1f5b3c3f 1307 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
ba395927
KA		 1308}
1309
1310static void iommu_flush_write_buffer(struct intel_iommu *iommu)
1311{
1312 u32 val;
1313 unsigned long flag;
1314
9af88143 1315 if (!rwbf_quirk && !cap_rwbf(iommu->cap))
ba395927 1316 return;
ba395927 1317
1f5b3c3f 1318 raw_spin_lock_irqsave(&iommu->register_lock, flag);
462b60f6 1319 writel(iommu->gcmd | DMA_GCMD_WBF, iommu->reg + DMAR_GCMD_REG);
ba395927
KA		 1320
1321 /* Make sure hardware complete it */
1322 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
c416daa9 1323 readl, (!(val & DMA_GSTS_WBFS)), val);
ba395927 1324
1f5b3c3f 1325 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
ba395927
KA		 1326}
1327
1328/* return value determine if we need a write buffer flush */
4c25a2c1
DW		 1329static void __iommu_flush_context(struct intel_iommu *iommu,
1330 u16 did, u16 source_id, u8 function_mask,
1331 u64 type)
ba395927
KA		 1332{
1333 u64 val = 0;
1334 unsigned long flag;
1335
ba395927
KA		 1336 switch (type) {
1337 case DMA_CCMD_GLOBAL_INVL:
1338 val = DMA_CCMD_GLOBAL_INVL;
1339 break;
1340 case DMA_CCMD_DOMAIN_INVL:
1341 val = DMA_CCMD_DOMAIN_INVL|DMA_CCMD_DID(did);
1342 break;
1343 case DMA_CCMD_DEVICE_INVL:
1344 val = DMA_CCMD_DEVICE_INVL|DMA_CCMD_DID(did)
1345 | DMA_CCMD_SID(source_id) | DMA_CCMD_FM(function_mask);
1346 break;
1347 default:
1348 BUG();
1349 }
1350 val |= DMA_CCMD_ICC;
1351
1f5b3c3f 1352 raw_spin_lock_irqsave(&iommu->register_lock, flag);
ba395927
KA		 1353 dmar_writeq(iommu->reg + DMAR_CCMD_REG, val);
1354
1355 /* Make sure hardware complete it */
1356 IOMMU_WAIT_OP(iommu, DMAR_CCMD_REG,
1357 dmar_readq, (!(val & DMA_CCMD_ICC)), val);
1358
1f5b3c3f 1359 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
ba395927
KA		 1360}
1361
ba395927 1362/* return value determine if we need a write buffer flush */
1f0ef2aa
DW		 1363static void __iommu_flush_iotlb(struct intel_iommu *iommu, u16 did,
1364 u64 addr, unsigned int size_order, u64 type)
ba395927
KA		 1365{
1366 int tlb_offset = ecap_iotlb_offset(iommu->ecap);
1367 u64 val = 0, val_iva = 0;
1368 unsigned long flag;
1369
ba395927
KA		 1370 switch (type) {
1371 case DMA_TLB_GLOBAL_FLUSH:
1372 /* global flush doesn't need set IVA_REG */
1373 val = DMA_TLB_GLOBAL_FLUSH|DMA_TLB_IVT;
1374 break;
1375 case DMA_TLB_DSI_FLUSH:
1376 val = DMA_TLB_DSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
1377 break;
1378 case DMA_TLB_PSI_FLUSH:
1379 val = DMA_TLB_PSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
ea8ea460 1380 /* IH bit is passed in as part of address */
ba395927
KA		 1381 val_iva = size_order | addr;
1382 break;
1383 default:
1384 BUG();
1385 }
1386 /* Note: set drain read/write */
1387#if 0
1388 /*
1389 * This is probably to be super secure.. Looks like we can
1390 * ignore it without any impact.
1391 */
1392 if (cap_read_drain(iommu->cap))
1393 val |= DMA_TLB_READ_DRAIN;
1394#endif
1395 if (cap_write_drain(iommu->cap))
1396 val |= DMA_TLB_WRITE_DRAIN;
1397
1f5b3c3f 1398 raw_spin_lock_irqsave(&iommu->register_lock, flag);
ba395927
KA		 1399 /* Note: Only uses first TLB reg currently */
1400 if (val_iva)
1401 dmar_writeq(iommu->reg + tlb_offset, val_iva);
1402 dmar_writeq(iommu->reg + tlb_offset + 8, val);
1403
1404 /* Make sure hardware complete it */
1405 IOMMU_WAIT_OP(iommu, tlb_offset + 8,
1406 dmar_readq, (!(val & DMA_TLB_IVT)), val);
1407
1f5b3c3f 1408 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
ba395927
KA		 1409
1410 /* check IOTLB invalidation granularity */
1411 if (DMA_TLB_IAIG(val) == 0)
9f10e5bf 1412 pr_err("Flush IOTLB failed\n");
ba395927 1413 if (DMA_TLB_IAIG(val) != DMA_TLB_IIRG(type))
9f10e5bf 1414 pr_debug("TLB flush request %Lx, actual %Lx\n",
5b6985ce
FY		 1415 (unsigned long long)DMA_TLB_IIRG(type),
1416 (unsigned long long)DMA_TLB_IAIG(val));
ba395927
KA		 1417}
1418
64ae892b
DW		 1419static struct device_domain_info *
1420iommu_support_dev_iotlb (struct dmar_domain *domain, struct intel_iommu *iommu,
1421 u8 bus, u8 devfn)
93a23a72 1422{
2f119c78 1423 bool found = false;
93a23a72 1424 struct device_domain_info *info;
0bcb3e28 1425 struct pci_dev *pdev;
93a23a72 1426
55d94043
JR		 1427 assert_spin_locked(&device_domain_lock);
1428
93a23a72
YZ		 1429 if (!ecap_dev_iotlb_support(iommu->ecap))
1430 return NULL;
1431
1432 if (!iommu->qi)
1433 return NULL;
1434
93a23a72 1435 list_for_each_entry(info, &domain->devices, link)
c3b497c6
JL		 1436 if (info->iommu == iommu && info->bus == bus &&
1437 info->devfn == devfn) {
2f119c78 1438 found = true;
93a23a72
YZ		 1439 break;
1440 }
93a23a72 1441
0bcb3e28 1442 if (!found || !info->dev || !dev_is_pci(info->dev))
93a23a72
YZ		 1443 return NULL;
1444
0bcb3e28
DW		 1445 pdev = to_pci_dev(info->dev);
1446
1447 if (!pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_ATS))
93a23a72
YZ		 1448 return NULL;
1449
0bcb3e28 1450 if (!dmar_find_matched_atsr_unit(pdev))
93a23a72
YZ		 1451 return NULL;
1452
93a23a72
YZ		 1453 return info;
1454}
1455
1456static void iommu_enable_dev_iotlb(struct device_domain_info *info)
ba395927 1457{
fb0cc3aa
BH		 1458 struct pci_dev *pdev;
1459
0bcb3e28 1460 if (!info || !dev_is_pci(info->dev))
93a23a72
YZ		 1461 return;
1462
fb0cc3aa
BH		 1463 pdev = to_pci_dev(info->dev);
1464 if (pci_enable_ats(pdev, VTD_PAGE_SHIFT))
1465 return;
1466
1467 info->ats.enabled = 1;
1468 info->ats.qdep = pci_ats_queue_depth(pdev);
93a23a72
YZ		 1469}
1470
1471static void iommu_disable_dev_iotlb(struct device_domain_info *info)
1472{
fb0cc3aa 1473 if (!info->ats.enabled)
93a23a72
YZ		 1474 return;
1475
0bcb3e28 1476 pci_disable_ats(to_pci_dev(info->dev));
fb0cc3aa 1477 info->ats.enabled = 0;
93a23a72
YZ		 1478}
1479
1480static void iommu_flush_dev_iotlb(struct dmar_domain *domain,
1481 u64 addr, unsigned mask)
1482{
1483 u16 sid, qdep;
1484 unsigned long flags;
1485 struct device_domain_info *info;
1486
1487 spin_lock_irqsave(&device_domain_lock, flags);
1488 list_for_each_entry(info, &domain->devices, link) {
fb0cc3aa 1489 if (!info->ats.enabled)
93a23a72
YZ		 1490 continue;
1491
1492 sid = info->bus << 8 | info->devfn;
fb0cc3aa 1493 qdep = info->ats.qdep;
93a23a72
YZ		 1494 qi_flush_dev_iotlb(info->iommu, sid, qdep, addr, mask);
1495 }
1496 spin_unlock_irqrestore(&device_domain_lock, flags);
1497}
1498
a1ddcbe9
JR		 1499static void iommu_flush_iotlb_psi(struct intel_iommu *iommu,
1500 struct dmar_domain *domain,
1501 unsigned long pfn, unsigned int pages,
1502 int ih, int map)
ba395927 1503{
9dd2fe89 1504 unsigned int mask = ilog2(__roundup_pow_of_two(pages));
03d6a246 1505 uint64_t addr = (uint64_t)pfn << VTD_PAGE_SHIFT;
a1ddcbe9 1506 u16 did = domain->iommu_did[iommu->seq_id];
ba395927 1507
ba395927
KA		 1508 BUG_ON(pages == 0);
1509
ea8ea460
DW		 1510 if (ih)
1511 ih = 1 << 6;
ba395927 1512 /*
9dd2fe89
YZ		 1513 * Fallback to domain selective flush if no PSI support or the size is
1514 * too big.
ba395927
KA		 1515 * PSI requires page size to be 2 ^ x, and the base address is naturally
1516 * aligned to the size
1517 */
9dd2fe89
YZ		 1518 if (!cap_pgsel_inv(iommu->cap) || mask > cap_max_amask_val(iommu->cap))
1519 iommu->flush.flush_iotlb(iommu, did, 0, 0,
1f0ef2aa 1520 DMA_TLB_DSI_FLUSH);
9dd2fe89 1521 else
ea8ea460 1522 iommu->flush.flush_iotlb(iommu, did, addr | ih, mask,
9dd2fe89 1523 DMA_TLB_PSI_FLUSH);
bf92df30
YZ		 1524
1525 /*
82653633
NA		 1526 * In caching mode, changes of pages from non-present to present require
1527 * flush. However, device IOTLB doesn't need to be flushed in this case.
bf92df30 1528 */
82653633 1529 if (!cap_caching_mode(iommu->cap) || !map)
9452d5bf
JR		 1530 iommu_flush_dev_iotlb(get_iommu_domain(iommu, did),
1531 addr, mask);
ba395927
KA		 1532}
1533
f8bab735 1534static void iommu_disable_protect_mem_regions(struct intel_iommu *iommu)
1535{
1536 u32 pmen;
1537 unsigned long flags;
1538
1f5b3c3f 1539 raw_spin_lock_irqsave(&iommu->register_lock, flags);
f8bab735 1540 pmen = readl(iommu->reg + DMAR_PMEN_REG);
1541 pmen &= ~DMA_PMEN_EPM;
1542 writel(pmen, iommu->reg + DMAR_PMEN_REG);
1543
1544 /* wait for the protected region status bit to clear */
1545 IOMMU_WAIT_OP(iommu, DMAR_PMEN_REG,
1546 readl, !(pmen & DMA_PMEN_PRS), pmen);
1547
1f5b3c3f 1548 raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
f8bab735 1549}
1550
2a41ccee 1551static void iommu_enable_translation(struct intel_iommu *iommu)
ba395927
KA		 1552{
1553 u32 sts;
1554 unsigned long flags;
1555
1f5b3c3f 1556 raw_spin_lock_irqsave(&iommu->register_lock, flags);
c416daa9
DW		 1557 iommu->gcmd |= DMA_GCMD_TE;
1558 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
ba395927
KA		 1559
1560 /* Make sure hardware complete it */
1561 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
c416daa9 1562 readl, (sts & DMA_GSTS_TES), sts);
ba395927 1563
1f5b3c3f 1564 raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
ba395927
KA		 1565}
1566
2a41ccee 1567static void iommu_disable_translation(struct intel_iommu *iommu)
ba395927
KA		 1568{
1569 u32 sts;
1570 unsigned long flag;
1571
1f5b3c3f 1572 raw_spin_lock_irqsave(&iommu->register_lock, flag);
ba395927
KA		 1573 iommu->gcmd &= ~DMA_GCMD_TE;
1574 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1575
1576 /* Make sure hardware complete it */
1577 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
c416daa9 1578 readl, (!(sts & DMA_GSTS_TES)), sts);
ba395927 1579
1f5b3c3f 1580 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
ba395927
KA		 1581}
1582
3460a6d9 1583
ba395927
KA		 1584static int iommu_init_domains(struct intel_iommu *iommu)
1585{
8bf47816
JR		 1586 u32 ndomains, nlongs;
1587 size_t size;
ba395927
KA		 1588
1589 ndomains = cap_ndoms(iommu->cap);
8bf47816 1590 pr_debug("%s: Number of Domains supported <%d>\n",
9f10e5bf 1591 iommu->name, ndomains);
ba395927
KA		 1592 nlongs = BITS_TO_LONGS(ndomains);
1593
94a91b50
DD		 1594 spin_lock_init(&iommu->lock);
1595
ba395927
KA		 1596 iommu->domain_ids = kcalloc(nlongs, sizeof(unsigned long), GFP_KERNEL);
1597 if (!iommu->domain_ids) {
9f10e5bf
JR		 1598 pr_err("%s: Allocating domain id array failed\n",
1599 iommu->name);
ba395927
KA		 1600 return -ENOMEM;
1601 }
8bf47816
JR		 1602
1603 size = ((ndomains >> 8) + 1) * sizeof(struct dmar_domain **);
1604 iommu->domains = kzalloc(size, GFP_KERNEL);
1605
1606 if (iommu->domains) {
1607 size = 256 * sizeof(struct dmar_domain *);
1608 iommu->domains[0] = kzalloc(size, GFP_KERNEL);
1609 }
1610
1611 if (!iommu->domains || !iommu->domains[0]) {
9f10e5bf
JR		 1612 pr_err("%s: Allocating domain array failed\n",
1613 iommu->name);
852bdb04 1614 kfree(iommu->domain_ids);
8bf47816 1615 kfree(iommu->domains);
852bdb04 1616 iommu->domain_ids = NULL;
8bf47816 1617 iommu->domains = NULL;
ba395927
KA		 1618 return -ENOMEM;
1619 }
1620
8bf47816
JR		 1621
1622
ba395927 1623 /*
c0e8a6c8
JR		 1624 * If Caching mode is set, then invalid translations are tagged
1625 * with domain-id 0, hence we need to pre-allocate it. We also
1626 * use domain-id 0 as a marker for non-allocated domain-id, so
1627 * make sure it is not used for a real domain.
ba395927 1628 */
c0e8a6c8
JR		 1629 set_bit(0, iommu->domain_ids);
1630
ba395927
KA		 1631 return 0;
1632}
ba395927 1633
ffebeb46 1634static void disable_dmar_iommu(struct intel_iommu *iommu)
ba395927 1635{
29a27719 1636 struct device_domain_info *info, *tmp;
55d94043 1637 unsigned long flags;
ba395927 1638
29a27719
JR		 1639 if (!iommu->domains || !iommu->domain_ids)
1640 return;
a4eaa86c 1641
55d94043 1642 spin_lock_irqsave(&device_domain_lock, flags);
29a27719
JR		 1643 list_for_each_entry_safe(info, tmp, &device_domain_list, global) {
1644 struct dmar_domain *domain;
1645
1646 if (info->iommu != iommu)
1647 continue;
1648
1649 if (!info->dev || !info->domain)
1650 continue;
1651
1652 domain = info->domain;
1653
e6de0f8d 1654 dmar_remove_one_dev_info(domain, info->dev);
29a27719
JR		 1655
1656 if (!domain_type_is_vm_or_si(domain))
1657 domain_exit(domain);
ba395927 1658 }
55d94043 1659 spin_unlock_irqrestore(&device_domain_lock, flags);
ba395927
KA		 1660
1661 if (iommu->gcmd & DMA_GCMD_TE)
1662 iommu_disable_translation(iommu);
ffebeb46 1663}
ba395927 1664
ffebeb46
JL		 1665static void free_dmar_iommu(struct intel_iommu *iommu)
1666{
1667 if ((iommu->domains) && (iommu->domain_ids)) {
8bf47816
JR		 1668 int elems = (cap_ndoms(iommu->cap) >> 8) + 1;
1669 int i;
1670
1671 for (i = 0; i < elems; i++)
1672 kfree(iommu->domains[i]);
ffebeb46
JL		 1673 kfree(iommu->domains);
1674 kfree(iommu->domain_ids);
1675 iommu->domains = NULL;
1676 iommu->domain_ids = NULL;
1677 }
ba395927 1678
d9630fe9
WH		 1679 g_iommus[iommu->seq_id] = NULL;
1680
ba395927
KA		 1681 /* free context mapping */
1682 free_context_table(iommu);
8a94ade4
DW		 1683
1684#ifdef CONFIG_INTEL_IOMMU_SVM
1685 if (pasid_enabled(iommu))
1686 intel_svm_free_pasid_tables(iommu);
1687#endif
ba395927
KA		 1688}
1689
ab8dfe25 1690static struct dmar_domain *alloc_domain(int flags)
ba395927 1691{
ba395927 1692 struct dmar_domain *domain;
ba395927
KA		 1693
1694 domain = alloc_domain_mem();
1695 if (!domain)
1696 return NULL;
1697
ab8dfe25 1698 memset(domain, 0, sizeof(*domain));
4c923d47 1699 domain->nid = -1;
ab8dfe25 1700 domain->flags = flags;
92d03cc8 1701 INIT_LIST_HEAD(&domain->devices);
2c2e2c38
FY		 1702
1703 return domain;
1704}
1705
d160aca5
JR		 1706/* Must be called with iommu->lock */
1707static int domain_attach_iommu(struct dmar_domain *domain,
fb170fb4
JL		 1708 struct intel_iommu *iommu)
1709{
44bde614 1710 unsigned long ndomains;
55d94043 1711 int num;
44bde614 1712
55d94043 1713 assert_spin_locked(&device_domain_lock);
d160aca5 1714 assert_spin_locked(&iommu->lock);
ba395927 1715
29a27719
JR		 1716 domain->iommu_refcnt[iommu->seq_id] += 1;
1717 domain->iommu_count += 1;
1718 if (domain->iommu_refcnt[iommu->seq_id] == 1) {
fb170fb4 1719 ndomains = cap_ndoms(iommu->cap);
d160aca5
JR		 1720 num = find_first_zero_bit(iommu->domain_ids, ndomains);
1721
1722 if (num >= ndomains) {
1723 pr_err("%s: No free domain ids\n", iommu->name);
1724 domain->iommu_refcnt[iommu->seq_id] -= 1;
1725 domain->iommu_count -= 1;
55d94043 1726 return -ENOSPC;
2c2e2c38 1727 }
ba395927 1728
d160aca5
JR		 1729 set_bit(num, iommu->domain_ids);
1730 set_iommu_domain(iommu, num, domain);
1731
1732 domain->iommu_did[iommu->seq_id] = num;
1733 domain->nid = iommu->node;
fb170fb4 1734
fb170fb4
JL		 1735 domain_update_iommu_cap(domain);
1736 }
d160aca5 1737
55d94043 1738 return 0;
fb170fb4
JL		 1739}
1740
1741static int domain_detach_iommu(struct dmar_domain *domain,
1742 struct intel_iommu *iommu)
1743{
d160aca5 1744 int num, count = INT_MAX;
d160aca5 1745
55d94043 1746 assert_spin_locked(&device_domain_lock);
d160aca5 1747 assert_spin_locked(&iommu->lock);
fb170fb4 1748
29a27719
JR		 1749 domain->iommu_refcnt[iommu->seq_id] -= 1;
1750 count = --domain->iommu_count;
1751 if (domain->iommu_refcnt[iommu->seq_id] == 0) {
d160aca5
JR		 1752 num = domain->iommu_did[iommu->seq_id];
1753 clear_bit(num, iommu->domain_ids);
1754 set_iommu_domain(iommu, num, NULL);
fb170fb4 1755
fb170fb4 1756 domain_update_iommu_cap(domain);
c0e8a6c8 1757 domain->iommu_did[iommu->seq_id] = 0;
fb170fb4 1758 }
fb170fb4
JL		 1759
1760 return count;
1761}
1762
ba395927 1763static struct iova_domain reserved_iova_list;
8a443df4 1764static struct lock_class_key reserved_rbtree_key;
ba395927 1765
51a63e67 1766static int dmar_init_reserved_ranges(void)
ba395927
KA		 1767{
1768 struct pci_dev *pdev = NULL;
1769 struct iova *iova;
1770 int i;
ba395927 1771
0fb5fe87
RM		 1772 init_iova_domain(&reserved_iova_list, VTD_PAGE_SIZE, IOVA_START_PFN,
1773 DMA_32BIT_PFN);
ba395927 1774
8a443df4
MG		 1775 lockdep_set_class(&reserved_iova_list.iova_rbtree_lock,
1776 &reserved_rbtree_key);
1777
ba395927
KA		 1778 /* IOAPIC ranges shouldn't be accessed by DMA */
1779 iova = reserve_iova(&reserved_iova_list, IOVA_PFN(IOAPIC_RANGE_START),
1780 IOVA_PFN(IOAPIC_RANGE_END));
51a63e67 1781 if (!iova) {
9f10e5bf 1782 pr_err("Reserve IOAPIC range failed\n");
51a63e67
JC		 1783 return -ENODEV;
1784 }
ba395927
KA		 1785
1786 /* Reserve all PCI MMIO to avoid peer-to-peer access */
1787 for_each_pci_dev(pdev) {
1788 struct resource *r;
1789
1790 for (i = 0; i < PCI_NUM_RESOURCES; i++) {
1791 r = &pdev->resource[i];
1792 if (!r->flags || !(r->flags & IORESOURCE_MEM))
1793 continue;
1a4a4551
DW		 1794 iova = reserve_iova(&reserved_iova_list,
1795 IOVA_PFN(r->start),
1796 IOVA_PFN(r->end));
51a63e67 1797 if (!iova) {
9f10e5bf 1798 pr_err("Reserve iova failed\n");
51a63e67
JC		 1799 return -ENODEV;
1800 }
ba395927
KA		 1801 }
1802 }
51a63e67 1803 return 0;
ba395927
KA		 1804}
1805
1806static void domain_reserve_special_ranges(struct dmar_domain *domain)
1807{
1808 copy_reserved_iova(&reserved_iova_list, &domain->iovad);
1809}
1810
1811static inline int guestwidth_to_adjustwidth(int gaw)
1812{
1813 int agaw;
1814 int r = (gaw - 12) % 9;
1815
1816 if (r == 0)
1817 agaw = gaw;
1818 else
1819 agaw = gaw + 9 - r;
1820 if (agaw > 64)
1821 agaw = 64;
1822 return agaw;
1823}
1824
dc534b25
JR		 1825static int domain_init(struct dmar_domain *domain, struct intel_iommu *iommu,
1826 int guest_width)
ba395927 1827{
ba395927
KA		 1828 int adjust_width, agaw;
1829 unsigned long sagaw;
1830
0fb5fe87
RM		 1831 init_iova_domain(&domain->iovad, VTD_PAGE_SIZE, IOVA_START_PFN,
1832 DMA_32BIT_PFN);
ba395927
KA		 1833 domain_reserve_special_ranges(domain);
1834
1835 /* calculate AGAW */
ba395927
KA		 1836 if (guest_width > cap_mgaw(iommu->cap))
1837 guest_width = cap_mgaw(iommu->cap);
1838 domain->gaw = guest_width;
1839 adjust_width = guestwidth_to_adjustwidth(guest_width);
1840 agaw = width_to_agaw(adjust_width);
1841 sagaw = cap_sagaw(iommu->cap);
1842 if (!test_bit(agaw, &sagaw)) {
1843 /* hardware doesn't support it, choose a bigger one */
9f10e5bf 1844 pr_debug("Hardware doesn't support agaw %d\n", agaw);
ba395927
KA		 1845 agaw = find_next_bit(&sagaw, 5, agaw);
1846 if (agaw >= 5)
1847 return -ENODEV;
1848 }
1849 domain->agaw = agaw;
ba395927 1850
8e604097
WH		 1851 if (ecap_coherent(iommu->ecap))
1852 domain->iommu_coherency = 1;
1853 else
1854 domain->iommu_coherency = 0;
1855
58c610bd
SY		 1856 if (ecap_sc_support(iommu->ecap))
1857 domain->iommu_snooping = 1;
1858 else
1859 domain->iommu_snooping = 0;
1860
214e39aa
DW		 1861 if (intel_iommu_superpage)
1862 domain->iommu_superpage = fls(cap_super_page_val(iommu->cap));
1863 else
1864 domain->iommu_superpage = 0;
1865
4c923d47 1866 domain->nid = iommu->node;
c7151a8d 1867
ba395927 1868 /* always allocate the top pgd */
4c923d47 1869 domain->pgd = (struct dma_pte *)alloc_pgtable_page(domain->nid);
ba395927
KA		 1870 if (!domain->pgd)
1871 return -ENOMEM;
5b6985ce 1872 __iommu_flush_cache(iommu, domain->pgd, PAGE_SIZE);
ba395927
KA		 1873 return 0;
1874}
1875
1876static void domain_exit(struct dmar_domain *domain)
1877{
ea8ea460 1878 struct page *freelist = NULL;
ba395927
KA		 1879
1880 /* Domain 0 is reserved, so dont process it */
1881 if (!domain)
1882 return;
1883
7b668357
AW		 1884 /* Flush any lazy unmaps that may reference this domain */
1885 if (!intel_iommu_strict)
1886 flush_unmaps_timeout(0);
1887
d160aca5
JR		 1888 /* Remove associated devices and clear attached or cached domains */
1889 rcu_read_lock();
ba395927 1890 domain_remove_dev_info(domain);
d160aca5 1891 rcu_read_unlock();
92d03cc8 1892
ba395927
KA		 1893 /* destroy iovas */
1894 put_iova_domain(&domain->iovad);
ba395927 1895
ea8ea460 1896 freelist = domain_unmap(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
ba395927 1897
ea8ea460
DW		 1898 dma_free_pagelist(freelist);
1899
ba395927
KA		 1900 free_domain_mem(domain);
1901}
1902
64ae892b
DW		 1903static int domain_context_mapping_one(struct dmar_domain *domain,
1904 struct intel_iommu *iommu,
28ccce0d 1905 u8 bus, u8 devfn)
ba395927 1906{
c6c2cebd 1907 u16 did = domain->iommu_did[iommu->seq_id];
28ccce0d
JR		 1908 int translation = CONTEXT_TT_MULTI_LEVEL;
1909 struct device_domain_info *info = NULL;
ba395927 1910 struct context_entry *context;
ba395927 1911 unsigned long flags;
ea6606b0 1912 struct dma_pte *pgd;
55d94043 1913 int ret, agaw;
28ccce0d 1914
c6c2cebd
JR		 1915 WARN_ON(did == 0);
1916
28ccce0d
JR		 1917 if (hw_pass_through && domain_type_is_si(domain))
1918 translation = CONTEXT_TT_PASS_THROUGH;
ba395927
KA		 1919
1920 pr_debug("Set context mapping for %02x:%02x.%d\n",
1921 bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
4ed0d3e6 1922
ba395927 1923 BUG_ON(!domain->pgd);
5331fe6f 1924
55d94043
JR		 1925 spin_lock_irqsave(&device_domain_lock, flags);
1926 spin_lock(&iommu->lock);
1927
1928 ret = -ENOMEM;
03ecc32c 1929 context = iommu_context_addr(iommu, bus, devfn, 1);
ba395927 1930 if (!context)
55d94043 1931 goto out_unlock;
ba395927 1932
55d94043
JR		 1933 ret = 0;
1934 if (context_present(context))
1935 goto out_unlock;
cf484d0e 1936
ea6606b0
WH		 1937 pgd = domain->pgd;
1938
de24e553 1939 context_clear_entry(context);
c6c2cebd 1940 context_set_domain_id(context, did);
ea6606b0 1941
de24e553
JR		 1942 /*
1943 * Skip top levels of page tables for iommu which has less agaw
1944 * than default. Unnecessary for PT mode.
1945 */
93a23a72 1946 if (translation != CONTEXT_TT_PASS_THROUGH) {
de24e553 1947 for (agaw = domain->agaw; agaw != iommu->agaw; agaw--) {
55d94043 1948 ret = -ENOMEM;
de24e553 1949 pgd = phys_to_virt(dma_pte_addr(pgd));
55d94043
JR		 1950 if (!dma_pte_present(pgd))
1951 goto out_unlock;
ea6606b0 1952 }
4ed0d3e6 1953
64ae892b 1954 info = iommu_support_dev_iotlb(domain, iommu, bus, devfn);
93a23a72
YZ		 1955 translation = info ? CONTEXT_TT_DEV_IOTLB :
1956 CONTEXT_TT_MULTI_LEVEL;
de24e553 1957
93a23a72
YZ		 1958 context_set_address_root(context, virt_to_phys(pgd));
1959 context_set_address_width(context, iommu->agaw);
de24e553
JR		 1960 } else {
1961 /*
1962 * In pass through mode, AW must be programmed to
1963 * indicate the largest AGAW value supported by
1964 * hardware. And ASR is ignored by hardware.
1965 */
1966 context_set_address_width(context, iommu->msagaw);
93a23a72 1967 }
4ed0d3e6
FY		 1968
1969 context_set_translation_type(context, translation);
c07e7d21
MM		 1970 context_set_fault_enable(context);
1971 context_set_present(context);
5331fe6f 1972 domain_flush_cache(domain, context, sizeof(*context));
ba395927 1973
4c25a2c1
DW		 1974 /*
1975 * It's a non-present to present mapping. If hardware doesn't cache
1976 * non-present entry we only need to flush the write-buffer. If the
1977 * _does_ cache non-present entries, then it does so in the special
1978 * domain #0, which we have to flush:
1979 */
1980 if (cap_caching_mode(iommu->cap)) {
1981 iommu->flush.flush_context(iommu, 0,
1982 (((u16)bus) << 8) | devfn,
1983 DMA_CCMD_MASK_NOBIT,
1984 DMA_CCMD_DEVICE_INVL);
c6c2cebd 1985 iommu->flush.flush_iotlb(iommu, did, 0, 0, DMA_TLB_DSI_FLUSH);
4c25a2c1 1986 } else {
ba395927 1987 iommu_flush_write_buffer(iommu);
4c25a2c1 1988 }
93a23a72 1989 iommu_enable_dev_iotlb(info);
c7151a8d 1990
55d94043
JR		 1991 ret = 0;
1992
1993out_unlock:
1994 spin_unlock(&iommu->lock);
1995 spin_unlock_irqrestore(&device_domain_lock, flags);
fb170fb4 1996
ba395927
KA		 1997 return 0;
1998}
1999
579305f7
AW		 2000struct domain_context_mapping_data {
2001 struct dmar_domain *domain;
2002 struct intel_iommu *iommu;
579305f7
AW		 2003};
2004
2005static int domain_context_mapping_cb(struct pci_dev *pdev,
2006 u16 alias, void *opaque)
2007{
2008 struct domain_context_mapping_data *data = opaque;
2009
2010 return domain_context_mapping_one(data->domain, data->iommu,
28ccce0d 2011 PCI_BUS_NUM(alias), alias & 0xff);
579305f7
AW		 2012}
2013
ba395927 2014static int
28ccce0d 2015domain_context_mapping(struct dmar_domain *domain, struct device *dev)
ba395927 2016{
64ae892b 2017 struct intel_iommu *iommu;
156baca8 2018 u8 bus, devfn;
579305f7 2019 struct domain_context_mapping_data data;
64ae892b 2020
e1f167f3 2021 iommu = device_to_iommu(dev, &bus, &devfn);
64ae892b
DW		 2022 if (!iommu)
2023 return -ENODEV;
ba395927 2024
579305f7 2025 if (!dev_is_pci(dev))
28ccce0d 2026 return domain_context_mapping_one(domain, iommu, bus, devfn);
579305f7
AW		 2027
2028 data.domain = domain;
2029 data.iommu = iommu;
579305f7
AW		 2030
2031 return pci_for_each_dma_alias(to_pci_dev(dev),
2032 &domain_context_mapping_cb, &data);
2033}
2034
2035static int domain_context_mapped_cb(struct pci_dev *pdev,
2036 u16 alias, void *opaque)
2037{
2038 struct intel_iommu *iommu = opaque;
2039
2040 return !device_context_mapped(iommu, PCI_BUS_NUM(alias), alias & 0xff);
ba395927
KA		 2041}
2042
e1f167f3 2043static int domain_context_mapped(struct device *dev)
ba395927 2044{
5331fe6f 2045 struct intel_iommu *iommu;
156baca8 2046 u8 bus, devfn;
5331fe6f 2047
e1f167f3 2048 iommu = device_to_iommu(dev, &bus, &devfn);
5331fe6f
WH		 2049 if (!iommu)
2050 return -ENODEV;
ba395927 2051
579305f7
AW		 2052 if (!dev_is_pci(dev))
2053 return device_context_mapped(iommu, bus, devfn);
e1f167f3 2054
579305f7
AW		 2055 return !pci_for_each_dma_alias(to_pci_dev(dev),
2056 domain_context_mapped_cb, iommu);
ba395927
KA		 2057}
2058
f532959b
FY		 2059/* Returns a number of VTD pages, but aligned to MM page size */
2060static inline unsigned long aligned_nrpages(unsigned long host_addr,
2061 size_t size)
2062{
2063 host_addr &= ~PAGE_MASK;
2064 return PAGE_ALIGN(host_addr + size) >> VTD_PAGE_SHIFT;
2065}
2066
6dd9a7c7
YS		 2067/* Return largest possible superpage level for a given mapping */
2068static inline int hardware_largepage_caps(struct dmar_domain *domain,
2069 unsigned long iov_pfn,
2070 unsigned long phy_pfn,
2071 unsigned long pages)
2072{
2073 int support, level = 1;
2074 unsigned long pfnmerge;
2075
2076 support = domain->iommu_superpage;
2077
2078 /* To use a large page, the virtual *and* physical addresses
2079 must be aligned to 2MiB/1GiB/etc. Lower bits set in either
2080 of them will mean we have to use smaller pages. So just
2081 merge them and check both at once. */
2082 pfnmerge = iov_pfn | phy_pfn;
2083
2084 while (support && !(pfnmerge & ~VTD_STRIDE_MASK)) {
2085 pages >>= VTD_STRIDE_SHIFT;
2086 if (!pages)
2087 break;
2088 pfnmerge >>= VTD_STRIDE_SHIFT;
2089 level++;
2090 support--;
2091 }
2092 return level;
2093}
2094
9051aa02
DW		 2095static int __domain_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
2096 struct scatterlist *sg, unsigned long phys_pfn,
2097 unsigned long nr_pages, int prot)
e1605495
DW		 2098{
2099 struct dma_pte *first_pte = NULL, *pte = NULL;
9051aa02 2100 phys_addr_t uninitialized_var(pteval);
cc4f14aa 2101 unsigned long sg_res = 0;
6dd9a7c7
YS		 2102 unsigned int largepage_lvl = 0;
2103 unsigned long lvl_pages = 0;
e1605495 2104
162d1b10 2105 BUG_ON(!domain_pfn_supported(domain, iov_pfn + nr_pages - 1));
e1605495
DW		 2106
2107 if ((prot & (DMA_PTE_READ|DMA_PTE_WRITE)) == 0)
2108 return -EINVAL;
2109
2110 prot &= DMA_PTE_READ | DMA_PTE_WRITE | DMA_PTE_SNP;
2111
cc4f14aa
JL		 2112 if (!sg) {
2113 sg_res = nr_pages;
9051aa02
DW		 2114 pteval = ((phys_addr_t)phys_pfn << VTD_PAGE_SHIFT) | prot;
2115 }
2116
6dd9a7c7 2117 while (nr_pages > 0) {
c85994e4
DW		 2118 uint64_t tmp;
2119
e1605495 2120 if (!sg_res) {
f532959b 2121 sg_res = aligned_nrpages(sg->offset, sg->length);
e1605495
DW		 2122 sg->dma_address = ((dma_addr_t)iov_pfn << VTD_PAGE_SHIFT) + sg->offset;
2123 sg->dma_length = sg->length;
db0fa0cb 2124 pteval = (sg_phys(sg) & PAGE_MASK) | prot;
6dd9a7c7 2125 phys_pfn = pteval >> VTD_PAGE_SHIFT;
e1605495 2126 }
6dd9a7c7 2127
e1605495 2128 if (!pte) {
6dd9a7c7
YS		 2129 largepage_lvl = hardware_largepage_caps(domain, iov_pfn, phys_pfn, sg_res);
2130
5cf0a76f 2131 first_pte = pte = pfn_to_dma_pte(domain, iov_pfn, &largepage_lvl);
e1605495
DW		 2132 if (!pte)
2133 return -ENOMEM;
6dd9a7c7 2134 /* It is large page*/
6491d4d0 2135 if (largepage_lvl > 1) {
ba2374fd
CZ		 2136 unsigned long nr_superpages, end_pfn;
2137
6dd9a7c7 2138 pteval |= DMA_PTE_LARGE_PAGE;
d41a4adb 2139 lvl_pages = lvl_to_nr_pages(largepage_lvl);
ba2374fd
CZ		 2140
2141 nr_superpages = sg_res / lvl_pages;
2142 end_pfn = iov_pfn + nr_superpages * lvl_pages - 1;
2143
d41a4adb
JL		 2144 /*
2145 * Ensure that old small page tables are
ba2374fd 2146 * removed to make room for superpage(s).
d41a4adb 2147 */
ba2374fd 2148 dma_pte_free_pagetable(domain, iov_pfn, end_pfn);
6491d4d0 2149 } else {
6dd9a7c7 2150 pteval &= ~(uint64_t)DMA_PTE_LARGE_PAGE;
6491d4d0 2151 }
6dd9a7c7 2152
e1605495
DW		 2153 }
2154 /* We don't need lock here, nobody else
2155 * touches the iova range
2156 */
7766a3fb 2157 tmp = cmpxchg64_local(&pte->val, 0ULL, pteval);
c85994e4 2158 if (tmp) {
1bf20f0d 2159 static int dumps = 5;
9f10e5bf
JR		 2160 pr_crit("ERROR: DMA PTE for vPFN 0x%lx already set (to %llx not %llx)\n",
2161 iov_pfn, tmp, (unsigned long long)pteval);
1bf20f0d
DW		 2162 if (dumps) {
2163 dumps--;
2164 debug_dma_dump_mappings(NULL);
2165 }
2166 WARN_ON(1);
2167 }
6dd9a7c7
YS		 2168
2169 lvl_pages = lvl_to_nr_pages(largepage_lvl);
2170
2171 BUG_ON(nr_pages < lvl_pages);
2172 BUG_ON(sg_res < lvl_pages);
2173
2174 nr_pages -= lvl_pages;
2175 iov_pfn += lvl_pages;
2176 phys_pfn += lvl_pages;
2177 pteval += lvl_pages * VTD_PAGE_SIZE;
2178 sg_res -= lvl_pages;
2179
2180 /* If the next PTE would be the first in a new page, then we
2181 need to flush the cache on the entries we've just written.
2182 And then we'll need to recalculate 'pte', so clear it and
2183 let it get set again in the if (!pte) block above.
2184
2185 If we're done (!nr_pages) we need to flush the cache too.
2186
2187 Also if we've been setting superpages, we may need to
2188 recalculate 'pte' and switch back to smaller pages for the
2189 end of the mapping, if the trailing size is not enough to
2190 use another superpage (i.e. sg_res < lvl_pages). */
e1605495 2191 pte++;
6dd9a7c7
YS		 2192 if (!nr_pages || first_pte_in_page(pte) ||
2193 (largepage_lvl > 1 && sg_res < lvl_pages)) {
e1605495
DW		 2194 domain_flush_cache(domain, first_pte,
2195 (void *)pte - (void *)first_pte);
2196 pte = NULL;
2197 }
6dd9a7c7
YS		 2198
2199 if (!sg_res && nr_pages)
e1605495
DW		 2200 sg = sg_next(sg);
2201 }
2202 return 0;
2203}
2204
9051aa02
DW		 2205static inline int domain_sg_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
2206 struct scatterlist *sg, unsigned long nr_pages,
2207 int prot)
ba395927 2208{
9051aa02
DW		 2209 return __domain_mapping(domain, iov_pfn, sg, 0, nr_pages, prot);
2210}
6f6a00e4 2211
9051aa02
DW		 2212static inline int domain_pfn_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
2213 unsigned long phys_pfn, unsigned long nr_pages,
2214 int prot)
2215{
2216 return __domain_mapping(domain, iov_pfn, NULL, phys_pfn, nr_pages, prot);
ba395927
KA		 2217}
2218
2452d9db 2219static void domain_context_clear_one(struct intel_iommu *iommu, u8 bus, u8 devfn)
ba395927 2220{
c7151a8d
WH		 2221 if (!iommu)
2222 return;
8c11e798
WH		 2223
2224 clear_context_table(iommu, bus, devfn);
2225 iommu->flush.flush_context(iommu, 0, 0, 0,
4c25a2c1 2226 DMA_CCMD_GLOBAL_INVL);
1f0ef2aa 2227 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
ba395927
KA		 2228}
2229
109b9b04
DW		 2230static inline void unlink_domain_info(struct device_domain_info *info)
2231{
2232 assert_spin_locked(&device_domain_lock);
2233 list_del(&info->link);
2234 list_del(&info->global);
2235 if (info->dev)
0bcb3e28 2236 info->dev->archdata.iommu = NULL;
109b9b04
DW		 2237}
2238
ba395927
KA		 2239static void domain_remove_dev_info(struct dmar_domain *domain)
2240{
3a74ca01 2241 struct device_domain_info *info, *tmp;
fb170fb4 2242 unsigned long flags;
ba395927
KA		 2243
2244 spin_lock_irqsave(&device_domain_lock, flags);
76f45fe3 2245 list_for_each_entry_safe(info, tmp, &domain->devices, link)
127c7615 2246 __dmar_remove_one_dev_info(info);
ba395927
KA		 2247 spin_unlock_irqrestore(&device_domain_lock, flags);
2248}
2249
2250/*
2251 * find_domain
1525a29a 2252 * Note: we use struct device->archdata.iommu stores the info
ba395927 2253 */
1525a29a 2254static struct dmar_domain *find_domain(struct device *dev)
ba395927
KA		 2255{
2256 struct device_domain_info *info;
2257
2258 /* No lock here, assumes no domain exit in normal case */
1525a29a 2259 info = dev->archdata.iommu;
ba395927
KA		 2260 if (info)
2261 return info->domain;
2262 return NULL;
2263}
2264
5a8f40e8 2265static inline struct device_domain_info *
745f2586
JL		 2266dmar_search_domain_by_dev_info(int segment, int bus, int devfn)
2267{
2268 struct device_domain_info *info;
2269
2270 list_for_each_entry(info, &device_domain_list, global)
41e80dca 2271 if (info->iommu->segment == segment && info->bus == bus &&
745f2586 2272 info->devfn == devfn)
5a8f40e8 2273 return info;
745f2586
JL		 2274
2275 return NULL;
2276}
2277
5db31569
JR		 2278static struct dmar_domain *dmar_insert_one_dev_info(struct intel_iommu *iommu,
2279 int bus, int devfn,
2280 struct device *dev,
2281 struct dmar_domain *domain)
745f2586 2282{
5a8f40e8 2283 struct dmar_domain *found = NULL;
745f2586
JL		 2284 struct device_domain_info *info;
2285 unsigned long flags;
d160aca5 2286 int ret;
745f2586
JL		 2287
2288 info = alloc_devinfo_mem();
2289 if (!info)
b718cd3d 2290 return NULL;
745f2586 2291
745f2586
JL		 2292 info->bus = bus;
2293 info->devfn = devfn;
fb0cc3aa
BH		 2294 info->ats.enabled = 0;
2295 info->ats.qdep = 0;
745f2586
JL		 2296 info->dev = dev;
2297 info->domain = domain;
5a8f40e8 2298 info->iommu = iommu;
745f2586
JL		 2299
2300 spin_lock_irqsave(&device_domain_lock, flags);
2301 if (dev)
0bcb3e28 2302 found = find_domain(dev);
f303e507
JR		 2303
2304 if (!found) {
5a8f40e8 2305 struct device_domain_info *info2;
41e80dca 2306 info2 = dmar_search_domain_by_dev_info(iommu->segment, bus, devfn);
f303e507
JR		 2307 if (info2) {
2308 found = info2->domain;
2309 info2->dev = dev;
2310 }
5a8f40e8 2311 }
f303e507 2312
745f2586
JL		 2313 if (found) {
2314 spin_unlock_irqrestore(&device_domain_lock, flags);
2315 free_devinfo_mem(info);
b718cd3d
DW		 2316 /* Caller must free the original domain */
2317 return found;
745f2586
JL		 2318 }
2319
d160aca5
JR		 2320 spin_lock(&iommu->lock);
2321 ret = domain_attach_iommu(domain, iommu);
2322 spin_unlock(&iommu->lock);
2323
2324 if (ret) {
c6c2cebd
JR		 2325 spin_unlock_irqrestore(&device_domain_lock, flags);
2326 return NULL;
2327 }
c6c2cebd 2328
b718cd3d
DW		 2329 list_add(&info->link, &domain->devices);
2330 list_add(&info->global, &device_domain_list);
2331 if (dev)
2332 dev->archdata.iommu = info;
2333 spin_unlock_irqrestore(&device_domain_lock, flags);
2334
cc4e2575
JR		 2335 if (dev && domain_context_mapping(domain, dev)) {
2336 pr_err("Domain context map for %s failed\n", dev_name(dev));
e6de0f8d 2337 dmar_remove_one_dev_info(domain, dev);
cc4e2575
JR		 2338 return NULL;
2339 }
2340
b718cd3d 2341 return domain;
745f2586
JL		 2342}
2343
579305f7
AW		 2344static int get_last_alias(struct pci_dev *pdev, u16 alias, void *opaque)
2345{
2346 *(u16 *)opaque = alias;
2347 return 0;
2348}
2349
ba395927 2350/* domain is initialized */
146922ec 2351static struct dmar_domain *get_domain_for_dev(struct device *dev, int gaw)
ba395927 2352{
cc4e2575 2353 struct device_domain_info *info = NULL;
579305f7
AW		 2354 struct dmar_domain *domain, *tmp;
2355 struct intel_iommu *iommu;
08a7f456 2356 u16 req_id, dma_alias;
ba395927 2357 unsigned long flags;
aa4d066a 2358 u8 bus, devfn;
ba395927 2359
146922ec 2360 domain = find_domain(dev);
ba395927
KA		 2361 if (domain)
2362 return domain;
2363
579305f7
AW		 2364 iommu = device_to_iommu(dev, &bus, &devfn);
2365 if (!iommu)
2366 return NULL;
2367
08a7f456
JR		 2368 req_id = ((u16)bus << 8) | devfn;
2369
146922ec
DW		 2370 if (dev_is_pci(dev)) {
2371 struct pci_dev *pdev = to_pci_dev(dev);
276dbf99 2372
579305f7
AW		 2373 pci_for_each_dma_alias(pdev, get_last_alias, &dma_alias);
2374
2375 spin_lock_irqsave(&device_domain_lock, flags);
2376 info = dmar_search_domain_by_dev_info(pci_domain_nr(pdev->bus),
2377 PCI_BUS_NUM(dma_alias),
2378 dma_alias & 0xff);
2379 if (info) {
2380 iommu = info->iommu;
2381 domain = info->domain;
5a8f40e8 2382 }
579305f7 2383 spin_unlock_irqrestore(&device_domain_lock, flags);
ba395927 2384
579305f7
AW		 2385 /* DMA alias already has a domain, uses it */
2386 if (info)
2387 goto found_domain;
2388 }
ba395927 2389
146922ec 2390 /* Allocate and initialize new domain for the device */
ab8dfe25 2391 domain = alloc_domain(0);
745f2586 2392 if (!domain)
579305f7 2393 return NULL;
dc534b25 2394 if (domain_init(domain, iommu, gaw)) {
579305f7
AW		 2395 domain_exit(domain);
2396 return NULL;
2c2e2c38 2397 }
ba395927 2398
579305f7 2399 /* register PCI DMA alias device */
08a7f456 2400 if (req_id != dma_alias && dev_is_pci(dev)) {
5db31569
JR		 2401 tmp = dmar_insert_one_dev_info(iommu, PCI_BUS_NUM(dma_alias),
2402 dma_alias & 0xff, NULL, domain);
579305f7
AW		 2403
2404 if (!tmp || tmp != domain) {
2405 domain_exit(domain);
2406 domain = tmp;
2407 }
2408
b718cd3d 2409 if (!domain)
579305f7 2410 return NULL;
ba395927
KA		 2411 }
2412
2413found_domain:
5db31569 2414 tmp = dmar_insert_one_dev_info(iommu, bus, devfn, dev, domain);
579305f7
AW		 2415
2416 if (!tmp || tmp != domain) {
2417 domain_exit(domain);
2418 domain = tmp;
2419 }
b718cd3d
DW		 2420
2421 return domain;
ba395927
KA		 2422}
2423
b213203e
DW		 2424static int iommu_domain_identity_map(struct dmar_domain *domain,
2425 unsigned long long start,
2426 unsigned long long end)
ba395927 2427{
c5395d5c
DW		 2428 unsigned long first_vpfn = start >> VTD_PAGE_SHIFT;
2429 unsigned long last_vpfn = end >> VTD_PAGE_SHIFT;
2430
2431 if (!reserve_iova(&domain->iovad, dma_to_mm_pfn(first_vpfn),
2432 dma_to_mm_pfn(last_vpfn))) {
9f10e5bf 2433 pr_err("Reserving iova failed\n");
b213203e 2434 return -ENOMEM;
ba395927
KA		 2435 }
2436
af1089ce 2437 pr_debug("Mapping reserved region %llx-%llx\n", start, end);
ba395927
KA		 2438 /*
2439 * RMRR range might have overlap with physical memory range,
2440 * clear it first
2441 */
c5395d5c 2442 dma_pte_clear_range(domain, first_vpfn, last_vpfn);
ba395927 2443
c5395d5c
DW		 2444 return domain_pfn_mapping(domain, first_vpfn, first_vpfn,
2445 last_vpfn - first_vpfn + 1,
61df7443 2446 DMA_PTE_READ|DMA_PTE_WRITE);
b213203e
DW		 2447}
2448
0b9d9753 2449static int iommu_prepare_identity_map(struct device *dev,
b213203e
DW		 2450 unsigned long long start,
2451 unsigned long long end)
2452{
2453 struct dmar_domain *domain;
2454 int ret;
2455
0b9d9753 2456 domain = get_domain_for_dev(dev, DEFAULT_DOMAIN_ADDRESS_WIDTH);
b213203e
DW		 2457 if (!domain)
2458 return -ENOMEM;
2459
19943b0e
DW		 2460 /* For _hardware_ passthrough, don't bother. But for software
2461 passthrough, we do it anyway -- it may indicate a memory
2462 range which is reserved in E820, so which didn't get set
2463 up to start with in si_domain */
2464 if (domain == si_domain && hw_pass_through) {
9f10e5bf
JR		 2465 pr_warn("Ignoring identity map for HW passthrough device %s [0x%Lx - 0x%Lx]\n",
2466 dev_name(dev), start, end);
19943b0e
DW		 2467 return 0;
2468 }
2469
9f10e5bf
JR		 2470 pr_info("Setting identity map for device %s [0x%Lx - 0x%Lx]\n",
2471 dev_name(dev), start, end);
2472
5595b528
DW		 2473 if (end < start) {
2474 WARN(1, "Your BIOS is broken; RMRR ends before it starts!\n"
2475 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
2476 dmi_get_system_info(DMI_BIOS_VENDOR),
2477 dmi_get_system_info(DMI_BIOS_VERSION),
2478 dmi_get_system_info(DMI_PRODUCT_VERSION));
2479 ret = -EIO;
2480 goto error;
2481 }
2482
2ff729f5
DW		 2483 if (end >> agaw_to_width(domain->agaw)) {
2484 WARN(1, "Your BIOS is broken; RMRR exceeds permitted address width (%d bits)\n"
2485 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
2486 agaw_to_width(domain->agaw),
2487 dmi_get_system_info(DMI_BIOS_VENDOR),
2488 dmi_get_system_info(DMI_BIOS_VERSION),
2489 dmi_get_system_info(DMI_PRODUCT_VERSION));
2490 ret = -EIO;
2491 goto error;
2492 }
19943b0e 2493
b213203e 2494 ret = iommu_domain_identity_map(domain, start, end);
ba395927
KA		 2495 if (ret)
2496 goto error;
2497
b213203e
DW		 2498 return 0;
2499
2500 error:
ba395927
KA		 2501 domain_exit(domain);
2502 return ret;
ba395927
KA		 2503}
2504
2505static inline int iommu_prepare_rmrr_dev(struct dmar_rmrr_unit *rmrr,
0b9d9753 2506 struct device *dev)
ba395927 2507{
0b9d9753 2508 if (dev->archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO)
ba395927 2509 return 0;
0b9d9753
DW		 2510 return iommu_prepare_identity_map(dev, rmrr->base_address,
2511 rmrr->end_address);
ba395927
KA		 2512}
2513
d3f13810 2514#ifdef CONFIG_INTEL_IOMMU_FLOPPY_WA
49a0429e
KA		 2515static inline void iommu_prepare_isa(void)
2516{
2517 struct pci_dev *pdev;
2518 int ret;
2519
2520 pdev = pci_get_class(PCI_CLASS_BRIDGE_ISA << 8, NULL);
2521 if (!pdev)
2522 return;
2523
9f10e5bf 2524 pr_info("Prepare 0-16MiB unity mapping for LPC\n");
0b9d9753 2525 ret = iommu_prepare_identity_map(&pdev->dev, 0, 16*1024*1024 - 1);
49a0429e
KA		 2526
2527 if (ret)
9f10e5bf 2528 pr_err("Failed to create 0-16MiB identity map - floppy might not work\n");
49a0429e 2529
9b27e82d 2530 pci_dev_put(pdev);
49a0429e
KA		 2531}
2532#else
2533static inline void iommu_prepare_isa(void)
2534{
2535 return;
2536}
d3f13810 2537#endif /* !CONFIG_INTEL_IOMMU_FLPY_WA */
49a0429e 2538
2c2e2c38 2539static int md_domain_init(struct dmar_domain *domain, int guest_width);
c7ab48d2 2540
071e1374 2541static int __init si_domain_init(int hw)
2c2e2c38 2542{
c7ab48d2 2543 int nid, ret = 0;
2c2e2c38 2544
ab8dfe25 2545 si_domain = alloc_domain(DOMAIN_FLAG_STATIC_IDENTITY);
2c2e2c38
FY		 2546 if (!si_domain)
2547 return -EFAULT;
2548
2c2e2c38
FY		 2549 if (md_domain_init(si_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
2550 domain_exit(si_domain);
2551 return -EFAULT;
2552 }
2553
0dc79715 2554 pr_debug("Identity mapping domain allocated\n");
2c2e2c38 2555
19943b0e
DW		 2556 if (hw)
2557 return 0;
2558
c7ab48d2 2559 for_each_online_node(nid) {
5dfe8660
TH		 2560 unsigned long start_pfn, end_pfn;
2561 int i;
2562
2563 for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, NULL) {
2564 ret = iommu_domain_identity_map(si_domain,
2565 PFN_PHYS(start_pfn), PFN_PHYS(end_pfn));
2566 if (ret)
2567 return ret;
2568 }
c7ab48d2
DW		 2569 }
2570
2c2e2c38
FY		 2571 return 0;
2572}
2573
9b226624 2574static int identity_mapping(struct device *dev)
2c2e2c38
FY		 2575{
2576 struct device_domain_info *info;
2577
2578 if (likely(!iommu_identity_mapping))
2579 return 0;
2580
9b226624 2581 info = dev->archdata.iommu;
cb452a40
MT		 2582 if (info && info != DUMMY_DEVICE_DOMAIN_INFO)
2583 return (info->domain == si_domain);
2c2e2c38 2584
2c2e2c38
FY		 2585 return 0;
2586}
2587
28ccce0d 2588static int domain_add_dev_info(struct dmar_domain *domain, struct device *dev)
2c2e2c38 2589{
0ac72664 2590 struct dmar_domain *ndomain;
5a8f40e8 2591 struct intel_iommu *iommu;
156baca8 2592 u8 bus, devfn;
2c2e2c38 2593
5913c9bf 2594 iommu = device_to_iommu(dev, &bus, &devfn);
5a8f40e8
DW		 2595 if (!iommu)
2596 return -ENODEV;
2597
5db31569 2598 ndomain = dmar_insert_one_dev_info(iommu, bus, devfn, dev, domain);
0ac72664
DW		 2599 if (ndomain != domain)
2600 return -EBUSY;
2c2e2c38
FY		 2601
2602 return 0;
2603}
2604
0b9d9753 2605static bool device_has_rmrr(struct device *dev)
ea2447f7
TM		 2606{
2607 struct dmar_rmrr_unit *rmrr;
832bd858 2608 struct device *tmp;
ea2447f7
TM		 2609 int i;
2610
0e242612 2611 rcu_read_lock();
ea2447f7 2612 for_each_rmrr_units(rmrr) {
b683b230
JL		 2613 /*
2614 * Return TRUE if this RMRR contains the device that
2615 * is passed in.
2616 */
2617 for_each_active_dev_scope(rmrr->devices,
2618 rmrr->devices_cnt, i, tmp)
0b9d9753 2619 if (tmp == dev) {
0e242612 2620 rcu_read_unlock();
ea2447f7 2621 return true;
b683b230 2622 }
ea2447f7 2623 }
0e242612 2624 rcu_read_unlock();
ea2447f7
TM		 2625 return false;
2626}
2627
c875d2c1
AW		 2628/*
2629 * There are a couple cases where we need to restrict the functionality of
2630 * devices associated with RMRRs. The first is when evaluating a device for
2631 * identity mapping because problems exist when devices are moved in and out
2632 * of domains and their respective RMRR information is lost. This means that
2633 * a device with associated RMRRs will never be in a "passthrough" domain.
2634 * The second is use of the device through the IOMMU API. This interface
2635 * expects to have full control of the IOVA space for the device. We cannot
2636 * satisfy both the requirement that RMRR access is maintained and have an
2637 * unencumbered IOVA space. We also have no ability to quiesce the device's
2638 * use of the RMRR space or even inform the IOMMU API user of the restriction.
2639 * We therefore prevent devices associated with an RMRR from participating in
2640 * the IOMMU API, which eliminates them from device assignment.
2641 *
2642 * In both cases we assume that PCI USB devices with RMRRs have them largely
2643 * for historical reasons and that the RMRR space is not actively used post
2644 * boot. This exclusion may change if vendors begin to abuse it.
18436afd
DW		 2645 *
2646 * The same exception is made for graphics devices, with the requirement that
2647 * any use of the RMRR regions will be torn down before assigning the device
2648 * to a guest.
c875d2c1
AW		 2649 */
2650static bool device_is_rmrr_locked(struct device *dev)
2651{
2652 if (!device_has_rmrr(dev))
2653 return false;
2654
2655 if (dev_is_pci(dev)) {
2656 struct pci_dev *pdev = to_pci_dev(dev);
2657
18436afd 2658 if (IS_USB_DEVICE(pdev) || IS_GFX_DEVICE(pdev))
c875d2c1
AW		 2659 return false;
2660 }
2661
2662 return true;
2663}
2664
3bdb2591 2665static int iommu_should_identity_map(struct device *dev, int startup)
6941af28 2666{
ea2447f7 2667
3bdb2591
DW		 2668 if (dev_is_pci(dev)) {
2669 struct pci_dev *pdev = to_pci_dev(dev);
ea2447f7 2670
c875d2c1 2671 if (device_is_rmrr_locked(dev))
3bdb2591 2672 return 0;
e0fc7e0b 2673
3bdb2591
DW		 2674 if ((iommu_identity_mapping & IDENTMAP_AZALIA) && IS_AZALIA(pdev))
2675 return 1;
e0fc7e0b 2676
3bdb2591
DW		 2677 if ((iommu_identity_mapping & IDENTMAP_GFX) && IS_GFX_DEVICE(pdev))
2678 return 1;
6941af28 2679
3bdb2591 2680 if (!(iommu_identity_mapping & IDENTMAP_ALL))
3dfc813d 2681 return 0;
3bdb2591
DW		 2682
2683 /*
2684 * We want to start off with all devices in the 1:1 domain, and
2685 * take them out later if we find they can't access all of memory.
2686 *
2687 * However, we can't do this for PCI devices behind bridges,
2688 * because all PCI devices behind the same bridge will end up
2689 * with the same source-id on their transactions.
2690 *
2691 * Practically speaking, we can't change things around for these
2692 * devices at run-time, because we can't be sure there'll be no
2693 * DMA transactions in flight for any of their siblings.
2694 *
2695 * So PCI devices (unless they're on the root bus) as well as
2696 * their parent PCI-PCI or PCIe-PCI bridges must be left _out_ of
2697 * the 1:1 domain, just in _case_ one of their siblings turns out
2698 * not to be able to map all of memory.
2699 */
2700 if (!pci_is_pcie(pdev)) {
2701 if (!pci_is_root_bus(pdev->bus))
2702 return 0;
2703 if (pdev->class >> 8 == PCI_CLASS_BRIDGE_PCI)
2704 return 0;
2705 } else if (pci_pcie_type(pdev) == PCI_EXP_TYPE_PCI_BRIDGE)
3dfc813d 2706 return 0;
3bdb2591
DW		 2707 } else {
2708 if (device_has_rmrr(dev))
2709 return 0;
2710 }
3dfc813d 2711
3bdb2591 2712 /*
3dfc813d 2713 * At boot time, we don't yet know if devices will be 64-bit capable.
3bdb2591 2714 * Assume that they will — if they turn out not to be, then we can
3dfc813d
DW		 2715 * take them out of the 1:1 domain later.
2716 */
8fcc5372
CW		 2717 if (!startup) {
2718 /*
2719 * If the device's dma_mask is less than the system's memory
2720 * size then this is not a candidate for identity mapping.
2721 */
3bdb2591 2722 u64 dma_mask = *dev->dma_mask;
8fcc5372 2723
3bdb2591
DW		 2724 if (dev->coherent_dma_mask &&
2725 dev->coherent_dma_mask < dma_mask)
2726 dma_mask = dev->coherent_dma_mask;
8fcc5372 2727
3bdb2591 2728 return dma_mask >= dma_get_required_mask(dev);
8fcc5372 2729 }
6941af28
DW		 2730
2731 return 1;
2732}
2733
cf04eee8
DW		 2734static int __init dev_prepare_static_identity_mapping(struct device *dev, int hw)
2735{
2736 int ret;
2737
2738 if (!iommu_should_identity_map(dev, 1))
2739 return 0;
2740
28ccce0d 2741 ret = domain_add_dev_info(si_domain, dev);
cf04eee8 2742 if (!ret)
9f10e5bf
JR		 2743 pr_info("%s identity mapping for device %s\n",
2744 hw ? "Hardware" : "Software", dev_name(dev));
cf04eee8
DW		 2745 else if (ret == -ENODEV)
2746 /* device not associated with an iommu */
2747 ret = 0;
2748
2749 return ret;
2750}
2751
2752
071e1374 2753static int __init iommu_prepare_static_identity_mapping(int hw)
2c2e2c38 2754{
2c2e2c38 2755 struct pci_dev *pdev = NULL;
cf04eee8
DW		 2756 struct dmar_drhd_unit *drhd;
2757 struct intel_iommu *iommu;
2758 struct device *dev;
2759 int i;
2760 int ret = 0;
2c2e2c38 2761
2c2e2c38 2762 for_each_pci_dev(pdev) {
cf04eee8
DW		 2763 ret = dev_prepare_static_identity_mapping(&pdev->dev, hw);
2764 if (ret)
2765 return ret;
2766 }
2767
2768 for_each_active_iommu(iommu, drhd)
2769 for_each_active_dev_scope(drhd->devices, drhd->devices_cnt, i, dev) {
2770 struct acpi_device_physical_node *pn;
2771 struct acpi_device *adev;
2772
2773 if (dev->bus != &acpi_bus_type)
2774 continue;
86080ccc 2775
cf04eee8
DW		 2776 adev= to_acpi_device(dev);
2777 mutex_lock(&adev->physical_node_lock);
2778 list_for_each_entry(pn, &adev->physical_node_list, node) {
2779 ret = dev_prepare_static_identity_mapping(pn->dev, hw);
2780 if (ret)
2781 break;
eae460b6 2782 }
cf04eee8
DW		 2783 mutex_unlock(&adev->physical_node_lock);
2784 if (ret)
2785 return ret;
62edf5dc 2786 }
2c2e2c38
FY		 2787
2788 return 0;
2789}
2790
ffebeb46
JL		 2791static void intel_iommu_init_qi(struct intel_iommu *iommu)
2792{
2793 /*
2794 * Start from the sane iommu hardware state.
2795 * If the queued invalidation is already initialized by us
2796 * (for example, while enabling interrupt-remapping) then
2797 * we got the things already rolling from a sane state.
2798 */
2799 if (!iommu->qi) {
2800 /*
2801 * Clear any previous faults.
2802 */
2803 dmar_fault(-1, iommu);
2804 /*
2805 * Disable queued invalidation if supported and already enabled
2806 * before OS handover.
2807 */
2808 dmar_disable_qi(iommu);
2809 }
2810
2811 if (dmar_enable_qi(iommu)) {
2812 /*
2813 * Queued Invalidate not enabled, use Register Based Invalidate
2814 */
2815 iommu->flush.flush_context = __iommu_flush_context;
2816 iommu->flush.flush_iotlb = __iommu_flush_iotlb;
9f10e5bf 2817 pr_info("%s: Using Register based invalidation\n",
ffebeb46
JL		 2818 iommu->name);
2819 } else {
2820 iommu->flush.flush_context = qi_flush_context;
2821 iommu->flush.flush_iotlb = qi_flush_iotlb;
9f10e5bf 2822 pr_info("%s: Using Queued invalidation\n", iommu->name);
ffebeb46
JL		 2823 }
2824}
2825
091d42e4 2826static int copy_context_table(struct intel_iommu *iommu,
543c8dcf 2827 struct root_entry __iomem *old_re,
091d42e4
JR		 2828 struct context_entry **tbl,
2829 int bus, bool ext)
2830{
dbcd861f 2831 int tbl_idx, pos = 0, idx, devfn, ret = 0, did;
543c8dcf
JR		 2832 struct context_entry __iomem *old_ce = NULL;
2833 struct context_entry *new_ce = NULL, ce;
2834 struct root_entry re;
091d42e4
JR		 2835 phys_addr_t old_ce_phys;
2836
2837 tbl_idx = ext ? bus * 2 : bus;
543c8dcf 2838 memcpy_fromio(&re, old_re, sizeof(re));
091d42e4
JR		 2839
2840 for (devfn = 0; devfn < 256; devfn++) {
2841 /* First calculate the correct index */
2842 idx = (ext ? devfn * 2 : devfn) % 256;
2843
2844 if (idx == 0) {
2845 /* First save what we may have and clean up */
2846 if (new_ce) {
2847 tbl[tbl_idx] = new_ce;
2848 __iommu_flush_cache(iommu, new_ce,
2849 VTD_PAGE_SIZE);
2850 pos = 1;
2851 }
2852
2853 if (old_ce)
2854 iounmap(old_ce);
2855
2856 ret = 0;
2857 if (devfn < 0x80)
543c8dcf 2858 old_ce_phys = root_entry_lctp(&re);
091d42e4 2859 else
543c8dcf 2860 old_ce_phys = root_entry_uctp(&re);
091d42e4
JR		 2861
2862 if (!old_ce_phys) {
2863 if (ext && devfn == 0) {
2864 /* No LCTP, try UCTP */
2865 devfn = 0x7f;
2866 continue;
2867 } else {
2868 goto out;
2869 }
2870 }
2871
2872 ret = -ENOMEM;
2873 old_ce = ioremap_cache(old_ce_phys, PAGE_SIZE);
2874 if (!old_ce)
2875 goto out;
2876
2877 new_ce = alloc_pgtable_page(iommu->node);
2878 if (!new_ce)
2879 goto out_unmap;
2880
2881 ret = 0;
2882 }
2883
2884 /* Now copy the context entry */
543c8dcf 2885 memcpy_fromio(&ce, old_ce + idx, sizeof(ce));
091d42e4 2886
cf484d0e 2887 if (!__context_present(&ce))
091d42e4
JR		 2888 continue;
2889
dbcd861f
JR		 2890 did = context_domain_id(&ce);
2891 if (did >= 0 && did < cap_ndoms(iommu->cap))
2892 set_bit(did, iommu->domain_ids);
2893
cf484d0e
JR		 2894 /*
2895 * We need a marker for copied context entries. This
2896 * marker needs to work for the old format as well as
2897 * for extended context entries.
2898 *
2899 * Bit 67 of the context entry is used. In the old
2900 * format this bit is available to software, in the
2901 * extended format it is the PGE bit, but PGE is ignored
2902 * by HW if PASIDs are disabled (and thus still
2903 * available).
2904 *
2905 * So disable PASIDs first and then mark the entry
2906 * copied. This means that we don't copy PASID
2907 * translations from the old kernel, but this is fine as
2908 * faults there are not fatal.
2909 */
2910 context_clear_pasid_enable(&ce);
2911 context_set_copied(&ce);
2912
091d42e4
JR		 2913 new_ce[idx] = ce;
2914 }
2915
2916 tbl[tbl_idx + pos] = new_ce;
2917
2918 __iommu_flush_cache(iommu, new_ce, VTD_PAGE_SIZE);
2919
2920out_unmap:
2921 iounmap(old_ce);
2922
2923out:
2924 return ret;
2925}
2926
2927static int copy_translation_tables(struct intel_iommu *iommu)
2928{
543c8dcf 2929 struct root_entry __iomem *old_rt;
091d42e4 2930 struct context_entry **ctxt_tbls;
091d42e4
JR		 2931 phys_addr_t old_rt_phys;
2932 int ctxt_table_entries;
2933 unsigned long flags;
2934 u64 rtaddr_reg;
2935 int bus, ret;
c3361f2f 2936 bool new_ext, ext;
091d42e4
JR		 2937
2938 rtaddr_reg = dmar_readq(iommu->reg + DMAR_RTADDR_REG);
2939 ext = !!(rtaddr_reg & DMA_RTADDR_RTT);
c3361f2f
JR		 2940 new_ext = !!ecap_ecs(iommu->ecap);
2941
2942 /*
2943 * The RTT bit can only be changed when translation is disabled,
2944 * but disabling translation means to open a window for data
2945 * corruption. So bail out and don't copy anything if we would
2946 * have to change the bit.
2947 */
2948 if (new_ext != ext)
2949 return -EINVAL;
091d42e4
JR		 2950
2951 old_rt_phys = rtaddr_reg & VTD_PAGE_MASK;
2952 if (!old_rt_phys)
2953 return -EINVAL;
2954
2955 old_rt = ioremap_cache(old_rt_phys, PAGE_SIZE);
2956 if (!old_rt)
2957 return -ENOMEM;
2958
2959 /* This is too big for the stack - allocate it from slab */
2960 ctxt_table_entries = ext ? 512 : 256;
2961 ret = -ENOMEM;
2962 ctxt_tbls = kzalloc(ctxt_table_entries * sizeof(void *), GFP_KERNEL);
2963 if (!ctxt_tbls)
2964 goto out_unmap;
2965
2966 for (bus = 0; bus < 256; bus++) {
2967 ret = copy_context_table(iommu, &old_rt[bus],
2968 ctxt_tbls, bus, ext);
2969 if (ret) {
2970 pr_err("%s: Failed to copy context table for bus %d\n",
2971 iommu->name, bus);
2972 continue;
2973 }
2974 }
2975
2976 spin_lock_irqsave(&iommu->lock, flags);
2977
2978 /* Context tables are copied, now write them to the root_entry table */
2979 for (bus = 0; bus < 256; bus++) {
2980 int idx = ext ? bus * 2 : bus;
2981 u64 val;
2982
2983 if (ctxt_tbls[idx]) {
2984 val = virt_to_phys(ctxt_tbls[idx]) | 1;
2985 iommu->root_entry[bus].lo = val;
2986 }
2987
2988 if (!ext || !ctxt_tbls[idx + 1])
2989 continue;
2990
2991 val = virt_to_phys(ctxt_tbls[idx + 1]) | 1;
2992 iommu->root_entry[bus].hi = val;
2993 }
2994
2995 spin_unlock_irqrestore(&iommu->lock, flags);
2996
2997 kfree(ctxt_tbls);
2998
2999 __iommu_flush_cache(iommu, iommu->root_entry, PAGE_SIZE);
3000
3001 ret = 0;
3002
3003out_unmap:
3004 iounmap(old_rt);
3005
3006 return ret;
3007}
3008
b779260b 3009static int __init init_dmars(void)
ba395927
KA		 3010{
3011 struct dmar_drhd_unit *drhd;
3012 struct dmar_rmrr_unit *rmrr;
a87f4918 3013 bool copied_tables = false;
832bd858 3014 struct device *dev;
ba395927 3015 struct intel_iommu *iommu;
9d783ba0 3016 int i, ret;
2c2e2c38 3017
ba395927
KA		 3018 /*
3019 * for each drhd
3020 * allocate root
3021 * initialize and program root entry to not present
3022 * endfor
3023 */
3024 for_each_drhd_unit(drhd) {
5e0d2a6f 3025 /*
3026 * lock not needed as this is only incremented in the single
3027 * threaded kernel __init code path all other access are read
3028 * only
3029 */
78d8e704 3030 if (g_num_of_iommus < DMAR_UNITS_SUPPORTED) {
1b198bb0
MT		 3031 g_num_of_iommus++;
3032 continue;
3033 }
9f10e5bf 3034 pr_err_once("Exceeded %d IOMMUs\n", DMAR_UNITS_SUPPORTED);
5e0d2a6f 3035 }
3036
ffebeb46
JL		 3037 /* Preallocate enough resources for IOMMU hot-addition */
3038 if (g_num_of_iommus < DMAR_UNITS_SUPPORTED)
3039 g_num_of_iommus = DMAR_UNITS_SUPPORTED;
3040
d9630fe9
WH		 3041 g_iommus = kcalloc(g_num_of_iommus, sizeof(struct intel_iommu *),
3042 GFP_KERNEL);
3043 if (!g_iommus) {
9f10e5bf 3044 pr_err("Allocating global iommu array failed\n");
d9630fe9
WH		 3045 ret = -ENOMEM;
3046 goto error;
3047 }
3048
80b20dd8 3049 deferred_flush = kzalloc(g_num_of_iommus *
3050 sizeof(struct deferred_flush_tables), GFP_KERNEL);
3051 if (!deferred_flush) {
5e0d2a6f 3052 ret = -ENOMEM;
989d51fc 3053 goto free_g_iommus;
5e0d2a6f 3054 }
3055
7c919779 3056 for_each_active_iommu(iommu, drhd) {
d9630fe9 3057 g_iommus[iommu->seq_id] = iommu;
ba395927 3058
b63d80d1
JR		 3059 intel_iommu_init_qi(iommu);
3060
e61d98d8
SS		 3061 ret = iommu_init_domains(iommu);
3062 if (ret)
989d51fc 3063 goto free_iommu;
e61d98d8 3064
4158c2ec
JR		 3065 init_translation_status(iommu);
3066
091d42e4
JR		 3067 if (translation_pre_enabled(iommu) && !is_kdump_kernel()) {
3068 iommu_disable_translation(iommu);
3069 clear_translation_pre_enabled(iommu);
3070 pr_warn("Translation was enabled for %s but we are not in kdump mode\n",
3071 iommu->name);
3072 }
4158c2ec 3073
ba395927
KA		 3074 /*
3075 * TBD:
3076 * we could share the same root & context tables
25985edc 3077 * among all IOMMU's. Need to Split it later.
ba395927
KA		 3078 */
3079 ret = iommu_alloc_root_entry(iommu);
ffebeb46 3080 if (ret)
989d51fc 3081 goto free_iommu;
5f0a7f76 3082
091d42e4
JR		 3083 if (translation_pre_enabled(iommu)) {
3084 pr_info("Translation already enabled - trying to copy translation structures\n");
3085
3086 ret = copy_translation_tables(iommu);
3087 if (ret) {
3088 /*
3089 * We found the IOMMU with translation
3090 * enabled - but failed to copy over the
3091 * old root-entry table. Try to proceed
3092 * by disabling translation now and
3093 * allocating a clean root-entry table.
3094 * This might cause DMAR faults, but
3095 * probably the dump will still succeed.
3096 */
3097 pr_err("Failed to copy translation tables from previous kernel for %s\n",
3098 iommu->name);
3099 iommu_disable_translation(iommu);
3100 clear_translation_pre_enabled(iommu);
3101 } else {
3102 pr_info("Copied translation tables from previous kernel for %s\n",
3103 iommu->name);
a87f4918 3104 copied_tables = true;
091d42e4
JR		 3105 }
3106 }
3107
5f0a7f76
JR		 3108 iommu_flush_write_buffer(iommu);
3109 iommu_set_root_entry(iommu);
3110 iommu->flush.flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL);
3111 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
3112
4ed0d3e6 3113 if (!ecap_pass_through(iommu->ecap))
19943b0e 3114 hw_pass_through = 0;
8a94ade4
DW		 3115#ifdef CONFIG_INTEL_IOMMU_SVM
3116 if (pasid_enabled(iommu))
3117 intel_svm_alloc_pasid_tables(iommu);
3118#endif
ba395927
KA		 3119 }
3120
19943b0e 3121 if (iommu_pass_through)
e0fc7e0b
DW		 3122 iommu_identity_mapping |= IDENTMAP_ALL;
3123
d3f13810 3124#ifdef CONFIG_INTEL_IOMMU_BROKEN_GFX_WA
e0fc7e0b 3125 iommu_identity_mapping |= IDENTMAP_GFX;
19943b0e 3126#endif
e0fc7e0b 3127
86080ccc
JR		 3128 if (iommu_identity_mapping) {
3129 ret = si_domain_init(hw_pass_through);
3130 if (ret)
3131 goto free_iommu;
3132 }
3133
e0fc7e0b
DW		 3134 check_tylersburg_isoch();
3135
a87f4918
JR		 3136 /*
3137 * If we copied translations from a previous kernel in the kdump
3138 * case, we can not assign the devices to domains now, as that
3139 * would eliminate the old mappings. So skip this part and defer
3140 * the assignment to device driver initialization time.
3141 */
3142 if (copied_tables)
3143 goto domains_done;
3144
ba395927 3145 /*
19943b0e
DW		 3146 * If pass through is not set or not enabled, setup context entries for
3147 * identity mappings for rmrr, gfx, and isa and may fall back to static
3148 * identity mapping if iommu_identity_mapping is set.
ba395927 3149 */
19943b0e
DW		 3150 if (iommu_identity_mapping) {
3151 ret = iommu_prepare_static_identity_mapping(hw_pass_through);
4ed0d3e6 3152 if (ret) {
9f10e5bf 3153 pr_crit("Failed to setup IOMMU pass-through\n");
989d51fc 3154 goto free_iommu;
ba395927
KA		 3155 }
3156 }
ba395927 3157 /*
19943b0e
DW		 3158 * For each rmrr
3159 * for each dev attached to rmrr
3160 * do
3161 * locate drhd for dev, alloc domain for dev
3162 * allocate free domain
3163 * allocate page table entries for rmrr
3164 * if context not allocated for bus
3165 * allocate and init context
3166 * set present in root table for this bus
3167 * init context with domain, translation etc
3168 * endfor
3169 * endfor
ba395927 3170 */
9f10e5bf 3171 pr_info("Setting RMRR:\n");
19943b0e 3172 for_each_rmrr_units(rmrr) {
b683b230
JL		 3173 /* some BIOS lists non-exist devices in DMAR table. */
3174 for_each_active_dev_scope(rmrr->devices, rmrr->devices_cnt,
832bd858 3175 i, dev) {
0b9d9753 3176 ret = iommu_prepare_rmrr_dev(rmrr, dev);
19943b0e 3177 if (ret)
9f10e5bf 3178 pr_err("Mapping reserved region failed\n");
ba395927 3179 }
4ed0d3e6 3180 }
49a0429e 3181
19943b0e
DW		 3182 iommu_prepare_isa();
3183
a87f4918
JR		 3184domains_done:
3185
ba395927
KA		 3186 /*
3187 * for each drhd
3188 * enable fault log
3189 * global invalidate context cache
3190 * global invalidate iotlb
3191 * enable translation
3192 */
7c919779 3193 for_each_iommu(iommu, drhd) {
51a63e67
JC		 3194 if (drhd->ignored) {
3195 /*
3196 * we always have to disable PMRs or DMA may fail on
3197 * this device
3198 */
3199 if (force_on)
7c919779 3200 iommu_disable_protect_mem_regions(iommu);
ba395927 3201 continue;
51a63e67 3202 }
ba395927
KA		 3203
3204 iommu_flush_write_buffer(iommu);
3205
3460a6d9
KA		 3206 ret = dmar_set_interrupt(iommu);
3207 if (ret)
989d51fc 3208 goto free_iommu;
3460a6d9 3209
8939ddf6
JR		 3210 if (!translation_pre_enabled(iommu))
3211 iommu_enable_translation(iommu);
3212
b94996c9 3213 iommu_disable_protect_mem_regions(iommu);
ba395927
KA		 3214 }
3215
3216 return 0;
989d51fc
JL		 3217
3218free_iommu:
ffebeb46
JL		 3219 for_each_active_iommu(iommu, drhd) {
3220 disable_dmar_iommu(iommu);
a868e6b7 3221 free_dmar_iommu(iommu);
ffebeb46 3222 }
9bdc531e 3223 kfree(deferred_flush);
989d51fc 3224free_g_iommus:
d9630fe9 3225 kfree(g_iommus);
989d51fc 3226error:
ba395927
KA		 3227 return ret;
3228}
3229
5a5e02a6 3230/* This takes a number of _MM_ pages, not VTD pages */
875764de
DW		 3231static struct iova *intel_alloc_iova(struct device *dev,
3232 struct dmar_domain *domain,
3233 unsigned long nrpages, uint64_t dma_mask)
ba395927 3234{
ba395927 3235 struct iova *iova = NULL;
ba395927 3236
875764de
DW		 3237 /* Restrict dma_mask to the width that the iommu can handle */
3238 dma_mask = min_t(uint64_t, DOMAIN_MAX_ADDR(domain->gaw), dma_mask);
8f6429c7
RM		 3239 /* Ensure we reserve the whole size-aligned region */
3240 nrpages = __roundup_pow_of_two(nrpages);
875764de
DW		 3241
3242 if (!dmar_forcedac && dma_mask > DMA_BIT_MASK(32)) {
ba395927
KA		 3243 /*
3244 * First try to allocate an io virtual address in
284901a9 3245 * DMA_BIT_MASK(32) and if that fails then try allocating
3609801e 3246 * from higher range
ba395927 3247 */
875764de
DW		 3248 iova = alloc_iova(&domain->iovad, nrpages,
3249 IOVA_PFN(DMA_BIT_MASK(32)), 1);
3250 if (iova)
3251 return iova;
3252 }
3253 iova = alloc_iova(&domain->iovad, nrpages, IOVA_PFN(dma_mask), 1);
3254 if (unlikely(!iova)) {
9f10e5bf 3255 pr_err("Allocating %ld-page iova for %s failed",
207e3592 3256 nrpages, dev_name(dev));
f76aec76
KA		 3257 return NULL;
3258 }
3259
3260 return iova;
3261}
3262
d4b709f4 3263static struct dmar_domain *__get_valid_domain_for_dev(struct device *dev)
f76aec76
KA		 3264{
3265 struct dmar_domain *domain;
f76aec76 3266
d4b709f4 3267 domain = get_domain_for_dev(dev, DEFAULT_DOMAIN_ADDRESS_WIDTH);
f76aec76 3268 if (!domain) {
9f10e5bf 3269 pr_err("Allocating domain for %s failed\n",
d4b709f4 3270 dev_name(dev));
4fe05bbc 3271 return NULL;
ba395927
KA		 3272 }
3273
f76aec76
KA		 3274 return domain;
3275}
3276
d4b709f4 3277static inline struct dmar_domain *get_valid_domain_for_dev(struct device *dev)
147202aa
DW		 3278{
3279 struct device_domain_info *info;
3280
3281 /* No lock here, assumes no domain exit in normal case */
d4b709f4 3282 info = dev->archdata.iommu;
147202aa
DW		 3283 if (likely(info))
3284 return info->domain;
3285
3286 return __get_valid_domain_for_dev(dev);
3287}
3288
ecb509ec 3289/* Check if the dev needs to go through non-identity map and unmap process.*/
73676832 3290static int iommu_no_mapping(struct device *dev)
2c2e2c38
FY		 3291{
3292 int found;
3293
3d89194a 3294 if (iommu_dummy(dev))
1e4c64c4
DW		 3295 return 1;
3296
2c2e2c38 3297 if (!iommu_identity_mapping)
1e4c64c4 3298 return 0;
2c2e2c38 3299
9b226624 3300 found = identity_mapping(dev);
2c2e2c38 3301 if (found) {
ecb509ec 3302 if (iommu_should_identity_map(dev, 0))
2c2e2c38
FY		 3303 return 1;
3304 else {
3305 /*
3306 * 32 bit DMA is removed from si_domain and fall back
3307 * to non-identity mapping.
3308 */
e6de0f8d 3309 dmar_remove_one_dev_info(si_domain, dev);
9f10e5bf
JR		 3310 pr_info("32bit %s uses non-identity mapping\n",
3311 dev_name(dev));
2c2e2c38
FY		 3312 return 0;
3313 }
3314 } else {
3315 /*
3316 * In case of a detached 64 bit DMA device from vm, the device
3317 * is put into si_domain for identity mapping.
3318 */
ecb509ec 3319 if (iommu_should_identity_map(dev, 0)) {
2c2e2c38 3320 int ret;
28ccce0d 3321 ret = domain_add_dev_info(si_domain, dev);
2c2e2c38 3322 if (!ret) {
9f10e5bf
JR		 3323 pr_info("64bit %s uses identity mapping\n",
3324 dev_name(dev));
2c2e2c38
FY		 3325 return 1;
3326 }
3327 }
3328 }
3329
1e4c64c4 3330 return 0;
2c2e2c38
FY		 3331}
3332
5040a918 3333static dma_addr_t __intel_map_single(struct device *dev, phys_addr_t paddr,
bb9e6d65 3334 size_t size, int dir, u64 dma_mask)
f76aec76 3335{
f76aec76 3336 struct dmar_domain *domain;
5b6985ce 3337 phys_addr_t start_paddr;
f76aec76
KA		 3338 struct iova *iova;
3339 int prot = 0;
6865f0d1 3340 int ret;
8c11e798 3341 struct intel_iommu *iommu;
33041ec0 3342 unsigned long paddr_pfn = paddr >> PAGE_SHIFT;
f76aec76
KA		 3343
3344 BUG_ON(dir == DMA_NONE);
2c2e2c38 3345
5040a918 3346 if (iommu_no_mapping(dev))
6865f0d1 3347 return paddr;
f76aec76 3348
5040a918 3349 domain = get_valid_domain_for_dev(dev);
f76aec76
KA		 3350 if (!domain)
3351 return 0;
3352
8c11e798 3353 iommu = domain_get_iommu(domain);
88cb6a74 3354 size = aligned_nrpages(paddr, size);
f76aec76 3355
5040a918 3356 iova = intel_alloc_iova(dev, domain, dma_to_mm_pfn(size), dma_mask);
f76aec76
KA		 3357 if (!iova)
3358 goto error;
3359
ba395927
KA		 3360 /*
3361 * Check if DMAR supports zero-length reads on write only
3362 * mappings..
3363 */
3364 if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
8c11e798 3365 !cap_zlr(iommu->cap))
ba395927
KA		 3366 prot |= DMA_PTE_READ;
3367 if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
3368 prot |= DMA_PTE_WRITE;
3369 /*
6865f0d1 3370 * paddr - (paddr + size) might be partial page, we should map the whole
ba395927 3371 * page. Note: if two part of one page are separately mapped, we
6865f0d1 3372 * might have two guest_addr mapping to the same host paddr, but this
ba395927
KA		 3373 * is not a big problem
3374 */
0ab36de2 3375 ret = domain_pfn_mapping(domain, mm_to_dma_pfn(iova->pfn_lo),
33041ec0 3376 mm_to_dma_pfn(paddr_pfn), size, prot);
ba395927
KA		 3377 if (ret)
3378 goto error;
3379
1f0ef2aa
DW		 3380 /* it's a non-present to present mapping. Only flush if caching mode */
3381 if (cap_caching_mode(iommu->cap))
a1ddcbe9
JR		 3382 iommu_flush_iotlb_psi(iommu, domain,
3383 mm_to_dma_pfn(iova->pfn_lo),
3384 size, 0, 1);
1f0ef2aa 3385 else
8c11e798 3386 iommu_flush_write_buffer(iommu);
f76aec76 3387
03d6a246
DW		 3388 start_paddr = (phys_addr_t)iova->pfn_lo << PAGE_SHIFT;
3389 start_paddr += paddr & ~PAGE_MASK;
3390 return start_paddr;
ba395927 3391
ba395927 3392error:
f76aec76
KA		 3393 if (iova)
3394 __free_iova(&domain->iovad, iova);
9f10e5bf 3395 pr_err("Device %s request: %zx@%llx dir %d --- failed\n",
5040a918 3396 dev_name(dev), size, (unsigned long long)paddr, dir);
ba395927
KA		 3397 return 0;
3398}
3399
ffbbef5c
FT		 3400static dma_addr_t intel_map_page(struct device *dev, struct page *page,
3401 unsigned long offset, size_t size,
3402 enum dma_data_direction dir,
3403 struct dma_attrs *attrs)
bb9e6d65 3404{
ffbbef5c 3405 return __intel_map_single(dev, page_to_phys(page) + offset, size,
46333e37 3406 dir, *dev->dma_mask);
bb9e6d65
FT		 3407}
3408
5e0d2a6f 3409static void flush_unmaps(void)
3410{
80b20dd8 3411 int i, j;
5e0d2a6f 3412
5e0d2a6f 3413 timer_on = 0;
3414
3415 /* just flush them all */
3416 for (i = 0; i < g_num_of_iommus; i++) {
a2bb8459
WH		 3417 struct intel_iommu *iommu = g_iommus[i];
3418 if (!iommu)
3419 continue;
c42d9f32 3420
9dd2fe89
YZ		 3421 if (!deferred_flush[i].next)
3422 continue;
3423
78d5f0f5
NA		 3424 /* In caching mode, global flushes turn emulation expensive */
3425 if (!cap_caching_mode(iommu->cap))
3426 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
93a23a72 3427 DMA_TLB_GLOBAL_FLUSH);
9dd2fe89 3428 for (j = 0; j < deferred_flush[i].next; j++) {
93a23a72
YZ		 3429 unsigned long mask;
3430 struct iova *iova = deferred_flush[i].iova[j];
78d5f0f5
NA		 3431 struct dmar_domain *domain = deferred_flush[i].domain[j];
3432
3433 /* On real hardware multiple invalidations are expensive */
3434 if (cap_caching_mode(iommu->cap))
a1ddcbe9 3435 iommu_flush_iotlb_psi(iommu, domain,
a156ef99 3436 iova->pfn_lo, iova_size(iova),
ea8ea460 3437 !deferred_flush[i].freelist[j], 0);
78d5f0f5 3438 else {
a156ef99 3439 mask = ilog2(mm_to_dma_pfn(iova_size(iova)));
78d5f0f5
NA		 3440 iommu_flush_dev_iotlb(deferred_flush[i].domain[j],
3441 (uint64_t)iova->pfn_lo << PAGE_SHIFT, mask);
3442 }
93a23a72 3443 __free_iova(&deferred_flush[i].domain[j]->iovad, iova);
ea8ea460
DW		 3444 if (deferred_flush[i].freelist[j])
3445 dma_free_pagelist(deferred_flush[i].freelist[j]);
80b20dd8 3446 }
9dd2fe89 3447 deferred_flush[i].next = 0;
5e0d2a6f 3448 }
3449
5e0d2a6f 3450 list_size = 0;
5e0d2a6f 3451}
3452
3453static void flush_unmaps_timeout(unsigned long data)
3454{
80b20dd8 3455 unsigned long flags;
3456
3457 spin_lock_irqsave(&async_umap_flush_lock, flags);
5e0d2a6f 3458 flush_unmaps();
80b20dd8 3459 spin_unlock_irqrestore(&async_umap_flush_lock, flags);
5e0d2a6f 3460}
3461
ea8ea460 3462static void add_unmap(struct dmar_domain *dom, struct iova *iova, struct page *freelist)
5e0d2a6f 3463{
3464 unsigned long flags;
80b20dd8 3465 int next, iommu_id;
8c11e798 3466 struct intel_iommu *iommu;
5e0d2a6f 3467
3468 spin_lock_irqsave(&async_umap_flush_lock, flags);
80b20dd8 3469 if (list_size == HIGH_WATER_MARK)
3470 flush_unmaps();
3471
8c11e798
WH		 3472 iommu = domain_get_iommu(dom);
3473 iommu_id = iommu->seq_id;
c42d9f32 3474
80b20dd8 3475 next = deferred_flush[iommu_id].next;
3476 deferred_flush[iommu_id].domain[next] = dom;
3477 deferred_flush[iommu_id].iova[next] = iova;
ea8ea460 3478 deferred_flush[iommu_id].freelist[next] = freelist;
80b20dd8 3479 deferred_flush[iommu_id].next++;
5e0d2a6f 3480
3481 if (!timer_on) {
3482 mod_timer(&unmap_timer, jiffies + msecs_to_jiffies(10));
3483 timer_on = 1;
3484 }
3485 list_size++;
3486 spin_unlock_irqrestore(&async_umap_flush_lock, flags);
3487}
3488
d41a4adb 3489static void intel_unmap(struct device *dev, dma_addr_t dev_addr)
ba395927 3490{
f76aec76 3491 struct dmar_domain *domain;
d794dc9b 3492 unsigned long start_pfn, last_pfn;
ba395927 3493 struct iova *iova;
8c11e798 3494 struct intel_iommu *iommu;
ea8ea460 3495 struct page *freelist;
ba395927 3496
73676832 3497 if (iommu_no_mapping(dev))
f76aec76 3498 return;
2c2e2c38 3499
1525a29a 3500 domain = find_domain(dev);
ba395927
KA		 3501 BUG_ON(!domain);
3502
8c11e798
WH		 3503 iommu = domain_get_iommu(domain);
3504
ba395927 3505 iova = find_iova(&domain->iovad, IOVA_PFN(dev_addr));
85b98276
DW		 3506 if (WARN_ONCE(!iova, "Driver unmaps unmatched page at PFN %llx\n",
3507 (unsigned long long)dev_addr))
ba395927 3508 return;
ba395927 3509
d794dc9b
DW		 3510 start_pfn = mm_to_dma_pfn(iova->pfn_lo);
3511 last_pfn = mm_to_dma_pfn(iova->pfn_hi + 1) - 1;
ba395927 3512
d794dc9b 3513 pr_debug("Device %s unmapping: pfn %lx-%lx\n",
207e3592 3514 dev_name(dev), start_pfn, last_pfn);
ba395927 3515
ea8ea460 3516 freelist = domain_unmap(domain, start_pfn, last_pfn);
d794dc9b 3517
5e0d2a6f 3518 if (intel_iommu_strict) {
a1ddcbe9 3519 iommu_flush_iotlb_psi(iommu, domain, start_pfn,
ea8ea460 3520 last_pfn - start_pfn + 1, !freelist, 0);
5e0d2a6f 3521 /* free iova */
3522 __free_iova(&domain->iovad, iova);
ea8ea460 3523 dma_free_pagelist(freelist);
5e0d2a6f 3524 } else {
ea8ea460 3525 add_unmap(domain, iova, freelist);
5e0d2a6f 3526 /*
3527 * queue up the release of the unmap to save the 1/6th of the
3528 * cpu used up by the iotlb flush operation...
3529 */
5e0d2a6f 3530 }
ba395927
KA		 3531}
3532
d41a4adb
JL		 3533static void intel_unmap_page(struct device *dev, dma_addr_t dev_addr,
3534 size_t size, enum dma_data_direction dir,
3535 struct dma_attrs *attrs)
3536{
3537 intel_unmap(dev, dev_addr);
3538}
3539
5040a918 3540static void *intel_alloc_coherent(struct device *dev, size_t size,
baa676fc
AP		 3541 dma_addr_t *dma_handle, gfp_t flags,
3542 struct dma_attrs *attrs)
ba395927 3543{
36746436 3544 struct page *page = NULL;
ba395927
KA		 3545 int order;
3546
5b6985ce 3547 size = PAGE_ALIGN(size);
ba395927 3548 order = get_order(size);
e8bb910d 3549
5040a918 3550 if (!iommu_no_mapping(dev))
e8bb910d 3551 flags &= ~(GFP_DMA | GFP_DMA32);
5040a918
DW		 3552 else if (dev->coherent_dma_mask < dma_get_required_mask(dev)) {
3553 if (dev->coherent_dma_mask < DMA_BIT_MASK(32))
e8bb910d
AW		 3554 flags |= GFP_DMA;
3555 else
3556 flags |= GFP_DMA32;
3557 }
ba395927 3558
36746436
AM		 3559 if (flags & __GFP_WAIT) {
3560 unsigned int count = size >> PAGE_SHIFT;
3561
3562 page = dma_alloc_from_contiguous(dev, count, order);
3563 if (page && iommu_no_mapping(dev) &&
3564 page_to_phys(page) + size > dev->coherent_dma_mask) {
3565 dma_release_from_contiguous(dev, page, count);
3566 page = NULL;
3567 }
3568 }
3569
3570 if (!page)
3571 page = alloc_pages(flags, order);
3572 if (!page)
ba395927 3573 return NULL;
36746436 3574 memset(page_address(page), 0, size);
ba395927 3575
36746436 3576 *dma_handle = __intel_map_single(dev, page_to_phys(page), size,
bb9e6d65 3577 DMA_BIDIRECTIONAL,
5040a918 3578 dev->coherent_dma_mask);
ba395927 3579 if (*dma_handle)
36746436
AM		 3580 return page_address(page);
3581 if (!dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT))
3582 __free_pages(page, order);
3583
ba395927
KA		 3584 return NULL;
3585}
3586
5040a918 3587static void intel_free_coherent(struct device *dev, size_t size, void *vaddr,
baa676fc 3588 dma_addr_t dma_handle, struct dma_attrs *attrs)
ba395927
KA		 3589{
3590 int order;
36746436 3591 struct page *page = virt_to_page(vaddr);
ba395927 3592
5b6985ce 3593 size = PAGE_ALIGN(size);
ba395927
KA		 3594 order = get_order(size);
3595
d41a4adb 3596 intel_unmap(dev, dma_handle);
36746436
AM		 3597 if (!dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT))
3598 __free_pages(page, order);
ba395927
KA		 3599}
3600
5040a918 3601static void intel_unmap_sg(struct device *dev, struct scatterlist *sglist,
d7ab5c46
FT		 3602 int nelems, enum dma_data_direction dir,
3603 struct dma_attrs *attrs)
ba395927 3604{
d41a4adb 3605 intel_unmap(dev, sglist[0].dma_address);
ba395927
KA		 3606}
3607
ba395927 3608static int intel_nontranslate_map_sg(struct device *hddev,
c03ab37c 3609 struct scatterlist *sglist, int nelems, int dir)
ba395927
KA		 3610{
3611 int i;
c03ab37c 3612 struct scatterlist *sg;
ba395927 3613
c03ab37c 3614 for_each_sg(sglist, sg, nelems, i) {
12d4d40e 3615 BUG_ON(!sg_page(sg));
db0fa0cb 3616 sg->dma_address = sg_phys(sg);
c03ab37c 3617 sg->dma_length = sg->length;
ba395927
KA		 3618 }
3619 return nelems;
3620}
3621
5040a918 3622static int intel_map_sg(struct device *dev, struct scatterlist *sglist, int nelems,
d7ab5c46 3623 enum dma_data_direction dir, struct dma_attrs *attrs)
ba395927 3624{
ba395927 3625 int i;
ba395927 3626 struct dmar_domain *domain;
f76aec76
KA		 3627 size_t size = 0;
3628 int prot = 0;
f76aec76
KA		 3629 struct iova *iova = NULL;
3630 int ret;
c03ab37c 3631 struct scatterlist *sg;
b536d24d 3632 unsigned long start_vpfn;
8c11e798 3633 struct intel_iommu *iommu;
ba395927
KA		 3634
3635 BUG_ON(dir == DMA_NONE);
5040a918
DW		 3636 if (iommu_no_mapping(dev))
3637 return intel_nontranslate_map_sg(dev, sglist, nelems, dir);
ba395927 3638
5040a918 3639 domain = get_valid_domain_for_dev(dev);
f76aec76
KA		 3640 if (!domain)
3641 return 0;
3642
8c11e798
WH		 3643 iommu = domain_get_iommu(domain);
3644
b536d24d 3645 for_each_sg(sglist, sg, nelems, i)
88cb6a74 3646 size += aligned_nrpages(sg->offset, sg->length);
f76aec76 3647
5040a918
DW		 3648 iova = intel_alloc_iova(dev, domain, dma_to_mm_pfn(size),
3649 *dev->dma_mask);
f76aec76 3650 if (!iova) {
c03ab37c 3651 sglist->dma_length = 0;
f76aec76
KA		 3652 return 0;
3653 }
3654
3655 /*
3656 * Check if DMAR supports zero-length reads on write only
3657 * mappings..
3658 */
3659 if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
8c11e798 3660 !cap_zlr(iommu->cap))
f76aec76
KA		 3661 prot |= DMA_PTE_READ;
3662 if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
3663 prot |= DMA_PTE_WRITE;
3664
b536d24d 3665 start_vpfn = mm_to_dma_pfn(iova->pfn_lo);
e1605495 3666
f532959b 3667 ret = domain_sg_mapping(domain, start_vpfn, sglist, size, prot);
e1605495 3668 if (unlikely(ret)) {
e1605495
DW		 3669 dma_pte_free_pagetable(domain, start_vpfn,
3670 start_vpfn + size - 1);
e1605495
DW		 3671 __free_iova(&domain->iovad, iova);
3672 return 0;
ba395927
KA		 3673 }
3674
1f0ef2aa
DW		 3675 /* it's a non-present to present mapping. Only flush if caching mode */
3676 if (cap_caching_mode(iommu->cap))
a1ddcbe9 3677 iommu_flush_iotlb_psi(iommu, domain, start_vpfn, size, 0, 1);
1f0ef2aa 3678 else
8c11e798 3679 iommu_flush_write_buffer(iommu);
1f0ef2aa 3680
ba395927
KA		 3681 return nelems;
3682}
3683
dfb805e8
FT		 3684static int intel_mapping_error(struct device *dev, dma_addr_t dma_addr)
3685{
3686 return !dma_addr;
3687}
3688
160c1d8e 3689struct dma_map_ops intel_dma_ops = {
baa676fc
AP		 3690 .alloc = intel_alloc_coherent,
3691 .free = intel_free_coherent,
ba395927
KA		 3692 .map_sg = intel_map_sg,
3693 .unmap_sg = intel_unmap_sg,
ffbbef5c
FT		 3694 .map_page = intel_map_page,
3695 .unmap_page = intel_unmap_page,
dfb805e8 3696 .mapping_error = intel_mapping_error,
ba395927
KA		 3697};
3698
3699static inline int iommu_domain_cache_init(void)
3700{
3701 int ret = 0;
3702
3703 iommu_domain_cache = kmem_cache_create("iommu_domain",
3704 sizeof(struct dmar_domain),
3705 0,
3706 SLAB_HWCACHE_ALIGN,
3707
3708 NULL);
3709 if (!iommu_domain_cache) {
9f10e5bf 3710 pr_err("Couldn't create iommu_domain cache\n");
ba395927
KA		 3711 ret = -ENOMEM;
3712 }
3713
3714 return ret;
3715}
3716
3717static inline int iommu_devinfo_cache_init(void)
3718{
3719 int ret = 0;
3720
3721 iommu_devinfo_cache = kmem_cache_create("iommu_devinfo",
3722 sizeof(struct device_domain_info),
3723 0,
3724 SLAB_HWCACHE_ALIGN,
ba395927
KA		 3725 NULL);
3726 if (!iommu_devinfo_cache) {
9f10e5bf 3727 pr_err("Couldn't create devinfo cache\n");
ba395927
KA		 3728 ret = -ENOMEM;
3729 }
3730
3731 return ret;
3732}
3733
ba395927
KA		 3734static int __init iommu_init_mempool(void)
3735{
3736 int ret;
ae1ff3d6 3737 ret = iova_cache_get();
ba395927
KA		 3738 if (ret)
3739 return ret;
3740
3741 ret = iommu_domain_cache_init();
3742 if (ret)
3743 goto domain_error;
3744
3745 ret = iommu_devinfo_cache_init();
3746 if (!ret)
3747 return ret;
3748
3749 kmem_cache_destroy(iommu_domain_cache);
3750domain_error:
ae1ff3d6 3751 iova_cache_put();
ba395927
KA		 3752
3753 return -ENOMEM;
3754}
3755
3756static void __init iommu_exit_mempool(void)
3757{
3758 kmem_cache_destroy(iommu_devinfo_cache);
3759 kmem_cache_destroy(iommu_domain_cache);
ae1ff3d6 3760 iova_cache_put();
ba395927
KA		 3761}
3762
556ab45f
DW		 3763static void quirk_ioat_snb_local_iommu(struct pci_dev *pdev)
3764{
3765 struct dmar_drhd_unit *drhd;
3766 u32 vtbar;
3767 int rc;
3768
3769 /* We know that this device on this chipset has its own IOMMU.
3770 * If we find it under a different IOMMU, then the BIOS is lying
3771 * to us. Hope that the IOMMU for this device is actually
3772 * disabled, and it needs no translation...
3773 */
3774 rc = pci_bus_read_config_dword(pdev->bus, PCI_DEVFN(0, 0), 0xb0, &vtbar);
3775 if (rc) {
3776 /* "can't" happen */
3777 dev_info(&pdev->dev, "failed to run vt-d quirk\n");
3778 return;
3779 }
3780 vtbar &= 0xffff0000;
3781
3782 /* we know that the this iommu should be at offset 0xa000 from vtbar */
3783 drhd = dmar_find_matched_drhd_unit(pdev);
3784 if (WARN_TAINT_ONCE(!drhd || drhd->reg_base_addr - vtbar != 0xa000,
3785 TAINT_FIRMWARE_WORKAROUND,
3786 "BIOS assigned incorrect VT-d unit for Intel(R) QuickData Technology device\n"))
3787 pdev->dev.archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO;
3788}
3789DECLARE_PCI_FIXUP_ENABLE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB, quirk_ioat_snb_local_iommu);
3790
ba395927
KA		 3791static void __init init_no_remapping_devices(void)
3792{
3793 struct dmar_drhd_unit *drhd;
832bd858 3794 struct device *dev;
b683b230 3795 int i;
ba395927
KA		 3796
3797 for_each_drhd_unit(drhd) {
3798 if (!drhd->include_all) {
b683b230
JL		 3799 for_each_active_dev_scope(drhd->devices,
3800 drhd->devices_cnt, i, dev)
3801 break;
832bd858 3802 /* ignore DMAR unit if no devices exist */
ba395927
KA		 3803 if (i == drhd->devices_cnt)
3804 drhd->ignored = 1;
3805 }
3806 }
3807
7c919779 3808 for_each_active_drhd_unit(drhd) {
7c919779 3809 if (drhd->include_all)
ba395927
KA		 3810 continue;
3811
b683b230
JL		 3812 for_each_active_dev_scope(drhd->devices,
3813 drhd->devices_cnt, i, dev)
832bd858 3814 if (!dev_is_pci(dev) || !IS_GFX_DEVICE(to_pci_dev(dev)))
ba395927 3815 break;
ba395927
KA		 3816 if (i < drhd->devices_cnt)
3817 continue;
3818
c0771df8
DW		 3819 /* This IOMMU has *only* gfx devices. Either bypass it or
3820 set the gfx_mapped flag, as appropriate */
3821 if (dmar_map_gfx) {
3822 intel_iommu_gfx_mapped = 1;
3823 } else {
3824 drhd->ignored = 1;
b683b230
JL		 3825 for_each_active_dev_scope(drhd->devices,
3826 drhd->devices_cnt, i, dev)
832bd858 3827 dev->archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO;
ba395927
KA		 3828 }
3829 }
3830}
3831
f59c7b69
FY		 3832#ifdef CONFIG_SUSPEND
3833static int init_iommu_hw(void)
3834{
3835 struct dmar_drhd_unit *drhd;
3836 struct intel_iommu *iommu = NULL;
3837
3838 for_each_active_iommu(iommu, drhd)
3839 if (iommu->qi)
3840 dmar_reenable_qi(iommu);
3841
b779260b
JC		 3842 for_each_iommu(iommu, drhd) {
3843 if (drhd->ignored) {
3844 /*
3845 * we always have to disable PMRs or DMA may fail on
3846 * this device
3847 */
3848 if (force_on)
3849 iommu_disable_protect_mem_regions(iommu);
3850 continue;
3851 }
3852
f59c7b69
FY		 3853 iommu_flush_write_buffer(iommu);
3854
3855 iommu_set_root_entry(iommu);
3856
3857 iommu->flush.flush_context(iommu, 0, 0, 0,
1f0ef2aa 3858 DMA_CCMD_GLOBAL_INVL);
2a41ccee
JL		 3859 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
3860 iommu_enable_translation(iommu);
b94996c9 3861 iommu_disable_protect_mem_regions(iommu);
f59c7b69
FY		 3862 }
3863
3864 return 0;
3865}
3866
3867static void iommu_flush_all(void)
3868{
3869 struct dmar_drhd_unit *drhd;
3870 struct intel_iommu *iommu;
3871
3872 for_each_active_iommu(iommu, drhd) {
3873 iommu->flush.flush_context(iommu, 0, 0, 0,
1f0ef2aa 3874 DMA_CCMD_GLOBAL_INVL);
f59c7b69 3875 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
1f0ef2aa 3876 DMA_TLB_GLOBAL_FLUSH);
f59c7b69
FY		 3877 }
3878}
3879
134fac3f 3880static int iommu_suspend(void)
f59c7b69
FY		 3881{
3882 struct dmar_drhd_unit *drhd;
3883 struct intel_iommu *iommu = NULL;
3884 unsigned long flag;
3885
3886 for_each_active_iommu(iommu, drhd) {
3887 iommu->iommu_state = kzalloc(sizeof(u32) * MAX_SR_DMAR_REGS,
3888 GFP_ATOMIC);
3889 if (!iommu->iommu_state)
3890 goto nomem;
3891 }
3892
3893 iommu_flush_all();
3894
3895 for_each_active_iommu(iommu, drhd) {
3896 iommu_disable_translation(iommu);
3897
1f5b3c3f 3898 raw_spin_lock_irqsave(&iommu->register_lock, flag);
f59c7b69
FY		 3899
3900 iommu->iommu_state[SR_DMAR_FECTL_REG] =
3901 readl(iommu->reg + DMAR_FECTL_REG);
3902 iommu->iommu_state[SR_DMAR_FEDATA_REG] =
3903 readl(iommu->reg + DMAR_FEDATA_REG);
3904 iommu->iommu_state[SR_DMAR_FEADDR_REG] =
3905 readl(iommu->reg + DMAR_FEADDR_REG);
3906 iommu->iommu_state[SR_DMAR_FEUADDR_REG] =
3907 readl(iommu->reg + DMAR_FEUADDR_REG);
3908
1f5b3c3f 3909 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
f59c7b69
FY		 3910 }
3911 return 0;
3912
3913nomem:
3914 for_each_active_iommu(iommu, drhd)
3915 kfree(iommu->iommu_state);
3916
3917 return -ENOMEM;
3918}
3919
134fac3f 3920static void iommu_resume(void)
f59c7b69
FY		 3921{
3922 struct dmar_drhd_unit *drhd;
3923 struct intel_iommu *iommu = NULL;
3924 unsigned long flag;
3925
3926 if (init_iommu_hw()) {
b779260b
JC		 3927 if (force_on)
3928 panic("tboot: IOMMU setup failed, DMAR can not resume!\n");
3929 else
3930 WARN(1, "IOMMU setup failed, DMAR can not resume!\n");
134fac3f 3931 return;
f59c7b69
FY		 3932 }
3933
3934 for_each_active_iommu(iommu, drhd) {
3935
1f5b3c3f 3936 raw_spin_lock_irqsave(&iommu->register_lock, flag);
f59c7b69
FY		 3937
3938 writel(iommu->iommu_state[SR_DMAR_FECTL_REG],
3939 iommu->reg + DMAR_FECTL_REG);
3940 writel(iommu->iommu_state[SR_DMAR_FEDATA_REG],
3941 iommu->reg + DMAR_FEDATA_REG);
3942 writel(iommu->iommu_state[SR_DMAR_FEADDR_REG],
3943 iommu->reg + DMAR_FEADDR_REG);
3944 writel(iommu->iommu_state[SR_DMAR_FEUADDR_REG],
3945 iommu->reg + DMAR_FEUADDR_REG);
3946
1f5b3c3f 3947 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
f59c7b69
FY		 3948 }
3949
3950 for_each_active_iommu(iommu, drhd)
3951 kfree(iommu->iommu_state);
f59c7b69
FY		 3952}
3953
134fac3f 3954static struct syscore_ops iommu_syscore_ops = {
f59c7b69
FY		 3955 .resume = iommu_resume,
3956 .suspend = iommu_suspend,
3957};
3958
134fac3f 3959static void __init init_iommu_pm_ops(void)
f59c7b69 3960{
134fac3f 3961 register_syscore_ops(&iommu_syscore_ops);
f59c7b69
FY		 3962}
3963
3964#else
99592ba4 3965static inline void init_iommu_pm_ops(void) {}
f59c7b69
FY		 3966#endif /* CONFIG_PM */
3967
318fe7df 3968
c2a0b538 3969int __init dmar_parse_one_rmrr(struct acpi_dmar_header *header, void *arg)
318fe7df
SS		 3970{
3971 struct acpi_dmar_reserved_memory *rmrr;
3972 struct dmar_rmrr_unit *rmrru;
3973
3974 rmrru = kzalloc(sizeof(*rmrru), GFP_KERNEL);
3975 if (!rmrru)
3976 return -ENOMEM;
3977
3978 rmrru->hdr = header;
3979 rmrr = (struct acpi_dmar_reserved_memory *)header;
3980 rmrru->base_address = rmrr->base_address;
3981 rmrru->end_address = rmrr->end_address;
2e455289
JL		 3982 rmrru->devices = dmar_alloc_dev_scope((void *)(rmrr + 1),
3983 ((void *)rmrr) + rmrr->header.length,
3984 &rmrru->devices_cnt);
3985 if (rmrru->devices_cnt && rmrru->devices == NULL) {
3986 kfree(rmrru);
3987 return -ENOMEM;
3988 }
318fe7df 3989
2e455289 3990 list_add(&rmrru->list, &dmar_rmrr_units);
318fe7df 3991
2e455289 3992 return 0;
318fe7df
SS		 3993}
3994
6b197249
JL		 3995static struct dmar_atsr_unit *dmar_find_atsr(struct acpi_dmar_atsr *atsr)
3996{
3997 struct dmar_atsr_unit *atsru;
3998 struct acpi_dmar_atsr *tmp;
3999
4000 list_for_each_entry_rcu(atsru, &dmar_atsr_units, list) {
4001 tmp = (struct acpi_dmar_atsr *)atsru->hdr;
4002 if (atsr->segment != tmp->segment)
4003 continue;
4004 if (atsr->header.length != tmp->header.length)
4005 continue;
4006 if (memcmp(atsr, tmp, atsr->header.length) == 0)
4007 return atsru;
4008 }
4009
4010 return NULL;
4011}
4012
4013int dmar_parse_one_atsr(struct acpi_dmar_header *hdr, void *arg)
318fe7df
SS		 4014{
4015 struct acpi_dmar_atsr *atsr;
4016 struct dmar_atsr_unit *atsru;
4017
6b197249
JL		 4018 if (system_state != SYSTEM_BOOTING && !intel_iommu_enabled)
4019 return 0;
4020
318fe7df 4021 atsr = container_of(hdr, struct acpi_dmar_atsr, header);
6b197249
JL		 4022 atsru = dmar_find_atsr(atsr);
4023 if (atsru)
4024 return 0;
4025
4026 atsru = kzalloc(sizeof(*atsru) + hdr->length, GFP_KERNEL);
318fe7df
SS		 4027 if (!atsru)
4028 return -ENOMEM;
4029
6b197249
JL		 4030 /*
4031 * If memory is allocated from slab by ACPI _DSM method, we need to
4032 * copy the memory content because the memory buffer will be freed
4033 * on return.
4034 */
4035 atsru->hdr = (void *)(atsru + 1);
4036 memcpy(atsru->hdr, hdr, hdr->length);
318fe7df 4037 atsru->include_all = atsr->flags & 0x1;
2e455289
JL		 4038 if (!atsru->include_all) {
4039 atsru->devices = dmar_alloc_dev_scope((void *)(atsr + 1),
4040 (void *)atsr + atsr->header.length,
4041 &atsru->devices_cnt);
4042 if (atsru->devices_cnt && atsru->devices == NULL) {
4043 kfree(atsru);
4044 return -ENOMEM;
4045 }
4046 }
318fe7df 4047
0e242612 4048 list_add_rcu(&atsru->list, &dmar_atsr_units);
318fe7df
SS		 4049
4050 return 0;
4051}
4052
9bdc531e
JL		 4053static void intel_iommu_free_atsr(struct dmar_atsr_unit *atsru)
4054{
4055 dmar_free_dev_scope(&atsru->devices, &atsru->devices_cnt);
4056 kfree(atsru);
4057}
4058
6b197249
JL		 4059int dmar_release_one_atsr(struct acpi_dmar_header *hdr, void *arg)
4060{
4061 struct acpi_dmar_atsr *atsr;
4062 struct dmar_atsr_unit *atsru;
4063
4064 atsr = container_of(hdr, struct acpi_dmar_atsr, header);
4065 atsru = dmar_find_atsr(atsr);
4066 if (atsru) {
4067 list_del_rcu(&atsru->list);
4068 synchronize_rcu();
4069 intel_iommu_free_atsr(atsru);
4070 }
4071
4072 return 0;
4073}
4074
4075int dmar_check_one_atsr(struct acpi_dmar_header *hdr, void *arg)
4076{
4077 int i;
4078 struct device *dev;
4079 struct acpi_dmar_atsr *atsr;
4080 struct dmar_atsr_unit *atsru;
4081
4082 atsr = container_of(hdr, struct acpi_dmar_atsr, header);
4083 atsru = dmar_find_atsr(atsr);
4084 if (!atsru)
4085 return 0;
4086
4087 if (!atsru->include_all && atsru->devices && atsru->devices_cnt)
4088 for_each_active_dev_scope(atsru->devices, atsru->devices_cnt,
4089 i, dev)
4090 return -EBUSY;
4091
4092 return 0;
4093}
4094
ffebeb46
JL		 4095static int intel_iommu_add(struct dmar_drhd_unit *dmaru)
4096{
4097 int sp, ret = 0;
4098 struct intel_iommu *iommu = dmaru->iommu;
4099
4100 if (g_iommus[iommu->seq_id])
4101 return 0;
4102
4103 if (hw_pass_through && !ecap_pass_through(iommu->ecap)) {
9f10e5bf 4104 pr_warn("%s: Doesn't support hardware pass through.\n",
ffebeb46
JL		 4105 iommu->name);
4106 return -ENXIO;
4107 }
4108 if (!ecap_sc_support(iommu->ecap) &&
4109 domain_update_iommu_snooping(iommu)) {
9f10e5bf 4110 pr_warn("%s: Doesn't support snooping.\n",
ffebeb46
JL		 4111 iommu->name);
4112 return -ENXIO;
4113 }
4114 sp = domain_update_iommu_superpage(iommu) - 1;
4115 if (sp >= 0 && !(cap_super_page_val(iommu->cap) & (1 << sp))) {
9f10e5bf 4116 pr_warn("%s: Doesn't support large page.\n",
ffebeb46
JL		 4117 iommu->name);
4118 return -ENXIO;
4119 }
4120
4121 /*
4122 * Disable translation if already enabled prior to OS handover.
4123 */
4124 if (iommu->gcmd & DMA_GCMD_TE)
4125 iommu_disable_translation(iommu);
4126
4127 g_iommus[iommu->seq_id] = iommu;
4128 ret = iommu_init_domains(iommu);
4129 if (ret == 0)
4130 ret = iommu_alloc_root_entry(iommu);
4131 if (ret)
4132 goto out;
4133
8a94ade4
DW		 4134#ifdef CONFIG_INTEL_IOMMU_SVM
4135 if (pasid_enabled(iommu))
4136 intel_svm_alloc_pasid_tables(iommu);
4137#endif
4138
ffebeb46
JL		 4139 if (dmaru->ignored) {
4140 /*
4141 * we always have to disable PMRs or DMA may fail on this device
4142 */
4143 if (force_on)
4144 iommu_disable_protect_mem_regions(iommu);
4145 return 0;
4146 }
4147
4148 intel_iommu_init_qi(iommu);
4149 iommu_flush_write_buffer(iommu);
4150 ret = dmar_set_interrupt(iommu);
4151 if (ret)
4152 goto disable_iommu;
4153
4154 iommu_set_root_entry(iommu);
4155 iommu->flush.flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL);
4156 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
4157 iommu_enable_translation(iommu);
4158
ffebeb46
JL		 4159 iommu_disable_protect_mem_regions(iommu);
4160 return 0;
4161
4162disable_iommu:
4163 disable_dmar_iommu(iommu);
4164out:
4165 free_dmar_iommu(iommu);
4166 return ret;
4167}
4168
6b197249
JL		 4169int dmar_iommu_hotplug(struct dmar_drhd_unit *dmaru, bool insert)
4170{
ffebeb46
JL		 4171 int ret = 0;
4172 struct intel_iommu *iommu = dmaru->iommu;
4173
4174 if (!intel_iommu_enabled)
4175 return 0;
4176 if (iommu == NULL)
4177 return -EINVAL;
4178
4179 if (insert) {
4180 ret = intel_iommu_add(dmaru);
4181 } else {
4182 disable_dmar_iommu(iommu);
4183 free_dmar_iommu(iommu);
4184 }
4185
4186 return ret;
6b197249
JL		 4187}
4188
9bdc531e
JL		 4189static void intel_iommu_free_dmars(void)
4190{
4191 struct dmar_rmrr_unit *rmrru, *rmrr_n;
4192 struct dmar_atsr_unit *atsru, *atsr_n;
4193
4194 list_for_each_entry_safe(rmrru, rmrr_n, &dmar_rmrr_units, list) {
4195 list_del(&rmrru->list);
4196 dmar_free_dev_scope(&rmrru->devices, &rmrru->devices_cnt);
4197 kfree(rmrru);
318fe7df
SS		 4198 }
4199
9bdc531e
JL		 4200 list_for_each_entry_safe(atsru, atsr_n, &dmar_atsr_units, list) {
4201 list_del(&atsru->list);
4202 intel_iommu_free_atsr(atsru);
4203 }
318fe7df
SS		 4204}
4205
4206int dmar_find_matched_atsr_unit(struct pci_dev *dev)
4207{
b683b230 4208 int i, ret = 1;
318fe7df 4209 struct pci_bus *bus;
832bd858
DW		 4210 struct pci_dev *bridge = NULL;
4211 struct device *tmp;
318fe7df
SS		 4212 struct acpi_dmar_atsr *atsr;
4213 struct dmar_atsr_unit *atsru;
4214
4215 dev = pci_physfn(dev);
318fe7df 4216 for (bus = dev->bus; bus; bus = bus->parent) {
b5f82ddf 4217 bridge = bus->self;
d14053b3
DW		 4218 /* If it's an integrated device, allow ATS */
4219 if (!bridge)
4220 return 1;
4221 /* Connected via non-PCIe: no ATS */
4222 if (!pci_is_pcie(bridge) ||
62f87c0e 4223 pci_pcie_type(bridge) == PCI_EXP_TYPE_PCI_BRIDGE)
318fe7df 4224 return 0;
d14053b3 4225 /* If we found the root port, look it up in the ATSR */
b5f82ddf 4226 if (pci_pcie_type(bridge) == PCI_EXP_TYPE_ROOT_PORT)
318fe7df 4227 break;
318fe7df
SS		 4228 }
4229
0e242612 4230 rcu_read_lock();
b5f82ddf
JL		 4231 list_for_each_entry_rcu(atsru, &dmar_atsr_units, list) {
4232 atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header);
4233 if (atsr->segment != pci_domain_nr(dev->bus))
4234 continue;
4235
b683b230 4236 for_each_dev_scope(atsru->devices, atsru->devices_cnt, i, tmp)
832bd858 4237 if (tmp == &bridge->dev)
b683b230 4238 goto out;
b5f82ddf
JL		 4239
4240 if (atsru->include_all)
b683b230 4241 goto out;
b5f82ddf 4242 }
b683b230
JL		 4243 ret = 0;
4244out:
0e242612 4245 rcu_read_unlock();
318fe7df 4246
b683b230 4247 return ret;
318fe7df
SS		 4248}
4249
59ce0515
JL		 4250int dmar_iommu_notify_scope_dev(struct dmar_pci_notify_info *info)
4251{
4252 int ret = 0;
4253 struct dmar_rmrr_unit *rmrru;
4254 struct dmar_atsr_unit *atsru;
4255 struct acpi_dmar_atsr *atsr;
4256 struct acpi_dmar_reserved_memory *rmrr;
4257
4258 if (!intel_iommu_enabled && system_state != SYSTEM_BOOTING)
4259 return 0;
4260
4261 list_for_each_entry(rmrru, &dmar_rmrr_units, list) {
4262 rmrr = container_of(rmrru->hdr,
4263 struct acpi_dmar_reserved_memory, header);
4264 if (info->event == BUS_NOTIFY_ADD_DEVICE) {
4265 ret = dmar_insert_dev_scope(info, (void *)(rmrr + 1),
4266 ((void *)rmrr) + rmrr->header.length,
4267 rmrr->segment, rmrru->devices,
4268 rmrru->devices_cnt);
27e24950 4269 if(ret < 0)
59ce0515
JL		 4270 return ret;
4271 } else if (info->event == BUS_NOTIFY_DEL_DEVICE) {
27e24950
JL		 4272 dmar_remove_dev_scope(info, rmrr->segment,
4273 rmrru->devices, rmrru->devices_cnt);
59ce0515
JL		 4274 }
4275 }
4276
4277 list_for_each_entry(atsru, &dmar_atsr_units, list) {
4278 if (atsru->include_all)
4279 continue;
4280
4281 atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header);
4282 if (info->event == BUS_NOTIFY_ADD_DEVICE) {
4283 ret = dmar_insert_dev_scope(info, (void *)(atsr + 1),
4284 (void *)atsr + atsr->header.length,
4285 atsr->segment, atsru->devices,
4286 atsru->devices_cnt);
4287 if (ret > 0)
4288 break;
4289 else if(ret < 0)
4290 return ret;
4291 } else if (info->event == BUS_NOTIFY_DEL_DEVICE) {
4292 if (dmar_remove_dev_scope(info, atsr->segment,
4293 atsru->devices, atsru->devices_cnt))
4294 break;
4295 }
4296 }
4297
4298 return 0;
4299}
4300
99dcaded
FY		 4301/*
4302 * Here we only respond to action of unbound device from driver.
4303 *
4304 * Added device is not attached to its DMAR domain here yet. That will happen
4305 * when mapping the device to iova.
4306 */
4307static int device_notifier(struct notifier_block *nb,
4308 unsigned long action, void *data)
4309{
4310 struct device *dev = data;
99dcaded
FY		 4311 struct dmar_domain *domain;
4312
3d89194a 4313 if (iommu_dummy(dev))
44cd613c
DW		 4314 return 0;
4315
1196c2fb 4316 if (action != BUS_NOTIFY_REMOVED_DEVICE)
7e7dfab7
JL		 4317 return 0;
4318
1525a29a 4319 domain = find_domain(dev);
99dcaded
FY		 4320 if (!domain)
4321 return 0;
4322
e6de0f8d 4323 dmar_remove_one_dev_info(domain, dev);
ab8dfe25 4324 if (!domain_type_is_vm_or_si(domain) && list_empty(&domain->devices))
7e7dfab7 4325 domain_exit(domain);
a97590e5 4326
99dcaded
FY		 4327 return 0;
4328}
4329
4330static struct notifier_block device_nb = {
4331 .notifier_call = device_notifier,
4332};
4333
75f05569
JL		 4334static int intel_iommu_memory_notifier(struct notifier_block *nb,
4335 unsigned long val, void *v)
4336{
4337 struct memory_notify *mhp = v;
4338 unsigned long long start, end;
4339 unsigned long start_vpfn, last_vpfn;
4340
4341 switch (val) {
4342 case MEM_GOING_ONLINE:
4343 start = mhp->start_pfn << PAGE_SHIFT;
4344 end = ((mhp->start_pfn + mhp->nr_pages) << PAGE_SHIFT) - 1;
4345 if (iommu_domain_identity_map(si_domain, start, end)) {
9f10e5bf 4346 pr_warn("Failed to build identity map for [%llx-%llx]\n",
75f05569
JL		 4347 start, end);
4348 return NOTIFY_BAD;
4349 }
4350 break;
4351
4352 case MEM_OFFLINE:
4353 case MEM_CANCEL_ONLINE:
4354 start_vpfn = mm_to_dma_pfn(mhp->start_pfn);
4355 last_vpfn = mm_to_dma_pfn(mhp->start_pfn + mhp->nr_pages - 1);
4356 while (start_vpfn <= last_vpfn) {
4357 struct iova *iova;
4358 struct dmar_drhd_unit *drhd;
4359 struct intel_iommu *iommu;
ea8ea460 4360 struct page *freelist;
75f05569
JL		 4361
4362 iova = find_iova(&si_domain->iovad, start_vpfn);
4363 if (iova == NULL) {
9f10e5bf 4364 pr_debug("Failed get IOVA for PFN %lx\n",
75f05569
JL		 4365 start_vpfn);
4366 break;
4367 }
4368
4369 iova = split_and_remove_iova(&si_domain->iovad, iova,
4370 start_vpfn, last_vpfn);
4371 if (iova == NULL) {
9f10e5bf 4372 pr_warn("Failed to split IOVA PFN [%lx-%lx]\n",
75f05569
JL		 4373 start_vpfn, last_vpfn);
4374 return NOTIFY_BAD;
4375 }
4376
ea8ea460
DW		 4377 freelist = domain_unmap(si_domain, iova->pfn_lo,
4378 iova->pfn_hi);
4379
75f05569
JL		 4380 rcu_read_lock();
4381 for_each_active_iommu(iommu, drhd)
a1ddcbe9 4382 iommu_flush_iotlb_psi(iommu, si_domain,
a156ef99 4383 iova->pfn_lo, iova_size(iova),
ea8ea460 4384 !freelist, 0);
75f05569 4385 rcu_read_unlock();
ea8ea460 4386 dma_free_pagelist(freelist);
75f05569
JL		 4387
4388 start_vpfn = iova->pfn_hi + 1;
4389 free_iova_mem(iova);
4390 }
4391 break;
4392 }
4393
4394 return NOTIFY_OK;
4395}
4396
4397static struct notifier_block intel_iommu_memory_nb = {
4398 .notifier_call = intel_iommu_memory_notifier,
4399 .priority = 0
4400};
4401
a5459cfe
AW		 4402
4403static ssize_t intel_iommu_show_version(struct device *dev,
4404 struct device_attribute *attr,
4405 char *buf)
4406{
4407 struct intel_iommu *iommu = dev_get_drvdata(dev);
4408 u32 ver = readl(iommu->reg + DMAR_VER_REG);
4409 return sprintf(buf, "%d:%d\n",
4410 DMAR_VER_MAJOR(ver), DMAR_VER_MINOR(ver));
4411}
4412static DEVICE_ATTR(version, S_IRUGO, intel_iommu_show_version, NULL);
4413
4414static ssize_t intel_iommu_show_address(struct device *dev,
4415 struct device_attribute *attr,
4416 char *buf)
4417{
4418 struct intel_iommu *iommu = dev_get_drvdata(dev);
4419 return sprintf(buf, "%llx\n", iommu->reg_phys);
4420}
4421static DEVICE_ATTR(address, S_IRUGO, intel_iommu_show_address, NULL);
4422
4423static ssize_t intel_iommu_show_cap(struct device *dev,
4424 struct device_attribute *attr,
4425 char *buf)
4426{
4427 struct intel_iommu *iommu = dev_get_drvdata(dev);
4428 return sprintf(buf, "%llx\n", iommu->cap);
4429}
4430static DEVICE_ATTR(cap, S_IRUGO, intel_iommu_show_cap, NULL);
4431
4432static ssize_t intel_iommu_show_ecap(struct device *dev,
4433 struct device_attribute *attr,
4434 char *buf)
4435{
4436 struct intel_iommu *iommu = dev_get_drvdata(dev);
4437 return sprintf(buf, "%llx\n", iommu->ecap);
4438}
4439static DEVICE_ATTR(ecap, S_IRUGO, intel_iommu_show_ecap, NULL);
4440
2238c082
AW		 4441static ssize_t intel_iommu_show_ndoms(struct device *dev,
4442 struct device_attribute *attr,
4443 char *buf)
4444{
4445 struct intel_iommu *iommu = dev_get_drvdata(dev);
4446 return sprintf(buf, "%ld\n", cap_ndoms(iommu->cap));
4447}
4448static DEVICE_ATTR(domains_supported, S_IRUGO, intel_iommu_show_ndoms, NULL);
4449
4450static ssize_t intel_iommu_show_ndoms_used(struct device *dev,
4451 struct device_attribute *attr,
4452 char *buf)
4453{
4454 struct intel_iommu *iommu = dev_get_drvdata(dev);
4455 return sprintf(buf, "%d\n", bitmap_weight(iommu->domain_ids,
4456 cap_ndoms(iommu->cap)));
4457}
4458static DEVICE_ATTR(domains_used, S_IRUGO, intel_iommu_show_ndoms_used, NULL);
4459
a5459cfe
AW		 4460static struct attribute *intel_iommu_attrs[] = {
4461 &dev_attr_version.attr,
4462 &dev_attr_address.attr,
4463 &dev_attr_cap.attr,
4464 &dev_attr_ecap.attr,
2238c082
AW		 4465 &dev_attr_domains_supported.attr,
4466 &dev_attr_domains_used.attr,
a5459cfe
AW		 4467 NULL,
4468};
4469
4470static struct attribute_group intel_iommu_group = {
4471 .name = "intel-iommu",
4472 .attrs = intel_iommu_attrs,
4473};
4474
4475const struct attribute_group *intel_iommu_groups[] = {
4476 &intel_iommu_group,
4477 NULL,
4478};
4479
ba395927
KA		 4480int __init intel_iommu_init(void)
4481{
9bdc531e 4482 int ret = -ENODEV;
3a93c841 4483 struct dmar_drhd_unit *drhd;
7c919779 4484 struct intel_iommu *iommu;
ba395927 4485
a59b50e9
JC		 4486 /* VT-d is required for a TXT/tboot launch, so enforce that */
4487 force_on = tboot_force_iommu();
4488
3a5670e8
JL		 4489 if (iommu_init_mempool()) {
4490 if (force_on)
4491 panic("tboot: Failed to initialize iommu memory\n");
4492 return -ENOMEM;
4493 }
4494
4495 down_write(&dmar_global_lock);
a59b50e9
JC		 4496 if (dmar_table_init()) {
4497 if (force_on)
4498 panic("tboot: Failed to initialize DMAR table\n");
9bdc531e 4499 goto out_free_dmar;
a59b50e9 4500 }
ba395927 4501
c2c7286a 4502 if (dmar_dev_scope_init() < 0) {
a59b50e9
JC		 4503 if (force_on)
4504 panic("tboot: Failed to initialize DMAR device scope\n");
9bdc531e 4505 goto out_free_dmar;
a59b50e9 4506 }
1886e8a9 4507
75f1cdf1 4508 if (no_iommu || dmar_disabled)
9bdc531e 4509 goto out_free_dmar;
2ae21010 4510
318fe7df 4511 if (list_empty(&dmar_rmrr_units))
9f10e5bf 4512 pr_info("No RMRR found\n");
318fe7df
SS		 4513
4514 if (list_empty(&dmar_atsr_units))
9f10e5bf 4515 pr_info("No ATSR found\n");
318fe7df 4516
51a63e67
JC		 4517 if (dmar_init_reserved_ranges()) {
4518 if (force_on)
4519 panic("tboot: Failed to reserve iommu ranges\n");
3a5670e8 4520 goto out_free_reserved_range;
51a63e67 4521 }
ba395927
KA		 4522
4523 init_no_remapping_devices();
4524
b779260b 4525 ret = init_dmars();
ba395927 4526 if (ret) {
a59b50e9
JC		 4527 if (force_on)
4528 panic("tboot: Failed to initialize DMARs\n");
9f10e5bf 4529 pr_err("Initialization failed\n");
9bdc531e 4530 goto out_free_reserved_range;
ba395927 4531 }
3a5670e8 4532 up_write(&dmar_global_lock);
9f10e5bf 4533 pr_info("Intel(R) Virtualization Technology for Directed I/O\n");
ba395927 4534
5e0d2a6f 4535 init_timer(&unmap_timer);
75f1cdf1
FT		 4536#ifdef CONFIG_SWIOTLB
4537 swiotlb = 0;
4538#endif
19943b0e 4539 dma_ops = &intel_dma_ops;
4ed0d3e6 4540
134fac3f 4541 init_iommu_pm_ops();
a8bcbb0d 4542
a5459cfe
AW		 4543 for_each_active_iommu(iommu, drhd)
4544 iommu->iommu_dev = iommu_device_create(NULL, iommu,
4545 intel_iommu_groups,
2439d4aa 4546 "%s", iommu->name);
a5459cfe 4547
4236d97d 4548 bus_set_iommu(&pci_bus_type, &intel_iommu_ops);
99dcaded 4549 bus_register_notifier(&pci_bus_type, &device_nb);
75f05569
JL		 4550 if (si_domain && !hw_pass_through)
4551 register_memory_notifier(&intel_iommu_memory_nb);
99dcaded 4552
8bc1f85c
ED		 4553 intel_iommu_enabled = 1;
4554
ba395927 4555 return 0;
9bdc531e
JL		 4556
4557out_free_reserved_range:
4558 put_iova_domain(&reserved_iova_list);
9bdc531e
JL		 4559out_free_dmar:
4560 intel_iommu_free_dmars();
3a5670e8
JL		 4561 up_write(&dmar_global_lock);
4562 iommu_exit_mempool();
9bdc531e 4563 return ret;
ba395927 4564}
e820482c 4565
2452d9db 4566static int domain_context_clear_one_cb(struct pci_dev *pdev, u16 alias, void *opaque)
579305f7
AW		 4567{
4568 struct intel_iommu *iommu = opaque;
4569
2452d9db 4570 domain_context_clear_one(iommu, PCI_BUS_NUM(alias), alias & 0xff);
579305f7
AW		 4571 return 0;
4572}
4573
4574/*
4575 * NB - intel-iommu lacks any sort of reference counting for the users of
4576 * dependent devices. If multiple endpoints have intersecting dependent
4577 * devices, unbinding the driver from any one of them will possibly leave
4578 * the others unable to operate.
4579 */
2452d9db 4580static void domain_context_clear(struct intel_iommu *iommu, struct device *dev)
3199aa6b 4581{
0bcb3e28 4582 if (!iommu || !dev || !dev_is_pci(dev))
3199aa6b
HW		 4583 return;
4584
2452d9db 4585 pci_for_each_dma_alias(to_pci_dev(dev), &domain_context_clear_one_cb, iommu);
3199aa6b
HW		 4586}
4587
127c7615 4588static void __dmar_remove_one_dev_info(struct device_domain_info *info)
c7151a8d 4589{
c7151a8d
WH		 4590 struct intel_iommu *iommu;
4591 unsigned long flags;
c7151a8d 4592
55d94043
JR		 4593 assert_spin_locked(&device_domain_lock);
4594
127c7615 4595 if (WARN_ON(!info))
c7151a8d
WH		 4596 return;
4597
127c7615 4598 iommu = info->iommu;
c7151a8d 4599
127c7615
JR		 4600 if (info->dev) {
4601 iommu_disable_dev_iotlb(info);
4602 domain_context_clear(iommu, info->dev);
4603 }
c7151a8d 4604
b608ac3b 4605 unlink_domain_info(info);
c7151a8d 4606
d160aca5 4607 spin_lock_irqsave(&iommu->lock, flags);
127c7615 4608 domain_detach_iommu(info->domain, iommu);
d160aca5 4609 spin_unlock_irqrestore(&iommu->lock, flags);
c7151a8d 4610
127c7615 4611 free_devinfo_mem(info);
c7151a8d 4612}
c7151a8d 4613
55d94043
JR		 4614static void dmar_remove_one_dev_info(struct dmar_domain *domain,
4615 struct device *dev)
4616{
127c7615 4617 struct device_domain_info *info;
55d94043 4618 unsigned long flags;
3e7abe25 4619
55d94043 4620 spin_lock_irqsave(&device_domain_lock, flags);
127c7615
JR		 4621 info = dev->archdata.iommu;
4622 __dmar_remove_one_dev_info(info);
55d94043 4623 spin_unlock_irqrestore(&device_domain_lock, flags);
c7151a8d
WH		 4624}
4625
2c2e2c38 4626static int md_domain_init(struct dmar_domain *domain, int guest_width)
5e98c4b1
WH		 4627{
4628 int adjust_width;
4629
0fb5fe87
RM		 4630 init_iova_domain(&domain->iovad, VTD_PAGE_SIZE, IOVA_START_PFN,
4631 DMA_32BIT_PFN);
5e98c4b1
WH		 4632 domain_reserve_special_ranges(domain);
4633
4634 /* calculate AGAW */
4635 domain->gaw = guest_width;
4636 adjust_width = guestwidth_to_adjustwidth(guest_width);
4637 domain->agaw = width_to_agaw(adjust_width);
4638
5e98c4b1 4639 domain->iommu_coherency = 0;
c5b15255 4640 domain->iommu_snooping = 0;
6dd9a7c7 4641 domain->iommu_superpage = 0;
fe40f1e0 4642 domain->max_addr = 0;
5e98c4b1
WH		 4643
4644 /* always allocate the top pgd */
4c923d47 4645 domain->pgd = (struct dma_pte *)alloc_pgtable_page(domain->nid);
5e98c4b1
WH		 4646 if (!domain->pgd)
4647 return -ENOMEM;
4648 domain_flush_cache(domain, domain->pgd, PAGE_SIZE);
4649 return 0;
4650}
4651
00a77deb 4652static struct iommu_domain *intel_iommu_domain_alloc(unsigned type)
38717946 4653{
5d450806 4654 struct dmar_domain *dmar_domain;
00a77deb
JR		 4655 struct iommu_domain *domain;
4656
4657 if (type != IOMMU_DOMAIN_UNMANAGED)
4658 return NULL;
38717946 4659
ab8dfe25 4660 dmar_domain = alloc_domain(DOMAIN_FLAG_VIRTUAL_MACHINE);
5d450806 4661 if (!dmar_domain) {
9f10e5bf 4662 pr_err("Can't allocate dmar_domain\n");
00a77deb 4663 return NULL;
38717946 4664 }
2c2e2c38 4665 if (md_domain_init(dmar_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
9f10e5bf 4666 pr_err("Domain initialization failed\n");
92d03cc8 4667 domain_exit(dmar_domain);
00a77deb 4668 return NULL;
38717946 4669 }
8140a95d 4670 domain_update_iommu_cap(dmar_domain);
faa3d6f5 4671
00a77deb 4672 domain = &dmar_domain->domain;
8a0e715b
JR		 4673 domain->geometry.aperture_start = 0;
4674 domain->geometry.aperture_end = __DOMAIN_MAX_ADDR(dmar_domain->gaw);
4675 domain->geometry.force_aperture = true;
4676
00a77deb 4677 return domain;
38717946 4678}
38717946 4679
00a77deb 4680static void intel_iommu_domain_free(struct iommu_domain *domain)
38717946 4681{
00a77deb 4682 domain_exit(to_dmar_domain(domain));
38717946 4683}
38717946 4684
4c5478c9
JR		 4685static int intel_iommu_attach_device(struct iommu_domain *domain,
4686 struct device *dev)
38717946 4687{
00a77deb 4688 struct dmar_domain *dmar_domain = to_dmar_domain(domain);
fe40f1e0
WH		 4689 struct intel_iommu *iommu;
4690 int addr_width;
156baca8 4691 u8 bus, devfn;
faa3d6f5 4692
c875d2c1
AW		 4693 if (device_is_rmrr_locked(dev)) {
4694 dev_warn(dev, "Device is ineligible for IOMMU domain attach due to platform RMRR requirement. Contact your platform vendor.\n");
4695 return -EPERM;
4696 }
4697
7207d8f9
DW		 4698 /* normally dev is not mapped */
4699 if (unlikely(domain_context_mapped(dev))) {
faa3d6f5
WH		 4700 struct dmar_domain *old_domain;
4701
1525a29a 4702 old_domain = find_domain(dev);
faa3d6f5 4703 if (old_domain) {
d160aca5 4704 rcu_read_lock();
de7e8886 4705 dmar_remove_one_dev_info(old_domain, dev);
d160aca5 4706 rcu_read_unlock();
62c22167
JR		 4707
4708 if (!domain_type_is_vm_or_si(old_domain) &&
4709 list_empty(&old_domain->devices))
4710 domain_exit(old_domain);
faa3d6f5
WH		 4711 }
4712 }
4713
156baca8 4714 iommu = device_to_iommu(dev, &bus, &devfn);
fe40f1e0
WH		 4715 if (!iommu)
4716 return -ENODEV;
4717
4718 /* check if this iommu agaw is sufficient for max mapped address */
4719 addr_width = agaw_to_width(iommu->agaw);
a99c47a2
TL		 4720 if (addr_width > cap_mgaw(iommu->cap))
4721 addr_width = cap_mgaw(iommu->cap);
4722
4723 if (dmar_domain->max_addr > (1LL << addr_width)) {
9f10e5bf 4724 pr_err("%s: iommu width (%d) is not "
fe40f1e0 4725 "sufficient for the mapped address (%llx)\n",
a99c47a2 4726 __func__, addr_width, dmar_domain->max_addr);
fe40f1e0
WH		 4727 return -EFAULT;
4728 }
a99c47a2
TL		 4729 dmar_domain->gaw = addr_width;
4730
4731 /*
4732 * Knock out extra levels of page tables if necessary
4733 */
4734 while (iommu->agaw < dmar_domain->agaw) {
4735 struct dma_pte *pte;
4736
4737 pte = dmar_domain->pgd;
4738 if (dma_pte_present(pte)) {
25cbff16
SY		 4739 dmar_domain->pgd = (struct dma_pte *)
4740 phys_to_virt(dma_pte_addr(pte));
7a661013 4741 free_pgtable_page(pte);
a99c47a2
TL		 4742 }
4743 dmar_domain->agaw--;
4744 }
fe40f1e0 4745
28ccce0d 4746 return domain_add_dev_info(dmar_domain, dev);
38717946 4747}
38717946 4748
4c5478c9
JR		 4749static void intel_iommu_detach_device(struct iommu_domain *domain,
4750 struct device *dev)
38717946 4751{
e6de0f8d 4752 dmar_remove_one_dev_info(to_dmar_domain(domain), dev);
faa3d6f5 4753}
c7151a8d 4754
b146a1c9
JR		 4755static int intel_iommu_map(struct iommu_domain *domain,
4756 unsigned long iova, phys_addr_t hpa,
5009065d 4757 size_t size, int iommu_prot)
faa3d6f5 4758{
00a77deb 4759 struct dmar_domain *dmar_domain = to_dmar_domain(domain);
fe40f1e0 4760 u64 max_addr;
dde57a21 4761 int prot = 0;
faa3d6f5 4762 int ret;
fe40f1e0 4763
dde57a21
JR		 4764 if (iommu_prot & IOMMU_READ)
4765 prot |= DMA_PTE_READ;
4766 if (iommu_prot & IOMMU_WRITE)
4767 prot |= DMA_PTE_WRITE;
9cf06697
SY		 4768 if ((iommu_prot & IOMMU_CACHE) && dmar_domain->iommu_snooping)
4769 prot |= DMA_PTE_SNP;
dde57a21 4770
163cc52c 4771 max_addr = iova + size;
dde57a21 4772 if (dmar_domain->max_addr < max_addr) {
fe40f1e0
WH		 4773 u64 end;
4774
4775 /* check if minimum agaw is sufficient for mapped address */
8954da1f 4776 end = __DOMAIN_MAX_ADDR(dmar_domain->gaw) + 1;
fe40f1e0 4777 if (end < max_addr) {
9f10e5bf 4778 pr_err("%s: iommu width (%d) is not "
fe40f1e0 4779 "sufficient for the mapped address (%llx)\n",
8954da1f 4780 __func__, dmar_domain->gaw, max_addr);
fe40f1e0
WH		 4781 return -EFAULT;
4782 }
dde57a21 4783 dmar_domain->max_addr = max_addr;
fe40f1e0 4784 }
ad051221
DW		 4785 /* Round up size to next multiple of PAGE_SIZE, if it and
4786 the low bits of hpa would take us onto the next page */
88cb6a74 4787 size = aligned_nrpages(hpa, size);
ad051221
DW		 4788 ret = domain_pfn_mapping(dmar_domain, iova >> VTD_PAGE_SHIFT,
4789 hpa >> VTD_PAGE_SHIFT, size, prot);
faa3d6f5 4790 return ret;
38717946 4791}
38717946 4792
5009065d 4793static size_t intel_iommu_unmap(struct iommu_domain *domain,
ea8ea460 4794 unsigned long iova, size_t size)
38717946 4795{
00a77deb 4796 struct dmar_domain *dmar_domain = to_dmar_domain(domain);
ea8ea460
DW		 4797 struct page *freelist = NULL;
4798 struct intel_iommu *iommu;
4799 unsigned long start_pfn, last_pfn;
4800 unsigned int npages;
42e8c186 4801 int iommu_id, level = 0;
5cf0a76f
DW		 4802
4803 /* Cope with horrid API which requires us to unmap more than the
4804 size argument if it happens to be a large-page mapping. */
dc02e46e 4805 BUG_ON(!pfn_to_dma_pte(dmar_domain, iova >> VTD_PAGE_SHIFT, &level));
5cf0a76f
DW		 4806
4807 if (size < VTD_PAGE_SIZE << level_to_offset_bits(level))
4808 size = VTD_PAGE_SIZE << level_to_offset_bits(level);
4b99d352 4809
ea8ea460
DW		 4810 start_pfn = iova >> VTD_PAGE_SHIFT;
4811 last_pfn = (iova + size - 1) >> VTD_PAGE_SHIFT;
4812
4813 freelist = domain_unmap(dmar_domain, start_pfn, last_pfn);
4814
4815 npages = last_pfn - start_pfn + 1;
4816
29a27719 4817 for_each_domain_iommu(iommu_id, dmar_domain) {
a1ddcbe9 4818 iommu = g_iommus[iommu_id];
ea8ea460 4819
42e8c186
JR		 4820 iommu_flush_iotlb_psi(g_iommus[iommu_id], dmar_domain,
4821 start_pfn, npages, !freelist, 0);
ea8ea460
DW		 4822 }
4823
4824 dma_free_pagelist(freelist);
fe40f1e0 4825
163cc52c
DW		 4826 if (dmar_domain->max_addr == iova + size)
4827 dmar_domain->max_addr = iova;
b146a1c9 4828
5cf0a76f 4829 return size;
38717946 4830}
38717946 4831
d14d6577 4832static phys_addr_t intel_iommu_iova_to_phys(struct iommu_domain *domain,
bb5547ac 4833 dma_addr_t iova)
38717946 4834{
00a77deb 4835 struct dmar_domain *dmar_domain = to_dmar_domain(domain);
38717946 4836 struct dma_pte *pte;
5cf0a76f 4837 int level = 0;
faa3d6f5 4838 u64 phys = 0;
38717946 4839
5cf0a76f 4840 pte = pfn_to_dma_pte(dmar_domain, iova >> VTD_PAGE_SHIFT, &level);
38717946 4841 if (pte)
faa3d6f5 4842 phys = dma_pte_addr(pte);
38717946 4843
faa3d6f5 4844 return phys;
38717946 4845}
a8bcbb0d 4846
5d587b8d 4847static bool intel_iommu_capable(enum iommu_cap cap)
dbb9fd86 4848{
dbb9fd86 4849 if (cap == IOMMU_CAP_CACHE_COHERENCY)
5d587b8d 4850 return domain_update_iommu_snooping(NULL) == 1;
323f99cb 4851 if (cap == IOMMU_CAP_INTR_REMAP)
5d587b8d 4852 return irq_remapping_enabled == 1;
dbb9fd86 4853
5d587b8d 4854 return false;
dbb9fd86
SY		 4855}
4856
abdfdde2
AW		 4857static int intel_iommu_add_device(struct device *dev)
4858{
a5459cfe 4859 struct intel_iommu *iommu;
abdfdde2 4860 struct iommu_group *group;
156baca8 4861 u8 bus, devfn;
70ae6f0d 4862
a5459cfe
AW		 4863 iommu = device_to_iommu(dev, &bus, &devfn);
4864 if (!iommu)
70ae6f0d
AW		 4865 return -ENODEV;
4866
a5459cfe 4867 iommu_device_link(iommu->iommu_dev, dev);
a4ff1fc2 4868
e17f9ff4 4869 group = iommu_group_get_for_dev(dev);
783f157b 4870
e17f9ff4
AW		 4871 if (IS_ERR(group))
4872 return PTR_ERR(group);
bcb71abe 4873
abdfdde2 4874 iommu_group_put(group);
e17f9ff4 4875 return 0;
abdfdde2 4876}
70ae6f0d 4877
abdfdde2
AW		 4878static void intel_iommu_remove_device(struct device *dev)
4879{
a5459cfe
AW		 4880 struct intel_iommu *iommu;
4881 u8 bus, devfn;
4882
4883 iommu = device_to_iommu(dev, &bus, &devfn);
4884 if (!iommu)
4885 return;
4886
abdfdde2 4887 iommu_group_remove_device(dev);
a5459cfe
AW		 4888
4889 iommu_device_unlink(iommu->iommu_dev, dev);
70ae6f0d
AW		 4890}
4891
b22f6434 4892static const struct iommu_ops intel_iommu_ops = {
5d587b8d 4893 .capable = intel_iommu_capable,
00a77deb
JR		 4894 .domain_alloc = intel_iommu_domain_alloc,
4895 .domain_free = intel_iommu_domain_free,
a8bcbb0d
JR		 4896 .attach_dev = intel_iommu_attach_device,
4897 .detach_dev = intel_iommu_detach_device,
b146a1c9
JR		 4898 .map = intel_iommu_map,
4899 .unmap = intel_iommu_unmap,
315786eb 4900 .map_sg = default_iommu_map_sg,
a8bcbb0d 4901 .iova_to_phys = intel_iommu_iova_to_phys,
abdfdde2
AW		 4902 .add_device = intel_iommu_add_device,
4903 .remove_device = intel_iommu_remove_device,
6d1c56a9 4904 .pgsize_bitmap = INTEL_IOMMU_PGSIZES,
a8bcbb0d 4905};
9af88143 4906
9452618e
DV		 4907static void quirk_iommu_g4x_gfx(struct pci_dev *dev)
4908{
4909 /* G4x/GM45 integrated gfx dmar support is totally busted. */
9f10e5bf 4910 pr_info("Disabling IOMMU for graphics on this chipset\n");
9452618e
DV		 4911 dmar_map_gfx = 0;
4912}
4913
4914DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2a40, quirk_iommu_g4x_gfx);
4915DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e00, quirk_iommu_g4x_gfx);
4916DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e10, quirk_iommu_g4x_gfx);
4917DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e20, quirk_iommu_g4x_gfx);
4918DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e30, quirk_iommu_g4x_gfx);
4919DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e40, quirk_iommu_g4x_gfx);
4920DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e90, quirk_iommu_g4x_gfx);
4921
d34d6517 4922static void quirk_iommu_rwbf(struct pci_dev *dev)
9af88143
DW		 4923{
4924 /*
4925 * Mobile 4 Series Chipset neglects to set RWBF capability,
210561ff 4926 * but needs it. Same seems to hold for the desktop versions.
9af88143 4927 */
9f10e5bf 4928 pr_info("Forcing write-buffer flush capability\n");
9af88143
DW		 4929 rwbf_quirk = 1;
4930}
4931
4932DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2a40, quirk_iommu_rwbf);
210561ff
DV		 4933DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e00, quirk_iommu_rwbf);
4934DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e10, quirk_iommu_rwbf);
4935DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e20, quirk_iommu_rwbf);
4936DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e30, quirk_iommu_rwbf);
4937DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e40, quirk_iommu_rwbf);
4938DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e90, quirk_iommu_rwbf);
e0fc7e0b 4939
eecfd57f
AJ		 4940#define GGC 0x52
4941#define GGC_MEMORY_SIZE_MASK (0xf << 8)
4942#define GGC_MEMORY_SIZE_NONE (0x0 << 8)
4943#define GGC_MEMORY_SIZE_1M (0x1 << 8)
4944#define GGC_MEMORY_SIZE_2M (0x3 << 8)
4945#define GGC_MEMORY_VT_ENABLED (0x8 << 8)
4946#define GGC_MEMORY_SIZE_2M_VT (0x9 << 8)
4947#define GGC_MEMORY_SIZE_3M_VT (0xa << 8)
4948#define GGC_MEMORY_SIZE_4M_VT (0xb << 8)
4949
d34d6517 4950static void quirk_calpella_no_shadow_gtt(struct pci_dev *dev)
9eecabcb
DW		 4951{
4952 unsigned short ggc;
4953
eecfd57f 4954 if (pci_read_config_word(dev, GGC, &ggc))
9eecabcb
DW		 4955 return;
4956
eecfd57f 4957 if (!(ggc & GGC_MEMORY_VT_ENABLED)) {
9f10e5bf 4958 pr_info("BIOS has allocated no shadow GTT; disabling IOMMU for graphics\n");
9eecabcb 4959 dmar_map_gfx = 0;
6fbcfb3e
DW		 4960 } else if (dmar_map_gfx) {
4961 /* we have to ensure the gfx device is idle before we flush */
9f10e5bf 4962 pr_info("Disabling batched IOTLB flush on Ironlake\n");
6fbcfb3e
DW		 4963 intel_iommu_strict = 1;
4964 }
9eecabcb
DW		 4965}
4966DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0040, quirk_calpella_no_shadow_gtt);
4967DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0044, quirk_calpella_no_shadow_gtt);
4968DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0062, quirk_calpella_no_shadow_gtt);
4969DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x006a, quirk_calpella_no_shadow_gtt);
4970
e0fc7e0b
DW		 4971/* On Tylersburg chipsets, some BIOSes have been known to enable the
4972 ISOCH DMAR unit for the Azalia sound device, but not give it any
4973 TLB entries, which causes it to deadlock. Check for that. We do
4974 this in a function called from init_dmars(), instead of in a PCI
4975 quirk, because we don't want to print the obnoxious "BIOS broken"
4976 message if VT-d is actually disabled.
4977*/
4978static void __init check_tylersburg_isoch(void)
4979{
4980 struct pci_dev *pdev;
4981 uint32_t vtisochctrl;
4982
4983 /* If there's no Azalia in the system anyway, forget it. */
4984 pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x3a3e, NULL);
4985 if (!pdev)
4986 return;
4987 pci_dev_put(pdev);
4988
4989 /* System Management Registers. Might be hidden, in which case
4990 we can't do the sanity check. But that's OK, because the
4991 known-broken BIOSes _don't_ actually hide it, so far. */
4992 pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x342e, NULL);
4993 if (!pdev)
4994 return;
4995
4996 if (pci_read_config_dword(pdev, 0x188, &vtisochctrl)) {
4997 pci_dev_put(pdev);
4998 return;
4999 }
5000
5001 pci_dev_put(pdev);
5002
5003 /* If Azalia DMA is routed to the non-isoch DMAR unit, fine. */
5004 if (vtisochctrl & 1)
5005 return;
5006
5007 /* Drop all bits other than the number of TLB entries */
5008 vtisochctrl &= 0x1c;
5009
5010 /* If we have the recommended number of TLB entries (16), fine. */
5011 if (vtisochctrl == 0x10)
5012 return;
5013
5014 /* Zero TLB entries? You get to ride the short bus to school. */
5015 if (!vtisochctrl) {
5016 WARN(1, "Your BIOS is broken; DMA routed to ISOCH DMAR unit but no TLB space.\n"
5017 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
5018 dmi_get_system_info(DMI_BIOS_VENDOR),
5019 dmi_get_system_info(DMI_BIOS_VERSION),
5020 dmi_get_system_info(DMI_PRODUCT_VERSION));
5021 iommu_identity_mapping |= IDENTMAP_AZALIA;
5022 return;
5023 }
9f10e5bf
JR		 5024
5025 pr_warn("Recommended TLB entries for ISOCH unit is 16; your BIOS set %d\n",
e0fc7e0b
DW		 5026 vtisochctrl);
5027}
Linux kernel - Deliverables
This page took 1.345962 seconds and 5 git commands to generate.