projects / deliverable / linux.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Merge branch 'fix/rt5645' of git://git.kernel.org/pub/scm/linux/kernel/git/broonie...
[deliverable/linux.git] / drivers / iommu / intel-iommu.c
CommitLineData
ba395927 1/*
ea8ea460 2 * Copyright © 2006-2014 Intel Corporation.
ba395927
KA		 3 *
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms and conditions of the GNU General Public License,
6 * version 2, as published by the Free Software Foundation.
7 *
8 * This program is distributed in the hope it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
11 * more details.
12 *
ea8ea460
DW		 13 * Authors: David Woodhouse <dwmw2@infradead.org>,
14 * Ashok Raj <ashok.raj@intel.com>,
15 * Shaohua Li <shaohua.li@intel.com>,
16 * Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>,
17 * Fenghua Yu <fenghua.yu@intel.com>
9f10e5bf 18 * Joerg Roedel <jroedel@suse.de>
ba395927
KA		 19 */
20
9f10e5bf
JR		 21#define pr_fmt(fmt) "DMAR: " fmt
22
ba395927
KA		 23#include <linux/init.h>
24#include <linux/bitmap.h>
5e0d2a6f 25#include <linux/debugfs.h>
54485c30 26#include <linux/export.h>
ba395927
KA		 27#include <linux/slab.h>
28#include <linux/irq.h>
29#include <linux/interrupt.h>
ba395927
KA		 30#include <linux/spinlock.h>
31#include <linux/pci.h>
32#include <linux/dmar.h>
33#include <linux/dma-mapping.h>
34#include <linux/mempool.h>
75f05569 35#include <linux/memory.h>
5e0d2a6f 36#include <linux/timer.h>
38717946 37#include <linux/iova.h>
5d450806 38#include <linux/iommu.h>
38717946 39#include <linux/intel-iommu.h>
134fac3f 40#include <linux/syscore_ops.h>
69575d38 41#include <linux/tboot.h>
adb2fe02 42#include <linux/dmi.h>
5cdede24 43#include <linux/pci-ats.h>
0ee332c1 44#include <linux/memblock.h>
36746436 45#include <linux/dma-contiguous.h>
091d42e4 46#include <linux/crash_dump.h>
8a8f422d 47#include <asm/irq_remapping.h>
ba395927 48#include <asm/cacheflush.h>
46a7fa27 49#include <asm/iommu.h>
ba395927 50
078e1ee2
JR		 51#include "irq_remapping.h"
52
5b6985ce
FY		 53#define ROOT_SIZE VTD_PAGE_SIZE
54#define CONTEXT_SIZE VTD_PAGE_SIZE
55
ba395927 56#define IS_GFX_DEVICE(pdev) ((pdev->class >> 16) == PCI_BASE_CLASS_DISPLAY)
18436afd 57#define IS_USB_DEVICE(pdev) ((pdev->class >> 8) == PCI_CLASS_SERIAL_USB)
ba395927 58#define IS_ISA_DEVICE(pdev) ((pdev->class >> 8) == PCI_CLASS_BRIDGE_ISA)
e0fc7e0b 59#define IS_AZALIA(pdev) ((pdev)->vendor == 0x8086 && (pdev)->device == 0x3a3e)
ba395927
KA		 60
61#define IOAPIC_RANGE_START (0xfee00000)
62#define IOAPIC_RANGE_END (0xfeefffff)
63#define IOVA_START_ADDR (0x1000)
64
65#define DEFAULT_DOMAIN_ADDRESS_WIDTH 48
66
4ed0d3e6 67#define MAX_AGAW_WIDTH 64
5c645b35 68#define MAX_AGAW_PFN_WIDTH (MAX_AGAW_WIDTH - VTD_PAGE_SHIFT)
4ed0d3e6 69
2ebe3151
DW		 70#define __DOMAIN_MAX_PFN(gaw) ((((uint64_t)1) << (gaw-VTD_PAGE_SHIFT)) - 1)
71#define __DOMAIN_MAX_ADDR(gaw) ((((uint64_t)1) << gaw) - 1)
72
73/* We limit DOMAIN_MAX_PFN to fit in an unsigned long, and DOMAIN_MAX_ADDR
74 to match. That way, we can use 'unsigned long' for PFNs with impunity. */
75#define DOMAIN_MAX_PFN(gaw) ((unsigned long) min_t(uint64_t, \
76 __DOMAIN_MAX_PFN(gaw), (unsigned long)-1))
77#define DOMAIN_MAX_ADDR(gaw) (((uint64_t)__DOMAIN_MAX_PFN(gaw)) << VTD_PAGE_SHIFT)
ba395927 78
1b722500
RM		 79/* IO virtual address start page frame number */
80#define IOVA_START_PFN (1)
81
f27be03b 82#define IOVA_PFN(addr) ((addr) >> PAGE_SHIFT)
284901a9 83#define DMA_32BIT_PFN IOVA_PFN(DMA_BIT_MASK(32))
6a35528a 84#define DMA_64BIT_PFN IOVA_PFN(DMA_BIT_MASK(64))
5e0d2a6f 85
df08cdc7
AM		 86/* page table handling */
87#define LEVEL_STRIDE (9)
88#define LEVEL_MASK (((u64)1 << LEVEL_STRIDE) - 1)
89
6d1c56a9
OBC		 90/*
91 * This bitmap is used to advertise the page sizes our hardware support
92 * to the IOMMU core, which will then use this information to split
93 * physically contiguous memory regions it is mapping into page sizes
94 * that we support.
95 *
96 * Traditionally the IOMMU core just handed us the mappings directly,
97 * after making sure the size is an order of a 4KiB page and that the
98 * mapping has natural alignment.
99 *
100 * To retain this behavior, we currently advertise that we support
101 * all page sizes that are an order of 4KiB.
102 *
103 * If at some point we'd like to utilize the IOMMU core's new behavior,
104 * we could change this to advertise the real page sizes we support.
105 */
106#define INTEL_IOMMU_PGSIZES (~0xFFFUL)
107
df08cdc7
AM		 108static inline int agaw_to_level(int agaw)
109{
110 return agaw + 2;
111}
112
113static inline int agaw_to_width(int agaw)
114{
5c645b35 115 return min_t(int, 30 + agaw * LEVEL_STRIDE, MAX_AGAW_WIDTH);
df08cdc7
AM		 116}
117
118static inline int width_to_agaw(int width)
119{
5c645b35 120 return DIV_ROUND_UP(width - 30, LEVEL_STRIDE);
df08cdc7
AM		 121}
122
123static inline unsigned int level_to_offset_bits(int level)
124{
125 return (level - 1) * LEVEL_STRIDE;
126}
127
128static inline int pfn_level_offset(unsigned long pfn, int level)
129{
130 return (pfn >> level_to_offset_bits(level)) & LEVEL_MASK;
131}
132
133static inline unsigned long level_mask(int level)
134{
135 return -1UL << level_to_offset_bits(level);
136}
137
138static inline unsigned long level_size(int level)
139{
140 return 1UL << level_to_offset_bits(level);
141}
142
143static inline unsigned long align_to_level(unsigned long pfn, int level)
144{
145 return (pfn + level_size(level) - 1) & level_mask(level);
146}
fd18de50 147
6dd9a7c7
YS		 148static inline unsigned long lvl_to_nr_pages(unsigned int lvl)
149{
5c645b35 150 return 1 << min_t(int, (lvl - 1) * LEVEL_STRIDE, MAX_AGAW_PFN_WIDTH);
6dd9a7c7
YS		 151}
152
dd4e8319
DW		 153/* VT-d pages must always be _smaller_ than MM pages. Otherwise things
154 are never going to work. */
155static inline unsigned long dma_to_mm_pfn(unsigned long dma_pfn)
156{
157 return dma_pfn >> (PAGE_SHIFT - VTD_PAGE_SHIFT);
158}
159
160static inline unsigned long mm_to_dma_pfn(unsigned long mm_pfn)
161{
162 return mm_pfn << (PAGE_SHIFT - VTD_PAGE_SHIFT);
163}
164static inline unsigned long page_to_dma_pfn(struct page *pg)
165{
166 return mm_to_dma_pfn(page_to_pfn(pg));
167}
168static inline unsigned long virt_to_dma_pfn(void *p)
169{
170 return page_to_dma_pfn(virt_to_page(p));
171}
172
d9630fe9
WH		 173/* global iommu list, set NULL for ignored DMAR units */
174static struct intel_iommu **g_iommus;
175
e0fc7e0b 176static void __init check_tylersburg_isoch(void);
9af88143
DW		 177static int rwbf_quirk;
178
b779260b
JC		 179/*
180 * set to 1 to panic kernel if can't successfully enable VT-d
181 * (used when kernel is launched w/ TXT)
182 */
183static int force_on = 0;
184
46b08e1a
MM		 185/*
186 * 0: Present
187 * 1-11: Reserved
188 * 12-63: Context Ptr (12 - (haw-1))
189 * 64-127: Reserved
190 */
191struct root_entry {
03ecc32c
DW		 192 u64 lo;
193 u64 hi;
46b08e1a
MM		 194};
195#define ROOT_ENTRY_NR (VTD_PAGE_SIZE/sizeof(struct root_entry))
46b08e1a 196
091d42e4
JR		 197/*
198 * Take a root_entry and return the Lower Context Table Pointer (LCTP)
199 * if marked present.
200 */
201static phys_addr_t root_entry_lctp(struct root_entry *re)
202{
203 if (!(re->lo & 1))
204 return 0;
205
206 return re->lo & VTD_PAGE_MASK;
207}
208
209/*
210 * Take a root_entry and return the Upper Context Table Pointer (UCTP)
211 * if marked present.
212 */
213static phys_addr_t root_entry_uctp(struct root_entry *re)
214{
215 if (!(re->hi & 1))
216 return 0;
46b08e1a 217
091d42e4
JR		 218 return re->hi & VTD_PAGE_MASK;
219}
7a8fc25e
MM		 220/*
221 * low 64 bits:
222 * 0: present
223 * 1: fault processing disable
224 * 2-3: translation type
225 * 12-63: address space root
226 * high 64 bits:
227 * 0-2: address width
228 * 3-6: aval
229 * 8-23: domain id
230 */
231struct context_entry {
232 u64 lo;
233 u64 hi;
234};
c07e7d21 235
cf484d0e
JR		 236static inline void context_clear_pasid_enable(struct context_entry *context)
237{
238 context->lo &= ~(1ULL << 11);
239}
240
241static inline bool context_pasid_enabled(struct context_entry *context)
242{
243 return !!(context->lo & (1ULL << 11));
244}
245
246static inline void context_set_copied(struct context_entry *context)
247{
248 context->hi |= (1ull << 3);
249}
250
251static inline bool context_copied(struct context_entry *context)
252{
253 return !!(context->hi & (1ULL << 3));
254}
255
256static inline bool __context_present(struct context_entry *context)
c07e7d21
MM		 257{
258 return (context->lo & 1);
259}
cf484d0e
JR		 260
261static inline bool context_present(struct context_entry *context)
262{
263 return context_pasid_enabled(context) ?
264 __context_present(context) :
265 __context_present(context) && !context_copied(context);
266}
267
c07e7d21
MM		 268static inline void context_set_present(struct context_entry *context)
269{
270 context->lo |= 1;
271}
272
273static inline void context_set_fault_enable(struct context_entry *context)
274{
275 context->lo &= (((u64)-1) << 2) | 1;
276}
277
c07e7d21
MM		 278static inline void context_set_translation_type(struct context_entry *context,
279 unsigned long value)
280{
281 context->lo &= (((u64)-1) << 4) | 3;
282 context->lo |= (value & 3) << 2;
283}
284
285static inline void context_set_address_root(struct context_entry *context,
286 unsigned long value)
287{
1a2262f9 288 context->lo &= ~VTD_PAGE_MASK;
c07e7d21
MM		 289 context->lo |= value & VTD_PAGE_MASK;
290}
291
292static inline void context_set_address_width(struct context_entry *context,
293 unsigned long value)
294{
295 context->hi |= value & 7;
296}
297
298static inline void context_set_domain_id(struct context_entry *context,
299 unsigned long value)
300{
301 context->hi |= (value & ((1 << 16) - 1)) << 8;
302}
303
dbcd861f
JR		 304static inline int context_domain_id(struct context_entry *c)
305{
306 return((c->hi >> 8) & 0xffff);
307}
308
c07e7d21
MM		 309static inline void context_clear_entry(struct context_entry *context)
310{
311 context->lo = 0;
312 context->hi = 0;
313}
7a8fc25e 314
622ba12a
MM		 315/*
316 * 0: readable
317 * 1: writable
318 * 2-6: reserved
319 * 7: super page
9cf06697
SY		 320 * 8-10: available
321 * 11: snoop behavior
622ba12a
MM		 322 * 12-63: Host physcial address
323 */
324struct dma_pte {
325 u64 val;
326};
622ba12a 327
19c239ce
MM		 328static inline void dma_clear_pte(struct dma_pte *pte)
329{
330 pte->val = 0;
331}
332
19c239ce
MM		 333static inline u64 dma_pte_addr(struct dma_pte *pte)
334{
c85994e4
DW		 335#ifdef CONFIG_64BIT
336 return pte->val & VTD_PAGE_MASK;
337#else
338 /* Must have a full atomic 64-bit read */
1a8bd481 339 return __cmpxchg64(&pte->val, 0ULL, 0ULL) & VTD_PAGE_MASK;
c85994e4 340#endif
19c239ce
MM		 341}
342
19c239ce
MM		 343static inline bool dma_pte_present(struct dma_pte *pte)
344{
345 return (pte->val & 3) != 0;
346}
622ba12a 347
4399c8bf
AK		 348static inline bool dma_pte_superpage(struct dma_pte *pte)
349{
c3c75eb7 350 return (pte->val & DMA_PTE_LARGE_PAGE);
4399c8bf
AK		 351}
352
75e6bf96
DW		 353static inline int first_pte_in_page(struct dma_pte *pte)
354{
355 return !((unsigned long)pte & ~VTD_PAGE_MASK);
356}
357
2c2e2c38
FY		 358/*
359 * This domain is a statically identity mapping domain.
360 * 1. This domain creats a static 1:1 mapping to all usable memory.
361 * 2. It maps to each iommu if successful.
362 * 3. Each iommu mapps to this domain if successful.
363 */
19943b0e
DW		 364static struct dmar_domain *si_domain;
365static int hw_pass_through = 1;
2c2e2c38 366
1ce28feb
WH		 367/* domain represents a virtual machine, more than one devices
368 * across iommus may be owned in one domain, e.g. kvm guest.
369 */
ab8dfe25 370#define DOMAIN_FLAG_VIRTUAL_MACHINE (1 << 0)
1ce28feb 371
2c2e2c38 372/* si_domain contains mulitple devices */
ab8dfe25 373#define DOMAIN_FLAG_STATIC_IDENTITY (1 << 1)
2c2e2c38 374
99126f7c
MM		 375struct dmar_domain {
376 int id; /* domain id */
4c923d47 377 int nid; /* node id */
78d8e704 378 DECLARE_BITMAP(iommu_bmp, DMAR_UNITS_SUPPORTED);
1b198bb0 379 /* bitmap of iommus this domain uses*/
99126f7c 380
00a77deb 381 struct list_head devices; /* all devices' list */
99126f7c
MM		 382 struct iova_domain iovad; /* iova's that belong to this domain */
383
384 struct dma_pte *pgd; /* virtual address */
99126f7c
MM		 385 int gaw; /* max guest address width */
386
387 /* adjusted guest address width, 0 is level 2 30-bit */
388 int agaw;
389
3b5410e7 390 int flags; /* flags to find out type of domain */
8e604097
WH		 391
392 int iommu_coherency;/* indicate coherency of iommu access */
58c610bd 393 int iommu_snooping; /* indicate snooping control feature*/
c7151a8d 394 int iommu_count; /* reference count of iommu */
6dd9a7c7
YS		 395 int iommu_superpage;/* Level of superpages supported:
396 0 == 4KiB (no superpages), 1 == 2MiB,
397 2 == 1GiB, 3 == 512GiB, 4 == 1TiB */
c7151a8d 398 spinlock_t iommu_lock; /* protect iommu set in domain */
fe40f1e0 399 u64 max_addr; /* maximum mapped address */
00a77deb
JR		 400
401 struct iommu_domain domain; /* generic domain data structure for
402 iommu core */
99126f7c
MM		 403};
404
a647dacb
MM		 405/* PCI domain-device relationship */
406struct device_domain_info {
407 struct list_head link; /* link to domain siblings */
408 struct list_head global; /* link to global list */
276dbf99 409 u8 bus; /* PCI bus number */
a647dacb 410 u8 devfn; /* PCI devfn number */
0bcb3e28 411 struct device *dev; /* it's NULL for PCIe-to-PCI bridge */
93a23a72 412 struct intel_iommu *iommu; /* IOMMU used by this device */
a647dacb
MM		 413 struct dmar_domain *domain; /* pointer to domain */
414};
415
b94e4117
JL		 416struct dmar_rmrr_unit {
417 struct list_head list; /* list of rmrr units */
418 struct acpi_dmar_header *hdr; /* ACPI header */
419 u64 base_address; /* reserved base address*/
420 u64 end_address; /* reserved end address */
832bd858 421 struct dmar_dev_scope *devices; /* target devices */
b94e4117
JL		 422 int devices_cnt; /* target device count */
423};
424
425struct dmar_atsr_unit {
426 struct list_head list; /* list of ATSR units */
427 struct acpi_dmar_header *hdr; /* ACPI header */
832bd858 428 struct dmar_dev_scope *devices; /* target devices */
b94e4117
JL		 429 int devices_cnt; /* target device count */
430 u8 include_all:1; /* include all ports */
431};
432
433static LIST_HEAD(dmar_atsr_units);
434static LIST_HEAD(dmar_rmrr_units);
435
436#define for_each_rmrr_units(rmrr) \
437 list_for_each_entry(rmrr, &dmar_rmrr_units, list)
438
5e0d2a6f 439static void flush_unmaps_timeout(unsigned long data);
440
b707cb02 441static DEFINE_TIMER(unmap_timer, flush_unmaps_timeout, 0, 0);
5e0d2a6f 442
80b20dd8 443#define HIGH_WATER_MARK 250
444struct deferred_flush_tables {
445 int next;
446 struct iova *iova[HIGH_WATER_MARK];
447 struct dmar_domain *domain[HIGH_WATER_MARK];
ea8ea460 448 struct page *freelist[HIGH_WATER_MARK];
80b20dd8 449};
450
451static struct deferred_flush_tables *deferred_flush;
452
5e0d2a6f 453/* bitmap for indexing intel_iommus */
5e0d2a6f 454static int g_num_of_iommus;
455
456static DEFINE_SPINLOCK(async_umap_flush_lock);
457static LIST_HEAD(unmaps_to_do);
458
459static int timer_on;
460static long list_size;
5e0d2a6f 461
92d03cc8 462static void domain_exit(struct dmar_domain *domain);
ba395927 463static void domain_remove_dev_info(struct dmar_domain *domain);
b94e4117 464static void domain_remove_one_dev_info(struct dmar_domain *domain,
bf9c9eda 465 struct device *dev);
92d03cc8 466static void iommu_detach_dependent_devices(struct intel_iommu *iommu,
0bcb3e28 467 struct device *dev);
2a46ddf7
JL		 468static int domain_detach_iommu(struct dmar_domain *domain,
469 struct intel_iommu *iommu);
ba395927 470
d3f13810 471#ifdef CONFIG_INTEL_IOMMU_DEFAULT_ON
0cd5c3c8
KM		 472int dmar_disabled = 0;
473#else
474int dmar_disabled = 1;
d3f13810 475#endif /*CONFIG_INTEL_IOMMU_DEFAULT_ON*/
0cd5c3c8 476
8bc1f85c
ED		 477int intel_iommu_enabled = 0;
478EXPORT_SYMBOL_GPL(intel_iommu_enabled);
479
2d9e667e 480static int dmar_map_gfx = 1;
7d3b03ce 481static int dmar_forcedac;
5e0d2a6f 482static int intel_iommu_strict;
6dd9a7c7 483static int intel_iommu_superpage = 1;
c83b2f20
DW		 484static int intel_iommu_ecs = 1;
485
486/* We only actually use ECS when PASID support (on the new bit 40)
487 * is also advertised. Some early implementations — the ones with
488 * PASID support on bit 28 — have issues even when we *only* use
489 * extended root/context tables. */
490#define ecs_enabled(iommu) (intel_iommu_ecs && ecap_ecs(iommu->ecap) && \
491 ecap_pasid(iommu->ecap))
ba395927 492
c0771df8
DW		 493int intel_iommu_gfx_mapped;
494EXPORT_SYMBOL_GPL(intel_iommu_gfx_mapped);
495
ba395927
KA		 496#define DUMMY_DEVICE_DOMAIN_INFO ((struct device_domain_info *)(-1))
497static DEFINE_SPINLOCK(device_domain_lock);
498static LIST_HEAD(device_domain_list);
499
b22f6434 500static const struct iommu_ops intel_iommu_ops;
a8bcbb0d 501
4158c2ec
JR		 502static bool translation_pre_enabled(struct intel_iommu *iommu)
503{
504 return (iommu->flags & VTD_FLAG_TRANS_PRE_ENABLED);
505}
506
091d42e4
JR		 507static void clear_translation_pre_enabled(struct intel_iommu *iommu)
508{
509 iommu->flags &= ~VTD_FLAG_TRANS_PRE_ENABLED;
510}
511
4158c2ec
JR		 512static void init_translation_status(struct intel_iommu *iommu)
513{
514 u32 gsts;
515
516 gsts = readl(iommu->reg + DMAR_GSTS_REG);
517 if (gsts & DMA_GSTS_TES)
518 iommu->flags |= VTD_FLAG_TRANS_PRE_ENABLED;
519}
520
00a77deb
JR		 521/* Convert generic 'struct iommu_domain to private struct dmar_domain */
522static struct dmar_domain *to_dmar_domain(struct iommu_domain *dom)
523{
524 return container_of(dom, struct dmar_domain, domain);
525}
526
ba395927
KA		 527static int __init intel_iommu_setup(char *str)
528{
529 if (!str)
530 return -EINVAL;
531 while (*str) {
0cd5c3c8
KM		 532 if (!strncmp(str, "on", 2)) {
533 dmar_disabled = 0;
9f10e5bf 534 pr_info("IOMMU enabled\n");
0cd5c3c8 535 } else if (!strncmp(str, "off", 3)) {
ba395927 536 dmar_disabled = 1;
9f10e5bf 537 pr_info("IOMMU disabled\n");
ba395927
KA		 538 } else if (!strncmp(str, "igfx_off", 8)) {
539 dmar_map_gfx = 0;
9f10e5bf 540 pr_info("Disable GFX device mapping\n");
7d3b03ce 541 } else if (!strncmp(str, "forcedac", 8)) {
9f10e5bf 542 pr_info("Forcing DAC for PCI devices\n");
7d3b03ce 543 dmar_forcedac = 1;
5e0d2a6f 544 } else if (!strncmp(str, "strict", 6)) {
9f10e5bf 545 pr_info("Disable batched IOTLB flush\n");
5e0d2a6f 546 intel_iommu_strict = 1;
6dd9a7c7 547 } else if (!strncmp(str, "sp_off", 6)) {
9f10e5bf 548 pr_info("Disable supported super page\n");
6dd9a7c7 549 intel_iommu_superpage = 0;
c83b2f20
DW		 550 } else if (!strncmp(str, "ecs_off", 7)) {
551 printk(KERN_INFO
552 "Intel-IOMMU: disable extended context table support\n");
553 intel_iommu_ecs = 0;
ba395927
KA		 554 }
555
556 str += strcspn(str, ",");
557 while (*str == ',')
558 str++;
559 }
560 return 0;
561}
562__setup("intel_iommu=", intel_iommu_setup);
563
564static struct kmem_cache *iommu_domain_cache;
565static struct kmem_cache *iommu_devinfo_cache;
ba395927 566
4c923d47 567static inline void *alloc_pgtable_page(int node)
eb3fa7cb 568{
4c923d47
SS		 569 struct page *page;
570 void *vaddr = NULL;
eb3fa7cb 571
4c923d47
SS		 572 page = alloc_pages_node(node, GFP_ATOMIC | __GFP_ZERO, 0);
573 if (page)
574 vaddr = page_address(page);
eb3fa7cb 575 return vaddr;
ba395927
KA		 576}
577
578static inline void free_pgtable_page(void *vaddr)
579{
580 free_page((unsigned long)vaddr);
581}
582
583static inline void *alloc_domain_mem(void)
584{
354bb65e 585 return kmem_cache_alloc(iommu_domain_cache, GFP_ATOMIC);
ba395927
KA		 586}
587
38717946 588static void free_domain_mem(void *vaddr)
ba395927
KA		 589{
590 kmem_cache_free(iommu_domain_cache, vaddr);
591}
592
593static inline void * alloc_devinfo_mem(void)
594{
354bb65e 595 return kmem_cache_alloc(iommu_devinfo_cache, GFP_ATOMIC);
ba395927
KA		 596}
597
598static inline void free_devinfo_mem(void *vaddr)
599{
600 kmem_cache_free(iommu_devinfo_cache, vaddr);
601}
602
ab8dfe25
JL		 603static inline int domain_type_is_vm(struct dmar_domain *domain)
604{
605 return domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE;
606}
607
608static inline int domain_type_is_vm_or_si(struct dmar_domain *domain)
609{
610 return domain->flags & (DOMAIN_FLAG_VIRTUAL_MACHINE |
611 DOMAIN_FLAG_STATIC_IDENTITY);
612}
1b573683 613
162d1b10
JL		 614static inline int domain_pfn_supported(struct dmar_domain *domain,
615 unsigned long pfn)
616{
617 int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
618
619 return !(addr_width < BITS_PER_LONG && pfn >> addr_width);
620}
621
4ed0d3e6 622static int __iommu_calculate_agaw(struct intel_iommu *iommu, int max_gaw)
1b573683
WH		 623{
624 unsigned long sagaw;
625 int agaw = -1;
626
627 sagaw = cap_sagaw(iommu->cap);
4ed0d3e6 628 for (agaw = width_to_agaw(max_gaw);
1b573683
WH		 629 agaw >= 0; agaw--) {
630 if (test_bit(agaw, &sagaw))
631 break;
632 }
633
634 return agaw;
635}
636
4ed0d3e6
FY		 637/*
638 * Calculate max SAGAW for each iommu.
639 */
640int iommu_calculate_max_sagaw(struct intel_iommu *iommu)
641{
642 return __iommu_calculate_agaw(iommu, MAX_AGAW_WIDTH);
643}
644
645/*
646 * calculate agaw for each iommu.
647 * "SAGAW" may be different across iommus, use a default agaw, and
648 * get a supported less agaw for iommus that don't support the default agaw.
649 */
650int iommu_calculate_agaw(struct intel_iommu *iommu)
651{
652 return __iommu_calculate_agaw(iommu, DEFAULT_DOMAIN_ADDRESS_WIDTH);
653}
654
2c2e2c38 655/* This functionin only returns single iommu in a domain */
8c11e798
WH		 656static struct intel_iommu *domain_get_iommu(struct dmar_domain *domain)
657{
658 int iommu_id;
659
2c2e2c38 660 /* si_domain and vm domain should not get here. */
ab8dfe25 661 BUG_ON(domain_type_is_vm_or_si(domain));
1b198bb0 662 iommu_id = find_first_bit(domain->iommu_bmp, g_num_of_iommus);
8c11e798
WH		 663 if (iommu_id < 0 || iommu_id >= g_num_of_iommus)
664 return NULL;
665
666 return g_iommus[iommu_id];
667}
668
8e604097
WH		 669static void domain_update_iommu_coherency(struct dmar_domain *domain)
670{
d0501960
DW		 671 struct dmar_drhd_unit *drhd;
672 struct intel_iommu *iommu;
2f119c78
QL		 673 bool found = false;
674 int i;
2e12bc29 675
d0501960 676 domain->iommu_coherency = 1;
8e604097 677
1b198bb0 678 for_each_set_bit(i, domain->iommu_bmp, g_num_of_iommus) {
2f119c78 679 found = true;
8e604097
WH		 680 if (!ecap_coherent(g_iommus[i]->ecap)) {
681 domain->iommu_coherency = 0;
682 break;
683 }
8e604097 684 }
d0501960
DW		 685 if (found)
686 return;
687
688 /* No hardware attached; use lowest common denominator */
689 rcu_read_lock();
690 for_each_active_iommu(iommu, drhd) {
691 if (!ecap_coherent(iommu->ecap)) {
692 domain->iommu_coherency = 0;
693 break;
694 }
695 }
696 rcu_read_unlock();
8e604097
WH		 697}
698
161f6934 699static int domain_update_iommu_snooping(struct intel_iommu *skip)
58c610bd 700{
161f6934
JL		 701 struct dmar_drhd_unit *drhd;
702 struct intel_iommu *iommu;
703 int ret = 1;
58c610bd 704
161f6934
JL		 705 rcu_read_lock();
706 for_each_active_iommu(iommu, drhd) {
707 if (iommu != skip) {
708 if (!ecap_sc_support(iommu->ecap)) {
709 ret = 0;
710 break;
711 }
58c610bd 712 }
58c610bd 713 }
161f6934
JL		 714 rcu_read_unlock();
715
716 return ret;
58c610bd
SY		 717}
718
161f6934 719static int domain_update_iommu_superpage(struct intel_iommu *skip)
6dd9a7c7 720{
8140a95d 721 struct dmar_drhd_unit *drhd;
161f6934 722 struct intel_iommu *iommu;
8140a95d 723 int mask = 0xf;
6dd9a7c7
YS		 724
725 if (!intel_iommu_superpage) {
161f6934 726 return 0;
6dd9a7c7
YS		 727 }
728
8140a95d 729 /* set iommu_superpage to the smallest common denominator */
0e242612 730 rcu_read_lock();
8140a95d 731 for_each_active_iommu(iommu, drhd) {
161f6934
JL		 732 if (iommu != skip) {
733 mask &= cap_super_page_val(iommu->cap);
734 if (!mask)
735 break;
6dd9a7c7
YS		 736 }
737 }
0e242612
JL		 738 rcu_read_unlock();
739
161f6934 740 return fls(mask);
6dd9a7c7
YS		 741}
742
58c610bd
SY		 743/* Some capabilities may be different across iommus */
744static void domain_update_iommu_cap(struct dmar_domain *domain)
745{
746 domain_update_iommu_coherency(domain);
161f6934
JL		 747 domain->iommu_snooping = domain_update_iommu_snooping(NULL);
748 domain->iommu_superpage = domain_update_iommu_superpage(NULL);
58c610bd
SY		 749}
750
03ecc32c
DW		 751static inline struct context_entry *iommu_context_addr(struct intel_iommu *iommu,
752 u8 bus, u8 devfn, int alloc)
753{
754 struct root_entry *root = &iommu->root_entry[bus];
755 struct context_entry *context;
756 u64 *entry;
757
c83b2f20 758 if (ecs_enabled(iommu)) {
03ecc32c
DW		 759 if (devfn >= 0x80) {
760 devfn -= 0x80;
761 entry = &root->hi;
762 }
763 devfn *= 2;
764 }
765 entry = &root->lo;
766 if (*entry & 1)
767 context = phys_to_virt(*entry & VTD_PAGE_MASK);
768 else {
769 unsigned long phy_addr;
770 if (!alloc)
771 return NULL;
772
773 context = alloc_pgtable_page(iommu->node);
774 if (!context)
775 return NULL;
776
777 __iommu_flush_cache(iommu, (void *)context, CONTEXT_SIZE);
778 phy_addr = virt_to_phys((void *)context);
779 *entry = phy_addr | 1;
780 __iommu_flush_cache(iommu, entry, sizeof(*entry));
781 }
782 return &context[devfn];
783}
784
4ed6a540
DW		 785static int iommu_dummy(struct device *dev)
786{
787 return dev->archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO;
788}
789
156baca8 790static struct intel_iommu *device_to_iommu(struct device *dev, u8 *bus, u8 *devfn)
c7151a8d
WH		 791{
792 struct dmar_drhd_unit *drhd = NULL;
b683b230 793 struct intel_iommu *iommu;
156baca8
DW		 794 struct device *tmp;
795 struct pci_dev *ptmp, *pdev = NULL;
aa4d066a 796 u16 segment = 0;
c7151a8d
WH		 797 int i;
798
4ed6a540
DW		 799 if (iommu_dummy(dev))
800 return NULL;
801
156baca8
DW		 802 if (dev_is_pci(dev)) {
803 pdev = to_pci_dev(dev);
804 segment = pci_domain_nr(pdev->bus);
ca5b74d2 805 } else if (has_acpi_companion(dev))
156baca8
DW		 806 dev = &ACPI_COMPANION(dev)->dev;
807
0e242612 808 rcu_read_lock();
b683b230 809 for_each_active_iommu(iommu, drhd) {
156baca8 810 if (pdev && segment != drhd->segment)
276dbf99 811 continue;
c7151a8d 812
b683b230 813 for_each_active_dev_scope(drhd->devices,
156baca8
DW		 814 drhd->devices_cnt, i, tmp) {
815 if (tmp == dev) {
816 *bus = drhd->devices[i].bus;
817 *devfn = drhd->devices[i].devfn;
b683b230 818 goto out;
156baca8
DW		 819 }
820
821 if (!pdev || !dev_is_pci(tmp))
822 continue;
823
824 ptmp = to_pci_dev(tmp);
825 if (ptmp->subordinate &&
826 ptmp->subordinate->number <= pdev->bus->number &&
827 ptmp->subordinate->busn_res.end >= pdev->bus->number)
828 goto got_pdev;
924b6231 829 }
c7151a8d 830
156baca8
DW		 831 if (pdev && drhd->include_all) {
832 got_pdev:
833 *bus = pdev->bus->number;
834 *devfn = pdev->devfn;
b683b230 835 goto out;
156baca8 836 }
c7151a8d 837 }
b683b230 838 iommu = NULL;
156baca8 839 out:
0e242612 840 rcu_read_unlock();
c7151a8d 841
b683b230 842 return iommu;
c7151a8d
WH		 843}
844
5331fe6f
WH		 845static void domain_flush_cache(struct dmar_domain *domain,
846 void *addr, int size)
847{
848 if (!domain->iommu_coherency)
849 clflush_cache_range(addr, size);
850}
851
ba395927
KA		 852static int device_context_mapped(struct intel_iommu *iommu, u8 bus, u8 devfn)
853{
ba395927 854 struct context_entry *context;
03ecc32c 855 int ret = 0;
ba395927
KA		 856 unsigned long flags;
857
858 spin_lock_irqsave(&iommu->lock, flags);
03ecc32c
DW		 859 context = iommu_context_addr(iommu, bus, devfn, 0);
860 if (context)
861 ret = context_present(context);
ba395927
KA		 862 spin_unlock_irqrestore(&iommu->lock, flags);
863 return ret;
864}
865
866static void clear_context_table(struct intel_iommu *iommu, u8 bus, u8 devfn)
867{
ba395927
KA		 868 struct context_entry *context;
869 unsigned long flags;
870
871 spin_lock_irqsave(&iommu->lock, flags);
03ecc32c 872 context = iommu_context_addr(iommu, bus, devfn, 0);
ba395927 873 if (context) {
03ecc32c
DW		 874 context_clear_entry(context);
875 __iommu_flush_cache(iommu, context, sizeof(*context));
ba395927
KA		 876 }
877 spin_unlock_irqrestore(&iommu->lock, flags);
878}
879
880static void free_context_table(struct intel_iommu *iommu)
881{
ba395927
KA		 882 int i;
883 unsigned long flags;
884 struct context_entry *context;
885
886 spin_lock_irqsave(&iommu->lock, flags);
887 if (!iommu->root_entry) {
888 goto out;
889 }
890 for (i = 0; i < ROOT_ENTRY_NR; i++) {
03ecc32c 891 context = iommu_context_addr(iommu, i, 0, 0);
ba395927
KA		 892 if (context)
893 free_pgtable_page(context);
03ecc32c 894
c83b2f20 895 if (!ecs_enabled(iommu))
03ecc32c
DW		 896 continue;
897
898 context = iommu_context_addr(iommu, i, 0x80, 0);
899 if (context)
900 free_pgtable_page(context);
901
ba395927
KA		 902 }
903 free_pgtable_page(iommu->root_entry);
904 iommu->root_entry = NULL;
905out:
906 spin_unlock_irqrestore(&iommu->lock, flags);
907}
908
b026fd28 909static struct dma_pte *pfn_to_dma_pte(struct dmar_domain *domain,
5cf0a76f 910 unsigned long pfn, int *target_level)
ba395927 911{
ba395927
KA		 912 struct dma_pte *parent, *pte = NULL;
913 int level = agaw_to_level(domain->agaw);
4399c8bf 914 int offset;
ba395927
KA		 915
916 BUG_ON(!domain->pgd);
f9423606 917
162d1b10 918 if (!domain_pfn_supported(domain, pfn))
f9423606
JS		 919 /* Address beyond IOMMU's addressing capabilities. */
920 return NULL;
921
ba395927
KA		 922 parent = domain->pgd;
923
5cf0a76f 924 while (1) {
ba395927
KA		 925 void *tmp_page;
926
b026fd28 927 offset = pfn_level_offset(pfn, level);
ba395927 928 pte = &parent[offset];
5cf0a76f 929 if (!*target_level && (dma_pte_superpage(pte) || !dma_pte_present(pte)))
6dd9a7c7 930 break;
5cf0a76f 931 if (level == *target_level)
ba395927
KA		 932 break;
933
19c239ce 934 if (!dma_pte_present(pte)) {
c85994e4
DW		 935 uint64_t pteval;
936
4c923d47 937 tmp_page = alloc_pgtable_page(domain->nid);
ba395927 938
206a73c1 939 if (!tmp_page)
ba395927 940 return NULL;
206a73c1 941
c85994e4 942 domain_flush_cache(domain, tmp_page, VTD_PAGE_SIZE);
64de5af0 943 pteval = ((uint64_t)virt_to_dma_pfn(tmp_page) << VTD_PAGE_SHIFT) | DMA_PTE_READ | DMA_PTE_WRITE;
effad4b5 944 if (cmpxchg64(&pte->val, 0ULL, pteval))
c85994e4
DW		 945 /* Someone else set it while we were thinking; use theirs. */
946 free_pgtable_page(tmp_page);
effad4b5 947 else
c85994e4 948 domain_flush_cache(domain, pte, sizeof(*pte));
ba395927 949 }
5cf0a76f
DW		 950 if (level == 1)
951 break;
952
19c239ce 953 parent = phys_to_virt(dma_pte_addr(pte));
ba395927
KA		 954 level--;
955 }
956
5cf0a76f
DW		 957 if (!*target_level)
958 *target_level = level;
959
ba395927
KA		 960 return pte;
961}
962
6dd9a7c7 963
ba395927 964/* return address's pte at specific level */
90dcfb5e
DW		 965static struct dma_pte *dma_pfn_level_pte(struct dmar_domain *domain,
966 unsigned long pfn,
6dd9a7c7 967 int level, int *large_page)
ba395927
KA		 968{
969 struct dma_pte *parent, *pte = NULL;
970 int total = agaw_to_level(domain->agaw);
971 int offset;
972
973 parent = domain->pgd;
974 while (level <= total) {
90dcfb5e 975 offset = pfn_level_offset(pfn, total);
ba395927
KA		 976 pte = &parent[offset];
977 if (level == total)
978 return pte;
979
6dd9a7c7
YS		 980 if (!dma_pte_present(pte)) {
981 *large_page = total;
ba395927 982 break;
6dd9a7c7
YS		 983 }
984
e16922af 985 if (dma_pte_superpage(pte)) {
6dd9a7c7
YS		 986 *large_page = total;
987 return pte;
988 }
989
19c239ce 990 parent = phys_to_virt(dma_pte_addr(pte));
ba395927
KA		 991 total--;
992 }
993 return NULL;
994}
995
ba395927 996/* clear last level pte, a tlb flush should be followed */
5cf0a76f 997static void dma_pte_clear_range(struct dmar_domain *domain,
595badf5
DW		 998 unsigned long start_pfn,
999 unsigned long last_pfn)
ba395927 1000{
6dd9a7c7 1001 unsigned int large_page = 1;
310a5ab9 1002 struct dma_pte *first_pte, *pte;
66eae846 1003
162d1b10
JL		 1004 BUG_ON(!domain_pfn_supported(domain, start_pfn));
1005 BUG_ON(!domain_pfn_supported(domain, last_pfn));
59c36286 1006 BUG_ON(start_pfn > last_pfn);
ba395927 1007
04b18e65 1008 /* we don't need lock here; nobody else touches the iova range */
59c36286 1009 do {
6dd9a7c7
YS		 1010 large_page = 1;
1011 first_pte = pte = dma_pfn_level_pte(domain, start_pfn, 1, &large_page);
310a5ab9 1012 if (!pte) {
6dd9a7c7 1013 start_pfn = align_to_level(start_pfn + 1, large_page + 1);
310a5ab9
DW		 1014 continue;
1015 }
6dd9a7c7 1016 do {
310a5ab9 1017 dma_clear_pte(pte);
6dd9a7c7 1018 start_pfn += lvl_to_nr_pages(large_page);
310a5ab9 1019 pte++;
75e6bf96
DW		 1020 } while (start_pfn <= last_pfn && !first_pte_in_page(pte));
1021
310a5ab9
DW		 1022 domain_flush_cache(domain, first_pte,
1023 (void *)pte - (void *)first_pte);
59c36286
DW		 1024
1025 } while (start_pfn && start_pfn <= last_pfn);
ba395927
KA		 1026}
1027
3269ee0b
AW		 1028static void dma_pte_free_level(struct dmar_domain *domain, int level,
1029 struct dma_pte *pte, unsigned long pfn,
1030 unsigned long start_pfn, unsigned long last_pfn)
1031{
1032 pfn = max(start_pfn, pfn);
1033 pte = &pte[pfn_level_offset(pfn, level)];
1034
1035 do {
1036 unsigned long level_pfn;
1037 struct dma_pte *level_pte;
1038
1039 if (!dma_pte_present(pte) || dma_pte_superpage(pte))
1040 goto next;
1041
1042 level_pfn = pfn & level_mask(level - 1);
1043 level_pte = phys_to_virt(dma_pte_addr(pte));
1044
1045 if (level > 2)
1046 dma_pte_free_level(domain, level - 1, level_pte,
1047 level_pfn, start_pfn, last_pfn);
1048
1049 /* If range covers entire pagetable, free it */
1050 if (!(start_pfn > level_pfn ||
08336fd2 1051 last_pfn < level_pfn + level_size(level) - 1)) {
3269ee0b
AW		 1052 dma_clear_pte(pte);
1053 domain_flush_cache(domain, pte, sizeof(*pte));
1054 free_pgtable_page(level_pte);
1055 }
1056next:
1057 pfn += level_size(level);
1058 } while (!first_pte_in_page(++pte) && pfn <= last_pfn);
1059}
1060
ba395927
KA		 1061/* free page table pages. last level pte should already be cleared */
1062static void dma_pte_free_pagetable(struct dmar_domain *domain,
d794dc9b
DW		 1063 unsigned long start_pfn,
1064 unsigned long last_pfn)
ba395927 1065{
162d1b10
JL		 1066 BUG_ON(!domain_pfn_supported(domain, start_pfn));
1067 BUG_ON(!domain_pfn_supported(domain, last_pfn));
59c36286 1068 BUG_ON(start_pfn > last_pfn);
ba395927 1069
d41a4adb
JL		 1070 dma_pte_clear_range(domain, start_pfn, last_pfn);
1071
f3a0a52f 1072 /* We don't need lock here; nobody else touches the iova range */
3269ee0b
AW		 1073 dma_pte_free_level(domain, agaw_to_level(domain->agaw),
1074 domain->pgd, 0, start_pfn, last_pfn);
6660c63a 1075
ba395927 1076 /* free pgd */
d794dc9b 1077 if (start_pfn == 0 && last_pfn == DOMAIN_MAX_PFN(domain->gaw)) {
ba395927
KA		 1078 free_pgtable_page(domain->pgd);
1079 domain->pgd = NULL;
1080 }
1081}
1082
ea8ea460
DW		 1083/* When a page at a given level is being unlinked from its parent, we don't
1084 need to *modify* it at all. All we need to do is make a list of all the
1085 pages which can be freed just as soon as we've flushed the IOTLB and we
1086 know the hardware page-walk will no longer touch them.
1087 The 'pte' argument is the *parent* PTE, pointing to the page that is to
1088 be freed. */
1089static struct page *dma_pte_list_pagetables(struct dmar_domain *domain,
1090 int level, struct dma_pte *pte,
1091 struct page *freelist)
1092{
1093 struct page *pg;
1094
1095 pg = pfn_to_page(dma_pte_addr(pte) >> PAGE_SHIFT);
1096 pg->freelist = freelist;
1097 freelist = pg;
1098
1099 if (level == 1)
1100 return freelist;
1101
adeb2590
JL		 1102 pte = page_address(pg);
1103 do {
ea8ea460
DW		 1104 if (dma_pte_present(pte) && !dma_pte_superpage(pte))
1105 freelist = dma_pte_list_pagetables(domain, level - 1,
1106 pte, freelist);
adeb2590
JL		 1107 pte++;
1108 } while (!first_pte_in_page(pte));
ea8ea460
DW		 1109
1110 return freelist;
1111}
1112
1113static struct page *dma_pte_clear_level(struct dmar_domain *domain, int level,
1114 struct dma_pte *pte, unsigned long pfn,
1115 unsigned long start_pfn,
1116 unsigned long last_pfn,
1117 struct page *freelist)
1118{
1119 struct dma_pte *first_pte = NULL, *last_pte = NULL;
1120
1121 pfn = max(start_pfn, pfn);
1122 pte = &pte[pfn_level_offset(pfn, level)];
1123
1124 do {
1125 unsigned long level_pfn;
1126
1127 if (!dma_pte_present(pte))
1128 goto next;
1129
1130 level_pfn = pfn & level_mask(level);
1131
1132 /* If range covers entire pagetable, free it */
1133 if (start_pfn <= level_pfn &&
1134 last_pfn >= level_pfn + level_size(level) - 1) {
1135 /* These suborbinate page tables are going away entirely. Don't
1136 bother to clear them; we're just going to *free* them. */
1137 if (level > 1 && !dma_pte_superpage(pte))
1138 freelist = dma_pte_list_pagetables(domain, level - 1, pte, freelist);
1139
1140 dma_clear_pte(pte);
1141 if (!first_pte)
1142 first_pte = pte;
1143 last_pte = pte;
1144 } else if (level > 1) {
1145 /* Recurse down into a level that isn't *entirely* obsolete */
1146 freelist = dma_pte_clear_level(domain, level - 1,
1147 phys_to_virt(dma_pte_addr(pte)),
1148 level_pfn, start_pfn, last_pfn,
1149 freelist);
1150 }
1151next:
1152 pfn += level_size(level);
1153 } while (!first_pte_in_page(++pte) && pfn <= last_pfn);
1154
1155 if (first_pte)
1156 domain_flush_cache(domain, first_pte,
1157 (void *)++last_pte - (void *)first_pte);
1158
1159 return freelist;
1160}
1161
1162/* We can't just free the pages because the IOMMU may still be walking
1163 the page tables, and may have cached the intermediate levels. The
1164 pages can only be freed after the IOTLB flush has been done. */
1165struct page *domain_unmap(struct dmar_domain *domain,
1166 unsigned long start_pfn,
1167 unsigned long last_pfn)
1168{
ea8ea460
DW		 1169 struct page *freelist = NULL;
1170
162d1b10
JL		 1171 BUG_ON(!domain_pfn_supported(domain, start_pfn));
1172 BUG_ON(!domain_pfn_supported(domain, last_pfn));
ea8ea460
DW		 1173 BUG_ON(start_pfn > last_pfn);
1174
1175 /* we don't need lock here; nobody else touches the iova range */
1176 freelist = dma_pte_clear_level(domain, agaw_to_level(domain->agaw),
1177 domain->pgd, 0, start_pfn, last_pfn, NULL);
1178
1179 /* free pgd */
1180 if (start_pfn == 0 && last_pfn == DOMAIN_MAX_PFN(domain->gaw)) {
1181 struct page *pgd_page = virt_to_page(domain->pgd);
1182 pgd_page->freelist = freelist;
1183 freelist = pgd_page;
1184
1185 domain->pgd = NULL;
1186 }
1187
1188 return freelist;
1189}
1190
1191void dma_free_pagelist(struct page *freelist)
1192{
1193 struct page *pg;
1194
1195 while ((pg = freelist)) {
1196 freelist = pg->freelist;
1197 free_pgtable_page(page_address(pg));
1198 }
1199}
1200
ba395927
KA		 1201/* iommu handling */
1202static int iommu_alloc_root_entry(struct intel_iommu *iommu)
1203{
1204 struct root_entry *root;
1205 unsigned long flags;
1206
4c923d47 1207 root = (struct root_entry *)alloc_pgtable_page(iommu->node);
ffebeb46 1208 if (!root) {
9f10e5bf 1209 pr_err("Allocating root entry for %s failed\n",
ffebeb46 1210 iommu->name);
ba395927 1211 return -ENOMEM;
ffebeb46 1212 }
ba395927 1213
5b6985ce 1214 __iommu_flush_cache(iommu, root, ROOT_SIZE);
ba395927
KA		 1215
1216 spin_lock_irqsave(&iommu->lock, flags);
1217 iommu->root_entry = root;
1218 spin_unlock_irqrestore(&iommu->lock, flags);
1219
1220 return 0;
1221}
1222
ba395927
KA		 1223static void iommu_set_root_entry(struct intel_iommu *iommu)
1224{
03ecc32c 1225 u64 addr;
c416daa9 1226 u32 sts;
ba395927
KA		 1227 unsigned long flag;
1228
03ecc32c 1229 addr = virt_to_phys(iommu->root_entry);
c83b2f20 1230 if (ecs_enabled(iommu))
03ecc32c 1231 addr |= DMA_RTADDR_RTT;
ba395927 1232
1f5b3c3f 1233 raw_spin_lock_irqsave(&iommu->register_lock, flag);
03ecc32c 1234 dmar_writeq(iommu->reg + DMAR_RTADDR_REG, addr);
ba395927 1235
c416daa9 1236 writel(iommu->gcmd | DMA_GCMD_SRTP, iommu->reg + DMAR_GCMD_REG);
ba395927
KA		 1237
1238 /* Make sure hardware complete it */
1239 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
c416daa9 1240 readl, (sts & DMA_GSTS_RTPS), sts);
ba395927 1241
1f5b3c3f 1242 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
ba395927
KA		 1243}
1244
1245static void iommu_flush_write_buffer(struct intel_iommu *iommu)
1246{
1247 u32 val;
1248 unsigned long flag;
1249
9af88143 1250 if (!rwbf_quirk && !cap_rwbf(iommu->cap))
ba395927 1251 return;
ba395927 1252
1f5b3c3f 1253 raw_spin_lock_irqsave(&iommu->register_lock, flag);
462b60f6 1254 writel(iommu->gcmd | DMA_GCMD_WBF, iommu->reg + DMAR_GCMD_REG);
ba395927
KA		 1255
1256 /* Make sure hardware complete it */
1257 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
c416daa9 1258 readl, (!(val & DMA_GSTS_WBFS)), val);
ba395927 1259
1f5b3c3f 1260 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
ba395927
KA		 1261}
1262
1263/* return value determine if we need a write buffer flush */
4c25a2c1
DW		 1264static void __iommu_flush_context(struct intel_iommu *iommu,
1265 u16 did, u16 source_id, u8 function_mask,
1266 u64 type)
ba395927
KA		 1267{
1268 u64 val = 0;
1269 unsigned long flag;
1270
ba395927
KA		 1271 switch (type) {
1272 case DMA_CCMD_GLOBAL_INVL:
1273 val = DMA_CCMD_GLOBAL_INVL;
1274 break;
1275 case DMA_CCMD_DOMAIN_INVL:
1276 val = DMA_CCMD_DOMAIN_INVL|DMA_CCMD_DID(did);
1277 break;
1278 case DMA_CCMD_DEVICE_INVL:
1279 val = DMA_CCMD_DEVICE_INVL|DMA_CCMD_DID(did)
1280 | DMA_CCMD_SID(source_id) | DMA_CCMD_FM(function_mask);
1281 break;
1282 default:
1283 BUG();
1284 }
1285 val |= DMA_CCMD_ICC;
1286
1f5b3c3f 1287 raw_spin_lock_irqsave(&iommu->register_lock, flag);
ba395927
KA		 1288 dmar_writeq(iommu->reg + DMAR_CCMD_REG, val);
1289
1290 /* Make sure hardware complete it */
1291 IOMMU_WAIT_OP(iommu, DMAR_CCMD_REG,
1292 dmar_readq, (!(val & DMA_CCMD_ICC)), val);
1293
1f5b3c3f 1294 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
ba395927
KA		 1295}
1296
ba395927 1297/* return value determine if we need a write buffer flush */
1f0ef2aa
DW		 1298static void __iommu_flush_iotlb(struct intel_iommu *iommu, u16 did,
1299 u64 addr, unsigned int size_order, u64 type)
ba395927
KA		 1300{
1301 int tlb_offset = ecap_iotlb_offset(iommu->ecap);
1302 u64 val = 0, val_iva = 0;
1303 unsigned long flag;
1304
ba395927
KA		 1305 switch (type) {
1306 case DMA_TLB_GLOBAL_FLUSH:
1307 /* global flush doesn't need set IVA_REG */
1308 val = DMA_TLB_GLOBAL_FLUSH|DMA_TLB_IVT;
1309 break;
1310 case DMA_TLB_DSI_FLUSH:
1311 val = DMA_TLB_DSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
1312 break;
1313 case DMA_TLB_PSI_FLUSH:
1314 val = DMA_TLB_PSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
ea8ea460 1315 /* IH bit is passed in as part of address */
ba395927
KA		 1316 val_iva = size_order | addr;
1317 break;
1318 default:
1319 BUG();
1320 }
1321 /* Note: set drain read/write */
1322#if 0
1323 /*
1324 * This is probably to be super secure.. Looks like we can
1325 * ignore it without any impact.
1326 */
1327 if (cap_read_drain(iommu->cap))
1328 val |= DMA_TLB_READ_DRAIN;
1329#endif
1330 if (cap_write_drain(iommu->cap))
1331 val |= DMA_TLB_WRITE_DRAIN;
1332
1f5b3c3f 1333 raw_spin_lock_irqsave(&iommu->register_lock, flag);
ba395927
KA		 1334 /* Note: Only uses first TLB reg currently */
1335 if (val_iva)
1336 dmar_writeq(iommu->reg + tlb_offset, val_iva);
1337 dmar_writeq(iommu->reg + tlb_offset + 8, val);
1338
1339 /* Make sure hardware complete it */
1340 IOMMU_WAIT_OP(iommu, tlb_offset + 8,
1341 dmar_readq, (!(val & DMA_TLB_IVT)), val);
1342
1f5b3c3f 1343 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
ba395927
KA		 1344
1345 /* check IOTLB invalidation granularity */
1346 if (DMA_TLB_IAIG(val) == 0)
9f10e5bf 1347 pr_err("Flush IOTLB failed\n");
ba395927 1348 if (DMA_TLB_IAIG(val) != DMA_TLB_IIRG(type))
9f10e5bf 1349 pr_debug("TLB flush request %Lx, actual %Lx\n",
5b6985ce
FY		 1350 (unsigned long long)DMA_TLB_IIRG(type),
1351 (unsigned long long)DMA_TLB_IAIG(val));
ba395927
KA		 1352}
1353
64ae892b
DW		 1354static struct device_domain_info *
1355iommu_support_dev_iotlb (struct dmar_domain *domain, struct intel_iommu *iommu,
1356 u8 bus, u8 devfn)
93a23a72 1357{
2f119c78 1358 bool found = false;
93a23a72
YZ		 1359 unsigned long flags;
1360 struct device_domain_info *info;
0bcb3e28 1361 struct pci_dev *pdev;
93a23a72
YZ		 1362
1363 if (!ecap_dev_iotlb_support(iommu->ecap))
1364 return NULL;
1365
1366 if (!iommu->qi)
1367 return NULL;
1368
1369 spin_lock_irqsave(&device_domain_lock, flags);
1370 list_for_each_entry(info, &domain->devices, link)
c3b497c6
JL		 1371 if (info->iommu == iommu && info->bus == bus &&
1372 info->devfn == devfn) {
2f119c78 1373 found = true;
93a23a72
YZ		 1374 break;
1375 }
1376 spin_unlock_irqrestore(&device_domain_lock, flags);
1377
0bcb3e28 1378 if (!found || !info->dev || !dev_is_pci(info->dev))
93a23a72
YZ		 1379 return NULL;
1380
0bcb3e28
DW		 1381 pdev = to_pci_dev(info->dev);
1382
1383 if (!pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_ATS))
93a23a72
YZ		 1384 return NULL;
1385
0bcb3e28 1386 if (!dmar_find_matched_atsr_unit(pdev))
93a23a72
YZ		 1387 return NULL;
1388
93a23a72
YZ		 1389 return info;
1390}
1391
1392static void iommu_enable_dev_iotlb(struct device_domain_info *info)
ba395927 1393{
0bcb3e28 1394 if (!info || !dev_is_pci(info->dev))
93a23a72
YZ		 1395 return;
1396
0bcb3e28 1397 pci_enable_ats(to_pci_dev(info->dev), VTD_PAGE_SHIFT);
93a23a72
YZ		 1398}
1399
1400static void iommu_disable_dev_iotlb(struct device_domain_info *info)
1401{
0bcb3e28
DW		 1402 if (!info->dev || !dev_is_pci(info->dev) ||
1403 !pci_ats_enabled(to_pci_dev(info->dev)))
93a23a72
YZ		 1404 return;
1405
0bcb3e28 1406 pci_disable_ats(to_pci_dev(info->dev));
93a23a72
YZ		 1407}
1408
1409static void iommu_flush_dev_iotlb(struct dmar_domain *domain,
1410 u64 addr, unsigned mask)
1411{
1412 u16 sid, qdep;
1413 unsigned long flags;
1414 struct device_domain_info *info;
1415
1416 spin_lock_irqsave(&device_domain_lock, flags);
1417 list_for_each_entry(info, &domain->devices, link) {
0bcb3e28
DW		 1418 struct pci_dev *pdev;
1419 if (!info->dev || !dev_is_pci(info->dev))
1420 continue;
1421
1422 pdev = to_pci_dev(info->dev);
1423 if (!pci_ats_enabled(pdev))
93a23a72
YZ		 1424 continue;
1425
1426 sid = info->bus << 8 | info->devfn;
0bcb3e28 1427 qdep = pci_ats_queue_depth(pdev);
93a23a72
YZ		 1428 qi_flush_dev_iotlb(info->iommu, sid, qdep, addr, mask);
1429 }
1430 spin_unlock_irqrestore(&device_domain_lock, flags);
1431}
1432
1f0ef2aa 1433static void iommu_flush_iotlb_psi(struct intel_iommu *iommu, u16 did,
ea8ea460 1434 unsigned long pfn, unsigned int pages, int ih, int map)
ba395927 1435{
9dd2fe89 1436 unsigned int mask = ilog2(__roundup_pow_of_two(pages));
03d6a246 1437 uint64_t addr = (uint64_t)pfn << VTD_PAGE_SHIFT;
ba395927 1438
ba395927
KA		 1439 BUG_ON(pages == 0);
1440
ea8ea460
DW		 1441 if (ih)
1442 ih = 1 << 6;
ba395927 1443 /*
9dd2fe89
YZ		 1444 * Fallback to domain selective flush if no PSI support or the size is
1445 * too big.
ba395927
KA		 1446 * PSI requires page size to be 2 ^ x, and the base address is naturally
1447 * aligned to the size
1448 */
9dd2fe89
YZ		 1449 if (!cap_pgsel_inv(iommu->cap) || mask > cap_max_amask_val(iommu->cap))
1450 iommu->flush.flush_iotlb(iommu, did, 0, 0,
1f0ef2aa 1451 DMA_TLB_DSI_FLUSH);
9dd2fe89 1452 else
ea8ea460 1453 iommu->flush.flush_iotlb(iommu, did, addr | ih, mask,
9dd2fe89 1454 DMA_TLB_PSI_FLUSH);
bf92df30
YZ		 1455
1456 /*
82653633
NA		 1457 * In caching mode, changes of pages from non-present to present require
1458 * flush. However, device IOTLB doesn't need to be flushed in this case.
bf92df30 1459 */
82653633 1460 if (!cap_caching_mode(iommu->cap) || !map)
93a23a72 1461 iommu_flush_dev_iotlb(iommu->domains[did], addr, mask);
ba395927
KA		 1462}
1463
f8bab735 1464static void iommu_disable_protect_mem_regions(struct intel_iommu *iommu)
1465{
1466 u32 pmen;
1467 unsigned long flags;
1468
1f5b3c3f 1469 raw_spin_lock_irqsave(&iommu->register_lock, flags);
f8bab735 1470 pmen = readl(iommu->reg + DMAR_PMEN_REG);
1471 pmen &= ~DMA_PMEN_EPM;
1472 writel(pmen, iommu->reg + DMAR_PMEN_REG);
1473
1474 /* wait for the protected region status bit to clear */
1475 IOMMU_WAIT_OP(iommu, DMAR_PMEN_REG,
1476 readl, !(pmen & DMA_PMEN_PRS), pmen);
1477
1f5b3c3f 1478 raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
f8bab735 1479}
1480
2a41ccee 1481static void iommu_enable_translation(struct intel_iommu *iommu)
ba395927
KA		 1482{
1483 u32 sts;
1484 unsigned long flags;
1485
1f5b3c3f 1486 raw_spin_lock_irqsave(&iommu->register_lock, flags);
c416daa9
DW		 1487 iommu->gcmd |= DMA_GCMD_TE;
1488 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
ba395927
KA		 1489
1490 /* Make sure hardware complete it */
1491 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
c416daa9 1492 readl, (sts & DMA_GSTS_TES), sts);
ba395927 1493
1f5b3c3f 1494 raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
ba395927
KA		 1495}
1496
2a41ccee 1497static void iommu_disable_translation(struct intel_iommu *iommu)
ba395927
KA		 1498{
1499 u32 sts;
1500 unsigned long flag;
1501
1f5b3c3f 1502 raw_spin_lock_irqsave(&iommu->register_lock, flag);
ba395927
KA		 1503 iommu->gcmd &= ~DMA_GCMD_TE;
1504 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1505
1506 /* Make sure hardware complete it */
1507 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
c416daa9 1508 readl, (!(sts & DMA_GSTS_TES)), sts);
ba395927 1509
1f5b3c3f 1510 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
ba395927
KA		 1511}
1512
3460a6d9 1513
ba395927
KA		 1514static int iommu_init_domains(struct intel_iommu *iommu)
1515{
1516 unsigned long ndomains;
1517 unsigned long nlongs;
1518
1519 ndomains = cap_ndoms(iommu->cap);
9f10e5bf
JR		 1520 pr_debug("%s: Number of Domains supported <%ld>\n",
1521 iommu->name, ndomains);
ba395927
KA		 1522 nlongs = BITS_TO_LONGS(ndomains);
1523
94a91b50
DD		 1524 spin_lock_init(&iommu->lock);
1525
ba395927
KA		 1526 /* TBD: there might be 64K domains,
1527 * consider other allocation for future chip
1528 */
1529 iommu->domain_ids = kcalloc(nlongs, sizeof(unsigned long), GFP_KERNEL);
1530 if (!iommu->domain_ids) {
9f10e5bf
JR		 1531 pr_err("%s: Allocating domain id array failed\n",
1532 iommu->name);
ba395927
KA		 1533 return -ENOMEM;
1534 }
1535 iommu->domains = kcalloc(ndomains, sizeof(struct dmar_domain *),
1536 GFP_KERNEL);
1537 if (!iommu->domains) {
9f10e5bf
JR		 1538 pr_err("%s: Allocating domain array failed\n",
1539 iommu->name);
852bdb04
JL		 1540 kfree(iommu->domain_ids);
1541 iommu->domain_ids = NULL;
ba395927
KA		 1542 return -ENOMEM;
1543 }
1544
1545 /*
1546 * if Caching mode is set, then invalid translations are tagged
1547 * with domainid 0. Hence we need to pre-allocate it.
1548 */
1549 if (cap_caching_mode(iommu->cap))
1550 set_bit(0, iommu->domain_ids);
1551 return 0;
1552}
ba395927 1553
ffebeb46 1554static void disable_dmar_iommu(struct intel_iommu *iommu)
ba395927
KA		 1555{
1556 struct dmar_domain *domain;
2a46ddf7 1557 int i;
ba395927 1558
94a91b50 1559 if ((iommu->domains) && (iommu->domain_ids)) {
a45946ab 1560 for_each_set_bit(i, iommu->domain_ids, cap_ndoms(iommu->cap)) {
a4eaa86c
JL		 1561 /*
1562 * Domain id 0 is reserved for invalid translation
1563 * if hardware supports caching mode.
1564 */
1565 if (cap_caching_mode(iommu->cap) && i == 0)
1566 continue;
1567
94a91b50
DD		 1568 domain = iommu->domains[i];
1569 clear_bit(i, iommu->domain_ids);
129ad281
JL		 1570 if (domain_detach_iommu(domain, iommu) == 0 &&
1571 !domain_type_is_vm(domain))
92d03cc8 1572 domain_exit(domain);
5e98c4b1 1573 }
ba395927
KA		 1574 }
1575
1576 if (iommu->gcmd & DMA_GCMD_TE)
1577 iommu_disable_translation(iommu);
ffebeb46 1578}
ba395927 1579
ffebeb46
JL		 1580static void free_dmar_iommu(struct intel_iommu *iommu)
1581{
1582 if ((iommu->domains) && (iommu->domain_ids)) {
1583 kfree(iommu->domains);
1584 kfree(iommu->domain_ids);
1585 iommu->domains = NULL;
1586 iommu->domain_ids = NULL;
1587 }
ba395927 1588
d9630fe9
WH		 1589 g_iommus[iommu->seq_id] = NULL;
1590
ba395927
KA		 1591 /* free context mapping */
1592 free_context_table(iommu);
ba395927
KA		 1593}
1594
ab8dfe25 1595static struct dmar_domain *alloc_domain(int flags)
ba395927 1596{
92d03cc8
JL		 1597 /* domain id for virtual machine, it won't be set in context */
1598 static atomic_t vm_domid = ATOMIC_INIT(0);
ba395927 1599 struct dmar_domain *domain;
ba395927
KA		 1600
1601 domain = alloc_domain_mem();
1602 if (!domain)
1603 return NULL;
1604
ab8dfe25 1605 memset(domain, 0, sizeof(*domain));
4c923d47 1606 domain->nid = -1;
ab8dfe25 1607 domain->flags = flags;
92d03cc8
JL		 1608 spin_lock_init(&domain->iommu_lock);
1609 INIT_LIST_HEAD(&domain->devices);
ab8dfe25 1610 if (flags & DOMAIN_FLAG_VIRTUAL_MACHINE)
92d03cc8 1611 domain->id = atomic_inc_return(&vm_domid);
2c2e2c38
FY		 1612
1613 return domain;
1614}
1615
fb170fb4
JL		 1616static int __iommu_attach_domain(struct dmar_domain *domain,
1617 struct intel_iommu *iommu)
2c2e2c38
FY		 1618{
1619 int num;
1620 unsigned long ndomains;
2c2e2c38 1621
ba395927 1622 ndomains = cap_ndoms(iommu->cap);
ba395927 1623 num = find_first_zero_bit(iommu->domain_ids, ndomains);
fb170fb4
JL		 1624 if (num < ndomains) {
1625 set_bit(num, iommu->domain_ids);
1626 iommu->domains[num] = domain;
1627 } else {
1628 num = -ENOSPC;
ba395927
KA		 1629 }
1630
fb170fb4
JL		 1631 return num;
1632}
1633
1634static int iommu_attach_domain(struct dmar_domain *domain,
1635 struct intel_iommu *iommu)
1636{
1637 int num;
1638 unsigned long flags;
1639
1640 spin_lock_irqsave(&iommu->lock, flags);
1641 num = __iommu_attach_domain(domain, iommu);
44bde614 1642 spin_unlock_irqrestore(&iommu->lock, flags);
fb170fb4 1643 if (num < 0)
9f10e5bf 1644 pr_err("%s: No free domain ids\n", iommu->name);
ba395927 1645
fb170fb4 1646 return num;
ba395927
KA		 1647}
1648
44bde614
JL		 1649static int iommu_attach_vm_domain(struct dmar_domain *domain,
1650 struct intel_iommu *iommu)
1651{
1652 int num;
1653 unsigned long ndomains;
1654
1655 ndomains = cap_ndoms(iommu->cap);
1656 for_each_set_bit(num, iommu->domain_ids, ndomains)
1657 if (iommu->domains[num] == domain)
1658 return num;
1659
1660 return __iommu_attach_domain(domain, iommu);
1661}
1662
2c2e2c38
FY		 1663static void iommu_detach_domain(struct dmar_domain *domain,
1664 struct intel_iommu *iommu)
ba395927
KA		 1665{
1666 unsigned long flags;
2c2e2c38 1667 int num, ndomains;
ba395927 1668
8c11e798 1669 spin_lock_irqsave(&iommu->lock, flags);
fb170fb4
JL		 1670 if (domain_type_is_vm_or_si(domain)) {
1671 ndomains = cap_ndoms(iommu->cap);
1672 for_each_set_bit(num, iommu->domain_ids, ndomains) {
1673 if (iommu->domains[num] == domain) {
1674 clear_bit(num, iommu->domain_ids);
1675 iommu->domains[num] = NULL;
1676 break;
1677 }
2c2e2c38 1678 }
fb170fb4
JL		 1679 } else {
1680 clear_bit(domain->id, iommu->domain_ids);
1681 iommu->domains[domain->id] = NULL;
2c2e2c38 1682 }
8c11e798 1683 spin_unlock_irqrestore(&iommu->lock, flags);
ba395927
KA		 1684}
1685
fb170fb4
JL		 1686static void domain_attach_iommu(struct dmar_domain *domain,
1687 struct intel_iommu *iommu)
1688{
1689 unsigned long flags;
1690
1691 spin_lock_irqsave(&domain->iommu_lock, flags);
1692 if (!test_and_set_bit(iommu->seq_id, domain->iommu_bmp)) {
1693 domain->iommu_count++;
1694 if (domain->iommu_count == 1)
1695 domain->nid = iommu->node;
1696 domain_update_iommu_cap(domain);
1697 }
1698 spin_unlock_irqrestore(&domain->iommu_lock, flags);
1699}
1700
1701static int domain_detach_iommu(struct dmar_domain *domain,
1702 struct intel_iommu *iommu)
1703{
1704 unsigned long flags;
1705 int count = INT_MAX;
1706
1707 spin_lock_irqsave(&domain->iommu_lock, flags);
1708 if (test_and_clear_bit(iommu->seq_id, domain->iommu_bmp)) {
1709 count = --domain->iommu_count;
1710 domain_update_iommu_cap(domain);
1711 }
1712 spin_unlock_irqrestore(&domain->iommu_lock, flags);
1713
1714 return count;
1715}
1716
ba395927 1717static struct iova_domain reserved_iova_list;
8a443df4 1718static struct lock_class_key reserved_rbtree_key;
ba395927 1719
51a63e67 1720static int dmar_init_reserved_ranges(void)
ba395927
KA		 1721{
1722 struct pci_dev *pdev = NULL;
1723 struct iova *iova;
1724 int i;
ba395927 1725
0fb5fe87
RM		 1726 init_iova_domain(&reserved_iova_list, VTD_PAGE_SIZE, IOVA_START_PFN,
1727 DMA_32BIT_PFN);
ba395927 1728
8a443df4
MG		 1729 lockdep_set_class(&reserved_iova_list.iova_rbtree_lock,
1730 &reserved_rbtree_key);
1731
ba395927
KA		 1732 /* IOAPIC ranges shouldn't be accessed by DMA */
1733 iova = reserve_iova(&reserved_iova_list, IOVA_PFN(IOAPIC_RANGE_START),
1734 IOVA_PFN(IOAPIC_RANGE_END));
51a63e67 1735 if (!iova) {
9f10e5bf 1736 pr_err("Reserve IOAPIC range failed\n");
51a63e67
JC		 1737 return -ENODEV;
1738 }
ba395927
KA		 1739
1740 /* Reserve all PCI MMIO to avoid peer-to-peer access */
1741 for_each_pci_dev(pdev) {
1742 struct resource *r;
1743
1744 for (i = 0; i < PCI_NUM_RESOURCES; i++) {
1745 r = &pdev->resource[i];
1746 if (!r->flags || !(r->flags & IORESOURCE_MEM))
1747 continue;
1a4a4551
DW		 1748 iova = reserve_iova(&reserved_iova_list,
1749 IOVA_PFN(r->start),
1750 IOVA_PFN(r->end));
51a63e67 1751 if (!iova) {
9f10e5bf 1752 pr_err("Reserve iova failed\n");
51a63e67
JC		 1753 return -ENODEV;
1754 }
ba395927
KA		 1755 }
1756 }
51a63e67 1757 return 0;
ba395927
KA		 1758}
1759
1760static void domain_reserve_special_ranges(struct dmar_domain *domain)
1761{
1762 copy_reserved_iova(&reserved_iova_list, &domain->iovad);
1763}
1764
1765static inline int guestwidth_to_adjustwidth(int gaw)
1766{
1767 int agaw;
1768 int r = (gaw - 12) % 9;
1769
1770 if (r == 0)
1771 agaw = gaw;
1772 else
1773 agaw = gaw + 9 - r;
1774 if (agaw > 64)
1775 agaw = 64;
1776 return agaw;
1777}
1778
1779static int domain_init(struct dmar_domain *domain, int guest_width)
1780{
1781 struct intel_iommu *iommu;
1782 int adjust_width, agaw;
1783 unsigned long sagaw;
1784
0fb5fe87
RM		 1785 init_iova_domain(&domain->iovad, VTD_PAGE_SIZE, IOVA_START_PFN,
1786 DMA_32BIT_PFN);
ba395927
KA		 1787 domain_reserve_special_ranges(domain);
1788
1789 /* calculate AGAW */
8c11e798 1790 iommu = domain_get_iommu(domain);
ba395927
KA		 1791 if (guest_width > cap_mgaw(iommu->cap))
1792 guest_width = cap_mgaw(iommu->cap);
1793 domain->gaw = guest_width;
1794 adjust_width = guestwidth_to_adjustwidth(guest_width);
1795 agaw = width_to_agaw(adjust_width);
1796 sagaw = cap_sagaw(iommu->cap);
1797 if (!test_bit(agaw, &sagaw)) {
1798 /* hardware doesn't support it, choose a bigger one */
9f10e5bf 1799 pr_debug("Hardware doesn't support agaw %d\n", agaw);
ba395927
KA		 1800 agaw = find_next_bit(&sagaw, 5, agaw);
1801 if (agaw >= 5)
1802 return -ENODEV;
1803 }
1804 domain->agaw = agaw;
ba395927 1805
8e604097
WH		 1806 if (ecap_coherent(iommu->ecap))
1807 domain->iommu_coherency = 1;
1808 else
1809 domain->iommu_coherency = 0;
1810
58c610bd
SY		 1811 if (ecap_sc_support(iommu->ecap))
1812 domain->iommu_snooping = 1;
1813 else
1814 domain->iommu_snooping = 0;
1815
214e39aa
DW		 1816 if (intel_iommu_superpage)
1817 domain->iommu_superpage = fls(cap_super_page_val(iommu->cap));
1818 else
1819 domain->iommu_superpage = 0;
1820
4c923d47 1821 domain->nid = iommu->node;
c7151a8d 1822
ba395927 1823 /* always allocate the top pgd */
4c923d47 1824 domain->pgd = (struct dma_pte *)alloc_pgtable_page(domain->nid);
ba395927
KA		 1825 if (!domain->pgd)
1826 return -ENOMEM;
5b6985ce 1827 __iommu_flush_cache(iommu, domain->pgd, PAGE_SIZE);
ba395927
KA		 1828 return 0;
1829}
1830
1831static void domain_exit(struct dmar_domain *domain)
1832{
46ebb7af
AW		 1833 struct dmar_drhd_unit *drhd;
1834 struct intel_iommu *iommu;
ea8ea460 1835 struct page *freelist = NULL;
ba395927
KA		 1836
1837 /* Domain 0 is reserved, so dont process it */
1838 if (!domain)
1839 return;
1840
7b668357
AW		 1841 /* Flush any lazy unmaps that may reference this domain */
1842 if (!intel_iommu_strict)
1843 flush_unmaps_timeout(0);
1844
92d03cc8 1845 /* remove associated devices */
ba395927 1846 domain_remove_dev_info(domain);
92d03cc8 1847
ba395927
KA		 1848 /* destroy iovas */
1849 put_iova_domain(&domain->iovad);
ba395927 1850
ea8ea460 1851 freelist = domain_unmap(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
ba395927 1852
92d03cc8 1853 /* clear attached or cached domains */
0e242612 1854 rcu_read_lock();
46ebb7af
AW		 1855 for_each_active_iommu(iommu, drhd)
1856 if (domain_type_is_vm(domain) ||
1857 test_bit(iommu->seq_id, domain->iommu_bmp))
1858 iommu_detach_domain(domain, iommu);
0e242612 1859 rcu_read_unlock();
2c2e2c38 1860
ea8ea460
DW		 1861 dma_free_pagelist(freelist);
1862
ba395927
KA		 1863 free_domain_mem(domain);
1864}
1865
64ae892b
DW		 1866static int domain_context_mapping_one(struct dmar_domain *domain,
1867 struct intel_iommu *iommu,
1868 u8 bus, u8 devfn, int translation)
ba395927
KA		 1869{
1870 struct context_entry *context;
ba395927 1871 unsigned long flags;
ea6606b0 1872 struct dma_pte *pgd;
ea6606b0
WH		 1873 int id;
1874 int agaw;
93a23a72 1875 struct device_domain_info *info = NULL;
ba395927
KA		 1876
1877 pr_debug("Set context mapping for %02x:%02x.%d\n",
1878 bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
4ed0d3e6 1879
ba395927 1880 BUG_ON(!domain->pgd);
4ed0d3e6
FY		 1881 BUG_ON(translation != CONTEXT_TT_PASS_THROUGH &&
1882 translation != CONTEXT_TT_MULTI_LEVEL);
5331fe6f 1883
03ecc32c
DW		 1884 spin_lock_irqsave(&iommu->lock, flags);
1885 context = iommu_context_addr(iommu, bus, devfn, 1);
1886 spin_unlock_irqrestore(&iommu->lock, flags);
ba395927
KA		 1887 if (!context)
1888 return -ENOMEM;
1889 spin_lock_irqsave(&iommu->lock, flags);
c07e7d21 1890 if (context_present(context)) {
ba395927
KA		 1891 spin_unlock_irqrestore(&iommu->lock, flags);
1892 return 0;
1893 }
1894
cf484d0e
JR		 1895 context_clear_entry(context);
1896
ea6606b0
WH		 1897 id = domain->id;
1898 pgd = domain->pgd;
1899
ab8dfe25 1900 if (domain_type_is_vm_or_si(domain)) {
44bde614
JL		 1901 if (domain_type_is_vm(domain)) {
1902 id = iommu_attach_vm_domain(domain, iommu);
fb170fb4 1903 if (id < 0) {
ea6606b0 1904 spin_unlock_irqrestore(&iommu->lock, flags);
9f10e5bf 1905 pr_err("%s: No free domain ids\n", iommu->name);
ea6606b0
WH		 1906 return -EFAULT;
1907 }
ea6606b0
WH		 1908 }
1909
1910 /* Skip top levels of page tables for
1911 * iommu which has less agaw than default.
1672af11 1912 * Unnecessary for PT mode.
ea6606b0 1913 */
1672af11
CW		 1914 if (translation != CONTEXT_TT_PASS_THROUGH) {
1915 for (agaw = domain->agaw; agaw != iommu->agaw; agaw--) {
1916 pgd = phys_to_virt(dma_pte_addr(pgd));
1917 if (!dma_pte_present(pgd)) {
1918 spin_unlock_irqrestore(&iommu->lock, flags);
1919 return -ENOMEM;
1920 }
ea6606b0
WH		 1921 }
1922 }
1923 }
1924
1925 context_set_domain_id(context, id);
4ed0d3e6 1926
93a23a72 1927 if (translation != CONTEXT_TT_PASS_THROUGH) {
64ae892b 1928 info = iommu_support_dev_iotlb(domain, iommu, bus, devfn);
93a23a72
YZ		 1929 translation = info ? CONTEXT_TT_DEV_IOTLB :
1930 CONTEXT_TT_MULTI_LEVEL;
1931 }
4ed0d3e6
FY		 1932 /*
1933 * In pass through mode, AW must be programmed to indicate the largest
1934 * AGAW value supported by hardware. And ASR is ignored by hardware.
1935 */
93a23a72 1936 if (unlikely(translation == CONTEXT_TT_PASS_THROUGH))
4ed0d3e6 1937 context_set_address_width(context, iommu->msagaw);
93a23a72
YZ		 1938 else {
1939 context_set_address_root(context, virt_to_phys(pgd));
1940 context_set_address_width(context, iommu->agaw);
1941 }
4ed0d3e6
FY		 1942
1943 context_set_translation_type(context, translation);
c07e7d21
MM		 1944 context_set_fault_enable(context);
1945 context_set_present(context);
5331fe6f 1946 domain_flush_cache(domain, context, sizeof(*context));
ba395927 1947
4c25a2c1
DW		 1948 /*
1949 * It's a non-present to present mapping. If hardware doesn't cache
1950 * non-present entry we only need to flush the write-buffer. If the
1951 * _does_ cache non-present entries, then it does so in the special
1952 * domain #0, which we have to flush:
1953 */
1954 if (cap_caching_mode(iommu->cap)) {
1955 iommu->flush.flush_context(iommu, 0,
1956 (((u16)bus) << 8) | devfn,
1957 DMA_CCMD_MASK_NOBIT,
1958 DMA_CCMD_DEVICE_INVL);
18fd779a 1959 iommu->flush.flush_iotlb(iommu, id, 0, 0, DMA_TLB_DSI_FLUSH);
4c25a2c1 1960 } else {
ba395927 1961 iommu_flush_write_buffer(iommu);
4c25a2c1 1962 }
93a23a72 1963 iommu_enable_dev_iotlb(info);
ba395927 1964 spin_unlock_irqrestore(&iommu->lock, flags);
c7151a8d 1965
fb170fb4
JL		 1966 domain_attach_iommu(domain, iommu);
1967
ba395927
KA		 1968 return 0;
1969}
1970
579305f7
AW		 1971struct domain_context_mapping_data {
1972 struct dmar_domain *domain;
1973 struct intel_iommu *iommu;
1974 int translation;
1975};
1976
1977static int domain_context_mapping_cb(struct pci_dev *pdev,
1978 u16 alias, void *opaque)
1979{
1980 struct domain_context_mapping_data *data = opaque;
1981
1982 return domain_context_mapping_one(data->domain, data->iommu,
1983 PCI_BUS_NUM(alias), alias & 0xff,
1984 data->translation);
1985}
1986
ba395927 1987static int
e1f167f3
DW		 1988domain_context_mapping(struct dmar_domain *domain, struct device *dev,
1989 int translation)
ba395927 1990{
64ae892b 1991 struct intel_iommu *iommu;
156baca8 1992 u8 bus, devfn;
579305f7 1993 struct domain_context_mapping_data data;
64ae892b 1994
e1f167f3 1995 iommu = device_to_iommu(dev, &bus, &devfn);
64ae892b
DW		 1996 if (!iommu)
1997 return -ENODEV;
ba395927 1998
579305f7
AW		 1999 if (!dev_is_pci(dev))
2000 return domain_context_mapping_one(domain, iommu, bus, devfn,
4ed0d3e6 2001 translation);
579305f7
AW		 2002
2003 data.domain = domain;
2004 data.iommu = iommu;
2005 data.translation = translation;
2006
2007 return pci_for_each_dma_alias(to_pci_dev(dev),
2008 &domain_context_mapping_cb, &data);
2009}
2010
2011static int domain_context_mapped_cb(struct pci_dev *pdev,
2012 u16 alias, void *opaque)
2013{
2014 struct intel_iommu *iommu = opaque;
2015
2016 return !device_context_mapped(iommu, PCI_BUS_NUM(alias), alias & 0xff);
ba395927
KA		 2017}
2018
e1f167f3 2019static int domain_context_mapped(struct device *dev)
ba395927 2020{
5331fe6f 2021 struct intel_iommu *iommu;
156baca8 2022 u8 bus, devfn;
5331fe6f 2023
e1f167f3 2024 iommu = device_to_iommu(dev, &bus, &devfn);
5331fe6f
WH		 2025 if (!iommu)
2026 return -ENODEV;
ba395927 2027
579305f7
AW		 2028 if (!dev_is_pci(dev))
2029 return device_context_mapped(iommu, bus, devfn);
e1f167f3 2030
579305f7
AW		 2031 return !pci_for_each_dma_alias(to_pci_dev(dev),
2032 domain_context_mapped_cb, iommu);
ba395927
KA		 2033}
2034
f532959b
FY		 2035/* Returns a number of VTD pages, but aligned to MM page size */
2036static inline unsigned long aligned_nrpages(unsigned long host_addr,
2037 size_t size)
2038{
2039 host_addr &= ~PAGE_MASK;
2040 return PAGE_ALIGN(host_addr + size) >> VTD_PAGE_SHIFT;
2041}
2042
6dd9a7c7
YS		 2043/* Return largest possible superpage level for a given mapping */
2044static inline int hardware_largepage_caps(struct dmar_domain *domain,
2045 unsigned long iov_pfn,
2046 unsigned long phy_pfn,
2047 unsigned long pages)
2048{
2049 int support, level = 1;
2050 unsigned long pfnmerge;
2051
2052 support = domain->iommu_superpage;
2053
2054 /* To use a large page, the virtual *and* physical addresses
2055 must be aligned to 2MiB/1GiB/etc. Lower bits set in either
2056 of them will mean we have to use smaller pages. So just
2057 merge them and check both at once. */
2058 pfnmerge = iov_pfn | phy_pfn;
2059
2060 while (support && !(pfnmerge & ~VTD_STRIDE_MASK)) {
2061 pages >>= VTD_STRIDE_SHIFT;
2062 if (!pages)
2063 break;
2064 pfnmerge >>= VTD_STRIDE_SHIFT;
2065 level++;
2066 support--;
2067 }
2068 return level;
2069}
2070
9051aa02
DW		 2071static int __domain_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
2072 struct scatterlist *sg, unsigned long phys_pfn,
2073 unsigned long nr_pages, int prot)
e1605495
DW		 2074{
2075 struct dma_pte *first_pte = NULL, *pte = NULL;
9051aa02 2076 phys_addr_t uninitialized_var(pteval);
cc4f14aa 2077 unsigned long sg_res = 0;
6dd9a7c7
YS		 2078 unsigned int largepage_lvl = 0;
2079 unsigned long lvl_pages = 0;
e1605495 2080
162d1b10 2081 BUG_ON(!domain_pfn_supported(domain, iov_pfn + nr_pages - 1));
e1605495
DW		 2082
2083 if ((prot & (DMA_PTE_READ|DMA_PTE_WRITE)) == 0)
2084 return -EINVAL;
2085
2086 prot &= DMA_PTE_READ | DMA_PTE_WRITE | DMA_PTE_SNP;
2087
cc4f14aa
JL		 2088 if (!sg) {
2089 sg_res = nr_pages;
9051aa02
DW		 2090 pteval = ((phys_addr_t)phys_pfn << VTD_PAGE_SHIFT) | prot;
2091 }
2092
6dd9a7c7 2093 while (nr_pages > 0) {
c85994e4
DW		 2094 uint64_t tmp;
2095
e1605495 2096 if (!sg_res) {
f532959b 2097 sg_res = aligned_nrpages(sg->offset, sg->length);
e1605495
DW		 2098 sg->dma_address = ((dma_addr_t)iov_pfn << VTD_PAGE_SHIFT) + sg->offset;
2099 sg->dma_length = sg->length;
2100 pteval = page_to_phys(sg_page(sg)) | prot;
6dd9a7c7 2101 phys_pfn = pteval >> VTD_PAGE_SHIFT;
e1605495 2102 }
6dd9a7c7 2103
e1605495 2104 if (!pte) {
6dd9a7c7
YS		 2105 largepage_lvl = hardware_largepage_caps(domain, iov_pfn, phys_pfn, sg_res);
2106
5cf0a76f 2107 first_pte = pte = pfn_to_dma_pte(domain, iov_pfn, &largepage_lvl);
e1605495
DW		 2108 if (!pte)
2109 return -ENOMEM;
6dd9a7c7 2110 /* It is large page*/
6491d4d0 2111 if (largepage_lvl > 1) {
6dd9a7c7 2112 pteval |= DMA_PTE_LARGE_PAGE;
d41a4adb
JL		 2113 lvl_pages = lvl_to_nr_pages(largepage_lvl);
2114 /*
2115 * Ensure that old small page tables are
2116 * removed to make room for superpage,
2117 * if they exist.
2118 */
6491d4d0 2119 dma_pte_free_pagetable(domain, iov_pfn,
d41a4adb 2120 iov_pfn + lvl_pages - 1);
6491d4d0 2121 } else {
6dd9a7c7 2122 pteval &= ~(uint64_t)DMA_PTE_LARGE_PAGE;
6491d4d0 2123 }
6dd9a7c7 2124
e1605495
DW		 2125 }
2126 /* We don't need lock here, nobody else
2127 * touches the iova range
2128 */
7766a3fb 2129 tmp = cmpxchg64_local(&pte->val, 0ULL, pteval);
c85994e4 2130 if (tmp) {
1bf20f0d 2131 static int dumps = 5;
9f10e5bf
JR		 2132 pr_crit("ERROR: DMA PTE for vPFN 0x%lx already set (to %llx not %llx)\n",
2133 iov_pfn, tmp, (unsigned long long)pteval);
1bf20f0d
DW		 2134 if (dumps) {
2135 dumps--;
2136 debug_dma_dump_mappings(NULL);
2137 }
2138 WARN_ON(1);
2139 }
6dd9a7c7
YS		 2140
2141 lvl_pages = lvl_to_nr_pages(largepage_lvl);
2142
2143 BUG_ON(nr_pages < lvl_pages);
2144 BUG_ON(sg_res < lvl_pages);
2145
2146 nr_pages -= lvl_pages;
2147 iov_pfn += lvl_pages;
2148 phys_pfn += lvl_pages;
2149 pteval += lvl_pages * VTD_PAGE_SIZE;
2150 sg_res -= lvl_pages;
2151
2152 /* If the next PTE would be the first in a new page, then we
2153 need to flush the cache on the entries we've just written.
2154 And then we'll need to recalculate 'pte', so clear it and
2155 let it get set again in the if (!pte) block above.
2156
2157 If we're done (!nr_pages) we need to flush the cache too.
2158
2159 Also if we've been setting superpages, we may need to
2160 recalculate 'pte' and switch back to smaller pages for the
2161 end of the mapping, if the trailing size is not enough to
2162 use another superpage (i.e. sg_res < lvl_pages). */
e1605495 2163 pte++;
6dd9a7c7
YS		 2164 if (!nr_pages || first_pte_in_page(pte) ||
2165 (largepage_lvl > 1 && sg_res < lvl_pages)) {
e1605495
DW		 2166 domain_flush_cache(domain, first_pte,
2167 (void *)pte - (void *)first_pte);
2168 pte = NULL;
2169 }
6dd9a7c7
YS		 2170
2171 if (!sg_res && nr_pages)
e1605495
DW		 2172 sg = sg_next(sg);
2173 }
2174 return 0;
2175}
2176
9051aa02
DW		 2177static inline int domain_sg_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
2178 struct scatterlist *sg, unsigned long nr_pages,
2179 int prot)
ba395927 2180{
9051aa02
DW		 2181 return __domain_mapping(domain, iov_pfn, sg, 0, nr_pages, prot);
2182}
6f6a00e4 2183
9051aa02
DW		 2184static inline int domain_pfn_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
2185 unsigned long phys_pfn, unsigned long nr_pages,
2186 int prot)
2187{
2188 return __domain_mapping(domain, iov_pfn, NULL, phys_pfn, nr_pages, prot);
ba395927
KA		 2189}
2190
c7151a8d 2191static void iommu_detach_dev(struct intel_iommu *iommu, u8 bus, u8 devfn)
ba395927 2192{
c7151a8d
WH		 2193 if (!iommu)
2194 return;
8c11e798
WH		 2195
2196 clear_context_table(iommu, bus, devfn);
2197 iommu->flush.flush_context(iommu, 0, 0, 0,
4c25a2c1 2198 DMA_CCMD_GLOBAL_INVL);
1f0ef2aa 2199 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
ba395927
KA		 2200}
2201
109b9b04
DW		 2202static inline void unlink_domain_info(struct device_domain_info *info)
2203{
2204 assert_spin_locked(&device_domain_lock);
2205 list_del(&info->link);
2206 list_del(&info->global);
2207 if (info->dev)
0bcb3e28 2208 info->dev->archdata.iommu = NULL;
109b9b04
DW		 2209}
2210
ba395927
KA		 2211static void domain_remove_dev_info(struct dmar_domain *domain)
2212{
3a74ca01 2213 struct device_domain_info *info, *tmp;
fb170fb4 2214 unsigned long flags;
ba395927
KA		 2215
2216 spin_lock_irqsave(&device_domain_lock, flags);
3a74ca01 2217 list_for_each_entry_safe(info, tmp, &domain->devices, link) {
109b9b04 2218 unlink_domain_info(info);
ba395927
KA		 2219 spin_unlock_irqrestore(&device_domain_lock, flags);
2220
93a23a72 2221 iommu_disable_dev_iotlb(info);
7c7faa11 2222 iommu_detach_dev(info->iommu, info->bus, info->devfn);
ba395927 2223
ab8dfe25 2224 if (domain_type_is_vm(domain)) {
7c7faa11 2225 iommu_detach_dependent_devices(info->iommu, info->dev);
fb170fb4 2226 domain_detach_iommu(domain, info->iommu);
92d03cc8
JL		 2227 }
2228
2229 free_devinfo_mem(info);
ba395927
KA		 2230 spin_lock_irqsave(&device_domain_lock, flags);
2231 }
2232 spin_unlock_irqrestore(&device_domain_lock, flags);
2233}
2234
2235/*
2236 * find_domain
1525a29a 2237 * Note: we use struct device->archdata.iommu stores the info
ba395927 2238 */
1525a29a 2239static struct dmar_domain *find_domain(struct device *dev)
ba395927
KA		 2240{
2241 struct device_domain_info *info;
2242
2243 /* No lock here, assumes no domain exit in normal case */
1525a29a 2244 info = dev->archdata.iommu;
ba395927
KA		 2245 if (info)
2246 return info->domain;
2247 return NULL;
2248}
2249
5a8f40e8 2250static inline struct device_domain_info *
745f2586
JL		 2251dmar_search_domain_by_dev_info(int segment, int bus, int devfn)
2252{
2253 struct device_domain_info *info;
2254
2255 list_for_each_entry(info, &device_domain_list, global)
41e80dca 2256 if (info->iommu->segment == segment && info->bus == bus &&
745f2586 2257 info->devfn == devfn)
5a8f40e8 2258 return info;
745f2586
JL		 2259
2260 return NULL;
2261}
2262
5a8f40e8 2263static struct dmar_domain *dmar_insert_dev_info(struct intel_iommu *iommu,
41e80dca 2264 int bus, int devfn,
b718cd3d
DW		 2265 struct device *dev,
2266 struct dmar_domain *domain)
745f2586 2267{
5a8f40e8 2268 struct dmar_domain *found = NULL;
745f2586
JL		 2269 struct device_domain_info *info;
2270 unsigned long flags;
2271
2272 info = alloc_devinfo_mem();
2273 if (!info)
b718cd3d 2274 return NULL;
745f2586 2275
745f2586
JL		 2276 info->bus = bus;
2277 info->devfn = devfn;
2278 info->dev = dev;
2279 info->domain = domain;
5a8f40e8 2280 info->iommu = iommu;
745f2586
JL		 2281
2282 spin_lock_irqsave(&device_domain_lock, flags);
2283 if (dev)
0bcb3e28 2284 found = find_domain(dev);
5a8f40e8
DW		 2285 else {
2286 struct device_domain_info *info2;
41e80dca 2287 info2 = dmar_search_domain_by_dev_info(iommu->segment, bus, devfn);
5a8f40e8
DW		 2288 if (info2)
2289 found = info2->domain;
2290 }
745f2586
JL		 2291 if (found) {
2292 spin_unlock_irqrestore(&device_domain_lock, flags);
2293 free_devinfo_mem(info);
b718cd3d
DW		 2294 /* Caller must free the original domain */
2295 return found;
745f2586
JL		 2296 }
2297
b718cd3d
DW		 2298 list_add(&info->link, &domain->devices);
2299 list_add(&info->global, &device_domain_list);
2300 if (dev)
2301 dev->archdata.iommu = info;
2302 spin_unlock_irqrestore(&device_domain_lock, flags);
2303
2304 return domain;
745f2586
JL		 2305}
2306
579305f7
AW		 2307static int get_last_alias(struct pci_dev *pdev, u16 alias, void *opaque)
2308{
2309 *(u16 *)opaque = alias;
2310 return 0;
2311}
2312
ba395927 2313/* domain is initialized */
146922ec 2314static struct dmar_domain *get_domain_for_dev(struct device *dev, int gaw)
ba395927 2315{
579305f7
AW		 2316 struct dmar_domain *domain, *tmp;
2317 struct intel_iommu *iommu;
5a8f40e8 2318 struct device_domain_info *info;
579305f7 2319 u16 dma_alias;
ba395927 2320 unsigned long flags;
aa4d066a 2321 u8 bus, devfn;
ba395927 2322
146922ec 2323 domain = find_domain(dev);
ba395927
KA		 2324 if (domain)
2325 return domain;
2326
579305f7
AW		 2327 iommu = device_to_iommu(dev, &bus, &devfn);
2328 if (!iommu)
2329 return NULL;
2330
146922ec
DW		 2331 if (dev_is_pci(dev)) {
2332 struct pci_dev *pdev = to_pci_dev(dev);
276dbf99 2333
579305f7
AW		 2334 pci_for_each_dma_alias(pdev, get_last_alias, &dma_alias);
2335
2336 spin_lock_irqsave(&device_domain_lock, flags);
2337 info = dmar_search_domain_by_dev_info(pci_domain_nr(pdev->bus),
2338 PCI_BUS_NUM(dma_alias),
2339 dma_alias & 0xff);
2340 if (info) {
2341 iommu = info->iommu;
2342 domain = info->domain;
5a8f40e8 2343 }
579305f7 2344 spin_unlock_irqrestore(&device_domain_lock, flags);
ba395927 2345
579305f7
AW		 2346 /* DMA alias already has a domain, uses it */
2347 if (info)
2348 goto found_domain;
2349 }
ba395927 2350
146922ec 2351 /* Allocate and initialize new domain for the device */
ab8dfe25 2352 domain = alloc_domain(0);
745f2586 2353 if (!domain)
579305f7 2354 return NULL;
44bde614
JL		 2355 domain->id = iommu_attach_domain(domain, iommu);
2356 if (domain->id < 0) {
2fe9723d 2357 free_domain_mem(domain);
579305f7 2358 return NULL;
2c2e2c38 2359 }
fb170fb4 2360 domain_attach_iommu(domain, iommu);
579305f7
AW		 2361 if (domain_init(domain, gaw)) {
2362 domain_exit(domain);
2363 return NULL;
2c2e2c38 2364 }
ba395927 2365
579305f7
AW		 2366 /* register PCI DMA alias device */
2367 if (dev_is_pci(dev)) {
2368 tmp = dmar_insert_dev_info(iommu, PCI_BUS_NUM(dma_alias),
2369 dma_alias & 0xff, NULL, domain);
2370
2371 if (!tmp || tmp != domain) {
2372 domain_exit(domain);
2373 domain = tmp;
2374 }
2375
b718cd3d 2376 if (!domain)
579305f7 2377 return NULL;
ba395927
KA		 2378 }
2379
2380found_domain:
579305f7
AW		 2381 tmp = dmar_insert_dev_info(iommu, bus, devfn, dev, domain);
2382
2383 if (!tmp || tmp != domain) {
2384 domain_exit(domain);
2385 domain = tmp;
2386 }
b718cd3d
DW		 2387
2388 return domain;
ba395927
KA		 2389}
2390
2c2e2c38 2391static int iommu_identity_mapping;
e0fc7e0b
DW		 2392#define IDENTMAP_ALL 1
2393#define IDENTMAP_GFX 2
2394#define IDENTMAP_AZALIA 4
2c2e2c38 2395
b213203e
DW		 2396static int iommu_domain_identity_map(struct dmar_domain *domain,
2397 unsigned long long start,
2398 unsigned long long end)
ba395927 2399{
c5395d5c
DW		 2400 unsigned long first_vpfn = start >> VTD_PAGE_SHIFT;
2401 unsigned long last_vpfn = end >> VTD_PAGE_SHIFT;
2402
2403 if (!reserve_iova(&domain->iovad, dma_to_mm_pfn(first_vpfn),
2404 dma_to_mm_pfn(last_vpfn))) {
9f10e5bf 2405 pr_err("Reserving iova failed\n");
b213203e 2406 return -ENOMEM;
ba395927
KA		 2407 }
2408
c5395d5c
DW		 2409 pr_debug("Mapping reserved region %llx-%llx for domain %d\n",
2410 start, end, domain->id);
ba395927
KA		 2411 /*
2412 * RMRR range might have overlap with physical memory range,
2413 * clear it first
2414 */
c5395d5c 2415 dma_pte_clear_range(domain, first_vpfn, last_vpfn);
ba395927 2416
c5395d5c
DW		 2417 return domain_pfn_mapping(domain, first_vpfn, first_vpfn,
2418 last_vpfn - first_vpfn + 1,
61df7443 2419 DMA_PTE_READ|DMA_PTE_WRITE);
b213203e
DW		 2420}
2421
0b9d9753 2422static int iommu_prepare_identity_map(struct device *dev,
b213203e
DW		 2423 unsigned long long start,
2424 unsigned long long end)
2425{
2426 struct dmar_domain *domain;
2427 int ret;
2428
0b9d9753 2429 domain = get_domain_for_dev(dev, DEFAULT_DOMAIN_ADDRESS_WIDTH);
b213203e
DW		 2430 if (!domain)
2431 return -ENOMEM;
2432
19943b0e
DW		 2433 /* For _hardware_ passthrough, don't bother. But for software
2434 passthrough, we do it anyway -- it may indicate a memory
2435 range which is reserved in E820, so which didn't get set
2436 up to start with in si_domain */
2437 if (domain == si_domain && hw_pass_through) {
9f10e5bf
JR		 2438 pr_warn("Ignoring identity map for HW passthrough device %s [0x%Lx - 0x%Lx]\n",
2439 dev_name(dev), start, end);
19943b0e
DW		 2440 return 0;
2441 }
2442
9f10e5bf
JR		 2443 pr_info("Setting identity map for device %s [0x%Lx - 0x%Lx]\n",
2444 dev_name(dev), start, end);
2445
5595b528
DW		 2446 if (end < start) {
2447 WARN(1, "Your BIOS is broken; RMRR ends before it starts!\n"
2448 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
2449 dmi_get_system_info(DMI_BIOS_VENDOR),
2450 dmi_get_system_info(DMI_BIOS_VERSION),
2451 dmi_get_system_info(DMI_PRODUCT_VERSION));
2452 ret = -EIO;
2453 goto error;
2454 }
2455
2ff729f5
DW		 2456 if (end >> agaw_to_width(domain->agaw)) {
2457 WARN(1, "Your BIOS is broken; RMRR exceeds permitted address width (%d bits)\n"
2458 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
2459 agaw_to_width(domain->agaw),
2460 dmi_get_system_info(DMI_BIOS_VENDOR),
2461 dmi_get_system_info(DMI_BIOS_VERSION),
2462 dmi_get_system_info(DMI_PRODUCT_VERSION));
2463 ret = -EIO;
2464 goto error;
2465 }
19943b0e 2466
b213203e 2467 ret = iommu_domain_identity_map(domain, start, end);
ba395927
KA		 2468 if (ret)
2469 goto error;
2470
2471 /* context entry init */
0b9d9753 2472 ret = domain_context_mapping(domain, dev, CONTEXT_TT_MULTI_LEVEL);
b213203e
DW		 2473 if (ret)
2474 goto error;
2475
2476 return 0;
2477
2478 error:
ba395927
KA		 2479 domain_exit(domain);
2480 return ret;
ba395927
KA		 2481}
2482
2483static inline int iommu_prepare_rmrr_dev(struct dmar_rmrr_unit *rmrr,
0b9d9753 2484 struct device *dev)
ba395927 2485{
0b9d9753 2486 if (dev->archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO)
ba395927 2487 return 0;
0b9d9753
DW		 2488 return iommu_prepare_identity_map(dev, rmrr->base_address,
2489 rmrr->end_address);
ba395927
KA		 2490}
2491
d3f13810 2492#ifdef CONFIG_INTEL_IOMMU_FLOPPY_WA
49a0429e
KA		 2493static inline void iommu_prepare_isa(void)
2494{
2495 struct pci_dev *pdev;
2496 int ret;
2497
2498 pdev = pci_get_class(PCI_CLASS_BRIDGE_ISA << 8, NULL);
2499 if (!pdev)
2500 return;
2501
9f10e5bf 2502 pr_info("Prepare 0-16MiB unity mapping for LPC\n");
0b9d9753 2503 ret = iommu_prepare_identity_map(&pdev->dev, 0, 16*1024*1024 - 1);
49a0429e
KA		 2504
2505 if (ret)
9f10e5bf 2506 pr_err("Failed to create 0-16MiB identity map - floppy might not work\n");
49a0429e 2507
9b27e82d 2508 pci_dev_put(pdev);
49a0429e
KA		 2509}
2510#else
2511static inline void iommu_prepare_isa(void)
2512{
2513 return;
2514}
d3f13810 2515#endif /* !CONFIG_INTEL_IOMMU_FLPY_WA */
49a0429e 2516
2c2e2c38 2517static int md_domain_init(struct dmar_domain *domain, int guest_width);
c7ab48d2 2518
071e1374 2519static int __init si_domain_init(int hw)
2c2e2c38
FY		 2520{
2521 struct dmar_drhd_unit *drhd;
2522 struct intel_iommu *iommu;
c7ab48d2 2523 int nid, ret = 0;
44bde614 2524 bool first = true;
2c2e2c38 2525
ab8dfe25 2526 si_domain = alloc_domain(DOMAIN_FLAG_STATIC_IDENTITY);
2c2e2c38
FY		 2527 if (!si_domain)
2528 return -EFAULT;
2529
2c2e2c38
FY		 2530 for_each_active_iommu(iommu, drhd) {
2531 ret = iommu_attach_domain(si_domain, iommu);
fb170fb4 2532 if (ret < 0) {
2c2e2c38
FY		 2533 domain_exit(si_domain);
2534 return -EFAULT;
44bde614
JL		 2535 } else if (first) {
2536 si_domain->id = ret;
2537 first = false;
2538 } else if (si_domain->id != ret) {
2539 domain_exit(si_domain);
2540 return -EFAULT;
2c2e2c38 2541 }
fb170fb4 2542 domain_attach_iommu(si_domain, iommu);
2c2e2c38
FY		 2543 }
2544
2545 if (md_domain_init(si_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
2546 domain_exit(si_domain);
2547 return -EFAULT;
2548 }
2549
9f10e5bf 2550 pr_debug("Identity mapping domain is domain %d\n",
9544c003 2551 si_domain->id);
2c2e2c38 2552
19943b0e
DW		 2553 if (hw)
2554 return 0;
2555
c7ab48d2 2556 for_each_online_node(nid) {
5dfe8660
TH		 2557 unsigned long start_pfn, end_pfn;
2558 int i;
2559
2560 for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, NULL) {
2561 ret = iommu_domain_identity_map(si_domain,
2562 PFN_PHYS(start_pfn), PFN_PHYS(end_pfn));
2563 if (ret)
2564 return ret;
2565 }
c7ab48d2
DW		 2566 }
2567
2c2e2c38
FY		 2568 return 0;
2569}
2570
9b226624 2571static int identity_mapping(struct device *dev)
2c2e2c38
FY		 2572{
2573 struct device_domain_info *info;
2574
2575 if (likely(!iommu_identity_mapping))
2576 return 0;
2577
9b226624 2578 info = dev->archdata.iommu;
cb452a40
MT		 2579 if (info && info != DUMMY_DEVICE_DOMAIN_INFO)
2580 return (info->domain == si_domain);
2c2e2c38 2581
2c2e2c38
FY		 2582 return 0;
2583}
2584
2585static int domain_add_dev_info(struct dmar_domain *domain,
5913c9bf 2586 struct device *dev, int translation)
2c2e2c38 2587{
0ac72664 2588 struct dmar_domain *ndomain;
5a8f40e8 2589 struct intel_iommu *iommu;
156baca8 2590 u8 bus, devfn;
5fe60f4e 2591 int ret;
2c2e2c38 2592
5913c9bf 2593 iommu = device_to_iommu(dev, &bus, &devfn);
5a8f40e8
DW		 2594 if (!iommu)
2595 return -ENODEV;
2596
5913c9bf 2597 ndomain = dmar_insert_dev_info(iommu, bus, devfn, dev, domain);
0ac72664
DW		 2598 if (ndomain != domain)
2599 return -EBUSY;
2c2e2c38 2600
5913c9bf 2601 ret = domain_context_mapping(domain, dev, translation);
e2ad23d0 2602 if (ret) {
5913c9bf 2603 domain_remove_one_dev_info(domain, dev);
e2ad23d0
DW		 2604 return ret;
2605 }
2606
2c2e2c38
FY		 2607 return 0;
2608}
2609
0b9d9753 2610static bool device_has_rmrr(struct device *dev)
ea2447f7
TM		 2611{
2612 struct dmar_rmrr_unit *rmrr;
832bd858 2613 struct device *tmp;
ea2447f7
TM		 2614 int i;
2615
0e242612 2616 rcu_read_lock();
ea2447f7 2617 for_each_rmrr_units(rmrr) {
b683b230
JL		 2618 /*
2619 * Return TRUE if this RMRR contains the device that
2620 * is passed in.
2621 */
2622 for_each_active_dev_scope(rmrr->devices,
2623 rmrr->devices_cnt, i, tmp)
0b9d9753 2624 if (tmp == dev) {
0e242612 2625 rcu_read_unlock();
ea2447f7 2626 return true;
b683b230 2627 }
ea2447f7 2628 }
0e242612 2629 rcu_read_unlock();
ea2447f7
TM		 2630 return false;
2631}
2632
c875d2c1
AW		 2633/*
2634 * There are a couple cases where we need to restrict the functionality of
2635 * devices associated with RMRRs. The first is when evaluating a device for
2636 * identity mapping because problems exist when devices are moved in and out
2637 * of domains and their respective RMRR information is lost. This means that
2638 * a device with associated RMRRs will never be in a "passthrough" domain.
2639 * The second is use of the device through the IOMMU API. This interface
2640 * expects to have full control of the IOVA space for the device. We cannot
2641 * satisfy both the requirement that RMRR access is maintained and have an
2642 * unencumbered IOVA space. We also have no ability to quiesce the device's
2643 * use of the RMRR space or even inform the IOMMU API user of the restriction.
2644 * We therefore prevent devices associated with an RMRR from participating in
2645 * the IOMMU API, which eliminates them from device assignment.
2646 *
2647 * In both cases we assume that PCI USB devices with RMRRs have them largely
2648 * for historical reasons and that the RMRR space is not actively used post
2649 * boot. This exclusion may change if vendors begin to abuse it.
18436afd
DW		 2650 *
2651 * The same exception is made for graphics devices, with the requirement that
2652 * any use of the RMRR regions will be torn down before assigning the device
2653 * to a guest.
c875d2c1
AW		 2654 */
2655static bool device_is_rmrr_locked(struct device *dev)
2656{
2657 if (!device_has_rmrr(dev))
2658 return false;
2659
2660 if (dev_is_pci(dev)) {
2661 struct pci_dev *pdev = to_pci_dev(dev);
2662
18436afd 2663 if (IS_USB_DEVICE(pdev) || IS_GFX_DEVICE(pdev))
c875d2c1
AW		 2664 return false;
2665 }
2666
2667 return true;
2668}
2669
3bdb2591 2670static int iommu_should_identity_map(struct device *dev, int startup)
6941af28 2671{
ea2447f7 2672
3bdb2591
DW		 2673 if (dev_is_pci(dev)) {
2674 struct pci_dev *pdev = to_pci_dev(dev);
ea2447f7 2675
c875d2c1 2676 if (device_is_rmrr_locked(dev))
3bdb2591 2677 return 0;
e0fc7e0b 2678
3bdb2591
DW		 2679 if ((iommu_identity_mapping & IDENTMAP_AZALIA) && IS_AZALIA(pdev))
2680 return 1;
e0fc7e0b 2681
3bdb2591
DW		 2682 if ((iommu_identity_mapping & IDENTMAP_GFX) && IS_GFX_DEVICE(pdev))
2683 return 1;
6941af28 2684
3bdb2591 2685 if (!(iommu_identity_mapping & IDENTMAP_ALL))
3dfc813d 2686 return 0;
3bdb2591
DW		 2687
2688 /*
2689 * We want to start off with all devices in the 1:1 domain, and
2690 * take them out later if we find they can't access all of memory.
2691 *
2692 * However, we can't do this for PCI devices behind bridges,
2693 * because all PCI devices behind the same bridge will end up
2694 * with the same source-id on their transactions.
2695 *
2696 * Practically speaking, we can't change things around for these
2697 * devices at run-time, because we can't be sure there'll be no
2698 * DMA transactions in flight for any of their siblings.
2699 *
2700 * So PCI devices (unless they're on the root bus) as well as
2701 * their parent PCI-PCI or PCIe-PCI bridges must be left _out_ of
2702 * the 1:1 domain, just in _case_ one of their siblings turns out
2703 * not to be able to map all of memory.
2704 */
2705 if (!pci_is_pcie(pdev)) {
2706 if (!pci_is_root_bus(pdev->bus))
2707 return 0;
2708 if (pdev->class >> 8 == PCI_CLASS_BRIDGE_PCI)
2709 return 0;
2710 } else if (pci_pcie_type(pdev) == PCI_EXP_TYPE_PCI_BRIDGE)
3dfc813d 2711 return 0;
3bdb2591
DW		 2712 } else {
2713 if (device_has_rmrr(dev))
2714 return 0;
2715 }
3dfc813d 2716
3bdb2591 2717 /*
3dfc813d 2718 * At boot time, we don't yet know if devices will be 64-bit capable.
3bdb2591 2719 * Assume that they will — if they turn out not to be, then we can
3dfc813d
DW		 2720 * take them out of the 1:1 domain later.
2721 */
8fcc5372
CW		 2722 if (!startup) {
2723 /*
2724 * If the device's dma_mask is less than the system's memory
2725 * size then this is not a candidate for identity mapping.
2726 */
3bdb2591 2727 u64 dma_mask = *dev->dma_mask;
8fcc5372 2728
3bdb2591
DW		 2729 if (dev->coherent_dma_mask &&
2730 dev->coherent_dma_mask < dma_mask)
2731 dma_mask = dev->coherent_dma_mask;
8fcc5372 2732
3bdb2591 2733 return dma_mask >= dma_get_required_mask(dev);
8fcc5372 2734 }
6941af28
DW		 2735
2736 return 1;
2737}
2738
cf04eee8
DW		 2739static int __init dev_prepare_static_identity_mapping(struct device *dev, int hw)
2740{
2741 int ret;
2742
2743 if (!iommu_should_identity_map(dev, 1))
2744 return 0;
2745
2746 ret = domain_add_dev_info(si_domain, dev,
2747 hw ? CONTEXT_TT_PASS_THROUGH :
2748 CONTEXT_TT_MULTI_LEVEL);
2749 if (!ret)
9f10e5bf
JR		 2750 pr_info("%s identity mapping for device %s\n",
2751 hw ? "Hardware" : "Software", dev_name(dev));
cf04eee8
DW		 2752 else if (ret == -ENODEV)
2753 /* device not associated with an iommu */
2754 ret = 0;
2755
2756 return ret;
2757}
2758
2759
071e1374 2760static int __init iommu_prepare_static_identity_mapping(int hw)
2c2e2c38 2761{
2c2e2c38 2762 struct pci_dev *pdev = NULL;
cf04eee8
DW		 2763 struct dmar_drhd_unit *drhd;
2764 struct intel_iommu *iommu;
2765 struct device *dev;
2766 int i;
2767 int ret = 0;
2c2e2c38 2768
2c2e2c38 2769 for_each_pci_dev(pdev) {
cf04eee8
DW		 2770 ret = dev_prepare_static_identity_mapping(&pdev->dev, hw);
2771 if (ret)
2772 return ret;
2773 }
2774
2775 for_each_active_iommu(iommu, drhd)
2776 for_each_active_dev_scope(drhd->devices, drhd->devices_cnt, i, dev) {
2777 struct acpi_device_physical_node *pn;
2778 struct acpi_device *adev;
2779
2780 if (dev->bus != &acpi_bus_type)
2781 continue;
86080ccc 2782
cf04eee8
DW		 2783 adev= to_acpi_device(dev);
2784 mutex_lock(&adev->physical_node_lock);
2785 list_for_each_entry(pn, &adev->physical_node_list, node) {
2786 ret = dev_prepare_static_identity_mapping(pn->dev, hw);
2787 if (ret)
2788 break;
eae460b6 2789 }
cf04eee8
DW		 2790 mutex_unlock(&adev->physical_node_lock);
2791 if (ret)
2792 return ret;
62edf5dc 2793 }
2c2e2c38
FY		 2794
2795 return 0;
2796}
2797
ffebeb46
JL		 2798static void intel_iommu_init_qi(struct intel_iommu *iommu)
2799{
2800 /*
2801 * Start from the sane iommu hardware state.
2802 * If the queued invalidation is already initialized by us
2803 * (for example, while enabling interrupt-remapping) then
2804 * we got the things already rolling from a sane state.
2805 */
2806 if (!iommu->qi) {
2807 /*
2808 * Clear any previous faults.
2809 */
2810 dmar_fault(-1, iommu);
2811 /*
2812 * Disable queued invalidation if supported and already enabled
2813 * before OS handover.
2814 */
2815 dmar_disable_qi(iommu);
2816 }
2817
2818 if (dmar_enable_qi(iommu)) {
2819 /*
2820 * Queued Invalidate not enabled, use Register Based Invalidate
2821 */
2822 iommu->flush.flush_context = __iommu_flush_context;
2823 iommu->flush.flush_iotlb = __iommu_flush_iotlb;
9f10e5bf 2824 pr_info("%s: Using Register based invalidation\n",
ffebeb46
JL		 2825 iommu->name);
2826 } else {
2827 iommu->flush.flush_context = qi_flush_context;
2828 iommu->flush.flush_iotlb = qi_flush_iotlb;
9f10e5bf 2829 pr_info("%s: Using Queued invalidation\n", iommu->name);
ffebeb46
JL		 2830 }
2831}
2832
091d42e4
JR		 2833static int copy_context_table(struct intel_iommu *iommu,
2834 struct root_entry *old_re,
2835 struct context_entry **tbl,
2836 int bus, bool ext)
2837{
2838 struct context_entry *old_ce = NULL, *new_ce = NULL, ce;
dbcd861f 2839 int tbl_idx, pos = 0, idx, devfn, ret = 0, did;
091d42e4
JR		 2840 phys_addr_t old_ce_phys;
2841
2842 tbl_idx = ext ? bus * 2 : bus;
2843
2844 for (devfn = 0; devfn < 256; devfn++) {
2845 /* First calculate the correct index */
2846 idx = (ext ? devfn * 2 : devfn) % 256;
2847
2848 if (idx == 0) {
2849 /* First save what we may have and clean up */
2850 if (new_ce) {
2851 tbl[tbl_idx] = new_ce;
2852 __iommu_flush_cache(iommu, new_ce,
2853 VTD_PAGE_SIZE);
2854 pos = 1;
2855 }
2856
2857 if (old_ce)
2858 iounmap(old_ce);
2859
2860 ret = 0;
2861 if (devfn < 0x80)
2862 old_ce_phys = root_entry_lctp(old_re);
2863 else
2864 old_ce_phys = root_entry_uctp(old_re);
2865
2866 if (!old_ce_phys) {
2867 if (ext && devfn == 0) {
2868 /* No LCTP, try UCTP */
2869 devfn = 0x7f;
2870 continue;
2871 } else {
2872 goto out;
2873 }
2874 }
2875
2876 ret = -ENOMEM;
2877 old_ce = ioremap_cache(old_ce_phys, PAGE_SIZE);
2878 if (!old_ce)
2879 goto out;
2880
2881 new_ce = alloc_pgtable_page(iommu->node);
2882 if (!new_ce)
2883 goto out_unmap;
2884
2885 ret = 0;
2886 }
2887
2888 /* Now copy the context entry */
2889 ce = old_ce[idx];
2890
cf484d0e 2891 if (!__context_present(&ce))
091d42e4
JR		 2892 continue;
2893
dbcd861f
JR		 2894 did = context_domain_id(&ce);
2895 if (did >= 0 && did < cap_ndoms(iommu->cap))
2896 set_bit(did, iommu->domain_ids);
2897
cf484d0e
JR		 2898 /*
2899 * We need a marker for copied context entries. This
2900 * marker needs to work for the old format as well as
2901 * for extended context entries.
2902 *
2903 * Bit 67 of the context entry is used. In the old
2904 * format this bit is available to software, in the
2905 * extended format it is the PGE bit, but PGE is ignored
2906 * by HW if PASIDs are disabled (and thus still
2907 * available).
2908 *
2909 * So disable PASIDs first and then mark the entry
2910 * copied. This means that we don't copy PASID
2911 * translations from the old kernel, but this is fine as
2912 * faults there are not fatal.
2913 */
2914 context_clear_pasid_enable(&ce);
2915 context_set_copied(&ce);
2916
091d42e4
JR		 2917 new_ce[idx] = ce;
2918 }
2919
2920 tbl[tbl_idx + pos] = new_ce;
2921
2922 __iommu_flush_cache(iommu, new_ce, VTD_PAGE_SIZE);
2923
2924out_unmap:
2925 iounmap(old_ce);
2926
2927out:
2928 return ret;
2929}
2930
2931static int copy_translation_tables(struct intel_iommu *iommu)
2932{
2933 struct context_entry **ctxt_tbls;
2934 struct root_entry *old_rt;
2935 phys_addr_t old_rt_phys;
2936 int ctxt_table_entries;
2937 unsigned long flags;
2938 u64 rtaddr_reg;
2939 int bus, ret;
c3361f2f 2940 bool new_ext, ext;
091d42e4
JR		 2941
2942 rtaddr_reg = dmar_readq(iommu->reg + DMAR_RTADDR_REG);
2943 ext = !!(rtaddr_reg & DMA_RTADDR_RTT);
c3361f2f
JR		 2944 new_ext = !!ecap_ecs(iommu->ecap);
2945
2946 /*
2947 * The RTT bit can only be changed when translation is disabled,
2948 * but disabling translation means to open a window for data
2949 * corruption. So bail out and don't copy anything if we would
2950 * have to change the bit.
2951 */
2952 if (new_ext != ext)
2953 return -EINVAL;
091d42e4
JR		 2954
2955 old_rt_phys = rtaddr_reg & VTD_PAGE_MASK;
2956 if (!old_rt_phys)
2957 return -EINVAL;
2958
2959 old_rt = ioremap_cache(old_rt_phys, PAGE_SIZE);
2960 if (!old_rt)
2961 return -ENOMEM;
2962
2963 /* This is too big for the stack - allocate it from slab */
2964 ctxt_table_entries = ext ? 512 : 256;
2965 ret = -ENOMEM;
2966 ctxt_tbls = kzalloc(ctxt_table_entries * sizeof(void *), GFP_KERNEL);
2967 if (!ctxt_tbls)
2968 goto out_unmap;
2969
2970 for (bus = 0; bus < 256; bus++) {
2971 ret = copy_context_table(iommu, &old_rt[bus],
2972 ctxt_tbls, bus, ext);
2973 if (ret) {
2974 pr_err("%s: Failed to copy context table for bus %d\n",
2975 iommu->name, bus);
2976 continue;
2977 }
2978 }
2979
2980 spin_lock_irqsave(&iommu->lock, flags);
2981
2982 /* Context tables are copied, now write them to the root_entry table */
2983 for (bus = 0; bus < 256; bus++) {
2984 int idx = ext ? bus * 2 : bus;
2985 u64 val;
2986
2987 if (ctxt_tbls[idx]) {
2988 val = virt_to_phys(ctxt_tbls[idx]) | 1;
2989 iommu->root_entry[bus].lo = val;
2990 }
2991
2992 if (!ext || !ctxt_tbls[idx + 1])
2993 continue;
2994
2995 val = virt_to_phys(ctxt_tbls[idx + 1]) | 1;
2996 iommu->root_entry[bus].hi = val;
2997 }
2998
2999 spin_unlock_irqrestore(&iommu->lock, flags);
3000
3001 kfree(ctxt_tbls);
3002
3003 __iommu_flush_cache(iommu, iommu->root_entry, PAGE_SIZE);
3004
3005 ret = 0;
3006
3007out_unmap:
3008 iounmap(old_rt);
3009
3010 return ret;
3011}
3012
b779260b 3013static int __init init_dmars(void)
ba395927
KA		 3014{
3015 struct dmar_drhd_unit *drhd;
3016 struct dmar_rmrr_unit *rmrr;
a87f4918 3017 bool copied_tables = false;
832bd858 3018 struct device *dev;
ba395927 3019 struct intel_iommu *iommu;
9d783ba0 3020 int i, ret;
2c2e2c38 3021
ba395927
KA		 3022 /*
3023 * for each drhd
3024 * allocate root
3025 * initialize and program root entry to not present
3026 * endfor
3027 */
3028 for_each_drhd_unit(drhd) {
5e0d2a6f 3029 /*
3030 * lock not needed as this is only incremented in the single
3031 * threaded kernel __init code path all other access are read
3032 * only
3033 */
78d8e704 3034 if (g_num_of_iommus < DMAR_UNITS_SUPPORTED) {
1b198bb0
MT		 3035 g_num_of_iommus++;
3036 continue;
3037 }
9f10e5bf 3038 pr_err_once("Exceeded %d IOMMUs\n", DMAR_UNITS_SUPPORTED);
5e0d2a6f 3039 }
3040
ffebeb46
JL		 3041 /* Preallocate enough resources for IOMMU hot-addition */
3042 if (g_num_of_iommus < DMAR_UNITS_SUPPORTED)
3043 g_num_of_iommus = DMAR_UNITS_SUPPORTED;
3044
d9630fe9
WH		 3045 g_iommus = kcalloc(g_num_of_iommus, sizeof(struct intel_iommu *),
3046 GFP_KERNEL);
3047 if (!g_iommus) {
9f10e5bf 3048 pr_err("Allocating global iommu array failed\n");
d9630fe9
WH		 3049 ret = -ENOMEM;
3050 goto error;
3051 }
3052
80b20dd8 3053 deferred_flush = kzalloc(g_num_of_iommus *
3054 sizeof(struct deferred_flush_tables), GFP_KERNEL);
3055 if (!deferred_flush) {
5e0d2a6f 3056 ret = -ENOMEM;
989d51fc 3057 goto free_g_iommus;
5e0d2a6f 3058 }
3059
7c919779 3060 for_each_active_iommu(iommu, drhd) {
d9630fe9 3061 g_iommus[iommu->seq_id] = iommu;
ba395927 3062
b63d80d1
JR		 3063 intel_iommu_init_qi(iommu);
3064
e61d98d8
SS		 3065 ret = iommu_init_domains(iommu);
3066 if (ret)
989d51fc 3067 goto free_iommu;
e61d98d8 3068
4158c2ec
JR		 3069 init_translation_status(iommu);
3070
091d42e4
JR		 3071 if (translation_pre_enabled(iommu) && !is_kdump_kernel()) {
3072 iommu_disable_translation(iommu);
3073 clear_translation_pre_enabled(iommu);
3074 pr_warn("Translation was enabled for %s but we are not in kdump mode\n",
3075 iommu->name);
3076 }
4158c2ec 3077
ba395927
KA		 3078 /*
3079 * TBD:
3080 * we could share the same root & context tables
25985edc 3081 * among all IOMMU's. Need to Split it later.
ba395927
KA		 3082 */
3083 ret = iommu_alloc_root_entry(iommu);
ffebeb46 3084 if (ret)
989d51fc 3085 goto free_iommu;
5f0a7f76 3086
091d42e4
JR		 3087 if (translation_pre_enabled(iommu)) {
3088 pr_info("Translation already enabled - trying to copy translation structures\n");
3089
3090 ret = copy_translation_tables(iommu);
3091 if (ret) {
3092 /*
3093 * We found the IOMMU with translation
3094 * enabled - but failed to copy over the
3095 * old root-entry table. Try to proceed
3096 * by disabling translation now and
3097 * allocating a clean root-entry table.
3098 * This might cause DMAR faults, but
3099 * probably the dump will still succeed.
3100 */
3101 pr_err("Failed to copy translation tables from previous kernel for %s\n",
3102 iommu->name);
3103 iommu_disable_translation(iommu);
3104 clear_translation_pre_enabled(iommu);
3105 } else {
3106 pr_info("Copied translation tables from previous kernel for %s\n",
3107 iommu->name);
a87f4918 3108 copied_tables = true;
091d42e4
JR		 3109 }
3110 }
3111
5f0a7f76
JR		 3112 iommu_flush_write_buffer(iommu);
3113 iommu_set_root_entry(iommu);
3114 iommu->flush.flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL);
3115 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
3116
4ed0d3e6 3117 if (!ecap_pass_through(iommu->ecap))
19943b0e 3118 hw_pass_through = 0;
ba395927
KA		 3119 }
3120
19943b0e 3121 if (iommu_pass_through)
e0fc7e0b
DW		 3122 iommu_identity_mapping |= IDENTMAP_ALL;
3123
d3f13810 3124#ifdef CONFIG_INTEL_IOMMU_BROKEN_GFX_WA
e0fc7e0b 3125 iommu_identity_mapping |= IDENTMAP_GFX;
19943b0e 3126#endif
e0fc7e0b 3127
86080ccc
JR		 3128 if (iommu_identity_mapping) {
3129 ret = si_domain_init(hw_pass_through);
3130 if (ret)
3131 goto free_iommu;
3132 }
3133
e0fc7e0b
DW		 3134 check_tylersburg_isoch();
3135
a87f4918
JR		 3136 /*
3137 * If we copied translations from a previous kernel in the kdump
3138 * case, we can not assign the devices to domains now, as that
3139 * would eliminate the old mappings. So skip this part and defer
3140 * the assignment to device driver initialization time.
3141 */
3142 if (copied_tables)
3143 goto domains_done;
3144
ba395927 3145 /*
19943b0e
DW		 3146 * If pass through is not set or not enabled, setup context entries for
3147 * identity mappings for rmrr, gfx, and isa and may fall back to static
3148 * identity mapping if iommu_identity_mapping is set.
ba395927 3149 */
19943b0e
DW		 3150 if (iommu_identity_mapping) {
3151 ret = iommu_prepare_static_identity_mapping(hw_pass_through);
4ed0d3e6 3152 if (ret) {
9f10e5bf 3153 pr_crit("Failed to setup IOMMU pass-through\n");
989d51fc 3154 goto free_iommu;
ba395927
KA		 3155 }
3156 }
ba395927 3157 /*
19943b0e
DW		 3158 * For each rmrr
3159 * for each dev attached to rmrr
3160 * do
3161 * locate drhd for dev, alloc domain for dev
3162 * allocate free domain
3163 * allocate page table entries for rmrr
3164 * if context not allocated for bus
3165 * allocate and init context
3166 * set present in root table for this bus
3167 * init context with domain, translation etc
3168 * endfor
3169 * endfor
ba395927 3170 */
9f10e5bf 3171 pr_info("Setting RMRR:\n");
19943b0e 3172 for_each_rmrr_units(rmrr) {
b683b230
JL		 3173 /* some BIOS lists non-exist devices in DMAR table. */
3174 for_each_active_dev_scope(rmrr->devices, rmrr->devices_cnt,
832bd858 3175 i, dev) {
0b9d9753 3176 ret = iommu_prepare_rmrr_dev(rmrr, dev);
19943b0e 3177 if (ret)
9f10e5bf 3178 pr_err("Mapping reserved region failed\n");
ba395927 3179 }
4ed0d3e6 3180 }
49a0429e 3181
19943b0e
DW		 3182 iommu_prepare_isa();
3183
a87f4918
JR		 3184domains_done:
3185
ba395927
KA		 3186 /*
3187 * for each drhd
3188 * enable fault log
3189 * global invalidate context cache
3190 * global invalidate iotlb
3191 * enable translation
3192 */
7c919779 3193 for_each_iommu(iommu, drhd) {
51a63e67
JC		 3194 if (drhd->ignored) {
3195 /*
3196 * we always have to disable PMRs or DMA may fail on
3197 * this device
3198 */
3199 if (force_on)
7c919779 3200 iommu_disable_protect_mem_regions(iommu);
ba395927 3201 continue;
51a63e67 3202 }
ba395927
KA		 3203
3204 iommu_flush_write_buffer(iommu);
3205
3460a6d9
KA		 3206 ret = dmar_set_interrupt(iommu);
3207 if (ret)
989d51fc 3208 goto free_iommu;
3460a6d9 3209
8939ddf6
JR		 3210 if (!translation_pre_enabled(iommu))
3211 iommu_enable_translation(iommu);
3212
b94996c9 3213 iommu_disable_protect_mem_regions(iommu);
ba395927
KA		 3214 }
3215
3216 return 0;
989d51fc
JL		 3217
3218free_iommu:
ffebeb46
JL		 3219 for_each_active_iommu(iommu, drhd) {
3220 disable_dmar_iommu(iommu);
a868e6b7 3221 free_dmar_iommu(iommu);
ffebeb46 3222 }
9bdc531e 3223 kfree(deferred_flush);
989d51fc 3224free_g_iommus:
d9630fe9 3225 kfree(g_iommus);
989d51fc 3226error:
ba395927
KA		 3227 return ret;
3228}
3229
5a5e02a6 3230/* This takes a number of _MM_ pages, not VTD pages */
875764de
DW		 3231static struct iova *intel_alloc_iova(struct device *dev,
3232 struct dmar_domain *domain,
3233 unsigned long nrpages, uint64_t dma_mask)
ba395927 3234{
ba395927 3235 struct iova *iova = NULL;
ba395927 3236
875764de
DW		 3237 /* Restrict dma_mask to the width that the iommu can handle */
3238 dma_mask = min_t(uint64_t, DOMAIN_MAX_ADDR(domain->gaw), dma_mask);
3239
3240 if (!dmar_forcedac && dma_mask > DMA_BIT_MASK(32)) {
ba395927
KA		 3241 /*
3242 * First try to allocate an io virtual address in
284901a9 3243 * DMA_BIT_MASK(32) and if that fails then try allocating
3609801e 3244 * from higher range
ba395927 3245 */
875764de
DW		 3246 iova = alloc_iova(&domain->iovad, nrpages,
3247 IOVA_PFN(DMA_BIT_MASK(32)), 1);
3248 if (iova)
3249 return iova;
3250 }
3251 iova = alloc_iova(&domain->iovad, nrpages, IOVA_PFN(dma_mask), 1);
3252 if (unlikely(!iova)) {
9f10e5bf 3253 pr_err("Allocating %ld-page iova for %s failed",
207e3592 3254 nrpages, dev_name(dev));
f76aec76
KA		 3255 return NULL;
3256 }
3257
3258 return iova;
3259}
3260
d4b709f4 3261static struct dmar_domain *__get_valid_domain_for_dev(struct device *dev)
f76aec76
KA		 3262{
3263 struct dmar_domain *domain;
3264 int ret;
3265
d4b709f4 3266 domain = get_domain_for_dev(dev, DEFAULT_DOMAIN_ADDRESS_WIDTH);
f76aec76 3267 if (!domain) {
9f10e5bf 3268 pr_err("Allocating domain for %s failed\n",
d4b709f4 3269 dev_name(dev));
4fe05bbc 3270 return NULL;
ba395927
KA		 3271 }
3272
3273 /* make sure context mapping is ok */
d4b709f4
DW		 3274 if (unlikely(!domain_context_mapped(dev))) {
3275 ret = domain_context_mapping(domain, dev, CONTEXT_TT_MULTI_LEVEL);
f76aec76 3276 if (ret) {
9f10e5bf 3277 pr_err("Domain context map for %s failed\n",
d4b709f4 3278 dev_name(dev));
4fe05bbc 3279 return NULL;
f76aec76 3280 }
ba395927
KA		 3281 }
3282
f76aec76
KA		 3283 return domain;
3284}
3285
d4b709f4 3286static inline struct dmar_domain *get_valid_domain_for_dev(struct device *dev)
147202aa
DW		 3287{
3288 struct device_domain_info *info;
3289
3290 /* No lock here, assumes no domain exit in normal case */
d4b709f4 3291 info = dev->archdata.iommu;
147202aa
DW		 3292 if (likely(info))
3293 return info->domain;
3294
3295 return __get_valid_domain_for_dev(dev);
3296}
3297
ecb509ec 3298/* Check if the dev needs to go through non-identity map and unmap process.*/
73676832 3299static int iommu_no_mapping(struct device *dev)
2c2e2c38
FY		 3300{
3301 int found;
3302
3d89194a 3303 if (iommu_dummy(dev))
1e4c64c4
DW		 3304 return 1;
3305
2c2e2c38 3306 if (!iommu_identity_mapping)
1e4c64c4 3307 return 0;
2c2e2c38 3308
9b226624 3309 found = identity_mapping(dev);
2c2e2c38 3310 if (found) {
ecb509ec 3311 if (iommu_should_identity_map(dev, 0))
2c2e2c38
FY		 3312 return 1;
3313 else {
3314 /*
3315 * 32 bit DMA is removed from si_domain and fall back
3316 * to non-identity mapping.
3317 */
bf9c9eda 3318 domain_remove_one_dev_info(si_domain, dev);
9f10e5bf
JR		 3319 pr_info("32bit %s uses non-identity mapping\n",
3320 dev_name(dev));
2c2e2c38
FY		 3321 return 0;
3322 }
3323 } else {
3324 /*
3325 * In case of a detached 64 bit DMA device from vm, the device
3326 * is put into si_domain for identity mapping.
3327 */
ecb509ec 3328 if (iommu_should_identity_map(dev, 0)) {
2c2e2c38 3329 int ret;
5913c9bf 3330 ret = domain_add_dev_info(si_domain, dev,
5fe60f4e
DW		 3331 hw_pass_through ?
3332 CONTEXT_TT_PASS_THROUGH :
3333 CONTEXT_TT_MULTI_LEVEL);
2c2e2c38 3334 if (!ret) {
9f10e5bf
JR		 3335 pr_info("64bit %s uses identity mapping\n",
3336 dev_name(dev));
2c2e2c38
FY		 3337 return 1;
3338 }
3339 }
3340 }
3341
1e4c64c4 3342 return 0;
2c2e2c38
FY		 3343}
3344
5040a918 3345static dma_addr_t __intel_map_single(struct device *dev, phys_addr_t paddr,
bb9e6d65 3346 size_t size, int dir, u64 dma_mask)
f76aec76 3347{
f76aec76 3348 struct dmar_domain *domain;
5b6985ce 3349 phys_addr_t start_paddr;
f76aec76
KA		 3350 struct iova *iova;
3351 int prot = 0;
6865f0d1 3352 int ret;
8c11e798 3353 struct intel_iommu *iommu;
33041ec0 3354 unsigned long paddr_pfn = paddr >> PAGE_SHIFT;
f76aec76
KA		 3355
3356 BUG_ON(dir == DMA_NONE);
2c2e2c38 3357
5040a918 3358 if (iommu_no_mapping(dev))
6865f0d1 3359 return paddr;
f76aec76 3360
5040a918 3361 domain = get_valid_domain_for_dev(dev);
f76aec76
KA		 3362 if (!domain)
3363 return 0;
3364
8c11e798 3365 iommu = domain_get_iommu(domain);
88cb6a74 3366 size = aligned_nrpages(paddr, size);
f76aec76 3367
5040a918 3368 iova = intel_alloc_iova(dev, domain, dma_to_mm_pfn(size), dma_mask);
f76aec76
KA		 3369 if (!iova)
3370 goto error;
3371
ba395927
KA		 3372 /*
3373 * Check if DMAR supports zero-length reads on write only
3374 * mappings..
3375 */
3376 if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
8c11e798 3377 !cap_zlr(iommu->cap))
ba395927
KA		 3378 prot |= DMA_PTE_READ;
3379 if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
3380 prot |= DMA_PTE_WRITE;
3381 /*
6865f0d1 3382 * paddr - (paddr + size) might be partial page, we should map the whole
ba395927 3383 * page. Note: if two part of one page are separately mapped, we
6865f0d1 3384 * might have two guest_addr mapping to the same host paddr, but this
ba395927
KA		 3385 * is not a big problem
3386 */
0ab36de2 3387 ret = domain_pfn_mapping(domain, mm_to_dma_pfn(iova->pfn_lo),
33041ec0 3388 mm_to_dma_pfn(paddr_pfn), size, prot);
ba395927
KA		 3389 if (ret)
3390 goto error;
3391
1f0ef2aa
DW		 3392 /* it's a non-present to present mapping. Only flush if caching mode */
3393 if (cap_caching_mode(iommu->cap))
ea8ea460 3394 iommu_flush_iotlb_psi(iommu, domain->id, mm_to_dma_pfn(iova->pfn_lo), size, 0, 1);
1f0ef2aa 3395 else
8c11e798 3396 iommu_flush_write_buffer(iommu);
f76aec76 3397
03d6a246
DW		 3398 start_paddr = (phys_addr_t)iova->pfn_lo << PAGE_SHIFT;
3399 start_paddr += paddr & ~PAGE_MASK;
3400 return start_paddr;
ba395927 3401
ba395927 3402error:
f76aec76
KA		 3403 if (iova)
3404 __free_iova(&domain->iovad, iova);
9f10e5bf 3405 pr_err("Device %s request: %zx@%llx dir %d --- failed\n",
5040a918 3406 dev_name(dev), size, (unsigned long long)paddr, dir);
ba395927
KA		 3407 return 0;
3408}
3409
ffbbef5c
FT		 3410static dma_addr_t intel_map_page(struct device *dev, struct page *page,
3411 unsigned long offset, size_t size,
3412 enum dma_data_direction dir,
3413 struct dma_attrs *attrs)
bb9e6d65 3414{
ffbbef5c 3415 return __intel_map_single(dev, page_to_phys(page) + offset, size,
46333e37 3416 dir, *dev->dma_mask);
bb9e6d65
FT		 3417}
3418
5e0d2a6f 3419static void flush_unmaps(void)
3420{
80b20dd8 3421 int i, j;
5e0d2a6f 3422
5e0d2a6f 3423 timer_on = 0;
3424
3425 /* just flush them all */
3426 for (i = 0; i < g_num_of_iommus; i++) {
a2bb8459
WH		 3427 struct intel_iommu *iommu = g_iommus[i];
3428 if (!iommu)
3429 continue;
c42d9f32 3430
9dd2fe89
YZ		 3431 if (!deferred_flush[i].next)
3432 continue;
3433
78d5f0f5
NA		 3434 /* In caching mode, global flushes turn emulation expensive */
3435 if (!cap_caching_mode(iommu->cap))
3436 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
93a23a72 3437 DMA_TLB_GLOBAL_FLUSH);
9dd2fe89 3438 for (j = 0; j < deferred_flush[i].next; j++) {
93a23a72
YZ		 3439 unsigned long mask;
3440 struct iova *iova = deferred_flush[i].iova[j];
78d5f0f5
NA		 3441 struct dmar_domain *domain = deferred_flush[i].domain[j];
3442
3443 /* On real hardware multiple invalidations are expensive */
3444 if (cap_caching_mode(iommu->cap))
3445 iommu_flush_iotlb_psi(iommu, domain->id,
a156ef99 3446 iova->pfn_lo, iova_size(iova),
ea8ea460 3447 !deferred_flush[i].freelist[j], 0);
78d5f0f5 3448 else {
a156ef99 3449 mask = ilog2(mm_to_dma_pfn(iova_size(iova)));
78d5f0f5
NA		 3450 iommu_flush_dev_iotlb(deferred_flush[i].domain[j],
3451 (uint64_t)iova->pfn_lo << PAGE_SHIFT, mask);
3452 }
93a23a72 3453 __free_iova(&deferred_flush[i].domain[j]->iovad, iova);
ea8ea460
DW		 3454 if (deferred_flush[i].freelist[j])
3455 dma_free_pagelist(deferred_flush[i].freelist[j]);
80b20dd8 3456 }
9dd2fe89 3457 deferred_flush[i].next = 0;
5e0d2a6f 3458 }
3459
5e0d2a6f 3460 list_size = 0;
5e0d2a6f 3461}
3462
3463static void flush_unmaps_timeout(unsigned long data)
3464{
80b20dd8 3465 unsigned long flags;
3466
3467 spin_lock_irqsave(&async_umap_flush_lock, flags);
5e0d2a6f 3468 flush_unmaps();
80b20dd8 3469 spin_unlock_irqrestore(&async_umap_flush_lock, flags);
5e0d2a6f 3470}
3471
ea8ea460 3472static void add_unmap(struct dmar_domain *dom, struct iova *iova, struct page *freelist)
5e0d2a6f 3473{
3474 unsigned long flags;
80b20dd8 3475 int next, iommu_id;
8c11e798 3476 struct intel_iommu *iommu;
5e0d2a6f 3477
3478 spin_lock_irqsave(&async_umap_flush_lock, flags);
80b20dd8 3479 if (list_size == HIGH_WATER_MARK)
3480 flush_unmaps();
3481
8c11e798
WH		 3482 iommu = domain_get_iommu(dom);
3483 iommu_id = iommu->seq_id;
c42d9f32 3484
80b20dd8 3485 next = deferred_flush[iommu_id].next;
3486 deferred_flush[iommu_id].domain[next] = dom;
3487 deferred_flush[iommu_id].iova[next] = iova;
ea8ea460 3488 deferred_flush[iommu_id].freelist[next] = freelist;
80b20dd8 3489 deferred_flush[iommu_id].next++;
5e0d2a6f 3490
3491 if (!timer_on) {
3492 mod_timer(&unmap_timer, jiffies + msecs_to_jiffies(10));
3493 timer_on = 1;
3494 }
3495 list_size++;
3496 spin_unlock_irqrestore(&async_umap_flush_lock, flags);
3497}
3498
d41a4adb 3499static void intel_unmap(struct device *dev, dma_addr_t dev_addr)
ba395927 3500{
f76aec76 3501 struct dmar_domain *domain;
d794dc9b 3502 unsigned long start_pfn, last_pfn;
ba395927 3503 struct iova *iova;
8c11e798 3504 struct intel_iommu *iommu;
ea8ea460 3505 struct page *freelist;
ba395927 3506
73676832 3507 if (iommu_no_mapping(dev))
f76aec76 3508 return;
2c2e2c38 3509
1525a29a 3510 domain = find_domain(dev);
ba395927
KA		 3511 BUG_ON(!domain);
3512
8c11e798
WH		 3513 iommu = domain_get_iommu(domain);
3514
ba395927 3515 iova = find_iova(&domain->iovad, IOVA_PFN(dev_addr));
85b98276
DW		 3516 if (WARN_ONCE(!iova, "Driver unmaps unmatched page at PFN %llx\n",
3517 (unsigned long long)dev_addr))
ba395927 3518 return;
ba395927 3519
d794dc9b
DW		 3520 start_pfn = mm_to_dma_pfn(iova->pfn_lo);
3521 last_pfn = mm_to_dma_pfn(iova->pfn_hi + 1) - 1;
ba395927 3522
d794dc9b 3523 pr_debug("Device %s unmapping: pfn %lx-%lx\n",
207e3592 3524 dev_name(dev), start_pfn, last_pfn);
ba395927 3525
ea8ea460 3526 freelist = domain_unmap(domain, start_pfn, last_pfn);
d794dc9b 3527
5e0d2a6f 3528 if (intel_iommu_strict) {
03d6a246 3529 iommu_flush_iotlb_psi(iommu, domain->id, start_pfn,
ea8ea460 3530 last_pfn - start_pfn + 1, !freelist, 0);
5e0d2a6f 3531 /* free iova */
3532 __free_iova(&domain->iovad, iova);
ea8ea460 3533 dma_free_pagelist(freelist);
5e0d2a6f 3534 } else {
ea8ea460 3535 add_unmap(domain, iova, freelist);
5e0d2a6f 3536 /*
3537 * queue up the release of the unmap to save the 1/6th of the
3538 * cpu used up by the iotlb flush operation...
3539 */
5e0d2a6f 3540 }
ba395927
KA		 3541}
3542
d41a4adb
JL		 3543static void intel_unmap_page(struct device *dev, dma_addr_t dev_addr,
3544 size_t size, enum dma_data_direction dir,
3545 struct dma_attrs *attrs)
3546{
3547 intel_unmap(dev, dev_addr);
3548}
3549
5040a918 3550static void *intel_alloc_coherent(struct device *dev, size_t size,
baa676fc
AP		 3551 dma_addr_t *dma_handle, gfp_t flags,
3552 struct dma_attrs *attrs)
ba395927 3553{
36746436 3554 struct page *page = NULL;
ba395927
KA		 3555 int order;
3556
5b6985ce 3557 size = PAGE_ALIGN(size);
ba395927 3558 order = get_order(size);
e8bb910d 3559
5040a918 3560 if (!iommu_no_mapping(dev))
e8bb910d 3561 flags &= ~(GFP_DMA | GFP_DMA32);
5040a918
DW		 3562 else if (dev->coherent_dma_mask < dma_get_required_mask(dev)) {
3563 if (dev->coherent_dma_mask < DMA_BIT_MASK(32))
e8bb910d
AW		 3564 flags |= GFP_DMA;
3565 else
3566 flags |= GFP_DMA32;
3567 }
ba395927 3568
36746436
AM		 3569 if (flags & __GFP_WAIT) {
3570 unsigned int count = size >> PAGE_SHIFT;
3571
3572 page = dma_alloc_from_contiguous(dev, count, order);
3573 if (page && iommu_no_mapping(dev) &&
3574 page_to_phys(page) + size > dev->coherent_dma_mask) {
3575 dma_release_from_contiguous(dev, page, count);
3576 page = NULL;
3577 }
3578 }
3579
3580 if (!page)
3581 page = alloc_pages(flags, order);
3582 if (!page)
ba395927 3583 return NULL;
36746436 3584 memset(page_address(page), 0, size);
ba395927 3585
36746436 3586 *dma_handle = __intel_map_single(dev, page_to_phys(page), size,
bb9e6d65 3587 DMA_BIDIRECTIONAL,
5040a918 3588 dev->coherent_dma_mask);
ba395927 3589 if (*dma_handle)
36746436
AM		 3590 return page_address(page);
3591 if (!dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT))
3592 __free_pages(page, order);
3593
ba395927
KA		 3594 return NULL;
3595}
3596
5040a918 3597static void intel_free_coherent(struct device *dev, size_t size, void *vaddr,
baa676fc 3598 dma_addr_t dma_handle, struct dma_attrs *attrs)
ba395927
KA		 3599{
3600 int order;
36746436 3601 struct page *page = virt_to_page(vaddr);
ba395927 3602
5b6985ce 3603 size = PAGE_ALIGN(size);
ba395927
KA		 3604 order = get_order(size);
3605
d41a4adb 3606 intel_unmap(dev, dma_handle);
36746436
AM		 3607 if (!dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT))
3608 __free_pages(page, order);
ba395927
KA		 3609}
3610
5040a918 3611static void intel_unmap_sg(struct device *dev, struct scatterlist *sglist,
d7ab5c46
FT		 3612 int nelems, enum dma_data_direction dir,
3613 struct dma_attrs *attrs)
ba395927 3614{
d41a4adb 3615 intel_unmap(dev, sglist[0].dma_address);
ba395927
KA		 3616}
3617
ba395927 3618static int intel_nontranslate_map_sg(struct device *hddev,
c03ab37c 3619 struct scatterlist *sglist, int nelems, int dir)
ba395927
KA		 3620{
3621 int i;
c03ab37c 3622 struct scatterlist *sg;
ba395927 3623
c03ab37c 3624 for_each_sg(sglist, sg, nelems, i) {
12d4d40e 3625 BUG_ON(!sg_page(sg));
4cf2e75d 3626 sg->dma_address = page_to_phys(sg_page(sg)) + sg->offset;
c03ab37c 3627 sg->dma_length = sg->length;
ba395927
KA		 3628 }
3629 return nelems;
3630}
3631
5040a918 3632static int intel_map_sg(struct device *dev, struct scatterlist *sglist, int nelems,
d7ab5c46 3633 enum dma_data_direction dir, struct dma_attrs *attrs)
ba395927 3634{
ba395927 3635 int i;
ba395927 3636 struct dmar_domain *domain;
f76aec76
KA		 3637 size_t size = 0;
3638 int prot = 0;
f76aec76
KA		 3639 struct iova *iova = NULL;
3640 int ret;
c03ab37c 3641 struct scatterlist *sg;
b536d24d 3642 unsigned long start_vpfn;
8c11e798 3643 struct intel_iommu *iommu;
ba395927
KA		 3644
3645 BUG_ON(dir == DMA_NONE);
5040a918
DW		 3646 if (iommu_no_mapping(dev))
3647 return intel_nontranslate_map_sg(dev, sglist, nelems, dir);
ba395927 3648
5040a918 3649 domain = get_valid_domain_for_dev(dev);
f76aec76
KA		 3650 if (!domain)
3651 return 0;
3652
8c11e798
WH		 3653 iommu = domain_get_iommu(domain);
3654
b536d24d 3655 for_each_sg(sglist, sg, nelems, i)
88cb6a74 3656 size += aligned_nrpages(sg->offset, sg->length);
f76aec76 3657
5040a918
DW		 3658 iova = intel_alloc_iova(dev, domain, dma_to_mm_pfn(size),
3659 *dev->dma_mask);
f76aec76 3660 if (!iova) {
c03ab37c 3661 sglist->dma_length = 0;
f76aec76
KA		 3662 return 0;
3663 }
3664
3665 /*
3666 * Check if DMAR supports zero-length reads on write only
3667 * mappings..
3668 */
3669 if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
8c11e798 3670 !cap_zlr(iommu->cap))
f76aec76
KA		 3671 prot |= DMA_PTE_READ;
3672 if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
3673 prot |= DMA_PTE_WRITE;
3674
b536d24d 3675 start_vpfn = mm_to_dma_pfn(iova->pfn_lo);
e1605495 3676
f532959b 3677 ret = domain_sg_mapping(domain, start_vpfn, sglist, size, prot);
e1605495 3678 if (unlikely(ret)) {
e1605495
DW		 3679 dma_pte_free_pagetable(domain, start_vpfn,
3680 start_vpfn + size - 1);
e1605495
DW		 3681 __free_iova(&domain->iovad, iova);
3682 return 0;
ba395927
KA		 3683 }
3684
1f0ef2aa
DW		 3685 /* it's a non-present to present mapping. Only flush if caching mode */
3686 if (cap_caching_mode(iommu->cap))
ea8ea460 3687 iommu_flush_iotlb_psi(iommu, domain->id, start_vpfn, size, 0, 1);
1f0ef2aa 3688 else
8c11e798 3689 iommu_flush_write_buffer(iommu);
1f0ef2aa 3690
ba395927
KA		 3691 return nelems;
3692}
3693
dfb805e8
FT		 3694static int intel_mapping_error(struct device *dev, dma_addr_t dma_addr)
3695{
3696 return !dma_addr;
3697}
3698
160c1d8e 3699struct dma_map_ops intel_dma_ops = {
baa676fc
AP		 3700 .alloc = intel_alloc_coherent,
3701 .free = intel_free_coherent,
ba395927
KA		 3702 .map_sg = intel_map_sg,
3703 .unmap_sg = intel_unmap_sg,
ffbbef5c
FT		 3704 .map_page = intel_map_page,
3705 .unmap_page = intel_unmap_page,
dfb805e8 3706 .mapping_error = intel_mapping_error,
ba395927
KA		 3707};
3708
3709static inline int iommu_domain_cache_init(void)
3710{
3711 int ret = 0;
3712
3713 iommu_domain_cache = kmem_cache_create("iommu_domain",
3714 sizeof(struct dmar_domain),
3715 0,
3716 SLAB_HWCACHE_ALIGN,
3717
3718 NULL);
3719 if (!iommu_domain_cache) {
9f10e5bf 3720 pr_err("Couldn't create iommu_domain cache\n");
ba395927
KA		 3721 ret = -ENOMEM;
3722 }
3723
3724 return ret;
3725}
3726
3727static inline int iommu_devinfo_cache_init(void)
3728{
3729 int ret = 0;
3730
3731 iommu_devinfo_cache = kmem_cache_create("iommu_devinfo",
3732 sizeof(struct device_domain_info),
3733 0,
3734 SLAB_HWCACHE_ALIGN,
ba395927
KA		 3735 NULL);
3736 if (!iommu_devinfo_cache) {
9f10e5bf 3737 pr_err("Couldn't create devinfo cache\n");
ba395927
KA		 3738 ret = -ENOMEM;
3739 }
3740
3741 return ret;
3742}
3743
ba395927
KA		 3744static int __init iommu_init_mempool(void)
3745{
3746 int ret;
3747 ret = iommu_iova_cache_init();
3748 if (ret)
3749 return ret;
3750
3751 ret = iommu_domain_cache_init();
3752 if (ret)
3753 goto domain_error;
3754
3755 ret = iommu_devinfo_cache_init();
3756 if (!ret)
3757 return ret;
3758
3759 kmem_cache_destroy(iommu_domain_cache);
3760domain_error:
85b45456 3761 iommu_iova_cache_destroy();
ba395927
KA		 3762
3763 return -ENOMEM;
3764}
3765
3766static void __init iommu_exit_mempool(void)
3767{
3768 kmem_cache_destroy(iommu_devinfo_cache);
3769 kmem_cache_destroy(iommu_domain_cache);
85b45456 3770 iommu_iova_cache_destroy();
ba395927
KA		 3771}
3772
556ab45f
DW		 3773static void quirk_ioat_snb_local_iommu(struct pci_dev *pdev)
3774{
3775 struct dmar_drhd_unit *drhd;
3776 u32 vtbar;
3777 int rc;
3778
3779 /* We know that this device on this chipset has its own IOMMU.
3780 * If we find it under a different IOMMU, then the BIOS is lying
3781 * to us. Hope that the IOMMU for this device is actually
3782 * disabled, and it needs no translation...
3783 */
3784 rc = pci_bus_read_config_dword(pdev->bus, PCI_DEVFN(0, 0), 0xb0, &vtbar);
3785 if (rc) {
3786 /* "can't" happen */
3787 dev_info(&pdev->dev, "failed to run vt-d quirk\n");
3788 return;
3789 }
3790 vtbar &= 0xffff0000;
3791
3792 /* we know that the this iommu should be at offset 0xa000 from vtbar */
3793 drhd = dmar_find_matched_drhd_unit(pdev);
3794 if (WARN_TAINT_ONCE(!drhd || drhd->reg_base_addr - vtbar != 0xa000,
3795 TAINT_FIRMWARE_WORKAROUND,
3796 "BIOS assigned incorrect VT-d unit for Intel(R) QuickData Technology device\n"))
3797 pdev->dev.archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO;
3798}
3799DECLARE_PCI_FIXUP_ENABLE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB, quirk_ioat_snb_local_iommu);
3800
ba395927
KA		 3801static void __init init_no_remapping_devices(void)
3802{
3803 struct dmar_drhd_unit *drhd;
832bd858 3804 struct device *dev;
b683b230 3805 int i;
ba395927
KA		 3806
3807 for_each_drhd_unit(drhd) {
3808 if (!drhd->include_all) {
b683b230
JL		 3809 for_each_active_dev_scope(drhd->devices,
3810 drhd->devices_cnt, i, dev)
3811 break;
832bd858 3812 /* ignore DMAR unit if no devices exist */
ba395927
KA		 3813 if (i == drhd->devices_cnt)
3814 drhd->ignored = 1;
3815 }
3816 }
3817
7c919779 3818 for_each_active_drhd_unit(drhd) {
7c919779 3819 if (drhd->include_all)
ba395927
KA		 3820 continue;
3821
b683b230
JL		 3822 for_each_active_dev_scope(drhd->devices,
3823 drhd->devices_cnt, i, dev)
832bd858 3824 if (!dev_is_pci(dev) || !IS_GFX_DEVICE(to_pci_dev(dev)))
ba395927 3825 break;
ba395927
KA		 3826 if (i < drhd->devices_cnt)
3827 continue;
3828
c0771df8
DW		 3829 /* This IOMMU has *only* gfx devices. Either bypass it or
3830 set the gfx_mapped flag, as appropriate */
3831 if (dmar_map_gfx) {
3832 intel_iommu_gfx_mapped = 1;
3833 } else {
3834 drhd->ignored = 1;
b683b230
JL		 3835 for_each_active_dev_scope(drhd->devices,
3836 drhd->devices_cnt, i, dev)
832bd858 3837 dev->archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO;
ba395927
KA		 3838 }
3839 }
3840}
3841
f59c7b69
FY		 3842#ifdef CONFIG_SUSPEND
3843static int init_iommu_hw(void)
3844{
3845 struct dmar_drhd_unit *drhd;
3846 struct intel_iommu *iommu = NULL;
3847
3848 for_each_active_iommu(iommu, drhd)
3849 if (iommu->qi)
3850 dmar_reenable_qi(iommu);
3851
b779260b
JC		 3852 for_each_iommu(iommu, drhd) {
3853 if (drhd->ignored) {
3854 /*
3855 * we always have to disable PMRs or DMA may fail on
3856 * this device
3857 */
3858 if (force_on)
3859 iommu_disable_protect_mem_regions(iommu);
3860 continue;
3861 }
3862
f59c7b69
FY		 3863 iommu_flush_write_buffer(iommu);
3864
3865 iommu_set_root_entry(iommu);
3866
3867 iommu->flush.flush_context(iommu, 0, 0, 0,
1f0ef2aa 3868 DMA_CCMD_GLOBAL_INVL);
2a41ccee
JL		 3869 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
3870 iommu_enable_translation(iommu);
b94996c9 3871 iommu_disable_protect_mem_regions(iommu);
f59c7b69
FY		 3872 }
3873
3874 return 0;
3875}
3876
3877static void iommu_flush_all(void)
3878{
3879 struct dmar_drhd_unit *drhd;
3880 struct intel_iommu *iommu;
3881
3882 for_each_active_iommu(iommu, drhd) {
3883 iommu->flush.flush_context(iommu, 0, 0, 0,
1f0ef2aa 3884 DMA_CCMD_GLOBAL_INVL);
f59c7b69 3885 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
1f0ef2aa 3886 DMA_TLB_GLOBAL_FLUSH);
f59c7b69
FY		 3887 }
3888}
3889
134fac3f 3890static int iommu_suspend(void)
f59c7b69
FY		 3891{
3892 struct dmar_drhd_unit *drhd;
3893 struct intel_iommu *iommu = NULL;
3894 unsigned long flag;
3895
3896 for_each_active_iommu(iommu, drhd) {
3897 iommu->iommu_state = kzalloc(sizeof(u32) * MAX_SR_DMAR_REGS,
3898 GFP_ATOMIC);
3899 if (!iommu->iommu_state)
3900 goto nomem;
3901 }
3902
3903 iommu_flush_all();
3904
3905 for_each_active_iommu(iommu, drhd) {
3906 iommu_disable_translation(iommu);
3907
1f5b3c3f 3908 raw_spin_lock_irqsave(&iommu->register_lock, flag);
f59c7b69
FY		 3909
3910 iommu->iommu_state[SR_DMAR_FECTL_REG] =
3911 readl(iommu->reg + DMAR_FECTL_REG);
3912 iommu->iommu_state[SR_DMAR_FEDATA_REG] =
3913 readl(iommu->reg + DMAR_FEDATA_REG);
3914 iommu->iommu_state[SR_DMAR_FEADDR_REG] =
3915 readl(iommu->reg + DMAR_FEADDR_REG);
3916 iommu->iommu_state[SR_DMAR_FEUADDR_REG] =
3917 readl(iommu->reg + DMAR_FEUADDR_REG);
3918
1f5b3c3f 3919 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
f59c7b69
FY		 3920 }
3921 return 0;
3922
3923nomem:
3924 for_each_active_iommu(iommu, drhd)
3925 kfree(iommu->iommu_state);
3926
3927 return -ENOMEM;
3928}
3929
134fac3f 3930static void iommu_resume(void)
f59c7b69
FY		 3931{
3932 struct dmar_drhd_unit *drhd;
3933 struct intel_iommu *iommu = NULL;
3934 unsigned long flag;
3935
3936 if (init_iommu_hw()) {
b779260b
JC		 3937 if (force_on)
3938 panic("tboot: IOMMU setup failed, DMAR can not resume!\n");
3939 else
3940 WARN(1, "IOMMU setup failed, DMAR can not resume!\n");
134fac3f 3941 return;
f59c7b69
FY		 3942 }
3943
3944 for_each_active_iommu(iommu, drhd) {
3945
1f5b3c3f 3946 raw_spin_lock_irqsave(&iommu->register_lock, flag);
f59c7b69
FY		 3947
3948 writel(iommu->iommu_state[SR_DMAR_FECTL_REG],
3949 iommu->reg + DMAR_FECTL_REG);
3950 writel(iommu->iommu_state[SR_DMAR_FEDATA_REG],
3951 iommu->reg + DMAR_FEDATA_REG);
3952 writel(iommu->iommu_state[SR_DMAR_FEADDR_REG],
3953 iommu->reg + DMAR_FEADDR_REG);
3954 writel(iommu->iommu_state[SR_DMAR_FEUADDR_REG],
3955 iommu->reg + DMAR_FEUADDR_REG);
3956
1f5b3c3f 3957 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
f59c7b69
FY		 3958 }
3959
3960 for_each_active_iommu(iommu, drhd)
3961 kfree(iommu->iommu_state);
f59c7b69
FY		 3962}
3963
134fac3f 3964static struct syscore_ops iommu_syscore_ops = {
f59c7b69
FY		 3965 .resume = iommu_resume,
3966 .suspend = iommu_suspend,
3967};
3968
134fac3f 3969static void __init init_iommu_pm_ops(void)
f59c7b69 3970{
134fac3f 3971 register_syscore_ops(&iommu_syscore_ops);
f59c7b69
FY		 3972}
3973
3974#else
99592ba4 3975static inline void init_iommu_pm_ops(void) {}
f59c7b69
FY		 3976#endif /* CONFIG_PM */
3977
318fe7df 3978
c2a0b538 3979int __init dmar_parse_one_rmrr(struct acpi_dmar_header *header, void *arg)
318fe7df
SS		 3980{
3981 struct acpi_dmar_reserved_memory *rmrr;
3982 struct dmar_rmrr_unit *rmrru;
3983
3984 rmrru = kzalloc(sizeof(*rmrru), GFP_KERNEL);
3985 if (!rmrru)
3986 return -ENOMEM;
3987
3988 rmrru->hdr = header;
3989 rmrr = (struct acpi_dmar_reserved_memory *)header;
3990 rmrru->base_address = rmrr->base_address;
3991 rmrru->end_address = rmrr->end_address;
2e455289
JL		 3992 rmrru->devices = dmar_alloc_dev_scope((void *)(rmrr + 1),
3993 ((void *)rmrr) + rmrr->header.length,
3994 &rmrru->devices_cnt);
3995 if (rmrru->devices_cnt && rmrru->devices == NULL) {
3996 kfree(rmrru);
3997 return -ENOMEM;
3998 }
318fe7df 3999
2e455289 4000 list_add(&rmrru->list, &dmar_rmrr_units);
318fe7df 4001
2e455289 4002 return 0;
318fe7df
SS		 4003}
4004
6b197249
JL		 4005static struct dmar_atsr_unit *dmar_find_atsr(struct acpi_dmar_atsr *atsr)
4006{
4007 struct dmar_atsr_unit *atsru;
4008 struct acpi_dmar_atsr *tmp;
4009
4010 list_for_each_entry_rcu(atsru, &dmar_atsr_units, list) {
4011 tmp = (struct acpi_dmar_atsr *)atsru->hdr;
4012 if (atsr->segment != tmp->segment)
4013 continue;
4014 if (atsr->header.length != tmp->header.length)
4015 continue;
4016 if (memcmp(atsr, tmp, atsr->header.length) == 0)
4017 return atsru;
4018 }
4019
4020 return NULL;
4021}
4022
4023int dmar_parse_one_atsr(struct acpi_dmar_header *hdr, void *arg)
318fe7df
SS		 4024{
4025 struct acpi_dmar_atsr *atsr;
4026 struct dmar_atsr_unit *atsru;
4027
6b197249
JL		 4028 if (system_state != SYSTEM_BOOTING && !intel_iommu_enabled)
4029 return 0;
4030
318fe7df 4031 atsr = container_of(hdr, struct acpi_dmar_atsr, header);
6b197249
JL		 4032 atsru = dmar_find_atsr(atsr);
4033 if (atsru)
4034 return 0;
4035
4036 atsru = kzalloc(sizeof(*atsru) + hdr->length, GFP_KERNEL);
318fe7df
SS		 4037 if (!atsru)
4038 return -ENOMEM;
4039
6b197249
JL		 4040 /*
4041 * If memory is allocated from slab by ACPI _DSM method, we need to
4042 * copy the memory content because the memory buffer will be freed
4043 * on return.
4044 */
4045 atsru->hdr = (void *)(atsru + 1);
4046 memcpy(atsru->hdr, hdr, hdr->length);
318fe7df 4047 atsru->include_all = atsr->flags & 0x1;
2e455289
JL		 4048 if (!atsru->include_all) {
4049 atsru->devices = dmar_alloc_dev_scope((void *)(atsr + 1),
4050 (void *)atsr + atsr->header.length,
4051 &atsru->devices_cnt);
4052 if (atsru->devices_cnt && atsru->devices == NULL) {
4053 kfree(atsru);
4054 return -ENOMEM;
4055 }
4056 }
318fe7df 4057
0e242612 4058 list_add_rcu(&atsru->list, &dmar_atsr_units);
318fe7df
SS		 4059
4060 return 0;
4061}
4062
9bdc531e
JL		 4063static void intel_iommu_free_atsr(struct dmar_atsr_unit *atsru)
4064{
4065 dmar_free_dev_scope(&atsru->devices, &atsru->devices_cnt);
4066 kfree(atsru);
4067}
4068
6b197249
JL		 4069int dmar_release_one_atsr(struct acpi_dmar_header *hdr, void *arg)
4070{
4071 struct acpi_dmar_atsr *atsr;
4072 struct dmar_atsr_unit *atsru;
4073
4074 atsr = container_of(hdr, struct acpi_dmar_atsr, header);
4075 atsru = dmar_find_atsr(atsr);
4076 if (atsru) {
4077 list_del_rcu(&atsru->list);
4078 synchronize_rcu();
4079 intel_iommu_free_atsr(atsru);
4080 }
4081
4082 return 0;
4083}
4084
4085int dmar_check_one_atsr(struct acpi_dmar_header *hdr, void *arg)
4086{
4087 int i;
4088 struct device *dev;
4089 struct acpi_dmar_atsr *atsr;
4090 struct dmar_atsr_unit *atsru;
4091
4092 atsr = container_of(hdr, struct acpi_dmar_atsr, header);
4093 atsru = dmar_find_atsr(atsr);
4094 if (!atsru)
4095 return 0;
4096
4097 if (!atsru->include_all && atsru->devices && atsru->devices_cnt)
4098 for_each_active_dev_scope(atsru->devices, atsru->devices_cnt,
4099 i, dev)
4100 return -EBUSY;
4101
4102 return 0;
4103}
4104
ffebeb46
JL		 4105static int intel_iommu_add(struct dmar_drhd_unit *dmaru)
4106{
4107 int sp, ret = 0;
4108 struct intel_iommu *iommu = dmaru->iommu;
4109
4110 if (g_iommus[iommu->seq_id])
4111 return 0;
4112
4113 if (hw_pass_through && !ecap_pass_through(iommu->ecap)) {
9f10e5bf 4114 pr_warn("%s: Doesn't support hardware pass through.\n",
ffebeb46
JL		 4115 iommu->name);
4116 return -ENXIO;
4117 }
4118 if (!ecap_sc_support(iommu->ecap) &&
4119 domain_update_iommu_snooping(iommu)) {
9f10e5bf 4120 pr_warn("%s: Doesn't support snooping.\n",
ffebeb46
JL		 4121 iommu->name);
4122 return -ENXIO;
4123 }
4124 sp = domain_update_iommu_superpage(iommu) - 1;
4125 if (sp >= 0 && !(cap_super_page_val(iommu->cap) & (1 << sp))) {
9f10e5bf 4126 pr_warn("%s: Doesn't support large page.\n",
ffebeb46
JL		 4127 iommu->name);
4128 return -ENXIO;
4129 }
4130
4131 /*
4132 * Disable translation if already enabled prior to OS handover.
4133 */
4134 if (iommu->gcmd & DMA_GCMD_TE)
4135 iommu_disable_translation(iommu);
4136
4137 g_iommus[iommu->seq_id] = iommu;
4138 ret = iommu_init_domains(iommu);
4139 if (ret == 0)
4140 ret = iommu_alloc_root_entry(iommu);
4141 if (ret)
4142 goto out;
4143
4144 if (dmaru->ignored) {
4145 /*
4146 * we always have to disable PMRs or DMA may fail on this device
4147 */
4148 if (force_on)
4149 iommu_disable_protect_mem_regions(iommu);
4150 return 0;
4151 }
4152
4153 intel_iommu_init_qi(iommu);
4154 iommu_flush_write_buffer(iommu);
4155 ret = dmar_set_interrupt(iommu);
4156 if (ret)
4157 goto disable_iommu;
4158
4159 iommu_set_root_entry(iommu);
4160 iommu->flush.flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL);
4161 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
4162 iommu_enable_translation(iommu);
4163
4164 if (si_domain) {
4165 ret = iommu_attach_domain(si_domain, iommu);
4166 if (ret < 0 || si_domain->id != ret)
4167 goto disable_iommu;
4168 domain_attach_iommu(si_domain, iommu);
4169 }
4170
4171 iommu_disable_protect_mem_regions(iommu);
4172 return 0;
4173
4174disable_iommu:
4175 disable_dmar_iommu(iommu);
4176out:
4177 free_dmar_iommu(iommu);
4178 return ret;
4179}
4180
6b197249
JL		 4181int dmar_iommu_hotplug(struct dmar_drhd_unit *dmaru, bool insert)
4182{
ffebeb46
JL		 4183 int ret = 0;
4184 struct intel_iommu *iommu = dmaru->iommu;
4185
4186 if (!intel_iommu_enabled)
4187 return 0;
4188 if (iommu == NULL)
4189 return -EINVAL;
4190
4191 if (insert) {
4192 ret = intel_iommu_add(dmaru);
4193 } else {
4194 disable_dmar_iommu(iommu);
4195 free_dmar_iommu(iommu);
4196 }
4197
4198 return ret;
6b197249
JL		 4199}
4200
9bdc531e
JL		 4201static void intel_iommu_free_dmars(void)
4202{
4203 struct dmar_rmrr_unit *rmrru, *rmrr_n;
4204 struct dmar_atsr_unit *atsru, *atsr_n;
4205
4206 list_for_each_entry_safe(rmrru, rmrr_n, &dmar_rmrr_units, list) {
4207 list_del(&rmrru->list);
4208 dmar_free_dev_scope(&rmrru->devices, &rmrru->devices_cnt);
4209 kfree(rmrru);
318fe7df
SS		 4210 }
4211
9bdc531e
JL		 4212 list_for_each_entry_safe(atsru, atsr_n, &dmar_atsr_units, list) {
4213 list_del(&atsru->list);
4214 intel_iommu_free_atsr(atsru);
4215 }
318fe7df
SS		 4216}
4217
4218int dmar_find_matched_atsr_unit(struct pci_dev *dev)
4219{
b683b230 4220 int i, ret = 1;
318fe7df 4221 struct pci_bus *bus;
832bd858
DW		 4222 struct pci_dev *bridge = NULL;
4223 struct device *tmp;
318fe7df
SS		 4224 struct acpi_dmar_atsr *atsr;
4225 struct dmar_atsr_unit *atsru;
4226
4227 dev = pci_physfn(dev);
318fe7df 4228 for (bus = dev->bus; bus; bus = bus->parent) {
b5f82ddf 4229 bridge = bus->self;
318fe7df 4230 if (!bridge || !pci_is_pcie(bridge) ||
62f87c0e 4231 pci_pcie_type(bridge) == PCI_EXP_TYPE_PCI_BRIDGE)
318fe7df 4232 return 0;
b5f82ddf 4233 if (pci_pcie_type(bridge) == PCI_EXP_TYPE_ROOT_PORT)
318fe7df 4234 break;
318fe7df 4235 }
b5f82ddf
JL		 4236 if (!bridge)
4237 return 0;
318fe7df 4238
0e242612 4239 rcu_read_lock();
b5f82ddf
JL		 4240 list_for_each_entry_rcu(atsru, &dmar_atsr_units, list) {
4241 atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header);
4242 if (atsr->segment != pci_domain_nr(dev->bus))
4243 continue;
4244
b683b230 4245 for_each_dev_scope(atsru->devices, atsru->devices_cnt, i, tmp)
832bd858 4246 if (tmp == &bridge->dev)
b683b230 4247 goto out;
b5f82ddf
JL		 4248
4249 if (atsru->include_all)
b683b230 4250 goto out;
b5f82ddf 4251 }
b683b230
JL		 4252 ret = 0;
4253out:
0e242612 4254 rcu_read_unlock();
318fe7df 4255
b683b230 4256 return ret;
318fe7df
SS		 4257}
4258
59ce0515
JL		 4259int dmar_iommu_notify_scope_dev(struct dmar_pci_notify_info *info)
4260{
4261 int ret = 0;
4262 struct dmar_rmrr_unit *rmrru;
4263 struct dmar_atsr_unit *atsru;
4264 struct acpi_dmar_atsr *atsr;
4265 struct acpi_dmar_reserved_memory *rmrr;
4266
4267 if (!intel_iommu_enabled && system_state != SYSTEM_BOOTING)
4268 return 0;
4269
4270 list_for_each_entry(rmrru, &dmar_rmrr_units, list) {
4271 rmrr = container_of(rmrru->hdr,
4272 struct acpi_dmar_reserved_memory, header);
4273 if (info->event == BUS_NOTIFY_ADD_DEVICE) {
4274 ret = dmar_insert_dev_scope(info, (void *)(rmrr + 1),
4275 ((void *)rmrr) + rmrr->header.length,
4276 rmrr->segment, rmrru->devices,
4277 rmrru->devices_cnt);
27e24950 4278 if(ret < 0)
59ce0515
JL		 4279 return ret;
4280 } else if (info->event == BUS_NOTIFY_DEL_DEVICE) {
27e24950
JL		 4281 dmar_remove_dev_scope(info, rmrr->segment,
4282 rmrru->devices, rmrru->devices_cnt);
59ce0515
JL		 4283 }
4284 }
4285
4286 list_for_each_entry(atsru, &dmar_atsr_units, list) {
4287 if (atsru->include_all)
4288 continue;
4289
4290 atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header);
4291 if (info->event == BUS_NOTIFY_ADD_DEVICE) {
4292 ret = dmar_insert_dev_scope(info, (void *)(atsr + 1),
4293 (void *)atsr + atsr->header.length,
4294 atsr->segment, atsru->devices,
4295 atsru->devices_cnt);
4296 if (ret > 0)
4297 break;
4298 else if(ret < 0)
4299 return ret;
4300 } else if (info->event == BUS_NOTIFY_DEL_DEVICE) {
4301 if (dmar_remove_dev_scope(info, atsr->segment,
4302 atsru->devices, atsru->devices_cnt))
4303 break;
4304 }
4305 }
4306
4307 return 0;
4308}
4309
99dcaded
FY		 4310/*
4311 * Here we only respond to action of unbound device from driver.
4312 *
4313 * Added device is not attached to its DMAR domain here yet. That will happen
4314 * when mapping the device to iova.
4315 */
4316static int device_notifier(struct notifier_block *nb,
4317 unsigned long action, void *data)
4318{
4319 struct device *dev = data;
99dcaded
FY		 4320 struct dmar_domain *domain;
4321
3d89194a 4322 if (iommu_dummy(dev))
44cd613c
DW		 4323 return 0;
4324
1196c2fb 4325 if (action != BUS_NOTIFY_REMOVED_DEVICE)
7e7dfab7
JL		 4326 return 0;
4327
1525a29a 4328 domain = find_domain(dev);
99dcaded
FY		 4329 if (!domain)
4330 return 0;
4331
3a5670e8 4332 down_read(&dmar_global_lock);
bf9c9eda 4333 domain_remove_one_dev_info(domain, dev);
ab8dfe25 4334 if (!domain_type_is_vm_or_si(domain) && list_empty(&domain->devices))
7e7dfab7 4335 domain_exit(domain);
3a5670e8 4336 up_read(&dmar_global_lock);
a97590e5 4337
99dcaded
FY		 4338 return 0;
4339}
4340
4341static struct notifier_block device_nb = {
4342 .notifier_call = device_notifier,
4343};
4344
75f05569
JL		 4345static int intel_iommu_memory_notifier(struct notifier_block *nb,
4346 unsigned long val, void *v)
4347{
4348 struct memory_notify *mhp = v;
4349 unsigned long long start, end;
4350 unsigned long start_vpfn, last_vpfn;
4351
4352 switch (val) {
4353 case MEM_GOING_ONLINE:
4354 start = mhp->start_pfn << PAGE_SHIFT;
4355 end = ((mhp->start_pfn + mhp->nr_pages) << PAGE_SHIFT) - 1;
4356 if (iommu_domain_identity_map(si_domain, start, end)) {
9f10e5bf 4357 pr_warn("Failed to build identity map for [%llx-%llx]\n",
75f05569
JL		 4358 start, end);
4359 return NOTIFY_BAD;
4360 }
4361 break;
4362
4363 case MEM_OFFLINE:
4364 case MEM_CANCEL_ONLINE:
4365 start_vpfn = mm_to_dma_pfn(mhp->start_pfn);
4366 last_vpfn = mm_to_dma_pfn(mhp->start_pfn + mhp->nr_pages - 1);
4367 while (start_vpfn <= last_vpfn) {
4368 struct iova *iova;
4369 struct dmar_drhd_unit *drhd;
4370 struct intel_iommu *iommu;
ea8ea460 4371 struct page *freelist;
75f05569
JL		 4372
4373 iova = find_iova(&si_domain->iovad, start_vpfn);
4374 if (iova == NULL) {
9f10e5bf 4375 pr_debug("Failed get IOVA for PFN %lx\n",
75f05569
JL		 4376 start_vpfn);
4377 break;
4378 }
4379
4380 iova = split_and_remove_iova(&si_domain->iovad, iova,
4381 start_vpfn, last_vpfn);
4382 if (iova == NULL) {
9f10e5bf 4383 pr_warn("Failed to split IOVA PFN [%lx-%lx]\n",
75f05569
JL		 4384 start_vpfn, last_vpfn);
4385 return NOTIFY_BAD;
4386 }
4387
ea8ea460
DW		 4388 freelist = domain_unmap(si_domain, iova->pfn_lo,
4389 iova->pfn_hi);
4390
75f05569
JL		 4391 rcu_read_lock();
4392 for_each_active_iommu(iommu, drhd)
4393 iommu_flush_iotlb_psi(iommu, si_domain->id,
a156ef99 4394 iova->pfn_lo, iova_size(iova),
ea8ea460 4395 !freelist, 0);
75f05569 4396 rcu_read_unlock();
ea8ea460 4397 dma_free_pagelist(freelist);
75f05569
JL		 4398
4399 start_vpfn = iova->pfn_hi + 1;
4400 free_iova_mem(iova);
4401 }
4402 break;
4403 }
4404
4405 return NOTIFY_OK;
4406}
4407
4408static struct notifier_block intel_iommu_memory_nb = {
4409 .notifier_call = intel_iommu_memory_notifier,
4410 .priority = 0
4411};
4412
a5459cfe
AW		 4413
4414static ssize_t intel_iommu_show_version(struct device *dev,
4415 struct device_attribute *attr,
4416 char *buf)
4417{
4418 struct intel_iommu *iommu = dev_get_drvdata(dev);
4419 u32 ver = readl(iommu->reg + DMAR_VER_REG);
4420 return sprintf(buf, "%d:%d\n",
4421 DMAR_VER_MAJOR(ver), DMAR_VER_MINOR(ver));
4422}
4423static DEVICE_ATTR(version, S_IRUGO, intel_iommu_show_version, NULL);
4424
4425static ssize_t intel_iommu_show_address(struct device *dev,
4426 struct device_attribute *attr,
4427 char *buf)
4428{
4429 struct intel_iommu *iommu = dev_get_drvdata(dev);
4430 return sprintf(buf, "%llx\n", iommu->reg_phys);
4431}
4432static DEVICE_ATTR(address, S_IRUGO, intel_iommu_show_address, NULL);
4433
4434static ssize_t intel_iommu_show_cap(struct device *dev,
4435 struct device_attribute *attr,
4436 char *buf)
4437{
4438 struct intel_iommu *iommu = dev_get_drvdata(dev);
4439 return sprintf(buf, "%llx\n", iommu->cap);
4440}
4441static DEVICE_ATTR(cap, S_IRUGO, intel_iommu_show_cap, NULL);
4442
4443static ssize_t intel_iommu_show_ecap(struct device *dev,
4444 struct device_attribute *attr,
4445 char *buf)
4446{
4447 struct intel_iommu *iommu = dev_get_drvdata(dev);
4448 return sprintf(buf, "%llx\n", iommu->ecap);
4449}
4450static DEVICE_ATTR(ecap, S_IRUGO, intel_iommu_show_ecap, NULL);
4451
4452static struct attribute *intel_iommu_attrs[] = {
4453 &dev_attr_version.attr,
4454 &dev_attr_address.attr,
4455 &dev_attr_cap.attr,
4456 &dev_attr_ecap.attr,
4457 NULL,
4458};
4459
4460static struct attribute_group intel_iommu_group = {
4461 .name = "intel-iommu",
4462 .attrs = intel_iommu_attrs,
4463};
4464
4465const struct attribute_group *intel_iommu_groups[] = {
4466 &intel_iommu_group,
4467 NULL,
4468};
4469
ba395927
KA		 4470int __init intel_iommu_init(void)
4471{
9bdc531e 4472 int ret = -ENODEV;
3a93c841 4473 struct dmar_drhd_unit *drhd;
7c919779 4474 struct intel_iommu *iommu;
ba395927 4475
a59b50e9
JC		 4476 /* VT-d is required for a TXT/tboot launch, so enforce that */
4477 force_on = tboot_force_iommu();
4478
3a5670e8
JL		 4479 if (iommu_init_mempool()) {
4480 if (force_on)
4481 panic("tboot: Failed to initialize iommu memory\n");
4482 return -ENOMEM;
4483 }
4484
4485 down_write(&dmar_global_lock);
a59b50e9
JC		 4486 if (dmar_table_init()) {
4487 if (force_on)
4488 panic("tboot: Failed to initialize DMAR table\n");
9bdc531e 4489 goto out_free_dmar;
a59b50e9 4490 }
ba395927 4491
c2c7286a 4492 if (dmar_dev_scope_init() < 0) {
a59b50e9
JC		 4493 if (force_on)
4494 panic("tboot: Failed to initialize DMAR device scope\n");
9bdc531e 4495 goto out_free_dmar;
a59b50e9 4496 }
1886e8a9 4497
75f1cdf1 4498 if (no_iommu || dmar_disabled)
9bdc531e 4499 goto out_free_dmar;
2ae21010 4500
318fe7df 4501 if (list_empty(&dmar_rmrr_units))
9f10e5bf 4502 pr_info("No RMRR found\n");
318fe7df
SS		 4503
4504 if (list_empty(&dmar_atsr_units))
9f10e5bf 4505 pr_info("No ATSR found\n");
318fe7df 4506
51a63e67
JC		 4507 if (dmar_init_reserved_ranges()) {
4508 if (force_on)
4509 panic("tboot: Failed to reserve iommu ranges\n");
3a5670e8 4510 goto out_free_reserved_range;
51a63e67 4511 }
ba395927
KA		 4512
4513 init_no_remapping_devices();
4514
b779260b 4515 ret = init_dmars();
ba395927 4516 if (ret) {
a59b50e9
JC		 4517 if (force_on)
4518 panic("tboot: Failed to initialize DMARs\n");
9f10e5bf 4519 pr_err("Initialization failed\n");
9bdc531e 4520 goto out_free_reserved_range;
ba395927 4521 }
3a5670e8 4522 up_write(&dmar_global_lock);
9f10e5bf 4523 pr_info("Intel(R) Virtualization Technology for Directed I/O\n");
ba395927 4524
5e0d2a6f 4525 init_timer(&unmap_timer);
75f1cdf1
FT		 4526#ifdef CONFIG_SWIOTLB
4527 swiotlb = 0;
4528#endif
19943b0e 4529 dma_ops = &intel_dma_ops;
4ed0d3e6 4530
134fac3f 4531 init_iommu_pm_ops();
a8bcbb0d 4532
a5459cfe
AW		 4533 for_each_active_iommu(iommu, drhd)
4534 iommu->iommu_dev = iommu_device_create(NULL, iommu,
4535 intel_iommu_groups,
4536 iommu->name);
4537
4236d97d 4538 bus_set_iommu(&pci_bus_type, &intel_iommu_ops);
99dcaded 4539 bus_register_notifier(&pci_bus_type, &device_nb);
75f05569
JL		 4540 if (si_domain && !hw_pass_through)
4541 register_memory_notifier(&intel_iommu_memory_nb);
99dcaded 4542
8bc1f85c
ED		 4543 intel_iommu_enabled = 1;
4544
ba395927 4545 return 0;
9bdc531e
JL		 4546
4547out_free_reserved_range:
4548 put_iova_domain(&reserved_iova_list);
9bdc531e
JL		 4549out_free_dmar:
4550 intel_iommu_free_dmars();
3a5670e8
JL		 4551 up_write(&dmar_global_lock);
4552 iommu_exit_mempool();
9bdc531e 4553 return ret;
ba395927 4554}
e820482c 4555
579305f7
AW		 4556static int iommu_detach_dev_cb(struct pci_dev *pdev, u16 alias, void *opaque)
4557{
4558 struct intel_iommu *iommu = opaque;
4559
4560 iommu_detach_dev(iommu, PCI_BUS_NUM(alias), alias & 0xff);
4561 return 0;
4562}
4563
4564/*
4565 * NB - intel-iommu lacks any sort of reference counting for the users of
4566 * dependent devices. If multiple endpoints have intersecting dependent
4567 * devices, unbinding the driver from any one of them will possibly leave
4568 * the others unable to operate.
4569 */
3199aa6b 4570static void iommu_detach_dependent_devices(struct intel_iommu *iommu,
0bcb3e28 4571 struct device *dev)
3199aa6b 4572{
0bcb3e28 4573 if (!iommu || !dev || !dev_is_pci(dev))
3199aa6b
HW		 4574 return;
4575
579305f7 4576 pci_for_each_dma_alias(to_pci_dev(dev), &iommu_detach_dev_cb, iommu);
3199aa6b
HW		 4577}
4578
2c2e2c38 4579static void domain_remove_one_dev_info(struct dmar_domain *domain,
bf9c9eda 4580 struct device *dev)
c7151a8d 4581{
bca2b916 4582 struct device_domain_info *info, *tmp;
c7151a8d
WH		 4583 struct intel_iommu *iommu;
4584 unsigned long flags;
2f119c78 4585 bool found = false;
156baca8 4586 u8 bus, devfn;
c7151a8d 4587
bf9c9eda 4588 iommu = device_to_iommu(dev, &bus, &devfn);
c7151a8d
WH		 4589 if (!iommu)
4590 return;
4591
4592 spin_lock_irqsave(&device_domain_lock, flags);
bca2b916 4593 list_for_each_entry_safe(info, tmp, &domain->devices, link) {
bf9c9eda
DW		 4594 if (info->iommu == iommu && info->bus == bus &&
4595 info->devfn == devfn) {
109b9b04 4596 unlink_domain_info(info);
c7151a8d
WH		 4597 spin_unlock_irqrestore(&device_domain_lock, flags);
4598
93a23a72 4599 iommu_disable_dev_iotlb(info);
c7151a8d 4600 iommu_detach_dev(iommu, info->bus, info->devfn);
bf9c9eda 4601 iommu_detach_dependent_devices(iommu, dev);
c7151a8d
WH		 4602 free_devinfo_mem(info);
4603
4604 spin_lock_irqsave(&device_domain_lock, flags);
4605
4606 if (found)
4607 break;
4608 else
4609 continue;
4610 }
4611
4612 /* if there is no other devices under the same iommu
4613 * owned by this domain, clear this iommu in iommu_bmp
4614 * update iommu count and coherency
4615 */
8bbc4410 4616 if (info->iommu == iommu)
2f119c78 4617 found = true;
c7151a8d
WH		 4618 }
4619
3e7abe25
RD		 4620 spin_unlock_irqrestore(&device_domain_lock, flags);
4621
c7151a8d 4622 if (found == 0) {
fb170fb4
JL		 4623 domain_detach_iommu(domain, iommu);
4624 if (!domain_type_is_vm_or_si(domain))
4625 iommu_detach_domain(domain, iommu);
c7151a8d 4626 }
c7151a8d
WH		 4627}
4628
2c2e2c38 4629static int md_domain_init(struct dmar_domain *domain, int guest_width)
5e98c4b1
WH		 4630{
4631 int adjust_width;
4632
0fb5fe87
RM		 4633 init_iova_domain(&domain->iovad, VTD_PAGE_SIZE, IOVA_START_PFN,
4634 DMA_32BIT_PFN);
5e98c4b1
WH		 4635 domain_reserve_special_ranges(domain);
4636
4637 /* calculate AGAW */
4638 domain->gaw = guest_width;
4639 adjust_width = guestwidth_to_adjustwidth(guest_width);
4640 domain->agaw = width_to_agaw(adjust_width);
4641
5e98c4b1 4642 domain->iommu_coherency = 0;
c5b15255 4643 domain->iommu_snooping = 0;
6dd9a7c7 4644 domain->iommu_superpage = 0;
fe40f1e0 4645 domain->max_addr = 0;
5e98c4b1
WH		 4646
4647 /* always allocate the top pgd */
4c923d47 4648 domain->pgd = (struct dma_pte *)alloc_pgtable_page(domain->nid);
5e98c4b1
WH		 4649 if (!domain->pgd)
4650 return -ENOMEM;
4651 domain_flush_cache(domain, domain->pgd, PAGE_SIZE);
4652 return 0;
4653}
4654
00a77deb 4655static struct iommu_domain *intel_iommu_domain_alloc(unsigned type)
38717946 4656{
5d450806 4657 struct dmar_domain *dmar_domain;
00a77deb
JR		 4658 struct iommu_domain *domain;
4659
4660 if (type != IOMMU_DOMAIN_UNMANAGED)
4661 return NULL;
38717946 4662
ab8dfe25 4663 dmar_domain = alloc_domain(DOMAIN_FLAG_VIRTUAL_MACHINE);
5d450806 4664 if (!dmar_domain) {
9f10e5bf 4665 pr_err("Can't allocate dmar_domain\n");
00a77deb 4666 return NULL;
38717946 4667 }
2c2e2c38 4668 if (md_domain_init(dmar_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
9f10e5bf 4669 pr_err("Domain initialization failed\n");
92d03cc8 4670 domain_exit(dmar_domain);
00a77deb 4671 return NULL;
38717946 4672 }
8140a95d 4673 domain_update_iommu_cap(dmar_domain);
faa3d6f5 4674
00a77deb 4675 domain = &dmar_domain->domain;
8a0e715b
JR		 4676 domain->geometry.aperture_start = 0;
4677 domain->geometry.aperture_end = __DOMAIN_MAX_ADDR(dmar_domain->gaw);
4678 domain->geometry.force_aperture = true;
4679
00a77deb 4680 return domain;
38717946 4681}
38717946 4682
00a77deb 4683static void intel_iommu_domain_free(struct iommu_domain *domain)
38717946 4684{
00a77deb 4685 domain_exit(to_dmar_domain(domain));
38717946 4686}
38717946 4687
4c5478c9
JR		 4688static int intel_iommu_attach_device(struct iommu_domain *domain,
4689 struct device *dev)
38717946 4690{
00a77deb 4691 struct dmar_domain *dmar_domain = to_dmar_domain(domain);
fe40f1e0
WH		 4692 struct intel_iommu *iommu;
4693 int addr_width;
156baca8 4694 u8 bus, devfn;
faa3d6f5 4695
c875d2c1
AW		 4696 if (device_is_rmrr_locked(dev)) {
4697 dev_warn(dev, "Device is ineligible for IOMMU domain attach due to platform RMRR requirement. Contact your platform vendor.\n");
4698 return -EPERM;
4699 }
4700
7207d8f9
DW		 4701 /* normally dev is not mapped */
4702 if (unlikely(domain_context_mapped(dev))) {
faa3d6f5
WH		 4703 struct dmar_domain *old_domain;
4704
1525a29a 4705 old_domain = find_domain(dev);
faa3d6f5 4706 if (old_domain) {
ab8dfe25 4707 if (domain_type_is_vm_or_si(dmar_domain))
bf9c9eda 4708 domain_remove_one_dev_info(old_domain, dev);
faa3d6f5
WH		 4709 else
4710 domain_remove_dev_info(old_domain);
62c22167
JR		 4711
4712 if (!domain_type_is_vm_or_si(old_domain) &&
4713 list_empty(&old_domain->devices))
4714 domain_exit(old_domain);
faa3d6f5
WH		 4715 }
4716 }
4717
156baca8 4718 iommu = device_to_iommu(dev, &bus, &devfn);
fe40f1e0
WH		 4719 if (!iommu)
4720 return -ENODEV;
4721
4722 /* check if this iommu agaw is sufficient for max mapped address */
4723 addr_width = agaw_to_width(iommu->agaw);
a99c47a2
TL		 4724 if (addr_width > cap_mgaw(iommu->cap))
4725 addr_width = cap_mgaw(iommu->cap);
4726
4727 if (dmar_domain->max_addr > (1LL << addr_width)) {
9f10e5bf 4728 pr_err("%s: iommu width (%d) is not "
fe40f1e0 4729 "sufficient for the mapped address (%llx)\n",
a99c47a2 4730 __func__, addr_width, dmar_domain->max_addr);
fe40f1e0
WH		 4731 return -EFAULT;
4732 }
a99c47a2
TL		 4733 dmar_domain->gaw = addr_width;
4734
4735 /*
4736 * Knock out extra levels of page tables if necessary
4737 */
4738 while (iommu->agaw < dmar_domain->agaw) {
4739 struct dma_pte *pte;
4740
4741 pte = dmar_domain->pgd;
4742 if (dma_pte_present(pte)) {
25cbff16
SY		 4743 dmar_domain->pgd = (struct dma_pte *)
4744 phys_to_virt(dma_pte_addr(pte));
7a661013 4745 free_pgtable_page(pte);
a99c47a2
TL		 4746 }
4747 dmar_domain->agaw--;
4748 }
fe40f1e0 4749
5913c9bf 4750 return domain_add_dev_info(dmar_domain, dev, CONTEXT_TT_MULTI_LEVEL);
38717946 4751}
38717946 4752
4c5478c9
JR		 4753static void intel_iommu_detach_device(struct iommu_domain *domain,
4754 struct device *dev)
38717946 4755{
00a77deb 4756 domain_remove_one_dev_info(to_dmar_domain(domain), dev);
faa3d6f5 4757}
c7151a8d 4758
b146a1c9
JR		 4759static int intel_iommu_map(struct iommu_domain *domain,
4760 unsigned long iova, phys_addr_t hpa,
5009065d 4761 size_t size, int iommu_prot)
faa3d6f5 4762{
00a77deb 4763 struct dmar_domain *dmar_domain = to_dmar_domain(domain);
fe40f1e0 4764 u64 max_addr;
dde57a21 4765 int prot = 0;
faa3d6f5 4766 int ret;
fe40f1e0 4767
dde57a21
JR		 4768 if (iommu_prot & IOMMU_READ)
4769 prot |= DMA_PTE_READ;
4770 if (iommu_prot & IOMMU_WRITE)
4771 prot |= DMA_PTE_WRITE;
9cf06697
SY		 4772 if ((iommu_prot & IOMMU_CACHE) && dmar_domain->iommu_snooping)
4773 prot |= DMA_PTE_SNP;
dde57a21 4774
163cc52c 4775 max_addr = iova + size;
dde57a21 4776 if (dmar_domain->max_addr < max_addr) {
fe40f1e0
WH		 4777 u64 end;
4778
4779 /* check if minimum agaw is sufficient for mapped address */
8954da1f 4780 end = __DOMAIN_MAX_ADDR(dmar_domain->gaw) + 1;
fe40f1e0 4781 if (end < max_addr) {
9f10e5bf 4782 pr_err("%s: iommu width (%d) is not "
fe40f1e0 4783 "sufficient for the mapped address (%llx)\n",
8954da1f 4784 __func__, dmar_domain->gaw, max_addr);
fe40f1e0
WH		 4785 return -EFAULT;
4786 }
dde57a21 4787 dmar_domain->max_addr = max_addr;
fe40f1e0 4788 }
ad051221
DW		 4789 /* Round up size to next multiple of PAGE_SIZE, if it and
4790 the low bits of hpa would take us onto the next page */
88cb6a74 4791 size = aligned_nrpages(hpa, size);
ad051221
DW		 4792 ret = domain_pfn_mapping(dmar_domain, iova >> VTD_PAGE_SHIFT,
4793 hpa >> VTD_PAGE_SHIFT, size, prot);
faa3d6f5 4794 return ret;
38717946 4795}
38717946 4796
5009065d 4797static size_t intel_iommu_unmap(struct iommu_domain *domain,
ea8ea460 4798 unsigned long iova, size_t size)
38717946 4799{
00a77deb 4800 struct dmar_domain *dmar_domain = to_dmar_domain(domain);
ea8ea460
DW		 4801 struct page *freelist = NULL;
4802 struct intel_iommu *iommu;
4803 unsigned long start_pfn, last_pfn;
4804 unsigned int npages;
4805 int iommu_id, num, ndomains, level = 0;
5cf0a76f
DW		 4806
4807 /* Cope with horrid API which requires us to unmap more than the
4808 size argument if it happens to be a large-page mapping. */
4809 if (!pfn_to_dma_pte(dmar_domain, iova >> VTD_PAGE_SHIFT, &level))
4810 BUG();
4811
4812 if (size < VTD_PAGE_SIZE << level_to_offset_bits(level))
4813 size = VTD_PAGE_SIZE << level_to_offset_bits(level);
4b99d352 4814
ea8ea460
DW		 4815 start_pfn = iova >> VTD_PAGE_SHIFT;
4816 last_pfn = (iova + size - 1) >> VTD_PAGE_SHIFT;
4817
4818 freelist = domain_unmap(dmar_domain, start_pfn, last_pfn);
4819
4820 npages = last_pfn - start_pfn + 1;
4821
4822 for_each_set_bit(iommu_id, dmar_domain->iommu_bmp, g_num_of_iommus) {
4823 iommu = g_iommus[iommu_id];
4824
4825 /*
4826 * find bit position of dmar_domain
4827 */
4828 ndomains = cap_ndoms(iommu->cap);
4829 for_each_set_bit(num, iommu->domain_ids, ndomains) {
4830 if (iommu->domains[num] == dmar_domain)
4831 iommu_flush_iotlb_psi(iommu, num, start_pfn,
4832 npages, !freelist, 0);
4833 }
4834
4835 }
4836
4837 dma_free_pagelist(freelist);
fe40f1e0 4838
163cc52c
DW		 4839 if (dmar_domain->max_addr == iova + size)
4840 dmar_domain->max_addr = iova;
b146a1c9 4841
5cf0a76f 4842 return size;
38717946 4843}
38717946 4844
d14d6577 4845static phys_addr_t intel_iommu_iova_to_phys(struct iommu_domain *domain,
bb5547ac 4846 dma_addr_t iova)
38717946 4847{
00a77deb 4848 struct dmar_domain *dmar_domain = to_dmar_domain(domain);
38717946 4849 struct dma_pte *pte;
5cf0a76f 4850 int level = 0;
faa3d6f5 4851 u64 phys = 0;
38717946 4852
5cf0a76f 4853 pte = pfn_to_dma_pte(dmar_domain, iova >> VTD_PAGE_SHIFT, &level);
38717946 4854 if (pte)
faa3d6f5 4855 phys = dma_pte_addr(pte);
38717946 4856
faa3d6f5 4857 return phys;
38717946 4858}
a8bcbb0d 4859
5d587b8d 4860static bool intel_iommu_capable(enum iommu_cap cap)
dbb9fd86 4861{
dbb9fd86 4862 if (cap == IOMMU_CAP_CACHE_COHERENCY)
5d587b8d 4863 return domain_update_iommu_snooping(NULL) == 1;
323f99cb 4864 if (cap == IOMMU_CAP_INTR_REMAP)
5d587b8d 4865 return irq_remapping_enabled == 1;
dbb9fd86 4866
5d587b8d 4867 return false;
dbb9fd86
SY		 4868}
4869
abdfdde2
AW		 4870static int intel_iommu_add_device(struct device *dev)
4871{
a5459cfe 4872 struct intel_iommu *iommu;
abdfdde2 4873 struct iommu_group *group;
156baca8 4874 u8 bus, devfn;
70ae6f0d 4875
a5459cfe
AW		 4876 iommu = device_to_iommu(dev, &bus, &devfn);
4877 if (!iommu)
70ae6f0d
AW		 4878 return -ENODEV;
4879
a5459cfe 4880 iommu_device_link(iommu->iommu_dev, dev);
a4ff1fc2 4881
e17f9ff4 4882 group = iommu_group_get_for_dev(dev);
783f157b 4883
e17f9ff4
AW		 4884 if (IS_ERR(group))
4885 return PTR_ERR(group);
bcb71abe 4886
abdfdde2 4887 iommu_group_put(group);
e17f9ff4 4888 return 0;
abdfdde2 4889}
70ae6f0d 4890
abdfdde2
AW		 4891static void intel_iommu_remove_device(struct device *dev)
4892{
a5459cfe
AW		 4893 struct intel_iommu *iommu;
4894 u8 bus, devfn;
4895
4896 iommu = device_to_iommu(dev, &bus, &devfn);
4897 if (!iommu)
4898 return;
4899
abdfdde2 4900 iommu_group_remove_device(dev);
a5459cfe
AW		 4901
4902 iommu_device_unlink(iommu->iommu_dev, dev);
70ae6f0d
AW		 4903}
4904
b22f6434 4905static const struct iommu_ops intel_iommu_ops = {
5d587b8d 4906 .capable = intel_iommu_capable,
00a77deb
JR		 4907 .domain_alloc = intel_iommu_domain_alloc,
4908 .domain_free = intel_iommu_domain_free,
a8bcbb0d
JR		 4909 .attach_dev = intel_iommu_attach_device,
4910 .detach_dev = intel_iommu_detach_device,
b146a1c9
JR		 4911 .map = intel_iommu_map,
4912 .unmap = intel_iommu_unmap,
315786eb 4913 .map_sg = default_iommu_map_sg,
a8bcbb0d 4914 .iova_to_phys = intel_iommu_iova_to_phys,
abdfdde2
AW		 4915 .add_device = intel_iommu_add_device,
4916 .remove_device = intel_iommu_remove_device,
6d1c56a9 4917 .pgsize_bitmap = INTEL_IOMMU_PGSIZES,
a8bcbb0d 4918};
9af88143 4919
9452618e
DV		 4920static void quirk_iommu_g4x_gfx(struct pci_dev *dev)
4921{
4922 /* G4x/GM45 integrated gfx dmar support is totally busted. */
9f10e5bf 4923 pr_info("Disabling IOMMU for graphics on this chipset\n");
9452618e
DV		 4924 dmar_map_gfx = 0;
4925}
4926
4927DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2a40, quirk_iommu_g4x_gfx);
4928DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e00, quirk_iommu_g4x_gfx);
4929DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e10, quirk_iommu_g4x_gfx);
4930DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e20, quirk_iommu_g4x_gfx);
4931DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e30, quirk_iommu_g4x_gfx);
4932DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e40, quirk_iommu_g4x_gfx);
4933DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e90, quirk_iommu_g4x_gfx);
4934
d34d6517 4935static void quirk_iommu_rwbf(struct pci_dev *dev)
9af88143
DW		 4936{
4937 /*
4938 * Mobile 4 Series Chipset neglects to set RWBF capability,
210561ff 4939 * but needs it. Same seems to hold for the desktop versions.
9af88143 4940 */
9f10e5bf 4941 pr_info("Forcing write-buffer flush capability\n");
9af88143
DW		 4942 rwbf_quirk = 1;
4943}
4944
4945DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2a40, quirk_iommu_rwbf);
210561ff
DV		 4946DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e00, quirk_iommu_rwbf);
4947DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e10, quirk_iommu_rwbf);
4948DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e20, quirk_iommu_rwbf);
4949DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e30, quirk_iommu_rwbf);
4950DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e40, quirk_iommu_rwbf);
4951DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e90, quirk_iommu_rwbf);
e0fc7e0b 4952
eecfd57f
AJ		 4953#define GGC 0x52
4954#define GGC_MEMORY_SIZE_MASK (0xf << 8)
4955#define GGC_MEMORY_SIZE_NONE (0x0 << 8)
4956#define GGC_MEMORY_SIZE_1M (0x1 << 8)
4957#define GGC_MEMORY_SIZE_2M (0x3 << 8)
4958#define GGC_MEMORY_VT_ENABLED (0x8 << 8)
4959#define GGC_MEMORY_SIZE_2M_VT (0x9 << 8)
4960#define GGC_MEMORY_SIZE_3M_VT (0xa << 8)
4961#define GGC_MEMORY_SIZE_4M_VT (0xb << 8)
4962
d34d6517 4963static void quirk_calpella_no_shadow_gtt(struct pci_dev *dev)
9eecabcb
DW		 4964{
4965 unsigned short ggc;
4966
eecfd57f 4967 if (pci_read_config_word(dev, GGC, &ggc))
9eecabcb
DW		 4968 return;
4969
eecfd57f 4970 if (!(ggc & GGC_MEMORY_VT_ENABLED)) {
9f10e5bf 4971 pr_info("BIOS has allocated no shadow GTT; disabling IOMMU for graphics\n");
9eecabcb 4972 dmar_map_gfx = 0;
6fbcfb3e
DW		 4973 } else if (dmar_map_gfx) {
4974 /* we have to ensure the gfx device is idle before we flush */
9f10e5bf 4975 pr_info("Disabling batched IOTLB flush on Ironlake\n");
6fbcfb3e
DW		 4976 intel_iommu_strict = 1;
4977 }
9eecabcb
DW		 4978}
4979DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0040, quirk_calpella_no_shadow_gtt);
4980DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0044, quirk_calpella_no_shadow_gtt);
4981DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0062, quirk_calpella_no_shadow_gtt);
4982DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x006a, quirk_calpella_no_shadow_gtt);
4983
e0fc7e0b
DW		 4984/* On Tylersburg chipsets, some BIOSes have been known to enable the
4985 ISOCH DMAR unit for the Azalia sound device, but not give it any
4986 TLB entries, which causes it to deadlock. Check for that. We do
4987 this in a function called from init_dmars(), instead of in a PCI
4988 quirk, because we don't want to print the obnoxious "BIOS broken"
4989 message if VT-d is actually disabled.
4990*/
4991static void __init check_tylersburg_isoch(void)
4992{
4993 struct pci_dev *pdev;
4994 uint32_t vtisochctrl;
4995
4996 /* If there's no Azalia in the system anyway, forget it. */
4997 pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x3a3e, NULL);
4998 if (!pdev)
4999 return;
5000 pci_dev_put(pdev);
5001
5002 /* System Management Registers. Might be hidden, in which case
5003 we can't do the sanity check. But that's OK, because the
5004 known-broken BIOSes _don't_ actually hide it, so far. */
5005 pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x342e, NULL);
5006 if (!pdev)
5007 return;
5008
5009 if (pci_read_config_dword(pdev, 0x188, &vtisochctrl)) {
5010 pci_dev_put(pdev);
5011 return;
5012 }
5013
5014 pci_dev_put(pdev);
5015
5016 /* If Azalia DMA is routed to the non-isoch DMAR unit, fine. */
5017 if (vtisochctrl & 1)
5018 return;
5019
5020 /* Drop all bits other than the number of TLB entries */
5021 vtisochctrl &= 0x1c;
5022
5023 /* If we have the recommended number of TLB entries (16), fine. */
5024 if (vtisochctrl == 0x10)
5025 return;
5026
5027 /* Zero TLB entries? You get to ride the short bus to school. */
5028 if (!vtisochctrl) {
5029 WARN(1, "Your BIOS is broken; DMA routed to ISOCH DMAR unit but no TLB space.\n"
5030 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
5031 dmi_get_system_info(DMI_BIOS_VENDOR),
5032 dmi_get_system_info(DMI_BIOS_VERSION),
5033 dmi_get_system_info(DMI_PRODUCT_VERSION));
5034 iommu_identity_mapping |= IDENTMAP_AZALIA;
5035 return;
5036 }
9f10e5bf
JR		 5037
5038 pr_warn("Recommended TLB entries for ISOCH unit is 16; your BIOS set %d\n",
e0fc7e0b
DW		 5039 vtisochctrl);
5040}
Linux kernel - Deliverables
This page took 0.987318 seconds and 5 git commands to generate.