projects / deliverable / linux.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
iommu/vt-d: Calculate translation in domain_context_mapping_one
[deliverable/linux.git] / drivers / iommu / intel-iommu.c
CommitLineData
ba395927 1/*
ea8ea460 2 * Copyright © 2006-2014 Intel Corporation.
ba395927
KA		 3 *
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms and conditions of the GNU General Public License,
6 * version 2, as published by the Free Software Foundation.
7 *
8 * This program is distributed in the hope it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
11 * more details.
12 *
ea8ea460
DW		 13 * Authors: David Woodhouse <dwmw2@infradead.org>,
14 * Ashok Raj <ashok.raj@intel.com>,
15 * Shaohua Li <shaohua.li@intel.com>,
16 * Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>,
17 * Fenghua Yu <fenghua.yu@intel.com>
9f10e5bf 18 * Joerg Roedel <jroedel@suse.de>
ba395927
KA		 19 */
20
9f10e5bf
JR		 21#define pr_fmt(fmt) "DMAR: " fmt
22
ba395927
KA		 23#include <linux/init.h>
24#include <linux/bitmap.h>
5e0d2a6f 25#include <linux/debugfs.h>
54485c30 26#include <linux/export.h>
ba395927
KA		 27#include <linux/slab.h>
28#include <linux/irq.h>
29#include <linux/interrupt.h>
ba395927
KA		 30#include <linux/spinlock.h>
31#include <linux/pci.h>
32#include <linux/dmar.h>
33#include <linux/dma-mapping.h>
34#include <linux/mempool.h>
75f05569 35#include <linux/memory.h>
5e0d2a6f 36#include <linux/timer.h>
38717946 37#include <linux/iova.h>
5d450806 38#include <linux/iommu.h>
38717946 39#include <linux/intel-iommu.h>
134fac3f 40#include <linux/syscore_ops.h>
69575d38 41#include <linux/tboot.h>
adb2fe02 42#include <linux/dmi.h>
5cdede24 43#include <linux/pci-ats.h>
0ee332c1 44#include <linux/memblock.h>
36746436 45#include <linux/dma-contiguous.h>
091d42e4 46#include <linux/crash_dump.h>
8a8f422d 47#include <asm/irq_remapping.h>
ba395927 48#include <asm/cacheflush.h>
46a7fa27 49#include <asm/iommu.h>
ba395927 50
078e1ee2
JR		 51#include "irq_remapping.h"
52
5b6985ce
FY		 53#define ROOT_SIZE VTD_PAGE_SIZE
54#define CONTEXT_SIZE VTD_PAGE_SIZE
55
ba395927 56#define IS_GFX_DEVICE(pdev) ((pdev->class >> 16) == PCI_BASE_CLASS_DISPLAY)
18436afd 57#define IS_USB_DEVICE(pdev) ((pdev->class >> 8) == PCI_CLASS_SERIAL_USB)
ba395927 58#define IS_ISA_DEVICE(pdev) ((pdev->class >> 8) == PCI_CLASS_BRIDGE_ISA)
e0fc7e0b 59#define IS_AZALIA(pdev) ((pdev)->vendor == 0x8086 && (pdev)->device == 0x3a3e)
ba395927
KA		 60
61#define IOAPIC_RANGE_START (0xfee00000)
62#define IOAPIC_RANGE_END (0xfeefffff)
63#define IOVA_START_ADDR (0x1000)
64
65#define DEFAULT_DOMAIN_ADDRESS_WIDTH 48
66
4ed0d3e6 67#define MAX_AGAW_WIDTH 64
5c645b35 68#define MAX_AGAW_PFN_WIDTH (MAX_AGAW_WIDTH - VTD_PAGE_SHIFT)
4ed0d3e6 69
2ebe3151
DW		 70#define __DOMAIN_MAX_PFN(gaw) ((((uint64_t)1) << (gaw-VTD_PAGE_SHIFT)) - 1)
71#define __DOMAIN_MAX_ADDR(gaw) ((((uint64_t)1) << gaw) - 1)
72
73/* We limit DOMAIN_MAX_PFN to fit in an unsigned long, and DOMAIN_MAX_ADDR
74 to match. That way, we can use 'unsigned long' for PFNs with impunity. */
75#define DOMAIN_MAX_PFN(gaw) ((unsigned long) min_t(uint64_t, \
76 __DOMAIN_MAX_PFN(gaw), (unsigned long)-1))
77#define DOMAIN_MAX_ADDR(gaw) (((uint64_t)__DOMAIN_MAX_PFN(gaw)) << VTD_PAGE_SHIFT)
ba395927 78
1b722500
RM		 79/* IO virtual address start page frame number */
80#define IOVA_START_PFN (1)
81
f27be03b 82#define IOVA_PFN(addr) ((addr) >> PAGE_SHIFT)
284901a9 83#define DMA_32BIT_PFN IOVA_PFN(DMA_BIT_MASK(32))
6a35528a 84#define DMA_64BIT_PFN IOVA_PFN(DMA_BIT_MASK(64))
5e0d2a6f 85
df08cdc7
AM		 86/* page table handling */
87#define LEVEL_STRIDE (9)
88#define LEVEL_MASK (((u64)1 << LEVEL_STRIDE) - 1)
89
6d1c56a9
OBC		 90/*
91 * This bitmap is used to advertise the page sizes our hardware support
92 * to the IOMMU core, which will then use this information to split
93 * physically contiguous memory regions it is mapping into page sizes
94 * that we support.
95 *
96 * Traditionally the IOMMU core just handed us the mappings directly,
97 * after making sure the size is an order of a 4KiB page and that the
98 * mapping has natural alignment.
99 *
100 * To retain this behavior, we currently advertise that we support
101 * all page sizes that are an order of 4KiB.
102 *
103 * If at some point we'd like to utilize the IOMMU core's new behavior,
104 * we could change this to advertise the real page sizes we support.
105 */
106#define INTEL_IOMMU_PGSIZES (~0xFFFUL)
107
df08cdc7
AM		 108static inline int agaw_to_level(int agaw)
109{
110 return agaw + 2;
111}
112
113static inline int agaw_to_width(int agaw)
114{
5c645b35 115 return min_t(int, 30 + agaw * LEVEL_STRIDE, MAX_AGAW_WIDTH);
df08cdc7
AM		 116}
117
118static inline int width_to_agaw(int width)
119{
5c645b35 120 return DIV_ROUND_UP(width - 30, LEVEL_STRIDE);
df08cdc7
AM		 121}
122
123static inline unsigned int level_to_offset_bits(int level)
124{
125 return (level - 1) * LEVEL_STRIDE;
126}
127
128static inline int pfn_level_offset(unsigned long pfn, int level)
129{
130 return (pfn >> level_to_offset_bits(level)) & LEVEL_MASK;
131}
132
133static inline unsigned long level_mask(int level)
134{
135 return -1UL << level_to_offset_bits(level);
136}
137
138static inline unsigned long level_size(int level)
139{
140 return 1UL << level_to_offset_bits(level);
141}
142
143static inline unsigned long align_to_level(unsigned long pfn, int level)
144{
145 return (pfn + level_size(level) - 1) & level_mask(level);
146}
fd18de50 147
6dd9a7c7
YS		 148static inline unsigned long lvl_to_nr_pages(unsigned int lvl)
149{
5c645b35 150 return 1 << min_t(int, (lvl - 1) * LEVEL_STRIDE, MAX_AGAW_PFN_WIDTH);
6dd9a7c7
YS		 151}
152
dd4e8319
DW		 153/* VT-d pages must always be _smaller_ than MM pages. Otherwise things
154 are never going to work. */
155static inline unsigned long dma_to_mm_pfn(unsigned long dma_pfn)
156{
157 return dma_pfn >> (PAGE_SHIFT - VTD_PAGE_SHIFT);
158}
159
160static inline unsigned long mm_to_dma_pfn(unsigned long mm_pfn)
161{
162 return mm_pfn << (PAGE_SHIFT - VTD_PAGE_SHIFT);
163}
164static inline unsigned long page_to_dma_pfn(struct page *pg)
165{
166 return mm_to_dma_pfn(page_to_pfn(pg));
167}
168static inline unsigned long virt_to_dma_pfn(void *p)
169{
170 return page_to_dma_pfn(virt_to_page(p));
171}
172
d9630fe9
WH		 173/* global iommu list, set NULL for ignored DMAR units */
174static struct intel_iommu **g_iommus;
175
e0fc7e0b 176static void __init check_tylersburg_isoch(void);
9af88143
DW		 177static int rwbf_quirk;
178
b779260b
JC		 179/*
180 * set to 1 to panic kernel if can't successfully enable VT-d
181 * (used when kernel is launched w/ TXT)
182 */
183static int force_on = 0;
184
46b08e1a
MM		 185/*
186 * 0: Present
187 * 1-11: Reserved
188 * 12-63: Context Ptr (12 - (haw-1))
189 * 64-127: Reserved
190 */
191struct root_entry {
03ecc32c
DW		 192 u64 lo;
193 u64 hi;
46b08e1a
MM		 194};
195#define ROOT_ENTRY_NR (VTD_PAGE_SIZE/sizeof(struct root_entry))
46b08e1a 196
091d42e4
JR		 197/*
198 * Take a root_entry and return the Lower Context Table Pointer (LCTP)
199 * if marked present.
200 */
201static phys_addr_t root_entry_lctp(struct root_entry *re)
202{
203 if (!(re->lo & 1))
204 return 0;
205
206 return re->lo & VTD_PAGE_MASK;
207}
208
209/*
210 * Take a root_entry and return the Upper Context Table Pointer (UCTP)
211 * if marked present.
212 */
213static phys_addr_t root_entry_uctp(struct root_entry *re)
214{
215 if (!(re->hi & 1))
216 return 0;
46b08e1a 217
091d42e4
JR		 218 return re->hi & VTD_PAGE_MASK;
219}
7a8fc25e
MM		 220/*
221 * low 64 bits:
222 * 0: present
223 * 1: fault processing disable
224 * 2-3: translation type
225 * 12-63: address space root
226 * high 64 bits:
227 * 0-2: address width
228 * 3-6: aval
229 * 8-23: domain id
230 */
231struct context_entry {
232 u64 lo;
233 u64 hi;
234};
c07e7d21 235
cf484d0e
JR		 236static inline void context_clear_pasid_enable(struct context_entry *context)
237{
238 context->lo &= ~(1ULL << 11);
239}
240
241static inline bool context_pasid_enabled(struct context_entry *context)
242{
243 return !!(context->lo & (1ULL << 11));
244}
245
246static inline void context_set_copied(struct context_entry *context)
247{
248 context->hi |= (1ull << 3);
249}
250
251static inline bool context_copied(struct context_entry *context)
252{
253 return !!(context->hi & (1ULL << 3));
254}
255
256static inline bool __context_present(struct context_entry *context)
c07e7d21
MM		 257{
258 return (context->lo & 1);
259}
cf484d0e
JR		 260
261static inline bool context_present(struct context_entry *context)
262{
263 return context_pasid_enabled(context) ?
264 __context_present(context) :
265 __context_present(context) && !context_copied(context);
266}
267
c07e7d21
MM		 268static inline void context_set_present(struct context_entry *context)
269{
270 context->lo |= 1;
271}
272
273static inline void context_set_fault_enable(struct context_entry *context)
274{
275 context->lo &= (((u64)-1) << 2) | 1;
276}
277
c07e7d21
MM		 278static inline void context_set_translation_type(struct context_entry *context,
279 unsigned long value)
280{
281 context->lo &= (((u64)-1) << 4) | 3;
282 context->lo |= (value & 3) << 2;
283}
284
285static inline void context_set_address_root(struct context_entry *context,
286 unsigned long value)
287{
1a2262f9 288 context->lo &= ~VTD_PAGE_MASK;
c07e7d21
MM		 289 context->lo |= value & VTD_PAGE_MASK;
290}
291
292static inline void context_set_address_width(struct context_entry *context,
293 unsigned long value)
294{
295 context->hi |= value & 7;
296}
297
298static inline void context_set_domain_id(struct context_entry *context,
299 unsigned long value)
300{
301 context->hi |= (value & ((1 << 16) - 1)) << 8;
302}
303
dbcd861f
JR		 304static inline int context_domain_id(struct context_entry *c)
305{
306 return((c->hi >> 8) & 0xffff);
307}
308
c07e7d21
MM		 309static inline void context_clear_entry(struct context_entry *context)
310{
311 context->lo = 0;
312 context->hi = 0;
313}
7a8fc25e 314
622ba12a
MM		 315/*
316 * 0: readable
317 * 1: writable
318 * 2-6: reserved
319 * 7: super page
9cf06697
SY		 320 * 8-10: available
321 * 11: snoop behavior
622ba12a
MM		 322 * 12-63: Host physcial address
323 */
324struct dma_pte {
325 u64 val;
326};
622ba12a 327
19c239ce
MM		 328static inline void dma_clear_pte(struct dma_pte *pte)
329{
330 pte->val = 0;
331}
332
19c239ce
MM		 333static inline u64 dma_pte_addr(struct dma_pte *pte)
334{
c85994e4
DW		 335#ifdef CONFIG_64BIT
336 return pte->val & VTD_PAGE_MASK;
337#else
338 /* Must have a full atomic 64-bit read */
1a8bd481 339 return __cmpxchg64(&pte->val, 0ULL, 0ULL) & VTD_PAGE_MASK;
c85994e4 340#endif
19c239ce
MM		 341}
342
19c239ce
MM		 343static inline bool dma_pte_present(struct dma_pte *pte)
344{
345 return (pte->val & 3) != 0;
346}
622ba12a 347
4399c8bf
AK		 348static inline bool dma_pte_superpage(struct dma_pte *pte)
349{
c3c75eb7 350 return (pte->val & DMA_PTE_LARGE_PAGE);
4399c8bf
AK		 351}
352
75e6bf96
DW		 353static inline int first_pte_in_page(struct dma_pte *pte)
354{
355 return !((unsigned long)pte & ~VTD_PAGE_MASK);
356}
357
2c2e2c38
FY		 358/*
359 * This domain is a statically identity mapping domain.
360 * 1. This domain creats a static 1:1 mapping to all usable memory.
361 * 2. It maps to each iommu if successful.
362 * 3. Each iommu mapps to this domain if successful.
363 */
19943b0e
DW		 364static struct dmar_domain *si_domain;
365static int hw_pass_through = 1;
2c2e2c38 366
28ccce0d
JR		 367/*
368 * Domain represents a virtual machine, more than one devices
1ce28feb
WH		 369 * across iommus may be owned in one domain, e.g. kvm guest.
370 */
ab8dfe25 371#define DOMAIN_FLAG_VIRTUAL_MACHINE (1 << 0)
1ce28feb 372
2c2e2c38 373/* si_domain contains mulitple devices */
ab8dfe25 374#define DOMAIN_FLAG_STATIC_IDENTITY (1 << 1)
2c2e2c38 375
99126f7c
MM		 376struct dmar_domain {
377 int id; /* domain id */
4c923d47 378 int nid; /* node id */
78d8e704 379 DECLARE_BITMAP(iommu_bmp, DMAR_UNITS_SUPPORTED);
1b198bb0 380 /* bitmap of iommus this domain uses*/
99126f7c 381
c0e8a6c8
JR		 382 u16 iommu_did[DMAR_UNITS_SUPPORTED];
383 /* Domain ids per IOMMU. Use u16 since
384 * domain ids are 16 bit wide according
385 * to VT-d spec, section 9.3 */
386
00a77deb 387 struct list_head devices; /* all devices' list */
99126f7c
MM		 388 struct iova_domain iovad; /* iova's that belong to this domain */
389
390 struct dma_pte *pgd; /* virtual address */
99126f7c
MM		 391 int gaw; /* max guest address width */
392
393 /* adjusted guest address width, 0 is level 2 30-bit */
394 int agaw;
395
3b5410e7 396 int flags; /* flags to find out type of domain */
8e604097
WH		 397
398 int iommu_coherency;/* indicate coherency of iommu access */
58c610bd 399 int iommu_snooping; /* indicate snooping control feature*/
c7151a8d 400 int iommu_count; /* reference count of iommu */
6dd9a7c7
YS		 401 int iommu_superpage;/* Level of superpages supported:
402 0 == 4KiB (no superpages), 1 == 2MiB,
403 2 == 1GiB, 3 == 512GiB, 4 == 1TiB */
c7151a8d 404 spinlock_t iommu_lock; /* protect iommu set in domain */
fe40f1e0 405 u64 max_addr; /* maximum mapped address */
00a77deb
JR		 406
407 struct iommu_domain domain; /* generic domain data structure for
408 iommu core */
99126f7c
MM		 409};
410
a647dacb
MM		 411/* PCI domain-device relationship */
412struct device_domain_info {
413 struct list_head link; /* link to domain siblings */
414 struct list_head global; /* link to global list */
276dbf99 415 u8 bus; /* PCI bus number */
a647dacb 416 u8 devfn; /* PCI devfn number */
0bcb3e28 417 struct device *dev; /* it's NULL for PCIe-to-PCI bridge */
93a23a72 418 struct intel_iommu *iommu; /* IOMMU used by this device */
a647dacb
MM		 419 struct dmar_domain *domain; /* pointer to domain */
420};
421
b94e4117
JL		 422struct dmar_rmrr_unit {
423 struct list_head list; /* list of rmrr units */
424 struct acpi_dmar_header *hdr; /* ACPI header */
425 u64 base_address; /* reserved base address*/
426 u64 end_address; /* reserved end address */
832bd858 427 struct dmar_dev_scope *devices; /* target devices */
b94e4117
JL		 428 int devices_cnt; /* target device count */
429};
430
431struct dmar_atsr_unit {
432 struct list_head list; /* list of ATSR units */
433 struct acpi_dmar_header *hdr; /* ACPI header */
832bd858 434 struct dmar_dev_scope *devices; /* target devices */
b94e4117
JL		 435 int devices_cnt; /* target device count */
436 u8 include_all:1; /* include all ports */
437};
438
439static LIST_HEAD(dmar_atsr_units);
440static LIST_HEAD(dmar_rmrr_units);
441
442#define for_each_rmrr_units(rmrr) \
443 list_for_each_entry(rmrr, &dmar_rmrr_units, list)
444
5e0d2a6f 445static void flush_unmaps_timeout(unsigned long data);
446
b707cb02 447static DEFINE_TIMER(unmap_timer, flush_unmaps_timeout, 0, 0);
5e0d2a6f 448
80b20dd8 449#define HIGH_WATER_MARK 250
450struct deferred_flush_tables {
451 int next;
452 struct iova *iova[HIGH_WATER_MARK];
453 struct dmar_domain *domain[HIGH_WATER_MARK];
ea8ea460 454 struct page *freelist[HIGH_WATER_MARK];
80b20dd8 455};
456
457static struct deferred_flush_tables *deferred_flush;
458
5e0d2a6f 459/* bitmap for indexing intel_iommus */
5e0d2a6f 460static int g_num_of_iommus;
461
462static DEFINE_SPINLOCK(async_umap_flush_lock);
463static LIST_HEAD(unmaps_to_do);
464
465static int timer_on;
466static long list_size;
5e0d2a6f 467
92d03cc8 468static void domain_exit(struct dmar_domain *domain);
ba395927 469static void domain_remove_dev_info(struct dmar_domain *domain);
b94e4117 470static void domain_remove_one_dev_info(struct dmar_domain *domain,
bf9c9eda 471 struct device *dev);
92d03cc8 472static void iommu_detach_dependent_devices(struct intel_iommu *iommu,
0bcb3e28 473 struct device *dev);
2a46ddf7
JL		 474static int domain_detach_iommu(struct dmar_domain *domain,
475 struct intel_iommu *iommu);
ba395927 476
d3f13810 477#ifdef CONFIG_INTEL_IOMMU_DEFAULT_ON
0cd5c3c8
KM		 478int dmar_disabled = 0;
479#else
480int dmar_disabled = 1;
d3f13810 481#endif /*CONFIG_INTEL_IOMMU_DEFAULT_ON*/
0cd5c3c8 482
8bc1f85c
ED		 483int intel_iommu_enabled = 0;
484EXPORT_SYMBOL_GPL(intel_iommu_enabled);
485
2d9e667e 486static int dmar_map_gfx = 1;
7d3b03ce 487static int dmar_forcedac;
5e0d2a6f 488static int intel_iommu_strict;
6dd9a7c7 489static int intel_iommu_superpage = 1;
c83b2f20
DW		 490static int intel_iommu_ecs = 1;
491
492/* We only actually use ECS when PASID support (on the new bit 40)
493 * is also advertised. Some early implementations — the ones with
494 * PASID support on bit 28 — have issues even when we *only* use
495 * extended root/context tables. */
496#define ecs_enabled(iommu) (intel_iommu_ecs && ecap_ecs(iommu->ecap) && \
497 ecap_pasid(iommu->ecap))
ba395927 498
c0771df8
DW		 499int intel_iommu_gfx_mapped;
500EXPORT_SYMBOL_GPL(intel_iommu_gfx_mapped);
501
ba395927
KA		 502#define DUMMY_DEVICE_DOMAIN_INFO ((struct device_domain_info *)(-1))
503static DEFINE_SPINLOCK(device_domain_lock);
504static LIST_HEAD(device_domain_list);
505
b22f6434 506static const struct iommu_ops intel_iommu_ops;
a8bcbb0d 507
4158c2ec
JR		 508static bool translation_pre_enabled(struct intel_iommu *iommu)
509{
510 return (iommu->flags & VTD_FLAG_TRANS_PRE_ENABLED);
511}
512
091d42e4
JR		 513static void clear_translation_pre_enabled(struct intel_iommu *iommu)
514{
515 iommu->flags &= ~VTD_FLAG_TRANS_PRE_ENABLED;
516}
517
4158c2ec
JR		 518static void init_translation_status(struct intel_iommu *iommu)
519{
520 u32 gsts;
521
522 gsts = readl(iommu->reg + DMAR_GSTS_REG);
523 if (gsts & DMA_GSTS_TES)
524 iommu->flags |= VTD_FLAG_TRANS_PRE_ENABLED;
525}
526
00a77deb
JR		 527/* Convert generic 'struct iommu_domain to private struct dmar_domain */
528static struct dmar_domain *to_dmar_domain(struct iommu_domain *dom)
529{
530 return container_of(dom, struct dmar_domain, domain);
531}
532
ba395927
KA		 533static int __init intel_iommu_setup(char *str)
534{
535 if (!str)
536 return -EINVAL;
537 while (*str) {
0cd5c3c8
KM		 538 if (!strncmp(str, "on", 2)) {
539 dmar_disabled = 0;
9f10e5bf 540 pr_info("IOMMU enabled\n");
0cd5c3c8 541 } else if (!strncmp(str, "off", 3)) {
ba395927 542 dmar_disabled = 1;
9f10e5bf 543 pr_info("IOMMU disabled\n");
ba395927
KA		 544 } else if (!strncmp(str, "igfx_off", 8)) {
545 dmar_map_gfx = 0;
9f10e5bf 546 pr_info("Disable GFX device mapping\n");
7d3b03ce 547 } else if (!strncmp(str, "forcedac", 8)) {
9f10e5bf 548 pr_info("Forcing DAC for PCI devices\n");
7d3b03ce 549 dmar_forcedac = 1;
5e0d2a6f 550 } else if (!strncmp(str, "strict", 6)) {
9f10e5bf 551 pr_info("Disable batched IOTLB flush\n");
5e0d2a6f 552 intel_iommu_strict = 1;
6dd9a7c7 553 } else if (!strncmp(str, "sp_off", 6)) {
9f10e5bf 554 pr_info("Disable supported super page\n");
6dd9a7c7 555 intel_iommu_superpage = 0;
c83b2f20
DW		 556 } else if (!strncmp(str, "ecs_off", 7)) {
557 printk(KERN_INFO
558 "Intel-IOMMU: disable extended context table support\n");
559 intel_iommu_ecs = 0;
ba395927
KA		 560 }
561
562 str += strcspn(str, ",");
563 while (*str == ',')
564 str++;
565 }
566 return 0;
567}
568__setup("intel_iommu=", intel_iommu_setup);
569
570static struct kmem_cache *iommu_domain_cache;
571static struct kmem_cache *iommu_devinfo_cache;
ba395927 572
9452d5bf
JR		 573static struct dmar_domain* get_iommu_domain(struct intel_iommu *iommu, u16 did)
574{
8bf47816
JR		 575 struct dmar_domain **domains;
576 int idx = did >> 8;
577
578 domains = iommu->domains[idx];
579 if (!domains)
580 return NULL;
581
582 return domains[did & 0xff];
9452d5bf
JR		 583}
584
585static void set_iommu_domain(struct intel_iommu *iommu, u16 did,
586 struct dmar_domain *domain)
587{
8bf47816
JR		 588 struct dmar_domain **domains;
589 int idx = did >> 8;
590
591 if (!iommu->domains[idx]) {
592 size_t size = 256 * sizeof(struct dmar_domain *);
593 iommu->domains[idx] = kzalloc(size, GFP_ATOMIC);
594 }
595
596 domains = iommu->domains[idx];
597 if (WARN_ON(!domains))
598 return;
599 else
600 domains[did & 0xff] = domain;
9452d5bf
JR		 601}
602
4c923d47 603static inline void *alloc_pgtable_page(int node)
eb3fa7cb 604{
4c923d47
SS		 605 struct page *page;
606 void *vaddr = NULL;
eb3fa7cb 607
4c923d47
SS		 608 page = alloc_pages_node(node, GFP_ATOMIC | __GFP_ZERO, 0);
609 if (page)
610 vaddr = page_address(page);
eb3fa7cb 611 return vaddr;
ba395927
KA		 612}
613
614static inline void free_pgtable_page(void *vaddr)
615{
616 free_page((unsigned long)vaddr);
617}
618
619static inline void *alloc_domain_mem(void)
620{
354bb65e 621 return kmem_cache_alloc(iommu_domain_cache, GFP_ATOMIC);
ba395927
KA		 622}
623
38717946 624static void free_domain_mem(void *vaddr)
ba395927
KA		 625{
626 kmem_cache_free(iommu_domain_cache, vaddr);
627}
628
629static inline void * alloc_devinfo_mem(void)
630{
354bb65e 631 return kmem_cache_alloc(iommu_devinfo_cache, GFP_ATOMIC);
ba395927
KA		 632}
633
634static inline void free_devinfo_mem(void *vaddr)
635{
636 kmem_cache_free(iommu_devinfo_cache, vaddr);
637}
638
ab8dfe25
JL		 639static inline int domain_type_is_vm(struct dmar_domain *domain)
640{
641 return domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE;
642}
643
28ccce0d
JR		 644static inline int domain_type_is_si(struct dmar_domain *domain)
645{
646 return domain->flags & DOMAIN_FLAG_STATIC_IDENTITY;
647}
648
ab8dfe25
JL		 649static inline int domain_type_is_vm_or_si(struct dmar_domain *domain)
650{
651 return domain->flags & (DOMAIN_FLAG_VIRTUAL_MACHINE |
652 DOMAIN_FLAG_STATIC_IDENTITY);
653}
1b573683 654
162d1b10
JL		 655static inline int domain_pfn_supported(struct dmar_domain *domain,
656 unsigned long pfn)
657{
658 int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
659
660 return !(addr_width < BITS_PER_LONG && pfn >> addr_width);
661}
662
4ed0d3e6 663static int __iommu_calculate_agaw(struct intel_iommu *iommu, int max_gaw)
1b573683
WH		 664{
665 unsigned long sagaw;
666 int agaw = -1;
667
668 sagaw = cap_sagaw(iommu->cap);
4ed0d3e6 669 for (agaw = width_to_agaw(max_gaw);
1b573683
WH		 670 agaw >= 0; agaw--) {
671 if (test_bit(agaw, &sagaw))
672 break;
673 }
674
675 return agaw;
676}
677
4ed0d3e6
FY		 678/*
679 * Calculate max SAGAW for each iommu.
680 */
681int iommu_calculate_max_sagaw(struct intel_iommu *iommu)
682{
683 return __iommu_calculate_agaw(iommu, MAX_AGAW_WIDTH);
684}
685
686/*
687 * calculate agaw for each iommu.
688 * "SAGAW" may be different across iommus, use a default agaw, and
689 * get a supported less agaw for iommus that don't support the default agaw.
690 */
691int iommu_calculate_agaw(struct intel_iommu *iommu)
692{
693 return __iommu_calculate_agaw(iommu, DEFAULT_DOMAIN_ADDRESS_WIDTH);
694}
695
2c2e2c38 696/* This functionin only returns single iommu in a domain */
8c11e798
WH		 697static struct intel_iommu *domain_get_iommu(struct dmar_domain *domain)
698{
699 int iommu_id;
700
2c2e2c38 701 /* si_domain and vm domain should not get here. */
ab8dfe25 702 BUG_ON(domain_type_is_vm_or_si(domain));
1b198bb0 703 iommu_id = find_first_bit(domain->iommu_bmp, g_num_of_iommus);
8c11e798
WH		 704 if (iommu_id < 0 || iommu_id >= g_num_of_iommus)
705 return NULL;
706
707 return g_iommus[iommu_id];
708}
709
8e604097
WH		 710static void domain_update_iommu_coherency(struct dmar_domain *domain)
711{
d0501960
DW		 712 struct dmar_drhd_unit *drhd;
713 struct intel_iommu *iommu;
2f119c78
QL		 714 bool found = false;
715 int i;
2e12bc29 716
d0501960 717 domain->iommu_coherency = 1;
8e604097 718
1b198bb0 719 for_each_set_bit(i, domain->iommu_bmp, g_num_of_iommus) {
2f119c78 720 found = true;
8e604097
WH		 721 if (!ecap_coherent(g_iommus[i]->ecap)) {
722 domain->iommu_coherency = 0;
723 break;
724 }
8e604097 725 }
d0501960
DW		 726 if (found)
727 return;
728
729 /* No hardware attached; use lowest common denominator */
730 rcu_read_lock();
731 for_each_active_iommu(iommu, drhd) {
732 if (!ecap_coherent(iommu->ecap)) {
733 domain->iommu_coherency = 0;
734 break;
735 }
736 }
737 rcu_read_unlock();
8e604097
WH		 738}
739
161f6934 740static int domain_update_iommu_snooping(struct intel_iommu *skip)
58c610bd 741{
161f6934
JL		 742 struct dmar_drhd_unit *drhd;
743 struct intel_iommu *iommu;
744 int ret = 1;
58c610bd 745
161f6934
JL		 746 rcu_read_lock();
747 for_each_active_iommu(iommu, drhd) {
748 if (iommu != skip) {
749 if (!ecap_sc_support(iommu->ecap)) {
750 ret = 0;
751 break;
752 }
58c610bd 753 }
58c610bd 754 }
161f6934
JL		 755 rcu_read_unlock();
756
757 return ret;
58c610bd
SY		 758}
759
161f6934 760static int domain_update_iommu_superpage(struct intel_iommu *skip)
6dd9a7c7 761{
8140a95d 762 struct dmar_drhd_unit *drhd;
161f6934 763 struct intel_iommu *iommu;
8140a95d 764 int mask = 0xf;
6dd9a7c7
YS		 765
766 if (!intel_iommu_superpage) {
161f6934 767 return 0;
6dd9a7c7
YS		 768 }
769
8140a95d 770 /* set iommu_superpage to the smallest common denominator */
0e242612 771 rcu_read_lock();
8140a95d 772 for_each_active_iommu(iommu, drhd) {
161f6934
JL		 773 if (iommu != skip) {
774 mask &= cap_super_page_val(iommu->cap);
775 if (!mask)
776 break;
6dd9a7c7
YS		 777 }
778 }
0e242612
JL		 779 rcu_read_unlock();
780
161f6934 781 return fls(mask);
6dd9a7c7
YS		 782}
783
58c610bd
SY		 784/* Some capabilities may be different across iommus */
785static void domain_update_iommu_cap(struct dmar_domain *domain)
786{
787 domain_update_iommu_coherency(domain);
161f6934
JL		 788 domain->iommu_snooping = domain_update_iommu_snooping(NULL);
789 domain->iommu_superpage = domain_update_iommu_superpage(NULL);
58c610bd
SY		 790}
791
03ecc32c
DW		 792static inline struct context_entry *iommu_context_addr(struct intel_iommu *iommu,
793 u8 bus, u8 devfn, int alloc)
794{
795 struct root_entry *root = &iommu->root_entry[bus];
796 struct context_entry *context;
797 u64 *entry;
798
c83b2f20 799 if (ecs_enabled(iommu)) {
03ecc32c
DW		 800 if (devfn >= 0x80) {
801 devfn -= 0x80;
802 entry = &root->hi;
803 }
804 devfn *= 2;
805 }
806 entry = &root->lo;
807 if (*entry & 1)
808 context = phys_to_virt(*entry & VTD_PAGE_MASK);
809 else {
810 unsigned long phy_addr;
811 if (!alloc)
812 return NULL;
813
814 context = alloc_pgtable_page(iommu->node);
815 if (!context)
816 return NULL;
817
818 __iommu_flush_cache(iommu, (void *)context, CONTEXT_SIZE);
819 phy_addr = virt_to_phys((void *)context);
820 *entry = phy_addr | 1;
821 __iommu_flush_cache(iommu, entry, sizeof(*entry));
822 }
823 return &context[devfn];
824}
825
4ed6a540
DW		 826static int iommu_dummy(struct device *dev)
827{
828 return dev->archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO;
829}
830
156baca8 831static struct intel_iommu *device_to_iommu(struct device *dev, u8 *bus, u8 *devfn)
c7151a8d
WH		 832{
833 struct dmar_drhd_unit *drhd = NULL;
b683b230 834 struct intel_iommu *iommu;
156baca8
DW		 835 struct device *tmp;
836 struct pci_dev *ptmp, *pdev = NULL;
aa4d066a 837 u16 segment = 0;
c7151a8d
WH		 838 int i;
839
4ed6a540
DW		 840 if (iommu_dummy(dev))
841 return NULL;
842
156baca8
DW		 843 if (dev_is_pci(dev)) {
844 pdev = to_pci_dev(dev);
845 segment = pci_domain_nr(pdev->bus);
ca5b74d2 846 } else if (has_acpi_companion(dev))
156baca8
DW		 847 dev = &ACPI_COMPANION(dev)->dev;
848
0e242612 849 rcu_read_lock();
b683b230 850 for_each_active_iommu(iommu, drhd) {
156baca8 851 if (pdev && segment != drhd->segment)
276dbf99 852 continue;
c7151a8d 853
b683b230 854 for_each_active_dev_scope(drhd->devices,
156baca8
DW		 855 drhd->devices_cnt, i, tmp) {
856 if (tmp == dev) {
857 *bus = drhd->devices[i].bus;
858 *devfn = drhd->devices[i].devfn;
b683b230 859 goto out;
156baca8
DW		 860 }
861
862 if (!pdev || !dev_is_pci(tmp))
863 continue;
864
865 ptmp = to_pci_dev(tmp);
866 if (ptmp->subordinate &&
867 ptmp->subordinate->number <= pdev->bus->number &&
868 ptmp->subordinate->busn_res.end >= pdev->bus->number)
869 goto got_pdev;
924b6231 870 }
c7151a8d 871
156baca8
DW		 872 if (pdev && drhd->include_all) {
873 got_pdev:
874 *bus = pdev->bus->number;
875 *devfn = pdev->devfn;
b683b230 876 goto out;
156baca8 877 }
c7151a8d 878 }
b683b230 879 iommu = NULL;
156baca8 880 out:
0e242612 881 rcu_read_unlock();
c7151a8d 882
b683b230 883 return iommu;
c7151a8d
WH		 884}
885
5331fe6f
WH		 886static void domain_flush_cache(struct dmar_domain *domain,
887 void *addr, int size)
888{
889 if (!domain->iommu_coherency)
890 clflush_cache_range(addr, size);
891}
892
ba395927
KA		 893static int device_context_mapped(struct intel_iommu *iommu, u8 bus, u8 devfn)
894{
ba395927 895 struct context_entry *context;
03ecc32c 896 int ret = 0;
ba395927
KA		 897 unsigned long flags;
898
899 spin_lock_irqsave(&iommu->lock, flags);
03ecc32c
DW		 900 context = iommu_context_addr(iommu, bus, devfn, 0);
901 if (context)
902 ret = context_present(context);
ba395927
KA		 903 spin_unlock_irqrestore(&iommu->lock, flags);
904 return ret;
905}
906
907static void clear_context_table(struct intel_iommu *iommu, u8 bus, u8 devfn)
908{
ba395927
KA		 909 struct context_entry *context;
910 unsigned long flags;
911
912 spin_lock_irqsave(&iommu->lock, flags);
03ecc32c 913 context = iommu_context_addr(iommu, bus, devfn, 0);
ba395927 914 if (context) {
03ecc32c
DW		 915 context_clear_entry(context);
916 __iommu_flush_cache(iommu, context, sizeof(*context));
ba395927
KA		 917 }
918 spin_unlock_irqrestore(&iommu->lock, flags);
919}
920
921static void free_context_table(struct intel_iommu *iommu)
922{
ba395927
KA		 923 int i;
924 unsigned long flags;
925 struct context_entry *context;
926
927 spin_lock_irqsave(&iommu->lock, flags);
928 if (!iommu->root_entry) {
929 goto out;
930 }
931 for (i = 0; i < ROOT_ENTRY_NR; i++) {
03ecc32c 932 context = iommu_context_addr(iommu, i, 0, 0);
ba395927
KA		 933 if (context)
934 free_pgtable_page(context);
03ecc32c 935
c83b2f20 936 if (!ecs_enabled(iommu))
03ecc32c
DW		 937 continue;
938
939 context = iommu_context_addr(iommu, i, 0x80, 0);
940 if (context)
941 free_pgtable_page(context);
942
ba395927
KA		 943 }
944 free_pgtable_page(iommu->root_entry);
945 iommu->root_entry = NULL;
946out:
947 spin_unlock_irqrestore(&iommu->lock, flags);
948}
949
b026fd28 950static struct dma_pte *pfn_to_dma_pte(struct dmar_domain *domain,
5cf0a76f 951 unsigned long pfn, int *target_level)
ba395927 952{
ba395927
KA		 953 struct dma_pte *parent, *pte = NULL;
954 int level = agaw_to_level(domain->agaw);
4399c8bf 955 int offset;
ba395927
KA		 956
957 BUG_ON(!domain->pgd);
f9423606 958
162d1b10 959 if (!domain_pfn_supported(domain, pfn))
f9423606
JS		 960 /* Address beyond IOMMU's addressing capabilities. */
961 return NULL;
962
ba395927
KA		 963 parent = domain->pgd;
964
5cf0a76f 965 while (1) {
ba395927
KA		 966 void *tmp_page;
967
b026fd28 968 offset = pfn_level_offset(pfn, level);
ba395927 969 pte = &parent[offset];
5cf0a76f 970 if (!*target_level && (dma_pte_superpage(pte) || !dma_pte_present(pte)))
6dd9a7c7 971 break;
5cf0a76f 972 if (level == *target_level)
ba395927
KA		 973 break;
974
19c239ce 975 if (!dma_pte_present(pte)) {
c85994e4
DW		 976 uint64_t pteval;
977
4c923d47 978 tmp_page = alloc_pgtable_page(domain->nid);
ba395927 979
206a73c1 980 if (!tmp_page)
ba395927 981 return NULL;
206a73c1 982
c85994e4 983 domain_flush_cache(domain, tmp_page, VTD_PAGE_SIZE);
64de5af0 984 pteval = ((uint64_t)virt_to_dma_pfn(tmp_page) << VTD_PAGE_SHIFT) | DMA_PTE_READ | DMA_PTE_WRITE;
effad4b5 985 if (cmpxchg64(&pte->val, 0ULL, pteval))
c85994e4
DW		 986 /* Someone else set it while we were thinking; use theirs. */
987 free_pgtable_page(tmp_page);
effad4b5 988 else
c85994e4 989 domain_flush_cache(domain, pte, sizeof(*pte));
ba395927 990 }
5cf0a76f
DW		 991 if (level == 1)
992 break;
993
19c239ce 994 parent = phys_to_virt(dma_pte_addr(pte));
ba395927
KA		 995 level--;
996 }
997
5cf0a76f
DW		 998 if (!*target_level)
999 *target_level = level;
1000
ba395927
KA		 1001 return pte;
1002}
1003
6dd9a7c7 1004
ba395927 1005/* return address's pte at specific level */
90dcfb5e
DW		 1006static struct dma_pte *dma_pfn_level_pte(struct dmar_domain *domain,
1007 unsigned long pfn,
6dd9a7c7 1008 int level, int *large_page)
ba395927
KA		 1009{
1010 struct dma_pte *parent, *pte = NULL;
1011 int total = agaw_to_level(domain->agaw);
1012 int offset;
1013
1014 parent = domain->pgd;
1015 while (level <= total) {
90dcfb5e 1016 offset = pfn_level_offset(pfn, total);
ba395927
KA		 1017 pte = &parent[offset];
1018 if (level == total)
1019 return pte;
1020
6dd9a7c7
YS		 1021 if (!dma_pte_present(pte)) {
1022 *large_page = total;
ba395927 1023 break;
6dd9a7c7
YS		 1024 }
1025
e16922af 1026 if (dma_pte_superpage(pte)) {
6dd9a7c7
YS		 1027 *large_page = total;
1028 return pte;
1029 }
1030
19c239ce 1031 parent = phys_to_virt(dma_pte_addr(pte));
ba395927
KA		 1032 total--;
1033 }
1034 return NULL;
1035}
1036
ba395927 1037/* clear last level pte, a tlb flush should be followed */
5cf0a76f 1038static void dma_pte_clear_range(struct dmar_domain *domain,
595badf5
DW		 1039 unsigned long start_pfn,
1040 unsigned long last_pfn)
ba395927 1041{
6dd9a7c7 1042 unsigned int large_page = 1;
310a5ab9 1043 struct dma_pte *first_pte, *pte;
66eae846 1044
162d1b10
JL		 1045 BUG_ON(!domain_pfn_supported(domain, start_pfn));
1046 BUG_ON(!domain_pfn_supported(domain, last_pfn));
59c36286 1047 BUG_ON(start_pfn > last_pfn);
ba395927 1048
04b18e65 1049 /* we don't need lock here; nobody else touches the iova range */
59c36286 1050 do {
6dd9a7c7
YS		 1051 large_page = 1;
1052 first_pte = pte = dma_pfn_level_pte(domain, start_pfn, 1, &large_page);
310a5ab9 1053 if (!pte) {
6dd9a7c7 1054 start_pfn = align_to_level(start_pfn + 1, large_page + 1);
310a5ab9
DW		 1055 continue;
1056 }
6dd9a7c7 1057 do {
310a5ab9 1058 dma_clear_pte(pte);
6dd9a7c7 1059 start_pfn += lvl_to_nr_pages(large_page);
310a5ab9 1060 pte++;
75e6bf96
DW		 1061 } while (start_pfn <= last_pfn && !first_pte_in_page(pte));
1062
310a5ab9
DW		 1063 domain_flush_cache(domain, first_pte,
1064 (void *)pte - (void *)first_pte);
59c36286
DW		 1065
1066 } while (start_pfn && start_pfn <= last_pfn);
ba395927
KA		 1067}
1068
3269ee0b
AW		 1069static void dma_pte_free_level(struct dmar_domain *domain, int level,
1070 struct dma_pte *pte, unsigned long pfn,
1071 unsigned long start_pfn, unsigned long last_pfn)
1072{
1073 pfn = max(start_pfn, pfn);
1074 pte = &pte[pfn_level_offset(pfn, level)];
1075
1076 do {
1077 unsigned long level_pfn;
1078 struct dma_pte *level_pte;
1079
1080 if (!dma_pte_present(pte) || dma_pte_superpage(pte))
1081 goto next;
1082
1083 level_pfn = pfn & level_mask(level - 1);
1084 level_pte = phys_to_virt(dma_pte_addr(pte));
1085
1086 if (level > 2)
1087 dma_pte_free_level(domain, level - 1, level_pte,
1088 level_pfn, start_pfn, last_pfn);
1089
1090 /* If range covers entire pagetable, free it */
1091 if (!(start_pfn > level_pfn ||
08336fd2 1092 last_pfn < level_pfn + level_size(level) - 1)) {
3269ee0b
AW		 1093 dma_clear_pte(pte);
1094 domain_flush_cache(domain, pte, sizeof(*pte));
1095 free_pgtable_page(level_pte);
1096 }
1097next:
1098 pfn += level_size(level);
1099 } while (!first_pte_in_page(++pte) && pfn <= last_pfn);
1100}
1101
ba395927
KA		 1102/* free page table pages. last level pte should already be cleared */
1103static void dma_pte_free_pagetable(struct dmar_domain *domain,
d794dc9b
DW		 1104 unsigned long start_pfn,
1105 unsigned long last_pfn)
ba395927 1106{
162d1b10
JL		 1107 BUG_ON(!domain_pfn_supported(domain, start_pfn));
1108 BUG_ON(!domain_pfn_supported(domain, last_pfn));
59c36286 1109 BUG_ON(start_pfn > last_pfn);
ba395927 1110
d41a4adb
JL		 1111 dma_pte_clear_range(domain, start_pfn, last_pfn);
1112
f3a0a52f 1113 /* We don't need lock here; nobody else touches the iova range */
3269ee0b
AW		 1114 dma_pte_free_level(domain, agaw_to_level(domain->agaw),
1115 domain->pgd, 0, start_pfn, last_pfn);
6660c63a 1116
ba395927 1117 /* free pgd */
d794dc9b 1118 if (start_pfn == 0 && last_pfn == DOMAIN_MAX_PFN(domain->gaw)) {
ba395927
KA		 1119 free_pgtable_page(domain->pgd);
1120 domain->pgd = NULL;
1121 }
1122}
1123
ea8ea460
DW		 1124/* When a page at a given level is being unlinked from its parent, we don't
1125 need to *modify* it at all. All we need to do is make a list of all the
1126 pages which can be freed just as soon as we've flushed the IOTLB and we
1127 know the hardware page-walk will no longer touch them.
1128 The 'pte' argument is the *parent* PTE, pointing to the page that is to
1129 be freed. */
1130static struct page *dma_pte_list_pagetables(struct dmar_domain *domain,
1131 int level, struct dma_pte *pte,
1132 struct page *freelist)
1133{
1134 struct page *pg;
1135
1136 pg = pfn_to_page(dma_pte_addr(pte) >> PAGE_SHIFT);
1137 pg->freelist = freelist;
1138 freelist = pg;
1139
1140 if (level == 1)
1141 return freelist;
1142
adeb2590
JL		 1143 pte = page_address(pg);
1144 do {
ea8ea460
DW		 1145 if (dma_pte_present(pte) && !dma_pte_superpage(pte))
1146 freelist = dma_pte_list_pagetables(domain, level - 1,
1147 pte, freelist);
adeb2590
JL		 1148 pte++;
1149 } while (!first_pte_in_page(pte));
ea8ea460
DW		 1150
1151 return freelist;
1152}
1153
1154static struct page *dma_pte_clear_level(struct dmar_domain *domain, int level,
1155 struct dma_pte *pte, unsigned long pfn,
1156 unsigned long start_pfn,
1157 unsigned long last_pfn,
1158 struct page *freelist)
1159{
1160 struct dma_pte *first_pte = NULL, *last_pte = NULL;
1161
1162 pfn = max(start_pfn, pfn);
1163 pte = &pte[pfn_level_offset(pfn, level)];
1164
1165 do {
1166 unsigned long level_pfn;
1167
1168 if (!dma_pte_present(pte))
1169 goto next;
1170
1171 level_pfn = pfn & level_mask(level);
1172
1173 /* If range covers entire pagetable, free it */
1174 if (start_pfn <= level_pfn &&
1175 last_pfn >= level_pfn + level_size(level) - 1) {
1176 /* These suborbinate page tables are going away entirely. Don't
1177 bother to clear them; we're just going to *free* them. */
1178 if (level > 1 && !dma_pte_superpage(pte))
1179 freelist = dma_pte_list_pagetables(domain, level - 1, pte, freelist);
1180
1181 dma_clear_pte(pte);
1182 if (!first_pte)
1183 first_pte = pte;
1184 last_pte = pte;
1185 } else if (level > 1) {
1186 /* Recurse down into a level that isn't *entirely* obsolete */
1187 freelist = dma_pte_clear_level(domain, level - 1,
1188 phys_to_virt(dma_pte_addr(pte)),
1189 level_pfn, start_pfn, last_pfn,
1190 freelist);
1191 }
1192next:
1193 pfn += level_size(level);
1194 } while (!first_pte_in_page(++pte) && pfn <= last_pfn);
1195
1196 if (first_pte)
1197 domain_flush_cache(domain, first_pte,
1198 (void *)++last_pte - (void *)first_pte);
1199
1200 return freelist;
1201}
1202
1203/* We can't just free the pages because the IOMMU may still be walking
1204 the page tables, and may have cached the intermediate levels. The
1205 pages can only be freed after the IOTLB flush has been done. */
1206struct page *domain_unmap(struct dmar_domain *domain,
1207 unsigned long start_pfn,
1208 unsigned long last_pfn)
1209{
ea8ea460
DW		 1210 struct page *freelist = NULL;
1211
162d1b10
JL		 1212 BUG_ON(!domain_pfn_supported(domain, start_pfn));
1213 BUG_ON(!domain_pfn_supported(domain, last_pfn));
ea8ea460
DW		 1214 BUG_ON(start_pfn > last_pfn);
1215
1216 /* we don't need lock here; nobody else touches the iova range */
1217 freelist = dma_pte_clear_level(domain, agaw_to_level(domain->agaw),
1218 domain->pgd, 0, start_pfn, last_pfn, NULL);
1219
1220 /* free pgd */
1221 if (start_pfn == 0 && last_pfn == DOMAIN_MAX_PFN(domain->gaw)) {
1222 struct page *pgd_page = virt_to_page(domain->pgd);
1223 pgd_page->freelist = freelist;
1224 freelist = pgd_page;
1225
1226 domain->pgd = NULL;
1227 }
1228
1229 return freelist;
1230}
1231
1232void dma_free_pagelist(struct page *freelist)
1233{
1234 struct page *pg;
1235
1236 while ((pg = freelist)) {
1237 freelist = pg->freelist;
1238 free_pgtable_page(page_address(pg));
1239 }
1240}
1241
ba395927
KA		 1242/* iommu handling */
1243static int iommu_alloc_root_entry(struct intel_iommu *iommu)
1244{
1245 struct root_entry *root;
1246 unsigned long flags;
1247
4c923d47 1248 root = (struct root_entry *)alloc_pgtable_page(iommu->node);
ffebeb46 1249 if (!root) {
9f10e5bf 1250 pr_err("Allocating root entry for %s failed\n",
ffebeb46 1251 iommu->name);
ba395927 1252 return -ENOMEM;
ffebeb46 1253 }
ba395927 1254
5b6985ce 1255 __iommu_flush_cache(iommu, root, ROOT_SIZE);
ba395927
KA		 1256
1257 spin_lock_irqsave(&iommu->lock, flags);
1258 iommu->root_entry = root;
1259 spin_unlock_irqrestore(&iommu->lock, flags);
1260
1261 return 0;
1262}
1263
ba395927
KA		 1264static void iommu_set_root_entry(struct intel_iommu *iommu)
1265{
03ecc32c 1266 u64 addr;
c416daa9 1267 u32 sts;
ba395927
KA		 1268 unsigned long flag;
1269
03ecc32c 1270 addr = virt_to_phys(iommu->root_entry);
c83b2f20 1271 if (ecs_enabled(iommu))
03ecc32c 1272 addr |= DMA_RTADDR_RTT;
ba395927 1273
1f5b3c3f 1274 raw_spin_lock_irqsave(&iommu->register_lock, flag);
03ecc32c 1275 dmar_writeq(iommu->reg + DMAR_RTADDR_REG, addr);
ba395927 1276
c416daa9 1277 writel(iommu->gcmd | DMA_GCMD_SRTP, iommu->reg + DMAR_GCMD_REG);
ba395927
KA		 1278
1279 /* Make sure hardware complete it */
1280 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
c416daa9 1281 readl, (sts & DMA_GSTS_RTPS), sts);
ba395927 1282
1f5b3c3f 1283 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
ba395927
KA		 1284}
1285
1286static void iommu_flush_write_buffer(struct intel_iommu *iommu)
1287{
1288 u32 val;
1289 unsigned long flag;
1290
9af88143 1291 if (!rwbf_quirk && !cap_rwbf(iommu->cap))
ba395927 1292 return;
ba395927 1293
1f5b3c3f 1294 raw_spin_lock_irqsave(&iommu->register_lock, flag);
462b60f6 1295 writel(iommu->gcmd | DMA_GCMD_WBF, iommu->reg + DMAR_GCMD_REG);
ba395927
KA		 1296
1297 /* Make sure hardware complete it */
1298 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
c416daa9 1299 readl, (!(val & DMA_GSTS_WBFS)), val);
ba395927 1300
1f5b3c3f 1301 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
ba395927
KA		 1302}
1303
1304/* return value determine if we need a write buffer flush */
4c25a2c1
DW		 1305static void __iommu_flush_context(struct intel_iommu *iommu,
1306 u16 did, u16 source_id, u8 function_mask,
1307 u64 type)
ba395927
KA		 1308{
1309 u64 val = 0;
1310 unsigned long flag;
1311
ba395927
KA		 1312 switch (type) {
1313 case DMA_CCMD_GLOBAL_INVL:
1314 val = DMA_CCMD_GLOBAL_INVL;
1315 break;
1316 case DMA_CCMD_DOMAIN_INVL:
1317 val = DMA_CCMD_DOMAIN_INVL|DMA_CCMD_DID(did);
1318 break;
1319 case DMA_CCMD_DEVICE_INVL:
1320 val = DMA_CCMD_DEVICE_INVL|DMA_CCMD_DID(did)
1321 | DMA_CCMD_SID(source_id) | DMA_CCMD_FM(function_mask);
1322 break;
1323 default:
1324 BUG();
1325 }
1326 val |= DMA_CCMD_ICC;
1327
1f5b3c3f 1328 raw_spin_lock_irqsave(&iommu->register_lock, flag);
ba395927
KA		 1329 dmar_writeq(iommu->reg + DMAR_CCMD_REG, val);
1330
1331 /* Make sure hardware complete it */
1332 IOMMU_WAIT_OP(iommu, DMAR_CCMD_REG,
1333 dmar_readq, (!(val & DMA_CCMD_ICC)), val);
1334
1f5b3c3f 1335 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
ba395927
KA		 1336}
1337
ba395927 1338/* return value determine if we need a write buffer flush */
1f0ef2aa
DW		 1339static void __iommu_flush_iotlb(struct intel_iommu *iommu, u16 did,
1340 u64 addr, unsigned int size_order, u64 type)
ba395927
KA		 1341{
1342 int tlb_offset = ecap_iotlb_offset(iommu->ecap);
1343 u64 val = 0, val_iva = 0;
1344 unsigned long flag;
1345
ba395927
KA		 1346 switch (type) {
1347 case DMA_TLB_GLOBAL_FLUSH:
1348 /* global flush doesn't need set IVA_REG */
1349 val = DMA_TLB_GLOBAL_FLUSH|DMA_TLB_IVT;
1350 break;
1351 case DMA_TLB_DSI_FLUSH:
1352 val = DMA_TLB_DSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
1353 break;
1354 case DMA_TLB_PSI_FLUSH:
1355 val = DMA_TLB_PSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
ea8ea460 1356 /* IH bit is passed in as part of address */
ba395927
KA		 1357 val_iva = size_order | addr;
1358 break;
1359 default:
1360 BUG();
1361 }
1362 /* Note: set drain read/write */
1363#if 0
1364 /*
1365 * This is probably to be super secure.. Looks like we can
1366 * ignore it without any impact.
1367 */
1368 if (cap_read_drain(iommu->cap))
1369 val |= DMA_TLB_READ_DRAIN;
1370#endif
1371 if (cap_write_drain(iommu->cap))
1372 val |= DMA_TLB_WRITE_DRAIN;
1373
1f5b3c3f 1374 raw_spin_lock_irqsave(&iommu->register_lock, flag);
ba395927
KA		 1375 /* Note: Only uses first TLB reg currently */
1376 if (val_iva)
1377 dmar_writeq(iommu->reg + tlb_offset, val_iva);
1378 dmar_writeq(iommu->reg + tlb_offset + 8, val);
1379
1380 /* Make sure hardware complete it */
1381 IOMMU_WAIT_OP(iommu, tlb_offset + 8,
1382 dmar_readq, (!(val & DMA_TLB_IVT)), val);
1383
1f5b3c3f 1384 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
ba395927
KA		 1385
1386 /* check IOTLB invalidation granularity */
1387 if (DMA_TLB_IAIG(val) == 0)
9f10e5bf 1388 pr_err("Flush IOTLB failed\n");
ba395927 1389 if (DMA_TLB_IAIG(val) != DMA_TLB_IIRG(type))
9f10e5bf 1390 pr_debug("TLB flush request %Lx, actual %Lx\n",
5b6985ce
FY		 1391 (unsigned long long)DMA_TLB_IIRG(type),
1392 (unsigned long long)DMA_TLB_IAIG(val));
ba395927
KA		 1393}
1394
64ae892b
DW		 1395static struct device_domain_info *
1396iommu_support_dev_iotlb (struct dmar_domain *domain, struct intel_iommu *iommu,
1397 u8 bus, u8 devfn)
93a23a72 1398{
2f119c78 1399 bool found = false;
93a23a72
YZ		 1400 unsigned long flags;
1401 struct device_domain_info *info;
0bcb3e28 1402 struct pci_dev *pdev;
93a23a72
YZ		 1403
1404 if (!ecap_dev_iotlb_support(iommu->ecap))
1405 return NULL;
1406
1407 if (!iommu->qi)
1408 return NULL;
1409
1410 spin_lock_irqsave(&device_domain_lock, flags);
1411 list_for_each_entry(info, &domain->devices, link)
c3b497c6
JL		 1412 if (info->iommu == iommu && info->bus == bus &&
1413 info->devfn == devfn) {
2f119c78 1414 found = true;
93a23a72
YZ		 1415 break;
1416 }
1417 spin_unlock_irqrestore(&device_domain_lock, flags);
1418
0bcb3e28 1419 if (!found || !info->dev || !dev_is_pci(info->dev))
93a23a72
YZ		 1420 return NULL;
1421
0bcb3e28
DW		 1422 pdev = to_pci_dev(info->dev);
1423
1424 if (!pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_ATS))
93a23a72
YZ		 1425 return NULL;
1426
0bcb3e28 1427 if (!dmar_find_matched_atsr_unit(pdev))
93a23a72
YZ		 1428 return NULL;
1429
93a23a72
YZ		 1430 return info;
1431}
1432
1433static void iommu_enable_dev_iotlb(struct device_domain_info *info)
ba395927 1434{
0bcb3e28 1435 if (!info || !dev_is_pci(info->dev))
93a23a72
YZ		 1436 return;
1437
0bcb3e28 1438 pci_enable_ats(to_pci_dev(info->dev), VTD_PAGE_SHIFT);
93a23a72
YZ		 1439}
1440
1441static void iommu_disable_dev_iotlb(struct device_domain_info *info)
1442{
0bcb3e28
DW		 1443 if (!info->dev || !dev_is_pci(info->dev) ||
1444 !pci_ats_enabled(to_pci_dev(info->dev)))
93a23a72
YZ		 1445 return;
1446
0bcb3e28 1447 pci_disable_ats(to_pci_dev(info->dev));
93a23a72
YZ		 1448}
1449
1450static void iommu_flush_dev_iotlb(struct dmar_domain *domain,
1451 u64 addr, unsigned mask)
1452{
1453 u16 sid, qdep;
1454 unsigned long flags;
1455 struct device_domain_info *info;
1456
1457 spin_lock_irqsave(&device_domain_lock, flags);
1458 list_for_each_entry(info, &domain->devices, link) {
0bcb3e28
DW		 1459 struct pci_dev *pdev;
1460 if (!info->dev || !dev_is_pci(info->dev))
1461 continue;
1462
1463 pdev = to_pci_dev(info->dev);
1464 if (!pci_ats_enabled(pdev))
93a23a72
YZ		 1465 continue;
1466
1467 sid = info->bus << 8 | info->devfn;
0bcb3e28 1468 qdep = pci_ats_queue_depth(pdev);
93a23a72
YZ		 1469 qi_flush_dev_iotlb(info->iommu, sid, qdep, addr, mask);
1470 }
1471 spin_unlock_irqrestore(&device_domain_lock, flags);
1472}
1473
1f0ef2aa 1474static void iommu_flush_iotlb_psi(struct intel_iommu *iommu, u16 did,
ea8ea460 1475 unsigned long pfn, unsigned int pages, int ih, int map)
ba395927 1476{
9dd2fe89 1477 unsigned int mask = ilog2(__roundup_pow_of_two(pages));
03d6a246 1478 uint64_t addr = (uint64_t)pfn << VTD_PAGE_SHIFT;
ba395927 1479
ba395927
KA		 1480 BUG_ON(pages == 0);
1481
ea8ea460
DW		 1482 if (ih)
1483 ih = 1 << 6;
ba395927 1484 /*
9dd2fe89
YZ		 1485 * Fallback to domain selective flush if no PSI support or the size is
1486 * too big.
ba395927
KA		 1487 * PSI requires page size to be 2 ^ x, and the base address is naturally
1488 * aligned to the size
1489 */
9dd2fe89
YZ		 1490 if (!cap_pgsel_inv(iommu->cap) || mask > cap_max_amask_val(iommu->cap))
1491 iommu->flush.flush_iotlb(iommu, did, 0, 0,
1f0ef2aa 1492 DMA_TLB_DSI_FLUSH);
9dd2fe89 1493 else
ea8ea460 1494 iommu->flush.flush_iotlb(iommu, did, addr | ih, mask,
9dd2fe89 1495 DMA_TLB_PSI_FLUSH);
bf92df30
YZ		 1496
1497 /*
82653633
NA		 1498 * In caching mode, changes of pages from non-present to present require
1499 * flush. However, device IOTLB doesn't need to be flushed in this case.
bf92df30 1500 */
82653633 1501 if (!cap_caching_mode(iommu->cap) || !map)
9452d5bf
JR		 1502 iommu_flush_dev_iotlb(get_iommu_domain(iommu, did),
1503 addr, mask);
ba395927
KA		 1504}
1505
f8bab735 1506static void iommu_disable_protect_mem_regions(struct intel_iommu *iommu)
1507{
1508 u32 pmen;
1509 unsigned long flags;
1510
1f5b3c3f 1511 raw_spin_lock_irqsave(&iommu->register_lock, flags);
f8bab735 1512 pmen = readl(iommu->reg + DMAR_PMEN_REG);
1513 pmen &= ~DMA_PMEN_EPM;
1514 writel(pmen, iommu->reg + DMAR_PMEN_REG);
1515
1516 /* wait for the protected region status bit to clear */
1517 IOMMU_WAIT_OP(iommu, DMAR_PMEN_REG,
1518 readl, !(pmen & DMA_PMEN_PRS), pmen);
1519
1f5b3c3f 1520 raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
f8bab735 1521}
1522
2a41ccee 1523static void iommu_enable_translation(struct intel_iommu *iommu)
ba395927
KA		 1524{
1525 u32 sts;
1526 unsigned long flags;
1527
1f5b3c3f 1528 raw_spin_lock_irqsave(&iommu->register_lock, flags);
c416daa9
DW		 1529 iommu->gcmd |= DMA_GCMD_TE;
1530 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
ba395927
KA		 1531
1532 /* Make sure hardware complete it */
1533 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
c416daa9 1534 readl, (sts & DMA_GSTS_TES), sts);
ba395927 1535
1f5b3c3f 1536 raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
ba395927
KA		 1537}
1538
2a41ccee 1539static void iommu_disable_translation(struct intel_iommu *iommu)
ba395927
KA		 1540{
1541 u32 sts;
1542 unsigned long flag;
1543
1f5b3c3f 1544 raw_spin_lock_irqsave(&iommu->register_lock, flag);
ba395927
KA		 1545 iommu->gcmd &= ~DMA_GCMD_TE;
1546 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1547
1548 /* Make sure hardware complete it */
1549 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
c416daa9 1550 readl, (!(sts & DMA_GSTS_TES)), sts);
ba395927 1551
1f5b3c3f 1552 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
ba395927
KA		 1553}
1554
3460a6d9 1555
ba395927
KA		 1556static int iommu_init_domains(struct intel_iommu *iommu)
1557{
8bf47816
JR		 1558 u32 ndomains, nlongs;
1559 size_t size;
ba395927
KA		 1560
1561 ndomains = cap_ndoms(iommu->cap);
8bf47816 1562 pr_debug("%s: Number of Domains supported <%d>\n",
9f10e5bf 1563 iommu->name, ndomains);
ba395927
KA		 1564 nlongs = BITS_TO_LONGS(ndomains);
1565
94a91b50
DD		 1566 spin_lock_init(&iommu->lock);
1567
ba395927
KA		 1568 iommu->domain_ids = kcalloc(nlongs, sizeof(unsigned long), GFP_KERNEL);
1569 if (!iommu->domain_ids) {
9f10e5bf
JR		 1570 pr_err("%s: Allocating domain id array failed\n",
1571 iommu->name);
ba395927
KA		 1572 return -ENOMEM;
1573 }
8bf47816
JR		 1574
1575 size = ((ndomains >> 8) + 1) * sizeof(struct dmar_domain **);
1576 iommu->domains = kzalloc(size, GFP_KERNEL);
1577
1578 if (iommu->domains) {
1579 size = 256 * sizeof(struct dmar_domain *);
1580 iommu->domains[0] = kzalloc(size, GFP_KERNEL);
1581 }
1582
1583 if (!iommu->domains || !iommu->domains[0]) {
9f10e5bf
JR		 1584 pr_err("%s: Allocating domain array failed\n",
1585 iommu->name);
852bdb04 1586 kfree(iommu->domain_ids);
8bf47816 1587 kfree(iommu->domains);
852bdb04 1588 iommu->domain_ids = NULL;
8bf47816 1589 iommu->domains = NULL;
ba395927
KA		 1590 return -ENOMEM;
1591 }
1592
8bf47816
JR		 1593
1594
ba395927 1595 /*
c0e8a6c8
JR		 1596 * If Caching mode is set, then invalid translations are tagged
1597 * with domain-id 0, hence we need to pre-allocate it. We also
1598 * use domain-id 0 as a marker for non-allocated domain-id, so
1599 * make sure it is not used for a real domain.
ba395927 1600 */
c0e8a6c8
JR		 1601 set_bit(0, iommu->domain_ids);
1602
ba395927
KA		 1603 return 0;
1604}
ba395927 1605
ffebeb46 1606static void disable_dmar_iommu(struct intel_iommu *iommu)
ba395927
KA		 1607{
1608 struct dmar_domain *domain;
2a46ddf7 1609 int i;
ba395927 1610
94a91b50 1611 if ((iommu->domains) && (iommu->domain_ids)) {
a45946ab 1612 for_each_set_bit(i, iommu->domain_ids, cap_ndoms(iommu->cap)) {
a4eaa86c
JL		 1613 /*
1614 * Domain id 0 is reserved for invalid translation
c0e8a6c8
JR		 1615 * if hardware supports caching mode and used as
1616 * a non-allocated marker.
a4eaa86c 1617 */
c0e8a6c8 1618 if (i == 0)
a4eaa86c
JL		 1619 continue;
1620
9452d5bf 1621 domain = get_iommu_domain(iommu, i);
94a91b50 1622 clear_bit(i, iommu->domain_ids);
129ad281
JL		 1623 if (domain_detach_iommu(domain, iommu) == 0 &&
1624 !domain_type_is_vm(domain))
92d03cc8 1625 domain_exit(domain);
5e98c4b1 1626 }
ba395927
KA		 1627 }
1628
1629 if (iommu->gcmd & DMA_GCMD_TE)
1630 iommu_disable_translation(iommu);
ffebeb46 1631}
ba395927 1632
ffebeb46
JL		 1633static void free_dmar_iommu(struct intel_iommu *iommu)
1634{
1635 if ((iommu->domains) && (iommu->domain_ids)) {
8bf47816
JR		 1636 int elems = (cap_ndoms(iommu->cap) >> 8) + 1;
1637 int i;
1638
1639 for (i = 0; i < elems; i++)
1640 kfree(iommu->domains[i]);
ffebeb46
JL		 1641 kfree(iommu->domains);
1642 kfree(iommu->domain_ids);
1643 iommu->domains = NULL;
1644 iommu->domain_ids = NULL;
1645 }
ba395927 1646
d9630fe9
WH		 1647 g_iommus[iommu->seq_id] = NULL;
1648
ba395927
KA		 1649 /* free context mapping */
1650 free_context_table(iommu);
ba395927
KA		 1651}
1652
ab8dfe25 1653static struct dmar_domain *alloc_domain(int flags)
ba395927 1654{
92d03cc8
JL		 1655 /* domain id for virtual machine, it won't be set in context */
1656 static atomic_t vm_domid = ATOMIC_INIT(0);
ba395927 1657 struct dmar_domain *domain;
ba395927
KA		 1658
1659 domain = alloc_domain_mem();
1660 if (!domain)
1661 return NULL;
1662
ab8dfe25 1663 memset(domain, 0, sizeof(*domain));
4c923d47 1664 domain->nid = -1;
ab8dfe25 1665 domain->flags = flags;
92d03cc8
JL		 1666 spin_lock_init(&domain->iommu_lock);
1667 INIT_LIST_HEAD(&domain->devices);
ab8dfe25 1668 if (flags & DOMAIN_FLAG_VIRTUAL_MACHINE)
92d03cc8 1669 domain->id = atomic_inc_return(&vm_domid);
2c2e2c38
FY		 1670
1671 return domain;
1672}
1673
fb170fb4
JL		 1674static int __iommu_attach_domain(struct dmar_domain *domain,
1675 struct intel_iommu *iommu)
2c2e2c38
FY		 1676{
1677 int num;
1678 unsigned long ndomains;
2c2e2c38 1679
e2411427
JR		 1680 num = domain->iommu_did[iommu->seq_id];
1681 if (num)
1682 return num;
1683
ba395927 1684 ndomains = cap_ndoms(iommu->cap);
e2411427
JR		 1685 num = find_first_zero_bit(iommu->domain_ids, ndomains);
1686
fb170fb4
JL		 1687 if (num < ndomains) {
1688 set_bit(num, iommu->domain_ids);
9452d5bf 1689 set_iommu_domain(iommu, num, domain);
c0e8a6c8 1690 domain->iommu_did[iommu->seq_id] = num;
fb170fb4
JL		 1691 } else {
1692 num = -ENOSPC;
ba395927
KA		 1693 }
1694
e2411427
JR		 1695 if (num < 0)
1696 pr_err("%s: No free domain ids\n", iommu->name);
1697
fb170fb4
JL		 1698 return num;
1699}
1700
1701static int iommu_attach_domain(struct dmar_domain *domain,
1702 struct intel_iommu *iommu)
1703{
1704 int num;
1705 unsigned long flags;
1706
1707 spin_lock_irqsave(&iommu->lock, flags);
1708 num = __iommu_attach_domain(domain, iommu);
44bde614 1709 spin_unlock_irqrestore(&iommu->lock, flags);
ba395927 1710
fb170fb4 1711 return num;
ba395927
KA		 1712}
1713
2c2e2c38
FY		 1714static void iommu_detach_domain(struct dmar_domain *domain,
1715 struct intel_iommu *iommu)
ba395927
KA		 1716{
1717 unsigned long flags;
c0e8a6c8 1718 int num;
ba395927 1719
8c11e798 1720 spin_lock_irqsave(&iommu->lock, flags);
c0e8a6c8
JR		 1721
1722 num = domain->iommu_did[iommu->seq_id];
1723
1724 if (num == 0)
1725 return;
1726
1727 clear_bit(num, iommu->domain_ids);
9452d5bf 1728 set_iommu_domain(iommu, num, NULL);
c0e8a6c8 1729
8c11e798 1730 spin_unlock_irqrestore(&iommu->lock, flags);
ba395927
KA		 1731}
1732
fb170fb4
JL		 1733static void domain_attach_iommu(struct dmar_domain *domain,
1734 struct intel_iommu *iommu)
1735{
1736 unsigned long flags;
1737
1738 spin_lock_irqsave(&domain->iommu_lock, flags);
1739 if (!test_and_set_bit(iommu->seq_id, domain->iommu_bmp)) {
1740 domain->iommu_count++;
1741 if (domain->iommu_count == 1)
1742 domain->nid = iommu->node;
1743 domain_update_iommu_cap(domain);
1744 }
1745 spin_unlock_irqrestore(&domain->iommu_lock, flags);
1746}
1747
1748static int domain_detach_iommu(struct dmar_domain *domain,
1749 struct intel_iommu *iommu)
1750{
1751 unsigned long flags;
1752 int count = INT_MAX;
1753
1754 spin_lock_irqsave(&domain->iommu_lock, flags);
1755 if (test_and_clear_bit(iommu->seq_id, domain->iommu_bmp)) {
1756 count = --domain->iommu_count;
1757 domain_update_iommu_cap(domain);
c0e8a6c8 1758 domain->iommu_did[iommu->seq_id] = 0;
fb170fb4
JL		 1759 }
1760 spin_unlock_irqrestore(&domain->iommu_lock, flags);
1761
1762 return count;
1763}
1764
ba395927 1765static struct iova_domain reserved_iova_list;
8a443df4 1766static struct lock_class_key reserved_rbtree_key;
ba395927 1767
51a63e67 1768static int dmar_init_reserved_ranges(void)
ba395927
KA		 1769{
1770 struct pci_dev *pdev = NULL;
1771 struct iova *iova;
1772 int i;
ba395927 1773
0fb5fe87
RM		 1774 init_iova_domain(&reserved_iova_list, VTD_PAGE_SIZE, IOVA_START_PFN,
1775 DMA_32BIT_PFN);
ba395927 1776
8a443df4
MG		 1777 lockdep_set_class(&reserved_iova_list.iova_rbtree_lock,
1778 &reserved_rbtree_key);
1779
ba395927
KA		 1780 /* IOAPIC ranges shouldn't be accessed by DMA */
1781 iova = reserve_iova(&reserved_iova_list, IOVA_PFN(IOAPIC_RANGE_START),
1782 IOVA_PFN(IOAPIC_RANGE_END));
51a63e67 1783 if (!iova) {
9f10e5bf 1784 pr_err("Reserve IOAPIC range failed\n");
51a63e67
JC		 1785 return -ENODEV;
1786 }
ba395927
KA		 1787
1788 /* Reserve all PCI MMIO to avoid peer-to-peer access */
1789 for_each_pci_dev(pdev) {
1790 struct resource *r;
1791
1792 for (i = 0; i < PCI_NUM_RESOURCES; i++) {
1793 r = &pdev->resource[i];
1794 if (!r->flags || !(r->flags & IORESOURCE_MEM))
1795 continue;
1a4a4551
DW		 1796 iova = reserve_iova(&reserved_iova_list,
1797 IOVA_PFN(r->start),
1798 IOVA_PFN(r->end));
51a63e67 1799 if (!iova) {
9f10e5bf 1800 pr_err("Reserve iova failed\n");
51a63e67
JC		 1801 return -ENODEV;
1802 }
ba395927
KA		 1803 }
1804 }
51a63e67 1805 return 0;
ba395927
KA		 1806}
1807
1808static void domain_reserve_special_ranges(struct dmar_domain *domain)
1809{
1810 copy_reserved_iova(&reserved_iova_list, &domain->iovad);
1811}
1812
1813static inline int guestwidth_to_adjustwidth(int gaw)
1814{
1815 int agaw;
1816 int r = (gaw - 12) % 9;
1817
1818 if (r == 0)
1819 agaw = gaw;
1820 else
1821 agaw = gaw + 9 - r;
1822 if (agaw > 64)
1823 agaw = 64;
1824 return agaw;
1825}
1826
1827static int domain_init(struct dmar_domain *domain, int guest_width)
1828{
1829 struct intel_iommu *iommu;
1830 int adjust_width, agaw;
1831 unsigned long sagaw;
1832
0fb5fe87
RM		 1833 init_iova_domain(&domain->iovad, VTD_PAGE_SIZE, IOVA_START_PFN,
1834 DMA_32BIT_PFN);
ba395927
KA		 1835 domain_reserve_special_ranges(domain);
1836
1837 /* calculate AGAW */
8c11e798 1838 iommu = domain_get_iommu(domain);
ba395927
KA		 1839 if (guest_width > cap_mgaw(iommu->cap))
1840 guest_width = cap_mgaw(iommu->cap);
1841 domain->gaw = guest_width;
1842 adjust_width = guestwidth_to_adjustwidth(guest_width);
1843 agaw = width_to_agaw(adjust_width);
1844 sagaw = cap_sagaw(iommu->cap);
1845 if (!test_bit(agaw, &sagaw)) {
1846 /* hardware doesn't support it, choose a bigger one */
9f10e5bf 1847 pr_debug("Hardware doesn't support agaw %d\n", agaw);
ba395927
KA		 1848 agaw = find_next_bit(&sagaw, 5, agaw);
1849 if (agaw >= 5)
1850 return -ENODEV;
1851 }
1852 domain->agaw = agaw;
ba395927 1853
8e604097
WH		 1854 if (ecap_coherent(iommu->ecap))
1855 domain->iommu_coherency = 1;
1856 else
1857 domain->iommu_coherency = 0;
1858
58c610bd
SY		 1859 if (ecap_sc_support(iommu->ecap))
1860 domain->iommu_snooping = 1;
1861 else
1862 domain->iommu_snooping = 0;
1863
214e39aa
DW		 1864 if (intel_iommu_superpage)
1865 domain->iommu_superpage = fls(cap_super_page_val(iommu->cap));
1866 else
1867 domain->iommu_superpage = 0;
1868
4c923d47 1869 domain->nid = iommu->node;
c7151a8d 1870
ba395927 1871 /* always allocate the top pgd */
4c923d47 1872 domain->pgd = (struct dma_pte *)alloc_pgtable_page(domain->nid);
ba395927
KA		 1873 if (!domain->pgd)
1874 return -ENOMEM;
5b6985ce 1875 __iommu_flush_cache(iommu, domain->pgd, PAGE_SIZE);
ba395927
KA		 1876 return 0;
1877}
1878
1879static void domain_exit(struct dmar_domain *domain)
1880{
46ebb7af
AW		 1881 struct dmar_drhd_unit *drhd;
1882 struct intel_iommu *iommu;
ea8ea460 1883 struct page *freelist = NULL;
ba395927
KA		 1884
1885 /* Domain 0 is reserved, so dont process it */
1886 if (!domain)
1887 return;
1888
7b668357
AW		 1889 /* Flush any lazy unmaps that may reference this domain */
1890 if (!intel_iommu_strict)
1891 flush_unmaps_timeout(0);
1892
92d03cc8 1893 /* remove associated devices */
ba395927 1894 domain_remove_dev_info(domain);
92d03cc8 1895
ba395927
KA		 1896 /* destroy iovas */
1897 put_iova_domain(&domain->iovad);
ba395927 1898
ea8ea460 1899 freelist = domain_unmap(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
ba395927 1900
92d03cc8 1901 /* clear attached or cached domains */
0e242612 1902 rcu_read_lock();
46ebb7af
AW		 1903 for_each_active_iommu(iommu, drhd)
1904 if (domain_type_is_vm(domain) ||
1905 test_bit(iommu->seq_id, domain->iommu_bmp))
1906 iommu_detach_domain(domain, iommu);
0e242612 1907 rcu_read_unlock();
2c2e2c38 1908
ea8ea460
DW		 1909 dma_free_pagelist(freelist);
1910
ba395927
KA		 1911 free_domain_mem(domain);
1912}
1913
64ae892b
DW		 1914static int domain_context_mapping_one(struct dmar_domain *domain,
1915 struct intel_iommu *iommu,
28ccce0d 1916 u8 bus, u8 devfn)
ba395927 1917{
28ccce0d
JR		 1918 int translation = CONTEXT_TT_MULTI_LEVEL;
1919 struct device_domain_info *info = NULL;
ba395927 1920 struct context_entry *context;
ba395927 1921 unsigned long flags;
ea6606b0 1922 struct dma_pte *pgd;
ea6606b0
WH		 1923 int id;
1924 int agaw;
28ccce0d
JR		 1925
1926 if (hw_pass_through && domain_type_is_si(domain))
1927 translation = CONTEXT_TT_PASS_THROUGH;
ba395927
KA		 1928
1929 pr_debug("Set context mapping for %02x:%02x.%d\n",
1930 bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
4ed0d3e6 1931
ba395927 1932 BUG_ON(!domain->pgd);
5331fe6f 1933
03ecc32c
DW		 1934 spin_lock_irqsave(&iommu->lock, flags);
1935 context = iommu_context_addr(iommu, bus, devfn, 1);
1936 spin_unlock_irqrestore(&iommu->lock, flags);
ba395927
KA		 1937 if (!context)
1938 return -ENOMEM;
1939 spin_lock_irqsave(&iommu->lock, flags);
c07e7d21 1940 if (context_present(context)) {
ba395927
KA		 1941 spin_unlock_irqrestore(&iommu->lock, flags);
1942 return 0;
1943 }
1944
cf484d0e
JR		 1945 context_clear_entry(context);
1946
ea6606b0
WH		 1947 id = domain->id;
1948 pgd = domain->pgd;
1949
ab8dfe25 1950 if (domain_type_is_vm_or_si(domain)) {
44bde614 1951 if (domain_type_is_vm(domain)) {
e2411427 1952 id = __iommu_attach_domain(domain, iommu);
fb170fb4 1953 if (id < 0) {
ea6606b0 1954 spin_unlock_irqrestore(&iommu->lock, flags);
9f10e5bf 1955 pr_err("%s: No free domain ids\n", iommu->name);
ea6606b0
WH		 1956 return -EFAULT;
1957 }
ea6606b0
WH		 1958 }
1959
1960 /* Skip top levels of page tables for
1961 * iommu which has less agaw than default.
1672af11 1962 * Unnecessary for PT mode.
ea6606b0 1963 */
1672af11
CW		 1964 if (translation != CONTEXT_TT_PASS_THROUGH) {
1965 for (agaw = domain->agaw; agaw != iommu->agaw; agaw--) {
1966 pgd = phys_to_virt(dma_pte_addr(pgd));
1967 if (!dma_pte_present(pgd)) {
1968 spin_unlock_irqrestore(&iommu->lock, flags);
1969 return -ENOMEM;
1970 }
ea6606b0
WH		 1971 }
1972 }
1973 }
1974
1975 context_set_domain_id(context, id);
4ed0d3e6 1976
93a23a72 1977 if (translation != CONTEXT_TT_PASS_THROUGH) {
64ae892b 1978 info = iommu_support_dev_iotlb(domain, iommu, bus, devfn);
93a23a72
YZ		 1979 translation = info ? CONTEXT_TT_DEV_IOTLB :
1980 CONTEXT_TT_MULTI_LEVEL;
1981 }
4ed0d3e6
FY		 1982 /*
1983 * In pass through mode, AW must be programmed to indicate the largest
1984 * AGAW value supported by hardware. And ASR is ignored by hardware.
1985 */
93a23a72 1986 if (unlikely(translation == CONTEXT_TT_PASS_THROUGH))
4ed0d3e6 1987 context_set_address_width(context, iommu->msagaw);
93a23a72
YZ		 1988 else {
1989 context_set_address_root(context, virt_to_phys(pgd));
1990 context_set_address_width(context, iommu->agaw);
1991 }
4ed0d3e6
FY		 1992
1993 context_set_translation_type(context, translation);
c07e7d21
MM		 1994 context_set_fault_enable(context);
1995 context_set_present(context);
5331fe6f 1996 domain_flush_cache(domain, context, sizeof(*context));
ba395927 1997
4c25a2c1
DW		 1998 /*
1999 * It's a non-present to present mapping. If hardware doesn't cache
2000 * non-present entry we only need to flush the write-buffer. If the
2001 * _does_ cache non-present entries, then it does so in the special
2002 * domain #0, which we have to flush:
2003 */
2004 if (cap_caching_mode(iommu->cap)) {
2005 iommu->flush.flush_context(iommu, 0,
2006 (((u16)bus) << 8) | devfn,
2007 DMA_CCMD_MASK_NOBIT,
2008 DMA_CCMD_DEVICE_INVL);
18fd779a 2009 iommu->flush.flush_iotlb(iommu, id, 0, 0, DMA_TLB_DSI_FLUSH);
4c25a2c1 2010 } else {
ba395927 2011 iommu_flush_write_buffer(iommu);
4c25a2c1 2012 }
93a23a72 2013 iommu_enable_dev_iotlb(info);
ba395927 2014 spin_unlock_irqrestore(&iommu->lock, flags);
c7151a8d 2015
fb170fb4
JL		 2016 domain_attach_iommu(domain, iommu);
2017
ba395927
KA		 2018 return 0;
2019}
2020
579305f7
AW		 2021struct domain_context_mapping_data {
2022 struct dmar_domain *domain;
2023 struct intel_iommu *iommu;
579305f7
AW		 2024};
2025
2026static int domain_context_mapping_cb(struct pci_dev *pdev,
2027 u16 alias, void *opaque)
2028{
2029 struct domain_context_mapping_data *data = opaque;
2030
2031 return domain_context_mapping_one(data->domain, data->iommu,
28ccce0d 2032 PCI_BUS_NUM(alias), alias & 0xff);
579305f7
AW		 2033}
2034
ba395927 2035static int
28ccce0d 2036domain_context_mapping(struct dmar_domain *domain, struct device *dev)
ba395927 2037{
64ae892b 2038 struct intel_iommu *iommu;
156baca8 2039 u8 bus, devfn;
579305f7 2040 struct domain_context_mapping_data data;
64ae892b 2041
e1f167f3 2042 iommu = device_to_iommu(dev, &bus, &devfn);
64ae892b
DW		 2043 if (!iommu)
2044 return -ENODEV;
ba395927 2045
579305f7 2046 if (!dev_is_pci(dev))
28ccce0d 2047 return domain_context_mapping_one(domain, iommu, bus, devfn);
579305f7
AW		 2048
2049 data.domain = domain;
2050 data.iommu = iommu;
579305f7
AW		 2051
2052 return pci_for_each_dma_alias(to_pci_dev(dev),
2053 &domain_context_mapping_cb, &data);
2054}
2055
2056static int domain_context_mapped_cb(struct pci_dev *pdev,
2057 u16 alias, void *opaque)
2058{
2059 struct intel_iommu *iommu = opaque;
2060
2061 return !device_context_mapped(iommu, PCI_BUS_NUM(alias), alias & 0xff);
ba395927
KA		 2062}
2063
e1f167f3 2064static int domain_context_mapped(struct device *dev)
ba395927 2065{
5331fe6f 2066 struct intel_iommu *iommu;
156baca8 2067 u8 bus, devfn;
5331fe6f 2068
e1f167f3 2069 iommu = device_to_iommu(dev, &bus, &devfn);
5331fe6f
WH		 2070 if (!iommu)
2071 return -ENODEV;
ba395927 2072
579305f7
AW		 2073 if (!dev_is_pci(dev))
2074 return device_context_mapped(iommu, bus, devfn);
e1f167f3 2075
579305f7
AW		 2076 return !pci_for_each_dma_alias(to_pci_dev(dev),
2077 domain_context_mapped_cb, iommu);
ba395927
KA		 2078}
2079
f532959b
FY		 2080/* Returns a number of VTD pages, but aligned to MM page size */
2081static inline unsigned long aligned_nrpages(unsigned long host_addr,
2082 size_t size)
2083{
2084 host_addr &= ~PAGE_MASK;
2085 return PAGE_ALIGN(host_addr + size) >> VTD_PAGE_SHIFT;
2086}
2087
6dd9a7c7
YS		 2088/* Return largest possible superpage level for a given mapping */
2089static inline int hardware_largepage_caps(struct dmar_domain *domain,
2090 unsigned long iov_pfn,
2091 unsigned long phy_pfn,
2092 unsigned long pages)
2093{
2094 int support, level = 1;
2095 unsigned long pfnmerge;
2096
2097 support = domain->iommu_superpage;
2098
2099 /* To use a large page, the virtual *and* physical addresses
2100 must be aligned to 2MiB/1GiB/etc. Lower bits set in either
2101 of them will mean we have to use smaller pages. So just
2102 merge them and check both at once. */
2103 pfnmerge = iov_pfn | phy_pfn;
2104
2105 while (support && !(pfnmerge & ~VTD_STRIDE_MASK)) {
2106 pages >>= VTD_STRIDE_SHIFT;
2107 if (!pages)
2108 break;
2109 pfnmerge >>= VTD_STRIDE_SHIFT;
2110 level++;
2111 support--;
2112 }
2113 return level;
2114}
2115
9051aa02
DW		 2116static int __domain_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
2117 struct scatterlist *sg, unsigned long phys_pfn,
2118 unsigned long nr_pages, int prot)
e1605495
DW		 2119{
2120 struct dma_pte *first_pte = NULL, *pte = NULL;
9051aa02 2121 phys_addr_t uninitialized_var(pteval);
cc4f14aa 2122 unsigned long sg_res = 0;
6dd9a7c7
YS		 2123 unsigned int largepage_lvl = 0;
2124 unsigned long lvl_pages = 0;
e1605495 2125
162d1b10 2126 BUG_ON(!domain_pfn_supported(domain, iov_pfn + nr_pages - 1));
e1605495
DW		 2127
2128 if ((prot & (DMA_PTE_READ|DMA_PTE_WRITE)) == 0)
2129 return -EINVAL;
2130
2131 prot &= DMA_PTE_READ | DMA_PTE_WRITE | DMA_PTE_SNP;
2132
cc4f14aa
JL		 2133 if (!sg) {
2134 sg_res = nr_pages;
9051aa02
DW		 2135 pteval = ((phys_addr_t)phys_pfn << VTD_PAGE_SHIFT) | prot;
2136 }
2137
6dd9a7c7 2138 while (nr_pages > 0) {
c85994e4
DW		 2139 uint64_t tmp;
2140
e1605495 2141 if (!sg_res) {
f532959b 2142 sg_res = aligned_nrpages(sg->offset, sg->length);
e1605495
DW		 2143 sg->dma_address = ((dma_addr_t)iov_pfn << VTD_PAGE_SHIFT) + sg->offset;
2144 sg->dma_length = sg->length;
2145 pteval = page_to_phys(sg_page(sg)) | prot;
6dd9a7c7 2146 phys_pfn = pteval >> VTD_PAGE_SHIFT;
e1605495 2147 }
6dd9a7c7 2148
e1605495 2149 if (!pte) {
6dd9a7c7
YS		 2150 largepage_lvl = hardware_largepage_caps(domain, iov_pfn, phys_pfn, sg_res);
2151
5cf0a76f 2152 first_pte = pte = pfn_to_dma_pte(domain, iov_pfn, &largepage_lvl);
e1605495
DW		 2153 if (!pte)
2154 return -ENOMEM;
6dd9a7c7 2155 /* It is large page*/
6491d4d0 2156 if (largepage_lvl > 1) {
6dd9a7c7 2157 pteval |= DMA_PTE_LARGE_PAGE;
d41a4adb
JL		 2158 lvl_pages = lvl_to_nr_pages(largepage_lvl);
2159 /*
2160 * Ensure that old small page tables are
2161 * removed to make room for superpage,
2162 * if they exist.
2163 */
6491d4d0 2164 dma_pte_free_pagetable(domain, iov_pfn,
d41a4adb 2165 iov_pfn + lvl_pages - 1);
6491d4d0 2166 } else {
6dd9a7c7 2167 pteval &= ~(uint64_t)DMA_PTE_LARGE_PAGE;
6491d4d0 2168 }
6dd9a7c7 2169
e1605495
DW		 2170 }
2171 /* We don't need lock here, nobody else
2172 * touches the iova range
2173 */
7766a3fb 2174 tmp = cmpxchg64_local(&pte->val, 0ULL, pteval);
c85994e4 2175 if (tmp) {
1bf20f0d 2176 static int dumps = 5;
9f10e5bf
JR		 2177 pr_crit("ERROR: DMA PTE for vPFN 0x%lx already set (to %llx not %llx)\n",
2178 iov_pfn, tmp, (unsigned long long)pteval);
1bf20f0d
DW		 2179 if (dumps) {
2180 dumps--;
2181 debug_dma_dump_mappings(NULL);
2182 }
2183 WARN_ON(1);
2184 }
6dd9a7c7
YS		 2185
2186 lvl_pages = lvl_to_nr_pages(largepage_lvl);
2187
2188 BUG_ON(nr_pages < lvl_pages);
2189 BUG_ON(sg_res < lvl_pages);
2190
2191 nr_pages -= lvl_pages;
2192 iov_pfn += lvl_pages;
2193 phys_pfn += lvl_pages;
2194 pteval += lvl_pages * VTD_PAGE_SIZE;
2195 sg_res -= lvl_pages;
2196
2197 /* If the next PTE would be the first in a new page, then we
2198 need to flush the cache on the entries we've just written.
2199 And then we'll need to recalculate 'pte', so clear it and
2200 let it get set again in the if (!pte) block above.
2201
2202 If we're done (!nr_pages) we need to flush the cache too.
2203
2204 Also if we've been setting superpages, we may need to
2205 recalculate 'pte' and switch back to smaller pages for the
2206 end of the mapping, if the trailing size is not enough to
2207 use another superpage (i.e. sg_res < lvl_pages). */
e1605495 2208 pte++;
6dd9a7c7
YS		 2209 if (!nr_pages || first_pte_in_page(pte) ||
2210 (largepage_lvl > 1 && sg_res < lvl_pages)) {
e1605495
DW		 2211 domain_flush_cache(domain, first_pte,
2212 (void *)pte - (void *)first_pte);
2213 pte = NULL;
2214 }
6dd9a7c7
YS		 2215
2216 if (!sg_res && nr_pages)
e1605495
DW		 2217 sg = sg_next(sg);
2218 }
2219 return 0;
2220}
2221
9051aa02
DW		 2222static inline int domain_sg_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
2223 struct scatterlist *sg, unsigned long nr_pages,
2224 int prot)
ba395927 2225{
9051aa02
DW		 2226 return __domain_mapping(domain, iov_pfn, sg, 0, nr_pages, prot);
2227}
6f6a00e4 2228
9051aa02
DW		 2229static inline int domain_pfn_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
2230 unsigned long phys_pfn, unsigned long nr_pages,
2231 int prot)
2232{
2233 return __domain_mapping(domain, iov_pfn, NULL, phys_pfn, nr_pages, prot);
ba395927
KA		 2234}
2235
c7151a8d 2236static void iommu_detach_dev(struct intel_iommu *iommu, u8 bus, u8 devfn)
ba395927 2237{
c7151a8d
WH		 2238 if (!iommu)
2239 return;
8c11e798
WH		 2240
2241 clear_context_table(iommu, bus, devfn);
2242 iommu->flush.flush_context(iommu, 0, 0, 0,
4c25a2c1 2243 DMA_CCMD_GLOBAL_INVL);
1f0ef2aa 2244 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
ba395927
KA		 2245}
2246
109b9b04
DW		 2247static inline void unlink_domain_info(struct device_domain_info *info)
2248{
2249 assert_spin_locked(&device_domain_lock);
2250 list_del(&info->link);
2251 list_del(&info->global);
2252 if (info->dev)
0bcb3e28 2253 info->dev->archdata.iommu = NULL;
109b9b04
DW		 2254}
2255
ba395927
KA		 2256static void domain_remove_dev_info(struct dmar_domain *domain)
2257{
3a74ca01 2258 struct device_domain_info *info, *tmp;
fb170fb4 2259 unsigned long flags;
ba395927
KA		 2260
2261 spin_lock_irqsave(&device_domain_lock, flags);
3a74ca01 2262 list_for_each_entry_safe(info, tmp, &domain->devices, link) {
109b9b04 2263 unlink_domain_info(info);
ba395927
KA		 2264 spin_unlock_irqrestore(&device_domain_lock, flags);
2265
93a23a72 2266 iommu_disable_dev_iotlb(info);
7c7faa11 2267 iommu_detach_dev(info->iommu, info->bus, info->devfn);
ba395927 2268
ab8dfe25 2269 if (domain_type_is_vm(domain)) {
7c7faa11 2270 iommu_detach_dependent_devices(info->iommu, info->dev);
fb170fb4 2271 domain_detach_iommu(domain, info->iommu);
92d03cc8
JL		 2272 }
2273
2274 free_devinfo_mem(info);
ba395927
KA		 2275 spin_lock_irqsave(&device_domain_lock, flags);
2276 }
2277 spin_unlock_irqrestore(&device_domain_lock, flags);
2278}
2279
2280/*
2281 * find_domain
1525a29a 2282 * Note: we use struct device->archdata.iommu stores the info
ba395927 2283 */
1525a29a 2284static struct dmar_domain *find_domain(struct device *dev)
ba395927
KA		 2285{
2286 struct device_domain_info *info;
2287
2288 /* No lock here, assumes no domain exit in normal case */
1525a29a 2289 info = dev->archdata.iommu;
ba395927
KA		 2290 if (info)
2291 return info->domain;
2292 return NULL;
2293}
2294
5a8f40e8 2295static inline struct device_domain_info *
745f2586
JL		 2296dmar_search_domain_by_dev_info(int segment, int bus, int devfn)
2297{
2298 struct device_domain_info *info;
2299
2300 list_for_each_entry(info, &device_domain_list, global)
41e80dca 2301 if (info->iommu->segment == segment && info->bus == bus &&
745f2586 2302 info->devfn == devfn)
5a8f40e8 2303 return info;
745f2586
JL		 2304
2305 return NULL;
2306}
2307
5a8f40e8 2308static struct dmar_domain *dmar_insert_dev_info(struct intel_iommu *iommu,
41e80dca 2309 int bus, int devfn,
b718cd3d
DW		 2310 struct device *dev,
2311 struct dmar_domain *domain)
745f2586 2312{
5a8f40e8 2313 struct dmar_domain *found = NULL;
745f2586
JL		 2314 struct device_domain_info *info;
2315 unsigned long flags;
2316
2317 info = alloc_devinfo_mem();
2318 if (!info)
b718cd3d 2319 return NULL;
745f2586 2320
745f2586
JL		 2321 info->bus = bus;
2322 info->devfn = devfn;
2323 info->dev = dev;
2324 info->domain = domain;
5a8f40e8 2325 info->iommu = iommu;
745f2586
JL		 2326
2327 spin_lock_irqsave(&device_domain_lock, flags);
2328 if (dev)
0bcb3e28 2329 found = find_domain(dev);
5a8f40e8
DW		 2330 else {
2331 struct device_domain_info *info2;
41e80dca 2332 info2 = dmar_search_domain_by_dev_info(iommu->segment, bus, devfn);
5a8f40e8
DW		 2333 if (info2)
2334 found = info2->domain;
2335 }
745f2586
JL		 2336 if (found) {
2337 spin_unlock_irqrestore(&device_domain_lock, flags);
2338 free_devinfo_mem(info);
b718cd3d
DW		 2339 /* Caller must free the original domain */
2340 return found;
745f2586
JL		 2341 }
2342
b718cd3d
DW		 2343 list_add(&info->link, &domain->devices);
2344 list_add(&info->global, &device_domain_list);
2345 if (dev)
2346 dev->archdata.iommu = info;
2347 spin_unlock_irqrestore(&device_domain_lock, flags);
2348
2349 return domain;
745f2586
JL		 2350}
2351
579305f7
AW		 2352static int get_last_alias(struct pci_dev *pdev, u16 alias, void *opaque)
2353{
2354 *(u16 *)opaque = alias;
2355 return 0;
2356}
2357
ba395927 2358/* domain is initialized */
146922ec 2359static struct dmar_domain *get_domain_for_dev(struct device *dev, int gaw)
ba395927 2360{
579305f7
AW		 2361 struct dmar_domain *domain, *tmp;
2362 struct intel_iommu *iommu;
5a8f40e8 2363 struct device_domain_info *info;
579305f7 2364 u16 dma_alias;
ba395927 2365 unsigned long flags;
aa4d066a 2366 u8 bus, devfn;
ba395927 2367
146922ec 2368 domain = find_domain(dev);
ba395927
KA		 2369 if (domain)
2370 return domain;
2371
579305f7
AW		 2372 iommu = device_to_iommu(dev, &bus, &devfn);
2373 if (!iommu)
2374 return NULL;
2375
146922ec
DW		 2376 if (dev_is_pci(dev)) {
2377 struct pci_dev *pdev = to_pci_dev(dev);
276dbf99 2378
579305f7
AW		 2379 pci_for_each_dma_alias(pdev, get_last_alias, &dma_alias);
2380
2381 spin_lock_irqsave(&device_domain_lock, flags);
2382 info = dmar_search_domain_by_dev_info(pci_domain_nr(pdev->bus),
2383 PCI_BUS_NUM(dma_alias),
2384 dma_alias & 0xff);
2385 if (info) {
2386 iommu = info->iommu;
2387 domain = info->domain;
5a8f40e8 2388 }
579305f7 2389 spin_unlock_irqrestore(&device_domain_lock, flags);
ba395927 2390
579305f7
AW		 2391 /* DMA alias already has a domain, uses it */
2392 if (info)
2393 goto found_domain;
2394 }
ba395927 2395
146922ec 2396 /* Allocate and initialize new domain for the device */
ab8dfe25 2397 domain = alloc_domain(0);
745f2586 2398 if (!domain)
579305f7 2399 return NULL;
44bde614
JL		 2400 domain->id = iommu_attach_domain(domain, iommu);
2401 if (domain->id < 0) {
2fe9723d 2402 free_domain_mem(domain);
579305f7 2403 return NULL;
2c2e2c38 2404 }
fb170fb4 2405 domain_attach_iommu(domain, iommu);
579305f7
AW		 2406 if (domain_init(domain, gaw)) {
2407 domain_exit(domain);
2408 return NULL;
2c2e2c38 2409 }
ba395927 2410
579305f7
AW		 2411 /* register PCI DMA alias device */
2412 if (dev_is_pci(dev)) {
2413 tmp = dmar_insert_dev_info(iommu, PCI_BUS_NUM(dma_alias),
2414 dma_alias & 0xff, NULL, domain);
2415
2416 if (!tmp || tmp != domain) {
2417 domain_exit(domain);
2418 domain = tmp;
2419 }
2420
b718cd3d 2421 if (!domain)
579305f7 2422 return NULL;
ba395927
KA		 2423 }
2424
2425found_domain:
579305f7
AW		 2426 tmp = dmar_insert_dev_info(iommu, bus, devfn, dev, domain);
2427
2428 if (!tmp || tmp != domain) {
2429 domain_exit(domain);
2430 domain = tmp;
2431 }
b718cd3d
DW		 2432
2433 return domain;
ba395927
KA		 2434}
2435
2c2e2c38 2436static int iommu_identity_mapping;
e0fc7e0b
DW		 2437#define IDENTMAP_ALL 1
2438#define IDENTMAP_GFX 2
2439#define IDENTMAP_AZALIA 4
2c2e2c38 2440
b213203e
DW		 2441static int iommu_domain_identity_map(struct dmar_domain *domain,
2442 unsigned long long start,
2443 unsigned long long end)
ba395927 2444{
c5395d5c
DW		 2445 unsigned long first_vpfn = start >> VTD_PAGE_SHIFT;
2446 unsigned long last_vpfn = end >> VTD_PAGE_SHIFT;
2447
2448 if (!reserve_iova(&domain->iovad, dma_to_mm_pfn(first_vpfn),
2449 dma_to_mm_pfn(last_vpfn))) {
9f10e5bf 2450 pr_err("Reserving iova failed\n");
b213203e 2451 return -ENOMEM;
ba395927
KA		 2452 }
2453
c5395d5c
DW		 2454 pr_debug("Mapping reserved region %llx-%llx for domain %d\n",
2455 start, end, domain->id);
ba395927
KA		 2456 /*
2457 * RMRR range might have overlap with physical memory range,
2458 * clear it first
2459 */
c5395d5c 2460 dma_pte_clear_range(domain, first_vpfn, last_vpfn);
ba395927 2461
c5395d5c
DW		 2462 return domain_pfn_mapping(domain, first_vpfn, first_vpfn,
2463 last_vpfn - first_vpfn + 1,
61df7443 2464 DMA_PTE_READ|DMA_PTE_WRITE);
b213203e
DW		 2465}
2466
0b9d9753 2467static int iommu_prepare_identity_map(struct device *dev,
b213203e
DW		 2468 unsigned long long start,
2469 unsigned long long end)
2470{
2471 struct dmar_domain *domain;
2472 int ret;
2473
0b9d9753 2474 domain = get_domain_for_dev(dev, DEFAULT_DOMAIN_ADDRESS_WIDTH);
b213203e
DW		 2475 if (!domain)
2476 return -ENOMEM;
2477
19943b0e
DW		 2478 /* For _hardware_ passthrough, don't bother. But for software
2479 passthrough, we do it anyway -- it may indicate a memory
2480 range which is reserved in E820, so which didn't get set
2481 up to start with in si_domain */
2482 if (domain == si_domain && hw_pass_through) {
9f10e5bf
JR		 2483 pr_warn("Ignoring identity map for HW passthrough device %s [0x%Lx - 0x%Lx]\n",
2484 dev_name(dev), start, end);
19943b0e
DW		 2485 return 0;
2486 }
2487
9f10e5bf
JR		 2488 pr_info("Setting identity map for device %s [0x%Lx - 0x%Lx]\n",
2489 dev_name(dev), start, end);
2490
5595b528
DW		 2491 if (end < start) {
2492 WARN(1, "Your BIOS is broken; RMRR ends before it starts!\n"
2493 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
2494 dmi_get_system_info(DMI_BIOS_VENDOR),
2495 dmi_get_system_info(DMI_BIOS_VERSION),
2496 dmi_get_system_info(DMI_PRODUCT_VERSION));
2497 ret = -EIO;
2498 goto error;
2499 }
2500
2ff729f5
DW		 2501 if (end >> agaw_to_width(domain->agaw)) {
2502 WARN(1, "Your BIOS is broken; RMRR exceeds permitted address width (%d bits)\n"
2503 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
2504 agaw_to_width(domain->agaw),
2505 dmi_get_system_info(DMI_BIOS_VENDOR),
2506 dmi_get_system_info(DMI_BIOS_VERSION),
2507 dmi_get_system_info(DMI_PRODUCT_VERSION));
2508 ret = -EIO;
2509 goto error;
2510 }
19943b0e 2511
b213203e 2512 ret = iommu_domain_identity_map(domain, start, end);
ba395927
KA		 2513 if (ret)
2514 goto error;
2515
2516 /* context entry init */
28ccce0d 2517 ret = domain_context_mapping(domain, dev);
b213203e
DW		 2518 if (ret)
2519 goto error;
2520
2521 return 0;
2522
2523 error:
ba395927
KA		 2524 domain_exit(domain);
2525 return ret;
ba395927
KA		 2526}
2527
2528static inline int iommu_prepare_rmrr_dev(struct dmar_rmrr_unit *rmrr,
0b9d9753 2529 struct device *dev)
ba395927 2530{
0b9d9753 2531 if (dev->archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO)
ba395927 2532 return 0;
0b9d9753
DW		 2533 return iommu_prepare_identity_map(dev, rmrr->base_address,
2534 rmrr->end_address);
ba395927
KA		 2535}
2536
d3f13810 2537#ifdef CONFIG_INTEL_IOMMU_FLOPPY_WA
49a0429e
KA		 2538static inline void iommu_prepare_isa(void)
2539{
2540 struct pci_dev *pdev;
2541 int ret;
2542
2543 pdev = pci_get_class(PCI_CLASS_BRIDGE_ISA << 8, NULL);
2544 if (!pdev)
2545 return;
2546
9f10e5bf 2547 pr_info("Prepare 0-16MiB unity mapping for LPC\n");
0b9d9753 2548 ret = iommu_prepare_identity_map(&pdev->dev, 0, 16*1024*1024 - 1);
49a0429e
KA		 2549
2550 if (ret)
9f10e5bf 2551 pr_err("Failed to create 0-16MiB identity map - floppy might not work\n");
49a0429e 2552
9b27e82d 2553 pci_dev_put(pdev);
49a0429e
KA		 2554}
2555#else
2556static inline void iommu_prepare_isa(void)
2557{
2558 return;
2559}
d3f13810 2560#endif /* !CONFIG_INTEL_IOMMU_FLPY_WA */
49a0429e 2561
2c2e2c38 2562static int md_domain_init(struct dmar_domain *domain, int guest_width);
c7ab48d2 2563
071e1374 2564static int __init si_domain_init(int hw)
2c2e2c38
FY		 2565{
2566 struct dmar_drhd_unit *drhd;
2567 struct intel_iommu *iommu;
c7ab48d2 2568 int nid, ret = 0;
44bde614 2569 bool first = true;
2c2e2c38 2570
ab8dfe25 2571 si_domain = alloc_domain(DOMAIN_FLAG_STATIC_IDENTITY);
2c2e2c38
FY		 2572 if (!si_domain)
2573 return -EFAULT;
2574
2c2e2c38
FY		 2575 for_each_active_iommu(iommu, drhd) {
2576 ret = iommu_attach_domain(si_domain, iommu);
fb170fb4 2577 if (ret < 0) {
2c2e2c38
FY		 2578 domain_exit(si_domain);
2579 return -EFAULT;
44bde614
JL		 2580 } else if (first) {
2581 si_domain->id = ret;
2582 first = false;
2583 } else if (si_domain->id != ret) {
2584 domain_exit(si_domain);
2585 return -EFAULT;
2c2e2c38 2586 }
fb170fb4 2587 domain_attach_iommu(si_domain, iommu);
2c2e2c38
FY		 2588 }
2589
2590 if (md_domain_init(si_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
2591 domain_exit(si_domain);
2592 return -EFAULT;
2593 }
2594
9f10e5bf 2595 pr_debug("Identity mapping domain is domain %d\n",
9544c003 2596 si_domain->id);
2c2e2c38 2597
19943b0e
DW		 2598 if (hw)
2599 return 0;
2600
c7ab48d2 2601 for_each_online_node(nid) {
5dfe8660
TH		 2602 unsigned long start_pfn, end_pfn;
2603 int i;
2604
2605 for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, NULL) {
2606 ret = iommu_domain_identity_map(si_domain,
2607 PFN_PHYS(start_pfn), PFN_PHYS(end_pfn));
2608 if (ret)
2609 return ret;
2610 }
c7ab48d2
DW		 2611 }
2612
2c2e2c38
FY		 2613 return 0;
2614}
2615
9b226624 2616static int identity_mapping(struct device *dev)
2c2e2c38
FY		 2617{
2618 struct device_domain_info *info;
2619
2620 if (likely(!iommu_identity_mapping))
2621 return 0;
2622
9b226624 2623 info = dev->archdata.iommu;
cb452a40
MT		 2624 if (info && info != DUMMY_DEVICE_DOMAIN_INFO)
2625 return (info->domain == si_domain);
2c2e2c38 2626
2c2e2c38
FY		 2627 return 0;
2628}
2629
28ccce0d 2630static int domain_add_dev_info(struct dmar_domain *domain, struct device *dev)
2c2e2c38 2631{
0ac72664 2632 struct dmar_domain *ndomain;
5a8f40e8 2633 struct intel_iommu *iommu;
156baca8 2634 u8 bus, devfn;
5fe60f4e 2635 int ret;
2c2e2c38 2636
5913c9bf 2637 iommu = device_to_iommu(dev, &bus, &devfn);
5a8f40e8
DW		 2638 if (!iommu)
2639 return -ENODEV;
2640
5913c9bf 2641 ndomain = dmar_insert_dev_info(iommu, bus, devfn, dev, domain);
0ac72664
DW		 2642 if (ndomain != domain)
2643 return -EBUSY;
2c2e2c38 2644
28ccce0d 2645 ret = domain_context_mapping(domain, dev);
e2ad23d0 2646 if (ret) {
5913c9bf 2647 domain_remove_one_dev_info(domain, dev);
e2ad23d0
DW		 2648 return ret;
2649 }
2650
2c2e2c38
FY		 2651 return 0;
2652}
2653
0b9d9753 2654static bool device_has_rmrr(struct device *dev)
ea2447f7
TM		 2655{
2656 struct dmar_rmrr_unit *rmrr;
832bd858 2657 struct device *tmp;
ea2447f7
TM		 2658 int i;
2659
0e242612 2660 rcu_read_lock();
ea2447f7 2661 for_each_rmrr_units(rmrr) {
b683b230
JL		 2662 /*
2663 * Return TRUE if this RMRR contains the device that
2664 * is passed in.
2665 */
2666 for_each_active_dev_scope(rmrr->devices,
2667 rmrr->devices_cnt, i, tmp)
0b9d9753 2668 if (tmp == dev) {
0e242612 2669 rcu_read_unlock();
ea2447f7 2670 return true;
b683b230 2671 }
ea2447f7 2672 }
0e242612 2673 rcu_read_unlock();
ea2447f7
TM		 2674 return false;
2675}
2676
c875d2c1
AW		 2677/*
2678 * There are a couple cases where we need to restrict the functionality of
2679 * devices associated with RMRRs. The first is when evaluating a device for
2680 * identity mapping because problems exist when devices are moved in and out
2681 * of domains and their respective RMRR information is lost. This means that
2682 * a device with associated RMRRs will never be in a "passthrough" domain.
2683 * The second is use of the device through the IOMMU API. This interface
2684 * expects to have full control of the IOVA space for the device. We cannot
2685 * satisfy both the requirement that RMRR access is maintained and have an
2686 * unencumbered IOVA space. We also have no ability to quiesce the device's
2687 * use of the RMRR space or even inform the IOMMU API user of the restriction.
2688 * We therefore prevent devices associated with an RMRR from participating in
2689 * the IOMMU API, which eliminates them from device assignment.
2690 *
2691 * In both cases we assume that PCI USB devices with RMRRs have them largely
2692 * for historical reasons and that the RMRR space is not actively used post
2693 * boot. This exclusion may change if vendors begin to abuse it.
18436afd
DW		 2694 *
2695 * The same exception is made for graphics devices, with the requirement that
2696 * any use of the RMRR regions will be torn down before assigning the device
2697 * to a guest.
c875d2c1
AW		 2698 */
2699static bool device_is_rmrr_locked(struct device *dev)
2700{
2701 if (!device_has_rmrr(dev))
2702 return false;
2703
2704 if (dev_is_pci(dev)) {
2705 struct pci_dev *pdev = to_pci_dev(dev);
2706
18436afd 2707 if (IS_USB_DEVICE(pdev) || IS_GFX_DEVICE(pdev))
c875d2c1
AW		 2708 return false;
2709 }
2710
2711 return true;
2712}
2713
3bdb2591 2714static int iommu_should_identity_map(struct device *dev, int startup)
6941af28 2715{
ea2447f7 2716
3bdb2591
DW		 2717 if (dev_is_pci(dev)) {
2718 struct pci_dev *pdev = to_pci_dev(dev);
ea2447f7 2719
c875d2c1 2720 if (device_is_rmrr_locked(dev))
3bdb2591 2721 return 0;
e0fc7e0b 2722
3bdb2591
DW		 2723 if ((iommu_identity_mapping & IDENTMAP_AZALIA) && IS_AZALIA(pdev))
2724 return 1;
e0fc7e0b 2725
3bdb2591
DW		 2726 if ((iommu_identity_mapping & IDENTMAP_GFX) && IS_GFX_DEVICE(pdev))
2727 return 1;
6941af28 2728
3bdb2591 2729 if (!(iommu_identity_mapping & IDENTMAP_ALL))
3dfc813d 2730 return 0;
3bdb2591
DW		 2731
2732 /*
2733 * We want to start off with all devices in the 1:1 domain, and
2734 * take them out later if we find they can't access all of memory.
2735 *
2736 * However, we can't do this for PCI devices behind bridges,
2737 * because all PCI devices behind the same bridge will end up
2738 * with the same source-id on their transactions.
2739 *
2740 * Practically speaking, we can't change things around for these
2741 * devices at run-time, because we can't be sure there'll be no
2742 * DMA transactions in flight for any of their siblings.
2743 *
2744 * So PCI devices (unless they're on the root bus) as well as
2745 * their parent PCI-PCI or PCIe-PCI bridges must be left _out_ of
2746 * the 1:1 domain, just in _case_ one of their siblings turns out
2747 * not to be able to map all of memory.
2748 */
2749 if (!pci_is_pcie(pdev)) {
2750 if (!pci_is_root_bus(pdev->bus))
2751 return 0;
2752 if (pdev->class >> 8 == PCI_CLASS_BRIDGE_PCI)
2753 return 0;
2754 } else if (pci_pcie_type(pdev) == PCI_EXP_TYPE_PCI_BRIDGE)
3dfc813d 2755 return 0;
3bdb2591
DW		 2756 } else {
2757 if (device_has_rmrr(dev))
2758 return 0;
2759 }
3dfc813d 2760
3bdb2591 2761 /*
3dfc813d 2762 * At boot time, we don't yet know if devices will be 64-bit capable.
3bdb2591 2763 * Assume that they will — if they turn out not to be, then we can
3dfc813d
DW		 2764 * take them out of the 1:1 domain later.
2765 */
8fcc5372
CW		 2766 if (!startup) {
2767 /*
2768 * If the device's dma_mask is less than the system's memory
2769 * size then this is not a candidate for identity mapping.
2770 */
3bdb2591 2771 u64 dma_mask = *dev->dma_mask;
8fcc5372 2772
3bdb2591
DW		 2773 if (dev->coherent_dma_mask &&
2774 dev->coherent_dma_mask < dma_mask)
2775 dma_mask = dev->coherent_dma_mask;
8fcc5372 2776
3bdb2591 2777 return dma_mask >= dma_get_required_mask(dev);
8fcc5372 2778 }
6941af28
DW		 2779
2780 return 1;
2781}
2782
cf04eee8
DW		 2783static int __init dev_prepare_static_identity_mapping(struct device *dev, int hw)
2784{
2785 int ret;
2786
2787 if (!iommu_should_identity_map(dev, 1))
2788 return 0;
2789
28ccce0d 2790 ret = domain_add_dev_info(si_domain, dev);
cf04eee8 2791 if (!ret)
9f10e5bf
JR		 2792 pr_info("%s identity mapping for device %s\n",
2793 hw ? "Hardware" : "Software", dev_name(dev));
cf04eee8
DW		 2794 else if (ret == -ENODEV)
2795 /* device not associated with an iommu */
2796 ret = 0;
2797
2798 return ret;
2799}
2800
2801
071e1374 2802static int __init iommu_prepare_static_identity_mapping(int hw)
2c2e2c38 2803{
2c2e2c38 2804 struct pci_dev *pdev = NULL;
cf04eee8
DW		 2805 struct dmar_drhd_unit *drhd;
2806 struct intel_iommu *iommu;
2807 struct device *dev;
2808 int i;
2809 int ret = 0;
2c2e2c38 2810
2c2e2c38 2811 for_each_pci_dev(pdev) {
cf04eee8
DW		 2812 ret = dev_prepare_static_identity_mapping(&pdev->dev, hw);
2813 if (ret)
2814 return ret;
2815 }
2816
2817 for_each_active_iommu(iommu, drhd)
2818 for_each_active_dev_scope(drhd->devices, drhd->devices_cnt, i, dev) {
2819 struct acpi_device_physical_node *pn;
2820 struct acpi_device *adev;
2821
2822 if (dev->bus != &acpi_bus_type)
2823 continue;
86080ccc 2824
cf04eee8
DW		 2825 adev= to_acpi_device(dev);
2826 mutex_lock(&adev->physical_node_lock);
2827 list_for_each_entry(pn, &adev->physical_node_list, node) {
2828 ret = dev_prepare_static_identity_mapping(pn->dev, hw);
2829 if (ret)
2830 break;
eae460b6 2831 }
cf04eee8
DW		 2832 mutex_unlock(&adev->physical_node_lock);
2833 if (ret)
2834 return ret;
62edf5dc 2835 }
2c2e2c38
FY		 2836
2837 return 0;
2838}
2839
ffebeb46
JL		 2840static void intel_iommu_init_qi(struct intel_iommu *iommu)
2841{
2842 /*
2843 * Start from the sane iommu hardware state.
2844 * If the queued invalidation is already initialized by us
2845 * (for example, while enabling interrupt-remapping) then
2846 * we got the things already rolling from a sane state.
2847 */
2848 if (!iommu->qi) {
2849 /*
2850 * Clear any previous faults.
2851 */
2852 dmar_fault(-1, iommu);
2853 /*
2854 * Disable queued invalidation if supported and already enabled
2855 * before OS handover.
2856 */
2857 dmar_disable_qi(iommu);
2858 }
2859
2860 if (dmar_enable_qi(iommu)) {
2861 /*
2862 * Queued Invalidate not enabled, use Register Based Invalidate
2863 */
2864 iommu->flush.flush_context = __iommu_flush_context;
2865 iommu->flush.flush_iotlb = __iommu_flush_iotlb;
9f10e5bf 2866 pr_info("%s: Using Register based invalidation\n",
ffebeb46
JL		 2867 iommu->name);
2868 } else {
2869 iommu->flush.flush_context = qi_flush_context;
2870 iommu->flush.flush_iotlb = qi_flush_iotlb;
9f10e5bf 2871 pr_info("%s: Using Queued invalidation\n", iommu->name);
ffebeb46
JL		 2872 }
2873}
2874
091d42e4
JR		 2875static int copy_context_table(struct intel_iommu *iommu,
2876 struct root_entry *old_re,
2877 struct context_entry **tbl,
2878 int bus, bool ext)
2879{
2880 struct context_entry *old_ce = NULL, *new_ce = NULL, ce;
dbcd861f 2881 int tbl_idx, pos = 0, idx, devfn, ret = 0, did;
091d42e4
JR		 2882 phys_addr_t old_ce_phys;
2883
2884 tbl_idx = ext ? bus * 2 : bus;
2885
2886 for (devfn = 0; devfn < 256; devfn++) {
2887 /* First calculate the correct index */
2888 idx = (ext ? devfn * 2 : devfn) % 256;
2889
2890 if (idx == 0) {
2891 /* First save what we may have and clean up */
2892 if (new_ce) {
2893 tbl[tbl_idx] = new_ce;
2894 __iommu_flush_cache(iommu, new_ce,
2895 VTD_PAGE_SIZE);
2896 pos = 1;
2897 }
2898
2899 if (old_ce)
2900 iounmap(old_ce);
2901
2902 ret = 0;
2903 if (devfn < 0x80)
2904 old_ce_phys = root_entry_lctp(old_re);
2905 else
2906 old_ce_phys = root_entry_uctp(old_re);
2907
2908 if (!old_ce_phys) {
2909 if (ext && devfn == 0) {
2910 /* No LCTP, try UCTP */
2911 devfn = 0x7f;
2912 continue;
2913 } else {
2914 goto out;
2915 }
2916 }
2917
2918 ret = -ENOMEM;
2919 old_ce = ioremap_cache(old_ce_phys, PAGE_SIZE);
2920 if (!old_ce)
2921 goto out;
2922
2923 new_ce = alloc_pgtable_page(iommu->node);
2924 if (!new_ce)
2925 goto out_unmap;
2926
2927 ret = 0;
2928 }
2929
2930 /* Now copy the context entry */
2931 ce = old_ce[idx];
2932
cf484d0e 2933 if (!__context_present(&ce))
091d42e4
JR		 2934 continue;
2935
dbcd861f
JR		 2936 did = context_domain_id(&ce);
2937 if (did >= 0 && did < cap_ndoms(iommu->cap))
2938 set_bit(did, iommu->domain_ids);
2939
cf484d0e
JR		 2940 /*
2941 * We need a marker for copied context entries. This
2942 * marker needs to work for the old format as well as
2943 * for extended context entries.
2944 *
2945 * Bit 67 of the context entry is used. In the old
2946 * format this bit is available to software, in the
2947 * extended format it is the PGE bit, but PGE is ignored
2948 * by HW if PASIDs are disabled (and thus still
2949 * available).
2950 *
2951 * So disable PASIDs first and then mark the entry
2952 * copied. This means that we don't copy PASID
2953 * translations from the old kernel, but this is fine as
2954 * faults there are not fatal.
2955 */
2956 context_clear_pasid_enable(&ce);
2957 context_set_copied(&ce);
2958
091d42e4
JR		 2959 new_ce[idx] = ce;
2960 }
2961
2962 tbl[tbl_idx + pos] = new_ce;
2963
2964 __iommu_flush_cache(iommu, new_ce, VTD_PAGE_SIZE);
2965
2966out_unmap:
2967 iounmap(old_ce);
2968
2969out:
2970 return ret;
2971}
2972
2973static int copy_translation_tables(struct intel_iommu *iommu)
2974{
2975 struct context_entry **ctxt_tbls;
2976 struct root_entry *old_rt;
2977 phys_addr_t old_rt_phys;
2978 int ctxt_table_entries;
2979 unsigned long flags;
2980 u64 rtaddr_reg;
2981 int bus, ret;
c3361f2f 2982 bool new_ext, ext;
091d42e4
JR		 2983
2984 rtaddr_reg = dmar_readq(iommu->reg + DMAR_RTADDR_REG);
2985 ext = !!(rtaddr_reg & DMA_RTADDR_RTT);
c3361f2f
JR		 2986 new_ext = !!ecap_ecs(iommu->ecap);
2987
2988 /*
2989 * The RTT bit can only be changed when translation is disabled,
2990 * but disabling translation means to open a window for data
2991 * corruption. So bail out and don't copy anything if we would
2992 * have to change the bit.
2993 */
2994 if (new_ext != ext)
2995 return -EINVAL;
091d42e4
JR		 2996
2997 old_rt_phys = rtaddr_reg & VTD_PAGE_MASK;
2998 if (!old_rt_phys)
2999 return -EINVAL;
3000
3001 old_rt = ioremap_cache(old_rt_phys, PAGE_SIZE);
3002 if (!old_rt)
3003 return -ENOMEM;
3004
3005 /* This is too big for the stack - allocate it from slab */
3006 ctxt_table_entries = ext ? 512 : 256;
3007 ret = -ENOMEM;
3008 ctxt_tbls = kzalloc(ctxt_table_entries * sizeof(void *), GFP_KERNEL);
3009 if (!ctxt_tbls)
3010 goto out_unmap;
3011
3012 for (bus = 0; bus < 256; bus++) {
3013 ret = copy_context_table(iommu, &old_rt[bus],
3014 ctxt_tbls, bus, ext);
3015 if (ret) {
3016 pr_err("%s: Failed to copy context table for bus %d\n",
3017 iommu->name, bus);
3018 continue;
3019 }
3020 }
3021
3022 spin_lock_irqsave(&iommu->lock, flags);
3023
3024 /* Context tables are copied, now write them to the root_entry table */
3025 for (bus = 0; bus < 256; bus++) {
3026 int idx = ext ? bus * 2 : bus;
3027 u64 val;
3028
3029 if (ctxt_tbls[idx]) {
3030 val = virt_to_phys(ctxt_tbls[idx]) | 1;
3031 iommu->root_entry[bus].lo = val;
3032 }
3033
3034 if (!ext || !ctxt_tbls[idx + 1])
3035 continue;
3036
3037 val = virt_to_phys(ctxt_tbls[idx + 1]) | 1;
3038 iommu->root_entry[bus].hi = val;
3039 }
3040
3041 spin_unlock_irqrestore(&iommu->lock, flags);
3042
3043 kfree(ctxt_tbls);
3044
3045 __iommu_flush_cache(iommu, iommu->root_entry, PAGE_SIZE);
3046
3047 ret = 0;
3048
3049out_unmap:
3050 iounmap(old_rt);
3051
3052 return ret;
3053}
3054
b779260b 3055static int __init init_dmars(void)
ba395927
KA		 3056{
3057 struct dmar_drhd_unit *drhd;
3058 struct dmar_rmrr_unit *rmrr;
a87f4918 3059 bool copied_tables = false;
832bd858 3060 struct device *dev;
ba395927 3061 struct intel_iommu *iommu;
9d783ba0 3062 int i, ret;
2c2e2c38 3063
ba395927
KA		 3064 /*
3065 * for each drhd
3066 * allocate root
3067 * initialize and program root entry to not present
3068 * endfor
3069 */
3070 for_each_drhd_unit(drhd) {
5e0d2a6f 3071 /*
3072 * lock not needed as this is only incremented in the single
3073 * threaded kernel __init code path all other access are read
3074 * only
3075 */
78d8e704 3076 if (g_num_of_iommus < DMAR_UNITS_SUPPORTED) {
1b198bb0
MT		 3077 g_num_of_iommus++;
3078 continue;
3079 }
9f10e5bf 3080 pr_err_once("Exceeded %d IOMMUs\n", DMAR_UNITS_SUPPORTED);
5e0d2a6f 3081 }
3082
ffebeb46
JL		 3083 /* Preallocate enough resources for IOMMU hot-addition */
3084 if (g_num_of_iommus < DMAR_UNITS_SUPPORTED)
3085 g_num_of_iommus = DMAR_UNITS_SUPPORTED;
3086
d9630fe9
WH		 3087 g_iommus = kcalloc(g_num_of_iommus, sizeof(struct intel_iommu *),
3088 GFP_KERNEL);
3089 if (!g_iommus) {
9f10e5bf 3090 pr_err("Allocating global iommu array failed\n");
d9630fe9
WH		 3091 ret = -ENOMEM;
3092 goto error;
3093 }
3094
80b20dd8 3095 deferred_flush = kzalloc(g_num_of_iommus *
3096 sizeof(struct deferred_flush_tables), GFP_KERNEL);
3097 if (!deferred_flush) {
5e0d2a6f 3098 ret = -ENOMEM;
989d51fc 3099 goto free_g_iommus;
5e0d2a6f 3100 }
3101
7c919779 3102 for_each_active_iommu(iommu, drhd) {
d9630fe9 3103 g_iommus[iommu->seq_id] = iommu;
ba395927 3104
b63d80d1
JR		 3105 intel_iommu_init_qi(iommu);
3106
e61d98d8
SS		 3107 ret = iommu_init_domains(iommu);
3108 if (ret)
989d51fc 3109 goto free_iommu;
e61d98d8 3110
4158c2ec
JR		 3111 init_translation_status(iommu);
3112
091d42e4
JR		 3113 if (translation_pre_enabled(iommu) && !is_kdump_kernel()) {
3114 iommu_disable_translation(iommu);
3115 clear_translation_pre_enabled(iommu);
3116 pr_warn("Translation was enabled for %s but we are not in kdump mode\n",
3117 iommu->name);
3118 }
4158c2ec 3119
ba395927
KA		 3120 /*
3121 * TBD:
3122 * we could share the same root & context tables
25985edc 3123 * among all IOMMU's. Need to Split it later.
ba395927
KA		 3124 */
3125 ret = iommu_alloc_root_entry(iommu);
ffebeb46 3126 if (ret)
989d51fc 3127 goto free_iommu;
5f0a7f76 3128
091d42e4
JR		 3129 if (translation_pre_enabled(iommu)) {
3130 pr_info("Translation already enabled - trying to copy translation structures\n");
3131
3132 ret = copy_translation_tables(iommu);
3133 if (ret) {
3134 /*
3135 * We found the IOMMU with translation
3136 * enabled - but failed to copy over the
3137 * old root-entry table. Try to proceed
3138 * by disabling translation now and
3139 * allocating a clean root-entry table.
3140 * This might cause DMAR faults, but
3141 * probably the dump will still succeed.
3142 */
3143 pr_err("Failed to copy translation tables from previous kernel for %s\n",
3144 iommu->name);
3145 iommu_disable_translation(iommu);
3146 clear_translation_pre_enabled(iommu);
3147 } else {
3148 pr_info("Copied translation tables from previous kernel for %s\n",
3149 iommu->name);
a87f4918 3150 copied_tables = true;
091d42e4
JR		 3151 }
3152 }
3153
5f0a7f76
JR		 3154 iommu_flush_write_buffer(iommu);
3155 iommu_set_root_entry(iommu);
3156 iommu->flush.flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL);
3157 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
3158
4ed0d3e6 3159 if (!ecap_pass_through(iommu->ecap))
19943b0e 3160 hw_pass_through = 0;
ba395927
KA		 3161 }
3162
19943b0e 3163 if (iommu_pass_through)
e0fc7e0b
DW		 3164 iommu_identity_mapping |= IDENTMAP_ALL;
3165
d3f13810 3166#ifdef CONFIG_INTEL_IOMMU_BROKEN_GFX_WA
e0fc7e0b 3167 iommu_identity_mapping |= IDENTMAP_GFX;
19943b0e 3168#endif
e0fc7e0b 3169
86080ccc
JR		 3170 if (iommu_identity_mapping) {
3171 ret = si_domain_init(hw_pass_through);
3172 if (ret)
3173 goto free_iommu;
3174 }
3175
e0fc7e0b
DW		 3176 check_tylersburg_isoch();
3177
a87f4918
JR		 3178 /*
3179 * If we copied translations from a previous kernel in the kdump
3180 * case, we can not assign the devices to domains now, as that
3181 * would eliminate the old mappings. So skip this part and defer
3182 * the assignment to device driver initialization time.
3183 */
3184 if (copied_tables)
3185 goto domains_done;
3186
ba395927 3187 /*
19943b0e
DW		 3188 * If pass through is not set or not enabled, setup context entries for
3189 * identity mappings for rmrr, gfx, and isa and may fall back to static
3190 * identity mapping if iommu_identity_mapping is set.
ba395927 3191 */
19943b0e
DW		 3192 if (iommu_identity_mapping) {
3193 ret = iommu_prepare_static_identity_mapping(hw_pass_through);
4ed0d3e6 3194 if (ret) {
9f10e5bf 3195 pr_crit("Failed to setup IOMMU pass-through\n");
989d51fc 3196 goto free_iommu;
ba395927
KA		 3197 }
3198 }
ba395927 3199 /*
19943b0e
DW		 3200 * For each rmrr
3201 * for each dev attached to rmrr
3202 * do
3203 * locate drhd for dev, alloc domain for dev
3204 * allocate free domain
3205 * allocate page table entries for rmrr
3206 * if context not allocated for bus
3207 * allocate and init context
3208 * set present in root table for this bus
3209 * init context with domain, translation etc
3210 * endfor
3211 * endfor
ba395927 3212 */
9f10e5bf 3213 pr_info("Setting RMRR:\n");
19943b0e 3214 for_each_rmrr_units(rmrr) {
b683b230
JL		 3215 /* some BIOS lists non-exist devices in DMAR table. */
3216 for_each_active_dev_scope(rmrr->devices, rmrr->devices_cnt,
832bd858 3217 i, dev) {
0b9d9753 3218 ret = iommu_prepare_rmrr_dev(rmrr, dev);
19943b0e 3219 if (ret)
9f10e5bf 3220 pr_err("Mapping reserved region failed\n");
ba395927 3221 }
4ed0d3e6 3222 }
49a0429e 3223
19943b0e
DW		 3224 iommu_prepare_isa();
3225
a87f4918
JR		 3226domains_done:
3227
ba395927
KA		 3228 /*
3229 * for each drhd
3230 * enable fault log
3231 * global invalidate context cache
3232 * global invalidate iotlb
3233 * enable translation
3234 */
7c919779 3235 for_each_iommu(iommu, drhd) {
51a63e67
JC		 3236 if (drhd->ignored) {
3237 /*
3238 * we always have to disable PMRs or DMA may fail on
3239 * this device
3240 */
3241 if (force_on)
7c919779 3242 iommu_disable_protect_mem_regions(iommu);
ba395927 3243 continue;
51a63e67 3244 }
ba395927
KA		 3245
3246 iommu_flush_write_buffer(iommu);
3247
3460a6d9
KA		 3248 ret = dmar_set_interrupt(iommu);
3249 if (ret)
989d51fc 3250 goto free_iommu;
3460a6d9 3251
8939ddf6
JR		 3252 if (!translation_pre_enabled(iommu))
3253 iommu_enable_translation(iommu);
3254
b94996c9 3255 iommu_disable_protect_mem_regions(iommu);
ba395927
KA		 3256 }
3257
3258 return 0;
989d51fc
JL		 3259
3260free_iommu:
ffebeb46
JL		 3261 for_each_active_iommu(iommu, drhd) {
3262 disable_dmar_iommu(iommu);
a868e6b7 3263 free_dmar_iommu(iommu);
ffebeb46 3264 }
9bdc531e 3265 kfree(deferred_flush);
989d51fc 3266free_g_iommus:
d9630fe9 3267 kfree(g_iommus);
989d51fc 3268error:
ba395927
KA		 3269 return ret;
3270}
3271
5a5e02a6 3272/* This takes a number of _MM_ pages, not VTD pages */
875764de
DW		 3273static struct iova *intel_alloc_iova(struct device *dev,
3274 struct dmar_domain *domain,
3275 unsigned long nrpages, uint64_t dma_mask)
ba395927 3276{
ba395927 3277 struct iova *iova = NULL;
ba395927 3278
875764de
DW		 3279 /* Restrict dma_mask to the width that the iommu can handle */
3280 dma_mask = min_t(uint64_t, DOMAIN_MAX_ADDR(domain->gaw), dma_mask);
3281
3282 if (!dmar_forcedac && dma_mask > DMA_BIT_MASK(32)) {
ba395927
KA		 3283 /*
3284 * First try to allocate an io virtual address in
284901a9 3285 * DMA_BIT_MASK(32) and if that fails then try allocating
3609801e 3286 * from higher range
ba395927 3287 */
875764de
DW		 3288 iova = alloc_iova(&domain->iovad, nrpages,
3289 IOVA_PFN(DMA_BIT_MASK(32)), 1);
3290 if (iova)
3291 return iova;
3292 }
3293 iova = alloc_iova(&domain->iovad, nrpages, IOVA_PFN(dma_mask), 1);
3294 if (unlikely(!iova)) {
9f10e5bf 3295 pr_err("Allocating %ld-page iova for %s failed",
207e3592 3296 nrpages, dev_name(dev));
f76aec76
KA		 3297 return NULL;
3298 }
3299
3300 return iova;
3301}
3302
d4b709f4 3303static struct dmar_domain *__get_valid_domain_for_dev(struct device *dev)
f76aec76
KA		 3304{
3305 struct dmar_domain *domain;
3306 int ret;
3307
d4b709f4 3308 domain = get_domain_for_dev(dev, DEFAULT_DOMAIN_ADDRESS_WIDTH);
f76aec76 3309 if (!domain) {
9f10e5bf 3310 pr_err("Allocating domain for %s failed\n",
d4b709f4 3311 dev_name(dev));
4fe05bbc 3312 return NULL;
ba395927
KA		 3313 }
3314
3315 /* make sure context mapping is ok */
d4b709f4 3316 if (unlikely(!domain_context_mapped(dev))) {
28ccce0d 3317 ret = domain_context_mapping(domain, dev);
f76aec76 3318 if (ret) {
9f10e5bf 3319 pr_err("Domain context map for %s failed\n",
d4b709f4 3320 dev_name(dev));
4fe05bbc 3321 return NULL;
f76aec76 3322 }
ba395927
KA		 3323 }
3324
f76aec76
KA		 3325 return domain;
3326}
3327
d4b709f4 3328static inline struct dmar_domain *get_valid_domain_for_dev(struct device *dev)
147202aa
DW		 3329{
3330 struct device_domain_info *info;
3331
3332 /* No lock here, assumes no domain exit in normal case */
d4b709f4 3333 info = dev->archdata.iommu;
147202aa
DW		 3334 if (likely(info))
3335 return info->domain;
3336
3337 return __get_valid_domain_for_dev(dev);
3338}
3339
ecb509ec 3340/* Check if the dev needs to go through non-identity map and unmap process.*/
73676832 3341static int iommu_no_mapping(struct device *dev)
2c2e2c38
FY		 3342{
3343 int found;
3344
3d89194a 3345 if (iommu_dummy(dev))
1e4c64c4
DW		 3346 return 1;
3347
2c2e2c38 3348 if (!iommu_identity_mapping)
1e4c64c4 3349 return 0;
2c2e2c38 3350
9b226624 3351 found = identity_mapping(dev);
2c2e2c38 3352 if (found) {
ecb509ec 3353 if (iommu_should_identity_map(dev, 0))
2c2e2c38
FY		 3354 return 1;
3355 else {
3356 /*
3357 * 32 bit DMA is removed from si_domain and fall back
3358 * to non-identity mapping.
3359 */
bf9c9eda 3360 domain_remove_one_dev_info(si_domain, dev);
9f10e5bf
JR		 3361 pr_info("32bit %s uses non-identity mapping\n",
3362 dev_name(dev));
2c2e2c38
FY		 3363 return 0;
3364 }
3365 } else {
3366 /*
3367 * In case of a detached 64 bit DMA device from vm, the device
3368 * is put into si_domain for identity mapping.
3369 */
ecb509ec 3370 if (iommu_should_identity_map(dev, 0)) {
2c2e2c38 3371 int ret;
28ccce0d 3372 ret = domain_add_dev_info(si_domain, dev);
2c2e2c38 3373 if (!ret) {
9f10e5bf
JR		 3374 pr_info("64bit %s uses identity mapping\n",
3375 dev_name(dev));
2c2e2c38
FY		 3376 return 1;
3377 }
3378 }
3379 }
3380
1e4c64c4 3381 return 0;
2c2e2c38
FY		 3382}
3383
5040a918 3384static dma_addr_t __intel_map_single(struct device *dev, phys_addr_t paddr,
bb9e6d65 3385 size_t size, int dir, u64 dma_mask)
f76aec76 3386{
f76aec76 3387 struct dmar_domain *domain;
5b6985ce 3388 phys_addr_t start_paddr;
f76aec76
KA		 3389 struct iova *iova;
3390 int prot = 0;
6865f0d1 3391 int ret;
8c11e798 3392 struct intel_iommu *iommu;
33041ec0 3393 unsigned long paddr_pfn = paddr >> PAGE_SHIFT;
f76aec76
KA		 3394
3395 BUG_ON(dir == DMA_NONE);
2c2e2c38 3396
5040a918 3397 if (iommu_no_mapping(dev))
6865f0d1 3398 return paddr;
f76aec76 3399
5040a918 3400 domain = get_valid_domain_for_dev(dev);
f76aec76
KA		 3401 if (!domain)
3402 return 0;
3403
8c11e798 3404 iommu = domain_get_iommu(domain);
88cb6a74 3405 size = aligned_nrpages(paddr, size);
f76aec76 3406
5040a918 3407 iova = intel_alloc_iova(dev, domain, dma_to_mm_pfn(size), dma_mask);
f76aec76
KA		 3408 if (!iova)
3409 goto error;
3410
ba395927
KA		 3411 /*
3412 * Check if DMAR supports zero-length reads on write only
3413 * mappings..
3414 */
3415 if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
8c11e798 3416 !cap_zlr(iommu->cap))
ba395927
KA		 3417 prot |= DMA_PTE_READ;
3418 if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
3419 prot |= DMA_PTE_WRITE;
3420 /*
6865f0d1 3421 * paddr - (paddr + size) might be partial page, we should map the whole
ba395927 3422 * page. Note: if two part of one page are separately mapped, we
6865f0d1 3423 * might have two guest_addr mapping to the same host paddr, but this
ba395927
KA		 3424 * is not a big problem
3425 */
0ab36de2 3426 ret = domain_pfn_mapping(domain, mm_to_dma_pfn(iova->pfn_lo),
33041ec0 3427 mm_to_dma_pfn(paddr_pfn), size, prot);
ba395927
KA		 3428 if (ret)
3429 goto error;
3430
1f0ef2aa
DW		 3431 /* it's a non-present to present mapping. Only flush if caching mode */
3432 if (cap_caching_mode(iommu->cap))
ea8ea460 3433 iommu_flush_iotlb_psi(iommu, domain->id, mm_to_dma_pfn(iova->pfn_lo), size, 0, 1);
1f0ef2aa 3434 else
8c11e798 3435 iommu_flush_write_buffer(iommu);
f76aec76 3436
03d6a246
DW		 3437 start_paddr = (phys_addr_t)iova->pfn_lo << PAGE_SHIFT;
3438 start_paddr += paddr & ~PAGE_MASK;
3439 return start_paddr;
ba395927 3440
ba395927 3441error:
f76aec76
KA		 3442 if (iova)
3443 __free_iova(&domain->iovad, iova);
9f10e5bf 3444 pr_err("Device %s request: %zx@%llx dir %d --- failed\n",
5040a918 3445 dev_name(dev), size, (unsigned long long)paddr, dir);
ba395927
KA		 3446 return 0;
3447}
3448
ffbbef5c
FT		 3449static dma_addr_t intel_map_page(struct device *dev, struct page *page,
3450 unsigned long offset, size_t size,
3451 enum dma_data_direction dir,
3452 struct dma_attrs *attrs)
bb9e6d65 3453{
ffbbef5c 3454 return __intel_map_single(dev, page_to_phys(page) + offset, size,
46333e37 3455 dir, *dev->dma_mask);
bb9e6d65
FT		 3456}
3457
5e0d2a6f 3458static void flush_unmaps(void)
3459{
80b20dd8 3460 int i, j;
5e0d2a6f 3461
5e0d2a6f 3462 timer_on = 0;
3463
3464 /* just flush them all */
3465 for (i = 0; i < g_num_of_iommus; i++) {
a2bb8459
WH		 3466 struct intel_iommu *iommu = g_iommus[i];
3467 if (!iommu)
3468 continue;
c42d9f32 3469
9dd2fe89
YZ		 3470 if (!deferred_flush[i].next)
3471 continue;
3472
78d5f0f5
NA		 3473 /* In caching mode, global flushes turn emulation expensive */
3474 if (!cap_caching_mode(iommu->cap))
3475 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
93a23a72 3476 DMA_TLB_GLOBAL_FLUSH);
9dd2fe89 3477 for (j = 0; j < deferred_flush[i].next; j++) {
93a23a72
YZ		 3478 unsigned long mask;
3479 struct iova *iova = deferred_flush[i].iova[j];
78d5f0f5
NA		 3480 struct dmar_domain *domain = deferred_flush[i].domain[j];
3481
3482 /* On real hardware multiple invalidations are expensive */
3483 if (cap_caching_mode(iommu->cap))
3484 iommu_flush_iotlb_psi(iommu, domain->id,
a156ef99 3485 iova->pfn_lo, iova_size(iova),
ea8ea460 3486 !deferred_flush[i].freelist[j], 0);
78d5f0f5 3487 else {
a156ef99 3488 mask = ilog2(mm_to_dma_pfn(iova_size(iova)));
78d5f0f5
NA		 3489 iommu_flush_dev_iotlb(deferred_flush[i].domain[j],
3490 (uint64_t)iova->pfn_lo << PAGE_SHIFT, mask);
3491 }
93a23a72 3492 __free_iova(&deferred_flush[i].domain[j]->iovad, iova);
ea8ea460
DW		 3493 if (deferred_flush[i].freelist[j])
3494 dma_free_pagelist(deferred_flush[i].freelist[j]);
80b20dd8 3495 }
9dd2fe89 3496 deferred_flush[i].next = 0;
5e0d2a6f 3497 }
3498
5e0d2a6f 3499 list_size = 0;
5e0d2a6f 3500}
3501
3502static void flush_unmaps_timeout(unsigned long data)
3503{
80b20dd8 3504 unsigned long flags;
3505
3506 spin_lock_irqsave(&async_umap_flush_lock, flags);
5e0d2a6f 3507 flush_unmaps();
80b20dd8 3508 spin_unlock_irqrestore(&async_umap_flush_lock, flags);
5e0d2a6f 3509}
3510
ea8ea460 3511static void add_unmap(struct dmar_domain *dom, struct iova *iova, struct page *freelist)
5e0d2a6f 3512{
3513 unsigned long flags;
80b20dd8 3514 int next, iommu_id;
8c11e798 3515 struct intel_iommu *iommu;
5e0d2a6f 3516
3517 spin_lock_irqsave(&async_umap_flush_lock, flags);
80b20dd8 3518 if (list_size == HIGH_WATER_MARK)
3519 flush_unmaps();
3520
8c11e798
WH		 3521 iommu = domain_get_iommu(dom);
3522 iommu_id = iommu->seq_id;
c42d9f32 3523
80b20dd8 3524 next = deferred_flush[iommu_id].next;
3525 deferred_flush[iommu_id].domain[next] = dom;
3526 deferred_flush[iommu_id].iova[next] = iova;
ea8ea460 3527 deferred_flush[iommu_id].freelist[next] = freelist;
80b20dd8 3528 deferred_flush[iommu_id].next++;
5e0d2a6f 3529
3530 if (!timer_on) {
3531 mod_timer(&unmap_timer, jiffies + msecs_to_jiffies(10));
3532 timer_on = 1;
3533 }
3534 list_size++;
3535 spin_unlock_irqrestore(&async_umap_flush_lock, flags);
3536}
3537
d41a4adb 3538static void intel_unmap(struct device *dev, dma_addr_t dev_addr)
ba395927 3539{
f76aec76 3540 struct dmar_domain *domain;
d794dc9b 3541 unsigned long start_pfn, last_pfn;
ba395927 3542 struct iova *iova;
8c11e798 3543 struct intel_iommu *iommu;
ea8ea460 3544 struct page *freelist;
ba395927 3545
73676832 3546 if (iommu_no_mapping(dev))
f76aec76 3547 return;
2c2e2c38 3548
1525a29a 3549 domain = find_domain(dev);
ba395927
KA		 3550 BUG_ON(!domain);
3551
8c11e798
WH		 3552 iommu = domain_get_iommu(domain);
3553
ba395927 3554 iova = find_iova(&domain->iovad, IOVA_PFN(dev_addr));
85b98276
DW		 3555 if (WARN_ONCE(!iova, "Driver unmaps unmatched page at PFN %llx\n",
3556 (unsigned long long)dev_addr))
ba395927 3557 return;
ba395927 3558
d794dc9b
DW		 3559 start_pfn = mm_to_dma_pfn(iova->pfn_lo);
3560 last_pfn = mm_to_dma_pfn(iova->pfn_hi + 1) - 1;
ba395927 3561
d794dc9b 3562 pr_debug("Device %s unmapping: pfn %lx-%lx\n",
207e3592 3563 dev_name(dev), start_pfn, last_pfn);
ba395927 3564
ea8ea460 3565 freelist = domain_unmap(domain, start_pfn, last_pfn);
d794dc9b 3566
5e0d2a6f 3567 if (intel_iommu_strict) {
03d6a246 3568 iommu_flush_iotlb_psi(iommu, domain->id, start_pfn,
ea8ea460 3569 last_pfn - start_pfn + 1, !freelist, 0);
5e0d2a6f 3570 /* free iova */
3571 __free_iova(&domain->iovad, iova);
ea8ea460 3572 dma_free_pagelist(freelist);
5e0d2a6f 3573 } else {
ea8ea460 3574 add_unmap(domain, iova, freelist);
5e0d2a6f 3575 /*
3576 * queue up the release of the unmap to save the 1/6th of the
3577 * cpu used up by the iotlb flush operation...
3578 */
5e0d2a6f 3579 }
ba395927
KA		 3580}
3581
d41a4adb
JL		 3582static void intel_unmap_page(struct device *dev, dma_addr_t dev_addr,
3583 size_t size, enum dma_data_direction dir,
3584 struct dma_attrs *attrs)
3585{
3586 intel_unmap(dev, dev_addr);
3587}
3588
5040a918 3589static void *intel_alloc_coherent(struct device *dev, size_t size,
baa676fc
AP		 3590 dma_addr_t *dma_handle, gfp_t flags,
3591 struct dma_attrs *attrs)
ba395927 3592{
36746436 3593 struct page *page = NULL;
ba395927
KA		 3594 int order;
3595
5b6985ce 3596 size = PAGE_ALIGN(size);
ba395927 3597 order = get_order(size);
e8bb910d 3598
5040a918 3599 if (!iommu_no_mapping(dev))
e8bb910d 3600 flags &= ~(GFP_DMA | GFP_DMA32);
5040a918
DW		 3601 else if (dev->coherent_dma_mask < dma_get_required_mask(dev)) {
3602 if (dev->coherent_dma_mask < DMA_BIT_MASK(32))
e8bb910d
AW		 3603 flags |= GFP_DMA;
3604 else
3605 flags |= GFP_DMA32;
3606 }
ba395927 3607
36746436
AM		 3608 if (flags & __GFP_WAIT) {
3609 unsigned int count = size >> PAGE_SHIFT;
3610
3611 page = dma_alloc_from_contiguous(dev, count, order);
3612 if (page && iommu_no_mapping(dev) &&
3613 page_to_phys(page) + size > dev->coherent_dma_mask) {
3614 dma_release_from_contiguous(dev, page, count);
3615 page = NULL;
3616 }
3617 }
3618
3619 if (!page)
3620 page = alloc_pages(flags, order);
3621 if (!page)
ba395927 3622 return NULL;
36746436 3623 memset(page_address(page), 0, size);
ba395927 3624
36746436 3625 *dma_handle = __intel_map_single(dev, page_to_phys(page), size,
bb9e6d65 3626 DMA_BIDIRECTIONAL,
5040a918 3627 dev->coherent_dma_mask);
ba395927 3628 if (*dma_handle)
36746436
AM		 3629 return page_address(page);
3630 if (!dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT))
3631 __free_pages(page, order);
3632
ba395927
KA		 3633 return NULL;
3634}
3635
5040a918 3636static void intel_free_coherent(struct device *dev, size_t size, void *vaddr,
baa676fc 3637 dma_addr_t dma_handle, struct dma_attrs *attrs)
ba395927
KA		 3638{
3639 int order;
36746436 3640 struct page *page = virt_to_page(vaddr);
ba395927 3641
5b6985ce 3642 size = PAGE_ALIGN(size);
ba395927
KA		 3643 order = get_order(size);
3644
d41a4adb 3645 intel_unmap(dev, dma_handle);
36746436
AM		 3646 if (!dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT))
3647 __free_pages(page, order);
ba395927
KA		 3648}
3649
5040a918 3650static void intel_unmap_sg(struct device *dev, struct scatterlist *sglist,
d7ab5c46
FT		 3651 int nelems, enum dma_data_direction dir,
3652 struct dma_attrs *attrs)
ba395927 3653{
d41a4adb 3654 intel_unmap(dev, sglist[0].dma_address);
ba395927
KA		 3655}
3656
ba395927 3657static int intel_nontranslate_map_sg(struct device *hddev,
c03ab37c 3658 struct scatterlist *sglist, int nelems, int dir)
ba395927
KA		 3659{
3660 int i;
c03ab37c 3661 struct scatterlist *sg;
ba395927 3662
c03ab37c 3663 for_each_sg(sglist, sg, nelems, i) {
12d4d40e 3664 BUG_ON(!sg_page(sg));
4cf2e75d 3665 sg->dma_address = page_to_phys(sg_page(sg)) + sg->offset;
c03ab37c 3666 sg->dma_length = sg->length;
ba395927
KA		 3667 }
3668 return nelems;
3669}
3670
5040a918 3671static int intel_map_sg(struct device *dev, struct scatterlist *sglist, int nelems,
d7ab5c46 3672 enum dma_data_direction dir, struct dma_attrs *attrs)
ba395927 3673{
ba395927 3674 int i;
ba395927 3675 struct dmar_domain *domain;
f76aec76
KA		 3676 size_t size = 0;
3677 int prot = 0;
f76aec76
KA		 3678 struct iova *iova = NULL;
3679 int ret;
c03ab37c 3680 struct scatterlist *sg;
b536d24d 3681 unsigned long start_vpfn;
8c11e798 3682 struct intel_iommu *iommu;
ba395927
KA		 3683
3684 BUG_ON(dir == DMA_NONE);
5040a918
DW		 3685 if (iommu_no_mapping(dev))
3686 return intel_nontranslate_map_sg(dev, sglist, nelems, dir);
ba395927 3687
5040a918 3688 domain = get_valid_domain_for_dev(dev);
f76aec76
KA		 3689 if (!domain)
3690 return 0;
3691
8c11e798
WH		 3692 iommu = domain_get_iommu(domain);
3693
b536d24d 3694 for_each_sg(sglist, sg, nelems, i)
88cb6a74 3695 size += aligned_nrpages(sg->offset, sg->length);
f76aec76 3696
5040a918
DW		 3697 iova = intel_alloc_iova(dev, domain, dma_to_mm_pfn(size),
3698 *dev->dma_mask);
f76aec76 3699 if (!iova) {
c03ab37c 3700 sglist->dma_length = 0;
f76aec76
KA		 3701 return 0;
3702 }
3703
3704 /*
3705 * Check if DMAR supports zero-length reads on write only
3706 * mappings..
3707 */
3708 if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
8c11e798 3709 !cap_zlr(iommu->cap))
f76aec76
KA		 3710 prot |= DMA_PTE_READ;
3711 if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
3712 prot |= DMA_PTE_WRITE;
3713
b536d24d 3714 start_vpfn = mm_to_dma_pfn(iova->pfn_lo);
e1605495 3715
f532959b 3716 ret = domain_sg_mapping(domain, start_vpfn, sglist, size, prot);
e1605495 3717 if (unlikely(ret)) {
e1605495
DW		 3718 dma_pte_free_pagetable(domain, start_vpfn,
3719 start_vpfn + size - 1);
e1605495
DW		 3720 __free_iova(&domain->iovad, iova);
3721 return 0;
ba395927
KA		 3722 }
3723
1f0ef2aa
DW		 3724 /* it's a non-present to present mapping. Only flush if caching mode */
3725 if (cap_caching_mode(iommu->cap))
ea8ea460 3726 iommu_flush_iotlb_psi(iommu, domain->id, start_vpfn, size, 0, 1);
1f0ef2aa 3727 else
8c11e798 3728 iommu_flush_write_buffer(iommu);
1f0ef2aa 3729
ba395927
KA		 3730 return nelems;
3731}
3732
dfb805e8
FT		 3733static int intel_mapping_error(struct device *dev, dma_addr_t dma_addr)
3734{
3735 return !dma_addr;
3736}
3737
160c1d8e 3738struct dma_map_ops intel_dma_ops = {
baa676fc
AP		 3739 .alloc = intel_alloc_coherent,
3740 .free = intel_free_coherent,
ba395927
KA		 3741 .map_sg = intel_map_sg,
3742 .unmap_sg = intel_unmap_sg,
ffbbef5c
FT		 3743 .map_page = intel_map_page,
3744 .unmap_page = intel_unmap_page,
dfb805e8 3745 .mapping_error = intel_mapping_error,
ba395927
KA		 3746};
3747
3748static inline int iommu_domain_cache_init(void)
3749{
3750 int ret = 0;
3751
3752 iommu_domain_cache = kmem_cache_create("iommu_domain",
3753 sizeof(struct dmar_domain),
3754 0,
3755 SLAB_HWCACHE_ALIGN,
3756
3757 NULL);
3758 if (!iommu_domain_cache) {
9f10e5bf 3759 pr_err("Couldn't create iommu_domain cache\n");
ba395927
KA		 3760 ret = -ENOMEM;
3761 }
3762
3763 return ret;
3764}
3765
3766static inline int iommu_devinfo_cache_init(void)
3767{
3768 int ret = 0;
3769
3770 iommu_devinfo_cache = kmem_cache_create("iommu_devinfo",
3771 sizeof(struct device_domain_info),
3772 0,
3773 SLAB_HWCACHE_ALIGN,
ba395927
KA		 3774 NULL);
3775 if (!iommu_devinfo_cache) {
9f10e5bf 3776 pr_err("Couldn't create devinfo cache\n");
ba395927
KA		 3777 ret = -ENOMEM;
3778 }
3779
3780 return ret;
3781}
3782
ba395927
KA		 3783static int __init iommu_init_mempool(void)
3784{
3785 int ret;
3786 ret = iommu_iova_cache_init();
3787 if (ret)
3788 return ret;
3789
3790 ret = iommu_domain_cache_init();
3791 if (ret)
3792 goto domain_error;
3793
3794 ret = iommu_devinfo_cache_init();
3795 if (!ret)
3796 return ret;
3797
3798 kmem_cache_destroy(iommu_domain_cache);
3799domain_error:
85b45456 3800 iommu_iova_cache_destroy();
ba395927
KA		 3801
3802 return -ENOMEM;
3803}
3804
3805static void __init iommu_exit_mempool(void)
3806{
3807 kmem_cache_destroy(iommu_devinfo_cache);
3808 kmem_cache_destroy(iommu_domain_cache);
85b45456 3809 iommu_iova_cache_destroy();
ba395927
KA		 3810}
3811
556ab45f
DW		 3812static void quirk_ioat_snb_local_iommu(struct pci_dev *pdev)
3813{
3814 struct dmar_drhd_unit *drhd;
3815 u32 vtbar;
3816 int rc;
3817
3818 /* We know that this device on this chipset has its own IOMMU.
3819 * If we find it under a different IOMMU, then the BIOS is lying
3820 * to us. Hope that the IOMMU for this device is actually
3821 * disabled, and it needs no translation...
3822 */
3823 rc = pci_bus_read_config_dword(pdev->bus, PCI_DEVFN(0, 0), 0xb0, &vtbar);
3824 if (rc) {
3825 /* "can't" happen */
3826 dev_info(&pdev->dev, "failed to run vt-d quirk\n");
3827 return;
3828 }
3829 vtbar &= 0xffff0000;
3830
3831 /* we know that the this iommu should be at offset 0xa000 from vtbar */
3832 drhd = dmar_find_matched_drhd_unit(pdev);
3833 if (WARN_TAINT_ONCE(!drhd || drhd->reg_base_addr - vtbar != 0xa000,
3834 TAINT_FIRMWARE_WORKAROUND,
3835 "BIOS assigned incorrect VT-d unit for Intel(R) QuickData Technology device\n"))
3836 pdev->dev.archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO;
3837}
3838DECLARE_PCI_FIXUP_ENABLE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB, quirk_ioat_snb_local_iommu);
3839
ba395927
KA		 3840static void __init init_no_remapping_devices(void)
3841{
3842 struct dmar_drhd_unit *drhd;
832bd858 3843 struct device *dev;
b683b230 3844 int i;
ba395927
KA		 3845
3846 for_each_drhd_unit(drhd) {
3847 if (!drhd->include_all) {
b683b230
JL		 3848 for_each_active_dev_scope(drhd->devices,
3849 drhd->devices_cnt, i, dev)
3850 break;
832bd858 3851 /* ignore DMAR unit if no devices exist */
ba395927
KA		 3852 if (i == drhd->devices_cnt)
3853 drhd->ignored = 1;
3854 }
3855 }
3856
7c919779 3857 for_each_active_drhd_unit(drhd) {
7c919779 3858 if (drhd->include_all)
ba395927
KA		 3859 continue;
3860
b683b230
JL		 3861 for_each_active_dev_scope(drhd->devices,
3862 drhd->devices_cnt, i, dev)
832bd858 3863 if (!dev_is_pci(dev) || !IS_GFX_DEVICE(to_pci_dev(dev)))
ba395927 3864 break;
ba395927
KA		 3865 if (i < drhd->devices_cnt)
3866 continue;
3867
c0771df8
DW		 3868 /* This IOMMU has *only* gfx devices. Either bypass it or
3869 set the gfx_mapped flag, as appropriate */
3870 if (dmar_map_gfx) {
3871 intel_iommu_gfx_mapped = 1;
3872 } else {
3873 drhd->ignored = 1;
b683b230
JL		 3874 for_each_active_dev_scope(drhd->devices,
3875 drhd->devices_cnt, i, dev)
832bd858 3876 dev->archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO;
ba395927
KA		 3877 }
3878 }
3879}
3880
f59c7b69
FY		 3881#ifdef CONFIG_SUSPEND
3882static int init_iommu_hw(void)
3883{
3884 struct dmar_drhd_unit *drhd;
3885 struct intel_iommu *iommu = NULL;
3886
3887 for_each_active_iommu(iommu, drhd)
3888 if (iommu->qi)
3889 dmar_reenable_qi(iommu);
3890
b779260b
JC		 3891 for_each_iommu(iommu, drhd) {
3892 if (drhd->ignored) {
3893 /*
3894 * we always have to disable PMRs or DMA may fail on
3895 * this device
3896 */
3897 if (force_on)
3898 iommu_disable_protect_mem_regions(iommu);
3899 continue;
3900 }
3901
f59c7b69
FY		 3902 iommu_flush_write_buffer(iommu);
3903
3904 iommu_set_root_entry(iommu);
3905
3906 iommu->flush.flush_context(iommu, 0, 0, 0,
1f0ef2aa 3907 DMA_CCMD_GLOBAL_INVL);
2a41ccee
JL		 3908 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
3909 iommu_enable_translation(iommu);
b94996c9 3910 iommu_disable_protect_mem_regions(iommu);
f59c7b69
FY		 3911 }
3912
3913 return 0;
3914}
3915
3916static void iommu_flush_all(void)
3917{
3918 struct dmar_drhd_unit *drhd;
3919 struct intel_iommu *iommu;
3920
3921 for_each_active_iommu(iommu, drhd) {
3922 iommu->flush.flush_context(iommu, 0, 0, 0,
1f0ef2aa 3923 DMA_CCMD_GLOBAL_INVL);
f59c7b69 3924 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
1f0ef2aa 3925 DMA_TLB_GLOBAL_FLUSH);
f59c7b69
FY		 3926 }
3927}
3928
134fac3f 3929static int iommu_suspend(void)
f59c7b69
FY		 3930{
3931 struct dmar_drhd_unit *drhd;
3932 struct intel_iommu *iommu = NULL;
3933 unsigned long flag;
3934
3935 for_each_active_iommu(iommu, drhd) {
3936 iommu->iommu_state = kzalloc(sizeof(u32) * MAX_SR_DMAR_REGS,
3937 GFP_ATOMIC);
3938 if (!iommu->iommu_state)
3939 goto nomem;
3940 }
3941
3942 iommu_flush_all();
3943
3944 for_each_active_iommu(iommu, drhd) {
3945 iommu_disable_translation(iommu);
3946
1f5b3c3f 3947 raw_spin_lock_irqsave(&iommu->register_lock, flag);
f59c7b69
FY		 3948
3949 iommu->iommu_state[SR_DMAR_FECTL_REG] =
3950 readl(iommu->reg + DMAR_FECTL_REG);
3951 iommu->iommu_state[SR_DMAR_FEDATA_REG] =
3952 readl(iommu->reg + DMAR_FEDATA_REG);
3953 iommu->iommu_state[SR_DMAR_FEADDR_REG] =
3954 readl(iommu->reg + DMAR_FEADDR_REG);
3955 iommu->iommu_state[SR_DMAR_FEUADDR_REG] =
3956 readl(iommu->reg + DMAR_FEUADDR_REG);
3957
1f5b3c3f 3958 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
f59c7b69
FY		 3959 }
3960 return 0;
3961
3962nomem:
3963 for_each_active_iommu(iommu, drhd)
3964 kfree(iommu->iommu_state);
3965
3966 return -ENOMEM;
3967}
3968
134fac3f 3969static void iommu_resume(void)
f59c7b69
FY		 3970{
3971 struct dmar_drhd_unit *drhd;
3972 struct intel_iommu *iommu = NULL;
3973 unsigned long flag;
3974
3975 if (init_iommu_hw()) {
b779260b
JC		 3976 if (force_on)
3977 panic("tboot: IOMMU setup failed, DMAR can not resume!\n");
3978 else
3979 WARN(1, "IOMMU setup failed, DMAR can not resume!\n");
134fac3f 3980 return;
f59c7b69
FY		 3981 }
3982
3983 for_each_active_iommu(iommu, drhd) {
3984
1f5b3c3f 3985 raw_spin_lock_irqsave(&iommu->register_lock, flag);
f59c7b69
FY		 3986
3987 writel(iommu->iommu_state[SR_DMAR_FECTL_REG],
3988 iommu->reg + DMAR_FECTL_REG);
3989 writel(iommu->iommu_state[SR_DMAR_FEDATA_REG],
3990 iommu->reg + DMAR_FEDATA_REG);
3991 writel(iommu->iommu_state[SR_DMAR_FEADDR_REG],
3992 iommu->reg + DMAR_FEADDR_REG);
3993 writel(iommu->iommu_state[SR_DMAR_FEUADDR_REG],
3994 iommu->reg + DMAR_FEUADDR_REG);
3995
1f5b3c3f 3996 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
f59c7b69
FY		 3997 }
3998
3999 for_each_active_iommu(iommu, drhd)
4000 kfree(iommu->iommu_state);
f59c7b69
FY		 4001}
4002
134fac3f 4003static struct syscore_ops iommu_syscore_ops = {
f59c7b69
FY		 4004 .resume = iommu_resume,
4005 .suspend = iommu_suspend,
4006};
4007
134fac3f 4008static void __init init_iommu_pm_ops(void)
f59c7b69 4009{
134fac3f 4010 register_syscore_ops(&iommu_syscore_ops);
f59c7b69
FY		 4011}
4012
4013#else
99592ba4 4014static inline void init_iommu_pm_ops(void) {}
f59c7b69
FY		 4015#endif /* CONFIG_PM */
4016
318fe7df 4017
c2a0b538 4018int __init dmar_parse_one_rmrr(struct acpi_dmar_header *header, void *arg)
318fe7df
SS		 4019{
4020 struct acpi_dmar_reserved_memory *rmrr;
4021 struct dmar_rmrr_unit *rmrru;
4022
4023 rmrru = kzalloc(sizeof(*rmrru), GFP_KERNEL);
4024 if (!rmrru)
4025 return -ENOMEM;
4026
4027 rmrru->hdr = header;
4028 rmrr = (struct acpi_dmar_reserved_memory *)header;
4029 rmrru->base_address = rmrr->base_address;
4030 rmrru->end_address = rmrr->end_address;
2e455289
JL		 4031 rmrru->devices = dmar_alloc_dev_scope((void *)(rmrr + 1),
4032 ((void *)rmrr) + rmrr->header.length,
4033 &rmrru->devices_cnt);
4034 if (rmrru->devices_cnt && rmrru->devices == NULL) {
4035 kfree(rmrru);
4036 return -ENOMEM;
4037 }
318fe7df 4038
2e455289 4039 list_add(&rmrru->list, &dmar_rmrr_units);
318fe7df 4040
2e455289 4041 return 0;
318fe7df
SS		 4042}
4043
6b197249
JL		 4044static struct dmar_atsr_unit *dmar_find_atsr(struct acpi_dmar_atsr *atsr)
4045{
4046 struct dmar_atsr_unit *atsru;
4047 struct acpi_dmar_atsr *tmp;
4048
4049 list_for_each_entry_rcu(atsru, &dmar_atsr_units, list) {
4050 tmp = (struct acpi_dmar_atsr *)atsru->hdr;
4051 if (atsr->segment != tmp->segment)
4052 continue;
4053 if (atsr->header.length != tmp->header.length)
4054 continue;
4055 if (memcmp(atsr, tmp, atsr->header.length) == 0)
4056 return atsru;
4057 }
4058
4059 return NULL;
4060}
4061
4062int dmar_parse_one_atsr(struct acpi_dmar_header *hdr, void *arg)
318fe7df
SS		 4063{
4064 struct acpi_dmar_atsr *atsr;
4065 struct dmar_atsr_unit *atsru;
4066
6b197249
JL		 4067 if (system_state != SYSTEM_BOOTING && !intel_iommu_enabled)
4068 return 0;
4069
318fe7df 4070 atsr = container_of(hdr, struct acpi_dmar_atsr, header);
6b197249
JL		 4071 atsru = dmar_find_atsr(atsr);
4072 if (atsru)
4073 return 0;
4074
4075 atsru = kzalloc(sizeof(*atsru) + hdr->length, GFP_KERNEL);
318fe7df
SS		 4076 if (!atsru)
4077 return -ENOMEM;
4078
6b197249
JL		 4079 /*
4080 * If memory is allocated from slab by ACPI _DSM method, we need to
4081 * copy the memory content because the memory buffer will be freed
4082 * on return.
4083 */
4084 atsru->hdr = (void *)(atsru + 1);
4085 memcpy(atsru->hdr, hdr, hdr->length);
318fe7df 4086 atsru->include_all = atsr->flags & 0x1;
2e455289
JL		 4087 if (!atsru->include_all) {
4088 atsru->devices = dmar_alloc_dev_scope((void *)(atsr + 1),
4089 (void *)atsr + atsr->header.length,
4090 &atsru->devices_cnt);
4091 if (atsru->devices_cnt && atsru->devices == NULL) {
4092 kfree(atsru);
4093 return -ENOMEM;
4094 }
4095 }
318fe7df 4096
0e242612 4097 list_add_rcu(&atsru->list, &dmar_atsr_units);
318fe7df
SS		 4098
4099 return 0;
4100}
4101
9bdc531e
JL		 4102static void intel_iommu_free_atsr(struct dmar_atsr_unit *atsru)
4103{
4104 dmar_free_dev_scope(&atsru->devices, &atsru->devices_cnt);
4105 kfree(atsru);
4106}
4107
6b197249
JL		 4108int dmar_release_one_atsr(struct acpi_dmar_header *hdr, void *arg)
4109{
4110 struct acpi_dmar_atsr *atsr;
4111 struct dmar_atsr_unit *atsru;
4112
4113 atsr = container_of(hdr, struct acpi_dmar_atsr, header);
4114 atsru = dmar_find_atsr(atsr);
4115 if (atsru) {
4116 list_del_rcu(&atsru->list);
4117 synchronize_rcu();
4118 intel_iommu_free_atsr(atsru);
4119 }
4120
4121 return 0;
4122}
4123
4124int dmar_check_one_atsr(struct acpi_dmar_header *hdr, void *arg)
4125{
4126 int i;
4127 struct device *dev;
4128 struct acpi_dmar_atsr *atsr;
4129 struct dmar_atsr_unit *atsru;
4130
4131 atsr = container_of(hdr, struct acpi_dmar_atsr, header);
4132 atsru = dmar_find_atsr(atsr);
4133 if (!atsru)
4134 return 0;
4135
4136 if (!atsru->include_all && atsru->devices && atsru->devices_cnt)
4137 for_each_active_dev_scope(atsru->devices, atsru->devices_cnt,
4138 i, dev)
4139 return -EBUSY;
4140
4141 return 0;
4142}
4143
ffebeb46
JL		 4144static int intel_iommu_add(struct dmar_drhd_unit *dmaru)
4145{
4146 int sp, ret = 0;
4147 struct intel_iommu *iommu = dmaru->iommu;
4148
4149 if (g_iommus[iommu->seq_id])
4150 return 0;
4151
4152 if (hw_pass_through && !ecap_pass_through(iommu->ecap)) {
9f10e5bf 4153 pr_warn("%s: Doesn't support hardware pass through.\n",
ffebeb46
JL		 4154 iommu->name);
4155 return -ENXIO;
4156 }
4157 if (!ecap_sc_support(iommu->ecap) &&
4158 domain_update_iommu_snooping(iommu)) {
9f10e5bf 4159 pr_warn("%s: Doesn't support snooping.\n",
ffebeb46
JL		 4160 iommu->name);
4161 return -ENXIO;
4162 }
4163 sp = domain_update_iommu_superpage(iommu) - 1;
4164 if (sp >= 0 && !(cap_super_page_val(iommu->cap) & (1 << sp))) {
9f10e5bf 4165 pr_warn("%s: Doesn't support large page.\n",
ffebeb46
JL		 4166 iommu->name);
4167 return -ENXIO;
4168 }
4169
4170 /*
4171 * Disable translation if already enabled prior to OS handover.
4172 */
4173 if (iommu->gcmd & DMA_GCMD_TE)
4174 iommu_disable_translation(iommu);
4175
4176 g_iommus[iommu->seq_id] = iommu;
4177 ret = iommu_init_domains(iommu);
4178 if (ret == 0)
4179 ret = iommu_alloc_root_entry(iommu);
4180 if (ret)
4181 goto out;
4182
4183 if (dmaru->ignored) {
4184 /*
4185 * we always have to disable PMRs or DMA may fail on this device
4186 */
4187 if (force_on)
4188 iommu_disable_protect_mem_regions(iommu);
4189 return 0;
4190 }
4191
4192 intel_iommu_init_qi(iommu);
4193 iommu_flush_write_buffer(iommu);
4194 ret = dmar_set_interrupt(iommu);
4195 if (ret)
4196 goto disable_iommu;
4197
4198 iommu_set_root_entry(iommu);
4199 iommu->flush.flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL);
4200 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
4201 iommu_enable_translation(iommu);
4202
4203 if (si_domain) {
4204 ret = iommu_attach_domain(si_domain, iommu);
4205 if (ret < 0 || si_domain->id != ret)
4206 goto disable_iommu;
4207 domain_attach_iommu(si_domain, iommu);
4208 }
4209
4210 iommu_disable_protect_mem_regions(iommu);
4211 return 0;
4212
4213disable_iommu:
4214 disable_dmar_iommu(iommu);
4215out:
4216 free_dmar_iommu(iommu);
4217 return ret;
4218}
4219
6b197249
JL		 4220int dmar_iommu_hotplug(struct dmar_drhd_unit *dmaru, bool insert)
4221{
ffebeb46
JL		 4222 int ret = 0;
4223 struct intel_iommu *iommu = dmaru->iommu;
4224
4225 if (!intel_iommu_enabled)
4226 return 0;
4227 if (iommu == NULL)
4228 return -EINVAL;
4229
4230 if (insert) {
4231 ret = intel_iommu_add(dmaru);
4232 } else {
4233 disable_dmar_iommu(iommu);
4234 free_dmar_iommu(iommu);
4235 }
4236
4237 return ret;
6b197249
JL		 4238}
4239
9bdc531e
JL		 4240static void intel_iommu_free_dmars(void)
4241{
4242 struct dmar_rmrr_unit *rmrru, *rmrr_n;
4243 struct dmar_atsr_unit *atsru, *atsr_n;
4244
4245 list_for_each_entry_safe(rmrru, rmrr_n, &dmar_rmrr_units, list) {
4246 list_del(&rmrru->list);
4247 dmar_free_dev_scope(&rmrru->devices, &rmrru->devices_cnt);
4248 kfree(rmrru);
318fe7df
SS		 4249 }
4250
9bdc531e
JL		 4251 list_for_each_entry_safe(atsru, atsr_n, &dmar_atsr_units, list) {
4252 list_del(&atsru->list);
4253 intel_iommu_free_atsr(atsru);
4254 }
318fe7df
SS		 4255}
4256
4257int dmar_find_matched_atsr_unit(struct pci_dev *dev)
4258{
b683b230 4259 int i, ret = 1;
318fe7df 4260 struct pci_bus *bus;
832bd858
DW		 4261 struct pci_dev *bridge = NULL;
4262 struct device *tmp;
318fe7df
SS		 4263 struct acpi_dmar_atsr *atsr;
4264 struct dmar_atsr_unit *atsru;
4265
4266 dev = pci_physfn(dev);
318fe7df 4267 for (bus = dev->bus; bus; bus = bus->parent) {
b5f82ddf 4268 bridge = bus->self;
318fe7df 4269 if (!bridge || !pci_is_pcie(bridge) ||
62f87c0e 4270 pci_pcie_type(bridge) == PCI_EXP_TYPE_PCI_BRIDGE)
318fe7df 4271 return 0;
b5f82ddf 4272 if (pci_pcie_type(bridge) == PCI_EXP_TYPE_ROOT_PORT)
318fe7df 4273 break;
318fe7df 4274 }
b5f82ddf
JL		 4275 if (!bridge)
4276 return 0;
318fe7df 4277
0e242612 4278 rcu_read_lock();
b5f82ddf
JL		 4279 list_for_each_entry_rcu(atsru, &dmar_atsr_units, list) {
4280 atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header);
4281 if (atsr->segment != pci_domain_nr(dev->bus))
4282 continue;
4283
b683b230 4284 for_each_dev_scope(atsru->devices, atsru->devices_cnt, i, tmp)
832bd858 4285 if (tmp == &bridge->dev)
b683b230 4286 goto out;
b5f82ddf
JL		 4287
4288 if (atsru->include_all)
b683b230 4289 goto out;
b5f82ddf 4290 }
b683b230
JL		 4291 ret = 0;
4292out:
0e242612 4293 rcu_read_unlock();
318fe7df 4294
b683b230 4295 return ret;
318fe7df
SS		 4296}
4297
59ce0515
JL		 4298int dmar_iommu_notify_scope_dev(struct dmar_pci_notify_info *info)
4299{
4300 int ret = 0;
4301 struct dmar_rmrr_unit *rmrru;
4302 struct dmar_atsr_unit *atsru;
4303 struct acpi_dmar_atsr *atsr;
4304 struct acpi_dmar_reserved_memory *rmrr;
4305
4306 if (!intel_iommu_enabled && system_state != SYSTEM_BOOTING)
4307 return 0;
4308
4309 list_for_each_entry(rmrru, &dmar_rmrr_units, list) {
4310 rmrr = container_of(rmrru->hdr,
4311 struct acpi_dmar_reserved_memory, header);
4312 if (info->event == BUS_NOTIFY_ADD_DEVICE) {
4313 ret = dmar_insert_dev_scope(info, (void *)(rmrr + 1),
4314 ((void *)rmrr) + rmrr->header.length,
4315 rmrr->segment, rmrru->devices,
4316 rmrru->devices_cnt);
27e24950 4317 if(ret < 0)
59ce0515
JL		 4318 return ret;
4319 } else if (info->event == BUS_NOTIFY_DEL_DEVICE) {
27e24950
JL		 4320 dmar_remove_dev_scope(info, rmrr->segment,
4321 rmrru->devices, rmrru->devices_cnt);
59ce0515
JL		 4322 }
4323 }
4324
4325 list_for_each_entry(atsru, &dmar_atsr_units, list) {
4326 if (atsru->include_all)
4327 continue;
4328
4329 atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header);
4330 if (info->event == BUS_NOTIFY_ADD_DEVICE) {
4331 ret = dmar_insert_dev_scope(info, (void *)(atsr + 1),
4332 (void *)atsr + atsr->header.length,
4333 atsr->segment, atsru->devices,
4334 atsru->devices_cnt);
4335 if (ret > 0)
4336 break;
4337 else if(ret < 0)
4338 return ret;
4339 } else if (info->event == BUS_NOTIFY_DEL_DEVICE) {
4340 if (dmar_remove_dev_scope(info, atsr->segment,
4341 atsru->devices, atsru->devices_cnt))
4342 break;
4343 }
4344 }
4345
4346 return 0;
4347}
4348
99dcaded
FY		 4349/*
4350 * Here we only respond to action of unbound device from driver.
4351 *
4352 * Added device is not attached to its DMAR domain here yet. That will happen
4353 * when mapping the device to iova.
4354 */
4355static int device_notifier(struct notifier_block *nb,
4356 unsigned long action, void *data)
4357{
4358 struct device *dev = data;
99dcaded
FY		 4359 struct dmar_domain *domain;
4360
3d89194a 4361 if (iommu_dummy(dev))
44cd613c
DW		 4362 return 0;
4363
1196c2fb 4364 if (action != BUS_NOTIFY_REMOVED_DEVICE)
7e7dfab7
JL		 4365 return 0;
4366
1525a29a 4367 domain = find_domain(dev);
99dcaded
FY		 4368 if (!domain)
4369 return 0;
4370
3a5670e8 4371 down_read(&dmar_global_lock);
bf9c9eda 4372 domain_remove_one_dev_info(domain, dev);
ab8dfe25 4373 if (!domain_type_is_vm_or_si(domain) && list_empty(&domain->devices))
7e7dfab7 4374 domain_exit(domain);
3a5670e8 4375 up_read(&dmar_global_lock);
a97590e5 4376
99dcaded
FY		 4377 return 0;
4378}
4379
4380static struct notifier_block device_nb = {
4381 .notifier_call = device_notifier,
4382};
4383
75f05569
JL		 4384static int intel_iommu_memory_notifier(struct notifier_block *nb,
4385 unsigned long val, void *v)
4386{
4387 struct memory_notify *mhp = v;
4388 unsigned long long start, end;
4389 unsigned long start_vpfn, last_vpfn;
4390
4391 switch (val) {
4392 case MEM_GOING_ONLINE:
4393 start = mhp->start_pfn << PAGE_SHIFT;
4394 end = ((mhp->start_pfn + mhp->nr_pages) << PAGE_SHIFT) - 1;
4395 if (iommu_domain_identity_map(si_domain, start, end)) {
9f10e5bf 4396 pr_warn("Failed to build identity map for [%llx-%llx]\n",
75f05569
JL		 4397 start, end);
4398 return NOTIFY_BAD;
4399 }
4400 break;
4401
4402 case MEM_OFFLINE:
4403 case MEM_CANCEL_ONLINE:
4404 start_vpfn = mm_to_dma_pfn(mhp->start_pfn);
4405 last_vpfn = mm_to_dma_pfn(mhp->start_pfn + mhp->nr_pages - 1);
4406 while (start_vpfn <= last_vpfn) {
4407 struct iova *iova;
4408 struct dmar_drhd_unit *drhd;
4409 struct intel_iommu *iommu;
ea8ea460 4410 struct page *freelist;
75f05569
JL		 4411
4412 iova = find_iova(&si_domain->iovad, start_vpfn);
4413 if (iova == NULL) {
9f10e5bf 4414 pr_debug("Failed get IOVA for PFN %lx\n",
75f05569
JL		 4415 start_vpfn);
4416 break;
4417 }
4418
4419 iova = split_and_remove_iova(&si_domain->iovad, iova,
4420 start_vpfn, last_vpfn);
4421 if (iova == NULL) {
9f10e5bf 4422 pr_warn("Failed to split IOVA PFN [%lx-%lx]\n",
75f05569
JL		 4423 start_vpfn, last_vpfn);
4424 return NOTIFY_BAD;
4425 }
4426
ea8ea460
DW		 4427 freelist = domain_unmap(si_domain, iova->pfn_lo,
4428 iova->pfn_hi);
4429
75f05569
JL		 4430 rcu_read_lock();
4431 for_each_active_iommu(iommu, drhd)
4432 iommu_flush_iotlb_psi(iommu, si_domain->id,
a156ef99 4433 iova->pfn_lo, iova_size(iova),
ea8ea460 4434 !freelist, 0);
75f05569 4435 rcu_read_unlock();
ea8ea460 4436 dma_free_pagelist(freelist);
75f05569
JL		 4437
4438 start_vpfn = iova->pfn_hi + 1;
4439 free_iova_mem(iova);
4440 }
4441 break;
4442 }
4443
4444 return NOTIFY_OK;
4445}
4446
4447static struct notifier_block intel_iommu_memory_nb = {
4448 .notifier_call = intel_iommu_memory_notifier,
4449 .priority = 0
4450};
4451
a5459cfe
AW		 4452
4453static ssize_t intel_iommu_show_version(struct device *dev,
4454 struct device_attribute *attr,
4455 char *buf)
4456{
4457 struct intel_iommu *iommu = dev_get_drvdata(dev);
4458 u32 ver = readl(iommu->reg + DMAR_VER_REG);
4459 return sprintf(buf, "%d:%d\n",
4460 DMAR_VER_MAJOR(ver), DMAR_VER_MINOR(ver));
4461}
4462static DEVICE_ATTR(version, S_IRUGO, intel_iommu_show_version, NULL);
4463
4464static ssize_t intel_iommu_show_address(struct device *dev,
4465 struct device_attribute *attr,
4466 char *buf)
4467{
4468 struct intel_iommu *iommu = dev_get_drvdata(dev);
4469 return sprintf(buf, "%llx\n", iommu->reg_phys);
4470}
4471static DEVICE_ATTR(address, S_IRUGO, intel_iommu_show_address, NULL);
4472
4473static ssize_t intel_iommu_show_cap(struct device *dev,
4474 struct device_attribute *attr,
4475 char *buf)
4476{
4477 struct intel_iommu *iommu = dev_get_drvdata(dev);
4478 return sprintf(buf, "%llx\n", iommu->cap);
4479}
4480static DEVICE_ATTR(cap, S_IRUGO, intel_iommu_show_cap, NULL);
4481
4482static ssize_t intel_iommu_show_ecap(struct device *dev,
4483 struct device_attribute *attr,
4484 char *buf)
4485{
4486 struct intel_iommu *iommu = dev_get_drvdata(dev);
4487 return sprintf(buf, "%llx\n", iommu->ecap);
4488}
4489static DEVICE_ATTR(ecap, S_IRUGO, intel_iommu_show_ecap, NULL);
4490
2238c082
AW		 4491static ssize_t intel_iommu_show_ndoms(struct device *dev,
4492 struct device_attribute *attr,
4493 char *buf)
4494{
4495 struct intel_iommu *iommu = dev_get_drvdata(dev);
4496 return sprintf(buf, "%ld\n", cap_ndoms(iommu->cap));
4497}
4498static DEVICE_ATTR(domains_supported, S_IRUGO, intel_iommu_show_ndoms, NULL);
4499
4500static ssize_t intel_iommu_show_ndoms_used(struct device *dev,
4501 struct device_attribute *attr,
4502 char *buf)
4503{
4504 struct intel_iommu *iommu = dev_get_drvdata(dev);
4505 return sprintf(buf, "%d\n", bitmap_weight(iommu->domain_ids,
4506 cap_ndoms(iommu->cap)));
4507}
4508static DEVICE_ATTR(domains_used, S_IRUGO, intel_iommu_show_ndoms_used, NULL);
4509
a5459cfe
AW		 4510static struct attribute *intel_iommu_attrs[] = {
4511 &dev_attr_version.attr,
4512 &dev_attr_address.attr,
4513 &dev_attr_cap.attr,
4514 &dev_attr_ecap.attr,
2238c082
AW		 4515 &dev_attr_domains_supported.attr,
4516 &dev_attr_domains_used.attr,
a5459cfe
AW		 4517 NULL,
4518};
4519
4520static struct attribute_group intel_iommu_group = {
4521 .name = "intel-iommu",
4522 .attrs = intel_iommu_attrs,
4523};
4524
4525const struct attribute_group *intel_iommu_groups[] = {
4526 &intel_iommu_group,
4527 NULL,
4528};
4529
ba395927
KA		 4530int __init intel_iommu_init(void)
4531{
9bdc531e 4532 int ret = -ENODEV;
3a93c841 4533 struct dmar_drhd_unit *drhd;
7c919779 4534 struct intel_iommu *iommu;
ba395927 4535
a59b50e9
JC		 4536 /* VT-d is required for a TXT/tboot launch, so enforce that */
4537 force_on = tboot_force_iommu();
4538
3a5670e8
JL		 4539 if (iommu_init_mempool()) {
4540 if (force_on)
4541 panic("tboot: Failed to initialize iommu memory\n");
4542 return -ENOMEM;
4543 }
4544
4545 down_write(&dmar_global_lock);
a59b50e9
JC		 4546 if (dmar_table_init()) {
4547 if (force_on)
4548 panic("tboot: Failed to initialize DMAR table\n");
9bdc531e 4549 goto out_free_dmar;
a59b50e9 4550 }
ba395927 4551
c2c7286a 4552 if (dmar_dev_scope_init() < 0) {
a59b50e9
JC		 4553 if (force_on)
4554 panic("tboot: Failed to initialize DMAR device scope\n");
9bdc531e 4555 goto out_free_dmar;
a59b50e9 4556 }
1886e8a9 4557
75f1cdf1 4558 if (no_iommu || dmar_disabled)
9bdc531e 4559 goto out_free_dmar;
2ae21010 4560
318fe7df 4561 if (list_empty(&dmar_rmrr_units))
9f10e5bf 4562 pr_info("No RMRR found\n");
318fe7df
SS		 4563
4564 if (list_empty(&dmar_atsr_units))
9f10e5bf 4565 pr_info("No ATSR found\n");
318fe7df 4566
51a63e67
JC		 4567 if (dmar_init_reserved_ranges()) {
4568 if (force_on)
4569 panic("tboot: Failed to reserve iommu ranges\n");
3a5670e8 4570 goto out_free_reserved_range;
51a63e67 4571 }
ba395927
KA		 4572
4573 init_no_remapping_devices();
4574
b779260b 4575 ret = init_dmars();
ba395927 4576 if (ret) {
a59b50e9
JC		 4577 if (force_on)
4578 panic("tboot: Failed to initialize DMARs\n");
9f10e5bf 4579 pr_err("Initialization failed\n");
9bdc531e 4580 goto out_free_reserved_range;
ba395927 4581 }
3a5670e8 4582 up_write(&dmar_global_lock);
9f10e5bf 4583 pr_info("Intel(R) Virtualization Technology for Directed I/O\n");
ba395927 4584
5e0d2a6f 4585 init_timer(&unmap_timer);
75f1cdf1
FT		 4586#ifdef CONFIG_SWIOTLB
4587 swiotlb = 0;
4588#endif
19943b0e 4589 dma_ops = &intel_dma_ops;
4ed0d3e6 4590
134fac3f 4591 init_iommu_pm_ops();
a8bcbb0d 4592
a5459cfe
AW		 4593 for_each_active_iommu(iommu, drhd)
4594 iommu->iommu_dev = iommu_device_create(NULL, iommu,
4595 intel_iommu_groups,
2439d4aa 4596 "%s", iommu->name);
a5459cfe 4597
4236d97d 4598 bus_set_iommu(&pci_bus_type, &intel_iommu_ops);
99dcaded 4599 bus_register_notifier(&pci_bus_type, &device_nb);
75f05569
JL		 4600 if (si_domain && !hw_pass_through)
4601 register_memory_notifier(&intel_iommu_memory_nb);
99dcaded 4602
8bc1f85c
ED		 4603 intel_iommu_enabled = 1;
4604
ba395927 4605 return 0;
9bdc531e
JL		 4606
4607out_free_reserved_range:
4608 put_iova_domain(&reserved_iova_list);
9bdc531e
JL		 4609out_free_dmar:
4610 intel_iommu_free_dmars();
3a5670e8
JL		 4611 up_write(&dmar_global_lock);
4612 iommu_exit_mempool();
9bdc531e 4613 return ret;
ba395927 4614}
e820482c 4615
579305f7
AW		 4616static int iommu_detach_dev_cb(struct pci_dev *pdev, u16 alias, void *opaque)
4617{
4618 struct intel_iommu *iommu = opaque;
4619
4620 iommu_detach_dev(iommu, PCI_BUS_NUM(alias), alias & 0xff);
4621 return 0;
4622}
4623
4624/*
4625 * NB - intel-iommu lacks any sort of reference counting for the users of
4626 * dependent devices. If multiple endpoints have intersecting dependent
4627 * devices, unbinding the driver from any one of them will possibly leave
4628 * the others unable to operate.
4629 */
3199aa6b 4630static void iommu_detach_dependent_devices(struct intel_iommu *iommu,
0bcb3e28 4631 struct device *dev)
3199aa6b 4632{
0bcb3e28 4633 if (!iommu || !dev || !dev_is_pci(dev))
3199aa6b
HW		 4634 return;
4635
579305f7 4636 pci_for_each_dma_alias(to_pci_dev(dev), &iommu_detach_dev_cb, iommu);
3199aa6b
HW		 4637}
4638
2c2e2c38 4639static void domain_remove_one_dev_info(struct dmar_domain *domain,
bf9c9eda 4640 struct device *dev)
c7151a8d 4641{
bca2b916 4642 struct device_domain_info *info, *tmp;
c7151a8d
WH		 4643 struct intel_iommu *iommu;
4644 unsigned long flags;
2f119c78 4645 bool found = false;
156baca8 4646 u8 bus, devfn;
c7151a8d 4647
bf9c9eda 4648 iommu = device_to_iommu(dev, &bus, &devfn);
c7151a8d
WH		 4649 if (!iommu)
4650 return;
4651
4652 spin_lock_irqsave(&device_domain_lock, flags);
bca2b916 4653 list_for_each_entry_safe(info, tmp, &domain->devices, link) {
bf9c9eda
DW		 4654 if (info->iommu == iommu && info->bus == bus &&
4655 info->devfn == devfn) {
109b9b04 4656 unlink_domain_info(info);
c7151a8d
WH		 4657 spin_unlock_irqrestore(&device_domain_lock, flags);
4658
93a23a72 4659 iommu_disable_dev_iotlb(info);
c7151a8d 4660 iommu_detach_dev(iommu, info->bus, info->devfn);
bf9c9eda 4661 iommu_detach_dependent_devices(iommu, dev);
c7151a8d
WH		 4662 free_devinfo_mem(info);
4663
4664 spin_lock_irqsave(&device_domain_lock, flags);
4665
4666 if (found)
4667 break;
4668 else
4669 continue;
4670 }
4671
4672 /* if there is no other devices under the same iommu
4673 * owned by this domain, clear this iommu in iommu_bmp
4674 * update iommu count and coherency
4675 */
8bbc4410 4676 if (info->iommu == iommu)
2f119c78 4677 found = true;
c7151a8d
WH		 4678 }
4679
3e7abe25
RD		 4680 spin_unlock_irqrestore(&device_domain_lock, flags);
4681
c7151a8d 4682 if (found == 0) {
fb170fb4
JL		 4683 domain_detach_iommu(domain, iommu);
4684 if (!domain_type_is_vm_or_si(domain))
4685 iommu_detach_domain(domain, iommu);
c7151a8d 4686 }
c7151a8d
WH		 4687}
4688
2c2e2c38 4689static int md_domain_init(struct dmar_domain *domain, int guest_width)
5e98c4b1
WH		 4690{
4691 int adjust_width;
4692
0fb5fe87
RM		 4693 init_iova_domain(&domain->iovad, VTD_PAGE_SIZE, IOVA_START_PFN,
4694 DMA_32BIT_PFN);
5e98c4b1
WH		 4695 domain_reserve_special_ranges(domain);
4696
4697 /* calculate AGAW */
4698 domain->gaw = guest_width;
4699 adjust_width = guestwidth_to_adjustwidth(guest_width);
4700 domain->agaw = width_to_agaw(adjust_width);
4701
5e98c4b1 4702 domain->iommu_coherency = 0;
c5b15255 4703 domain->iommu_snooping = 0;
6dd9a7c7 4704 domain->iommu_superpage = 0;
fe40f1e0 4705 domain->max_addr = 0;
5e98c4b1
WH		 4706
4707 /* always allocate the top pgd */
4c923d47 4708 domain->pgd = (struct dma_pte *)alloc_pgtable_page(domain->nid);
5e98c4b1
WH		 4709 if (!domain->pgd)
4710 return -ENOMEM;
4711 domain_flush_cache(domain, domain->pgd, PAGE_SIZE);
4712 return 0;
4713}
4714
00a77deb 4715static struct iommu_domain *intel_iommu_domain_alloc(unsigned type)
38717946 4716{
5d450806 4717 struct dmar_domain *dmar_domain;
00a77deb
JR		 4718 struct iommu_domain *domain;
4719
4720 if (type != IOMMU_DOMAIN_UNMANAGED)
4721 return NULL;
38717946 4722
ab8dfe25 4723 dmar_domain = alloc_domain(DOMAIN_FLAG_VIRTUAL_MACHINE);
5d450806 4724 if (!dmar_domain) {
9f10e5bf 4725 pr_err("Can't allocate dmar_domain\n");
00a77deb 4726 return NULL;
38717946 4727 }
2c2e2c38 4728 if (md_domain_init(dmar_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
9f10e5bf 4729 pr_err("Domain initialization failed\n");
92d03cc8 4730 domain_exit(dmar_domain);
00a77deb 4731 return NULL;
38717946 4732 }
8140a95d 4733 domain_update_iommu_cap(dmar_domain);
faa3d6f5 4734
00a77deb 4735 domain = &dmar_domain->domain;
8a0e715b
JR		 4736 domain->geometry.aperture_start = 0;
4737 domain->geometry.aperture_end = __DOMAIN_MAX_ADDR(dmar_domain->gaw);
4738 domain->geometry.force_aperture = true;
4739
00a77deb 4740 return domain;
38717946 4741}
38717946 4742
00a77deb 4743static void intel_iommu_domain_free(struct iommu_domain *domain)
38717946 4744{
00a77deb 4745 domain_exit(to_dmar_domain(domain));
38717946 4746}
38717946 4747
4c5478c9
JR		 4748static int intel_iommu_attach_device(struct iommu_domain *domain,
4749 struct device *dev)
38717946 4750{
00a77deb 4751 struct dmar_domain *dmar_domain = to_dmar_domain(domain);
fe40f1e0
WH		 4752 struct intel_iommu *iommu;
4753 int addr_width;
156baca8 4754 u8 bus, devfn;
faa3d6f5 4755
c875d2c1
AW		 4756 if (device_is_rmrr_locked(dev)) {
4757 dev_warn(dev, "Device is ineligible for IOMMU domain attach due to platform RMRR requirement. Contact your platform vendor.\n");
4758 return -EPERM;
4759 }
4760
7207d8f9
DW		 4761 /* normally dev is not mapped */
4762 if (unlikely(domain_context_mapped(dev))) {
faa3d6f5
WH		 4763 struct dmar_domain *old_domain;
4764
1525a29a 4765 old_domain = find_domain(dev);
faa3d6f5 4766 if (old_domain) {
ab8dfe25 4767 if (domain_type_is_vm_or_si(dmar_domain))
bf9c9eda 4768 domain_remove_one_dev_info(old_domain, dev);
faa3d6f5
WH		 4769 else
4770 domain_remove_dev_info(old_domain);
62c22167
JR		 4771
4772 if (!domain_type_is_vm_or_si(old_domain) &&
4773 list_empty(&old_domain->devices))
4774 domain_exit(old_domain);
faa3d6f5
WH		 4775 }
4776 }
4777
156baca8 4778 iommu = device_to_iommu(dev, &bus, &devfn);
fe40f1e0
WH		 4779 if (!iommu)
4780 return -ENODEV;
4781
4782 /* check if this iommu agaw is sufficient for max mapped address */
4783 addr_width = agaw_to_width(iommu->agaw);
a99c47a2
TL		 4784 if (addr_width > cap_mgaw(iommu->cap))
4785 addr_width = cap_mgaw(iommu->cap);
4786
4787 if (dmar_domain->max_addr > (1LL << addr_width)) {
9f10e5bf 4788 pr_err("%s: iommu width (%d) is not "
fe40f1e0 4789 "sufficient for the mapped address (%llx)\n",
a99c47a2 4790 __func__, addr_width, dmar_domain->max_addr);
fe40f1e0
WH		 4791 return -EFAULT;
4792 }
a99c47a2
TL		 4793 dmar_domain->gaw = addr_width;
4794
4795 /*
4796 * Knock out extra levels of page tables if necessary
4797 */
4798 while (iommu->agaw < dmar_domain->agaw) {
4799 struct dma_pte *pte;
4800
4801 pte = dmar_domain->pgd;
4802 if (dma_pte_present(pte)) {
25cbff16
SY		 4803 dmar_domain->pgd = (struct dma_pte *)
4804 phys_to_virt(dma_pte_addr(pte));
7a661013 4805 free_pgtable_page(pte);
a99c47a2
TL		 4806 }
4807 dmar_domain->agaw--;
4808 }
fe40f1e0 4809
28ccce0d 4810 return domain_add_dev_info(dmar_domain, dev);
38717946 4811}
38717946 4812
4c5478c9
JR		 4813static void intel_iommu_detach_device(struct iommu_domain *domain,
4814 struct device *dev)
38717946 4815{
00a77deb 4816 domain_remove_one_dev_info(to_dmar_domain(domain), dev);
faa3d6f5 4817}
c7151a8d 4818
b146a1c9
JR		 4819static int intel_iommu_map(struct iommu_domain *domain,
4820 unsigned long iova, phys_addr_t hpa,
5009065d 4821 size_t size, int iommu_prot)
faa3d6f5 4822{
00a77deb 4823 struct dmar_domain *dmar_domain = to_dmar_domain(domain);
fe40f1e0 4824 u64 max_addr;
dde57a21 4825 int prot = 0;
faa3d6f5 4826 int ret;
fe40f1e0 4827
dde57a21
JR		 4828 if (iommu_prot & IOMMU_READ)
4829 prot |= DMA_PTE_READ;
4830 if (iommu_prot & IOMMU_WRITE)
4831 prot |= DMA_PTE_WRITE;
9cf06697
SY		 4832 if ((iommu_prot & IOMMU_CACHE) && dmar_domain->iommu_snooping)
4833 prot |= DMA_PTE_SNP;
dde57a21 4834
163cc52c 4835 max_addr = iova + size;
dde57a21 4836 if (dmar_domain->max_addr < max_addr) {
fe40f1e0
WH		 4837 u64 end;
4838
4839 /* check if minimum agaw is sufficient for mapped address */
8954da1f 4840 end = __DOMAIN_MAX_ADDR(dmar_domain->gaw) + 1;
fe40f1e0 4841 if (end < max_addr) {
9f10e5bf 4842 pr_err("%s: iommu width (%d) is not "
fe40f1e0 4843 "sufficient for the mapped address (%llx)\n",
8954da1f 4844 __func__, dmar_domain->gaw, max_addr);
fe40f1e0
WH		 4845 return -EFAULT;
4846 }
dde57a21 4847 dmar_domain->max_addr = max_addr;
fe40f1e0 4848 }
ad051221
DW		 4849 /* Round up size to next multiple of PAGE_SIZE, if it and
4850 the low bits of hpa would take us onto the next page */
88cb6a74 4851 size = aligned_nrpages(hpa, size);
ad051221
DW		 4852 ret = domain_pfn_mapping(dmar_domain, iova >> VTD_PAGE_SHIFT,
4853 hpa >> VTD_PAGE_SHIFT, size, prot);
faa3d6f5 4854 return ret;
38717946 4855}
38717946 4856
5009065d 4857static size_t intel_iommu_unmap(struct iommu_domain *domain,
ea8ea460 4858 unsigned long iova, size_t size)
38717946 4859{
00a77deb 4860 struct dmar_domain *dmar_domain = to_dmar_domain(domain);
ea8ea460
DW		 4861 struct page *freelist = NULL;
4862 struct intel_iommu *iommu;
4863 unsigned long start_pfn, last_pfn;
4864 unsigned int npages;
4865 int iommu_id, num, ndomains, level = 0;
5cf0a76f
DW		 4866
4867 /* Cope with horrid API which requires us to unmap more than the
4868 size argument if it happens to be a large-page mapping. */
4869 if (!pfn_to_dma_pte(dmar_domain, iova >> VTD_PAGE_SHIFT, &level))
4870 BUG();
4871
4872 if (size < VTD_PAGE_SIZE << level_to_offset_bits(level))
4873 size = VTD_PAGE_SIZE << level_to_offset_bits(level);
4b99d352 4874
ea8ea460
DW		 4875 start_pfn = iova >> VTD_PAGE_SHIFT;
4876 last_pfn = (iova + size - 1) >> VTD_PAGE_SHIFT;
4877
4878 freelist = domain_unmap(dmar_domain, start_pfn, last_pfn);
4879
4880 npages = last_pfn - start_pfn + 1;
4881
4882 for_each_set_bit(iommu_id, dmar_domain->iommu_bmp, g_num_of_iommus) {
4883 iommu = g_iommus[iommu_id];
4884
4885 /*
4886 * find bit position of dmar_domain
4887 */
4888 ndomains = cap_ndoms(iommu->cap);
4889 for_each_set_bit(num, iommu->domain_ids, ndomains) {
9452d5bf 4890 if (get_iommu_domain(iommu, num) == dmar_domain)
ea8ea460
DW		 4891 iommu_flush_iotlb_psi(iommu, num, start_pfn,
4892 npages, !freelist, 0);
4893 }
4894
4895 }
4896
4897 dma_free_pagelist(freelist);
fe40f1e0 4898
163cc52c
DW		 4899 if (dmar_domain->max_addr == iova + size)
4900 dmar_domain->max_addr = iova;
b146a1c9 4901
5cf0a76f 4902 return size;
38717946 4903}
38717946 4904
d14d6577 4905static phys_addr_t intel_iommu_iova_to_phys(struct iommu_domain *domain,
bb5547ac 4906 dma_addr_t iova)
38717946 4907{
00a77deb 4908 struct dmar_domain *dmar_domain = to_dmar_domain(domain);
38717946 4909 struct dma_pte *pte;
5cf0a76f 4910 int level = 0;
faa3d6f5 4911 u64 phys = 0;
38717946 4912
5cf0a76f 4913 pte = pfn_to_dma_pte(dmar_domain, iova >> VTD_PAGE_SHIFT, &level);
38717946 4914 if (pte)
faa3d6f5 4915 phys = dma_pte_addr(pte);
38717946 4916
faa3d6f5 4917 return phys;
38717946 4918}
a8bcbb0d 4919
5d587b8d 4920static bool intel_iommu_capable(enum iommu_cap cap)
dbb9fd86 4921{
dbb9fd86 4922 if (cap == IOMMU_CAP_CACHE_COHERENCY)
5d587b8d 4923 return domain_update_iommu_snooping(NULL) == 1;
323f99cb 4924 if (cap == IOMMU_CAP_INTR_REMAP)
5d587b8d 4925 return irq_remapping_enabled == 1;
dbb9fd86 4926
5d587b8d 4927 return false;
dbb9fd86
SY		 4928}
4929
abdfdde2
AW		 4930static int intel_iommu_add_device(struct device *dev)
4931{
a5459cfe 4932 struct intel_iommu *iommu;
abdfdde2 4933 struct iommu_group *group;
156baca8 4934 u8 bus, devfn;
70ae6f0d 4935
a5459cfe
AW		 4936 iommu = device_to_iommu(dev, &bus, &devfn);
4937 if (!iommu)
70ae6f0d
AW		 4938 return -ENODEV;
4939
a5459cfe 4940 iommu_device_link(iommu->iommu_dev, dev);
a4ff1fc2 4941
e17f9ff4 4942 group = iommu_group_get_for_dev(dev);
783f157b 4943
e17f9ff4
AW		 4944 if (IS_ERR(group))
4945 return PTR_ERR(group);
bcb71abe 4946
abdfdde2 4947 iommu_group_put(group);
e17f9ff4 4948 return 0;
abdfdde2 4949}
70ae6f0d 4950
abdfdde2
AW		 4951static void intel_iommu_remove_device(struct device *dev)
4952{
a5459cfe
AW		 4953 struct intel_iommu *iommu;
4954 u8 bus, devfn;
4955
4956 iommu = device_to_iommu(dev, &bus, &devfn);
4957 if (!iommu)
4958 return;
4959
abdfdde2 4960 iommu_group_remove_device(dev);
a5459cfe
AW		 4961
4962 iommu_device_unlink(iommu->iommu_dev, dev);
70ae6f0d
AW		 4963}
4964
b22f6434 4965static const struct iommu_ops intel_iommu_ops = {
5d587b8d 4966 .capable = intel_iommu_capable,
00a77deb
JR		 4967 .domain_alloc = intel_iommu_domain_alloc,
4968 .domain_free = intel_iommu_domain_free,
a8bcbb0d
JR		 4969 .attach_dev = intel_iommu_attach_device,
4970 .detach_dev = intel_iommu_detach_device,
b146a1c9
JR		 4971 .map = intel_iommu_map,
4972 .unmap = intel_iommu_unmap,
315786eb 4973 .map_sg = default_iommu_map_sg,
a8bcbb0d 4974 .iova_to_phys = intel_iommu_iova_to_phys,
abdfdde2
AW		 4975 .add_device = intel_iommu_add_device,
4976 .remove_device = intel_iommu_remove_device,
6d1c56a9 4977 .pgsize_bitmap = INTEL_IOMMU_PGSIZES,
a8bcbb0d 4978};
9af88143 4979
9452618e
DV		 4980static void quirk_iommu_g4x_gfx(struct pci_dev *dev)
4981{
4982 /* G4x/GM45 integrated gfx dmar support is totally busted. */
9f10e5bf 4983 pr_info("Disabling IOMMU for graphics on this chipset\n");
9452618e
DV		 4984 dmar_map_gfx = 0;
4985}
4986
4987DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2a40, quirk_iommu_g4x_gfx);
4988DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e00, quirk_iommu_g4x_gfx);
4989DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e10, quirk_iommu_g4x_gfx);
4990DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e20, quirk_iommu_g4x_gfx);
4991DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e30, quirk_iommu_g4x_gfx);
4992DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e40, quirk_iommu_g4x_gfx);
4993DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e90, quirk_iommu_g4x_gfx);
4994
d34d6517 4995static void quirk_iommu_rwbf(struct pci_dev *dev)
9af88143
DW		 4996{
4997 /*
4998 * Mobile 4 Series Chipset neglects to set RWBF capability,
210561ff 4999 * but needs it. Same seems to hold for the desktop versions.
9af88143 5000 */
9f10e5bf 5001 pr_info("Forcing write-buffer flush capability\n");
9af88143
DW		 5002 rwbf_quirk = 1;
5003}
5004
5005DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2a40, quirk_iommu_rwbf);
210561ff
DV		 5006DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e00, quirk_iommu_rwbf);
5007DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e10, quirk_iommu_rwbf);
5008DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e20, quirk_iommu_rwbf);
5009DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e30, quirk_iommu_rwbf);
5010DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e40, quirk_iommu_rwbf);
5011DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e90, quirk_iommu_rwbf);
e0fc7e0b 5012
eecfd57f
AJ		 5013#define GGC 0x52
5014#define GGC_MEMORY_SIZE_MASK (0xf << 8)
5015#define GGC_MEMORY_SIZE_NONE (0x0 << 8)
5016#define GGC_MEMORY_SIZE_1M (0x1 << 8)
5017#define GGC_MEMORY_SIZE_2M (0x3 << 8)
5018#define GGC_MEMORY_VT_ENABLED (0x8 << 8)
5019#define GGC_MEMORY_SIZE_2M_VT (0x9 << 8)
5020#define GGC_MEMORY_SIZE_3M_VT (0xa << 8)
5021#define GGC_MEMORY_SIZE_4M_VT (0xb << 8)
5022
d34d6517 5023static void quirk_calpella_no_shadow_gtt(struct pci_dev *dev)
9eecabcb
DW		 5024{
5025 unsigned short ggc;
5026
eecfd57f 5027 if (pci_read_config_word(dev, GGC, &ggc))
9eecabcb
DW		 5028 return;
5029
eecfd57f 5030 if (!(ggc & GGC_MEMORY_VT_ENABLED)) {
9f10e5bf 5031 pr_info("BIOS has allocated no shadow GTT; disabling IOMMU for graphics\n");
9eecabcb 5032 dmar_map_gfx = 0;
6fbcfb3e
DW		 5033 } else if (dmar_map_gfx) {
5034 /* we have to ensure the gfx device is idle before we flush */
9f10e5bf 5035 pr_info("Disabling batched IOTLB flush on Ironlake\n");
6fbcfb3e
DW		 5036 intel_iommu_strict = 1;
5037 }
9eecabcb
DW		 5038}
5039DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0040, quirk_calpella_no_shadow_gtt);
5040DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0044, quirk_calpella_no_shadow_gtt);
5041DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0062, quirk_calpella_no_shadow_gtt);
5042DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x006a, quirk_calpella_no_shadow_gtt);
5043
e0fc7e0b
DW		 5044/* On Tylersburg chipsets, some BIOSes have been known to enable the
5045 ISOCH DMAR unit for the Azalia sound device, but not give it any
5046 TLB entries, which causes it to deadlock. Check for that. We do
5047 this in a function called from init_dmars(), instead of in a PCI
5048 quirk, because we don't want to print the obnoxious "BIOS broken"
5049 message if VT-d is actually disabled.
5050*/
5051static void __init check_tylersburg_isoch(void)
5052{
5053 struct pci_dev *pdev;
5054 uint32_t vtisochctrl;
5055
5056 /* If there's no Azalia in the system anyway, forget it. */
5057 pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x3a3e, NULL);
5058 if (!pdev)
5059 return;
5060 pci_dev_put(pdev);
5061
5062 /* System Management Registers. Might be hidden, in which case
5063 we can't do the sanity check. But that's OK, because the
5064 known-broken BIOSes _don't_ actually hide it, so far. */
5065 pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x342e, NULL);
5066 if (!pdev)
5067 return;
5068
5069 if (pci_read_config_dword(pdev, 0x188, &vtisochctrl)) {
5070 pci_dev_put(pdev);
5071 return;
5072 }
5073
5074 pci_dev_put(pdev);
5075
5076 /* If Azalia DMA is routed to the non-isoch DMAR unit, fine. */
5077 if (vtisochctrl & 1)
5078 return;
5079
5080 /* Drop all bits other than the number of TLB entries */
5081 vtisochctrl &= 0x1c;
5082
5083 /* If we have the recommended number of TLB entries (16), fine. */
5084 if (vtisochctrl == 0x10)
5085 return;
5086
5087 /* Zero TLB entries? You get to ride the short bus to school. */
5088 if (!vtisochctrl) {
5089 WARN(1, "Your BIOS is broken; DMA routed to ISOCH DMAR unit but no TLB space.\n"
5090 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
5091 dmi_get_system_info(DMI_BIOS_VENDOR),
5092 dmi_get_system_info(DMI_BIOS_VERSION),
5093 dmi_get_system_info(DMI_PRODUCT_VERSION));
5094 iommu_identity_mapping |= IDENTMAP_AZALIA;
5095 return;
5096 }
9f10e5bf
JR		 5097
5098 pr_warn("Recommended TLB entries for ISOCH unit is 16; your BIOS set %d\n",
e0fc7e0b
DW		 5099 vtisochctrl);
5100}
Linux kernel - Deliverables
This page took 1.096056 seconds and 5 git commands to generate.